diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 2cfde526d..8e93cf8c3 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -2,6 +2,80 @@ version: 2 updates: - package-ecosystem: github-actions directory: "/" - target-branch: "pre-1.35.6" schedule: interval: weekly + + # Every Cargo.toml that has its own Cargo.lock. tools/unitctl is one + # workspace, so its root covers the three member crates. The tests build + # a copy of each test/wasm_component fixture with "cargo component build", + # which writes src/bindings.rs again. The exports come from + # wasi::http::proxy::export!, so a bump cannot break a fixture through + # that file. + - package-ecosystem: cargo + directories: + - "/tools/unitctl" + - "/src/otel" + - "/src/wasm-wasi-component" + - "/pkg/eol" + - "/test/fake_otlp" + - "/test/fake_upstream" + - "/test/wasm_component/hello_world" + - "/test/wasm_component/spin_loop" + schedule: + interval: weekly + # Every version update below belongs to a group, and a group is one + # pull request for all directories. So at most three version-update + # pull requests are open, whether the limit counts per directory or for + # the whole block. + open-pull-requests-limit: 3 + commit-message: + prefix: "build(deps)" + # A group is one pull request for all directories, so src/otel and + # test/fake_otlp move together in one otel pull request. + groups: + otel: + patterns: + - "opentelemetry*" + - "tonic*" + - "prost*" + # A group covers version updates only, so security updates need their + # own group to keep the otel crates together. + otel-security: + applies-to: security-updates + patterns: + - "opentelemetry*" + - "tonic*" + - "prost*" + # A group without "patterns" outranks "opentelemetry*" when Dependabot + # picks the most specific group, so the two groups below must exclude + # the otel crates. Otherwise they claim them and split the stack. + minor-and-patch: + exclude-patterns: + - "opentelemetry*" + - "tonic*" + - "prost*" + update-types: + - "minor" + - "patch" + # Major updates, including a 0.x minor, which cargo rates as major. + # They come together in one pull request. + major: + exclude-patterns: + - "opentelemetry*" + - "tonic*" + - "prost*" + update-types: + - "major" + ignore: + # Wasmtime is updated by hand. Its version is also pinned in + # pkg/contrib/src/wasmtime/version, EOL.md and pkg/eol.json, which a + # Dependabot pull request cannot change. It stays on the 48 LTS line. + # This also stops Dependabot security pull requests for Wasmtime; the + # weekly audit.yml run reports its advisories instead. + - dependency-name: "wasmtime*" + # bindgen 0.68 emitted opaque structs under clang 22; 0.72 is the + # tested version. A 0.x minor can break, so bump it by hand. + - dependency-name: "bindgen" + update-types: + - "version-update:semver-minor" + - "version-update:semver-major" diff --git a/.github/sanitizer-allowlist.txt b/.github/sanitizer-allowlist.txt new file mode 100644 index 000000000..fb4c3d71c --- /dev/null +++ b/.github/sanitizer-allowlist.txt @@ -0,0 +1,17 @@ +# Known UndefinedBehaviorSanitizer violations, waived by +# .github/scripts/check-sanitizer-reports.sh so that the sanitizer gate can be +# made blocking without first fixing every pre-existing site. +# +# Rules: +# - One POSIX extended regular expression per line, matched against the +# report line as it appears in the harness's unit.log. +# - Every entry names the issue tracking its fix. An entry without one is a +# bug in this file: the point of the list is to shrink, and an unattributed +# waiver never does. +# - ASan/LSan/TSan reports cannot be waived here. The script refuses to +# match them on purpose -- a memory-safety report fails the leg, always. +# - Pin the source location. A bare message would waive the same class of +# violation everywhere in the tree, including a new one. +# +# No active waivers. The last four (freeunitorg/freeunit#188, #200) are +# fixed at their source. diff --git a/.github/scripts/check-rust-version.py b/.github/scripts/check-rust-version.py new file mode 100755 index 000000000..e6c228096 --- /dev/null +++ b/.github/scripts/check-rust-version.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +# +# Check every package of a Cargo project with the Rust release that its +# "rust-version" names, so that a dependency update which raises the floor +# fails here instead of in a build that pins an older toolchain. +# +# Usage: check-rust-version.py [] +# +# A package without "rust-version" is an error. + +import os +import subprocess +import sys +import tomllib + + +def load(path): + with open(path, 'rb') as f: + return tomllib.load(f) + + +def run(cmd): + print('+ ' + ' '.join(cmd), flush=True) + return subprocess.run(cmd).returncode + + +def main(): + root = sys.argv[1] + target = ['--target', sys.argv[2]] if len(sys.argv) > 2 else [] + manifest = os.path.join(root, 'Cargo.toml') + top = load(manifest) + workspace = top.get('workspace', {}) + + dirs = [os.path.join(root, m) for m in workspace.get('members', [])] + if 'package' in top: + dirs.insert(0, root) + + failed = False + + for d in dirs: + path = os.path.join(d, 'Cargo.toml') + package = load(path)['package'] + version = package.get('rust-version') + + if isinstance(version, dict) and version.get('workspace'): + version = workspace.get('package', {}).get('rust-version') + + if not isinstance(version, str): + print(f'::error file={path}::{package["name"]} declares no ' + 'rust-version') + failed = True + continue + + if (run(['rustup', 'toolchain', 'install', version, + '--profile', 'minimal', *target]) != 0 + or run(['cargo', f'+{version}', 'check', '--locked', + '--manifest-path', manifest, '-p', package['name'], + *target]) != 0): + print(f'::error file={path}::{package["name"]} does not build ' + f'with Rust {version}, its rust-version') + failed = True + + sys.exit(1 if failed else 0) + + +main() diff --git a/.github/scripts/check-sanitizer-reports.sh b/.github/scripts/check-sanitizer-reports.sh new file mode 100755 index 000000000..63542cbba --- /dev/null +++ b/.github/scripts/check-sanitizer-reports.sh @@ -0,0 +1,126 @@ +#!/bin/sh +# +# Fail a sanitizer CI leg on any ASan/UBSan report the test session produced. +# +# Why this scans the harness's unit.log rather than a sanitizer log_path +# directory: ASAN_OPTIONS/UBSAN_OPTIONS log_path does not work for unitd. +# The daemon relocates argv[]/environ[] in nxt_process_title() +# (src/nxt_process_title.c) and overwrites the original block, so by the time a +# forked router or application worker writes a report the options string the +# sanitizer runtime kept a pointer into is gone. Measured on this tree: an +# ordinary binary run with the same UBSAN_OPTIONS honours log_path, +# halt_on_error and print_stacktrace; inside unitd none of the three take +# effect and the log directory stays empty on every run -- including runs that +# produced a genuine heap-use-after-free. Reports do reach unitd's stderr, +# which test/conftest.py wires to /unit.log, so that is what we read. +# +# The caller must pass --save-log to pytest: without it conftest.py removes the +# per-restart temp dir (and with it unit.log) after every test. +# +# Usage: check-sanitizer-reports.sh [allowlist-file] +# directory holding the harness's unit-test-* temp dirs +# (tempfile.mkdtemp(prefix='unit-test-'), i.e. $TMPDIR or /tmp) +# [allowlist-file] optional; see .github/sanitizer-allowlist.txt + +set -eu + +log_root=${1:?usage: check-sanitizer-reports.sh [allowlist-file]} +allowlist=${2:-} + +# A sanitizer report line, anchored tightly enough that a request body echoed +# into the --debug log cannot forge one. +# UBSan: "src/nxt_conf.c:664:26: runtime error: ..." +# ASan: "==123==ERROR: AddressSanitizer: heap-use-after-free ..." +ubsan_re='[^[:space:]]+:[0-9]+:[0-9]+: runtime error: ' +asan_re='(ERROR|SUMMARY): [A-Za-z]*Sanitizer:' + +logs=$(find "$log_root" -maxdepth 2 -name unit.log -type f 2>/dev/null | sort) + +if [ -z "$logs" ]; then + echo "::error::No unit.log found under ${log_root}. The sanitizer gate" \ + "cannot certify this leg: either the test session never started, or" \ + "pytest was invoked without --save-log and the harness removed the" \ + "temp dirs. Refusing to report a clean run." + exit 1 +fi + +echo "Scanning $(echo "$logs" | wc -l) unit.log file(s) under ${log_root}:" +echo "$logs" | sed 's/^/ /' +echo + +findings=$(mktemp) +patterns=$(mktemp) +trap 'rm -f "$findings" "$patterns"' EXIT + +# -a is load-bearing, not defensive. unitd's --debug log carries raw payload +# bytes, tens of thousands of NULs in a typical run, so grep classifies +# unit.log as binary: it prints the matches it found before the first NUL and +# then silently stops. Measured on a churn log: 3 of 8 report lines survived, +# and the two AddressSanitizer lines -- which come later in the file than the +# first NUL -- were among the 5 dropped. Without -a this gate reproduces the +# exact bug it was written to remove. +for f in $logs; do + grep -aEH "$ubsan_re|$asan_re" "$f" 2>/dev/null || true +done > "$findings" + +if [ ! -s "$findings" ]; then + echo "No sanitizer reports." + exit 0 +fi + +# An ASan/LSan/TSan report is never allowlistable -- those are memory-safety +# bugs, not style debt. Only UBSan lines can be waived, and only by an entry +# that names an open issue. +hard=$(grep -aE "$asan_re" "$findings" || true) + +soft=$(grep -aEv "$asan_re" "$findings" || true) +known='' +unknown=$soft + +if [ -n "$allowlist" ] && [ -f "$allowlist" ] && [ -n "$soft" ]; then + # Patterns go through a file, not a heredoc: `grep -f -` would take its + # pattern list from the same stdin the findings are piped in on, match + # nothing, and report a clean leg -- the exact failure this gate exists to + # remove. + grep -Ev '^[[:space:]]*(#|$)' "$allowlist" > "$patterns" || true + + if [ -s "$patterns" ]; then + known=$(printf '%s\n' "$soft" | grep -aE -f "$patterns" || true) + unknown=$(printf '%s\n' "$soft" | grep -aE -v -f "$patterns" || true) + fi +fi + +if [ -n "$known" ]; then + echo "Known UBSan violations waived by ${allowlist} (tracked, not fixed here):" + printf '%s\n' "$known" | sed 's/.*unit\.log://' | sort | uniq -c | sort -rn | + sed 's/^/ /' + echo +fi + +if [ -z "$hard" ] && [ -z "$unknown" ]; then + echo "No unwaived sanitizer reports." + exit 0 +fi + +if [ -n "$hard" ]; then + echo "AddressSanitizer/LeakSanitizer report(s) -- never waivable:" + printf '%s\n' "$hard" | sed 's/^/ /' + echo + echo "Stack frames from the same log(s):" + for f in $logs; do + grep -aE '^ *#[0-9]+ 0x' "$f" 2>/dev/null | head -40 | sed 's/^/ /' + done + echo +fi + +if [ -n "$unknown" ]; then + echo "UBSan violation(s) not in ${allowlist:-(no allowlist)}:" + printf '%s\n' "$unknown" | sed 's/.*unit\.log://' | sort | uniq -c | sort -rn | + sed 's/^/ /' + echo +fi + +echo "::error::Sanitizer report(s) in this leg; see the step output above and" \ + "the uploaded unit-logs artifact. A new UBSan site must be fixed, or" \ + "waived in .github/sanitizer-allowlist.txt with an issue reference." +exit 1 diff --git a/.github/scripts/check-test-hooks.sh b/.github/scripts/check-test-hooks.sh new file mode 100755 index 000000000..32178c321 --- /dev/null +++ b/.github/scripts/check-test-hooks.sh @@ -0,0 +1,271 @@ +#!/bin/sh +# +# Check that the NXT_TESTS hooks reach the test programs and nothing else. +# +# A "hook" is a fault-injection branch or counter compiled into the core or +# into libunit only for the C test suite, inside "#if (NXT_TESTS)". With +# "./configure --tests", "make tests" compiles every core and libunit source +# twice: build/src/X.o without the macro and build/src/X.test.o with it +# (auto/make). unitd, libnxt.a, libunit.a and the language modules are built +# from the plain objects; the test programs link libnxt-tests.a and +# libunit-tests.a, built from the instrumented ones. +# +# The script derives everything from the build. It keeps no list of hook +# names or hooked sources, so a new hook needs no change here. +# +# hook symbols: the symbols defined in X.test.o and not in X.o. Names that +# contain a "." are compiler-made clones (foo.isra.0, foo.cold) +# and are skipped; a hook is a C identifier. +# members: the global symbols of X.test.o. A program that defines one +# of them linked object X, from one of the two archives. +# +# Checks, in order: +# +# 0. NXT_TESTS is not in build/include/nxt_auto_config.h, and at least one +# hook symbol exists. If the macro leaked into the plain objects, X.o and +# X.test.o are the same and the hook set is empty; that is a failure, not +# a pass. +# 1. Every src/**/*.c outside src/test/ that mentions NXT_TESTS has a +# build/src/X.test.o, and that object differs from X.o once debug info +# is removed. A hook in a source that is compiled once (for example +# src/nxt_main.c or a module source) would ship; a plain object that +# equals its instrumented twin was compiled with the macro. +# 2. No shipped artifact defines a hook symbol: build/sbin/unitd, +# build/lib/libnxt.a, build/lib/libunit.a, build/src/nxt_unit.o (the +# object the language modules link), and, when present, build/lib/libnxt.so +# and every build/*.unit.so. +# 3. Every test program that linked object X defines every hook of X. This +# proves it took X from the instrumented archive, not the plain one. +# +# Usage: .github/scripts/check-test-hooks.sh [build-dir] +# +# Run it after "./configure --tests", "make" and "make tests". It needs +# nm(1), objcopy(1) and cmp(1). + +set -eu + +# sort(1) and comm(1) must agree on the collation order +LC_ALL=C +export LC_ALL + +build=${1:-build} + +tmp=$(mktemp -d) +finished= + +# With "set -e" a failing nm, comm or awk would end the script without a +# word; say so, and only then is a silent exit impossible. +finish() +{ + status=$? + rm -rf "$tmp" + + if [ -z "$finished" ] && [ $status -ne 0 ]; then + echo "::error::check-test-hooks.sh aborted with status $status" >&2 + fi +} + +trap finish EXIT + +fail=0 + +error() +{ + echo "::error::$*" >&2 + fail=1 +} + +# defined symbol names of one file (an object, an archive or a program) +names() +{ + nm --defined-only -p "$1" 2>/dev/null | awk 'NF >= 3 { print $NF }' \ + | sort -u +} + +# defined global symbol names of one object +global_names() +{ + nm --defined-only -p "$1" 2>/dev/null \ + | awk 'NF >= 3 && $(NF - 1) ~ /^[A-Z]$/ { print $NF }' | sort -u +} + + +# 0. The macro is per object, and the derivation works. + +if grep -q 'NXT_TESTS' "$build/include/nxt_auto_config.h"; then + error "NXT_TESTS is defined in $build/include/nxt_auto_config.h;" \ + "it must stay a per-object macro (see auto/make)" +fi + +test_objs=$(find "$build/src" -name '*.test.o' | sort) + +if [ -z "$test_objs" ]; then + error "no *.test.o under $build/src; run 'make tests' first" + finished=1 + exit 1 +fi + +: > "$tmp/hooks" + +for t in $test_objs; do + p=${t%.test.o}.o + + if [ ! -f "$p" ]; then + error "$t has no plain object $p" + continue + fi + + names "$p" > "$tmp/plain" + names "$t" > "$tmp/instr" + + # symbols only the instrumented object defines + comm -13 "$tmp/plain" "$tmp/instr" | grep -v '\.' > "$tmp/hooks.$$" || true + + if [ -s "$tmp/hooks.$$" ]; then + rel=${t#"$build"/} + rel=${rel%.test.o} + awk -v o="$rel" '{ print o, $0 }' "$tmp/hooks.$$" >> "$tmp/hooks" + fi +done + +nhooks=$(wc -l < "$tmp/hooks") + +if [ "$nhooks" -eq 0 ]; then + error "no symbol differs between the plain and the instrumented objects;" \ + "NXT_TESTS leaked into the plain build or is missing from the" \ + "instrumented one" + finished=1 + exit 1 +fi + +echo "hook symbols: $nhooks" +sed 's/^/ /' "$tmp/hooks" + + +# 1. Every hooked source is compiled twice, and the two objects differ. +# +# The symbol diff above cannot see a hook that adds no symbol, and it goes +# empty for an object that was compiled with the macro both times. So for +# every source that tests NXT_TESTS in a preprocessor directive the plain and +# the instrumented object, debug info removed, must not be the same file. A +# mention in a comment is not a hook and is not counted. + +for src in $(find src -name '*.c' -not -path 'src/test/*' | sort); do + grep -Eq '^[[:space:]]*#[[:space:]]*(if|ifdef|ifndef|elif)\b.*NXT_TESTS' \ + "$src" || continue + + p=$build/${src%.c}.o + t=$build/${src%.c}.test.o + + if [ ! -f "$t" ]; then + error "$src tests NXT_TESTS but is compiled once, without the" \ + "macro; its hooks would ship or never link. Hooks belong in" \ + "a source of NXT_LIB_SRCS or NXT_LIB_UNIT_SRCS (auto/sources)" + continue + fi + + objcopy --strip-debug "$p" "$tmp/plain.o" + objcopy --strip-debug "$t" "$tmp/instr.o" + + if cmp -s "$tmp/plain.o" "$tmp/instr.o"; then + error "$src tests NXT_TESTS but $p and $t are the same object;" \ + "the plain object was compiled with the macro" + fi +done + + +# 2. Shipped artifacts carry no hook symbol. +# +# A static function that the plain compile inlines and the instrumented one +# does not is in the hook set too. Another plain object may define a static +# of the same name, and that name is then legitimately in unitd. Such names +# cannot be checked by name; they are reported and left out. This hides no +# leak: a plain object compiled with the macro is caught by check 0, because +# auto/make has one rule template for all plain objects. + +awk '{ print $2 }' "$tmp/hooks" | sort -u > "$tmp/hooknames" + +for t in $test_objs; do + names "${t%.test.o}.o" +done | sort -u > "$tmp/plainnames" + +ambiguous=$(comm -12 "$tmp/hooknames" "$tmp/plainnames") + +if [ -n "$ambiguous" ]; then + echo "not checked by name, defined by another plain object:" \ + "$(echo $ambiguous)" + comm -23 "$tmp/hooknames" "$tmp/plainnames" > "$tmp/hooknames.$$" + mv "$tmp/hooknames.$$" "$tmp/hooknames" +fi + +shipped="$build/sbin/unitd $build/lib/libnxt.a $build/lib/libunit.a \ + $build/src/nxt_unit.o" + +if [ -f "$build/lib/libnxt.so" ]; then + shipped="$shipped $build/lib/libnxt.so" +fi + +shipped="$shipped $(find "$build" -name '*.unit.so' | sort)" + +for f in $shipped; do + if [ ! -f "$f" ]; then + error "$f is missing; run 'make', 'make $build/lib/libunit.a' and" \ + "'make tests' first" + continue + fi + + names "$f" > "$tmp/defined" + + leaked=$(comm -12 "$tmp/hooknames" "$tmp/defined") + + if [ -n "$leaked" ]; then + error "$f defines NXT_TESTS hook symbols: $(echo $leaked)" + fi +done + + +# 3. The test programs took every linked object from the instrumented archive. + +programs="$build/tests $build/ncq_test $build/vbcq_test $build/unit_app_test \ + $build/unit_close_test $build/unit_port_recv_test $build/unit_msg_test \ + $build/unit_websocket_chat $build/unit_websocket_echo" + +for prog in $programs; do + if [ ! -f "$prog" ]; then + error "$prog is missing; run 'make tests' first" + continue + fi + + names "$prog" > "$tmp/defined" + + for t in $test_objs; do + rel=${t#"$build"/} + rel=${rel%.test.o} + + awk -v o="$rel" '$1 == o { print $2 }' "$tmp/hooks" | sort -u \ + > "$tmp/want" + + [ -s "$tmp/want" ] || continue + + global_names "$t" > "$tmp/members" + + # not linked at all: nothing to prove + [ -n "$(comm -12 "$tmp/members" "$tmp/defined")" ] || continue + + missing=$(comm -23 "$tmp/want" "$tmp/defined") + + if [ -n "$missing" ]; then + error "$prog linked $rel from the plain archive:" \ + "missing hook symbols $(echo $missing)" + fi + done +done + + +finished=1 + +if [ $fail -ne 0 ]; then + exit 1 +fi + +echo "ok: hooks are in the test programs only" diff --git a/.github/workflows/analysis-clang-ast.yml b/.github/workflows/analysis-clang-ast.yml index cf4d61305..12bbfe74b 100644 --- a/.github/workflows/analysis-clang-ast.yml +++ b/.github/workflows/analysis-clang-ast.yml @@ -20,9 +20,18 @@ on: - 'pkg/contrib/**' - '.github/workflows/analysis-clang-ast.yml' +concurrency: + # Cancel superseded runs for the same pull request. Non-PR events fall back + # to the run id, which is unique, so a push is never grouped with another: + # Actions keeps only one PENDING run per group and would drop a queued commit + # even with cancel-in-progress false, which would break bisectability. + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: clang-ast: runs-on: ubuntu-latest + timeout-minutes: 20 container: image: debian:testing @@ -49,7 +58,7 @@ jobs: [ -f "$f" ] && ln -sf "$f" /usr/lib/$(basename "$f") || true done - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - name: Checkout and build clang-ast run: | diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml new file mode 100644 index 000000000..63ae09a53 --- /dev/null +++ b/.github/workflows/audit.yml @@ -0,0 +1,184 @@ + +name: "Audit (cargo)" + +# Every tracked Cargo.lock is checked against the RustSec advisory database, +# and every pkg/contrib source-tarball pin is checked against OSV. The weekly +# trigger is what makes this a gate rather than a formality: a new advisory can +# land against a file nobody has touched, and without the schedule the audit +# would only run when a lock or a pin changes. +on: + pull_request: + paths: + - '**/Cargo.lock' + - '**/Cargo.toml' + # Only wasmtime: both scripts hardcode that pin, so firing this job + # for an njs or wasi-sysroot change would report a green that covers + # nothing. The Makefile is here because its build flags decide which + # crates the audit gate treats as linked. + - 'pkg/contrib/src/wasmtime/version' + - 'pkg/contrib/src/wasmtime/SHA512SUMS' + - 'pkg/contrib/src/wasmtime/Makefile' + - 'pkg/contrib/check-advisories.py' + - 'pkg/contrib/check-lock-audit.py' + - 'pkg/contrib/test_check_lock_audit.py' + - 'pkg/contrib/wasmtime-lock-audit.allow.json' + - 'pkg/eol.json' + # A change to this file must run this file. + - .github/workflows/audit.yml + push: + branches: + - master + paths: + - '**/Cargo.lock' + - '**/Cargo.toml' + # Only wasmtime: both scripts hardcode that pin, so firing this job + # for an njs or wasi-sysroot change would report a green that covers + # nothing. The Makefile is here because its build flags decide which + # crates the audit gate treats as linked. + - 'pkg/contrib/src/wasmtime/version' + - 'pkg/contrib/src/wasmtime/SHA512SUMS' + - 'pkg/contrib/src/wasmtime/Makefile' + - 'pkg/contrib/check-advisories.py' + - 'pkg/contrib/check-lock-audit.py' + - 'pkg/contrib/test_check_lock_audit.py' + - 'pkg/contrib/wasmtime-lock-audit.allow.json' + - 'pkg/eol.json' + - .github/workflows/audit.yml + schedule: + - cron: "0 6 * * 1" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + # Cancel superseded runs for the same pull request. Non-PR events fall back + # to the run id, which is unique, so a push is never grouped with another: + # Actions keeps only one PENDING run per group and would drop a queued commit + # even with cancel-in-progress false, which would break bisectability. + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + locks: + # The lock list is discovered instead of hard-coded, so a lock added by a + # future tool is covered the moment it is committed. Passing it to the + # audit job also makes that a matrix, so every failing lock is reported, + # not just the first. + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + locks: ${{ steps.list.outputs.locks }} + steps: + - uses: actions/checkout@v7 + + - name: List tracked lock files + id: list + run: | + locks=$(git ls-files '*Cargo.lock' \ + | jq -R -s -c 'split("\n") | map(select(length > 0))') + test "$(echo "$locks" | jq 'length')" -gt 0 + echo "locks=$locks" >> "$GITHUB_OUTPUT" + + audit: + needs: locks + runs-on: ubuntu-latest + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + lock: ${{ fromJSON(needs.locks.outputs.locks) }} + steps: + - uses: actions/checkout@v7 + + - run: rustup update stable + + # Prebuilt binary instead of `cargo install cargo-audit`, which would + # compile it from source on every run. + - name: Install cargo-audit + uses: taiki-e/install-action@e407f7bafb71fd004bc5c2da3032e5470cbb6ef0 # v2.87.24 + with: + tool: cargo-audit + + # Without --deny warnings the command exits 0 on an unmaintained, + # unsound or yanked crate, so the job would pass while an advisory is + # open. + - name: Audit ${{ matrix.lock }} + run: cargo audit --deny warnings --file '${{ matrix.lock }}' + + pins: + # A dependency pinned as a source tarball has a lock file inside the + # archive, but it is not tracked in this repository. Check the direct + # source-package pins against OSV, then audit the archived lock. That + # lock covers the whole Wasmtime workspace while Unit links one C API + # library, so check-lock-audit.py fails only on a finding against a crate + # that library actually links, and only when the allowlist does not carry + # a reason for that exact version. + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v7 + + - run: rustup update stable + + - name: Check source-tarball pins + run: python3 pkg/contrib/check-advisories.py + + - name: Test the lock audit gate + run: | + pip install --disable-pip-version-check --quiet pytest + python3 -m pytest pkg/contrib/test_check_lock_audit.py + + - name: Install cargo-audit + uses: taiki-e/install-action@e407f7bafb71fd004bc5c2da3032e5470cbb6ef0 # v2.87.24 + with: + tool: cargo-audit + + - name: Resolve the Wasmtime pin + id: pin + run: | + version="$(sed -n -E 's/^WASMTIME_VERSION := ([0-9.]+)$/\1/p' \ + pkg/contrib/src/wasmtime/version)" + test -n "$version" + echo "archive=wasmtime-v${version}-src.tar.gz" >> "$GITHUB_OUTPUT" + + # The archive is immutable and its digest is committed, so this is one + # download per pin instead of 190 MB on every run and every Monday. The + # digest is checked below either way, so a poisoned entry still fails. + - name: Cache the Wasmtime source archive + uses: actions/cache@v6 + with: + path: ~/wasmtime-archive + key: >- + wasmtime-src-${{ hashFiles('pkg/contrib/src/wasmtime/SHA512SUMS') }} + + - name: Unpack the Wasmtime source archive + run: | + archive="${{ steps.pin.outputs.archive }}" + mkdir -p ~/wasmtime-archive + # --retry because on a scheduled gate a mirror blip and an advisory + # must not both show up as a red job. + test -f ~/"wasmtime-archive/$archive" || \ + curl --fail --location --show-error --retry 3 --retry-delay 5 \ + --output ~/"wasmtime-archive/$archive" \ + "https://packages.freeunit.org/wasmtime/$archive" + digest="$(awk -v archive="$archive" '$2 == archive { print $1 }' \ + pkg/contrib/src/wasmtime/SHA512SUMS)" + test -n "$digest" + printf '%s %s\n' "$digest" ~/"wasmtime-archive/$archive" \ + | sha512sum --check + mkdir "$RUNNER_TEMP/wasmtime" + tar -xzf ~/"wasmtime-archive/$archive" --strip-components=1 \ + --directory "$RUNNER_TEMP/wasmtime" + + - name: Audit the Wasmtime source lock + run: | + # cargo audit exits non-zero on any finding, and the archived lock + # covers the whole Wasmtime workspace, so the report is the input to + # the gate rather than the verdict. An empty report is a failure: + # check-lock-audit.py rejects anything that is not a cargo audit run. + cargo audit --json --file "$RUNNER_TEMP/wasmtime/Cargo.lock" \ + > "$RUNNER_TEMP/audit.json" || true + python3 pkg/contrib/check-lock-audit.py \ + --report "$RUNNER_TEMP/audit.json" \ + --tree "$RUNNER_TEMP/wasmtime" diff --git a/.github/workflows/build-deb.yml b/.github/workflows/build-deb.yml index 3539fe0e5..06ec578fa 100644 --- a/.github/workflows/build-deb.yml +++ b/.github/workflows/build-deb.yml @@ -69,6 +69,7 @@ env: jobs: build-trixie: runs-on: ubuntu-latest + timeout-minutes: 60 container: image: debian:trixie @@ -84,7 +85,7 @@ jobs: apt-get update apt-get install -y --no-install-recommends git ca-certificates - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -160,6 +161,7 @@ jobs: smoke-test: needs: build-trixie runs-on: ubuntu-latest + timeout-minutes: 20 container: image: debian:trixie strategy: @@ -186,7 +188,7 @@ jobs: apt-get update apt-get install -y --no-install-recommends git ca-certificates - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -298,6 +300,7 @@ jobs: package-qa: needs: build-trixie runs-on: ubuntu-latest + timeout-minutes: 20 container: image: debian:trixie strategy: @@ -316,7 +319,7 @@ jobs: apt-get update apt-get install -y --no-install-recommends git ca-certificates - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -355,6 +358,7 @@ jobs: needs: [build-trixie, smoke-test, package-qa] if: startsWith(github.ref, 'refs/tags/') runs-on: ubuntu-latest + timeout-minutes: 15 permissions: contents: write # required to upload assets to the release steps: @@ -448,7 +452,7 @@ jobs: NOTES - name: Attach .deb to GitHub Release - uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v3.0.2 + uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v3.0.3 with: body_path: RELEASE_NOTES.md files: | diff --git a/.github/workflows/build-distros.yml b/.github/workflows/build-distros.yml index 1255c319b..d8334427d 100644 --- a/.github/workflows/build-distros.yml +++ b/.github/workflows/build-distros.yml @@ -1,28 +1,23 @@ name: "Build (Fedora/Alpine · GCC/Clang)" on: - workflow_dispatch: # manual only -# push: -# branches: master -# paths: -# - configure -# - 'auto/**' -# - 'src/**' -# - 'test/**' -# - '.github/workflows/build-distros.yml' -# pull_request: -# branches: master -# paths: -# - configure -# - 'auto/**' -# - 'src/**' -# - 'test/**' -# - '.github/workflows/build-distros.yml' + # Rolling distributions break on their own schedule, not on ours, so this is + # a weekly gate rather than a PR gate: Fedora rawhide and Alpine edge move + # under us between pull requests, and a red check nobody caused is a check + # nobody reads. A failure opens or updates a tracking issue instead. + schedule: + - cron: '17 4 * * 1' + workflow_dispatch: + +permissions: + contents: read + issues: write jobs: fedora-rawhide: runs-on: ubuntu-latest + timeout-minutes: 45 strategy: fail-fast: false @@ -42,13 +37,13 @@ jobs: python-unversioned-command python3 python3-devel \ php-devel php-embedded perl-devel perl-ExtUtils-Embed \ ruby-devel java-devel nodejs-devel nodejs-npm golang \ - cargo rust cmake + cargo rust cmake clang-devel if [ "${{ matrix.compiler }}" = "clang" ]; then dnf -y install --setopt=install_weak_deps=False clang fi npm install -g node-gyp - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - name: configure unit CC=${{ matrix.compiler }} run: | @@ -61,6 +56,22 @@ jobs: - name: make unit run: make -j 4 + # --tests has always been configured here, but only "all" was built, + # so a toolchain that breaks the tests binary alone went unnoticed + # (musl's CMSG_NXTHDR under clang, #475). build/tests itself is built, + # not run: the clone and credential cases need capabilities this + # container may not have, and nxt_tests.c has no per-test filter. + - name: make tests + run: make -j 4 tests + + # The two standalone binaries need no privileges and run here as they do + # in build-test's c-tests. See the same steps in alpine-edge for why. + - name: Run libunit close-provenance test + run: ./build/unit_close_test + + - name: Run libunit port_recv fd-ownership test + run: ./build/unit_port_recv_test + - name: configure unit-php run: ./configure php @@ -128,8 +139,25 @@ jobs: - name: make unit-wasm run: make wasm + # The Rust component module is the reason this workflow is worth running + # weekly: bindgen parses Unit's headers with libclang, and a new libclang + # breaks that silently -- it emits opaque `_address` placeholder structs + # rather than failing, so the crate then fails to compile with errors that + # point at Unit's code. That is what #250 fixed for clang 22, which this + # distribution already ships. Only the gcc leg builds it: bindgen uses + # libclang whatever the C compiler is, so the clang leg would test the + # same parse twice. + - name: configure unit-wasm-wasi-component + run: ./configure wasm-wasi-component + if: matrix.compiler == 'gcc' + + - name: make unit-wasm-wasi-component + run: make wasm-wasi-component + if: matrix.compiler == 'gcc' + alpine-edge: runs-on: ubuntu-latest + timeout-minutes: 45 strategy: fail-fast: false @@ -146,12 +174,12 @@ jobs: apk add gcc make musl-dev openssl-dev pcre2-dev curl \ zlib-dev zstd-dev brotli-dev \ php83-dev php83-embed python3-dev perl-dev ruby-dev \ - openjdk21-jdk cargo rust + openjdk21-jdk cargo rust clang-dev linux-headers if [ "${{ matrix.compiler }}" = "clang" ]; then apk add clang fi - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - name: configure unit CC=${{ matrix.compiler }} run: | @@ -164,10 +192,28 @@ jobs: - name: make unit run: make -j 4 + # --tests has always been configured here, but only "all" was built, + # so a toolchain that breaks the tests binary alone went unnoticed + # (musl's CMSG_NXTHDR under clang, #475). build/tests itself is built, + # not run: the clone and credential cases need capabilities this + # container may not have, and nxt_tests.c has no per-test filter. + - name: make tests + run: make -j 4 tests + + # The two standalone binaries need no privileges, so they run here as + # they do in build-test's c-tests. unit_port_recv_test moves control + # messages over socketpair()s, so it executes the NXT_CMSG_NXTHDR() loop + # #475 changed against musl's macro; nothing else runs that on musl. + - name: Run libunit close-provenance test + run: ./build/unit_close_test + + - name: Run libunit port_recv fd-ownership test + run: ./build/unit_port_recv_test + - name: configure unit-php run: | ln -s /usr/lib/libphp83.so /usr/lib/libphp.so - ./configure php --php-config=/usr/bin/php-config83 + ./configure php --config=/usr/bin/php-config83 - name: make unit-php run: make -j 4 @@ -194,4 +240,101 @@ jobs: run: ./configure java - name: make unit-java - run: make -j 4 java \ No newline at end of file + run: make -j 4 java + + # See the note on the same steps in fedora-rawhide: this is the gate that + # would have caught the bindgen/libclang breakage #250 fixed. Alpine + # matters twice over here -- it defaults clang-dev to clang 22 in edge AND + # in stable 3.24, and it is where the module could not be packaged at all + # before #250. gcc leg only: bindgen goes through libclang either way. + - name: configure unit-wasm-wasi-component + run: ./configure wasm-wasi-component + if: matrix.compiler == 'gcc' + + - name: make unit-wasm-wasi-component + run: make wasm-wasi-component + if: matrix.compiler == 'gcc' + + # Alpine without "linux-headers". The full alpine-edge leg installs it (the + # Rust/bindgen and JDK steps are happier with the kernel UAPI headers around), + # and that is exactly what masked #273 for as long as it existed: the + # memfd_create() probe used to include , which Alpine ships in + # linux-headers rather than in musl-dev, so an ordinary "apk add build-base" + # build silently fell back to shm_open() and every isolation.rootfs app died. + # This leg is deliberately minimal -- core only, no modules -- so that it + # tests the probe and nothing else, and it asserts the probe result instead of + # trusting a green build, which would stay green with the fallback in place. + alpine-no-linux-headers: + runs-on: ubuntu-latest + timeout-minutes: 20 + + container: + image: alpine:edge + + steps: + - name: Install tools/deps + run: | + apk update && apk upgrade + apk add gcc make musl-dev openssl-dev pcre2-dev + if apk info -e linux-headers; then + echo "linux-headers must not be installed on this leg" >&2 + exit 1 + fi + + - uses: actions/checkout@v7 + + - name: configure unit + run: | + set -o pipefail + ./configure --openssl --tests | tee configure.log + + - name: assert memfd_create() was detected + run: | + grep -q 'checking for memfd_create() \.\.\. found' configure.log || { + echo 'memfd_create() not detected without linux-headers (#273)' >&2 + grep -n 'memfd' configure.log build/autoconf.err >&2 || true + exit 1 + } + + - name: make unit + run: make -j 4 + + # A weekly red check that nobody caused is a check nobody reads, so a + # scheduled failure is reported as a tracking issue instead, the same way + # eol-check.yml handles its weekly run. + report: + runs-on: ubuntu-latest + timeout-minutes: 10 + needs: [ fedora-rawhide, alpine-edge, alpine-no-linux-headers ] + if: failure() && github.event_name == 'schedule' + + steps: + - name: Open or update tracking issue + env: + GH_TOKEN: ${{ github.token }} + # This job does not check out the repository, so gh has no local git + # remote to infer the target from and would exit before creating the + # issue -- silently defeating the point of the job. + GH_REPO: ${{ github.repository }} + run: | + title='Rolling distro build is failing (Fedora rawhide / Alpine edge)' + { + echo 'The weekly rolling-distribution build failed. These distributions' + echo 'move between our pull requests, so a failure here is usually a' + echo 'toolchain or package change rather than a regression in Unit.' + echo + echo "Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + echo + echo 'Reproduce locally, e.g. for Alpine:' + echo + echo '```' + echo 'docker run --rm -it --platform linux/amd64 alpine:edge sh' + echo '```' + } > issue-body.md + existing=$(gh issue list --state open --search "in:title ${title}" \ + --json number,title --jq ".[] | select(.title == \"${title}\") | .number" | head -n1) + if [ -n "${existing}" ]; then + gh issue comment "${existing}" --body-file issue-body.md + else + gh issue create --title "${title}" --body-file issue-body.md + fi diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 15be67055..365610709 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -11,6 +11,7 @@ on: - 'pkg/contrib/**' - 'pkg/eol.json' - '.github/workflows/build-test.yml' + - '.github/scripts/check-test-hooks.sh' push: branches: master paths: @@ -22,12 +23,25 @@ on: - 'pkg/contrib/**' - 'pkg/eol.json' - '.github/workflows/build-test.yml' + - '.github/scripts/check-test-hooks.sh' env: # OpenSSL patch version built from source (see "Build OpenSSL" step). Only the # patch level lives here; the 3.6 "slot" (/opt/openssl-3.6, OPENSSL36_* vars) # is stable across patch bumps and intentionally not templated. - OPENSSL_VERSION: 3.6.2 + OPENSSL_VERSION: 3.6.5 + # OpenSSL 4.0 for the openssl4 leg below. No distribution ships 4.0 yet, so + # like the 3.6 slot it is built from source; the /opt/openssl-4.0 slot is + # stable across patch bumps. + OPENSSL4_VERSION: 4.0.3 + +concurrency: + # Cancel superseded runs for the same pull request. Non-PR events fall back + # to the run id, which is unique, so a push is never grouped with another: + # Actions keeps only one PENDING run per group and would drop a queued commit + # even with cancel-in-progress false, which would break bisectability. + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: # Single source of truth for the test matrix. The version-tested language @@ -37,25 +51,226 @@ jobs: # and the "minimal" image — those are Docker packaging flavors, not test axes. prepare: runs-on: ubuntu-latest + timeout-minutes: 10 outputs: builds: ${{ steps.list.outputs.builds }} steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - name: Derive build list from pkg/eol.json id: list + # Slow legs come first in the list. All 25 legs are created at + # once. The hosted runner pool is smaller than the matrix, so some + # legs wait for a runner. GitHub does not promise to start legs in + # matrix order. Two runs (36318226905 and 36322425869) showed no + # such order: python-3.12 was last in the old matrix and started + # last; it was first in this matrix and started 12th of 23. The + # order is a hypothesis and has no cost. The set of legs does not + # change. Times from run 36318226905: python-3.12 372 s (pytest + # 332 s; 243 s and 170 s on 3.13 and 3.14), wasm-wasi-component + # 322 s, java 250 to 273 s, php-8.4 255 s. Short legs: perl 109 s, + # wasm 88 s, node about 150 s, ruby about 125 s. run: | builds=$(jq -c ' - ["unit", "perl", "wasm", "wasm-wasi-component"] - + [ .runtimes | to_entries[] - | select(.key | IN("go", "java", "node", "php", "ruby")) - | .key as $rt | .value[] | $rt + "-" + .version ] - + [ .runtimes.python[] | "python-" + .version ] + def legs($rt): [ (.runtimes[$rt] // [])[] | $rt + "-" + .version ]; + legs("python") + + ["wasm-wasi-component"] + + legs("java") + + legs("php") + + ["unit", "perl", "wasm"] + + legs("go") + + legs("node") + + legs("ruby") ' pkg/eol.json) echo "builds=${builds}" >> "$GITHUB_OUTPUT" + # The core build, once for the whole run. Every matrix leg below used to + # run the same "Configure unit" and "Make unit" steps with the same flags, + # so the same objects were compiled 26 times. This job does it once and + # ships the result as an artifact; the legs unpack it and build only their + # own module. Flags here are a copy of what the legs ran before -- keep + # the two in step if they ever change. + core: + runs-on: ubuntu-latest + timeout-minutes: 20 + + steps: + - uses: actions/checkout@v7 + + - name: Read njs version + id: metadata + run: | + NJS_VERSION=$(sed -n "s/NJS_VERSION := \(.*\)/\1/p" pkg/contrib/src/njs/version) + echo "njs_version=${NJS_VERSION}" >> "$GITHUB_OUTPUT" + + - name: Install packages + run: sudo apt-get -y install libbrotli-dev ccache + + - name: Cache ccache + uses: actions/cache@v6 + with: + path: ~/.ccache + key: ccache-${{ runner.os }}-${{ runner.arch }}-core-${{ github.run_id }} + restore-keys: | + ccache-${{ runner.os }}-${{ runner.arch }}-core- + + - name: Enable ccache + run: | + echo "/usr/lib/ccache" >> "$GITHUB_PATH" + echo "CCACHE_DIR=$HOME/.ccache" >> "$GITHUB_ENV" + echo "CCACHE_MAXSIZE=500M" >> "$GITHUB_ENV" + + - name: Cache OpenSSL 3.6 + id: cache-openssl36 + uses: actions/cache@v6 + with: + path: /opt/openssl-3.6 + key: openssl-${{ env.OPENSSL_VERSION }}-${{ runner.os }}-${{ runner.arch }} + + - name: Build OpenSSL 3.6 + if: steps.cache-openssl36.outputs.cache-hit != 'true' + run: | + sudo apt-get -y install build-essential + cd /tmp + wget -q "https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz" + tar -xzf "openssl-${OPENSSL_VERSION}.tar.gz" + cd "openssl-${OPENSSL_VERSION}" + ./Configure --prefix=/opt/openssl-3.6 shared no-docs + make -j$(nproc) + sudo mkdir -p /opt/openssl-3.6 + sudo make install_sw + + - name: Set OpenSSL 3.6 build environment + run: | + OSSL=/opt/openssl-3.6 + LIBDIR=$( [ -d "$OSSL/lib64" ] && echo "$OSSL/lib64" || echo "$OSSL/lib" ) + echo "OPENSSL36_INCDIR=${OSSL}/include" >> $GITHUB_ENV + echo "OPENSSL36_LIBDIR=${LIBDIR}" >> $GITHUB_ENV + + - name: Cache njs + id: cache-njs + uses: actions/cache@v6 + with: + path: njs + key: njs-freeunitorg-${{ steps.metadata.outputs.njs_version }}-${{ runner.os }}-${{ runner.arch }} + + - name: Clone njs repository + if: steps.cache-njs.outputs.cache-hit != 'true' + uses: actions/checkout@v7 + with: + repository: freeunitorg/njs + ref: '${{ steps.metadata.outputs.njs_version }}' + path: njs + + - name: Make njs + if: steps.cache-njs.outputs.cache-hit != 'true' + run: | + ./configure --no-libxml2 --no-zlib + make -j4 -k + working-directory: njs + + # --otel makes "Make unit" run "cargo rustc" on src/otel. ccache + # covers only C. In run 36318226905 that cargo build took 44.5 s of + # the 55 s of "Make unit", on every run. This step caches the crates + # and the target directory. The target directory is cargo's default, + # src/otel/target: auto/make gives no --target-dir. The key has the + # rustc version: the runner image updates rust, and a different rustc + # rebuilds the target anyway. The otel crate itself is rebuilt on + # every run, because checkout gives its sources a new mtime. Local + # test with a warm target: 52 s -> 2 s. + - name: Read rustc version + id: rustc + run: echo "version=$(rustc -V | tr -c 'A-Za-z0-9.\n' '-')" >> "$GITHUB_OUTPUT" + + - name: Cache the otel Rust build + uses: actions/cache@v6 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + src/otel/target + key: otel-cargo-${{ runner.os }}-${{ runner.arch }}-${{ steps.rustc.outputs.version }}-${{ hashFiles('src/otel/Cargo.lock') }} + restore-keys: | + otel-cargo-${{ runner.os }}-${{ runner.arch }}-${{ steps.rustc.outputs.version }}- + + # An --otel build without --openssl must not link OpenSSL. The + # src/otel crate links no TLS stack. This check reads the link + # line that configure writes. readelf on unitd cannot show the + # fault here: the gcc of this runner passes --as-needed, and that + # drops an unused -lssl. The next step runs configure again, so + # this step removes its output. + - name: Check that --otel without --openssl links no OpenSSL + run: | + ./configure --otel + if grep -n -E -- '-l(ssl|crypto)' build/Makefile; then + echo "::error::an --otel build without --openssl links OpenSSL" + exit 1 + fi + rm -rf build Makefile + + - name: Configure unit + # LD_LIBRARY_PATH is set only for this step so that auto/feature + # test binaries load OpenSSL 3.6 at runtime. It is NOT exported + # globally — doing so would break sudo (sudoers.so links against + # system libssl 3.3 which requires OPENSSL_3.3.0 from libcrypto, + # a symbol version that 3.6 does not export in the same slot). + env: + LD_LIBRARY_PATH: ${{ env.OPENSSL36_LIBDIR }} + run: | + ./configure \ + --tests \ + --openssl \ + --njs \ + --zlib \ + --zstd \ + --brotli \ + --otel \ + --cc-opt="-I${OPENSSL36_INCDIR} -I njs/src/ -I njs/build" \ + --ld-opt="-L${OPENSSL36_LIBDIR} -Wl,-rpath,${OPENSSL36_LIBDIR} -L njs/build" + + # CARGO_INCREMENTAL=0, as for wasm-wasi-component: incremental data + # only makes the cached src/otel/target bigger. This build uses + # --release, which already sets it off. This keeps it off if the + # profile changes. + - name: Make unit + env: + CARGO_INCREMENTAL: 0 + run: | + make -j$(nproc) -k || make + + # The default target is `all: unitd manpage`, so --tests above only + # makes this target available -- it does not build it. Without it + # build/unit_app_test does not exist and test_app_lifecycle.py's + # `libunit` parameter self-skips in every leg, silently: the runtime + # that needs no language toolchain and links libunit directly, i.e. + # the one that exercises the port/fd lifecycle with the least in the + # way, would contribute nothing while the legs still reported green. + - name: Make the libunit test application + run: make -j$(nproc) build/unit_app_test + + # tar, not a bare upload: actions/upload-artifact stores a zip and + # restores neither the executable bit nor mtimes. Both matter here -- + # the legs run build/unit_app_test directly, and `make ` decides + # what to rebuild from mtimes. The root Makefile is generated by + # configure (it just includes build/Makefile), so it travels too, and + # build/autoconf.data is what a module configure sources for the core's + # flags. + - name: Pack the core build + run: tar -czf /tmp/unit-core-build.tar.gz Makefile build + + - name: Upload the core build + uses: actions/upload-artifact@v7 + with: + name: unit-core-build + path: /tmp/unit-core-build.tar.gz + retention-days: 1 + compression-level: 0 + test: - needs: prepare + needs: [prepare, core] runs-on: ${{ matrix.os }} + # 45, not 30: the wasm leg now runs the suite on a cold contrib wasmtime + # build (the cache key moves with the pin) and builds the guest toolchain. + timeout-minutes: 45 strategy: fail-fast: false matrix: @@ -63,7 +278,7 @@ jobs: os: [ubuntu-latest] steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 # Provides module, language version and testpath from build name - name: Output build metadata @@ -76,14 +291,149 @@ jobs: IFS='-' read -r module version <<< "${{ matrix.build }}" fi - testpath="test/test_${module}*" - - # Run all tests for "unit" and "python" - # Python is the default module for tests - if [ "$module" = "unit" ] || [ "$module" = "python" ]; then + # test_app_lifecycle.py is runtime-agnostic: it is parametrized over + # every runtime and self-skips the ones this leg did not build, so + # each language leg must be told about it explicitly -- the + # test_* glob below would otherwise leave the app-process + # libunit port/fd lifecycle covered in the python leg only. + testpath="test/test_${module}* test/test_app_lifecycle.py" + + # "unit" is the only leg that runs the language-agnostic suite in + # one piece; the Python-application tests self-skip there without + # the module, and every other test runs as written. + if [ "$module" = "unit" ]; then testpath="test" fi + # A Python leg runs its interpreter-specific suites whole. That is + # what the three versions are for. It also takes a share of the + # other files, balanced by measured time (see below). Three runs + # of the whole language-agnostic suite cost about 12 minutes per + # leg, twice any other language, for tests that no interpreter + # distinguishes. Each shared file now runs on one Python version + # instead of three. The union over the legs does not change. + # + # Both lists are built from the files that exist, never from a glob + # handed to pytest. There is no test_wsgi_*.py -- WSGI is exercised + # through the python module in test_python_* -- and an unmatched + # pattern survives shell expansion literally, so pytest aborts the + # whole leg with "file or directory not found" instead of running the + # rest. `test/test_*.py` is depth 1 while the unit leg runs the whole + # `test` tree, so a nested test file would run on unit and on no + # python leg; keep the two definitions of "the suite" in step. + if [ "$module" = "python" ]; then + versions=$(jq -r '.runtimes.python | length' pkg/eol.json) + shard=$(jq -r --arg v "${version}" \ + '[.runtimes.python[].version] | index($v)' pkg/eol.json) + + # A jq miss prints "null" rather than failing, and a non-integer + # in either comparison below fails open: the leg would run its + # interpreter files, skip every shared one, and still report + # green. Fail loudly instead, like the OpenSSL 4.0 assertion. + case "${versions}" in + ''|0|*[!0-9]*) + echo "::error::no python runtimes in pkg/eol.json" >&2 + exit 1 + ;; + esac + + case "${shard}" in + ''|*[!0-9]*) + echo "::error::no shard for python-${version} in pkg/eol.json" >&2 + exit 1 + ;; + esac + + # Shared files go to the shards by measured time, not by count. + # Round-robin by count gave pytest 332 s / 243 s / 170 s on + # python-3.12 / 3.13 / 3.14 (run 36318226905, PR #448), and the + # 3.12 leg ended the run. The list below has every shared file + # that took 10 s or more in that run, with its time in seconds. + # Each listed file goes to the shard with the least load (the + # first such shard on a tie). All other shared files are + # round-robin, as before. test_app_lifecycle.py took 73 s + # there; 13 s is its time after the keep-alive reads stopped + # waiting for read_timeout. It skips runtimes that are not + # built, so one Python leg is enough for it. The other language + # legs still run it. A file that is not in this list only costs + # balance, not coverage. + heavy=" + test/test_app_start_timeout.py:72 + test/test_settings.py:64 + test/test_otel.py:34 + test/test_proxy.py:27 + test/test_tls.py:22 + test/test_procman.py:21 + test/test_app_lifecycle.py:13 + test/test_idle_close_wait.py:12 + test/test_status.py:11 + test/test_access_log.py:11 + " + + shared="" + whole="" + for f in $(ls test/test_*.py | sort); do + case "$f" in + test/test_python*|test/test_asgi*|test/test_wsgi*) + whole="${whole} ${f}" + ;; + *) + shared="${shared} ${f}" + ;; + esac + done + + declare -A owner + load=() + for ((s = 0; s < versions; s++)); do + load[s]=0 + done + + # A heavy entry that names no shared test file is a stale table: + # the file was renamed or removed, and its time would silently + # stop counting. Fail instead. + for entry in ${heavy}; do + f="${entry%:*}" + case " ${shared} " in + *" ${f} "*) ;; + *) + echo "::error::python shards: ${f} in the heavy table" \ + "is not a shared test file" >&2 + exit 1 + ;; + esac + + min=0 + for ((s = 1; s < versions; s++)); do + if [ "${load[s]}" -lt "${load[min]}" ]; then + min=$s + fi + done + + owner[$f]=$min + load[min]=$((load[min] + ${entry##*:})) + done + + i=0 + for f in ${shared}; do + if [ -z "${owner[$f]+set}" ]; then + owner[$f]=$((i % versions)) + i=$((i + 1)) + fi + done + + # Each shared file has one owner by construction: the heavy loop + # above, then round-robin. Collect this shard's files. + files="" + for f in ${shared}; do + if [ "${owner[$f]}" -eq "$shard" ]; then + files="${files} ${f}" + fi + done + + testpath="${files}${whole}" + fi + echo "module=${module}" >> "$GITHUB_OUTPUT" echo "version=${version}" >> "$GITHUB_OUTPUT" echo "testpath=${testpath}" >> "$GITHUB_OUTPUT" @@ -178,7 +528,7 @@ jobs: - name: Clone njs repository if: steps.cache-njs.outputs.cache-hit != 'true' - uses: actions/checkout@v5 + uses: actions/checkout@v7 with: repository: freeunitorg/njs ref: '${{ steps.metadata.outputs.njs_version }}' @@ -195,29 +545,29 @@ jobs: ## Unit ## - - name: Configure unit - # LD_LIBRARY_PATH is set only for this step so that auto/feature - # test binaries load OpenSSL 3.6 at runtime. It is NOT exported - # globally — doing so would break sudo (sudoers.so links against - # system libssl 3.3 which requires OPENSSL_3.3.0 from libcrypto, - # a symbol version that 3.6 does not export in the same slot). - env: - LD_LIBRARY_PATH: ${{ env.OPENSSL36_LIBDIR }} - run: | - ./configure \ - --tests \ - --openssl \ - --njs \ - --zlib \ - --zstd \ - --brotli \ - --otel \ - --cc-opt="-I${OPENSSL36_INCDIR} -I njs/src/ -I njs/build" \ - --ld-opt="-L${OPENSSL36_LIBDIR} -Wl,-rpath,${OPENSSL36_LIBDIR} -L njs/build" - - - name: Make unit + # The core build comes from the `core` job above: the same configure + # flags and the same objects, compiled once for the whole run instead + # of once per leg. Only this leg's own module is built below. + - name: Download the core build + uses: actions/download-artifact@v8 + with: + name: unit-core-build + path: /tmp + + - name: Unpack the core build + # actions/checkout stamps the sources with the time it ran, which is + # after the core job built them, so make would judge every core object + # out of date and rebuild the very work this job exists to reuse. + # Touching the whole build tree to one timestamp fixes that: make + # rebuilds on a *newer* prerequisite, and equal mtimes count as up to + # date. Everything under build/ came from this same commit, so there + # is no real staleness to mask. run: | - make -j4 -k || make + tar -xzf /tmp/unit-core-build.tar.gz + find Makefile build -exec touch {} + + test -f build/autoconf.data + test -x build/sbin/unitd + test -x build/unit_app_test ## ## Go @@ -244,7 +594,7 @@ jobs: ## Java ## - - uses: actions/setup-java@v5 + - uses: actions/setup-java@v6 with: distribution: temurin java-version: '${{ steps.metadata.outputs.version }}' @@ -252,7 +602,7 @@ jobs: - name: Configure java run: | - sudo ./configure java + sudo ./configure java --home="$(readlink -f "${JAVA_HOME}")" if: steps.metadata.outputs.module == 'java' - name: Make java @@ -293,7 +643,8 @@ jobs: - name: Install libperl-dev run: | - sudo apt-get install libperl-dev + sudo apt-get -y update + sudo apt-get -y install libperl-dev if: steps.metadata.outputs.module == 'perl' - name: Configure perl @@ -318,6 +669,16 @@ jobs: update: true if: steps.metadata.outputs.module == 'php' + # Informational. A package built --with-system-tzdata reports + # "Version => 0.system" and reads /usr/share/zoneinfo at request time, + # i.e. inside an isolation.rootfs chroot; a bundled-timelib build + # reports a real release and never touches the filesystem. Which one + # the runner has decides how test_php_isolation_rootfs behaves. + - name: Report php timezone database + run: | + php -i | grep -i 'timezone database' || true + if: steps.metadata.outputs.module == 'php' + - name: Configure php run: | ./configure php @@ -337,9 +698,19 @@ jobs: python-version: '${{ steps.metadata.outputs.version }}' if: steps.metadata.outputs.module == 'python' + # A second setup-python below installs python-version '3' for pytest and + # overwrites pythonLocation with whatever the newest 3.x is. Record the + # matrix interpreter's prefix while it is still the current one. + - name: Record python prefix + run: | + echo "UNIT_PYTHON_PREFIX=${pythonLocation}" >> "$GITHUB_ENV" + if: steps.metadata.outputs.module == 'python' + - name: Configure python3 run: | - sudo ./configure python --config=python3-config + sudo ./configure python \ + --config="${UNIT_PYTHON_PREFIX}/bin/python3-config" \ + --lib-path="${UNIT_PYTHON_PREFIX}/lib" if: steps.metadata.outputs.module == 'python' - name: Make python3 @@ -381,7 +752,13 @@ jobs: uses: actions/cache@v6 with: path: pkg/contrib/wasmtime/artifacts - key: wasmtime-${{ steps.metadata.outputs.wasmtime_version }}-${{ runner.os }}-${{ runner.arch }} + # Every file that decides what gets downloaded, verified and + # built, not just the version. The Makefile matters most: its cmake + # flags decide which features the library carries, so without it a + # feature change with no version bump would restore the old library + # and skip the rebuild, leaving the tests and the audit gate + # disagreeing about what was linked. + key: wasmtime-${{ steps.metadata.outputs.wasmtime_version }}-${{ hashFiles('pkg/contrib/src/wasmtime/*') }}-${{ runner.os }}-${{ runner.arch }} - name: Make wasmtime run: | @@ -390,7 +767,7 @@ jobs: - name: Configure wasm run: | - ./configure wasm --include-path=pkg/contrib/wasmtime/artifacts/include --lib-path=pkg/contrib/wasmtime/artifacts/lib + ./configure wasm --rpath --include-path=pkg/contrib/wasmtime/artifacts/include --lib-path=pkg/contrib/wasmtime/artifacts/lib if: steps.metadata.outputs.module == 'wasm' - name: Make wasm @@ -398,6 +775,32 @@ jobs: make wasm if: steps.metadata.outputs.module == 'wasm' + # The guest fixture is compiled at test time from pkg/contrib sources: + # wasi-sysroot for wasi-libc and libunit-wasm for the request ABI. + - name: Cache wasi-sysroot + id: cache-wasi-sysroot + if: steps.metadata.outputs.module == 'wasm' + uses: actions/cache@v6 + with: + path: | + pkg/contrib/wasi-sysroot + pkg/contrib/libunit-wasm + # Every file that decides what gets downloaded, verified and built, + # not just the version: otherwise a checksum or build-rule change + # restores the old tree and skips the make below. + key: wasi-sysroot-${{ hashFiles('pkg/contrib/src/wasi-sysroot/*', 'pkg/contrib/src/libunit-wasm/*') }}-${{ runner.os }}-${{ runner.arch }} + + - name: Make wasi-sysroot + run: | + make -C pkg/contrib .wasi-sysroot libunit-wasm + if: steps.metadata.outputs.module == 'wasm' && steps.cache-wasi-sysroot.outputs.cache-hit != 'true' + + - name: Install wasm guest toolchain + run: | + sudo apt install -y clang lld + sudo apt install -y "libclang-rt-$(clang -dumpversion | cut -d. -f1)-dev-wasm32" + if: steps.metadata.outputs.module == 'wasm' + ## ## wasm-wasi-component ## @@ -410,19 +813,48 @@ jobs: # Prebuilt binary instead of `cargo install cargo-component` (which # compiles it from source on every run). Same tool, much faster. - name: Install cargo-component - uses: taiki-e/install-action@v2 + uses: taiki-e/install-action@e407f7bafb71fd004bc5c2da3032e5470cbb6ef0 # v2.87.24 with: tool: cargo-component if: steps.metadata.outputs.module == 'wasm-wasi-component' + # In run 36318226905 "Make wasm-wasi-component" took 189 s: 188 + # crates built from nothing on every run. This step caches the + # crates and the target directory, like the otel build in the core + # job. "Setup rust" above installs the latest stable, so the key has + # the rustc version. This crate is rebuilt on every run, because + # checkout gives its sources a new mtime. + - name: Read rustc version + id: rustc + run: echo "version=$(rustc -V | tr -c 'A-Za-z0-9.\n' '-')" >> "$GITHUB_OUTPUT" + if: steps.metadata.outputs.module == 'wasm-wasi-component' + + - name: Cache the wasm-wasi-component Rust build + uses: actions/cache@v6 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + src/wasm-wasi-component/target + key: wasm-wasi-component-cargo-${{ runner.os }}-${{ runner.arch }}-${{ steps.rustc.outputs.version }}-${{ hashFiles('src/wasm-wasi-component/Cargo.lock') }} + restore-keys: | + wasm-wasi-component-cargo-${{ runner.os }}-${{ runner.arch }}-${{ steps.rustc.outputs.version }}- + if: steps.metadata.outputs.module == 'wasm-wasi-component' + - name: Configure wasm-wasi-component run: | ./configure wasm-wasi-component if: steps.metadata.outputs.module == 'wasm-wasi-component' + # bindgen picks clang up from PATH. The libclang-drift check against + # new distribution compilers lives in build-distros.yml. + # CARGO_INCREMENTAL=0: incremental data only makes the cached target + # bigger. The release profile already sets it off. This keeps it + # off if the profile changes. - name: Make wasm-wasi-component + env: + CARGO_INCREMENTAL: 0 run: | - CLANG_PATH=/usr/bin/clang-15 \ BINDGEN_EXTRA_CLANG_ARGS="-I../../njs/src -I../../njs/build" \ make wasm-wasi-component if: steps.metadata.outputs.module == 'wasm-wasi-component' @@ -477,16 +909,91 @@ jobs: run: | sudo apt install -y python3-pytest - if: steps.metadata.outputs.module != 'wasm' - - name: Run ${{ steps.metadata.outputs.module }} tests + # GitHub runners are UTC, where localtime() and gmtime() agree, so a + # UTC-only run cannot tell a correct HTTP-date from one rendered in + # local time and labelled "GMT" (see test_static_last_modified_is_gmt, + # which skips itself, loudly, when the offset is zero). Kolkata is + # +05:30 all year: never UTC, no DST to shift it, and a half-hour + # offset that also catches whole-hour assumptions. + # "sudo -E" below keeps it for unitd, which inherits pytest's env. + env: + TZ: Asia/Kolkata run: | - if [ "${{ matrix.build }}" == "wasm-wasi-component" ]; then - pytest-3 --print-log ${{ steps.metadata.outputs.testpath }} + # Pin unitd's embedded interpreter to the toolcache prefix it was + # built against; conftest turns this into PYTHONHOME for unitd only. + if [ "${{ steps.metadata.outputs.module }}" == "python" ]; then + export UNIT_PYTHONHOME="${UNIT_PYTHON_PREFIX}" + fi + # The wasm legs build their guest fixture as the runner user, so + # they run unprivileged to keep the result reachable by pytest; the + # others install applications as root. + # --unsafe runs the long Autobahn 9.x websocket cases. + if [ "${{ matrix.build }}" == "wasm-wasi-component" ] || [ "${{ matrix.build }}" == "wasm" ]; then + set -o pipefail + pytest-3 --print-log --unsafe ${{ steps.metadata.outputs.testpath }} 2>&1 | tee "$RUNNER_TEMP/pytest.out" else - sudo -E pytest-3 --print-log ${{ steps.metadata.outputs.testpath }} + sudo -E pytest-3 --print-log --unsafe ${{ steps.metadata.outputs.testpath }} fi - if: steps.metadata.outputs.module != 'wasm' + + # check_prerequisites() skips the whole file when the clang/wasi-sysroot + # fixture cannot be built, so an incomplete toolchain would leave this + # leg green while it covers nothing. At least one smoke test must have + # actually passed. The wasm-wasi-component leg has the same trap: a + # failed "cargo component build" turns into a missing cargo_component + # feature, and test_wasm-wasi-component.py skips as a whole. + - name: Fail if the wasm smoke test skipped + if: ${{ !cancelled() && (steps.metadata.outputs.module == 'wasm' || steps.metadata.outputs.module == 'wasm-wasi-component') }} + run: | + file="test_${{ steps.metadata.outputs.module }}.py" + grep -qE "${file//./\\.}::.*PASSED" "$RUNNER_TEMP/pytest.out" || { + echo "no $file test passed; the guest toolchain skipped it:" >&2 + grep -E 'SKIPPED|feature' "$RUNNER_TEMP/pytest.out" >&2 || true + exit 1 + } + + # test_capget_fallback.py needs a unitd that actually calls capget(), + # which a root one never does, so every leg above skips it -- the file + # is inside "test" but has never once run. Repeat it here without + # sudo. The python leg is the one chosen because the test drives a + # real application to show it inherits the shim harmlessly, and this + # leg is where a language module exists. + # + # test_configuration.py runs here too: test_unprivileged_user_error + # needs a unitd that is not root. + # + # pytest-3 is named absolutely: setup-python puts a toolcache bin + # directory ahead of /usr/bin for this step and not for the sudo'd one + # above, and both must run the pytest that "apt install python3-pytest" + # provided. + # + # !cancelled() rather than the implicit success(): this is the only leg + # that ever runs these files, and gating it on the whole sudo'd suite + # above would let one unrelated flake silently take that coverage away + # again -- the very problem this step exists to fix. + - name: Run capget fallback and configuration tests unprivileged + run: | + export UNIT_PYTHONHOME="${UNIT_PYTHON_PREFIX}" + /usr/bin/pytest-3 --print-log test/test_capget_fallback.py test/test_configuration.py + if: ${{ !cancelled() && steps.metadata.outputs.module == 'python' }} + + # test_asgi_lifespan.py stops unitd from inside the test to check the + # shutdown event, test_tls_certificate_update_restart restarts it to + # check that a replaced bundle survives, and test_state_store.py runs + # unitd on its own state directory and stops it while a store runs. + # test_njs_modules.py stores njs modules and starts unitd again to + # check what it loads from the state directory. + # All need --restart, which the run above does not pass, so they skip + # there. Run them again. + - name: Run the tests that restart unitd, with --restart + run: | + export UNIT_PYTHONHOME="${UNIT_PYTHON_PREFIX}" + sudo -E /usr/bin/pytest-3 --print-log --restart \ + test/test_asgi_lifespan.py \ + test/test_njs_modules.py \ + test/test_state_store.py \ + test/test_tls.py::test_tls_certificate_update_restart + if: ${{ !cancelled() && steps.metadata.outputs.module == 'python' }} # C unit-test suite (src/test/ -> build/tests): HTTP parser, port/IPC # fault-injection, lvlhsh, mp, rbtree, base64, utf8, clone creds, etc. @@ -495,9 +1002,10 @@ jobs: # privilege-sensitive cases (clone creds) work. c-tests: runs-on: ubuntu-latest + timeout-minutes: 20 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - name: Install packages run: | @@ -507,14 +1015,280 @@ jobs: - name: Configure unit (--tests) run: ./configure --openssl --tests + - name: Build unitd and libunit + run: | + make -j4 + make -j4 build/lib/libunit.a + + - name: Build C tests + run: make -j4 tests + + # The hooks (fault-injection branches and counters inside + # "#if (NXT_TESTS)") must reach the test programs and nothing else. + # The script derives the hook symbols from the build itself, so a new + # hook needs no change here. It fails when a hook symbol is in unitd, + # libnxt.a, libunit.a or nxt_unit.o, when a test program linked a plain + # object, or when a source outside the twice-compiled set mentions + # NXT_TESTS. + - name: Check the test hooks are in the test programs only + run: .github/scripts/check-test-hooks.sh + + - name: Run C tests + run: sudo ./build/tests + + - name: Run libunit close-provenance test + run: ./build/unit_close_test + + - name: Run libunit port_recv fd-ownership test + run: ./build/unit_port_recv_test + + # Also run in c-tests-asan, but that leg is a --debug build and this one + # is not. They execute different code: NXT_DEBUG makes nxt_queue_remove() + # clear the removed link and enables nxt_assert(), so a debug-only run can + # hide a use-after-unlink that the release build would show. + - name: Run libunit malformed port message test + run: ./build/unit_msg_test + + # The C unit-test suite with PCRE 1 instead of PCRE2. nxt_pcre.c has its + # own match limits, recursion limit and callout. No other job compiles + # it. ubuntu-24.04 is pinned because it still ships libpcre3-dev. + c-tests-pcre1: + runs-on: ubuntu-24.04 + timeout-minutes: 20 + + steps: + - uses: actions/checkout@v7 + + - name: Install packages + run: | + sudo apt-get -y update + sudo apt-get -y install build-essential libpcre3-dev + + - name: Configure unit (--no-pcre2 --tests) + run: ./configure --no-pcre2 --tests + + - name: Build unitd + run: make -j4 + + - name: Build C tests + run: make -j4 tests + + - name: Run C tests + run: sudo ./build/tests + + # Same C unit-test suite as c-tests above, built with AddressSanitizer and + # run with LeakSanitizer's default ASAN_OPTIONS (detect_leaks is on unless + # something turns it off). Separate from sanitize.yml: that workflow runs + # unitd itself, a fork-heavy daemon where leak detection is noisy enough + # that it is turned off there deliberately (see its "Configure unit" + # comment); build/tests is a single non-forking process exercising + # nxt_lvlhsh, nxt_mp, the port/IPC fixtures, etc. directly, so a leak here + # is a real one in that fixture's own teardown, not daemon-lifecycle noise. + # No suppressions file: every known leak was fixed at its allocation site + # (test fixtures freeing what they allocated -- lvlhsh buckets, ports, + # runtime processes). + c-tests-asan: + runs-on: ubuntu-latest + timeout-minutes: 20 + + steps: + - uses: actions/checkout@v7 + + - name: Install packages + run: | + sudo apt-get -y update + sudo apt-get -y install build-essential libssl-dev libpcre2-dev clang + + # -fno-sanitize-recover: the first UBSan report fails the job. + # nxt_conf_map_object_test fails only under UBSan. + - name: Configure unit (ASan+LSan+UBSan, --tests) + run: | + ./configure \ + --cc=clang \ + --debug \ + --tests \ + --cc-opt="-fsanitize=address,undefined -fno-sanitize-recover=undefined -fno-omit-frame-pointer -g" \ + --ld-opt="-fsanitize=address,undefined" + - name: Build C tests run: make -j4 tests + # sudo for the same reason as c-tests above: the clone creds cases + # need real root. Running as root does not disable LeakSanitizer -- + # this repo's own sandboxed reproduction ran as root throughout. - name: Run C tests run: sudo ./build/tests + # The standalone libunit test programs are not part of build/tests: each + # has its own main() and links libunit's instrumented archive, and + # "make tests" builds them. c-tests and build-distros run the first two + # without a sanitizer; unit_msg_test runs nowhere else. No sudo: they + # use socketpair()s, fork() and shared memory, and build-distros runs + # them unprivileged. unit_msg_test's children leave with _exit(), which + # skips the leak check, so only its parent process is checked. + # + # LSAN_OPTIONS=use_stacks=0, for these steps only: LeakSanitizer scans + # thread stacks conservatively, so a stale pointer left on the stack at + # exit keeps a leaked object reachable, and whether one is left depends + # on stack layout. With the nxt_unit_done() calls at the end of its + # main() removed, unit_port_recv_test reported the leak in 3 of 10 runs + # with the default options and in 10 of 10 with use_stacks=0. Globals, + # TLS and registers are still roots, so unit_port_recv_test and + # unit_msg_test release the library with nxt_unit_done() before they + # exit; a leak inside libunit is then unreachable and reported every + # time. - name: Run libunit close-provenance test + env: + LSAN_OPTIONS: use_stacks=0 run: ./build/unit_close_test - name: Run libunit port_recv fd-ownership test + env: + LSAN_OPTIONS: use_stacks=0 run: ./build/unit_port_recv_test + + - name: Run libunit malformed port message test + env: + LSAN_OPTIONS: use_stacks=0 + run: ./build/unit_msg_test + + # OpenSSL 4.0 compile gate. 4.0 removed nothing Unit uses, but it deprecated + # more of what Unit still calls, and 5.0 is where the deprecated symbols + # actually go away -- so the value here is having a leg that fails when a new + # deprecation lands, instead of noticing at removal time. Deliberately cheap: + # a build plus the C test suite, no language modules and no pytest run, since + # nothing above the TLS layer changes with the OpenSSL version. + openssl4: + runs-on: ubuntu-latest + timeout-minutes: 30 + + steps: + - uses: actions/checkout@v7 + + - name: Install packages + run: | + sudo apt-get -y update + sudo apt-get -y install build-essential libpcre2-dev + + - name: Cache OpenSSL 4.0 + id: cache-openssl4 + uses: actions/cache@v6 + with: + path: /opt/openssl-4.0 + key: openssl-${{ env.OPENSSL4_VERSION }}-${{ runner.os }}-${{ runner.arch }} + + - name: Build OpenSSL 4.0 + if: steps.cache-openssl4.outputs.cache-hit != 'true' + run: | + cd /tmp + wget -q "https://www.openssl.org/source/openssl-${OPENSSL4_VERSION}.tar.gz" + tar -xzf "openssl-${OPENSSL4_VERSION}.tar.gz" + cd "openssl-${OPENSSL4_VERSION}" + ./Configure --prefix=/opt/openssl-4.0 shared no-docs + make -j$(nproc) + sudo mkdir -p /opt/openssl-4.0 + sudo make install_sw + + - name: Set OpenSSL 4.0 build environment + # Same isolation as the 3.6 slot: never overwrite the system libraries, + # because sudo links against them. + run: | + OSSL=/opt/openssl-4.0 + LIBDIR=$( [ -d "$OSSL/lib64" ] && echo "$OSSL/lib64" || echo "$OSSL/lib" ) + echo "OPENSSL4_INCDIR=${OSSL}/include" >> $GITHUB_ENV + echo "OPENSSL4_LIBDIR=${LIBDIR}" >> $GITHUB_ENV + + - name: Configure unit (--openssl --tests) + # LD_LIBRARY_PATH is scoped to this step only so that auto/feature test + # binaries load OpenSSL 4.0; exporting it globally would break sudo. + # The build itself needs no such help -- -Wl,-rpath below is baked into + # the binaries, and is honoured under sudo too. + env: + LD_LIBRARY_PATH: ${{ env.OPENSSL4_LIBDIR }} + run: | + ./configure \ + --tests \ + --openssl \ + --cc-opt="-I${OPENSSL4_INCDIR} -DOPENSSL_NO_DEPRECATED" \ + --ld-opt="-L${OPENSSL4_LIBDIR} -Wl,-rpath,${OPENSSL4_LIBDIR}" + + # Assert rather than print: if the include/lib flags ever stop taking + # effect, configure falls back to the runner's system OpenSSL 3.x and the + # leg would pass while testing nothing it was added for. + - name: Assert configure found OpenSSL 4.0 + run: | + grep NXT_HAVE_OPENSSL_VERSION build/include/nxt_auto_config.h + grep -q '#define NXT_HAVE_OPENSSL_VERSION "OpenSSL 4\.' \ + build/include/nxt_auto_config.h + + - name: Build unit + run: make -j4 + + - name: Build C tests + run: make -j4 tests + + - name: Run C tests + run: sudo ./build/tests + + # --hardening=strict compile gate, with gcc and with clang. auto/cc/test + # already passes -Werror to both, so a new warning from -Wformat=2, -Wvla or + # -Wimplicit-fallthrough (a fall-through without nxt_fallthrough) fails the + # job. The assert step fails when a compiler stops accepting one of the + # probed flags: auto/cc/hardening only drops such a flag, so a plain build + # would pass while testing nothing. Cheap on purpose: a build plus the C + # test suite, no language modules. + hardening-strict: + runs-on: ubuntu-latest + timeout-minutes: 20 + + strategy: + fail-fast: false + matrix: + cc: [gcc, clang] + + steps: + - uses: actions/checkout@v7 + + - name: Install packages + run: | + sudo apt-get -y update + sudo apt-get -y install build-essential libssl-dev libpcre2-dev clang + + - name: Configure unit (--hardening=strict --tests --openssl) + run: | + ./configure \ + --cc=${{ matrix.cc }} \ + --hardening=strict \ + --tests \ + --openssl 2>&1 | tee configure.log + test "${PIPESTATUS[0]}" -eq 0 + + # Every flag that this job exists for must be found. The FORTIFY probe + # falls back from level 3 to 2, so either level passes. The CET and + # branch-protection flags are not asserted: each applies to one target + # only. + - name: Assert configure found the hardening flags + run: | + grep -q 'hardening: .* strict$' configure.log + for flag in -fstack-protector-strong -fstack-clash-protection \ + -Wl,-z,relro -Wl,-z,now -Wl,-z,noexecstack \ + -Wformat=2 -Wvla -Wimplicit-fallthrough \ + -ftrivial-auto-var-init=pattern + do + grep -q "checking for $flag \.\.\. found" configure.log \ + || { echo "$flag was not found"; exit 1; } + done + grep -q 'checking for -D_FORTIFY_SOURCE=[23] \.\.\. found' \ + configure.log \ + || { echo "_FORTIFY_SOURCE was not found"; exit 1; } + grep -q 'define NXT_HAVE_BUILTIN_OVERFLOW' \ + build/include/nxt_auto_config.h + + - name: Build unit + run: make -j4 + + - name: Build C tests + run: make -j4 tests + + - name: Run C tests + run: sudo ./build/tests diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml index ac66d0164..0bde3de3c 100644 --- a/.github/workflows/claude.yml +++ b/.github/workflows/claude.yml @@ -31,6 +31,7 @@ jobs: (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')) && contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.issue.author_association)) runs-on: ubuntu-latest + timeout-minutes: 30 permissions: contents: read pull-requests: read @@ -39,13 +40,13 @@ jobs: actions: read # Required for Claude to read CI results on PRs steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v7 with: fetch-depth: 1 - name: Run Claude Code id: claude - uses: anthropics/claude-code-action@v1 + uses: anthropics/claude-code-action@cab360f6565aa35a51d6ce9e43f1f4287c0a32ea # v1.0.241 with: claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} @@ -86,6 +87,7 @@ jobs: (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')) && !contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.issue.author_association)) runs-on: ubuntu-latest + timeout-minutes: 5 permissions: issues: write pull-requests: write diff --git a/.github/workflows/eol-check.yml b/.github/workflows/eol-check.yml index ba28091c8..da8338a0e 100644 --- a/.github/workflows/eol-check.yml +++ b/.github/workflows/eol-check.yml @@ -27,15 +27,24 @@ on: permissions: contents: read +concurrency: + # Cancel superseded runs for the same pull request. Non-PR events fall back + # to the run id, which is unique, so a push is never grouped with another: + # Actions keeps only one PENDING run per group and would drop a queued commit + # even with cancel-in-progress false, which would break bisectability. + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: eol-check: runs-on: ubuntu-latest + timeout-minutes: 10 permissions: contents: read issues: write # scheduled run may open/update a tracking issue steps: - name: Checkout - uses: actions/checkout@v5 + uses: actions/checkout@v7 # ubuntu-latest ships a Rust toolchain preinstalled — no setup step needed. # Live-fetch endoflife.date. The CLI degrades gracefully: a single fetch diff --git a/.github/workflows/fuzzing.yml b/.github/workflows/fuzzing.yml index 34e2a1914..d90e4741f 100644 --- a/.github/workflows/fuzzing.yml +++ b/.github/workflows/fuzzing.yml @@ -1,42 +1,110 @@ -name: "Fuzzing (OSS-Fuzz)" +name: "Fuzzing (libFuzzer)" + on: - workflow_dispatch: # manual only - # pull_request: - # paths: - # - 'src/**' - # - 'fuzzing/**' - # - '.github/workflows/fuzzing.yml' - -permissions: {} + # The HTTP parsers are the reason this runs on pull requests: they are the + # code an attacker reaches first, and the seed corpus already covers them. + pull_request: + paths: + - 'src/nxt_http*' + - 'src/nxt_h1proto*' + - 'src/nxt_conf*' + # nxt_basic_fuzz.c drives nxt_utf8_decode() and nxt_utf8_length(), so a + # change to the decoder is a change to what this job fuzzes. + - 'src/nxt_utf8*' + # nxt_http_controller_fuzz.c #includes nxt_controller.c, so a change + # there is a change to what this job fuzzes. + - 'src/nxt_controller*' + - 'src/nxt_unit*' + - 'src/nxt_port_memory_int.h' + - 'src/nxt_span.h' + - 'src/nxt_router*' + # configure --fuzz, auto/fuzzing and auto/sources are what produce the + # binaries; break them and the job has nothing to run. + - configure + - 'auto/**' + - 'fuzzing/**' + - '.github/workflows/fuzzing.yml' + workflow_dispatch: + inputs: + seconds: + description: 'libFuzzer budget per target, in seconds' + required: false + default: '60' + targets: + description: 'Space-separated fuzzer names (empty = all ten)' + required: false + default: '' + +permissions: + contents: read + +concurrency: + # Cancel superseded runs for the same pull request. Non-PR events fall back + # to the run id, which is unique, so a dispatch is never grouped with + # another: Actions keeps only one PENDING run per group and would drop a + # queued one even with cancel-in-progress false. + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: - Fuzzing: + libfuzzer: + name: libFuzzer (ASan + UBSan) runs-on: ubuntu-latest - permissions: - security-events: write + # Ten targets at a 60s budget is a little over ten minutes -- each + # finishes within a second or two of its budget, and run-ci.sh caps the + # worst case at two minutes each, twenty in all. The clang/ASan build is + # a few minutes on top. A dispatch can ask for a much longer budget + # across all ten targets, so it gets a much longer bound; a pull request + # must not, or a hung job holds a runner for an hour. + timeout-minutes: ${{ github.event_name == 'workflow_dispatch' && 120 || 30 }} + steps: - - name: Build Fuzzers - id: build - uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master - with: - oss-fuzz-project-name: 'unit' - language: c - - name: Run Fuzzers - uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master - with: - oss-fuzz-project-name: 'unit' - language: c - fuzz-seconds: 300 - output-sarif: true - - name: Upload Crash - uses: actions/upload-artifact@v4 - if: failure() && steps.build.outcome == 'success' - with: - name: artifacts - path: ./out/artifacts - - name: Upload Sarif - if: always() && steps.build.outcome == 'success' - uses: github/codeql-action/upload-sarif@v4 - with: - # Path to SARIF file relative to the root of the repository - sarif_file: cifuzz-sarif/results.sarif - checkout_path: cifuzz-sarif + - uses: actions/checkout@v7 + + # libclang-rt-dev carries libclang_rt.fuzzer, which -fsanitize=fuzzer + # links against; clang alone is not enough. + - name: Install clang and the sanitizer runtimes + run: | + sudo apt-get -qq update + sudo apt-get -qq install -y --no-install-recommends \ + clang llvm libclang-rt-dev + + - name: Build the fuzzers + env: + CC: clang + CXX: clang++ + CFLAGS: >- + -g -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + -fsanitize=address,undefined -fsanitize=fuzzer-no-link + CXXFLAGS: >- + -g -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + -fsanitize=address,undefined -fsanitize=fuzzer-no-link + # configure compiles and runs small probe programs. Built with ASan + # they report leaks and exit non-zero, and configure then records the + # feature as absent -- a silently different build, not a failure. + ASAN_OPTIONS: detect_leaks=0 + run: | + ./configure --no-regex --no-pcre2 --fuzz=-fsanitize=fuzzer + make fuzz -j"$(nproc)" + + # All ten targets, on a pull request and on a dispatch alike. Picking a + # subset per changed path was tried and dropped: every path filter above + # already maps to a target, `fuzzing/**` maps to all of them, and a + # target left out of the list is a harness that CI compiles but never + # runs. At about 61 seconds each the whole set is close to ten minutes. + - name: Run the fuzzers + env: + FUZZ_SECONDS: ${{ github.event.inputs.seconds || '60' }} + FUZZ_TARGETS: ${{ github.event.inputs.targets }} + run: | + # shellcheck disable=SC2086 # deliberate word splitting + ./fuzzing/run-ci.sh -t "${FUZZ_SECONDS}" ${FUZZ_TARGETS} + + # A reproducer is the only thing that makes a fuzzing failure actionable. + - name: Upload reproducers + if: failure() + uses: actions/upload-artifact@v7 + with: + name: fuzz-artifacts + path: build/fuzz-artifacts/ + if-no-files-found: ignore diff --git a/.github/workflows/lint-whitespace.yml b/.github/workflows/lint-whitespace.yml index beb805b0c..ebd96e834 100644 --- a/.github/workflows/lint-whitespace.yml +++ b/.github/workflows/lint-whitespace.yml @@ -7,11 +7,20 @@ on: pull_request: types: [ opened, synchronize ] +concurrency: + # Cancel superseded runs for the same pull request. Non-PR events fall back + # to the run id, which is unique, so a push is never grouped with another: + # Actions keeps only one PENDING run per group and would drop a queued commit + # even with cancel-in-progress false, which would break bisectability. + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: check-whitespace: runs-on: ubuntu-latest + timeout-minutes: 10 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 with: fetch-depth: 0 diff --git a/.github/workflows/release-docker.yml b/.github/workflows/release-docker.yml index 34519b2d2..05688f312 100644 --- a/.github/workflows/release-docker.yml +++ b/.github/workflows/release-docker.yml @@ -51,12 +51,13 @@ jobs: # consumed by both `build` and `merge` via fromJSON(). setup: runs-on: ubuntu-latest + timeout-minutes: 10 outputs: version: ${{ steps.meta.outputs.version }} variants: ${{ steps.meta.outputs.variants }} steps: - name: Checkout - uses: actions/checkout@v5 + uses: actions/checkout@v7 - name: Resolve version and variant list id: meta @@ -84,6 +85,7 @@ jobs: build: needs: setup runs-on: ${{ matrix.arch.runner }} + timeout-minutes: 90 permissions: contents: read packages: write @@ -102,7 +104,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v5 + uses: actions/checkout@v7 - name: Pin version in Dockerfile run: | @@ -113,14 +115,14 @@ jobs: pkg/docker/Dockerfile.${{ matrix.variant }} - name: Log in to GHCR - uses: docker/login-action@v4 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: registry: ${{ env.GHCR_REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Log in to Docker Hub - uses: docker/login-action@v4 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -146,6 +148,7 @@ jobs: merge: needs: [setup, build] runs-on: ubuntu-latest + timeout-minutes: 20 permissions: contents: read packages: write @@ -157,14 +160,14 @@ jobs: steps: - name: Log in to GHCR - uses: docker/login-action@v4 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: registry: ${{ env.GHCR_REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Log in to Docker Hub - uses: docker/login-action@v4 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} diff --git a/.github/workflows/rust-version.yml b/.github/workflows/rust-version.yml new file mode 100644 index 000000000..3592e44af --- /dev/null +++ b/.github/workflows/rust-version.yml @@ -0,0 +1,81 @@ +name: "Rust (rust-version)" + +# Builds every crate with the Rust release its "rust-version" declares. +# The other pull request jobs build the crates with stable Rust only. +# build-deb.yml pins Rust 1.88.0, but it does not run on pull requests. So +# a dependency update that raises a crate's floor could pass its pull +# request and then break the deb build. There is one leg per directory +# with its own Cargo.lock, the same directories as the cargo entries in +# .github/dependabot.yml. A package without "rust-version" fails its leg. + +on: + pull_request: + paths: + - '**/Cargo.toml' + - '**/Cargo.lock' + - '**/rust-toolchain.toml' + - 'tools/unitctl/unit-openapi/openapi-templates/**' + - 'docs/unit-openapi.yaml' + - '.github/scripts/check-rust-version.py' + - '.github/workflows/rust-version.yml' + push: + branches: master + paths: + - '**/Cargo.toml' + - '**/Cargo.lock' + - '**/rust-toolchain.toml' + - 'tools/unitctl/unit-openapi/openapi-templates/**' + - 'docs/unit-openapi.yaml' + - '.github/scripts/check-rust-version.py' + - '.github/workflows/rust-version.yml' + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + contents: read + +jobs: + check: + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: + - dir: tools/unitctl + setup: openapi + - dir: src/otel + - dir: src/wasm-wasi-component + setup: libunit + - dir: pkg/eol + - dir: test/fake_otlp + - dir: test/fake_upstream + - dir: test/wasm_component/hello_world + target: wasm32-wasip1 + - dir: test/wasm_component/spin_loop + target: wasm32-wasip1 + name: ${{ matrix.dir }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + # The OpenAPI client is generated, not tracked. The generator also + # rewrites unit-openapi/Cargo.toml, so the tracked file is restored. + - name: Generate the OpenAPI client + if: matrix.setup == 'openapi' + run: | + make -C tools/unitctl CARGO=true openapi-generate + git checkout -- tools/unitctl/unit-openapi/Cargo.toml + + # build.rs runs bindgen on the Unit headers and links nxt_unit.o. + - name: Build nxt_unit.o + if: matrix.setup == 'libunit' + run: | + sudo apt-get update + sudo apt-get install -y libclang-dev + ./configure --no-regex + make -j"$(nproc)" build/src/nxt_unit.o + + - name: cargo check with rust-version + run: python3 .github/scripts/check-rust-version.py ${{ matrix.dir }} ${{ matrix.target }} diff --git a/.github/workflows/sanitize.yml b/.github/workflows/sanitize.yml index 58887a126..956bb13e2 100644 --- a/.github/workflows/sanitize.yml +++ b/.github/workflows/sanitize.yml @@ -23,7 +23,13 @@ on: - 'go/**' - 'src/**' - 'test/**' + # The gate is three files, not one: a change to the scan script or to + # the waiver list can break report detection or widen what is ignored + # just as effectively as a change to this workflow, and must not skip + # the job that would catch it. - '.github/workflows/sanitize.yml' + - '.github/scripts/check-sanitizer-reports.sh' + - '.github/sanitizer-allowlist.txt' push: # master is the release/default branch; pre-* are the release-prep # integration branches (e.g. pre-1.35.6) where the hardening actually @@ -39,12 +45,39 @@ on: - 'go/**' - 'src/**' - 'test/**' + # The gate is three files, not one: a change to the scan script or to + # the waiver list can break report detection or widen what is ignored + # just as effectively as a change to this workflow, and must not skip + # the job that would catch it. - '.github/workflows/sanitize.yml' + - '.github/scripts/check-sanitizer-reports.sh' + - '.github/sanitizer-allowlist.txt' + +concurrency: + # Cancel superseded runs for the same pull request. Non-PR events fall back + # to the run id, which is unique, so a push is never grouped with another: + # Actions keeps only one PENDING run per group and would drop a queued commit + # even with cancel-in-progress false, which would break bisectability. + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: sanitize: name: "sanitize (${{ matrix.runtime }})" runs-on: ubuntu-latest + timeout-minutes: 40 + + # NOTE: `continue-on-error` is deliberately NOT set at job level here. + # It is set on every step an experimental leg can fail in: the + # runtime-specific configure/make steps, the Go package install, the test + # run and the gate. Anything earlier -- apt, the core unitd build -- is + # shared infrastructure, and a failure there is a real problem on every + # leg rather than something the experimental runtime is allowed to have. + # Job-level continue-on-error only stops the WORKFLOW RUN from failing; the + # job keeps its own `failure` conclusion, GitHub creates one check run per + # matrix job mirroring it, and the check still shows red in the PR rollup. + # To make an experimental leg genuinely non-blocking the flag has to be on + # the steps that can fail -- see the run and gate steps below. # Blocking on purpose: this job exists to red-line lifecycle teardown # regressions. Its first run caught the controller's raw c->link unlinks @@ -64,8 +97,12 @@ jobs: # unitd master and self-skips without it. It also makes conftest # rmtree the temp dir on teardown, which is fine for these tests. pytest_opts: "--restart" - # test_static is module-free; the graceful/listener drivers need - # the python module, so the shared teardown coverage lives here. + # test_static and test_conn_recycle are module-free; the + # graceful/listener drivers need the python module, so the shared + # teardown coverage lives here. test_conn_recycle exercises the + # connection-struct freelist recycler under ASan (recycle churn + + # engine teardown at listen_threads reconfig). + # # test_conn_task_race drives a send-timeout mid-stream RST abort # straddling keep-alive reuse (freeunit#156): pre-fix a deferred # conn write item dereferenced a freed request-pool task, which @@ -84,13 +121,45 @@ jobs: # buffer reaches the parser's recycle path. Without ASan a stale # buffer is usually silently reused, so only this leg turns that # back into a hard failure. + # + # test_app_lifecycle drives the APPLICATION process's libunit + # port/fd lifecycle (process-count churn, control restart storms, + # whole-application remove/re-add, churn under keep-alive). The + # language module is linked into the app process with the same + # instrumentation as unitd, so a double close(), a use-after-free + # of an nxt_unit_port_t or a stale descriptor in libunit becomes a + # hard ASan failure here instead of an intermittent "[alert]" line + # in one runtime's log. It self-skips every runtime that is not + # built into this leg's unitd. + # + # test_configuration drives the control API only and is here for + # test_listeners_close_before_reply: it reads the router's + # nxt_debug() records to assert that a removed listener's + # descriptor is closed before the removal is acknowledged + # (#276), and skips on any build without --debug. Every + # build-test leg is a release build, so this leg is the only one + # in which that test runs rather than skips. + # + # test_state_store forks a store child from main and stops main + # while the child runs (#516); it needs --restart, and this is + # the leg that runs that path under ASan. + # + # test_settings runs the body_min_rate and send_min_rate floors. + # A floor closes the connection from a read or a write handler, + # in the middle of a body or a response. It also has a TLS case, + # which this leg runs because it is configured with --openssl. tests: >- test/test_graceful_reload.py + test/test_state_store.py test/test_listener_drain.py test/test_static.py test/test_conn_task_race.py test/test_proxy.py test/test_chunked.py + test/test_conn_recycle.py + test/test_app_lifecycle.py + test/test_configuration.py + test/test_settings.py - runtime: php apt: "" configure: php @@ -98,11 +167,39 @@ jobs: # No --restart: test_php_application.py does not signal the master, # and --restart's teardown rmtree trips on test_php_application_ # forbidden's restricted-permission fixture (rmtree PermissionError). + # + # test_php_targets is here for test_php_application_targets_cwd. + # On the old code it fails only when libunit fills released + # memory, and only a --debug build does that. Every build-test + # leg is a release build. + pytest_opts: "" + tests: >- + test/test_php_application.py + test/test_php_targets.py + test/test_app_lifecycle.py + # Go is where the app-side port-fd double-close actually manifested + # (a flaky "close(N) failed: Bad file descriptor" in the go isolation + # leg) and it is the only module that registers libunit add_port / + # remove_port callbacks, i.e. the only one that exercises the + # ownership handoff those callbacks create. Running its churn tests + # under ASan turns a silently-reused descriptor into a hard failure. + # + # experimental: cgo + ASan is a new combination for this repo (the Go + # binary links the sanitized libunit.a through cgo). Kept + # non-blocking for its first green runs; drop `experimental` once it + # is stable. + - runtime: go + apt: "" + configure: go --go-path= + make: go pytest_opts: "" - tests: test/test_php_application.py + experimental: true + tests: >- + test/test_go_application.py + test/test_app_lifecycle.py steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 # Base tools + pytest (needed to run any leg) plus this runtime's extra # apt packages. Runtimes whose toolchain is not in apt add a setup step @@ -155,6 +252,7 @@ jobs: run: | ./configure \ --debug \ + --tests \ --openssl \ --cc-opt="-fsanitize=address,undefined -fno-omit-frame-pointer -O1" \ --ld-opt="-fsanitize=address,undefined" @@ -162,70 +260,143 @@ jobs: - name: Make unit run: make -j$(nproc) unitd + # --tests above only makes this target available; the default target + # does not build it. Without it the `libunit` parameter of + # test_app_lifecycle.py self-skips, and the runtime that links libunit + # directly -- the one worth the most under a sanitizer, since nothing + # else sits between the test and the port/fd paths -- would be absent + # from precisely the legs built to instrument them. + - name: Make the libunit test application + run: make -j$(nproc) build/unit_app_test + # Same probe-leak rationale as the core configure step above. - name: Configure ${{ matrix.runtime }} module + continue-on-error: ${{ matrix.experimental == true }} env: ASAN_OPTIONS: "detect_leaks=0" run: ./configure ${{ matrix.configure }} - name: Make ${{ matrix.runtime }} module + continue-on-error: ${{ matrix.experimental == true }} run: make -j$(nproc) ${{ matrix.make }} - - name: Create ASan log directory - run: mkdir -p "${GITHUB_WORKSPACE}/asan-logs" + # The Go package is consumed from build/go/src/unit.nginx.org/go by + # test/unit/applications/lang/go.py (it writes a go.mod `replace` + # pointing there), so the test apps only build after go-install. The + # generated env.go carries the tree's --cc-opt/--ld-opt into cgo, which + # is how the sanitized libunit.a reaches the Go app process. + - name: Install Go package + continue-on-error: ${{ matrix.experimental == true }} + if: matrix.runtime == 'go' + run: make go-install # ASAN_OPTIONS / UBSAN_OPTIONS are step-level env so pytest and every # forked unitd process inherit them. - # log_path=/asan-logs/{asan,ubsan} - # ASan and UBSan reports from the forked daemons land in - # asan-logs/. files rather than only the per-test unit.log - # the harness rotates away, so the guard step below can fail the job - # even when pytest itself is green. UBSan needs its OWN log_path: - # with only ASAN_OPTIONS set, a UBSan-only violation that does not - # escalate to a fatal signal writes to neither the asan-logs dir nor - # a place the harness scans (it greps unit.log for "Sanitizer", not - # UBSan's "runtime error"), so half the gate could pass silently. # detect_leaks=0 # the fork-heavy daemon makes LeakSanitizer extremely noisy; turning # leak detection on is tracked as a follow-up, once the teardown # paths are use-after-free / null-deref clean. - # pytest_opts is per-runtime (e.g. --restart for the graceful tests). + # print_stacktrace + # honoured wherever the options string survives. It does NOT take + # effect inside unitd itself -- see the header of + # .github/scripts/check-sanitizer-reports.sh -- which is exactly why + # the gate below reads unit.log instead of trusting a log_path + # directory. Kept because it costs nothing and does apply to the + # application processes a `"type": "external"` module execve()s, + # which start a fresh sanitizer runtime with the options intact. + # halt_on_error is deliberately NOT set, for that same reason: in an + # exec'd app process it is live, and it aborts the worker on the + # FIRST violation -- including ones the allowlist waives, since that + # list is a post-hoc log scan and cannot stop a runtime abort. With + # it on, the known nxt_cpymem() zero-length copy (issue #200) killed + # the Go app mid-response and failed unrelated tests with a missing + # response header. Unwaived reports are failed by the gate step + # below, which is the mechanism that is actually load-bearing. + # + # --save-log keeps /unit.log after each test. Without it + # test/conftest.py removes the per-restart temp dir, and with it the only + # copy of any report a forked daemon wrote -- which is what the gate + # below scans. pytest_opts is per-runtime (e.g. --restart for the + # graceful tests). - name: Run ${{ matrix.runtime }} sanitizer subset + continue-on-error: ${{ matrix.experimental == true }} + # TZ: same pin as build-test.yml's pytest step, and for the same + # reason (see its comment) -- without it this leg's + # test_static_last_modified_is_gmt is as vacuous under UTC as that + # one was. env: - ASAN_OPTIONS: "detect_leaks=0:log_path=${{ github.workspace }}/asan-logs/asan" - UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1:log_path=${{ github.workspace }}/asan-logs/ubsan" + ASAN_OPTIONS: "detect_leaks=0" + UBSAN_OPTIONS: "print_stacktrace=1" + TZ: Asia/Kolkata run: | - python3 -m pytest -v ${{ matrix.pytest_opts }} ${{ matrix.tests }} + set -o pipefail + # An image without tzdata reads an unknown zone as UTC, where the + # test above skips itself and the leg stays green -- the pin would + # be silently absent. Assert the offset instead of trusting it. + offset=$(date +%z) + if [ "$offset" != "+0530" ]; then + echo "TZ=$TZ resolved to $offset, expected +0530:" \ + "the zone is missing from this image and the pin does nothing" + exit 1 + fi + python3 -m pytest -v --save-log ${{ matrix.pytest_opts }} \ + ${{ matrix.tests }} | tee "${RUNNER_TEMP}/pytest.out" - - name: Show ASan/UBSan reports + # A leg in which every test skipped exits 0, writes clean logs, and is + # therefore green -- indistinguishable from one that ran and passed. + # That is how a module which failed to build hides: check_*() marks the + # runtime unavailable at session start, every parametrized test skips, + # and nothing anywhere says so. It matters most for a leg being + # watched for "first green runs" before dropping `experimental`, where + # a skipped-everything green is precisely the wrong signal. + - name: Fail on an empty test run if: always() + continue-on-error: ${{ matrix.experimental == true }} run: | - if [ -n "$(ls -A asan-logs 2>/dev/null)" ]; then - for f in asan-logs/*; do - echo "===== ${f} =====" - cat "${f}" - done - else - echo "No ASan/UBSan report files were produced." + if [ ! -s "${RUNNER_TEMP}/pytest.out" ]; then + echo "::error::The ${{ matrix.runtime }} leg produced no pytest" \ + "output; the run step did not get that far." + exit 1 fi - - name: Upload ASan/UBSan reports + # The pass has to belong to THIS runtime. A bare "N passed" is + # satisfied by the runtime-agnostic parameters that run in every + # leg -- libunit above all, which needs no language toolchain and + # so passes even when this leg's own module failed to build and + # every one of its tests skipped. That is the precise condition + # this guard exists to catch, so it looks for a passing test + # parametrized with the runtime's own name. + if ! grep -qE "\[${{ matrix.runtime }}\].*PASSED" \ + "${RUNNER_TEMP}/pytest.out"; then + echo "::error::No ${{ matrix.runtime }} test passed in the" \ + "${{ matrix.runtime }} leg. Its tests skipped or errored," \ + "which exits 0 and would otherwise report this leg green" \ + "while covering nothing of the runtime it is named for." + tail -30 "${RUNNER_TEMP}/pytest.out" + exit 1 + fi + + - name: Upload unit logs if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: - name: asan-ubsan-logs-${{ matrix.runtime }} - path: asan-logs/ - if-no-files-found: ignore - - # Sanitizer reports from forked children do not reliably fail pytest: the - # crash aborts a child while the session continues, and with log_path set - # the report never reaches the unit.log the harness scans. Fail the job - # explicitly on any report file, even when pytest was green. + name: unit-logs-${{ matrix.runtime }} + path: ${{ env.TMPDIR || '/tmp' }}/unit-test-*/unit.log + if-no-files-found: warn + + # Sanitizer reports do not reliably fail pytest. A UBSan violation that + # does not escalate to a signal fails nothing at all: the harness's + # Log.check_alerts() greps unit.log for "[alert]" and "Sanitizer", not + # for UBSan's "runtime error:". An ASan report does abort the router, + # but the session then fails with connection errors that read as an + # infrastructure flake rather than as a memory-safety bug. Measured on + # master: 24 runs of the configuration-churn workload produced a UBSan + # violation in every run with pytest green in 23 of them, and the one + # ASan heap-use-after-free surfaced only as "Client can't connect to the + # server". Fail the job explicitly on what the logs actually contain. - name: Fail on sanitizer reports if: always() + continue-on-error: ${{ matrix.experimental == true }} run: | - if [ -n "$(ls -A asan-logs 2>/dev/null)" ]; then - echo "::error::AddressSanitizer/UBSan produced report(s) in the ${{ matrix.runtime }} leg; see the asan-ubsan-logs-${{ matrix.runtime }} artifact and the 'Show ASan/UBSan reports' step." - exit 1 - fi - echo "No sanitizer reports; ${{ matrix.runtime }} lifecycle paths are clean." + .github/scripts/check-sanitizer-reports.sh \ + "${TMPDIR:-/tmp}" .github/sanitizer-allowlist.txt diff --git a/.github/workflows/security-seccomp.yml b/.github/workflows/security-seccomp.yml index e4c640b52..e079ab1a9 100644 --- a/.github/workflows/security-seccomp.yml +++ b/.github/workflows/security-seccomp.yml @@ -5,26 +5,69 @@ on: paths: - 'pkg/docker/seccomp-no-af-alg.json' - 'pkg/docker/Dockerfile.*' + - 'pkg/docker/docker-entrypoint.sh' - 'test/security/seccomp/**' + - 'test/security/docker/**' + # The hardening matrix documents what the daemon does with credentials + # and capabilities; these are the files that decide it. + - 'src/nxt_capability.*' + - 'src/nxt_credential.*' + - 'src/nxt_isolation.*' + - 'src/nxt_process.*' + # nxt_runtime_conf_init() sets rt->user_cred/rt->group and calls + # nxt_capability_set(), which is the decision this table describes. + - 'src/nxt_runtime.*' + # nxt_main_process.c:556 fills app_conf->user from rt->user_cred, and + # nxt_application.c:669 sets process->user_cred -- the credentials the + # canary reports back. + - 'src/nxt_main_process.*' + - 'src/nxt_application.*' - '.github/workflows/security-seccomp.yml' pull_request: paths: - 'pkg/docker/seccomp-no-af-alg.json' - 'pkg/docker/Dockerfile.*' + - 'pkg/docker/docker-entrypoint.sh' - 'test/security/seccomp/**' + - 'test/security/docker/**' + # The hardening matrix documents what the daemon does with credentials + # and capabilities; these are the files that decide it. + - 'src/nxt_capability.*' + - 'src/nxt_credential.*' + - 'src/nxt_isolation.*' + - 'src/nxt_process.*' + # nxt_runtime_conf_init() sets rt->user_cred/rt->group and calls + # nxt_capability_set(), which is the decision this table describes. + - 'src/nxt_runtime.*' + # nxt_main_process.c:556 fills app_conf->user from rt->user_cred, and + # nxt_application.c:669 sets process->user_cred -- the credentials the + # canary reports back. + - 'src/nxt_main_process.*' + - 'src/nxt_application.*' - '.github/workflows/security-seccomp.yml' workflow_dispatch: permissions: contents: read +concurrency: + # Cancel superseded runs for the same pull request. Non-PR events fall back + # to the run id, which is unique, so a push is never grouped with another: + # Actions keeps only one PENDING run per group and would drop a queued commit + # even with cancel-in-progress false, which would break bisectability. + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: seccomp-af-alg: name: Verify AF_ALG seccomp mitigation runs-on: ubuntu-latest + # Raised from 15 with the hardening matrix added at the end. The image + # build dominates; the matrix itself measured 57 seconds locally. + timeout-minutes: 20 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - name: Pull test image run: docker pull python:3.13-slim-trixie @@ -70,3 +113,12 @@ jobs: - name: Run seccomp tests (FreeUnit image) run: bash test/security/seccomp/test-af-alg.sh freeunit:seccomp-test + + # Reuses the image built above rather than building or pulling another + # one: this is the second security property of the same artifact, and an + # image build is the expensive part of this workflow. + # + # (The workflow is still named after AF_ALG. Renaming it would rename + # the check, so that is left to a deliberate change of its own.) + - name: Run Docker hardening matrix (FreeUnit image) + run: bash test/security/docker/test-hardening.sh freeunit:seccomp-test diff --git a/.github/workflows/security-syscalls.yml b/.github/workflows/security-syscalls.yml new file mode 100644 index 000000000..c75615615 --- /dev/null +++ b/.github/workflows/security-syscalls.yml @@ -0,0 +1,77 @@ +name: "Security (Syscall Drift)" + +on: + pull_request: + paths: + - configure + - 'auto/**' + - 'src/**' + - 'test/syscalls/**' + - '.github/workflows/security-syscalls.yml' + push: + branches: master + paths: + - configure + - 'auto/**' + - 'src/**' + - 'test/syscalls/**' + - '.github/workflows/security-syscalls.yml' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + # Cancel superseded runs for the same pull request. Non-PR events fall back + # to the run id, which is unique, so a push is never grouped with another: + # Actions keeps only one PENDING run per group and would drop a queued commit + # even with cancel-in-progress false, which would break bisectability. + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + syscalls: + name: Syscall drift, advisory (${{ matrix.libc }}) + runs-on: ubuntu-latest + timeout-minutes: 20 + + # Advisory until a baseline has been captured from a CI run and committed. + # The container pins the libc by digest, but which syscalls glibc picks + # still depends partly on the host kernel, and these baselines were taken + # on a developer machine -- so a runner on an older kernel could add a + # fallback syscall the baseline has never seen, and a red gate here would + # more likely be wrong than right. test/syscalls/README.md has the + # four-step sequence for making it blocking; it is one commit. + continue-on-error: true + + strategy: + fail-fast: false + matrix: + libc: [ glibc, musl ] + + steps: + - uses: actions/checkout@v7 + + # Everything happens inside a pinned container (debian:trixie-slim / + # alpine:3.22), so the runner image can move without moving the baseline: + # the captured set depends on the libc, not on the host. The runner's own + # kernel is still in play -- see the artifact note below. + - name: Capture the syscall set + run: test/syscalls/run-in-docker.sh ${{ matrix.libc }} + -o "syscalls-${{ matrix.libc }}.txt" + + # Uploaded even on failure: reconciling a baseline against what CI + # actually observed is the documented update path (test/syscalls/README.md) + # and it only works if the list survives a red run. + - name: Upload the captured list + if: always() + uses: actions/upload-artifact@v7 + with: + name: syscalls-${{ matrix.libc }} + path: syscalls-${{ matrix.libc }}.txt + if-no-files-found: warn + + - name: Compare against the baseline + run: test/syscalls/check.sh + "test/syscalls/baseline-linux-${{ matrix.libc }}.txt" + "syscalls-${{ matrix.libc }}.txt" diff --git a/.github/workflows/unitctl.yml b/.github/workflows/unitctl.yml index 8d285ec36..1c0c7e2c0 100644 --- a/.github/workflows/unitctl.yml +++ b/.github/workflows/unitctl.yml @@ -6,6 +6,9 @@ on: paths: - tools/unitctl/** - docs/unit-openapi.yaml + # A change to this file must run this file. Without it a pull request + # that only edits the workflow is never checked by it. + - .github/workflows/unitctl.yml push: branches: - master @@ -21,9 +24,18 @@ on: permissions: contents: write +concurrency: + # Cancel superseded runs for the same pull request. Non-PR events fall back + # to the run id, which is unique, so a push is never grouped with another: + # Actions keeps only one PENDING run per group and would drop a queued commit + # even with cancel-in-progress false, which would break bisectability. + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: test: runs-on: ${{ matrix.os }} + timeout-minutes: 25 defaults: run: working-directory: tools/unitctl @@ -43,14 +55,14 @@ jobs: os: macos-latest target: aarch64-apple-darwin steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - run: rustup update stable - run: rustup target add ${{ matrix.target }} - name: Install cross if: matrix.target == 'aarch64-unknown-linux-gnu' - uses: taiki-e/install-action@v2 + uses: taiki-e/install-action@e407f7bafb71fd004bc5c2da3032e5470cbb6ef0 # v2.87.24 with: tool: cross @@ -60,7 +72,7 @@ jobs: brew install make gnu-sed grep gawk echo "MAKE=gmake" >> $GITHUB_ENV - - uses: actions/setup-java@v5 + - uses: actions/setup-java@v6 with: distribution: temurin java-version: 21 @@ -72,8 +84,101 @@ jobs: - name: Test ${{ matrix.os }} run: ${{ env.MAKE }} test + musl: + # The only leg that compiles unitctl without the "tls" feature. The + # feature-off code -- the #[cfg(not(feature = "tls"))] blocks and the test + # that an "https://" control socket is refused -- is built nowhere else, so + # without this job it can break and no run would say so. + # + # The musl target is also the one shipped without TLS, because TLS is what + # pulls in aws-lc-sys and its C toolchain. + # + # What this job does not cover: every runner has a gnu host, so a musl + # *host* -- "make release" on Alpine -- is not built here. The make rule + # for that host is checked below instead, which is the part that can be + # checked without a musl runner. + runs-on: ubuntu-latest + timeout-minutes: 30 + defaults: + run: + working-directory: tools/unitctl + steps: + - uses: actions/checkout@v7 + + - run: rustup update stable + - run: rustup target add x86_64-unknown-linux-musl + + # The build needs the generated OpenAPI client, which is produced by a + # Java tool. + - uses: actions/setup-java@v6 + with: + distribution: temurin + java-version: 21 + + # The GNUmakefile drops the default features for a musl target, so this + # is the feature-off build. + - name: Make unitctl (x86_64-unknown-linux-musl) + run: make x86_64-unknown-linux-musl + + # Assert on the binary, not on the build log. Without TLS there is no + # aws-lc-sys, so the binary carries no aws_lc symbol. The symbol count is + # checked first, because a stripped binary would make the aws_lc test pass + # while proving nothing. + # + # A program that links a C library needs an interpreter to load it, so the + # absence of a PT_INTERP segment is what says none was linked. readelf + # names the segment; "file" would say "static-pie linked", which is that + # tool's wording and changes with the runner image. + - name: Check the binary carries no TLS stack + run: | + bin=target/x86_64-unknown-linux-musl/release/unitctl + file "$bin" + test "$(nm "$bin" | wc -l)" -gt 1000 + ! nm "$bin" | grep -q aws_lc + ! readelf -lW "$bin" | grep -q INTERP + + # The claim behind the musl build is that TLS is the only thing needing a + # C toolchain. This runner has cc and cmake, so the job cannot show that + # a build works without them. What it can show is that no crate which + # would use them is in the feature-off graph. The line count guards + # against an empty tree passing the grep. + - name: Check no C-building crate is in the graph + run: | + tree=$(cargo tree --no-default-features \ + --target x86_64-unknown-linux-musl --edges normal,build) + test "$(echo "$tree" | wc -l)" -gt 50 + ! echo "$tree" | grep -E '(^|[^a-z-])(cc|cmake|aws-lc-sys) v' + + # Run the host tests in the same feature state, so the feature-off + # branches are executed and not merely compiled. + - name: Test without the tls feature + run: cargo test --no-default-features + + # The step above stays green if the refusal test is deleted or marked + # #[ignore]. Name the test and require that one test ran: a filter that + # matches nothing also reports "ok", with "0 passed". + - name: Check the https refusal test ran + run: | + cargo test --no-default-features -p unit-client-rs \ + control_socket_address::tests::refuses_https_without_the_tls_feature \ + -- --exact | tee result.log + grep -q 'test result: ok. 1 passed' result.log + rm result.log + + # A native build on a musl host takes the same feature flag, by a + # different rule (GNUmakefile HOST_TARGET). No runner has a musl host, + # so ask make what it would run. Both directions are checked: a rule + # that always passed the flag would break every gnu build. + - name: Check the native musl build drops the feature too + run: | + make -n release HOST_TARGET=x86_64-unknown-linux-musl \ + | grep -q -- '--no-default-features' + ! make -n release HOST_TARGET=x86_64-unknown-linux-gnu \ + | grep -q -- '--no-default-features' + build: runs-on: ${{ matrix.os }} + timeout-minutes: 30 defaults: run: working-directory: tools/unitctl @@ -100,18 +205,18 @@ jobs: target: x86_64-apple-darwin steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - run: rustup update stable - run: rustup target add ${{ matrix.target }} - name: Install cross if: matrix.target == 'aarch64-unknown-linux-gnu' - uses: taiki-e/install-action@v2 + uses: taiki-e/install-action@e407f7bafb71fd004bc5c2da3032e5470cbb6ef0 # v2.87.24 with: tool: cross - - uses: Swatinem/rust-cache@v2 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: prefix-key: rust-${{ matrix.build }} workspaces: ./tools/unitctl -> target @@ -137,7 +242,7 @@ jobs: brew install make gnu-sed grep gawk echo "MAKE=gmake" >> $GITHUB_ENV - - uses: actions/setup-java@v5 + - uses: actions/setup-java@v6 with: distribution: temurin java-version: 21 @@ -161,13 +266,13 @@ jobs: mv ./target/${{ matrix.target }}/release/unitctl unitctl-${{ env.SHORT_VERSION }}-${{ matrix.target }} - name: Upload sha256 sum - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: unitctl-${{ env.SHORT_VERSION }}-${{ matrix.target }}.sha256 path: tools/unitctl/unitctl-${{ env.SHORT_VERSION }}-${{ matrix.target }}.sha256 - name: Upload unitctl - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: unitctl-${{ env.SHORT_VERSION }}-${{ matrix.target }} path: tools/unitctl/unitctl-${{ env.SHORT_VERSION }}-${{ matrix.target }} @@ -177,13 +282,14 @@ jobs: if: startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' needs: [build] runs-on: ubuntu-latest + timeout-minutes: 15 steps: - uses: actions/download-artifact@v8 with: merge-multiple: true - name: Create GitHub release - uses: ncipollo/release-action@v1 + uses: ncipollo/release-action@339a81892b84b4eeb0f6e744e4574d79d0d9b8dd # v1.21.0 with: artifacts: "unitctl-*" prerelease: ${{ github.event_name == 'workflow_dispatch' }} diff --git a/CHANGES b/CHANGES index 69f92f593..db2d177f7 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,1177 @@ -Changes with FreeUnit 1.36.1 02 Aug 2026 +Changes with FreeUnit 1.37.1 04 Oct 2026 + + *) Bugfix: with "chunked_transform", a valid chunked request body got + 400 when a read ended right after the data of a chunk, before the + CRLF after it. The data stayed in the read buffer, and the next read + was parsed after it. So the result depended on how the client's + writes reached the server. The bug had appeared in 1.35.5. + + *) Bugfix: unitd did not load the stored configuration when the + "version" file in the state directory ended with a line end, as + "echo 13700 > version" and most editors write it. It logged + "invalid version string" and started with no configuration. A line + end, a space or a tab after the number is now ignored. Any other + content is still an error. unitd itself writes the file with no + line end. The bug had appeared in 1.26.0. + + *) Change: "--otel" no longer needs OpenSSL. Before, configure + stopped with "OpenTelemetry support requires OpenSSL." when the + OpenSSL headers were missing, and it linked unitd with OpenSSL + even without "--openssl". The OTel exporter has linked no TLS + stack since 1.36.0. macOS builds did not have the problem. + + *) Bugfix: the worker of a wasm application crashed when the guest + called the import nxt_wasm_send_response, nxt_wasm_send_headers or + nxt_wasm_response_end before the first request. This could happen + in the start function or "_initialize" of the guest, in the malloc + handler, or in the module_init hook. These imports now trap when + the worker has no request yet. The worker then does not start, + except in the hook, where the trap ends only the hook. + + *) Bugfix: the wasm module took any export with a handler name as a + function. If that export was not a function, such as a global or + a memory, the worker crashed on the first call of the handler. The + module now refuses such a guest when the worker starts. + + *) Bugfix: with "large_header_buffers" set to 256 or more, the limit + did not work. The router counted the large buffers of a request + header in 8 bits, so the count went back to 0 after 255. A request + header with many fields could then take memory until + "header_read_timeout" ended it. The bug had appeared in 1.29.0. + + +Changes with FreeUnit 1.37.0 04 Oct 2026 + + *) Bugfix: a chunked request body that grew over "max_body_size" + after the read that carried the request header got no response. + The router closed the connection without a status line. Now it + answers 413 and closes. A body that crossed the limit in the same + read as the header already got 413. A later read that fails to + allocate memory in the chunk parser, or to write the body to the + temporary file, now answers 500 and closes in the same way. The + 400 for a malformed chunk in a later read now clears keepalive + and closes the connection, like the other error paths. A short + write of a body with a Content-Length to the temporary file now + answers 500 instead of closing silently. + + *) Bugfix: the router could take a message of an application response + before an earlier message of the same response. With a header over + 1024 bytes and a first body write of 16 to 1024 bytes, it parsed the + body as the header, logged "response buffer too small for fields + count", and answered 503. With a body write over 1024 bytes and a + later smaller one, the client got the two parts in the wrong order, + and nothing was logged. This needed two or more application + processes or threads. The bug had appeared in 1.19.0. The fix is in + libunit: an application of type "external", such as Go or + Node.js, gets it when it is built with the new libunit. + + *) Bugfix: the router did not answer "Expect: 100-continue". A client + that sent it, as curl and Guzzle do for large uploads, waited for its + own timeout before it sent the body. For curl, that timeout is 1 + second. The router now sends "100 Continue" before it waits for the + body, also when a part of the body came with the header. A request + that the router refuses from its header gets its final status with no + 100. A proxied request no longer carries the Expect field to the + upstream. + + *) Bugfix: the PHP module read the target index of a request after the + request was released. libunit fills released request memory with + 0xA5, so the module called chdir(2) on every request to a "script" + target. A request could also run in the directory of another + target. This happened with 166 or more targets, and after + fastcgi_finish_request() when the released memory already held a + request for another target. The bug had appeared in 1.18.0. + + *) Bugfix: flush() in a PHP application did not send the response + header. If the application had no output yet, or its output was + in an output buffer, the client got the header only at the end of + the request, and headers_sent() returned false after flush(). Now + flush() sends the header. As in mod_php, flush() does not empty + the output buffers; ob_flush() does. The PHP module has had no + flush handler since 1.4. + + *) Bugfix: the router sent the header of a chunked HTTP/1.1 response + without the empty line that ends it, and sent that line with the + first body bytes. If an application sent the header and then + waited before its body, the client could not use the header until + the body began. Now the header ends at once. A complete response + has the same bytes as before. $body_bytes_sent no longer counts the + empty line. + + *) Security: with response compression on, the router kept one + compressor stream for each router thread. But it compresses an + application response in several steps. So two application + responses on one thread used the same stream. A client could get + bytes of the response to another client, a body that was not + valid, or a body that stopped early. The router could also crash + when it used a stream that another response had freed. Now each + response has its own stream, and the router frees it when the + response ends or stops early. + + *) Change: the main process now writes the state files in a + short-lived child process, so a slow fsync(2) no longer delays + the start of application processes after a reconfiguration, or + the exit of the main process. One store runs at a time; a newer + configuration waits for it, and only the latest one waiting is + stored. At exit, the main process waits for the store to finish; + when a newer configuration waits, it stops the running store + first. When the store fails, the main process logs an alert. + + *) Change: an uploaded certificate bundle is now also stored by that + child process, so its two fsync(2) calls no longer stop the main + process. The upload is still answered only when the bundle is on + disk. A certificate deletion also runs in the child process, after + every earlier store. A controller process that exits during a store + is started again only when the store ends, so that it reads the + stored files. + + *) Bugfix: a regular expression in the configuration could run for up + to 10,000,000 match steps on one request. A match now stops after + at most 100,000 steps and the request gets a 500 response. The + error log then gets a warning with the pattern and the subject + length, not the subject. In "compression" "types", a match that + stops is not a match, and the response is not compressed. A + repeated group takes about 2 steps per byte. So a subject much + longer than 8 KiB can reach the limit and get a 500 response. Such + a subject needs a "large_header_buffer_size" above its default of + 8192. With PCRE 1 and with PCRE2 before 10.30, a match also stops + at 2,000 nested calls and does not overflow the thread stack. + There, a repeated group on a subject of more than about 1,000 + bytes can get a 500 response. + + *) Security: the wasm module did not check the offset that the malloc + handler of the guest returned. A negative or large offset put the + request buffer outside the linear memory of the guest. The worker + now refuses such an offset, or an offset that is not aligned, and + exits at startup. + + *) Bugfix: an application with a rootfs created the missing + directories of each mount destination with mkdir(), which follows + symlinks. A symlink inside the rootfs could make unitd create empty + directories outside it. The directories are now created without + following symlinks. On a kernel without openat2() (older than 5.6) + a rootfs with a symlinked component in a mount destination (for + example usr -> real_usr) is now refused. A missing rootfs + directory is now refused on every system. Builds without + openat2(), such as on FreeBSD, created it before. + + *) Bugfix: with isolation.namespaces.cgroup and isolation.cgroup.path, + the cgroup namespace of an application was rooted at the cgroup of + the main process, not at the configured cgroup. The application saw + its own cgroup as "/scope/python" instead of "/". The namespace is + now created after the process is moved into its cgroup. + + *) Change: on macOS, a stored state file is now flushed with + fcntl(F_BARRIERFSYNC), and its directory with fcntl(F_FULLFSYNC) + after the rename, so the file also leaves the cache of the drive. + Plain fsync(2) there does not flush that cache, and a power loss + could lose a stored configuration. + + *) Security: the router and libunit read records from shared memory + that the other process can still write, without enough checks. A + buffer that was not a whole number of records was read past its + end. In libunit, a segment id of UINT32_MAX wrapped the size of the + segment array, a segment of any size was accepted, a duplicate + segment id replaced a segment that buffers still used, and a field + could point before the field array. The router trusted a busy chunk + marker that the application can clear. All of these are now + checked. + + *) Bugfix: after 2^32 RPC registrations the stream counter wrapped + and handed out stream 0, which callers read as a failure. The + registration stayed. Stream 0 is now skipped. + + *) Security: the wasm module read the base address of the linear + memory of the guest only once, at startup. A guest with a 64-bit + memory could grow it past 4 GiB, and wasmtime then moved it. The + module then read and wrote the old address, and the worker crashed. + The module now refuses a 64-bit memory. It also reads the base + address again after the guest runs. + + *) Bugfix: an uploaded njs module was written into its file without + truncating it, and without fsync(2). When a longer file of that + name was already on disk, for example one that a failed write had + left, the old tail stayed after the new module. The router read + that file and the configuration failed. The upload was also + answered with 200 when the write failed. Now the main process + stores the module through a temporary file, flushes it and renames + it, in its store child, and the upload is answered only when the + module is on disk. A deletion also runs in the store child. A + module name can no longer start with ".", and a module larger than + 16 MiB is answered with 413. Earlier versions accepted a name that + starts with "."; such a module is no longer loaded or listed after + the upgrade. + + *) Bugfix: at startup, a stored njs module that did not compile hid + every stored module that was read after it. Those modules were + missing from /js_modules and could not be used. Now only the module + that does not compile is left out. + + *) Bugfix: a keep-alive TLS connection that was idle across a + reconfiguration could use the TLS settings of the old configuration + after they were freed, when it was closed. The connection now keeps + the configuration until it is freed. + + *) Bugfix: a read or write that was already queued when a connection + was closed could run on the closed connection: a write could call a + missing error handler, and a read could read the closing socket and + close the connection again. Such a queued handler now does + nothing. + + *) Bugfix: a static file response that failed after the request was + closed could close the file descriptor twice. The second close could + close a descriptor that another connection had got in the + meantime. + + *) Change: on Linux, the default number of router threads + ("listen_threads") is now at most the CPU limit of the cgroup v2 + group of unitd and of its parent groups ("cpu.max"), rounded up to + whole CPUs. Before, it was the number of CPUs that unitd could run + on, so a container limited to 2 CPUs on a 64-CPU host started 64 + router threads. Now it starts 2. A limit of 1.5 CPUs also gives 2 + threads. An explicit "listen_threads" is not changed. With a private + cgroup namespace, only the limits inside the namespace are seen. The + limit is read once, when unitd starts. cgroup v1 is not read. + + *) Bugfix: a proxied request with a chunked body, or an HTTP/2 request + without a content-length, was sent to the upstream with two + Content-Length fields. + + *) Bugfix: with "limits": {"timeout"} set, a WebSocket session was + closed about one timeout after the upgrade, because the request + deadline stayed armed for every frame. The deadline now stops at the + upgrade. + + *) Bugfix: a WebSocket client that sent PINGs and did not read the + PONGs could make the router keep a PONG in memory for each PING. + Now, while a PONG is not sent, the router keeps only the PONG for + the most recent PING, and sends it after the queued PONG (RFC 6455 + Section 5.5.3). + + *) Bugfix: the router crashed when it validated a configuration that + gave "compressors" in the object form. + + *) Change: with telemetry on, the application now gets a traceparent + that names FreeUnit's own span as its parent, so the application's + spans are children of FreeUnit's span. Before, the application got + the client's traceparent unchanged. + + *) Security: an application process could send control messages to + the router. It could make the router quit, load a configuration, + restart applications, forget a process, redirect the error log, or + reopen the access log. The router now checks the sender pid that + the kernel gives. QUIT, CHANGE_FILE and ACCESS_LOG must come from + main. REMOVE_PID must come from main or from a prototype. DATA, + APP_RESTART and STATUS must come from the controller. The router + closes the descriptors of a refused message and does not reply. + + *) Security: an application process could send a false NEW_PORT to + the router, register its own pid as main or as the controller, and + then pass the checks above. It could also get the port of main or + of the controller with GET_PORT, attach a shared memory segment to + another process with MMAP, make a debug router abort with GET_MMAP, + and answer a pending request of the router with RPC_READY or + RPC_ERROR, for example with a false certificate. The router now + accepts NEW_PORT from main, from a prototype for an application + port, and from an application for its own application port only. + GET_PORT, GET_MMAP, MMAP and OOSM must come from the application + that the message names, and GET_PORT gives out only ports of the + router. RPC_READY and RPC_ERROR must come from main, the controller + or a prototype. Every process now refuses a NEW_PORT message that is + too short. Limits: the router does not check that an RPC reply comes + from the process that got the request. The ports of the router + threads get messages through a shared memory queue, which gives no + sender pid, so the router does not check senders there. + + *) Bugfix: when unitd ran as root and the control socket was given to + another user or group with --control-user or --control-group, that + user passed the socket file permissions but the controller then + closed the connection. The controller now also accepts the + configured user, and a peer in the configured group. + + *) Bugfix: a worker that got a QUIT before it was ready did not act on + it, and ran until it was killed 15 seconds later, so unitd was slow + to exit. The worker now exits at once. + + *) Bugfix: at shutdown, the router could ask a prototype that was + already exiting to start a new worker. The prototype then forked a + worker that nobody stopped, and unitd did not exit. If the + prototype was gone, the router logged a false alert. The prototype + now refuses the start, and the router logs a gone prototype at the + debug level. + + *) Change: a QUIT that fails because the worker already exited is now + logged at the info level, not as an alert. + + *) Bugfix: when a router thread ran out of connections or file + descriptors, the log got an "epoll_ctl() failed (2: No such file or + directory)" alert every 100 milliseconds until a connection was + freed. + + *) Bugfix: when a router thread could not allocate the listen event + while a listener was added, a configuration request could wait + forever, or a listening socket shared by several router threads + could be closed too early. When the thread had no free connection + slot instead, the listener never accepted connections. Now the + request completes, and a listener without a free slot starts to + accept connections when a slot is free. + + *) Bugfix: at shutdown, idle client connections were closed but not + freed. They are now closed through their protocol handlers. + + *) Bugfix: on systems without accept4(), an accepted socket that could + not be made non-blocking was closed and then still used. + + *) Security: an application could write an unexpected value into an + entry of a shared memory queue. The router then retried that entry + for ever. Each enqueue and dequeue now stops after one queue size + of retries. The router refuses the message and writes an alert. + + *) Security: a port kept every fragment of a message until the last + one arrived, so a peer process could make it hold memory without + limit. A fragment that named a shared memory segment but carried no + record also left a receive buffer in the stream while the same + buffer was reused for the next read. Reassembly is now limited to + 64 streams, 128 MiB per stream and 256 MiB per port, a new stream + drops the oldest one when 64 are open, and such a fragment is + dropped. A port linked without a process no longer crashes a + release build when it is freed. + + *) Bugfix: an application used a shared memory segment id from the + other process to grow its array of segments without a limit. An id + like 100000000 in one message made it allocate and fill an array of + that many slots, so the process could run out of memory. It now + refuses an id of 65536 or more. The router already refused such ids + from applications; it now uses the same limit for the segments it + creates. + + *) Bugfix: the proxy recognised an upstream Transfer-Encoding only when + its value was exactly "chunked". "Chunked" or "gzip, chunked" was + not recognised, so the body was framed by Content-Length or by + connection close, and the chunk framing reached the client as body + bytes. The value is now parsed as a list of codings, and coding + names are compared without case. One "chunked" is decoded. Any + other transfer coding, and chunked together with Content-Length, + fails the request with 502: the proxy cannot decode such a body, + and it never forwards Transfer-Encoding to the client. A response + to HEAD, and any 204 or 304, is exempt from both 502s: it ends at + the first empty line, so no Transfer-Encoding frames it. Before, + such a response gave 502 when it carried chunked together with + Content-Length. + + *) Change: ".svgz" added to the default MIME type list as + "image/svg+xml". A share "types" rule that names image/svg+xml or + image/* now matches a .svgz file; before, the file had no type and + such a rule refused it. + + *) Feature: a static ".svgz" file is served as "image/svg+xml" with + "Content-Encoding: gzip". Before, it had no Content-Type, and a + browser did not show it. The stored gzip bytes are sent + unchanged: compression does not code them again, the ETag stays + strong, and a Range applies to the stored bytes. A client without + gzip in Accept-Encoding also gets the gzip bytes, as with + "AddEncoding gzip svgz" in Apache. A "mime_types" setting that + maps ".svgz" to another type adds no coding. The ETag of a .svgz + file ends in "-gzip", so a cache that stored the old response + gets a full response, not a 304. A .svgz file never answers 304 + to If-Modified-Since, and a date in If-Range never resumes it, so + a client that revalidates by date only downloads the file again + each time. + + *) Feature: the "body_min_rate" and "send_min_rate" options in + "settings/http" set a minimum client rate in bytes per second on + HTTP/1 connections. The body_read_timeout and send_timeout timers + start again after each read or write, so a client that sends or + reads one byte before each timeout kept a connection for ever. + After a grace time equal to the related timeout, a client below + the rate gets 408 (body) or the connection closes (response). The + rate is checked for each window of at least the grace time, so + bytes sent early give no credit for a slow transfer later. The + default is 0, which disables the check. For body_min_rate, 256 is + a good value. send_min_rate counts the bytes that the kernel + accepts into the socket send buffer. The kernel sizes this buffer + to the link, so the rate is near the real download rate of the + client. A floor above the bandwidth of honest slow clients stops + their downloads; use a small value, for example 16384. + + *) Feature: the --hardening=[off|default|strict] configure option adds + compiler and linker hardening flags. Each flag is probed, and a + compiler that does not support a flag only loses that flag. The + default is "off", so a build without the option does not change. + + *) Feature: "GET /certificates" shows the SHA-256 fingerprint of the + server certificate of every bundle as "fingerprint", in the form + "openssl x509 -fingerprint -sha256" prints it. + + *) Feature: "PUT /certificates/" replaces an existing certificate + bundle in place. Main writes the bundle to a temporary file and + renames it over the old one, so a crash never leaves a partial + bundle. When the current configuration names the bundle, the + controller applies the configuration again before it answers. New + handshakes get the new certificate; accepted connections finish + with the old one. An external ACME client can renew a certificate + with one request and no restart; see docs/acme.md. A bundle over + 1 MiB is refused with 413, and a name over 255 bytes with 400, for + PUT and for DELETE. A bundle with the same certificates as the + stored one is not stored again and causes no reconfiguration. The + router opens bundles read-only. + + *) Bugfix: the router leaked the TLS contexts it had built for a new + configuration when applying that configuration failed later, for + example in a bundle of a later listener or in an application. + + *) Bugfix: with "isolation": {"namespaces": {"pid": true}}, a worker + that died before it was fully started left its record, port and + descriptor in the main process until the prototype exited. The + prototype now reports such a worker to main, which drops them. + + *) Bugfix: the main process accepted a second WHOAMI from an + application process, and a parent that is not a prototype. The + process was then linked into its parent's list twice, which + corrupted the list in main. Such a message is now refused. + + *) Bugfix: a request could reach the application with a truncated method + or header name length. The application protocol stores method and + field name lengths in a single byte. When a method exceeded 255 + bytes, the router wrapped the length. For PHP, Perl, and Ruby, header + names carry the "HTTP_" prefix, causing names of 251 to 255 bytes to + wrap. The router now refuses a method longer than 255 bytes with 501, + and an overflowing header name with 431. Both are logged at "info". + Other languages continue to accept header names up to 255 bytes. + + *) Bugfix: an upstream 1xx interim response ("100 Continue", "103 Early + Hints") was relayed to the client as the response, with the final + response relayed as its body. The proxy now drops interim responses + (101 excepted) and relays the final one; more than ten of them, or + one larger than the proxy header buffer, fails the request with 502. + + *) Change: a certificate bundle whose private key does not belong to + its first certificate is now refused at upload with 400 "Invalid + certificate.". Before, the bundle was stored, and the mismatch + broke the next reconfiguration of every listener that named it. + Certificate names that start with "." are reserved for the store's + own files, and files with such names are ignored at startup. + After an upgrade, a bundle already stored with a mismatched key is + not loaded (an alert at startup) and leaves /certificates; its + file stays in the state directory and cannot be deleted through + the API, so remove it by hand. + + *) Bugfix: an application worker that served a WebSocket upgrade never + became idle again. The router counts a session on the worker and + keeps a worker with a session out of the idle queues, but nothing + uncounted the session when it ended, so the process stayed out of + them for good: its idle timer never ran and its slot counted against + "processes": {"max"} until it died, which an application that serves + WebSockets reaches one upgrade at a time. A session is now uncounted + when it ends, whichever side ends it; the bug had appeared in 1.19.0. + + *) Bugfix: an application that answered a request and kept running was + counted idle. PHP does this whenever a script calls + fastcgi_finish_request(): the response goes out, the script runs on, + and the router returned the worker's slot to "processes": {"max"} and + let its idle timer reap a process that was still executing. Such a + worker now reports itself busy until the work ends, so "max" bounds + live processes and "/status" reports them as "detached". The same + holds for a request whose "limits": {"timeout"} deadline has passed: + the worker still running it is no longer returned to the idle pool + either, and keeps counting against "max" until it answers or exits; + the bug had appeared in 1.29.0. + + *) Change: "limits": {"timeout"} now also bounds how long a request waits + for an application process, not only how long a worker may spend on + it. A request that waited for capacity previously had no deadline at + all. The default remains no timeout. + + *) Security: the router decoded the response an application returned + in shared memory without checking it: the field count was read + twice, and the name, value and body offsets were followed without a + bounds check. An application could point the router anywhere. + libunit read the request from shared memory more than once after + checking it. A sibling process of the same application, which maps + the same segment, could change a value in between. Both sides now + read every value once and check every offset against the buffer. + + *) Security: the string form of "access_log" "format" wrote what a + variable expanded to without escaping it. "$uri" is the + percent-decoded target, so a request for "/%0d%0a..." put a real + CRLF in the log and the rest of the line read as a record of a + request that never happened. Bytes a variable expands to are now + escaped the way nginx escapes them: the quote, the backslash, every + byte below 0x20, 0x7F and every byte above it all become "\xHH". + nginx emits no short forms, so a reader that already unescapes an + nginx log reads this one. The text of the format itself is + unchanged, so a tab or a quote an operator wrote still reaches the + log as itself; an "njs" format is the exception, because it + produces the whole record and all of it is escaped. A value that + carries a quote of its own now shows it escaped: an entity-tag + logged through "$response_header_etag" reads \x22abc\x22 rather + than "abc", and a non-ASCII path reads "\xD0\xBF..." rather than + its bytes. A log parser that expected the raw bytes has to be + updated, though "\xHH" is reversible and the bytes can be recovered + from it. The object form of "format" is unaffected by this change, + but it is not a way to keep bytes either: it replaces anything that + is not valid UTF-8 with U+FFFD before it serializes. + + *) Security: when "access_log" "format" was a JSON object without an + njs expression, the router wrote each variable's value into the + record without escaping it. A request header with a quote could + close a member and add members to the record. Every value is now + escaped. A variable in a member name is no longer expanded; the + name is written as configured. + + *) Change: an object-format access log replaces each byte that begins + no valid UTF-8 sequence with U+FFFD. Request headers may carry such + bytes, and the record was then not valid JSON. The string form + keeps them, written as \xHH. + + *) Change: upgrade contrib njs to 1.0.1. + + *) Bugfix: a port that was closed and freed could leave a pending epoll + or kqueue change behind it. The change is held by a pointer into the + port, and is applied when the batch is committed -- at the top of the + next poll at the latest, which is after the port's memory pool is + released. The commit then read freed memory, and could also name a + descriptor number the kernel had already handed to somebody else. A + port now drops its pending changes before it is freed (#414). + + *) Bugfix: a process could stop hearing from a peer for good. When the + wake-up that tells a peer to read the shared queue failed because + the kernel could not allocate for it, the port treated the peer as + dead: the wake-up was dropped and, since a later message raises one + only when the queue was empty, no later message on that port raised + one either. The wake-up is now kept and retried, and the retry no + longer waits for a socket event that a socket that never filled up + will not deliver (#392). + + *) Security: a port message that was queued because the peer's socket was + not writable borrowed the descriptors it carried -- for NEW_PORT, + another port's socket and queue -- so a port closed before the queue + drained left the deferred sendmsg() naming a closed number, or an + unrelated descriptor that had since reused it. The queued copy now + owns duplicates and closes them after the send (#388). + + *) Bugfix: the router died on the first request after a configuration + that had a "compression" block was replaced by one that had none. + The compression state was held in process globals allocated from the + router configuration that parsed them, and a later configuration + neither refilled nor cleared them, so the next request read a freed + pool. The state is now held per configuration and reached through + the request, and the compressor a response is already using is kept + with that response, so a body still being compressed when the + configuration is replaced also finishes (#167). + + *) Bugfix: a static response that was subject to content negotiation now + carries "Vary: Accept-Encoding". Without it a shared cache could store + whichever representation it saw first and serve it to every client -- + gzip bytes to one that cannot decode them, or the uncompressed copy to + one that could have had the small version. The header is sent on the + identity response as well as the compressed one, since the identity + response is the one a cache must not reuse for a client that accepts + gzip. + + *) Change: removed the unused job cluster (nxt_job.c, nxt_job.h, and + nxt_event_conn_job_sendfile.c). The offload path they implemented + had no caller anywhere in the tree and its thread pool handoff was + structurally unreachable, since the connection field it read was + never assigned. + + *) Feature: byte-range requests for static files. A "share" action now + answers a satisfiable single-range "Range: bytes=..." request with + "206 Partial Content" and a matching "Content-Range", and an + unsatisfiable one with "416 Range Not Satisfiable"; "Accept-Ranges: + bytes" is now sent on a plain 200 as well. The three single-range + forms are supported -- "A-B", "A-" and the suffix form "-N" -- with + the end clamped to the last byte of the file; a multi-range request + is ignored and answered with the full 200, as RFC 9110 permits. + "If-Range" is honoured against the file's entity-tag (strong + comparison) or its "Last-Modified" date, falling back to the full + 200 on a mismatch. Preconditions (RFC 9110 Sect. 13.2.2) are still + evaluated first, so a 304 or 412 outcome wins over any Range. A + response served from a Range is not content-coded; compression + still applies to a full 200. + *) Change: unitctl builds TLS support behind a "tls" Cargo feature, on by + default. The musl build turns it off, so the static binary no longer + links a C TLS library and no longer accepts an "https://" control + socket address; it refuses one with a message naming the feature. A + Unix socket or a plain "http://" address is unaffected, and a build + with default features behaves as before. + *) Change: unitctl sends a JSON configuration file as it was written. It + parsed the file and re-serialized it before, which kept the last of two + object members with the same name and re-spelled numbers, so the server + never saw what the operator wrote. Unit now reports the duplicate + itself, with the line and column of the file. A file that is not + valid UTF-8 is no longer refused by unitctl; Unit refuses it and names + the member. The + "edit" command no longer accepts JSON5 comments in the temporary file. + *) Change: unitctl no longer reads JSON5. A ".json5" file is refused by + name, with a message saying to convert it first; "json5 -o config.json + config.json5" does that, and "jq" reformats ordinary JSON. Nothing + parsed JSON5 after the configuration upload path stopped parsing its + input, so this removes a parser that only the refusal path still + reached, and two crates with it. + + *) Feature: conditional requests for static files. A "share" action now + answers a GET or HEAD carrying "If-None-Match" or "If-Modified-Since" + with "304 Not Modified" when the file's validators still match, so a + client that already holds the file is no longer sent it again. Unit + emitted "ETag" and "Last-Modified" before, but ignored both when they + came back. Entity-tag comparison is weak and "*" matches any + representation; a present "If-None-Match" takes precedence over + "If-Modified-Since", per RFC 9110. "If-Match" and + "If-Unmodified-Since" are evaluated first and answer "412 Precondition + Failed" when they do not hold. + + *) Change: a content-coded response now carries a weak entity-tag, so + "If-Range" cannot resume a coded representation; "If-None-Match" + revalidation is unaffected. Unit derives the tag from the file's mtime + and size, which do not change with the content coding, so a gzip + response and the identity one previously advertised the same strong + validator -- 1.36.1 and earlier shipped that. RFC 9110 requires a + strong validator to change whenever the selected representation + changes, and nginx weakens the tag for the same reason. + + *) Security: a file descriptor is no longer leaked when a static request + cannot be satisfied by any acceptable representation. A share with + compression configured answered "406 Not Acceptable" without closing + the file it had opened, so a single unauthenticated request carrying + "Accept-Encoding: identity;q=0, *;q=0" cost the router a descriptor, + and repeating it exhausted the table, stopping every application + behind that router. Present since compression was added in 1.35.0: + 1.35.0, 1.35.1, 1.35.2, 1.35.3, 1.35.4, 1.35.5, 1.36.0 and 1.36.1 + are all affected. + + *) Security: a file descriptor sent to a port is no longer leaked when + the message is dispatched and nothing takes ownership of it. An + application holds the write end of the router's main port and the + message header is unauthenticated, so a compromised application could + attach a descriptor to messages the router does not expect one on -- + or to a fragment that never reaches a handler -- and consume the + router's descriptor table, stopping every application behind it. A + fragment stream that is started and never completed still retains its + descriptor; that needs a bound on pending fragments and is tracked + in freeunitorg/freeunit#343. + + *) Bugfix: unitctl gave up on a configuration it could not decode as + UTF-8. Unit stores the bytes it is given, so a configuration can + hold bytes no UTF-8 decoder accepts; "unitctl execute" and "unitctl + export" then failed with "JSON error [path=/config]" and left the + operator no way to read or back up the configuration. Both work + now: the document is shown with the undecodable bytes replaced and + the members they fall in named, and "export" stores the server's + bytes verbatim. "unitctl edit" refuses such a configuration rather + than writing the replacements back over it. + + *) Bugfix: unitctl reordered the members of a configuration. "unitctl + edit" and "unitctl export" passed the document through an unordered + map, so members came back in an arbitrary order and an edit rewrote + the whole configuration even where nothing had changed. The order + the server sent is now kept. + + *) Change: two source files that the build never used have been + removed. src/nxt_job_cache_file.c is named nowhere in auto/, so no + configuration ever compiled it, and nothing in the tree referenced + it or its symbols. src/nxt_source.h was removed because nothing + includes it: of the 106 headers in src/, it was the only one with + no #include anywhere in the tree. Nothing observable changes. + + *) Change: the fiber implementation has been removed. It had not been + compiled since 2017: the only allocation of the engine's fiber + stack sat under "#if 0", so the pointer was always NULL and the + setjmp() branch that tested it could not run. The thread, log and + sprintf sites were already disabled. Nothing observable changes. + + *) Change: the nxt_dyld wrapper around dlopen()/dlsym()/dlclose() has + been removed. It shipped in the initial import and was never called: + the language module loader in src/nxt_application.c calls dlopen() + and dlsym() itself and never used this wrapper. libnxt.a loses one + object file; there are no other callers. + + *) Bugfix: do not exit when the capget() syscall is denied. A syscall + filter that omits capget -- a systemd "SystemCallFilter=" allowlist, + a hand-written seccomp profile -- made a non-root unitd fail to start + with "failed to get process capabilities". Unit now warns and runs + without assuming any capability: user and group switching and + "rootfs" isolation are disabled for applications that do not enable + the "credential" namespace, and applications run as unitd's own uid. + Every other capget() failure remains fatal. + + *) Bugfix: the main process did not answer a START_PROCESS request it + could not carry out -- an unknown router port, a sender that is not + the router, or an allocation that failed -- so the start stayed + pending. The application then parked every later start on a + prototype that was not coming, and the requests waiting on it were + never failed. + + *) Bugfix: a message that could not be sent because the receiving process + had died was reported to the sender as sent. A sender that answers a + request and then sends a fallback lost both, so an application could + wait for an answer that nothing would ever deliver. + + *) Bugfix: eight places whose memory nothing else reclaims handed a + message to a port and did not take it back when the port refused it. + The port layer accepts a buffer only when it answers success, so on a + refusal -- which happens when memory runs out, or when the receiving + process has stopped reading a full shared queue -- the memory was + lost. Two of them lost shared memory that no cleanup reclaims, so an + application could lose the capacity to answer at all. + + *) Change: a websocket connection is now closed when a frame cannot be + handed to the application. Such a frame was previously dropped in + silence, leaving the peer's stream short a frame it was never told + about -- which for a fragmented message the client cannot detect. + + *) Bugfix: a port could stop sending after it ran out of memory. When a + write had to be retried later, the port asked its own thread to watch + the socket again, and that request needed memory of its own. If it + failed, nothing watched the socket and every later message waited in + the queue until the port closed. The port now carries the request it + needs, so it cannot fail for lack of memory. + + *) Bugfix: the port layer reported a message as sent when it had been + neither sent nor queued. This happened when a socket answered + EAGAIN and no memory was left to hold the message for a later + attempt, so it takes memory pressure to reach. The sender then + cleared the state that would have retried or reported it, so a + request could wait for an answer that nothing would ever deliver. + + *) Bugfix: an application worker that died before it finished starting + was reported to nobody, so the start it was forked for stayed pending + forever. Once the application reached its "processes" maximum it + started no further process, and because an application's "timeout" + defaults to 0 the requests waiting on it hung instead of failing. + + *) Security: application processes no longer inherit unitd's + capabilities. A unitd started as a non-root user that had been + granted capabilities -- systemd "AmbientCapabilities=", file + capabilities -- passed them through fork() to every application it + ran, because switching between two nonzero uids does not clear them + and a filtered capget() skips the switch entirely. Each forked + process now empties its permitted, effective, inheritable and + ambient sets once its credentials are final, before the worker + serves any request. The language module's own startup still runs + before the drop: the mount, pivot_root and chroot work for "rootfs" + isolation happens after it and needs the capabilities that are being + given up. Where a syscall filter denies capset() as well, an + application start is now refused rather than allowed to proceed with + capabilities still held: the request is answered 503 and the log + names the denied syscall. A process with an empty permitted, + effective and ambient set -- which is every process of a root unitd + once it has switched away from uid 0, and every process of an + unprivileged one that was granted no capabilities -- keeps behaving + as before, since there is nothing an application could have + inherited from it. Its inheritable set is emptied as well, which on + its own is not counted and grants nothing without an execve() of a + file carrying a matching capability. The + main process is unchanged, since it needs CAP_NET_BIND_SERVICE to + bind listening sockets on every reconfiguration; the router, + controller and discovery processes warn and carry on rather than + refuse, because they run no application code. Applications + configured with "user": "root" are also unchanged, since a root + process regains capabilities on the next execve() anyway. One + configuration does lose something it previously kept: an + application in a "credential" namespace began with a full + capability set inside that namespace, because the default uid_map + maps unitd's euid to the application's uid, so the setuid() is not + a transition out of uid 0 and nothing cleared it. Those + capabilities were valid only inside the namespace and Unit's own + use of them is finished before the drop, but an application that + relied on them now needs "user": "root". + + *) Security: unitctl no longer depends on rustls-pemfile or on hyper's + "http2" feature. rustls-pemfile is unmaintained; the parser is replaced + by rustls-pki-types, already in the dependency graph. PEM parsing + behavior is unchanged: both parsers were run over 1884 PEM documents and + produced zero differences. This closes RUSTSEC-2025-0134. Dropping the + "http2" feature from hyper, hyper-rustls and hyper-util removes the h2 + and fnv crates as well, closing RUSTSEC-2026-0258, an unbounded empty + DATA frame flood in h2 0.4.15. The control API speaks HTTP/1.1 only -- + unitd writes "HTTP/1.1" into every response line -- so HTTP/2 support + was never usable against the server itself. + + *) Change: unitctl no longer offers HTTP/2 in the TLS handshake it makes to + a remote control API. This follows from the "http2" feature removal + above. A TLS-terminating proxy placed in front of a remote control API + must now speak HTTP/1.1 to unitctl; one that only accepts HTTP/2 can no + longer be reached. + + *) Change: unitctl no longer reads hjson or YAML, and no longer writes + YAML. Input piped to "/config" on stdin is parsed as JSON now, not + hjson; hjson is a superset of JSON, so piping JSON still works, but + comments and unquoted keys do not. A ".hjson" or ".cjson" file is + refused with "hjson is no longer supported: convert the file to JSON + first", instead of being handed to the JSON parser, which reports a + misleading error at the first comment. A ".yaml" or ".yml" file is + refused with "YAML is no longer supported: convert the file to JSON + first, for example with "yq -o=json"". Supported input formats are + now json and pem. "unitctl status -t yaml" is gone; + --output-format now accepts json, json-pretty and text. This removes + the nu-json dependency and the 30 crates of Nushell utility code it + pulled in for a tool that speaks one JSON control API. It also + removes serde_yaml and unsafe-libyaml; the latter is a c2rust + transpile of libyaml with 218 unsafe functions and no RustSec + advisory. + tools/unitc is unaffected: it still converts configuration to and + from YAML, using yq. + + *) Change: removed src/nxt_queue.c, the unused half of a queue utility + library carried over from nginx. It defined nxt_queue_sort() and + nxt_queue_middle(), both exported but never called anywhere in the + tree; src/nxt_queue.h and the nxt_queue_* macros it provides are + unaffected and untouched. + + *) Change: nxt_mem_zone, a second allocator, is removed. It was a + zone allocator over a mapped region, distinct from nxt_mp, the + pool allocator product code actually uses; nothing in the router, + applications, or any language module called it. Its only caller + was its own unit test, run by "./build/tests"; that test is + removed too, dropping the three "mem zone test passed" lines from + the suite's output. Upstream nginx/unit had already reached the + same conclusion and deleted the file as part of an unrelated + cleanup. + + *) Change: the entity-tag and last-modified date of a static file are + now weak while the request falls inside the second the file was + last written. Both are derived from the whole-second modification + time and the size, so a rewrite to the same size during that second + changes neither, and RFC 9110 Sect. 8.8.1 reserves a strong + validator for one that cannot miss a change. "If-Match" and + "If-Range" stop matching inside that window, so a range request + conditioned on "If-Range" is answered with the full 200 rather than + a slice keyed to a validator Unit cannot vouch for; an unconditional + "Range" is unaffected. "If-Match: *" is unaffected too, since "*" + asks whether a representation exists, not whether a validator + matches. The tag keeps its format, so nothing already held in a + cache is invalidated. + + *) Change: removed the GnuTLS, CyaSSL, and PolarSSL backends + (nxt_gnutls.c, nxt_cyassl.c, nxt_polarssl.c) and their configure + plumbing ("--gnutls", "--cyassl", "--polarssl"). None of the + three ever built: they read a "c->u.ssltls" union member that + does not exist, and reference types such as "nxt_ssltls_conf_t" + and "nxt_event_conn_io_t" that predate a refactor still visible + in the current tree, so passing the flag reaches a compiler + error instead of a clean "unsupported". CyaSSL was renamed + wolfSSL and PolarSSL became mbedTLS, both in 2015, so the flags + also target products that no longer exist under these names. + The OpenSSL backend is unaffected; none of these files were + ever part of a default build. + + *) Change: removed the PHP TrueAsync code path. It was written for a + "zend_async_event_t" API expected in PHP 8.5 core, which did not ship + it, so the configure probe never succeeded and the code sat behind + "#if NXT_PHP_TRUEASYNC" in every build. It could not have compiled had + the probe succeeded: it reads "async" and "entrypoint" members that + "nxt_php_app_conf_t" does not have and calls "nxt_php_extension_init()", + which no file in the tree defines. The probe, the "entrypoint" target + key and the TrueAsync test suite are removed with it; a PHP application + builds and behaves as before. + + *) Bugfix: the java module split every outgoing WebSocket text message + larger than 8 KiB into 8 KiB frames, whatever "max_frame_size" was set + to. A 64 KiB message arrived as eight frames and a 16 MiB one as about + two thousand, each carrying its own frame header and costing its own + message to the router, while every other language module sends a + complete message as a single frame. A complete text message, from + getBasicRemote() or getAsyncRemote(), is now encoded into a buffer of + its own and sent as one frame; a message above + "nginx.unit.websocket.MAX_SEND_BUFFER_SIZE", 16 MiB by default, still + fragments. A Writer from getSendWriter() still sends each chunk it + flushes as a frame of its own, but that chunk is now one frame instead + of up to four. An OutputStream from getSendStream() still sends each + 8 KiB it fills as a frame of its own. This does not make the module + honour "max_frame_size", which remains an inbound limit only. + + *) Bugfix: the java module sent nothing through the asynchronous + WebSocket remote. Every send from getAsyncRemote() -- sendText(), + sendBinary() and sendObject(), with a SendHandler or through the + Future they return -- ended in a doWrite() that had no body, so no + frame left, the handler was never called and Future.get() blocked + for good; getBasicRemote() sends through another path and was not + affected. Such a send now goes out as one from the blocking remote + does, and its handler or future completes on the sending thread with + the outcome, which is a failure once the session is closed (#434). + Batching works as well: flushBatch() and setBatchingAllowed(false) + threw instead of sending what was batched, and a batched frame from a + buffer whose position was not 0 had a header that overstated its + length and corrupted the frames after it. + + *) Bugfix: the java module sent the wrong bytes, or nothing, for a + WebSocket message from a heap ByteBuffer that does not start at its + array's first byte. The payload was read from the backing array at + position(), ignoring arrayOffset(), so a buffer from + ByteBuffer.wrap(a, off, n).slice() sent bytes from the start of the + array instead of its own; and a read-only heap buffer threw + ReadOnlyBufferException instead of being sent. Both the blocking and + the asynchronous remote were affected (#486). + + *) Bugfix: the Python ASGI module refused a WebSocket message larger + than 1 MB whatever "max_frame_size" was set to. The module carried a + private 1 MB limit that no configuration ever wrote and charged the + fragments of a message against it, while the configured value is + checked by the router per frame: a 4 MB message was closed with 1009 + where "max_frame_size" was 32 MB, and at the default size a client + that split a larger message into small frames was refused as well. + Each frame is now limited by "max_frame_size" alone. The module + only limits a connection to 10 MB of payload that the application + has not received yet (fragments of a message in progress and + complete messages waiting for receive()); a single frame larger + than that is still accepted when nothing else is waiting. A + fragmented message also left a count behind that lowered the limit + applied to the next message. + + *) Bugfix: a byte-range request from a client that refused the identity + coding is no longer answered with identity bytes. A range is served + as identity, so "Range" together with "Accept-Encoding: + identity;q=0" returned a "206 Partial Content" carrying exactly the + bytes the client said it would not take. The range is now ignored + and the full "200 OK" is sent in a coding the client does accept. + Ignoring the range is all or nothing, so an unsatisfiable range from + such a client is no longer answered with "416" either. A request + that accepts no coding at all is still "406 Not Acceptable", and a + range from a client that does accept identity is still a 206. + Several "Accept-Encoding" lines in one request are now read as the + single comma-joined value RFC 9110 defines them to be; previously + only the first was consulted. + + *) Bugfix: a response that no compressor applies to is no longer sent + as identity to a client that refused the identity coding. With a + compressor configured, "Accept-Encoding: gzip, identity;q=0" was + accepted because gzip was acceptable. When the media type of the + response was outside "types", or the response had no media type, + the response was then sent uncompressed: the identity bytes that + the request had refused. Such a request now gets "406 Not + Acceptable", with a "Range" header and without one. A 406 from + this negotiation carries "Vary: Accept-Encoding". A 1xx, 204 or + 304 response is never negotiated, so an application that sends one + of these with a "Content-Length" is not affected. A response with + no body, and a response that carries its own "Content-Encoding", + are not affected either. + + *) Bugfix: "*" in "Accept-Encoding" now stands for each enabled + compressor that the client did not name. Before, it stood for the + identity coding only. "identity;q=0, *;q=1" was answered "406 Not + Acceptable" with gzip enabled and applicable, because the wildcard + could select only identity, and the same field refused identity. A + coding that the field names keeps its own weight. The wildcard does + not match it, and the named coding wins a tie: "gzip, *" now sends + gzip. Before, it sent the response uncompressed. + + *) Bugfix: a compressor below its own "min_length" is no longer selected + before a compressor with a lower weight that can be applied. + "min_length" belongs to each compressor. With gzip at 1000 and + deflate at 0, a 100-byte response was sent uncompressed, because + gzip had the higher weight. A client that refused the identity + coding got the same uncompressed bytes. Such a client now gets + deflate. When each compressor that the client accepts is below its + "min_length", the client gets "406 Not Acceptable" instead of the + bytes it refused. + + *) Change: a server with no compression configured now answers "406 Not + Acceptable" to a request whose "Accept-Encoding" refuses the identity + coding: "identity;q=0", or "*;q=0" with no more specific entry for + identity. Identity is the only coding that such a server has. + Before, it sent the response, or the requested range of it, in the + bytes that the client had refused. To read that refusal, each + response with a known length above zero now looks at its own + "Content-Encoding" and at the request "Accept-Encoding". Before, + these checks ran only with a compressor configured. + + *) Bugfix: a malformed weight in "Accept-Encoding" is no longer read as a + refusal. The qvalue was converted with strtod(), which takes no digits + at all from "identity;q=" and from "identity;q=abc" and reports zero, + so either one answered "406 Not Acceptable" to a client that had + refused nothing; "q=0x0" and "q=0e0" did the same. An element whose + weight does not match the grammar of RFC 9110 Sect. 12.4.2 is now + ignored, as an element naming an unknown coding already was. This + also removes "q=nan", which passed the 0-to-1 range check -- a NaN + compares false against both bounds -- and then outranked every real + weight. + + *) Bugfix: a negotiation failure on "Accept-Encoding" was answered with + "503 Service Unavailable" instead of "406 Not Acceptable" when the + response came from an application. The router treated every result of + the acceptability check other than success as a server error; only the + static path reported the 406. + + *) Bugfix: a "Content-Encoding" set with "response_headers" no longer + makes the body coded twice. With a compressor configured, the + response was compressed first, and "response_headers" then replaced + the "Content-Encoding" of the compressor. So a share of "$uri.gz" + with "Content-Encoding: gzip" sent gzip of the stored gzip file to a + client that accepts gzip, with a weak ETag. Now such a response is + not compressed, as with a "Content-Encoding" from an application. + This applies to static files and to application responses. A + "Content-Encoding" removed with null in "response_headers" also keeps + the compressor out, so bytes that Unit compressed are no longer sent + without a "Content-Encoding". Such a response is identity, so a + client that refused identity gets "406 Not Acceptable". For a + response that has its own "Content-Encoding", the null still removes + only the field, as before. + + *) Change: unitd, libunit.a and the language modules no longer contain + the test hooks compiled under "#if (NXT_TESTS)". "./configure + --tests" used to define the macro for the whole build, and the deb + and rpm packages pass "--tests", so every released package from 0.2 + to 1.36.1 carried them: the nxt_random_test() entry point in all of + them, and in 1.36.0 and 1.36.1 the fault-injection branches and + counters in nxt_mp.c, nxt_port_rpc.c and nxt_port_socket.c, two of + which were exported symbols. The hooks now go only into the test + programs. A build without "--tests" is unchanged. + + *) Change: update wasmtime to 48.0.5. The WebAssembly module moves from + 43.0.1, and the WebAssembly component module from 47.0.2. Version 48 + is a long-term support release of wasmtime. It fixes twelve wasmtime + security advisories published on 24 September and 2 October 2026. + Each one needs a hostile WebAssembly guest. With the features that + FreeUnit builds, such a guest could panic the host, exhaust host + memory, read uninitialized host memory, or corrupt the GC heap. + Building the two modules now needs Rust 1.95.0 or newer. + + *) Bugfix: the PHP module checked that "script" is under "root" by + comparing the first bytes of the two real paths. With "root": + "/srv/app", a script in "/srv/app2" passed the check and was served. + The script must now be in "root" or in a directory below it. A + "script" that resolves to "root" itself is also rejected; it was + served as an empty 200 response. The bug had appeared in 0.4. + + *) Change: the Java module now bundles Apache Tomcat 9.0.122 (from + 9.0.121). The upstream release fixes CVE-2026-87022 (WebSocket + message smuggling with per-message-deflate) and eleven other CVEs. + None of the twelve fixes is in a jar that FreeUnit bundles. The + FreeUnit WebSocket code is derived from Tomcat, but it does not + decompress received frames, so CVE-2026-87022 does not affect it. + + *) Feature: Docker images for Go 1.27, Perl 5.44 and Java 27. The Java + images are now based on Ubuntu 26.04 (resolute) instead of 24.04 + (noble), because Eclipse Temurin publishes Java 27 only for 26.04. + + *) Change: the Docker images now install the pending updates of their + base image when they are built. Before, an image kept the packages + of its base image as they were when that image was published. + +Changes with FreeUnit 1.36.1 28 Aug 2026 + + *) Feature: configuration validation errors now say where they are. A + rejected configuration request carries an RFC 6901 JSON Pointer in + the "path" member of "location": it points at the member whose value + failed validation, or at the containing object when the failure is an + unknown parameter, whose name the message itself already carries; "" + is the document root. On an unknown parameter whose name is a close + and unambiguous typo of a known one, the intended name is reported in + a top-level "suggestion" member. The addition is strictly additive: + the existing response fields and the error message wording are + unchanged, and each new member is omitted where it does not apply. + + *) Feature: the /status API reports OpenTelemetry span export health as + "telemetry": {"spans": {"exported": N, "failed": M}} -- the spans the + collector accepted, and the spans whose export failed. Both are + reported because a failure count alone cannot distinguish a healthy + pipeline from one that has never exported anything. The object is + omitted entirely unless Unit was built with OpenTelemetry support, + "/config/settings/telemetry" is set, and the exporter was built + successfully, so its absence is how a client tells telemetry is off + rather than idle. The counters are cumulative since the exporter was + last built, so changing any telemetry setting restarts them, while + re-applying an identical configuration does not. + + *) Bugfix: validate the shared memory segment identifier and geometry an + application process supplies in its NXT_PORT_MSG_MMAP message. The + router checked only the pid fields, then used the peer-authored + header id directly as an array index and mapped the descriptor at + whatever length fstat() reported; a compromised or faulty application + could drive an out-of-range write and an oversized mapping. + + *) Bugfix: bounds-check the router-supplied chunk identifier and size in + libunit before consuming them, so an out-of-range size can no longer + make every serialized-pointer check in the arrival-time validation + block pass and be read out of bounds in the application process. + + *) Bugfix: do not replay a stale SCM_RIGHTS control message in libunit + when the embedder's port_recv callback returns without receiving + anything. The out parameter carrying the control-data length was + stored unconditionally, so a descriptor from an earlier message could + be adopted a second time. + + *) Bugfix: check the shared memory buffer allocation when forwarding a + websocket frame. Exhausting the outgoing shared memory crashed the + router with a NULL dereference; every other caller already checked. + + *) Bugfix: keep the chunked request body buffer alive across framing + reads. A buffer drained without producing a data slice was handed to + its completion handler while the parser still referenced it, a + use-after-free reachable from an ordinary chunked request. + + *) Bugfix: parse the HTTP/1.1 trailer section (RFC 9112 7.1.2) instead + of erroring on the first byte after the terminal chunk. In the proxy + response relay this aborted an already-streaming response and + truncated the body at a racy point; chunked requests carrying a + trailer were rejected the same way. Trailer field names and values + are validated as they are parsed. + + *) Bugfix: reject an application response that disagrees with itself + about its own body length. A duplicate Content-Length was neither + detected nor parsed, so every such header was forwarded verbatim and + the body was framed by the advertised length instead of chunked. + + *) Bugfix: release the response buffers adopted from the port on the + error paths of the router's response handler, and drain them from the + request output chain when the failure happens after they were linked + into it. Both leaked the chain the handler had taken ownership of. + + *) Bugfix: keep the memory pool cursor aligned in nxt_mp_get() by + rounding the requested size up to the maximum alignment rather than + flooring it. The small allocation path is a bump allocator with no + per-allocation alignment step, so any request whose size was not a + multiple of that alignment left the page cursor misaligned, and + allocations from that page kept violating the documented alignment + guarantee until a later size happened to bring the cursor back into + step, which crashed with SIGBUS on strict alignment targets such as + 32-bit ARM. + + *) Bugfix: serialize the process reference count on the runtime process + mutex and take a reference during cross-thread process lookups. The + count was mutated non-atomically from every engine thread and a + lookup returned an unreferenced process, so a router worker could + operate on an nxt_process_t -- including its shared memory segments + and its mutex -- that the main thread had already torn down. + + *) Bugfix: keep the socket and timer tasks of a connection scoped to + that connection, so a work item cannot run against a task belonging + to a different connection. + + *) Change: remove the dead chunk-tracking bitmap from the shared memory + segment header. Its trailing fields ran past the header area and + aliased the payload of the segment's first chunk, which was harmless + only because nothing read or wrote them; the field offsets a peer of + either vintage uses are unchanged, so old and new builds still + interoperate. A static assertion now pins the header struct inside + the area reserved for it. + + *) Change: the shared memory chunk range check is now a single shared + implementation used by both the router and libunit, with the chunk + count computed so that a peer-supplied size in the top of the 32-bit + range cannot wrap to zero chunks and be accepted. + + *) Change: recycle connection structures through a per-engine freelist, + embed the static file context in the request, recycle static file + buffer descriptors through a thread-local freelist, and resolve the + first eight routing variables and the first sixteen request and + response header fields from arrays embedded in the request. Each + avoids or reuses an allocation on a per-connection or per-request hot + path. + + *) Change: the WebAssembly component module no longer supports + guest-initiated outbound HTTP. Serving Unit never needed it, and the + feature that provided it linked an entire TLS stack -- rustls, + tokio-rustls, webpki-roots and rustls-webpki -- into the module. A + guest that only imports wasi:http/outgoing-handler, as the + wasi:http/proxy world requires, still instantiates and runs; an + actual outbound call now fails with HttpRequestDenied. + + *) Change: update wasmtime from 36.0.12 to 47.0.2 in the WebAssembly + component module, keeping outbound HTTP denied and the TLS stack out + of the dependency graph. *) Change: the "if" option of a route "match" object, added in 1.33.0, is now described in the OpenAPI specification and covered by the test @@ -11,6 +1183,152 @@ Changes with FreeUnit 1.36.1 02 Aug 2026 njs template strings express comparisons and regular expressions ("`${uri == '/admin'}`", "`${/^\/admin/.test(uri)}`"). + *) Change: build against OpenSSL 4.0, which constified the return types + of the X509 name accessors and deprecated + X509_NAME_get_text_by_NID(); SNI certificate matching and the + "certificates" control API now read the certificate name through + X509_NAME_get_index_by_NID() and the entry accessors instead, and a + new CI job builds and runs the C test suite against OpenSSL 4.0 with + deprecated APIs disabled, so it fails on any new use of a deprecated + function rather than only on a removal. + + *) Change: OpenSSL 1.1.1 or later is now required to build with + "--openssl"; configure fails with an explicit message on older + releases. Builds against OpenSSL 1.0.x have not worked since 1.35.2, + when the TLS module started to use OpenSSL_version_num(); the + unreachable pre-1.1.0 compatibility code is removed. + + *) Bugfix: close the engine port when a router worker thread exits. + Lowering settings/listen_threads deletes engines, and the thread exit + handler only dropped a port reference, and the runtime held another, + so nothing ever called nxt_port_close(): every deleted engine leaked + its socket pair and its queue descriptor, three per engine, for the + life of the router. The count only ever rose, so a configuration that + raises and lowers the thread count walked the router towards its + descriptor limit. + + *) Bugfix: close the descriptors a forged or malformed port message + leaves behind. A message carries up to two descriptors whatever its + type normally uses, and the dispatcher reclaims nothing once a + handler has returned. The main process kept the second descriptor on + the whoami path, the router lost both when a conf data message named + no reply port, and a PROCESS_READY naming an unknown process closed + neither. A peer that is already compromised could exhaust the + descriptor table of the receiving process -- the main process on the + whoami path -- by repeating any of them. + + *) Bugfix: bound the log file slot a CHANGE_FILE message claims, and + stop leaking the descriptor it carries. The slot index arrived from a + peer and was used to walk the runtime's log file list without a range + check, and a message whose payload was shorter than the index was + read past its end. The rejection paths now close what the message + attached, and nxt_file_redirect() releases the descriptor when dup2() + fails rather than only on success -- which also ends a leak of one + descriptor per unrotatable log file on every SIGUSR1. + + *) Bugfix: a NEW_PORT or PROCESS_READY message carrying a shared-memory + queue shorter than the queue type faulted the receiving process + (main, the prototype, or the router) with SIGBUS on first access; the + object size is now checked before mapping, a short queue is refused + with a warning, and the router no longer leaks the descriptor of a + queue it refuses. + + *) Bugfix: a NEW_PORT message announcing a port that already existed, or + one that could not be created, left an uninitialized stack value + behind; the controller dereferenced it unconditionally on every + NEW_PORT, and main and the router relied on a NULL check that could + not catch non-NULL garbage. The result is now cleared explicitly + whenever no port is created. + + *) Bugfix: close the file descriptors a NEW_PORT or MMAP message left + behind instead of leaking them. The discovery and application + prototype processes kept every port queue descriptor open for the + life of the process, the main process kept the queue descriptor of + any port it had not just created, the router lost it on ports it does + not map a queue for, the controller never closed it, and the mmap + handler leaked its second descriptor on every path and returned + closing nothing at all when the first was missing; a message type + with no handler also went uncollected. A NEW_PORT for a port that + already has a queue now refuses the new descriptor instead of mapping + it over the live queue and leaking the mapping it replaced, which had + let a repeated announcement re-point an established port at memory of + the sender's choosing. + + *) Bugfix: session tickets were silently disabled when Unit was built + against an OpenSSL 3.x library compiled with the deprecated 3.0 API + removed (OPENSSL_NO_DEPRECATED_3_0), because the feature guard tested + for a control macro that is itself declared only under that API and + so was never defined; the session ticket key callback now registers + through SSL_CTX_set_tlsext_ticket_key_evp_cb() and keys an EVP_MAC + context on OpenSSL 3.0 and later, independent of the deprecated + HMAC_CTX API. + + *) Bugfix: stop building OpenTelemetry span attributes for spans the + sampler has already dropped. set_attribute() on a non-recording span + discards its argument, but the argument was built first -- the + sprintf round-trips, the pool allocations and two owned strings per + attribute -- and nothing asked whether the span was recording. + Lowering sampling_ratio therefore bought back the exporter and + nothing on the request path. The sampler's verdict is now read once + when the span is created and the attribute stages consult it, which + removes most of telemetry's request-path cost at a low sampling + ratio. Context propagation is deliberately outside that check: the + traceparent must reach the peer and the application whatever the + sampling decision was. + + *) Bugfix: do not attach the traceparent to an OpenTelemetry span as a + "Unit Attribute" event. It duplicated the trace and span identifiers + the span context already carries, nothing consumed it, and it was + added twice on a request that took the error path with a trace it had + started itself, because the error path re-runs header propagation. + + *) Bugfix: OpenTelemetry spans were lost at the edges of a request's + life. A request that ended before its span was collected -- an error + path, a dropped connection -- left the span allocated and never + exported it; the span is now tied to the memory pool that holds it + and is exported with an error status when that pool is released, so + an abandoned request appears in the trace instead of vanishing. + + *) Bugfix: flush the OpenTelemetry span pipeline before the router + exits. Exit and reload discarded the pipeline with up to 4096 spans + still queued, losing every span produced since the last export. The + pipeline is now flushed with a 2 second bound, so a slow or + unreachable collector delays shutdown by at most that long rather + than either hanging it or costing the queued spans. + + *) Bugfix: rebuild the OpenTelemetry exporter only when the telemetry + settings have changed. Every configuration change rebuilt it, so + adding a route or changing an application stalled reconfiguration + against an unreachable collector even though it did not touch + "/config/settings/telemetry". The exporter is now rebuilt only when + the endpoint, protocol, sampling ratio or batch size actually differs + from the running one. + + *) Change: the OpenTelemetry span attributes "http.response.status_code" + and "http.request.body.size" are now emitted as OTLP integer values, + as the HTTP semantic conventions specify, instead of strings. A + collector or query that matches these attributes as strings must be + adapted; one that reads them as integers, or that relied on the + convention rather than on Unit's previous output, needs no change. + + *) Change: the work a traced request does on the request path is + reduced: attribute keys are no longer copied onto the heap per + request, the two attributes above no longer round-trip through + strings, and a span stage now crosses into the tracer once instead of + once per attribute. Together with the sampling fix above, the + telemetry overhead of a request measured 27% lower at + "sampling_ratio" 1.0 and 53% lower at 0.1. + + *) Change: the Java module now bundles Apache Tomcat 9.0.121 (from + 9.0.110) and ClassGraph 4.8.186 (from 4.8.181). The upstream Tomcat + 9.0.121 release fixes CVE-2026-66299 (unbounded buffer for + undelivered WebSocket messages, a memory exhaustion by a slow + client), CVE-2026-73180, CVE-2026-68569, CVE-2026-66422 and + CVE-2026-65927. FreeUnit bundles only the servlet, JSP and EL API + jars, Jasper and the Tomcat utility jars -- not catalina, the + WebSocket implementation or the example webapps -- so the bump keeps + the jars it ships on a supported release. + Changes with FreeUnit 1.36.0 16 Jul 2026 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 61383d9b1..20a82bf7a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -65,6 +65,35 @@ Run the test suite before submitting: $ sudo pytest-3 --print-log test/ ``` +If you changed core C code or libunit, also run the C test suite: + +```console +$ ./configure --tests --openssl +$ make +$ make build/lib/libunit.a +$ make tests +$ ./build/tests +``` + +`--tests` builds the test programs (`build/tests`, +`build/unit_port_recv_test`, and the rest). It does not change `unitd`, +`libnxt.a`, `libunit.a`, or any language module: those come from the +same objects with or without `--tests`. + +If you touch a source file under `#if (NXT_TESTS)`, run the gate script +after `make tests`. It fails if a test hook reaches a shipped artifact: + +```console +$ .github/scripts/check-test-hooks.sh +``` + +Build once with `./configure --hardening=strict` before you submit. +New code must compile warning-free under it: no variable-length +arrays, and `nxt_fallthrough;` instead of a fall-through comment. For +length arithmetic on untrusted input, use `nxt_size_add()` and +`nxt_size_mul()` from `src/nxt_checked.h`. For record parsing, use +`nxt_span_t` from `src/nxt_span.h`. + ## Commit Messages Use conventional commits format: diff --git a/EOL.md b/EOL.md index d7f8d63f7..01c29372c 100644 --- a/EOL.md +++ b/EOL.md @@ -11,19 +11,22 @@ EOL dates are tracked at [endoflife.date](https://endoflife.date). | Runtime | Version | Upstream EOL | FreeUnit Drops | |---------|---------|--------------|----------------| | Go | 1.24 (EOL) † | Feb 2026 | Feb 2027 | -| Go | 1.25 | Aug 2026 | Aug 2027 | +| Go | 1.25 (EOL) † | Aug 2026 | Aug 2027 | | Go | 1.26 | Feb 2027 | Feb 2028 | +| Go | 1.27 | Aug 2027 | Aug 2028 | | Java (JSC) | 17 (LTS) | Oct 2027 | Oct 2028 | | Java (JSC) | 21 (LTS) | Dec 2029 | Dec 2030 | | Java (JSC) | 25 (LTS) | Sep 2031 | Sep 2032 | -| Java (JSC) | 26 | Sep 2026 | Sep 2027 | +| Java (JSC) | 26 (EOL) † | Sep 2026 | Sep 2027 | +| Java (JSC) | 27 | Mar 2027 | Mar 2028 | | Node.js | 20 (LTS) (EOL) † | Apr 2026 | Apr 2027 | | Node.js | 22 (LTS) | Apr 2027 | Apr 2028 | | Node.js | 24 (LTS) | Apr 2028 | Apr 2029 | | Node.js | 26 (LTS) | Apr 2029 | Apr 2030 | -| Perl | 5.38 | Jul 2026 | Jul 2027 | +| Perl | 5.38 (EOL) † | Jul 2026 | Jul 2027 | | Perl | 5.40 | Jun 2027 | Jun 2028 | | Perl | 5.42 | Jul 2028 | Jul 2029 | +| Perl | 5.44 | Jul 2029 | Jul 2030 | | PHP | 8.3 | Dec 2027 | Dec 2028 | | PHP | 8.4 | Dec 2028 | Dec 2029 | | PHP | 8.5 | Dec 2029 | Dec 2030 | @@ -45,14 +48,14 @@ EOL dates are tracked at [endoflife.date](https://endoflife.date). | Fedora | 43 | 2026-12 | 2029-12 | 3.14 | | Fedora | 44 | 2027-06 | 2030-06 | 3.14 | | CentOS Stream | 9 | 2027-05 | 2030-05 | 3.11 | -| CentOS Stream | 10 | 2030-01 | 2033-01 | 3.13 | -| Amazon Linux | 2 | 2026-06 | 2029-06 | 3.7 ‡ | +| CentOS Stream | 10 | 2030-05 | 2033-05 | 3.13 | +| Amazon Linux | 2 (EOL) † | 2026-06 | 2029-06 | 3.7 ‡ | | Amazon Linux | 2023 | 2029-06 | 2032-06 | 3.11 | -| Ubuntu (LTS) | 22.04 | 2027-04 | 2030-04 | 3.10 | +| Ubuntu (LTS) | 22.04 | 2027-06 | 2030-06 | 3.10 | | Ubuntu (LTS) | 24.04 | 2029-05 | 2032-05 | 3.12 | -| Ubuntu (LTS) | 26.04 | 2031-04 | 2034-04 | 3.13 | +| Ubuntu (LTS) | 26.04 | 2031-05 | 2034-05 | 3.13 | | Debian | 11 (bullseye) (EOL) † | 2024-08 | 2027-08 | 3.9 | -| Debian | 12 (bookworm) | 2026-07 | 2029-07 | 3.11 | +| Debian | 12 (bookworm) (EOL) † | 2026-07 | 2029-07 | 3.11 | | Debian | 13 (trixie) | 2028-08 | 2031-08 | 3.13 | | RHEL | 8 | 2029-05 | 2032-05 | 3.8 ‡ | | RHEL | 9 | 2032-05 | 2035-05 | 3.11 | @@ -66,6 +69,35 @@ EOL dates are tracked at [endoflife.date](https://endoflife.date). † Past upstream EOL; in FreeUnit grace period. ‡ Default Python shipped by this OS is itself past upstream EOL. FreeUnit does not backport fixes to that Python version. +The "Upstream EOL" column is always the end of *standard* support, per the LTS +rule below — never an extended-maintenance date. Which endoflife.date field that +is depends on the vendor, so `pkg/eol/` picks per category: Ubuntu's `eol` is the +end of standard security maintenance (ESM excluded), while Debian's `eol` became +the end of **LTS** in September 2026, and the Debian Security Team's window — the +one this table tracks — moved to that API's `support` field. + +## Dependency Support + +Build-time and bundled libraries. These are **not** matrix variants — nothing in +Docker or CI derives an image from this table — so the 1-year grace rule does not +apply to them. Machine-readable copy: the `dependencies` key in `pkg/eol.json`. + +† Bundled software that is already past upstream EOL. It carries no FreeUnit "drops" +date, because the grace policy applies to matrix variants, not to vendored jars; it +needs an upgrade or replacement decision instead. + +| Dependency | Role | Floor / Version | Upstream EOL | Notes | +|------------|------|-----------------|--------------|-------| +| OpenSSL | TLS backend (`--openssl`) | 1.1.1 (floor) | Sep 2023 | The floor stays at 1.1.1 rather than 3.x because RHEL 8 still ships a vendor-patched 1.1.1 and Red Hat maintains it into 2029 -- from 2026-09 it is the last platform in the matrix still receiving vendor patches for it. Amazon Linux 2's ended 2026-06-30 and Debian 11's LTS ends 2026-08-31; both remain inside FreeUnit's own three-year post-EOL support window (see `_grace_os` in `pkg/eol.json`). Floor declared by the `auto/ssltls` probe (PR #224). | +| OpenSSL | tested ceiling | 3.6.5 (CI build) | Nov 2026 | `build-test.yml` builds it into `/opt/openssl-3.6` (`OPENSSL_VERSION`). **The pin needs a bump before 2026-11-01** (3.5 is the LTS, EOL Apr 2030). | +| OpenSSL | tested ceiling | 4.0.3 (CI build) | May 2027 | The `openssl4` job builds it into `/opt/openssl-4.0` (`OPENSSL4_VERSION`) and compiles with `-DOPENSSL_NO_DEPRECATED`, so any use of an API 4.0 deprecates fails the build. | +| Apache Tomcat | Servlet/JSP/EL API + Jasper jars bundled by the Java module | 9.0.x | Mar 2027 ([no earlier than](https://tomcat.apache.org/whichversion.html)) | Bundled by `auto/modules/java`; independent of the JDK variant. | +| Eclipse Jetty | `jetty-util` / `jetty-server` / `jetty-http` jars bundled by the Java module | 9.4.58.v20250814 | **Aug 2025 (EOL)** † | Jetty 9.4 lost community support on 2025-08-14 and the bundled build is its last release. Needs an upgrade-or-replace decision — Jetty 10/11 are EOL too; 12.x is the supported line. | +| Eclipse ECJ (JDT batch compiler) | JSP compilation jar bundled by the Java module | 3.26.0 | none published | Tracks Eclipse releases; endoflife.date has no product for it. Pinned build is from Jun 2021; current is 3.42.0 (Jun 2025). | +| ClassGraph | classpath scanning jar bundled by the Java module | latest | — | No upstream EOL schedule; pinned and bumped as needed. | +| Wasmtime (C API) | WebAssembly runtime built by `pkg/contrib` and linked by the `wasm` module | 48.0.5 | none published | Fetched as a source tarball (`pkg/contrib/src/wasmtime/version`), so `cargo audit` never sees it; the `Audit (cargo)` pins job queries OSV for the pinned version instead. Same 48.0.5 as the Rust pin below; 48 is a long-term support line. The 47.0.4 to 48.0.5 bump closed twelve advisories from 2026-09-24 and 2026-10-02. The 43.0.1 bump closed RUSTSEC-2026-0269, -0222, -0114 and three wasmtime-wasi path/permission advisories — see #400. | +| Wasmtime (Rust crate) | WebAssembly runtime used by the `wasm-wasi-component` module | 48.0.5 | none published | `src/wasm-wasi-component/Cargo.lock`, audited by the `Audit (cargo)` workflow on every lock. Bumped with the advisories it closes. | + ## Rules - **Adding a runtime version:** when new upstream release reaches stable, FreeUnit adds @@ -77,6 +109,13 @@ EOL dates are tracked at [endoflife.date](https://endoflife.date). - **Security-only mode:** versions within 6 months of the FreeUnit drop date receive security fixes only — no new features backported. - **LTS runtimes (Java 17/21/25):** follow the upstream LTS schedule strictly. +- **Dependencies:** the floors in the Dependency Support section are not part of the + EOL + grace policy; they track what FreeUnit builds and bundles, not what it ships + as a variant. +- **A dependency pinned twice needs both rows.** Wasmtime is pinned once as a Rust + crate and once as a source tarball, and only the crate is reachable by `cargo + audit`. List every pin separately and say which one an automated check covers, so + a gap between them is visible here rather than only in a lock file. - **LTS OS (Ubuntu, RHEL, Debian):** 3-year extension applies to standard EOL, not extended security maintenance dates. diff --git a/README.md b/README.md index 9f3111518..05c6e6fee 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,7 @@ and are available for `linux/amd64` and `linux/arm64`. | Node.js 24 | `ghcr.io/freeunitorg/freeunit:latest-node-24` | | Node.js 22 | `ghcr.io/freeunitorg/freeunit:latest-node-22` | | Node.js 20 | `ghcr.io/freeunitorg/freeunit:latest-node-20` | +| Go 1.27 | `ghcr.io/freeunitorg/freeunit:latest-go-1.27` | | Go 1.26 | `ghcr.io/freeunitorg/freeunit:latest-go-1.26` | | Go 1.25 | `ghcr.io/freeunitorg/freeunit:latest-go-1.25` | | Ruby 4.0 | `ghcr.io/freeunitorg/freeunit:latest-ruby-4.0` | diff --git a/RELEASE-PROCESS.md b/RELEASE-PROCESS.md index 77a07c67d..3aad4ee7d 100644 --- a/RELEASE-PROCESS.md +++ b/RELEASE-PROCESS.md @@ -44,11 +44,18 @@ Create a commit that updates the docs/changes.xml for this release. As well as adding the various entries also update the 'date' and 'time' fields. -## Generate the CHANGES file +## Update the CHANGES file - $ make -C docs/ changes && mv build/CHANGES . +CHANGES is maintained by hand: every changelog commit adds its entry to +both docs/changes.xml and CHANGES. For the release, retitle the top block +and set its date, keeping the leading blank line and the column the date +sits in: -See 24ed91f40 for an example. + Changes with FreeUnit X.Y.Z DD Mon YYYY + +Do not regenerate it with `make -C docs/ changes`: the generator titles the +block "Changes with Unit" and wraps every entry differently from the file +in the tree, so the result is a rewrite rather than an update. # Merge it @@ -70,6 +77,13 @@ The tag can be pushed just as the branch is. E.g. $ git push 1.33.0 +Do not create the `unitctl/X.Y.Z` tag by hand. Pushing `X.Y.Z` runs +.github/workflows/unitctl.yml, whose release job creates `unitctl/X.Y.Z` +and its release itself; a tag that already exists makes that step fail. +The same push runs build-deb.yml and release-docker.yml; the latter logs +in to Docker Hub, so the DOCKERHUB_USERNAME and DOCKERHUB_TOKEN secrets +have to be set before the tag is pushed. + # A new 'Release' @@ -107,12 +121,12 @@ release number: # Copyright (C) FreeUnit Community - NXT_VERSION=1.36.0 - NXT_VERNUM=13600 + NXT_VERSION=1.36.1 + NXT_VERNUM=13601 - **`NXT_VERSION`** — human-readable dotted string (`MAJOR.MINOR.PATCH`). - **`NXT_VERNUM`** — the same number for compile-time comparisons: - `MAJOR * 10000 + MINOR * 100 + PATCH` (e.g. `1.36.0` → `13600`). Keep both in + `MAJOR * 10000 + MINOR * 100 + PATCH` (e.g. `1.36.1` → `13601`). Keep both in lockstep; a mismatch is a silent bug. ## Consumers diff --git a/TODO.md b/TODO.md index e6e7d6ceb..4efc41686 100644 --- a/TODO.md +++ b/TODO.md @@ -47,8 +47,6 @@ **Open questions (decide before August):** -- [ ] PHP TrueAsync — determine source of `nxt_php_extension.c` (fork EdmondDantes or write from scratch) -- [ ] PHP 8.5 `rootfs` SIGSEGV — needs diagnostics on real hardware - [ ] OpenSSL 3.6 migration — verify clang-ast compatibility - [ ] Proxy request buffering (#58) — define scope (per-action vs global) @@ -319,130 +317,6 @@ In-repo audit (2026-05-31): --- -## PHP TrueAsync mode (branch: php-graceful-shutdown) - -Items below must be resolved before the branch can be merged and before -`test/test_php_trueasync.py` can run in CI. - -### 1. `nxt_php_app_conf_t` is missing `async` and `entrypoint` fields - -**Problem.** `src/nxt_php_sapi.c` (commit 58ffa236) references `c->async` -and `c->entrypoint` where `c` is a pointer to `nxt_php_app_conf_t` -(defined in `src/nxt_application.h`). The struct currently has only -`targets` and `options`, so the code **will not compile** as-is. - -The original upstream commit (EdmondDantes/052afd8) likely also modified -`nxt_application.h` and the JSON config validator; those changes were not -included when the commit was cherry-picked into this branch. - -**Risk.** Build failure on the first C compile step; CI is entirely broken. - -**Fix.** -- Add the missing fields to `nxt_php_app_conf_t` in `src/nxt_application.h`: - ```c - typedef struct { - nxt_conf_value_t *targets; - nxt_conf_value_t *options; - nxt_bool_t async; /* true when TrueAsync mode requested */ - nxt_str_t entrypoint; /* path to the PHP entrypoint script */ - } nxt_php_app_conf_t; - ``` -- Wire them up in the config parser (find where other PHP config fields are - read from JSON — probably `nxt_router.c` or a PHP-specific conf handler). -- Add validation rules in `nxt_conf_validation.c` so the REST API rejects - `entrypoint` without `async: true` and vice-versa. - ---- - -### 2. `nxt_php_extension.c` / `nxt_php_extension_init()` are missing - -**Problem.** `PHP_MINIT_FUNCTION(nxt_php_ext)` calls -`nxt_php_extension_init()` (line 221 of `nxt_php_sapi.c`) and the async -request handler references `nxt_php_request_callback` (declared `extern` -at line 2187). Both symbols are expected to come from `nxt_php_extension.c`, -which does not exist in the repository. - -Without it the linker will fail and there is no PHP-land API for user -scripts to register a request handler. The `\Unit\Server::setHandler()` -calls in `test/php/async_*/entrypoint.php` are placeholders until the -real extension is in place. - -**Risk.** Linker failure; all TrueAsync tests will skip until resolved. - -**Fix.** -- Locate the file in the EdmondDantes fork or write it from scratch. - Minimum required: - - `zval *nxt_php_request_callback` global (zeroed at startup). - - `nxt_php_extension_init()` — registers `\Unit\Server` with a static - `setHandler(callable $cb)` that stores `$cb` into the global above. - - `\Unit\Request` class with methods `body()`, `query()`, `headers()`, - `respond(int $status, array $headers, string $body)`. -- Add the new `.c` file to the PHP module build rules in `auto/modules/php`. -- Document the final PHP API surface so entrypoint authors have a stable - contract. - ---- - -### 3. No timeout / force-kill fallback in `nxt_php_quit_handler` - -**Problem.** `nxt_php_quit_handler()` calls `ZEND_ASYNC_SHUTDOWN()` and -returns immediately. If user coroutines are stuck in a tight loop or -blocking I/O that the TrueAsync scheduler cannot interrupt, the worker -hangs forever. - -**Risk.** Silent hang in production; container runtime must SIGKILL instead. - -**Fix.** -- After `ZEND_ASYNC_SHUTDOWN()`, arm a timer for a configurable grace - period (default 30 s). On expiry call `exit(1)`. -- Expose as a PHP application config option: - ```json - { "type": "php", "async": true, "entrypoint": "server.php", - "shutdown_timeout": 30 } - ``` -- Add a test that verifies the force-kill fires when a coroutine refuses - to stop. - ---- - -### 4. Behavior of in-flight requests on shutdown is undefined - -**Problem.** It is not documented whether `ZEND_ASYNC_SHUTDOWN()` lets -active request coroutines run to completion or cancels them immediately. - -`test_php_trueasync_inflight_request_completes` assumes completion -semantics. If cancellation is the designed behavior, the test assertion -must be changed to `pytest.xfail`. - -**Action.** -- Read the TrueAsync scheduler shutdown semantics. -- Document the behavior. Update the test accordingly. - ---- - -### 5. Pending writes in `drain_queue` are silently abandoned on shutdown - -**Problem.** If `ZEND_ASYNC_SHUTDOWN()` fires while `drain_queue` is -non-empty, buffered response bytes are never sent and the client receives -a truncated body. - -**Fix.** Drain the queue synchronously in `nxt_php_quit_handler()` before -calling `ZEND_ASYNC_SHUTDOWN()`. - ---- - -### 6. Test fixtures use a placeholder PHP API - -The entrypoint scripts in `test/php/async_*/entrypoint.php` use -`\Unit\Server::setHandler()` and `\Unit\Request` — these are provisional -names. Once item 2 is resolved: -- Update all three fixture files to match the real class/method names. -- Replace the `_check_trueasync_available()` runtime probe in - `test_php_trueasync.py` with a proper `prerequisites` feature flag - (requires updating `unit/check/discover_available.py`). - ---- - ## OpenSSL 3.6 — test openssl-3.x branch Before the OpenSSL 3.6 migration can be considered fully validated: @@ -485,24 +359,6 @@ causing `test_php_application_disable_classes` and `test_php_application_disable --- -### `rootfs` isolation SIGSEGV (PHP 8.5) - -`test_php_isolation_rootfs` fails with signal 11 (SIGSEGV) when running PHP 8.5 -inside a chroot/rootfs-isolated Unit application. - -**Test:** `test/test_php_isolation.py` — skipped for PHP >= 8.5 with explicit reason. - -**Likely causes:** -- PHP 8.5 introduced new shared library dependencies not present in the minimal rootfs fixture -- Or PHP 8.5 accesses paths at startup that are unavailable in the chroot - -**Investigation steps:** -1. Run `ldd $(which php)` with PHP 8.5 and compare against the rootfs fixture contents -2. Check `unit.log` for the full path that caused the segfault (needs core dump or `strace`) -3. Check if `php 8.5 --define open_basedir=...` reproduces outside of Unit - ---- - ## Test Infrastructure ### Download statistics for packages.freeunit.org diff --git a/auto/cc/hardening b/auto/cc/hardening new file mode 100644 index 000000000..b83c9bd36 --- /dev/null +++ b/auto/cc/hardening @@ -0,0 +1,158 @@ + +# Copyright (C) FreeUnit Community + + +# --hardening=[off|default|strict] (auto/options). The default is "off". +# With "off", NXT_CFLAGS and NXT_LD_OPT stay as auto/cc/test left them. +# A build that does not pass the option does not change. +# +# Each flag below is probed before it is added. The probe compiles a small +# program with the flag under -Werror. For a linker flag, the probe also +# links the program. When an older compiler, a different target or a libc +# does not support a flag, the build does not fail. It only does not get +# that one flag. This follows the "try it" approach of auto/feature. The +# value and run reporting of auto/feature is not used, because these flags +# do not need it. + +case "$NXT_HARDENING" in + off | default | strict) + ;; + + *) + $echo + $echo $0: error: invalid value \"$NXT_HARDENING\" for --hardening. + $echo Valid values are: off, default, strict. + $echo + exit 1 + ;; +esac + + +if [ $NXT_HARDENING != off ]; then + +# The flags are in the order that auto/make uses. The flag under test goes +# into NXT_CFLAGS. CFLAGS from the environment comes last. Thus a -O0 in +# CFLAGS wins over the built-in -O in the probe, as it does in the build. +# The probe includes a libc header. With it, the #warning of glibc about +# a _FORTIFY_SOURCE level that it treats as a lower one fails the probe, +# and not the real sources. $2 is extra source for the probe. +nxt_hardening_try_cflag() +{ + nxt_flag=$1 + + $echo -n "checking for $nxt_flag ..." + + cat << END > $NXT_AUTOTEST.c +#include +${2-} +int main(void) { return 0; } +END + + if /bin/sh -c "($CC $NXT_CFLAGS $nxt_flag $NXT_CC_OPT $CFLAGS -Werror \ + -o $NXT_AUTOTEST $NXT_AUTOTEST.c)" \ + >> $NXT_AUTOCONF_ERR 2>&1 + then + $echo " found" + NXT_CFLAGS="$NXT_CFLAGS $nxt_flag" + nxt_ret=0 + else + $echo " not found" + nxt_ret=1 + fi + + rm -rf $NXT_AUTOTEST $NXT_AUTOTEST.c + + return $nxt_ret +} + + +# Links a small program with the given linker flags. The exit status +# tells if the link succeeded. This function adds nothing to NXT_LD_OPT. +nxt_hardening_link() +{ + cat << END > $NXT_AUTOTEST.c +int main(void) { return 0; } +END + + /bin/sh -c "($CC $CFLAGS $NXT_CFLAGS $NXT_CC_OPT \ + -o $NXT_AUTOTEST $NXT_AUTOTEST.c $NXT_LD_OPT $*)" \ + >> $NXT_AUTOCONF_ERR 2>&1 + nxt_ret=$? + + rm -rf $NXT_AUTOTEST $NXT_AUTOTEST.c + + return $nxt_ret +} + + +# GNU ld only warns about a -z keyword that it does not know, and it links +# anyway. Thus a plain probe would "find" every -z flag. When the linker +# accepts -Wl,--fatal-warnings, the probes below run with it. Then a +# linker warning fails the probe. macOS ld does not accept the flag. +# There the probes run without it. +nxt_hardening_ld_fatal= + +nxt_hardening_try_ldflag() +{ + nxt_flag=$1 + + $echo -n "checking for $nxt_flag ..." + + if nxt_hardening_link $nxt_hardening_ld_fatal $nxt_flag; then + $echo " found" + NXT_LD_OPT="$NXT_LD_OPT $nxt_flag" + else + $echo " not found" + fi +} + + +# Do not stop on an error exit status while the probes run, the same as in +# auto/cc/test. A probe that fails to compile is expected and handled. It +# must not abort the whole configure run. The "set -e" after the last +# probe restores the default. +set +e + +nxt_hardening_try_cflag -fstack-protector-strong +nxt_hardening_try_cflag -fstack-clash-protection + +# Each flag applies to one target only: -fcf-protection to x86, and +# -mbranch-protection=standard to arm64. The compiler rejects the flag on +# the other target. Thus the probe alone selects the correct one. +nxt_hardening_try_cflag -fcf-protection +nxt_hardening_try_cflag -mbranch-protection=standard + +# _FORTIFY_SOURCE does nothing without optimization, and not every libc +# warns about that. Thus the probe checks the optimization level itself. +# When the libc treats a level as a lower one, it gives a warning. Then +# the probe tries the lower level. +nxt_fortify_check='#ifndef __OPTIMIZE__ +#error _FORTIFY_SOURCE needs an optimizing build +#endif' + +nxt_hardening_try_cflag -D_FORTIFY_SOURCE=3 "$nxt_fortify_check" \ + || nxt_hardening_try_cflag -D_FORTIFY_SOURCE=2 "$nxt_fortify_check" + +$echo -n "checking for -Wl,--fatal-warnings ..." + +if nxt_hardening_link -Wl,--fatal-warnings; then + $echo " found" + nxt_hardening_ld_fatal=-Wl,--fatal-warnings +else + $echo " not found" +fi + +nxt_hardening_try_ldflag -Wl,-z,relro +nxt_hardening_try_ldflag -Wl,-z,now +nxt_hardening_try_ldflag -Wl,-z,noexecstack + +if [ $NXT_HARDENING = strict ]; then + nxt_hardening_try_cflag -Wformat=2 + nxt_hardening_try_cflag -Wvla + nxt_hardening_try_cflag -Wimplicit-fallthrough + nxt_hardening_try_cflag -ftrivial-auto-var-init=pattern +fi + +set -e + +fi diff --git a/auto/clang b/auto/clang index fea8602ad..51698d28c 100644 --- a/auto/clang +++ b/auto/clang @@ -114,6 +114,24 @@ nxt_feature_test="int main(void) { . auto/feature +nxt_feature="GCC __builtin_add_overflow()" +nxt_feature_name=NXT_HAVE_BUILTIN_OVERFLOW +nxt_feature_run= +nxt_feature_incs= +nxt_feature_libs= +nxt_feature_test="#include + + int main(void) { + size_t r; + + if (__builtin_add_overflow((size_t) 2, (size_t) 3, &r) + || __builtin_mul_overflow((size_t) 2, (size_t) 3, &r)) + return 1; + return 0; + }" +. auto/feature + + nxt_feature="GCC __attribute__ visibility" nxt_feature_name=NXT_HAVE_GCC_ATTRIBUTE_VISIBILITY nxt_feature_run= @@ -196,6 +214,25 @@ nxt_feature_test="static void f(void) __attribute__ ((__unused__)); . auto/feature +nxt_feature="GCC __attribute__ fallthrough" +nxt_feature_name=NXT_HAVE_GCC_ATTRIBUTE_FALLTHROUGH +nxt_feature_run= +nxt_feature_incs= +nxt_feature_libs= +nxt_feature_test="int main(int argc, char **argv) { + switch (argc) { + case 1: + argc++; + __attribute__ ((__fallthrough__)); + default: + break; + } + + return !argc; + }" +. auto/feature + + nxt_feature="GCC __attribute__ nonstring" nxt_feature_name=NXT_HAVE_GCC_ATTRIBUTE_NONSTRING nxt_feature_run= diff --git a/auto/files b/auto/files index 4912ed802..6eedead30 100644 --- a/auto/files +++ b/auto/files @@ -69,6 +69,56 @@ nxt_feature_test="#define _GNU_SOURCE . auto/feature +# close_range(), Linux 5.9 with glibc 2.34, FreeBSD 12.2. + +nxt_feature="close_range()" +nxt_feature_name=NXT_HAVE_CLOSE_RANGE +nxt_feature_run= +nxt_feature_incs= +nxt_feature_libs= +nxt_feature_test="#define _GNU_SOURCE + #include + + int main(void) { + return close_range(3, ~0U, 0); + }" +. auto/feature + + +if [ $nxt_found = no ]; then + + # A libc without the wrapper, such as musl, may still have the number. + + nxt_feature="close_range() syscall" + nxt_feature_name=NXT_HAVE_SYS_CLOSE_RANGE + nxt_feature_test="#include + #include + + int main(void) { + return syscall(SYS_close_range, 3, ~0U, 0); + }" + . auto/feature +fi + + +# closefrom(), FreeBSD 8.0, NetBSD 3.0, OpenBSD 3.5, Solaris 9. +# Solaris and illumos declare it in , the others in . + +nxt_feature="closefrom()" +nxt_feature_name=NXT_HAVE_CLOSEFROM +nxt_feature_run= +nxt_feature_incs= +nxt_feature_libs= +nxt_feature_test="#include + #include + + int main(void) { + closefrom(3); + return 0; + }" +. auto/feature + + nxt_feature="openat2()" nxt_feature_name=NXT_HAVE_OPENAT2 nxt_feature_run= diff --git a/auto/fuzzing b/auto/fuzzing index f792941bd..6f68f6c14 100644 --- a/auto/fuzzing +++ b/auto/fuzzing @@ -20,10 +20,15 @@ if [ -n "$NXT_FUZZ" ]; then .PHONY: fuzz fuzz: $NXT_BUILD_DIR/fuzz_basic \\ + $NXT_BUILD_DIR/fuzz_http_chunk \\ $NXT_BUILD_DIR/fuzz_http_controller \\ $NXT_BUILD_DIR/fuzz_http_h1p \\ $NXT_BUILD_DIR/fuzz_http_h1p_peer \\ - $NXT_BUILD_DIR/fuzz_json + $NXT_BUILD_DIR/fuzz_http_h1p_peer_response \\ + $NXT_BUILD_DIR/fuzz_http_ws_utf8 \\ + $NXT_BUILD_DIR/fuzz_json \\ + $NXT_BUILD_DIR/fuzz_router_app_response \\ + $NXT_BUILD_DIR/fuzz_unit_msg $NXT_BUILD_DIR/fuzz_basic: \$(NXT_FUZZ_OBJS) \\ $NXT_BUILD_DIR/lib/$NXT_LIB_STATIC @@ -34,6 +39,15 @@ $NXT_BUILD_DIR/fuzz_basic: \$(NXT_FUZZ_OBJS) \\ $NXT_LD_OPT $NXT_LIBM $NXT_LIBS $NXT_LIB_AUX_LIBS \\ $NXT_FUZZ +$NXT_BUILD_DIR/fuzz_http_chunk: \$(NXT_FUZZ_OBJS) \\ + $NXT_BUILD_DIR/lib/$NXT_LIB_STATIC + \$(PP_LD) \$@ + \$(v)\$(NXT_EXEC_LINK) -o $NXT_BUILD_DIR/fuzz_http_chunk \\ + \$(CFLAGS) $NXT_BUILD_DIR/fuzzing/nxt_http_chunk_fuzz.o \\ + $NXT_BUILD_DIR/lib/$NXT_LIB_STATIC \\ + $NXT_LD_OPT $NXT_LIBM $NXT_LIBS $NXT_LIB_AUX_LIBS \\ + $NXT_FUZZ + $NXT_BUILD_DIR/fuzz_http_controller: \$(NXT_FUZZ_OBJS) \\ $NXT_BUILD_DIR/lib/$NXT_LIB_STATIC \$(PP_LD) \$@ @@ -61,6 +75,24 @@ $NXT_BUILD_DIR/fuzz_http_h1p_peer: \$(NXT_FUZZ_OBJS) \\ $NXT_LD_OPT $NXT_LIBM $NXT_LIBS $NXT_LIB_AUX_LIBS \\ $NXT_FUZZ +$NXT_BUILD_DIR/fuzz_http_h1p_peer_response: \$(NXT_FUZZ_OBJS) \\ + $NXT_BUILD_DIR/lib/$NXT_LIB_STATIC + \$(PP_LD) \$@ + \$(v)\$(NXT_EXEC_LINK) -o $NXT_BUILD_DIR/fuzz_http_h1p_peer_response \\ + \$(CFLAGS) $NXT_BUILD_DIR/fuzzing/nxt_http_h1p_peer_response_fuzz.o \\ + $NXT_BUILD_DIR/lib/$NXT_LIB_STATIC \\ + $NXT_LD_OPT $NXT_LIBM $NXT_LIBS $NXT_LIB_AUX_LIBS \\ + $NXT_FUZZ + +$NXT_BUILD_DIR/fuzz_http_ws_utf8: \$(NXT_FUZZ_OBJS) \\ + $NXT_BUILD_DIR/lib/$NXT_LIB_STATIC + \$(PP_LD) \$@ + \$(v)\$(NXT_EXEC_LINK) -o $NXT_BUILD_DIR/fuzz_http_ws_utf8 \\ + \$(CFLAGS) $NXT_BUILD_DIR/fuzzing/nxt_http_ws_utf8_fuzz.o \\ + $NXT_BUILD_DIR/lib/$NXT_LIB_STATIC \\ + $NXT_LD_OPT $NXT_LIBM $NXT_LIBS $NXT_LIB_AUX_LIBS \\ + $NXT_FUZZ + $NXT_BUILD_DIR/fuzz_json: \$(NXT_FUZZ_OBJS) \\ $NXT_BUILD_DIR/lib/$NXT_LIB_STATIC \$(PP_LD) \$@ @@ -70,6 +102,25 @@ $NXT_BUILD_DIR/fuzz_json: \$(NXT_FUZZ_OBJS) \\ $NXT_LD_OPT $NXT_LIBM $NXT_LIBS $NXT_LIB_AUX_LIBS \\ $NXT_FUZZ +$NXT_BUILD_DIR/fuzz_router_app_response: \$(NXT_FUZZ_OBJS) \\ + $NXT_BUILD_DIR/lib/$NXT_LIB_STATIC + \$(PP_LD) \$@ + \$(v)\$(NXT_EXEC_LINK) -o $NXT_BUILD_DIR/fuzz_router_app_response \\ + \$(CFLAGS) $NXT_BUILD_DIR/fuzzing/nxt_router_app_response_fuzz.o \\ + $NXT_BUILD_DIR/lib/$NXT_LIB_STATIC \\ + $NXT_LD_OPT $NXT_LIBM $NXT_LIBS $NXT_LIB_AUX_LIBS \\ + $NXT_FUZZ + +# libunit's receive side: linked against libunit, not the core library. +$NXT_BUILD_DIR/fuzz_unit_msg: \$(NXT_FUZZ_OBJS) \\ + $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_STATIC + \$(PP_LD) \$@ + \$(v)\$(NXT_EXEC_LINK) -o $NXT_BUILD_DIR/fuzz_unit_msg \\ + \$(CFLAGS) $NXT_BUILD_DIR/fuzzing/nxt_unit_msg_fuzz.o \\ + $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_STATIC \\ + $NXT_LD_OPT $NXT_LIBM $NXT_LIBS $NXT_LIB_AUX_LIBS \\ + $NXT_FUZZ + END fi diff --git a/auto/help b/auto/help index 4e10d2844..c851e75b1 100644 --- a/auto/help +++ b/auto/help @@ -61,6 +61,16 @@ cat << END --debug enable debug logging + --hardening=MODE add compiler and linker hardening flags, probed for + compiler support (default: off, unchanged build): + off - no extra flags (default) + default - stack protector, _FORTIFY_SOURCE=3 (or 2), + stack-clash and control-flow protection, + RELRO/BIND_NOW/noexecstack at link time + strict - default, plus -Wformat=2, -Wvla, + -Wimplicit-fallthrough and + -ftrivial-auto-var-init=pattern + --fuzz=ENGINE enable fuzz testing diff --git a/auto/make b/auto/make index e15469bec..7d8ae3527 100644 --- a/auto/make +++ b/auto/make @@ -178,6 +178,7 @@ for nxt_src in $NXT_LIB_SRCS $NXT_TEST_SRCS $NXT_FUZZ_SRCS $NXT_LIB_UNIT_SRCS \ src/test/nxt_unit_app_test.c \ src/test/nxt_unit_close_test.c \ src/test/nxt_unit_port_recv_test.c \ + src/test/nxt_unit_msg_test.c \ src/test/nxt_unit_websocket_chat.c \ src/test/nxt_unit_websocket_echo.c do @@ -185,11 +186,21 @@ do nxt_dep=${nxt_src%.c}.dep nxt_dep_flags=`nxt_gen_dep_flags` nxt_dep_post=`nxt_gen_dep_post` + + # Only the test sources see NXT_TESTS here. The core and libunit + # objects built by this loop are the ones unitd, libnxt.a, libunit.a + # and the language modules link, so they never get the macro. The + # instrumented copies the test programs link are built further down. + case "$nxt_src" in + src/test/*) nxt_test_cflags="-DNXT_TESTS=1" ;; + *) nxt_test_cflags= ;; + esac + cat << END >> $NXT_MAKEFILE $NXT_BUILD_DIR/$nxt_obj: $nxt_src $NXT_VERSION_H \$(PP_CC) \$@ - \$(v)\$(CC) -c \$(CFLAGS) \$(NXT_LIB_INCS) $NXT_LIB_AUX_CFLAGS \\ + \$(v)\$(CC) -c \$(CFLAGS) $nxt_test_cflags \$(NXT_LIB_INCS) $NXT_LIB_AUX_CFLAGS \\ -o $NXT_BUILD_DIR/$nxt_obj \\ $nxt_dep_flags \\ $nxt_src @@ -210,7 +221,7 @@ cat << END >> $NXT_MAKEFILE $NXT_BUILD_DIR/$nxt_obj: $nxt_src $NXT_VERSION_H \$(PP_CC) \$@ - \$(v)\$(CC) -c \$(CFLAGS) -DNXT_NCQ_TEST=1 \$(NXT_LIB_INCS) $NXT_LIB_AUX_CFLAGS \\ + \$(v)\$(CC) -c \$(CFLAGS) -DNXT_TESTS=1 -DNXT_NCQ_TEST=1 \$(NXT_LIB_INCS) $NXT_LIB_AUX_CFLAGS \\ -o $NXT_BUILD_DIR/$nxt_obj \\ $nxt_dep_flags \\ $nxt_src @@ -229,7 +240,7 @@ cat << END >> $NXT_MAKEFILE $NXT_BUILD_DIR/$nxt_obj: $nxt_src $NXT_VERSION_H \$(PP_CC) \$@ - \$(v)\$(CC) -c \$(CFLAGS) -DNXT_NCQ_TEST=0 \$(NXT_LIB_INCS) $NXT_LIB_AUX_CFLAGS \\ + \$(v)\$(CC) -c \$(CFLAGS) -DNXT_TESTS=1 -DNXT_NCQ_TEST=0 \$(NXT_LIB_INCS) $NXT_LIB_AUX_CFLAGS \\ -o $NXT_BUILD_DIR/$nxt_obj \\ $nxt_dep_flags \\ $nxt_src @@ -244,6 +255,73 @@ $echo >> $NXT_MAKEFILE if [ $NXT_TESTS = YES ]; then + # Instrumented core and libunit. + # + # Every source of libnxt.a and libunit.a is compiled a second time with + # -DNXT_TESTS=1 into .test.o and archived as libnxt-tests.a and + # libunit-tests.a. The test programs link only these archives; the plain + # archives above are what unitd and the language modules link. Compiling + # the whole set, not a list of the sources that carry hooks, means a hook + # added to any core or libunit source is covered without a build change. + + $echo "NXT_LIB_TEST_OBJS = \\" >> $NXT_MAKEFILE + + for nxt_src in $NXT_LIB_SRCS + do + nxt_obj=${nxt_src%.c}.test.o + $echo " $NXT_BUILD_DIR/$nxt_obj \\" >> $NXT_MAKEFILE + done + + $echo >> $NXT_MAKEFILE + + $echo "NXT_LIB_UNIT_TEST_OBJS = \\" >> $NXT_MAKEFILE + $echo " $NXT_BUILD_DIR/src/nxt_lvlhsh.test.o \\" >> $NXT_MAKEFILE + $echo " $NXT_BUILD_DIR/src/nxt_murmur_hash.test.o \\" >> $NXT_MAKEFILE + $echo " $NXT_BUILD_DIR/src/nxt_socket_msg.test.o \\" >> $NXT_MAKEFILE + $echo " $NXT_BUILD_DIR/src/nxt_websocket.test.o \\" >> $NXT_MAKEFILE + + for nxt_src in $NXT_LIB_UNIT_SRCS + do + nxt_obj=${nxt_src%.c}.test.o + $echo " $NXT_BUILD_DIR/$nxt_obj \\" >> $NXT_MAKEFILE + done + + $echo >> $NXT_MAKEFILE + + for nxt_src in $NXT_LIB_SRCS $NXT_LIB_UNIT_SRCS + do + nxt_obj=${nxt_src%.c}.test.o + nxt_dep=${nxt_src%.c}.test.dep + nxt_dep_flags=`nxt_gen_dep_flags` + nxt_dep_post=`nxt_gen_dep_post` + cat << END >> $NXT_MAKEFILE + +$NXT_BUILD_DIR/$nxt_obj: $nxt_src $NXT_VERSION_H + \$(PP_CC) \$@ + \$(v)\$(CC) -c \$(CFLAGS) -DNXT_TESTS=1 \$(NXT_LIB_INCS) $NXT_LIB_AUX_CFLAGS \\ + -o $NXT_BUILD_DIR/$nxt_obj \\ + $nxt_dep_flags \\ + $nxt_src + $nxt_dep_post + +-include $NXT_BUILD_DIR/$nxt_dep + +END + + done + + cat << END >> $NXT_MAKEFILE + +$NXT_BUILD_DIR/lib/$NXT_LIB_TEST_STATIC: \$(NXT_LIB_TEST_OBJS) + \$(PP_AR) \$@ + \$(v)$NXT_STATIC_LINK \$@ \$(NXT_LIB_TEST_OBJS) + +$NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_TEST_STATIC: \$(NXT_LIB_UNIT_TEST_OBJS) + \$(PP_AR) \$@ + \$(v)$NXT_STATIC_LINK \$@ \$(NXT_LIB_UNIT_TEST_OBJS) + +END + # Test object files list. $echo "NXT_TEST_OBJS = \\" >> $NXT_MAKEFILE @@ -259,92 +337,92 @@ if [ $NXT_TESTS = YES ]; then cat << END >> $NXT_MAKEFILE .PHONY: tests -tests: $NXT_BUILD_DIR/tests $NXT_BUILD_DIR/utf8_file_name_test \\ +tests: $NXT_BUILD_DIR/tests \\ $NXT_BUILD_DIR/ncq_test \\ $NXT_BUILD_DIR/vbcq_test \\ $NXT_BUILD_DIR/unit_app_test $NXT_BUILD_DIR/unit_websocket_chat \\ $NXT_BUILD_DIR/unit_websocket_echo \\ $NXT_BUILD_DIR/unit_close_test \\ - $NXT_BUILD_DIR/unit_port_recv_test + $NXT_BUILD_DIR/unit_port_recv_test \\ + $NXT_BUILD_DIR/unit_msg_test $NXT_BUILD_DIR/tests: \$(NXT_TEST_OBJS) \\ - $NXT_BUILD_DIR/lib/$NXT_LIB_STATIC \\ + $NXT_BUILD_DIR/lib/$NXT_LIB_TEST_STATIC \\ \$(NXT_OTEL_LIB_STATIC) \$(PP_LD) \$@ \$(v)\$(NXT_EXEC_LINK) -o $NXT_BUILD_DIR/tests \\ \$(CFLAGS) \$(NXT_TEST_OBJS) \\ - $NXT_BUILD_DIR/lib/$NXT_LIB_STATIC \$(NXT_OTEL_LIB_STATIC) \\ - $NXT_LD_OPT $NXT_LIBM $NXT_LIBS $NXT_LIB_AUX_LIBS - -$NXT_BUILD_DIR/utf8_file_name_test: $NXT_LIB_UTF8_FILE_NAME_TEST_SRCS \\ - $NXT_BUILD_DIR/lib/$NXT_LIB_STATIC \\ - \$(NXT_OTEL_LIB_STATIC) - \$(PP_LD) \$@ - \$(v)\$(CC) \$(CFLAGS) \$(NXT_LIB_INCS) $NXT_LIB_AUX_CFLAGS \\ - -o $NXT_BUILD_DIR/utf8_file_name_test \\ - $NXT_LIB_UTF8_FILE_NAME_TEST_SRCS \\ - $NXT_BUILD_DIR/lib/$NXT_LIB_STATIC \$(NXT_OTEL_LIB_STATIC) \\ + $NXT_BUILD_DIR/lib/$NXT_LIB_TEST_STATIC \$(NXT_OTEL_LIB_STATIC) \\ $NXT_LD_OPT $NXT_LIBM $NXT_LIBS $NXT_LIB_AUX_LIBS $NXT_BUILD_DIR/ncq_test: $NXT_BUILD_DIR/src/test/nxt_ncq_test.o \\ - $NXT_BUILD_DIR/lib/$NXT_LIB_STATIC \\ + $NXT_BUILD_DIR/lib/$NXT_LIB_TEST_STATIC \\ \$(NXT_OTEL_LIB_STATIC) \$(PP_LD) \$@ \$(v)\$(NXT_EXEC_LINK) -o $NXT_BUILD_DIR/ncq_test \\ \$(CFLAGS) $NXT_BUILD_DIR/src/test/nxt_ncq_test.o \\ - $NXT_BUILD_DIR/lib/$NXT_LIB_STATIC \$(NXT_OTEL_LIB_STATIC) \\ + $NXT_BUILD_DIR/lib/$NXT_LIB_TEST_STATIC \$(NXT_OTEL_LIB_STATIC) \\ $NXT_LD_OPT $NXT_LIBM $NXT_LIBS $NXT_LIB_AUX_LIBS $NXT_BUILD_DIR/vbcq_test: $NXT_BUILD_DIR/src/test/nxt_vbcq_test.o \\ - $NXT_BUILD_DIR/lib/$NXT_LIB_STATIC \\ + $NXT_BUILD_DIR/lib/$NXT_LIB_TEST_STATIC \\ \$(NXT_OTEL_LIB_STATIC) \$(PP_LD) \$@ \$(v)\$(NXT_EXEC_LINK) -o $NXT_BUILD_DIR/vbcq_test \\ \$(CFLAGS) $NXT_BUILD_DIR/src/test/nxt_vbcq_test.o \\ - $NXT_BUILD_DIR/lib/$NXT_LIB_STATIC \$(NXT_OTEL_LIB_STATIC) \\ + $NXT_BUILD_DIR/lib/$NXT_LIB_TEST_STATIC \$(NXT_OTEL_LIB_STATIC) \\ $NXT_LD_OPT $NXT_LIBM $NXT_LIBS $NXT_LIB_AUX_LIBS $NXT_BUILD_DIR/unit_app_test: $NXT_BUILD_DIR/src/test/nxt_unit_app_test.o \\ - $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_STATIC + $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_TEST_STATIC \$(PP_LD) \$@ \$(v)\$(NXT_EXEC_LINK) -o $NXT_BUILD_DIR/unit_app_test \\ \$(CFLAGS) $NXT_BUILD_DIR/src/test/nxt_unit_app_test.o \\ - $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_STATIC \\ + $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_TEST_STATIC \\ $NXT_LD_OPT $NXT_LIBM $NXT_LIBS $NXT_LIB_AUX_LIBS $NXT_BUILD_DIR/unit_close_test: $NXT_BUILD_DIR/src/test/nxt_unit_close_test.o \\ - $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_STATIC + $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_TEST_STATIC \$(PP_LD) \$@ \$(v)\$(NXT_EXEC_LINK) -o $NXT_BUILD_DIR/unit_close_test \\ \$(CFLAGS) $NXT_BUILD_DIR/src/test/nxt_unit_close_test.o \\ - $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_STATIC \\ + $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_TEST_STATIC \\ $NXT_LD_OPT $NXT_LIBM $NXT_LIBS $NXT_LIB_AUX_LIBS $NXT_BUILD_DIR/unit_port_recv_test: \\ $NXT_BUILD_DIR/src/test/nxt_unit_port_recv_test.o \\ - $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_STATIC + $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_TEST_STATIC \$(PP_LD) \$@ \$(v)\$(NXT_EXEC_LINK) -o $NXT_BUILD_DIR/unit_port_recv_test \\ \$(CFLAGS) $NXT_BUILD_DIR/src/test/nxt_unit_port_recv_test.o \\ - $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_STATIC \\ + $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_TEST_STATIC \\ + $NXT_LD_OPT $NXT_LIBM $NXT_LIBS $NXT_LIB_AUX_LIBS + +$NXT_BUILD_DIR/unit_msg_test: \\ + $NXT_BUILD_DIR/src/test/nxt_unit_msg_test.o \\ + $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_TEST_STATIC + \$(PP_LD) \$@ + \$(v)\$(NXT_EXEC_LINK) -o $NXT_BUILD_DIR/unit_msg_test \\ + \$(CFLAGS) $NXT_BUILD_DIR/src/test/nxt_unit_msg_test.o \\ + $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_TEST_STATIC \\ $NXT_LD_OPT $NXT_LIBM $NXT_LIBS $NXT_LIB_AUX_LIBS $NXT_BUILD_DIR/unit_websocket_chat: \\ $NXT_BUILD_DIR/src/test/nxt_unit_websocket_chat.o \\ - $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_STATIC + $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_TEST_STATIC \$(PP_LD) \$@ \$(v)\$(NXT_EXEC_LINK) -o $NXT_BUILD_DIR/unit_websocket_chat \\ \$(CFLAGS) $NXT_BUILD_DIR/src/test/nxt_unit_websocket_chat.o \\ - $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_STATIC \\ + $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_TEST_STATIC \\ $NXT_LD_OPT $NXT_LIBM $NXT_LIBS $NXT_LIB_AUX_LIBS $NXT_BUILD_DIR/unit_websocket_echo: \\ $NXT_BUILD_DIR/src/test/nxt_unit_websocket_echo.o \\ - $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_STATIC + $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_TEST_STATIC \$(PP_LD) \$@ \$(v)\$(NXT_EXEC_LINK) -o $NXT_BUILD_DIR/unit_websocket_echo \\ \$(CFLAGS) $NXT_BUILD_DIR/src/test/nxt_unit_websocket_echo.o \\ - $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_STATIC \\ + $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_TEST_STATIC \\ $NXT_LD_OPT $NXT_LIBM $NXT_LIBS $NXT_LIB_AUX_LIBS END diff --git a/auto/modules/java b/auto/modules/java index 30bbd30d7..c74a337dc 100644 --- a/auto/modules/java +++ b/auto/modules/java @@ -238,7 +238,7 @@ cat << END > $NXT_JAVA_JARS static const char *nxt_java_system_jars[] = { END -NXT_TOMCAT_VERSION=9.0.110 +NXT_TOMCAT_VERSION=9.0.122 NXT_JAR_VERSION=$NXT_TOMCAT_VERSION @@ -297,7 +297,7 @@ NXT_JAR_NAME=jetty-http . auto/modules/java_get_jar NXT_JAR_NAME=classgraph -NXT_JAR_VERSION=4.8.181 +NXT_JAR_VERSION=4.8.186 NXT_JAR_NAMESPACE=io/github/classgraph/ . auto/modules/java_get_jar diff --git a/auto/modules/java_chk_sha512 b/auto/modules/java_chk_sha512 index 10891cee0..fe259c582 100644 --- a/auto/modules/java_chk_sha512 +++ b/auto/modules/java_chk_sha512 @@ -37,6 +37,22 @@ fi if [ -f "$NXT_JAR_CHK_FILE" ]; then NXT_JAR_SHA512=`grep -F $NXT_JAR_FILE auto/modules/java_jar.sha512 | head -c 128` + + # An unpinned jar produced an empty expected value, a malformed check + # line, and a "SHA512 not matched" that reads exactly like a corrupt + # download. It is neither: it means the version was bumped without + # updating auto/modules/java_jar.sha512. + if [ -z "$NXT_JAR_SHA512" ]; then + $echo + $echo $0: error: no pinned SHA512 for "$NXT_JAR_FILE". + $echo + $echo " Add its checksum to auto/modules/java_jar.sha512;" + $echo " a version bump needs the pinned checksums updated too." + $echo + $echo "error: no pinned SHA512 for $NXT_JAR_FILE" >> $NXT_AUTOCONF_ERR + exit 1 + fi + NXT_JAR_CHK=${NXT_JAR_CHK_FILE}.sha512.$$ $echo "$NXT_JAR_SHA512 $NXT_JAR_CHK_FILE" > $NXT_JAR_CHK diff --git a/auto/modules/java_get_jar b/auto/modules/java_get_jar index 81b300f91..ad2348d1e 100644 --- a/auto/modules/java_get_jar +++ b/auto/modules/java_get_jar @@ -25,10 +25,73 @@ if [ ! -f "$NXT_BUILD_DIR/$NXT_JAR_FILE" ]; then $echo "getting remote $NXT_JAR_FILE ..." >> $NXT_AUTOCONF_ERR mkdir -p "${NXT_JAR_LOCAL_DIR}" - curl --progress-bar "$NXT_JAR_URL" -o "$NXT_JAR_LOCAL_TMP" - NXT_JAR_CHK_FILE=$NXT_JAR_LOCAL_TMP - . auto/modules/java_chk_sha512 + # The observed CI failure is a COMPLETE transfer of the wrong bytes: + # a "100.0%" progress bar in 0.07s for a 280 KB jar, then a checksum + # mismatch -- an error page or a poisoned cache, not a truncated + # download. curl's own --retry does not cover that, since as far as + # curl is concerned the transfer succeeded, so the retry has to span + # the verification too. --fail keeps an error page from being written + # out as if it were the jar, and --location follows a redirect. + NXT_JAR_TRY=1 + NXT_JAR_FAIL=download + + while [ $NXT_JAR_TRY -le 3 ]; do + + if curl --fail --location --retry 2 --retry-delay 2 \ + --progress-bar "$NXT_JAR_URL" -o "$NXT_JAR_LOCAL_TMP" + then + NXT_JAR_CHK_FILE=$NXT_JAR_LOCAL_TMP + . auto/modules/java_chk_sha512 + + # java_chk_sha512 deletes the file it rejects, so the file + # surviving means the checksum matched. + if [ -f "$NXT_JAR_LOCAL_TMP" ]; then + break + fi + + NXT_JAR_FAIL=checksum + + $echo "retrying $NXT_JAR_FILE: checksum did not match" + else + rm -f "$NXT_JAR_LOCAL_TMP" + + NXT_JAR_FAIL=download + + $echo "retrying $NXT_JAR_FILE: download failed" + fi + + NXT_JAR_TRY=$((NXT_JAR_TRY + 1)) + done + + # Every attempt was rejected. Say why, rather than letting the mv + # below fail with a "cannot stat" that hides the cause, and keep the + # two causes apart: a download that never arrived is a different + # problem from bytes that arrived and did not verify. + if [ ! -f "$NXT_JAR_LOCAL_TMP" ] && [ "$NXT_JAR_FAIL" = "download" ] + then + $echo + $echo $0: error: failed to download "$NXT_JAR_URL". + $echo + $echo "error: failed to download $NXT_JAR_URL" >> $NXT_AUTOCONF_ERR + exit 1 + fi + + if [ ! -f "$NXT_JAR_LOCAL_TMP" ]; then + $echo + $echo $0: error: SHA512 mismatch for "$NXT_JAR_FILE". + $echo + $echo " downloaded from: $NXT_JAR_URL" + $echo " expected SHA512: $NXT_JAR_SHA512" + $echo + $echo " The pinned checksum is in auto/modules/java_jar.sha512." + $echo " A mismatch means the download was truncated or proxied," + $echo " or the artifact itself changed." + $echo + $echo "error: SHA512 mismatch for $NXT_JAR_FILE" \ + >> $NXT_AUTOCONF_ERR + exit 1 + fi mv "$NXT_JAR_LOCAL_TMP" "$NXT_JAR_LOCAL" else diff --git a/auto/modules/java_jar.sha512 b/auto/modules/java_jar.sha512 index 29e0f6199..c4de2261a 100644 --- a/auto/modules/java_jar.sha512 +++ b/auto/modules/java_jar.sha512 @@ -1,23 +1,14 @@ -56ddba7508c2e56f710af1d4c619ccdfda3acc378cfde71d7e5a13b62c1388b424fa91133e528edc16e9619093fb888b2ec7aaa9721d8e14e101ee2a4a00877a classgraph-4.8.181.jar +1e750bb08bff586c1023ddf6af9f3987bf060a5adb3cf1b79d1d1c9ca662ebb3e5aaf8a70a158ecb8b00a836884b4d93ea1bcfe116c92a5e613c2c294451fd49 classgraph-4.8.186.jar ab441acf5551a7dc81c353eaccb3b3df9e89a48987294d19e39acdb83a5b640fcdff7414cee29f5b96eaa8826647f1d5323e185018fe33a64c402d69c73c9158 ecj-3.26.0.jar 609516a5c3b7fa9e256b768124d3c822f28ce426687a479eb57b0f2f6dc82ee6c3560f8e790394b71feebc409eda78a04d6c03254bb41127506ffcf6b7905a61 jetty-http-9.4.58.v20250814.jar 5f2b1efe5f6279ed9c3a25d8c3a6413b195a6957500752cb7b59fc8e8bc43d01eeb2e97b619bfb74ff5ec96cd8893ba1d9d372f8a226ff197d3b452febafa146 jetty-server-9.4.58.v20250814.jar c4505826e68be8d8736eba5cd29fd86ada455ee5765f8d46d33085b788eb3295023b167354178441e80c97edde9f988f89bf3959c59ef8aada56e5d863673021 jetty-util-9.4.58.v20250814.jar -65cfee9beb10eb9831af55f039d8084dcd500a9718f4c753acf99fb52d4ab8b1bd70c8a6807a4cae7096466ea344c79d366c4b115d8b1e641bc6c86e49e2c8c3 tomcat-api-9.0.108.jar -b1970b9af36882c351fc0913ca39d33e184dd97274a36d6856a9a52f930ea0e9f3aa04758905ee7b416526322fe06e846c7f041fcbbeb561be82cc0a30bed647 tomcat-api-9.0.110.jar -772f68efbd68d010eb72d043c115364d65083c9a95887df79bf26feed6aa332348424ec3fbab9155dab508c11a40a31f1d1aea1205d04a180b0619497628ed32 tomcat-el-api-9.0.108.jar -1957abb8d4bf8e51d98a9a4064b157d00d26d0b8fd9513669745c3e61645d9dcc07a570305bd22ca1f0d1685efb8e652b17aa2e6a30cac8f136fda9e15f1baf7 tomcat-el-api-9.0.110.jar -fee3afbb966efdbe49347f1942a3ba5aa275e7193de6a14a0355f49a8138b255f4123189435613976e18c55cb37f4f53e5bc8dd1b9cc178551b4fb0978e545a4 tomcat-jasper-9.0.108.jar -e482fd98ce80176b106a726e2d54506a40a9add016549b35f1f5faff4a95a3a14e5f2ca471ad458a0907064c983693c48a1146650bd1144b1f2b624f37b11d02 tomcat-jasper-9.0.110.jar -93c6ac9876599a1a7c7dfc09458edcc109de0748db77514b765132814d80621a4bca431378acc916306e928119cea1157288ff81f8d88fd1661366aded8213b2 tomcat-jasper-el-9.0.108.jar -d461d63ecaa824298bc72983297bd1f0b108d20c182675dcdd44090a5390cc8889c7a50e50d03d7367939cfc77a33128372ddd6907387ee6d84789e457aa9af7 tomcat-jasper-el-9.0.110.jar -c3bef13c90043fa179c3b7ab599c7ab227e5bf28132f314049222096a1af80b34fc28737f40973c0dc4f10f5899da524fefafc5a97981b0f18e87062a5041766 tomcat-jsp-api-9.0.108.jar -8ab91979e09581009d1fa26b2a99dae59454791095015d818a6db44fe383935a1758d705c405fe7de3633561546d6a6417005d7f9a4e6cb96a818520bf67659b tomcat-jsp-api-9.0.110.jar -3ea930f5da6f2e4ba90208ef4cede22decf737aad252f9c74dcf4ffaa7351ab52e153b31f41fe64fc0c7a1bae626e1e1da2de679975583740975cce4391c67d9 tomcat-juli-9.0.108.jar -313900c4b8c65e232da9431f7b1e2745c81d2a6ece7be06d5bc51da3b5c321afc0080959b48e47a84f99558ecfbbbc642685cb171d2bac85c86d0cb50cfdc103 tomcat-juli-9.0.110.jar -a1e93a66ffe517baa37878d085b18537c18284c8ddf3872d9b4c2ef0d6579c68f0eac57d5d15303cde485b3286bafd20e1b03136e9c86712002a31edd6081052 tomcat-servlet-api-9.0.108.jar -6cce38b6e786538ce09d0098bc75923cea9a08b726dfcdac66625c8d4b103c4d546a037351008d95be026c4ceb99e7db1ee9d605bb5e13a78eff0431e8f7e03c tomcat-servlet-api-9.0.110.jar -cfcabcd79c56f33415b8f4095ac894c0dfa5f392fd272e2e66d61d67051b1eac026740dd21657179a651fa0eeb2f39351781771b0e38799ab1fdf277ebda9883 tomcat-util-9.0.108.jar -f1d183cf8568cab72dcb72dbe052656b448f95c546288237c8548577e51e975dfb371e8152557d4b094ed98035a1bfc5de6edbeca09e5751ca56c9677ae792e5 tomcat-util-9.0.110.jar -0cd88e452ead14c89cd9c70b965795f5793355bc61a0cbf58355585afedcc8a24cb62968da757e253ef1b119021a543aea0058cf9fd5ab152bc185e338c3d15f tomcat-util-scan-9.0.108.jar -0059c8f5bb338d4e4492f15537c9ec9cb337472c878d46b32dbc5811fe517f35917358a23777b2e2eea7f6c9b5c21d07532ddc25f9a82defd73c3c1e11800ada tomcat-util-scan-9.0.110.jar +36383f02b50b0243a78a71500bbd885ec699b429fa2d1d8e1723276d4aa525e0e4fc1cb7dc0b977e48d182dd02feb6fd4db29e6af348cd274c1a10a10ebb8f39 tomcat-api-9.0.122.jar +bc5890603c5fa3e4dd817fd6fcf7d196a01a9c30ddab35f276d15239b5efd1d74df3ee3468dc3a2cfd8c366e7dcc26361f400a47faf32486c4453e53e153c2be tomcat-el-api-9.0.122.jar +20aba1db626aff6b66446721ae5a8a99c8040e285da01a4e0c649841957304bef43538939f65a98749be0cfc2e1ea3a000c3b733948fbf9a081f616fdb51a7e4 tomcat-jasper-9.0.122.jar +94f731d2e044a17c41483853f43c07ab48be45effb87678fd8f0cf6d5b6ff48b8f0482349ccb5b47c38e586577e51695727c97287cadb34f49e3c6ae851715bc tomcat-jasper-el-9.0.122.jar +188126266f4d1db7ebe2a5ca48ce0abb4b7ea75c53c79a18a117bfac8afecb43324fdbe933459239989533142d8d246bf2798545e63b638c8955b7de5dc8178b tomcat-jsp-api-9.0.122.jar +f7a756e6e91d378d8da41dc906372b8f3dcd2a8e985393eff6a79ea94d7b31e631b9ebe5661681f5cfd5072362d0097e6b15c8444c87f66af13efe08e78ee82d tomcat-juli-9.0.122.jar +f81255c4d696ef156c82e5b6ea543105712327e8ebcd95e33171f7d4cf5d380ce86f88350e08054ba6743675e76760915bb6fd20d6d72d34ea8d2c72b6dc7d43 tomcat-servlet-api-9.0.122.jar +78f5cb17044f8ed3a837756d6600048eb326a00512a9ef45bbd9bbd0840e51234cb18847af4e55c0545e412ac9b6a736ab113f6efa0096ea5b2843c2d8ed5bf9 tomcat-util-9.0.122.jar +94aa42c471809a543fe99017f3ad873956cf3fc00fb17f34e7e37950b7948e784bb68d7e3fb005190c6d0db1eba85be60d7db2c70ee09e19bd1f50744b35978f tomcat-util-scan-9.0.122.jar diff --git a/auto/modules/php b/auto/modules/php index 0e63179d7..c4094561a 100644 --- a/auto/modules/php +++ b/auto/modules/php @@ -211,31 +211,6 @@ else fi -nxt_feature="PHP TrueAsync API (zend_async_event_t)" -nxt_feature_name="" -nxt_feature_run=no -nxt_feature_incs="${NXT_PHP_INCLUDE}" -nxt_feature_libs="${NXT_PHP_LIB} ${NXT_PHP_LDFLAGS}" -nxt_feature_test=" - #include - - int main(void) { - zend_async_event_t *e = NULL; - (void) e; - return 0; - }" - -. auto/feature - -if [ $nxt_found = yes ]; then - NXT_PHP_TRUEASYNC=1 - $echo " + PHP TrueAsync: enabled" -else - NXT_PHP_TRUEASYNC=0 - $echo " + PHP TrueAsync: not available (standard mode only)" -fi - - nxt_feature="PHP sapi_module_struct pre_request_init field" nxt_feature_name="" nxt_feature_run=no @@ -269,7 +244,352 @@ if grep ^$NXT_PHP_MODULE: $NXT_MAKEFILE 2>&1 > /dev/null; then fi +NXT_PHP_MOUNTS_HEADER=nxt_${NXT_PHP_MODULE}_mounts.h + +# A PHP built with the --with-system-tzdata patch -- Debian, Ubuntu and Fedora +# packages all carry it -- reads the timezone database from the filesystem +# lazily, at request time, that is, after Unit has already chroot()ed into +# "isolation.rootfs". When the database is not there such a build does not +# raise an error, it dereferences NULL: the application worker dies with +# SIGSEGV on the first date() call. Bind mount the database so that PHP gets +# the same automount.language_deps treatment the other language modules get. +# +# PHP does not expose the path at runtime, so derive it here: +# +# --with-system-tzdata=DIR a build that spells the path out; trust it +# even if DIR is absent on the build host. +# --with-system-tzdata Debian, Ubuntu and Fedora pass the flag bare +# and rely on the patch's built-in default, +# /usr/share/zoneinfo: probe for it, and fall +# back to that default when the build host +# has no database at all. +# no such option A build using timelib's bundled database (the +# official php:*-cli images, Alpine's packages); +# it never opens the directory, so mount nothing. +# options unavailable php-config too old to report them: probe, since +# a spurious mount beats a SIGSEGV. +# +# A php-config that does not know --configure-options prints its usage to +# *stdout* and exits nonzero, so the query has to be judged by exit status; +# taking the output alone would read the usage text as a real answer and skip +# the probe. + +# A PHP configured with no options at all reports an empty list and exits +# successfully: that is a bundled-timezonedb build, not an unknown one. Only +# a failed query (a php-config without --configure-options) leaves the +# question open, and that is the one case worth probing the host for. +nxt_php_configure_options_known=yes + +if NXT_PHP_CONFIGURE_OPTIONS=`$NXT_PHP_CONFIG --configure-options 2>/dev/null` +then + NXT_PHP_CONFIGURE_OPTIONS=`printf '%s' "$NXT_PHP_CONFIGURE_OPTIONS" \ + | tr '\n' ' '` +else + NXT_PHP_CONFIGURE_OPTIONS= + nxt_php_configure_options_known=no +fi + +NXT_PHP_TZDATA= +nxt_php_tzprobe=no +nxt_php_tzdata_default=no +nxt_php_tzdata_relative= +nxt_php_tzdata_guessed= + +# Some builds hand back $ac_configure_args verbatim, with each argument +# shell-quoted, and a quoted --with-system-tzdata=DIR may carry a space. +# Split the list into one argument per line the way a shell would -- on +# unquoted whitespace only, quotes removed -- so a quoted argument survives +# whole instead of matching no arm below or being cut at its first space. +# Autoconf quotes with single quotes alone, so a double quote is a pathname +# character here and not quoting: reading it as quoting would swallow the +# rest of the line into one argument and drop the quote from the path. + +nxt_php_configure_args=`printf '%s' "$NXT_PHP_CONFIGURE_OPTIONS" \ + | awk ' + { + n = length($0); q = ""; tok = ""; has = 0; + for (i = 1; i <= n; i++) { + c = substr($0, i, 1); + if (q != "") { + if (c == q) { q = ""; } else { tok = tok c; } + } else if (c == "\047") { + q = c; has = 1; + } else if (c == " " || c == "\t") { + if (has || tok != "") { print tok; } + tok = ""; has = 0; + } else { + tok = tok c; + } + } + if (has || tok != "") { print tok; } + }'` + +# The stock php-config strips the quotes from $configure_options before it +# echoes them (scripts/php-config.in: sed -e "s#'##g"), so a directory with +# whitespace in it arrives already split into words. Recover it: a word +# that starts an absolute tzdata path absorbs the words after it up to the +# next option (-x) or Autoconf variable (NAME=value). It deliberately +# absorbs the positional system type too: no pattern can tell a system +# alias from a path component, since config.sub takes shortened aliases +# ("x86_64-linux") and single words as readily as the cpu-vendor-os +# triple, and a directory name may hold a hyphen just as happily. So this +# pass only has to produce a candidate the true path is a word prefix of; +# the arm below settles which prefix against the build host's filesystem. +# Repeated whitespace is already collapsed by php-config's unquoted echo, +# so it cannot be recovered. + +nxt_php_configure_args=`printf '%s\n' "$nxt_php_configure_args" \ + | awk ' + function flush() { if (path != "") { print path; path = ""; } } + /^--?with-system-tzdata=\// { + flush(); path = $0; next; + } + path != "" && !/^-/ && !/^[A-Za-z_][A-Za-z0-9_]*=/ { + path = path " " $0; next; + } + { flush(); print; } + END { flush(); }'` + +# Walk the arguments in order: Autoconf keeps the last occurrence of a +# repeated option, so a bare --with-system-tzdata appended after an "=no" or +# "=DIR" form is what PHP's ext/date/config.m4 actually saw. It reads the +# value the way Autoconf hands it over: "no" (or --without-system-tzdata) +# disables the patch, "yes" enables it with the built-in default, and only an +# absolute path overrides that default. Every arm therefore clears what the +# others set -- the relative note included, or a later absolute path would be +# mounted under an earlier relative value's "no mount can be derived" notice. + +nxt_php_ifs_saved=$IFS +IFS=' +' +set -f +for nxt_php_opt in $nxt_php_configure_args +do + # Autoconf accepts the single-dash spelling of every option too. + case "$nxt_php_opt" in + -with-* | -without-*) + nxt_php_opt="-$nxt_php_opt" + ;; + esac + + case "$nxt_php_opt" in + --with-system-tzdata | --with-system-tzdata=yes) + NXT_PHP_TZDATA=; nxt_php_tzprobe=yes + nxt_php_tzdata_relative=; nxt_php_tzdata_guessed= + ;; + --with-system-tzdata=no | --without-system-tzdata) + NXT_PHP_TZDATA=; nxt_php_tzprobe=no + nxt_php_tzdata_relative=; nxt_php_tzdata_guessed= + ;; + --with-system-tzdata=/*) + nxt_php_tzprobe=no + nxt_php_tzdata_relative=; nxt_php_tzdata_guessed= + NXT_PHP_TZDATA=`printf '%s' "$nxt_php_opt" \ + | sed 's/^--with-system-tzdata=//'` + + # A candidate carrying whitespace was reassembled from words, + # and only the filesystem knows where the directory ends: take + # the longest word prefix that is one. If no prefix is, keep + # the candidate less a trailing system type and say it was + # reconstructed -- the cross-host case, where the probe loop + # already trusts the build host to look like the rootfs. + case "$NXT_PHP_TZDATA" in + *\ *) + nxt_php_tzcand=$NXT_PHP_TZDATA + + while : + do + if [ -d "$nxt_php_tzcand" ]; then + NXT_PHP_TZDATA=$nxt_php_tzcand + break + fi + + case "$nxt_php_tzcand" in + *\ *) + nxt_php_tzcand=${nxt_php_tzcand% *} + ;; + *) + # Nothing exists. The absorbed tail may + # still be a system type, and dropping a + # trailing alias-shaped word ([a-z0-9_.] + # and at least one hyphen) is the best + # reading left for the message and for a + # rootfs that has what this host lacks. + nxt_php_tzlast=${NXT_PHP_TZDATA##* } + + case "$nxt_php_tzlast" in + *-*) + case "$nxt_php_tzlast" in + *[!a-z0-9_.-]*) ;; + *) + NXT_PHP_TZDATA=${NXT_PHP_TZDATA% *} + ;; + esac + ;; + esac + + nxt_php_tzdata_guessed=$NXT_PHP_TZDATA + break + ;; + esac + done + ;; + esac + ;; + --with-system-tzdata=*) + # A relative value is embedded verbatim and resolved against the + # worker's working directory at run time, so no host directory + # can be derived for a mount; say so instead of mounting an + # unrelated one, and leave the database to the rootfs author. + NXT_PHP_TZDATA=; nxt_php_tzprobe=no + nxt_php_tzdata_guessed= + nxt_php_tzdata_relative=`printf '%s' "$nxt_php_opt" \ + | sed 's/^--with-system-tzdata=//'` + ;; + esac +done +set +f +IFS=$nxt_php_ifs_saved + +if [ $nxt_php_configure_options_known = no ]; then + nxt_php_tzprobe=yes +fi + +if [ $nxt_php_tzprobe = yes ]; then + for nxt_tzdir in /usr/share/zoneinfo /usr/lib/zoneinfo \ + /usr/share/lib/zoneinfo /etc/zoneinfo + do + if [ -f "$nxt_tzdir/UTC" ]; then + NXT_PHP_TZDATA=$nxt_tzdir + break + fi + done + + # When the option itself asked for the probe, PHP does read a system + # database, and the patch compiles /usr/share/zoneinfo in as its + # default -- Debian's and Fedora's php binaries both carry the string. + # A build host without any database (a minimal build chroot, a + # sysroot) must not turn that into "bundled, no mount": emit the + # default, which the worker skips with a warning if the running host + # lacks it too. A probe that only php-config's silence asked for has + # no such certainty, so it still mounts nothing when it finds nothing. + if [ ".$NXT_PHP_TZDATA" = "." ] \ + && [ $nxt_php_configure_options_known = yes ] + then + NXT_PHP_TZDATA=/usr/share/zoneinfo + nxt_php_tzdata_default=yes + fi +fi + +# The path reaches the header as data, never as shell text: escape +# backslash, double quote and -- against -Wtrigraphs, which -Wall raises +# and the default -Werror makes fatal -- the question mark for the C string +# literal, and write the one line that carries it with printf, because an +# unquoted heredoc would re-read "$", "`" and "\" in the path. A control +# character has no representation here -- a newline would break the +# header's line structure as well -- so refuse those. +# +# A double quote and a backslash are refused too, for now: the C literal +# could carry them escaped, but the discovery process hands the mount +# table to the main process as JSON built with a plain %s +# (nxt_discovery_modules(), src/nxt_application.c), where either byte +# breaks the parse and takes every language module down with it. Lift +# this once that serializer escapes its strings. A single quote in the +# path is caught by the same test: the stock php-config's quote strip +# turns Autoconf's '\'' into a bare backslash before it gets here. +nxt_php_tzdata_refused=no + +if [ ".$nxt_php_tzdata_relative" != "." ]; then + printf '%s\n' " + PHP timezone database: relative path \"$nxt_php_tzdata_relative\"" \ + " is resolved against the worker's working directory; no" \ + " mount can be derived, provide it inside the rootfs" + nxt_php_tzdata_refused=yes +fi + +if [ $nxt_php_tzdata_default = yes ]; then + printf '%s\n' " + PHP timezone database: none found on this host;" \ + " mounting the patch's default \"$NXT_PHP_TZDATA\"" +fi + +if [ ".$nxt_php_tzdata_guessed" != "." ] \ + && [ ".$NXT_PHP_TZDATA" = ".$nxt_php_tzdata_guessed" ] +then + printf '%s\n' " + PHP timezone database path \"$NXT_PHP_TZDATA\" was" \ + " reconstructed from a whitespace-split configure option" \ + " and is not a directory here; check it before relying" \ + " on the mount" +fi + +nxt_php_tzdata_printable=$(printf '%s' "$NXT_PHP_TZDATA" \ + | tr -d '\001-\037\177') + +if [ "x$nxt_php_tzdata_printable" != "x$NXT_PHP_TZDATA" ]; then + printf '%s\n' " + PHP timezone database path \"$nxt_php_tzdata_printable\"" \ + " contains control characters that cannot be embedded;" \ + " no mount" + NXT_PHP_TZDATA= + nxt_php_tzdata_refused=yes +fi + +case "$NXT_PHP_TZDATA" in + *\"* | *\\*) + printf '%s\n' " + PHP timezone database path \"$NXT_PHP_TZDATA\"" \ + " contains a double quote or a backslash, which the" \ + " module discovery message cannot carry; no mount" + NXT_PHP_TZDATA= + nxt_php_tzdata_refused=yes + ;; +esac + +if [ ".$NXT_PHP_TZDATA" != "." ]; then + +nxt_php_tzdata_c=$(printf '%s' "$NXT_PHP_TZDATA" | sed 's/[\\"?]/\\&/g') + +cat << 'END' > $NXT_BUILD_DIR/include/$NXT_PHP_MOUNTS_HEADER +#ifndef _NXT_PHP_MOUNTS_H_INCLUDED_ +#define _NXT_PHP_MOUNTS_H_INCLUDED_ + + +static const nxt_fs_mount_t nxt_php_mounts[] = { +END + +printf ' {(u_char *) "%s", (u_char *) "%s",\n' \ + "$nxt_php_tzdata_c" "$nxt_php_tzdata_c" \ + >> $NXT_BUILD_DIR/include/$NXT_PHP_MOUNTS_HEADER + +cat << 'END' >> $NXT_BUILD_DIR/include/$NXT_PHP_MOUNTS_HEADER + NXT_FS_BIND, (u_char *) "bind", 0, NULL, 1, 1}, +}; + +#define NXT_PHP_MOUNTS nxt_php_mounts +#define NXT_PHP_NMOUNTS nxt_nitems(nxt_php_mounts) + + +#endif /* _NXT_PHP_MOUNTS_H_INCLUDED_ */ +END + +else + +cat << 'END' > $NXT_BUILD_DIR/include/$NXT_PHP_MOUNTS_HEADER +#ifndef _NXT_PHP_MOUNTS_H_INCLUDED_ +#define _NXT_PHP_MOUNTS_H_INCLUDED_ + + +#define NXT_PHP_MOUNTS NULL +#define NXT_PHP_NMOUNTS 0 + + +#endif /* _NXT_PHP_MOUNTS_H_INCLUDED_ */ +END + +fi + + $echo " + PHP module: ${NXT_PHP_MODULE}.unit.so" +if [ $nxt_php_tzdata_refused = no ]; then + printf '%s\n' " + PHP timezone database:" \ + " ${NXT_PHP_TZDATA:-bundled, no mount required}" +fi . auto/cc/deps @@ -297,8 +617,8 @@ $NXT_BUILD_DIR/$nxt_obj: $nxt_src $NXT_VERSION_H \$(PP_CC) \$@ \$(v)\$(CC) -c \$(CFLAGS) $NXT_PHP_ADDITIONAL_FLAGS \$(NXT_INCS) \\ $NXT_PHP_INCLUDE -DNXT_ZEND_SIGNAL_STARTUP=$NXT_ZEND_SIGNAL_STARTUP \\ - -DNXT_PHP_TRUEASYNC=$NXT_PHP_TRUEASYNC \\ -DNXT_PHP_PRE_REQUEST_INIT=$NXT_PHP_PRE_REQUEST_INIT \\ + -DNXT_PHP_MOUNTS_H=\"$NXT_PHP_MOUNTS_HEADER\" \\ $nxt_dep_flags \\ -o $NXT_BUILD_DIR/$nxt_obj $nxt_src $nxt_dep_post diff --git a/auto/options b/auto/options index f19948588..78f8f0d5b 100644 --- a/auto/options +++ b/auto/options @@ -11,6 +11,8 @@ NXT_LD_OPT= NXT_DEBUG=NO +NXT_HARDENING=off + NXT_INET6=YES NXT_UNIX_DOMAIN=YES @@ -22,9 +24,6 @@ NXT_TRY_PCRE2=YES NXT_TLS=NO NXT_OPENSSL=NO -NXT_GNUTLS=NO -NXT_CYASSL=NO -NXT_POLARSSL=NO NXT_ZLIB=NO NXT_ZSTD=NO @@ -105,6 +104,8 @@ do --debug) NXT_DEBUG=YES ;; + --hardening=*) NXT_HARDENING="$value" ;; + --no-ipv6) NXT_INET6=NO ;; --no-unix-sockets) NXT_UNIX_DOMAIN=NO ;; @@ -112,9 +113,6 @@ do --no-pcre2) NXT_TRY_PCRE2=NO ;; --openssl) NXT_OPENSSL=YES ;; - --gnutls) NXT_GNUTLS=YES ;; - --cyassl) NXT_CYASSL=YES ;; - --polarssl) NXT_POLARSSL=YES ;; --zlib) NXT_ZLIB=YES ;; --zstd) NXT_ZSTD=YES ;; diff --git a/auto/otel b/auto/otel index 4d059a285..9a75e7dfc 100644 --- a/auto/otel +++ b/auto/otel @@ -19,44 +19,25 @@ if [ $NXT_OTEL = YES ]; then fi $echo "found" + # The src/otel crate links no TLS stack, so it needs no OpenSSL. + # On Linux, its other libraries come from NXT_LIBM, NXT_LIBS and + # the compiler driver (libc, libgcc_s). + case "$NXT_SYSTEM" in Darwin) NXT_OTEL_LIBS="-framework CoreFoundation -framework Security -framework SystemConfiguration" + ;; + *) + NXT_OTEL_LIBS= + ;; + esac - cat << END >> $NXT_AUTO_CONFIG_H + cat << END >> $NXT_AUTO_CONFIG_H #ifndef NXT_HAVE_OTEL #define NXT_HAVE_OTEL 1 #endif END - ;; - *) - - $echo -n " - " - - nxt_feature="OpenSSL library" - nxt_feature_name=NXT_HAVE_OTEL - nxt_feature_run=yes - nxt_feature_incs= - nxt_feature_libs="-lssl -lcrypto" - nxt_feature_test="#include - - int main(void) { - SSL_library_init(); - return 0; - }" - . auto/feature - - if [ ! $nxt_found = yes ]; then - $echo - $echo $0: error: OpenTelemetry support requires OpenSSL. - $echo - exit 1; - fi - - NXT_OTEL_LIBS="-lssl -lcrypto" - ;; - esac fi diff --git a/auto/shmem b/auto/shmem index cc85d1408..421943be3 100644 --- a/auto/shmem +++ b/auto/shmem @@ -109,12 +109,20 @@ fi # Linux +# MFD_CLOEXEC and the memfd_create() declaration live in under +# _GNU_SOURCE on glibc >= 2.27 and on musl >= 1.1.20. Probing +# instead tests for the kernel UAPI headers package rather than for the +# syscall: Alpine ships that header in "linux-headers", not in musl-dev, so a +# plain "apk add build-base" build used to fall back to shm_open(), which then +# fails inside an isolation.rootfs because /dev/shm does not exist there. + nxt_feature="memfd_create()" nxt_feature_name=NXT_HAVE_MEMFD_CREATE nxt_feature_run=yes nxt_feature_incs= nxt_feature_libs= -nxt_feature_test="#include +nxt_feature_test="#define _GNU_SOURCE + #include #include #include @@ -130,6 +138,33 @@ nxt_feature_test="#include . auto/feature +if [ $nxt_found = no ]; then + + # Older libc: MFD_CLOEXEC is only in the kernel UAPI header. + + nxt_feature="memfd_create() in " + nxt_feature_test="#include + #include + #include + + int main(void) { + static char name[] = \"/unit.configure\"; + + int fd = syscall(SYS_memfd_create, name, + MFD_CLOEXEC); + if (fd == -1) + return 1; + + return 0; + }" + . auto/feature + + if [ $nxt_found = yes ]; then + nxt_have=NXT_HAVE_LINUX_MEMFD_H . auto/have + fi +fi + + if [ "$nxt_shm_open_found$nxt_found" = nono ]; then $echo $echo $0: error: no shared memory implementation found. diff --git a/auto/sources b/auto/sources index 117910888..d94bacdb6 100644 --- a/auto/sources +++ b/auto/sources @@ -23,12 +23,9 @@ NXT_LIB_SRCS=" \ src/nxt_port_memory.c \ src/nxt_port_rpc.c \ src/nxt_port.c \ - src/nxt_dyld.c \ src/nxt_random.c \ - src/nxt_queue.c \ src/nxt_rbtree.c \ src/nxt_mp.c \ - src/nxt_mem_zone.c \ src/nxt_string.c \ src/nxt_utf8.c \ src/nxt_parse.c \ @@ -66,9 +63,7 @@ NXT_LIB_SRCS=" \ src/nxt_conn_read.c \ src/nxt_conn_write.c \ src/nxt_conn_close.c \ - src/nxt_event_conn_job_sendfile.c \ src/nxt_conn_proxy.c \ - src/nxt_job.c \ src/nxt_sockaddr.c \ src/nxt_listen_socket.c \ src/nxt_upstream.c \ @@ -114,12 +109,20 @@ NXT_LIB_SRCS=" \ NXT_LIB_UNIT_SRCS="src/nxt_unit.c" +# The C test suite needs the NXT_TESTS hooks: fault-injection branches and +# counters compiled into the core and into libunit. NXT_TESTS is never a +# global macro. "make tests" compiles every core and libunit source a second +# time with -DNXT_TESTS=1 into these two archives, and only the test programs +# link them. unitd, libnxt.a, libunit.a and the language modules are built +# from the plain objects and never see the macro. +# +NXT_LIB_TEST_STATIC="libnxt-tests.a" +NXT_LIB_UNIT_TEST_STATIC="libunit-tests.a" + + NXT_LIB_TLS_DEPS="src/nxt_tls.h" NXT_LIB_TLS_SRCS="src/nxt_cert.c" NXT_LIB_OPENSSL_SRCS="src/nxt_openssl.c" -NXT_LIB_GNUTLS_SRCS="src/nxt_gnutls.c" -NXT_LIB_CYASSL_SRCS="src/nxt_cyassl.c" -NXT_LIB_POLARSSL_SRCS="src/nxt_polarssl.c" NXT_LIB_PCRE_SRCS="src/nxt_pcre.c" NXT_LIB_PCRE2_SRCS="src/nxt_pcre2.c" @@ -153,6 +156,8 @@ NXT_TEST_BUILD_SRCS="src/nxt_test_build.c" NXT_TEST_DEPS="src/test/nxt_tests.h \ src/test/nxt_rbtree1.h \ + src/nxt_checked.h \ + src/nxt_span.h \ " NXT_TEST_SRCS=" \ @@ -162,21 +167,66 @@ NXT_TEST_SRCS=" \ src/test/nxt_term_parse_test.c \ src/test/nxt_msec_diff_test.c \ src/test/nxt_mp_test.c \ - src/test/nxt_mem_zone_test.c \ src/test/nxt_lvlhsh_test.c \ src/test/nxt_gmtime_test.c \ src/test/nxt_sprintf_test.c \ src/test/nxt_malloc_test.c \ src/test/nxt_utf8_test.c \ + src/test/nxt_utf8_sanitize_test.c \ src/test/nxt_rbtree1_test.c \ src/test/nxt_http_parse_test.c \ src/test/nxt_strverscmp_test.c \ src/test/nxt_base64_test.c \ src/test/nxt_string_test.c \ + src/test/nxt_buf_test.c \ src/test/nxt_http_chunk_parse_test.c \ + src/test/nxt_http_validate_host_test.c \ + src/test/nxt_http_request_body_alloc_test.c \ src/test/nxt_conf_parse_test.c \ src/test/nxt_port_fail_test.c \ + src/test/nxt_fd_event_change_test.c \ + src/test/nxt_port_use_unless_zero_test.c \ src/test/nxt_port_mmap_range_test.c \ + src/test/nxt_port_mmap_read_test.c \ + src/test/nxt_port_mmaps_max_test.c \ + src/test/nxt_port_ready_test.c \ + src/test/nxt_conn_close_idle_test.c \ + src/test/nxt_listen_event_test.c \ + src/test/nxt_conn_io_accept_test.c \ + src/test/nxt_runtime_idle_close_test.c \ + src/test/nxt_router_new_port_test.c \ + src/test/nxt_router_start_fail_test.c \ + src/test/nxt_router_start_fail_soak_test.c \ + src/test/nxt_router_start_proto_gone_test.c \ + src/test/nxt_router_proto_wedge_test.c \ + src/test/nxt_router_proto_death_test.c \ + src/test/nxt_router_start_timeout_test.c \ + src/test/nxt_router_app_timeout_test.c \ + src/test/nxt_router_remove_pid_soak_test.c \ + src/test/nxt_router_detached_test.c \ + src/test/nxt_router_sender_test.c \ + src/test/nxt_router_stale_joint_test.c \ + src/test/nxt_router_websocket_test.c \ + src/test/nxt_main_start_process_reply_test.c \ + src/test/nxt_main_file_store_test.c \ + src/test/nxt_main_store_test.c \ + src/test/nxt_proto_creating_wedge_test.c \ + src/test/nxt_main_whoami_test.c \ + src/test/nxt_main_remove_child_pid_test.c \ + src/test/nxt_port_change_file_test.c \ + src/test/nxt_port_ctrunc_test.c \ + src/test/nxt_port_fd_test.c \ + src/test/nxt_port_rpc_fd_test.c \ + src/test/nxt_port_queued_fd_test.c \ + src/test/nxt_controller_peer_test.c \ + src/test/nxt_checked_test.c \ + src/test/nxt_conn_close_test.c \ + src/test/nxt_router_prepare_msg_test.c \ + src/test/nxt_router_response_parse_test.c \ + src/test/nxt_port_frag_test.c \ + src/test/nxt_port_release_test.c \ + src/test/nxt_nncq_bound_test.c \ + src/test/nxt_cpu_limit_test.c \ " @@ -189,13 +239,9 @@ if [ "$NXT_HAVE_CGROUP" = "YES" ]; then fi -NXT_LIB_UTF8_FILE_NAME_TEST_SRCS=" \ - src/test/nxt_utf8_file_name_test.c \ -" - - if [ $NXT_HAVE_ROOTFS = YES ]; then NXT_LIB_SRCS="$NXT_LIB_SRCS src/nxt_fs_mount.c" + NXT_TEST_SRCS="$NXT_TEST_SRCS src/test/nxt_isolation_mount_dst_test.c" fi @@ -207,21 +253,7 @@ fi if [ $NXT_OPENSSL = YES ]; then NXT_LIB_SRCS="$NXT_LIB_SRCS $NXT_LIB_OPENSSL_SRCS" -fi - - -if [ $NXT_GNUTLS = YES ]; then - NXT_LIB_SRCS="$NXT_LIB_SRCS $NXT_LIB_GNUTLS_SRCS" -fi - - -if [ $NXT_CYASSL = YES ]; then - NXT_LIB_SRCS="$NXT_LIB_SRCS $NXT_LIB_CYASSL_SRCS" -fi - - -if [ $NXT_POLARSSL = YES ]; then - NXT_LIB_SRCS="$NXT_LIB_SRCS $NXT_LIB_POLARSSL_SRCS" + NXT_TEST_SRCS="$NXT_TEST_SRCS src/test/nxt_openssl_server_init_test.c" fi @@ -246,6 +278,8 @@ if [ "$NXT_REGEX" = "YES" ]; then else NXT_LIB_SRCS="$NXT_LIB_SRCS $NXT_LIB_PCRE_SRCS" fi + + NXT_TEST_SRCS="$NXT_TEST_SRCS src/test/nxt_regex_test.c" fi if [ "$NXT_HAVE_EPOLL" = "YES" -o "$NXT_TEST_BUILD_EPOLL" = "YES" ]; then @@ -331,17 +365,23 @@ if [ "$NXT_TEST_BUILD" = "YES" ]; then fi -if [ $NXT_TESTS = YES ]; then - nxt_have=NXT_TESTS . auto/have +# libunit's message-feeding hook is compiled in for the fuzz targets too. +if [ -n "$NXT_FUZZ" ]; then + nxt_have=NXT_FUZZ_BUILD . auto/have fi NXT_FUZZ_SRCS=" \ fuzzing/nxt_basic_fuzz.c \ + fuzzing/nxt_http_chunk_fuzz.c \ fuzzing/nxt_http_controller_fuzz.c \ fuzzing/nxt_http_h1p_fuzz.c \ fuzzing/nxt_http_h1p_peer_fuzz.c \ + fuzzing/nxt_http_h1p_peer_response_fuzz.c \ + fuzzing/nxt_http_ws_utf8_fuzz.c \ fuzzing/nxt_json_fuzz.c \ + fuzzing/nxt_router_app_response_fuzz.c \ + fuzzing/nxt_unit_msg_fuzz.c \ " diff --git a/auto/ssltls b/auto/ssltls index b275e6140..ac829176f 100644 --- a/auto/ssltls +++ b/auto/ssltls @@ -5,13 +5,6 @@ NXT_OPENSSL_CFLAGS= NXT_OPENSSL_LIBS= -NXT_GNUTLS_CFLAGS= -NXT_GNUTLS_LIBS= -NXT_OPENSSL_LIBS= -NXT_CYASSL_CFLAGS= -NXT_CYASSL_LIBS= -NXT_POLARSSL_CFLAGS= -NXT_POLARSSL_LIBS= if [ $NXT_OPENSSL = YES ]; then @@ -22,9 +15,11 @@ if [ $NXT_OPENSSL = YES ]; then nxt_feature_incs= nxt_feature_libs="-lssl -lcrypto" nxt_feature_test="#include + #include int main(void) { - SSL_library_init(); + SSL_CTX_free(NULL); + (void) ERR_get_error(); return 0; }" . auto/feature @@ -34,6 +29,43 @@ if [ $NXT_OPENSSL = YES ]; then NXT_TLS=YES NXT_OPENSSL_LIBS="$nxt_feature_libs" + # The probe above uses only symbols that every OpenSSL provides and + # none of them deprecates, so it compiles against a too-old library + # as well; that is deliberate, so that the floor probe below is the + # one to report a too-old library by name instead of configure + # bailing out with a misleading "no OpenSSL library found". The + # floor is not a build fact but a support decision: it stays + # at 1.1.1 rather than 3.x because RHEL 8 still ships a + # vendor-patched 1.1.1 that Red Hat maintains into 2029, and from + # 2026-09 it is the last platform in the matrix still receiving + # those patches: Amazon Linux 2's ended 2026-06-30 and Debian + # 11's LTS ends 2026-08-31. Both stay in FreeUnit's own grace + # window; the OS matrix and that window both live in + # pkg/eol.json (see EOL.md). + # Checking it here fails configure with a clear message instead of + # leaving a too-old library to surface as an implicit declaration + # deep in the build. + nxt_feature="OpenSSL 1.1.1 or later" + nxt_feature_name= + nxt_feature_run=no + nxt_feature_test="#include + + #if OPENSSL_VERSION_NUMBER < 0x1010100fL + #error OpenSSL 1.1.1 or later is required. + #endif + + int main(void) { + return 0; + }" + . auto/feature + + if [ $nxt_found = no ]; then + $echo + $echo $0: error: OpenSSL 1.1.1 or later is required. + $echo + exit 1; + fi + nxt_feature="OpenSSL version" nxt_feature_name=NXT_HAVE_OPENSSL_VERSION nxt_feature_run=value @@ -85,132 +117,3 @@ if [ $NXT_OPENSSL = YES ]; then }" . auto/feature fi - - -if [ $NXT_GNUTLS = YES ]; then - - if /bin/sh -c "(pkg-config gnutls --exists)" >> $NXT_AUTOCONF_ERR 2>&1; - then - NXT_GNUTLS_CFLAGS=`pkg-config gnutls --cflags` - NXT_GNUTLS_LIBS=`pkg-config gnutls --libs` - - nxt_feature="GnuTLS library" - nxt_feature_name=NXT_HAVE_GNUTLS - nxt_feature_run=yes - nxt_feature_incs=$NXT_GNUTLS_CFLAGS - nxt_feature_libs=$NXT_GNUTLS_LIBS - nxt_feature_test="#include - - int main(void) { - gnutls_global_init(); - gnutls_global_deinit(); - return 0; - }" - . auto/feature - - - if [ $nxt_found = yes ]; then - NXT_TLS=YES - - $echo " + GnuTLS version: `pkg-config gnutls --modversion`" - - - nxt_feature="gnutls_transport_set_vec_push_function" - nxt_feature_name=NXT_HAVE_GNUTLS_VEC_PUSH - nxt_feature_run=no - nxt_feature_incs=$NXT_GNUTLS_CFLAGS - nxt_feature_libs=$NXT_GNUTLS_LIBS - nxt_feature_test="#include - - int main(void) { - gnutls_transport_set_vec_push_function(NULL, NULL); - return 0; - }" - . auto/feature - - - nxt_feature="gnutls_global_set_time_function" - nxt_feature_name=NXT_HAVE_GNUTLS_SET_TIME - nxt_feature_run=no - nxt_feature_incs=$NXT_GNUTLS_CFLAGS - nxt_feature_libs=$NXT_GNUTLS_LIBS - nxt_feature_test="#include - - int main(void) { - gnutls_global_set_time_function(NULL); - return 0; - }" - . auto/feature - - else - $echo - $echo $0: error: no GnuTLS library found. - $echo - exit 1; - fi - fi -fi - - -if [ $NXT_CYASSL = YES ]; then - - nxt_feature="CyaSSL library" - nxt_feature_name=NXT_HAVE_CYASSL - nxt_feature_run=yes - nxt_feature_incs= - nxt_feature_libs="-lcyassl" - nxt_feature_test="#include - - int main(void) { - CyaSSL_Init(); - CyaSSL_Cleanup(); - return 0; - }" - . auto/feature - - - if [ $nxt_found = yes ]; then - NXT_TLS=YES - NXT_CYASSL_CFLAGS="$nxt_feature_incs" - NXT_CYASSL_LIBS="$nxt_feature_libs" - - else - $echo - $echo $0: error: no CyaSSL library found. - $echo - exit 1; - fi -fi - - -if [ $NXT_POLARSSL = YES ]; then - - nxt_feature="PolarSSL library" - nxt_feature_name=NXT_HAVE_POLARSSL - nxt_feature_run=yes - nxt_feature_incs= - nxt_feature_libs="-lpolarssl" - nxt_feature_test="#include - - int main(void) { - ssl_context ssl; - memset(&ssl, '\0', sizeof(ssl)); - ssl_init(&ssl); - ssl_free(&ssl); - return 0; - }" - . auto/feature - - - if [ $nxt_found = yes ]; then - NXT_TLS=YES - NXT_POLARSSL_CFLAGS="$nxt_feature_incs" - NXT_POLARSSL_LIBS="$nxt_feature_libs" - - else - $echo - $echo $0: error: no PolarSSL library found. - $echo - exit 1; - fi -fi diff --git a/auto/summary b/auto/summary index 4a0888a62..2b05d788b 100644 --- a/auto/summary +++ b/auto/summary @@ -39,6 +39,7 @@ Unit configuration summary: cgroupv2: .................. $NXT_HAVE_CGROUP debug logging: ............. $NXT_DEBUG + hardening: ................. $NXT_HARDENING fuzz engine: ............... "$NXT_FUZZ" diff --git a/configure b/configure index aba56e8ae..e844a9006 100755 --- a/configure +++ b/configure @@ -74,6 +74,7 @@ mkdir -p $NXT_BUILD_DIR/var/log/unit > $NXT_AUTO_CONFIG_H . auto/cc/test +. auto/cc/hardening cat << END >> $NXT_AUTO_CONFIG_H @@ -168,13 +169,11 @@ END # LOOK -NXT_LIB_AUX_CFLAGS="$NXT_OPENSSL_CFLAGS $NXT_GNUTLS_CFLAGS \\ - $NXT_CYASSL_CFLAGS $NXT_POLARSSL_CFLAGS \\ +NXT_LIB_AUX_CFLAGS="$NXT_OPENSSL_CFLAGS \\ $NXT_PCRE_CFLAGS $NXT_ZLIB_CFLAGS $NXT_ZSTD_CFLAGS \\ $NXT_BROTLI_CFLAGS" -NXT_LIB_AUX_LIBS="$NXT_OPENSSL_LIBS $NXT_GNUTLS_LIBS \\ - $NXT_CYASSL_LIBS $NXT_POLARSSL_LIBS \\ +NXT_LIB_AUX_LIBS="$NXT_OPENSSL_LIBS \\ $NXT_PCRE_LIB $NXT_ZLIB_LIBS $NXT_ZSTD_LIBS \\ $NXT_BROTLI_LIBS" diff --git a/docs/acme.md b/docs/acme.md new file mode 100644 index 000000000..c8078f83f --- /dev/null +++ b/docs/acme.md @@ -0,0 +1,214 @@ +# ACME certificates with an external client + +ACME is the protocol that certificate authorities use to issue and renew +certificates. Let's Encrypt is one of these authorities. FreeUnit does not +connect to a certificate authority. An external ACME client gets and renews +the certificate. Two examples are certbot and lego. A deploy hook then +uploads the new bundle through the control API. A bundle is one PEM body +that holds the server certificate, its chain, and the private key. + +Renewal needs no restart. `PUT /certificates/` on an existing name +replaces the bundle in place. When a listener names the bundle, FreeUnit +applies the configuration again. Connections that were already accepted +finish with the old certificate. New handshakes get the new certificate. + +This page gives the procedure for operators. + +## What the control API does + +- `PUT /certificates/` creates the bundle, or replaces it when the + name exists. The body is a PEM bundle. The first certificate must be the + server certificate. The chain certificates and the private key follow. +- Main writes the bundle to a temporary file in the certificate store and + renames it over the old file. A crash never leaves a partial bundle. +- The key must belong to the first certificate. The answer to a bundle + with the wrong key is `400 Invalid certificate.`. The old bundle stays in + use. +- A name that starts with `.` is reserved, and a name is at most 255 + bytes. The answer to any other name is `400 Invalid certificate name.`. +- The answer to a bundle over 1 MiB is `413 Certificate bundle is too + large.`. +- When the current configuration names the bundle, the answer is + `200 Certificate chain updated.`. The router has loaded the new bundle + at that point. Otherwise the answer is `200 Certificate chain uploaded.`. +- `500 Failed to store certificate.` means main could not write the file. + Nothing changed. +- `500 Certificate stored but not applied.` means the new bundle is on + disk, but the router refused the configuration with it. One cause is a + `conf_commands` option that does not accept the new key type. The router + keeps the old certificate in use. `GET /certificates/` already + shows the metadata of the new bundle. The next reconfiguration or restart + loads the new bundle. Upload a working bundle again, or correct the + configuration. +- A bundle with the same certificates as the stored one is not stored + again and causes no reconfiguration. The answer is the one a store + gives. After `500 Certificate stored but not applied.` or + `500 Failed to store certificate.`, the same bundle is stored and + applied again. The controller keeps this state only in memory: after a + restart of the controller, or after a change of a file under + `state/certs/` by hand, the same bundle can be skipped. Then upload a + different bundle first, or restart Unit. +- The request waits while another configuration change is in progress, as + `PUT /config` does. +- `GET /certificates//fingerprint` gives the SHA-256 fingerprint of + the server certificate, in the form `openssl x509 -fingerprint -sha256` + prints. A script that polls for a renewed file compares the two and + uploads only a new certificate. + +## HTTP-01 route on port 80 + +HTTP-01 is an ACME challenge type. The authority gets a token file over +plain HTTP from `http:///.well-known/acme-challenge/`. The +ACME client writes the token file into a directory. FreeUnit serves that +directory with `share`. The router runs as the `--user` user. The router +must be able to read the token files. It must also be able to enter each +directory above them. + +```json +{ + "listeners": { + "*:80": { "pass": "routes/http" }, + "*:443": { "pass": "routes/app", "tls": { "certificate": "example.org" } } + }, + "routes": { + "http": [ + { + "match": { "uri": "/.well-known/acme-challenge/*" }, + "action": { + "share": "/var/lib/unit-acme$uri", + "chroot": "/var/lib/unit-acme/.well-known/acme-challenge/", + "fallback": { "return": 404 } + } + }, + { "action": { "return": 301, "location": "https://$host$request_uri" } } + ], + "app": [ { "action": { "pass": "applications/app" } } ] + } +} +``` + +Leave out the `*:443` listener until the first bundle exists. FreeUnit +refuses a configuration where a listener names a missing bundle. + +## The deploy hook + +The hook uploads the renewed bundle under the same name every time. It +needs only `curl` and write access to the control socket. + +```sh +#!/bin/sh +# Deploy hook for certbot. lego uses it through the wrapper below. +# certbot sets RENEWED_LINEAGE, for example /etc/letsencrypt/live/example.org +# lego sets CERT_FULLCHAIN, CERT_KEY and UNIT_CERT_NAME through the wrapper +set -eu + +SOCK=${UNIT_CONTROL:-/var/run/control.freeunit.sock} +FULLCHAIN=${CERT_FULLCHAIN:-${RENEWED_LINEAGE:-}/fullchain.pem} +KEY=${CERT_KEY:-${RENEWED_LINEAGE:-}/privkey.pem} +NAME=${UNIT_CERT_NAME:-$(basename "${RENEWED_LINEAGE:?set RENEWED_LINEAGE or UNIT_CERT_NAME}")} + +cat "$FULLCHAIN" "$KEY" \ + | curl -fsS --unix-socket "$SOCK" -X PUT --data-binary @- \ + "http://localhost/certificates/$NAME" +``` + +Install it as `/usr/local/sbin/freeunit-deploy-hook`, mode 0755. + +The default socket is the socket of the FreeUnit Debian package. For a +different installation, set `UNIT_CONTROL` to its control socket: + +| Installation | Control socket | +|---|---| +| FreeUnit Debian package | `/var/run/control.freeunit.sock` | +| RPM package | `/var/run/unit/control.sock` | +| Docker image | `/var/run/control.unit.sock` | +| Source build | `/control.unit.sock`, or the `--control` option of `configure` | + +`unitd --help` shows the default socket of the binary under `--control`. +A service can set a different socket with `--control`. Look for this +option in the unit file of the service. + +`curl -f` returns an error for each answer of 400 or higher. The ACME +client then reports a failed upload. + +### certbot + +```sh +mkdir -p /etc/letsencrypt/renewal-hooks/deploy +ln -s /usr/local/sbin/freeunit-deploy-hook \ + /etc/letsencrypt/renewal-hooks/deploy/freeunit +certbot certonly --webroot -w /var/lib/unit-acme -d example.org -d www.example.org +RENEWED_LINEAGE=/etc/letsencrypt/live/example.org /usr/local/sbin/freeunit-deploy-hook +``` + +With the link, certbot runs the hook after each renewal. certbot sets +`RENEWED_LINEAGE` for the hook. The last line does the first upload. Add +the `*:443` listener after it. The `certbot.timer` unit of the distribution +runs `certbot renew` two times each day. + +### lego + +```sh +cat > /usr/local/sbin/lego-freeunit <<'EOF' +#!/bin/sh +CERT_FULLCHAIN=$LEGO_CERT_PATH CERT_KEY=$LEGO_CERT_KEY_PATH \ +UNIT_CERT_NAME=$LEGO_CERT_DOMAIN exec /usr/local/sbin/freeunit-deploy-hook +EOF +chmod 0755 /usr/local/sbin/lego-freeunit + +lego --email ops@example.org --accept-tos --path /var/lib/lego \ + --domains example.org --http --http.webroot /var/lib/unit-acme \ + run --run-hook /usr/local/sbin/lego-freeunit +``` + +lego sets `LEGO_CERT_PATH`, `LEGO_CERT_KEY_PATH` and `LEGO_CERT_DOMAIN` for +its hook. The wrapper maps them to the variables of the deploy hook. + +For a wildcard certificate, use `--dns ` instead of `--http`. +FreeUnit then needs no challenge route. + +### A renewal timer for lego + +lego has no timer. Use a systemd timer that runs one time each day. lego +renews a certificate only when it is close to expiry. + +```ini +# /etc/systemd/system/lego-renew.service +[Unit] +Description=Renew certificates with lego and deploy them to FreeUnit + +[Service] +Type=oneshot +ExecStart=/usr/bin/lego --email ops@example.org --accept-tos \ + --path /var/lib/lego --domains example.org \ + --http --http.webroot /var/lib/unit-acme \ + renew --no-random-sleep --renew-hook /usr/local/sbin/lego-freeunit +``` + +```ini +# /etc/systemd/system/lego-renew.timer +[Unit] +Description=Daily certificate renewal + +[Timer] +OnCalendar=daily +RandomizedDelaySec=1h +Persistent=true + +[Install] +WantedBy=timers.target +``` + +```sh +systemctl enable --now lego-renew.timer +``` + +To use cron instead, add this line: `17 3 * * * root /usr/bin/lego ... renew --no-random-sleep --renew-hook /usr/local/sbin/lego-freeunit`. + +## Notes + +- Do not manually edit the files in the state directory. The next + reconfiguration loads a bundle that you changed there, but + `GET /certificates` shows the old metadata until a restart. +- The hook must run as a user that can write to the control socket. Do + not give an application process access to the socket. diff --git a/docs/changes.xml b/docs/changes.xml index 3f0d2387b..043d4cce5 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -5,11 +5,1661 @@ + + + + +with "chunked_transform", a valid chunked request body got 400 when a read +ended right after the data of a chunk, before the CRLF after it. + + + + + +unitd did not load the stored configuration when the "version" file in the +state directory ended with a line end, as "echo 13700 > version" and most +editors write it. It logged "invalid version string" and started with no +configuration. A line end, a space or a tab after the number is now ignored. +Any other content is still an error. The bug had appeared in 1.26.0. + + + + + +"--otel" no longer needs OpenSSL. Before, configure stopped with +"OpenTelemetry support requires OpenSSL." when the OpenSSL headers were +missing, and it linked unitd with OpenSSL even without "--openssl". The OTel +exporter has linked no TLS stack since 1.36.0. macOS builds did not have the +problem. + + + + + +the worker of a wasm application crashed when the guest called the import +nxt_wasm_send_response, nxt_wasm_send_headers or nxt_wasm_response_end before +the first request. This could happen in the start function or "_initialize" +of the guest, in the malloc handler, or in the module_init hook. These +imports now trap when the worker has no request yet. The worker then does +not start, except in the hook, where the trap ends only the hook. + + + + + +the wasm module took any export with a handler name as a function. If that +export was not a function, such as a global or a memory, the worker crashed on +the first call of the handler. The module now refuses such a guest when the +worker starts. + + + + + +with "large_header_buffers" set to 256 or more, the limit did not work. The +router counted the large buffers of a request header in 8 bits, so the count +went back to 0 after 255. A request header with many fields could then take +memory until "header_read_timeout" ended it. The bug had appeared in 1.29.0. + + + + + + + + + + +FreeUnit updated to 1.37.1. + + + + + + + + + + +on Linux, the default number of router threads ("listen_threads") is now at +most the CPU limit of the cgroup v2 group of unitd and of its parent groups +("cpu.max"), rounded up to whole CPUs. Before, it was the number of CPUs that +unitd could run on, so a container limited to 2 CPUs on a 64-CPU host started +64 router threads. Now it starts 2. A limit of 1.5 CPUs also gives 2 +threads. An explicit "listen_threads" is not changed. With a private cgroup +namespace, only the limits inside the namespace are seen. The limit is read +once, when unitd starts. cgroup v1 is not read. + + + + + +after 2^32 RPC registrations the stream counter wrapped and handed out +stream 0, which callers read as a failure. The registration stayed. +Stream 0 is now skipped. + + + + + +a chunked request body that grew over "max_body_size" after the read that +carried the request header got no response. The router closed the connection +without a status line. Now it answers 413 and closes. A body that crossed the +limit in the same read as the header already got 413. A later read that fails +to allocate memory in the chunk parser, or to write the body to the temporary +file, now answers 500 and closes in the same way. The 400 for a malformed chunk +in a later read now clears keepalive and closes the connection, like the other +error paths. A short write of a body with a Content-Length to the temporary +file now answers 500 instead of closing silently. + + + + + +the router could take a message of an application response before an earlier +message of the same response. With a header over 1024 bytes and a first body +write of 16 to 1024 bytes, it parsed the body as the header, logged "response +buffer too small for fields count", and answered 503. With a body write over +1024 bytes and a later smaller one, the client got the two parts in the wrong +order, and nothing was logged. This needed two or more application processes +or threads. The bug had appeared in 1.19.0. The fix is in libunit: an +application of type "external", such as Go or Node.js, gets it when it is +built with the new libunit. + + + + + +the router did not answer "Expect: 100-continue". A client that sent it, as +curl and Guzzle do for large uploads, waited for its own timeout before it +sent the body. For curl, that timeout is 1 second. The router now sends +"100 Continue" before it waits for the body, also when a part of the body +came with the header. A request that the router refuses from its header gets +its final status with no 100. A proxied request no longer carries the Expect +field to the upstream. + + + + + +the PHP module read the target index of a request after the request was +released. libunit fills released request memory with 0xA5, so the module +called chdir(2) on every request to a "script" target. A request could also +run in the directory of another target. This happened with 166 or more +targets, and after fastcgi_finish_request() when the released memory already +held a request for another target. The bug had appeared in 1.18.0. + + + + + +flush() in a PHP application did not send the response header. If the +application had no output yet, or its output was in an output buffer, the +client got the header only at the end of the request, and headers_sent() +returned false after flush(). Now flush() sends the header. As in mod_php, +flush() does not empty the output buffers; ob_flush() does. The PHP module +has had no flush handler since 1.4. + + + + + +the router sent the header of a chunked HTTP/1.1 response without the empty +line that ends it, and sent that line with the first body bytes. If an +application sent the header and then waited before its body, the client could +not use the header until the body began. Now the header ends at once. A +complete response has the same bytes as before. $body_bytes_sent no longer +counts the empty line. + + + + + +with response compression on, the router kept one compressor stream for each +router thread. But it compresses an application response in several steps. +So two application responses on one thread used the same stream. A client +could get bytes of the response to another client, a body that was not valid, +or a body that stopped early. The router could also crash when it used a +stream that another response had freed. Now each response has its own stream, +and the router frees it when the response ends or stops early. + + + + + +the main process now writes the state files in a short-lived child +process, so a slow fsync(2) no longer delays the start of application +processes after a reconfiguration, or the exit of the main process. One +store runs at a time; a newer configuration waits for it, and only the +latest one waiting is stored. At exit, the main process waits for the +store to finish; when a newer configuration waits, it stops the running +store first. When the store fails, the main process logs an alert. + + + + + +an uploaded certificate bundle is now also stored by that child process, so +its two fsync(2) calls no longer stop the main process. The upload is still +answered only when the bundle is on disk. A certificate deletion also runs +in the child process, after every earlier store. A controller process that +exits during a store is started again only when the store ends, so that it +reads the stored files. + + + + + +the wasm module did not check the offset that the malloc handler of the guest +returned. A negative or large offset put the request buffer outside the +linear memory of the guest. The worker now refuses such an offset, or an +offset that is not aligned, and exits at startup. + + + + + +a regular expression in the configuration could run for up to 10,000,000 +match steps on one request. A match now stops after at most 100,000 steps +and the request gets a 500 response. The error log then gets a warning with +the pattern and the subject length, not the subject. In "compression" +"types", a match that stops is not a match, and the response is not +compressed. A repeated group takes about 2 steps per byte. So a subject +much longer than 8 KiB can reach the limit and get a 500 response. Such a +subject needs a "large_header_buffer_size" above its default of 8192. With +PCRE 1 and with PCRE2 before 10.30, a match also stops at 2,000 nested calls +and does not overflow the thread stack. There, a repeated group on a subject +of more than about 1,000 bytes can get a 500 response. + + + + + +an application with a rootfs created the missing directories of each mount +destination with mkdir(), which follows symlinks. A symlink inside the rootfs +could make unitd create empty directories outside it. The directories are +now created without following symlinks. On a kernel without openat2() (older +than 5.6) a rootfs with a symlinked component in a mount destination (for +example usr -> real_usr) is now refused. A missing rootfs directory is now +refused on every system. Builds without openat2(), such as on FreeBSD, +created it before. + + + + + +with isolation.namespaces.cgroup and isolation.cgroup.path, the cgroup +namespace of an application was rooted at the cgroup of the main process, not +at the configured cgroup. The application saw its own cgroup as +"/scope/python" instead of "/". The namespace is now created after the process +is moved into its cgroup. + + + + + +an uploaded njs module was written into its file without truncating it, and +without fsync(2). When a longer file of that name was already on disk, for +example one that a failed write had left, the old tail stayed after the new +module. The router read that file and the configuration failed. The upload +was also answered with 200 when the write failed. Now the main process stores +the module through a temporary file, flushes it and renames it, in its store +child, and the upload is answered only when the module is on disk. A deletion +also runs in the store child. A module name can no longer start with ".", and +a module larger than 16 MiB is answered with 413. Earlier versions accepted a +name that starts with "."; such a module is no longer loaded or listed after +the upgrade. + + + + + +at startup, a stored njs module that did not compile hid every stored module +that was read after it. Those modules were missing from /js_modules and could +not be used. Now only the module that does not compile is left out. + + + + + +on macOS, a stored state file is now flushed with fcntl(F_BARRIERFSYNC), and +its directory with fcntl(F_FULLFSYNC) after the rename, so the file also leaves +the cache of the drive. Plain fsync(2) there does not flush that cache, and a +power loss could lose a stored configuration. + + + + + +when unitd ran as root and the control socket was given to another user or +group with --control-user or --control-group, that user passed the socket file +permissions but the controller then closed the connection. The controller now +also accepts the configured user, and a peer in the configured group. + + + + + +a worker that got a QUIT before it was ready did not act on it, and ran until +it was killed 15 seconds later, so unitd was slow to exit. The worker now +exits at once. + + + + + +at shutdown, the router could ask a prototype that was already exiting to +start a new worker. The prototype then forked a worker that nobody stopped, +and unitd did not exit. If the prototype was gone, the router logged a false +alert. The prototype now refuses the start, and the router logs a gone +prototype at the debug level. + + + + + +a QUIT that fails because the worker already exited is now logged at the info +level, not as an alert. + + + + + +when a router thread ran out of connections or file descriptors, the log got +an "epoll_ctl() failed (2: No such file or directory)" alert every 100 +milliseconds until a connection was freed. + + + + + +when a router thread could not allocate the listen event while a listener was +added, a configuration request could wait forever, or a listening socket +shared by several router threads could be closed too early. When the thread +had no free connection slot instead, the listener never accepted connections. +Now the request completes, and a listener without a free slot starts to accept +connections when a slot is free. + + + + + +at shutdown, idle client connections were closed but not freed. They are now +closed through their protocol handlers. + + + + + +on systems without accept4(), an accepted socket that could not be made +non-blocking was closed and then still used. + + + + + +the router and libunit read records from shared memory that the other +process can still write, without enough checks. A buffer that was not a whole +number of records was read past its end. In libunit, a segment id of +UINT32_MAX wrapped the size of the segment array, a segment of any size was +accepted, a duplicate segment id replaced a segment that buffers still used, +and a field could point before the field array. The router trusted a busy +chunk marker that the application can clear. All of these are now checked. + + + + + +the wasm module read the base address of the linear memory of the guest only +once, at startup. A guest with a 64-bit memory could grow it past 4 GiB, and +wasmtime then moved it. The module then read and wrote the old address, and +the worker crashed. The module now refuses a 64-bit memory. It also reads +the base address again after the guest runs. + + + + + +a keep-alive TLS connection that was idle across a reconfiguration could +use the TLS settings of the old configuration after they were freed, when it +was closed. The connection now keeps the configuration until it is freed. + + + + + +a read or write that was already queued when a connection was closed could +run on the closed connection: a write could call a missing error handler, and +a read could read the closing socket and close the connection again. Such a +queued handler now does nothing. + + + + + +a static file response that failed after the request was closed could close +the file descriptor twice. The second close could close a descriptor that +another connection had got in the meantime. + + + + + +a proxied request with a chunked body, or an HTTP/2 request without a +content-length, was sent to the upstream with two Content-Length fields. + + + + + +with "limits": {"timeout"} set, a WebSocket session was closed about one +timeout after the upgrade, because the request deadline stayed armed for every +frame. The deadline now stops at the upgrade. + + + + + +a WebSocket client that sent PINGs and did not read the PONGs could make the +router keep a PONG in memory for each PING. Now, while a PONG is not sent, the +router keeps only the PONG for the most recent PING, and sends it after the +queued PONG (RFC 6455 Section 5.5.3). + + + + + +the router crashed when it validated a configuration that gave +"compressors" in the object form. + + + + + +with telemetry on, the application now gets a traceparent that names +FreeUnit's own span as its parent, so the application's spans are children of +FreeUnit's span. Before, the application got the client's traceparent +unchanged. + + + + + +an application process could send control messages to the router. It could +make the router quit, load a configuration, restart applications, forget a +process, redirect the error log, or reopen the access log. The router now +checks the sender pid that the kernel gives. QUIT, CHANGE_FILE and ACCESS_LOG +must come from main. REMOVE_PID must come from main or from a prototype. +DATA, APP_RESTART and STATUS must come from the controller. The router closes +the descriptors of a refused message and does not reply. + + + + + +an application process could send a false NEW_PORT to the router, register +its own pid as main or as the controller, and then pass the checks above. It +could also get the port of main or of the controller with GET_PORT, attach a +shared memory segment to another process with MMAP, make a debug router abort +with GET_MMAP, and answer a pending request of the router with RPC_READY or +RPC_ERROR, for example with a false certificate. The router now accepts +NEW_PORT from main, from a prototype for an application port, and from an +application for its own application port only. GET_PORT, GET_MMAP, MMAP and +OOSM must come from the application that the message names, and GET_PORT gives +out only ports of the router. RPC_READY and RPC_ERROR must come from main, the +controller or a prototype. Every process now refuses a NEW_PORT message that +is too short. Limits: the router does not check that an RPC reply comes from +the process that got the request. The ports of the router threads get +messages through a shared memory queue, which gives no sender pid, so the +router does not check senders there. + + + + + +an application could write an unexpected value into an entry of a shared +memory queue. The router then retried that entry for ever. Each enqueue and +dequeue now stops after one queue size of retries. The router refuses the +message and writes an alert. + + + + + +a port kept every fragment of a message until the last one arrived, so a +peer process could make it hold memory without limit. A fragment that named +a shared memory segment but carried no record also left a receive buffer in +the stream while the same buffer was reused for the next read. Reassembly +is now limited to 64 streams, 128 MiB per stream and 256 MiB per port, a new +stream drops the oldest one when 64 are open, and such a fragment is dropped. +A port linked without a process no longer crashes a release build when it is +freed. + + + + + +an application used a shared memory segment id from the other process to grow +its array of segments without a limit. An id like 100000000 in one message +made it allocate and fill an array of that many slots, so the process could +run out of memory. It now refuses an id of 65536 or more. The router already +refused such ids from applications; it now uses the same limit for the +segments it creates. + + + + + +The proxy recognised an upstream Transfer-Encoding only when its value was +exactly "chunked". "Chunked" or "gzip, chunked" was not recognised, so the +body was framed by Content-Length or by connection close, and the chunk +framing reached the client as body bytes. The value is now parsed as a list +of codings, and coding names are compared without case. One "chunked" is +decoded. Any other transfer coding, and chunked together with +Content-Length, fails the request with 502: the proxy cannot decode such a +body, and it never forwards Transfer-Encoding to the client. A response to +HEAD, and any 204 or 304, is exempt from both 502s: it ends at the first empty +line, so no Transfer-Encoding frames it. Before, such a response gave 502 when +it carried chunked together with Content-Length. + + + + + +".svgz" added to the default MIME type list as "image/svg+xml". A share "types" +rule that names image/svg+xml or image/* now matches a .svgz file; before, the +file had no type and such a rule refused it. + + + + + +a static ".svgz" file is served as "image/svg+xml" with "Content-Encoding: +gzip". Before, it had no Content-Type, and a browser did not show it. The +stored gzip bytes are sent unchanged: compression does not code them again, the +ETag stays strong, and a Range applies to the stored bytes. A client without +gzip in Accept-Encoding also gets the gzip bytes, as with "AddEncoding gzip +svgz" in Apache. A "mime_types" setting that maps ".svgz" to another type adds +no coding. The ETag of a .svgz file ends in "-gzip", so a cache that stored the +old response gets a full response, not a 304. A .svgz file never answers 304 to +If-Modified-Since, and a date in If-Range never resumes it, so a client that +revalidates by date only downloads the file again each time. + + + + + +the "body_min_rate" and "send_min_rate" options in "settings/http" set +a minimum client rate in bytes per second on HTTP/1 connections. The +body_read_timeout and send_timeout timers start again after each read or +write, so a client that sends or reads one byte before each timeout kept +a connection for ever. +After a grace time equal to the related timeout, a client below the rate +gets 408 (body) or the connection closes (response). The rate is checked for +each window of at least the grace time, so bytes sent early give no credit for +a slow transfer later. The default is 0, which disables the check. For +body_min_rate, 256 is a good value. send_min_rate counts the bytes that the +kernel accepts into the socket send buffer. The kernel sizes this buffer to +the link, so the rate is near the real download rate of the client. A floor +above the bandwidth of honest slow clients stops their downloads; use a small +value, for example 16384. + + + + + +the --hardening=[off|default|strict] configure option adds compiler and +linker hardening flags. Each flag is probed, and a compiler that does not +support a flag only loses that flag. The default is "off", so a build +without the option does not change. + + + + + +"GET /certificates" shows the SHA-256 fingerprint of the server certificate of +every bundle as "fingerprint", in the form "openssl x509 -fingerprint -sha256" +prints it. + + + + + +"PUT /certificates/<name>" replaces an existing certificate bundle in +place. Main writes the bundle to a temporary file and renames it over the +old one, so a crash never leaves a partial bundle. When the current +configuration names the bundle, the controller applies the configuration +again before it answers. New handshakes get the new certificate; accepted +connections finish with the old one. An external ACME client can renew a +certificate with one request and no restart; see docs/acme.md. A bundle over +1 MiB is refused with 413, and a name over 255 bytes with 400, for PUT and +for DELETE. A bundle with the same certificates as the stored one is not +stored again and causes no reconfiguration. The router opens bundles +read-only. + + + + + +the router leaked the TLS contexts it had built for a new configuration when +applying that configuration failed later, for example in a bundle of a later +listener or in an application. + + + + + +with "isolation": {"namespaces": {"pid": true}}, a worker that died before it +was fully started left its record, port and descriptor in the main process +until the prototype exited. The prototype now reports such a worker to main, +which drops them. + + + + + +the main process accepted a second WHOAMI from an application process, and a +parent that is not a prototype. The process was then linked into its parent's +list twice, which corrupted the list in main. Such a message is now refused. + + + + + +a request could reach the application with a truncated method or header name +length. The application protocol stores method and field name lengths in a +single byte. When a method exceeded 255 bytes, the router wrapped the length. +For PHP, Perl, and Ruby, header names carry the "HTTP_" prefix, causing names +of 251 to 255 bytes to wrap. The router now refuses a method longer than +255 bytes with 501, and an overflowing header name with 431. Both are logged +at "info". Other languages continue to accept header names up to 255 bytes. + + + + + +an upstream 1xx interim response ("100 Continue", "103 Early Hints") was +relayed to the client as the response, with the final response relayed as its +body. The proxy now drops interim responses (101 excepted) and relays the +final one; more than ten of them, or one larger than the proxy header buffer, +fails the request with 502. + + + + + +a certificate bundle whose private key does not belong to its first +certificate is now refused at upload with 400 "Invalid certificate.". Before, +the bundle was stored, and the mismatch broke the next reconfiguration of every +listener that named it. Certificate names that start with "." are reserved +for the store's own files, and files with such names are ignored at startup. +After an upgrade, a bundle already stored with a mismatched key is not loaded +(an alert at startup) and leaves /certificates; its file stays in the state +directory and cannot be deleted through the API, so remove it by hand. + + + + + +an application worker that served a WebSocket upgrade never became idle again. +The router counts a session on the worker and keeps a worker with a session out +of the idle queues, but nothing uncounted the session when it ended, so the +process stayed out of them for good: its idle timer never ran and its slot +counted against "processes": {"max"} until it died, which an application that +serves WebSockets reaches one upgrade at a time. A session is now uncounted +when it ends, whichever side ends it; the bug had appeared in 1.19.0. + + + + + +an application that answered a request and kept running was counted idle. PHP +does this whenever a script calls fastcgi_finish_request(): the response goes +out, the script runs on, and the router returned the worker's slot to +"processes": {"max"} and let its idle timer reap a process that was still +executing. Such a worker now reports itself busy until the work ends, so "max" +bounds live processes and "/status" reports them as "detached". The same holds +for a request whose "limits": {"timeout"} deadline has passed: the worker still +running it is no longer returned to the idle pool either, and keeps counting +against "max" until it answers or exits; +the bug had appeared in 1.29.0. + + + + + +"limits": {"timeout"} now also bounds how long a request waits for an +application process, not only how long a worker may spend on it. A request +that waited for capacity previously had no deadline at all. The default +remains no timeout. + + + + + +the router decoded the response an application returned in shared memory +without checking it: the field count was read twice, and the name, value and +body offsets were followed without a bounds check. An application could point +the router anywhere. libunit read the request from shared memory more than +once after checking it. A sibling process of the same application, which maps +the same segment, could change a value in between. Both sides now read every +value once and check every offset against the buffer. + + + + + +the string form of "access_log" "format" wrote what a variable expanded to +without escaping it. "$uri" is the percent-decoded target, so a request for +"/%0d%0a..." put a real CRLF in the log and the rest of the line read as a +record of a request that never happened. Bytes a variable expands to are now +escaped the way nginx escapes them: the quote, the backslash, every byte below +0x20, 0x7F and every byte above it all become "\xHH". nginx emits no short +forms, so a reader that already unescapes an nginx log reads this one. The +text of the +format itself is unchanged, except for an "njs" format, which produces the +whole record and is escaped whole. A value that carries a quote of its own now +shows it escaped: an entity-tag logged through "$response_header_etag" reads +\x22abc\x22 rather than "abc", and a non-ASCII path reads "\xD0\xBF..." rather than +its bytes. A log parser that expected the raw bytes has to be updated, though "\xHH" is +reversible and the bytes can be recovered from it. The object form of "format" +is unaffected by this change, but it is not a way to keep bytes either: it +replaces anything that is not valid UTF-8 with U+FFFD before it serializes. + + + + + +when "access_log" "format" was a JSON object without an njs expression, the +router wrote each variable's value into the record without escaping it. A +request header with a quote could close a member and add members to the +record. Every value is now escaped. A variable in a member name is no longer +expanded; the name is written as configured. + + + + + +an object-format access log replaces each byte that begins no valid UTF-8 +sequence with U+FFFD. Request headers may carry such bytes, and the record was +then not valid JSON. The string form keeps them, written as \xHH. + + + + + +a port message that was queued because the peer's socket was not writable +borrowed the descriptors it carried -- for NEW_PORT, another port's socket +and queue -- so a port closed before the queue drained left the deferred +sendmsg() naming a closed number, or an unrelated descriptor that had since +reused it. The queued copy now owns duplicates and closes them after the +send (#388). + + + + + +a process could stop hearing from a peer for good. When the wake-up that +tells a peer to read the shared queue failed because the kernel could not +allocate for it, the port treated the peer as dead: the wake-up was dropped +and, since a later message raises one only when the queue was empty, no +later message on that port raised one either. The wake-up is now kept and +retried, and the retry no longer waits for a socket event that a socket that +never filled up will not deliver (#392). + + + + + +a port that was closed and freed could leave a pending epoll or kqueue change +behind it. The change is held by a pointer into the port, and is applied when +the batch is committed -- at the top of the next poll at the latest, which is +after the port's memory pool is released. The commit then read freed memory, +and could also name a descriptor number the kernel had already handed to +somebody else. A port now drops its pending changes before it is freed +(#414). + + + + + +upgrade contrib njs to 1.0.1. + + + + + +removed the unused job cluster (nxt_job.c, nxt_job.h, and +nxt_event_conn_job_sendfile.c). The offload path they implemented had no +caller anywhere in the tree and its thread pool handoff was structurally +unreachable, since the connection field it read was never assigned. + + + + + +byte-range requests for static files. A "share" action answers a "Range" +header naming a single range with "206 Partial Content" and a +"Content-Range" header, an unsatisfiable one with "416 Range Not +Satisfiable", and advertises support with "Accept-Ranges: bytes". +"If-Range" is honoured, so a range is served only while the client's copy +is still current. A request naming several ranges is answered in full, as +a server that does not implement multipart ranges may. + + + + + +conditional requests for static files. A "share" action now answers a GET +or HEAD carrying "If-None-Match" or "If-Modified-Since" with "304 Not +Modified" when the file's validators still match, so a client that already +holds the file is not sent it again. Unit emitted "ETag" and +"Last-Modified" before, but ignored both when they came back. "If-Match" +and "If-Unmodified-Since" are evaluated first and answer "412 Precondition +Failed" when they do not hold. Entity-tag comparison is weak for +"If-None-Match" and strong for "If-Match", and "*" matches any +representation, per RFC 9110. + + + + + +the router died on the first request after a configuration that had a +"compression" block was replaced by one that had none. The compression +state was held in process globals allocated from the router configuration +that parsed them, and a later configuration neither refilled nor cleared +them, so the next request read a freed pool. The state is now held per +configuration and reached through the request, and the compressor a +response is already using is kept with that response, so a body still +being compressed when the configuration is replaced also finishes (#167). + + + + + +a static response that was subject to content negotiation now carries +"Vary: Accept-Encoding". Without it a shared cache could store whichever +representation it saw first and serve it to every client -- gzip bytes to one +that cannot decode them, or the uncompressed copy to one that could have had +the small version. The header is sent on the identity response as well as +the compressed one, since the identity response is the one a cache must not +reuse for a client that accepts gzip. + + + + + +a content-coded response now carries a weak entity-tag, so "If-Range" cannot +resume a coded representation; "If-None-Match" revalidation is unaffected. +Unit derives the tag from the file's mtime and size, which do not change with +the content coding, so a gzip response and the identity one previously +advertised the same strong validator -- 1.36.1 and earlier shipped that. +RFC 9110 requires a strong validator to change whenever the selected +representation changes, and nginx weakens the tag for the same reason. + + + + + +a file descriptor is no longer leaked when a static request cannot be +satisfied by any acceptable representation. A share with compression +configured answered "406 Not Acceptable" without closing the file it had +opened, so a single unauthenticated request carrying +"Accept-Encoding: identity;q=0, *;q=0" cost the router a descriptor, and +repeating it exhausted the table, stopping every application behind that +router. Present since compression was added in 1.35.0: 1.35.0, 1.35.1, +1.35.2, 1.35.3, 1.35.4, 1.35.5, 1.36.0 and 1.36.1 are all affected. + + + + + +a file descriptor sent to a port is no longer leaked when the message is +dispatched and nothing takes ownership of it. An application holds the write +end of the router's main port and the message header is unauthenticated, so a +compromised application could attach a descriptor to messages the router does +not expect one on -- or to a fragment that never reaches a handler -- and +consume the router's descriptor table, stopping every application behind it. +A fragment stream that is started and never completed still retains its +descriptor; that needs a bound on pending fragments and is tracked in +freeunitorg/freeunit#343. + + + + + +two source files that the build never used have been removed. +src/nxt_job_cache_file.c is named nowhere in auto/, so no configuration ever +compiled it, and nothing in the tree referenced it or its symbols. +src/nxt_source.h was removed because nothing includes it: of the 106 headers +in src/, it was the only one with no #include anywhere in the tree. Nothing +observable changes. + + + + + +the fiber implementation has been removed. It had not been compiled since +2017: the only allocation of the engine's fiber stack sat under "#if 0", so +the pointer was always NULL and the setjmp() branch that tested it could not +run. The thread, log and sprintf sites were already disabled. Nothing +observable changes. + + + + + +the nxt_dyld wrapper around dlopen()/dlsym()/dlclose() has been removed. +It shipped in the initial import and was never called: the language +module loader in src/nxt_application.c calls dlopen() and dlsym() itself +and never used this wrapper. libnxt.a loses one object file; there are +no other callers. + + + + + +do not exit when the capget() syscall is denied. A syscall filter that +omits capget -- a systemd "SystemCallFilter=" allowlist, a hand-written +seccomp profile -- made a non-root unitd fail to start with "failed to get +process capabilities". Unit now warns and runs without assuming any +capability: user and group switching and "rootfs" isolation are disabled +for applications that do not enable the "credential" namespace, and +applications run as unitd's own uid. Every other capget() failure +remains fatal. + + + + + +the main process did not answer a START_PROCESS request it could not carry +out -- an unknown router port, a sender that is not the router, or an +allocation that failed -- so the start stayed pending. The application then +parked every later start on a prototype that was not coming, and the +requests waiting on it were never failed. + + + + + +a message that could not be sent because the receiving process had died was +reported to the sender as sent. A sender that answers a request and then sends +a fallback lost both, so an application could wait for an answer that nothing +would ever deliver. + + + + + +eight places whose memory nothing else reclaims handed a message to a port and +did not take it back when the port refused it. The port layer accepts a buffer +only when it answers success, so on a refusal -- which happens when memory runs +out, or when the receiving process has stopped reading a full shared queue -- +the memory was lost. Two of them lost shared memory that no cleanup reclaims, +so an application could lose the capacity to answer at all. + + + + + +a websocket connection is now closed when a frame cannot be handed to the +application. Such a frame was previously dropped in silence, leaving the +peer's stream short a frame it was never told about -- which for a fragmented +message the client cannot detect. + + + + + +a port could stop sending after it ran out of memory. When a write had to be +retried later, the port asked its own thread to watch the socket again, and +that request needed memory of its own. If it failed, nothing watched the +socket and every later message waited in the queue until the port closed. The +port now carries the request it needs, so it cannot fail for lack of +memory. + + + + + +the port layer reported a message as sent when it had been neither sent nor +queued. This happened when a socket answered EAGAIN and no memory was left to +hold the message for a later attempt, so it takes memory pressure to reach. +The sender then cleared the state that would have +retried or reported it, so a request could wait for an answer that nothing +would ever deliver. + + + + + +an application worker that died before it finished starting was reported to +nobody, so the start it was forked for stayed pending forever. Once the +application reached its "processes" maximum it started no further process, +and because an application's "timeout" defaults to 0 the requests waiting on +it hung instead of failing. + + + + + +application processes no longer inherit unitd's capabilities. A unitd +started as a non-root user that had been granted capabilities -- systemd +"AmbientCapabilities=", file capabilities -- passed them through fork() +to every application it ran, because switching between two nonzero uids +does not clear them and a filtered capget() skips the switch entirely. +Each forked process now empties its permitted, effective, inheritable +and ambient sets once its credentials are final, before the worker +serves any request. The language module's own startup still runs before +the drop: the mount, pivot_root and chroot work for "rootfs" isolation +happens after it and needs the capabilities that are being given up. +Where a syscall filter denies capset() as well, an application start is +now refused rather than allowed to proceed with capabilities still held: +the request is answered 503 and the log names the denied syscall. A +process with an empty permitted, effective and ambient set -- which is +every process of a root unitd once it has switched away from uid 0, and +every process of an unprivileged one that was granted no capabilities -- +keeps behaving as before, since there is nothing an application could +have inherited from it. Its inheritable set is emptied as well, which +on its own is not counted and grants nothing without an execve() of a +file carrying a matching capability. The main process is +unchanged, since it needs CAP_NET_BIND_SERVICE to bind listening sockets +on every reconfiguration; the router, controller and discovery processes +warn and carry on rather than refuse, because they run no application +code. Applications configured with "user": "root" are also unchanged, +since a root process regains capabilities on the next execve() anyway. +One configuration does lose something it previously kept: an application +in a "credential" namespace began with a full capability set inside that +namespace, because the default uid_map maps unitd's euid to the +application's uid, so the setuid() is not a transition out of uid 0 and +nothing cleared it. Those capabilities were valid only inside the +namespace and Unit's own use of them is finished before the drop, but an +application that relied on them now needs "user": "root". + + + + + +unitctl gave up on a configuration it could not decode as UTF-8. Unit stores +the bytes it is given, so a configuration can hold bytes no UTF-8 decoder +accepts; "unitctl execute" and "unitctl export" then failed with "JSON error +[path=/config]" and left the operator no way to read or back up the +configuration. Both work now: the document is shown with the undecodable +bytes replaced and the members they fall in named, and "export" stores the +server's bytes verbatim. "unitctl edit" refuses such a configuration rather +than writing the replacements back over it. + + + + + +unitctl reordered the members of a configuration. "unitctl edit" and "unitctl +export" passed the document through an unordered map, so members came back in +an arbitrary order and an edit rewrote the whole configuration even where +nothing had changed. The order the server sent is now kept. + + + + + +unitctl no longer depends on rustls-pemfile or on hyper's "http2" feature. +rustls-pemfile is unmaintained; the parser is replaced by rustls-pki-types, +already in the dependency graph. PEM parsing behavior is unchanged: both +parsers were run over 1884 PEM documents and produced zero differences. This +closes RUSTSEC-2025-0134. Dropping the "http2" feature from hyper, hyper-rustls +and hyper-util removes the h2 and fnv crates as well, closing +RUSTSEC-2026-0258, an unbounded empty DATA frame flood in h2 0.4.15. The +control API speaks HTTP/1.1 only -- unitd writes "HTTP/1.1" into every response +line -- so HTTP/2 support was never usable against the server itself. + + + + + +unitctl no longer offers HTTP/2 in the TLS handshake it makes to a remote +control API. This follows from the "http2" feature removal above. A TLS- +terminating proxy placed in front of a remote control API must now speak +HTTP/1.1 to unitctl; one that only accepts HTTP/2 can no longer be reached. + + + + + +unitctl no longer reads hjson or YAML, and no longer writes YAML. Input piped +to "/config" on stdin is parsed as JSON now, not hjson; hjson is a superset of +JSON, so piping JSON still works, but comments and unquoted keys do not. A +".hjson" or ".cjson" file is refused with "hjson is no longer supported: +convert the file to JSON first", instead of being handed to the JSON parser, +which reports a misleading error at the first comment. A ".yaml" or ".yml" +file is refused with "YAML is no longer supported: convert the file to JSON +first, for example with "yq -o=json"". Supported input formats are now json +and pem. "unitctl status -t yaml" is gone; --output-format now accepts +json, json-pretty and text. This removes the nu-json dependency and the 30 +crates of Nushell utility code it pulled in for a tool that speaks one JSON +control API. It also removes serde_yaml and unsafe-libyaml; the latter is a +c2rust transpile of libyaml with 218 unsafe functions and no RustSec advisory. +tools/unitc is unaffected: it still converts configuration to and from YAML, +using yq. + + + + + +unitctl builds TLS support behind a "tls" Cargo feature, on by default. The +musl build turns it off, so the static binary no longer links a C TLS library +and no longer accepts an "https://" control socket address; it refuses one with +a message naming the feature. A Unix socket or a plain "http://" address is +unaffected, and a build with default features behaves as before. + + + + + +removed src/nxt_queue.c, the unused half of a queue utility library carried +over from nginx. It defined nxt_queue_sort() and nxt_queue_middle(), both +exported but never called anywhere in the tree; src/nxt_queue.h and the +nxt_queue_* macros it provides are unaffected and untouched. + + + + + +nxt_mem_zone, a second allocator, is removed. It was a zone allocator over a +mapped region, distinct from nxt_mp, the pool allocator product code actually +uses; nothing in the router, applications, or any language module called it. +Its only caller was its own unit test, run by "./build/tests"; that test is +removed too, dropping the three "mem zone test passed" lines from the suite's +output. Upstream nginx/unit had already reached the same conclusion and +deleted the file as part of an unrelated cleanup. + + + + + +the entity-tag and last-modified date of a static file are now weak while +the request falls inside the second the file was last written, because a +rewrite to the same size during that second changes neither validator. +If-Match and the If-Range forms stop matching in that window, so a range +conditioned on If-Range falls back to the full response; an unconditional +Range is unaffected. The tag keeps its format, so nothing already cached is +invalidated. + + + + + +unitctl sends a JSON configuration file as it was written. It parsed the file +and re-serialized it before, which kept the last of two object members with the +same name and re-spelled numbers, so the server never saw what the operator +wrote. Unit now reports the duplicate itself, with the line and column of the +file. A file that is not valid UTF-8 is no longer refused by unitctl; Unit +refuses it and names the member. The "edit" command no longer accepts JSON5 comments in the temporary +file. + + + + + +removed the GnuTLS, CyaSSL, and PolarSSL backends (nxt_gnutls.c, +nxt_cyassl.c, nxt_polarssl.c) and their configure plumbing ("--gnutls", +"--cyassl", "--polarssl"). None of the three ever built: they read a +"c->u.ssltls" union member that does not exist, and reference types such +as "nxt_ssltls_conf_t" and "nxt_event_conn_io_t" that predate a refactor +still visible in the current tree, so passing the flag reaches a compiler +error instead of a clean "unsupported". CyaSSL was renamed wolfSSL and +PolarSSL became mbedTLS, both in 2015, so the flags also target products +that no longer exist under these names. The OpenSSL backend is +unaffected; none of these files were ever part of a default build. + + + + + +unitctl no longer reads JSON5. A ".json5" file is refused by name, with a +message saying to convert it first; "json5 -o config.json config.json5" does +that, and "jq" reformats ordinary JSON. Nothing parsed JSON5 after the +configuration upload path stopped parsing its input, so this removes a parser +that only the refusal path still reached, and two crates with it. + + + + + +removed the PHP TrueAsync code path. It was written for a "zend_async_event_t" +API expected in PHP 8.5 core, which did not ship it, so the configure probe +never succeeded and the code sat behind "#if NXT_PHP_TRUEASYNC" in every build. +It could not have compiled had the probe succeeded: it reads "async" and +"entrypoint" members that "nxt_php_app_conf_t" does not have and calls +"nxt_php_extension_init()", which no file in the tree defines. The probe, the +"entrypoint" target key and the TrueAsync test suite are removed with it; a PHP +application builds and behaves as before. + + + + + +the java module split every outgoing WebSocket text message larger than 8 KiB +into 8 KiB frames, whatever "max_frame_size" was set to. A 64 KiB message +arrived as eight frames and a 16 MiB one as about two thousand, each carrying +its own frame header and costing its own message to the router, while every +other language module sends a complete message as a single frame. A complete +text message, from getBasicRemote() or getAsyncRemote(), is now encoded into a +buffer of its own and sent as one frame; a message above +"nginx.unit.websocket.MAX_SEND_BUFFER_SIZE", 16 MiB by default, still +fragments. A Writer from getSendWriter() still sends each chunk it flushes as +a frame of its own, but that chunk is now one frame instead of up to four. An +OutputStream from getSendStream() still sends each 8 KiB it fills as a frame of +its own. This does not make the module honour "max_frame_size", which remains +an inbound limit only. + + + + + +the java module sent nothing through the asynchronous WebSocket remote. Every +send from getAsyncRemote() -- sendText(), sendBinary() and sendObject(), with a +SendHandler or through the Future they return -- ended in a doWrite() that had +no body, so no frame left, the handler was never called and Future.get() +blocked for good; getBasicRemote() sends through another path and was not +affected. Such a send now goes out as one from the blocking remote does, and +its handler or future completes on the sending thread with the outcome, which +is a failure once the session is closed (#434). Batching works as well: +flushBatch() and setBatchingAllowed(false) threw instead of sending what was +batched, and a batched frame from a buffer whose position was not 0 had a +header that overstated its length and corrupted the frames after it. + + + + + +the java module sent the wrong bytes, or nothing, for a WebSocket message +from a heap ByteBuffer that does not start at its array's first byte. The +payload was read from the backing array at position(), ignoring arrayOffset(), +so a buffer from ByteBuffer.wrap(a, off, n).slice() sent bytes from the start +of the array instead of its own; and a read-only heap buffer threw +ReadOnlyBufferException instead of being sent. Both the blocking and the +asynchronous remote were affected (#486). + + + + + +the Python ASGI module refused a WebSocket message larger than 1 MB whatever +"max_frame_size" was set to. The module carried a private 1 MB limit that no +configuration ever wrote and charged the fragments of a message against it, +while the configured value is checked by the router per frame: a 4 MB message +was closed with 1009 where "max_frame_size" was 32 MB, and at the default size +a client that split a larger message into small frames was refused as well. +Each frame is now limited by "max_frame_size" alone. The module only limits a +connection to 10 MB of payload that the application has not received yet +(fragments of a message in progress and complete messages waiting for +receive()); a single frame larger than that is still accepted when nothing else +is waiting. A fragmented message also left a count behind that lowered the +limit applied to the next message. + + + + + +a byte-range request from a client that refused the identity coding is no +longer answered with identity bytes. A range is served as identity, so +Range together with "Accept-Encoding: identity;q=0" returned a 206 carrying +exactly the bytes the client said it would not take; the range is now +ignored and the full 200 is sent in a coding the client accepts. An +unsatisfiable range from such a client is no longer answered with 416 either. +A request that accepts no coding at all is still 406. Several Accept-Encoding +lines in one request are now read as one comma-joined value; previously only the +first was consulted. + + + + + +a response that no compressor applies to is no longer sent as identity to a +client that refused the identity coding. With a compressor configured, +"Accept-Encoding: gzip, identity;q=0" was accepted because gzip was +acceptable. When the media type of the response was outside "types", or the +response had no media type, the response was then sent uncompressed: the +identity bytes that the request had refused. Such a request now gets 406, +with a Range header and without one. A 406 from this negotiation carries +"Vary: Accept-Encoding". A 1xx, 204 or 304 response is never negotiated, so +an application that sends one of these with a Content-Length is not affected. +A response with no body, and a response that carries its own +Content-Encoding, are not affected either. + + + + + +"*" in Accept-Encoding now stands for each enabled compressor that the client +did not name. Before, it stood for the identity coding only. +"identity;q=0, *;q=1" was answered 406 with gzip enabled and applicable, +because the wildcard could select only identity, and the same field refused +identity. A coding that the field names keeps its own weight. The wildcard +does not match it, and the named coding wins a tie: "gzip, *" now sends gzip. +Before, it sent the response uncompressed. + + + + + +a compressor below its own min_length is no longer selected before a +compressor with a lower weight that can be applied. min_length belongs to +each compressor. With gzip at 1000 and deflate at 0, a 100-byte response was +sent uncompressed, because gzip had the higher weight. A client that refused +the identity coding got the same uncompressed bytes. Such a client now gets +deflate. When each compressor that the client accepts is below its +min_length, the client gets 406 instead of the bytes it refused. + + + + + +a server with no compression configured now answers 406 to a request whose +Accept-Encoding refuses the identity coding: "identity;q=0", or "*;q=0" with +no more specific entry for identity. Identity is the only coding that such a +server has. Before, it sent the response, or the requested range of it, in +the bytes that the client had refused. To read that refusal, each response +with a known length above zero now looks at its own Content-Encoding and at +the request Accept-Encoding. Before, these checks ran only with a compressor +configured. + + + + + +a malformed weight in Accept-Encoding is no longer read as a refusal. The +qvalue was converted with strtod(), which takes no digits at all from +"identity;q=" and from "identity;q=abc" and reports zero, so either one +answered 406 to a client that had refused nothing; "q=0x0" and "q=0e0" did the +same. An element whose weight does not match the grammar of RFC 9110 +Sect. 12.4.2 is now ignored, as an element naming an unknown coding already +was. This also removes "q=nan", which passed the 0-to-1 range check and then +outranked every real weight. + + + + + +a negotiation failure on Accept-Encoding was answered with 503 instead of 406 +when the response came from an application. The router treated every result +of the acceptability check other than success as a server error; only the +static path reported the 406. + + + + + +a "Content-Encoding" set with "response_headers" no longer makes the body +coded twice. With a compressor configured, the response was compressed first, +and "response_headers" then replaced the "Content-Encoding" of the compressor. +So a share of "$uri.gz" with "Content-Encoding: gzip" sent gzip of the stored +gzip file to a client that accepts gzip, with a weak ETag. Now such a +response is not compressed, as with a "Content-Encoding" from an application. +This applies to static files and to application responses. A +"Content-Encoding" removed with null in "response_headers" also keeps the +compressor out, so bytes that Unit compressed are no longer sent without a +"Content-Encoding". Such a response is identity, so a client that refused +identity gets "406 Not Acceptable". For a response that has its own +"Content-Encoding", the null still removes only the field, as before. + + + + + +unitd, libunit.a and the language modules no longer contain the test hooks +compiled under "#if (NXT_TESTS)". "./configure --tests" used to define the +macro for the whole build, and the deb and rpm packages pass "--tests", so +every released package from 0.2 to 1.36.1 carried them: the nxt_random_test() +entry point in all of them, and in 1.36.0 and 1.36.1 the fault-injection +branches and counters in nxt_mp.c, nxt_port_rpc.c and nxt_port_socket.c, two +of which were exported symbols. The hooks now go only into the test programs. +A build without "--tests" is unchanged. + + + + + +update wasmtime to 48.0.5. The WebAssembly module moves from 43.0.1, and the +WebAssembly component module from 47.0.2. Version 48 is a long-term support +release of wasmtime. It fixes twelve wasmtime security advisories published on +24 September and 2 October 2026. Each one needs a hostile WebAssembly guest. +With the features that FreeUnit builds, such a guest could panic the host, +exhaust host memory, read uninitialized host memory, or corrupt the GC heap. +Building the two modules now needs Rust 1.95.0 or newer. + + + + + +the PHP module checked that "script" is under "root" by comparing the first +bytes of the two real paths. With "root": "/srv/app", a script in "/srv/app2" +passed the check and was served. The script must now be in "root" or in a +directory below it. A "script" that resolves to "root" itself is also +rejected; it was served as an empty 200 response. The bug had appeared in 0.4. + + + + + +the Java module now bundles Apache Tomcat 9.0.122 (from 9.0.121). The +upstream release fixes CVE-2026-87022 (WebSocket message smuggling with +per-message-deflate) and eleven other CVEs. None of the twelve fixes is in a +jar that FreeUnit bundles. The FreeUnit WebSocket code is derived from Tomcat, +but it does not decompress received frames, so CVE-2026-87022 does not affect +it. + + + + + +Docker images for Go 1.27, Perl 5.44 and Java 27. The Java images are now +based on Ubuntu 26.04 (resolute) instead of 24.04 (noble), because Eclipse +Temurin publishes Java 27 only for 26.04. + + + + + +the Docker images now install the pending updates of their base image when +they are built. Before, an image kept the packages of its base image as they +were when that image was published. + + + + + + + + + + +FreeUnit updated to 1.37.0. + + + + + + + + +configuration validation errors now say where they are. A rejected +configuration request carries an RFC 6901 JSON Pointer in the "path" member +of "location": it points at the member whose value failed validation, or at +the containing object when the failure is an unknown parameter, whose name +the message itself already carries; "" is the document root. On an unknown +parameter whose name is a close and unambiguous typo of a known one, the +intended name is reported in a top-level "suggestion" member. The addition +is strictly additive: the existing response fields and the error message +wording are unchanged, and each new member is omitted where it does not +apply. + + + + + +the /status API reports OpenTelemetry span export health as "telemetry": +{"spans": {"exported": N, "failed": M}} -- the spans the collector accepted, +and the spans whose export failed. Both are reported because a failure count +alone cannot distinguish a healthy pipeline from one that has never exported +anything. The object is omitted entirely unless Unit was built with +OpenTelemetry support, "/config/settings/telemetry" is set, and the exporter +was built successfully, so its absence is how a client tells telemetry is off +rather than idle. The counters are cumulative since the exporter was last +built, so changing any telemetry setting restarts them, while re-applying an +identical configuration does not. + + + + + +validate the shared memory segment identifier and geometry an application +process supplies in its NXT_PORT_MSG_MMAP message. The router checked only +the pid fields, then used the peer-authored header id directly as an array +index and mapped the descriptor at whatever length fstat() reported; a +compromised or faulty application could drive an out-of-range write and an +oversized mapping. + + + + + +bounds-check the router-supplied chunk identifier and size in libunit before +consuming them, so an out-of-range size can no longer make every +serialized-pointer check in the arrival-time validation block pass and be +read out of bounds in the application process. + + + + + +do not replay a stale SCM_RIGHTS control message in libunit when the +embedder's port_recv callback returns without receiving anything. The out +parameter carrying the control-data length was stored unconditionally, so a +descriptor from an earlier message could be adopted a second time. + + + + + +check the shared memory buffer allocation when forwarding a websocket frame. +Exhausting the outgoing shared memory crashed the router with a NULL +dereference; every other caller already checked. + + + + + +keep the chunked request body buffer alive across framing reads. A buffer +drained without producing a data slice was handed to its completion handler +while the parser still referenced it, a use-after-free reachable from an +ordinary chunked request. + + + + + +parse the HTTP/1.1 trailer section (RFC 9112 7.1.2) instead of erroring on +the first byte after the terminal chunk. In the proxy response relay this +aborted an already-streaming response and truncated the body at a racy +point; chunked requests carrying a trailer were rejected the same way. +Trailer field names and values are validated as they are parsed. + + + + + +reject an application response that disagrees with itself about its own body +length. A duplicate Content-Length was neither detected nor parsed, so every +such header was forwarded verbatim and the body was framed by the advertised +length instead of chunked. + + + + + +release the response buffers adopted from the port on the error paths of the +router's response handler, and drain them from the request output chain when +the failure happens after they were linked into it. Both leaked the chain +the handler had taken ownership of. + + + + + +keep the memory pool cursor aligned in nxt_mp_get() by rounding the requested +size up to the maximum alignment rather than flooring it. The small +allocation path is a bump allocator with no per-allocation alignment step, so +any request whose size was not a multiple of that alignment left the page +cursor misaligned, and allocations from that page kept violating the +documented alignment guarantee until a later size happened to bring the +cursor back into step, which crashed with SIGBUS on strict alignment targets +such as 32-bit ARM. + + + + + +serialize the process reference count on the runtime process mutex and take a +reference during cross-thread process lookups. The count was mutated +non-atomically from every engine thread and a lookup returned an unreferenced +process, so a router worker could operate on an nxt_process_t -- including its +shared memory segments and its mutex -- that the main thread had already torn +down. + + + + + +keep the socket and timer tasks of a connection scoped to that connection, so +a work item cannot run against a task belonging to a different connection. + + + + + +remove the dead chunk-tracking bitmap from the shared memory segment header. +Its trailing fields ran past the header area and aliased the payload of the +segment's first chunk, which was harmless only because nothing read or wrote +them; the field offsets a peer of either vintage uses are unchanged, so old +and new builds still interoperate. A static assertion now pins the header +struct inside the area reserved for it. + + + + + +the shared memory chunk range check is now a single shared implementation +used by both the router and libunit, with the chunk count computed so that a +peer-supplied size in the top of the 32-bit range cannot wrap to zero chunks +and be accepted. + + + + + +recycle connection structures through a per-engine freelist, embed the static +file context in the request, recycle static file buffer descriptors through a +thread-local freelist, and resolve the first eight routing variables and the +first sixteen request and response header fields from arrays embedded in the +request. Each avoids or reuses an allocation on a per-connection or +per-request hot path. + + + + + +the WebAssembly component module no longer supports guest-initiated outbound +HTTP. Serving Unit never needed it, and the feature that provided it linked +an entire TLS stack -- rustls, tokio-rustls, webpki-roots and rustls-webpki -- +into the module. A guest that only imports wasi:http/outgoing-handler, as the +wasi:http/proxy world requires, still instantiates and runs; an actual +outbound call now fails with HttpRequestDenied. + + + + + +update wasmtime from 36.0.12 to 47.0.2 in the WebAssembly component module, +keeping outbound HTTP denied and the TLS stack out of the dependency graph. + + + the "if" option of a route "match" object, added in 1.33.0, is now @@ -24,6 +1674,230 @@ comparisons and regular expressions + + +build against OpenSSL 4.0, which constified the return types of the X509 name +accessors and deprecated X509_NAME_get_text_by_NID(); SNI certificate matching +and the "certificates" control API now read the certificate name through +X509_NAME_get_index_by_NID() and the entry accessors instead, and a new CI job +builds and runs the C test suite against OpenSSL 4.0 with deprecated APIs +disabled, so it fails on any new use of a deprecated function rather than only +on a removal. + + + + + +OpenSSL 1.1.1 or later is now required to build with "--openssl"; configure +fails with an explicit message on older releases. Builds against OpenSSL 1.0.x +have not worked since 1.35.2, when the TLS module started to use +OpenSSL_version_num(); the unreachable pre-1.1.0 compatibility code is +removed. + + + + + +close the engine port when a router worker thread exits. Lowering +settings/listen_threads deletes engines, and the thread exit handler only +dropped a port reference, and the runtime held another, so nothing ever +called nxt_port_close(): every deleted engine leaked its socket pair and its +queue descriptor, three per engine, for the life of the router. The count +only ever rose, so a configuration that raises and lowers the thread count +walked the router towards its descriptor limit. + + + + + +close the descriptors a forged or malformed port message leaves behind. A +message carries up to two descriptors whatever its type normally uses, and +the dispatcher reclaims nothing once a handler has returned. The main +process kept the second descriptor on the whoami path, the router lost both +when a conf data message named no reply port, and a PROCESS_READY naming an +unknown process closed neither. A peer that is already compromised could +exhaust the descriptor table of the receiving process -- the main process on +the whoami path -- by repeating any of them. + + + + + +bound the log file slot a CHANGE_FILE message claims, and stop leaking the +descriptor it carries. The slot index arrived from a peer and was used to +walk the runtime's log file list without a range check, and a message whose +payload was shorter than the index was read past its end. The rejection +paths now close what the message attached, and nxt_file_redirect() releases +the descriptor when dup2() fails rather than only on success -- which also +ends a leak of one descriptor per unrotatable log file on every SIGUSR1. + + + + + +a NEW_PORT or PROCESS_READY message carrying a shared-memory queue shorter +than the queue type faulted the receiving process (main, the prototype, or the +router) with SIGBUS on first access; the object size is now checked before +mapping, a short queue is refused with a warning, and the router no longer +leaks the descriptor of a queue it refuses. + + + + + +a NEW_PORT message announcing a port that already existed, or one that could +not be created, left an uninitialized stack value behind; the controller +dereferenced it unconditionally on every NEW_PORT, and main and the router +relied on a NULL check that could not catch non-NULL garbage. The result is +now cleared explicitly whenever no port is created. + + + + + +close the file descriptors a NEW_PORT or MMAP message left behind instead of +leaking them. The discovery and application prototype processes kept every +port queue descriptor open for the life of the process, the main process kept +the queue descriptor of any port it had not just created, the router lost it +on ports it does not map a queue for, the controller never closed it, and the +mmap handler leaked its second descriptor on every path and returned closing +nothing at all when the first was missing; a message type with no handler also +went uncollected. A NEW_PORT for +a port that already has a queue now refuses the new descriptor instead of +mapping it over the live queue and leaking the mapping it replaced, which had +let a repeated announcement re-point an established port at memory of the +sender's choosing. + + + + + +session tickets were silently disabled when Unit was built against an OpenSSL +3.x library compiled with the deprecated 3.0 API removed +(OPENSSL_NO_DEPRECATED_3_0), because the feature guard tested for a control +macro that is itself declared only under that API and so was never defined; +the session ticket key callback now registers through +SSL_CTX_set_tlsext_ticket_key_evp_cb() and keys an EVP_MAC context on OpenSSL +3.0 and later, independent of the deprecated HMAC_CTX API. + + + + + +stop building OpenTelemetry span attributes for spans the sampler has +already dropped. set_attribute() on a non-recording span discards its +argument, but the argument was built first -- the sprintf round-trips, the +pool allocations and two owned strings per attribute -- and nothing asked +whether the span was recording. Lowering sampling_ratio therefore bought +back the exporter and nothing on the request path. The sampler's verdict is +now read once when the span is created and the attribute stages consult it, +which removes most of telemetry's request-path cost at a low sampling +ratio. Context propagation is deliberately outside that check: the +traceparent must reach the peer and the application whatever the sampling +decision was. + + + + + +do not attach the traceparent to an OpenTelemetry span as a "Unit Attribute" +event. It duplicated the trace and span identifiers the span context +already carries, nothing consumed it, and it was added twice on a request +that took the error path with a trace it had started itself, because the +error path re-runs header propagation. + + + + + +OpenTelemetry spans were lost at the edges of a request's life. A request +that ended before its span was collected -- an error path, a dropped +connection -- left the span allocated and never exported it; the span is now +tied to the memory pool that holds it and is exported with an error status +when that pool is released, so an abandoned request appears in the trace +instead of vanishing. + + + + + +flush the OpenTelemetry span pipeline before the router exits. Exit and +reload discarded the pipeline with up to 4096 spans still queued, losing every +span produced since the last export. The pipeline is now flushed with a 2 +second bound, so a slow or unreachable collector delays shutdown by at most +that long rather than either hanging it or costing the queued spans. + + + + + +rebuild the OpenTelemetry exporter only when the telemetry settings have +changed. Every configuration change rebuilt it, so adding a route or changing +an application stalled reconfiguration against an unreachable collector even +though it did not touch "/config/settings/telemetry". The exporter is now +rebuilt only when the endpoint, protocol, sampling ratio or batch size +actually differs from the running one. + + + + + +the OpenTelemetry span attributes "http.response.status_code" and +"http.request.body.size" are now emitted as OTLP integer values, as the HTTP +semantic conventions specify, instead of strings. A collector or query that +matches these attributes as strings must be adapted; one that reads them as +integers, or that relied on the convention rather than on Unit's previous +output, needs no change. + + + + + +the work a traced request does on the request path is reduced: attribute keys +are no longer copied onto the heap per request, the two attributes above no +longer round-trip through strings, and a span stage now crosses into the +tracer once instead of once per attribute. Together with the sampling fix +above, the telemetry overhead of a request measured 27% lower at +"sampling_ratio" 1.0 and 53% lower at 0.1. + + + + + +the Java module now bundles Apache Tomcat 9.0.121 (from 9.0.110) and +ClassGraph 4.8.186 (from 4.8.181). The upstream Tomcat 9.0.121 release fixes +CVE-2026-66299 (unbounded buffer for undelivered WebSocket messages, a memory +exhaustion by a slow client), CVE-2026-73180, CVE-2026-68569, CVE-2026-66422 +and CVE-2026-65927. FreeUnit bundles only the servlet, JSP and EL API jars, +Jasper and the Tomcat utility jars -- not catalina, the WebSocket +implementation or the example webapps -- so the bump keeps the jars it ships +on a supported release. + + + + + + + + + + +FreeUnit updated to 1.36.1. + + + diff --git a/docs/man/man8/unitd.8.in b/docs/man/man8/unitd.8.in index 517d1a957..cb0868bdc 100644 --- a/docs/man/man8/unitd.8.in +++ b/docs/man/man8/unitd.8.in @@ -72,10 +72,15 @@ Sets the group of the UNIX-domain control socket. .Pp On a UNIX-domain control socket, and where the platform provides peer credentials (Linux and the BSDs), the control API additionally requires -the connecting peer's effective user ID to be either 0 (root) or the -user ID +the connecting peer's effective user ID to be 0 (root), the user ID .Nm -runs as; other local users are rejected before the request is read. +runs as, or the user set with +.Fl Fl control-user , +or its group ID to be the group set with +.Fl Fl control-group . +On Linux 4.13 and later, a supplementary group of the peer also matches +.Fl Fl control-group . +Other local users are rejected before the request is read. On those builds the .Fl Fl control-user , .Fl Fl control-group , diff --git a/docs/unit-openapi.yaml b/docs/unit-openapi.yaml index eb515305e..b9cb6ab95 100644 --- a/docs/unit-openapi.yaml +++ b/docs/unit-openapi.yaml @@ -1,6 +1,6 @@ openapi: 3.0.0 info: - title: "FreeUnit 1.36.0 (ex NGINX Unit)" + title: "FreeUnit 1.37.0 (ex NGINX Unit)" description: "FreeUnit (ex NGINX Unit) is a lightweight and versatile application runtime that provides the essential components for your web application as a single open-source server: running application code, serving static assets, @@ -112,9 +112,28 @@ paths: put: operationId: putCertBundle - summary: "Create or overwrite the actual certificate bundle" - description: "Creates or overwrites the [stored certificate bundle] - (https://unit.nginx.org/certificates/) in Unit." + summary: "Create or replace the actual certificate bundle" + description: "Creates the [stored certificate bundle] + (https://unit.nginx.org/certificates/) in Unit, or replaces it in + place when the name exists. The body is a PEM bundle: the server + certificate first, then its chain, then the private key. Main + writes the bundle to a temporary file and renames it over the old + one. When the current configuration names the bundle, the + controller applies the configuration again before it answers with + `Certificate chain updated.`; new handshakes get the new + certificate, accepted connections finish with the old one. + Otherwise the answer is `Certificate chain uploaded.`. A bundle + whose key does not belong to the first certificate is refused with + 400 `Invalid certificate.`, a name that starts with `.` or is over + 255 bytes with 400 `Invalid certificate name.`, and a bundle over + 1 MiB with 413 + `Certificate bundle is too large.`; the old bundle stays in use. + 500 `Certificate stored but not applied.` means the bundle is on + disk, but the router refused the configuration with it and keeps + the old certificate. A bundle with the same certificates as the + stored one is not stored again and causes no reconfiguration; the + answer is the one a store gives. The request waits while a + reconfiguration is in progress, as a configuration change does." tags: - certificates @@ -140,6 +159,9 @@ paths: "404": $ref: "#/components/responses/responseNotFound" + "413": + $ref: "#/components/responses/responsePayloadTooLarge" + "500": $ref: "#/components/responses/responseInternalError" @@ -174,6 +196,40 @@ paths: "404": $ref: "#/components/responses/responseNotFound" + /certificates/{bundleName}/fingerprint: + summary: "Endpoint for the certificate bundle fingerprint" + get: + operationId: getCertBundleFingerprint + summary: "Retrieve the fingerprint of the server certificate" + description: "Retrieves the SHA-256 fingerprint of the first + certificate in a [stored certificate bundle] + (https://unit.nginx.org/certificates/), in the form + `openssl x509 -fingerprint -sha256` prints it." + + tags: + - certificates + + parameters: + - $ref: "#/components/parameters/bundleName" + + responses: + "200": + description: "OK; the certificate bundle exists in the + configuration." + + content: + application/json: + schema: + type: string + + examples: + Fingerprint: + value: "5F:0A:3E:8B:1C:27:9D:44:B6:E1:72:C8:0D:35:9A:F4:\ + 6B:E8:21:7C:D3:90:4E:AB:15:66:F2:8D:3C:71:B9:02" + + "404": + $ref: "#/components/responses/responseNotFound" + /certificates/{bundleName}/chain: summary: "Endpoint for the certificate bundle chain" get: @@ -3229,6 +3285,87 @@ paths: "404": $ref: "#/components/responses/responseNotFound" + /config/settings/http/body_min_rate: + summary: "Endpoint for the `body_min_rate` option in `http`" + get: + operationId: getSettingsHttpBodyMinRate + summary: "Retrieve the body_min_rate option from http settings" + description: "Retrieves the `body_min_rate` option that represents + the minimum rate of the client request body, in bytes per second. + The value 0 disables the check." + + tags: + - settings + - config + + responses: + "200": + description: "OK; the `body_min_rate` option exists in the + configuration." + + content: + application/json: + schema: + type: integer + + examples: + BodyMinRate: + value: 256 + + "404": + $ref: "#/components/responses/responseNotFound" + + put: + operationId: updateSettingsHttpBodyMinRate + summary: "Create or overwrite the body_min_rate option" + description: "Creates or overwrites the `body_min_rate` option in + the configuration." + + tags: + - settings + - config + + requestBody: + required: true + content: + application/json: + schema: + type: integer + + examples: + BodyMinRate: + value: 256 + + responses: + "200": + $ref: "#/components/responses/responseOkUpdated" + + "400": + $ref: "#/components/responses/responseBadRequest" + + "404": + $ref: "#/components/responses/responseNotFound" + + "500": + $ref: "#/components/responses/responseInternalError" + + delete: + operationId: deleteSettingsHttpBodyMinRate + summary: "Delete the body_min_rate option" + description: "Deletes the `body_min_rate` option from the + configuration." + + tags: + - settings + - config + + responses: + "200": + $ref: "#/components/responses/responseOkDeleted" + + "404": + $ref: "#/components/responses/responseNotFound" + /config/settings/http/body_read_timeout: summary: "Endpoint for the `body_read_timeout` option in `http`" get: @@ -4469,6 +4606,87 @@ paths: "404": $ref: "#/components/responses/responseNotFound" + /config/settings/http/send_min_rate: + summary: "Endpoint for the `send_min_rate` option in `http`" + get: + operationId: getSettingsHttpSendMinRate + summary: "Retrieve the send_min_rate option from http settings" + description: "Retrieves the `send_min_rate` option that represents + the minimum rate at which the client reads the response, in bytes + per second. The value 0 disables the check." + + tags: + - settings + - config + + responses: + "200": + description: "OK; the `send_min_rate` option exists in the + configuration." + + content: + application/json: + schema: + type: integer + + examples: + SendMinRate: + value: 16384 + + "404": + $ref: "#/components/responses/responseNotFound" + + put: + operationId: updateSettingsHttpSendMinRate + summary: "Create or overwrite the send_min_rate option" + description: "Creates or overwrites the `send_min_rate` option in + the configuration." + + tags: + - settings + - config + + requestBody: + required: true + content: + application/json: + schema: + type: integer + + examples: + SendMinRate: + value: 16384 + + responses: + "200": + $ref: "#/components/responses/responseOkUpdated" + + "400": + $ref: "#/components/responses/responseBadRequest" + + "404": + $ref: "#/components/responses/responseNotFound" + + "500": + $ref: "#/components/responses/responseInternalError" + + delete: + operationId: deleteSettingsHttpSendMinRate + summary: "Delete the send_min_rate option" + description: "Deletes the `send_min_rate` option from the + configuration." + + tags: + - settings + - config + + responses: + "200": + $ref: "#/components/responses/responseOkDeleted" + + "404": + $ref: "#/components/responses/responseNotFound" + /config/settings/http/send_timeout: summary: "Endpoint for the `send_timeout` option in `http`" get: @@ -5051,6 +5269,7 @@ paths: application/json: schema: type: integer + format: int64 examples: Accepted: @@ -5075,6 +5294,7 @@ paths: application/json: schema: type: integer + format: int64 examples: Active: @@ -5098,6 +5318,7 @@ paths: application/json: schema: type: integer + format: int64 examples: Idle: @@ -5122,6 +5343,7 @@ paths: application/json: schema: type: integer + format: int64 examples: Closed: @@ -5171,11 +5393,128 @@ paths: application/json: schema: type: integer + format: int64 examples: Closed: value: 1307 + /status/telemetry: + summary: "Endpoint for the `telemetry` status object" + get: + operationId: getStatusTelemetry + summary: "Retrieve the telemetry status object" + description: "Retrieves the `telemetry` status object that represents + Unit's [OpenTelemetry export statistics] + (https://unit.nginx.org/statusapi/)." + + tags: + - status + + responses: + "200": + description: "OK; the `telemetry` object exists in the + configuration." + + content: + application/json: + schema: + $ref: "#/components/schemas/statusTelemetry" + + examples: + example1: + $ref: "#/components/examples/statusTelemetry" + + "404": + $ref: "#/components/responses/responseNotFound" + + /status/telemetry/spans: + summary: "Endpoint for the `spans` telemetry object" + get: + operationId: getStatusTelemetrySpans + summary: "Retrieve the telemetry spans object" + description: "Retrieves the `spans` object that represents Unit's + [OpenTelemetry span export counters] + (https://unit.nginx.org/statusapi/)." + + tags: + - status + + responses: + "200": + description: "OK; the `spans` object exists in the configuration." + + content: + application/json: + schema: + $ref: "#/components/schemas/statusTelemetrySpans" + + examples: + example1: + $ref: "#/components/examples/statusTelemetrySpans" + + "404": + $ref: "#/components/responses/responseNotFound" + + /status/telemetry/spans/exported: + summary: "Endpoint for the `exported` spans number" + get: + operationId: getStatusTelemetrySpansExported + summary: "Retrieve the exported spans number" + description: "Retrieves the `exported` spans number from Unit's + [telemetry statistics](https://unit.nginx.org/statusapi/), the spans + the collector accepted." + + tags: + - status + + responses: + "200": + description: "OK; the `exported` number exists in the + configuration." + + content: + application/json: + schema: + type: integer + format: int64 + + examples: + Exported: + value: 4096 + + "404": + $ref: "#/components/responses/responseNotFound" + + /status/telemetry/spans/failed: + summary: "Endpoint for the `failed` spans number" + get: + operationId: getStatusTelemetrySpansFailed + summary: "Retrieve the failed spans number" + description: "Retrieves the `failed` spans number from Unit's + [telemetry statistics](https://unit.nginx.org/statusapi/), the spans + whose export failed." + + tags: + - status + + responses: + "200": + description: "OK; the `failed` number exists in the configuration." + + content: + application/json: + schema: + type: integer + format: int64 + + examples: + Failed: + value: 0 + + "404": + $ref: "#/components/responses/responseNotFound" + /status/applications: summary: "Endpoint for the `applications` status object" get: @@ -5324,6 +5663,72 @@ paths: "404": $ref: "#/components/responses/responseNotFound" + /status/applications/{appName}/processes/unaccounted: + summary: "Endpoint for the `unaccounted` processes number" + get: + operationId: getStatusApplicationsAppProcessesUnaccounted + summary: "Retrieve the unaccounted processes app status number" + description: "Retrieves the `unaccounted` processes number that + represents Unit's per-app + [process statistics](https://unit.nginx.org/statusapi/)." + + tags: + - status + + parameters: + - $ref: "#/components/parameters/appName" + + responses: + "200": + description: "OK; the `unaccounted` number exists in the + configuration." + + content: + application/json: + schema: + type: integer + format: int64 + + examples: + Unaccounted: + value: 0 + + "404": + $ref: "#/components/responses/responseNotFound" + + /status/applications/{appName}/processes/detached: + summary: "Endpoint for the `detached` processes number" + get: + operationId: getStatusApplicationsAppProcessesDetached + summary: "Retrieve the detached processes app status number" + description: "Retrieves the `detached` processes number that + represents Unit's per-app + [process statistics](https://unit.nginx.org/statusapi/)." + + tags: + - status + + parameters: + - $ref: "#/components/parameters/appName" + + responses: + "200": + description: "OK; the `detached` number exists in the + configuration." + + content: + application/json: + schema: + type: integer + format: int64 + + examples: + Detached: + value: 0 + + "404": + $ref: "#/components/responses/responseNotFound" + /status/applications/{appName}/processes/idle: summary: "Endpoint for the `idle` processes number" get: @@ -5443,11 +5848,17 @@ components: bundleName: in: path - description: "A certificate bundle's name" + description: "A certificate bundle's name. It is not empty, it is + 255 bytes or less, and it does not start with \".\" (such names + are reserved for the certificate store). An invalid name gives + 400 on PUT and DELETE." name: bundleName required: true schema: type: string + minLength: 1 + maxLength: 255 + pattern: "^[^.].*" langMod: in: path @@ -5495,6 +5906,15 @@ components: line: 1 column: 0 + errorUnknownParameter: + summary: "400 error response for an unknown parameter" + value: + error: "Invalid configuration." + detail: 'Unknown parameter "pas".' + location: + path: "/routes/0/action" + suggestion: "pass" + errorValueDoesntExist: summary: "404 error response" value: @@ -5518,6 +5938,8 @@ components: value: bundle: key: "RSA (4096 bits)" + fingerprint: "5F:0A:3E:8B:1C:27:9D:44:B6:E1:72:C8:0D:35:9A:F4:\ + 6B:E8:21:7C:D3:90:4E:AB:15:66:F2:8D:3C:71:B9:02" chain: - subject: common_name: "example.com" @@ -5559,6 +5981,8 @@ components: summary: "Single certificate bundle" value: key: "RSA (4096 bits)" + fingerprint: "5F:0A:3E:8B:1C:27:9D:44:B6:E1:72:C8:0D:35:9A:F4:\ + 6B:E8:21:7C:D3:90:4E:AB:15:66:F2:8D:3C:71:B9:02" chain: - subject: common_name: "example.com" @@ -5757,12 +6181,14 @@ components: settings: http: + body_min_rate: 256 body_read_timeout: 30 discard_unsafe_fields: true header_read_timeout: 30 idle_timeout: 180 log_route: true max_body_size: 8388608 + send_min_rate: 16384 send_timeout: 30 server_version: false @@ -6029,6 +6455,7 @@ components: summary: "Global settings" value: http: + body_min_rate: 256 body_read_timeout: 30 compression: types: @@ -6043,6 +6470,7 @@ components: idle_timeout: 180 log_route: true max_body_size: 8388608 + send_min_rate: 16384 send_timeout: 30 server_version: false static: @@ -6055,6 +6483,7 @@ components: configSettingsHttp: summary: "HTTP settings" value: + body_min_rate: 256 body_read_timeout: 30 compression: types: @@ -6069,6 +6498,7 @@ components: idle_timeout: 180 log_route: true max_body_size: 8388608 + send_min_rate: 16384 send_timeout: 30 server_version: false static: @@ -6170,12 +6600,18 @@ components: closed: 1050 requests: total: 1307 + telemetry: + spans: + exported: 4096 + failed: 0 applications: wp: processes: running: 9 starting: 1 + unaccounted: 0 idle: 0 + detached: 0 requests: active: 15 @@ -6235,7 +6671,9 @@ components: processes: running: 9 starting: 1 + unaccounted: 0 idle: 0 + detached: 0 requests: active: 15 @@ -6246,7 +6684,9 @@ components: processes: running: 9 starting: 1 + unaccounted: 0 idle: 0 + detached: 0 requests: active: 15 @@ -6256,7 +6696,9 @@ components: value: running: 9 starting: 1 + unaccounted: 0 idle: 0 + detached: 0 # /status/applications/{appName}/requests statusApplicationsAppRequests: @@ -6270,6 +6712,21 @@ components: value: total: 1307 + # /status/telemetry + statusTelemetry: + summary: "Regular telemetry status object" + value: + spans: + exported: 4096 + failed: 0 + + # /status/telemetry/spans + statusTelemetrySpans: + summary: "Regular telemetry spans status object" + value: + exported: 4096 + failed: 0 + # -- RESPONSES -- responses: @@ -6309,6 +6766,9 @@ components: example1: $ref: "#/components/examples/errorInvalidJson" + example2: + $ref: "#/components/examples/errorUnknownParameter" + responseNotFound: description: "Not Found; the value does not exist in the configuration. This may occur if any part of the path is non-existent." @@ -6322,6 +6782,14 @@ components: example1: $ref: "#/components/examples/errorValueDoesntExist" + responsePayloadTooLarge: + description: "Payload Too Large. The certificate bundle is over 1 MiB." + + content: + application/json: + schema: + $ref: "#/components/schemas/jsonErrorMessage" + responseInternalError: description: "Internal server error; the configuration wasn't applied. This may occur with misconfigured paths, wrong permissions, etc." @@ -6360,10 +6828,67 @@ components: jsonErrorMessage: type: object - description: "JSON message on error." + description: "JSON message on error. Only `error` is always present; + each of the others is omitted where it does not apply." + required: + - error + properties: + error: + type: string + description: "What went wrong, in a human-readable sentence." + + detail: + type: string + description: "A longer explanation of the error, where one adds + anything to the message." + + location: + $ref: "#/components/schemas/errorLocation" + + suggestion: + type: string + description: "The name the request most likely meant, reported when + an unknown parameter is a close and unambiguous typo of a known + one." + additionalProperties: type: string + errorLocation: + type: object + description: "Where the error is. Present when it can be placed. Which + members appear depends on how it was found, and they do not index the + same document: `offset`, `line` and `column` locate the error in the + request body, while `path` locates it in the resulting configuration." + properties: + offset: + type: integer + description: "Byte offset into the request body, reported only for + an error found while parsing it. A leading UTF-8 byte order mark + is skipped before parsing, so the offset is relative to the JSON + that follows it rather than to the bytes sent." + + line: + type: integer + description: "Line number, reported with `column` and only for an + error found while parsing." + + column: + type: integer + description: "Column number, reported with `line`." + + path: + type: string + description: "An RFC 6901 JSON Pointer to the member whose value + failed validation, to the member a required parameter is missing + from, or to the containing object when the failure is an unknown + parameter, whose name the message itself carries. It points into + the configuration the request would produce, not into the request + body: a request to a path below \"/config\" is validated against a + copy of the whole configuration with the change applied, so the + pointer is rooted at the configuration and an empty string is the + configuration itself." + # Configuration sections as data types; hugely reliant on each other # /certificates @@ -6382,6 +6907,12 @@ components: type: string description: "Certificate bundle's key type, i. e. RSA, ECDSA, etc." + fingerprint: + type: string + description: "SHA-256 fingerprint of the first certificate in the + bundle, in the form `openssl x509 -fingerprint -sha256` prints + it: uppercase hex, a colon between the bytes." + chain: $ref: "#/components/schemas/certBundleChain" @@ -6694,6 +7225,12 @@ components: type: integer description: "Request timeout in seconds." + start_timeout: + type: integer + description: "Time in seconds an application process may take to + announce itself before Unit fails the start request; 0, the + default, means no limit." + processes: description: "Governs the behavior of app processes." anyOf: @@ -6884,6 +7421,17 @@ components: $ref: "#/components/schemas/stringArray" description: "Host directories this application may have access to." + execution_timeout: + type: integer + description: "Time in seconds a component may run for one request + before Unit stops it; 0, the default, means no limit. Unit + answers 500 if the component has not begun its response; once + its headers are sent, the client sees a truncated response + instead. This bounds the guest, not the request: it runs from + the guest's first instruction until it returns, waits inside + host calls included. To bound what the client waits for, use + limits/timeout, which answers 503." + configApplicationPHP: description: "PHP application on Unit." allOf: @@ -7493,6 +8041,31 @@ components: in Unit." properties: + body_min_rate: + type: integer + minimum: 0 + maximum: 2147483647 + description: "Minimum rate of the client request body, in bytes + per second. Only HTTP/1 connections use this check. The check + starts after a grace time. The grace + time is equal to `body_read_timeout`. After the grace time, + if the client sent fewer bytes than `body_min_rate` multiplied + by the elapsed seconds, Unit sends 408 and closes the + connection. The rate is checked for each window of at least + the grace time, so bytes sent early give no credit later. Unit + counts the body bytes that it reads after the request header. + The read that completes the body is not checked, because the + body is then complete. If `body_read_timeout` is 0, there is + no grace time. Each read is then checked against the time + since the previous read, or since Unit started to read the + body. A `body_read_timeout` of 0 also turns off the timer that + stops a client that sends nothing. The check runs only when + a read completes, so a client that stops sending is then never + stopped. A `body_min_rate` of 0 disables the check. Use the + value 256 to stop slow clients (\"slow POST\")." + + default: 0 + body_read_timeout: type: integer description: "Maximum number of seconds to read data from the body of @@ -7537,6 +8110,34 @@ components: default: 8388608 + send_min_rate: + type: integer + minimum: 0 + maximum: 2147483647 + description: "Minimum rate at which the client reads the response, + in bytes per second. Only HTTP/1 connections use this check. + Unit measures only the time when response + data waits for the client. The check starts after a grace time. + The grace time is equal to `send_timeout`. After the grace + time, if the client accepted fewer bytes than `send_min_rate` + multiplied by the elapsed seconds, Unit closes the connection. + The rate is checked for each window of at least the grace time. + Unit counts the bytes that the kernel accepts into the socket + send buffer. The kernel sizes this buffer to the link, so the + measured rate is near the real download rate of the client. A + floor above the bandwidth of honest slow clients stops their + downloads. Use a small value, for example 16384. If + `send_timeout` is 0, there is no grace time. Each write is + then checked against the time since the previous write, but + only the time when response data waits for the client is + counted. A `send_timeout` of 0 also turns off the timer that + stops a client that reads nothing. The check runs only when + a write completes, so a client that stops reading is then never + stopped. A `send_min_rate` of 0 disables the check. WebSocket + connections do not use this check." + + default: 0 + send_timeout: type: integer description: "Maximum number of seconds to transmit data as a @@ -7654,6 +8255,9 @@ components: requests: $ref: "#/components/schemas/statusRequests" + telemetry: + $ref: "#/components/schemas/statusTelemetry" + applications: $ref: "#/components/schemas/statusApplications" @@ -7711,6 +8315,23 @@ components: type: integer description: "Current starting app processes." + unaccounted: + type: integer + format: int64 + description: "App processes Unit started and can no longer see: a + worker that a `limits/start_timeout` deadline gave up on is still + running, but never announced itself, so Unit has no pid for it. + They count against `processes/max` until they announce themselves + after all, or exit." + + detached: + type: integer + format: int64 + description: "App processes that answered a request and kept + running, such as a PHP script after `fastcgi_finish_request()`. + They are counted in `running`, never in `idle`, and hold their + slot under `processes/max` until the work ends." + idle: type: integer description: "Current idle app processes." @@ -7731,8 +8352,39 @@ components: properties: total: type: integer + format: int64 description: "Total non-API requests during the instance’s lifetime." + # /status/telemetry + statusTelemetry: + description: "Represents the health of Unit's OpenTelemetry span + export. The object is omitted entirely unless Unit was built with + OpenTelemetry support, `/config/settings/telemetry` is set, and the + exporter was built successfully, so its absence is how a client tells + telemetry that is off from telemetry that is idle." + type: object + properties: + spans: + $ref: "#/components/schemas/statusTelemetrySpans" + + # /status/telemetry/spans + statusTelemetrySpans: + description: "Represents Unit's OpenTelemetry span export counters. + Both are cumulative since the exporter was last built, so changing + any telemetry setting restarts them, while re-applying an identical + configuration does not." + type: object + properties: + exported: + type: integer + format: int64 + description: "Spans the collector accepted." + + failed: + type: integer + format: int64 + description: "Spans whose export failed." + # /status/connections statusConnections: description: "Represents Unit's per-instance connection statistics." @@ -7740,19 +8392,23 @@ components: properties: accepted: type: integer + format: int64 description: "Total accepted connections during the instance’s lifetime." active: type: integer + format: int64 description: "Current active connections for the instance." idle: type: integer + format: int64 description: "Current idle connections for the instance." closed: type: integer + format: int64 description: "Total closed connections during the instance’s lifetime." diff --git a/fuzzing/README.md b/fuzzing/README.md index 9c70c8016..2ce2a02ba 100644 --- a/fuzzing/README.md +++ b/fuzzing/README.md @@ -38,18 +38,51 @@ $ make fuzz -j$(nproc) ```shell $ mkdir -p build/fuzz_basic_seed +$ mkdir -p build/fuzz_http_chunk_seed $ mkdir -p build/fuzz_http_controller_seed $ mkdir -p build/fuzz_http_h1p_seed $ mkdir -p build/fuzz_http_h1p_peer_seed +$ mkdir -p build/fuzz_http_h1p_peer_response_seed +$ mkdir -p build/fuzz_http_ws_utf8_seed $ mkdir -p build/fuzz_json_seed +$ mkdir -p build/fuzz_router_app_response_seed +$ mkdir -p build/fuzz_unit_msg_seed $ ./build/fuzz_basic build/fuzz_basic_seed fuzzing/fuzz_basic_seed_corpus +$ ./build/fuzz_http_chunk build/fuzz_http_chunk_seed fuzzing/fuzz_http_chunk_seed_corpus $ ./build/fuzz_http_controller build/fuzz_http_controller_seed fuzzing/fuzz_http_seed_corpus $ ./build/fuzz_http_h1p build/fuzz_http_h1p_seed fuzzing/fuzz_http_seed_corpus $ ./build/fuzz_http_h1p_peer build/fuzz_http_h1p_peer_seed fuzzing/fuzz_http_seed_corpus +$ ./build/fuzz_http_h1p_peer_response \ + build/fuzz_http_h1p_peer_response_seed \ + fuzzing/fuzz_http_h1p_peer_response_seed_corpus +$ ./build/fuzz_http_ws_utf8 build/fuzz_http_ws_utf8_seed fuzzing/fuzz_http_ws_utf8_seed_corpus $ ./build/fuzz_json build/fuzz_json_seed fuzzing/fuzz_json_seed_corpus +$ ./build/fuzz_router_app_response build/fuzz_router_app_response_seed fuzzing/fuzz_router_app_response_seed_corpus +$ ./build/fuzz_unit_msg build/fuzz_unit_msg_seed fuzzing/fuzz_unit_msg_seed_corpus ``` +### Differential targets. + +Three targets check a result, not only that the code does not crash. Each +one aborts on a difference, so a wrong answer is reported like a crash. + +- `fuzz_http_chunk` decodes a chunked body two times: one time as a single + buffer, and one time cut into pieces given to the decoder in chains of one + to three buffers. The two runs must give the same verdict, the same end + of the message, and the same decoded bytes. The first input byte seeds + the cuts. +- `fuzz_http_h1p_peer_response` parses an upstream response header in the + same way as the proxy: it continues after each read that returned + `NXT_AGAIN`. A run that gets the header a few bytes at a time must give + the same status, header size and fields as a run that gets it in one pass. + The first input byte seeds the read sizes. +- `fuzz_http_ws_utf8` runs the UTF-8 check for WebSocket text messages and + close reasons on masked frames that are split into several buffers. It + compares the verdict after each frame with a small validator written from + Unicode Table 3-7. Byte 0 selects a close reason, byte 1 seeds the cuts, + bytes 2-5 are the masking key. + Here is more information about [LibFuzzer](https://llvm.org/docs/LibFuzzer.html). ## Build fuzzers using other fuzzing engines. @@ -66,3 +99,61 @@ may vary). ``` clang, llvm & compiler-rt ``` + +`fuzz_unit_msg` (`nxt_unit_msg_fuzz.c`) is linked against libunit rather than +the core library: it feeds one router-to-application port message through +`nxt_unit_process_msg()` and the request handler. Its input format and what +the harness steers are described at the top of the file. + +`fuzz_router_app_response` (`nxt_router_app_response_fuzz.c`) is the other +side of that trust boundary: the router decoding an `nxt_unit_response_t` an +application handed back, through `nxt_router_response_header_parse()` in +`src/nxt_router.c` (extracted from `nxt_router_response_ready_handler()` so +it can be driven without a port message or a live application). The fuzz +input *is* the response buffer, byte for byte -- no framing on top. + +## In CI + +`.github/workflows/fuzzing.yml` builds these targets with clang, ASan and +UBSan and runs `fuzzing/run-ci.sh`, which gives each target a bounded +libFuzzer budget against its seed corpus and fails on the first crash. + +```shell +$ fuzzing/run-ci.sh # all ten, 60s each +$ fuzzing/run-ci.sh -t 120 fuzz_http_h1p # one target, longer +``` + +A pull request that touches `src/nxt_http*`, `src/nxt_h1proto*`, +`src/nxt_controller*`, `src/nxt_conf*`, `src/nxt_unit*`, `src/nxt_router*` or +`fuzzing/` runs all ten targets -- roughly ten minutes; a manual +`workflow_dispatch` takes a budget and a target list as inputs. Reproducers +land in `build/fuzz-artifacts/` and are uploaded as an artifact when the job +fails. + +Each target finishes within a second or two of its budget. `run-ci.sh` caps +it at the budget plus a minute anyway -- libFuzzer only checks the budget +between runs -- and treats hitting that cap as a warning rather than a +failure, while distinguishing it from a kill that arrives early: an OOM is a +finding, not slowness. + +It sets `UBSAN_OPTIONS=halt_on_error=1`. `-fsanitize=undefined` is +recoverable by default -- UBSan prints the report and carries on, libFuzzer +exits 0, and the check goes green on a finding. It deliberately does *not* +set `print_stacktrace=1`, which reintroduces the symbolizer cost below; the +report keeps its file:line and the reproducer is saved, so rerun the +reproducer with `UBSAN_OPTIONS=print_stacktrace=1` when you want the trace. + +It also passes `-print_funcs=0`, which is worth knowing about if you run the +fuzzers by hand. libFuzzer symbolizes every newly covered function to print +its `NEW_FUNC` line, and llvm-symbolizer needs about thirteen seconds per call +against a 14 MB ASan+UBSan binary. On a fresh corpus -- which is what CI has +every time -- that is the whole run: `fuzz_json` managed **7 executions in 90 +seconds** with the default, and **1,190,884 in 31 seconds** without it. Crash +reports keep their symbols either way; those come from the sanitizer's stack +printer, not from libFuzzer's coverage output. + +This is a smoke gate, not a campaign. It replaced an OSS-Fuzz CIFuzz job +that could not have worked here: the OSS-Fuzz `unit` project is marked +`disabled: true` ("Project is archived") and its `main_repo` is +`https://github.com/nginx/unit`, so a run from this repository would have +built upstream's source rather than the pull request's. diff --git a/fuzzing/build-fuzz.sh b/fuzzing/build-fuzz.sh index 62f7a6761..72820cb0c 100644 --- a/fuzzing/build-fuzz.sh +++ b/fuzzing/build-fuzz.sh @@ -10,10 +10,15 @@ export LIB_FUZZING_ENGINE="-fsanitize=fuzzer" make fuzz -j$(nproc) mkdir -p build/fuzz_basic_seed +mkdir -p build/fuzz_http_chunk_seed mkdir -p build/fuzz_http_controller_seed mkdir -p build/fuzz_http_h1p_seed mkdir -p build/fuzz_http_h1p_peer_seed +mkdir -p build/fuzz_http_h1p_peer_response_seed +mkdir -p build/fuzz_http_ws_utf8_seed mkdir -p build/fuzz_json_seed +mkdir -p build/fuzz_router_app_response_seed +mkdir -p build/fuzz_unit_msg_seed echo "" echo "Run: ./build/\${fuzzer} build/\${fuzzer}_seed fuzzing/\${fuzzer}_seed_corpus" diff --git a/fuzzing/fuzz_http_chunk_seed_corpus/hex_lower_ext.bin b/fuzzing/fuzz_http_chunk_seed_corpus/hex_lower_ext.bin new file mode 100644 index 000000000..8f4b6195f --- /dev/null +++ b/fuzzing/fuzz_http_chunk_seed_corpus/hex_lower_ext.bin @@ -0,0 +1,4 @@ +oa;name=value +0123456789 +0;x + diff --git a/fuzzing/fuzz_http_chunk_seed_corpus/hex_upper.bin b/fuzzing/fuzz_http_chunk_seed_corpus/hex_upper.bin new file mode 100644 index 000000000..311021e7c --- /dev/null +++ b/fuzzing/fuzz_http_chunk_seed_corpus/hex_upper.bin @@ -0,0 +1,4 @@ +JA +0123456789 +0 + diff --git a/fuzzing/fuzz_http_chunk_seed_corpus/incomplete.bin b/fuzzing/fuzz_http_chunk_seed_corpus/incomplete.bin new file mode 100644 index 000000000..8ce123b02 --- /dev/null +++ b/fuzzing/fuzz_http_chunk_seed_corpus/incomplete.bin @@ -0,0 +1,2 @@ +á65 +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA \ No newline at end of file diff --git a/fuzzing/fuzz_http_chunk_seed_corpus/last_chunk.bin b/fuzzing/fuzz_http_chunk_seed_corpus/last_chunk.bin new file mode 100644 index 000000000..cd96a933a Binary files /dev/null and b/fuzzing/fuzz_http_chunk_seed_corpus/last_chunk.bin differ diff --git a/fuzzing/fuzz_http_chunk_seed_corpus/long_data.bin b/fuzzing/fuzz_http_chunk_seed_corpus/long_data.bin new file mode 100644 index 000000000..c5620c711 --- /dev/null +++ b/fuzzing/fuzz_http_chunk_seed_corpus/long_data.bin @@ -0,0 +1,6 @@ +¼40 +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +40 +BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB +0 + diff --git a/fuzzing/fuzz_http_chunk_seed_corpus/pipelined.bin b/fuzzing/fuzz_http_chunk_seed_corpus/pipelined.bin new file mode 100644 index 000000000..bf8dedfc1 --- /dev/null +++ b/fuzzing/fuzz_http_chunk_seed_corpus/pipelined.bin @@ -0,0 +1,7 @@ +5 +hello +0 + +GET / HTTP/1.1 +Host: localhost + diff --git a/fuzzing/fuzz_http_chunk_seed_corpus/short_data.bin b/fuzzing/fuzz_http_chunk_seed_corpus/short_data.bin new file mode 100644 index 000000000..401263434 --- /dev/null +++ b/fuzzing/fuzz_http_chunk_seed_corpus/short_data.bin @@ -0,0 +1,4 @@ +—2 +X +0 + diff --git a/fuzzing/fuzz_http_chunk_seed_corpus/size_max.bin b/fuzzing/fuzz_http_chunk_seed_corpus/size_max.bin new file mode 100644 index 000000000..f151b4124 --- /dev/null +++ b/fuzzing/fuzz_http_chunk_seed_corpus/size_max.bin @@ -0,0 +1,2 @@ +rfffffffffffffff +AAAA \ No newline at end of file diff --git a/fuzzing/fuzz_http_chunk_seed_corpus/size_overflow.bin b/fuzzing/fuzz_http_chunk_seed_corpus/size_overflow.bin new file mode 100644 index 000000000..db8379ef9 --- /dev/null +++ b/fuzzing/fuzz_http_chunk_seed_corpus/size_overflow.bin @@ -0,0 +1,2 @@ +M10000000000000000 +A diff --git a/fuzzing/fuzz_http_chunk_seed_corpus/smuggle_colonless.bin b/fuzzing/fuzz_http_chunk_seed_corpus/smuggle_colonless.bin new file mode 100644 index 000000000..4da71a974 --- /dev/null +++ b/fuzzing/fuzz_http_chunk_seed_corpus/smuggle_colonless.bin @@ -0,0 +1,3 @@ +(0 +GET /admin HTTP/1.1 + diff --git a/fuzzing/fuzz_http_chunk_seed_corpus/trailer.bin b/fuzzing/fuzz_http_chunk_seed_corpus/trailer.bin new file mode 100644 index 000000000..6393620b7 --- /dev/null +++ b/fuzzing/fuzz_http_chunk_seed_corpus/trailer.bin @@ -0,0 +1,5 @@ +”3 +abc +0 +X-Trailer-Test: trailed + diff --git a/fuzzing/fuzz_http_chunk_seed_corpus/trailer_obs_text.bin b/fuzzing/fuzz_http_chunk_seed_corpus/trailer_obs_text.bin new file mode 100644 index 000000000..936b3910c --- /dev/null +++ b/fuzzing/fuzz_http_chunk_seed_corpus/trailer_obs_text.bin @@ -0,0 +1,3 @@ +Þ0 +X: € + diff --git a/fuzzing/fuzz_http_chunk_seed_corpus/trailers_two.bin b/fuzzing/fuzz_http_chunk_seed_corpus/trailers_two.bin new file mode 100644 index 000000000..ca22f01da --- /dev/null +++ b/fuzzing/fuzz_http_chunk_seed_corpus/trailers_two.bin @@ -0,0 +1,6 @@ +¹1 +z +0 +A: 1 +B: 2 + diff --git a/fuzzing/fuzz_http_chunk_seed_corpus/two_chunks.bin b/fuzzing/fuzz_http_chunk_seed_corpus/two_chunks.bin new file mode 100644 index 000000000..dc568458a --- /dev/null +++ b/fuzzing/fuzz_http_chunk_seed_corpus/two_chunks.bin @@ -0,0 +1,6 @@ +%2 +ab +2 +cd +0 + diff --git a/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/bad_length.bin b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/bad_length.bin new file mode 100644 index 000000000..16e2a1cfb --- /dev/null +++ b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/bad_length.bin @@ -0,0 +1,3 @@ +ÔHTTP/1.1 200 OK +Content-Length: 99999999999999999999999 + diff --git a/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/bad_version.bin b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/bad_version.bin new file mode 100644 index 000000000..1851c2bb1 --- /dev/null +++ b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/bad_version.bin @@ -0,0 +1,2 @@ +|HTTP/2.0 200 OK + diff --git a/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/chunked_and_length.bin b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/chunked_and_length.bin new file mode 100644 index 000000000..2baf93fbf --- /dev/null +++ b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/chunked_and_length.bin @@ -0,0 +1,4 @@ +jHTTP/1.1 200 OK +Transfer-Encoding: chunked +Content-Length: 5 + diff --git a/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/dup_length.bin b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/dup_length.bin new file mode 100644 index 000000000..c31c721ca --- /dev/null +++ b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/dup_length.bin @@ -0,0 +1,5 @@ +ŸHTTP/1.1 200 OK +Content-Length: 5 +Content-Length: 6 + +hello \ No newline at end of file diff --git a/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/http10_close.bin b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/http10_close.bin new file mode 100644 index 000000000..398fbc51f --- /dev/null +++ b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/http10_close.bin @@ -0,0 +1,4 @@ + HTTP/1.0 200 OK +Connection: close +X-Custom: value + diff --git a/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/interim.bin b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/interim.bin new file mode 100644 index 000000000..11fe728ac --- /dev/null +++ b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/interim.bin @@ -0,0 +1,5 @@ +¨HTTP/1.1 100 Continue + +HTTP/1.1 200 OK +Content-Length: 0 + diff --git a/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/lf_only.bin b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/lf_only.bin new file mode 100644 index 000000000..77f5df5ab --- /dev/null +++ b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/lf_only.bin @@ -0,0 +1,4 @@ +ÝHTTP/1.1 502 Bad Gateway +Content-Type: text/plain +X-A: b + diff --git a/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/many_fields.bin b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/many_fields.bin new file mode 100644 index 000000000..78a340c3e --- /dev/null +++ b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/many_fields.bin @@ -0,0 +1,22 @@ +HTTP/1.1 200 OK +X-Field-0: v0 +X-Field-1: v1 +X-Field-2: v2 +X-Field-3: v3 +X-Field-4: v4 +X-Field-5: v5 +X-Field-6: v6 +X-Field-7: v7 +X-Field-8: v8 +X-Field-9: v9 +X-Field-10: v10 +X-Field-11: v11 +X-Field-12: v12 +X-Field-13: v13 +X-Field-14: v14 +X-Field-15: v15 +X-Field-16: v16 +X-Field-17: v17 +X-Field-18: v18 +X-Field-19: v19 + diff --git a/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/no_content.bin b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/no_content.bin new file mode 100644 index 000000000..0aeae015c --- /dev/null +++ b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/no_content.bin @@ -0,0 +1,3 @@ +>HTTP/1.1 204 No Content +Content-Length: 10 + diff --git a/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/not_modified.bin b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/not_modified.bin new file mode 100644 index 000000000..a30a9c427 --- /dev/null +++ b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/not_modified.bin @@ -0,0 +1,3 @@ +sHTTP/1.1 304 Not Modified +Transfer-Encoding: chunked + diff --git a/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/ok_chunked.bin b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/ok_chunked.bin new file mode 100644 index 000000000..8af2544e2 --- /dev/null +++ b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/ok_chunked.bin @@ -0,0 +1,8 @@ +5HTTP/1.1 200 OK +Transfer-Encoding: chunked +Date: Thu, 01 Jan 1970 00:00:00 GMT + +2 +ab +0 + diff --git a/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/ok_length.bin b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/ok_length.bin new file mode 100644 index 000000000..54c5b5208 Binary files /dev/null and b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/ok_length.bin differ diff --git a/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/short_status.bin b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/short_status.bin new file mode 100644 index 000000000..2aeb892ec --- /dev/null +++ b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/short_status.bin @@ -0,0 +1 @@ +±HTTP/1.1 20 \ No newline at end of file diff --git a/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/te_gzip_chunked.bin b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/te_gzip_chunked.bin new file mode 100644 index 000000000..691fa799d --- /dev/null +++ b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/te_gzip_chunked.bin @@ -0,0 +1,5 @@ +9HTTP/1.1 200 OK +Transfer-Encoding: gzip, chunked + +0 + diff --git a/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/te_upper.bin b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/te_upper.bin new file mode 100644 index 000000000..88709a92f --- /dev/null +++ b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/te_upper.bin @@ -0,0 +1,5 @@ +7HTTP/1.1 200 OK +Transfer-Encoding: Chunked + +0 + diff --git a/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/underscore.bin b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/underscore.bin new file mode 100644 index 000000000..759802f1d --- /dev/null +++ b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/underscore.bin @@ -0,0 +1,4 @@ +GHTTP/1.1 200 OK +X_Under: 1 +Set-Cookie: a=b + diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/ascii.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/ascii.bin new file mode 100644 index 000000000..af11e225c Binary files /dev/null and b/fuzzing/fuzz_http_ws_utf8_seed_corpus/ascii.bin differ diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/beyond_max.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/beyond_max.bin new file mode 100644 index 000000000..41167b29b Binary files /dev/null and b/fuzzing/fuzz_http_ws_utf8_seed_corpus/beyond_max.bin differ diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/close_bad_reason.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/close_bad_reason.bin new file mode 100644 index 000000000..a84765e3d --- /dev/null +++ b/fuzzing/fuzz_http_ws_utf8_seed_corpus/close_bad_reason.bin @@ -0,0 +1,2 @@ + +7ú!=èÿ \ No newline at end of file diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/close_code_only.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/close_code_only.bin new file mode 100644 index 000000000..06180a038 --- /dev/null +++ b/fuzzing/fuzz_http_ws_utf8_seed_corpus/close_code_only.bin @@ -0,0 +1 @@ + 7ú!=è \ No newline at end of file diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/close_reason.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/close_reason.bin new file mode 100644 index 000000000..a386754cf --- /dev/null +++ b/fuzzing/fuzz_http_ws_utf8_seed_corpus/close_reason.bin @@ -0,0 +1 @@ + 7ú!=èκόσμε \ No newline at end of file diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/four_byte.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/four_byte.bin new file mode 100644 index 000000000..5aeb69dea Binary files /dev/null and b/fuzzing/fuzz_http_ws_utf8_seed_corpus/four_byte.bin differ diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/kosme.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/kosme.bin new file mode 100644 index 000000000..74ce0e86b Binary files /dev/null and b/fuzzing/fuzz_http_ws_utf8_seed_corpus/kosme.bin differ diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/kosme_surrogate.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/kosme_surrogate.bin new file mode 100644 index 000000000..e232c8332 Binary files /dev/null and b/fuzzing/fuzz_http_ws_utf8_seed_corpus/kosme_surrogate.bin differ diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/long_mixed.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/long_mixed.bin new file mode 100644 index 000000000..ea43304a1 Binary files /dev/null and b/fuzzing/fuzz_http_ws_utf8_seed_corpus/long_mixed.bin differ diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_close_bad_cont_ascii-3b_01.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_close_bad_cont_ascii-3b_01.bin new file mode 100644 index 000000000..b73a0b228 Binary files /dev/null and b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_close_bad_cont_ascii-3b_01.bin differ diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_close_bad_start_ascii-3b_01.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_close_bad_start_ascii-3b_01.bin new file mode 100644 index 000000000..3ad279b27 --- /dev/null +++ b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_close_bad_start_ascii-3b_01.bin @@ -0,0 +1 @@ +Š7\77777777777777777777777777ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ77!!èÿ \ No newline at end of file diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_close_valid_empty_01.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_close_valid_empty_01.bin new file mode 100644 index 000000000..907417b27 --- /dev/null +++ b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_close_valid_empty_01.bin @@ -0,0 +1,2 @@ + +7ú!èÿ \ No newline at end of file diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_close_valid_empty_02.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_close_valid_empty_02.bin new file mode 100644 index 000000000..6a65d311b --- /dev/null +++ b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_close_valid_empty_02.bin @@ -0,0 +1 @@ + < \ No newline at end of file diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-3b-4b_01.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-3b-4b_01.bin new file mode 100644 index 000000000..0ec1623f6 Binary files /dev/null and b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-3b-4b_01.bin differ diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-3b-4b_02.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-3b-4b_02.bin new file mode 100644 index 000000000..34dadffd8 Binary files /dev/null and b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-3b-4b_02.bin differ diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-3b-4b_03.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-3b-4b_03.bin new file mode 100644 index 000000000..a0c61be46 Binary files /dev/null and b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-3b-4b_03.bin differ diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-3b-4b_04.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-3b-4b_04.bin new file mode 100644 index 000000000..f3e4787ed Binary files /dev/null and b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-3b-4b_04.bin differ diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-3b-4b_05.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-3b-4b_05.bin new file mode 100644 index 000000000..aac3d1a38 Binary files /dev/null and b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-3b-4b_05.bin differ diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-3b_01.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-3b_01.bin new file mode 100644 index 000000000..4be939488 Binary files /dev/null and b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-3b_01.bin differ diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-4b_01.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-4b_01.bin new file mode 100644 index 000000000..9cef8b702 Binary files /dev/null and b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-4b_01.bin differ diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-4b_02.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-4b_02.bin new file mode 100644 index 000000000..c11d6c7d8 Binary files /dev/null and b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-4b_02.bin differ diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-4b_03.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-4b_03.bin new file mode 100644 index 000000000..7f79da1c8 Binary files /dev/null and b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-4b_03.bin differ diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-4b_04.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-4b_04.bin new file mode 100644 index 000000000..68e08418f Binary files /dev/null and b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-4b_04.bin differ diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b_01.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b_01.bin new file mode 100644 index 000000000..da1c1c21f Binary files /dev/null and b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b_01.bin differ diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-3b_01.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-3b_01.bin new file mode 100644 index 000000000..9d80b3512 Binary files /dev/null and b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-3b_01.bin differ diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_2b-3b_01.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_2b-3b_01.bin new file mode 100644 index 000000000..bb1b29138 Binary files /dev/null and b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_2b-3b_01.bin differ diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_2b_01.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_2b_01.bin new file mode 100644 index 000000000..cd765d40a Binary files /dev/null and b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_2b_01.bin differ diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_ascii-2b-3b-4b_01.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_ascii-2b-3b-4b_01.bin new file mode 100644 index 000000000..5a3a64588 Binary files /dev/null and b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_ascii-2b-3b-4b_01.bin differ diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_ascii-2b-3b-4b_02.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_ascii-2b-3b-4b_02.bin new file mode 100644 index 000000000..c0d29c982 Binary files /dev/null and b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_ascii-2b-3b-4b_02.bin differ diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_ascii-2b-3b-4b_03.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_ascii-2b-3b-4b_03.bin new file mode 100644 index 000000000..fab4dfd80 Binary files /dev/null and b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_ascii-2b-3b-4b_03.bin differ diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_ascii-2b_01.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_ascii-2b_01.bin new file mode 100644 index 000000000..1b71a3a8d Binary files /dev/null and b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_ascii-2b_01.bin differ diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_ascii-2b_02.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_ascii-2b_02.bin new file mode 100644 index 000000000..18ccd5992 Binary files /dev/null and b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_ascii-2b_02.bin differ diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_ascii-2b_03.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_ascii-2b_03.bin new file mode 100644 index 000000000..9fa87c43b Binary files /dev/null and b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_ascii-2b_03.bin differ diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_ascii_01.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_ascii_01.bin new file mode 100644 index 000000000..1169d79c3 Binary files /dev/null and b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_ascii_01.bin differ diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_ascii_02.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_ascii_02.bin new file mode 100644 index 000000000..46ef263fa Binary files /dev/null and b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_ascii_02.bin differ diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_ascii_03.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_ascii_03.bin new file mode 100644 index 000000000..d09280971 Binary files /dev/null and b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_ascii_03.bin differ diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_ascii_04.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_ascii_04.bin new file mode 100644 index 000000000..f964a5b95 Binary files /dev/null and b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_ascii_04.bin differ diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_truncated_4b_01.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_truncated_4b_01.bin new file mode 100644 index 000000000..f92ca0050 Binary files /dev/null and b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_truncated_4b_01.bin differ diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_valid_ascii-2b-3b-4b_01.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_valid_ascii-2b-3b-4b_01.bin new file mode 100644 index 000000000..3f838f649 Binary files /dev/null and b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_valid_ascii-2b-3b-4b_01.bin differ diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_valid_ascii-2b_01.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_valid_ascii-2b_01.bin new file mode 100644 index 000000000..af3f7af5e Binary files /dev/null and b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_valid_ascii-2b_01.bin differ diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_valid_ascii-2b_02.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_valid_ascii-2b_02.bin new file mode 100644 index 000000000..dacd92046 Binary files /dev/null and b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_valid_ascii-2b_02.bin differ diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_valid_ascii-2b_03.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_valid_ascii-2b_03.bin new file mode 100644 index 000000000..fefba9674 Binary files /dev/null and b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_valid_ascii-2b_03.bin differ diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_valid_ascii_01.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_valid_ascii_01.bin new file mode 100644 index 000000000..f7a8de0e1 Binary files /dev/null and b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_valid_ascii_01.bin differ diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_valid_ascii_02.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_valid_ascii_02.bin new file mode 100644 index 000000000..01f8ad905 Binary files /dev/null and b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_valid_ascii_02.bin differ diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_valid_empty_01.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_valid_empty_01.bin new file mode 100644 index 000000000..3cd6113aa Binary files /dev/null and b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_valid_empty_01.bin differ diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/overlong.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/overlong.bin new file mode 100644 index 000000000..36705d50f Binary files /dev/null and b/fuzzing/fuzz_http_ws_utf8_seed_corpus/overlong.bin differ diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/truncated.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/truncated.bin new file mode 100644 index 000000000..5144de22e Binary files /dev/null and b/fuzzing/fuzz_http_ws_utf8_seed_corpus/truncated.bin differ diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/zero_mask.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/zero_mask.bin new file mode 100644 index 000000000..48ae70390 Binary files /dev/null and b/fuzzing/fuzz_http_ws_utf8_seed_corpus/zero_mask.bin differ diff --git a/fuzzing/fuzz_router_app_response_seed_corpus/no_fields.bin b/fuzzing/fuzz_router_app_response_seed_corpus/no_fields.bin new file mode 100644 index 000000000..8443c13f6 Binary files /dev/null and b/fuzzing/fuzz_router_app_response_seed_corpus/no_fields.bin differ diff --git a/fuzzing/fuzz_router_app_response_seed_corpus/piggyback.bin b/fuzzing/fuzz_router_app_response_seed_corpus/piggyback.bin new file mode 100644 index 000000000..1885f9114 Binary files /dev/null and b/fuzzing/fuzz_router_app_response_seed_corpus/piggyback.bin differ diff --git a/fuzzing/fuzz_router_app_response_seed_corpus/valid_simple.bin b/fuzzing/fuzz_router_app_response_seed_corpus/valid_simple.bin new file mode 100644 index 000000000..f670b8954 Binary files /dev/null and b/fuzzing/fuzz_router_app_response_seed_corpus/valid_simple.bin differ diff --git a/fuzzing/fuzz_unit_msg_seed_corpus/mmap_partial_tail.bin b/fuzzing/fuzz_unit_msg_seed_corpus/mmap_partial_tail.bin new file mode 100644 index 000000000..7372259db Binary files /dev/null and b/fuzzing/fuzz_unit_msg_seed_corpus/mmap_partial_tail.bin differ diff --git a/fuzzing/fuzz_unit_msg_seed_corpus/port_ack.bin b/fuzzing/fuzz_unit_msg_seed_corpus/port_ack.bin new file mode 100644 index 000000000..8e0b7ecb4 Binary files /dev/null and b/fuzzing/fuzz_unit_msg_seed_corpus/port_ack.bin differ diff --git a/fuzzing/fuzz_unit_msg_seed_corpus/req_body.bin b/fuzzing/fuzz_unit_msg_seed_corpus/req_body.bin new file mode 100644 index 000000000..05da4caba Binary files /dev/null and b/fuzzing/fuzz_unit_msg_seed_corpus/req_body.bin differ diff --git a/fuzzing/fuzz_unit_msg_seed_corpus/req_headers_dup_field_inside_fields.bin b/fuzzing/fuzz_unit_msg_seed_corpus/req_headers_dup_field_inside_fields.bin new file mode 100644 index 000000000..75c9e47f3 Binary files /dev/null and b/fuzzing/fuzz_unit_msg_seed_corpus/req_headers_dup_field_inside_fields.bin differ diff --git a/fuzzing/fuzz_unit_msg_seed_corpus/req_headers_fields_body.bin b/fuzzing/fuzz_unit_msg_seed_corpus/req_headers_fields_body.bin new file mode 100644 index 000000000..4da3c0408 Binary files /dev/null and b/fuzzing/fuzz_unit_msg_seed_corpus/req_headers_fields_body.bin differ diff --git a/fuzzing/fuzz_unit_msg_seed_corpus/req_headers_min.bin b/fuzzing/fuzz_unit_msg_seed_corpus/req_headers_min.bin new file mode 100644 index 000000000..9ac6e5e12 Binary files /dev/null and b/fuzzing/fuzz_unit_msg_seed_corpus/req_headers_min.bin differ diff --git a/fuzzing/fuzz_unit_msg_seed_corpus/req_headers_two_records.bin b/fuzzing/fuzz_unit_msg_seed_corpus/req_headers_two_records.bin new file mode 100644 index 000000000..900b78c9e Binary files /dev/null and b/fuzzing/fuzz_unit_msg_seed_corpus/req_headers_two_records.bin differ diff --git a/fuzzing/fuzz_unit_msg_seed_corpus/req_headers_ws_handshake.bin b/fuzzing/fuzz_unit_msg_seed_corpus/req_headers_ws_handshake.bin new file mode 100644 index 000000000..53889aaf3 Binary files /dev/null and b/fuzzing/fuzz_unit_msg_seed_corpus/req_headers_ws_handshake.bin differ diff --git a/fuzzing/fuzz_unit_msg_seed_corpus/rpc_ready_mmap.bin b/fuzzing/fuzz_unit_msg_seed_corpus/rpc_ready_mmap.bin new file mode 100644 index 000000000..5812bfb2b Binary files /dev/null and b/fuzzing/fuzz_unit_msg_seed_corpus/rpc_ready_mmap.bin differ diff --git a/fuzzing/fuzz_unit_msg_seed_corpus/shm_ack.bin b/fuzzing/fuzz_unit_msg_seed_corpus/shm_ack.bin new file mode 100644 index 000000000..e45448a76 Binary files /dev/null and b/fuzzing/fuzz_unit_msg_seed_corpus/shm_ack.bin differ diff --git a/fuzzing/fuzz_unit_msg_seed_corpus/websocket.bin b/fuzzing/fuzz_unit_msg_seed_corpus/websocket.bin new file mode 100644 index 000000000..fb7368938 Binary files /dev/null and b/fuzzing/fuzz_unit_msg_seed_corpus/websocket.bin differ diff --git a/fuzzing/fuzz_unit_msg_seed_corpus/websocket_masked_ext.bin b/fuzzing/fuzz_unit_msg_seed_corpus/websocket_masked_ext.bin new file mode 100644 index 000000000..2ef6610f5 Binary files /dev/null and b/fuzzing/fuzz_unit_msg_seed_corpus/websocket_masked_ext.bin differ diff --git a/fuzzing/nxt_basic_fuzz.c b/fuzzing/nxt_basic_fuzz.c index 5f71a909c..35e21b6ae 100644 --- a/fuzzing/nxt_basic_fuzz.c +++ b/fuzzing/nxt_basic_fuzz.c @@ -190,10 +190,9 @@ nxt_utf8_fuzz(const u_char *data, size_t size) in = data; nxt_utf8_decode(&in, data + size); - nxt_utf8_casecmp((const u_char *)"ABC АБВ ΑΒΓ", - data, - nxt_length("ABC АБВ ΑΒΓ"), - size); + /* nxt_utf8_decode() stops after one sequence; walk the whole input. */ + + nxt_utf8_length(data, size); } diff --git a/fuzzing/nxt_http_chunk_fuzz.c b/fuzzing/nxt_http_chunk_fuzz.c new file mode 100644 index 000000000..6010d9f19 --- /dev/null +++ b/fuzzing/nxt_http_chunk_fuzz.c @@ -0,0 +1,328 @@ +/* + * Copyright (C) FreeUnit contributors. + */ + +#include + + +/* + * This target fuzzes the chunked transfer-coding decoder, + * nxt_http_chunk_parse(). The decoder reads a client request body and, in + * the proxy, an upstream response body. Thus both ends of a connection + * send data to it. + * + * The decoder keeps its state between buffers and between calls, and a + * network read can stop at any byte. Thus the target decodes each input + * two times: one time as a single buffer, and one time cut into pieces that + * it gives to the decoder as chains of one to three buffers for each call. + * The two runs must give the same verdict, the same end position of the + * message, and the same decoded bytes. A difference causes an abort. Thus + * a state that is lost or read incorrectly at a buffer border is a crash, + * not a silent difference. + * + * Input layout: the first byte seeds the cut points, the other bytes are the + * body. + */ + + +#define KMININPUTLENGTH 2 +#define KMAXINPUTLENGTH 4096 + +#define NXT_CHUNK_FUZZ_MAX_PIECES 64 + + +typedef struct { + nxt_uint_t verdict; + size_t consumed; + size_t length; + size_t capacity; + u_char *data; +} nxt_chunk_fuzz_result_t; + + +extern int LLVMFuzzerInitialize(int *argc, char ***argv); +extern int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size); + +static nxt_int_t nxt_chunk_fuzz_run(nxt_task_t *task, nxt_mp_t *mp, + const u_char *body, size_t size, uint32_t seed, + nxt_chunk_fuzz_result_t *res); +static nxt_int_t nxt_chunk_fuzz_collect(nxt_buf_t *out, + nxt_chunk_fuzz_result_t *res); +static nxt_uint_t nxt_chunk_fuzz_verdict(nxt_http_chunk_parse_t *hcp); +static uint32_t nxt_chunk_fuzz_rand(uint32_t *state); + + +extern char **environ; + + +int +LLVMFuzzerInitialize(int *argc, char ***argv) +{ + if (nxt_lib_start("fuzzing", NULL, &environ) != NXT_OK) { + return NXT_ERROR; + } + + return 0; +} + + +int +LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) +{ + nxt_mp_t *mp; + nxt_task_t *task; + nxt_thread_t *thr; + nxt_chunk_fuzz_result_t whole, split; + + if (size < KMININPUTLENGTH || size > KMAXINPUTLENGTH) { + return 0; + } + + thr = nxt_thread(); + task = thr->task; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (mp == NULL) { + return 0; + } + + /* A seed of 0 puts the full body in one buffer: the reference run. */ + + if (nxt_chunk_fuzz_run(task, mp, data + 1, size - 1, 0, &whole) != NXT_OK + || nxt_chunk_fuzz_run(task, mp, data + 1, size - 1, + (uint32_t) data[0] + 1, &split) + != NXT_OK) + { + goto done; + } + + if (whole.verdict != split.verdict + || whole.consumed != split.consumed + || whole.length != split.length + || memcmp(whole.data, split.data, whole.length) != 0) + { + nxt_thread_log_alert("chunk fuzz: split run differs: " + "verdict %ui/%ui consumed %uz/%uz " + "length %uz/%uz", + whole.verdict, split.verdict, + whole.consumed, split.consumed, + whole.length, split.length); + abort(); + } + +done: + + nxt_mp_destroy(mp); + + return 0; +} + + +/* + * Each piece is a separate heap allocation of its exact size. Thus a read + * after the end of a buffer goes into an ASan redzone, not into the next + * piece. The buffer headers and the output buffers of the decoder come from + * "mp". The caller destroys the pool after the two runs. This also frees + * the output buffers that the decoder kept in the pool. + */ + +static nxt_int_t +nxt_chunk_fuzz_run(nxt_task_t *task, nxt_mp_t *mp, const u_char *body, + size_t size, uint32_t seed, nxt_chunk_fuzz_result_t *res) +{ + size_t offset, len, cut; + size_t offsets[NXT_CHUNK_FUZZ_MAX_PIECES]; + nxt_int_t ret; + nxt_buf_t *b, *out, *pieces[NXT_CHUNK_FUZZ_MAX_PIECES]; + nxt_uint_t i, j, n, group; + nxt_http_chunk_parse_t hcp; + + nxt_memzero(res, sizeof(nxt_chunk_fuzz_result_t)); + + res->data = nxt_mp_nget(mp, size); + if (res->data == NULL) { + return NXT_ERROR; + } + + res->capacity = size; + + ret = NXT_ERROR; + + /* Cut the body. */ + + n = 0; + offset = 0; + + while (offset < size) { + len = size - offset; + + if (seed != 0 && n < NXT_CHUNK_FUZZ_MAX_PIECES - 1) { + cut = nxt_chunk_fuzz_rand(&seed) % 16 + 1; + len = nxt_min(len, cut); + } + + b = nxt_buf_mem_alloc(mp, 0, 0); + if (b == NULL) { + goto done; + } + + b->mem.start = nxt_malloc(len); + if (b->mem.start == NULL) { + goto done; + } + + nxt_memcpy(b->mem.start, body + offset, len); + + b->mem.pos = b->mem.start; + b->mem.free = b->mem.start + len; + b->mem.end = b->mem.free; + + offsets[n] = offset; + pieces[n++] = b; + + offset += len; + } + + /* Decode. */ + + nxt_memzero(&hcp, sizeof(nxt_http_chunk_parse_t)); + + hcp.mem_pool = mp; + + /* + * Without this flag, the decoder puts a buffer that gave no data on the + * work queue of the engine. There is no engine here. + */ + hcp.retain_buffers = 1; + + for (i = 0; i < n; i += group) { + group = 1; + + if (seed != 0) { + cut = nxt_chunk_fuzz_rand(&seed) % 3 + 1; + group = nxt_min(n - i, cut); + } + + for (j = i; j < i + group - 1; j++) { + pieces[j]->next = pieces[j + 1]; + } + + pieces[i + group - 1]->next = NULL; + + out = nxt_http_chunk_parse(task, &hcp, pieces[i]); + + if (nxt_chunk_fuzz_collect(out, res) != NXT_OK) { + nxt_thread_log_alert("chunk fuzz: bad output buffer"); + abort(); + } + + if (hcp.error) { + /* No memory. This is not a verdict on the input. */ + goto done; + } + + if (hcp.chunk_error || hcp.last) { + break; + } + } + + res->verdict = nxt_chunk_fuzz_verdict(&hcp); + + /* + * The decoder stops in the buffer that holds the byte that ended the + * message or broke it. It has consumed that byte. The decoder does not + * touch the buffers after it. A buffer before it can keep "pos" at + * "start" when it held only chunk data. Thus the last piece with a + * moved "pos" is the one that holds the byte. + */ + if (hcp.chunk_error || hcp.last) { + for (j = n; j > 0; j--) { + b = pieces[j - 1]; + + if (b->mem.pos != b->mem.start) { + res->consumed = offsets[j - 1] + (b->mem.pos - b->mem.start); + break; + } + } + } + + ret = NXT_OK; + +done: + + for (i = 0; i < n; i++) { + nxt_free(pieces[i]->mem.start); + } + + return ret; +} + + +/* + * A decoded slice must be inside the buffer it was cut from, which is its + * parent. All the slices together cannot hold more bytes than the body. + */ + +static nxt_int_t +nxt_chunk_fuzz_collect(nxt_buf_t *out, nxt_chunk_fuzz_result_t *res) +{ + size_t len; + nxt_buf_t *b, *parent; + + for (b = out; b != NULL; b = b->next) { + parent = b->parent; + + if (parent == NULL + || b->mem.pos > b->mem.free + || b->mem.pos < parent->mem.start + || b->mem.free > parent->mem.end) + { + return NXT_ERROR; + } + + len = b->mem.free - b->mem.pos; + + if (len > res->capacity - res->length) { + return NXT_ERROR; + } + + nxt_memcpy(res->data + res->length, b->mem.pos, len); + res->length += len; + } + + return NXT_OK; +} + + +static nxt_uint_t +nxt_chunk_fuzz_verdict(nxt_http_chunk_parse_t *hcp) +{ + if (hcp->chunk_error) { + return 1; + } + + if (hcp->last) { + return 2; + } + + return 0; +} + + +/* xorshift32: fast, and the same sequence for the same input. */ + +static uint32_t +nxt_chunk_fuzz_rand(uint32_t *state) +{ + uint32_t x; + + x = *state; + + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + + *state = x; + + return x; +} diff --git a/fuzzing/nxt_http_h1p_fuzz.c b/fuzzing/nxt_http_h1p_fuzz.c index 5b7f323ff..459941bef 100644 --- a/fuzzing/nxt_http_h1p_fuzz.c +++ b/fuzzing/nxt_http_h1p_fuzz.c @@ -28,6 +28,9 @@ LLVMFuzzerInitialize(int *argc, char ***argv) return NXT_ERROR; } + /* Keep a fuzzing run quiet: nothing below alert is worth printing. */ + nxt_main_log.level = NXT_LOG_ALERT; + ret = nxt_http_fields_hash(&nxt_h1p_fields_hash, nxt_h1p_fields, nxt_nitems(nxt_h1p_fields)); if (ret != NXT_OK) { @@ -76,6 +79,13 @@ LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) goto failed; } + /* + * A field handler may log, and nxt_log() dereferences task->log. The + * request is zeroed memory here, so without this any handler that logs + * is a null dereference in the harness rather than a finding. + */ + req->task.log = &nxt_main_log; + buf.start = (u_char *)data; buf.end = (u_char *)data + size; buf.pos = buf.start; diff --git a/fuzzing/nxt_http_h1p_peer_fuzz.c b/fuzzing/nxt_http_h1p_peer_fuzz.c index 79145ac78..6774286db 100644 --- a/fuzzing/nxt_http_h1p_peer_fuzz.c +++ b/fuzzing/nxt_http_h1p_peer_fuzz.c @@ -28,6 +28,9 @@ LLVMFuzzerInitialize(int *argc, char ***argv) return NXT_ERROR; } + /* Keep a fuzzing run quiet: nothing below alert is worth printing. */ + nxt_main_log.level = NXT_LOG_ALERT; + ret = nxt_http_fields_hash(&nxt_h1p_peer_fields_hash, nxt_h1p_peer_fields, nxt_nitems(nxt_h1p_peer_fields)); @@ -72,6 +75,13 @@ LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) goto failed; } + /* + * A field handler may log, and nxt_log() dereferences task->log. The + * request is zeroed memory here, so without this any handler that logs + * is a null dereference in the harness rather than a finding. + */ + req->task.log = &nxt_main_log; + buf.start = (u_char *)data; buf.end = (u_char *)data + size; buf.pos = buf.start; diff --git a/fuzzing/nxt_http_h1p_peer_response_fuzz.c b/fuzzing/nxt_http_h1p_peer_response_fuzz.c new file mode 100644 index 000000000..79eaced9b --- /dev/null +++ b/fuzzing/nxt_http_h1p_peer_response_fuzz.c @@ -0,0 +1,358 @@ +/* + * Copyright (C) FreeUnit contributors. + */ + +#include + +/* DO NOT TRY THIS AT HOME! */ +#include "nxt_h1proto.c" + + +/* + * This target fuzzes how the proxy reads an upstream response header: the + * status line and the header fields that nxt_h1p_peer_header_parse() takes + * from the upstream socket, and the peer field handlers that run on them. + * fuzz_http_h1p_peer gets to those handlers through the request parser and + * in one call. This target gets to them in the same way as the proxy. + * + * The proxy reads the header into one buffer. After each read that + * returned NXT_AGAIN, it parses again, and the parser continues from its + * saved state. Thus the target parses each input two times: one time + * complete, and one time a few bytes at a time. The two runs must give the + * same result, the same status, the same end of the header and the same + * fields. A difference causes an abort. + * + * Input layout: the first byte seeds the read sizes, the other bytes are + * the response. + */ + + +#define KMININPUTLENGTH 2 +#define KMAXINPUTLENGTH 4096 + + +typedef struct { + nxt_int_t ret; + nxt_int_t status; + size_t header_size; + nxt_uint_t nfields; + nxt_http_field_t fields[64]; +} nxt_peer_fuzz_result_t; + + +extern int LLVMFuzzerInitialize(int *argc, char ***argv); +extern int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size); + +static nxt_int_t nxt_peer_fuzz_run(nxt_mp_t *mp, u_char *start, + const u_char *src, size_t size, uint32_t seed, nxt_peer_fuzz_result_t *res); +static nxt_int_t nxt_peer_fuzz_result(nxt_int_t ret); +static nxt_bool_t nxt_peer_fuzz_field_eq(nxt_http_field_t *f1, u_char *base1, + nxt_http_field_t *f2, u_char *base2); +static uint32_t nxt_peer_fuzz_rand(uint32_t *state); + + +extern char **environ; + + +int +LLVMFuzzerInitialize(int *argc, char ***argv) +{ + nxt_int_t ret; + + if (nxt_lib_start("fuzzing", NULL, &environ) != NXT_OK) { + return NXT_ERROR; + } + + /* Keep a fuzzing run quiet: nothing below alert is worth printing. */ + nxt_main_log.level = NXT_LOG_ALERT; + + ret = nxt_http_fields_hash(&nxt_h1p_peer_fields_hash, + nxt_h1p_peer_fields, + nxt_nitems(nxt_h1p_peer_fields)); + if (ret != NXT_OK) { + return NXT_ERROR; + } + + return 0; +} + + +int +LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) +{ + u_char *whole_buf, *split_buf; + nxt_mp_t *mp; + nxt_uint_t i; + nxt_peer_fuzz_result_t *whole, *split; + + if (size < KMININPUTLENGTH || size > KMAXINPUTLENGTH) { + return 0; + } + + size--; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (mp == NULL) { + return 0; + } + + /* + * Each run gets its own heap buffer of the exact size. Thus ASan catches + * a read after the received bytes at the end of the input at least, and + * the addresses of the fields show which run they came from. + */ + whole_buf = nxt_malloc(size); + split_buf = nxt_malloc(size); + whole = nxt_mp_zget(mp, sizeof(nxt_peer_fuzz_result_t)); + split = nxt_mp_zget(mp, sizeof(nxt_peer_fuzz_result_t)); + + if (whole_buf == NULL || split_buf == NULL + || whole == NULL || split == NULL) + { + goto done; + } + + /* A seed of 0 gives the full response in one pass: the reference run. */ + + if (nxt_peer_fuzz_run(mp, whole_buf, data + 1, size, 0, whole) != NXT_OK + || nxt_peer_fuzz_run(mp, split_buf, data + 1, size, + (uint32_t) data[0] + 1, split) + != NXT_OK) + { + goto done; + } + + if (whole->ret != split->ret) { + goto differs; + } + + if (whole->ret != NXT_DONE) { + goto done; + } + + if (whole->status != split->status + || whole->header_size != split->header_size + || whole->nfields != split->nfields) + { + goto differs; + } + + for (i = 0; i < whole->nfields; i++) { + if (!nxt_peer_fuzz_field_eq(&whole->fields[i], whole_buf, + &split->fields[i], split_buf)) + { + goto differs; + } + } + +done: + + nxt_free(whole_buf); + nxt_free(split_buf); + + nxt_mp_destroy(mp); + + return 0; + +differs: + + nxt_thread_log_alert("peer response fuzz: split run differs: " + "ret %i/%i status %i/%i header %uz/%uz " + "fields %ui/%ui", + whole->ret, split->ret, + whole->status, split->status, + whole->header_size, split->header_size, + whole->nfields, split->nfields); + abort(); +} + + +static nxt_int_t +nxt_peer_fuzz_run(nxt_mp_t *mp, u_char *start, const u_char *src, size_t size, + uint32_t seed, nxt_peer_fuzz_result_t *res) +{ + size_t len, cut, done; + nxt_int_t ret; + nxt_buf_mem_t bm; + nxt_h1proto_t *h1p; + nxt_http_field_t *f; + nxt_http_peer_t *peer; + nxt_http_request_t *r; + + r = nxt_mp_zget(mp, sizeof(nxt_http_request_t)); + peer = nxt_mp_zget(mp, sizeof(nxt_http_peer_t)); + h1p = nxt_mp_zget(mp, sizeof(nxt_h1proto_t)); + + if (r == NULL || peer == NULL || h1p == NULL) { + return NXT_ERROR; + } + + /* + * A field handler can log, and nxt_log() reads task->log. The request + * is zeroed memory here. Without this line, a handler that logs causes + * a null dereference in the harness, which is not a finding. + */ + r->task.log = &nxt_main_log; + r->mem_pool = mp; + r->peer = peer; + r->resp.content_length_n = -1; + + peer->request = r; + peer->proto.h1 = h1p; + + /* The same values that nxt_h1p_peer_header_send() leaves. */ + peer->status = NXT_HTTP_UNSET; + + ret = nxt_http_parse_request_init(&h1p->parser, mp); + if (ret != NXT_OK) { + return NXT_ERROR; + } + + bm.start = start; + bm.pos = start; + bm.free = start; + bm.end = start + size; + + /* + * Each pass copies the next bytes of the input to bm.free, in the same + * way as a read from the upstream. After a 1xx, the parser can move the + * unread bytes to bm.start and move bm.free back. Thus bm.free is not a + * cursor into the input. "done" is the cursor. + */ + done = 0; + + for ( ;; ) { + len = size - done; + + if (seed != 0) { + cut = nxt_peer_fuzz_rand(&seed) % 32 + 1; + len = nxt_min(len, cut); + } + + nxt_memcpy(bm.free, src + done, len); + bm.free += len; + done += len; + + ret = nxt_h1p_peer_header_parse(peer, &bm); + + if (ret != NXT_AGAIN || done == size) { + break; + } + } + + if (bm.pos < start || bm.pos > bm.free) { + nxt_thread_log_alert("peer response fuzz: pos out of the buffer"); + abort(); + } + + res->ret = nxt_peer_fuzz_result(ret); + + if (ret != NXT_DONE) { + return NXT_OK; + } + + res->status = peer->status; + res->header_size = bm.pos - start; + + /* The same steps that nxt_h1p_peer_header_read_done() does. */ + + peer->num_inline_fields = h1p->parser.num_inline_fields; + if (peer->num_inline_fields > 0) { + nxt_memcpy(peer->inline_fields, h1p->parser.inline_fields, + sizeof(nxt_http_field_t) * peer->num_inline_fields); + } + peer->fields = h1p->parser.fields; + + nxt_http_fields_each(f, peer->inline_fields, peer->num_inline_fields, + peer->fields) + { + if (f->name < start || f->name + f->name_length > bm.pos + || f->value < start || f->value + f->value_length > bm.pos) + { + nxt_thread_log_alert("peer response fuzz: field out of header"); + abort(); + } + + if (res->nfields < nxt_nitems(res->fields)) { + res->fields[res->nfields++] = *f; + } + + } nxt_http_fields_loop; + + ret = nxt_http_fields_process(peer->inline_fields, peer->num_inline_fields, + peer->fields, &nxt_h1p_peer_fields_hash, r); + if (ret != NXT_OK) { + return NXT_ERROR; + } + + /* + * The framing decision of nxt_h1p_peer_header_read_done(), in the same + * order: a response without a body ends at the empty line, whatever the + * fields say. After that, a Transfer-Encoding the proxy cannot decode, + * and chunked together with Content-Length, each end in 502. + */ + + h1p->chunked = (h1p->transfer_encoding == NXT_HTTP_TE_CHUNKED); + + if (nxt_http_request_is_bodyless_final(r, peer->status)) { + h1p->chunked = 0; + + } else if (h1p->transfer_encoding == NXT_HTTP_TE_UNSUPPORTED + || (h1p->chunked && r->resp.content_length != NULL)) + { + res->status = NXT_HTTP_BAD_GATEWAY; + } + + return NXT_OK; +} + + +/* + * The error that a bad header ends in can depend on where it was cut. A + * field name that is longer than its limit is "too large" when the parser + * stops in it, but "invalid" when the bad byte after it is already there. + * Only the outcome must be the same. + */ + +static nxt_int_t +nxt_peer_fuzz_result(nxt_int_t ret) +{ + switch (ret) { + case NXT_DONE: + case NXT_AGAIN: + return ret; + default: + return NXT_ERROR; + } +} + + +static nxt_bool_t +nxt_peer_fuzz_field_eq(nxt_http_field_t *f1, u_char *base1, + nxt_http_field_t *f2, u_char *base2) +{ + return f1->hash == f2->hash + && f1->name_length == f2->name_length + && f1->value_length == f2->value_length + && f1->name - base1 == f2->name - base2 + && f1->value - base1 == f2->value - base2; +} + + +/* xorshift32: fast, and the same sequence for the same input. */ + +static uint32_t +nxt_peer_fuzz_rand(uint32_t *state) +{ + uint32_t x; + + x = *state; + + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + + *state = x; + + return x; +} diff --git a/fuzzing/nxt_http_ws_utf8_fuzz.c b/fuzzing/nxt_http_ws_utf8_fuzz.c new file mode 100644 index 000000000..76020faf5 --- /dev/null +++ b/fuzzing/nxt_http_ws_utf8_fuzz.c @@ -0,0 +1,324 @@ +/* + * Copyright (C) FreeUnit contributors. + */ + +#include + +/* DO NOT TRY THIS AT HOME! */ +#include "nxt_h1proto_websocket.c" + + +/* + * This target fuzzes the UTF-8 check that the router does on WebSocket text + * messages and close reasons, nxt_h1p_ws_utf8_validate(). The client + * controls each byte of the input: the text, the mask, and how the message + * is split into frames. The check reads masked bytes, walks a frame that + * spans several buffers, and keeps a character that a frame border cut for + * the next frame. + * + * The target compares the verdict with a small independent validator + * written from the table of well-formed byte sequences in the Unicode + * standard (Table 3-7). RFC 6455 Section 8.1 says that the connection must + * fail as soon as the text cannot become valid. Thus the verdict must be + * the same after each frame, not only at the end of the message. A + * difference causes an abort. + * + * Input layout: + * byte 0 flags; bit 0 checks a close frame reason, not a message + * byte 1 seeds the frame and buffer cuts + * bytes 2-5 the masking key + * the rest the unmasked payload + */ + + +#define KMININPUTLENGTH 6 +#define KMAXINPUTLENGTH 4096 + +#define NXT_WS_FUZZ_MAX_BUFS 8 + + +typedef struct { + uint8_t need; + u_char lo; + u_char hi; + uint8_t failed; +} nxt_ws_fuzz_ref_t; + + +extern int LLVMFuzzerInitialize(int *argc, char ***argv); +extern int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size); + +static nxt_int_t nxt_ws_fuzz_frame(nxt_mp_t *mp, nxt_h1p_ws_utf8_t *state, + const u_char *payload, size_t size, size_t skip, const u_char *mask, + nxt_uint_t final, uint32_t *seed, nxt_int_t *res); +static void nxt_ws_fuzz_ref(nxt_ws_fuzz_ref_t *ref, const u_char *p, + size_t size); +static uint32_t nxt_ws_fuzz_rand(uint32_t *state); + + +extern char **environ; + + +int +LLVMFuzzerInitialize(int *argc, char ***argv) +{ + if (nxt_lib_start("fuzzing", NULL, &environ) != NXT_OK) { + return NXT_ERROR; + } + + return 0; +} + + +int +LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) +{ + size_t len, cut, skip; + uint32_t seed; + nxt_mp_t *mp; + nxt_int_t res, expect; + nxt_uint_t close, final; + const u_char *mask, *p, *end; + nxt_ws_fuzz_ref_t ref; + nxt_h1p_ws_utf8_t state; + + if (size < KMININPUTLENGTH || size > KMAXINPUTLENGTH) { + return 0; + } + + close = data[0] & 1; + seed = (uint32_t) data[1] + 1; + mask = &data[2]; + + p = data + 6; + end = data + size; + + /* A close frame has a code of two bytes before the reason. */ + if (close && end - p < 2) { + return 0; + } + + mp = nxt_mp_create(1024, 128, 256, 32); + if (mp == NULL) { + return 0; + } + + nxt_memzero(&state, sizeof(nxt_h1p_ws_utf8_t)); + nxt_memzero(&ref, sizeof(nxt_ws_fuzz_ref_t)); + + do { + len = end - p; + skip = 0; + + if (close) { + /* A close frame is one complete frame. */ + skip = 2; + + } else { + /* Empty frames are permitted, so a cut can be 0. */ + cut = nxt_ws_fuzz_rand(&seed) % 64; + len = nxt_min(len, cut); + } + + final = (p + len == end); + + if (nxt_ws_fuzz_frame(mp, &state, p, len, skip, mask, final, &seed, + &res) + != NXT_OK) + { + goto done; + } + + nxt_ws_fuzz_ref(&ref, p + skip, len - skip); + + expect = (ref.failed || (final && ref.need != 0)) ? NXT_ERROR : NXT_OK; + + if (res != expect) { + nxt_thread_log_alert("ws utf8 fuzz: frame at %uz of %uz bytes: " + "%i, expected %i", + (size_t) (p - data - 6), len, res, expect); + abort(); + } + + p += len; + + } while (res == NXT_OK && p < end); + +done: + + nxt_mp_destroy(mp); + + return 0; +} + + +/* + * Masks the payload of one frame into a chain of buffers, then runs the + * check on it. Each buffer is a heap allocation of its exact size, so that + * ASan catches a read after the end of any buffer. The first buffer starts + * with a few bytes that stand for the frame header: the check must start at + * "start", not at the buffer. + */ + +static nxt_int_t +nxt_ws_fuzz_frame(nxt_mp_t *mp, nxt_h1p_ws_utf8_t *state, const u_char *payload, + size_t size, size_t skip, const u_char *mask, nxt_uint_t final, + uint32_t *seed, nxt_int_t *res) +{ + size_t hsize, len, cut, i, offset; + u_char *start; + nxt_int_t ret; + nxt_buf_t *b, *first, **prev; + nxt_uint_t n, k; + u_char *mem[NXT_WS_FUZZ_MAX_BUFS]; + + ret = NXT_ERROR; + + first = NULL; + prev = &first; + start = NULL; + + n = 0; + offset = 0; + hsize = nxt_ws_fuzz_rand(seed) % 13 + 2; + + do { + len = size - offset; + + if (n < NXT_WS_FUZZ_MAX_BUFS - 1) { + cut = nxt_ws_fuzz_rand(seed) % 24 + 1; + len = nxt_min(len, cut); + } + + if (n == 0) { + len += hsize; + } + + b = nxt_buf_mem_alloc(mp, 0, 0); + if (b == NULL) { + goto done; + } + + mem[n] = nxt_malloc(len); + if (mem[n] == NULL) { + goto done; + } + + b->mem.start = mem[n++]; + b->mem.pos = b->mem.start; + b->mem.free = b->mem.start + len; + b->mem.end = b->mem.free; + + i = 0; + + if (n == 1) { + nxt_memset(b->mem.start, 0x81, hsize); + start = b->mem.start + hsize; + i = hsize; + } + + for ( /* void */ ; i < len; i++) { + b->mem.start[i] = payload[offset] ^ mask[offset % 4]; + offset++; + } + + *prev = b; + prev = &b->next; + + } while (offset < size); + + *res = nxt_h1p_ws_utf8_validate(state, first, start, skip, size - skip, + mask, final); + + ret = NXT_OK; + +done: + + for (k = 0; k < n; k++) { + nxt_free(mem[k]); + } + + return ret; +} + + +/* Unicode Table 3-7, Well-Formed UTF-8 Byte Sequences. */ + +static void +nxt_ws_fuzz_ref(nxt_ws_fuzz_ref_t *ref, const u_char *p, size_t size) +{ + u_char c; + const u_char *end; + + for (end = p + size; p < end && !ref->failed; p++) { + c = *p; + + if (ref->need != 0) { + if (c < ref->lo || c > ref->hi) { + ref->failed = 1; + return; + } + + ref->need--; + ref->lo = 0x80; + ref->hi = 0xBF; + continue; + } + + ref->lo = 0x80; + ref->hi = 0xBF; + + if (c < 0x80) { + continue; + } + + if (c >= 0xC2 && c <= 0xDF) { + ref->need = 1; + + } else if (c == 0xE0) { + ref->need = 2; + ref->lo = 0xA0; + + } else if (c == 0xED) { + ref->need = 2; + ref->hi = 0x9F; + + } else if (c >= 0xE1 && c <= 0xEF) { + ref->need = 2; + + } else if (c == 0xF0) { + ref->need = 3; + ref->lo = 0x90; + + } else if (c >= 0xF1 && c <= 0xF3) { + ref->need = 3; + + } else if (c == 0xF4) { + ref->need = 3; + ref->hi = 0x8F; + + } else { + ref->failed = 1; + return; + } + } +} + + +/* xorshift32: fast, and the same sequence for the same input. */ + +static uint32_t +nxt_ws_fuzz_rand(uint32_t *state) +{ + uint32_t x; + + x = *state; + + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + + *state = x; + + return x; +} diff --git a/fuzzing/nxt_router_app_response_fuzz.c b/fuzzing/nxt_router_app_response_fuzz.c new file mode 100644 index 000000000..b3bf634d0 --- /dev/null +++ b/fuzzing/nxt_router_app_response_fuzz.c @@ -0,0 +1,95 @@ +/* + * Copyright (C) FreeUnit contributors. + */ + +/* + * libFuzzer target for nxt_router_response_header_parse() in + * src/nxt_router.c: the router decoding the nxt_unit_response_t that an + * untrusted application handed back. The fuzz input is the response buffer, + * byte for byte: the header, the field array and whatever the sptr offsets + * point at. + */ + +#include + +/* DO NOT TRY THIS AT HOME! */ +#include "nxt_router.c" + + +#define KMININPUTLENGTH sizeof(nxt_unit_response_t) +#define KMAXINPUTLENGTH (64 * 1024) + + +extern int LLVMFuzzerInitialize(int *argc, char ***argv); +extern int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size); + +extern char **environ; + + +/* A bad response is answered with nxt_alert(), which no log level silences. */ +static void nxt_cdecl +nxt_fuzz_log_handler(nxt_uint_t level, nxt_log_t *log, const char *fmt, ...) +{ +} + + +int +LLVMFuzzerInitialize(int *argc, char ***argv) +{ + if (nxt_lib_start("fuzzing", NULL, &environ) != NXT_OK) { + return NXT_ERROR; + } + + nxt_main_log.handler = nxt_fuzz_log_handler; + + if (nxt_http_response_hash_init(NULL) != NXT_OK) { + return NXT_ERROR; + } + + return 0; +} + + +int +LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) +{ + nxt_mp_t *mp; + u_char *buf; + nxt_buf_t b; + nxt_http_request_t r; + + if (size < KMININPUTLENGTH || size > KMAXINPUTLENGTH) { + return 0; + } + + mp = nxt_mp_create(1024, 128, 256, 32); + if (mp == NULL) { + return 0; + } + + /* Exactly the fuzzer's bytes, so ASan catches a sptr resolved outside. */ + buf = nxt_mp_alloc(mp, size); + if (buf == NULL) { + goto failed; + } + + memcpy(buf, data, size); + + nxt_memzero(&b, sizeof(nxt_buf_t)); + b.mem.start = buf; + b.mem.pos = buf; + b.mem.free = buf + size; + b.mem.end = buf + size; + + nxt_memzero(&r, sizeof(nxt_http_request_t)); + r.mem_pool = mp; + r.task.log = &nxt_main_log; + + (void) nxt_router_response_header_parse(&r.task, &r, &b); + +failed: + + nxt_mp_destroy(mp); + + return 0; +} diff --git a/fuzzing/nxt_unit_msg_fuzz.c b/fuzzing/nxt_unit_msg_fuzz.c new file mode 100644 index 000000000..04e10225d --- /dev/null +++ b/fuzzing/nxt_unit_msg_fuzz.c @@ -0,0 +1,481 @@ +/* + * Copyright (C) FreeUnit contributors. + */ + +/* + * libFuzzer target for libunit's receive side: one port message, as the + * router would send it, through nxt_unit_process_msg() -- the mmap record + * walk (nxt_unit_mmap_read()), request arrival validation + * (nxt_unit_process_req_headers(), nxt_unit_sptr_in_buf()), request body + * and websocket frames -- and, for a request that comes out ready, the + * application's request handler, which reads every field and finishes it, + * or upgrades it when it is a websocket handshake so that frames on the + * same stream reach the frame parser (nxt_unit_process_websocket()). + * + * Input layout: + * + * [0..1] little-endian length L of the port message (capped at the + * 16 KB read buffer) + * [2..2+L) the port message: nxt_port_msg_t, then its payload -- for + * an mmap message an array of nxt_port_mmap_msg_t + * [2+L..) the shared memory payload (up to 1 MB), placed in chunks + * of segment 0 that the records then point into + * + * The trust model is libunit's: the router is the peer. The harness plays + * a correct router for everything that is its job rather than the + * message's, and leaves the message itself to the fuzzer: + * + * - Segment 0 is a real shared memory segment, handed to libunit the way + * the router hands one over. The payload goes into chunks the harness + * allocates in the segment's free map, as the router does, and a record + * whose (chunk_id, size) is in range is moved into that allocation: + * without that, the next input overwrites chunks a request that is + * still waiting for its body holds -- memory the router would never + * reuse -- and the "finding" is the harness's own. Out-of-range + * records are left alone, to exercise the range check. + * - Every whole record's mmap_id is forced to 0 unless it is 0xFFFFFFFF. + * Any other id names a segment libunit does not have, and libunit then + * (correctly) parks the message and asks the router for it; parked + * messages would pile up across runs. The id is not capped by design + * (#172), so an id near 2^32 would also grow the segment array to that + * size -- an allocation, not a finding. + * - QUIT and REMOVE_PID are skipped: they tear down state every later run + * depends on. + * - The sender pid and reply port are set to the router port libunit was + * started with, so that a request can complete instead of waiting for a + * port that never arrives. + * + * Requests that wait for a body keep their chunks, and nothing may ever + * send that body; when the segment runs out, the whole libunit instance is + * torn down (which finishes those requests) and set up again. + * + * Built only with --fuzz, which also compiles in the + * nxt_unit_test_process_msg() hook (NXT_FUZZ_BUILD). + */ + +#include +#include +#include +#include +#include +#include + +#include +#include + + +#define NXT_UNIT_MSG_FUZZ_MSG_MAX 16384 +#define NXT_UNIT_MSG_FUZZ_DATA_MAX (1024 * 1024) +#define NXT_UNIT_MSG_FUZZ_ROUTER_PORT 2 + + +extern int LLVMFuzzerInitialize(int *argc, char ***argv); +extern int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size); + + +static nxt_unit_ctx_t *nxt_unit_msg_fuzz_ctx; +static nxt_port_mmap_header_t *nxt_unit_msg_fuzz_hdr; +static int nxt_unit_msg_fuzz_fds[3] = { -1, -1, -1 }; +static int nxt_unit_msg_fuzz_null = -1; +static volatile uint64_t nxt_unit_msg_fuzz_sink; + + +static u_char +nxt_unit_msg_fuzz_last(nxt_unit_sptr_t *sptr, uint32_t length) +{ + return length == 0 ? 0 + : ((u_char *) nxt_unit_sptr_get(sptr))[length - 1]; +} + + +static void +nxt_unit_msg_fuzz_handler(nxt_unit_request_info_t *req) +{ + uint32_t i; + uint64_t sum; + nxt_unit_field_t *f; + nxt_unit_request_t *r; + + /* As the Python and Java modules do before reading the fields. */ + nxt_unit_request_group_dup_fields(req); + + r = req->request; + sum = 0; + + /* + * Touch what an application reads: the last byte of every string the + * arrival checks vetted, at the length they vetted. The request lives + * in the shared segment, which ASan does not shadow, so what this + * catches is a read that leaves the mapping altogether. + */ + sum += nxt_unit_msg_fuzz_last(&r->method, r->method_length); + sum += nxt_unit_msg_fuzz_last(&r->version, r->version_length); + sum += nxt_unit_msg_fuzz_last(&r->remote, r->remote_length); + sum += nxt_unit_msg_fuzz_last(&r->local_addr, r->local_addr_length); + sum += nxt_unit_msg_fuzz_last(&r->local_port, r->local_port_length); + sum += nxt_unit_msg_fuzz_last(&r->server_name, r->server_name_length); + sum += nxt_unit_msg_fuzz_last(&r->target, r->target_length); + sum += nxt_unit_msg_fuzz_last(&r->path, r->path_length); + sum += nxt_unit_msg_fuzz_last(&r->query, r->query_length); + + for (i = 0; i < r->fields_count; i++) { + f = &r->fields[i]; + + sum += nxt_unit_msg_fuzz_last(&f->name, f->name_length); + sum += nxt_unit_msg_fuzz_last(&f->value, f->value_length); + } + + if (r->content_length_field != NXT_UNIT_NONE_FIELD) { + sum += r->fields[r->content_length_field].hash; + } + + if (r->content_type_field != NXT_UNIT_NONE_FIELD) { + sum += r->fields[r->content_type_field].hash; + } + + if (r->cookie_field != NXT_UNIT_NONE_FIELD) { + sum += r->fields[r->cookie_field].hash; + } + + if (r->authorization_field != NXT_UNIT_NONE_FIELD) { + sum += r->fields[r->authorization_field].hash; + } + + nxt_unit_msg_fuzz_sink = sum; + + /* + * A handshake is answered with an upgrade, as a websocket application + * would: the request then waits in the hash for WEBSOCKET messages on + * its stream, which is the only way a frame reaches the frame parser. + * The close frame (or the reset) finishes it. + */ + if (r->websocket_handshake + && nxt_unit_response_init(req, 101, 0, 0) == NXT_UNIT_OK + && nxt_unit_response_upgrade(req) == NXT_UNIT_OK + && nxt_unit_response_send(req) == NXT_UNIT_OK) + { + return; + } + + nxt_unit_request_done(req, NXT_UNIT_ERROR); +} + + +static void +nxt_unit_msg_fuzz_websocket(nxt_unit_websocket_frame_t *ws) +{ + u_char buf[64]; + ssize_t n; + + /* Reads the payload through the mask, as an application would. */ + n = nxt_unit_websocket_read(ws, buf, sizeof(buf)); + + nxt_unit_msg_fuzz_sink = ws->payload_len + ws->header->opcode + + (n > 0 ? buf[n - 1] : 0); + + nxt_unit_websocket_done(ws); +} + + +static void +nxt_unit_msg_fuzz_close(nxt_unit_request_info_t *req) +{ + nxt_unit_request_done(req, NXT_UNIT_ERROR); +} + + +static ssize_t +nxt_unit_msg_fuzz_send(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, + const void *buf, size_t buf_size, const void *oob, size_t oob_size) +{ + return buf_size; +} + + +static ssize_t +nxt_unit_msg_fuzz_recv(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, void *buf, + size_t buf_size, void *oob, size_t *oob_size) +{ + *oob_size = 0; + + return 0; +} + + +static int +nxt_unit_msg_fuzz_shm(size_t size) +{ + int fd; + + fd = syscall(SYS_memfd_create, "unit.msg.fuzz", 0); + if (fd == -1) { + return -1; + } + + if (ftruncate(fd, size) == -1) { + close(fd); + return -1; + } + + return fd; +} + + +/* A libunit instance with segment 0 handed over, as a router would. */ +static int +nxt_unit_msg_fuzz_setup(void) +{ + int fd, ready[2], router[2], read[2], shared[2]; + nxt_port_msg_t msg; + nxt_unit_init_t init; + nxt_port_mmap_header_t *hdr; + + if (socketpair(AF_UNIX, SOCK_DGRAM, 0, ready) == -1 + || socketpair(AF_UNIX, SOCK_DGRAM, 0, router) == -1 + || socketpair(AF_UNIX, SOCK_DGRAM, 0, read) == -1 + || socketpair(AF_UNIX, SOCK_DGRAM, 0, shared) == -1) + { + return -1; + } + + /* The peer ends, which libunit is not given and does not close. */ + nxt_unit_msg_fuzz_fds[0] = ready[1]; + nxt_unit_msg_fuzz_fds[1] = router[1]; + nxt_unit_msg_fuzz_fds[2] = shared[1]; + + memset(&init, 0, sizeof(init)); + + init.callbacks.request_handler = nxt_unit_msg_fuzz_handler; + init.callbacks.websocket_handler = nxt_unit_msg_fuzz_websocket; + init.callbacks.close_handler = nxt_unit_msg_fuzz_close; + init.callbacks.port_send = nxt_unit_msg_fuzz_send; + init.callbacks.port_recv = nxt_unit_msg_fuzz_recv; + + init.ready_port.id.pid = getpid(); + init.ready_port.id.id = 1; + init.ready_port.in_fd = -1; + init.ready_port.out_fd = ready[0]; + init.ready_stream = 1; + + init.router_port.id.pid = getpid(); + init.router_port.id.id = NXT_UNIT_MSG_FUZZ_ROUTER_PORT; + init.router_port.in_fd = -1; + init.router_port.out_fd = router[0]; + + init.read_port.id.pid = getpid(); + init.read_port.id.id = 3; + init.read_port.in_fd = read[0]; + init.read_port.out_fd = read[1]; + + init.shared_port_fd = shared[0]; + init.shared_queue_fd = nxt_unit_msg_fuzz_shm(sizeof(nxt_app_queue_t)); + + /* libunit's own log lines would drown libFuzzer's. */ + init.log_fd = nxt_unit_msg_fuzz_null; + + if (init.shared_queue_fd == -1) { + return -1; + } + + nxt_unit_msg_fuzz_ctx = nxt_unit_init(&init); + if (nxt_unit_msg_fuzz_ctx == NULL) { + return -1; + } + + fd = nxt_unit_msg_fuzz_shm(PORT_MMAP_SIZE); + if (fd == -1) { + return -1; + } + + hdr = mmap(NULL, PORT_MMAP_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, fd, + 0); + if (hdr == MAP_FAILED) { + return -1; + } + + /* As nxt_port_new_port_mmap() initialises a segment. */ + memset(hdr->free_map, 0xFF, sizeof(hdr->free_map)); + nxt_port_mmap_set_chunk_busy(hdr->free_map, PORT_MMAP_CHUNK_COUNT); + + hdr->id = 0; + hdr->src_pid = getpid(); + hdr->dst_pid = getpid(); + hdr->sent_over = 0xFFFFu; + + nxt_unit_msg_fuzz_hdr = hdr; + + memset(&msg, 0, sizeof(msg)); + + msg.pid = getpid(); + msg.type = _NXT_PORT_MSG_MMAP; + msg.last = 1; + + /* libunit maps the segment itself and closes the descriptor. */ + if (nxt_unit_test_process_msg(nxt_unit_msg_fuzz_ctx, &msg, sizeof(msg), + fd) + != NXT_UNIT_OK) + { + return -1; + } + + return 0; +} + + +static int +nxt_unit_msg_fuzz_reset(void) +{ + int i; + + nxt_unit_done(nxt_unit_msg_fuzz_ctx); + + munmap(nxt_unit_msg_fuzz_hdr, PORT_MMAP_SIZE); + + for (i = 0; i < 3; i++) { + close(nxt_unit_msg_fuzz_fds[i]); + nxt_unit_msg_fuzz_fds[i] = -1; + } + + return nxt_unit_msg_fuzz_setup(); +} + + +/* "n" consecutive free chunks, marked busy, as the router allocates. */ +static int +nxt_unit_msg_fuzz_alloc(size_t n, nxt_chunk_id_t *base) +{ + nxt_free_map_t *m; + nxt_chunk_id_t c, i; + + m = nxt_unit_msg_fuzz_hdr->free_map; + + for (c = 0; c + n <= PORT_MMAP_CHUNK_COUNT; c += i + 1) { + + for (i = 0; i < n; i++) { + if ((m[FREE_IDX(c + i)] & FREE_MASK(c + i)) == 0) { + break; + } + } + + if (i == n) { + for (i = 0; i < n; i++) { + nxt_port_mmap_set_chunk_busy(m, c + i); + } + + *base = c; + + return 0; + } + } + + return -1; +} + + +int +LLVMFuzzerInitialize(int *argc, char ***argv) +{ + nxt_unit_msg_fuzz_null = open("/dev/null", O_WRONLY); + + if (nxt_unit_msg_fuzz_null == -1 || nxt_unit_msg_fuzz_setup() != 0) { + abort(); + } + + return 0; +} + + +int +LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) +{ + u_char buf[NXT_UNIT_MSG_FUZZ_MSG_MAX]; + size_t len, dlen, n, off, nchunks, room; + nxt_chunk_id_t base, c; + nxt_port_msg_t *msg; + nxt_port_mmap_msg_t rec; + + if (size < 2) { + return 0; + } + + len = data[0] | (data[1] << 8); + data += 2; + size -= 2; + + len = nxt_min(len, nxt_min(size, sizeof(buf))); + + memcpy(buf, data, len); + + if (len < sizeof(nxt_port_msg_t)) { + /* Still worth sending: the "too small" path. */ + (void) nxt_unit_test_process_msg(nxt_unit_msg_fuzz_ctx, buf, len, -1); + return 0; + } + + msg = (nxt_port_msg_t *) buf; + + if (msg->type == _NXT_PORT_MSG_QUIT + || msg->type == _NXT_PORT_MSG_REMOVE_PID) + { + return 0; + } + + /* + * The reply goes to the router port libunit was started with; any other + * (pid, port) is unknown, and a request for it waits for a NEW_PORT + * that never comes. + */ + msg->pid = getpid(); + msg->reply_port = NXT_UNIT_MSG_FUZZ_ROUTER_PORT; + + base = 0; + n = 0; + + if (msg->mmap && len >= sizeof(nxt_port_msg_t) + + sizeof(nxt_port_mmap_msg_t)) + { + dlen = nxt_min(size - len, NXT_UNIT_MSG_FUZZ_DATA_MAX); + n = (dlen + PORT_MMAP_CHUNK_SIZE - 1) / PORT_MMAP_CHUNK_SIZE; + n = nxt_max(n, 1); + + if (nxt_unit_msg_fuzz_alloc(n, &base) != 0) { + if (nxt_unit_msg_fuzz_reset() != 0) { + abort(); + } + + if (nxt_unit_msg_fuzz_alloc(n, &base) != 0) { + abort(); + } + } + + memcpy(nxt_port_mmap_chunk_start(nxt_unit_msg_fuzz_hdr, base), + data + len, dlen); + + for (off = sizeof(nxt_port_msg_t); + off + sizeof(nxt_port_mmap_msg_t) <= len; + off += sizeof(nxt_port_mmap_msg_t)) + { + memcpy(&rec, buf + off, sizeof(rec)); + + if (rec.mmap_id != 0xFFFFFFFF) { + rec.mmap_id = 0; + } + + if (nxt_port_mmap_chunk_range_valid(rec.chunk_id, rec.size, + &nchunks)) + { + c = base + rec.chunk_id % n; + room = (base + n - c) * PORT_MMAP_CHUNK_SIZE; + + rec.chunk_id = c; + rec.size = nxt_min(rec.size, room); + } + + memcpy(buf + off, &rec, sizeof(rec)); + } + } + + (void) nxt_unit_test_process_msg(nxt_unit_msg_fuzz_ctx, buf, len, -1); + + return 0; +} diff --git a/fuzzing/oss-fuzz.sh b/fuzzing/oss-fuzz.sh index 1fe5ecda8..abd142f8a 100644 --- a/fuzzing/oss-fuzz.sh +++ b/fuzzing/oss-fuzz.sh @@ -12,6 +12,7 @@ pushd fuzzing/ cp fuzz_http.dict $OUT/fuzz_http_controller.dict cp fuzz_http.dict $OUT/fuzz_http_h1p.dict cp fuzz_http.dict $OUT/fuzz_http_h1p_peer.dict +cp fuzz_http.dict $OUT/fuzz_http_h1p_peer_response.dict # Create temporary directories. cp -r fuzz_http_seed_corpus/ fuzz_http_controller_seed_corpus/ @@ -23,6 +24,9 @@ zip -r $OUT/fuzz_http_controller_seed_corpus.zip fuzz_http_controller_seed_corp zip -r $OUT/fuzz_http_h1p_seed_corpus.zip fuzz_http_h1p_seed_corpus/ zip -r $OUT/fuzz_http_h1p_peer_seed_corpus.zip fuzz_http_h1p_peer_seed_corpus/ zip -r $OUT/fuzz_json_seed_corpus.zip fuzz_json_seed_corpus/ +zip -r $OUT/fuzz_http_chunk_seed_corpus.zip fuzz_http_chunk_seed_corpus/ +zip -r $OUT/fuzz_http_h1p_peer_response_seed_corpus.zip fuzz_http_h1p_peer_response_seed_corpus/ +zip -r $OUT/fuzz_http_ws_utf8_seed_corpus.zip fuzz_http_ws_utf8_seed_corpus/ # Delete temporary directories. rm -r fuzz_http_controller_seed_corpus/ fuzz_http_h1p_seed_corpus/ fuzz_http_h1p_peer_seed_corpus/ diff --git a/fuzzing/run-ci.sh b/fuzzing/run-ci.sh new file mode 100755 index 000000000..61e0127ef --- /dev/null +++ b/fuzzing/run-ci.sh @@ -0,0 +1,193 @@ +#!/bin/sh +# +# Run libFuzzer targets for a bounded time against their seed corpora and fail +# on the first crash. This is a smoke gate, not a fuzzing campaign: the +# question it answers is "does the HTTP parser still survive its own seed +# corpus and a minute of shallow mutation", which is the part worth blocking a +# pull request on. +# +# Usage: +# fuzzing/run-ci.sh [-t SECONDS] [TARGET ...] +# +# -t SECONDS libFuzzer budget per target (default 60) +# TARGET ... fuzzer names; default is all ten +# +# Expects the fuzzers to be built already: +# CC=clang CXX=clang++ \ +# CFLAGS="-g -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION \ +# -fsanitize=address,undefined -fsanitize=fuzzer-no-link" \ +# ./configure --no-regex --no-pcre2 --fuzz=-fsanitize=fuzzer && make fuzz +# +# Any crash, leak or sanitizer report leaves the reproducer in +# build/fuzz-artifacts/ for CI to upload. + +set -eu + +BUDGET=60 +BUILD=${BUILD_DIR:-build} + +die() { echo "run-ci.sh: $*" >&2; exit 1; } + +while getopts 't:' opt; do + case "$opt" in + t) BUDGET=$OPTARG ;; + *) die "usage: $0 [-t SECONDS] [TARGET ...]" ;; + esac +done +shift $((OPTIND - 1)) + +# The budget reaches this from a workflow input and is then used in shell +# arithmetic, where a non-numeric value is at best a confusing error. +case "$BUDGET" in + ''|*[!0-9]*) die "budget must be a whole number of seconds, got '$BUDGET'" ;; +esac + +# Seed corpus and dictionary per target. The fuzz_http_* targets that take a +# request share the HTTP corpus. The other targets have a corpus of their own. +corpus_for() { + case "$1" in + fuzz_basic) echo fuzz_basic_seed_corpus ;; + fuzz_json) echo fuzz_json_seed_corpus ;; + fuzz_unit_msg) echo fuzz_unit_msg_seed_corpus ;; + fuzz_http_chunk | fuzz_http_h1p_peer_response | fuzz_http_ws_utf8) + echo "$1_seed_corpus" ;; + fuzz_router_app_response) + echo fuzz_router_app_response_seed_corpus ;; + fuzz_http_*) echo fuzz_http_seed_corpus ;; + *) die "unknown target: $1" ;; + esac +} + +# fuzz_http.dict holds request words. They also help the upstream response +# header, but not a chunked body or WebSocket text. +dict_for() { + case "$1" in + fuzz_http_chunk | fuzz_http_ws_utf8) + echo "" ;; + fuzz_http_*) echo fuzz_http.dict ;; + *) echo "" ;; + esac +} + +[ $# -gt 0 ] || set -- fuzz_basic fuzz_json fuzz_http_controller \ + fuzz_http_h1p fuzz_http_h1p_peer \ + fuzz_http_h1p_peer_response fuzz_http_chunk \ + fuzz_http_ws_utf8 fuzz_router_app_response \ + fuzz_unit_msg + +# Freeze the list: the loop below rebuilds "$@" to hold each target's libFuzzer +# arguments, so it cannot also be the list being iterated. +TARGETS=$* + +# -fsanitize=undefined is recoverable by default: UBSan prints the report and +# execution continues, so libFuzzer exits 0 and a green check hides the +# finding. halt_on_error turns a report into an abort, which libFuzzer +# records as a crash and saves a reproducer for. Set here rather than in the +# build flags so the guarantee travels with the runner, whoever built the +# binaries. +# +# Deliberately without print_stacktrace=1: that pulls in llvm-symbolizer, the +# same thirteen-seconds-a-call cost that -print_funcs=0 exists to avoid, and +# it turns the abort into a stall the wall-clock cap then files as "slow". +# The report keeps its file:line and the reproducer is saved, which is what a +# rerun needs. +# Appended rather than defaulted: sanitizer flags are parsed left to right and +# the last wins, so this stays authoritative even when the caller already has +# UBSAN_OPTIONS set to something (including halt_on_error=0). +UBSAN_OPTIONS="${UBSAN_OPTIONS:+$UBSAN_OPTIONS:}halt_on_error=1" +export UBSAN_OPTIONS + +ART="$BUILD/fuzz-artifacts" +CORPORA="$BUILD/fuzz-corpora" +mkdir -p "$ART" "$CORPORA" || die "cannot write under $BUILD -- wrong BUILD_DIR?" + +# Check every binary up front: a build that silently produced nothing must not +# be mistaken for a clean run. +for target in $TARGETS; do + [ -x "$BUILD/$target" ] || die "no fuzzer at $BUILD/$target -- was 'make fuzz' run?" + [ -d "fuzzing/$(corpus_for "$target")" ] \ + || die "no seed corpus fuzzing/$(corpus_for "$target") for $target" +done + +# -print_funcs=0 is not cosmetic. libFuzzer symbolizes every newly covered +# function to print its NEW_FUNC line, and llvm-symbolizer takes about thirteen +# seconds per call against a 14 MB ASan+UBSan binary, so the exploration phase +# -- exactly the part a fresh CI corpus is all of -- stalls on the symbolizer. +# Measured here, 30-second budget, before and after: +# +# fuzz_json 7 runs / 90s -> 1190884 runs / 31s +# fuzz_http_h1p 161 runs / 91s -> 1267717 runs / 31s +# fuzz_http_controller -> 1567704 runs / 31s +# fuzz_http_h1p_peer -> 1442178 runs / 32s +# +# Crash reports keep their symbols either way: those go through the sanitizer's +# own stack printer, not libFuzzer's coverage output. Verified on a purpose- +# built crashing target. +PRINT_FUNCS=-print_funcs=0 + +# -max_total_time is only checked between runs and libFuzzer's own -timeout +# defaults to twenty minutes, so a single pathological unit can still run far +# past the budget. Bound the wall clock rather than tightening -timeout: a +# slow target is not a finding, and turning slowness into a red check would +# only teach people to ignore the job. A minute of headroom is ample now that +# a target finishes within a second or two of its budget. +# +# FUZZ_WALL_MARGIN exists so the two branches below can be exercised in a +# test without waiting a minute for the cap. +WALL=$((BUDGET + ${FUZZ_WALL_MARGIN:-60})) + +status=0 +attempted=0 +ran=0 +slow=0 + +for target in $TARGETS; do + seed=$(corpus_for "$target") + dict=$(dict_for "$target") + + mkdir -p "$CORPORA/$target" + set -- "$CORPORA/$target" "fuzzing/$seed" \ + "-max_total_time=$BUDGET" \ + "-artifact_prefix=$ART/$target-" \ + "$PRINT_FUNCS" -print_final_stats=1 -rss_limit_mb=2560 + [ -n "$dict" ] && set -- "$@" "-dict=fuzzing/$dict" + + attempted=$((attempted + 1)) + echo "=== $target (${BUDGET}s budget, ${WALL}s wall cap, seeds: $seed${dict:+, dict: $dict}) ===" + rc=0 + started=$(date +%s) + timeout -k 5 "$WALL" "$BUILD/$target" "$@" || rc=$? + elapsed=$(( $(date +%s) - started )) + + # 137 is 128+SIGKILL, which is what `timeout -k` reports for a target it + # had to kill -- and also what the kernel's OOM killer produces mid-run. + # Distinguishing them matters: an OOM is a finding and must not be filed + # under "slow". Only a kill at or past the cap is the cap. + case "$rc" in + 0) + ran=$((ran + 1)) + ;; + 124|137) + if [ "$elapsed" -ge "$WALL" ]; then + echo "run-ci.sh: $target hit the ${WALL}s wall cap (not a finding)" >&2 + slow=$((slow + 1)) + ran=$((ran + 1)) + else + echo "run-ci.sh: $target KILLED after ${elapsed}s, well inside the" \ + "${WALL}s cap -- treating as a failure (OOM killer?)" >&2 + status=1 + fi + ;; + *) + echo "run-ci.sh: $target FAILED (exit $rc after ${elapsed}s)" >&2 + status=1 + ;; + esac + echo "" +done + +# Guard on what was attempted, not on what succeeded: an empty target list +# would otherwise be a green run that fuzzed nothing. +[ "$attempted" -gt 0 ] || die "no fuzzer actually ran" +echo "run-ci.sh: $attempted target(s) attempted, $ran finished, $slow hit the wall cap, ${BUDGET}s budget each" +exit "$status" diff --git a/go/port.go b/go/port.go index ed8283fe3..020289ce0 100644 --- a/go/port.go +++ b/go/port.go @@ -215,6 +215,29 @@ func nxt_go_port_send(pid C.int, id C.int, buf unsafe.Pointer, buf_size C.int, return C.ssize_t(n) } +// closeUnixRights closes every descriptor an SCM_RIGHTS block carries. It is +// used on paths that refuse a message after the kernel has already installed +// its descriptors, where nothing else will ever see them. Parse failures are +// ignored deliberately: the block is known to be damaged, and whatever can +// still be read out of it is worth closing. +func closeUnixRights(oob []byte) { + msgs, err := syscall.ParseSocketControlMessage(oob) + if err != nil { + return + } + + for _, m := range msgs { + fds, err := syscall.ParseUnixRights(&m) + if err != nil { + continue + } + + for _, fd := range fds { + syscall.Close(fd) + } + } +} + //export nxt_go_port_recv func nxt_go_port_recv(pid C.int, id C.int, buf unsafe.Pointer, buf_size C.int, oob unsafe.Pointer, oob_size *C.size_t) C.ssize_t { @@ -240,9 +263,36 @@ func nxt_go_port_recv(pid C.int, id C.int, buf unsafe.Pointer, buf_size C.int, return 0 } - n, oobn, _, _, err := p.rcv.ReadMsgUnix(GoBytes(buf, buf_size), + n, oobn, flags, _, err := p.rcv.ReadMsgUnix(GoBytes(buf, buf_size), GoBytes(oob, oob_capacity)) + // The control data was cut short, so a descriptor this message was + // meant to carry may not be here; libunit would take its fd-less path + // on a message that otherwise looks whole. The callback reports a + // length rather than msg_flags, so there is no way to hand the + // truncation across as such: report it as a read error, which + // nxt_unit_port_recv() turns into NXT_UNIT_ERROR before it parses any + // control data. This is the one place the Go wrapper can see + // MSG_CTRUNC at all -- ReadMsgUnix is what consumed it. + if err == nil && flags&syscall.MSG_CTRUNC != 0 { + // Truncation does not imply that nothing was delivered: + // scm_detach_fds() installs the descriptors that fit and only + // then raises MSG_CTRUNC. Those are open in this process + // already, and reporting an error leaves *oob_size at 0, so + // libunit never parses the block and never closes them -- + // a descriptor leak under exactly the pressure that caused + // the truncation. Close them here instead; best effort, + // since a block the kernel cut may not parse. + if oobn > 0 { + closeUnixRights(GoBytes(oob, C.int(oobn))) + } + + nxt_go_alert("control data truncated on a %d byte message; "+ + "message dropped", n) + + return C.ssize_t(-1) + } + if err != nil { if nerr, ok := err.(*net.OpError); ok { if nerr.Err == io.EOF { diff --git a/pkg/community/CENTOS.ALMALINUX.md b/pkg/community/CENTOS.ALMALINUX.md index 6d9ef7ec6..360907e8b 100644 --- a/pkg/community/CENTOS.ALMALINUX.md +++ b/pkg/community/CENTOS.ALMALINUX.md @@ -15,7 +15,7 @@ dnf install -y epel-release dnf install -y gcc make git openssl-devel pcre2-devel \ zlib-devel libzstd-devel brotli-devel -# For OTEL support: Rust >= 1.94.1 (rustup.rs) +# For OTEL support: Rust >= 1.88 (rustup.rs) ``` ### PHP version options @@ -113,7 +113,7 @@ git checkout v1.35.5 # or master / latest tag make -j$(nproc) ``` -> **Note:** `--otel` requires Rust ≥ 1.94.1 (`curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh`). +> **Note:** `--otel` requires Rust ≥ 1.88 (`curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh`). > Remove `--otel` if you don't need OpenTelemetry support. > > `--user` / `--group` are omitted because the systemd service runs unitd directly. diff --git a/pkg/contrib/check-advisories.py b/pkg/contrib/check-advisories.py new file mode 100644 index 000000000..bc3eca6b8 --- /dev/null +++ b/pkg/contrib/check-advisories.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +"""Check tarball-pinned dependencies against OSV advisories. + +`cargo audit` reads Cargo.lock files only, so a dependency pinned as a source +tarball under pkg/contrib is invisible to it. This script reads the version +pin and asks OSV, which mirrors the RustSec advisory database and also carries +the GHSA/CVE aliases. + +It also checks that the three places naming the pin agree: + - pkg/contrib/src/wasmtime/version + - pkg/contrib/src/wasmtime/SHA512SUMS + - pkg/eol.json (dependencies.wasmtime_c_api) + +Exit codes: 0 clean, 1 advisories found, 2 configuration or query failure. +""" +import json +import re +import sys +import urllib.error +import urllib.request +from pathlib import Path + +REPO = Path(__file__).resolve().parents[2] +VERSION_FILE = REPO / "pkg/contrib/src/wasmtime/version" +SUMS_FILE = REPO / "pkg/contrib/src/wasmtime/SHA512SUMS" +EOL_FILE = REPO / "pkg/eol.json" +OSV_URL = "https://api.osv.dev/v1/query" + +# The contrib tarball is the wasmtime workspace; pinning it pins both crates +# that the C API links. +CRATES = ("wasmtime", "wasmtime-wasi") + + +def read_version() -> str: + match = re.search(r"WASMTIME_VERSION\s*:=\s*(\S+)", VERSION_FILE.read_text()) + if not match: + fail(f"no WASMTIME_VERSION in {VERSION_FILE.relative_to(REPO)}") + return match.group(1) + + +def fail(message: str) -> None: + print(f"error: {message}", file=sys.stderr) + sys.exit(2) + + +def check_sums(version: str) -> None: + tarball = f"wasmtime-v{version}-src.tar.gz" + pattern = re.compile(rf"\s{tarball}$", re.MULTILINE) + if not pattern.search(SUMS_FILE.read_text()): + fail(f"{SUMS_FILE.relative_to(REPO)} has no digest for {tarball}") + + +def check_eol_json(version: str) -> None: + config = json.loads(EOL_FILE.read_text()) + pinned = config["dependencies"]["wasmtime_c_api"]["version"] + if pinned != version: + fail( + f"{EOL_FILE.relative_to(REPO)} says wasmtime_c_api {pinned}, but the " + f"pin is {version}" + ) + + +def query(crate: str, version: str) -> list: + body = json.dumps( + {"package": {"name": crate, "ecosystem": "crates.io"}, "version": version} + ).encode() + request = urllib.request.Request( + OSV_URL, data=body, headers={"Content-Type": "application/json"} + ) + try: + with urllib.request.urlopen(request, timeout=30) as response: + return json.load(response).get("vulns", []) + except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as error: + fail(f"OSV query for {crate} {version} failed: {error}") + + +def main() -> int: + version = read_version() + check_sums(version) + check_eol_json(version) + + found = False + seen = set() + for crate in CRATES: + for vuln in query(crate, version): + aliases = vuln.get("aliases", []) + # OSV carries the GHSA and RUSTSEC records of one advisory as two + # entries that alias each other; report each advisory once. + key = tuple(sorted({vuln["id"], *aliases})) + if key in seen: + continue + seen.add(key) + found = True + alias_text = ", ".join(aliases) + suffix = f" ({alias_text})" if alias_text else "" + print(f"{crate} {version}: {vuln['id']}{suffix}") + if vuln.get("summary"): + print(f" {vuln['summary']}") + + if found: + return 1 + + print(f"{', '.join(CRATES)} {version}: no advisories") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/pkg/contrib/check-lock-audit.py b/pkg/contrib/check-lock-audit.py new file mode 100755 index 000000000..310ea1070 --- /dev/null +++ b/pkg/contrib/check-lock-audit.py @@ -0,0 +1,293 @@ +#!/usr/bin/env python3 +"""Gate a `cargo audit --json` report on the pinned Wasmtime source lock. + +The archived Cargo.lock covers the whole Wasmtime workspace: the CLI, the +fuzzing targets and wasi-http are all in it, while Unit links one C API +library built from a subset of the features. A finding against a crate that +the library does not link is not a finding against Unit, so auditing the lock +as a whole reports far more than it can act on. + +The archive cannot be repaired either. It ships `.cargo/config.toml` with +`replace-with = "vendored-sources"` over a `vendor/` directory, so +`cargo update --precise` inside it is a no-op: the vendor set holds the pinned +versions and nothing else. + +So this script splits the report in two. A finding against a crate that the C +API links fails the job unless the allowlist accepts that exact version with a +reason. A finding outside the linked graph is reported and does not fail. + +The linked graph is computed, never declared: `cargo tree` resolves it offline +against the vendored archive, with the feature set that pkg/contrib builds. A +Wasmtime bump that starts linking an affected crate therefore turns the job red +on its own, which a hand-written "not linked" note would not. + +Exit codes: 0 clean, 1 a linked finding is not allowed, 2 configuration or +tooling failure. +""" +import argparse +import json +import re +import subprocess +import sys +from pathlib import Path + +REPO = Path(__file__).resolve().parents[2] +MAKEFILE = REPO / "pkg/contrib/src/wasmtime/Makefile" +ALLOWLIST = REPO / "pkg/contrib/wasmtime-lock-audit.allow.json" +FEATURES_CMAKE = Path("crates/c-api/cmake/features.cmake") +CMAKELISTS = Path("crates/c-api/CMakeLists.txt") + +PACKAGE_RE = re.compile(r"^(?P[\w.+-]+) v(?P[^\s]+)") +FEATURE_RE = re.compile( + r"^\s*feature\((?P[\w-]+)\s+(?PON|OFF)\)", re.MULTILINE +) +CMAKE_FLAG_RE = re.compile(r"-D(?PWASMTIME_[A-Z0-9_]+)=(?PON|OFF)") + + +def fail(message: str) -> None: + print(f"error: {message}", file=sys.stderr) + sys.exit(2) + + +def rel(path: Path) -> Path: + """Shorten a path for a message, unless a test moved it out of the tree.""" + try: + return path.relative_to(REPO) + except ValueError: + return path + + +def cmake_name(feature: str) -> str: + return f"wasmtime_feature_{feature}".upper().replace("-", "_") + + +def read_cmake_flags() -> dict: + """Read the -DWASMTIME_* flags that pkg/contrib passes to cmake.""" + if not MAKEFILE.is_file(): + fail(f"{MAKEFILE} is missing") + return { + match.group("name"): match.group("value") == "ON" + for match in CMAKE_FLAG_RE.finditer(MAKEFILE.read_text()) + } + + +def read_features(tree: Path) -> list: + """Resolve the Cargo features of the C API library as cmake would.""" + source = tree / FEATURES_CMAKE + if not source.is_file(): + fail(f"{source} is missing; is {tree} an unpacked Wasmtime archive?") + + # The feature list below is only the whole story while cmake still starts + # from nothing and passes exactly what features.cmake collected. If a + # release changes either, the resolved graph shifts and findings move + # quietly in or out of it. + text = source.read_text() + if "--no-default-features" not in text: + fail(f"{FEATURES_CMAKE} no longer starts from --no-default-features") + + cmakelists = tree / CMAKELISTS + if not cmakelists.is_file(): + fail(f"{cmakelists} is missing") + if "${WASMTIME_FEATURES}" not in cmakelists.read_text(): + fail(f"{CMAKELISTS} no longer passes ${{WASMTIME_FEATURES}} to cargo") + + flags = read_cmake_flags() + disable_all = flags.get("WASMTIME_DISABLE_ALL_FEATURES", False) + + features = [] + declared = set() + for match in FEATURE_RE.finditer(text): + name = match.group("name") + declared.add(cmake_name(name)) + default = match.group("default") == "ON" and not disable_all + if flags.get(cmake_name(name), default): + features.append(name) + + if not features: + fail(f"no features parsed from {source}") + + unknown = set(flags) - declared - {"WASMTIME_DISABLE_ALL_FEATURES"} + if unknown: + fail( + f"{rel(MAKEFILE)} sets {', '.join(sorted(unknown))}, " + f"which {FEATURES_CMAKE} does not declare" + ) + + return features + + +def linked_packages(tree: Path) -> set: + """Return the (name, version) pairs the C API library links.""" + command = [ + "cargo", + "tree", + "--offline", + "--locked", + "--package=wasmtime-c-api", + "--edges=normal,build", + # Without this cargo resolves the host triple alone, and a crate + # linked only on another architecture would read as not linked. + "--target=all", + "--prefix=none", + "--no-default-features", + ] + command += [f"--features={feature}" for feature in read_features(tree)] + + try: + result = subprocess.run( + command, cwd=tree, capture_output=True, text=True, check=True + ) + except OSError as error: + fail(f"cannot run cargo tree: {error}") + except subprocess.CalledProcessError as error: + fail(f"cargo tree failed:\n{error.stderr.strip()}") + + packages = set() + for line in result.stdout.splitlines(): + match = PACKAGE_RE.match(line) + if match: + packages.add((match.group("name"), match.group("version"))) + + if not packages: + fail("cargo tree reported no packages") + + return packages + + +def read_findings(report: Path) -> list: + """Flatten a cargo audit report into (kind, id, package, version) tuples. + + A yanked crate carries no advisory, so the identifier is None there and + the allowlist has to key on the kind and the exact version instead. + """ + try: + data = json.loads(report.read_text()) + except OSError as error: + fail(f"cannot read {report}: {error}") + except json.JSONDecodeError as error: + fail(f"{report} is not valid JSON: {error}") + + if not isinstance(data, dict) or "vulnerabilities" not in data: + fail(f"{report} is not a cargo audit report") + + def package(entry: dict) -> tuple: + try: + return entry["package"]["name"], entry["package"]["version"] + except (KeyError, TypeError): + fail(f"{report} has an entry without a package name and version") + + findings = [] + for entry in data["vulnerabilities"].get("list", []): + advisory = entry.get("advisory") or {} + findings.append(("vulnerability", advisory.get("id"), *package(entry))) + + # Warnings are grouped by kind, and a kind this script has never seen is + # still a warning: iterate whatever the report carries. + for kind, entries in sorted(data.get("warnings", {}).items()): + for entry in entries: + advisory = entry.get("advisory") or {} + findings.append((kind, advisory.get("id"), *package(entry))) + + return findings + + +def read_allowlist(path: Path) -> dict: + try: + data = json.loads(path.read_text()) + except OSError as error: + fail(f"cannot read {path}: {error}") + except json.JSONDecodeError as error: + fail(f"{path} is not valid JSON: {error}") + + allowed = {} + for entry in data.get("allow", []): + missing = {"kind", "package", "version", "reason"} - set(entry) + if missing: + fail(f"{path} has an entry without {', '.join(sorted(missing))}") + key = (entry["kind"], entry.get("id"), entry["package"], entry["version"]) + if key in allowed: + fail(f"{path} lists {entry['package']} {entry['version']} twice") + allowed[key] = entry["reason"] + + return allowed + + +def describe(finding: tuple) -> str: + kind, identifier, package, version = finding + label = identifier or kind + return f"{package} {version}: {label}" + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--report", required=True, type=Path, help="cargo audit --json output" + ) + parser.add_argument( + "--tree", type=Path, help="the unpacked Wasmtime source archive" + ) + parser.add_argument( + "--linked", type=Path, help="a cargo tree listing, instead of --tree" + ) + parser.add_argument("--allow", type=Path, default=ALLOWLIST) + args = parser.parse_args() + + if (args.tree is None) == (args.linked is None): + fail("pass either --tree or --linked") + + if args.linked: + packages = { + (match.group("name"), match.group("version")) + for line in args.linked.read_text().splitlines() + if (match := PACKAGE_RE.match(line)) + } + else: + packages = linked_packages(args.tree) + + findings = read_findings(args.report) + allowed = read_allowlist(args.allow) + + unlinked = [] + accepted = [] + denied = [] + for finding in findings: + _, _, package, version = finding + if (package, version) not in packages: + unlinked.append(finding) + elif finding in allowed: + accepted.append(finding) + else: + denied.append(finding) + + print(f"{len(packages)} packages linked by the C API library") + + for finding in unlinked: + print(f"not linked: {describe(finding)}") + + for finding in accepted: + print(f"accepted: {describe(finding)}") + print(f" {allowed[finding]}") + + # An advisory id is None for a yank, so sort on a string in its place. + stale = sorted(set(allowed) - set(accepted), key=lambda f: (f[0], f[1] or "", *f[2:])) + for finding in stale: + print(f"stale: {describe(finding)} is no longer reported as linked") + + for finding in denied: + print(f"error: {describe(finding)} is linked and not allowed") + + if denied: + print( + f"\n{len(denied)} linked finding(s) are not in " + f"{args.allow.name}. Bump the pin, or add an entry with the " + f"reason the build is not affected.", + file=sys.stderr, + ) + return 1 + + print(f"\n{len(findings)} finding(s), none linked and unaccounted for") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/pkg/contrib/src/njs/Makefile b/pkg/contrib/src/njs/Makefile index 431ab4ec9..e5bd83c36 100644 --- a/pkg/contrib/src/njs/Makefile +++ b/pkg/contrib/src/njs/Makefile @@ -14,7 +14,7 @@ njs: njs-$(NJS_VERSION).tar.gz .sum-njs $(UNPACK) $(MOVE) -# njs 1.0.0 recommends QuickJS, but that requires an external QuickJS / +# njs 1.0.1 recommends QuickJS, but that requires an external QuickJS / # QuickJS-NG library, so build --no-quickjs to keep the baseline deterministic. .njs: njs cd $< && ./configure \ diff --git a/pkg/contrib/src/njs/SHA512SUMS b/pkg/contrib/src/njs/SHA512SUMS index ab99c9d18..04d2143ff 100644 --- a/pkg/contrib/src/njs/SHA512SUMS +++ b/pkg/contrib/src/njs/SHA512SUMS @@ -1 +1 @@ -fde328f79869821fd288a0a19afee44e4089959513aa79f5831f44f4be4a562312cd04d3d27467bae4b909341a962533f54920b71c533ed58122bdb3bf9b335d njs-1.0.0.tar.gz +0339134b30a382820ff7678975425bf1f7040f3f52304b698f7cf08717d07cd9988508982bcd7a8e5e1d64fbe2ad634e38587d37f1afdbe9e9a8ffe8915ac667 njs-1.0.1.tar.gz diff --git a/pkg/contrib/src/njs/version b/pkg/contrib/src/njs/version index 5e8307059..fa2b1d135 100644 --- a/pkg/contrib/src/njs/version +++ b/pkg/contrib/src/njs/version @@ -1 +1 @@ -NJS_VERSION := 1.0.0 +NJS_VERSION := 1.0.1 diff --git a/pkg/contrib/src/wasmtime/Makefile b/pkg/contrib/src/wasmtime/Makefile index dce9ad3b8..996cfd4ab 100644 --- a/pkg/contrib/src/wasmtime/Makefile +++ b/pkg/contrib/src/wasmtime/Makefile @@ -23,6 +23,7 @@ wasmtime: wasmtime-v$(WASMTIME_VERSION)-src.tar.gz .sum-wasmtime .wasmtime: wasmtime cd $< && cmake \ -DCMAKE_INSTALL_LIBDIR=lib \ + -DWASMTIME_FEATURE_WASI_HTTP=OFF \ -S crates/c-api \ -B target/c-api \ --install-prefix "$(TOPSRC)/wasmtime/artifacts" diff --git a/pkg/contrib/src/wasmtime/SHA512SUMS b/pkg/contrib/src/wasmtime/SHA512SUMS index 0a0d35eed..d462e4761 100644 --- a/pkg/contrib/src/wasmtime/SHA512SUMS +++ b/pkg/contrib/src/wasmtime/SHA512SUMS @@ -1 +1 @@ -1a7d3eba2e39ecb148771908714a799ec8511b9fff63c5e8ab30000ddb6a5380a575b9d479b4fc72eb5aa52bed78ee6de82df7437bf2c7e0c357ab2cd2aebef0 wasmtime-v43.0.1-src.tar.gz \ No newline at end of file +a354ccf737a0727d8c83944ca5d909da53cc95637646cd80e0a7c70d54ffc809531c3ef96bcd4bdc40754f20dfefe6c1d4027f0c77b608e3b2699d95b53a2baa wasmtime-v48.0.5-src.tar.gz \ No newline at end of file diff --git a/pkg/contrib/src/wasmtime/version b/pkg/contrib/src/wasmtime/version index 382960944..631363286 100644 --- a/pkg/contrib/src/wasmtime/version +++ b/pkg/contrib/src/wasmtime/version @@ -1 +1 @@ -WASMTIME_VERSION := 43.0.1 +WASMTIME_VERSION := 48.0.5 diff --git a/pkg/contrib/test_check_lock_audit.py b/pkg/contrib/test_check_lock_audit.py new file mode 100644 index 000000000..c665b3be7 --- /dev/null +++ b/pkg/contrib/test_check_lock_audit.py @@ -0,0 +1,348 @@ +"""Tests for check-lock-audit.py. + +These run without a Unit binary, so they live next to the script instead of in +test/, where conftest.py starts a server for every session. Run them with +`python3 -m pytest pkg/contrib/test_check_lock_audit.py`. +""" +import importlib.util +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +SCRIPT = Path(__file__).with_name("check-lock-audit.py") + +LINKED = """\ +wasmtime-c-api v47.0.4 (/src/crates/c-api/artifact) +crossbeam-epoch v0.9.18 +chacha20 v0.10.0 +anyhow v1.0.100 +wasmtime v47.0.4 (/src/crates/wasmtime) +""" + +ALLOW = { + "allow": [ + { + "kind": "vulnerability", + "id": "RUSTSEC-2026-0204", + "package": "crossbeam-epoch", + "version": "0.9.18", + "reason": "not reached", + }, + { + "kind": "yanked", + "id": None, + "package": "chacha20", + "version": "0.10.0", + "reason": "a yank is not an advisory", + }, + ] +} + + +def vulnerability(identifier, package, version): + return { + "advisory": {"id": identifier}, + "package": {"name": package, "version": version}, + } + + +def report(vulnerabilities=(), warnings=None): + return { + "vulnerabilities": { + "count": len(vulnerabilities), + "list": list(vulnerabilities), + }, + "warnings": warnings or {}, + } + + +BASELINE = report( + vulnerabilities=[vulnerability("RUSTSEC-2026-0204", "crossbeam-epoch", "0.9.18")], + warnings={ + "yanked": [{"advisory": None, "package": {"name": "chacha20", "version": "0.10.0"}}], + "unmaintained": [ + { + "advisory": {"id": "RUSTSEC-2024-0436"}, + "package": {"name": "paste", "version": "1.0.7"}, + } + ], + }, +) + + +@pytest.fixture +def run(tmp_path): + def call(report_text, allow=ALLOW, linked_text=LINKED): + linked = tmp_path / "linked.txt" + linked.write_text(linked_text) + report_file = tmp_path / "audit.json" + if report_text is not None: + report_file.write_text(report_text) + allow_file = tmp_path / "allow.json" + allow_file.write_text(json.dumps(allow)) + return subprocess.run( + [ + sys.executable, + str(SCRIPT), + "--report", + str(report_file), + "--linked", + str(linked), + "--allow", + str(allow_file), + ], + capture_output=True, + text=True, + ) + + return call + + +def test_allowlisted_findings_pass(run): + result = run(json.dumps(BASELINE)) + assert result.returncode == 0, result.stdout + result.stderr + assert "accepted: crossbeam-epoch 0.9.18" in result.stdout + assert "accepted: chacha20 0.10.0: yanked" in result.stdout + + +def test_unlinked_finding_is_reported_but_passes(run): + result = run(json.dumps(BASELINE)) + assert result.returncode == 0 + assert "not linked: paste 1.0.7: RUSTSEC-2024-0436" in result.stdout + + +def test_unknown_linked_vulnerability_fails(run): + data = report( + vulnerabilities=[ + vulnerability("RUSTSEC-2026-0204", "crossbeam-epoch", "0.9.18"), + vulnerability("RUSTSEC-2099-0001", "anyhow", "1.0.100"), + ] + ) + result = run(json.dumps(data)) + assert result.returncode == 1 + assert "error: anyhow 1.0.100: RUSTSEC-2099-0001 is linked" in result.stdout + + +def test_unknown_linked_yank_fails(run): + data = report( + warnings={ + "yanked": [ + {"advisory": None, "package": {"name": "anyhow", "version": "1.0.100"}} + ] + } + ) + result = run(json.dumps(data)) + assert result.returncode == 1 + assert "error: anyhow 1.0.100: yanked is linked" in result.stdout + + +def test_an_unseen_warning_kind_is_still_gated(run): + data = report( + warnings={ + "unsound": [ + { + "advisory": {"id": "RUSTSEC-2099-0002"}, + "package": {"name": "anyhow", "version": "1.0.100"}, + } + ] + } + ) + result = run(json.dumps(data)) + assert result.returncode == 1 + assert "RUSTSEC-2099-0002" in result.stdout + + +def test_the_allowlist_version_is_part_of_the_key(run): + # An acceptance is written against one version; the next pin has to earn + # its own entry rather than inherit this one. + linked = LINKED + "crossbeam-epoch v0.9.20\n" + data = report( + vulnerabilities=[ + vulnerability("RUSTSEC-2026-0204", "crossbeam-epoch", "0.9.20") + ] + ) + result = run(json.dumps(data), linked_text=linked) + assert result.returncode == 1 + assert "error: crossbeam-epoch 0.9.20: RUSTSEC-2026-0204" in result.stdout + + +def test_a_stale_entry_warns_without_failing(run): + data = report( + vulnerabilities=[vulnerability("RUSTSEC-2026-0204", "crossbeam-epoch", "0.9.18")] + ) + result = run(json.dumps(data)) + assert result.returncode == 0 + assert "stale: chacha20 0.10.0: yanked is no longer reported" in result.stdout + + +def test_malformed_report_fails(run): + result = run("{not json") + assert result.returncode == 2 + assert "not valid JSON" in result.stderr + + +def test_an_empty_report_fails(run): + result = run("{}") + assert result.returncode == 2 + assert "not a cargo audit report" in result.stderr + + +def test_a_missing_report_fails(run): + result = run(None) + assert result.returncode == 2 + assert "cannot read" in result.stderr + + +def test_an_allowlist_entry_needs_a_reason(run): + allow = {"allow": [{"kind": "yanked", "package": "chacha20", "version": "0.10.0"}]} + result = run(json.dumps(BASELINE), allow) + assert result.returncode == 2 + assert "without reason" in result.stderr + + +def module(): + spec = importlib.util.spec_from_file_location("check_lock_audit", SCRIPT) + loaded = importlib.util.module_from_spec(spec) + spec.loader.exec_module(loaded) + return loaded + + +def archive(tmp_path, check, features_cmake): + """Lay out the parts of an unpacked archive that read_features() reads.""" + cmake = tmp_path / check.FEATURES_CMAKE + cmake.parent.mkdir(parents=True, exist_ok=True) + cmake.write_text(features_cmake) + cmakelists = tmp_path / check.CMAKELISTS + cmakelists.write_text("${WASMTIME_CARGO_BINARY} build ${WASMTIME_FEATURES}\n") + return tmp_path + + +def test_the_feature_set_follows_the_cmake_override(tmp_path, monkeypatch): + check = module() + makefile = tmp_path / "Makefile" + makefile.write_text("-DWASMTIME_FEATURE_WASI_HTTP=OFF \\\n") + monkeypatch.setattr(check, "MAKEFILE", makefile) + tree = archive( + tmp_path, + check, + 'set(WASMTIME_FEATURES "--no-default-features")\n' + "feature(wasi ON)\nfeature(wasi-http ON)\nfeature(all-arch OFF)\n", + ) + assert check.read_features(tree) == ["wasi"] + + +@pytest.mark.parametrize( + "features_cmake, cmakelists", + [ + ('set(WASMTIME_FEATURES "")\nfeature(wasi ON)\n', None), + (None, "${WASMTIME_CARGO_BINARY} build\n"), + ], +) +def test_a_changed_cargo_invocation_fails( + tmp_path, monkeypatch, features_cmake, cmakelists +): + """The reconstruction is only valid while cmake drives cargo this way.""" + check = module() + makefile = tmp_path / "Makefile" + makefile.write_text("") + monkeypatch.setattr(check, "MAKEFILE", makefile) + tree = archive( + tmp_path, + check, + features_cmake + or 'set(WASMTIME_FEATURES "--no-default-features")\nfeature(wasi ON)\n', + ) + if cmakelists: + (tree / check.CMAKELISTS).write_text(cmakelists) + with pytest.raises(SystemExit) as error: + check.read_features(tree) + assert error.value.code == 2 + + +def test_an_undeclared_cmake_flag_fails(tmp_path, monkeypatch): + check = module() + makefile = tmp_path / "Makefile" + makefile.write_text("-DWASMTIME_FEATURE_NO_SUCH_THING=OFF\n") + monkeypatch.setattr(check, "MAKEFILE", makefile) + tree = archive( + tmp_path, + check, + 'set(WASMTIME_FEATURES "--no-default-features")\nfeature(wasi ON)\n', + ) + with pytest.raises(SystemExit) as error: + check.read_features(tree) + assert error.value.code == 2 + + +def test_a_cargo_tree_listing_parses(tmp_path): + """The shapes cargo tree --prefix=none actually prints.""" + check = module() + listing = tmp_path / "tree.txt" + listing.write_text( + "wasmtime v47.0.4 (/src/crates/wasmtime)\n" + "anyhow v1.0.100\n" + "anyhow v1.0.100 (*)\n" + "serde_derive v1.0.228 (proc-macro)\n" + "wasi v0.11.0+wasi-snapshot-preview1\n" + "\n" + ) + found = { + (match.group("name"), match.group("version")) + for line in listing.read_text().splitlines() + if (match := check.PACKAGE_RE.match(line)) + } + assert found == { + ("wasmtime", "47.0.4"), + ("anyhow", "1.0.100"), + ("serde_derive", "1.0.228"), + ("wasi", "0.11.0+wasi-snapshot-preview1"), + } + + +def test_a_duplicate_allowlist_entry_fails(run): + entry = { + "kind": "yanked", + "id": None, + "package": "chacha20", + "version": "0.10.0", + "reason": "first", + } + allow = {"allow": [entry, dict(entry, reason="second")]} + result = run(json.dumps(BASELINE), allow) + assert result.returncode == 2 + assert "twice" in result.stderr + + +def test_a_warning_without_a_package_fails(run): + data = report(warnings={"yanked": [{"advisory": None}]}) + result = run(json.dumps(data)) + assert result.returncode == 2 + assert "without a package name and version" in result.stderr + + +def test_stale_entries_sort_with_and_without_an_advisory_id(run): + # Sorting the stale list used to compare None against a string. + allow = { + "allow": [ + { + "kind": "yanked", + "id": None, + "package": "aaa", + "version": "1.0.0", + "reason": "no advisory", + }, + { + "kind": "yanked", + "id": "RUSTSEC-2099-0003", + "package": "bbb", + "version": "2.0.0", + "reason": "with advisory", + }, + ] + } + result = run(json.dumps(report()), allow) + assert result.returncode == 0 + assert result.stdout.count("stale:") == 2 diff --git a/pkg/contrib/wasmtime-lock-audit.allow.json b/pkg/contrib/wasmtime-lock-audit.allow.json new file mode 100644 index 000000000..b73c602c1 --- /dev/null +++ b/pkg/contrib/wasmtime-lock-audit.allow.json @@ -0,0 +1,19 @@ +{ + "comment": [ + "Findings against crates that the Wasmtime C API library links, and that", + "pkg/contrib cannot resolve by bumping the pin. check-lock-audit.py fails", + "the audit on any other linked finding. A finding outside the linked graph", + "needs no entry here; it is reported and ignored.", + "The version is part of the key, so a pin bump forces this file to be", + "reviewed again rather than carrying an acceptance forward silently." + ], + "allow": [ + { + "kind": "yanked", + "id": null, + "package": "chacha20", + "version": "0.10.0", + "reason": "Linked through rand 0.10.1, which wasmtime-wasi uses to seed guest randomness. A yank is not an advisory: no RustSec record applies to this version. 0.10.1 is yanked as well and 0.10.2 is clean, but the Wasmtime 48.0.5 archive is vendored and its lock still names 0.10.0, so the version moves only when the pin does." + } + ] +} diff --git a/pkg/docker/Dockerfile.builder-php8.5 b/pkg/docker/Dockerfile.builder-php8.5 index 51dc1e892..bbb2d670d 100644 --- a/pkg/docker/Dockerfile.builder-php8.5 +++ b/pkg/docker/Dockerfile.builder-php8.5 @@ -21,7 +21,7 @@ RUN set -ex \ && export CARGO_HOME=/usr/local/cargo \ && export PATH=/usr/local/cargo/bin:$PATH \ && ./rustup-init -y --no-modify-path --profile minimal \ - --default-toolchain 1.94.1 --default-host "${rustArch}" \ + --default-toolchain 1.95.0 --default-host "${rustArch}" \ && rm rustup-init \ && rustc --version \ && cargo --version diff --git a/pkg/docker/Dockerfile.builder-trixie b/pkg/docker/Dockerfile.builder-trixie index 6bfc75343..c2f9940dc 100644 --- a/pkg/docker/Dockerfile.builder-trixie +++ b/pkg/docker/Dockerfile.builder-trixie @@ -21,7 +21,7 @@ RUN set -ex \ && export CARGO_HOME=/usr/local/cargo \ && export PATH=/usr/local/cargo/bin:$PATH \ && ./rustup-init -y --no-modify-path --profile minimal \ - --default-toolchain 1.94.1 --default-host "${rustArch}" \ + --default-toolchain 1.95.0 --default-host "${rustArch}" \ && rm rustup-init \ && rustc --version \ && cargo --version diff --git a/pkg/docker/Dockerfile.go-1.25 b/pkg/docker/Dockerfile.go-1.25 index a917c3281..1bd99149a 100644 --- a/pkg/docker/Dockerfile.go-1.25 +++ b/pkg/docker/Dockerfile.go-1.25 @@ -6,14 +6,15 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ @@ -35,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.go-1.26 b/pkg/docker/Dockerfile.go-1.26 index d5330ff1e..6478d6aa2 100644 --- a/pkg/docker/Dockerfile.go-1.26 +++ b/pkg/docker/Dockerfile.go-1.26 @@ -6,14 +6,15 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ @@ -35,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.go-1.27 b/pkg/docker/Dockerfile.go-1.27 new file mode 100644 index 000000000..ff866d2a6 --- /dev/null +++ b/pkg/docker/Dockerfile.go-1.27 @@ -0,0 +1,116 @@ +FROM golang:1.27-trixie + +LABEL org.opencontainers.image.title="FreeUnit (go-1.27)" +LABEL org.opencontainers.image.description="Community build of FreeUnit for Docker." +LABEL org.opencontainers.image.url="https://freeunit.org" +LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" +LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" +LABEL org.opencontainers.image.vendor="FreeUnit Community " +LABEL org.opencontainers.image.version="1.36.1" + +RUN set -ex \ + && savedAptMark="$(apt-mark showmanual)" \ + && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ + && apt-get install --no-install-recommends --no-install-suggests -y \ + ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ + && export RUST_VERSION=1.95.0 \ + && export RUSTUP_HOME=/usr/src/unit/rustup \ + && export CARGO_HOME=/usr/src/unit/cargo \ + && export PATH=/usr/src/unit/cargo/bin:$PATH \ + && dpkgArch="$(dpkg --print-architecture)" \ + && case "${dpkgArch##*-}" in \ + amd64) rustArch="x86_64-unknown-linux-gnu"; rustupSha256="6aeece6993e902708983b209d04c0d1dbb14ebb405ddb87def578d41f920f56d" ;; \ + arm64) rustArch="aarch64-unknown-linux-gnu"; rustupSha256="1cffbf51e63e634c746f741de50649bbbcbd9dbe1de363c9ecef64e278dba2b2" ;; \ + *) echo >&2 "unsupported architecture: ${dpkgArch}"; exit 1 ;; \ + esac \ + && url="https://static.rust-lang.org/rustup/archive/1.27.1/${rustArch}/rustup-init" \ + && curl -L -O "$url" \ + && echo "${rustupSha256} *rustup-init" | sha256sum -c - \ + && chmod +x rustup-init \ + && ./rustup-init -y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION --default-host ${rustArch} \ + && rm rustup-init \ + && rustup --version \ + && cargo --version \ + && rustc --version \ + && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ + && mkdir -p /usr/src/unit \ + && cd /usr/src/unit \ + && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ + && cd unit \ + && NCPU="$(getconf _NPROCESSORS_ONLN)" \ + && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ + && CC_OPT="$(DEB_BUILD_MAINT_OPTIONS="hardening=+all,-pie" DEB_CFLAGS_MAINT_APPEND="-fPIC" dpkg-buildflags --get CFLAGS | tr ' ' '\n' | grep -v '^-specs=' | tr '\n' ' ')" \ + && LD_OPT="$(DEB_BUILD_MAINT_OPTIONS="hardening=+all,-pie" DEB_LDFLAGS_MAINT_APPEND="-Wl,--as-needed -pie" dpkg-buildflags --get LDFLAGS | tr ' ' '\n' | grep -v '^-specs=' | tr '\n' ' ')" \ + && CONFIGURE_ARGS_MODULES="--prefix=/usr \ + --statedir=/var/lib/unit \ + --control=unix:/var/run/control.unit.sock \ + --runstatedir=/var/run \ + --pid=/var/run/unit.pid \ + --logdir=/var/log \ + --log=/var/log/unit.log \ + --tmpdir=/var/tmp \ + --user=unit \ + --group=unit \ + --openssl \ + --libdir=/usr/lib/$DEB_HOST_MULTIARCH" \ + && CONFIGURE_ARGS="$CONFIGURE_ARGS_MODULES \ + --njs \ + --otel \ + --zlib \ + --zstd \ + --brotli" \ + && make -j $NCPU -C pkg/contrib .njs \ + && export PKG_CONFIG_PATH=$(pwd)/pkg/contrib/njs/build \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --ld-opt="$LD_OPT" --modulesdir=/usr/lib/unit/debug-modules --debug \ + && make -j $NCPU unitd \ + && install -pm755 build/sbin/unitd /usr/sbin/unitd-debug \ + && make clean \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --ld-opt="$LD_OPT" --modulesdir=/usr/lib/unit/modules \ + && make -j $NCPU unitd \ + && install -pm755 build/sbin/unitd /usr/sbin/unitd \ + && make clean \ + && /bin/true \ + && ./configure $CONFIGURE_ARGS_MODULES --cc-opt="$CC_OPT" --modulesdir=/usr/lib/unit/debug-modules --debug \ + && ./configure go --go-path=$GOPATH \ + && make -j $NCPU go-install-src libunit-install \ + && make clean \ + && ./configure $CONFIGURE_ARGS_MODULES --cc-opt="$CC_OPT" --modulesdir=/usr/lib/unit/modules \ + && ./configure go --go-path=$GOPATH \ + && make -j $NCPU go-install-src libunit-install \ + && cd \ + && rm -rf /usr/src/unit \ + && for f in /usr/sbin/unitd /usr/lib/unit/modules/*.unit.so; do \ + [ -f "$f" ] || continue; \ + ldd "$f" | awk '/=>/{print $(NF-1)}' | while read n; do dpkg-query -S $(realpath $n 2>/dev/null || echo $n) 2>/dev/null; done | grep -v '^diversion' | sed 's/^\([^:]\+\):.*$/\1/' | sort | uniq >> /requirements.apt; \ + done \ + && apt-mark showmanual | xargs apt-mark auto > /dev/null \ + && { [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark; } \ + && /bin/true \ + && mkdir -p /var/lib/unit/ \ + && mkdir -p /docker-entrypoint.d/ \ + && groupadd --gid 999 unit \ + && useradd \ + --uid 999 \ + --gid unit \ + --no-create-home \ + --home /nonexistent \ + --comment "unit user" \ + --shell /bin/false \ + unit \ + && apt-get update \ + && apt-get --no-install-recommends --no-install-suggests -y install curl $(cat /requirements.apt) \ + && apt-get purge -y --auto-remove build-essential \ + && rm -rf /var/lib/apt/lists/* \ + && rm -f /requirements.apt \ + && ln -sf /dev/stderr /var/log/unit.log + +COPY docker-entrypoint.sh /usr/local/bin/ +COPY welcome.* /usr/share/unit/welcome/ +COPY seccomp-no-af-alg.json /usr/share/unit/ + +STOPSIGNAL SIGTERM + +ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] +EXPOSE 80 +CMD ["unitd", "--no-daemon", "--control", "unix:/var/run/control.unit.sock"] diff --git a/pkg/docker/Dockerfile.java-17 b/pkg/docker/Dockerfile.java-17 index 77142f771..520771b66 100644 --- a/pkg/docker/Dockerfile.java-17 +++ b/pkg/docker/Dockerfile.java-17 @@ -1,4 +1,4 @@ -FROM eclipse-temurin:17-jdk-noble +FROM eclipse-temurin:17-jdk-resolute LABEL org.opencontainers.image.title="FreeUnit (java-17)" LABEL org.opencontainers.image.description="Community build of FreeUnit for Docker." @@ -6,14 +6,15 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ @@ -35,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.java-21 b/pkg/docker/Dockerfile.java-21 index 8a77c3eb4..ecb9d13b7 100644 --- a/pkg/docker/Dockerfile.java-21 +++ b/pkg/docker/Dockerfile.java-21 @@ -1,4 +1,4 @@ -FROM eclipse-temurin:21-jdk-noble +FROM eclipse-temurin:21-jdk-resolute LABEL org.opencontainers.image.title="FreeUnit (java-21)" LABEL org.opencontainers.image.description="Community build of FreeUnit for Docker." @@ -6,14 +6,15 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ @@ -35,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.java-25 b/pkg/docker/Dockerfile.java-25 index 2e92da7ba..faf03ca7e 100644 --- a/pkg/docker/Dockerfile.java-25 +++ b/pkg/docker/Dockerfile.java-25 @@ -1,4 +1,4 @@ -FROM eclipse-temurin:25-jdk-noble +FROM eclipse-temurin:25-jdk-resolute LABEL org.opencontainers.image.title="FreeUnit (java-25)" LABEL org.opencontainers.image.description="Community build of FreeUnit for Docker." @@ -6,14 +6,15 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ @@ -35,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.java-26 b/pkg/docker/Dockerfile.java-26 index 807e5178d..8ef41b59f 100644 --- a/pkg/docker/Dockerfile.java-26 +++ b/pkg/docker/Dockerfile.java-26 @@ -1,4 +1,4 @@ -FROM eclipse-temurin:26-jdk-noble +FROM eclipse-temurin:26-jdk-resolute LABEL org.opencontainers.image.title="FreeUnit (java-26)" LABEL org.opencontainers.image.description="Community build of FreeUnit for Docker." @@ -6,14 +6,15 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ @@ -35,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.java-27 b/pkg/docker/Dockerfile.java-27 new file mode 100644 index 000000000..b82003368 --- /dev/null +++ b/pkg/docker/Dockerfile.java-27 @@ -0,0 +1,116 @@ +FROM eclipse-temurin:27-jdk-resolute + +LABEL org.opencontainers.image.title="FreeUnit (java-27)" +LABEL org.opencontainers.image.description="Community build of FreeUnit for Docker." +LABEL org.opencontainers.image.url="https://freeunit.org" +LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" +LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" +LABEL org.opencontainers.image.vendor="FreeUnit Community " +LABEL org.opencontainers.image.version="1.36.1" + +RUN set -ex \ + && savedAptMark="$(apt-mark showmanual)" \ + && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ + && apt-get install --no-install-recommends --no-install-suggests -y \ + ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ + && export RUST_VERSION=1.95.0 \ + && export RUSTUP_HOME=/usr/src/unit/rustup \ + && export CARGO_HOME=/usr/src/unit/cargo \ + && export PATH=/usr/src/unit/cargo/bin:$PATH \ + && dpkgArch="$(dpkg --print-architecture)" \ + && case "${dpkgArch##*-}" in \ + amd64) rustArch="x86_64-unknown-linux-gnu"; rustupSha256="6aeece6993e902708983b209d04c0d1dbb14ebb405ddb87def578d41f920f56d" ;; \ + arm64) rustArch="aarch64-unknown-linux-gnu"; rustupSha256="1cffbf51e63e634c746f741de50649bbbcbd9dbe1de363c9ecef64e278dba2b2" ;; \ + *) echo >&2 "unsupported architecture: ${dpkgArch}"; exit 1 ;; \ + esac \ + && url="https://static.rust-lang.org/rustup/archive/1.27.1/${rustArch}/rustup-init" \ + && curl -L -O "$url" \ + && echo "${rustupSha256} *rustup-init" | sha256sum -c - \ + && chmod +x rustup-init \ + && ./rustup-init -y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION --default-host ${rustArch} \ + && rm rustup-init \ + && rustup --version \ + && cargo --version \ + && rustc --version \ + && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ + && mkdir -p /usr/src/unit \ + && cd /usr/src/unit \ + && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ + && cd unit \ + && NCPU="$(getconf _NPROCESSORS_ONLN)" \ + && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ + && CC_OPT="$(DEB_BUILD_MAINT_OPTIONS="hardening=+all,-pie" DEB_CFLAGS_MAINT_APPEND="-fPIC" dpkg-buildflags --get CFLAGS | tr ' ' '\n' | grep -v '^-specs=' | tr '\n' ' ')" \ + && LD_OPT="$(DEB_BUILD_MAINT_OPTIONS="hardening=+all,-pie" DEB_LDFLAGS_MAINT_APPEND="-Wl,--as-needed -pie" dpkg-buildflags --get LDFLAGS | tr ' ' '\n' | grep -v '^-specs=' | tr '\n' ' ')" \ + && CONFIGURE_ARGS_MODULES="--prefix=/usr \ + --statedir=/var/lib/unit \ + --control=unix:/var/run/control.unit.sock \ + --runstatedir=/var/run \ + --pid=/var/run/unit.pid \ + --logdir=/var/log \ + --log=/var/log/unit.log \ + --tmpdir=/var/tmp \ + --user=unit \ + --group=unit \ + --openssl \ + --libdir=/usr/lib/$DEB_HOST_MULTIARCH" \ + && CONFIGURE_ARGS="$CONFIGURE_ARGS_MODULES \ + --njs \ + --otel \ + --zlib \ + --zstd \ + --brotli" \ + && make -j $NCPU -C pkg/contrib .njs \ + && export PKG_CONFIG_PATH=$(pwd)/pkg/contrib/njs/build \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --ld-opt="$LD_OPT" --modulesdir=/usr/lib/unit/debug-modules --debug \ + && make -j $NCPU unitd \ + && install -pm755 build/sbin/unitd /usr/sbin/unitd-debug \ + && make clean \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --ld-opt="$LD_OPT" --modulesdir=/usr/lib/unit/modules \ + && make -j $NCPU unitd \ + && install -pm755 build/sbin/unitd /usr/sbin/unitd \ + && make clean \ + && /bin/true \ + && ./configure $CONFIGURE_ARGS_MODULES --cc-opt="$CC_OPT" --modulesdir=/usr/lib/unit/debug-modules --debug \ + && ./configure java --jars=/usr/share/unit-jsc-common/ \ + && make -j $NCPU java-shared-install java-install \ + && make clean \ + && ./configure $CONFIGURE_ARGS_MODULES --cc-opt="$CC_OPT" --modulesdir=/usr/lib/unit/modules \ + && ./configure java --jars=/usr/share/unit-jsc-common/ \ + && make -j $NCPU java-shared-install java-install \ + && cd \ + && rm -rf /usr/src/unit \ + && for f in /usr/sbin/unitd /usr/lib/unit/modules/*.unit.so; do \ + [ -f "$f" ] || continue; \ + ldd "$f" | awk '/=>/{print $(NF-1)}' | while read n; do dpkg-query -S $(realpath $n 2>/dev/null || echo $n) 2>/dev/null; done | grep -v '^diversion' | sed 's/^\([^:]\+\):.*$/\1/' | sort | uniq >> /requirements.apt; \ + done \ + && apt-mark showmanual | xargs apt-mark auto > /dev/null \ + && { [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark; } \ + && rm -rf /root/.m2 \ + && mkdir -p /var/lib/unit/ \ + && mkdir -p /docker-entrypoint.d/ \ + && groupadd --gid 999 unit \ + && useradd \ + --uid 999 \ + --gid unit \ + --no-create-home \ + --home /nonexistent \ + --comment "unit user" \ + --shell /bin/false \ + unit \ + && apt-get update \ + && apt-get --no-install-recommends --no-install-suggests -y install curl $(cat /requirements.apt) \ + && apt-get purge -y --auto-remove build-essential \ + && rm -rf /var/lib/apt/lists/* \ + && rm -f /requirements.apt \ + && ln -sf /dev/stderr /var/log/unit.log + +COPY docker-entrypoint.sh /usr/local/bin/ +COPY welcome.* /usr/share/unit/welcome/ +COPY seccomp-no-af-alg.json /usr/share/unit/ + +STOPSIGNAL SIGTERM + +ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] +EXPOSE 80 +CMD ["unitd", "--no-daemon", "--control", "unix:/var/run/control.unit.sock"] diff --git a/pkg/docker/Dockerfile.minimal b/pkg/docker/Dockerfile.minimal index 150793e58..fee881187 100644 --- a/pkg/docker/Dockerfile.minimal +++ b/pkg/docker/Dockerfile.minimal @@ -6,14 +6,15 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ @@ -35,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.node-20 b/pkg/docker/Dockerfile.node-20 index 215867433..4c03fd673 100644 --- a/pkg/docker/Dockerfile.node-20 +++ b/pkg/docker/Dockerfile.node-20 @@ -6,14 +6,15 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ @@ -35,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.node-22 b/pkg/docker/Dockerfile.node-22 index 8793cc9a1..691352bab 100644 --- a/pkg/docker/Dockerfile.node-22 +++ b/pkg/docker/Dockerfile.node-22 @@ -6,14 +6,15 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ @@ -35,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.node-24 b/pkg/docker/Dockerfile.node-24 index 138ccdeb5..7f613141b 100644 --- a/pkg/docker/Dockerfile.node-24 +++ b/pkg/docker/Dockerfile.node-24 @@ -6,14 +6,15 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ @@ -35,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.node-26 b/pkg/docker/Dockerfile.node-26 index da4d58ef7..c0755af5b 100644 --- a/pkg/docker/Dockerfile.node-26 +++ b/pkg/docker/Dockerfile.node-26 @@ -6,14 +6,15 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ @@ -35,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.perl-5.38 b/pkg/docker/Dockerfile.perl-5.38 index fcce62dd0..75982c269 100644 --- a/pkg/docker/Dockerfile.perl-5.38 +++ b/pkg/docker/Dockerfile.perl-5.38 @@ -6,14 +6,15 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ @@ -35,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.perl-5.40 b/pkg/docker/Dockerfile.perl-5.40 index 90ae2fdef..9dbaab13f 100644 --- a/pkg/docker/Dockerfile.perl-5.40 +++ b/pkg/docker/Dockerfile.perl-5.40 @@ -6,14 +6,15 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ @@ -35,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.perl-5.42 b/pkg/docker/Dockerfile.perl-5.42 index 1efa58be8..c4ae23512 100644 --- a/pkg/docker/Dockerfile.perl-5.42 +++ b/pkg/docker/Dockerfile.perl-5.42 @@ -6,14 +6,15 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ @@ -35,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.perl-5.44 b/pkg/docker/Dockerfile.perl-5.44 new file mode 100644 index 000000000..0b966945d --- /dev/null +++ b/pkg/docker/Dockerfile.perl-5.44 @@ -0,0 +1,116 @@ +FROM perl:5.44-trixie + +LABEL org.opencontainers.image.title="FreeUnit (perl-5.44)" +LABEL org.opencontainers.image.description="Community build of FreeUnit for Docker." +LABEL org.opencontainers.image.url="https://freeunit.org" +LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" +LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" +LABEL org.opencontainers.image.vendor="FreeUnit Community " +LABEL org.opencontainers.image.version="1.36.1" + +RUN set -ex \ + && savedAptMark="$(apt-mark showmanual)" \ + && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ + && apt-get install --no-install-recommends --no-install-suggests -y \ + ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ + && export RUST_VERSION=1.95.0 \ + && export RUSTUP_HOME=/usr/src/unit/rustup \ + && export CARGO_HOME=/usr/src/unit/cargo \ + && export PATH=/usr/src/unit/cargo/bin:$PATH \ + && dpkgArch="$(dpkg --print-architecture)" \ + && case "${dpkgArch##*-}" in \ + amd64) rustArch="x86_64-unknown-linux-gnu"; rustupSha256="6aeece6993e902708983b209d04c0d1dbb14ebb405ddb87def578d41f920f56d" ;; \ + arm64) rustArch="aarch64-unknown-linux-gnu"; rustupSha256="1cffbf51e63e634c746f741de50649bbbcbd9dbe1de363c9ecef64e278dba2b2" ;; \ + *) echo >&2 "unsupported architecture: ${dpkgArch}"; exit 1 ;; \ + esac \ + && url="https://static.rust-lang.org/rustup/archive/1.27.1/${rustArch}/rustup-init" \ + && curl -L -O "$url" \ + && echo "${rustupSha256} *rustup-init" | sha256sum -c - \ + && chmod +x rustup-init \ + && ./rustup-init -y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION --default-host ${rustArch} \ + && rm rustup-init \ + && rustup --version \ + && cargo --version \ + && rustc --version \ + && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ + && mkdir -p /usr/src/unit \ + && cd /usr/src/unit \ + && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ + && cd unit \ + && NCPU="$(getconf _NPROCESSORS_ONLN)" \ + && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ + && CC_OPT="$(DEB_BUILD_MAINT_OPTIONS="hardening=+all,-pie" DEB_CFLAGS_MAINT_APPEND="-fPIC" dpkg-buildflags --get CFLAGS | tr ' ' '\n' | grep -v '^-specs=' | tr '\n' ' ')" \ + && LD_OPT="$(DEB_BUILD_MAINT_OPTIONS="hardening=+all,-pie" DEB_LDFLAGS_MAINT_APPEND="-Wl,--as-needed -pie" dpkg-buildflags --get LDFLAGS | tr ' ' '\n' | grep -v '^-specs=' | tr '\n' ' ')" \ + && CONFIGURE_ARGS_MODULES="--prefix=/usr \ + --statedir=/var/lib/unit \ + --control=unix:/var/run/control.unit.sock \ + --runstatedir=/var/run \ + --pid=/var/run/unit.pid \ + --logdir=/var/log \ + --log=/var/log/unit.log \ + --tmpdir=/var/tmp \ + --user=unit \ + --group=unit \ + --openssl \ + --libdir=/usr/lib/$DEB_HOST_MULTIARCH" \ + && CONFIGURE_ARGS="$CONFIGURE_ARGS_MODULES \ + --njs \ + --otel \ + --zlib \ + --zstd \ + --brotli" \ + && make -j $NCPU -C pkg/contrib .njs \ + && export PKG_CONFIG_PATH=$(pwd)/pkg/contrib/njs/build \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --ld-opt="$LD_OPT" --modulesdir=/usr/lib/unit/debug-modules --debug \ + && make -j $NCPU unitd \ + && install -pm755 build/sbin/unitd /usr/sbin/unitd-debug \ + && make clean \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --ld-opt="$LD_OPT" --modulesdir=/usr/lib/unit/modules \ + && make -j $NCPU unitd \ + && install -pm755 build/sbin/unitd /usr/sbin/unitd \ + && make clean \ + && /bin/true \ + && ./configure $CONFIGURE_ARGS_MODULES --cc-opt="$CC_OPT" --modulesdir=/usr/lib/unit/debug-modules --debug \ + && ./configure perl \ + && make -j $NCPU perl-install \ + && make clean \ + && ./configure $CONFIGURE_ARGS_MODULES --cc-opt="$CC_OPT" --modulesdir=/usr/lib/unit/modules \ + && ./configure perl \ + && make -j $NCPU perl-install \ + && cd \ + && rm -rf /usr/src/unit \ + && for f in /usr/sbin/unitd /usr/lib/unit/modules/*.unit.so; do \ + [ -f "$f" ] || continue; \ + ldd "$f" | awk '/=>/{print $(NF-1)}' | while read n; do dpkg-query -S $(realpath $n 2>/dev/null || echo $n) 2>/dev/null; done | grep -v '^diversion' | sed 's/^\([^:]\+\):.*$/\1/' | sort | uniq >> /requirements.apt; \ + done \ + && apt-mark showmanual | xargs apt-mark auto > /dev/null \ + && { [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark; } \ + && /bin/true \ + && mkdir -p /var/lib/unit/ \ + && mkdir -p /docker-entrypoint.d/ \ + && groupadd --gid 999 unit \ + && useradd \ + --uid 999 \ + --gid unit \ + --no-create-home \ + --home /nonexistent \ + --comment "unit user" \ + --shell /bin/false \ + unit \ + && apt-get update \ + && apt-get --no-install-recommends --no-install-suggests -y install curl $(cat /requirements.apt) \ + && apt-get purge -y --auto-remove build-essential \ + && rm -rf /var/lib/apt/lists/* \ + && rm -f /requirements.apt \ + && ln -sf /dev/stderr /var/log/unit.log + +COPY docker-entrypoint.sh /usr/local/bin/ +COPY welcome.* /usr/share/unit/welcome/ +COPY seccomp-no-af-alg.json /usr/share/unit/ + +STOPSIGNAL SIGTERM + +ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] +EXPOSE 80 +CMD ["unitd", "--no-daemon", "--control", "unix:/var/run/control.unit.sock"] diff --git a/pkg/docker/Dockerfile.php-8.3 b/pkg/docker/Dockerfile.php-8.3 index 545834817..e3b049ef6 100644 --- a/pkg/docker/Dockerfile.php-8.3 +++ b/pkg/docker/Dockerfile.php-8.3 @@ -6,14 +6,15 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ @@ -35,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.php-8.4 b/pkg/docker/Dockerfile.php-8.4 index 94084852a..779ef2bef 100644 --- a/pkg/docker/Dockerfile.php-8.4 +++ b/pkg/docker/Dockerfile.php-8.4 @@ -6,14 +6,15 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ @@ -35,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.php-8.5 b/pkg/docker/Dockerfile.php-8.5 index 23f7c69af..d9bc4d514 100644 --- a/pkg/docker/Dockerfile.php-8.5 +++ b/pkg/docker/Dockerfile.php-8.5 @@ -6,14 +6,15 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ @@ -35,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.python-3.12 b/pkg/docker/Dockerfile.python-3.12 index a6dffb0a1..257cd6124 100644 --- a/pkg/docker/Dockerfile.python-3.12 +++ b/pkg/docker/Dockerfile.python-3.12 @@ -6,14 +6,15 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ @@ -35,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.python-3.12-slim b/pkg/docker/Dockerfile.python-3.12-slim index ca7821af3..bb237321a 100644 --- a/pkg/docker/Dockerfile.python-3.12-slim +++ b/pkg/docker/Dockerfile.python-3.12-slim @@ -6,14 +6,15 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ @@ -35,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.python-3.13 b/pkg/docker/Dockerfile.python-3.13 index aca936a09..1087c0b86 100644 --- a/pkg/docker/Dockerfile.python-3.13 +++ b/pkg/docker/Dockerfile.python-3.13 @@ -6,14 +6,15 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ @@ -35,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.python-3.13-slim b/pkg/docker/Dockerfile.python-3.13-slim index 7127ab3fa..59ac35ab5 100644 --- a/pkg/docker/Dockerfile.python-3.13-slim +++ b/pkg/docker/Dockerfile.python-3.13-slim @@ -6,14 +6,15 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ @@ -35,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.python-3.14 b/pkg/docker/Dockerfile.python-3.14 index a684dfe82..1652a1a6b 100644 --- a/pkg/docker/Dockerfile.python-3.14 +++ b/pkg/docker/Dockerfile.python-3.14 @@ -6,14 +6,15 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ @@ -35,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.python-3.14-slim b/pkg/docker/Dockerfile.python-3.14-slim index 6241bda51..37daae820 100644 --- a/pkg/docker/Dockerfile.python-3.14-slim +++ b/pkg/docker/Dockerfile.python-3.14-slim @@ -6,14 +6,15 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ @@ -35,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.ruby-3.3 b/pkg/docker/Dockerfile.ruby-3.3 index a5c2f876e..bf4d0e64c 100644 --- a/pkg/docker/Dockerfile.ruby-3.3 +++ b/pkg/docker/Dockerfile.ruby-3.3 @@ -6,14 +6,15 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ @@ -35,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.ruby-3.4 b/pkg/docker/Dockerfile.ruby-3.4 index 36ea02925..922815cf6 100644 --- a/pkg/docker/Dockerfile.ruby-3.4 +++ b/pkg/docker/Dockerfile.ruby-3.4 @@ -6,14 +6,15 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ @@ -35,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.ruby-4.0 b/pkg/docker/Dockerfile.ruby-4.0 index 7d4ca2e38..fe11f9723 100644 --- a/pkg/docker/Dockerfile.ruby-4.0 +++ b/pkg/docker/Dockerfile.ruby-4.0 @@ -6,14 +6,15 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ @@ -35,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.wasm b/pkg/docker/Dockerfile.wasm index fa6dbec6b..096e1020d 100644 --- a/pkg/docker/Dockerfile.wasm +++ b/pkg/docker/Dockerfile.wasm @@ -6,14 +6,15 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ @@ -35,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Makefile b/pkg/docker/Makefile index 0d45fccf4..ef0915ea4 100644 --- a/pkg/docker/Makefile +++ b/pkg/docker/Makefile @@ -37,7 +37,7 @@ RUN_go ?= /bin/true MODULE_PREBUILD_go ?= /bin/true VERSIONS_java ?= $(call eol_versions,java) -VARIANT_java ?= noble +VARIANT_java ?= resolute $(foreach javaversion, $(VERSIONS_java), $(eval CONTAINER_java-$(javaversion) = eclipse-temurin:$(javaversion)-jdk-$(VARIANT_java))) CONFIGURE_java ?= java --jars=/usr/share/unit-jsc-common/ INSTALL_java ?= java-shared-install java-install diff --git a/pkg/docker/README.md b/pkg/docker/README.md index a506112dd..372aadcfa 100644 --- a/pkg/docker/README.md +++ b/pkg/docker/README.md @@ -166,8 +166,8 @@ Builder images (`Dockerfile.builder-*`) are pulled from GHCR automatically, or built locally on first use if unavailable: ``` -Dockerfile.builder-trixie → ghcr.io/freeunitorg/freeunit-builder:trixie-rust1.94.1 -Dockerfile.builder-php8.5 → ghcr.io/freeunitorg/freeunit-builder:php8.5-rust1.94.1 +Dockerfile.builder-trixie → ghcr.io/freeunitorg/freeunit-builder:trixie-rust1.95.0 +Dockerfile.builder-php8.5 → ghcr.io/freeunitorg/freeunit-builder:php8.5-rust1.95.0 ``` The `local/` subdirectory contains the corresponding multi-stage Dockerfiles used @@ -187,8 +187,12 @@ A summary (OK / FAILED) is printed at the end. | `wasm` | debian:trixie-slim | | `go-1.25` | golang:1.25 | | `go-1.26` | golang:1.26 | -| `java-17` | eclipse-temurin:17-jdk-noble | -| `java-21` | eclipse-temurin:21-jdk-noble | +| `go-1.27` | golang:1.27 | +| `java-17` | eclipse-temurin:17-jdk-resolute | +| `java-21` | eclipse-temurin:21-jdk-resolute | +| `java-25` | eclipse-temurin:25-jdk-resolute | +| `java-26` | eclipse-temurin:26-jdk-resolute | +| `java-27` | eclipse-temurin:27-jdk-resolute | | `node-20` | node:20 | | `node-22` | node:22 | | `node-24` | node:24 | @@ -196,6 +200,7 @@ A summary (OK / FAILED) is printed at the end. | `perl-5.38` | perl:5.38 | | `perl-5.40` | perl:5.40 | | `perl-5.42` | perl:5.42 | +| `perl-5.44` | perl:5.44 | | `php-8.3` | php:8.3-cli | | `php-8.4` | php:8.4-cli | | `php-8.5` | php:8.5-cli-trixie | diff --git a/pkg/docker/build-local.sh b/pkg/docker/build-local.sh index bb82c013d..21543a6ba 100755 --- a/pkg/docker/build-local.sh +++ b/pkg/docker/build-local.sh @@ -175,9 +175,9 @@ ensure_builder() { # silently fall back to a local build. local IMG BDF case "$VARIANT" in - minimal|wasm) IMG="ghcr.io/freeunitorg/freeunit-builder:trixie-rust1.94.1" + minimal|wasm) IMG="ghcr.io/freeunitorg/freeunit-builder:trixie-rust1.95.0" BDF="${SCRIPT_DIR}/Dockerfile.builder-trixie" ;; - php-8.5) IMG="ghcr.io/freeunitorg/freeunit-builder:php8.5-rust1.94.1" + php-8.5) IMG="ghcr.io/freeunitorg/freeunit-builder:php8.5-rust1.95.0" BDF="${SCRIPT_DIR}/Dockerfile.builder-php8.5" ;; *) return 0 ;; # no builder for this variant esac diff --git a/pkg/docker/local/Dockerfile.minimal b/pkg/docker/local/Dockerfile.minimal index 5399e34fa..1833623d9 100644 --- a/pkg/docker/local/Dockerfile.minimal +++ b/pkg/docker/local/Dockerfile.minimal @@ -1,11 +1,11 @@ # Fast local build — uses pre-built builder image (no apt/Rust download). # Build builder first: -# docker build -t ghcr.io/freeunitorg/freeunit-builder:trixie-rust1.94.1 \ +# docker build -t ghcr.io/freeunitorg/freeunit-builder:trixie-rust1.95.0 \ # -f ../Dockerfile.builder-trixie .. # Then build this variant: # ../build-local.sh -b minimal -FROM ghcr.io/freeunitorg/freeunit-builder:trixie-rust1.94.1 AS build +FROM ghcr.io/freeunitorg/freeunit-builder:trixie-rust1.95.0 AS build RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ @@ -77,6 +77,7 @@ COPY --from=build /requirements.apt /requirements.apt RUN set -ex \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get --no-install-recommends --no-install-suggests -y install \ curl $(cat /requirements.apt) \ && rm -rf /var/lib/apt/lists/* \ diff --git a/pkg/docker/local/Dockerfile.php-8.5 b/pkg/docker/local/Dockerfile.php-8.5 index e3643c52f..6a73c9c3d 100644 --- a/pkg/docker/local/Dockerfile.php-8.5 +++ b/pkg/docker/local/Dockerfile.php-8.5 @@ -1,11 +1,11 @@ # Fast local build — uses pre-built builder image (no apt/Rust download). # Build builder first: -# docker build -t ghcr.io/freeunitorg/freeunit-builder:php8.5-rust1.94.1 \ +# docker build -t ghcr.io/freeunitorg/freeunit-builder:php8.5-rust1.95.0 \ # -f ../Dockerfile.builder-php8.5 .. # Then build this variant: # ../build-local.sh -b php-8.5 -FROM ghcr.io/freeunitorg/freeunit-builder:php8.5-rust1.94.1 AS build +FROM ghcr.io/freeunitorg/freeunit-builder:php8.5-rust1.95.0 AS build RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ @@ -77,6 +77,7 @@ COPY --from=build /requirements.apt /requirements.apt RUN set -ex \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get --no-install-recommends --no-install-suggests -y install \ curl $(cat /requirements.apt) \ && rm -rf /var/lib/apt/lists/* \ diff --git a/pkg/docker/local/Dockerfile.wasm b/pkg/docker/local/Dockerfile.wasm index 726db14ab..9a3767bf3 100644 --- a/pkg/docker/local/Dockerfile.wasm +++ b/pkg/docker/local/Dockerfile.wasm @@ -1,8 +1,8 @@ # Fast local build — uses pre-built builder image (no apt/Rust download). -# Builder: ghcr.io/freeunitorg/freeunit-builder:trixie-rust1.94.1 +# Builder: ghcr.io/freeunitorg/freeunit-builder:trixie-rust1.95.0 # (same as minimal — wasmtime Rust build is included in that builder) -FROM ghcr.io/freeunitorg/freeunit-builder:trixie-rust1.94.1 AS build +FROM ghcr.io/freeunitorg/freeunit-builder:trixie-rust1.95.0 AS build RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ @@ -81,6 +81,7 @@ COPY --from=build /requirements.apt /requirements.apt RUN set -ex \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get --no-install-recommends --no-install-suggests -y install \ curl $(cat /requirements.apt) \ && rm -rf /var/lib/apt/lists/* \ diff --git a/pkg/docker/template.Dockerfile b/pkg/docker/template.Dockerfile index 1d9355393..44c257ad2 100644 --- a/pkg/docker/template.Dockerfile +++ b/pkg/docker/template.Dockerfile @@ -11,9 +11,10 @@ LABEL org.opencontainers.image.version="@@VERSION@@" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ diff --git a/pkg/eol.json b/pkg/eol.json index 97ccd6db3..62b7c3bfe 100644 --- a/pkg/eol.json +++ b/pkg/eol.json @@ -15,20 +15,20 @@ ], "centos_stream": [ {"version": "9", "eol": "2027-05", "supported_until": "2030-05", "default_python": "3.11"}, - {"version": "10", "eol": "2030-01", "supported_until": "2033-01", "default_python": "3.13"} + {"version": "10", "eol": "2030-05", "supported_until": "2033-05", "default_python": "3.13"} ], "amazonlinux": [ - {"version": "2", "eol": "2026-06", "supported_until": "2029-06", "default_python": "3.7", "default_python_note": "OS-shipped; Python 3.7 upstream EOL Jun 2023"}, + {"version": "2", "eol": "2026-06", "supported_until": "2029-06", "default_python": "3.7", "default_python_note": "OS-shipped; Python 3.7 upstream EOL Jun 2023", "note": "EOL"}, {"version": "2023", "eol": "2029-06", "supported_until": "2032-06", "default_python": "3.11"} ], "ubuntu": [ - {"version": "22.04", "eol": "2027-04", "supported_until": "2030-04", "default_python": "3.10", "lts": true}, + {"version": "22.04", "eol": "2027-06", "supported_until": "2030-06", "default_python": "3.10", "lts": true}, {"version": "24.04", "eol": "2029-05", "supported_until": "2032-05", "default_python": "3.12", "lts": true}, - {"version": "26.04", "eol": "2031-04", "supported_until": "2034-04", "default_python": "3.13", "lts": true} + {"version": "26.04", "eol": "2031-05", "supported_until": "2034-05", "default_python": "3.13", "lts": true} ], "debian": [ {"version": "11", "eol": "2024-08", "supported_until": "2027-08", "default_python": "3.9", "codename": "bullseye", "note": "EOL"}, - {"version": "12", "eol": "2026-07", "supported_until": "2029-07", "default_python": "3.11", "codename": "bookworm"}, + {"version": "12", "eol": "2026-07", "supported_until": "2029-07", "default_python": "3.11", "codename": "bookworm", "note": "EOL"}, {"version": "13", "eol": "2028-08", "supported_until": "2031-08", "default_python": "3.13", "codename": "trixie"} ], "rhel": [ @@ -46,14 +46,16 @@ }, "runtimes": { "go": [ - {"version": "1.25", "eol": "2026-08", "supported_until": "2027-08"}, - {"version": "1.26", "eol": "2027-02", "supported_until": "2028-02"} + {"version": "1.25", "eol": "2026-08", "supported_until": "2027-08", "note": "EOL"}, + {"version": "1.26", "eol": "2027-02", "supported_until": "2028-02"}, + {"version": "1.27", "eol": "2027-08", "supported_until": "2028-08"} ], "java": [ - {"version": "17", "eol": "2027-10", "supported_until": "2028-10", "base": "eclipse-temurin"}, - {"version": "21", "eol": "2029-12", "supported_until": "2030-12", "base": "eclipse-temurin"}, + {"version": "17", "eol": "2027-10", "supported_until": "2028-10", "base": "eclipse-temurin", "lts": true}, + {"version": "21", "eol": "2029-12", "supported_until": "2030-12", "base": "eclipse-temurin", "lts": true}, {"version": "25", "eol": "2031-09", "supported_until": "2032-09", "base": "eclipse-temurin", "lts": true}, - {"version": "26", "eol": "2026-09", "supported_until": "2027-09", "base": "eclipse-temurin"} + {"version": "26", "eol": "2026-09", "supported_until": "2027-09", "base": "eclipse-temurin", "note": "EOL"}, + {"version": "27", "eol": "2027-03", "supported_until": "2028-03", "base": "eclipse-temurin"} ], "node": [ {"version": "20", "eol": "2026-04", "supported_until": "2027-04", "note": "EOL"}, @@ -62,9 +64,10 @@ {"version": "26", "eol": "2029-04", "supported_until": "2030-04", "lts": true} ], "perl": [ - {"version": "5.38", "eol": "2026-07", "supported_until": "2027-07"}, + {"version": "5.38", "eol": "2026-07", "supported_until": "2027-07", "note": "EOL"}, {"version": "5.40", "eol": "2027-06", "supported_until": "2028-06"}, - {"version": "5.42", "eol": "2028-07", "supported_until": "2029-07"} + {"version": "5.42", "eol": "2028-07", "supported_until": "2029-07"}, + {"version": "5.44", "eol": "2029-07", "supported_until": "2030-07"} ], "php": [ {"version": "8.3", "eol": "2027-12", "supported_until": "2028-12"}, @@ -87,5 +90,59 @@ "minimal": [ {"version": null, "eol": null, "supported_until": null} ] + }, + "_comment_dependencies": "Build-time and bundled library floors/ceilings. NOT a matrix: only .os and .runtimes feed the Docker/CI variant lists (build-test.yml, release-docker.yml, pkg/docker/Makefile, pkg/docker/build-local.sh), and pkg/eol/src/main.rs reads only _grace_*, .os and .runtimes — unknown top-level keys are ignored. Entries here are documentation for humans; the EOL + grace policy does not apply to them, so they carry no supported_until.", + "dependencies": { + "openssl": { + "role": "TLS backend (./configure --openssl)", + "floor": "1.1.1", + "floor_eol": "2023-09", + "floor_note": "The floor stays at 1.1.1 rather than 3.x because RHEL 8 (openssl-devel) still ships a vendor-patched 1.1.1 that Red Hat maintains into 2029, and from 2026-09 it is the last platform in the matrix still receiving those patches; Amazon Linux 2 builds against openssl11-devel (see pkg/rpm/Makefile and pkg/rpm/unit.spec.in). Amazon Linux 2 and Debian 11 remain inside FreeUnit's own three-year post-EOL support window (see _grace_os).", + "floor_declared_by": "auto/ssltls probe (PR #224), which also drops the pre-1.1.0 compatibility code in src/nxt_openssl.c", + "tested_ceiling": "3.6.5 and 4.0.3", + "tested_ceiling_note": "build-test.yml tests two ceilings from source: 3.6.5 (OPENSSL_VERSION env, built into /opt/openssl-3.6) and 4.0.3 (OPENSSL4_VERSION env, /opt/openssl-4.0, built by the openssl4 job with --cc-opt=-DOPENSSL_NO_DEPRECATED so any use of an API OpenSSL 4.0 deprecates fails the build). ACTION: OpenSSL 3.6 goes upstream-EOL 2026-11-01, so the OPENSSL_VERSION pin needs a bump to a supported branch (3.5 is the LTS, EOL 2030-04); 4.0 is supported to 2027-05.", + "source": "https://endoflife.date/api/openssl.json" + }, + "tomcat": { + "role": "Servlet/JSP/EL API + Jasper jars bundled by the java module (auto/modules/java, NXT_TOMCAT_VERSION)", + "version": "9.0.x", + "eol": "2027-03", + "eol_note": "Apache: \"End of support for Tomcat 9.0.x is expected no earlier than 31 March 2027\", with at least 12 months notice. Not a Docker variant — the jars are bundled into the java module regardless of JDK version.", + "source": "https://tomcat.apache.org/whichversion.html" + }, + "jetty": { + "role": "HTTP parsing/server jars bundled by the java module (auto/modules/java, NXT_JAR_VERSION under org/eclipse/jetty/): jetty-util, jetty-server, jetty-http", + "version": "9.4.58.v20250814", + "eol": "2025-08", + "eol_note": "ALREADY EOL. Eclipse ended community support for the Jetty 9.4 line on 2025-08-14, and 9.4.58.v20250814 — the exact build bundled here — is its final release (jetty.org/download.html lists it as (EOL); the 9.4 row reads \"2016-2025 ... EOL / Unsupported\"). No supported_until is recorded: the EOL + grace policy covers matrix variants, not bundled jars, and this needs an upgrade-or-replace decision (Jetty 10/11 are also EOL as of 2025-01-01; 12.0/12.1 are the supported lines and require Java 11+/17+), not a grace window.", + "source": "https://endoflife.date/api/eclipse-jetty.json" + }, + "ecj": { + "role": "Eclipse JDT batch compiler jar bundled by the java module (auto/modules/java, org/eclipse/jdt/), used to compile JSPs at runtime", + "version": "3.26.0", + "eol": null, + "eol_note": "Tracks Eclipse releases; no published EOL. endoflife.date has no product for it (checked /api/all.json — the only Eclipse products are eclipse-jetty and eclipse-temurin), so there is no upstream date to validate against. The pinned 3.26.0 is from 2021-06-14; the current release is 3.42.0 (2025-06-07), so this is stale rather than expired.", + "source": "https://central.sonatype.com/artifact/org.eclipse.jdt/ecj/versions" + }, + "classgraph": { + "role": "classpath scanning jar bundled by the java module (auto/modules/java, NXT_JAR_VERSION)", + "version": "latest", + "note": "No floor and no EOL schedule upstream: pinned to the current release and bumped as needed.", + "source": "https://github.com/classgraph/classgraph/releases" + }, + "wasmtime_c_api": { + "role": "WebAssembly runtime built by pkg/contrib and linked by the wasm module (pkg/contrib/src/wasmtime/version, WASMTIME_VERSION)", + "version": "48.0.5", + "eol": null, + "eol_note": "Fetched as a source tarball from packages.freeunit.org, so its Cargo.lock is not among the repository locks. The Audit (cargo) pins job verifies the tarball digest, queries OSV for the direct source-package pin, and audits the archived Cargo.lock, so an advisory here arrives as a red job rather than silence. That lock is the whole Wasmtime workspace, so pkg/contrib/check-lock-audit.py fails it only for a crate the C API library links, with pkg/contrib/wasmtime-lock-audit.allow.json recording the accepted ones; the archive is vendored, so a finding moves only when the pin does. Same 48.0.5 as the wasmtime_crate pin. Wasmtime 48 is a long-term support line: upstream supports a release whose number divides by 12 for 24 months and backports security fixes to it. The 47.0.4 to 48.0.5 bump closed twelve advisories published on 2026-09-24 and 2026-10-02. The 43.0.1 bump closed RUSTSEC-2026-0269 (filesystem sandbox escape), -0222, -0114, and wasmtime-wasi -0149, -0182 and -0188 (WASI path and file-permission bypasses). Tracked in freeunitorg/freeunit#400.", + "source": "https://github.com/bytecodealliance/wasmtime/releases" + }, + "wasmtime_crate": { + "role": "WebAssembly runtime used by the wasm-wasi-component module (src/wasm-wasi-component/Cargo.lock)", + "version": "48.0.5", + "eol": null, + "eol_note": "Audited by the Audit (cargo) workflow, which checks every tracked Cargo.lock against the RustSec database, so an advisory here arrives as a red job rather than silence. Bumped with the advisories it closes.", + "source": "https://github.com/bytecodealliance/wasmtime/releases" + } } } diff --git a/pkg/eol/Cargo.lock b/pkg/eol/Cargo.lock index 19e2a6f70..93fc6d1b4 100644 --- a/pkg/eol/Cargo.lock +++ b/pkg/eol/Cargo.lock @@ -34,27 +34,27 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", ] [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", @@ -63,9 +63,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.149" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ "itoa", "memchr", @@ -76,9 +76,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.117" +version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" dependencies = [ "proc-macro2", "quote", diff --git a/pkg/eol/Cargo.toml b/pkg/eol/Cargo.toml index c72f878b7..68b514eec 100644 --- a/pkg/eol/Cargo.toml +++ b/pkg/eol/Cargo.toml @@ -2,6 +2,7 @@ name = "unit-eol-check" version = "1.0.0" edition = "2021" +rust-version = "1.78" [dependencies] serde_json = "1" diff --git a/pkg/eol/README.md b/pkg/eol/README.md index 453a3b576..a5fda0a1b 100644 --- a/pkg/eol/README.md +++ b/pkg/eol/README.md @@ -32,27 +32,29 @@ Exit codes: `0` = clean or warnings only · `1` = errors found (drift, expiry, o Errors include the **expiry gate**: any runtime or OS entry whose `supported_until` is already in the past (EOL + grace elapsed) is a hard error and must be dropped from `pkg/eol.json`. -## Current EOL Status (2026-07-13) +## Current EOL Status (2026-09-08) All dates verified against endoflife.date API. No errors. Warnings below are proximity/past-EOL alerts only — every entry is still inside its grace window. +Dates are compared per category against the field that carries *standard* support: `support` for Debian (its `eol` became the end of LTS in September 2026), `eol` for everything else. + ### Runtimes — warnings -None. All shipped runtimes are within (or ahead of) upstream EOL; `node 20` is already flagged `(EOL)` in the matrix. +None. All shipped runtimes are within (or ahead of) upstream EOL; `go 1.25`, `node 20` and `perl 5.38` are already flagged `(EOL)` in the matrix. ### OS — warnings | Entry | Note | |-------|------| | debian 11 | Past EOL (2024-08), grace until 2027-08 | -| fedora 40, 41, 42 | Past EOL (2025-05 / 2025-12 / 2026-05), grace active | +| fedora 40, 41, 42 | Past EOL (2025-05 / 2025-12 / 2026-05), grace until 2028-05 / 2028-12 / 2029-05 | | amazonlinux 2 | Past EOL (2026-06), grace until 2029-06 | | alpine 3.20 | Past EOL (2026-04), grace until 2029-04 | -| debian 12 | EOL now (2026-07), grace until 2029-07 | -| fedora 43, 44 | EOL in ~5 / ~11 months | -| alpine 3.21, 3.22 | EOL in ~4 / ~10 months | -| centos_stream 9 | EOL in ~10 months (2027-05) | -| ubuntu 22.04 | EOL in ~9 months (2027-04) | +| debian 12 | Past EOL (2026-07), grace until 2029-07 | +| fedora 43, 44 | EOL in ~3 / ~9 months | +| alpine 3.21, 3.22 | EOL in ~2 / ~8 months | +| centos_stream 9 | EOL in ~8 months (2027-05) | +| ubuntu 22.04 | EOL in ~9 months (2027-06) | ## Architecture diff --git a/pkg/eol/src/main.rs b/pkg/eol/src/main.rs index d83fccc07..87bf8a4fc 100644 --- a/pkg/eol/src/main.rs +++ b/pkg/eol/src/main.rs @@ -204,6 +204,22 @@ fn fetch_api(category: &str) -> Result { // API date lookup // --------------------------------------------------------------------------- +/// endoflife.date field holding the date FreeUnit's policy is anchored to. +/// +/// EOL.md: the three-year OS extension runs from *standard* EOL, never from +/// extended maintenance. For every vendor but one that is the API's `eol`. +/// Debian is the exception: in September 2026 endoflife.date redefined Debian's +/// `eol` to the end of LTS (`eolFrom` in the v1 API) and moved the end of +/// regular security-team support — the date the policy means — to `support` +/// (`eoasFrom`). Reading `eol` for Debian would silently re-anchor the grace +/// window to LTS and keep bullseye in the matrix two years past policy. +fn api_eol_field(category: &str) -> &'static str { + match category { + "debian" => "support", + _ => "eol", + } +} + /// Result of an API EOL lookup: Ok(Some(date)), Ok(None) (version not found in API), /// or Err (network/parse failure). fn api_eol_date(category: &str, version: &str) -> Result, String> { @@ -223,7 +239,7 @@ fn api_eol_date(category: &str, version: &str) -> Result, String> None => continue, }; if cycle == version { - if let Some(eol_val) = entry.get("eol") { + if let Some(eol_val) = entry.get(api_eol_field(category)) { match eol_val { serde_json::Value::String(s) => { // Normalize to YYYY-MM (API may return YYYY-MM or YYYY-MM-DD) @@ -1149,4 +1165,15 @@ mod tests { assert_eq!(r[0].severity, Severity::Error); assert!(r[0].message.contains("unparseable")); } + + #[test] + fn debian_is_compared_against_the_support_field() { + // Debian's API `eol` became the end of LTS in September 2026; the + // policy anchors on the end of regular security support, which the API + // now calls `support`. Every other category still means `eol`. + assert_eq!(api_eol_field("debian"), "support"); + assert_eq!(api_eol_field("ubuntu"), "eol"); + assert_eq!(api_eol_field("alpine"), "eol"); + assert_eq!(api_eol_field("python"), "eol"); + } } diff --git a/src/java/nginx/unit/Request.java b/src/java/nginx/unit/Request.java index 335d79805..0352957ef 100644 --- a/src/java/nginx/unit/Request.java +++ b/src/java/nginx/unit/Request.java @@ -1310,8 +1310,20 @@ public void sendWsFrame(ByteBuffer payload, byte opCode, boolean last, sendWsFrame(req_info_ptr, payload, payload.position(), payload.limit() - payload.position(), opCode, last); } else { - sendWsFrame(req_info_ptr, payload.array(), payload.position(), - payload.limit() - payload.position(), opCode, last); + if (!payload.hasArray()) { + /* + * A read-only heap buffer hides its array, so each frame + * sent from one costs a copy of its remaining bytes. + */ + ByteBuffer copy = ByteBuffer.allocate(payload.remaining()); + copy.put(payload.duplicate()); + copy.flip(); + payload = copy; + } + + sendWsFrame(req_info_ptr, payload.array(), + payload.arrayOffset() + payload.position(), + payload.remaining(), opCode, last); } } diff --git a/src/java/nginx/unit/websocket/Constants.java b/src/java/nginx/unit/websocket/Constants.java index 38b22fe08..bef5bb399 100644 --- a/src/java/nginx/unit/websocket/Constants.java +++ b/src/java/nginx/unit/websocket/Constants.java @@ -45,6 +45,15 @@ public class Constants { "nginx.unit.websocket.DEFAULT_BUFFER_SIZE", 8 * 1024) .intValue(); + /* + * The largest text message that is encoded into a buffer of its own, and + * therefore sent as a single frame. A larger message is encoded through + * DEFAULT_BUFFER_SIZE and fragments. + */ + static final int MAX_SEND_BUFFER_SIZE = Integer.getInteger( + "nginx.unit.websocket.MAX_SEND_BUFFER_SIZE", 16 * 1024 * 1024) + .intValue(); + // Client connection /** * Property name to set to configure the value that is passed to diff --git a/src/java/nginx/unit/websocket/LocalStrings.properties b/src/java/nginx/unit/websocket/LocalStrings.properties index aeafe0823..a814a11c0 100644 --- a/src/java/nginx/unit/websocket/LocalStrings.properties +++ b/src/java/nginx/unit/websocket/LocalStrings.properties @@ -57,6 +57,7 @@ wsFrame.alreadySuspended=Message receiving has already been suspended. wsFrame.bufferTooSmall=No async message support and buffer too small. Buffer size: [{0}], Message size: [{1}] wsFrame.byteToLongFail=Too many bytes ([{0}]) were provided to be converted into a long wsFrame.closed=New frame received after a close control frame +wsFrame.binaryMessageTooBig=The binary message was too big for the output buffer and the endpoint does not support partial messages wsFrame.controlFragmented=A fragmented control frame was received but control frames may not be fragmented wsFrame.controlPayloadTooBig=A control frame was sent with a payload of size [{0}] which is larger than the maximum permitted of 125 bytes wsFrame.controlNoFin=A control frame was sent that did not have the fin bit set. Control frames are not permitted to use continuation frames. diff --git a/src/java/nginx/unit/websocket/WsRemoteEndpointImplBase.java b/src/java/nginx/unit/websocket/WsRemoteEndpointImplBase.java index d451db7d2..3e2fd9a79 100644 --- a/src/java/nginx/unit/websocket/WsRemoteEndpointImplBase.java +++ b/src/java/nginx/unit/websocket/WsRemoteEndpointImplBase.java @@ -28,9 +28,11 @@ import java.util.ArrayList; import java.util.List; import java.util.Queue; +import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import javax.websocket.CloseReason; @@ -120,9 +122,27 @@ public boolean getBatchingAllowed() { } + /* + * The batched frames sit in outputBuffer, which only writeMessagePart() + * knows how to hand to doWrite(), so the flush goes through startMessage() + * like any other message part. sendMessageBlock() writes straight to the + * request instead, and with no payload it has nothing to write. + */ @Override public void flushBatch() throws IOException { - sendMessageBlock(Constants.INTERNAL_OPCODE_FLUSH, null, true); + FutureToSendHandler f2sh = new FutureToSendHandler(wsSession); + startMessage(Constants.INTERNAL_OPCODE_FLUSH, null, true, f2sh); + + try { + long timeout = getBlockingSendTimeout(); + if (timeout < 0) { + f2sh.get(); + } else { + f2sh.get(timeout, TimeUnit.MILLISECONDS); + } + } catch (InterruptedException | ExecutionException | TimeoutException e) { + throw new IOException(e); + } } @@ -211,8 +231,9 @@ public void sendStringByCompletion(String text, SendHandler handler) { throw new IllegalArgumentException(sm.getString("wsRemoteEndpoint.nullHandler")); } stateMachine.textStart(); + CharBuffer part = CharBuffer.wrap(text); TextMessageSendHandler tmsh = new TextMessageSendHandler(handler, - CharBuffer.wrap(text), true, encoder, encoderBuffer, this); + part, true, encoder, encodeBuffer(part), this); tmsh.write(); // TextMessageSendHandler will update stateMachine when it completes } @@ -240,18 +261,95 @@ public Writer getSendWriter() { } + private static final long OVER_CAP = -1; + + /* + * The exact UTF-8 encoded length of part, without consuming it, or + * OVER_CAP once the length passes cap. The scan stops at the cap, so an + * oversized message does not pay for a full pass. + * + * The widths below are Utf8Encoder's own: one byte up to 0x7F, two up to + * 0x7FF, four for a high surrogate followed by a low one, three for the + * rest. An unpaired surrogate lands in the last branch, and the width + * charged for it there is arbitrary, because such a message never reaches + * the wire: Utf8Encoder returns malformedForLength(1) for an unpaired + * surrogate inside the buffer, and UNDERFLOW for one at its very end, + * which CharsetEncoder.encode(in, out, true) turns into malformed input + * under the default REPORT action. Either way the encode below reports an + * error and throws. + */ + private static long utf8Length(CharBuffer part, long cap) { + long len = 0; + int limit = part.limit(); + + for (int i = part.position(); i < limit; i++) { + char c = part.get(i); + + if (c < 0x80) { + len += 1; + } else if (c < 0x800) { + len += 2; + } else if (Character.isHighSurrogate(c) && i + 1 < limit + && Character.isLowSurrogate(part.get(i + 1))) + { + len += 4; + i++; + } else { + len += 3; + } + + /* + * Inclusive, and load-bearing: a message of exactly cap bytes + * still gets a buffer of its own. Autobahn case 9.1.6 sends + * 16777216 bytes, which is the default cap to the byte, so a + * ">=" here would fragment the largest case back into 2048 frames + * and fail with the very fin mismatch this method exists to fix. + */ + if (len > cap) { + return OVER_CAP; + } + } + + return len; + } + + + /* + * A message that fits under the cap gets a buffer of its own, so it leaves + * as one frame: Utf8Encoder returns UNDERFLOW when input and output run + * out together. Any other message uses encoderBuffer and fragments as + * before. The buffer is on the heap, like encoderBuffer, so both take the + * same JNI path. + * + * The price, on the blocking and the asynchronous path alike: a full + * utf8Length() pass before the first byte is sent, and a buffer of the + * message's encoded length, up to 16 MiB by default. + */ + private ByteBuffer encodeBuffer(CharBuffer part) { + long needed = utf8Length(part, Constants.MAX_SEND_BUFFER_SIZE); + + if (needed != OVER_CAP && needed > encoderBuffer.capacity()) { + return ByteBuffer.allocate((int) needed); + } + + return encoderBuffer; + } + + void sendMessageBlock(CharBuffer part, boolean last) throws IOException { long timeoutExpiry = getTimeoutExpiry(); boolean isDone = false; + ByteBuffer buffer = encodeBuffer(part); + while (!isDone) { - encoderBuffer.clear(); - CoderResult cr = encoder.encode(part, encoderBuffer, true); + buffer.clear(); + CoderResult cr = encoder.encode(part, buffer, true); if (cr.isError()) { throw new IllegalArgumentException(cr.toString()); } isDone = !cr.isOverflow(); - encoderBuffer.flip(); - sendMessageBlock(Constants.OPCODE_TEXT, encoderBuffer, last && isDone, timeoutExpiry); + buffer.flip(); + sendMessageBlock(Constants.OPCODE_TEXT, buffer, last && isDone, timeoutExpiry); } stateMachine.complete(last); } @@ -307,7 +405,16 @@ void startMessage(byte opCode, ByteBuffer payload, boolean last, intermediateMessageHandler, new EndMessageHandler(this, handler), -1)); - messageParts = transformation.sendMessagePart(messageParts); + /* + * On the server side there is no transformation: Tomcat's + * WsHttpUpgradeHandler.init() takes it from the WsFrameServer it + * creates, and Unit parses the frames itself, so no WsFrameServer is + * ever created and no transformation is installed. The message + * parts then go out as they are. + */ + if (transformation != null) { + messageParts = transformation.sendMessagePart(messageParts); + } // Some extensions/transformations may buffer messages so it is possible // that no message parts will be returned. If this is the case the @@ -642,6 +749,22 @@ protected void setRequest(Request request) { this.request = request; } + + /* + * The transport doWrite() sends through. Unit writes the frame header + * itself, so a frame is handed over as its payload, opcode and fin bit + * rather than as the serialised header writeHeader() built. + */ + protected final void sendWsFrame(ByteBuffer payload, byte opCode, + boolean fin, long timeoutExpiry) throws IOException { + request.sendWsFrame(payload, opCode, fin, timeoutExpiry); + } + + + protected final boolean isSessionOpen() { + return wsSession.isOpen(); + } + protected void setEncoders(EndpointConfig endpointConfig) throws DeploymentException { encoderEntries.clear(); @@ -714,13 +837,20 @@ private static void writeHeader(ByteBuffer headerBuffer, boolean fin, b = 0; } + /* + * The frame carries position()..limit(), which is what both write + * paths send; limit() alone overstates it for a buffer that does not + * start at 0, such as a slice or a partly read one. + */ + int len = payload.remaining(); + // Next write the mask && length length - if (payload.limit() < 126) { - headerBuffer.put((byte) (payload.limit() | b)); - } else if (payload.limit() < 65536) { + if (len < 126) { + headerBuffer.put((byte) (len | b)); + } else if (len < 65536) { headerBuffer.put((byte) (126 | b)); - headerBuffer.put((byte) (payload.limit() >>> 8)); - headerBuffer.put((byte) (payload.limit() & 0xFF)); + headerBuffer.put((byte) (len >>> 8)); + headerBuffer.put((byte) (len & 0xFF)); } else { // Will never be more than 2^31-1 headerBuffer.put((byte) (127 | b)); @@ -728,10 +858,10 @@ private static void writeHeader(ByteBuffer headerBuffer, boolean fin, headerBuffer.put((byte) 0); headerBuffer.put((byte) 0); headerBuffer.put((byte) 0); - headerBuffer.put((byte) (payload.limit() >>> 24)); - headerBuffer.put((byte) (payload.limit() >>> 16)); - headerBuffer.put((byte) (payload.limit() >>> 8)); - headerBuffer.put((byte) (payload.limit() & 0xFF)); + headerBuffer.put((byte) (len >>> 24)); + headerBuffer.put((byte) (len >>> 16)); + headerBuffer.put((byte) (len >>> 8)); + headerBuffer.put((byte) (len & 0xFF)); } if (masked) { headerBuffer.put(mask[0]); diff --git a/src/java/nginx/unit/websocket/WsSession.java b/src/java/nginx/unit/websocket/WsSession.java index b654eb372..3bd3064c0 100644 --- a/src/java/nginx/unit/websocket/WsSession.java +++ b/src/java/nginx/unit/websocket/WsSession.java @@ -989,7 +989,7 @@ public void onMessage(ByteBuffer buf, boolean last) if (binaryBuffer.remaining() < buf.remaining()) { throw new WsIOException(new CloseReason( CloseCodes.TOO_BIG, - sm.getString("wsFrame.textMessageTooBig"))); + sm.getString("wsFrame.binaryMessageTooBig"))); } binaryBuffer.put(buf); diff --git a/src/java/nginx/unit/websocket/server/LocalStrings.properties b/src/java/nginx/unit/websocket/server/LocalStrings.properties index 5bc125017..4fd7b2810 100644 --- a/src/java/nginx/unit/websocket/server/LocalStrings.properties +++ b/src/java/nginx/unit/websocket/server/LocalStrings.properties @@ -41,3 +41,5 @@ wsHttpUpgradeHandler.noPreInit=The preInit() method must be called to configure wsHttpUpgradeHandler.serverStop=The server is stopping wsRemoteEndpointServer.closeFailed=Failed to close the ServletOutputStream connection cleanly +wsRemoteEndpointServer.closed=Message will not be sent because the WebSocket session has been closed +wsRemoteEndpointServer.badFrame=The output buffer holds a frame of [{0}] bytes, more than a message may carry diff --git a/src/java/nginx/unit/websocket/server/WsRemoteEndpointImplServer.java b/src/java/nginx/unit/websocket/server/WsRemoteEndpointImplServer.java index 6d10a3bee..c4f1f4428 100644 --- a/src/java/nginx/unit/websocket/server/WsRemoteEndpointImplServer.java +++ b/src/java/nginx/unit/websocket/server/WsRemoteEndpointImplServer.java @@ -22,6 +22,8 @@ import java.nio.ByteBuffer; import java.nio.channels.CompletionHandler; import java.nio.channels.InterruptedByTimeoutException; +import java.util.ArrayDeque; +import java.util.Queue; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.TimeUnit; @@ -50,6 +52,38 @@ public class WsRemoteEndpointImplServer extends WsRemoteEndpointImplBase { private volatile long timeoutExpiry = -1; private volatile boolean close; + /* + * A frame that the batching path split across two doWrite() calls: the + * base class fills its output buffer and flushes it wherever it happens + * to be full, so a frame's header, or its payload, may arrive in parts. + * A server frame is not masked, so its header is at most 10 bytes. + */ + private final byte[] header = new byte[10]; + private int headerLength = 0; + private byte frameOpCode; + private boolean frameFin; + private long frameLeft = 0; + private ByteBuffer framePayload = null; + + /* + * A completion runs on the thread that sent. A handler that sends from + * onResult(), as TextMessageSendHandler does for each 8 KiB of a text + * message, nests a doWrite() inside the completion, so a large message + * would nest thousands deep. Completions nest inline up to MAX_DEPTH and + * are queued beyond it, to be run by the outermost completion on this + * thread. Inline, not always queued: a handler that waits on a send it + * made itself (Future.get(), flushBatch(), close() with batching on) needs + * that send's completion to run before it returns. + * + * The state is per thread: once a completion releases the message part, + * another thread may send on this endpoint while the first is still + * inside its completion. + */ + private static final int MAX_DEPTH = 32; + + private static final ThreadLocal completions = + new ThreadLocal<>(); + public WsRemoteEndpointImplServer( WsServerContainer serverContainer) { } @@ -60,9 +94,217 @@ protected final boolean isMasked() { return false; } + /* + * Sends the frames the base class serialised into buffers. Unit writes + * the frame header itself, so each frame is decoded back into its opcode, + * fin bit and payload and handed to sendWsFrame(). + * + * writeMessagePart() sends a frame directly as its header and its payload, + * in that order. With batching allowed it copies frames into its output + * buffer instead and passes that buffer whenever it fills or is flushed, + * so that path is decoded as a byte stream that may split a frame across + * calls. + * + * The write is synchronous: sendWsFrame() has copied the payload to the + * router by the time it returns, so the handler is completed here, on the + * calling thread, with the outcome. + */ @Override protected void doWrite(SendHandler handler, long blockingWriteTimeoutExpiry, ByteBuffer... buffers) { + SendResult result = SENDRESULT_OK; + + try { + if (!isSessionOpen()) { + throw new IOException(sm.getString("wsRemoteEndpointServer.closed")); + } + + if (buffers.length == 2 && headerLength == 0 && frameLeft == 0) { + // Direct path: the frame's header, then its payload. + ByteBuffer hdr = buffers[0]; + ByteBuffer payload = buffers[1]; + byte b = hdr.get(hdr.position()); + + sendWsFrame(payload, (byte) (b & 0x0F), (b & 0x80) != 0, + blockingWriteTimeoutExpiry); + + hdr.position(hdr.limit()); + payload.position(payload.limit()); + } else { + for (ByteBuffer buffer : buffers) { + sendFrames(buffer, blockingWriteTimeoutExpiry); + } + } + } catch (IOException | RuntimeException e) { + /* + * A frame the stream decoder had started is lost with the send; + * start the next doWrite() at a frame boundary. + */ + headerLength = 0; + frameLeft = 0; + framePayload = null; + + result = new SendResult(e); + } + + complete(handler, result); + } + + + private void sendFrames(ByteBuffer buffer, long timeoutExpiry) + throws IOException { + while (buffer.hasRemaining()) { + if (frameLeft == 0 && framePayload == null) { + if (!readHeader(buffer)) { + // The rest of the header is in a later buffer + return; + } + if (frameLeft == 0) { + sendWsFrame(ByteBuffer.allocate(0), frameOpCode, frameFin, + timeoutExpiry); + continue; + } + } + + int limit = buffer.limit(); + + if (framePayload == null && buffer.remaining() >= frameLeft) { + // The whole payload is here: send it in place + buffer.limit(buffer.position() + (int) frameLeft); + sendWsFrame(buffer, frameOpCode, frameFin, timeoutExpiry); + buffer.position(buffer.limit()); + buffer.limit(limit); + frameLeft = 0; + continue; + } + + // The payload is split across buffers: gather it + if (framePayload == null) { + framePayload = ByteBuffer.allocate((int) frameLeft); + } + + int n = Math.min(buffer.remaining(), framePayload.remaining()); + buffer.limit(buffer.position() + n); + framePayload.put(buffer); + buffer.limit(limit); + frameLeft -= n; + + if (frameLeft == 0) { + framePayload.flip(); + ByteBuffer payload = framePayload; + framePayload = null; + sendWsFrame(payload, frameOpCode, frameFin, timeoutExpiry); + } + } + } + + + /* + * Reads header bytes from buffer until the header is complete. Returns + * false when buffer ran out first; the bytes read so far are kept for the + * next call. + */ + private boolean readHeader(ByteBuffer buffer) throws IOException { + while (buffer.hasRemaining()) { + header[headerLength++] = buffer.get(); + + if (headerLength < 2) { + continue; + } + + int len = header[1] & 0x7F; + int need = len == 126 ? 4 : len == 127 ? 10 : 2; + + if (headerLength < need) { + continue; + } + + frameFin = (header[0] & 0x80) != 0; + frameOpCode = (byte) (header[0] & 0x0F); + + if (len == 126) { + frameLeft = ((header[2] & 0xFF) << 8) | (header[3] & 0xFF); + } else if (len == 127) { + frameLeft = 0; + for (int i = 2; i < 10; i++) { + frameLeft = (frameLeft << 8) | (header[i] & 0xFF); + } + if (frameLeft < 0 || frameLeft > Integer.MAX_VALUE) { + headerLength = 0; + throw new IOException(sm.getString( + "wsRemoteEndpointServer.badFrame", + Long.valueOf(frameLeft))); + } + } else { + frameLeft = len; + } + + headerLength = 0; + return true; + } + + return false; + } + + + private void complete(SendHandler handler, SendResult result) { + Completions c = completions.get(); + + if (c == null) { + c = new Completions(); + completions.set(c); + + } else if (c.depth >= MAX_DEPTH) { + c.queue.add(new Completion(handler, result)); + return; + } + + boolean outermost = c.depth == 0; + + c.depth++; + try { + handler.onResult(result); + } finally { + c.depth--; + + if (outermost) { + drain(c); + } + } + } + + + private static void drain(Completions c) { + try { + Completion q; + while ((q = c.queue.poll()) != null) { + c.depth++; + try { + q.handler.onResult(q.result); + } finally { + c.depth--; + } + } + } finally { + completions.remove(); + } + } + + + private static class Completions { + private int depth = 0; + private final Queue queue = new ArrayDeque<>(); + } + + + private static class Completion { + private final SendHandler handler; + private final SendResult result; + + private Completion(SendHandler handler, SendResult result) { + this.handler = handler; + this.result = result; + } } @Override diff --git a/src/java/nxt_jni_Request.c b/src/java/nxt_jni_Request.c index 4c3ce534e..1e31f57b0 100644 --- a/src/java/nxt_jni_Request.c +++ b/src/java/nxt_jni_Request.c @@ -597,27 +597,12 @@ nxt_java_Request_getScheme(JNIEnv *env, jclass cls, jlong req_ptr) static jstring JNICALL nxt_java_Request_getServerName(JNIEnv *env, jclass cls, jlong req_ptr) { - char *host, *colon; - nxt_unit_field_t *f; nxt_unit_request_t *r; r = nxt_jlong2ptr(req_ptr); - f = nxt_java_findHeader(r->fields, r->fields + r->fields_count, - "Host", 4); - if (f != NULL) { - host = nxt_unit_sptr_get(&f->value); - - colon = memchr(host, ':', f->value_length); - - if (colon == NULL) { - colon = host + f->value_length; - } - - return nxt_java_newString(env, host, colon - host); - } - - return nxt_java_Request_getLocalName(env, cls, req_ptr); + return nxt_java_newString(env, nxt_unit_sptr_get(&r->server_name), + r->server_name_length); } diff --git a/src/nodejs/unit-http/unit.cpp b/src/nodejs/unit-http/unit.cpp index 7d9395bb6..a55733ae7 100644 --- a/src/nodejs/unit-http/unit.cpp +++ b/src/nodejs/unit-http/unit.cpp @@ -20,6 +20,7 @@ struct port_data_t { port_data_t(nxt_unit_ctx_t *c, nxt_unit_port_t *p); void process_port_msg(); + void cancel(); void stop(); template @@ -31,10 +32,12 @@ struct port_data_t { nxt_unit_ctx_t *ctx; nxt_unit_port_t *port; + port_data_t *next; uv_poll_t poll; uv_timer_t timer; int ref_count; bool scheduled; + bool delayed; bool stopped; }; @@ -48,7 +51,8 @@ struct req_data_t { port_data_t::port_data_t(nxt_unit_ctx_t *c, nxt_unit_port_t *p) : - ctx(c), port(p), ref_count(0), scheduled(false), stopped(false) + ctx(c), port(p), next(NULL), ref_count(0), scheduled(false), + delayed(false), stopped(false) { timer.type = UV_UNKNOWN_HANDLE; } @@ -57,7 +61,9 @@ port_data_t::port_data_t(nxt_unit_ctx_t *c, nxt_unit_port_t *p) : void port_data_t::process_port_msg() { - int rc, err; + int rc, err, delay; + Unit *obj; + port_data_t *data; rc = nxt_unit_process_port_msg(ctx, port); @@ -65,6 +71,27 @@ port_data_t::process_port_msg() return; } + /* + * A QUIT in this call runs Unit::quit(), which stops all ports and + * calls nxt_unit_done(). The context can be freed now. Do not use it. + * The port data stays until the uv close callbacks run. + */ + + if (stopped) { + return; + } + + /* + * NXT_UNIT_OK also comes when the call found no message and a FINISH + * retry is pending. The retry is not due before a deadline. Wait for + * it with the timer, and do not spin the loop with zero timeouts. + */ + + delay = nxt_unit_detached_retry_timeout(ctx); + if (delay < 0) { + delay = 0; + } + if (timer.type == UV_UNKNOWN_HANDLE) { err = uv_timer_init(poll.loop, &timer); if (err < 0) { @@ -76,19 +103,79 @@ port_data_t::process_port_msg() timer.data = this; } - if (!scheduled && !stopped) { - uv_timer_start(&timer, timer_callback, 0, 0); + /* + * Any call runs the retry of the context, whatever the port. So one + * delayed call for each context is enough. A delayed call is pending + * on this port or on the other port of the context. Keep it, unless + * this call processed a message. Then more messages can wait, so + * call again at once, and cancel the delayed call of the other port. + */ + + obj = reinterpret_cast(ctx->unit->data); + + if (delay > 0) { + if (scheduled) { + return; + } + + for (data = obj->ports_; data != NULL; data = data->next) { + if (data->delayed) { + return; + } + } + + } else { + if (scheduled && !delayed) { + return; + } - scheduled = true; + for (data = obj->ports_; data != NULL; data = data->next) { + if (data != this && data->delayed) { + data->cancel(); + } + } } + + uv_timer_start(&timer, timer_callback, delay, 0); + + scheduled = true; + delayed = (delay > 0); +} + + +void +port_data_t::cancel() +{ + uv_timer_stop(&timer); + + scheduled = false; + delayed = false; } void port_data_t::stop() { + Unit *obj; + port_data_t **link; + + if (stopped) { + return; + } + stopped = true; + /* The context can be freed after this. Unlink from its ports. */ + + obj = reinterpret_cast(ctx->unit->data); + + for (link = &obj->ports_; *link != NULL; link = &(*link)->next) { + if (*link == this) { + *link = next; + break; + } + } + uv_poll_stop(&poll); uv_close((uv_handle_t *) &poll, delete_data); @@ -126,6 +213,8 @@ port_data_t::timer_callback(uv_timer_t *handle) data = get(handle); data->scheduled = false; + data->delayed = false; + if (data->stopped) { return; } @@ -150,7 +239,8 @@ port_data_t::delete_data(uv_handle_t* handle) Unit::Unit(napi_env env, napi_value jsthis): nxt_napi(env), wrapper_(wrap(jsthis, this, destroy)), - unit_ctx_(nullptr) + unit_ctx_(nullptr), + ports_(nullptr) { nxt_unit_debug(NULL, "Unit::Unit()"); } @@ -512,6 +602,9 @@ Unit::add_port(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port) data->ref_count++; data->poll.data = data; + + data->next = obj->ports_; + obj->ports_ = data; } return NXT_UNIT_OK; @@ -563,6 +656,16 @@ Unit::quit(nxt_unit_ctx_t *ctx) nxt_unit_debug(ctx, "quit: %s", e.str); } + /* + * nxt_unit_done() can free the context. No timer of its ports may + * call into libunit after that. Stop the ports that libunit did not + * remove yet. A later remove_port() call finds them stopped. + */ + + while (ports_ != NULL) { + ports_->stop(); + } + nxt_unit_done(ctx); } diff --git a/src/nodejs/unit-http/unit.h b/src/nodejs/unit-http/unit.h index 1aa93073a..1ce2c6534 100644 --- a/src/nodejs/unit-http/unit.h +++ b/src/nodejs/unit-http/unit.h @@ -9,11 +9,16 @@ #include "nxt_napi.h" +struct port_data_t; + + class Unit : public nxt_napi { public: static napi_value init(napi_env env, napi_value exports); private: + friend struct port_data_t; + Unit(napi_env env, napi_value jsthis); ~Unit(); @@ -81,6 +86,7 @@ class Unit : public nxt_napi { napi_ref wrapper_; nxt_unit_ctx_t *unit_ctx_; + port_data_t *ports_; }; diff --git a/src/nxt_app_log.c b/src/nxt_app_log.c index ae57c2a2f..7d7d4f8e8 100644 --- a/src/nxt_app_log.c +++ b/src/nxt_app_log.c @@ -41,28 +41,8 @@ nxt_log_time_handler(nxt_uint_t level, nxt_log_t *log, const char *fmt, ...) syslogmsg = p; #endif -#if 0 - nxt_fid_t fid; - const char *id; - nxt_fiber_t *fib; - - fib = nxt_fiber_self(thr); - - if (fib != NULL) { - id = "[%V] %PI#%PT#%PF "; - fid = nxt_fiber_id(fib); - - } else { - id = "[%V] %PI#%PT "; - fid = 0; - } - - p = nxt_sprintf(p, end, id, &nxt_log_levels[level], nxt_pid, - nxt_thread_tid(thr), fid); -#else p = nxt_sprintf(p, end, "[%V] %PI#%PT ", &nxt_log_levels[level], nxt_pid, nxt_thread_tid(thr)); -#endif if (log->ident != 0) { p = nxt_sprintf(p, end, "*%D ", log->ident); diff --git a/src/nxt_app_nncq.h b/src/nxt_app_nncq.h index f9b8ce0c8..38414e4df 100644 --- a/src/nxt_app_nncq.h +++ b/src/nxt_app_nncq.h @@ -11,6 +11,9 @@ #define NXT_APP_NNCQ_SIZE 131072 +/* The retry budget of the loops below, see nxt_nncq.h. */ +#define NXT_APP_NNCQ_RETRIES NXT_APP_NNCQ_SIZE + typedef uint32_t nxt_app_nncq_atomic_t; typedef uint16_t nxt_app_nncq_cycle_t; @@ -97,13 +100,13 @@ nxt_app_nncq_init(nxt_app_nncq_t volatile *q) } -static void +static nxt_int_t nxt_app_nncq_enqueue(nxt_app_nncq_t volatile *q, nxt_app_nncq_atomic_t val) { nxt_app_nncq_cycle_t e_cycle, t_cycle; - nxt_app_nncq_atomic_t n, t, e, j; + nxt_app_nncq_atomic_t n, t, e, j, r; - for ( ;; ) { + for (r = 0; r < NXT_APP_NNCQ_RETRIES; r++) { t = nxt_app_nncq_tail(q); j = nxt_app_nncq_map(q, t); e = q->entries[j]; @@ -123,11 +126,13 @@ nxt_app_nncq_enqueue(nxt_app_nncq_t volatile *q, nxt_app_nncq_atomic_t val) n = nxt_app_nncq_new_entry(q, t_cycle, val); if (nxt_atomic_cmp_set(&q->entries[j], e, n)) { - break; + nxt_app_nncq_tail_cmp_inc(q, t); + + return NXT_OK; } } - nxt_app_nncq_tail_cmp_inc(q, t); + return NXT_ERROR; } @@ -135,9 +140,9 @@ static nxt_app_nncq_atomic_t nxt_app_nncq_dequeue(nxt_app_nncq_t volatile *q) { nxt_app_nncq_cycle_t e_cycle, h_cycle; - nxt_app_nncq_atomic_t h, j, e; + nxt_app_nncq_atomic_t h, j, e, r; - for ( ;; ) { + for (r = 0; r < NXT_APP_NNCQ_RETRIES; r++) { h = nxt_app_nncq_head(q); j = nxt_app_nncq_map(q, h); e = q->entries[j]; @@ -154,11 +159,11 @@ nxt_app_nncq_dequeue(nxt_app_nncq_t volatile *q) } if (nxt_atomic_cmp_set(&q->head, h, h + 1)) { - break; + return nxt_app_nncq_index(q, e); } } - return nxt_app_nncq_index(q, e); + return nxt_app_nncq_empty(q); } diff --git a/src/nxt_app_queue.h b/src/nxt_app_queue.h index 747611d45..4364bf35a 100644 --- a/src/nxt_app_queue.h +++ b/src/nxt_app_queue.h @@ -75,7 +75,11 @@ nxt_app_queue_send(nxt_app_queue_t volatile *q, const void *p, qi->tracking = tracking; *cookie = i; - nxt_app_nncq_enqueue(&q->queue, i); + if (nxt_slow_path(nxt_app_nncq_enqueue(&q->queue, i) != NXT_OK)) { + (void) nxt_app_nncq_enqueue(&q->free_items, i); + + return NXT_ERROR; + } n = nxt_atomic_cmp_set(&q->notified, 0, 1); @@ -138,7 +142,7 @@ nxt_app_queue_recv(nxt_app_queue_t volatile *q, void *p, uint32_t *cookie) nxt_memcpy(p, qi->data, size); *cookie = i; - nxt_app_nncq_enqueue(&q->free_items, i); + (void) nxt_app_nncq_enqueue(&q->free_items, i); return size; } diff --git a/src/nxt_application.c b/src/nxt_application.c index 9f476c996..6ab9c58c6 100644 --- a/src/nxt_application.c +++ b/src/nxt_application.c @@ -60,6 +60,9 @@ static void nxt_proto_quit_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); static void nxt_proto_process_created_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); static void nxt_proto_quit_children(nxt_task_t *task); +static void nxt_proto_report_child_pid(nxt_task_t *task, + nxt_process_t *process, nxt_uint_t level); +static void nxt_proto_kill_silent(nxt_task_t *task, void *obj, void *data); static nxt_process_t *nxt_proto_process_find(nxt_task_t *task, nxt_pid_t pid); static void nxt_proto_process_add(nxt_task_t *task, nxt_process_t *process); static nxt_process_t *nxt_proto_process_remove(nxt_task_t *task, nxt_pid_t pid); @@ -67,6 +70,8 @@ static u_char *nxt_cstr_dup(nxt_mp_t *mp, u_char *dst, u_char *src); static void nxt_proto_signal_handler(nxt_task_t *task, void *obj, void *data); static void nxt_proto_sigterm_handler(nxt_task_t *task, void *obj, void *data); static void nxt_proto_sigchld_handler(nxt_task_t *task, void *obj, void *data); +static void nxt_app_new_port_handler(nxt_task_t *task, + nxt_port_recv_msg_t *msg); nxt_str_t nxt_server = nxt_string(NXT_SERVER); @@ -81,13 +86,20 @@ static nxt_lvlhsh_t nxt_proto_processes; static nxt_queue_t nxt_proto_children; static nxt_bool_t nxt_proto_exiting; +/* + * The prototype's own deadline on a quit; see nxt_proto_quit_children(). + * File scope, so its node can never outlive its storage and + * nxt_timer_disable() is never owed. + */ +static nxt_timer_t nxt_proto_kill_timer; + static nxt_app_module_t *nxt_app; static nxt_common_app_conf_t *nxt_app_conf; static const nxt_port_handlers_t nxt_discovery_process_port_handlers = { .quit = nxt_signal_quit_handler, - .new_port = nxt_port_new_port_handler, + .new_port = nxt_app_new_port_handler, .change_file = nxt_port_change_log_file_handler, .mmap = nxt_port_mmap_handler, .data = nxt_port_data_handler, @@ -110,7 +122,7 @@ const nxt_sig_event_t nxt_prototype_signals[] = { static const nxt_port_handlers_t nxt_proto_process_port_handlers = { .quit = nxt_proto_quit_handler, .change_file = nxt_port_change_log_file_handler, - .new_port = nxt_port_new_port_handler, + .new_port = nxt_app_new_port_handler, .process_created = nxt_proto_process_created_handler, .process_ready = nxt_port_process_ready_handler, .remove_pid = nxt_port_remove_pid_handler, @@ -161,6 +173,23 @@ const nxt_process_init_t nxt_app_process = { }; +/* + * nxt_port_new_port_handler() hands the queue descriptor of a newly created + * port to its caller, because only the caller knows whether this process + * maps port queues. Neither the discovery nor the prototype process does, + * so both used the base handler directly and kept every queue descriptor + * their peer sent open for the life of the process. + */ + +static void +nxt_app_new_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) +{ + nxt_port_new_port_handler(task, msg); + + nxt_port_recv_msg_close_fds(msg); +} + + static nxt_int_t nxt_discovery_start(nxt_task_t *task, nxt_process_data_t *data) { @@ -206,17 +235,28 @@ nxt_discovery_start(nxt_task_t *task, nxt_process_data_t *data) static nxt_buf_t * nxt_discovery_modules(nxt_task_t *task, const char *path) { - char *name; - u_char *p, *end; - size_t size; - glob_t glb; - nxt_mp_t *mp; - nxt_buf_t *b; - nxt_int_t ret; - nxt_uint_t i, n, j; - nxt_array_t *modules, *mounts; - nxt_module_t *module; - nxt_fs_mount_t *mnt; + char *name; + size_t size; + glob_t glb; + nxt_mp_t *mp; + nxt_str_t str; + nxt_buf_t *b; + nxt_int_t ret; + nxt_uint_t i, n, j; + nxt_array_t *modules, *mounts; + nxt_module_t *module; + nxt_fs_mount_t *mnt; + nxt_conf_value_t *root, *obj, *array, *mount; + + static const nxt_str_t type_str = nxt_string("type"); + static const nxt_str_t name_str = nxt_string("name"); + static const nxt_str_t version_str = nxt_string("version"); + static const nxt_str_t file_str = nxt_string("file"); + static const nxt_str_t mounts_str = nxt_string("mounts"); + static const nxt_str_t src_str = nxt_string("src"); + static const nxt_str_t dst_str = nxt_string("dst"); + static const nxt_str_t flags_str = nxt_string("flags"); + static const nxt_str_t data_str = nxt_string("data"); b = NULL; @@ -249,86 +289,100 @@ nxt_discovery_modules(nxt_task_t *task, const char *path) } } - size = nxt_length("[]"); module = modules->elts; n = modules->nelts; + /* + * The main process parses this message as JSON, so build it through + * nxt_conf, which escapes every string. Formatted with "%s", a double + * quote or a backslash in a module path or a mount source broke the + * parse, and the main process then started with no language modules + * at all and nothing above debug level in the log to say why. + */ + root = nxt_conf_create_array(mp, n); + if (nxt_slow_path(root == NULL)) { + goto fail; + } + for (i = 0; i < n; i++) { nxt_debug(task, "module: %d %V %V", module[i].type, &module[i].version, &module[i].file); - size += nxt_length("{\"type\": ,"); - size += nxt_length(" \"name\": \"\","); - size += nxt_length(" \"version\": \"\","); - size += nxt_length(" \"file\": \"\","); - size += nxt_length(" \"mounts\": []},"); + obj = nxt_conf_create_object(mp, 5); + if (nxt_slow_path(obj == NULL)) { + goto fail; + } - size += NXT_INT_T_LEN - + module[i].version.length - + module[i].name.length - + module[i].file.length; + nxt_conf_set_member_integer(obj, &type_str, module[i].type, 0); + nxt_conf_set_member_string(obj, &name_str, &module[i].name, 1); + nxt_conf_set_member_string(obj, &version_str, &module[i].version, 2); + nxt_conf_set_member_string(obj, &file_str, &module[i].file, 3); mounts = module[i].mounts; - - size += mounts->nelts * nxt_length("{\"src\": \"\", \"dst\": \"\", " - "\"type\": , \"name\": \"\", " - "\"flags\": , \"data\": \"\"},"); - mnt = mounts->elts; - for (j = 0; j < mounts->nelts; j++) { - size += nxt_strlen(mnt[j].src) + nxt_strlen(mnt[j].dst) - + nxt_strlen(mnt[j].name) + (2 * NXT_INT_T_LEN) - + (mnt[j].data == NULL ? 0 : nxt_strlen(mnt[j].data)); + array = nxt_conf_create_array(mp, mounts->nelts); + if (nxt_slow_path(array == NULL)) { + goto fail; } - } - b = nxt_buf_mem_alloc(mp, size, 0); - if (b == NULL) { - goto fail; - } + for (j = 0; j < mounts->nelts; j++) { + mount = nxt_conf_create_object(mp, 6); + if (nxt_slow_path(mount == NULL)) { + goto fail; + } - b->completion_handler = nxt_discovery_completion_handler; + str.start = mnt[j].src; + str.length = nxt_strlen(mnt[j].src); + nxt_conf_set_member_string(mount, &src_str, &str, 0); - p = b->mem.free; - end = b->mem.end; - *p++ = '['; + str.start = mnt[j].dst; + str.length = nxt_strlen(mnt[j].dst); + nxt_conf_set_member_string(mount, &dst_str, &str, 1); - for (i = 0; i < n; i++) { - mounts = module[i].mounts; + str.start = mnt[j].name; + str.length = nxt_strlen(mnt[j].name); + nxt_conf_set_member_string(mount, &name_str, &str, 2); - p = nxt_sprintf(p, end, "{\"type\": %d, \"name\": \"%V\", " - "\"version\": \"%V\", \"file\": \"%V\", \"mounts\": [", - module[i].type, &module[i].name, &module[i].version, - &module[i].file); + nxt_conf_set_member_integer(mount, &type_str, mnt[j].type, 3); + nxt_conf_set_member_integer(mount, &flags_str, mnt[j].flags, 4); - mnt = mounts->elts; - for (j = 0; j < mounts->nelts; j++) { - p = nxt_sprintf(p, end, - "{\"src\": \"%s\", \"dst\": \"%s\", " - "\"name\": \"%s\", \"type\": %d, \"flags\": %d, " - "\"data\": \"%s\"},", - mnt[j].src, mnt[j].dst, mnt[j].name, mnt[j].type, - mnt[j].flags, - mnt[j].data == NULL ? (u_char *) "" : mnt[j].data); + str.start = (mnt[j].data == NULL) ? (u_char *) "" : mnt[j].data; + str.length = nxt_strlen(str.start); + nxt_conf_set_member_string(mount, &data_str, &str, 5); + + nxt_conf_set_element(array, j, mount); } - *p++ = ']'; - *p++ = '}'; - *p++ = ','; + nxt_conf_set_member(obj, &mounts_str, array, 4); + nxt_conf_set_element(root, i, obj); } - *p++ = ']'; + size = nxt_conf_json_length(root, NULL); - if (nxt_slow_path(p > end)) { - nxt_alert(task, "discovery write past the buffer"); + b = nxt_buf_mem_alloc(mp, size, 0); + if (b == NULL) { goto fail; } - b->mem.free = p; + b->completion_handler = nxt_discovery_completion_handler; + + b->mem.free = nxt_conf_json_print(b->mem.free, root, NULL); fail: + /* + * This is the success path too -- b is NULL only if one of the allocations + * above failed. Say so: the caller turns a NULL into NXT_ERROR, and the + * main process then starts the controller and the router with no language + * modules registered, so without a line here the only symptom an operator + * sees is every application type reported as "not found". + */ + if (nxt_slow_path(b == NULL)) { + nxt_alert(task, "discovery failed to build the module list"); + nxt_mp_destroy(mp); + } + globfree(&glb); return b; @@ -600,6 +654,90 @@ nxt_proto_start(nxt_task_t *task, nxt_process_data_t *data) } +#if (NXT_USE_CMSG_PID) + +/* + * Is the kernel-validated sender of this START_PROCESS allowed to make the + * prototype fork a worker? + * + * Only the router ever sends one (nxt_router_start_app_process_handler(), + * src/nxt_router.c:469, and nxt_router_app_prefork(), :3379), but every + * worker the prototype forks inherits the write end of the prototype's own + * port socket -- nxt_proc_keep_matrix[] does not list it, and + * nxt_process_close_ports() keeps the parent's port unconditionally -- and + * nxt_port_handler() dispatches on the wire type alone. So without this + * test one compromised worker can spend the prototype's process budget, and, + * once #268 has the prototype answer for a child that dies, drive the + * router's start bookkeeping from the inside. + * + * The test has two arms because the prototype has two pid views of the + * router, and the credential the kernel writes is always relative to the + * receiver's pid namespace: + * + * - Sharing main's namespace, the router is an ordinary visible process and + * the credential is its global pid, which is what the inherited + * rt->port_by_type[NXT_PROCESS_ROUTER] is keyed on. This is the same + * whitelist nxt_main_start_process_handler() applies + * (src/nxt_main_process.c:532). + * + * - Under "isolation": {"namespaces": {"pid": true}} the prototype is the + * init of its own namespace (nxt_process.c:765) and the router lives in + * an ancestor of it, so the router has no pid there at all: the kernel + * translates an untranslatable sender to 0 (pid_vnr() returns 0 outside + * the receiver's namespace; measured, not assumed). Comparing against + * the router's global pid would then refuse every legitimate start. A + * credential of 0 is therefore what "from outside this namespace" looks + * like, and it is exactly the set the prototype does not fork: its + * workers are forked with no clone flags of their own + * (nxt_proto_start_process_handler() leaves process->isolation zeroed), + * so each of them is inside this namespace and presents a non-zero + * namespace-local pid. What this arm authenticates is therefore a + * namespace boundary, not an identity: it admits every holder of the + * write end that sits outside this namespace. By construction that is + * main and the router (nxt_proc_keep_matrix[] keeps only those two in a + * worker, and nxt_proc_send_matrix[] lets the prototype answer only + * those two); main is trusted and does not send START_PROCESS. A worker + * that passes its inherited port out of the namespace over SCM_RIGHTS + * turns an accomplice into an admitted sender, which needs a second + * foothold and is the limit of what a credential can decide here. + * + * The reply addressing at "failed:" is deliberately left keyed on + * msg->port_msg.pid: the runtime port hash is keyed on the global pid, which + * in the isolated arm is precisely the number the credential cannot supply. + * That is why a refused message is not answered at all -- answering it would + * let the forger name any port and stream it likes and have the prototype + * cancel somebody else's RPC. A forgery strands nothing: the router never + * armed an RPC for a message it did not send. A genuine start refused by + * the router-port-is-NULL branch would be a different matter -- the router's + * RPC is keyed on the prototype (nxt_port_rpc_ex_set_peer(), src/nxt_router.c), + * so it would stay armed until the prototype dies -- but that branch is + * reachable only once the router is already gone, and a restarted router + * cannot address a pre-existing prototype at all: nxt_port_send_new_port() + * announces a newcomer outward and never announces existing peers to it. + */ +static nxt_bool_t +nxt_proto_start_process_sender_ok(nxt_task_t *task, nxt_runtime_t *rt, + nxt_port_recv_msg_t *msg) +{ + nxt_port_t *router_port; + + if (rt->is_pid_isolated) { + return nxt_recv_msg_cmsg_pid(msg) == 0; + } + + router_port = rt->port_by_type[NXT_PROCESS_ROUTER]; + + if (nxt_slow_path(router_port == NULL)) { + nxt_alert(task, "router port not found"); + return 0; + } + + return nxt_recv_msg_cmsg_pid(msg) == router_port->pid; +} + +#endif + + static void nxt_proto_start_process_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) { @@ -612,6 +750,45 @@ nxt_proto_start_process_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) rt = task->thread->runtime; +#if (NXT_USE_CMSG_PID) + /* + * Before anything is allocated or forked, and without a reply: see + * nxt_proto_start_process_sender_ok(). The descriptors are closed here + * because this handler owns whatever the message carried -- the port + * read loop does not reclaim them (src/nxt_port_socket.c:1381). The + * router's own START_PROCESS to a prototype carries none (both fds are + * -1 at src/nxt_router.c:469 and :3379; the shared port and queue only + * travel in the one it sends main), so this is for what a forger + * attaches, not for anything a legitimate message brings. + */ + if (nxt_slow_path(!nxt_proto_start_process_sender_ok(task, rt, msg))) { + nxt_alert(task, "process %PI cannot start processes", + nxt_recv_msg_cmsg_pid(msg)); + + nxt_port_recv_msg_close_fds(msg); + + return; + } +#endif + + /* + * A prototype that got QUIT only waits for its workers to exit. The + * router can still send START_PROCESS at that point. On shutdown main + * sends QUIT to the prototype directly. The prototype reports the exit + * of a worker to the router. The router starts a replacement before + * its own QUIT arrives. A worker forked now gets no QUIT from anyone. + * The prototype waits for that worker, so Unit does not exit. Refuse + * the start. The reply fails the RPC of the router and gives the slot + * back. + */ + if (nxt_slow_path(nxt_proto_exiting)) { + nxt_debug(task, "prototype is exiting, start process refused"); + + nxt_port_recv_msg_close_fds(msg); + + goto failed; + } + process = nxt_process_new(rt); if (nxt_slow_path(process == NULL)) { goto failed; @@ -650,7 +827,16 @@ nxt_proto_start_process_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) process->user_cred = &rt->user_cred; process->data.app = nxt_app_conf; + + /* + * Remember who to answer, not just which stream: the RPC is registered on + * the initiator's port, and by the time this child is reaped the message + * that named it is long gone. nxt_proto_child_exited() is then the only + * thing left that can report a child which died before it was announced. + */ process->stream = msg->port_msg.stream; + process->stream_pid = msg->port_msg.pid; + process->stream_port = msg->port_msg.reply_port; init->siblings = &nxt_proto_children; @@ -677,6 +863,25 @@ nxt_proto_start_process_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) } +#if (NXT_TESTS) + +void +nxt_proto_test_run_start_process_handler(nxt_task_t *task, + nxt_port_recv_msg_t *msg) +{ + nxt_proto_start_process_handler(task, msg); +} + + +void +nxt_proto_test_set_exiting(nxt_bool_t exiting) +{ + nxt_proto_exiting = exiting; +} + +#endif + + static void nxt_proto_quit_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) { @@ -716,21 +921,147 @@ nxt_proto_quit_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) } +/* + * Tell every worker to go, and wait for the SIGCHLD of each: only when + * nxt_proto_children is empty does nxt_proto_sigchld_handler() run + * nxt_process_quit() for the prototype itself. + * + * A port message is how a worker is told, and a worker that has never sent + * PROCESS_READY is not reading its port yet. That alone does not lose the + * message: the QUIT sits in the socket, and a module worker still inside the + * module's own start function reads it the moment nxt_unit_init() starts + * reading -- which is why an application that merely takes its time to start + * shuts down cleanly, and has to keep doing so. + * + * What is lost is the QUIT to a worker that will never read that port at + * all: a "type": "external" one exec'd the user's binary out of + * nxt_app_setup() before any of ours ran, and a module worker whose start + * function never returns is no better. Such a worker outlives the request + * that asked for it and the prototype waits on it for good, so the pair + * survives every drain there is, including unitd's own exit. Measured with + * an external application running /bin/sleep and a "limits": + * {"start_timeout"} that gives up on it: five rejected configuration PUTs + * left five prototypes and five workers alive, and replacing the + * configuration did not collect them either. + * + * The two are the same worker until one of them announces itself, and + * nothing here can tell them apart -- that is precisely the question + * "start_timeout" exists to answer. So the QUIT goes to every worker as it + * always did, and where the application declared a bound, the prototype arms + * one of its own: SIGKILL for whatever is still silent when it expires, in + * nxt_proto_kill_silent(). + * + * That deadline is strictly later than the router's, and by a whole + * "start_timeout": it is armed here, and this runs on a QUIT the router only + * sends once it has itself given up on the application. A worker that would + * have announced itself within the bound the user asked for therefore gets + * that whole bound over again before anything is signalled. + * + * With no "start_timeout" nothing is armed and this is exactly what it was, + * unbounded -- as the wait for an application start is unbounded by default. + */ + static void nxt_proto_quit_children(nxt_task_t *task) { - nxt_port_t *port; - nxt_process_t *process; - nxt_runtime_t *rt; + nxt_bool_t silent; + nxt_port_t *port; + nxt_process_t *process; + nxt_runtime_t *rt; + nxt_event_engine_t *engine; rt = task->thread->runtime; + silent = 0; + nxt_queue_each(process, &nxt_proto_children, nxt_process_t, link) { + + if (nxt_slow_path(process->state != NXT_PROCESS_STATE_READY)) { + silent = 1; + } + port = nxt_process_port_first(process); + /* + * The router stops its own workers before it quits the prototype. + * A worker can exit before the prototype handles its SIGCHLD. + * The QUIT to that worker then fails with EPIPE. + * nxt_socketpair_send_ex() logs this failure at info for a QUIT. + */ nxt_runtime_port_send_quit(task, rt, port); } nxt_queue_loop; + + if (!silent || nxt_app_conf == NULL || nxt_app_conf->start_timeout == 0) { + return; + } + + engine = task->thread->engine; + + nxt_proto_kill_timer.bias = NXT_TIMER_DEFAULT_BIAS; + nxt_proto_kill_timer.work_queue = &engine->fast_work_queue; + nxt_proto_kill_timer.handler = nxt_proto_kill_silent; + nxt_proto_kill_timer.task = &engine->task; + nxt_proto_kill_timer.log = nxt_proto_kill_timer.task->log; + + nxt_timer_add(engine, &nxt_proto_kill_timer, nxt_app_conf->start_timeout); + + nxt_debug(task, "app \"%V\" quit deadline %M ms for a worker that has not " + "announced itself", + &nxt_app_conf->name, nxt_app_conf->start_timeout); +} + + +/* + * The prototype's deadline expired: a worker it told to quit has still not + * announced itself, so it never read that QUIT and never will. + * + * Signal it, which is possible precisely because it never announced itself: + * the prototype forked it and knows its pid, where the router only ever + * learns one from PROCESS_READY. SIGKILL, not SIGTERM, because what such a + * worker does with a catchable signal is the user's binary's business and one + * that ignores SIGTERM is the case this exists for; nothing is lost, because + * a worker that has not announced itself holds no port anyone can reach, no + * mapped queue (nxt_port_process_ready_handler() maps it at PROCESS_READY) + * and no request. + * + * Read process->isolated_pid, not ->pid: under "isolation": {"namespaces": + * {"pid": true}} nxt_proto_process_created_handler() rewrites ->pid to the + * global pid and only ->isolated_pid stays valid in this namespace. It is + * the number waitpid() reports in nxt_proto_sigchld_handler(), and the one + * nxt_port_process_ready_handler() signals for the same reason. + */ + +static void +nxt_proto_kill_silent(nxt_task_t *task, void *obj, void *data) +{ + nxt_process_t *process; + + nxt_queue_each(process, &nxt_proto_children, nxt_process_t, link) { + + if (process->state == NXT_PROCESS_STATE_READY) { + continue; + } + + nxt_log(task, NXT_LOG_INFO, + "app process %PI neither announced itself nor acted on the " + "quit it was sent within \"start_timeout\"; killing it", + process->isolated_pid); + + if (nxt_fast_path(process->isolated_pid > 0) + && kill(process->isolated_pid, SIGKILL) == -1) + { + /* + * ESRCH is ordinary: the worker may already have exited with its + * SIGCHLD still pending, and that SIGCHLD drains it. + */ + nxt_log(task, (nxt_errno == ESRCH) ? NXT_LOG_INFO : NXT_LOG_ALERT, + "kill(%PI, SIGKILL) failed for an app process that never " + "announced itself %E", + process->isolated_pid, nxt_errno); + } + } + nxt_queue_loop; } @@ -885,9 +1216,7 @@ nxt_proto_sigchld_handler(nxt_task_t *task, void *obj, void *data) port = nxt_process_port_first(process); } - if (process->state != NXT_PROCESS_STATE_CREATING) { - nxt_port_remove_notify_others(task, process); - } + nxt_proto_child_exited(task, process); nxt_process_close_ports(task, process); @@ -903,6 +1232,283 @@ nxt_proto_sigchld_handler(nxt_task_t *task, void *obj, void *data) } +/* + * Report a reaped child of the prototype. Every child is forked to satisfy a + * START_PROCESS request, and that request has an RPC handler armed on the + * initiator's port; something has to retire it when the child dies or the + * initiator waits forever. This closes the case where the child dies. + * + * The case where the *prototype* dies part-way through an on-demand start is + * closed elsewhere and no longer belongs on this list: + * nxt_router_start_app_process_handler() keys that RPC on the prototype + * (nxt_port_rpc_ex_set_peer(), src/nxt_router.c), so the REMOVE_PID for the + * prototype reaches nxt_port_rpc_remove_peer() and fails the start, releasing + * the app->pending_processes slot with it. + * + * What that leaves is this function's case alone: a child that dies before + * PROCESS_CREATED, which no REMOVE_PID describes, because until the handshake + * completes the prototype does not necessarily know a globally valid pid for + * it. + * + * A child that got as far as PROCESS_CREATED is reported by REMOVE_PID, which + * carries ->stream: nxt_router_remove_pid_handler() turns a stream-bearing + * REMOVE_PID into an RPC error. A child still in the CREATING state cannot be + * announced that way. The pid is the whole content of REMOVE_PID, and until + * the PROCESS_CREATED exchange completes the prototype does not necessarily + * know a globally valid one: under pid isolation ->pid is still the + * namespace-local pid nxt_process_create() got from fork(), and the global pid + * only arrives with PROCESS_CREATED (see nxt_proto_process_created_handler() + * and 900828cc, which is why such a process is deliberately kept out of the + * global pid hash). Broadcasting that pid would ask every receiver to remove + * whatever unrelated process happens to hold it. + * + * So the gate stays exactly as it was, and the CREATING case is answered + * directly instead: an RPC error to the port the start request came from, + * addressed by ->stream_pid/->stream_port rather than by any pid of the dead + * child. It is the same message nxt_proto_start_process_handler() sends when + * the fork itself fails. + * + * Answering the initiator is not the whole job, though, because the initiator + * is not the only process left holding the child. A worker that got as far as + * WHOAMI made main create a process record and a port for it, with main's end + * of the worker's port socket in it (nxt_main_process_whoami_handler()), and + * linked that record into the prototype's ->children. REMOVE_PID is what + * retires it -- nxt_proc_remove_notify_matrix pairs a dying APP with MAIN -- + * so a CREATING child that is only answered on the RPC leaves main holding a + * record and an fd until the prototype itself exits. Before this whole fix + * that leak was capped by the wedge: the application stopped starting + * processes, so at most one could leak. Now that the start RPC is retired and + * requests retry, a worker that keeps dying in that window costs main one + * record and one descriptor per attempt, without bound. + * + * The pid problem is the same one the gate exists for, so the answer is the + * same test the rest of the code already uses for "is this pid a usable global + * key": rt->is_pid_isolated, which nxt_process_create() consults to decide + * whether a forked child may go into the runtime hash at all. When it is + * clear, the prototype shares main's pid namespace, ->pid is the fork() return + * in that namespace, and it is by construction the same number main read from + * SCM_CREDENTIALS on the WHOAMI message -- so REMOVE_PID is safe. The + * notification is deliberately made after ->stream has been cleared, so it + * carries no stream: the initiator has already been answered directly, and a + * stream-bearing REMOVE_PID would make nxt_router_remove_pid_handler() fail + * the same RPC a second time. + * + * Ordering is not a race even though the two messages come from two processes: + * the worker's WHOAMI and the prototype's REMOVE_PID are both written to the + * single write end of main's port socketpair, inherited by every descendant, + * so they share one kernel queue. The worker's WHOAMI write completes before + * it exits, and the prototype writes only after waitpid() has reaped it. If + * the WHOAMI never reached the socket it died with the worker, and main has no + * record to retire. + * + * When rt->is_pid_isolated is set there is no safe pid to send: ->pid is the + * namespace-local one, and the global pid main and the router keyed their + * records on cannot be derived from it here. Sending it anyway would ask + * every receiver to remove whatever unrelated process holds that number -- and + * a prototype's namespace-local counter climbs with each worker it forks, so + * it walks into the range the daemon's own pids occupy. Removing a live + * sibling's ports drops requests, which is worse than the leak, so no + * REMOVE_PID is sent there. + * + * So the report goes to main alone, as REMOVE_CHILD_PID carrying the + * namespace-local pid. Main holds both names -- the global pid from + * SCM_CREDENTIALS and the local one from the WHOAMI header -- and resolves + * the report among the sender's own children only. It then tells the router + * by the global pid, so the router sees what a REMOVE_PID would have given it. + */ + +void +nxt_proto_child_exited(nxt_task_t *task, nxt_process_t *process) +{ + nxt_int_t ret; + nxt_uint_t level; + nxt_port_t *port; + nxt_runtime_t *rt; + + if (process->state != NXT_PROCESS_STATE_CREATING) { + nxt_port_remove_notify_others(task, process); + + return; + } + + rt = task->thread->runtime; + + /* + * A CREATING child whose initiator is already gone is the expected shape + * of a teardown rather than an anomaly of one, and the record leak noted + * below is moot once the prototype itself is on its way out. + */ + level = nxt_proto_exiting ? NXT_LOG_WARN : NXT_LOG_ALERT; + + if (process->stream != 0) { + port = nxt_runtime_port_find(rt, process->stream_pid, + process->stream_port); + + if (nxt_slow_path(port == NULL)) { + if (rt->is_pid_isolated) { + nxt_log(task, level, "app process (isolated %PI) died before " + "it was created and its start initiator %PI port %d " + "is gone (stream %uD)", process->isolated_pid, + process->stream_pid, (int) process->stream_port, + process->stream); + + } else { + nxt_log(task, level, "app process %PI died before it was " + "created and its start initiator %PI port %d is gone " + "(stream %uD)", process->pid, process->stream_pid, + (int) process->stream_port, process->stream); + } + + /* Nothing can carry the answer; do not leave it armed. */ + process->stream = 0; + + } else { + ret = nxt_port_socket_write(task, port, NXT_PORT_MSG_RPC_ERROR, + -1, process->stream, 0, NULL); + + /* + * One answer per start request: nxt_port_rpc_handler() drops a + * stream it no longer knows, but the identifiers come from a + * shared counter and are reused, so a second error for a retired + * stream could land on somebody else's RPC. So ->stream is + * cleared only once the answer has actually been written or + * queued, which is what makes the REMOVE_PID below streamless. + * + * Three failures reach this call site, and none of them leaves + * a message behind. The prototype maps no queue for a router + * port -- nxt_app_new_port_handler() closes the queue descriptor + * every NEW_PORT carries, and main maps a queue only for an + * application port -- so nxt_port_socket_write2() never takes + * the shared-ring path that answers NXT_AGAIN. What is left is + * nxt_port_msg_chk_insert() failing to allocate; an inline + * write that hit EAGAIN with no memory left to hold it for a + * later attempt; and an inline write to a router port whose + * peer has died. + * + * Keeping ->stream then lets the REMOVE_PID carry it, and + * nxt_router_remove_pid_handler() turns that into the same RPC + * error -- the fallback the CREATED path has always used. + * Clearing it regardless would leave the start RPC armed and + * rebuild the wedge this function exists to close, silently. + * + * That fallback is a second chance, not a guarantee: the + * REMOVE_PID allocates a buffer and a message of its own from + * the same pools and can fail the same way, and then only the + * application's "limits.start_timeout" retires the start. The + * alert below is what makes that case visible instead of + * silent. After an EAGAIN, REMOVE_PID cannot rescue it either: + * the same EAGAIN cleared write_ready, so REMOVE_PID is queued + * rather than sent, and the error handler that runs next drains + * it too. + */ + if (nxt_fast_path(ret == NXT_OK)) { + process->stream = 0; + + } else if (rt->is_pid_isolated) { + nxt_log(task, level, "app process (isolated %PI) died before " + "it was created and could not be reported to its " + "start initiator %PI port %d (stream %uD)", + process->isolated_pid, process->stream_pid, + (int) process->stream_port, process->stream); + + } else { + nxt_log(task, level, "app process %PI died before it was " + "created and could not be reported to its start " + "initiator %PI port %d (stream %uD); the REMOVE_PID " + "that follows carries the report unless it cannot be " + "allocated either", process->pid, process->stream_pid, + (int) process->stream_port, process->stream); + } + } + } + + /* + * Under pid isolation the direct answer is the only vehicle: ->pid is the + * namespace-local one, so no REMOVE_PID may be sent for this child at + * all. When that answer could not be allocated above, the start RPC is + * therefore left armed. What retires it then is the prototype's own + * death -- the router keys that RPC on the process it sent START_PROCESS + * to (nxt_port_rpc_ex_set_peer(), src/nxt_router.c), so main's REMOVE_PID + * for the prototype reaches nxt_port_rpc_remove_peer() -- or, sooner and + * per application, "limits": {"start_timeout"}, which defaults to none. + * Closing it here needs an answer that cannot fail to allocate, which + * this layer has no way to reserve; tracked with the record leak in #310. + */ + + if (nxt_slow_path(rt->is_pid_isolated)) { + nxt_debug(task, "app process (isolated %PI) died before it was " + "created", process->isolated_pid); + + nxt_proto_report_child_pid(task, process, level); + + /* No REMOVE_PID will carry it; do not leave it armed. */ + process->stream = 0; + + return; + } + + nxt_port_remove_notify_others(task, process); +} + + +/* + * Tell main that a child of this prototype died, by its namespace-local pid. + * The message carries no stream: only the router turns one into an RPC error, + * and this goes to main alone. + * + * Ordering: the worker's WHOAMI and this message go to the same write end of + * main's port socketpair, and this runs only after waitpid() reaped the + * worker, so main reads the WHOAMI first. This holds because main's port has + * no shared-memory queue. + */ + +static void +nxt_proto_report_child_pid(nxt_task_t *task, nxt_process_t *process, + nxt_uint_t level) +{ + nxt_buf_t *buf; + nxt_port_t *main_port; + nxt_runtime_t *rt; + + rt = task->thread->runtime; + + main_port = rt->port_by_type[NXT_PROCESS_MAIN]; + if (nxt_slow_path(main_port == NULL)) { + goto fail; + } + + buf = nxt_buf_mem_ts_alloc(task, task->thread->engine->mem_pool, + sizeof(nxt_pid_t)); + if (nxt_slow_path(buf == NULL)) { + goto fail; + } + + buf->mem.free = nxt_cpymem(buf->mem.free, &process->isolated_pid, + sizeof(nxt_pid_t)); + + if (nxt_fast_path(nxt_port_socket_write(task, main_port, + NXT_PORT_MSG_REMOVE_CHILD_PID, + -1, 0, 0, buf) + == NXT_OK)) + { + return; + } + + /* Still ours: the port layer takes the buffer only on NXT_OK. */ + + nxt_work_queue_add(&task->thread->engine->fast_work_queue, + buf->completion_handler, task, buf, buf->parent); + +fail: + + /* Main then keeps the record until this prototype exits (#310). */ + + nxt_log(task, level, "app process (isolated %PI) died before it was " + "created and could not be reported to the main process", + process->isolated_pid); +} + + static nxt_app_module_t * nxt_app_module_load(nxt_task_t *task, const char *name) { @@ -1294,8 +1900,7 @@ nxt_proto_process_remove(nxt_task_t *task, nxt_pid_t pid) process = lhq.value; - nxt_queue_remove(&process->link); - process->link.next = NULL; + nxt_process_unlink(process); break; diff --git a/src/nxt_application.h b/src/nxt_application.h index a3b4230a0..36ad6b8d3 100644 --- a/src/nxt_application.h +++ b/src/nxt_application.h @@ -110,6 +110,22 @@ typedef struct { const char *component; nxt_conf_value_t *access; + + /* + * "execution_timeout" in milliseconds, 0 for unbounded. It bounds one + * invocation of the component's "handle", and only the wasm module acts + * on it: the guest is preempted inside the module's own process, so + * unlike "limits": {"start_timeout"} the router has nothing to arm. + * + * It is wall clock from the guest's first instruction until "handle" + * returns, not CPU time, so time the guest spends parked in a host + * call -- waiting for a slow client to drain the response body, say -- + * is spent too. And the deadline is only checked while wasm executes, + * so a guest parked in a host call is not preempted, only trapped when + * it resumes. This is not a request timeout; that is + * "limits": {"timeout"}, which the router arms for every app type. + */ + nxt_msec_t execution_timeout; } nxt_wasm_wc_app_conf_t; @@ -131,6 +147,15 @@ struct nxt_common_app_conf_s { size_t shm_limit; uint32_t request_limit; + /* + * "limits": {"start_timeout"} in milliseconds, 0 for unbounded. The + * router arms it on the start itself (nxt_router_start_timer_arm()); + * the prototype is given the same number because it is the one process + * that can act on a worker the router gave up on -- it knows its pid. + * See nxt_proto_quit_children(). + */ + nxt_msec_t start_timeout; + nxt_fd_t shared_port_fd; nxt_fd_t shared_queue_fd; @@ -167,6 +192,16 @@ struct nxt_app_module_s { nxt_app_lang_module_t *nxt_app_lang_module(nxt_runtime_t *rt, nxt_str_t *name); nxt_app_type_t nxt_app_parse_type(u_char *p, size_t length); +#if (NXT_TESTS) +/* + * Invokes the static nxt_proto_start_process_handler() with a synthesised + * runtime and message; see src/test/nxt_main_start_process_reply_test.c. + */ +void nxt_proto_test_run_start_process_handler(nxt_task_t *task, + nxt_port_recv_msg_t *msg); +void nxt_proto_test_set_exiting(nxt_bool_t exiting); +#endif + NXT_EXPORT extern nxt_str_t nxt_server; extern nxt_app_module_t nxt_external_module; @@ -175,4 +210,7 @@ NXT_EXPORT nxt_int_t nxt_unit_default_init(nxt_task_t *task, NXT_EXPORT nxt_int_t nxt_app_set_logs(void); +/* Reports a reaped child of the prototype; see nxt_proto_sigchld_handler(). */ +void nxt_proto_child_exited(nxt_task_t *task, nxt_process_t *process); + #endif /* _NXT_APPLICATION_H_INCLIDED_ */ diff --git a/src/nxt_brotli.c b/src/nxt_brotli.c index c69d977e0..30c0b840a 100644 --- a/src/nxt_brotli.c +++ b/src/nxt_brotli.c @@ -46,14 +46,14 @@ nxt_brotli_compress(nxt_http_comp_compressor_ctx_t *ctx, const uint8_t *in_buf, &in_len, &in_buf, &out_bytes, &out_buf, NULL); if (!ok) { - goto out_err_free; + return -1; } ok = BrotliEncoderCompressStream(brotli, BROTLI_OPERATION_FLUSH, &in_len, &in_buf, &out_bytes, &out_buf, NULL); if (!ok) { - goto out_err_free; + return -1; } if (last) { @@ -61,18 +61,19 @@ nxt_brotli_compress(nxt_http_comp_compressor_ctx_t *ctx, const uint8_t *in_buf, &in_len, &in_buf, &out_bytes, &out_buf, NULL); if (!ok) { - goto out_err_free; + return -1; } - - BrotliEncoderDestroyInstance(brotli); } return out_len - out_bytes; +} -out_err_free: - BrotliEncoderDestroyInstance(brotli); - return -1; +static void +nxt_brotli_free(nxt_http_comp_compressor_ctx_t *ctx) +{ + BrotliEncoderDestroyInstance(ctx->brotli_ctx); + ctx->brotli_ctx = NULL; } @@ -80,4 +81,5 @@ const nxt_http_comp_operations_t nxt_http_comp_brotli_ops = { .init = nxt_brotli_init, .bound = nxt_brotli_bound, .deflate = nxt_brotli_compress, + .free = nxt_brotli_free, }; diff --git a/src/nxt_buf.h b/src/nxt_buf.h index 2f2cbaa68..9ba911599 100644 --- a/src/nxt_buf.h +++ b/src/nxt_buf.h @@ -247,8 +247,8 @@ NXT_EXPORT size_t nxt_buf_chain_length(nxt_buf_t *b); nxt_inline nxt_buf_t * nxt_buf_cpy(nxt_buf_t *b, const void *src, size_t length) { - nxt_memcpy(b->mem.free, src, length); - b->mem.free += length; + /* nxt_cpymem() skips a zero-length copy, whose "src" may be NULL. */ + b->mem.free = nxt_cpymem(b->mem.free, src, length); return b; } diff --git a/src/nxt_capability.c b/src/nxt_capability.c index 157c1eadb..e341b3551 100644 --- a/src/nxt_capability.c +++ b/src/nxt_capability.c @@ -4,15 +4,21 @@ */ /* - * NOTE: this module currently exposes capability *detection* - * (capget), used by nxt_isolation.c and the credential machinery to - * decide whether a non-root build can honor setuid/setgid in the - * "user"/"group" config keys. Capabilities are NOT dropped - * programmatically via capset(2): the privilege barrier for app - * processes is the existing setuid + PR_SET_NO_NEW_PRIVS dance in - * nxt_credential.c / nxt_process.c. Operators expecting an explicit - * capability-drop step should not assume one exists; isolation in - * Unit relies on uid/namespace/seccomp boundaries, not capset. + * NOTE: this module does two separate things, in two different + * processes. + * + * nxt_capability_set() is *detection* (capget), run once in the main + * process from nxt_runtime_conf_init(). nxt_isolation.c and the + * credential machinery use its answer to decide whether a non-root + * unitd can honour setuid/setgid in the "user"/"group" config keys + * and "rootfs" isolation. + * + * nxt_capability_drop() is *disposal* (capset), run in every forked + * process from the tail of nxt_process_apply_creds(). It is never + * run in main, which keeps CAP_NET_BIND_SERVICE for the bind() in + * nxt_main_listening_socket() -- listeners are created on every + * reconfiguration, long after conf_init, so main cannot drop + * anything. See nxt_capability_drop() for the rest. */ #include @@ -20,7 +26,9 @@ #if (NXT_HAVE_LINUX_CAPABILITY) #include +#include #include +#include #if (_LINUX_CAPABILITY_VERSION_3) @@ -49,7 +57,7 @@ nxt_capability_set(nxt_task_t *task, nxt_capabilities_t *cap) { nxt_assert(cap->setid == 0); - if (geteuid() == 0) { + if (nxt_euid == 0) { cap->setid = 1; cap->chroot = 1; return NXT_OK; @@ -77,6 +85,7 @@ nxt_capability_linux_get_version(void) static nxt_int_t nxt_capability_specific_set(nxt_task_t *task, nxt_capabilities_t *cap) { + nxt_err_t err; struct __user_cap_data_struct *val, data[2]; struct __user_cap_header_struct hdr; @@ -100,7 +109,43 @@ nxt_capability_specific_set(nxt_task_t *task, nxt_capabilities_t *cap) hdr.pid = nxt_pid; if (nxt_slow_path(nxt_capget(&hdr, val) == -1)) { - nxt_alert(task, "failed to get process capabilities: %E", nxt_errno); + err = nxt_errno; + + /* + * A syscall filter that does not allow capget() reports it + * through SECCOMP_RET_ERRNO, conventionally as EPERM or ENOSYS. + * Capabilities then cannot be observed at all, which is not a + * reason to refuse to start: leave both flags clear and run the + * way an ordinary unprivileged unitd already does. + * + * Any other errno means the call itself is wrong -- EINVAL, + * which a working version probe would have prevented, or EFAULT + * for a bad pointer -- so it is a bug in Unit rather than a + * policy decision made by the operator. Keep failing closed + * there, rather than silently downgrading privileges. + */ + + if (err == NXT_EPERM || err == NXT_ENOSYS) { + nxt_log(task, NXT_LOG_WARN, "capget() failed %E; process " + "capabilities are unknown and will not be used: user and " + "group switching and \"rootfs\" isolation are disabled " + "for applications that do not enable the \"credential\" " + "namespace, and applications run as uid %d", + err, (int) nxt_euid); + + /* + * The log file is not open yet -- this runs from + * nxt_runtime_conf_init() -- so the warning above reaches + * stderr only. Record it, and nxt_runtime_start() repeats it + * once unit.log exists, or a daemonised unitd would keep no + * trace of running without knowing its own capabilities. + */ + cap->unknown = 1; + + return NXT_OK; + } + + nxt_alert(task, "failed to get process capabilities: %E", err); return NXT_ERROR; } @@ -120,6 +165,607 @@ nxt_capability_specific_set(nxt_task_t *task, nxt_capabilities_t *cap) return NXT_OK; } + +/* + * The magic number rather than : one constant, stable + * since procfs existed, against a header that is not present on every + * libc Unit builds against. + */ +#define NXT_PROC_SUPER_MAGIC 0x9fa0 + + +static nxt_int_t +nxt_capability_status_line(u_char *p, u_char *end, uint64_t *bits) +{ + u_char *q; + uint64_t v; + nxt_uint_t digits; + + if (end - p <= (ssize_t) nxt_length("CapXxx:") + || memcmp(p, "Cap", 3) != 0) + { + return 0; + } + + if (memcmp(p, "CapPrm:", 7) != 0 + && memcmp(p, "CapEff:", 7) != 0 + && memcmp(p, "CapAmb:", 7) != 0) + { + /* CapInh and CapBnd, deliberately: see nxt_capability_still_held(). */ + return 0; + } + + q = p + nxt_length("CapXxx:"); + + while (q < end && (*q == ' ' || *q == '\t')) { + q++; + } + + v = 0; + digits = 0; + + while (q < end) { + if (*q >= '0' && *q <= '9') { + v = (v << 4) | (uint64_t) (*q - '0'); + + } else if (*q >= 'a' && *q <= 'f') { + v = (v << 4) | (uint64_t) (*q - 'a' + 10); + + } else if (*q >= 'A' && *q <= 'F') { + v = (v << 4) | (uint64_t) (*q - 'A' + 10); + + } else { + break; + } + + q++; + digits++; + } + + if (digits == 0 || digits > 16) { + return -1; + } + + *bits |= v; + + return 1; +} + + +static nxt_int_t +nxt_capability_proc_status_held(void) +{ + int fd; + u_char *p, *end, *eol; + size_t len; + ssize_t n; + uint64_t bits; + nxt_int_t ret; + nxt_uint_t found; + nxt_bool_t skipping; + struct statfs fs; + u_char buf[1024]; + + fd = open("/proc/self/status", O_RDONLY | O_CLOEXEC); + + if (fd < 0) { + return -1; + } + + /* + * Prove it is procfs before believing a word of it. An application + * that enables a "credential" namespace may also use "rootfs" even + * when capabilities are unknown, and nxt_proto_setup() has already + * pivot_root()ed into it by the time this runs -- with + * "automount": {"procfs": false} there may be no /proc there at all, + * or there may be a regular file of that name supplied with the + * application image. Reading capability sets out of one would let + * the image decide whether the prototype it is about to run keeps + * unitd's capabilities. + */ + + if (fstatfs(fd, &fs) != 0 || fs.f_type != NXT_PROC_SUPER_MAGIC) { + close(fd); + return -1; + } + + bits = 0; + found = 0; + len = 0; + skipping = 0; + + /* + * Streamed over a small buffer, with the tail of a partial line + * carried across reads, because this file has no bounded size: a + * process in enough supplementary groups pushes the Cap* lines past + * any fixed buffer on the strength of its "Groups:" line alone, and + * reading only the first N bytes would turn "holds nothing" into + * "cannot tell" and refuse it. A line too long for the buffer is + * skipped rather than grown into -- a Cap* line is 25 bytes. + */ + + for ( ;; ) { + n = read(fd, buf + len, sizeof(buf) - len); + + if (n < 0) { + + /* + * A signal that arrives mid-read is not an answer about + * capabilities. Without the retry it would become one: + * read() fails, the reader reports "cannot tell", and the + * caller refuses an application over a SIGCHLD. Every other + * errno is a real failure to read the file, and saying so is + * the honest reply. + */ + + if (nxt_errno == NXT_EINTR) { + continue; + } + + close(fd); + return -1; + } + + if (n == 0) { + break; + } + + end = buf + len + n; + p = buf; + + for ( ;; ) { + eol = memchr(p, '\n', (size_t) (end - p)); + + if (eol == NULL) { + break; + } + + if (!skipping) { + ret = nxt_capability_status_line(p, eol, &bits); + + if (nxt_slow_path(ret < 0)) { + close(fd); + return -1; + } + + found += ret; + } + + skipping = 0; + p = eol + 1; + } + + len = (size_t) (end - p); + + if (len == sizeof(buf)) { + skipping = 1; + len = 0; + + } else { + memmove(buf, p, len); + } + } + + /* + * Whatever is left when the file ends is a last line with no + * trailing newline. procfs always terminates the one it writes, so + * this is unreachable for the real /proc/self/status -- but the + * fstatfs() above is the only thing standing between this parser and + * a file supplied with an application image, and dropping the tail + * would silently turn a "CapPrm: 00000000000000c0" written without a + * newline into a file whose Cap* lines simply do not count. + */ + + if (len > 0 && !skipping) { + ret = nxt_capability_status_line(buf, buf + len, &bits); + + if (nxt_slow_path(ret < 0)) { + close(fd); + return -1; + } + + found += ret; + } + + close(fd); + + /* + * CapPrm and CapEff, at least. CapAmb needs Linux 4.3 and is + * allowed to be missing; fewer than two means this is not the file + * we think it is, and guessing from it would be worse than saying + * so. + */ + + if (found < 2) { + return -1; + } + + return bits != 0; +} + + +/* + * Does this process still hold anything an application could inherit? + * 1 yes, 0 no, -1 could not be determined. + * + * Consulted only after capset() has already been denied, to separate + * "the operator filtered a call that had nothing left to do" from "the + * sets really are still full". Without it the refusal in + * nxt_process_created_ok() would fire in the two commonest deployments + * there are, neither of which is unsafe: a root unitd, whose setuid() + * away from uid 0 empties permitted, effective and ambient before this + * function is ever reached, and an ordinary unprivileged unitd that was + * granted nothing. Both would lose every application to a filter that + * costs them nothing. + * + * Two readers are tried, cheapest first, because neither covers every + * deployment on its own. + * + * - capget(). A filter that denies capset() often still permits it, + * and this is the exact answer with no filesystem involved. It does + * not report the ambient set, which does not matter here: + * PR_CAP_AMBIENT_CLEAR_ALL in nxt_capability_drop() has already + * run, needs no privilege and cannot be refused by the capability + * model. pid 0 rather than nxt_pid for the same reason capset() + * uses it -- nxt_pid is the global pid, which does not resolve from + * inside a PID namespace. + * + * - /proc/self/status, when capget() is filtered too -- the shape this + * whole fallback exists for. A syscall filter cannot see through + * open() and read(). + * + * Both are readings. Nothing here is inferred from what this process + * ought to be holding, and an earlier draft that did -- a root unitd's + * setuid() away from uid 0 empties permitted, effective and ambient, so + * the transition alone looks like an answer -- was removed rather than + * patched. It was wrong two separate ways: SECBIT_KEEP_CAPS or + * SECBIT_NO_SETUID_FIXUP, either of which a service manager can set, + * suppress the clearing entirely; and inside a "credential" namespace a + * process can reach a nonzero euid while holding a full capability set + * in that namespace, never having made the transition at all. Both + * leave a process that looks disarmed and is not. It bought only the + * case where capget() is filtered *and* procfs is unreadable, and being + * wrong there is exactly where it costs the most. + * + * If neither reader answers, the caller refuses: this is a security + * fallback, and "could not tell" is not "there is nothing there". + * + * "Anything" means the permitted, effective and ambient sets, and only + * those three. The capset() this reader is consulted after still + * empties the inheritable set along with them whenever it is allowed to + * run -- what this function decides is narrower, and it is the only + * question worth asking here: is anything left that an application + * could use? + * + * An inheritable bit is not. It confers nothing in the process holding + * it; capabilities(7) turns it into privilege only on an execve() of a + * file whose own inheritable capability set intersects it. Unit's + * language modules are fork()ed and never exec()ed, so they cannot + * reach that rule at all, and an "external" application execs the + * binary named in its configuration -- an ordinary file, carrying no + * file capabilities unless an operator deliberately gave it some, in + * which case they asked for exactly this. Refusing over inheritable + * alone would therefore be refusing over something strictly narrower + * than the escalation this fallback exists to stop, and it is also + * precisely what an unprivileged unitd handed its workers before + * nxt_capability_drop() existed: nothing in Unit has ever cleared an + * inheritable set before now. + * + * That leaves a residual, and it is left deliberately: an "external" + * application whose own binary an operator gave a matching file + * inheritable capability does get that bit promoted on execve(). + * Refusing would not take it away -- capset() has just been denied, so + * nothing here can clear anything -- it would only take away the + * application. The trade is a certain outage on every host with a + * stray inheritable bit against a bit the operator had to place twice, + * once on unitd and once on the file it execs. + * + * Counting it would cost a great deal, too. Inheritable bits turn up on + * ordinary hosts without anyone meaning them to matter -- pam_cap exists + * to set exactly them, and the desktop this was measured on carries + * CapInh=0000000800000000 over entirely empty permitted, effective and + * ambient sets. A root unitd is no safer either, because setuid() away + * from uid 0 clears permitted, effective and ambient and leaves + * inheritable exactly as it was. Both would lose every application to a + * filter that costs them nothing. + * Conditioning it on PR_SET_NO_NEW_PRIVS does not rescue the idea + * either: no_new_privs is off in the default configuration -- + * nxt_isolation_main_prefork() sets isolation.new_privs to 1 and + * nxt_process_apply_creds() issues the prctl only when it is 0 -- so + * the condition would be false on exactly the hosts that have the + * stray bit. + * + * The bounding set is never counted either, for the reason it is never + * emptied: it is a ceiling on what an execve() may add, not something + * this process holds, and capset() would not have cleared it either. + * + * The ambient set is read where a reader offers it, even though the + * kernel keeps it a subset of permitted -- so an empty permitted + * already implies an empty ambient -- because the + * PR_CAP_AMBIENT_CLEAR_ALL in nxt_capability_drop() is only warned + * about when it fails. + */ +static nxt_int_t +nxt_capability_still_held(void) +{ + uint32_t bits; + struct __user_cap_data_struct data[2]; + struct __user_cap_header_struct hdr; + + nxt_memzero(data, sizeof(data)); + + hdr.version = NXT_CAPABILITY_VERSION; + hdr.pid = 0; + + if (nxt_fast_path(nxt_capget(&hdr, data) == 0)) { + bits = data[0].effective | data[0].permitted + | data[1].effective | data[1].permitted; + + return bits != 0; + } + + return nxt_capability_proc_status_held(); +} + + +/* + * Drop every capability the calling process still holds. Called from + * the tail of nxt_process_apply_creds(), which is the last thing that + * runs in a freshly forked prototype, router, controller or discovery + * process before it starts doing work on an application's behalf -- and + * after setgid()/setuid(), after unshare() and main's uid_map write, + * and after all the mount/pivot_root/chroot work nxt_proto_setup() does + * for "rootfs". Application workers never reach this function: the + * prototype forks them and nxt_app_setup() calls init->start directly, + * so they inherit sets this call already emptied. + * + * It is not the earliest possible point. nxt_proto_setup() calls the + * language module's setup first, and for PHP that is + * php_module_startup() with the php.ini named in the configuration, + * which loads and initialises whatever extensions that file lists. + * Nothing can be dropped before that: the mount, pivot_root and + * chroot calls later in the same function need CAP_SYS_ADMIN and + * CAP_SYS_CHROOT. The window is unchanged by this code and is not + * widened by it -- setgid()/setuid() have not happened yet either, so + * module startup has always run with unitd's own uid, which is the + * larger privilege of the two. + * + * Capabilities are per-thread, and capset() with pid 0 sets only the + * calling thread's, so this has to run while the process is still + * single-threaded -- and it does. The router's engine threads are + * created from init->start, which runs after this and inherits the + * emptied sets; the prototype and its workers never leave one thread; + * thread pools spawn on their first posted work item, which is later + * still. The one thread that could pre-exist is the signal thread, + * and only for an event engine that cannot poll signals itself + * (nxt_signal_thread_start(), reached from nxt_event_engine_create() + * when !interface->signal_support). Linux never selects one: epoll + * declares NXT_SIGNAL_EVENTS wherever signalfd exists, and rt->engine + * is only ever set to the interface Unit itself chose + * (nxt_runtime.c), never by an operator. Measured: with the router at + * nine threads, every /proc//task//status in every child + * reports empty sets. + * + * Why here and not in main: main binds listening sockets in + * nxt_main_listening_socket() every time the configuration changes, so + * it needs CAP_NET_BIND_SERVICE for the whole life of the process -- + * and also chown()s sockets, writes uid_map and moves children into + * cgroups. nxt_capability_set() runs from nxt_runtime_conf_init(), + * long before the first bind(), so a drop there would turn "unitd fails + * to start" into "unitd starts but cannot listen on :80". + * + * fork() copies the permitted, effective, inheritable, ambient and + * bounding sets verbatim, so without this a non-root unitd that was + * granted, say, CAP_SETUID/CAP_SETGID/CAP_SYS_CHROOT would hand those + * to every application worker. Two deployments reach that state: + * + * - capget() filtered. nxt_capability_specific_set() leaves setid + * clear, so nxt_process_apply_creds() skips setgid()/setuid() + * entirely and nothing lowers anything. + * - capget() working. The credentials are switched, but from one + * nonzero uid to another, and capabilities(7) only clears the sets + * on a transition *out of* uid 0 -- so the capabilities survive + * the setuid() as well. + * + * A root unitd is the case that was already safe: its setuid() away + * from 0 clears the permitted, effective and ambient sets by itself, + * which makes the capset() below a verified no-op there rather than a + * behaviour change. + * + * The bounding set is deliberately left alone: emptying it needs + * CAP_SETPCAP, which this process is about to give up and usually + * never had. It is a ceiling on what an execve() may add to the + * permitted set, and with inheritable emptied below the only thing + * left for it to admit is a file capability on the binary an + * application execve()s -- which is the operator's own choice, not + * something inherited from unitd. PR_SET_NO_NEW_PRIVS would close + * even that, but it is off in the default configuration (see the + * comment on nxt_capability_still_held() above), so it is not what + * makes this safe. + * + * The return value is three-way and advisory in the middle case: + * NXT_OK when nothing is left to inherit, NXT_ERROR when the call was + * malformed, and NXT_DECLINED when a syscall filter denied capset() + * and this process is still holding something as a result. + * NXT_DECLINED is deliberately not a refusal here. This function is + * reached by the router, the controller and discovery as well, through + * nxt_process_core_setup(), and failing those closed is worse than the + * bug it would be protecting against: router and controller carry + * .restart = 1 and are re-forked immediately and without backoff, so a + * filter that is going to deny every attempt turns into an unbounded + * respawn loop; and discovery notifies no one when it dies, leaving + * unitd running with no modules and no control socket. Only the caller + * knows whether application code is about to inherit what could not be + * dropped, so only the caller can decide. + */ +nxt_int_t +nxt_capability_drop(nxt_task_t *task) +{ + nxt_int_t held; + nxt_err_t err; + struct __user_cap_data_struct data[2]; + struct __user_cap_header_struct hdr; + + if (geteuid() == 0) { + /* + * An application configured with "user": "root" -- and the + * initial state of a root unitd, before apply_creds() has + * switched user -- keeps what root has. Dropping there would + * be a compatibility break with no security value: euid 0 + * regains a full permitted set on the next execve() of any + * ordinary file anyway (capabilities(7), "Capabilities and + * execution of programs by root"), so the drop could not hold + * for an exec'd application and would only make an embedded + * one inconsistently crippled. This mirrors the geteuid()==0 + * short-circuit in nxt_capability_set() above. + * + * It is also the escape hatch for the one case that does lose + * something here: an application in a "credential" namespace + * starts with a full capability set in that namespace + * (user_namespaces(7)), and unless it is mapped to uid 0 it + * now loses it. Those capabilities were only ever valid + * inside the namespace, and everything Unit itself does with + * them -- the mounts, pivot_root and chroot for "rootfs" -- + * has already happened in nxt_proto_setup() by the time this + * runs. An application that wants to keep them can ask for + * uid 0 within its own namespace. + * + * geteuid() rather than nxt_euid: nxt_euid is sampled once in + * nxt_lib_start() and is stale in a process that has just + * called setuid(). + */ + + return NXT_OK; + } + +#ifdef PR_CAP_AMBIENT + + /* + * Clear the ambient set first, and separately. For a non-root + * process executing a file with no capabilities, capabilities(7) + * reduces to P'(permitted) = P'(effective) = P(ambient): the + * ambient set is the only one that carries real privilege across + * execve(), inheritable survives but does nothing without a + * matching file inheritable set. So ambient is precisely what an + * "external" application -- Go, Node, anything Unit runs through + * the execve() in nxt_external.c -- would otherwise inherit. + * PR_CAP_AMBIENT_CLEAR_ALL needs no privilege and no prior read of + * the set, so it still works when capget() and capset() are both + * denied; emptying the permitted set below clears ambient + * implicitly, but only if that call succeeds. + * + * A plain #ifdef rather than an auto/ feature test: PR_CAP_AMBIENT + * is a macro in , so this compiles out on headers too + * old to define it, and a kernel older than 4.3 has no ambient set + * to clear and answers EINVAL, which is not worth a warning. + */ + + if (nxt_slow_path(prctl(PR_CAP_AMBIENT, PR_CAP_AMBIENT_CLEAR_ALL, 0, 0, 0) + != 0)) + { + nxt_debug(task, "prctl(PR_CAP_AMBIENT_CLEAR_ALL) failed %E", + nxt_errno); + } + +#endif + + nxt_memzero(data, sizeof(data)); + + /* + * The compiled-in version constant, not + * nxt_capability_linux_get_version(): the probe is a capget() call, + * and this function has to work in the deployment where capget() is + * exactly what is filtered. On every + * header Unit builds against this is _LINUX_CAPABILITY_VERSION_3, + * which the kernel has accepted since 2.6.26 and which describes the + * two-element data array above. + * + * pid 0, not nxt_pid: capset() takes only 0 or the caller's own pid, + * and nxt_pid holds the *global* pid in a process that was moved into + * a PID namespace (nxt_process_whoami()), which would be EPERM here. + */ + + hdr.version = NXT_CAPABILITY_VERSION; + hdr.pid = 0; + + if (nxt_fast_path(nxt_capset(&hdr, data) == 0)) { + return NXT_OK; + } + + err = nxt_errno; + + /* + * Same errno split as nxt_capability_specific_set(): EPERM and + * ENOSYS are what a syscall filter returns, and a filter is an + * operator's decision, not an error. It is also the only way this + * call can fail with EPERM: lowering all three sets to empty is + * always permitted for any process (capabilities(7), the transition + * rules for a caller without CAP_SETPCAP), so a refusal cannot have + * come from the capability model itself. + * + * A denial is not automatically a problem, though, which is why + * nxt_capability_still_held() gets the last word: a root unitd has + * already emptied every set by switching away from uid 0, and an + * unprivileged one never held anything, so in both the call that + * was just refused would have done nothing. Only a process that + * demonstrably still holds something -- or cannot find out -- has + * anything to report. + * + * NXT_DECLINED rather than NXT_ERROR, and rather than NXT_OK: the + * consequence of a denied capset() is not the same for every + * caller, so this function reports "the sets are still full" and + * lets nxt_process_apply_creds()'s callers decide. A prototype + * refuses to start, because its whole remaining job is to fork + * application workers that would inherit those sets; the router, + * controller and discovery processes warn and continue, because + * they run no application code and killing them is worse than the + * bug -- see nxt_process_created_ok() and nxt_process_core_setup(). + * + * Anything else -- EINVAL, EFAULT -- means the call we just made is + * malformed, which can only be a bug in Unit, since neither the + * version nor the pointer depends on anything an operator controls. + * Fail closed there, for every caller alike. + */ + + if (err == NXT_EPERM || err == NXT_ENOSYS) { + held = nxt_capability_still_held(); + + if (held == 0) { + /* + * The overwhelmingly common case, and the reason this is + * measured rather than assumed. A root unitd has already + * cleared every set by switching away from uid 0, and an + * ordinary unprivileged unitd never had anything -- in both + * the call that was just denied would have been a no-op, and + * refusing over it would take the application down for no + * security benefit at all. Not even a warning: nothing + * happened. + */ + + nxt_debug(task, "capset() failed %E, and no capability is held", + err); + + return NXT_OK; + } + + if (held > 0) { + nxt_log(task, NXT_LOG_WARN, "capset() failed %E; the capabilities " + "this process holds could not be dropped", err); + + } else { + nxt_log(task, NXT_LOG_WARN, "capset() failed %E, and whether this " + "process holds any capability could not be determined " + "either", err); + } + + return NXT_DECLINED; + } + + nxt_alert(task, "failed to drop process capabilities: %E", err); + + return NXT_ERROR; +} + #else static nxt_int_t @@ -128,4 +774,11 @@ nxt_capability_specific_set(nxt_task_t *task, nxt_capabilities_t *cap) return NXT_OK; } + +nxt_int_t +nxt_capability_drop(nxt_task_t *task) +{ + return NXT_OK; +} + #endif diff --git a/src/nxt_capability.h b/src/nxt_capability.h index 1575d4094..0364db179 100644 --- a/src/nxt_capability.h +++ b/src/nxt_capability.h @@ -9,10 +9,20 @@ typedef struct { uint8_t setid; /* 1 bit */ uint8_t chroot; /* 1 bit */ + uint8_t unknown; /* 1 bit */ } nxt_capabilities_t; NXT_EXPORT nxt_int_t nxt_capability_set(nxt_task_t *task, nxt_capabilities_t *cap); +/* + * NXT_OK: nothing is left to inherit -- either the sets were emptied, + * or they were already empty when a filter denied the attempt. + * NXT_DECLINED: capset() is filtered and this process is still holding + * capabilities, or cannot determine that it is not -- advisory, the + * caller decides whether that is fatal. NXT_ERROR: the call was + * malformed. + */ +NXT_EXPORT nxt_int_t nxt_capability_drop(nxt_task_t *task); #endif /* _NXT_CAPABILITY_INCLUDED_ */ diff --git a/src/nxt_cert.c b/src/nxt_cert.c index ece5e9fd5..accdddcde 100644 --- a/src/nxt_cert.c +++ b/src/nxt_cert.c @@ -7,6 +7,8 @@ #include #include #include +#include +#include #include @@ -19,6 +21,26 @@ #include +/* The buffer X509_NAME_get_text_by_NID() used to be given, less the NUL. */ +#define NXT_CERT_NAME_TEXT_MAX 255 + +/* + * The X509 name accessors were constified in stages, and not together: + * X509_NAME_get_entry() and X509_NAME_ENTRY_get_data() took const arguments + * from 1.1.0, but X509_NAME_get_index_by_NID() only from 3.0, and it is 4.0 + * that constified the values all three return. Qualifying from 3.0 is what + * satisfies every version at once -- an older header takes these pointers + * non-const, and 4.0 hands them back const -- so the boundary is 3.0 rather + * than the 1.1.0 the other guards in this file use. + */ + +#if OPENSSL_VERSION_NUMBER >= 0x30000000L +#define NXT_X509_CONST const +#else +#define NXT_X509_CONST +#endif + + struct nxt_cert_s { EVP_PKEY *key; nxt_uint_t count; @@ -26,11 +48,29 @@ struct nxt_cert_s { }; -typedef struct { +/* The size of a SHA-256 digest. */ +#define NXT_CERT_DIGEST_LENGTH 32 + + +struct nxt_cert_info_s { nxt_str_t name; nxt_conf_value_t *value; nxt_mp_t *mp; -} nxt_cert_info_t; + + /* See nxt_cert_digest(). */ + u_char digest[NXT_CERT_DIGEST_LENGTH]; + + /* + * The disk or the router may not hold this bundle: the router refused + * the configuration with it, or a store failed and the file may hold + * either bundle. See nxt_cert_info_equal(). The flag is only in the + * memory of the controller. A new controller reads the bundles from + * disk with the flag clear, and a file changed by hand is not seen. In + * these cases an equal upload is skipped until a different bundle is + * stored. + */ + uint8_t unapplied; /* 1 bit */ +}; typedef struct { @@ -43,9 +83,13 @@ static nxt_cert_t *nxt_cert_fd(nxt_task_t *task, nxt_fd_t fd); static nxt_cert_t *nxt_cert_bio(nxt_task_t *task, BIO *bio); static int nxt_nxt_cert_pem_suffix(char *pem_str, const char *suffix); +static nxt_int_t nxt_cert_digest(nxt_cert_t *cert, u_char *digest); +static nxt_cert_info_t *nxt_cert_info_find(nxt_str_t *name); static nxt_conf_value_t *nxt_cert_details(nxt_mp_t *mp, nxt_cert_t *cert); static nxt_conf_value_t *nxt_cert_name_details(nxt_mp_t *mp, X509 *x509, nxt_bool_t issuer); +static nxt_int_t nxt_cert_name_text(NXT_X509_CONST X509_NAME *x509_name, + int nid, nxt_str_t *str); static nxt_conf_value_t *nxt_cert_alt_names_details(nxt_mp_t *mp, STACK_OF(GENERAL_NAME) *alt_names); static void nxt_cert_buf_completion(nxt_task_t *task, void *obj, void *data); @@ -302,6 +346,17 @@ nxt_cert_bio(nxt_task_t *task, BIO *bio) goto fail; } + /* + * The leaf comes first in the bundle. A key that belongs to another + * certificate is refused here, at upload, instead of at the next + * reconfiguration of every listener that names the bundle. + */ + if (X509_check_private_key(cert->chain[0], cert->key) != 1) { + nxt_openssl_log_error(task, NXT_LOG_ALERT, + "certificate and private key do not match"); + goto fail; + } + return cert; fail: @@ -406,15 +461,39 @@ nxt_cert_info_init(nxt_task_t *task, nxt_array_t *certs) nxt_int_t nxt_cert_info_save(nxt_str_t *name, nxt_cert_t *cert) { - nxt_mp_t *mp; - nxt_int_t ret; - nxt_cert_info_t *info; - nxt_conf_value_t *value; - nxt_lvlhsh_query_t lhq; + nxt_cert_info_t *info, *old; + + info = nxt_cert_info_create(name, cert); + if (nxt_slow_path(info == NULL)) { + return NXT_ERROR; + } + + if (nxt_slow_path(nxt_cert_info_replace(info, &old) != NXT_OK)) { + nxt_cert_info_release(info); + return NXT_ERROR; + } + + nxt_cert_info_release(old); + + return NXT_OK; +} + + +/* + * Make the metadata of a bundle without publishing it. The controller calls + * this and nxt_cert_info_replace() before main replaces the file, so that a + * failed allocation does not change the stored bundle. + */ +nxt_cert_info_t * +nxt_cert_info_create(nxt_str_t *name, nxt_cert_t *cert) +{ + nxt_mp_t *mp; + nxt_cert_info_t *info; + nxt_conf_value_t *value; mp = nxt_mp_create(1024, 128, 256, 32); if (nxt_slow_path(mp == NULL)) { - return NXT_ERROR; + return NULL; } info = nxt_mp_get(mp, sizeof(nxt_cert_info_t)); @@ -432,39 +511,193 @@ nxt_cert_info_save(nxt_str_t *name, nxt_cert_t *cert) goto fail; } + if (nxt_slow_path(nxt_cert_digest(cert, info->digest) != NXT_OK)) { + goto fail; + } + info->mp = mp; info->value = value; + info->unapplied = 0; - lhq.key_hash = nxt_djb_hash(name->start, name->length); + return info; + +fail: + + nxt_mp_destroy(mp); + return NULL; +} + + +/* + * "digest" gets the SHA-256 digest of the SHA-256 digests of the + * certificates, in bundle order. Two bundles with the same digest hold the + * same certificates. They also hold the same key: nxt_cert_bio() has + * checked the key against the first certificate, and one public key has + * one private key. The PEM text may still differ, in line breaks or in + * the key format. The router loads the same contexts from either. + */ +static nxt_int_t +nxt_cert_digest(nxt_cert_t *cert, u_char *digest) +{ + u_char *p, *buf; + nxt_int_t ret; + nxt_uint_t i; + unsigned int len; + + buf = nxt_malloc(cert->count * EVP_MAX_MD_SIZE); + if (nxt_slow_path(buf == NULL)) { + return NXT_ERROR; + } + + ret = NXT_ERROR; + p = buf; + + for (i = 0; i < cert->count; i++) { + if (nxt_slow_path(X509_digest(cert->chain[i], EVP_sha256(), p, &len) + != 1)) + { + nxt_thread_log_alert("X509_digest() failed"); + goto done; + } + + p += len; + } + + if (nxt_fast_path(EVP_Digest(buf, p - buf, digest, &len, EVP_sha256(), + NULL) == 1 + && len == NXT_CERT_DIGEST_LENGTH)) + { + ret = NXT_OK; + + } else { + nxt_thread_log_alert("EVP_Digest() failed"); + } + +done: + + nxt_free(buf); + + return ret; +} + + +/* + * Whether "cert" holds the same certificates as the stored bundle "name", + * and the router has that bundle. The controller then stores nothing and + * applies nothing. After a store whose configuration the router refused, + * this is false until the same name is stored and applied once more, so a + * repeated upload of that bundle is a full one. + */ +nxt_bool_t +nxt_cert_info_equal(nxt_str_t *name, nxt_cert_t *cert) +{ + u_char digest[NXT_CERT_DIGEST_LENGTH]; + nxt_cert_info_t *info; + + info = nxt_cert_info_find(name); + + if (info == NULL || info->unapplied) { + return 0; + } + + if (nxt_slow_path(nxt_cert_digest(cert, digest) != NXT_OK)) { + return 0; + } + + return (memcmp(digest, info->digest, NXT_CERT_DIGEST_LENGTH) == 0); +} + + +/* The controller records whether the router took the stored bundle. */ +void +nxt_cert_info_applied(nxt_str_t *name, nxt_bool_t applied) +{ + nxt_cert_info_t *info; + + info = nxt_cert_info_find(name); + + if (info != NULL) { + info->unapplied = !applied; + } +} + + +/* + * Put the metadata in the hash. "old" gets the old metadata of the same + * name, or NULL when the name is new. The caller keeps "old" until it calls + * nxt_cert_info_release() or nxt_cert_info_restore(). On failure, the hash + * does not change. + */ +nxt_int_t +nxt_cert_info_replace(nxt_cert_info_t *info, nxt_cert_info_t **old) +{ + nxt_int_t ret; + nxt_lvlhsh_query_t lhq; + + lhq.key_hash = nxt_djb_hash(info->name.start, info->name.length); lhq.replace = 1; - lhq.key = *name; + lhq.key = info->name; lhq.value = info; lhq.proto = &nxt_cert_info_hash_proto; ret = nxt_lvlhsh_insert(&nxt_cert_info, &lhq); if (nxt_slow_path(ret != NXT_OK)) { - goto fail; + return NXT_ERROR; } - if (lhq.value != info) { - info = lhq.value; - nxt_mp_destroy(info->mp); - } + *old = (lhq.value != info) ? lhq.value : NULL; return NXT_OK; +} -fail: - nxt_mp_destroy(mp); - return NXT_ERROR; +/* + * Undo nxt_cert_info_replace() after a failed store: put the old metadata + * back, or remove the name when it was new. Both only free memory, so this + * cannot fail. Releases "info". + * + * Main can fail or die after it renamed the new file over the old one, so + * the file can hold either bundle. The old metadata is thus marked as not + * applied, and the next upload of either bundle is a full store. + */ +void +nxt_cert_info_restore(nxt_cert_info_t *info, nxt_cert_info_t *old) +{ + nxt_lvlhsh_query_t lhq; + + lhq.key_hash = nxt_djb_hash(info->name.start, info->name.length); + lhq.key = info->name; + lhq.proto = &nxt_cert_info_hash_proto; + + if (old != NULL) { + old->unapplied = 1; + + lhq.replace = 1; + lhq.value = old; + + (void) nxt_lvlhsh_insert(&nxt_cert_info, &lhq); + + } else { + (void) nxt_lvlhsh_delete(&nxt_cert_info, &lhq); + } + + nxt_cert_info_release(info); } -nxt_conf_value_t * -nxt_cert_info_get(nxt_str_t *name) +void +nxt_cert_info_release(nxt_cert_info_t *info) +{ + if (info != NULL) { + nxt_mp_destroy(info->mp); + } +} + + +static nxt_cert_info_t * +nxt_cert_info_find(nxt_str_t *name) { nxt_int_t ret; - nxt_cert_info_t *info; nxt_lvlhsh_query_t lhq; lhq.key_hash = nxt_djb_hash(name->start, name->length); @@ -476,9 +709,18 @@ nxt_cert_info_get(nxt_str_t *name) return NULL; } - info = lhq.value; + return lhq.value; +} + + +nxt_conf_value_t * +nxt_cert_info_get(nxt_str_t *name) +{ + nxt_cert_info_t *info; - return info->value; + info = nxt_cert_info_find(name); + + return (info != NULL) ? info->value : NULL; } @@ -536,22 +778,26 @@ nxt_cert_details(nxt_mp_t *mp, nxt_cert_t *cert) X509 *x509; u_char *end; EVP_PKEY *key; - ASN1_TIME *asn1_time; + const ASN1_TIME *asn1_time; nxt_str_t str; nxt_int_t ret; nxt_uint_t i; + unsigned int md_len; nxt_conf_value_t *object, *chain, *element, *value; - u_char buf[256]; + u_char buf[256], md[EVP_MAX_MD_SIZE]; + + static const u_char hex[] = "0123456789ABCDEF"; static const nxt_str_t key_str = nxt_string("key"); static const nxt_str_t chain_str = nxt_string("chain"); + static const nxt_str_t fingerprint_str = nxt_string("fingerprint"); static const nxt_str_t since_str = nxt_string("since"); static const nxt_str_t until_str = nxt_string("until"); static const nxt_str_t issuer_str = nxt_string("issuer"); static const nxt_str_t subject_str = nxt_string("subject"); static const nxt_str_t validity_str = nxt_string("validity"); - object = nxt_conf_create_object(mp, 2); + object = nxt_conf_create_object(mp, 3); if (nxt_slow_path(object == NULL)) { return NULL; } @@ -594,6 +840,40 @@ nxt_cert_details(nxt_mp_t *mp, nxt_cert_t *cert) nxt_conf_set_member_null(object, &key_str, 0); } + /* + * The SHA-256 fingerprint of the server certificate, in the form + * "openssl x509 -fingerprint -sha256" prints: uppercase hex, a colon + * between the bytes. A renewal script compares it with the + * fingerprint of the file it has, and uploads only a new one. + */ + + if (nxt_slow_path(X509_digest(cert->chain[0], EVP_sha256(), md, &md_len) + != 1)) + { + nxt_thread_log_alert("X509_digest() failed"); + return NULL; + } + + end = buf; + + for (i = 0; i < md_len; i++) { + if (i != 0) { + *end++ = ':'; + } + + *end++ = hex[md[i] >> 4]; + *end++ = hex[md[i] & 0x0f]; + } + + str.length = end - buf; + str.start = buf; + + ret = nxt_conf_set_member_string_dup(object, mp, &fingerprint_str, &str, + 1); + if (nxt_slow_path(ret != NXT_OK)) { + return NULL; + } + chain = nxt_conf_create_array(mp, cert->count); if (nxt_slow_path(chain == NULL)) { return NULL; @@ -631,7 +911,7 @@ nxt_cert_details(nxt_mp_t *mp, nxt_cert_t *cert) return NULL; } - asn1_time = X509_get_notBefore(x509); + asn1_time = X509_get0_notBefore(x509); ret = ASN1_TIME_print(bio, asn1_time); @@ -654,7 +934,7 @@ nxt_cert_details(nxt_mp_t *mp, nxt_cert_t *cert) return NULL; } - asn1_time = X509_get_notAfter(x509); + asn1_time = X509_get0_notAfter(x509); ret = ASN1_TIME_print(bio, asn1_time); @@ -677,7 +957,7 @@ nxt_cert_details(nxt_mp_t *mp, nxt_cert_t *cert) nxt_conf_set_element(chain, i, element); } - nxt_conf_set_member(object, &chain_str, chain, 1); + nxt_conf_set_member(object, &chain_str, chain, 2); return object; } @@ -692,14 +972,12 @@ typedef struct { static nxt_conf_value_t * nxt_cert_name_details(nxt_mp_t *mp, X509 *x509, nxt_bool_t issuer) { - int len; - X509_NAME *x509_name; + NXT_X509_CONST X509_NAME *x509_name; nxt_str_t str; - nxt_int_t ret; + nxt_int_t ret, found; nxt_uint_t i, n, count; nxt_conf_value_t *object, *names; STACK_OF(GENERAL_NAME) *alt_names; - u_char buf[256]; static const nxt_cert_nid_t nids[] = { { NID_commonName, nxt_string("common_name") }, @@ -752,19 +1030,25 @@ nxt_cert_name_details(nxt_mp_t *mp, X509 *x509, nxt_bool_t issuer) for (n = 0, i = 0; n != nxt_nitems(nids) && i != count; n++) { - len = X509_NAME_get_text_by_NID(x509_name, nids[n].nid, - (char *) buf, sizeof(buf)); + found = nxt_cert_name_text(x509_name, nids[n].nid, &str); if (n == 1 && names != NULL) { nxt_conf_set_member(object, &alt_names_str, names, i++); } - if (len < 0) { + if (found != NXT_OK) { continue; } - str.length = len; - str.start = buf; + /* + * X509_NAME_get_text_by_NID() used to copy into a 256 byte buffer and + * report the truncated length. Keep that cap: it is what the control + * API has always reported for pathologically long names. + */ + + if (str.length > NXT_CERT_NAME_TEXT_MAX) { + str.length = NXT_CERT_NAME_TEXT_MAX; + } ret = nxt_conf_set_member_string_dup(object, mp, &nids[n].name, &str, i++); @@ -777,6 +1061,51 @@ nxt_cert_name_details(nxt_mp_t *mp, X509 *x509, nxt_bool_t issuer) } +/* + * Return the raw contents of the first entry of x509_name that carries nid, + * or NXT_DECLINED if there is none. + * + * This replaces X509_NAME_get_text_by_NID(), deprecated in OpenSSL 4.0. The + * old function copied the entry verbatim into the caller's buffer; the entry + * accessors hand out the same bytes without the copy, pointing into the + * certificate, which outlives the returned string here. + * + * As before the bytes are neither NUL terminated nor guaranteed free of + * embedded NULs, so the caller must keep treating them as a counted string. + */ + +static nxt_int_t +nxt_cert_name_text(NXT_X509_CONST X509_NAME *x509_name, int nid, + nxt_str_t *str) +{ + int i, len; + NXT_X509_CONST ASN1_STRING *data; + NXT_X509_CONST X509_NAME_ENTRY *entry; + + i = X509_NAME_get_index_by_NID(x509_name, nid, -1); + if (i < 0) { + return NXT_DECLINED; + } + + entry = X509_NAME_get_entry(x509_name, i); + if (nxt_slow_path(entry == NULL)) { + return NXT_DECLINED; + } + + data = X509_NAME_ENTRY_get_data(entry); + + len = ASN1_STRING_length(data); + if (nxt_slow_path(len < 0)) { + return NXT_DECLINED; + } + + str->length = len; + str->start = (u_char *) ASN1_STRING_get0_data(data); + + return NXT_OK; +} + + static nxt_conf_value_t * nxt_cert_alt_names_details(nxt_mp_t *mp, STACK_OF(GENERAL_NAME) *alt_names) { @@ -812,11 +1141,7 @@ nxt_cert_alt_names_details(nxt_mp_t *mp, STACK_OF(GENERAL_NAME) *alt_names) } str.length = ASN1_STRING_length(name->d.dNSName); -#if OPENSSL_VERSION_NUMBER > 0x10100000L str.start = (u_char *) ASN1_STRING_get0_data(name->d.dNSName); -#else - str.start = ASN1_STRING_data(name->d.dNSName); -#endif ret = nxt_conf_set_element_string_dup(array, mp, i++, &str); if (nxt_slow_path(ret != NXT_OK)) { @@ -898,7 +1223,8 @@ nxt_cert_store_load(nxt_task_t *task, nxt_mp_t *mp) name.length = nxt_strlen(de->d_name); name.start = (u_char *) de->d_name; - if (nxt_str_eq(&name, ".", 1) || nxt_str_eq(&name, "..", 2)) { + /* ".", "..", and the names the store keeps for itself. */ + if (name.start[0] == '.') { continue; } @@ -1178,8 +1504,13 @@ nxt_cert_store_get_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) /* * Look up the sender's port via the kernel-validated PID * (SCM_CREDENTIALS). msg->port_msg.pid is self-declared, so using - * it would let a compromised worker spoof the controller / router - * and pull arbitrary certificate material out of main. + * it would let a compromised worker spoof the router and pull + * arbitrary certificate material out of main. + * + * Only the router reads bundles: it opens every bundle a listener + * names during a reconfiguration. The controller parses a bundle + * from the request body and sends it to main with CERT_STORE; it + * never reads one back. */ port = nxt_runtime_port_find(task->thread->runtime, nxt_recv_msg_cmsg_pid(msg), @@ -1192,10 +1523,8 @@ nxt_cert_store_get_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) return; } - if (nxt_slow_path(port->type != NXT_PROCESS_CONTROLLER - && port->type != NXT_PROCESS_ROUTER)) - { - nxt_alert(task, "process %PI cannot store certificates", + if (nxt_slow_path(port->type != NXT_PROCESS_ROUTER)) { + nxt_alert(task, "process %PI cannot read certificates", nxt_recv_msg_cmsg_pid(msg)); nxt_port_recv_msg_close_fds(msg); return; @@ -1225,7 +1554,7 @@ nxt_cert_store_get_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) p = nxt_cpymem(file.name, rt->certs.start, rt->certs.length); p = nxt_cpymem(p, name.start, name.length + 1); - ret = nxt_file_open(task, &file, NXT_FILE_RDWR, NXT_FILE_CREATE_OR_OPEN, + ret = nxt_file_open(task, &file, NXT_FILE_RDONLY, NXT_FILE_OPEN, NXT_FILE_OWNER_ACCESS); nxt_free(file.name); @@ -1255,6 +1584,243 @@ nxt_cert_store_get_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) } +/* + * Send "mbuf" to main to store it as the bundle "name". The bundle goes in + * a shared memory segment, as conf.json does. The port message holds the + * NUL-terminated name, followed by the size. "handler" gets RPC_READY when + * the file is in place, and RPC_ERROR or a NULL message otherwise. + */ +void +nxt_cert_store_put(nxt_task_t *task, nxt_str_t *name, nxt_buf_mem_t *mbuf, + nxt_mp_t *mp, nxt_port_rpc_handler_t handler, void *ctx) +{ + void *mem; + size_t size; + uint32_t stream; + nxt_fd_t fd; + nxt_int_t ret; + nxt_buf_t *b; + nxt_port_t *main_port, *ctl_port; + nxt_runtime_t *rt; + + size = nxt_buf_mem_used_size(mbuf); + + fd = nxt_shm_open(task, size); + if (nxt_slow_path(fd == -1)) { + goto fail; + } + + mem = nxt_mem_mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (nxt_slow_path(mem == MAP_FAILED)) { + goto fail; + } + + nxt_memcpy(mem, mbuf->pos, size); + nxt_mem_munmap(mem, size); + + b = nxt_buf_mem_alloc(mp, name->length + 1 + sizeof(size_t), 0); + if (nxt_slow_path(b == NULL)) { + goto fail; + } + + b->completion_handler = nxt_cert_buf_completion; + + nxt_buf_cpystr(b, name); + *b->mem.free++ = '\0'; + b->mem.free = nxt_cpymem(b->mem.free, &size, sizeof(size_t)); + + rt = task->thread->runtime; + main_port = rt->port_by_type[NXT_PROCESS_MAIN]; + ctl_port = rt->port_by_type[NXT_PROCESS_CONTROLLER]; + + stream = nxt_port_rpc_register_handler(task, ctl_port, handler, handler, + -1, ctx); + if (nxt_slow_path(stream == 0)) { + goto fail; + } + + ret = nxt_port_socket_write(task, main_port, + NXT_PORT_MSG_CERT_STORE | NXT_PORT_MSG_CLOSE_FD, + fd, stream, ctl_port->id, b); + + if (nxt_slow_path(ret != NXT_OK)) { + nxt_port_rpc_cancel(task, ctl_port, stream); + goto fail; + } + + /* See nxt_cert_store_get(). */ + nxt_mp_retain(mp); + + return; + +fail: + + /* The port layer owns the descriptor only after a successful write. */ + if (fd != -1) { + nxt_fd_close(fd); + } + + handler(task, NULL, ctx); +} + + +/* + * The name of a bundle in a message of the controller: NUL-terminated, not + * empty, at most NXT_CERT_NAME_MAX_LENGTH bytes, without "/", and not + * starting with ".". The controller checks the name already, but it is + * unprivileged, and main builds a path from the name. "span" is left at + * the first byte after the NUL. + */ +static nxt_int_t +nxt_cert_store_name(nxt_buf_t *b, nxt_span_t *span, nxt_str_t *name) +{ + u_char *nul; + const u_char *start; + + if (b == NULL) { + return NXT_ERROR; + } + + nxt_span_init(span, b->mem.pos, b->mem.free); + + nul = memchr(b->mem.pos, '\0', nxt_span_len(span)); + if (nul == NULL) { + return NXT_ERROR; + } + + name->length = nul - b->mem.pos; + + if (nxt_span_take(span, name->length + 1, &start) != 0 + || name->length == 0 + || name->length > NXT_CERT_NAME_MAX_LENGTH + || start[0] == '.' + || memchr(start, '/', name->length) != NULL) + { + return NXT_ERROR; + } + + name->start = b->mem.pos; + + return NXT_OK; +} + + +/* + * Main's side of nxt_cert_store_put(). The store child of main writes the + * bundle to "certs/.store.tmp" and renames it over "certs/", so a + * reader sees the old bundle or the new one, never a partial file. The + * temporary name is fixed, so a name of NAME_MAX bytes still works, and one + * name is enough: one store child runs at a time, and a bundle name cannot + * start with ".". The controller is unprivileged, so main checks the + * sender and the name again. + * + * Main answers when the store child exits (nxt_main_store_submit()), so + * the controller answers 200 only when the bundle is on disk. The two + * fsync(2) calls of the store do not stop main. + */ +void +nxt_cert_store_put_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) +{ + size_t size; + u_char *mem; + nxt_str_t name; + nxt_file_t file; + nxt_port_t *port; + nxt_span_t span; + nxt_runtime_t *rt; + nxt_main_store_job_t *job; + + port = nxt_runtime_port_find(task->thread->runtime, + nxt_recv_msg_cmsg_pid(msg), + msg->port_msg.reply_port); + + if (nxt_slow_path(port == NULL || port->type != NXT_PROCESS_CONTROLLER)) { + nxt_alert(task, "process %PI cannot store certificates", + nxt_recv_msg_cmsg_pid(msg)); + nxt_port_recv_msg_close_fds(msg); + return; + } + + rt = task->thread->runtime; + + mem = NULL; + + if (nxt_slow_path(rt->certs.start == NULL)) { + nxt_alert(task, "no certificates storage directory"); + goto fail; + } + + if (nxt_slow_path(msg->fd[0] == -1)) { + nxt_alert(task, "cert_store_handler: invalid shm fd"); + goto fail; + } + + /* The message holds the NUL-terminated name, followed by the size. */ + + if (nxt_slow_path(nxt_cert_store_name(msg->buf, &span, &name) != NXT_OK + || nxt_span_len(&span) != sizeof(size_t))) + { + nxt_alert(task, "cert_store_handler: invalid certificate name"); + goto fail; + } + + (void) nxt_span_copy(&span, &size, sizeof(size_t)); + + if (nxt_slow_path(size == 0 || size > NXT_CERT_STORE_MAX_SIZE)) { + nxt_alert(task, "cert_store_handler: invalid bundle size %uz", size); + goto fail; + } + + /* + * Copy the bundle with pread() instead of mapping the segment: a + * sender can shrink a mapped segment, and a read past its end is + * SIGBUS in main. A short pread() is only an error. + */ + + mem = nxt_malloc(size); + if (nxt_slow_path(mem == NULL)) { + goto fail; + } + + nxt_memzero(&file, sizeof(nxt_file_t)); + file.fd = msg->fd[0]; + + /* nxt_file_read() logs the name when pread() fails; it must not be NULL. */ + file.name = name.start; + + if (nxt_slow_path(nxt_file_read(&file, mem, size, 0) != (ssize_t) size)) { + nxt_alert(task, "cert_store_handler: short bundle read"); + goto fail; + } + + job = nxt_main_store_job_create(NXT_MAIN_STORE_PUT, "certificate", + &rt->certs, &name); + if (nxt_slow_path(job == NULL)) { + goto fail; + } + + job->data = mem; + job->size = size; + job->reply_pid = port->pid; + job->reply_port = port->id; + job->stream = msg->port_msg.stream; + + nxt_port_recv_msg_close_fds(msg); + + nxt_main_store_submit(task, job); + + return; + +fail: + + (void) nxt_port_socket_write(task, port, NXT_PORT_MSG_RPC_ERROR, -1, + msg->port_msg.stream, 0, NULL); + + nxt_free(mem); + nxt_port_recv_msg_close_fds(msg); +} + + void nxt_cert_store_delete(nxt_task_t *task, nxt_str_t *name, nxt_mp_t *mp) { @@ -1277,14 +1843,20 @@ nxt_cert_store_delete(nxt_task_t *task, nxt_str_t *name, nxt_mp_t *mp) } +/* + * The store child of main deletes the bundle, after any store that waits + * before it. A delete in main could run before the rename() of an earlier + * store child of the same bundle, and that rename() would bring the bundle + * back. + */ void nxt_cert_store_delete_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) { - u_char *p; - nxt_str_t name; - nxt_port_t *ctl_port; - nxt_runtime_t *rt; - nxt_file_name_t *path; + nxt_str_t name; + nxt_port_t *ctl_port; + nxt_span_t span; + nxt_runtime_t *rt; + nxt_main_store_job_t *job; rt = task->thread->runtime; ctl_port = rt->port_by_type[NXT_PROCESS_CONTROLLER]; @@ -1307,17 +1879,17 @@ nxt_cert_store_delete_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) return; } - name.start = msg->buf->mem.pos; - name.length = nxt_strlen(name.start); - - path = nxt_malloc(rt->certs.length + name.length + 1); - - if (nxt_fast_path(path != NULL)) { - p = nxt_cpymem(path, rt->certs.start, rt->certs.length); - p = nxt_cpymem(p, name.start, name.length + 1); + if (nxt_slow_path(nxt_cert_store_name(msg->buf, &span, &name) != NXT_OK + || nxt_span_len(&span) != 0)) + { + nxt_alert(task, "cert_delete_handler: invalid certificate name"); + return; + } - (void) nxt_file_delete(path); + job = nxt_main_store_job_create(NXT_MAIN_STORE_DELETE, "certificate", + &rt->certs, &name); - nxt_free(path); + if (nxt_fast_path(job != NULL)) { + nxt_main_store_submit(task, job); } } diff --git a/src/nxt_cert.h b/src/nxt_cert.h index dbaddcf93..3615d5024 100644 --- a/src/nxt_cert.h +++ b/src/nxt_cert.h @@ -8,25 +8,45 @@ #define _NXT_CERT_INCLUDED_ -typedef struct nxt_cert_s nxt_cert_t; +typedef struct nxt_cert_s nxt_cert_t; +typedef struct nxt_cert_info_s nxt_cert_info_t; nxt_cert_t *nxt_cert_mem(nxt_task_t *task, nxt_buf_mem_t *mbuf); void nxt_cert_destroy(nxt_cert_t *cert); void nxt_cert_info_init(nxt_task_t *task, nxt_array_t *certs); nxt_int_t nxt_cert_info_save(nxt_str_t *name, nxt_cert_t *cert); +nxt_cert_info_t *nxt_cert_info_create(nxt_str_t *name, nxt_cert_t *cert); +nxt_int_t nxt_cert_info_replace(nxt_cert_info_t *info, nxt_cert_info_t **old); +void nxt_cert_info_restore(nxt_cert_info_t *info, nxt_cert_info_t *old); +void nxt_cert_info_release(nxt_cert_info_t *info); +nxt_bool_t nxt_cert_info_equal(nxt_str_t *name, nxt_cert_t *cert); +void nxt_cert_info_applied(nxt_str_t *name, nxt_bool_t applied); nxt_conf_value_t *nxt_cert_info_get(nxt_str_t *name); nxt_conf_value_t *nxt_cert_info_get_all(nxt_mp_t *mp); nxt_int_t nxt_cert_info_delete(nxt_str_t *name); +/* A stored bundle is a chain and a key. 1 MiB can hold hundreds. */ +#define NXT_CERT_STORE_MAX_SIZE (1024 * 1024) + +/* + * A bundle is the file "certs/". 255 is NAME_MAX on Linux, the BSDs, + * macOS and Solaris. The temporary file of main has a fixed name, so the + * store needs no room for a suffix. + */ +#define NXT_CERT_NAME_MAX_LENGTH 255 + nxt_array_t *nxt_cert_store_load(nxt_task_t *task, nxt_mp_t *mem_pool); void nxt_cert_store_release(nxt_array_t *certs); void nxt_cert_store_get(nxt_task_t *task, nxt_str_t *name, nxt_mp_t *mp, nxt_port_rpc_handler_t handler, void *ctx); +void nxt_cert_store_put(nxt_task_t *task, nxt_str_t *name, nxt_buf_mem_t *mbuf, + nxt_mp_t *mp, nxt_port_rpc_handler_t handler, void *ctx); void nxt_cert_store_delete(nxt_task_t *task, nxt_str_t *name, nxt_mp_t *mp); void nxt_cert_store_get_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); +void nxt_cert_store_put_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); void nxt_cert_store_delete_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); #endif /* _NXT_CERT_INCLUDED_ */ diff --git a/src/nxt_cgroup.c b/src/nxt_cgroup.c index fd9d72449..86896599a 100644 --- a/src/nxt_cgroup.c +++ b/src/nxt_cgroup.c @@ -87,7 +87,7 @@ nxt_cgroup_rmdir_up(char *cgpath, const char *cgroot) nxt_int_t -nxt_cgroup_proc_add(nxt_task_t *task, nxt_process_t *process) +nxt_cgroup_proc_add(nxt_task_t *task, nxt_process_t *process, nxt_pid_t pid) { int len; size_t old_len; @@ -103,15 +103,12 @@ nxt_cgroup_proc_add(nxt_task_t *task, nxt_process_t *process) } /* - * Resolve the cgroup path against /proc//cgroup rather than - * /proc/self/cgroup: the parent's cgroup view may differ from the - * just-forked child's, particularly when CLONE_NEWCGROUP is in play - * and the configured path is relative. Reading the child's own - * /proc entry avoids a TOCTOU where the parent moves between cgroups - * after fork() but before this write. + * Resolve the cgroup path from /proc//cgroup, not from + * /proc/self/cgroup. The parent can move to a different cgroup after + * fork() and before this write. A relative path that uses the + * parent's entry then points to the wrong cgroup. */ - ret = nxt_mk_cgpath(task, process->isolation.cgroup.path, cgprocs, - process->pid); + ret = nxt_mk_cgpath(task, process->isolation.cgroup.path, cgprocs, pid); if (nxt_slow_path(ret == NXT_ERROR)) { return NXT_ERROR; } @@ -168,7 +165,7 @@ nxt_cgroup_proc_add(nxt_task_t *task, nxt_process_t *process) } setvbuf(fp, NULL, _IONBF, 0); - len = fprintf(fp, "%d\n", process->pid); + len = fprintf(fp, "%d\n", pid); nxt_file_fclose(task, fp); if (nxt_slow_path(len < 0)) { diff --git a/src/nxt_cgroup.h b/src/nxt_cgroup.h index 9d8c98a47..3ff26a78b 100644 --- a/src/nxt_cgroup.h +++ b/src/nxt_cgroup.h @@ -8,7 +8,8 @@ nxt_int_t nxt_cgroup_make_procs_path(char *cgprocs, size_t len); -nxt_int_t nxt_cgroup_proc_add(nxt_task_t *task, nxt_process_t *process); +nxt_int_t nxt_cgroup_proc_add(nxt_task_t *task, nxt_process_t *process, + nxt_pid_t pid); void nxt_cgroup_cleanup(nxt_task_t *task, const nxt_process_t *process); diff --git a/src/nxt_checked.h b/src/nxt_checked.h new file mode 100644 index 000000000..d9a0c9c14 --- /dev/null +++ b/src/nxt_checked.h @@ -0,0 +1,79 @@ +/* + * Copyright (C) FreeUnit Community + */ + +#ifndef _NXT_CHECKED_H_INCLUDED_ +#define _NXT_CHECKED_H_INCLUDED_ + + +/* + * Checked size_t arithmetic. Each function returns 0 and stores the + * result in "out". When the operation overflows, the function returns 1. + * Then the value of "out" is not defined. Callers must not use it. + * + * With GCC and Clang, nxt_size_add() and nxt_size_mul() use the compiler's + * __builtin_*_overflow() (probed in auto/clang). Other compilers get the + * portable functions. The portable functions are always compiled, so + * that the test suite covers them under every compiler. + */ + +nxt_inline int +nxt_size_add_portable(size_t a, size_t b, size_t *out) +{ + if (b > SIZE_MAX - a) { + return 1; + } + + *out = a + b; + + return 0; +} + + +nxt_inline int +nxt_size_mul_portable(size_t a, size_t b, size_t *out) +{ + if (a != 0 && b > SIZE_MAX / a) { + return 1; + } + + *out = a * b; + + return 0; +} + + +#if (NXT_HAVE_BUILTIN_OVERFLOW) + +nxt_inline int +nxt_size_add(size_t a, size_t b, size_t *out) +{ + return __builtin_add_overflow(a, b, out); +} + + +nxt_inline int +nxt_size_mul(size_t a, size_t b, size_t *out) +{ + return __builtin_mul_overflow(a, b, out); +} + +#else + +nxt_inline int +nxt_size_add(size_t a, size_t b, size_t *out) +{ + return nxt_size_add_portable(a, b, out); +} + + +nxt_inline int +nxt_size_mul(size_t a, size_t b, size_t *out) +{ + return nxt_size_mul_portable(a, b, out); +} + +#endif + + +#endif /* _NXT_CHECKED_H_INCLUDED_ */ diff --git a/src/nxt_clang.h b/src/nxt_clang.h index 872f15271..363ce25ea 100644 --- a/src/nxt_clang.h +++ b/src/nxt_clang.h @@ -12,6 +12,48 @@ #define nxt_noinline __attribute__((noinline)) #define nxt_cdecl +/* + * An intended switch fall-through. It is a statement and not a comment. + * Thus -Wimplicit-fallthrough accepts it under both GCC and Clang. That + * warning is part of --hardening=strict, see auto/cc/hardening. A "Fall + * through." comment alone satisfies the default heuristic of GCC. It does + * not satisfy Clang. + */ +#if (NXT_HAVE_GCC_ATTRIBUTE_FALLTHROUGH) +#define nxt_fallthrough __attribute__((__fallthrough__)) +#else +#define nxt_fallthrough +#endif + + +/* + * nxt_static_assert() fails the compilation when the condition does not + * hold. auto/cc/test adds -std=gnu11 only on its gcc and clang branches + * and leaves the generic "cc" branch alone, so a compiler defaulting to + * pre-C11 has to fall back to declaring an array with a negative size; + * there the message survives only in the type name, which has to be unique + * per use, hence the token pasting. + */ + +#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) + +#define nxt_static_assert(cond, msg) \ + _Static_assert(cond, msg) + +#else + +#define nxt_static_assert_paste(prefix, line) \ + prefix##line + +#define nxt_static_assert_name(prefix, line) \ + nxt_static_assert_paste(prefix, line) + +#define nxt_static_assert(cond, msg) \ + typedef char nxt_static_assert_name(nxt_static_assert_failed_, \ + __LINE__)[(cond) ? 1 : -1] + +#endif + #if (NXT_CLANG) diff --git a/src/nxt_conf.c b/src/nxt_conf.c index ec4ed7b07..3a66462f0 100644 --- a/src/nxt_conf.c +++ b/src/nxt_conf.c @@ -607,10 +607,15 @@ nxt_conf_map_object(nxt_mp_t *mp, const nxt_conf_value_t *value, const nxt_conf_map_t *map, nxt_uint_t n, void *data) { double num; + size_t len; nxt_str_t str, *s; nxt_uint_t i; nxt_conf_value_t *v; + /* + * A mapped field can have less alignment than its type needs. + * Build the value here and copy it to the field with nxt_memcpy(). + */ union { uint8_t ui8; int32_t i32; @@ -623,7 +628,7 @@ nxt_conf_map_object(nxt_mp_t *mp, const nxt_conf_value_t *value, nxt_str_t str; char *cstrz; void *v; - } *ptr; + } val; for (i = 0; i < n; i++) { @@ -633,16 +638,16 @@ nxt_conf_map_object(nxt_mp_t *mp, const nxt_conf_value_t *value, continue; } - ptr = nxt_pointer_to(data, map[i].offset); - switch (map[i].type) { case NXT_CONF_MAP_INT8: - if (v->type == NXT_CONF_VALUE_BOOLEAN) { - ptr->ui8 = v->u.boolean; + if (v->type != NXT_CONF_VALUE_BOOLEAN) { + continue; } + val.ui8 = v->u.boolean; + len = sizeof(val.ui8); break; case NXT_CONF_MAP_INT32: @@ -653,7 +658,7 @@ nxt_conf_map_object(nxt_mp_t *mp, const nxt_conf_value_t *value, case NXT_CONF_MAP_MSEC: if (v->type != NXT_CONF_VALUE_INTEGER) { - break; + continue; } num = nxt_strtod(v->u.number, NULL); @@ -661,41 +666,50 @@ nxt_conf_map_object(nxt_mp_t *mp, const nxt_conf_value_t *value, switch (map[i].type) { case NXT_CONF_MAP_INT32: - ptr->i32 = num; + val.i32 = num; + len = sizeof(val.i32); break; case NXT_CONF_MAP_INT64: - ptr->i64 = num; + val.i64 = num; + len = sizeof(val.i64); break; case NXT_CONF_MAP_INT: - ptr->i = num; + val.i = num; + len = sizeof(val.i); break; case NXT_CONF_MAP_SIZE: - ptr->size = num; + val.size = num; + len = sizeof(val.size); break; case NXT_CONF_MAP_OFF: - ptr->off = num; + val.off = num; + len = sizeof(val.off); break; case NXT_CONF_MAP_MSEC: - ptr->msec = (nxt_msec_t) num * 1000; + val.msec = (nxt_msec_t) num * 1000; + len = sizeof(val.msec); break; default: nxt_unreachable(); + continue; } break; case NXT_CONF_MAP_DOUBLE: - if (v->type == NXT_CONF_VALUE_NUMBER) { - ptr->dbl = nxt_strtod(v->u.number, NULL); + if (v->type != NXT_CONF_VALUE_NUMBER) { + continue; } + val.dbl = nxt_strtod(v->u.number, NULL); + len = sizeof(val.dbl); break; case NXT_CONF_MAP_STR: @@ -705,7 +719,7 @@ nxt_conf_map_object(nxt_mp_t *mp, const nxt_conf_value_t *value, if (v->type != NXT_CONF_VALUE_SHORT_STRING && v->type != NXT_CONF_VALUE_STRING) { - break; + continue; } nxt_conf_get_string(v, &str); @@ -713,41 +727,49 @@ nxt_conf_map_object(nxt_mp_t *mp, const nxt_conf_value_t *value, switch (map[i].type) { case NXT_CONF_MAP_STR: - ptr->str = str; + val.str = str; + len = sizeof(val.str); break; case NXT_CONF_MAP_STR_COPY: - s = nxt_str_dup(mp, &ptr->str, &str); + s = nxt_str_dup(mp, &val.str, &str); if (nxt_slow_path(s == NULL)) { return NXT_ERROR; } + len = sizeof(val.str); break; case NXT_CONF_MAP_CSTRZ: - ptr->cstrz = nxt_str_cstrz(mp, &str); + val.cstrz = nxt_str_cstrz(mp, &str); - if (nxt_slow_path(ptr->cstrz == NULL)) { + if (nxt_slow_path(val.cstrz == NULL)) { return NXT_ERROR; } + len = sizeof(val.cstrz); break; default: nxt_unreachable(); + continue; } break; case NXT_CONF_MAP_PTR: - - ptr->v = v; - + val.v = v; + len = sizeof(val.v); break; + + default: + continue; } + + nxt_memcpy(nxt_pointer_to(data, map[i].offset), &val, len); } return NXT_OK; diff --git a/src/nxt_conf.h b/src/nxt_conf.h index 493d64801..a69ab7bad 100644 --- a/src/nxt_conf.h +++ b/src/nxt_conf.h @@ -68,6 +68,14 @@ typedef struct { } nxt_conf_json_pretty_t; +typedef struct nxt_conf_vldt_path_s nxt_conf_vldt_path_t; + +struct nxt_conf_vldt_path_s { + nxt_conf_vldt_path_t *prev; + nxt_str_t seg; +}; + + typedef struct { nxt_conf_value_t *conf; nxt_mp_t *pool; @@ -76,6 +84,9 @@ typedef struct { nxt_tstr_state_t *tstr_state; nxt_mp_t *conf_pool; nxt_uint_t ver; + nxt_conf_vldt_path_t *path; /* Stack (top = deepest segment). */ + nxt_str_t pointer; /* RFC 6901 pointer at error site. */ + nxt_str_t suggestion; /* "Did you mean X" member name. */ } nxt_conf_validation_t; @@ -117,6 +128,14 @@ void nxt_conf_json_position(u_char *start, const u_char *pos, nxt_uint_t *line, nxt_int_t nxt_conf_validate(nxt_conf_validation_t *vldt); +/* + * Checks every string in the tree -- values and member names alike -- for + * valid UTF-8. Separate from nxt_conf_validate() because + * the two callers want different answers: the control API refuses such a + * configuration, while a state file that already holds one still has to load. + */ +nxt_int_t nxt_conf_validate_encoding(nxt_conf_validation_t *vldt); + NXT_EXPORT void nxt_conf_get_string(const nxt_conf_value_t *value, nxt_str_t *str); NXT_EXPORT nxt_str_t *nxt_conf_get_string_dup(const nxt_conf_value_t *value, diff --git a/src/nxt_conf_validation.c b/src/nxt_conf_validation.c index 8334ee10e..67579d2cf 100644 --- a/src/nxt_conf_validation.c +++ b/src/nxt_conf_validation.c @@ -75,8 +75,12 @@ struct nxt_conf_vldt_object_s { static nxt_int_t nxt_conf_vldt_type(nxt_conf_validation_t *vldt, const nxt_str_t *name, nxt_conf_value_t *value, nxt_conf_vldt_type_t type); +static nxt_int_t nxt_conf_vldt_verror(nxt_conf_validation_t *vldt, + const char *fmt, va_list args); static nxt_int_t nxt_conf_vldt_error(nxt_conf_validation_t *vldt, const char *fmt, ...); +static nxt_int_t nxt_conf_vldt_member_error(nxt_conf_validation_t *vldt, + const nxt_str_t *member, const char *fmt, ...); static nxt_int_t nxt_conf_vldt_var(nxt_conf_validation_t *vldt, const nxt_str_t *name, nxt_str_t *value); static nxt_int_t nxt_conf_vldt_if(nxt_conf_validation_t *vldt, @@ -119,6 +123,8 @@ static nxt_int_t nxt_conf_vldt_pass(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); static nxt_int_t nxt_conf_vldt_return(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); +static nxt_int_t nxt_conf_vldt_index(nxt_conf_validation_t *vldt, + nxt_conf_value_t *value, void *data); static nxt_int_t nxt_conf_vldt_share(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); static nxt_int_t nxt_conf_vldt_share_element(nxt_conf_validation_t *vldt, @@ -137,6 +143,8 @@ static nxt_int_t nxt_conf_vldt_python_prefix(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); static nxt_int_t nxt_conf_vldt_listen_threads(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); +static nxt_int_t nxt_conf_vldt_min_rate(nxt_conf_validation_t *vldt, + nxt_conf_value_t *value, void *data); static nxt_int_t nxt_conf_vldt_threads(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); static nxt_int_t nxt_conf_vldt_thread_stack_size(nxt_conf_validation_t *vldt, @@ -191,6 +199,8 @@ static nxt_int_t nxt_conf_vldt_app(nxt_conf_validation_t *vldt, nxt_str_t *name, nxt_conf_value_t *value); static nxt_int_t nxt_conf_vldt_object(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); +static nxt_int_t nxt_conf_vldt_app_limits(nxt_conf_validation_t *vldt, + nxt_conf_value_t *value, void *data); static nxt_int_t nxt_conf_vldt_processes(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); static nxt_int_t nxt_conf_vldt_object_iterator(nxt_conf_validation_t *vldt, @@ -201,6 +211,8 @@ static nxt_int_t nxt_conf_vldt_environment(nxt_conf_validation_t *vldt, nxt_str_t *name, nxt_conf_value_t *value); static nxt_int_t nxt_conf_vldt_c_string(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); +static nxt_int_t nxt_conf_vldt_wasm_wc_timeout(nxt_conf_validation_t *vldt, + nxt_conf_value_t *value, void *data); static nxt_int_t nxt_conf_vldt_targets_exclusive( nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); static nxt_int_t nxt_conf_vldt_targets(nxt_conf_validation_t *vldt, @@ -253,6 +265,12 @@ static nxt_int_t nxt_conf_vldt_rootfs_path(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); #endif +#if (NXT_HAVE_ISOLATION_ROOTFS) && (NXT_HAVE_CLONE_NEWUSER) \ + && (NXT_HAVE_CLONE_NEWNS) +static nxt_int_t nxt_conf_vldt_isolation_mounts(nxt_conf_validation_t *vldt, + nxt_conf_value_t *value, nxt_app_lang_module_t *lang); +#endif + #if (NXT_HAVE_NJS) static nxt_int_t nxt_conf_vldt_js_module(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); @@ -379,6 +397,16 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_http_members[] = { }, { .name = nxt_string("send_timeout"), .type = NXT_CONF_VLDT_INTEGER, + }, { + .name = nxt_string("body_min_rate"), + .type = NXT_CONF_VLDT_INTEGER, + .validator = nxt_conf_vldt_min_rate, + .u.string = "body_min_rate", + }, { + .name = nxt_string("send_min_rate"), + .type = NXT_CONF_VLDT_INTEGER, + .validator = nxt_conf_vldt_min_rate, + .u.string = "send_min_rate", }, { .name = nxt_string("idle_timeout"), .type = NXT_CONF_VLDT_INTEGER, @@ -864,6 +892,7 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_share_action_members[] = { }, { .name = nxt_string("index"), .type = NXT_CONF_VLDT_STRING, + .validator = nxt_conf_vldt_index, }, { .name = nxt_string("types"), .type = NXT_CONF_VLDT_STRING | NXT_CONF_VLDT_ARRAY, @@ -1102,6 +1131,7 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_php_target_members[] = { }, { .name = nxt_string("index"), .type = NXT_CONF_VLDT_STRING, + .validator = nxt_conf_vldt_index, }, NXT_CONF_VLDT_END @@ -1119,6 +1149,7 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_php_notargets_members[] = { }, { .name = nxt_string("index"), .type = NXT_CONF_VLDT_STRING, + .validator = nxt_conf_vldt_index, }, NXT_CONF_VLDT_NEXT(nxt_conf_vldt_php_common_members) @@ -1273,6 +1304,10 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_wasm_wc_members[] = { .type = NXT_CONF_VLDT_OBJECT, .validator = nxt_conf_vldt_object, .u.members = nxt_conf_vldt_wasm_access_members, + }, { + .name = nxt_string("execution_timeout"), + .type = NXT_CONF_VLDT_INTEGER, + .validator = nxt_conf_vldt_wasm_wc_timeout, }, NXT_CONF_VLDT_NEXT(nxt_conf_vldt_common_members) @@ -1296,7 +1331,7 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_common_members[] = { }, { .name = nxt_string("limits"), .type = NXT_CONF_VLDT_OBJECT, - .validator = nxt_conf_vldt_object, + .validator = nxt_conf_vldt_app_limits, .u.members = nxt_conf_vldt_app_limits_members, }, { .name = nxt_string("processes"), @@ -1348,6 +1383,9 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_app_limits_members[] = { { .name = nxt_string("timeout"), .type = NXT_CONF_VLDT_INTEGER, + }, { + .name = nxt_string("start_timeout"), + .type = NXT_CONF_VLDT_INTEGER, }, { .name = nxt_string("requests"), .type = NXT_CONF_VLDT_INTEGER, @@ -1381,8 +1419,9 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_app_processes_members[] = { * lets the operator disable every isolation feature here. This is * intentional: any peer who can write to the control socket already * has the same authority as the unitd main process (see the - * SO_PEERCRED check landed in andypost/unit#14 — non-root local - * users are rejected at the socket layer, not by this validator). + * SO_PEERCRED check landed in andypost/unit#14 — local users other + * than root, unitd's user and the --control-user / --control-group + * principals are rejected at the socket layer, not by this validator). * Allow-listing executable paths or forcing isolation = true here * is a deployment policy decision, not a config-schema concern; * deployments needing that should add a wrapping admission gate @@ -1615,6 +1654,14 @@ nxt_conf_validate(nxt_conf_validation_t *vldt) ret = nxt_tstr_state_done(vldt->tstr_state, error); if (ret != NXT_OK) { ret = nxt_conf_vldt_error(vldt, "%s", error); + + /* + * Deferred tstr compilation runs once the traversal has unwound, so + * vldt->path no longer names the value that failed. An empty pointer + * here would claim the document root; report no location instead. + */ + vldt->pointer.start = NULL; + return ret; } @@ -1774,17 +1821,219 @@ nxt_conf_vldt_type(nxt_conf_validation_t *vldt, const nxt_str_t *name, } +/* + * Serialize vldt->path into vldt->pointer as an RFC 6901 JSON Pointer. + * Root is the empty string "". Segments are separated by '/'. Within each + * segment, '~' is encoded as "~0" and '/' as "~1" (in that order). + * + * Never fails validation: on allocation failure the pointer is left empty. + */ +/* + * A configuration string is JSON text, and RFC 8259 Sect. 8.1 defines JSON + * text as UTF-8. The parser does not enforce it: nxt_conf_json_parse_string() + * copies every byte above 0x1F through untouched, so a PUT can store bytes that + * no JSON parser reads back -- including the body of GET /config, which is + * served as "application/json", a media type that carries no charset parameter. + * + * This walks the parsed tree rather than the schema. A configuration option + * added later cannot escape the check by forgetting to ask for it, which is the + * property that matters here: the zero-byte guards this fork already carries + * were written one sink at a time, and the sink nobody reported kept its hole. + * + * A zero byte is deliberately NOT rejected. It is valid UTF-8, and "location" + * accepts one today and percent-encodes it (test/test_return.py:137), so + * refusing it here would break a documented behaviour to fix a different + * problem. The path sinks guard themselves; see nxt_http_static.c. + */ + static nxt_int_t -nxt_conf_vldt_error(nxt_conf_validation_t *vldt, const char *fmt, ...) +nxt_conf_vldt_encoding_str(nxt_conf_validation_t *vldt, const nxt_str_t *str, + const char *what) +{ + if (nxt_slow_path(!nxt_utf8_is_valid(str->start, str->length))) { + return nxt_conf_vldt_error(vldt, "The %s is not valid UTF-8. JSON " + "text is UTF-8 (RFC 8259 Sect. 8.1), so " + "these bytes have no JSON representation " + "and could not be read back.", what); + } + + return NXT_OK; +} + + +static nxt_int_t +nxt_conf_vldt_encoding_value(nxt_conf_validation_t *vldt, + nxt_conf_value_t *value) +{ + u_char *p; + uint32_t index, next, count; + nxt_str_t name, str; + nxt_int_t ret; + nxt_conf_value_t *member; + nxt_conf_vldt_path_t seg; + + u_char buf[sizeof("4294967295") - 1]; + + switch (nxt_conf_type(value)) { + + case NXT_CONF_STRING: + nxt_conf_get_string(value, &str); + + return nxt_conf_vldt_encoding_str(vldt, &str, "value"); + + case NXT_CONF_ARRAY: + count = nxt_conf_array_elements_count(value); + + for (index = 0; index < count; index++) { + p = nxt_sprintf(buf, buf + sizeof(buf), "%uD", index); + + seg.prev = vldt->path; + seg.seg.start = buf; + seg.seg.length = p - buf; + vldt->path = &seg; + + ret = nxt_conf_vldt_encoding_value(vldt, + nxt_conf_get_array_element(value, index)); + + vldt->path = seg.prev; + + if (nxt_slow_path(ret != NXT_OK)) { + return ret; + } + } + + return NXT_OK; + + case NXT_CONF_OBJECT: + next = 0; + + for ( ;; ) { + member = nxt_conf_next_object_member(value, &name, &next); + + if (member == NULL) { + break; + } + + /* + * Check the name before it becomes a path segment. The pointer + * rendered for an error is echoed in the response body, so putting + * the offending bytes there would make the error report itself + * unreadable for the same reason it is being reported. A bad name + * is reported against the object that holds it. + */ + + ret = nxt_conf_vldt_encoding_str(vldt, &name, "member name"); + + if (nxt_slow_path(ret != NXT_OK)) { + return ret; + } + + seg.prev = vldt->path; + seg.seg = name; + vldt->path = &seg; + + ret = nxt_conf_vldt_encoding_value(vldt, member); + + vldt->path = seg.prev; + + if (nxt_slow_path(ret != NXT_OK)) { + return ret; + } + } + + return NXT_OK; + + default: + return NXT_OK; + } +} + + +nxt_int_t +nxt_conf_validate_encoding(nxt_conf_validation_t *vldt) +{ + return nxt_conf_vldt_encoding_value(vldt, vldt->conf); +} + + +static void +nxt_conf_vldt_render_pointer(nxt_conf_validation_t *vldt) +{ + u_char *p; + size_t size, n; + u_char c; + nxt_conf_vldt_path_t *node; + + static u_char empty[1] = ""; + + /* + * Always produce a non-NULL pointer: the empty string "" is the + * RFC 6901 pointer to the document root. Callers distinguish + * "validation error, pointer available" from "no pointer" via + * start != NULL. + */ + vldt->pointer.length = 0; + vldt->pointer.start = empty; + + if (vldt->path == NULL) { + return; + } + + size = 0; + for (node = vldt->path; node != NULL; node = node->prev) { + if (nxt_slow_path(size == NXT_SIZE_T_MAX)) { + vldt->pointer.start = NULL; + return; + } + size += 1; /* leading '/' */ + for (n = 0; n < node->seg.length; n++) { + c = node->seg.start[n]; + if (nxt_slow_path(size >= NXT_SIZE_T_MAX - 1)) { + vldt->pointer.start = NULL; + return; + } + size += (c == '~' || c == '/') ? 2 : 1; + } + } + + p = nxt_mp_nget(vldt->pool, size); + if (p == NULL) { + vldt->pointer.start = NULL; + return; + } + + vldt->pointer.start = p; + vldt->pointer.length = size; + + p += size; + + for (node = vldt->path; node != NULL; node = node->prev) { + for (n = node->seg.length; n > 0; n--) { + c = node->seg.start[n - 1]; + if (c == '~') { + *--p = '0'; + *--p = '~'; + } else if (c == '/') { + *--p = '1'; + *--p = '~'; + } else { + *--p = c; + } + } + *--p = '/'; + } +} + + +static nxt_int_t +nxt_conf_vldt_verror(nxt_conf_validation_t *vldt, const char *fmt, + va_list args) { u_char *p, *end; size_t size; - va_list args; u_char error[NXT_MAX_ERROR_STR]; - va_start(args, fmt); end = nxt_vsprintf(error, error + NXT_MAX_ERROR_STR, fmt, args); - va_end(args); size = end - error; @@ -1798,10 +2047,187 @@ nxt_conf_vldt_error(nxt_conf_validation_t *vldt, const char *fmt, ...) vldt->error.length = size; vldt->error.start = p; + nxt_conf_vldt_render_pointer(vldt); + return NXT_DECLINED; } +static nxt_int_t +nxt_conf_vldt_error(nxt_conf_validation_t *vldt, const char *fmt, ...) +{ + va_list args; + nxt_int_t ret; + + va_start(args, fmt); + ret = nxt_conf_vldt_verror(vldt, fmt, args); + va_end(args); + + return ret; +} + + +/* + * Report an error against a member of the object being validated, for the + * semantic checks that run once nxt_conf_vldt_object() has finished the + * traversal and already popped that member's path segment. Without this the + * location would name the containing object instead of the value that failed. + */ + +static nxt_int_t +nxt_conf_vldt_member_error(nxt_conf_validation_t *vldt, + const nxt_str_t *member, const char *fmt, ...) +{ + va_list args; + nxt_int_t ret; + nxt_conf_vldt_path_t seg; + + seg.prev = vldt->path; + seg.seg = *member; + vldt->path = &seg; + + va_start(args, fmt); + ret = nxt_conf_vldt_verror(vldt, fmt, args); + va_end(args); + + vldt->path = seg.prev; + + return ret; +} + + +/* + * Damerau-Levenshtein distance between two byte strings. Rejects inputs + * that would overflow the stack buffer by returning SIZE_MAX. + * + * Used only inside validation error paths; input lengths are bounded by + * configuration member-name lengths. + */ +#define NXT_CONF_VLDT_EDIT_MAX 64 + +static size_t +nxt_conf_vldt_edit_distance(const nxt_str_t *a, const nxt_str_t *b) +{ + size_t i, j, cost, ins, del, sub, tr, min; + size_t prev2[NXT_CONF_VLDT_EDIT_MAX + 1]; + size_t prev[NXT_CONF_VLDT_EDIT_MAX + 1]; + size_t curr[NXT_CONF_VLDT_EDIT_MAX + 1]; + + if (a->length > NXT_CONF_VLDT_EDIT_MAX + || b->length > NXT_CONF_VLDT_EDIT_MAX) + { + return (size_t) -1; + } + + for (j = 0; j <= b->length; j++) { + prev[j] = j; + prev2[j] = 0; + } + + for (i = 1; i <= a->length; i++) { + curr[0] = i; + + for (j = 1; j <= b->length; j++) { + cost = (a->start[i - 1] == b->start[j - 1]) ? 0 : 1; + + ins = curr[j - 1] + 1; /* left neighbour */ + del = prev[j] + 1; /* upper neighbour */ + sub = prev[j - 1] + cost; /* diagonal */ + + min = ins < del ? ins : del; + if (sub < min) { + min = sub; + } + + if (i > 1 && j > 1 + && a->start[i - 1] == b->start[j - 2] + && a->start[i - 2] == b->start[j - 1]) + { + tr = prev2[j - 2] + 1; + if (tr < min) { + min = tr; + } + } + + curr[j] = min; + } + + for (j = 0; j <= b->length; j++) { + prev2[j] = prev[j]; + prev[j] = curr[j]; + } + } + + return prev[b->length]; +} + + +/* + * Emit "Unknown parameter" error for a member not present in the schema + * table. If a sufficiently-close, uniquely-best candidate exists, store + * its name in vldt->suggestion so the response body can surface it. + * + * Does NOT modify the legacy error-message wording. + */ +static nxt_int_t +nxt_conf_vldt_unknown_member(nxt_conf_validation_t *vldt, + const nxt_str_t *name, nxt_conf_vldt_object_t *vals) +{ + size_t d, best, second; + size_t threshold, shorter; + nxt_conf_vldt_object_t *v, *best_v; + + best = (size_t) -1; + second = (size_t) -1; + best_v = NULL; + + v = vals; + + for ( ;; ) { + if (v->name.length == 0) { + if (v->u.members != NULL) { + v = v->u.members; + continue; + } + break; + } + + d = nxt_conf_vldt_edit_distance(name, &v->name); + if (d < best) { + second = best; + best = d; + best_v = v; + } else if (d < second) { + second = d; + } + + v++; + } + + threshold = nxt_max((size_t) 2, name->length / 3); + + /* + * Require the edit to cover at most half of the shorter name as well as + * to stay under the threshold. The threshold alone is too permissive for + * short names: at two bytes it admits a distance of 2, which is every + * two-letter member in the schema, so "zz" would "did you mean" its way to + * an unrelated name it shares no characters with. + * + * best < second keeps the match unambiguous. + */ + + if (best_v != NULL && best <= threshold && best < second) { + shorter = nxt_min(name->length, best_v->name.length); + + if (best * 2 <= shorter) { + vldt->suggestion = best_v->name; + } + } + + return nxt_conf_vldt_error(vldt, "Unknown parameter \"%V\".", name); +} + + nxt_inline nxt_int_t nxt_conf_vldt_unsupported(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data) @@ -2170,6 +2596,39 @@ nxt_conf_vldt_return(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, } +/* + * "index" is appended to a "share" directory and the result is handed to + * open() as a NUL-terminated C string (nxt_http_static.c, where the name is + * assembled). An embedded NUL truncates the name there, so "a\u0000b" opens + * "a". Unlike "share" this value is not a template, so it can be refused once + * at configuration time instead of on every request. + * + * The neighbouring "share" has the same guard applied per request, and + * "component" and the environment names and values are refused the same way; + * this is the sink that was missed. + */ + +static nxt_int_t +nxt_conf_vldt_index(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, + void *data) +{ + nxt_str_t str; + + nxt_conf_get_string(value, &str); + + /* memchr() on a null pointer is undefined even for a zero length. */ + + if (str.length != 0 + && nxt_slow_path(memchr(str.start, '\0', str.length) != NULL)) + { + return nxt_conf_vldt_error(vldt, "The \"index\" must not contain " + "null character."); + } + + return NXT_OK; +} + + static nxt_int_t nxt_conf_vldt_share(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data) @@ -2350,6 +2809,36 @@ nxt_conf_vldt_listen_threads(nxt_conf_validation_t *vldt, } +/* + * The rate is in bytes per second. The router compares it with the + * elapsed time in milliseconds as a 64-bit product. The upper limit + * keeps this product in range. + */ + +static nxt_int_t +nxt_conf_vldt_min_rate(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, + void *data) +{ + int64_t rate; + const char *name; + + name = data; + rate = nxt_conf_get_number(value); + + if (rate < 0) { + return nxt_conf_vldt_error(vldt, "The \"%s\" number must be " + "equal to or greater than 0.", name); + } + + if (rate > NXT_INT32_T_MAX) { + return nxt_conf_vldt_error(vldt, "The \"%s\" number must " + "not exceed %d.", name, NXT_INT32_T_MAX); + } + + return NXT_OK; +} + + static nxt_int_t nxt_conf_vldt_threads(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data) @@ -2408,8 +2897,7 @@ nxt_conf_vldt_compressors(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, /* NXT_CONF_OBJECT */ - return nxt_conf_vldt_object_iterator(vldt, value, - &nxt_conf_vldt_compressor_members); + return nxt_conf_vldt_object(vldt, value, nxt_conf_vldt_compressor_members); } @@ -2883,10 +3371,11 @@ static nxt_int_t nxt_conf_vldt_object_conf_commands(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data) { - uint32_t index; - nxt_int_t ret; - nxt_str_t name; - nxt_conf_value_t *member; + uint32_t index; + nxt_int_t ret; + nxt_str_t name; + nxt_conf_value_t *member; + nxt_conf_vldt_path_t seg; index = 0; @@ -2897,7 +3386,21 @@ nxt_conf_vldt_object_conf_commands(nxt_conf_validation_t *vldt, break; } + /* + * This iterates the object itself rather than going through + * nxt_conf_vldt_object_iterator(), so the command name has to be + * pushed here; without it the error below would point at + * conf_commands rather than at the command that failed. + */ + + seg.prev = vldt->path; + seg.seg = name; + vldt->path = &seg; + ret = nxt_conf_vldt_type(vldt, &name, member, NXT_CONF_VLDT_STRING); + + vldt->path = seg.prev; + if (ret != NXT_OK) { return ret; } @@ -3097,8 +3600,8 @@ nxt_conf_vldt_listen_backlog(nxt_conf_validation_t *vldt, static nxt_int_t -nxt_conf_vldt_app(nxt_conf_validation_t *vldt, nxt_str_t *name, - nxt_conf_value_t *value) +nxt_conf_vldt_app_type(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, + nxt_app_lang_module_t **langp) { nxt_int_t ret; nxt_str_t type; @@ -3108,6 +3611,47 @@ nxt_conf_vldt_app(nxt_conf_validation_t *vldt, nxt_str_t *name, static const nxt_str_t type_str = nxt_string("type"); + type_value = nxt_conf_get_object_member(value, &type_str, NULL); + + if (type_value == NULL) { + return nxt_conf_vldt_error(vldt, + "Application must have the \"type\" property set."); + } + + ret = nxt_conf_vldt_type(vldt, &type_str, type_value, NXT_CONF_VLDT_STRING); + + if (ret != NXT_OK) { + return ret; + } + + nxt_conf_get_string(type_value, &type); + + thread = nxt_thread(); + + lang = nxt_app_lang_module(thread->runtime, &type); + if (lang == NULL) { + return nxt_conf_vldt_error(vldt, + "The module to run \"%V\" is not found " + "among the available application modules.", + &type); + } + + *langp = lang; + + return NXT_OK; +} + + +static nxt_int_t +nxt_conf_vldt_app(nxt_conf_validation_t *vldt, nxt_str_t *name, + nxt_conf_value_t *value) +{ + nxt_int_t ret; + nxt_conf_vldt_path_t seg; + nxt_app_lang_module_t *lang; + + static const nxt_str_t type_str = nxt_string("type"); + static const struct { nxt_conf_vldt_handler_t validator; nxt_conf_vldt_object_t *members; @@ -3123,38 +3667,44 @@ nxt_conf_vldt_app(nxt_conf_validation_t *vldt, nxt_str_t *name, { nxt_conf_vldt_object, nxt_conf_vldt_wasm_wc_members }, }; + lang = NULL; + ret = nxt_conf_vldt_type(vldt, name, value, NXT_CONF_VLDT_OBJECT); if (ret != NXT_OK) { return ret; } - type_value = nxt_conf_get_object_member(value, &type_str, NULL); + /* + * The three checks below all concern the "type" member, so they report + * against it rather than against the application object that contains + * it. The nested validator that follows must run with the path back at + * the application, so the segment is popped before it. + */ - if (type_value == NULL) { - return nxt_conf_vldt_error(vldt, - "Application must have the \"type\" property set."); - } + seg.prev = vldt->path; + seg.seg = type_str; + vldt->path = &seg; - ret = nxt_conf_vldt_type(vldt, &type_str, type_value, NXT_CONF_VLDT_STRING); + ret = nxt_conf_vldt_app_type(vldt, value, &lang); + + vldt->path = seg.prev; if (ret != NXT_OK) { return ret; } - nxt_conf_get_string(type_value, &type); - - thread = nxt_thread(); - - lang = nxt_app_lang_module(thread->runtime, &type); - if (lang == NULL) { - return nxt_conf_vldt_error(vldt, - "The module to run \"%V\" is not found " - "among the available application modules.", - &type); + ret = types[lang->type].validator(vldt, value, types[lang->type].members); + if (ret != NXT_OK) { + return ret; } - return types[lang->type].validator(vldt, value, types[lang->type].members); +#if (NXT_HAVE_ISOLATION_ROOTFS) && (NXT_HAVE_CLONE_NEWUSER) \ + && (NXT_HAVE_CLONE_NEWNS) + return nxt_conf_vldt_isolation_mounts(vldt, value, lang); +#else + return NXT_OK; +#endif } @@ -3167,6 +3717,7 @@ nxt_conf_vldt_object(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, nxt_str_t name, var; nxt_conf_value_t *member; nxt_conf_vldt_object_t *vals; + nxt_conf_vldt_path_t seg; vals = data; @@ -3185,8 +3736,16 @@ nxt_conf_vldt_object(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, member = nxt_conf_get_object_member(value, &vals->name, NULL); if (member == NULL) { - return nxt_conf_vldt_error(vldt, "Required parameter \"%V\" " - "is missing.", &vals->name); + seg.prev = vldt->path; + seg.seg = vals->name; + vldt->path = &seg; + + ret = nxt_conf_vldt_error(vldt, "Required parameter \"%V\" " + "is missing.", &vals->name); + + vldt->path = seg.prev; + + return ret; } } @@ -3212,8 +3771,7 @@ nxt_conf_vldt_object(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, continue; } - return nxt_conf_vldt_error(vldt, "Unknown parameter \"%V\".", - &name); + return nxt_conf_vldt_unknown_member(vldt, &name, data); } if (!nxt_strstr_eq(&vals->name, &name)) { @@ -3227,7 +3785,14 @@ nxt_conf_vldt_object(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, nxt_conf_get_string(member, &var); if (nxt_is_tstr(&var)) { + seg.prev = vldt->path; + seg.seg = name; + vldt->path = &seg; + ret = nxt_conf_vldt_var(vldt, &name, &var); + + vldt->path = seg.prev; + if (ret != NXT_OK) { return ret; } @@ -3236,8 +3801,13 @@ nxt_conf_vldt_object(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, } } + seg.prev = vldt->path; + seg.seg = name; + vldt->path = &seg; + ret = nxt_conf_vldt_type(vldt, &name, member, vals->type); if (ret != NXT_OK) { + vldt->path = seg.prev; return ret; } @@ -3245,16 +3815,84 @@ nxt_conf_vldt_object(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, ret = vals->validator(vldt, member, vals->u.members); if (ret != NXT_OK) { + vldt->path = seg.prev; return ret; } } + vldt->path = seg.prev; break; } } } +typedef struct { + int64_t start_timeout; +} nxt_conf_vldt_app_limits_conf_t; + + +static nxt_conf_map_t nxt_conf_vldt_app_limits_conf_map[] = { + { + nxt_string("start_timeout"), + NXT_CONF_MAP_INT64, + offsetof(nxt_conf_vldt_app_limits_conf_t, start_timeout), + }, +}; + + +/* + * "start_timeout" reaches the router through NXT_CONF_MAP_MSEC + * (nxt_router_app_limits_conf[], src/nxt_router.c:1733), which computes + * (nxt_msec_t) seconds * 1000 and range-checks nothing. Seconds above + * NXT_INT32_T_MAX / 1000 therefore land past the sign bit of the 32-bit + * millisecond clock, where nxt_msec_diff() (src/nxt_time.h:103) reads the + * deadline as already past, so the bound fires at once instead of far in the + * future; a negative value is an out-of-range conversion to an unsigned type + * before it even gets that far. Bound it here, exactly as "idle_timeout" is + * bounded in nxt_conf_vldt_processes() below. + */ + +static nxt_int_t +nxt_conf_vldt_app_limits(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, + void *data) +{ + nxt_int_t ret; + nxt_conf_vldt_app_limits_conf_t limits; + + static const nxt_str_t start_timeout_str = nxt_string("start_timeout"); + + ret = nxt_conf_vldt_object(vldt, value, data); + if (ret != NXT_OK) { + return ret; + } + + limits.start_timeout = 0; + + ret = nxt_conf_map_object(vldt->pool, value, + nxt_conf_vldt_app_limits_conf_map, + nxt_nitems(nxt_conf_vldt_app_limits_conf_map), + &limits); + if (ret != NXT_OK) { + return ret; + } + + if (limits.start_timeout < 0) { + return nxt_conf_vldt_member_error(vldt, &start_timeout_str, + "The \"start_timeout\" number must not " + "be negative."); + } + + if (limits.start_timeout > NXT_INT32_T_MAX / 1000) { + return nxt_conf_vldt_member_error(vldt, &start_timeout_str, + "The \"start_timeout\" number must not " + "exceed %d.", NXT_INT32_T_MAX / 1000); + } + + return NXT_OK; +} + + typedef struct { int64_t spare; int64_t max; @@ -3291,6 +3929,10 @@ nxt_conf_vldt_processes(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, nxt_int_t ret; nxt_conf_vldt_processes_conf_t proc; + static const nxt_str_t spare_str = nxt_string("spare"); + static const nxt_str_t max_str = nxt_string("max"); + static const nxt_str_t idle_timeout_str = nxt_string("idle_timeout"); + if (nxt_conf_type(value) == NXT_CONF_INTEGER) { int_value = nxt_conf_get_number(value); @@ -3325,37 +3967,44 @@ nxt_conf_vldt_processes(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, } if (proc.spare < 0) { - return nxt_conf_vldt_error(vldt, "The \"spare\" number must not be " + return nxt_conf_vldt_member_error(vldt, &spare_str, + "The \"spare\" number must not be " "negative."); } if (proc.spare > NXT_INT32_T_MAX) { - return nxt_conf_vldt_error(vldt, "The \"spare\" number must not " + return nxt_conf_vldt_member_error(vldt, &spare_str, + "The \"spare\" number must not " "exceed %d.", NXT_INT32_T_MAX); } if (proc.max < 1) { - return nxt_conf_vldt_error(vldt, "The \"max\" number must be equal " + return nxt_conf_vldt_member_error(vldt, &max_str, + "The \"max\" number must be equal " "to or greater than 1."); } if (proc.max > NXT_INT32_T_MAX) { - return nxt_conf_vldt_error(vldt, "The \"max\" number must not " + return nxt_conf_vldt_member_error(vldt, &max_str, + "The \"max\" number must not " "exceed %d.", NXT_INT32_T_MAX); } if (proc.max < proc.spare) { - return nxt_conf_vldt_error(vldt, "The \"spare\" number must be " + return nxt_conf_vldt_member_error(vldt, &spare_str, + "The \"spare\" number must be " "less than or equal to \"max\"."); } if (proc.idle_timeout < 0) { - return nxt_conf_vldt_error(vldt, "The \"idle_timeout\" number must not " + return nxt_conf_vldt_member_error(vldt, &idle_timeout_str, + "The \"idle_timeout\" number must not " "be negative."); } if (proc.idle_timeout > NXT_INT32_T_MAX / 1000) { - return nxt_conf_vldt_error(vldt, "The \"idle_timeout\" number must not " + return nxt_conf_vldt_member_error(vldt, &idle_timeout_str, + "The \"idle_timeout\" number must not " "exceed %d.", NXT_INT32_T_MAX / 1000); } @@ -3371,6 +4020,7 @@ nxt_conf_vldt_object_iterator(nxt_conf_validation_t *vldt, nxt_int_t ret; nxt_str_t name; nxt_conf_value_t *member; + nxt_conf_vldt_path_t seg; nxt_conf_vldt_member_t validator; validator = (nxt_conf_vldt_member_t) data; @@ -3382,7 +4032,14 @@ nxt_conf_vldt_object_iterator(nxt_conf_validation_t *vldt, return NXT_OK; } + seg.prev = vldt->path; + seg.seg = name; + vldt->path = &seg; + ret = validator(vldt, &name, member); + + vldt->path = seg.prev; + if (ret != NXT_OK) { return ret; } @@ -3394,10 +4051,13 @@ static nxt_int_t nxt_conf_vldt_array_iterator(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data) { + u_char *p; uint32_t index; nxt_int_t ret; nxt_conf_value_t *element; + nxt_conf_vldt_path_t seg; nxt_conf_vldt_element_t validator; + u_char buf[NXT_INT_T_LEN]; validator = (nxt_conf_vldt_element_t) data; @@ -3408,8 +4068,17 @@ nxt_conf_vldt_array_iterator(nxt_conf_validation_t *vldt, return NXT_OK; } + p = nxt_sprintf(buf, buf + sizeof(buf), "%uD", index); + + seg.prev = vldt->path; + seg.seg.start = buf; + seg.seg.length = p - buf; + vldt->path = &seg; + ret = validator(vldt, element); + vldt->path = seg.prev; + if (ret != NXT_OK) { return ret; } @@ -3454,6 +4123,40 @@ nxt_conf_vldt_environment(nxt_conf_validation_t *vldt, nxt_str_t *name, } +/* + * "execution_timeout" bounds one invocation of the component's "handle". + * It reaches the module through NXT_CONF_MAP_MSEC (nxt_wasm_wc_app_conf[], + * src/nxt_main_process.c), which computes (nxt_msec_t) seconds * 1000 and + * range-checks nothing, so bound it here exactly as "start_timeout" is + * bounded in nxt_conf_vldt_app_limits() above: a negative value is an + * out-of-range conversion to an unsigned type, and seconds above + * NXT_INT32_T_MAX / 1000 land past the sign bit of the 32-bit millisecond + * value. 0 means unbounded. + */ + +static nxt_int_t +nxt_conf_vldt_wasm_wc_timeout(nxt_conf_validation_t *vldt, + nxt_conf_value_t *value, void *data) +{ + int64_t timeout; + + timeout = nxt_conf_get_number(value); + + if (timeout < 0) { + return nxt_conf_vldt_error(vldt, "The \"execution_timeout\" number " + "must not be negative."); + } + + if (timeout > NXT_INT32_T_MAX / 1000) { + return nxt_conf_vldt_error(vldt, "The \"execution_timeout\" number " + "must not exceed %d.", + NXT_INT32_T_MAX / 1000); + } + + return NXT_OK; +} + + /* * Reject empty and embedded-NUL values for options that are later consumed * as NUL-terminated C strings (NXT_CONF_MAP_CSTRZ or a direct sink): @@ -3683,6 +4386,179 @@ nxt_conf_vldt_isolation(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, } +#if (NXT_HAVE_ISOLATION_ROOTFS) && (NXT_HAVE_CLONE_NEWUSER) \ + && (NXT_HAVE_CLONE_NEWNS) + +static nxt_bool_t +nxt_conf_vldt_boolean_member(nxt_conf_value_t *object, const nxt_str_t *name, + nxt_bool_t dflt) +{ + nxt_conf_value_t *member; + + if (object == NULL) { + return dflt; + } + + member = nxt_conf_get_object_member(object, name, NULL); + + if (member == NULL || nxt_conf_type(member) != NXT_CONF_BOOLEAN) { + return dflt; + } + + return nxt_conf_get_boolean(member); +} + + +/* + * A prototype that enters a new user namespace (CLONE_NEWUSER, from + * "namespaces": {"credential": true}) without a new mount namespace + * (CLONE_NEWNS, from "namespaces": {"mount": true}) stays in the parent's + * mount namespace, which is owned by the initial user namespace. mount(2) + * there requires CAP_SYS_ADMIN in *that* namespace, which the child does not + * have, so the automount loop in nxt_isolation_prepare_rootfs() + * (src/nxt_isolation.c:926) fails with EPERM at src/nxt_fs_mount.c:93 and the + * prototype dies before it can serve -- with only "Failed to apply new + * configuration." on the control API. This is a kernel invariant, so refuse + * the combination here instead. + * + * The rootfs switch itself does not need CLONE_NEWNS: + * nxt_isolation_change_root() (src/nxt_isolation.c:1058) falls back to + * chroot(2), which needs only CAP_SYS_CHROOT in the new user namespace. So + * refuse only the configurations that have a mount to perform: the "tmpfs" + * and "procfs" automounts (unconditional builtins, added in + * nxt_isolation_set_lang_mounts(), src/nxt_isolation.c:703 and :729), and the + * "language_deps" automount when the module declares language dependency + * mounts (every lang mount carries deps = 1, src/nxt_main_process.c:1768, and + * is skipped when "language_deps" is false, src/nxt_isolation.c:926). A + * config that disables every applicable automount does start and serve, and + * stays valid. + * + * The "language_deps" arm asks what the module declares, not what the runtime + * would end up mounting: the same loop also skips a bind mount whose source is + * missing on the host (src/nxt_isolation.c:930), so a module whose every + * declared dependency happens to be absent -- copied into the rootfs already, + * say -- would in fact have started. Answering that here would mean stat()ing + * host paths from the validator, which runs in the controller as the unitd + * user while the prototype mounts as root, so it would be wrong in the other + * direction just as easily. Declared-but-absent is therefore the line: it + * needs no filesystem access, it does not change under the app's feet, and the + * error names the automount to switch off. + */ +static nxt_int_t +nxt_conf_vldt_isolation_mounts(nxt_conf_validation_t *vldt, + nxt_conf_value_t *value, nxt_app_lang_module_t *lang) +{ + u_char *p, *q; + u_char names_buf[64], disable_buf[96]; + nxt_str_t names, disable; + nxt_uint_t i, n; + nxt_conf_value_t *isolation, *namespaces, *automount; + const nxt_str_t *enabled[3]; + + static const nxt_str_t isolation_str = nxt_string("isolation"); + static const nxt_str_t namespaces_str = nxt_string("namespaces"); + static const nxt_str_t automount_str = nxt_string("automount"); + static const nxt_str_t rootfs_str = nxt_string("rootfs"); + static const nxt_str_t credential_str = nxt_string("credential"); + static const nxt_str_t mount_str = nxt_string("mount"); + static const nxt_str_t tmpfs_str = nxt_string("tmpfs"); + static const nxt_str_t procfs_str = nxt_string("procfs"); + static const nxt_str_t langdeps_str = nxt_string("language_deps"); + + isolation = nxt_conf_get_object_member(value, &isolation_str, + NULL); + if (isolation == NULL + || nxt_conf_type(isolation) != NXT_CONF_OBJECT + || nxt_conf_get_object_member(isolation, &rootfs_str, + NULL) == NULL) + { + return NXT_OK; + } + + namespaces = nxt_conf_get_object_member(isolation, + &namespaces_str, + NULL); + if (namespaces == NULL || nxt_conf_type(namespaces) != NXT_CONF_OBJECT) { + return NXT_OK; + } + + if (!nxt_conf_vldt_boolean_member(namespaces, &credential_str, 0) + || nxt_conf_vldt_boolean_member(namespaces, &mount_str, 0)) + { + return NXT_OK; + } + + automount = nxt_conf_get_object_member(isolation, + &automount_str, NULL); + if (automount != NULL && nxt_conf_type(automount) != NXT_CONF_OBJECT) { + automount = NULL; + } + + /* + * Collect every automount that would be attempted, so that one error + * names them all and the operator can turn them off in a single edit. + */ + + n = 0; + + if (nxt_conf_vldt_boolean_member(automount, &procfs_str, 1)) { + enabled[n++] = &procfs_str; + } + + if (nxt_conf_vldt_boolean_member(automount, &tmpfs_str, 1)) { + enabled[n++] = &tmpfs_str; + } + + if (nxt_conf_vldt_boolean_member(automount, &langdeps_str, 1) + && lang->mounts != NULL && lang->mounts->nelts > 0) + { + enabled[n++] = &langdeps_str; + } + + if (n == 0) { + return NXT_OK; + } + + p = names_buf; + q = disable_buf; + + for (i = 0; i < n; i++) { + p = nxt_sprintf(p, names_buf + sizeof(names_buf), "%s\"%V\"", + (i == 0) ? (u_char *) "" : + (i + 1 == n) ? (u_char *) " and " : (u_char *) ", ", + enabled[i]); + + q = nxt_sprintf(q, disable_buf + sizeof(disable_buf), + "%s\"%V\": false", + (i == 0) ? (u_char *) "" : (u_char *) ", ", + enabled[i]); + } + + names.start = names_buf; + names.length = p - names_buf; + + disable.start = disable_buf; + disable.length = q - disable_buf; + + return nxt_conf_vldt_error(vldt, + "The \"isolation\" object sets \"rootfs\" with " + "\"namespaces\": {\"credential\": true} but " + "without \"namespaces\": {\"mount\": true}; a " + "process in a new user namespace but in the " + "parent mount namespace cannot mount the %V " + "%s (EPERM), so the application could never " + "start. Set \"namespaces\": {\"mount\": true}, " + "drop \"namespaces\": {\"credential\": true}, " + "or set \"automount\": {%V}.", + &names, + (n == 1) ? (u_char *) "automount" + : (u_char *) "automounts", + &disable); +} + +#endif + + #if (NXT_HAVE_CLONE_NEWUSER) static nxt_int_t @@ -3832,8 +4708,9 @@ static nxt_int_t nxt_conf_vldt_upstream(nxt_conf_validation_t *vldt, nxt_str_t *name, nxt_conf_value_t *value) { - nxt_int_t ret; - nxt_conf_value_t *conf; + nxt_int_t ret; + nxt_conf_value_t *conf; + nxt_conf_vldt_path_t seg; static const nxt_str_t servers = nxt_string("servers"); @@ -3851,8 +4728,19 @@ nxt_conf_vldt_upstream(nxt_conf_validation_t *vldt, nxt_str_t *name, conf = nxt_conf_get_object_member(value, &servers, NULL); if (conf == NULL) { - return nxt_conf_vldt_error(vldt, "The \"%V\" upstream must contain " - "\"servers\" object value.", name); + /* Same convention as the NXT_CONF_VLDT_REQUIRED check: a missing + * member is reported against the member, not its container. */ + + seg.prev = vldt->path; + seg.seg = servers; + vldt->path = &seg; + + ret = nxt_conf_vldt_error(vldt, "The \"%V\" upstream must contain " + "\"servers\" object value.", name); + + vldt->path = seg.prev; + + return ret; } return NXT_OK; @@ -3989,9 +4877,11 @@ nxt_conf_vldt_access_log(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data) { nxt_int_t ret; + nxt_conf_vldt_path_t seg; nxt_conf_vldt_access_log_conf_t conf; static const nxt_str_t format_str = nxt_string("format"); + static const nxt_str_t path_str = nxt_string("path"); if (nxt_conf_type(value) == NXT_CONF_STRING) { return nxt_conf_vldt_c_string(vldt, value, (void *) "access_log"); @@ -4013,19 +4903,28 @@ nxt_conf_vldt_access_log(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, } if (conf.path.length == 0) { - return nxt_conf_vldt_error(vldt, + return nxt_conf_vldt_member_error(vldt, &path_str, "The \"path\" string must not be empty."); } /* The log path is opened as a NUL-terminated C string. */ if (memchr(conf.path.start, '\0', conf.path.length) != NULL) { - return nxt_conf_vldt_error(vldt, "The \"path\" value must not " + return nxt_conf_vldt_member_error(vldt, &path_str, + "The \"path\" value must not " "contain null character."); } if (nxt_is_tstr(&conf.format)) { - return nxt_conf_vldt_var(vldt, &format_str, &conf.format); + seg.prev = vldt->path; + seg.seg = format_str; + vldt->path = &seg; + + ret = nxt_conf_vldt_var(vldt, &format_str, &conf.format); + + vldt->path = seg.prev; + + return ret; } return NXT_OK; diff --git a/src/nxt_conn.c b/src/nxt_conn.c index b4950d17a..8c796407b 100644 --- a/src/nxt_conn.c +++ b/src/nxt_conn.c @@ -7,6 +7,13 @@ #include +#if (NXT_DEBUG) +static nxt_bool_t nxt_conn_is_parked(nxt_event_engine_t *engine, + nxt_conn_t *c); +#endif +static nxt_bool_t nxt_conn_timer_quiescent(nxt_timer_t *timer); + + nxt_conn_io_t nxt_unix_conn_io = { .connect = nxt_conn_io_connect, .accept = nxt_conn_io_accept, @@ -45,9 +52,35 @@ nxt_conn_create(nxt_mp_t *mp, nxt_task_t *task) nxt_conn_t *c; nxt_thread_t *thr; - c = nxt_mp_zget(mp, sizeof(nxt_conn_t)); - if (nxt_slow_path(c == NULL)) { - return NULL; + thr = nxt_thread(); + + /* + * The freelist only recycles connection structs backed by the persistent + * engine->mem_pool, so a recycled struct outlives every per-connection + * pool. When engine->mem_pool is NULL (e.g. an OOM at engine setup) the + * struct is allocated from the per-connection pool mp and is therefore + * NOT recyclable; nxt_conn_free() must not push it onto the freelist. + */ + + if (thr->engine->mem_pool != NULL) { + + if (thr->engine->free_connections != NULL) { + c = thr->engine->free_connections; + thr->engine->free_connections = c->next; + nxt_memzero(c, sizeof(nxt_conn_t)); + + } else { + c = nxt_mp_zget(thr->engine->mem_pool, sizeof(nxt_conn_t)); + if (nxt_slow_path(c == NULL)) { + return NULL; + } + } + + } else { + c = nxt_mp_zget(mp, sizeof(nxt_conn_t)); + if (nxt_slow_path(c == NULL)) { + return NULL; + } } c->mem_pool = mp; @@ -63,7 +96,6 @@ nxt_conn_create(nxt_mp_t *mp, nxt_task_t *task) c->log.ident = nxt_task_next_ident(); } - thr = nxt_thread(); thr->engine->connections++; c->task.thread = thr; @@ -104,12 +136,148 @@ nxt_conn_create(nxt_mp_t *mp, nxt_task_t *task) void nxt_conn_free(nxt_task_t *task, nxt_conn_t *c) { - nxt_mp_t *mp; + nxt_mp_t *mp; + nxt_event_engine_t *engine; - task->thread->engine->connections--; + engine = task->thread->engine; + engine->connections--; mp = c->mem_pool; - nxt_mp_release(mp); + + /* + * Only recycle structs backed by the persistent engine->mem_pool. A + * freelist entry must outlive every per-connection pool, so when + * engine->mem_pool is NULL the struct lives in mp (== c->mem_pool) and + * must die with it -- pushing it would leave a dangling pointer on the + * freelist once mp is released below. In the recycled case the struct + * does not live in mp, so releasing mp does not free the pushed struct. + * + * A conn must be freed on the engine that created it: recycling a struct + * backed by another engine's mem_pool would dangle once that engine is + * torn down. The pre-existing connections-- accounting already relies on + * this; assert it so the freelist can too. + */ + + nxt_assert(c->task.thread == task->thread); + + /* An open fd would leak and leave a stale fd-event registered. */ + nxt_assert(c->socket.fd == -1); + + /* + * Take both timers out of the engine's timer machinery, and take the conn + * off whichever tracking queue it is on. Callers only ever + * nxt_timer_disable() a timer, which clears the enabled bit but leaves the + * node in the engine's rbtree, and they rely on nxt_conn_close_handler() + * having untracked the conn -- both leave a live node inside a struct that + * nxt_conn_create() re-zeroes on reuse, silently corrupting the rbtree or + * the idle/active queue and drifting their counters. nxt_conn_untrack() is + * idempotent and nxt_timer_delete()'s removal can be deferred as a queued + * change, so quiescence is rechecked in nxt_conn_recycle_pending(). + * + * Both run outside the recycling branch below: on the non-recycling path + * the struct dies with mp, so a node left in the rbtree or on a tracking + * queue would dangle into released memory rather than be re-zeroed -- the + * same defect with a worse failure mode. + */ + + nxt_assert(c->idle == NXT_CONN_TRACK_NONE); + nxt_conn_untrack(engine, c); + + nxt_timer_delete(engine, &c->read_timer); + nxt_timer_delete(engine, &c->write_timer); + + if (engine->mem_pool != NULL) { + nxt_assert(!nxt_conn_is_parked(engine, c)); + + /* + * Park the struct rather than publishing it to the freelist directly. + * A conn can be freed while work items that reference it are still + * queued -- nxt_h1p_peer_close() sets block_read/closing for + * exactly that reason, then reaches nxt_conn_free() synchronously on + * its fd == -1 path. Handing the struct out now would let + * nxt_conn_create()'s re-zero clear those guards from under the queued + * item, which would then act on an unrelated live connection. + * nxt_conn_recycle_pending() promotes the struct only once the engine + * has drained every work queue, so no queued item can still reach it. + */ + c->next = engine->pending_connections; + engine->pending_connections = c; + } + + if (mp != NULL) { + nxt_mp_release(mp); + } +} + + +static nxt_bool_t +nxt_conn_timer_quiescent(nxt_timer_t *timer) +{ + return !nxt_timer_is_in_tree(timer) + && !timer->queued + && timer->change == NXT_TIMER_NO_CHANGE; +} + + +#if (NXT_DEBUG) + +/* + * A conn freed twice would appear on pending_connections twice; the promotion + * walk then splices it onto free_connections twice and the two lists end up + * aliasing, so distinct live connections can be handed the same struct. Catch + * it at the source in debug builds rather than at the far-away symptom. + */ + +static nxt_bool_t +nxt_conn_is_parked(nxt_event_engine_t *engine, nxt_conn_t *c) +{ + nxt_conn_t *p; + + for (p = engine->pending_connections; p != NULL; p = p->next) { + if (p == c) { + return 1; + } + } + + return 0; +} + +#endif + + +/* + * Publish parked connection structs to the reusable freelist. Called from the + * event engine once nxt_event_engine_queue_pop() reports every engine-local + * work queue empty: at that point no queued work item can reference any parked + * struct, so reuse cannot race a handler that is still to run. A struct whose + * timers have not settled yet stays parked and is retried on a later cycle. + * + * See nxt_event_engine_start() for why the cross-thread locked_work_queue is + * outside this barrier and why that is currently safe. + */ + +void +nxt_conn_recycle_pending(nxt_event_engine_t *engine) +{ + nxt_conn_t *c, *next, **link; + + link = &engine->pending_connections; + + for (c = *link; c != NULL; c = next) { + next = c->next; + + if (nxt_conn_timer_quiescent(&c->read_timer) + && nxt_conn_timer_quiescent(&c->write_timer)) + { + *link = next; + + c->next = engine->free_connections; + engine->free_connections = c; + + } else { + link = &c->next; + } + } } diff --git a/src/nxt_conn.h b/src/nxt_conn.h index c193713ea..4df52c053 100644 --- a/src/nxt_conn.h +++ b/src/nxt_conn.h @@ -146,12 +146,11 @@ struct nxt_conn_s { nxt_conn_io_t *io; - union { #if (NXT_TLS) + union { void *tls; -#endif - nxt_thread_pool_t *thread_pool; } u; +#endif nxt_mp_t *mem_pool; @@ -168,6 +167,9 @@ struct nxt_conn_s { uint8_t block_write; /* 1 bit */ uint8_t delayed; /* 1 bit */ + /* Makes a repeated nxt_conn_close() a no-op; never cleared. */ + uint8_t closing; /* 1 bit */ + #define NXT_CONN_TRACK_NONE 0 #define NXT_CONN_TRACK_IDLE 1 #define NXT_CONN_TRACK_ACTIVE 2 @@ -193,6 +195,7 @@ struct nxt_conn_s { uint8_t tcp_nodelay; /* 1 bit */ nxt_queue_link_t link; + nxt_conn_t *next; }; @@ -244,6 +247,7 @@ struct nxt_conn_s { NXT_EXPORT nxt_conn_t *nxt_conn_create(nxt_mp_t *mp, nxt_task_t *task); NXT_EXPORT void nxt_conn_free(nxt_task_t *task, nxt_conn_t *c); +NXT_EXPORT void nxt_conn_recycle_pending(nxt_event_engine_t *engine); NXT_EXPORT void nxt_conn_close(nxt_event_engine_t *engine, nxt_conn_t *c); NXT_EXPORT void nxt_conn_timer(nxt_event_engine_t *engine, nxt_conn_t *c, @@ -261,6 +265,12 @@ void nxt_conn_connect_error(nxt_task_t *task, void *obj, void *data); NXT_EXPORT nxt_listen_event_t *nxt_listen_event(nxt_task_t *task, nxt_listen_socket_t *ls); void nxt_conn_io_accept(nxt_task_t *task, void *obj, void *data); + +#if (NXT_TESTS) +extern nxt_uint_t nxt_conn_test_nonblocking_failures; +extern nxt_uint_t nxt_listen_event_test_alloc_failures; +#endif + NXT_EXPORT void nxt_conn_accept(nxt_task_t *task, nxt_listen_event_t *lev, nxt_conn_t *c); void nxt_conn_accept_error(nxt_task_t *task, nxt_listen_event_t *lev, @@ -287,9 +297,6 @@ ssize_t nxt_event_conn_io_writev(nxt_conn_t *c, nxt_iobuf_t *iob, nxt_uint_t niob); ssize_t nxt_event_conn_io_send(nxt_conn_t *c, void *buf, size_t size); -NXT_EXPORT void nxt_event_conn_job_sendfile(nxt_task_t *task, - nxt_conn_t *c); - #define nxt_conn_connect(engine, c) \ nxt_work_queue_add(&engine->socket_work_queue, nxt_conn_sys_socket, \ diff --git a/src/nxt_conn_accept.c b/src/nxt_conn_accept.c index 7e2576705..d27a23eb6 100644 --- a/src/nxt_conn_accept.c +++ b/src/nxt_conn_accept.c @@ -34,12 +34,27 @@ static void nxt_conn_listen_timer_handler(nxt_task_t *task, void *obj, void *data); +#if (NXT_TESTS) + +/* For src/test/nxt_router_stale_joint_test.c: fail the next N allocations. */ +nxt_uint_t nxt_listen_event_test_alloc_failures; + +#endif + + nxt_listen_event_t * nxt_listen_event(nxt_task_t *task, nxt_listen_socket_t *ls) { nxt_listen_event_t *lev; nxt_event_engine_t *engine; +#if (NXT_TESTS) + if (nxt_listen_event_test_alloc_failures > 0) { + nxt_listen_event_test_alloc_failures--; + return NULL; + } +#endif + lev = nxt_zalloc(sizeof(nxt_listen_event_t)); if (nxt_fast_path(lev != NULL)) { @@ -69,16 +84,29 @@ nxt_listen_event(nxt_task_t *task, nxt_listen_socket_t *ls) lev->socket.task = &lev->task; lev->timer.task = &lev->task; - if (nxt_conn_accept_alloc(task, lev) != NULL) { + nxt_queue_insert_tail(&engine->listen_connections, &lev->link); + + if (nxt_fast_path(nxt_conn_accept_alloc(task, lev) != NULL)) { nxt_fd_event_enable_accept(engine, &lev->socket); - nxt_queue_insert_tail(&engine->listen_connections, &lev->link); + } else { + /* + * The spare conn could not be allocated (the engine is at + * max_connections, or memory is short). The listener is + * still valid: it is linked above, so the router can find it + * for later updates and closes, and it is simply not armed + * yet. Treat this exactly like the same failure on the accept + * path (nxt_conn_accept_next()): close idle conns to make room + * and let the 100ms listen timer retry the spare, which arms + * accept once it succeeds (nxt_conn_listen_timer_handler()). + * The nxt_fd_event_is_active() check in there keeps its + * disable_read() from acting on this never-armed listener. + */ + nxt_conn_accept_close_idle(task, lev); } - - return lev; } - return NULL; + return lev; } @@ -132,9 +160,20 @@ nxt_conn_listen_handler(nxt_task_t *task, void *obj, void *data) } +#if (NXT_TESTS) + +/* For src/test/nxt_conn_io_accept_test.c: fail the next N nonblocking calls. */ +nxt_uint_t nxt_conn_test_nonblocking_failures; + +#endif + + void nxt_conn_io_accept(nxt_task_t *task, void *obj, void *data) { +#if (NXT_LINUX) + nxt_int_t ret; +#endif socklen_t socklen; nxt_conn_t *c; nxt_socket_t s; @@ -177,19 +216,33 @@ nxt_conn_io_accept(nxt_task_t *task, void *obj, void *data) return; } - c->socket.fd = s; - #if (NXT_LINUX) /* * Linux does not inherit non-blocking mode * from listen socket for accept()ed socket. */ - if (nxt_slow_path(nxt_socket_nonblocking(task, s) != NXT_OK)) { + ret = nxt_socket_nonblocking(task, s); + +#if (NXT_TESTS) + if (nxt_conn_test_nonblocking_failures > 0) { + nxt_conn_test_nonblocking_failures--; + ret = NXT_ERROR; + } +#endif + + if (nxt_slow_path(ret != NXT_OK)) { nxt_socket_close(task, s); + return; } #endif + /* + * Only a socket that is kept goes into the spare conn: on the error + * returns above the spare keeps fd -1, which nxt_conn_free() asserts. + */ + c->socket.fd = s; + nxt_debug(task, "accept(%d): %d", lev->socket.fd, s); nxt_conn_accept(task, lev, c); @@ -269,14 +322,31 @@ nxt_conn_accept_close_idle(nxt_task_t *task, nxt_listen_event_t *lev) engine = task->thread->engine; + /* + * The work runs later, so it gets the engine task. The task of the + * caller can be temporary: nxt_listen_event() runs in a router job, + * whose task is freed with the configuration after the job is posted. + */ nxt_work_queue_add(&engine->close_work_queue, - nxt_conn_accept_close_idle_handler, task, NULL, NULL); + nxt_conn_accept_close_idle_handler, &engine->task, + NULL, NULL); nxt_timer_add(engine, &lev->timer, 100); - nxt_fd_event_disable_read(engine, &lev->socket); + /* + * Alert only when an armed listener stops accepting. The timer retries + * of the same stall, and a new listener that was never armed, come + * here with the read side inactive. + */ + if (nxt_fd_event_is_active(lev->socket.read)) { + nxt_fd_event_disable_read(engine, &lev->socket); + + nxt_alert(task, "new connections are not accepted within 100ms"); - nxt_alert(task, "new connections are not accepted within 100ms"); + } else { + nxt_debug(task, "listener %d: no spare conn, retry in 100ms", + lev->socket.fd); + } } @@ -308,6 +378,16 @@ nxt_conn_accept_close_idle_handler(nxt_task_t *task, void *obj, void *data) nxt_debug(c->socket.task, "idle connection: %d rdy:%d", c->socket.fd, c->socket.read_ready); + /* + * Not reached today: the accept-init work of a conn is queued + * before this pass, and the close queue runs after the read and + * write queues. Checked as nxt_runtime_close_idle_connections() + * does, so a later change of the queue order cannot crash here. + */ + if (c->read_state == NULL || c->read_state->close_handler == NULL) { + continue; + } + /* * Close regardless of read_ready: a pending FIN from the peer * (CLOSE-WAIT) sets read_ready = 1 but the connection is still @@ -386,6 +466,13 @@ nxt_conn_listen_timer_handler(nxt_task_t *task, void *obj, void *data) nxt_fd_event_enable_accept(task->thread->engine, &lev->socket); + /* + * Start a new batch, as nxt_conn_listen_handler() does. A listener + * that was never armed has ready 0 from nxt_zalloc() here, and the + * accept would wrap it. + */ + lev->ready = lev->batch; + lev->accept(task, lev, c); } diff --git a/src/nxt_conn_close.c b/src/nxt_conn_close.c index 80e28fc34..f5fed1412 100644 --- a/src/nxt_conn_close.c +++ b/src/nxt_conn_close.c @@ -27,13 +27,24 @@ nxt_conn_close(nxt_event_engine_t *engine, nxt_conn_t *c) .l_linger = 0, }; - nxt_debug(c->socket.task, "conn close fd:%d, to:%d", - c->socket.fd, c->socket.timedout); + nxt_debug(c->socket.task, "conn close fd:%d, to:%d, cl:%d", + c->socket.fd, c->socket.timedout, c->closing); /* - * Disable all pending write operations because on success they - * will incorrectly call a ready handler set for nxt_conn_close(). + * A handler queued before the first close may close again; a second + * close handler would run the release handler twice. */ + if (c->closing) { + return; + } + + c->closing = 1; + + /* + * Read or write handlers already queued must not touch the socket or + * call the previous state's handlers. + */ + c->block_read = 1; c->write = NULL; if (c->socket.timedout) { diff --git a/src/nxt_conn_write.c b/src/nxt_conn_write.c index c29a83f89..6ccff40ff 100644 --- a/src/nxt_conn_write.c +++ b/src/nxt_conn_write.c @@ -27,6 +27,14 @@ nxt_conn_io_write(nxt_task_t *task, void *obj, void *data) nxt_debug(task, "conn write fd:%d er:%d bl:%d", c->socket.fd, c->socket.error, c->block_write); + /* + * A write queued before nxt_conn_close() returns here: the error path + * would queue the close state's NULL error_handler. + */ + if (c->closing) { + return; + } + if (c->socket.error != 0 || c->block_write) { goto error; } diff --git a/src/nxt_controller.c b/src/nxt_controller.c index 6c6c691af..751570f0b 100644 --- a/src/nxt_controller.c +++ b/src/nxt_controller.c @@ -29,6 +29,24 @@ typedef struct { } nxt_controller_request_t; +#if (NXT_TLS) +typedef struct { + nxt_str_t name; + nxt_cert_info_t *info; + nxt_cert_info_t *old; + nxt_controller_request_t *req; +} nxt_controller_cert_store_t; +#endif + + +#if (NXT_HAVE_NJS) +typedef struct { + nxt_str_t name; + nxt_controller_request_t *req; +} nxt_controller_script_store_t; +#endif + + typedef struct { nxt_uint_t status; nxt_conf_value_t *conf; @@ -38,6 +56,8 @@ typedef struct { ssize_t offset; nxt_uint_t line; nxt_uint_t column; + nxt_str_t pointer; /* RFC 6901 path to validation error. */ + nxt_str_t suggestion; /* "Did you mean" member name. */ } nxt_controller_response_t; @@ -86,6 +106,7 @@ static void nxt_controller_process_request(nxt_task_t *task, nxt_controller_request_t *req); static void nxt_controller_process_config(nxt_task_t *task, nxt_controller_request_t *req, nxt_str_t *path); +static nxt_bool_t nxt_controller_store_in_flight(void); static nxt_bool_t nxt_controller_check_postpone_request(nxt_task_t *task); static void nxt_controller_process_status(nxt_task_t *task, nxt_controller_request_t *req); @@ -96,7 +117,11 @@ static void nxt_controller_status_response(nxt_task_t *task, #if (NXT_TLS) static void nxt_controller_process_cert(nxt_task_t *task, nxt_controller_request_t *req, nxt_str_t *path); -static void nxt_controller_process_cert_save(nxt_task_t *task, +static void nxt_controller_cert_store_handler(nxt_task_t *task, + nxt_port_recv_msg_t *msg, void *data); +static void nxt_controller_cert_apply(nxt_task_t *task, + nxt_controller_cert_store_t *store); +static void nxt_controller_cert_apply_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, void *data); static nxt_bool_t nxt_controller_cert_in_use(nxt_str_t *name); static void nxt_controller_cert_cleanup(nxt_task_t *task, void *obj, @@ -137,6 +162,14 @@ static nxt_uint_t nxt_controller_router_ready; static nxt_controller_conf_t nxt_controller_conf; static nxt_queue_t nxt_controller_waiting_requests; static nxt_bool_t nxt_controller_waiting_init_conf; +#if (NXT_TLS) +static nxt_bool_t nxt_controller_cert_storing; +/* A stored bundle that waits for the first configuration of a router. */ +static nxt_controller_cert_store_t *nxt_controller_cert_stored; +#endif +#if (NXT_HAVE_NJS) +static nxt_bool_t nxt_controller_script_storing; +#endif static nxt_conf_value_t *nxt_controller_status; @@ -203,7 +236,10 @@ nxt_controller_prefork(nxt_task_t *task, nxt_process_t *process, nxt_mp_t *mp) } if (ret == NXT_OK) { - num = nxt_int_parse(ver.start, ver.length); + /* unitd writes no line end, but an editor may add one. */ + num = nxt_int_parse(ver.start, + nxt_str_strip(ver.start, + ver.start + ver.length)); if (nxt_slow_path(num < 0)) { nxt_alert(task, "failed to restore previous configuration: " @@ -377,6 +413,27 @@ nxt_controller_start(nxt_task_t *task, nxt_process_data_t *data) vldt.conf_pool = mp; vldt.ver = nxt_conf_ver; + /* + * A state file written before this check existed can hold bytes the + * control API would now refuse. Rejecting it here would drop the whole + * configuration on the next restart -- the daemon would come back serving + * nothing -- and repairing it would silently rewrite an operator's value, + * which is how a working non-UTF-8 "share" path becomes a broken one. So + * it is loaded exactly as written and reported, once, with the pointer + * that names it. The API refuses to store any more of them. + */ + + if (nxt_conf_validate_encoding(&vldt) == NXT_DECLINED) { + nxt_log(task, NXT_LOG_WARN, "the restored configuration holds a value " + "at \"%V\" that the control API would now reject: %V It is " + "kept as written, and the configuration is running; correct it " + "to be able to update the configuration.", + &vldt.pointer, &vldt.error); + + nxt_memzero(&vldt.error, sizeof(nxt_str_t)); + nxt_memzero(&vldt.pointer, sizeof(nxt_str_t)); + } + ret = nxt_conf_validate(&vldt); if (nxt_slow_path(ret != NXT_OK)) { @@ -409,9 +466,20 @@ static void nxt_controller_process_new_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) { + nxt_port_t *port; + nxt_port_new_port_handler(task, msg); - if (msg->u.new_port->type != NXT_PROCESS_ROUTER + port = msg->u.new_port; + + /* + * The controller never maps a port queue, so the descriptor + * nxt_port_new_port_handler() leaves to its caller has no use here. + */ + nxt_port_recv_msg_close_fds(msg); + + if (port == NULL + || port->type != NXT_PROCESS_ROUTER || !nxt_controller_router_ready) { return; @@ -495,6 +563,8 @@ nxt_controller_remove_pid_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) nxt_memcpy(&pid, msg->buf->mem.pos, sizeof(pid)); + /* Unreferenced: the controller process runs a single engine. */ + process = nxt_runtime_process_find(rt, pid); if (process != NULL && nxt_process_type(process) == NXT_PROCESS_ROUTER) { nxt_controller_router_ready = 0; @@ -537,7 +607,10 @@ static void nxt_controller_conf_init_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, void *data) { - nxt_runtime_t *rt; + nxt_runtime_t *rt; +#if (NXT_TLS) + nxt_controller_cert_store_t *store; +#endif nxt_controller_waiting_init_conf = 0; @@ -563,6 +636,18 @@ nxt_controller_conf_init_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, nxt_controller_listening = 1; } +#if (NXT_TLS) + store = nxt_controller_cert_stored; + + if (store != NULL) { + /* It runs the waiting requests when it ends. */ + nxt_controller_cert_stored = NULL; + + nxt_controller_cert_apply(task, store); + return; + } +#endif + nxt_controller_flush_requests(task); } @@ -696,6 +781,17 @@ nxt_runtime_controller_socket(nxt_task_t *task, nxt_runtime_t *rt) const char *path = ls->sockaddr->u.sockaddr_un.sun_path; nxt_fs_mkdir_p_dirname((const u_char *) path, 0755); + + /* + * Resolve --control-user / --control-group once. The same ids + * are used to chown the socket and to check the peer credentials. + */ + if (nxt_file_owner_ids(rt->control_user, rt->control_group, + &rt->control_uid, &rt->control_gid) + != NXT_OK) + { + return NXT_ERROR; + } } #endif @@ -709,6 +805,88 @@ nxt_runtime_controller_socket(nxt_task_t *task, nxt_runtime_t *rt) } +/* + * A peer may use the control API if it is root, runs as unitd's effective + * UID, or is one of the principals the control socket was delegated to with + * --control-user / --control-group. The socket's file permissions are + * still enforced by the kernel on connect(), so this never grants more + * than those permissions already do. + */ + +nxt_bool_t +nxt_controller_peer_allowed(nxt_uid_t uid, nxt_gid_t gid, + const nxt_gid_t *groups, nxt_uint_t ngroups, nxt_uid_t euid, + nxt_uid_t ctl_uid, nxt_gid_t ctl_gid) +{ + nxt_uint_t i; + + if (uid == 0 || uid == euid) { + return 1; + } + + if (ctl_uid != (nxt_uid_t) -1 && uid == ctl_uid) { + return 1; + } + + if (ctl_gid != (nxt_gid_t) -1) { + if (gid == ctl_gid) { + return 1; + } + + for (i = 0; i < ngroups; i++) { + if (groups[i] == ctl_gid) { + return 1; + } + } + } + + return 0; +} + + +#if (NXT_HAVE_UNIX_DOMAIN && NXT_HAVE_UCRED && defined SO_PEERGROUPS) + +static nxt_gid_t * +nxt_controller_peer_groups(nxt_socket_t fd, nxt_uint_t *ngroups) +{ + nxt_err_t err; + nxt_uint_t try; + nxt_gid_t *groups; + socklen_t len; + + len = 64 * sizeof(nxt_gid_t); + + for (try = 0; try < 2; try++) { + groups = nxt_malloc(len); + if (nxt_slow_path(groups == NULL)) { + return NULL; + } + + if (getsockopt(fd, SOL_SOCKET, SO_PEERGROUPS, groups, &len) == 0) { + *ngroups = len / sizeof(nxt_gid_t); + return groups; + } + + err = nxt_errno; + + nxt_free(groups); + + /* + * On ERANGE the kernel stores the required size in len. A kernel + * older than Linux 4.13 returns ENOPROTOOPT: then only the primary + * group is checked. + */ + if (err != ERANGE) { + break; + } + } + + return NULL; +} + +#endif + + static nxt_int_t nxt_controller_check_peer_cred(nxt_task_t *task, nxt_conn_t *c) { @@ -717,8 +895,9 @@ nxt_controller_check_peer_cred(nxt_task_t *task, nxt_conn_t *c) * The control socket is the privilege boundary for the REST API: * filesystem perms on `control.unit.sock` are defense-in-depth, not the * boundary itself. Require the peer's effective UID to match unitd's - * (or be root) so a local user with directory-write permission cannot - * mutate config by hand-crafting connections. + * (or be root, or the configured --control-user / --control-group) so + * a local user with directory-write permission cannot mutate config by + * hand-crafting connections. */ if (c->remote == NULL || c->remote->u.sockaddr.sa_family != AF_UNIX) @@ -728,8 +907,10 @@ nxt_controller_check_peer_cred(nxt_task_t *task, nxt_conn_t *c) #if (NXT_HAVE_UCRED) { - struct ucred cred; - socklen_t len = sizeof(cred); + nxt_bool_t allowed; + struct ucred cred; + socklen_t len = sizeof(cred); + nxt_runtime_t *rt = task->thread->runtime; if (getsockopt(c->socket.fd, SOL_SOCKET, SO_PEERCRED, &cred, &len) != 0 || len < (socklen_t) sizeof(cred)) @@ -738,7 +919,29 @@ nxt_controller_check_peer_cred(nxt_task_t *task, nxt_conn_t *c) return NXT_ERROR; } - if (cred.uid != 0 && cred.uid != nxt_euid) { + allowed = nxt_controller_peer_allowed(cred.uid, cred.gid, NULL, 0, + nxt_euid, rt->control_uid, + rt->control_gid); + +#if (defined SO_PEERGROUPS) + if (!allowed && rt->control_gid != (nxt_gid_t) -1) { + nxt_gid_t *groups; + nxt_uint_t ngroups; + + groups = nxt_controller_peer_groups(c->socket.fd, &ngroups); + + if (groups != NULL) { + allowed = nxt_controller_peer_allowed(cred.uid, cred.gid, + groups, ngroups, + nxt_euid, + rt->control_uid, + rt->control_gid); + nxt_free(groups); + } + } +#endif + + if (!allowed) { nxt_alert(task, "controller: rejecting connection from uid %d " "(unitd uid %d); set socket permissions accordingly", (int) cred.uid, (int) nxt_euid); @@ -750,15 +953,18 @@ nxt_controller_check_peer_cred(nxt_task_t *task, nxt_conn_t *c) #elif (defined(__FreeBSD__) || defined(__APPLE__) || defined(__OpenBSD__) \ || defined(__NetBSD__) || defined(__DragonFly__)) { - uid_t euid; - gid_t egid; + uid_t euid; + gid_t egid; + nxt_runtime_t *rt = task->thread->runtime; if (getpeereid(c->socket.fd, &euid, &egid) != 0) { nxt_alert(task, "controller: getpeereid failed %E", nxt_errno); return NXT_ERROR; } - if (euid != 0 && euid != nxt_euid) { + if (!nxt_controller_peer_allowed(euid, egid, NULL, 0, nxt_euid, + rt->control_uid, rt->control_gid)) + { nxt_alert(task, "controller: rejecting connection from uid %d " "(unitd uid %d)", (int) euid, (int) nxt_euid); return NXT_ERROR; @@ -1292,6 +1498,16 @@ nxt_controller_process_request(nxt_task_t *task, nxt_controller_request_t *req) goto invalid_method; } + /* + * The answer has the certificates and the modules. Wait, as in + * their handlers. + */ + if (nxt_controller_store_in_flight()) { + nxt_queue_insert_tail(&nxt_controller_waiting_requests, + &req->link); + return; + } + if (nxt_controller_status == NULL) { nxt_controller_process_status(task, req); return; @@ -1495,13 +1711,30 @@ nxt_controller_process_config(nxt_task_t *task, nxt_controller_request_t *req, vldt.conf_pool = mp; vldt.ver = NXT_VERNUM; - rc = nxt_conf_validate(&vldt); + /* + * Before nxt_conf_validate(), which quotes an offending member name + * into its own error text: a name that is not UTF-8 would otherwise + * reach the response body and make the error report unreadable for + * the very reason it is being reported. + * + * This runs on the tree that would be installed, so a configuration + * that already holds such a value has to have it corrected before any + * other part of it can be updated. The pointer in the error names it. + */ + + rc = nxt_conf_validate_encoding(&vldt); + + if (nxt_fast_path(rc == NXT_OK)) { + rc = nxt_conf_validate(&vldt); + } if (nxt_slow_path(rc != NXT_OK)) { nxt_mp_destroy(mp); if (rc == NXT_DECLINED) { resp.detail = vldt.error; + resp.pointer = vldt.pointer; + resp.suggestion = vldt.suggestion; goto invalid_conf; } @@ -1578,13 +1811,30 @@ nxt_controller_process_config(nxt_task_t *task, nxt_controller_request_t *req, vldt.conf_pool = mp; vldt.ver = NXT_VERNUM; - rc = nxt_conf_validate(&vldt); + /* + * Before nxt_conf_validate(), which quotes an offending member name + * into its own error text: a name that is not UTF-8 would otherwise + * reach the response body and make the error report unreadable for + * the very reason it is being reported. + * + * This runs on the tree that would be installed, so a configuration + * that already holds such a value has to have it corrected before any + * other part of it can be updated. The pointer in the error names it. + */ + + rc = nxt_conf_validate_encoding(&vldt); + + if (nxt_fast_path(rc == NXT_OK)) { + rc = nxt_conf_validate(&vldt); + } if (nxt_slow_path(rc != NXT_OK)) { nxt_mp_destroy(mp); if (rc == NXT_DECLINED) { resp.detail = vldt.error; + resp.pointer = vldt.pointer; + resp.suggestion = vldt.suggestion; goto invalid_conf; } @@ -1647,6 +1897,33 @@ nxt_controller_process_config(nxt_task_t *task, nxt_controller_request_t *req, } +/* + * Whether main is storing a file for a request that waits for the answer. + * The store runs in a child of main and can take as long as a few fsync(2) + * calls. The request is not in the waiting queue meanwhile: a router that + * restarts flushes the queue (nxt_controller_conf_init_handler()), and a + * request in it would run a second time, and be freed while main still + * answers it. So the other requests wait while this is true. + */ +static nxt_bool_t +nxt_controller_store_in_flight(void) +{ + nxt_bool_t storing; + + storing = 0; + +#if (NXT_TLS) + storing |= nxt_controller_cert_storing; +#endif + +#if (NXT_HAVE_NJS) + storing |= nxt_controller_script_storing; +#endif + + return storing; +} + + static nxt_bool_t nxt_controller_check_postpone_request(nxt_task_t *task) { @@ -1654,6 +1931,7 @@ nxt_controller_check_postpone_request(nxt_task_t *task) nxt_runtime_t *rt; if (!nxt_queue_is_empty(&nxt_controller_waiting_requests) + || nxt_controller_store_in_flight() || nxt_controller_waiting_init_conf || !nxt_controller_router_ready) { @@ -1796,13 +2074,13 @@ static void nxt_controller_process_cert(nxt_task_t *task, nxt_controller_request_t *req, nxt_str_t *path) { - u_char *p; - nxt_str_t name; - nxt_int_t ret; - nxt_conn_t *c; - nxt_cert_t *cert; - nxt_conf_value_t *value; - nxt_controller_response_t resp; + u_char *p; + nxt_str_t name; + nxt_conn_t *c; + nxt_cert_t *cert; + nxt_conf_value_t *value; + nxt_controller_response_t resp; + nxt_controller_cert_store_t *store; name.length = path->length - 1; name.start = path->start + 1; @@ -1825,6 +2103,13 @@ nxt_controller_process_cert(nxt_task_t *task, if (nxt_str_eq(&req->parser.method, "GET", 3)) { + /* While main stores a bundle, its metadata can roll back. Wait. */ + if (nxt_controller_cert_storing) { + nxt_queue_insert_tail(&nxt_controller_waiting_requests, + &req->link); + return; + } + if (name.length != 0) { value = nxt_cert_info_get(&name); if (value == NULL) { @@ -1852,14 +2137,30 @@ nxt_controller_process_cert(nxt_task_t *task, return; } - if (name.length == 0 || path != NULL) { + /* + * Names starting with "." are reserved for the store's own files. A + * name over NAME_MAX cannot be stored, so it is refused here with 400, + * not by main with ENAMETOOLONG and 500. + */ + if (name.length == 0 || path != NULL || name.start[0] == '.' + || name.length > NXT_CERT_NAME_MAX_LENGTH) + { goto invalid_name; } + /* + * A store or a delete waits for the current reconfiguration: the router + * reads the bundles while it applies a configuration. + */ + if (nxt_controller_check_postpone_request(task)) { + nxt_queue_insert_tail(&nxt_controller_waiting_requests, &req->link); + return; + } + if (nxt_str_eq(&req->parser.method, "PUT", 3)) { - value = nxt_cert_info_get(&name); - if (value != NULL) { - goto exists_cert; + /* Main refuses a larger bundle. Send 413 here, not 500. */ + if (nxt_buf_mem_used_size(&c->read->mem) > NXT_CERT_STORE_MAX_SIZE) { + goto too_large; } cert = nxt_cert_mem(task, &c->read->mem); @@ -1867,16 +2168,56 @@ nxt_controller_process_cert(nxt_task_t *task, goto invalid_cert; } - ret = nxt_cert_info_save(&name, cert); + /* + * The same certificates as the stored bundle, and the router has + * them. A store would write the same file, and a reconfiguration + * would build the same contexts. A script that uploads on a timer + * causes no work here. + */ + if (nxt_cert_info_equal(&name, cert)) { + nxt_cert_destroy(cert); + goto unchanged; + } + + store = nxt_mp_get(c->mem_pool, sizeof(nxt_controller_cert_store_t)); + if (nxt_slow_path(store == NULL)) { + nxt_cert_destroy(cert); + goto alloc_fail; + } + + /* + * Publish the new metadata before main replaces the file. All the + * allocations are here, so a failure changes nothing on disk, and + * the rollback after a failed store cannot fail. + */ + store->info = nxt_cert_info_create(&name, cert); nxt_cert_destroy(cert); - if (nxt_slow_path(ret != NXT_OK)) { + if (nxt_slow_path(store->info == NULL)) { + goto alloc_fail; + } + + if (nxt_slow_path(nxt_cert_info_replace(store->info, &store->old) + != NXT_OK)) + { + nxt_cert_info_release(store->info); goto alloc_fail; } - nxt_cert_store_get(task, &name, c->mem_pool, - nxt_controller_process_cert_save, req); + nxt_controller_cert_storing = 1; + + store->name = name; + store->req = req; + + /* + * Any other change waits until main answers: see + * nxt_controller_store_in_flight(). The request itself is not in + * the waiting queue, so a flush of the queue cannot run it again. + */ + + nxt_cert_store_put(task, &name, &c->read->mem, c->mem_pool, + nxt_controller_cert_store_handler, store); return; } @@ -1906,6 +2247,25 @@ nxt_controller_process_cert(nxt_task_t *task, nxt_controller_response(task, req, &resp); return; +unchanged: + + /* + * The answer a store gives, so that a script keyed on either text + * keeps working. The postpone check above has found a ready router, + * so a store of a bundle in use would have applied the configuration. + */ + resp.status = 200; + + if (nxt_controller_cert_in_use(&name)) { + resp.title = (u_char *) "Certificate chain updated."; + + } else { + resp.title = (u_char *) "Certificate chain uploaded."; + } + + nxt_controller_response(task, req, &resp); + return; + invalid_name: resp.status = 400; @@ -1924,10 +2284,10 @@ nxt_controller_process_cert(nxt_task_t *task, nxt_controller_response(task, req, &resp); return; -exists_cert: +too_large: - resp.status = 400; - resp.title = (u_char *) "Certificate already exists."; + resp.status = 413; + resp.title = (u_char *) "Certificate bundle is too large."; resp.offset = -1; nxt_controller_response(task, req, &resp); @@ -1971,41 +2331,146 @@ nxt_controller_process_cert(nxt_task_t *task, } +/* + * Main stored the bundle, or the store failed. On failure, the old metadata + * goes back, so a failed store leaves no metadata without a file. + */ static void -nxt_controller_process_cert_save(nxt_task_t *task, nxt_port_recv_msg_t *msg, +nxt_controller_cert_store_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, void *data) { - nxt_conn_t *c; - nxt_buf_mem_t *mbuf; - nxt_controller_request_t *req; - nxt_controller_response_t resp; + nxt_controller_response_t resp; + nxt_controller_cert_store_t *store; - req = data; + store = data; - nxt_memzero(&resp, sizeof(nxt_controller_response_t)); + nxt_controller_cert_storing = 0; + + if (msg == NULL || msg->port_msg.type != NXT_PORT_MSG_RPC_READY) { + nxt_cert_info_restore(store->info, store->old); + + nxt_memzero(&resp, sizeof(nxt_controller_response_t)); - if (msg == NULL || msg->port_msg.type == _NXT_PORT_MSG_RPC_ERROR) { resp.status = 500; resp.title = (u_char *) "Failed to store certificate."; + resp.offset = -1; - nxt_controller_response(task, req, &resp); + nxt_controller_response(task, store->req, &resp); + + nxt_controller_flush_requests(task); return; } - c = req->conn; + nxt_cert_info_release(store->old); + + /* + * The router restarted during the store, and the new router applies + * its first configuration now. A second configuration sent before + * that one ends would run at the same time in the router, and the + * router does not support that. nxt_controller_conf_init_handler() + * goes on with this store. The other changes wait meanwhile, because + * nxt_controller_waiting_init_conf is set. + */ + if (nxt_controller_waiting_init_conf) { + nxt_controller_cert_stored = store; + return; + } + + nxt_controller_cert_apply(task, store); +} + + +/* + * When the current configuration names the stored bundle, the controller + * sends the configuration to the router again. The router reads every + * bundle during a reconfiguration, so new handshakes get the new + * certificate. Accepted connections keep their old TLS context. The + * first configuration of a new router may have read the old bundle, so a + * bundle that waited for it is applied as well. + */ +static void +nxt_controller_cert_apply(nxt_task_t *task, nxt_controller_cert_store_t *store) +{ + nxt_int_t rc; + nxt_runtime_t *rt; + nxt_controller_request_t *req; + nxt_controller_response_t resp; + + req = store->req; + + nxt_memzero(&resp, sizeof(nxt_controller_response_t)); + + rt = task->thread->runtime; + + /* When no router is ready, the next router loads the new bundle. */ + + if (nxt_controller_cert_in_use(&store->name) + && nxt_controller_router_ready + && rt->port_by_type[NXT_PROCESS_ROUTER] != NULL) + { + rc = nxt_controller_conf_send(task, req->conn->mem_pool, + nxt_controller_conf.root, + nxt_controller_cert_apply_handler, + store); + + if (nxt_fast_path(rc == NXT_OK)) { + nxt_queue_insert_head(&nxt_controller_waiting_requests, + &req->link); + return; + } + + nxt_cert_info_applied(&store->name, 0); + + resp.status = 500; + resp.title = (u_char *) "Certificate stored but not applied."; + resp.offset = -1; + + } else { + resp.status = 200; + resp.title = (u_char *) "Certificate chain uploaded."; + } + + nxt_controller_response(task, req, &resp); + + nxt_controller_flush_requests(task); +} + - mbuf = &c->read->mem; +static void +nxt_controller_cert_apply_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, + void *data) +{ + nxt_bool_t applied; + nxt_controller_request_t *req; + nxt_controller_response_t resp; + nxt_controller_cert_store_t *store; - nxt_fd_write(msg->fd[0], mbuf->pos, nxt_buf_mem_used_size(mbuf)); + store = data; + req = store->req; - nxt_fd_close(msg->fd[0]); + nxt_queue_remove(&req->link); nxt_memzero(&resp, sizeof(nxt_controller_response_t)); - resp.status = 200; - resp.title = (u_char *) "Certificate chain uploaded."; + applied = (msg->port_msg.type == NXT_PORT_MSG_RPC_READY); + + /* Other changes waited, so the name still holds this bundle. */ + nxt_cert_info_applied(&store->name, applied); + + if (applied) { + resp.status = 200; + resp.title = (u_char *) "Certificate chain updated."; + + } else { + /* The router kept the old configuration and the old contexts. */ + resp.status = 500; + resp.title = (u_char *) "Certificate stored but not applied."; + resp.offset = -1; + } nxt_controller_response(task, req, &resp); + + nxt_controller_flush_requests(task); } @@ -2072,15 +2537,16 @@ static void nxt_controller_process_script(nxt_task_t *task, nxt_controller_request_t *req, nxt_str_t *path) { - u_char *p; - nxt_int_t ret; - nxt_str_t name; - nxt_conn_t *c; - nxt_script_t *script; - nxt_buf_mem_t *bm; - nxt_conf_value_t *value; - nxt_controller_response_t resp; - u_char error[NXT_MAX_ERROR_STR]; + u_char *p; + nxt_int_t ret; + nxt_str_t name; + nxt_conn_t *c; + nxt_script_t *script; + nxt_buf_mem_t *bm; + nxt_conf_value_t *value; + nxt_controller_response_t resp; + nxt_controller_script_store_t *store; + u_char error[NXT_MAX_ERROR_STR]; name.length = path->length - 1; name.start = path->start + 1; @@ -2103,6 +2569,13 @@ nxt_controller_process_script(nxt_task_t *task, if (nxt_str_eq(&req->parser.method, "GET", 3)) { + /* While main stores a module, its metadata can roll back. Wait. */ + if (nxt_controller_script_storing) { + nxt_queue_insert_tail(&nxt_controller_waiting_requests, + &req->link); + return; + } + if (name.length != 0) { value = nxt_script_info_get(&name); if (value == NULL) { @@ -2130,10 +2603,26 @@ nxt_controller_process_script(nxt_task_t *task, return; } - if (name.length == 0 || path != NULL) { + /* + * The name is a file name in the scripts directory. Main keeps its + * temporary file there under a name that starts with ".", and refuses + * such a name. Answer 400 here, not 500 from main. + */ + if (name.length == 0 || path != NULL || name.start[0] == '.' + || name.length > NXT_SCRIPT_NAME_MAX_LENGTH) + { goto invalid_name; } + /* + * A store or a delete waits for the current reconfiguration: the router + * reads the modules while it applies a configuration. + */ + if (nxt_controller_check_postpone_request(task)) { + nxt_queue_insert_tail(&nxt_controller_waiting_requests, &req->link); + return; + } + if (nxt_str_eq(&req->parser.method, "PUT", 3)) { value = nxt_script_info_get(&name); if (value != NULL) { @@ -2142,12 +2631,23 @@ nxt_controller_process_script(nxt_task_t *task, bm = &c->read->mem; + /* Main refuses a larger module. Send 413 here, not 500. */ + if (nxt_buf_mem_used_size(bm) > NXT_SCRIPT_STORE_MAX_SIZE) { + goto too_large; + } + + store = nxt_mp_get(c->mem_pool, sizeof(nxt_controller_script_store_t)); + if (nxt_slow_path(store == NULL)) { + goto alloc_fail; + } + script = nxt_script_new(task, &name, bm->pos, nxt_buf_mem_used_size(bm), error); if (script == NULL) { goto invalid_script; } + /* A failed store deletes the metadata again. */ ret = nxt_script_info_save(&name, script); nxt_script_destroy(script); @@ -2156,8 +2656,19 @@ nxt_controller_process_script(nxt_task_t *task, goto alloc_fail; } - nxt_script_store_get(task, &name, c->mem_pool, - nxt_controller_process_script_save, req); + nxt_controller_script_storing = 1; + + store->name = name; + store->req = req; + + /* + * Any other change waits until main answers: see + * nxt_controller_store_in_flight(). The request itself is not in + * the waiting queue, so a flush of the queue cannot run it again. + */ + + nxt_script_store_put(task, &name, bm, c->mem_pool, + nxt_controller_process_script_save, store); return; } @@ -2217,6 +2728,15 @@ nxt_controller_process_script(nxt_task_t *task, nxt_controller_response(task, req, &resp); return; +too_large: + + resp.status = 413; + resp.title = (u_char *) "JS module is too large."; + resp.offset = -1; + + nxt_controller_response(task, req, &resp); + return; + script_in_use: resp.status = 400; @@ -2258,37 +2778,36 @@ static void nxt_controller_process_script_save(nxt_task_t *task, nxt_port_recv_msg_t *msg, void *data) { - nxt_conn_t *c; - nxt_buf_mem_t *mbuf; - nxt_controller_request_t *req; - nxt_controller_response_t resp; + nxt_controller_request_t *req; + nxt_controller_response_t resp; + nxt_controller_script_store_t *store; - req = data; + store = data; + req = store->req; + + nxt_controller_script_storing = 0; nxt_memzero(&resp, sizeof(nxt_controller_response_t)); - if (msg == NULL || msg->port_msg.type == _NXT_PORT_MSG_RPC_ERROR) { + if (msg == NULL || msg->port_msg.type != NXT_PORT_MSG_RPC_READY) { + /* + * A store child that failed after rename(2) leaves the file, and + * the next start loads it. + */ + (void) nxt_script_info_delete(&store->name); + resp.status = 500; resp.title = (u_char *) "Failed to store script."; + resp.offset = -1; - nxt_controller_response(task, req, &resp); - return; + } else { + resp.status = 200; + resp.title = (u_char *) "JS module uploaded."; } - c = req->conn; - - mbuf = &c->read->mem; - - nxt_fd_write(msg->fd[0], mbuf->pos, nxt_buf_mem_used_size(mbuf)); - - nxt_fd_close(msg->fd[0]); - - nxt_memzero(&resp, sizeof(nxt_controller_response_t)); - - resp.status = 200; - resp.title = (u_char *) "JS module uploaded."; - nxt_controller_response(task, req, &resp); + + nxt_controller_flush_requests(task); } @@ -2628,6 +3147,8 @@ nxt_controller_response(nxt_task_t *task, nxt_controller_request_t *req, static const nxt_str_t offset_str = nxt_string("offset"); static const nxt_str_t line_str = nxt_string("line"); static const nxt_str_t column_str = nxt_string("column"); + static const nxt_str_t path_str = nxt_string("path"); + static const nxt_str_t suggestion_str = nxt_string("suggestion"); static nxt_time_string_t date_cache = { (nxt_atomic_uint_t) -1, @@ -2656,6 +3177,10 @@ nxt_controller_response(nxt_task_t *task, nxt_controller_request_t *req, nxt_str_set(&status_line, "405 Method Not Allowed"); break; + case 413: + nxt_str_set(&status_line, "413 Payload Too Large"); + break; + default: nxt_str_set(&status_line, "500 Internal Server Error"); break; @@ -2665,9 +3190,15 @@ nxt_controller_response(nxt_task_t *task, nxt_controller_request_t *req, value = resp->conf; if (value == NULL) { + nxt_uint_t loc_n, loc_i, have_offset, have_pointer; + + have_offset = (resp->status >= 400 && resp->offset != -1); + have_pointer = (resp->status >= 400 && resp->pointer.start != NULL); + n = 1 + (resp->detail.length != 0) - + (resp->status >= 400 && resp->offset != -1); + + (have_offset || have_pointer) + + (resp->suggestion.length != 0); value = nxt_conf_create_object(c->mem_pool, n); @@ -2694,24 +3225,47 @@ nxt_controller_response(nxt_task_t *task, nxt_controller_request_t *req, nxt_conf_set_member_string(value, &detail_str, &resp->detail, n); } - if (resp->status >= 400 && resp->offset != -1) { + if (have_offset || have_pointer) { n++; - location = nxt_conf_create_object(c->mem_pool, - resp->line != 0 ? 3 : 1); + loc_n = (have_offset ? (resp->line != 0 ? 3 : 1) : 0) + have_pointer; + + location = nxt_conf_create_object(c->mem_pool, loc_n); + + if (nxt_slow_path(location == NULL)) { + nxt_controller_conn_close(task, c, req); + return; + } nxt_conf_set_member(value, &location_str, location, n); - nxt_conf_set_member_integer(location, &offset_str, resp->offset, 0); + loc_i = 0; + + if (have_offset) { + nxt_conf_set_member_integer(location, &offset_str, + resp->offset, loc_i++); - if (resp->line != 0) { - nxt_conf_set_member_integer(location, &line_str, - resp->line, 1); + if (resp->line != 0) { + nxt_conf_set_member_integer(location, &line_str, + resp->line, loc_i++); + + nxt_conf_set_member_integer(location, &column_str, + resp->column, loc_i++); + } + } - nxt_conf_set_member_integer(location, &column_str, - resp->column, 2); + if (have_pointer) { + nxt_conf_set_member_string(location, &path_str, + &resp->pointer, loc_i++); } } + + if (resp->suggestion.length != 0) { + n++; + + nxt_conf_set_member_string(value, &suggestion_str, + &resp->suggestion, n); + } } nxt_memzero(&pretty, sizeof(nxt_conf_json_pretty_t)); diff --git a/src/nxt_cyassl.c b/src/nxt_cyassl.c deleted file mode 100644 index 91039e0c5..000000000 --- a/src/nxt_cyassl.c +++ /dev/null @@ -1,621 +0,0 @@ - -/* - * Copyright (C) NGINX, Inc. - * Copyright (C) Igor Sysoev - */ - -#include -#include -#include - - -typedef struct { - CYASSL *session; - - int ssl_error; - uint8_t times; /* 2 bits */ - - nxt_buf_mem_t buffer; -} nxt_cyassl_conn_t; - - -static nxt_int_t nxt_cyassl_server_init(nxt_ssltls_conf_t *conf); -static void nxt_cyassl_conn_init(nxt_thread_t *thr, nxt_ssltls_conf_t *conf, - nxt_event_conn_t *c); -static void nxt_cyassl_session_cleanup(void *data); -static int nxt_cyassl_io_recv(CYASSL *ssl, char *buf, int size, void *data); -static int nxt_cyassl_io_send(CYASSL *ssl, char *buf, int size, void *data); -static void nxt_cyassl_conn_handshake(nxt_thread_t *thr, void *obj, void *data); -static void nxt_cyassl_conn_io_read(nxt_thread_t *thr, void *obj, void *data); -static void nxt_cyassl_conn_io_shutdown(nxt_thread_t *thr, void *obj, - void *data); -static ssize_t nxt_cyassl_conn_io_write_chunk(nxt_thread_t *thr, - nxt_event_conn_t *c, nxt_buf_t *b, size_t limit); -static ssize_t nxt_cyassl_conn_io_send(nxt_event_conn_t *c, void *buf, - size_t size); -static nxt_int_t nxt_cyassl_conn_test_error(nxt_thread_t *thr, - nxt_event_conn_t *c, int err, nxt_work_handler_t handler); -static void nxt_cdecl nxt_cyassl_conn_error(nxt_event_conn_t *c, nxt_err_t err, - const char *fmt, ...); -static nxt_uint_t nxt_cyassl_log_error_level(nxt_event_conn_t *c, nxt_err_t err, - int ssl_error); -static void nxt_cdecl nxt_cyassl_log_error(nxt_uint_t level, nxt_log_t *log, - int ret, const char *fmt, ...); -static u_char *nxt_cyassl_copy_error(int err, u_char *p, u_char *end); - - -const nxt_ssltls_lib_t nxt_cyassl_lib = { - nxt_cyassl_server_init, - NULL, -}; - - -static nxt_event_conn_io_t nxt_cyassl_event_conn_io = { - NULL, - NULL, - - nxt_cyassl_conn_io_read, - NULL, - NULL, - - nxt_event_conn_io_write, - nxt_cyassl_conn_io_write_chunk, - NULL, - NULL, - nxt_cyassl_conn_io_send, - - nxt_cyassl_conn_io_shutdown, -}; - - -static nxt_int_t -nxt_cyassl_start(void) -{ - int err; - nxt_thread_t *thr; - static nxt_bool_t started; - - if (nxt_fast_path(started)) { - return NXT_OK; - } - - started = 1; - - thr = nxt_thread(); - - /* TODO: CyaSSL_Cleanup() */ - - err = CyaSSL_Init(); - if (err != SSL_SUCCESS) { - nxt_cyassl_log_error(NXT_LOG_ALERT, thr->log, err, - "CyaSSL_Init() failed"); - return NXT_ERROR; - } - - nxt_thread_log_error(NXT_LOG_INFO, "CyaSSL version: %s", - LIBCYASSL_VERSION_STRING); - - /* CyaSSL_SetLoggingCb */ - /* CyaSSL_SetAllocators */ - - return NXT_OK; -} - - -static nxt_int_t -nxt_cyassl_server_init(nxt_ssltls_conf_t *conf) -{ - int err; - char *certificate, *key; - CYASSL_CTX *ctx; - nxt_thread_t *thr; - - thr = nxt_thread(); - - if (nxt_slow_path(nxt_cyassl_start() != NXT_OK)) { - return NXT_ERROR; - } - - ctx = CyaSSL_CTX_new(CyaSSLv23_server_method()); - if (ctx == NULL) { - nxt_cyassl_log_error(NXT_LOG_ALERT, thr->log, 0, - "CyaSSL_CTX_new() failed"); - return NXT_ERROR; - } - - conf->ctx = ctx; - conf->conn_init = nxt_cyassl_conn_init; - - certificate = conf->certificate; - - err = CyaSSL_CTX_use_certificate_file(ctx, certificate, SSL_FILETYPE_PEM); - if (err != SSL_SUCCESS) { - nxt_cyassl_log_error(NXT_LOG_ALERT, thr->log, err, - "CyaSSL_CTX_use_certificate_file(\"%s\") failed", - certificate); - goto fail; - } - - key = conf->certificate_key; - - err = CyaSSL_CTX_use_PrivateKey_file(ctx, key, SSL_FILETYPE_PEM); - if (err != SSL_SUCCESS) { - nxt_cyassl_log_error(NXT_LOG_ALERT, thr->log, err, - "CyaSSL_CTX_use_PrivateKey_file(\"%s\") failed", - key); - goto fail; - } - - if (conf->ciphers != NULL) { - err = CyaSSL_CTX_set_cipher_list(ctx, conf->ciphers); - if (err != SSL_SUCCESS) { - nxt_cyassl_log_error(NXT_LOG_ALERT, thr->log, err, - "CyaSSL_CTX_set_cipher_list(\"%s\") failed", - conf->ciphers); - goto fail; - } - } - - /* TODO: ca_certificate */ - - CyaSSL_SetIORecv(ctx, nxt_cyassl_io_recv); - CyaSSL_SetIOSend(ctx, nxt_cyassl_io_send); - - return NXT_OK; - -fail: - - CyaSSL_CTX_free(ctx); - - return NXT_ERROR; -} - - -static void -nxt_cyassl_conn_init(nxt_thread_t *thr, nxt_ssltls_conf_t *conf, - nxt_event_conn_t *c) -{ - CYASSL *s; - CYASSL_CTX *ctx; - nxt_cyassl_conn_t *ssltls; - nxt_mem_pool_cleanup_t *mpcl; - - nxt_log_debug(c->socket.log, "cyassl conn init"); - - ssltls = nxt_mp_zget(c->mem_pool, sizeof(nxt_cyassl_conn_t)); - if (ssltls == NULL) { - goto fail; - } - - c->u.ssltls = ssltls; - nxt_buf_mem_set_size(&ssltls->buffer, conf->buffer_size); - - mpcl = nxt_mem_pool_cleanup(c->mem_pool, 0); - if (mpcl == NULL) { - goto fail; - } - - ctx = conf->ctx; - - s = CyaSSL_new(ctx); - if (s == NULL) { - nxt_cyassl_log_error(NXT_LOG_ALERT, c->socket.log, 0, - "CyaSSL_new() failed"); - goto fail; - } - - ssltls->session = s; - mpcl->handler = nxt_cyassl_session_cleanup; - mpcl->data = ssltls; - - CyaSSL_SetIOReadCtx(s, c); - CyaSSL_SetIOWriteCtx(s, c); - - c->io = &nxt_cyassl_event_conn_io; - c->sendfile = NXT_CONN_SENDFILE_OFF; - - nxt_cyassl_conn_handshake(thr, c, c->socket.data); - return; - -fail: - - nxt_event_conn_io_handle(thr, c->read_work_queue, - c->read_state->error_handler, c, c->socket.data); -} - - -static void -nxt_cyassl_session_cleanup(void *data) -{ - nxt_cyassl_conn_t *ssltls; - - ssltls = data; - - nxt_thread_log_debug("cyassl session cleanup"); - - nxt_free(ssltls->buffer.start); - - CyaSSL_free(ssltls->session); -} - - -static int -nxt_cyassl_io_recv(CYASSL *ssl, char *buf, int size, void *data) -{ - ssize_t n; - nxt_thread_t *thr; - nxt_event_conn_t *c; - - c = data; - thr = nxt_thread(); - - n = thr->engine->event->io->recv(c, (u_char *) buf, size, 0); - - if (n > 0) { - return n; - } - - if (n == 0) { - return CYASSL_CBIO_ERR_CONN_CLOSE; - } - - if (n == NXT_AGAIN) { - return CYASSL_CBIO_ERR_WANT_READ; - } - - return CYASSL_CBIO_ERR_GENERAL; -} - - -static int -nxt_cyassl_io_send(CYASSL *ssl, char *buf, int size, void *data) -{ - ssize_t n; - nxt_thread_t *thr; - nxt_event_conn_t *c; - - c = data; - thr = nxt_thread(); - - n = thr->engine->event->io->send(c, (u_char *) buf, size); - - if (n > 0) { - return n; - } - - if (n == NXT_AGAIN) { - return CYASSL_CBIO_ERR_WANT_WRITE; - } - - return CYASSL_CBIO_ERR_GENERAL; -} - - -static void -nxt_cyassl_conn_handshake(nxt_thread_t *thr, void *obj, void *data) -{ - int ret; - nxt_int_t n; - nxt_err_t err; - nxt_event_conn_t *c; - nxt_cyassl_conn_t *ssltls; - - c = obj; - ssltls = c->u.ssltls; - - nxt_log_debug(thr->log, "cyassl conn handshake: %d", ssltls->times); - - /* "ssltls->times == 1" is suitable to run CyaSSL_negotiate() in job. */ - - ret = CyaSSL_negotiate(ssltls->session); - - err = (ret != 0) ? nxt_socket_errno : 0; - - nxt_thread_time_debug_update(thr); - - nxt_log_debug(thr->log, "CyaSSL_negotiate(%d): %d", c->socket.fd, ret); - - if (ret == 0) { - nxt_cyassl_conn_io_read(thr, c, data); - return; - } - - n = nxt_cyassl_conn_test_error(thr, c, ret, nxt_cyassl_conn_handshake); - - if (n == NXT_ERROR) { - nxt_cyassl_conn_error(c, err, "CyaSSL_negotiate(%d) failed", - c->socket.fd); - - nxt_event_conn_io_handle(thr, c->read_work_queue, - c->read_state->error_handler, c, data); - - } else if (ssltls->ssl_error == SSL_ERROR_WANT_READ && ssltls->times < 2) { - ssltls->times++; - } -} - - -static void -nxt_cyassl_conn_io_read(nxt_thread_t *thr, void *obj, void *data) -{ - int ret; - nxt_buf_t *b; - nxt_err_t err; - nxt_int_t n; - nxt_event_conn_t *c; - nxt_cyassl_conn_t *ssltls; - nxt_work_handler_t handler; - - c = obj; - - nxt_log_debug(thr->log, "cyassl conn read"); - - handler = c->read_state->ready_handler; - b = c->read; - - /* b == NULL is used to test descriptor readiness. */ - - if (b != NULL) { - ssltls = c->u.ssltls; - - ret = CyaSSL_read(ssltls->session, b->mem.free, - b->mem.end - b->mem.free); - - err = (ret <= 0) ? nxt_socket_errno : 0; - - nxt_log_debug(thr->log, "CyaSSL_read(%d, %p, %uz): %d", - c->socket.fd, b->mem.free, b->mem.end - b->mem.free, ret); - - if (ret > 0) { - /* c->socket.read_ready is kept. */ - b->mem.free += ret; - handler = c->read_state->ready_handler; - - } else { - n = nxt_cyassl_conn_test_error(thr, c, ret, - nxt_cyassl_conn_io_read); - - if (nxt_fast_path(n != NXT_ERROR)) { - return; - } - - nxt_cyassl_conn_error(c, err, "CyaSSL_read(%d, %p, %uz) failed", - c->socket.fd, b->mem.free, - b->mem.end - b->mem.free); - - handler = c->read_state->error_handler; - } - } - - nxt_event_conn_io_handle(thr, c->read_work_queue, handler, c, data); -} - - -static ssize_t -nxt_cyassl_conn_io_write_chunk(nxt_thread_t *thr, nxt_event_conn_t *c, - nxt_buf_t *b, size_t limit) -{ - nxt_cyassl_conn_t *ssltls; - - nxt_log_debug(thr->log, "cyassl conn write chunk"); - - ssltls = c->u.ssltls; - - return nxt_sendbuf_copy_coalesce(c, &ssltls->buffer, b, limit); -} - - -static ssize_t -nxt_cyassl_conn_io_send(nxt_event_conn_t *c, void *buf, size_t size) -{ - int ret; - nxt_err_t err; - nxt_int_t n; - nxt_cyassl_conn_t *ssltls; - - nxt_log_debug(c->socket.log, "cyassl send"); - - ssltls = c->u.ssltls; - - ret = CyaSSL_write(ssltls->session, buf, size); - - if (ret <= 0) { - err = nxt_socket_errno; - c->socket.error = err; - - } else { - err = 0; - } - - nxt_log_debug(c->socket.log, "CyaSSL_write(%d, %p, %uz): %d", - c->socket.fd, buf, size, ret); - - if (ret > 0) { - return ret; - } - - n = nxt_cyassl_conn_test_error(nxt_thread(), c, ret, - nxt_event_conn_io_write); - - if (nxt_slow_path(n == NXT_ERROR)) { - nxt_cyassl_conn_error(c, err, "CyaSSL_write(%d, %p, %uz) failed", - c->socket.fd, buf, size); - } - - return n; -} - - -static void -nxt_cyassl_conn_io_shutdown(nxt_thread_t *thr, void *obj, void *data) -{ - int ret; - nxt_event_conn_t *c; - nxt_cyassl_conn_t *ssltls; - - c = obj; - - nxt_log_debug(thr->log, "cyassl conn shutdown"); - - ssltls = c->u.ssltls; - - ret = CyaSSL_shutdown(ssltls->session); - - nxt_log_debug(thr->log, "CyaSSL_shutdown(%d): %d", c->socket.fd, ret); - - if (nxt_slow_path(ret != SSL_SUCCESS)) { - nxt_cyassl_conn_error(c, 0, "CyaSSL_shutdown(%d) failed", c->socket.fd); - } - - nxt_event_conn_io_handle(thr, c->write_work_queue, - c->write_state->close_handler, c, data); -} - - -static nxt_int_t -nxt_cyassl_conn_test_error(nxt_thread_t *thr, nxt_event_conn_t *c, int ret, - nxt_work_handler_t handler) -{ - nxt_work_queue_t *wq; - nxt_cyassl_conn_t *ssltls; - - ssltls = c->u.ssltls; - ssltls->ssl_error = CyaSSL_get_error(ssltls->session, ret); - - nxt_log_debug(thr->log, "CyaSSL_get_error(): %d", ssltls->ssl_error); - - switch (ssltls->ssl_error) { - - case SSL_ERROR_WANT_READ: - nxt_event_fd_block_write(thr->engine, &c->socket); - - c->socket.read_ready = 0; - c->socket.read_handler = handler; - - if (nxt_event_fd_is_disabled(c->socket.read)) { - nxt_event_fd_enable_read(thr->engine, &c->socket); - } - - return NXT_AGAIN; - - case SSL_ERROR_WANT_WRITE: - nxt_event_fd_block_read(thr->engine, &c->socket); - - c->socket.write_ready = 0; - c->socket.write_handler = handler; - - if (nxt_event_fd_is_disabled(c->socket.write)) { - nxt_event_fd_enable_write(thr->engine, &c->socket); - } - - return NXT_AGAIN; - - case SSL_ERROR_ZERO_RETURN: - /* A "close notify" alert */ - - if (c->read_state != NULL) { - wq = c->read_work_queue; - handler = c->read_state->close_handler; - - } else { - wq = c->write_work_queue; - handler = c->write_state->close_handler; - } - - nxt_event_conn_io_handle(thr, wq, handler, c, c->socket.data); - - return 0; - - default: - return NXT_ERROR; - } -} - - -static void nxt_cdecl -nxt_cyassl_conn_error(nxt_event_conn_t *c, nxt_err_t err, const char *fmt, ...) -{ - u_char *p, *end; - va_list args; - nxt_uint_t level; - nxt_cyassl_conn_t *ssltls; - u_char msg[NXT_MAX_ERROR_STR]; - - ssltls = c->u.ssltls; - - level = nxt_cyassl_log_error_level(c, err, ssltls->ssl_error); - - if (nxt_log_level_enough(c->socket.log, level)) { - - end = msg + sizeof(msg); - - va_start(args, fmt); - p = nxt_vsprintf(msg, end, fmt, args); - va_end(args); - - if (err != 0) { - p = nxt_sprintf(p, end, " %E", err); - } - - p = nxt_cyassl_copy_error(ssltls->ssl_error, p, end); - - nxt_log_error(level, c->socket.log, "%*s", p - msg, msg); - } -} - - -static nxt_uint_t -nxt_cyassl_log_error_level(nxt_event_conn_t *c, nxt_err_t err, int ssl_error) -{ - switch (ssl_error) { - - case SOCKET_ERROR_E: /* -208 */ - case MATCH_SUITE_ERROR: /* -261 */ - break; - - default: - return NXT_LOG_ALERT; - } - - return NXT_LOG_INFO; -} - - -static void nxt_cdecl -nxt_cyassl_log_error(nxt_uint_t level, nxt_log_t *log, int err, - const char *fmt, ...) -{ - u_char *p, *end; - va_list args; - u_char msg[NXT_MAX_ERROR_STR]; - - if (nxt_log_level_enough(log, level)) { - - end = msg + sizeof(msg); - - va_start(args, fmt); - p = nxt_vsprintf(msg, end, fmt, args); - va_end(args); - - p = nxt_cyassl_copy_error(err, p, end); - - nxt_log_error(level, log, "%*s", p - msg, msg); - } -} - - -static u_char * -nxt_cyassl_copy_error(int err, u_char *p, u_char *end) -{ - p = nxt_sprintf(p, end, " (SSL:%d ", err); - - CyaSSL_ERR_error_string_n(err, (char *) p, end - p); - - p += nxt_strlen(p); - - if (p < end) { - *p++ = ')'; - } - - return p; -} diff --git a/src/nxt_devpoll_engine.c b/src/nxt_devpoll_engine.c index 71ab11a38..2470d0594 100644 --- a/src/nxt_devpoll_engine.c +++ b/src/nxt_devpoll_engine.c @@ -34,6 +34,8 @@ static void nxt_devpoll_enable(nxt_event_engine_t *engine, nxt_fd_event_t *ev); static void nxt_devpoll_disable(nxt_event_engine_t *engine, nxt_fd_event_t *ev); static nxt_bool_t nxt_devpoll_close(nxt_event_engine_t *engine, nxt_fd_event_t *ev); +static void nxt_devpoll_cancel_changes(nxt_event_engine_t *engine, + nxt_fd_event_t *ev); static void nxt_devpoll_enable_read(nxt_event_engine_t *engine, nxt_fd_event_t *ev); static void nxt_devpoll_enable_write(nxt_event_engine_t *engine, @@ -70,6 +72,7 @@ const nxt_event_interface_t nxt_devpoll_engine = { nxt_devpoll_disable, nxt_devpoll_disable, nxt_devpoll_close, + nxt_devpoll_cancel_changes, nxt_devpoll_enable_read, nxt_devpoll_enable_write, nxt_devpoll_disable_read, @@ -390,6 +393,46 @@ nxt_devpoll_change(nxt_event_engine_t *engine, nxt_fd_event_t *ev, } +/* + * Take this event's pending changes out of the batch, so that a struct that + * is about to be freed is not dereferenced by nxt_devpoll_commit_changes(). + * + * The change is dropped rather than committed: the descriptor is closed, or + * is about to be, so the change would name a descriptor number that may + * already belong to somebody else. + */ + +static void +nxt_devpoll_cancel_changes(nxt_event_engine_t *engine, nxt_fd_event_t *ev) +{ + nxt_devpoll_change_t *change, *dst, *end; + + if (!ev->changing) { + return; + } + + dst = engine->u.devpoll.changes; + end = dst + engine->u.devpoll.nchanges; + + for (change = dst; change < end; change++) { + + if (change->event == ev) { + continue; + } + + if (dst != change) { + *dst = *change; + } + + dst++; + } + + engine->u.devpoll.nchanges = (int) (dst - engine->u.devpoll.changes); + + ev->changing = 0; +} + + static nxt_int_t nxt_devpoll_commit_changes(nxt_event_engine_t *engine) { diff --git a/src/nxt_dyld.c b/src/nxt_dyld.c deleted file mode 100644 index 63e6be145..000000000 --- a/src/nxt_dyld.c +++ /dev/null @@ -1,86 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) NGINX, Inc. - */ - -#include - - -nxt_int_t -nxt_dyld_load(nxt_dyld_t *dyld) -{ - const char *err; - - dyld->handle = dlopen(dyld->name, RTLD_NOW | RTLD_GLOBAL); - - if (dyld->handle != NULL) { - nxt_thread_log_debug("dlopen(\"%s\")", dyld->name); - return NXT_OK; - } - - err = dlerror(); - if (err == NULL) { - err = "(null)"; - } - - nxt_thread_log_alert("dlopen(\"%s\") failed: %s", dyld->name, err); - - return NXT_ERROR; -} - - -void * -nxt_dyld_symbol(nxt_dyld_t *dyld, const char *symbol) -{ - void *handle, *s; - const char *name; - const char *err; - - if (dyld == NXT_DYLD_ANY) { - handle = RTLD_DEFAULT; - name = "RTLD_DEFAULT"; - - } else { - handle = dyld->handle; - name = dyld->name; - } - - s = dlsym(handle, symbol); - - if (s != NULL) { - nxt_thread_log_debug("dlsym(\"%s\", \"%s\")", name, symbol); - return s; - } - - err = dlerror(); - if (err == NULL) { - err = "(null)"; - } - - nxt_thread_log_alert("dlsym(\"%s\", \"%s\") failed: %s", name, symbol, err); - - return s; -} - - -nxt_int_t -nxt_dyld_unload(nxt_dyld_t *dyld) -{ - const char *err; - - if (dlclose(dyld->handle) == 0) { - nxt_thread_log_debug("dlclose(\"%s\")", dyld->name); - return NXT_OK; - } - - err = dlerror(); - - if (err == NULL) { - err = "(null)"; - } - - nxt_thread_log_alert("dlclose(\"%s\") failed: %s", dyld->name, err); - - return NXT_ERROR; -} diff --git a/src/nxt_dyld.h b/src/nxt_dyld.h deleted file mode 100644 index 65ce1874a..000000000 --- a/src/nxt_dyld.h +++ /dev/null @@ -1,29 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) NGINX, Inc. - */ - -#ifndef _NXT_UNIX_DYLD_H_INCLUDED_ -#define _NXT_UNIX_DYLD_H_INCLUDED_ - - -typedef struct { - void *handle; - char *name; -} nxt_dyld_t; - - -#define NXT_DYLD_ANY RTLD_DEFAULT - - -#define nxt_dyld_is_valid(dyld) \ - ((dyld)->handle != NULL) - - -NXT_EXPORT nxt_int_t nxt_dyld_load(nxt_dyld_t *dyld); -NXT_EXPORT void *nxt_dyld_symbol(nxt_dyld_t *dyld, const char *symbol); -NXT_EXPORT nxt_int_t nxt_dyld_unload(nxt_dyld_t *dyld); - - -#endif /* _NXT_UNIX_DYLD_H_INCLUDED_ */ diff --git a/src/nxt_epoll_engine.c b/src/nxt_epoll_engine.c index a4d49ceb0..e1e07b6e5 100644 --- a/src/nxt_epoll_engine.c +++ b/src/nxt_epoll_engine.c @@ -47,6 +47,8 @@ static void nxt_epoll_disable(nxt_event_engine_t *engine, nxt_fd_event_t *ev); static void nxt_epoll_delete(nxt_event_engine_t *engine, nxt_fd_event_t *ev); static nxt_bool_t nxt_epoll_close(nxt_event_engine_t *engine, nxt_fd_event_t *ev); +static void nxt_epoll_cancel_changes(nxt_event_engine_t *engine, + nxt_fd_event_t *ev); static void nxt_epoll_enable_read(nxt_event_engine_t *engine, nxt_fd_event_t *ev); static void nxt_epoll_enable_write(nxt_event_engine_t *engine, @@ -126,6 +128,7 @@ const nxt_event_interface_t nxt_epoll_edge_engine = { nxt_epoll_disable, nxt_epoll_delete, nxt_epoll_close, + nxt_epoll_cancel_changes, nxt_epoll_enable_read, nxt_epoll_enable_write, nxt_epoll_disable_read, @@ -172,6 +175,7 @@ const nxt_event_interface_t nxt_epoll_level_engine = { nxt_epoll_disable, nxt_epoll_delete, nxt_epoll_close, + nxt_epoll_cancel_changes, nxt_epoll_enable_read, nxt_epoll_enable_write, nxt_epoll_disable_read, @@ -412,6 +416,47 @@ nxt_epoll_close(nxt_event_engine_t *engine, nxt_fd_event_t *ev) } +/* + * Take this event's pending changes out of the batch, so that a struct that + * is about to be freed is not dereferenced by nxt_epoll_commit_changes(). + * + * The change is dropped rather than committed: the descriptor is closed, or + * is about to be, and close() removes it from the epoll set on its own. + * Committing instead would epoll_ctl() a descriptor number that may already + * name somebody else's file. + */ + +static void +nxt_epoll_cancel_changes(nxt_event_engine_t *engine, nxt_fd_event_t *ev) +{ + nxt_epoll_change_t *change, *dst, *end; + + if (!ev->changing) { + return; + } + + dst = engine->u.epoll.changes; + end = dst + engine->u.epoll.nchanges; + + for (change = dst; change < end; change++) { + + if (change->event.data.ptr == ev) { + continue; + } + + if (dst != change) { + *dst = *change; + } + + dst++; + } + + engine->u.epoll.nchanges = (nxt_uint_t) (dst - engine->u.epoll.changes); + + ev->changing = 0; +} + + static void nxt_epoll_enable_read(nxt_event_engine_t *engine, nxt_fd_event_t *ev) { diff --git a/src/nxt_event_conn_job_sendfile.c b/src/nxt_event_conn_job_sendfile.c deleted file mode 100644 index 0f6f9353a..000000000 --- a/src/nxt_event_conn_job_sendfile.c +++ /dev/null @@ -1,259 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) NGINX, Inc. - */ - -#include - - -typedef struct { - nxt_job_t job; - nxt_buf_t *out; - size_t sent; - size_t limit; - nxt_work_handler_t ready_handler; -} nxt_job_sendfile_t; - - -static void nxt_event_conn_job_sendfile_start(nxt_task_t *task, void *obj, - void *data); -static void nxt_event_conn_job_sendfile_handler(nxt_task_t *task, void *obj, - void *data); -static void nxt_event_conn_job_sendfile_return(nxt_task_t *task, void *obj, - void *data); -static nxt_buf_t *nxt_event_conn_job_sendfile_completion(nxt_task_t *task, - nxt_conn_t *c, nxt_buf_t *b); - - -void -nxt_event_conn_job_sendfile(nxt_task_t *task, nxt_conn_t *c) -{ - nxt_fd_event_disable(task->thread->engine, &c->socket); - - /* A work item data is not used in nxt_event_conn_job_sendfile_start(). */ - nxt_event_conn_job_sendfile_start(task, c, NULL); -} - - -static void -nxt_event_conn_job_sendfile_start(nxt_task_t *task, void *obj, void *data) -{ - nxt_conn_t *c; - nxt_iobuf_t b; - nxt_job_sendfile_t *jbs; - nxt_sendbuf_coalesce_t sb; - - c = obj; - - nxt_debug(task, "event conn sendfile fd:%d", c->socket.fd); - - jbs = nxt_job_create(c->mem_pool, sizeof(nxt_job_sendfile_t)); - - if (nxt_slow_path(jbs == NULL)) { - c->write_state->error_handler(task, c, NULL); - return; - } - - c->socket.write_handler = nxt_event_conn_job_sendfile_start; - c->socket.error_handler = c->write_state->error_handler; - - jbs->job.data = c; - nxt_job_set_name(&jbs->job, "job sendfile"); - - jbs->limit = nxt_event_conn_write_limit(c); - - if (jbs->limit != 0) { - - sb.buf = c->write; - sb.iobuf = &b; - sb.nmax = 1; - sb.sync = 0; - sb.size = 0; - sb.limit = jbs->limit; - - if (nxt_sendbuf_mem_coalesce(c->socket.task, &sb) != 0 || !sb.sync) { - - jbs->job.thread_pool = c->u.thread_pool; - jbs->job.log = c->socket.log; - jbs->out = c->write; - c->write = NULL; - jbs->ready_handler = nxt_event_conn_job_sendfile_return; - - c->block_read = 1; - c->block_write = 1; - - nxt_job_start(task, &jbs->job, nxt_event_conn_job_sendfile_handler); - return; - } - } - - nxt_event_conn_job_sendfile_return(task, jbs, c); -} - - -static void -nxt_event_conn_job_sendfile_handler(nxt_task_t *task, void *obj, void *data) -{ - ssize_t ret; - nxt_buf_t *b; - nxt_bool_t first; - nxt_conn_t *c; - nxt_job_sendfile_t *jbs; - - jbs = obj; - c = data; - - nxt_debug(task, "event conn job sendfile fd:%d", c->socket.fd); - - first = c->socket.write_ready; - b = jbs->out; - - do { - ret = c->io->old_sendbuf(c, b, jbs->limit); - - if (ret == NXT_AGAIN) { - break; - } - - if (nxt_slow_path(ret == NXT_ERROR)) { - goto done; - } - - jbs->sent += ret; - jbs->limit -= ret; - - b = nxt_sendbuf_update(b, ret); - - if (b == NULL) { - goto done; - } - - if (jbs->limit == 0) { - - if (c->rate == NULL) { - jbs->limit = c->max_chunk; - goto fast; - } - - goto done; - } - - } while (c->socket.write_ready); - - if (first && task->thread->thread_pool->work_queue.head != NULL) { - goto fast; - } - -done: - - nxt_job_return(task, &jbs->job, jbs->ready_handler); - return; - -fast: - - nxt_work_set(&jbs->job.work, nxt_event_conn_job_sendfile_handler, - jbs->job.task, jbs, c); - - nxt_thread_pool_post(task->thread->thread_pool, &jbs->job.work); -} - - -static void -nxt_event_conn_job_sendfile_return(nxt_task_t *task, void *obj, void *data) -{ - size_t sent; - nxt_buf_t *b; - nxt_bool_t done; - nxt_conn_t *c; - nxt_job_sendfile_t *jbs; - - jbs = obj; - c = data; - - c->block_read = 0; - c->block_write = 0; - - sent = jbs->sent; - c->sent += sent; - - nxt_debug(task, "event conn sendfile sent:%z", sent); - - b = jbs->out; - - /* The job must be destroyed before connection error handler. */ - nxt_job_destroy(task, jbs); - - if (0 /* STUB: c->write_state->process_buffers */) { - b = nxt_event_conn_job_sendfile_completion(task, c, b); - - done = (b == NULL); - - /* Add data which might be added after sendfile job has started. */ - nxt_buf_chain_add(&b, c->write); - c->write = b; - - if (done) { - /* All data has been sent. */ - - if (b != NULL) { - /* But new data has been added. */ - nxt_event_conn_job_sendfile_start(task, c, NULL); - } - - return; - } - } - - if (sent != 0 && c->write_state->timer_autoreset) { - nxt_timer_disable(task->thread->engine, &c->write_timer); - } - - if (c->socket.error == 0 - && !nxt_event_conn_write_delayed(task->thread->engine, c, sent)) - { - nxt_conn_timer(task->thread->engine, c, c->write_state, - &c->write_timer); - - nxt_fd_event_oneshot_write(task->thread->engine, &c->socket); - } - - if (sent != 0) { - nxt_work_queue_add(c->write_work_queue, c->write_state->ready_handler, - task, c, c->socket.data); - /* - * Fall through if first operations were - * successful but the last one failed. - */ - } - - if (nxt_slow_path(c->socket.error != 0)) { - nxt_work_queue_add(c->write_work_queue, c->write_state->error_handler, - task, c, c->socket.data); - } -} - - -static nxt_buf_t * -nxt_event_conn_job_sendfile_completion(nxt_task_t *task, nxt_conn_t *c, - nxt_buf_t *b) -{ - while (b != NULL) { - - nxt_prefetch(b->next); - - if (nxt_buf_is_mem(b) && b->mem.pos != b->mem.free) { - break; - - } else if (nxt_buf_is_file(b) && b->file_pos != b->file_end) { - break; - } - - nxt_work_queue_add(c->write_work_queue, - b->completion_handler, task, b, b->parent); - - b = b->next; - } - - return b; -} diff --git a/src/nxt_event_engine.c b/src/nxt_event_engine.c index 9a37b34c4..3d5dea8a6 100644 --- a/src/nxt_event_engine.c +++ b/src/nxt_event_engine.c @@ -62,15 +62,6 @@ nxt_event_engine_create(nxt_task_t *task, engine->batch = batch; -#if 0 - if (flags & NXT_ENGINE_FIBERS) { - engine->fibers = nxt_fiber_main_create(engine); - if (engine->fibers == NULL) { - goto fibers_fail; - } - } -#endif - engine->current_work_queue = &engine->fast_work_queue; nxt_work_queue_cache_create(&engine->work_queue_cache, 0); @@ -153,12 +144,6 @@ nxt_event_engine_create(nxt_task_t *task, nxt_free(engine->signals); nxt_work_queue_cache_destroy(&engine->work_queue_cache); - nxt_free(engine->fibers); - -#if 0 -fibers_fail: -#endif - nxt_free(engine); return NULL; @@ -529,16 +514,6 @@ nxt_event_engine_start(nxt_event_engine_t *engine) thr = nxt_thread(); - if (engine->fibers) { - /* - * _setjmp() cannot be wrapped in a function since return from - * the function clobbers stack used by future _setjmp() returns. - */ - _setjmp(engine->fibers->fiber.jmp); - - /* A return point from fibers. */ - } - thr->log = engine->task.log; for ( ;; ) { @@ -547,6 +522,21 @@ nxt_event_engine_start(nxt_event_engine_t *engine) handler = nxt_event_engine_queue_pop(engine, &task, &obj, &data); if (handler == NULL) { + /* + * Every engine-local work queue is empty here, so no queued + * item can still reference a connection struct freed during + * this drain. This is the only point at which parked structs + * become reusable. + * + * engine->locked_work_queue is deliberately not covered: it is + * the cross-thread post queue, drained separately by + * nxt_event_engine_post_handler() after poll, and reading its + * head here would race its spinlock. Nothing posted through + * nxt_event_engine_post() carries an nxt_conn_t today, so the + * barrier holds -- but a future post that does would need this + * revisited rather than assumed covered. + */ + nxt_conn_recycle_pending(engine); break; } diff --git a/src/nxt_event_engine.h b/src/nxt_event_engine.h index 8f5666851..4d85786e1 100644 --- a/src/nxt_event_engine.h +++ b/src/nxt_event_engine.h @@ -67,6 +67,37 @@ typedef struct { nxt_bool_t (*close)(nxt_event_engine_t *engine, nxt_fd_event_t *ev); + /* + * Drop every change still pending for this event from the engine's + * change batch. + * + * The batching engines hold the change by pointer and dereference it + * when the batch is committed, at the top of the next poll at the + * latest. Nothing else keeps the struct the pointer names alive until + * then, so anything that frees a struct holding an nxt_fd_event_t has to + * take its pending changes out of the batch first. + * + * The change is dropped, not committed: the descriptor is being closed + * or is closed already, so committing would act on a descriptor number + * that may already name somebody else's file. A caller that wants the + * kernel told must delete the event before it closes, which is what + * nxt_fd_event_close() is for. + * + * For epoll and kqueue that leaves nothing behind: the kernel drops a + * descriptor from its set when the descriptor is closed. The poll, + * devpoll and pollset engines instead keep the set in user space, in + * ->fd_hash and the engine's own array, and only an applied delete + * removes an entry from it. Dropping a pending delete there leaves a + * stale entry pointing at the freed event -- the same entry those + * engines are already left with whenever an event is freed without a + * delete, which is the common case for a port. Not committing is still + * the safer half: committing dereferences the freed event first. + * + * Runs on the engine's own thread, like every other operation here. + */ + void (*cancel_changes)(nxt_event_engine_t *engine, + nxt_fd_event_t *ev); + /* * Add a file descriptor to an event set and enable the most effective * read event notification method provided by underlying event facility. @@ -363,6 +394,10 @@ void nxt_fd_event_hash_destroy(nxt_lvlhsh_t *lvlhsh); (engine)->event.close(engine, ev) +#define nxt_fd_event_cancel_changes(engine, ev) \ + (engine)->event.cancel_changes(engine, ev) + + #define nxt_fd_event_enable_read(engine, ev) \ (engine)->event.enable_read(engine, ev) @@ -407,9 +442,6 @@ void nxt_fd_event_hash_destroy(nxt_lvlhsh_t *lvlhsh); (engine)->event.enable_accept(engine, ev) -#define NXT_ENGINE_FIBERS 1 - - typedef struct { nxt_fd_t fds[2]; nxt_fd_event_t event; @@ -465,8 +497,6 @@ struct nxt_event_engine_s { nxt_event_signals_t *signals; - nxt_fiber_main_t *fibers; - /* The engine ID, the main engine has ID 0. */ uint32_t id; @@ -484,6 +514,9 @@ struct nxt_event_engine_s { nxt_queue_t active_connections; nxt_array_t *mem_cache; + nxt_conn_t *free_connections; + nxt_conn_t *pending_connections; + nxt_atomic_uint_t accepted_conns_cnt; nxt_atomic_uint_t idle_conns_cnt; nxt_atomic_uint_t active_conns_cnt; diff --git a/src/nxt_eventport_engine.c b/src/nxt_eventport_engine.c index 7ac7f44de..a92c96c77 100644 --- a/src/nxt_eventport_engine.c +++ b/src/nxt_eventport_engine.c @@ -23,6 +23,8 @@ static void nxt_eventport_disable(nxt_event_engine_t *engine, nxt_fd_event_t *ev); static nxt_bool_t nxt_eventport_close(nxt_event_engine_t *engine, nxt_fd_event_t *ev); +static void nxt_eventport_cancel_changes(nxt_event_engine_t *engine, + nxt_fd_event_t *ev); static void nxt_eventport_enable_read(nxt_event_engine_t *engine, nxt_fd_event_t *ev); static void nxt_eventport_enable_write(nxt_event_engine_t *engine, @@ -63,6 +65,7 @@ const nxt_event_interface_t nxt_eventport_engine = { nxt_eventport_disable, nxt_eventport_disable, nxt_eventport_close, + nxt_eventport_cancel_changes, nxt_eventport_enable_read, nxt_eventport_enable_write, nxt_eventport_disable_read, @@ -293,6 +296,47 @@ nxt_eventport_disable_event(nxt_event_engine_t *engine, nxt_fd_event_t *ev) } +/* + * Take this event's pending changes out of the batch, so that a struct that + * is about to be freed is not dereferenced by nxt_eventport_commit_changes(). + * + * The change is dropped rather than committed: the descriptor is closed, or + * is about to be, so the change would name a descriptor number that may + * already belong to somebody else. + */ + +static void +nxt_eventport_cancel_changes(nxt_event_engine_t *engine, nxt_fd_event_t *ev) +{ + nxt_eventport_change_t *change, *dst, *end; + + if (!ev->changing) { + return; + } + + dst = engine->u.eventport.changes; + end = dst + engine->u.eventport.nchanges; + + for (change = dst; change < end; change++) { + + if (change->event == ev) { + continue; + } + + if (dst != change) { + *dst = *change; + } + + dst++; + } + + engine->u.eventport.nchanges = (nxt_uint_t) + (dst - engine->u.eventport.changes); + + ev->changing = 0; +} + + static nxt_int_t nxt_eventport_commit_changes(nxt_event_engine_t *engine) { diff --git a/src/nxt_fiber.c b/src/nxt_fiber.c deleted file mode 100644 index d6cac8930..000000000 --- a/src/nxt_fiber.c +++ /dev/null @@ -1,465 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) NGINX, Inc. - */ - -#include - - -static char *nxt_fiber_create_stack(nxt_task_t *task, nxt_fiber_t *fib); -static void nxt_fiber_switch_stack(nxt_fiber_t *fib, jmp_buf *parent); -static void nxt_fiber_switch_handler(nxt_task_t *task, void *obj, void *data); -static void nxt_fiber_switch(nxt_task_t *task, nxt_fiber_t *fib); -static void nxt_fiber_timer_handler(nxt_task_t *task, void *obj, void *data); - - -#define nxt_fiber_enqueue(thr, task, fib) \ - nxt_work_queue_add(&(thr)->engine->fast_work_queue, \ - nxt_fiber_switch_handler, task, fib, NULL) - - -nxt_fiber_main_t * -nxt_fiber_main_create(nxt_event_engine_t *engine) -{ - nxt_fiber_main_t *fm; - - fm = nxt_zalloc(sizeof(nxt_fiber_main_t)); - if (nxt_slow_path(fm == NULL)) { - return NULL; - } - - fm->engine = engine; - fm->stack_size = 512 * 1024 - nxt_pagesize; - fm->idle = NULL; - - return fm; -} - - -nxt_int_t -nxt_fiber_create(nxt_fiber_start_t start, void *data, size_t stack) -{ - int ret; - jmp_buf parent; - nxt_fid_t fid; - nxt_fiber_t *fib; - nxt_thread_t *thr; - nxt_fiber_main_t *fm; - - thr = nxt_thread(); - fm = thr->engine->fibers; - - fid = ++fm->fid; - - if (fid == 0) { - fid = ++fm->fid; - } - - fib = fm->idle; - - if (fib != NULL) { - fm->idle = fib->next; - fib->fid = fid; - fib->start = start; - fib->data = data; - fib->main = fm; - - fib->task.thread = thr; - fib->task.log = thr->log; - fib->task.ident = nxt_task_next_ident(); - - nxt_debug(&fib->task, "fiber create cached: %PF", fib->fid); - - nxt_fiber_enqueue(thr, &fm->engine->task, fib); - - return NXT_OK; - } - - nxt_log_debug(thr->log, "fiber create"); - - fib = nxt_malloc(sizeof(nxt_fiber_t)); - if (nxt_slow_path(fib == NULL)) { - return NXT_ERROR; - } - - fib->fid = fid; - fib->start = start; - fib->data = data; - fib->stack_size = fm->stack_size; - fib->main = fm; - - fib->task.thread = thr; - fib->task.log = thr->log; - fib->task.ident = nxt_task_next_ident(); - - fib->stack = nxt_fiber_create_stack(&fib->task, fib); - - if (nxt_fast_path(fib->stack != NULL)) { - - if (_setjmp(parent) != 0) { - nxt_log_debug(thr->log, "fiber create: %PF", fib->fid); - return NXT_OK; - } - - nxt_fiber_switch_stack(fib, &parent); - /* It does not return if the switch was successful. */ - } - - ret = munmap(fib->stack - nxt_pagesize, fib->stack_size + nxt_pagesize); - - if (nxt_slow_path(ret != 0)) { - nxt_log_alert(thr->log, "munmap() failed %E", nxt_errno); - } - - nxt_free(fib); - - return NXT_ERROR; -} - - -#if (NXT_LINUX) - -static char * -nxt_fiber_create_stack(nxt_task_t *task, nxt_fiber_t *fib) -{ - char *s; - size_t size; - - size = fib->stack_size + nxt_pagesize; - - s = mmap(NULL, size, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANON | MAP_GROWSDOWN, -1, 0); - - if (nxt_slow_path(s == MAP_FAILED)) { - nxt_alert(task, "fiber stack " - "mmap(%uz, MAP_PRIVATE|MAP_ANON|MAP_GROWSDOWN) failed %E", - size, nxt_errno); - - return NULL; - } - - if (nxt_slow_path(mprotect(s, nxt_pagesize, PROT_NONE) != 0)) { - nxt_alert(task, "fiber stack mprotect(%uz, PROT_NONE) failed %E", - size, nxt_errno); - - return NULL; - } - - s += nxt_pagesize; - - nxt_debug(task, "fiber stack mmap: %p", s); - - return s; -} - -#else /* Generic version. */ - -static char * -nxt_fiber_create_stack(nxt_task_t *task, nxt_fiber_t *fib) -{ - char *s; - size_t size; - - size = fib->stack_size + nxt_pagesize; - - s = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); - - if (nxt_slow_path(s == MAP_FAILED)) { - nxt_alert(task, "fiber stack mmap(%uz, MAP_PRIVATE|MAP_ANON) failed %E", - size, nxt_errno); - - return NULL; - } - - if (nxt_slow_path(mprotect(s, nxt_pagesize, PROT_NONE) != 0)) { - nxt_alert(task, "fiber stack mprotect(%uz, PROT_NONE) failed %E", - size, nxt_errno); - - return NULL; - } - - s += nxt_pagesize; - - nxt_debug(task, "fiber stack mmap: %p", s); - - return s; -} - -#endif - - -#if (NXT_LINUX && NXT_64BIT) - -/* - * Linux 64-bit ucontext version. 64-bit glibc makecontext() passes - * pointers as signed int's. The bug has been fixed in glibc 2.8. - */ - -static void nxt_fiber_trampoline(uint32_t fh, uint32_t fl, uint32_t ph, - uint32_t pl); - - -static void -nxt_fiber_switch_stack(nxt_fiber_t *fib, jmp_buf *parent) -{ - ucontext_t uc; - - nxt_debug(&fib->task, "fiber switch to stack: %p", fib->stack); - - if (nxt_slow_path(getcontext(&uc) != 0)) { - nxt_alert(&fib->task, "getcontext() failed"); - return; - } - - uc.uc_link = NULL; - uc.uc_stack.ss_sp = fib->stack; - uc.uc_stack.ss_size = fib->stack_size; - - makecontext(&uc, (void (*)(void)) nxt_fiber_trampoline, 4, - (uint32_t) ((uintptr_t) fib >> 32), - (uint32_t) ((uintptr_t) fib & 0xFFFFFFFF), - (uint32_t) ((uintptr_t) parent >> 32), - (uint32_t) ((uintptr_t) parent & 0xFFFFFFFF)); - - setcontext(&uc); - - nxt_alert(&fib->task, "setcontext() failed"); -} - - -static void -nxt_fiber_trampoline(uint32_t fh, uint32_t fl, uint32_t ph, uint32_t pl) -{ - jmp_buf *parent; - nxt_task_t *task; - nxt_fiber_t *fib; - - fib = (nxt_fiber_t *) (((uintptr_t) fh << 32) + fl); - parent = (jmp_buf *) (((uintptr_t) ph << 32) + pl); - - task = &fib->task; - - if (_setjmp(fib->jmp) == 0) { - nxt_debug(task, "fiber return to parent stack"); - - nxt_fiber_enqueue(task->thread, task, fib); - - _longjmp(*parent, 1); - - nxt_unreachable(); - } - - nxt_debug(task, "fiber start"); - - fib->start(fib->data); - - nxt_fiber_exit(task, &fib->main->fiber, NULL); - - nxt_unreachable(); -} - -#elif (NXT_HAVE_UCONTEXT) - -/* Generic ucontext version. */ - -static void nxt_fiber_trampoline(nxt_fiber_t *fib, jmp_buf *parent); - - -static void -nxt_fiber_switch_stack(nxt_fiber_t *fib, jmp_buf *parent) -{ - ucontext_t uc; - - nxt_debug(&fib->task, "fiber switch to stack: %p", fib->stack); - - if (nxt_slow_path(getcontext(&uc) != 0)) { - nxt_alert(&fib->task, "getcontext() failed"); - return; - } - - uc.uc_link = NULL; - uc.uc_stack.ss_sp = fib->stack; - uc.uc_stack.ss_size = fib->stack_size; - - makecontext(&uc, (void (*)(void)) nxt_fiber_trampoline, 2, fib, parent); - - setcontext(&uc); - -#if !(NXT_SOLARIS) - /* Solaris declares setcontext() as __NORETURN. */ - - nxt_alert(&fib->task, "setcontext() failed"); -#endif -} - - -static void -nxt_fiber_trampoline(nxt_fiber_t *fib, jmp_buf *parent) -{ - nxt_task_t *task; - - task = &fib->task; - - if (_setjmp(fib->jmp) == 0) { - nxt_debug(task, "fiber return to parent stack"); - - nxt_fiber_enqueue(task->thread, task, fib); - - _longjmp(*parent, 1); - - nxt_unreachable(); - } - - nxt_debug(task, "fiber start"); - - fib->start(fib->data); - - nxt_fiber_exit(task, &fib->main->fiber, NULL); - - nxt_unreachable(); -} - -#else - -#error No ucontext(3) interface. - -#endif - - -static void -nxt_fiber_switch_handler(nxt_task_t *task, void *obj, void *data) -{ - nxt_fiber_t *fib; - - fib = obj; - - nxt_fiber_switch(task, fib); - nxt_unreachable(); -} - - -static void -nxt_fiber_switch(nxt_task_t *task, nxt_fiber_t *fib) -{ - nxt_debug(task, "fiber switch: %PF", fib->fid); - - task->thread->fiber = fib; - - _longjmp(fib->jmp, 1); - - nxt_unreachable(); -} - - -nxt_fiber_t * -nxt_fiber_self(nxt_thread_t *thr) -{ - return (nxt_fast_path(thr != NULL)) ? thr->fiber : NULL; -} - - -void -nxt_fiber_yield(nxt_task_t *task) -{ - nxt_fiber_t *fib; - - fib = task->thread->fiber; - - if (_setjmp(fib->jmp) == 0) { - - nxt_debug(task, "fiber yield"); - - nxt_fiber_enqueue(task->thread, &fib->main->engine->task, fib); - - nxt_fiber_switch(task, &fib->main->fiber); - - nxt_unreachable(); - } - - nxt_debug(task, "fiber yield return"); -} - - -void -nxt_fiber_sleep(nxt_task_t *task, nxt_msec_t timeout) -{ - nxt_fiber_t *fib; - - fib = task->thread->fiber; - - fib->timer.work_queue = &task->thread->engine->fast_work_queue; - fib->timer.handler = nxt_fiber_timer_handler; - fib->timer.log = &nxt_main_log; - - task = &fib->task; - - nxt_timer_add(task->thread->engine, &fib->timer, timeout); - - if (_setjmp(fib->jmp) == 0) { - - nxt_debug(task, "fiber sleep: %T", timeout); - - nxt_fiber_switch(task, &fib->main->fiber); - - nxt_unreachable(); - } - - nxt_debug(task, "fiber sleep return"); -} - - -static void -nxt_fiber_timer_handler(nxt_task_t *task, void *obj, void *data) -{ - nxt_fiber_t *fib; - nxt_timer_t *ev; - - ev = obj; - - nxt_debug(task, "fiber timer handler"); - - fib = nxt_timer_data(ev, nxt_fiber_t, timer); - - nxt_fiber_switch(task, fib); - - nxt_unreachable(); -} - - -void -nxt_fiber_wait(nxt_task_t *task) -{ - nxt_fiber_t *fib; - - fib = task->thread->fiber; - - if (_setjmp(fib->jmp) == 0) { - nxt_debug(task, "fiber wait"); - - nxt_fiber_switch(task, &fib->main->fiber); - - nxt_unreachable(); - } - - nxt_debug(task, "fiber wait return"); -} - - -void -nxt_fiber_exit(nxt_task_t *task, nxt_fiber_t *next, void *data) -{ - nxt_fiber_t *fib; - - fib = task->thread->fiber; - - nxt_debug(task, "fiber exit"); - - /* TODO: limit idle fibers. */ - fib->next = fib->main->idle; - fib->main->idle = fib; - - nxt_fiber_switch(task, next); - - nxt_unreachable(); -} diff --git a/src/nxt_fiber.h b/src/nxt_fiber.h deleted file mode 100644 index 7f1beace6..000000000 --- a/src/nxt_fiber.h +++ /dev/null @@ -1,57 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) NGINX, Inc. - */ - -#ifndef _NXT_FIBER_H_INCLUDED_ -#define _NXT_FIBER_H_INCLUDED_ - - -typedef struct nxt_fiber_main_s nxt_fiber_main_t; -typedef void (*nxt_fiber_start_t)(void *data); - - -typedef uint32_t nxt_fid_t; -#define nxt_fiber_id(f) (f)->fid; - - -typedef struct nxt_fiber_s nxt_fiber_t; - -struct nxt_fiber_s { - jmp_buf jmp; - nxt_fid_t fid; - nxt_fiber_start_t start; - void *data; - char *stack; - size_t stack_size; - nxt_err_t err; - - nxt_task_t task; - - nxt_fiber_main_t *main; - nxt_fiber_t *next; - - nxt_timer_t timer; -}; - - -struct nxt_fiber_main_s { - nxt_fiber_t fiber; - nxt_fiber_t *idle; - nxt_event_engine_t *engine; - size_t stack_size; - nxt_fid_t fid; -}; - - -nxt_fiber_main_t *nxt_fiber_main_create(nxt_event_engine_t *engine); -nxt_int_t nxt_fiber_create(nxt_fiber_start_t start, void *data, size_t stack); -void nxt_fiber_yield(nxt_task_t *task); -void nxt_fiber_sleep(nxt_task_t *task, nxt_msec_t timeout); -void nxt_fiber_wait(nxt_task_t *task); -void nxt_fiber_exit(nxt_task_t *task, nxt_fiber_t *next, void *data); -NXT_EXPORT nxt_fiber_t *nxt_fiber_self(nxt_thread_t *thr); - - -#endif /* _NXT_FIBER_H_INCLUDED_ */ diff --git a/src/nxt_file.c b/src/nxt_file.c index 66edb1adc..85574de35 100644 --- a/src/nxt_file.c +++ b/src/nxt_file.c @@ -325,83 +325,114 @@ nxt_file_set_access(nxt_file_name_t *name, nxt_file_access_t access) } +/* + * Resolves the user and group names with the reentrant calls. An id stays + * -1 when its name is NULL. The buffer grows on ERANGE, up to 1M. + */ + +#define NXT_FILE_OWNER_BUF_MAX (1024 * 1024) + + nxt_int_t -nxt_file_chown(nxt_file_name_t *name, const char *owner, const char *group) +nxt_file_owner_ids(const char *owner, const char *group, uid_t *uid, + gid_t *gid) { - int err; - char *buf; - long bufsize; - gid_t gid = ~0; - uid_t uid = ~0; + int err; + long size; + char *buf; + struct group grp, *gr; + struct passwd pwd, *pw; - if (owner == NULL && group == NULL) { - return NXT_OK; - } + *uid = (uid_t) -1; + *gid = (gid_t) -1; if (owner != NULL) { - struct passwd pwd, *result; - - bufsize = sysconf(_SC_GETPW_R_SIZE_MAX); - if (bufsize == -1) { - bufsize = 32768; + /* sysconf() returns -1 when the system has no limit. */ + size = sysconf(_SC_GETPW_R_SIZE_MAX); + if (size <= 0) { + size = 16384; } - buf = nxt_malloc(bufsize); - if (buf == NULL) { - return NXT_ERROR; + for ( ;; ) { + buf = nxt_malloc(size); + if (buf == NULL) { + return NXT_ERROR; + } + + err = getpwnam_r(owner, &pwd, buf, size, &pw); + if (err != ERANGE || size >= NXT_FILE_OWNER_BUF_MAX) { + break; + } + + nxt_free(buf); + size *= 2; } - err = getpwnam_r(owner, &pwd, buf, bufsize, &result); - if (result == NULL) { + if (pw == NULL) { nxt_thread_log_alert("getpwnam_r(\"%s\", ...) failed %E %s", - owner, nxt_errno, + owner, err, err == 0 ? "(User not found)" : ""); - goto out_err_free; + nxt_free(buf); + return NXT_ERROR; } - uid = pwd.pw_uid; + *uid = pwd.pw_uid; nxt_free(buf); } if (group != NULL) { - struct group grp, *result; - - bufsize = sysconf(_SC_GETGR_R_SIZE_MAX); - if (bufsize == -1) { - bufsize = 32768; + size = sysconf(_SC_GETGR_R_SIZE_MAX); + if (size <= 0) { + size = 16384; } - buf = nxt_malloc(bufsize); - if (buf == NULL) { - return NXT_ERROR; + for ( ;; ) { + buf = nxt_malloc(size); + if (buf == NULL) { + return NXT_ERROR; + } + + err = getgrnam_r(group, &grp, buf, size, &gr); + if (err != ERANGE || size >= NXT_FILE_OWNER_BUF_MAX) { + break; + } + + nxt_free(buf); + size *= 2; } - err = getgrnam_r(group, &grp, buf, bufsize, &result); - if (result == NULL) { + if (gr == NULL) { nxt_thread_log_alert("getgrnam_r(\"%s\", ...) failed %E %s", - group, nxt_errno, + group, err, err == 0 ? "(Group not found)" : ""); - goto out_err_free; + nxt_free(buf); + return NXT_ERROR; } - gid = grp.gr_gid; + *gid = grp.gr_gid; nxt_free(buf); } + return NXT_OK; +} + + +nxt_int_t +nxt_file_chown(nxt_file_name_t *name, uid_t uid, gid_t gid) +{ + if (uid == (uid_t) -1 && gid == (gid_t) -1) { + return NXT_OK; + } + if (nxt_fast_path(chown((const char *) name, uid, gid) == 0)) { return NXT_OK; } nxt_thread_log_alert("chown(\"%FN\", %l, %l) failed %E", name, - owner != NULL ? (long) uid : -1, - group != NULL ? (long) gid : -1, nxt_errno); - - return NXT_ERROR; - -out_err_free: - nxt_free(buf); + uid != (uid_t) -1 ? (long) uid : -1, + gid != (gid_t) -1 ? (long) gid : -1, nxt_errno); return NXT_ERROR; } @@ -426,6 +457,152 @@ nxt_file_rename(nxt_file_name_t *old_name, nxt_file_name_t *new_name) } +/* + * The errors of a file system that cannot flush a descriptor in the asked + * way. ENOTSUP and EOPNOTSUPP are one value on Linux and FreeBSD, and two + * values on macOS. + */ +static nxt_bool_t +nxt_file_sync_unsupported(nxt_err_t err) +{ + switch (err) { + + case NXT_EINVAL: + case NXT_EOPNOTSUPP: +#if (ENOTSUP != EOPNOTSUPP) + case ENOTSUP: +#endif + case ENOTTY: + return 1; + + default: + return 0; + } +} + + +#if defined(F_FULLFSYNC) + +/* + * Runs a macOS flush command. EINTR repeats the call. If the file system + * does not support the command, for example an SMB mount with ENOTSUP, + * fsync() does the flush and its result counts. Any other error, such as + * EIO, is the result: the flush failed. + */ +static int +nxt_file_fcntl_sync(nxt_fd_t fd, int cmd) +{ + nxt_err_t err; + + for ( ;; ) { + if (fcntl(fd, cmd) == 0) { + return 0; + } + + err = nxt_errno; + + if (err != NXT_EINTR) { + break; + } + } + + if (nxt_file_sync_unsupported(err)) { + return fsync(fd); + } + + /* errno still holds the error of fcntl(). */ + return -1; +} + +#endif + + +/* + * On macOS, fsync(2) sends the data to the drive, but the drive can keep it + * in its cache, so a power loss can still lose it. F_FULLFSYNC also + * flushes the cache of the drive, and it waits for the whole queue of the + * device. F_BARRIERFSYNC only orders the data before any later write to + * the device. So a file gets the barrier, and the directory gets the full + * flush after the rename. That one full flush makes the data and the + * rename durable. Without barrier support fsync() takes the place of the + * barrier, and only the later full flush orders the data, as before. + * Elsewhere both are fsync(). + */ +static int +nxt_file_fsync(nxt_fd_t fd, nxt_bool_t full) +{ +#if defined(F_FULLFSYNC) +#if defined(F_BARRIERFSYNC) + if (!full) { + return nxt_file_fcntl_sync(fd, F_BARRIERFSYNC); + } +#endif + return nxt_file_fcntl_sync(fd, F_FULLFSYNC); +#else + return fsync(fd); +#endif +} + + +nxt_int_t +nxt_file_sync(nxt_task_t *task, nxt_file_t *file) +{ + nxt_debug(task, "fsync(%FD, \"%FN\")", file->fd, file->name); + + if (nxt_fast_path(nxt_file_fsync(file->fd, 0) == 0)) { + return NXT_OK; + } + + file->error = nxt_errno; + + nxt_alert(task, "fsync(%FD, \"%FN\") failed %E", + file->fd, file->name, file->error); + + return NXT_ERROR; +} + + +nxt_int_t +nxt_file_dir_sync(nxt_task_t *task, nxt_file_name_t *name) +{ + int fd; + nxt_int_t ret; + nxt_err_t err; + + fd = open((char *) name, O_RDONLY | NXT_FILE_DIR_SYNC_FLAGS); + if (nxt_slow_path(fd == -1)) { + nxt_alert(task, "open(\"%FN\", O_RDONLY) failed %E", name, nxt_errno); + return NXT_ERROR; + } + + nxt_debug(task, "fsync(%FD, \"%FN\") directory", fd, name); + + ret = NXT_OK; + + if (nxt_slow_path(nxt_file_fsync(fd, 1) != 0)) { + err = nxt_errno; + + /* + * Some file systems refuse to flush a directory descriptor at all; + * the rename() has still happened, so this is not a store failure. + */ + if (nxt_file_sync_unsupported(err)) { + nxt_debug(task, "fsync(%FD, \"%FN\") directory unsupported %E", + fd, name, err); + + } else { + nxt_alert(task, "fsync(%FD, \"%FN\") directory failed %E", + fd, name, err); + ret = NXT_ERROR; + } + } + + nxt_fd_close(fd); + + return ret; +} + + /* * ioctl(FIONBIO) sets a non-blocking mode using one syscall, * thereas fcntl(F_SETFL, O_NONBLOCK) needs to learn the current state @@ -621,7 +798,8 @@ nxt_file_fclose(nxt_task_t *task, FILE *fp) /* * nxt_file_redirect() redirects the file to the fd descriptor. - * Then the fd descriptor is closed. + * Then the fd descriptor is closed -- on the failing paths as well, so that + * the caller can hand the descriptor over unconditionally. */ nxt_int_t @@ -632,6 +810,14 @@ nxt_file_redirect(nxt_file_t *file, nxt_fd_t fd) if (dup2(fd, file->fd) == -1) { nxt_thread_log_alert("dup2(%FD, %FD, \"%FN\") failed %E", fd, file->fd, file->name, nxt_errno); + + /* + * The descriptor is consumed whatever the outcome, as the comment + * above states: callers hand it over and never close it on success, + * so failing without closing it here leaked it instead. + */ + (void) close(fd); + return NXT_ERROR; } diff --git a/src/nxt_file.h b/src/nxt_file.h index 0c03a5f90..ed98212e7 100644 --- a/src/nxt_file.h +++ b/src/nxt_file.h @@ -141,11 +141,44 @@ NXT_EXPORT nxt_int_t nxt_file_openat2(nxt_task_t *task, nxt_file_t *file, #endif /* NXT_HAVE_OPENAT2 */ +/* + * O_DIRECTORY, where the platform has it, keeps nxt_file_dir_sync() from + * opening something that is not a directory. It is not in POSIX, so the + * flag is optional. + */ +#if defined(O_DIRECTORY) +#define NXT_FILE_DIR_SYNC_FLAGS O_DIRECTORY +#else +#define NXT_FILE_DIR_SYNC_FLAGS 0 +#endif + + +/* + * O_NOFOLLOW, where the platform has it, keeps a create from opening a + * symbolic link left in place of the file. O_EXCL already refuses one -- + * POSIX has open() fail when O_CREAT and O_EXCL are set and the path names + * a link -- so this is the belt to that pair of braces, and optional. + */ +#if defined(O_NOFOLLOW) +#define NXT_FILE_NOFOLLOW O_NOFOLLOW +#else +#define NXT_FILE_NOFOLLOW 0 +#endif + + /* The file creation modes. */ #define NXT_FILE_CREATE_OR_OPEN O_CREAT #define NXT_FILE_OPEN 0 #define NXT_FILE_TRUNCATE (O_CREAT | O_TRUNC) +/* + * Create the file or fail: never open something that is already there, and + * never follow a link to it. For a path an unprivileged user could have + * got to first -- see nxt_main_file_store(). + */ +#define NXT_FILE_CREATE_EXCLUSIVE \ + (O_CREAT | O_EXCL | NXT_FILE_NOFOLLOW) + /* The file access rights. */ #define NXT_FILE_DEFAULT_ACCESS 0644 #define NXT_FILE_OWNER_ACCESS 0600 @@ -177,11 +210,25 @@ NXT_EXPORT nxt_int_t nxt_file_info(nxt_file_t *file, nxt_file_info_t *fi); NXT_EXPORT nxt_int_t nxt_file_delete(nxt_file_name_t *name); NXT_EXPORT nxt_int_t nxt_file_set_access(nxt_file_name_t *name, nxt_file_access_t access); -NXT_EXPORT nxt_int_t nxt_file_chown(nxt_file_name_t *name, const char *owner, - const char *group); +NXT_EXPORT nxt_int_t nxt_file_owner_ids(const char *owner, const char *group, + uid_t *uid, gid_t *gid); +NXT_EXPORT nxt_int_t nxt_file_chown(nxt_file_name_t *name, uid_t uid, + gid_t gid); NXT_EXPORT nxt_int_t nxt_file_rename(nxt_file_name_t *old_name, nxt_file_name_t *new_name); +/* + * Flush an open file, respectively a directory named by path, to stable + * storage. A durable replace is: write the temporary file, nxt_file_sync() + * it, nxt_file_rename() it over the destination, then nxt_file_dir_sync() + * the directory that holds them -- without the last step the rename itself + * may not survive a power loss. On macOS, nxt_file_sync() only orders the + * data before the rename, and nxt_file_dir_sync() makes both durable. + */ +NXT_EXPORT nxt_int_t nxt_file_sync(nxt_task_t *task, nxt_file_t *file); +NXT_EXPORT nxt_int_t nxt_file_dir_sync(nxt_task_t *task, + nxt_file_name_t *name); + NXT_EXPORT nxt_int_t nxt_fd_nonblocking(nxt_task_t *task, nxt_fd_t fd); NXT_EXPORT nxt_int_t nxt_fd_blocking(nxt_task_t *task, nxt_fd_t fd); NXT_EXPORT ssize_t nxt_fd_write(nxt_fd_t fd, u_char *buf, size_t size); diff --git a/src/nxt_gmtime.c b/src/nxt_gmtime.c index 9c3fa190c..01b6840f4 100644 --- a/src/nxt_gmtime.c +++ b/src/nxt_gmtime.c @@ -7,19 +7,44 @@ #include -/* The function is valid for positive nxt_time_t only. */ +/* + * The function is valid for any nxt_time_t that lands on a calendar date + * from year 1 onward -- that is the whole range of a 32-bit time_t, and + * every pre-epoch timestamp a filesystem can hold. + */ void nxt_gmtime(nxt_time_t s, struct tm *tm) { nxt_int_t yday; nxt_uint_t daytime, mday, mon, year, days, leap; + nxt_time_t dayno, secs; + + dayno = s / 86400; + secs = s % 86400; + + if (secs < 0) { + /* + * C integer division truncates toward zero, but a calendar floors: + * for a pre-epoch time the remainder comes out negative and the day + * number one too high. + */ + secs += 86400; + dayno--; + } - days = (nxt_uint_t) (s / 86400); - daytime = (nxt_uint_t) (s % 86400); + daytime = (nxt_uint_t) secs; /* January 1, 1970 was Thursday. */ - tm->tm_wday = (4 + days) % 7; + tm->tm_wday = (nxt_int_t) (((dayno + 4) % 7 + 7) % 7); + + /* + * A pre-epoch day number wraps when it is made unsigned, but every step + * below is modular addition and the result lands back in range, because + * the Gauss' formula counts from March 1, 1 BCE and so is positive for + * every date this function accepts. + */ + days = (nxt_uint_t) dayno; /* The algorithm based on Gauss' formula. */ diff --git a/src/nxt_gnutls.c b/src/nxt_gnutls.c deleted file mode 100644 index aab4699c6..000000000 --- a/src/nxt_gnutls.c +++ /dev/null @@ -1,742 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) NGINX, Inc. - */ - -#include -#include - - -typedef struct { - gnutls_session_t session; - - uint8_t times; /* 2 bits */ - uint8_t no_shutdown; /* 1 bit */ - - nxt_buf_mem_t buffer; -} nxt_gnutls_conn_t; - - -typedef struct { - gnutls_priority_t ciphers; - gnutls_certificate_credentials_t certificate; -} nxt_gnutls_ctx_t; - - - -#if (NXT_HAVE_GNUTLS_SET_TIME) -time_t nxt_gnutls_time(time_t *tp); -#endif -static nxt_int_t nxt_gnutls_server_init(nxt_ssltls_conf_t *conf); -static nxt_int_t nxt_gnutls_set_ciphers(nxt_ssltls_conf_t *conf); - -static void nxt_gnutls_conn_init(nxt_thread_t *thr, nxt_ssltls_conf_t *conf, - nxt_event_conn_t *c); -static void nxt_gnutls_session_cleanup(void *data); -static ssize_t nxt_gnutls_pull(gnutls_transport_ptr_t data, void *buf, - size_t size); -static ssize_t nxt_gnutls_push(gnutls_transport_ptr_t data, const void *buf, - size_t size); -#if (NXT_HAVE_GNUTLS_VEC_PUSH) -static ssize_t nxt_gnutls_vec_push(gnutls_transport_ptr_t data, - const giovec_t *iov, int iovcnt); -#endif -static void nxt_gnutls_conn_handshake(nxt_thread_t *thr, void *obj, void *data); -static void nxt_gnutls_conn_io_read(nxt_thread_t *thr, void *obj, void *data); -static ssize_t nxt_gnutls_conn_io_write_chunk(nxt_thread_t *thr, - nxt_event_conn_t *c, nxt_buf_t *b, size_t limit); -static ssize_t nxt_gnutls_conn_io_send(nxt_event_conn_t *c, void *buf, - size_t size); -static void nxt_gnutls_conn_io_shutdown(nxt_thread_t *thr, void *obj, - void *data); -static nxt_int_t nxt_gnutls_conn_test_error(nxt_thread_t *thr, - nxt_event_conn_t *c, ssize_t err, nxt_work_handler_t handler); -static void nxt_cdecl nxt_gnutls_conn_log_error(nxt_event_conn_t *c, - ssize_t err, const char *fmt, ...); -static nxt_uint_t nxt_gnutls_log_error_level(nxt_event_conn_t *c, ssize_t err); -static void nxt_cdecl nxt_gnutls_log_error(nxt_uint_t level, nxt_log_t *log, - int err, const char *fmt, ...); - - -const nxt_ssltls_lib_t nxt_gnutls_lib = { - nxt_gnutls_server_init, - NULL, -}; - - -static nxt_event_conn_io_t nxt_gnutls_event_conn_io = { - NULL, - NULL, - - nxt_gnutls_conn_io_read, - NULL, - NULL, - - nxt_event_conn_io_write, - nxt_gnutls_conn_io_write_chunk, - NULL, - NULL, - nxt_gnutls_conn_io_send, - - nxt_gnutls_conn_io_shutdown, -}; - - -static nxt_int_t -nxt_gnutls_start(void) -{ - int ret; - static nxt_bool_t started; - - if (nxt_fast_path(started)) { - return NXT_OK; - } - - started = 1; - - /* TODO: gnutls_global_deinit */ - - ret = gnutls_global_init(); - if (ret != GNUTLS_E_SUCCESS) { - nxt_gnutls_log_error(NXT_LOG_ALERT, nxt_thread_log(), ret, - "gnutls_global_init() failed"); - return NXT_ERROR; - } - - nxt_thread_log_error(NXT_LOG_INFO, "GnuTLS version: %s", - gnutls_check_version(NULL)); - -#if (NXT_HAVE_GNUTLS_SET_TIME) - gnutls_global_set_time_function(nxt_gnutls_time); -#endif - - return NXT_OK; -} - - -#if (NXT_HAVE_GNUTLS_SET_TIME) - -/* GnuTLS 2.12.0 */ - -time_t -nxt_gnutls_time(time_t *tp) -{ - time_t t; - nxt_thread_t *thr; - - thr = nxt_thread(); - nxt_log_debug(thr->log, "gnutls time"); - - t = (time_t) nxt_thread_time(thr); - - if (tp != NULL) { - *tp = t; - } - - return t; -} - -#endif - - -static nxt_int_t -nxt_gnutls_server_init(nxt_ssltls_conf_t *conf) -{ - int ret; - char *certificate, *key, *ca_certificate; - nxt_thread_t *thr; - nxt_gnutls_ctx_t *ctx; - - if (nxt_slow_path(nxt_gnutls_start() != NXT_OK)) { - return NXT_ERROR; - } - - /* TODO: mem_pool, cleanup: gnutls_certificate_free_credentials, - gnutls_priority_deinit */ - - ctx = nxt_zalloc(sizeof(nxt_gnutls_ctx_t)); - if (ctx == NULL) { - return NXT_ERROR; - } - - conf->ctx = ctx; - conf->conn_init = nxt_gnutls_conn_init; - - thr = nxt_thread(); - - ret = gnutls_certificate_allocate_credentials(&ctx->certificate); - if (ret != GNUTLS_E_SUCCESS) { - nxt_gnutls_log_error(NXT_LOG_ALERT, thr->log, ret, - "gnutls_certificate_allocate_credentials() failed"); - return NXT_ERROR; - } - - certificate = conf->certificate; - key = conf->certificate_key; - - ret = gnutls_certificate_set_x509_key_file(ctx->certificate, certificate, - key, GNUTLS_X509_FMT_PEM); - if (ret != GNUTLS_E_SUCCESS) { - nxt_gnutls_log_error(NXT_LOG_ALERT, thr->log, ret, - "gnutls_certificate_set_x509_key_file(\"%s\", \"%s\") failed", - certificate, key); - goto certificate_fail; - } - - if (nxt_gnutls_set_ciphers(conf) != NXT_OK) { - goto ciphers_fail; - } - - if (conf->ca_certificate != NULL) { - ca_certificate = conf->ca_certificate; - - ret = gnutls_certificate_set_x509_trust_file(ctx->certificate, - ca_certificate, - GNUTLS_X509_FMT_PEM); - if (ret < 0) { - nxt_gnutls_log_error(NXT_LOG_ALERT, thr->log, ret, - "gnutls_certificate_set_x509_trust_file(\"%s\") failed", - ca_certificate); - goto ca_certificate_fail; - } - } - - return NXT_OK; - -ca_certificate_fail: - - gnutls_priority_deinit(ctx->ciphers); - -ciphers_fail: - -certificate_fail: - - gnutls_certificate_free_credentials(ctx->certificate); - - return NXT_ERROR; -} - - -static nxt_int_t -nxt_gnutls_set_ciphers(nxt_ssltls_conf_t *conf) -{ - int ret; - const char *ciphers; - const char *err; - nxt_gnutls_ctx_t *ctx; - - ciphers = (conf->ciphers != NULL) ? conf->ciphers : "NORMAL:!COMP-DEFLATE"; - ctx = conf->ctx; - - ret = gnutls_priority_init(&ctx->ciphers, ciphers, &err); - - switch (ret) { - - case GNUTLS_E_SUCCESS: - return NXT_OK; - - case GNUTLS_E_INVALID_REQUEST: - nxt_gnutls_log_error(NXT_LOG_ALERT, nxt_thread_log(), ret, - "gnutls_priority_init(\"%s\") failed at \"%s\"", - ciphers, err); - return NXT_ERROR; - - default: - nxt_gnutls_log_error(NXT_LOG_ALERT, nxt_thread_log(), ret, - "gnutls_priority_init() failed"); - return NXT_ERROR; - } -} - - -static void -nxt_gnutls_conn_init(nxt_thread_t *thr, nxt_ssltls_conf_t *conf, - nxt_event_conn_t *c) -{ - int ret; - gnutls_session_t sess; - nxt_gnutls_ctx_t *ctx; - nxt_gnutls_conn_t *ssltls; - nxt_mem_pool_cleanup_t *mpcl; - - nxt_log_debug(c->socket.log, "gnutls conn init"); - - ssltls = nxt_mp_zget(c->mem_pool, sizeof(nxt_gnutls_conn_t)); - if (ssltls == NULL) { - goto fail; - } - - c->u.ssltls = ssltls; - nxt_buf_mem_set_size(&ssltls->buffer, conf->buffer_size); - - mpcl = nxt_mem_pool_cleanup(c->mem_pool, 0); - if (mpcl == NULL) { - goto fail; - } - - ret = gnutls_init(&ssltls->session, GNUTLS_SERVER); - if (ret != GNUTLS_E_SUCCESS) { - nxt_gnutls_log_error(NXT_LOG_ALERT, c->socket.log, ret, - "gnutls_init() failed"); - goto fail; - } - - sess = ssltls->session; - mpcl->handler = nxt_gnutls_session_cleanup; - mpcl->data = ssltls; - - ctx = conf->ctx; - - ret = gnutls_priority_set(sess, ctx->ciphers); - if (ret != GNUTLS_E_SUCCESS) { - nxt_gnutls_log_error(NXT_LOG_ALERT, c->socket.log, ret, - "gnutls_priority_set() failed"); - goto fail; - } - - /* - * Disable TLS random padding of records in CBC ciphers, - * which may be up to 255 bytes. - */ - gnutls_record_disable_padding(sess); - - ret = gnutls_credentials_set(sess, GNUTLS_CRD_CERTIFICATE, - ctx->certificate); - if (ret != GNUTLS_E_SUCCESS) { - nxt_gnutls_log_error(NXT_LOG_ALERT, c->socket.log, ret, - "gnutls_credentials_set() failed"); - goto fail; - } - - if (conf->ca_certificate != NULL) { - gnutls_certificate_server_set_request(sess, GNUTLS_CERT_REQUEST); - } - - gnutls_transport_set_ptr(sess, (gnutls_transport_ptr_t) c); - gnutls_transport_set_pull_function(sess, nxt_gnutls_pull); - gnutls_transport_set_push_function(sess, nxt_gnutls_push); -#if (NXT_HAVE_GNUTLS_VEC_PUSH) - gnutls_transport_set_vec_push_function(sess, nxt_gnutls_vec_push); -#endif - - c->io = &nxt_gnutls_event_conn_io; - c->sendfile = NXT_CONN_SENDFILE_OFF; - - nxt_gnutls_conn_handshake(thr, c, c->socket.data); - return; - -fail: - - nxt_event_conn_io_handle(thr, c->read_work_queue, - c->read_state->error_handler, c, c->socket.data); -} - - -static void -nxt_gnutls_session_cleanup(void *data) -{ - nxt_gnutls_conn_t *ssltls; - - ssltls = data; - - nxt_thread_log_debug("gnutls session cleanup"); - - nxt_free(ssltls->buffer.start); - - gnutls_deinit(ssltls->session); -} - - -static ssize_t -nxt_gnutls_pull(gnutls_transport_ptr_t data, void *buf, size_t size) -{ - ssize_t n; - nxt_thread_t *thr; - nxt_event_conn_t *c; - - c = data; - thr = nxt_thread(); - - n = thr->engine->event->io->recv(c, buf, size, 0); - - if (n == NXT_AGAIN) { - nxt_set_errno(NXT_EAGAIN); - return -1; - } - - return n; -} - - -static ssize_t -nxt_gnutls_push(gnutls_transport_ptr_t data, const void *buf, size_t size) -{ - ssize_t n; - nxt_thread_t *thr; - nxt_event_conn_t *c; - - c = data; - thr = nxt_thread(); - - n = thr->engine->event->io->send(c, (u_char *) buf, size); - - if (n == NXT_AGAIN) { - nxt_set_errno(NXT_EAGAIN); - return -1; - } - - return n; -} - - -#if (NXT_HAVE_GNUTLS_VEC_PUSH) - -/* GnuTLS 2.12.0 */ - -static ssize_t -nxt_gnutls_vec_push(gnutls_transport_ptr_t data, const giovec_t *iov, - int iovcnt) -{ - ssize_t n; - nxt_thread_t *thr; - nxt_event_conn_t *c; - - c = data; - thr = nxt_thread(); - - /* - * This code assumes that giovec_t is the same as "struct iovec" - * and nxt_iobuf_t. It is not true for Windows. - */ - n = thr->engine->event->io->writev(c, (nxt_iobuf_t *) iov, iovcnt); - - if (n == NXT_AGAIN) { - nxt_set_errno(NXT_EAGAIN); - return -1; - } - - return n; -} - -#endif - - -static void -nxt_gnutls_conn_handshake(nxt_thread_t *thr, void *obj, void *data) -{ - int err; - nxt_int_t ret; - nxt_event_conn_t *c; - nxt_gnutls_conn_t *ssltls; - - c = obj; - ssltls = c->u.ssltls; - - nxt_log_debug(thr->log, "gnutls conn handshake: %d", ssltls->times); - - /* "ssltls->times == 1" is suitable to run gnutls_handshake() in job. */ - - err = gnutls_handshake(ssltls->session); - - nxt_thread_time_debug_update(thr); - - nxt_log_debug(thr->log, "gnutls_handshake(): %d", err); - - if (err == GNUTLS_E_SUCCESS) { - nxt_gnutls_conn_io_read(thr, c, data); - return; - } - - ret = nxt_gnutls_conn_test_error(thr, c, err, nxt_gnutls_conn_handshake); - - if (ret == NXT_ERROR) { - nxt_gnutls_conn_log_error(c, err, "gnutls_handshake() failed"); - - nxt_event_conn_io_handle(thr, c->read_work_queue, - c->read_state->error_handler, c, data); - - } else if (err == GNUTLS_E_AGAIN - && ssltls->times < 2 - && gnutls_record_get_direction(ssltls->session) == 0) - { - ssltls->times++; - } -} - - -static void -nxt_gnutls_conn_io_read(nxt_thread_t *thr, void *obj, void *data) -{ - ssize_t n; - nxt_buf_t *b; - nxt_int_t ret; - nxt_event_conn_t *c; - nxt_gnutls_conn_t *ssltls; - nxt_work_handler_t handler; - - c = obj; - - nxt_log_debug(thr->log, "gnutls conn read"); - - handler = c->read_state->ready_handler; - b = c->read; - - /* b == NULL is used to test descriptor readiness. */ - - if (b != NULL) { - ssltls = c->u.ssltls; - - n = gnutls_record_recv(ssltls->session, b->mem.free, - b->mem.end - b->mem.free); - - nxt_log_debug(thr->log, "gnutls_record_recv(%d, %p, %uz): %z", - c->socket.fd, b->mem.free, b->mem.end - b->mem.free, n); - - if (n > 0) { - /* c->socket.read_ready is kept. */ - b->mem.free += n; - handler = c->read_state->ready_handler; - - } else if (n == 0) { - handler = c->read_state->close_handler; - - } else { - ret = nxt_gnutls_conn_test_error(thr, c, n, - nxt_gnutls_conn_io_read); - - if (nxt_fast_path(ret != NXT_ERROR)) { - return; - } - - nxt_gnutls_conn_log_error(c, n, - "gnutls_record_recv(%d, %p, %uz): failed", - c->socket.fd, b->mem.free, - b->mem.end - b->mem.free); - - handler = c->read_state->error_handler; - } - } - - nxt_event_conn_io_handle(thr, c->read_work_queue, handler, c, data); -} - - -static ssize_t -nxt_gnutls_conn_io_write_chunk(nxt_thread_t *thr, nxt_event_conn_t *c, - nxt_buf_t *b, size_t limit) -{ - nxt_gnutls_conn_t *ssltls; - - nxt_log_debug(thr->log, "gnutls conn write chunk"); - - ssltls = c->u.ssltls; - - return nxt_sendbuf_copy_coalesce(c, &ssltls->buffer, b, limit); -} - - -static ssize_t -nxt_gnutls_conn_io_send(nxt_event_conn_t *c, void *buf, size_t size) -{ - ssize_t n; - nxt_int_t ret; - nxt_gnutls_conn_t *ssltls; - - ssltls = c->u.ssltls; - - n = gnutls_record_send(ssltls->session, buf, size); - - nxt_log_debug(c->socket.log, "gnutls_record_send(%d, %p, %uz): %z", - c->socket.fd, buf, size, n); - - if (n > 0) { - return n; - } - - ret = nxt_gnutls_conn_test_error(nxt_thread(), c, n, - nxt_event_conn_io_write); - - if (nxt_slow_path(ret == NXT_ERROR)) { - nxt_gnutls_conn_log_error(c, n, - "gnutls_record_send(%d, %p, %uz): failed", - c->socket.fd, buf, size); - } - - return ret; -} - - -static void -nxt_gnutls_conn_io_shutdown(nxt_thread_t *thr, void *obj, void *data) -{ - int err; - nxt_int_t ret; - nxt_event_conn_t *c; - nxt_gnutls_conn_t *ssltls; - nxt_work_handler_t handler; - gnutls_close_request_t how; - - c = obj; - ssltls = c->u.ssltls; - - if (ssltls->session == NULL || ssltls->no_shutdown) { - handler = c->write_state->close_handler; - goto done; - } - - nxt_log_debug(c->socket.log, "gnutls conn shutdown"); - - if (c->socket.timedout || c->socket.error != 0) { - how = GNUTLS_SHUT_WR; - - } else if (c->socket.closed) { - how = GNUTLS_SHUT_RDWR; - - } else { - how = GNUTLS_SHUT_RDWR; - } - - err = gnutls_bye(ssltls->session, how); - - nxt_log_debug(c->socket.log, "gnutls_bye(%d, %d): %d", - c->socket.fd, how, err); - - if (err == GNUTLS_E_SUCCESS) { - handler = c->write_state->close_handler; - - } else { - ret = nxt_gnutls_conn_test_error(thr, c, err, - nxt_gnutls_conn_io_shutdown); - - if (ret != NXT_ERROR) { /* ret == NXT_AGAIN */ - c->socket.error_handler = c->read_state->error_handler; - nxt_event_timer_add(thr->engine, &c->read_timer, 5000); - return; - } - - nxt_gnutls_conn_log_error(c, err, "gnutls_bye(%d) failed", - c->socket.fd); - - handler = c->write_state->error_handler; - } - -done: - - nxt_event_conn_io_handle(thr, c->write_work_queue, handler, c, data); -} - - -static nxt_int_t -nxt_gnutls_conn_test_error(nxt_thread_t *thr, nxt_event_conn_t *c, ssize_t err, - nxt_work_handler_t handler) -{ - int ret; - nxt_gnutls_conn_t *ssltls; - - switch (err) { - - case GNUTLS_E_REHANDSHAKE: - case GNUTLS_E_AGAIN: - ssltls = c->u.ssltls; - ret = gnutls_record_get_direction(ssltls->session); - - nxt_log_debug(thr->log, "gnutls_record_get_direction(): %d", ret); - - if (ret == 0) { - /* A read direction. */ - - nxt_event_fd_block_write(thr->engine, &c->socket); - - c->socket.read_ready = 0; - c->socket.read_handler = handler; - - if (nxt_event_fd_is_disabled(c->socket.read)) { - nxt_event_fd_enable_read(thr->engine, &c->socket); - } - - } else { - /* A write direction. */ - - nxt_event_fd_block_read(thr->engine, &c->socket); - - c->socket.write_ready = 0; - c->socket.write_handler = handler; - - if (nxt_event_fd_is_disabled(c->socket.write)) { - nxt_event_fd_enable_write(thr->engine, &c->socket); - } - } - - return NXT_AGAIN; - - default: - c->socket.error = 1000; /* Nonexistent errno code. */ - return NXT_ERROR; - } -} - - -static void -nxt_gnutls_conn_log_error(nxt_event_conn_t *c, ssize_t err, - const char *fmt, ...) -{ - va_list args; - nxt_uint_t level; - u_char *p, msg[NXT_MAX_ERROR_STR]; - - level = nxt_gnutls_log_error_level(c, err); - - if (nxt_log_level_enough(c->socket.log, level)) { - - va_start(args, fmt); - p = nxt_vsprintf(msg, msg + sizeof(msg), fmt, args); - va_end(args); - - nxt_log_error(level, c->socket.log, "%*s (%d: %s)", - p - msg, msg, err, gnutls_strerror(err)); - } -} - - -static nxt_uint_t -nxt_gnutls_log_error_level(nxt_event_conn_t *c, ssize_t err) -{ - nxt_gnutls_conn_t *ssltls; - - switch (err) { - - case GNUTLS_E_UNKNOWN_CIPHER_SUITE: /* -21 */ - - /* Disable gnutls_bye(), because it returns GNUTLS_E_INTERNAL_ERROR. */ - ssltls = c->u.ssltls; - ssltls->no_shutdown = 1; - - /* Fall through. */ - - case GNUTLS_E_UNEXPECTED_PACKET_LENGTH: /* -9 */ - c->socket.error = 1000; /* Nonexistent errno code. */ - break; - - default: - return NXT_LOG_ALERT; - } - - return NXT_LOG_INFO; -} - - -static void -nxt_gnutls_log_error(nxt_uint_t level, nxt_log_t *log, int err, - const char *fmt, ...) -{ - va_list args; - u_char *p, msg[NXT_MAX_ERROR_STR]; - - va_start(args, fmt); - p = nxt_vsprintf(msg, msg + sizeof(msg), fmt, args); - va_end(args); - - nxt_log_error(level, log, "%*s (%d: %s)", - p - msg, msg, err, gnutls_strerror(err)); -} diff --git a/src/nxt_h1proto.c b/src/nxt_h1proto.c index 57d8cb9b2..5cc289f21 100644 --- a/src/nxt_h1proto.c +++ b/src/nxt_h1proto.c @@ -21,6 +21,8 @@ #if (NXT_TLS) static ssize_t nxt_http_idle_io_read_handler(nxt_task_t *task, nxt_conn_t *c); static void nxt_http_conn_test(nxt_task_t *task, void *obj, void *data); +static void nxt_http_conn_tls_conf_release(nxt_task_t *task, void *obj, + void *data); #endif static ssize_t nxt_h1p_idle_io_read_handler(nxt_task_t *task, nxt_conn_t *c); static void nxt_h1p_conn_proto_init(nxt_task_t *task, void *obj, void *data); @@ -41,7 +43,13 @@ static nxt_int_t nxt_h1p_websocket_version(void *ctx, nxt_http_field_t *field, uintptr_t data); static nxt_int_t nxt_h1p_transfer_encoding(void *ctx, nxt_http_field_t *field, uintptr_t data); +static nxt_int_t nxt_h1p_expect(void *ctx, nxt_http_field_t *field, + uintptr_t data); static void nxt_h1p_request_body_read(nxt_task_t *task, nxt_http_request_t *r); +static nxt_int_t nxt_h1p_request_continue(nxt_task_t *task, + nxt_h1proto_t *h1p); +static void nxt_h1p_conn_continue_sent(nxt_task_t *task, void *obj, + void *data); static void nxt_h1p_conn_request_body_read(nxt_task_t *task, void *obj, void *data); static void nxt_h1p_request_local_addr(nxt_task_t *task, nxt_http_request_t *r); @@ -56,6 +64,12 @@ static nxt_off_t nxt_h1p_request_body_bytes_sent(nxt_task_t *task, static void nxt_h1p_request_discard(nxt_task_t *task, nxt_http_request_t *r, nxt_buf_t *last); static void nxt_h1p_conn_request_error(nxt_task_t *task, void *obj, void *data); +static nxt_bool_t nxt_h1p_rate_too_low(uint64_t bytes, uint64_t msec, + nxt_msec_t grace, int32_t rate); +static void nxt_h1p_send_rate_start(nxt_task_t *task, nxt_h1proto_t *h1p, + nxt_http_request_t *r); +static nxt_bool_t nxt_h1p_send_rate_check(nxt_task_t *task, nxt_conn_t *c); +static void nxt_h1p_request_timedout(nxt_task_t *task, nxt_conn_t *c); static void nxt_h1p_conn_request_timeout(nxt_task_t *task, void *obj, void *data); static void nxt_h1p_conn_request_send_timeout(nxt_task_t *task, void *obj, @@ -118,6 +132,7 @@ static const nxt_conn_state_t nxt_h1p_shutdown_state; #endif static const nxt_conn_state_t nxt_h1p_idle_state; static const nxt_conn_state_t nxt_h1p_header_parse_state; +static const nxt_conn_state_t nxt_h1p_continue_state; static const nxt_conn_state_t nxt_h1p_read_body_state; static const nxt_conn_state_t nxt_h1p_request_send_state; static const nxt_conn_state_t nxt_h1p_timeout_response_state; @@ -165,6 +180,7 @@ static nxt_http_field_proc_t nxt_h1p_fields[] = { { nxt_string("Sec-WebSocket-Version"), &nxt_h1p_websocket_version, 0 }, { nxt_string("Transfer-Encoding"), &nxt_h1p_transfer_encoding, 0 }, + { nxt_string("Expect"), &nxt_h1p_expect, 0 }, { nxt_string("Host"), &nxt_http_request_host, 0 }, { nxt_string("Cookie"), &nxt_http_request_field, @@ -379,9 +395,43 @@ nxt_http_conn_test(nxt_task_t *task, void *obj, void *data) tls = joint->socket_conf->tls; + /* + * The connection holds the listener configuration until it is freed. + * The TLS connection reads its nxt_tls_conf_t, which lives in the memory + * pool of the router configuration, in the handshake callbacks and in + * the TLS shutdown. Only a request references the configuration, so + * without this reference a reconfiguration would destroy it under a + * connection in the handshake, and under a keep-alive connection that + * is closed after its last request has released its own reference. + * The cleanup runs in nxt_conn_free(), after the TLS shutdown. + */ + if (nxt_slow_path(nxt_mp_cleanup(c->mem_pool, + nxt_http_conn_tls_conf_release, + &engine->task, joint, NULL) + != NXT_OK)) + { + nxt_h1p_closing(task, c); + return; + } + + joint->count++; + tls->conn_init(task, tls, c); } + +static void +nxt_http_conn_tls_conf_release(nxt_task_t *task, void *obj, void *data) +{ + nxt_socket_conf_joint_t *joint; + + joint = obj; + + nxt_debug(task, "http conn tls conf release"); + + nxt_router_conf_release(task, joint); +} + #endif @@ -891,19 +941,45 @@ nxt_h1p_transfer_encoding(void *ctx, nxt_http_field_t *field, uintptr_t data) } +/* + * RFC 9110, 10.1.1. The router meets "100-continue" itself and ignores any + * other expectation, as nginx does. An HTTP/1.0 client gets no 100. The + * proxy has the whole body before it connects, so it does not forward the + * field. The application still gets it. + */ + +static nxt_int_t +nxt_h1p_expect(void *ctx, nxt_http_field_t *field, uintptr_t data) +{ + nxt_http_request_t *r; + + r = ctx; + field->hopbyhop = 1; + + if (field->value_length == nxt_length("100-continue") + && nxt_memcasecmp(field->value, "100-continue", + nxt_length("100-continue")) == 0 + && nxt_h1p_is_http11(r->proto.h1)) + { + r->proto.h1->continue_pending = 1; + } + + return NXT_OK; +} + + static void nxt_h1p_request_body_read(nxt_task_t *task, nxt_http_request_t *r) { - size_t size, body_length, body_buffer_size, body_rest; + size_t size, body_length, body_rest; ssize_t res; nxt_buf_t *in, *b, *out, *chunk; + nxt_int_t ret; nxt_conn_t *c; nxt_h1proto_t *h1p; nxt_socket_conf_t *skcf; nxt_http_status_t status; - static const nxt_str_t tmp_name_pattern = nxt_string("/req-XXXXXXXX"); - h1p = r->proto.h1; skcf = r->conf->socket_conf; @@ -953,61 +1029,13 @@ nxt_h1p_request_body_read(nxt_task_t *task, nxt_http_request_t *r) body_length = (size_t) r->content_length_n; - body_buffer_size = nxt_min(skcf->body_buffer_size, body_length); - - if (body_length > body_buffer_size) { - nxt_str_t *tmp_path, tmp_name; - - tmp_path = &skcf->body_temp_path; - - tmp_name.length = tmp_path->length + tmp_name_pattern.length; - - b = nxt_buf_file_alloc(r->mem_pool, - body_buffer_size + sizeof(nxt_file_t) - + tmp_name.length + 1, 0); - if (nxt_slow_path(b == NULL)) { - status = NXT_HTTP_INTERNAL_SERVER_ERROR; - goto error; - } - - tmp_name.start = nxt_pointer_to(b->mem.start, sizeof(nxt_file_t)); - - memcpy(tmp_name.start, tmp_path->start, tmp_path->length); - memcpy(tmp_name.start + tmp_path->length, tmp_name_pattern.start, - tmp_name_pattern.length); - tmp_name.start[tmp_name.length] = '\0'; - - b->file = (nxt_file_t *) b->mem.start; - nxt_memzero(b->file, sizeof(nxt_file_t)); - b->file->fd = -1; - b->file->size = body_length; - - b->mem.start += sizeof(nxt_file_t) + tmp_name.length + 1; - b->mem.pos = b->mem.start; - b->mem.free = b->mem.start; - - b->file->fd = mkstemp((char *) tmp_name.start); - if (nxt_slow_path(b->file->fd == -1)) { - nxt_alert(task, "mkstemp(%s) failed %E", tmp_name.start, nxt_errno); - - status = NXT_HTTP_INTERNAL_SERVER_ERROR; - goto error; - } - - nxt_debug(task, "create body tmp file \"%V\", %d", - &tmp_name, b->file->fd); - - unlink((char *) tmp_name.start); - - } else { - b = nxt_buf_mem_alloc(r->mem_pool, body_buffer_size, 0); - if (nxt_slow_path(b == NULL)) { - status = NXT_HTTP_INTERNAL_SERVER_ERROR; - goto error; - } + ret = nxt_http_request_body_alloc(task, r, body_length); + if (nxt_slow_path(ret != NXT_OK)) { + status = NXT_HTTP_INTERNAL_SERVER_ERROR; + goto error; } - r->body = b; + b = r->body; body_rest = r->chunked ? 1 : body_length; @@ -1066,7 +1094,7 @@ nxt_h1p_request_body_read(nxt_task_t *task, nxt_http_request_t *r) } } else { - size = nxt_min(body_buffer_size, size); + size = nxt_min(size, (size_t) nxt_buf_mem_free_size(&b->mem)); b->mem.free = nxt_cpymem(b->mem.free, in->mem.pos, size); in->mem.pos += size; @@ -1081,8 +1109,41 @@ nxt_h1p_request_body_read(nxt_task_t *task, nxt_http_request_t *r) h1p->buffers = in; h1p->nbuffers++; + /* + * The body_min_rate floor counts each byte that a read returns + * in the body read state, chunk framing included. The framing + * costs the client the same bandwidth, and max_body_size limits + * the payload. The header block, and the body bytes that came + * with it, are not counted, because the first body read comes + * after this point. Thus the first window is a little stricter. + * This is intentional. The time starts at the start of the body + * read state. After a 100 (Continue), + * nxt_h1p_conn_continue_sent() starts it again. + */ + h1p->body_rate_on = (skcf->body_min_rate > 0); + h1p->body_rate_start = task->thread->engine->timers.now; + h1p->body_rate_bytes = 0; + c = h1p->conn; c->read = b; + + /* + * The 100 goes out also if a part of the body came with the header, + * as in nginx. That part can be only chunk framing, and the client + * can still wait for the 100. Nothing is written before the body + * is read, so the 100 cannot follow a response. The flag is cleared + * when the 100 is queued, so a request gets one 100 at most. + */ + if (h1p->continue_pending) { + ret = nxt_h1p_request_continue(task, h1p); + if (nxt_slow_path(ret != NXT_OK)) { + status = NXT_HTTP_INTERNAL_SERVER_ERROR; + goto error; + } + + return; + } + c->read_state = &nxt_h1p_read_body_state; nxt_conn_read(task->thread->engine, c); @@ -1110,6 +1171,93 @@ nxt_h1p_request_body_read(nxt_task_t *task, nxt_http_request_t *r) } +/* + * Sends "100 Continue" on the normal write path. The body is read only + * after the 100 is sent. A write error or a send timeout closes the + * request, as for a response. + */ + +static nxt_int_t +nxt_h1p_request_continue(nxt_task_t *task, nxt_h1proto_t *h1p) +{ + nxt_buf_t *b; + nxt_conn_t *c; + + static const char continue_response[] = "HTTP/1.1 100 Continue\r\n\r\n"; + + nxt_debug(task, "h1p request continue"); + + c = h1p->conn; + + /* The connection pool frees the buffer also if it is never sent. */ + b = nxt_buf_mem_alloc(c->mem_pool, nxt_length(continue_response), 0); + if (nxt_slow_path(b == NULL)) { + return NXT_ERROR; + } + + b->mem.free = nxt_cpymem(b->mem.free, continue_response, + nxt_length(continue_response)); + + /* $body_bytes_sent does not count the 100. */ + h1p->sent_before_body += nxt_length(continue_response); + h1p->continue_pending = 0; + + c->write = b; + c->write_state = &nxt_h1p_continue_state; + + nxt_conn_write(task->thread->engine, c); + + return NXT_OK; +} + + +static const nxt_conn_state_t nxt_h1p_continue_state + nxt_aligned(64) = +{ + .ready_handler = nxt_h1p_conn_continue_sent, + .error_handler = nxt_h1p_conn_request_error, + + .timer_handler = nxt_h1p_conn_request_send_timeout, + .timer_value = nxt_h1p_conn_request_timer_value, + .timer_data = offsetof(nxt_socket_conf_t, send_timeout), + .timer_autoreset = 1, +}; + + +static void +nxt_h1p_conn_continue_sent(nxt_task_t *task, void *obj, void *data) +{ + nxt_conn_t *c; + nxt_h1proto_t *h1p; + nxt_event_engine_t *engine; + + c = obj; + + nxt_debug(task, "h1p conn continue sent"); + + engine = task->thread->engine; + + c->write = nxt_sendbuf_completion(task, &engine->fast_work_queue, c->write); + + if (c->write != NULL) { + nxt_conn_write(engine, c); + return; + } + + /* + * The body read state starts now, and so does the body_read_timeout + * timer. Thus the body_min_rate time starts again here. The time + * when the 100 waited for space in the send buffer is not counted. + */ + h1p = c->socket.data; + h1p->body_rate_start = engine->timers.now; + + c->read_state = &nxt_h1p_read_body_state; + + nxt_conn_read(engine, c); +} + + static const nxt_conn_state_t nxt_h1p_read_body_state nxt_aligned(64) = { @@ -1130,6 +1278,7 @@ nxt_h1p_conn_request_body_read(nxt_task_t *task, void *obj, void *data) size_t size, body_rest; ssize_t res; nxt_buf_t *b, *out, *chunk; + nxt_msec_t msec; nxt_conn_t *c; nxt_h1proto_t *h1p; nxt_socket_conf_t *skcf; @@ -1156,11 +1305,14 @@ nxt_h1p_conn_request_body_read(nxt_task_t *task, void *obj, void *data) out = nxt_http_chunk_parse(task, &h1p->chunked_parse, b); if (h1p->chunked_parse.error) { - nxt_h1p_request_error(task, h1p, r); + h1p->keepalive = 0; + nxt_http_request_error(task, r, + NXT_HTTP_INTERNAL_SERVER_ERROR); return; } if (h1p->chunked_parse.chunk_error) { + h1p->keepalive = 0; nxt_http_request_error(task, r, NXT_HTTP_BAD_REQUEST); return; } @@ -1169,14 +1321,18 @@ nxt_h1p_conn_request_body_read(nxt_task_t *task, void *obj, void *data) size = nxt_buf_mem_used_size(&chunk->mem); res = nxt_fd_write(b->file->fd, chunk->mem.pos, size); if (nxt_slow_path(res < (ssize_t) size)) { - nxt_h1p_request_error(task, h1p, r); + h1p->keepalive = 0; + nxt_http_request_error(task, r, + NXT_HTTP_INTERNAL_SERVER_ERROR); return; } b->file_end += size; if ((size_t) b->file_end > skcf->max_body_size) { - nxt_h1p_request_error(task, h1p, r); + h1p->keepalive = 0; + nxt_http_request_error(task, r, + NXT_HTTP_PAYLOAD_TOO_LARGE); return; } } @@ -1215,7 +1371,9 @@ nxt_h1p_conn_request_body_read(nxt_task_t *task, void *obj, void *data) res = nxt_fd_write(b->file->fd, b->mem.pos, size); if (nxt_slow_path(res < (ssize_t) size)) { - nxt_h1p_request_error(task, h1p, r); + h1p->keepalive = 0; + nxt_http_request_error(task, r, + NXT_HTTP_INTERNAL_SERVER_ERROR); return; } @@ -1244,6 +1402,36 @@ nxt_h1p_conn_request_body_read(nxt_task_t *task, void *obj, void *data) nxt_debug(task, "h1p body rest: %uz", body_rest); if (body_rest != 0) { + + if (h1p->body_rate_on) { + /* + * The bytes are counted only here. The flag does not change + * in the body read state, and only a read that continues + * the body is checked. + */ + h1p->body_rate_bytes += c->nbytes; + + msec = (nxt_msec_t) (engine->timers.now - h1p->body_rate_start); + + if (nxt_h1p_rate_too_low(h1p->body_rate_bytes, msec, + skcf->body_read_timeout, + skcf->body_min_rate)) + { + nxt_log(task, NXT_LOG_INFO, "client body rate is less than " + "body_min_rate %d: %uL bytes in %M ms", + skcf->body_min_rate, h1p->body_rate_bytes, msec); + + nxt_h1p_request_timedout(task, c); + return; + } + + if (msec >= skcf->body_read_timeout) { + /* The window passed: the next window counts from now. */ + h1p->body_rate_start = engine->timers.now; + h1p->body_rate_bytes = 0; + } + } + nxt_conn_read(engine, c); } else { @@ -1456,18 +1644,27 @@ nxt_h1p_request_header_send(nxt_task_t *task, nxt_http_request_t *r, if (r->resp.content_length == NULL || r->resp.content_length->skip) { if (http11) { - if (n != NXT_HTTP_NOT_MODIFIED + /* + * r->no_body already covers 204 and 304, plus 1xx and every + * response to HEAD; the two status tests are kept so the + * framing rule stays readable at the point it is applied. + */ + if (!r->no_body + && n != NXT_HTTP_NOT_MODIFIED && n != NXT_HTTP_NO_CONTENT && body_handler != NULL && !h1p->websocket) { h1p->chunked = 1; size += nxt_length(chunked); - /* Trailing CRLF will be added by the first chunk header. */ - size -= nxt_length("\r\n"); } - } else { + } else if (!r->no_body) { + /* + * A pre-HTTP/1.1 client needs the close to delimit a body; + * a response that has no body needs no such delimiter, so + * keep-alive stays as negotiated. + */ h1p->keepalive = 0; } } @@ -1534,15 +1731,18 @@ nxt_h1p_request_header_send(nxt_task_t *task, nxt_http_request_t *r, if (h1p->chunked) { p = nxt_cpymem(p, chunked, nxt_length(chunked)); - /* Trailing CRLF will be added by the first chunk header. */ - - } else { - *p++ = '\r'; *p++ = '\n'; } + /* + * The header ends here, also for a chunked response. Thus the client + * can use the header before the first body bytes come. + */ + *p++ = '\r'; *p++ = '\n'; + header->mem.free = p; - h1p->header_size = nxt_buf_mem_used_size(&header->mem); + /* $body_bytes_sent does not count the header, or a 100 sent before. */ + h1p->sent_before_body += nxt_buf_mem_used_size(&header->mem); c = h1p->conn; @@ -1550,6 +1750,8 @@ nxt_h1p_request_header_send(nxt_task_t *task, nxt_http_request_t *r, h1p->conn_write_tail = &header->next; c->write_state = &nxt_h1p_request_send_state; + nxt_h1p_send_rate_start(task, h1p, r); + if (body_handler != NULL) { /* * The body handler will run before c->io->write() handler, @@ -1655,6 +1857,8 @@ nxt_h1p_request_send(nxt_task_t *task, nxt_http_request_t *r, nxt_buf_t *out) c->write = out; c->write_state = &nxt_h1p_request_send_state; + nxt_h1p_send_rate_start(task, h1p, r); + nxt_conn_write(task->thread->engine, c); } else { @@ -1669,15 +1873,24 @@ nxt_h1p_request_send(nxt_task_t *task, nxt_http_request_t *r, nxt_buf_t *out) } +/* + * The data of a chunk ends with CRLF. This CRLF starts the next chunk + * header or the last chunk. Before the first chunk there is no data to end: + * the response header ends with its own CRLF. + */ + static nxt_buf_t * nxt_h1p_chunk_create(nxt_task_t *task, nxt_http_request_t *r, nxt_buf_t *out) { + u_char *p; nxt_off_t size; nxt_buf_t *b, **prev, *header, *tail; + nxt_h1proto_t *h1p; const size_t chunk_size = 2 * nxt_length("\r\n") + NXT_OFF_T_HEXLEN; static const char tail_chunk[] = "\r\n0\r\n\r\n"; + h1p = r->proto.h1; size = 0; prev = &out; @@ -1716,6 +1929,11 @@ nxt_h1p_chunk_create(nxt_task_t *task, nxt_http_request_t *r, nxt_buf_t *out) nxt_memcpy(tail->mem.free, tail_chunk, sizeof(tail_chunk)); tail->mem.free += nxt_length(tail_chunk); + if (!h1p->chunk_sent && size == 0) { + /* No chunk data comes before the last chunk. */ + tail->mem.pos += nxt_length("\r\n"); + } + break; } @@ -1733,8 +1951,16 @@ nxt_h1p_chunk_create(nxt_task_t *task, nxt_http_request_t *r, nxt_buf_t *out) } header->next = out; - header->mem.free = nxt_sprintf(header->mem.free, header->mem.end, - "\r\n%xO\r\n", size); + p = header->mem.free; + + if (h1p->chunk_sent) { + *p++ = '\r'; *p++ = '\n'; + } + + header->mem.free = nxt_sprintf(p, header->mem.end, "%xO\r\n", size); + + h1p->chunk_sent = 1; + return header; } @@ -1747,7 +1973,7 @@ nxt_h1p_request_body_bytes_sent(nxt_task_t *task, nxt_http_proto_t proto) h1p = proto.h1; - sent = h1p->conn->sent - h1p->header_size; + sent = h1p->conn->sent - h1p->sent_before_body; return (sent > 0) ? sent : 0; } @@ -1807,19 +2033,198 @@ nxt_h1p_conn_request_error(nxt_task_t *task, void *obj, void *data) } +/* + * The minimum transfer rate floor: body_min_rate and send_min_rate. + * + * The body_read_timeout and send_timeout timers are gap timers. Each + * read or write of one or more bytes starts the timer again. Thus the + * timers stop a client that sends or reads nothing, but they do not stop + * a client that transfers one byte just before each timeout ("slow POST" + * and "slow read"). Such a client can keep a connection for ever. + * + * The rate floor stops this client. The floor is added to the gap timers, + * it does not replace them. The floor check runs only when a read or + * a write completes. A client that stops fully causes no events, and the + * gap timer stops it. A client that is slower than the floor still causes + * events, and the floor check stops it. + * + * The check starts after a grace time. The grace time is equal to the gap + * timeout (body_read_timeout or send_timeout). It lets TCP slow start and + * short network stops occur. A total time limit is not used, because it + * also stops honest large transfers on slow links. + * + * A timeout of 0 turns the gap timer off: nxt_conn_timer() arms no timer + * for the value 0. Then nothing stops a client that stops fully, because + * the floor check needs a read or a write. The grace time is 0 too, so + * a new window starts after each check. A body read is checked against + * the time since the previous read, or since the start of the body read + * state. A write is checked against the time since the previous write, + * or since the start of the send period if that is later. Thus the time + * between send periods is not counted. A check in the same millisecond + * passes. + * + * The check is "bytes * 1000 < rate * msec" in 64-bit integers. The + * validator keeps the rate at or below 2^31 - 1, and the check limits + * msec to 2^31 - 1. Thus the product cannot overflow. + */ + +static nxt_bool_t +nxt_h1p_rate_too_low(uint64_t bytes, uint64_t msec, nxt_msec_t grace, + int32_t rate) +{ + if (rate <= 0 || msec < grace) { + return 0; + } + + msec = nxt_min(msec, (uint64_t) NXT_INT32_T_MAX); + + if (bytes >= UINT64_MAX / 1000) { + return 0; + } + + return (bytes * 1000 < (uint64_t) rate * msec); +} + + +/* + * The send rate is measured only while response data waits for the client. + * A send period starts when the router gives data to an empty connection + * write queue. The period stops when the client accepted all the queued + * data. The time between periods is not counted: then the router waits + * for the application or for the upstream server, not for the client. + * Thus a slow response source (for example a stream of events) is not + * stopped by the floor. The bytes and the time of the periods of one + * request are added together until a window of at least the grace time + * passes the check; then the next window starts from zero. Thus bytes + * sent early in a response give no credit for a slow read later. + * + * The check runs after each write, also after the write that stops + * a period. When the last data of the response goes out, the response + * is complete in the socket buffer. Then a failed check only closes the + * connection instead of keeping it alive. + * + * WebSocket frames also use the request send state. The floor is not + * used for a WebSocket connection. + */ + static void -nxt_h1p_conn_request_timeout(nxt_task_t *task, void *obj, void *data) +nxt_h1p_send_rate_start(nxt_task_t *task, nxt_h1proto_t *h1p, + nxt_http_request_t *r) { - nxt_conn_t *c; - nxt_timer_t *timer; + if (h1p->send_rate_on + || h1p->websocket + || r->conf->socket_conf->send_min_rate <= 0) + { + return; + } + + h1p->send_rate_on = 1; + h1p->send_rate_start = task->thread->engine->timers.now; + h1p->send_rate_sent = h1p->conn->sent; +} + + +static nxt_bool_t +nxt_h1p_send_rate_check(nxt_task_t *task, nxt_conn_t *c) +{ + uint64_t bytes, msec; nxt_h1proto_t *h1p; + nxt_socket_conf_t *skcf; nxt_http_request_t *r; + h1p = c->socket.data; + + if (!h1p->send_rate_on) { + return 0; + } + + r = h1p->request; + + if (nxt_slow_path(r == NULL)) { + return 0; + } + + skcf = r->conf->socket_conf; + + msec = h1p->send_rate_time + + (nxt_msec_t) (task->thread->engine->timers.now + - h1p->send_rate_start); + bytes = h1p->send_rate_bytes; + + if (c->sent > h1p->send_rate_sent) { + bytes += c->sent - h1p->send_rate_sent; + } + + /* + * The check also runs when the write empties the queue. An application + * or an upstream server that gives one buffer at a time can make each + * write event empty the queue. Without the check there, the router + * never checks the rate of a client that keeps the socket buffer full. + */ + + if (!nxt_h1p_rate_too_low(bytes, msec, skcf->send_timeout, + skcf->send_min_rate)) + { + if (msec >= skcf->send_timeout) { + /* The window passed: the next window counts from now. */ + msec = 0; + bytes = 0; + + h1p->send_rate_time = 0; + h1p->send_rate_bytes = 0; + h1p->send_rate_start = task->thread->engine->timers.now; + h1p->send_rate_sent = c->sent; + } + + if (c->write == NULL) { + /* The client accepted all the queued data: the period stops. */ + h1p->send_rate_on = 0; + h1p->send_rate_time = msec; + h1p->send_rate_bytes = bytes; + } + + return 0; + } + + nxt_log(task, NXT_LOG_INFO, "client send rate is less than " + "send_min_rate %d: %uL bytes in %uL ms", + skcf->send_min_rate, bytes, msec); + + /* The same steps as nxt_h1p_conn_request_send_timeout(). */ + + nxt_timer_disable(task->thread->engine, &c->write_timer); + c->block_write = 1; + + nxt_h1p_request_error(task, h1p, r); + + return 1; +} + + +static void +nxt_h1p_conn_request_timeout(nxt_task_t *task, void *obj, void *data) +{ + nxt_timer_t *timer; + timer = obj; nxt_debug(task, "h1p conn request timeout"); - c = nxt_read_timer_conn(timer); + nxt_h1p_request_timedout(task, nxt_read_timer_conn(timer)); +} + + +/* + * The steps for a request read timeout: header_read_timeout, + * body_read_timeout, and the body_min_rate floor. + */ + +static void +nxt_h1p_request_timedout(nxt_task_t *task, nxt_conn_t *c) +{ + nxt_h1proto_t *h1p; + nxt_http_request_t *r; + c->block_read = 1; /* * Disable SO_LINGER off during socket closing @@ -1931,6 +2336,12 @@ nxt_h1p_conn_sent(nxt_task_t *task, void *obj, void *data) c->write = nxt_sendbuf_completion(task, &engine->fast_work_queue, c->write); + if (c->write_state == &nxt_h1p_request_send_state + && nxt_slow_path(nxt_h1p_send_rate_check(task, c))) + { + return; + } + if (c->write != NULL) { nxt_conn_write(engine, c); } @@ -2482,9 +2893,14 @@ nxt_h1p_peer_header_send(nxt_task_t *task, nxt_http_peer_t *peer) + sizeof("Connection: close\r\n") + sizeof("\r\n"); - /* Emit Content-Length after chunked_transform; NULL body → value 0. */ + /* + * Emit Content-Length after chunked_transform; NULL body → value 0. + * The transform adds a Content-Length field (r->content_length) that + * goes out with the other fields; a second one would make the + * upstream answer 400. + */ content_length = -1; - if (r->chunked) { + if (r->chunked && r->content_length == NULL) { if (r->body == NULL) { content_length = 0; } else { @@ -2840,12 +3256,92 @@ nxt_h1p_peer_header_read_done(nxt_task_t *task, void *obj, void *data) h1p = peer->proto.h1; - if (h1p->chunked) { - if (r->resp.content_length != NULL) { - peer->status = NXT_HTTP_BAD_GATEWAY; + h1p->chunked = (h1p->transfer_encoding == NXT_HTTP_TE_CHUNKED); + + /* + * RFC 9112 Sect. 6.3: a response to HEAD, and any 204 or 304 response, + * is terminated by the first empty line after the header fields no + * matter what Content-Length or Transfer-Encoding say. Such a response + * is complete right here, so neither arm the chunked parser nor set a + * remainder from a Content-Length that describes a body the upstream + * will never send. + * + * Without this the upstream -- which nxt_h1p_peer_header_send() always + * asks to "Connection: close" -- closes with h1p->remainder still at + * the advertised length, or with chunked_parse.last still clear; + * nxt_h1p_peer_closed() then reads that as a truncated body and sets + * r->truncated and r->inconsistent, and nxt_h1p_request_close() drops + * the client keep-alive over a response that was never short. Any + * pipelined request already in the client's socket buffer is lost. + * + * Complete the response the way a body that reaches its declared + * length completes it in nxt_h1p_peer_body_process(): hand the + * request's last buffer to the ready handler and mark the peer closed, + * so nxt_http_proxy_send_body() closes the upstream connection and + * releases the request pool. + * + * The predicate is the "final response" one, not the full RFC list: + * nxt_h1p_peer_header_parse() drops a 1xx and reads on, so a 1xx + * never reaches here. + * + * "b" is not forwarded: bytes an upstream put after the header of a + * bodyless response are not a body. It is handed to + * nxt_http_proxy_buf_mem_hold() rather than freed, because the + * response fields point their name/value into it and are read until + * the request is logged and closed; see the comment there. + */ + if (nxt_http_request_is_bodyless_final(r, peer->status)) { + h1p->chunked = 0; + h1p->remainder = 0; + + if (nxt_slow_path(nxt_http_proxy_buf_mem_hold(task, r, b) + != NXT_OK)) + { + peer->status = NXT_HTTP_INTERNAL_SERVER_ERROR; break; } + peer->body = nxt_http_buf_last(r); + peer->closed = 1; + + r->state->ready_handler(task, r, peer); + return; + } + + /* + * See nxt_h1p_peer_transfer_encoding(). A Transfer-Encoding the + * proxy cannot decode fails the response with 502. With any + * Transfer-Encoding, Content-Length does not frame the body, and both + * together is an error (RFC 9112 Sect. 6.3). The upstream connection + * is closed after every response, so a framing error never reaches a + * reused connection. + * + * Both checks come after the bodyless branch above, not before it. A + * response to HEAD, and any 204 or 304, ends at the first empty line + * no matter what Content-Length and Transfer-Encoding say -- the same + * Sect. 6.3 -- so there is no body for the proxy to decode, and no + * framing for these two fields to disagree about. Such a response has + * already been completed above. Failing it with 502 would lose a + * response over a field that frames nothing: the proxy never reads a + * body there, whatever coding the upstream named. + */ + if (h1p->transfer_encoding == NXT_HTTP_TE_UNSUPPORTED) { + nxt_log(task, NXT_LOG_WARN, + "upstream sent unsupported Transfer-Encoding"); + + peer->status = NXT_HTTP_BAD_GATEWAY; + break; + } + + if (h1p->chunked && r->resp.content_length != NULL) { + nxt_log(task, NXT_LOG_WARN, "upstream sent both " + "Transfer-Encoding and Content-Length"); + + peer->status = NXT_HTTP_BAD_GATEWAY; + break; + } + + if (h1p->chunked) { h1p->chunked_parse.mem_pool = c->mem_pool; } else if (r->resp.content_length_n > 0) { @@ -2857,6 +3353,20 @@ nxt_h1p_peer_header_read_done(nxt_task_t *task, void *obj, void *data) return; } + /* + * No body bytes arrived with the header, so nothing will relay "b" and + * nothing will run its completion handler. Dropping it here -- which + * is what this path did -- stranded both the buffer and the + * r->mem_pool retain nxt_http_proxy_buf_mem_alloc() took, so the whole + * request pool was never destroyed. Measured on the parent commit: + * two pools reach a zero retain per request on this path against three + * on the path where the body shares the header's read. + */ + if (nxt_slow_path(nxt_http_proxy_buf_mem_hold(task, r, b) != NXT_OK)) { + peer->status = NXT_HTTP_INTERNAL_SERVER_ERROR; + break; + } + r->state->ready_handler(task, r, peer); return; @@ -2866,7 +3376,7 @@ nxt_h1p_peer_header_read_done(nxt_task_t *task, void *obj, void *data) return; } - /* Fall through. */ + nxt_fallthrough; default: case NXT_ERROR: @@ -2883,12 +3393,28 @@ nxt_h1p_peer_header_read_done(nxt_task_t *task, void *obj, void *data) } +/* + * A 1xx other than 101 is an interim response (RFC 9110, 15.2). + * Limit how many one upstream response may have. Apache also uses 10. + */ +#define NXT_HTTP_MAX_INTERIM_RESPONSES 10 + +#define nxt_h1p_peer_status_interim(status) \ + ((status) >= NXT_HTTP_CONTINUE && (status) < NXT_HTTP_OK \ + && (status) != NXT_HTTP_SWITCHING_PROTOCOLS) + + static nxt_int_t nxt_h1p_peer_header_parse(nxt_http_peer_t *peer, nxt_buf_mem_t *bm) { - u_char *p; - size_t length; - nxt_int_t status; + u_char *p; + size_t length; + nxt_int_t ret, status; + nxt_http_request_parse_t *rp; + + rp = &peer->proto.h1->parser; + +again: if (peer->status < 0) { length = nxt_buf_mem_used_size(bm); @@ -2922,9 +3448,44 @@ nxt_h1p_peer_header_parse(nxt_http_peer_t *peer, nxt_buf_mem_t *bm) bm->pos = p + 1; peer->status = status; + + /* Do not store the fields of a 1xx. */ + rp->discard_fields = nxt_h1p_peer_status_interim(status); + } + + ret = nxt_http_parse_fields(rp, bm); + + if (ret != NXT_DONE || !nxt_h1p_peer_status_interim(peer->status)) { + return ret; } - return nxt_http_parse_fields(&peer->proto.h1->parser, bm); + /* + * Drop the 1xx and read the next response. The client gets only + * the final response. NXT_ERROR becomes 502. + */ + if (nxt_slow_path(++peer->num_interim > NXT_HTTP_MAX_INTERIM_RESPONSES)) { + nxt_log(&peer->request->task, NXT_LOG_WARN, + "upstream sent more than %d interim responses", + NXT_HTTP_MAX_INTERIM_RESPONSES); + + return NXT_ERROR; + } + + /* The handler can point to the end of the empty line. */ + rp->handler = NULL; + peer->status = NXT_HTTP_UNSET; + + /* + * Free the 1xx bytes, so the final header can use the whole buffer. + * This is safe only after NXT_DONE: no pointer into the buffer is left. + */ + length = bm->free - bm->pos; + nxt_memmove(bm->start, bm->pos, length); + + bm->pos = bm->start; + bm->free = bm->start + length; + + goto again; } @@ -3260,12 +3821,14 @@ nxt_h1p_peer_close(nxt_task_t *task, nxt_http_peer_t *peer) * nxt_h1p_peer_read_done()/nxt_h1p_peer_send_timeout()/etc. and dereference * the freed peer -- a use-after-free that crashes the router. Both paths * are at risk: the read side (response relay) and the write side (the - * request body upload uses an autoreset send timer). Setting block_read / - * block_write makes a queued nxt_conn_io_read()/nxt_conn_io_write() bail out - * early; nxt_conn_close() still emits the FIN via its work-queue handler. + * request body upload uses an autoreset send timer). block_read stops + * a queued nxt_conn_io_read(), and the closing flag makes a queued + * nxt_conn_io_write() return. nxt_conn_close() sets both; the fd == -1 + * branch skips it and sets them here. block_write would not do: it sends + * a queued write to the write state's error_handler, which uses the peer. + * nxt_conn_close() still emits the FIN via its work-queue handler. */ c->block_read = 1; - c->block_write = 1; nxt_timer_disable(task->thread->engine, &c->read_timer); nxt_timer_disable(task->thread->engine, &c->write_timer); @@ -3275,6 +3838,8 @@ nxt_h1p_peer_close(nxt_task_t *task, nxt_http_peer_t *peer) nxt_conn_close(task->thread->engine, c); } else { + c->closing = 1; + nxt_h1p_peer_free(task, c, NULL); } } @@ -3300,20 +3865,89 @@ nxt_h1p_peer_free(nxt_task_t *task, void *obj, void *data) } +/* + * Transfer-Encoding is a comma-separated list of transfer codings + * (RFC 9112 Sect. 6.1). Coding names are case-insensitive. "chunked" must + * be the last coding, and it must appear only once. Several + * Transfer-Encoding lines form one list, in order. So each line starts from + * the state that the previous lines left in h1p->transfer_encoding. + * + * The proxy decodes only chunked, and it never forwards Transfer-Encoding to + * the client. So the result is NXT_HTTP_TE_CHUNKED only when the whole list + * is one "chunked", in any letter case. Every other list is + * NXT_HTTP_TE_UNSUPPORTED: an unknown coding, "gzip, chunked", + * "chunked, gzip", chunked twice, an empty value, or a coding with + * parameters. nxt_h1p_peer_header_read_done() then fails the response with + * 502. RFC 9112 Sect. 6.3 would read such a body until the connection + * closes. But the proxy cannot decode that body, and the client would get it + * in a coding that no header names. + */ + static nxt_int_t nxt_h1p_peer_transfer_encoding(void *ctx, nxt_http_field_t *field, uintptr_t data) { + u_char *p, *end, *start, *last; + nxt_bool_t empty; + nxt_h1proto_t *h1p; + nxt_http_te_t te; nxt_http_request_t *r; r = ctx; field->skip = 1; - if (field->value_length == 7 - && memcmp(field->value, "chunked", 7) == 0) - { - r->peer->proto.h1->chunked = 1; + h1p = r->peer->proto.h1; + te = h1p->transfer_encoding; + + p = field->value; + end = p + field->value_length; + empty = 1; + + while (p < end) { + start = p; + + while (p < end && *p != ',') { + p++; + } + + last = p; + + if (p < end) { + p++; /* Skip the comma. */ + } + + while (start < last && (*start == ' ' || *start == '\t')) { + start++; + } + + while (last > start && (last[-1] == ' ' || last[-1] == '\t')) { + last--; + } + + /* RFC 9110 Sect. 5.6.1: empty list elements are ignored. */ + if (start == last) { + continue; + } + + empty = 0; + + if (te == NXT_HTTP_TE_NONE + && last - start == nxt_length("chunked") + && nxt_memcasecmp(start, "chunked", nxt_length("chunked")) == 0) + { + te = NXT_HTTP_TE_CHUNKED; + + } else { + te = NXT_HTTP_TE_UNSUPPORTED; + } } + /* A field line must carry at least one transfer coding. */ + if (empty) { + te = NXT_HTTP_TE_UNSUPPORTED; + } + + h1p->transfer_encoding = te; + return NXT_OK; } diff --git a/src/nxt_h1proto.h b/src/nxt_h1proto.h index b324db8d1..6a45f9f54 100644 --- a/src/nxt_h1proto.h +++ b/src/nxt_h1proto.h @@ -16,16 +16,28 @@ typedef struct nxt_h1p_websocket_timer_s nxt_h1p_websocket_timer_t; +/* + * The trailing bytes of a UTF-8 sequence the end of a frame cut in half. + * A sequence is at most four bytes, so three of them can be incomplete; + * they are completed by the frame that continues the message. + */ +typedef struct { + uint8_t len; + u_char tail[3]; +} nxt_h1p_ws_utf8_t; + + struct nxt_h1proto_s { nxt_http_request_parse_t parser; nxt_http_chunk_parse_t chunked_parse; nxt_off_t remainder; - uint8_t nbuffers; + size_t nbuffers; uint8_t header_buffer_slot; uint8_t large_buffer_slot; uint8_t keepalive; /* 1 bit */ uint8_t chunked; /* 1 bit */ + uint8_t chunk_sent; /* 1 bit */ uint8_t websocket; /* 1 bit */ uint8_t connection_upgrade; /* 1 bit */ uint8_t upgrade_websocket; /* 1 bit */ @@ -34,11 +46,34 @@ struct nxt_h1proto_s { uint8_t websocket_cont_expected; /* 1 bit */ uint8_t websocket_closed; /* 1 bit */ + uint8_t websocket_text; /* 1 bit */ + uint8_t websocket_pong_queued; /* 1 bit */ + uint8_t continue_pending; /* 1 bit */ + + nxt_h1p_ws_utf8_t websocket_utf8; - uint32_t header_size; + /* + * Bytes written before the body: a 100 (Continue) and the response + * header. $body_bytes_sent does not count them. + */ + uint32_t sent_before_body; + + /* + * The minimum transfer rate state of the current request. The + * keep-alive code zeroes these fields before the next request. + */ + uint8_t body_rate_on; /* 1 bit */ + uint8_t send_rate_on; /* 1 bit */ + nxt_msec_t body_rate_start; + nxt_msec_t send_rate_start; + uint64_t body_rate_bytes; + uint64_t send_rate_bytes; + uint64_t send_rate_time; + nxt_off_t send_rate_sent; nxt_http_field_t *websocket_key; nxt_h1p_websocket_timer_t *websocket_timer; + nxt_buf_t *websocket_pong_next; nxt_http_request_t *request; nxt_buf_t *buffers; diff --git a/src/nxt_h1proto_websocket.c b/src/nxt_h1proto_websocket.c index d2cbee7ce..5e8c90aa0 100644 --- a/src/nxt_h1proto_websocket.c +++ b/src/nxt_h1proto_websocket.c @@ -34,6 +34,15 @@ static void hxt_h1p_send_ws_error(nxt_task_t *task, nxt_http_request_t *r, const nxt_ws_error_t *err, ...); static void nxt_h1p_conn_ws_error_sent(nxt_task_t *task, void *obj, void *data); static void nxt_h1p_conn_ws_pong(nxt_task_t *task, void *obj, void *data); +static void nxt_h1p_conn_ws_pong_send(nxt_task_t *task, nxt_http_request_t *r, + nxt_buf_t *out); +static void nxt_h1p_conn_ws_pong_sent(nxt_task_t *task, void *obj, + void *data); +static size_t nxt_h1p_ws_utf8_seqlen(u_char c); +static nxt_int_t nxt_h1p_ws_utf8_begin(const u_char *seq, size_t n); +static nxt_int_t nxt_h1p_ws_utf8_validate(nxt_h1p_ws_utf8_t *state, + nxt_buf_t *b, u_char *start, size_t skip, size_t length, + const u_char *mask, nxt_uint_t final); static const nxt_conn_state_t nxt_h1p_read_ws_frame_header_state; static const nxt_conn_state_t nxt_h1p_read_ws_frame_payload_state; @@ -71,6 +80,12 @@ static const nxt_ws_error_t nxt_ws_err_cont_expected = { static const nxt_ws_error_t nxt_ws_err_invalid_length = { NXT_WEBSOCKET_CR_PROTOCOL_ERROR, 0, nxt_string("Invalid extended payload length") }; +static const nxt_ws_error_t nxt_ws_err_invalid_utf8 = { + NXT_WEBSOCKET_CR_INVALID_DATA, + 0, nxt_string("Invalid UTF-8") }; +static const nxt_ws_error_t nxt_ws_err_truncated = { + NXT_WEBSOCKET_CR_PROTOCOL_ERROR, + 0, nxt_string("Frame shorter than its header") }; void nxt_h1p_websocket_first_frame_start(nxt_task_t *task, nxt_http_request_t *r, @@ -326,6 +341,21 @@ nxt_h1p_conn_ws_frame_header_read(nxt_task_t *task, void *obj, void *data) wsh->opcode); return; } + + /* + * A message starts here: remember whether its payload is text, + * and drop a sequence the previous message left incomplete. A + * frame with an RSV bit set belongs to an extension this router + * does not implement (RFC 6455 Section 5.2; RFC 7692 Section 6 + * puts the "Per-Message Compressed" bit, RSV1, on the first + * fragment of a compressed data message), so its bytes are not + * text until the application has decoded them. + */ + h1p->websocket_text = (wsh->opcode == NXT_WEBSOCKET_OP_TEXT + && wsh->rsv1 == 0 + && wsh->rsv2 == 0 + && wsh->rsv3 == 0); + nxt_memzero(&h1p->websocket_utf8, sizeof(nxt_h1p_ws_utf8_t)); } h1p->websocket_cont_expected = !wsh->fin; @@ -410,13 +440,190 @@ nxt_h1p_conn_ws_keepalive_enable(nxt_task_t *task, nxt_h1proto_t *h1p) } +/* + * Returns the length of the UTF-8 sequence a lead byte announces, or 0 when + * the byte cannot begin one. C0 and C1 have only overlong two-byte + * encodings and F5..FF are beyond U+10FFFF, so neither can lead a sequence + * (RFC 3629 Section 3). Shortest form, surrogates, and the byte ranges that + * apply only after E0, ED, F0 and F4 are left to nxt_utf8_decode(). + */ +static size_t +nxt_h1p_ws_utf8_seqlen(u_char c) +{ + if (c >= 0xF0) { + return (c <= 0xF4) ? 4 : 0; + } + + if (c >= 0xE0) { + return 3; + } + + return (c >= 0xC2) ? 2 : 0; +} + + +/* + * Decides whether the "n" bytes at "seq", the whole or the beginning of one + * UTF-8 sequence, can be a valid character: NXT_OK when they already are, + * NXT_AGAIN when a continuation is still needed, NXT_ERROR when no byte + * following them can repair the sequence. + * + * A partial sequence is completed with the smallest continuation bytes a + * valid encoding could have and handed to nxt_utf8_decode(), so that this + * test cannot drift from the decoder that finally accepts the bytes: E0 must + * be followed by A0..BF, F0 by 90..BF and F4 by 80..8F, or the character is + * overlong or beyond U+10FFFF however the sequence ends. + */ +static nxt_int_t +nxt_h1p_ws_utf8_begin(const u_char *seq, size_t n) +{ + u_char buf[4]; + size_t len; + const u_char *p; + + len = nxt_h1p_ws_utf8_seqlen(seq[0]); + + if (nxt_slow_path(len == 0)) { + return NXT_ERROR; + } + + if (n < len) { + nxt_memcpy(buf, seq, n); + + if (n == 1) { + switch (seq[0]) { + case 0xE0: + buf[n++] = 0xA0; + break; + case 0xF0: + buf[n++] = 0x90; + break; + default: + buf[n++] = 0x80; + } + } + + while (n < 4) { + buf[n++] = 0x80; + } + + p = buf; + + if (nxt_utf8_decode(&p, buf + 4) == 0xFFFFFFFF) { + return NXT_ERROR; + } + + return NXT_AGAIN; + } + + p = seq; + + if (nxt_utf8_decode(&p, seq + n) == 0xFFFFFFFF) { + return NXT_ERROR; + } + + return NXT_OK; +} + + +/* + * Validates the payload of a data or close frame as UTF-8. "length" bytes + * are read from "start", the first payload byte in "b"; the rest of the + * frame, if it did not fit in one buffer, follows in the "next" chain. The + * payload is raw, so every byte is unmasked with mask[i % 4] as it is read, + * "i" being its offset from the first byte of the payload. "skip" bytes + * before "start" are stepped over: they are the close code, which is not + * text. + * + * A sequence the end of the frame cut short is kept in "state" and completed + * by the frame that continues the message. With "final" set the payload + * must end on a character boundary, as a message and a close reason do. + */ +static nxt_int_t +nxt_h1p_ws_utf8_validate(nxt_h1p_ws_utf8_t *state, nxt_buf_t *b, u_char *start, + size_t skip, size_t length, const u_char *mask, nxt_uint_t final) +{ + u_char c, seq[4]; + size_t i, n; + const u_char *p, *end; + nxt_buf_t *in; + + in = b; + p = start; + end = in->mem.free; + + for (i = 0; i < skip + length; /* void */) { + + while (nxt_slow_path(p >= end)) { + in = in->next; + + if (nxt_slow_path(in == NULL)) { + /* + * The frame is shorter than its header says, which cannot + * happen: a frame is buffered whole before it is handled + * here. Answered apart from invalid UTF-8 so that a reader + * fault is not reported as an encoding one. + */ + return NXT_DECLINED; + } + + p = in->mem.start; + end = in->mem.free; + } + + c = *p++ ^ mask[i % 4]; + i++; + + if (nxt_slow_path(i <= skip)) { + continue; + } + + if (nxt_fast_path(state->len == 0 && c < 0x80)) { + continue; + } + + nxt_memcpy(seq, state->tail, state->len); + seq[state->len] = c; + n = state->len + 1; + + switch (nxt_h1p_ws_utf8_begin(seq, n)) { + + case NXT_OK: + state->len = 0; + break; + + case NXT_AGAIN: + if (nxt_slow_path(n > sizeof(state->tail))) { + /* Cannot be taken: NXT_AGAIN means n < len <= 4. */ + return NXT_ERROR; + } + + nxt_memcpy(state->tail, seq, n); + state->len = n; + break; + + default: + return NXT_ERROR; + } + } + + if (nxt_slow_path(final && state->len != 0)) { + return NXT_ERROR; + } + + return NXT_OK; +} + + static void nxt_h1p_conn_ws_frame_process(nxt_task_t *task, nxt_conn_t *c, nxt_h1proto_t *h1p, nxt_websocket_header_t *wsh) { - size_t hsize; + size_t hsize, payload_len; uint8_t *p, *mask; uint16_t code; + nxt_int_t res; + nxt_h1p_ws_utf8_t utf8; nxt_http_request_t *r; r = h1p->request; @@ -444,9 +651,74 @@ nxt_h1p_conn_ws_frame_process(nxt_task_t *task, nxt_conn_t *c, code); return; } + + /* + * RFC 6455 Section 5.5.1: the reason is UTF-8-encoded. It does + * not continue the message in progress, so it is validated + * against its own state and must be complete. Unlike a data + * frame it is read with an RSV bit set: permessage-deflate and + * its relatives apply to data messages only (RFC 7692 + * Section 6), and the 2-byte code above is read the same way. + */ + if (wsh->payload_len > 2) { + payload_len = (size_t) nxt_websocket_frame_payload_len(wsh); + + nxt_memzero(&utf8, sizeof(nxt_h1p_ws_utf8_t)); + + res = nxt_h1p_ws_utf8_validate(&utf8, r->ws_frame, p, 2, + payload_len - 2, mask, 1); + + if (nxt_slow_path(res != NXT_OK)) { + hxt_h1p_send_ws_error(task, r, + (res == NXT_DECLINED) + ? &nxt_ws_err_truncated + : &nxt_ws_err_invalid_utf8); + return; + } + } } h1p->websocket_closed = 1; + nxt_memzero(&h1p->websocket_utf8, sizeof(nxt_h1p_ws_utf8_t)); + + } else if (h1p->websocket_text + && (wsh->opcode == NXT_WEBSOCKET_OP_TEXT + || wsh->opcode == NXT_WEBSOCKET_OP_CONT)) + { + if (nxt_slow_path(wsh->rsv1 != 0 || wsh->rsv2 != 0 + || wsh->rsv3 != 0)) + { + /* + * Some extension defines this frame (RFC 6455 Section 5.2), so + * neither it nor what follows in the message can be read as + * text here. + */ + h1p->websocket_text = 0; + + } else { + /* + * RFC 6455 Section 5.6: a text message is UTF-8 text however + * many frames it is split into, and Section 8.1 fails the + * connection on data it cannot interpret; Section 7.4.1 gives + * the close code for that, 1007. Binary messages are not text + * and are passed on unread. + */ + hsize = nxt_websocket_frame_header_size(wsh); + mask = nxt_pointer_to(wsh, hsize - 4); + payload_len = (size_t) nxt_websocket_frame_payload_len(wsh); + + res = nxt_h1p_ws_utf8_validate(&h1p->websocket_utf8, r->ws_frame, + nxt_pointer_to(wsh, hsize), 0, + payload_len, mask, wsh->fin); + + if (nxt_slow_path(res != NXT_OK)) { + hxt_h1p_send_ws_error(task, r, + (res == NXT_DECLINED) + ? &nxt_ws_err_truncated + : &nxt_ws_err_invalid_utf8); + return; + } + } } r->state->ready_handler(task, r, NULL); @@ -662,6 +934,8 @@ nxt_h1p_conn_ws_pong(nxt_task_t *task, void *obj, void *data) { uint8_t payload_len, i; nxt_buf_t *b, *out, *next; + nxt_bool_t deferred; + nxt_h1proto_t *h1p; nxt_http_request_t *r; nxt_websocket_header_t *wsh; uint8_t mask[4]; @@ -669,6 +943,7 @@ nxt_h1p_conn_ws_pong(nxt_task_t *task, void *obj, void *data) nxt_debug(task, "h1p conn ws pong"); r = obj; + h1p = r->proto.h1; b = r->ws_frame; wsh = (nxt_websocket_header_t *) b->mem.pos; @@ -680,10 +955,42 @@ nxt_h1p_conn_ws_pong(nxt_task_t *task, void *obj, void *data) b->mem.pos += 4; - out = nxt_http_buf_mem(task, r, 2 + payload_len); - if (nxt_slow_path(out == NULL)) { - nxt_http_request_error_handler(task, r, r->proto.any); - return; + /* + * RFC 6455 Section 5.5.3 lets an endpoint send a PONG only for the most + * recent PING when it did not send the PONGs for the earlier PINGs yet. + * A client that sends PINGs and does not read the PONGs could otherwise + * make the router keep a PONG in memory for each PING. + * + * Thus, while a PONG is queued, the router does not queue a new PONG. + * It keeps the PONG for the most recent PING in a buffer that the next + * PING overwrites, and queues it when the queued PONG is sent. The + * router then keeps at most two PONGs for a connection. + */ + + deferred = h1p->websocket_pong_queued; + + if (deferred) { + out = h1p->websocket_pong_next; + + if (out == NULL) { + /* A control frame payload is 125 bytes or less. */ + out = nxt_buf_mem_alloc(r->mem_pool, 2 + 125, 0); + if (nxt_slow_path(out == NULL)) { + nxt_http_request_error_handler(task, r, r->proto.any); + return; + } + + h1p->websocket_pong_next = out; + } + + nxt_debug(task, "h1p conn ws pong deferred"); + + } else { + out = nxt_http_buf_mem(task, r, 2 + payload_len); + if (nxt_slow_path(out == NULL)) { + nxt_http_request_error_handler(task, r, r->proto.any); + return; + } } out->mem.start[0] = 0; @@ -711,7 +1018,75 @@ nxt_h1p_conn_ws_pong(nxt_task_t *task, void *obj, void *data) r->ws_frame = b; - nxt_http_request_send(task, r, out); + if (!deferred) { + nxt_h1p_conn_ws_pong_send(task, r, out); + } nxt_http_request_ws_frame_start(task, r, r->ws_frame); } + + +static void +nxt_h1p_conn_ws_pong_send(nxt_task_t *task, nxt_http_request_t *r, + nxt_buf_t *out) +{ + out->completion_handler = nxt_h1p_conn_ws_pong_sent; + out->parent = r; + + r->proto.h1->websocket_pong_queued = 1; + + nxt_http_request_send(task, r, out); +} + + +/* + * The queued PONG is sent, or it is discarded. If a PING came in the + * meantime, queue the PONG for the most recent PING now. Do not queue it + * when the request is closed or failed, or when its last buffer is sent: + * no frame can follow the last buffer. + */ + +static void +nxt_h1p_conn_ws_pong_sent(nxt_task_t *task, void *obj, void *data) +{ + nxt_buf_t *b, *next, *out; + nxt_h1proto_t *h1p; + nxt_http_request_t *r; + + b = obj; + r = data; + + nxt_debug(task, "h1p conn ws pong sent"); + + h1p = r->proto.h1; + + if (h1p != NULL) { + h1p->websocket_pong_queued = 0; + + out = h1p->websocket_pong_next; + h1p->websocket_pong_next = NULL; + + if (out != NULL) { + if (r->error || r->last == NULL || h1p->conn->closing) { + nxt_mp_free(r->mem_pool, out); + + } else { + /* The completion handler releases the pool. */ + nxt_mp_retain(r->mem_pool); + + nxt_h1p_conn_ws_pong_send(task, r, out); + } + } + } + + /* The pool can be freed with the last release, and r with it. */ + + do { + next = b->next; + + nxt_mp_free(r->mem_pool, b); + nxt_mp_release(r->mem_pool); + + b = next; + } while (b != NULL); +} diff --git a/src/nxt_http.h b/src/nxt_http.h index df24e7ebc..49531ae70 100644 --- a/src/nxt_http.h +++ b/src/nxt_http.h @@ -19,6 +19,7 @@ typedef enum { NXT_HTTP_SWITCHING_PROTOCOLS = 101, NXT_HTTP_OK = 200, + NXT_HTTP_PARTIAL_CONTENT = 206, NXT_HTTP_NO_CONTENT = 204, NXT_HTTP_MULTIPLE_CHOICES = 300, @@ -36,8 +37,10 @@ typedef enum { NXT_HTTP_NOT_ACCEPTABLE = 406, NXT_HTTP_REQUEST_TIMEOUT = 408, NXT_HTTP_LENGTH_REQUIRED = 411, + NXT_HTTP_PRECONDITION_FAILED = 412, NXT_HTTP_PAYLOAD_TOO_LARGE = 413, NXT_HTTP_URI_TOO_LONG = 414, + NXT_HTTP_RANGE_NOT_SATISFIABLE = 416, NXT_HTTP_UPGRADE_REQUIRED = 426, NXT_HTTP_REQUEST_HEADER_FIELDS_TOO_LARGE = 431, @@ -62,6 +65,18 @@ typedef enum { } nxt_http_te_t; +/* What "response_headers" will do to one response field. */ +typedef enum { + NXT_HTTP_SET_HEADER_NONE = 0, + NXT_HTTP_SET_HEADER_REPLACE, + NXT_HTTP_SET_HEADER_REMOVE, + NXT_HTTP_SET_HEADER_ERROR, +} nxt_http_set_header_op_t; + + +typedef struct nxt_http_set_headers_ctx_s nxt_http_set_headers_ctx_t; + + typedef enum { NXT_HTTP_PROTO_H1 = 0, NXT_HTTP_PROTO_H2, @@ -115,6 +130,14 @@ typedef struct { nxt_http_field_t *content_length; nxt_off_t content_length_n; const nxt_str_t *mime_type; + + /* + * The response was chosen by negotiation on Accept-Encoding, so its Vary + * must name that header. Kept on the response because "response_headers" + * is applied after the header is generated and may replace or remove it; + * nxt_http_request_header_send() re-asserts it once that has run. + */ + uint8_t vary_accept_encoding; /* 1 bit */ } nxt_http_response_t; @@ -165,6 +188,8 @@ typedef struct { nxt_http_protocol_t protocol:8; /* 2 bits */ uint8_t header_received; /* 1 bit */ uint8_t closed; /* 1 bit */ + /* number of 1xx responses dropped */ + uint8_t num_interim; } nxt_http_peer_t; @@ -227,6 +252,7 @@ struct nxt_http_request_s { nxt_tstr_query_t *tstr_query; nxt_tstr_cache_t tstr_cache; + nxt_http_set_headers_ctx_t *set_headers; nxt_http_action_t *action; void *req_rpc_data; @@ -250,6 +276,9 @@ struct nxt_http_request_s { nxt_http_static_ctx_t static_ctx; + /* Set by nxt_http_comp_check_acceptable(), from mem_pool. */ + nxt_http_comp_ctx_t *comp_ctx; + nxt_http_status_t status:16; uint8_t log_route; /* 1 bit */ @@ -267,6 +296,7 @@ struct nxt_http_request_s { uint8_t error; /* 1 bit */ uint8_t websocket_handshake; /* 1 bit */ uint8_t chunked; /* 1 bit */ + uint8_t no_body; /* 1 bit */ }; @@ -424,12 +454,17 @@ nxt_http_request_t *nxt_http_request_create(nxt_task_t *task); void nxt_http_request_error(nxt_task_t *task, nxt_http_request_t *r, nxt_http_status_t status); void nxt_http_request_read_body(nxt_task_t *task, nxt_http_request_t *r); +nxt_int_t nxt_http_request_body_alloc(nxt_task_t *task, nxt_http_request_t *r, + size_t body_length); void nxt_http_request_header_send(nxt_task_t *task, nxt_http_request_t *r, nxt_work_handler_t body_handler, void *data); void nxt_http_request_ws_frame_start(nxt_task_t *task, nxt_http_request_t *r, nxt_buf_t *ws_frame); void nxt_http_request_send(nxt_task_t *task, nxt_http_request_t *r, nxt_buf_t *out); +nxt_bool_t nxt_http_request_is_bodyless_final(nxt_http_request_t *r, + nxt_http_status_t status); +nxt_bool_t nxt_http_status_no_representation(nxt_http_status_t status); nxt_buf_t *nxt_http_buf_mem(nxt_task_t *task, nxt_http_request_t *r, size_t size); nxt_buf_t *nxt_http_buf_last(nxt_http_request_t *r); @@ -438,6 +473,7 @@ void nxt_http_request_close_handler(nxt_task_t *task, void *obj, void *data); nxt_int_t nxt_http_request_host(void *ctx, nxt_http_field_t *field, uintptr_t data); +nxt_int_t nxt_http_validate_host(nxt_str_t *host, nxt_mp_t *mp); nxt_int_t nxt_http_request_field(void *ctx, nxt_http_field_t *field, uintptr_t offset); nxt_int_t nxt_http_request_content_length(void *ctx, nxt_http_field_t *field, @@ -485,6 +521,10 @@ nxt_int_t nxt_http_rewrite_init(nxt_router_conf_t *rtcf, nxt_http_action_t *action, nxt_http_action_conf_t *acf); nxt_int_t nxt_http_rewrite(nxt_task_t *task, nxt_http_request_t *r); +nxt_bool_t nxt_http_set_headers_override_validators(nxt_http_request_t *r); +nxt_http_set_header_op_t nxt_http_set_headers_field_op(nxt_http_request_t *r, + const char *name, size_t length); +nxt_int_t nxt_http_comp_merge_vary(nxt_http_request_t *r); nxt_int_t nxt_http_set_headers_init(nxt_router_conf_t *rtcf, nxt_http_action_t *action, nxt_http_action_conf_t *acf); nxt_int_t nxt_http_set_headers(nxt_http_request_t *r); @@ -518,6 +558,8 @@ nxt_int_t nxt_http_proxy_skip(void *ctx, nxt_http_field_t *field, uintptr_t data); nxt_buf_t *nxt_http_proxy_buf_mem_alloc(nxt_task_t *task, nxt_http_request_t *r, size_t size); +nxt_int_t nxt_http_proxy_buf_mem_hold(nxt_task_t *task, nxt_http_request_t *r, + nxt_buf_t *b); void nxt_http_proxy_buf_mem_free(nxt_task_t *task, nxt_http_request_t *r, nxt_buf_t *b); diff --git a/src/nxt_http_chunk_parse.c b/src/nxt_http_chunk_parse.c index 559d704ae..a8c703c34 100644 --- a/src/nxt_http_chunk_parse.c +++ b/src/nxt_http_chunk_parse.c @@ -398,6 +398,13 @@ nxt_http_chunk_buffer(nxt_http_chunk_parse_t *hcp, nxt_buf_t ***tail, hcp->chunk_size -= size; if (hcp->chunk_size == 0) { + /* + * The whole chunk data is in the output buffer. Consume it, so + * that a caller that keeps the bytes after pos for the next read + * does not parse the data again as framing. + */ + in->mem.pos = in->mem.free; + return NXT_HTTP_CHUNK_END_ON_BORDER; } diff --git a/src/nxt_http_compression.c b/src/nxt_http_compression.c index 646cdfb95..8d800f5f5 100644 --- a/src/nxt_http_compression.c +++ b/src/nxt_http_compression.c @@ -18,12 +18,14 @@ #define NXT_COMP_LEVEL_UNSET INT8_MIN +/* Headroom for the flush marker a compressor emits per call. */ +#define NXT_HTTP_COMP_FLUSH_SLACK 64 + typedef enum nxt_http_comp_scheme_e nxt_http_comp_scheme_t; typedef struct nxt_http_comp_type_s nxt_http_comp_type_t; typedef struct nxt_http_comp_opts_s nxt_http_comp_opts_t; typedef struct nxt_http_comp_compressor_s nxt_http_comp_compressor_t; -typedef struct nxt_http_comp_ctx_s nxt_http_comp_ctx_t; enum nxt_http_comp_scheme_e { NXT_HTTP_COMP_SCHEME_IDENTITY = 0, @@ -63,25 +65,63 @@ struct nxt_http_comp_compressor_s { nxt_http_comp_opts_t opts; }; +/* + * One per request: nxt_http_comp_check_acceptable() allocates it from the + * request pool and stores it in r->comp_ctx. It was one object per router + * thread. But an application response is compressed in several event loop + * turns, one for each message from the application. So two responses on + * one thread used the same stream. A response could get the bytes of + * another response, or use a stream that the other response had freed. + */ struct nxt_http_comp_ctx_s { nxt_uint_t idx; + /* + * The compressor's type table entry, copied out of the configuration + * when the choice is applied. The table is static and lives as long as + * the process, so a body that is still being compressed when the + * configuration is replaced keeps working from here instead of indexing + * a per-configuration array that may already be freed. + */ + const nxt_http_comp_type_t *type; + + /* + * The compressor nxt_http_comp_check_acceptable() chose, or -1 when the + * request will not be compressed. Kept apart from "idx", which is only + * set once the choice has actually been applied. + */ + nxt_int_t sel_idx; + + /* + * The client sent "identity;q=0": it will not take the file's own bytes. + * Recorded separately from sel_idx because a request that refuses + * identity and accepts gzip selects gzip and is perfectly serveable -- + * until a Range enters, which is served as identity. + */ + bool identity_refused; + nxt_off_t resp_clen; nxt_off_t clen_sent; + /* The stream in "ctx" is initialised and not yet freed. */ + bool live; + nxt_http_comp_compressor_ctx_t ctx; }; -static nxt_tstr_t *nxt_http_comp_accept_encoding_query; -static nxt_http_route_rule_t *nxt_http_comp_mime_types_rule; -static nxt_http_comp_compressor_t *nxt_http_comp_enabled_compressors; -static nxt_uint_t nxt_http_comp_nr_enabled_compressors; - -static nxt_thread_declare_data(nxt_http_comp_ctx_t, - nxt_http_comp_compressor_ctx); - -#define nxt_http_comp_ctx() nxt_thread_get_data(nxt_http_comp_compressor_ctx) +/* + * Everything here is allocated from the router configuration's pools, so it + * is held per configuration and reached through the request. It used to be + * four process-global pointers, which dangled as soon as the configuration + * they came from was freed -- reconfiguring compression away killed the + * router on the next request (#167). + */ +struct nxt_http_comp_conf_s { + nxt_http_route_rule_t *mime_types_rule; + nxt_http_comp_compressor_t *enabled; + nxt_uint_t nr_enabled; +}; static const nxt_conf_map_t nxt_http_comp_compressors_opts_map[] = { { @@ -138,30 +178,72 @@ static const nxt_http_comp_type_t nxt_http_comp_compressors[] = { }; +nxt_inline nxt_http_comp_conf_t * +nxt_http_comp_request_conf(const nxt_http_request_t *r) +{ + return r->conf->socket_conf->router_conf->compression; +} + + +/* + * Frees the stream of one response. This occurs after the last deflate() + * call, after a failed call, and from the request pool cleanup when the + * response stops before its end: the client closed the connection, the + * application failed, or a timer expired. Only the first of these calls + * frees the stream. + */ + +static void +nxt_http_comp_stream_free(nxt_http_comp_ctx_t *ctx) +{ + if (!ctx->live) { + return; + } + + ctx->live = false; + + ctx->type->cops->free(&ctx->ctx); +} + + +static void +nxt_http_comp_ctx_cleanup(nxt_task_t *task, void *obj, void *data) +{ + nxt_http_comp_stream_free(obj); +} + + static ssize_t -nxt_http_comp_compress(uint8_t *dst, size_t dst_size, const uint8_t *src, - size_t src_size, bool last) +nxt_http_comp_compress(nxt_http_comp_ctx_t *ctx, uint8_t *dst, + size_t dst_size, const uint8_t *src, size_t src_size, + bool last) { - nxt_http_comp_ctx_t *ctx = nxt_http_comp_ctx(); - nxt_http_comp_compressor_t *compressor; + ssize_t ret; const nxt_http_comp_operations_t *cops; - compressor = &nxt_http_comp_enabled_compressors[ctx->idx]; - cops = compressor->type->cops; + /* Input after the end of the stream, or after a failed call. */ + if (nxt_slow_path(!ctx->live)) { + return -1; + } + + cops = ctx->type->cops; - return cops->deflate(&ctx->ctx, src, src_size, dst, dst_size, last); + ret = cops->deflate(&ctx->ctx, src, src_size, dst, dst_size, last); + + if (last || ret == -1) { + nxt_http_comp_stream_free(ctx); + } + + return ret; } static size_t -nxt_http_comp_bound(size_t size) +nxt_http_comp_bound(nxt_http_comp_ctx_t *ctx, size_t size) { - nxt_http_comp_ctx_t *ctx = nxt_http_comp_ctx(); - nxt_http_comp_compressor_t *compressor; const nxt_http_comp_operations_t *cops; - compressor = &nxt_http_comp_enabled_compressors[ctx->idx]; - cops = compressor->type->cops; + cops = ctx->type->cops; return cops->bound(&ctx->ctx, size); } @@ -174,59 +256,112 @@ nxt_http_comp_compress_app_response(nxt_task_t *task, nxt_http_request_t *r, bool last; size_t buf_len; ssize_t cbytes; - nxt_buf_t *buf; + nxt_buf_t *in, *next, *buf, *out, **tail; nxt_off_t in_len; - nxt_http_comp_ctx_t *ctx = nxt_http_comp_ctx(); + nxt_http_comp_ctx_t *ctx = r->comp_ctx; - if (ctx->idx == NXT_HTTP_COMP_SCHEME_IDENTITY) { + if (ctx == NULL || ctx->idx == NXT_HTTP_COMP_SCHEME_IDENTITY) { return NXT_OK; } - if (!nxt_buf_is_port_mmap(*b)) { - return NXT_OK; - } + /* + * What arrives here is a chain, not a single buffer: + * nxt_port_mmap_read() (src/nxt_port_memory.c) makes one nxt_buf_t per + * nxt_port_mmap_msg_t, the call site links it into r->out with + * nxt_buf_chain_add(), and a sync buffer may sit at the tail. Walk it. + * + * The previous version compressed the head and then replaced the whole + * chain with that one output buffer, so anything behind the head was + * dropped from the response and never released, and ctx->clen_sent + * counted only the head -- which is what decides whether the compressor + * is told it is finishing the stream. + */ - in_len = (*b)->mem.free - (*b)->mem.pos; - buf_len = nxt_http_comp_bound(in_len); + out = NULL; + tail = &out; - buf = nxt_buf_mem_ts_alloc(task, (*b)->data, buf_len); - if (nxt_slow_path(buf == NULL)) { - return NXT_ERROR; - } + for (in = *b; in != NULL; in = next) { + next = in->next; + in->next = NULL; + + /* + * Buffers that did not come through shared memory are passed + * through untouched: the trailing sync/last buffer carries no data, + * and a plain-mode response is not compressed at all (the same + * condition the single-buffer version tested on the head). + */ + if (!nxt_buf_is_port_mmap(in)) { + *tail = in; + tail = &in->next; + continue; + } - buf->data = (*b)->data; + in_len = in->mem.free - in->mem.pos; - last = ctx->clen_sent + in_len == ctx->resp_clen; + last = ctx->clen_sent + in_len == ctx->resp_clen; - cbytes = nxt_http_comp_compress(buf->mem.start, buf_len, - (*b)->mem.pos, in_len, last); - if (cbytes == -1) { - nxt_buf_free(buf->data, buf); - return NXT_ERROR; - } + if (in_len == 0 && !last) { + goto release; + } + + /* + * The per-call flush marker each compressor emits after the input + * is not part of what bound() promises for the input alone, so the + * output buffer gets a small fixed margin on top. + */ + buf_len = nxt_http_comp_bound(ctx, in_len) + + NXT_HTTP_COMP_FLUSH_SLACK; + + buf = nxt_buf_mem_ts_alloc(task, in->data, buf_len); + if (nxt_slow_path(buf == NULL)) { + goto fail; + } + + cbytes = nxt_http_comp_compress(ctx, buf->mem.start, buf_len, + in->mem.pos, in_len, last); + if (nxt_slow_path(cbytes == -1)) { + nxt_buf_free(buf->data, buf); + goto fail; + } - buf->mem.free += cbytes; + buf->mem.free += cbytes; - ctx->clen_sent += in_len; + ctx->clen_sent += in_len; -#define nxt_swap_buf(db, sb) \ - do { \ - nxt_buf_t **db_ = (db); \ - nxt_buf_t **sb_ = (sb); \ - nxt_buf_t *tmp_; \ - \ - tmp_ = *db_; \ - *db_ = *sb_; \ - *sb_ = tmp_; \ - } while (0) + *tail = buf; + tail = &buf->next; - nxt_swap_buf(b, &buf); + release: -#undef nxt_swap_buf + /* + * The compressed bytes have been copied out, so the shared memory + * chunk can go back to the application. This has to run the + * buffer's own completion handler -- nxt_buf_free() is a plain pool + * free, and using it here (as the single-buffer version did) leaked + * the chunk, the mmap_handler reference and the port mem_pool + * reference on every compressed response. in->next was cleared + * above because nxt_port_mmap_buf_completion() walks the chain. + */ + nxt_work_queue_add(&task->thread->engine->fast_work_queue, + in->completion_handler, task, in, in->parent); + } - nxt_buf_free(buf->data, buf); + *b = out; return NXT_OK; + +fail: + + /* + * Keep whatever has been produced so far, and re-attach the input that + * has not been consumed, so that the caller's error path owns the whole + * chain and releases it. + */ + in->next = next; + *tail = in; + *b = out; + + return NXT_ERROR; } @@ -235,12 +370,13 @@ nxt_http_comp_compress_static_response(nxt_task_t *task, nxt_http_request_t *r, nxt_file_t **f, nxt_file_info_t *fi, size_t static_buf_len, size_t *out_total) { - size_t in_size, out_size, rest; - char *tmp_path, *p; - uint8_t *in, *out; - nxt_int_t ret; - nxt_file_t tfile; - nxt_runtime_t *rt = task->thread->runtime; + size_t in_size, out_size, rest; + char *tmp_path, *p; + uint8_t *in, *out; + nxt_int_t ret; + nxt_file_t tfile; + nxt_runtime_t *rt = task->thread->runtime; + nxt_http_comp_ctx_t *ctx = r->comp_ctx; static const char *template = "unit-compr-XXXXXX"; @@ -267,7 +403,7 @@ nxt_http_comp_compress_static_response(nxt_task_t *task, nxt_http_request_t *r, tfile.name = (nxt_file_name_t *)tmp_path; in_size = nxt_file_size(fi); - out_size = nxt_http_comp_bound(in_size); + out_size = nxt_http_comp_bound(ctx, in_size); ret = ftruncate(tfile.fd, out_size); if (nxt_slow_path(ret == -1)) { @@ -310,7 +446,8 @@ nxt_http_comp_compress_static_response(nxt_task_t *task, nxt_http_request_t *r, last = n == rest; - cbytes = nxt_http_comp_compress(out + *out_total, out_size - *out_total, + cbytes = nxt_http_comp_compress(ctx, out + *out_total, + out_size - *out_total, in + in_size - rest, n, last); if (cbytes == -1) { nxt_file_close(task, &tfile); @@ -347,26 +484,120 @@ nxt_http_comp_compress_static_response(nxt_task_t *task, nxt_http_request_t *r, } +/* + * Tells whether the negotiation chose a compressor for this response. It + * stays true after the stream is freed, so it does not say that a stream is + * live. + */ bool -nxt_http_comp_wants_compression(void) +nxt_http_comp_wants_compression(nxt_http_request_t *r) +{ + return r->comp_ctx != NULL && r->comp_ctx->idx; +} + + +bool +nxt_http_comp_identity_refused(nxt_http_request_t *r) +{ + return r->comp_ctx != NULL && r->comp_ctx->identity_refused; +} + + +/* + * Finds the ";q=" weight parameter in one Accept-Encoding element. + * + * RFC 9110 Sect. 12.4.2 spells the parameter "weight = OWS ';' OWS ('q' / + * 'Q') '=' qvalue", and an ABNF literal is case-insensitive besides, so + * "identity;Q=0" is as valid as "identity;q=0". A plain strstr() for ";q=" + * misses it, and for identity that means missing a refusal. Spaces are + * already gone by the time this runs. + */ + +static char * +nxt_http_comp_find_weight(char *tkn) +{ + for (char *p = tkn; (p = strchr(p, ';')) != NULL; p++) { + if ((p[1] == 'q' || p[1] == 'Q') && p[2] == '=') { + return p; + } + } + + return NULL; +} + + +/* + * Reads the qvalue that follows a weight's "=". + * + * RFC 9110 Sect. 12.4.2 spells it "( '0' [ '.' 0*3DIGIT ] ) / ( '1' [ '.' + * 0*3( '0' ) ] )": one leading digit, then at most a fraction. strtod() on + * its own is far looser and every way it is looser is a bug here. It takes + * no digits at all from "q=" and from "q=abc" and reports 0, which this + * function's caller reads as a refusal; it reads "q=0x10" as hexadecimal; + * and it turns "q=nan" into a NaN that compares false against both range + * bounds, so the element is kept and then outranks every real weight. + * + * So check the shape first and only then convert. The digit count in the + * fraction is not enforced: rejecting "q=0.0000" would read a client's + * refusal as an acceptance, which is the wrong way to be strict. + * + * An element whose weight does not parse is ignored, exactly like an element + * naming a coding this build does not have. Reading it as q=0 is the other + * defensible answer -- nginx reads a bad quantity as zero -- but a zero is a + * refusal here, so "identity;q=" would answer 406 to a client that refused + * nothing. + */ + +static bool +nxt_http_comp_parse_weight(const char *qvalue, double *qval) { - nxt_http_comp_ctx_t *ctx = nxt_http_comp_ctx(); + const char *p = qvalue; + + if (*p != '0' && *p != '1') { + return false; + } - return ctx->idx; + p++; + + if (*p == '.') { + p++; + + while (*p >= '0' && *p <= '9') { + p++; + } + } + + /* Anything may follow the weight, but only as a further parameter. */ + + if (*p != '\0' && *p != ';') { + return false; + } + + *qval = strtod(qvalue, NULL); + + return *qval <= 1.0; } static nxt_uint_t -nxt_http_comp_compressor_lookup_enabled(const nxt_str_t *token) +nxt_http_comp_compressor_lookup_enabled(const nxt_http_comp_conf_t *conf, + const nxt_str_t *token) { - if (token->start[0] == '*') { - return NXT_HTTP_COMP_SCHEME_IDENTITY; + /* With compression off, there is no enabled array to search. */ + + if (conf == NULL) { + return NXT_HTTP_COMP_SCHEME_UNKNOWN; } - for (nxt_uint_t i = 0; i < nxt_http_comp_nr_enabled_compressors; i++) { - if (nxt_strstr_eq(token, - &nxt_http_comp_enabled_compressors[i].type->token)) - { + /* + * RFC 9110 Sect. 8.4.1: a content coding is a token, and tokens are + * compared case-insensitively. "Identity;q=0" and "GZIP" are as valid + * as the lowercase spellings, and a case-sensitive compare silently + * ignores them -- which, for identity, means missing a refusal. + */ + + for (nxt_uint_t i = 0; i < conf->nr_enabled; i++) { + if (nxt_strcasestr_eq(token, &conf->enabled[i].type->token)) { return i; } } @@ -375,6 +606,24 @@ nxt_http_comp_compressor_lookup_enabled(const nxt_str_t *token) } +/* + * Tells whether the token names the identity coding. Identity is a + * representation that each response has. So it is recognised from the + * static table, not from the enabled compressors: it must be recognised + * with no compressor enabled. + */ + +static bool +nxt_http_comp_token_is_identity(const nxt_str_t *token) +{ + const nxt_http_comp_type_t *identity; + + identity = &nxt_http_comp_compressors[NXT_HTTP_COMP_SCHEME_IDENTITY]; + + return nxt_strcasestr_eq(token, &identity->token); +} + + /* * We need to parse the 'Accept-Encoding` header as described by * @@ -391,14 +640,176 @@ nxt_http_comp_compressor_lookup_enabled(const nxt_str_t *token) * * 'identity;q=0' seems to basically mean the same thing... */ +/* + * Collects every Accept-Encoding field into one value. + * + * RFC 9110 Sect. 5.3: a field that may carry a comma-separated list can be + * sent as several lines, and a recipient must treat them as one value joined + * by commas. A variable query answers with the first matching field only + * (nxt_http_var_header()), so "Accept-Encoding: gzip" followed by + * "Accept-Encoding: identity;q=0" lost the refusal and the request was served + * the identity bytes it had declined -- and in the other order the gzip it + * would have accepted was never seen, so it drew a 406. + * + * The single-field case, which is every ordinary request, points straight at + * the field and copies nothing. + */ + +static nxt_int_t +nxt_http_comp_accept_encoding(nxt_http_request_t *r, nxt_str_t *value) +{ + u_char *p; + size_t len; + nxt_uint_t n; + nxt_http_field_t *f, *first; + nxt_http_fields_iter_t iter; + + static const nxt_str_t accept_encoding = nxt_string("Accept-Encoding"); + + n = 0; + len = 0; + first = NULL; + + for (f = nxt_http_fields_first(&iter, r->inline_fields, + r->num_inline_fields, r->fields); + f != NULL; + f = nxt_http_fields_next(&iter)) + { + if (f->skip || f->name_length != accept_encoding.length + || nxt_strncasecmp(f->name, accept_encoding.start, + accept_encoding.length) != 0) + { + continue; + } + + if (n == 0) { + first = f; + + } else { + len += nxt_length(", "); + } + + len += f->value_length; + n++; + } + + if (n == 0) { + nxt_str_null(value); + return NXT_OK; + } + + if (n == 1) { + value->start = first->value; + value->length = first->value_length; + + return NXT_OK; + } + + p = nxt_mp_nget(r->mem_pool, len); + if (nxt_slow_path(p == NULL)) { + return NXT_ERROR; + } + + value->start = p; + value->length = len; + + n = 0; + + for (f = nxt_http_fields_first(&iter, r->inline_fields, + r->num_inline_fields, r->fields); + f != NULL; + f = nxt_http_fields_next(&iter)) + { + if (f->skip || f->name_length != accept_encoding.length + || nxt_strncasecmp(f->name, accept_encoding.start, + accept_encoding.length) != 0) + { + continue; + } + + if (n++ != 0) { + p = nxt_cpymem(p, ", ", nxt_length(", ")); + } + + p = nxt_cpymem(p, f->value, f->value_length); + } + + return NXT_OK; +} + + +/* + * The length that the compressor decision uses. Returns -1 when the length + * of the body is not known yet. + */ + +static nxt_off_t +nxt_http_comp_resp_length(const nxt_http_request_t *r) +{ + if (r->resp.content_length_n > -1) { + return r->resp.content_length_n; + } + + if (r->resp.content_length != NULL) { + return strtol((char *) r->resp.content_length->value, NULL, 10); + } + + return -1; +} + + +/* + * Tells whether "min_length" refuses this response. A body of unknown + * length is compressed, so it is never below the minimum. + * + * nxt_http_comp_select_compressor() and nxt_http_comp_apply_compression() + * both call this, and they must get the same answer. The first decides + * which coding is selected. The second decides whether the selected coding + * is applied. + */ + +static bool +nxt_http_comp_below_min_len(nxt_off_t clen, + const nxt_http_comp_compressor_t *compressor) +{ + return clen > -1 && clen < compressor->opts.min_len; +} + + static nxt_int_t -nxt_http_comp_select_compressor(nxt_http_request_t *r, const nxt_str_t *token) +nxt_http_comp_select_compressor(const nxt_http_comp_conf_t *conf, + nxt_http_request_t *r, const nxt_str_t *token, + bool *identity_refused) { + /* + * What the field said about identity. "identity_named" and + * "identity_named_ok" record an explicit "identity" token. The + * wildcard pair below records "*". They are kept apart because the + * explicit token wins. "named" records the enabled codings that the + * field listed. The wildcard does not stand for these codings. + */ bool identity_allowed = true; + bool identity_named = false; + bool identity_named_ok = false; + bool wildcard_seen = false; char *str, *tkn, *tail, *cur; double weight = 0.0; + double wildcard_qval = 0.0; + uint32_t named = 0; + nxt_off_t clen = nxt_http_comp_resp_length(r); nxt_int_t idx = NXT_HTTP_COMP_SCHEME_IDENTITY; + *identity_refused = false; + + /* + * No Accept-Encoding field: identity is acceptable. This also keeps + * nxt_str_cstrz() away from the NULL start of an absent field. + */ + + if (token->length == 0) { + return NXT_HTTP_COMP_SCHEME_IDENTITY; + } + str = nxt_str_cstrz(r->mem_pool, token); if (str == NULL) { return NXT_HTTP_COMP_SCHEME_IDENTITY; @@ -406,11 +817,17 @@ nxt_http_comp_select_compressor(nxt_http_request_t *r, const nxt_str_t *token) cur = tail = str; /* - * To ease parsing the Accept-Encoding header, remove all spaces, - * which hold no semantic meaning. + * To ease parsing the Accept-Encoding header, remove all optional + * whitespace, which holds no semantic meaning. + * + * OWS is SP or HTAB (RFC 9110 Sect. 5.6.3), and it is legal on either + * side of the weight's semicolon. Removing only the space left + * "identity;q=0" unparsed, so the element read as an unknown + * coding and the refusal it carried was lost -- which handed a client + * that refused identity a 206 of exactly those bytes. */ for (; *cur != '\0'; cur++) { - if (*cur == ' ') { + if (*cur == ' ' || *cur == '\t') { continue; } @@ -419,35 +836,84 @@ nxt_http_comp_select_compressor(nxt_http_request_t *r, const nxt_str_t *token) *tail = '\0'; while ((tkn = strsep(&str, ","))) { - char *qptr; - double qval = 1.0; - nxt_str_t enc; - nxt_uint_t ecidx; - nxt_http_comp_scheme_t scheme; - - qptr = strstr(tkn, ";q="); - if (qptr != NULL) { - nxt_errno = 0; - - qval = strtod(qptr + 3, NULL); - - if (nxt_errno == ERANGE || qval < 0.0 || qval > 1.0) { - continue; - } + bool wildcard; + char *qptr; + double qval = 1.0; + nxt_str_t enc; + nxt_uint_t ecidx; + + qptr = nxt_http_comp_find_weight(tkn); + if (qptr != NULL && !nxt_http_comp_parse_weight(qptr + 3, &qval)) { + continue; } enc.start = (u_char *)tkn; enc.length = qptr != NULL ? (size_t)(qptr - tkn) : strlen(tkn); - ecidx = nxt_http_comp_compressor_lookup_enabled(&enc); - if (ecidx == NXT_HTTP_COMP_SCHEME_UNKNOWN) { + /* + * The wildcard is the whole token, not merely its first character: + * "*" is a tchar, so "*foo" is a legal (and unknown) coding name, and + * matching on the first byte alone made it stand for every coding. + */ + + wildcard = (enc.length == 1 && enc.start[0] == '*'); + + /* + * The wildcard names no coding of its own. It is only recorded + * here. The pass after the loop turns it into candidates. + */ + + if (wildcard) { + wildcard_seen = true; + wildcard_qval = qval; continue; } - scheme = nxt_http_comp_enabled_compressors[ecidx].type->scheme; + /* + * Identity says whether the own bytes of the response are + * acceptable. That does not depend on the configured compressors. + * Read it before the lookup, so that a refusal still arrives when + * compression is off. That is the one case where identity is all + * the server can offer. + */ - if (qval == 0.0 && scheme == NXT_HTTP_COMP_SCHEME_IDENTITY) { - identity_allowed = false; + if (nxt_http_comp_token_is_identity(&enc)) { + ecidx = NXT_HTTP_COMP_SCHEME_IDENTITY; + identity_named = true; + identity_named_ok = (qval != 0.0); + + } else { + ecidx = nxt_http_comp_compressor_lookup_enabled(conf, &enc); + if (ecidx == NXT_HTTP_COMP_SCHEME_UNKNOWN) { + continue; + } + + /* + * Record the coding as named, whether or not it can be applied: + * the wildcard stands only for the codings that the field did + * not name. An index past the width of the mask is left out of + * the wildcard. That is the safe direction. "compressors" can + * repeat an encoding, so the index is not bounded by the number + * of distinct codings. + */ + + if (ecidx < sizeof(named) * 8) { + named |= (uint32_t) 1 << ecidx; + } + + /* + * A coding below its own "min_length" is never applied. So it + * cannot replace a refused identity, and it must not outrank a + * coding that can be applied. "min_length" is set per + * compressor. With gzip at 1000 and zstd at 0, a 100-byte + * response can be served as zstd. Before, gzip was selected on + * its weight alone, and a request that could be satisfied was + * refused. + */ + + if (nxt_http_comp_below_min_len(clen, &conf->enabled[ecidx])) { + continue; + } } if (qval == 0.0 || qval < weight) { @@ -458,6 +924,79 @@ nxt_http_comp_select_compressor(nxt_http_request_t *r, const nxt_str_t *token) weight = qval; } + /* + * Sect. 12.5.3: identity "is acceptable by default unless specifically + * excluded by the Accept-Encoding header field stating either + * 'identity;q=0' or '*;q=0' without a more specific entry for + * 'identity'". So the explicit token wins. In + * "gzip, identity;q=0.5, *;q=0" the client refused everything it did + * not name, and then named identity as acceptable. So identity is + * acceptable. + */ + + if (identity_named) { + identity_allowed = identity_named_ok; + + } else if (wildcard_seen) { + identity_allowed = (wildcard_qval != 0.0); + } + + *identity_refused = !identity_allowed; + + /* + * Sect. 12.5.3: "*" matches any available content coding not explicitly + * listed in the field. So it stands for each enabled coding that the + * client did not name, and for identity, at its own weight. Before, + * the wildcard was read as identity only. Then it could not select a + * compressor, and "identity;q=0, *;q=1" was answered 406 although gzip + * was enabled and applicable. + * + * This pass runs after the loop. So a named coding keeps its own + * weight at any place in the field. The comparison is strict, so a + * named coding wins a tie with the wildcard. Identity is taken before + * any compressor: all of these have the one wildcard weight, and + * identity needs nothing applied. + */ + + if (wildcard_qval > weight) { + if (!identity_named) { + idx = NXT_HTTP_COMP_SCHEME_IDENTITY; + weight = wildcard_qval; + + } else if (conf != NULL) { + for (nxt_uint_t i = 1; i < conf->nr_enabled; i++) { + const nxt_str_t *tok = &conf->enabled[i].type->token; + + if (i >= sizeof(named) * 8 + || (named & ((uint32_t) 1 << i)) != 0) + { + continue; + } + + /* + * A lookup by name finds only the first entry for a coding. + * So a repeated "compressors" entry cannot be reached by + * name. The wildcard must not reach it either. Otherwise + * "*" would run a coding with options that no named request + * can ask for. + */ + + if (nxt_http_comp_compressor_lookup_enabled(conf, tok) != i) { + continue; + } + + if (nxt_http_comp_below_min_len(clen, &conf->enabled[i])) { + continue; + } + + idx = i; + weight = wildcard_qval; + + break; + } + } + } + if (idx == NXT_HTTP_COMP_SCHEME_IDENTITY && !identity_allowed) { return -1; } @@ -467,7 +1006,8 @@ nxt_http_comp_select_compressor(nxt_http_request_t *r, const nxt_str_t *token) static nxt_int_t -nxt_http_comp_set_header(nxt_http_request_t *r, nxt_uint_t comp_idx) +nxt_http_comp_set_header(const nxt_http_comp_conf_t *conf, + nxt_http_request_t *r, nxt_uint_t comp_idx) { const nxt_str_t *token; nxt_http_field_t *f; @@ -480,7 +1020,7 @@ nxt_http_comp_set_header(nxt_http_request_t *r, nxt_uint_t comp_idx) return NXT_ERROR; } - token = &nxt_http_comp_enabled_compressors[comp_idx].type->token; + token = &conf->enabled[comp_idx].type->token; *f = (nxt_http_field_t){}; @@ -541,23 +1081,245 @@ nxt_http_comp_is_resp_content_encoded(const nxt_http_request_t *r) } +/* + * Adds "Vary: Accept-Encoding", so a shared cache keys on the header that + * chose this representation. + * + * RFC 9110 Sect. 12.5.5: a response that was subject to proactive negotiation + * must say which request headers it varied on, or a cache is entitled to + * serve it to a client that would have been given a different representation + * -- gzip bytes to a client that cannot decode them, or identity to one that + * could have had the small copy. + * + * Emitted on the identity response as well as the coded one. The identity + * response is precisely the one a cache must not reuse for a gzip-capable + * client, so omitting it there would leave the hole open from the other side. + * + * This is the companion of weakening the entity-tag for a coded + * representation: that makes revalidation distinguish the two, this makes the + * cache key distinguish them. Either alone leaves shared caches able to mix + * them. + */ + nxt_int_t -nxt_http_comp_check_compression(nxt_task_t *task, nxt_http_request_t *r) +nxt_http_comp_merge_vary(nxt_http_request_t *r) { - int err; - nxt_int_t ret, idx; - nxt_off_t min_len; - nxt_str_t accept_encoding, mime_type = {}; - nxt_router_conf_t *rtcf; - nxt_http_comp_ctx_t *ctx = nxt_http_comp_ctx(); - nxt_http_comp_compressor_t *compressor; + u_char *p, *end, *tok; + nxt_int_t len; + size_t keep; + nxt_http_field_t *f, *vary; + nxt_http_fields_iter_t iter; + + static const nxt_str_t accept_encoding = nxt_string("Accept-Encoding"); + + /* + * An existing Vary is merged into, not replaced and not deferred to. + * Something else naming a different header -- "Vary: Origin", say -- + * still needs Accept-Encoding added, because the response varies on both; + * treating any existing Vary as sufficient would leave the coding out of + * the cache key, which is the hole this function exists to close. + */ + + vary = NULL; + + for (f = nxt_http_fields_first(&iter, r->resp.inline_fields, + r->resp.num_inline_fields, r->resp.fields); + f != NULL; + f = nxt_http_fields_next(&iter)) + { + if (!f->skip && f->name_length == nxt_length("Vary") + && nxt_strncasecmp(f->name, (u_char *) "Vary", + nxt_length("Vary")) == 0) + { + vary = f; + break; + } + } + + if (vary != NULL) { + p = vary->value; + end = p + vary->value_length; + + /* + * "Vary: *" already varies on everything; adding to it says less. + * + * Trim both ends before the test. These are response fields an + * application handed to libunit, not request headers the parser has + * normalised, so "Vary: * " arrives with its trailing space intact -- + * and appending to that would emit "* , Accept-Encoding", which is + * not a valid field value. + */ + + while (p < end && (*p == ' ' || *p == '\t')) { + p++; + } + + while (end > p && (end[-1] == ' ' || end[-1] == '\t')) { + end--; + } + + if (end - p == 1 && *p == '*') { + return NXT_OK; + } + + /* + * An empty value carries no tokens, so there is nothing to append to + * and nothing to search: replacing it avoids emitting ", A-E" with a + * leading comma. RFC 9110 Sect. 5.6.1.2 permits the empty element, + * but there is no reason to produce one. + */ + + end = vary->value + vary->value_length; + + if (end == vary->value) { + vary->value = accept_encoding.start; + vary->value_length = accept_encoding.length; + + return NXT_OK; + } + + /* + * Already listed? Compare per token, so "X-Accept-Encoding" misses. + * + * A while loop rather than a for with p++: the inner scan can leave p + * at end, and incrementing there would form a pointer past + * one-past-the-end, which C does not define even where it is + * harmless in practice. + */ + + p = vary->value; + + while (p < end) { + while (p < end && (*p == ' ' || *p == '\t' || *p == ',')) { + p++; + } + + tok = p; + + while (p < end && *p != ',') { + p++; + } + + len = p - tok; + + while (len > 0 && (tok[len - 1] == ' ' || tok[len - 1] == '\t')) { + len--; + } + + if (len == (nxt_int_t) accept_encoding.length + && nxt_strncasecmp(tok, accept_encoding.start, + accept_encoding.length) == 0) + { + return NXT_OK; + } + + if (p < end) { + p++; + } + } + + /* + * Append after the last real token, not after whatever the value + * happens to end with: "Origin," would otherwise become + * "Origin,, Accept-Encoding". Empty list elements are legal and + * ignored (Sect. 5.6.1.2), but there is no reason to emit one. + */ - *ctx = (nxt_http_comp_ctx_t){ .resp_clen = -1 }; + keep = vary->value_length; + + while (keep > 0 + && (vary->value[keep - 1] == ' ' + || vary->value[keep - 1] == '\t' + || vary->value[keep - 1] == ',')) + { + keep--; + } + + len = keep + nxt_length(", ") + accept_encoding.length; + + p = nxt_mp_nget(r->mem_pool, len); + if (nxt_slow_path(p == NULL)) { + return NXT_ERROR; + } + + nxt_memcpy(p, vary->value, keep); + nxt_memcpy(p + keep, ", ", nxt_length(", ")); + nxt_memcpy(p + keep + nxt_length(", "), + accept_encoding.start, accept_encoding.length); + + vary->value = p; + vary->value_length = len; - if (nxt_http_comp_nr_enabled_compressors == 0) { return NXT_OK; } + f = nxt_http_resp_field_zero_add(&r->resp, r->mem_pool); + if (nxt_slow_path(f == NULL)) { + return NXT_ERROR; + } + + nxt_http_field_name_set(f, "Vary"); + + f->value = accept_encoding.start; + f->value_length = accept_encoding.length; + + return NXT_OK; +} + + +/* + * A 406 depends on Accept-Encoding as much as the representation that it + * replaces. So it carries "Vary: Accept-Encoding" too. Without it, a cache + * that stores error responses can give this 406 to a later client that could + * be served. nxt_http_request_error() drops each response field, but + * nxt_http_request_header_send() adds Vary again from this flag. + * + * Only the 406 carries Vary. When the types rule withdraws the coding, + * when the response has no media type, or when no compressor is + * configured, the 200 to a client that accepts identity has no Vary. + * A shared cache can give that 200 to a later client that refused + * identity. Vary on each such 200 is a larger change, and it is not + * made here. + */ + +static nxt_int_t +nxt_http_comp_not_acceptable(nxt_http_request_t *r) +{ + r->resp.vary_accept_encoding = 1; + + return NXT_HTTP_NOT_ACCEPTABLE; +} + + +/* + * Decides whether an acceptable representation exists, and remembers which + * compressor would be used, without touching the response or allocating a + * compressor context. + * + * Separated from applying that decision because RFC 9110 Sect. 13.2.1 puts + * this ahead of precondition evaluation: a request that cannot be satisfied + * at all must be answered 406, not 304 or 412. The caller therefore asks + * this first, evaluates preconditions, and only then applies -- so a 304 + * neither carries a Content-Encoding header nor initialises a compressor + * that no deflate() call uses. + */ + +nxt_int_t +nxt_http_comp_check_acceptable(nxt_task_t *task, nxt_http_request_t *r) +{ + bool identity_refused; + nxt_int_t ret, idx; + nxt_str_t accept_encoding, mime_type = {}; + nxt_http_comp_ctx_t *ctx; + nxt_http_comp_conf_t *conf = nxt_http_comp_request_conf(r); + nxt_http_set_header_op_t conf_op; + + /* + * No context means no compression. A context from an earlier call is + * dropped, not reused: its pool cleanup still frees its stream. + */ + r->comp_ctx = NULL; + if (r->resp.content_length == NULL && r->resp.content_length_n == -1) { return NXT_OK; } @@ -566,6 +1328,96 @@ nxt_http_comp_check_compression(nxt_task_t *task, nxt_http_request_t *r) return NXT_OK; } + /* + * A 1xx, 204 or 304 describes no representation. So there is nothing + * that the client could refuse, and no 406 to give: negotiation is + * skipped, whatever the request asked for. The length tests above do + * not cover this. An application can send Content-Length with such a + * status. A 304 carries the length of the body that the client already + * has. nxt_http_response_content_length() stores that field without + * setting content_length_n, which stays -1. + * + * r->status holds the response status at both callers. The application + * path copies it out of the response before it asks. The static path + * is at 200 here, before precondition evaluation, so a 406 still + * outranks the 304 or 412 that a validator would give. + */ + + if (nxt_http_status_no_representation(r->status)) { + return NXT_OK; + } + + if (nxt_http_comp_is_resp_content_encoded(r)) { + return NXT_OK; + } + + /* + * "response_headers" are applied after this, when the header is sent. + * A Content-Encoding there then replaces the one that a compressor adds, + * or removes it. Both make a coded body wrong: coded twice under one + * coding name, or coded under no name at all (#557). + * + * A value means that the operator says the body is already coded. This + * is the usual way to serve a precompressed file. It is the same case + * as a Content-Encoding in the response, so it returns here too. + * + * A null value removes the field. The body then has no coding, so it is + * identity, and a refusal of identity is still answered below. Only the + * compressor is kept out. This applies only to a coding that Unit would + * apply. A response that has its own Content-Encoding returned above, + * and a null value then only removes that field from the coded body. + * + * A template value is resolved now, once for the request. A variable + * that is not set gives an empty string, and an empty string is a value. + * A value that is not safe in a field is not sent, so it counts as + * absent. + */ + + conf_op = nxt_http_set_headers_field_op(r, "Content-Encoding", + nxt_length("Content-Encoding")); + + if (nxt_slow_path(conf_op == NXT_HTTP_SET_HEADER_ERROR)) { + return NXT_ERROR; + } + + if (conf_op == NXT_HTTP_SET_HEADER_REPLACE) { + return NXT_OK; + } + + ret = nxt_http_comp_accept_encoding(r, &accept_encoding); + if (nxt_slow_path(ret != NXT_OK)) { + return NXT_ERROR; + } + + /* + * Ask what the client accepts before anything below rules compression + * out. Identity is the one representation that the server always has. + * So "identity;q=0" is a refusal that must be answered even where no + * compressor can run. Before, this was asked afterwards, when the + * response was already declared serveable. That is how the own bytes + * of the file reached a client that had refused them (#390). + */ + + idx = nxt_http_comp_select_compressor(conf, r, &accept_encoding, + &identity_refused); + if (idx == -1) { + return nxt_http_comp_not_acceptable(r); + } + + if (conf_op == NXT_HTTP_SET_HEADER_REMOVE) { + return identity_refused ? nxt_http_comp_not_acceptable(r) : NXT_OK; + } + + /* + * A built configuration always holds identity. So NULL is the only "no + * compression" state, and identity is then the only coding. A client + * that refused it left through the 406 above. + */ + + if (conf == NULL) { + return NXT_OK; + } + if (r->resp.mime_type != NULL) { mime_type = *r->resp.mime_type; } else if (r->resp.content_type != NULL) { @@ -573,71 +1425,192 @@ nxt_http_comp_check_compression(nxt_task_t *task, nxt_http_request_t *r) mime_type.length = r->resp.content_type->value_length; } + /* + * A coding was selected, but the "types" rule below can still withdraw + * it, and the fallback is always identity. When the client refused + * identity, that fallback does not exist. So the answer is 406, not + * the bytes that the client refused. "min_length" cannot reach here: a + * coding below it is not selected. + */ + if (mime_type.start == NULL) { - return NXT_OK; + return identity_refused ? nxt_http_comp_not_acceptable(r) : NXT_OK; } - if (nxt_http_comp_mime_types_rule != NULL) { - ret = nxt_http_route_test_rule(r, nxt_http_comp_mime_types_rule, + if (conf->mime_types_rule != NULL) { + ret = nxt_http_route_test_rule(r, conf->mime_types_rule, mime_type.start, mime_type.length); - if (ret == 0) { - return NXT_OK; + + /* + * NXT_ERROR is not a match. A regex can reach its match limit on a + * long media type. The response then takes the no-match path. + */ + + if (ret == 0 || nxt_slow_path(ret == NXT_ERROR)) { + return identity_refused ? nxt_http_comp_not_acceptable(r) : NXT_OK; } } - rtcf = r->conf->socket_conf->router_conf; + /* + * Past every early return above, so this response really was subject to + * negotiation on Accept-Encoding, whichever coding is chosen below. + */ - if (nxt_http_comp_is_resp_content_encoded(r)) { - return NXT_OK; - } + r->resp.vary_accept_encoding = 1; - ret = nxt_tstr_query_init(&r->tstr_query, rtcf->tstr_state, &r->tstr_cache, - r, r->mem_pool); - if (nxt_slow_path(ret == NXT_ERROR)) { + if (nxt_slow_path(nxt_http_comp_merge_vary(r) != NXT_OK)) { return NXT_ERROR; } - ret = nxt_tstr_query(task, r->tstr_query, - nxt_http_comp_accept_encoding_query, &accept_encoding); - if (nxt_slow_path(ret != NXT_OK)) { + ctx = nxt_mp_get(r->mem_pool, sizeof(nxt_http_comp_ctx_t)); + if (nxt_slow_path(ctx == NULL)) { return NXT_ERROR; } - idx = nxt_http_comp_select_compressor(r, &accept_encoding); - if (idx == -1) { - return NXT_HTTP_NOT_ACCEPTABLE; + *ctx = (nxt_http_comp_ctx_t){ .resp_clen = -1, .sel_idx = idx, + .identity_refused = identity_refused }; + + r->comp_ctx = ctx; + + return NXT_OK; +} + + +/* + * Weakens the response's ETag, so that a coded representation does not carry + * the same strong validator as the identity one. + * + * RFC 9110 Sect. 8.8.3: a strong validator must change whenever the selected + * representation changes, and a content coding selects a different + * representation. Unit derives the tag from the file's mtime and size, which + * are the same whichever coding is served, so without this a client that took + * its tag from a gzip response and then sent it back in If-Range -- which + * Sect. 13.1.5 compares strongly -- would match, and be handed identity bytes + * for offsets it believes are gzip offsets. Weakening makes that strong + * comparison fail, so the range is simply not applied. + * + * If-None-Match keeps working: it compares weakly, so revalidation of a coded + * representation still answers 304. + * + * nginx does the same thing (ngx_http_weak_etag(), called from its gzip + * header filter); Apache appends "-gzip" and Go's net/http appends the coding + * name. All three make the tag differ per coding; weakening is the smallest + * of the three and needs no extra allocation beyond the prefix. + */ + +static nxt_int_t +nxt_http_comp_weaken_etag(nxt_http_request_t *r) +{ + u_char *p; + nxt_http_field_t *f; + nxt_http_fields_iter_t iter; + + for (f = nxt_http_fields_first(&iter, r->resp.inline_fields, + r->resp.num_inline_fields, r->resp.fields); + f != NULL; + f = nxt_http_fields_next(&iter)) + { + if (f->skip || f->name_length != nxt_length("ETag") + || nxt_strncasecmp(f->name, (u_char *) "ETag", + nxt_length("ETag")) != 0) + { + continue; + } + + if (f->value_length >= 2 + && f->value[0] == 'W' && f->value[1] == '/') + { + return NXT_OK; + } + + p = nxt_mp_nget(r->mem_pool, f->value_length + nxt_length("W/")); + if (nxt_slow_path(p == NULL)) { + return NXT_ERROR; + } + + /* Copy out of the old value before the field is repointed. */ + nxt_memcpy(p, "W/", nxt_length("W/")); + nxt_memcpy(p + nxt_length("W/"), f->value, f->value_length); + + f->value = p; + f->value_length += nxt_length("W/"); + + return NXT_OK; } - if (idx == NXT_HTTP_COMP_SCHEME_IDENTITY) { + return NXT_OK; +} + + +/* + * Applies the decision nxt_http_comp_check_acceptable() reached: adds the + * Content-Encoding header and initialises the compressor. Call it only on a + * path that will actually send a body. + */ + +nxt_int_t +nxt_http_comp_apply_compression(nxt_task_t *task, nxt_http_request_t *r) +{ + int err; + nxt_int_t idx; + nxt_http_comp_ctx_t *ctx = r->comp_ctx; + nxt_http_comp_conf_t *conf; + nxt_http_comp_compressor_t *compressor; + + if (ctx == NULL) { return NXT_OK; } - compressor = &nxt_http_comp_enabled_compressors[idx]; + idx = ctx->sel_idx; - if (r->resp.content_length_n > -1) { - ctx->resp_clen = r->resp.content_length_n; - } else if (r->resp.content_length != NULL) { - ctx->resp_clen = - strtol((char *)r->resp.content_length->value, NULL, 10); + if (idx == -1 || idx == NXT_HTTP_COMP_SCHEME_IDENTITY) { + return NXT_OK; } - min_len = compressor->opts.min_len; + /* + * Reached only when nxt_http_comp_check_acceptable() chose a compressor, + * which it does only from a non-NULL configuration. + */ + conf = nxt_http_comp_request_conf(r); + compressor = &conf->enabled[idx]; - if (ctx->resp_clen > -1 && ctx->resp_clen < min_len) { + ctx->resp_clen = nxt_http_comp_resp_length(r); + + if (nxt_http_comp_below_min_len(ctx->resp_clen, compressor)) { return NXT_OK; } - nxt_http_comp_set_header(r, idx); + nxt_http_comp_set_header(conf, r, idx); + + if (nxt_slow_path(nxt_http_comp_weaken_etag(r) != NXT_OK)) { + return NXT_ERROR; + } ctx->idx = idx; - ctx->ctx.level = nxt_http_comp_enabled_compressors[idx].opts.level; + ctx->type = compressor->type; + ctx->ctx.level = compressor->opts.level; + + /* + * Registered before the stream exists, so that no stream is left without + * a cleanup. The cleanup frees the stream when the response stops before + * the last deflate() call. &r->task, not "task": the cleanup keeps the + * pointer, and r->task lives as long as the pool. + */ + if (nxt_slow_path(nxt_mp_cleanup(r->mem_pool, nxt_http_comp_ctx_cleanup, + &r->task, ctx, NULL) + != NXT_OK)) + { + return NXT_ERROR; + } err = compressor->type->cops->init(&ctx->ctx); if (nxt_slow_path(err)) { return NXT_ERROR; } + ctx->live = true; + return NXT_OK; } @@ -671,6 +1644,7 @@ nxt_http_comp_compressor_is_valid(const nxt_str_t *token) static nxt_int_t nxt_http_comp_set_compressor(nxt_task_t *task, nxt_router_conf_t *rtcf, + nxt_http_comp_conf_t *conf, const nxt_conf_value_t *comp, nxt_uint_t index) { nxt_int_t ret; @@ -689,7 +1663,7 @@ nxt_http_comp_set_compressor(nxt_task_t *task, nxt_router_conf_t *rtcf, nxt_conf_get_string(obj, &token); cidx = nxt_http_comp_compressor_token2idx(&token); - compr = &nxt_http_comp_enabled_compressors[index]; + compr = &conf->enabled[index]; compr->type = &nxt_http_comp_compressors[cidx]; compr->opts.level = compr->type->def_compr; @@ -721,32 +1695,29 @@ nxt_int_t nxt_http_comp_compression_init(nxt_task_t *task, nxt_router_conf_t *rtcf, const nxt_conf_value_t *comp_conf) { - nxt_int_t ret; - nxt_uint_t n = 1; /* 'identity' */ - nxt_conf_value_t *comps, *mimes; + nxt_int_t ret; + nxt_uint_t n = 1; /* 'identity' */ + nxt_conf_value_t *comps, *mimes; + nxt_http_comp_conf_t *conf; - static const nxt_str_t accept_enc_str = - nxt_string("$header_accept_encoding"); static const nxt_str_t comps_str = nxt_string("compressors"); static const nxt_str_t mimes_str = nxt_string("types"); + conf = nxt_mp_zalloc(rtcf->mem_pool, sizeof(nxt_http_comp_conf_t)); + if (nxt_slow_path(conf == NULL)) { + return NXT_ERROR; + } + mimes = nxt_conf_get_object_member(comp_conf, &mimes_str, NULL); if (mimes != NULL) { - nxt_http_comp_mime_types_rule = + conf->mime_types_rule = nxt_http_route_types_rule_create(task, rtcf->mem_pool, mimes); - if (nxt_slow_path(nxt_http_comp_mime_types_rule == NULL)) { + if (nxt_slow_path(conf->mime_types_rule == NULL)) { return NXT_ERROR; } } - nxt_http_comp_accept_encoding_query = - nxt_tstr_compile(rtcf->tstr_state, &accept_enc_str, - NXT_TSTR_STRZ); - if (nxt_slow_path(nxt_http_comp_accept_encoding_query == NULL)) { - return NXT_ERROR; - } - comps = nxt_conf_get_object_member(comp_conf, &comps_str, NULL); if (nxt_slow_path(comps == NULL)) { return NXT_ERROR; @@ -757,30 +1728,40 @@ nxt_http_comp_compression_init(nxt_task_t *task, nxt_router_conf_t *rtcf, } else { n += nxt_conf_object_members_count(comps); } - nxt_http_comp_nr_enabled_compressors = n; + conf->nr_enabled = n; - nxt_http_comp_enabled_compressors = - nxt_mp_zalloc(rtcf->mem_pool, - sizeof(nxt_http_comp_compressor_t) * n); + conf->enabled = nxt_mp_zalloc(rtcf->mem_pool, + sizeof(nxt_http_comp_compressor_t) * n); + if (nxt_slow_path(conf->enabled == NULL)) { + return NXT_ERROR; + } - nxt_http_comp_enabled_compressors[0] = + conf->enabled[0] = (nxt_http_comp_compressor_t){ .type = &nxt_http_comp_compressors[0], .opts.level = NXT_COMP_LEVEL_UNSET, .opts.min_len = -1 }; if (nxt_conf_type(comps) == NXT_CONF_OBJECT) { - return nxt_http_comp_set_compressor(task, rtcf, comps, 1); - } + ret = nxt_http_comp_set_compressor(task, rtcf, conf, comps, 1); + if (nxt_slow_path(ret == NXT_ERROR)) { + return NXT_ERROR; + } - for (nxt_uint_t i = 1; i < nxt_http_comp_nr_enabled_compressors; i++) { - nxt_conf_value_t *obj; + } else { + for (nxt_uint_t i = 1; i < conf->nr_enabled; i++) { + nxt_conf_value_t *obj; - obj = nxt_conf_get_array_element(comps, i - 1); - ret = nxt_http_comp_set_compressor(task, rtcf, obj, i); - if (ret == NXT_ERROR) { - return NXT_ERROR; + obj = nxt_conf_get_array_element(comps, i - 1); + ret = nxt_http_comp_set_compressor(task, rtcf, conf, obj, i); + if (ret == NXT_ERROR) { + return NXT_ERROR; + } } } + /* One publish point: a failure above leaves rtcf->compression NULL. */ + + rtcf->compression = conf; + return NXT_OK; } diff --git a/src/nxt_http_compression.h b/src/nxt_http_compression.h index 99af8a669..5cf3a087d 100644 --- a/src/nxt_http_compression.h +++ b/src/nxt_http_compression.h @@ -73,6 +73,7 @@ struct nxt_http_comp_operations_s { ssize_t (*deflate)(nxt_http_comp_compressor_ctx_t *ctx, const uint8_t *in_buf, size_t in_len, uint8_t *out_buf, size_t out_len, bool last); + void (*free)(nxt_http_comp_compressor_ctx_t *ctx); }; @@ -95,9 +96,12 @@ extern nxt_int_t nxt_http_comp_compress_app_response(nxt_task_t *task, extern nxt_int_t nxt_http_comp_compress_static_response(nxt_task_t *task, nxt_http_request_t *r, nxt_file_t **f, nxt_file_info_t *fi, size_t static_buf_len, size_t *out_total); -extern bool nxt_http_comp_wants_compression(void); +extern bool nxt_http_comp_wants_compression(nxt_http_request_t *r); +extern bool nxt_http_comp_identity_refused(nxt_http_request_t *r); extern bool nxt_http_comp_compressor_is_valid(const nxt_str_t *token); -extern nxt_int_t nxt_http_comp_check_compression(nxt_task_t *task, +extern nxt_int_t nxt_http_comp_check_acceptable(nxt_task_t *task, + nxt_http_request_t *r); +extern nxt_int_t nxt_http_comp_apply_compression(nxt_task_t *task, nxt_http_request_t *r); extern nxt_int_t nxt_http_comp_compression_init(nxt_task_t *task, nxt_router_conf_t *rtcf, const nxt_conf_value_t *comp_conf); diff --git a/src/nxt_http_parse.c b/src/nxt_http_parse.c index e8f71334c..61f1367c3 100644 --- a/src/nxt_http_parse.c +++ b/src/nxt_http_parse.c @@ -358,17 +358,17 @@ nxt_http_parse_request_line(nxt_http_request_parse_t *rp, u_char **pos, if (p[7] < '0' || p[7] > '9') { break; } - /* Fall through. */ + nxt_fallthrough; case 7: if (p[6] != '.') { break; } - /* Fall through. */ + nxt_fallthrough; case 6: if (p[5] < '0' || p[5] > '9') { break; } - /* Fall through. */ + nxt_fallthrough; default: return NXT_AGAIN; } @@ -743,13 +743,13 @@ nxt_http_lookup_field_end(u_char *p, const u_char *end) switch (end - p) { case 3: nxt_field_end_test_char(*p); p++; - /* Fall through. */ + nxt_fallthrough; case 2: nxt_field_end_test_char(*p); p++; - /* Fall through. */ + nxt_fallthrough; case 1: nxt_field_end_test_char(*p); p++; - /* Fall through. */ + nxt_fallthrough; case 0: break; default: @@ -785,7 +785,7 @@ nxt_http_parse_field_end(nxt_http_request_parse_t *rp, u_char **pos, if (rp->skip_field) { rp->skip_field = 0; - } else { + } else if (!rp->discard_fields) { if (rp->num_inline_fields < 16) { field = &rp->inline_fields[rp->num_inline_fields++]; diff --git a/src/nxt_http_parse.h b/src/nxt_http_parse.h index c0d5c6226..150929684 100644 --- a/src/nxt_http_parse.h +++ b/src/nxt_http_parse.h @@ -72,6 +72,8 @@ struct nxt_http_request_parse_s { uint8_t skip_field; /* 1 bit */ uint8_t discard_unsafe_fields; /* 1 bit */ + /* check the fields, but do not store them */ + uint8_t discard_fields; /* 1 bit */ /* target with "/." */ uint8_t complex_target; /* 1 bit */ diff --git a/src/nxt_http_proxy.c b/src/nxt_http_proxy.c index f7874ec47..d3b36a11e 100644 --- a/src/nxt_http_proxy.c +++ b/src/nxt_http_proxy.c @@ -374,6 +374,84 @@ nxt_http_proxy_buf_mem_free(nxt_task_t *task, nxt_http_request_t *r, } +static void +nxt_http_proxy_buf_mem_cleanup(nxt_task_t *task, void *obj, void *data) +{ + nxt_buf_t *b; + nxt_event_engine_t *engine; + + b = obj; + engine = data; + + /* + * The engine is carried in "data": nxt_mp_destroy() has none to hand. + * "task" is passed only so the pointer nxt_mp_cleanup() stores stays + * inside the pool being destroyed -- do not dereference it here. + */ + + nxt_event_engine_buf_mem_free(engine, b); +} + + +/* + * Keep an upstream read buffer alive for the rest of the request without + * holding the request pool hostage. + * + * A buffer that is relayed downstream needs none of this: its completion + * handler frees it and drops the retain. One that is *not* relayed does -- + * an upstream header block with no body bytes behind it, and the header of a + * response that RFC 9112 Sect. 6.3 gives no body at all. Two things then have + * to hold at once: + * + * - It must outlive the response. peer->fields point their name/value into + * this buffer (nxt_http_parse_fields) and nxt_http_proxy_header_read() + * shallow-copies those field structs into r->resp, where + * $response_header_* and "match" conditions read them until the request is + * logged and closed. Returning it to the engine cache when the response + * completes would leave them pointing at reusable memory. + * + * - It must not keep the retain nxt_http_proxy_buf_mem_alloc() took. That + * retain is dropped by a buffer completion, and a buffer that is never + * relayed has no completion to run -- the pool would never reach zero and + * the entire request pool would be stranded. + * + * A pool cleanup satisfies both: nxt_mp_destroy() runs it before freeing the + * pool's own blocks, so the field bytes stay valid for exactly as long as + * anything can read them, and not one request longer. + */ + +nxt_int_t +nxt_http_proxy_buf_mem_hold(nxt_task_t *task, nxt_http_request_t *r, + nxt_buf_t *b) +{ + nxt_int_t ret; + + /* + * &r->task, not "task": nxt_mp_cleanup() stores the task pointer in the + * work item and hands it back at destroy time, and the peer connection's + * task is freed with that connection well before the request pool. + * &r->task lives in the pool being destroyed, which nxt_mp_destroy() runs + * its cleanups before freeing. + */ + + ret = nxt_mp_cleanup(r->mem_pool, nxt_http_proxy_buf_mem_cleanup, &r->task, + b, task->thread->engine); + if (nxt_slow_path(ret != NXT_OK)) { + return NXT_ERROR; + } + + /* + * The request itself holds a retain until nxt_http_request_close_handler(), + * so this cannot destroy the pool here -- which it must not, the cleanup + * just registered lives in it. + */ + + nxt_mp_release(r->mem_pool); + + return NXT_OK; +} + + static void nxt_http_proxy_error(nxt_task_t *task, void *obj, void *data) { diff --git a/src/nxt_http_request.c b/src/nxt_http_request.c index c15abd99d..ea6af760c 100644 --- a/src/nxt_http_request.c +++ b/src/nxt_http_request.c @@ -9,7 +9,6 @@ #include -static nxt_int_t nxt_http_validate_host(nxt_str_t *host, nxt_mp_t *mp); static void nxt_http_request_start(nxt_task_t *task, void *obj, void *data); static nxt_int_t nxt_http_request_forward(nxt_task_t *task, nxt_http_request_t *r, nxt_http_forward_t *forward); @@ -22,6 +21,10 @@ static void nxt_http_request_forward_protocol(nxt_http_request_t *r, static void nxt_http_request_ready(nxt_task_t *task, void *obj, void *data); static void nxt_http_request_proto_info(nxt_task_t *task, nxt_http_request_t *r); +static nxt_bool_t nxt_http_request_is_bodyless(nxt_http_request_t *r); +static void nxt_http_request_drop_framing_fields(nxt_http_request_t *r); +static nxt_buf_t *nxt_http_request_body_drop(nxt_task_t *task, + nxt_http_request_t *r, nxt_buf_t *out); static void nxt_http_request_mem_buf_completion(nxt_task_t *task, void *obj, void *data); static void nxt_http_request_done(nxt_task_t *task, void *obj, void *data); @@ -104,7 +107,7 @@ nxt_http_request_host(void *ctx, nxt_http_field_t *field, uintptr_t data) } -static nxt_int_t +nxt_int_t nxt_http_validate_host(nxt_str_t *host, nxt_mp_t *mp) { u_char *h, ch; @@ -678,6 +681,83 @@ nxt_http_request_read_body(nxt_task_t *task, nxt_http_request_t *r) } +/* + * Allocates r->body for a request body of body_length bytes, or of a + * length not known in advance when body_length is (size_t) -1. A body + * that fits in body_buffer_size is kept in memory; a longer one goes to + * an unlinked temporary file under body_temp_path, with a memory part of + * body_buffer_size for the reads. The protocol layer that fills the + * buffer enforces max_body_size on what it stores. + */ + +nxt_int_t +nxt_http_request_body_alloc(nxt_task_t *task, nxt_http_request_t *r, + size_t body_length) +{ + size_t body_buffer_size; + nxt_buf_t *b; + nxt_socket_conf_t *skcf; + + static const nxt_str_t tmp_name_pattern = nxt_string("/req-XXXXXXXX"); + + skcf = r->conf->socket_conf; + + body_buffer_size = nxt_min(skcf->body_buffer_size, body_length); + + if (body_length > body_buffer_size) { + nxt_str_t *tmp_path, tmp_name; + + tmp_path = &skcf->body_temp_path; + + tmp_name.length = tmp_path->length + tmp_name_pattern.length; + + b = nxt_buf_file_alloc(r->mem_pool, + body_buffer_size + sizeof(nxt_file_t) + + tmp_name.length + 1, 0); + if (nxt_slow_path(b == NULL)) { + return NXT_ERROR; + } + + tmp_name.start = nxt_pointer_to(b->mem.start, sizeof(nxt_file_t)); + + memcpy(tmp_name.start, tmp_path->start, tmp_path->length); + memcpy(tmp_name.start + tmp_path->length, tmp_name_pattern.start, + tmp_name_pattern.length); + tmp_name.start[tmp_name.length] = '\0'; + + b->file = (nxt_file_t *) b->mem.start; + nxt_memzero(b->file, sizeof(nxt_file_t)); + b->file->fd = -1; + b->file->size = body_length; + + b->mem.start += sizeof(nxt_file_t) + tmp_name.length + 1; + b->mem.pos = b->mem.start; + b->mem.free = b->mem.start; + + b->file->fd = mkstemp((char *) tmp_name.start); + if (nxt_slow_path(b->file->fd == -1)) { + nxt_alert(task, "mkstemp(%s) failed %E", tmp_name.start, nxt_errno); + return NXT_ERROR; + } + + nxt_debug(task, "create body tmp file \"%V\", %d", + &tmp_name, b->file->fd); + + unlink((char *) tmp_name.start); + + } else { + b = nxt_buf_mem_alloc(r->mem_pool, body_buffer_size, 0); + if (nxt_slow_path(b == NULL)) { + return NXT_ERROR; + } + } + + r->body = b; + + return NXT_OK; +} + + void nxt_http_request_header_send(nxt_task_t *task, nxt_http_request_t *r, nxt_work_handler_t body_handler, void *data) @@ -692,6 +772,63 @@ nxt_http_request_header_send(nxt_task_t *task, nxt_http_request_t *r, goto fail; } + /* + * "response_headers" has just run, and it replaces or removes a field it + * names outright. A response chosen by negotiation still varies on + * Accept-Encoding whatever an operator wrote there -- almost always they + * are adding Origin for CORS, unaware Unit generates the field at all -- + * so dropping it here would hand a shared cache the licence to serve one + * coding to every client. Re-assert it. The merge is idempotent: it + * leaves "*" alone, leaves a list that already names the header alone, + * and re-adds the field if it was removed. + */ + + if (r->resp.vary_accept_encoding) { + ret = nxt_http_comp_merge_vary(r); + if (nxt_slow_path(ret != NXT_OK)) { + goto fail; + } + } + + /* + * RFC 9112 Sect. 6.3: a 1xx, 204 or 304 response, and any response to a + * HEAD request, never has a message body, no matter what the response + * headers say. Record that here, in the layer every response source + * (application, proxy, static, "return") passes through, so that the + * per-protocol header_send() can drop the framing and + * nxt_http_request_send() can drop the body buffers. Without this an + * application that writes a body on 204/304 has those bytes forwarded + * verbatim and unframed, which a downstream parser reads as the start of + * the next response. + */ + r->no_body = nxt_http_request_is_bodyless(r); + + /* + * RFC 9110 Sect. 8.6: a server must not send Content-Length in a 1xx or + * 204 response. A 304 keeps it -- there it describes the body the client + * already has -- and so does a HEAD response, where it describes the body + * the equivalent GET would return. + */ + if (r->no_body + && (r->status < NXT_HTTP_OK || r->status == NXT_HTTP_NO_CONTENT)) + { + if (r->resp.content_length != NULL) { + r->resp.content_length->skip = 1; + } + + r->resp.content_length_n = -1; + + /* + * r->resp.content_length only tracks the field the application sent. + * A Content-Length added by "response_headers" (nxt_http_set_headers.c) + * is a generic field, and an application-supplied Transfer-Encoding is + * one too; both are forbidden here and both would otherwise be + * serialized verbatim, desyncing the next response on a connection + * whose keep-alive this change preserves. Sweep the whole field store. + */ + nxt_http_request_drop_framing_fields(r); + } + /* * TODO: "Server", "Date", and "Content-Length" processing should be moved * to the last header filter. @@ -774,9 +911,190 @@ nxt_http_request_ws_frame_start(nxt_task_t *task, nxt_http_request_t *r, } +/* + * A *final* response that RFC 9112 Sect. 6.3 gives no message body: 204, 304, + * or any response to HEAD. 1xx is excluded on purpose. A 1xx is an interim + * response -- the exchange continues with a final response after it -- so + * "carries no body" and "ends the exchange" are different questions for it, + * and a caller that needs the second one must not be answered with the first. + * + * The status is a parameter rather than a read of r->status because the proxy + * has to answer this question before r->status exists. The h1 peer reader + * decides how to frame the upstream body while the upstream status still lives + * in peer->status; nxt_http_proxy_header_read() copies it into r->status only + * one step later. + */ + +nxt_bool_t +nxt_http_request_is_bodyless_final(nxt_http_request_t *r, + nxt_http_status_t status) +{ + if (status < NXT_HTTP_OK) { + return 0; + } + + /* 204 and 304; the 1xx arm of that test is excluded above. */ + if (nxt_http_status_no_representation(status)) { + return 1; + } + + return r->method != NULL && nxt_str_eq(r->method, "HEAD", 4); +} + + +/* + * Tells whether a status describes no representation of its own: a 1xx + * interim response, a 204, and a 304. The Content-Length of a 304 describes + * the body that the client already has. Content negotiation has nothing to + * select for such a response, so the response cannot be 406. + * + * The method is not read on purpose. That separates this from + * nxt_http_request_is_bodyless_final() above. A HEAD response carries no + * body, but it still describes the representation that the equivalent GET + * would return (RFC 9110 Sect. 9.3.2). So it is negotiated like that GET. + */ + +nxt_bool_t +nxt_http_status_no_representation(nxt_http_status_t status) +{ + return status < NXT_HTTP_OK + || status == NXT_HTTP_NO_CONTENT + || status == NXT_HTTP_NOT_MODIFIED; +} + + +static nxt_bool_t +nxt_http_request_is_bodyless(nxt_http_request_t *r) +{ + /* + * A 101 upgrade is a 1xx status, but the bytes that follow its header are + * not a message body -- they are the upgraded protocol (WebSocket), and + * nxt_h1proto_websocket.c pushes them through nxt_http_request_send(). + * + * The status matters as much as the flag. websocket_handshake is set when + * the request headers are parsed (src/nxt_h1proto.c), long before the + * application chooses a status, and the h1 sender only treats the response + * as an upgrade when it is also 101 (src/nxt_h1proto.c). Testing the flag + * alone would leave an application that answers a WebSocket-upgrade + * request with 204 plus a body on the unframed path. + */ + if (r->websocket_handshake && r->status == NXT_HTTP_SWITCHING_PROTOCOLS) { + return 0; + } + + if (r->status >= NXT_HTTP_CONTINUE && r->status < NXT_HTTP_OK) { + return 1; + } + + return nxt_http_request_is_bodyless_final(r, r->status); +} + + +/* + * Mark every Content-Length and Transfer-Encoding response field skipped. + * Only for 1xx and 204, where RFC 9110 Sect. 8.6 and RFC 9112 Sect. 6.1 forbid + * both: a 304 keeps Content-Length, and so does a response to HEAD. + */ + +static void +nxt_http_request_drop_framing_fields(nxt_http_request_t *r) +{ + nxt_http_field_t *field; + + nxt_http_fields_each(field, r->resp.inline_fields, r->resp.num_inline_fields, + r->resp.fields) + { + if (field->skip) { + continue; + } + + if ((field->name_length == nxt_length("Content-Length") + && nxt_strncasecmp(field->name, (u_char *) "Content-Length", + nxt_length("Content-Length")) == 0) + || (field->name_length == nxt_length("Transfer-Encoding") + && nxt_strncasecmp(field->name, (u_char *) "Transfer-Encoding", + nxt_length("Transfer-Encoding")) == 0)) + { + field->skip = 1; + } + + } nxt_http_fields_loop; +} + + +/* + * Strip the payload from a response that must not carry one, keeping the + * sync/last markers that drive request completion. Data-only buffers are + * unlinked and drained so their completion handlers run and the shared memory + * they hold is released; a buffer that also carries the "last" marker stays in + * the chain but is emptied in place, so the request still finishes normally. + */ + +static nxt_buf_t * +nxt_http_request_body_drop(nxt_task_t *task, nxt_http_request_t *r, + nxt_buf_t *out) +{ + size_t dropped; + nxt_buf_t *b, *next, **prev; + nxt_work_queue_t *wq; + + dropped = 0; + prev = &out; + wq = &task->thread->engine->fast_work_queue; + + for (b = out; b != NULL; b = next) { + next = b->next; + + if (nxt_buf_is_sync(b)) { + /* + * A sync buffer may be allocated with NXT_BUF_SYNC_SIZE, so it + * carries no payload and its mem.free/file fields must not even + * be read. Keep it as is. + */ + prev = &b->next; + continue; + } + + dropped += nxt_buf_used_size(b); + + if (nxt_buf_is_last(b)) { + /* Empty it in place; the "last" marker still ends the request. */ + b->mem.pos = b->mem.free; + + if (nxt_buf_is_file(b)) { + b->file_pos = b->file_end; + } + + prev = &b->next; + continue; + } + + *prev = next; + b->next = NULL; + + nxt_sendbuf_drain(task, wq, b); + } + + if (dropped != 0) { + nxt_debug(task, "http request body dropped on status %d: %uz bytes", + (int) r->status, dropped); + } + + return out; +} + + void nxt_http_request_send(nxt_task_t *task, nxt_http_request_t *r, nxt_buf_t *out) { + if (r->no_body) { + out = nxt_http_request_body_drop(task, r, out); + + if (out == NULL) { + return; + } + } + if (nxt_fast_path(r->proto.any != NULL)) { nxt_http_proto[r->protocol].send(task, r, out); } diff --git a/src/nxt_http_route.c b/src/nxt_http_route.c index 5b937dd7d..9c1dd5a32 100644 --- a/src/nxt_http_route.c +++ b/src/nxt_http_route.c @@ -1184,13 +1184,31 @@ nxt_http_route_pattern_create(nxt_task_t *task, nxt_mp_t *mp, /* - * Configured pattern strings are decoded once here at compile time, - * while request URIs are decoded by the HTTP parser before they reach - * the matcher. Both paths funnel through the same nxt_decode_uri / - * nxt_decode_uri_plus helpers; %XX semantics are symmetric, so a - * "%2e%2e" in the configured pattern will compare against the same - * bytes a request "%2e%2e" decoded into. If either helper's behaviour - * changes, route patterns and request URIs MUST be updated in lock-step. + * Configured pattern strings are decoded once here, at configuration time, + * by nxt_decode_uri() ("uri") or nxt_decode_uri_plus() ("arguments" and + * "query"). A regex pattern ("~...") is compiled as written and is not + * decoded. + * + * The request side does not use these helpers, and the two sides are not + * symmetric. + * + * The request path is decoded by nxt_http_parse_complex_target() in + * src/nxt_http_parse.c: + * - most decoded bytes go back through its state machine, so an encoded + * dot segment such as "/%2e%2e/" is normalized like "/../"; + * - a decoded '%', '#' or '?' is copied as a plain byte; + * - "%00" is rejected; + * - "/./" and "/../" segments and a trailing "/." or "/.." are removed, + * and repeated slashes are merged; + * - "encoded_slashes" keeps "%25" and "%2F" encoded, but only the + * controller sets it. The router never does. + * So a pattern that spells "%2e%2e" or "//" as a whole path segment can + * never match a normalized path. A ".." inside a segment ("/a..b") is + * kept on both sides and compares as plain bytes. + * + * Request arguments and the query are decoded by nxt_http_arguments_parse() + * in src/nxt_http_request.c: '+' becomes a space, a bad or short "%XX" is + * copied as is, and "%00" is accepted. */ static nxt_int_t nxt_http_route_decode_str(nxt_str_t *str, nxt_http_uri_encoding_t encoding) @@ -2164,12 +2182,14 @@ nxt_http_route_pattern(nxt_http_request_t *r, nxt_http_route_pattern_t *pattern, if (pattern->regex) { if (r->regex_match == NULL) { /* - * Reuse one match-data struct across every pattern compiled - * against this request, so size it for the minimum ovector - * (one offset pair — the overall match). Captures are not - * consulted by the matcher. Passing 0 to PCRE2's - * pcre2_match_data_create() is undefined per the public - * docs; 1 is the documented minimum. + * One match object serves every pattern tested against this + * request. Its match data holds the minimum ovector: one + * offset pair, the overall match. The matcher does not read + * captures. pcre2_match_data_create() treats an ovecsize of 0 + * as 1 (pcre2_match_data_create(3)); 1 is passed to make the + * minimum explicit. With PCRE1 the size counts ints, not + * pairs; pcre_exec() then returns 0 on a match, and + * nxt_regex_match() counts 0 as a match. */ r->regex_match = nxt_regex_match_create(r->mem_pool, 1); if (nxt_slow_path(r->regex_match == NULL)) { @@ -2253,6 +2273,11 @@ nxt_http_route_memcmp(u_char *start, u_char *test, size_t test_length, { nxt_int_t n; + /* A "" pattern slice has a NULL start: no memcmp(NULL, p, 0). */ + if (test_length == 0) { + return 1; + } + if (case_sensitive) { n = memcmp(start, test, test_length); diff --git a/src/nxt_http_set_headers.c b/src/nxt_http_set_headers.c index 049abe62d..9caef0d1f 100644 --- a/src/nxt_http_set_headers.c +++ b/src/nxt_http_set_headers.c @@ -14,6 +14,167 @@ typedef struct { } nxt_http_header_val_t; +/* The keys of r->action resolved for this request, one entry per key. */ + +struct nxt_http_set_headers_ctx_s { + nxt_http_action_t *action; + nxt_str_t *value; + uint8_t *state; +}; + + +typedef enum { + NXT_HTTP_SET_HEADERS_UNRESOLVED = 0, + NXT_HTTP_SET_HEADERS_RESOLVED, + NXT_HTTP_SET_HEADERS_REJECTED, +} nxt_http_set_headers_state_t; + + +static nxt_http_set_headers_ctx_t *nxt_http_set_headers_ctx( + nxt_http_request_t *r); +static nxt_int_t nxt_http_set_headers_value(nxt_http_request_t *r, + nxt_http_set_headers_ctx_t *ctx, nxt_uint_t i); + + +/* + * Whether the matched action replaces or removes one of the validators the + * static handler generates. + * + * A conditional request has to be judged against the validator the client was + * actually given. When "response_headers" sets ETag or Last-Modified, the + * value Unit derives from the file is not what went out, so comparing against + * it answers the wrong question -- it refuses an If-Match carrying the tag the + * server itself advertised. The static handler asks this and declines to + * evaluate preconditions at all in that case, which loses the 304 but is never + * wrong. + * + * Only the name matters here, so no template value is resolved. + */ + +nxt_bool_t +nxt_http_set_headers_override_validators(nxt_http_request_t *r) +{ + nxt_uint_t i, n; + nxt_http_action_t *action; + nxt_http_header_val_t *header; + + action = r->action; + + if (action == NULL || action->set_headers == NULL) { + return 0; + } + + header = action->set_headers->elts; + n = action->set_headers->nelts; + + for (i = 0; i < n; i++) { + if (header[i].name.length == nxt_length("ETag") + && nxt_strncasecmp(header[i].name.start, (u_char *) "ETag", + nxt_length("ETag")) == 0) + { + return 1; + } + + if (header[i].name.length == nxt_length("Last-Modified") + && nxt_strncasecmp(header[i].name.start, + (u_char *) "Last-Modified", + nxt_length("Last-Modified")) == 0) + { + return 1; + } + } + + return 0; +} + + +/* + * What nxt_http_set_headers() will later do to the response field "name". + * + * Code that runs before the header is sent can ask this. Compression needs + * it: a Content-Encoding from "response_headers" replaces the one that a + * compressor adds. So the compressor must not run. Otherwise the body is + * coded twice and the field names one coding only. + * + * The value is resolved here with nxt_http_set_headers_value(). It stores + * the result on the request, and nxt_http_set_headers() uses that result. + * So each key is resolved once, and both see the same value, also for a + * JavaScript template. A template value can resolve to bytes that are not + * safe in a field. nxt_http_set_headers() then skips that key, so it + * counts as absent here too. A null value removes the field. An empty + * string is a value: the field is sent empty. A variable that is not set + * gives an empty string. A failed query gives NXT_HTTP_SET_HEADER_ERROR. + * + * Resolving a template this early is safe. The query is synchronous. + * nxt_tstr_query_init() reuses r->tstr_query. A cacheable variable keeps + * its value in r->tstr_cache for the rest of the request. + * + * The status test is the same as in nxt_http_set_headers(). Keys are + * applied in order, so the last key that matches and is not skipped gives + * the result. The search goes back from the end and stops at that key. + */ + +nxt_http_set_header_op_t +nxt_http_set_headers_field_op(nxt_http_request_t *r, const char *name, + size_t length) +{ + nxt_int_t ret; + nxt_uint_t i; + nxt_http_action_t *action; + nxt_http_header_val_t *header; + nxt_http_set_headers_ctx_t *ctx; + + action = r->action; + + if (action == NULL || action->set_headers == NULL) { + return NXT_HTTP_SET_HEADER_NONE; + } + + if (r->status < NXT_HTTP_OK || r->status >= NXT_HTTP_BAD_REQUEST) { + return NXT_HTTP_SET_HEADER_NONE; + } + + /* The context is made only when a key matches. */ + + ctx = NULL; + + header = action->set_headers->elts; + i = action->set_headers->nelts; + + while (i > 0) { + i--; + + if (header[i].name.length != length + || nxt_memcasecmp(header[i].name.start, name, length) != 0) + { + continue; + } + + if (ctx == NULL) { + ctx = nxt_http_set_headers_ctx(r); + if (nxt_slow_path(ctx == NULL)) { + return NXT_HTTP_SET_HEADER_ERROR; + } + } + + ret = nxt_http_set_headers_value(r, ctx, i); + + if (ret == NXT_DECLINED) { + continue; + } + + if (nxt_slow_path(ret != NXT_OK)) { + return NXT_HTTP_SET_HEADER_ERROR; + } + + return (ctx->value[i].start != NULL) ? NXT_HTTP_SET_HEADER_REPLACE + : NXT_HTTP_SET_HEADER_REMOVE; + } + + return NXT_HTTP_SET_HEADER_NONE; +} + + nxt_int_t nxt_http_set_headers_init(nxt_router_conf_t *rtcf, nxt_http_action_t *action, nxt_http_action_conf_t *acf) @@ -99,6 +260,126 @@ nxt_http_header_value_is_safe(const nxt_str_t *v) } +/* + * The resolved keys of r->action for this request. The context is made on + * first use and kept in r->set_headers. A request is allocated zeroed for + * each request, also on a keep-alive connection, so the pointer never comes + * from an earlier request. The context is made again if r->action is not + * the action that it was made for. + */ + +static nxt_http_set_headers_ctx_t * +nxt_http_set_headers_ctx(nxt_http_request_t *r) +{ + nxt_uint_t n; + nxt_http_set_headers_ctx_t *ctx; + + ctx = r->set_headers; + + /* + * Today the action does not change between the compression check and + * the header: the static fallback runs only before the check. This + * test is a guard for a later path that changes r->action. + */ + + if (ctx != NULL && ctx->action == r->action) { + return ctx; + } + + n = r->action->set_headers->nelts; + + ctx = nxt_mp_zget(r->mem_pool, sizeof(nxt_http_set_headers_ctx_t)); + if (nxt_slow_path(ctx == NULL)) { + return NULL; + } + + ctx->value = nxt_mp_zalloc(r->mem_pool, sizeof(nxt_str_t) * n); + if (nxt_slow_path(ctx->value == NULL)) { + return NULL; + } + + ctx->state = nxt_mp_zalloc(r->mem_pool, n); + if (nxt_slow_path(ctx->state == NULL)) { + return NULL; + } + + ctx->action = r->action; + r->set_headers = ctx; + + return ctx; +} + + +/* + * Resolve key "i" of "response_headers" once for this request, and store + * the result in ctx. A later call returns the stored result. So a + * template, a JavaScript one too, is run only once. + * + * A null value gives a null string, which removes the field. A value that + * is not safe in a field gives NXT_DECLINED, and the key is skipped. + * nxt_http_set_headers() and nxt_http_set_headers_field_op() both use this, + * so they agree on what each key does. + */ + +static nxt_int_t +nxt_http_set_headers_value(nxt_http_request_t *r, + nxt_http_set_headers_ctx_t *ctx, nxt_uint_t i) +{ + nxt_int_t ret; + nxt_str_t *value; + nxt_router_conf_t *rtcf; + nxt_http_header_val_t *hv; + + switch (ctx->state[i]) { + + case NXT_HTTP_SET_HEADERS_RESOLVED: + return NXT_OK; + + case NXT_HTTP_SET_HEADERS_REJECTED: + return NXT_DECLINED; + + default: + break; + } + + hv = ctx->action->set_headers->elts; + hv = &hv[i]; + value = &ctx->value[i]; + + if (hv->value == NULL) { + nxt_str_null(value); + + } else if (nxt_tstr_is_const(hv->value)) { + nxt_tstr_str(hv->value, value); + + } else { + rtcf = r->conf->socket_conf->router_conf; + + ret = nxt_tstr_query_init(&r->tstr_query, rtcf->tstr_state, + &r->tstr_cache, r, r->mem_pool); + if (nxt_slow_path(ret != NXT_OK)) { + return NXT_ERROR; + } + + ret = nxt_tstr_query(&r->task, r->tstr_query, hv->value, value); + if (nxt_slow_path(ret != NXT_OK)) { + return NXT_ERROR; + } + } + + if (value->start != NULL + && nxt_slow_path(!nxt_http_header_value_is_safe(value))) + { + ctx->state[i] = NXT_HTTP_SET_HEADERS_REJECTED; + return NXT_DECLINED; + } + + ctx->state[i] = NXT_HTTP_SET_HEADERS_RESOLVED; + + return NXT_OK; +} + + static nxt_http_field_t * nxt_http_resp_header_find(nxt_http_request_t *r, u_char *name, size_t length) { @@ -127,14 +408,13 @@ nxt_http_resp_header_find(nxt_http_request_t *r, u_char *name, size_t length) nxt_int_t nxt_http_set_headers(nxt_http_request_t *r) { - u_char *rejected; - nxt_int_t ret; - nxt_uint_t i, n; - nxt_str_t *value; - nxt_http_field_t *f; - nxt_router_conf_t *rtcf; - nxt_http_action_t *action; - nxt_http_header_val_t *hv, *header; + nxt_int_t ret; + nxt_uint_t i, n; + nxt_str_t *value; + nxt_http_field_t *f; + nxt_http_action_t *action; + nxt_http_header_val_t *hv, *header; + nxt_http_set_headers_ctx_t *ctx; action = r->action; @@ -146,66 +426,50 @@ nxt_http_set_headers(nxt_http_request_t *r) return NXT_OK; } - rtcf = r->conf->socket_conf->router_conf; - header = action->set_headers->elts; n = action->set_headers->nelts; - value = nxt_mp_zalloc(r->mem_pool, sizeof(nxt_str_t) * n); - if (nxt_slow_path(value == NULL)) { - return NXT_ERROR; - } + /* + * nxt_http_set_headers_field_op() can have resolved some keys already. + * Their stored results are used here, and they are not resolved again. + */ - rejected = nxt_mp_zalloc(r->mem_pool, n); - if (nxt_slow_path(rejected == NULL)) { + ctx = nxt_http_set_headers_ctx(r); + if (nxt_slow_path(ctx == NULL)) { return NXT_ERROR; } + value = ctx->value; + for (i = 0; i < n; i++) { hv = &header[i]; - if (hv->value == NULL) { - continue; - } - - if (nxt_tstr_is_const(hv->value)) { - nxt_tstr_str(hv->value, &value[i]); - - } else { - ret = nxt_tstr_query_init(&r->tstr_query, rtcf->tstr_state, - &r->tstr_cache, r, r->mem_pool); - if (nxt_slow_path(ret != NXT_OK)) { - return NXT_ERROR; - } + ret = nxt_http_set_headers_value(r, ctx, i); - ret = nxt_tstr_query(&r->task, r->tstr_query, hv->value, &value[i]); - if (nxt_slow_path(ret != NXT_OK)) { - return NXT_ERROR; - } - } - - if (value[i].start != NULL - && nxt_slow_path(!nxt_http_header_value_is_safe(&value[i]))) - { + if (nxt_slow_path(ret == NXT_DECLINED)) { nxt_log(&r->task, NXT_LOG_INFO, "set_headers \"%V\": dropping value containing control " "bytes (HTTP response-splitting protection)", &hv->name); /* - * Mark the entry as rejected instead of clearing the value: + * The entry is marked as rejected instead of clearing the value: * a NULL value means "delete this header", and letting an * attacker-triggered rejection remove an existing response * header (e.g. an app-emitted X-Frame-Options) would fail * open. Rejected entries are skipped entirely below, so a * pre-existing same-named header survives untouched. */ - rejected[i] = 1; + continue; + } + + if (nxt_slow_path(ret != NXT_OK)) { + return NXT_ERROR; } } for (i = 0; i < n; i++) { - if (rejected[i]) { + if (ctx->state[i] == NXT_HTTP_SET_HEADERS_REJECTED) { continue; } diff --git a/src/nxt_http_static.c b/src/nxt_http_static.c index 7d6e9deef..b4e0621a0 100644 --- a/src/nxt_http_static.c +++ b/src/nxt_http_static.c @@ -167,6 +167,13 @@ static void nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, nxt_http_static_ctx_t *ctx); static void nxt_http_static_next(nxt_task_t *task, nxt_http_request_t *r, nxt_http_static_ctx_t *ctx, nxt_http_status_t status); +static nxt_http_status_t nxt_http_static_preconditions(nxt_http_request_t *r, + nxt_str_t *etag, nxt_bool_t weak, nxt_time_t mtime, nxt_bool_t date_ok); +static nxt_bool_t nxt_http_static_etag_match(nxt_str_t *list, nxt_str_t *etag, + nxt_bool_t own_weak, nxt_bool_t strong); +static nxt_http_status_t nxt_http_static_range(nxt_http_request_t *r, + nxt_str_t *etag, nxt_bool_t weak, nxt_time_t mtime, nxt_bool_t date_ok, + nxt_off_t size, nxt_off_t *start, nxt_off_t *end); #if (NXT_HAVE_OPENAT2) static u_char *nxt_http_static_chroot_match(u_char *chr, u_char *shr); #endif @@ -481,27 +488,34 @@ nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, nxt_http_static_ctx_t *ctx) { size_t length, encode; - u_char *p, *fname; + u_char *p, *end, *fname; struct tm tm; nxt_buf_t *fb; nxt_int_t ret; - nxt_str_t *shr, *index, exten, *mtype; + nxt_str_t *shr, *index, exten, *mtype, etag; nxt_uint_t level; nxt_file_t *f, file; nxt_file_info_t fi; - nxt_http_field_t *field; - nxt_http_status_t status; + nxt_off_t range_start, range_end; + nxt_bool_t is_range, etag_weak, svgz; + nxt_http_status_t rstatus; + nxt_http_field_t *field, *coding; + nxt_http_status_t status, pcond; nxt_router_conf_t *rtcf; nxt_http_action_t *action; nxt_work_handler_t body_handler; nxt_http_static_conf_t *conf; + static const nxt_str_t svgz_exten = nxt_string(".svgz"); + static const nxt_str_t svg_mtype = nxt_string("image/svg+xml"); + action = ctx->action; conf = action->u.conf; rtcf = r->conf->socket_conf->router_conf; f = NULL; mtype = NULL; + coding = NULL; shr = &ctx->share; index = &conf->index; @@ -701,11 +715,35 @@ nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, goto fail; } - nxt_localtime(nxt_file_mtime(&fi), &tm); + nxt_gmtime(nxt_file_mtime(&fi), &tm); field->value = p; field->value_length = nxt_http_date(p, &tm) - p; + if (exten.start == NULL) { + nxt_http_static_extract_extension(shr, &exten); + } + + if (mtype == NULL) { + mtype = nxt_http_static_mtype_get(&rtcf->mtypes_hash, &exten); + } + + /* + * A ".svgz" file is an SVG image stored with gzip coding, so it is + * sent with "Content-Encoding: gzip", as Apache does with + * "AddEncoding gzip svgz". The coding is not negotiated. A client + * without gzip in Accept-Encoding still gets the gzip bytes, because + * no other representation exists. + * + * The extension must also resolve to image/svg+xml. An operator who + * maps ".svgz" to another type in "mime_types" gets no coding. + * + * The decision is made here because it also changes the ETag below. + */ + + svgz = (nxt_strcasestr_eq(&exten, &svgz_exten) + && nxt_strcasestr_eq(mtype, &svg_mtype)); + field = nxt_http_resp_field_zero_add(&r->resp, r->mem_pool); if (nxt_slow_path(field == NULL)) { goto fail; @@ -713,7 +751,49 @@ nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, nxt_http_field_name_set(field, "ETag"); - length = NXT_TIME_T_HEXLEN + NXT_OFF_T_HEXLEN + 3; + /* + * RFC 9110 Sect. 8.8.1: a validator is strong only when the + * representation cannot change again without the validator changing + * with it. Both of ours are derived from the whole-second mtime and + * the size, so a rewrite to the same size during the second the file + * was last written is invisible to both. While the clock is still + * inside that second another write can still land there, so the tag + * cannot be promised strong; once the second has passed, no later + * write can reproduce this mtime and the tag is strong for good. + * + * Apache weakens on the same hazard (modules/http/http_etag.c, in + * ap_make_etag_ex()), though on a sliding second against a + * microsecond mtime rather than the calendar second a whole-second + * mtime gives us. The tag's format does not change, so nothing + * already in a cache is invalidated by this. + * + * This reasons about the clock that stamped the file, so it holds + * only where that is the clock Unit reads. On a remote filesystem + * the mtime comes from the server: if that clock trails this one, + * Unit can call a tag strong while the server can still write into + * the second it names. Apache carries the same caveat. + * + * nxt_thread_time() is the cached per-thread clock. If it lags, it + * reports the request as still inside the second and the tag is + * weakened when it need not have been -- the safe direction. + */ + + etag_weak = ((nxt_time_t) nxt_thread_time(task->thread) + <= (nxt_time_t) nxt_file_mtime(&fi)); + + /* + * The tag of a ".svgz" file ends in "-gzip". Before the file was + * sent with gzip coding, its tag had no suffix. The response had no + * coding, or it was compressed a second time under the weak form of + * the tag. A cache that stored either response sends that tag back + * in If-None-Match. The new tag does not match it, so the cache + * gets a full 200 with the new header fields instead of a 304 that + * keeps the old entry. Apache's mod_deflate adds the same suffix + * to the tag of a response it compresses. + */ + + length = nxt_length("W/") + NXT_TIME_T_HEXLEN + NXT_OFF_T_HEXLEN + + nxt_length("-gzip") + 3; p = nxt_mp_nget(r->mem_pool, length); if (nxt_slow_path(p == NULL)) { @@ -721,18 +801,35 @@ nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, } field->value = p; - field->value_length = nxt_sprintf(p, p + length, "\"%xT-%xO\"", - nxt_file_mtime(&fi), - nxt_file_size(&fi)) - - p; - if (exten.start == NULL) { - nxt_http_static_extract_extension(shr, &exten); + if (etag_weak) { + *p++ = 'W'; + *p++ = '/'; } + /* + * nxt_file_mtime() yields a native time_t, which need not be + * nxt_time_t: on QNX it is a 32-bit unsigned type against a 64-bit + * nxt_time_t. "%T" reads an nxt_time_t from the argument list, so + * the value has to be converted before it is passed, not after. + */ + end = field->value + length; - if (mtype == NULL) { - mtype = nxt_http_static_mtype_get(&rtcf->mtypes_hash, &exten); - } + field->value_length = nxt_sprintf(p, end, "\"%xT-%xO%s\"", + (nxt_time_t) nxt_file_mtime(&fi), + nxt_file_size(&fi), + svgz ? (u_char *) "-gzip" + : (u_char *) "") + - field->value; + + /* + * The comparison functions work on the opaque tag, so "etag" skips + * the prefix; weakness travels beside it as a flag rather than in + * the string. Leaving "W/" in here would make every If-None-Match + * miss, since the client sends back the opaque tag it was given. + */ + + etag.start = p; + etag.length = field->value_length - (p - field->value); if (mtype->length != 0) { field = nxt_http_resp_field_zero_add(&r->resp, r->mem_pool); @@ -748,33 +845,285 @@ nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, r->resp.mime_type = mtype; - if (ctx->need_body && nxt_file_size(&fi) > 0) { - ret = nxt_http_comp_check_compression(task, r); - if (ret == NXT_HTTP_NOT_ACCEPTABLE) { - nxt_http_request_error(task, r, NXT_HTTP_NOT_ACCEPTABLE); + /* + * The Content-Encoding of a ".svgz" file is added before the + * acceptability check below. That check skips a response that + * already has a Content-Encoding. So the file is not compressed a + * second time, its ETag stays strong, and a Range applies to the + * stored bytes. + */ + + if (svgz) { + field = nxt_http_resp_field_zero_add(&r->resp, r->mem_pool); + if (nxt_slow_path(field == NULL)) { + goto fail; + } + + nxt_http_field_name_set(field, "Content-Encoding"); + + field->value = (u_char *) "gzip"; + field->value_length = nxt_length("gzip"); + + coding = field; + } + + /* + * RFC 9110 Sect. 13.2.1: an ordinary failure outranks a precondition. + * If no acceptable representation exists the answer is 406, and it + * must not be displaced by the 304 or 412 a validator would give -- + * so ask about acceptability here, and apply the decision further + * down, on the path that actually sends a body. + */ + + ret = nxt_http_comp_check_acceptable(task, r); + if (ret == NXT_HTTP_NOT_ACCEPTABLE) { + /* + * Every other exit that answers without a body closes the file + * first -- the 304 and 416 branches below, and "fail:". This one + * returns rather than reaching either, so it has to close its own, + * or one unauthenticated request costs the router a descriptor. + */ + nxt_file_close(task, f); + f = NULL; + + nxt_http_request_error(task, r, NXT_HTTP_NOT_ACCEPTABLE); + return; + } else if (ret != NXT_OK) { + goto fail; + } + + /* + * A response with the stored coding of a ".svgz" file has a coding + * that an older cached response of the same file may lack. The + * Last-Modified date did not change when the coding was added, so a + * date cannot tell the two apart. Only the ETag can. So a date + * gives no 304 and no range for such a response. A client that + * sends If-None-Match or an entity-tag in If-Range is not affected. + */ + + pcond = nxt_http_static_preconditions(r, &etag, etag_weak, + nxt_file_mtime(&fi), + coding == NULL); + + if (pcond != NXT_HTTP_OK) { + nxt_file_close(task, f); + f = NULL; + + if (pcond == NXT_HTTP_PRECONDITION_FAILED) { + nxt_http_request_error(task, r, pcond); return; - } else if (ret != NXT_OK) { + } + + /* + * A 304 carries the validators and nothing else. + * content_length_n is reset so no Content-Length is emitted, and + * no body handler is scheduled; the h1 framing already + * special-cases 304, so keep-alive survives and the response is + * never chunked (src/nxt_h1proto.c). + */ + r->status = NXT_HTTP_NOT_MODIFIED; + r->resp.content_length_n = -1; + + /* + * RFC 9110 Sect. 15.4.5: a 304 should not carry representation + * metadata such as Content-Encoding. A cache keeps the field it + * stored with the 200. The compressed path also sends a 304 + * without it. + */ + + if (coding != NULL) { + coding->skip = 1; + } + + body_handler = NULL; + + goto send; + } + + field = nxt_http_resp_field_zero_add(&r->resp, r->mem_pool); + if (nxt_slow_path(field == NULL)) { + goto fail; + } + + nxt_http_field_name_set(field, "Accept-Ranges"); + + field->value = (u_char *) "bytes"; + field->value_length = nxt_length("bytes"); + + /* + * A HEAD is range-processed exactly like the GET it stands for, and + * differs only in sending no body. RFC 9110 Sect. 14.2 says range + * handling is defined for GET, which reads like an argument for + * ignoring Range here -- but Sect. 9.3.2 asks a HEAD to send the same + * header fields the GET would have sent, and nginx, Apache and Go's + * net/http all answer 206 with Content-Range to a HEAD. "curl -I -r" + * relies on it. Matching them is worth more than the stricter + * reading of a sentence about methods that do not define ranges. + */ + + rstatus = nxt_http_static_range(r, &etag, etag_weak, + nxt_file_mtime(&fi), coding == NULL, + nxt_file_size(&fi), &range_start, + &range_end); + + /* + * A range is served as identity, so a client that sent + * "identity;q=0" must not be given one -- see the comment on the + * drop below. Tested here rather than after the 416 branch so that + * ignoring the Range is all or nothing: an unsatisfiable range from + * such a client would otherwise still draw a 416 whose + * "Content-Range: bytes STAR/size" reports the size of exactly the + * representation it refused. + */ + + if (nxt_http_comp_identity_refused(r)) { + rstatus = NXT_HTTP_OK; + } + + if (rstatus == NXT_HTTP_RANGE_NOT_SATISFIABLE) { + nxt_file_close(task, f); + f = NULL; + + r->status = NXT_HTTP_RANGE_NOT_SATISFIABLE; + r->resp.content_length_n = 0; + + /* + * A 416 has no body, so it has no content coding. The + * error responses (406, 412, 500) need no such step: + * nxt_http_request_error() drops every field added here. + */ + + if (coding != NULL) { + coding->skip = 1; + } + + field = nxt_http_resp_field_zero_add(&r->resp, r->mem_pool); + if (nxt_slow_path(field == NULL)) { goto fail; } - if (nxt_http_comp_wants_compression()) { - size_t out_total; - nxt_int_t ret; + nxt_http_field_name_set(field, "Content-Range"); - ret = nxt_http_comp_compress_static_response( - task, r, &f, &fi, - NXT_HTTP_STATIC_BUF_SIZE, - &out_total); - if (ret == NXT_ERROR) { - goto fail; - } + length = nxt_length("bytes */") + NXT_OFF_T_LEN; + + p = nxt_mp_nget(r->mem_pool, length); + if (nxt_slow_path(p == NULL)) { + goto fail; + } + + field->value = p; + field->value_length = nxt_sprintf(p, p + length, "bytes */%O", + nxt_file_size(&fi)) + - p; + + body_handler = NULL; + goto send; + } + + is_range = (rstatus == NXT_HTTP_PARTIAL_CONTENT); + + /* + * Why the Range is dropped for a client that refused identity. + * + * Not because a coded range is impossible: Sect. 3.2 defines + * representation data as the bytes after content coding and Sect. + * 14.1.2 defines a byte range over that, so bytes 0-9 of the gzip + * representation is well defined -- and Unit codes the whole file + * into a temp file before sending, so it could slice that instead. + * + * Because a coded range is never resumable, and costs a whole file + * to answer. A coded representation carries a weak entity-tag + * (nxt_http_comp_weaken_etag()), and nxt_http_static_range() above + * compares an "If-Range" strongly, so a resumed coded range always + * falls back to the full response -- the one thing a range request + * exists to avoid. Meanwhile Unit compresses the whole file into a + * temp file to hand back ten bytes of it. + * + * So ignore the Range (Sect. 14.2 permits it) and send the full 200 + * in the coding the client did accept. That is already how this + * function answers a malformed range, a multi-range request and an + * If-Range mismatch, so it needs no new shape of response. A 406 + * would be the other reading, but it refuses a request that can be + * satisfied. + * + * Not reached when nothing is acceptable: that is already 406, from + * nxt_http_comp_check_acceptable() above. That includes a response + * that no coding is applied to, where the full 200 would be + * identity again. + */ + + if (is_range) { + r->status = NXT_HTTP_PARTIAL_CONTENT; + r->resp.content_length_n = range_end - range_start + 1; + + field = nxt_http_resp_field_zero_add(&r->resp, r->mem_pool); + if (nxt_slow_path(field == NULL)) { + goto fail; + } + + nxt_http_field_name_set(field, "Content-Range"); + + /* + * Three %O conversions, so three times NXT_OFF_T_LEN. With two, + * 40 digits are available across start, end and size: 13 digits + * each still fits, 14 does not, so a file of about 10 TB with a + * range near its end truncates the header. nxt_sprintf() clamps + * at the boundary, so the value is cut rather than the buffer + * overrun -- a wrong Content-Range rather than a crash. + */ + length = nxt_length("bytes -/") + 3 * NXT_OFF_T_LEN; - ret = nxt_file_info(f, &fi); + p = nxt_mp_nget(r->mem_pool, length); + if (nxt_slow_path(p == NULL)) { + goto fail; + } + + field->value = p; + field->value_length = nxt_sprintf(p, p + length, + "bytes %O-%O/%O", range_start, + range_end, nxt_file_size(&fi)) + - p; + } + + if (ctx->need_body && nxt_file_size(&fi) > 0) { + + /* + * A satisfiable Range request is served as identity partial + * content: content-coding a byte slice would either compress + * the wrong bytes (coding the whole file, then slicing, defeats + * the point of a range request) or require re-deriving which + * coded bytes correspond to the requested identity range, which + * most codings do not support at all. Skipping compression + * here is a plain read of the file, so it does not touch the + * temp-file swap that nxt_http_comp_compress_static_response() + * performs, nor r->resp.mime_type (already set above). + */ + + if (!is_range) { + ret = nxt_http_comp_apply_compression(task, r); if (nxt_slow_path(ret != NXT_OK)) { goto fail; } - r->resp.content_length_n = out_total; + if (nxt_http_comp_wants_compression(r)) { + size_t out_total; + nxt_int_t ret; + + ret = nxt_http_comp_compress_static_response( + task, r, &f, &fi, + NXT_HTTP_STATIC_BUF_SIZE, + &out_total); + if (ret == NXT_ERROR) { + goto fail; + } + + ret = nxt_file_info(f, &fi); + if (nxt_slow_path(ret != NXT_OK)) { + goto fail; + } + + r->resp.content_length_n = out_total; + } } fb = nxt_http_static_buf_alloc(task, r->mem_pool); @@ -783,7 +1132,14 @@ nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, } fb->file = f; - fb->file_end = nxt_file_size(&fi); + + if (is_range) { + fb->file_pos = range_start; + fb->file_end = range_end + 1; + + } else { + fb->file_end = nxt_file_size(&fi); + } r->out = fb; @@ -872,6 +1228,8 @@ nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, body_handler = NULL; } +send: + nxt_http_request_header_send(task, r, body_handler, NULL); r->state = &nxt_http_static_send_state; @@ -887,6 +1245,520 @@ nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, } +/* + * Conditional requests, RFC 9110 Sect. 13.2.2, evaluated in the order that + * section mandates: If-Match, then If-Unmodified-Since, then If-None-Match, + * then If-Modified-Since. Returns NXT_HTTP_OK to serve the file normally, + * NXT_HTTP_NOT_MODIFIED for a 304, or NXT_HTTP_PRECONDITION_FAILED for a 412. + * + * The later step in each pair is consulted only when the earlier one is + * absent: a client that sends both an entity-tag and a date is asking to be + * judged by the entity-tag, even when the tag does not match. + * + * "date_ok" is 0 when the date does not identify the representation. Then + * If-Modified-Since never gives a 304, because a 304 tells a cache to reuse + * the entry it has, and that entry can be stale. If-Unmodified-Since needs + * no such gate. A 412 makes no cache reuse an entry. A request that passes + * gets the full 200 with the current header fields. + */ + +static nxt_http_status_t +nxt_http_static_preconditions(nxt_http_request_t *r, nxt_str_t *etag, + nxt_bool_t weak, nxt_time_t mtime, nxt_bool_t date_ok) +{ + nxt_str_t value; + nxt_time_t date; + nxt_bool_t im_seen, im_match, inm_seen, inm_match; + nxt_http_field_t *f, *ium, *ims; + nxt_http_fields_iter_t iter; + + /* + * RFC 9110 Sect. 5.3: repeated field lines are equivalent to one line + * holding the comma-separated concatenation. If-Match and If-None-Match + * are both "#entity-tag" lists whose members are OR'd, so evaluating + * every line and remembering whether ANY of them matched is exactly that + * concatenation -- whereas keeping only the last line seen would refuse a + * legitimate request with 412 when an earlier If-Match line matched. + */ + + /* + * Judge a conditional request only against validators the client was + * actually given. If "response_headers" replaces or removes ETag or + * Last-Modified, what this function would compare is not what went out + * (src/nxt_http_set_headers.c), so decline rather than answer 304 or 412 + * on the strength of a tag the client never saw. Serving the full + * response is always a correct answer to a conditional request. + */ + + if (nxt_http_set_headers_override_validators(r)) { + return NXT_HTTP_OK; + } + + im_seen = 0; + im_match = 0; + inm_seen = 0; + inm_match = 0; + ium = NULL; + ims = NULL; + + /* + * Request headers land in r->inline_fields and only spill into the + * r->fields list, so a list-only walk silently sees nothing on an + * ordinary request. Use the iterator that covers both + * (src/nxt_http_parse.h). + */ + + for (f = nxt_http_fields_first(&iter, r->inline_fields, + r->num_inline_fields, r->fields); + f != NULL; + f = nxt_http_fields_next(&iter)) + { + if (f->skip) { + continue; + } + + switch (f->name_length) { + + case nxt_length("If-Match"): + if (nxt_strncasecmp(f->name, (u_char *) "If-Match", + nxt_length("If-Match")) == 0) + { + im_seen = 1; + + if (!im_match) { + value.start = f->value; + value.length = f->value_length; + + /* Sect. 13.1.1: If-Match compares strongly. */ + + im_match = nxt_http_static_etag_match(&value, etag, weak, + 1); + } + } + + break; + + case nxt_length("If-None-Match"): + if (nxt_strncasecmp(f->name, (u_char *) "If-None-Match", + nxt_length("If-None-Match")) == 0) + { + inm_seen = 1; + + if (!inm_match) { + value.start = f->value; + value.length = f->value_length; + + inm_match = nxt_http_static_etag_match(&value, etag, weak, + 0); + } + } + + break; + + case nxt_length("If-Modified-Since"): + if (nxt_strncasecmp(f->name, (u_char *) "If-Modified-Since", + nxt_length("If-Modified-Since")) == 0) + { + ims = f; + } + + break; + + case nxt_length("If-Unmodified-Since"): + if (nxt_strncasecmp(f->name, (u_char *) "If-Unmodified-Since", + nxt_length("If-Unmodified-Since")) == 0) + { + ium = f; + } + + break; + + default: + break; + } + } + + if (im_seen) { + if (!im_match) { + return NXT_HTTP_PRECONDITION_FAILED; + } + + } else if (ium != NULL) { + date = nxt_time_parse(ium->value, ium->value_length); + + if (date != (nxt_time_t) -1 && mtime > date) { + return NXT_HTTP_PRECONDITION_FAILED; + } + } + + if (inm_seen) { + if (inm_match) { + return NXT_HTTP_NOT_MODIFIED; + } + + return NXT_HTTP_OK; + } + + if (ims != NULL && date_ok) { + date = nxt_time_parse(ims->value, ims->value_length); + + /* + * An unparsable date is ignored rather than treated as an error. + * Sect. 13.1.3 makes "earlier than or equal to" the unmodified case; + * nginx offers an "exact" mode to survive a rollback to an older + * mtime, but the RFC comparison is what Unit implements. + */ + + if (date != (nxt_time_t) -1 && mtime <= date) { + return NXT_HTTP_NOT_MODIFIED; + } + } + + return NXT_HTTP_OK; +} + + +/* + * Matches an entity-tag against an If-Match or If-None-Match list. + * + * "etag" is the opaque tag without any "W/"; "own_weak" says whether the tag + * Unit generated for this representation is weak, which it is while the + * request falls inside the second the file was last written. + * + * Sect. 8.8.3.2: the strong function matches only when both tags are strong, + * so a weak tag on either side fails it -- the client's, or our own. The + * weak function strips the prefix and compares the opaque tags. + * + * "*" matches any existing representation, but only as the entire field + * value -- the grammar is "*" / #entity-tag, so it is not a list member. + */ + +static nxt_bool_t +nxt_http_static_etag_match(nxt_str_t *list, nxt_str_t *etag, + nxt_bool_t own_weak, nxt_bool_t strong) +{ + u_char *p, *end, *start; + nxt_bool_t weak; + nxt_str_t tag; + + p = list->start; + end = p + list->length; + + while (p < end && (*p == ' ' || *p == '\t')) { + p++; + } + + if (end - p == 1 && *p == '*') { + return 1; + } + + if (own_weak && strong) { + /* + * A strong comparison needs both tags strong (Sect. 8.8.3.2), and + * ours is not, so no entity-tag in the list can match. This is + * below the "*" test on purpose: "*" asks whether a representation + * exists at all, not whether a validator matches, so weakness does + * not bear on it. + */ + + return 0; + } + + while (p < end) { + + while (p < end && (*p == ' ' || *p == '\t' || *p == ',')) { + p++; + } + + if (p == end) { + break; + } + + weak = 0; + + if (end - p >= 2 && p[0] == 'W' && p[1] == '/') { + weak = 1; + p += 2; + } + + if (p == end || *p != '"') { + /* Not a valid entity-tag; skip to the next comma. */ + + while (p < end && *p != ',') { + p++; + } + + continue; + } + + start = p++; + + while (p < end && *p != '"') { + p++; + } + + if (p == end) { + break; + } + + p++; + + if (weak && strong) { + continue; + } + + tag.start = start; + tag.length = p - start; + + if (nxt_strstr_eq(&tag, etag)) { + return 1; + } + } + + return 0; +} + + +/* + * Parses a decimal run of digits starting at *p (bounded by end), advancing + * *p past what it consumed. Returns the parsed value, or -1 if *p pointed + * at a non-digit (no digits consumed, *p left unchanged). + */ + +static nxt_off_t +nxt_http_static_range_number(u_char **p, u_char *end) +{ + u_char *start; + nxt_off_t value; + + start = *p; + + if (*p == end || **p < '0' || **p > '9') { + return -1; + } + + value = 0; + + while (*p < end && **p >= '0' && **p <= '9') { + /* + * Saturate rather than wrap. A wrapped value goes NEGATIVE, and a + * negative first-pos passes both the "a >= size" and "b < a" tests, + * so the range is accepted and "rest = file_end - file_pos" in + * nxt_http_static_body_handler() comes out negative: nxt_min() casts + * it to a huge size_t, the buffer allocation fails, and the request + * is abandoned with the file still open in r->out. One header per + * leaked descriptor is an unauthenticated denial of service. + * + * Saturating is also what the RFC asks for at both ends: a first-pos + * of NXT_OFF_T_MAX is >= size, so Sect. 14.1.2 gives 416, while a + * suffix that large means "the whole representation". + */ + + if (value > (NXT_OFF_T_MAX - (**p - '0')) / 10) { + value = NXT_OFF_T_MAX; + + while (*p < end && **p >= '0' && **p <= '9') { + (*p)++; + } + + return value; + } + + value = value * 10 + (*(*p)++ - '0'); + } + + if (*p == start) { + return -1; + } + + return value; +} + + +/* + * RFC 9110 Sect. 14.1-14.4: parses a "Range" request header and, when + * "If-Range" (Sect. 13.1.5) is present, applies it only if the precondition + * matches the current representation. + * + * Returns: + * NXT_HTTP_OK -- no Range applies; serve the full 200. + * (No Range header, a malformed Range, a multi-range request -- a + * server may legally ignore Range entirely -- or an If-Range mismatch.) + * NXT_HTTP_PARTIAL_CONTENT -- "start"/"end" name an inclusive byte range + * to serve as a 206; both are clamped to [0, size - 1]. + * NXT_HTTP_RANGE_NOT_SATISFIABLE -- the single range is out of bounds; the + * caller answers 416 with a "Content-Range: bytes STAR/size" header. + * + * "date_ok" is 0 when the date does not identify the representation. Then + * a date in If-Range never matches. + */ + +static nxt_http_status_t +nxt_http_static_range(nxt_http_request_t *r, nxt_str_t *etag, + nxt_bool_t weak, nxt_time_t mtime, nxt_bool_t date_ok, nxt_off_t size, + nxt_off_t *start, nxt_off_t *end) +{ + u_char *p, *last; + nxt_off_t a, b, suffix; + nxt_time_t date; + nxt_bool_t match; + nxt_str_t value; + nxt_http_field_t *f, *range, *if_range; + nxt_http_fields_iter_t iter; + + range = NULL; + if_range = NULL; + + for (f = nxt_http_fields_first(&iter, r->inline_fields, + r->num_inline_fields, r->fields); + f != NULL; + f = nxt_http_fields_next(&iter)) + { + if (f->skip) { + continue; + } + + switch (f->name_length) { + + case nxt_length("Range"): + if (nxt_strncasecmp(f->name, (u_char *) "Range", + nxt_length("Range")) == 0) + { + range = f; + } + + break; + + case nxt_length("If-Range"): + if (nxt_strncasecmp(f->name, (u_char *) "If-Range", + nxt_length("If-Range")) == 0) + { + if_range = f; + } + + break; + + default: + break; + } + } + + if (range == NULL) { + return NXT_HTTP_OK; + } + + if (if_range != NULL) { + value.start = if_range->value; + value.length = if_range->value_length; + + if (value.length > 0 && (value.start[0] == '"' + || (value.length > 1 + && value.start[0] == 'W' + && value.start[1] == '/'))) + { + /* An entity-tag: Sect. 13.1.5 requires the strong comparison. */ + + match = nxt_http_static_etag_match(&value, etag, weak, 1); + + } else { + date = nxt_time_parse(value.start, value.length); + + /* + * Sect. 13.1.5: an exact match against the last modification, + * and only while that date is a strong validator. Inside the + * second the file was written it is not: the client would splice + * a range from one version onto a copy of another, and unlike a + * bad conditional GET that produces a corrupt file rather than a + * stale one. Refusing the If-Range costs a full response. + */ + + match = (date_ok && !weak && date != (nxt_time_t) -1 + && date == mtime); + } + + if (!match) { + return NXT_HTTP_OK; + } + } + + p = range->value; + last = p + range->value_length; + + if ((size_t) (last - p) <= nxt_length("bytes=") + || nxt_strncasecmp(p, (u_char *) "bytes=", nxt_length("bytes=")) != 0) + { + return NXT_HTTP_OK; + } + + p += nxt_length("bytes="); + + /* + * Only a single range-spec is supported; a comma anywhere in the + * remainder marks a multi-range request, which a server may ignore. + */ + + if (memchr(p, ',', last - p) != NULL) { + return NXT_HTTP_OK; + } + + if (*p == '-') { + p++; + + suffix = nxt_http_static_range_number(&p, last); + + if (suffix == -1 || p != last) { + return NXT_HTTP_OK; + } + + /* + * A suffix range is unsatisfiable when it asks for nothing, and also + * against a zero-length representation: "size - suffix" would clamp + * to 0 while "size - 1" is -1, yielding "Content-Range: bytes 0--1/0". + */ + + if (suffix == 0 || size == 0) { + return NXT_HTTP_RANGE_NOT_SATISFIABLE; + } + + a = (suffix < size) ? size - suffix : 0; + b = size - 1; + + } else { + a = nxt_http_static_range_number(&p, last); + + if (a == -1 || p == last || *p != '-') { + return NXT_HTTP_OK; + } + + p++; + + if (a >= size) { + return NXT_HTTP_RANGE_NOT_SATISFIABLE; + } + + if (p == last) { + b = size - 1; + + } else { + b = nxt_http_static_range_number(&p, last); + + if (b == -1 || p != last || b < a) { + return NXT_HTTP_OK; + } + + if (b >= size) { + b = size - 1; + } + } + } + + *start = a; + *end = b; + + return NXT_HTTP_PARTIAL_CONTENT; +} + + static void nxt_http_static_next(nxt_task_t *task, nxt_http_request_t *r, nxt_http_static_ctx_t *ctx, nxt_http_status_t status) @@ -990,7 +1862,7 @@ nxt_http_static_extract_extension(nxt_str_t *path, nxt_str_t *exten) switch (ch) { case '/': p++; - /* Fall through. */ + nxt_fallthrough; case '.': goto extension; } @@ -1107,6 +1979,11 @@ nxt_http_static_buf_completion(nxt_task_t *task, void *obj, void *data) b->next = nxt_http_buf_last(r); } else { + if (nxt_slow_path(n == 0)) { + /* file truncated since it was stat(2)'d */ + nxt_http_request_error_handler(task, r, r->proto.any); + goto clean; + } fb->file_pos += n; b->next = NULL; } @@ -1125,6 +2002,18 @@ nxt_http_static_buf_completion(nxt_task_t *task, void *obj, void *data) clean: + /* + * The file goes first: the buffers below may hold the last references + * to the request pool, and when the pool is destroyed r is gone and + * nxt_http_static_buf_cleanup() would close the file a second time. + */ + if (fb != NULL) { + nxt_file_close(task, fb->file); + r->out = NULL; + + nxt_http_static_buf_free(fb); + } + do { next = b->next; @@ -1133,13 +2022,6 @@ nxt_http_static_buf_completion(nxt_task_t *task, void *obj, void *data) b = next; } while (b != NULL); - - if (fb != NULL) { - nxt_file_close(task, fb->file); - r->out = NULL; - - nxt_http_static_buf_free(fb); - } } @@ -1185,6 +2067,7 @@ nxt_http_static_mtypes_init(nxt_mp_t *mp, nxt_lvlhsh_t *hash) { nxt_string("text/css"), ".css" }, { nxt_string("image/svg+xml"), ".svg" }, + { nxt_string("image/svg+xml"), ".svgz" }, { nxt_string("image/webp"), ".webp" }, { nxt_string("image/png"), ".png" }, { nxt_string("image/apng"), ".apng" }, diff --git a/src/nxt_http_websocket.c b/src/nxt_http_websocket.c index 39477e2dc..4b84d73eb 100644 --- a/src/nxt_http_websocket.c +++ b/src/nxt_http_websocket.c @@ -115,7 +115,44 @@ nxt_http_websocket_client(nxt_task_t *task, void *obj, void *data) req_rpc_data->stream, task->thread->engine->port->id, out); if (nxt_slow_path(res != NXT_OK)) { - // TODO: handle + /* + * The port layer did not take the chain, so it is still ours to + * return -- and nothing else will: these are chunks of the + * application's outgoing shared memory, which no pool teardown + * reclaims. A frame lost this way costs the segment capacity + * permanently, and the application starves once enough have gone. + * + * Queued rather than run here, as nxt_port_msg_drop() queues its + * own: nxt_port_mmap_buf_completion() can post a SHM_ACK, and + * re-entering the port layer from inside this call is worth + * avoiding even on the way out. + */ + + while (out != NULL) { + next = out->next; + out->next = NULL; + + nxt_work_queue_add(&task->thread->engine->fast_work_queue, + out->completion_handler, task, out, + out->parent); + + out = next; + } + + /* + * And fail the connection, the way the allocation failure above + * does. A non-OK answer here means the application port is gone or + * its shared queue is full, so this frame is not reaching the + * application at all; carrying on would leave the peer's stream + * short a frame it is never told about, which for a fragmented + * message is a protocol error the client cannot detect. This is + * what the "// TODO: handle" that used to stand here was asking + * for. + */ + + nxt_http_websocket_error_handler(task, r, r->proto.any); + + return; } b = r->ws_frame; diff --git a/src/nxt_isolation.c b/src/nxt_isolation.c index d5ea563f0..b9f3a9322 100644 --- a/src/nxt_isolation.c +++ b/src/nxt_isolation.c @@ -846,21 +846,263 @@ nxt_isolation_unmount_all(nxt_task_t *task, nxt_process_t *process) } +#if (NXT_HAVE_OPENAT2) +static nxt_bool_t nxt_isolation_no_openat2; +#endif + + +/* + * Open the mount destination rel, a path relative to rootfs_fd, as a + * directory. Create each missing component with mkdirat() against its + * parent fd. mkdirat() does not follow a symlink in the last component, + * so no directory is ever created outside the rootfs. With openat2() each + * prefix is resolved from rootfs_fd with RESOLVE_BENEATH: relative in-tree + * links such as "lib -> usr/lib" still work; absolute links and links that + * leave the rootfs fail with EXDEV. Without openat2() a symlinked + * component is refused. + * + * The returned O_PATH fd pins the directory inode, not its name: the + * directory can still be renamed after resolution. This is a known limit + * of fd walks. + */ + +static nxt_int_t +nxt_isolation_mount_dst_open(nxt_task_t *task, int rootfs_fd, + const u_char *dst, const u_char *rel, int *fdp) +{ + int fd, parent, flags; +#if (NXT_HAVE_OPENAT2) + int tries; +#endif + char *path, *p, *comp, save; + size_t len; + nxt_err_t err; +#if (NXT_HAVE_OPENAT2) + struct open_how how; +#endif + +#if defined(O_PATH) + flags = O_PATH | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC; +#else + flags = O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC; +#endif + + /* + * A module path such as "//usr/lib" leaves a '/' at the start of rel. + * openat2() with RESOLVE_BENEATH refuses an absolute path. + */ + while (*rel == '/') { + rel++; + } + + len = nxt_strlen(rel); + + path = nxt_malloc(len + 1); + if (nxt_slow_path(path == NULL)) { + return NXT_ERROR; + } + + nxt_memcpy(path, rel, len + 1); + + parent = rootfs_fd; + p = path; + + for ( ;; ) { + while (*p == '/') { + p++; + } + + if (*p == '\0') { + break; + } + + comp = p; + + while (*p != '\0' && *p != '/') { + p++; + } + + save = *p; + *p = '\0'; + + /* + * A module path from the build (for example sys.path) can contain + * "." or "..". Without openat2() ".." would climb out of the rootfs. + */ + if (nxt_slow_path(nxt_strcmp(comp, ".") == 0 + || nxt_strcmp(comp, "..") == 0)) + { + nxt_alert(task, "mount destination %s: component \"%s\" " + "is not allowed", dst, comp); + goto fail; + } + + if (mkdirat(parent, comp, 0777) != 0 && nxt_errno != EEXIST) { + nxt_alert(task, "mount destination %s: mkdirat(\"%s\") %E", + dst, comp, nxt_errno); + goto fail; + } + + fd = -1; + +#if (NXT_HAVE_OPENAT2) + if (!nxt_isolation_no_openat2) { + nxt_memzero(&how, sizeof(how)); + how.flags = O_PATH | O_DIRECTORY | O_CLOEXEC; + how.resolve = RESOLVE_BENEATH | RESOLVE_NO_MAGICLINKS; + + /* + * Resolve the full prefix from rootfs_fd, not comp from parent. + * From parent, an in-tree link such as "usr/lib64 -> ../lib" + * fails. A rename race can only cause ENOENT, not an escape. + * + * With RESOLVE_BENEATH the kernel returns EAGAIN when a rename + * or a mount occurs on the system during a ".." lookup. Try + * again a small number of times. + */ + for (tries = 3; tries > 0; tries--) { + fd = syscall(SYS_openat2, rootfs_fd, path, &how, sizeof(how)); + + if (fd != -1 || nxt_errno != EAGAIN) { + break; + } + } + + if (fd == -1) { + err = nxt_errno; + + if (err == ENOENT) { + nxt_alert(task, "mount destination %s: component \"%s\" " + "cannot be resolved beneath rootfs", dst, comp); + goto fail; + } + + /* + * ENOSYS: openat2() is Linux 5.6+. EPERM: a seccomp + * profile or gVisor denies the syscall. EINVAL: the + * kernel does not know a resolve flag. The fallback + * below is stricter, so it is safe in all three cases. + */ + if (err != ENOSYS && err != EPERM && err != EINVAL) { + nxt_alert(task, "mount destination %s: component \"%s\" " + "escapes rootfs or is not a directory: %E", + dst, comp, err); + goto fail; + } + + nxt_isolation_no_openat2 = 1; + + /* Logged once per prototype start. */ + nxt_log(task, NXT_LOG_WARN, + "openat2() is not supported by the running kernel; " + "rootfs mount destinations must not contain " + "symlinks"); + } + } +#endif + + if (fd == -1) { + fd = openat(parent, comp, flags); + + if (fd == -1) { + err = nxt_errno; + + /* + * Linux returns ENOTDIR for a symlink opened with + * O_NOFOLLOW | O_DIRECTORY. POSIX allows ELOOP, and + * other systems can return it. + */ + if (err == ENOTDIR || err == ELOOP) { + nxt_alert(task, "mount destination %s: component \"%s\" " + "is a symlink or not a directory", dst, comp); + + } else { + nxt_alert(task, "mount destination %s: openat(\"%s\") %E", + dst, comp, err); + } + + goto fail; + } + } + + *p = save; + + if (parent != rootfs_fd) { + close(parent); + } + + parent = fd; + } + + nxt_free(path); + + if (nxt_slow_path(parent == rootfs_fd)) { + nxt_alert(task, "mount destination %s is the rootfs", dst); + return NXT_ERROR; + } + + *fdp = parent; + + return NXT_OK; + +fail: + + if (parent != rootfs_fd) { + close(parent); + } + + nxt_free(path); + + return NXT_ERROR; +} + + +#if (NXT_TESTS) + +/* + * Lets src/test/nxt_isolation_mount_dst_test.c run the walk above. When + * fallback is set, the walk does not call openat2(). It takes the + * O_NOFOLLOW branch, as after ENOSYS, EPERM or EINVAL. *openat2_used is + * set when the walk did not switch to that branch. + */ + +nxt_int_t +nxt_isolation_test_mount_dst_open(nxt_task_t *task, int rootfs_fd, + const char *rel, nxt_bool_t fallback, nxt_bool_t *openat2_used, int *fdp) +{ + nxt_int_t ret; + +#if (NXT_HAVE_OPENAT2) + nxt_isolation_no_openat2 = fallback; +#endif + + ret = nxt_isolation_mount_dst_open(task, rootfs_fd, (const u_char *) rel, + (const u_char *) rel, fdp); + +#if (NXT_HAVE_OPENAT2) + *openat2_used = !nxt_isolation_no_openat2; +#else + *openat2_used = 0; +#endif + + return ret; +} + +#endif + + nxt_int_t nxt_isolation_prepare_rootfs(nxt_task_t *task, nxt_process_t *process) { - size_t i, n; + int rootfs_fd, mnt_dst_fd; + u_char *mounted; + size_t i, n, rootfs_len; nxt_int_t ret; struct stat st; nxt_array_t *mounts; - const u_char *dst; + const u_char *dst, *rootfs_path; nxt_fs_mount_t *mnt; nxt_process_automount_t *automount; -#if (NXT_HAVE_OPENAT2) - int rootfs_fd; - const u_char *rootfs_path; - size_t rootfs_len; -#endif automount = &process->isolation.automount; mounts = process->isolation.mounts; @@ -892,35 +1134,42 @@ nxt_isolation_prepare_rootfs(nxt_task_t *task, nxt_process_t *process) } #endif -#if (NXT_HAVE_OPENAT2) /* - * Mount destinations live under a (possibly user-owned) rootfs. An - * attacker with write access to that tree can race mkdir->mount(2) - * by swapping a path component for a symlink pointing outside the - * rootfs. Re-open the rootfs and resolve each destination with - * RESOLVE_BENEATH so a tampered destination that escapes the - * rootfs fails the open, before we hand a path to mount(2). - * RESOLVE_NO_SYMLINKS is intentionally NOT requested: legitimate - * rootfs setups commonly contain in-tree symlinks (e.g. - * /lib -> /usr/lib) that must still resolve. + * Mount destinations live under a rootfs that may be writable by the + * application user. nxt_isolation_mount_dst_open() creates them + * without following symlinks and returns an fd. On Linux the mount + * targets that fd, not the path. */ rootfs_path = process->isolation.rootfs; - rootfs_len = (rootfs_path != NULL) ? nxt_strlen(rootfs_path) : 0; - - rootfs_fd = -1; - if (rootfs_len > 0) { - rootfs_fd = open((const char *) rootfs_path, - O_PATH | O_DIRECTORY | O_CLOEXEC); - if (rootfs_fd == -1) { - nxt_alert(task, "open rootfs(%s) %E", rootfs_path, nxt_errno); - return NXT_ERROR; - } - } + rootfs_len = nxt_strlen(rootfs_path); + + /* + * The rootfs is operator config. It must exist: a missing rootfs is + * refused, not created as root. + */ +#if defined(O_PATH) + rootfs_fd = open((const char *) rootfs_path, + O_PATH | O_DIRECTORY | O_CLOEXEC); +#else + rootfs_fd = open((const char *) rootfs_path, + O_RDONLY | O_DIRECTORY | O_CLOEXEC); #endif + if (nxt_slow_path(rootfs_fd == -1)) { + nxt_alert(task, "open rootfs(%s) %E", rootfs_path, nxt_errno); + return NXT_ERROR; + } - for (i = 0; i < n; i++) { - int mnt_dst_fd = -1; + /* + * mounted[i] is set when this call has mounted mnt[i]. The undo path + * unmounts only those. One extra byte, because n can be 0. + */ + mounted = nxt_zalloc(n + 1); + if (nxt_slow_path(mounted == NULL)) { + close(rootfs_fd); + return NXT_ERROR; + } + for (i = 0; i < n; i++) { dst = mnt[i].dst; if (mnt[i].deps && !automount->language_deps) { @@ -934,117 +1183,58 @@ nxt_isolation_prepare_rootfs(nxt_task_t *task, nxt_process_t *process) continue; } - ret = nxt_fs_mkdir_p(dst, 0777); - if (nxt_slow_path(ret != NXT_OK)) { - nxt_alert(task, "mkdir(%s) %E", dst, nxt_errno); - goto undo; - } - -#if (NXT_HAVE_OPENAT2) /* - * Require a path separator right after the rootfs prefix so a - * sibling like "-helper" is not treated as living under - * "" (which would resolve a bogus relative path and abort - * startup with a spurious ENOENT). dst is always built as - * rootfs + a component starting with '/', so this byte is '/' for - * every real destination; the preceding length check makes the - * index safe to read. + * nxt_isolation_set_lang_mounts() builds every destination as + * rootfs + module path. A module path from the build (for example + * sys.path) can be relative, so check that it starts with '/'. */ - if (rootfs_fd != -1 - && nxt_strlen(dst) > rootfs_len - && memcmp(dst, rootfs_path, rootfs_len) == 0 - && dst[rootfs_len] == '/') - { - struct open_how how; - const char *rel; - int fd; - - rel = (const char *) dst + rootfs_len; - while (*rel == '/') { - rel++; - } + nxt_assert(memcmp(dst, rootfs_path, rootfs_len) == 0); - nxt_memzero(&how, sizeof(how)); - how.flags = O_PATH | O_CLOEXEC | O_DIRECTORY; - how.resolve = RESOLVE_BENEATH; - - fd = syscall(SYS_openat2, rootfs_fd, rel, &how, sizeof(how)); - if (fd == -1) { - if (nxt_errno == ENOSYS) { - /* - * openat2(2) is Linux 5.6+; older kernels return - * ENOSYS even when the userspace headers are - * present. Skip the symlink-resolution check on - * such kernels rather than failing every isolation - * setup; mount(2) below still operates on the - * caller-supplied path, just without the extra - * RESOLVE_BENEATH guard. Warn once so operators - * see the reduced security posture. - */ - static nxt_bool_t openat2_warned; - if (!openat2_warned) { - nxt_log(task, NXT_LOG_WARN, - "openat2(SYS_openat2) not supported by the " - "running kernel; rootfs mount destinations " - "are not validated against symlinks"); - openat2_warned = 1; - } - - } else { - nxt_alert(task, "mount destination %s escapes rootfs %s " - "or contains symlinks: %E", dst, rootfs_path, - nxt_errno); - ret = NXT_ERROR; - goto undo; - } + if (nxt_slow_path(dst[rootfs_len] != '/')) { + nxt_alert(task, "mount destination %s is not an absolute path " + "under rootfs %s", dst, rootfs_path); + goto undo; + } - } else { - /* - * Keep the validated fd open and mount onto it directly - * (via /proc/self/fd/ in nxt_fs_mount) so the mount - * targets the exact inode openat2 resolved, rather than - * re-resolving dst as a path and reopening the check-then- - * use window. Closed after the mount below. - */ - mnt_dst_fd = fd; - } + ret = nxt_isolation_mount_dst_open(task, rootfs_fd, dst, + dst + rootfs_len + 1, &mnt_dst_fd); + if (nxt_slow_path(ret != NXT_OK)) { + goto undo; } -#endif ret = nxt_fs_mount(task, &mnt[i], mnt_dst_fd); -#if (NXT_HAVE_OPENAT2) - if (mnt_dst_fd != -1) { - close(mnt_dst_fd); - } -#endif + close(mnt_dst_fd); if (nxt_slow_path(ret != NXT_OK)) { goto undo; } - } -#if (NXT_HAVE_OPENAT2) - if (rootfs_fd != -1) { - close(rootfs_fd); + mounted[i] = 1; } -#endif + + nxt_free(mounted); + close(rootfs_fd); return NXT_OK; undo: - n = i + 1; + /* + * Unmount only the destinations that this call mounted. A skipped or + * failed destination is not mounted, and its path can go through a + * symlink. + */ + n = i; for (i = 0; i < n; i++) { - nxt_fs_unmount(mnt[i].dst); + if (mounted[i]) { + nxt_fs_unmount(mnt[i].dst); + } } -#if (NXT_HAVE_OPENAT2) - if (rootfs_fd != -1) { - close(rootfs_fd); - } -#endif + nxt_free(mounted); + close(rootfs_fd); return NXT_ERROR; } diff --git a/src/nxt_isolation.h b/src/nxt_isolation.h index b1bfc33c7..172db3317 100644 --- a/src/nxt_isolation.h +++ b/src/nxt_isolation.h @@ -13,6 +13,12 @@ nxt_int_t nxt_isolation_main_prefork(nxt_task_t *task, nxt_process_t *process, nxt_int_t nxt_isolation_prepare_rootfs(nxt_task_t *task, nxt_process_t *process); nxt_int_t nxt_isolation_change_root(nxt_task_t *task, nxt_process_t *process); + +#if (NXT_TESTS) +/* The mount destination walk, for the mount destination test. */ +nxt_int_t nxt_isolation_test_mount_dst_open(nxt_task_t *task, int rootfs_fd, + const char *rel, nxt_bool_t fallback, nxt_bool_t *openat2_used, int *fdp); +#endif #endif #endif /* _NXT_ISOLATION_H_INCLUDED_ */ diff --git a/src/nxt_job.c b/src/nxt_job.c deleted file mode 100644 index 560739537..000000000 --- a/src/nxt_job.c +++ /dev/null @@ -1,199 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) NGINX, Inc. - */ - -#include - - -static void nxt_job_thread_trampoline(nxt_task_t *task, void *obj, void *data); -static void nxt_job_thread_return_handler(nxt_task_t *task, void *obj, - void *data); - - -void * -nxt_job_create(nxt_mp_t *mp, size_t size) -{ - size_t cache_size; - nxt_job_t *job; - - if (mp == NULL) { - mp = nxt_mp_create(1024, 128, 256, 32); - if (nxt_slow_path(mp == NULL)) { - return NULL; - } - - job = nxt_mp_zget(mp, size); - cache_size = 0; - - } else { - job = nxt_mp_zalloc(mp, size); - cache_size = size; - } - - if (nxt_slow_path(job == NULL)) { - return NULL; - } - - job->cache_size = (uint16_t) cache_size; - job->mem_pool = mp; - nxt_job_set_name(job, "job"); - - /* Allow safe nxt_queue_remove() in nxt_job_destroy(). */ - nxt_queue_self(&job->link); - - return job; -} - - -void -nxt_job_init(nxt_job_t *job, size_t size) -{ - nxt_memzero(job, size); - - nxt_job_set_name(job, "job"); - - nxt_queue_self(&job->link); -} - - -void -nxt_job_destroy(nxt_task_t *task, void *data) -{ - nxt_job_t *job; - - job = data; - - nxt_queue_remove(&job->link); - - if (job->cache_size == 0) { - - if (job->mem_pool != NULL) { - nxt_mp_destroy(job->mem_pool); - } - - } else { - nxt_mp_free(job->mem_pool, job); - } -} - - -#if 0 - -nxt_int_t -nxt_job_cleanup_add(nxt_mp_t *mp, nxt_job_t *job) -{ - nxt_mem_pool_cleanup_t *mpcl; - - mpcl = nxt_mem_pool_cleanup(mp, 0); - - if (nxt_fast_path(mpcl != NULL)) { - mpcl->handler = nxt_job_destroy; - mpcl->data = job; - return NXT_OK; - } - - return NXT_ERROR; -} - -#endif - - -/* - * The (void *) casts in nxt_thread_pool_post() and nxt_event_engine_post() - * calls and to the "nxt_work_handler_t" are required by Sun C. - */ - -void -nxt_job_start(nxt_task_t *task, nxt_job_t *job, nxt_work_handler_t handler) -{ - nxt_debug(task, "%s start", job->name); - - if (job->thread_pool != NULL) { - nxt_int_t ret; - - job->engine = task->thread->engine; - - nxt_work_set(&job->work, nxt_job_thread_trampoline, - job->task, job, (void *) handler); - - ret = nxt_thread_pool_post(job->thread_pool, &job->work); - - if (ret == NXT_OK) { - return; - } - - handler = job->abort_handler; - } - - handler(job->task, job, job->data); -} - - -/* A trampoline function is called by a thread pool thread. */ - -static void -nxt_job_thread_trampoline(nxt_task_t *task, void *obj, void *data) -{ - nxt_job_t *job; - nxt_work_handler_t handler; - - job = obj; - handler = (nxt_work_handler_t) data; - - nxt_debug(task, "%s thread", job->name); - - if (nxt_slow_path(job->cancel)) { - nxt_job_return(task, job, job->abort_handler); - - } else { - handler(job->task, job, job->data); - } -} - - -void -nxt_job_return(nxt_task_t *task, nxt_job_t *job, nxt_work_handler_t handler) -{ - nxt_debug(task, "%s return", job->name); - - if (job->engine != NULL) { - /* A return function is called in thread pool thread context. */ - - nxt_work_set(&job->work, nxt_job_thread_return_handler, - job->task, job, (void *) handler); - - nxt_event_engine_post(job->engine, &job->work); - - return; - } - - if (nxt_slow_path(job->cancel)) { - nxt_debug(task, "%s cancellation", job->name); - handler = job->abort_handler; - } - - nxt_work_queue_add(&task->thread->engine->fast_work_queue, - handler, job->task, job, job->data); -} - - -static void -nxt_job_thread_return_handler(nxt_task_t *task, void *obj, void *data) -{ - nxt_job_t *job; - nxt_work_handler_t handler; - - job = obj; - handler = (nxt_work_handler_t) data; - - job->task->thread = task->thread; - - if (nxt_slow_path(job->cancel)) { - nxt_debug(task, "%s cancellation", job->name); - handler = job->abort_handler; - } - - handler(job->task, job, job->data); -} diff --git a/src/nxt_job.h b/src/nxt_job.h deleted file mode 100644 index 0495c4844..000000000 --- a/src/nxt_job.h +++ /dev/null @@ -1,86 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) NGINX, Inc. - */ - -#ifndef _NXT_JOB_H_INCLUDED_ -#define _NXT_JOB_H_INCLUDED_ - - -/* - * A job may run by separate thread, so each job should have its - * its own mem_pool. A job itself is allocated from this mem_pool. - * On job completion a job initiator can destroy the job at once - * with nxt_job_destroy() or can postpone the destruction with - * nxt_job_cleanup_add(), if the initiator uses data from the job's - * mem_pool. - * - * Several child jobs may run in context of another job in the same - * thread. In this case the child job may use a mem_pool of the - * parent job and the child job is allocated using the mem_pool's cache. - * nxt_job_destroy() just returns the job to the cache. All job - * allocations however still remain in the parent mem_pool. - * - * The first thread in job thread pool is created on demand. If this - * operation fails the job abort handler is called. It also is called - * if the job is canceled. To avoid race condition the abort handler - * always runs in context of a thread initiated the job. The abort - * handler may be as simple as nxt_job_destroy(). - */ - - -typedef struct { - void *data; - - nxt_task_t *task; - - nxt_work_handler_t abort_handler; - - uint16_t cache_size; - uint8_t cancel; /* 1 bit */ - - nxt_mp_t *mem_pool; - nxt_queue_link_t link; - - nxt_thread_pool_t *thread_pool; - nxt_event_engine_t *engine; - nxt_log_t *log; - - nxt_work_t work; - -#if (NXT_DEBUG) - const char *name; -#endif - -} nxt_job_t; - - -NXT_EXPORT void *nxt_job_create(nxt_mp_t *mp, size_t size); -NXT_EXPORT void nxt_job_init(nxt_job_t *job, size_t size); -NXT_EXPORT void nxt_job_destroy(nxt_task_t *task, void *data); -NXT_EXPORT nxt_int_t nxt_job_cleanup_add(nxt_mp_t *mp, nxt_job_t *job); - -NXT_EXPORT void nxt_job_start(nxt_task_t *task, nxt_job_t *job, - nxt_work_handler_t handler); -NXT_EXPORT void nxt_job_return(nxt_task_t *task, nxt_job_t *job, - nxt_work_handler_t handler); - - -#define nxt_job_cancel(job) \ - (job)->cancel = 1 - - -#if (NXT_DEBUG) - -#define nxt_job_set_name(job, text) \ - (job)->name = text - -#else - -#define nxt_job_set_name(job, text) - -#endif - - -#endif /* _NXT_JOB_H_INCLUDED_ */ diff --git a/src/nxt_job_cache_file.c b/src/nxt_job_cache_file.c deleted file mode 100644 index 13f01a453..000000000 --- a/src/nxt_job_cache_file.c +++ /dev/null @@ -1,24 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) NGINX, Inc. - */ - - -#include - - -nxt_job_cache_file_t * -nxt_job_cache_file_create(nxt_mp_t *mp) -{ - nxt_job_cache_file_t *jbc; - - jbc = nxt_job_create(mp, sizeof(nxt_job_cache_file_t)); - - if (nxt_fast_path(jbc != NULL)) { - jbc->file.fd = NXT_FILE_INVALID; - jbc->read_required = nxt_job_file_read_required; - } - - return jbc; -} diff --git a/src/nxt_kqueue_engine.c b/src/nxt_kqueue_engine.c index e8e282437..a39fbdaff 100644 --- a/src/nxt_kqueue_engine.c +++ b/src/nxt_kqueue_engine.c @@ -59,6 +59,8 @@ static void nxt_kqueue_disable(nxt_event_engine_t *engine, nxt_fd_event_t *ev); static void nxt_kqueue_delete(nxt_event_engine_t *engine, nxt_fd_event_t *ev); static nxt_bool_t nxt_kqueue_close(nxt_event_engine_t *engine, nxt_fd_event_t *ev); +static void nxt_kqueue_cancel_changes(nxt_event_engine_t *engine, + nxt_fd_event_t *ev); static void nxt_kqueue_enable_read(nxt_event_engine_t *engine, nxt_fd_event_t *ev); static void nxt_kqueue_enable_write(nxt_event_engine_t *engine, @@ -142,6 +144,7 @@ const nxt_event_interface_t nxt_kqueue_engine = { nxt_kqueue_disable, nxt_kqueue_delete, nxt_kqueue_close, + nxt_kqueue_cancel_changes, nxt_kqueue_enable_read, nxt_kqueue_enable_write, nxt_kqueue_disable_read, @@ -313,6 +316,54 @@ nxt_kqueue_close(nxt_event_engine_t *engine, nxt_fd_event_t *ev) } +/* + * Take this event's pending changes out of the batch, so that a struct that + * is about to be freed is not dereferenced when the batch is flushed -- by + * nxt_kqueue_get_kevent() when it fills, or by nxt_kqueue_poll(). + * + * The change is dropped rather than flushed: the descriptor is closed, or is + * about to be, and kqueue(2) removes every kevent that references a + * descriptor when the descriptor is closed. Flushing instead would name a + * descriptor number that may already belong to somebody else. + * + * Matching is on ->udata, which nxt_kqueue_fd_set() sets to the event; that + * names the struct being freed exactly, where ->ident names only a + * descriptor number. nxt_kqueue_enable_file() puts a file event in the + * same field, and a pointer comparison tells the two apart on its own. + * + * Unlike the other engines this does not return early when ->changing is + * clear. Here the flag is only ever a "maybe" -- a flush leaves it set, + * for the reason nxt_kqueue_fd_set() gives -- so the scan is what decides, + * and it costs a walk of at most ->mchanges entries. + */ + +static void +nxt_kqueue_cancel_changes(nxt_event_engine_t *engine, nxt_fd_event_t *ev) +{ + struct kevent *kev, *dst, *end; + + dst = engine->u.kqueue.changes; + end = dst + engine->u.kqueue.nchanges; + + for (kev = dst; kev < end; kev++) { + + if (nxt_kevent_get_udata(kev->udata) == (void *) ev) { + continue; + } + + if (dst != kev) { + *dst = *kev; + } + + dst++; + } + + engine->u.kqueue.nchanges = (int) (dst - engine->u.kqueue.changes); + + ev->changing = 0; +} + + /* * The kqueue event engine uses only three states: inactive, blocked, and * active. An active oneshot event is marked as it is in the default @@ -451,6 +502,25 @@ nxt_kqueue_fd_set(nxt_event_engine_t *engine, nxt_fd_event_t *ev, kev = nxt_kqueue_get_kevent(engine); + /* + * ->changing says a change for this event may be buffered. It is what + * the generic callers test before they ask for a cancel, so kqueue has + * to set it even though no kqueue path reads it -- nxt_kqueue_close() + * scans by descriptor, and nxt_kqueue_cancel_changes() scans by ->udata + * without testing the flag. + * + * A flush does not clear it. It could: ->filter tells an fd event from + * the file events nxt_kqueue_enable_file() puts in the same ->udata + * field, as nxt_kqueue_error() relies on. But clearing would walk the + * whole batch on every flush and every poll, to spare a scan that runs + * only when a port is released. So the flag only ever says "maybe", and + * the cost of that is a scan that finds nothing. It is never stale in + * the unsafe direction: it is set whenever a change is queued, and + * cleared only once the batch has been scanned. + */ + + ev->changing = 1; + kev->ident = ev->fd; kev->filter = filter; kev->flags = flags; @@ -990,6 +1060,15 @@ nxt_kqueue_conn_io_read(nxt_task_t *task, void *obj, void *data) nxt_debug(task, "kqueue conn read fd:%d", c->socket.fd); + /* + * As in nxt_conn_io_read(): a closed conn must not reach the EOF path. + * An EOF with socket.error set runs only the error path, not also the + * read state's close_handler. + */ + if (c->socket.error != 0 || c->block_read) { + return; + } + if (c->socket.kq_available == 0 && c->socket.kq_eof) { nxt_debug(task, "kevent fd:%d eof", c->socket.fd); diff --git a/src/nxt_lib.c b/src/nxt_lib.c index aaec1e8ea..c2bcaa46a 100644 --- a/src/nxt_lib.c +++ b/src/nxt_lib.c @@ -5,6 +5,7 @@ */ #include +#include nxt_uint_t nxt_ncpu = 1; @@ -15,6 +16,12 @@ nxt_atomic_t nxt_task_ident; nxt_thread_declare_data(nxt_thread_t, nxt_thread_context); +#if (NXT_LINUX) +static nxt_uint_t nxt_cgroup_cpu_limit(void); +static nxt_uint_t nxt_cgroup_cpu_max_read(const char *dir); +#endif + + #if (NXT_DEBUG && NXT_FREEBSD) /* * Fill memory with 0xA5 after malloc() and with 0x5A before free(). @@ -36,6 +43,9 @@ nxt_lib_start(const char *app, char **argv, char ***envp) nxt_int_t flags; nxt_bool_t update; nxt_thread_t *thread; +#if (NXT_LINUX) + nxt_uint_t limit; +#endif flags = nxt_stderr_start(); @@ -115,6 +125,25 @@ nxt_lib_start(const char *app, char **argv, char ***envp) #endif +#if (NXT_LINUX) + /* + * A CPU bandwidth limit (cgroup v2 "cpu.max") does not change the + * affinity mask. A container limited to 2 CPUs on a 64-CPU host + * otherwise starts 64 router threads. + */ + if (n > 1) { + limit = nxt_cgroup_cpu_limit(); + + if (limit != 0 && limit < (nxt_uint_t) n) { + nxt_log(&nxt_main_task, NXT_LOG_INFO, + "the cgroup CPU limit lowers the CPU count from %d to %ui", + n, limit); + + n = (int) limit; + } + } +#endif + nxt_debug(&nxt_main_task, "ncpu: %d", n); if (n > 1) { @@ -143,6 +172,335 @@ nxt_lib_start(const char *app, char **argv, char ***envp) } +/* + * Parses the contents of a cgroup v2 "cpu.max" file: " ", + * or "max " for no limit. Returns the quota in whole CPUs, rounded + * up, or 0 if there is no limit or the contents are not valid. + */ + +nxt_uint_t +nxt_cgroup_cpu_max_parse(const u_char *p, size_t len) +{ + size_t n; + nxt_off_t quota, period; + nxt_span_t span; + const u_char *sp, *field; + + /* A trailing newline or space is not part of the value. */ + + while (len != 0 && (p[len - 1] == '\n' || p[len - 1] == ' ')) { + len--; + } + + nxt_span_init(&span, p, p + len); + + /* The quota is the bytes before the first space. */ + + sp = memchr(p, ' ', len); + if (sp == NULL) { + return 0; + } + + n = sp - p; + + if (nxt_span_take(&span, n, &field) != 0) { + return 0; + } + + quota = nxt_off_t_parse(field, n); + + /* The space. */ + + if (nxt_span_take(&span, 1, &field) != 0) { + return 0; + } + + /* The period is the rest. */ + + n = nxt_span_len(&span); + + if (nxt_span_take(&span, n, &field) != 0) { + return 0; + } + + period = nxt_off_t_parse(field, n); + + if (quota <= 0 || period <= 0) { + return 0; + } + + /* Rounds up without an addition that could overflow. */ + quota = quota / period + (quota % period != 0); + + return (quota > (nxt_off_t) NXT_INT32_T_MAX) ? 0 : (nxt_uint_t) quota; +} + + +/* + * /proc/self/mountinfo writes a space, a tab, a newline and a backslash in + * a path as a backslash and three octal digits, such as "\040" for a space. + * Decodes these escapes in place. /proc/self/cgroup does not escape a + * path, so the decoded mount root can be compared with it. + */ + +void +nxt_cgroup_mountinfo_unescape(char *s) +{ + char *d; + + d = s; + + while (*s != '\0') { + + if (s[0] == '\\' + && s[1] >= '0' && s[1] <= '3' + && s[2] >= '0' && s[2] <= '7' + && s[3] >= '0' && s[3] <= '7') + { + *d++ = (char) (((s[1] - '0') << 6) | ((s[2] - '0') << 3) + | (s[3] - '0')); + s += 4; + continue; + } + + *d++ = *s++; + } + + *d = '\0'; +} + + +/* + * Returns the path of "cgroup" below the root "root" of the cgroup2 mount: + * "" for the root itself, else a path that starts with "/". Returns NULL + * if the cgroup is not below the root, or if its path has a ".." component. + * Such a cgroup is outside the cgroup namespace, and the mount does not + * show it. + */ + +const char * +nxt_cgroup_relative_path(const char *cgroup, const char *root) +{ + size_t len; + const char *rel, *p; + + rel = cgroup; + + if (strcmp(root, "/") != 0) { + len = strlen(root); + + if (strncmp(cgroup, root, len) != 0 + || (cgroup[len] != '/' && cgroup[len] != '\0')) + { + return NULL; + } + + rel = cgroup + len; + } + + /* Only a whole ".." component: a cgroup can be named "..foo". */ + + for (p = strstr(rel, "/.."); p != NULL; p = strstr(p + 1, "/..")) { + + if (p[3] == '/' || p[3] == '\0') { + return NULL; + } + } + + if (strcmp(rel, "/") == 0) { + return ""; + } + + return rel; +} + + +#if (NXT_LINUX) + +/* + * Returns the lowest CPU limit of the cgroup v2 directories from the + * cgroup of this process up to the root of the cgroup2 mount, or 0 if + * no level sets a limit. The root of the mount is the root of the cgroup + * namespace of the process. With a private cgroup namespace, as in a + * container, a limit on a parent of the namespace root is not seen. The + * cgroup v1 "cpu" controller is not read. + */ + +static nxt_uint_t +nxt_cgroup_cpu_limit(void) +{ + int i, len; + char *line, *p, *cgroup, *root, *mnt, *slash; + FILE *fp; + size_t size, base; + ssize_t nread; + nxt_uint_t limit, cpus; + const char *rel; + char dir[NXT_MAX_PATH_LEN]; + + line = NULL; + size = 0; + cgroup = NULL; + limit = 0; + + fp = fopen("/proc/self/cgroup", "re"); + if (fp == NULL) { + return 0; + } + + while ((nread = getline(&line, &size, fp)) > 0) { + if (nread > 3 && memcmp(line, "0::", 3) == 0) { + if (line[nread - 1] == '\n') { + line[nread - 1] = '\0'; + } + + cgroup = strdup(line + 3); + break; + } + } + + fclose(fp); + + if (cgroup == NULL) { + goto done; + } + + fp = fopen("/proc/self/mountinfo", "re"); + if (fp == NULL) { + goto done; + } + + root = NULL; + mnt = NULL; + + while (getline(&line, &size, fp) > 0) { + p = strstr(line, " - cgroup2 "); + if (p == NULL) { + continue; + } + + *p = '\0'; + + /* " ..." */ + + p = line; + + for (i = 0; i < 3 && p != NULL; i++) { + p = strchr(p, ' '); + + if (p != NULL) { + p++; + } + } + + if (p == NULL) { + continue; + } + + root = p; + + p = strchr(p, ' '); + if (p == NULL) { + continue; + } + + *p++ = '\0'; + mnt = p; + + p = strchr(p, ' '); + if (p != NULL) { + *p = '\0'; + } + + break; + } + + fclose(fp); + + if (mnt == NULL) { + goto done; + } + + nxt_cgroup_mountinfo_unescape(root); + nxt_cgroup_mountinfo_unescape(mnt); + + /* The cgroup path is relative to the root of the cgroup namespace. */ + + rel = nxt_cgroup_relative_path(cgroup, root); + if (rel == NULL) { + goto done; + } + + len = snprintf(dir, sizeof(dir), "%s%s", mnt, rel); + if (len < 0 || (size_t) len >= sizeof(dir)) { + goto done; + } + + base = strlen(mnt); + + for ( ;; ) { + cpus = nxt_cgroup_cpu_max_read(dir); + + if (cpus != 0 && (limit == 0 || cpus < limit)) { + limit = cpus; + } + + if ((size_t) len <= base) { + break; + } + + slash = strrchr(dir + base, '/'); + if (slash == NULL) { + break; + } + + *slash = '\0'; + len = slash - dir; + } + +done: + + nxt_free(cgroup); + nxt_free(line); + + return limit; +} + + +static nxt_uint_t +nxt_cgroup_cpu_max_read(const char *dir) +{ + int fd, len; + ssize_t n; + u_char buf[64]; + char path[NXT_MAX_PATH_LEN]; + + len = snprintf(path, sizeof(path), "%s/cpu.max", dir); + if (len < 0 || (size_t) len >= sizeof(path)) { + return 0; + } + + /* The root cgroup and a cgroup without the "cpu" controller have none. */ + + fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd == -1) { + return 0; + } + + n = read(fd, buf, sizeof(buf)); + + close(fd); + + if (n <= 0) { + return 0; + } + + return nxt_cgroup_cpu_max_parse(buf, n); +} + +#endif + + void nxt_lib_stop(void) { diff --git a/src/nxt_listen_socket.c b/src/nxt_listen_socket.c index 4fe3e20b6..4a562fda9 100644 --- a/src/nxt_listen_socket.c +++ b/src/nxt_listen_socket.c @@ -127,8 +127,6 @@ nxt_listen_socket_create(nxt_task_t *task, nxt_mp_t *mp, #if (NXT_HAVE_UNIX_DOMAIN) if (family == AF_UNIX) { - const char *user; - const char *group; nxt_runtime_t *rt = thr->runtime; name = (nxt_file_name_t *) sa->u.sockaddr_un.sun_path; @@ -138,10 +136,8 @@ nxt_listen_socket_create(nxt_task_t *task, nxt_mp_t *mp, goto listen_fail; } - user = rt->control_user; - group = rt->control_group; - - if (nxt_file_chown(name, user, group) != NXT_OK) { + /* The ids were resolved once, in nxt_runtime_controller_socket(). */ + if (nxt_file_chown(name, rt->control_uid, rt->control_gid) != NXT_OK) { goto listen_fail; } } diff --git a/src/nxt_main.h b/src/nxt_main.h index 7880e55f8..ba4057388 100644 --- a/src/nxt_main.h +++ b/src/nxt_main.h @@ -55,7 +55,6 @@ typedef uint16_t nxt_port_id_t; #include #include #include -#include #include #include #include @@ -83,7 +82,6 @@ typedef struct nxt_sockaddr_s nxt_sockaddr_t; #include #include #include -#include typedef void *(*nxt_mem_proto_alloc_t)(void *pool, size_t size); @@ -95,7 +93,6 @@ typedef struct { } nxt_mem_proto_t; -#include #include #include @@ -146,7 +143,6 @@ typedef void (*nxt_event_conn_handler_t)(nxt_thread_t *thr, nxt_conn_t *c); #include #include -#include #include @@ -161,6 +157,10 @@ typedef void (*nxt_event_conn_handler_t)(nxt_thread_t *thr, nxt_conn_t *c); */ NXT_EXPORT nxt_int_t nxt_lib_start(const char *app, char **argv, char ***envp); NXT_EXPORT void nxt_lib_stop(void); +NXT_EXPORT nxt_uint_t nxt_cgroup_cpu_max_parse(const u_char *p, size_t len); +NXT_EXPORT void nxt_cgroup_mountinfo_unescape(char *s); +NXT_EXPORT const char *nxt_cgroup_relative_path(const char *cgroup, + const char *root); NXT_EXPORT extern nxt_uint_t nxt_ncpu; diff --git a/src/nxt_main_process.c b/src/nxt_main_process.c index dca79e711..4d1dd3701 100644 --- a/src/nxt_main_process.c +++ b/src/nxt_main_process.c @@ -11,6 +11,7 @@ #include #include #include +#include #if (NXT_TLS) #include #endif @@ -19,6 +20,7 @@ #endif #include +#include typedef struct { @@ -35,6 +37,20 @@ typedef struct { } nxt_conf_app_map_t; +/* The state store runs in a short-lived child of main, one at a time. */ +typedef struct { + nxt_pid_t pid; /* The running store child, or 0. */ + nxt_bool_t version; /* It also stores the version file. */ + nxt_bool_t killed; /* Main sent it SIGKILL. */ + nxt_main_store_job_t *job; /* Its job, or NULL for conf.json. */ + u_char *pending; /* The next conf.json store, or NULL. */ + size_t pending_size; + nxt_uint_t ahead; /* Queued jobs older than "pending". */ + nxt_main_store_job_t *jobs; /* The jobs that wait, in order. */ + nxt_bool_t controller; /* Start it when the store ends. */ +} nxt_main_store_t; + + static nxt_int_t nxt_main_process_port_create(nxt_task_t *task, nxt_runtime_t *rt); static void nxt_main_process_title(nxt_task_t *task); @@ -60,13 +76,78 @@ static void nxt_main_port_modules_handler(nxt_task_t *task, static int nxt_cdecl nxt_app_lang_compare(const void *v1, const void *v2); static void nxt_main_process_whoami_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); +#if (NXT_USE_CMSG_PID) +static void nxt_main_process_name_child(nxt_task_t *task, + nxt_process_t *pprocess, nxt_process_t *process, nxt_pid_t ns_pid); +static void nxt_main_remove_child_pid_handler(nxt_task_t *task, + nxt_port_recv_msg_t *msg); +#endif static void nxt_main_port_conf_store_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); -static nxt_int_t nxt_main_file_store(nxt_task_t *task, const char *tmp_name, - const char *name, u_char *buf, size_t size); +static void nxt_main_store_schedule(nxt_task_t *task, u_char *p, + size_t size); +static void nxt_main_store_next(nxt_task_t *task); +static nxt_pid_t nxt_main_store_fork(nxt_task_t *task); +static void nxt_main_store_start(nxt_task_t *task, u_char *p, size_t size); +static void nxt_main_store_start_job(nxt_task_t *task, + nxt_main_store_job_t *job); +static nxt_int_t nxt_main_store_job_run(nxt_task_t *task, + nxt_main_store_job_t *job); +static void nxt_main_store_job_done(nxt_task_t *task, + nxt_main_store_job_t *job, nxt_int_t ret); +static void nxt_main_store_close_fds(void); +#if (NXT_LINUX) +static nxt_int_t nxt_main_store_close_proc_fds(void); +#endif +static nxt_int_t nxt_main_store_files(nxt_task_t *task, u_char *p, + size_t size, nxt_bool_t version); +static void nxt_main_start_exit(nxt_task_t *task); +static void nxt_main_store_cancel(nxt_task_t *task); +static nxt_bool_t nxt_main_store_cancelled(nxt_pid_t pid, int status); +static nxt_bool_t nxt_main_store_exited(nxt_task_t *task, nxt_pid_t pid, + int status); +static nxt_int_t nxt_main_file_store_inherit(nxt_task_t *task, + nxt_file_t *tmp, const char *name); static void nxt_main_port_access_log_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); +#if (NXT_TESTS) +static nxt_uint_t nxt_main_test_process_new_failure_count; +static nxt_msec_t nxt_main_test_store_delay; + + +void +nxt_main_test_process_new_failures(nxt_uint_t failures) +{ + nxt_main_test_process_new_failure_count = failures; +} + + +/* + * nxt_process_new() with an allocation-failure hook in front of it, in the + * shape of nxt_port_test_msg_alloc_failures() (src/nxt_port_socket.c): the + * count is decremented per call, so one armed failure fires once and the + * handler then behaves exactly as it does when the process record cannot + * be allocated. The wrapper stands in front of the call rather than in + * place of the branch that follows it, so the code the test reaches stays + * the handler's own -- however this file happens to spell its response to + * a NULL process. + */ +nxt_inline nxt_process_t * +nxt_main_process_new(nxt_runtime_t *rt) +{ + if (nxt_slow_path(nxt_main_test_process_new_failure_count != 0)) { + nxt_main_test_process_new_failure_count--; + return NULL; + } + + return nxt_process_new(rt); +} + +#else +#define nxt_main_process_new(rt) nxt_process_new(rt) +#endif + const nxt_sig_event_t nxt_main_process_signals[] = { nxt_event_signal(SIGHUP, nxt_main_process_signal_handler), nxt_event_signal(SIGINT, nxt_main_process_sigterm_handler), @@ -82,6 +163,8 @@ nxt_uint_t nxt_conf_ver; static nxt_bool_t nxt_exiting; +static nxt_main_store_t nxt_main_store; + nxt_int_t nxt_main_process_start(nxt_thread_t *thr, nxt_task_t *task, @@ -175,6 +258,12 @@ static nxt_conf_map_t nxt_common_app_limits_conf[] = { offsetof(nxt_common_app_conf_t, request_limit), }, + { + nxt_string("start_timeout"), + NXT_CONF_MAP_MSEC, + offsetof(nxt_common_app_conf_t, start_timeout), + }, + }; @@ -388,6 +477,11 @@ static nxt_conf_map_t nxt_wasm_wc_app_conf[] = { NXT_CONF_MAP_PTR, offsetof(nxt_common_app_conf_t, u.wasm_wc.access), }, + { + nxt_string("execution_timeout"), + NXT_CONF_MAP_MSEC, + offsetof(nxt_common_app_conf_t, u.wasm_wc.execution_timeout), + }, }; @@ -425,15 +519,34 @@ nxt_main_new_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) && port->type == NXT_PROCESS_APP && msg->fd[1] != -1) { - mem = nxt_mem_mmap(NULL, sizeof(nxt_port_queue_t), - PROT_READ | PROT_WRITE, MAP_SHARED, msg->fd[1], 0); - if (nxt_fast_path(mem != MAP_FAILED)) { + mem = nxt_port_queue_mmap(task, msg->fd[1], sizeof(nxt_port_queue_t)); + + if (nxt_fast_path(mem != NULL)) { port->queue = mem; - } - nxt_fd_close(msg->fd[1]); - msg->fd[1] = -1; + } else { + /* + * Not a fallback to the socket: a libunit process delivers a + * socket message only after dequeuing the READ_SOCKET marker + * that only a queue-holding sender emits, so everything main + * sends on this port -- CHANGE_FILE on log rotation, QUIT on + * shutdown -- would be suspended undelivered, and the second + * message fails the worker's context. See issue #231. + */ + nxt_alert(task, "cannot map the queue of port %PI:%d; the " + "process cannot be reached on this port from main", + port->pid, (int) port->id); + } } + + /* + * nxt_port_new_port_handler() leaves the queue descriptor to its caller, + * and only a new application port has a use for it here. Anything else + * -- a port that already existed, a port that could not be created, a + * type whose queue main never maps -- used to keep the descriptor open + * in the most privileged process of all. + */ + nxt_port_recv_msg_close_fds(msg); } @@ -454,28 +567,41 @@ nxt_main_start_process_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) rt = task->thread->runtime; + /* + * Every exit below the fork() is either the successful one, which lets + * the new process answer for itself, or "failed:", which answers for it. + * Nothing may leave this handler in between: START_PROCESS carries the + * stream of an RPC the router has already armed + * (nxt_router_start_app_process_handler()), and only a reply retires it. + * A silent return leaves that RPC outstanding forever -- neither + * nxt_router_app_port_ready() nor nxt_router_app_port_error() ever runs, + * so app->proto_port_requests is never cleared, every later start for + * the application parks on the prototype that is not coming, and the + * requests waiting on it are never failed. See issue #257. + */ + process = NULL; + port = rt->port_by_type[NXT_PROCESS_ROUTER]; if (nxt_slow_path(port == NULL)) { nxt_alert(task, "router port not found"); - goto close_fds; + goto failed; } if (nxt_slow_path(port->pid != nxt_recv_msg_cmsg_pid(msg))) { nxt_alert(task, "process %PI cannot start processes", nxt_recv_msg_cmsg_pid(msg)); - goto close_fds; + goto failed; } - process = nxt_process_new(rt); + process = nxt_main_process_new(rt); if (nxt_slow_path(process == NULL)) { - goto close_fds; + goto failed; } process->mem_pool = nxt_mp_create(1024, 128, 256, 32); if (process->mem_pool == NULL) { - nxt_process_use(task, process, -1); - goto close_fds; + goto failed; } process->parent_port = rt->port_by_type[NXT_PROCESS_MAIN]; @@ -603,17 +729,41 @@ nxt_main_start_process_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) failed: - nxt_process_use(task, process, -1); + if (process != NULL) { + nxt_process_use(task, process, -1); + } - port = nxt_runtime_port_find(rt, msg->port_msg.pid, + /* + * Answer on the port of the process the kernel says sent this, not the + * one the message claims to come from: msg->port_msg.pid is filled in by + * the sender (nxt_port_socket_write2()) and is not authenticated, and + * the two branches above now reach this reply before the router identity + * check has run -- or after it has failed. Keyed on the credential, the + * RPC_ERROR can only ever retire an RPC of the sender's own, so a worker + * that forges a START_PROCESS cannot use main to cancel a stream of the + * router's choosing. For a legitimate sender the two pids are equal, + * because nxt_port_socket_write2() sets port_msg.pid to its own nxt_pid; + * where SCM_CREDENTIALS is unavailable nxt_recv_msg_cmsg_pid() is + * defined as port_msg.pid, so the lookup is unchanged there. + */ + port = nxt_runtime_port_find(rt, nxt_recv_msg_cmsg_pid(msg), msg->port_msg.reply_port); if (nxt_fast_path(port != NULL)) { - nxt_port_socket_write(task, port, NXT_PORT_MSG_RPC_ERROR, - -1, msg->port_msg.stream, 0, NULL); - } + (void) nxt_port_socket_write(task, port, NXT_PORT_MSG_RPC_ERROR, + -1, msg->port_msg.stream, 0, NULL); -close_fds: + } else { + /* + * Nothing to answer on: the sender is gone, or never had the port it + * named. A dead sender's RPCs die with it, so this is only worth a + * diagnostic -- but a silent drop here is exactly the shape of the + * defect above, so it is not left silent. + */ + nxt_alert(task, "cannot report a failed start back: reply port %d of " + "process %PI not found", (int) msg->port_msg.reply_port, + nxt_recv_msg_cmsg_pid(msg)); + } nxt_fd_close(msg->fd[0]); msg->fd[0] = -1; @@ -623,6 +773,100 @@ nxt_main_start_process_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) } +#if (NXT_TESTS) + +/* + * Public wrapper that lets src/test/nxt_main_start_process_reply_test.c + * invoke the static nxt_main_start_process_handler() directly with a + * synthesised runtime and message -- used to verify that every exit above + * the fork() answers the router's START_PROCESS RPC instead of returning + * silently and stranding it (issue #257). + */ +void +nxt_main_test_run_start_process_handler(nxt_task_t *task, + nxt_port_recv_msg_t *msg) +{ + nxt_main_start_process_handler(task, msg); +} + + +/* Lets src/test/nxt_main_whoami_test.c drive the WHOAMI handler. */ +void +nxt_main_test_run_whoami_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) +{ + nxt_main_process_whoami_handler(task, msg); +} + + +/* + * Let src/test/nxt_main_store_test.c drive the state store. The delay + * makes each store child sleep before it stores, so that the test can act + * while the child runs. The test reaps the child with waitpid() and + * gives the status to nxt_main_test_store_exited(). + */ +void +nxt_main_test_store_set_delay(nxt_msec_t delay) +{ + nxt_main_test_store_delay = delay; +} + + +void +nxt_main_test_store_schedule(nxt_task_t *task, u_char *p, size_t size) +{ + nxt_main_store_schedule(task, p, size); +} + + +nxt_pid_t +nxt_main_test_store_pid(void) +{ + return nxt_main_store.pid; +} + + +nxt_bool_t +nxt_main_test_store_exited(nxt_task_t *task, nxt_pid_t pid, int status) +{ + return nxt_main_store_exited(task, pid, status); +} + + +/* Start the exit as the SIGTERM and SIGQUIT handlers do, or end it. */ +void +nxt_main_test_store_set_exiting(nxt_task_t *task, nxt_bool_t exiting) +{ + if (exiting) { + nxt_main_start_exit(task); + + } else { + nxt_exiting = 0; + } +} + + +#if (NXT_USE_CMSG_PID) + +void +nxt_main_test_run_name_child(nxt_task_t *task, nxt_process_t *pprocess, + nxt_process_t *process, nxt_pid_t ns_pid) +{ + nxt_main_process_name_child(task, pprocess, process, ns_pid); +} + + +void +nxt_main_test_run_remove_child_pid_handler(nxt_task_t *task, + nxt_port_recv_msg_t *msg) +{ + nxt_main_remove_child_pid_handler(task, msg); +} + +#endif + +#endif + + static void nxt_main_process_created_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) { @@ -699,6 +943,9 @@ static nxt_port_handlers_t nxt_main_process_port_handlers = { .process_ready = nxt_port_process_ready_handler, .whoami = nxt_main_process_whoami_handler, .remove_pid = nxt_port_remove_pid_handler, +#if (NXT_USE_CMSG_PID) + .remove_child_pid = nxt_main_remove_child_pid_handler, +#endif .start_process = nxt_main_start_process_handler, .socket = nxt_main_port_socket_handler, .socket_unlink = nxt_main_port_socket_unlink_handler, @@ -707,10 +954,12 @@ static nxt_port_handlers_t nxt_main_process_port_handlers = { #if (NXT_TLS) .cert_get = nxt_cert_store_get_handler, .cert_delete = nxt_cert_store_delete_handler, + .cert_store = nxt_cert_store_put_handler, #endif #if (NXT_HAVE_NJS) .script_get = nxt_script_store_get_handler, .script_delete = nxt_script_store_delete_handler, + .script_store = nxt_script_store_put_handler, #endif .access_log = nxt_main_port_access_log_handler, .rpc_ready = nxt_port_rpc_handler, @@ -718,6 +967,59 @@ static nxt_port_handlers_t nxt_main_process_port_handlers = { }; +#if (NXT_USE_CMSG_PID) + +/* + * Keep a child's pid in the pid namespace of the prototype that forked it. + * + * Under "isolation": {"namespaces": {"pid": true}} a worker has two pids: the + * global one from SCM_CREDENTIALS, which keys main's record, and the local one + * the prototype got from fork(). The WHOAMI header carries the local one, and + * it is the only name the prototype has for a worker that dies before + * PROCESS_CREATED. nxt_main_remove_child_pid_handler() resolves that name. + * + * The pair is kept even when the two numbers are equal: the counters are + * independent, so equality proves nothing. A name a live sibling still holds + * is refused, and the new worker stays unnamed. This happens without a + * forger too: the prototype forks a new worker with the local pid of one + * whose report main has not read yet. Either way the cost is the #310 leak + * for that one worker. Taking the name over would be worse: a late report + * for the old worker would then remove the live new one. + */ + +static void +nxt_main_process_name_child(nxt_task_t *task, nxt_process_t *pprocess, + nxt_process_t *process, nxt_pid_t ns_pid) +{ + nxt_process_t *child; + + /* 0 means "no name"; no namespace hands out 0 or a negative pid. */ + + if (nxt_slow_path(ns_pid <= 0)) { + return; + } + + nxt_queue_each(child, &pprocess->children, nxt_process_t, link) { + + if (child != process && child->parent_ns_pid == ns_pid) { + nxt_log(task, NXT_LOG_WARN, "pid %PI inside %PI is still held " + "by process %PI; process %PI stays unnamed", ns_pid, + pprocess->pid, child->pid, process->pid); + + return; + } + + } nxt_queue_loop; + + nxt_debug(task, "process %PI is pid %PI inside %PI", process->pid, + ns_pid, pprocess->pid); + + process->parent_ns_pid = ns_pid; +} + +#endif + + static void nxt_main_process_whoami_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) { @@ -740,6 +1042,14 @@ nxt_main_process_whoami_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) rt = task->thread->runtime; + /* + * Unreferenced: the main process runs a single engine. That matters + * here specifically because the result outlives the call -- it is + * linked into pprocess->children below, a weak link cleaned up in + * nxt_runtime_process_free(). With one engine nothing can drop the + * last reference concurrently, so the link cannot outlive the process. + */ + pprocess = nxt_runtime_process_find(rt, ppid); if (nxt_slow_path(pprocess == NULL)) { nxt_alert(task, "whoami: parent process %PI not found", ppid); @@ -751,6 +1061,41 @@ nxt_main_process_whoami_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) nxt_debug(task, "whoami: from %PI, parent %PI, fd %d", pid, ppid, msg->fd[0]); + /* + * The sender is not trusted, and "ppid" comes from its message. Only + * two shapes are valid: a process forked by main sends no descriptor, + * and a worker forked by a prototype sends its port. A process sends + * WHOAMI once, so a second one with a port is refused too; without + * that, it would get a second port and be linked into "children" + * twice, which corrupts the queue. + */ + + if (ppid == nxt_pid) { + if (nxt_slow_path(msg->fd[0] != -1)) { + nxt_alert(task, "whoami: process %PI sent a port", pid); + goto fail; + } + + } else { + if (nxt_slow_path(nxt_process_type(pprocess) + != NXT_PROCESS_PROTOTYPE)) + { + nxt_alert(task, "whoami: process %PI named %PI as its parent, " + "which is not a prototype", pid, ppid); + goto fail; + } + + if (nxt_slow_path(msg->fd[0] == -1)) { + nxt_alert(task, "whoami: worker %PI sent no port", pid); + goto fail; + } + + if (nxt_slow_path(nxt_runtime_port_find(rt, pid, 0) != NULL)) { + nxt_alert(task, "whoami: process %PI sent WHOAMI again", pid); + goto fail; + } + } + if (msg->fd[0] != -1) { port = nxt_runtime_process_port_create(task, rt, pid, 0, NXT_PROCESS_APP); @@ -778,7 +1123,15 @@ nxt_main_process_whoami_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) } if (ppid != nxt_pid) { + /* A new record: the checks above make it so. */ + nxt_assert(port->process->link.next == NULL); + nxt_queue_insert_tail(&pprocess->children, &port->process->link); + +#if (NXT_USE_CMSG_PID) + nxt_main_process_name_child(task, pprocess, port->process, + msg->port_msg.pid); +#endif } buf = nxt_buf_mem_alloc(task->thread->engine->mem_pool, @@ -789,16 +1142,126 @@ nxt_main_process_whoami_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) buf->mem.free = nxt_cpymem(buf->mem.free, &pid, sizeof(nxt_pid_t)); - (void) nxt_port_socket_write(task, port, NXT_PORT_MSG_RPC_READY_LAST, -1, - msg->port_msg.stream, 0, buf); + if (nxt_slow_path(nxt_port_socket_write(task, port, + NXT_PORT_MSG_RPC_READY_LAST, -1, + msg->port_msg.stream, 0, buf) + != NXT_OK)) + { + /* Still ours: the port layer takes the buffer only on NXT_OK. */ + + nxt_work_queue_add(&task->thread->engine->fast_work_queue, + buf->completion_handler, task, buf, buf->parent); + } fail: - if (msg->fd[0] != -1) { - nxt_fd_close(msg->fd[0]); + /* + * Close both descriptors: WHOAMI carries one, but a compromised sender + * can attach a second to any message, and leaving it open here would + * leak a descriptor of the main process on every forged message. + */ + nxt_port_recv_msg_close_fds(msg); +} + + +#if (NXT_USE_CMSG_PID) + +/* + * A prototype reports a worker of its own that died before PROCESS_CREATED, + * by the worker's namespace-local pid. REMOVE_PID cannot carry that pid: it + * is broadcast, and elsewhere the number names an unrelated process. + * + * The sender is the pid from SCM_CREDENTIALS, and the name is looked up among + * that sender's own children only, so a prototype can retire its own workers + * and nothing else. Main then tells the router by the global pid, as a + * REMOVE_PID would. Main cannot check that the worker is really dead, so a + * bad prototype can drop the record of a live worker of its own; it can do + * the same with REMOVE_PID outside a pid namespace. + * + * Without a sender credential the handler is not registered at all: a pid + * namespace needs Linux, which has SO_PASSCRED. + */ + +static void +nxt_main_remove_child_pid_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) +{ + size_t size; + nxt_buf_t *buf; + nxt_pid_t pid, sender; + nxt_runtime_t *rt; + nxt_process_t *pprocess, *child; + + buf = msg->buf; + size = (buf != NULL) ? (size_t) nxt_buf_used_size(buf) : 0; + + if (nxt_slow_path(size != sizeof(nxt_pid_t))) { + nxt_log(task, NXT_LOG_WARN, "REMOVE_CHILD_PID with a %uz byte " + "payload", size); + goto done; + } + + nxt_memcpy(&pid, buf->mem.pos, sizeof(nxt_pid_t)); + + /* 0 is how "no name" is stored, so it must not match anything. */ + + if (nxt_slow_path(pid <= 0)) { + nxt_log(task, NXT_LOG_WARN, "REMOVE_CHILD_PID naming pid %PI", pid); + goto done; + } + + sender = nxt_recv_msg_cmsg_pid(msg); + + rt = task->thread->runtime; + + pprocess = nxt_runtime_process_find(rt, sender); + + if (pprocess == NULL) { + /* The prototype may have exited before this was read. */ + nxt_debug(task, "REMOVE_CHILD_PID from unknown process %PI", sender); + goto done; } + + if (nxt_slow_path(nxt_process_type(pprocess) != NXT_PROCESS_PROTOTYPE)) { + nxt_alert(task, "process %PI is not a prototype and cannot report " + "a child pid", sender); + goto done; + } + + nxt_queue_each(child, &pprocess->children, nxt_process_t, link) { + + if (child->parent_ns_pid != pid) { + continue; + } + + nxt_debug(task, "remove child pid %PI (aka %PI) of %PI", pid, + child->pid, sender); + + /* As in nxt_main_process_sigchld_handler(). */ + + if (!nxt_exiting) { + nxt_port_remove_notify_others(task, child); + } + + nxt_process_unlink(child); + + nxt_process_close_ports(task, child); + + goto done; + + } nxt_queue_loop; + + /* Not an error: a worker that died before WHOAMI left no record. */ + + nxt_debug(task, "process %PI reported child pid %PI, which it has no " + "record for", sender, pid); + +done: + + nxt_port_recv_msg_close_fds(msg); } +#endif + static nxt_int_t nxt_main_process_port_create(nxt_task_t *task, nxt_runtime_t *rt) @@ -876,7 +1339,7 @@ nxt_main_process_sigterm_handler(nxt_task_t *task, void *obj, void *data) */ rt->quit_mode = NXT_PORT_QUIT_NORMAL; - nxt_exiting = 1; + nxt_main_start_exit(task); nxt_runtime_quit(task, 0); } @@ -898,7 +1361,7 @@ nxt_main_process_sigquit_handler(nxt_task_t *task, void *obj, void *data) */ rt->quit_mode = NXT_PORT_QUIT_GRACEFUL; - nxt_exiting = 1; + nxt_main_start_exit(task); nxt_runtime_quit(task, 0); } @@ -1045,7 +1508,12 @@ nxt_main_process_sigchld_handler(nxt_task_t *task, void *obj, void *data) return; } - if (WTERMSIG(status)) { + if (nxt_main_store_cancelled(pid, status)) { + /* Main stopped it; see nxt_main_store_cancel(). */ + nxt_trace(task, "process %PI exited on signal %d", + pid, WTERMSIG(status)); + + } else if (WTERMSIG(status)) { #ifdef WCOREDUMP nxt_alert(task, "process %PI exited on signal %d%s", pid, WTERMSIG(status), @@ -1060,15 +1528,35 @@ nxt_main_process_sigchld_handler(nxt_task_t *task, void *obj, void *data) pid, WEXITSTATUS(status)); } - process = nxt_runtime_process_find(rt, pid); + /* + * Unreferenced: the main process runs a single engine. That + * matters here specifically because nxt_process_close_ports() + * below takes its own +1/-1 around the port loop, which on a + * process already at zero would be a second drop to zero and so a + * second teardown -- the defect fixed in nxt_port_remove_pid(). + * With one engine, find() returning non-NULL implies use_count >= 1 + * for the whole handler. + */ - if (process != NULL) { - nxt_main_process_cleanup(task, process); + if (!nxt_main_store_exited(task, pid, status)) { + process = nxt_runtime_process_find(rt, pid); - if (process->state == NXT_PROCESS_STATE_READY) { - process->stream = 0; + if (process == NULL) { + continue; } + nxt_main_process_cleanup(task, process); + + /* + * ->stream is no longer cleared here. It is cleared where the + * start RPC is actually answered -- on a successful NEW_PORT + * announcement in nxt_port_process_ready_handler() -- which is + * both earlier and narrower: the READY state is set before that + * announcement is written, so clearing on the state dropped the + * REMOVE_PID fallback for a start whose reply never went out. + * See issue #271. + */ + nxt_queue_init(&children); if (!nxt_queue_is_empty(&process->children)) { @@ -1088,42 +1576,63 @@ nxt_main_process_sigchld_handler(nxt_task_t *task, void *obj, void *data) nxt_process_close_ports(task, process); nxt_queue_each(child, &children, nxt_process_t, link) { - nxt_queue_remove(&child->link); - child->link.next = NULL; + nxt_process_unlink(child); nxt_process_close_ports(task, child); } nxt_queue_loop; - if (rt->nprocesses <= 1) { - nxt_runtime_quit(task, 0); - - return; - } + } else { + nxt_port_remove_notify_others(task, process); - continue; - } + nxt_queue_each(child, &children, nxt_process_t, link) { + nxt_port_remove_notify_others(task, child); - nxt_port_remove_notify_others(task, process); + nxt_process_unlink(child); - nxt_queue_each(child, &children, nxt_process_t, link) { - nxt_port_remove_notify_others(task, child); + nxt_process_close_ports(task, child); + } nxt_queue_loop; - nxt_queue_remove(&child->link); - child->link.next = NULL; + init = *(nxt_process_init_t *) nxt_process_init(process); - nxt_process_close_ports(task, child); - } nxt_queue_loop; + nxt_process_close_ports(task, process); - init = *(nxt_process_init_t *) nxt_process_init(process); + if (init.restart) { + if (init.type == NXT_PROCESS_CONTROLLER + && nxt_main_store.pid != 0) + { + /* See nxt_main_store_exited(). */ + nxt_log(task, NXT_LOG_INFO, "controller starts when " + "state store child %PI ends", + nxt_main_store.pid); + + nxt_main_store.controller = 1; + continue; + } + + ret = nxt_process_init_start(task, init); + if (nxt_slow_path(ret == NXT_ERROR)) { + nxt_alert(task, "failed to restart %s", init.name); + } + } - nxt_process_close_ports(task, process); + continue; + } + } - if (init.restart) { - ret = nxt_process_init_start(task, init); - if (nxt_slow_path(ret == NXT_ERROR)) { - nxt_alert(task, "failed to restart %s", init.name); - } + /* + * At exit, main also waits for the store child. In a container + * main is PID 1, and when it exits the kernel kills the child. + */ + if (nxt_exiting && rt->nprocesses <= 1) { + if (nxt_main_store.pid != 0) { + nxt_debug(task, "waiting for state store child %PI", + nxt_main_store.pid); + continue; } + + nxt_runtime_quit(task, 0); + + return; } } } @@ -1573,11 +2082,15 @@ nxt_main_port_modules_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) conf = nxt_conf_json_parse(mp, b->mem.pos, b->mem.free, NULL); if (conf == NULL) { + nxt_alert(task, "discovery message is not valid JSON; " + "no application modules will be available"); goto fail; } root = nxt_conf_get_path(conf, &root_path); if (root == NULL) { + nxt_alert(task, "discovery message has no module list; " + "no application modules will be available"); goto fail; } @@ -1589,6 +2102,7 @@ nxt_main_port_modules_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) lang = nxt_array_zero_add(rt->languages); if (lang == NULL) { + nxt_alert(task, "failed to record the module at index %uD", index); goto fail; } @@ -1598,6 +2112,8 @@ nxt_main_port_modules_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) nxt_nitems(nxt_app_lang_module_map), lang); if (ret != NXT_OK) { + nxt_alert(task, "unexpected members in the module at index %uD", + index); goto fail; } @@ -1661,7 +2177,7 @@ nxt_main_port_modules_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) } if (nxt_slow_path(ret == NXT_ERROR)) { - nxt_exiting = 1; + nxt_main_start_exit(task); nxt_runtime_quit(task, 1); } @@ -1695,11 +2211,9 @@ static void nxt_main_port_conf_store_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) { void *p; - size_t n, size; - nxt_int_t ret; + size_t size; nxt_port_t *ctl_port; nxt_runtime_t *rt; - u_char ver[NXT_INT_T_LEN]; rt = task->thread->runtime; @@ -1754,29 +2268,15 @@ nxt_main_port_conf_store_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) nxt_debug(task, "conf_store_handler(%uz): %*s", size, size, p); - if (nxt_conf_ver != NXT_VERNUM) { - n = nxt_sprintf(ver, ver + NXT_INT_T_LEN, "%d", NXT_VERNUM) - ver; - - ret = nxt_main_file_store(task, rt->ver_tmp, rt->ver, ver, n); - if (nxt_slow_path(ret != NXT_OK)) { - goto error; - } - - nxt_conf_ver = NXT_VERNUM; - } + /* The scheduler owns the mapping from here on. */ + nxt_main_store_schedule(task, p, size); - ret = nxt_main_file_store(task, rt->conf_tmp, rt->conf, p, size); - - if (nxt_fast_path(ret == NXT_OK)) { - goto cleanup; - } + return; error: nxt_alert(task, "failed to store current configuration"); -cleanup: - if (p != MAP_FAILED) { nxt_mem_munmap(p, size); } @@ -1788,34 +2288,878 @@ nxt_main_port_conf_store_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) } -static nxt_int_t -nxt_main_file_store(nxt_task_t *task, const char *tmp_name, const char *name, - u_char *buf, size_t size) +/* + * A store runs fsync(2) twice for each file, which can take tens of + * milliseconds. A short-lived child does it, so main forks workers and + * handles signals in the meantime (issue #516). + * + * Only one store child runs at a time: two would race on the same + * temporary name. A conf.json store that arrives while a child runs + * becomes the pending store, and a newer store replaces it. Other stores + * are jobs (nxt_main_store_submit()). They wait in a queue, and each one + * runs. nxt_main_store_next() starts the next store when the child exits. + * + * The stores start in the order they came. A newer conf.json store takes + * the place of the pending store that it replaces. The order matters for + * a certificate DELETE: it must not run before an older conf.json store + * that stops naming the bundle. Otherwise a crash between the two leaves + * a conf.json that names a deleted bundle, and the next start refuses + * that configuration. + * + * "p" is a mapping of "size" bytes; this function takes ownership of it. + */ +static void +nxt_main_store_schedule(nxt_task_t *task, u_char *p, size_t size) { - ssize_t n; - nxt_int_t ret; - nxt_file_t file; + nxt_main_store_job_t *job; - nxt_memzero(&file, sizeof(nxt_file_t)); + if (nxt_main_store.pending != NULL) { + nxt_debug(task, "state store: a newer store replaces the pending one"); - file.name = (nxt_file_name_t *) name; + nxt_mem_munmap(nxt_main_store.pending, nxt_main_store.pending_size); - ret = nxt_file_open(task, &file, NXT_FILE_WRONLY, NXT_FILE_TRUNCATE, - NXT_FILE_OWNER_ACCESS); + } else { + /* The jobs that wait now came first. */ + nxt_main_store.ahead = 0; + + for (job = nxt_main_store.jobs; job != NULL; job = job->next) { + nxt_main_store.ahead++; + } + } + + nxt_main_store.pending = p; + nxt_main_store.pending_size = size; + + if (nxt_main_store.pid == 0) { + nxt_main_store_next(task); + return; + } + + if (nxt_exiting) { + nxt_main_store_cancel(task); + } +} + + +/* + * Main starts to exit. The SIGTERM and SIGQUIT handlers call this, and so + * does the start failure. The test hook nxt_main_test_store_set_exiting() + * calls it too, so src/test/nxt_main_store_test.c runs this code. + */ +static void +nxt_main_start_exit(nxt_task_t *task) +{ + nxt_exiting = 1; + + nxt_main_store_cancel(task); +} + + +/* + * At exit, main waits for the store child and then for the pending store. + * When a conf.json store is pending and a conf.json store child runs, that + * child stores a configuration that is no longer the last one, so main + * stops it with SIGKILL and waits for one store instead of two. The next + * store starts when the child is reaped, so two stores still do not + * overlap. Main never kills a job child: each job runs. + * + * Main does this only at exit. While main runs, the child finishes, and + * the pending store waits for it. A kill on each new store would let + * stores that come faster than one store takes keep conf.json old until + * they stop. + * + * A killed child leaves at most the temporary file, and the next store + * unlinks it first. If the child is killed after a rename(), the file it + * renamed is complete, because it was flushed before the rename(); the + * next store flushes the directory again. A killed child does not set + * nxt_conf_ver, so the next store writes the version file again. + */ +static void +nxt_main_store_cancel(nxt_task_t *task) +{ + if (nxt_main_store.pid == 0 + || nxt_main_store.job != NULL + || nxt_main_store.killed + || nxt_main_store.pending == NULL) + { + return; + } + + /* + * The child is not reaped until the SIGCHLD handler runs, so its pid + * cannot belong to another process yet. + */ + if (nxt_slow_path(kill(nxt_main_store.pid, SIGKILL) != 0)) { + nxt_alert(task, "kill(%PI, SIGKILL) failed %E", + nxt_main_store.pid, nxt_errno); + return; + } + + nxt_debug(task, "state store child %PI cancelled", nxt_main_store.pid); + + nxt_main_store.killed = 1; +} + + +/* + * Allocate a job that changes "". "dir" ends with "/", its + * "start" is NUL-terminated, and it must live as long as main, as + * rt->certs and rt->scripts do. A PUT job writes through the temporary + * ".store.tmp". One temporary name per directory is enough, because + * only one store runs at a time. So the caller must refuse a "name" that + * starts with ".". "what" names the kind of file in the alerts, for + * example "certificate". + * + * The caller then sets "data" and "size" of a PUT job (nxt_malloc()ed + * memory, which the job owns from here on), and "reply_pid", "reply_port" + * and "stream" when the sender waits for an answer. It gives the job to + * nxt_main_store_submit(), or frees it with nxt_main_store_job_free(). + */ +nxt_main_store_job_t * +nxt_main_store_job_create(nxt_main_store_op_t op, const char *what, + const nxt_str_t *dir, const nxt_str_t *name) +{ + u_char *p; + size_t size, path; + nxt_main_store_job_t *job; + + /* "" and its NUL. */ + if (nxt_size_add(dir->length, name->length, &path) != 0 + || nxt_size_add(path, 1, &path) != 0 + || nxt_size_add(sizeof(nxt_main_store_job_t), path, &size) != 0) + { + return NULL; + } + + /* ".store.tmp" and its NUL. */ + if (op == NXT_MAIN_STORE_PUT + && (nxt_size_add(size, dir->length, &size) != 0 + || nxt_size_add(size, sizeof(NXT_MAIN_STORE_TMP), &size) != 0)) + { + return NULL; + } + + job = nxt_malloc(size); + if (nxt_slow_path(job == NULL)) { + return NULL; + } + + nxt_memzero(job, sizeof(nxt_main_store_job_t)); + + job->op = op; + job->what = what; + job->dir = (const char *) dir->start; + + p = (u_char *) job + sizeof(nxt_main_store_job_t); + + job->name = (char *) p; + job->base = (char *) p + dir->length; + + p = nxt_cpymem(p, dir->start, dir->length); + p = nxt_cpymem(p, name->start, name->length); + *p++ = '\0'; + + if (op == NXT_MAIN_STORE_PUT) { + job->tmp = (char *) p; + + p = nxt_cpymem(p, dir->start, dir->length); + nxt_memcpy(p, NXT_MAIN_STORE_TMP, sizeof(NXT_MAIN_STORE_TMP)); + } + + return job; +} + + +void +nxt_main_store_job_free(nxt_main_store_job_t *job) +{ + nxt_free(job->data); + nxt_free(job); +} + + +/* + * Queue a job after the jobs that wait already. Main answers the sender + * when the job ends: RPC_READY_LAST when the child exited with 0, and + * RPC_ERROR otherwise. When fork() fails, main runs the job itself and + * answers at once. The job is freed after the answer. + */ +void +nxt_main_store_submit(nxt_task_t *task, nxt_main_store_job_t *job) +{ + nxt_main_store_job_t **last; + + job->next = NULL; + + for (last = &nxt_main_store.jobs; *last != NULL; last = &(*last)->next) { + /* void */ + } + + *last = job; + + if (nxt_main_store.pid == 0) { + nxt_main_store_next(task); + } +} + + +/* + * Start stores until a child runs or nothing waits, in the order they came: + * see nxt_main_store_schedule(). A store that cannot fork runs in main, + * and then the next one starts. + */ +static void +nxt_main_store_next(nxt_task_t *task) +{ + u_char *p; + size_t size; + nxt_main_store_job_t *job; + + while (nxt_main_store.pid == 0) { + job = nxt_main_store.jobs; + p = nxt_main_store.pending; + + if (job != NULL && (p == NULL || nxt_main_store.ahead != 0)) { + nxt_main_store.jobs = job->next; + job->next = NULL; + + if (nxt_main_store.ahead != 0) { + nxt_main_store.ahead--; + } + + nxt_main_store_start_job(task, job); + continue; + } + + if (p == NULL) { + return; + } + + size = nxt_main_store.pending_size; + + nxt_main_store.pending = NULL; + nxt_main_store.pending_size = 0; + + nxt_main_store_start(task, p, size); + } +} + + +/* + * Fork a store child. Returns the pid in main, 0 in the child and -1 when + * fork() fails. + * + * The signals of main stay blocked in the child, so SIGTERM or SIGINT sent + * to the process group does not stop the store. The child ends with + * _exit(), which runs no atexit() handler of main. + */ +static nxt_pid_t +nxt_main_store_fork(nxt_task_t *task) +{ + nxt_pid_t pid; + + pid = fork(); + + if (nxt_slow_path(pid < 0)) { + nxt_alert(task, "fork() failed for the state store %E", nxt_errno); + return -1; + } + + if (pid == 0) { + nxt_pid = getpid(); + task->thread->tid = 0; + + nxt_main_store_close_fds(); + +#if (NXT_TESTS) + if (nxt_main_test_store_delay != 0) { + nxt_nanosleep(nxt_main_test_store_delay * 1000000); + } +#endif + + return 0; + } + + nxt_debug(task, "state store child %PI", pid); + + nxt_main_store.pid = pid; + + return pid; +} + + +static void +nxt_main_store_start(nxt_task_t *task, u_char *p, size_t size) +{ + nxt_pid_t pid; + nxt_bool_t version; + + /* + * The version file is stored again with each store until a child that + * stored it exits with 0. Only then does main set nxt_conf_ver. + */ + version = (nxt_conf_ver != NXT_VERNUM); + + pid = nxt_main_store_fork(task); + + if (nxt_slow_path(pid < 0)) { + /* Do not lose the store: do it in main, as before the child. */ + if (nxt_main_store_files(task, p, size, version) == NXT_OK + && version) + { + nxt_conf_ver = NXT_VERNUM; + } + + nxt_mem_munmap(p, size); + return; + } + + if (pid == 0) { + if (nxt_main_store_files(task, p, size, version) != NXT_OK) { + _exit(1); + } + + _exit(0); + } + + /* The child has its own copy of the mapping. */ + nxt_mem_munmap(p, size); + + nxt_main_store.version = version; +} + + +static void +nxt_main_store_start_job(nxt_task_t *task, nxt_main_store_job_t *job) +{ + nxt_pid_t pid; + + pid = nxt_main_store_fork(task); + + if (nxt_slow_path(pid < 0)) { + nxt_main_store_job_done(task, job, nxt_main_store_job_run(task, job)); + return; + } + + if (pid == 0) { + if (nxt_main_store_job_run(task, job) != NXT_OK) { + _exit(1); + } + + _exit(0); + } + + /* The child has its own copy of the data. */ + nxt_free(job->data); + job->data = NULL; + + nxt_main_store.job = job; +} + + +static nxt_int_t +nxt_main_store_job_run(nxt_task_t *task, nxt_main_store_job_t *job) +{ + if (job->op == NXT_MAIN_STORE_PUT) { + return nxt_main_file_store(task, job->dir, job->tmp, job->name, + job->data, job->size); + } + + /* A file that is not there is deleted already. */ + if (unlink(job->name) != 0) { + if (nxt_errno == NXT_ENOENT) { + return NXT_OK; + } + + nxt_alert(task, "unlink(\"%s\") failed %E", job->name, nxt_errno); + + return NXT_ERROR; + } + + /* The file is gone; a failed flush only costs durability. */ + (void) nxt_file_dir_sync(task, (nxt_file_name_t *) job->dir); + + return NXT_OK; +} + + +/* + * Log a failed job, answer the sender and free the job. Main logs the + * failure: the alerts of the child can go to a rotated log, because + * SIGUSR1 reopens the log in main only. A sender that has gone has no + * port in the runtime any more, and gets no answer. + */ +static void +nxt_main_store_job_done(nxt_task_t *task, nxt_main_store_job_t *job, + nxt_int_t ret) +{ + nxt_port_t *port; + nxt_port_msg_type_t type; + + if (ret == NXT_OK) { + type = NXT_PORT_MSG_RPC_READY_LAST; + + } else { + type = NXT_PORT_MSG_RPC_ERROR; + + nxt_alert(task, "failed to %s %s \"%s\"", + (job->op == NXT_MAIN_STORE_PUT) ? "store" : "delete", + job->what, job->base); + } + + if (job->reply_pid != 0) { + port = nxt_runtime_port_find(task->thread->runtime, job->reply_pid, + job->reply_port); + + if (port != NULL) { + (void) nxt_port_socket_write(task, port, type, -1, job->stream, + 0, NULL); + } + } + + nxt_main_store_job_free(job); +} + + +/* + * The store child inherits every descriptor of main: the listen sockets, + * the ports and the log files. A copy of a listen socket keeps the address + * bound while the child runs, so a listener that the router closes cannot + * be bound again until the store ends. The child needs none of them. It + * logs to stderr (nxt_log_time_handler() writes to nxt_stderr, which + * nxt_runtime_log_files_create() redirects to the log file), and it opens + * the files it writes. So it closes every descriptor from 3 upwards. + * + * close_range() fails with ENOSYS on Linux before 5.9, and a seccomp + * profile can refuse it. Then, on Linux, the child closes the descriptors + * that /proc/self/fd lists. Only when that directory cannot be read does + * it close each number up to the limit: with a limit of 1048576 that loop + * took 862 to 892 ms per store. All of this runs in the child, not in + * main. + */ +static void +nxt_main_store_close_fds(void) +{ +#if (NXT_HAVE_CLOSEFROM && !NXT_HAVE_CLOSE_RANGE && !NXT_HAVE_SYS_CLOSE_RANGE) + + closefrom(3); + +#else + + long fd, max; + +#if (NXT_HAVE_CLOSE_RANGE) + + if (close_range(3, ~0U, 0) == 0) { + return; + } + +#elif (NXT_HAVE_SYS_CLOSE_RANGE) + + if (syscall(SYS_close_range, 3, ~0U, 0) == 0) { + return; + } + +#endif + +#if (NXT_LINUX) + + if (nxt_main_store_close_proc_fds() == NXT_OK) { + return; + } + +#endif + + max = sysconf(_SC_OPEN_MAX); + + if (max <= 0) { + max = 1024; + } + + for (fd = 3; fd < max; fd++) { + (void) close((int) fd); + } + +#endif +} + + +#if (NXT_LINUX) + +/* + * Close the descriptors from 3 upwards that /proc/self/fd lists, except + * the one that reads the directory. A close changes the directory while + * it is read, and readdir() can then skip an entry. So the scan runs + * again from the start until a scan closes nothing, as glibc does in + * __closefrom_fallback(). + */ +static nxt_int_t +nxt_main_store_close_proc_fds(void) +{ + int dfd; + DIR *dir; + nxt_int_t fd; + nxt_bool_t closed; + struct dirent *de; + + dir = opendir("/proc/self/fd"); + if (dir == NULL) { + return NXT_ERROR; + } + + dfd = dirfd(dir); + + do { + closed = 0; + + for ( ;; ) { + de = readdir(dir); + if (de == NULL) { + break; + } + + /* "." and ".." do not parse. */ + fd = nxt_int_parse((u_char *) de->d_name, + nxt_strlen(de->d_name)); + + if (fd < 3 || fd == dfd) { + continue; + } + + (void) close((int) fd); + closed = 1; + } + + rewinddir(dir); + + } while (closed); + + (void) closedir(dir); + + return NXT_OK; +} + +#endif + + +static nxt_int_t +nxt_main_store_files(nxt_task_t *task, u_char *p, size_t size, + nxt_bool_t version) +{ + size_t n; + nxt_int_t ret; + nxt_runtime_t *rt; + u_char ver[NXT_INT_T_LEN]; + + rt = task->thread->runtime; + + if (version) { + n = nxt_sprintf(ver, ver + NXT_INT_T_LEN, "%d", NXT_VERNUM) - ver; + + ret = nxt_main_file_store(task, rt->state, rt->ver_tmp, rt->ver, ver, n); + if (nxt_slow_path(ret != NXT_OK)) { + goto fail; + } + } + + ret = nxt_main_file_store(task, rt->state, rt->conf_tmp, rt->conf, p, size); + + if (nxt_fast_path(ret == NXT_OK)) { + return NXT_OK; + } + +fail: + + nxt_alert(task, "failed to store current configuration"); + + return NXT_ERROR; +} + + +/* Whether "pid" is the store child that main killed, and it died of that. */ +static nxt_bool_t +nxt_main_store_cancelled(nxt_pid_t pid, int status) +{ + return (pid == nxt_main_store.pid + && nxt_main_store.killed + && WIFSIGNALED(status) + && WTERMSIG(status) == SIGKILL); +} + + +/* + * Called by the SIGCHLD handler for every reaped pid. Returns 1 when "pid" + * was the store child. The child is not in the process registry and is not + * counted in rt->nprocesses. + * + * Main logs a failed store again: the alerts of the child can go to a + * rotated log, because SIGUSR1 reopens the log in main only. A child that + * main killed is not a failure: a newer store is pending. A child that + * exited before the signal arrived keeps its exit code. + * + * A controller that exits while a store child runs starts again here, when no + * store runs or waits. nxt_controller_prefork() reads conf.json, the + * certificate bundles and the njs modules from disk. Started at once, the new + * controller could read them before the child renames its file. It would then + * keep the old state, and main would send the answer of the store to the + * controller that has gone. Only the controller sends stores, so no store is + * added after main has reaped it. + */ +static nxt_bool_t +nxt_main_store_exited(nxt_task_t *task, nxt_pid_t pid, int status) +{ + nxt_int_t ret; + nxt_bool_t ok; + nxt_main_store_job_t *job; + + if (pid != nxt_main_store.pid) { + return 0; + } + + ok = (WIFEXITED(status) && WEXITSTATUS(status) == 0); + + job = nxt_main_store.job; + + if (job != NULL) { + nxt_main_store_job_done(task, job, ok ? NXT_OK : NXT_ERROR); + + } else if (ok) { + if (nxt_main_store.version) { + nxt_conf_ver = NXT_VERNUM; + } + + } else if (nxt_main_store_cancelled(pid, status)) { + nxt_debug(task, "state store child %PI stopped, a newer store follows", + pid); + + } else { + nxt_alert(task, "state store child %PI failed, " + "the configuration was not stored", pid); + } + + nxt_main_store.pid = 0; + nxt_main_store.killed = 0; + nxt_main_store.job = NULL; + + nxt_main_store_next(task); + + if (nxt_main_store.pid == 0 && nxt_main_store.controller) { + nxt_main_store.controller = 0; + + if (!nxt_exiting) { + ret = nxt_process_init_start(task, nxt_controller_process); + if (nxt_slow_path(ret == NXT_ERROR)) { + nxt_alert(task, "failed to restart controller"); + } + } + } + + return 1; +} + + +/* + * Replace "name" with "size" bytes of "buf" atomically: the content goes to + * "tmp_name", which the caller places in "dir" beside the destination, is + * flushed, and only then rename(2)d over "name", so a reader sees either + * the whole old file or the whole new one. "dir" is flushed after the + * rename, without which the rename may not survive a power loss. Every + * failure before the rename unlinks the temporary and leaves the existing + * "name" untouched. + * + * Because the replacement arrives by rename(), a "name" that was a symlink + * is replaced by a regular file rather than followed -- the price of + * atomicity. A symlinked state *directory* is unaffected: "dir" and + * "tmp_name" resolve through it just as "name" does. + * + * The temporary is created 0600 as before; when the destination already + * exists, its mode and ownership are carried over to the replacement, so a + * state file an administrator has re-permissioned keeps its settings. + */ +nxt_int_t +nxt_main_file_store(nxt_task_t *task, const char *dir, const char *tmp_name, + const char *name, u_char *buf, size_t size) +{ + size_t written; + ssize_t n; + nxt_int_t ret; + nxt_file_t file; + + nxt_memzero(&file, sizeof(nxt_file_t)); + + file.name = (nxt_file_name_t *) tmp_name; + + /* + * Without a log level nxt_file_open() reports nothing, and a failed + * store would be diagnosable only from the caller's summary alert. + * NXT_LOG_ALERT is zero, which is the "say nothing" value, so the + * loudest usable level here is NXT_LOG_ERR. + */ + file.log_level = NXT_LOG_ERR; + + /* + * Unlink first, then create exclusively, rather than opening the + * temporary with O_TRUNC. The name is predictable and the store runs + * as root, so if --statedir is writable by anyone else that user can + * put a symbolic link there ahead of us: O_TRUNC would follow it and + * truncate whatever it addresses, and the rename below would then + * install the link as the state file. O_EXCL refuses a link outright + * and refuses a regular file somebody hard-linked to a target we must + * not write, which O_NOFOLLOW alone would not. + * + * A leftover temporary from an interrupted store is still simply + * replaced -- that is what the unlink is for. The window between the + * two is not a hole: something recreated in it makes the create fail, + * which loses the store and keeps the target, rather than the other + * way round. + */ + if (nxt_slow_path(unlink(tmp_name) != 0 && nxt_errno != NXT_ENOENT)) { + nxt_alert(task, "unlink(\"%FN\") failed %E", file.name, nxt_errno); + + return NXT_ERROR; + } + + ret = nxt_file_open(task, &file, NXT_FILE_WRONLY, + NXT_FILE_CREATE_EXCLUSIVE, NXT_FILE_OWNER_ACCESS); if (nxt_slow_path(ret != NXT_OK)) { return NXT_ERROR; } - n = nxt_file_write(&file, buf, size, 0); + /* + * pwrite() is allowed to store less than it was asked for, and this + * store exists to survive a partial one: resume from where it stopped + * rather than turning a short write into a failed store. A zero return + * carries no errno and cannot make progress, so it ends the loop. + */ + for (written = 0; written < size; written += n) { + n = nxt_file_write(&file, buf + written, size - written, written); + + if (nxt_slow_path(n <= 0)) { + /* nxt_file_write() logs the errno; a zero return has none. */ + if (n == 0) { + nxt_alert(task, "write(\"%FN\") stored %uz of %uz bytes", + file.name, written, size); + } + + goto fail; + } + } + + if (nxt_slow_path(nxt_main_file_store_inherit(task, &file, name) + != NXT_OK)) + { + goto fail; + } + + /* + * rename() already orders the name change for readers; what this buys + * is the data reaching stable storage before the name points at it, so + * a crash cannot leave conf.json naming a file whose blocks were never + * written. + */ + if (nxt_slow_path(nxt_file_sync(task, &file) != NXT_OK)) { + goto fail; + } nxt_file_close(task, &file); + file.fd = NXT_FILE_INVALID; - if (nxt_slow_path(n != (ssize_t) size)) { + if (nxt_slow_path(nxt_file_rename(file.name, (nxt_file_name_t *) name) + != NXT_OK)) + { (void) nxt_file_delete(file.name); return NXT_ERROR; } - return nxt_file_rename(file.name, (nxt_file_name_t *) name); + /* The destination is in place; a failed flush only costs durability. */ + (void) nxt_file_dir_sync(task, (nxt_file_name_t *) dir); + + return NXT_OK; + +fail: + + if (file.fd != NXT_FILE_INVALID) { + nxt_file_close(task, &file); + } + + (void) nxt_file_delete(file.name); + + return NXT_ERROR; +} + + +/* + * Carry the destination's mode and ownership onto the temporary that is + * about to replace it. A destination that does not exist yet, or that is + * not a regular file, keeps the 0600 the temporary was created with. + */ +static nxt_int_t +nxt_main_file_store_inherit(nxt_task_t *task, nxt_file_t *tmp, + const char *name) +{ + nxt_file_info_t fi; + + /* + * lstat(), not nxt_file_info(), which stat()s a name and therefore + * follows a symbolic link. The temporary is created exclusively + * because --statedir may be writable by another user; that same user + * can point the destination at a file of their own, and a stat() here + * would copy its ownership and mode onto the replacement -- aim it at + * something 0666 and conf.json comes back world-writable, and stays + * that way, because every later store inherits it again. + * + * On the path this is for -- an administrator's re-permissioned state + * file -- lstat() and stat() agree. + */ + if (lstat(name, &fi) != 0) { + if (nxt_errno == NXT_ENOENT) { + return NXT_OK; + } + + /* + * The temporary is written and 0600 is a serviceable result, so + * an unreadable destination costs the inheritance and not the + * store: refusing to persist the configuration at all is the worse + * outcome, which is the same trade the fchown() below makes. + */ + nxt_log(task, NXT_LOG_INFO, "lstat(\"%FN\") failed %E", + (nxt_file_name_t *) name, nxt_errno); + + return NXT_OK; + } + + /* + * Only a regular file donates anything. A symbolic link, or a + * destination somebody replaced with a device or a directory, leaves + * the temporary on the 0600 it was created with; the rename replaces + * whatever is there either way. + */ + if (!S_ISREG(fi.st_mode)) { + return NXT_OK; + } + + /* + * Ownership first, mode second. A successful chown() clears set-user-ID + * and set-group-ID on a regular file -- Linux does it to root's chown() + * too -- so doing it the other way round would drop exactly the bits + * "fi.st_mode & 07777" is here to carry over. + * + * Main runs as root, so this normally succeeds; when it cannot change + * the owner the store still proceeds -- refusing to persist the + * configuration would be the worse failure. + */ + if (nxt_slow_path(fchown(tmp->fd, fi.st_uid, fi.st_gid) != 0)) { + nxt_log(task, NXT_LOG_INFO, "fchown(%FD, \"%FN\") failed %E", + tmp->fd, tmp->name, nxt_errno); + } + + if (nxt_slow_path(fchmod(tmp->fd, fi.st_mode & 07777) != 0)) { + nxt_alert(task, "fchmod(%FD, \"%FN\") failed %E", + tmp->fd, tmp->name, nxt_errno); + return NXT_ERROR; + } + + return NXT_OK; } diff --git a/src/nxt_main_process.h b/src/nxt_main_process.h index ef083d630..682aba9d9 100644 --- a/src/nxt_main_process.h +++ b/src/nxt_main_process.h @@ -21,6 +21,43 @@ typedef enum { nxt_int_t nxt_main_process_start(nxt_thread_t *thr, nxt_task_t *task, nxt_runtime_t *runtime); +nxt_int_t nxt_main_file_store(nxt_task_t *task, const char *dir, + const char *tmp_name, const char *name, u_char *buf, size_t size); + + +/* + * A change of a file in a directory of the state, run by the store child + * of main. The conf.json store is not a job. See nxt_main_store_submit(). + */ + +#define NXT_MAIN_STORE_TMP ".store.tmp" + +typedef enum { + NXT_MAIN_STORE_PUT = 0, + NXT_MAIN_STORE_DELETE, +} nxt_main_store_op_t; + +typedef struct nxt_main_store_job_s nxt_main_store_job_t; + +struct nxt_main_store_job_s { + nxt_main_store_job_t *next; + nxt_main_store_op_t op; + const char *what; /* For the alerts: "certificate". */ + const char *dir; /* Flushed after the change. */ + char *name; /* "". */ + const char *base; + char *tmp; /* PUT: ".store.tmp". */ + u_char *data; /* PUT: nxt_malloc()ed, owned. */ + size_t size; + nxt_pid_t reply_pid; /* 0: nobody waits for an answer. */ + nxt_port_id_t reply_port; + uint32_t stream; +}; + +nxt_main_store_job_t *nxt_main_store_job_create(nxt_main_store_op_t op, + const char *what, const nxt_str_t *dir, const nxt_str_t *name); +void nxt_main_store_job_free(nxt_main_store_job_t *job); +void nxt_main_store_submit(nxt_task_t *task, nxt_main_store_job_t *job); NXT_EXPORT extern nxt_uint_t nxt_conf_ver; @@ -33,5 +70,25 @@ NXT_EXPORT extern const nxt_process_init_t nxt_app_process; extern const nxt_sig_event_t nxt_main_process_signals[]; extern const nxt_sig_event_t nxt_process_signals[]; +#if (NXT_TESTS) +void nxt_main_test_process_new_failures(nxt_uint_t failures); +void nxt_main_test_run_start_process_handler(nxt_task_t *task, + nxt_port_recv_msg_t *msg); +void nxt_main_test_run_whoami_handler(nxt_task_t *task, + nxt_port_recv_msg_t *msg); +void nxt_main_test_store_set_delay(nxt_msec_t delay); +void nxt_main_test_store_schedule(nxt_task_t *task, u_char *p, size_t size); +nxt_pid_t nxt_main_test_store_pid(void); +nxt_bool_t nxt_main_test_store_exited(nxt_task_t *task, nxt_pid_t pid, + int status); +void nxt_main_test_store_set_exiting(nxt_task_t *task, nxt_bool_t exiting); +#if (NXT_USE_CMSG_PID) +void nxt_main_test_run_name_child(nxt_task_t *task, nxt_process_t *pprocess, + nxt_process_t *process, nxt_pid_t ns_pid); +void nxt_main_test_run_remove_child_pid_handler(nxt_task_t *task, + nxt_port_recv_msg_t *msg); +#endif +#endif + #endif /* _NXT_MAIN_PROCESS_H_INCLUDED_ */ diff --git a/src/nxt_mem_zone.c b/src/nxt_mem_zone.c deleted file mode 100644 index a3ba37007..000000000 --- a/src/nxt_mem_zone.c +++ /dev/null @@ -1,948 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) NGINX, Inc. - */ - -#include - - -#define NXT_MEM_ZONE_PAGE_FREE 0 -/* - * A page was never allocated before so it should be filled with - * junk on the first time allocation if memory debugging is enabled. - */ -#define NXT_MEM_ZONE_PAGE_FRESH 1 - -/* An entire page is currently used, no chunks inside the page. */ -#define NXT_MEM_ZONE_PAGE_USED 2 - - -typedef struct nxt_mem_zone_page_s nxt_mem_zone_page_t; - -struct nxt_mem_zone_page_s { - /* - * A size of page chunks if value is greater than or equal to 16. - * Otherwise it is used to mark page state: NXT_MEM_ZONE_PAGE_FREE, - * NXT_MEM_ZONE_PAGE_FRESH, and NXT_MEM_ZONE_PAGE_USED. - */ - uint16_t size; - - /* A number of free chunks of a chunked page. */ - uint16_t chunks; - - union { - /* A chunk bitmap if a number of chunks is lesser than 32. */ - uint8_t map[4]; - /* - * The count is a number of successive occupied pages in the first - * page. In the next occupied pages and in all free pages the count - * is zero, because a number of successive free pages is stored in - * free block size resided in beginning of the first free page. - */ - uint32_t count; - } u; - - /* Used for slot list of pages with free chunks. */ - nxt_mem_zone_page_t *next; - - /* - * Used to link of all pages including free, chunked and occupied - * pages to coalesce free pages. - */ - nxt_queue_link_t link; -}; - - -typedef struct { - uint32_t size; - uint32_t chunks; - uint32_t start; - uint32_t map_size; - nxt_mem_zone_page_t *pages; -} nxt_mem_zone_slot_t; - - -typedef struct { - NXT_RBTREE_NODE (node); - uint32_t size; -} nxt_mem_zone_free_block_t; - - -struct nxt_mem_zone_s { - nxt_thread_spinlock_t lock; - nxt_mem_zone_page_t *pages; - nxt_mem_zone_page_t sentinel_page; - nxt_rbtree_t free_pages; - - uint32_t page_size_shift; - uint32_t page_size_mask; - uint32_t max_chunk_size; - uint32_t small_bitmap_min_size; - - u_char *start; - u_char *end; - - nxt_mem_zone_slot_t slots[]; -}; - - -#define nxt_mem_zone_page_addr(zone, page) \ - (void *) (zone->start + ((page - zone->pages) << zone->page_size_shift)) - - -#define nxt_mem_zone_addr_page(zone, addr) \ - &zone->pages[((u_char *) addr - zone->start) >> zone->page_size_shift] - - -#define nxt_mem_zone_page_is_free(page) \ - (page->size < NXT_MEM_ZONE_PAGE_USED) - - -#define nxt_mem_zone_page_is_chunked(page) \ - (page->size >= 16) - - -#define nxt_mem_zone_page_bitmap(zone, slot) \ - (slot->size < zone->small_bitmap_min_size) - - -#define nxt_mem_zone_set_chunk_free(map, chunk) \ - map[chunk / 8] &= ~(0x80 >> (chunk & 7)) - - -#define nxt_mem_zone_chunk_is_free(map, chunk) \ - ((map[chunk / 8] & (0x80 >> (chunk & 7))) == 0) - - -#define nxt_mem_zone_fresh_junk(p, size) \ - nxt_memset((p), 0xA5, size) - - -#define nxt_mem_zone_free_junk(p, size) \ - nxt_memset((p), 0x5A, size) - - -static uint32_t nxt_mem_zone_pages(u_char *start, size_t zone_size, - nxt_uint_t page_size); -static void *nxt_mem_zone_slots_init(nxt_mem_zone_t *zone, - nxt_uint_t page_size); -static void nxt_mem_zone_slot_init(nxt_mem_zone_slot_t *slot, - nxt_uint_t page_size); -static intptr_t nxt_mem_zone_rbtree_compare(nxt_rbtree_node_t *node1, - nxt_rbtree_node_t *node2); -static void *nxt_mem_zone_alloc_small(nxt_mem_zone_t *zone, - nxt_mem_zone_slot_t *slot, size_t size); -static nxt_uint_t nxt_mem_zone_alloc_chunk(uint8_t *map, nxt_uint_t offset, - nxt_uint_t size); -static void *nxt_mem_zone_alloc_large(nxt_mem_zone_t *zone, size_t alignment, - size_t size); -static nxt_mem_zone_page_t *nxt_mem_zone_alloc_pages(nxt_mem_zone_t *zone, - size_t alignment, uint32_t pages); -static nxt_mem_zone_free_block_t * - nxt_mem_zone_find_free_block(nxt_mem_zone_t *zone, nxt_rbtree_node_t *node, - uint32_t alignment, uint32_t pages); -static const char *nxt_mem_zone_free_chunk(nxt_mem_zone_t *zone, - nxt_mem_zone_page_t *page, void *p); -static void nxt_mem_zone_free_pages(nxt_mem_zone_t *zone, - nxt_mem_zone_page_t *page, nxt_uint_t count); - - -static nxt_log_moderation_t nxt_mem_zone_log_moderation = { - NXT_LOG_ALERT, 2, "mem_zone_alloc() failed, not enough memory", - NXT_LOG_MODERATION -}; - - -nxt_mem_zone_t * -nxt_mem_zone_init(u_char *start, size_t zone_size, nxt_uint_t page_size) -{ - uint32_t pages; - nxt_uint_t n; - nxt_mem_zone_t *zone; - nxt_mem_zone_page_t *page; - nxt_mem_zone_free_block_t *block; - - if (nxt_slow_path((page_size & (page_size - 1)) != 0)) { - nxt_thread_log_alert("mem zone page size must be a power of 2"); - return NULL; - } - - pages = nxt_mem_zone_pages(start, zone_size, page_size); - if (pages == 0) { - return NULL; - } - - zone = (nxt_mem_zone_t *) start; - - /* The function returns address after all slots. */ - page = nxt_mem_zone_slots_init(zone, page_size); - - zone->pages = page; - - for (n = 0; n < pages; n++) { - page[n].size = NXT_MEM_ZONE_PAGE_FRESH; - } - - /* - * A special sentinel page entry marked as used does not correspond - * to a real page. The entry simplifies neighbour queue nodes check - * in nxt_mem_zone_free_pages(). - */ - zone->sentinel_page.size = NXT_MEM_ZONE_PAGE_USED; - nxt_queue_sentinel(&zone->sentinel_page.link); - nxt_queue_insert_after(&zone->sentinel_page.link, &page->link); - - /* rbtree of free pages. */ - - nxt_rbtree_init(&zone->free_pages, nxt_mem_zone_rbtree_compare); - - block = (nxt_mem_zone_free_block_t *) zone->start; - block->size = pages; - - nxt_rbtree_insert(&zone->free_pages, &block->node); - - return zone; -} - - -static uint32_t -nxt_mem_zone_pages(u_char *start, size_t zone_size, nxt_uint_t page_size) -{ - u_char *end; - size_t reserved; - nxt_uint_t n, pages, size, chunks, last; - nxt_mem_zone_t *zone; - - /* - * Find all maximum chunk sizes which zone page can be split on - * with minimum 16-byte step. - */ - last = page_size / 16; - n = 0; - size = 32; - - do { - chunks = page_size / size; - - if (last != chunks) { - last = chunks; - n++; - } - - size += 16; - - } while (chunks > 1); - - /* - * Find number of usable zone pages except zone bookkeeping data, - * slots, and pages entries. - */ - reserved = sizeof(nxt_mem_zone_t) + (n * sizeof(nxt_mem_zone_slot_t)); - - end = nxt_trunc_ptr(start + zone_size, page_size); - zone_size = end - start; - - pages = (zone_size - reserved) / (page_size + sizeof(nxt_mem_zone_page_t)); - - if (reserved > zone_size || pages == 0) { - nxt_thread_log_alert("mem zone size is too small: %uz", zone_size); - return 0; - } - - reserved += pages * sizeof(nxt_mem_zone_page_t); - nxt_memzero(start, reserved); - - zone = (nxt_mem_zone_t *) start; - - zone->start = nxt_align_ptr(start + reserved, page_size); - zone->end = end; - - nxt_thread_log_debug("mem zone pages: %uD, unused:%z", pages, - end - (zone->start + pages * page_size)); - - /* - * If a chunk size is lesser than zone->small_bitmap_min_size - * bytes, a page's chunk bitmap is larger than 32 bits and the - * bimap is placed at the start of the page. - */ - zone->small_bitmap_min_size = page_size / 32; - - zone->page_size_mask = page_size - 1; - zone->max_chunk_size = page_size / 2; - - n = zone->max_chunk_size; - - do { - zone->page_size_shift++; - n /= 2; - } while (n != 0); - - return (uint32_t) pages; -} - - -static void * -nxt_mem_zone_slots_init(nxt_mem_zone_t *zone, nxt_uint_t page_size) -{ - nxt_uint_t n, size, chunks; - nxt_mem_zone_slot_t *slot; - - slot = zone->slots; - - slot[0].chunks = page_size / 16; - slot[0].size = 16; - - n = 0; - size = 32; - - for ( ;; ) { - chunks = page_size / size; - - if (slot[n].chunks != chunks) { - - nxt_mem_zone_slot_init(&slot[n], page_size); - - nxt_thread_log_debug( - "mem zone size:%uD chunks:%uD start:%uD map:%uD", - slot[n].size, slot[n].chunks + 1, - slot[n].start, slot[n].map_size); - - n++; - - if (chunks == 1) { - return &slot[n]; - } - } - - slot[n].chunks = chunks; - slot[n].size = size; - size += 16; - } -} - - -static void -nxt_mem_zone_slot_init(nxt_mem_zone_slot_t *slot, nxt_uint_t page_size) -{ - /* - * Calculate number of bytes required to store a chunk bitmap - * and align it to 4 bytes. - */ - slot->map_size = nxt_align_size(((slot->chunks + 7) / 8), 4); - - /* If chunk size is not a multiple of zone page size, there - * is surplus space which can be used for the chunk's bitmap. - */ - slot->start = page_size - slot->chunks * slot->size; - - /* slot->chunks should be one less than actual number of chunks. */ - slot->chunks--; - - if (slot->map_size > 4) { - /* A page's chunks bitmap is placed at the start of the page. */ - - if (slot->start < slot->map_size) { - /* - * There is no surplus space or the space is too - * small for chunks bitmap, so use the first chunks. - */ - if (slot->size < slot->map_size) { - /* The first chunks are occupied by bitmap. */ - slot->chunks -= slot->map_size / slot->size; - slot->start = nxt_align_size(slot->map_size, 16); - - } else { - /* The first chunk is occupied by bitmap. */ - slot->chunks--; - slot->start = slot->size; - } - } - } -} - - -/* - * Round up to the next highest power of 2. The algorithm is - * described in "Bit Twiddling Hacks" by Sean Eron Anderson. - */ - -nxt_inline uint32_t -nxt_next_highest_power_of_two(uint32_t n) -{ - n--; - n |= n >> 1; - n |= n >> 2; - n |= n >> 4; - n |= n >> 8; - n |= n >> 16; - n++; - - return n; -} - - -static intptr_t -nxt_mem_zone_rbtree_compare(nxt_rbtree_node_t *node1, nxt_rbtree_node_t *node2) -{ - u_char *start1, *end1, *start2, *end2; - uint32_t n, size, size1, size2; - nxt_mem_zone_free_block_t *block1, *block2; - - block1 = (nxt_mem_zone_free_block_t *) node1; - block2 = (nxt_mem_zone_free_block_t *) node2; - - size1 = block1->size; - size2 = block2->size; - - /* - * This subtractions do not overflow if number of pages of a free - * block is below 2^31-1. This allows to use blocks up to 128G if - * a zone page size is just 64 bytes. - */ - n = size1 - size2; - - if (n != 0) { - return n; - } - - /* - * Sort equally sized blocks by their capability to allocate memory with - * alignment equal to the size rounded the previous higest power of 2. - */ - - /* Round the size to the previous higest power of two. */ - size = nxt_next_highest_power_of_two(size1) >> 1; - - /* Align the blocks' start and end to the rounded size. */ - start1 = nxt_align_ptr(block1, size); - end1 = nxt_trunc_ptr((u_char *) block1 + size1, size); - - start2 = nxt_align_ptr(block2, size); - end2 = nxt_trunc_ptr((u_char *) block2 + size2, size); - - return (end1 - start1) - (end2 - start2); -} - - -void * -nxt_mem_zone_zalloc(nxt_mem_zone_t *zone, size_t size) -{ - void *p; - - p = nxt_mem_zone_align(zone, 1, size); - - if (nxt_fast_path(p != NULL)) { - nxt_memzero(p, size); - } - - return p; -} - - -void * -nxt_mem_zone_align(nxt_mem_zone_t *zone, size_t alignment, size_t size) -{ - void *p; - nxt_mem_zone_slot_t *slot; - - if (nxt_slow_path((alignment - 1) & alignment) != 0) { - /* Alignment must be a power of 2. */ - return NULL; - } - - if (size <= zone->max_chunk_size && alignment <= zone->max_chunk_size) { - /* All chunks are aligned to 16. */ - - if (alignment > 16) { - /* - * Chunks which size is power of 2 are aligned to the size. - * So allocation size should be increased to the next highest - * power of two. This can waste memory, but a main consumer - * of aligned allocations is lvlhsh which anyway allocates - * memory with alignment equal to size. - */ - size = nxt_next_highest_power_of_two(size); - size = nxt_max(size, alignment); - } - - /* - * Find a zone slot with appropriate chunk size. - * This operation can be performed without holding lock. - */ - for (slot = zone->slots; slot->size < size; slot++) { /* void */ } - - nxt_thread_log_debug("mem zone alloc: @%uz:%uz chunk:%uD", - alignment, size, slot->size); - - nxt_thread_spin_lock(&zone->lock); - - p = nxt_mem_zone_alloc_small(zone, slot, size); - - } else { - - nxt_thread_log_debug("mem zone alloc: @%uz:%uz", alignment, size); - - nxt_thread_spin_lock(&zone->lock); - - p = nxt_mem_zone_alloc_large(zone, alignment, size); - } - - nxt_thread_spin_unlock(&zone->lock); - - if (nxt_fast_path(p != NULL)) { - nxt_thread_log_debug("mem zone alloc: %p", p); - - } else { - nxt_log_alert_moderate(&nxt_mem_zone_log_moderation, nxt_thread_log(), - "nxt_mem_zone_alloc(%uz, %uz) failed, not enough memory", - alignment, size); - } - - return p; -} - - -static void * -nxt_mem_zone_alloc_small(nxt_mem_zone_t *zone, nxt_mem_zone_slot_t *slot, - size_t size) -{ - u_char *p; - uint8_t *map; - nxt_mem_zone_page_t *page; - - page = slot->pages; - - if (nxt_fast_path(page != NULL)) { - - p = nxt_mem_zone_page_addr(zone, page); - - if (nxt_mem_zone_page_bitmap(zone, slot)) { - /* A page's chunks bitmap is placed at the start of the page. */ - map = p; - - } else { - map = page->u.map; - } - - p += nxt_mem_zone_alloc_chunk(map, slot->start, slot->size); - - page->chunks--; - - if (page->chunks == 0) { - /* - * Remove full page from the zone slot list of pages with - * free chunks. - */ - slot->pages = page->next; -#if (NXT_DEBUG) - page->next = NULL; -#endif - } - - return p; - } - - page = nxt_mem_zone_alloc_pages(zone, 1, 1); - - if (nxt_fast_path(page != NULL)) { - - slot->pages = page; - - page->size = slot->size; - /* slot->chunks are already one less. */ - page->chunks = slot->chunks; - page->u.count = 0; - page->next = NULL; - - p = nxt_mem_zone_page_addr(zone, page); - - if (nxt_mem_zone_page_bitmap(zone, slot)) { - /* A page's chunks bitmap is placed at the start of the page. */ - map = p; - nxt_memzero(map, slot->map_size); - - } else { - map = page->u.map; - } - - /* Mark the first chunk as busy. */ - map[0] = 0x80; - - return p + slot->start; - } - - return NULL; -} - - -static nxt_uint_t -nxt_mem_zone_alloc_chunk(uint8_t *map, nxt_uint_t offset, nxt_uint_t size) -{ - uint8_t mask; - nxt_uint_t n; - - n = 0; - - /* The page must have at least one free chunk. */ - - for ( ;; ) { - /* The bitmap is always aligned to uint32_t. */ - - if (*(uint32_t *) &map[n] != 0xFFFFFFFF) { - - do { - if (map[n] != 0xFF) { - - mask = 0x80; - - do { - if ((map[n] & mask) == 0) { - /* The free chunk is found. */ - map[n] |= mask; - return offset; - } - - offset += size; - mask >>= 1; - - } while (mask != 0); - - } else { - /* Fast-forward: all 8 chunks are occupied. */ - offset += size * 8; - } - - n++; - - } while (n % 4 != 0); - - } else { - /* Fast-forward: all 32 chunks are occupied. */ - offset += size * 32; - n += 4; - } - } -} - - -static void * -nxt_mem_zone_alloc_large(nxt_mem_zone_t *zone, size_t alignment, size_t size) -{ - uint32_t pages; - nxt_mem_zone_page_t *page; - - pages = (size + zone->page_size_mask) >> zone->page_size_shift; - - page = nxt_mem_zone_alloc_pages(zone, alignment, pages); - - if (nxt_fast_path(page != NULL)) { - return nxt_mem_zone_page_addr(zone, page); - } - - return NULL; -} - - -static nxt_mem_zone_page_t * -nxt_mem_zone_alloc_pages(nxt_mem_zone_t *zone, size_t alignment, uint32_t pages) -{ - u_char *p; - size_t prev_size; - uint32_t prev_pages, node_pages, next_pages; - nxt_uint_t n; - nxt_mem_zone_page_t *prev_page, *page, *next_page; - nxt_mem_zone_free_block_t *block, *next_block; - - block = nxt_mem_zone_find_free_block(zone, - nxt_rbtree_root(&zone->free_pages), - alignment, pages); - - if (nxt_slow_path(block == NULL)) { - return NULL; - } - - node_pages = block->size; - - nxt_rbtree_delete(&zone->free_pages, &block->node); - - p = nxt_align_ptr(block, alignment); - page = nxt_mem_zone_addr_page(zone, p); - - prev_size = p - (u_char *) block; - - if (prev_size != 0) { - prev_pages = prev_size >> zone->page_size_shift; - node_pages -= prev_pages; - - block->size = prev_pages; - nxt_rbtree_insert(&zone->free_pages, &block->node); - - prev_page = nxt_mem_zone_addr_page(zone, block); - nxt_queue_insert_after(&prev_page->link, &page->link); - } - - next_pages = node_pages - pages; - - if (next_pages != 0) { - next_page = &page[pages]; - next_block = nxt_mem_zone_page_addr(zone, next_page); - next_block->size = next_pages; - - nxt_rbtree_insert(&zone->free_pages, &next_block->node); - nxt_queue_insert_after(&page->link, &next_page->link); - } - - /* Go through pages after all rbtree operations to not trash CPU cache. */ - - page[0].u.count = pages; - - for (n = 0; n < pages; n++) { - - if (page[n].size == NXT_MEM_ZONE_PAGE_FRESH) { - nxt_mem_zone_fresh_junk(nxt_mem_zone_page_addr(zone, &page[n]), - zone->page_size_mask + 1); - } - - page[n].size = NXT_MEM_ZONE_PAGE_USED; - } - - return page; -} - - -/* - * Free blocks are sorted by size and then if the sizes are equal - * by aligned allocation capabilty. The former criterion is just - * comparison with a requested size and it can be used for iteractive - * search. The later criterion cannot be tested only by the requested - * size and alignment, so recursive in-order tree traversal is required - * to find a suitable free block. nxt_mem_zone_find_free_block() uses - * only recursive in-order tree traversal because anyway the slowest part - * of the algorithm are CPU cache misses. Besides the last tail recursive - * call may be optimized by compiler into iteractive search. - */ - -static nxt_mem_zone_free_block_t * -nxt_mem_zone_find_free_block(nxt_mem_zone_t *zone, nxt_rbtree_node_t *node, - uint32_t alignment, uint32_t pages) -{ - u_char *aligned, *end; - nxt_mem_zone_free_block_t *block, *free_block; - - if (node == nxt_rbtree_sentinel(&zone->free_pages)) { - return NULL; - } - - block = (nxt_mem_zone_free_block_t *) node; - - if (pages <= block->size) { - - free_block = nxt_mem_zone_find_free_block(zone, block->node.left, - alignment, pages); - if (free_block != NULL) { - return free_block; - } - - aligned = nxt_align_ptr(block, alignment); - - if (pages == block->size) { - if (aligned == (u_char *) block) { - /* Exact match. */ - return block; - } - - } else { /* pages < block->size */ - aligned += pages << zone->page_size_shift; - end = nxt_pointer_to(block, block->size << zone->page_size_shift); - - if (aligned <= end) { - return block; - } - } - } - - return nxt_mem_zone_find_free_block(zone, block->node.right, - alignment, pages); -} - - -void -nxt_mem_zone_free(nxt_mem_zone_t *zone, void *p) -{ - nxt_uint_t count; - const char *err; - nxt_mem_zone_page_t *page; - - nxt_thread_log_debug("mem zone free: %p", p); - - if (nxt_fast_path(zone->start <= (u_char *) p - && (u_char *) p < zone->end)) - { - page = nxt_mem_zone_addr_page(zone, p); - - nxt_thread_spin_lock(&zone->lock); - - if (nxt_mem_zone_page_is_chunked(page)) { - err = nxt_mem_zone_free_chunk(zone, page, p); - - } else if (nxt_slow_path(nxt_mem_zone_page_is_free(page))) { - err = "page is already free"; - - } else if (nxt_slow_path((uintptr_t) p & zone->page_size_mask) != 0) { - err = "invalid pointer to chunk"; - - } else { - count = page->u.count; - - if (nxt_fast_path(count != 0)) { - nxt_mem_zone_free_junk(p, count * zone->page_size_mask + 1); - nxt_mem_zone_free_pages(zone, page, count); - err = NULL; - - } else { - /* Not the first allocated page. */ - err = "pointer to wrong page"; - } - } - - nxt_thread_spin_unlock(&zone->lock); - - } else { - err = "pointer is out of zone"; - } - - if (nxt_slow_path(err != NULL)) { - nxt_thread_log_alert("nxt_mem_zone_free(%p): %s", p, err); - } -} - - -static const char * -nxt_mem_zone_free_chunk(nxt_mem_zone_t *zone, nxt_mem_zone_page_t *page, - void *p) -{ - u_char *map; - uint32_t size, offset, chunk; - nxt_mem_zone_page_t *pg, **ppg; - nxt_mem_zone_slot_t *slot; - - size = page->size; - - /* Find a zone slot with appropriate chunk size. */ - for (slot = zone->slots; slot->size < size; slot++) { /* void */ } - - offset = (uintptr_t) p & zone->page_size_mask; - offset -= slot->start; - - chunk = offset / size; - - if (nxt_slow_path(offset != chunk * size)) { - return "pointer to wrong chunk"; - } - - if (nxt_mem_zone_page_bitmap(zone, slot)) { - /* A page's chunks bitmap is placed at the start of the page. */ - map = (u_char *) ((uintptr_t) p & ~((uintptr_t) zone->page_size_mask)); - - } else { - map = page->u.map; - } - - if (nxt_mem_zone_chunk_is_free(map, chunk)) { - return "chunk is already free"; - } - - nxt_mem_zone_set_chunk_free(map, chunk); - - nxt_mem_zone_free_junk(p, page->size); - - if (page->chunks == 0) { - page->chunks = 1; - - /* Add the page to the head of slot list of pages with free chunks. */ - page->next = slot->pages; - slot->pages = page; - - } else if (page->chunks != slot->chunks) { - page->chunks++; - - } else { - - if (map != page->u.map) { - nxt_mem_zone_free_junk(map, slot->map_size); - } - - /* - * All chunks are free, remove the page from the slot list of pages - * with free chunks and add the page to the free pages tree. - */ - ppg = &slot->pages; - - for (pg = slot->pages; pg != NULL; pg = pg->next) { - - if (pg == page) { - *ppg = page->next; - break; - } - - ppg = &pg->next; - } - - nxt_mem_zone_free_pages(zone, page, 1); - } - - return NULL; -} - - -static void -nxt_mem_zone_free_pages(nxt_mem_zone_t *zone, nxt_mem_zone_page_t *page, - nxt_uint_t count) -{ - nxt_mem_zone_page_t *prev_page, *next_page; - nxt_mem_zone_free_block_t *block, *prev_block, *next_block; - - page->size = NXT_MEM_ZONE_PAGE_FREE; - page->chunks = 0; - page->u.count = 0; - page->next = NULL; - - nxt_memzero(&page[1], (count - 1) * sizeof(nxt_mem_zone_page_t)); - - next_page = nxt_queue_link_data(page->link.next, nxt_mem_zone_page_t, link); - - if (nxt_mem_zone_page_is_free(next_page)) { - - /* Coalesce with the next free pages. */ - - nxt_queue_remove(&next_page->link); - nxt_memzero(next_page, sizeof(nxt_mem_zone_page_t)); - - next_block = nxt_mem_zone_page_addr(zone, next_page); - count += next_block->size; - nxt_rbtree_delete(&zone->free_pages, &next_block->node); - } - - prev_page = nxt_queue_link_data(page->link.prev, nxt_mem_zone_page_t, link); - - if (nxt_mem_zone_page_is_free(prev_page)) { - - /* Coalesce with the previous free pages. */ - - nxt_queue_remove(&page->link); - - prev_block = nxt_mem_zone_page_addr(zone, prev_page); - count += prev_block->size; - nxt_rbtree_delete(&zone->free_pages, &prev_block->node); - - prev_block->size = count; - nxt_rbtree_insert(&zone->free_pages, &prev_block->node); - - return; - } - - block = nxt_mem_zone_page_addr(zone, page); - block->size = count; - nxt_rbtree_insert(&zone->free_pages, &block->node); -} diff --git a/src/nxt_mem_zone.h b/src/nxt_mem_zone.h deleted file mode 100644 index 89d73ca22..000000000 --- a/src/nxt_mem_zone.h +++ /dev/null @@ -1,28 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) NGINX, Inc. - */ - -#ifndef _NXT_MEM_ZONE_H_INCLUDED_ -#define _NXT_MEM_ZONE_H_INCLUDED_ - - -typedef struct nxt_mem_zone_s nxt_mem_zone_t; - - -NXT_EXPORT nxt_mem_zone_t *nxt_mem_zone_init(u_char *start, size_t zone_size, - nxt_uint_t page_size); - -#define nxt_mem_zone_alloc(zone, size) \ - nxt_mem_zone_align((zone), 1, (size)) - -NXT_EXPORT void *nxt_mem_zone_align(nxt_mem_zone_t *zone, size_t alignment, - size_t size) - NXT_MALLOC_LIKE; -NXT_EXPORT void *nxt_mem_zone_zalloc(nxt_mem_zone_t *zone, size_t size) - NXT_MALLOC_LIKE; -NXT_EXPORT void nxt_mem_zone_free(nxt_mem_zone_t *zone, void *p); - - -#endif /* _NXT_MEM_ZONE_H_INCLUDED_ */ diff --git a/src/nxt_murmur_hash.c b/src/nxt_murmur_hash.c index 56bf2d4b6..434b2a0a8 100644 --- a/src/nxt_murmur_hash.c +++ b/src/nxt_murmur_hash.c @@ -37,10 +37,10 @@ nxt_murmur_hash2(const void *data, size_t len) switch (len) { case 3: h ^= p[2] << 16; - /* Fall through. */ + nxt_fallthrough; case 2: h ^= p[1] << 8; - /* Fall through. */ + nxt_fallthrough; case 1: h ^= p[0]; h *= m; diff --git a/src/nxt_nncq.h b/src/nxt_nncq.h index 6c9ab3262..28585b227 100644 --- a/src/nxt_nncq.h +++ b/src/nxt_nncq.h @@ -11,6 +11,14 @@ #define NXT_NNCQ_SIZE 16384 +/* + * The peer maps the queue writable and can put a foreign cycle under head + * or tail. Each loop below retries at most this many times, so the peer + * cannot make this side spin for ever. Legitimate contention needs far + * fewer: each retry means another party made progress. + */ +#define NXT_NNCQ_RETRIES NXT_NNCQ_SIZE + typedef uint32_t nxt_nncq_atomic_t; typedef uint16_t nxt_nncq_cycle_t; @@ -94,13 +102,13 @@ nxt_nncq_init(nxt_nncq_t volatile *q) } -static inline void +static inline nxt_int_t nxt_nncq_enqueue(nxt_nncq_t volatile *q, nxt_nncq_atomic_t val) { nxt_nncq_cycle_t e_cycle, t_cycle; - nxt_nncq_atomic_t n, t, e, j; + nxt_nncq_atomic_t n, t, e, j, r; - for ( ;; ) { + for (r = 0; r < NXT_NNCQ_RETRIES; r++) { t = nxt_nncq_tail(q); j = nxt_nncq_map(q, t); e = q->entries[j]; @@ -120,11 +128,13 @@ nxt_nncq_enqueue(nxt_nncq_t volatile *q, nxt_nncq_atomic_t val) n = nxt_nncq_new_entry(q, t_cycle, val); if (nxt_atomic_cmp_set(&q->entries[j], e, n)) { - break; + nxt_nncq_tail_cmp_inc(q, t); + + return NXT_OK; } } - nxt_nncq_tail_cmp_inc(q, t); + return NXT_ERROR; } @@ -132,9 +142,9 @@ static inline nxt_nncq_atomic_t nxt_nncq_dequeue(nxt_nncq_t volatile *q) { nxt_nncq_cycle_t e_cycle, h_cycle; - nxt_nncq_atomic_t h, j, e; + nxt_nncq_atomic_t h, j, e, r; - for ( ;; ) { + for (r = 0; r < NXT_NNCQ_RETRIES; r++) { h = nxt_nncq_head(q); j = nxt_nncq_map(q, h); e = q->entries[j]; @@ -151,11 +161,11 @@ nxt_nncq_dequeue(nxt_nncq_t volatile *q) } if (nxt_atomic_cmp_set(&q->head, h, h + 1)) { - break; + return nxt_nncq_index(q, e); } } - return nxt_nncq_index(q, e); + return nxt_nncq_empty(q); } diff --git a/src/nxt_openssl.c b/src/nxt_openssl.c index 7902640da..88d4b8ff8 100644 --- a/src/nxt_openssl.c +++ b/src/nxt_openssl.c @@ -17,6 +17,22 @@ #include #include +/* + * The X509 name accessors were constified in stages, and not together: + * X509_NAME_get_entry() and X509_NAME_ENTRY_get_data() took const arguments + * from 1.1.0, but X509_NAME_get_index_by_NID() only from 3.0, and it is 4.0 + * that constified the values all three return. Qualifying from 3.0 is what + * satisfies every version at once -- an older header takes these pointers + * non-const, and 4.0 hands them back const -- so the boundary is 3.0 rather + * than the 1.1.0 the other guards in this file use. + */ + +#if OPENSSL_VERSION_NUMBER >= 0x30000000L +#define NXT_X509_CONST const +#else +#define NXT_X509_CONST +#endif + typedef struct { SSL *session; @@ -31,6 +47,38 @@ typedef struct { } nxt_openssl_conn_t; +#if (NXT_HAVE_OPENSSL_TLSEXT) + +/* + * OpenSSL 3.0 deprecated the HMAC_CTX flavour of the session ticket key + * callback in favour of SSL_CTX_set_tlsext_ticket_key_evp_cb(), which hands + * the callback an EVP_MAC_CTX instead. The deprecation covers the + * SSL_CTRL_SET_TLSEXT_TICKET_KEY_CB control behind the old registration + * macro as well, so the feature guard has to follow the callback flavour: + * once OpenSSL 5.0 drops the deprecated symbols, testing for the control + * would silently turn Session Tickets off. + */ + +#if OPENSSL_VERSION_NUMBER >= 0x30000000L + +#include +#include + +#define NXT_OPENSSL_TICKET_KEY_CB 1 +typedef EVP_MAC_CTX nxt_tls_ticket_mac_t; + +#elif defined(SSL_CTRL_SET_TLSEXT_TICKET_KEY_CB) + +#include + +#define NXT_OPENSSL_TICKET_KEY_CB 1 +typedef HMAC_CTX nxt_tls_ticket_mac_t; + +#endif + +#endif /* NXT_HAVE_OPENSSL_TLSEXT */ + + struct nxt_tls_ticket_s { u_char name[16]; u_char hmac_key[32]; @@ -55,12 +103,6 @@ typedef enum { static nxt_int_t nxt_openssl_library_init(nxt_task_t *task); static void nxt_openssl_library_free(nxt_task_t *task); -#if OPENSSL_VERSION_NUMBER < 0x10100004L -static nxt_int_t nxt_openssl_locks_init(void); -static void nxt_openssl_lock(int mode, int type, const char *file, int line); -static unsigned long nxt_openssl_thread_id(void); -static void nxt_openssl_locks_free(void); -#endif static nxt_int_t nxt_openssl_server_init(nxt_task_t *task, nxt_mp_t *mp, nxt_tls_init_t *tls_init, nxt_bool_t last); static nxt_int_t nxt_openssl_chain_file(nxt_task_t *task, SSL_CTX *ctx, @@ -72,13 +114,18 @@ static nxt_int_t nxt_ssl_conf_commands(nxt_task_t *task, SSL_CTX *ctx, #if (NXT_HAVE_OPENSSL_TLSEXT) static nxt_int_t nxt_tls_ticket_keys(nxt_task_t *task, SSL_CTX *ctx, nxt_tls_init_t *tls_init, nxt_mp_t *mp); +#ifdef NXT_OPENSSL_TICKET_KEY_CB static int nxt_tls_ticket_key_callback(SSL *s, unsigned char *name, - unsigned char *iv, EVP_CIPHER_CTX *ectx, HMAC_CTX *hctx, int enc); + unsigned char *iv, EVP_CIPHER_CTX *ectx, nxt_tls_ticket_mac_t *hctx, + int enc); +#endif #endif static void nxt_ssl_session_cache(SSL_CTX *ctx, size_t cache_size, time_t timeout); static nxt_uint_t nxt_openssl_cert_get_names(nxt_task_t *task, X509 *cert, nxt_tls_conf_t *conf, nxt_mp_t *mp); +static nxt_int_t nxt_openssl_name_text(NXT_X509_CONST X509_NAME *x509_name, + int nid, nxt_str_t *str); static nxt_int_t nxt_openssl_bundle_hash_test(nxt_lvlhsh_query_t *lhq, void *data); static nxt_int_t nxt_openssl_bundle_hash_insert(nxt_task_t *task, @@ -114,6 +161,11 @@ const nxt_tls_lib_t nxt_openssl_lib = { }; +#if (NXT_TESTS) +nxt_uint_t nxt_openssl_test_fail; +#endif + + static nxt_conn_io_t nxt_openssl_conn_io = { .read = nxt_conn_io_read, .recvbuf = nxt_openssl_conn_io_recvbuf, @@ -138,55 +190,13 @@ nxt_openssl_library_init(nxt_task_t *task) return NXT_OK; } -#if OPENSSL_VERSION_NUMBER >= 0x10100003L - OPENSSL_init_ssl(OPENSSL_INIT_LOAD_CONFIG, NULL); -#else - { - nxt_int_t ret; - - SSL_load_error_strings(); - - OPENSSL_config(NULL); - - /* - * SSL_library_init(3): - * - * SSL_library_init() always returns "1", - * so it is safe to discard the return value. - */ - (void) SSL_library_init(); - - ret = nxt_openssl_locks_init(); - if (nxt_slow_path(ret != NXT_OK)) { - return ret; - } - } - -#endif - nxt_openssl_version = OpenSSL_version_num(); nxt_log(task, NXT_LOG_INFO, "%s, %xl", OpenSSL_version(OPENSSL_VERSION), nxt_openssl_version); -#ifndef SSL_OP_NO_COMPRESSION - { - /* - * Disable gzip compression in OpenSSL prior to 1.0.0 - * version, this saves about 522K per connection. - */ - int n; - STACK_OF(SSL_COMP) *ssl_comp_methods; - - ssl_comp_methods = SSL_COMP_get_compression_methods(); - - for (n = sk_SSL_COMP_num(ssl_comp_methods); n != 0; n--) { - (void) sk_SSL_COMP_pop(ssl_comp_methods); - } - } -#endif index = SSL_get_ex_new_index(0, NULL, NULL, NULL, NULL); @@ -202,93 +212,11 @@ nxt_openssl_library_init(nxt_task_t *task) } -#if OPENSSL_VERSION_NUMBER >= 0x10100003L - static void nxt_openssl_library_free(nxt_task_t *task) { } -#else - -static nxt_thread_mutex_t *nxt_openssl_locks; - -static nxt_int_t -nxt_openssl_locks_init(void) -{ - int i, n; - nxt_int_t ret; - - n = CRYPTO_num_locks(); - - nxt_openssl_locks = OPENSSL_malloc(n * sizeof(nxt_thread_mutex_t)); - if (nxt_slow_path(nxt_openssl_locks == NULL)) { - return NXT_ERROR; - } - - for (i = 0; i < n; i++) { - ret = nxt_thread_mutex_create(&nxt_openssl_locks[i]); - if (nxt_slow_path(ret != NXT_OK)) { - return ret; - } - } - - CRYPTO_set_locking_callback(nxt_openssl_lock); - - CRYPTO_set_id_callback(nxt_openssl_thread_id); - - return NXT_OK; -} - - -static void -nxt_openssl_lock(int mode, int type, const char *file, int line) -{ - nxt_thread_mutex_t *lock; - - lock = &nxt_openssl_locks[type]; - - if ((mode & CRYPTO_LOCK) != 0) { - (void) nxt_thread_mutex_lock(lock); - - } else { - (void) nxt_thread_mutex_unlock(lock); - } -} - - -static u_long -nxt_openssl_thread_id(void) -{ - return (u_long) nxt_thread_handle(); -} - - -static void -nxt_openssl_library_free(nxt_task_t *task) -{ - nxt_openssl_locks_free(); -} - - -static void -nxt_openssl_locks_free(void) -{ - int i, n; - - n = CRYPTO_num_locks(); - - CRYPTO_set_locking_callback(NULL); - - for (i = 0; i < n; i++) { - nxt_thread_mutex_destroy(&nxt_openssl_locks[i]); - } - - OPENSSL_free(nxt_openssl_locks); -} - -#endif - static nxt_int_t nxt_openssl_server_init(nxt_task_t *task, nxt_mp_t *mp, @@ -300,16 +228,32 @@ nxt_openssl_server_init(nxt_task_t *task, nxt_mp_t *mp, STACK_OF(X509_NAME) *list; nxt_tls_bundle_conf_t *bundle; + conf = tls_init->conf; + + bundle = conf->bundle; + nxt_assert(bundle != NULL); + + /* nxt_router_conf_error() frees the context of every bundle. */ + bundle->ctx = NULL; + ctx = SSL_CTX_new(SSLv23_server_method()); + +#if (NXT_TESTS) + if (nxt_openssl_test_fail == NXT_OPENSSL_TEST_FAIL_CTX_NEW) { + SSL_CTX_free(ctx); + ctx = NULL; + } +#endif + if (ctx == NULL) { nxt_openssl_log_error(task, NXT_LOG_ALERT, "SSL_CTX_new() failed"); - return NXT_ERROR; - } - conf = tls_init->conf; + /* nxt_openssl_chain_file() does not run, so close the file here. */ + nxt_fd_close(bundle->chain_file); + bundle->chain_file = -1; - bundle = conf->bundle; - nxt_assert(bundle != NULL); + return NXT_ERROR; + } bundle->ctx = ctx; @@ -437,6 +381,9 @@ nxt_openssl_server_init(nxt_task_t *task, nxt_mp_t *mp, SSL_CTX_free(ctx); + /* nxt_router_conf_error() frees the contexts of the other bundles. */ + bundle->ctx = NULL; + #if (OPENSSL_VERSION_NUMBER >= 0x1010100fL \ && OPENSSL_VERSION_NUMBER < 0x1010101fL) RAND_keep_random_devices_open(0); @@ -460,14 +407,28 @@ nxt_openssl_chain_file(nxt_task_t *task, SSL_CTX *ctx, nxt_tls_conf_t *conf, ret = NXT_ERROR; cert = NULL; + bundle = conf->bundle; + bio = BIO_new(BIO_s_fd()); + +#if (NXT_TESTS) + if (nxt_openssl_test_fail == NXT_OPENSSL_TEST_FAIL_BIO_NEW) { + BIO_free(bio); + bio = NULL; + } +#endif + if (bio == NULL) { + /* No BIO owns the file yet, so close it here. */ + nxt_fd_close(bundle->chain_file); + bundle->chain_file = -1; + goto end; } - bundle = conf->bundle; - + /* The BIO owns the file now: BIO_free() below closes it. */ BIO_set_fd(bio, bundle->chain_file, BIO_CLOSE); + bundle->chain_file = -1; cert = PEM_read_bio_X509_AUX(bio, NULL, NULL, NULL); if (cert == NULL) { @@ -660,7 +621,7 @@ nxt_tls_ticket_keys(nxt_task_t *task, SSL_CTX *ctx, nxt_tls_init_t *tls_init, goto no_ticket; } -#ifdef SSL_CTRL_SET_TLSEXT_TICKET_KEY_CB +#ifdef NXT_OPENSSL_TICKET_KEY_CB tickets = nxt_mp_get(mp, sizeof(nxt_tls_tickets_t) + count * sizeof(nxt_tls_ticket_t)); @@ -701,8 +662,13 @@ nxt_tls_ticket_keys(nxt_task_t *task, SSL_CTX *ctx, nxt_tls_init_t *tls_init, } while (i < count); +#if OPENSSL_VERSION_NUMBER >= 0x30000000L + if (SSL_CTX_set_tlsext_ticket_key_evp_cb(ctx, nxt_tls_ticket_key_callback) + == 0) +#else if (SSL_CTX_set_tlsext_ticket_key_cb(ctx, nxt_tls_ticket_key_callback) == 0) +#endif { nxt_openssl_log_error(task, NXT_LOG_ALERT, "Unit was built with Session Tickets support, however, " @@ -731,11 +697,11 @@ nxt_tls_ticket_keys(nxt_task_t *task, SSL_CTX *ctx, nxt_tls_init_t *tls_init, } -#ifdef SSL_CTRL_SET_TLSEXT_TICKET_KEY_CB +#ifdef NXT_OPENSSL_TICKET_KEY_CB static int nxt_tls_ticket_key_callback(SSL *s, unsigned char *name, unsigned char *iv, - EVP_CIPHER_CTX *ectx, HMAC_CTX *hctx, int enc) + EVP_CIPHER_CTX *ectx, nxt_tls_ticket_mac_t *hctx, int enc) { nxt_uint_t i; nxt_conn_t *c; @@ -743,6 +709,9 @@ nxt_tls_ticket_key_callback(SSL *s, unsigned char *name, unsigned char *iv, const EVP_CIPHER *cipher; nxt_tls_ticket_t *ticket; nxt_openssl_conn_t *tls; +#if OPENSSL_VERSION_NUMBER >= 0x30000000L + OSSL_PARAM params[2]; +#endif c = SSL_get_ex_data(s, nxt_openssl_connection_index); @@ -815,6 +784,28 @@ nxt_tls_ticket_key_callback(SSL *s, unsigned char *name, unsigned char *iv, digest = EVP_sha256(); #endif +#if OPENSSL_VERSION_NUMBER >= 0x30000000L + + /* + * The EVP_MAC_CTX handed to this callback is an already created HMAC + * context; it only needs the digest and the key. EVP_MAC_init() takes + * both, so the key material, its length and the digest are exactly the + * ones HMAC_Init_ex() used to be given, and the tag length still follows + * from the digest. + */ + + params[0] = OSSL_PARAM_construct_utf8_string(OSSL_MAC_PARAM_DIGEST, + (char *) EVP_MD_get0_name(digest), 0); + params[1] = OSSL_PARAM_construct_end(); + + if (EVP_MAC_init(hctx, ticket[i].hmac_key, ticket[i].size, params) != 1) { + nxt_openssl_log_error(c->socket.task, NXT_LOG_ALERT, + "EVP_MAC_init() failed"); + return -1; + } + +#else + if (HMAC_Init_ex(hctx, ticket[i].hmac_key, ticket[i].size, digest, NULL) != 1) { @@ -823,10 +814,12 @@ nxt_tls_ticket_key_callback(SSL *s, unsigned char *name, unsigned char *iv, return -1; } +#endif + return enc; } -#endif /* SSL_CTRL_SET_TLSEXT_TICKET_KEY_CB */ +#endif /* NXT_OPENSSL_TICKET_KEY_CB */ #endif /* NXT_HAVE_OPENSSL_TLSEXT */ @@ -851,9 +844,8 @@ static nxt_uint_t nxt_openssl_cert_get_names(nxt_task_t *task, X509 *cert, nxt_tls_conf_t *conf, nxt_mp_t *mp) { - int len; nxt_str_t domain, str; - X509_NAME *x509_name; + NXT_X509_CONST X509_NAME *x509_name; nxt_uint_t i, n; GENERAL_NAME *name; nxt_tls_bundle_conf_t *bundle; @@ -875,11 +867,7 @@ nxt_openssl_cert_get_names(nxt_task_t *task, X509 *cert, nxt_tls_conf_t *conf, } str.length = ASN1_STRING_length(name->d.dNSName); -#if OPENSSL_VERSION_NUMBER > 0x10100000L str.start = (u_char *) ASN1_STRING_get0_data(name->d.dNSName); -#else - str.start = ASN1_STRING_data(name->d.dNSName); -#endif domain.start = nxt_mp_nget(mp, str.length); if (nxt_slow_path(domain.start == NULL)) { @@ -909,23 +897,22 @@ nxt_openssl_cert_get_names(nxt_task_t *task, X509 *cert, nxt_tls_conf_t *conf, } else { x509_name = X509_get_subject_name(cert); - len = X509_NAME_get_text_by_NID(x509_name, NID_commonName, - NULL, 0); - if (len <= 0) { + + if (nxt_openssl_name_text(x509_name, NID_commonName, &str) != NXT_OK + || str.length == 0) + { nxt_log(task, NXT_LOG_WARN, "certificate \"%V\" has neither " "Subject Alternative Name nor Common Name", &bundle->name); return NXT_OK; } - domain.start = nxt_mp_nget(mp, len + 1); + domain.start = nxt_mp_nget(mp, str.length); if (nxt_slow_path(domain.start == NULL)) { return NXT_ERROR; } - domain.length = X509_NAME_get_text_by_NID(x509_name, NID_commonName, - (char *) domain.start, - len + 1); - nxt_memcpy_lowcase(domain.start, domain.start, domain.length); + domain.length = str.length; + nxt_memcpy_lowcase(domain.start, str.start, str.length); item = nxt_mp_get(mp, sizeof(nxt_tls_bundle_hash_item_t)); if (nxt_slow_path(item == NULL)) { @@ -953,6 +940,53 @@ nxt_openssl_cert_get_names(nxt_task_t *task, X509 *cert, nxt_tls_conf_t *conf, } +/* + * Return the raw contents of the first entry of x509_name that carries nid, + * or NXT_DECLINED if there is none. + * + * This replaces X509_NAME_get_text_by_NID(), deprecated in OpenSSL 4.0. The + * only property of that function the callers ever used was "the bytes of the + * first matching entry, verbatim", which is what the entry accessors give + * directly and without the copy: the returned string points into the + * certificate and stays valid for as long as it does. + * + * The bytes are not NUL terminated and, exactly as before, may contain + * embedded NULs -- the callers must keep handling them with an explicit + * length and never as a C string. + */ + +static nxt_int_t +nxt_openssl_name_text(NXT_X509_CONST X509_NAME *x509_name, int nid, + nxt_str_t *str) +{ + int i, len; + NXT_X509_CONST ASN1_STRING *data; + NXT_X509_CONST X509_NAME_ENTRY *entry; + + i = X509_NAME_get_index_by_NID(x509_name, nid, -1); + if (i < 0) { + return NXT_DECLINED; + } + + entry = X509_NAME_get_entry(x509_name, i); + if (nxt_slow_path(entry == NULL)) { + return NXT_DECLINED; + } + + data = X509_NAME_ENTRY_get_data(entry); + + len = ASN1_STRING_length(data); + if (nxt_slow_path(len < 0)) { + return NXT_DECLINED; + } + + str->length = len; + str->start = (u_char *) ASN1_STRING_get0_data(data); + + return NXT_OK; +} + + static const nxt_lvlhsh_proto_t nxt_openssl_bundle_hash_proto nxt_aligned(64) = { @@ -1675,6 +1709,53 @@ nxt_openssl_conn_error(nxt_task_t *task, nxt_err_t err, const char *fmt, ...) } +/* + * OpenSSL 4.0 renamed the SSLv3-era alert reason codes to SSL_R_TLS_ALERT_* + * and removed the old spellings, so a build against it with deprecated APIs + * disabled cannot see SSL_R_SSLV3_ALERT_*. 1.1.1 and 3.x have only the old + * names. Define the new spelling in terms of the old one where it is + * missing, so the switch below needs one spelling rather than a version + * conditional per label. + * + * The SSL_R_TLSV1_ALERT_* codes are deliberately not bridged: 4.0 kept those + * names, and no SSL_R_TLS_ALERT_ spelling exists for them. + */ + +#ifndef SSL_R_TLS_ALERT_UNEXPECTED_MESSAGE +#define SSL_R_TLS_ALERT_UNEXPECTED_MESSAGE SSL_R_SSLV3_ALERT_UNEXPECTED_MESSAGE +#endif +#ifndef SSL_R_TLS_ALERT_BAD_RECORD_MAC +#define SSL_R_TLS_ALERT_BAD_RECORD_MAC SSL_R_SSLV3_ALERT_BAD_RECORD_MAC +#endif +#ifndef SSL_R_TLS_ALERT_DECOMPRESSION_FAILURE +#define SSL_R_TLS_ALERT_DECOMPRESSION_FAILURE SSL_R_SSLV3_ALERT_DECOMPRESSION_FAILURE +#endif +#ifndef SSL_R_TLS_ALERT_HANDSHAKE_FAILURE +#define SSL_R_TLS_ALERT_HANDSHAKE_FAILURE SSL_R_SSLV3_ALERT_HANDSHAKE_FAILURE +#endif +#ifndef SSL_R_TLS_ALERT_ILLEGAL_PARAMETER +#define SSL_R_TLS_ALERT_ILLEGAL_PARAMETER SSL_R_SSLV3_ALERT_ILLEGAL_PARAMETER +#endif +#ifndef SSL_R_TLS_ALERT_NO_CERTIFICATE +#define SSL_R_TLS_ALERT_NO_CERTIFICATE SSL_R_SSLV3_ALERT_NO_CERTIFICATE +#endif +#ifndef SSL_R_TLS_ALERT_BAD_CERTIFICATE +#define SSL_R_TLS_ALERT_BAD_CERTIFICATE SSL_R_SSLV3_ALERT_BAD_CERTIFICATE +#endif +#ifndef SSL_R_TLS_ALERT_UNSUPPORTED_CERTIFICATE +#define SSL_R_TLS_ALERT_UNSUPPORTED_CERTIFICATE SSL_R_SSLV3_ALERT_UNSUPPORTED_CERTIFICATE +#endif +#ifndef SSL_R_TLS_ALERT_CERTIFICATE_REVOKED +#define SSL_R_TLS_ALERT_CERTIFICATE_REVOKED SSL_R_SSLV3_ALERT_CERTIFICATE_REVOKED +#endif +#ifndef SSL_R_TLS_ALERT_CERTIFICATE_EXPIRED +#define SSL_R_TLS_ALERT_CERTIFICATE_EXPIRED SSL_R_SSLV3_ALERT_CERTIFICATE_EXPIRED +#endif +#ifndef SSL_R_TLS_ALERT_CERTIFICATE_UNKNOWN +#define SSL_R_TLS_ALERT_CERTIFICATE_UNKNOWN SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN +#endif + + static nxt_uint_t nxt_openssl_log_error_level(nxt_err_t err) { @@ -1732,21 +1813,21 @@ nxt_openssl_log_error_level(nxt_err_t err) case SSL_R_SCSV_RECEIVED_WHEN_RENEGOTIATING: /* 345 */ #endif case 1000:/* SSL_R_SSLV3_ALERT_CLOSE_NOTIFY */ - case SSL_R_SSLV3_ALERT_UNEXPECTED_MESSAGE: /* 1010 */ - case SSL_R_SSLV3_ALERT_BAD_RECORD_MAC: /* 1020 */ + case SSL_R_TLS_ALERT_UNEXPECTED_MESSAGE: /* 1010 */ + case SSL_R_TLS_ALERT_BAD_RECORD_MAC: /* 1020 */ case SSL_R_TLSV1_ALERT_DECRYPTION_FAILED: /* 1021 */ case SSL_R_TLSV1_ALERT_RECORD_OVERFLOW: /* 1022 */ - case SSL_R_SSLV3_ALERT_DECOMPRESSION_FAILURE: /* 1030 */ - case SSL_R_SSLV3_ALERT_HANDSHAKE_FAILURE: /* 1040 */ - case SSL_R_SSLV3_ALERT_ILLEGAL_PARAMETER: /* 1047 */ + case SSL_R_TLS_ALERT_DECOMPRESSION_FAILURE: /* 1030 */ + case SSL_R_TLS_ALERT_HANDSHAKE_FAILURE: /* 1040 */ + case SSL_R_TLS_ALERT_ILLEGAL_PARAMETER: /* 1047 */ break; - case SSL_R_SSLV3_ALERT_NO_CERTIFICATE: /* 1041 */ - case SSL_R_SSLV3_ALERT_BAD_CERTIFICATE: /* 1042 */ - case SSL_R_SSLV3_ALERT_UNSUPPORTED_CERTIFICATE: /* 1043 */ - case SSL_R_SSLV3_ALERT_CERTIFICATE_REVOKED: /* 1044 */ - case SSL_R_SSLV3_ALERT_CERTIFICATE_EXPIRED: /* 1045 */ - case SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN: /* 1046 */ + case SSL_R_TLS_ALERT_NO_CERTIFICATE: /* 1041 */ + case SSL_R_TLS_ALERT_BAD_CERTIFICATE: /* 1042 */ + case SSL_R_TLS_ALERT_UNSUPPORTED_CERTIFICATE: /* 1043 */ + case SSL_R_TLS_ALERT_CERTIFICATE_REVOKED: /* 1044 */ + case SSL_R_TLS_ALERT_CERTIFICATE_EXPIRED: /* 1045 */ + case SSL_R_TLS_ALERT_CERTIFICATE_UNKNOWN: /* 1046 */ case SSL_R_TLSV1_ALERT_UNKNOWN_CA: /* 1048 */ case SSL_R_TLSV1_ALERT_ACCESS_DENIED: /* 1049 */ case SSL_R_TLSV1_ALERT_DECODE_ERROR: /* 1050 */ diff --git a/src/nxt_otel.c b/src/nxt_otel.c index 2d2eb1883..2d9c33c9c 100644 --- a/src/nxt_otel.c +++ b/src/nxt_otel.c @@ -3,8 +3,6 @@ * Copyright (C) F5, Inc. */ -#include - #include #include #include @@ -22,24 +20,6 @@ #define NXT_OTEL_TRACEPARENT_LEN 55 -/* - * Span attribute keys. These follow the stable OpenTelemetry HTTP semantic - * conventions (https://opentelemetry.io/docs/specs/semconv/http/http-spans/); - * "http.flavor"/"http.user_agent" from older drafts are superseded by - * "network.protocol.version"/"user_agent.original". "unit.application.*" are - * FreeUnit-specific: a reverse proxy can't know the served app, but Unit can. - */ -#define NXT_OTEL_BODY_SIZE_TAG "http.request.body.size" -#define NXT_OTEL_METHOD_TAG "http.request.method" -#define NXT_OTEL_PATH_TAG "url.path" -#define NXT_OTEL_SCHEME_TAG "url.scheme" -#define NXT_OTEL_FLAVOR_TAG "network.protocol.version" -#define NXT_OTEL_USER_AGENT_TAG "user_agent.original" -#define NXT_OTEL_SERVER_ADDR_TAG "server.address" -#define NXT_OTEL_CLIENT_ADDR_TAG "client.address" -#define NXT_OTEL_APP_NAME_TAG "unit.application.name" -#define NXT_OTEL_APP_TYPE_TAG "unit.application.type" -#define NXT_OTEL_STATUS_CODE_TAG "http.response.status_code" static void @@ -54,27 +34,111 @@ nxt_otel_state_transition(nxt_otel_state_t *state, nxt_otel_status_t status) /* - * Set a semantic-convention span attribute from a string-literal key and an - * nxt_str_t value. Empty or absent values are skipped so we don't emit blank - * attributes for headers the request didn't carry. + * A request's span attributes, accumulated on the stack and handed to Rust in + * one call. + * + * Each attribute used to cost its own FFI crossing and two heap allocations -- + * one for the key, one for the value -- and the keys were compile-time + * constants being strlen'd and copied per request. Now the key is an id into a + * static table on the Rust side, and a whole stage's attributes cross once. + * + * The batch never outlives the stage that fills it, so the nxt_str_t values it + * holds are borrowed, not copied: they stay valid until the flush, and Rust + * copies what it needs to keep. + */ +typedef struct { + nxt_otel_attr_t attrs[NXT_OTEL_ATTR_MAX]; + nxt_uint_t n; +} nxt_otel_attr_batch_t; + + +static void +nxt_otel_attr_batch_init(nxt_otel_attr_batch_t *batch) +{ + batch->n = 0; +} + + +static nxt_otel_attr_t * +nxt_otel_attr_next(nxt_otel_attr_batch_t *batch, nxt_otel_attr_id_t id) +{ + nxt_otel_attr_t *attr; + + /* + * Every stage adds at most one attribute per id, so the batch cannot + * overflow; the check is here so that a future caller adding a second + * attribute for one id truncates rather than writing off the end. + */ + if (nxt_slow_path(batch->n >= NXT_OTEL_ATTR_MAX)) { + return NULL; + } + + attr = &batch->attrs[batch->n++]; + attr->key_id = id; + + return attr; +} + + +/* + * Add a string attribute. Empty or absent values are skipped so we don't emit + * blank attributes for headers the request didn't carry. */ static void -nxt_otel_add_attr(nxt_http_request_t *r, const char *key, nxt_str_t *val) +nxt_otel_attr_str(nxt_otel_attr_batch_t *batch, nxt_otel_attr_id_t id, + nxt_str_t *val) { - nxt_str_t k; + nxt_otel_attr_t *attr; if (val == NULL || val->start == NULL || val->length == 0) { return; } - if (r->otel == NULL || r->otel->trace == NULL) { + attr = nxt_otel_attr_next(batch, id); + if (attr == NULL) { return; } - k.start = (u_char *) key; - k.length = nxt_strlen(key); + attr->type = NXT_OTEL_ATTR_TYPE_STR; + attr->ival = 0; + attr->sval = *val; +} + - nxt_otel_rs_add_attr(r->otel->trace, &k, val); +/* + * Add an integer attribute. The semantic conventions type these as integers, + * and passing one as an integer avoids a sprintf round-trip on the request + * path as well as being the correct value type on the wire. + */ +static void +nxt_otel_attr_i64(nxt_otel_attr_batch_t *batch, nxt_otel_attr_id_t id, + int64_t val) +{ + nxt_otel_attr_t *attr; + + attr = nxt_otel_attr_next(batch, id); + if (attr == NULL) { + return; + } + + attr->type = NXT_OTEL_ATTR_TYPE_I64; + attr->ival = val; + nxt_memzero(&attr->sval, sizeof(nxt_str_t)); +} + + +static void +nxt_otel_attr_flush(nxt_http_request_t *r, nxt_otel_attr_batch_t *batch) +{ + if (batch->n == 0) { + return; + } + + if (r->otel == NULL || r->otel->trace == NULL) { + return; + } + + nxt_otel_rs_add_attrs(r->otel->trace, batch->attrs, batch->n); } @@ -107,7 +171,6 @@ static void nxt_otel_propagate_header(nxt_task_t *task, nxt_http_request_t *r) { u_char *traceval; - nxt_str_t traceparent_name, traceparent; nxt_http_field_t *f; traceval = nxt_mp_zalloc(r->mem_pool, NXT_OTEL_TRACEPARENT_LEN + 1); @@ -123,49 +186,46 @@ nxt_otel_propagate_header(nxt_task_t *task, nxt_http_request_t *r) return; } - if (r->otel->trace_id != NULL) { - // copy in the pre-existing traceparent for the response - sprintf((char *) traceval, "%s-%s-%s-%s", - (char *) r->otel->version, - (char *) r->otel->trace_id, - (char *) r->otel->parent_id, - (char *) r->otel->trace_flags); - /* - * if we didn't inherit a trace id then we need to add the - * traceparent header to the request + * The traceparent forwarded to the peer/app -- and echoed in the + * response -- always carries this request's own span as parent-id, so + * that the app's spans are children of ours rather than siblings of the + * inbound parent. nxt_otel_rs_copy_traceparent() reads trace-id, + * span-id and flags off the span's own context, which already carries + * the inherited trace-id/flags when a valid inbound traceparent was + * accepted, or freshly generated ones otherwise -- so one call covers + * both cases. */ - } else { - - nxt_otel_rs_copy_traceparent(traceval, r->otel->trace); + nxt_otel_rs_copy_traceparent(traceval, r->otel->trace); - /* - * nxt_list_add() hands out non-zeroed memory: garbage skip/hopbyhop - * bits make the peer/app serializers randomly drop the field. - */ - f = nxt_http_req_field_zero_add(r); - if (nxt_slow_path(f == NULL)) { - return; + /* + * Any inbound traceparent, valid or not, is replaced by the value + * below, so the peer/app never sees two of them. + */ + nxt_http_fields_each(f, r->inline_fields, r->num_inline_fields, r->fields) + { + if (f->name_length == nxt_length("traceparent") + && nxt_memcasecmp(f->name, "traceparent", + nxt_length("traceparent")) == 0) + { + f->skip = 1; } - nxt_http_field_name_set(f, "traceparent"); - f->value = traceval; - f->value_length = nxt_strlen(traceval); - - traceparent_name = (nxt_str_t) { - .start = f->name, - .length = f->name_length, - }; - - traceparent = (nxt_str_t) { - .start = f->value, - .length = f->value_length, - }; + } nxt_http_fields_loop; - nxt_otel_rs_add_event_to_trace(r->otel->trace, - &traceparent_name, &traceparent); + /* + * nxt_list_add() hands out non-zeroed memory: garbage skip/hopbyhop + * bits make the peer/app serializers randomly drop the field. + */ + f = nxt_http_req_field_zero_add(r); + if (nxt_slow_path(f == NULL)) { + return; } + nxt_http_field_name_set(f, "traceparent"); + f->value = traceval; + f->value_length = nxt_strlen(traceval); + f = nxt_http_resp_field_zero_add(&r->resp, r->mem_pool); if (nxt_slow_path(f == NULL)) { nxt_log(task, NXT_LOG_ERR, @@ -180,31 +240,26 @@ nxt_otel_propagate_header(nxt_task_t *task, nxt_http_request_t *r) static void -nxt_otel_span_add_headers(nxt_task_t *task, nxt_http_request_t *r) +nxt_otel_span_add_request_attrs(nxt_http_request_t *r) { - nxt_str_t val; - - nxt_log(task, NXT_LOG_DEBUG, "adding attributes to trace"); - - if (r->otel == NULL || r->otel->trace == NULL) { - nxt_log(task, NXT_LOG_ERR, "no trace to add attributes to!"); - nxt_otel_state_transition(r->otel, NXT_OTEL_ERROR_STATE); - return; - } + nxt_str_t val; + nxt_otel_attr_batch_t batch; /* * Record only well-defined semconv attributes. We deliberately do NOT * iterate every request header: that would leak sensitive values * (Authorization, Cookie, ...) into the telemetry backend. */ - nxt_otel_add_attr(r, NXT_OTEL_METHOD_TAG, r->method); - nxt_otel_add_attr(r, NXT_OTEL_PATH_TAG, r->path); + nxt_otel_attr_batch_init(&batch); + + nxt_otel_attr_str(&batch, NXT_OTEL_ATTR_METHOD, r->method); + nxt_otel_attr_str(&batch, NXT_OTEL_ATTR_PATH, r->path); nxt_str_set(&val, "http"); if (r->tls) { nxt_str_set(&val, "https"); } - nxt_otel_add_attr(r, NXT_OTEL_SCHEME_TAG, &val); + nxt_otel_attr_str(&batch, NXT_OTEL_ATTR_SCHEME, &val); /* "HTTP/1.1" -> "1.1" for network.protocol.version */ val = r->version; @@ -214,69 +269,76 @@ nxt_otel_span_add_headers(nxt_task_t *task, nxt_http_request_t *r) val.start += nxt_length("HTTP/"); val.length -= nxt_length("HTTP/"); } - nxt_otel_add_attr(r, NXT_OTEL_FLAVOR_TAG, &val); + nxt_otel_attr_str(&batch, NXT_OTEL_ATTR_FLAVOR, &val); if (r->user_agent != NULL) { val.start = r->user_agent->value; val.length = r->user_agent->value_length; - nxt_otel_add_attr(r, NXT_OTEL_USER_AGENT_TAG, &val); + nxt_otel_attr_str(&batch, NXT_OTEL_ATTR_USER_AGENT, &val); } - nxt_otel_add_attr(r, NXT_OTEL_SERVER_ADDR_TAG, &r->host); + nxt_otel_attr_str(&batch, NXT_OTEL_ATTR_SERVER_ADDR, &r->host); if (r->remote != NULL) { val.start = nxt_sockaddr_address(r->remote); val.length = r->remote->address_length; - nxt_otel_add_attr(r, NXT_OTEL_CLIENT_ADDR_TAG, &val); + nxt_otel_attr_str(&batch, NXT_OTEL_ATTR_CLIENT_ADDR, &val); } - nxt_otel_propagate_header(task, r); - - nxt_otel_state_transition(r->otel, NXT_OTEL_BODY_STATE); + nxt_otel_attr_flush(r, &batch); } static void -nxt_otel_span_add_body(nxt_http_request_t *r) +nxt_otel_span_add_headers(nxt_task_t *task, nxt_http_request_t *r) { - size_t body_size = 0; - size_t buf_size; - u_char *body_buf, *body_size_buf; - nxt_int_t cur; - nxt_str_t body_val; + nxt_log(task, NXT_LOG_DEBUG, "adding attributes to trace"); - if (r->body != NULL) { - body_size = nxt_buf_used_size(r->body); + if (r->otel == NULL || r->otel->trace == NULL) { + nxt_log(task, NXT_LOG_ERR, "no trace to add attributes to!"); + nxt_otel_state_transition(r->otel, NXT_OTEL_ERROR_STATE); + return; } - buf_size = 1; // first digit - if (body_size != 0) { - buf_size += log10(body_size); // subsequent digits - } - buf_size += 1; // \0 - buf_size += nxt_strlen(NXT_OTEL_BODY_SIZE_TAG); - buf_size += 1; // \0 + /* + * A span the sampler dropped records nothing, yet every attribute value + * is still assembled before the call that discards it. Skip that work. + * + * Propagation below is not part of the bargain: the traceparent must + * reach the peer and the application whatever the sampling decision was, + * so that a downstream service can continue -- or deliberately not + * continue -- the same trace. + */ - body_buf = nxt_mp_alloc(r->mem_pool, buf_size); - if (nxt_slow_path(body_buf == NULL)) { - return; + if (r->otel->recording) { + nxt_otel_span_add_request_attrs(r); } - cur = sprintf((char *) body_buf, "%lu", body_size); - if (cur < 0) { + nxt_otel_propagate_header(task, r); + + nxt_otel_state_transition(r->otel, NXT_OTEL_BODY_STATE); +} + + +static void +nxt_otel_span_add_body(nxt_http_request_t *r) +{ + size_t body_size = 0; + nxt_otel_attr_batch_t batch; + + if (!r->otel->recording) { + nxt_otel_state_transition(r->otel, NXT_OTEL_COLLECT_STATE); return; } - cur += 1; - body_size_buf = body_buf + cur; - nxt_cpystr(body_buf + cur, (const u_char *) NXT_OTEL_BODY_SIZE_TAG); + if (r->body != NULL) { + body_size = nxt_buf_used_size(r->body); + } - body_val = (nxt_str_t) { - .start = body_buf, - .length = nxt_strlen(body_buf), - }; + nxt_otel_attr_batch_init(&batch); + nxt_otel_attr_i64(&batch, NXT_OTEL_ATTR_BODY_SIZE, (int64_t) body_size); + nxt_otel_attr_flush(r, &batch); - nxt_otel_add_attr(r, (const char *) body_size_buf, &body_val); nxt_otel_state_transition(r->otel, NXT_OTEL_COLLECT_STATE); } @@ -284,17 +346,18 @@ nxt_otel_span_add_body(nxt_http_request_t *r) static void nxt_otel_span_add_status(nxt_task_t *task, nxt_http_request_t *r) { - int n; - u_char status_buf[8]; const char *type_name; nxt_str_t val; nxt_app_t *app; + nxt_otel_attr_batch_t batch; nxt_request_rpc_data_t *rpc; - if (r->otel == NULL || r->otel->trace == NULL) { + if (r->otel == NULL || r->otel->trace == NULL || !r->otel->recording) { return; } + nxt_otel_attr_batch_init(&batch); + /* * Application identity is resolved during routing, so it is only known by * the time the span is collected (after the response). A reverse proxy @@ -303,32 +366,21 @@ nxt_otel_span_add_status(nxt_task_t *task, nxt_http_request_t *r) rpc = r->req_rpc_data; if (rpc != NULL && rpc->app != NULL) { app = rpc->app; - nxt_otel_add_attr(r, NXT_OTEL_APP_NAME_TAG, &app->name); + nxt_otel_attr_str(&batch, NXT_OTEL_ATTR_APP_NAME, &app->name); type_name = nxt_otel_app_type_name(app->type); val.start = (u_char *) type_name; val.length = nxt_strlen(type_name); - nxt_otel_add_attr(r, NXT_OTEL_APP_TYPE_TAG, &val); + nxt_otel_attr_str(&batch, NXT_OTEL_ATTR_APP_TYPE, &val); } // dont bother logging an unset status - if (r->status == 0) { - return; + if (r->status != 0) { + nxt_otel_attr_i64(&batch, NXT_OTEL_ATTR_STATUS_CODE, + (int64_t) r->status); } - n = snprintf((char *) status_buf, sizeof(status_buf), "%d", - (int) r->status); - /* - * snprintf() returns the length it *would* have written; on truncation - * that exceeds the buffer, so only record the attribute when the value - * fully fit -- a truncated status code is meaningless anyway, and using - * the would-have-been length as val.length would read past status_buf. - */ - if (n > 0 && n < (int) sizeof(status_buf)) { - val.start = status_buf; - val.length = (size_t) n; - nxt_otel_add_attr(r, NXT_OTEL_STATUS_CODE_TAG, &val); - } + nxt_otel_attr_flush(r, &batch); /* Flag server errors so the span shows Status::Error in the collector. */ if (r->status >= NXT_HTTP_INTERNAL_SERVER_ERROR) { @@ -354,6 +406,63 @@ nxt_otel_span_collect(nxt_task_t *task, nxt_http_request_t *r) } +/* + * Release the span of a request that is being torn down. + * + * Registered as an r->mem_pool cleanup at span creation, so it runs from + * nxt_mp_destroy() on *every* request exit: the normal completion path, the + * client-abort/error path that goes straight to + * nxt_http_request_close_handler() without ever reaching COLLECT, and any exit + * path added in the future. Tying the span's lifetime to the pool that holds + * it makes the release structural rather than something each new exit path has + * to remember. + * + * nxt_otel_span_collect() NULLs r->otel->trace after handing the span to Rust, + * and it is the only other place that does so; that NULL is what keeps this + * idempotent, i.e. a no-op for a request that was collected normally. + */ +static void +nxt_otel_span_pool_cleanup(nxt_task_t *task, void *obj, void *data) +{ + nxt_thread_t *thr; + nxt_otel_state_t *state; + + state = obj; + + if (state->trace == NULL) { + return; + } + + thr = nxt_thread(); + + /* + * The task recorded at registration belongs to the connection, which may + * already have been recycled by the time the pool is destroyed, so log + * against the current thread's task instead of the one passed in. + */ + nxt_log(thr->task, NXT_LOG_DEBUG, + "otel: releasing the span of an unfinished request"); + + /* + * The request never produced a response. Mark the span failed and export + * it rather than dropping it silently: an aborted or timed out request is + * usually the interesting trace, and a span that simply vanishes is + * indistinguishable from telemetry being broken. Exporting also keeps the + * span consistent with the 5xx case, which is already flagged + * Status::Error in nxt_otel_span_add_status(). + * + * This cannot stall the engine thread: nxt_otel_rs_send_trace() only ends + * the span, which enqueues it on the Rust-side batch processor; the actual + * OTLP export runs later on that processor's own thread. + */ + nxt_otel_rs_set_error(state->trace); + nxt_otel_rs_send_trace(state->trace); + + state->trace = NULL; + state->status = NXT_OTEL_UNINIT_STATE; +} + + static void nxt_otel_error(nxt_task_t *task, nxt_http_request_t *r) { @@ -428,6 +537,29 @@ nxt_otel_trace_and_span_init(nxt_task_t *task, nxt_http_request_t *r) return; } + /* + * Bind the span to the request memory pool right away, so that from here + * on no exit path can drop the request without releasing the span. + */ + if (nxt_slow_path(nxt_mp_cleanup(r->mem_pool, nxt_otel_span_pool_cleanup, + task, r->otel, NULL) != NXT_OK)) + { + /* + * Without the cleanup nothing guarantees the span is released on an + * abort, so end it now instead of tracing this request and risking a + * leak. + */ + nxt_log(task, NXT_LOG_ERR, "couldn't register otel span cleanup"); + + nxt_otel_rs_send_trace(r->otel->trace); + r->otel->trace = NULL; + + nxt_otel_state_transition(r->otel, NXT_OTEL_ERROR_STATE); + return; + } + + r->otel->recording = nxt_otel_rs_is_recording(r->otel->trace); + nxt_otel_state_transition(r->otel, NXT_OTEL_HEADER_STATE); } @@ -648,6 +780,63 @@ nxt_otel_parse_tracestate(void *ctx, nxt_http_field_t *field, uintptr_t data) } +/* + * Flush the telemetry pipeline on the way out of the process. + * + * Called from nxt_runtime_exit(), the single funnel every router exit passes + * through (signal handlers, the "quit" port message and internal failures all + * converge on nxt_runtime_quit() -> nxt_runtime_exit() -> exit()). Without + * this the batch processor's queue -- up to MAX_QUEUE_SIZE (4096) spans plus + * whatever is in the batch being assembled -- is simply discarded. + * + * The flush is bounded rather than unconditional. A blocking shutdown waits + * for the exporter's own 10s timeout when the collector is unreachable, which + * would trade "loses spans on exit" for "takes ten seconds to stop", a far + * more visible regression for anything supervising unitd. The budget below is + * comfortably more than a reachable collector needs (a local OTLP endpoint + * answers in milliseconds) and short enough that a dead one is not felt. + */ + +#define NXT_OTEL_EXIT_FLUSH_TIMEOUT_MS 2000 + +void +nxt_otel_shutdown(nxt_task_t *task) +{ + if (!nxt_otel_rs_is_init()) { + return; + } + + nxt_log(task, NXT_LOG_DEBUG, "otel: flushing spans before exit"); + + /* + * NXT_OTEL_SHUTDOWN_FLUSHED is not a promise that nothing was lost: the + * router's worker engines are still live here, so a span ended after the + * flush reaches an already shut down provider and is dropped silently. + * That one stays unreportable until producers can be quiesced, which is + * the P5 graceful-shutdown work, tracked in issue #219. + */ + switch (nxt_otel_rs_shutdown_bounded(NXT_OTEL_EXIT_FLUSH_TIMEOUT_MS)) { + + case NXT_OTEL_SHUTDOWN_TIMEOUT: + nxt_log(task, NXT_LOG_WARN, + "otel: the final span flush did not complete within %d ms, " + "exiting anyway; the spans it held are lost", + NXT_OTEL_EXIT_FLUSH_TIMEOUT_MS); + break; + + case NXT_OTEL_SHUTDOWN_FAILED: + nxt_log(task, NXT_LOG_WARN, + "otel: span export to the collector failed, exiting anyway; " + "the spans of at least one batch are lost"); + break; + + default: + nxt_log(task, NXT_LOG_DEBUG, "otel: span flush complete"); + break; + } +} + + void nxt_otel_log_callback(nxt_uint_t log_level, const char *arg) { diff --git a/src/nxt_otel.h b/src/nxt_otel.h index 3e64983d1..eec16332f 100644 --- a/src/nxt_otel.h +++ b/src/nxt_otel.h @@ -16,6 +16,81 @@ #endif +/* + * Span attribute keys, as ids rather than strings. + * + * The key text lives on the Rust side in ATTR_KEYS, as a const table of + * Key::from_static_str -- so a key costs no allocation and no strlen per + * request. The order of this enum and of that table is the contract between + * the two sides: the id indexes the table directly. Add to the end, and add + * to ATTR_KEYS in src/otel/src/lib.rs in the same commit. + */ +typedef enum { + NXT_OTEL_ATTR_METHOD = 0, + NXT_OTEL_ATTR_PATH, + NXT_OTEL_ATTR_SCHEME, + NXT_OTEL_ATTR_FLAVOR, + NXT_OTEL_ATTR_USER_AGENT, + NXT_OTEL_ATTR_SERVER_ADDR, + NXT_OTEL_ATTR_CLIENT_ADDR, + NXT_OTEL_ATTR_APP_NAME, + NXT_OTEL_ATTR_APP_TYPE, + NXT_OTEL_ATTR_STATUS_CODE, + NXT_OTEL_ATTR_BODY_SIZE, + NXT_OTEL_ATTR_MAX +} nxt_otel_attr_id_t; + + +/* Which member of nxt_otel_attr_t carries the value. */ +#define NXT_OTEL_ATTR_TYPE_STR 0 +#define NXT_OTEL_ATTR_TYPE_I64 1 + + +/* + * One span attribute in a batch. Mirrored by nxt_otel_attr_t in + * src/otel/src/lib.rs; the layouts must stay identical. + * + * The Rust struct is #[repr(C)], so both sides lay their fields out under + * the target's C ABI: a change mirrored on both sides, reordering included, + * stays in agreement on every target. What breaks the FFI is editing one + * side only, or pairing fields whose types are not the same size and + * alignment everywhere we ship -- C long is 4 bytes on ILP32 where Rust i64 + * is always 8, and C enum sizes are implementation-defined, which is why + * key_id and type are uint32_t here rather than nxt_otel_attr_id_t. + * + * The offsets below are the same on x86-64, armv7 and i386 (the total size + * is not: 32 on 64-bit, 24 on 32-bit, where int64_t needs only 4-byte + * alignment). They are asserted rather than described, so that a field + * added, reordered or appended here fails the build and prompts the matching + * edit on the Rust side, which these assertions cannot see. + * + * The size assertion is what covers a field appended after sval: the offset + * checks alone would all still hold, while the struct grew. That matters + * more than it looks, because these are passed as an array -- a C-side-only + * trailing field changes the stride C writes with and not the one Rust reads + * with, so every attribute after the first would be misread. + */ +typedef struct { + uint32_t key_id; + uint32_t type; + int64_t ival; + nxt_str_t sval; +} nxt_otel_attr_t; + +nxt_static_assert(offsetof(nxt_otel_attr_t, key_id) == 0, + "nxt_otel_attr_t layout drifted from src/otel/src/lib.rs"); +nxt_static_assert(offsetof(nxt_otel_attr_t, type) == 4, + "nxt_otel_attr_t layout drifted from src/otel/src/lib.rs"); +nxt_static_assert(offsetof(nxt_otel_attr_t, ival) == 8, + "nxt_otel_attr_t layout drifted from src/otel/src/lib.rs"); +nxt_static_assert(offsetof(nxt_otel_attr_t, sval) == 2 * sizeof(uint32_t) + + sizeof(int64_t), + "nxt_otel_attr_t layout drifted from src/otel/src/lib.rs"); +nxt_static_assert(sizeof(nxt_otel_attr_t) == offsetof(nxt_otel_attr_t, sval) + + sizeof(nxt_str_t), + "nxt_otel_attr_t grew a field Rust does not know about"); + + #if (NXT_HAVE_OTEL) extern void nxt_otel_rs_send_trace(void *trace); extern void * nxt_otel_rs_get_or_create_trace(const u_char *trace_id, @@ -26,12 +101,36 @@ extern void nxt_otel_rs_init( const nxt_str_t *endpoint, const nxt_str_t *protocol, double sample_fraction, double batch_size); extern void nxt_otel_rs_copy_traceparent(u_char *buffer, void *span); -extern void nxt_otel_rs_add_event_to_trace(void *trace, nxt_str_t *key, - nxt_str_t *val); -extern void nxt_otel_rs_add_attr(void *trace, nxt_str_t *key, nxt_str_t *val); +extern void nxt_otel_rs_add_attrs(void *trace, + const nxt_otel_attr_t *attrs, size_t n); extern void nxt_otel_rs_set_error(void *trace); +extern uint8_t nxt_otel_rs_is_recording(void *trace); extern uint8_t nxt_otel_rs_is_init(void); extern void nxt_otel_rs_uninit(void); +extern uint8_t nxt_otel_rs_shutdown_bounded(uint64_t timeout_ms); +/* + * Span export health for the /status API: writes the number of spans the + * exporter accepted and the number it rejected since the live provider was + * installed, and returns non-zero when a provider is installed at all. When + * it returns 0 the two counters are meaningless and must not be reported. + */ +extern uint8_t nxt_otel_rs_export_stats(uint64_t *exported, uint64_t *failed); + +/* + * Return values of nxt_otel_rs_shutdown_bounded(). Defined on the Rust side + * as NXT_OTEL_SHUTDOWN_* in src/otel/src/lib.rs; the two lists must be kept + * in step. + * + * TIMEOUT the flush did not answer within the caller's budget. + * FLUSHED the flush completed and no export has failed since the provider + * was installed. Not a promise that no spans were lost: see the + * producer race documented on the Rust side and in issue #219. + * FAILED an export failed -- either the one this call forced, or an earlier + * batch recorded by the sticky failure flag. + */ +#define NXT_OTEL_SHUTDOWN_TIMEOUT 0 +#define NXT_OTEL_SHUTDOWN_FLUSHED 1 +#define NXT_OTEL_SHUTDOWN_FAILED 2 #endif @@ -66,6 +165,8 @@ struct nxt_otel_state_s { void *trace; nxt_otel_status_t status; nxt_str_t trace_state; + /* The sampler's verdict on this span, cached at creation. */ + uint8_t recording; }; @@ -78,6 +179,7 @@ void nxt_otel_log_callback(nxt_uint_t log_level, const char *arg); void nxt_otel_test_and_call_state(nxt_task_t *task, nxt_http_request_t *r); void nxt_otel_request_error_path(nxt_task_t *task, nxt_http_request_t *r); +void nxt_otel_shutdown(nxt_task_t *task); #endif /* _NXT_OTEL_H_INCLUDED_ */ diff --git a/src/nxt_pcre.c b/src/nxt_pcre.c index 737fc7cf7..28e137af2 100644 --- a/src/nxt_pcre.c +++ b/src/nxt_pcre.c @@ -9,30 +9,46 @@ struct nxt_regex_s { - pcre *code; - pcre_extra *extra; - nxt_str_t pattern; + pcre *code; + nxt_str_t pattern; }; +/* + * The limits and the match data of one request. "extra" passes the + * structure to the callout, which counts down "budget". + */ + struct nxt_regex_match_s { - int ovecsize; - int ovec[]; + pcre_extra extra; + uint32_t budget; + int ovecsize; + int ovec[]; }; static void *nxt_pcre_malloc(size_t size); static void nxt_pcre_free(void *p); +static int nxt_pcre_callout(pcre_callout_block *block); static nxt_mp_t *nxt_pcre_mp; +/* + * The most nested calls that one match may make. PCRE 1 makes them on the + * stack, 512 bytes each in libpcre3 8.39 on x86-64. 2,000 calls use about + * 1 MB. A request matches on a router thread with the default glibc stack: + * 8 MB, or 2 MB when the stack rlimit is unlimited. + */ +#define NXT_PCRE_RECURSION_LIMIT 2000 + nxt_regex_t * nxt_regex_compile(nxt_mp_t *mp, nxt_str_t *source, nxt_regex_err_t *err) { - int erroffset; - char *pattern; - void *saved_malloc, *saved_free; - nxt_regex_t *re; + int erroffset; + char *pattern; + void *saved_malloc, *saved_free; + nxt_regex_t *re; + unsigned long options; err->offset = source->length; @@ -58,21 +74,41 @@ nxt_regex_compile(nxt_mp_t *mp, nxt_str_t *source, nxt_regex_err_t *err) nxt_pcre_mp = mp; re->code = pcre_compile(pattern, 0, &err->msg, &erroffset, NULL); - if (nxt_fast_path(re->code != NULL)) { -#if 0 - re->extra = pcre_study(re->code, PCRE_STUDY_JIT_COMPILE, &err->msg); - if (nxt_slow_path(re->extra == NULL && err->msg != NULL)) { - nxt_log_warn(thr->log, "pcre_study(%V) failed: %s", source, err->msg); - } -#else - re->extra = NULL; -#endif - } else { + /* + * The library counts match steps from zero again at each start position + * of a pattern that is not anchored. So the match limit does not bound + * one whole match of it. Such a pattern is compiled again with a + * callout before each item. One match may make at most + * NXT_REGEX_MATCH_LIMIT callouts in total. The callouts do not change + * the result. An anchored pattern has one start position and gets no + * callouts. + * + * PCRE_INFO_OPTIONS writes an unsigned long. + */ + + if (re->code != NULL + && pcre_fullinfo(re->code, NULL, PCRE_INFO_OPTIONS, &options) == 0 + && (options & PCRE_ANCHORED) == 0) + { + re->code = pcre_compile(pattern, PCRE_AUTO_CALLOUT, &err->msg, + &erroffset, NULL); + } + + if (nxt_slow_path(re->code == NULL)) { err->offset = erroffset; re = NULL; } + /* + * The callout function is global in PCRE 1. It is set here, before any + * match with the pattern. No other code in the process sets it. + */ + + if (pcre_callout != nxt_pcre_callout) { + pcre_callout = nxt_pcre_callout; + } + pcre_malloc = saved_malloc; pcre_free = saved_free; @@ -101,13 +137,38 @@ nxt_pcre_free(void *p) } +static int +nxt_pcre_callout(pcre_callout_block *block) +{ + nxt_regex_match_t *match; + + match = block->callout_data; + + /* A negative value stops the whole match with this value. */ + + if (match->budget == 0) { + return PCRE_ERROR_CALLOUT; + } + + match->budget--; + + return 0; +} + + nxt_regex_match_t * nxt_regex_match_create(nxt_mp_t *mp, size_t size) { nxt_regex_match_t *match; - match = nxt_mp_get(mp, sizeof(nxt_regex_match_t) + sizeof(int) * size); + match = nxt_mp_zget(mp, sizeof(nxt_regex_match_t) + sizeof(int) * size); if (nxt_fast_path(match != NULL)) { + match->extra.flags = PCRE_EXTRA_MATCH_LIMIT + | PCRE_EXTRA_MATCH_LIMIT_RECURSION + | PCRE_EXTRA_CALLOUT_DATA; + match->extra.match_limit = NXT_REGEX_MATCH_LIMIT; + match->extra.match_limit_recursion = NXT_PCRE_RECURSION_LIMIT; + match->extra.callout_data = match; match->ovecsize = size; } @@ -121,9 +182,34 @@ nxt_regex_match(nxt_regex_t *re, u_char *subject, size_t length, { int ret; - ret = pcre_exec(re->code, re->extra, (const char *) subject, length, 0, 0, - match->ovec, match->ovecsize); + match->budget = NXT_REGEX_MATCH_LIMIT; + + ret = pcre_exec(re->code, &match->extra, (const char *) subject, length, + 0, 0, match->ovec, match->ovecsize); if (nxt_slow_path(ret < PCRE_ERROR_NOMATCH)) { + + /* + * A client can cause these errors on each request. So the log gets + * no subject bytes for a limit, and at most 64 for others. The + * callout returns PCRE_ERROR_CALLOUT when one match makes more + * than NXT_REGEX_MATCH_LIMIT callouts. + */ + + if (ret == PCRE_ERROR_MATCHLIMIT || ret == PCRE_ERROR_CALLOUT + || ret == PCRE_ERROR_RECURSIONLIMIT) + { + nxt_thread_log_error(NXT_LOG_WARN, + "pcre_exec() reached the %s limit on %uz " + "bytes using \"%V\"", + (ret == PCRE_ERROR_RECURSIONLIMIT) + ? "recursion" : "match", + length, &re->pattern); + + return NXT_ERROR; + } + + length = nxt_min(length, NXT_REGEX_LOG_SUBJECT); + nxt_thread_log_error(NXT_LOG_ERR, "pcre_exec() failed: %d on \"%*s\" using \"%V\"", ret, length, subject, &re->pattern); diff --git a/src/nxt_pcre2.c b/src/nxt_pcre2.c index cb51062c0..387f6fbc6 100644 --- a/src/nxt_pcre2.c +++ b/src/nxt_pcre2.c @@ -10,8 +10,31 @@ #include +/* + * The most heap memory in KiB that one match may use for backtracking + * frames. In PCRE2 10.42 on x86-64, a frame takes 128 bytes, plus 16 + * bytes for each capturing group. 100,000 frames of 160 bytes take about + * 16 MB. The match limit permits no more frames than that, so this limit + * can stop a pattern with several capturing groups first: three with + * PCRE2 10.42, two with 10.46, where a frame is 8 bytes larger. The + * library default is about 20 GB. + */ +#define NXT_PCRE2_HEAP_LIMIT 16384 + +/* + * The most nested calls that one match may make with PCRE2 before 10.30. + * These versions make the calls on the stack, 448 bytes each in PCRE2 10.23 + * of Amazon Linux 2 on x86-64. 2,000 calls use about 900 KB. A request + * matches on a router thread with the default glibc stack: 8 MB, or 2 MB + * when the stack rlimit is unlimited. The library default is 10,000,000. + */ +#define NXT_PCRE2_RECURSION_LIMIT 2000 + + static void *nxt_pcre2_malloc(PCRE2_SIZE size, void *memory_data); static void nxt_pcre2_free(void *p, void *memory_data); +static int nxt_pcre2_callout(pcre2_callout_block *block, void *data); +static void nxt_pcre2_match_cleanup(nxt_task_t *task, void *obj, void *data); struct nxt_regex_s { @@ -20,12 +43,26 @@ struct nxt_regex_s { }; +/* + * The match data and the match context of one request. The callout of the + * context gets the structure and counts down "budget". + */ + +struct nxt_regex_match_s { + pcre2_match_data *data; + pcre2_match_context *ctx; + uint32_t budget; +}; + + nxt_regex_t * nxt_regex_compile(nxt_mp_t *mp, nxt_str_t *source, nxt_regex_err_t *err) { int errcode; + uint32_t options; nxt_int_t ret; PCRE2_SIZE erroffset; + pcre2_code *code; nxt_regex_t *re; pcre2_general_context *general_ctx; pcre2_compile_context *compile_ctx; @@ -55,20 +92,40 @@ nxt_regex_compile(nxt_mp_t *mp, nxt_str_t *source, nxt_regex_err_t *err) re->code = pcre2_compile((PCRE2_SPTR) source->start, source->length, 0, &errcode, &erroffset, compile_ctx); if (nxt_slow_path(re->code == NULL)) { - err->offset = erroffset; + goto compile_fail; + } - ret = pcre2_get_error_message(errcode, (PCRE2_UCHAR *) err->msg, - ERR_BUF_SIZE); - if (ret < 0) { - (void) nxt_sprintf(err->msg, err->msg + ERR_BUF_SIZE, - "compilation failed with unknown " - "error code: %d%Z", errcode); + /* + * The library counts match steps from zero again at each start position + * of a pattern that is not anchored. So the match limit does not bound + * one whole match of it. Such a pattern is compiled again with a + * callout before each item. One match may make at most + * NXT_REGEX_MATCH_LIMIT callouts in total. The callouts do not change + * the result. An anchored pattern has one start position and gets no + * callouts. + */ + + if (pcre2_pattern_info(re->code, PCRE2_INFO_ALLOPTIONS, &options) == 0 + && (options & PCRE2_ANCHORED) == 0) + { + code = pcre2_compile((PCRE2_SPTR) source->start, source->length, + PCRE2_AUTO_CALLOUT, &errcode, &erroffset, + compile_ctx); + if (nxt_slow_path(code == NULL)) { + goto compile_fail; } - return NULL; + pcre2_code_free(re->code); + re->code = code; } #if 0 + /* + * The JIT honours the match limit and callouts but not the heap limit, + * and it has its own stack limit. This matters if the JIT is enabled + * again. + */ + errcode = pcre2_jit_compile(re, PCRE2_JIT_COMPLETE); if (nxt_slow_path(errcode != 0 && errcode != PCRE2_ERROR_JIT_BADOPTION)) { ret = pcre2_get_error_message(errcode, (PCRE2_UCHAR *) err->msg, @@ -85,6 +142,20 @@ nxt_regex_compile(nxt_mp_t *mp, nxt_str_t *source, nxt_regex_err_t *err) return re; +compile_fail: + + err->offset = erroffset; + + ret = pcre2_get_error_message(errcode, (PCRE2_UCHAR *) err->msg, + ERR_BUF_SIZE); + if (ret < 0) { + (void) nxt_sprintf(err->msg, err->msg + ERR_BUF_SIZE, + "compilation failed with unknown " + "error code: %d%Z", errcode); + } + + return NULL; + alloc_fail: err->offset = source->length; @@ -107,24 +178,90 @@ nxt_pcre2_free(void *p, void *mp) } +static int +nxt_pcre2_callout(pcre2_callout_block *block, void *data) +{ + nxt_regex_match_t *match; + + match = data; + + /* A negative value stops the whole match with this value. */ + + if (match->budget == 0) { + return PCRE2_ERROR_CALLOUT; + } + + match->budget--; + + return 0; +} + + +static void +nxt_pcre2_match_cleanup(nxt_task_t *task, void *obj, void *data) +{ + pcre2_match_context_free(obj); +} + + nxt_regex_match_t * nxt_regex_match_create(nxt_mp_t *mp, size_t size) { nxt_regex_match_t *match; - pcre2_general_context *ctx; + pcre2_match_context *ctx; + pcre2_general_context *general_ctx; - ctx = pcre2_general_context_create(nxt_pcre2_malloc, nxt_pcre2_free, mp); - if (nxt_slow_path(ctx == NULL)) { + match = nxt_mp_get(mp, sizeof(nxt_regex_match_t)); + if (nxt_slow_path(match == NULL)) { + return NULL; + } + + general_ctx = pcre2_general_context_create(nxt_pcre2_malloc, + nxt_pcre2_free, mp); + if (nxt_slow_path(general_ctx == NULL)) { nxt_thread_log_alert("pcre2_general_context_create() failed"); return NULL; } - match = pcre2_match_data_create(size, ctx); - if (nxt_slow_path(match == NULL)) { + match->data = pcre2_match_data_create(size, general_ctx); + if (nxt_slow_path(match->data == NULL)) { nxt_thread_log_alert("pcre2_match_data_create(%uz) failed", size); return NULL; } + /* + * PCRE2 10.30 to 10.40 allocate the backtracking frames of a match with + * the allocator of the match context, and free them after the match. + * The pool does not free memory. So the context uses the system + * allocator, and the pool frees the context when the pool is destroyed. + */ + + ctx = pcre2_match_context_create(NULL); + if (nxt_slow_path(ctx == NULL)) { + nxt_thread_log_alert("pcre2_match_context_create() failed"); + return NULL; + } + + if (nxt_slow_path(nxt_mp_cleanup(mp, nxt_pcre2_match_cleanup, NULL, ctx, + NULL) + != NXT_OK)) + { + pcre2_match_context_free(ctx); + return NULL; + } + + (void) pcre2_set_match_limit(ctx, NXT_REGEX_MATCH_LIMIT); + +#if (PCRE2_MAJOR > 10 || PCRE2_MINOR >= 30) + (void) pcre2_set_heap_limit(ctx, NXT_PCRE2_HEAP_LIMIT); +#else + (void) pcre2_set_recursion_limit(ctx, NXT_PCRE2_RECURSION_LIMIT); +#endif + + (void) pcre2_set_callout(ctx, nxt_pcre2_callout, match); + + match->ctx = ctx; + return match; } @@ -134,13 +271,58 @@ nxt_regex_match(nxt_regex_t *re, u_char *subject, size_t length, nxt_regex_match_t *match) { nxt_int_t ret; + const char *limit; PCRE2_UCHAR errptr[ERR_BUF_SIZE]; - ret = pcre2_match(re->code, (PCRE2_SPTR) subject, length, 0, 0, match, - NULL); + match->budget = NXT_REGEX_MATCH_LIMIT; + + ret = pcre2_match(re->code, (PCRE2_SPTR) subject, length, 0, 0, + match->data, match->ctx); if (nxt_slow_path(ret < PCRE2_ERROR_NOMATCH)) { + /* + * A client can cause these errors on each request. So the log gets + * no subject bytes for a limit, and at most 64 for others. The + * callout returns PCRE2_ERROR_CALLOUT when one match makes more + * than NXT_REGEX_MATCH_LIMIT callouts. + */ + + switch (ret) { + + case PCRE2_ERROR_MATCHLIMIT: + case PCRE2_ERROR_CALLOUT: + limit = "match"; + break; + +#if defined(PCRE2_ERROR_HEAPLIMIT) + case PCRE2_ERROR_HEAPLIMIT: + limit = "heap"; + break; +#endif + + /* PCRE2 10.30 and later call it the depth limit. */ + + case PCRE2_ERROR_RECURSIONLIMIT: + limit = "recursion"; + break; + + default: + limit = NULL; + break; + } + + if (limit != NULL) { + nxt_thread_log_error(NXT_LOG_WARN, + "pcre2_match() reached the %s limit on %uz " + "bytes using \"%V\"", limit, length, + &re->pattern); + + return NXT_ERROR; + } + + length = nxt_min(length, NXT_REGEX_LOG_SUBJECT); + if (pcre2_get_error_message(ret, errptr, ERR_BUF_SIZE) < 0) { nxt_thread_log_error(NXT_LOG_ERR, "pcre2_match() failed: %d on \"%*s\" " diff --git a/src/nxt_php_sapi.c b/src/nxt_php_sapi.c index 65121ec52..557cf78dc 100644 --- a/src/nxt_php_sapi.c +++ b/src/nxt_php_sapi.c @@ -16,6 +16,8 @@ #include #include +#include NXT_PHP_MOUNTS_H + #if (PHP_VERSION_ID >= 50400) #define NXT_HAVE_PHP_IGNORE_CWD 1 @@ -83,15 +85,6 @@ static nxt_int_t nxt_php_setup(nxt_task_t *task, nxt_process_t *process, nxt_common_app_conf_t *conf); static nxt_int_t nxt_php_start(nxt_task_t *task, nxt_process_data_t *data); static void nxt_php_cleanup_targets(void); -#if NXT_PHP_TRUEASYNC -static nxt_int_t nxt_php_async_load_entrypoint(nxt_task_t *task, nxt_str_t *entrypoint); -static bool nxt_php_activate_true_async(nxt_task_t *task); -static void nxt_php_suspend_coroutine(nxt_unit_ctx_t *ctx); -static int nxt_php_add_port(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port); -static void nxt_php_remove_port(nxt_unit_t *unit, nxt_unit_ctx_t *ctx, nxt_unit_port_t *port); -static void nxt_php_quit_handler(nxt_unit_ctx_t *ctx); -static void nxt_php_shm_ack_handler(nxt_unit_ctx_t *ctx); -#endif /* NXT_PHP_TRUEASYNC */ static nxt_int_t nxt_php_set_target(nxt_task_t *task, nxt_php_target_t *target, nxt_conf_value_t *conf); static nxt_int_t nxt_php_set_ini_path(nxt_task_t *task, nxt_str_t *path, @@ -111,21 +104,16 @@ static void nxt_php_disable(nxt_task_t *task, const char *type, static nxt_int_t nxt_php_dirname(const nxt_str_t *file, nxt_str_t *dir); static void nxt_php_str_trim_trail(nxt_str_t *str, u_char t); static void nxt_php_str_trim_lead(nxt_str_t *str, u_char t); +static nxt_bool_t nxt_php_path_is_under(const nxt_str_t *root, + const nxt_str_t *path); nxt_inline u_char *nxt_realpath(const void *c); static nxt_int_t nxt_php_do_301(nxt_unit_request_info_t *req); static nxt_int_t nxt_php_handle_fs_err(nxt_unit_request_info_t *req); static void nxt_php_request_handler(nxt_unit_request_info_t *req); -#if NXT_PHP_TRUEASYNC -static void nxt_php_request_handler_async(nxt_unit_request_info_t *req); -#endif /* NXT_PHP_TRUEASYNC */ static void nxt_php_dynamic_request(nxt_php_run_ctx_t *ctx, nxt_unit_request_t *r); -#if NXT_PHP_TRUEASYNC -static void nxt_php_scope_init_superglobals(zend_async_scope_t *scope); -static void nxt_php_scope_populate_superglobals(zend_async_scope_t *scope); -#endif /* NXT_PHP_TRUEASYNC */ #if (PHP_VERSION_ID < 70400) static void nxt_zend_stream_init_fp(zend_file_handle *handle, FILE *fp, const char *filename); @@ -145,10 +133,6 @@ nxt_inline void nxt_php_set_str(nxt_unit_request_info_t *req, const char *name, static void nxt_php_set_cstr(nxt_unit_request_info_t *req, const char *name, const char *str, uint32_t len, zval *track_vars_array TSRMLS_DC); void nxt_php_register_variables(zval *track_vars_array TSRMLS_DC); -#if NXT_PHP_TRUEASYNC -static void nxt_php_register_variables_async(nxt_unit_request_info_t *req, - nxt_php_run_ctx_t *ctx, zval *track_vars_array TSRMLS_DC); -#endif /* NXT_PHP_TRUEASYNC */ #if NXT_PHP8 static void nxt_php_log_message(const char *message, int syslog_type_int); #else @@ -167,6 +151,7 @@ static size_t nxt_php_read_post(char *buffer, size_t count_bytes TSRMLS_DC); static int nxt_php_unbuffered_write(const char *str, uint str_length TSRMLS_DC); static int nxt_php_read_post(char *buffer, uint count_bytes TSRMLS_DC); #endif +static void nxt_php_flush(void *server_context); #ifdef NXT_PHP7 @@ -225,13 +210,6 @@ PHP_MINIT_FUNCTION(nxt_php_ext) nxt_php_chdir_handler = func->internal_function.handler; func->internal_function.handler = nxt_php_chdir; -#if NXT_PHP_TRUEASYNC - /* Register NginxUnit PHP extension classes (TrueAsync only) */ - if (nxt_php_extension_init() != NXT_OK) { - return FAILURE; - } -#endif /* NXT_PHP_TRUEASYNC */ - return SUCCESS; } @@ -292,7 +270,16 @@ PHP_FUNCTION(fastcgi_finish_request) #endif } - nxt_unit_request_done(ctx->req, NXT_UNIT_OK); + /* + * Detached, not plain done: the script keeps running after this returns. + * Without telling the router that, the worker is counted idle from here + * -- its slot goes back to "processes": {"max"} and its idle timer can + * reap a process that is still executing. libunit reports the work + * finished by itself when this request handler returns, which covers the + * exit() and fatal paths as well as an ordinary return. + */ + + nxt_unit_request_done_detached(ctx->req, NXT_UNIT_OK); ctx->req = NULL; PG(connection_status) = PHP_CONNECTION_ABORTED; @@ -322,7 +309,7 @@ static sapi_module_struct nxt_php_sapi_module = NULL, /* deactivate */ nxt_php_unbuffered_write, /* unbuffered write */ - NULL, /* flush */ + nxt_php_flush, /* flush */ NULL, /* get uid */ NULL, /* getenv */ @@ -384,8 +371,8 @@ NXT_EXPORT nxt_app_module_t nxt_app_module = { compat, nxt_string("php"), PHP_VERSION, - NULL, - 0, + NXT_PHP_MOUNTS, + NXT_PHP_NMOUNTS, nxt_php_setup, nxt_php_start, }; @@ -395,8 +382,7 @@ static nxt_php_target_t *nxt_php_targets; static nxt_uint_t nxt_php_targets_count; static nxt_int_t nxt_php_last_target = -1; -/* Global Unit context - needed by nxt_php_extension.c */ -nxt_unit_ctx_t *nxt_php_unit_ctx; +static nxt_unit_ctx_t *nxt_php_unit_ctx; #if defined(ZTS) && (PHP_VERSION_ID < 70400) static void ***tsrm_ls; @@ -416,7 +402,6 @@ nxt_php_setup(nxt_task_t *task, nxt_process_t *process, static const nxt_str_t user_str = nxt_string("user"); static const nxt_str_t admin_str = nxt_string("admin"); - c = &conf->u.php; #ifdef ZTS @@ -487,205 +472,6 @@ nxt_php_setup(nxt_task_t *task, nxt_process_t *process, } -#if NXT_PHP_TRUEASYNC - -/** - * Add pending write to drain_queue - */ -nxt_php_pending_write_t * -nxt_php_drain_queue_add(nxt_unit_ctx_t *ctx, - nxt_unit_request_info_t *req, - zend_string *str, - size_t offset) -{ - nxt_php_async_ctx_data_t *async_data = ctx->data; - nxt_php_pending_write_t *pw; - - if (async_data == NULL) { - return NULL; - } - - pw = malloc(sizeof(nxt_php_pending_write_t)); - if (pw == NULL) { - return NULL; - } - - pw->req = req; - pw->offset = offset; - - /* Increase refcount - string won't be freed */ - pw->data = zend_string_copy(str); - - nxt_queue_insert_tail(&async_data->drain_queue, &pw->link); - - return pw; -} - - -/** - * Remove and free pending write from drain_queue - */ -static void -nxt_php_drain_queue_remove(nxt_php_pending_write_t *pw) -{ - nxt_queue_remove(&pw->link); - - /* Decrease refcount - will be freed when refcount reaches 0 */ - zend_string_release(pw->data); - - free(pw); -} - - -/** - * Callback function called when port fd becomes readable - */ -static void -nxt_php_port_event_callback(zend_async_event_t *event, zend_async_event_callback_t *callback, void *result, zend_object *exception) -{ - nxt_php_port_event_data_t *event_data; - - /* Get pointer to our extra data using extra_offset */ - event_data = (nxt_php_port_event_data_t *)((char *)event + event->extra_offset); - - /* Process messages from this port */ - nxt_unit_process_port_msg(event_data->ctx, event_data->port); -} - - -/** - * Callback to add port - called by Unit for each port - */ -static int -nxt_php_add_port(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port) -{ - zend_async_poll_event_t *poll_event; - nxt_php_port_event_data_t *event_data; - - nxt_php_unit_ctx = ctx; - - /* Skip ports without input fd */ - if (port->in_fd == -1) { - return NXT_UNIT_OK; - } - - /* Set port to non-blocking mode */ - if (fcntl(port->in_fd, F_SETFL, O_NONBLOCK) == -1) { - nxt_unit_warn(ctx, "fcntl(%d, O_NONBLOCK) failed: %s (%d)", - port->in_fd, strerror(errno), errno); - return NXT_UNIT_ERROR; - } - - /* Create TrueAsync poll event with extra space for our data */ - poll_event = ZEND_ASYNC_NEW_POLL_EVENT_EX( - port->in_fd, false, ASYNC_READABLE, sizeof(nxt_php_port_event_data_t) - ); - - if (poll_event == NULL) { - nxt_unit_alert(ctx, "Failed to create TrueAsync poll event for port fd=%d", port->in_fd); - return NXT_UNIT_ERROR; - } - - /* Get pointer to our extra data */ - event_data = (nxt_php_port_event_data_t *)((char *)poll_event + poll_event->base.extra_offset); - event_data->ctx = ctx; - event_data->port = port; - - /* Register callback with poll event */ - if (!poll_event->base.add_callback(&poll_event->base, ZEND_ASYNC_EVENT_CALLBACK(nxt_php_port_event_callback))) { - nxt_unit_alert(ctx, "Failed to add callback for port fd=%d", port->in_fd); - poll_event->base.dispose(&poll_event->base); - return NXT_UNIT_ERROR; - } - - /* Start polling */ - if (!poll_event->base.start(&poll_event->base)) { - nxt_unit_alert(ctx, "Failed to start polling for port fd=%d", port->in_fd); - poll_event->base.dispose(&poll_event->base); - return NXT_UNIT_ERROR; - } - - /* Save poll_event in port->data for cleanup */ - port->data = poll_event; - - return NXT_UNIT_OK; -} - - -/** - * Callback to remove port - */ -static void -nxt_php_remove_port(nxt_unit_t *unit, nxt_unit_ctx_t *ctx, nxt_unit_port_t *port) -{ - zend_async_poll_event_t *poll_event; - - if (port->data != NULL) { - poll_event = (zend_async_poll_event_t *) port->data; - - /* Dispose using TrueAsync cleanup */ - poll_event->base.dispose(&poll_event->base); - - port->data = NULL; - } -} - - -/** - * SHM ACK handler - called when Router releases shared memory - */ -static void -nxt_php_shm_ack_handler(nxt_unit_ctx_t *ctx) -{ - nxt_php_async_ctx_data_t *async_data = ctx->data; - nxt_queue_link_t *lnk; - nxt_php_pending_write_t *pw; - ssize_t res; - size_t remaining; - - if (async_data == NULL) { - return; - } - - /* Process all pending writes in drain_queue */ - lnk = nxt_queue_first(&async_data->drain_queue); - - while (lnk != nxt_queue_tail(&async_data->drain_queue)) { - pw = nxt_container_of(lnk, nxt_php_pending_write_t, link); - lnk = nxt_queue_next(lnk); /* Save next element before potential removal */ - - remaining = ZSTR_LEN(pw->data) - pw->offset; - - /* Try to send remaining data */ - res = nxt_unit_response_write_nb(pw->req, - ZSTR_VAL(pw->data) + pw->offset, - remaining, - 0); - - if (res < 0) { - /* Error - remove from queue */ - nxt_unit_warn(ctx, "drain_queue: write error, removing pending write"); - nxt_php_drain_queue_remove(pw); - continue; - } - - if (res == 0) { - /* Still no space - stop processing */ - return; - } - - pw->offset += res; - - if (pw->offset >= ZSTR_LEN(pw->data)) { - /* ALL SENT! Remove from queue */ - nxt_php_drain_queue_remove(pw); - } - } -} - -#endif /* NXT_PHP_TRUEASYNC */ - - static nxt_int_t nxt_php_start(nxt_task_t *task, nxt_process_data_t *data) { @@ -699,7 +485,6 @@ nxt_php_start(nxt_task_t *task, nxt_process_data_t *data) nxt_php_app_conf_t *c; nxt_common_app_conf_t *conf; - conf = data->app; c = &conf->u.php; @@ -723,88 +508,33 @@ nxt_php_start(nxt_task_t *task, nxt_process_data_t *data) ret = nxt_php_set_target(task, &nxt_php_targets[n], value); if (nxt_slow_path(ret != NXT_OK)) { - return NXT_ERROR; + goto fail; } } } else { ret = nxt_php_set_target(task, &nxt_php_targets[0], conf->self); if (nxt_slow_path(ret != NXT_OK)) { - return NXT_ERROR; + goto fail; } } ret = nxt_unit_default_init(task, &php_init, conf); if (nxt_slow_path(ret != NXT_OK)) { nxt_alert(task, "nxt_unit_default_init() failed"); - return ret; + goto fail; } - /* Choose request handler based on mode */ -#if NXT_PHP_TRUEASYNC - if (c->async && c->entrypoint.length > 0) { - nxt_debug(task, "PHP HTTP Server mode enabled with entrypoint: %V", &c->entrypoint); - php_init.callbacks.add_port = nxt_php_add_port; - php_init.callbacks.remove_port = nxt_php_remove_port; - php_init.callbacks.request_handler = nxt_php_request_handler_async; - php_init.callbacks.quit = nxt_php_quit_handler; - php_init.callbacks.shm_ack_handler = nxt_php_shm_ack_handler; - - if (nxt_slow_path(nxt_php_async_load_entrypoint(task, &c->entrypoint) != NXT_OK)) { - nxt_alert(task, "failed to load entrypoint script"); - return NXT_ERROR; - } - - if (nxt_php_request_callback == NULL) { - nxt_alert(task, "TrueAsync: Request callback not registered in the entrypoint script!"); - return NXT_ERROR; - } - - if(nxt_slow_path(!nxt_php_activate_true_async(task))) { - return NXT_ERROR; - } - } else { -#endif /* NXT_PHP_TRUEASYNC */ - nxt_debug(task, "PHP standard mode"); - php_init.callbacks.request_handler = nxt_php_request_handler; -#if NXT_PHP_TRUEASYNC - } -#endif /* NXT_PHP_TRUEASYNC */ + php_init.callbacks.request_handler = nxt_php_request_handler; unit_ctx = nxt_unit_init(&php_init); if (nxt_slow_path(unit_ctx == NULL)) { - return NXT_ERROR; + goto fail; } nxt_php_unit_ctx = unit_ctx; -#if NXT_PHP_TRUEASYNC - /* Initialize async context data for drain_queue */ - if (c->async && c->entrypoint.length > 0) { - nxt_php_async_ctx_data_t *async_data; - - async_data = malloc(sizeof(nxt_php_async_ctx_data_t)); - if (async_data == NULL) { - nxt_alert(task, "Failed to allocate async context data"); - nxt_unit_done(unit_ctx); - return NXT_ERROR; - } - - nxt_queue_init(&async_data->drain_queue); - async_data->ctx = unit_ctx; - - unit_ctx->data = async_data; - } - - if (c->async && c->entrypoint.length > 0) { - /* Suspend main coroutine until the server continues to operate successfully */ - nxt_php_suspend_coroutine(nxt_php_unit_ctx); - } else { - nxt_unit_run(nxt_php_unit_ctx); - } -#else nxt_unit_run(nxt_php_unit_ctx); -#endif /* NXT_PHP_TRUEASYNC */ nxt_unit_done(nxt_php_unit_ctx); /* Clean up allocated memory before exit */ @@ -813,6 +543,12 @@ nxt_php_start(nxt_task_t *task, nxt_process_data_t *data) exit(0); return NXT_OK; + +fail: + + nxt_php_cleanup_targets(); + + return NXT_ERROR; } @@ -831,7 +567,6 @@ nxt_php_set_target(nxt_task_t *task, nxt_php_target_t *target, static const nxt_str_t root_str = nxt_string("root"); static const nxt_str_t script_str = nxt_string("script"); static const nxt_str_t index_str = nxt_string("index"); - static const nxt_str_t entrypoint_str = nxt_string("entrypoint"); value = nxt_conf_get_object_member(conf, &root_str, NULL); @@ -850,6 +585,7 @@ nxt_php_set_target(nxt_task_t *task, nxt_php_target_t *target, p = nxt_realpath(tmp); if (nxt_slow_path(p == NULL)) { nxt_alert(task, "root realpath(%s) failed %E", tmp, nxt_errno); + nxt_free(tmp); return NXT_ERROR; } @@ -869,7 +605,7 @@ nxt_php_set_target(nxt_task_t *task, nxt_php_target_t *target, tmp = nxt_malloc(target->root.length + 1 + str.length + 1); if (nxt_slow_path(tmp == NULL)) { - return NXT_ERROR; + goto fail; } p = tmp; @@ -884,7 +620,7 @@ nxt_php_set_target(nxt_task_t *task, nxt_php_target_t *target, if (nxt_slow_path(p == NULL)) { nxt_alert(task, "script realpath(%s) failed %E", tmp, nxt_errno); nxt_free(tmp); - return NXT_ERROR; + goto fail; } nxt_free(tmp); @@ -892,19 +628,20 @@ nxt_php_set_target(nxt_task_t *task, nxt_php_target_t *target, target->script_filename.length = nxt_strlen(p); target->script_filename.start = p; - if (!nxt_str_start(&target->script_filename, - target->root.start, target->root.length)) - { + if (!nxt_php_path_is_under(&target->root, &target->script_filename)) { nxt_alert(task, "script is not under php root"); nxt_free(p); - return NXT_ERROR; + nxt_str_null(&target->script_filename); + goto fail; } ret = nxt_php_dirname(&target->script_filename, &target->script_dirname); if (nxt_slow_path(ret != NXT_OK)) { nxt_free(target->script_filename.start); - return NXT_ERROR; + nxt_str_null(&target->script_filename); + nxt_str_null(&target->script_dirname); + goto fail; } target->script_name.length = target->script_filename.length @@ -913,74 +650,6 @@ nxt_php_set_target(nxt_task_t *task, nxt_php_target_t *target, + target->root.length; } else { - /* Check for entrypoint (async mode) */ - value = nxt_conf_get_object_member(conf, &entrypoint_str, NULL); - - if (value != NULL) { - nxt_conf_get_string(value, &str); - - /* Check if entrypoint is an absolute path */ - if (str.length > 0 && str.start[0] == '/') { - /* Absolute path - use it directly */ - tmp = nxt_malloc(str.length + 1); - if (nxt_slow_path(tmp == NULL)) { - return NXT_ERROR; - } - - nxt_memcpy(tmp, str.start, str.length); - tmp[str.length] = '\0'; - - } else { - /* Relative path - prepend root */ - nxt_php_str_trim_lead(&str, '/'); - - tmp = nxt_malloc(target->root.length + 1 + str.length + 1); - if (nxt_slow_path(tmp == NULL)) { - return NXT_ERROR; - } - - p = tmp; - - p = nxt_cpymem(p, target->root.start, target->root.length); - *p++ = '/'; - - p = nxt_cpymem(p, str.start, str.length); - *p = '\0'; - } - - p = nxt_realpath(tmp); - if (nxt_slow_path(p == NULL)) { - nxt_alert(task, "entrypoint realpath(%s) failed %E", tmp, nxt_errno); - nxt_free(tmp); - return NXT_ERROR; - } - - nxt_free(tmp); - - target->script_filename.length = nxt_strlen(p); - target->script_filename.start = p; - - if (!nxt_str_start(&target->script_filename, - target->root.start, target->root.length)) - { - nxt_alert(task, "entrypoint is not under php root"); - nxt_free(p); - return NXT_ERROR; - } - - ret = nxt_php_dirname(&target->script_filename, - &target->script_dirname); - if (nxt_slow_path(ret != NXT_OK)) { - nxt_free(p); - return NXT_ERROR; - } - - target->script_name.length = target->script_filename.length - - target->root.length; - target->script_name.start = target->script_filename.start - + target->root.length; - } - value = nxt_conf_get_object_member(conf, &index_str, NULL); if (value != NULL) { @@ -988,7 +657,7 @@ nxt_php_set_target(nxt_task_t *task, nxt_php_target_t *target, tmp = nxt_malloc(str.length + 1); if (nxt_slow_path(tmp == NULL)) { - return NXT_ERROR; + goto fail; } nxt_memcpy(tmp, str.start, str.length); @@ -1004,6 +673,13 @@ nxt_php_set_target(nxt_task_t *task, nxt_php_target_t *target, } return NXT_OK; + +fail: + + nxt_free(target->root.start); + nxt_str_null(&target->root); + + return NXT_ERROR; } @@ -1338,6 +1014,28 @@ nxt_php_str_trim_lead(nxt_str_t *str, u_char t) } +/* + * Checks that "path" names something below "root": "root", then '/', then + * at least one more byte. A plain prefix test would accept "/srv/app2/x.php" + * for the root "/srv/app". It would also accept the root itself. + * + * Both paths come from realpath(), so they are absolute and have no "." + * or ".." components, no symbolic links and no repeated or trailing '/'. + * nxt_php_str_trim_trail() trims the root "/" to length 0. Every absolute + * path except "/" itself is below that root. + */ + +static nxt_bool_t +nxt_php_path_is_under(const nxt_str_t *root, const nxt_str_t *path) +{ + return path->length > root->length + && path->length - root->length >= 2 + && path->start[root->length] == '/' + && (root->length == 0 + || memcmp(path->start, root->start, root->length) == 0); +} + + nxt_inline u_char * nxt_realpath(const void *c) { @@ -1423,12 +1121,14 @@ nxt_php_handle_fs_err(nxt_unit_request_info_t *req) static void nxt_php_request_handler(nxt_unit_request_info_t *req) { + uint8_t app_target; nxt_php_target_t *target; nxt_php_run_ctx_t ctx; nxt_unit_request_t *r; r = req->request; - target = &nxt_php_targets[r->app_target]; + app_target = r->app_target; + target = &nxt_php_targets[app_target]; nxt_memzero(&ctx, sizeof(ctx)); @@ -1445,11 +1145,12 @@ nxt_php_request_handler(nxt_unit_request_info_t *req) ctx.script_dirname = target->script_dirname; ctx.script_name = target->script_name; - ctx.chdir = (r->app_target != nxt_php_last_target); + ctx.chdir = (app_target != nxt_php_last_target); nxt_php_execute(&ctx, r); - nxt_php_last_target = ctx.chdir ? -1 : r->app_target; + /* nxt_php_execute() ended the request; r points to released memory. */ + nxt_php_last_target = ctx.chdir ? -1 : app_target; } @@ -1719,9 +1420,8 @@ nxt_php_unbuffered_write(const char *str, uint str_length TSRMLS_DC) ctx = SG(server_context); - /* During entrypoint execution there's no request context */ + /* Module startup, opcache.preload included, runs before any request. */ if (ctx == NULL || ctx->req == NULL) { - /* Log output from entrypoint to unit log */ nxt_unit_log(nxt_php_unit_ctx, NXT_UNIT_LOG_INFO, "PHP output: %.*s", (int)str_length, str); return str_length; @@ -1737,6 +1437,38 @@ nxt_php_unbuffered_write(const char *str, uint str_length TSRMLS_DC) } +/* + * PHP calls this for flush(), and after each write when implicit_flush is + * on. nxt_php_unbuffered_write() sends each write to the router at once. + * Thus only the response header can wait here. This function does not + * empty the output buffers of PHP; ob_flush() does that, as in mod_php. + */ + +static void +nxt_php_flush(void *server_context) +{ + nxt_php_run_ctx_t *ctx; + + ctx = server_context; + + /* + * Module startup has no context. fastcgi_finish_request() leaves the + * context without a request. + */ + if (ctx == NULL || ctx->req == NULL) { + return; + } + + /* + * If the send fails, SG(headers_sent) stays 0. The next output then + * tries again and handles the error, as it does without flush(). + */ + if (!SG(headers_sent)) { + (void) sapi_send_headers(TSRMLS_C); + } +} + + static int nxt_php_send_headers(sapi_headers_struct *sapi_headers TSRMLS_DC) { @@ -1751,7 +1483,7 @@ nxt_php_send_headers(sapi_headers_struct *sapi_headers TSRMLS_DC) ctx = SG(server_context); - /* During entrypoint execution there's no request context */ + /* Module startup, opcache.preload included, runs before any request. */ if (ctx == NULL || ctx->req == NULL) { return SAPI_HEADER_SENT_SUCCESSFULLY; } @@ -1832,7 +1564,10 @@ nxt_php_read_post(char *buffer, uint count_bytes TSRMLS_DC) ctx = SG(server_context); - /* During entrypoint execution there's no request context */ + /* + * fastcgi_finish_request() clears the request while PHP keeps running, + * and request_parse_body() can reach this callback afterwards. + */ if (ctx == NULL || ctx->req == NULL) { return 0; } @@ -1850,7 +1585,7 @@ nxt_php_read_cookies(TSRMLS_D) ctx = SG(server_context); - /* During entrypoint execution there's no request context */ + /* Module startup runs before any request. */ if (ctx == NULL) { return NULL; } @@ -1872,7 +1607,11 @@ nxt_php_register_variables(zval *track_vars_array TSRMLS_DC) ctx = SG(server_context); - /* During entrypoint execution there's no request context */ + /* + * PHP creates $_SERVER, and with it calls this callback, from script + * content rather than from request state, so it can be reached before + * the first request. + */ if (ctx == NULL || ctx->req == NULL) { return; } @@ -1984,130 +1723,6 @@ nxt_php_register_variables(zval *track_vars_array TSRMLS_DC) } -#if NXT_PHP_TRUEASYNC - -static void -nxt_php_register_variables_async(nxt_unit_request_info_t *req, - nxt_php_run_ctx_t *ctx, zval *track_vars_array TSRMLS_DC) -{ - const char *name; - char *str; - nxt_unit_field_t *f, *f_end; - nxt_unit_request_t *r; - - r = req->request; - - nxt_unit_req_debug(req, "nxt_php_register_variables_async"); - - /* Register SERVER_SOFTWARE */ - php_register_variable_safe((char *) "SERVER_SOFTWARE", - (char *) nxt_server.start, - nxt_server.length, track_vars_array TSRMLS_CC); - - /* Register SERVER_PROTOCOL */ - str = nxt_unit_sptr_get(&r->version); - php_register_variable_safe((char *) "SERVER_PROTOCOL", str, - r->version_length, track_vars_array TSRMLS_CC); - - /* Register PHP_SELF and PATH_INFO */ - if (ctx->path_info.length != 0) { - str = nxt_unit_sptr_get(&r->path); - php_register_variable_safe((char *) "PHP_SELF", str, - r->path_length, track_vars_array TSRMLS_CC); - php_register_variable_safe((char *) "PATH_INFO", - (char *) ctx->path_info.start, - ctx->path_info.length, track_vars_array TSRMLS_CC); - } else { - php_register_variable_safe((char *) "PHP_SELF", - (char *) ctx->script_name.start, - ctx->script_name.length, track_vars_array TSRMLS_CC); - } - - /* Register SCRIPT_NAME */ - php_register_variable_safe((char *) "SCRIPT_NAME", - (char *) ctx->script_name.start, - ctx->script_name.length, track_vars_array TSRMLS_CC); - - /* Register SCRIPT_FILENAME */ - php_register_variable_safe((char *) "SCRIPT_FILENAME", - (char *) ctx->script_filename.start, - ctx->script_filename.length, track_vars_array TSRMLS_CC); - - /* Register DOCUMENT_ROOT */ - php_register_variable_safe((char *) "DOCUMENT_ROOT", - (char *) ctx->root->start, - ctx->root->length, track_vars_array TSRMLS_CC); - - /* Register REQUEST_METHOD */ - str = nxt_unit_sptr_get(&r->method); - php_register_variable_safe((char *) "REQUEST_METHOD", str, - r->method_length, track_vars_array TSRMLS_CC); - - /* Register REQUEST_URI */ - str = nxt_unit_sptr_get(&r->target); - php_register_variable_safe((char *) "REQUEST_URI", str, - r->target_length, track_vars_array TSRMLS_CC); - - /* Register QUERY_STRING */ - str = nxt_unit_sptr_get(&r->query); - php_register_variable_safe((char *) "QUERY_STRING", str, - r->query_length, track_vars_array TSRMLS_CC); - - /* Register REMOTE_ADDR */ - str = nxt_unit_sptr_get(&r->remote); - php_register_variable_safe((char *) "REMOTE_ADDR", str, - r->remote_length, track_vars_array TSRMLS_CC); - - /* Register SERVER_ADDR */ - str = nxt_unit_sptr_get(&r->local_addr); - php_register_variable_safe((char *) "SERVER_ADDR", str, - r->local_addr_length, track_vars_array TSRMLS_CC); - - /* Register SERVER_NAME */ - str = nxt_unit_sptr_get(&r->server_name); - php_register_variable_safe((char *) "SERVER_NAME", str, - r->server_name_length, track_vars_array TSRMLS_CC); - - /* Register SERVER_PORT */ - str = nxt_unit_sptr_get(&r->local_port); - php_register_variable_safe((char *) "SERVER_PORT", str, - r->local_port_length, track_vars_array TSRMLS_CC); - - /* Register HTTPS if TLS is enabled */ - if (r->tls) { - php_register_variable_safe((char *) "HTTPS", (char *) "on", - 2, track_vars_array TSRMLS_CC); - } - - /* Register HTTP headers */ - f_end = r->fields + r->fields_count; - for (f = r->fields; f < f_end; f++) { - name = nxt_unit_sptr_get(&f->name); - str = nxt_unit_sptr_get(&f->value); - php_register_variable_safe((char *) name, str, - f->value_length, track_vars_array TSRMLS_CC); - } - - /* Register CONTENT_LENGTH */ - if (r->content_length_field != NXT_UNIT_NONE_FIELD) { - f = r->fields + r->content_length_field; - str = nxt_unit_sptr_get(&f->value); - php_register_variable_safe((char *) "CONTENT_LENGTH", str, - f->value_length, track_vars_array TSRMLS_CC); - } - - /* Register CONTENT_TYPE */ - if (r->content_type_field != NXT_UNIT_NONE_FIELD) { - f = r->fields + r->content_type_field; - str = nxt_unit_sptr_get(&f->value); - php_register_variable_safe((char *) "CONTENT_TYPE", str, - f->value_length, track_vars_array TSRMLS_CC); - } -} - -#endif /* NXT_PHP_TRUEASYNC */ - - static void nxt_php_set_sptr(nxt_unit_request_info_t *req, const char *name, nxt_unit_sptr_t *v, uint32_t len, zval *track_vars_array TSRMLS_DC) @@ -2208,6 +1823,7 @@ nxt_php_log_message(char *message TSRMLS_DC) ctx = SG(server_context); + /* fastcgi_finish_request() leaves the context without a request. */ if (ctx != NULL && ctx->req != NULL) { nxt_unit_req_log(ctx->req, NXT_UNIT_LOG_NOTICE, "php message: %s", message); @@ -2217,463 +1833,3 @@ nxt_php_log_message(char *message TSRMLS_DC) "php message: %s", message); } } - - -#if NXT_PHP_TRUEASYNC - -/* TrueAsync Mode Implementation */ - -/* External globals */ -extern zval *nxt_php_request_callback; - -/** - * The request handler invoked from NGINX UNIT. - * The handler uses a PHP callback function (nxt_php_request_callback) to process the request. - * - **/ -static void -nxt_php_request_handler_async(nxt_unit_request_info_t *req) -{ - zend_async_scope_t *request_scope; - zend_coroutine_t *coroutine; - nxt_php_run_ctx_t run_ctx; - nxt_php_target_t *target; - nxt_unit_request_t *r; - nxt_unit_field_t *f; - void *saved_server_context; - - r = req->request; - target = &nxt_php_targets[r->app_target]; - - /* 1. Create new Scope for this request with its own superglobals */ - request_scope = ZEND_ASYNC_NEW_SCOPE(ZEND_ASYNC_CURRENT_SCOPE); - - if (request_scope == NULL) { - nxt_unit_req_alert(req, "Failed to create request scope"); - nxt_unit_request_done(req, NXT_UNIT_ERROR); - return; - } - - /* 2. DISABLE inheriting global superglobals - each request has its own */ - ZEND_ASYNC_SCOPE_CLR_INHERIT_SUPERGLOBALS(request_scope); - - /* 3. Setup SG(server_context) and SG(request_info) for superglobals population */ - saved_server_context = SG(server_context); - - /* Prepare run context */ - nxt_memzero(&run_ctx, sizeof(run_ctx)); - run_ctx.req = req; - run_ctx.root = &target->root; - run_ctx.index = &target->index; - run_ctx.script_filename = target->script_filename; - run_ctx.script_dirname = target->script_dirname; - run_ctx.script_name = target->script_name; - - /* Extract cookie if present */ - if (r->cookie_field != NXT_UNIT_NONE_FIELD) { - f = r->fields + r->cookie_field; - run_ctx.cookie = nxt_unit_sptr_get(&f->value); - } - - /* Setup SG(server_context) */ - SG(server_context) = &run_ctx; - - /* Setup SG(request_info) for php_default_treat_data */ - SG(request_info).request_method = nxt_unit_sptr_get(&r->method); - SG(request_info).query_string = r->query.offset ? nxt_unit_sptr_get(&r->query) : NULL; - SG(request_info).content_length = r->content_length; - - if (r->content_type_field != NXT_UNIT_NONE_FIELD) { - f = r->fields + r->content_type_field; - SG(request_info).content_type = nxt_unit_sptr_get(&f->value); - } else { - SG(request_info).content_type = NULL; - } - - /* 4. Populate superglobals with data from HTTP request */ - nxt_php_scope_populate_superglobals(request_scope); - - /* 5. Restore original SG(server_context) */ - SG(server_context) = saved_server_context; - - /* 6. Create coroutine within this Scope */ - coroutine = ZEND_ASYNC_NEW_COROUTINE(request_scope); - if (coroutine == NULL) { - nxt_unit_req_alert(req, "Failed to create coroutine"); - /* Scope will be automatically cleaned up */ - nxt_unit_request_done(req, NXT_UNIT_ERROR); - return; - } - - /* 5. Setup coroutine entry point and data */ - coroutine->internal_entry = nxt_php_request_coroutine_entry; - coroutine->extended_data = req; - - /* 6. Enqueue coroutine for execution */ - if (!ZEND_ASYNC_ENQUEUE_COROUTINE(coroutine)) { - nxt_unit_req_alert(req, "Failed to enqueue coroutine"); - nxt_unit_request_done(req, NXT_UNIT_ERROR); - return; - } -} - -/* Load and execute entrypoint script in async mode */ -static nxt_int_t -nxt_php_async_load_entrypoint(nxt_task_t *task, nxt_str_t *entrypoint) -{ - FILE *fp; - const char *filename; - zend_file_handle file_handle; - - nxt_log(task, NXT_LOG_INFO, "TrueAsync: nxt_php_async_load_entrypoint called, PID=%d", getpid()); - - - /* Create null-terminated filename */ - char *filename_buf = nxt_malloc(entrypoint->length + 1); - if (filename_buf == NULL) { - nxt_alert(task, "TrueAsync: Failed to allocate filename buffer"); - return NXT_ERROR; - } - - nxt_memcpy(filename_buf, entrypoint->start, entrypoint->length); - filename_buf[entrypoint->length] = '\0'; - filename = filename_buf; - - - fp = fopen(filename, "re"); - if (fp == NULL) { - nxt_alert(task, "TrueAsync: Failed to open entrypoint %s", filename); - nxt_free(filename_buf); - return NXT_ERROR; - } - - /* Initialize minimal SG() fields like CLI does */ - SG(request_info).argc = 0; - SG(request_info).argv = NULL; - SG(request_info).path_translated = (char*)filename; - SG(server_context) = NULL; - - /* Call php_request_startup() like CLI does */ -#ifdef NXT_PHP7 - if (nxt_slow_path(php_request_startup() == FAILURE)) { -#else - if (nxt_slow_path(php_request_startup(TSRMLS_C) == FAILURE)) { -#endif - nxt_alert(task, "TrueAsync: php_request_startup() failed"); - fclose(fp); - nxt_free(filename_buf); - return NXT_ERROR; - } - - nxt_zend_stream_init_fp(&file_handle, fp, filename); - - /* Execute entrypoint script */ - int result = php_execute_script(&file_handle TSRMLS_CC); - - if (result == FAILURE) { - nxt_alert(task, "TrueAsync: php_execute_script() FAILED!"); - fclose(fp); - nxt_free(filename_buf); - return NXT_ERROR; - } - -#if (PHP_VERSION_ID >= 80100) - zend_destroy_file_handle(&file_handle); -#endif - - nxt_free(filename_buf); - - /* Check if callback was registered */ - if (nxt_php_request_callback == NULL) { - nxt_alert(task, "TrueAsync: No request handler registered! Call HttpServer->onRequest() in your entrypoint."); - return NXT_ERROR; - } - - /* Check callback is callable */ - if (nxt_php_request_callback != NULL && Z_TYPE_P(nxt_php_request_callback) == IS_OBJECT) { - nxt_log(task, NXT_LOG_INFO, "TrueAsync: Callback is object, type OK"); - } else { - nxt_alert(task, "TrueAsync: Callback has wrong type: %d", - nxt_php_request_callback ? Z_TYPE_P(nxt_php_request_callback) : -1); - } - - /* - * DON'T call php_request_shutdown() — we want the callback zval to - * persist across the prototype → worker fork. But scrub any - * pending exception left in EG: if the entrypoint script raised - * something that wasn't caught before HttpServer->onRequest() - * stored the callback, every forked worker would inherit the - * exception on its first request. The callback zval itself - * (nxt_php_request_callback) is not on the exception path; it - * was registered explicitly by the userland code. - * - * Other EG globals (output buffers, error_reporting, the symbol - * table) are also inherited, but those are reset implicitly when - * the worker enters a fresh request_init. Exception state is - * the one that bites hardest because the next request's - * php_execute_script() can early-exit on a stale EG(exception). - */ - if (EG(exception) != NULL) { - zend_clear_exception(); - } - - return NXT_OK; -} - - -/* Quit handler for graceful shutdown */ -static void -nxt_php_quit_handler(nxt_unit_ctx_t *ctx) -{ - nxt_unit_debug(ctx, "TrueAsync: quit handler called, triggering graceful shutdown"); - - /* Call ZEND_ASYNC_SHUTDOWN to trigger graceful shutdown of all coroutines */ - ZEND_ASYNC_SHUTDOWN(); -} - - -/* Server wait event methods */ -static bool -nxt_php_server_wait_event_start(zend_async_event_t *event) -{ - /* No action needed - event waits indefinitely */ - return true; -} - -static bool -nxt_php_server_wait_event_stop(zend_async_event_t *event) -{ - /* No action needed */ - return true; -} - -static bool -nxt_php_server_wait_event_add_callback(zend_async_event_t *event, zend_async_event_callback_t *callback) -{ - return zend_async_callbacks_push(event, callback); -} - -static bool -nxt_php_server_wait_event_del_callback(zend_async_event_t *event, zend_async_event_callback_t *callback) -{ - return zend_async_callbacks_remove(event, callback); -} - -static bool -nxt_php_server_wait_event_replay(zend_async_event_t *event, zend_async_event_callback_t *callback, zval *result, zend_object **exception) -{ - /* Event never resolves - cannot replay */ - return false; -} - -static zend_string * -nxt_php_server_wait_event_info(zend_async_event_t *event) -{ - return zend_string_init("NGINX Unit server waiting", sizeof("NGINX Unit server waiting") - 1, 0); -} - -static bool -nxt_php_server_wait_event_dispose(zend_async_event_t *event) -{ - if (ZEND_ASYNC_EVENT_REFCOUNT(event) > 1) { - ZEND_ASYNC_EVENT_DEL_REF(event); - return true; - } - - if (ZEND_ASYNC_EVENT_REFCOUNT(event) == 1) { - ZEND_ASYNC_EVENT_DEL_REF(event); - } - - /* Notify all callbacks that event is disposed */ - ZEND_ASYNC_CALLBACKS_NOTIFY(event, NULL, NULL); - - /* Free the event */ - efree(event); - - return true; -} - -static bool -nxt_php_activate_true_async(nxt_task_t *task) -{ - if (ZEND_ASYNC_IS_OFF) { - nxt_alert(task, "TrueAsync: async mode is off"); - return false; - } - - if (ZEND_ASYNC_IS_READY) { - if (!ZEND_ASYNC_SCHEDULER_LAUNCH()) { - nxt_alert(task, "TrueAsync: Failed to launch scheduler"); - return false; - } - } - - return ZEND_ASYNC_IS_ACTIVE; -} - -/** - * Suspend coroutine until application should finish - * Event handlers registered via add_port/remove_port will be triggered by scheduler - */ -static void -nxt_php_suspend_coroutine(nxt_unit_ctx_t *ctx) -{ - zend_coroutine_t *coroutine = ZEND_ASYNC_CURRENT_COROUTINE; - if (coroutine == NULL) { - nxt_unit_alert(ctx, "TrueAsync: Failed to get current coroutine"); - return; - } - - /* Create server wait event (it's custom nginx unit trigger for TrueAsync) */ - zend_async_event_t *event = emalloc(sizeof(zend_async_event_t)); - if (event == NULL) { - nxt_unit_alert(ctx, "TrueAsync: Failed to allocate server wait event"); - return; - } - - /* Initialize event structure */ - memset(event, 0, sizeof(zend_async_event_t)); - - /* Set event methods */ - event->start = nxt_php_server_wait_event_start; - event->stop = nxt_php_server_wait_event_stop; - event->add_callback = nxt_php_server_wait_event_add_callback; - event->del_callback = nxt_php_server_wait_event_del_callback; - event->replay = nxt_php_server_wait_event_replay; - event->info = nxt_php_server_wait_event_info; - event->dispose = nxt_php_server_wait_event_dispose; - - /* Create waker for coroutine */ - if (UNEXPECTED(zend_async_waker_new(coroutine) == NULL)) { - nxt_unit_alert(ctx, "TrueAsync: Failed to create waker"); - event->dispose(event); - return; - } - - /* Attach coroutine to wait event - it will suspend until GRACEFUL_SHUTDOWN */ - zend_async_resume_when(coroutine, event, true, zend_async_waker_callback_resolve, NULL); - - if (UNEXPECTED(EG(exception) != NULL)) { - nxt_unit_alert(ctx, "TrueAsync: Failed to attach coroutine to wait event"); - zend_async_waker_clean(coroutine); - event->dispose(event); - return; - } - - ZEND_ASYNC_SUSPEND(); - zend_async_waker_clean(coroutine); -} - - -static void -nxt_php_scope_init_superglobals(zend_async_scope_t *scope) -{ - zval tmp; - - if (scope->superglobals != NULL) { - return; - } - - ALLOC_HASHTABLE(scope->superglobals); - zend_hash_init(scope->superglobals, 8, NULL, ZVAL_PTR_DTOR, 0); - - array_init(&tmp); - zend_hash_str_add_new(scope->superglobals, "_GET", sizeof("_GET") - 1, &tmp); - - array_init(&tmp); - zend_hash_str_add_new(scope->superglobals, "_POST", sizeof("_POST") - 1, &tmp); - - array_init(&tmp); - zend_hash_str_add_new(scope->superglobals, "_COOKIE", sizeof("_COOKIE") - 1, &tmp); - - array_init(&tmp); - zend_hash_str_add_new(scope->superglobals, "_SERVER", sizeof("_SERVER") - 1, &tmp); - - array_init(&tmp); - zend_hash_str_add_new(scope->superglobals, "_ENV", sizeof("_ENV") - 1, &tmp); - - array_init(&tmp); - zend_hash_str_add_new(scope->superglobals, "_FILES", sizeof("_FILES") - 1, &tmp); - - array_init(&tmp); - zend_hash_str_add_new(scope->superglobals, "_REQUEST", sizeof("_REQUEST") - 1, &tmp); -} - - -static void -nxt_php_scope_populate_superglobals(zend_async_scope_t *scope) -{ - zval *server_array, *get_array, *post_array, *cookie_array; - nxt_php_run_ctx_t *ctx; - nxt_unit_request_t *r; - nxt_unit_field_t *f; - - if (scope->superglobals == NULL) { - nxt_php_scope_init_superglobals(scope); - } - - ctx = SG(server_context); - if (ctx == NULL || ctx->req == NULL) { - return; - } - - r = ctx->req->request; - - server_array = zend_hash_str_find(scope->superglobals, "_SERVER", sizeof("_SERVER") - 1); - get_array = zend_hash_str_find(scope->superglobals, "_GET", sizeof("_GET") - 1); - post_array = zend_hash_str_find(scope->superglobals, "_POST", sizeof("_POST") - 1); - cookie_array = zend_hash_str_find(scope->superglobals, "_COOKIE", sizeof("_COOKIE") - 1); - - /* Populate $_SERVER */ - if (server_array != NULL) { - nxt_php_register_variables_async(ctx->req, ctx, server_array); - } - - /* Populate $_GET - use PARSE_STRING to write into our array */ - if (get_array != NULL && r->query_length > 0) { - char *query = estrndup((char *)nxt_unit_sptr_get(&r->query), r->query_length); - php_default_treat_data(PARSE_STRING, query, get_array); - } - - /* Populate $_COOKIE - use PARSE_STRING to write into our array */ - if (cookie_array != NULL && ctx->cookie != NULL) { - char *cookie = estrdup(ctx->cookie); - php_default_treat_data(PARSE_STRING, cookie, cookie_array); - } - - /* Populate $_POST - use PARSE_STRING for application/x-www-form-urlencoded */ - if (post_array != NULL && - SG(request_info).request_method && - !strcasecmp(SG(request_info).request_method, "POST")) - { - /* Check if content type is application/x-www-form-urlencoded */ - if (r->content_type_field != NXT_UNIT_NONE_FIELD) { - f = r->fields + r->content_type_field; - const char *content_type = nxt_unit_sptr_get(&f->value); - - /* Only parse if it's form-urlencoded (simple POST data) */ - if (content_type && - strncasecmp(content_type, "application/x-www-form-urlencoded", 33) == 0) - { - /* Read POST body into memory */ - size_t post_len = r->content_length; - if (post_len > 0) { - char *post_data = nxt_malloc(post_len + 1); - if (post_data != NULL) { - size_t read_bytes = nxt_unit_request_read(ctx->req, post_data, post_len); - if (read_bytes > 0) { - post_data[read_bytes] = '\0'; - /* php_default_treat_data will modify and free the string, so use estrndup */ - char *post_copy = estrndup(post_data, read_bytes); - php_default_treat_data(PARSE_STRING, post_copy, post_array); - } - nxt_free(post_data); - } - } - } - /* For multipart/form-data and others - not supported yet in async mode */ - } - } -} - -#endif /* NXT_PHP_TRUEASYNC */ diff --git a/src/nxt_polarssl.c b/src/nxt_polarssl.c deleted file mode 100644 index 4a8144731..000000000 --- a/src/nxt_polarssl.c +++ /dev/null @@ -1,118 +0,0 @@ - -/* - * Copyright (C) NGINX, Inc. - * Copyright (C) Igor Sysoev - */ - -#include -#include -#include -#include -#include - - -typedef struct { - ssl_context ssl; - x509_cert certificate; - rsa_context key; -} nxt_polarssl_ctx_t; - - -static nxt_int_t nxt_polarssl_server_init(nxt_ssltls_conf_t *conf); -static void nxt_polarssl_conn_init(nxt_thread_t *thr, nxt_ssltls_conf_t *conf, - nxt_event_conn_t *c); -static void nxt_polarssl_log_error(nxt_uint_t level, nxt_log_t *log, int err, - const char *fmt, ...); - - -nxt_ssltls_lib_t nxt_polarssl_lib = { - nxt_polarssl_server_init, - NULL, -}; - - -static nxt_int_t -nxt_polarssl_server_init(nxt_ssltls_conf_t *conf) -{ - int n; - nxt_thread_t *thr; - nxt_polarssl_ctx_t *ctx; - - thr = nxt_thread(); - - /* TODO: mem_pool */ - - ctx = nxt_zalloc(sizeof(nxt_polarssl_ctx_t)); - if (ctx == NULL) { - return NXT_ERROR; - } - - n = ssl_init(&ctx->ssl); - if (n != 0) { - nxt_polarssl_log_error(NXT_LOG_ALERT, thr->log, n, "ssl_init() failed"); - return NXT_ERROR; - } - - ssl_set_endpoint(&ctx->ssl, SSL_IS_SERVER ); - - conf->ctx = ctx; - conf->conn_init = nxt_polarssl_conn_init; - - n = x509parse_crtfile(&ctx->certificate, conf->certificate); - if (n != 0) { - nxt_polarssl_log_error(NXT_LOG_ALERT, thr->log, n, - "x509parse_crt(\"%V\") failed", - &conf->certificate); - goto fail; - } - - rsa_init(&ctx->key, RSA_PKCS_V15, 0); - - n = x509parse_keyfile(&ctx->key, conf->certificate_key, NULL); - if (n != 0) { - nxt_polarssl_log_error(NXT_LOG_ALERT, thr->log, n, - "x509parse_key(\"%V\") failed", - &conf->certificate_key); - goto fail; - } - - ssl_set_own_cert(&ctx->ssl, &ctx->certificate, &ctx->key); - - /* TODO: ciphers */ - - /* TODO: ca_certificate */ - - return NXT_OK; - -fail: - - return NXT_ERROR; -} - - -static void -nxt_polarssl_conn_init(nxt_thread_t *thr, nxt_ssltls_conf_t *conf, - nxt_event_conn_t *c) -{ -} - - -static void -nxt_polarssl_log_error(nxt_uint_t level, nxt_log_t *log, int err, - const char *fmt, ...) -{ - va_list args; - u_char *p, *end, msg[NXT_MAX_ERROR_STR]; - - end = msg + NXT_MAX_ERROR_STR; - - va_start(args, fmt); - p = nxt_vsprintf(msg, end, fmt, args); - va_end(args); - - p = nxt_sprintf(p, end, " (%d: ", err); - - error_strerror(err, (char *) msg, p - msg); - - nxt_log_error(level, log, "%*s)", p - msg, msg); -} diff --git a/src/nxt_poll_engine.c b/src/nxt_poll_engine.c index f514e0a9a..63d126c33 100644 --- a/src/nxt_poll_engine.c +++ b/src/nxt_poll_engine.c @@ -34,6 +34,8 @@ static void nxt_poll_disable(nxt_event_engine_t *engine, nxt_fd_event_t *ev); static nxt_bool_t nxt_poll_close(nxt_event_engine_t *engine, nxt_fd_event_t *ev); +static void nxt_poll_cancel_changes(nxt_event_engine_t *engine, + nxt_fd_event_t *ev); static void nxt_poll_enable_read(nxt_event_engine_t *engine, nxt_fd_event_t *ev); static void nxt_poll_enable_write(nxt_event_engine_t *engine, @@ -73,6 +75,7 @@ const nxt_event_interface_t nxt_poll_engine = { nxt_poll_disable, nxt_poll_disable, nxt_poll_close, + nxt_poll_cancel_changes, nxt_poll_enable_read, nxt_poll_enable_write, nxt_poll_disable_read, @@ -318,6 +321,46 @@ nxt_poll_change(nxt_event_engine_t *engine, nxt_fd_event_t *ev, nxt_uint_t op, } +/* + * Take this event's pending changes out of the batch, so that a struct that + * is about to be freed is not dereferenced by nxt_poll_commit_changes(). + * + * The change is dropped rather than committed: the descriptor is closed, or + * is about to be, so the change would name a descriptor number that may + * already belong to somebody else. + */ + +static void +nxt_poll_cancel_changes(nxt_event_engine_t *engine, nxt_fd_event_t *ev) +{ + nxt_poll_change_t *change, *dst, *end; + + if (!ev->changing) { + return; + } + + dst = engine->u.poll.changes; + end = dst + engine->u.poll.nchanges; + + for (change = dst; change < end; change++) { + + if (change->event == ev) { + continue; + } + + if (dst != change) { + *dst = *change; + } + + dst++; + } + + engine->u.poll.nchanges = (nxt_uint_t) (dst - engine->u.poll.changes); + + ev->changing = 0; +} + + static nxt_int_t nxt_poll_commit_changes(nxt_event_engine_t *engine) { diff --git a/src/nxt_pollset_engine.c b/src/nxt_pollset_engine.c index e2c1a2d01..1d439b457 100644 --- a/src/nxt_pollset_engine.c +++ b/src/nxt_pollset_engine.c @@ -29,6 +29,8 @@ static void nxt_pollset_enable(nxt_event_engine_t *engine, nxt_fd_event_t *ev); static void nxt_pollset_disable(nxt_event_engine_t *engine, nxt_fd_event_t *ev); static nxt_bool_t nxt_pollset_close(nxt_event_engine_t *engine, nxt_fd_event_t *ev); +static void nxt_pollset_cancel_changes(nxt_event_engine_t *engine, + nxt_fd_event_t *ev); static void nxt_pollset_enable_read(nxt_event_engine_t *engine, nxt_fd_event_t *ev); static void nxt_pollset_enable_write(nxt_event_engine_t *engine, @@ -64,6 +66,7 @@ const nxt_event_interface_t nxt_pollset_engine = { nxt_pollset_disable, nxt_pollset_disable, nxt_pollset_close, + nxt_pollset_cancel_changes, nxt_pollset_enable_read, nxt_pollset_enable_write, nxt_pollset_disable_read, @@ -376,6 +379,46 @@ nxt_pollset_change(nxt_event_engine_t *engine, nxt_fd_event_t *ev, } +/* + * Take this event's pending changes out of the batch, so that a struct that + * is about to be freed is not dereferenced by nxt_pollset_commit_changes(). + * + * The change is dropped rather than committed: the descriptor is closed, or + * is about to be, so the change would name a descriptor number that may + * already belong to somebody else. + */ + +static void +nxt_pollset_cancel_changes(nxt_event_engine_t *engine, nxt_fd_event_t *ev) +{ + nxt_pollset_change_t *change, *dst, *end; + + if (!ev->changing) { + return; + } + + dst = engine->u.pollset.changes; + end = dst + engine->u.pollset.nchanges; + + for (change = dst; change < end; change++) { + + if (change->event == ev) { + continue; + } + + if (dst != change) { + *dst = *change; + } + + dst++; + } + + engine->u.pollset.nchanges = (int) (dst - engine->u.pollset.changes); + + ev->changing = 0; +} + + static nxt_int_t nxt_pollset_commit_changes(nxt_event_engine_t *engine) { diff --git a/src/nxt_port.c b/src/nxt_port.c index 15a6c59c9..29ad3e602 100644 --- a/src/nxt_port.c +++ b/src/nxt_port.c @@ -10,6 +10,7 @@ #include #include #include +#include static void nxt_port_remove_pid(nxt_task_t *task, nxt_port_recv_msg_t *msg, @@ -19,6 +20,73 @@ static void nxt_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); static nxt_atomic_uint_t nxt_port_last_id = 1; +/* + * Map a queue a peer sent over a port. + * + * mmap() succeeds when the descriptor refers to an object shorter than the + * mapping, and the first access past the object's last page raises SIGBUS + * in this process -- main, the prototype or the router, depending on which + * handler took the message. That is cheaper for a hostile peer than + * exhausting the descriptor table, so the size has to be validated before + * the mapping is made. + * + * Only a short object is refused. Both senders do size the queue with + * ftruncate() to exactly the size the receiver maps -- nxt_shm_open() in + * the runtime and nxt_unit_shm_open() in libunit -- but fstat() does not + * have to report that size back: a shm object is rounded up to a page on + * some systems, and the queue is not a whole number of pages, so requiring + * the exact size would refuse every legitimate queue there. A larger + * object is harmless -- only the first size bytes are mapped, and every + * access uses fixed offsets derived from the same type. + * + * The check does not make a hostile peer harmless: the sender keeps the + * descriptor and can shrink the object after the check, and neither + * MAP_POPULATE nor a probe read would close that window -- both only touch + * the pages before the shrink. Sealing the object is not available to the + * receiver, and is Linux-only. What the check does close is the whole + * class of short objects a peer can simply hand over, which is what the + * receiver can decide on its own. + */ + +void * +nxt_port_queue_mmap(nxt_task_t *task, nxt_fd_t fd, size_t size) +{ + void *mem; + struct stat queue_stat; + + if (nxt_slow_path(fstat(fd, &queue_stat) == -1)) { + nxt_log(task, NXT_LOG_WARN, "fstat(%FD) failed %E", fd, nxt_errno); + + return NULL; + } + + /* + * Only a short object is refused, not a differing one: the producers + * size the queue exactly, but fstat() does not have to report that. + * A shm object is rounded up to a page on some systems, and the queue + * is not a whole number of pages, so requiring equality would refuse + * every legitimate queue there. + */ + if (nxt_slow_path(queue_stat.st_size < (off_t) size)) { + nxt_log(task, NXT_LOG_WARN, "port queue on descriptor %FD is %O " + "bytes, less than the %uz it must hold", fd, + queue_stat.st_size, size); + + return NULL; + } + + mem = nxt_mem_mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + + if (nxt_slow_path(mem == MAP_FAILED)) { + nxt_log(task, NXT_LOG_WARN, "mmap(%FD) failed %E", fd, nxt_errno); + + return NULL; + } + + return mem; +} + + static void nxt_port_mp_cleanup(nxt_task_t *task, void *obj, void *data) { @@ -70,6 +138,7 @@ nxt_port_new(nxt_task_t *task, nxt_port_id_t id, nxt_pid_t pid, nxt_mp_cleanup(mp, nxt_port_mp_cleanup, task, port, mp); nxt_queue_init(&port->messages); + nxt_queue_init(&port->frag_queue); nxt_thread_mutex_create(&port->write_mutex); port->queue_fd = -1; @@ -123,20 +192,127 @@ nxt_port_close(nxt_task_t *task, nxt_port_t *port) } +/* + * Take a reference, unless the port has already dropped its last one. + * + * This is the counterpart of a lookup in process->ports: a reference derived + * from a pointer that was found rather than from one the caller already + * holds cannot use nxt_port_use(), because the port may be in teardown by + * the time the increment lands. Callers must hold rt->processes_mutex, + * which nxt_port_release() also holds while it unlinks the port, so that + * {find, ref} and {last drop, unlink} are mutually exclusive: a port still + * reachable through process->ports either still has a reference, or has + * already had its count latched at zero and fails here. + * + * Returns 1 when a reference was taken -- the caller then owns it and must + * drop it with nxt_port_use(task, port, -1), outside the mutex. + */ + +nxt_bool_t +nxt_port_use_unless_zero(nxt_port_t *port) +{ + nxt_atomic_int_t c; + + for ( ;; ) { + c = port->use_count; + + if (c <= 0) { + return 0; + } + + if (nxt_atomic_cmp_set(&port->use_count, c, c + 1)) { + return 1; + } + } +} + + static void nxt_port_release(nxt_task_t *task, nxt_port_t *port) { + nxt_runtime_t *rt; + nxt_debug(task, "port %p %d:%d release, type %d", port, port->pid, port->id, port->type); port->app = NULL; + /* + * port->socket is embedded in the port, so it dies with the memory pool + * released at the end of this function. A change queued for it is held + * by pointer in the engine's change batch and is dereferenced when that + * batch is committed, which happens at the top of the next poll at the + * latest -- after this release. nxt_port_rearm_now() queues exactly such + * a change, and nxt_port_write_msgs() calls it immediately before the + * nxt_port_use() that can bring the count to zero and reach here. + * + * Unlike a connection, a port has no close handler to defer its own + * teardown behind the next poll (nxt_conn_close_handler() does that with + * a zero timer), and it never deletes its event from the engine. So the + * pending change is dropped instead. + * + * A port with an engine is released on that engine's thread: the only + * other path through nxt_port_use() carries the last drop across with a + * work item. A port that never reached nxt_port_read_enable() or + * nxt_port_write_enable() has no engine and has queued nothing. + * + * ->socket.changing is the engines' own "a change may be buffered" bit. + * On kqueue it is only ever a maybe -- set when a change is queued and + * cleared only by the cancel, never by a flush -- so the test here can + * ask for a scan that finds nothing, but it never skips a cancel that + * was needed. + * + * port->engine is read as the engine the change was queued on, which it + * is everywhere but one path: nxt_router_thread_exit_handler() points a + * departed worker's port at the router's own engine before it drops the + * last reference. A change left in the departed engine's batch is not + * found by the scan below, and is not committed either -- that engine is + * never polled again and nxt_event_engine_free() releases it, batch and + * all, a few lines further on. + */ + + if (port->engine != NULL && port->socket.changing) { + nxt_fd_event_cancel_changes(port->engine, &port->socket); + } + if (port->link.next != NULL) { - nxt_assert(port->process != NULL); + rt = task->thread->runtime; + + /* + * The unlink happens under rt->processes_mutex so that it cannot + * race a lookup in process->ports on another engine -- see + * nxt_port_use_unless_zero(). use_count is zero here and stays + * zero, so every reader that reaches the port through the queue + * before the unlink takes the mutex and fails the try-ref, and every + * reader after it cannot reach the port at all. + * + * rt->processes_mutex must stay a leaf (src/nxt_process.c:150), so + * the process reference is dropped after the unlock: + * nxt_process_use() takes the same mutex itself. + */ + + nxt_thread_mutex_lock(&rt->processes_mutex); nxt_process_port_remove(port); - nxt_process_use(task, port->process, -1); + nxt_thread_mutex_unlock(&rt->processes_mutex); + + /* + * nxt_process_port_add() is the only production path that links a + * port, and it sets ->process and takes the reference with it. A + * port linked any other way (#425) has no reference to drop: this + * used to be an nxt_assert(), which release builds compile out, and + * nxt_process_use() then dereferenced NULL. The unlink above still + * has to happen -- the list would otherwise point into the pool + * freed below. + */ + if (nxt_fast_path(port->process != NULL)) { + nxt_process_use(task, port->process, -1); + + } else { + nxt_alert(task, "port %p %d:%d was linked to a process without " + "a process reference", port, port->pid, port->id); + } } nxt_mp_release(port->mem_pool); @@ -174,15 +350,28 @@ nxt_port_enable(nxt_task_t *task, nxt_port_t *port, * * The audit recommends rejecting application workers / prototypes * that forge cert/script/socket/access-log/etc. messages to the - * main process. A first draft was implemented but caused - * regressions in the isolation test suite — the kernel-validated - * sender PID (SCM_CREDENTIALS) is not consistently available on - * the existing socketpair setup, and the message-dispatch path - * runs before the prototype's process registration completes. + * main process. A first draft was reverted because the + * kernel-validated sender PID was not available on the existing + * socketpair setup. That blocker is gone: nxt_socketpair_create() + * sets SO_PASSCRED on both ends (and libunit does the same on its + * own sockets), NXT_HAVE_MSGHDR_CMSGCRED platforms get an SCM_CREDS + * control message attached to every send by + * nxt_socket_msg_oob_init(), and both receive paths in + * nxt_port_socket.c fill msg->cmsg_pid. Individual handlers are + * being gated on nxt_recv_msg_cmsg_pid() one at a time -- + * nxt_main_process_created_handler(), nxt_main_start_process_handler(), + * nxt_port_process_ready_handler(), nxt_proto_process_created_handler(), + * nxt_proto_start_process_handler() and the cert/script/socket/ + * access-log handlers already are. * - * Reverted to the pre-audit dispatch path here and deferred the - * ACL to a follow-up that introduces SO_PASSCRED on the relevant - * socketpairs and updates the process-registration ordering. + * What is left is the table-driven part: a per-message-type sender + * ACL applied here, in the dispatcher, so that a handler which + * forgets its own gate is still covered. That still needs the + * process-registration ordering sorted out -- dispatch can run + * before the prototype has finished registering a child, so the ACL + * cannot simply require an already registered sender. Platforms + * with neither SCM_CREDENTIALS nor SCM_CREDS (macOS, NetBSD, + * OpenBSD, illumos) stay unauthenticated either way. * See https://github.com/andypost/unit/pull/14 review thread. */ @@ -190,22 +379,37 @@ nxt_port_enable(nxt_task_t *task, nxt_port_t *port, static void nxt_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) { - nxt_port_handler_t *handlers; + nxt_port_handler_t handler, *handlers; if (nxt_fast_path(msg->port_msg.type < NXT_PORT_MSG_MAX)) { - nxt_debug(task, "port %d: message type:%uD fds:%d,%d", - msg->port->socket.fd, msg->port_msg.type, - msg->fd[0], msg->fd[1]); - handlers = msg->port->data; - handlers[msg->port_msg.type](task, msg); + handler = handlers[msg->port_msg.type]; - return; + /* + * The tables are designated initializers over a struct of named + * slots, and most of them set only a few: an in-range type whose + * slot the receiving process never filled is NULL, not a handler. + */ + if (nxt_fast_path(handler != NULL)) { + nxt_debug(task, "port %d: message type:%uD fds:%d,%d", + msg->port->socket.fd, msg->port_msg.type, + msg->fd[0], msg->fd[1]); + + handler(task, msg); + + return; + } } nxt_alert(task, "port %d: unknown message type:%uD", msg->port->socket.fd, msg->port_msg.type); + + /* + * The type is a byte off the wire, so any peer can name one that has + * no handler. Nothing runs to take the descriptors it attached. + */ + nxt_port_recv_msg_close_fds(msg); } @@ -216,30 +420,94 @@ nxt_port_quit_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) } +/* + * Announce a port to every peer the send matrix pairs it with, carrying + * "stream" so that the announcement doubles as the reply to the start RPC + * the initiator armed. + * + * Returns NXT_OK when the announcement that answers that RPC was accepted + * for delivery, so that the caller can read it as "the start has been + * answered". + * + * That is the announcement addressed to the router, and only that one. The + * router owns the registration for every start: it registers the RPC on its + * own port and then sends START_PROCESS elsewhere -- to main when a + * prototype has to be forked and to the prototype when a worker has, but the + * handler is armed on the router port in both cases + * (nxt_router_start_app_process_handler() registers on the router port it + * was called with, src/nxt_router.c:5069 and :459; + * nxt_router_app_prefork() on rt->port_by_type[NXT_PROCESS_ROUTER], + * src/nxt_router.c:3365). Main forks the prototype but never owns the RPC + * for it. nxt_router_new_port_handler() then feeds a stream-bearing + * NEW_PORT to nxt_port_rpc_handler() (src/nxt_router.c:788-791), which is + * what retires the registration -- when it gets that far. NXT_OK here means + * the announcement was queued for the router, not that the router acted on + * it: nxt_router_new_port_handler() returns at src/nxt_router.c:782 when it + * cannot map the new port's queue, walking past the RPC dispatch and leaving + * the start outstanding. Retiring the stream on a queued write is therefore + * the best answer available here and not a guarantee; issue #223 is where + * the router learns to fail the start it refused. + * + * Aggregating every peer instead would be wrong in the direction that costs + * correctness: when the prototype announces a worker it writes to main as + * well as to the router, and a failure to main with the router's write + * accepted would keep a stream the router has already retired -- exactly the + * wrapped-counter hazard the caller clears it to avoid. A failed write to + * any other peer is logged and does not change the answer; it is a delivery + * problem for that peer, not evidence about the start. + * + * With no router peer at all there is nothing to report on and NXT_OK is the + * honest answer: an initiator that is not in this runtime has no + * registration here to keep alive for. That is the boot-time shape, where + * main announces the core processes with stream 0 and clearing is a no-op. + */ + /* TODO join with process_ready and move to nxt_main_process.c */ -nxt_inline void +nxt_inline nxt_int_t nxt_port_send_new_port(nxt_task_t *task, nxt_runtime_t *rt, nxt_port_t *new_port, uint32_t stream) { + nxt_int_t ret; nxt_port_t *port; nxt_process_t *process; +#if (NXT_TESTS) + nxt_port_test_broadcasts++; +#endif + nxt_debug(task, "new port %d for process %PI", new_port->pair[1], new_port->pid); + ret = NXT_OK; + nxt_runtime_process_each(rt, process) { - if (process->pid == new_port->pid || process->pid == nxt_pid) { + if (process->pid == new_port->pid || process->pid == nxt_pid + || nxt_queue_is_empty(&process->ports)) + { continue; } port = nxt_process_port_first(process); if (nxt_proc_send_matrix[port->type][new_port->type]) { - (void) nxt_port_send_port(task, port, new_port, stream); + if (nxt_slow_path(nxt_port_send_port(task, port, new_port, stream) + != NXT_OK)) + { + if (port->type == NXT_PROCESS_ROUTER) { + ret = NXT_ERROR; + + } else { + nxt_log(task, NXT_LOG_WARN, "failed to announce the port " + "of process %PI to process %PI", new_port->pid, + port->pid); + } + } } } nxt_runtime_process_loop; + + return ret; } @@ -247,6 +515,7 @@ nxt_int_t nxt_port_send_port(nxt_task_t *task, nxt_port_t *port, nxt_port_t *new_port, uint32_t stream) { + nxt_int_t ret; nxt_buf_t *b; nxt_port_msg_new_port_t *msg; @@ -268,9 +537,50 @@ nxt_port_send_port(nxt_task_t *task, nxt_port_t *port, nxt_port_t *new_port, msg->max_share = port->max_share; msg->type = new_port->type; - return nxt_port_socket_write2(task, port, NXT_PORT_MSG_NEW_PORT, - new_port->pair[1], new_port->queue_fd, - stream, 0, b); + ret = nxt_port_socket_write2(task, port, NXT_PORT_MSG_NEW_PORT, + new_port->pair[1], new_port->queue_fd, + stream, 0, b); + + if (nxt_slow_path(ret != NXT_OK)) { + /* + * Still ours, and nobody else can reach it: b never leaves this + * function, so a caller that sees the failure has nothing to + * complete. The descriptors are borrowed from new_port and stay + * with it, as they do on the success path -- NEW_PORT carries no + * NXT_PORT_MSG_CLOSE_FD. + */ + + nxt_work_queue_add(&task->thread->engine->fast_work_queue, + b->completion_handler, task, b, b->parent); + } + + return ret; +} + + +/* + * Copies the body of a NEW_PORT message. The sender sets the size. A body + * shorter than nxt_port_msg_new_port_t is an error: the missing bytes would + * come from an earlier message in the same buffer. + */ + +nxt_int_t +nxt_port_new_port_msg(const nxt_port_recv_msg_t *msg, + nxt_port_msg_new_port_t *out) +{ + nxt_span_t span; + + if (nxt_slow_path(msg->buf == NULL)) { + return NXT_ERROR; + } + + nxt_span_init(&span, msg->buf->mem.pos, msg->buf->mem.free); + + if (nxt_slow_path(nxt_span_copy(&span, out, sizeof(*out)) != 0)) { + return NXT_ERROR; + } + + return NXT_OK; } @@ -279,33 +589,75 @@ nxt_port_new_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) { nxt_port_t *port; nxt_runtime_t *rt; - nxt_port_msg_new_port_t *new_port_msg; + nxt_port_msg_new_port_t new_port_msg; rt = task->thread->runtime; - new_port_msg = (nxt_port_msg_new_port_t *) msg->buf->mem.pos; + /* + * The message arrives on the stack of nxt_port_read_handler() with the + * union uninitialized, and every caller of this handler reads + * msg->u.new_port on return, so a path that creates no port has to say + * so rather than leave the garbage the stack happened to hold. + */ + msg->u.new_port = NULL; + msg->new_port_created = 0; - /* TODO check b size and make plain */ + if (nxt_slow_path(nxt_port_new_port_msg(msg, &new_port_msg) != NXT_OK)) { + nxt_alert(task, "process %PI sent a short new port message; refused", + msg->port_msg.pid); + + nxt_port_recv_msg_close_fds(msg); + return; + } nxt_debug(task, "new port %d received for process %PI:%d", - msg->fd[0], new_port_msg->pid, new_port_msg->id); + msg->fd[0], new_port_msg.pid, new_port_msg.id); - port = nxt_runtime_port_find(rt, new_port_msg->pid, new_port_msg->id); + port = nxt_runtime_port_find(rt, new_port_msg.pid, new_port_msg.id); if (port != NULL) { - nxt_debug(task, "port %PI:%d already exists", new_port_msg->pid, - new_port_msg->id); + nxt_debug(task, "port %PI:%d already exists", new_port_msg.pid, + new_port_msg.id); msg->u.new_port = port; - nxt_fd_close(msg->fd[0]); - msg->fd[0] = -1; + /* The socket is refused: the port keeps the pair it was built with. */ + if (msg->fd[0] != -1) { + nxt_fd_close(msg->fd[0]); + msg->fd[0] = -1; + } + + /* + * The queue descriptor is left to the caller unless the port has a + * queue already. An existing port without one is not an anomaly + * but the normal path in the main process: + * nxt_main_process_whoami_handler() creates every application port + * at WHOAMI time, before the NEW_PORT + * announcing that same port -- and its queue -- arrives, so this + * branch is the only place main can ever be handed the queue of a + * worker. Refusing fd[1] here outright leaves port->queue NULL, and + * a queueless sender falls back to a plain socket write that libunit + * parks waiting for a READ_SOCKET marker which never comes, stranding + * every CHANGE_FILE and QUIT main sends to its workers. + * + * A port that already has a queue does refuse the descriptor: mapping + * it again would re-point a live port's queue at memory this message + * chose, dropping the mapping the port was built with. Every caller + * maps fd[1] only while it is still set, so clearing it here + * suppresses that without racing them. + */ + if (port->queue != NULL && msg->fd[1] != -1) { + nxt_fd_close(msg->fd[1]); + msg->fd[1] = -1; + } + return; } - port = nxt_runtime_process_port_create(task, rt, new_port_msg->pid, - new_port_msg->id, - new_port_msg->type); + port = nxt_runtime_process_port_create(task, rt, new_port_msg.pid, + new_port_msg.id, + new_port_msg.type); if (nxt_slow_path(port == NULL)) { + nxt_port_recv_msg_close_fds(msg); return; } @@ -313,49 +665,412 @@ nxt_port_new_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) port->pair[0] = -1; port->pair[1] = msg->fd[0]; - port->max_size = new_port_msg->max_size; - port->max_share = new_port_msg->max_share; + + /* The port owns the descriptor now. */ + msg->fd[0] = -1; + + port->max_size = new_port_msg.max_size; + port->max_share = new_port_msg.max_share; port->socket.task = task; nxt_port_write_enable(task, port); msg->u.new_port = port; + msg->new_port_created = 1; + + /* + * fd[1] is deliberately left in the message: it is the queue of the + * port just created, and only the caller knows whether this process + * maps port queues at all and with which size. The same applies on + * the branch above when the port exists but has no queue yet. Every + * caller therefore has to close it once it is done -- which is why the + * refusing paths clear it instead, so that "still set" means "yours". + */ } /* TODO move to nxt_main_process.c */ void nxt_port_process_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) { + void *mem; + uint32_t stream; nxt_port_t *port; nxt_process_t *process; nxt_runtime_t *rt; rt = task->thread->runtime; + /* Unreferenced: main and prototype processes run a single engine. */ + + /* + * The lookup stays on msg->port_msg.pid: it is the key of the runtime + * hash. The kernel-validated sender PID cannot replace it, because the + * prototype installs this handler too, and there SCM_CREDENTIALS + * carries the namespace-local pid of a pid-isolated worker while the + * hash is keyed on the global one (see nxt_proto_process_created_handler(), + * which uses the two as distinct values, and nxt_process_create() + * refusing to register an isolated pid). It authenticates the resolved + * process instead, right below. + */ process = nxt_runtime_process_find(rt, msg->port_msg.pid); if (nxt_slow_path(process == NULL)) { + nxt_port_recv_msg_close_fds(msg); return; } - nxt_assert(process->state != NXT_PROCESS_STATE_READY); +#if (NXT_USE_CMSG_PID) + /* + * Authenticate the sender before anything else is read or written: the + * port socket of the main or the prototype process is duplicated into + * every child (nxt_proc_keep_matrix[]), the wire pid in the message is + * chosen by the sender, and the handler below re-points the named + * process's port at the queue this message carries. Without this gate + * any worker can announce a sibling and hand the receiver a shared + * memory queue of its own making. + * + * The credential is compared against process->isolated_pid, and only + * against it. isolated_pid is by construction the pid of the process + * as the receiver sees it -- the fork() return value in + * nxt_process_create(), left alone when nxt_proto_process_created_handler() + * rewrites process->pid to the global pid of a pid-isolated worker. + * Accepting process->pid as well would be a bypass: inside a pid + * namespace another task whose namespace-local pid happens to equal the + * victim's global pid would pass, and a compromised worker can fork + * towards such a collision. + * + * A non-positive isolated_pid means the record was not created by a + * fork() in this process -- nxt_runtime_process_get() only fills + * ->pid -- so there is nothing to authenticate against and no + * legitimate PROCESS_READY to lose: a worker announces itself to its + * parent prototype, and main's own children are all forked by main. + * Reject rather than compare, so that a cmsg_pid of 0 or the -1 that + * the shared-memory queue path leaves in place (see + * nxt_port_queue_read_handler()) cannot match by accident. + */ + if (nxt_slow_path(process->isolated_pid <= 0 + || nxt_recv_msg_cmsg_pid(msg) != process->isolated_pid)) + { + nxt_alert(task, "process %PI sent PROCESS_READY claiming process %PI", + nxt_recv_msg_cmsg_pid(msg), msg->port_msg.pid); + nxt_port_recv_msg_close_fds(msg); + return; + } +#endif - process->state = NXT_PROCESS_STATE_READY; + /* + * A repeated PROCESS_READY from the authenticated owner is not + * rejected: on ucred and cmsgcred platforms the gate above has already + * turned a forged repeat into a rejection, and a genuine one has to + * keep working -- the process may legitimately re-announce a new queue, + * and refusing it would strand the process on the mapping it has. It + * replaces the mapping instead, as it did before this check existed, + * and the previous mapping is released below rather than leaked. + * nxt_assert() is not used because it compiles out in release builds, + * leaving no diagnostic at all. + */ + if (nxt_slow_path(process->state == NXT_PROCESS_STATE_READY)) { + nxt_log(task, NXT_LOG_WARN, "repeated PROCESS_READY claiming " + "process %PI", msg->port_msg.pid); + } + + /* + * Every guard here rejects before the state is set: a message that is + * not acted upon must leave no trace. Marking a process ready and only + * then rejecting it would make the next, legitimate PROCESS_READY trip + * the guard above -- and, before the start stream was tied to the + * announcement at the tail of this handler, would have let + * nxt_main_process_sigchld_handler() clear the start stream of a process + * that never finished starting, dropping the REMOVE_PID that cancels the + * pending start RPC. + */ + if (nxt_slow_path(nxt_queue_is_empty(&process->ports))) { + nxt_log(task, NXT_LOG_WARN, "PROCESS_READY claiming process %PI, " + "which has no ports", msg->port_msg.pid); + nxt_port_recv_msg_close_fds(msg); + return; + } - nxt_assert(!nxt_queue_is_empty(&process->ports)); + /* + * The start already failed and the process was killed for it. Acting on + * a retransmission would signal that pid a second time, and once it has + * been reaped the pid may name an unrelated process. + */ + if (nxt_slow_path(process->start_failed)) { + nxt_log(task, NXT_LOG_WARN, "PROCESS_READY claiming process %PI, " + "whose start has already failed", msg->port_msg.pid); + nxt_port_recv_msg_close_fds(msg); + return; + } port = nxt_process_port_first(process); + if (msg->fd[0] != -1) { + mem = nxt_port_queue_mmap(task, msg->fd[0], sizeof(nxt_port_queue_t)); + + if (nxt_fast_path(mem != NULL)) { + /* + * Release what a previous PROCESS_READY installed, so that + * replacing the queue does not leak the old mapping and its + * descriptor. The size follows nxt_port_close() -- an app + * queue is mapped for the port with the reserved id. + */ + if (port->queue_fd != -1) { + nxt_fd_close(port->queue_fd); + port->queue_fd = -1; + } + + if (port->queue != NULL) { + nxt_mem_munmap(port->queue, + (port->id == (nxt_port_id_t) -1) + ? sizeof(nxt_app_queue_t) + : sizeof(nxt_port_queue_t)); + port->queue = NULL; + } + + port->queue_fd = msg->fd[0]; + port->queue = mem; + + /* The port owns the descriptor now. */ + msg->fd[0] = -1; + + } else if (port->queue != NULL) { + nxt_log(task, NXT_LOG_WARN, "process %PI ready: cannot map " + "the replacement queue, keeping the current one", + msg->port_msg.pid); + } + } + + /* + * One test for both ways an application worker can end up with no queue: + * a descriptor that could not be mapped, and no descriptor at all. + * + * The second is not hypothetical. Only libunit attaches a queue to + * PROCESS_READY, and it always does (nxt_unit.c:1050) -- the processes + * that legitimately send fd -1 are the core ones, which announce to main + * and are not NXT_PROCESS_APP. But at the receiver's RLIMIT_NOFILE the + * kernel delivers the payload and the credential while discarding + * SCM_RIGHTS and setting MSG_CTRUNC, which nothing here inspects, so an + * authenticated READY can arrive with its descriptor silently gone. + * Treating that as ready would broadcast the same unreachable port that + * issue #231 is about, by a different road. + * + * A queueless APP port is therefore a failed start either way, and the + * distinction that matters is not how the descriptor was lost but that + * the worker cannot be reached on it: no fallback to the socket exists, + * because a libunit process only delivers a socket message after + * dequeuing the READ_SOCKET marker that nxt_port_socket_write2() emits + * under port->queue != NULL. A queueless sender never emits the marker, + * its messages are suspended undelivered, and the second one fails the + * worker with "too many port socket messages". + * + * A repeated READY whose replacement mapping failed keeps the queue it + * has and is not a failed start -- it is excluded here by port->queue. + */ + + if (port->queue == NULL && port->type == NXT_PROCESS_APP) { +#if (NXT_USE_CMSG_PID) + /* + * The whole arm, not only the kill, is confined to the + * platforms that authenticate the sender. Every step of it + * is destructive -- it refuses the announcement, strands the + * process at CREATED and signals it -- and none of that is + * safe to do on a message that cannot be attributed. + * + * Acting on an unauthenticated message would also be worse + * than the bug it fixes: without the kill below there is + * nothing to reap the worker, so the prototype never sends + * the REMOVE_PID that fails the start, and the latch refuses + * every retransmission -- a start pending forever rather + * than one that fails. Elsewhere the old behaviour stands + * until NEW_PORT and PROCESS_READY carry provenance + * everywhere; that is issue #223. + */ + + nxt_alert(task, "process %PI ready: cannot map the queue; " + "failing the start", msg->port_msg.pid); + + /* + * The tail below is not reached, so close what the sender + * attached here instead: the descriptor that could not be + * mapped, and fd[1], which this handler never uses. + */ + nxt_port_recv_msg_close_fds(msg); + + /* + * A retransmitted PROCESS_READY must not signal again: the + * pid has been killed and, once reaped, may name an + * unrelated process. The state cannot carry this, because + * the process has to stay at CREATED. + */ + process->start_failed = 1; + + /* + * The kill only fails the start if somebody notices the + * death, and the prototype's SIGCHLD handler notifies the + * others for any state but CREATING + * (nxt_application.c:907). + * + * The record is normally CREATED by then: the worker sends + * PROCESS_CREATED first (nxt_process.c:918) and the + * prototype sets the state when it handles it + * (nxt_application.c:769). Both messages travel the same + * socketpair, so READY cannot overtake CREATED. It can + * however be LOST: if that write hits EAGAIN the message is + * buffered on the port (nxt_port_socket.c:331-359) and the + * worker then enters init->start and never returns to its + * loop to flush it, while libunit makes the descriptor + * blocking and sends READY regardless. The record is then + * still CREATING here. + * + * Advancing it is what guarantees the REMOVE_PID that + * carries the start stream; leaving it at CREATING would + * kill the worker and still leave the start pending, which + * is the wedge this arm exists to end. + * + * CREATED, not READY: it never became usable. The start + * stream survives either way -- this arm returns above the + * announcement at the tail of this handler, which is the + * only thing that clears it -- so the REMOVE_PID this + * guarantees still carries it. + */ + if (process->state == NXT_PROCESS_STATE_CREATING) { + process->state = NXT_PROCESS_STATE_CREATED; + } + + /* + * The process is left at CREATED because nothing about it + * ever became ready, and that costs the release nothing: + * an app worker announces itself to the prototype, whose + * SIGCHLD handler notifies the others whenever the state is + * anything but CREATING (nxt_application.c:907), so CREATED + * and READY travel the same path. The stream that REMOVE_PID + * carries -- the one the router turns into the start's + * RPC_ERROR -- was set by the prototype when it forked the + * worker (nxt_application.c:672) and is untouched here. + * + * isolated_pid is the pid as this receiver sees it, which + * inside a pid namespace is a different number from the + * global one; nxt_process.c uses the same identity rule + * when it kills a child whose cgroup setup failed. + * + * The positive test is redundant -- the sender gate above + * rejects a non-positive isolated_pid before this arm is + * reachable, under the same #if -- and is kept only because + * what it guards is kill(0, SIGKILL), which would signal + * the caller's whole process group. A later change that + * moved or relaxed that gate would otherwise turn this into + * exactly that. + */ + if (nxt_fast_path(process->isolated_pid > 0)) { + if (kill(process->isolated_pid, SIGKILL) == -1) { + /* + * ESRCH is ordinary here: the process may already + * have exited with a SIGCHLD still pending. + */ + nxt_log(task, (nxt_errno == ESRCH) ? NXT_LOG_INFO + : NXT_LOG_ALERT, + "kill(%PI, SIGKILL) failed for a process " + "whose queue could not be mapped %E", + process->isolated_pid, nxt_errno); + } + } + + return; +#else + /* + * See above: with no kernel-validated sender pid the start + * cannot be failed safely, so the announcement stands and + * the port stays unreachable. This is issue #231 as it was. + */ + nxt_alert(task, "process %PI ready: cannot map the queue; " + "the process cannot be reached on this port", + msg->port_msg.pid); +#endif + } + + /* + * Set below the mapping, not above it: a PROCESS_READY that cannot be + * acted upon must leave the process at CREATED (see the failure arm). + * Core processes -- router, controller, prototype -- send fd -1 and skip + * the block entirely, so they become READY here as they always did. + */ + process->state = NXT_PROCESS_STATE_READY; + nxt_debug(task, "process %PI ready", msg->port_msg.pid); - if (msg->fd[0] != -1) { - port->queue_fd = msg->fd[0]; - port->queue = nxt_mem_mmap(NULL, sizeof(nxt_port_queue_t), - PROT_READ | PROT_WRITE, MAP_SHARED, - msg->fd[0], 0); + /* + * Close whatever the sender attached and the port did not take over: + * fd[1] is never used here, and fd[0] survives a failed mapping. + */ + nxt_port_recv_msg_close_fds(msg); + + /* + * Retire the start stream, but only once the announcement that answers + * it has actually gone out. + * + * ->stream is the RPC the initiator armed for this start. While it is + * set, nxt_port_remove_notify_others() puts it into the REMOVE_PID that + * reports this process's death, and nxt_router_remove_pid_handler() + * turns a stream-bearing REMOVE_PID into an RPC_ERROR + * (src/nxt_router.c). That is the right fallback for a start + * that never completed, and a liability afterwards: stream identifiers + * come from one 32-bit counter (nxt_stream_ident, src/nxt_port_rpc.c, + * bumped in nxt_port_rpc_register_handler_ex()) that every request also + * draws on (nxt_router_process_http_request()), so once it wraps, an + * ordinary worker exit would fail whatever live RPC has inherited the + * number. The reachable collision set is small -- the retype lands on + * the router's main port, which holds start, prefork, listen-socket and + * access-log registrations, while request RPCs live on the worker + * threads' engine ports -- but it is not empty. + * + * Zeroing on the READY state alone is what this deliberately is not. + * The state is set above and the announcement is sent here, and in + * between the start RPC has not been retired by anything -- so a + * PROCESS_READY whose NEW_PORT could not be written would lose both the + * reply and the REMOVE_PID fallback, and leave the initiator waiting + * forever. That is the wedge issue #231 is about, re-entered through a + * corner. Keyed on the send instead, the stream survives exactly the + * cases that still need it. See issue #271. + * + * nxt_main_process_sigchld_handler() used to do this, later and on the + * state; it no longer needs to, and main gets the same treatment here + * because it runs the same handler for its own children. + * + * The stream announced is process->stream, the value this process itself + * registered when it forked the child (nxt_main_start_process_handler() + * and nxt_proto_start_process_handler()), not the one the message + * carries. A child echoes the stream it inherited across the fork + * (nxt_process_send_ready(), src/nxt_process.c:1203), so for a healthy + * child the two are the same number -- but the wire field is written by + * the sender, and the sender gate above authenticates who sent the + * message, not what it says. A child that named a different stream, or + * zero, would otherwise have the announcement carry a number no + * registration is waiting on while this cleared the one that is: the + * router would never retire the real start, and the REMOVE_PID that is + * the remaining fallback would carry nothing. Announcing the registered + * value and clearing that same value keeps the reply and the fallback + * describing one start. + * + * A mismatch is logged rather than refused. Refusing would strand the + * start the receiver knows about, which is the failure this is avoiding; + * announcing the authoritative stream answers it correctly and leaves a + * record that a child sent something it should not have. + */ + stream = process->stream; + + if (nxt_slow_path(msg->port_msg.stream != stream)) { + nxt_log(task, NXT_LOG_WARN, "process %PI sent PROCESS_READY naming " + "start stream #%uD, but it was started for stream #%uD; " + "answering the latter", msg->port_msg.pid, + msg->port_msg.stream, stream); } - nxt_port_send_new_port(task, rt, port, msg->port_msg.stream); + if (nxt_fast_path(nxt_port_send_new_port(task, rt, port, stream) + == NXT_OK)) + { + process->stream = 0; + } } @@ -370,10 +1085,23 @@ nxt_port_mmap_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) if (nxt_slow_path(msg->fd[0] == -1)) { nxt_log(task, NXT_LOG_WARN, "invalid fd passed with mmap message"); + /* + * A message can carry two descriptors whichever its type needs, so + * a missing fd[0] does not mean the message carried nothing. + */ + nxt_port_recv_msg_close_fds(msg); return; } - process = nxt_runtime_process_find(rt, msg->port_msg.pid); + /* + * Referenced, not just found: this handler is in the router worker port + * handler table, so it runs on engines other than the one that can free + * the process. nxt_port_incoming_port_mmap() takes and releases + * process->incoming.mutex without touching the reference count, so the + * reference has to span the whole call. + */ + + process = nxt_runtime_process_ref(rt, msg->port_msg.pid); if (nxt_slow_path(process == NULL)) { nxt_log(task, NXT_LOG_WARN, "failed to get process #%PI", msg->port_msg.pid); @@ -383,9 +1111,21 @@ nxt_port_mmap_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) nxt_port_incoming_port_mmap(task, process, msg->fd[0]); + /* + * Before fail_close: that label is shared with the process == NULL path, + * where no reference was taken. + */ + + nxt_process_use(task, process, -1); + fail_close: - nxt_fd_close(msg->fd[0]); + /* + * nxt_port_incoming_port_mmap() maps the segment rather than keeping the + * descriptor, so fd[0] is released on the success path too. fd[1] is + * never used by this handler and was leaked on every one of them. + */ + nxt_port_recv_msg_close_fds(msg); } @@ -415,8 +1155,15 @@ nxt_port_change_log_file(nxt_task_t *task, nxt_runtime_t *rt, nxt_uint_t slot, b->mem.free = nxt_cpymem(b->mem.free, &slot, sizeof(nxt_uint_t)); - (void) nxt_port_socket_write(task, port, NXT_PORT_MSG_CHANGE_FILE, - fd, 0, 0, b); + if (nxt_slow_path(nxt_port_socket_write(task, port, + NXT_PORT_MSG_CHANGE_FILE, + fd, 0, 0, b) != NXT_OK)) + { + /* Still ours: the port layer takes the buffer only on NXT_OK. */ + + nxt_work_queue_add(&task->thread->engine->fast_work_queue, + b->completion_handler, task, b, b->parent); + } } nxt_runtime_process_loop; } @@ -425,7 +1172,9 @@ nxt_port_change_log_file(nxt_task_t *task, nxt_runtime_t *rt, nxt_uint_t slot, void nxt_port_change_log_file_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) { + size_t size; nxt_buf_t *b; + nxt_int_t ret; nxt_uint_t slot; nxt_file_t *log_file; nxt_runtime_t *rt; @@ -433,18 +1182,82 @@ nxt_port_change_log_file_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) rt = task->thread->runtime; b = msg->buf; - slot = *(nxt_uint_t *) b->mem.pos; + /* + * The payload is whatever the sender chose to write: a message with a + * short -- or empty -- buffer would have the slot read past its end. + * The size the sender declared is what nxt_port_read_msg_process() + * advanced b->mem.free by, so the used size is the only bound here. + * nxt_assert() is not used because it compiles out in release builds, + * which is exactly where this message arrives unauthenticated. + */ + size = 0; + + if (nxt_fast_path(b != NULL)) { + size = (size_t) nxt_buf_mem_used_size(&b->mem); + } + + if (nxt_slow_path(size < sizeof(nxt_uint_t))) { + nxt_log(task, NXT_LOG_WARN, "CHANGE_FILE with a %uz byte payload, " + "which is too short to name a log file slot", size); + + nxt_port_recv_msg_close_fds(msg); + return; + } + + /* The payload is not aligned for a nxt_uint_t load. */ + nxt_memcpy(&slot, b->mem.pos, sizeof(nxt_uint_t)); + + /* + * nxt_list_elt() returns NULL past the end of the list, and every + * process that installs this handler -- the discovery and prototype + * processes, the router and the controller -- would then dereference + * it. + */ log_file = nxt_list_elt(rt->log_files, slot); + if (nxt_slow_path(log_file == NULL)) { + nxt_log(task, NXT_LOG_WARN, "CHANGE_FILE claiming log file slot " + "%ui, which does not exist", slot); + + nxt_port_recv_msg_close_fds(msg); + return; + } + + if (nxt_slow_path(msg->fd[0] == -1)) { + nxt_log(task, NXT_LOG_WARN, "CHANGE_FILE claiming log file slot " + "%ui without a descriptor to redirect it to", slot); + + nxt_port_recv_msg_close_fds(msg); + return; + } + nxt_debug(task, "change log file %FD:%FD", msg->fd[0], log_file->fd); + /* + * A second descriptor is never used on this path, but a message can + * carry one whatever its type: close it before the redirect, so that + * neither outcome of the redirect leaves it behind. + */ + if (msg->fd[1] != -1) { + nxt_fd_close(msg->fd[1]); + msg->fd[1] = -1; + } + /* * The old log file descriptor must be closed at the moment when no * other threads use it. dup2() allows to use the old file descriptor * for new log file. This change is performed atomically in the kernel. + * + * nxt_file_redirect() consumes the descriptor on every outcome, so drop + * it from the message rather than letting close_fds() reach a number + * that has already been reused. */ - if (nxt_file_redirect(log_file, msg->fd[0]) == NXT_OK) { + ret = nxt_file_redirect(log_file, msg->fd[0]); + + msg->fd[0] = -1; + + if (nxt_fast_path(ret == NXT_OK)) { if (slot == 0) { (void) nxt_file_stderr(log_file); } @@ -509,8 +1322,17 @@ nxt_port_remove_notify_others(nxt_task_t *task, nxt_process_t *process) buf->mem.free = nxt_cpymem(buf->mem.free, &pid, sizeof(pid)); - nxt_port_socket_write(task, port, NXT_PORT_MSG_REMOVE_PID, -1, - process->stream, 0, buf); + if (nxt_slow_path(nxt_port_socket_write(task, port, + NXT_PORT_MSG_REMOVE_PID, -1, + process->stream, 0, buf) + != NXT_OK)) + { + /* Still ours: the port layer takes the buffer only on NXT_OK. */ + + nxt_work_queue_add(&task->thread->engine->fast_work_queue, + buf->completion_handler, task, buf, + buf->parent); + } } nxt_runtime_process_loop; } @@ -547,10 +1369,21 @@ nxt_port_remove_pid(nxt_task_t *task, nxt_port_recv_msg_t *msg, nxt_port_rpc_remove_peer(task, msg->port, pid); - process = nxt_runtime_process_find(rt, pid); + /* + * Referenced even though this only runs on main engines: the constraint + * that matters here is ownership, not which thread runs. A router worker + * can drop the last reference to the process at any point, and + * nxt_process_close_ports() takes its own reference around the port loop + * -- on a process already at zero that would be a second drop to zero and + * so a second teardown, racing the one already posted to this engine. + */ + + process = nxt_runtime_process_ref(rt, pid); if (process) { nxt_process_close_ports(task, process); + + nxt_process_use(task, process, -1); } } @@ -623,27 +1456,117 @@ nxt_port_post(nxt_task_t *task, nxt_port_t *port, static void -nxt_port_release_handler(nxt_task_t *task, nxt_port_t *port, void *data) +nxt_port_release_work_handler(nxt_task_t *task, void *obj, void *data) { - /* no op */ + /* + * Drop the reference nxt_port_use() handed to this item, rather than + * releasing outright: this runs on port->engine, so the drop takes the + * on-engine branch of nxt_port_use() and releases only if nobody else + * took a reference while the item was in flight. That re-check is the + * whole point of holding a reference across the post -- see + * nxt_port_use(). + */ + + nxt_port_use(task, obj, -1); } void nxt_port_use(nxt_task_t *task, nxt_port_t *port, int i) { - int c; - - c = nxt_atomic_fetch_add(&port->use_count, i); + nxt_atomic_int_t c; - if (i < 0 && c == -i) { + if (i >= 0 + || port->engine == NULL + || task->thread->engine == port->engine) + { + c = nxt_atomic_fetch_add(&port->use_count, i); - if (port->engine == NULL || task->thread->engine == port->engine) { + if (i < 0 && c == -i) { nxt_port_release(task, port); + } - return; + return; + } + + /* + * A drop on another thread cannot release the port here -- the release + * frees port->mem_pool, and may drop the port's process reference, + * neither of which this thread owns -- so the last one is carried to + * port->engine by a work item. + * + * The item is embedded in the port and posted straight to the engine + * rather than routed through nxt_port_post(), which allocates one with + * nxt_zalloc() and can return NXT_ERROR. This call site has nowhere to + * put that error: the reference it is dropping is the last, so refusing + * to defer would leak the port, its memory pool and the process + * reference it holds -- unbounded, under exactly the memory pressure + * that caused the failure -- while releasing here would free another + * engine's memory from this thread, which is what the deferral exists + * to prevent. A deferral that cannot allocate cannot fail. This + * mirrors nxt_runtime_process_release(). + * + * The reference is handed to the item instead of being given up: the + * count is left standing at 1 and the posted handler drops it on + * port->engine. The port stays reachable through process->ports until + * then, so another engine can still take a reference in that window -- + * nxt_process_broadcast_shm_ack() walks a process's ports through bare + * pointers and nxt_port_socket_write() takes a reference on each -- and + * only a drop re-checked at the far end can tell that apart from a port + * nobody wants. Releasing from the handler unconditionally would free + * a port somebody holds. Routing through nxt_port_post() used to + * provide exactly this: it took a reference of its own and ended in + * nxt_port_use(port, -1). + * + * The hand-over is why the loop below is a compare-and-set that stores + * the item's one reference in place of the last drop, rather than a + * fetch-and-add to zero followed by a compensating increment. use_count + * reaches zero only on port->engine, immediately before + * nxt_port_release(), so while the item is in flight no other thread can + * observe a last drop: a reference taken in that window takes the count + * to 2, and the holder's drop finds the item's own reference still + * standing, so it is not the last and cannot reach this branch. At most + * one post is therefore ever in flight, and the item cannot be posted + * twice. That matters because nxt_locked_work_queue_add() links the + * item onto the queue tail: an item posted while it is already queued + * becomes its own successor, and the engine draining the queue then + * spins forever. + * + * The item becomes free again only once the handler's drop can run, and + * that is after nxt_locked_work_queue_move() has taken it off the + * locked queue and copied it into the engine's own work queue -- which + * is also why the release may free the pool the item lives in. + */ + + for ( ;; ) { + c = port->use_count; + + if (c != -i) { + nxt_assert(c > -i); + + if (nxt_atomic_cmp_set(&port->use_count, c, c + i)) { + return; + } + + continue; } - nxt_port_post(task, port, nxt_port_release_handler, NULL); + /* + * Exactly one reference is left standing, whatever the size of the + * batch this drop carries, because the item's handler drops exactly + * one. + */ + + if (nxt_atomic_cmp_set(&port->use_count, c, 1)) { + break; + } } + + port->release_work.handler = nxt_port_release_work_handler; + port->release_work.task = &port->engine->task; + port->release_work.obj = port; + port->release_work.data = NULL; + port->release_work.next = NULL; + + nxt_event_engine_post(port->engine, &port->release_work); } diff --git a/src/nxt_port.h b/src/nxt_port.h index d859ebcec..4caf36194 100644 --- a/src/nxt_port.h +++ b/src/nxt_port.h @@ -63,6 +63,26 @@ struct nxt_port_handlers_s { nxt_port_handler_t shm_ack; nxt_port_handler_t read_queue; nxt_port_handler_t read_socket; + + /* + * An application that answered a request and kept running, and later + * that it has finished. Appended, and appended is the only safe edit + * here: every _NXT_PORT_MSG_* value is this struct's offset, so + * inserting or reordering a slot renumbers the wire protocol. + */ + nxt_port_handler_t detached; + + /* + * A prototype reports a child that died before PROCESS_CREATED, by its + * namespace-local pid. Appended for the same reason as the slot above. + */ + nxt_port_handler_t remove_child_pid; + + /* The controller asks main to store a certificate bundle. Appended. */ + nxt_port_handler_t cert_store; + + /* The controller asks main to store an njs module. Appended. */ + nxt_port_handler_t script_store; }; @@ -120,6 +140,14 @@ typedef enum { _NXT_PORT_MSG_READ_QUEUE = nxt_port_handler_idx(read_queue), _NXT_PORT_MSG_READ_SOCKET = nxt_port_handler_idx(read_socket), + _NXT_PORT_MSG_DETACHED = nxt_port_handler_idx(detached), + + _NXT_PORT_MSG_REMOVE_CHILD_PID + = nxt_port_handler_idx(remove_child_pid), + + _NXT_PORT_MSG_CERT_STORE = nxt_port_handler_idx(cert_store), + _NXT_PORT_MSG_SCRIPT_STORE = nxt_port_handler_idx(script_store), + NXT_PORT_MSG_MAX = sizeof(nxt_port_handlers_t) / sizeof(nxt_port_handler_t), @@ -164,9 +192,24 @@ typedef enum { NXT_PORT_MSG_SHM_ACK = nxt_msg_last(_NXT_PORT_MSG_SHM_ACK), NXT_PORT_MSG_READ_QUEUE = _NXT_PORT_MSG_READ_QUEUE, NXT_PORT_MSG_READ_SOCKET = _NXT_PORT_MSG_READ_SOCKET, + NXT_PORT_MSG_DETACHED = nxt_msg_last(_NXT_PORT_MSG_DETACHED), + NXT_PORT_MSG_REMOVE_CHILD_PID + = nxt_msg_last(_NXT_PORT_MSG_REMOVE_CHILD_PID), + NXT_PORT_MSG_CERT_STORE = nxt_msg_last(_NXT_PORT_MSG_CERT_STORE), + NXT_PORT_MSG_SCRIPT_STORE = nxt_msg_last(_NXT_PORT_MSG_SCRIPT_STORE), } nxt_port_msg_type_t; +/* + * The message numbers are on the wire, and a language module built from + * another release reads the same numbers. Pin the last slot, so that a slot + * inserted before it fails the build. Add new slots after it and move the + * pin to the new last slot. + */ +nxt_static_assert(_NXT_PORT_MSG_SCRIPT_STORE == 37, + "a port message slot was inserted, not appended"); + + /* * Wire-format payload for NXT_PORT_MSG_QUIT. A single byte selects * between fast and graceful exit on the receiving side. When the @@ -179,6 +222,31 @@ typedef enum { } nxt_port_quit_mode_t; +/* + * Wire-format payload for NXT_PORT_MSG_DETACHED. A single byte says which + * edge this is: an application that has answered a request and is still + * running, or the same application reporting that work done. + * + * A byte rather than a flag on nxt_port_msg_t: that header has no spare + * bit that is reliably zeroed. Its four "1 bit" fields are whole bytes, + * the trailing pad byte is never cleared by the senders that build the + * header field by field, and nxt_port_socket_write() ORs into ->last. A + * new type is bounds-checked on both sides instead, so an older peer + * refuses the message rather than misreading a flag. + * + * The two edges are a balanced pair: one start and one finish per unit of + * detached work. The payload names neither a request nor a context, so the + * router can only count the edges per process; it takes the worker out of + * the idle economy on the first start and gives it back on the last finish. + * The payload may grow later, and the handler requires at least one byte and + * reads the first, so an older sender stays readable. + */ +typedef enum { + NXT_PORT_DETACHED_START = 0, + NXT_PORT_DETACHED_FINISH = 1, +} nxt_port_detached_t; + + /* Passed as a first iov chunk. */ typedef struct { uint32_t stream; @@ -211,6 +279,15 @@ typedef struct { nxt_port_msg_t port_msg; uint8_t close_fd; /* 1 bit */ uint8_t allocated; /* 1 bit */ + + /* + * The message is a QUIT, so its peer can have exited already. + * A send that fails because the peer is gone is logged at info, not + * as an alert. The flag is local to this process. It is not on the + * wire and not in shared memory. The flag stays set when the socket + * carries only the READ_QUEUE wake-up for a QUIT in the shared queue. + */ + uint8_t peer_may_be_gone; /* 1 bit */ } nxt_port_send_msg_t; #if (NXT_HAVE_UCRED) || (NXT_HAVE_MSGHDR_CMSGCRED) @@ -223,10 +300,22 @@ struct nxt_port_recv_msg_s { nxt_port_t *port; nxt_port_msg_t port_msg; size_t size; + /* Of a stream being reassembled: what it holds, nxt_port_frag_cost(). */ + size_t frag_held; + /* Of a stream being reassembled: its place in port->frag_queue. */ + nxt_queue_link_t frag_link; #if (NXT_USE_CMSG_PID) nxt_pid_t cmsg_pid; #endif nxt_bool_t cancelled; + /* + * Set by nxt_port_new_port_handler() when it had to create the port + * u.new_port names, clear when it found one already registered. A + * caller that refuses the announcement needs the difference: an existing + * port is live and must be left alone, while one this message brought + * into the runtime is the caller's to undo. + */ + nxt_bool_t new_port_created; union { nxt_port_t *new_port; nxt_pid_t removed_pid; @@ -246,12 +335,18 @@ struct nxt_port_recv_msg_s { /* * Close any file descriptors the peer attached to a received message via - * SCM_RIGHTS. A privileged handler that rejects a message (unauthorized - * or malformed sender) must call this before returning: the port - * dispatcher does not reclaim descriptors once the handler returns, and - * a compromised peer can attach fds to a forged message, so leaving them - * open on the reject path would let it exhaust the receiver's descriptor - * table. + * SCM_RIGHTS. + * + * The ownership contract: nxt_port_read_msg_process() closes whatever is + * left in msg->fd[] once the message has been dispatched, so a handler that + * KEEPS a descriptor must set its slot to -1. A kept descriptor whose slot + * was left set is closed under the handler, and the number is then handed + * out again by the next open() or accept() on that thread -- the retained + * handle silently refers to something else, which is a good deal worse than + * the leak this arrangement replaced. + * + * Calling this explicitly is still right on a reject path, where it makes + * the intent local and obvious, and it is idempotent. */ nxt_inline void nxt_port_recv_msg_close_fds(nxt_port_recv_msg_t *msg) @@ -268,11 +363,39 @@ nxt_port_recv_msg_close_fds(nxt_port_recv_msg_t *msg) } +/* + * How many queued messages on one port may carry a file descriptor. + * + * port->messages itself stays unbounded. A message with no descriptor costs + * only memory, which was true before a queued message took ownership of what + * it names, and bounding it would change behaviour on paths that never hold a + * descriptor at all -- every ordinary reply and every request body fragment. + * + * A message that does carry one is different: it holds up to two descriptors + * of this process open for as long as it waits, so a peer that stops reading + * turns into RLIMIT_NOFILE pressure on the sender rather than memory pressure + * alone. That is the cost this bounds. + * + * The traffic being bounded is control-plane and one message per event: a new + * port, a process start, a listening socket, a certificate, a script, a shared + * memory segment. A port with 128 of them outstanding is not a busy port, it + * is a peer that has stopped reading, so the bound is far above any legitimate + * burst. At two descriptors an entry it caps one stalled port at 256 open + * descriptors, which leaves room for several of them under the 1024 soft + * RLIMIT_NOFILE that is still the common default. + */ +#define NXT_PORT_MAX_FD_MSGS 128 + + typedef struct nxt_app_s nxt_app_t; struct nxt_port_s { nxt_fd_event_t socket; + /* + * Set only by nxt_process_port_add(). nxt_port_release() unlinks the + * port and drops the process reference; it does not clear ->process. + */ nxt_queue_link_t link; /* for nxt_process_t.ports */ nxt_process_t *process; @@ -286,12 +409,81 @@ struct nxt_port_s { nxt_queue_t messages; /* of nxt_port_send_msg_t */ nxt_thread_mutex_t write_mutex; + /* + * How many entries of ->messages still carry a file descriptor. + * + * A queued message owns the descriptors it names, so each such entry + * holds up to two of this process's descriptors open for as long as it + * waits. The count exists to bound that; it is maintained under + * ->write_mutex and is described with the bound in src/nxt_port_socket.c. + * + * ->fd_refusing is set by the first send the bound refuses and cleared + * by the next descriptor it takes, also under ->write_mutex. A peer + * that stops reading keeps the port at the bound for as long as it + * stalls, so the refusal is logged once when the port gets there rather + * than once for every send. + */ + uint32_t fd_messages; + uint8_t fd_refusing; + /* Maximum size of message part. */ uint32_t max_size; /* Maximum interleave of message parts. */ uint32_t max_share; + /* + * Websocket sessions upgraded from a request this worker answered. A + * session is counted by NXT_APR_UPGRADE and uncounted by + * NXT_APR_WEBSOCKET_CLOSE, both in nxt_router_app_port_release(). + */ uint32_t active_websockets; + + /* + * The application answered a request on this port and kept running. + * Treated exactly like active_websockets by the idle transition in + * nxt_router_app_port_idle(): the port stays in app->ports and in + * app->processes, and stays out of the idle queues, so the reaper + * never sees it and it keeps counting against "processes": {"max"}. + */ + uint8_t detached; + + /* + * An edge the accounting refused -- an unmatched FINISH, or a START at + * the maximum count -- was already logged as an alert for this port. + * Such an edge is a valid message the application can send in a loop, + * so only the first one is an alert and the rest go to the debug log. + * Touched only by nxt_router_detached_apply(), on the main thread. + */ + uint8_t detached_alerted; + + /* + * How many units of detached work the application reported of its own: + * one for each START edge whose FINISH edge has not arrived. Until the + * last of them does, the port stays out of the idle economy even with no + * request left. A count rather than a flag because the edges carry no + * context id and a worker may run several contexts -- see + * nxt_port_detached_t above -- so the first context's FINISH must not + * speak for the rest. It saturates at UINT32_MAX and detached_router + * below does not: this one moves on edges the application sends, which + * the router cannot bound, while detached_router moves only on requests + * the router itself gave up on, one per request it has in flight. + */ + uint32_t detached_app; + + /* + * The router put the port in the detached state itself: one for each + * request it has given up on that the worker is still running. A + * "limits": {"timeout"} expiry answers the client while the worker keeps + * executing, and the port may not rejoin the idle economy until every + * such request has been answered or the port closes. A count rather + * than a flag because one worker can run several of them at once, and + * kept apart from detached_app so that neither clear drops the other's + * reason. As wide as active_requests below it, which counts the same + * population: "threads" is validated up to NXT_INT32_T_MAX, and a wrap + * would leave a settle unable to clear the state at all. + */ + uint32_t detached_router; + uint32_t active_requests; nxt_port_handler_t handler; @@ -300,6 +492,74 @@ struct nxt_port_s { nxt_mp_t *mem_pool; nxt_event_engine_t *engine; + /* + * The deferral that carries the last reference drop to port->engine. + * Embedded rather than allocated, so that nxt_port_use() has no failure + * path -- see the comment there. Single-instance: use_count reaches + * zero only on port->engine, so only one thread at a time can hand the + * last reference over, and at most one post is ever in flight. + */ + nxt_work_t release_work; + + /* + * The write event has to be re-armed on port->engine, and a caller on + * another engine cannot do it directly. nxt_port_post() would allocate + * the item it posts, which fails exactly when the re-arm is needed most: + * the write that could not be held for a later attempt ran out of memory + * too. So the item lives here. + * + * ->release_work gets its uniqueness from the reference count -- only one + * thread can hand over the last reference -- and nothing like that holds + * for a re-arm, which any thread can want at any time. ->rearm_pending + * is that guarantee instead: the poster takes it from 0 to 1 and the + * handler clears it, so the item is on the engine's queue at most once. + * Re-arming twice would cost nothing, but linking the same item twice + * makes it its own successor. + * + * ->announce says the shared queue holds items whose READ_QUEUE marker + * was never sent. It is a fact about the port rather than work to run, + * so it is a flag and not a second item: any thread may set it, only + * port->engine clears it, and it is cleared only once a marker has gone + * out. + */ + nxt_work_t rearm_work; + nxt_atomic_t rearm_pending; + nxt_atomic_t announce; + + /* + * A QUIT was sent to this port, so its peer can be gone by now. + * Any thread can set the flag in nxt_port_socket_write2(). The flag + * is never cleared. The sender reads it when a send fails. + * A READ_QUEUE wake-up can be pending already when the QUIT goes into + * the shared queue (notify == 0). That wake-up wakes the peer for + * the QUIT too, so its failure is logged at info like the QUIT's own. + */ + nxt_atomic_t quit_sent; + + /* + * The pacing of a re-arm that follows a send which failed for want of + * kernel memory. + * + * ENOBUFS and ENOMEM leave the socket writable, so no edge is coming and + * the retry has to force the readiness re-check itself. Done on the + * spot, that re-check comes straight back: a shortage that lasts turns + * the engine thread into a tight epoll_wait/sendmsg/epoll_ctl loop, + * which is the worst moment to burn a core on. The timer paces it + * instead -- the retry still happens, just not as fast as the kernel can + * refuse it. See issue #407. + * + * ->retry_delay is what the next arming waits. It doubles up to a cap + * and a successful send puts it back to zero. + * + * ->retry_timer.enabled says the timer is holding a port reference: set + * by nxt_timer_add(), left set while the expired timer waits in the work + * queue, and cleared by nxt_timer_handler() on the line before it calls + * nxt_port_retry_handler(). Both fields are touched on port->engine + * only. + */ + nxt_timer_t retry_timer; + nxt_msec_t retry_delay; + nxt_buf_t *free_bufs; nxt_socket_t pair[2]; @@ -311,6 +571,15 @@ struct nxt_port_s { nxt_lvlhsh_t frags; + /* + * The fragment streams in ->frags and the bytes they hold, kept against + * the NXT_PORT_FRAG_* limits below. Touched only by the port's reader. + */ + uint32_t frag_streams; + uint32_t frag_size; /* <= NXT_PORT_FRAG_TOTAL_MAX */ + /* The same streams, the oldest first. */ + nxt_queue_t frag_queue; + nxt_atomic_t use_count; nxt_process_type_t type; @@ -323,6 +592,27 @@ struct nxt_port_s { }; +/* + * Limits on fragment reassembly at a receiving port (#394). A sender + * controls how many fragmented messages it opens and how long it keeps + * each one going, and the receiver holds every fragment until the last + * one arrives: without a bound, a peer that never sends the last fragment + * -- or sends a new stream id for each message -- grows the receiver + * without limit. libunit never fragments, so the legitimate senders are + * Unit's own processes, one message at a time per destination port; the + * largest such message is a configuration pushed from the controller. + * + * A stream that would pass a size limit is dropped as a whole, with an + * alert: what it had accumulated is released and its later fragments are + * discarded as belonging to no stream. A new stream when the port already + * has NXT_PORT_FRAG_STREAMS_MAX open drops the oldest one the same way, + * so streams that are never ended cannot block the port for good. + */ +#define NXT_PORT_FRAG_STREAMS_MAX 64 /* per port */ +#define NXT_PORT_FRAG_SIZE_MAX (128 * 1024 * 1024) /* per stream */ +#define NXT_PORT_FRAG_TOTAL_MAX (256 * 1024 * 1024) /* per port */ + + typedef struct { nxt_port_id_t id; nxt_pid_t pid; @@ -386,9 +676,71 @@ nxt_int_t nxt_port_socket_write2(nxt_task_t *task, nxt_port_t *port, nxt_uint_t type, nxt_fd_t fd, nxt_fd_t fd2, uint32_t stream, nxt_port_id_t reply_port, nxt_buf_t *b); + +typedef enum { + /* + * The message was still whole in port->messages and has been taken back + * out of it. Its buffers have been completed and the queue's reference + * to the port released, so nothing the caller handed to write2() is + * reachable from the port any more. + */ + NXT_PORT_MSG_CANCELLED = 0, + /* + * Found, but a fragment of it has already gone out (port_msg.nf), so the + * peer is mid-stream and the descriptors have been handed off. Left + * queued: taking it back now would strand the peer on an unfinished + * stream. + */ + NXT_PORT_MSG_STARTED, + /* + * Not in port->messages. Either it was never queued or it has been sent + * in full. NOT a lifetime boundary on its own: the write handler removes + * the message and only then queues the buffer completion, which can land + * behind work already queued ahead of it. + */ + NXT_PORT_MSG_NOT_FOUND, +} nxt_port_msg_cancel_t; + +/* + * Take a not-yet-started message back out of a port's send queue. + * + * Callable only on the thread that owns the port's engine, and only against a + * message this caller queued: it identifies the message by (type, stream, + * reply_port) and, when b is not NULL, by buffer identity as well, since a + * stream number alone is only unique per reply port. + * + * Descriptors are treated exactly as the send path treats them, through + * close_fd: a message that owns its descriptors has them closed here, and one + * that merely borrows them (START_PROCESS borrows the application's shared + * port) does not, so cancelling can never close a descriptor its owner is + * still using. + */ +nxt_port_msg_cancel_t nxt_port_socket_cancel(nxt_task_t *task, + nxt_port_t *port, nxt_uint_t type, uint32_t stream, + nxt_port_id_t reply_port, nxt_buf_t *b); + #if (NXT_TESTS) void nxt_port_test_msg_alloc_failures(nxt_uint_t failures); void nxt_port_test_run_error_handler(nxt_task_t *task, nxt_port_t *port); +void nxt_port_test_run_read_msg_process(nxt_task_t *task, nxt_port_t *port, + nxt_port_recv_msg_t *msg); + +/* + * Counts entries into nxt_port_send_new_port(). It is nxt_inline and + * skips the announced process and the receiver itself, so a fixture with + * one process cannot tell "not broadcast" from "broadcast to nobody" by + * observing peers; counting the call itself distinguishes them. + */ +NXT_EXPORT extern nxt_uint_t nxt_port_test_broadcasts; + +/* + * Counts entries into the event-loop pass of nxt_port_write_msgs() -- the + * dispatches of the port's write handler. A retry that is not paced makes + * this climb with the number of poll passes rather than with elapsed time, + * which is the whole of what the backoff is there to stop and is not + * observable from the port's state afterwards. + */ +NXT_EXPORT extern nxt_uint_t nxt_port_test_write_dispatches; #endif nxt_inline nxt_int_t @@ -407,8 +759,12 @@ nxt_int_t nxt_port_send_port(nxt_task_t *task, nxt_port_t *port, void nxt_port_change_log_file(nxt_task_t *task, nxt_runtime_t *rt, nxt_uint_t slot, nxt_fd_t fd); void nxt_port_remove_notify_others(nxt_task_t *task, nxt_process_t *process); +void nxt_port_rearm(nxt_task_t *task, nxt_port_t *port); +void *nxt_port_queue_mmap(nxt_task_t *task, nxt_fd_t fd, size_t size); void nxt_port_quit_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); +nxt_int_t nxt_port_new_port_msg(const nxt_port_recv_msg_t *msg, + nxt_port_msg_new_port_t *out); void nxt_port_new_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); void nxt_port_process_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); void nxt_port_change_log_file_handler(nxt_task_t *task, @@ -421,6 +777,7 @@ void nxt_port_empty_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); nxt_int_t nxt_port_post(nxt_task_t *task, nxt_port_t *port, nxt_port_post_handler_t handler, void *data); void nxt_port_use(nxt_task_t *task, nxt_port_t *port, int i); +nxt_bool_t nxt_port_use_unless_zero(nxt_port_t *port); nxt_inline void nxt_port_inc_use(nxt_port_t *port) { diff --git a/src/nxt_port_memory.c b/src/nxt_port_memory.c index 386407415..315e84382 100644 --- a/src/nxt_port_memory.c +++ b/src/nxt_port_memory.c @@ -5,10 +5,15 @@ */ #include +#include #if (NXT_HAVE_MEMFD_CREATE) +#if (NXT_HAVE_LINUX_MEMFD_H) #include +#else +#include +#endif #include #include @@ -54,10 +59,11 @@ nxt_port_mmap_at(nxt_port_mmaps_t *port_mmaps, uint32_t i) } /* - * A capacity able to hold slot i is i + 1 elements, which is not - * representable in uint32_t for i == UINT32_MAX. + * "i" is a segment id from the peer, or the next id of an outgoing + * segment. Past NXT_PORT_MMAPS_MAX it is refused before any growth, + * so a huge id costs nothing. This also keeps i + 1 in uint32_t. */ - if (nxt_slow_path(i == UINT32_MAX)) { + if (nxt_slow_path(i >= NXT_PORT_MMAPS_MAX)) { return NULL; } @@ -151,8 +157,10 @@ nxt_port_mmaps_destroy(nxt_port_mmaps_t *port_mmaps, nxt_bool_t free_elts) } +#if (NXT_DEBUG) #define nxt_port_mmap_free_junk(p, size) \ memset((p), 0xA5, size) +#endif static void @@ -214,7 +222,9 @@ nxt_port_mmap_buf_completion(nxt_task_t *task, void *obj, void *data) c = nxt_port_mmap_chunk_id(hdr, p); } +#if (NXT_DEBUG) nxt_port_mmap_free_junk(p, b->mem.end - p); +#endif nxt_debug(task, "mmap buf completion: %p [%p,%uz] (sent=%d), " "%PI->%PI,%d,%d", b, b->mem.start, b->mem.end - b->mem.start, @@ -230,9 +240,20 @@ nxt_port_mmap_buf_completion(nxt_task_t *task, void *obj, void *data) if (dst_pid == nxt_pid && nxt_atomic_cmp_set(&hdr->oosm, 1, 0)) { - process = nxt_runtime_process_find(task->thread->runtime, src_pid); + process = nxt_runtime_process_ref(task->thread->runtime, src_pid); nxt_process_broadcast_shm_ack(task, process); + + /* + * Released here rather than at the end of the function: the + * complete_buf back-edge below re-enters this block once per buffer + * of the chain, so a release outside it would leak every reference + * but the last. + */ + + if (process != NULL) { + nxt_process_use(task, process, -1); + } } release_buf: @@ -379,17 +400,26 @@ nxt_port_incoming_port_mmap(nxt_task_t *task, nxt_process_t *process, static nxt_port_mmap_handler_t * -nxt_port_new_port_mmap(nxt_task_t *task, nxt_port_mmaps_t *mmaps, - nxt_bool_t tracking, nxt_int_t n) +nxt_port_new_port_mmap(nxt_task_t *task, nxt_port_mmaps_t *mmaps, nxt_int_t n) { void *mem; nxt_fd_t fd; nxt_int_t i; - nxt_free_map_t *free_map; nxt_port_mmap_t *port_mmap; nxt_port_mmap_header_t *hdr; nxt_port_mmap_handler_t *mmap_handler; + /* + * The id of the new segment is its index, mmaps->size. Refused here + * with its own message; nxt_port_mmap_at() below refuses it too. + */ + if (nxt_slow_path(mmaps->size >= NXT_PORT_MMAPS_MAX)) { + nxt_alert(task, "too many port mmaps (%uD), limit is %uD", + mmaps->size, NXT_PORT_MMAPS_MAX); + + return NULL; + } + mmap_handler = nxt_zalloc(sizeof(nxt_port_mmap_handler_t)); if (nxt_slow_path(mmap_handler == NULL)) { nxt_alert(task, "failed to allocate mmap_handler"); @@ -427,22 +457,18 @@ nxt_port_new_port_mmap(nxt_task_t *task, nxt_port_mmaps_t *mmaps, hdr = mmap_handler->hdr; nxt_memset(hdr->free_map, 0xFFU, sizeof(hdr->free_map)); - nxt_memset(hdr->free_tracking_map, 0xFFU, sizeof(hdr->free_tracking_map)); hdr->id = mmaps->size - 1; hdr->src_pid = nxt_pid; hdr->sent_over = 0xFFFFu; /* Mark first chunk as busy */ - free_map = tracking ? hdr->free_tracking_map : hdr->free_map; - for (i = 0; i < n; i++) { - nxt_port_mmap_set_chunk_busy(free_map, i); + nxt_port_mmap_set_chunk_busy(hdr->free_map, i); } /* Mark as busy chunk followed the last available chunk. */ nxt_port_mmap_set_chunk_busy(hdr->free_map, PORT_MMAP_CHUNK_COUNT); - nxt_port_mmap_set_chunk_busy(hdr->free_tracking_map, PORT_MMAP_CHUNK_COUNT); nxt_log(task, NXT_LOG_DEBUG, "new mmap #%D created for %PI -> ...", hdr->id, nxt_pid); @@ -536,7 +562,7 @@ nxt_shm_open(nxt_task_t *task, size_t size) static nxt_port_mmap_handler_t * nxt_port_mmap_get(nxt_task_t *task, nxt_port_mmaps_t *mmaps, nxt_chunk_id_t *c, - nxt_int_t n, nxt_bool_t tracking) + nxt_int_t n) { nxt_int_t i, res, nchunks; nxt_free_map_t *free_map; @@ -566,13 +592,16 @@ nxt_port_mmap_get(nxt_task_t *task, nxt_port_mmaps_t *mmaps, nxt_chunk_id_t *c, *c = 0; - free_map = tracking ? hdr->free_tracking_map : hdr->free_map; + free_map = hdr->free_map; while (nxt_port_mmap_get_free_chunk(free_map, c)) { nchunks = 1; while (nchunks < n) { - res = nxt_port_mmap_chk_set_chunk_busy(free_map, *c + nchunks); + /* Not up to the sentinel: see nxt_port_mmap_increase_buf(). */ + res = *c + nchunks < PORT_MMAP_CHUNK_COUNT + && nxt_port_mmap_chk_set_chunk_busy(free_map, + *c + nchunks); if (res == 0) { for (i = 0; i < nchunks; i++) { @@ -600,7 +629,7 @@ nxt_port_mmap_get(nxt_task_t *task, nxt_port_mmaps_t *mmaps, nxt_chunk_id_t *c, end: *c = 0; - mmap_handler = nxt_port_new_port_mmap(task, mmaps, tracking, n); + mmap_handler = nxt_port_new_port_mmap(task, mmaps, n); unlock_return: @@ -616,7 +645,14 @@ nxt_port_get_port_incoming_mmap(nxt_task_t *task, nxt_pid_t spid, uint32_t id) nxt_process_t *process; nxt_port_mmap_handler_t *mmap_handler; - process = nxt_runtime_process_find(task->thread->runtime, spid); + /* + * Referenced, not just found. This runs on whichever router worker + * engine received the message, while the router main engine can be + * releasing the same process; without the reference the mutex locked on + * the next line can already have been destroyed and freed underneath us. + */ + + process = nxt_runtime_process_ref(task->thread->runtime, spid); if (nxt_slow_path(process == NULL)) { return NULL; } @@ -644,6 +680,17 @@ nxt_port_get_port_incoming_mmap(nxt_task_t *task, nxt_pid_t spid, uint32_t id) nxt_thread_mutex_unlock(&process->incoming.mutex); + /* + * Strictly after the unlock. If this is the last reference, the release + * runs nxt_thread_mutex_destroy(&process->incoming.mutex) -- dropping it + * before the unlock would destroy the mutex this function still holds. + * + * The returned handler is unaffected: the reference bumped above is its + * own, independent of the process, and the caller releases it. + */ + + nxt_process_use(task, process, -1); + return mmap_handler; } @@ -670,13 +717,16 @@ nxt_port_mmap_get_buf(nxt_task_t *task, nxt_port_mmaps_t *mmaps, size_t size) b = nxt_buf_mem_ts_alloc(task, task->thread->engine->mem_pool, 0); if (nxt_slow_path(b == NULL)) { + nxt_alert(task, "failed to allocate a buffer for %z bytes of shared " + "memory", size); + return NULL; } b->completion_handler = nxt_port_mmap_buf_completion; nxt_buf_set_port_mmap(b); - mmap_handler = nxt_port_mmap_get(task, mmaps, &c, nchunks, 0); + mmap_handler = nxt_port_mmap_get(task, mmaps, &c, nchunks); if (nxt_slow_path(mmap_handler == NULL)) { mp = task->thread->engine->mem_pool; nxt_mp_free(mp, b); @@ -737,8 +787,11 @@ nxt_port_mmap_increase_buf(nxt_task_t *task, nxt_buf_t *b, size_t size, c = start; - /* Try to acquire as much chunks as required. */ - while (nchunks > 0) { + /* + * Try to acquire as much chunks as required. Not up to the busy + * sentinel: the peer maps the segment writable and can clear it. + */ + while (nchunks > 0 && c < PORT_MMAP_CHUNK_COUNT) { if (nxt_port_mmap_chk_set_chunk_busy(hdr->free_map, c) == 0) { break; @@ -884,27 +937,51 @@ nxt_port_mmap_write(nxt_task_t *task, nxt_port_t *port, } +/* + * The buffers of an mmap message carry an array of nxt_port_mmap_msg_t + * written by the peer, which may be an untrusted application process. The + * array is walked with nxt_span_copy(), so a buffer whose length is not a + * whole number of records -- a partial tail of 1 to 11 bytes -- is refused + * at the tail instead of being read as a record that runs past mem.free. + * Each record is copied out before any field is used; the fields are then + * bounds-checked by nxt_port_mmap_get_incoming_buf(): mmap_id against the + * sender's incoming segments, chunk_id and size by + * nxt_port_mmap_chunk_range_valid(). + */ + void nxt_port_mmap_read(nxt_task_t *task, nxt_port_recv_msg_t *msg) { nxt_buf_t *b, **pb; - nxt_port_mmap_msg_t *end, *mmap_msg; + nxt_span_t span; + nxt_port_mmap_msg_t mmap_msg; pb = &msg->buf; msg->size = 0; for (b = msg->buf; b != NULL; b = b->next) { - mmap_msg = (nxt_port_mmap_msg_t *) b->mem.pos; - end = (nxt_port_mmap_msg_t *) b->mem.free; + nxt_span_init(&span, b->mem.pos, b->mem.free); + + while (nxt_span_len(&span) != 0) { + + if (nxt_slow_path(nxt_span_copy(&span, &mmap_msg, + sizeof(nxt_port_mmap_msg_t)) + != 0)) + { + nxt_alert(task, "invalid mmap message from pid %PI: " + "%uz trailing bytes are not a whole record", + msg->port_msg.pid, nxt_span_len(&span)); + + break; + } - while (mmap_msg < end) { nxt_debug(task, "mmap_msg={%D, %D, %D} from %PI", - mmap_msg->mmap_id, mmap_msg->chunk_id, mmap_msg->size, + mmap_msg.mmap_id, mmap_msg.chunk_id, mmap_msg.size, msg->port_msg.pid); *pb = nxt_port_mmap_get_incoming_buf(task, msg->port, - msg->port_msg.pid, mmap_msg); + msg->port_msg.pid, &mmap_msg); if (nxt_slow_path(*pb == NULL)) { nxt_log_error(NXT_LOG_ERR, task->log, "failed to get mmap buffer"); @@ -912,9 +989,8 @@ nxt_port_mmap_read(nxt_task_t *task, nxt_port_recv_msg_t *msg) break; } - msg->size += mmap_msg->size; + msg->size += mmap_msg.size; pb = &(*pb)->next; - mmap_msg++; /* Mark original buf as complete. */ b->mem.pos += sizeof(nxt_port_mmap_msg_t); @@ -976,18 +1052,83 @@ nxt_port_mmap_get_method(nxt_task_t *task, nxt_port_t *port, nxt_buf_t *b) void nxt_process_broadcast_shm_ack(nxt_task_t *task, nxt_process_t *process) { - nxt_port_t *port; + nxt_port_t *port; + nxt_runtime_t *rt; - if (nxt_slow_path(process == NULL || nxt_queue_is_empty(&process->ports))) - { + if (nxt_slow_path(process == NULL)) { return; } - port = nxt_process_port_first(process); + rt = task->thread->runtime; - if (port->type == NXT_PROCESS_APP) { - nxt_port_post(task, port, nxt_port_broadcast_shm_ack, process); + /* + * This runs on any engine -- most often a worker engine, from + * nxt_port_mmap_buf_completion() -- while the app port's teardown runs on + * the router's main thread. The caller holds a reference to the process, + * not to the port, so the port has to be looked up, and a reference taken + * from a lookup can land on a port whose count already reached zero: + * nxt_port_post() below does an unconditional increment, and + * nxt_port_release() would by then be destroying the port's memory pool + * (issue #195 -- the `port->use_count == 0` assertion in a debug build, + * a use-after-free of the port and of the work item in a release one). + * + * rt->processes_mutex is the lock nxt_port_release() unlinks the port + * under, so reading process->ports and try-referencing the port under it + * cannot observe a port that is being released: either the try-ref wins + * and the port is alive for as long as this function holds it, or the + * port is already dying and there is nothing to acknowledge -- the + * process is going away and the app will not wait for shared memory + * again. + * + * Only the lookup is done under the mutex. Everything after it takes + * other locks (nxt_process_use() takes this very mutex), and + * rt->processes_mutex is a leaf -- see src/nxt_process.c:150. + */ + + nxt_thread_mutex_lock(&rt->processes_mutex); + + port = NULL; + + if (!nxt_queue_is_empty(&process->ports)) { + port = nxt_process_port_first(process); + + if (port->type != NXT_PROCESS_APP || !nxt_port_use_unless_zero(port)) { + port = NULL; + } + } + + nxt_thread_mutex_unlock(&rt->processes_mutex); + + if (port == NULL) { + return; } + + /* + * The process is handed to another engine as a bare pointer and is + * dereferenced there, so the posted handler needs a reference of its + * own and drops it when it is done. nxt_port_post() returns NXT_OK + * both when it queues the handler and when it calls it inline on the + * current engine, and the handler owns the reference either way; only + * the NXT_ERROR case leaves nothing to run, so only that case has to + * be undone here. The inline call is safe: the caller still holds + * its own reference, so this one cannot be the last. + */ + + nxt_process_use(task, process, 1); + + if (nxt_slow_path(nxt_port_post(task, port, nxt_port_broadcast_shm_ack, + process) != NXT_OK)) + { + nxt_process_use(task, process, -1); + } + + /* + * nxt_port_post() took its own reference on success, so the lookup + * reference is dropped here either way. It is dropped after the post so + * that the port cannot be released between the try-ref and the post. + */ + + nxt_port_use(task, port, -1); } @@ -1002,4 +1143,30 @@ nxt_port_broadcast_shm_ack(nxt_task_t *task, nxt_port_t *port, void *data) (void) nxt_port_socket_write(task, port, NXT_PORT_MSG_SHM_ACK, -1, 0, 0, NULL); } nxt_queue_loop; + + nxt_process_use(task, process, -1); +} + + +#if (NXT_TESTS) + +/* + * The static growth and creation paths, for + * src/test/nxt_port_mmaps_max_test.c. + */ + +nxt_port_mmap_t * +nxt_port_test_mmap_at(nxt_port_mmaps_t *mmaps, uint32_t i) +{ + return nxt_port_mmap_at(mmaps, i); } + + +nxt_port_mmap_handler_t * +nxt_port_test_new_port_mmap(nxt_task_t *task, nxt_port_mmaps_t *mmaps, + nxt_int_t n) +{ + return nxt_port_new_port_mmap(task, mmaps, n); +} + +#endif diff --git a/src/nxt_port_memory.h b/src/nxt_port_memory.h index f1e709646..476af6117 100644 --- a/src/nxt_port_memory.h +++ b/src/nxt_port_memory.h @@ -52,4 +52,11 @@ nxt_int_t nxt_shm_open(nxt_task_t *task, size_t size); void nxt_process_broadcast_shm_ack(nxt_task_t *task, nxt_process_t *process); +#if (NXT_TESTS) +/* The segment array growth and the segment creation, for the limit test. */ +nxt_port_mmap_t *nxt_port_test_mmap_at(nxt_port_mmaps_t *mmaps, uint32_t i); +nxt_port_mmap_handler_t *nxt_port_test_new_port_mmap(nxt_task_t *task, + nxt_port_mmaps_t *mmaps, nxt_int_t n); +#endif + #endif /* _NXT_PORT_MEMORY_H_INCLUDED_ */ diff --git a/src/nxt_port_memory_int.h b/src/nxt_port_memory_int.h index 63ac09e19..846cf1763 100644 --- a/src/nxt_port_memory_int.h +++ b/src/nxt_port_memory_int.h @@ -10,6 +10,7 @@ #include #include +#include #ifdef NXT_MMAP_TINY_CHUNK @@ -47,6 +48,32 @@ */ #define NXT_PORT_MMAP_MAX_SEGMENTS (UINT32_MAX / PORT_MMAP_DATA_SIZE + 1) +/* + * The most shared memory segments one process keeps for one peer, in each + * direction. A segment id is an index into that array, and the array is + * grown to hold the id. On both sides, the router and libunit, an incoming + * id is taken from the peer: from an mmap record in a port message, or from + * a segment header in memory the peer can still write. Without a limit an + * id like 100000000 makes the receiver allocate and initialise an array of + * that many slots from one message. + * + * NXT_PORT_MMAP_MAX_SEGMENTS above bounds only the libunit-authored ids the + * router receives. The router's own outgoing segments are not bounded by + * shm_limit, so libunit needs a limit of its own, and this one is used by + * both sides, in nxt_port_mmap_at() and nxt_unit_mmap_at(). Each segment + * holds PORT_MMAP_SIZE bytes, about 10 MiB, so 65536 segments are about + * 640 GiB of shared memory between one pair of processes: no real peer + * reaches it. The arrays a bad id can grow are bounded too: at the limit + * the router's array of nxt_port_mmap_t (one pointer) is 512 KiB, and + * libunit's array of nxt_unit_mmap_t (a pointer, a pthread_t and a queue, + * 32 bytes on 64-bit) is 2 MiB. The router refuses to create a segment + * past the limit, and both sides refuse an incoming id past it. + * + * In an NXT_MMAP_TINY_CHUNK build a segment is 2 KiB, so the limit is only + * 128 MiB of shared memory per pair; that build is for debugging. + */ +#define NXT_PORT_MMAPS_MAX 65536U + typedef uint32_t nxt_chunk_id_t; @@ -70,13 +97,42 @@ struct nxt_port_mmap_header_s { nxt_port_id_t sent_over; nxt_atomic_t oosm; nxt_free_map_t free_map[MAX_FREE_IDX]; + /* + * Not padding in the alignment sense: nxt_port_mmap_set_chunk_busy() is + * called with PORT_MMAP_CHUNK_COUNT to plant a permanently-busy sentinel + * one word past the last real word of free_map[], so that a multi-chunk + * allocation walking off the end of the segment fails to claim its + * continuation instead of reading past the end of the struct. + */ nxt_free_map_t free_map_padding; - nxt_free_map_t free_tracking_map[MAX_FREE_IDX]; - nxt_free_map_t free_tracking_map_padding; - nxt_atomic_t tracking[PORT_MMAP_CHUNK_COUNT]; + + /* + * Not a live field. It reserves the window that a peer built before the + * tracking bitmap was dropped still writes: such a peer memsets + * MAX_FREE_IDX words of free_tracking_map starting here and plants its + * sentinel one word past them. A libunit of that vintage can create + * segments this build maps, so the window has to stay unused while those + * peers are supported. + * + * Reserved in the struct rather than only described in a comment, so + * that the next field added lands after it by construction instead of + * by the author having read this. Removing the reservation is what + * turns the removal of the tracking bitmap into the same latent + * corruption it was meant to fix. + */ + nxt_free_map_t legacy_tracking_window[MAX_FREE_IDX + 1]; }; +/* + * The header struct is mapped over the first PORT_MMAP_HEADER_SIZE bytes of + * the segment and chunk 0 starts right after it, so anything the struct + * declares beyond that boundary silently aliases payload. + */ +nxt_static_assert(sizeof(nxt_port_mmap_header_t) <= PORT_MMAP_HEADER_SIZE, + "nxt_port_mmap_header_t overflows the segment header area"); + + struct nxt_port_mmap_handler_s { nxt_port_mmap_header_t *hdr; nxt_atomic_t use_count; diff --git a/src/nxt_port_queue.h b/src/nxt_port_queue.h index 50151fb64..ff6d7be7e 100644 --- a/src/nxt_port_queue.h +++ b/src/nxt_port_queue.h @@ -74,7 +74,12 @@ nxt_port_queue_send(nxt_port_queue_t volatile *q, const void *p, uint8_t size, qi->size = size; nxt_memcpy(qi->data, p, size); - nxt_nncq_enqueue(&q->queue, i); + if (nxt_slow_path(nxt_nncq_enqueue(&q->queue, i) != NXT_OK)) { + (void) nxt_nncq_enqueue(&q->free_items, i); + + *notify = 0; + return NXT_ERROR; + } i = nxt_atomic_fetch_add(&q->nitems, 1); @@ -114,7 +119,7 @@ nxt_port_queue_recv(nxt_port_queue_t volatile *q, void *p) nxt_memcpy(p, qi->data, size); - nxt_nncq_enqueue(&q->free_items, i); + (void) nxt_nncq_enqueue(&q->free_items, i); nxt_atomic_fetch_add(&q->nitems, -1); diff --git a/src/nxt_port_rpc.c b/src/nxt_port_rpc.c index 87815faae..9a5f295ac 100644 --- a/src/nxt_port_rpc.c +++ b/src/nxt_port_rpc.c @@ -25,6 +25,9 @@ struct nxt_port_rpc_reg_s { }; +static void *nxt_port_rpc_reg_add(nxt_task_t *task, nxt_port_t *port, + nxt_port_rpc_handler_t ready_handler, nxt_port_rpc_handler_t error_handler, + uint32_t stream, size_t ex_size); static void nxt_port_rpc_remove_from_peers(nxt_task_t *task, nxt_port_t *port, nxt_port_rpc_reg_t *reg); @@ -48,6 +51,24 @@ nxt_port_rpc_test_insert_failures(nxt_uint_t failures) } +/* + * Set the next stream identifier and return the previous one. A test uses + * it to reach the 32-bit wrap and to restore the counter afterwards. It is + * a plain store: the tests are single-threaded. + */ + +uint32_t +nxt_port_rpc_test_set_stream_ident(uint32_t stream) +{ + uint32_t old; + + old = *nxt_stream_ident; + *nxt_stream_ident = stream; + + return old; +} + + static nxt_bool_t nxt_port_rpc_test_should_fail(nxt_uint_t *failures) { @@ -155,14 +176,57 @@ nxt_port_rpc_register_handler_ex(nxt_task_t *task, nxt_port_t *port, nxt_port_rpc_handler_t ready_handler, nxt_port_rpc_handler_t error_handler, size_t ex_size) { - uint32_t stream; + uint32_t stream; + + /* Stream 0 means "no handler" to the callers: skip it after a wrap. */ + + do { + stream = nxt_atomic_fetch_add(nxt_stream_ident, 1); + } while (nxt_slow_path(stream == 0)); + + return nxt_port_rpc_reg_add(task, port, ready_handler, error_handler, + stream, ex_size); +} + + +/* + * Register a handler on a stream the caller names rather than on a fresh one. + * + * The one caller is a router that has just been told, by its own deadline, + * that a start it asked for will not be answered, and that wants to keep + * watching the stream for the reply that may still come: the worker's late + * PROCESS_READY, or the REMOVE_PID for it that the process's death produces. + * See nxt_router_app_start_expired() in src/nxt_router.c. + * + * Only safe from inside that stream's own handler, because + * nxt_port_rpc_handler() deletes the registration before it runs the handler + * and does not delete again afterwards -- so the key is free at that point, + * and nothing removes what this puts back. Anywhere else the insert would + * either collide with a live registration or be undone by the caller's own + * cleanup, and the stream identifier itself is only unique until + * nxt_stream_ident wraps. + */ + +void * +nxt_port_rpc_register_handler_at(nxt_task_t *task, nxt_port_t *port, + nxt_port_rpc_handler_t ready_handler, nxt_port_rpc_handler_t error_handler, + uint32_t stream, size_t ex_size) +{ + return nxt_port_rpc_reg_add(task, port, ready_handler, error_handler, + stream, ex_size); +} + + +static void * +nxt_port_rpc_reg_add(nxt_task_t *task, nxt_port_t *port, + nxt_port_rpc_handler_t ready_handler, nxt_port_rpc_handler_t error_handler, + uint32_t stream, size_t ex_size) +{ nxt_port_rpc_reg_t *reg; nxt_lvlhsh_query_t lhq; nxt_assert(port->pair[0] != -1); - stream = nxt_atomic_fetch_add(nxt_stream_ident, 1); - #if (NXT_TESTS) if (nxt_slow_path(nxt_port_rpc_test_should_fail( &nxt_port_rpc_test_alloc_failure_count))) @@ -387,6 +451,15 @@ nxt_port_rpc_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) if (ret != NXT_OK) { nxt_debug(task, "rpc: stream #%uD no handler found", stream); + /* + * No handler ran, so nothing can have taken the descriptors this + * message arrived with. The dispatcher would close them anyway; + * doing it here keeps the reject path self-evidently safe, since + * the stream number is a value off the wire and any peer can name + * one that was never registered. + */ + nxt_port_recv_msg_close_fds(msg); + return; } @@ -544,11 +617,55 @@ nxt_port_rpc_cancel(nxt_task_t *task, nxt_port_t *port, uint32_t stream) static nxt_buf_t nxt_port_close_dummy_buf; + +static void +nxt_port_rpc_error_msg(nxt_task_t *task, nxt_port_t *port, uint32_t stream) +{ + nxt_port_recv_msg_t msg; + + msg.fd[0] = -1; + msg.fd[1] = -1; + msg.buf = &nxt_port_close_dummy_buf; + msg.port = port; + msg.port_msg.stream = stream; + msg.port_msg.pid = nxt_pid; + msg.port_msg.type = _NXT_PORT_MSG_RPC_ERROR; + msg.port_msg.last = 1; + msg.port_msg.mmap = 0; + msg.port_msg.nf = 0; + msg.port_msg.mf = 0; + msg.size = 0; + msg.cancelled = 0; + msg.u.data = NULL; + + nxt_port_rpc_handler(task, &msg); +} + + +/* + * Fail one registered RPC as though the peer had answered RPC_ERROR: the + * registered error handler runs and the registration is retired, so a caller + * that has decided the answer will never come reuses that handler's recovery + * instead of duplicating it. + * + * Safe on a stream that is already gone -- nxt_port_rpc_handler() drops an + * unknown stream with a debug line -- which is what makes it usable from a + * deadline timer racing a real reply. + */ + +void +nxt_port_rpc_error(nxt_task_t *task, nxt_port_t *port, uint32_t stream) +{ + nxt_debug(task, "rpc: stream #%uD fail registration", stream); + + nxt_port_rpc_error_msg(task, port, stream); +} + + void nxt_port_rpc_close(nxt_task_t *task, nxt_port_t *port) { - nxt_port_rpc_reg_t *reg; - nxt_port_recv_msg_t msg; + nxt_port_rpc_reg_t *reg; for ( ;; ) { reg = nxt_lvlhsh_peek(&port->rpc_streams, &lvlhsh_rpc_reg_proto); @@ -556,21 +673,6 @@ nxt_port_rpc_close(nxt_task_t *task, nxt_port_t *port) return; } - msg.fd[0] = -1; - msg.fd[1] = -1; - msg.buf = &nxt_port_close_dummy_buf; - msg.port = port; - msg.port_msg.stream = reg->stream; - msg.port_msg.pid = nxt_pid; - msg.port_msg.type = _NXT_PORT_MSG_RPC_ERROR; - msg.port_msg.last = 1; - msg.port_msg.mmap = 0; - msg.port_msg.nf = 0; - msg.port_msg.mf = 0; - msg.size = 0; - msg.cancelled = 0; - msg.u.data = NULL; - - nxt_port_rpc_handler(task, &msg); + nxt_port_rpc_error_msg(task, port, reg->stream); } } diff --git a/src/nxt_port_rpc.h b/src/nxt_port_rpc.h index 8218c4e3a..1f2de1dc0 100644 --- a/src/nxt_port_rpc.h +++ b/src/nxt_port_rpc.h @@ -19,10 +19,14 @@ uint32_t nxt_port_rpc_register_handler(nxt_task_t *task, nxt_port_t *port, void *nxt_port_rpc_register_handler_ex(nxt_task_t *task, nxt_port_t *port, nxt_port_rpc_handler_t ready_handler, nxt_port_rpc_handler_t error_handler, size_t ex_size); +void *nxt_port_rpc_register_handler_at(nxt_task_t *task, nxt_port_t *port, + nxt_port_rpc_handler_t ready_handler, nxt_port_rpc_handler_t error_handler, + uint32_t stream, size_t ex_size); #if (NXT_TESTS) void nxt_port_rpc_test_alloc_failures(nxt_uint_t failures); void nxt_port_rpc_test_insert_failures(nxt_uint_t failures); +uint32_t nxt_port_rpc_test_set_stream_ident(uint32_t stream); #endif uint32_t nxt_port_rpc_ex_stream(void *ex); @@ -33,6 +37,7 @@ void nxt_port_rpc_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); void nxt_port_rpc_remove_peer(nxt_task_t *task, nxt_port_t *port, nxt_pid_t peer); void nxt_port_rpc_cancel(nxt_task_t *task, nxt_port_t *port, uint32_t stream); +void nxt_port_rpc_error(nxt_task_t *task, nxt_port_t *port, uint32_t stream); void nxt_port_rpc_close(nxt_task_t *task, nxt_port_t *port); diff --git a/src/nxt_port_socket.c b/src/nxt_port_socket.c index 6677dd735..537fc029d 100644 --- a/src/nxt_port_socket.c +++ b/src/nxt_port_socket.c @@ -14,14 +14,34 @@ (int) (NXT_PORT_QUEUE_MSG_SIZE - sizeof(nxt_port_msg_t)) +/* + * The first and the last delay a paced retry waits, in milliseconds. The + * first is short because most shortages are: the common case is one failed + * send and then a retry that works, and a millisecond costs that case + * nothing measurable. The cap is what a shortage that lasts settles at, and + * is the retry latency this trades for not spinning -- see the comment on + * nxt_port_retry_later(). + */ +#define NXT_PORT_RETRY_MIN_DELAY 1 +#define NXT_PORT_RETRY_MAX_DELAY 32 + + static nxt_bool_t nxt_port_can_enqueue_buf(nxt_buf_t *b); static uint8_t nxt_port_enqueue_buf(nxt_task_t *task, nxt_port_msg_t *pm, void *qbuf, nxt_buf_t *b); static nxt_int_t nxt_port_msg_chk_insert(nxt_task_t *task, nxt_port_t *port, nxt_port_send_msg_t *msg); static nxt_port_send_msg_t *nxt_port_msg_alloc(const nxt_port_send_msg_t *m); +static nxt_int_t nxt_port_msg_dup_fds(nxt_port_send_msg_t *msg); +static nxt_int_t nxt_port_write_msgs(nxt_task_t *task, void *obj, + void *data, nxt_bool_t *send_failed); static void nxt_port_write_handler(nxt_task_t *task, void *obj, void *data); static nxt_port_send_msg_t *nxt_port_msg_first(nxt_port_t *port); +nxt_inline nxt_bool_t nxt_port_msg_has_fd(const nxt_port_send_msg_t *msg); +nxt_inline void nxt_port_msg_fd_uncount_locked(nxt_port_t *port, + nxt_port_send_msg_t *msg); +static void nxt_port_msg_fd_uncount(nxt_port_t *port, + nxt_port_send_msg_t *msg); nxt_inline void nxt_port_msg_close_fd(nxt_port_send_msg_t *msg); nxt_inline void nxt_port_close_fds(nxt_fd_t *fd); static nxt_buf_t *nxt_port_buf_completion(nxt_task_t *task, @@ -35,11 +55,29 @@ static void nxt_port_read_msg_process(nxt_task_t *task, nxt_port_t *port, nxt_port_recv_msg_t *msg); static nxt_buf_t *nxt_port_buf_alloc(nxt_port_t *port); static void nxt_port_buf_free(nxt_port_t *port, nxt_buf_t *b); +static void nxt_port_frag_unaccount(nxt_port_t *port, + nxt_port_recv_msg_t *fmsg); +static void nxt_port_frag_evict(nxt_task_t *task, nxt_port_t *port, + nxt_port_recv_msg_t *msg); +static void nxt_port_frag_drop(nxt_task_t *task, nxt_port_t *port, + nxt_port_recv_msg_t *fmsg, nxt_bool_t in_hash); +static void nxt_port_frag_bufs_release(nxt_task_t *task, nxt_port_t *port, + nxt_buf_t *b); +static nxt_int_t nxt_port_announce(nxt_task_t *task, nxt_port_t *port); +static void nxt_port_rearm_now(nxt_task_t *task, nxt_port_t *port); +static void nxt_port_retry_later(nxt_task_t *task, nxt_port_t *port); +static void nxt_port_retry_handler(nxt_task_t *task, void *obj, void *data); +static void nxt_port_rearm_work_handler(nxt_task_t *task, void *obj, + void *data); static void nxt_port_error_handler(nxt_task_t *task, void *obj, void *data); #if (NXT_TESTS) static nxt_uint_t nxt_port_test_msg_alloc_failure_count; +nxt_uint_t nxt_port_test_broadcasts; + +nxt_uint_t nxt_port_test_write_dispatches; + void nxt_port_test_msg_alloc_failures(nxt_uint_t failures) @@ -64,6 +102,23 @@ nxt_port_test_run_error_handler(nxt_task_t *task, nxt_port_t *port) nxt_port_error_handler(task, &port->socket, NULL); } + +/* + * Public wrapper that lets src/test/nxt_port_fd_test.c invoke the static + * nxt_port_read_msg_process() directly with a synthesised port and message + * -- used to verify the tail close of whatever a dispatch leaves in + * msg->fd[], and the close in the last-fragment merge that runs before + * msg->fd[0] is overwritten by the assembled message's own descriptors + * (freeunitorg/freeunit#342, "port: close the descriptors a handler does + * not take"). + */ +void +nxt_port_test_run_read_msg_process(nxt_task_t *task, nxt_port_t *port, + nxt_port_recv_msg_t *msg) +{ + nxt_port_read_msg_process(task, port, msg); +} + #endif @@ -185,6 +240,55 @@ nxt_port_release_send_msg(nxt_port_send_msg_t *msg) } +/* + * Give up on a message that nxt_port_write_msgs() is dropping. Complete + * what it still owns. + * + * This applies only to a message not yet in port->messages, msg->link.next + * == NULL: the caller's stack copy from the nxt_port_msg_chk_insert() + * NXT_DECLINED branch. It cannot double-complete, because the + * nxt_port_error_handler() the caller raises next completes only what the + * queue holds, and this message was never in it. + * + * nxt_port_socket_write2() answers NXT_OK on the paths that still reach + * here, on purpose. Callers that ignore the return rely on the port to + * finish what they handed it, such as the START_PROCESS deadline in + * src/nxt_router.c; skipping this call would orphan those buffers instead. + * The inline first fragment of a send to a dead peer no longer comes this + * way: nothing of it was consumed, so it is reported and stays the + * caller's. + * + * Close descriptors first, then complete buffers -- the order + * nxt_port_error_handler() uses. + */ + +static void +nxt_port_msg_drop(nxt_task_t *task, nxt_port_send_msg_t *msg) +{ + nxt_buf_t *b, *next; + nxt_work_queue_t *wq; + + nxt_port_msg_close_fd(msg); + + wq = &task->thread->engine->fast_work_queue; + + for (b = msg->buf; b != NULL; b = next) { + next = b->next; + b->next = NULL; + + if (nxt_buf_is_sync(b)) { + continue; + } + + nxt_work_queue_add(wq, b->completion_handler, task, b, b->parent); + } + + msg->buf = NULL; + + nxt_port_release_send_msg(msg); +} + + nxt_int_t nxt_port_socket_write2(nxt_task_t *task, nxt_port_t *port, nxt_uint_t type, nxt_fd_t fd, nxt_fd_t fd2, uint32_t stream, nxt_port_id_t reply_port, @@ -193,12 +297,16 @@ nxt_port_socket_write2(nxt_task_t *task, nxt_port_t *port, nxt_uint_t type, int notify; uint8_t qmsg_size; nxt_int_t res; + nxt_bool_t enqueued, send_failed; nxt_port_send_msg_t msg; struct { nxt_port_msg_t pm; uint8_t buf[NXT_PORT_MAX_ENQUEUE_BUF_SIZE]; } qmsg; + enqueued = 0; + send_failed = 0; + msg.link.next = NULL; msg.link.prev = NULL; @@ -209,6 +317,24 @@ nxt_port_socket_write2(nxt_task_t *task, nxt_port_t *port, nxt_uint_t type, msg.close_fd = (type & NXT_PORT_MSG_CLOSE_FD) != 0; msg.allocated = 0; + /* + * Set the flag here, from the type the caller asked for. The shared + * queue branch below can change the socket message into a READ_QUEUE + * wake-up, and nxt_port_msg_chk_insert() copies the message. The flag + * is written once, before the message is published under + * port->write_mutex. Only the sender reads it, after that. So the + * flag needs no lock of its own. + */ + msg.peer_may_be_gone = ((type & NXT_PORT_MSG_MASK) == _NXT_PORT_MSG_QUIT); + + /* + * Set port->quit_sent before the QUIT can reach the shared queue. + * A wake-up that is already pending then covers the QUIT as well. + */ + if (msg.peer_may_be_gone) { + (void) nxt_atomic_cmp_set(&port->quit_sent, 0, 1); + } + msg.port_msg.stream = stream; msg.port_msg.pid = nxt_pid; msg.port_msg.reply_port = reply_port; @@ -220,6 +346,15 @@ nxt_port_socket_write2(nxt_task_t *task, nxt_port_t *port, nxt_uint_t type, if (port->queue != NULL && type != _NXT_PORT_MSG_READ_QUEUE) { + /* + * A QUIT goes into the shared queue like any other message. + * A worker that is still in nxt_unit_init() holds only one socket + * message with no queue marker ("too many port socket messages"). + * A QUIT on the socket would be a second one. So the wake-up for + * a QUIT is a plain READ_QUEUE. msg.peer_may_be_gone, set above, + * still marks it as a QUIT. A failed wake-up to a worker that is + * gone is then logged at info. + */ if (fd == -1 && nxt_port_can_enqueue_buf(b)) { qmsg.pm = msg.port_msg; @@ -235,6 +370,10 @@ nxt_port_socket_write2(nxt_task_t *task, nxt_port_t *port, nxt_uint_t type, (int) port->pid, (int) port->id, port->socket.fd, (int) qmsg_size, notify, res); + if (nxt_slow_path(res == NXT_ERROR)) { + goto queue_broken; + } + if (b != NULL && nxt_fast_path(res == NXT_OK)) { if (qmsg.pm.mmap) { b->is_port_mmap_sent = 1; @@ -250,6 +389,13 @@ nxt_port_socket_write2(nxt_task_t *task, nxt_port_t *port, nxt_uint_t type, return res; } + /* + * The payload is in the shared queue and its completion is + * already queued: b belongs to the port from here on, whatever + * becomes of the wake-up below. + */ + enqueued = 1; + msg.port_msg.type = _NXT_PORT_MSG_READ_QUEUE; msg.buf = NULL; @@ -262,6 +408,10 @@ nxt_port_socket_write2(nxt_task_t *task, nxt_port_t *port, nxt_uint_t type, (int) port->pid, (int) port->id, port->socket.fd, notify, res); + if (nxt_slow_path(res == NXT_ERROR)) { + goto queue_broken; + } + if (nxt_slow_path(res == NXT_AGAIN)) { return NXT_AGAIN; } @@ -270,11 +420,82 @@ nxt_port_socket_write2(nxt_task_t *task, nxt_port_t *port, nxt_uint_t type, res = nxt_port_msg_chk_insert(task, port, &msg); if (nxt_fast_path(res == NXT_DECLINED)) { - nxt_port_write_handler(task, &port->socket, &msg); + /* + * Inline, with this stack copy: the answer is whether the port took + * the message, not whether the call ran. On NXT_ERROR nothing of it + * was consumed, so the caller still owns fd, fd2 and b -- the + * contract in src/nxt_port.h, and the same state this function + * already answers for the allocation failure above. + */ + res = nxt_port_write_msgs(task, &port->socket, &msg, &send_failed); + } + + if (nxt_slow_path(res != NXT_OK && enqueued)) { + /* + * Only the wake-up failed. The buffer already went into the shared + * queue above, with its completion queued, so answering anything + * but NXT_OK would hand back a buffer this function no longer owns + * -- and a caller that cleans up after a failed write would + * complete it a second time. nxt_runtime_port_send_quit() + * (src/nxt_runtime.c) is such a caller. + * + * The message sits in the queue, but nothing told the peer to look. + * nxt_port_queue_send() raises "notify" only on the 0-to-1 + * transition (src/nxt_port_queue.h), so later enqueues on this port + * stay silent while this item keeps nitems above zero. + * + * After EAGAIN the peer still gets it. That errno means the socket + * holds data the peer has not read, and nxt_port_queue_read_handler() + * drains the ring before it reads the socket, so the next read pass + * takes this item too. + * + * ENOBUFS and ENOMEM promise nothing: nxt_socketpair_send() maps + * both to NXT_AGAIN as well (src/nxt_socketpair.c), and they mean + * the system could not allocate for the call, not that the peer + * owes this port a read. + * + * So record that a marker is owed and re-arm. nxt_port_announce() + * sends it from the stack with no allocation, either here on this + * port's own engine or from the item nxt_port_rearm() posts. + * + * Unless the send failed outright. Then it did not hit a transient + * condition: nxt_socketpair_send() answers NXT_AGAIN for EAGAIN, + * ENOBUFS and ENOMEM and retries EINTR, so everything that reaches + * here is an error that will not get better -- a dead peer + * routinely, and EMSGSIZE or EBADF otherwise. Every one of them + * ends this port: the marker can never be delivered, and re-arming + * the write event only brings this function back to fail the same + * send, leaving another queued error handler each pass while the + * event stays armed on a port that is about to be freed. + */ + + if (nxt_fast_path(!send_failed)) { + nxt_atomic_fetch_add(&port->announce, 1); + + nxt_port_rearm(task, port); + } + res = NXT_OK; } return res; + +queue_broken: + + /* + * The peer broke the shared queue and nxt_nncq.h gave up on it. + * Nothing of the message was consumed, so the caller still owns fd, + * fd2 and b, as after a send to a dead peer. Later messages meet the + * same bounded retries: the queue cannot be dropped from a port that + * other engines send on. + */ + + nxt_alert(task, "port{%d,%d} %d: shared queue is broken; " + "message type %d stream #%uD refused", + (int) port->pid, (int) port->id, port->socket.fd, + (int) msg.port_msg.type, stream); + + return NXT_ERROR; } @@ -332,7 +553,11 @@ static nxt_int_t nxt_port_msg_chk_insert(nxt_task_t *task, nxt_port_t *port, nxt_port_send_msg_t *msg) { - nxt_int_t res; + nxt_int_t res; + nxt_bool_t has_fd, over_bound; + + has_fd = nxt_port_msg_has_fd(msg); + over_bound = 0; nxt_thread_mutex_lock(&port->write_mutex); @@ -341,11 +566,34 @@ nxt_port_msg_chk_insert(nxt_task_t *task, nxt_port_t *port, { res = NXT_DECLINED; + } else if (nxt_slow_path(has_fd + && port->fd_messages >= NXT_PORT_MAX_FD_MSGS)) + { + /* + * Refuse rather than drop. NXT_ERROR from here says nothing of the + * message was consumed and it is still the caller's -- the same + * answer this function already gives when nxt_port_msg_alloc() + * fails, and the contract in src/nxt_port.h -- so every caller that + * sends a descriptor already has cleanup for it. + * + * Logged once per stall, not once per refusal: see ->fd_refusing. + */ + + over_bound = !port->fd_refusing; + port->fd_refusing = 1; + res = NXT_ERROR; + } else { msg = nxt_port_msg_alloc(msg); if (nxt_fast_path(msg != NULL)) { nxt_queue_insert_tail(&port->messages, &msg->link); + + if (has_fd) { + port->fd_messages++; + port->fd_refusing = 0; + } + nxt_port_use(task, port, 1); res = NXT_OK; @@ -356,6 +604,13 @@ nxt_port_msg_chk_insert(nxt_task_t *task, nxt_port_t *port, nxt_thread_mutex_unlock(&port->write_mutex); + if (nxt_slow_path(over_bound)) { + nxt_alert(task, "port{%d,%d} %d: %d queued messages already hold a " + "descriptor, refusing to queue another", + (int) port->pid, (int) port->id, port->socket.fd, + NXT_PORT_MAX_FD_MSGS); + } + return res; } @@ -381,47 +636,521 @@ nxt_port_msg_alloc(const nxt_port_send_msg_t *m) msg->allocated = 1; + /* + * A queued message must own the descriptors it names. + * + * The caller's copy borrows them: NEW_PORT names another port's pair[1] + * and queue_fd, START_PROCESS the application's shared port. The + * sendmsg() that puts them into SCM_RIGHTS runs later, from + * nxt_port_write_msgs(), and by then the owner may have closed them -- + * nxt_port_close() (src/nxt_port.c) does when a port goes away -- and + * the kernel may have given the numbers to something else. The deferred + * send would then fail with EBADF, or hand the peer whatever descriptor + * now sits at that number. + * + * So duplicate them here, and mark the copy as owning what it sends. + * From here on the send path closes them the way it closes any owned + * descriptor: after the send in nxt_port_write_msgs(), or from + * nxt_port_error_handler() and nxt_port_socket_cancel() when the message + * is dropped instead. The originals stay with their owner. + * + * The dup holds the descriptor until the message goes out, so a peer + * that stops reading holds this side's descriptor table open in + * proportion to what it was sent. That is what NXT_PORT_MAX_FD_MSGS + * bounds; the callers of this function check it before they get here. + */ + + if (!msg->close_fd && nxt_slow_path(nxt_port_msg_dup_fds(msg) != NXT_OK)) { + nxt_free(msg); + return NULL; + } + return msg; } +/* + * Replace a borrowed fd[0]/fd[1] with duplicates the message owns. On + * failure nothing is changed: a first duplicate is closed again, so the + * caller keeps exactly its originals. + */ + +static nxt_int_t +nxt_port_msg_dup_fds(nxt_port_send_msg_t *msg) +{ + nxt_fd_t fd0, fd1; + + if (msg->fd[0] == -1 && msg->fd[1] == -1) { + return NXT_OK; + } + + fd0 = -1; + fd1 = -1; + + if (msg->fd[0] != -1) { + fd0 = fcntl(msg->fd[0], F_DUPFD_CLOEXEC, 0); + if (nxt_slow_path(fd0 == -1)) { + nxt_thread_log_alert("dup(%FD) failed %E", msg->fd[0], nxt_errno); + return NXT_ERROR; + } + } + + if (msg->fd[1] != -1) { + fd1 = fcntl(msg->fd[1], F_DUPFD_CLOEXEC, 0); + if (nxt_slow_path(fd1 == -1)) { + nxt_thread_log_alert("dup(%FD) failed %E", msg->fd[1], nxt_errno); + + if (fd0 != -1) { + nxt_fd_close(fd0); + } + + return NXT_ERROR; + } + } + + msg->fd[0] = fd0; + msg->fd[1] = fd1; + msg->close_fd = 1; + + return NXT_OK; +} + + +/* + * Disable, not block. Blocking leaves the descriptor registered and only + * has the poller ignore what it reports; in edge-triggered mode that report + * is then lost, and the enable that later re-arms a blocked event touches + * the registration not at all. That is fine after EAGAIN, where the + * socket is full and the peer's next read raises a fresh edge. After a + * send that failed for want of kernel memory it is not: the socket stayed + * writable, no edge is coming, and the message queued for a later attempt + * would wait on an event that never fires. Disabling drops the + * registration, so the re-arm adds it back and the poller re-checks + * readiness on the spot (#392). + * + * The cost is one epoll_ctl() here and one on the next EAGAIN, both off + * the inline path. Only the event-loop pass of nxt_port_write_msgs() -- + * data == NULL, the queue drained -- asks for this, and that pass runs on + * port->engine, where nxt_port_post() calls the handler directly: the + * post that allocates, and whose failure would skip this, is not reached + * from here. + * + * Both tests below are repeated from the caller, for the same reason + * nxt_port_announce() repeats its own: the descriptor, because a port can + * be closed while this item is in flight and nxt_port_write_close() leaves + * socket.fd at the number it had, so the disable would go out on a closed + * or already reused descriptor (see nxt_port_rearm_now()); the event state, + * because nxt_fd_event_disable_write() carries no guard of its own, unlike + * the nxt_fd_event_block_write() this replaced, and an EV_DISABLE on a + * kqueue knote that was never added is ENOENT. + */ + static void -nxt_port_fd_block_write(nxt_task_t *task, nxt_port_t *port, void *data) +nxt_port_fd_disable_write(nxt_task_t *task, nxt_port_t *port, void *data) +{ + if (port->pair[1] == -1 || !nxt_fd_event_is_active(port->socket.write)) { + return; + } + + nxt_fd_event_disable_write(task->thread->engine, &port->socket); +} + + +/* + * Tell the peer the shared queue has something in it, when the marker that + * should have said so could not be written. + * + * The marker is a bare header with no buffer and no descriptor, so it goes out + * from the stack with one sendmsg() and allocates nothing -- which is the + * point, because this runs after an allocation failure. The flag is cleared + * only once the marker has left: a peer that reads one drains the whole ring, + * so anything enqueued in between is covered by the same marker. + * + * Return values: + * + * NXT_AGAIN indicates the kernel rejected the marker due to insufficient + * memory; the caller must delay the next attempt. NXT_OK indicates the + * port owes nothing, the marker was sent, or the write returned EAGAIN. + * After EAGAIN, the socket buffer is full, so the next read by the peer + * triggers a write event. If the port is already closed (pair[1] is -1), + * the function also returns NXT_OK. NXT_ERROR indicates an unrecoverable + * socket error. In that case, the function transfers queued messages to + * nxt_port_error_handler(), and the caller must neither enable the write + * event nor schedule a retry. + */ + +static nxt_int_t +nxt_port_announce(nxt_task_t *task, nxt_port_t *port) { - nxt_fd_event_block_write(task->thread->engine, &port->socket); + ssize_t n; + nxt_fd_t fd[2]; + struct iovec iov; + nxt_port_msg_t msg; + + /* + * pair[1], not socket.fd: nxt_port_write_close() clears the first and + * leaves the second at the number it had, so socket.fd tells you nothing + * about whether this port may still be written to. The caller checks the + * same thing before it arms the event; this repeats it because the test + * is cheap and the consequence of getting it wrong is writing to somebody + * else's descriptor. + */ + + if (port->announce == 0 || port->pair[1] == -1) { + return NXT_OK; + } + + /* nxt_socketpair_send() reads both, whether or not it sends them. */ + + fd[0] = -1; + fd[1] = -1; + + nxt_memzero(&msg, sizeof(nxt_port_msg_t)); + + msg.type = _NXT_PORT_MSG_READ_QUEUE; + msg.pid = nxt_pid; + msg.last = 1; + + iov.iov_base = &msg; + iov.iov_len = sizeof(nxt_port_msg_t); + + n = nxt_socketpair_send(&port->socket, fd, &iov, 1); + + if (n == (ssize_t) sizeof(nxt_port_msg_t)) { + nxt_atomic_fetch_add(&port->announce, -1); + + /* + * The shortage is over, as after a queued send that went out. This + * runs on port->engine, from nxt_port_rearm_now(), so the reset + * keeps the ownership rule the one in nxt_port_write_msgs() keeps. + */ + + port->retry_delay = 0; + + nxt_debug(task, "port{%d,%d} %d: queue announced", (int) port->pid, + (int) port->id, port->socket.fd); + + return NXT_OK; + } + + /* + * port->socket.error is this send's and not a stale one: the tests above + * guarantee the send ran, and nxt_socketpair_send() writes the field on + * every outcome. + */ + + if (n == NXT_AGAIN) { + return (port->socket.error == NXT_EAGAIN) ? NXT_OK : NXT_AGAIN; + } + + /* + * If the peer closed the socket or an unrecoverable error occurred, + * do not schedule a retry. Delaying retries would leave the timer + * running indefinitely without re-enabling the write event. As a + * result, queued messages would never reach the error handler. + * + * Transfer queued messages to nxt_port_error_handler() now, matching + * the fail path in nxt_port_write_msgs(). Decrement announce by 1; + * any leftover count on a failed port is harmless because NXT_ERROR + * never arms the retry timer. + */ + + nxt_atomic_fetch_add(&port->announce, -1); + + nxt_port_inc_use(port); + + nxt_work_queue_add(&task->thread->engine->fast_work_queue, + nxt_port_error_handler, task, &port->socket, + &port->socket); + + return NXT_ERROR; } +/* + * The work itself, on port->engine and touching neither the flag nor the + * reference count: both belong to whoever posted, and a caller already on + * this engine posts nothing. + * + * The descriptor is checked because a port can be closed while an item is in + * flight: nxt_port_write_close() sets pair[1] to -1 but leaves socket.fd at + * the number it had, so enabling would register that number -- possibly + * somebody else's by then -- against this port's handlers. Every close runs + * on port->engine, which is this thread, so the read is stable. + */ + static void -nxt_port_fd_enable_write(nxt_task_t *task, nxt_port_t *port, void *data) +nxt_port_rearm_now(nxt_task_t *task, nxt_port_t *port) { + nxt_int_t ret; + + if (port->pair[1] == -1) { + return; + } + + /* + * A paced retry already owes this port both the enable and the marker: + * the timer ends here. Enabling now would bring the write handler back + * on a socket that stayed writable, which is the spin the timer is there + * to stop, and it would also leave the event active under the timer, so + * the timer's own enable could not force a fresh readiness check. The + * new message waits at most the current delay, as a paced retry does. + */ + + if (port->retry_timer.enabled) { + return; + } + + /* + * A marker that could not be written is the second way this port can + * spin: nxt_port_announce() clears ->announce only on a send that fully + * succeeded, and an enabled event brings the write handler straight + * back, which finds nothing to send, disables the event, and re-arms on + * the owed marker again. Pacing it here rather than at that caller + * covers every one of them, since this is the only place the marker is + * ever attempted. The marker goes first so that a failed one leaves + * the event down, instead of an enable that the retry takes back in the + * same pass -- two epoll_ctl() calls that cancel out. If sending the + * marker fails permanently, nxt_port_announce() hands the port to + * nxt_port_error_handler(), and there is nothing left to arm. + */ + + ret = nxt_port_announce(task, port); + + if (nxt_slow_path(ret != NXT_OK)) { + if (ret == NXT_AGAIN) { + nxt_port_retry_later(task, port); + } + + return; + } + nxt_fd_event_enable_write(task->thread->engine, &port->socket); } +/* + * Arm the retry instead of making it now. + * + * The event is disabled and a timer re-arms it after a delay. What that + * buys is in the comment on ->retry_timer in src/nxt_port.h; what it costs + * is retry latency, up to NXT_PORT_RETRY_MAX_DELAY once a shortage has + * lasted long enough for the delay to reach the cap. That is a deliberate + * loss of immediacy on a path #393 made immediate on purpose, and the trade + * is against burning a core in a tight retry loop at the one moment the + * machine has nothing to spare. + * + * Disabling, rather than blocking, for the reason given above + * nxt_port_fd_disable_write(): a registration left in place reports nothing + * more on a socket that never stopped being writable, so the re-arm has to + * add it back. That function also carries the test for a caller that has + * disabled the event already. + * + * Callable on port->engine only. Both callers are there: the drained pass + * of nxt_port_write_msgs() runs on the port's own engine, and + * nxt_port_rearm_now() is what nxt_port_rearm() posts to it. That is what + * makes the timer safe without a lock. + */ + static void -nxt_port_write_handler(nxt_task_t *task, void *obj, void *data) +nxt_port_retry_later(nxt_task_t *task, nxt_port_t *port) +{ + nxt_event_engine_t *engine; + + if (port->retry_timer.enabled || port->pair[1] == -1) { + return; + } + + engine = port->engine; + + nxt_port_fd_disable_write(task, port, NULL); + + port->retry_delay = (port->retry_delay == 0) + ? NXT_PORT_RETRY_MIN_DELAY + : nxt_min(port->retry_delay * 2, + NXT_PORT_RETRY_MAX_DELAY); + + port->retry_timer.work_queue = &engine->fast_work_queue; + port->retry_timer.handler = nxt_port_retry_handler; + port->retry_timer.task = &engine->task; + port->retry_timer.log = engine->task.log; + + /* + * The reference is what keeps the port -- and the timer, which lives in + * it -- alive until the handler runs. It is also the whole of the close + * handling: an armed timer cannot be cancelled from another engine, and + * cancelling it here would leave nxt_timer_handler() with a pointer into + * a pool this drop had just released. So nothing cancels it. The timer + * fires within the cap whatever became of the port, finds a closed one + * by its pair[1] and does nothing with it, and drops this reference. + * + * The one case this does not cover is an engine that is freed with the + * timer still in its tree: the handler never runs, and the port leaks + * with this reference. That is a leak, not a use-after-free, and an + * engine is freed only when its thread exits (a router thread that a + * lower "listen_threads" removed) or when the process does. + */ + + nxt_port_inc_use(port); + + nxt_timer_add(engine, &port->retry_timer, port->retry_delay); + + nxt_debug(task, "port{%d,%d} %d: retry in %M ms", (int) port->pid, + (int) port->id, port->socket.fd, port->retry_delay); +} + + +static void +nxt_port_retry_handler(nxt_task_t *task, void *obj, void *data) +{ + nxt_port_t *port; + + port = nxt_timer_data(obj, nxt_port_t, retry_timer); + + nxt_port_rearm_now(task, port); + + /* + * Last, because nxt_port_rearm_now() may have armed the timer again -- + * on its own reference, which is why this one can be the last without + * taking the timer down with it. + */ + + nxt_port_use(task, port, -1); +} + + +/* + * Clear ->rearm_pending before the work, not after: a poster that arrives + * while this runs then posts again rather than dropping the re-arm, and + * enabling the same write event twice costs nothing. The reverse order can + * lose one, because a poster that sees the flag set cannot know whether the + * enable it skipped has happened yet. + * + * Only this handler clears it, and only because the post that queued this + * item set it. The flag counts posts, not re-arms. + */ + +static void +nxt_port_rearm_work_handler(nxt_task_t *task, void *obj, void *data) +{ + nxt_port_t *port; + + port = obj; + + nxt_atomic_fetch_add(&port->rearm_pending, -1); + + nxt_port_rearm_now(task, port); + + /* + * The reference the post took. It runs on port->engine, so this drop + * takes the on-engine branch of nxt_port_use() and releases the port only + * if nobody else holds it -- the same discipline nxt_port_post() uses. + */ + + nxt_port_use(task, port, -1); +} + + +/* + * Re-arm the write event, and announce the queue if a marker is owed. + * + * On port->engine both are immediate. From another engine they have to be + * posted, and nxt_port_post() would allocate the item -- which fails exactly + * when this is needed, since the write that brought us here ran out of memory + * too. So the port carries its own item, and ->rearm_pending keeps it off the + * engine's queue twice: linking one item as its own successor would spin + * nxt_locked_work_queue_add() forever. + */ + +void +nxt_port_rearm(nxt_task_t *task, nxt_port_t *port) +{ + if (task->thread->engine == port->engine) { + nxt_port_rearm_now(task, port); + + return; + } + + if (!nxt_atomic_cmp_set(&port->rearm_pending, 0, 1)) { + return; + } + + nxt_atomic_fetch_add(&port->use_count, 1); + + port->rearm_work.handler = nxt_port_rearm_work_handler; + port->rearm_work.task = &port->engine->task; + port->rearm_work.obj = port; + port->rearm_work.data = NULL; + port->rearm_work.next = NULL; + + nxt_event_engine_post(port->engine, &port->rearm_work); +} + + +/* + * Write what the port has and report whether an inline message was taken. + * + * The event loop never sets the port socket's own ->data, so data != NULL + * marks the inline call from nxt_port_socket_write2(), with "msg" as that + * caller's stack copy. Only then does the return value matter -- a queued + * message belongs to the port either way, with nobody left to tell. + * + * NXT_ERROR means the message was neither sent nor queued, and nothing of + * it was consumed. See the ownership contract this promises the caller in + * the comment above the inline call in nxt_port_socket_write2(). + * + * Two exits answer NXT_ERROR, both for a first fragment sent inline: an + * EAGAIN with no memory to hold it for a later attempt, and a send that + * fails outright, which routinely means the peer is gone. + * + * "send_failed" separates them for the one caller that has to tell them + * apart, and is set on that second exit alone. It says what it tests: the + * sendmsg() failed with something nxt_socketpair_send() does not retry, of + * which a dead peer is the routine case. Such a port arms nothing -- it + * owes no queue marker, and re-arming its write event only schedules the + * same failing send again. + * + * The other three exits that raise the error handler -- a short write, and + * two allocation failures after a fragment has gone out -- leave a socket + * that still works, so they keep re-arming on an owed marker. The handler + * itself does not decide this either way: it drains the port's queued + * messages and drops one reference, and closes nothing. + */ + +static nxt_int_t +nxt_port_write_msgs(nxt_task_t *task, void *obj, void *data, + nxt_bool_t *send_failed) { int use_delta; size_t plain_size; ssize_t n; + nxt_int_t ret; uint32_t mmsg_buf[3 * NXT_IOBUF_MAX * 10]; - nxt_bool_t block_write, enable_write; + nxt_bool_t failed, block_write, enable_write, retry_later; nxt_port_t *port; struct iovec iov[NXT_IOBUF_MAX * 10]; nxt_work_queue_t *wq; nxt_port_method_t m; - nxt_port_send_msg_t *msg; + nxt_port_send_msg_t *msg, *qmsg; nxt_sendbuf_coalesce_t sb; port = nxt_container_of(obj, nxt_port_t, socket); + ret = NXT_OK; + failed = 0; block_write = 0; enable_write = 0; + retry_later = 0; use_delta = 0; wq = &task->thread->engine->fast_work_queue; +#if (NXT_TESTS) + if (data == NULL) { + nxt_port_test_write_dispatches++; + } +#endif + do { if (data) { msg = data; @@ -467,8 +1196,12 @@ nxt_port_write_handler(nxt_task_t *task, void *obj, void *data) /* * Send through mmap enabled only when payload - * is bigger than PORT_MMAP_MIN_SIZE. + * is bigger than PORT_MMAP_MIN_SIZE. The bit is per fragment: a + * fragment sent plain after one sent through mmap must not carry + * it, or the receiver reads its payload as mmap records. */ + msg->port_msg.mmap = 0; + if (m == NXT_PORT_METHOD_MMAP && plain_size > PORT_MMAP_MIN_SIZE) { nxt_port_mmap_write(task, port, msg, &sb, mmsg_buf); @@ -479,15 +1212,33 @@ nxt_port_write_handler(nxt_task_t *task, void *obj, void *data) msg->port_msg.last |= sb.last; msg->port_msg.mf = sb.limit_reached || sb.nmax_reached; - n = nxt_socketpair_send(&port->socket, msg->fd, iov, sb.niov + 1); + n = nxt_socketpair_send_ex(&port->socket, msg->fd, iov, sb.niov + 1, + msg->peer_may_be_gone + || port->quit_sent != 0); if (n > 0) { + /* + * The shortage, if there was one, is over: start the next one + * from the short delay again rather than from wherever the last + * one left off. On the drained pass alone, because that is the + * one that runs on port->engine, and ->retry_delay is owned by + * that engine. An inline send is a poor signal anyway: it can + * come from any thread at any moment, including between two + * retries that are both about to fail. + */ + + if (data == NULL) { + port->retry_delay = 0; + } + if (nxt_slow_path((size_t) n != sb.size + iov[0].iov_len)) { nxt_alert(task, "port %d: short write: %z instead of %uz", port->socket.fd, n, sb.size + iov[0].iov_len); goto fail; } + nxt_port_msg_fd_uncount(port, msg); + nxt_port_msg_close_fd(msg); msg->buf = nxt_port_buf_completion(task, wq, msg->buf, plain_size, @@ -518,11 +1269,15 @@ nxt_port_write_handler(nxt_task_t *task, void *obj, void *data) nxt_thread_mutex_unlock(&port->write_mutex); } else { - msg = nxt_port_msg_insert_tail(port, msg); - if (nxt_slow_path(msg == NULL)) { + qmsg = nxt_port_msg_insert_tail(port, msg); + if (nxt_slow_path(qmsg == NULL)) { + nxt_port_msg_drop(task, msg); + goto fail; } + msg = qmsg; + use_delta++; } @@ -550,54 +1305,251 @@ nxt_port_write_handler(nxt_task_t *task, void *obj, void *data) } } else { + /* + * A send that failed outright -- routinely a dead peer, but + * nxt_socketpair_send() retries or defers everything except a + * non-recoverable errno, so this covers all of them. Nothing of + * the message was consumed -- nxt_socketpair_send() sent no + * bytes and no descriptor, and nxt_port_msg_close_fd() runs only + * after a successful send -- so an inline first fragment is + * still whole and still the caller's. Saying so is what lets + * the caller stop waiting for a reply that is not coming. + * + * Once a fragment has gone out the message is no longer whole, + * and a queued message has no caller to tell, so both keep the + * old treatment: the port owns them, and nxt_port_msg_drop() + * finishes them. + */ + if (nxt_slow_path(n == NXT_ERROR)) { + failed = 1; + if (msg->link.next == NULL) { - nxt_port_msg_close_fd(msg); + if (data != NULL && msg->port_msg.nf == 0) { + ret = NXT_ERROR; - nxt_port_release_send_msg(msg); + } else { + nxt_port_msg_drop(task, msg); + } } - goto fail; - } + goto fail; + } + + /* + * NXT_AGAIN. After EAGAIN the socket is full, and the peer's + * next read raises the edge that brings this handler back. + * After ENOBUFS or ENOMEM it is not full: the kernel could not + * allocate for the call, and no edge is coming. This pass + * finds the event active, and an active edge-triggered event + * is never re-added, so a retry must force a readiness re-check. + * + * The retry is paced rather than attempted on the spot. The + * socket is writable, so an immediate re-check fails again + * while the memory shortage lasts (#407). Instead, + * nxt_port_retry_later() disables the event and sets a timer + * to re-enable it. Because it handles the event disable directly, + * block_write is not set here. + */ + + if (data == NULL && port->socket.error != NXT_EAGAIN) { + retry_later = 1; + } + + if (msg->link.next == NULL) { + qmsg = nxt_port_msg_insert_tail(port, msg); + if (nxt_slow_path(qmsg == NULL)) { + /* + * EAGAIN, then no memory to hold the message for a later + * attempt. The socket is alive and nothing of the + * message was consumed: nxt_socketpair_send() sent no + * bytes and no descriptor, and nxt_port_msg_close_fd() + * runs only after a successful send. So an inline + * caller can be told, and can act on it: keep its stream + * armed, or complete its own payload. + * + * This applies only to the first fragment. Once a + * fragment has gone out, ->nf is set, the descriptor is + * closed, and the sent buffers are completed, so the + * port owns the cleanup, not the caller. + */ + + if (data != NULL && msg->port_msg.nf == 0) { + ret = NXT_ERROR; + + /* + * Leave the loop the ordinary way instead of raising + * the error handler. The socket is alive, and the + * ordinary exit re-arms the write event that + * nxt_socketpair_send() just cleared. Without that + * re-arm, everything queued on this port from here + * on -- including work another thread inserted + * during this send -- would wait for a writable + * event that never comes. + */ + + break; + } + + nxt_port_msg_drop(task, msg); + + goto fail; + } + + msg = qmsg; + + use_delta++; + } + } + + } while (port->socket.write_ready); + + if (nxt_fd_event_is_disabled(port->socket.write)) { + enable_write = 1; + } + + goto cleanup; + +fail: + + use_delta++; + + nxt_work_queue_add(wq, nxt_port_error_handler, task, &port->socket, + &port->socket); + +cleanup: + + if (block_write && nxt_fd_event_is_active(port->socket.write)) { + nxt_port_post(task, port, nxt_port_fd_disable_write, NULL); + } + + /* + * Not on a port whose send failed outright: the event-loop pass and the + * inline one both reach this through "fail:", and neither has anything + * to gain from arming a socket that has just refused to carry a message. + * + * A paced retry answers for the owed marker as well -- the timer ends in + * nxt_port_rearm_now(), which sends it -- and arming both would put the + * write event back up at once and undo the pacing. Otherwise an owed + * marker is a reason to come back even when the event needs no arming, + * since the re-arm handler is the only place that attempts it. + */ + + if (!failed) { + if (retry_later) { + nxt_port_retry_later(task, port); + + } else if (enable_write || port->announce != 0) { + nxt_port_rearm(task, port); + } + } + + if (use_delta != 0) { + nxt_port_use(task, port, use_delta); + } + + if (send_failed != NULL) { + *send_failed = failed; + } + + return ret; +} + + +static void +nxt_port_write_handler(nxt_task_t *task, void *obj, void *data) +{ + (void) nxt_port_write_msgs(task, obj, data, NULL); +} + + +nxt_port_msg_cancel_t +nxt_port_socket_cancel(nxt_task_t *task, nxt_port_t *port, nxt_uint_t type, + uint32_t stream, nxt_port_id_t reply_port, nxt_buf_t *b) +{ + nxt_buf_t *buf, *next; + nxt_work_queue_t *wq; + nxt_port_send_msg_t *msg, *found; + nxt_port_msg_cancel_t ret; + + found = NULL; + ret = NXT_PORT_MSG_NOT_FOUND; + + nxt_thread_mutex_lock(&port->write_mutex); + + nxt_queue_each(msg, &port->messages, nxt_port_send_msg_t, link) { + + if (msg->port_msg.stream != stream + || msg->port_msg.type != (type & NXT_PORT_MSG_MASK) + || msg->port_msg.reply_port != reply_port) + { + continue; + } + + if (b != NULL && msg->buf != b) { + continue; + } + + /* + * nxt_port_write_handler() sets nf on the message once it has sent a + * fragment, and clears msg->fd[] at the same time: from then on the + * peer is reading a stream this message must finish. + */ + + if (msg->port_msg.nf) { + ret = NXT_PORT_MSG_STARTED; + break; + } + + nxt_port_msg_fd_uncount_locked(port, msg); + + nxt_queue_remove(&msg->link); + msg->link.next = NULL; + + found = msg; + ret = NXT_PORT_MSG_CANCELLED; - if (msg->link.next == NULL) { - msg = nxt_port_msg_insert_tail(port, msg); - if (nxt_slow_path(msg == NULL)) { - goto fail; - } + break; - use_delta++; - } - } + } nxt_queue_loop; - } while (port->socket.write_ready); + nxt_thread_mutex_unlock(&port->write_mutex); - if (nxt_fd_event_is_disabled(port->socket.write)) { - enable_write = 1; + if (found == NULL) { + return ret; } - goto cleanup; + /* + * Outside the mutex from here: a completion handler may reach arbitrary + * router code, and nxt_port_use() can destroy the port. + */ -fail: + buf = found->buf; + found->buf = NULL; - use_delta++; + nxt_port_msg_close_fd(found); - nxt_work_queue_add(wq, nxt_port_error_handler, task, &port->socket, - &port->socket); + nxt_port_release_send_msg(found); -cleanup: + wq = &task->thread->engine->fast_work_queue; - if (block_write && nxt_fd_event_is_active(port->socket.write)) { - nxt_port_post(task, port, nxt_port_fd_block_write, NULL); - } + while (buf != NULL) { + next = buf->next; + buf->next = NULL; - if (enable_write) { - nxt_port_post(task, port, nxt_port_fd_enable_write, NULL); - } + if (!nxt_buf_is_sync(buf)) { + nxt_work_queue_add(wq, buf->completion_handler, task, buf, + buf->parent); + } - if (use_delta != 0) { - nxt_port_use(task, port, use_delta); + buf = next; } + + /* The reference nxt_port_msg_chk_insert() took for the queued message. */ + + nxt_port_use(task, port, -1); + + return ret; } @@ -624,6 +1576,67 @@ nxt_port_msg_first(nxt_port_t *port) } +nxt_inline nxt_bool_t +nxt_port_msg_has_fd(const nxt_port_send_msg_t *msg) +{ + return msg->fd[0] != -1 || msg->fd[1] != -1; +} + + +/* + * Stop counting a queued message against NXT_PORT_MAX_FD_MSGS. + * + * The count follows one invariant: an entry of port->messages is counted + * exactly while it carries a descriptor. So this runs at the two moments + * that end it -- the message leaves the queue, or it is still queued but has + * just had its descriptors sent -- and must run before whatever clears + * msg->fd[], since that is what it reads. msg->link.next is the queue's own + * "is it in the list" marker, which keeps the inline stack copy in + * nxt_port_socket_write2() out of the count. + * + * The _locked form is for the callers that already hold port->write_mutex. + */ + +nxt_inline void +nxt_port_msg_fd_uncount_locked(nxt_port_t *port, nxt_port_send_msg_t *msg) +{ + /* + * The count is never below zero by the invariant, so the test on it + * only guards against a bug elsewhere. It is cheap, and a wrap is + * worse than the leak the bound fixes: at UINT32_MAX the port would + * refuse every descriptor for the rest of its life. + */ + + if (msg->link.next != NULL && nxt_port_msg_has_fd(msg) + && nxt_fast_path(port->fd_messages != 0)) + { + port->fd_messages--; + } +} + + +/* + * The test before the lock only spares the mutex for the common message + * with no descriptor. It is repeated under the lock, because + * nxt_port_socket_cancel() can take the same message out of the queue, and + * uncount it, between the two. + */ + +static void +nxt_port_msg_fd_uncount(nxt_port_t *port, nxt_port_send_msg_t *msg) +{ + if (msg->link.next == NULL || !nxt_port_msg_has_fd(msg)) { + return; + } + + nxt_thread_mutex_lock(&port->write_mutex); + + nxt_port_msg_fd_uncount_locked(port, msg); + + nxt_thread_mutex_unlock(&port->write_mutex); +} + + nxt_inline void nxt_port_msg_close_fd(nxt_port_send_msg_t *msg) { @@ -714,21 +1727,56 @@ nxt_port_buf_completion(nxt_task_t *task, nxt_work_queue_t *wq, nxt_buf_t *b, } +/* + * The bound applies here as well: this is the other way a descriptor-carrying + * message enters port->messages, when an inline first fragment hits EAGAIN and + * has to be held for a later attempt. Answering NULL is what an allocation + * failure already answers, and nxt_port_write_msgs() turns both into the same + * "nothing was consumed" NXT_ERROR for a first fragment. + */ + static nxt_port_send_msg_t * nxt_port_msg_insert_tail(nxt_port_t *port, nxt_port_send_msg_t *msg) { + nxt_bool_t has_fd, log; + + has_fd = nxt_port_msg_has_fd(msg); + + nxt_thread_mutex_lock(&port->write_mutex); + + if (nxt_slow_path(has_fd && port->fd_messages >= NXT_PORT_MAX_FD_MSGS)) { + log = !port->fd_refusing; + port->fd_refusing = 1; + + nxt_thread_mutex_unlock(&port->write_mutex); + + if (log) { + nxt_thread_log_alert("port{%d,%d}: %d queued messages already " + "hold a descriptor, refusing to queue " + "another", (int) port->pid, (int) port->id, + NXT_PORT_MAX_FD_MSGS); + } + + return NULL; + } + if (msg->allocated == 0) { msg = nxt_port_msg_alloc(msg); if (nxt_slow_path(msg == NULL)) { + nxt_thread_mutex_unlock(&port->write_mutex); + return NULL; } } - nxt_thread_mutex_lock(&port->write_mutex); - nxt_queue_insert_tail(&port->messages, &msg->link); + if (has_fd) { + port->fd_messages++; + port->fd_refusing = 0; + } + nxt_thread_mutex_unlock(&port->write_mutex); return msg; @@ -763,6 +1811,45 @@ nxt_port_read_close(nxt_port_t *port) } +/* + * Report a message whose control data could not be taken, on the way to + * dropping it. + * + * Truncation is called out separately because it is the case a reader of the + * log cannot otherwise infer: the payload and, on Linux, the kernel + * credential arrived intact, and only the SCM_RIGHTS was cut -- so what a + * handler would see is a well-formed, authenticated message whose descriptor + * is -1. The message is named by type and stream, which is what ties the + * loss to the start, mmap or port hand-off that will not complete. + * + * "n" is the payload length recvmsg() returned; below sizeof(nxt_port_msg_t) + * there is no header to name the message by. + */ + +static void +nxt_port_oob_error_alert(nxt_task_t *task, nxt_port_t *port, + nxt_recv_oob_t *oob, nxt_port_recv_msg_t *msg, ssize_t n) +{ + if (nxt_fast_path(!oob->truncated)) { + nxt_alert(task, "failed to get oob data from %d", port->socket.fd); + + return; + } + + if (n >= (ssize_t) sizeof(nxt_port_msg_t)) { + nxt_alert(task, "port{%d,%d} %d: control data truncated on message " + "type %d stream #%uD; message dropped", + (int) port->pid, (int) port->id, port->socket.fd, + (int) msg->port_msg.type, msg->port_msg.stream); + + } else { + nxt_alert(task, "port{%d,%d} %d: control data truncated on a %z byte " + "message; message dropped", + (int) port->pid, (int) port->id, port->socket.fd, n); + } +} + + static void nxt_port_read_handler(nxt_task_t *task, void *obj, void *data) { @@ -824,9 +1911,13 @@ nxt_port_read_handler(nxt_task_t *task, void *obj, void *data) ret = nxt_socket_msg_oob_get(&oob, msg.fd, nxt_recv_msg_cmsg_pid_ref(&msg)); if (nxt_slow_path(ret != NXT_OK)) { - nxt_alert(task, "failed to get oob data from %d", - port->socket.fd); + nxt_port_oob_error_alert(task, port, &oob, &msg, n); + /* + * Whatever part of the SCM_RIGHTS did arrive is in msg.fd; + * the dispatcher does not reclaim descriptors once a message + * is refused here, so close them before dropping it. + */ nxt_port_close_fds(msg.fd); goto fail; @@ -855,7 +1946,11 @@ nxt_port_read_handler(nxt_task_t *task, void *obj, void *data) if (n == NXT_AGAIN) { nxt_port_buf_free(port, b); - nxt_fd_event_enable_read(task->thread->engine, &port->socket); + /* See the comment in nxt_port_queue_read_handler(). */ + if (!nxt_fd_event_is_active(port->socket.read)) { + nxt_fd_event_enable_read(task->thread->engine, &port->socket); + } + return; } @@ -1020,18 +2115,53 @@ nxt_port_queue_read_handler(nxt_task_t *task, void *obj, void *data) ret = nxt_socket_msg_oob_get(&oob, msg.fd, nxt_recv_msg_cmsg_pid_ref(&msg)); if (nxt_slow_path(ret != NXT_OK)) { - nxt_alert(task, "failed to get oob data from %d", - port->socket.fd); + nxt_port_oob_error_alert(task, port, &oob, &msg, n); + /* See nxt_port_read_handler(). */ nxt_port_close_fds(msg.fd); - return; + /* + * Unwind what this iteration took, then keep draining. + * Returning bare -- which is what this site did -- strands + * the buffer and the READ_SOCKET marker this read was + * consuming, and it strands whatever was queued behind + * them: queue->nitems is non-zero for the whole + * invocation, which is precisely how nxt_port_queue_send() + * decides a reader is already awake and skips the socket + * notification. A message enqueued in that window is + * therefore never announced, and returning here means + * nobody ever comes back for it -- one refused datagram + * costing the port every message queued behind it. + * + * nitems is deliberately not decremented here. The + * handler holds exactly one nitems credit for the whole + * invocation, and whichever exit it finally takes releases + * it -- the empty-queue check above, or the loop's own + * termination. Decrementing here as well as there would + * give back one credit twice. + */ + if (port->from_socket > 0) { + port->from_socket--; + } + + nxt_port_buf_free(port, b); + + continue; } } if (n == (ssize_t) sizeof(nxt_port_msg_t) && msg.port_msg.type == _NXT_PORT_MSG_READ_QUEUE) { + /* + * A wake-up marker carries no payload and needs no + * descriptor, but the sender chooses the type byte and + * SCM_RIGHTS is already attached by the time it is read. + * This exit never reaches nxt_port_read_msg_process(), so + * the close at its tail cannot cover it. + */ + nxt_port_close_fds(msg.fd); + nxt_port_buf_free(port, b); nxt_debug(task, "port{%d,%d} %d: recv %d read_queue", @@ -1063,6 +2193,8 @@ nxt_port_queue_read_handler(nxt_task_t *task, void *obj, void *data) (int) port->pid, (int) port->id, port->socket.fd); + nxt_port_close_fds(msg.fd); + return; } @@ -1075,6 +2207,8 @@ nxt_port_queue_read_handler(nxt_task_t *task, void *obj, void *data) (int) port->pid, (int) port->id, port->socket.fd); + nxt_port_close_fds(msg.fd); + return; } } @@ -1118,7 +2252,16 @@ nxt_port_queue_read_handler(nxt_task_t *task, void *obj, void *data) if (n == NXT_AGAIN) { nxt_port_buf_free(port, b); - nxt_fd_event_enable_read(task->thread->engine, &port->socket); + /* + * An active read event stays registered, so EAGAIN needs no + * re-arm: the next datagram raises a new event. On epoll the + * enable is an EPOLL_CTL_MOD even when the event is active, and + * this point is reached once per drain of the port. Enable only + * an event that is not active. + */ + if (!nxt_fd_event_is_active(port->socket.read)) { + nxt_fd_event_enable_read(task->thread->engine, &port->socket); + } continue; } @@ -1180,10 +2323,46 @@ static const nxt_lvlhsh_proto_t lvlhsh_frag_proto nxt_aligned(64) = { }; +/* + * An upper bound on what nxt_port_mmap_read() allocates for each record: + * nxt_buf_mem_ts_alloc() takes the memory part of an nxt_buf_t followed by + * its thread-safe completion state, a work item and an engine pointer. + */ +#define NXT_PORT_FRAG_MMAP_BUF_COST \ + (sizeof(nxt_buf_t) + sizeof(nxt_work_t) + sizeof(void *)) + + +/* + * What a fragment kept for reassembly costs the receiver: the whole buffer + * it came in, port->max_size, however little of it carries. Counting the + * payload alone let a stream of empty fragments hold buffers without limit. + * An mmap fragment also keeps a buffer per record, and a record may name + * no bytes at all: a max_size fragment of empty records is some 1,300 + * buffers, so each of them is charged too. + */ +static size_t +nxt_port_frag_cost(nxt_port_t *port, nxt_port_recv_msg_t *msg) +{ + size_t cost; + nxt_buf_t *b; + + cost = nxt_max(msg->size, port->max_size); + + if (msg->port_msg.mmap) { + for (b = msg->buf; b != NULL; b = b->next) { + cost += NXT_PORT_FRAG_MMAP_BUF_COST; + } + } + + return cost; +} + + static nxt_port_recv_msg_t * nxt_port_frag_start(nxt_task_t *task, nxt_port_t *port, nxt_port_recv_msg_t *msg) { + size_t cost; nxt_int_t res; nxt_lvlhsh_query_t lhq; nxt_port_recv_msg_t *fmsg; @@ -1191,6 +2370,21 @@ nxt_port_frag_start(nxt_task_t *task, nxt_port_t *port, nxt_debug(task, "start frag stream #%uD", msg->port_msg.stream); + if (nxt_slow_path(port->frag_streams >= NXT_PORT_FRAG_STREAMS_MAX)) { + nxt_port_frag_evict(task, port, msg); + } + + cost = nxt_port_frag_cost(port, msg); + + if (nxt_slow_path(msg->size > NXT_PORT_FRAG_SIZE_MAX + || cost > NXT_PORT_FRAG_TOTAL_MAX - port->frag_size)) + { + nxt_alert(task, "port %d: fragmented message #%uD from pid %PI " + "exceeds the reassembly limit, dropped", port->socket.fd, + msg->port_msg.stream, msg->port_msg.pid); + return NULL; + } + fmsg = nxt_mp_alloc(port->mem_pool, sizeof(nxt_port_recv_msg_t)); if (nxt_slow_path(fmsg == NULL)) { @@ -1215,6 +2409,13 @@ nxt_port_frag_start(nxt_task_t *task, nxt_port_t *port, switch (res) { case NXT_OK: + fmsg->frag_held = cost; + + port->frag_streams++; + port->frag_size += cost; + + nxt_queue_insert_tail(&port->frag_queue, &fmsg->frag_link); + return fmsg; case NXT_DECLINED: @@ -1264,6 +2465,10 @@ nxt_port_frag_find(nxt_task_t *task, nxt_port_t *port, nxt_port_recv_msg_t *msg) switch (res) { case NXT_OK: + if (last) { + nxt_port_frag_unaccount(port, lhq.value); + } + return lhq.value; default: @@ -1275,10 +2480,180 @@ nxt_port_frag_find(nxt_task_t *task, nxt_port_t *port, nxt_port_recv_msg_t *msg) } +static void +nxt_port_frag_unaccount(nxt_port_t *port, nxt_port_recv_msg_t *fmsg) +{ + port->frag_streams--; + port->frag_size -= fmsg->frag_held; + + nxt_queue_remove(&fmsg->frag_link); +} + + +/* + * Makes room for the new stream in "msg" when NXT_PORT_FRAG_STREAMS_MAX are + * open: the oldest stream is dropped as a whole. A peer that opens streams + * and never ends them thus cannot block reassembly for good. To keep out a + * message of another sender, it must open that many new streams between + * two fragments of that message, for every message. + */ +static void +nxt_port_frag_evict(nxt_task_t *task, nxt_port_t *port, + nxt_port_recv_msg_t *msg) +{ + nxt_port_recv_msg_t *old; + + old = nxt_queue_link_data(nxt_queue_first(&port->frag_queue), + nxt_port_recv_msg_t, frag_link); + + nxt_alert(task, "port %d: %uD fragmented messages already in progress, " + "dropping the oldest, stream #%uD from pid %PI, for stream " + "#%uD from pid %PI", port->socket.fd, port->frag_streams, + old->port_msg.stream, old->port_msg.pid, msg->port_msg.stream, + msg->port_msg.pid); + + nxt_port_frag_drop(task, port, old, 1); +} + + +/* + * Would appending "msg" to the stream being reassembled in "fmsg" pass a + * limit? The per-port total is checked only for a fragment that keeps the + * stream open, against its "cost": the last one hands the whole message + * over at once. A stream and the port's total are kept under their + * limits, so neither subtraction can wrap. + */ +static nxt_bool_t +nxt_port_frag_fits(nxt_task_t *task, nxt_port_t *port, + nxt_port_recv_msg_t *fmsg, nxt_port_recv_msg_t *msg, size_t cost) +{ + if (nxt_slow_path(msg->size > NXT_PORT_FRAG_SIZE_MAX - fmsg->size)) { + nxt_alert(task, "port %d: fragmented message #%uD from pid %PI " + "exceeds %d bytes, dropped", port->socket.fd, + msg->port_msg.stream, msg->port_msg.pid, + NXT_PORT_FRAG_SIZE_MAX); + return 0; + } + + if (msg->port_msg.mf != 0 + && nxt_slow_path(cost > NXT_PORT_FRAG_TOTAL_MAX - port->frag_size)) + { + nxt_alert(task, "port %d: fragmented messages in progress exceed " + "%d bytes, dropping stream #%uD from pid %PI", + port->socket.fd, NXT_PORT_FRAG_TOTAL_MAX, + msg->port_msg.stream, msg->port_msg.pid); + return 0; + } + + return 1; +} + + +/* + * Gives back the buffers of a reassembled message, each the way it came: + * a shared memory buffer is completed, a read buffer returns to the port. + * The fragments of one stream need not all have come the same way -- the + * sender picks plain or mmap per fragment -- so the last fragment's mmap + * bit does not say how to release what the earlier ones brought. + */ +static void +nxt_port_frag_bufs_release(nxt_task_t *task, nxt_port_t *port, nxt_buf_t *b) +{ + nxt_buf_t *next; + + for ( /* void */ ; b != NULL; b = next) { + next = b->next; + b->next = NULL; + + if (nxt_buf_is_port_mmap(b)) { + nxt_work_queue_add(port->socket.read_work_queue, + b->completion_handler, task, b, b->parent); + + } else { + nxt_port_buf_free(port, b); + } + } +} + + +/* + * Drops a stream that passed a limit, or the oldest one when a new stream + * needs its place (nxt_port_frag_evict()): out of ->frags if it is still there, + * then everything it had accumulated -- buffers, the first fragment's + * descriptors, the stream itself. Its later fragments find no stream and + * are discarded by the "not found" path. + */ +static void +nxt_port_frag_drop(nxt_task_t *task, nxt_port_t *port, + nxt_port_recv_msg_t *fmsg, nxt_bool_t in_hash) +{ + nxt_lvlhsh_query_t lhq; + nxt_port_frag_key_t frag_key; + + if (in_hash) { + frag_key.stream = fmsg->port_msg.stream; + frag_key.pid = fmsg->port_msg.pid; + + lhq.key_hash = nxt_murmur_hash2(&frag_key, + sizeof(nxt_port_frag_key_t)); + lhq.key.length = sizeof(nxt_port_frag_key_t); + lhq.key.start = (u_char *) &frag_key; + lhq.proto = &lvlhsh_frag_proto; + lhq.pool = port->mem_pool; + + if (nxt_lvlhsh_delete(&port->frags, &lhq) == NXT_OK) { + nxt_port_frag_unaccount(port, fmsg); + } + } + + /* + * A stream started while cancelled holds copies only: the caller freed + * its buffers and closed its descriptors. + */ + if (fmsg->cancelled == 0) { + nxt_port_frag_bufs_release(task, port, fmsg->buf); + + nxt_port_close_fds(fmsg->fd); + } + + nxt_mp_free(port->mem_pool, fmsg); +} + + +/* + * Did nxt_port_mmap_read() turn an mmap fragment into shared memory + * buffers? A fragment that is empty, carries only part of a record, or + * whose first record names no segment leaves msg->buf at the read buffer + * "orig_b", or at NULL. Kept in a stream, the read buffer would also be + * given back to port->free_bufs by the caller and reused for the next + * message while the stream still held it; the sender never sends such a + * fragment, so the fragment is refused. msg->buf is set back to "orig_b" + * for the caller to free. + */ +static nxt_bool_t +nxt_port_frag_mmap_valid(nxt_task_t *task, nxt_port_t *port, + nxt_port_recv_msg_t *msg, nxt_buf_t *orig_b) +{ + if (nxt_fast_path(msg->buf != orig_b && msg->buf != NULL)) { + return 1; + } + + nxt_alert(task, "port %d: mmap fragment of message #%uD from pid %PI " + "carries no shared memory buffer, dropped", port->socket.fd, + msg->port_msg.stream, msg->port_msg.pid); + + msg->buf = orig_b; + msg->size = 0; + + return 0; +} + + static void nxt_port_read_msg_process(nxt_task_t *task, nxt_port_t *port, nxt_port_recv_msg_t *msg) { + size_t cost; nxt_buf_t *b, *orig_b, *next; nxt_port_recv_msg_t *fmsg; @@ -1311,11 +2686,44 @@ nxt_port_read_msg_process(nxt_task_t *task, nxt_port_t *port, if (msg->port_msg.mmap) { nxt_port_mmap_read(task, msg); + + if (nxt_slow_path(!nxt_port_frag_mmap_valid(task, port, msg, + orig_b))) + { + /* The last fragment's lookup took it out already. */ + nxt_port_frag_drop(task, port, fmsg, + msg->port_msg.mf != 0); + + b = orig_b; + + goto fmsg_failed; + } + } + + /* The last fragment is not kept, so it is not charged. */ + cost = (msg->port_msg.mf != 0) ? nxt_port_frag_cost(port, msg) : 0; + + if (nxt_slow_path(!nxt_port_frag_fits(task, port, fmsg, msg, + cost))) + { + /* The last fragment's lookup took the stream out already. */ + nxt_port_frag_drop(task, port, fmsg, msg->port_msg.mf != 0); + + /* This fragment's own buffers complete on the common path. */ + b = msg->buf; + + goto fmsg_failed; } nxt_buf_chain_add(&fmsg->buf, msg->buf); fmsg->size += msg->size; + + if (msg->port_msg.mf != 0) { + fmsg->frag_held += cost; + port->frag_size += cost; + } + msg->buf = NULL; b = NULL; @@ -1325,16 +2733,28 @@ nxt_port_read_msg_process(nxt_task_t *task, nxt_port_t *port, port->handler(task, fmsg); + /* + * The assembled message's descriptors are about to overwrite + * this fragment's own, which would drop them unnoticed. + */ + nxt_port_close_fds(msg->fd); + msg->buf = fmsg->buf; msg->fd[0] = fmsg->fd[0]; msg->fd[1] = fmsg->fd[1]; /* * To disable instant completion or buffer re-usage, - * handler should reset 'msg.buf'. + * handler should reset 'msg.buf'. Whatever it leaves is + * released here, buffer by buffer: this fragment's own read + * buffer is in the chain only when it came plain, and an + * mmap fragment's is freed by the caller once msg->buf is + * restored below. */ - if (!msg->port_msg.mmap && msg->buf == b) { - nxt_port_buf_free(port, b); + if (msg->buf == b) { + nxt_port_frag_bufs_release(task, port, b); + + msg->buf = NULL; } } } @@ -1347,6 +2767,15 @@ nxt_port_read_msg_process(nxt_task_t *task, nxt_port_t *port, if (msg->port_msg.mmap && msg->cancelled == 0) { nxt_port_mmap_read(task, msg); + + if (nxt_slow_path(!nxt_port_frag_mmap_valid(task, port, msg, + orig_b))) + { + b = orig_b; + + goto fmsg_failed; + } + b = msg->buf; } @@ -1406,6 +2835,26 @@ nxt_port_read_msg_process(nxt_task_t *task, nxt_port_t *port, /* restore original buf */ msg->buf = orig_b; } + + /* + * Close whatever nothing took ownership of. A handler that keeps a + * descriptor sets its slot to -1, so this closes only what is genuinely + * unclaimed, and it is idempotent for the paths that already closed. + * + * This sits at the tail rather than beside the handler call because the + * paths that drop descriptors are not all handler paths: "goto + * fmsg_failed" above skips the dispatch entirely when a fragment cannot + * be found or started, and a middle fragment is chained into the + * assembled message without its own fd[] being carried across. Neither + * runs a handler, so no per-handler close can reach them. + * + * The message header is attacker-chosen -- type, stream and the nf/mf + * fragment bits are bytes off the wire, and an application holds the + * write end of the router's main port -- so every one of those paths is + * reachable from a compromised application, one leaked descriptor per + * message until the table is exhausted. + */ + nxt_port_close_fds(msg->fd); } @@ -1472,6 +2921,8 @@ nxt_port_error_handler(nxt_task_t *task, void *obj, void *data) nxt_queue_each(msg, &port->messages, nxt_port_send_msg_t, link) { + nxt_port_msg_fd_uncount_locked(port, msg); + nxt_port_msg_close_fd(msg); for (b = msg->buf; b != NULL; b = next) { diff --git a/src/nxt_process.c b/src/nxt_process.c index 62c00e6e4..4dba7fd64 100644 --- a/src/nxt_process.c +++ b/src/nxt_process.c @@ -31,15 +31,18 @@ #if (NXT_HAVE_LINUX_NS) static nxt_int_t nxt_process_pipe_timer(nxt_fd_t fd, short event); -static nxt_int_t nxt_process_check_pid_status(const nxt_fd_t *gc_pipe); -static nxt_pid_t nxt_process_recv_pid(const nxt_fd_t *pid_pipe, - const nxt_fd_t *gc_pipe); +static void nxt_process_fd_close(nxt_fd_t *fd); +static void nxt_process_pipe_close(nxt_fd_t *pp); +static nxt_int_t nxt_process_recv_status(const nxt_fd_t *gc_pipe); +static void nxt_process_send_status(const nxt_fd_t *gc_pipe, int8_t status); +static nxt_pid_t nxt_process_recv_pid(const nxt_fd_t *pid_pipe); static void nxt_process_send_pid(const nxt_fd_t *pid_pipe, nxt_pid_t pid); static nxt_int_t nxt_process_unshare(nxt_task_t *task, nxt_process_t *process, - nxt_fd_t *pid_pipe, nxt_fd_t *gc_pipe, nxt_bool_t use_pidns); -static nxt_int_t nxt_process_init_pidns(nxt_task_t *task, + nxt_fd_t *pid_pipe, nxt_fd_t *gc_pipe, nxt_bool_t use_pidns, + nxt_bool_t use_cgroup); +static nxt_int_t nxt_process_init_pipes(nxt_task_t *task, const nxt_process_t *process, nxt_fd_t *pid_pipe, nxt_fd_t *gc_pipe, - nxt_bool_t *use_pidns); + nxt_bool_t *use_pidns, nxt_bool_t *use_cgroup); #endif static nxt_pid_t nxt_process_create(nxt_task_t *task, nxt_process_t *process); @@ -347,7 +350,25 @@ nxt_process_child_fixup(nxt_task_t *task, nxt_process_t *process) rt = task->thread->runtime; - /* Remove not ready processes. */ + /* + * Remove not ready processes. + * + * These walks hand unreferenced processes to nxt_process_close_ports(), + * which takes its own +1/-1 -- the shape that made nxt_port_remove_pid() + * a double release once the refcount became cross-thread. It is safe + * here for a different reason than everywhere else, and not because of + * the refcount: this runs immediately after fork() in the child, which + * is single-threaded by definition, so no other thread exists to have + * dropped the last reference. nxt_runtime_process_each() iterating + * rt->processes without the mutex is safe for the same reason. + * + * The other half of that, now that nxt_process_use() locks + * rt->processes_mutex: the mutex is inherited across fork() and would be + * held forever in the child if any other thread of the parent held it at + * the call. None can -- only main and prototype fork, both take the + * mutex on their single event engine, and their thread-pool threads + * never touch it. + */ nxt_runtime_process_each(rt, p) { if (nxt_proc_keep_matrix[ptype][nxt_process_type(p)] == 0 @@ -427,51 +448,74 @@ nxt_process_pipe_timer(nxt_fd_t fd, short event) } +static void +nxt_process_fd_close(nxt_fd_t *fd) +{ + if (*fd != -1) { + close(*fd); + *fd = -1; + } +} + + +static void +nxt_process_pipe_close(nxt_fd_t *pp) +{ + /* A pipe that nxt_process_init_pipes() did not create is -1. */ + + nxt_process_fd_close(&pp[0]); + nxt_process_fd_close(&pp[1]); +} + + static nxt_int_t -nxt_process_check_pid_status(const nxt_fd_t *gc_pipe) +nxt_process_recv_status(const nxt_fd_t *gc_pipe) { int8_t status = -1; ssize_t ret; - close(gc_pipe[1]); - ret = nxt_process_pipe_timer(gc_pipe[0], POLLIN); if (ret == NXT_OK) { read(gc_pipe[0], &status, sizeof(int8_t)); } - close(gc_pipe[0]); - return status; } +static void +nxt_process_send_status(const nxt_fd_t *gc_pipe, int8_t status) +{ + /* + * The parent keeps its read end open until nxt_process_pipe_close(). + * So this write does not get EPIPE when the child has exited. + */ + if (gc_pipe[1] != -1) { + write(gc_pipe[1], &status, sizeof(int8_t)); + } +} + + static nxt_pid_t -nxt_process_recv_pid(const nxt_fd_t *pid_pipe, const nxt_fd_t *gc_pipe) +nxt_process_recv_pid(const nxt_fd_t *pid_pipe) { - int8_t status; ssize_t ret; nxt_pid_t pid; close(pid_pipe[1]); - close(gc_pipe[0]); - status = 0; + pid = -1; ret = nxt_process_pipe_timer(pid_pipe[0], POLLIN); if (ret == NXT_OK) { ret = read(pid_pipe[0], &pid, sizeof(nxt_pid_t)); - } - if (ret <= 0) { - status = -1; - pid = -1; + if (ret <= 0) { + pid = -1; + } } - write(gc_pipe[1], &status, sizeof(int8_t)); - close(pid_pipe[0]); - close(gc_pipe[1]); return pid; } @@ -496,86 +540,132 @@ nxt_process_send_pid(const nxt_fd_t *pid_pipe, nxt_pid_t pid) static nxt_int_t nxt_process_unshare(nxt_task_t *task, nxt_process_t *process, nxt_fd_t *pid_pipe, nxt_fd_t *gc_pipe, - nxt_bool_t use_pidns) + nxt_bool_t use_pidns, nxt_bool_t use_cgroup) { int ret; nxt_pid_t pid; - if (process->isolation.clone.flags == 0) { - return NXT_OK; - } - - ret = unshare(process->isolation.clone.flags); - if (nxt_slow_path(ret == -1)) { - nxt_alert(task, "unshare() failed for %s %E", process->name, - nxt_errno); + /* + * The child only reads gc_pipe. Without its own write end, a read + * gets end of file when the parent exits. + */ + nxt_process_fd_close(&gc_pipe[1]); - if (use_pidns) { - nxt_pipe_close(task, gc_pipe); - nxt_pipe_close(task, pid_pipe); + if (use_cgroup) { + /* + * Wait until the parent has moved this process into its cgroup. + * A new cgroup namespace is rooted at the cgroup of the process + * at unshare() time. A process from the fork(2) below starts in + * the cgroup of this process. + */ + if (nxt_process_recv_status(gc_pipe) != 0) { + goto fail; } - - return NXT_ERROR; - } - - if (!use_pidns) { - return NXT_OK; } /* - * PID namespace requested. Employ a double fork(2) technique - * so that the prototype process will be placed into the new - * namespace and end up with PID 1 (as before with clone). + * Unshare all flags in one call. With CLONE_NEWUSER, the kernel checks + * the other flags with the credentials from before the call. A second + * unshare() after CLONE_NEWUSER uses the new credentials and can fail + * with EPERM, for example in the unprivileged_userns AppArmor profile + * of Ubuntu 24.04. */ - pid = fork(); - if (nxt_slow_path(pid < 0)) { - nxt_alert(task, "fork() failed for %s %E", process->name, nxt_errno); - nxt_pipe_close(task, gc_pipe); - nxt_pipe_close(task, pid_pipe); + if (process->isolation.clone.flags != 0) { + ret = unshare(process->isolation.clone.flags); + if (nxt_slow_path(ret == -1)) { + nxt_alert(task, "unshare() failed for %s %E", process->name, + nxt_errno); + goto fail; + } + } - return NXT_ERROR; + if (use_pidns) { + /* + * PID namespace requested. Employ a double fork(2) technique + * so that the prototype process will be placed into the new + * namespace and end up with PID 1 (as before with clone). + */ + pid = fork(); + if (nxt_slow_path(pid < 0)) { + nxt_alert(task, "fork() failed for %s %E", process->name, + nxt_errno); + goto fail; - } else if (pid > 0) { - nxt_pipe_close(task, gc_pipe); - nxt_process_send_pid(pid_pipe, pid); + } else if (pid > 0) { + nxt_process_pipe_close(gc_pipe); + nxt_process_send_pid(pid_pipe, pid); - _exit(EXIT_SUCCESS); - } + _exit(EXIT_SUCCESS); + } - nxt_pipe_close(task, pid_pipe); - ret = nxt_process_check_pid_status(gc_pipe); - if (ret == -1) { - return NXT_ERROR; + nxt_process_pipe_close(pid_pipe); + + /* Wait until the parent has the pid of this process. */ + + if (nxt_process_recv_status(gc_pipe) != 0) { + goto fail; + } } + nxt_process_pipe_close(gc_pipe); + return NXT_OK; + +fail: + + nxt_process_pipe_close(gc_pipe); + nxt_process_pipe_close(pid_pipe); + + return NXT_ERROR; } static nxt_int_t -nxt_process_init_pidns(nxt_task_t *task, const nxt_process_t *process, +nxt_process_init_pipes(nxt_task_t *task, const nxt_process_t *process, nxt_fd_t *pid_pipe, nxt_fd_t *gc_pipe, - nxt_bool_t *use_pidns) + nxt_bool_t *use_pidns, nxt_bool_t *use_cgroup) { - int ret; + int ret; + + pid_pipe[0] = pid_pipe[1] = -1; + gc_pipe[0] = gc_pipe[1] = -1; *use_pidns = 0; + *use_cgroup = 0; #if (NXT_HAVE_CLONE_NEWPID) *use_pidns = nxt_is_pid_isolated(process); #endif - if (!*use_pidns) { +#if (NXT_HAVE_CGROUP) + *use_cgroup = (process->isolation.cgroup.path != NULL); +#endif + + if (!*use_pidns && !*use_cgroup) { return NXT_OK; } - ret = nxt_pipe_create(task, pid_pipe, 0, 0); + /* + * gc_pipe carries status bytes from the parent to the new process. + * With a cgroup path, the parent sends 0 after it has moved the child + * into the cgroup. The child waits for it before unshare() and before + * the pid isolation fork(2). With pid isolation, the parent sends 0 + * after it has read the pid of the grandchild from pid_pipe. The + * grandchild waits for it. -1, end of file or a timeout make the + * process exit. + */ + ret = nxt_pipe_create(task, gc_pipe, 0, 0); if (nxt_slow_path(ret == NXT_ERROR)) { return NXT_ERROR; } - ret = nxt_pipe_create(task, gc_pipe, 0, 0); + if (!*use_pidns) { + return NXT_OK; + } + + ret = nxt_pipe_create(task, pid_pipe, 0, 0); if (nxt_slow_path(ret == NXT_ERROR)) { + nxt_process_pipe_close(gc_pipe); return NXT_ERROR; } @@ -602,9 +692,10 @@ nxt_process_create(nxt_task_t *task, nxt_process_t *process) #if (NXT_HAVE_LINUX_NS) nxt_fd_t pid_pipe[2], gc_pipe[2]; - nxt_bool_t use_pidns; + nxt_bool_t use_pidns, use_cgroup; - ret = nxt_process_init_pidns(task, process, pid_pipe, gc_pipe, &use_pidns); + ret = nxt_process_init_pipes(task, process, pid_pipe, gc_pipe, + &use_pidns, &use_cgroup); if (ret == NXT_ERROR) { return -1; } @@ -613,6 +704,12 @@ nxt_process_create(nxt_task_t *task, nxt_process_t *process) pid = fork(); if (nxt_slow_path(pid < 0)) { nxt_alert(task, "fork() failed for %s %E", process->name, nxt_errno); + +#if (NXT_HAVE_LINUX_NS) + nxt_process_pipe_close(gc_pipe); + nxt_process_pipe_close(pid_pipe); +#endif + return pid; } @@ -620,7 +717,8 @@ nxt_process_create(nxt_task_t *task, nxt_process_t *process) /* Child. */ #if (NXT_HAVE_LINUX_NS) - ret = nxt_process_unshare(task, process, pid_pipe, gc_pipe, use_pidns); + ret = nxt_process_unshare(task, process, pid_pipe, gc_pipe, + use_pidns, use_cgroup); if (ret == NXT_ERROR) { _exit(EXIT_FAILURE); } @@ -648,12 +746,56 @@ nxt_process_create(nxt_task_t *task, nxt_process_t *process) nxt_debug(task, "fork(%s): %PI", process->name, pid); +#if (NXT_HAVE_CGROUP) + /* + * Move the child into its cgroup by the pid from fork(). The child + * waits for the status byte below before unshare() and before the pid + * isolation fork(2). So a new cgroup namespace is rooted at this + * cgroup, and the grandchild starts in it. + */ + ret = nxt_cgroup_proc_add(task, process, pid); + if (nxt_slow_path(ret != NXT_OK)) { + nxt_alert(task, "cgroup: failed to add process %s to %s %E", + process->name, process->isolation.cgroup.path, nxt_errno); + nxt_cgroup_cleanup(task, process); + #if (NXT_HAVE_LINUX_NS) + /* The child waits for the status byte. -1 makes it exit. */ + + nxt_process_send_status(gc_pipe, -1); + nxt_process_pipe_close(gc_pipe); + nxt_process_pipe_close(pid_pipe); +#else + kill(pid, SIGTERM); +#endif + + return -1; + } +#endif + +#if (NXT_HAVE_LINUX_NS) + if (use_cgroup) { + nxt_process_send_status(gc_pipe, 0); + } + if (use_pidns) { - pid = nxt_process_recv_pid(pid_pipe, gc_pipe); - if (pid == -1) { - return pid; + pid = nxt_process_recv_pid(pid_pipe); + nxt_process_send_status(gc_pipe, (pid == -1) ? -1 : 0); + } + + nxt_process_pipe_close(gc_pipe); + + if (pid == -1) { +#if (NXT_HAVE_CGROUP) + if (use_cgroup) { + /* + * The child can still be in the cgroup. Then rmdir() fails + * and the directory stays. + */ + nxt_cgroup_cleanup(task, process); } +#endif + return pid; } #endif @@ -675,17 +817,6 @@ nxt_process_create(nxt_task_t *task, nxt_process_t *process) nxt_runtime_process_add(task, process); } -#if (NXT_HAVE_CGROUP) - ret = nxt_cgroup_proc_add(task, process); - if (nxt_slow_path(ret != NXT_OK)) { - nxt_alert(task, "cgroup: failed to add process %s to %s %E", - process->name, process->isolation.cgroup.path, nxt_errno); - nxt_cgroup_cleanup(task, process); - kill(pid, SIGTERM); - return -1; - } -#endif - return pid; } @@ -973,6 +1104,40 @@ nxt_process_created_ok(nxt_task_t *task, nxt_port_recv_msg_t *msg, void *data) init = nxt_process_init(process); ret = nxt_process_apply_creds(task, process); + + if (nxt_slow_path(ret == NXT_DECLINED)) { + /* + * capset() is filtered and nxt_capability_still_held() found + * this process carrying capabilities because of it, or could + * not establish that it is not. Only a prototype reaches + * nxt_process_created_ok(): the core processes set + * NXT_PROCESS_STATE_READY in nxt_process_core_setup() and never + * send PROCESS_CREATED, and an application worker never sends + * one either, because nxt_app_setup() returns init->start() + * directly and that call does not come back with NXT_OK. The + * entire remaining job of a prototype is to fork workers that + * would inherit those sets. Refuse the start + * instead, and say which syscall and which consequence, so the + * operator's next step is to allow capset() rather than to + * hunt for a broken application. + * + * The refusal is visible: this exits nonzero, main's SIGCHLD + * reaper notifies the router with the start's stream still + * attached (main clears ->stream only once the NEW_PORT that + * answers the start has gone out, and a prototype that died + * here never sent the PROCESS_READY that would have caused + * one), the router turns that REMOVE_PID into an RPC error for + * the start attempt, and the requests waiting on the application + * are answered 503 rather than left to time out. + */ + + nxt_alert(task, "%s refused to start: capset() is denied, so the " + "capabilities this process holds cannot be dropped and " + "would be inherited by application code", process->name); + + goto fail; + } + if (nxt_slow_path(ret != NXT_OK)) { goto fail; } @@ -1021,7 +1186,24 @@ nxt_process_core_setup(nxt_task_t *task, nxt_process_t *process) nxt_int_t ret; ret = nxt_process_apply_creds(task, process); - if (nxt_slow_path(ret != NXT_OK)) { + + /* + * NXT_DECLINED -- capset() filtered, capabilities still held -- is + * warn-and-continue here, unlike on the nxt_process_created_ok() + * path. This is the router, the controller and discovery: they run + * no application code, so there is nothing to inherit what they + * keep, and refusing costs far more than it buys. Router and + * controller declare .restart = 1 and nxt_main_process_sigchld_ + * handler() re-forks them immediately and without backoff, so a + * filter that denies every attempt would spin a respawn loop + * instead of reporting anything; discovery's row in + * nxt_proc_remove_notify_matrix is all zeroes and it is the main + * process's only startup action, so its death would leave unitd up + * with no modules, no control socket and no error. + * nxt_capability_drop() has already logged the warning. + */ + + if (nxt_slow_path(ret != NXT_OK && ret != NXT_DECLINED)) { return NXT_ERROR; } @@ -1110,7 +1292,23 @@ nxt_process_apply_creds(nxt_task_t *task, nxt_process_t *process) } #endif - return NXT_OK; + /* + * Last: nothing after this point in any child needs a capability, + * and everything before it might. PR_SET_NO_NEW_PRIVS above stops + * this process *gaining* privilege across execve(); it does not + * take away what the process already carries, and fork() copies + * every capability set verbatim, so a unitd that was granted + * capabilities would otherwise pass them straight to application + * code. Main never calls apply_creds() and so never drops -- it + * binds listeners on every reconfiguration. + * + * Its NXT_DECLINED -- a filtered capset(), sets still full -- is + * passed through rather than folded into either NXT_OK or + * NXT_ERROR, because the two callers answer it differently: see + * nxt_process_created_ok() and nxt_process_core_setup(). + */ + + return nxt_capability_drop(task); } @@ -1271,10 +1469,31 @@ nxt_nanosleep(nxt_nsec_t ns) } +/* + * The only way a port is linked into a process's port list. ->process, + * ->link and the reference the port holds on the process are set together + * here and dropped together by nxt_port_release(), so they cannot diverge + * (#425). A port that is already paired is refused rather than linked a + * second time, in every build: a second insert corrupts the list, and an + * nxt_assert() would compile out of a release build. + */ + void nxt_process_port_add(nxt_task_t *task, nxt_process_t *process, nxt_port_t *port) { - nxt_assert(port->process == NULL); + /* + * A double add is a bug: trap it in a debug build, refuse it in any. + * link.next reads as "already linked" only because a port is never + * linked again after nxt_port_release() unlinks it: in a release build + * nxt_queue_remove() does not clear the link. + */ + nxt_assert(port->process == NULL && port->link.next == NULL); + + if (nxt_slow_path(port->process != NULL || port->link.next != NULL)) { + nxt_alert(task, "port %p %d:%d is already paired with a process", + port, port->pid, port->id); + return; + } port->process = process; nxt_queue_insert_tail(&process->ports, &port->link); @@ -1291,6 +1510,13 @@ nxt_process_type(nxt_process_t *process) } +/* + * The caller must hold a reference to the process. Closing the ports drops + * the reference each of them holds, so this takes one of its own to survive + * the loop -- but on a process that has already reached zero that same pair + * is a fresh 0 -> 1 -> 0 transition, which runs the teardown a second time. + */ + void nxt_process_close_ports(nxt_task_t *task, nxt_process_t *process) { diff --git a/src/nxt_process.h b/src/nxt_process.h index 1c589d7b4..0ed9f371b 100644 --- a/src/nxt_process.h +++ b/src/nxt_process.h @@ -123,14 +123,47 @@ struct nxt_process_s { */ nxt_bool_t released; + /* + * Latched when a PROCESS_READY could not be acted upon and the process + * was killed for it (nxt_port_process_ready_handler()). A retransmitted + * PROCESS_READY must not re-enter that arm and signal a pid that has + * already been killed -- and, once reaped, may name an unrelated + * process. + * + * The state cannot carry this, because the process is left at CREATED. + * Set only where the sender of a PROCESS_READY can be authenticated + * (NXT_USE_CMSG_PID); elsewhere that arm is not reachable, so nothing + * ever latches it. + */ + nxt_bool_t start_failed; + nxt_port_mmaps_t incoming; nxt_pid_t isolated_pid; + + /* + * The pid of this process inside the pid namespace of the prototype + * that forked it. Only main sets it, from the WHOAMI header; 0 means + * "no name". ->pid stays the global key. See REMOVE_CHILD_PID in + * src/nxt_main_process.c. + */ + nxt_pid_t parent_ns_pid; + const char *name; nxt_port_t *parent_port; + /* + * The start request this process was forked to satisfy: ->stream is the + * initiator's RPC stream, and ->stream_pid/->stream_port address the port + * that RPC is registered on. A creator that can be left holding the only + * armed handler for a child records all three -- the prototype does, in + * nxt_proto_start_process_handler() -- so that a child which dies before + * anything else can report it still gets answered. + */ uint32_t stream; + nxt_pid_t stream_pid; + nxt_port_id_t stream_port; nxt_mp_t *mem_pool; nxt_credential_t *user_cred; @@ -201,6 +234,20 @@ NXT_EXPORT void nxt_process_arguments(nxt_task_t *task, char **orig_argv, #define nxt_process_port_first(process) \ nxt_queue_link_data(nxt_queue_first(&process->ports), nxt_port_t, link) +/* + * Take a process off its parent's "children" queue. link.next == NULL means + * "not linked", which the release build of nxt_queue_remove() does not set, + * so it is set here. Safe to call twice. + */ +nxt_inline void +nxt_process_unlink(nxt_process_t *process) +{ + if (process->link.next != NULL) { + nxt_queue_remove(&process->link); + process->link.next = NULL; + } +} + NXT_EXPORT void nxt_process_port_add(nxt_task_t *task, nxt_process_t *process, nxt_port_t *port); diff --git a/src/nxt_queue.c b/src/nxt_queue.c deleted file mode 100644 index c81356dbd..000000000 --- a/src/nxt_queue.c +++ /dev/null @@ -1,85 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) NGINX, Inc. - */ - - -#include - - -/* - * Find the middle queue element if the queue has odd number of elements, - * or the first element of the queue's second part otherwise. - */ - -nxt_queue_link_t * -nxt_queue_middle(nxt_queue_t *queue) -{ - nxt_queue_link_t *middle, *next; - - middle = nxt_queue_first(queue); - - if (middle == nxt_queue_last(queue)) { - return middle; - } - - next = middle; - - for ( ;; ) { - middle = nxt_queue_next(middle); - - next = nxt_queue_next(next); - - if (next == nxt_queue_last(queue)) { - return middle; - } - - next = nxt_queue_next(next); - - if (next == nxt_queue_last(queue)) { - return middle; - } - } -} - - -/* - * nxt_queue_sort() provides a stable sort because it uses the insertion - * sort algorithm. Its worst and average computational complexity is O^2. - */ - -void -nxt_queue_sort(nxt_queue_t *queue, - nxt_int_t (*cmp)(const void *data, const nxt_queue_link_t *, - const nxt_queue_link_t *), const void *data) -{ - nxt_queue_link_t *link, *prev, *next; - - link = nxt_queue_first(queue); - - if (link == nxt_queue_last(queue)) { - return; - } - - for (link = nxt_queue_next(link); - link != nxt_queue_tail(queue); - link = next) - { - prev = nxt_queue_prev(link); - next = nxt_queue_next(link); - - nxt_queue_remove(link); - - do { - if (cmp(data, prev, link) <= 0) { - break; - } - - prev = nxt_queue_prev(prev); - - } while (prev != nxt_queue_head(queue)); - - nxt_queue_insert_after(prev, link); - } -} diff --git a/src/nxt_queue.h b/src/nxt_queue.h index 6b7f5d577..4fa8eaac5 100644 --- a/src/nxt_queue.h +++ b/src/nxt_queue.h @@ -194,12 +194,6 @@ typedef struct { nxt_container_of(lnk, type, link) -NXT_EXPORT nxt_queue_link_t *nxt_queue_middle(nxt_queue_t *queue); -NXT_EXPORT void nxt_queue_sort(nxt_queue_t *queue, - nxt_int_t (*cmp)(const void *, const nxt_queue_link_t *, - const nxt_queue_link_t *), const void *data); - - #define nxt_queue_each(elt, queue, type, link) \ do { \ nxt_queue_link_t *_lnk, *_nxt; \ diff --git a/src/nxt_regex.h b/src/nxt_regex.h index a24c50f80..70dbaa6c3 100644 --- a/src/nxt_regex.h +++ b/src/nxt_regex.h @@ -9,13 +9,24 @@ #if (NXT_HAVE_REGEX) -typedef struct nxt_regex_s nxt_regex_t; +/* + * The most match steps that one match of one pattern may use. The library + * default is 10,000,000. The library counts steps from zero again at each + * start position of a pattern that is not anchored. So such a pattern also + * gets a callout before each item, and one match may make at most this + * number of callouts in total. + */ +#define NXT_REGEX_MATCH_LIMIT 100000 - #if (NXT_HAVE_PCRE2) -typedef void nxt_regex_match_t; -#else +/* + * The most bytes of a subject that a match error writes to the log. The + * subject can come from a client. + */ +#define NXT_REGEX_LOG_SUBJECT 64 + + +typedef struct nxt_regex_s nxt_regex_t; typedef struct nxt_regex_match_s nxt_regex_match_t; -#endif typedef struct { size_t offset; diff --git a/src/nxt_router.c b/src/nxt_router.c index ba3b1a0e2..e00bc6dc9 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -21,10 +21,9 @@ #include #include #include +#include #include -#define NXT_SHARED_PORT_ID 0xFFFFu - #if (NXT_HAVE_OTEL) #define NXT_OTEL_BATCH_DEFAULT 128 #define NXT_OTEL_SAMPLING_DEFAULT 1 @@ -40,6 +39,7 @@ typedef struct { nxt_conf_value_t *limits_value; nxt_conf_value_t *processes_value; nxt_conf_value_t *targets_value; + nxt_msec_t start_timeout; } nxt_router_app_conf_t; @@ -84,20 +84,129 @@ typedef struct { } nxt_socket_rpc_t; +typedef struct nxt_router_start_timer_s nxt_router_start_timer_t; + + typedef struct { - nxt_app_t *app; - nxt_router_temp_conf_t *temp_conf; - uint8_t proto; /* 1 bit */ + nxt_app_t *app; + nxt_router_temp_conf_t *temp_conf; + nxt_router_start_timer_t *start_timer; + uint8_t proto; /* 1 bit */ } nxt_app_rpc_t; typedef struct { - nxt_app_joint_t *app_joint; - uint32_t generation; - uint8_t proto; /* 1 bit */ + nxt_app_joint_t *app_joint; + nxt_router_start_timer_t *start_timer; + uint32_t generation; + uint8_t proto; /* 1 bit */ + /* + * Not a start attempt at all any more: the registration was put back on + * the stream by nxt_router_app_start_expired() to reap the process the + * expired attempt left behind, and it owns an unaccounted_processes slot + * rather than a pending_processes one. + */ + uint8_t expired; /* 1 bit */ } nxt_app_joint_rpc_t; +/* + * Deadline for one START_PROCESS RPC. + * + * The only thing that can answer a start is the new worker itself, through + * nxt_unit_init() -> PROCESS_READY. A process that is forked successfully but + * never gets there -- a "type": "external" binary that blocks before exec'ing + * anything of ours, a runtime stuck in its own init -- answers nothing and + * dies of nothing, so the RPC stays armed for the life of the router. With + * the default "processes" that RPC is the sole continuation of + * nxt_router_conf_apply(), so the configuration PUT never returns and the + * controller queues every later request behind it, GET /status included. + * + * On expiry the RPC is failed through nxt_port_rpc_error(), which runs the + * handler that a real failure would have run. Nothing here duplicates that + * recovery: the pending_processes and proto_port_requests accounting stays in + * nxt_router_app_port_error() / nxt_router_app_prefork_error() and runs once, + * driven by the RPC layer. + * + * The struct owns nothing but a copy of the application name, deliberately: + * holding an nxt_app_t or an nxt_app_joint_t reference would let a deadline + * extend the lifetime of the very object whose start it is giving up on. Both + * ports are referenced, because the stream is only meaningful against the + * router port and the send queue is only reachable through the destination. + * + * Expiry cannot simply fail the RPC, because the START_PROCESS it is giving up + * on may not have left the router yet. nxt_port_socket_write2() reports NXT_OK + * once the message is in dport->messages, and what it queued there is a shallow + * copy: the payload pointer, which on the config-apply path lives in + * tmcf->mem_pool, and the application's shared-port descriptor numbers, which + * it borrows. Failing the RPC runs nxt_router_conf_error(), which unlinks the + * application (closing that shared port) and releases that pool -- and the + * queued copy would still be there, to be dereferenced and sent with + * SCM_RIGHTS when the destination finally drains. + * + * So the deadline is still armed at the write, and expiry resolves against the + * send queue first: + * + * not yet started nxt_port_socket_cancel() takes the message back and + * completes its buffer, and the failure is driven from + * that completion. + * partially sent left queued -- a fragment and the descriptors are + * already with the peer -- and the failure waits for the + * completion of the last fragment. See the note on + * ->expired below. + * fully sent the message is gone from the queue but its completion + * can still be pending: nxt_port_write_handler() removes + * it and only then queues the buffer completion, which + * can land behind a timer handler already queued ahead of + * it. Absence from the queue is not a lifetime boundary, + * so the failure again waits for the completion. + * completed ->send_done: nothing references the payload any more, + * so the RPC is failed inline. + * + * Everything but the last case therefore funnels through + * nxt_router_start_buf_completion(), which is the single point where the + * payload is known to be unreachable from the port layer. + * + * References: one for the RPC that holds it, one for the engine while the + * timer node is live, and one for a buffer completion that has yet to run. + */ + +struct nxt_router_start_timer_s { + nxt_timer_t timer; + /* The port the RPC is registered on, and the one it was written to. */ + nxt_port_t *port; + nxt_port_t *dport; + /* The queued message's identity, for nxt_port_socket_cancel(). */ + nxt_buf_t *buf; + nxt_port_id_t reply_port; + uint32_t stream; + nxt_msec_t timeout; + int32_t refs; + /* The timer node is live in the engine. */ + uint8_t armed; /* 1 bit */ + /* The deadline fired; the RPC must be failed once the payload is free. */ + uint8_t expired; /* 1 bit */ + /* The buffer completion has run: the port layer is done with the msg. */ + uint8_t send_done; /* 1 bit */ + /* The RPC has retired, so nothing may fail it again. */ + uint8_t retired; /* 1 bit */ + /* + * The START_PROCESS was taken back out of the send queue before the + * destination ever saw it, so no process was forked for it and there is + * nothing to account for. See nxt_router_app_port_error(). + */ + uint8_t recalled; /* 1 bit */ + /* + * The payload's completion found bytes of it still unsent, so the + * destination never saw the whole message and forked nothing for it. + * The same conclusion as ->recalled, reached at the other end of a send + * the deadline could not take back. See nxt_router_app_port_error(). + */ + uint8_t dropped; /* 1 bit */ + nxt_str_t app_name; +}; + + static nxt_int_t nxt_router_prefork(nxt_task_t *task, nxt_process_t *process, nxt_mp_t *mp); static nxt_int_t nxt_router_start(nxt_task_t *task, nxt_process_data_t *data); @@ -106,7 +215,23 @@ static void nxt_router_greet_controller(nxt_task_t *task, static nxt_int_t nxt_router_start_app_process(nxt_task_t *task, nxt_app_t *app); -static void nxt_router_new_port_handler(nxt_task_t *task, +static nxt_bool_t nxt_router_msg_from(nxt_task_t *task, + nxt_port_recv_msg_t *msg, nxt_process_type_t type); +static nxt_bool_t nxt_router_pid_is(nxt_task_t *task, nxt_pid_t pid, + nxt_process_type_t type); +static nxt_bool_t nxt_router_msg_from_self(nxt_task_t *task, + nxt_port_recv_msg_t *msg); +static nxt_bool_t nxt_router_msg_sender_ok(nxt_task_t *task, + nxt_port_recv_msg_t *msg); +static void nxt_router_msg_sender_refuse(nxt_task_t *task, + nxt_port_recv_msg_t *msg, const char *reason); +static const char *nxt_router_new_port_check(nxt_task_t *task, + nxt_port_recv_msg_t *msg); +static const char *nxt_router_get_port_check(nxt_task_t *task, + nxt_port_recv_msg_t *msg); +static const char *nxt_router_get_mmap_check(nxt_task_t *task, + nxt_port_recv_msg_t *msg); +static void nxt_router_gate_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); static void nxt_router_conf_data_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); @@ -122,6 +247,9 @@ static void nxt_router_conf_ready(nxt_task_t *task, nxt_router_temp_conf_t *tmcf); static void nxt_router_conf_send(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, nxt_port_msg_type_t type); +#if (NXT_TLS) +static void nxt_router_conf_error_tls(nxt_task_t *task, nxt_queue_t *sockets); +#endif static nxt_int_t nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, u_char *start, u_char *end); @@ -213,6 +341,9 @@ static void nxt_router_listen_socket_create(nxt_task_t *task, void *obj, void *data); static void nxt_router_listen_socket_update(nxt_task_t *task, void *obj, void *data); +static void nxt_router_listen_socket_release_stale(nxt_task_t *task, + nxt_event_engine_t *engine, nxt_listen_socket_t *ls, + nxt_socket_conf_joint_t *keep); static void nxt_router_listen_socket_delete(nxt_task_t *task, void *obj, void *data); static void nxt_router_worker_thread_quit(nxt_task_t *task, void *obj, @@ -227,16 +358,43 @@ static void nxt_router_req_headers_ack_handler(nxt_task_t *task, static void nxt_router_listen_socket_release(nxt_task_t *task, nxt_socket_conf_t *skcf); +static nxt_router_start_timer_t *nxt_router_start_timer_create(nxt_task_t *task, + nxt_app_t *app, nxt_port_t *port, nxt_port_t *dport, nxt_buf_t *b); +static void nxt_router_start_timer_arm(nxt_task_t *task, + nxt_router_start_timer_t *st, uint32_t stream); +static void nxt_router_start_timer_cancel(nxt_task_t *task, + nxt_router_start_timer_t *st); +static void nxt_router_start_timer_use(nxt_task_t *task, + nxt_router_start_timer_t *st, int32_t delta); +static void nxt_router_start_timeout(nxt_task_t *task, void *obj, void *data); +static void nxt_router_start_timer_release(nxt_task_t *task, void *obj, + void *data); +static void nxt_router_start_buf_completion(nxt_task_t *task, + nxt_router_start_timer_t *st, nxt_bool_t dropped); +static void nxt_router_start_conf_buf_completion(nxt_task_t *task, void *obj, + void *data); +static void nxt_router_start_app_buf_completion(nxt_task_t *task, void *obj, + void *data); + static void nxt_router_app_port_ready(nxt_task_t *task, nxt_port_recv_msg_t *msg, void *data); static void nxt_router_app_port_error(nxt_task_t *task, nxt_port_recv_msg_t *msg, void *data); +static void nxt_router_app_start_failed(nxt_task_t *task, nxt_app_t *app, + uint32_t count, nxt_bool_t unaccounted); +static void nxt_router_app_start_expired(nxt_task_t *task, nxt_app_t *app, + nxt_port_recv_msg_t *msg, nxt_app_joint_t *app_joint, uint32_t generation, + nxt_bool_t proto); +static void nxt_router_app_unaccounted_release(nxt_task_t *task, + nxt_app_t *app); static void nxt_router_app_use(nxt_task_t *task, nxt_app_t *app, int i); static void nxt_router_app_unlink(nxt_task_t *task, nxt_app_t *app); static void nxt_router_app_port_release(nxt_task_t *task, nxt_app_t *app, nxt_port_t *port, nxt_apr_action_t action); +static void nxt_router_app_abandoned_settle(nxt_task_t *task, + nxt_request_rpc_data_t *req_rpc_data); static void nxt_router_app_port_get(nxt_task_t *task, nxt_app_t *app, nxt_request_rpc_data_t *req_rpc_data); static void nxt_router_http_request_error(nxt_task_t *task, void *obj, @@ -247,7 +405,8 @@ static void nxt_router_http_request_done(nxt_task_t *task, void *obj, static void nxt_router_app_prepare_request(nxt_task_t *task, nxt_request_rpc_data_t *req_rpc_data); static nxt_buf_t *nxt_router_prepare_msg(nxt_task_t *task, - nxt_http_request_t *r, nxt_app_t *app, const nxt_str_t *prefix); + nxt_http_request_t *r, nxt_app_t *app, const nxt_str_t *prefix, + nxt_http_status_t *status); static void nxt_router_app_timeout(nxt_task_t *task, void *obj, void *data); static void nxt_router_adjust_idle_timer(nxt_task_t *task, void *obj, @@ -269,6 +428,14 @@ static void nxt_router_http_request_release_post(nxt_task_t *task, static void nxt_router_http_request_release(nxt_task_t *task, void *obj, void *data); static void nxt_router_oosm_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); +static void nxt_router_detached_handler(nxt_task_t *task, + nxt_port_recv_msg_t *msg); +static void nxt_router_detached_apply(nxt_task_t *task, nxt_pid_t pid, + uint8_t state); +static nxt_bool_t nxt_router_app_port_idle(nxt_task_t *task, nxt_app_t *app, + nxt_port_t *port); +static nxt_bool_t nxt_router_app_port_busy(nxt_task_t *task, nxt_app_t *app, + nxt_port_t *port, const char *reason); static void nxt_router_get_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); static void nxt_router_get_mmap_handler(nxt_task_t *task, @@ -293,21 +460,114 @@ static const nxt_str_t *nxt_app_msg_prefix[] = { }; +/* + * Each application process has the write end of the main port of the + * router, and the sender sets the message type. Thus the router accepts + * a message on this port only from its normal sender. This table lists + * the gated types. nxt_router_process_port_handlers points each of them + * to nxt_router_gate_handler(). That handler checks the sender of the + * row, then runs the check of the row if there is one, and then runs the + * handler of the row. To gate one more type, add its row here and point + * its slot to nxt_router_gate_handler(). + * + * The main port of the router has no shared memory queue. Thus each + * message comes through the socket, and the kernel gives the pid of the + * sender (SCM_CREDENTIALS). Without SCM_CREDENTIALS, the pid comes from + * the message header, which the sender sets. + * + * The sender checks read rt->port_by_type[]. NEW_PORT sets that slot, so + * NEW_PORT is gated too: only main can announce a port that is not an + * application port. + * + * The ports of the router engines are not gated. Their messages can come + * through the shared memory queue, which carries no pid. + */ + +typedef enum { + NXT_ROUTER_SENDER_NONE = 0, + NXT_ROUTER_SENDER_MAIN, + NXT_ROUTER_SENDER_CONTROLLER, + NXT_ROUTER_SENDER_MAIN_OR_PROTO, + /* Main, the controller or a registered prototype. */ + NXT_ROUTER_SENDER_NOT_APP, + /* A registered application process that names itself in the header. */ + NXT_ROUTER_SENDER_SELF, + /* Any sender. The check of the row decides. */ + NXT_ROUTER_SENDER_ANY, +} nxt_router_sender_t; + + +/* Returns NULL to accept the message, or the reason to refuse it. */ + +typedef const char *(*nxt_router_gate_check_t)(nxt_task_t *task, + nxt_port_recv_msg_t *msg); + + +typedef struct { + nxt_router_sender_t sender; + nxt_router_gate_check_t check; + nxt_port_handler_t handler; +} nxt_router_gate_t; + + +static const nxt_router_gate_t nxt_router_gates[NXT_PORT_MSG_MAX] = { + [_NXT_PORT_MSG_QUIT] = + { NXT_ROUTER_SENDER_MAIN, NULL, nxt_signal_quit_handler }, + [_NXT_PORT_MSG_CHANGE_FILE] = + { NXT_ROUTER_SENDER_MAIN, NULL, nxt_port_change_log_file_handler }, + [_NXT_PORT_MSG_ACCESS_LOG] = + { NXT_ROUTER_SENDER_MAIN, NULL, nxt_router_access_log_reopen_handler }, + [_NXT_PORT_MSG_REMOVE_PID] = + { NXT_ROUTER_SENDER_MAIN_OR_PROTO, NULL, + nxt_router_remove_pid_handler }, + [_NXT_PORT_MSG_DATA] = + { NXT_ROUTER_SENDER_CONTROLLER, NULL, nxt_router_conf_data_handler }, + [_NXT_PORT_MSG_APP_RESTART] = + { NXT_ROUTER_SENDER_CONTROLLER, NULL, + nxt_router_app_restart_handler }, + [_NXT_PORT_MSG_STATUS] = + { NXT_ROUTER_SENDER_CONTROLLER, NULL, nxt_router_status_handler }, + [_NXT_PORT_MSG_NEW_PORT] = + { NXT_ROUTER_SENDER_ANY, nxt_router_new_port_check, + nxt_router_new_port_handler }, + [_NXT_PORT_MSG_GET_PORT] = + { NXT_ROUTER_SENDER_SELF, nxt_router_get_port_check, + nxt_router_get_port_handler }, + [_NXT_PORT_MSG_GET_MMAP] = + { NXT_ROUTER_SENDER_SELF, nxt_router_get_mmap_check, + nxt_router_get_mmap_handler }, + [_NXT_PORT_MSG_MMAP] = + { NXT_ROUTER_SENDER_SELF, NULL, nxt_port_mmap_handler }, + [_NXT_PORT_MSG_OOSM] = + { NXT_ROUTER_SENDER_SELF, NULL, nxt_router_oosm_handler }, + /* + * An application sends its RPC replies to its own ports, never here. + * nxt_port_rpc_handler() does not compare the sender with the peer of + * the registration, so any process that passes can answer any stream. + */ + [_NXT_PORT_MSG_RPC_READY] = + { NXT_ROUTER_SENDER_NOT_APP, NULL, nxt_port_rpc_handler }, + [_NXT_PORT_MSG_RPC_ERROR] = + { NXT_ROUTER_SENDER_NOT_APP, NULL, nxt_port_rpc_handler }, +}; + + static const nxt_port_handlers_t nxt_router_process_port_handlers = { - .quit = nxt_signal_quit_handler, - .new_port = nxt_router_new_port_handler, - .get_port = nxt_router_get_port_handler, - .change_file = nxt_port_change_log_file_handler, - .mmap = nxt_port_mmap_handler, - .get_mmap = nxt_router_get_mmap_handler, - .data = nxt_router_conf_data_handler, - .app_restart = nxt_router_app_restart_handler, - .status = nxt_router_status_handler, - .remove_pid = nxt_router_remove_pid_handler, - .access_log = nxt_router_access_log_reopen_handler, - .rpc_ready = nxt_port_rpc_handler, - .rpc_error = nxt_port_rpc_handler, - .oosm = nxt_router_oosm_handler, + .quit = nxt_router_gate_handler, + .new_port = nxt_router_gate_handler, + .get_port = nxt_router_gate_handler, + .change_file = nxt_router_gate_handler, + .mmap = nxt_router_gate_handler, + .get_mmap = nxt_router_gate_handler, + .data = nxt_router_gate_handler, + .app_restart = nxt_router_gate_handler, + .status = nxt_router_gate_handler, + .remove_pid = nxt_router_gate_handler, + .access_log = nxt_router_gate_handler, + .rpc_ready = nxt_router_gate_handler, + .rpc_error = nxt_router_gate_handler, + .oosm = nxt_router_gate_handler, + .detached = nxt_router_detached_handler, }; @@ -397,22 +657,28 @@ nxt_router_greet_controller(nxt_task_t *task, nxt_port_t *controller_port) } -static void +void nxt_router_start_app_process_handler(nxt_task_t *task, nxt_port_t *port, void *data) { - size_t size; - uint32_t stream; - nxt_fd_t port_fd, queue_fd; - nxt_int_t ret; - nxt_app_t *app; - nxt_buf_t *b; - nxt_port_t *dport; - nxt_runtime_t *rt; - nxt_app_joint_rpc_t *app_joint_rpc; + size_t size; + uint32_t stream; + nxt_fd_t port_fd, queue_fd; + nxt_err_t err; + nxt_int_t ret; + nxt_app_t *app; + nxt_buf_t *b; + nxt_bool_t proto_gone; + nxt_port_t *dport; + nxt_runtime_t *rt; + nxt_app_joint_rpc_t *app_joint_rpc; + nxt_router_start_timer_t *st; app = data; + st = NULL; + proto_gone = 0; + nxt_thread_mutex_lock(&app->mutex); dport = app->proto_port; @@ -466,17 +732,77 @@ nxt_router_start_app_process_handler(nxt_task_t *task, nxt_port_t *port, stream = nxt_port_rpc_ex_stream(app_joint_rpc); + /* + * Before the write: the deadline resolves through the payload's completion + * handler, which nxt_port_socket_write2() may run before it returns. + */ + + st = nxt_router_start_timer_create(task, app, port, dport, b); + + if (st != NULL && b != NULL) { + b->completion_handler = nxt_router_start_app_buf_completion; + } + ret = nxt_port_socket_write2(task, dport, NXT_PORT_MSG_START_PROCESS, port_fd, queue_fd, stream, port->id, b); if (nxt_slow_path(ret != NXT_OK)) { nxt_port_rpc_cancel(task, port, stream); + /* + * A worker start goes to the prototype. The prototype can exit + * without a request from the router. On shutdown main sends QUIT + * to every process directly. The prototype reports the exit of its + * workers to the router (REMOVE_PID) and then exits. The router + * can handle such a REMOVE_PID before its own QUIT. It then starts + * a replacement worker in nxt_router_app_port_close(), and the + * socket of the prototype is already closed. A prototype that + * crashes gives the same result. The router cannot know this in + * advance: it clears ->proto_port only when the REMOVE_PID of the + * prototype arrives, or when it sends the QUIT itself. + * + * Nothing was sent, so nothing was forked. The slot goes back + * below in the usual way. Only the log level is different. A + * prototype start goes to main. Main is never expected to be gone, + * so that start keeps the alert. + */ + + err = dport->socket.error; + + proto_gone = (b == NULL + && (err == NXT_EPIPE || err == NXT_ECONNRESET + || err == NXT_ECONNREFUSED)); + goto failed; } app_joint_rpc->app_joint = app->joint; app_joint_rpc->generation = app->generation; app_joint_rpc->proto = (b != NULL); + app_joint_rpc->start_timer = st; + + if (st != NULL) { + nxt_router_start_timer_arm(task, st, stream); + } + + /* + * Key the registration by the process the START_PROCESS was sent to, so + * that its death retires the RPC. Without a peer the registration is + * only ever retired by a reply on its own stream: when the prototype + * dies between forking the worker and that worker's PROCESS_READY, the + * REMOVE_PID the router gets for the prototype carries no stream (it + * had already reached READY itself, see nxt_main_process.c) and + * nxt_port_rpc_remove_peer() finds nothing to fail, so the attempt's + * pending_processes slot is never given back. + * + * For a prototype start the peer is the main process, which is + * harmless: its death is fatal anyway, and a prototype dying before + * READY is already reported by the stream-bearing REMOVE_PID. + * + * The setter cannot report a failed insert: it logs and leaves ->peer + * at -1 (src/nxt_port_rpc.c:301), so a pool allocation failure here + * restores the peerless registration -- as at every other call site. + */ + nxt_port_rpc_ex_set_peer(task, port, app_joint_rpc, dport->pid); if (b != NULL) { app->proto_port_requests++; @@ -486,281 +812,1115 @@ nxt_router_start_app_process_handler(nxt_task_t *task, nxt_port_t *port, nxt_router_app_joint_use(task, app->joint, 1); + goto skip; + failed: + /* + * The attempt is over and no handler remains armed for it -- the two + * earlier sites never armed one, and the third cancels its registration + * with nxt_port_rpc_cancel() -- so nothing will ever report it back. + * Release the pending_processes slot the initiator took, exactly as + * nxt_router_app_port_error() does for the attempts that do get that far. + */ + + if (proto_gone) { + nxt_debug(task, "app '%V' start attempt cancelled: prototype %PI " + "is gone", &app->name, dport->pid); + + } else { + nxt_alert(task, "app '%V' failed to start a process", &app->name); + } + + if (st != NULL) { + /* + * Never armed, and the buffer below is freed without its completion + * handler ever running, so both references are dropped here. + */ + + if (b != NULL) { + nxt_router_start_timer_use(task, st, -1); + } + + nxt_router_start_timer_use(task, st, -1); + } + if (b != NULL) { nxt_mp_free(b->data, b); } + nxt_router_app_start_failed(task, app, 1, 0); + skip: nxt_router_app_use(task, app, -1); } -static void -nxt_router_app_joint_use(nxt_task_t *task, nxt_app_joint_t *app_joint, int i) +/* + * Create the deadline for a START_PROCESS RPC, before the write. + * + * Before, because the payload buffer has to carry a completion handler that + * knows about this struct: the deadline resolves through that completion, and + * nxt_port_socket_write2() can run it before it returns. + * + * The timer runs on the engine of the port the RPC is registered on -- always + * the router's own port, for both the config-apply prefork path + * (nxt_router_app_rpc_create()) and the on-demand path + * (nxt_router_start_app_process_handler(), reached by nxt_port_post() to that + * same port) -- so the expiry handler, the reply handlers it races and the + * buffer completion cannot run concurrently. + * + * A NULL return means no deadline, which is the configured behaviour for + * "start_timeout": 0 and the fallback if the allocation fails; the start is + * then exactly as unbounded as it was before this existed, which is worse than + * the alternative but not worse than failing a start over a small malloc. + */ + +static nxt_router_start_timer_t * +nxt_router_start_timer_create(nxt_task_t *task, nxt_app_t *app, + nxt_port_t *port, nxt_port_t *dport, nxt_buf_t *b) { - app_joint->use_count += i; + nxt_msec_t timeout; + nxt_router_start_timer_t *st; - if (app_joint->use_count == 0) { - nxt_assert(app_joint->app == NULL); + /* Immutable once the application is configured, so read unlocked. */ + timeout = app->start_timeout; - nxt_free(app_joint); + if (timeout == 0) { + return NULL; } -} + st = nxt_malloc(sizeof(nxt_router_start_timer_t) + app->name.length); + if (nxt_slow_path(st == NULL)) { + nxt_alert(task, "app \"%V\" start deadline not armed: out of memory", + &app->name); -static nxt_int_t -nxt_router_start_app_process(nxt_task_t *task, nxt_app_t *app) -{ - nxt_int_t res; - nxt_port_t *router_port; - nxt_runtime_t *rt; - - nxt_debug(task, "app '%V' start process", &app->name); + return NULL; + } - rt = task->thread->runtime; - router_port = rt->port_by_type[NXT_PROCESS_ROUTER]; + nxt_memzero(st, sizeof(nxt_router_start_timer_t)); - nxt_router_app_use(task, app, 1); + st->app_name.start = nxt_pointer_to(st, sizeof(nxt_router_start_timer_t)); + st->app_name.length = app->name.length; + nxt_memcpy(st->app_name.start, app->name.start, app->name.length); - res = nxt_port_post(task, router_port, nxt_router_start_app_process_handler, - app); + st->timeout = timeout; + st->port = port; + st->dport = dport; + st->reply_port = port->id; + st->buf = b; - if (res == NXT_OK) { - return res; - } + nxt_port_inc_use(port); + nxt_port_inc_use(dport); - nxt_thread_mutex_lock(&app->mutex); + /* The reference of the caller, which hands it to the RPC. */ + st->refs = 1; - app->pending_processes--; + if (b == NULL) { + /* + * Nothing to wait for: a message with no payload and no descriptors + * (the worker-start leg, where the prototype is already up) leaves + * nothing of ours reachable from the send queue, so expiry can fail + * the RPC directly. + */ + st->send_done = 1; - nxt_thread_mutex_unlock(&app->mutex); + } else { + /* The reference of the pending buffer completion. */ + st->refs++; - nxt_router_app_use(task, app, -1); + b->parent = st; + } - return NXT_ERROR; + return st; } -nxt_inline nxt_bool_t -nxt_router_msg_cancel(nxt_task_t *task, nxt_request_rpc_data_t *req_rpc_data) +/* + * Arm it. Separate from creation because the stream is only known once the + * RPC is registered, and because a write that fails must be able to drop the + * struct without ever having put a node in the engine's timer tree. + */ + +static void +nxt_router_start_timer_arm(nxt_task_t *task, nxt_router_start_timer_t *st, + uint32_t stream) { - nxt_buf_t *b, *next; - nxt_bool_t cancelled; - nxt_port_t *app_port; - nxt_msg_info_t *msg_info; + nxt_event_engine_t *engine; - msg_info = &req_rpc_data->msg_info; + st->stream = stream; - if (msg_info->buf == NULL) { - return 0; - } + engine = task->thread->engine; - app_port = req_rpc_data->app_port; + st->timer.bias = NXT_TIMER_DEFAULT_BIAS; + st->timer.work_queue = &engine->fast_work_queue; + st->timer.handler = nxt_router_start_timeout; + st->timer.task = &engine->task; + st->timer.log = st->timer.task->log; - if (app_port != NULL && app_port->id == NXT_SHARED_PORT_ID) { - cancelled = nxt_app_queue_cancel(app_port->queue, - msg_info->tracking_cookie, - req_rpc_data->stream); + /* The reference of the engine, while the timer node is live. */ + st->refs++; + st->armed = 1; - if (cancelled) { - nxt_debug(task, "stream #%uD: cancelled by router", - req_rpc_data->stream); - } + nxt_timer_add(engine, &st->timer, st->timeout); - } else { - cancelled = 0; - } + nxt_debug(task, "app \"%V\" stream #%uD start deadline %M ms", + &st->app_name, stream, st->timeout); +} - for (b = msg_info->buf; b != NULL; b = next) { - next = b->next; - b->next = NULL; - if (b->is_port_mmap_sent) { - b->is_port_mmap_sent = cancelled == 0; - } +static void +nxt_router_start_timer_use(nxt_task_t *task, nxt_router_start_timer_t *st, + int32_t delta) +{ + st->refs += delta; - b->completion_handler(task, b, b->parent); + nxt_assert(st->refs >= 0); + + if (st->refs > 0) { + return; } - msg_info->buf = NULL; + nxt_port_use(task, st->port, -1); + nxt_port_use(task, st->dport, -1); - return cancelled; + nxt_free(st); } -nxt_inline nxt_bool_t -nxt_queue_chk_remove(nxt_queue_link_t *lnk) +/* + * The start answered, or its write failed: drop the RPC's reference and, with + * it, the deadline. + * + * nxt_timer_disable() is not enough -- it clears the enabled bit but leaves + * the node in the engine's rbtree, which would dangle the moment this struct + * is freed. nxt_timer_delete() removes it, but its removal can itself be a + * queued change referencing the timer, so a non-zero return means the struct + * is still reachable from the engine: the engine's reference is then handed to + * a release handler instead of being dropped here. Re-arming at zero is the + * same trick nxt_router_free_app() uses for app_joint->idle_timer. + * + * When the expiry handler is the caller's own caller, ->armed is already + * clear: the node has fired, so it owns the engine's reference and drops it + * itself. + */ + +static void +nxt_router_start_timer_cancel(nxt_task_t *task, nxt_router_start_timer_t *st) { - if (lnk->next != NULL) { - nxt_queue_remove(lnk); + nxt_event_engine_t *engine; - lnk->next = NULL; + st->retired = 1; - return 1; + if (st->armed) { + engine = task->thread->engine; + + st->armed = 0; + + if (nxt_timer_delete(engine, &st->timer)) { + st->timer.handler = nxt_router_start_timer_release; + nxt_timer_add(engine, &st->timer, 0); + + } else { + nxt_router_start_timer_use(task, st, -1); + } } - return 0; + nxt_router_start_timer_use(task, st, -1); } -nxt_inline void -nxt_request_rpc_data_unlink(nxt_task_t *task, - nxt_request_rpc_data_t *req_rpc_data) +static void +nxt_router_start_timer_release(nxt_task_t *task, void *obj, void *data) { - nxt_app_t *app; - nxt_bool_t unlinked; - nxt_http_request_t *r; + nxt_timer_t *timer; - nxt_router_msg_cancel(task, req_rpc_data); - - app = req_rpc_data->app; + timer = obj; - if (req_rpc_data->app_port != NULL) { - nxt_router_app_port_release(task, app, req_rpc_data->app_port, - req_rpc_data->apr_action); + nxt_router_start_timer_use(task, + nxt_timer_data(timer, nxt_router_start_timer_t, timer), -1); +} - req_rpc_data->app_port = NULL; - } - r = req_rpc_data->request; +/* + * The deadline fired. What it may do depends on where the START_PROCESS it is + * giving up on has got to; see the comment on nxt_router_start_timer_s. + */ - if (r != NULL) { - r->timer_data = NULL; +static void +nxt_router_start_timeout(nxt_task_t *task, void *obj, void *data) +{ + nxt_timer_t *timer; + nxt_port_msg_cancel_t cancelled; + nxt_router_start_timer_t *st; - nxt_router_http_request_release_post(task, r); + timer = obj; + st = nxt_timer_data(timer, nxt_router_start_timer_t, timer); - r->req_rpc_data = NULL; - req_rpc_data->request = NULL; + /* + * "unless a reply lands first" is not hedging. The failure is driven + * from the payload's completion, which for a partly sent message waits + * for the rest of it to leave (see below), and a PROCESS_READY that + * arrives in that window retires the stream before the deadline can. + */ - if (app != NULL) { - unlinked = 0; + nxt_alert(task, "app \"%V\" process did not become ready in time; the " + "deadline expired, and the start request (stream #%uD) " + "will be failed unless a reply lands first. The process, " + "if it is still running, never called nxt_unit_init() -- " + "raise \"limits\": {\"start_timeout\"} if the application " + "simply needs longer to start", + &st->app_name, st->stream); - nxt_thread_mutex_lock(&app->mutex); + /* + * The node has fired, so it is out of the engine's tree: clearing ->armed + * both records that and keeps the error handler reached below from trying + * to delete it again through nxt_router_start_timer_cancel(). The + * engine's reference is this frame's now, and is dropped at the end. + */ - if (r->app_link.next != NULL) { - nxt_queue_remove(&r->app_link); - r->app_link.next = NULL; + st->armed = 0; + st->expired = 1; - unlinked = 1; - } + /* + * Take the message back if there is still anything to take back. Both + * shapes of START_PROCESS go through this, for different reasons. + * + * The one that carries a payload (a prototype start) must be recalled + * before its RPC can be failed at all: failing it releases the pool the + * payload lives in and the descriptors it borrows. + * + * The header-only one (a worker start, sent to a prototype that is already + * up) carries no router memory and no descriptors, so nothing about it is + * a lifetime hazard. Recalling it is still worth doing: delivered after + * the stream is retired, it starts a worker whose PROCESS_READY answers + * nobody -- nxt_port_rpc_handler() drops an unknown stream -- leaving a + * process and a port the router is not tracking. That is the same orphan + * the deadline already leaves when a worker is merely slow, since the + * router never learns its pid and so cannot kill it either, and reaping in + * the prototype is what will collect both; but there is no reason to + * create one while the message has not left yet. + * + * A destination whose port has a shared-memory queue is the case this + * cannot reach: nxt_port_socket_write2() puts the message straight into + * the ring, where it is already delivered, and only a READ_QUEUE + * notification can be left in port->messages. Matching on the message + * type is what keeps this from recalling that notification and stranding a + * message that is genuinely on its way. + */ - nxt_thread_mutex_unlock(&app->mutex); + cancelled = nxt_port_socket_cancel(task, st->dport, + NXT_PORT_MSG_START_PROCESS, + st->stream, st->reply_port, st->buf); - if (unlinked) { - nxt_mp_release(r->mem_pool); - } - } - } + /* + * Recalled before the destination saw it: no worker was forked for this + * start, so there is no process to account for and the slot goes back the + * ordinary way. Accounting it would move a slot to unaccounted_processes + * that nothing can ever drain -- no PROCESS_READY and no REMOVE_PID are + * coming for a process that was never created -- which would turn the + * bound into a ratchet. See nxt_router_app_port_error(). + */ - if (app != NULL) { - nxt_router_app_use(task, app, -1); + st->recalled = (cancelled == NXT_PORT_MSG_CANCELLED); + + if (st->send_done) { + /* + * Nothing of ours is in the port layer's hands -- either the payload's + * completion has already run, or there was never a payload -- so the + * RPC is failed here. There is nothing left to drive it otherwise: a + * header-only message has no buffers for the recall to complete. The + * stream may meanwhile have been retired by a reply that landed in the + * same turn of the event loop; nxt_port_rpc_error() drops an unknown + * stream, so the race costs a debug line. + */ + + nxt_port_rpc_error(task, st->port, st->stream); + + } else { + /* + * A payload still in flight: whichever of the three states it is in, + * the RPC is failed from nxt_router_start_buf_completion(). + * NXT_PORT_MSG_CANCELLED has just queued that completion, and the + * other two mean the message is still being sent, so the completion is + * still to come. Waiting is what keeps nxt_router_conf_error() from + * releasing a payload the send queue still points at. + * + * NXT_PORT_MSG_STARTED is the limitation of a protocol without + * cancellation: a destination that took one fragment and then stopped + * draining holds the failure off for as long as it stays stuck, and it + * is worth saying so, because from the outside that looks like a + * deadline that did not fire. The wedge this deadline exists for -- a + * destination that is draining fine and a worker that never announces + * itself -- is unaffected, because there the message goes out whole. + */ + + if (cancelled == NXT_PORT_MSG_STARTED) { + nxt_alert(task, "app \"%V\" stream #%uD start request is partly " + "sent; failing it has to wait for the rest of it " + "to leave, and a reply that lands meanwhile wins", + &st->app_name, st->stream); + } + } + + /* The engine's reference: the timer node is gone. */ + + nxt_router_start_timer_use(task, st, -1); +} + + +/* + * Which way it left: the port layer advances the payload as it sends it, so + * bytes still unconsumed at its completion mean the destination never saw the + * whole message. It was taken back by nxt_port_socket_cancel(), or dropped by + * nxt_port_msg_drop() because the destination died or the port layer could not + * requeue it -- and nothing was forked for a start that never arrived. + * + * One shape does not reach here. An inline first fragment that hits EAGAIN + * and cannot be held for a later attempt is refused outright, with + * NXT_ERROR and the payload untouched, so the caller fails the start itself. + * + * ->recalled cannot answer this on its own: a message the deadline finds + * partly sent stays queued, and only its completion knows whether the rest of + * it ever left. See nxt_router_app_port_error(). + */ + +nxt_inline nxt_bool_t +nxt_router_start_buf_dropped(nxt_buf_t *b) +{ + return nxt_buf_mem_used_size(&b->mem) != 0; +} + + +/* + * The payload has left the port layer, one way or another: written in full, + * dropped by nxt_port_socket_cancel(), or released by nxt_port_error_handler() + * when the destination died. This is the only point at which nothing in + * dport->messages can reach the buffer or the borrowed descriptors any more, + * so it is where an expired start is failed. + */ + +static void +nxt_router_start_buf_completion(nxt_task_t *task, nxt_router_start_timer_t *st, + nxt_bool_t dropped) +{ + nxt_bool_t fail; + + st->send_done = 1; + st->buf = NULL; + st->dropped = dropped; + + fail = (st->expired && !st->retired); + + if (fail) { + /* + * Nothing may touch b after this: failing the RPC reaches + * nxt_router_conf_error(), which releases the pool the buffer was + * allocated from. + */ + + nxt_port_rpc_error(task, st->port, st->stream); + } + + nxt_router_start_timer_use(task, st, -1); +} + + +/* + * The config-apply payload is allocated from tmcf->mem_pool and freed with it, + * so this replaces nxt_buf_dummy_completion() and frees nothing. + */ + +static void +nxt_router_start_conf_buf_completion(nxt_task_t *task, void *obj, void *data) +{ + nxt_router_start_buf_completion(task, data, + nxt_router_start_buf_dropped(obj)); +} + + +/* + * The on-demand payload is allocated from the engine's pool and owns itself, + * so this replaces nxt_buf_completion(): it returns the buffer to that pool + * first, because failing the RPC below may not return at all cheaply. The + * buffer never has a parent, so none of nxt_buf_parent_completion()'s work + * applies -- which is also why the context travels in b->parent and this + * handler is not nxt_buf_completion(). + */ + +static void +nxt_router_start_app_buf_completion(nxt_task_t *task, void *obj, void *data) +{ + nxt_buf_t *b; + nxt_bool_t dropped; + nxt_router_start_timer_t *st; + + b = obj; + st = data; + + nxt_assert(b->next == NULL); + nxt_assert(b->parent == st); + + /* Read before the free: the answer is in the buffer's own pointers. */ + + dropped = nxt_router_start_buf_dropped(b); + + nxt_mp_free(b->data, b); + + nxt_router_start_buf_completion(task, st, dropped); +} + + + +static void +nxt_router_app_joint_use(nxt_task_t *task, nxt_app_joint_t *app_joint, int i) +{ + app_joint->use_count += i; + + if (app_joint->use_count == 0) { + nxt_assert(app_joint->app == NULL); + + nxt_free(app_joint); + } +} + + +static nxt_int_t +nxt_router_start_app_process(nxt_task_t *task, nxt_app_t *app) +{ + nxt_int_t res; + nxt_port_t *router_port; + nxt_runtime_t *rt; + + nxt_debug(task, "app '%V' start process", &app->name); + + rt = task->thread->runtime; + router_port = rt->port_by_type[NXT_PROCESS_ROUTER]; + + nxt_router_app_use(task, app, 1); + + res = nxt_port_post(task, router_port, nxt_router_start_app_process_handler, + app); + + if (res == NXT_OK) { + return res; + } + + nxt_thread_mutex_lock(&app->mutex); + + app->pending_processes--; + + nxt_thread_mutex_unlock(&app->mutex); + + nxt_router_app_use(task, app, -1); + + return NXT_ERROR; +} + + +/* + * Take the queued request message back from the shared port queue. + * nxt_app_queue_cancel() (src/nxt_app_queue.h) is a CAS on the queue item's + * tracking word; nxt_unit_app_queue_recv() runs the same CAS from the worker + * side, so exactly one of the two wins and a false answer here means a worker + * claimed the slot first. + * + * The CAS runs at most once per request, and the answer is kept in + * ->msg_info.cancel for the callers that need it afterwards: both outcomes + * leave the tracking word at 0, so a second CAS would report a claim that + * never happened and clear ->is_port_mmap_sent on a chunk the worker owns. + */ + +nxt_inline nxt_bool_t +nxt_router_msg_retract(nxt_task_t *task, nxt_request_rpc_data_t *req_rpc_data) +{ + nxt_port_t *app_port; + nxt_msg_info_t *msg_info; + + msg_info = &req_rpc_data->msg_info; + + if (msg_info->buf == NULL) { + return 0; + } + + if (msg_info->cancel == NXT_MSG_QUEUED) { + app_port = req_rpc_data->app_port; + + if (app_port == NULL || app_port->id != NXT_SHARED_PORT_ID) { + /* Acknowledged: the message is the worker's, not the queue's. */ + return 0; + } + + if (nxt_app_queue_cancel(app_port->queue, msg_info->tracking_cookie, + req_rpc_data->stream)) + { + msg_info->cancel = NXT_MSG_RETRACTED; + + nxt_debug(task, "stream #%uD: cancelled by router", + req_rpc_data->stream); + + } else { + msg_info->cancel = NXT_MSG_CLAIMED; + + nxt_debug(task, "stream #%uD: claimed by a worker", + req_rpc_data->stream); + } + } + + return msg_info->cancel == NXT_MSG_RETRACTED; +} + + +nxt_inline nxt_bool_t +nxt_router_msg_cancel(nxt_task_t *task, nxt_request_rpc_data_t *req_rpc_data) +{ + nxt_buf_t *b, *next; + nxt_bool_t cancelled; + nxt_msg_info_t *msg_info; + + msg_info = &req_rpc_data->msg_info; + + if (msg_info->buf == NULL) { + return 0; + } + + cancelled = nxt_router_msg_retract(task, req_rpc_data); + + for (b = msg_info->buf; b != NULL; b = next) { + next = b->next; + b->next = NULL; + + if (b->is_port_mmap_sent) { + b->is_port_mmap_sent = cancelled == 0; + } + + b->completion_handler(task, b, b->parent); + } + + msg_info->buf = NULL; + + return cancelled; +} + + +nxt_inline nxt_bool_t +nxt_queue_chk_remove(nxt_queue_link_t *lnk) +{ + if (lnk->next != NULL) { + nxt_queue_remove(lnk); + + lnk->next = NULL; + + return 1; + } + + return 0; +} + + +nxt_inline void +nxt_request_rpc_data_unlink(nxt_task_t *task, + nxt_request_rpc_data_t *req_rpc_data) +{ + nxt_app_t *app; + nxt_bool_t unlinked; + nxt_http_request_t *r; + + nxt_router_msg_cancel(task, req_rpc_data); + + app = req_rpc_data->app; + + if (req_rpc_data->app_port != NULL) { + nxt_router_app_port_release(task, app, req_rpc_data->app_port, + req_rpc_data->apr_action); + + req_rpc_data->app_port = NULL; + } + + r = req_rpc_data->request; + + if (r != NULL) { + r->timer_data = NULL; + + nxt_router_http_request_release_post(task, r); + + r->req_rpc_data = NULL; + req_rpc_data->request = NULL; + + if (app != NULL) { + unlinked = 0; + + nxt_thread_mutex_lock(&app->mutex); + + if (r->app_link.next != NULL) { + nxt_queue_remove(&r->app_link); + r->app_link.next = NULL; + + unlinked = 1; + } + + nxt_thread_mutex_unlock(&app->mutex); + + if (unlinked) { + nxt_mp_release(r->mem_pool); + } + } + } + + if (app != NULL) { + nxt_router_app_use(task, app, -1); req_rpc_data->app = NULL; } - if (req_rpc_data->msg_info.body_fd != -1) { - nxt_fd_close(req_rpc_data->msg_info.body_fd); + if (req_rpc_data->msg_info.body_fd != -1) { + nxt_fd_close(req_rpc_data->msg_info.body_fd); + + req_rpc_data->msg_info.body_fd = -1; + } + + if (req_rpc_data->rpc_cancel) { + req_rpc_data->rpc_cancel = 0; + + nxt_port_rpc_cancel(task, task->thread->engine->port, + req_rpc_data->stream); + } +} + + +void +nxt_router_new_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) +{ + nxt_int_t res; + nxt_app_t *app; + nxt_port_t *port, *main_app_port; + nxt_runtime_t *rt; + + nxt_port_new_port_handler(task, msg); + + port = msg->u.new_port; + + if (port != NULL && port->type == NXT_PROCESS_CONTROLLER) { + nxt_router_greet_controller(task, msg->u.new_port); + } + + /* + * The router maps a queue for application ports only, so every other + * branch below has to release the queue descriptor that + * nxt_port_new_port_handler() left behind. None of the RPC handlers + * reachable from here reads msg->fd, so closing before dispatching them + * takes nothing away. + */ + + if (port != NULL && port->type == NXT_PROCESS_PROTOTYPE) { + nxt_port_recv_msg_close_fds(msg); + + nxt_port_rpc_handler(task, msg); + + return; + } + + if (port == NULL || port->type != NXT_PROCESS_APP) { + nxt_port_recv_msg_close_fds(msg); + + if (msg->port_msg.stream == 0) { + return; + } + + msg->port_msg.type = _NXT_PORT_MSG_RPC_ERROR; + + } else { + if (msg->fd[1] != -1) { + res = nxt_router_port_queue_map(task, port, msg->fd[1]); + + /* + * Closed whether or not the mapping succeeded: the mapping does + * not keep the descriptor, and the dispatcher reclaims nothing a + * handler leaves behind. The size check above is what makes this + * matter -- it lets a peer choose to be refused, so the failure + * return below handed that peer a descriptor of ours per message, + * which is the exhaustion the check exists to prevent. + * + * The return itself is left alone. It also walks past the RPC + * dispatch below, so a refused queue leaves the pending start + * outstanding, and the port stays registered although a port + * whose queue was refused can never be reached. Both are worth + * fixing and neither is fixable here: the port this handler is + * holding may be one nxt_port_new_port_handler() just created or + * one that was already live and serving, and nothing at this call + * site can tell them apart -- so failing the start or releasing + * the port would, for a forged duplicate NEW_PORT, do it to a + * working application port. That needs the two handlers to agree + * on provenance first; see #223. + */ + + nxt_fd_close(msg->fd[1]); + msg->fd[1] = -1; + + if (nxt_slow_path(res != NXT_OK)) { + return; + } + } + } + + if (msg->port_msg.stream != 0) { + nxt_port_rpc_handler(task, msg); + return; + } + + nxt_debug(task, "new port id %d (%d)", port->id, port->type); + + /* + * An application's "main" port has id 0 and is announced carrying the + * stream of the start it answers, which the dispatch above consumes. + * Arriving here means id 0 with no stream, and the arm below cannot + * serve that pair: it looks the announcement up as a *sibling* of an + * already-known main port, and for id 0 that lookup finds the port + * itself. Which of the two cases this is decides the remedy, and only + * nxt_port_new_port_handler() knows -- hence msg->new_port_created. + * + * Already registered: a worker that sent PROCESS_READY a second time. + * Its start stream was retired by the first announcement + * (nxt_port_process_ready_handler(), src/nxt_port.c), so the repeat + * carries none. The port is live and its queue, if it brought one, was + * mapped above -- leave it alone. Falling through would re-add it to + * the application hash: nxt_port_hash_add() declines the duplicate key, + * but ->port_hash_count is incremented regardless, and + * nxt_router_app_need_start() reads that count, so the router would + * believe it has workers it does not have. It would also send a second + * PORT_ACK for one port. + * + * Created by this message: an application main port announced with no + * stream at all, which no start is waiting on and nothing else will + * complete. Keeping it would leave a port registered in the runtime + * that no application owns and no PORT_ACK was ever sent for. Undo the + * registration this message caused instead. + * + * An assertion is not enough for either: nxt_assert() compiles out in a + * release build, which is exactly where the miscount and the orphan + * would do their damage. + */ + if (nxt_slow_path(port->id == 0)) { + + if (msg->new_port_created) { + nxt_alert(task, "new port of process %PI has id 0 and no start " + "stream; refused", port->pid); + + nxt_port_close(task, port); + nxt_runtime_port_remove(task, port); + + return; + } + + nxt_log(task, NXT_LOG_WARN, "process %PI announced its main port " + "again with no start stream; already registered, ignored", + port->pid); + + return; + } + + /* Find 'main' app port and get app reference. */ + rt = task->thread->runtime; + + /* + * It is safe to access 'runtime->ports' hash because 'NEW_PORT' + * sent to main port (with id == 0) and processed in main thread. + */ + main_app_port = nxt_port_hash_find(&rt->ports, port->pid, 0); + nxt_assert(main_app_port != NULL); + + app = main_app_port->app; + + if (nxt_fast_path(app != NULL)) { + nxt_thread_mutex_lock(&app->mutex); + + /* TODO here should be find-and-add code because there can be + port waiters in port_hash */ + nxt_port_hash_add(&app->port_hash, port); + app->port_hash_count++; + + nxt_thread_mutex_unlock(&app->mutex); + + port->app = app; + } + + port->main_app_port = main_app_port; + + nxt_port_socket_write(task, port, NXT_PORT_MSG_PORT_ACK, -1, 0, 0, NULL); +} + + +/* The sender pid against the port of the process type; see the gate table. */ + +static nxt_bool_t +nxt_router_msg_from(nxt_task_t *task, nxt_port_recv_msg_t *msg, + nxt_process_type_t type) +{ + nxt_port_t *port; + + /* + * Read at message time, because a restarted router gets the ports + * again through NEW_PORT. A NULL port refuses the message. + */ + + port = task->thread->runtime->port_by_type[type]; + + return (port != NULL && nxt_recv_msg_cmsg_pid(msg) == port->pid); +} + + +/* + * Whether the pid is a registered process of the type. A prototype sends + * REMOVE_PID for a child that it forked (nxt_proto_child_exited()). The + * router cannot check that the pid is a child of that prototype: a child + * that stopped during its start has no record in the router. Thus any + * registered prototype passes. + */ + +static nxt_bool_t +nxt_router_pid_is(nxt_task_t *task, nxt_pid_t pid, nxt_process_type_t type) +{ + nxt_bool_t is; + nxt_process_t *process; + + process = nxt_runtime_process_ref(task->thread->runtime, pid); + if (process == NULL) { + return 0; + } + + is = (nxt_process_type(process) == type); + + nxt_process_use(task, process, -1); + + return is; +} + + +/* + * The handlers of GET_PORT, GET_MMAP, MMAP and OOSM act for the process + * that the header names. Only libunit sends them, for its own process. + */ + +static nxt_bool_t +nxt_router_msg_from_self(nxt_task_t *task, nxt_port_recv_msg_t *msg) +{ + nxt_pid_t pid; - req_rpc_data->msg_info.body_fd = -1; + pid = nxt_recv_msg_cmsg_pid(msg); + + return (pid > 0 && pid == msg->port_msg.pid + && nxt_router_pid_is(task, pid, NXT_PROCESS_APP)); +} + + +static nxt_bool_t +nxt_router_msg_sender_ok(nxt_task_t *task, nxt_port_recv_msg_t *msg) +{ + nxt_bool_t ok; + const char *reason; + const nxt_router_gate_t *gate; + + /* nxt_port_handler() dispatches only a type below NXT_PORT_MSG_MAX. */ + + gate = &nxt_router_gates[msg->port_msg.type]; + + switch (gate->sender) { + + case NXT_ROUTER_SENDER_MAIN: + ok = nxt_router_msg_from(task, msg, NXT_PROCESS_MAIN); + break; + + case NXT_ROUTER_SENDER_CONTROLLER: + ok = nxt_router_msg_from(task, msg, NXT_PROCESS_CONTROLLER); + break; + + case NXT_ROUTER_SENDER_MAIN_OR_PROTO: + ok = nxt_router_msg_from(task, msg, NXT_PROCESS_MAIN) + || nxt_router_pid_is(task, nxt_recv_msg_cmsg_pid(msg), + NXT_PROCESS_PROTOTYPE); + break; + + case NXT_ROUTER_SENDER_NOT_APP: + ok = nxt_router_msg_from(task, msg, NXT_PROCESS_MAIN) + || nxt_router_msg_from(task, msg, NXT_PROCESS_CONTROLLER) + || nxt_router_pid_is(task, nxt_recv_msg_cmsg_pid(msg), + NXT_PROCESS_PROTOTYPE); + break; + + case NXT_ROUTER_SENDER_SELF: + ok = nxt_router_msg_from_self(task, msg); + break; + + case NXT_ROUTER_SENDER_ANY: + ok = 1; + break; + + default: + /* A type without a row has no sender that passes. */ + ok = 0; + break; } - if (req_rpc_data->rpc_cancel) { - req_rpc_data->rpc_cancel = 0; + reason = "wrong sender"; - nxt_port_rpc_cancel(task, task->thread->engine->port, - req_rpc_data->stream); + if (nxt_fast_path(ok)) { + reason = (gate->check != NULL) ? gate->check(task, msg) : NULL; + + if (nxt_fast_path(reason == NULL)) { + return 1; + } } + + nxt_router_msg_sender_refuse(task, msg, reason); + + return 0; } +/* + * No reply: the sender selects the reply port and the stream, so a reply + * would let a false sender make the router write to any stream. + */ + static void -nxt_router_new_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) +nxt_router_msg_sender_refuse(nxt_task_t *task, nxt_port_recv_msg_t *msg, + const char *reason) { - nxt_int_t res; - nxt_app_t *app; - nxt_port_t *port, *main_app_port; - nxt_runtime_t *rt; + nxt_alert(task, "process %PI sent message type %uD claiming process %PI; " + "refused: %s", nxt_recv_msg_cmsg_pid(msg), + (uint32_t) msg->port_msg.type, msg->port_msg.pid, reason); - nxt_port_new_port_handler(task, msg); + nxt_port_recv_msg_close_fds(msg); - port = msg->u.new_port; +#if (NXT_TESTS) + nxt_router_test_senders_refused++; +#endif +} - if (port != NULL && port->type == NXT_PROCESS_CONTROLLER) { - nxt_router_greet_controller(task, msg->u.new_port); + +/* + * Main announces every process. A prototype announces the main port of + * each worker it starts (nxt_port_process_ready_handler()), with the stream + * of the start. libunit announces the port of each context it adds, for + * its own process and with no stream (nxt_unit_send_port()). Thus only + * main can announce a port that is not an application port. + */ + +static const char * +nxt_router_new_port_check(nxt_task_t *task, nxt_port_recv_msg_t *msg) +{ + nxt_port_msg_new_port_t new_port; + + if (nxt_slow_path(nxt_port_new_port_msg(msg, &new_port) != NXT_OK)) { + return "short new port message"; } - if (port != NULL && port->type == NXT_PROCESS_PROTOTYPE) { - nxt_port_rpc_handler(task, msg); + if (nxt_router_msg_from(task, msg, NXT_PROCESS_MAIN)) { + return NULL; + } - return; + if (nxt_slow_path(new_port.type != NXT_PROCESS_APP)) { + return "not an application port"; } - if (port == NULL || port->type != NXT_PROCESS_APP) { + if (nxt_router_pid_is(task, nxt_recv_msg_cmsg_pid(msg), + NXT_PROCESS_PROTOTYPE)) + { + return NULL; + } - if (msg->port_msg.stream == 0) { - return; - } + if (nxt_slow_path(!nxt_router_msg_from_self(task, msg) + || new_port.pid != msg->port_msg.pid + || msg->port_msg.stream != 0)) + { + return "not a new port of the sender"; + } - msg->port_msg.type = _NXT_PORT_MSG_RPC_ERROR; + return NULL; +} - } else { - if (msg->fd[1] != -1) { - res = nxt_router_port_queue_map(task, port, msg->fd[1]); - if (nxt_slow_path(res != NXT_OK)) { - return; - } - nxt_fd_close(msg->fd[1]); - msg->fd[1] = -1; - } +/* + * libunit asks for the port of a router engine that sent it a request + * (nxt_unit_get_port()). The router does not give out other ports. + */ + +static const char * +nxt_router_get_port_check(nxt_task_t *task, nxt_port_recv_msg_t *msg) +{ + nxt_port_t *port; + nxt_span_t span; + nxt_port_msg_get_port_t get_port; + + if (nxt_slow_path(msg->buf == NULL)) { + return "short get port message"; } - if (msg->port_msg.stream != 0) { - nxt_port_rpc_handler(task, msg); - return; + nxt_span_init(&span, msg->buf->mem.pos, msg->buf->mem.free); + + if (nxt_slow_path(nxt_span_copy(&span, &get_port, sizeof(get_port)) != 0)) + { + return "short get port message"; } - nxt_debug(task, "new port id %d (%d)", port->id, port->type); + if (get_port.pid != nxt_pid) { + return "not a router port"; + } - /* - * Port with "id == 0" is application 'main' port and it always - * should come with non-zero stream. - */ - nxt_assert(port->id != 0); + port = nxt_runtime_port_find(task->thread->runtime, get_port.pid, + get_port.id); - /* Find 'main' app port and get app reference. */ - rt = task->thread->runtime; + if (port != NULL && port->type != NXT_PROCESS_ROUTER) { + return "not a router port"; + } - /* - * It is safe to access 'runtime->ports' hash because 'NEW_PORT' - * sent to main port (with id == 0) and processed in main thread. - */ - main_app_port = nxt_port_hash_find(&rt->ports, port->pid, 0); - nxt_assert(main_app_port != NULL); + return NULL; +} - app = main_app_port->app; - if (nxt_fast_path(app != NULL)) { - nxt_thread_mutex_lock(&app->mutex); +/* The reply goes to a port that the header names. */ - /* TODO here should be find-and-add code because there can be - port waiters in port_hash */ - nxt_port_hash_add(&app->port_hash, port); - app->port_hash_count++; +static const char * +nxt_router_get_mmap_check(nxt_task_t *task, nxt_port_recv_msg_t *msg) +{ + nxt_port_t *port; - nxt_thread_mutex_unlock(&app->mutex); + port = nxt_runtime_port_find(task->thread->runtime, msg->port_msg.pid, + msg->port_msg.reply_port); - port->app = app; + if (port != NULL && port->type != NXT_PROCESS_APP) { + return "reply port is not an application port"; } - port->main_app_port = main_app_port; + return NULL; +} - nxt_port_socket_write(task, port, NXT_PORT_MSG_PORT_ACK, -1, 0, 0, NULL); + +static void +nxt_router_gate_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) +{ + if (nxt_fast_path(nxt_router_msg_sender_ok(task, msg))) { + nxt_router_gates[msg->port_msg.type].handler(task, msg); + } +} + + +#if (NXT_TESTS) + +/* For src/test/nxt_router_sender_test.c. */ + +nxt_uint_t nxt_router_test_senders_refused; + + +nxt_bool_t +nxt_router_test_msg_sender_ok(nxt_task_t *task, nxt_port_recv_msg_t *msg) +{ + return nxt_router_msg_sender_ok(task, msg); } +#endif + static void nxt_router_conf_data_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) @@ -776,6 +1936,7 @@ nxt_router_conf_data_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) msg->port_msg.reply_port); if (nxt_slow_path(port == NULL)) { nxt_alert(task, "conf_data_handler: reply port not found"); + nxt_port_recv_msg_close_fds(msg); return; } @@ -848,10 +2009,12 @@ nxt_router_conf_data_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) nxt_mem_munmap(p, size); } - if (msg->fd[0] != -1) { - nxt_fd_close(msg->fd[0]); - msg->fd[0] = -1; - } + /* + * Close both descriptors: the configuration arrives on fd[0], but a + * compromised sender can attach a second one to any message and would + * otherwise leak a descriptor of the router on every forged message. + */ + nxt_port_recv_msg_close_fds(msg); } @@ -1001,6 +2164,19 @@ nxt_router_status_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) } nxt_queue_loop; +#if (NXT_HAVE_OTEL) + /* + * Span export health. The counters live in the Rust exporter wrapper, + * which runs in this process, so the router is the only place they can be + * read from. A build without OTel, or a configuration with no + * "settings/telemetry", leaves the memzero'd zeros in place and + * nxt_status_get() then omits the "telemetry" object entirely. + */ + report->otel_configured = nxt_otel_rs_export_stats( + &report->otel_spans_exported, + &report->otel_spans_failed); +#endif + report->apps_count = 0; app_stat = report->apps; p = b->mem.end; @@ -1016,7 +2192,9 @@ nxt_router_status_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) app_stat->active_requests = app->active_requests; app_stat->pending_processes = app->pending_processes; app_stat->processes = app->processes; + app_stat->unaccounted_processes = app->unaccounted_processes; app_stat->idle_processes = app->idle_processes; + app_stat->detached_processes = app->detached_processes; report->apps_count++; app_stat++; @@ -1026,7 +2204,20 @@ nxt_router_status_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) fail: - nxt_port_socket_write(task, port, type, -1, msg->port_msg.stream, 0, b); + if (nxt_slow_path(nxt_port_socket_write(task, port, type, -1, + msg->port_msg.stream, 0, b) + != NXT_OK) + && b != NULL) + { + /* + * Still ours: the port layer takes the buffer only on NXT_OK, and + * this one lives in the reply port's pool, so it would sit there + * until the controller's port is released. + */ + + nxt_work_queue_add(&task->thread->engine->fast_work_queue, + b->completion_handler, task, b, b->parent); + } } @@ -1157,13 +2348,24 @@ nxt_router_temp_conf(nxt_task_t *task) } -nxt_inline nxt_bool_t -nxt_router_app_can_start(nxt_app_t *app) +#if (NXT_TESTS) + +/* + * src/test/nxt_router_start_timeout_test.c drives the config-apply start + * against a real temporary configuration, because that is the one whose + * failure path releases the pool the START_PROCESS payload lives in. Building + * one by hand would not do: nxt_router_temp_conf() is also what initialises the + * socket queues nxt_router_conf_error() walks. + */ + +nxt_router_temp_conf_t * +nxt_router_test_temp_conf(nxt_task_t *task) { - return app->processes + app->pending_processes < app->max_processes - && app->pending_processes < app->max_pending_processes; + return nxt_router_temp_conf(task); } +#endif + nxt_inline nxt_bool_t nxt_router_app_need_start(nxt_app_t *app) @@ -1314,6 +2516,26 @@ nxt_router_conf_wait(nxt_task_t *task, void *obj, void *data) } +/* + * Hand a joint job's temporary configuration reference back to the + * configuration thread. Resetting work.next is not optional: the job was + * chained on recf->jobs when it was built, and posting a work item that still + * points at its old neighbour silently splices that neighbour into the target + * engine's locked queue. The job must not be touched after the post -- the + * configuration thread may run nxt_router_conf_wait() and release the pool the + * job lives in at any point from here on. + */ + +nxt_inline void +nxt_router_conf_wait_post(nxt_joint_job_t *job) +{ + job->work.next = NULL; + job->work.handler = nxt_router_conf_wait; + + nxt_event_engine_post(job->tmcf->engine, &job->work); +} + + static void nxt_router_conf_ready(nxt_task_t *task, nxt_router_temp_conf_t *tmcf) { @@ -1398,6 +2620,17 @@ nxt_router_conf_error(nxt_task_t *task, nxt_router_temp_conf_t *tmcf) nxt_router_access_log_release(task, &router->lock, rtcf->access_log); +#if (NXT_TLS) + /* + * The new socket confs go with rtcf->mem_pool, but the TLS contexts + * built for them so far are OpenSSL allocations. No connection uses + * them: the listen joints are posted to the engines only on success. + * The confs in keeping_sockets are the old ones and stay in use. + */ + nxt_router_conf_error_tls(task, &creating_sockets); + nxt_router_conf_error_tls(task, &updating_sockets); +#endif + nxt_mp_destroy(rtcf->mem_pool); nxt_router_conf_send(task, tmcf, NXT_PORT_MSG_RPC_ERROR); @@ -1406,6 +2639,34 @@ nxt_router_conf_error(nxt_task_t *task, nxt_router_temp_conf_t *tmcf) } +#if (NXT_TLS) + +static void +nxt_router_conf_error_tls(nxt_task_t *task, nxt_queue_t *sockets) +{ + nxt_queue_link_t *qlk; + nxt_socket_conf_t *skcf; + + for (qlk = nxt_queue_first(sockets); + qlk != nxt_queue_tail(sockets); + qlk = nxt_queue_next(qlk)) + { + skcf = nxt_queue_link_data(qlk, nxt_socket_conf_t, link); + + /* + * A conf without a bundle failed before its first context. A + * bundle whose server_init() failed has a NULL context, and + * SSL_CTX_free(NULL) does nothing. + */ + if (skcf->tls != NULL && skcf->tls->bundle != NULL) { + task->thread->runtime->tls->server_free(task, skcf->tls); + } + } +} + +#endif + + static void nxt_router_conf_send(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, nxt_port_msg_type_t type) @@ -1466,6 +2727,12 @@ static nxt_conf_map_t nxt_router_app_limits_conf[] = { NXT_CONF_MAP_MSEC, offsetof(nxt_router_app_conf_t, timeout), }, + + { + nxt_string("start_timeout"), + NXT_CONF_MAP_MSEC, + offsetof(nxt_router_app_conf_t, start_timeout), + }, }; @@ -1566,6 +2833,18 @@ static nxt_conf_map_t nxt_router_http_conf[] = { offsetof(nxt_socket_conf_t, send_timeout), }, + { + nxt_string("body_min_rate"), + NXT_CONF_MAP_INT32, + offsetof(nxt_socket_conf_t, body_min_rate), + }, + + { + nxt_string("send_min_rate"), + NXT_CONF_MAP_INT32, + offsetof(nxt_socket_conf_t, send_min_rate), + }, + { nxt_string("body_temp_path"), NXT_CONF_MAP_STR, @@ -1620,6 +2899,104 @@ static nxt_conf_map_t nxt_router_websocket_conf[] = { }; +#if (NXT_HAVE_OTEL) + +/* + * The telemetry settings currently applied to the Rust exporter. + * + * nxt_otel_rs_init() flushes the live provider and builds a new one, which + * blocks the router's control thread for as long as the export of the pending + * batch takes -- up to the exporter's 10s timeout against an unreachable + * collector -- and drops whatever is in flight across the swap. Every config + * apply used to pay that, including applies that do not touch + * /settings/telemetry at all (adding a route, changing an app). Caching what + * was applied lets an unchanged telemetry section be a no-op. + * + * Only exact equality is treated as unchanged; anything the comparison cannot + * establish falls through to the rebuild, which is the old behaviour. + */ +typedef struct { + nxt_bool_t configured; + nxt_str_t endpoint; + nxt_str_t protocol; + double sample_fraction; + double batch_size; +} nxt_otel_applied_conf_t; + + +static nxt_otel_applied_conf_t nxt_otel_applied_conf; + + +static nxt_bool_t +nxt_router_otel_conf_unchanged(nxt_str_t *endpoint, nxt_str_t *protocol, + double sample_fraction, double batch_size) +{ + /* + * The doubles are re-parsed from the same JSON text on every apply, so an + * unchanged section reproduces them bit for bit; an exact comparison is + * therefore the right test here, and any inequality only costs a rebuild. + */ + return nxt_otel_applied_conf.configured + && nxt_strstr_eq(&nxt_otel_applied_conf.endpoint, endpoint) + && nxt_strstr_eq(&nxt_otel_applied_conf.protocol, protocol) + && nxt_otel_applied_conf.sample_fraction == sample_fraction + && nxt_otel_applied_conf.batch_size == batch_size; +} + + +static nxt_int_t +nxt_router_otel_conf_store_str(nxt_str_t *dst, const nxt_str_t *src) +{ + u_char *p; + + p = NULL; + + if (src->length != 0) { + p = nxt_malloc(src->length); + if (nxt_slow_path(p == NULL)) { + return NXT_ERROR; + } + + memcpy(p, src->start, src->length); + } + + nxt_free(dst->start); + + dst->start = p; + dst->length = src->length; + + return NXT_OK; +} + + +/* + * Remember what was just handed to nxt_otel_rs_init(). The strings point into + * the configuration memory pool, which is released once this apply completes, + * so they have to be copied. On any failure the cache is invalidated, which + * only means the next apply rebuilds unconditionally. + */ +static void +nxt_router_otel_conf_remember(nxt_str_t *endpoint, nxt_str_t *protocol, + double sample_fraction, double batch_size) +{ + if (nxt_slow_path(nxt_router_otel_conf_store_str( + &nxt_otel_applied_conf.endpoint, endpoint) != NXT_OK + || nxt_router_otel_conf_store_str( + &nxt_otel_applied_conf.protocol, + protocol) != NXT_OK)) + { + nxt_otel_applied_conf.configured = 0; + return; + } + + nxt_otel_applied_conf.sample_fraction = sample_fraction; + nxt_otel_applied_conf.batch_size = batch_size; + nxt_otel_applied_conf.configured = 1; +} + +#endif + + static nxt_int_t nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, u_char *start, u_char *end) @@ -1650,6 +3027,7 @@ nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, *otel_batching, *otel_proto; #endif nxt_conf_value_t *root, *conf, *http, *value, *websocket; + nxt_conf_value_t *comp; nxt_conf_value_t *applications, *application, *settings; nxt_conf_value_t *listeners, *listener; nxt_socket_conf_t *skcf; @@ -1791,6 +3169,7 @@ nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, apcf.max_processes = 1; apcf.spare_processes = 0; apcf.timeout = 0; + apcf.start_timeout = NXT_APP_START_TIMEOUT; apcf.idle_timeout = 15000; apcf.limits_value = NULL; apcf.processes_value = NULL; @@ -1900,6 +3279,7 @@ nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, app->max_pending_processes = apcf.spare_processes ? apcf.spare_processes : 1; app->timeout = apcf.timeout; + app->start_timeout = apcf.start_timeout; app->idle_timeout = apcf.idle_timeout; app->targets = targets; @@ -1990,6 +3370,18 @@ nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, websocket = nxt_conf_get_path(root, &websocket_path); + /* + * Compression is configured once for the whole router, not per listener. + * The return is deliberately not checked: a "compression" block the + * validator accepts but this cannot build -- one with no "compressors", + * which is not a required member -- leaves compression off rather than + * rejecting the configuration, which is what it has always done. + */ + comp = nxt_conf_get_path(root, &compression_path); + if (comp != NULL) { + (void) nxt_http_comp_compression_init(task, rtcf, comp); + } + listeners = nxt_conf_get_path(root, &listeners_path); if (listeners != NULL) { @@ -2037,6 +3429,8 @@ nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, skcf->proxy_timeout = 60 * 1000; skcf->proxy_send_timeout = 30 * 1000; skcf->proxy_read_timeout = 30 * 1000; + skcf->body_min_rate = 0; + skcf->send_min_rate = 0; skcf->server_version = 1; skcf->chunked_transform = 0; @@ -2048,7 +3442,6 @@ nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, nxt_str_null(&skcf->body_temp_path); if (http != NULL) { - nxt_conf_value_t *comp; ret = nxt_conf_map_object(mp, http, nxt_router_http_conf, nxt_nitems(nxt_router_http_conf), @@ -2058,10 +3451,6 @@ nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, goto fail; } - comp = nxt_conf_get_path(root, &compression_path); - if (comp != NULL) { - nxt_http_comp_compression_init(task, rtcf, comp); - } } if (websocket != NULL) { @@ -2225,11 +3614,33 @@ nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, ? nxt_conf_get_number(otel_sampling) : NXT_OTEL_SAMPLING_DEFAULT; - nxt_otel_rs_init(&nxt_otel_log_callback, &telemetry_endpoint, - &telemetry_proto, telemetry_sample_fraction, - telemetry_batching); - } else { + /* + * Rebuild the exporter only when the telemetry settings actually + * differ. nxt_otel_rs_is_init() is required as well so that a + * previous init that failed inside the Rust side (an exporter that + * could not be built) is retried rather than remembered as applied. + */ + if (!nxt_router_otel_conf_unchanged(&telemetry_endpoint, + &telemetry_proto, + telemetry_sample_fraction, + telemetry_batching) + || !nxt_otel_rs_is_init()) + { + nxt_otel_rs_init(&nxt_otel_log_callback, &telemetry_endpoint, + &telemetry_proto, telemetry_sample_fraction, + telemetry_batching); + + nxt_router_otel_conf_remember(&telemetry_endpoint, &telemetry_proto, + telemetry_sample_fraction, + telemetry_batching); + } + + } else if (nxt_otel_applied_conf.configured || nxt_otel_rs_is_init()) { nxt_otel_rs_uninit(); + + nxt_free(nxt_otel_applied_conf.endpoint.start); + nxt_free(nxt_otel_applied_conf.protocol.start); + nxt_memzero(&nxt_otel_applied_conf, sizeof(nxt_otel_applied_conf)); } #endif @@ -2310,6 +3721,7 @@ nxt_router_js_module_rpc_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, ret = nxt_script_file_read(msg->fd[0], &text); nxt_fd_close(msg->fd[0]); + msg->fd[0] = -1; if (nxt_slow_path(ret == NXT_ERROR)) { goto fail; @@ -2615,9 +4027,8 @@ nxt_router_port_queue_map(nxt_task_t *task, nxt_port_t *port, nxt_fd_t fd) nxt_assert(fd != -1); - mem = nxt_mem_mmap(NULL, sizeof(nxt_port_queue_t), - PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); - if (nxt_slow_path(mem == MAP_FAILED)) { + mem = nxt_port_queue_mmap(task, fd, sizeof(nxt_port_queue_t)); + if (nxt_slow_path(mem == NULL)) { return NXT_ERROR; } @@ -2667,7 +4078,7 @@ nxt_router_apps_hash_add(nxt_router_conf_t *rtcf, nxt_app_t *app) case NXT_DECLINED: nxt_thread_log_alert("router app hash adding failed: " "\"%V\" is already in hash", &lhq.key); - /* Fall through. */ + nxt_fallthrough; default: return NXT_ERROR; } @@ -2942,6 +4353,9 @@ nxt_router_listen_socket_ready(nxt_task_t *task, nxt_port_recv_msg_t *msg, s = msg->fd[0]; + /* The listener owns the descriptor now. */ + msg->fd[0] = -1; + ret = nxt_socket_nonblocking(task, s); if (nxt_slow_path(ret != NXT_OK)) { goto fail; @@ -3069,7 +4483,12 @@ nxt_router_tls_rpc_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, tls->tls_init->conf = tlscf; - bundle = nxt_mp_get(mp, sizeof(nxt_tls_bundle_conf_t)); + /* + * The error path frees bundle->ctx of every bundle in the chain, so + * the bundle must not hold a garbage pointer before server_init() + * sets it. + */ + bundle = nxt_mp_zget(mp, sizeof(nxt_tls_bundle_conf_t)); if (nxt_slow_path(bundle == NULL)) { goto fail; } @@ -3078,7 +4497,13 @@ nxt_router_tls_rpc_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, goto fail; } + /* + * The bundle owns the descriptor now. server_init() closes it on + * every path and sets bundle->chain_file to -1. + */ bundle->chain_file = msg->fd[0]; + msg->fd[0] = -1; + bundle->next = tlscf->bundle; tlscf->bundle = bundle; @@ -3104,17 +4529,20 @@ static void nxt_router_app_rpc_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, nxt_app_t *app) { - size_t size; - uint32_t stream; - nxt_fd_t port_fd, queue_fd; - nxt_int_t ret; - nxt_buf_t *b; - nxt_port_t *router_port, *dport; - nxt_runtime_t *rt; - nxt_app_rpc_t *rpc; + size_t size; + uint32_t stream; + nxt_fd_t port_fd, queue_fd; + nxt_int_t ret; + nxt_buf_t *b; + nxt_port_t *router_port, *dport; + nxt_runtime_t *rt; + nxt_app_rpc_t *rpc; + nxt_router_start_timer_t *st; rt = task->thread->runtime; + st = NULL; + dport = app->proto_port; if (dport == NULL) { @@ -3162,6 +4590,17 @@ nxt_router_app_rpc_create(nxt_task_t *task, stream = nxt_port_rpc_ex_stream(rpc); + /* + * Before the write: the deadline resolves through the payload's completion + * handler, which nxt_port_socket_write2() may run before it returns. + */ + + st = nxt_router_start_timer_create(task, app, router_port, dport, b); + + if (st != NULL && b != NULL) { + b->completion_handler = nxt_router_start_conf_buf_completion; + } + ret = nxt_port_socket_write2(task, dport, NXT_PORT_MSG_START_PROCESS, port_fd, queue_fd, stream, router_port->id, b); if (nxt_slow_path(ret != NXT_OK)) { @@ -3175,14 +4614,58 @@ nxt_router_app_rpc_create(nxt_task_t *task, app->pending_processes++; } + rpc->start_timer = st; + + if (st != NULL) { + nxt_router_start_timer_arm(task, st, stream); + } + return; fail: + if (st != NULL) { + /* + * Never armed, and the payload dies with tmcf->mem_pool below without + * its completion handler ever running, so both references go here. + */ + + if (b != NULL) { + nxt_router_start_timer_use(task, st, -1); + } + + nxt_router_start_timer_use(task, st, -1); + } + nxt_router_conf_error(task, tmcf); } +#if (NXT_TESTS) + +void +nxt_router_test_app_rpc_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, + nxt_app_t *app) +{ + nxt_router_app_rpc_create(task, tmcf, app); +} + + +/* + * The request deadline, for src/test/nxt_router_app_timeout_test.c. That test + * drives it through the engine's timer machinery, so the wrapper is installed + * as the timer handler rather than called. + */ + +void +nxt_router_test_app_timeout(nxt_task_t *task, void *obj, void *data) +{ + nxt_router_app_timeout(task, obj, data); +} + +#endif + + static void nxt_router_app_prefork_ready(nxt_task_t *task, nxt_port_recv_msg_t *msg, void *data) @@ -3195,6 +4678,12 @@ nxt_router_app_prefork_ready(nxt_task_t *task, nxt_port_recv_msg_t *msg, rpc = data; app = rpc->app; + if (rpc->start_timer != NULL) { + nxt_router_start_timer_cancel(task, rpc->start_timer); + + rpc->start_timer = NULL; + } + port = msg->u.new_port; nxt_assert(port != NULL); @@ -3257,6 +4746,12 @@ nxt_router_app_prefork_error(nxt_task_t *task, nxt_port_recv_msg_t *msg, app = rpc->app; tmcf = rpc->temp_conf; + if (rpc->start_timer != NULL) { + nxt_router_start_timer_cancel(task, rpc->start_timer); + + rpc->start_timer = NULL; + } + if (rpc->proto) { nxt_log(task, NXT_LOG_WARN, "failed to start prototype \"%V\"", &app->name); @@ -3459,6 +4954,7 @@ nxt_router_engine_joints_create(nxt_router_temp_conf_t *tmcf, } joint->count = 1; + joint->stale = 0; skcf = nxt_queue_link_data(qlk, nxt_socket_conf_t, link); skcf->count++; @@ -3488,9 +4984,15 @@ nxt_router_engine_quit(nxt_router_temp_conf_t *tmcf, job->task = tmcf->engine->task; job->work.handler = nxt_router_worker_thread_quit; job->work.task = &job->task; - job->work.obj = NULL; + job->work.obj = job; job->work.data = NULL; - job->tmcf = NULL; + job->tmcf = tmcf; + + /* + * The job outlives this call on the target engine's locked queue, so hold + * the pool it lives in, as the joints create/delete jobs do. + */ + tmcf->count++; return NXT_OK; } @@ -3682,9 +5184,6 @@ nxt_router_thread_start(void *data) engine->task.log = thread->log; thread->engine = engine; thread->task = &engine->task; -#if 0 - thread->fiber = &engine->fibers->fiber; -#endif engine->mem_pool = nxt_mp_create(4096, 128, 1024, 64); if (nxt_slow_path(engine->mem_pool == NULL)) { @@ -3767,24 +5266,46 @@ nxt_router_listen_socket_create(nxt_task_t *task, void *obj, void *data) skcf = joint->socket_conf; ls = skcf->listen; - lev = nxt_listen_event(task, ls); - if (nxt_slow_path(lev == NULL)) { - nxt_router_listen_socket_release(task, skcf); - return; - } - - lev->socket.data = joint; - + /* + * This engine holds one reference on the listen socket, also when the + * listen event cannot be allocated below. The last engine that + * releases its reference closes the socket, see + * nxt_router_listen_socket_release(). + */ lock = &skcf->router_conf->router->lock; nxt_thread_spin_lock(lock); ls->count++; nxt_thread_spin_unlock(lock); - job->work.next = NULL; - job->work.handler = nxt_router_conf_wait; + lev = nxt_listen_event(task, ls); + if (nxt_slow_path(lev == NULL)) { + /* + * Only the lev allocation itself can fail here: a missing spare + * conn keeps the listener and is retried by its own timer + * (nxt_listen_event()). Acknowledge the job so the configuration + * request completes instead of waiting forever. The joint and + * the listen socket reference stay: releasing the joint here could + * drop the last reference to the configuration that is being + * applied. A later update or delete of this listener releases + * both, see nxt_router_listen_socket_release_stale(). The release + * is deferred, not skipped: until that job, the stale joint keeps + * this configuration (its memory pool and TLS contexts) and the + * listen socket, so the address stays bound. + */ + nxt_alert(task, "engine %p: listen socket %d: no memory for the " + "listen event, no connections will be accepted on it", + task->thread->engine, ls->socket); - nxt_event_engine_post(job->tmcf->engine, &job->work); + joint->stale = 1; + + nxt_router_conf_wait_post(job); + return; + } + + lev->socket.data = joint; + + nxt_router_conf_wait_post(job); } @@ -3831,14 +5352,35 @@ nxt_router_listen_socket_update(nxt_task_t *task, void *obj, void *data) lev = nxt_router_listen_event(&engine->listen_connections, joint->socket_conf); + if (nxt_slow_path(lev == NULL)) { + /* + * The previous configuration could not allocate the listen event + * on this engine (nxt_router_listen_socket_create()), so there is + * nothing to update: create it now instead. The create path + * links the joint itself, so unlink it again first. + */ + nxt_queue_remove(&joint->link); + + nxt_router_listen_socket_create(task, obj, data); + + /* + * The job is posted: its task and data can be freed now, so the + * engine task is used below. An update keeps the listen socket, + * so the stale joint of the failed create has the same one. The + * create above took a new reference on it, so this release does + * not close it. + */ + nxt_router_listen_socket_release_stale(&engine->task, engine, + joint->socket_conf->listen, + joint); + return; + } + old = lev->socket.data; lev->socket.data = joint; lev->listen = joint->socket_conf->listen; - job->work.next = NULL; - job->work.handler = nxt_router_conf_wait; - - nxt_event_engine_post(job->tmcf->engine, &job->work); + nxt_router_conf_wait_post(job); /* * The task is allocated from configuration temporary @@ -3863,6 +5405,30 @@ nxt_router_listen_socket_delete(nxt_task_t *task, void *obj, void *data) lev = nxt_router_listen_event(&engine->listen_connections, skcf); + if (nxt_slow_path(lev == NULL)) { + /* + * The listen event was never allocated on this engine + * (nxt_router_listen_socket_create()), so there is no listener to + * close. Release the stale joint and its listen socket reference + * first: if this was the last reference, the socket is closed, and + * the acknowledgement then means that the port is free, as after + * nxt_router_listen_socket_close_finish(). The listen socket is + * matched, not skcf: when an engine is removed + * (nxt_router_engine_conf_delete()), skcf is the socket conf of + * the new configuration, and the stale joint has the old one. + */ + nxt_router_listen_socket_release_stale(&engine->task, engine, + skcf->listen, NULL); + + nxt_router_conf_wait_post(obj); + + if (engine->shutdown && nxt_queue_is_empty(&engine->joints)) { + nxt_router_worker_thread_exit(&engine->task); + } + + return; + } + nxt_fd_event_delete(engine, &lev->socket); nxt_debug(task, "engine %p: listen socket delete: %d", engine, @@ -3878,19 +5444,108 @@ nxt_router_listen_socket_delete(nxt_task_t *task, void *obj, void *data) } +/* + * Releases the joints that a failed nxt_router_listen_socket_create() left + * in engine->joints for the listen socket ls, and the listen socket + * reference that each of them holds. Such a joint has the stale flag. + * Only the flag is trusted, not ls alone: a request can hold the joint of + * a closed listener in engine->joints, and a new listen socket can have + * the address of the freed one. "keep" is the joint of the current job. + * Each update or delete of ls releases its stale joints, so an engine has + * at most one of them for ls when this runs; the loop does not depend on + * that. The order is the same as in nxt_router_listen_socket_close_finish(): + * first the listen socket, which the last reference closes, then the + * joint. + */ + +static void +nxt_router_listen_socket_release_stale(nxt_task_t *task, + nxt_event_engine_t *engine, nxt_listen_socket_t *ls, + nxt_socket_conf_joint_t *keep) +{ + nxt_queue_link_t *qlk, *next; + nxt_socket_conf_joint_t *joint; + + for (qlk = nxt_queue_first(&engine->joints); + qlk != nxt_queue_tail(&engine->joints); + qlk = next) + { + next = nxt_queue_next(qlk); + + joint = nxt_queue_link_data(qlk, nxt_socket_conf_joint_t, link); + + if (joint == keep) { + continue; + } + + if (joint->stale && joint->socket_conf->listen == ls) { + nxt_debug(task, "engine %p: release stale joint %p", engine, + joint); + + nxt_router_listen_socket_release(task, joint->socket_conf); + nxt_router_conf_release(task, joint); + } + } +} + + +#if (NXT_TESTS) + +/* For src/test/nxt_router_stale_joint_test.c. */ + +void +nxt_router_test_listen_socket_release_stale(nxt_task_t *task, + nxt_event_engine_t *engine, nxt_listen_socket_t *ls, + nxt_socket_conf_joint_t *keep) +{ + nxt_router_listen_socket_release_stale(task, engine, ls, keep); +} + + +/* The listen socket jobs, for src/test/nxt_router_stale_joint_test.c. */ + +void +nxt_router_test_listen_socket_create(nxt_task_t *task, nxt_joint_job_t *job, + nxt_socket_conf_joint_t *joint) +{ + nxt_router_listen_socket_create(task, job, joint); +} + + +void +nxt_router_test_listen_socket_update(nxt_task_t *task, nxt_joint_job_t *job, + nxt_socket_conf_joint_t *joint) +{ + nxt_router_listen_socket_update(task, job, joint); +} + +#endif + + static void nxt_router_worker_thread_quit(nxt_task_t *task, void *obj, void *data) { + nxt_joint_job_t *job; nxt_event_engine_t *engine; nxt_debug(task, "router worker thread quit"); + job = obj; + engine = task->thread->engine; engine->shutdown = 1; + /* + * Give the reference nxt_router_engine_quit() took back. The task this + * handler was called with is allocated from the same pool as the job, so + * the exit below has to continue on the engine's own task. + */ + + nxt_router_conf_wait_post(job); + if (nxt_queue_is_empty(&engine->joints)) { - nxt_router_worker_thread_exit(task); + nxt_router_worker_thread_exit(&engine->task); } } @@ -3914,10 +5569,13 @@ nxt_router_worker_thread_exit(nxt_task_t *task) * * Accepting -> Draining: nxt_router_listen_socket_close() (phase 1) * disarms accept(2), marks lev->draining = 1. + * If connections are in flight it also posts the + * close_job back to the configuration thread, so + * the control request does not wait for them. * Draining -> Closed: nxt_router_listen_socket_close_finish() (phase 2) * runs once lev->count == 1 (no in-flight accepted - * connections), releases the FD and posts the - * close_job back to the configuration thread. + * connections), releases the FD and then posts the + * close_job if phase 1 did not already. * * The intermediate state lets in-flight TLS handshakes and accepted-but- * not-yet-handled connections complete cleanly instead of being RST when @@ -3925,7 +5583,8 @@ nxt_router_worker_thread_exit(nxt_task_t *task) * pattern in nxt_router_worker_thread_quit()). */ -static void nxt_router_listen_socket_close_ready(nxt_socket_conf_joint_t *joint); +static void nxt_router_listen_socket_close_ready(nxt_task_t *task, + nxt_socket_conf_joint_t *joint); static void nxt_router_listen_socket_close_finish(nxt_task_t *task, nxt_listen_event_t *lev); @@ -3970,31 +5629,36 @@ nxt_router_listen_socket_close(nxt_task_t *task, void *obj, void *data) } } - /* - * Configuration is ready once accept(2) has been disarmed. The - * listener FD and old socket configuration stay alive below until - * accepted connections release their listener refs, but the control - * request must not wait for idle client timing. - */ - nxt_router_listen_socket_close_ready(joint); - /* * Phase 2 gating: wait for accepted connections to release their * refs. lev->count == 1 means only the original listener ref * remains (see nxt_listen_event() and nxt_conn_accept()). * - * Known limitation: the listening FD stays bound until the drain - * completes (nxt_router_listen_socket_release() closes it on the - * last engine's phase 2), so a configuration that re-adds the same - * address while the old listener drains fails with EADDRINUSE at - * bind time in the main process -- a visible, retryable config - * error. Closing the FD in phase 1 instead requires reworking the - * ls->count contract across engines and the nxt_process_quit() - * listen-queue walk; that belongs to the full-drain follow-up. + * When connections are still in flight the configuration is + * acknowledged here, before the FD is released: the control request + * must not wait for idle client timing. The listening FD then stays + * bound until the drain completes + * (nxt_router_listen_socket_release() closes it on the last engine's + * phase 2), so a configuration that re-adds the same address while + * the old listener drains can fail with EADDRINUSE at bind time in + * the main process -- a visible, retryable config error. Closing + * the FD in phase 1 instead requires reworking the ls->count + * contract across engines and the nxt_process_quit() listen-queue + * walk; that belongs to the full-drain follow-up. + * + * With nothing in flight there is no drain to wait for, so the reply + * is deferred to nxt_router_listen_socket_close_finish() below, + * which sends it after the descriptor is closed. That keeps the + * pre-drain contract -- a successful reconfiguration means the port + * is free -- for every listener that has no accepted connection on + * it. */ if (lev->count > 1) { nxt_debug(task, "engine %p: listen socket %d drain pending, " "in-flight: %D", engine, lev->socket.fd, lev->count - 1); + + nxt_router_listen_socket_close_ready(task, joint); + return; } @@ -4003,7 +5667,8 @@ nxt_router_listen_socket_close(nxt_task_t *task, void *obj, void *data) static void -nxt_router_listen_socket_close_ready(nxt_socket_conf_joint_t *joint) +nxt_router_listen_socket_close_ready(nxt_task_t *task, + nxt_socket_conf_joint_t *joint) { nxt_joint_job_t *job; @@ -4014,10 +5679,22 @@ nxt_router_listen_socket_close_ready(nxt_socket_conf_joint_t *joint) joint->close_job = NULL; - job->work.next = NULL; - job->work.handler = nxt_router_conf_wait; + /* + * The only record of the acknowledgement on the thread that issues + * it. The configuration thread's "temp conf ... count" line is + * written after a cross-thread post, so where it lands in the log is + * a function of that thread's wake-up latency, not of where in the + * close path the acknowledgement was issued -- it stays in the same + * place whether this call is made before the descriptor is released + * or after it. This line does not: it is written by the closing + * engine itself, between its own records, so the order of the close + * and the acknowledgement is readable from the log. + * test_listeners_close_before_reply asserts exactly that. + */ + nxt_debug(task, "engine %p: listen socket close acknowledged", + task->thread->engine); - nxt_event_engine_post(job->tmcf->engine, &job->work); + nxt_router_conf_wait_post(job); } @@ -4035,13 +5712,33 @@ nxt_router_listen_socket_close_finish(nxt_task_t *task, joint = lev->socket.data; lev->socket.data = NULL; - nxt_router_listen_socket_close_ready(joint); - /* 'task' refers to lev->task and we cannot use after nxt_free() */ task = &task->thread->engine->task; nxt_router_listen_socket_release(task, joint->socket_conf); + /* + * Acknowledge only now. nxt_router_listen_socket_release() above + * drops this engine's reference to the listen socket, and on the + * last engine to reach phase 2 it performs the single + * nxt_socket_close() of the descriptor. The configuration is not + * reported successful until every engine has acknowledged + * (nxt_router_conf_ready()), so acknowledging after the release -- + * rather than before it, as this function used to -- puts the reply + * strictly after the close for a listener with nothing in flight. + * + * A listener that did have connections to drain acknowledged early + * in phase 1 and left joint->close_job NULL, so this is the no-op it + * has always been on that path, and the still-bound descriptor + * documented there remains its known limitation. + * + * The joint outlives the call: its listener reference is dropped by + * nxt_router_listen_event_release() below, and + * nxt_router_listen_socket_release() frees the nxt_listen_socket_t, + * not the socket configuration the joint points at. + */ + nxt_router_listen_socket_close_ready(task, joint); + nxt_router_listen_event_release(task, lev, joint); } @@ -4099,8 +5796,15 @@ nxt_router_listen_socket_release(nxt_task_t *task, nxt_socket_conf_t *skcf) rt = task->thread->runtime; main_port = rt->port_by_type[NXT_PROCESS_MAIN]; - (void) nxt_port_socket_write(task, main_port, NXT_PORT_MSG_SOCKET_UNLINK, - -1, 0, 0, b); + if (nxt_slow_path(nxt_port_socket_write(task, main_port, + NXT_PORT_MSG_SOCKET_UNLINK, + -1, 0, 0, b) != NXT_OK)) + { + /* Still ours: the port layer takes the buffer only on NXT_OK. */ + + nxt_work_queue_add(&task->thread->engine->fast_work_queue, + b->completion_handler, task, b, b->parent); + } out_free_ls: #endif @@ -4202,79 +5906,239 @@ nxt_router_conf_release(nxt_task_t *task, nxt_socket_conf_joint_t *joint) if (skcf != NULL && skcf->tls != NULL) { task->thread->runtime->tls->server_free(task, skcf->tls); } -#endif +#endif + + /* TODO remove engine->port */ + + if (rtcf != NULL) { + nxt_debug(task, "old router conf is destroyed"); + + nxt_router_apps_hash_use(task, rtcf, -1); + + nxt_router_access_log_release(task, lock, rtcf->access_log); + + nxt_tstr_state_release(rtcf->tstr_state); + + nxt_mp_thread_adopt(rtcf->mem_pool); + + nxt_mp_destroy(rtcf->mem_pool); + } +} + + +static void +nxt_router_thread_exit_handler(nxt_task_t *task, void *obj, void *data) +{ + nxt_port_t *port; + nxt_thread_link_t *link; + nxt_event_engine_t *engine; + nxt_thread_handle_t handle; + + handle = (nxt_thread_handle_t) (uintptr_t) obj; + link = data; + + nxt_thread_wait(handle); + + engine = link->engine; + + nxt_queue_remove(&engine->link); + + port = engine->port; + + // TODO notify all apps + + port->engine = task->thread->engine; + nxt_mp_thread_adopt(port->mem_pool); + + /* + * The worker thread is gone, so nothing reads port->pair[0] any more: + * close the socket pair and the queue memfd, and drop the reference + * nxt_router_rt_add_port() took when it published the port in + * rt->ports. Without this every engine removed by a listen_threads + * decrease leaks 3 descriptors (2 socketpair + 1 memfd) for the + * router's lifetime. + */ + nxt_port_close(task, port); + nxt_runtime_port_remove(task, port); + + nxt_port_use(task, port, -1); + + nxt_mp_thread_adopt(engine->mem_pool); + nxt_mp_destroy(engine->mem_pool); + + nxt_event_engine_free(engine); + + nxt_free(link); +} + + +/* + * Decodes the nxt_unit_response_t that an application put into "b". The + * buffer is shared memory the application can still write, so every value + * is read once, through volatile, into a local and checked before use: the + * field count against the buffer, each sptr with nxt_unit_sptr_in_buf(). + * It is a separate function so that a test and the fuzz target + * (fuzzing/nxt_router_app_response_fuzz.c) can call it with a bare request. + */ + +static nxt_int_t +nxt_router_response_header_parse(nxt_task_t *task, nxt_http_request_t *r, + nxt_buf_t *b) +{ + size_t b_size, count; + u_char *p; + uint32_t piggyback_length; + nxt_int_t ret; + nxt_unit_field_t *f, uf; + nxt_http_field_t *field; + nxt_unit_response_t *resp; + + b_size = nxt_buf_is_mem(b) ? nxt_buf_mem_used_size(&b->mem) : 0; + + if (nxt_slow_path(b_size < sizeof(nxt_unit_response_t) + || b_size > UINT32_MAX)) + { + nxt_alert(task, "response buffer too small: %z", b_size); + return NXT_ERROR; + } + + resp = (void *) b->mem.pos; + count = *(volatile uint32_t *) &resp->fields_count; + + if (nxt_slow_path(count > (b_size - sizeof(nxt_unit_response_t)) + / sizeof(nxt_unit_field_t))) + { + nxt_alert(task, "response buffer too small for fields count: %uz", + count); + return NXT_ERROR; + } - /* TODO remove engine->port */ + for (f = resp->fields; f < resp->fields + count; f++) { + uf = *(volatile nxt_unit_field_t *) f; - if (rtcf != NULL) { - nxt_debug(task, "old router conf is destroyed"); + if (uf.skip) { + continue; + } - nxt_router_apps_hash_use(task, rtcf, -1); + field = nxt_http_resp_field_add(&r->resp, r->mem_pool); - nxt_router_access_log_release(task, lock, rtcf->access_log); + if (nxt_slow_path(field == NULL)) { + return NXT_ERROR; + } - nxt_tstr_state_release(rtcf->tstr_state); + field->hash = uf.hash; + field->skip = 0; + field->hopbyhop = 0; - nxt_mp_thread_adopt(rtcf->mem_pool); + field->name_length = uf.name_length; + field->value_length = uf.value_length; + field->name = nxt_unit_sptr_in_buf(&f->name, uf.name_length, + b->mem.pos, b_size); + field->value = nxt_unit_sptr_in_buf(&f->value, uf.value_length, + b->mem.pos, b_size); - nxt_mp_destroy(rtcf->mem_pool); - } -} + if (nxt_slow_path(field->name == NULL || field->value == NULL)) { + nxt_alert(task, "response field sptr out of bounds"); + return NXT_ERROR; + } + ret = nxt_http_field_process(field, &nxt_response_fields_hash, r); + if (nxt_slow_path(ret != NXT_OK)) { + return NXT_ERROR; + } -static void -nxt_router_thread_exit_handler(nxt_task_t *task, void *obj, void *data) -{ - nxt_port_t *port; - nxt_thread_link_t *link; - nxt_event_engine_t *engine; - nxt_thread_handle_t handle; + nxt_debug(task, "header%s: %*s: %*s", + (field->skip ? " skipped" : ""), + (size_t) field->name_length, field->name, + (size_t) field->value_length, field->value); - handle = (nxt_thread_handle_t) (uintptr_t) obj; - link = data; + if (field->skip) { + if (r->resp.num_inline_fields > 0 + && field == &r->resp.inline_fields[r->resp.num_inline_fields - 1]) + { + r->resp.num_inline_fields--; + } else if (r->resp.fields != NULL && r->resp.fields->last != NULL) { + r->resp.fields->last->nelts--; + } + } + } - nxt_thread_wait(handle); + r->status = *(volatile uint16_t *) &resp->status; - engine = link->engine; + piggyback_length = *(volatile uint32_t *) &resp->piggyback_content_length; - nxt_queue_remove(&engine->link); + if (piggyback_length != 0) { + p = nxt_unit_sptr_in_buf(&resp->piggyback_content, piggyback_length, + b->mem.pos, b_size); + if (nxt_slow_path(p == NULL)) { + nxt_alert(task, "response piggyback content out of bounds"); + return NXT_ERROR; + } - port = engine->port; + b->mem.pos = p; + b->mem.free = p + piggyback_length; - // TODO notify all apps + } else { + b->mem.pos = b->mem.free; + } - port->engine = task->thread->engine; - nxt_mp_thread_adopt(port->mem_pool); - nxt_port_use(task, port, -1); + return NXT_OK; +} - nxt_mp_thread_adopt(engine->mem_pool); - nxt_mp_destroy(engine->mem_pool); - nxt_event_engine_free(engine); +#if (NXT_TESTS) - nxt_free(link); +/* For src/test/nxt_router_response_parse_test.c. */ + +nxt_int_t +nxt_router_test_response_header_parse(nxt_task_t *task, + nxt_http_request_t *r, nxt_buf_t *b) +{ + return nxt_router_response_header_parse(task, r, b); } +#endif + static void nxt_router_response_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, void *data) { - size_t b_size, count; nxt_int_t ret; nxt_app_t *app; nxt_buf_t *b, *next, *out, *last_b, *owned_b; nxt_port_t *app_port; - nxt_unit_field_t *f; - nxt_http_field_t *field; + nxt_http_status_t status; nxt_http_request_t *r; - nxt_unit_response_t *resp; nxt_request_rpc_data_t *req_rpc_data; + /* What "fail:" answers with, unless a branch below knows better. */ + status = NXT_HTTP_SERVICE_UNAVAILABLE; + + /* + * An application response never legitimately carries a descriptor and + * this handler does not read msg->fd. The dispatcher closes what a + * handler leaves behind, so this is belt and braces; it is here because + * an application owns its end of the port socket, which makes this the + * one handler a hostile peer reaches on every single request. + */ + nxt_port_recv_msg_close_fds(msg); + req_rpc_data = data; r = req_rpc_data->request; if (nxt_slow_path(r == NULL)) { + /* + * A request the router gave up on. The answer belongs to nobody and + * needs no reply, but the last message is the worker saying it is + * done: this is where the port it was running on may rejoin the idle + * economy. A message that is not the last one proves nothing yet; + * the response can still be streaming. + */ + if (msg->port_msg.last != 0) { + nxt_router_app_abandoned_settle(task, req_rpc_data); + } + return; } @@ -4313,7 +6177,12 @@ nxt_router_response_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, nxt_request_rpc_data_unlink(task, req_rpc_data); } else { - if (app->timeout != 0) { + /* + * The deadline bounds the worker's answer, and an upgraded + * WebSocket has had its answer: the frames that follow are a + * session, however quiet (#422). + */ + if (app->timeout != 0 && r->state != &nxt_http_websocket) { r->timer.handler = nxt_router_app_timeout; r->timer_data = req_rpc_data; nxt_timer_add(task->thread->engine, &r->timer, app->timeout); @@ -4347,76 +6216,11 @@ nxt_router_response_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, nxt_http_request_send_body(task, r, NULL); } else { - b_size = nxt_buf_is_mem(b) ? nxt_buf_mem_used_size(&b->mem) : 0; - - if (nxt_slow_path(b_size < sizeof(nxt_unit_response_t))) { - nxt_alert(task, "response buffer too small: %z", b_size); - goto fail; - } - - resp = (void *) b->mem.pos; - count = (b_size - sizeof(nxt_unit_response_t)) - / sizeof(nxt_unit_field_t); - - if (nxt_slow_path(count < resp->fields_count)) { - nxt_alert(task, "response buffer too small for fields count: %D", - resp->fields_count); + ret = nxt_router_response_header_parse(task, r, b); + if (nxt_slow_path(ret != NXT_OK)) { goto fail; } - field = NULL; - - for (f = resp->fields; f < resp->fields + resp->fields_count; f++) { - if (f->skip) { - continue; - } - - field = nxt_http_resp_field_add(&r->resp, r->mem_pool); - - if (nxt_slow_path(field == NULL)) { - goto fail; - } - - field->hash = f->hash; - field->skip = 0; - field->hopbyhop = 0; - - field->name_length = f->name_length; - field->value_length = f->value_length; - field->name = nxt_unit_sptr_get(&f->name); - field->value = nxt_unit_sptr_get(&f->value); - - ret = nxt_http_field_process(field, &nxt_response_fields_hash, r); - if (nxt_slow_path(ret != NXT_OK)) { - goto fail; - } - - nxt_debug(task, "header%s: %*s: %*s", - (field->skip ? " skipped" : ""), - (size_t) field->name_length, field->name, - (size_t) field->value_length, field->value); - - if (field->skip) { - if (r->resp.num_inline_fields > 0 - && field == &r->resp.inline_fields[r->resp.num_inline_fields - 1]) - { - r->resp.num_inline_fields--; - } else if (r->resp.fields != NULL && r->resp.fields->last != NULL) { - r->resp.fields->last->nelts--; - } - } - } - - r->status = resp->status; - - if (resp->piggyback_content_length != 0) { - b->mem.pos = nxt_unit_sptr_get(&resp->piggyback_content); - b->mem.free = b->mem.pos + resp->piggyback_content_length; - - } else { - b->mem.pos = b->mem.free; - } - if (nxt_buf_mem_used_size(&b->mem) == 0) { next = b->next; b->next = NULL; @@ -4433,11 +6237,33 @@ nxt_router_response_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, /* * Check compression before handing the chain over to r->out, so that - * the rejection below still owns it and releases it inline. - * nxt_http_comp_check_compression() reads r->resp and the request - * configuration, never r->out, so the order is free. + * the rejection below still owns it and releases it inline. Both + * calls read r->resp and the request configuration, never r->out, so + * the order is free. + * + * An application response is always sent with a body, so the two + * halves run together here. They are separate calls because the + * static path has to put precondition evaluation between them: a 406 + * outranks a precondition, but a 304 must not leave an initialised + * compressor behind (RFC 9110 Sect. 13.2.1). */ - ret = nxt_http_comp_check_compression(task, r); + ret = nxt_http_comp_check_acceptable(task, r); + if (ret != NXT_OK) { + /* + * No acceptable representation is a fault of the request, not of + * the server, and it is the one non-NXT_OK result here that is + * not an error: the release below is still what the chain needs, + * but the answer is 406 rather than the 503 "fail:" gives + * everything else. + */ + if (ret == NXT_HTTP_NOT_ACCEPTABLE) { + status = NXT_HTTP_NOT_ACCEPTABLE; + } + + goto fail; + } + + ret = nxt_http_comp_apply_compression(task, r); if (ret != NXT_OK) { goto fail; } @@ -4465,12 +6291,26 @@ nxt_router_response_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, nxt_thread_mutex_unlock(&app->mutex); nxt_router_app_port_release(task, app, app_port, NXT_APR_UPGRADE); - req_rpc_data->apr_action = NXT_APR_CLOSE; + + /* + * The count above is undone by this action, and only by it: + * nxt_request_rpc_data_unlink() is the one path that runs for a + * websocket that ends, whichever side ends it, and ->app_port is + * still this port when it does. + */ + req_rpc_data->apr_action = NXT_APR_WEBSOCKET_CLOSE; nxt_debug(task, "stream #%uD upgrade", req_rpc_data->stream); r->state = &nxt_http_websocket; + /* + * The 101 arrived as a non-last message, which re-armed the + * request deadline above; the upgrade ends the request, so the + * deadline goes with it (#422). + */ + nxt_timer_disable(task->thread->engine, &r->timer); + } else { r->state = &nxt_http_request_send_state; } @@ -4505,7 +6345,7 @@ nxt_router_response_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, r->last = last_b; } - nxt_http_request_error(task, r, NXT_HTTP_SERVICE_UNAVAILABLE); + nxt_http_request_error(task, r, status); /* * Complete the buffers adopted from the port above, if any: nobody else @@ -4560,10 +6400,9 @@ nxt_router_req_headers_ack_handler(nxt_task_t *task, { int res; nxt_app_t *app; - nxt_buf_t *b; + nxt_buf_t *b, *next; nxt_bool_t start_process, unlinked; - nxt_port_t *app_port, *main_app_port, *idle_port; - nxt_queue_link_t *idle_lnk; + nxt_port_t *app_port, *main_app_port; nxt_http_request_t *r; nxt_debug(task, "stream #%uD: got ack from %PI:%d", @@ -4576,7 +6415,6 @@ nxt_router_req_headers_ack_handler(nxt_task_t *task, app = req_rpc_data->app; r = req_rpc_data->request; - start_process = 0; unlinked = 0; nxt_thread_mutex_lock(&app->mutex); @@ -4604,41 +6442,7 @@ nxt_router_req_headers_ack_handler(nxt_task_t *task, main_app_port = app_port->main_app_port; - if (nxt_queue_chk_remove(&main_app_port->idle_link)) { - app->idle_processes--; - - nxt_debug(task, "app '%V' move port %PI:%d out of %s (ack)", - &app->name, main_app_port->pid, main_app_port->id, - (main_app_port->idle_start ? "idle_ports" : "spare_ports")); - - /* Check port was in 'spare_ports' using idle_start field. */ - if (main_app_port->idle_start == 0 - && app->idle_processes >= app->spare_processes) - { - /* - * If there is a vacant space in spare ports, - * move the last idle to spare_ports. - */ - nxt_assert(!nxt_queue_is_empty(&app->idle_ports)); - - idle_lnk = nxt_queue_last(&app->idle_ports); - idle_port = nxt_queue_link_data(idle_lnk, nxt_port_t, idle_link); - nxt_queue_remove(idle_lnk); - - nxt_queue_insert_tail(&app->spare_ports, idle_lnk); - - idle_port->idle_start = 0; - - nxt_debug(task, "app '%V' move port %PI:%d from idle_ports " - "to spare_ports", - &app->name, idle_port->pid, idle_port->id); - } - - if (nxt_router_app_can_start(app) && nxt_router_app_need_start(app)) { - app->pending_processes++; - start_process = 1; - } - } + start_process = nxt_router_app_port_busy(task, app, main_app_port, "ack"); main_app_port->active_requests++; @@ -4681,6 +6485,25 @@ nxt_router_req_headers_ack_handler(nxt_task_t *task, task->thread->engine->port->id, b); if (nxt_slow_path(res != NXT_OK)) { + /* + * This tail was cut from msg_info.buf above, so + * nxt_router_msg_cancel() walks a chain that no longer reaches + * it and nothing else completes it. Its buffers are chunks of + * shared memory that no pool teardown reclaims, so return them + * here. Queued rather than run inline, as the port layer does + * when it drops a message of its own. + */ + + while (b != NULL) { + next = b->next; + b->next = NULL; + + nxt_work_queue_add(&task->thread->engine->fast_work_queue, + b->completion_handler, task, b, b->parent); + + b = next; + } + nxt_http_request_error(task, r, NXT_HTTP_INTERNAL_SERVER_ERROR); } } @@ -4727,8 +6550,13 @@ nxt_router_response_error_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, req_rpc_data->rpc_cancel = 0; + /* + * The worker is gone, so whatever the router gave up on it is over. The + * settle does nothing unless this request is one of those. + */ + nxt_router_app_abandoned_settle(task, req_rpc_data); + /* TODO cancel message and return if cancelled. */ - // nxt_router_msg_cancel(task, &req_rpc_data->msg_info, req_rpc_data->stream); if (req_rpc_data->request != NULL) { nxt_http_request_error(task, req_rpc_data->request, @@ -4745,7 +6573,7 @@ nxt_router_app_port_ready(nxt_task_t *task, nxt_port_recv_msg_t *msg, { uint32_t n; nxt_app_t *app; - nxt_bool_t start_process, restarted; + nxt_bool_t start_process, restarted, superseded; nxt_port_t *port; nxt_app_joint_t *app_joint; nxt_app_joint_rpc_t *app_joint_rpc; @@ -4756,6 +6584,12 @@ nxt_router_app_port_ready(nxt_task_t *task, nxt_port_recv_msg_t *msg, app_joint = app_joint_rpc->app_joint; port = msg->u.new_port; + if (app_joint_rpc->start_timer != NULL) { + nxt_router_start_timer_cancel(task, app_joint_rpc->start_timer); + + app_joint_rpc->start_timer = NULL; + } + nxt_assert(app_joint != NULL); nxt_assert(port != NULL); nxt_assert(port->id == 0); @@ -4777,16 +6611,49 @@ nxt_router_app_port_ready(nxt_task_t *task, nxt_port_recv_msg_t *msg, restarted = (app->generation != app_joint_rpc->generation); if (app_joint_rpc->proto) { - nxt_assert(app->proto_port == NULL); nxt_assert(port->type == NXT_PROCESS_PROTOTYPE); - n = app->proto_port_requests; - app->proto_port_requests = 0; + if (app_joint_rpc->expired) { + /* + * Not a start attempt any more: the reaper armed by + * nxt_router_app_start_expired() for the prototype the deadline + * gave up on, which has announced itself after all. The cohort + * it owned was released when the attempt was failed, so there is + * nothing to replay here, and the slot it holds is an + * unaccounted one. Guarded rather than asserted for the reason + * given below, where a worker takes the same step. + */ + + nxt_assert(app->unaccounted_processes != 0); + + if (nxt_fast_path(app->unaccounted_processes != 0)) { + app->unaccounted_processes--; + } + + n = 0; + + } else { + n = app->proto_port_requests; + app->proto_port_requests = 0; + } + + /* + * A prototype that only just missed its deadline is adopted, exactly + * as a late worker is below -- unless the application has acquired a + * prototype meanwhile. It can: failing the attempt clears + * proto_port_requests, so the next request starts a second prototype + * and two of them can be up at once, of which only one may ever be + * app->proto_port. The other is QUIT here, which is also how the + * router disposes of one that belongs to a superseded generation. + */ + + superseded = restarted || app->proto_port != NULL; - if (nxt_slow_path(restarted)) { + if (nxt_slow_path(superseded)) { nxt_thread_mutex_unlock(&app->mutex); - nxt_debug(task, "proto port ready for restarted app, send QUIT"); + nxt_debug(task, "proto port ready for an app that has no use for " + "it, send QUIT"); nxt_port_socket_write(task, port, NXT_PORT_MSG_QUIT, -1, 0, 0, NULL); @@ -4814,9 +6681,41 @@ nxt_router_app_port_ready(nxt_task_t *task, nxt_port_recv_msg_t *msg, } nxt_assert(port->type == NXT_PROCESS_APP); - nxt_assert(app->pending_processes != 0); - app->pending_processes--; + if (app_joint_rpc->expired) { + /* + * The worker beat the reaper: "limits": {"start_timeout"} gave up on + * its start and moved the slot to unaccounted_processes, and the + * process the router had no pid for has now announced itself. Move + * the slot back and adopt the worker exactly as an in-time start is + * adopted below -- the sum nxt_router_app_can_start() is taken over + * does not change, and this is the only drain that recovers a slot + * without the process having to die. The request that waited for it + * was answered when the deadline fired; the worker serves the next + * one. + */ + + /* + * Guarded rather than asserted, for the reason given in + * nxt_router_app_start_failed(): nxt_assert() is nothing in a release + * build, and of the two ways to be wrong here, decrementing a counter + * that is already zero is much the worse. It wraps to UINT32_MAX and + * pins nxt_router_app_can_start() false for the application's + * lifetime, where failing to decrement merely leaks the slot. + * nxt_router_app_unaccounted_release() takes the same care. + */ + + nxt_assert(app->unaccounted_processes != 0); + + if (nxt_fast_path(app->unaccounted_processes != 0)) { + app->unaccounted_processes--; + } + + } else { + nxt_assert(app->pending_processes != 0); + + app->pending_processes--; + } if (nxt_slow_path(restarted)) { nxt_debug(task, "new port ready for restarted app, send QUIT"); @@ -4862,10 +6761,10 @@ static void nxt_router_app_port_error(nxt_task_t *task, nxt_port_recv_msg_t *msg, void *data) { + uint32_t n; + nxt_bool_t reap, restarted, expired, unsent; nxt_app_t *app; nxt_app_joint_t *app_joint; - nxt_queue_link_t *link; - nxt_http_request_t *r; nxt_app_joint_rpc_t *app_joint_rpc; nxt_assert(data != NULL); @@ -4873,11 +6772,54 @@ nxt_router_app_port_error(nxt_task_t *task, nxt_port_recv_msg_t *msg, app_joint_rpc = data; app_joint = app_joint_rpc->app_joint; + expired = 0; + unsent = 0; + + if (app_joint_rpc->start_timer != NULL) { + /* Read before the cancel, which is what drops the deadline. */ + expired = app_joint_rpc->start_timer->expired; + + /* The two ways the destination never saw the whole message. */ + unsent = app_joint_rpc->start_timer->recalled + || app_joint_rpc->start_timer->dropped; + + nxt_router_start_timer_cancel(task, app_joint_rpc->start_timer); + + app_joint_rpc->start_timer = NULL; + } + nxt_assert(app_joint != NULL); app = app_joint->app; - nxt_router_app_joint_use(task, app_joint, -1); + /* + * A start the deadline gave up on keeps its stream watched, and the + * reaper that watches it inherits this attempt's reference rather than + * taking one of its own -- so that the joint cannot be released between + * the two. + * + * Both shapes of start are reaped, because both leave a process behind. + * A worker start leaves the worker the prototype forked for it. A + * prototype start leaves the prototype itself: main forks it before it + * can answer, and one that never reaches PROCESS_READY forks no worker, + * so exactly one process comes of the attempt whatever the size of the + * cohort parked on it. The cohort stands for requests, not for + * processes; only the reaped slot stands for a process. + * + * A start the destination never saw whole is the exception -- recalled + * out of the send queue, or dropped by the port layer with part of it + * still unsent. Nothing was forked for either, so there is nothing to + * reap and the slot goes back the ordinary way. Accounting one of those + * would move a slot to unaccounted_processes that nothing can drain: no + * PROCESS_READY and no REMOVE_PID are coming for a process that was + * never created, which would make the bound a ratchet. + */ + + reap = (expired && !unsent && app != NULL); + + if (!reap) { + nxt_router_app_joint_use(task, app_joint, -1); + } if (nxt_slow_path(app == NULL)) { nxt_debug(task, "start error for released app"); @@ -4885,15 +6827,217 @@ nxt_router_app_port_error(nxt_task_t *task, nxt_port_recv_msg_t *msg, return; } - nxt_debug(task, "app '%V' %p start error", &app->name, app); + if (app_joint_rpc->expired) { + /* + * Not a start attempt: the reaper armed by + * nxt_router_app_start_expired() for a process the router had no + * pid for. Being here is the whole point -- the process is gone, + * and its unaccounted_processes slot goes with it. + */ + + nxt_router_app_unaccounted_release(task, app); + + return; + } + + /* + * Two ways in are not failures at all, and both are excluded rather + * than tolerated. + * + * A restart: nxt_router_app_restart_handler() bumps app->generation + * and sends QUIT to the prototype, and a worker that prototype had + * already forked for an in-flight start goes with it. The REMOVE_PID + * for that worker still carries the start's stream -- the prototype + * set it when it forked (nxt_application.c) and nothing clears it for + * a process that never became ready -- so the router retypes it into + * an RPC error and it lands here, reporting a start the operator + * themselves cancelled. The RPC peer is not what brings it here: that + * is the process the START_PROCESS was sent to, and this arrives for + * the worker. Generation is how nxt_router_app_port_ready() tells the + * same two apart before it quietly QUITs a port that arrives for a + * superseded one. + * + * And a shutdown: nxt_port_close() runs nxt_port_rpc_close() over the + * router port, which turns every registration still on it into an + * RPC error, so a start that was merely still pending when Unit was + * asked to stop would report itself as a failure. Nothing bumps the + * generation there, so it needs its own test -- engine->shutdown, the + * same flag this file already consults before deciding to start + * another process at all. + * + * Only the log line is conditional either way: the cleanup below has + * to run regardless, since the slots and the parked requests belong + * to an attempt that is over whatever ended it. + */ + + nxt_thread_mutex_lock(&app->mutex); + + restarted = (app->generation != app_joint_rpc->generation); + + nxt_thread_mutex_unlock(&app->mutex); + + /* + * Otherwise: logged, not nxt_debug()d. This handler is armed for one + * start attempt and disarmed the moment the port arrives, so it + * cannot fire for an ordinary worker exit or an idle timeout: every + * remaining call means an application process that was asked for will + * never exist, and the requests parked on it are about to be answered + * 503. At debug level that was invisible in a default configuration, + * which left "the application returns 503" with no corresponding line + * anywhere in the log. + * + * NXT_LOG_ERR rather than nxt_alert(), which the early-failure path + * in nxt_router_start_app_process_handler() uses. That one can only + * be Unit's own fault; this one is usually the application's -- a + * module that fails to import, a worker that exits during startup -- + * and Unit is working exactly as designed when it reports one. + * [error] is emitted at the default log level, which is the whole + * point of the change. One exception: a write to a prototype that is + * already gone is logged at debug level there, not as an alert. + * + * A different sentence from that alert, deliberately, even though + * the two describe the same disappointment. The test suite skips + * expected alerts by an unanchored regex over the log + * (Log.check_alerts()), so a test that skipped this [error] line by + * quoting "failed to start a process" would silently swallow the + * alert as well, in every test that ran alongside it. The two are + * worth telling apart; the level alone is a thin thing to rely on. + * + * It cannot say *why*: this is usually a REMOVE_PID that + * nxt_router_remove_pid_handler() retyped, and its payload is the + * dead pid and nothing else. The reason is in whatever the dead + * process logged before exiting; this line is what tells the operator + * there is something to go and look for. + */ + + if (nxt_slow_path(restarted || task->thread->engine->shutdown)) { + nxt_debug(task, "app '%V' start attempt cancelled", &app->name); + + } else if (expired) { + /* + * A third way in that is not what the line above describes: the + * "limits": {"start_timeout"} deadline failed this RPC itself. The + * process it was waiting for very probably does exist -- the router + * never learns its pid, so it cannot say -- it simply never called + * nxt_unit_init(). "produced no process" would be the wrong thing + * to send an operator looking through the logs, and the deadline has + * already said the right one, naming the application and the knob. + */ + + nxt_debug(task, "app '%V' start attempt bounded by \"start_timeout\"", + &app->name); + + } else { + nxt_log(task, NXT_LOG_ERR, + "app '%V' start attempt produced no process", &app->name); + } + + if (app_joint_rpc->proto) { + /* + * The prototype this RPC was armed for will never arrive, and only + * nxt_router_app_port_ready() clears proto_port_requests. Left set, + * it makes every later nxt_router_start_app_process_handler() call + * for this application take the "wait for prototype process" branch + * and park without sending anything -- so no further port_ready or + * port_error can ever fire to clear it. Only replacing the + * nxt_app_t recovers, and a reload does that only when the + * application's own config text changes. Clear it here, and take + * the whole parked cohort: the initiator's slot, plus one for each + * caller that joined the wait. This mirrors port_ready(), which + * takes the same count and replays that many starts on success. + * + * Where those slots go is decided below. All of them go back when + * the attempt forked nothing; when it did, one of them stays behind + * with the prototype, which is the process it forked. + */ + + nxt_thread_mutex_lock(&app->mutex); + + n = app->proto_port_requests; + app->proto_port_requests = 0; + + nxt_thread_mutex_unlock(&app->mutex); + + /* + * The counter is incremented in the same branch that sets ->proto, + * so an armed prototype RPC always owns at least its own slot. + */ + nxt_assert(n != 0); + + } else { + n = 1; + } + + if (reap) { + /* + * One process was forked for this attempt, and the reaper accounts + * for it. The rest of a prototype start's cohort goes back the + * ordinary way: those slots were taken by requests that joined the + * wait, and no process was forked for any of them. + */ + + if (n > 1) { + nxt_router_app_start_failed(task, app, n - 1, 0); + } + + nxt_router_app_start_expired(task, app, msg, app_joint, + app_joint_rpc->generation, + app_joint_rpc->proto); + + return; + } + + nxt_router_app_start_failed(task, app, n, 0); +} + + +/* + * A start attempt that will never yield a port: give back the "count" + * pending_processes slots it owns and, if the application is left with no way + * to answer at all, fail the requests waiting for an acknowledgement instead + * of letting them sit until they time out. + * + * "count" is 1 for an ordinary attempt, which owns only its initiator's slot. + * A failed prototype attempt owns the whole parked cohort: see + * nxt_router_app_port_error(). Zero is the caller that owns no slot at all + * and only wants the requests answered: nxt_router_app_port_get(), for a + * request that arrives when the application can neither serve it nor start + * anything to serve it. + * + * "unaccounted" moves the slots to app->unaccounted_processes instead of + * giving them back, for the one caller whose attempt very probably did leave + * an OS process behind: nxt_router_app_start_expired(). + */ + +static void +nxt_router_app_start_failed(nxt_task_t *task, nxt_app_t *app, uint32_t count, + nxt_bool_t unaccounted) +{ + nxt_queue_link_t *link; + nxt_http_request_t *r; link = NULL; nxt_thread_mutex_lock(&app->mutex); - nxt_assert(app->pending_processes != 0); + nxt_assert(app->pending_processes >= count); - app->pending_processes--; + /* + * The accounting above guarantees this, but nxt_assert() is nothing in a + * release build and the two directions are not equally bad: releasing one + * slot too few merely leaks it, while releasing one too many wraps the + * counter to UINT32_MAX and makes nxt_router_app_can_start() false + * forever -- a worse wedge than the one this path exists to clear. + */ + if (nxt_slow_path(count > app->pending_processes)) { + count = app->pending_processes; + } + + app->pending_processes -= count; + + if (unaccounted) { + app->unaccounted_processes += count; + } if (app->processes == 0 && !nxt_queue_is_empty(&app->ack_waiting_req)) { link = nxt_queue_first(&app->ack_waiting_req); @@ -4927,6 +7071,138 @@ nxt_router_app_port_error(nxt_task_t *task, nxt_port_recv_msg_t *msg, } +/* + * A start that "limits": {"start_timeout"} gave up on. + * + * The attempt is over -- its requests are answered here, through + * nxt_router_app_start_failed() -- but unlike every other way an attempt can + * end, this one very probably leaves a process running. Something was forked + * (only a forked process can fail to announce itself), and the router has no + * pid for it, because a pid is exactly what PROCESS_READY carries. Giving + * the pending_processes slot back would therefore mean the next request forks + * another one, with nothing counting either: "processes": {"max"} would bound + * only the processes that work. The slot moves to unaccounted_processes + * instead, where nxt_router_app_can_start() still sees it. + * + * "proto" says what was forked, and only nxt_router_app_port_ready() cares: + * a worker start leaves the worker its prototype forked, a prototype start + * leaves the prototype main forked for it. A stuck prototype forks no worker + * of its own, so either way the attempt leaves exactly one process, and one + * slot accounts for it. That a stuck prototype spends worker slots is + * deliberate: while it is stuck the application has no way to run a worker + * anyway, and the alternative -- a counter of its own -- would let prototypes + * and workers each grow to "max". + * + * That is a bound, not a leak, because the stream stays watched. The reply + * this start gave up on can still arrive, and both shapes of it retire the + * slot: + * + * - the process announces itself late, missing only the deadline; + * nxt_router_app_port_ready() retires the slot and adopts it -- a worker + * into ->processes, a prototype into ->proto_port unless the application + * has one by then -- so a merely slow application costs one 503 and + * nothing else; + * + * - the process dies and the router is told with the start's own stream + * still attached: for a worker by the prototype that forked it and + * reaps it, for a prototype by main, which clears process->stream only + * for a process that reached READY (see nxt_port_remove_notify_others(), + * and nxt_router_remove_pid_handler(), which retypes that REMOVE_PID + * into an RPC error) -- and nxt_router_app_port_error() gives the slot + * back. + * + * Neither is guaranteed: a process wedged forever is exactly the case this + * exists for, and it holds its slot for as long as it lives. That is the + * intended trade. A configuration reload that replaces the nxt_app_t is the + * other way back. + */ + +static void +nxt_router_app_start_expired(nxt_task_t *task, nxt_app_t *app, + nxt_port_recv_msg_t *msg, nxt_app_joint_t *app_joint, uint32_t generation, + nxt_bool_t proto) +{ + uint32_t stream; + nxt_app_joint_rpc_t *reaper; + + stream = msg->port_msg.stream; + + nxt_router_app_start_failed(task, app, 1, 1); + + /* + * Only from inside this stream's own handler is the key free to take + * again; see nxt_port_rpc_register_handler_at(). + */ + + reaper = nxt_port_rpc_register_handler_at(task, msg->port, + nxt_router_app_port_ready, + nxt_router_app_port_error, + stream, + sizeof(nxt_app_joint_rpc_t)); + + if (nxt_slow_path(reaper == NULL)) { + /* + * The bound holds -- the slot is already counted -- but nothing will + * hand it back, so say so: from here only a reload recovers it. + */ + + nxt_alert(task, "app \"%V\" cannot watch the process its expired " + "start left behind; that \"processes\" slot is held " + "until the application is reconfigured", &app->name); + + nxt_router_app_joint_use(task, app_joint, -1); + + return; + } + + /* The reaper inherits the failed attempt's app_joint reference. */ + + reaper->app_joint = app_joint; + reaper->generation = generation; + reaper->expired = 1; + + /* + * Which kind of process the slot stands for: nxt_router_app_port_ready() + * branches on ->proto before anything else, and a late prototype must + * not be adopted as though it were a worker. + */ + + reaper->proto = proto; + + nxt_debug(task, "app '%V' stream #%uD kept to account for the process an " + "expired start left behind", &app->name, stream); +} + + +/* + * The process an expired start left behind is accounted for at last: it died + * and the router was told. Only nxt_router_app_start_expired() puts a slot + * here, and only this and nxt_router_app_port_ready() take one back. + */ + +static void +nxt_router_app_unaccounted_release(nxt_task_t *task, nxt_app_t *app) +{ + nxt_thread_mutex_lock(&app->mutex); + + /* + * One reaper owns exactly one slot, so this cannot underflow; the test + * is here because a wrapped counter would pin nxt_router_app_can_start() + * false for the application's lifetime, which is worse than leaking the + * slot it is meant to release. + */ + + if (nxt_fast_path(app->unaccounted_processes != 0)) { + app->unaccounted_processes--; + } + + nxt_thread_mutex_unlock(&app->mutex); + + nxt_debug(task, "app '%V' unaccounted process is gone, %d left", + &app->name, app->unaccounted_processes); +} + + nxt_inline nxt_port_t * nxt_router_app_get_port_for_quit(nxt_task_t *task, nxt_app_t *app) { @@ -4995,12 +7271,307 @@ nxt_router_app_unlink(nxt_task_t *task, nxt_app_t *app) } +/* + * Refill spare_ports from the tail of idle_ports when a port leaves + * spare_ports. Called with app->mutex held. + */ +static void +nxt_router_app_spare_rebalance(nxt_task_t *task, nxt_app_t *app, + nxt_port_t *port) +{ + nxt_port_t *idle_port; + nxt_queue_link_t *idle_lnk; + + /* Check port was in 'spare_ports' using idle_start field. */ + if (port->idle_start == 0 && app->idle_processes >= app->spare_processes) { + /* + * If there is a vacant space in spare ports, + * move the last idle to spare_ports. + */ + nxt_assert(!nxt_queue_is_empty(&app->idle_ports)); + + idle_lnk = nxt_queue_last(&app->idle_ports); + idle_port = nxt_queue_link_data(idle_lnk, nxt_port_t, idle_link); + nxt_queue_remove(idle_lnk); + + nxt_queue_insert_tail(&app->spare_ports, idle_lnk); + + idle_port->idle_start = 0; + + nxt_debug(task, "app '%V' move port %PI:%d from idle_ports " + "to spare_ports", + &app->name, idle_port->pid, idle_port->id); + } +} + + +/* + * Take a port out of the idle economy. Called with app->mutex held; answers + * whether the caller must start a replacement process once it has dropped the + * lock. Does nothing to a port that is not in one of the idle queues. + * + * Two events reach this transition. An acknowledgement, where a worker has + * taken a request the router was still holding for it, and the start of + * detached work, where a worker the router has already parked as idle says it + * is in fact still running. The second is why this is a function rather than + * a block: setting ->detached without unwinding leaves a port that + * nxt_router_app_port_idle() will refuse to insert again and that + * nxt_router_adjust_idle_timer() will still reap, so the flag and the unwind + * have to happen in one critical section. + * + * "reason" is for the debug log only. + */ + +static nxt_bool_t +nxt_router_app_port_busy(nxt_task_t *task, nxt_app_t *app, nxt_port_t *port, + const char *reason) +{ + nxt_bool_t start_process; + + start_process = 0; + + if (!nxt_queue_chk_remove(&port->idle_link)) { + return 0; + } + + app->idle_processes--; + + nxt_debug(task, "app '%V' move port %PI:%d out of %s (%s)", + &app->name, port->pid, port->id, + (port->idle_start ? "idle_ports" : "spare_ports"), reason); + + nxt_router_app_spare_rebalance(task, app, port); + + if (nxt_router_app_can_start(app) && nxt_router_app_need_start(app)) { + app->pending_processes++; + start_process = 1; + } + + return start_process; +} + + +/* + * Hand a port back to the idle economy, if it is really idle. Called with + * app->mutex held; answers whether the caller must post app->adjust_idle_work + * once it has dropped the lock, which is how the reaper's timer gets armed. + * + * Extracted from nxt_router_app_port_release() because the end of detached + * work reaches the same transition from nxt_router_detached_handler(), and a + * second copy of a block that touches three queues and two counters is how + * they drift apart. + */ + +static nxt_bool_t +nxt_router_app_port_idle(nxt_task_t *task, nxt_app_t *app, nxt_port_t *port) +{ + nxt_bool_t adjust_idle_timer; + + adjust_idle_timer = 0; + + if (port->pair[1] != -1 + && port->active_requests == 0 + && port->active_websockets == 0 + && port->detached == 0 + && port->idle_link.next == NULL) + { + if (app->idle_processes == app->spare_processes + && app->adjust_idle_work.data == NULL) + { + adjust_idle_timer = 1; + app->adjust_idle_work.data = app; + app->adjust_idle_work.next = NULL; + } + + if (app->idle_processes < app->spare_processes) { + nxt_queue_insert_tail(&app->spare_ports, &port->idle_link); + + nxt_debug(task, "app '%V' move port %PI:%d to spare_ports", + &app->name, port->pid, port->id); + + } else { + nxt_queue_insert_tail(&app->idle_ports, &port->idle_link); + + port->idle_start = task->thread->engine->timers.now; + + nxt_debug(task, "app '%V' move port %PI:%d to idle_ports", + &app->name, port->pid, port->id); + } + + app->idle_processes++; + } + + return adjust_idle_timer; +} + + +/* + * The router gave up on a request a worker is running. The 503 is the + * client's answer, but the worker does not know that and keeps executing, so + * the port must not go back into the idle economy when the request's + * accounting is released a moment from now: the reaper would QUIT a worker + * in the middle of a request, and the port would count as free against + * "processes": {"max"} while it is not. + * + * Enter the state a detached worker already has -- see + * nxt_router_detached_apply() -- and settle it when the worker answers, when + * the port closes, or when the application's own FINISH edge says its work is + * over. "app_port" is the port the acknowledgement moved the request to: + * the worker's, never the shared queue. Called before the unlink that + * releases the request. + * + * The state lands on app_port->main_app_port. That is the same object the + * acknowledgement and a release reach through the same field, while a + * detached edge finds it by pid (nxt_router_detached_apply()); a worker's + * ports share one main port, so both identities name it. + */ + +static void +nxt_router_app_abandon(nxt_task_t *task, nxt_app_t *app, + nxt_request_rpc_data_t *req_rpc_data, nxt_port_t *app_port) +{ + nxt_bool_t changed, start_process; + nxt_port_t *port; + + port = app_port->main_app_port; + + /* + * Both the detached state and the rpc_data's own pointer outlive this + * call, and the state is published to another thread: the port close + * path takes the same mutex and drops the state's reference when it + * settles. So both application references and the port reference are + * taken before the state can be seen, the way + * nxt_router_detached_apply() takes its own before the mutex; a close + * that arrived in between would otherwise drop a reference this call has + * not taken yet. The request still holds an application reference of + * its own here, so the increment cannot resurrect a freed one. + */ + nxt_router_app_use(task, app, 2); + + nxt_thread_mutex_lock(&app->mutex); + + changed = 0; + + if (port->detached == 0) { + port->detached = 1; + app->detached_processes++; + changed = 1; + } + + port->detached_router++; + + nxt_port_inc_use(port); + + /* + * Defensive, and symmetric with the detached START edge: a port with a + * live request is not in an idle queue, so this normally finds nothing + * to unwind and starts nothing. It is here so that the flag and the + * unwind share this critical section whichever way the port got here. + */ + start_process = nxt_router_app_port_busy(task, app, port, "abandoned"); + + nxt_thread_mutex_unlock(&app->mutex); + + if (!changed) { + /* + * The application was detached already, so its own edge owns that + * reference; this call keeps only the rpc_data's. + */ + nxt_router_app_use(task, app, -1); + } + + /* + * Keep the registration. nxt_router_response_ready_handler() is what + * drops the answer of a request nobody tracks any more, and that is also + * where the port is taken back; the unlink that follows would cancel the + * registration first and the answer would never reach it. + */ + req_rpc_data->rpc_cancel = 0; + + req_rpc_data->abandoned_app = app; + req_rpc_data->abandoned_port = port; + + if (start_process) { + nxt_router_start_app_process(task, app); + } +} + + +/* + * A worker the router gave up on has answered, or is gone. Its port rejoins + * the idle economy if the router's own mark is the only thing holding it out + * -- an application START edge clears that mark and its FINISH edge is what + * ends the state then -- and the references the abandonment took are dropped + * either way. + */ + +static void +nxt_router_app_abandoned_settle(nxt_task_t *task, + nxt_request_rpc_data_t *req_rpc_data) +{ + nxt_port_t *port; + nxt_app_t *app; + nxt_bool_t adjust_idle_timer, ours; + + port = req_rpc_data->abandoned_port; + + if (port == NULL) { + return; + } + + app = req_rpc_data->abandoned_app; + + req_rpc_data->abandoned_port = NULL; + req_rpc_data->abandoned_app = NULL; + + ours = 0; + adjust_idle_timer = 0; + + nxt_thread_mutex_lock(&app->mutex); + + if (port->detached_router != 0) { + port->detached_router--; + + /* + * The port is free only once no reason is left: this was the last + * request the router gave up on, and the application is not running + * detached work of its own either. + */ + if (port->detached_router == 0 && port->detached_app == 0 + && port->detached != 0) + { + port->detached = 0; + + app->detached_processes--; + + adjust_idle_timer = nxt_router_app_port_idle(task, app, port); + + ours = 1; + } + } + + nxt_thread_mutex_unlock(&app->mutex); + + if (adjust_idle_timer) { + nxt_router_app_use(task, app, 1); + nxt_event_engine_post(app->engine, &app->adjust_idle_work); + } + + if (ours) { + nxt_router_app_use(task, app, -1); + } + + nxt_router_app_use(task, app, -1); + nxt_port_use(task, port, -1); +} + + static void nxt_router_app_port_release(nxt_task_t *task, nxt_app_t *app, nxt_port_t *port, nxt_apr_action_t action) { int inc_use; - uint32_t got_response, dec_requests; + uint32_t got_response, dec_requests, dec_websockets; nxt_bool_t adjust_idle_timer; nxt_port_t *main_app_port; @@ -5009,6 +7580,7 @@ nxt_router_app_port_release(nxt_task_t *task, nxt_app_t *app, nxt_port_t *port, inc_use = 0; got_response = 0; dec_requests = 0; + dec_websockets = 0; switch (action) { case NXT_APR_NEW_PORT: @@ -5027,11 +7599,21 @@ nxt_router_app_port_release(nxt_task_t *task, nxt_app_t *app, nxt_port_t *port, case NXT_APR_CLOSE: inc_use = -1; break; + case NXT_APR_WEBSOCKET_CLOSE: + dec_websockets = 1; + inc_use = -1; + break; } - nxt_debug(task, "app '%V' release port %PI:%d: %d %d", &app->name, + nxt_debug(task, "app '%V' release port %PI:%d: %d %d %d", &app->name, port->pid, port->id, - (int) inc_use, (int) got_response); + (int) inc_use, (int) got_response, (int) dec_websockets); + + /* + * A websocket is upgraded from a request a worker has answered, so the + * port it is released on is that worker's own port, never the shared one. + */ + nxt_assert(dec_websockets == 0 || port->id != NXT_SHARED_PORT_ID); if (port->id == NXT_SHARED_PORT_ID) { nxt_thread_mutex_lock(&app->mutex); @@ -5048,46 +7630,17 @@ nxt_router_app_port_release(nxt_task_t *task, nxt_app_t *app, nxt_port_t *port, nxt_thread_mutex_lock(&app->mutex); main_app_port->active_requests -= got_response + dec_requests; + main_app_port->active_websockets -= dec_websockets; app->active_requests -= got_response + dec_requests; if (main_app_port->pair[1] != -1 && main_app_port->app_link.next == NULL) { nxt_queue_insert_tail(&app->ports, &main_app_port->app_link); - nxt_port_inc_use(main_app_port); - } - - adjust_idle_timer = 0; - - if (main_app_port->pair[1] != -1 - && main_app_port->active_requests == 0 - && main_app_port->active_websockets == 0 - && main_app_port->idle_link.next == NULL) - { - if (app->idle_processes == app->spare_processes - && app->adjust_idle_work.data == NULL) - { - adjust_idle_timer = 1; - app->adjust_idle_work.data = app; - app->adjust_idle_work.next = NULL; - } - - if (app->idle_processes < app->spare_processes) { - nxt_queue_insert_tail(&app->spare_ports, &main_app_port->idle_link); - - nxt_debug(task, "app '%V' move port %PI:%d to spare_ports", - &app->name, main_app_port->pid, main_app_port->id); - } else { - nxt_queue_insert_tail(&app->idle_ports, &main_app_port->idle_link); - - main_app_port->idle_start = task->thread->engine->timers.now; - - nxt_debug(task, "app '%V' move port %PI:%d to idle_ports", - &app->name, main_app_port->pid, main_app_port->id); - } - - app->idle_processes++; + nxt_port_inc_use(main_app_port); } + adjust_idle_timer = nxt_router_app_port_idle(task, app, main_app_port); + nxt_thread_mutex_unlock(&app->mutex); if (adjust_idle_timer) { @@ -5112,18 +7665,32 @@ nxt_router_app_port_release(nxt_task_t *task, nxt_app_t *app, nxt_port_t *port, } +#if (NXT_TESTS) + +/* For src/test/nxt_router_websocket_test.c. */ + +void +nxt_router_test_app_port_release(nxt_task_t *task, nxt_app_t *app, + nxt_port_t *port, nxt_apr_action_t action) +{ + nxt_router_app_port_release(task, app, port, action); +} + +#endif + + void nxt_router_app_port_close(nxt_task_t *task, nxt_port_t *port) { - nxt_app_t *app; - nxt_bool_t unchain, start_process; - nxt_port_t *idle_port; - nxt_queue_link_t *idle_lnk; + nxt_app_t *app; + nxt_bool_t unchain, start_process, detached; app = port->app; nxt_assert(app != NULL); + detached = 0; + nxt_thread_mutex_lock(&app->mutex); if (port == app->proto_port) { @@ -5161,27 +7728,31 @@ nxt_router_app_port_close(nxt_task_t *task, nxt_port_t *port) &app->name, port->pid, port->id, (port->idle_start ? "idle_ports" : "spare_ports")); - if (port->idle_start == 0 - && app->idle_processes >= app->spare_processes) - { - nxt_assert(!nxt_queue_is_empty(&app->idle_ports)); + nxt_router_app_spare_rebalance(task, app, port); + } - idle_lnk = nxt_queue_last(&app->idle_ports); - idle_port = nxt_queue_link_data(idle_lnk, nxt_port_t, idle_link); - nxt_queue_remove(idle_lnk); + app->processes--; - nxt_queue_insert_tail(&app->spare_ports, idle_lnk); + /* + * A worker can die in the middle of its detached work -- a fatal error + * after the response went out, or a kill. This is the only path that + * runs for it then, so the count has to be settled here or the + * application never reaches the zero nxt_router_free_app() asserts on. + * The process is gone, so every unit of work it was running goes with + * it, however many the counts hold. The matching application + * reference, the one the first reason took, is dropped below, outside + * the lock. + */ - idle_port->idle_start = 0; + detached = port->detached; - nxt_debug(task, "app '%V' move port %PI:%d from idle_ports " - "to spare_ports", - &app->name, idle_port->pid, idle_port->id); - } + if (detached) { + port->detached = 0; + port->detached_app = 0; + port->detached_router = 0; + app->detached_processes--; } - app->processes--; - start_process = !task->thread->engine->shutdown && nxt_router_app_can_start(app) && nxt_router_app_need_start(app); @@ -5201,6 +7772,16 @@ nxt_router_app_port_close(nxt_task_t *task, nxt_port_t *port) if (start_process) { nxt_router_start_app_process(task, app); } + + /* + * Keep the detached reference until the replacement start has taken its + * own. After configuration removal this can be the last reference, so + * dropping it earlier frees app before nxt_router_start_app_process(). + */ + + if (detached) { + nxt_router_app_use(task, app, -1); + } } @@ -5244,6 +7825,33 @@ nxt_router_adjust_idle_timer(nxt_task_t *task, void *obj, void *data) lnk = nxt_queue_first(&app->idle_ports); port = nxt_queue_link_data(lnk, nxt_port_t, idle_link); + /* + * A worker running detached work must never be reachable from here: + * this loop clears ->app, which stops nxt_port_close() from ever + * running nxt_router_app_port_close() for the port, and the count + * and the application reference would then never be settled. + * nxt_router_app_port_idle() refuses to insert such a port and + * nxt_router_app_port_busy() takes one that is already here back + * out, both under this same mutex. + */ + + nxt_assert(port->detached == 0); + + if (nxt_slow_path(port->detached != 0)) { + /* + * A release build. A bare continue selects the same link again + * and spins with app->mutex held. Take the port out of the idle + * queue only: the worker is alive, so it keeps ->app, stays in + * app->processes and gets no QUIT. nxt_router_app_port_idle() + * inserts it again when the detached work ends. + */ + + nxt_queue_remove(lnk); + lnk->next = NULL; + app->idle_processes--; + continue; + } + timeout = port->idle_start + app->idle_timeout; nxt_debug(task, "app '%V' pid %PI, start %M, timeout %M, threshold %M", @@ -5383,6 +7991,7 @@ nxt_router_free_app(nxt_task_t *task, void *obj, void *data) nxt_assert(app->proto_port == NULL); nxt_assert(app->processes == 0); + nxt_assert(app->detached_processes == 0); nxt_assert(app->active_requests == 0); nxt_assert(app->port_hash_count == 0); nxt_assert(app->idle_processes == 0); @@ -5421,7 +8030,7 @@ static void nxt_router_app_port_get(nxt_task_t *task, nxt_app_t *app, nxt_request_rpc_data_t *req_rpc_data) { - nxt_bool_t start_process; + nxt_bool_t start_process, unanswerable; nxt_port_t *port; nxt_http_request_t *r; @@ -5439,6 +8048,21 @@ nxt_router_app_port_get(nxt_task_t *task, nxt_app_t *app, start_process = 1; } + /* + * Nothing is running, nothing is starting, and nothing may be started: + * every "processes" slot is held by a process an expired start left + * behind (nxt_router_app_start_expired()). Only a start can take a + * request back out of ack_waiting_req, so parking this one would leave + * it there until it timed out. Answer it below instead, at once. + * + * Reachable only through unaccounted_processes: with no process and + * none pending, nxt_router_app_can_start() is otherwise false only for + * "processes": {"max": 0}, which the configuration rejects. + */ + + unanswerable = (start_process == 0 && app->processes == 0 + && app->pending_processes == 0); + r = req_rpc_data->request; /* @@ -5458,8 +8082,42 @@ nxt_router_app_port_get(nxt_task_t *task, nxt_app_t *app, req_rpc_data->app_port = port; req_rpc_data->apr_action = NXT_APR_REQUEST_FAILED; + /* + * Bound the wait for a process. Until a worker acknowledges the request + * the two dispatch paths that arm this timer have not run, so a request + * parked in ack_waiting_req has no deadline of its own: it waits for a + * worker that may never ask for it. With detached work that wait is as + * long as the application chooses to run, which is what makes the + * deadline worth having -- "limits": {"timeout"} now bounds waiting for + * capacity, not only the time a worker spends on the request. + * + * The handler is the one those paths use, and it is shared with the + * post-acknowledgement deadline, where 503 is simply the right answer. + * Here it is right only if the request is still in the queue, so the + * handler retracts it first and tells the two deadlines apart by the + * answer; see nxt_router_app_timeout(). + * + * An acknowledgement re-arms the same timer with the same handler, which + * nxt_timer_add() treats as a change, not a second timer. + */ + + if (app->timeout != 0) { + r->timer.handler = nxt_router_app_timeout; + r->timer_data = req_rpc_data; + + nxt_timer_add(task->thread->engine, &r->timer, app->timeout); + } + if (start_process) { nxt_router_start_app_process(task, app); + + } else if (nxt_slow_path(unanswerable)) { + nxt_debug(task, "app '%V' has no process and may not start one", + &app->name); + + /* No slot of its own to give back: only the requests are wanted. */ + + nxt_router_app_start_failed(task, app, 0, 0); } } @@ -5565,10 +8223,11 @@ static void nxt_router_app_prepare_request(nxt_task_t *task, nxt_request_rpc_data_t *req_rpc_data) { - nxt_app_t *app; - nxt_buf_t *buf, *body; - nxt_int_t res; - nxt_port_t *port, *reply_port; + nxt_app_t *app; + nxt_buf_t *buf, *body; + nxt_int_t res; + nxt_port_t *port, *reply_port; + nxt_http_status_t status; int notify; struct { @@ -5589,13 +8248,9 @@ nxt_router_app_prepare_request(nxt_task_t *task, reply_port = task->thread->engine->port; buf = nxt_router_prepare_msg(task, req_rpc_data->request, app, - nxt_app_msg_prefix[app->type]); + nxt_app_msg_prefix[app->type], &status); if (nxt_slow_path(buf == NULL)) { - nxt_alert(task, "stream #%uD, app '%V': failed to prepare app message", - req_rpc_data->stream, &app->name); - - nxt_http_request_error(task, req_rpc_data->request, - NXT_HTTP_INTERNAL_SERVER_ERROR); + nxt_http_request_error(task, req_rpc_data->request, status); return; } @@ -5663,9 +8318,32 @@ nxt_router_app_prepare_request(nxt_task_t *task, +/* + * Builds the nxt_unit_request_t for the application in shared memory. + * + * Every length that lands in a narrow field of the libunit protocol is + * checked before the buffer is allocated, and the request is refused rather + * than having the field truncated: a truncated length makes the application + * see a different string from the one that was copied (a 256-byte method + * arrived as an empty one), with the NUL terminator somewhere else. + * + * - method_length is uint8_t, and the HTTP parser does not bound a method + * (it is limited only by the header buffer): 501. + * - a field's name_length is uint8_t. nxt_http_parse_field_name() caps a + * name at 255 bytes (NXT_HTTP_MAX_FIELD_NAME), which fits alone, but the + * PHP/Perl/Ruby prefix "HTTP_" is added on top: names of 251..255 bytes + * wrapped to 0..4. 431. + * - version, the address and port texts are uint8_t as well but produced + * by Unit itself; they are checked all the same, and fail with 500. + * - the uint32_t lengths (server name, target, path, query, values) are + * bounded by req_size <= PORT_MMAP_DATA_SIZE. + * + * On failure NULL is returned and *status says what to answer. + */ + static nxt_buf_t * nxt_router_prepare_msg(nxt_task_t *task, nxt_http_request_t *r, - nxt_app_t *app, const nxt_str_t *prefix) + nxt_app_t *app, const nxt_str_t *prefix, nxt_http_status_t *status) { void *target_pos, *query_pos; u_char *pos, *end, *p, c; @@ -5678,6 +8356,30 @@ nxt_router_prepare_msg(nxt_task_t *task, nxt_http_request_t *r, nxt_http_fields_iter_t iter, dup_iter; nxt_unit_request_t *req; + *status = NXT_HTTP_INTERNAL_SERVER_ERROR; + + if (nxt_slow_path(r->method->length > UINT8_MAX)) { + nxt_log(task, NXT_LOG_INFO, "app '%V': request method of %uz bytes " + "is too long for the application protocol", &app->name, + r->method->length); + + *status = NXT_HTTP_NOT_IMPLEMENTED; + return NULL; + } + + if (nxt_slow_path(r->version.length > UINT8_MAX + || r->remote->address_length > UINT8_MAX + || r->local->address_length > UINT8_MAX + || nxt_sockaddr_port_length(r->local) > UINT8_MAX)) + { + nxt_alert(task, "app '%V': request version or address too long for " + "the application protocol", &app->name); + + return NULL; + } + + /* Every length is of data the request holds, so the sum cannot wrap. */ + req_size = sizeof(nxt_unit_request_t) + r->method->length + 1 + r->version.length + 1 @@ -5694,8 +8396,22 @@ nxt_router_prepare_msg(nxt_task_t *task, nxt_http_request_t *r, nxt_http_fields_each(field, r->inline_fields, r->num_inline_fields, r->fields) { + if (field->skip) { + continue; + } + fields_count++; + if (nxt_slow_path(field->name_length + prefix->length > UINT8_MAX)) { + nxt_log(task, NXT_LOG_INFO, "app '%V': header field name of %d " + "bytes is too long for the application protocol with " + "prefix \"%V\"", &app->name, (int) field->name_length, + prefix); + + *status = NXT_HTTP_REQUEST_HEADER_FIELDS_TOO_LARGE; + return NULL; + } + req_size += field->name_length + prefix->length + 1 + field->value_length + 1; } nxt_http_fields_loop; @@ -5703,8 +8419,8 @@ nxt_router_prepare_msg(nxt_task_t *task, nxt_http_request_t *r, req_size += fields_count * sizeof(nxt_unit_field_t); if (nxt_slow_path(req_size > PORT_MMAP_DATA_SIZE)) { - nxt_alert(task, "headers to big to fit in shared memory (%d)", - (int) req_size); + nxt_alert(task, "app '%V': headers too big to fit in shared memory " + "(%uz)", &app->name, req_size); return NULL; } @@ -5806,6 +8522,7 @@ nxt_router_prepare_msg(nxt_task_t *task, nxt_http_request_t *r, dst_field->hash = field->hash; dst_field->skip = 0; + /* Checked to fit uint8_t when req_size was summed. */ dst_field->name_length = field->name_length + prefix->length; dst_field->value_length = field->value_length; @@ -5947,10 +8664,30 @@ nxt_router_prepare_msg(nxt_task_t *task, nxt_http_request_t *r, } +#if (NXT_TESTS) + +nxt_buf_t * +nxt_router_test_prepare_msg(nxt_task_t *task, nxt_http_request_t *r, + nxt_app_t *app, nxt_uint_t *status) +{ + nxt_buf_t *b; + nxt_http_status_t st; + + b = nxt_router_prepare_msg(task, r, app, nxt_app_msg_prefix[app->type], + &st); + *status = st; + + return b; +} + +#endif + + static void nxt_router_app_timeout(nxt_task_t *task, void *obj, void *data) { nxt_timer_t *timer; + nxt_msg_info_t *msg_info; nxt_http_request_t *r; nxt_request_rpc_data_t *req_rpc_data; @@ -5961,6 +8698,74 @@ nxt_router_app_timeout(nxt_task_t *task, void *obj, void *data) r = nxt_timer_data(timer, nxt_http_request_t, timer); req_rpc_data = r->timer_data; + msg_info = &req_rpc_data->msg_info; + + /* + * Two deadlines share this handler. After an acknowledgement a worker + * holds the request and 503 is the answer: it is too slow. Before one + * the request is still in the shared port queue, and only the CAS in + * nxt_router_msg_retract() says whether it is still there. + * + * So retract before answering. A retraction that loses means a worker + * took the request in that same instant: it is running, and a 503 here + * would both fail the request and run it -- twice, if the client + * retries. + * + * Leave a claimed request alone until its acknowledgement, and do not + * re-arm: the acknowledgement arms this timer again. Do not answer it + * at a later expiry either. The chain's tail is the request body that + * nxt_router_req_headers_ack_handler() still has to send, and libunit + * keeps the request until that body arrives. An unlink here would drop + * the body and cancel the RPC, so a late acknowledgement is ignored and + * the request stays in the worker for ever. + * + * The acknowledgement replaces ->app_port with the worker's own port, so + * the shared port here means that it has not arrived yet. + */ + + if (msg_info->cancel == NXT_MSG_QUEUED) { + (void) nxt_router_msg_retract(task, req_rpc_data); + } + + if (msg_info->cancel == NXT_MSG_CLAIMED + && req_rpc_data->app_port != NULL + && req_rpc_data->app_port->id == NXT_SHARED_PORT_ID) + { + nxt_debug(task, "stream #%uD: claimed, waiting for the ack", + req_rpc_data->stream); + + return; + } + + /* + * The acknowledgement moves the request's port off the shared queue and + * onto the worker that took it; nxt_router_msg_retract() reads the same + * fact. A request a worker is running is not the router's to release: + * its port has to stay out of the idle economy until the worker answers + * or closes. See nxt_router_app_abandon(). + * + * An upgraded stream is not such a request. Its accounting was already + * given back by NXT_APR_UPGRADE, so no worker is running it and there is + * no slot to hold; what does keep the port out of the idle economy is the + * session count, and that one is settled by the unlink below, because the + * upgrade left NXT_APR_WEBSOCKET_CLOSE as the action. The mark would + * instead report the worker "detached" for as long as the connection + * lives. The websocket state is the upgrade itself -- it is assigned + * beside that release, and nowhere else -- so a handshake still in flight + * or one that failed, both of which do hold their accounting, still reach + * the abandon. + */ + if (req_rpc_data->app_port != NULL + && req_rpc_data->app_port->id != NXT_SHARED_PORT_ID + && r->state != &nxt_http_websocket) + { + /* The request is still linked, so it holds the application. */ + nxt_assert(req_rpc_data->app != NULL); + + nxt_router_app_abandon(task, req_rpc_data->app, req_rpc_data, + req_rpc_data->app_port); + } + nxt_http_request_error(task, r, NXT_HTTP_SERVICE_UNAVAILABLE); nxt_request_rpc_data_unlink(task, req_rpc_data); @@ -5988,6 +8793,300 @@ nxt_router_http_request_release(nxt_task_t *task, void *obj, void *data) } +/* + * An application says it answered a request and kept running, or that such + * work has finished. PHP's fastcgi_finish_request() is the case this + * exists for: libunit reports the request done while the script runs on, so + * without this the router counts the worker idle, hands its slot back to + * "processes": {"max"}, and reaps a process that is still executing. + * + * Runs on the router's main thread, because the message arrives on the + * router's own port; see nxt_router_detached_handler(). The port is + * identified by pid alone: the accounting hangs off the application's main + * port, the one with id 0, and a worker has exactly one however many + * contexts it runs. Looking it up rather than following ->main_app_port + * from the sender's port is also what keeps this off a stale pointer -- that + * field is never cleared, and the main port can be released while a sibling + * port of the same process is still registered. + * + * The application's edges are counted, not latched. Several contexts of one + * worker can run detached work at once and the edge names none of them, so + * the port leaves the idle economy on the first start and rejoins it on the + * last finish, once the router has no abandoned request of its own left + * either. app->detached_processes counts processes, so it moves on those two + * transitions only. A finish with nothing to match -- a worker whose death + * has already settled the count, or a forged one -- changes nothing. + */ + +static void +nxt_router_detached_apply(nxt_task_t *task, nxt_pid_t pid, uint8_t state) +{ + int drop; + nxt_app_t *app; + nxt_port_t *port; + nxt_bool_t changed, start_process, adjust_idle_timer; + const char *alert; + nxt_runtime_t *rt; + nxt_atomic_int_t c; + + rt = task->thread->runtime; + + nxt_assert(task->thread->engine == rt->main_engine); + + port = nxt_runtime_port_find(rt, pid, 0); + + if (nxt_slow_path(port == NULL)) { + nxt_debug(task, "detached_handler: %PI has no main port", pid); + return; + } + + app = port->app; + + if (nxt_slow_path(app == NULL)) { + nxt_debug(task, "detached_handler: port %PI:%d has no application", + pid, port->id); + return; + } + + /* + * A reference, taken only if the application still has one. The + * count reaching zero is what posts nxt_router_free_app() to this + * thread, and the last drop can happen on a worker engine while this + * edge is already on its way here: the request that held the + * application is released there, and a reconfiguration has already + * taken the configuration's reference. A plain increment would + * resurrect an application whose free is queued behind this handler, + * and the start it may post below would then lock a mutex in freed + * memory. Zero is treated exactly like no application at all; the + * free that is coming clears ->app and QUITs the worker. + */ + + for ( ;; ) { + c = app->use_count; + + if (c == 0) { + nxt_debug(task, "detached_handler: app '%V' is being freed", + &app->name); + return; + } + + if (nxt_atomic_cmp_set(&app->use_count, c, c + 1)) { + break; + } + } + + nxt_debug(task, "app '%V' port %PI:%d detached state %d", + &app->name, port->pid, port->id, (int) state); + + alert = NULL; + changed = 0; + start_process = 0; + adjust_idle_timer = 0; + + nxt_thread_mutex_lock(&app->mutex); + + if (state == NXT_PORT_DETACHED_START) { + + /* + * A reason of the application's own, counted beside the router's + * count of requests it gave up on: neither may settle the other. + * + * Saturate rather than wrap. The count is bounded by the contexts + * of one worker, so the maximum is out of reach for a sound sender; + * a wrap would read as "no detached work" and hand a running worker + * back to the reaper. + */ + + if (nxt_slow_path(port->detached_app == UINT32_MAX)) { + alert = "sent a detached start at the maximum count"; + + } else { + port->detached_app++; + + if (port->detached == 0) { + port->detached = 1; + app->detached_processes++; + changed = 1; + + /* + * The worker may already have been parked as idle. libunit + * sends the start edge before the last response message, but + * that only orders the two on the wire: the response is + * answered by the engine that owns the request, this by the + * main thread, and a request that was failed rather than + * answered -- a "limits": {"timeout"} expiry, an error -- + * runs nxt_router_app_port_release() with no start edge in + * sight at all, so the port can have been sitting in + * idle_ports for as long as the application chose to keep + * running. + * + * Unwind it here, exactly as an acknowledgement does. + * Leaving it in place would be worse than not having this + * message: the reaper would QUIT a worker that is still + * executing -- the bug this exists to fix -- and would clear + * ->app on the way, so nxt_router_app_port_close() would + * never run and the count and the application reference + * below would never be settled. + */ + + start_process = nxt_router_app_port_busy(task, app, port, + "detached"); + } + } + + } else if (nxt_slow_path(port->detached_app == 0)) { + + /* + * An unmatched finish: a worker whose death has already settled the + * count, one otherwise out of step with the router, or a forged + * edge. Leave the count at zero and touch neither + * app->detached_processes nor the reference. + */ + + alert = "sent an unmatched detached finish"; + + } else if (--port->detached_app == 0) { + + /* + * The port leaves the detached state only when no reason is left: + * nothing here while a request the router gave up on is still + * running, and nothing for that request's answer while the + * application's own work is. + */ + if (port->detached_router == 0 && port->detached != 0) { + port->detached = 0; + app->detached_processes--; + changed = 1; + + /* + * Only now may this worker rejoin the idle economy. Its last + * response went out long ago, so nothing else will run this + * transition for it. + */ + + adjust_idle_timer = nxt_router_app_port_idle(task, app, port); + } + } + + nxt_thread_mutex_unlock(&app->mutex); + + /* + * The application can send a refused edge in a loop, so it is an alert + * once per port and debug output after that. + */ + + if (nxt_slow_path(alert != NULL)) { + if (port->detached_alerted == 0) { + port->detached_alerted = 1; + nxt_alert(task, "detached_handler: %PI %s", pid, alert); + + } else { + nxt_debug(task, "detached_handler: %PI %s", pid, alert); + } + } + + if (adjust_idle_timer) { + nxt_router_app_use(task, app, 1); + nxt_event_engine_post(app->engine, &app->adjust_idle_work); + } + + /* + * Holding the reference taken above: this is the only caller that can + * reach nxt_router_start_app_process() with no reference of its own. + */ + + if (start_process) { + nxt_router_start_app_process(task, app); + } + + /* + * A start edge that changed the state keeps the reference taken above: + * it is what keeps this worker's application alive for as long as the + * work runs. A detached worker holds no request, and a request is what + * otherwise holds the application -- by the time the work starts + * nxt_request_rpc_data_unlink() has already dropped its reference, so a + * configuration reload could free the application out from under a + * process still executing. The finish edge that changed the state + * returns it, and an edge that changed nothing returns only its own. + * One reference per process, then, not per counted edge: the inner + * starts and finishes take one and give it straight back, and a start + * that found the router's own mark already set leaves that reference to + * nxt_router_app_abandoned_settle(). + */ + + drop = 1; + + if (changed) { + drop = (state == NXT_PORT_DETACHED_START) ? 0 : 2; + } + + if (drop != 0) { + nxt_router_app_use(task, app, -drop); + } +} + + +/* + * The edge arrives on the router's own port, whichever engine answered the + * request: libunit sends both edges to its router port, the way it sends + * OOSM. That port is read on the main thread, which is where everything + * the edge touches lives -- rt->ports is read without a lock, which is only + * safe on the thread that adds to and removes from it, and an application + * is freed on app->engine, the same thread. One socket per worker also + * keeps a finish edge behind the start edge that preceded it, which the + * ports of two engines could not: their reads race, and the flag would be + * cleared under the next request's work. + */ + +static void +nxt_router_detached_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) +{ + /* + * The pid in the message is chosen by the sender, and every worker holds + * a copy of this port. Accept an edge only for the process that sent it, + * or a worker could pin another worker as detached or release one. The + * router port has no shared memory queue, so the message always comes + * from the socket and carries the kernel's credentials. + */ + if (nxt_slow_path(nxt_recv_msg_cmsg_pid(msg) != msg->port_msg.pid)) { + nxt_alert(task, "process %PI sent a detached edge for process %PI", + nxt_recv_msg_cmsg_pid(msg), msg->port_msg.pid); + return; + } + + if (nxt_slow_path(msg->buf == NULL + || nxt_buf_used_size(msg->buf) < (int) sizeof(uint8_t))) + { + nxt_alert(task, "detached_handler: %PI sent no state byte", + msg->port_msg.pid); + return; + } + + if (*msg->buf->mem.pos != NXT_PORT_DETACHED_START + && *msg->buf->mem.pos != NXT_PORT_DETACHED_FINISH) + { + nxt_alert(task, "detached_handler: invalid state byte %d", + (int) *msg->buf->mem.pos); + return; + } + + nxt_router_detached_apply(task, msg->port_msg.pid, *msg->buf->mem.pos); +} + + +#if (NXT_TESTS) + +/* For src/test/nxt_router_detached_test.c. */ + +void +nxt_router_test_detached_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) +{ + nxt_router_detached_handler(task, msg); +} + +#endif + + static void nxt_router_oosm_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) { @@ -6000,8 +9099,24 @@ nxt_router_oosm_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) nxt_debug(task, "oosm in %PI", msg->port_msg.pid); - process = nxt_runtime_process_find(task->thread->runtime, - msg->port_msg.pid); + /* + * Referenced, not just found: .oosm is in both the router main and the + * router worker port handler tables, and the reference has to span the + * incoming.mutex critical section below. Which engine actually receives + * a given OOSM depends on the port libunit sends it to, so this is not + * assumed either way -- what makes the reference necessary is that a + * worker engine can drop the last reference to the same process while + * this runs. + * + * It has to span the broadcast too, but note what that does and does not + * buy: it keeps the nxt_process_t allocated, and nothing more. The ports + * nxt_process_broadcast_shm_ack() reaches through process->ports are + * refcounted separately and that queue is walked here without a lock, so + * this reference does not make the walk itself safe. + */ + + process = nxt_runtime_process_ref(task->thread->runtime, + msg->port_msg.pid); if (nxt_slow_path(process == NULL)) { return; } @@ -6042,6 +9157,13 @@ nxt_router_oosm_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) if (ack) { nxt_process_broadcast_shm_ack(task, process); } + + /* + * The only release point: the process == NULL return above precedes the + * reference, and the loop breaks and continues but never returns. + */ + + nxt_process_use(task, process, -1); } @@ -6077,7 +9199,7 @@ nxt_router_get_mmap_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) get_mmap_msg = (nxt_port_msg_get_mmap_t *) msg->buf->mem.pos; - nxt_assert(port->type == NXT_PROCESS_APP); + /* nxt_router_get_mmap_check() refused a port of another type. */ if (nxt_slow_path(port->app == NULL)) { nxt_alert(task, "get_mmap_handler: app == NULL for reply port %PI:%d", diff --git a/src/nxt_router.h b/src/nxt_router.h index e2c5b83bc..735c21c52 100644 --- a/src/nxt_router.h +++ b/src/nxt_router.h @@ -14,6 +14,7 @@ typedef struct nxt_http_request_s nxt_http_request_t; #include +#include typedef struct nxt_http_action_s nxt_http_action_t; @@ -23,10 +24,20 @@ typedef struct nxt_upstream_s nxt_upstream_t; typedef struct nxt_upstreams_s nxt_upstreams_t; typedef struct nxt_router_access_log_s nxt_router_access_log_t; typedef struct nxt_router_access_log_format_s nxt_router_access_log_format_t; +typedef struct nxt_http_comp_conf_s nxt_http_comp_conf_t; +typedef struct nxt_http_comp_ctx_s nxt_http_comp_ctx_t; #define NXT_HTTP_ACTION_ERROR ((nxt_http_action_t *) -1) +/* + * The id of an application's shared port: the one whose queue every worker of + * the application reads, as opposed to a single worker's own port. A request + * message still sitting in that queue can be retracted; one sent to a worker + * port cannot. + */ +#define NXT_SHARED_PORT_ID 0xFFFFu + typedef struct { nxt_thread_spinlock_t lock; @@ -56,6 +67,15 @@ typedef struct { nxt_tstr_cond_t log_cond; nxt_router_access_log_t *access_log; nxt_router_access_log_format_t *log_format; + + /* + * Compression state, allocated from mem_pool above and reachable from a + * request as r->conf->socket_conf->router_conf->compression. NULL when + * this configuration has no "compression" block. Per-configuration + * rather than process-global so that reconfiguring compression away + * cannot leave a pointer into a freed pool behind (#167). + */ + nxt_http_comp_conf_t *compression; } nxt_router_conf_t; @@ -102,6 +122,28 @@ typedef struct { } nxt_joint_job_t; +/* + * Default "limits": {"start_timeout"}, in milliseconds: how long a worker may + * take to reach nxt_unit_init() before its start request is failed. Zero + * disables the bound, and that is the default deliberately. + * + * The window this measures is the whole of the application's own startup, not + * just Unit's part of it: every module runs user code before nxt_unit_init() + * -- nxt_python_start() imports the application module + * (src/python/nxt_python.c:303, via nxt_python_set_target()) before + * nxt_unit_init() (:365), nxt_java_start() deploys the webapp + * (src/nxt_java.c:417) before :441, the Node module loads the script before + * src/nodejs/unit-http/unit.cpp:279, and Go and "external" run main() first. + * A default bound would + * therefore fail a Django or ML import, or a JVM webapp deployment, that + * works today -- and a start failed while the worker is merely slow leaves + * that worker alive and orphaned, because the router cannot kill a process it + * has no pid for. So the bound is opt-in, and choosing a value is the + * operator's judgement about their own application's startup. + */ +#define NXT_APP_START_TIMEOUT 0 + + typedef struct { uint32_t use_count; nxt_app_t *app; @@ -129,6 +171,30 @@ struct nxt_app_s { uint32_t processes; uint32_t idle_processes; + /* + * Workers that answered a request and kept running. Counted in + * ->processes like any live worker -- a subset of it, not a separate + * population -- but never in ->idle_processes, because the reaper walks + * idle_ports and asserts that queue is non-empty while idle_processes + * exceeds spare_processes. + */ + uint32_t detached_processes; + + /* + * Application processes the router asked for, that were forked, and that + * it has neither a port nor a pid for: the ones a "limits": + * {"start_timeout"} deadline gave up on. Such a worker is not in + * ->processes, because only PROCESS_READY puts one there, and it is no + * longer in ->pending_processes, because its start attempt is over -- + * so without a third counter "processes": {"max"} would bound nothing, + * and every expired start would fork another OS child. + * + * A slot leaves again when the process it stands for is finally + * accounted for: it announces itself late, or it dies and the router is + * told. See nxt_router_app_start_expired() in src/nxt_router.c. + */ + uint32_t unaccounted_processes; + uint32_t max_processes; uint32_t spare_processes; uint32_t max_pending_processes; @@ -139,6 +205,14 @@ struct nxt_app_s { nxt_msec_t timeout; nxt_msec_t idle_timeout; + /* + * Bound on a START_PROCESS RPC: a worker that is forked but never calls + * nxt_unit_init() sends no PROCESS_READY, so without this nothing ever + * retires the RPC. Zero disables the bound. See + * nxt_router_start_timer_t in nxt_router.c. + */ + nxt_msec_t start_timeout; + nxt_str_t *targets; nxt_app_type_t type:8; @@ -159,6 +233,23 @@ struct nxt_app_s { }; +/* + * The gate on every application start. It lives here rather than in + * src/nxt_router.c so that the regression tests which turn on it -- the ones + * for the pending_processes accounting, #214 and #269 -- can call the very + * predicate the router uses instead of keeping a copy that silently drifts + * from it. + */ + +nxt_inline nxt_bool_t +nxt_router_app_can_start(nxt_app_t *app) +{ + return app->processes + app->pending_processes + + app->unaccounted_processes < app->max_processes + && app->pending_processes < app->max_pending_processes; +} + + typedef struct { size_t max_frame_size; nxt_msec_t read_timeout; @@ -199,6 +290,10 @@ typedef struct { nxt_msec_t proxy_send_timeout; nxt_msec_t proxy_read_timeout; + /* Minimum client transfer rates in bytes per second, 0 is off. */ + int32_t body_min_rate; + int32_t send_min_rate; + nxt_websocket_conf_t websocket_conf; nxt_str_t body_temp_path; @@ -227,6 +322,12 @@ typedef struct { nxt_joint_job_t *close_job; + /* + * The create job of this joint could not allocate the listen event, + * see nxt_router_listen_socket_release_stale(). + */ + uint8_t stale; /* 1 bit */ + nxt_upstream_t **upstreams; /* Modules configuraitons. */ @@ -262,9 +363,68 @@ void nxt_router_access_log_use(nxt_thread_spinlock_t *lock, nxt_router_access_log_t *access_log); void nxt_router_access_log_release(nxt_task_t *task, nxt_thread_spinlock_t *lock, nxt_router_access_log_t *access_log); +/* Not static so the descriptor-ownership test can drive it directly. */ +void nxt_router_new_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); + +/* Not static so the start-failure test can drive it directly. */ +void nxt_router_start_app_process_handler(nxt_task_t *task, nxt_port_t *port, + void *data); + void nxt_router_access_log_reopen_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); +#if (NXT_TESTS) +/* The sender check of the router main port, for the router sender test. */ +nxt_bool_t nxt_router_test_msg_sender_ok(nxt_task_t *task, + nxt_port_recv_msg_t *msg); + +extern nxt_uint_t nxt_router_test_senders_refused; + +/* The detached edge handler, for the sender check test. */ +void nxt_router_test_detached_handler(nxt_task_t *task, + nxt_port_recv_msg_t *msg); + +/* The port release, for the websocket accounting test. */ +void nxt_router_test_app_port_release(nxt_task_t *task, nxt_app_t *app, + nxt_port_t *port, nxt_apr_action_t action); +#endif + +#if (NXT_TESTS) +/* + * The config-apply start and the temporary configuration its failure path + * releases, for the start-deadline test. + */ +nxt_router_temp_conf_t *nxt_router_test_temp_conf(nxt_task_t *task); +void nxt_router_test_app_rpc_create(nxt_task_t *task, + nxt_router_temp_conf_t *tmcf, nxt_app_t *app); + +/* The release of stale listen joints, for the stale joint test. */ +void nxt_router_test_listen_socket_release_stale(nxt_task_t *task, + nxt_event_engine_t *engine, nxt_listen_socket_t *ls, + nxt_socket_conf_joint_t *keep); + +/* The listen socket create and update jobs, for the stale joint test. */ +void nxt_router_test_listen_socket_create(nxt_task_t *task, + nxt_joint_job_t *job, nxt_socket_conf_joint_t *joint); +void nxt_router_test_listen_socket_update(nxt_task_t *task, + nxt_joint_job_t *job, nxt_socket_conf_joint_t *joint); + +/* The request deadline handler, for the app-timeout test. */ +void nxt_router_test_app_timeout(nxt_task_t *task, void *obj, void *data); + +/* The application response decoder, for the response parse test. */ +nxt_int_t nxt_router_test_response_header_parse(nxt_task_t *task, + nxt_http_request_t *r, nxt_buf_t *b); + +/* + * The app message builder with the prefix of app->type, for the protocol + * length test. The status is an nxt_uint_t: nxt_http_status_t is not + * visible here. + */ +nxt_buf_t *nxt_router_test_prepare_msg(nxt_task_t *task, nxt_http_request_t *r, + nxt_app_t *app, nxt_uint_t *status); +#endif + extern nxt_router_t *nxt_router; diff --git a/src/nxt_router_access_log.c b/src/nxt_router_access_log.c index 1ec1a2d2c..45ae8f61a 100644 --- a/src/nxt_router_access_log.c +++ b/src/nxt_router_access_log.c @@ -158,10 +158,8 @@ static nxt_router_access_log_format_t * nxt_router_access_log_format_create(nxt_task_t *task, nxt_router_conf_t *rtcf, nxt_conf_value_t *value) { - size_t size; uint32_t i, n, next; nxt_str_t name, str, *dst; - nxt_bool_t has_js; nxt_conf_value_t *cv; nxt_router_access_log_member_t *member; nxt_router_access_log_format_t *format; @@ -179,68 +177,40 @@ nxt_router_access_log_format_create(nxt_task_t *task, nxt_router_conf_t *rtcf, if (value != NULL) { if (nxt_conf_type(value) == NXT_CONF_OBJECT) { - next = 0; - has_js = 0; - n = nxt_conf_object_members_count(value); - for ( ;; ) { + member = nxt_mp_alloc(rtcf->mem_pool, + n * sizeof(nxt_router_access_log_member_t)); + if (nxt_slow_path(member == NULL)) { + return NULL; + } + + next = 0; + + for (i = 0; i < n; i++) { cv = nxt_conf_next_object_member(value, &name, &next); if (cv == NULL) { break; } - nxt_conf_get_string(cv, &str); - - if (nxt_tstr_is_js(&str)) { - has_js = 1; - } - } - - if (has_js) { - member = nxt_mp_alloc(rtcf->mem_pool, - n * sizeof(nxt_router_access_log_member_t)); - if (nxt_slow_path(member == NULL)) { + dst = nxt_str_dup(rtcf->mem_pool, &member[i].name, &name); + if (nxt_slow_path(dst == NULL)) { return NULL; } - next = 0; - - for (i = 0; i < n; i++) { - cv = nxt_conf_next_object_member(value, &name, &next); - if (cv == NULL) { - break; - } - - dst = nxt_str_dup(rtcf->mem_pool, &member[i].name, &name); - if (nxt_slow_path(dst == NULL)) { - return NULL; - } - - nxt_conf_get_string(cv, &str); + nxt_conf_get_string(cv, &str); - member[i].tstr = nxt_tstr_compile(rtcf->tstr_state, &str, - NXT_TSTR_LOGGING); - if (nxt_slow_path(member[i].tstr == NULL)) { - return NULL; - } + member[i].tstr = nxt_tstr_compile(rtcf->tstr_state, &str, + NXT_TSTR_LOGGING); + if (nxt_slow_path(member[i].tstr == NULL)) { + return NULL; } - - format->nmembers = n; - format->member = member; - - return format; } - size = nxt_conf_json_length(value, NULL); - - str.start = nxt_mp_nget(rtcf->mem_pool, size); - if (nxt_slow_path(str.start == NULL)) { - return NULL; - } + format->nmembers = n; + format->member = member; - str.length = nxt_conf_json_print(str.start, value, NULL) - - str.start; + return format; } else { nxt_conf_get_string(value, &str); @@ -250,8 +220,16 @@ nxt_router_access_log_format_create(nxt_task_t *task, nxt_router_conf_t *rtcf, str = default_format; } + /* + * NXT_TSTR_ESCAPE is set here and not for the object format above: the + * object format prints through nxt_conf_json_print(), which escapes what + * it prints, and escaping twice would show the escape instead of the + * value. + */ + format->tstr = nxt_tstr_compile(rtcf->tstr_state, &str, - NXT_TSTR_LOGGING | NXT_TSTR_NEWLINE); + NXT_TSTR_LOGGING | NXT_TSTR_NEWLINE + | NXT_TSTR_ESCAPE); if (nxt_slow_path(format->tstr == NULL)) { return NULL; } @@ -355,6 +333,27 @@ nxt_router_access_log_json(nxt_task_t *task, nxt_http_request_t *r, } } + /* + * The value is whatever the variable resolved to, and for a request + * header that is bytes the client chose: RFC 9110 Sect. 5.5 admits + * obs-text, so 0x80-0xFF reach here unfiltered. JSON text is UTF-8 + * (RFC 8259 Sect. 8.1), and nxt_conf_json_escape() escapes the quote + * and the backslash but copies every other byte through -- so an + * unencodable one leaves a record that is unforgeable and still + * unreadable, which a strict consumer drops whole. + * + * Sanitize here rather than in the serializer. The same serializer + * writes state/conf.json and answers GET /config, where the bytes are + * the operator's and have to round-trip exactly: a "share" path is + * allowed to be non-UTF-8 on Linux, and rewriting one would point the + * router at a different file after a restart. + */ + + ret = nxt_utf8_sanitize(r->mem_pool, &str, &str); + if (nxt_slow_path(ret != NXT_OK)) { + return NXT_ERROR; + } + nxt_conf_set_member_string(value, &member->name, &str, i); } @@ -449,6 +448,9 @@ nxt_router_access_log_ready(nxt_task_t *task, nxt_port_recv_msg_t *msg, access_log->fd = msg->fd[0]; + /* The access log owns the descriptor now. */ + msg->fd[0] = -1; + nxt_work_queue_add(&task->thread->engine->fast_work_queue, nxt_router_conf_apply, task, tmcf, NULL); } @@ -619,6 +621,7 @@ nxt_router_access_log_reopen_ready(nxt_task_t *task, nxt_port_recv_msg_t *msg, } nxt_fd_close(msg->fd[0]); + msg->fd[0] = -1; nxt_mp_release(reopen->mem_pool); } diff --git a/src/nxt_router_request.h b/src/nxt_router_request.h index 6e226bae8..993f3c433 100644 --- a/src/nxt_router_request.h +++ b/src/nxt_router_request.h @@ -7,10 +7,27 @@ #define _NXT_ROUTER_REQUEST_H_INCLUDED_ +typedef enum { + /* Not offered to nxt_app_queue_cancel() yet. */ + NXT_MSG_QUEUED = 0, + /* The router took the message back; no worker will ever see it. */ + NXT_MSG_RETRACTED, + /* A worker claimed the queue slot first; the request is running. */ + NXT_MSG_CLAIMED, +} nxt_msg_cancel_t; + + typedef struct { nxt_buf_t *buf; nxt_fd_t body_fd; uint32_t tracking_cookie; + + /* + * What nxt_app_queue_cancel() answered, kept because the CAS cannot be + * asked twice: a win and a loss both leave the queue item's tracking + * word at 0, so a second call would report a claim that never happened. + */ + nxt_msg_cancel_t cancel; } nxt_msg_info_t; @@ -20,6 +37,13 @@ typedef enum { NXT_APR_GOT_RESPONSE, NXT_APR_UPGRADE, NXT_APR_CLOSE, + /* + * The close of a request that NXT_APR_UPGRADE turned into a websocket. + * It is the counterpart of that action: it drops the reference the way + * NXT_APR_CLOSE does and also takes the session off the worker's + * active_websockets count. + */ + NXT_APR_WEBSOCKET_CLOSE, } nxt_apr_action_t; @@ -33,6 +57,17 @@ typedef struct { nxt_http_request_t *request; nxt_msg_info_t msg_info; + /* + * The worker's main port and its application, when the router gave up on + * a request that worker was running. Each is held by a reference of its + * own until the worker answers, because neither the request nor its + * accounting outlives the decision to stop waiting: the port stays out of + * the idle economy until then and the application has to stay alive for + * that state to be settled at all. + */ + nxt_port_t *abandoned_port; + nxt_app_t *abandoned_app; + nxt_bool_t rpc_cancel; } nxt_request_rpc_data_t; diff --git a/src/nxt_runtime.c b/src/nxt_runtime.c index 67efddcf6..91de56231 100644 --- a/src/nxt_runtime.c +++ b/src/nxt_runtime.c @@ -11,6 +11,9 @@ #include #include #include +#if (NXT_HAVE_OTEL) +#include +#endif static nxt_int_t nxt_runtime_inherited_listen_sockets(nxt_task_t *task, @@ -65,6 +68,9 @@ nxt_runtime_create(nxt_task_t *task) task->thread->runtime = rt; rt->mem_pool = mp; + rt->control_uid = (nxt_uid_t) -1; + rt->control_gid = (nxt_gid_t) -1; + nxt_thread_mutex_create(&rt->processes_mutex); rt->services = nxt_services_init(mp); @@ -131,10 +137,6 @@ nxt_runtime_create(nxt_task_t *task) goto fail; } - if (nxt_slow_path(nxt_var_index_init() != NXT_OK)) { - goto fail; - } - nxt_work_queue_add(&task->thread->engine->fast_work_queue, nxt_runtime_start, task, rt, NULL); @@ -298,9 +300,6 @@ nxt_runtime_event_engines(nxt_task_t *task, nxt_runtime_t *rt) thread = task->thread; thread->engine = engine; -#if 0 - thread->fiber = &engine->fibers->fiber; -#endif engine->id = rt->last_engine_id++; engine->mem_pool = nxt_mp_create(1024, 128, 256, 32); @@ -354,6 +353,19 @@ nxt_runtime_start(nxt_task_t *task, void *obj, void *data) goto fail; } + if (rt->capabilities.unknown) { + /* + * nxt_capability_set() could only reach stderr: it runs from + * nxt_runtime_conf_init(), before the log file exists. Repeat it + * here so the record survives in unit.log too. + */ + nxt_log(task, NXT_LOG_WARN, "capget() failed; process " + "capabilities are unknown and will not be used: user and " + "group switching and \"rootfs\" isolation are disabled " + "for applications that do not enable the \"credential\" " + "namespace, and applications run as uid %d", (int) nxt_euid); + } + if (nxt_runtime_event_engine_change(task, rt) != NXT_OK) { goto fail; } @@ -467,6 +479,32 @@ nxt_runtime_quit(nxt_task_t *task, nxt_uint_t status) } +/* + * Close every idle conn of the engine on shutdown. + * + * Each conn is handed to its own protocol through c->read_state's + * close_handler, the same contract nxt_conn_accept_close_idle_handler() + * (nxt_conn_accept.c) uses to reclaim idle conns under fd pressure. That + * handler ends in the protocol's free path -- nxt_h1p_conn_free() for a + * router conn, nxt_controller_conn_free() for a controller conn -- which + * releases everything the runtime cannot know about: the sockaddr from the + * engine cache (nxt_conn_accept_alloc()), and for the router the listener + * reference taken in nxt_conn_accept(), without which a draining listener + * never reaches its final release. It also satisfies CONN-INV-2 (a + * write_state whose ready_handler releases the conn is installed before + * nxt_conn_close()), which a bare nxt_conn_close() here would violate: a + * keep-alive h1p conn still carries its request-send state, whose + * ready_handler is a no-op with c->write == NULL, so the conn would leak. + * + * A conn with no read_state has been accepted but its listen handler + * (nxt_conn_accept(): lev->listen->handler, queued on the read work queue) + * has not run yet, so no protocol owns it. It is left alone: that queued + * work item still references it, so closing or freeing it here would be a + * use-after-free once the item runs, and nxt_conn_close() on a conn with + * write_state == NULL crashes in nxt_conn_close_handler(). It gets its + * protocol state when that item runs and is closed by the next pass. + */ + static void nxt_runtime_close_idle_connections(nxt_event_engine_t *engine) { @@ -485,24 +523,53 @@ nxt_runtime_close_idle_connections(nxt_event_engine_t *engine) next = nxt_queue_next(link); c = nxt_queue_link_data(link, nxt_conn_t, link); - if (!c->socket.read_ready) { - /* - * Unlink and clear the tracking state immediately, before - * scheduling the async close. nxt_runtime_quit() calls this on - * every shutdown continuation, so a conn left on idle_connections - * would be re-selected and re-closed on the next pass before its - * async close handler runs -- a double nxt_conn_close(), i.e. a - * use-after-free. Clearing c->idle to TRACK_NONE also stops the - * close handler from unlinking the same conn a second time - * (P4.5). Iteration stays safe: `next` was captured above. - */ - nxt_conn_untrack(engine, c); - nxt_conn_close(engine, c); + if (c->socket.read_ready) { + continue; } + + if (c->read_state == NULL || c->read_state->close_handler == NULL) { + nxt_debug(c->socket.task, "idle connection: %d not yet " + "initialized, skipped", c->socket.fd); + continue; + } + + /* + * Unlink and clear the tracking state immediately, before the + * protocol schedules its async close. nxt_runtime_quit() calls + * this on every shutdown continuation, so a conn left on + * idle_connections would be re-selected and re-closed on the next + * pass before its async close handler runs -- a double + * nxt_conn_close(), i.e. a use-after-free. Clearing c->idle to + * TRACK_NONE also stops the close handler from unlinking the same + * conn a second time (P4.5), and makes the nxt_conn_active() at + * the top of the h1p close handlers a no-op. Iteration stays + * safe: `next` was captured above. + */ + nxt_conn_untrack(engine, c); + + c->read_state->close_handler(c->socket.task, c, c->socket.data); } } +#if (NXT_TESTS) + +/* + * src/test/nxt_runtime_idle_close_test.c drives the idle-connection close + * path (M-11) without going through a full nxt_runtime_quit(): it checks + * that every idle conn is closed through its own protocol close handler, + * and that a conn without one is left alone. + */ + +void +nxt_runtime_test_close_idle_connections(nxt_event_engine_t *engine) +{ + nxt_runtime_close_idle_connections(engine); +} + +#endif + + /* * Allocate a one-byte port-message body carrying rt->quit_mode (a * nxt_port_quit_mode_t value, see nxt_port.h). libunit parses this @@ -614,6 +681,16 @@ nxt_runtime_exit(nxt_task_t *task, void *obj, void *data) return; } +#if (NXT_HAVE_OTEL) + /* + * Only the router builds spans, and this is the last point before exit() + * at which the batch processor can still be flushed. + */ + if (rt->type == NXT_PROCESS_ROUTER) { + nxt_otel_shutdown(task); + } +#endif + if (rt->type == NXT_PROCESS_MAIN) { if (rt->pid_file != NULL) { nxt_file_delete(rt->pid_file); @@ -1537,8 +1614,9 @@ nxt_current_directory(nxt_mp_t *mp) static nxt_int_t nxt_runtime_pid_file_create(nxt_task_t *task, nxt_file_name_t *pid_file) { - ssize_t length; - nxt_int_t n; + size_t size, written; + ssize_t n; + nxt_int_t ret; nxt_file_t file; u_char pid[NXT_INT64_T_LEN + nxt_length("\n")]; @@ -1548,17 +1626,35 @@ nxt_runtime_pid_file_create(nxt_task_t *task, nxt_file_name_t *pid_file) nxt_fs_mkdir_p_dirname(pid_file, 0755); - n = nxt_file_open(task, &file, O_WRONLY, O_CREAT | O_TRUNC, - NXT_FILE_DEFAULT_ACCESS); + ret = nxt_file_open(task, &file, O_WRONLY, O_CREAT | O_TRUNC, + NXT_FILE_DEFAULT_ACCESS); - if (n != NXT_OK) { + if (ret != NXT_OK) { return NXT_ERROR; } - length = nxt_sprintf(pid, pid + sizeof(pid), "%PI%n", nxt_pid) - pid; + size = nxt_sprintf(pid, pid + sizeof(pid), "%PI%n", nxt_pid) - pid; - if (nxt_file_write(&file, pid, length, 0) != length) { - return NXT_ERROR; + /* + * pwrite() may store less than it was asked for, so resume from where it + * stopped rather than report a short write as a failure. A zero return + * carries no errno and cannot make progress, so it ends the loop. + */ + + for (written = 0; written < size; written += n) { + n = nxt_file_write(&file, pid + written, size - written, written); + + if (nxt_slow_path(n <= 0)) { + /* nxt_file_write() logs the errno; a zero return has none. */ + if (n == 0) { + nxt_alert(task, "write(\"%FN\") stored %uz of %uz bytes", + file.name, written, size); + } + + nxt_file_close(task, &file); + + return NXT_ERROR; + } } nxt_file_close(task, &file); @@ -1572,13 +1668,10 @@ nxt_runtime_process_free(nxt_runtime_t *rt, nxt_process_t *process) { nxt_process_t *child; - if (process->link.next != NULL) { - nxt_queue_remove(&process->link); - } + nxt_process_unlink(process); nxt_queue_each(child, &process->children, nxt_process_t, link) { - nxt_queue_remove(&child->link); - child->link.next = NULL; + nxt_process_unlink(child); } nxt_queue_loop; nxt_assert(process->use_count == 0); @@ -1727,6 +1820,16 @@ nxt_runtime_process_lhq_pid(nxt_lvlhsh_query_t *lhq, nxt_pid_t *pid) } +/* + * Returns the process without a reference: the pointer is only valid for as + * long as nothing can drop the last reference to it. That holds when the + * caller runs on the single engine that owns the process -- the main, the + * prototype or the controller process, each of which has exactly one event + * engine -- and nowhere else. Any caller that can run on a router worker + * engine must use nxt_runtime_process_ref() instead; the router's main + * engine can be freeing the process concurrently. + */ + nxt_process_t * nxt_runtime_process_find(nxt_runtime_t *rt, nxt_pid_t pid) { @@ -1752,6 +1855,47 @@ nxt_runtime_process_find(nxt_runtime_t *rt, nxt_pid_t pid) } +/* + * Same lookup, but the caller adopts a reference to the result and must drop + * it with nxt_process_use(task, process, -1). The find and the increment + * share one rt->processes_mutex critical section, and nxt_process_use() + * unlinks the process from rt->processes under that same mutex when the count + * reaches zero -- so a lookup that still sees the process in the hash cannot + * be racing its teardown, and the reference it takes cannot be an increment + * on an object that is already being freed. + * + * Cost, since this sits on the shared-memory message path: the find already + * took the mutex, so the increment itself is free; the matching release in + * nxt_process_use() is one additional acquisition of the same uncontended + * global mutex per lookup. + */ + +nxt_process_t * +nxt_runtime_process_ref(nxt_runtime_t *rt, nxt_pid_t pid) +{ + nxt_process_t *process; + nxt_lvlhsh_query_t lhq; + + process = NULL; + + nxt_runtime_process_lhq_pid(&lhq, &pid); + + nxt_thread_mutex_lock(&rt->processes_mutex); + + if (nxt_lvlhsh_find(&rt->processes, &lhq) == NXT_OK) { + process = lhq.value; + process->use_count++; + + } else { + nxt_thread_log_debug("process %PI not found", pid); + } + + nxt_thread_mutex_unlock(&rt->processes_mutex); + + return process; +} + + static nxt_process_t * nxt_runtime_process_get(nxt_runtime_t *rt, nxt_pid_t pid) { diff --git a/src/nxt_runtime.h b/src/nxt_runtime.h index 82ba71bb9..e24624382 100644 --- a/src/nxt_runtime.h +++ b/src/nxt_runtime.h @@ -78,6 +78,8 @@ struct nxt_runtime_s { mode_t control_mode; const char *control_user; const char *control_group; + nxt_uid_t control_uid; /* resolved, or -1 */ + nxt_gid_t control_gid; /* resolved, or -1 */ nxt_str_t certs; nxt_str_t scripts; @@ -116,6 +118,7 @@ void nxt_runtime_process_unlink_locked(nxt_runtime_t *rt, nxt_process_t *process); nxt_process_t *nxt_runtime_process_find(nxt_runtime_t *rt, nxt_pid_t pid); +nxt_process_t *nxt_runtime_process_ref(nxt_runtime_t *rt, nxt_pid_t pid); nxt_process_t *nxt_runtime_process_first(nxt_runtime_t *rt, nxt_lvlhsh_each_t *lhe); @@ -140,6 +143,9 @@ NXT_EXPORT nxt_port_t *nxt_runtime_port_find(nxt_runtime_t *rt, nxt_pid_t pid, /* STUB */ nxt_int_t nxt_runtime_controller_socket(nxt_task_t *task, nxt_runtime_t *rt); +nxt_bool_t nxt_controller_peer_allowed(nxt_uid_t uid, nxt_gid_t gid, + const nxt_gid_t *groups, nxt_uint_t ngroups, nxt_uid_t euid, + nxt_uid_t ctl_uid, nxt_gid_t ctl_gid); nxt_str_t *nxt_current_directory(nxt_mp_t *mp); @@ -183,4 +189,10 @@ extern nxt_module_init_t nxt_init_modules[]; extern nxt_uint_t nxt_init_modules_n; +#if (NXT_TESTS) +/* The idle-connection close pass, for the idle close dispatch test. */ +void nxt_runtime_test_close_idle_connections(nxt_event_engine_t *engine); +#endif + + #endif /* _NXT_RUNTIME_H_INCLIDED_ */ diff --git a/src/nxt_script.c b/src/nxt_script.c index 0b36f289a..967b82fe1 100644 --- a/src/nxt_script.c +++ b/src/nxt_script.c @@ -7,6 +7,9 @@ #include #include #include +#include +#include +#include #include @@ -32,6 +35,8 @@ static nxt_script_t *nxt_script_get(nxt_task_t *task, nxt_str_t *name, nxt_fd_t fd); static nxt_conf_value_t *nxt_script_details(nxt_mp_t *mp, nxt_script_t *cert); static void nxt_script_buf_completion(nxt_task_t *task, void *obj, void *data); +static nxt_int_t nxt_script_store_name(nxt_buf_t *b, size_t rest, + nxt_str_t *name); static nxt_lvlhsh_t nxt_script_info; @@ -159,17 +164,15 @@ nxt_script_info_init(nxt_task_t *task, nxt_array_t *scripts) item = scripts->elts; for (i = 0; i < scripts->nelts; i++) { - script = nxt_script_get(task, &item->name, item->fd); + script = nxt_script_get(task, &item[i].name, item[i].fd); if (nxt_slow_path(script == NULL)) { continue; } - (void) nxt_script_info_save(&item->name, script); + (void) nxt_script_info_save(&item[i].name, script); nxt_script_destroy(script); - - item++; } } @@ -378,7 +381,11 @@ nxt_script_store_load(nxt_task_t *task, nxt_mp_t *mp) name.length = nxt_strlen(de->d_name); name.start = (u_char *) de->d_name; - if (nxt_str_eq(&name, ".", 1) || nxt_str_eq(&name, "..", 2)) { + /* + * No module name starts with ".": this skips "." and "..", and + * the temporary file of a store that did not finish. + */ + if (name.start[0] == '.') { continue; } @@ -530,10 +537,59 @@ nxt_script_buf_completion(nxt_task_t *task, void *obj, void *data) } +/* + * Check the name at the start of "b": it ends with a NUL inside the + * message, and "rest" bytes follow the NUL. A name is not empty, is at most + * NXT_SCRIPT_NAME_MAX_LENGTH bytes, has no "/" and does not start with ".". + * The sender is a process of Unit, but main puts the name into a path. + */ +static nxt_int_t +nxt_script_store_name(nxt_buf_t *b, size_t rest, nxt_str_t *name) +{ + u_char *nul; + nxt_span_t span; + const u_char *start; + + if (b == NULL) { + return NXT_ERROR; + } + + nxt_span_init(&span, b->mem.pos, b->mem.free); + + nul = memchr(b->mem.pos, '\0', nxt_span_len(&span)); + if (nul == NULL) { + return NXT_ERROR; + } + + name->length = nul - b->mem.pos; + + if (nxt_span_take(&span, name->length + 1, &start) != 0 + || nxt_span_len(&span) != rest + || name->length == 0 + || name->length > NXT_SCRIPT_NAME_MAX_LENGTH + || start[0] == '.' + || memchr(start, '/', name->length) != NULL) + { + return NXT_ERROR; + } + + name->start = b->mem.pos; + + return NXT_OK; +} + + +/* + * Only the router reads modules: it opens every module a configuration + * uses. The file is opened read-only and is never created, so a read + * cannot change the store. The controller stores a module with + * nxt_script_store_put(). + */ void nxt_script_store_get_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) { u_char *p; + size_t size; nxt_int_t ret; nxt_str_t name; nxt_file_t file; @@ -544,8 +600,8 @@ nxt_script_store_get_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) /* * Look up the sender's port via the kernel-validated PID * (SCM_CREDENTIALS). msg->port_msg.pid is self-declared, so using - * it would let a compromised worker spoof the controller / router - * and pull arbitrary script material out of main. + * it would let a compromised worker spoof the router and pull + * arbitrary script material out of main. */ port = nxt_runtime_port_find(task->thread->runtime, nxt_recv_msg_cmsg_pid(msg), @@ -558,10 +614,8 @@ nxt_script_store_get_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) return; } - if (nxt_slow_path(port->type != NXT_PROCESS_CONTROLLER - && port->type != NXT_PROCESS_ROUTER)) - { - nxt_alert(task, "process %PI cannot store scripts", + if (nxt_slow_path(port->type != NXT_PROCESS_ROUTER)) { + nxt_alert(task, "process %PI cannot read scripts", nxt_recv_msg_cmsg_pid(msg)); nxt_port_recv_msg_close_fds(msg); return; @@ -574,15 +628,24 @@ nxt_script_store_get_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) rt = task->thread->runtime; - if (nxt_slow_path(rt->certs.start == NULL)) { + if (nxt_slow_path(rt->scripts.start == NULL)) { nxt_alert(task, "no scripts storage directory"); goto error; } - name.start = msg->buf->mem.pos; - name.length = nxt_strlen(name.start); + if (nxt_slow_path(nxt_script_store_name(msg->buf, 0, &name) != NXT_OK)) { + nxt_alert(task, "script_get_handler: invalid script name"); + goto error; + } + + /* "" and its NUL. */ + if (nxt_slow_path(nxt_size_add(rt->scripts.length, name.length, &size) != 0 + || nxt_size_add(size, 1, &size) != 0)) + { + goto error; + } - file.name = nxt_malloc(rt->scripts.length + name.length + 1); + file.name = nxt_malloc(size); if (nxt_slow_path(file.name == NULL)) { goto error; } @@ -590,7 +653,7 @@ nxt_script_store_get_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) p = nxt_cpymem(file.name, rt->scripts.start, rt->scripts.length); p = nxt_cpymem(p, name.start, name.length + 1); - ret = nxt_file_open(task, &file, NXT_FILE_RDWR, NXT_FILE_CREATE_OR_OPEN, + ret = nxt_file_open(task, &file, NXT_FILE_RDONLY, NXT_FILE_OPEN, NXT_FILE_OWNER_ACCESS); nxt_free(file.name); @@ -620,6 +683,205 @@ nxt_script_store_get_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) } +/* + * Send "mbuf" to main to store it as the module "name". The module goes in + * a shared memory segment, as a certificate bundle does. The port message + * holds the NUL-terminated name, followed by the size. "handler" gets + * RPC_READY when the file is on disk, and RPC_ERROR or a NULL message + * otherwise. + */ +void +nxt_script_store_put(nxt_task_t *task, nxt_str_t *name, nxt_buf_mem_t *mbuf, + nxt_mp_t *mp, nxt_port_rpc_handler_t handler, void *ctx) +{ + void *mem; + size_t size; + uint32_t stream; + nxt_fd_t fd; + nxt_int_t ret; + nxt_buf_t *b; + nxt_port_t *main_port, *ctl_port; + nxt_runtime_t *rt; + + size = nxt_buf_mem_used_size(mbuf); + + fd = nxt_shm_open(task, size); + if (nxt_slow_path(fd == -1)) { + goto fail; + } + + mem = nxt_mem_mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (nxt_slow_path(mem == MAP_FAILED)) { + goto fail; + } + + nxt_memcpy(mem, mbuf->pos, size); + nxt_mem_munmap(mem, size); + + b = nxt_buf_mem_alloc(mp, name->length + 1 + sizeof(size_t), 0); + if (nxt_slow_path(b == NULL)) { + goto fail; + } + + b->completion_handler = nxt_script_buf_completion; + + nxt_buf_cpystr(b, name); + *b->mem.free++ = '\0'; + b->mem.free = nxt_cpymem(b->mem.free, &size, sizeof(size_t)); + + rt = task->thread->runtime; + main_port = rt->port_by_type[NXT_PROCESS_MAIN]; + ctl_port = rt->port_by_type[NXT_PROCESS_CONTROLLER]; + + stream = nxt_port_rpc_register_handler(task, ctl_port, handler, handler, + -1, ctx); + if (nxt_slow_path(stream == 0)) { + goto fail; + } + + ret = nxt_port_socket_write(task, main_port, + NXT_PORT_MSG_SCRIPT_STORE + | NXT_PORT_MSG_CLOSE_FD, + fd, stream, ctl_port->id, b); + + if (nxt_slow_path(ret != NXT_OK)) { + nxt_port_rpc_cancel(task, ctl_port, stream); + goto fail; + } + + /* See nxt_script_store_get(). */ + nxt_mp_retain(mp); + + return; + +fail: + + /* The port layer owns the descriptor only after a successful write. */ + if (fd != -1) { + nxt_fd_close(fd); + } + + handler(task, NULL, ctx); +} + + +/* + * Main's side of nxt_script_store_put(). The store child of main writes + * the module to "scripts/.store.tmp", flushes it and renames it over + * "scripts/", so a reader sees the old file or the new one, never a + * partial file. Main answers the controller when the child exits. The + * controller is unprivileged, so main checks the sender, the name and the + * size again. + */ +void +nxt_script_store_put_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) +{ + size_t size; + u_char *mem; + nxt_str_t name; + nxt_file_t file; + nxt_port_t *port; + nxt_span_t span; + nxt_runtime_t *rt; + nxt_main_store_job_t *job; + + port = nxt_runtime_port_find(task->thread->runtime, + nxt_recv_msg_cmsg_pid(msg), + msg->port_msg.reply_port); + + if (nxt_slow_path(port == NULL || port->type != NXT_PROCESS_CONTROLLER)) { + nxt_alert(task, "process %PI cannot store scripts", + nxt_recv_msg_cmsg_pid(msg)); + nxt_port_recv_msg_close_fds(msg); + return; + } + + rt = task->thread->runtime; + + mem = NULL; + + if (nxt_slow_path(rt->scripts.start == NULL)) { + nxt_alert(task, "no scripts storage directory"); + goto fail; + } + + if (nxt_slow_path(msg->fd[0] == -1)) { + nxt_alert(task, "script_store_handler: invalid shm fd"); + goto fail; + } + + /* The message holds the NUL-terminated name, followed by the size. */ + + if (nxt_slow_path(nxt_script_store_name(msg->buf, sizeof(size_t), &name) + != NXT_OK)) + { + nxt_alert(task, "script_store_handler: invalid script name"); + goto fail; + } + + nxt_span_init(&span, name.start + name.length + 1, msg->buf->mem.free); + + if (nxt_slow_path(nxt_span_copy(&span, &size, sizeof(size_t)) != 0)) { + nxt_alert(task, "script_store_handler: no script size"); + goto fail; + } + + if (nxt_slow_path(size == 0 || size > NXT_SCRIPT_STORE_MAX_SIZE)) { + nxt_alert(task, "script_store_handler: invalid script size %uz", + size); + goto fail; + } + + /* + * Copy the module with pread() instead of mapping the segment: a + * sender can shrink a mapped segment, and a read past its end is + * SIGBUS in main. A short pread() is only an error. + */ + + mem = nxt_malloc(size); + if (nxt_slow_path(mem == NULL)) { + goto fail; + } + + nxt_memzero(&file, sizeof(nxt_file_t)); + file.fd = msg->fd[0]; + + /* nxt_file_read() logs the name when pread() fails; it must not be NULL. */ + file.name = name.start; + + if (nxt_slow_path(nxt_file_read(&file, mem, size, 0) != (ssize_t) size)) { + nxt_alert(task, "script_store_handler: short script read"); + goto fail; + } + + job = nxt_main_store_job_create(NXT_MAIN_STORE_PUT, "JS module", + &rt->scripts, &name); + if (nxt_slow_path(job == NULL)) { + goto fail; + } + + job->data = mem; + job->size = size; + job->reply_pid = port->pid; + job->reply_port = port->id; + job->stream = msg->port_msg.stream; + + nxt_port_recv_msg_close_fds(msg); + + nxt_main_store_submit(task, job); + + return; + +fail: + + (void) nxt_port_socket_write(task, port, NXT_PORT_MSG_RPC_ERROR, -1, + msg->port_msg.stream, 0, NULL); + + nxt_free(mem); + nxt_port_recv_msg_close_fds(msg); +} + + void nxt_script_store_delete(nxt_task_t *task, nxt_str_t *name, nxt_mp_t *mp) { @@ -642,14 +904,18 @@ nxt_script_store_delete(nxt_task_t *task, nxt_str_t *name, nxt_mp_t *mp) } +/* + * The delete is a job of the store child, as a store is. Jobs run in the + * order they come, so a delete cannot run before the rename(2) of an + * earlier store of the same module, which would bring the module back. + */ void nxt_script_store_delete_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) { - u_char *p; - nxt_str_t name; - nxt_port_t *ctl_port; - nxt_runtime_t *rt; - nxt_file_name_t *path; + nxt_str_t name; + nxt_port_t *ctl_port; + nxt_runtime_t *rt; + nxt_main_store_job_t *job; rt = task->thread->runtime; ctl_port = rt->port_by_type[NXT_PROCESS_CONTROLLER]; @@ -672,18 +938,16 @@ nxt_script_store_delete_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) return; } - name.start = msg->buf->mem.pos; - name.length = nxt_strlen(name.start); - - path = nxt_malloc(rt->scripts.length + name.length + 1); - - if (nxt_fast_path(path != NULL)) { - p = nxt_cpymem(path, rt->scripts.start, rt->scripts.length); - p = nxt_cpymem(p, name.start, name.length + 1); + if (nxt_slow_path(nxt_script_store_name(msg->buf, 0, &name) != NXT_OK)) { + nxt_alert(task, "script_delete_handler: invalid script name"); + return; + } - (void) nxt_file_delete(path); + job = nxt_main_store_job_create(NXT_MAIN_STORE_DELETE, "JS module", + &rt->scripts, &name); - nxt_free(path); + if (nxt_fast_path(job != NULL)) { + nxt_main_store_submit(task, job); } } diff --git a/src/nxt_script.h b/src/nxt_script.h index ffefc108f..a300dfb2d 100644 --- a/src/nxt_script.h +++ b/src/nxt_script.h @@ -8,6 +8,16 @@ #define _NXT_SCRIPT_INCLUDED_ +/* + * A module name is a file name in the scripts directory of the state, so + * NAME_MAX bounds it. + */ +#define NXT_SCRIPT_NAME_MAX_LENGTH 255 + +/* Main copies a module into its memory to store it. */ +#define NXT_SCRIPT_STORE_MAX_SIZE (16 * 1024 * 1024) + + typedef struct nxt_script_s nxt_script_t; nxt_script_t *nxt_script_new(nxt_task_t *task, nxt_str_t *name, u_char *data, @@ -25,9 +35,13 @@ void nxt_script_store_release(nxt_array_t *scripts); void nxt_script_store_get(nxt_task_t *task, nxt_str_t *name, nxt_mp_t *mp, nxt_port_rpc_handler_t handler, void *ctx); +void nxt_script_store_put(nxt_task_t *task, nxt_str_t *name, + nxt_buf_mem_t *mbuf, nxt_mp_t *mp, nxt_port_rpc_handler_t handler, + void *ctx); void nxt_script_store_delete(nxt_task_t *task, nxt_str_t *name, nxt_mp_t *mp); void nxt_script_store_get_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); +void nxt_script_store_put_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); void nxt_script_store_delete_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); diff --git a/src/nxt_select_engine.c b/src/nxt_select_engine.c index 6f7600125..8f268a802 100644 --- a/src/nxt_select_engine.c +++ b/src/nxt_select_engine.c @@ -14,6 +14,8 @@ static void nxt_select_enable(nxt_event_engine_t *engine, nxt_fd_event_t *ev); static void nxt_select_disable(nxt_event_engine_t *engine, nxt_fd_event_t *ev); static nxt_bool_t nxt_select_close(nxt_event_engine_t *engine, nxt_fd_event_t *ev); +static void nxt_select_cancel_changes(nxt_event_engine_t *engine, + nxt_fd_event_t *ev); static void nxt_select_enable_read(nxt_event_engine_t *engine, nxt_fd_event_t *ev); static void nxt_select_enable_write(nxt_event_engine_t *engine, @@ -42,6 +44,7 @@ const nxt_event_interface_t nxt_select_engine = { nxt_select_disable, nxt_select_disable, nxt_select_close, + nxt_select_cancel_changes, nxt_select_enable_read, nxt_select_enable_write, nxt_select_disable_read, @@ -124,6 +127,17 @@ nxt_select_close(nxt_event_engine_t *engine, nxt_fd_event_t *ev) } +/* + * select() applies every change as it is made and batches nothing, so this + * engine never sets ->changing and holds no pending change to drop. + */ + +static void +nxt_select_cancel_changes(nxt_event_engine_t *engine, nxt_fd_event_t *ev) +{ +} + + static void nxt_select_enable_read(nxt_event_engine_t *engine, nxt_fd_event_t *ev) { diff --git a/src/nxt_service.c b/src/nxt_service.c index cc9f8d894..33bc8f978 100644 --- a/src/nxt_service.c +++ b/src/nxt_service.c @@ -44,16 +44,6 @@ static const nxt_service_t nxt_services[] = { { "SSL/TLS", "openssl", &nxt_openssl_lib }, #endif -#if (NXT_HAVE_GNUTLS) - { "SSL/TLS", "GnuTLS", &nxt_gnutls_lib }, - { "SSL/TLS", "gnutls", &nxt_gnutls_lib }, -#endif - -#if (NXT_HAVE_CYASSL) - { "SSL/TLS", "CyaSSL", &nxt_cyassl_lib }, - { "SSL/TLS", "cyassl", &nxt_cyassl_lib }, -#endif - }; diff --git a/src/nxt_socket.h b/src/nxt_socket.h index 69b09039f..d2fdebcff 100644 --- a/src/nxt_socket.h +++ b/src/nxt_socket.h @@ -114,9 +114,15 @@ NXT_EXPORT nxt_int_t nxt_socketpair_create(nxt_task_t *task, NXT_EXPORT void nxt_socketpair_close(nxt_task_t *task, nxt_socket_t *pair); NXT_EXPORT ssize_t nxt_socketpair_send(nxt_fd_event_t *ev, nxt_fd_t *fd, nxt_iobuf_t *iob, nxt_uint_t niob); +ssize_t nxt_socketpair_send_ex(nxt_fd_event_t *ev, nxt_fd_t *fd, + nxt_iobuf_t *iob, nxt_uint_t niob, nxt_bool_t peer_may_be_gone); NXT_EXPORT ssize_t nxt_socketpair_recv(nxt_fd_event_t *ev, nxt_iobuf_t *iob, nxt_uint_t niob, void *oob); +#if (NXT_TESTS) +NXT_EXPORT void nxt_socketpair_test_send_fail(nxt_err_t err, nxt_uint_t n); +#endif + #define nxt_socket_nonblocking(task, fd) \ nxt_fd_nonblocking(task, fd) diff --git a/src/nxt_socket_msg.c b/src/nxt_socket_msg.c index 3b35ab292..5b019970d 100644 --- a/src/nxt_socket_msg.c +++ b/src/nxt_socket_msg.c @@ -47,10 +47,16 @@ nxt_recvmsg(nxt_socket_t s, nxt_iobuf_t *iob, nxt_uint_t niob, msg.msg_control = oob->buf; msg.msg_controllen = sizeof(oob->buf); + /* Cleared so that MSG_CTRUNC below is read from what recvmsg() wrote. */ + msg.msg_flags = 0; + n = recvmsg(s, &msg, 0); if (nxt_fast_path(n != -1)) { oob->size = msg.msg_controllen; + + /* Recorded for nxt_socket_msg_oob_get(); see NXT_OOB_RECV_SIZE. */ + oob->truncated = (msg.msg_flags & MSG_CTRUNC) != 0; } return n; diff --git a/src/nxt_socket_msg.h b/src/nxt_socket_msg.h index 8a36245cc..fcc82139f 100644 --- a/src/nxt_socket_msg.h +++ b/src/nxt_socket_msg.h @@ -42,9 +42,56 @@ typedef struct cmsgcred nxt_socket_cred_t; #endif +/* + * NXT_OOB_RECV_SIZE is the largest control block Unit can legitimately be + * handed: the two descriptors a message may carry -- nxt_port_recv_msg_t + * has exactly two fd slots and nxt_socket_msg_oob_init() never attaches + * more -- plus, where the platform passes credentials, the one credential + * cmsg the kernel or the sender adds. A receive buffer of that size can + * therefore never truncate a well-formed message: MSG_CTRUNC means either + * that the kernel could not deliver all of the control data it had for this + * message (an SCM_RIGHTS it could not install in full, typically under + * RLIMIT_NOFILE pressure), or that a peer sent more control data than any + * Unit message type uses. Both are receive errors, not conditions a + * handler can be expected to notice: what reaches it is a well-formed, + * authenticated message whose descriptor is simply -1. + * + * What a kernel does with the descriptors it could not fit is not uniform, + * and only the Linux behaviour is verified here: scm_detach_fds() installs + * the ones that fit into this process and only then raises MSG_CTRUNC, so + * they are open, owned, and unreachable unless the accessor reports them -- + * which is why nxt_socket_msg_oob_get() and nxt_socket_msg_oob_get_fds() + * fill fd[] before failing the message. A kernel that instead discards the + * rights leaves nothing to leak, and the same code is correct there because + * fd[] simply stays -1. The refusal itself does not depend on which way a + * platform goes; the reporting is what the Linux semantics make necessary. + * MSG_CTRUNC is POSIX, so the flag is available wherever Unit builds, but + * only the Linux side of this is covered by a test (see + * src/test/nxt_port_ctrunc_test.c, whose kernel-driven cases skip on macOS + * and whose queue case is gated on NXT_HAVE_UCRED). + */ + +#define NXT_OOB_ALIGN (sizeof(size_t)) + + +/* + * "buf" holds a control block that CMSG_FIRSTHDR()/CMSG_NXTHDR() walk as + * struct cmsghdr, so its address has to carry that struct's alignment. + * Nothing here states that alignment: it comes from "buf" following a size_t + * directly, since alignof(struct cmsghdr) == alignof(size_t) on the ABIs Unit + * builds for (cmsg_len is a size_t on glibc, socklen_t elsewhere). A field + * inserted between the two moves "buf" off that alignment and every cmsg + * access becomes undefined -- UBSan reports it as "member access within + * misaligned address ... for type 'struct cmsghdr'". Keep new members after + * "buf", as "truncated" is. nxt_aligned() is not a substitute: it expands to + * nothing outside GCC and clang. + */ + typedef struct { - size_t size; - u_char buf[NXT_OOB_RECV_SIZE]; + size_t size; + u_char buf[NXT_OOB_RECV_SIZE] nxt_aligned(NXT_OOB_ALIGN); + /* recvmsg() reported MSG_CTRUNC: the control data is incomplete. */ + nxt_bool_t truncated; } nxt_recv_oob_t; @@ -83,6 +130,21 @@ NXT_CMSG_NXTHDR(struct msghdr *msgh, struct cmsghdr *cmsg) } +/* + * Put a receive buffer into the "carries nothing" state. Callers that fill + * oob themselves, or that recycle a buffer, use this rather than assigning + * ->size alone, so that every field describing the control data is retired + * together. + */ + +nxt_inline void +nxt_socket_msg_oob_reset(nxt_recv_oob_t *oob) +{ + oob->size = 0; + oob->truncated = 0; +} + + nxt_inline void nxt_socket_msg_oob_init(nxt_send_oob_t *oob, int *fds) { @@ -153,6 +215,21 @@ nxt_socket_msg_oob_get_fds(nxt_recv_oob_t *oob, nxt_fd_t *fd) cmsg != NULL; cmsg = NXT_CMSG_NXTHDR(&msg, cmsg)) { + /* + * Fail closed on a header that does not fit the bytes actually + * received. CMSG_FIRSTHDR() only checks that msg_controllen covers + * a struct cmsghdr, so the first header's cmsg_len can still claim + * more data than arrived, and a block the kernel cut short is + * exactly where that happens. Reading CMSG_DATA() over that claim + * would run off the end of oob->buf. + */ + if (nxt_slow_path(cmsg->cmsg_len < CMSG_LEN(0) + || (size_t) ((u_char *) cmsg - oob->buf) + + cmsg->cmsg_len > oob->size)) + { + return NXT_ERROR; + } + size = cmsg->cmsg_len - CMSG_LEN(0); if (cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SCM_RIGHTS) { @@ -162,10 +239,19 @@ nxt_socket_msg_oob_get_fds(nxt_recv_oob_t *oob, nxt_fd_t *fd) nxt_memcpy(fd, CMSG_DATA(cmsg), size); - return NXT_OK; + break; } } + /* + * Checked after the loop, not before it: a truncated control block can + * still have delivered a descriptor, and the caller can only close what + * it has been told about. + */ + if (nxt_slow_path(oob->truncated)) { + return NXT_ERROR; + } + return NXT_OK; } @@ -177,7 +263,7 @@ nxt_socket_msg_oob_get(nxt_recv_oob_t *oob, nxt_fd_t *fd, nxt_pid_t *pid) struct msghdr msg; struct cmsghdr *cmsg; - if (oob->size == 0) { + if (oob->size == 0 && !oob->truncated) { return NXT_OK; } @@ -192,6 +278,21 @@ nxt_socket_msg_oob_get(nxt_recv_oob_t *oob, nxt_fd_t *fd, nxt_pid_t *pid) cmsg != NULL; cmsg = NXT_CMSG_NXTHDR(&msg, cmsg)) { + /* + * Fail closed on a header that does not fit the bytes actually + * received. CMSG_FIRSTHDR() only checks that msg_controllen covers + * a struct cmsghdr, so the first header's cmsg_len can still claim + * more data than arrived, and a block the kernel cut short is + * exactly where that happens. Reading CMSG_DATA() over that claim + * would run off the end of oob->buf. + */ + if (nxt_slow_path(cmsg->cmsg_len < CMSG_LEN(0) + || (size_t) ((u_char *) cmsg - oob->buf) + + cmsg->cmsg_len > oob->size)) + { + return NXT_ERROR; + } + size = cmsg->cmsg_len - CMSG_LEN(0); if (cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SCM_RIGHTS) { @@ -222,6 +323,11 @@ nxt_socket_msg_oob_get(nxt_recv_oob_t *oob, nxt_fd_t *fd, nxt_pid_t *pid) #endif } + /* See nxt_socket_msg_oob_get_fds() on why this follows the loop. */ + if (nxt_slow_path(oob->truncated)) { + return NXT_ERROR; + } + #if (NXT_CRED_USECMSG) /* For platforms supporting credential passing, it's enforced */ if (nxt_slow_path(*pid == -1)) { diff --git a/src/nxt_socketpair.c b/src/nxt_socketpair.c index 45274b788..da386b92c 100644 --- a/src/nxt_socketpair.c +++ b/src/nxt_socketpair.c @@ -82,9 +82,77 @@ nxt_socketpair_close(nxt_task_t *task, nxt_socket_t *pair) } +#if (NXT_TESTS) +static nxt_err_t nxt_socketpair_test_send_err; +static nxt_uint_t nxt_socketpair_test_send_err_left; + + +/* + * Make the next "n" calls of sendmsg() in nxt_socketpair_send() fail with + * "err" without running it. The call after that runs the real sendmsg(). + * Two is what expresses "the retry fails too", and zero injects nothing. + * src/test/nxt_port_fail_test.c uses it for the errors a socketpair between + * two live processes will not produce on demand. + */ + +void +nxt_socketpair_test_send_fail(nxt_err_t err, nxt_uint_t n) +{ + nxt_socketpair_test_send_err = err; + nxt_socketpair_test_send_err_left = (err != 0) ? n : 0; +} + +#endif + + +/* + * The level to log a failed send at. A QUIT goes to a process that can + * have exited already, at once or later from the queue. So EPIPE and the + * like on a QUIT are expected. They are logged at the level that + * nxt_socket_error_level() gives, which is info. A SOCK_DGRAM pair (the + * fallback in nxt_socketpair_create()) reports a gone peer as ECONNREFUSED. + * That is info too. All other failures are an alert. + * + * The caller marks a QUIT with "peer_may_be_gone" + * (nxt_port_send_msg_t.peer_may_be_gone). This is the only way to know + * it for the READ_QUEUE wake-up that nxt_port_socket_write2() sends for a + * QUIT in a shared queue. A QUIT header in iob[0] counts as well, because + * nxt_port_write_msgs() sends the port message header first. + */ +static nxt_uint_t +nxt_socketpair_send_error_level(nxt_iobuf_t *iob, nxt_uint_t niob, + nxt_bool_t peer_may_be_gone, nxt_err_t err) +{ + nxt_port_msg_t *msg; + + if (!peer_may_be_gone + && niob != 0 && iob[0].iov_len >= sizeof(nxt_port_msg_t)) + { + msg = iob[0].iov_base; + + peer_may_be_gone = (msg->type == _NXT_PORT_MSG_QUIT); + } + + if (peer_may_be_gone) { + return (err == NXT_ECONNREFUSED) ? NXT_LOG_INFO + : nxt_socket_error_level(err); + } + + return NXT_LOG_ALERT; +} + + ssize_t nxt_socketpair_send(nxt_fd_event_t *ev, nxt_fd_t *fd, nxt_iobuf_t *iob, nxt_uint_t niob) +{ + return nxt_socketpair_send_ex(ev, fd, iob, niob, 0); +} + + +ssize_t +nxt_socketpair_send_ex(nxt_fd_event_t *ev, nxt_fd_t *fd, nxt_iobuf_t *iob, + nxt_uint_t niob, nxt_bool_t peer_may_be_gone) { ssize_t n; nxt_err_t err; @@ -93,14 +161,41 @@ nxt_socketpair_send(nxt_fd_event_t *ev, nxt_fd_t *fd, nxt_iobuf_t *iob, nxt_socket_msg_oob_init(&oob, fd); for ( ;; ) { - n = nxt_sendmsg(ev->fd, iob, niob, &oob); + n = -1; + err = 0; - err = (n == -1) ? nxt_socket_errno : 0; +#if (NXT_TESTS) + if (nxt_socketpair_test_send_err_left > 0) { + err = nxt_socketpair_test_send_err; + + nxt_socketpair_test_send_err_left--; + + if (nxt_socketpair_test_send_err_left == 0) { + nxt_socketpair_test_send_err = 0; + } + } +#endif + + if (nxt_fast_path(err == 0)) { + n = nxt_sendmsg(ev->fd, iob, niob, &oob); + + err = (n == -1) ? nxt_socket_errno : 0; + } nxt_debug(ev->task, "sendmsg(%d, %FD, %FD, %ui): %z", ev->fd, fd[0], fd[1], niob, n); if (n > 0) { + /* + * Everywhere else in the tree a non-zero socket.error means the + * socket is broken (nxt_conn_write.c, nxt_conn_read.c, + * nxt_conn_close.c), and nxt_conn_io_write() stores 0 on a + * successful write for that reason. Keep the field meaning the + * same thing here: the errno recorded below is only for the + * NXT_AGAIN that follows it. + */ + ev->error = 0; + return n; } @@ -114,9 +209,19 @@ nxt_socketpair_send(nxt_fd_event_t *ev, nxt_fd_t *fd, nxt_iobuf_t *iob, /* * Returned (at least on OSX) when trying to send many small messages. + * + * ENOMEM is the same condition by another name: the kernel could not + * allocate for this call. Linux answers it when the SCM_RIGHTS + * copy of a descriptor-carrying message cannot be allocated, and + * nothing of the message has gone out. The socket and the peer are + * fine, so the caller must keep the message and try again, as it + * does after ENOBUFS. The "default" arm below is for errors the + * socket will not recover from; answering NXT_ERROR here made a + * caller drop a message a live peer was still owed (#392). */ case NXT_ENOBUFS: - nxt_debug(ev->task, "sendmsg(%d) no buffers", ev->fd); + case NXT_ENOMEM: + nxt_debug(ev->task, "sendmsg(%d) cannot allocate %E", ev->fd, err); break; case NXT_EINTR: @@ -124,12 +229,23 @@ nxt_socketpair_send(nxt_fd_event_t *ev, nxt_fd_t *fd, nxt_iobuf_t *iob, continue; default: - nxt_alert(ev->task, "sendmsg(%d, %FD, %FD, %ui) failed %E", - ev->fd, fd[0], fd[1], niob, err); + nxt_log(ev->task, + nxt_socketpair_send_error_level(iob, niob, + peer_may_be_gone, err), + "sendmsg(%d, %FD, %FD, %ui) failed %E", + ev->fd, fd[0], fd[1], niob, err); + + /* + * The one exit where the socket really is broken, so it is the + * one that must leave the errno behind: see the note above on + * what a non-zero socket.error means. + */ + ev->error = err; return NXT_ERROR; } + ev->error = err; ev->write_ready = 0; return NXT_AGAIN; diff --git a/src/nxt_source.h b/src/nxt_source.h deleted file mode 100644 index 0b4658dd7..000000000 --- a/src/nxt_source.h +++ /dev/null @@ -1,42 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) NGINX, Inc. - */ - -#ifndef _NXT_SOURCE_H_INCLUDED_ -#define _NXT_SOURCE_H_INCLUDED_ - - -/* - * A source handler should store a pointer to a passed source hook, but not - * the hook's values themselves, because a source filter may change the values. - */ -typedef struct { - void *context; - nxt_work_handler_t filter; -} nxt_source_hook_t; - - -typedef void (*nxt_source_handler_t)(void *source_context, - nxt_source_hook_t *query); - - -#define nxt_source_filter(thr, wq, task, next, out) \ - do { \ - if (thr->engine->batch != 0) { \ - nxt_thread_work_queue_add(thr, wq, nxt_source_filter_handler, \ - task, next, out); \ - \ - } else { \ - (next)->filter(task, (next)->context, out); \ - } \ - \ - } while (0) - - -NXT_EXPORT void nxt_source_filter_handler(nxt_task_t *task, void *obj, - void *data); - - -#endif /* _NXT_SOURCE_H_INCLUDED_ */ diff --git a/src/nxt_span.h b/src/nxt_span.h new file mode 100644 index 000000000..6418e2d14 --- /dev/null +++ b/src/nxt_span.h @@ -0,0 +1,97 @@ + +/* + * Copyright (C) FreeUnit Community + */ + +#ifndef _NXT_SPAN_H_INCLUDED_ +#define _NXT_SPAN_H_INCLUDED_ + + +/* + * nxt_span_t is a bounded read cursor over a buffer. "pos" is the next + * byte to consume. "end" is one past the last valid byte. It is for + * parsers that walk untrusted, length-prefixed input, such as shared + * memory port messages and libunit request buffers. Such a parser must + * check each advance against the remaining bytes. It must not trust a + * length field that comes from the peer. + * + * A short read means that not enough bytes are left. A partial tail means + * that the span ends in the middle of a whole record. Both are reported + * as a failure. nxt_span_take() never returns a pointer that reads past + * "end". + */ + +typedef struct { + const u_char *pos; + const u_char *end; +} nxt_span_t; + + +nxt_inline void +nxt_span_init(nxt_span_t *span, const u_char *start, const u_char *end) +{ + span->pos = start; + span->end = end; +} + + +nxt_inline size_t +nxt_span_len(const nxt_span_t *span) +{ + return (size_t) (span->end - span->pos); +} + + +/* + * Takes "size" bytes off the front of the span. Returns a pointer to them + * in "out". Returns 1 when fewer than "size" bytes remain. This covers a + * short buffer. It also covers a partial tail that is left after whole + * records were taken. On failure, "span" and "out" do not change. + */ + +nxt_inline int +nxt_span_take(nxt_span_t *span, size_t size, const u_char **out) +{ + size_t avail; + + avail = nxt_span_len(span); + + if (nxt_slow_path(size > avail)) { + return 1; + } + + *out = span->pos; + span->pos += size; + + return 0; +} + + +/* + * Copies "size" bytes off the front of the span into "dst". It fails in + * the same way as nxt_span_take(). On a short or partial tail, the span + * and "dst" do not change. A copy of 0 bytes always succeeds and does + * not touch "dst". It also does not call memcpy(), because "pos" of an + * empty span may be NULL and memcpy(dst, NULL, 0) is undefined. + */ + +nxt_inline int +nxt_span_copy(nxt_span_t *span, void *dst, size_t size) +{ + const u_char *src; + + if (size == 0) { + return 0; + } + + if (nxt_span_take(span, size, &src) != 0) { + return 1; + } + + nxt_memcpy(dst, src, size); + + return 0; +} + + +#endif /* _NXT_SPAN_H_INCLUDED_ */ diff --git a/src/nxt_sprintf.c b/src/nxt_sprintf.c index 2e29e80ef..8b1db8b25 100644 --- a/src/nxt_sprintf.c +++ b/src/nxt_sprintf.c @@ -28,7 +28,6 @@ * * %PI nxt_pid_t, process id * %PT nxt_tid_t, thread id - * %PF nxt_fid_t, fiber id * %PH pthread_t handle returned by pthread_self() * * %s null-terminated string @@ -507,12 +506,6 @@ nxt_vsprintf(u_char *buf, u_char *end, const char *fmt, va_list args) ui64 = (uint64_t) (uintptr_t) va_arg(args, nxt_tid_t); sign = 0; goto number; -#if 0 - case 'F': - ui64 = (uint64_t) va_arg(args, nxt_fid_t); - sign = 0; - goto number; -#endif case 'H': ui64 = (uint64_t) (uintptr_t) va_arg(args, pthread_t); spf.hex = HEXADECIMAL; diff --git a/src/nxt_status.c b/src/nxt_status.c index 92cbf2e65..213c05792 100644 --- a/src/nxt_status.c +++ b/src/nxt_status.c @@ -35,11 +35,23 @@ nxt_status_get(nxt_status_report_t *report, nxt_mp_t *mp) static const nxt_str_t reqs_str = nxt_string("requests"); static const nxt_str_t total_str = nxt_string("total"); static const nxt_str_t apps_str = nxt_string("applications"); + static const nxt_str_t telemetry_str = nxt_string("telemetry"); + static const nxt_str_t spans_str = nxt_string("spans"); + static const nxt_str_t exported_str = nxt_string("exported"); + static const nxt_str_t failed_str = nxt_string("failed"); static const nxt_str_t procs_str = nxt_string("processes"); static const nxt_str_t run_str = nxt_string("running"); static const nxt_str_t start_str = nxt_string("starting"); + static const nxt_str_t unacc_str = nxt_string("unaccounted"); + static const nxt_str_t detached_str = nxt_string("detached"); - status = nxt_conf_create_object(mp, 4); + /* + * modules, connections, requests, applications -- plus "telemetry" when + * OTel is built in and currently configured. It is omitted rather than + * zeroed otherwise, so a build or a configuration without telemetry + * reports exactly what it did before. + */ + status = nxt_conf_create_object(mp, 4 + (report->otel_configured != 0)); if (nxt_slow_path(status == NULL)) { return NULL; } @@ -148,6 +160,36 @@ nxt_status_get(nxt_status_report_t *report, nxt_mp_t *mp) nxt_conf_set_member_integer(obj, &total_str, report->requests, 0); + if (report->otel_configured) { + nxt_conf_value_t *spans; + + /* + * "is my telemetry reaching the collector?" needs both halves: a + * failure count alone cannot tell a healthy pipeline from one that + * has never exported anything. Both are cumulative since the live + * tracer provider was installed, so a telemetry reconfiguration + * restarts them. + */ + obj = nxt_conf_create_object(mp, 1); + if (nxt_slow_path(obj == NULL)) { + return NULL; + } + + nxt_conf_set_member(status, &telemetry_str, obj, idx++); + + spans = nxt_conf_create_object(mp, 2); + if (nxt_slow_path(spans == NULL)) { + return NULL; + } + + nxt_conf_set_member(obj, &spans_str, spans, 0); + + nxt_conf_set_member_integer(spans, &exported_str, + report->otel_spans_exported, 0); + nxt_conf_set_member_integer(spans, &failed_str, + report->otel_spans_failed, 1); + } + apps = nxt_conf_create_object(mp, report->apps_count); if (nxt_slow_path(apps == NULL)) { return NULL; @@ -171,7 +213,7 @@ nxt_status_get(nxt_status_report_t *report, nxt_mp_t *mp) return NULL; } - obj = nxt_conf_create_object(mp, 3); + obj = nxt_conf_create_object(mp, 5); if (nxt_slow_path(obj == NULL)) { return NULL; } @@ -180,7 +222,11 @@ nxt_status_get(nxt_status_report_t *report, nxt_mp_t *mp) nxt_conf_set_member_integer(obj, &run_str, app->processes, 0); nxt_conf_set_member_integer(obj, &start_str, app->pending_processes, 1); - nxt_conf_set_member_integer(obj, &idle_str, app->idle_processes, 2); + nxt_conf_set_member_integer(obj, &unacc_str, + app->unaccounted_processes, 2); + nxt_conf_set_member_integer(obj, &idle_str, app->idle_processes, 3); + nxt_conf_set_member_integer(obj, &detached_str, + app->detached_processes, 4); obj = nxt_conf_create_object(mp, 1); if (nxt_slow_path(obj == NULL)) { diff --git a/src/nxt_status.h b/src/nxt_status.h index a99ac7d0e..cdfb6fb3a 100644 --- a/src/nxt_status.h +++ b/src/nxt_status.h @@ -12,7 +12,9 @@ typedef struct { uint32_t active_requests; uint32_t pending_processes; uint32_t processes; + uint32_t unaccounted_processes; uint32_t idle_processes; + uint32_t detached_processes; } nxt_status_app_t; @@ -22,6 +24,19 @@ typedef struct { uint64_t closed_conns; uint64_t requests; + /* + * OpenTelemetry span export health, filled in by the router. The two + * counters are only meaningful when otel_configured is set: a build + * without OTel support, or one where "settings/telemetry" is absent, + * leaves them zero and reports no "telemetry" object in /status at all. + * + * Counted in spans since the live tracer provider was installed, i.e. + * reset by a telemetry reconfiguration. + */ + uint64_t otel_spans_exported; + uint64_t otel_spans_failed; + uint8_t otel_configured; + size_t apps_count; nxt_status_app_t apps[]; } nxt_status_report_t; diff --git a/src/nxt_string.c b/src/nxt_string.c index 551ba72b9..aa1c2a4fc 100644 --- a/src/nxt_string.c +++ b/src/nxt_string.c @@ -341,18 +341,20 @@ nxt_rmemstrn(const u_char *s, const u_char *end, const char *ss, size_t length) } +/* The length of the string without trailing CR, LF, SP and HTAB. */ + size_t -nxt_str_strip(const u_char *start, u_char *end) +nxt_str_strip(const u_char *start, const u_char *end) { - u_char *p; + const u_char *p; - for (p = end - 1; p >= start; p--) { - if (*p != '\r' && *p != '\n') { + for (p = end; p != start; p--) { + if (p[-1] != '\r' && p[-1] != '\n' && p[-1] != ' ' && p[-1] != '\t') { break; } } - return (p + 1) - start; + return p - start; } @@ -412,7 +414,7 @@ nxt_strverscmp(const u_char *s1, const u_char *s2) c1 = *s1++; c2 = *s2++; - /* Fall through. */ + nxt_fallthrough; case st_num: @@ -441,7 +443,7 @@ nxt_strverscmp(const u_char *s1, const u_char *s2) return 1; } - /* Fall through. */ + nxt_fallthrough; case st_frac: default: diff --git a/src/nxt_string.h b/src/nxt_string.h index a5bb1f79c..bfa667735 100644 --- a/src/nxt_string.h +++ b/src/nxt_string.h @@ -58,6 +58,15 @@ NXT_EXPORT void nxt_memcpy_upcase(u_char *dst, const u_char *src, nxt_inline void * nxt_cpymem(void *dst, const void *src, size_t length) { + /* + * memcpy() declares "src" non-null even for a zero-length copy, and + * callers reach here with the NULL start of an empty nxt_str_t. The + * copy has nothing to do in that case and dst + 0 is dst. + */ + if (length == 0) { + return dst; + } + return memcpy(dst, src, length) + length; } @@ -88,7 +97,7 @@ NXT_EXPORT u_char *nxt_memcasestrn(const u_char *s, const u_char *end, const char *ss, size_t length); NXT_EXPORT u_char *nxt_rmemstrn(const u_char *s, const u_char *end, const char *ss, size_t length); -NXT_EXPORT size_t nxt_str_strip(const u_char *start, u_char *end); +NXT_EXPORT size_t nxt_str_strip(const u_char *start, const u_char *end); typedef struct { @@ -122,9 +131,16 @@ NXT_EXPORT nxt_str_t *nxt_str_dup(nxt_mp_t *mp, nxt_str_t *dst, NXT_EXPORT char *nxt_str_cstrz(nxt_mp_t *mp, const nxt_str_t *src); +/* + * memcmp() declares both pointers non-null whatever the length, and a + * zero-length nxt_str_t carries a NULL start, so the length test has to + * short-circuit the call rather than only decide its result. Two empty + * strings are equal, which is what falling through to "true" gives. + */ #define nxt_strstr_eq(s1, s2) \ (((s1)->length == (s2)->length) \ - && (memcmp((s1)->start, (s2)->start, (s1)->length) == 0)) + && ((s1)->length == 0 \ + || memcmp((s1)->start, (s2)->start, (s1)->length) == 0)) #define nxt_strcasestr_eq(s1, s2) \ @@ -132,12 +148,16 @@ NXT_EXPORT char *nxt_str_cstrz(nxt_mp_t *mp, const nxt_str_t *src); && (nxt_memcasecmp((s1)->start, (s2)->start, (s1)->length) == 0)) +/* The same zero-length short circuit as nxt_strstr_eq(), for memcmp(). */ + #define nxt_str_eq(s, p, _length) \ - (((s)->length == _length) && (memcmp((s)->start, p, _length) == 0)) + (((s)->length == _length) \ + && ((s)->length == 0 || memcmp((s)->start, p, _length) == 0)) #define nxt_str_start(s, p, _length) \ - (((s)->length >= _length) && (memcmp((s)->start, p, _length) == 0)) + (((s)->length >= _length) \ + && ((_length) == 0 || memcmp((s)->start, p, _length) == 0)) #define nxt_strchr_eq(s, c) \ diff --git a/src/nxt_test_build.h b/src/nxt_test_build.h index 6513e748f..2c563f2c3 100644 --- a/src/nxt_test_build.h +++ b/src/nxt_test_build.h @@ -74,8 +74,12 @@ struct signalfd_siginfo { uint64_t ssi_stime; /* System CPU time consumed (SIGCHLD) */ uint64_t ssi_addr; /* Address that generated signal (for hardware-generated signals) */ - uint8_t pad[8]; /* Pad size to 128 bytes (allow for - additional fields in the future) */ + uint8_t pad[8]; /* Padding; the real kernel struct pads + to 128 bytes, but the exact size does + not matter here since this is a + syntax-check-only stub for non-Linux + test builds, never used for a real + signalfd(2) read */ }; diff --git a/src/nxt_thread.h b/src/nxt_thread.h index 53b2d8c0f..80fdd477d 100644 --- a/src/nxt_thread.h +++ b/src/nxt_thread.h @@ -148,14 +148,6 @@ struct nxt_thread_s { nxt_event_engine_t *engine; void *data; -#if 0 - /* - * Although pointer to a current fiber should be a property of - * engine->fibers, its placement here eliminates 2 memory accesses. - */ - nxt_fiber_t *fiber; -#endif - nxt_random_t random; }; diff --git a/src/nxt_timer.c b/src/nxt_timer.c index cb94b77c8..47f61511d 100644 --- a/src/nxt_timer.c +++ b/src/nxt_timer.c @@ -192,7 +192,7 @@ nxt_timer_changes_commit(nxt_event_engine_t *engine) break; } - /* Fall through. */ + nxt_fallthrough; case NXT_TIMER_DELETE: nxt_debug(timer->task, "timer rbtree delete: %M±%d", diff --git a/src/nxt_tls.h b/src/nxt_tls.h index 0667ade3c..fbe878006 100644 --- a/src/nxt_tls.h +++ b/src/nxt_tls.h @@ -98,19 +98,17 @@ extern const nxt_tls_lib_t nxt_openssl_lib; void nxt_cdecl nxt_openssl_log_error(nxt_task_t *task, nxt_uint_t level, const char *fmt, ...); u_char *nxt_openssl_copy_error(u_char *p, u_char *end); -#endif -#if (NXT_HAVE_GNUTLS) -extern const nxt_tls_lib_t nxt_gnutls_lib; -#endif +#if (NXT_TESTS) +/* + * Makes nxt_openssl_server_init() take one of its failure paths, for + * src/test/nxt_openssl_server_init_test.c. + */ +#define NXT_OPENSSL_TEST_FAIL_CTX_NEW 1 +#define NXT_OPENSSL_TEST_FAIL_BIO_NEW 2 -#if (NXT_HAVE_CYASSL) -extern const nxt_tls_lib_t nxt_cyassl_lib; +NXT_EXPORT extern nxt_uint_t nxt_openssl_test_fail; #endif - -#if (NXT_HAVE_POLARSSL) -extern const nxt_tls_lib_t nxt_polar_lib; #endif - #endif /* _NXT_TLS_H_INCLUDED_ */ diff --git a/src/nxt_tstr.c b/src/nxt_tstr.c index d939d44e2..b99ef1de6 100644 --- a/src/nxt_tstr.c +++ b/src/nxt_tstr.c @@ -261,11 +261,66 @@ nxt_tstr_query_init(nxt_tstr_query_t **query_p, nxt_tstr_state_t *state, } +#if (NXT_HAVE_NJS) + +/* + * A JS template produces the whole record, so there is no template text to + * keep apart from the values -- escape all of it. The trailing newline is + * part of that value: nxt_js_add_tpl() appends it inside the generated + * function, so escaping it too would turn every record into one line. + */ + +static nxt_int_t +nxt_tstr_escape(nxt_mp_t *mp, nxt_str_t *val, nxt_bool_t newline) +{ + size_t length, size; + u_char *p; + nxt_str_t escaped; + + length = val->length; + + if (newline && length > 0 && val->start[length - 1] == '\n') { + length--; + + } else { + newline = 0; + } + + size = nxt_var_escape_length(val->start, length); + + if (size == length) { + return NXT_OK; + } + + escaped.length = size + newline; + + escaped.start = nxt_mp_nget(mp, escaped.length); + if (nxt_slow_path(escaped.start == NULL)) { + return NXT_ERROR; + } + + p = nxt_var_escape(escaped.start, val->start, length); + + if (newline) { + *p++ = '\n'; + } + + nxt_assert(p == escaped.start + escaped.length); + + *val = escaped; + + return NXT_OK; +} + +#endif + + nxt_int_t nxt_tstr_query(nxt_task_t *task, nxt_tstr_query_t *query, nxt_tstr_t *tstr, nxt_str_t *val) { - nxt_int_t ret; + nxt_int_t ret; + nxt_uint_t flags; if (nxt_tstr_is_const(tstr)) { nxt_tstr_str(tstr, val); @@ -273,9 +328,18 @@ nxt_tstr_query(nxt_task_t *task, nxt_tstr_query_t *query, nxt_tstr_t *tstr, } if (tstr->type == NXT_TSTR_VAR) { + flags = 0; + + if (tstr->flags & NXT_TSTR_LOGGING) { + flags |= NXT_VAR_LOGGING; + } + + if (tstr->flags & NXT_TSTR_ESCAPE) { + flags |= NXT_VAR_ESCAPE; + } + ret = nxt_var_interpreter(task, query->state, &query->cache->var, - tstr->u.var, val, query->ctx, - tstr->flags & NXT_TSTR_LOGGING); + tstr->u.var, val, query->ctx, flags); if (nxt_slow_path(ret != NXT_OK)) { return NXT_ERROR; @@ -289,6 +353,19 @@ nxt_tstr_query(nxt_task_t *task, nxt_tstr_query_t *query, nxt_tstr_t *tstr, if (nxt_slow_path(ret != NXT_OK)) { return NXT_ERROR; } + + if (tstr->flags & NXT_TSTR_ESCAPE) { + /* + * (flags & BIT) != 0, not the masked bit: nxt_bool_t is + * nxt_uint_t, and nxt_tstr_escape() adds this value to a length. + * nxt_tstr_compile() normalises the same flag the same way. + */ + ret = nxt_tstr_escape(query->cache->var.pool, val, + (tstr->flags & NXT_TSTR_NEWLINE) != 0); + if (nxt_slow_path(ret != NXT_OK)) { + return NXT_ERROR; + } + } #endif } diff --git a/src/nxt_tstr.h b/src/nxt_tstr.h index 7fb02631e..b96d04a31 100644 --- a/src/nxt_tstr.h +++ b/src/nxt_tstr.h @@ -35,6 +35,13 @@ typedef enum { NXT_TSTR_STRZ = 1 << 0, NXT_TSTR_LOGGING = 1 << 1, NXT_TSTR_NEWLINE = 1 << 2, + /* + * Escape what a variable expands to, leaving the template around it as + * the operator wrote it. The access log's string format sets this: its + * values come from the client, and a raw newline in one of them ends the + * record and starts a line the client chose. + */ + NXT_TSTR_ESCAPE = 1 << 3, } nxt_tstr_flags_t; @@ -73,6 +80,11 @@ nxt_is_tstr(nxt_str_t *str) { u_char *p; + /* memchr() declares its pointer non-null; an empty string has none. */ + if (str->length == 0) { + return 0; + } + p = memchr(str->start, '`', str->length); if (p != NULL) { return 1; diff --git a/src/nxt_types.h b/src/nxt_types.h index 723346d9f..e89fd4074 100644 --- a/src/nxt_types.h +++ b/src/nxt_types.h @@ -54,10 +54,23 @@ typedef off_t nxt_off_t; */ #if (NXT_QNX) /* - * QNX defines time_t as uint32_t. - * Y2038 fix: "typedef int64_t nxt_time_t". + * QNX before SDP 8.0 defines time_t as an unsigned 32-bit type, so the + * signed type of the same width this arm used could not represent any + * instant past 2038-01-19. SDP 8.0 and later use a signed 64-bit time_t + * (https://www.qnx.com/developers/docs/8.0/com.qnx.doc.neutrino.prog/topic/ + * timing_Time_functions.html), where this arm matches the generic branch + * and the typedef below is a no-op. + * + * NXT_TIME_T_SIZE is probed by auto/types from the native sizeof(time_t), + * which is 4 on a pre-8.0 QNX, but every user of the macro -- the NXT_TIME_T_* + * macros below, the Y2038 guard in nxt_time_parse(), and the gmtime test -- + * asks it about nxt_time_t rather than about time_t. Redefine it to match + * the type it is read as. */ -typedef int32_t nxt_time_t; +typedef int64_t nxt_time_t; + +#undef NXT_TIME_T_SIZE +#define NXT_TIME_T_SIZE 8 #else /* Y2038, if time_t is 32-bit integer. */ diff --git a/src/nxt_unicode_lowcase.h b/src/nxt_unicode_lowcase.h deleted file mode 100644 index 2f25ab69e..000000000 --- a/src/nxt_unicode_lowcase.h +++ /dev/null @@ -1,1043 +0,0 @@ - -/* - * 26 128-bytes blocks, 521 pointers. - * 14920 bytes on 32-bit platforms, 17000 bytes on 64-bit platforms. - */ - -#define NXT_UNICODE_MAX_LOWCASE 0x10427 - -#define NXT_UNICODE_BLOCK_SIZE 128 - - -static const uint32_t nxt_unicode_block_000[128] nxt_aligned(64) = { - 0x00000, 0x00001, 0x00002, 0x00003, 0x00004, 0x00005, 0x00006, 0x00007, - 0x00008, 0x00009, 0x0000A, 0x0000B, 0x0000C, 0x0000D, 0x0000E, 0x0000F, - 0x00010, 0x00011, 0x00012, 0x00013, 0x00014, 0x00015, 0x00016, 0x00017, - 0x00018, 0x00019, 0x0001A, 0x0001B, 0x0001C, 0x0001D, 0x0001E, 0x0001F, - 0x00020, 0x00021, 0x00022, 0x00023, 0x00024, 0x00025, 0x00026, 0x00027, - 0x00028, 0x00029, 0x0002A, 0x0002B, 0x0002C, 0x0002D, 0x0002E, 0x0002F, - 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, - 0x00038, 0x00039, 0x0003A, 0x0003B, 0x0003C, 0x0003D, 0x0003E, 0x0003F, - 0x00040, 0x00061, 0x00062, 0x00063, 0x00064, 0x00065, 0x00066, 0x00067, - 0x00068, 0x00069, 0x0006A, 0x0006B, 0x0006C, 0x0006D, 0x0006E, 0x0006F, - 0x00070, 0x00071, 0x00072, 0x00073, 0x00074, 0x00075, 0x00076, 0x00077, - 0x00078, 0x00079, 0x0007A, 0x0005B, 0x0005C, 0x0005D, 0x0005E, 0x0005F, - 0x00060, 0x00061, 0x00062, 0x00063, 0x00064, 0x00065, 0x00066, 0x00067, - 0x00068, 0x00069, 0x0006A, 0x0006B, 0x0006C, 0x0006D, 0x0006E, 0x0006F, - 0x00070, 0x00071, 0x00072, 0x00073, 0x00074, 0x00075, 0x00076, 0x00077, - 0x00078, 0x00079, 0x0007A, 0x0007B, 0x0007C, 0x0007D, 0x0007E, 0x0007F, -}; - - -static const uint32_t nxt_unicode_block_001[128] nxt_aligned(64) = { - 0x00080, 0x00081, 0x00082, 0x00083, 0x00084, 0x00085, 0x00086, 0x00087, - 0x00088, 0x00089, 0x0008A, 0x0008B, 0x0008C, 0x0008D, 0x0008E, 0x0008F, - 0x00090, 0x00091, 0x00092, 0x00093, 0x00094, 0x00095, 0x00096, 0x00097, - 0x00098, 0x00099, 0x0009A, 0x0009B, 0x0009C, 0x0009D, 0x0009E, 0x0009F, - 0x000A0, 0x000A1, 0x000A2, 0x000A3, 0x000A4, 0x000A5, 0x000A6, 0x000A7, - 0x000A8, 0x000A9, 0x000AA, 0x000AB, 0x000AC, 0x000AD, 0x000AE, 0x000AF, - 0x000B0, 0x000B1, 0x000B2, 0x000B3, 0x000B4, 0x003BC, 0x000B6, 0x000B7, - 0x000B8, 0x000B9, 0x000BA, 0x000BB, 0x000BC, 0x000BD, 0x000BE, 0x000BF, - 0x000E0, 0x000E1, 0x000E2, 0x000E3, 0x000E4, 0x000E5, 0x000E6, 0x000E7, - 0x000E8, 0x000E9, 0x000EA, 0x000EB, 0x000EC, 0x000ED, 0x000EE, 0x000EF, - 0x000F0, 0x000F1, 0x000F2, 0x000F3, 0x000F4, 0x000F5, 0x000F6, 0x000D7, - 0x000F8, 0x000F9, 0x000FA, 0x000FB, 0x000FC, 0x000FD, 0x000FE, 0x000DF, - 0x000E0, 0x000E1, 0x000E2, 0x000E3, 0x000E4, 0x000E5, 0x000E6, 0x000E7, - 0x000E8, 0x000E9, 0x000EA, 0x000EB, 0x000EC, 0x000ED, 0x000EE, 0x000EF, - 0x000F0, 0x000F1, 0x000F2, 0x000F3, 0x000F4, 0x000F5, 0x000F6, 0x000F7, - 0x000F8, 0x000F9, 0x000FA, 0x000FB, 0x000FC, 0x000FD, 0x000FE, 0x000FF, -}; - - -static const uint32_t nxt_unicode_block_002[128] nxt_aligned(64) = { - 0x00101, 0x00101, 0x00103, 0x00103, 0x00105, 0x00105, 0x00107, 0x00107, - 0x00109, 0x00109, 0x0010B, 0x0010B, 0x0010D, 0x0010D, 0x0010F, 0x0010F, - 0x00111, 0x00111, 0x00113, 0x00113, 0x00115, 0x00115, 0x00117, 0x00117, - 0x00119, 0x00119, 0x0011B, 0x0011B, 0x0011D, 0x0011D, 0x0011F, 0x0011F, - 0x00121, 0x00121, 0x00123, 0x00123, 0x00125, 0x00125, 0x00127, 0x00127, - 0x00129, 0x00129, 0x0012B, 0x0012B, 0x0012D, 0x0012D, 0x0012F, 0x0012F, - 0x00130, 0x00131, 0x00133, 0x00133, 0x00135, 0x00135, 0x00137, 0x00137, - 0x00138, 0x0013A, 0x0013A, 0x0013C, 0x0013C, 0x0013E, 0x0013E, 0x00140, - 0x00140, 0x00142, 0x00142, 0x00144, 0x00144, 0x00146, 0x00146, 0x00148, - 0x00148, 0x00149, 0x0014B, 0x0014B, 0x0014D, 0x0014D, 0x0014F, 0x0014F, - 0x00151, 0x00151, 0x00153, 0x00153, 0x00155, 0x00155, 0x00157, 0x00157, - 0x00159, 0x00159, 0x0015B, 0x0015B, 0x0015D, 0x0015D, 0x0015F, 0x0015F, - 0x00161, 0x00161, 0x00163, 0x00163, 0x00165, 0x00165, 0x00167, 0x00167, - 0x00169, 0x00169, 0x0016B, 0x0016B, 0x0016D, 0x0016D, 0x0016F, 0x0016F, - 0x00171, 0x00171, 0x00173, 0x00173, 0x00175, 0x00175, 0x00177, 0x00177, - 0x000FF, 0x0017A, 0x0017A, 0x0017C, 0x0017C, 0x0017E, 0x0017E, 0x00073, -}; - - -static const uint32_t nxt_unicode_block_003[128] nxt_aligned(64) = { - 0x00180, 0x00253, 0x00183, 0x00183, 0x00185, 0x00185, 0x00254, 0x00188, - 0x00188, 0x00256, 0x00257, 0x0018C, 0x0018C, 0x0018D, 0x001DD, 0x00259, - 0x0025B, 0x00192, 0x00192, 0x00260, 0x00263, 0x00195, 0x00269, 0x00268, - 0x00199, 0x00199, 0x0019A, 0x0019B, 0x0026F, 0x00272, 0x0019E, 0x00275, - 0x001A1, 0x001A1, 0x001A3, 0x001A3, 0x001A5, 0x001A5, 0x00280, 0x001A8, - 0x001A8, 0x00283, 0x001AA, 0x001AB, 0x001AD, 0x001AD, 0x00288, 0x001B0, - 0x001B0, 0x0028A, 0x0028B, 0x001B4, 0x001B4, 0x001B6, 0x001B6, 0x00292, - 0x001B9, 0x001B9, 0x001BA, 0x001BB, 0x001BD, 0x001BD, 0x001BE, 0x001BF, - 0x001C0, 0x001C1, 0x001C2, 0x001C3, 0x001C6, 0x001C6, 0x001C6, 0x001C9, - 0x001C9, 0x001C9, 0x001CC, 0x001CC, 0x001CC, 0x001CE, 0x001CE, 0x001D0, - 0x001D0, 0x001D2, 0x001D2, 0x001D4, 0x001D4, 0x001D6, 0x001D6, 0x001D8, - 0x001D8, 0x001DA, 0x001DA, 0x001DC, 0x001DC, 0x001DD, 0x001DF, 0x001DF, - 0x001E1, 0x001E1, 0x001E3, 0x001E3, 0x001E5, 0x001E5, 0x001E7, 0x001E7, - 0x001E9, 0x001E9, 0x001EB, 0x001EB, 0x001ED, 0x001ED, 0x001EF, 0x001EF, - 0x001F0, 0x001F3, 0x001F3, 0x001F3, 0x001F5, 0x001F5, 0x00195, 0x001BF, - 0x001F9, 0x001F9, 0x001FB, 0x001FB, 0x001FD, 0x001FD, 0x001FF, 0x001FF, -}; - - -static const uint32_t nxt_unicode_block_004[128] nxt_aligned(64) = { - 0x00201, 0x00201, 0x00203, 0x00203, 0x00205, 0x00205, 0x00207, 0x00207, - 0x00209, 0x00209, 0x0020B, 0x0020B, 0x0020D, 0x0020D, 0x0020F, 0x0020F, - 0x00211, 0x00211, 0x00213, 0x00213, 0x00215, 0x00215, 0x00217, 0x00217, - 0x00219, 0x00219, 0x0021B, 0x0021B, 0x0021D, 0x0021D, 0x0021F, 0x0021F, - 0x0019E, 0x00221, 0x00223, 0x00223, 0x00225, 0x00225, 0x00227, 0x00227, - 0x00229, 0x00229, 0x0022B, 0x0022B, 0x0022D, 0x0022D, 0x0022F, 0x0022F, - 0x00231, 0x00231, 0x00233, 0x00233, 0x00234, 0x00235, 0x00236, 0x00237, - 0x00238, 0x00239, 0x02C65, 0x0023C, 0x0023C, 0x0019A, 0x02C66, 0x0023F, - 0x00240, 0x00242, 0x00242, 0x00180, 0x00289, 0x0028C, 0x00247, 0x00247, - 0x00249, 0x00249, 0x0024B, 0x0024B, 0x0024D, 0x0024D, 0x0024F, 0x0024F, - 0x00250, 0x00251, 0x00252, 0x00253, 0x00254, 0x00255, 0x00256, 0x00257, - 0x00258, 0x00259, 0x0025A, 0x0025B, 0x0025C, 0x0025D, 0x0025E, 0x0025F, - 0x00260, 0x00261, 0x00262, 0x00263, 0x00264, 0x00265, 0x00266, 0x00267, - 0x00268, 0x00269, 0x0026A, 0x0026B, 0x0026C, 0x0026D, 0x0026E, 0x0026F, - 0x00270, 0x00271, 0x00272, 0x00273, 0x00274, 0x00275, 0x00276, 0x00277, - 0x00278, 0x00279, 0x0027A, 0x0027B, 0x0027C, 0x0027D, 0x0027E, 0x0027F, -}; - - -static const uint32_t nxt_unicode_block_006[128] nxt_aligned(64) = { - 0x00300, 0x00301, 0x00302, 0x00303, 0x00304, 0x00305, 0x00306, 0x00307, - 0x00308, 0x00309, 0x0030A, 0x0030B, 0x0030C, 0x0030D, 0x0030E, 0x0030F, - 0x00310, 0x00311, 0x00312, 0x00313, 0x00314, 0x00315, 0x00316, 0x00317, - 0x00318, 0x00319, 0x0031A, 0x0031B, 0x0031C, 0x0031D, 0x0031E, 0x0031F, - 0x00320, 0x00321, 0x00322, 0x00323, 0x00324, 0x00325, 0x00326, 0x00327, - 0x00328, 0x00329, 0x0032A, 0x0032B, 0x0032C, 0x0032D, 0x0032E, 0x0032F, - 0x00330, 0x00331, 0x00332, 0x00333, 0x00334, 0x00335, 0x00336, 0x00337, - 0x00338, 0x00339, 0x0033A, 0x0033B, 0x0033C, 0x0033D, 0x0033E, 0x0033F, - 0x00340, 0x00341, 0x00342, 0x00343, 0x00344, 0x003B9, 0x00346, 0x00347, - 0x00348, 0x00349, 0x0034A, 0x0034B, 0x0034C, 0x0034D, 0x0034E, 0x0034F, - 0x00350, 0x00351, 0x00352, 0x00353, 0x00354, 0x00355, 0x00356, 0x00357, - 0x00358, 0x00359, 0x0035A, 0x0035B, 0x0035C, 0x0035D, 0x0035E, 0x0035F, - 0x00360, 0x00361, 0x00362, 0x00363, 0x00364, 0x00365, 0x00366, 0x00367, - 0x00368, 0x00369, 0x0036A, 0x0036B, 0x0036C, 0x0036D, 0x0036E, 0x0036F, - 0x00371, 0x00371, 0x00373, 0x00373, 0x00374, 0x00375, 0x00377, 0x00377, - 0x00378, 0x00379, 0x0037A, 0x0037B, 0x0037C, 0x0037D, 0x0037E, 0x0037F, -}; - - -static const uint32_t nxt_unicode_block_007[128] nxt_aligned(64) = { - 0x00380, 0x00381, 0x00382, 0x00383, 0x00384, 0x00385, 0x003AC, 0x00387, - 0x003AD, 0x003AE, 0x003AF, 0x0038B, 0x003CC, 0x0038D, 0x003CD, 0x003CE, - 0x00390, 0x003B1, 0x003B2, 0x003B3, 0x003B4, 0x003B5, 0x003B6, 0x003B7, - 0x003B8, 0x003B9, 0x003BA, 0x003BB, 0x003BC, 0x003BD, 0x003BE, 0x003BF, - 0x003C0, 0x003C1, 0x003A2, 0x003C3, 0x003C4, 0x003C5, 0x003C6, 0x003C7, - 0x003C8, 0x003C9, 0x003CA, 0x003CB, 0x003AC, 0x003AD, 0x003AE, 0x003AF, - 0x003B0, 0x003B1, 0x003B2, 0x003B3, 0x003B4, 0x003B5, 0x003B6, 0x003B7, - 0x003B8, 0x003B9, 0x003BA, 0x003BB, 0x003BC, 0x003BD, 0x003BE, 0x003BF, - 0x003C0, 0x003C1, 0x003C3, 0x003C3, 0x003C4, 0x003C5, 0x003C6, 0x003C7, - 0x003C8, 0x003C9, 0x003CA, 0x003CB, 0x003CC, 0x003CD, 0x003CE, 0x003D7, - 0x003B2, 0x003B8, 0x003D2, 0x003D3, 0x003D4, 0x003C6, 0x003C0, 0x003D7, - 0x003D9, 0x003D9, 0x003DB, 0x003DB, 0x003DD, 0x003DD, 0x003DF, 0x003DF, - 0x003E1, 0x003E1, 0x003E3, 0x003E3, 0x003E5, 0x003E5, 0x003E7, 0x003E7, - 0x003E9, 0x003E9, 0x003EB, 0x003EB, 0x003ED, 0x003ED, 0x003EF, 0x003EF, - 0x003BA, 0x003C1, 0x003F2, 0x003F3, 0x003B8, 0x003B5, 0x003F6, 0x003F8, - 0x003F8, 0x003F2, 0x003FB, 0x003FB, 0x003FC, 0x0037B, 0x0037C, 0x0037D, -}; - - -static const uint32_t nxt_unicode_block_008[128] nxt_aligned(64) = { - 0x00450, 0x00451, 0x00452, 0x00453, 0x00454, 0x00455, 0x00456, 0x00457, - 0x00458, 0x00459, 0x0045A, 0x0045B, 0x0045C, 0x0045D, 0x0045E, 0x0045F, - 0x00430, 0x00431, 0x00432, 0x00433, 0x00434, 0x00435, 0x00436, 0x00437, - 0x00438, 0x00439, 0x0043A, 0x0043B, 0x0043C, 0x0043D, 0x0043E, 0x0043F, - 0x00440, 0x00441, 0x00442, 0x00443, 0x00444, 0x00445, 0x00446, 0x00447, - 0x00448, 0x00449, 0x0044A, 0x0044B, 0x0044C, 0x0044D, 0x0044E, 0x0044F, - 0x00430, 0x00431, 0x00432, 0x00433, 0x00434, 0x00435, 0x00436, 0x00437, - 0x00438, 0x00439, 0x0043A, 0x0043B, 0x0043C, 0x0043D, 0x0043E, 0x0043F, - 0x00440, 0x00441, 0x00442, 0x00443, 0x00444, 0x00445, 0x00446, 0x00447, - 0x00448, 0x00449, 0x0044A, 0x0044B, 0x0044C, 0x0044D, 0x0044E, 0x0044F, - 0x00450, 0x00451, 0x00452, 0x00453, 0x00454, 0x00455, 0x00456, 0x00457, - 0x00458, 0x00459, 0x0045A, 0x0045B, 0x0045C, 0x0045D, 0x0045E, 0x0045F, - 0x00461, 0x00461, 0x00463, 0x00463, 0x00465, 0x00465, 0x00467, 0x00467, - 0x00469, 0x00469, 0x0046B, 0x0046B, 0x0046D, 0x0046D, 0x0046F, 0x0046F, - 0x00471, 0x00471, 0x00473, 0x00473, 0x00475, 0x00475, 0x00477, 0x00477, - 0x00479, 0x00479, 0x0047B, 0x0047B, 0x0047D, 0x0047D, 0x0047F, 0x0047F, -}; - - -static const uint32_t nxt_unicode_block_009[128] nxt_aligned(64) = { - 0x00481, 0x00481, 0x00482, 0x00483, 0x00484, 0x00485, 0x00486, 0x00487, - 0x00488, 0x00489, 0x0048B, 0x0048B, 0x0048D, 0x0048D, 0x0048F, 0x0048F, - 0x00491, 0x00491, 0x00493, 0x00493, 0x00495, 0x00495, 0x00497, 0x00497, - 0x00499, 0x00499, 0x0049B, 0x0049B, 0x0049D, 0x0049D, 0x0049F, 0x0049F, - 0x004A1, 0x004A1, 0x004A3, 0x004A3, 0x004A5, 0x004A5, 0x004A7, 0x004A7, - 0x004A9, 0x004A9, 0x004AB, 0x004AB, 0x004AD, 0x004AD, 0x004AF, 0x004AF, - 0x004B1, 0x004B1, 0x004B3, 0x004B3, 0x004B5, 0x004B5, 0x004B7, 0x004B7, - 0x004B9, 0x004B9, 0x004BB, 0x004BB, 0x004BD, 0x004BD, 0x004BF, 0x004BF, - 0x004CF, 0x004C2, 0x004C2, 0x004C4, 0x004C4, 0x004C6, 0x004C6, 0x004C8, - 0x004C8, 0x004CA, 0x004CA, 0x004CC, 0x004CC, 0x004CE, 0x004CE, 0x004CF, - 0x004D1, 0x004D1, 0x004D3, 0x004D3, 0x004D5, 0x004D5, 0x004D7, 0x004D7, - 0x004D9, 0x004D9, 0x004DB, 0x004DB, 0x004DD, 0x004DD, 0x004DF, 0x004DF, - 0x004E1, 0x004E1, 0x004E3, 0x004E3, 0x004E5, 0x004E5, 0x004E7, 0x004E7, - 0x004E9, 0x004E9, 0x004EB, 0x004EB, 0x004ED, 0x004ED, 0x004EF, 0x004EF, - 0x004F1, 0x004F1, 0x004F3, 0x004F3, 0x004F5, 0x004F5, 0x004F7, 0x004F7, - 0x004F9, 0x004F9, 0x004FB, 0x004FB, 0x004FD, 0x004FD, 0x004FF, 0x004FF, -}; - - -static const uint32_t nxt_unicode_block_00a[128] nxt_aligned(64) = { - 0x00501, 0x00501, 0x00503, 0x00503, 0x00505, 0x00505, 0x00507, 0x00507, - 0x00509, 0x00509, 0x0050B, 0x0050B, 0x0050D, 0x0050D, 0x0050F, 0x0050F, - 0x00511, 0x00511, 0x00513, 0x00513, 0x00515, 0x00515, 0x00517, 0x00517, - 0x00519, 0x00519, 0x0051B, 0x0051B, 0x0051D, 0x0051D, 0x0051F, 0x0051F, - 0x00521, 0x00521, 0x00523, 0x00523, 0x00525, 0x00525, 0x00527, 0x00527, - 0x00528, 0x00529, 0x0052A, 0x0052B, 0x0052C, 0x0052D, 0x0052E, 0x0052F, - 0x00530, 0x00561, 0x00562, 0x00563, 0x00564, 0x00565, 0x00566, 0x00567, - 0x00568, 0x00569, 0x0056A, 0x0056B, 0x0056C, 0x0056D, 0x0056E, 0x0056F, - 0x00570, 0x00571, 0x00572, 0x00573, 0x00574, 0x00575, 0x00576, 0x00577, - 0x00578, 0x00579, 0x0057A, 0x0057B, 0x0057C, 0x0057D, 0x0057E, 0x0057F, - 0x00580, 0x00581, 0x00582, 0x00583, 0x00584, 0x00585, 0x00586, 0x00557, - 0x00558, 0x00559, 0x0055A, 0x0055B, 0x0055C, 0x0055D, 0x0055E, 0x0055F, - 0x00560, 0x00561, 0x00562, 0x00563, 0x00564, 0x00565, 0x00566, 0x00567, - 0x00568, 0x00569, 0x0056A, 0x0056B, 0x0056C, 0x0056D, 0x0056E, 0x0056F, - 0x00570, 0x00571, 0x00572, 0x00573, 0x00574, 0x00575, 0x00576, 0x00577, - 0x00578, 0x00579, 0x0057A, 0x0057B, 0x0057C, 0x0057D, 0x0057E, 0x0057F, -}; - - -static const uint32_t nxt_unicode_block_021[128] nxt_aligned(64) = { - 0x01080, 0x01081, 0x01082, 0x01083, 0x01084, 0x01085, 0x01086, 0x01087, - 0x01088, 0x01089, 0x0108A, 0x0108B, 0x0108C, 0x0108D, 0x0108E, 0x0108F, - 0x01090, 0x01091, 0x01092, 0x01093, 0x01094, 0x01095, 0x01096, 0x01097, - 0x01098, 0x01099, 0x0109A, 0x0109B, 0x0109C, 0x0109D, 0x0109E, 0x0109F, - 0x02D00, 0x02D01, 0x02D02, 0x02D03, 0x02D04, 0x02D05, 0x02D06, 0x02D07, - 0x02D08, 0x02D09, 0x02D0A, 0x02D0B, 0x02D0C, 0x02D0D, 0x02D0E, 0x02D0F, - 0x02D10, 0x02D11, 0x02D12, 0x02D13, 0x02D14, 0x02D15, 0x02D16, 0x02D17, - 0x02D18, 0x02D19, 0x02D1A, 0x02D1B, 0x02D1C, 0x02D1D, 0x02D1E, 0x02D1F, - 0x02D20, 0x02D21, 0x02D22, 0x02D23, 0x02D24, 0x02D25, 0x010C6, 0x02D27, - 0x010C8, 0x010C9, 0x010CA, 0x010CB, 0x010CC, 0x02D2D, 0x010CE, 0x010CF, - 0x010D0, 0x010D1, 0x010D2, 0x010D3, 0x010D4, 0x010D5, 0x010D6, 0x010D7, - 0x010D8, 0x010D9, 0x010DA, 0x010DB, 0x010DC, 0x010DD, 0x010DE, 0x010DF, - 0x010E0, 0x010E1, 0x010E2, 0x010E3, 0x010E4, 0x010E5, 0x010E6, 0x010E7, - 0x010E8, 0x010E9, 0x010EA, 0x010EB, 0x010EC, 0x010ED, 0x010EE, 0x010EF, - 0x010F0, 0x010F1, 0x010F2, 0x010F3, 0x010F4, 0x010F5, 0x010F6, 0x010F7, - 0x010F8, 0x010F9, 0x010FA, 0x010FB, 0x010FC, 0x010FD, 0x010FE, 0x010FF, -}; - - -static const uint32_t nxt_unicode_block_03c[128] nxt_aligned(64) = { - 0x01E01, 0x01E01, 0x01E03, 0x01E03, 0x01E05, 0x01E05, 0x01E07, 0x01E07, - 0x01E09, 0x01E09, 0x01E0B, 0x01E0B, 0x01E0D, 0x01E0D, 0x01E0F, 0x01E0F, - 0x01E11, 0x01E11, 0x01E13, 0x01E13, 0x01E15, 0x01E15, 0x01E17, 0x01E17, - 0x01E19, 0x01E19, 0x01E1B, 0x01E1B, 0x01E1D, 0x01E1D, 0x01E1F, 0x01E1F, - 0x01E21, 0x01E21, 0x01E23, 0x01E23, 0x01E25, 0x01E25, 0x01E27, 0x01E27, - 0x01E29, 0x01E29, 0x01E2B, 0x01E2B, 0x01E2D, 0x01E2D, 0x01E2F, 0x01E2F, - 0x01E31, 0x01E31, 0x01E33, 0x01E33, 0x01E35, 0x01E35, 0x01E37, 0x01E37, - 0x01E39, 0x01E39, 0x01E3B, 0x01E3B, 0x01E3D, 0x01E3D, 0x01E3F, 0x01E3F, - 0x01E41, 0x01E41, 0x01E43, 0x01E43, 0x01E45, 0x01E45, 0x01E47, 0x01E47, - 0x01E49, 0x01E49, 0x01E4B, 0x01E4B, 0x01E4D, 0x01E4D, 0x01E4F, 0x01E4F, - 0x01E51, 0x01E51, 0x01E53, 0x01E53, 0x01E55, 0x01E55, 0x01E57, 0x01E57, - 0x01E59, 0x01E59, 0x01E5B, 0x01E5B, 0x01E5D, 0x01E5D, 0x01E5F, 0x01E5F, - 0x01E61, 0x01E61, 0x01E63, 0x01E63, 0x01E65, 0x01E65, 0x01E67, 0x01E67, - 0x01E69, 0x01E69, 0x01E6B, 0x01E6B, 0x01E6D, 0x01E6D, 0x01E6F, 0x01E6F, - 0x01E71, 0x01E71, 0x01E73, 0x01E73, 0x01E75, 0x01E75, 0x01E77, 0x01E77, - 0x01E79, 0x01E79, 0x01E7B, 0x01E7B, 0x01E7D, 0x01E7D, 0x01E7F, 0x01E7F, -}; - - -static const uint32_t nxt_unicode_block_03d[128] nxt_aligned(64) = { - 0x01E81, 0x01E81, 0x01E83, 0x01E83, 0x01E85, 0x01E85, 0x01E87, 0x01E87, - 0x01E89, 0x01E89, 0x01E8B, 0x01E8B, 0x01E8D, 0x01E8D, 0x01E8F, 0x01E8F, - 0x01E91, 0x01E91, 0x01E93, 0x01E93, 0x01E95, 0x01E95, 0x01E96, 0x01E97, - 0x01E98, 0x01E99, 0x01E9A, 0x01E61, 0x01E9C, 0x01E9D, 0x000DF, 0x01E9F, - 0x01EA1, 0x01EA1, 0x01EA3, 0x01EA3, 0x01EA5, 0x01EA5, 0x01EA7, 0x01EA7, - 0x01EA9, 0x01EA9, 0x01EAB, 0x01EAB, 0x01EAD, 0x01EAD, 0x01EAF, 0x01EAF, - 0x01EB1, 0x01EB1, 0x01EB3, 0x01EB3, 0x01EB5, 0x01EB5, 0x01EB7, 0x01EB7, - 0x01EB9, 0x01EB9, 0x01EBB, 0x01EBB, 0x01EBD, 0x01EBD, 0x01EBF, 0x01EBF, - 0x01EC1, 0x01EC1, 0x01EC3, 0x01EC3, 0x01EC5, 0x01EC5, 0x01EC7, 0x01EC7, - 0x01EC9, 0x01EC9, 0x01ECB, 0x01ECB, 0x01ECD, 0x01ECD, 0x01ECF, 0x01ECF, - 0x01ED1, 0x01ED1, 0x01ED3, 0x01ED3, 0x01ED5, 0x01ED5, 0x01ED7, 0x01ED7, - 0x01ED9, 0x01ED9, 0x01EDB, 0x01EDB, 0x01EDD, 0x01EDD, 0x01EDF, 0x01EDF, - 0x01EE1, 0x01EE1, 0x01EE3, 0x01EE3, 0x01EE5, 0x01EE5, 0x01EE7, 0x01EE7, - 0x01EE9, 0x01EE9, 0x01EEB, 0x01EEB, 0x01EED, 0x01EED, 0x01EEF, 0x01EEF, - 0x01EF1, 0x01EF1, 0x01EF3, 0x01EF3, 0x01EF5, 0x01EF5, 0x01EF7, 0x01EF7, - 0x01EF9, 0x01EF9, 0x01EFB, 0x01EFB, 0x01EFD, 0x01EFD, 0x01EFF, 0x01EFF, -}; - - -static const uint32_t nxt_unicode_block_03e[128] nxt_aligned(64) = { - 0x01F00, 0x01F01, 0x01F02, 0x01F03, 0x01F04, 0x01F05, 0x01F06, 0x01F07, - 0x01F00, 0x01F01, 0x01F02, 0x01F03, 0x01F04, 0x01F05, 0x01F06, 0x01F07, - 0x01F10, 0x01F11, 0x01F12, 0x01F13, 0x01F14, 0x01F15, 0x01F16, 0x01F17, - 0x01F10, 0x01F11, 0x01F12, 0x01F13, 0x01F14, 0x01F15, 0x01F1E, 0x01F1F, - 0x01F20, 0x01F21, 0x01F22, 0x01F23, 0x01F24, 0x01F25, 0x01F26, 0x01F27, - 0x01F20, 0x01F21, 0x01F22, 0x01F23, 0x01F24, 0x01F25, 0x01F26, 0x01F27, - 0x01F30, 0x01F31, 0x01F32, 0x01F33, 0x01F34, 0x01F35, 0x01F36, 0x01F37, - 0x01F30, 0x01F31, 0x01F32, 0x01F33, 0x01F34, 0x01F35, 0x01F36, 0x01F37, - 0x01F40, 0x01F41, 0x01F42, 0x01F43, 0x01F44, 0x01F45, 0x01F46, 0x01F47, - 0x01F40, 0x01F41, 0x01F42, 0x01F43, 0x01F44, 0x01F45, 0x01F4E, 0x01F4F, - 0x01F50, 0x01F51, 0x01F52, 0x01F53, 0x01F54, 0x01F55, 0x01F56, 0x01F57, - 0x01F58, 0x01F51, 0x01F5A, 0x01F53, 0x01F5C, 0x01F55, 0x01F5E, 0x01F57, - 0x01F60, 0x01F61, 0x01F62, 0x01F63, 0x01F64, 0x01F65, 0x01F66, 0x01F67, - 0x01F60, 0x01F61, 0x01F62, 0x01F63, 0x01F64, 0x01F65, 0x01F66, 0x01F67, - 0x01F70, 0x01F71, 0x01F72, 0x01F73, 0x01F74, 0x01F75, 0x01F76, 0x01F77, - 0x01F78, 0x01F79, 0x01F7A, 0x01F7B, 0x01F7C, 0x01F7D, 0x01F7E, 0x01F7F, -}; - - -static const uint32_t nxt_unicode_block_03f[128] nxt_aligned(64) = { - 0x01F80, 0x01F81, 0x01F82, 0x01F83, 0x01F84, 0x01F85, 0x01F86, 0x01F87, - 0x01F80, 0x01F81, 0x01F82, 0x01F83, 0x01F84, 0x01F85, 0x01F86, 0x01F87, - 0x01F90, 0x01F91, 0x01F92, 0x01F93, 0x01F94, 0x01F95, 0x01F96, 0x01F97, - 0x01F90, 0x01F91, 0x01F92, 0x01F93, 0x01F94, 0x01F95, 0x01F96, 0x01F97, - 0x01FA0, 0x01FA1, 0x01FA2, 0x01FA3, 0x01FA4, 0x01FA5, 0x01FA6, 0x01FA7, - 0x01FA0, 0x01FA1, 0x01FA2, 0x01FA3, 0x01FA4, 0x01FA5, 0x01FA6, 0x01FA7, - 0x01FB0, 0x01FB1, 0x01FB2, 0x01FB3, 0x01FB4, 0x01FB5, 0x01FB6, 0x01FB7, - 0x01FB0, 0x01FB1, 0x01F70, 0x01F71, 0x01FB3, 0x01FBD, 0x003B9, 0x01FBF, - 0x01FC0, 0x01FC1, 0x01FC2, 0x01FC3, 0x01FC4, 0x01FC5, 0x01FC6, 0x01FC7, - 0x01F72, 0x01F73, 0x01F74, 0x01F75, 0x01FC3, 0x01FCD, 0x01FCE, 0x01FCF, - 0x01FD0, 0x01FD1, 0x01FD2, 0x01FD3, 0x01FD4, 0x01FD5, 0x01FD6, 0x01FD7, - 0x01FD0, 0x01FD1, 0x01F76, 0x01F77, 0x01FDC, 0x01FDD, 0x01FDE, 0x01FDF, - 0x01FE0, 0x01FE1, 0x01FE2, 0x01FE3, 0x01FE4, 0x01FE5, 0x01FE6, 0x01FE7, - 0x01FE0, 0x01FE1, 0x01F7A, 0x01F7B, 0x01FE5, 0x01FED, 0x01FEE, 0x01FEF, - 0x01FF0, 0x01FF1, 0x01FF2, 0x01FF3, 0x01FF4, 0x01FF5, 0x01FF6, 0x01FF7, - 0x01F78, 0x01F79, 0x01F7C, 0x01F7D, 0x01FF3, 0x01FFD, 0x01FFE, 0x01FFF, -}; - - -static const uint32_t nxt_unicode_block_042[128] nxt_aligned(64) = { - 0x02100, 0x02101, 0x02102, 0x02103, 0x02104, 0x02105, 0x02106, 0x02107, - 0x02108, 0x02109, 0x0210A, 0x0210B, 0x0210C, 0x0210D, 0x0210E, 0x0210F, - 0x02110, 0x02111, 0x02112, 0x02113, 0x02114, 0x02115, 0x02116, 0x02117, - 0x02118, 0x02119, 0x0211A, 0x0211B, 0x0211C, 0x0211D, 0x0211E, 0x0211F, - 0x02120, 0x02121, 0x02122, 0x02123, 0x02124, 0x02125, 0x003C9, 0x02127, - 0x02128, 0x02129, 0x0006B, 0x000E5, 0x0212C, 0x0212D, 0x0212E, 0x0212F, - 0x02130, 0x02131, 0x0214E, 0x02133, 0x02134, 0x02135, 0x02136, 0x02137, - 0x02138, 0x02139, 0x0213A, 0x0213B, 0x0213C, 0x0213D, 0x0213E, 0x0213F, - 0x02140, 0x02141, 0x02142, 0x02143, 0x02144, 0x02145, 0x02146, 0x02147, - 0x02148, 0x02149, 0x0214A, 0x0214B, 0x0214C, 0x0214D, 0x0214E, 0x0214F, - 0x02150, 0x02151, 0x02152, 0x02153, 0x02154, 0x02155, 0x02156, 0x02157, - 0x02158, 0x02159, 0x0215A, 0x0215B, 0x0215C, 0x0215D, 0x0215E, 0x0215F, - 0x02170, 0x02171, 0x02172, 0x02173, 0x02174, 0x02175, 0x02176, 0x02177, - 0x02178, 0x02179, 0x0217A, 0x0217B, 0x0217C, 0x0217D, 0x0217E, 0x0217F, - 0x02170, 0x02171, 0x02172, 0x02173, 0x02174, 0x02175, 0x02176, 0x02177, - 0x02178, 0x02179, 0x0217A, 0x0217B, 0x0217C, 0x0217D, 0x0217E, 0x0217F, -}; - - -static const uint32_t nxt_unicode_block_043[128] nxt_aligned(64) = { - 0x02180, 0x02181, 0x02182, 0x02184, 0x02184, 0x02185, 0x02186, 0x02187, - 0x02188, 0x02189, 0x0218A, 0x0218B, 0x0218C, 0x0218D, 0x0218E, 0x0218F, - 0x02190, 0x02191, 0x02192, 0x02193, 0x02194, 0x02195, 0x02196, 0x02197, - 0x02198, 0x02199, 0x0219A, 0x0219B, 0x0219C, 0x0219D, 0x0219E, 0x0219F, - 0x021A0, 0x021A1, 0x021A2, 0x021A3, 0x021A4, 0x021A5, 0x021A6, 0x021A7, - 0x021A8, 0x021A9, 0x021AA, 0x021AB, 0x021AC, 0x021AD, 0x021AE, 0x021AF, - 0x021B0, 0x021B1, 0x021B2, 0x021B3, 0x021B4, 0x021B5, 0x021B6, 0x021B7, - 0x021B8, 0x021B9, 0x021BA, 0x021BB, 0x021BC, 0x021BD, 0x021BE, 0x021BF, - 0x021C0, 0x021C1, 0x021C2, 0x021C3, 0x021C4, 0x021C5, 0x021C6, 0x021C7, - 0x021C8, 0x021C9, 0x021CA, 0x021CB, 0x021CC, 0x021CD, 0x021CE, 0x021CF, - 0x021D0, 0x021D1, 0x021D2, 0x021D3, 0x021D4, 0x021D5, 0x021D6, 0x021D7, - 0x021D8, 0x021D9, 0x021DA, 0x021DB, 0x021DC, 0x021DD, 0x021DE, 0x021DF, - 0x021E0, 0x021E1, 0x021E2, 0x021E3, 0x021E4, 0x021E5, 0x021E6, 0x021E7, - 0x021E8, 0x021E9, 0x021EA, 0x021EB, 0x021EC, 0x021ED, 0x021EE, 0x021EF, - 0x021F0, 0x021F1, 0x021F2, 0x021F3, 0x021F4, 0x021F5, 0x021F6, 0x021F7, - 0x021F8, 0x021F9, 0x021FA, 0x021FB, 0x021FC, 0x021FD, 0x021FE, 0x021FF, -}; - - -static const uint32_t nxt_unicode_block_049[128] nxt_aligned(64) = { - 0x02480, 0x02481, 0x02482, 0x02483, 0x02484, 0x02485, 0x02486, 0x02487, - 0x02488, 0x02489, 0x0248A, 0x0248B, 0x0248C, 0x0248D, 0x0248E, 0x0248F, - 0x02490, 0x02491, 0x02492, 0x02493, 0x02494, 0x02495, 0x02496, 0x02497, - 0x02498, 0x02499, 0x0249A, 0x0249B, 0x0249C, 0x0249D, 0x0249E, 0x0249F, - 0x024A0, 0x024A1, 0x024A2, 0x024A3, 0x024A4, 0x024A5, 0x024A6, 0x024A7, - 0x024A8, 0x024A9, 0x024AA, 0x024AB, 0x024AC, 0x024AD, 0x024AE, 0x024AF, - 0x024B0, 0x024B1, 0x024B2, 0x024B3, 0x024B4, 0x024B5, 0x024D0, 0x024D1, - 0x024D2, 0x024D3, 0x024D4, 0x024D5, 0x024D6, 0x024D7, 0x024D8, 0x024D9, - 0x024DA, 0x024DB, 0x024DC, 0x024DD, 0x024DE, 0x024DF, 0x024E0, 0x024E1, - 0x024E2, 0x024E3, 0x024E4, 0x024E5, 0x024E6, 0x024E7, 0x024E8, 0x024E9, - 0x024D0, 0x024D1, 0x024D2, 0x024D3, 0x024D4, 0x024D5, 0x024D6, 0x024D7, - 0x024D8, 0x024D9, 0x024DA, 0x024DB, 0x024DC, 0x024DD, 0x024DE, 0x024DF, - 0x024E0, 0x024E1, 0x024E2, 0x024E3, 0x024E4, 0x024E5, 0x024E6, 0x024E7, - 0x024E8, 0x024E9, 0x024EA, 0x024EB, 0x024EC, 0x024ED, 0x024EE, 0x024EF, - 0x024F0, 0x024F1, 0x024F2, 0x024F3, 0x024F4, 0x024F5, 0x024F6, 0x024F7, - 0x024F8, 0x024F9, 0x024FA, 0x024FB, 0x024FC, 0x024FD, 0x024FE, 0x024FF, -}; - - -static const uint32_t nxt_unicode_block_058[128] nxt_aligned(64) = { - 0x02C30, 0x02C31, 0x02C32, 0x02C33, 0x02C34, 0x02C35, 0x02C36, 0x02C37, - 0x02C38, 0x02C39, 0x02C3A, 0x02C3B, 0x02C3C, 0x02C3D, 0x02C3E, 0x02C3F, - 0x02C40, 0x02C41, 0x02C42, 0x02C43, 0x02C44, 0x02C45, 0x02C46, 0x02C47, - 0x02C48, 0x02C49, 0x02C4A, 0x02C4B, 0x02C4C, 0x02C4D, 0x02C4E, 0x02C4F, - 0x02C50, 0x02C51, 0x02C52, 0x02C53, 0x02C54, 0x02C55, 0x02C56, 0x02C57, - 0x02C58, 0x02C59, 0x02C5A, 0x02C5B, 0x02C5C, 0x02C5D, 0x02C5E, 0x02C2F, - 0x02C30, 0x02C31, 0x02C32, 0x02C33, 0x02C34, 0x02C35, 0x02C36, 0x02C37, - 0x02C38, 0x02C39, 0x02C3A, 0x02C3B, 0x02C3C, 0x02C3D, 0x02C3E, 0x02C3F, - 0x02C40, 0x02C41, 0x02C42, 0x02C43, 0x02C44, 0x02C45, 0x02C46, 0x02C47, - 0x02C48, 0x02C49, 0x02C4A, 0x02C4B, 0x02C4C, 0x02C4D, 0x02C4E, 0x02C4F, - 0x02C50, 0x02C51, 0x02C52, 0x02C53, 0x02C54, 0x02C55, 0x02C56, 0x02C57, - 0x02C58, 0x02C59, 0x02C5A, 0x02C5B, 0x02C5C, 0x02C5D, 0x02C5E, 0x02C5F, - 0x02C61, 0x02C61, 0x0026B, 0x01D7D, 0x0027D, 0x02C65, 0x02C66, 0x02C68, - 0x02C68, 0x02C6A, 0x02C6A, 0x02C6C, 0x02C6C, 0x00251, 0x00271, 0x00250, - 0x00252, 0x02C71, 0x02C73, 0x02C73, 0x02C74, 0x02C76, 0x02C76, 0x02C77, - 0x02C78, 0x02C79, 0x02C7A, 0x02C7B, 0x02C7C, 0x02C7D, 0x0023F, 0x00240, -}; - - -static const uint32_t nxt_unicode_block_059[128] nxt_aligned(64) = { - 0x02C81, 0x02C81, 0x02C83, 0x02C83, 0x02C85, 0x02C85, 0x02C87, 0x02C87, - 0x02C89, 0x02C89, 0x02C8B, 0x02C8B, 0x02C8D, 0x02C8D, 0x02C8F, 0x02C8F, - 0x02C91, 0x02C91, 0x02C93, 0x02C93, 0x02C95, 0x02C95, 0x02C97, 0x02C97, - 0x02C99, 0x02C99, 0x02C9B, 0x02C9B, 0x02C9D, 0x02C9D, 0x02C9F, 0x02C9F, - 0x02CA1, 0x02CA1, 0x02CA3, 0x02CA3, 0x02CA5, 0x02CA5, 0x02CA7, 0x02CA7, - 0x02CA9, 0x02CA9, 0x02CAB, 0x02CAB, 0x02CAD, 0x02CAD, 0x02CAF, 0x02CAF, - 0x02CB1, 0x02CB1, 0x02CB3, 0x02CB3, 0x02CB5, 0x02CB5, 0x02CB7, 0x02CB7, - 0x02CB9, 0x02CB9, 0x02CBB, 0x02CBB, 0x02CBD, 0x02CBD, 0x02CBF, 0x02CBF, - 0x02CC1, 0x02CC1, 0x02CC3, 0x02CC3, 0x02CC5, 0x02CC5, 0x02CC7, 0x02CC7, - 0x02CC9, 0x02CC9, 0x02CCB, 0x02CCB, 0x02CCD, 0x02CCD, 0x02CCF, 0x02CCF, - 0x02CD1, 0x02CD1, 0x02CD3, 0x02CD3, 0x02CD5, 0x02CD5, 0x02CD7, 0x02CD7, - 0x02CD9, 0x02CD9, 0x02CDB, 0x02CDB, 0x02CDD, 0x02CDD, 0x02CDF, 0x02CDF, - 0x02CE1, 0x02CE1, 0x02CE3, 0x02CE3, 0x02CE4, 0x02CE5, 0x02CE6, 0x02CE7, - 0x02CE8, 0x02CE9, 0x02CEA, 0x02CEC, 0x02CEC, 0x02CEE, 0x02CEE, 0x02CEF, - 0x02CF0, 0x02CF1, 0x02CF3, 0x02CF3, 0x02CF4, 0x02CF5, 0x02CF6, 0x02CF7, - 0x02CF8, 0x02CF9, 0x02CFA, 0x02CFB, 0x02CFC, 0x02CFD, 0x02CFE, 0x02CFF, -}; - - -static const uint32_t nxt_unicode_block_14c[128] nxt_aligned(64) = { - 0x0A600, 0x0A601, 0x0A602, 0x0A603, 0x0A604, 0x0A605, 0x0A606, 0x0A607, - 0x0A608, 0x0A609, 0x0A60A, 0x0A60B, 0x0A60C, 0x0A60D, 0x0A60E, 0x0A60F, - 0x0A610, 0x0A611, 0x0A612, 0x0A613, 0x0A614, 0x0A615, 0x0A616, 0x0A617, - 0x0A618, 0x0A619, 0x0A61A, 0x0A61B, 0x0A61C, 0x0A61D, 0x0A61E, 0x0A61F, - 0x0A620, 0x0A621, 0x0A622, 0x0A623, 0x0A624, 0x0A625, 0x0A626, 0x0A627, - 0x0A628, 0x0A629, 0x0A62A, 0x0A62B, 0x0A62C, 0x0A62D, 0x0A62E, 0x0A62F, - 0x0A630, 0x0A631, 0x0A632, 0x0A633, 0x0A634, 0x0A635, 0x0A636, 0x0A637, - 0x0A638, 0x0A639, 0x0A63A, 0x0A63B, 0x0A63C, 0x0A63D, 0x0A63E, 0x0A63F, - 0x0A641, 0x0A641, 0x0A643, 0x0A643, 0x0A645, 0x0A645, 0x0A647, 0x0A647, - 0x0A649, 0x0A649, 0x0A64B, 0x0A64B, 0x0A64D, 0x0A64D, 0x0A64F, 0x0A64F, - 0x0A651, 0x0A651, 0x0A653, 0x0A653, 0x0A655, 0x0A655, 0x0A657, 0x0A657, - 0x0A659, 0x0A659, 0x0A65B, 0x0A65B, 0x0A65D, 0x0A65D, 0x0A65F, 0x0A65F, - 0x0A661, 0x0A661, 0x0A663, 0x0A663, 0x0A665, 0x0A665, 0x0A667, 0x0A667, - 0x0A669, 0x0A669, 0x0A66B, 0x0A66B, 0x0A66D, 0x0A66D, 0x0A66E, 0x0A66F, - 0x0A670, 0x0A671, 0x0A672, 0x0A673, 0x0A674, 0x0A675, 0x0A676, 0x0A677, - 0x0A678, 0x0A679, 0x0A67A, 0x0A67B, 0x0A67C, 0x0A67D, 0x0A67E, 0x0A67F, -}; - - -static const uint32_t nxt_unicode_block_14d[128] nxt_aligned(64) = { - 0x0A681, 0x0A681, 0x0A683, 0x0A683, 0x0A685, 0x0A685, 0x0A687, 0x0A687, - 0x0A689, 0x0A689, 0x0A68B, 0x0A68B, 0x0A68D, 0x0A68D, 0x0A68F, 0x0A68F, - 0x0A691, 0x0A691, 0x0A693, 0x0A693, 0x0A695, 0x0A695, 0x0A697, 0x0A697, - 0x0A698, 0x0A699, 0x0A69A, 0x0A69B, 0x0A69C, 0x0A69D, 0x0A69E, 0x0A69F, - 0x0A6A0, 0x0A6A1, 0x0A6A2, 0x0A6A3, 0x0A6A4, 0x0A6A5, 0x0A6A6, 0x0A6A7, - 0x0A6A8, 0x0A6A9, 0x0A6AA, 0x0A6AB, 0x0A6AC, 0x0A6AD, 0x0A6AE, 0x0A6AF, - 0x0A6B0, 0x0A6B1, 0x0A6B2, 0x0A6B3, 0x0A6B4, 0x0A6B5, 0x0A6B6, 0x0A6B7, - 0x0A6B8, 0x0A6B9, 0x0A6BA, 0x0A6BB, 0x0A6BC, 0x0A6BD, 0x0A6BE, 0x0A6BF, - 0x0A6C0, 0x0A6C1, 0x0A6C2, 0x0A6C3, 0x0A6C4, 0x0A6C5, 0x0A6C6, 0x0A6C7, - 0x0A6C8, 0x0A6C9, 0x0A6CA, 0x0A6CB, 0x0A6CC, 0x0A6CD, 0x0A6CE, 0x0A6CF, - 0x0A6D0, 0x0A6D1, 0x0A6D2, 0x0A6D3, 0x0A6D4, 0x0A6D5, 0x0A6D6, 0x0A6D7, - 0x0A6D8, 0x0A6D9, 0x0A6DA, 0x0A6DB, 0x0A6DC, 0x0A6DD, 0x0A6DE, 0x0A6DF, - 0x0A6E0, 0x0A6E1, 0x0A6E2, 0x0A6E3, 0x0A6E4, 0x0A6E5, 0x0A6E6, 0x0A6E7, - 0x0A6E8, 0x0A6E9, 0x0A6EA, 0x0A6EB, 0x0A6EC, 0x0A6ED, 0x0A6EE, 0x0A6EF, - 0x0A6F0, 0x0A6F1, 0x0A6F2, 0x0A6F3, 0x0A6F4, 0x0A6F5, 0x0A6F6, 0x0A6F7, - 0x0A6F8, 0x0A6F9, 0x0A6FA, 0x0A6FB, 0x0A6FC, 0x0A6FD, 0x0A6FE, 0x0A6FF, -}; - - -static const uint32_t nxt_unicode_block_14e[128] nxt_aligned(64) = { - 0x0A700, 0x0A701, 0x0A702, 0x0A703, 0x0A704, 0x0A705, 0x0A706, 0x0A707, - 0x0A708, 0x0A709, 0x0A70A, 0x0A70B, 0x0A70C, 0x0A70D, 0x0A70E, 0x0A70F, - 0x0A710, 0x0A711, 0x0A712, 0x0A713, 0x0A714, 0x0A715, 0x0A716, 0x0A717, - 0x0A718, 0x0A719, 0x0A71A, 0x0A71B, 0x0A71C, 0x0A71D, 0x0A71E, 0x0A71F, - 0x0A720, 0x0A721, 0x0A723, 0x0A723, 0x0A725, 0x0A725, 0x0A727, 0x0A727, - 0x0A729, 0x0A729, 0x0A72B, 0x0A72B, 0x0A72D, 0x0A72D, 0x0A72F, 0x0A72F, - 0x0A730, 0x0A731, 0x0A733, 0x0A733, 0x0A735, 0x0A735, 0x0A737, 0x0A737, - 0x0A739, 0x0A739, 0x0A73B, 0x0A73B, 0x0A73D, 0x0A73D, 0x0A73F, 0x0A73F, - 0x0A741, 0x0A741, 0x0A743, 0x0A743, 0x0A745, 0x0A745, 0x0A747, 0x0A747, - 0x0A749, 0x0A749, 0x0A74B, 0x0A74B, 0x0A74D, 0x0A74D, 0x0A74F, 0x0A74F, - 0x0A751, 0x0A751, 0x0A753, 0x0A753, 0x0A755, 0x0A755, 0x0A757, 0x0A757, - 0x0A759, 0x0A759, 0x0A75B, 0x0A75B, 0x0A75D, 0x0A75D, 0x0A75F, 0x0A75F, - 0x0A761, 0x0A761, 0x0A763, 0x0A763, 0x0A765, 0x0A765, 0x0A767, 0x0A767, - 0x0A769, 0x0A769, 0x0A76B, 0x0A76B, 0x0A76D, 0x0A76D, 0x0A76F, 0x0A76F, - 0x0A770, 0x0A771, 0x0A772, 0x0A773, 0x0A774, 0x0A775, 0x0A776, 0x0A777, - 0x0A778, 0x0A77A, 0x0A77A, 0x0A77C, 0x0A77C, 0x01D79, 0x0A77F, 0x0A77F, -}; - - -static const uint32_t nxt_unicode_block_14f[128] nxt_aligned(64) = { - 0x0A781, 0x0A781, 0x0A783, 0x0A783, 0x0A785, 0x0A785, 0x0A787, 0x0A787, - 0x0A788, 0x0A789, 0x0A78A, 0x0A78C, 0x0A78C, 0x00265, 0x0A78E, 0x0A78F, - 0x0A791, 0x0A791, 0x0A793, 0x0A793, 0x0A794, 0x0A795, 0x0A796, 0x0A797, - 0x0A798, 0x0A799, 0x0A79A, 0x0A79B, 0x0A79C, 0x0A79D, 0x0A79E, 0x0A79F, - 0x0A7A1, 0x0A7A1, 0x0A7A3, 0x0A7A3, 0x0A7A5, 0x0A7A5, 0x0A7A7, 0x0A7A7, - 0x0A7A9, 0x0A7A9, 0x00266, 0x0A7AB, 0x0A7AC, 0x0A7AD, 0x0A7AE, 0x0A7AF, - 0x0A7B0, 0x0A7B1, 0x0A7B2, 0x0A7B3, 0x0A7B4, 0x0A7B5, 0x0A7B6, 0x0A7B7, - 0x0A7B8, 0x0A7B9, 0x0A7BA, 0x0A7BB, 0x0A7BC, 0x0A7BD, 0x0A7BE, 0x0A7BF, - 0x0A7C0, 0x0A7C1, 0x0A7C2, 0x0A7C3, 0x0A7C4, 0x0A7C5, 0x0A7C6, 0x0A7C7, - 0x0A7C8, 0x0A7C9, 0x0A7CA, 0x0A7CB, 0x0A7CC, 0x0A7CD, 0x0A7CE, 0x0A7CF, - 0x0A7D0, 0x0A7D1, 0x0A7D2, 0x0A7D3, 0x0A7D4, 0x0A7D5, 0x0A7D6, 0x0A7D7, - 0x0A7D8, 0x0A7D9, 0x0A7DA, 0x0A7DB, 0x0A7DC, 0x0A7DD, 0x0A7DE, 0x0A7DF, - 0x0A7E0, 0x0A7E1, 0x0A7E2, 0x0A7E3, 0x0A7E4, 0x0A7E5, 0x0A7E6, 0x0A7E7, - 0x0A7E8, 0x0A7E9, 0x0A7EA, 0x0A7EB, 0x0A7EC, 0x0A7ED, 0x0A7EE, 0x0A7EF, - 0x0A7F0, 0x0A7F1, 0x0A7F2, 0x0A7F3, 0x0A7F4, 0x0A7F5, 0x0A7F6, 0x0A7F7, - 0x0A7F8, 0x0A7F9, 0x0A7FA, 0x0A7FB, 0x0A7FC, 0x0A7FD, 0x0A7FE, 0x0A7FF, -}; - - -static const uint32_t nxt_unicode_block_1fe[128] nxt_aligned(64) = { - 0x0FF00, 0x0FF01, 0x0FF02, 0x0FF03, 0x0FF04, 0x0FF05, 0x0FF06, 0x0FF07, - 0x0FF08, 0x0FF09, 0x0FF0A, 0x0FF0B, 0x0FF0C, 0x0FF0D, 0x0FF0E, 0x0FF0F, - 0x0FF10, 0x0FF11, 0x0FF12, 0x0FF13, 0x0FF14, 0x0FF15, 0x0FF16, 0x0FF17, - 0x0FF18, 0x0FF19, 0x0FF1A, 0x0FF1B, 0x0FF1C, 0x0FF1D, 0x0FF1E, 0x0FF1F, - 0x0FF20, 0x0FF41, 0x0FF42, 0x0FF43, 0x0FF44, 0x0FF45, 0x0FF46, 0x0FF47, - 0x0FF48, 0x0FF49, 0x0FF4A, 0x0FF4B, 0x0FF4C, 0x0FF4D, 0x0FF4E, 0x0FF4F, - 0x0FF50, 0x0FF51, 0x0FF52, 0x0FF53, 0x0FF54, 0x0FF55, 0x0FF56, 0x0FF57, - 0x0FF58, 0x0FF59, 0x0FF5A, 0x0FF3B, 0x0FF3C, 0x0FF3D, 0x0FF3E, 0x0FF3F, - 0x0FF40, 0x0FF41, 0x0FF42, 0x0FF43, 0x0FF44, 0x0FF45, 0x0FF46, 0x0FF47, - 0x0FF48, 0x0FF49, 0x0FF4A, 0x0FF4B, 0x0FF4C, 0x0FF4D, 0x0FF4E, 0x0FF4F, - 0x0FF50, 0x0FF51, 0x0FF52, 0x0FF53, 0x0FF54, 0x0FF55, 0x0FF56, 0x0FF57, - 0x0FF58, 0x0FF59, 0x0FF5A, 0x0FF5B, 0x0FF5C, 0x0FF5D, 0x0FF5E, 0x0FF5F, - 0x0FF60, 0x0FF61, 0x0FF62, 0x0FF63, 0x0FF64, 0x0FF65, 0x0FF66, 0x0FF67, - 0x0FF68, 0x0FF69, 0x0FF6A, 0x0FF6B, 0x0FF6C, 0x0FF6D, 0x0FF6E, 0x0FF6F, - 0x0FF70, 0x0FF71, 0x0FF72, 0x0FF73, 0x0FF74, 0x0FF75, 0x0FF76, 0x0FF77, - 0x0FF78, 0x0FF79, 0x0FF7A, 0x0FF7B, 0x0FF7C, 0x0FF7D, 0x0FF7E, 0x0FF7F, -}; - - -static const uint32_t nxt_unicode_block_208[40] nxt_aligned(64) = { - 0x10428, 0x10429, 0x1042A, 0x1042B, 0x1042C, 0x1042D, 0x1042E, 0x1042F, - 0x10430, 0x10431, 0x10432, 0x10433, 0x10434, 0x10435, 0x10436, 0x10437, - 0x10438, 0x10439, 0x1043A, 0x1043B, 0x1043C, 0x1043D, 0x1043E, 0x1043F, - 0x10440, 0x10441, 0x10442, 0x10443, 0x10444, 0x10445, 0x10446, 0x10447, - 0x10448, 0x10449, 0x1044A, 0x1044B, 0x1044C, 0x1044D, 0x1044E, 0x1044F, -}; - - -static const uint32_t *nxt_unicode_blocks[] nxt_aligned(64) = { - nxt_unicode_block_000, - nxt_unicode_block_001, - nxt_unicode_block_002, - nxt_unicode_block_003, - nxt_unicode_block_004, - NULL, - nxt_unicode_block_006, - nxt_unicode_block_007, - nxt_unicode_block_008, - nxt_unicode_block_009, - nxt_unicode_block_00a, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - nxt_unicode_block_021, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - nxt_unicode_block_03c, - nxt_unicode_block_03d, - nxt_unicode_block_03e, - nxt_unicode_block_03f, - NULL, - NULL, - nxt_unicode_block_042, - nxt_unicode_block_043, - NULL, - NULL, - NULL, - NULL, - NULL, - nxt_unicode_block_049, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - nxt_unicode_block_058, - nxt_unicode_block_059, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - nxt_unicode_block_14c, - nxt_unicode_block_14d, - nxt_unicode_block_14e, - nxt_unicode_block_14f, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - nxt_unicode_block_1fe, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - nxt_unicode_block_208, -}; diff --git a/src/nxt_unicode_lowcase.pl b/src/nxt_unicode_lowcase.pl deleted file mode 100644 index abf649651..000000000 --- a/src/nxt_unicode_lowcase.pl +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/perl - -use warnings; -use strict; - -# BLOCK_SIZE should be 128, 256, 512, etc. The value 128 provides -# the minimum memory footprint for both 32-bit and 64-bit platforms. -use constant BLOCK_SIZE => 128; - -my %lowcase; -my %blocks; -my $max_block = 0; -my $max_lowcase = 0; - -while (<>) { - if (/^(\w+); (C|S); (\w+);/) { - my ($symbol, $folding) = (hex $1, hex $3); - $lowcase{$symbol} = $folding; - $blocks{int($symbol / BLOCK_SIZE)} = 1; - - if ($max_lowcase < $symbol) { - $max_lowcase = $symbol; - } - } -} - - -my $last_block_size = $max_lowcase % BLOCK_SIZE + 1; - - -for my $block (sort { $a <=> $b } keys %blocks) { - if ($max_block < $block) { - $max_block = $block; - } -} - - -my $blocks = scalar keys %blocks; - -printf("\n/*\n" . - " * %d %s-bytes blocks, %d pointers.\n" . - " * %d bytes on 32-bit platforms, %d bytes on 64-bit platforms.\n" . - " */\n\n", - $blocks, BLOCK_SIZE, $max_block + 1, - ($blocks - 1) * BLOCK_SIZE * 4 + $last_block_size + $max_block * 4, - ($blocks - 1) * BLOCK_SIZE * 4 + $last_block_size+ $max_block * 8); - -printf("#define NXT_UNICODE_MAX_LOWCASE 0x%05x\n\n", $max_lowcase); -printf("#define NXT_UNICODE_BLOCK_SIZE %d\n\n\n", BLOCK_SIZE); - - -for my $block (sort { $a <=> $b } keys %blocks) { - my $block_size = ($block != $max_block) ? BLOCK_SIZE : $last_block_size; - - print "static const uint32_t "; - printf("nxt_unicode_block_%03x[%d] nxt_aligned(64) = {", - $block, $block_size); - - for my $c (0 .. $block_size - 1) { - printf "\n " if $c % 8 == 0; - - my $n = $block * BLOCK_SIZE + $c; - - if (exists $lowcase{$n}) { - printf(" 0x%05x,", $lowcase{$n}); - - } else { - #print " .......,"; - printf(" 0x%05x,", $n); - } - } - - print "\n};\n\n\n"; -} - - -print "static const uint32_t *nxt_unicode_blocks[] nxt_aligned(64) = {\n"; - -for my $block (0 .. $max_block) { - if (exists($blocks{$block})) { - printf(" nxt_unicode_block_%03x,\n", $block); - - } else { - print " NULL,\n"; - } -} - -print "};\n"; diff --git a/src/nxt_unicode_macosx_lowcase.h b/src/nxt_unicode_macosx_lowcase.h deleted file mode 100644 index d0cc15186..000000000 --- a/src/nxt_unicode_macosx_lowcase.h +++ /dev/null @@ -1,816 +0,0 @@ - -/* - * 15 128-bytes blocks, 511 pointers. - * 9267 bytes on 32-bit platforms, 11307 bytes on 64-bit platforms. - */ - -#define NXT_UNICODE_MAX_LOWCASE 0x0FF3A - -#define NXT_UNICODE_BLOCK_SIZE 128 - - -static const uint32_t nxt_unicode_block_000[128] nxt_aligned(64) = { - 0x00000, 0x00001, 0x00002, 0x00003, 0x00004, 0x00005, 0x00006, 0x00007, - 0x00008, 0x00009, 0x0000A, 0x0000B, 0x0000C, 0x0000D, 0x0000E, 0x0000F, - 0x00010, 0x00011, 0x00012, 0x00013, 0x00014, 0x00015, 0x00016, 0x00017, - 0x00018, 0x00019, 0x0001A, 0x0001B, 0x0001C, 0x0001D, 0x0001E, 0x0001F, - 0x00020, 0x00021, 0x00022, 0x00023, 0x00024, 0x00025, 0x00026, 0x00027, - 0x00028, 0x00029, 0x0002A, 0x0002B, 0x0002C, 0x0002D, 0x0002E, 0x0002F, - 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, - 0x00038, 0x00039, 0x0003A, 0x0003B, 0x0003C, 0x0003D, 0x0003E, 0x0003F, - 0x00040, 0x00061, 0x00062, 0x00063, 0x00064, 0x00065, 0x00066, 0x00067, - 0x00068, 0x00069, 0x0006A, 0x0006B, 0x0006C, 0x0006D, 0x0006E, 0x0006F, - 0x00070, 0x00071, 0x00072, 0x00073, 0x00074, 0x00075, 0x00076, 0x00077, - 0x00078, 0x00079, 0x0007A, 0x0005B, 0x0005C, 0x0005D, 0x0005E, 0x0005F, - 0x00060, 0x00061, 0x00062, 0x00063, 0x00064, 0x00065, 0x00066, 0x00067, - 0x00068, 0x00069, 0x0006A, 0x0006B, 0x0006C, 0x0006D, 0x0006E, 0x0006F, - 0x00070, 0x00071, 0x00072, 0x00073, 0x00074, 0x00075, 0x00076, 0x00077, - 0x00078, 0x00079, 0x0007A, 0x0007B, 0x0007C, 0x0007D, 0x0007E, 0x0007F, -}; - - -static const uint32_t nxt_unicode_block_001[128] nxt_aligned(64) = { - 0x00080, 0x00081, 0x00082, 0x00083, 0x00084, 0x00085, 0x00086, 0x00087, - 0x00088, 0x00089, 0x0008A, 0x0008B, 0x0008C, 0x0008D, 0x0008E, 0x0008F, - 0x00090, 0x00091, 0x00092, 0x00093, 0x00094, 0x00095, 0x00096, 0x00097, - 0x00098, 0x00099, 0x0009A, 0x0009B, 0x0009C, 0x0009D, 0x0009E, 0x0009F, - 0x000A0, 0x000A1, 0x000A2, 0x000A3, 0x000A4, 0x000A5, 0x000A6, 0x000A7, - 0x000A8, 0x000A9, 0x000AA, 0x000AB, 0x000AC, 0x000AD, 0x000AE, 0x000AF, - 0x000B0, 0x000B1, 0x000B2, 0x000B3, 0x000B4, 0x000B5, 0x000B6, 0x000B7, - 0x000B8, 0x000B9, 0x000BA, 0x000BB, 0x000BC, 0x000BD, 0x000BE, 0x000BF, - 0x000E0, 0x000E1, 0x000E2, 0x000E3, 0x000E4, 0x000E5, 0x000E6, 0x000E7, - 0x000E8, 0x000E9, 0x000EA, 0x000EB, 0x000EC, 0x000ED, 0x000EE, 0x000EF, - 0x000F0, 0x000F1, 0x000F2, 0x000F3, 0x000F4, 0x000F5, 0x000F6, 0x000D7, - 0x000F8, 0x000F9, 0x000FA, 0x000FB, 0x000FC, 0x000FD, 0x000FE, 0x000DF, - 0x000E0, 0x000E1, 0x000E2, 0x000E3, 0x000E4, 0x000E5, 0x000E6, 0x000E7, - 0x000E8, 0x000E9, 0x000EA, 0x000EB, 0x000EC, 0x000ED, 0x000EE, 0x000EF, - 0x000F0, 0x000F1, 0x000F2, 0x000F3, 0x000F4, 0x000F5, 0x000F6, 0x000F7, - 0x000F8, 0x000F9, 0x000FA, 0x000FB, 0x000FC, 0x000FD, 0x000FE, 0x000FF, -}; - - -static const uint32_t nxt_unicode_block_002[128] nxt_aligned(64) = { - 0x00101, 0x00101, 0x00103, 0x00103, 0x00105, 0x00105, 0x00107, 0x00107, - 0x00109, 0x00109, 0x0010B, 0x0010B, 0x0010D, 0x0010D, 0x0010F, 0x0010F, - 0x00111, 0x00111, 0x00113, 0x00113, 0x00115, 0x00115, 0x00117, 0x00117, - 0x00119, 0x00119, 0x0011B, 0x0011B, 0x0011D, 0x0011D, 0x0011F, 0x0011F, - 0x00121, 0x00121, 0x00123, 0x00123, 0x00125, 0x00125, 0x00127, 0x00127, - 0x00129, 0x00129, 0x0012B, 0x0012B, 0x0012D, 0x0012D, 0x0012F, 0x0012F, - 0x00130, 0x00131, 0x00133, 0x00133, 0x00135, 0x00135, 0x00137, 0x00137, - 0x00138, 0x0013A, 0x0013A, 0x0013C, 0x0013C, 0x0013E, 0x0013E, 0x00140, - 0x00140, 0x00142, 0x00142, 0x00144, 0x00144, 0x00146, 0x00146, 0x00148, - 0x00148, 0x00149, 0x0014B, 0x0014B, 0x0014D, 0x0014D, 0x0014F, 0x0014F, - 0x00151, 0x00151, 0x00153, 0x00153, 0x00155, 0x00155, 0x00157, 0x00157, - 0x00159, 0x00159, 0x0015B, 0x0015B, 0x0015D, 0x0015D, 0x0015F, 0x0015F, - 0x00161, 0x00161, 0x00163, 0x00163, 0x00165, 0x00165, 0x00167, 0x00167, - 0x00169, 0x00169, 0x0016B, 0x0016B, 0x0016D, 0x0016D, 0x0016F, 0x0016F, - 0x00171, 0x00171, 0x00173, 0x00173, 0x00175, 0x00175, 0x00177, 0x00177, - 0x000FF, 0x0017A, 0x0017A, 0x0017C, 0x0017C, 0x0017E, 0x0017E, 0x0017F, -}; - - -static const uint32_t nxt_unicode_block_003[128] nxt_aligned(64) = { - 0x00180, 0x00253, 0x00183, 0x00183, 0x00185, 0x00185, 0x00254, 0x00188, - 0x00188, 0x00256, 0x00257, 0x0018C, 0x0018C, 0x0018D, 0x001DD, 0x00259, - 0x0025B, 0x00192, 0x00192, 0x00260, 0x00263, 0x00195, 0x00269, 0x00268, - 0x00199, 0x00199, 0x0019A, 0x0019B, 0x0026F, 0x00272, 0x0019E, 0x00275, - 0x001A1, 0x001A1, 0x001A3, 0x001A3, 0x001A5, 0x001A5, 0x001A6, 0x001A8, - 0x001A8, 0x00283, 0x001AA, 0x001AB, 0x001AD, 0x001AD, 0x00288, 0x001B0, - 0x001B0, 0x0028A, 0x0028B, 0x001B4, 0x001B4, 0x001B6, 0x001B6, 0x00292, - 0x001B9, 0x001B9, 0x001BA, 0x001BB, 0x001BD, 0x001BD, 0x001BE, 0x001BF, - 0x001C0, 0x001C1, 0x001C2, 0x001C3, 0x001C6, 0x001C6, 0x001C6, 0x001C9, - 0x001C9, 0x001C9, 0x001CC, 0x001CC, 0x001CC, 0x001CE, 0x001CE, 0x001D0, - 0x001D0, 0x001D2, 0x001D2, 0x001D4, 0x001D4, 0x001D6, 0x001D6, 0x001D8, - 0x001D8, 0x001DA, 0x001DA, 0x001DC, 0x001DC, 0x001DD, 0x001DF, 0x001DF, - 0x001E1, 0x001E1, 0x001E3, 0x001E3, 0x001E5, 0x001E5, 0x001E7, 0x001E7, - 0x001E9, 0x001E9, 0x001EB, 0x001EB, 0x001ED, 0x001ED, 0x001EF, 0x001EF, - 0x001F0, 0x001F3, 0x001F3, 0x001F3, 0x001F5, 0x001F5, 0x001F6, 0x001F7, - 0x001F9, 0x001F9, 0x001FB, 0x001FB, 0x001FD, 0x001FD, 0x001FF, 0x001FF, -}; - - -static const uint32_t nxt_unicode_block_004[128] nxt_aligned(64) = { - 0x00201, 0x00201, 0x00203, 0x00203, 0x00205, 0x00205, 0x00207, 0x00207, - 0x00209, 0x00209, 0x0020B, 0x0020B, 0x0020D, 0x0020D, 0x0020F, 0x0020F, - 0x00211, 0x00211, 0x00213, 0x00213, 0x00215, 0x00215, 0x00217, 0x00217, - 0x00219, 0x00219, 0x0021B, 0x0021B, 0x0021C, 0x0021D, 0x0021F, 0x0021F, - 0x00220, 0x00221, 0x00222, 0x00223, 0x00224, 0x00225, 0x00227, 0x00227, - 0x00229, 0x00229, 0x0022B, 0x0022B, 0x0022D, 0x0022D, 0x0022F, 0x0022F, - 0x00231, 0x00231, 0x00233, 0x00233, 0x00234, 0x00235, 0x00236, 0x00237, - 0x00238, 0x00239, 0x0023A, 0x0023B, 0x0023C, 0x0023D, 0x0023E, 0x0023F, - 0x00240, 0x00241, 0x00242, 0x00243, 0x00244, 0x00245, 0x00246, 0x00247, - 0x00248, 0x00249, 0x0024A, 0x0024B, 0x0024C, 0x0024D, 0x0024E, 0x0024F, - 0x00250, 0x00251, 0x00252, 0x00253, 0x00254, 0x00255, 0x00256, 0x00257, - 0x00258, 0x00259, 0x0025A, 0x0025B, 0x0025C, 0x0025D, 0x0025E, 0x0025F, - 0x00260, 0x00261, 0x00262, 0x00263, 0x00264, 0x00265, 0x00266, 0x00267, - 0x00268, 0x00269, 0x0026A, 0x0026B, 0x0026C, 0x0026D, 0x0026E, 0x0026F, - 0x00270, 0x00271, 0x00272, 0x00273, 0x00274, 0x00275, 0x00276, 0x00277, - 0x00278, 0x00279, 0x0027A, 0x0027B, 0x0027C, 0x0027D, 0x0027E, 0x0027F, -}; - - -static const uint32_t nxt_unicode_block_007[128] nxt_aligned(64) = { - 0x00380, 0x00381, 0x00382, 0x00383, 0x00384, 0x00385, 0x003AC, 0x00387, - 0x003AD, 0x003AE, 0x003AF, 0x0038B, 0x003CC, 0x0038D, 0x003CD, 0x003CE, - 0x00390, 0x003B1, 0x003B2, 0x003B3, 0x003B4, 0x003B5, 0x003B6, 0x003B7, - 0x003B8, 0x003B9, 0x003BA, 0x003BB, 0x003BC, 0x003BD, 0x003BE, 0x003BF, - 0x003C0, 0x003C1, 0x003A2, 0x003C3, 0x003C4, 0x003C5, 0x003C6, 0x003C7, - 0x003C8, 0x003C9, 0x003CA, 0x003CB, 0x003AC, 0x003AD, 0x003AE, 0x003AF, - 0x003B0, 0x003B1, 0x003B2, 0x003B3, 0x003B4, 0x003B5, 0x003B6, 0x003B7, - 0x003B8, 0x003B9, 0x003BA, 0x003BB, 0x003BC, 0x003BD, 0x003BE, 0x003BF, - 0x003C0, 0x003C1, 0x003C2, 0x003C3, 0x003C4, 0x003C5, 0x003C6, 0x003C7, - 0x003C8, 0x003C9, 0x003CA, 0x003CB, 0x003CC, 0x003CD, 0x003CE, 0x003CF, - 0x003D0, 0x003D1, 0x003D2, 0x003D3, 0x003D4, 0x003D5, 0x003D6, 0x003D7, - 0x003D8, 0x003D9, 0x003DA, 0x003DB, 0x003DC, 0x003DD, 0x003DE, 0x003DF, - 0x003E0, 0x003E1, 0x003E3, 0x003E3, 0x003E5, 0x003E5, 0x003E7, 0x003E7, - 0x003E9, 0x003E9, 0x003EB, 0x003EB, 0x003ED, 0x003ED, 0x003EF, 0x003EF, - 0x003F0, 0x003F1, 0x003F2, 0x003F3, 0x003F4, 0x003F5, 0x003F6, 0x003F7, - 0x003F8, 0x003F9, 0x003FA, 0x003FB, 0x003FC, 0x003FD, 0x003FE, 0x003FF, -}; - - -static const uint32_t nxt_unicode_block_008[128] nxt_aligned(64) = { - 0x00450, 0x00451, 0x00452, 0x00453, 0x00454, 0x00455, 0x00456, 0x00457, - 0x00458, 0x00459, 0x0045A, 0x0045B, 0x0045C, 0x0045D, 0x0045E, 0x0045F, - 0x00430, 0x00431, 0x00432, 0x00433, 0x00434, 0x00435, 0x00436, 0x00437, - 0x00438, 0x00439, 0x0043A, 0x0043B, 0x0043C, 0x0043D, 0x0043E, 0x0043F, - 0x00440, 0x00441, 0x00442, 0x00443, 0x00444, 0x00445, 0x00446, 0x00447, - 0x00448, 0x00449, 0x0044A, 0x0044B, 0x0044C, 0x0044D, 0x0044E, 0x0044F, - 0x00430, 0x00431, 0x00432, 0x00433, 0x00434, 0x00435, 0x00436, 0x00437, - 0x00438, 0x00439, 0x0043A, 0x0043B, 0x0043C, 0x0043D, 0x0043E, 0x0043F, - 0x00440, 0x00441, 0x00442, 0x00443, 0x00444, 0x00445, 0x00446, 0x00447, - 0x00448, 0x00449, 0x0044A, 0x0044B, 0x0044C, 0x0044D, 0x0044E, 0x0044F, - 0x00450, 0x00451, 0x00452, 0x00453, 0x00454, 0x00455, 0x00456, 0x00457, - 0x00458, 0x00459, 0x0045A, 0x0045B, 0x0045C, 0x0045D, 0x0045E, 0x0045F, - 0x00461, 0x00461, 0x00463, 0x00463, 0x00465, 0x00465, 0x00467, 0x00467, - 0x00469, 0x00469, 0x0046B, 0x0046B, 0x0046D, 0x0046D, 0x0046F, 0x0046F, - 0x00471, 0x00471, 0x00473, 0x00473, 0x00475, 0x00475, 0x00477, 0x00477, - 0x00479, 0x00479, 0x0047B, 0x0047B, 0x0047D, 0x0047D, 0x0047F, 0x0047F, -}; - - -static const uint32_t nxt_unicode_block_009[128] nxt_aligned(64) = { - 0x00481, 0x00481, 0x00482, 0x00483, 0x00484, 0x00485, 0x00486, 0x00487, - 0x00488, 0x00489, 0x0048A, 0x0048B, 0x0048C, 0x0048D, 0x0048E, 0x0048F, - 0x00491, 0x00491, 0x00493, 0x00493, 0x00495, 0x00495, 0x00497, 0x00497, - 0x00499, 0x00499, 0x0049B, 0x0049B, 0x0049D, 0x0049D, 0x0049F, 0x0049F, - 0x004A1, 0x004A1, 0x004A3, 0x004A3, 0x004A5, 0x004A5, 0x004A7, 0x004A7, - 0x004A9, 0x004A9, 0x004AB, 0x004AB, 0x004AD, 0x004AD, 0x004AF, 0x004AF, - 0x004B1, 0x004B1, 0x004B3, 0x004B3, 0x004B5, 0x004B5, 0x004B7, 0x004B7, - 0x004B9, 0x004B9, 0x004BB, 0x004BB, 0x004BD, 0x004BD, 0x004BF, 0x004BF, - 0x004C0, 0x004C2, 0x004C2, 0x004C4, 0x004C4, 0x004C5, 0x004C6, 0x004C8, - 0x004C8, 0x004C9, 0x004CA, 0x004CC, 0x004CC, 0x004CD, 0x004CE, 0x004CF, - 0x004D1, 0x004D1, 0x004D3, 0x004D3, 0x004D4, 0x004D5, 0x004D7, 0x004D7, - 0x004D8, 0x004D9, 0x004DA, 0x004DB, 0x004DD, 0x004DD, 0x004DF, 0x004DF, - 0x004E0, 0x004E1, 0x004E3, 0x004E3, 0x004E5, 0x004E5, 0x004E7, 0x004E7, - 0x004E8, 0x004E9, 0x004EA, 0x004EB, 0x004ED, 0x004ED, 0x004EF, 0x004EF, - 0x004F1, 0x004F1, 0x004F3, 0x004F3, 0x004F5, 0x004F5, 0x004F6, 0x004F7, - 0x004F9, 0x004F9, 0x004FA, 0x004FB, 0x004FC, 0x004FD, 0x004FE, 0x004FF, -}; - - -static const uint32_t nxt_unicode_block_00a[128] nxt_aligned(64) = { - 0x00500, 0x00501, 0x00502, 0x00503, 0x00504, 0x00505, 0x00506, 0x00507, - 0x00508, 0x00509, 0x0050A, 0x0050B, 0x0050C, 0x0050D, 0x0050E, 0x0050F, - 0x00510, 0x00511, 0x00512, 0x00513, 0x00514, 0x00515, 0x00516, 0x00517, - 0x00518, 0x00519, 0x0051A, 0x0051B, 0x0051C, 0x0051D, 0x0051E, 0x0051F, - 0x00520, 0x00521, 0x00522, 0x00523, 0x00524, 0x00525, 0x00526, 0x00527, - 0x00528, 0x00529, 0x0052A, 0x0052B, 0x0052C, 0x0052D, 0x0052E, 0x0052F, - 0x00530, 0x00561, 0x00562, 0x00563, 0x00564, 0x00565, 0x00566, 0x00567, - 0x00568, 0x00569, 0x0056A, 0x0056B, 0x0056C, 0x0056D, 0x0056E, 0x0056F, - 0x00570, 0x00571, 0x00572, 0x00573, 0x00574, 0x00575, 0x00576, 0x00577, - 0x00578, 0x00579, 0x0057A, 0x0057B, 0x0057C, 0x0057D, 0x0057E, 0x0057F, - 0x00580, 0x00581, 0x00582, 0x00583, 0x00584, 0x00585, 0x00586, 0x00557, - 0x00558, 0x00559, 0x0055A, 0x0055B, 0x0055C, 0x0055D, 0x0055E, 0x0055F, - 0x00560, 0x00561, 0x00562, 0x00563, 0x00564, 0x00565, 0x00566, 0x00567, - 0x00568, 0x00569, 0x0056A, 0x0056B, 0x0056C, 0x0056D, 0x0056E, 0x0056F, - 0x00570, 0x00571, 0x00572, 0x00573, 0x00574, 0x00575, 0x00576, 0x00577, - 0x00578, 0x00579, 0x0057A, 0x0057B, 0x0057C, 0x0057D, 0x0057E, 0x0057F, -}; - - -static const uint32_t nxt_unicode_block_03c[128] nxt_aligned(64) = { - 0x01E01, 0x01E01, 0x01E03, 0x01E03, 0x01E05, 0x01E05, 0x01E07, 0x01E07, - 0x01E09, 0x01E09, 0x01E0B, 0x01E0B, 0x01E0D, 0x01E0D, 0x01E0F, 0x01E0F, - 0x01E11, 0x01E11, 0x01E13, 0x01E13, 0x01E15, 0x01E15, 0x01E17, 0x01E17, - 0x01E19, 0x01E19, 0x01E1B, 0x01E1B, 0x01E1D, 0x01E1D, 0x01E1F, 0x01E1F, - 0x01E21, 0x01E21, 0x01E23, 0x01E23, 0x01E25, 0x01E25, 0x01E27, 0x01E27, - 0x01E29, 0x01E29, 0x01E2B, 0x01E2B, 0x01E2D, 0x01E2D, 0x01E2F, 0x01E2F, - 0x01E31, 0x01E31, 0x01E33, 0x01E33, 0x01E35, 0x01E35, 0x01E37, 0x01E37, - 0x01E39, 0x01E39, 0x01E3B, 0x01E3B, 0x01E3D, 0x01E3D, 0x01E3F, 0x01E3F, - 0x01E41, 0x01E41, 0x01E43, 0x01E43, 0x01E45, 0x01E45, 0x01E47, 0x01E47, - 0x01E49, 0x01E49, 0x01E4B, 0x01E4B, 0x01E4D, 0x01E4D, 0x01E4F, 0x01E4F, - 0x01E51, 0x01E51, 0x01E53, 0x01E53, 0x01E55, 0x01E55, 0x01E57, 0x01E57, - 0x01E59, 0x01E59, 0x01E5B, 0x01E5B, 0x01E5D, 0x01E5D, 0x01E5F, 0x01E5F, - 0x01E61, 0x01E61, 0x01E63, 0x01E63, 0x01E65, 0x01E65, 0x01E67, 0x01E67, - 0x01E69, 0x01E69, 0x01E6B, 0x01E6B, 0x01E6D, 0x01E6D, 0x01E6F, 0x01E6F, - 0x01E71, 0x01E71, 0x01E73, 0x01E73, 0x01E75, 0x01E75, 0x01E77, 0x01E77, - 0x01E79, 0x01E79, 0x01E7B, 0x01E7B, 0x01E7D, 0x01E7D, 0x01E7F, 0x01E7F, -}; - - -static const uint32_t nxt_unicode_block_03d[128] nxt_aligned(64) = { - 0x01E81, 0x01E81, 0x01E83, 0x01E83, 0x01E85, 0x01E85, 0x01E87, 0x01E87, - 0x01E89, 0x01E89, 0x01E8B, 0x01E8B, 0x01E8D, 0x01E8D, 0x01E8F, 0x01E8F, - 0x01E91, 0x01E91, 0x01E93, 0x01E93, 0x01E95, 0x01E95, 0x01E96, 0x01E97, - 0x01E98, 0x01E99, 0x01E9A, 0x01E9B, 0x01E9C, 0x01E9D, 0x01E9E, 0x01E9F, - 0x01EA1, 0x01EA1, 0x01EA3, 0x01EA3, 0x01EA5, 0x01EA5, 0x01EA7, 0x01EA7, - 0x01EA9, 0x01EA9, 0x01EAB, 0x01EAB, 0x01EAD, 0x01EAD, 0x01EAF, 0x01EAF, - 0x01EB1, 0x01EB1, 0x01EB3, 0x01EB3, 0x01EB5, 0x01EB5, 0x01EB7, 0x01EB7, - 0x01EB9, 0x01EB9, 0x01EBB, 0x01EBB, 0x01EBD, 0x01EBD, 0x01EBF, 0x01EBF, - 0x01EC1, 0x01EC1, 0x01EC3, 0x01EC3, 0x01EC5, 0x01EC5, 0x01EC7, 0x01EC7, - 0x01EC9, 0x01EC9, 0x01ECB, 0x01ECB, 0x01ECD, 0x01ECD, 0x01ECF, 0x01ECF, - 0x01ED1, 0x01ED1, 0x01ED3, 0x01ED3, 0x01ED5, 0x01ED5, 0x01ED7, 0x01ED7, - 0x01ED9, 0x01ED9, 0x01EDB, 0x01EDB, 0x01EDD, 0x01EDD, 0x01EDF, 0x01EDF, - 0x01EE1, 0x01EE1, 0x01EE3, 0x01EE3, 0x01EE5, 0x01EE5, 0x01EE7, 0x01EE7, - 0x01EE9, 0x01EE9, 0x01EEB, 0x01EEB, 0x01EED, 0x01EED, 0x01EEF, 0x01EEF, - 0x01EF1, 0x01EF1, 0x01EF3, 0x01EF3, 0x01EF5, 0x01EF5, 0x01EF7, 0x01EF7, - 0x01EF9, 0x01EF9, 0x01EFA, 0x01EFB, 0x01EFC, 0x01EFD, 0x01EFE, 0x01EFF, -}; - - -static const uint32_t nxt_unicode_block_03e[128] nxt_aligned(64) = { - 0x01F00, 0x01F01, 0x01F02, 0x01F03, 0x01F04, 0x01F05, 0x01F06, 0x01F07, - 0x01F00, 0x01F01, 0x01F02, 0x01F03, 0x01F04, 0x01F05, 0x01F06, 0x01F07, - 0x01F10, 0x01F11, 0x01F12, 0x01F13, 0x01F14, 0x01F15, 0x01F16, 0x01F17, - 0x01F10, 0x01F11, 0x01F12, 0x01F13, 0x01F14, 0x01F15, 0x01F1E, 0x01F1F, - 0x01F20, 0x01F21, 0x01F22, 0x01F23, 0x01F24, 0x01F25, 0x01F26, 0x01F27, - 0x01F20, 0x01F21, 0x01F22, 0x01F23, 0x01F24, 0x01F25, 0x01F26, 0x01F27, - 0x01F30, 0x01F31, 0x01F32, 0x01F33, 0x01F34, 0x01F35, 0x01F36, 0x01F37, - 0x01F30, 0x01F31, 0x01F32, 0x01F33, 0x01F34, 0x01F35, 0x01F36, 0x01F37, - 0x01F40, 0x01F41, 0x01F42, 0x01F43, 0x01F44, 0x01F45, 0x01F46, 0x01F47, - 0x01F40, 0x01F41, 0x01F42, 0x01F43, 0x01F44, 0x01F45, 0x01F4E, 0x01F4F, - 0x01F50, 0x01F51, 0x01F52, 0x01F53, 0x01F54, 0x01F55, 0x01F56, 0x01F57, - 0x01F58, 0x01F51, 0x01F5A, 0x01F53, 0x01F5C, 0x01F55, 0x01F5E, 0x01F57, - 0x01F60, 0x01F61, 0x01F62, 0x01F63, 0x01F64, 0x01F65, 0x01F66, 0x01F67, - 0x01F60, 0x01F61, 0x01F62, 0x01F63, 0x01F64, 0x01F65, 0x01F66, 0x01F67, - 0x01F70, 0x01F71, 0x01F72, 0x01F73, 0x01F74, 0x01F75, 0x01F76, 0x01F77, - 0x01F78, 0x01F79, 0x01F7A, 0x01F7B, 0x01F7C, 0x01F7D, 0x01F7E, 0x01F7F, -}; - - -static const uint32_t nxt_unicode_block_03f[128] nxt_aligned(64) = { - 0x01F80, 0x01F81, 0x01F82, 0x01F83, 0x01F84, 0x01F85, 0x01F86, 0x01F87, - 0x01F80, 0x01F81, 0x01F82, 0x01F83, 0x01F84, 0x01F85, 0x01F86, 0x01F87, - 0x01F90, 0x01F91, 0x01F92, 0x01F93, 0x01F94, 0x01F95, 0x01F96, 0x01F97, - 0x01F90, 0x01F91, 0x01F92, 0x01F93, 0x01F94, 0x01F95, 0x01F96, 0x01F97, - 0x01FA0, 0x01FA1, 0x01FA2, 0x01FA3, 0x01FA4, 0x01FA5, 0x01FA6, 0x01FA7, - 0x01FA0, 0x01FA1, 0x01FA2, 0x01FA3, 0x01FA4, 0x01FA5, 0x01FA6, 0x01FA7, - 0x01FB0, 0x01FB1, 0x01FB2, 0x01FB3, 0x01FB4, 0x01FB5, 0x01FB6, 0x01FB7, - 0x01FB0, 0x01FB1, 0x01F70, 0x01F71, 0x01FB3, 0x01FBD, 0x003B9, 0x01FBF, - 0x01FC0, 0x01FC1, 0x01FC2, 0x01FC3, 0x01FC4, 0x01FC5, 0x01FC6, 0x01FC7, - 0x01F72, 0x01F73, 0x01F74, 0x01F75, 0x01FC3, 0x01FCD, 0x01FCE, 0x01FCF, - 0x01FD0, 0x01FD1, 0x01FD2, 0x01FD3, 0x01FD4, 0x01FD5, 0x01FD6, 0x01FD7, - 0x01FD0, 0x01FD1, 0x01F76, 0x01F77, 0x01FDC, 0x01FDD, 0x01FDE, 0x01FDF, - 0x01FE0, 0x01FE1, 0x01FE2, 0x01FE3, 0x01FE4, 0x01FE5, 0x01FE6, 0x01FE7, - 0x01FE0, 0x01FE1, 0x01F7A, 0x01F7B, 0x01FE5, 0x01FED, 0x01FEE, 0x01FEF, - 0x01FF0, 0x01FF1, 0x01FF2, 0x01FF3, 0x01FF4, 0x01FF5, 0x01FF6, 0x01FF7, - 0x01F78, 0x01F79, 0x01F7C, 0x01F7D, 0x01FF3, 0x01FFD, 0x01FFE, 0x01FFF, -}; - - -static const uint32_t nxt_unicode_block_042[128] nxt_aligned(64) = { - 0x02100, 0x02101, 0x02102, 0x02103, 0x02104, 0x02105, 0x02106, 0x02107, - 0x02108, 0x02109, 0x0210A, 0x0210B, 0x0210C, 0x0210D, 0x0210E, 0x0210F, - 0x02110, 0x02111, 0x02112, 0x02113, 0x02114, 0x02115, 0x02116, 0x02117, - 0x02118, 0x02119, 0x0211A, 0x0211B, 0x0211C, 0x0211D, 0x0211E, 0x0211F, - 0x02120, 0x02121, 0x02122, 0x02123, 0x02124, 0x02125, 0x02126, 0x02127, - 0x02128, 0x02129, 0x0212A, 0x0212B, 0x0212C, 0x0212D, 0x0212E, 0x0212F, - 0x02130, 0x02131, 0x02132, 0x02133, 0x02134, 0x02135, 0x02136, 0x02137, - 0x02138, 0x02139, 0x0213A, 0x0213B, 0x0213C, 0x0213D, 0x0213E, 0x0213F, - 0x02140, 0x02141, 0x02142, 0x02143, 0x02144, 0x02145, 0x02146, 0x02147, - 0x02148, 0x02149, 0x0214A, 0x0214B, 0x0214C, 0x0214D, 0x0214E, 0x0214F, - 0x02150, 0x02151, 0x02152, 0x02153, 0x02154, 0x02155, 0x02156, 0x02157, - 0x02158, 0x02159, 0x0215A, 0x0215B, 0x0215C, 0x0215D, 0x0215E, 0x0215F, - 0x02170, 0x02171, 0x02172, 0x02173, 0x02174, 0x02175, 0x02176, 0x02177, - 0x02178, 0x02179, 0x0217A, 0x0217B, 0x0217C, 0x0217D, 0x0217E, 0x0217F, - 0x02170, 0x02171, 0x02172, 0x02173, 0x02174, 0x02175, 0x02176, 0x02177, - 0x02178, 0x02179, 0x0217A, 0x0217B, 0x0217C, 0x0217D, 0x0217E, 0x0217F, -}; - - -static const uint32_t nxt_unicode_block_1fe[59] nxt_aligned(64) = { - 0x0FF00, 0x0FF01, 0x0FF02, 0x0FF03, 0x0FF04, 0x0FF05, 0x0FF06, 0x0FF07, - 0x0FF08, 0x0FF09, 0x0FF0A, 0x0FF0B, 0x0FF0C, 0x0FF0D, 0x0FF0E, 0x0FF0F, - 0x0FF10, 0x0FF11, 0x0FF12, 0x0FF13, 0x0FF14, 0x0FF15, 0x0FF16, 0x0FF17, - 0x0FF18, 0x0FF19, 0x0FF1A, 0x0FF1B, 0x0FF1C, 0x0FF1D, 0x0FF1E, 0x0FF1F, - 0x0FF20, 0x0FF41, 0x0FF42, 0x0FF43, 0x0FF44, 0x0FF45, 0x0FF46, 0x0FF47, - 0x0FF48, 0x0FF49, 0x0FF4A, 0x0FF4B, 0x0FF4C, 0x0FF4D, 0x0FF4E, 0x0FF4F, - 0x0FF50, 0x0FF51, 0x0FF52, 0x0FF53, 0x0FF54, 0x0FF55, 0x0FF56, 0x0FF57, - 0x0FF58, 0x0FF59, 0x0FF5A, -}; - - -static const uint32_t *nxt_unicode_blocks[] nxt_aligned(64) = { - nxt_unicode_block_000, - nxt_unicode_block_001, - nxt_unicode_block_002, - nxt_unicode_block_003, - nxt_unicode_block_004, - NULL, - NULL, - nxt_unicode_block_007, - nxt_unicode_block_008, - nxt_unicode_block_009, - nxt_unicode_block_00a, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - nxt_unicode_block_03c, - nxt_unicode_block_03d, - nxt_unicode_block_03e, - nxt_unicode_block_03f, - NULL, - NULL, - nxt_unicode_block_042, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - nxt_unicode_block_1fe, -}; diff --git a/src/nxt_unit.c b/src/nxt_unit.c index dd33408b0..c62d5300f 100644 --- a/src/nxt_unit.c +++ b/src/nxt_unit.c @@ -5,6 +5,8 @@ #include "nxt_main.h" #include "nxt_port_memory_int.h" +#include "nxt_checked.h" +#include "nxt_span.h" #include "nxt_socket_msg.h" #include "nxt_port_queue.h" #include "nxt_app_queue.h" @@ -17,7 +19,11 @@ #include "nxt_websocket.h" #if (NXT_HAVE_MEMFD_CREATE) +#if (NXT_HAVE_LINUX_MEMFD_H) #include +#else +#include +#endif #endif #define NXT_UNIT_MAX_PLAIN_SIZE 1024 @@ -73,6 +79,17 @@ static int nxt_unit_process_new_port(nxt_unit_ctx_t *ctx, static int nxt_unit_ctx_ready(nxt_unit_ctx_t *ctx); static int nxt_unit_process_req_headers(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg, nxt_unit_request_info_t **preq); +static void nxt_unit_ctx_detached_start(nxt_unit_ctx_t *ctx); +static void nxt_unit_ctx_detached_done(nxt_unit_ctx_t *ctx); +static int nxt_unit_ctx_detached_retry(nxt_unit_ctx_t *ctx); +static int nxt_unit_detached_timeout(nxt_unit_ctx_impl_t *ctx_impl); +static int nxt_unit_detached_poll(nxt_unit_ctx_t *ctx, int fd); +static void nxt_unit_detached_sleep(nxt_unit_ctx_impl_t *ctx_impl); +static uint64_t nxt_unit_detached_now(void); +static uint64_t nxt_unit_detached_delay(nxt_unit_ctx_impl_t *ctx_impl, + uint64_t now); +static int nxt_unit_detached_wake(nxt_unit_ctx_impl_t *ctx_impl); +static int nxt_unit_send_detached(nxt_unit_ctx_t *ctx, uint8_t state); static int nxt_unit_process_req_body(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg); static int nxt_unit_request_check_response_port(nxt_unit_request_info_t *req, @@ -132,6 +149,8 @@ static int nxt_unit_check_rbuf_mmap(nxt_unit_ctx_t *ctx, static int nxt_unit_mmap_read(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg, nxt_unit_read_buf_t *rbuf); static int nxt_unit_get_mmap(nxt_unit_ctx_t *ctx, pid_t pid, uint32_t id); +static void nxt_unit_unpark_rbufs(nxt_unit_ctx_t *ctx, + nxt_queue_t *awaiting_rbuf); static void nxt_unit_mmap_release(nxt_unit_ctx_t *ctx, nxt_port_mmap_header_t *hdr, void *start, uint32_t size); static int nxt_unit_send_shm_ack(nxt_unit_ctx_t *ctx, pid_t pid); @@ -149,6 +168,7 @@ nxt_inline int nxt_unit_is_read_queue(nxt_unit_read_buf_t *rbuf); nxt_inline int nxt_unit_is_read_socket(nxt_unit_read_buf_t *rbuf); nxt_inline int nxt_unit_is_shm_ack(nxt_unit_read_buf_t *rbuf); nxt_inline int nxt_unit_is_quit(nxt_unit_read_buf_t *rbuf); +nxt_inline int nxt_unit_is_socket_quit(nxt_unit_read_buf_t *rbuf); static int nxt_unit_process_port_msg_impl(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port); static void nxt_unit_ctx_free(nxt_unit_ctx_impl_t *ctx_impl); @@ -175,8 +195,8 @@ static int nxt_unit_get_port(nxt_unit_ctx_t *ctx, nxt_unit_port_id_t *port_id); static ssize_t nxt_unit_port_send(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, const void *buf, size_t buf_size, const nxt_send_oob_t *oob); -static ssize_t nxt_unit_sendmsg(nxt_unit_ctx_t *ctx, int fd, - const void *buf, size_t buf_size, const nxt_send_oob_t *oob); +static ssize_t nxt_unit_sendmsg(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, + int fd, const void *buf, size_t buf_size, const nxt_send_oob_t *oob); static int nxt_unit_ctx_port_recv(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, nxt_unit_read_buf_t *rbuf); nxt_inline void nxt_unit_rbuf_cpy(nxt_unit_read_buf_t *dst, @@ -217,6 +237,7 @@ static char * nxt_unit_snprint_prefix(char *p, char *end, pid_t pid, static void *nxt_unit_lvlhsh_alloc(void *data, size_t size); static void nxt_unit_lvlhsh_free(void *data, void *p); static int nxt_unit_memcasecmp(const void *p1, const void *p2, size_t length); +nxt_inline void nxt_unit_shm_copy(void *dst, const void *src, size_t size); /* @@ -256,52 +277,29 @@ nxt_unit_response_buf_size(uint32_t max_fields_count, /* - * Validate that an sptr field within a peer-supplied buffer dereferences - * to a [length]-byte range that is wholly inside the buffer. Used at - * request-arrival time to vet every sptr in nxt_unit_request_t before - * the application sees it. - * - * sptr->base aliases the address of the sptr itself (the union encodes - * an offset relative to that location), so this also implicitly checks - * that the sptr is inside the buffer. + * nxt_unit_sptr_in_buf() moved to nxt_unit_sptr.h: the router's response + * parser (src/nxt_router.c) needs the exact same bounds check against a + * peer-supplied buffer, on the other side of the trust boundary. */ -static int -nxt_unit_sptr_in_buf(nxt_unit_sptr_t *sptr, uint32_t length, - void *buf_start, uint32_t buf_size) -{ - size_t sptr_off, end_off; - if ((uint8_t *) sptr < (uint8_t *) buf_start) { - return 0; - } - sptr_off = (uint8_t *) sptr - (uint8_t *) buf_start; +/* + * Copies "size" bytes out of shared memory, each read once through volatile. + * For a record whose size is not known before its first bytes are read. + */ - /* - * The sptr struct itself must fit inside the buffer before we - * dereference sptr->offset. Reject a buffer too small to hold an - * sptr first: buf_size is uint32_t and sizeof() is size_t, so - * "buf_size - sizeof(nxt_unit_sptr_t)" is evaluated in size_t and - * underflows to a huge value -- wrongly passing the bound check -- - * when buf_size is smaller than the struct. The short-circuit keeps - * the subtraction below from ever underflowing. - */ - if (buf_size < sizeof(nxt_unit_sptr_t) - || sptr_off > buf_size - sizeof(nxt_unit_sptr_t)) - { - return 0; - } +nxt_inline void +nxt_unit_shm_copy(void *dst, const void *src, size_t size) +{ + u_char *d; + const volatile u_char *s; - if (sptr->offset > buf_size - sptr_off) { - return 0; - } + d = dst; + s = src; - end_off = sptr_off + sptr->offset; - if (length > buf_size - end_off) { - return 0; + while (size-- != 0) { + *d++ = *s++; } - - return 1; } @@ -386,6 +384,12 @@ struct nxt_unit_read_buf_s { }; +enum { + NXT_UNIT_DETACHED_NONE = 0, + NXT_UNIT_DETACHED_RUNNING = 1, +}; + + struct nxt_unit_ctx_impl_s { nxt_unit_ctx_t ctx; @@ -425,6 +429,41 @@ struct nxt_unit_ctx_impl_s { uint8_t ready; /* 1 bit */ uint8_t quit_param; + /* + * The application answered a request with + * nxt_unit_request_done_detached() and has not returned from its + * request handler yet. Holds the router's view of this worker as + * busy, and holds off a graceful quit, until it does. + */ + uint8_t detached; /* 1 bit */ + + /* Failed FINISH sends so far; 0 when none is pending. */ + uint8_t detached_retries; + + /* + * Set while nxt_unit_process_port_msg() runs. The embedder owns the + * event loop, so a pending retry must not block it. + */ + uint8_t detached_nowait; /* 1 bit */ + + /* + * The last nxt_unit_process_port_msg() call found no message, and it + * returned NXT_UNIT_OK only for a pending FINISH retry. + */ + uint8_t detached_idle; /* 1 bit */ + + /* + * The earliest time for the next FINISH retry from + * nxt_unit_process_port_msg(), in milliseconds of a monotonic clock. + */ + uint64_t detached_deadline; + + /* + * The START edge was never delivered. The worker retires when the + * request handler returns. + */ + uint8_t detached_unreported; /* 1 bit */ + nxt_unit_mmap_buf_t ctx_buf[2]; nxt_unit_read_buf_t ctx_read_buf; @@ -798,6 +837,21 @@ nxt_unit_ctx_init(nxt_unit_impl_t *lib, nxt_unit_ctx_impl_t *ctx_impl, ctx_impl->ready = 0; ctx_impl->quit_param = NXT_QUIT_GRACEFUL; + /* + * Explicitly, like every field above it: this function initialises the + * context field by field and never memsets it, so anything left out + * starts as whatever the allocator left behind -- 0xAA in a debug + * build, which reads as "detached" and sends the finish report to a + * poison pointer. + */ + + ctx_impl->detached = NXT_UNIT_DETACHED_NONE; + ctx_impl->detached_retries = 0; + ctx_impl->detached_nowait = 0; + ctx_impl->detached_idle = 0; + ctx_impl->detached_deadline = 0; + ctx_impl->detached_unreported = 0; + nxt_queue_init(&ctx_impl->free_req); nxt_queue_init(&ctx_impl->free_ws); nxt_queue_init(&ctx_impl->active_req); @@ -1058,7 +1112,7 @@ nxt_unit_ready(nxt_unit_ctx_t *ctx, int ready_fd, uint32_t stream, int queue_fd) nxt_socket_msg_oob_init(&oob, fds); - res = nxt_unit_sendmsg(ctx, ready_fd, &msg, sizeof(msg), &oob); + res = nxt_unit_sendmsg(ctx, NULL, ready_fd, &msg, sizeof(msg), &oob); if (res != sizeof(msg)) { return NXT_UNIT_ERROR; } @@ -1086,7 +1140,20 @@ nxt_unit_process_msg(nxt_unit_ctx_t *ctx, nxt_unit_read_buf_t *rbuf, rc = nxt_socket_msg_oob_get_fds(&rbuf->oob, recv_msg.fd); if (nxt_slow_path(rc != NXT_OK)) { - nxt_unit_alert(ctx, "failed to receive file descriptor over cmsg"); + if (rbuf->oob.truncated) { + /* + * The descriptor this message was meant to carry may not be + * here: NEW_PORT and MMAP would take their fd-less path on a + * message that otherwise looks whole. Refuse it; "done" closes + * whatever part of the SCM_RIGHTS did arrive. + */ + nxt_unit_alert(ctx, "control data truncated on a %d byte " + "message; message dropped", (int) rbuf->size); + + } else { + nxt_unit_alert(ctx, "failed to receive file descriptor over cmsg"); + } + rc = NXT_UNIT_ERROR; goto done; } @@ -1276,6 +1343,73 @@ nxt_unit_process_msg(nxt_unit_ctx_t *ctx, nxt_unit_read_buf_t *rbuf, } +#if (NXT_TESTS || NXT_FUZZ_BUILD) + +/* + * Feeds one message through nxt_unit_process_msg() as if it had just been + * read from a port of "ctx", then reads the buffers it made pending and + * runs whatever request it made ready through callbacks.request_handler, + * the way the read loops do. "fd", when + * not -1, arrives as the message's SCM_RIGHTS descriptor and is owned by + * libunit from here on. Used by src/test/nxt_unit_msg_test.c and + * fuzzing/nxt_unit_msg_fuzz.c. + */ + +int +nxt_unit_test_process_msg(nxt_unit_ctx_t *ctx, const void *msg, size_t size, + int fd) +{ + int rc; + struct cmsghdr *cmsg; + nxt_unit_read_buf_t *rbuf; + + rbuf = nxt_unit_read_buf_get(ctx); + if (nxt_slow_path(rbuf == NULL)) { + return NXT_UNIT_ERROR; + } + + if (nxt_slow_path(size > sizeof(rbuf->buf))) { + nxt_unit_read_buf_release(ctx, rbuf); + + return NXT_UNIT_ERROR; + } + + /* Deterministic bytes past the message, whatever the buffer held. */ + memset(rbuf->buf, 0, sizeof(rbuf->buf)); + memcpy(rbuf->buf, msg, size); + + rbuf->size = size; + rbuf->oob.size = 0; + rbuf->oob.truncated = 0; + + if (fd != -1) { + cmsg = (struct cmsghdr *) rbuf->oob.buf; + + memset(cmsg, 0, CMSG_SPACE(sizeof(int))); + + cmsg->cmsg_len = CMSG_LEN(sizeof(int)); + cmsg->cmsg_level = SOL_SOCKET; + cmsg->cmsg_type = SCM_RIGHTS; + + memcpy(CMSG_DATA(cmsg), &fd, sizeof(int)); + + rbuf->oob.size = CMSG_SPACE(sizeof(int)); + } + + rc = nxt_unit_process_msg(ctx, rbuf, NULL); + + if (rc != NXT_UNIT_ERROR) { + rc = nxt_unit_process_pending_rbuf(ctx); + } + + nxt_unit_process_ready_req(ctx); + + return rc; +} + +#endif + + static int nxt_unit_process_new_port(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg) { @@ -1338,6 +1472,39 @@ nxt_unit_process_new_port(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg) } +#if (NXT_TESTS) + +/* + * Adds a port with a shared memory queue, as NEW_PORT does for a router + * engine port. libunit owns "out_fd" and "queue" from here on; "queue" is + * a mapping of sizeof(nxt_port_queue_t) bytes. + */ + +int +nxt_unit_test_add_queue_port(nxt_unit_ctx_t *ctx, pid_t pid, uint16_t id, + int out_fd, void *queue) +{ + nxt_unit_port_t new_port, *port; + + nxt_unit_port_id_init(&new_port.id, pid, id); + + new_port.in_fd = -1; + new_port.out_fd = out_fd; + new_port.data = NULL; + + port = nxt_unit_add_port(ctx, &new_port, queue); + if (nxt_slow_path(port == NULL)) { + return NXT_UNIT_ERROR; + } + + nxt_unit_port_release(port); + + return NXT_UNIT_OK; +} + +#endif + + static int nxt_unit_ctx_ready(nxt_unit_ctx_t *ctx) { @@ -1383,9 +1550,10 @@ nxt_unit_process_req_headers(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg, nxt_unit_request_info_t **preq) { int res; + char *method, *target, *preread; nxt_unit_impl_t *lib; nxt_unit_port_id_t port_id; - nxt_unit_request_t *r; + nxt_unit_request_t hdr; nxt_unit_mmap_buf_t *b; nxt_unit_request_info_t *req; nxt_unit_request_info_impl_t *req_impl; @@ -1406,17 +1574,23 @@ nxt_unit_process_req_headers(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg, } /* - * Validate every sptr in the request struct before any code path - * dereferences it. Offsets originate from the router (a more - * privileged peer) but the libunit ABI is also reachable from - * attacker-influenced input shapes; keeping the validation - * co-located with arrival makes the trust boundary explicit. + * The request is in a segment that every process of the application + * maps writable: the router wrote it, but a sibling process can change + * it at any time. So every value is read once. The fixed part and + * each field are copied out through volatile, the checks and the uses + * take the copies, and every sptr is resolved with + * nxt_unit_sptr_in_buf(), which reads the offset once. The pointers it + * returns are the ones used. */ { + void *name, *value, *end; uint32_t i; + nxt_unit_field_t uf; nxt_unit_request_t *vr = recv_msg->start; uint32_t vsize = recv_msg->size; + hdr = *(volatile nxt_unit_request_t *) vr; + /* * The fields[] array trails the fixed request struct; its region * (fields_count * sizeof(nxt_unit_field_t)) must lie within the @@ -1426,36 +1600,39 @@ nxt_unit_process_req_headers(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg, * multiplication from overflowing a 32-bit fields_count. */ if (nxt_slow_path(sizeof(nxt_unit_request_t) - + (uint64_t) vr->fields_count + + (uint64_t) hdr.fields_count * sizeof(nxt_unit_field_t) > vsize)) { nxt_unit_warn(ctx, "#%"PRIu32": malformed request: fields_count " "%"PRIu32" exceeds buffer", recv_msg->stream, - vr->fields_count); + hdr.fields_count); return NXT_UNIT_ERROR; } + method = nxt_unit_sptr_in_buf(&vr->method, hdr.method_length, + recv_msg->start, vsize); + target = nxt_unit_sptr_in_buf(&vr->target, hdr.target_length, + recv_msg->start, vsize); + preread = nxt_unit_sptr_in_buf(&vr->preread_content, 0, + recv_msg->start, vsize); + if (nxt_slow_path( - !nxt_unit_sptr_in_buf(&vr->method, vr->method_length, - recv_msg->start, vsize) - || !nxt_unit_sptr_in_buf(&vr->version, vr->version_length, + method == NULL || target == NULL || preread == NULL + || !nxt_unit_sptr_in_buf(&vr->version, hdr.version_length, recv_msg->start, vsize) - || !nxt_unit_sptr_in_buf(&vr->remote, vr->remote_length, + || !nxt_unit_sptr_in_buf(&vr->remote, hdr.remote_length, recv_msg->start, vsize) - || !nxt_unit_sptr_in_buf(&vr->local_addr, vr->local_addr_length, + || !nxt_unit_sptr_in_buf(&vr->local_addr, hdr.local_addr_length, recv_msg->start, vsize) - || !nxt_unit_sptr_in_buf(&vr->local_port, vr->local_port_length, + || !nxt_unit_sptr_in_buf(&vr->local_port, hdr.local_port_length, recv_msg->start, vsize) - || !nxt_unit_sptr_in_buf(&vr->server_name, vr->server_name_length, + || !nxt_unit_sptr_in_buf(&vr->server_name, + hdr.server_name_length, recv_msg->start, vsize) - || !nxt_unit_sptr_in_buf(&vr->target, vr->target_length, + || !nxt_unit_sptr_in_buf(&vr->path, hdr.path_length, recv_msg->start, vsize) - || !nxt_unit_sptr_in_buf(&vr->path, vr->path_length, - recv_msg->start, vsize) - || !nxt_unit_sptr_in_buf(&vr->query, vr->query_length, - recv_msg->start, vsize) - || !nxt_unit_sptr_in_buf(&vr->preread_content, 0, + || !nxt_unit_sptr_in_buf(&vr->query, hdr.query_length, recv_msg->start, vsize))) { nxt_unit_warn(ctx, "#%"PRIu32": malformed request: " @@ -1463,14 +1640,25 @@ nxt_unit_process_req_headers(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg, return NXT_UNIT_ERROR; } - for (i = 0; i < vr->fields_count; i++) { - if (nxt_slow_path( - !nxt_unit_sptr_in_buf(&vr->fields[i].name, - vr->fields[i].name_length, - recv_msg->start, vsize) - || !nxt_unit_sptr_in_buf(&vr->fields[i].value, - vr->fields[i].value_length, - recv_msg->start, vsize))) + /* + * Field strings must also lie past fields[]: the router puts them + * there, and nxt_unit_request_group_dup_fields() moves fields one + * slot on by subtracting sizeof(nxt_unit_field_t) from their + * offsets, which a target inside fields[] would underflow. + */ + end = &vr->fields[hdr.fields_count]; + + for (i = 0; i < hdr.fields_count; i++) { + uf = *(volatile nxt_unit_field_t *) &vr->fields[i]; + + name = nxt_unit_sptr_in_buf(&vr->fields[i].name, uf.name_length, + recv_msg->start, vsize); + value = nxt_unit_sptr_in_buf(&vr->fields[i].value, + uf.value_length, + recv_msg->start, vsize); + + if (nxt_slow_path(name == NULL || value == NULL + || name < end || value < end)) { nxt_unit_warn(ctx, "#%"PRIu32": malformed request: field " "%"PRIu32" sptr out of buffer", @@ -1487,14 +1675,14 @@ nxt_unit_process_req_headers(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg, * read. Reject any that is neither "unset" nor a valid index. */ if (nxt_slow_path( - (vr->content_length_field != NXT_UNIT_NONE_FIELD - && vr->content_length_field >= vr->fields_count) - || (vr->content_type_field != NXT_UNIT_NONE_FIELD - && vr->content_type_field >= vr->fields_count) - || (vr->cookie_field != NXT_UNIT_NONE_FIELD - && vr->cookie_field >= vr->fields_count) - || (vr->authorization_field != NXT_UNIT_NONE_FIELD - && vr->authorization_field >= vr->fields_count))) + (hdr.content_length_field != NXT_UNIT_NONE_FIELD + && hdr.content_length_field >= hdr.fields_count) + || (hdr.content_type_field != NXT_UNIT_NONE_FIELD + && hdr.content_type_field >= hdr.fields_count) + || (hdr.cookie_field != NXT_UNIT_NONE_FIELD + && hdr.cookie_field >= hdr.fields_count) + || (hdr.authorization_field != NXT_UNIT_NONE_FIELD + && hdr.authorization_field >= hdr.fields_count))) { nxt_unit_warn(ctx, "#%"PRIu32": malformed request: cached field " "index out of range", recv_msg->stream); @@ -1520,12 +1708,11 @@ nxt_unit_process_req_headers(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg, req->response = NULL; req->response_buf = NULL; - r = req->request; - - req->content_length = r->content_length; + req->content_length = hdr.content_length; + /* The pointer from the check, not the sptr read again. */ req->content_buf = req->request_buf; - req->content_buf->free = nxt_unit_sptr_get(&r->preread_content); + req->content_buf->free = preread; req_impl->stream = recv_msg->stream; @@ -1549,11 +1736,9 @@ nxt_unit_process_req_headers(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg, req_impl->in_hash = 0; nxt_unit_debug(ctx, "#%"PRIu32": %.*s %.*s (%d)", recv_msg->stream, - (int) r->method_length, - (char *) nxt_unit_sptr_get(&r->method), - (int) r->target_length, - (char *) nxt_unit_sptr_get(&r->target), - (int) r->content_length); + (int) hdr.method_length, method, + (int) hdr.target_length, target, + (int) hdr.content_length); nxt_unit_port_id_init(&port_id, recv_msg->pid, recv_msg->reply_port); @@ -1596,6 +1781,8 @@ nxt_unit_process_req_headers(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg, if (preq == NULL) { lib->callbacks.request_handler(req); + nxt_unit_ctx_detached_done(ctx); + } else { *preq = req; } @@ -1652,6 +1839,8 @@ nxt_unit_process_req_body(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg) if (req->content_fd != -1 || l == req->content_length) { lib->callbacks.request_handler(req); + + nxt_unit_ctx_detached_done(ctx); } return NXT_UNIT_OK; @@ -1815,8 +2004,9 @@ nxt_unit_send_req_headers_ack(nxt_unit_request_info_t *req) static int nxt_unit_process_websocket(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg) { - size_t hsize; + size_t size, hsize; nxt_unit_impl_t *lib; + nxt_websocket_header_t wsh; nxt_unit_mmap_buf_t *b; nxt_unit_callbacks_t *cb; nxt_unit_request_info_t *req; @@ -1879,14 +2069,24 @@ nxt_unit_process_websocket(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg) ws_impl->ws.header = (void *) b->buf.start; - if (nxt_slow_path((size_t) (b->buf.end - b->buf.start) < 2)) { + size = b->buf.end - b->buf.start; + + if (nxt_slow_path(size < 2)) { nxt_unit_warn(ctx, "#%"PRIu32": truncated websocket frame header", req_impl->stream); nxt_unit_websocket_frame_release(&ws_impl->ws); return NXT_UNIT_ERROR; } - hsize = nxt_websocket_frame_header_size(ws_impl->ws.header); + /* + * The frame is in shared memory a sibling process can write, so the + * header is copied out once: the two fixed bytes first, which give + * the header size, then the rest. The length and the mask flag + * come from the copy. + */ + nxt_unit_shm_copy(&wsh, b->buf.start, 2); + + hsize = nxt_websocket_frame_header_size(&wsh); /* * Reject truncated frames before reading the extended length / @@ -1895,21 +2095,24 @@ nxt_unit_process_websocket(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg) * otherwise OOB-read b->buf.start + hsize - 4 (mask) and the * 8-byte extended length, and break the buffer invariant. */ - if (nxt_slow_path((size_t) (b->buf.end - b->buf.start) < hsize)) { + if (nxt_slow_path(size < hsize)) { nxt_unit_warn(ctx, "#%"PRIu32": truncated websocket frame: " "hsize %zu > buf size %zu", - req_impl->stream, hsize, - (size_t) (b->buf.end - b->buf.start)); + req_impl->stream, hsize, size); nxt_unit_websocket_frame_release(&ws_impl->ws); return NXT_UNIT_ERROR; } - ws_impl->ws.payload_len = nxt_websocket_frame_payload_len( - ws_impl->ws.header); + if (hsize > 2) { + nxt_unit_shm_copy(wsh.payload_len_, b->buf.start + 2, + nxt_min(hsize, sizeof(wsh)) - 2); + } + + ws_impl->ws.payload_len = nxt_websocket_frame_payload_len(&wsh); - if (ws_impl->ws.header->mask) { + if (wsh.mask) { ws_impl->ws.mask = (uint8_t *) b->buf.start + hsize - 4; } else { @@ -2164,9 +2367,10 @@ nxt_unit_field_hash(const char *name, size_t name_length) void nxt_unit_request_group_dup_fields(nxt_unit_request_info_t *req) { - char *name; - uint32_t i, j; - nxt_unit_field_t *fields, f; + char *name, *jname; + void *start; + uint32_t i, j, n, size; + nxt_unit_field_t *fields, f, fi, fj; nxt_unit_request_t *r; static const nxt_str_t content_length = nxt_string("content-length"); @@ -2178,12 +2382,41 @@ nxt_unit_request_group_dup_fields(nxt_unit_request_info_t *req) r = req->request; fields = r->fields; - for (i = 0; i < r->fields_count; i++) { - name = nxt_unit_sptr_get(&fields[i].name); + /* + * The request was checked on arrival, but it is in shared memory a + * sibling process can write, and this runs later. So the count is read + * once and checked again, each field is copied out before use, and the + * names are resolved with nxt_unit_sptr_in_buf(). A request that does + * not pass is left as it is. + */ + start = req->request_buf->start; + size = req->request_buf->end - req->request_buf->start; + + n = *(volatile uint32_t *) &r->fields_count; + + if (nxt_slow_path(sizeof(nxt_unit_request_t) + + (uint64_t) n * sizeof(nxt_unit_field_t) + > size)) + { + nxt_unit_req_warn(req, "group_dup_fields: fields_count %"PRIu32 + " exceeds buffer", n); + return; + } + + for (i = 0; i < n; i++) { + fi = *(volatile nxt_unit_field_t *) &fields[i]; + + name = nxt_unit_sptr_in_buf(&fields[i].name, fi.name_length, + start, size); + if (nxt_slow_path(name == NULL)) { + nxt_unit_req_warn(req, "group_dup_fields: field %"PRIu32 + " name out of buffer", i); + return; + } - switch (fields[i].hash) { + switch (fi.hash) { case NXT_UNIT_HASH_CONTENT_LENGTH: - if (fields[i].name_length == content_length.length + if (fi.name_length == content_length.length && nxt_unit_memcasecmp(name, content_length.start, content_length.length) == 0) { @@ -2193,7 +2426,7 @@ nxt_unit_request_group_dup_fields(nxt_unit_request_info_t *req) break; case NXT_UNIT_HASH_CONTENT_TYPE: - if (fields[i].name_length == content_type.length + if (fi.name_length == content_type.length && nxt_unit_memcasecmp(name, content_type.start, content_type.length) == 0) { @@ -2203,7 +2436,7 @@ nxt_unit_request_group_dup_fields(nxt_unit_request_info_t *req) break; case NXT_UNIT_HASH_COOKIE: - if (fields[i].name_length == cookie.length + if (fi.name_length == cookie.length && nxt_unit_memcasecmp(name, cookie.start, cookie.length) == 0) { @@ -2213,17 +2446,26 @@ nxt_unit_request_group_dup_fields(nxt_unit_request_info_t *req) break; } - for (j = i + 1; j < r->fields_count; j++) { - if (fields[i].hash != fields[j].hash - || fields[i].name_length != fields[j].name_length - || nxt_unit_memcasecmp(name, - nxt_unit_sptr_get(&fields[j].name), - fields[j].name_length) != 0) - { + for (j = i + 1; j < n; j++) { + fj = *(volatile nxt_unit_field_t *) &fields[j]; + + if (fi.hash != fj.hash || fi.name_length != fj.name_length) { + continue; + } + + jname = nxt_unit_sptr_in_buf(&fields[j].name, fj.name_length, + start, size); + if (nxt_slow_path(jname == NULL)) { + nxt_unit_req_warn(req, "group_dup_fields: field %"PRIu32 + " name out of buffer", j); + return; + } + + if (nxt_unit_memcasecmp(name, jname, fj.name_length) != 0) { continue; } - f = fields[j]; + f = fj; f.value.offset += (j - (i + 1)) * sizeof(f); while (j > i + 1) { @@ -3019,7 +3261,7 @@ nxt_unit_read_buf_get(nxt_unit_ctx_t *ctx) pthread_mutex_unlock(&ctx_impl->mutex); - rbuf->oob.size = 0; + nxt_socket_msg_oob_reset(&rbuf->oob); return rbuf; } @@ -3479,102 +3721,728 @@ nxt_unit_buf_read(nxt_unit_buf_t **b, uint64_t *len, void *dst, size_t size) } +/* + * Tell the router this worker is, or is no longer, running work of its own + * after a response. One byte of payload says which edge; see + * nxt_port_detached_t in src/nxt_port.h for why it is a byte and a new + * message type rather than a flag on the header. + * + * Sent to the router's own port, the way OOSM is, not to the port the + * response went to. The router reads that port on its main thread, which + * is the thread that owns the accounting, and one socket keeps the finish + * edge behind the start edge that preceded it: the next request's response + * may belong to another router engine, and two engines' reads of two + * sockets race. The router looks the worker up by pid. + */ + +#if (NXT_TESTS) +static unsigned int nxt_unit_test_send_detached_failure_count; + + void -nxt_unit_request_done(nxt_unit_request_info_t *req, int rc) +nxt_unit_test_send_detached_failures(unsigned int failures) { - uint32_t size; - nxt_port_msg_t msg = {}; - nxt_unit_impl_t *lib; - nxt_unit_request_info_impl_t *req_impl; + nxt_unit_test_send_detached_failure_count = failures; +} +#endif - req_impl = nxt_container_of(req, nxt_unit_request_info_impl_t, req); - nxt_unit_req_debug(req, "done: %d", rc); +static int +nxt_unit_send_detached(nxt_unit_ctx_t *ctx, uint8_t state) +{ + int res; + nxt_unit_impl_t *lib; + struct { + nxt_port_msg_t msg; + uint8_t state; + } m; - if (nxt_slow_path(rc != NXT_UNIT_OK)) { - goto skip_response_send; +#if (NXT_TESTS) + if (nxt_slow_path(nxt_unit_test_send_detached_failure_count > 0)) { + nxt_unit_test_send_detached_failure_count--; + return NXT_UNIT_ERROR; } +#endif - if (nxt_slow_path(req_impl->state < NXT_UNIT_RS_RESPONSE_INIT)) { - - size = nxt_length("Content-Type") + nxt_length("text/plain"); - - rc = nxt_unit_response_init(req, 200, 1, size); - if (nxt_slow_path(rc != NXT_UNIT_OK)) { - goto skip_response_send; - } + lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); - rc = nxt_unit_response_add_field(req, "Content-Type", - nxt_length("Content-Type"), - "text/plain", nxt_length("text/plain")); - if (nxt_slow_path(rc != NXT_UNIT_OK)) { - goto skip_response_send; - } + if (nxt_slow_path(lib->router_port == NULL)) { + nxt_unit_debug(ctx, "detached %d: no router port to report on", + (int) state); + return NXT_UNIT_ERROR; } - if (nxt_slow_path(req_impl->state < NXT_UNIT_RS_RESPONSE_SENT)) { - - req_impl->state = NXT_UNIT_RS_RESPONSE_SENT; + memset(&m, 0, sizeof(m)); - nxt_unit_buf_send_done(req->response_buf); + m.msg.pid = lib->pid; + m.msg.type = _NXT_PORT_MSG_DETACHED; + m.msg.last = 1; + m.state = state; - return; + res = nxt_unit_port_send(ctx, lib->router_port, &m, sizeof(m), NULL); + if (nxt_slow_path(res != sizeof(m))) { + return NXT_UNIT_ERROR; } -skip_response_send: + return NXT_UNIT_OK; +} - lib = nxt_container_of(req->unit, nxt_unit_impl_t, unit); - msg.stream = req_impl->stream; - msg.pid = lib->pid; - msg.type = (rc == NXT_UNIT_OK) ? _NXT_PORT_MSG_DATA - : _NXT_PORT_MSG_RPC_ERROR; - msg.last = 1; +void +nxt_unit_request_done_detached(nxt_unit_request_info_t *req, int rc) +{ + /* + * Before the response, not after. The two go to different ports, so + * this orders nothing on the router's side by itself -- the router + * takes a worker it has already parked as idle back out when the edge + * arrives -- but it does keep the window in which the port sits in the + * idle queue as short as the router's own read makes it. + * + * A failure here is not fatal to the request: the response still goes + * out below. + */ - (void) nxt_unit_port_send(req->ctx, req->response_port, - &msg, sizeof(msg), NULL); + nxt_unit_ctx_detached_start(req->ctx); - nxt_unit_request_info_release(req); + nxt_unit_request_done(req, rc); } -int -nxt_unit_websocket_send(nxt_unit_request_info_t *req, uint8_t opcode, - uint8_t last, const void *start, size_t size) -{ - const struct iovec iov = { (void *) start, size }; - - return nxt_unit_websocket_sendv(req, opcode, last, &iov, 1); -} +/* + * Report the START edge. The application continues inside its request + * handler, and the read loop does not run again until the handler returns, + * so a retry from the read loop is too late. Retry here: 8 attempts, with + * sleeps of 1, 2, ... 64 ms between them, 127 ms in total. + * + * If all attempts fail, the router counts this worker idle while the + * application runs. Do not set ->detached, because the router does not + * know about the work. Retire the worker when the handler returns. + */ +#define NXT_UNIT_DETACHED_START_ATTEMPTS 8 -int -nxt_unit_websocket_sendv(nxt_unit_request_info_t *req, uint8_t opcode, - uint8_t last, const struct iovec *iov, int iovcnt) +static void +nxt_unit_ctx_detached_start(nxt_unit_ctx_t *ctx) { - int i, rc; - size_t l, copy; - uint32_t payload_len, buf_size, alloc_size; - const uint8_t *b; - nxt_unit_buf_t *buf; - nxt_unit_mmap_buf_t mmap_buf; - nxt_websocket_header_t *wh; - char local_buf[NXT_UNIT_LOCAL_BUF_SIZE]; + int i; + struct timespec ts; + nxt_unit_ctx_impl_t *ctx_impl; - payload_len = 0; + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); - for (i = 0; i < iovcnt; i++) { - payload_len += iov[i].iov_len; + if (ctx_impl->detached != NXT_UNIT_DETACHED_NONE + || ctx_impl->detached_unreported) + { + return; } - buf_size = 10 + payload_len; - alloc_size = nxt_min(buf_size, PORT_MMAP_DATA_SIZE); + for (i = 0; i < NXT_UNIT_DETACHED_START_ATTEMPTS; i++) { - rc = nxt_unit_get_outgoing_buf(req->ctx, req->response_port, - alloc_size, alloc_size, - &mmap_buf, local_buf); - if (nxt_slow_path(rc != NXT_UNIT_OK)) { - return rc; + if (i > 0) { + ts.tv_sec = 0; + ts.tv_nsec = (1L << (i - 1)) * 1000000L; + + (void) nanosleep(&ts, NULL); + } + + if (nxt_unit_send_detached(ctx, NXT_PORT_DETACHED_START) + == NXT_UNIT_OK) + { + ctx_impl->detached = NXT_UNIT_DETACHED_RUNNING; + return; + } + } + + ctx_impl->detached_unreported = 1; + + nxt_unit_alert(ctx, "failed to report a detached response, " + "retiring the worker when the request handler returns"); +} + + +/* + * The application's request handler has returned, so whatever it was doing + * after its response is over. Runs on every return path -- a normal return, + * and the ones PHP reaches through exit() and a fatal error, which all come + * back through the handler call site. + */ + +static void +nxt_unit_ctx_detached_done(nxt_unit_ctx_t *ctx) +{ + int rc; + nxt_unit_ctx_impl_t *ctx_impl; + + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + + if (nxt_slow_path(ctx_impl->detached_unreported)) { + ctx_impl->detached_unreported = 0; + + /* + * The START edge was never delivered. Quit gracefully, the same + * way as the deferred quit below: the router then settles the + * process and starts a replacement if one is needed. + */ + + nxt_unit_quit(ctx, NXT_QUIT_GRACEFUL); + + return; + } + + if (ctx_impl->detached == NXT_UNIT_DETACHED_NONE) { + return; + } + + rc = nxt_unit_send_detached(ctx, NXT_PORT_DETACHED_FINISH); + if (nxt_fast_path(rc == NXT_UNIT_OK)) { + ctx_impl->detached = NXT_UNIT_DETACHED_NONE; + ctx_impl->detached_retries = 0; + + /* + * A graceful QUIT that arrived during the work was deferred by + * nxt_unit_quit() on this flag, the way one that arrives during a + * request is deferred on active_req. That one is retried when the + * request is released; this is the equivalent. + */ + + if (nxt_slow_path(!nxt_unit_chk_ready(ctx))) { + nxt_unit_quit(ctx, NXT_QUIT_GRACEFUL); + } + + return; + } + + /* + * The FINISH edge was not delivered. Keep ->detached set and let the + * read loop retry; its blocking waits are bounded while a retry is + * pending (see nxt_unit_detached_timeout()). The give-up is not run + * here, inside nxt_unit_process_ready_req(). + * + * Only arm the retry if it is not armed. Otherwise each request handler + * that returns while the FINISH is pending resets the budget, and a + * worker that serves traffic never reaches the give-up. + */ + + if (ctx_impl->detached_retries == 0) { + ctx_impl->detached_retries = 1; + ctx_impl->detached_deadline = 0; + } +} + + +/* + * Retry the FINISH edge. Called from the read loop, outside request + * processing. After 10 failed retries, close the worker. + */ + +static int +nxt_unit_ctx_detached_retry(nxt_unit_ctx_t *ctx) +{ + int res; + nxt_unit_impl_t *lib; + nxt_unit_ctx_impl_t *ctx_impl; + + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + + if (nxt_fast_path(ctx_impl->detached_retries == 0)) { + return NXT_UNIT_OK; + } + + /* + * A resend is safe only because a failed send delivered nothing. The + * router counts the edges and they name no context, so a FINISH that + * arrived twice would settle another context's START and hand a worker + * that is still executing back to the idle economy. + * nxt_unit_send_detached() fails on a missing router port, on a queue + * overflow, which enqueues nothing, and on a short send, which a port + * socket (SOCK_DGRAM or SOCK_SEQPACKET) makes all or nothing; none of + * these delivers the edge. Keep it that way, or give the edge a + * context id first. + */ + + res = nxt_unit_send_detached(ctx, NXT_PORT_DETACHED_FINISH); + if (nxt_fast_path(res == NXT_UNIT_OK)) { + ctx_impl->detached = NXT_UNIT_DETACHED_NONE; + ctx_impl->detached_retries = 0; + + if (nxt_slow_path(!nxt_unit_chk_ready(ctx))) { + nxt_unit_quit(ctx, NXT_QUIT_GRACEFUL); + } + + return NXT_UNIT_OK; + } + + if (++ctx_impl->detached_retries > 10) { + lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); + + /* + * Close the main context: the router keeps the detached count on + * the main port (id 0), and only closing that port settles the + * count and the application reference. This assumes one context, + * as in PHP, the only caller of nxt_unit_request_done_detached(). + * A non-main context closing alone would leave this context's start + * counted for the life of the process, and the main context is + * changed here without cross-thread synchronization. This is a + * comment and not nxt_assert(): libunit does not link the thread + * context that nxt_assert() needs. + */ + + nxt_unit_alert(ctx, "failed to report detached finish, closing worker"); + nxt_unit_quit(&lib->main_ctx.ctx, NXT_QUIT_NORMAL); + return NXT_UNIT_ERROR; + } + + return NXT_UNIT_OK; +} + + +/* + * How long a read may block while a FINISH retry is pending: 2 ms after the + * first failed retry, doubling up to 256 ms. The ten retries then span + * about 0.8 s without traffic. + */ + +static int +nxt_unit_detached_timeout(nxt_unit_ctx_impl_t *ctx_impl) +{ + return 1 << nxt_min(ctx_impl->detached_retries - 1, 8); +} + + +/* + * Wait for "fd" to become readable while a FINISH retry is pending. + * Returns NXT_UNIT_AGAIN when the wait expires, so that the caller returns + * to the read loop, which retries. Inside nxt_unit_process_port_msg() the + * wait is zero. The embedder's event loop must not block, so the backoff + * is a deadline there. + */ + +static int +nxt_unit_detached_poll(nxt_unit_ctx_t *ctx, int fd) +{ + int nevents, timeout; + struct pollfd pfd; + nxt_unit_ctx_impl_t *ctx_impl; + + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + + pfd.fd = fd; + pfd.events = POLLIN; + pfd.revents = 0; + + timeout = ctx_impl->detached_nowait ? 0 + : nxt_unit_detached_timeout(ctx_impl); + + nevents = poll(&pfd, 1, timeout); + + if (nevents == 0 || (nevents == -1 && errno == EINTR)) { + return NXT_UNIT_AGAIN; + } + + return NXT_UNIT_OK; +} + + +/* + * Sleep for the backoff before the next FINISH retry. This is for a caller + * that has no descriptor to wait on. nxt_unit_detached_poll() does the + * same for a loop that has one. + */ + +static void +nxt_unit_detached_sleep(nxt_unit_ctx_impl_t *ctx_impl) +{ + struct timespec ts; + + ts.tv_sec = 0; + ts.tv_nsec = nxt_unit_detached_timeout(ctx_impl) * 1000000L; + + (void) nanosleep(&ts, NULL); +} + + +/* + * The time for the FINISH retry deadline, in milliseconds. libunit does + * not link nxt_monotonic_time(), so this makes the same choice of clock, + * in the same order, with one difference. nxt_monotonic_time() prefers + * CLOCK_MONOTONIC_COARSE on Linux and CLOCK_MONOTONIC_FAST on FreeBSD. + * Those clocks lag up to one kernel tick behind the fine clock of the + * embedder's timers. An embedder that waits for the delay from + * nxt_unit_detached_retry_timeout() would then find the retry not due + * yet, and wake up once more for each retry. So the fine clock is used. + * See src/nxt_time.c for the other choices. + */ + +static uint64_t +nxt_unit_detached_now(void) +{ +#if (NXT_HAVE_HG_GETHRTIME) + + return (uint64_t) hg_gethrtime() / 1000000; + +#elif (NXT_SOLARIS || NXT_HPUX) + + return (uint64_t) gethrtime() / 1000000; + +#elif (NXT_HAVE_CLOCK_MONOTONIC) + + struct timespec ts; + + (void) clock_gettime(CLOCK_MONOTONIC, &ts); + + return (uint64_t) ts.tv_sec * 1000 + ts.tv_nsec / 1000000; + +#elif (NXT_MACOSX) + + /* + * mach_absolute_time() returns ticks. The timebase gives their length + * in nanoseconds as a fraction: 1/1 on Intel Macs, 125/3 on Apple + * silicon. The quotient and the remainder are scaled apart, so the + * result is exact and no product can overflow before the nanoseconds + * themselves do, in centuries. + */ + + uint64_t ns, ticks; + mach_timebase_info_data_t tb; + + (void) mach_timebase_info(&tb); + + ticks = mach_absolute_time(); + + ns = ticks / tb.denom * tb.numer + ticks % tb.denom * tb.numer / tb.denom; + + return ns / 1000000; + +#else + + /* + * No monotonic clock. nxt_monotonic_time() emulates one with a + * per-thread state. This helper keeps no state, so a step of the + * clock moves the deadline. nxt_unit_detached_delay() bounds that. + */ + + struct timeval tv; + + (void) gettimeofday(&tv, NULL); + + return (uint64_t) tv.tv_sec * 1000 + tv.tv_usec / 1000; + +#endif +} + + +/* + * The time from "now" to the FINISH retry deadline, in milliseconds, or 0 + * when the retry is due. A deadline that is more than one backoff step + * ahead can only come from a clock that went back. The retry is due then: + * an early retry is harmless, and a late one holds the worker detached. + */ + +static uint64_t +nxt_unit_detached_delay(nxt_unit_ctx_impl_t *ctx_impl, uint64_t now) +{ + uint64_t delay; + + if (now >= ctx_impl->detached_deadline) { + return 0; + } + + delay = ctx_impl->detached_deadline - now; + + if (nxt_slow_path(delay > (uint64_t) nxt_unit_detached_timeout(ctx_impl))) { + return 0; + } + + return delay; +} + + +/* + * Return the code for a call that found no message. A pending FINISH + * retry needs another call, and the embedder's descriptor stays quiet + * until unrelated traffic arrives. So report NXT_UNIT_OK: an integration + * that drives its own event loop calls again on that code, and + * NXT_UNIT_AGAIN stops it. Nothing waits here. The backoff is + * ->detached_deadline, and a call before the deadline only receives. + */ + +static int +nxt_unit_detached_wake(nxt_unit_ctx_impl_t *ctx_impl) +{ + if (nxt_fast_path(ctx_impl->detached_retries == 0 || !ctx_impl->online)) { + return NXT_UNIT_AGAIN; + } + + ctx_impl->detached_idle = 1; + + return NXT_UNIT_OK; +} + + +/* + * Tell an integration with its own event loop when to call + * nxt_unit_process_port_msg() again. A delay is returned only after a + * call that found no message. After a call that processed a message, + * more messages can wait, so the next call must come at once. + */ + +int +nxt_unit_detached_retry_timeout(nxt_unit_ctx_t *ctx) +{ + uint64_t now; + nxt_unit_ctx_impl_t *ctx_impl; + + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + + if (ctx_impl->detached_retries == 0 + || ctx_impl->detached_retries > 10 + || !ctx_impl->online) + { + return -1; + } + + if (!ctx_impl->detached_idle) { + return 0; + } + + now = nxt_unit_detached_now(); + + return (int) nxt_min(nxt_unit_detached_delay(ctx_impl, now), INT_MAX); +} + + +#if (NXT_TESTS) +uint8_t +nxt_unit_test_ctx_detached(nxt_unit_ctx_t *ctx) +{ + nxt_unit_ctx_impl_t *ctx_impl; + + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + + return ctx_impl->detached; +} + + +uint8_t +nxt_unit_test_ctx_detached_retries(nxt_unit_ctx_t *ctx) +{ + nxt_unit_ctx_impl_t *ctx_impl; + + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + + return ctx_impl->detached_retries; +} + + +void +nxt_unit_test_ctx_set_detached(nxt_unit_ctx_t *ctx, uint8_t val) +{ + nxt_unit_ctx_impl_t *ctx_impl; + + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + + ctx_impl->detached = val; +} + + +void +nxt_unit_test_ctx_set_detached_retries(nxt_unit_ctx_t *ctx, uint8_t val) +{ + nxt_unit_ctx_impl_t *ctx_impl; + + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + + ctx_impl->detached_retries = val; +} + + +uint8_t +nxt_unit_test_ctx_detached_unreported(nxt_unit_ctx_t *ctx) +{ + nxt_unit_ctx_impl_t *ctx_impl; + + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + + return ctx_impl->detached_unreported; +} + + +void +nxt_unit_test_ctx_detached_start(nxt_unit_ctx_t *ctx) +{ + nxt_unit_ctx_detached_start(ctx); +} + + +void +nxt_unit_test_ctx_detached_done(nxt_unit_ctx_t *ctx) +{ + nxt_unit_ctx_detached_done(ctx); +} + + +int +nxt_unit_test_ctx_detached_retry(nxt_unit_ctx_t *ctx) +{ + return nxt_unit_ctx_detached_retry(ctx); +} + + +nxt_unit_port_t * +nxt_unit_test_ctx_read_port(nxt_unit_ctx_t *ctx) +{ + nxt_unit_ctx_impl_t *ctx_impl; + + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + + return ctx_impl->read_port; +} + + +uint64_t +nxt_unit_test_detached_now(void) +{ + return nxt_unit_detached_now(); +} + + +uint8_t +nxt_unit_test_ctx_online(nxt_unit_ctx_t *ctx) +{ + nxt_unit_ctx_impl_t *ctx_impl; + + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + + return ctx_impl->online; +} + + +uint8_t +nxt_unit_test_ctx_ready(nxt_unit_ctx_t *ctx) +{ + nxt_unit_ctx_impl_t *ctx_impl; + + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + + return ctx_impl->ready; +} + + +void +nxt_unit_test_ctx_set_ready(nxt_unit_ctx_t *ctx, uint8_t val) +{ + nxt_unit_ctx_impl_t *ctx_impl; + + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + + ctx_impl->ready = val; +} + + +void +nxt_unit_test_ctx_quit_graceful(nxt_unit_ctx_t *ctx) +{ + nxt_unit_quit(ctx, NXT_QUIT_GRACEFUL); +} +#endif + + +void +nxt_unit_request_done(nxt_unit_request_info_t *req, int rc) +{ + uint32_t size; + nxt_port_msg_t msg = {}; + nxt_unit_impl_t *lib; + nxt_unit_request_info_impl_t *req_impl; + + req_impl = nxt_container_of(req, nxt_unit_request_info_impl_t, req); + + nxt_unit_req_debug(req, "done: %d", rc); + + if (nxt_slow_path(rc != NXT_UNIT_OK)) { + goto skip_response_send; + } + + if (nxt_slow_path(req_impl->state < NXT_UNIT_RS_RESPONSE_INIT)) { + + size = nxt_length("Content-Type") + nxt_length("text/plain"); + + rc = nxt_unit_response_init(req, 200, 1, size); + if (nxt_slow_path(rc != NXT_UNIT_OK)) { + goto skip_response_send; + } + + rc = nxt_unit_response_add_field(req, "Content-Type", + nxt_length("Content-Type"), + "text/plain", nxt_length("text/plain")); + if (nxt_slow_path(rc != NXT_UNIT_OK)) { + goto skip_response_send; + } + } + + if (nxt_slow_path(req_impl->state < NXT_UNIT_RS_RESPONSE_SENT)) { + + req_impl->state = NXT_UNIT_RS_RESPONSE_SENT; + + nxt_unit_buf_send_done(req->response_buf); + + return; + } + +skip_response_send: + + lib = nxt_container_of(req->unit, nxt_unit_impl_t, unit); + + msg.stream = req_impl->stream; + msg.pid = lib->pid; + msg.type = (rc == NXT_UNIT_OK) ? _NXT_PORT_MSG_DATA + : _NXT_PORT_MSG_RPC_ERROR; + msg.last = 1; + + (void) nxt_unit_port_send(req->ctx, req->response_port, + &msg, sizeof(msg), NULL); + + nxt_unit_request_info_release(req); +} + + +int +nxt_unit_websocket_send(nxt_unit_request_info_t *req, uint8_t opcode, + uint8_t last, const void *start, size_t size) +{ + const struct iovec iov = { (void *) start, size }; + + return nxt_unit_websocket_sendv(req, opcode, last, &iov, 1); +} + + +int +nxt_unit_websocket_sendv(nxt_unit_request_info_t *req, uint8_t opcode, + uint8_t last, const struct iovec *iov, int iovcnt) +{ + int i, rc; + size_t l, copy; + uint32_t payload_len, buf_size, alloc_size; + const uint8_t *b; + nxt_unit_buf_t *buf; + nxt_unit_mmap_buf_t mmap_buf; + nxt_websocket_header_t *wh; + char local_buf[NXT_UNIT_LOCAL_BUF_SIZE]; + + payload_len = 0; + + for (i = 0; i < iovcnt; i++) { + payload_len += iov[i].iov_len; + } + + buf_size = 10 + payload_len; + alloc_size = nxt_min(buf_size, PORT_MMAP_DATA_SIZE); + + rc = nxt_unit_get_outgoing_buf(req->ctx, req->response_port, + alloc_size, alloc_size, + &mmap_buf, local_buf); + if (nxt_slow_path(rc != NXT_UNIT_OK)) { + return rc; } buf = &mmap_buf.buf; @@ -3929,6 +4797,7 @@ nxt_unit_wait_shm_ack(nxt_unit_ctx_t *ctx) static nxt_unit_mmap_t * nxt_unit_mmap_at(nxt_unit_mmaps_t *mmaps, uint32_t i) { + size_t bytes; uint32_t cap, n; nxt_unit_mmap_t *e; @@ -3936,25 +4805,51 @@ nxt_unit_mmap_at(nxt_unit_mmaps_t *mmaps, uint32_t i) return mmaps->elts + i; } + /* + * The same guards as nxt_port_mmap_at() on the router side. "i" is a + * segment id taken from the peer (an mmap record or a segment header), + * or the next id of an outgoing segment. Past NXT_PORT_MMAPS_MAX it is + * refused before any growth: the array is grown to hold slot i, so an + * id like 100000000 would otherwise cost a huge allocation and its + * initialisation from one message. The limit also keeps i + 1 in + * uint32_t; for i == UINT32_MAX "i + 1 > cap" wraps to 0, skips the + * growth, and the element pointer lands 4G elements past the array. + */ + if (nxt_slow_path(i >= NXT_PORT_MMAPS_MAX)) { + return NULL; + } + cap = mmaps->cap; if (cap == 0) { cap = i + 1; } - while (i + 1 > cap) { + while (cap <= i) { if (cap < 16) { cap = cap * 2; } else { + /* The 1.5x step would wrap below the target and spin. */ + if (nxt_slow_path(cap > UINT32_MAX - cap / 2)) { + return NULL; + } + cap = cap + cap / 2; } } if (cap != mmaps->cap) { - e = realloc(mmaps->elts, cap * sizeof(nxt_unit_mmap_t)); + if (nxt_slow_path(nxt_size_mul(cap, sizeof(nxt_unit_mmap_t), + &bytes) + != 0)) + { + return NULL; + } + + e = realloc(mmaps->elts, bytes); if (nxt_slow_path(e == NULL)) { return NULL; } @@ -4016,7 +4911,6 @@ nxt_unit_new_mmap(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, int n) hdr = mem; memset(hdr->free_map, 0xFFU, sizeof(hdr->free_map)); - memset(hdr->free_tracking_map, 0xFFU, sizeof(hdr->free_tracking_map)); hdr->id = lib->outgoing.size - 1; hdr->src_pid = lib->pid; @@ -4031,7 +4925,6 @@ nxt_unit_new_mmap(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, int n) /* Mark as busy chunk followed the last available chunk. */ nxt_port_mmap_set_chunk_busy(hdr->free_map, PORT_MMAP_CHUNK_COUNT); - nxt_port_mmap_set_chunk_busy(hdr->free_tracking_map, PORT_MMAP_CHUNK_COUNT); pthread_mutex_unlock(&lib->outgoing.mutex); @@ -4238,12 +5131,11 @@ nxt_unit_incoming_mmap(nxt_unit_ctx_t *ctx, pid_t pid, int fd) { int rc; void *mem; + uint32_t id; nxt_queue_t awaiting_rbuf; struct stat mmap_stat; nxt_unit_mmap_t *mm; nxt_unit_impl_t *lib; - nxt_unit_ctx_impl_t *ctx_impl; - nxt_unit_read_buf_t *rbuf; nxt_port_mmap_header_t *hdr; lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); @@ -4257,7 +5149,20 @@ nxt_unit_incoming_mmap(nxt_unit_ctx_t *ctx, pid_t pid, int fd) return NXT_UNIT_ERROR; } - mem = mmap(NULL, mmap_stat.st_size, PROT_READ | PROT_WRITE, + /* + * Every chunk offset is computed against PORT_MMAP_SIZE, and the + * munmap() calls use it too: a shorter object faults on access, a + * longer one leaks the excess mapping. The router side requires the + * same (nxt_port_incoming_port_mmap()). + */ + if (nxt_slow_path(mmap_stat.st_size != (off_t) PORT_MMAP_SIZE)) { + nxt_unit_alert(ctx, "incoming_mmap: unexpected segment size: %d != %d", + (int) mmap_stat.st_size, (int) PORT_MMAP_SIZE); + + return NXT_UNIT_ERROR; + } + + mem = mmap(NULL, PORT_MMAP_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); if (nxt_slow_path(mem == MAP_FAILED)) { nxt_unit_alert(ctx, "incoming_mmap: mmap() failed: %s (%d)", @@ -4279,11 +5184,19 @@ nxt_unit_incoming_mmap(nxt_unit_ctx_t *ctx, pid_t pid, int fd) return NXT_UNIT_ERROR; } + /* + * The segment id lives in memory the sender keeps mapped writable: read + * it once and use only the copy. It indexes lib->incoming, which + * nxt_unit_mmap_at() grows to fit and which refuses an id past + * NXT_PORT_MMAPS_MAX. + */ + id = hdr->id; + nxt_queue_init(&awaiting_rbuf); pthread_mutex_lock(&lib->incoming.mutex); - mm = nxt_unit_mmap_at(&lib->incoming, hdr->id); + mm = nxt_unit_mmap_at(&lib->incoming, id); if (nxt_slow_path(mm == NULL)) { nxt_unit_alert(ctx, "incoming_mmap: failed to add to incoming array"); @@ -4291,6 +5204,18 @@ nxt_unit_incoming_mmap(nxt_unit_ctx_t *ctx, pid_t pid, int fd) rc = NXT_UNIT_ERROR; + } else if (nxt_slow_path(mm->hdr != NULL)) { + /* + * A duplicate id: buffers may point into the segment already + * there, so it stays and the new mapping goes. + */ + nxt_unit_warn(ctx, "incoming_mmap: duplicate segment id %"PRIu32, + id); + + munmap(mem, PORT_MMAP_SIZE); + + rc = NXT_UNIT_OK; + } else { mm->hdr = hdr; @@ -4304,7 +5229,24 @@ nxt_unit_incoming_mmap(nxt_unit_ctx_t *ctx, pid_t pid, int fd) pthread_mutex_unlock(&lib->incoming.mutex); - nxt_queue_each(rbuf, &awaiting_rbuf, nxt_unit_read_buf_t, link) { + nxt_unit_unpark_rbufs(ctx, &awaiting_rbuf); + + return rc; +} + + +/* + * Hands the read buffers parked on a segment back to their contexts as + * pending, so each context reads its message again. + */ + +static void +nxt_unit_unpark_rbufs(nxt_unit_ctx_t *ctx, nxt_queue_t *awaiting_rbuf) +{ + nxt_unit_ctx_impl_t *ctx_impl; + nxt_unit_read_buf_t *rbuf; + + nxt_queue_each(rbuf, awaiting_rbuf, nxt_unit_read_buf_t, link) { ctx_impl = rbuf->ctx_impl; @@ -4319,8 +5261,6 @@ nxt_unit_incoming_mmap(nxt_unit_ctx_t *ctx, pid_t pid, int fd) nxt_unit_awake_ctx(ctx, ctx_impl); } nxt_queue_loop; - - return rc; } @@ -4409,6 +5349,7 @@ nxt_unit_check_rbuf_mmap(nxt_unit_ctx_t *ctx, nxt_unit_mmaps_t *mmaps, nxt_unit_read_buf_t *rbuf) { int res, need_rbuf; + nxt_queue_t awaiting_rbuf; nxt_unit_mmap_t *mm; nxt_unit_ctx_impl_t *ctx_impl; @@ -4442,7 +5383,40 @@ nxt_unit_check_rbuf_mmap(nxt_unit_ctx_t *ctx, nxt_unit_mmaps_t *mmaps, if (need_rbuf) { res = nxt_unit_get_mmap(ctx, pid, id); if (nxt_slow_path(res == NXT_UNIT_ERROR)) { - return NXT_UNIT_ERROR; + /* + * The caller releases rbuf on ERROR, so take it back off the + * wait queue first -- unless the segment arrived meanwhile and + * nxt_unit_incoming_mmap() already took it, in which case it + * is pending and still ours to wait on. + * + * Buffers other contexts parked behind this one did not ask + * for the segment themselves and nobody will ask for it now, + * so they go back to their contexts: the next read asks again. + */ + nxt_queue_init(&awaiting_rbuf); + + pthread_mutex_lock(&mmaps->mutex); + + if (mmaps->elts[id].hdr == NULL) { + nxt_queue_remove(&rbuf->link); + res = NXT_UNIT_ERROR; + + nxt_queue_add(&awaiting_rbuf, &mmaps->elts[id].awaiting_rbuf); + nxt_queue_init(&mmaps->elts[id].awaiting_rbuf); + + } else { + res = NXT_UNIT_AGAIN; + } + + pthread_mutex_unlock(&mmaps->mutex); + + if (res == NXT_UNIT_ERROR) { + nxt_atomic_fetch_add(&ctx_impl->wait_items, -1); + + nxt_unit_unpark_rbufs(ctx, &awaiting_rbuf); + } + + return res; } } @@ -4450,6 +5424,17 @@ nxt_unit_check_rbuf_mmap(nxt_unit_ctx_t *ctx, nxt_unit_mmaps_t *mmaps, } +/* + * The payload of an mmap message is an array of nxt_port_mmap_msg_t written + * by the router. It is walked with nxt_span_copy(): a payload that is not a + * whole number of records -- a partial tail -- is refused as a whole before + * anything is allocated, instead of its last "record" being read past the + * end of the message. Each record is copied out before use, and every + * field is bounds-checked before it reaches an index or pointer arithmetic: + * mmap_id by nxt_unit_mmap_at(), which refuses an id past NXT_PORT_MMAPS_MAX + * (see there), chunk_id and size by nxt_port_mmap_chunk_range_valid(). + */ + static int nxt_unit_mmap_read(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg, nxt_unit_read_buf_t *rbuf) @@ -4458,10 +5443,12 @@ nxt_unit_mmap_read(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg, void *start; size_t nchunks; uint32_t size; + nxt_bool_t first; + nxt_span_t span; nxt_unit_impl_t *lib; nxt_unit_mmaps_t *mmaps; nxt_unit_mmap_buf_t *b, **incoming_tail; - nxt_port_mmap_msg_t *mmap_msg, *end; + nxt_port_mmap_msg_t mmap_msg; nxt_port_mmap_header_t *hdr; if (nxt_slow_path(recv_msg->size < sizeof(nxt_port_mmap_msg_t))) { @@ -4471,13 +5458,21 @@ nxt_unit_mmap_read(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg, return NXT_UNIT_ERROR; } - mmap_msg = recv_msg->start; - end = nxt_pointer_to(recv_msg->start, recv_msg->size); + if (nxt_slow_path(recv_msg->size % sizeof(nxt_port_mmap_msg_t) != 0)) { + nxt_unit_alert(ctx, "#%"PRIu32": mmap_read: message size %d is not " + "a whole number of mmap records", + recv_msg->stream, (int) recv_msg->size); + + return NXT_UNIT_ERROR; + } incoming_tail = &recv_msg->incoming_buf; - /* Allocating buffer structures. */ - for (; mmap_msg < end; mmap_msg++) { + /* Allocating buffer structures, one per whole record. */ + for (size = 0; + size < recv_msg->size; + size += sizeof(nxt_port_mmap_msg_t)) + { b = nxt_unit_mmap_buf_get(ctx); if (nxt_slow_path(b == NULL)) { nxt_unit_warn(ctx, "#%"PRIu32": mmap_read: failed to allocate buf", @@ -4495,7 +5490,10 @@ nxt_unit_mmap_read(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg, } b = recv_msg->incoming_buf; - mmap_msg = recv_msg->start; + first = 1; + + nxt_span_init(&span, recv_msg->start, + nxt_pointer_to(recv_msg->start, recv_msg->size)); lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); @@ -4503,9 +5501,10 @@ nxt_unit_mmap_read(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg, pthread_mutex_lock(&mmaps->mutex); - for (; mmap_msg < end; mmap_msg++) { + while (nxt_span_copy(&span, &mmap_msg, sizeof(nxt_port_mmap_msg_t)) == 0) { + res = nxt_unit_check_rbuf_mmap(ctx, mmaps, - recv_msg->pid, mmap_msg->mmap_id, + recv_msg->pid, mmap_msg.mmap_id, &hdr, rbuf); if (nxt_slow_path(res != NXT_UNIT_OK)) { @@ -4521,39 +5520,26 @@ nxt_unit_mmap_read(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg, * would point outside the mapped data area before they reach the * pointer arithmetic below. */ - if (nxt_slow_path(!nxt_port_mmap_chunk_range_valid(mmap_msg->chunk_id, - mmap_msg->size, + if (nxt_slow_path(!nxt_port_mmap_chunk_range_valid(mmap_msg.chunk_id, + mmap_msg.size, &nchunks))) { nxt_unit_alert(ctx, "#%"PRIu32": mmap_read: invalid mmap message: " "chunk_id %"PRIu32", size %"PRIu32 " (chunks %zu, max %d)", - recv_msg->stream, mmap_msg->chunk_id, - mmap_msg->size, nchunks, PORT_MMAP_CHUNK_COUNT); - - pthread_mutex_unlock(&mmaps->mutex); + recv_msg->stream, mmap_msg.chunk_id, + mmap_msg.size, nchunks, PORT_MMAP_CHUNK_COUNT); - /* - * Entries before the rejected one are already populated, and - * this message is dropped rather than retried, so their chunks - * have to be marked free as well: nxt_unit_mmap_buf_release() - * alone would recycle the wrappers and leave the chunks busy in - * the peer's segment for good. Entries not reached yet have a - * NULL hdr and are skipped by nxt_unit_free_outgoing_buf(). - */ - while (recv_msg->incoming_buf != NULL) { - nxt_unit_mmap_buf_free(recv_msg->incoming_buf); - } - - return NXT_UNIT_ERROR; + goto invalid; } - start = nxt_port_mmap_chunk_start(hdr, mmap_msg->chunk_id); - size = mmap_msg->size; + start = nxt_port_mmap_chunk_start(hdr, mmap_msg.chunk_id); + size = mmap_msg.size; - if (recv_msg->start == mmap_msg) { + if (first) { recv_msg->start = start; recv_msg->size = size; + first = 0; } b->buf.start = start; @@ -4567,13 +5553,31 @@ nxt_unit_mmap_read(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg, recv_msg->stream, start, (int) size, (int) hdr->src_pid, (int) hdr->dst_pid, - (int) hdr->id, (int) mmap_msg->chunk_id, - (int) mmap_msg->size); + (int) hdr->id, (int) mmap_msg.chunk_id, + (int) mmap_msg.size); } pthread_mutex_unlock(&mmaps->mutex); return NXT_UNIT_OK; + +invalid: + + pthread_mutex_unlock(&mmaps->mutex); + + /* + * Entries before the rejected one are already populated, and this + * message is dropped rather than retried, so their chunks have to be + * marked free as well: nxt_unit_mmap_buf_release() alone would recycle + * the wrappers and leave the chunks busy in the peer's segment for good. + * Entries not reached yet have a NULL hdr and are skipped by + * nxt_unit_free_outgoing_buf(). + */ + while (recv_msg->incoming_buf != NULL) { + nxt_unit_mmap_buf_free(recv_msg->incoming_buf); + } + + return NXT_UNIT_ERROR; } @@ -4620,7 +5624,9 @@ nxt_unit_mmap_release(nxt_unit_ctx_t *ctx, nxt_port_mmap_header_t *hdr, nxt_chunk_id_t c; nxt_unit_impl_t *lib; +#if (NXT_DEBUG) memset(start, 0xA5, size); +#endif p = start; end = p + size; @@ -4820,6 +5826,17 @@ nxt_unit_run(nxt_unit_ctx_t *ctx) nxt_unit_quit(ctx, NXT_QUIT_NORMAL); break; } + + /* + * A read that returned no message because the context went offline + * ends the loop the same way a QUIT message does, so report it the + * same way: callers exit the process with this code. + */ + + if (nxt_slow_path(rc == NXT_UNIT_AGAIN && !ctx_impl->online)) { + rc = NXT_UNIT_OK; + break; + } } nxt_unit_ctx_release(ctx); @@ -4880,7 +5897,7 @@ nxt_unit_run_once_impl(nxt_unit_ctx_t *ctx) static int nxt_unit_read_buf(nxt_unit_ctx_t *ctx, nxt_unit_read_buf_t *rbuf) { - int nevents, res, err; + int nevents, res, err, timeout; nxt_uint_t nfds; nxt_unit_impl_t *lib; nxt_unit_ctx_impl_t *ctx_impl; @@ -4889,6 +5906,33 @@ nxt_unit_read_buf(nxt_unit_ctx_t *ctx, nxt_unit_read_buf_t *rbuf) ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + timeout = -1; + + if (nxt_slow_path(ctx_impl->detached_retries > 0)) { + res = nxt_unit_ctx_detached_retry(ctx); + if (nxt_slow_path(res != NXT_UNIT_OK)) { + return res; + } + + if (nxt_slow_path(!ctx_impl->online)) { + /* + * The retry completed a graceful quit that was deferred on the + * detached flag, so the read port is removed. Report "no + * message": the caller releases the buffer and its loop stops + * on ->online. A receive here would wait for a message the + * router will never send. + */ + + rbuf->size = -1; + + return NXT_UNIT_AGAIN; + } + + if (ctx_impl->detached_retries > 0) { + timeout = nxt_unit_detached_timeout(ctx_impl); + } + } + if (ctx_impl->wait_items > 0 || !nxt_unit_chk_ready(ctx)) { return nxt_unit_ctx_port_recv(ctx, ctx_impl->read_port, rbuf); } @@ -4943,7 +5987,15 @@ nxt_unit_read_buf(nxt_unit_ctx_t *ctx, nxt_unit_read_buf_t *rbuf) fds[1].revents = 0; - nevents = poll(fds, nfds, -1); + nevents = poll(fds, nfds, timeout); + + if (nxt_slow_path(nevents == 0)) { + /* A pending FINISH retry bounded the wait; the caller retries. */ + rbuf->size = -1; + + return NXT_UNIT_AGAIN; + } + if (nxt_slow_path(nevents == -1)) { err = errno; @@ -5080,6 +6132,10 @@ nxt_unit_process_ready_req(nxt_unit_ctx_t *ctx) nxt_queue_each(req_impl, &ready_req, nxt_unit_request_info_impl_t, port_wait_link) { + if (nxt_slow_path(!ctx_impl->online)) { + break; + } + lib = nxt_container_of(ctx_impl->ctx.unit, nxt_unit_impl_t, unit); req = &req_impl->req; @@ -5114,6 +6170,8 @@ nxt_unit_process_ready_req(nxt_unit_ctx_t *ctx) lib->callbacks.request_handler(&req_impl->req); + nxt_unit_ctx_detached_done(ctx); + } nxt_queue_loop; } @@ -5132,6 +6190,23 @@ nxt_unit_run_ctx(nxt_unit_ctx_t *ctx) rc = NXT_UNIT_OK; while (nxt_fast_path(ctx_impl->online)) { + if (nxt_slow_path(ctx_impl->detached_retries > 0)) { + rc = nxt_unit_ctx_detached_retry(ctx); + if (nxt_slow_path(rc != NXT_UNIT_OK)) { + break; + } + + /* + * The retry may have completed a deferred graceful quit, which + * removes the read port. Leave with the retry's NXT_UNIT_OK, + * before a buffer is taken; "rc" is that value. + */ + + if (nxt_slow_path(!ctx_impl->online)) { + break; + } + } + rbuf = nxt_unit_read_buf_get(ctx); if (nxt_slow_path(rbuf == NULL)) { rc = NXT_UNIT_ERROR; @@ -5142,6 +6217,11 @@ nxt_unit_run_ctx(nxt_unit_ctx_t *ctx) rc = nxt_unit_ctx_port_recv(ctx, ctx_impl->read_port, rbuf); if (rc == NXT_UNIT_AGAIN) { + if (nxt_slow_path(ctx_impl->detached_retries > 0)) { + nxt_unit_read_buf_release(ctx, rbuf); + continue; + } + goto retry; } @@ -5220,20 +6300,92 @@ nxt_unit_is_quit(nxt_unit_read_buf_t *rbuf) } +/* + * A QUIT as nxt_runtime_port_send_quit() writes it. It is the header and + * one byte for the quit mode. The byte is absent when the runtime had no + * memory for it. + */ + +nxt_inline int +nxt_unit_is_socket_quit(nxt_unit_read_buf_t *rbuf) +{ + nxt_port_msg_t *port_msg; + + if (rbuf->size == (ssize_t) sizeof(nxt_port_msg_t) + || rbuf->size == (ssize_t) sizeof(nxt_port_msg_t) + 1) + { + port_msg = (nxt_port_msg_t *) rbuf->buf; + + return port_msg->type == _NXT_PORT_MSG_QUIT && !port_msg->mmap; + } + + return 0; +} + + int nxt_unit_run_shared(nxt_unit_ctx_t *ctx) { int rc; nxt_unit_impl_t *lib; nxt_unit_read_buf_t *rbuf; + nxt_unit_ctx_impl_t *ctx_impl; nxt_unit_ctx_use(ctx); lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); rc = NXT_UNIT_OK; - while (nxt_fast_path(nxt_unit_chk_ready(ctx))) { + while (nxt_fast_path(ctx_impl->online)) { + + /* + * A request from the shared port also runs the application's + * handler. So this loop can end detached work, and the finish + * report can fail. Retry it here, as nxt_unit_run_ctx() does. The + * wait below is bounded while a retry is pending. + */ + + if (nxt_slow_path(ctx_impl->detached_retries > 0)) { + rc = nxt_unit_ctx_detached_retry(ctx); + if (nxt_slow_path(rc != NXT_UNIT_OK)) { + break; + } + + /* + * The retry can complete a deferred graceful quit. That + * removes the read port. Leave with the retry's NXT_UNIT_OK. + */ + + if (nxt_slow_path(!ctx_impl->online)) { + break; + } + } + + if (nxt_slow_path(!nxt_unit_chk_ready(ctx))) { + + /* + * No request can arrive now, so the loop ends here. The loop + * runs on ->online and not on nxt_unit_chk_ready() for one + * reason: a pending retry must not be left behind here. Both + * causes of a false nxt_unit_chk_ready() can occur with a retry + * pending. The request that ended the detached work can be the + * one that reached "request_limit". A graceful quit deferred on + * the detached state has already cleared ->ready. There is no + * descriptor to wait on, so sleep for the backoff and retry + * above. This goes on up to the give-up that closes the worker. + */ + + if (nxt_fast_path(ctx_impl->detached_retries == 0)) { + break; + } + + nxt_unit_detached_sleep(ctx_impl); + + continue; + } + rbuf = nxt_unit_read_buf_get(ctx); if (nxt_slow_path(rbuf == NULL)) { rc = NXT_UNIT_ERROR; @@ -5244,6 +6396,11 @@ nxt_unit_run_shared(nxt_unit_ctx_t *ctx) rc = nxt_unit_shared_port_recv(ctx, lib->shared_port, rbuf); if (rc == NXT_UNIT_AGAIN) { + if (nxt_slow_path(ctx_impl->detached_retries > 0)) { + nxt_unit_read_buf_release(ctx, rbuf); + continue; + } + goto retry; } @@ -5308,10 +6465,31 @@ nxt_unit_process_port_msg(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port) { int rc; + nxt_unit_ctx_impl_t *ctx_impl; + + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + nxt_unit_ctx_use(ctx); + ctx_impl->detached_nowait = 1; + ctx_impl->detached_idle = 0; + rc = nxt_unit_process_port_msg_impl(ctx, port); + ctx_impl->detached_nowait = 0; + + /* + * A QUIT in this call runs the quit callback. When the callback calls + * nxt_unit_done(), the reference above is the last one, and the release + * below frees the context. The caller cannot know this from ctx, so + * tell it here: NXT_UNIT_AGAIN means "nothing to schedule, stop". An + * offline context has no read port, so a call again has no use. + */ + + if (rc == NXT_UNIT_OK && nxt_slow_path(!ctx_impl->online)) { + rc = NXT_UNIT_AGAIN; + } + nxt_unit_ctx_release(ctx); return rc; @@ -5322,13 +6500,58 @@ static int nxt_unit_process_port_msg_impl(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port) { int rc; + uint64_t now; nxt_unit_impl_t *lib; nxt_unit_read_buf_t *rbuf; + nxt_unit_ctx_impl_t *ctx_impl; lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + + /* + * The embedder runs its own event loop, so this call is the only + * wake-up that libunit gets. The read loops retry from their own + * waits. Run the retry before anything that can return early. If + * not, a worker whose finish report failed stays detached in the + * router until traffic arrives. The backoff is a deadline and not a + * wait, because a wait would block the embedder's loop. A call before + * the deadline only receives. + */ + + if (nxt_slow_path(ctx_impl->detached_retries > 0 && ctx_impl->online)) { + now = nxt_unit_detached_now(); + + if (nxt_unit_detached_delay(ctx_impl, now) > 0) { + goto recv; + } + + rc = nxt_unit_ctx_detached_retry(ctx); + if (nxt_slow_path(rc != NXT_UNIT_OK)) { + return rc; + } + + if (ctx_impl->detached_retries > 0) { + ctx_impl->detached_deadline = now + + nxt_unit_detached_timeout(ctx_impl); + } + + /* + * The retry can complete a graceful quit that was deferred on the + * detached state. That removes the read port. Report "no + * message" here. A receive would wait for a message that the + * router will never send, and NXT_UNIT_AGAIN makes the embedder + * stop calling. + */ + + if (nxt_slow_path(!ctx_impl->online)) { + return NXT_UNIT_AGAIN; + } + } + +recv: if (port == lib->shared_port && !nxt_unit_chk_ready(ctx)) { - return NXT_UNIT_AGAIN; + return nxt_unit_detached_wake(ctx_impl); } rbuf = nxt_unit_read_buf_get(ctx); @@ -5345,6 +6568,11 @@ nxt_unit_process_port_msg_impl(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port) if (rc != NXT_UNIT_OK) { nxt_unit_read_buf_release(ctx, rbuf); + + if (rc == NXT_UNIT_AGAIN) { + return nxt_unit_detached_wake(ctx_impl); + } + return rc; } @@ -6085,7 +7313,8 @@ nxt_unit_quit(nxt_unit_ctx_t *ctx, uint8_t quit_param) quit = nxt_queue_is_empty(&ctx_impl->active_req) && nxt_queue_is_empty(&ctx_impl->pending_rbuf) - && ctx_impl->wait_items == 0; + && ctx_impl->wait_items == 0 + && ctx_impl->detached == NXT_UNIT_DETACHED_NONE; pthread_mutex_unlock(&ctx_impl->mutex); @@ -6199,13 +7428,31 @@ nxt_unit_port_send(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, nxt_int_t rc; nxt_port_msg_t msg; nxt_unit_impl_t *lib; + const nxt_port_msg_t *pm; nxt_unit_port_impl_t *port_impl; lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); port_impl = nxt_container_of(port, nxt_unit_port_impl_t, port); + + /* + * A message that goes to the socket puts only a READ_SOCKET marker into + * the queue. The marker does not tell who sent it. For each marker, + * the reader takes the next datagram, and that datagram can be from + * another process (nxt_port_queue_read_handler()). Thus a datagram can + * be read before a queue message that its sender sent earlier. A + * datagram is never read after a queue message that its sender sent + * later. So a message of a request goes into the queue only if it is + * the last message of the request. A message of no request (stream 0) + * can also go into the queue, even if "last" is clear. QUIT and + * RPC_READY to the port of a context are such messages. + */ + pm = buf; + if (port_impl->queue != NULL && (oob == NULL || oob->size == 0) - && buf_size <= NXT_PORT_QUEUE_MSG_SIZE) + && buf_size <= NXT_PORT_QUEUE_MSG_SIZE + && buf_size >= sizeof(nxt_port_msg_t) + && (pm->stream == 0 || pm->last)) { rc = nxt_port_queue_send(port_impl->queue, buf, buf_size, ¬ify); if (nxt_slow_path(rc != NXT_OK)) { @@ -6225,7 +7472,7 @@ nxt_unit_port_send(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, msg.type = _NXT_PORT_MSG_READ_QUEUE; if (lib->callbacks.port_send == NULL) { - ret = nxt_unit_sendmsg(ctx, port->out_fd, &msg, + ret = nxt_unit_sendmsg(ctx, port, port->out_fd, &msg, sizeof(nxt_port_msg_t), NULL); nxt_unit_debug(ctx, "port{%d,%d} send %d read_queue", @@ -6271,7 +7518,7 @@ nxt_unit_port_send(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, (int) ret); } else { - ret = nxt_unit_sendmsg(ctx, port->out_fd, buf, buf_size, oob); + ret = nxt_unit_sendmsg(ctx, port, port->out_fd, buf, buf_size, oob); nxt_unit_debug(ctx, "port{%d,%d} sendmsg %d", (int) port->id.pid, (int) port->id.id, @@ -6283,12 +7530,13 @@ nxt_unit_port_send(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, static ssize_t -nxt_unit_sendmsg(nxt_unit_ctx_t *ctx, int fd, +nxt_unit_sendmsg(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, int fd, const void *buf, size_t buf_size, const nxt_send_oob_t *oob) { int err; ssize_t n; struct iovec iov[1]; + nxt_unit_impl_t *lib; nxt_unit_ctx_impl_t *ctx_impl; iov[0].iov_base = (void *) buf; @@ -6325,9 +7573,36 @@ nxt_unit_sendmsg(nxt_unit_ctx_t *ctx, int fd, * too and cannot be used to distinguish steady state from * shutdown. The unambiguous flag is ctx_impl->online. */ + lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); - if (ctx_impl->online) { + /* + * A router engine port whose peer has already closed is not an + * application fault even while online: the router closes an + * engine's port pair when a "listen_threads" decrease retires + * that engine (nxt_router_thread_exit_handler()), and it does + * not tell the applications that still hold a copy of the port, + * so a response to a request that engine had handed out fails: + * with ECONNREFUSED on the SOCK_DGRAM pairs used on Linux + * (src/nxt_socketpair.c), with EPIPE or ECONNRESET on a stream + * pair. The request was abandoned on the router side already; + * say so at warn level. + * + * That reasoning covers only the per-engine ports learned through + * NEW_PORT/GET_PORT. The main router port, the shared port and + * the readiness descriptor have no such lifecycle: a broken one + * of those is a real problem and keeps the alert, because for + * some of its messages this log line is the only visible signal + * (nxt_unit_mmap_release() ignores nxt_unit_send_shm_ack()'s + * return value). + */ + if (ctx_impl->online + && !(port != NULL + && port != lib->router_port + && port != lib->shared_port + && (err == ECONNREFUSED || err == EPIPE + || err == ECONNRESET))) + { nxt_unit_alert(ctx, "sendmsg(%d, %d) failed: %s (%d)", fd, (int) buf_size, strerror(err), err); @@ -6350,8 +7625,10 @@ nxt_unit_ctx_port_recv(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, nxt_unit_read_buf_t *rbuf) { int res, read; + nxt_unit_ctx_impl_t *ctx_impl; nxt_unit_port_impl_t *port_impl; + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); port_impl = nxt_container_of(port, nxt_unit_port_impl_t, port); read = 0; @@ -6373,7 +7650,7 @@ nxt_unit_ctx_port_recv(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, * a leftover oob.size would offer them again. */ port_impl->socket_rbuf->size = 0; - port_impl->socket_rbuf->oob.size = 0; + nxt_socket_msg_oob_reset(&port_impl->socket_rbuf->oob); nxt_unit_debug(ctx, "port{%d,%d} use suspended message %d", (int) port->id.pid, (int) port->id.id, @@ -6408,6 +7685,13 @@ nxt_unit_ctx_port_recv(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, return NXT_UNIT_AGAIN; } + if (nxt_slow_path(ctx_impl->detached_retries > 0 && port->in_fd != -1)) { + res = nxt_unit_detached_poll(ctx, port->in_fd); + if (res != NXT_UNIT_OK) { + return res; + } + } + res = nxt_unit_port_recv(ctx, port, rbuf); if (nxt_slow_path(res == NXT_UNIT_ERROR)) { return NXT_UNIT_ERROR; @@ -6436,6 +7720,22 @@ nxt_unit_ctx_port_recv(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, return NXT_UNIT_OK; } + /* + * A QUIT with no READ_SOCKET mark ahead of it was not sent through + * the queue, and no mark comes for it later. The prototype writes a + * QUIT to the bare socket of a worker whose queue it has not mapped + * yet. That worker has not sent PROCESS_READY. A suspended QUIT + * keeps the worker alive until SIGTERM. A QUIT needs no order with + * the queued messages, so act on it now. + */ + if (nxt_unit_is_socket_quit(rbuf)) { + nxt_unit_debug(ctx, "port{%d,%d} recv %d quit", + (int) port->id.pid, (int) port->id.id, + (int) rbuf->size); + + return NXT_UNIT_OK; + } + nxt_unit_debug(ctx, "port{%d,%d} suspend message %d", (int) port->id.pid, (int) port->id.id, (int) rbuf->size); @@ -6458,7 +7758,7 @@ nxt_unit_ctx_port_recv(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, nxt_unit_rbuf_cpy(port_impl->socket_rbuf, rbuf); - rbuf->oob.size = 0; + nxt_socket_msg_oob_reset(&rbuf->oob); goto retry; } @@ -6470,6 +7770,7 @@ nxt_unit_rbuf_cpy(nxt_unit_read_buf_t *dst, nxt_unit_read_buf_t *src) memcpy(dst->buf, src->buf, src->size); dst->size = src->size; dst->oob.size = src->oob.size; + dst->oob.truncated = src->oob.truncated; memcpy(dst->oob.buf, src->oob.buf, src->oob.size); } @@ -6479,8 +7780,10 @@ nxt_unit_shared_port_recv(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, nxt_unit_read_buf_t *rbuf) { int res; + nxt_unit_ctx_impl_t *ctx_impl; nxt_unit_port_impl_t *port_impl; + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); port_impl = nxt_container_of(port, nxt_unit_port_impl_t, port); retry: @@ -6492,6 +7795,23 @@ nxt_unit_shared_port_recv(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, } if (res == NXT_UNIT_AGAIN) { + + /* + * Bound the wait while a detached finish retry is pending, as + * nxt_unit_ctx_port_recv() does. Then the caller returns to its + * loop and retries. Without this, the caller blocks here until a + * request comes. + */ + + if (nxt_slow_path(ctx_impl->detached_retries > 0 + && port->in_fd != -1)) + { + res = nxt_unit_detached_poll(ctx, port->in_fd); + if (res != NXT_UNIT_OK) { + return res; + } + } + res = nxt_unit_port_recv(ctx, port, rbuf); if (nxt_slow_path(res == NXT_UNIT_ERROR)) { return NXT_UNIT_ERROR; @@ -6562,6 +7882,17 @@ nxt_unit_port_recv(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, } rbuf->oob.size = oob_size; + /* + * The callback reports a length, not msg_flags, so a truncation + * cannot be carried across as such: a wrapper that sees MSG_CTRUNC + * reports a read error instead, and the rbuf->size < 0 arm above + * returns before any control data is parsed (go/port.go does exactly + * this). Nothing can therefore be inferred about truncation here; + * clear the flag so that a previous message's MSG_CTRUNC, left in + * this recycled buffer, cannot be read as this one's. + */ + rbuf->oob.truncated = 0; + return NXT_UNIT_OK; } @@ -6582,7 +7913,7 @@ nxt_unit_port_recv(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, * clear it here to keep "size 0 implies no control data" holding on * every exit of this function rather than by inspection of callers. */ - rbuf->oob.size = 0; + nxt_socket_msg_oob_reset(&rbuf->oob); if (err == EINTR) { goto retry; diff --git a/src/nxt_unit.h b/src/nxt_unit.h index a50046a52..6588c65a9 100644 --- a/src/nxt_unit.h +++ b/src/nxt_unit.h @@ -228,8 +228,51 @@ nxt_unit_request_info_t *nxt_unit_dequeue_request(nxt_unit_ctx_t *ctx); */ int nxt_unit_run_once(nxt_unit_ctx_t *ctx); +/* + * Process one message from "port", for an integration that drives its own + * event loop. The call does not wait for a message. It returns: + * + * - NXT_UNIT_OK: call again. The call processed a message, or it found + * no message and a FINISH retry is pending. Get the delay before the + * next call from nxt_unit_detached_retry_timeout(). + * - NXT_UNIT_AGAIN: do not call again until the port becomes readable. + * The call found no message and no retry is pending, or the context is + * offline after the call: a QUIT was processed, or a retry completed a + * deferred graceful quit, or the retries gave up. + * - NXT_UNIT_ERROR: the worker is closing. Do not call again. + * + * A QUIT runs the quit callback inside the call. When that callback calls + * nxt_unit_done(), the context can be freed when the call returns. So + * after NXT_UNIT_AGAIN or NXT_UNIT_ERROR the caller must not use ctx or + * port again. Only after NXT_UNIT_OK may it call + * nxt_unit_detached_retry_timeout(). + */ int nxt_unit_process_port_msg(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port); +/* + * Get the delay before the next nxt_unit_process_port_msg() call, in + * milliseconds, for an integration that drives its own event loop. Use it + * only after nxt_unit_process_port_msg() returns NXT_UNIT_OK. After + * NXT_UNIT_AGAIN or NXT_UNIT_ERROR the context can be freed already, see + * nxt_unit_process_port_msg(). NXT_UNIT_OK means "call again". Usually + * the call processed a message and more can wait. But the call also + * returns NXT_UNIT_OK when it found no message and a FINISH retry is + * pending (see nxt_unit_request_done_detached()). Then the next retry is + * not due until a deadline, and a call before it only receives. + * + * The function returns: + * + * - -1: no retry is pending, or the context is offline. Call again at + * once, as for any NXT_UNIT_OK. + * - 0: call again at once. The retry is due now, or the last + * nxt_unit_process_port_msg() call on the context processed a message. + * - more than 0: the last call found no message. Call again after this + * many milliseconds, or earlier when the port becomes readable. + * + * The value is not more than INT_MAX. + */ +int nxt_unit_detached_retry_timeout(nxt_unit_ctx_t *ctx); + /* Destroy application library object. */ void nxt_unit_done(nxt_unit_ctx_t *); @@ -245,10 +288,6 @@ void nxt_unit_port_id_init(nxt_unit_port_id_t *port_id, pid_t pid, uint16_t id); /* Calculates hash for given field name. */ uint16_t nxt_unit_field_hash(const char* name, size_t name_length); -/* Split host for server name and port. */ -void nxt_unit_split_host(char *host_start, uint32_t host_length, - char **name, uint32_t *name_length, char **port, uint32_t *port_length); - /* Group duplicate fields for easy enumeration. */ void nxt_unit_request_group_dup_fields(nxt_unit_request_info_t *req); @@ -318,6 +357,39 @@ ssize_t nxt_unit_request_readline_size(nxt_unit_request_info_t *req, void nxt_unit_request_done(nxt_unit_request_info_t *req, int rc); +/* + * Finish a request the way nxt_unit_request_done() does, and tell the router + * that this worker is still running: the application has answered the client + * but has not returned, so the process is busy even though it has no request. + * + * Without this the router counts such a worker idle the moment the response + * goes out. It then hands the worker's slot back to "processes": {"max"}, + * and its idle timer can reap a process that is still executing. + * + * libunit reports the work finished by itself, once the request handler + * returns; the application does not have to pair this call with anything. + * That covers the requests libunit hands to the handler, not one taken with + * nxt_unit_dequeue_request(). libunit tracks the state per context and the + * router counts the reports per worker, so several contexts of one worker + * may run detached work at once: the worker stays busy until the last of + * them returns. PHP's fastcgi_finish_request() is the caller this exists + * for, and PHP runs one context. + * + * The report to the router can fail. Then libunit retries it from its + * read loops and from nxt_unit_process_port_msg(). The router holds the + * worker busy until a retry succeeds. In that time nothing wakes an + * integration that drives its own event loop. So while a retry is + * pending, nxt_unit_process_port_msg() returns NXT_UNIT_OK, and not + * NXT_UNIT_AGAIN, to ask for the next call. The call never waits for the + * retry backoff, because a wait would block the event loop. It keeps a + * deadline instead, and a call before the deadline only receives. An + * integration that calls again on NXT_UNIT_OK must not call at once each + * time, or its loop spins until the deadline. It gets the delay for the + * next call from nxt_unit_detached_retry_timeout(). The retries give up + * after about 0.8 s and close the worker. + */ +void nxt_unit_request_done_detached(nxt_unit_request_info_t *req, int rc); + int nxt_unit_websocket_send(nxt_unit_request_info_t *req, uint8_t opcode, uint8_t last, const void *start, size_t size); @@ -397,4 +469,31 @@ void nxt_unit_req_log(nxt_unit_request_info_t *req, int level, nxt_unit_req_log(req, NXT_UNIT_LOG_ALERT, fmt, ##ARGS) +#if (NXT_TESTS) +void nxt_unit_test_send_detached_failures(unsigned int failures); +uint8_t nxt_unit_test_ctx_detached(nxt_unit_ctx_t *ctx); +uint8_t nxt_unit_test_ctx_detached_retries(nxt_unit_ctx_t *ctx); +void nxt_unit_test_ctx_set_detached(nxt_unit_ctx_t *ctx, uint8_t val); +void nxt_unit_test_ctx_set_detached_retries(nxt_unit_ctx_t *ctx, + uint8_t val); +uint8_t nxt_unit_test_ctx_detached_unreported(nxt_unit_ctx_t *ctx); +void nxt_unit_test_ctx_detached_start(nxt_unit_ctx_t *ctx); +void nxt_unit_test_ctx_detached_done(nxt_unit_ctx_t *ctx); +int nxt_unit_test_ctx_detached_retry(nxt_unit_ctx_t *ctx); +nxt_unit_port_t *nxt_unit_test_ctx_read_port(nxt_unit_ctx_t *ctx); +uint64_t nxt_unit_test_detached_now(void); +uint8_t nxt_unit_test_ctx_online(nxt_unit_ctx_t *ctx); +uint8_t nxt_unit_test_ctx_ready(nxt_unit_ctx_t *ctx); +void nxt_unit_test_ctx_set_ready(nxt_unit_ctx_t *ctx, uint8_t val); +void nxt_unit_test_ctx_quit_graceful(nxt_unit_ctx_t *ctx); +int nxt_unit_test_add_queue_port(nxt_unit_ctx_t *ctx, pid_t pid, + uint16_t id, int out_fd, void *queue); +#endif + +#if (NXT_TESTS || NXT_FUZZ_BUILD) +int nxt_unit_test_process_msg(nxt_unit_ctx_t *ctx, const void *msg, + size_t size, int fd); +#endif + + #endif /* _NXT_UNIT_H_INCLUDED_ */ diff --git a/src/nxt_unit_sptr.h b/src/nxt_unit_sptr.h index 314416e4e..071391148 100644 --- a/src/nxt_unit_sptr.h +++ b/src/nxt_unit_sptr.h @@ -35,4 +35,53 @@ nxt_unit_sptr_get(nxt_unit_sptr_t *sptr) } +/* + * Resolves an sptr field of a peer-supplied buffer, returning the pointer + * when the [length]-byte range it addresses is wholly inside the buffer and + * NULL otherwise. Shared by both directions of the trust boundary: libunit + * vets every sptr of an nxt_unit_request_t before the application sees it + * (src/nxt_unit.c), and the router vets an application's nxt_unit_response_t + * (src/nxt_router.c). + * + * sptr->base aliases the address of the sptr itself (the union encodes an + * offset relative to that location), so this also implicitly checks that + * the sptr is inside the buffer. The offset is read once: the buffer may be + * shared memory the peer keeps writing, so it is read through volatile and + * the caller uses the returned pointer rather than resolving the sptr again. + */ +static inline void * +nxt_unit_sptr_in_buf(nxt_unit_sptr_t *sptr, uint32_t length, + void *buf_start, uint32_t buf_size) +{ + size_t sptr_off; + uint32_t offset; + + if ((uint8_t *) sptr < (uint8_t *) buf_start) { + return NULL; + } + + sptr_off = (uint8_t *) sptr - (uint8_t *) buf_start; + + /* + * The sptr itself must fit inside the buffer before its offset is read; + * the first test keeps the subtraction from underflowing. + */ + if (buf_size < sizeof(nxt_unit_sptr_t) + || sptr_off > buf_size - sizeof(nxt_unit_sptr_t)) + { + return NULL; + } + + offset = *(volatile uint32_t *) &sptr->offset; + + if (offset > buf_size - sptr_off + || length > buf_size - sptr_off - offset) + { + return NULL; + } + + return sptr->base + offset; +} + + #endif /* _NXT_UNIT_SPTR_H_INCLUDED_ */ diff --git a/src/nxt_unix.h b/src/nxt_unix.h index 6bc6be5ee..cf2402d36 100644 --- a/src/nxt_unix.h +++ b/src/nxt_unix.h @@ -153,7 +153,6 @@ #include #include #include -#include #include #include #include diff --git a/src/nxt_utf8.c b/src/nxt_utf8.c index d983be841..b4179eaa6 100644 --- a/src/nxt_utf8.c +++ b/src/nxt_utf8.c @@ -6,28 +6,8 @@ #include -/* - * The nxt_unicode_lowcase.h file is the auto-generated file from - * the CaseFolding-6.3.0.txt file provided by Unicode, Inc.: - * - * ./lib/src/nxt_unicode_lowcase.pl CaseFolding-6.3.0.txt - * - * This file should be copied to system specific nxt_unicode_SYSTEM_lowcase.h - * file and utf8_file_name_test should be built with this file. - * Then a correct system specific file should be generated: - * - * ./build/utf8_file_name_test | ./lib/src/nxt_unicode_lowcase.pl - * - * Only common and simple case foldings are supported. Full case foldings - * is not supported. Combined characters are also not supported. - */ -#if (NXT_MACOSX) -#include - -#else -#include -#endif +static uint32_t nxt_utf8_decode2(const u_char **start, const u_char *end); u_char * @@ -91,7 +71,7 @@ nxt_utf8_decode(const u_char **start, const u_char *end) * invalid or overlong UTF-8 sequence. */ -uint32_t +static uint32_t nxt_utf8_decode2(const u_char **start, const u_char *end) { u_char c; @@ -158,7 +138,18 @@ nxt_utf8_decode2(const u_char **start, const u_char *end) } while (n != 0); - if (overlong < u && u < 0x110000) { + /* + * Shortest form, inside the Unicode range, and not a surrogate: + * U+D800-U+DFFF exist only to be paired inside UTF-16 and have no + * UTF-8 encoding at all (Unicode 15.0 Sect. 3.9, D92), so a decoder + * that returns them hands its caller a code point that cannot be + * re-encoded -- and, for anything that then writes the bytes back + * out, output no strict reader will take. + */ + + if (overlong < u && u < 0x110000 + && !(u >= 0xD800 && u <= 0xDFFF)) + { *start = p; return u; } @@ -168,91 +159,128 @@ nxt_utf8_decode2(const u_char **start, const u_char *end) } +ssize_t +nxt_utf8_length(const u_char *p, size_t len) +{ + ssize_t length; + const u_char *end; + + length = 0; + + end = p + len; + + while (p < end) { + if (nxt_slow_path(nxt_utf8_decode(&p, end) == 0xFFFFFFFF)) { + return -1; + } + + length++; + } + + return length; +} + + /* - * nxt_utf8_casecmp() tests only up to the minimum of given lengths, but - * requires lengths of both strings because otherwise nxt_utf8_decode2() - * may fail due to incomplete sequence. + * Copy "src" into the pool, replacing every byte that begins no valid UTF-8 + * sequence with U+FFFD. Returns "src" itself when there is nothing to + * replace, so the ordinary path neither allocates nor copies. + * + * The replacement is per byte rather than per maximal subpart (Unicode 15.0 + * Sect. 3.9): a truncated four-byte sequence therefore yields up to four + * U+FFFD where a maximal-subpart resync yields one. That changes how many + * replacement characters a reader sees, never whether the result is valid, + * and it keeps the worst-case expansion at the three bytes of U+FFFD per + * input byte -- which matters when the input is a request header somebody + * else chose. */ nxt_int_t -nxt_utf8_casecmp(const u_char *start1, const u_char *start2, size_t len1, - size_t len2) +nxt_utf8_sanitize(nxt_mp_t *mp, nxt_str_t *dst, const nxt_str_t *src) { - int32_t n; - uint32_t u1, u2; - const u_char *end1, *end2; + size_t len; + u_char *d, *out; + const u_char *p, *seq, *end, *start; - end1 = start1 + len1; - end2 = start2 + len2; + /* + * "dst" and "src" may be the same nxt_str_t -- the access log sanitizes + * a value in place -- so the input is read through locals captured here + * and "dst" is not written until both passes are done. + */ - while (start1 < end1 && start2 < end2) { + start = src->start; + end = start + src->length; - u1 = nxt_utf8_lowcase(&start1, end1); + p = start; + len = 0; - u2 = nxt_utf8_lowcase(&start2, end2); + while (p < end) { - if (nxt_slow_path((u1 | u2) == 0xFFFFFFFF)) { - return NXT_UTF8_SORT_INVALID; - } + /* + * nxt_utf8_decode() is a call into another translation unit, and a + * logged value is mostly ASCII, so the single-byte case is decided + * here rather than paid for at that price once per character. + */ - n = u1 - u2; + if (nxt_fast_path(*p < 0x80)) { + p++; + len++; - if (n != 0) { - return (nxt_int_t) n; + continue; } - } - - return 0; -} + seq = p; -uint32_t -nxt_utf8_lowcase(const u_char **start, const u_char *end) -{ - uint32_t u; - const uint32_t *block; + if (nxt_utf8_decode(&seq, end) == 0xFFFFFFFF) { + len += nxt_length("\xEF\xBF\xBD"); + p++; - u = (uint32_t) **start; - - if (nxt_fast_path(u < 0x80)) { - (*start)++; + continue; + } - return nxt_unicode_block_000[u]; + len += seq - p; + p = seq; } - u = nxt_utf8_decode2(start, end); + if (nxt_fast_path(len == src->length)) { + *dst = *src; - if (u <= NXT_UNICODE_MAX_LOWCASE) { - block = nxt_unicode_blocks[u / NXT_UNICODE_BLOCK_SIZE]; + return NXT_OK; + } - if (block != NULL) { - return block[u % NXT_UNICODE_BLOCK_SIZE]; - } + out = nxt_mp_nget(mp, len); + if (nxt_slow_path(out == NULL)) { + return NXT_ERROR; } - return u; -} + d = out; + p = start; + + while (p < end) { + if (nxt_fast_path(*p < 0x80)) { + *d++ = *p++; -ssize_t -nxt_utf8_length(const u_char *p, size_t len) -{ - ssize_t length; - const u_char *end; + continue; + } - length = 0; + seq = p; - end = p + len; + if (nxt_utf8_decode(&seq, end) == 0xFFFFFFFF) { + *d++ = 0xEF; *d++ = 0xBF; *d++ = 0xBD; + p++; - while (p < end) { - if (nxt_slow_path(nxt_utf8_decode(&p, end) == 0xFFFFFFFF)) { - return -1; + continue; } - length++; + d = nxt_cpymem(d, p, seq - p); + p = seq; } - return length; + dst->start = out; + dst->length = len; + + return NXT_OK; } diff --git a/src/nxt_utf8.h b/src/nxt_utf8.h index 13f42e16a..2b581a70e 100644 --- a/src/nxt_utf8.h +++ b/src/nxt_utf8.h @@ -1,4 +1,3 @@ - /* * Copyright (C) Igor Sysoev * Copyright (C) NGINX, Inc. @@ -8,53 +7,12 @@ #define _NXT_UTF8_H_INCLUDED_ -/* - * Since the maximum valid Unicode character is 0x0010FFFF, the maximum - * difference between Unicode characters is lesser 0x0010FFFF and - * 0x0EEE0EEE can be used as value to indicate UTF-8 encoding error. - */ -#define NXT_UTF8_SORT_INVALID 0x0EEE0EEE - - NXT_EXPORT u_char *nxt_utf8_encode(u_char *p, uint32_t u); +NXT_EXPORT nxt_int_t nxt_utf8_sanitize(nxt_mp_t *mp, nxt_str_t *dst, + const nxt_str_t *src); NXT_EXPORT uint32_t nxt_utf8_decode(const u_char **start, const u_char *end); -NXT_EXPORT uint32_t nxt_utf8_decode2(const u_char **start, const u_char *end); -NXT_EXPORT nxt_int_t nxt_utf8_casecmp(const u_char *start1, - const u_char *start2, size_t len1, size_t len2); -NXT_EXPORT uint32_t nxt_utf8_lowcase(const u_char **start, const u_char *end); NXT_EXPORT ssize_t nxt_utf8_length(const u_char *p, size_t len); NXT_EXPORT nxt_bool_t nxt_utf8_is_valid(const u_char *p, size_t len); -/* nxt_utf8_next() expects a valid UTF-8 string. */ - -nxt_inline const u_char * -nxt_utf8_next(const u_char *p, const u_char *end) -{ - u_char c; - - c = *p++; - - if ((c & 0x80) != 0) { - - do { - /* - * The first UTF-8 byte is either 0xxxxxxx or 11xxxxxx. - * The next UTF-8 bytes are 10xxxxxx. - */ - c = *p; - - if ((c & 0xC0) != 0x80) { - return p; - } - - p++; - - } while (p < end); - } - - return p; -} - - #endif /* _NXT_UTF8_H_INCLUDED_ */ diff --git a/src/nxt_var.c b/src/nxt_var.c index cb422380f..4851bedbb 100644 --- a/src/nxt_var.c +++ b/src/nxt_var.c @@ -70,9 +70,6 @@ static const nxt_lvlhsh_proto_t nxt_var_cache_proto nxt_aligned(64) = { static nxt_lvlhsh_t nxt_var_hash; -static uint32_t nxt_var_count; - -static nxt_var_decl_t **nxt_vars; static nxt_int_t @@ -286,33 +283,6 @@ nxt_var_register(nxt_var_decl_t *decl, size_t n) } } - nxt_var_count += n; - - return NXT_OK; -} - - -nxt_int_t -nxt_var_index_init(void) -{ - nxt_uint_t i; - nxt_var_decl_t *decl, **vars; - nxt_lvlhsh_each_t lhe; - - vars = nxt_memalign(64, nxt_var_count * sizeof(nxt_var_decl_t *)); - if (vars == NULL) { - return NXT_ERROR; - } - - nxt_lvlhsh_each_init(&lhe, &nxt_var_hash_proto); - - for (i = 0; i < nxt_var_count; i++) { - decl = nxt_lvlhsh_each(&nxt_var_hash, &lhe); - vars[i] = decl; - } - - nxt_vars = vars; - return NXT_OK; } @@ -493,10 +463,75 @@ nxt_var_next_part(u_char *start, u_char *end, nxt_str_t *part) } +/* + * Escape one byte of a variable's value the way nginx's access log does + * (ngx_http_log_escape()): the quote, the backslash and everything outside + * printable ASCII become "\xHH". nginx has one emit path and no short forms + * such as "\\\"", so a reader that already unescapes an nginx log reads this + * one correctly; emitting "\\\"" here would make such a reader see a literal + * backslash followed by a field-terminating quote. + * + * The high half is escaped too, not only the control characters. A reader + * that decodes the log as UTF-8 and splits it on Unicode line breaks treats + * U+0085, U+2028 and U+2029 as terminators, so a raw 0xC2 0x85 in a value + * would end a record just as a raw newline does. + */ + +nxt_inline nxt_bool_t +nxt_var_escaped(u_char c) +{ + return c < 0x20 || c >= 0x7F || c == '"' || c == '\\'; +} + + +size_t +nxt_var_escape_length(const u_char *p, size_t length) +{ + size_t size, i; + + size = length; + + for (i = 0; i < length; i++) { + if (nxt_var_escaped(p[i])) { + /* One byte becomes the four of "\xHH". */ + size += 3; + } + } + + return size; +} + + +u_char * +nxt_var_escape(u_char *dst, const u_char *src, size_t length) +{ + static const u_char hex[] = "0123456789ABCDEF"; + + size_t i; + u_char c; + + for (i = 0; i < length; i++) { + c = src[i]; + + if (!nxt_var_escaped(c)) { + *dst++ = c; + continue; + } + + *dst++ = '\\'; + *dst++ = 'x'; + *dst++ = hex[c >> 4]; + *dst++ = hex[c & 0xf]; + } + + return dst; +} + + nxt_int_t nxt_var_interpreter(nxt_task_t *task, nxt_tstr_state_t *state, nxt_var_cache_t *cache, nxt_var_t *var, nxt_str_t *str, void *ctx, - nxt_bool_t logging) + nxt_uint_t flags) { u_char *p, *src; size_t length, last, next; @@ -529,9 +564,16 @@ nxt_var_interpreter(nxt_task_t *task, nxt_tstr_state_t *state, *part = value; - length += value->length - subs[i].length; + length -= subs[i].length; + + if (flags & NXT_VAR_ESCAPE) { + length += nxt_var_escape_length(value->start, value->length); + + } else { + length += value->length; + } - if (logging && value->start == NULL) { + if ((flags & NXT_VAR_LOGGING) && value->start == NULL) { length += 1; } } @@ -556,9 +598,14 @@ nxt_var_interpreter(nxt_task_t *task, nxt_tstr_state_t *state, p = nxt_cpymem(p, &src[last], next - last); } - p = nxt_cpymem(p, part[i]->start, part[i]->length); + if (flags & NXT_VAR_ESCAPE) { + p = nxt_var_escape(p, part[i]->start, part[i]->length); + + } else { + p = nxt_cpymem(p, part[i]->start, part[i]->length); + } - if (logging && part[i]->start == NULL) { + if ((flags & NXT_VAR_LOGGING) && part[i]->start == NULL) { *p++ = '-'; } @@ -566,9 +613,16 @@ nxt_var_interpreter(nxt_task_t *task, nxt_tstr_state_t *state, } if (last != var->length) { - nxt_cpymem(p, &src[last], var->length - last); + p = nxt_cpymem(p, &src[last], var->length - last); } + /* + * The pass above and the length computed for nxt_mp_nget() are written by + * hand and have to agree. Say so, rather than letting them drift apart + * quietly: what a variable expands to is escaped in one of them. + */ + nxt_assert(p == str->start + str->length); + return NXT_OK; } diff --git a/src/nxt_var.h b/src/nxt_var.h index ca86513cb..b84ca212e 100644 --- a/src/nxt_var.h +++ b/src/nxt_var.h @@ -49,7 +49,6 @@ typedef struct { nxt_int_t nxt_var_register(nxt_var_decl_t *decl, size_t n); -nxt_int_t nxt_var_index_init(void); nxt_var_field_t *nxt_var_field_get(nxt_array_t *fields, uint16_t index); nxt_var_field_t *nxt_var_field_new(nxt_mp_t *mp, nxt_str_t *name, @@ -58,9 +57,20 @@ nxt_var_field_t *nxt_var_field_new(nxt_mp_t *mp, nxt_str_t *name, nxt_var_t *nxt_var_compile(nxt_tstr_state_t *state, nxt_str_t *str); nxt_int_t nxt_var_test(nxt_tstr_state_t *state, nxt_str_t *str, u_char *error); +typedef enum { + /* Write "-" for a variable that has no value, as an access log does. */ + NXT_VAR_LOGGING = 1 << 0, + /* Escape the bytes a variable expands to; see nxt_var_escape(). */ + NXT_VAR_ESCAPE = 1 << 1, +} nxt_var_flags_t; + + +size_t nxt_var_escape_length(const u_char *p, size_t length); +u_char *nxt_var_escape(u_char *dst, const u_char *src, size_t length); + nxt_int_t nxt_var_interpreter(nxt_task_t *task, nxt_tstr_state_t *state, nxt_var_cache_t *cache, nxt_var_t *var, nxt_str_t *str, void *ctx, - nxt_bool_t logging); + nxt_uint_t flags); nxt_str_t *nxt_var_get(nxt_task_t *task, nxt_tstr_state_t *state, nxt_var_cache_t *cache, nxt_str_t *name, void *ctx); diff --git a/src/nxt_zlib.c b/src/nxt_zlib.c index ad1957ead..ea2871b88 100644 --- a/src/nxt_zlib.c +++ b/src/nxt_zlib.c @@ -62,22 +62,25 @@ nxt_zlib_deflate(nxt_http_comp_compressor_ctx_t *ctx, const uint8_t *in_buf, ret = deflate(z, last ? Z_FINISH : Z_SYNC_FLUSH); if (ret == Z_STREAM_ERROR || ret == Z_BUF_ERROR) { - deflateEnd(z); return -1; } - if (last) { - deflateEnd(z); - } - return z->total_out - compressed_bytes; } +static void +nxt_zlib_free(nxt_http_comp_compressor_ctx_t *ctx) +{ + deflateEnd(&ctx->zlib_ctx); +} + + const nxt_http_comp_operations_t nxt_http_comp_deflate_ops = { .init = nxt_zlib_deflate_init, .bound = nxt_zlib_bound, .deflate = nxt_zlib_deflate, + .free = nxt_zlib_free, }; @@ -85,4 +88,5 @@ const nxt_http_comp_operations_t nxt_http_comp_gzip_ops = { .init = nxt_zlib_gzip_init, .bound = nxt_zlib_bound, .deflate = nxt_zlib_deflate, + .free = nxt_zlib_free, }; diff --git a/src/nxt_zstd.c b/src/nxt_zstd.c index 8fb379448..b4d8ad2cc 100644 --- a/src/nxt_zstd.c +++ b/src/nxt_zstd.c @@ -51,7 +51,6 @@ nxt_zstd_compress(nxt_http_comp_compressor_ctx_t *ctx, const uint8_t *in_buf, if (last) { ret = ZSTD_endStream(zstd, &zoutb); - ZSTD_freeCStream(zstd); } if (ZSTD_isError(ret)) { @@ -62,8 +61,17 @@ nxt_zstd_compress(nxt_http_comp_compressor_ctx_t *ctx, const uint8_t *in_buf, } +static void +nxt_zstd_free(nxt_http_comp_compressor_ctx_t *ctx) +{ + ZSTD_freeCStream(ctx->zstd_ctx); + ctx->zstd_ctx = NULL; +} + + const nxt_http_comp_operations_t nxt_http_comp_zstd_ops = { .init = nxt_zstd_init, .bound = nxt_zstd_bound, .deflate = nxt_zstd_compress, + .free = nxt_zstd_free, }; diff --git a/src/otel/Cargo.lock b/src/otel/Cargo.lock index 365e14b77..a52cfc5f5 100644 --- a/src/otel/Cargo.lock +++ b/src/otel/Cargo.lock @@ -4,9 +4,9 @@ version = 4 [[package]] name = "anyhow" -version = "1.0.102" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "async-trait" @@ -74,11 +74,17 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "bitflags" -version = "2.11.1" +version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" [[package]] name = "bumpalo" @@ -88,9 +94,9 @@ checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" [[package]] name = "bytes" -version = "1.11.1" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "cfg-if" @@ -222,9 +228,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.14" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" +checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" dependencies = [ "atomic-waker", "bytes", @@ -247,9 +253,9 @@ checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" [[package]] name = "http" -version = "1.4.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8be7462df143984c4598a256ef469b251d7d7f9e271135073e78fc535414f3d0" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" dependencies = [ "bytes", "itoa", @@ -257,9 +263,9 @@ dependencies = [ [[package]] name = "http-body" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" dependencies = [ "bytes", "http", @@ -267,9 +273,9 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.3" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" dependencies = [ "bytes", "futures-core", @@ -292,9 +298,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hyper" -version = "1.10.1" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" dependencies = [ "atomic-waker", "bytes", @@ -327,16 +333,17 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.20" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" dependencies = [ - "base64", + "base64 0.23.1", "bytes", "futures-channel", "futures-util", "http", "http-body", + "httparse", "hyper", "ipnet", "libc", @@ -549,9 +556,9 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "opentelemetry" -version = "0.32.0" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0142c63252a9e054e68a4c61a5778f7b14f576274d593f8ce883d191a099682" +checksum = "6cdb0b1b267eb9db3331b434ed9ddab10d50e280a9adf9d13e5233e2002b61b5" dependencies = [ "futures-core", "futures-sink", @@ -563,9 +570,9 @@ dependencies = [ [[package]] name = "opentelemetry-http" -version = "0.32.0" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5683015d09e2df236ef005b17f6f196f0d5f6313c4fa43a7b6a53b52776e4331" +checksum = "ee2c3625b8aa04209f7e01e513bc8044da9687fa38a9eacf59628ca5f3b87300" dependencies = [ "async-trait", "bytes", @@ -576,11 +583,12 @@ dependencies = [ [[package]] name = "opentelemetry-otlp" -version = "0.32.0" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9966929966d17620d7c316c643ba62631826e10021409357772d5eea84f62c35" +checksum = "699a67345e21962a231955b9059a157e428b219fa5df8efe11f08346fb23a344" dependencies = [ "http", + "httpdate", "opentelemetry", "opentelemetry-http", "opentelemetry-proto", @@ -595,9 +603,9 @@ dependencies = [ [[package]] name = "opentelemetry-proto" -version = "0.32.0" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56d658ba1faf63f7b9c492cfbe6e0ec365440a16132d3270c1065f7b33f1b638" +checksum = "25da1ac11a0aeccf38d7f77ee0348715adaf8340f65ad46c94a02c6b20e2f65d" dependencies = [ "opentelemetry", "opentelemetry_sdk", @@ -608,15 +616,15 @@ dependencies = [ [[package]] name = "opentelemetry-semantic-conventions" -version = "0.32.0" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ca2f98a0437b427b4b08f19f1caa3c44db885a202bc12cfea13d6c702243d68" +checksum = "d288b1e1bc3590052fbb8f0f635ef248dd9022cf1fbaeb6582b7ad6081a17b87" [[package]] name = "opentelemetry_sdk" -version = "0.32.1" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b59f80e1ac4d5ff7a2db8fb6c80badb7f0f3f858211fba08dd9aaec750894f9" +checksum = "cb39533d9d1c912123efd7d41d7e0c29d16917b60ce15b4c8d87cb1af7f67520" dependencies = [ "futures-channel", "futures-executor", @@ -707,9 +715,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" dependencies = [ "bytes", "prost-derive", @@ -717,9 +725,9 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", "itertools", @@ -787,7 +795,7 @@ version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-channel", "futures-core", @@ -938,9 +946,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.52.3" +version = "1.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" dependencies = [ "bytes", "libc", @@ -994,7 +1002,7 @@ checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" dependencies = [ "async-trait", "axum", - "base64", + "base64 0.22.1", "bytes", "h2", "http", diff --git a/src/otel/Cargo.toml b/src/otel/Cargo.toml index a8c2ab834..f3b47e1be 100644 --- a/src/otel/Cargo.toml +++ b/src/otel/Cargo.toml @@ -2,6 +2,7 @@ name = "otel" version = "0.1.0" edition = "2021" +rust-version = "1.88" [lib] crate-type = ["staticlib"] @@ -24,14 +25,14 @@ bench = false # staticlib), so both transports are plaintext — the endpoint must be plain # http:// (h2c for grpc). [dependencies] -opentelemetry = "0.32" -opentelemetry_sdk = { version = "0.32", features = ["trace"] } -opentelemetry-otlp = { version = "0.32", default-features = false, features = [ +opentelemetry = "0.33" +opentelemetry_sdk = { version = "0.33", features = ["trace"] } +opentelemetry-otlp = { version = "0.33", default-features = false, features = [ "trace", "http-proto", "reqwest-blocking-client", "grpc-tonic", ] } -opentelemetry-semantic-conventions = "0.32" +opentelemetry-semantic-conventions = "0.33" tonic = "0.14" tokio = { version = "1", features = ["rt-multi-thread"] } diff --git a/src/otel/src/lib.rs b/src/otel/src/lib.rs index 9311491dd..c4b6b26d0 100644 --- a/src/otel/src/lib.rs +++ b/src/otel/src/lib.rs @@ -14,20 +14,27 @@ //! which exports it from its own background thread. use opentelemetry::global; -use opentelemetry::global::BoxedSpan; +use opentelemetry::global::{BoxedSpan, BoxedTracer}; use opentelemetry::trace::{ Span, SpanContext, SpanId, SpanKind, Status, TraceContextExt, TraceFlags, TraceId, TraceState, Tracer, TracerProvider, }; -use opentelemetry::{Context, KeyValue}; -use opentelemetry_otlp::{Protocol, SpanExporter, WithExportConfig}; +use opentelemetry::{Context, Key, KeyValue, Value}; +use opentelemetry_otlp::{ + Protocol, RetryPolicy, SpanExporter, WithExportConfig, WithHttpConfig, + WithTonicConfig, +}; +use opentelemetry_sdk::error::OTelSdkResult; use opentelemetry_sdk::trace::{ - BatchConfigBuilder, BatchSpanProcessor, Sampler, SdkTracerProvider, + BatchConfigBuilder, BatchSpanProcessor, Sampler, SdkTracerProvider, SpanData, + SpanExporter as SdkSpanExporter, }; use opentelemetry_sdk::Resource; use std::ffi::{c_char, CStr, CString}; use std::str::FromStr; -use std::sync::Mutex; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::mpsc; +use std::sync::{Mutex, RwLock}; use std::time::Duration; use std::{ptr, slice}; @@ -40,6 +47,63 @@ const SPAN_NAME: &str = "request"; const NXT_LOG_ERR: nxt_uint_t = 1; +/// Return values of `nxt_otel_rs_shutdown_bounded`. +/// +/// These are mirrored as `NXT_OTEL_SHUTDOWN_*` in `src/nxt_otel.h`, which is +/// the only consumer; the two lists must be kept in step. +const NXT_OTEL_SHUTDOWN_TIMEOUT: u8 = 0; +const NXT_OTEL_SHUTDOWN_FLUSHED: u8 = 1; +const NXT_OTEL_SHUTDOWN_FAILED: u8 = 2; + +/// Span attribute keys, indexed by the `nxt_otel_attr_id_t` id C sends. +/// +/// `Key::from_static_str` is a `const fn` over a `&'static str`, so a key +/// costs no allocation and no `strlen` -- where the previous string-key FFI +/// copied all eleven of these compile-time constants onto the heap on every +/// traced request. +/// +/// This table and `nxt_otel_attr_id_t` in `src/nxt_otel.h` are one contract: +/// the id indexes this array directly, so the order must match exactly. +/// The `[Key; NXT_OTEL_ATTR_MAX]` annotation makes a mismatched element count +/// a compile error, so a key added here without extending the count cannot +/// ship. It cannot check the C enum itself -- the ids stay a reviewed +/// contract -- but it catches the half of the mistake that lives on this side. +/// +/// These follow the stable OpenTelemetry HTTP semantic conventions +/// (); +/// "http.flavor"/"http.user_agent" from older drafts are superseded by +/// "network.protocol.version"/"user_agent.original". "unit.application.*" are +/// FreeUnit-specific: a reverse proxy can't know the served app, but Unit can. +static ATTR_KEYS: [Key; NXT_OTEL_ATTR_MAX] = [ + Key::from_static_str("http.request.method"), + Key::from_static_str("url.path"), + Key::from_static_str("url.scheme"), + Key::from_static_str("network.protocol.version"), + Key::from_static_str("user_agent.original"), + Key::from_static_str("server.address"), + Key::from_static_str("client.address"), + Key::from_static_str("unit.application.name"), + Key::from_static_str("unit.application.type"), + Key::from_static_str("http.response.status_code"), + Key::from_static_str("http.request.body.size"), +]; + +/// Must equal `NXT_OTEL_ATTR_MAX` in `src/nxt_otel.h`. +const NXT_OTEL_ATTR_MAX: usize = 11; + +const NXT_OTEL_ATTR_TYPE_STR: u32 = 0; +const NXT_OTEL_ATTR_TYPE_I64: u32 = 1; + +/// One span attribute in a batch. Mirrors `nxt_otel_attr_t` in +/// `src/nxt_otel.h`; the layouts must stay identical. +#[repr(C)] +pub struct nxt_otel_attr_t { + pub key_id: u32, + pub r#type: u32, + pub ival: i64, + pub sval: nxt_str_t, +} + #[repr(C)] pub struct nxt_str_t { pub length: usize, @@ -61,6 +125,19 @@ fn provider_slot() -> &'static Mutex> { &PROVIDER } +/// The tracer built from the live provider. +/// +/// `global::tracer_provider().tracer()` takes the global provider's lock, +/// clones the provider and heap-allocates a fresh tracer on every call, but a +/// tracer is meant to be built once. Caching it leaves the request path with +/// one uncontended read lock instead. `None` means no provider is installed; +/// the request path then falls back to the global one, so behaviour is +/// unchanged either way. +fn tracer_slot() -> &'static RwLock> { + static TRACER: RwLock> = RwLock::new(None); + &TRACER +} + /// The tokio runtime owning the gRPC exporter's tonic channel. The blocking /// batch processor drives export RPCs onto it from its own thread, so it must /// outlive the provider; it is dropped in `nxt_otel_rs_shutdown_tracer`. @@ -69,13 +146,88 @@ fn runtime_slot() -> &'static Mutex> { &RT } +/// Set whenever an export returns `Err`, and cleared when a new provider is +/// installed. Sticky because the `BatchSpanProcessor` throws away the result +/// of every batch it exports on its own schedule (it only propagates the one +/// forced by `force_flush`), so without this a batch that failed minutes +/// before exit would leave nothing at all for the shutdown path to report. +static EXPORT_FAILED: AtomicBool = AtomicBool::new(false); + +/// Spans in batches the exporter accepted, and spans in batches it rejected, +/// since the live provider was installed. Read out by +/// `nxt_otel_rs_export_stats` for the `/status` API, which is the only way an +/// operator can see export health without waiting for the process to exit. +/// +/// Counted in spans rather than batches because the number an operator needs +/// is "how much telemetry did I lose", and a batch is a variable-size unit +/// that answers that only by accident. Neither counter includes spans the +/// processor dropped before an export was attempted (a full queue): that +/// happens inside the SDK, which reports it nowhere we can observe. +static SPANS_EXPORTED: AtomicU64 = AtomicU64::new(0); +static SPANS_FAILED: AtomicU64 = AtomicU64::new(0); + +/// Wraps the configured OTLP exporter so a failed export is remembered in +/// `EXPORT_FAILED` even when the processor discards the result. +/// +/// Everything other than `export` is delegated verbatim to the inner +/// exporter: taking the trait's defaults here would silently turn shutdown, +/// force-flush and resource propagation into no-ops. +#[derive(Debug)] +struct FailureTrackingExporter { + inner: E, +} + +impl SdkSpanExporter for FailureTrackingExporter { + async fn export(&self, batch: Vec) -> OTelSdkResult { + // Count before the batch is moved into the inner exporter. + let spans = batch.len() as u64; + + let res = self.inner.export(batch).await; + + if res.is_err() { + // The boolean is kept alongside the counter rather than derived + // from it: it is the fact the shutdown path asks for ("has any + // export failed"), and it stays true even for the degenerate + // empty batch that would add nothing to SPANS_FAILED. + EXPORT_FAILED.store(true, Ordering::Release); + SPANS_FAILED.fetch_add(spans, Ordering::Relaxed); + } else { + SPANS_EXPORTED.fetch_add(spans, Ordering::Relaxed); + } + + res + } + + fn shutdown_with_timeout(&self, timeout: Duration) -> OTelSdkResult { + self.inner.shutdown_with_timeout(timeout) + } + + fn shutdown(&self) -> OTelSdkResult { + self.inner.shutdown() + } + + fn force_flush(&self) -> OTelSdkResult { + self.inner.force_flush() + } + + fn set_resource(&mut self, resource: &Resource) { + self.inner.set_resource(resource); + } +} + /// Build the OTLP/HTTP exporter: the blocking reqwest client, no async runtime. +/// +/// opentelemetry-otlp 0.33 retries a failed export up to 3 times by default. +/// Both builders disable this. The batch worker blocks on each export, so the +/// retries would hold the queue while the collector fails. A failed batch is +/// counted as failed in /status and dropped. fn build_http_exporter(endpoint: String) -> Result { SpanExporter::builder() .with_http() .with_endpoint(endpoint) .with_protocol(Protocol::HttpBinary) .with_timeout(EXPORT_TIMEOUT) + .with_retry_policy(RetryPolicy::disabled()) .build() .map_err(|e| format!("couldn't build otel http exporter: {e}")) } @@ -97,6 +249,7 @@ fn build_grpc_exporter(endpoint: String) -> Result { .with_tonic() .with_endpoint(endpoint) .with_timeout(EXPORT_TIMEOUT) + .with_retry_policy(RetryPolicy::disabled()) .build() .map_err(|e| format!("couldn't build otel grpc exporter: {e}"))? }; @@ -138,6 +291,44 @@ pub unsafe extern "C" fn nxt_otel_rs_is_init() -> u8 { .unwrap_or(0) } +/// Report span export health for the `/status` API. +/// +/// Writes the number of spans the exporter accepted and the number it +/// rejected since the live provider was installed, and returns 1 when a +/// provider is installed at all. +/// +/// The counters are written whether or not a provider is installed, so on a +/// 0 return they hold whatever the previous provider left behind -- they are +/// only zeroed when a new one is installed. A 0 return therefore means the +/// values are stale, not absent: the caller must ignore them rather than +/// report them. +/// +/// The counters are read with two independent `Relaxed` loads, so a report +/// taken while an export is completing can be off by one batch. That is +/// deliberate: this is a health gauge, not an accounting ledger, and taking a +/// lock on the export path to make the pair atomic would cost more than the +/// skew is worth. +/// +/// # Safety +/// +/// `exported` and `failed` must each be either null or a valid, aligned, +/// writable `uint64_t`. +#[no_mangle] +pub unsafe extern "C" fn nxt_otel_rs_export_stats( + exported: *mut u64, + failed: *mut u64, +) -> u8 { + if !exported.is_null() { + *exported = SPANS_EXPORTED.load(Ordering::Relaxed); + } + + if !failed.is_null() { + *failed = SPANS_FAILED.load(Ordering::Relaxed); + } + + nxt_otel_rs_is_init() +} + #[no_mangle] pub unsafe extern "C" fn nxt_otel_rs_uninit() { nxt_otel_rs_shutdown_tracer(); @@ -187,7 +378,19 @@ pub unsafe extern "C" fn nxt_otel_rs_init( } }; - let processor = BatchSpanProcessor::builder(exporter) + // A new provider starts with a clean slate, so the shutdown path does not + // normally report a failure that belonged to the exporter we replaced: + // nxt_otel_rs_shutdown_tracer() above joins the old worker thread before + // this runs. It joins on a 5s budget though, and an export gets 10s, so + // a worker still blocked on a dead collector can outlive the join and set + // the flag after this store. The cost is one spurious FAILED at the next + // exit, describing a failure that was real but belonged to the old + // exporter -- not worth a second flag to suppress. + EXPORT_FAILED.store(false, Ordering::Release); + SPANS_EXPORTED.store(0, Ordering::Relaxed); + SPANS_FAILED.store(0, Ordering::Relaxed); + + let processor = BatchSpanProcessor::builder(FailureTrackingExporter { inner: exporter }) .with_batch_config( BatchConfigBuilder::default() .with_max_export_batch_size(batch_size as usize) @@ -210,6 +413,10 @@ pub unsafe extern "C" fn nxt_otel_rs_init( global::set_tracer_provider(provider.clone()); + if let Ok(mut slot) = tracer_slot().write() { + *slot = Some(global::tracer_provider().tracer(TRACER_NAME)); + } + if let Ok(mut slot) = provider_slot().lock() { *slot = Some(provider); } @@ -241,39 +448,74 @@ pub unsafe extern "C" fn nxt_otel_rs_copy_traceparent(buf: *mut c_char, span: *c *buf.add(TRACEPARENT_HEADER_LEN as usize) = 0; } +/// Set a stage's semantic-convention span attributes in one call. +/// +/// C accumulates a stage's attributes on its stack and crosses once, rather +/// than once per attribute: a traced request used to make eight or nine FFI +/// crossings here and allocate two `String`s at each of them. Only the values +/// still allocate, and only those that are strings. +/// +/// Note this deliberately does NOT use `SpanBuilder::with_attributes`, which +/// issue #221 suggests. That would assemble the attributes while the span is +/// being built -- before C can observe the sampling decision -- which would +/// undo the `is_recording` gate that is the measured win. Attributes are set +/// after the gate, on a span already known to be recording. +/// +/// # Safety +/// +/// `attrs` must point at `n` initialised `nxt_otel_attr_t`, and each entry's +/// `sval` must reference bytes that stay valid for the duration of the call. #[no_mangle] -pub unsafe extern "C" fn nxt_otel_rs_add_event_to_trace( +pub unsafe extern "C" fn nxt_otel_rs_add_attrs( trace: *mut BoxedSpan, - key: *const nxt_str_t, - val: *const nxt_str_t, + attrs: *const nxt_otel_attr_t, + n: usize, ) { - if trace.is_null() || key.is_null() || val.is_null() { + if trace.is_null() || attrs.is_null() || n == 0 { return; } - let key = nxt_str_to_string(&*key); - let val = nxt_str_to_string(&*val); - - (*trace).add_event("Unit Attribute".to_string(), vec![KeyValue::new(key, val)]); + let attrs = slice::from_raw_parts(attrs, n); + + // Set each attribute directly rather than collecting into a Vec first. + // `Span::set_attributes` is a default trait method that just loops over + // `set_attribute`, and `BoxedSpan` does not override it, so a Vec here + // would be an allocation that buys nothing. The saving this function + // exists for is the single FFI crossing and the static keys, not batching + // inside the SDK. + for attr in attrs { + // A key_id C and Rust disagree about would index out of bounds; + // drop the attribute rather than panic across the FFI boundary. + let Some(key) = ATTR_KEYS.get(attr.key_id as usize) else { + continue; + }; + + let value = match attr.r#type { + NXT_OTEL_ATTR_TYPE_I64 => Value::I64(attr.ival), + NXT_OTEL_ATTR_TYPE_STR => Value::String(nxt_str_to_string(&attr.sval).into()), + _ => continue, + }; + + // Cloning a Key built by `from_static_str` copies a &'static str -- + // no allocation, which is the point of the ATTR_KEYS table. + (*trace).set_attribute(KeyValue::new(key.clone(), value)); + } } -/// Set a semantic-convention span attribute (e.g. `http.request.method`). -/// Unlike `add_event_to_trace`, this records structured span attributes the -/// collector can index and query, not timestamped events. +/// Whether the sampler kept this span. +/// +/// C asks once, right after the span is built, and skips the attribute work +/// for a span that is not recording. `set_attribute` on such a span is +/// already a no-op, but its arguments are built before the call that throws +/// them away -- so without this gate, lowering `sampling_ratio` buys back the +/// exporter and nothing on the request path. #[no_mangle] -pub unsafe extern "C" fn nxt_otel_rs_add_attr( - trace: *mut BoxedSpan, - key: *const nxt_str_t, - val: *const nxt_str_t, -) { - if trace.is_null() || key.is_null() || val.is_null() { - return; +pub unsafe extern "C" fn nxt_otel_rs_is_recording(trace: *mut BoxedSpan) -> u8 { + if trace.is_null() { + return 0; } - let key = nxt_str_to_string(&*key); - let val = nxt_str_to_string(&*val); - - (*trace).set_attribute(KeyValue::new(key, val)); + (*trace).is_recording() as u8 } /// Mark the span as errored. Called by C for 5xx responses so the trace is @@ -326,6 +568,12 @@ unsafe fn nxt_otel_parent_context( Some(Context::new().with_remote_span_context(sc)) } +fn nxt_otel_build_span(tracer: &BoxedTracer, parent: &Context) -> BoxedSpan { + let builder = tracer.span_builder(SPAN_NAME).with_kind(SpanKind::Server); + + tracer.build_with_context(builder, parent) +} + #[no_mangle] pub unsafe extern "C" fn nxt_otel_rs_get_or_create_trace( trace_id: *const c_char, @@ -333,12 +581,23 @@ pub unsafe extern "C" fn nxt_otel_rs_get_or_create_trace( trace_flags: *const c_char, trace_state: *const nxt_str_t, ) -> *mut BoxedSpan { - let tracer = global::tracer_provider().tracer(TRACER_NAME); - let builder = tracer.span_builder(SPAN_NAME).with_kind(SpanKind::Server); - let parent = nxt_otel_parent_context(trace_id, parent_id, trace_flags, trace_state) .unwrap_or_else(Context::new); - let span = tracer.build_with_context(builder, &parent); + + // The read guard is held across the build so the cached tracer cannot be + // dropped by a concurrent reconfigure while a span is being made from it. + let cached = tracer_slot().read().ok(); + + let span = match cached.as_ref().and_then(|slot| slot.as_ref()) { + Some(tracer) => nxt_otel_build_span(tracer, &parent), + None => { + // No provider installed, or the lock is poisoned: fall back to the + // global provider, which hands out a no-op tracer in that case. + let tracer = global::tracer_provider().tracer(TRACER_NAME); + + nxt_otel_build_span(&tracer, &parent) + } + }; Box::into_raw(Box::new(span)) } @@ -356,9 +615,115 @@ pub unsafe extern "C" fn nxt_otel_rs_send_trace(trace: *mut BoxedSpan) { span.end(); } +/// Flush and tear down the live tracer provider with a caller-supplied bound, +/// for use on the process exit path. +/// +/// `nxt_otel_rs_shutdown_tracer` can block for as long as the exporter's own +/// `EXPORT_TIMEOUT` (10s) when the collector is unreachable, which is far too +/// long to sit in front of `exit()`. The provider and runtime are taken out of +/// their slots here and *moved* into a helper thread, so the statics are left +/// empty and unlocked whatever happens; we then wait up to `timeout_ms` for +/// that thread to finish flushing. On timeout we give up and return: the +/// helper thread is left running and dies with the process. Best effort by +/// construction — spans that could not be flushed in the budget are lost, +/// exactly as they are today, but a dead collector can no longer delay exit. +/// +/// Returns one of three statuses, mirrored in `src/nxt_otel.h`: +/// +/// * `NXT_OTEL_SHUTDOWN_FLUSHED` (1) — the flush completed within the budget +/// and no export has failed since this provider was installed. +/// * `NXT_OTEL_SHUTDOWN_FAILED` (2) — an export failed. Either the flush this +/// call forced failed, or `EXPORT_FAILED` records an earlier batch that did: +/// the processor exports on its own schedule and discards the result, so the +/// only way that failure is visible at all is the sticky flag. The helper +/// thread failing to spawn reports here too; it is a failure, not a timeout. +/// * `NXT_OTEL_SHUTDOWN_TIMEOUT` (0) — the helper thread did not answer inside +/// `timeout_ms`. Kept distinct from the above because a rejected export +/// fails in milliseconds, and calling that a timeout sends an operator +/// looking for latency that is not there. +/// +/// One loss remains unreportable from here: the router's worker engines are +/// still live when `nxt_runtime_exit` calls this, so a span they end after the +/// flush reaches a provider that is already shut down and is dropped without a +/// word. Nothing at this layer can see it — the engine threads are never +/// joined (`nxt_thread_join` has no call sites), so producers cannot be +/// quiesced first. That needs the P5 graceful-shutdown work and is tracked in +/// issue #219. A `FLUSHED` return therefore means "this flush succeeded and no +/// export has failed", not "no spans were lost". +#[no_mangle] +pub unsafe extern "C" fn nxt_otel_rs_shutdown_bounded(timeout_ms: u64) -> u8 { + // This path takes the provider without going through + // nxt_otel_rs_shutdown_tracer(), so the cached tracer has to be dropped + // here too: leaving it would hand out a tracer whose provider has + // already been shut down. + if let Ok(mut slot) = tracer_slot().write() { + *slot = None; + } + + let provider = provider_slot().lock().ok().and_then(|mut g| g.take()); + let rt = runtime_slot().lock().ok().and_then(|mut g| g.take()); + + if provider.is_none() && rt.is_none() { + return if EXPORT_FAILED.load(Ordering::Acquire) { + NXT_OTEL_SHUTDOWN_FAILED + } else { + NXT_OTEL_SHUTDOWN_FLUSHED + }; + } + + let (tx, rx) = mpsc::channel::(); + + let spawned = std::thread::Builder::new() + .name("otel-shutdown".to_string()) + .spawn(move || { + let mut flushed = true; + + if let Some(provider) = provider { + // Needs the runtime alive on a grpc build, hence the ordering. + // + // force_flush() first, because it is the only one of the two + // that reports whether the spans actually left: the batch + // processor hands shutdown() the export result and shutdown() + // throws it away, reporting only that the worker answered. A + // collector that refuses the connection fails in milliseconds, + // so shutdown() alone would call that a clean flush and exit + // without a word. Both run whatever the first one says. + let exported = provider.force_flush().is_ok(); + let stopped = provider.shutdown().is_ok(); + + flushed = exported && stopped; + } + if let Some(rt) = rt { + rt.shutdown_background(); + } + // Read the sticky flag only after the flush above, so a batch the + // flush itself pushed out and failed on is counted here too. + let _ = tx.send(flushed && !EXPORT_FAILED.load(Ordering::Acquire)); + }); + + // The caller logs these, so they must not be conflated: a collector that + // rejects the export fails in milliseconds, and reporting that as a + // timeout sends an operator looking for latency that is not there. + match spawned { + Ok(_) => match rx.recv_timeout(Duration::from_millis(timeout_ms)) { + Ok(true) => NXT_OTEL_SHUTDOWN_FLUSHED, + Ok(false) => NXT_OTEL_SHUTDOWN_FAILED, + Err(_) => NXT_OTEL_SHUTDOWN_TIMEOUT, + }, + // Nothing flushed at all, and no time was spent trying. + Err(_) => NXT_OTEL_SHUTDOWN_FAILED, + } +} + /// Flush and tear down the live tracer provider, if any. #[no_mangle] pub unsafe extern "C" fn nxt_otel_rs_shutdown_tracer() { + // Dropped first: a tracer handed out after this point would belong to a + // provider that is already being torn down. + if let Ok(mut slot) = tracer_slot().write() { + *slot = None; + } + let provider = provider_slot().lock().ok().and_then(|mut g| g.take()); if let Some(provider) = provider { // Flushes pending spans; on a grpc build this still needs the runtime, diff --git a/src/python/nxt_python_asgi.c b/src/python/nxt_python_asgi.c index a123e9411..5cd145957 100644 --- a/src/python/nxt_python_asgi.c +++ b/src/python/nxt_python_asgi.c @@ -46,6 +46,8 @@ static void nxt_py_asgi_quit(nxt_unit_ctx_t *ctx); static void nxt_py_asgi_shm_ack_handler(nxt_unit_ctx_t *ctx); static PyObject *nxt_py_asgi_port_read(PyObject *self, PyObject *args); +static void nxt_py_asgi_retry_timer_cancel(nxt_unit_ctx_t *ctx, + nxt_py_asgi_ctx_data_t *ctx_data); static void nxt_python_asgi_done(void); static PyObject *nxt_py_port_read; @@ -269,6 +271,7 @@ nxt_python_asgi_ctx_data_alloc(void **pdata, int main) { "add_reader", &ctx_data->loop_add_reader }, { "remove_reader", &ctx_data->loop_remove_reader }, { "call_soon", &ctx_data->loop_call_soon }, + { "call_later", &ctx_data->loop_call_later }, { "run_until_complete", &ctx_data->loop_run_until_complete }, { "create_future", &ctx_data->loop_create_future }, }; @@ -366,10 +369,15 @@ nxt_python_asgi_ctx_data_free(void *data) ctx_data = data; + /* The delayed call keeps the pointer of the context. It must not run. */ + + nxt_py_asgi_retry_timer_cancel(NULL, ctx_data); + Py_XDECREF(ctx_data->loop_run_until_complete); Py_XDECREF(ctx_data->loop_create_future); Py_XDECREF(ctx_data->loop_create_task); Py_XDECREF(ctx_data->loop_call_soon); + Py_XDECREF(ctx_data->loop_call_later); Py_XDECREF(ctx_data->loop_add_reader); Py_XDECREF(ctx_data->loop_remove_reader); Py_XDECREF(ctx_data->quit_future); @@ -1162,8 +1170,8 @@ nxt_py_asgi_shm_ack_handler(nxt_unit_ctx_t *ctx) static PyObject * nxt_py_asgi_port_read(PyObject *self, PyObject *args) { - int rc; - PyObject *arg0, *arg1, *res; + int rc, delay, port_id; + PyObject *arg0, *arg1, *res, *timeout; Py_ssize_t n; nxt_unit_ctx_t *ctx; nxt_unit_port_t *port; @@ -1195,24 +1203,82 @@ nxt_py_asgi_port_read(PyObject *self, PyObject *args) port = PyLong_AsVoidPtr(arg1); + /* + * When the FINISH retries give up, the call quits the context and can + * free the port. Keep its id for the error message. + */ + + port_id = port->id.id; + rc = nxt_unit_process_port_msg(ctx, port); nxt_unit_debug(ctx, "asgi_port_read(%p,%p): %d", ctx, port, rc); + ctx_data = ctx->data; + + /* + * Each call can run the pending FINISH retry of the context, whatever + * the port. So one delayed call for each context is enough. Cancel + * the old one before a new call is scheduled. Otherwise each burst of + * traffic adds one more timer, and all of them wake the loop. Cancel + * it also before the error return. After the give-up the loop still + * runs the lifespan shutdown, and the old call must not run then. + */ + + nxt_py_asgi_retry_timer_cancel(ctx, ctx_data); + if (nxt_slow_path(rc == NXT_UNIT_ERROR)) { return PyErr_Format(PyExc_RuntimeError, - "error processing port %d message", port->id.id); + "error processing port %d message", port_id); } if (rc == NXT_UNIT_OK) { - ctx_data = ctx->data; - res = PyObject_CallFunctionObjArgs(ctx_data->loop_call_soon, - nxt_py_port_read, - arg0, arg1, NULL); - if (nxt_slow_path(res == NULL)) { - nxt_unit_alert(ctx, "Python failed to call 'loop.call_soon'"); - nxt_python_print_exception(); + /* + * NXT_UNIT_OK also comes when the call found no message and a + * FINISH retry is pending. The retry is not due before a deadline. + * Call again at that deadline, and do not spin the loop with + * call_soon(). + */ + + delay = nxt_unit_detached_retry_timeout(ctx); + + if (delay > 0) { + timeout = PyFloat_FromDouble(delay / 1000.0); + if (nxt_slow_path(timeout == NULL)) { + nxt_unit_alert(ctx, "Python failed to create a float"); + nxt_python_print_exception(); + + Py_RETURN_NONE; + } + + res = PyObject_CallFunctionObjArgs(ctx_data->loop_call_later, + timeout, nxt_py_port_read, + arg0, arg1, NULL); + + Py_DECREF(timeout); + + if (nxt_slow_path(res == NULL)) { + nxt_unit_alert(ctx, "Python failed to call 'loop.call_later'"); + nxt_python_print_exception(); + + Py_RETURN_NONE; + } + + /* Keep the TimerHandle, so that a later call can cancel it. */ + + ctx_data->retry_timer = res; + + Py_RETURN_NONE; + + } else { + res = PyObject_CallFunctionObjArgs(ctx_data->loop_call_soon, + nxt_py_port_read, + arg0, arg1, NULL); + if (nxt_slow_path(res == NULL)) { + nxt_unit_alert(ctx, "Python failed to call 'loop.call_soon'"); + nxt_python_print_exception(); + } } Py_XDECREF(res); @@ -1222,6 +1288,35 @@ nxt_py_asgi_port_read(PyObject *self, PyObject *args) } +static void +nxt_py_asgi_retry_timer_cancel(nxt_unit_ctx_t *ctx, + nxt_py_asgi_ctx_data_t *ctx_data) +{ + PyObject *timer, *res; + + timer = ctx_data->retry_timer; + + if (timer == NULL) { + return; + } + + ctx_data->retry_timer = NULL; + + /* Cancel of a TimerHandle that already ran does nothing. */ + + res = PyObject_CallMethod(timer, "cancel", NULL); + if (nxt_slow_path(res == NULL)) { + nxt_unit_alert(ctx, "Python failed to cancel the retry timer"); + nxt_python_print_exception(); + + } else { + Py_DECREF(res); + } + + Py_DECREF(timer); +} + + PyObject * nxt_py_asgi_enum_headers(PyObject *headers, nxt_py_asgi_enum_header_cb cb, void *data) diff --git a/src/python/nxt_python_asgi.h b/src/python/nxt_python_asgi.h index 94478a361..a9783536c 100644 --- a/src/python/nxt_python_asgi.h +++ b/src/python/nxt_python_asgi.h @@ -29,6 +29,8 @@ typedef struct { PyObject *loop_create_future; PyObject *loop_create_task; PyObject *loop_call_soon; + PyObject *loop_call_later; + PyObject *retry_timer; PyObject *loop_add_reader; PyObject *loop_remove_reader; PyObject *quit_future; diff --git a/src/python/nxt_python_asgi_websocket.c b/src/python/nxt_python_asgi_websocket.c index 195864b4a..3590aaa0b 100644 --- a/src/python/nxt_python_asgi_websocket.c +++ b/src/python/nxt_python_asgi_websocket.c @@ -40,7 +40,6 @@ typedef struct { int state; nxt_queue_t pending_frames; uint64_t pending_payload_len; - uint64_t pending_frame_len; int pending_fins; } nxt_py_asgi_websocket_t; @@ -94,8 +93,17 @@ static PyTypeObject nxt_py_asgi_websocket_type = { .tp_methods = nxt_py_asgi_websocket_methods, }; -static uint64_t nxt_py_asgi_ws_max_frame_size = 1024 * 1024; -static uint64_t nxt_py_asgi_ws_max_buffer_size = 10 * 1024 * 1024; +/* + * Bounds the payload bytes a connection holds that the application has not + * yet received: the fragments of a message in progress, and complete messages + * waiting for receive(). The per-frame limit is the operator's + * settings/http/websocket/max_frame_size, applied by the router before the + * frame reaches this worker (the max_frame_size check in + * nxt_h1p_conn_ws_frame_header_read()); the router does no accounting across + * frames, which is what is bounded here. A suspended frame pins router shared + * memory, in 16 KiB chunks per frame, not worker heap. + */ +#define NXT_PY_ASGI_WS_MAX_BUFFER (10 * 1024 * 1024) int @@ -125,7 +133,6 @@ nxt_py_asgi_websocket_create(nxt_unit_request_info_t *req) ws->state = NXT_WS_INIT; nxt_queue_init(&ws->pending_frames); ws->pending_payload_len = 0; - ws->pending_frame_len = 0; ws->pending_fins = 0; } @@ -490,7 +497,7 @@ nxt_py_asgi_websocket_handler(nxt_unit_websocket_frame_t *frame) { uint8_t opcode; uint16_t status_code; - uint64_t rest; + uint64_t limit; PyObject *msg, *exc; nxt_py_asgi_websocket_t *ws; @@ -519,17 +526,23 @@ nxt_py_asgi_websocket_handler(nxt_unit_websocket_frame_t *frame) goto bad_state; } - rest = nxt_py_asgi_ws_max_frame_size - ws->pending_frame_len; - - if (nxt_slow_path(frame->payload_len > rest)) { - nxt_unit_websocket_done(frame); + /* + * The floor admits one whole router-approved frame into an empty queue, so + * this module never refuses a frame the configured max_frame_size allowed. + * It is also why the subtraction cannot wrap: limit >= payload_len by + * construction. CLOSE is exempt: the router caps a control frame at 125 + * bytes (nxt_h1p_conn_ws_frame_header_read()), and refusing a CLOSE would + * eat the ASGI disconnect message. + */ + limit = NXT_PY_ASGI_WS_MAX_BUFFER; - goto too_big; + if (frame->payload_len > limit) { + limit = frame->payload_len; } - rest = nxt_py_asgi_ws_max_buffer_size - ws->pending_payload_len; - - if (nxt_slow_path(frame->payload_len > rest)) { + if (nxt_slow_path(opcode != NXT_WEBSOCKET_OP_CLOSE + && ws->pending_payload_len > limit - frame->payload_len)) + { nxt_unit_websocket_done(frame); goto too_big; @@ -708,9 +721,9 @@ nxt_py_asgi_websocket_suspend_frame(nxt_unit_websocket_frame_t *frame) /* * Guard against uint64 wraparound across many fragmented frames; - * otherwise the eventual max_buffer_size check is bypassed. Run - * the check BEFORE inserting p into the pending_frames queue so a - * failure exit doesn't leak the suspended-frame slot. + * otherwise the NXT_PY_ASGI_WS_MAX_BUFFER check in the handler is + * bypassed. Run the check BEFORE inserting p into the pending_frames + * queue so a failure exit doesn't leak the suspended-frame slot. */ if (nxt_slow_path(frame->payload_len > UINT64_MAX - ws->pending_payload_len)) @@ -731,18 +744,6 @@ nxt_py_asgi_websocket_suspend_frame(nxt_unit_websocket_frame_t *frame) ws->pending_payload_len += frame->payload_len; ws->pending_fins += frame->header->fin; - - if (frame->header->fin) { - ws->pending_frame_len = 0; - - } else { - if (frame->header->opcode == NXT_WEBSOCKET_OP_CONT) { - ws->pending_frame_len += frame->payload_len; - - } else { - ws->pending_frame_len = frame->payload_len; - } - } } diff --git a/src/test/nxt_buf_test.c b/src/test/nxt_buf_test.c new file mode 100644 index 000000000..f9e3159d9 --- /dev/null +++ b/src/test/nxt_buf_test.c @@ -0,0 +1,46 @@ +/* + * Copyright (C) FreeUnit contributors. + */ + +#include +#include "nxt_tests.h" + + +/* An empty nxt_str_t has a NULL start: memcpy() must not be given it. */ + +nxt_int_t +nxt_buf_test(nxt_thread_t *thr) +{ + nxt_int_t ret; + nxt_mp_t *mp; + nxt_buf_t *b; + static nxt_str_t empty = nxt_null_string; + + mp = nxt_mp_create(1024, 128, 512, 16); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + ret = NXT_ERROR; + + b = nxt_buf_mem_alloc(mp, 1, 0); + + if (b != NULL) { + nxt_buf_cpystr(b, &empty); + + if (nxt_buf_mem_used_size(&b->mem) == 0) { + ret = NXT_OK; + } + } + + nxt_mp_destroy(mp); + + if (ret != NXT_OK) { + nxt_log_alert(thr->log, "nxt_buf test failed"); + return NXT_ERROR; + } + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "nxt_buf test passed"); + + return NXT_OK; +} diff --git a/src/test/nxt_checked_test.c b/src/test/nxt_checked_test.c new file mode 100644 index 000000000..c8aaa3480 --- /dev/null +++ b/src/test/nxt_checked_test.c @@ -0,0 +1,153 @@ +/* + * Copyright (C) FreeUnit Community + */ + +#include +#include +#include +#include "nxt_tests.h" + + +typedef struct { + size_t a; + size_t b; + size_t result; + int overflow; +} nxt_checked_case_t; + + +static const nxt_checked_case_t nxt_checked_add_cases[] = { + { 0, 0, 0, 0 }, + { 2, 3, 5, 0 }, + { SIZE_MAX, 0, SIZE_MAX, 0 }, + { 0, SIZE_MAX, SIZE_MAX, 0 }, + { SIZE_MAX - 1, 1, SIZE_MAX, 0 }, + { SIZE_MAX, 1, 0, 1 }, + { 1, SIZE_MAX, 0, 1 }, + { SIZE_MAX, SIZE_MAX, 0, 1 }, +}; + + +static const nxt_checked_case_t nxt_checked_mul_cases[] = { + { 0, 0, 0, 0 }, + { 0, SIZE_MAX, 0, 0 }, + { SIZE_MAX, 0, 0, 0 }, + { 6, 7, 42, 0 }, + { SIZE_MAX, 1, SIZE_MAX, 0 }, + { 1, SIZE_MAX, SIZE_MAX, 0 }, + { SIZE_MAX / 2, 2, SIZE_MAX - 1, 0 }, + { SIZE_MAX / 2 + 1, 2, 0, 1 }, + { SIZE_MAX, 2, 0, 1 }, + { 2, SIZE_MAX, 0, 1 }, + { SIZE_MAX, SIZE_MAX, 0, 1 }, +}; + + +/* + * Runs one add or mul function over its table. The builtin and the + * portable functions must agree on every row, so both are run over the + * same table. + */ + +static nxt_int_t +nxt_checked_run_table(nxt_thread_t *thr, const char *name, + int (*fn)(size_t, size_t, size_t *), const nxt_checked_case_t *cases, + nxt_uint_t n) +{ + size_t r; + nxt_uint_t i; + + for (i = 0; i < n; i++) { + r = 0xAB; + + if (cases[i].overflow) { + NXT_TEST_CHECK(thr->log, fn(cases[i].a, cases[i].b, &r) != 0, + "%s(%uz, %uz) did not overflow", + name, cases[i].a, cases[i].b); + continue; + } + + NXT_TEST_CHECK(thr->log, fn(cases[i].a, cases[i].b, &r) == 0 + && r == cases[i].result, + "%s(%uz, %uz) failed: %uz, expected %uz", + name, cases[i].a, cases[i].b, r, cases[i].result); + } + + return NXT_OK; +} + + +nxt_int_t +nxt_checked_test(nxt_thread_t *thr) +{ + size_t extra; + u_char dst[4]; + nxt_span_t span; + const u_char *p; + + static const u_char buf[11] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; + + if (nxt_checked_run_table(thr, "nxt_size_add", nxt_size_add, + nxt_checked_add_cases, + nxt_nitems(nxt_checked_add_cases)) != NXT_OK + || nxt_checked_run_table(thr, "nxt_size_add_portable", + nxt_size_add_portable, nxt_checked_add_cases, + nxt_nitems(nxt_checked_add_cases)) != NXT_OK + || nxt_checked_run_table(thr, "nxt_size_mul", nxt_size_mul, + nxt_checked_mul_cases, + nxt_nitems(nxt_checked_mul_cases)) != NXT_OK + || nxt_checked_run_table(thr, "nxt_size_mul_portable", + nxt_size_mul_portable, nxt_checked_mul_cases, + nxt_nitems(nxt_checked_mul_cases)) != NXT_OK) + { + return NXT_ERROR; + } + + /* Two whole 4-byte records, then a partial tail of 0..3 bytes. */ + + for (extra = 0; extra <= 3; extra++) { + nxt_span_init(&span, buf, buf + 8 + extra); + + NXT_TEST_CHECK(thr->log, nxt_span_take(&span, 4, &p) == 0 && p == buf + && nxt_span_take(&span, 4, &p) == 0 && p == buf + 4 + && nxt_span_len(&span) == extra, + "nxt_span_take() rejected a whole record"); + NXT_TEST_CHECK(thr->log, nxt_span_take(&span, 4, &p) != 0, + "nxt_span_take() accepted a %uz-byte tail", extra); + } + + nxt_span_init(&span, buf, buf + 6); + + NXT_TEST_CHECK(thr->log, nxt_span_copy(&span, dst, 4) == 0 + && memcmp(dst, buf, 4) == 0, + "nxt_span_copy() failed on a whole record"); + + dst[0] = 0xAA; + + NXT_TEST_CHECK(thr->log, nxt_span_copy(&span, dst, 4) != 0 + && dst[0] == 0xAA, + "nxt_span_copy() accepted or wrote a partial tail"); + + /* A copy of 0 bytes succeeds and changes neither the span nor dst. */ + + NXT_TEST_CHECK(thr->log, nxt_span_copy(&span, dst, 0) == 0 + && span.pos == buf + 4 && span.end == buf + 6 + && dst[0] == 0xAA, + "nxt_span_copy() of 0 bytes failed on a non-empty span"); + + /* The same on an empty span with pos == end == NULL. */ + + nxt_span_init(&span, NULL, NULL); + + NXT_TEST_CHECK(thr->log, nxt_span_copy(&span, dst, 0) == 0 + && span.pos == NULL && span.end == NULL + && dst[0] == 0xAA, + "nxt_span_copy() of 0 bytes failed on an empty span"); + + NXT_TEST_CHECK(thr->log, nxt_span_copy(&span, dst, 1) != 0 + && span.pos == NULL && dst[0] == 0xAA, + "nxt_span_copy() accepted a byte from an empty span"); + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "nxt_checked test passed"); + return NXT_OK; +} diff --git a/src/test/nxt_conf_parse_test.c b/src/test/nxt_conf_parse_test.c index 1df388edd..7e51e05da 100644 --- a/src/test/nxt_conf_parse_test.c +++ b/src/test/nxt_conf_parse_test.c @@ -147,3 +147,161 @@ nxt_http_route_addr_test(nxt_thread_t *thr) return NXT_OK; } + + + +/* Packed: the fields after "flag" are misaligned, for UBSan to catch. */ + +typedef struct { + uint8_t flag; + int32_t i32; + int64_t i64; + int i; + ssize_t size; + off_t off; + nxt_msec_t msec; + double dbl; + nxt_str_t str; + char *cstrz; + nxt_str_t rstr; + void *ptr; + uint8_t bad8; + int32_t bad32; +} nxt_packed nxt_conf_map_test_t; + + +#define nxt_conf_map_test_field(field, type) \ + { nxt_string(#field), type, offsetof(nxt_conf_map_test_t, field) } + + +nxt_int_t +nxt_conf_map_object_test(nxt_thread_t *thr) +{ + void *ptr; + nxt_mp_t *mp; + nxt_str_t str, rstr, orig; + nxt_int_t ret; + nxt_conf_value_t *cv, *member; + nxt_conf_map_test_t dst; + + static const nxt_str_t str_name = nxt_string("str"); + static const nxt_str_t rstr_name = nxt_string("rstr"); + static const nxt_str_t ptr_name = nxt_string("ptr"); + + static const nxt_str_t json = nxt_string( + "{\"flag\": true, \"i32\": -123456, \"i64\": -9876543210, \"i\": -7," + " \"size\": 424242, \"off\": 131072, \"msec\": 3, \"dbl\": 1.5," + " \"str\": \"hello\", \"cstrz\": \"world\", \"rstr\": \"raw\"," + " \"ptr\": [1], \"bad8\": 1, \"bad32\": \"x\"}"); + + static const nxt_conf_map_t map[] = { + nxt_conf_map_test_field(flag, NXT_CONF_MAP_INT8), + nxt_conf_map_test_field(i32, NXT_CONF_MAP_INT32), + nxt_conf_map_test_field(i64, NXT_CONF_MAP_INT64), + nxt_conf_map_test_field(i, NXT_CONF_MAP_INT), + nxt_conf_map_test_field(size, NXT_CONF_MAP_SIZE), + nxt_conf_map_test_field(off, NXT_CONF_MAP_OFF), + nxt_conf_map_test_field(msec, NXT_CONF_MAP_MSEC), + nxt_conf_map_test_field(dbl, NXT_CONF_MAP_DOUBLE), + nxt_conf_map_test_field(str, NXT_CONF_MAP_STR_COPY), + nxt_conf_map_test_field(cstrz, NXT_CONF_MAP_CSTRZ), + nxt_conf_map_test_field(rstr, NXT_CONF_MAP_STR), + nxt_conf_map_test_field(ptr, NXT_CONF_MAP_PTR), + /* Wrong JSON types: the fields keep their sentinels. */ + nxt_conf_map_test_field(bad8, NXT_CONF_MAP_INT8), + nxt_conf_map_test_field(bad32, NXT_CONF_MAP_INT32), + }; + + nxt_thread_time_update(thr); + + mp = nxt_mp_create(1024, 128, 256, 32); + if (mp == NULL) { + return NXT_ERROR; + } + + ret = NXT_ERROR; + nxt_memzero(&dst, sizeof(dst)); + dst.bad8 = 0xA5; + dst.bad32 = 0x5A5A5A5A; + + cv = nxt_conf_json_parse(mp, json.start, json.start + json.length, NULL); + + if (cv == NULL + || nxt_conf_map_object(mp, cv, map, nxt_nitems(map), &dst) != NXT_OK) + { + nxt_log_alert(thr->log, "nxt_conf_map_object() test failed"); + goto done; + } + + if (dst.flag != 1 + || dst.i32 != -123456 + || dst.i64 != -9876543210LL + || dst.i != -7 + || dst.size != 424242 + || dst.off != 131072 + || dst.msec != 3000 + || dst.dbl != 1.5 + || dst.str.length != 5 + || memcmp(dst.str.start, "hello", 5) != 0 + || dst.cstrz == NULL + || strcmp(dst.cstrz, "world") != 0 + || dst.rstr.length != 3 + || memcmp(dst.rstr.start, "raw", 3) != 0 + || dst.ptr == NULL + || nxt_conf_type(dst.ptr) != NXT_CONF_ARRAY + || dst.bad8 != 0xA5 + || dst.bad32 != 0x5A5A5A5A) + { + nxt_log_alert(thr->log, "nxt_conf_map_object() mapped wrong values"); + goto done; + } + + /* + * The non-scalar copies, read back with memcpy() from their misaligned + * fields. NXT_CONF_MAP_STR stores the member's own string, without a + * copy. NXT_CONF_MAP_STR_COPY stores a copy from the pool. + * NXT_CONF_MAP_PTR stores the member's nxt_conf_value_t itself. + */ + + nxt_memcpy(&rstr, &dst.rstr, sizeof(rstr)); + nxt_memcpy(&str, &dst.str, sizeof(str)); + nxt_memcpy(&ptr, &dst.ptr, sizeof(ptr)); + + member = nxt_conf_get_object_member(cv, &rstr_name, NULL); + nxt_conf_get_string(member, &orig); + + if (rstr.start != orig.start || rstr.length != orig.length) { + nxt_log_alert(thr->log, "nxt_conf_map_object() NXT_CONF_MAP_STR " + "did not store the member's own string"); + goto done; + } + + member = nxt_conf_get_object_member(cv, &str_name, NULL); + nxt_conf_get_string(member, &orig); + + if (str.start == orig.start + || str.length != orig.length + || memcmp(str.start, orig.start, orig.length) != 0) + { + nxt_log_alert(thr->log, "nxt_conf_map_object() NXT_CONF_MAP_STR_COPY " + "did not store a copy"); + goto done; + } + + if (ptr != nxt_conf_get_object_member(cv, &ptr_name, NULL)) { + nxt_log_alert(thr->log, "nxt_conf_map_object() NXT_CONF_MAP_PTR " + "did not store the member's value"); + goto done; + } + + ret = NXT_OK; + + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "nxt_conf_map_object() alignment test passed"); + +done: + + nxt_mp_destroy(mp); + + return ret; +} diff --git a/src/test/nxt_conn_close_idle_test.c b/src/test/nxt_conn_close_idle_test.c new file mode 100644 index 000000000..a46dd2e07 --- /dev/null +++ b/src/test/nxt_conn_close_idle_test.c @@ -0,0 +1,125 @@ +/* + * Copyright (C) FreeUnit contributors. + */ + +/* + * Regression test for M-5: nxt_conn_accept_close_idle() (nxt_conn_accept.c) + * used to call nxt_fd_event_disable_read() unconditionally. When the 100ms + * listen timer fires and the spare conn still cannot be allocated, this + * function runs again on a listener that is already inactive; disabling an + * inactive epoll read is what turns into an epoll_ctl() ENOENT alert every + * 100ms (see nxt_epoll_engine.c). The fix guards the call with + * nxt_fd_event_is_active(), matching nxt_router_listen_socket_close(). + * + * nxt_conn_accept_close_idle() itself is static; nxt_conn_accept_error() is + * the one entry point into it, for the EMFILE/ENFILE/ENOBUFS/ENOMEM cases. + */ + +#include +#include +#include +#include "nxt_tests.h" + + +static nxt_uint_t nxt_conn_close_idle_test_disable_calls; + + +static void +nxt_conn_close_idle_test_disable_read(nxt_event_engine_t *engine, + nxt_fd_event_t *ev) +{ + nxt_conn_close_idle_test_disable_calls++; + + /* What a real disable does: the read side becomes inactive. */ + ev->read = NXT_EVENT_INACTIVE; +} + + +nxt_int_t +nxt_conn_close_idle_test(nxt_thread_t *thr) +{ + nxt_int_t ret; + nxt_task_t *task; + nxt_listen_event_t lev; + nxt_event_engine_t engine, *saved_engine; + nxt_work_queue_cache_t cache; + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "conn close idle test started"); + + ret = NXT_ERROR; + + task = thr->task; + task->thread = thr; + + nxt_memzero(&engine, sizeof(nxt_event_engine_t)); + nxt_memzero(&lev, sizeof(nxt_listen_event_t)); + + if (nxt_slow_path(nxt_timers_init(&engine.timers, 4) != NXT_OK)) { + nxt_log_alert(thr->log, "conn close idle test failed to init " + "timers"); + return NXT_ERROR; + } + + engine.event.disable_read = nxt_conn_close_idle_test_disable_read; + + nxt_work_queue_cache_create(&cache, 1024); + + engine.close_work_queue.cache = &cache; + nxt_work_queue_name(&engine.close_work_queue, "close"); + + /* nxt_conn_accept_close_idle() reads the engine off task->thread. */ + saved_engine = thr->engine; + thr->engine = &engine; + + lev.socket.task = task; + lev.socket.log = thr->log; + lev.socket.read = NXT_EVENT_DEFAULT; /* armed, as a live listener is */ + lev.timer.task = task; + lev.timer.log = thr->log; + + nxt_conn_close_idle_test_disable_calls = 0; + + /* + * First round: the listener is active, so the guard must still let the + * disable through -- this is not a "never disable" fix. + */ + nxt_conn_accept_error(task, &lev, "test", NXT_ENOMEM); + + if (nxt_slow_path(nxt_conn_close_idle_test_disable_calls != 1)) { + nxt_log_alert(thr->log, "conn close idle test: an active listener " + "was not disabled (%ui calls)", + nxt_conn_close_idle_test_disable_calls); + goto done; + } + + /* + * Second round: the 100ms timer fired again and the spare conn still + * cannot be allocated, so this runs again on the now-inactive listener. + * Before the fix, disable_read() would be called again here. + */ + nxt_conn_accept_error(task, &lev, "test", NXT_ENOMEM); + + if (nxt_slow_path(nxt_conn_close_idle_test_disable_calls != 1)) { + nxt_log_alert(thr->log, "conn close idle test: disable_read() was " + "called on an already inactive listener (%ui calls " + "total) -- this is the epoll_ctl ENOENT storm", + nxt_conn_close_idle_test_disable_calls); + goto done; + } + + ret = NXT_OK; + +done: + + thr->engine = saved_engine; + + nxt_work_queue_cache_destroy(&cache); + nxt_free(engine.timers.changes); + + if (nxt_fast_path(ret == NXT_OK)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "conn close idle test passed"); + } + + return ret; +} diff --git a/src/test/nxt_conn_close_test.c b/src/test/nxt_conn_close_test.c new file mode 100644 index 000000000..203f024ca --- /dev/null +++ b/src/test/nxt_conn_close_test.c @@ -0,0 +1,553 @@ +/* + * Copyright (C) F5, Inc. + */ + +/* + * An I/O handler queued before nxt_conn_close() in the same engine cycle + * cannot be taken back, so it must be harmless. Each leg queues handlers + * on a real conn over a socketpair, closes, and drains the work queues in + * engine order. The fd is never registered, so the close is synchronous. + */ + +#include +#include "nxt_tests.h" + +#include +#include + + +#if (NXT_HAVE_EPOLL_EDGE) + +typedef struct { + nxt_event_engine_t *engine; + nxt_event_engine_t *saved_engine; + nxt_conn_t *c; + nxt_fd_t peer; + + nxt_uint_t released; + nxt_uint_t read_handlers; + nxt_uint_t null_handlers; + nxt_fd_t fd_at_release; +} nxt_conn_close_test_t; + + +static nxt_int_t nxt_conn_close_test_write(nxt_thread_t *thr); +static nxt_int_t nxt_conn_close_test_read(nxt_thread_t *thr); +static nxt_int_t nxt_conn_close_test_twice(nxt_thread_t *thr); +static nxt_int_t nxt_conn_close_test_race(nxt_thread_t *thr); +static nxt_int_t nxt_conn_close_test_setup(nxt_thread_t *thr, + nxt_conn_close_test_t *t); +static nxt_int_t nxt_conn_close_test_finish(nxt_thread_t *thr, + nxt_conn_close_test_t *t, nxt_int_t ret); +static void nxt_conn_close_test_drain(nxt_conn_close_test_t *t); +static nxt_bool_t nxt_conn_close_test_drain_wq(nxt_conn_close_test_t *t, + nxt_work_queue_t *wq); +static void nxt_conn_close_test_release(nxt_task_t *task, void *obj, + void *data); +static void nxt_conn_close_test_read_handler(nxt_task_t *task, void *obj, + void *data); + + +static nxt_conn_close_test_t nxt_conn_close_test_fixture; + + +/* Like every close state in the tree: no error_handler. */ + +static const nxt_conn_state_t nxt_conn_close_test_close_state + nxt_aligned(64) = +{ + .ready_handler = nxt_conn_close_test_release, +}; + + +static const nxt_conn_state_t nxt_conn_close_test_read_state + nxt_aligned(64) = +{ + .ready_handler = nxt_conn_close_test_read_handler, + .close_handler = nxt_conn_close_test_read_handler, + .error_handler = nxt_conn_close_test_read_handler, +}; + +#endif + + +nxt_int_t +nxt_conn_close_test(nxt_thread_t *thr) +{ +#if (NXT_HAVE_EPOLL_EDGE) + + nxt_int_t ret; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "conn close test started"); + + ret = nxt_conn_close_test_write(thr); + + if (nxt_conn_close_test_read(thr) != NXT_OK) { + ret = NXT_ERROR; + } + + if (nxt_conn_close_test_twice(thr) != NXT_OK) { + ret = NXT_ERROR; + } + + if (nxt_conn_close_test_race(thr) != NXT_OK) { + ret = NXT_ERROR; + } + + if (ret != NXT_OK) { + return NXT_ERROR; + } + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "conn close test passed"); + +#endif + + return NXT_OK; +} + + +#if (NXT_HAVE_EPOLL_EDGE) + +/* Queues as nxt_h1p_conn_init() and the fd dispatch set them. */ + +static nxt_int_t +nxt_conn_close_test_setup(nxt_thread_t *thr, nxt_conn_close_test_t *t) +{ + nxt_mp_t *mp; + nxt_fd_t pair[2]; + nxt_conn_t *c; + nxt_task_t *task; + nxt_event_engine_t *engine; + + task = thr->task; + task->thread = thr; + + nxt_memzero(t, sizeof(nxt_conn_close_test_t)); + t->peer = -1; + t->fd_at_release = -2; + + t->saved_engine = thr->engine; + + engine = nxt_event_engine_create(task, &nxt_epoll_edge_engine, NULL, 0, 0); + if (nxt_slow_path(engine == NULL)) { + return NXT_ERROR; + } + + t->engine = engine; + thr->engine = engine; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + c = nxt_conn_create(mp, task); + if (nxt_slow_path(c == NULL)) { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + t->c = c; + + if (nxt_slow_path(socketpair(AF_UNIX, SOCK_STREAM, 0, pair) != 0)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "conn close test: socketpair failed"); + return NXT_ERROR; + } + + if (nxt_slow_path(fcntl(pair[0], F_SETFL, O_NONBLOCK) == -1 + || fcntl(pair[1], F_SETFL, O_NONBLOCK) == -1)) + { + nxt_fd_close(pair[0]); + nxt_fd_close(pair[1]); + return NXT_ERROR; + } + + c->socket.fd = pair[0]; + t->peer = pair[1]; + + c->socket.read_ready = 1; + c->socket.write_ready = 1; + + c->read_work_queue = &engine->fast_work_queue; + c->write_work_queue = &engine->fast_work_queue; + c->socket.read_work_queue = &engine->read_work_queue; + c->socket.write_work_queue = &engine->write_work_queue; + + c->read = nxt_buf_mem_alloc(mp, 64, 0); + if (nxt_slow_path(c->read == NULL)) { + return NXT_ERROR; + } + + c->read_state = &nxt_conn_close_test_read_state; + c->write_state = &nxt_conn_close_test_close_state; + + return NXT_OK; +} + + +static nxt_int_t +nxt_conn_close_test_finish(nxt_thread_t *thr, nxt_conn_close_test_t *t, + nxt_int_t ret) +{ + nxt_conn_t *c; + + c = t->c; + + if (c != NULL) { + if (c->socket.fd != -1) { + nxt_fd_close(c->socket.fd); + c->socket.fd = -1; + } + + nxt_conn_free(thr->task, c); + } + + if (t->peer != -1) { + nxt_fd_close(t->peer); + } + + if (t->engine != NULL) { + nxt_event_engine_free(t->engine); + } + + thr->engine = t->saved_engine; + + return ret; +} + + +/* + * Drain in nxt_event_engine_start() order, firing close timers between + * passes. A NULL handler is counted instead of called. + */ + +static void +nxt_conn_close_test_drain(nxt_conn_close_test_t *t) +{ + nxt_bool_t ran; + nxt_event_engine_t *engine; + + engine = t->engine; + + for ( ;; ) { + do { + ran = nxt_conn_close_test_drain_wq(t, &engine->fast_work_queue) + || nxt_conn_close_test_drain_wq(t, &engine->accept_work_queue) + || nxt_conn_close_test_drain_wq(t, &engine->read_work_queue) + || nxt_conn_close_test_drain_wq(t, &engine->socket_work_queue) + || nxt_conn_close_test_drain_wq(t, + &engine->connect_work_queue) + || nxt_conn_close_test_drain_wq(t, &engine->write_work_queue) + || nxt_conn_close_test_drain_wq(t, + &engine->shutdown_work_queue) + || nxt_conn_close_test_drain_wq(t, &engine->close_work_queue); + } while (ran); + + nxt_timer_find(engine); + + if (nxt_rbtree_is_empty(&engine->timers.tree)) { + return; + } + + /* + * nxt_conn_close_handler() adds the close timer with 0 ms, so a + * 100 ms step fires it without advancing the clock. + */ + nxt_timer_expire(engine, engine->timers.now + 100); + + if (engine->fast_work_queue.head == NULL) { + return; + } + } +} + + +static nxt_bool_t +nxt_conn_close_test_drain_wq(nxt_conn_close_test_t *t, nxt_work_queue_t *wq) +{ + void *obj, *data; + nxt_task_t *task; + nxt_work_handler_t handler; + + if (wq->head == NULL) { + return 0; + } + + handler = nxt_work_queue_pop(wq, &task, &obj, &data); + + if (handler == NULL) { + t->null_handlers++; + return 1; + } + + handler(task, obj, data); + + return 1; +} + + +static void +nxt_conn_close_test_release(nxt_task_t *task, void *obj, void *data) +{ + nxt_conn_t *c; + nxt_conn_close_test_t *t; + + c = obj; + t = &nxt_conn_close_test_fixture; + + t->released++; + t->fd_at_release = c->socket.fd; +} + + +/* Like the h1proto read handlers, which end in nxt_h1p_shutdown(). */ + +static void +nxt_conn_close_test_read_handler(nxt_task_t *task, void *obj, void *data) +{ + nxt_conn_t *c; + nxt_conn_close_test_t *t; + + c = obj; + t = &nxt_conn_close_test_fixture; + + t->read_handlers++; + + nxt_conn_close(task->thread->engine, c); +} + + +/* + * A queued write, then block_write and close, as nxt_h1p_peer_close() and + * the h1p timeouts do: the close state's NULL error_handler must not run. + */ + +static nxt_int_t +nxt_conn_close_test_write(nxt_thread_t *thr) +{ + ssize_t n; + u_char byte; + nxt_conn_t *c; + nxt_conn_close_test_t *t; + + t = &nxt_conn_close_test_fixture; + + if (nxt_conn_close_test_setup(thr, t) != NXT_OK) { + return nxt_conn_close_test_finish(thr, t, NXT_ERROR); + } + + c = t->c; + + c->write = c->read; + + nxt_work_queue_add(c->socket.write_work_queue, nxt_conn_io_write, + c->socket.task, c, c->socket.data); + + c->block_write = 1; + + nxt_conn_close(t->engine, c); + + nxt_conn_close_test_drain(t); + + if (t->null_handlers != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "conn close test: write: a queued nxt_conn_io_write() " + "on a closed conn queued the NULL error_handler of the " + "close state (%ui times); the engine would call it", + t->null_handlers); + return nxt_conn_close_test_finish(thr, t, NXT_ERROR); + } + + if (t->released != 1 || t->fd_at_release != -1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "conn close test: write: release handler ran %ui " + "times (expected 1), fd at release %d (expected -1)", + t->released, (int) t->fd_at_release); + return nxt_conn_close_test_finish(thr, t, NXT_ERROR); + } + + /* Nothing was written; the peer sees only the FIN. */ + + n = recv(t->peer, &byte, 1, 0); + + if (n != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "conn close test: write: peer recv() %z, expected 0", + n); + return nxt_conn_close_test_finish(thr, t, NXT_ERROR); + } + + return nxt_conn_close_test_finish(thr, t, NXT_OK); +} + + +/* + * A queued read, then a close without block_read, as nxt_h1p_conn_close() + * does: nothing may be read and no read state handler may run. + */ + +static nxt_int_t +nxt_conn_close_test_read(nxt_thread_t *thr) +{ + nxt_conn_t *c; + nxt_conn_close_test_t *t; + + t = &nxt_conn_close_test_fixture; + + if (nxt_conn_close_test_setup(thr, t) != NXT_OK) { + return nxt_conn_close_test_finish(thr, t, NXT_ERROR); + } + + c = t->c; + + if (send(t->peer, "ping", 4, 0) != 4) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "conn close test: read: send() failed"); + return nxt_conn_close_test_finish(thr, t, NXT_ERROR); + } + + nxt_work_queue_add(c->socket.read_work_queue, nxt_conn_io_read, + c->socket.task, c, c->socket.data); + + nxt_conn_close(t->engine, c); + + nxt_conn_close_test_drain(t); + + if (t->read_handlers != 0 || c->nbytes != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "conn close test: read: a queued nxt_conn_io_read() " + "on a closed conn read %uD bytes and ran %ui read " + "state handlers (expected 0 and 0)", + c->nbytes, t->read_handlers); + return nxt_conn_close_test_finish(thr, t, NXT_ERROR); + } + + if (t->null_handlers != 0 || t->released != 1 || t->fd_at_release != -1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "conn close test: read: %ui NULL handlers, release " + "handler ran %ui times (expected 1), fd at release %d " + "(expected -1)", t->null_handlers, t->released, + (int) t->fd_at_release); + return nxt_conn_close_test_finish(thr, t, NXT_ERROR); + } + + return nxt_conn_close_test_finish(thr, t, NXT_OK); +} + + +/* Two closes in one cycle must release once. */ + +static nxt_int_t +nxt_conn_close_test_twice(nxt_thread_t *thr) +{ + nxt_conn_t *c; + nxt_conn_close_test_t *t; + + t = &nxt_conn_close_test_fixture; + + if (nxt_conn_close_test_setup(thr, t) != NXT_OK) { + return nxt_conn_close_test_finish(thr, t, NXT_ERROR); + } + + c = t->c; + + nxt_conn_close(t->engine, c); + nxt_conn_close(t->engine, c); + + nxt_conn_close_test_drain(t); + + if (t->released != 1 || t->fd_at_release != -1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "conn close test: twice: release handler ran %ui " + "times (expected 1), fd at release %d (expected -1)", + t->released, (int) t->fd_at_release); + return nxt_conn_close_test_finish(thr, t, NXT_ERROR); + } + + if (c->socket.fd != -1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "conn close test: twice: fd %d still open after the " + "close", (int) c->socket.fd); + return nxt_conn_close_test_finish(thr, t, NXT_ERROR); + } + + return nxt_conn_close_test_finish(thr, t, NXT_OK); +} + + +/* + * The full race: a read ran and queued its ready handler, a new read event + * is queued, then the conn is closed. The ready handler closes again (the + * second close must be a no-op) and the new read must not reach the FIN + * and run the close handler, which would close once more. + */ + +static nxt_int_t +nxt_conn_close_test_race(nxt_thread_t *thr) +{ + nxt_conn_t *c; + nxt_conn_close_test_t *t; + + t = &nxt_conn_close_test_fixture; + + if (nxt_conn_close_test_setup(thr, t) != NXT_OK) { + return nxt_conn_close_test_finish(thr, t, NXT_ERROR); + } + + c = t->c; + + if (send(t->peer, "ping", 4, 0) != 4 || shutdown(t->peer, SHUT_WR) != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "conn close test: race: send() or shutdown() failed"); + return nxt_conn_close_test_finish(thr, t, NXT_ERROR); + } + + /* The first read runs and queues the ready handler. */ + + nxt_work_queue_add(c->socket.read_work_queue, nxt_conn_io_read, + c->socket.task, c, c->socket.data); + + (void) nxt_conn_close_test_drain_wq(t, c->socket.read_work_queue); + + if (c->nbytes != 4 || c->read_work_queue->head == NULL) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "conn close test: race: the first read got %uD bytes " + "(expected 4) or queued no handler", c->nbytes); + return nxt_conn_close_test_finish(thr, t, NXT_ERROR); + } + + /* A new edge event: the FIN is readable. */ + + c->socket.read_ready = 1; + + nxt_work_queue_add(c->socket.read_work_queue, nxt_conn_io_read, + c->socket.task, c, c->socket.data); + + nxt_conn_close(t->engine, c); + + nxt_conn_close_test_drain(t); + + if (t->read_handlers != 1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "conn close test: race: %ui read state handlers ran " + "(expected 1); the queued read after the close reached " + "the socket", t->read_handlers); + return nxt_conn_close_test_finish(thr, t, NXT_ERROR); + } + + if (t->null_handlers != 0 || t->released != 1 || t->fd_at_release != -1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "conn close test: race: %ui NULL handlers, release " + "handler ran %ui times (expected 1), fd at release %d " + "(expected -1)", t->null_handlers, t->released, + (int) t->fd_at_release); + return nxt_conn_close_test_finish(thr, t, NXT_ERROR); + } + + return nxt_conn_close_test_finish(thr, t, NXT_OK); +} + +#endif diff --git a/src/test/nxt_conn_io_accept_test.c b/src/test/nxt_conn_io_accept_test.c new file mode 100644 index 000000000..44117d342 --- /dev/null +++ b/src/test/nxt_conn_io_accept_test.c @@ -0,0 +1,243 @@ +/* + * Copyright (C) FreeUnit contributors. + */ + +/* + * Regression test for M-4: when nxt_conn_io_accept() (nxt_conn_accept.c) + * cannot make an accepted socket non-blocking, it closes the socket and + * returns. The spare conn (lev->next) must then keep fd -1. Before the + * fix it kept the number of the closed socket, and nxt_conn_free() of the + * spare, at a listener release, asserts that the fd is -1. + * + * The test accepts a real TCP connection on 127.0.0.1 and makes the + * non-blocking step fail with the NXT_TESTS hook. It checks the spare fd, + * that the rejected socket was not passed to nxt_conn_accept(), and that + * the accepted socket was closed: the client reads end of file. + * + * Before the fix, the code went on into nxt_conn_accept(). The fixture + * engine, listen socket and work queue let that path run, so the test + * reports the failure instead of a crash. + */ + +#include +#include +#include +#include "nxt_tests.h" + + +static void +nxt_conn_io_accept_test_handler(nxt_task_t *task, void *obj, void *data) +{ +} + + +/* + * Before the fix, the code went on into nxt_conn_accept() and then into + * nxt_conn_accept_close_idle(), which can disable the listener. With this + * handler the M-4 test does not depend on the M-5 fix. + */ + +static void +nxt_conn_io_accept_test_disable_read(nxt_event_engine_t *engine, + nxt_fd_event_t *ev) +{ + ev->read = NXT_EVENT_INACTIVE; +} + + +nxt_int_t +nxt_conn_io_accept_test(nxt_thread_t *thr) +{ +#if (NXT_LINUX) + int lfd, cfd; + char ch; + void *obj, *data; + size_t size; + ssize_t n; + nxt_int_t ret; + socklen_t len; + nxt_conn_t c; + nxt_task_t *t; + nxt_sockaddr_t *remote; + nxt_work_queue_t wq; + nxt_event_engine_t engine, *saved_engine; + nxt_work_handler_t handler; + nxt_listen_event_t lev; + struct sockaddr_in sin; + nxt_listen_socket_t ls; + nxt_work_queue_cache_t cache; + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "conn io accept test started"); + + ret = NXT_ERROR; + cfd = -1; + remote = NULL; + + lfd = socket(AF_INET, SOCK_STREAM, 0); + if (lfd == -1) { + nxt_log_alert(thr->log, "conn io accept test: socket() failed %E", + nxt_errno); + return NXT_ERROR; + } + + /* + * The fixture for nxt_conn_accept(): an engine at max_connections, so + * no new spare conn is allocated, a listen socket with a handler that + * does nothing, and a work queue. + */ + nxt_memzero(&engine, sizeof(nxt_event_engine_t)); + + if (nxt_timers_init(&engine.timers, 4) != NXT_OK) { + (void) close(lfd); + return NXT_ERROR; + } + + engine.task = *thr->task; + engine.event.disable_read = nxt_conn_io_accept_test_disable_read; + nxt_queue_init(&engine.idle_connections); + nxt_queue_init(&engine.active_connections); + + nxt_work_queue_cache_create(&cache, 0); + + engine.close_work_queue.cache = &cache; + + nxt_memzero(&wq, sizeof(nxt_work_queue_t)); + wq.cache = &cache; + + nxt_memzero(&ls, sizeof(nxt_listen_socket_t)); + ls.handler = nxt_conn_io_accept_test_handler; + + saved_engine = thr->engine; + thr->engine = &engine; + + /* The cleanup at "done" reads c and lev, also after an early failure. */ + + nxt_memzero(&c, sizeof(nxt_conn_t)); + c.socket.fd = -1; + + nxt_memzero(&lev, sizeof(nxt_listen_event_t)); + lev.timer.task = &engine.task; + lev.timer.log = thr->log; + + nxt_memzero(&sin, sizeof(struct sockaddr_in)); + sin.sin_family = AF_INET; + sin.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + + len = sizeof(struct sockaddr_in); + + if (bind(lfd, (struct sockaddr *) &sin, len) != 0 + || listen(lfd, 1) != 0 + || getsockname(lfd, (struct sockaddr *) &sin, &len) != 0) + { + nxt_log_alert(thr->log, "conn io accept test: listen failed %E", + nxt_errno); + goto done; + } + + cfd = socket(AF_INET, SOCK_STREAM, 0); + if (cfd == -1 + || connect(cfd, (struct sockaddr *) &sin, len) != 0) + { + nxt_log_alert(thr->log, "conn io accept test: connect failed %E", + nxt_errno); + goto done; + } + + /* Room for the address text that nxt_conn_accept() writes. */ + size = offsetof(nxt_sockaddr_t, u) + sizeof(struct sockaddr_in) + + NXT_INET_ADDR_STR_LEN; + + remote = nxt_zalloc(size); + if (remote == NULL) { + goto done; + } + + remote->socklen = sizeof(struct sockaddr_in); + remote->length = NXT_INET_ADDR_STR_LEN; + + c.remote = remote; + + lev.socket.fd = lfd; + lev.listen = &ls; + lev.work_queue = &wq; + lev.next = &c; + lev.ready = 1; + + nxt_conn_test_nonblocking_failures = 1; + + nxt_conn_io_accept(thr->task, &lev, NULL); + + if (nxt_conn_test_nonblocking_failures != 0) { + nxt_log_alert(thr->log, "conn io accept test failed: the " + "non-blocking step was not reached"); + goto done; + } + + if (wq.head != NULL || c.listen != NULL) { + nxt_log_alert(thr->log, "conn io accept test failed: the rejected " + "socket was passed to nxt_conn_accept()"); + goto done; + } + + if (c.socket.fd != -1) { + nxt_log_alert(thr->log, "conn io accept test failed: the spare conn " + "kept fd %d of a closed socket", c.socket.fd); + goto done; + } + + /* The accepted socket is closed: the client reads end of file. */ + + n = read(cfd, &ch, 1); + if (n != 0) { + nxt_log_alert(thr->log, "conn io accept test failed: the accepted " + "socket was not closed (read %z)", n); + goto done; + } + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "conn io accept test passed"); + + ret = NXT_OK; + +done: + + nxt_conn_test_nonblocking_failures = 0; + + /* Undo what nxt_conn_accept() did, if it ran. */ + + nxt_conn_untrack(&engine, (&c)); + + nxt_timer_delete(&engine, &lev.timer); + + while (wq.head != NULL) { + (void) nxt_work_queue_pop(&wq, &t, &obj, &data); + } + + while (engine.close_work_queue.head != NULL) { + handler = nxt_work_queue_pop(&engine.close_work_queue, &t, &obj, + &data); + handler(t, obj, data); + } + + thr->engine = saved_engine; + + nxt_work_queue_cache_destroy(&cache); + nxt_free(engine.timers.changes); + + if (remote != NULL) { + nxt_free(remote); + } + + if (cfd != -1) { + (void) close(cfd); + } + + (void) close(lfd); + + return ret; + +#else + + return NXT_OK; + +#endif +} diff --git a/src/test/nxt_controller_peer_test.c b/src/test/nxt_controller_peer_test.c new file mode 100644 index 000000000..b32c3f1a0 --- /dev/null +++ b/src/test/nxt_controller_peer_test.c @@ -0,0 +1,80 @@ +/* + * Copyright (C) FreeUnit + */ + +#include +#include +#include "nxt_tests.h" + + +#define NONE_UID ((nxt_uid_t) -1) +#define NONE_GID ((nxt_gid_t) -1) + + +typedef struct { + nxt_uid_t uid; + nxt_gid_t gid; + const nxt_gid_t *groups; + nxt_uint_t ngroups; + nxt_uid_t euid; + nxt_uid_t ctl_uid; + nxt_gid_t ctl_gid; + nxt_bool_t allowed; +} nxt_controller_peer_test_t; + + +static const nxt_gid_t groups_with_ctl[] = { 10, 20, 500 }; +static const nxt_gid_t groups_without[] = { 10, 20 }; + + +static const nxt_controller_peer_test_t tests[] = { + /* root and unitd's own euid are always allowed */ + { 0, 0, NULL, 0, 1000, NONE_UID, NONE_GID, 1 }, + { 1000, 1000, NULL, 0, 1000, NONE_UID, NONE_GID, 1 }, + + /* unrelated uid without delegation is rejected */ + { 1001, 1001, NULL, 0, 0, NONE_UID, NONE_GID, 0 }, + + /* --control-user alice (uid 1001) with unitd running as root */ + { 1001, 1001, NULL, 0, 0, 1001, NONE_GID, 1 }, + { 1002, 1002, NULL, 0, 0, 1001, NONE_GID, 0 }, + + /* --control-group (gid 500): primary gid match */ + { 1002, 500, NULL, 0, 0, NONE_UID, 500, 1 }, + + /* --control-group: supplementary group match */ + { 1002, 1002, groups_with_ctl, 3, 0, NONE_UID, 500, 1 }, + { 1002, 1002, groups_without, 2, 0, NONE_UID, 500, 0 }, + + /* supplementary groups are ignored without --control-group */ + { 1002, 1002, groups_with_ctl, 3, 0, NONE_UID, NONE_GID, 0 }, +}; + + +nxt_int_t +nxt_controller_peer_test(nxt_thread_t *thr) +{ + nxt_uint_t i; + nxt_bool_t allowed; + const nxt_controller_peer_test_t *t; + + for (i = 0; i < nxt_nitems(tests); i++) { + t = &tests[i]; + + allowed = nxt_controller_peer_allowed(t->uid, t->gid, t->groups, + t->ngroups, t->euid, + t->ctl_uid, t->ctl_gid); + + if (allowed != t->allowed) { + nxt_log_alert(thr->log, "nxt_controller_peer_allowed() test #%ui " + "failed: got %d, expected %d", + i, (int) allowed, (int) t->allowed); + return NXT_ERROR; + } + } + + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "nxt_controller_peer_allowed() test passed"); + + return NXT_OK; +} diff --git a/src/test/nxt_cpu_limit_test.c b/src/test/nxt_cpu_limit_test.c new file mode 100644 index 000000000..9640c8253 --- /dev/null +++ b/src/test/nxt_cpu_limit_test.c @@ -0,0 +1,129 @@ + +/* + * Copyright (C) FreeUnit + */ + +#include +#include "nxt_tests.h" + + +typedef struct { + const char *in; + nxt_uint_t cpus; +} nxt_cpu_limit_case_t; + + +typedef struct { + const char *in; + const char *out; +} nxt_cpu_limit_str_case_t; + + +typedef struct { + const char *cgroup; + const char *root; + const char *rel; +} nxt_cpu_limit_path_case_t; + + +nxt_int_t +nxt_cpu_limit_test(nxt_thread_t *thr) +{ + nxt_uint_t i, cpus; + nxt_bool_t ok; + const char *rel; + char buf[32]; + + static const nxt_cpu_limit_case_t cases[] = { + { "max 100000\n", 0 }, + { "200000 100000\n", 2 }, + { "150000 100000\n", 2 }, + { "50000 100000\n", 1 }, + { "1000 1000000\n", 1 }, + { "6400000 100000", 64 }, + { "100000 0\n", 0 }, + { "0 100000\n", 0 }, + { "-1 100000\n", 0 }, + { "100000\n", 0 }, + { "100000 \n", 0 }, + { "\n", 0 }, + { "", 0 }, + { "1x0000 100000\n", 0 }, + { "9223372036854775807 1\n", 0 }, + { "100001 100000\n", 2 }, + { "2147483647 1\n", 2147483647 }, + { "2147483648 1\n", 0 }, + { "200000 100000 \n \n", 2 }, + { " 200000 100000\n", 0 }, + { "200000 100000\n", 0 }, + { "200000 100000 1\n", 0 }, + { "200000\n100000\n", 0 }, + { " \n", 0 }, + }; + + static const nxt_cpu_limit_str_case_t unescape[] = { + { "/sys/fs/cgroup", "/sys/fs/cgroup" }, + { "/mnt/a\\040b", "/mnt/a b" }, + { "\\011\\012\\134\\040", "\t\n\\ " }, + { "/a\\04", "/a\\04" }, + { "/a\\0x0", "/a\\0x0" }, + { "/a\\400", "/a\\400" }, + { "/a\\", "/a\\" }, + { "", "" }, + }; + + static const nxt_cpu_limit_path_case_t paths[] = { + { "/", "/", "" }, + { "/a/b", "/", "/a/b" }, + { "/a/b", "/a", "/b" }, + { "/a", "/a", "" }, + { "/ab", "/a", NULL }, + { "/b", "/a", NULL }, + { "/..", "/", NULL }, + { "/../b", "/", NULL }, + { "/a/..", "/", NULL }, + { "/a/../b", "/", NULL }, + { "/..a", "/", "/..a" }, + { "/a/..b/c", "/", "/a/..b/c" }, + { "/a..", "/", "/a.." }, + { "/a/b..c", "/a", "/b..c" }, + }; + + for (i = 0; i < nxt_nitems(cases); i++) { + cpus = nxt_cgroup_cpu_max_parse((const u_char *) cases[i].in, + strlen(cases[i].in)); + + NXT_TEST_CHECK(thr->log, cpus == cases[i].cpus, + "cpu.max \"%s\": %ui CPUs, expected %ui", + cases[i].in, cpus, cases[i].cpus); + } + + for (i = 0; i < nxt_nitems(unescape); i++) { + nxt_cpystrn((u_char *) buf, (const u_char *) unescape[i].in, + sizeof(buf)); + + nxt_cgroup_mountinfo_unescape(buf); + + NXT_TEST_CHECK(thr->log, strcmp(buf, unescape[i].out) == 0, + "mountinfo \"%s\": \"%s\", expected \"%s\"", + unescape[i].in, buf, unescape[i].out); + } + + for (i = 0; i < nxt_nitems(paths); i++) { + rel = nxt_cgroup_relative_path(paths[i].cgroup, paths[i].root); + + ok = (rel == NULL || paths[i].rel == NULL) + ? rel == paths[i].rel + : strcmp(rel, paths[i].rel) == 0; + + NXT_TEST_CHECK(thr->log, ok, + "cgroup \"%s\" below \"%s\": \"%s\", expected \"%s\"", + paths[i].cgroup, paths[i].root, + (rel != NULL) ? rel : "NULL", + (paths[i].rel != NULL) ? paths[i].rel : "NULL"); + } + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "cpu limit test passed"); + + return NXT_OK; +} diff --git a/src/test/nxt_fd_event_change_test.c b/src/test/nxt_fd_event_change_test.c new file mode 100644 index 000000000..89417c760 --- /dev/null +++ b/src/test/nxt_fd_event_change_test.c @@ -0,0 +1,576 @@ +/* + * Copyright (C) NGINX, Inc. + */ + +#include +#include +#include +#include "nxt_tests.h" + + +/* + * A queued fd-event change is held by pointer in the engine's change batch + * and dereferenced when that batch is committed, at the top of the next poll + * at the latest. Nothing else keeps the struct the pointer names alive, so + * anything that frees such a struct has to drop its pending changes first -- + * nxt_fd_event_cancel_changes(). + * + * These legs run against the platform's real event engine, not a stub: the + * batch and its commit are the thing under test. Each asserts on the + * engine's change count, so a leg that stops doing what it claims fails + * rather than passing quietly. Under a sanitized build the final poll in + * each leg is the second oracle: it commits whatever is left, so a change + * that was not dropped is reported as a use-after-free. + */ + +/* + * ->changing is not a cross-engine contract. The epoll engine keeps it and + * clears it when it commits, so the legs below can assert on it there. The + * kqueue engine does not keep it at all -- its paths work by descriptor or + * by ->udata -- so on kqueue the change count is the only oracle. + */ + +#if (NXT_HAVE_EPOLL) +#define nxt_fd_event_change_test_nchanges(engine) \ + ((nxt_uint_t) (engine)->u.epoll.nchanges) +#define NXT_FD_EVENT_CHANGE_TEST 1 +#define NXT_FD_EVENT_CHANGE_TEST_CHANGING 1 + +#elif (NXT_HAVE_KQUEUE) +#define nxt_fd_event_change_test_nchanges(engine) \ + ((nxt_uint_t) (engine)->u.kqueue.nchanges) +#define NXT_FD_EVENT_CHANGE_TEST 1 +#define NXT_FD_EVENT_CHANGE_TEST_CHANGING 0 + +#else +#define NXT_FD_EVENT_CHANGE_TEST 0 +#define NXT_FD_EVENT_CHANGE_TEST_CHANGING 0 +#endif + + +#if (NXT_FD_EVENT_CHANGE_TEST) + +static nxt_event_engine_t *nxt_fd_event_change_test_engine(nxt_task_t *task); +static nxt_int_t nxt_fd_event_change_test_drop(nxt_thread_t *thr, + nxt_event_engine_t *engine); +static nxt_int_t nxt_fd_event_change_test_keeps_others(nxt_thread_t *thr, + nxt_event_engine_t *engine); +static nxt_int_t nxt_fd_event_change_test_port(nxt_thread_t *thr, + nxt_event_engine_t *engine); +static nxt_fd_event_t *nxt_fd_event_change_test_event(nxt_task_t *task, + nxt_fd_t fd); +static void nxt_fd_event_change_test_handler(nxt_task_t *task, void *obj, + void *data); + + +nxt_int_t +nxt_fd_event_change_test(nxt_thread_t *thr) +{ + nxt_int_t ret; + nxt_task_t *task; + nxt_event_engine_t *engine, *saved_engine; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "fd event change test started"); + + task = thr->task; + task->thread = thr; + + engine = nxt_fd_event_change_test_engine(task); + if (nxt_slow_path(engine == NULL)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "fd event change test: no event engine"); + return NXT_ERROR; + } + + saved_engine = thr->engine; + thr->engine = engine; + + /* + * Without eventfd (epoll) or EVFILT_USER (kqueue) the engine posts + * through a signal pipe, and nxt_event_engine_create() enables its read + * event in this same batch. Commit it, so that every leg counts only + * the changes it queued itself. + */ + + engine->event.poll(engine, 0); + + if (nxt_slow_path(nxt_fd_event_change_test_nchanges(engine) != 0)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "fd event change test: the engine's own changes were " + "not committed, %ui left", + nxt_fd_event_change_test_nchanges(engine)); + ret = NXT_ERROR; + goto done; + } + + ret = nxt_fd_event_change_test_drop(thr, engine); + + if (ret == NXT_OK) { + ret = nxt_fd_event_change_test_keeps_others(thr, engine); + } + + if (ret == NXT_OK) { + ret = nxt_fd_event_change_test_port(thr, engine); + } + +done: + + thr->engine = saved_engine; + + nxt_event_engine_free(engine); + + if (ret != NXT_OK) { + return NXT_ERROR; + } + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "fd event change test passed"); + + return NXT_OK; +} + + +static nxt_event_engine_t * +nxt_fd_event_change_test_engine(nxt_task_t *task) +{ + nxt_mp_t *mp; + nxt_array_t *services; + nxt_event_engine_t *engine; + const nxt_event_interface_t *interface; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NULL; + } + + services = nxt_services_init(mp); + if (nxt_slow_path(services == NULL)) { + nxt_mp_destroy(mp); + return NULL; + } + + interface = nxt_service_get(services, "engine", NULL); + if (nxt_slow_path(interface == NULL)) { + nxt_mp_destroy(mp); + return NULL; + } + + /* + * No signal set: this fixture drives the engine by hand and never runs + * nxt_event_engine_start(), so nothing here delivers a signal. + */ + + engine = nxt_event_engine_create(task, interface, NULL, 0, 0); + + nxt_mp_destroy(mp); + + return engine; +} + + +static nxt_fd_event_t * +nxt_fd_event_change_test_event(nxt_task_t *task, nxt_fd_t fd) +{ + nxt_fd_event_t *ev; + + /* + * nxt_malloc(), not the mem pool a port uses, so that the poll at the + * end of the leg reads freed memory outright under a sanitizer rather + * than memory a pool happens to keep mapped. + */ + + ev = nxt_zalloc(sizeof(nxt_fd_event_t)); + if (nxt_slow_path(ev == NULL)) { + return NULL; + } + + ev->fd = fd; + ev->task = task; + ev->log = task->log; + ev->read_work_queue = &task->thread->engine->fast_work_queue; + ev->write_work_queue = &task->thread->engine->fast_work_queue; + ev->read_handler = nxt_fd_event_change_test_handler; + ev->write_handler = nxt_fd_event_change_test_handler; + ev->error_handler = nxt_fd_event_change_test_handler; + + return ev; +} + + +static void +nxt_fd_event_change_test_handler(nxt_task_t *task, void *obj, void *data) +{ +} + + +/* + * The bare mechanism: a change queued for an event, then dropped, leaves the + * batch empty, so freeing the event afterwards cannot be reached by a commit. + */ + +static nxt_int_t +nxt_fd_event_change_test_drop(nxt_thread_t *thr, nxt_event_engine_t *engine) +{ + nxt_int_t ret; + nxt_fd_event_t *ev; + nxt_socket_t pair[2]; + + ret = NXT_ERROR; + ev = NULL; + pair[0] = -1; + pair[1] = -1; + + if (nxt_slow_path(socketpair(AF_UNIX, SOCK_DGRAM, 0, pair) != 0)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "fd event change test: socketpair failed"); + return NXT_ERROR; + } + + ev = nxt_fd_event_change_test_event(thr->task, pair[1]); + if (nxt_slow_path(ev == NULL)) { + goto done; + } + + nxt_fd_event_enable_write(engine, ev); + + if (nxt_slow_path(nxt_fd_event_change_test_nchanges(engine) != 1)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "fd event change test: enabling the write event queued " + "%ui changes, expected 1", + nxt_fd_event_change_test_nchanges(engine)); + goto done; + } + +#if (NXT_FD_EVENT_CHANGE_TEST_CHANGING) + if (nxt_slow_path(ev->changing != 1)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "fd event change test: a queued change left changing:%d", + (int) ev->changing); + goto done; + } +#endif + + nxt_fd_event_cancel_changes(engine, ev); + + if (nxt_slow_path(nxt_fd_event_change_test_nchanges(engine) != 0)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "fd event change test: cancelling left %ui changes", + nxt_fd_event_change_test_nchanges(engine)); + goto done; + } + +#if (NXT_FD_EVENT_CHANGE_TEST_CHANGING) + if (nxt_slow_path(ev->changing != 0)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "fd event change test: cancelling left changing:%d", + (int) ev->changing); + goto done; + } +#endif + + nxt_free(ev); + ev = NULL; + + /* Commits whatever is left. Nothing is, so nothing reads the free. */ + + engine->event.poll(engine, 0); + + ret = NXT_OK; + +done: + + nxt_free(ev); + + if (pair[0] != -1) { + nxt_socket_close(thr->task, pair[0]); + } + + if (pair[1] != -1) { + nxt_socket_close(thr->task, pair[1]); + } + + return ret; +} + + +/* + * Cancelling one event's change leaves every other event's change in the + * batch, and still commits it. The compaction is the part of this that can + * silently go wrong. + */ + +static nxt_int_t +nxt_fd_event_change_test_keeps_others(nxt_thread_t *thr, + nxt_event_engine_t *engine) +{ + nxt_int_t ret; + nxt_uint_t i; + nxt_fd_event_t *ev[3]; + nxt_socket_t pair[3][2]; + + ret = NXT_ERROR; + + for (i = 0; i < 3; i++) { + ev[i] = NULL; + pair[i][0] = -1; + pair[i][1] = -1; + } + + for (i = 0; i < 3; i++) { + if (nxt_slow_path(socketpair(AF_UNIX, SOCK_DGRAM, 0, pair[i]) != 0)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "fd event change test: socketpair failed"); + goto done; + } + + ev[i] = nxt_fd_event_change_test_event(thr->task, pair[i][1]); + if (nxt_slow_path(ev[i] == NULL)) { + goto done; + } + + nxt_fd_event_enable_write(engine, ev[i]); + } + + if (nxt_slow_path(nxt_fd_event_change_test_nchanges(engine) != 3)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "fd event change test: three events queued %ui changes", + nxt_fd_event_change_test_nchanges(engine)); + goto done; + } + + /* The middle one, so that compaction has to move a tail entry. */ + + nxt_fd_event_cancel_changes(engine, ev[1]); + + if (nxt_slow_path(nxt_fd_event_change_test_nchanges(engine) != 2)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "fd event change test: cancelling one of three left %ui " + "changes, expected 2", + nxt_fd_event_change_test_nchanges(engine)); + goto done; + } + +#if (NXT_FD_EVENT_CHANGE_TEST_CHANGING) + if (nxt_slow_path(ev[0]->changing != 1 || ev[2]->changing != 1)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "fd event change test: cancelling one event cleared " + "changing on another (%d, %d)", + (int) ev[0]->changing, (int) ev[2]->changing); + goto done; + } +#endif + + nxt_free(ev[1]); + ev[1] = NULL; + + /* + * Commits the two that were kept. Both are still allocated, and both + * descriptors are still open, so this is the check that compaction left + * usable entries rather than merely the right count. + * + * Both are writable, so the poll also queues their handlers on + * fast_work_queue with ev[0] and ev[2] as the object. Nothing runs that + * queue before nxt_event_engine_free() drops it; a leg that does run it + * has to do so before the events are freed below. + */ + + engine->event.poll(engine, 0); + + if (nxt_slow_path(nxt_fd_event_change_test_nchanges(engine) != 0)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "fd event change test: the poll left %ui changes", + nxt_fd_event_change_test_nchanges(engine)); + goto done; + } + +#if (NXT_FD_EVENT_CHANGE_TEST_CHANGING) + if (nxt_slow_path(ev[0]->changing != 0 || ev[2]->changing != 0)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "fd event change test: a committed change left " + "changing set (%d, %d)", + (int) ev[0]->changing, (int) ev[2]->changing); + goto done; + } +#endif + +#if (NXT_HAVE_EPOLL) + /* + * The count above says two entries survived; this says they are the + * right two. A swapped entry commits silently and keeps the count, so + * ask the kernel instead: the kept descriptors are in the epoll set and + * EPOLL_CTL_MOD finds them, the cancelled one was never added and + * EPOLL_CTL_MOD reports ENOENT. + */ + + { + int err; + nxt_err_t errn; + nxt_uint_t k; + struct epoll_event ee; + + for (k = 0; k < 3; k++) { + nxt_memzero(&ee, sizeof(struct epoll_event)); + ee.events = EPOLLOUT; + ee.data.ptr = ev[k]; + + err = epoll_ctl(engine->u.epoll.fd, EPOLL_CTL_MOD, pair[k][1], + &ee); + errn = (err == 0) ? 0 : nxt_errno; + + if (k == 1) { + if (nxt_slow_path(err == 0 || errn != NXT_ENOENT)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "fd event change test: the cancelled event " + "reached the epoll set (fd:%d) %E", + pair[k][1], errn); + goto done; + } + + } else if (nxt_slow_path(err != 0)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "fd event change test: a kept event is not in " + "the epoll set (fd:%d) %E", + pair[k][1], errn); + goto done; + } + } + } +#endif + + ret = NXT_OK; + +done: + + for (i = 0; i < 3; i++) { + nxt_free(ev[i]); + + if (pair[i][0] != -1) { + nxt_socket_close(thr->task, pair[i][0]); + } + + if (pair[i][1] != -1) { + nxt_socket_close(thr->task, pair[i][1]); + } + } + + return ret; +} + + +/* + * The reachable case. nxt_port_rearm_now() arms the write event, and + * nxt_port_write_msgs() calls it immediately before the nxt_port_use() that + * can take the count to zero -- which releases the port's memory pool, and + * port->socket lives in it. This leg is that pair of calls with nothing in + * between: arm, then drop the last reference. + * + * Before the fix the poll below commits a change into freed memory. Under a + * sanitized build that is a heap-use-after-free report; in a plain build it + * is silent, which is why the change count is asserted as well. + */ + +static nxt_int_t +nxt_fd_event_change_test_port(nxt_thread_t *thr, nxt_event_engine_t *engine) +{ + nxt_int_t ret; + nxt_task_t *task; + nxt_port_t *port; + nxt_socket_t pair[2]; + + ret = NXT_ERROR; + task = thr->task; + + if (nxt_slow_path(socketpair(AF_UNIX, SOCK_DGRAM, 0, pair) != 0)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "fd event change test: socketpair failed"); + return NXT_ERROR; + } + + port = nxt_port_new(task, 1, nxt_pid, NXT_PROCESS_MAIN); + if (nxt_slow_path(port == NULL)) { + nxt_socket_close(task, pair[0]); + nxt_socket_close(task, pair[1]); + return NXT_ERROR; + } + + port->pair[0] = pair[0]; + port->pair[1] = pair[1]; + port->socket.task = task; + + nxt_port_write_enable(task, port); + + if (nxt_slow_path(port->engine != engine)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "fd event change test: the port took another engine"); + goto done; + } + + /* Same engine, so this arms the write event inline. */ + + nxt_port_rearm(task, port); + + if (nxt_slow_path(nxt_fd_event_change_test_nchanges(engine) != 1)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "fd event change test: re-arming the port queued %ui " + "changes, expected 1", + nxt_fd_event_change_test_nchanges(engine)); + goto done; + } + + ret = NXT_OK; + +done: + + /* + * What nxt_port_close() does to the descriptors, which every release + * path runs first -- nxt_port_mp_cleanup() asserts on it. port->socket + * keeps the descriptor number it had, so a change left in the batch now + * names a number the next open() may hand to somebody else. + */ + + nxt_socket_close(task, port->pair[0]); + port->pair[0] = -1; + + nxt_socket_close(task, port->pair[1]); + port->pair[1] = -1; + + /* + * The last reference. nxt_port_release() runs from here and releases + * port->mem_pool, which is where port->socket lives. + */ + + nxt_port_use(task, port, -1); + + if (ret != NXT_OK) { + return ret; + } + + if (nxt_slow_path(nxt_fd_event_change_test_nchanges(engine) != 0)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "fd event change test: releasing the port left %ui " + "changes pointing into its freed memory pool", + nxt_fd_event_change_test_nchanges(engine)); + return NXT_ERROR; + } + + /* Would commit that change, if one were left. */ + + engine->event.poll(engine, 0); + + return NXT_OK; +} + +#else /* !NXT_FD_EVENT_CHANGE_TEST */ + +nxt_int_t +nxt_fd_event_change_test(nxt_thread_t *thr) +{ + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "fd event change test skipped: the change batch is asserted " + "on only for epoll and kqueue"); + + return NXT_OK; +} + +#endif diff --git a/src/test/nxt_gmtime_test.c b/src/test/nxt_gmtime_test.c index cec81ab45..5f9fda0c9 100644 --- a/src/test/nxt_gmtime_test.c +++ b/src/test/nxt_gmtime_test.c @@ -22,20 +22,102 @@ #endif +/* + * Pre-epoch expectations, checked against fixed calendar fields rather than + * against gmtime(). Do not "simplify" these into the differential loops + * below: handing a nxt_time_t to gmtime(), which takes a time_t, is only + * safe where the two types agree, and for a negative value they need not. + * Where nxt_time_t is 8 bytes against a 4-byte time_t -- QNX before SDP + * 8.0, whose native time_t is uint32_t, and the case the NXT_TIME_T_SIZE + * test above already contemplates -- the library reads half the object, + * the low half on a little-endian machine, which often looks right by + * accident, and the high half on a big-endian one, which is garbage. + * Fixed fields test the arithmetic on every platform instead. + * + * For the same reason the loops below never hand gmtime() the address of + * the nxt_time_t itself: they convert into a real time_t first, and skip + * the values that do not survive the conversion. + * + * A file's mtime can be negative ("touch -d 1969-07-20"), and the day-time + * step used to wrap and render an hour of 1193046. -432000 and beyond + * matter on their own: a weekday taken from an unsigned day number is still + * right for the first four days before the epoch and wrong after them. + * + * Rows derived with "date -u -d @" on a signed 64-bit time_t host, + * never from nxt_gmtime() itself. + */ + +static const struct { + nxt_time_t time; + int year, mon, mday, wday, yday, hour, min, sec; +} nxt_gmtime_pre_epoch[] = { + { -1, 69, 11, 31, 3, 364, 23, 59, 59 }, /* Wed, 31 Dec 1969 */ + { -86400, 69, 11, 31, 3, 364, 0, 0, 0 }, /* Wed, 31 Dec 1969 */ + { -432000, 69, 11, 27, 6, 360, 0, 0, 0 }, /* Sat, 27 Dec 1969 */ + { -432001, 69, 11, 26, 5, 359, 23, 59, 59 }, /* Fri, 26 Dec 1969 */ + { -14182940, 69, 6, 20, 0, 200, 20, 17, 40 }, /* Sun, 20 Jul 1969 */ + { -2147472000, 1, 11, 14, 6, 347, 0, 0, 0 }, /* Sat, 14 Dec 1901 */ +#if (NXT_TIME_T_SIZE == 8) + /* The oldest date the Gauss' formula supports. */ + { -62135596800, -1899, 0, 1, 1, 0, 0, 0, 0 }, /* Mon, 01 Jan 0001 */ +#endif +}; + + nxt_int_t nxt_gmtime_test(nxt_thread_t *thr) { + time_t native; struct tm tm0, *tm1; + nxt_uint_t i; nxt_time_t s; nxt_nsec_t start, end; nxt_thread_time_update(thr); nxt_log_error(NXT_LOG_NOTICE, thr->log, "gmtime test started"); + for (i = 0; i < nxt_nitems(nxt_gmtime_pre_epoch); i++) { + + s = nxt_gmtime_pre_epoch[i].time; + + nxt_gmtime(s, &tm0); + + if (tm0.tm_year != nxt_gmtime_pre_epoch[i].year + || tm0.tm_mon != nxt_gmtime_pre_epoch[i].mon + || tm0.tm_mday != nxt_gmtime_pre_epoch[i].mday + || tm0.tm_wday != nxt_gmtime_pre_epoch[i].wday + || tm0.tm_yday != nxt_gmtime_pre_epoch[i].yday + || tm0.tm_hour != nxt_gmtime_pre_epoch[i].hour + || tm0.tm_min != nxt_gmtime_pre_epoch[i].min + || tm0.tm_sec != nxt_gmtime_pre_epoch[i].sec) + { + nxt_log_alert(thr->log, + "gmtime test failed: %T @ wday %d, " + "%02d.%02d.%d %02d:%02d:%02d", + s, tm0.tm_wday, tm0.tm_mday, tm0.tm_mon + 1, + tm0.tm_year + 1900, tm0.tm_hour, tm0.tm_min, + tm0.tm_sec); + return NXT_ERROR; + } + } + for (s = 0; s < NXT_GMTIME_MAX; s += 86400) { nxt_gmtime(s, &tm0); - tm1 = gmtime(&s); + + native = (time_t) s; + + if ((nxt_time_t) native != s) { + /* + * The native time_t is narrower than nxt_time_t and cannot hold + * s, so gmtime() would be given a truncated value and there is + * nothing left to compare against. Where the two types agree + * this never triggers. + */ + break; + } + + tm1 = gmtime(&native); if (tm0.tm_mday != tm1->tm_mday || tm0.tm_mon != tm1->tm_mon @@ -51,6 +133,36 @@ nxt_gmtime_test(nxt_thread_t *thr) } } + /* + * The loop above steps whole days, so it never exercises the time of + * day. Walk a day's worth of seconds either side of the epoch. + */ + + for (s = 0; s < 90000; s++) { + + nxt_gmtime(s, &tm0); + + native = (time_t) s; + + if ((nxt_time_t) native != s) { + break; + } + + tm1 = gmtime(&native); + + if (tm0.tm_hour != tm1->tm_hour + || tm0.tm_min != tm1->tm_min + || tm0.tm_sec != tm1->tm_sec + || tm0.tm_mday != tm1->tm_mday + || tm0.tm_wday != tm1->tm_wday) + { + nxt_log_alert(thr->log, + "gmtime test failed: %T @ %02d:%02d:%02d", + s, tm1->tm_hour, tm1->tm_min, tm1->tm_sec); + return NXT_ERROR; + } + } + nxt_thread_time_update(thr); start = nxt_thread_monotonic_time(thr); @@ -70,7 +182,8 @@ nxt_gmtime_test(nxt_thread_t *thr) start = nxt_thread_monotonic_time(thr); for (s = 0; s < 10000000; s++) { - (void) gmtime(&s); + native = (time_t) s; + (void) gmtime(&native); } nxt_thread_time_update(thr); diff --git a/src/test/nxt_http_parse_test.c b/src/test/nxt_http_parse_test.c index 520bd4184..7c6bc4ca1 100644 --- a/src/test/nxt_http_parse_test.c +++ b/src/test/nxt_http_parse_test.c @@ -64,11 +64,13 @@ static nxt_int_t nxt_http_parse_test_run(nxt_http_request_parse_t *rp, nxt_str_t *request); static nxt_int_t nxt_http_parse_test_bench(nxt_thread_t *thr, nxt_str_t *request, nxt_lvlhsh_t *hash, const char *name, nxt_uint_t n); +static void nxt_http_parse_test_hash_destroy(nxt_lvlhsh_t *hash); static nxt_int_t nxt_http_parse_test_request_line(nxt_http_request_parse_t *rp, nxt_http_parse_test_data_t *data, nxt_str_t *request, nxt_log_t *log); static nxt_int_t nxt_http_parse_test_fields(nxt_http_request_parse_t *rp, nxt_http_parse_test_data_t *data, nxt_str_t *request, nxt_log_t *log); +static nxt_int_t nxt_http_parse_test_discard_fields(nxt_thread_t *thr); static nxt_int_t nxt_http_parse_test_headers(nxt_http_request_parse_t *rp, nxt_http_parse_test_data_t *data, nxt_str_t *request, nxt_log_t *log); @@ -726,6 +728,24 @@ static nxt_str_t nxt_http_test_big_request = nxt_string( ); +/* + * nxt_http_fields_hash() and nxt_http_fields_hash_collisions() build a + * lvlhsh with the malloc-based nxt_http_fields_hash_proto (pool is NULL). + * The test drives them directly, so it must drain the hash itself here, + * the same way nxt_lvlhsh_retrieve() is used to tear down a lvlhsh in + * nxt_lvlhsh_test(), otherwise every bucket and level it allocated leaks. + */ +static void +nxt_http_parse_test_hash_destroy(nxt_lvlhsh_t *hash) +{ + while (nxt_lvlhsh_retrieve(hash, &nxt_http_fields_hash_proto, NULL) + != NULL) + { + continue; + } +} + + nxt_int_t nxt_http_parse_test(nxt_thread_t *thr) { @@ -783,6 +803,10 @@ nxt_http_parse_test(nxt_thread_t *thr) nxt_mp_destroy(mp_temp); } + if (nxt_http_parse_test_discard_fields(thr) != NXT_OK) { + return NXT_ERROR; + } + nxt_log_error(NXT_LOG_NOTICE, thr->log, "http parse test passed"); nxt_memzero(&hash, sizeof(nxt_lvlhsh_t)); @@ -792,6 +816,8 @@ nxt_http_parse_test(nxt_thread_t *thr) nxt_nitems(nxt_http_test_bench_fields), 0); + nxt_http_parse_test_hash_destroy(&hash); + nxt_memzero(&hash, sizeof(nxt_lvlhsh_t)); lvl_colls = nxt_http_fields_hash_collisions(&hash, @@ -799,6 +825,8 @@ nxt_http_parse_test(nxt_thread_t *thr) nxt_nitems(nxt_http_test_bench_fields), 1); + nxt_http_parse_test_hash_destroy(&hash); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "http parse test hash collisions %ui out of %uz, level: %ui", colls, nxt_nitems(nxt_http_test_bench_fields), lvl_colls); @@ -815,6 +843,7 @@ nxt_http_parse_test(nxt_thread_t *thr) &hash, "simple", 1000000) != NXT_OK) { + nxt_http_parse_test_hash_destroy(&hash); return NXT_ERROR; } @@ -822,13 +851,103 @@ nxt_http_parse_test(nxt_thread_t *thr) &hash, "big", 100000) != NXT_OK) { + nxt_http_parse_test_hash_destroy(&hash); return NXT_ERROR; } + nxt_http_parse_test_hash_destroy(&hash); + return NXT_OK; } +/* + * With "discard_fields", fields are checked but not stored. Then a reset of + * the handler is enough to parse the next block. 24 fields is more than the + * 16 inline fields. + */ + +static nxt_int_t +nxt_http_parse_test_discard_fields(nxt_thread_t *thr) +{ + u_char *p; + nxt_mp_t *mp; + nxt_int_t rc, ret; + nxt_uint_t i, nfields; + nxt_buf_mem_t buf; + nxt_http_field_t *field; + nxt_http_request_parse_t rp; + u_char block[24 * nxt_length("X-Field-00: v\r\n") + 2]; + + p = block; + + for (i = 0; i < 24; i++) { + p = nxt_sprintf(p, block + sizeof(block), "X-Field-%02ui: v\r\n", i); + } + + *p++ = '\r'; + *p++ = '\n'; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (mp == NULL) { + return NXT_ERROR; + } + + ret = NXT_ERROR; + + nxt_memzero(&rp, sizeof(nxt_http_request_parse_t)); + + if (nxt_http_parse_request_init(&rp, mp) != NXT_OK) { + goto done; + } + + rp.discard_fields = 1; + + buf.start = block; + buf.pos = block; + buf.free = p; + buf.end = p; + + rc = nxt_http_parse_fields(&rp, &buf); + + if (rc != NXT_DONE || rp.num_inline_fields != 0 || rp.fields != NULL) { + nxt_log_alert(thr->log, "http parse discard fields test failed: " + "rc %i, %ui inline fields, list %p", + rc, (nxt_uint_t) rp.num_inline_fields, rp.fields); + goto done; + } + + rp.handler = NULL; + rp.discard_fields = 0; + + buf.pos = block; + + rc = nxt_http_parse_fields(&rp, &buf); + + nfields = 0; + + nxt_http_fields_each(field, rp.inline_fields, rp.num_inline_fields, + rp.fields) + { + nfields++; + } nxt_http_fields_loop; + + if (rc != NXT_DONE || nfields != 24) { + nxt_log_alert(thr->log, "http parse fields reset test failed: " + "rc %i, %ui fields", rc, nfields); + goto done; + } + + ret = NXT_OK; + +done: + + nxt_mp_destroy(mp); + + return ret; +} + + static nxt_int_t nxt_http_parse_test_run(nxt_http_request_parse_t *rp, nxt_str_t *request) { diff --git a/src/test/nxt_http_request_body_alloc_test.c b/src/test/nxt_http_request_body_alloc_test.c new file mode 100644 index 000000000..72cdecdcf --- /dev/null +++ b/src/test/nxt_http_request_body_alloc_test.c @@ -0,0 +1,208 @@ + +/* + * Copyright (C) FreeUnit + */ + +#include +#include +#include +#include "nxt_tests.h" + + +/* + * nxt_http_request_body_alloc() picks the request body buffer from the + * listener's body_buffer_size: a body that fits stays in memory, a longer + * or unknown one gets an unlinked temporary file with a memory part of + * body_buffer_size. The HTTP/1 path used to do this inline; other + * frontends allocate the body the same way, so the table pins the choice + * and the sizes. + */ + +typedef struct { + const char *name; + size_t body_length; + nxt_bool_t file; + size_t mem_size; +} nxt_http_request_body_alloc_test_case_t; + + +static const nxt_http_request_body_alloc_test_case_t + nxt_http_request_body_alloc_test_cases[] = +{ + { "a body below body_buffer_size", 1000, 0, 1000 }, + { "a body of body_buffer_size", 4096, 0, 4096 }, + { "a body above body_buffer_size", 4097, 1, 4096 }, + { "a body of unknown length", (size_t) -1, 1, 4096 }, +}; + + +nxt_int_t +nxt_http_request_body_alloc_test(nxt_thread_t *thr) +{ + nxt_fd_t fd; + const char *tmpdir; + size_t len; + u_char *notdir; + nxt_mp_t *mp; + nxt_buf_t *b; + nxt_int_t ret; + nxt_uint_t i; + nxt_socket_conf_t skcf; + nxt_http_request_t r; + nxt_socket_conf_joint_t joint; + + const nxt_http_request_body_alloc_test_case_t *tc; + + nxt_memzero(&skcf, sizeof(nxt_socket_conf_t)); + nxt_memzero(&joint, sizeof(nxt_socket_conf_joint_t)); + + /* An empty TMPDIR is the same as no TMPDIR. */ + + tmpdir = getenv("TMPDIR"); + if (tmpdir == NULL || tmpdir[0] == '\0') { + tmpdir = "/tmp"; + } + + skcf.body_buffer_size = 4096; + skcf.body_temp_path.start = (u_char *) tmpdir; + skcf.body_temp_path.length = nxt_strlen(tmpdir); + + joint.socket_conf = &skcf; + + fd = -1; + + for (i = 0; i < nxt_nitems(nxt_http_request_body_alloc_test_cases); i++) { + tc = &nxt_http_request_body_alloc_test_cases[i]; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (mp == NULL) { + return NXT_ERROR; + } + + nxt_memzero(&r, sizeof(nxt_http_request_t)); + r.mem_pool = mp; + r.conf = &joint; + + ret = nxt_http_request_body_alloc(thr->task, &r, tc->body_length); + + b = r.body; + + if (ret != NXT_OK || b == NULL) { + nxt_log_alert(thr->log, "http request body alloc test failed: " + "%s: no buffer", tc->name); + goto fail; + } + + if (nxt_buf_is_file(b) != tc->file) { + nxt_log_alert(thr->log, "http request body alloc test failed: " + "%s: file %d expected %d", + tc->name, nxt_buf_is_file(b), tc->file); + goto fail; + } + + if ((size_t) nxt_buf_mem_size(&b->mem) != tc->mem_size + || b->mem.pos != b->mem.start + || b->mem.free != b->mem.start) + { + nxt_log_alert(thr->log, "http request body alloc test failed: " + "%s: memory part %uz expected %uz", + tc->name, nxt_buf_mem_size(&b->mem), tc->mem_size); + goto fail; + } + + if (tc->file) { + fd = b->file->fd; + + if (b->file->fd == -1 || b->file_end != 0 + || b->file->size != (nxt_off_t) tc->body_length) + { + nxt_log_alert(thr->log, "http request body alloc test " + "failed: %s: file fd %d size %O", + tc->name, b->file->fd, b->file->size); + goto fail; + } + + if (!nxt_test_fd_is_open(b->file->fd)) { + nxt_log_alert(thr->log, "http request body alloc test " + "failed: %s: file descriptor is not open", + tc->name); + goto fail; + } + + nxt_fd_close(fd); + fd = -1; + } + + nxt_mp_destroy(mp); + } + + /* + * A temporary path that cannot be opened is an error, not a buffer. + * A regular file is used as the directory: a file under it cannot be + * made on any host (ENOTDIR). + */ + + len = nxt_strlen(tmpdir) + nxt_length("/nxt_body_alloc_XXXXXX") + 1; + + notdir = nxt_malloc(len); + if (notdir == NULL) { + return NXT_ERROR; + } + + (void) nxt_sprintf(notdir, notdir + len, + "%s/nxt_body_alloc_XXXXXX%Z", tmpdir); + + fd = mkstemp((char *) notdir); + if (fd == -1) { + nxt_log_alert(thr->log, "http request body alloc test failed: " + "mkstemp(\"%s\") %E", notdir, nxt_errno); + nxt_free(notdir); + return NXT_ERROR; + } + + nxt_fd_close(fd); + fd = -1; + + skcf.body_temp_path.start = notdir; + skcf.body_temp_path.length = nxt_strlen(notdir); + + mp = nxt_mp_create(1024, 128, 256, 32); + if (mp == NULL) { + (void) unlink((char *) notdir); + nxt_free(notdir); + return NXT_ERROR; + } + + nxt_memzero(&r, sizeof(nxt_http_request_t)); + r.mem_pool = mp; + r.conf = &joint; + + ret = nxt_http_request_body_alloc(thr->task, &r, 8192); + + if (ret != NXT_ERROR || r.body != NULL) { + nxt_log_alert(thr->log, "http request body alloc test failed: " + "a bad temporary path returned %i", ret); + (void) unlink((char *) notdir); + nxt_free(notdir); + goto fail; + } + + nxt_mp_destroy(mp); + (void) unlink((char *) notdir); + nxt_free(notdir); + + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "http request body alloc test passed"); + + return NXT_OK; + +fail: + + if (fd != -1) { + nxt_fd_close(fd); + } + + nxt_mp_destroy(mp); + + return NXT_ERROR; +} diff --git a/src/test/nxt_http_validate_host_test.c b/src/test/nxt_http_validate_host_test.c new file mode 100644 index 000000000..b93eef3e6 --- /dev/null +++ b/src/test/nxt_http_validate_host_test.c @@ -0,0 +1,88 @@ + +/* + * Copyright (C) FreeUnit + */ + +#include +#include +#include "nxt_tests.h" + + +/* + * nxt_http_validate_host() checks and normalises the value of a Host + * field. It is shared with other frontends, which validate the + * :authority pseudo-header the same way, so the table pins the contract: + * the port is cut off, a trailing dot is dropped, upper case is lowered + * into a pool copy, and an empty label or a slash is a 400. + */ + +nxt_int_t +nxt_http_validate_host_test(nxt_thread_t *thr) +{ + nxt_mp_t *mp; + nxt_str_t host; + nxt_int_t ret; + nxt_uint_t i; + + static const struct { + nxt_str_t input; + nxt_str_t output; + nxt_int_t ret; + } tests[] = { + { nxt_string("example.com"), nxt_string("example.com"), NXT_OK }, + { nxt_string("Example.COM"), nxt_string("example.com"), NXT_OK }, + { nxt_string("example.com."), nxt_string("example.com"), NXT_OK }, + { nxt_string("example.com:8080"), nxt_string("example.com"), NXT_OK }, + { nxt_string("example.com.:8080"), nxt_string("example.com"), NXT_OK }, + { nxt_string("[::1]:8080"), nxt_string("[::1]"), NXT_OK }, + { nxt_string("[::1]"), nxt_string("[::1]"), NXT_OK }, + { nxt_string(""), nxt_string(""), NXT_OK }, + { nxt_string("a..b"), nxt_null_string, + NXT_HTTP_BAD_REQUEST }, + { nxt_string("example.com/path"), nxt_null_string, + NXT_HTTP_BAD_REQUEST }, + { nxt_string("[::1]/"), nxt_null_string, + NXT_HTTP_BAD_REQUEST }, + }; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (mp == NULL) { + return NXT_ERROR; + } + + for (i = 0; i < nxt_nitems(tests); i++) { + host = tests[i].input; + + ret = nxt_http_validate_host(&host, mp); + + if (ret != tests[i].ret) { + nxt_log_alert(thr->log, "nxt_http_validate_host(\"%V\") test " + "failed: %i expected %i", + &tests[i].input, ret, tests[i].ret); + goto fail; + } + + if (ret != NXT_OK) { + continue; + } + + if (!nxt_strstr_eq(&host, &tests[i].output)) { + nxt_log_alert(thr->log, "nxt_http_validate_host(\"%V\") test " + "failed: \"%V\" expected \"%V\"", + &tests[i].input, &host, &tests[i].output); + goto fail; + } + } + + nxt_mp_destroy(mp); + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "http validate host test passed"); + + return NXT_OK; + +fail: + + nxt_mp_destroy(mp); + + return NXT_ERROR; +} diff --git a/src/test/nxt_isolation_mount_dst_test.c b/src/test/nxt_isolation_mount_dst_test.c new file mode 100644 index 000000000..e8411d427 --- /dev/null +++ b/src/test/nxt_isolation_mount_dst_test.c @@ -0,0 +1,378 @@ +/* + * Copyright (C) FreeUnit + */ + +/* + * The mount destination walk in src/nxt_isolation.c has two branches. With + * openat2() it resolves each prefix beneath the rootfs. Without openat2() + * it opens each component with O_NOFOLLOW and refuses a symlink. Every CI + * kernel has openat2(), so the Python isolation tests run only the first + * branch. This test runs each case twice: once as unitd runs it, and once + * with the second branch forced through nxt_isolation_test_mount_dst_open(). + * + * The walk needs no privileges, so the test runs as any user. + * + * The rootfs of each case: + * + * decoy/ outside the rootfs, must stay empty + * r/usr -> ../decoy a link out of the rootfs + * r/a/esc -> ../../decoy the same link one level down + * r/real/lib/ + * r/lib -> real/lib a link inside the rootfs + */ + +#include +#include +#include +#include +#include "nxt_tests.h" + +#include +#include + + +typedef struct { + const char *name; + const char *rel; + /* The directory that the walk opens, relative to the rootfs. */ + const char *opened; + /* The component that the walk refuses. */ + const char *comp; + /* Only openat2() resolves the destination. */ + nxt_bool_t openat2_only; +} nxt_isolation_mount_dst_case_t; + + +typedef struct { + const nxt_isolation_mount_dst_case_t *tc; + nxt_bool_t fallback; + nxt_bool_t have_openat2; + nxt_log_t *log; + const char *tmpdir; + u_char msg[NXT_MAX_ERROR_STR]; +} nxt_isolation_mount_dst_ctx_t; + + +static const nxt_isolation_mount_dst_case_t nxt_mount_dst_cases[] = { + { "symlink out of rootfs", "usr/lib/python3.12", NULL, "usr", 0 }, + { "symlink out of rootfs below a directory", "a/esc/lib", NULL, "esc", + 0 }, + { "symlink inside rootfs", "lib/x", "real/lib/x", "lib", 1 }, + { "no symlink", "a/b/c", "a/b/c", NULL, 0 }, +}; + + +static void nxt_cdecl +nxt_isolation_mount_dst_test_log_handler(nxt_uint_t level, nxt_log_t *log, + const char *fmt, ...) +{ + u_char *p, *end; + va_list args; + nxt_isolation_mount_dst_ctx_t *ctx; + + if (level != NXT_LOG_ALERT) { + return; + } + + ctx = log->ctx; + + p = ctx->msg; + end = ctx->msg + sizeof(ctx->msg) - 1; + + va_start(args, fmt); + p = nxt_vsprintf(p, end, fmt, args); + va_end(args); + + *p = '\0'; +} + + +static int +nxt_isolation_mount_dst_test_remove(const char *path, const struct stat *st, + int flag, struct FTW *ftw) +{ + (void) remove(path); + + return 0; +} + + +static nxt_bool_t +nxt_isolation_mount_dst_test_empty(int dir_fd, const char *name) +{ + int fd; + DIR *dir; + nxt_bool_t empty; + struct dirent *de; + + fd = openat(dir_fd, name, O_RDONLY | O_DIRECTORY | O_CLOEXEC); + if (fd == -1) { + return 0; + } + + dir = fdopendir(fd); + if (dir == NULL) { + close(fd); + return 0; + } + + empty = 1; + + while ((de = readdir(dir)) != NULL) { + if (nxt_strcmp(de->d_name, ".") != 0 + && nxt_strcmp(de->d_name, "..") != 0) + { + empty = 0; + } + } + + closedir(dir); + + return empty; +} + + +static nxt_int_t +nxt_isolation_mount_dst_test_setup(int tmp_fd) +{ + if (mkdirat(tmp_fd, "decoy", 0777) != 0 + || mkdirat(tmp_fd, "r", 0777) != 0 + || mkdirat(tmp_fd, "r/a", 0777) != 0 + || mkdirat(tmp_fd, "r/real", 0777) != 0 + || mkdirat(tmp_fd, "r/real/lib", 0777) != 0 + || symlinkat("../decoy", tmp_fd, "r/usr") != 0 + || symlinkat("../../decoy", tmp_fd, "r/a/esc") != 0 + || symlinkat("real/lib", tmp_fd, "r/lib") != 0) + { + return NXT_ERROR; + } + + return NXT_OK; +} + + +static nxt_int_t +nxt_isolation_mount_dst_test_run(nxt_isolation_mount_dst_ctx_t *ctx, + int tmp_fd) +{ + int fd, rootfs_fd; + nxt_int_t ret; + nxt_log_t log; + nxt_bool_t openat2_used, use_openat2, refused; + nxt_task_t task; + struct stat st, want; + const char *mode; + const nxt_isolation_mount_dst_case_t *tc; + + tc = ctx->tc; + mode = ctx->fallback ? "fallback" : "default"; + + NXT_TEST_CHECK(ctx->log, + nxt_isolation_mount_dst_test_setup(tmp_fd) == NXT_OK, + "mount dst test: setup in \"%s\" failed %E", ctx->tmpdir, + nxt_errno); + + rootfs_fd = openat(tmp_fd, "r", O_RDONLY | O_DIRECTORY | O_CLOEXEC); + + NXT_TEST_CHECK(ctx->log, rootfs_fd != -1, + "mount dst test: open rootfs failed %E", nxt_errno); + + log.level = NXT_LOG_INFO; + log.ident = 0; + log.handler = nxt_isolation_mount_dst_test_log_handler; + log.ctx_handler = NULL; + log.ctx = ctx; + + nxt_memzero(&task, sizeof(task)); + task.thread = nxt_thread(); + task.log = &log; + + fd = -1; + + ret = nxt_isolation_test_mount_dst_open(&task, rootfs_fd, tc->rel, + ctx->fallback, &openat2_used, &fd); + + use_openat2 = !ctx->fallback && ctx->have_openat2; + refused = (tc->comp != NULL && (!tc->openat2_only || !use_openat2)); + + NXT_TEST_CHECK(ctx->log, nxt_isolation_mount_dst_test_empty(tmp_fd, + "decoy"), + "mount dst test: %s, %s: a directory was created " + "outside the rootfs", tc->name, mode); + + if (refused) { + NXT_TEST_CHECK(ctx->log, ret == NXT_ERROR, + "mount dst test: %s, %s: \"%s\" is not refused", + tc->name, mode, tc->rel); + + nxt_log_error(NXT_LOG_NOTICE, ctx->log, "mount dst test: %s, %s: %s", + tc->name, mode, ctx->msg); + + /* + * Each branch has its own message. Only Linux is checked: for a + * symlink opened with O_NOFOLLOW, FreeBSD returns EMLINK, and the + * fallback logs the errno instead. + */ +#if (NXT_LINUX) + { + u_char want_msg[128]; + + (void) nxt_sprintf(want_msg, want_msg + sizeof(want_msg), + "component \"%s\" %s%Z", tc->comp, + use_openat2 + ? "escapes rootfs or is not a directory: " + : "is a symlink or not a directory"); + + NXT_TEST_CHECK(ctx->log, + strstr((char *) ctx->msg, (char *) want_msg) + != NULL, + "mount dst test: %s, %s: the log is \"%s\", " + "wanted \"%s\"", tc->name, mode, ctx->msg, + want_msg); + } +#endif + + if (tc->opened != NULL) { + NXT_TEST_CHECK(ctx->log, + fstatat(rootfs_fd, tc->opened, &st, 0) != 0, + "mount dst test: %s, %s: \"%s\" was created", + tc->name, mode, tc->opened); + } + + } else { + NXT_TEST_CHECK(ctx->log, ret == NXT_OK, + "mount dst test: %s, %s: \"%s\" is refused: %s", + tc->name, mode, tc->rel, ctx->msg); + + NXT_TEST_CHECK(ctx->log, + fstat(fd, &st) == 0 + && fstatat(rootfs_fd, tc->opened, &want, 0) == 0 + && st.st_dev == want.st_dev + && st.st_ino == want.st_ino, + "mount dst test: %s, %s: the walk did not open " + "\"%s\"", tc->name, mode, tc->opened); + + close(fd); + } + + NXT_TEST_CHECK(ctx->log, openat2_used == use_openat2, + "mount dst test: %s, %s: openat2 used %d, wanted %d", + tc->name, mode, (int) openat2_used, (int) use_openat2); + + close(rootfs_fd); + + return NXT_OK; +} + + +static int +nxt_isolation_mount_dst_test_child(void *data) +{ + int tmp_fd; + char dir[256]; + nxt_int_t ret; + nxt_isolation_mount_dst_ctx_t *ctx; + + ctx = data; + + (void) nxt_sprintf((u_char *) dir, (u_char *) dir + sizeof(dir), + "%s/nxt_isolation_mount_dst_test.XXXXXX%Z", + ctx->tmpdir); + + if (mkdtemp(dir) == NULL) { + nxt_log_alert(ctx->log, "mount dst test: mkdtemp(\"%s\") failed %E", + dir, nxt_errno); + return 1; + } + + tmp_fd = open(dir, O_RDONLY | O_DIRECTORY | O_CLOEXEC); + + if (tmp_fd == -1) { + nxt_log_alert(ctx->log, "mount dst test: open(\"%s\") failed %E", + dir, nxt_errno); + ret = NXT_ERROR; + + } else { + ret = nxt_isolation_mount_dst_test_run(ctx, tmp_fd); + close(tmp_fd); + } + + (void) nftw(dir, nxt_isolation_mount_dst_test_remove, 16, + FTW_DEPTH | FTW_PHYS); + + return (ret == NXT_OK) ? 0 : 1; +} + + +#if (NXT_HAVE_OPENAT2) + +static nxt_bool_t +nxt_isolation_mount_dst_test_have_openat2(void) +{ + int fd; + struct open_how how; + + nxt_memzero(&how, sizeof(how)); + how.flags = O_PATH | O_DIRECTORY | O_CLOEXEC; + how.resolve = RESOLVE_BENEATH | RESOLVE_NO_MAGICLINKS; + + fd = syscall(SYS_openat2, AT_FDCWD, ".", &how, sizeof(how)); + if (fd == -1) { + return 0; + } + + close(fd); + + return 1; +} + +#endif + + +nxt_int_t +nxt_isolation_mount_dst_test(nxt_thread_t *thr) +{ + nxt_uint_t i, fallback; + nxt_isolation_mount_dst_ctx_t ctx; + + nxt_thread_time_update(thr); + + nxt_memzero(&ctx, sizeof(ctx)); + + ctx.log = thr->log; + + ctx.tmpdir = getenv("TMPDIR"); + if (ctx.tmpdir == NULL || ctx.tmpdir[0] != '/') { + ctx.tmpdir = "/tmp"; + } + +#if (NXT_HAVE_OPENAT2) + ctx.have_openat2 = nxt_isolation_mount_dst_test_have_openat2(); +#endif + + if (!ctx.have_openat2) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, "mount dst test: openat2() " + "is not available, both runs take the fallback"); + } + + for (fallback = 0; fallback <= 1; fallback++) { + for (i = 0; i < nxt_nitems(nxt_mount_dst_cases); i++) { + ctx.tc = &nxt_mount_dst_cases[i]; + ctx.fallback = fallback; + + if (nxt_test_in_child(thr, ctx.tc->name, + nxt_isolation_mount_dst_test_child, &ctx) + != 0) + { + nxt_log_alert(thr->log, "mount dst test: \"%s\" failed " + "with the %s branch", ctx.tc->name, + fallback ? "fallback" : "default"); + return NXT_ERROR; + } + } + } + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "mount destination test passed"); + + return NXT_OK; +} diff --git a/src/test/nxt_listen_event_test.c b/src/test/nxt_listen_event_test.c new file mode 100644 index 000000000..3e15f9459 --- /dev/null +++ b/src/test/nxt_listen_event_test.c @@ -0,0 +1,321 @@ +/* + * Copyright (C) FreeUnit contributors. + */ + +/* + * Regression test for M-6: nxt_listen_event() (nxt_conn_accept.c) used to + * return a non-NULL lev even when the spare conn could not be allocated, + * and that lev was neither armed (enable_accept) nor linked into + * engine->listen_connections. Every caller (nxt_router.c, nxt_controller.c, + * nxt_runtime.c) only checks for NULL, so the listener silently never + * accepted on that engine, and the router could not even find it again for + * a later update or close. + * + * The fix keeps the listener: lev is linked, left unarmed, and handed to the + * same recovery the accept path already uses when a spare cannot be + * allocated -- nxt_conn_accept_close_idle(), which closes idle conns and + * arms the 100ms listen timer. nxt_conn_listen_timer_handler() then + * retries the spare and, once it succeeds, arms accept. + * + * Both halves are driven here on a fixture engine: + * + * 1. with engine->connections == engine->max_connections the spare + * allocation fails; nxt_listen_event() must still return a lev that is + * linked, unarmed, without a spare, with the retry timer pending; + * + * 2. a slot is freed (max_connections raised), the listen timer handler + * installed on lev->timer is run; it must allocate the spare, arm + * accept once and invoke the accept handler with that spare. + */ + +#include +#include +#include +#include "nxt_tests.h" + + +static nxt_uint_t nxt_listen_event_test_enable_accept_calls; +static nxt_uint_t nxt_listen_event_test_accept_calls; +static nxt_conn_t *nxt_listen_event_test_accepted_spare; +static uint32_t nxt_listen_event_test_accepted_ready; + + +static void +nxt_listen_event_test_enable_accept(nxt_event_engine_t *engine, + nxt_fd_event_t *ev) +{ + nxt_listen_event_test_enable_accept_calls++; + + /* What a real enable_accept does: the read side becomes active. */ + ev->read = NXT_EVENT_ACTIVE; +} + + +static void +nxt_listen_event_test_disable_read(nxt_event_engine_t *engine, + nxt_fd_event_t *ev) +{ + /* + * Must not be reached: the listener is never armed before the retry + * succeeds, and the M-5 guard in nxt_conn_accept_close_idle() skips an + * inactive listener. Counted through the enable_accept counter going + * negative would be obscure, so use a separate alert instead. + */ + nxt_thread_log_alert("listen event test: disable_read() on a listener " + "that was never armed"); + + ev->read = NXT_EVENT_INACTIVE; +} + + +static void +nxt_listen_event_test_accept(nxt_task_t *task, void *obj, void *data) +{ + nxt_listen_event_t *lev; + + /* + * Stands in for nxt_conn_io_accept(): the timer handler calls + * lev->accept with the spare it just allocated. Record it instead of + * running accept(2) on the fixture's closed listen fd. + */ + lev = obj; + + nxt_listen_event_test_accept_calls++; + nxt_listen_event_test_accepted_spare = data; + nxt_listen_event_test_accepted_ready = lev->ready; +} + + +static nxt_conn_io_t nxt_listen_event_test_io = { + .accept = nxt_listen_event_test_accept, +}; + + +nxt_int_t +nxt_listen_event_test(nxt_thread_t *thr) +{ + nxt_int_t ret; + nxt_task_t *task; + nxt_sockaddr_t sa; + nxt_listen_event_t *lev; + nxt_listen_socket_t ls; + nxt_event_engine_t engine, *saved_engine; + nxt_work_queue_cache_t cache; + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "listen event test started"); + + ret = NXT_ERROR; + lev = NULL; + + task = thr->task; + task->thread = thr; + + nxt_memzero(&engine, sizeof(nxt_event_engine_t)); + nxt_memzero(&ls, sizeof(nxt_listen_socket_t)); + nxt_memzero(&sa, sizeof(nxt_sockaddr_t)); + + if (nxt_slow_path(nxt_timers_init(&engine.timers, 4) != NXT_OK)) { + nxt_log_alert(thr->log, "listen event test failed to init timers"); + return NXT_ERROR; + } + + /* Backs the conn structs and the sockaddr cache of the retry below. */ + engine.mem_pool = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(engine.mem_pool == NULL)) { + nxt_free(engine.timers.changes); + return NXT_ERROR; + } + + engine.task = *task; + engine.event.io = &nxt_listen_event_test_io; + engine.event.enable_accept = nxt_listen_event_test_enable_accept; + engine.event.disable_read = nxt_listen_event_test_disable_read; + + nxt_queue_init(&engine.listen_connections); + nxt_queue_init(&engine.idle_connections); + nxt_queue_init(&engine.active_connections); + + nxt_work_queue_cache_create(&cache, 1024); + + engine.fast_work_queue.cache = &cache; + nxt_work_queue_name(&engine.fast_work_queue, "fast"); + + engine.close_work_queue.cache = &cache; + nxt_work_queue_name(&engine.close_work_queue, "close"); + + /* + * Enough of a listen socket for nxt_conn_accept_alloc(): the spare's + * sockaddr is sized and typed from it. The fd is never used, since + * accept is stubbed above. + */ + sa.type = SOCK_STREAM; + sa.u.sockaddr.sa_family = AF_INET; + + ls.socket = -1; + ls.sockaddr = &sa; + ls.socklen = sizeof(struct sockaddr_in); + ls.address_length = NXT_INET_ADDR_STR_LEN; + + /* + * engine->connections < engine->max_connections is false (0 < 0), so + * nxt_conn_accept_alloc() fails immediately without touching memory + * pools or the listen socket's sockaddr cache -- the failure this test + * needs, reached the cheapest way. + */ + engine.connections = 0; + engine.max_connections = 0; + + /* The timer-driven first accept must start a batch of this size. */ + engine.batch = 3; + + saved_engine = thr->engine; + thr->engine = &engine; + + nxt_listen_event_test_enable_accept_calls = 0; + nxt_listen_event_test_accept_calls = 0; + nxt_listen_event_test_accepted_spare = NULL; + + lev = nxt_listen_event(task, &ls); + + if (nxt_slow_path(lev == NULL)) { + nxt_log_alert(thr->log, "listen event test: nxt_listen_event() " + "returned NULL although only the spare conn could " + "not be allocated"); + goto done; + } + + if (nxt_slow_path(nxt_queue_is_empty(&engine.listen_connections) + || nxt_queue_first(&engine.listen_connections) + != &lev->link)) + { + nxt_log_alert(thr->log, "listen event test: the listener was not " + "linked into listen_connections"); + goto done; + } + + if (nxt_slow_path(lev->next != NULL)) { + nxt_log_alert(thr->log, "listen event test: a spare conn is set " + "although its allocation must have failed"); + goto done; + } + + if (nxt_slow_path(nxt_listen_event_test_enable_accept_calls != 0 + || nxt_fd_event_is_active(lev->socket.read))) + { + nxt_log_alert(thr->log, "listen event test: accept was armed " + "without a spare conn"); + goto done; + } + + if (nxt_slow_path(!nxt_timer_is_in_tree(&lev->timer) + && lev->timer.change == NXT_TIMER_NO_CHANGE)) + { + nxt_log_alert(thr->log, "listen event test: the listen timer was " + "not armed, the spare would never be retried"); + goto done; + } + + if (nxt_slow_path(lev->timer.handler == NULL)) { + nxt_log_alert(thr->log, "listen event test: no listen timer " + "handler installed"); + goto done; + } + + /* + * The idle-close pass runs later, so it must have the engine task: the + * task of the caller is the task of a router job, which is freed with + * the configuration after the job is posted. + */ + if (nxt_slow_path(engine.close_work_queue.head == NULL + || engine.close_work_queue.head->task != &engine.task)) + { + nxt_log_alert(thr->log, "listen event test: the idle-close pass " + "was not queued with the engine task"); + goto done; + } + + /* + * nxt_conn_accept_close_idle() also queued the idle-close pass. There + * is nothing idle on the fixture, so it is a no-op here; run it anyway + * so the queue is empty when the cache is destroyed below. + */ + while (engine.close_work_queue.head != NULL) { + void *obj, *data; + nxt_task_t *t; + nxt_work_handler_t handler; + + handler = nxt_work_queue_pop(&engine.close_work_queue, &t, &obj, + &data); + handler(t, obj, data); + } + + /* + * A connection slot frees up. Run what the 100ms timer would run: + * nxt_conn_listen_timer_handler(), installed on lev->timer by + * nxt_listen_event(). It must now get the spare, arm accept and hand + * the spare to the accept handler. + */ + engine.max_connections = 1; + + lev->timer.handler(task, &lev->timer, NULL); + + if (nxt_slow_path(lev->next == NULL)) { + nxt_log_alert(thr->log, "listen event test: the retry did not " + "allocate the spare conn"); + goto done; + } + + if (nxt_slow_path(nxt_listen_event_test_enable_accept_calls != 1 + || !nxt_fd_event_is_active(lev->socket.read))) + { + nxt_log_alert(thr->log, "listen event test: the retry did not arm " + "accept exactly once (%ui calls)", + nxt_listen_event_test_enable_accept_calls); + goto done; + } + + if (nxt_slow_path(nxt_listen_event_test_accept_calls != 1 + || nxt_listen_event_test_accepted_spare != lev->next)) + { + nxt_log_alert(thr->log, "listen event test: the retry did not " + "call the accept handler with the new spare " + "(%ui calls)", nxt_listen_event_test_accept_calls); + goto done; + } + + if (nxt_slow_path(nxt_listen_event_test_accepted_ready != 3)) { + nxt_log_alert(thr->log, "listen event test: the first accept after " + "the retry got ready %uD, expected the batch 3", + nxt_listen_event_test_accepted_ready); + goto done; + } + + ret = NXT_OK; + +done: + + if (lev != NULL) { + /* What nxt_router_listen_event_release() does on the last ref. */ + + if (lev->next != NULL) { + nxt_sockaddr_cache_free(&engine, lev->next); + nxt_conn_free(task, lev->next); + } + + nxt_timer_delete(&engine, &lev->timer); + nxt_queue_remove(&lev->link); + nxt_free(lev); + } + + thr->engine = saved_engine; + + nxt_work_queue_cache_destroy(&cache); + nxt_mp_destroy(engine.mem_pool); + nxt_free(engine.timers.changes); + + if (nxt_fast_path(ret == NXT_OK)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, "listen event test passed"); + } + + return ret; +} diff --git a/src/test/nxt_lvlhsh_test.c b/src/test/nxt_lvlhsh_test.c index baa6d0e1e..4db4653dd 100644 --- a/src/test/nxt_lvlhsh_test.c +++ b/src/test/nxt_lvlhsh_test.c @@ -57,7 +57,7 @@ nxt_lvlhsh_test_add(nxt_lvlhsh_t *lh, const nxt_lvlhsh_proto_t *proto, case NXT_DECLINED: nxt_thread_log_alert("lvlhsh test failed: " "key %p is already in hash", key); - /* Fall through. */ + nxt_fallthrough; default: return NXT_ERROR; } diff --git a/src/test/nxt_main_file_store_test.c b/src/test/nxt_main_file_store_test.c new file mode 100644 index 000000000..6c9e49af4 --- /dev/null +++ b/src/test/nxt_main_file_store_test.c @@ -0,0 +1,458 @@ +/* + * Copyright (C) F5, Inc. + */ + +/* + * Regression test for issue #215: nxt_main_file_store() + * (src/nxt_main_process.c) ignored its "tmp_name" argument, opening the + * destination itself with NXT_FILE_TRUNCATE and finishing with a rename() + * of the file onto itself. The state file was rewritten in place, and the + * short-write path unlinked what was then the live conf.json. + * + * Four invariants, each observed rather than inferred: + * + * - a store replaces the destination by rename(2), pinned down by the + * inode number changing. An in-place rewrite keeps the inode, so this + * case alone separates the fixed function from the old one; + * - no temporary survives a successful store, including a stale one from + * an interrupted run left in the way; + * - a store that cannot create its temporary reports NXT_ERROR and leaves + * the destination byte-for-byte intact -- the shape of an ENOSPC or a + * read-only state directory, where the old code destroyed the + * configuration it was asked to update; + * - an existing destination keeps its mode across the replacement. + * + * Ownership is not asserted: the test does not require root, and chown() to + * another uid is not available to an unprivileged run. + */ + +#include +#include +#include +#include "nxt_tests.h" + +#include + + +#define nxt_main_file_store_test_fail(thr, ...) \ + do { \ + nxt_log_alert((thr)->log, "main file store test: " __VA_ARGS__); \ + goto done; \ + } while (0) + + +static nxt_int_t +nxt_main_file_store_test_content(const char *name, char *buf, size_t size, + ssize_t *len) +{ + int fd; + ssize_t n; + + fd = open(name, O_RDONLY); + if (nxt_slow_path(fd == -1)) { + return NXT_ERROR; + } + + n = read(fd, buf, size); + + close(fd); + + if (nxt_slow_path(n < 0)) { + return NXT_ERROR; + } + + *len = n; + + return NXT_OK; +} + + +static nxt_int_t +nxt_main_file_store_test_holds(const char *name, const char *expected) +{ + char buf[256]; + size_t len; + ssize_t n; + + if (nxt_slow_path(nxt_main_file_store_test_content(name, buf, sizeof(buf), + &n) + != NXT_OK)) + { + return NXT_ERROR; + } + + len = nxt_strlen(expected); + + if (nxt_slow_path((size_t) n != len || memcmp(buf, expected, len) != 0)) { + return NXT_ERROR; + } + + return NXT_OK; +} + + +static nxt_int_t +nxt_main_file_store_test_plant(const char *name, const char *content, + mode_t mode) +{ + int fd; + size_t len; + ssize_t n; + + fd = open(name, O_WRONLY | O_CREAT | O_TRUNC, mode); + if (nxt_slow_path(fd == -1)) { + return NXT_ERROR; + } + + len = nxt_strlen(content); + + n = write(fd, content, len); + + close(fd); + + /* open() honours the umask; the test needs the exact mode. */ + if (nxt_slow_path(n != (ssize_t) len || chmod(name, mode) != 0)) { + return NXT_ERROR; + } + + return NXT_OK; +} + + +static nxt_int_t +nxt_main_file_store_test_store(nxt_task_t *task, const char *dir, + const char *tmp_name, const char *name, const char *content) +{ + return nxt_main_file_store(task, dir, tmp_name, name, (u_char *) content, + nxt_strlen(content)); +} + + +nxt_int_t +nxt_main_file_store_test(nxt_thread_t *thr) +{ + char dir[NXT_MAX_PATH_LEN], name[NXT_MAX_PATH_LEN]; + char tmp_name[NXT_MAX_PATH_LEN], nowhere[NXT_MAX_PATH_LEN]; + char victim[NXT_MAX_PATH_LEN]; + ino_t first_ino; + const char *tmpdir; + nxt_int_t ret; + nxt_task_t *task; + struct stat st; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "main file store test started"); + + task = thr->task; + task->thread = thr; + + tmpdir = getenv("TMPDIR"); + if (tmpdir == NULL || tmpdir[0] != '/') { + tmpdir = "/tmp"; + } + + (void) nxt_sprintf((u_char *) dir, (u_char *) dir + sizeof(dir), + "%s/nxt_main_file_store_test.XXXXXX%Z", tmpdir); + + if (nxt_slow_path(mkdtemp(dir) == NULL)) { + nxt_log_alert(thr->log, "main file store test: mkdtemp(\"%s\") " + "failed %E", dir, nxt_errno); + return NXT_ERROR; + } + + ret = NXT_ERROR; + + (void) nxt_sprintf((u_char *) name, (u_char *) name + sizeof(name), + "%s/conf.json%Z", dir); + (void) nxt_sprintf((u_char *) tmp_name, + (u_char *) tmp_name + sizeof(tmp_name), + "%s/conf.json.tmp%Z", dir); + (void) nxt_sprintf((u_char *) nowhere, + (u_char *) nowhere + sizeof(nowhere), + "%s/absent/conf.json.tmp%Z", dir); + (void) nxt_sprintf((u_char *) victim, + (u_char *) victim + sizeof(victim), + "%s/victim%Z", dir); + + /* A first store into an empty directory creates the destination. */ + + if (nxt_main_file_store_test_store(task, dir, tmp_name, name, + "{\"one\":1}") != NXT_OK) + { + nxt_main_file_store_test_fail(thr, "the first store failed"); + } + + if (nxt_main_file_store_test_holds(name, "{\"one\":1}") != NXT_OK) { + nxt_main_file_store_test_fail(thr, "\"%s\" does not hold what the " + "first store wrote", name); + } + + if (stat(name, &st) != 0) { + nxt_main_file_store_test_fail(thr, "stat(\"%s\") failed %E", name, + nxt_errno); + } + + first_ino = st.st_ino; + + /* + * A stale temporary sits in the way, standing in for a store that was + * interrupted before its rename: it must be overwritten silently and + * must not survive. + */ + + if (nxt_main_file_store_test_plant(tmp_name, "stale", 0600) != NXT_OK) { + nxt_main_file_store_test_fail(thr, "could not plant a stale " + "temporary"); + } + + if (nxt_main_file_store_test_store(task, dir, tmp_name, name, + "{\"two\":2}") != NXT_OK) + { + nxt_main_file_store_test_fail(thr, "the second store failed"); + } + + if (nxt_main_file_store_test_holds(name, "{\"two\":2}") != NXT_OK) { + nxt_main_file_store_test_fail(thr, "\"%s\" does not hold what the " + "second store wrote", name); + } + + if (stat(tmp_name, &st) == 0) { + nxt_main_file_store_test_fail(thr, "\"%s\" was left behind by a " + "successful store", tmp_name); + } + + if (stat(name, &st) != 0) { + nxt_main_file_store_test_fail(thr, "stat(\"%s\") failed %E", name, + nxt_errno); + } + + if (st.st_ino == first_ino) { + nxt_main_file_store_test_fail(thr, "\"%s\" kept inode %uL across a " + "store -- the file was rewritten in " + "place, so a crash mid-write leaves " + "it truncated", name, + (uint64_t) first_ino); + } + + /* + * A store that cannot create its temporary -- the path names a + * directory that does not exist, the shape of a full or read-only state + * directory -- must report the failure and leave the destination as it + * was. + */ + + if (nxt_main_file_store_test_store(task, dir, nowhere, name, + "{\"three\":3}") == NXT_OK) + { + nxt_main_file_store_test_fail(thr, "a store whose temporary cannot " + "be created reported success"); + } + + if (nxt_main_file_store_test_holds(name, "{\"two\":2}") != NXT_OK) { + nxt_main_file_store_test_fail(thr, "a failed store damaged \"%s\"", + name); + } + + /* An existing destination keeps its mode across the replacement. */ + + if (chmod(name, 0640) != 0) { + nxt_main_file_store_test_fail(thr, "chmod(\"%s\") failed %E", name, + nxt_errno); + } + + if (nxt_main_file_store_test_store(task, dir, tmp_name, name, + "{\"four\":4}") != NXT_OK) + { + nxt_main_file_store_test_fail(thr, "the mode-preserving store " + "failed"); + } + + if (stat(name, &st) != 0 || (st.st_mode & 07777) != 0640) { + nxt_main_file_store_test_fail(thr, "\"%s\" came back with mode " + "0x%04xd, expected the 0640 (0x1a0) it had", + name, (unsigned) (st.st_mode & 07777)); + } + + /* + * Set-user-ID survives too. The replacement carries the mode over with + * fchmod() and the ownership with fchown(), and a chown() of a regular + * file clears set-user-ID -- so doing them in that order would drop the + * bit while appearing to preserve the mode. An unchanged owner is + * enough to trigger it, which is why this is assertable without root. + */ + + if (chmod(name, 04640) != 0) { + nxt_main_file_store_test_fail(thr, "chmod(\"%s\") failed %E", name, + nxt_errno); + } + + if (nxt_main_file_store_test_store(task, dir, tmp_name, name, + "{\"setid\":1}") != NXT_OK) + { + nxt_main_file_store_test_fail(thr, "the set-user-ID preserving store " + "failed"); + } + + if (stat(name, &st) != 0 || (st.st_mode & 07777) != 04640) { + nxt_main_file_store_test_fail(thr, "\"%s\" came back with mode " + "0x%04xd, expected the 04640 (0x9a0) " + "it had -- chown() after chmod() " + "clears set-user-ID", name, + (unsigned) (st.st_mode & 07777)); + } + + if (chmod(name, 0640) != 0) { + nxt_main_file_store_test_fail(thr, "chmod(\"%s\") failed %E", name, + nxt_errno); + } + + /* + * The temporary's name is predictable and the store runs as root, so a + * state directory writable by anyone else is a place to leave a symbolic + * link under that name and have the store follow it. Opening with + * O_TRUNC did: it truncated whatever the link addressed and the rename + * then installed the link as the state file. The store must write the + * state file and leave the link's target alone. + */ + + if (nxt_main_file_store_test_plant(victim, "precious", 0600) != NXT_OK) { + nxt_main_file_store_test_fail(thr, "could not plant the link target"); + } + + (void) unlink(tmp_name); + + if (symlink(victim, tmp_name) != 0) { + nxt_main_file_store_test_fail(thr, "symlink(\"%s\", \"%s\") failed " + "%E", victim, tmp_name, nxt_errno); + } + + if (nxt_main_file_store_test_store(task, dir, tmp_name, name, + "{\"five\":5}") != NXT_OK) + { + nxt_main_file_store_test_fail(thr, "a store whose temporary was a " + "symbolic link failed"); + } + + if (nxt_main_file_store_test_holds(victim, "precious") != NXT_OK) { + nxt_main_file_store_test_fail(thr, "the store followed the link and " + "wrote through it to \"%s\"", victim); + } + + if (nxt_main_file_store_test_holds(name, "{\"five\":5}") != NXT_OK) { + nxt_main_file_store_test_fail(thr, "\"%s\" does not hold what the " + "store past a link wrote", name); + } + + if (lstat(name, &st) != 0 || !S_ISREG(st.st_mode)) { + nxt_main_file_store_test_fail(thr, "\"%s\" is not a regular file -- " + "the link was installed as the state " + "file", name); + } + + (void) unlink(victim); + + /* + * The destination can be a symbolic link as well, and there the danger + * is not the write -- the rename replaces the link whatever it points + * at -- but the inheritance. Reading the destination's mode with + * stat() follows the link, so a link to a file of the other user's own + * hands them the mode of the conf.json that replaces it: aim it at + * something 0666 and every later store inherits that again. + */ + + if (nxt_main_file_store_test_plant(victim, "bait", 0666) != NXT_OK) { + nxt_main_file_store_test_fail(thr, "could not plant the mode bait"); + } + + (void) unlink(name); + + if (symlink(victim, name) != 0) { + nxt_main_file_store_test_fail(thr, "symlink(\"%s\", \"%s\") failed " + "%E", victim, name, nxt_errno); + } + + if (nxt_main_file_store_test_store(task, dir, tmp_name, name, + "{\"six\":6}") != NXT_OK) + { + nxt_main_file_store_test_fail(thr, "a store onto a linked " + "destination failed"); + } + + if (nxt_main_file_store_test_holds(victim, "bait") != NXT_OK) { + nxt_main_file_store_test_fail(thr, "the store wrote through the " + "linked destination to \"%s\"", victim); + } + + if (nxt_main_file_store_test_holds(name, "{\"six\":6}") != NXT_OK) { + nxt_main_file_store_test_fail(thr, "\"%s\" does not hold what the " + "store onto a link wrote", name); + } + + if (lstat(name, &st) != 0 || !S_ISREG(st.st_mode)) { + nxt_main_file_store_test_fail(thr, "\"%s\" is not a regular file " + "after a store onto a link", name); + } + + if ((st.st_mode & 07777) != 0600) { + nxt_main_file_store_test_fail(thr, "\"%s\" came back with mode " + "0x%04xd, expected the 0600 (0x180) " + "the temporary was created with -- the " + "store inherited the link target's " + "mode", name, + (unsigned) (st.st_mode & 07777)); + } + + (void) unlink(victim); + + /* + * Replace a certificate bundle under its own name. The temporary file + * is in the same directory and has a reserved dot name. A short bundle + * that replaces a long bundle must not keep the tail of the long bundle. + * The old in-place open without O_TRUNC kept that tail. + */ + + (void) unlink(name); + + (void) nxt_sprintf((u_char *) name, (u_char *) name + sizeof(name), + "%s/example.org%Z", dir); + (void) nxt_sprintf((u_char *) tmp_name, + (u_char *) tmp_name + sizeof(tmp_name), + "%s/.example.org.tmp%Z", dir); + + if (nxt_main_file_store_test_store(task, dir, tmp_name, name, + "-----BEGIN LONG BUNDLE-----") + != NXT_OK) + { + nxt_main_file_store_test_fail(thr, "the long bundle store failed"); + } + + if (nxt_main_file_store_test_store(task, dir, tmp_name, name, + "-----SHORT-----") != NXT_OK) + { + nxt_main_file_store_test_fail(thr, "the short bundle store failed"); + } + + if (nxt_main_file_store_test_holds(name, "-----SHORT-----") != NXT_OK) { + nxt_main_file_store_test_fail(thr, "\"%s\" kept a tail of the longer " + "bundle it replaced", name); + } + + if (stat(tmp_name, &st) == 0) { + nxt_main_file_store_test_fail(thr, "\"%s\" was left behind by a " + "bundle store", tmp_name); + } + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "main file store test passed"); + + ret = NXT_OK; + +done: + + (void) unlink(name); + (void) unlink(tmp_name); + (void) unlink(victim); + (void) rmdir(dir); + + return ret; +} diff --git a/src/test/nxt_main_remove_child_pid_test.c b/src/test/nxt_main_remove_child_pid_test.c new file mode 100644 index 000000000..7c1eddd96 --- /dev/null +++ b/src/test/nxt_main_remove_child_pid_test.c @@ -0,0 +1,463 @@ +/* + * Copyright (C) FreeUnit Community + */ + +/* + * Issue #310: under "isolation": {"namespaces": {"pid": true}} a worker that + * died before PROCESS_CREATED left its record, port and descriptor in main + * until the prototype exited. The prototype knows such a worker only by its + * namespace-local pid, which REMOVE_PID must not broadcast. + * + * Main keeps that pid next to the global one at WHOAMI time + * (nxt_main_process_name_child()) and resolves REMOVE_CHILD_PID against it, + * among the sender's own children only. Both halves are driven here. + * + * Naming must refuse 0, a negative pid and a name a live sibling holds, and + * must keep a name equal to the global pid. Resolving must refuse a short + * payload, pid 0, a negative pid, an unknown sender, a sender that is not a + * prototype and a pid no child of the sender holds; when it accepts, it must + * retire the record and tell the router by the global pid. + * + * Without NXT_USE_CMSG_PID the handler is not registered, and nothing is + * tested. + */ + +#include +#include +#include +#include +#include +#include "nxt_tests.h" + + +#if (NXT_USE_CMSG_PID) + +/* Namespace-local pids of two workers. */ +#define NXT_MAIN_RCP_NS_PID 7 +#define NXT_MAIN_RCP_OTHER_NS_PID 8 + + +typedef struct { + nxt_pid_t sender; + nxt_pid_t pid; + nxt_bool_t payload; + const char *label; +} nxt_main_rcp_case_t; + + +static nxt_port_t * +nxt_main_rcp_port(nxt_task_t *task, nxt_runtime_t *rt, nxt_pid_t pid, + nxt_process_type_t type) +{ + nxt_port_t *port; + + port = nxt_runtime_process_port_create(task, rt, pid, 0, type); + if (port != NULL) { + port->pair[0] = -1; + port->pair[1] = -1; + port->socket.fd = -1; + } + + return port; +} + + +static void +nxt_main_rcp_port_release(nxt_task_t *task, nxt_port_t *port) +{ + if (port != NULL) { + nxt_port_close(task, port); + nxt_runtime_port_remove(task, port); + } +} + + +/* + * Drive the handler once. The fixture ports are never write_ready, so what + * the handler sends stays on port->messages for the test to read. + */ + +static nxt_int_t +nxt_main_rcp_report(nxt_thread_t *thr, nxt_task_t *task, nxt_mp_t *mp, + nxt_pid_t sender, nxt_pid_t pid, nxt_bool_t payload) +{ + nxt_buf_t *buf; + nxt_port_recv_msg_t msg; + + buf = NULL; + + if (payload) { + buf = nxt_buf_mem_alloc(mp, sizeof(nxt_pid_t), 0); + if (nxt_slow_path(buf == NULL)) { + nxt_log_alert(thr->log, "main remove child pid test: no buffer"); + return NXT_ERROR; + } + + buf->mem.free = nxt_cpymem(buf->mem.free, &pid, sizeof(nxt_pid_t)); + } + + nxt_memzero(&msg, sizeof(nxt_port_recv_msg_t)); + + msg.buf = buf; + msg.fd[0] = -1; + msg.fd[1] = -1; + msg.port_msg.type = _NXT_PORT_MSG_REMOVE_CHILD_PID; + msg.port_msg.pid = sender; + msg.cmsg_pid = sender; + + nxt_main_test_run_remove_child_pid_handler(task, &msg); + + if (buf != NULL) { + nxt_mp_free(mp, buf); + } + + return NXT_OK; +} + + +/* The router got one REMOVE_PID naming pid, and nothing else. */ + +static nxt_int_t +nxt_main_rcp_router_told(nxt_thread_t *thr, nxt_port_t *router_port, + nxt_pid_t pid, const char *label) +{ + nxt_pid_t sent_pid; + nxt_port_send_msg_t *sent; + + if (nxt_slow_path(nxt_queue_is_empty(&router_port->messages))) { + nxt_log_alert(thr->log, "main remove child pid test: %s: the router " + "was not told", label); + return NXT_ERROR; + } + + sent = nxt_queue_link_data(nxt_queue_first(&router_port->messages), + nxt_port_send_msg_t, link); + + if (nxt_slow_path(sent->port_msg.type != _NXT_PORT_MSG_REMOVE_PID + || sent->buf == NULL + || nxt_buf_used_size(sent->buf) != sizeof(nxt_pid_t))) + { + nxt_log_alert(thr->log, "main remove child pid test: %s: the router " + "got a message of type %d, not a REMOVE_PID", label, + (int) sent->port_msg.type); + return NXT_ERROR; + } + + nxt_memcpy(&sent_pid, sent->buf->mem.pos, sizeof(nxt_pid_t)); + + if (nxt_slow_path(sent_pid != pid)) { + nxt_log_alert(thr->log, "main remove child pid test: %s: the router " + "was told to remove pid %PI, not %PI", label, sent_pid, + pid); + return NXT_ERROR; + } + + if (nxt_slow_path(nxt_queue_next(nxt_queue_first(&router_port->messages)) + != nxt_queue_tail(&router_port->messages))) + { + nxt_log_alert(thr->log, "main remove child pid test: %s: the router " + "got more than one message", label); + return NXT_ERROR; + } + + /* Queued messages hold a port reference each; give them back. */ + nxt_port_test_run_error_handler(thr->task, router_port); + + return NXT_OK; +} + + +nxt_int_t +nxt_main_remove_child_pid_test(nxt_thread_t *thr) +{ + nxt_mp_t *mp; + nxt_int_t ret; + nxt_pid_t proto_pid, child_pid, other_pid, twin_pid; + nxt_task_t *task; + nxt_uint_t i; + nxt_port_t *router_port, *proto_port, *child_port, *other_port; + nxt_port_t *twin_port; + nxt_runtime_t *rt, *saved_rt; + nxt_process_t *proto, *child, *other, *twin; + nxt_event_engine_t engine, *saved_engine; + nxt_port_send_msg_t *sent; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "main remove child pid test started"); + + ret = NXT_ERROR; + router_port = NULL; + proto_port = NULL; + child_port = NULL; + other_port = NULL; + twin_port = NULL; + + task = thr->task; + task->thread = thr; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + rt = nxt_mp_zalloc(mp, sizeof(nxt_runtime_t)); + if (nxt_slow_path(rt == NULL)) { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + rt->mem_pool = mp; + + if (nxt_slow_path(nxt_thread_mutex_create(&rt->processes_mutex) != NXT_OK)) + { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + /* nxt_port_remove_notify_others() allocates from engine->mem_pool. */ + nxt_memzero(&engine, sizeof(engine)); + nxt_work_queue_cache_create(&engine.work_queue_cache, 1024); + engine.fast_work_queue.cache = &engine.work_queue_cache; + nxt_work_queue_name(&engine.fast_work_queue, "fast"); + engine.mem_pool = mp; + + saved_rt = thr->runtime; + saved_engine = thr->engine; + thr->runtime = rt; + thr->engine = &engine; + rt->main_engine = &engine; + + /* + * This process stands in for main, so every fixture pid must differ from + * nxt_pid: nxt_port_remove_notify_others() skips the running process. + * twin is the worker whose local pid equals its global pid. + */ + + proto_pid = nxt_pid + 1; + child_pid = nxt_pid + 2; + other_pid = nxt_pid + 3; + twin_pid = nxt_pid + 5; + + router_port = nxt_main_rcp_port(task, rt, nxt_pid + 4, NXT_PROCESS_ROUTER); + proto_port = nxt_main_rcp_port(task, rt, proto_pid, NXT_PROCESS_PROTOTYPE); + child_port = nxt_main_rcp_port(task, rt, child_pid, NXT_PROCESS_APP); + other_port = nxt_main_rcp_port(task, rt, other_pid, NXT_PROCESS_APP); + twin_port = nxt_main_rcp_port(task, rt, twin_pid, NXT_PROCESS_APP); + + if (nxt_slow_path(router_port == NULL || proto_port == NULL + || child_port == NULL || other_port == NULL + || twin_port == NULL)) + { + goto done; + } + + proto = proto_port->process; + child = child_port->process; + other = other_port->process; + twin = twin_port->process; + + /* As nxt_main_process_whoami_handler() leaves the prototype's workers. */ + + nxt_queue_insert_tail(&proto->children, &child->link); + nxt_queue_insert_tail(&proto->children, &other->link); + nxt_queue_insert_tail(&proto->children, &twin->link); + + /* Naming: 0 and a negative pid are refused. */ + + nxt_main_test_run_name_child(task, proto, child, 0); + nxt_main_test_run_name_child(task, proto, child, -1); + + if (nxt_slow_path(child->parent_ns_pid != 0)) { + nxt_log_alert(thr->log, "main remove child pid test: a worker was " + "named %PI", child->parent_ns_pid); + goto done; + } + + nxt_main_test_run_name_child(task, proto, child, NXT_MAIN_RCP_NS_PID); + nxt_main_test_run_name_child(task, proto, other, + NXT_MAIN_RCP_OTHER_NS_PID); + + if (nxt_slow_path(child->parent_ns_pid != NXT_MAIN_RCP_NS_PID + || other->parent_ns_pid != NXT_MAIN_RCP_OTHER_NS_PID)) + { + nxt_log_alert(thr->log, "main remove child pid test: a worker was " + "not named"); + goto done; + } + + /* A name a live sibling holds is refused; the sibling keeps it. */ + + nxt_main_test_run_name_child(task, proto, twin, NXT_MAIN_RCP_NS_PID); + + if (nxt_slow_path(twin->parent_ns_pid != 0 + || child->parent_ns_pid != NXT_MAIN_RCP_NS_PID)) + { + nxt_log_alert(thr->log, "main remove child pid test: a worker was " + "allowed to claim a live sibling's name"); + goto done; + } + + /* + * Reports that must retire nothing and tell the router nothing. twin + * still has no name, so a handler that reads 0 as a name would retire + * it. The router holds a named child of its own for the non-prototype + * case, so that only the type check keeps that record. + */ + + nxt_queue_remove(&other->link); + nxt_queue_insert_tail(&router_port->process->children, &other->link); + + { + const nxt_main_rcp_case_t refused[] = { + { proto_pid, NXT_MAIN_RCP_NS_PID, 0, "a payload with no pid" }, + { proto_pid, 0, 1, "pid 0" }, + { proto_pid, -1, 1, "a negative pid" }, + { nxt_pid + 9, NXT_MAIN_RCP_NS_PID, 1, "an unknown sender" }, + { router_port->pid, NXT_MAIN_RCP_OTHER_NS_PID, 1, + "a sender that is not a prototype" }, + { proto_pid, NXT_MAIN_RCP_NS_PID + 100, 1, "a pid of no child" }, + }; + + for (i = 0; i < nxt_nitems(refused); i++) { + ret = nxt_main_rcp_report(thr, task, mp, refused[i].sender, + refused[i].pid, refused[i].payload); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + ret = NXT_ERROR; + + if (nxt_slow_path(nxt_runtime_process_find(rt, child_pid) != child + || nxt_runtime_process_find(rt, other_pid) + != other + || nxt_runtime_process_find(rt, twin_pid) + != twin)) + { + nxt_log_alert(thr->log, "main remove child pid test: %s " + "retired a record", refused[i].label); + goto done; + } + + if (nxt_slow_path(!nxt_queue_is_empty(&router_port->messages))) { + sent = nxt_queue_link_data( + nxt_queue_first(&router_port->messages), + nxt_port_send_msg_t, link); + + nxt_log_alert(thr->log, "main remove child pid test: %s made " + "the router a message of type %d", + refused[i].label, (int) sent->port_msg.type); + goto done; + } + } + } + + nxt_queue_remove(&other->link); + nxt_queue_insert_tail(&proto->children, &other->link); + + /* A name equal to the global pid is kept. */ + + nxt_main_test_run_name_child(task, proto, twin, twin_pid); + + if (nxt_slow_path(twin->parent_ns_pid != twin_pid)) { + nxt_log_alert(thr->log, "main remove child pid test: a worker whose " + "two pids are equal was left unnamed"); + goto done; + } + + /* Accepted: the record goes and the router is told by the global pid. */ + + ret = nxt_main_rcp_report(thr, task, mp, proto_pid, twin_pid, 1); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + ret = NXT_ERROR; + twin_port = NULL; + + if (nxt_slow_path(nxt_runtime_process_find(rt, twin_pid) != NULL)) { + nxt_log_alert(thr->log, "main remove child pid test: a worker whose " + "two pids are equal was not retired"); + goto done; + } + + if (nxt_slow_path(nxt_main_rcp_router_told(thr, router_port, twin_pid, + "equal pids") != NXT_OK)) + { + goto done; + } + + ret = nxt_main_rcp_report(thr, task, mp, proto_pid, NXT_MAIN_RCP_NS_PID, + 1); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + ret = NXT_ERROR; + child_port = NULL; + + if (nxt_slow_path(nxt_runtime_process_find(rt, child_pid) != NULL)) { + nxt_log_alert(thr->log, "main remove child pid test: the record of a " + "reported worker outlived the report"); + goto done; + } + + if (nxt_slow_path(nxt_runtime_process_find(rt, other_pid) != other)) { + nxt_log_alert(thr->log, "main remove child pid test: the report took " + "a sibling's record with it"); + goto done; + } + + if (nxt_slow_path(nxt_main_rcp_router_told(thr, router_port, child_pid, + "a reported worker") != NXT_OK)) + { + goto done; + } + + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "main remove child pid test passed"); + + ret = NXT_OK; + +done: + + /* + * The ports carry pools of their own; nxt_port_close() is a no-op on the + * -1 descriptors and nxt_runtime_port_remove() drops the last reference. + * nxt_runtime_process_free() unlinks a child still in a children queue. + */ + + if (router_port != NULL) { + nxt_port_test_run_error_handler(task, router_port); + } + + nxt_main_rcp_port_release(task, router_port); + nxt_main_rcp_port_release(task, child_port); + nxt_main_rcp_port_release(task, twin_port); + nxt_main_rcp_port_release(task, other_port); + nxt_main_rcp_port_release(task, proto_port); + + thr->runtime = saved_rt; + thr->engine = saved_engine; + + nxt_work_queue_cache_destroy(&engine.work_queue_cache); + + nxt_thread_mutex_destroy(&rt->processes_mutex); + + nxt_mp_destroy(mp); + + return ret; +} + +#else + +nxt_int_t +nxt_main_remove_child_pid_test(nxt_thread_t *thr) +{ + nxt_log_error(NXT_LOG_NOTICE, thr->log, "main remove child pid test " + "skipped: no sender credentials on this platform"); + + return NXT_OK; +} + +#endif diff --git a/src/test/nxt_main_start_process_reply_test.c b/src/test/nxt_main_start_process_reply_test.c new file mode 100644 index 000000000..f751f8431 --- /dev/null +++ b/src/test/nxt_main_start_process_reply_test.c @@ -0,0 +1,828 @@ +/* + * Copyright (C) F5, Inc. + */ + +/* + * Regression test for issue #257: nxt_main_start_process_handler() + * (src/nxt_main_process.c) had exits above the fork() that returned + * without answering the router. + * + * START_PROCESS carries the stream of an RPC the router has already armed + * in nxt_router_start_app_process_handler(), having incremented + * app->proto_port_requests for it. Only a reply retires that RPC: with + * none, neither nxt_router_app_port_ready() nor nxt_router_app_port_error() + * ever runs, the counter is never cleared, and every later start for the + * application parks on a prototype that is not coming. The three exits + * covered here -- no router port, a sender that is not the router, and a + * process record that cannot be allocated -- all used to "goto close_fds", + * which closes the descriptors and returns in silence. + * + * The reply is observed rather than inferred: the fixture ports carry no + * queue and leave write_ready unset, so nxt_port_socket_write() takes the + * nxt_port_msg_chk_insert() path and the message stays on port->messages + * for the test to read -- the arrangement src/test/nxt_router_start_fail_ + * test.c and src/test/nxt_router_proto_wedge_test.c already rely on. Each + * case requires exactly one message, of type RPC_ERROR and on the stream + * the incoming message named: a handler that answered twice would retire + * an RPC that is no longer the caller's. + * + * The foreign-sender case additionally pins down where the answer goes. + * The reply port is looked up by the kernel-validated sender pid, not by + * the self-declared msg->port_msg.pid, so a forged START_PROCESS can only + * ever cancel a stream of the forger's own; that case gives the two pids + * different values and requires the router's port to stay untouched. + * + * The prototype's own START_PROCESS handler is covered here too. It is the + * other end of the same message -- the router sends one to main to get a + * prototype and one to the prototype to get a worker + * (src/nxt_router.c:469 and :3379) -- and until issue #270 it checked + * nothing at all, although every worker it forks inherits the write end of + * the prototype's port. Its cases are refusals only, and each is run in a + * forked child: a handler that does not refuse goes on to fork a worker + * from a fixture that has no application loaded, and containing that keeps + * one broken gate from taking the whole test binary down. The parent + * reports a child that died as the fall-through it is. + * + * Not covered: the nxt_mp_create() failure a few lines below the process + * allocation. Arming it needs a malloc-failure hook that does not exist, + * and the sites are not independent -- both reach "failed:" through the + * same "goto", so the process-allocation case already exercises the reply + * they share. What it does not exercise is the "process != NULL" guard at + * "failed:", which only the mem_pool site can reach. + */ + +#include +#include +#include +#include +#include +#include +#include +#include "nxt_tests.h" + +#include +#include + + +typedef void (*nxt_main_start_process_reply_test_handler_t)(nxt_task_t *task, + nxt_port_recv_msg_t *msg); + + +static nxt_int_t nxt_main_start_process_reply_test_answer(nxt_thread_t *thr, + nxt_task_t *task, nxt_port_recv_msg_t *msg, + nxt_main_start_process_reply_test_handler_t handler, + nxt_port_t *reply_port, nxt_port_t *silent_port, const char *name); + + +/* + * Run the handler once and require that it answered: exactly one RPC_ERROR + * on reply_port, carrying the stream of the incoming message, and nothing + * at all on silent_port (which may be NULL when there is no second port to + * rule out). Both ports are left drained, so the next case starts from + * empty queues. + */ +static nxt_int_t +nxt_main_start_process_reply_test_case(nxt_thread_t *thr, nxt_task_t *task, + nxt_port_recv_msg_t *msg, nxt_port_t *reply_port, nxt_port_t *silent_port, + const char *name) +{ + return nxt_main_start_process_reply_test_answer(thr, task, msg, + nxt_main_test_run_start_process_handler, + reply_port, silent_port, name); +} + + +/* The same check for any START_PROCESS handler. */ + +static nxt_int_t +nxt_main_start_process_reply_test_answer(nxt_thread_t *thr, nxt_task_t *task, + nxt_port_recv_msg_t *msg, + nxt_main_start_process_reply_test_handler_t handler, + nxt_port_t *reply_port, nxt_port_t *silent_port, const char *name) +{ + nxt_fd_t fd0, fd1; + nxt_int_t ret; + nxt_uint_t n; + nxt_queue_link_t *link; + nxt_port_send_msg_t *sent; + + ret = NXT_ERROR; + sent = NULL; + n = 0; + + fd0 = open("/dev/null", O_RDONLY); + fd1 = open("/dev/null", O_RDONLY); + + if (nxt_slow_path(fd0 == -1 || fd1 == -1)) { + nxt_log_alert(thr->log, "main start process reply test: %s failed to " + "open the descriptors", name); + goto done; + } + + msg->fd[0] = fd0; + msg->fd[1] = fd1; + + handler(task, msg); + + /* + * Both exits own the descriptors the message carried, and both spellings + * of the failure tail closed them -- so this is not what discriminates + * the fix, only a guard against the cleanup below closing a number that + * has already been reused. + */ + if (msg->fd[0] == -1 && !nxt_test_fd_is_open(fd0)) + { + fd0 = -1; + } + + if (msg->fd[1] == -1 && !nxt_test_fd_is_open(fd1)) + { + fd1 = -1; + } + + if (nxt_slow_path(fd0 != -1 || fd1 != -1)) { + nxt_log_alert(thr->log, "main start process reply test: %s leaked a " + "descriptor", name); + goto done; + } + + for (link = nxt_queue_first(&reply_port->messages); + link != nxt_queue_tail(&reply_port->messages); + link = nxt_queue_next(link)) + { + n++; + sent = nxt_queue_link_data(link, nxt_port_send_msg_t, link); + } + + if (nxt_slow_path(n != 1)) { + nxt_log_alert(thr->log, "main start process reply test: %s left %ui " + "replies on the reply port (expected 1) -- the " + "START_PROCESS RPC is stranded", name, n); + goto done; + } + + if (nxt_slow_path(sent->port_msg.type != _NXT_PORT_MSG_RPC_ERROR)) { + nxt_log_alert(thr->log, "main start process reply test: %s answered " + "with message type %d (expected %d, RPC_ERROR)", name, + (int) sent->port_msg.type, + (int) _NXT_PORT_MSG_RPC_ERROR); + goto done; + } + + if (nxt_slow_path(sent->port_msg.last != 1)) { + nxt_log_alert(thr->log, "main start process reply test: %s answered " + "with a non-final message, so the RPC stays armed", + name); + goto done; + } + + if (nxt_slow_path(sent->port_msg.stream != msg->port_msg.stream)) { + nxt_log_alert(thr->log, "main start process reply test: %s answered " + "on stream %uD (expected %uD)", name, + sent->port_msg.stream, msg->port_msg.stream); + goto done; + } + + if (silent_port != NULL && !nxt_queue_is_empty(&silent_port->messages)) { + nxt_log_alert(thr->log, "main start process reply test: %s answered " + "on a port of the process it was not sent by", name); + goto done; + } + + ret = NXT_OK; + +done: + + /* + * Close only what is still open: a consumed descriptor is gone already, + * and closing its number again could reach an unrelated file. + */ + if (fd0 != -1 && nxt_test_fd_is_open(fd0)) { + (void) close(fd0); + } + + if (fd1 != -1 && nxt_test_fd_is_open(fd1)) { + (void) close(fd1); + } + + /* + * A queued message is malloc'd and takes a port reference + * (nxt_port_msg_chk_insert()). Left alone the port never reaches + * nxt_port_mp_cleanup(), so its pool leaks and the assertions there + * never run -- the same teardown src/test/nxt_router_proto_wedge_test.c + * performs, and for the same reason. + */ + nxt_port_test_run_error_handler(task, reply_port); + + if (silent_port != NULL) { + nxt_port_test_run_error_handler(task, silent_port); + } + + return ret; +} + + +/* + * Run the handler once where there is no port to answer on: the sender is a + * pid the runtime has never seen, so nxt_runtime_port_find() returns NULL + * and the handler can only log. Nothing may be queued on any port, and the + * descriptors the message carried must still be released. + */ +static nxt_int_t +nxt_main_start_process_reply_test_no_reply_case(nxt_thread_t *thr, + nxt_task_t *task, nxt_port_recv_msg_t *msg, nxt_port_t *port_a, + nxt_port_t *port_b, const char *name) +{ + nxt_fd_t fd0, fd1; + nxt_int_t ret; + + ret = NXT_ERROR; + + fd0 = open("/dev/null", O_RDONLY); + fd1 = open("/dev/null", O_RDONLY); + + if (nxt_slow_path(fd0 == -1 || fd1 == -1)) { + nxt_log_alert(thr->log, "main start process reply test: %s failed to " + "open the descriptors", name); + goto done; + } + + msg->fd[0] = fd0; + msg->fd[1] = fd1; + + nxt_main_test_run_start_process_handler(task, msg); + + if (msg->fd[0] == -1 && !nxt_test_fd_is_open(fd0)) + { + fd0 = -1; + } + + if (msg->fd[1] == -1 && !nxt_test_fd_is_open(fd1)) + { + fd1 = -1; + } + + if (nxt_slow_path(fd0 != -1 || fd1 != -1)) { + nxt_log_alert(thr->log, "main start process reply test: %s leaked a " + "descriptor", name); + goto done; + } + + if (nxt_slow_path(!nxt_queue_is_empty(&port_a->messages) + || (port_b != NULL + && !nxt_queue_is_empty(&port_b->messages)))) + { + nxt_log_alert(thr->log, "main start process reply test: %s answered " + "on a port, but no port belongs to the sender", name); + goto done; + } + + ret = NXT_OK; + +done: + + if (fd0 != -1 && nxt_test_fd_is_open(fd0)) { + (void) close(fd0); + } + + if (fd1 != -1 && nxt_test_fd_is_open(fd1)) { + (void) close(fd1); + } + + nxt_port_test_run_error_handler(task, port_a); + + if (port_b != NULL) { + nxt_port_test_run_error_handler(task, port_b); + } + + return ret; +} + +#if (NXT_USE_CMSG_PID) + +/* + * One refusal case for nxt_proto_start_process_handler(), run inside a + * forked child. + * + * The child does the whole check and reports through its exit status: 0 when + * the handler refused, 1 when it was reached and left a trace it should not + * have. A handler with no gate does not return at all -- it allocates a + * process and reads nxt_app_conf, which no fixture here sets -- so the + * parent treats a signalled child as the failure it is, and names it. + * + * What "refused" means, in the order the child tests it: nothing was queued + * on any port (a refusal must not answer, because the reply is addressed by + * the sender-chosen msg->port_msg.pid and answering would let a forger + * cancel a stream of its choosing), and both descriptors the message carried + * were closed (this handler owns them; the port read loop reclaims nothing). + */ +static nxt_int_t +nxt_main_start_process_reply_test_proto_case(nxt_thread_t *thr, + nxt_task_t *task, nxt_port_recv_msg_t *msg, nxt_port_t *port_a, + nxt_port_t *port_b, const char *name) +{ + int status; + pid_t child; + nxt_fd_t fd0, fd1; + nxt_int_t ret; + + fd0 = open("/dev/null", O_RDONLY); + fd1 = open("/dev/null", O_RDONLY); + + if (nxt_slow_path(fd0 == -1 || fd1 == -1)) { + nxt_log_alert(thr->log, "main start process reply test: %s failed to " + "open the descriptors", name); + + if (fd0 != -1) { + (void) close(fd0); + } + + if (fd1 != -1) { + (void) close(fd1); + } + + return NXT_ERROR; + } + + msg->fd[0] = fd0; + msg->fd[1] = fd1; + + child = fork(); + + if (nxt_slow_path(child == -1)) { + nxt_log_alert(thr->log, "main start process reply test: %s failed to " + "fork %E", name, nxt_errno); + (void) close(fd0); + (void) close(fd1); + + return NXT_ERROR; + } + + if (child == 0) { + ret = NXT_OK; + + nxt_proto_test_run_start_process_handler(task, msg); + + if (!nxt_queue_is_empty(&port_a->messages) + || (port_b != NULL && !nxt_queue_is_empty(&port_b->messages))) + { + nxt_log_alert(thr->log, "main start process reply test: %s was " + "answered; a refused START_PROCESS must not reply, " + "because the reply is addressed by the pid the " + "sender chose", name); + ret = NXT_ERROR; + } + + if (msg->fd[0] != -1 || msg->fd[1] != -1 + || nxt_test_fd_is_open(fd0) + || nxt_test_fd_is_open(fd1)) + { + nxt_log_alert(thr->log, "main start process reply test: %s leaked " + "a descriptor", name); + ret = NXT_ERROR; + } + + /* + * _exit(), not exit(): the child shares the parent's stdio and its + * pools, and must run no destructor or atexit handler of theirs. + */ + _exit(ret == NXT_OK ? 0 : 1); + } + + /* The child owns them now; this frame only has to stop naming them. */ + (void) close(fd0); + (void) close(fd1); + + msg->fd[0] = -1; + msg->fd[1] = -1; + + while (waitpid(child, &status, 0) == -1) { + if (nxt_errno != NXT_EINTR) { + nxt_log_alert(thr->log, "main start process reply test: %s failed " + "to reap the child %E", name, nxt_errno); + return NXT_ERROR; + } + } + + if (nxt_slow_path(!WIFEXITED(status))) { + nxt_log_alert(thr->log, "main start process reply test: %s did not " + "refuse -- the handler ran past the sender check and " + "into the start path, and died there on signal %d", + name, WTERMSIG(status)); + return NXT_ERROR; + } + + if (nxt_slow_path(WEXITSTATUS(status) != 0)) { + return NXT_ERROR; + } + + return NXT_OK; +} + +#endif + + +/* + * A START_PROCESS from the router reaches a prototype after its QUIT. On + * shutdown this happens when the exiting prototype reports the exit of a + * worker, and the router starts a replacement for that worker. A worker + * forked at that point gets no QUIT from anyone, and the prototype waits for + * it for ever. The prototype must refuse the start. It must answer, so + * that the router retires its RPC. + * + * The case runs in a forked child, like the refusal cases above. A handler + * that does not refuse forks a worker from a fixture that has no application + * loaded. + */ +static nxt_int_t +nxt_main_start_process_reply_test_proto_exiting(nxt_thread_t *thr, + nxt_task_t *task, nxt_port_recv_msg_t *msg, nxt_port_t *reply_port, + nxt_port_t *silent_port) +{ + int status; + pid_t child; + nxt_int_t ret; + + child = fork(); + + if (nxt_slow_path(child == -1)) { + nxt_log_alert(thr->log, "main start process reply test: an exiting " + "prototype failed to fork %E", nxt_errno); + return NXT_ERROR; + } + + if (child == 0) { + nxt_proto_test_set_exiting(1); + + ret = nxt_main_start_process_reply_test_answer(thr, task, msg, + nxt_proto_test_run_start_process_handler, + reply_port, silent_port, + "an exiting prototype"); + + _exit(ret == NXT_OK ? 0 : 1); + } + + while (waitpid(child, &status, 0) == -1) { + if (nxt_errno != NXT_EINTR) { + nxt_log_alert(thr->log, "main start process reply test: an " + "exiting prototype failed to reap the child %E", + nxt_errno); + return NXT_ERROR; + } + } + + if (nxt_slow_path(!WIFEXITED(status))) { + nxt_log_alert(thr->log, "main start process reply test: an exiting " + "prototype did not refuse -- the handler went on to " + "start a worker, and died there on signal %d", + WTERMSIG(status)); + return NXT_ERROR; + } + + if (nxt_slow_path(WEXITSTATUS(status) != 0)) { + nxt_log_alert(thr->log, "main start process reply test: an exiting " + "prototype failed, exit status %d", + WEXITSTATUS(status)); + return NXT_ERROR; + } + + return NXT_OK; +} + + +nxt_int_t +nxt_main_start_process_reply_test(nxt_thread_t *thr) +{ + nxt_mp_t *mp; + nxt_int_t ret; + nxt_task_t *task; + nxt_port_t *router_port, *foreign_port; + nxt_runtime_t *rt, *saved_rt; + nxt_event_engine_t engine, *saved_engine; + nxt_port_recv_msg_t msg; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "main start process reply test started"); + + ret = NXT_ERROR; + router_port = NULL; + foreign_port = NULL; + + task = thr->task; + task->thread = thr; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + rt = nxt_mp_zalloc(mp, sizeof(nxt_runtime_t)); + if (nxt_slow_path(rt == NULL)) { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + rt->mem_pool = mp; + + if (nxt_slow_path(nxt_thread_mutex_create(&rt->processes_mutex) != NXT_OK)) + { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + /* + * Only the members the reached code touches are set; the engine is what + * the queued-message teardown needs (see src/test/nxt_router_new_port_ + * test.c for the same fixture). + */ + nxt_memzero(&engine, sizeof(engine)); + nxt_work_queue_cache_create(&engine.work_queue_cache, 1024); + engine.fast_work_queue.cache = &engine.work_queue_cache; + nxt_work_queue_name(&engine.fast_work_queue, "fast"); + engine.mem_pool = mp; + + saved_rt = thr->runtime; + saved_engine = thr->engine; + thr->runtime = rt; + thr->engine = &engine; + rt->main_engine = &engine; + + /* + * The router's port: both the one the handler checks the sender + * against, and -- since the router is the sender in every case but the + * forgery below -- the one the reply is written to. No queue and + * write_ready unset, so a reply is queued rather than written. + */ + + router_port = nxt_port_new(task, 0, nxt_pid, NXT_PROCESS_ROUTER); + if (nxt_slow_path(router_port == NULL)) { + goto done; + } + + router_port->pair[0] = -1; + router_port->pair[1] = -1; + router_port->socket.fd = -1; + + /* + * What nxt_runtime_port_add() does, spelled out: that helper is static + * to src/nxt_runtime.c, and it also takes a port reference the fixture + * would then have to give back. The port stays owned by this frame. + */ + rt->port_by_type[NXT_PROCESS_ROUTER] = router_port; + + if (nxt_slow_path(nxt_port_hash_add(&rt->ports, router_port) != NXT_OK)) { + goto done; + } + + nxt_memzero(&msg, sizeof(nxt_port_recv_msg_t)); + + msg.port_msg.pid = nxt_pid; + msg.port_msg.reply_port = router_port->id; + msg.port_msg.type = _NXT_PORT_MSG_START_PROCESS; +#if (NXT_USE_CMSG_PID) + msg.cmsg_pid = nxt_pid; +#endif + + /* + * No router port at all. The runtime still knows the sender's port, so + * there is something to answer on -- the handler used to return without + * doing so. + */ + + rt->port_by_type[NXT_PROCESS_ROUTER] = NULL; + + msg.port_msg.stream = 0x11111111; + + ret = nxt_main_start_process_reply_test_case(thr, task, &msg, router_port, + NULL, "a missing router " + "port"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + rt->port_by_type[NXT_PROCESS_ROUTER] = router_port; + +#if (NXT_USE_CMSG_PID) + + /* + * A sender that is not the router. The message claims to come from the + * router -- port_msg.pid is filled in by the sender and is not + * authenticated -- while the credential the kernel attached names + * somebody else. The start is refused, and the refusal has to be + * reported to the process that actually sent it: answering the pid the + * message claimed would let a worker cancel a stream of the router's + * choosing. + */ + + foreign_port = nxt_port_new(task, router_port->id, nxt_pid + 1, + NXT_PROCESS_APP); + if (nxt_slow_path(foreign_port == NULL)) { + ret = NXT_ERROR; + goto done; + } + + foreign_port->pair[0] = -1; + foreign_port->pair[1] = -1; + foreign_port->socket.fd = -1; + + if (nxt_slow_path(nxt_port_hash_add(&rt->ports, foreign_port) != NXT_OK)) { + ret = NXT_ERROR; + goto done; + } + + msg.cmsg_pid = nxt_pid + 1; + msg.port_msg.stream = 0x22222222; + + ret = nxt_main_start_process_reply_test_case(thr, task, &msg, + foreign_port, router_port, + "a foreign sender"); + + msg.cmsg_pid = nxt_pid; + + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + +#endif + + /* + * The process record cannot be allocated. This is the exit furthest + * into the handler that still owes the router an answer, and the one + * that leaves "process" NULL for the guard at "failed:". + */ + + nxt_main_test_process_new_failures(1); + + msg.port_msg.stream = 0x33333333; + + ret = nxt_main_start_process_reply_test_case(thr, task, &msg, router_port, + foreign_port, + "a failed process " + "allocation"); + + nxt_main_test_process_new_failures(0); + + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + /* + * Nothing to answer on. The sender is a pid the runtime knows no port + * for, so the reply-port lookup fails; the handler logs and must leave + * every queue untouched rather than answering somebody else. Both pid + * fields move together so the case holds however + * nxt_recv_msg_cmsg_pid() is defined. + */ + + msg.port_msg.pid = nxt_pid + 2; +#if (NXT_USE_CMSG_PID) + msg.cmsg_pid = nxt_pid + 2; +#endif + msg.port_msg.stream = 0x44444444; + + ret = nxt_main_start_process_reply_test_no_reply_case(thr, task, &msg, + router_port, + foreign_port, + "an unknown " + "sender"); + + msg.port_msg.pid = nxt_pid; +#if (NXT_USE_CMSG_PID) + msg.cmsg_pid = nxt_pid; +#endif + + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + +#if (NXT_USE_CMSG_PID) + + /* + * The prototype's START_PROCESS handler, issue #270. Each case is a + * refusal; the fixture is the one above, read as the prototype's own + * runtime -- rt->port_by_type[NXT_PROCESS_ROUTER] is the router port the + * prototype inherited from main, which nxt_proc_keep_matrix[] keeps. + */ + + /* + * A worker forging a START_PROCESS. It claims to be the router in the + * one field it controls, and the credential names it. This is the whole + * exposure: nxt_process_close_ports() keeps the parent's port + * unconditionally, so every worker holds the write end of the port this + * message arrived on. + */ + + msg.cmsg_pid = nxt_pid + 1; + msg.port_msg.stream = 0x55555555; + + ret = nxt_main_start_process_reply_test_proto_case(thr, task, &msg, + router_port, + foreign_port, + "a foreign sender to " + "the prototype"); + + msg.cmsg_pid = nxt_pid; + + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + /* + * No router port to compare against. The prototype cannot tell who sent + * this, and refusing is the only safe answer -- the same conclusion + * nxt_main_start_process_handler() reaches, minus the reply it can + * address and the prototype cannot. + */ + + rt->port_by_type[NXT_PROCESS_ROUTER] = NULL; + + msg.port_msg.stream = 0x66666666; + + ret = nxt_main_start_process_reply_test_proto_case(thr, task, &msg, + router_port, + foreign_port, + "a missing router port " + "in the prototype"); + + rt->port_by_type[NXT_PROCESS_ROUTER] = router_port; + + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + /* + * A pid-isolated prototype. There the router is in an ancestor pid + * namespace and has no pid at all in this one, so the kernel writes a + * credential of 0 and the router's global pid is the wrong thing to + * compare against; the prototype's own workers, forked with no clone + * flags of their own, are inside the namespace and carry non-zero + * namespace-local pids. This case is such a worker -- a credential that + * is neither 0 nor the router's -- and pins the arm down from the side + * that matters: a gate that simply skipped the check when isolated would + * pass every other case here and fail this one. + */ + + rt->is_pid_isolated = 1; + + msg.cmsg_pid = 2; + msg.port_msg.stream = 0x77777777; + + ret = nxt_main_start_process_reply_test_proto_case(thr, task, &msg, + router_port, + foreign_port, + "a worker inside a " + "pid-isolated " + "prototype"); + + rt->is_pid_isolated = 0; + msg.cmsg_pid = nxt_pid; + + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + +#endif + + msg.port_msg.stream = 0x88888888; + + ret = nxt_main_start_process_reply_test_proto_exiting(thr, task, &msg, + router_port, + foreign_port); + + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "main start process reply test passed"); + + ret = NXT_OK; + +done: + + /* Whatever a failed case left armed must not follow the test out. */ + nxt_main_test_process_new_failures(0); + + if (foreign_port != NULL) { + (void) nxt_port_hash_remove(&rt->ports, foreign_port); + + nxt_port_use(task, foreign_port, -1); + } + + if (router_port != NULL) { + (void) nxt_port_hash_remove(&rt->ports, router_port); + + nxt_port_use(task, router_port, -1); + } + + thr->engine = saved_engine; + thr->runtime = saved_rt; + + nxt_work_queue_cache_destroy(&engine.work_queue_cache); + nxt_thread_mutex_destroy(&rt->processes_mutex); + nxt_mp_destroy(mp); + + return ret; +} diff --git a/src/test/nxt_main_store_test.c b/src/test/nxt_main_store_test.c new file mode 100644 index 000000000..f8e1da78b --- /dev/null +++ b/src/test/nxt_main_store_test.c @@ -0,0 +1,1404 @@ +/* + * Copyright (C) F5, Inc. + */ + +/* + * Tests of the state store child of main (src/nxt_main_process.c, + * https://github.com/freeunitorg/freeunit/issues/516). + * + * The test drives the scheduler directly. An NXT_TESTS hook makes each + * store child sleep before it stores, so the test can act while a child + * runs. The test reaps each child with waitpid() and gives the status to + * the handler that the SIGCHLD handler of main calls. + * + * Cases: + * + * - While main runs, a newer store waits for the running store child, + * which finishes its store. + * - At exit, main stops a running store child with SIGKILL when a newer + * store waits: when the exit starts, and when a store comes during + * the exit. The killed child is not logged as a failed store, it does + * not set nxt_conf_ver, and the newer store runs when the child is + * reaped. + * - A child that exited before the signal arrived keeps its exit code: + * its store counts, and nxt_conf_ver is set. + * - The store child closes the descriptors it inherits from 3 upwards + * before it stores. The test holds the read end of a pipe and gives + * the write end to the child only. The read end must see the end of + * file while the child still sleeps. + * - Stores start in the order they came. A conf.json store that + * replaces a pending one takes its place. So a DELETE never runs + * before an older conf.json store. + * - Certificate bundles (with TLS). nxt_cert_store_put_handler() gives + * the bundle to a store child and answers the controller only when the + * child exits: nothing is on the reply port while the child runs. Two + * uploads in quick succession both land, each gets its own answer, in + * order. A conf.json store that arrives between them runs between + * them, and it does not kill the certificate child, even when main + * exits. A DELETE that follows a PUT of the same bundle runs after + * it, so the bundle is gone at the end. + */ + +#include +#include +#include +#include +#include +#include +#if (NXT_TLS) +#include +#endif +#include "nxt_tests.h" + +#include +#include +#include + + +#define NXT_MAIN_STORE_TEST_DELAY 500 + + +#define nxt_main_store_test_fail(thr, ...) \ + do { \ + nxt_log_alert((thr)->log, "main store test: " __VA_ARGS__); \ + goto done; \ + } while (0) + + +typedef struct { + char dir[NXT_MAX_PATH_LEN]; + char conf[NXT_MAX_PATH_LEN]; + char conf_tmp[NXT_MAX_PATH_LEN]; + char ver[NXT_MAX_PATH_LEN]; + char ver_tmp[NXT_MAX_PATH_LEN]; + char certs[NXT_MAX_PATH_LEN]; + char bundle[NXT_MAX_PATH_LEN]; +} nxt_main_store_test_paths_t; + + +static nxt_uint_t nxt_main_store_test_alerts; +static nxt_log_handler_t nxt_main_store_test_next_handler; + + +/* Counts the alerts of this process, and logs as before. */ +static void nxt_cdecl +nxt_main_store_test_log(nxt_uint_t level, nxt_log_t *log, const char *fmt, + ...) +{ + u_char *p; + va_list args; + u_char msg[NXT_MAX_ERROR_STR]; + + if (level == NXT_LOG_ALERT) { + nxt_main_store_test_alerts++; + } + + va_start(args, fmt); + p = nxt_vsprintf(msg, msg + sizeof(msg), fmt, args); + va_end(args); + + nxt_main_store_test_next_handler(level, log, "%*s", (size_t) (p - msg), + msg); +} + + +/* The mapping that the scheduler takes and unmaps. */ +static u_char * +nxt_main_store_test_map(const char *content, size_t *size) +{ + u_char *p; + + *size = nxt_strlen(content); + + p = nxt_mem_mmap(NULL, *size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANON, -1, 0); + if (nxt_slow_path(p == MAP_FAILED)) { + return NULL; + } + + nxt_memcpy(p, content, *size); + + return p; +} + + +static nxt_int_t +nxt_main_store_test_schedule(nxt_task_t *task, const char *content) +{ + u_char *p; + size_t size; + + p = nxt_main_store_test_map(content, &size); + if (nxt_slow_path(p == NULL)) { + return NXT_ERROR; + } + + nxt_main_test_store_schedule(task, p, size); + + return NXT_OK; +} + + +static int +nxt_main_store_test_wait(nxt_pid_t pid, int *status) +{ + while (waitpid(pid, status, 0) != pid) { + if (nxt_errno != NXT_EINTR) { + return -1; + } + } + + return 0; +} + + +static nxt_bool_t +nxt_main_store_test_holds(const char *name, const char *expected) +{ + int fd; + char buf[256]; + size_t len; + ssize_t n; + + fd = open(name, O_RDONLY); + if (fd == -1) { + return 0; + } + + n = read(fd, buf, sizeof(buf)); + + close(fd); + + len = nxt_strlen(expected); + + return (n == (ssize_t) len && memcmp(buf, expected, len) == 0); +} + + +/* Stop and reap a store child that a failed case left behind. */ +static void +nxt_main_store_test_reap_all(nxt_task_t *task) +{ + int status; + nxt_pid_t pid; + + nxt_main_test_store_set_delay(0); + nxt_main_test_store_set_exiting(task, 0); + + for ( ;; ) { + pid = nxt_main_test_store_pid(); + if (pid == 0) { + return; + } + + (void) kill(pid, SIGKILL); + + if (nxt_main_store_test_wait(pid, &status) != 0) { + return; + } + + (void) nxt_main_test_store_exited(task, pid, status); + } +} + + +/* + * While main runs, a newer store does not stop the running child. Stores + * that come faster than one store takes would otherwise keep conf.json old + * until they stop. + */ +static nxt_int_t +nxt_main_store_test_running(nxt_thread_t *thr, nxt_task_t *task, + nxt_main_store_test_paths_t *paths) +{ + int status; + nxt_int_t ret; + nxt_pid_t first, second; + + ret = NXT_ERROR; + + nxt_main_store_test_alerts = 0; + + nxt_main_test_store_set_delay(NXT_MAIN_STORE_TEST_DELAY); + + if (nxt_main_store_test_schedule(task, "{\"r\":1}") != NXT_OK) { + nxt_main_store_test_fail(thr, "cannot map the first store"); + } + + first = nxt_main_test_store_pid(); + if (first == 0) { + nxt_main_store_test_fail(thr, "the first store started no child"); + } + + if (nxt_main_store_test_schedule(task, "{\"r\":2}") != NXT_OK) { + nxt_main_store_test_fail(thr, "cannot map the second store"); + } + + nxt_main_test_store_set_delay(0); + + if (nxt_main_store_test_wait(first, &status) != 0) { + nxt_main_store_test_fail(thr, "waitpid(%PI) failed %E", first, + nxt_errno); + } + + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + nxt_main_store_test_fail(thr, "the running store child %PI did not " + "finish while main runs: status 0x%Xi", + first, (nxt_uint_t) status); + } + + (void) nxt_main_test_store_exited(task, first, status); + + if (!nxt_main_store_test_holds(paths->conf, "{\"r\":1}")) { + nxt_main_store_test_fail(thr, "\"%s\" does not hold the first " + "configuration", paths->conf); + } + + second = nxt_main_test_store_pid(); + if (second == 0 || second == first) { + nxt_main_store_test_fail(thr, "the pending store did not start"); + } + + if (nxt_main_store_test_wait(second, &status) != 0) { + nxt_main_store_test_fail(thr, "waitpid(%PI) failed %E", second, + nxt_errno); + } + + (void) nxt_main_test_store_exited(task, second, status); + + if (!nxt_main_store_test_holds(paths->conf, "{\"r\":2}")) { + nxt_main_store_test_fail(thr, "\"%s\" does not hold the last " + "configuration", paths->conf); + } + + if (nxt_main_store_test_alerts != 0) { + nxt_main_store_test_fail(thr, "%ui alerts", nxt_main_store_test_alerts); + } + + ret = NXT_OK; + +done: + + nxt_main_store_test_reap_all(task); + + return ret; +} + + +/* Reap "pid" and require that main killed it, without an alert. */ +static nxt_int_t +nxt_main_store_test_killed(nxt_thread_t *thr, nxt_task_t *task, + nxt_pid_t pid) +{ + int status; + + if (nxt_main_store_test_wait(pid, &status) != 0) { + nxt_log_alert(thr->log, "main store test: waitpid(%PI) failed %E", + pid, nxt_errno); + return NXT_ERROR; + } + + if (!WIFSIGNALED(status) || WTERMSIG(status) != SIGKILL) { + nxt_log_alert(thr->log, "main store test: the superseded store child " + "%PI was not killed at exit: it exited with status " + "0x%Xi", pid, (nxt_uint_t) status); + return NXT_ERROR; + } + + if (!nxt_main_test_store_exited(task, pid, status)) { + nxt_log_alert(thr->log, "main store test: child %PI is not the store " + "child", pid); + return NXT_ERROR; + } + + if (nxt_main_store_test_alerts != 0) { + nxt_log_alert(thr->log, "main store test: the killed child %PI was " + "logged as a failure (%ui alerts)", pid, + nxt_main_store_test_alerts); + return NXT_ERROR; + } + + if (nxt_conf_ver != 0) { + nxt_log_alert(thr->log, "main store test: a killed child set " + "nxt_conf_ver"); + return NXT_ERROR; + } + + return NXT_OK; +} + + +/* + * At exit, a newer store stops the running child: first when the exit + * starts with a store pending, then when a store comes during the exit. + */ +static nxt_int_t +nxt_main_store_test_cancel(nxt_thread_t *thr, nxt_task_t *task, + nxt_main_store_test_paths_t *paths) +{ + int status; + char ver[NXT_INT_T_LEN + 1]; + nxt_int_t ret; + nxt_pid_t first, second, third; + struct stat st; + + ret = NXT_ERROR; + + nxt_conf_ver = 0; + nxt_main_store_test_alerts = 0; + + nxt_main_test_store_set_delay(NXT_MAIN_STORE_TEST_DELAY); + + if (nxt_main_store_test_schedule(task, "{\"a\":1}") != NXT_OK) { + nxt_main_store_test_fail(thr, "cannot map the first store"); + } + + first = nxt_main_test_store_pid(); + if (first == 0) { + nxt_main_store_test_fail(thr, "the first store started no child"); + } + + if (nxt_main_store_test_schedule(task, "{\"b\":2}") != NXT_OK) { + nxt_main_store_test_fail(thr, "cannot map the second store"); + } + + if (nxt_main_test_store_pid() != first) { + nxt_main_store_test_fail(thr, "a second child started while the " + "first one runs"); + } + + /* SIGTERM or SIGQUIT: the exit starts with a store pending. */ + nxt_main_test_store_set_exiting(task, 1); + + if (nxt_main_store_test_killed(thr, task, first) != NXT_OK) { + goto done; + } + + second = nxt_main_test_store_pid(); + if (second == 0 || second == first) { + nxt_main_store_test_fail(thr, "the pending store did not start when " + "the killed child was reaped"); + } + + /* A store that comes during the exit. */ + + nxt_main_test_store_set_delay(0); + + if (nxt_main_store_test_schedule(task, "{\"c\":3}") != NXT_OK) { + nxt_main_store_test_fail(thr, "cannot map the third store"); + } + + if (nxt_main_store_test_killed(thr, task, second) != NXT_OK) { + goto done; + } + + third = nxt_main_test_store_pid(); + if (third == 0 || third == second) { + nxt_main_store_test_fail(thr, "the store that came during the exit " + "did not start"); + } + + if (nxt_main_store_test_wait(third, &status) != 0) { + nxt_main_store_test_fail(thr, "waitpid(%PI) failed %E", third, + nxt_errno); + } + + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + nxt_main_store_test_fail(thr, "the last store failed: status " + "0x%Xi", (nxt_uint_t) status); + } + + (void) nxt_main_test_store_exited(task, third, status); + + if (nxt_main_test_store_pid() != 0) { + nxt_main_store_test_fail(thr, "a store child runs with nothing " + "pending"); + } + + if (nxt_conf_ver != NXT_VERNUM) { + nxt_main_store_test_fail(thr, "nxt_conf_ver is not set after the " + "version was stored"); + } + + if (!nxt_main_store_test_holds(paths->conf, "{\"c\":3}")) { + nxt_main_store_test_fail(thr, "\"%s\" does not hold the last " + "configuration", paths->conf); + } + + (void) nxt_sprintf((u_char *) ver, (u_char *) ver + sizeof(ver), "%d%Z", + NXT_VERNUM); + + if (!nxt_main_store_test_holds(paths->ver, ver)) { + nxt_main_store_test_fail(thr, "\"%s\" does not hold the version", + paths->ver); + } + + if (stat(paths->conf_tmp, &st) == 0 || stat(paths->ver_tmp, &st) == 0) { + nxt_main_store_test_fail(thr, "a temporary file was left behind"); + } + + if (nxt_main_store_test_alerts != 0) { + nxt_main_store_test_fail(thr, "%ui alerts", nxt_main_store_test_alerts); + } + + ret = NXT_OK; + +done: + + nxt_main_store_test_reap_all(task); + + return ret; +} + + +/* + * The child finished its store before main killed it. kill() of a zombie + * succeeds, and the exit code 0 still counts. + */ +static nxt_int_t +nxt_main_store_test_finished(nxt_thread_t *thr, nxt_task_t *task, + nxt_main_store_test_paths_t *paths) +{ + int status; + nxt_int_t ret; + nxt_pid_t first, second; + siginfo_t info; + + ret = NXT_ERROR; + + nxt_conf_ver = 0; + nxt_main_store_test_alerts = 0; + + nxt_main_test_store_set_delay(0); + + if (nxt_main_store_test_schedule(task, "{\"c\":3}") != NXT_OK) { + nxt_main_store_test_fail(thr, "cannot map the first store"); + } + + first = nxt_main_test_store_pid(); + if (first == 0) { + nxt_main_store_test_fail(thr, "the first store started no child"); + } + + /* Wait for the exit, and leave the child a zombie. */ + + nxt_memzero(&info, sizeof(info)); + + while (waitid(P_PID, first, &info, WEXITED | WNOWAIT) != 0) { + if (nxt_errno != NXT_EINTR) { + nxt_main_store_test_fail(thr, "waitid(%PI) failed %E", first, + nxt_errno); + } + } + + nxt_main_test_store_set_exiting(task, 1); + + if (nxt_main_store_test_schedule(task, "{\"d\":4}") != NXT_OK) { + nxt_main_store_test_fail(thr, "cannot map the second store"); + } + + if (nxt_main_store_test_wait(first, &status) != 0) { + nxt_main_store_test_fail(thr, "waitpid(%PI) failed %E", first, + nxt_errno); + } + + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + nxt_main_store_test_fail(thr, "the first store failed: status 0x%Xi", + (nxt_uint_t) status); + } + + (void) nxt_main_test_store_exited(task, first, status); + + if (nxt_conf_ver != NXT_VERNUM) { + nxt_main_store_test_fail(thr, "the store of a child that exited " + "with 0 did not count"); + } + + second = nxt_main_test_store_pid(); + if (second == 0) { + nxt_main_store_test_fail(thr, "the pending store did not start"); + } + + if (nxt_main_store_test_wait(second, &status) != 0) { + nxt_main_store_test_fail(thr, "waitpid(%PI) failed %E", second, + nxt_errno); + } + + (void) nxt_main_test_store_exited(task, second, status); + + if (!nxt_main_store_test_holds(paths->conf, "{\"d\":4}")) { + nxt_main_store_test_fail(thr, "\"%s\" does not hold the last " + "configuration", paths->conf); + } + + if (nxt_main_store_test_alerts != 0) { + nxt_main_store_test_fail(thr, "%ui alerts", nxt_main_store_test_alerts); + } + + ret = NXT_OK; + +done: + + nxt_main_store_test_reap_all(task); + + return ret; +} + + +static nxt_msec_t +nxt_main_store_test_now(void) +{ + struct timespec ts; + + (void) clock_gettime(CLOCK_MONOTONIC, &ts); + + return (nxt_msec_t) (ts.tv_sec * 1000 + ts.tv_nsec / 1000000); +} + + +/* + * The child closes the write end of the pipe before its delay. Before the + * change, it held the write end until it exited, after the delay. + */ +static nxt_int_t +nxt_main_store_test_fds(nxt_thread_t *thr, nxt_task_t *task, + nxt_main_store_test_paths_t *paths) +{ + int pp[2], n, status; + char c; + ssize_t r; + nxt_int_t ret; + nxt_pid_t pid; + nxt_msec_t start, elapsed; + struct pollfd pfd; + + ret = NXT_ERROR; + + pp[0] = -1; + pp[1] = -1; + + if (pipe(pp) != 0) { + nxt_main_store_test_fail(thr, "pipe() failed %E", nxt_errno); + } + + nxt_main_test_store_set_delay(NXT_MAIN_STORE_TEST_DELAY); + + start = nxt_main_store_test_now(); + + if (nxt_main_store_test_schedule(task, "{\"e\":5}") != NXT_OK) { + nxt_main_store_test_fail(thr, "cannot map the store"); + } + + nxt_main_test_store_set_delay(0); + + pid = nxt_main_test_store_pid(); + if (pid == 0) { + nxt_main_store_test_fail(thr, "the store started no child"); + } + + /* Now only the child can hold the write end. */ + + (void) close(pp[1]); + pp[1] = -1; + + pfd.fd = pp[0]; + pfd.events = POLLIN; + pfd.revents = 0; + + do { + n = poll(&pfd, 1, 4 * NXT_MAIN_STORE_TEST_DELAY); + } while (n == -1 && nxt_errno == NXT_EINTR); + + elapsed = nxt_main_store_test_now() - start; + + if (n != 1) { + nxt_main_store_test_fail(thr, "the pipe did not become readable: " + "poll() returned %d", n); + } + + r = read(pp[0], &c, 1); + + if (r != 0) { + nxt_main_store_test_fail(thr, "read() of the pipe returned %z", r); + } + + if (waitpid(pid, &status, WNOHANG) != 0 + || elapsed >= NXT_MAIN_STORE_TEST_DELAY / 2) + { + nxt_main_store_test_fail(thr, "the store child %PI held an inherited " + "descriptor until it exited (%M ms)", pid, + elapsed); + } + + if (nxt_main_store_test_wait(pid, &status) != 0) { + nxt_main_store_test_fail(thr, "waitpid(%PI) failed %E", pid, + nxt_errno); + } + + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + nxt_main_store_test_fail(thr, "the store failed: status 0x%Xi", + (nxt_uint_t) status); + } + + (void) nxt_main_test_store_exited(task, pid, status); + + if (!nxt_main_store_test_holds(paths->conf, "{\"e\":5}")) { + nxt_main_store_test_fail(thr, "\"%s\" does not hold the " + "configuration", paths->conf); + } + + ret = NXT_OK; + +done: + + nxt_main_store_test_reap_all(task); + + if (pp[0] != -1) { + (void) close(pp[0]); + } + + if (pp[1] != -1) { + (void) close(pp[1]); + } + + return ret; +} + + +static nxt_int_t +nxt_main_store_test_write(const char *name, const char *content) +{ + int fd; + size_t len; + ssize_t n; + + fd = open(name, O_WRONLY | O_CREAT | O_TRUNC, 0600); + if (fd == -1) { + return NXT_ERROR; + } + + len = nxt_strlen(content); + + n = write(fd, content, len); + + close(fd); + + return (n == (ssize_t) len) ? NXT_OK : NXT_ERROR; +} + + +/* Reap the store child that runs now, and require that it exited with 0. */ +static nxt_int_t +nxt_main_store_test_reap(nxt_thread_t *thr, nxt_task_t *task, + const char *what) +{ + int status; + nxt_pid_t pid; + + pid = nxt_main_test_store_pid(); + if (pid == 0) { + nxt_log_alert(thr->log, "main store test: no store child for %s", + what); + return NXT_ERROR; + } + + if (nxt_main_store_test_wait(pid, &status) != 0) { + nxt_log_alert(thr->log, "main store test: waitpid(%PI) failed %E", + pid, nxt_errno); + return NXT_ERROR; + } + + (void) nxt_main_test_store_exited(task, pid, status); + + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + nxt_log_alert(thr->log, "main store test: the store child of %s " + "ended with status 0x%Xi", what, (nxt_uint_t) status); + return NXT_ERROR; + } + + return NXT_OK; +} + + +/* "" exists. */ +static nxt_bool_t +nxt_main_store_test_exists(const char *dir, const char *name) +{ + char path[NXT_MAX_PATH_LEN]; + struct stat st; + + (void) nxt_sprintf((u_char *) path, (u_char *) path + sizeof(path), + "%s%s%Z", dir, name); + + return (stat(path, &st) == 0); +} + + +/* Queue a DELETE of "" that nobody waits for. */ +static nxt_int_t +nxt_main_store_test_submit_delete(nxt_task_t *task, const char *dir, + const char *name) +{ + nxt_str_t d, n; + nxt_main_store_job_t *job; + + d.start = (u_char *) dir; + d.length = nxt_strlen(dir); + n.start = (u_char *) name; + n.length = nxt_strlen(name); + + job = nxt_main_store_job_create(NXT_MAIN_STORE_DELETE, "file", &d, &n); + if (nxt_slow_path(job == NULL)) { + return NXT_ERROR; + } + + nxt_main_store_submit(task, job); + + return NXT_OK; +} + + +/* + * Stores start in the order they came. A conf.json store "A" runs. Then + * come a DELETE of "first", a conf.json store "B", a DELETE of "old", and a + * conf.json store "C" that replaces "B". The order must be A, the DELETE + * of "first", C, the DELETE of "old". Before the change, both DELETEs ran + * before C, so "old" was gone while conf.json still held A. + */ +static nxt_int_t +nxt_main_store_test_order(nxt_thread_t *thr, nxt_task_t *task, + nxt_main_store_test_paths_t *paths) +{ + nxt_int_t ret; + char dir[NXT_MAX_PATH_LEN], name[NXT_MAX_PATH_LEN]; + + ret = NXT_ERROR; + + nxt_main_store_test_alerts = 0; + + nxt_main_test_store_set_delay(0); + + (void) nxt_sprintf((u_char *) dir, (u_char *) dir + sizeof(dir), "%s/%Z", + paths->dir); + + (void) nxt_sprintf((u_char *) name, (u_char *) name + sizeof(name), + "%sfirst%Z", dir); + + if (nxt_main_store_test_write(name, "first") != NXT_OK) { + nxt_main_store_test_fail(thr, "cannot write \"%s\"", name); + } + + (void) nxt_sprintf((u_char *) name, (u_char *) name + sizeof(name), + "%sold%Z", dir); + + if (nxt_main_store_test_write(name, "old") != NXT_OK) { + nxt_main_store_test_fail(thr, "cannot write \"%s\"", name); + } + + /* The child of A runs; the rest waits for it. */ + + if (nxt_main_store_test_schedule(task, "{\"A\":1}") != NXT_OK) { + nxt_main_store_test_fail(thr, "cannot map A"); + } + + if (nxt_main_test_store_pid() == 0) { + nxt_main_store_test_fail(thr, "A started no child"); + } + + if (nxt_main_store_test_submit_delete(task, dir, "first") != NXT_OK) { + nxt_main_store_test_fail(thr, "cannot queue the DELETE of first"); + } + + if (nxt_main_store_test_schedule(task, "{\"B\":2}") != NXT_OK) { + nxt_main_store_test_fail(thr, "cannot map B"); + } + + if (nxt_main_store_test_submit_delete(task, dir, "old") != NXT_OK) { + nxt_main_store_test_fail(thr, "cannot queue the DELETE of old"); + } + + if (nxt_main_store_test_schedule(task, "{\"C\":3}") != NXT_OK) { + nxt_main_store_test_fail(thr, "cannot map C"); + } + + if (nxt_main_store_test_reap(thr, task, "A") != NXT_OK) { + goto done; + } + + if (!nxt_main_store_test_holds(paths->conf, "{\"A\":1}")) { + nxt_main_store_test_fail(thr, "\"%s\" does not hold A", paths->conf); + } + + /* The DELETE of "first" came before B, so it runs before C. */ + + if (nxt_main_store_test_reap(thr, task, "the second store") != NXT_OK) { + goto done; + } + + if (nxt_main_store_test_exists(dir, "first") + || !nxt_main_store_test_holds(paths->conf, "{\"A\":1}")) + { + nxt_main_store_test_fail(thr, "the second store was not the DELETE " + "of \"first\", which came before B"); + } + + /* C took the place of B, so it runs before the DELETE of "old". */ + + if (nxt_main_store_test_reap(thr, task, "the third store") != NXT_OK) { + goto done; + } + + if (!nxt_main_store_test_exists(dir, "old")) { + nxt_main_store_test_fail(thr, "\"%sold\" was deleted before C, which " + "took the place of the older B", dir); + } + + if (!nxt_main_store_test_holds(paths->conf, "{\"C\":3}")) { + nxt_main_store_test_fail(thr, "\"%s\" does not hold C after the " + "third store", paths->conf); + } + + if (nxt_main_store_test_reap(thr, task, "the DELETE of old") != NXT_OK) { + goto done; + } + + if (nxt_main_store_test_exists(dir, "old")) { + nxt_main_store_test_fail(thr, "\"%sold\" was not deleted", dir); + } + + if (nxt_main_test_store_pid() != 0) { + nxt_main_store_test_fail(thr, "a store child runs with nothing " + "pending"); + } + + if (nxt_main_store_test_alerts != 0) { + nxt_main_store_test_fail(thr, "%ui alerts", nxt_main_store_test_alerts); + } + + ret = NXT_OK; + +done: + + nxt_main_store_test_reap_all(task); + + (void) nxt_sprintf((u_char *) name, (u_char *) name + sizeof(name), + "%sfirst%Z", dir); + (void) unlink(name); + + (void) nxt_sprintf((u_char *) name, (u_char *) name + sizeof(name), + "%sold%Z", dir); + (void) unlink(name); + + return ret; +} + + +#if (NXT_TLS) + +/* A message of the controller, as the port layer gives it to main. */ +static void +nxt_main_store_test_msg(nxt_port_recv_msg_t *msg, nxt_buf_t *b, u_char *buf, + size_t len, nxt_port_t *port, uint32_t stream, nxt_fd_t fd) +{ + nxt_memzero(msg, sizeof(nxt_port_recv_msg_t)); + nxt_memzero(b, sizeof(nxt_buf_t)); + + b->mem.start = buf; + b->mem.pos = buf; + b->mem.free = buf + len; + b->mem.end = buf + len; + + msg->buf = b; + msg->fd[0] = fd; + msg->fd[1] = -1; + msg->port_msg.pid = nxt_pid; + msg->port_msg.reply_port = port->id; + msg->port_msg.stream = stream; +#if (NXT_USE_CMSG_PID) + msg->cmsg_pid = nxt_pid; +#endif +} + + +/* PUT /certificates/, as nxt_cert_store_put() sends it. */ +static nxt_int_t +nxt_main_store_test_put(nxt_task_t *task, nxt_main_store_test_paths_t *paths, + nxt_port_t *port, const char *name, const char *content, uint32_t stream) +{ + u_char *p; + size_t size; + nxt_fd_t fd; + nxt_buf_t b; + nxt_port_recv_msg_t msg; + u_char buf[64]; + + if (nxt_main_store_test_write(paths->bundle, content) != NXT_OK) { + return NXT_ERROR; + } + + fd = open(paths->bundle, O_RDONLY); + if (fd == -1) { + return NXT_ERROR; + } + + size = nxt_strlen(content); + + p = nxt_cpymem(buf, name, nxt_strlen(name) + 1); + p = nxt_cpymem(p, &size, sizeof(size_t)); + + nxt_main_store_test_msg(&msg, &b, buf, p - buf, port, stream, fd); + + nxt_cert_store_put_handler(task, &msg); + + if (msg.fd[0] != -1 && nxt_test_fd_is_open(fd)) { + (void) close(fd); + } + + return NXT_OK; +} + + +/* DELETE /certificates/, as nxt_cert_store_delete() sends it. */ +static void +nxt_main_store_test_delete(nxt_task_t *task, nxt_port_t *port, + const char *name) +{ + size_t len; + nxt_buf_t b; + nxt_port_recv_msg_t msg; + u_char buf[64]; + + len = nxt_strlen(name) + 1; + + nxt_memcpy(buf, name, len); + + nxt_main_store_test_msg(&msg, &b, buf, len, port, 0, -1); + + nxt_cert_store_delete_handler(task, &msg); +} + + +/* The number of answers on the port, and the last one. */ +static nxt_uint_t +nxt_main_store_test_replies(nxt_port_t *port, nxt_port_send_msg_t **last) +{ + nxt_uint_t n; + nxt_queue_link_t *link; + + n = 0; + *last = NULL; + + for (link = nxt_queue_first(&port->messages); + link != nxt_queue_tail(&port->messages); + link = nxt_queue_next(link)) + { + n++; + *last = nxt_queue_link_data(link, nxt_port_send_msg_t, link); + } + + return n; +} + + +static nxt_int_t +nxt_main_store_test_reply(nxt_thread_t *thr, nxt_port_t *port, nxt_uint_t n, + uint32_t stream, const char *what) +{ + nxt_uint_t count; + nxt_port_send_msg_t *sent; + + count = nxt_main_store_test_replies(port, &sent); + + if (count != n) { + nxt_log_alert(thr->log, "main store test: %ui answers after %s " + "(expected %ui)", count, what, n); + return NXT_ERROR; + } + + if (sent->port_msg.type != _NXT_PORT_MSG_RPC_READY + || sent->port_msg.last != 1 + || sent->port_msg.stream != stream) + { + nxt_log_alert(thr->log, "main store test: %s answered type %d, " + "last %d, stream %uD (expected %d, 1, %uD)", what, + (int) sent->port_msg.type, (int) sent->port_msg.last, + sent->port_msg.stream, (int) _NXT_PORT_MSG_RPC_READY, + stream); + return NXT_ERROR; + } + + return NXT_OK; +} + + +static nxt_int_t +nxt_main_store_test_certs(nxt_thread_t *thr, nxt_task_t *task, + nxt_runtime_t *rt, nxt_main_store_test_paths_t *paths) +{ + char name[NXT_MAX_PATH_LEN]; + nxt_mp_t *mp; + nxt_int_t ret; + nxt_port_t *port; + struct stat st; + nxt_event_engine_t engine, *saved_engine; + nxt_port_send_msg_t *sent; + + ret = NXT_ERROR; + port = NULL; + + if (mkdir(paths->certs, 0700) != 0) { + nxt_log_alert(thr->log, "main store test: mkdir(\"%s\") failed %E", + paths->certs, nxt_errno); + return NXT_ERROR; + } + + rt->certs.start = (u_char *) paths->certs; + rt->certs.length = nxt_strlen(paths->certs); + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + /* The engine is what the teardown of queued messages needs. */ + nxt_memzero(&engine, sizeof(engine)); + nxt_work_queue_cache_create(&engine.work_queue_cache, 1024); + engine.fast_work_queue.cache = &engine.work_queue_cache; + nxt_work_queue_name(&engine.fast_work_queue, "fast"); + engine.mem_pool = mp; + + saved_engine = thr->engine; + thr->engine = &engine; + rt->main_engine = &engine; + + /* + * The port of the controller. No queue and write_ready unset, so an + * answer is queued on port->messages rather than written. + */ + port = nxt_port_new(task, 0, nxt_pid, NXT_PROCESS_CONTROLLER); + if (nxt_slow_path(port == NULL)) { + goto done; + } + + port->pair[0] = -1; + port->pair[1] = -1; + port->socket.fd = -1; + + rt->port_by_type[NXT_PROCESS_CONTROLLER] = port; + + if (nxt_slow_path(nxt_port_hash_add(&rt->ports, port) != NXT_OK)) { + nxt_port_use(task, port, -1); + port = NULL; + goto done; + } + + nxt_main_store_test_alerts = 0; + + /* The first upload. Its child sleeps before it stores. */ + + nxt_main_test_store_set_delay(NXT_MAIN_STORE_TEST_DELAY); + + if (nxt_main_store_test_put(task, paths, port, "one", "bundle one", + 0x11111111) + != NXT_OK) + { + nxt_main_store_test_fail(thr, "cannot send the first bundle"); + } + + if (!nxt_queue_is_empty(&port->messages)) { + nxt_main_store_test_fail(thr, "main answered the upload before the " + "bundle was stored"); + } + + if (nxt_main_test_store_pid() == 0) { + nxt_main_store_test_fail(thr, "the upload started no store child"); + } + + /* + * A conf.json store and a second upload arrive while it runs. Main + * starts to exit then, so a conf.json store child would be killed. + */ + + if (nxt_main_store_test_schedule(task, "{\"f\":6}") != NXT_OK) { + nxt_main_store_test_fail(thr, "cannot map the conf.json store"); + } + + nxt_main_test_store_set_exiting(task, 1); + + if (nxt_main_store_test_put(task, paths, port, "two", "bundle two", + 0x22222222) + != NXT_OK) + { + nxt_main_store_test_fail(thr, "cannot send the second bundle"); + } + + nxt_main_test_store_set_delay(0); + + if (!nxt_queue_is_empty(&port->messages)) { + nxt_main_store_test_fail(thr, "main answered an upload before its " + "store child exited"); + } + + (void) nxt_sprintf((u_char *) name, (u_char *) name + sizeof(name), + "%sone%Z", paths->certs); + + if (nxt_main_store_test_reap(thr, task, "the first bundle") != NXT_OK) { + goto done; + } + + if (nxt_main_store_test_reply(thr, port, 1, 0x11111111, + "the first bundle") + != NXT_OK) + { + goto done; + } + + if (!nxt_main_store_test_holds(name, "bundle one")) { + nxt_main_store_test_fail(thr, "\"%s\" does not hold the first bundle " + "after the answer", name); + } + + /* The conf.json store came before the second upload. */ + + if (nxt_main_store_test_reap(thr, task, "conf.json") != NXT_OK) { + goto done; + } + + if (!nxt_main_store_test_holds(paths->conf, "{\"f\":6}")) { + nxt_main_store_test_fail(thr, "\"%s\" does not hold the " + "configuration", paths->conf); + } + + if (nxt_main_store_test_replies(port, &sent) != 1) { + nxt_main_store_test_fail(thr, "the second bundle was stored before " + "the older conf.json store"); + } + + (void) nxt_sprintf((u_char *) name, (u_char *) name + sizeof(name), + "%stwo%Z", paths->certs); + + if (nxt_main_store_test_reap(thr, task, "the second bundle") != NXT_OK) { + goto done; + } + + if (nxt_main_store_test_reply(thr, port, 2, 0x22222222, + "the second bundle") + != NXT_OK) + { + goto done; + } + + if (!nxt_main_store_test_holds(name, "bundle two")) { + nxt_main_store_test_fail(thr, "\"%s\" does not hold the second " + "bundle after the answer", name); + } + + if (nxt_main_test_store_pid() != 0) { + nxt_main_store_test_fail(thr, "a store child runs with nothing " + "pending"); + } + + nxt_main_test_store_set_exiting(task, 0); + + /* A DELETE right after a PUT of the same bundle runs after it. */ + + nxt_main_test_store_set_delay(NXT_MAIN_STORE_TEST_DELAY); + + if (nxt_main_store_test_put(task, paths, port, "three", "bundle three", + 0x33333333) + != NXT_OK) + { + nxt_main_store_test_fail(thr, "cannot send the third bundle"); + } + + nxt_main_test_store_set_delay(0); + + nxt_main_store_test_delete(task, port, "three"); + + if (nxt_main_store_test_reap(thr, task, "the third bundle") != NXT_OK) { + goto done; + } + + /* A delete that main ran at once started no child. */ + if (nxt_main_test_store_pid() != 0 + && nxt_main_store_test_reap(thr, task, "the delete") != NXT_OK) + { + goto done; + } + + if (nxt_main_store_test_reply(thr, port, 3, 0x33333333, + "the third bundle") + != NXT_OK) + { + goto done; + } + + (void) nxt_sprintf((u_char *) name, (u_char *) name + sizeof(name), + "%sthree%Z", paths->certs); + + if (stat(name, &st) == 0) { + nxt_main_store_test_fail(thr, "\"%s\" is back: the delete ran before " + "the store", name); + } + + (void) nxt_sprintf((u_char *) name, (u_char *) name + sizeof(name), + "%s" NXT_MAIN_STORE_TMP "%Z", paths->certs); + + if (stat(name, &st) == 0) { + nxt_main_store_test_fail(thr, "\"%s\" was left behind", name); + } + + if (nxt_main_store_test_alerts != 0) { + nxt_main_store_test_fail(thr, "%ui alerts", nxt_main_store_test_alerts); + } + + ret = NXT_OK; + +done: + + nxt_main_store_test_reap_all(task); + + if (port != NULL) { + /* Frees the queued answers and their port references. */ + nxt_port_test_run_error_handler(task, port); + + (void) nxt_port_hash_remove(&rt->ports, port); + rt->port_by_type[NXT_PROCESS_CONTROLLER] = NULL; + + nxt_port_use(task, port, -1); + } + + thr->engine = saved_engine; + rt->main_engine = NULL; + + nxt_work_queue_cache_destroy(&engine.work_queue_cache); + nxt_mp_destroy(mp); + + (void) unlink(paths->bundle); + + (void) nxt_sprintf((u_char *) name, (u_char *) name + sizeof(name), + "%sone%Z", paths->certs); + (void) unlink(name); + + (void) nxt_sprintf((u_char *) name, (u_char *) name + sizeof(name), + "%stwo%Z", paths->certs); + (void) unlink(name); + + (void) nxt_sprintf((u_char *) name, (u_char *) name + sizeof(name), + "%sthree%Z", paths->certs); + (void) unlink(name); + + (void) nxt_sprintf((u_char *) name, (u_char *) name + sizeof(name), + "%s" NXT_MAIN_STORE_TMP "%Z", paths->certs); + (void) unlink(name); + + (void) rmdir(paths->certs); + + return ret; +} + +#endif /* NXT_TLS */ + + +static void +nxt_main_store_test_path(char *buf, const char *dir, const char *name) +{ + (void) nxt_sprintf((u_char *) buf, (u_char *) buf + NXT_MAX_PATH_LEN, + "%s/%s%Z", dir, name); +} + + +nxt_int_t +nxt_main_store_test(nxt_thread_t *thr) +{ + nxt_uint_t conf_ver; + nxt_int_t ret; + nxt_task_t *task; + const char *tmpdir; + nxt_runtime_t rt, *saved_rt; + nxt_main_store_test_paths_t paths; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "main store test started"); + + task = thr->task; + task->thread = thr; + + tmpdir = getenv("TMPDIR"); + if (tmpdir == NULL || tmpdir[0] != '/') { + tmpdir = "/tmp"; + } + + (void) nxt_sprintf((u_char *) paths.dir, + (u_char *) paths.dir + sizeof(paths.dir), + "%s/nxt_main_store_test.XXXXXX%Z", tmpdir); + + if (nxt_slow_path(mkdtemp(paths.dir) == NULL)) { + nxt_log_alert(thr->log, "main store test: mkdtemp(\"%s\") failed %E", + paths.dir, nxt_errno); + return NXT_ERROR; + } + + nxt_main_store_test_path(paths.conf, paths.dir, "conf.json"); + nxt_main_store_test_path(paths.conf_tmp, paths.dir, "conf.json.tmp"); + nxt_main_store_test_path(paths.ver, paths.dir, "version"); + nxt_main_store_test_path(paths.ver_tmp, paths.dir, "version.tmp"); + nxt_main_store_test_path(paths.certs, paths.dir, "certs/"); + nxt_main_store_test_path(paths.bundle, paths.dir, "bundle"); + + nxt_memzero(&rt, sizeof(nxt_runtime_t)); + + rt.state = paths.dir; + rt.conf = paths.conf; + rt.conf_tmp = paths.conf_tmp; + rt.ver = paths.ver; + rt.ver_tmp = paths.ver_tmp; + + saved_rt = thr->runtime; + thr->runtime = &rt; + + conf_ver = nxt_conf_ver; + + nxt_main_store_test_next_handler = thr->log->handler; + thr->log->handler = nxt_main_store_test_log; + + ret = nxt_main_store_test_running(thr, task, &paths); + + if (ret == NXT_OK) { + ret = nxt_main_store_test_cancel(thr, task, &paths); + } + + if (ret == NXT_OK) { + ret = nxt_main_store_test_finished(thr, task, &paths); + } + + if (ret == NXT_OK) { + ret = nxt_main_store_test_fds(thr, task, &paths); + } + + if (ret == NXT_OK) { + ret = nxt_main_store_test_order(thr, task, &paths); + } + +#if (NXT_TLS) + if (ret == NXT_OK) { + ret = nxt_main_store_test_certs(thr, task, &rt, &paths); + } +#endif + + thr->log->handler = nxt_main_store_test_next_handler; + thr->runtime = saved_rt; + nxt_conf_ver = conf_ver; + + (void) unlink(paths.conf); + (void) unlink(paths.conf_tmp); + (void) unlink(paths.ver); + (void) unlink(paths.ver_tmp); + (void) rmdir(paths.dir); + + if (ret == NXT_OK) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, "main store test passed"); + } + + return ret; +} diff --git a/src/test/nxt_main_whoami_test.c b/src/test/nxt_main_whoami_test.c new file mode 100644 index 000000000..3d2534b31 --- /dev/null +++ b/src/test/nxt_main_whoami_test.c @@ -0,0 +1,261 @@ +/* + * Copyright (C) FreeUnit Community + */ + +/* + * The WHOAMI handler in main must refuse a message whose shape does not + * match how the sender was started. "ppid" comes from the sender, which is + * not trusted. + * + * - a worker already recorded sends WHOAMI again, with or without a port: + * before the fix it got a second port and was linked into its parent's + * "children" twice, which corrupts the queue; + * - a process names a parent that is not a prototype; + * - a process names main as its parent and sends a port. + * + * Each refused message must leave the records as they were and close the + * descriptor it carried. The accepted shapes run in every pytest that + * starts an application. + */ + +#include +#include +#include +#include +#include "nxt_tests.h" + + +typedef struct { + nxt_pid_t pid; + nxt_pid_t ppid; + nxt_bool_t with_fd; + const char *label; +} nxt_main_whoami_case_t; + + +static nxt_port_t * +nxt_main_whoami_port(nxt_task_t *task, nxt_runtime_t *rt, nxt_pid_t pid, + nxt_process_type_t type) +{ + nxt_port_t *port; + + port = nxt_runtime_process_port_create(task, rt, pid, 0, type); + if (port != NULL) { + port->pair[0] = -1; + port->pair[1] = -1; + port->socket.fd = -1; + } + + return port; +} + + +static void +nxt_main_whoami_port_release(nxt_task_t *task, nxt_port_t *port) +{ + if (port != NULL) { + nxt_port_close(task, port); + nxt_runtime_port_remove(task, port); + } +} + + +static nxt_uint_t +nxt_main_whoami_count(nxt_queue_t *queue) +{ + nxt_uint_t n; + nxt_queue_link_t *lnk; + + n = 0; + + for (lnk = nxt_queue_first(queue); + lnk != nxt_queue_tail(queue) && n < 100; + lnk = nxt_queue_next(lnk)) + { + n++; + } + + return n; +} + + +static nxt_int_t +nxt_main_whoami_refused(nxt_thread_t *thr, nxt_mp_t *mp, nxt_runtime_t *rt, + nxt_process_t *proto, nxt_process_t *child, nxt_main_whoami_case_t *c) +{ + int pair[2]; + nxt_buf_t *buf; + nxt_process_t *process; + nxt_port_recv_msg_t msg; + + pair[0] = -1; + pair[1] = -1; + + if (c->with_fd && socketpair(AF_UNIX, SOCK_DGRAM, 0, pair) != 0) { + nxt_log_alert(thr->log, "main whoami test: socketpair() failed"); + return NXT_ERROR; + } + + buf = nxt_buf_mem_alloc(mp, sizeof(nxt_pid_t), 0); + if (nxt_slow_path(buf == NULL)) { + return NXT_ERROR; + } + + buf->mem.free = nxt_cpymem(buf->mem.free, &c->ppid, sizeof(nxt_pid_t)); + + nxt_memzero(&msg, sizeof(nxt_port_recv_msg_t)); + + msg.buf = buf; + msg.fd[0] = pair[0]; + msg.fd[1] = -1; + msg.port_msg.type = _NXT_PORT_MSG_WHOAMI; + msg.port_msg.pid = c->pid; +#if (NXT_USE_CMSG_PID) + msg.cmsg_pid = c->pid; +#endif + + nxt_main_test_run_whoami_handler(thr->task, &msg); + + nxt_mp_free(mp, buf); + + if (pair[1] != -1) { + nxt_fd_close(pair[1]); + } + + if (c->with_fd && nxt_test_fd_is_open(pair[0])) { + nxt_log_alert(thr->log, "main whoami test: %s: the descriptor was " + "kept", c->label); + nxt_fd_close(pair[0]); + return NXT_ERROR; + } + + if (nxt_main_whoami_count(&proto->children) != 1 + || nxt_main_whoami_count(&child->ports) != 1) + { + nxt_log_alert(thr->log, "main whoami test: %s: %ui children, %ui " + "ports of the worker; expected 1 and 1", c->label, + nxt_main_whoami_count(&proto->children), + nxt_main_whoami_count(&child->ports)); + return NXT_ERROR; + } + + if (c->pid != child->pid) { + process = nxt_runtime_process_find(rt, c->pid); + + if (process != NULL) { + nxt_log_alert(thr->log, "main whoami test: %s: a record was " + "created", c->label); + return NXT_ERROR; + } + } + + return NXT_OK; +} + + +nxt_int_t +nxt_main_whoami_test(nxt_thread_t *thr) +{ + nxt_mp_t *mp; + nxt_int_t ret; + nxt_pid_t proto_pid, child_pid, router_pid, new_pid; + nxt_uint_t i; + nxt_port_t *main_port, *router_port, *proto_port; + nxt_port_t *child_port; + nxt_runtime_t *rt, *saved_rt; + nxt_process_t *proto, *child; + nxt_main_whoami_case_t cases[4]; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "main whoami test started"); + + ret = NXT_ERROR; + main_port = NULL; + router_port = NULL; + proto_port = NULL; + child_port = NULL; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + rt = nxt_mp_zalloc(mp, sizeof(nxt_runtime_t)); + if (nxt_slow_path(rt == NULL)) { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + rt->mem_pool = mp; + + if (nxt_slow_path(nxt_thread_mutex_create(&rt->processes_mutex) != NXT_OK)) + { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + saved_rt = thr->runtime; + thr->runtime = rt; + + proto_pid = nxt_pid + 1; + child_pid = nxt_pid + 2; + router_pid = nxt_pid + 3; + new_pid = nxt_pid + 4; + + main_port = nxt_main_whoami_port(thr->task, rt, nxt_pid, + NXT_PROCESS_MAIN); + router_port = nxt_main_whoami_port(thr->task, rt, router_pid, + NXT_PROCESS_ROUTER); + proto_port = nxt_main_whoami_port(thr->task, rt, proto_pid, + NXT_PROCESS_PROTOTYPE); + child_port = nxt_main_whoami_port(thr->task, rt, child_pid, + NXT_PROCESS_APP); + + if (nxt_slow_path(main_port == NULL || router_port == NULL + || proto_port == NULL || child_port == NULL)) + { + goto done; + } + + proto = proto_port->process; + child = child_port->process; + + /* As an accepted WHOAMI leaves a worker. */ + nxt_queue_insert_tail(&proto->children, &child->link); + + cases[0] = (nxt_main_whoami_case_t) { + child_pid, proto_pid, 1, "second WHOAMI with a port" }; + cases[1] = (nxt_main_whoami_case_t) { + child_pid, proto_pid, 0, "second WHOAMI without a port" }; + cases[2] = (nxt_main_whoami_case_t) { + new_pid, router_pid, 1, "parent is not a prototype" }; + cases[3] = (nxt_main_whoami_case_t) { + new_pid, nxt_pid, 1, "child of main sends a port" }; + + for (i = 0; i < nxt_nitems(cases); i++) { + if (nxt_main_whoami_refused(thr, mp, rt, proto, child, &cases[i]) + != NXT_OK) + { + goto done; + } + } + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "main whoami test passed"); + + ret = NXT_OK; + +done: + + nxt_main_whoami_port_release(thr->task, child_port); + nxt_main_whoami_port_release(thr->task, proto_port); + nxt_main_whoami_port_release(thr->task, router_port); + nxt_main_whoami_port_release(thr->task, main_port); + + thr->runtime = saved_rt; + + nxt_thread_mutex_destroy(&rt->processes_mutex); + + nxt_mp_destroy(mp); + + return ret; +} diff --git a/src/test/nxt_malloc_test.c b/src/test/nxt_malloc_test.c index daa609b14..e7c34cdf3 100644 --- a/src/test/nxt_malloc_test.c +++ b/src/test/nxt_malloc_test.c @@ -119,5 +119,7 @@ nxt_malloc_test(nxt_thread_t *thr) s[-1].size + 1, s->size, s->alignment, s->tight); } + nxt_log_error(NXT_LOG_NOTICE, thr->log, "malloc test passed"); + return NXT_OK; } diff --git a/src/test/nxt_mem_zone_test.c b/src/test/nxt_mem_zone_test.c deleted file mode 100644 index faf0feee6..000000000 --- a/src/test/nxt_mem_zone_test.c +++ /dev/null @@ -1,74 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) NGINX, Inc. - */ - -#include -#include "nxt_tests.h" - - -nxt_int_t -nxt_mem_zone_test(nxt_thread_t *thr, nxt_uint_t runs, nxt_uint_t nblocks, - size_t max_size) -{ - void *start, **blocks; - size_t total, zone_size; - uint32_t size; - nxt_uint_t i, n; - nxt_mem_zone_t *zone; - const size_t page_size = 4096; - - nxt_thread_time_update(thr); - nxt_log_error(NXT_LOG_NOTICE, thr->log, - "mem zone test started, max:%uz", max_size); - - zone_size = (max_size + 1) * nblocks; - - start = nxt_memalign(page_size, zone_size); - if (start == NULL) { - return NXT_ERROR; - } - - zone = nxt_mem_zone_init(start, zone_size, page_size); - if (zone == NULL) { - return NXT_ERROR; - } - - blocks = nxt_malloc(nblocks * sizeof(void *)); - if (blocks == NULL) { - return NXT_ERROR; - } - - size = 0; - - for (i = 0; i < runs; i++) { - - total = 0; - - for (n = 0; n < nblocks; n++) { - size = nxt_murmur_hash2(&size, sizeof(uint32_t)); - - total += size & max_size; - blocks[n] = nxt_mem_zone_alloc(zone, size & max_size); - - if (blocks[n] == NULL) { - nxt_log_error(NXT_LOG_NOTICE, thr->log, - "mem zone test failed: %uz", total); - return NXT_ERROR; - } - } - - for (n = 0; n < nblocks; n++) { - nxt_mem_zone_free(zone, blocks[n]); - } - } - - nxt_free(blocks); - nxt_free(zone); - - nxt_thread_time_update(thr); - nxt_log_error(NXT_LOG_NOTICE, thr->log, "mem zone test passed"); - - return NXT_OK; -} diff --git a/src/test/nxt_nncq_bound_test.c b/src/test/nxt_nncq_bound_test.c new file mode 100644 index 000000000..d187d6206 --- /dev/null +++ b/src/test/nxt_nncq_bound_test.c @@ -0,0 +1,182 @@ +/* + * Copyright (C) F5, Inc. + */ + +/* + * The application maps the shared memory queues (src/nxt_nncq.h, + * src/nxt_app_nncq.h) writable. A foreign cycle under head or tail used + * to keep the router retrying for ever. Each case poisons one queue and + * runs the router-side operation in a child under alarm(): it must return + * and report the queue empty or the send failed. + */ + +#include +#include +#include +#include "nxt_tests.h" + + +static const uint8_t nxt_nncq_bound_test_msg[1] = { 0x5A }; + + +static int +nxt_nncq_bound_test_port_recv(void *mem) +{ + uint8_t buf[NXT_PORT_QUEUE_MSG_SIZE]; + nxt_port_queue_t *q; + + q = mem; + q->queue.entries[0] = 7 * NXT_NNCQ_SIZE; + + return nxt_port_queue_recv(q, buf) == -1; +} + + +static int +nxt_nncq_bound_test_port_send(nxt_port_queue_t *q, nxt_nncq_t *poison) +{ + int notify; + + poison->entries[0] = 7 * NXT_NNCQ_SIZE; + + return nxt_port_queue_send(q, nxt_nncq_bound_test_msg, 1, ¬ify); +} + + +static int +nxt_nncq_bound_test_port_send_full(void *mem) +{ + nxt_port_queue_t *q = mem; + + return nxt_nncq_bound_test_port_send(q, &q->free_items) == NXT_AGAIN; +} + + +/* The refused message must give its slot back: all of them stay free. */ + +static int +nxt_nncq_bound_test_port_send_slot(void *mem) +{ + nxt_uint_t n; + nxt_port_queue_t *q = mem; + + if (nxt_nncq_bound_test_port_send(q, &q->queue) != NXT_ERROR) { + return 0; + } + + for (n = 0; nxt_nncq_dequeue(&q->free_items) + != nxt_nncq_empty(&q->free_items); n++); + + return n == NXT_PORT_QUEUE_SIZE; +} + + +static int +nxt_nncq_bound_test_app_send(nxt_app_queue_t *q, nxt_app_nncq_t *poison) +{ + int notify; + uint32_t cookie; + + poison->entries[0] = 7 * NXT_APP_NNCQ_SIZE; + + return nxt_app_queue_send(q, nxt_nncq_bound_test_msg, 1, 1, ¬ify, + &cookie); +} + + +static int +nxt_nncq_bound_test_app_send_full(void *mem) +{ + nxt_app_queue_t *q = mem; + + return nxt_nncq_bound_test_app_send(q, &q->free_items) == NXT_AGAIN; +} + + +static int +nxt_nncq_bound_test_app_send_slot(void *mem) +{ + nxt_uint_t n; + nxt_app_queue_t *q = mem; + + if (nxt_nncq_bound_test_app_send(q, &q->queue) != NXT_ERROR) { + return 0; + } + + for (n = 0; nxt_app_nncq_dequeue(&q->free_items) + != nxt_app_nncq_empty(&q->free_items); n++); + + return n == NXT_APP_QUEUE_SIZE; +} + + +typedef struct { + const char *name; + nxt_bool_t app; + int (*op)(void *q); +} nxt_nncq_bound_test_case_t; + + +static const nxt_nncq_bound_test_case_t nxt_nncq_bound_test_cases[] = { + { "port recv", 0, nxt_nncq_bound_test_port_recv }, + { "port send, poisoned free list", 0, nxt_nncq_bound_test_port_send_full }, + { "port send, poisoned slot", 0, nxt_nncq_bound_test_port_send_slot }, + { "app send, poisoned free list", 1, nxt_nncq_bound_test_app_send_full }, + { "app send, poisoned slot", 1, nxt_nncq_bound_test_app_send_slot }, +}; + + +static void *nxt_nncq_bound_test_mem; + + +static int +nxt_nncq_bound_test_child(void *data) +{ + const nxt_nncq_bound_test_case_t *tc = data; + + alarm(5); + + return !tc->op(nxt_nncq_bound_test_mem); +} + + +nxt_int_t +nxt_nncq_bound_test(nxt_thread_t *thr) +{ + int rc; + size_t size; + nxt_uint_t i; + + const nxt_nncq_bound_test_case_t *tc; + + nxt_thread_time_update(thr); + + for (i = 0; i < nxt_nitems(nxt_nncq_bound_test_cases); i++) { + tc = &nxt_nncq_bound_test_cases[i]; + size = tc->app ? sizeof(nxt_app_queue_t) : sizeof(nxt_port_queue_t); + + nxt_nncq_bound_test_mem = nxt_malloc(size); + if (nxt_nncq_bound_test_mem == NULL) { + return NXT_ERROR; + } + + if (tc->app) { + nxt_app_queue_init(nxt_nncq_bound_test_mem); + + } else { + nxt_port_queue_init(nxt_nncq_bound_test_mem); + } + + rc = nxt_test_in_child(thr, tc->name, nxt_nncq_bound_test_child, + (void *) tc); + + nxt_free(nxt_nncq_bound_test_mem); + + NXT_TEST_CHECK(thr->log, rc == 0, "nncq bound test: %s did not " + "return the failure", tc->name); + } + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "nncq bound test passed"); + + return NXT_OK; +} diff --git a/src/test/nxt_openssl_server_init_test.c b/src/test/nxt_openssl_server_init_test.c new file mode 100644 index 000000000..9497fc909 --- /dev/null +++ b/src/test/nxt_openssl_server_init_test.c @@ -0,0 +1,378 @@ +/* + * Copyright (C) FreeUnit Community + */ + +/* + * Regression test for the failure paths of nxt_openssl_server_init() + * (src/nxt_openssl.c). + * + * nxt_router_tls_rpc_handler() links a new bundle into the chain of the + * socket conf and calls server_init(). When that fails, the router calls + * nxt_router_conf_error(), which calls server_free() on the conf, and + * server_free() calls SSL_CTX_free() on the context of every bundle in the + * chain. Two bugs were on this path: + * + * - The bundle came from nxt_mp_get(), and server_init() returned right + * after a failed SSL_CTX_new() without a write to bundle->ctx. So + * server_free() gave a garbage pointer to SSL_CTX_free(). + * + * - The bundle owns the chain file. Only the BIO in + * nxt_openssl_chain_file() closed it, so the descriptor leaked when + * SSL_CTX_new() or BIO_new() failed. + * + * The test calls server_init() and server_free() directly, not through the + * router: a router conf needs an engine, a port and a controller peer, and + * none of them has a part in these two bugs. Each bundle is allocated the + * way the router allocated it before the fix, with nxt_mp_get(), and filled + * with 0xa5, so the test also covers a caller that does not zero it. + * NXT_TESTS hooks in src/nxt_openssl.c make SSL_CTX_new() or BIO_new() + * fail. + * + * After each call the test opens a probe descriptor. It usually gets the + * number of the closed chain file, so a second close of that number from a + * later path closes the probe, and the test sees it. + */ + +#include +#include "nxt_tests.h" + +#include + +#include +#include +#include + + +typedef struct { + nxt_thread_t *thr; + nxt_task_t *task; + nxt_mp_t *mp; + nxt_tls_conf_t *conf; + nxt_tls_init_t tls_init; + const char *path; +} nxt_openssl_si_test_t; + + +static nxt_int_t +nxt_openssl_si_test_write_bundle(nxt_thread_t *thr, const char *path) +{ + int ok; + FILE *f; + X509 *x; + EVP_PKEY *key; + X509_NAME *name; + EVP_PKEY_CTX *kctx; + + ok = 0; + x = NULL; + key = NULL; + name = NULL; + f = NULL; + + kctx = EVP_PKEY_CTX_new_id(EVP_PKEY_ED25519, NULL); + if (kctx == NULL + || EVP_PKEY_keygen_init(kctx) != 1 + || EVP_PKEY_keygen(kctx, &key) != 1) + { + goto done; + } + + x = X509_new(); + name = X509_NAME_new(); + if (x == NULL || name == NULL) { + goto done; + } + + if (X509_set_version(x, 2) != 1 + || ASN1_INTEGER_set(X509_get_serialNumber(x), 1) != 1 + || X509_gmtime_adj(X509_getm_notBefore(x), 0) == NULL + || X509_gmtime_adj(X509_getm_notAfter(x), 3600) == NULL + || X509_set_pubkey(x, key) != 1 + || X509_NAME_add_entry_by_txt(name, "CN", MBSTRING_ASC, + (const u_char *) "localhost", -1, -1, 0) + != 1 + || X509_set_subject_name(x, name) != 1 + || X509_set_issuer_name(x, name) != 1 + || X509_sign(x, key, NULL) == 0) + { + goto done; + } + + f = fopen(path, "w"); + if (f == NULL) { + goto done; + } + + ok = PEM_write_X509(f, x) == 1 + && PEM_write_PrivateKey(f, key, NULL, NULL, 0, NULL, NULL) == 1; + +done: + + if (f != NULL && fclose(f) != 0) { + ok = 0; + } + + X509_NAME_free(name); + X509_free(x); + EVP_PKEY_free(key); + EVP_PKEY_CTX_free(kctx); + + if (!ok) { + nxt_log_alert(thr->log, "openssl server init test: cannot write " + "a test bundle to \"%s\"", path); + return NXT_ERROR; + } + + return NXT_OK; +} + + +/* + * Links a new bundle into the chain the way nxt_router_tls_rpc_handler() + * does, and returns its chain file descriptor. + */ + +static nxt_fd_t +nxt_openssl_si_test_add_bundle(nxt_openssl_si_test_t *t) +{ + nxt_fd_t fd; + nxt_tls_bundle_conf_t *bundle; + + fd = open(t->path, O_RDONLY | O_CLOEXEC); + if (fd == -1) { + nxt_log_alert(t->thr->log, "openssl server init test: " + "open(\"%s\") failed %E", t->path, nxt_errno); + return -1; + } + + bundle = nxt_mp_get(t->mp, sizeof(nxt_tls_bundle_conf_t)); + if (bundle == NULL) { + (void) close(fd); + return -1; + } + + /* Garbage, as nxt_mp_get() can give it. */ + nxt_memset(bundle, 0xa5, sizeof(nxt_tls_bundle_conf_t)); + + nxt_str_set(&bundle->name, "test"); + bundle->chain_file = fd; + + bundle->next = t->conf->bundle; + t->conf->bundle = bundle; + + return fd; +} + + +/* + * Runs server_init() on a new bundle and checks the result, the context of + * the bundle and that the chain file is closed. + */ + +static nxt_int_t +nxt_openssl_si_test_init(nxt_openssl_si_test_t *t, const char *name, + nxt_uint_t fail, nxt_bool_t last, nxt_int_t expect) +{ + nxt_fd_t fd; + nxt_int_t ret; + nxt_tls_bundle_conf_t *bundle; + + fd = nxt_openssl_si_test_add_bundle(t); + if (fd == -1) { + return NXT_ERROR; + } + + bundle = t->conf->bundle; + + nxt_openssl_test_fail = fail; + + ret = nxt_openssl_lib.server_init(t->task, t->mp, &t->tls_init, last); + + nxt_openssl_test_fail = 0; + + NXT_TEST_CHECK(t->thr->log, ret == expect, + "openssl server init test: %s: server_init() returned " + "%i, expected %i", name, ret, expect); + + if (nxt_test_fd_is_open(fd)) { + (void) close(fd); + + nxt_log_alert(t->thr->log, "openssl server init test: %s: " + "the chain file %FD is still open", name, fd); + return NXT_ERROR; + } + + NXT_TEST_CHECK(t->thr->log, (bundle->ctx != NULL) == (expect == NXT_OK), + "openssl server init test: %s: bundle->ctx is %p", + name, bundle->ctx); + + /* Nothing may close the number again. */ + NXT_TEST_CHECK(t->thr->log, bundle->chain_file == -1, + "openssl server init test: %s: bundle->chain_file is %FD, " + "expected -1", name, bundle->chain_file); + + return NXT_OK; +} + + +/* + * Opens a probe, calls server_free() as nxt_router_conf_error() does, and + * checks that the probe is still open. + */ + +static nxt_int_t +nxt_openssl_si_test_free(nxt_openssl_si_test_t *t, const char *name) +{ + nxt_fd_t probe; + nxt_bool_t is_open; + + probe = open(t->path, O_RDONLY | O_CLOEXEC); + if (probe == -1) { + nxt_log_alert(t->thr->log, "openssl server init test: " + "open(\"%s\") failed %E", t->path, nxt_errno); + return NXT_ERROR; + } + + nxt_openssl_lib.server_free(t->task, t->conf); + + is_open = nxt_test_fd_is_open(probe); + + if (is_open) { + (void) close(probe); + } + + NXT_TEST_CHECK(t->thr->log, is_open, + "openssl server init test: %s: server_free() closed " + "the probe descriptor %FD", name, probe); + + return NXT_OK; +} + + +static nxt_int_t +nxt_openssl_si_test_case(nxt_openssl_si_test_t *t, const char *name, + nxt_bool_t first_ok, nxt_uint_t fail) +{ + nxt_int_t ret; + + t->conf = nxt_mp_zget(t->mp, sizeof(nxt_tls_conf_t)); + if (t->conf == NULL) { + return NXT_ERROR; + } + + t->conf->no_wait_shutdown = 1; + + nxt_memzero(&t->tls_init, sizeof(nxt_tls_init_t)); + t->tls_init.conf = t->conf; + + if (first_ok) { + /* An earlier certificate of the same socket, with a context. */ + ret = nxt_openssl_si_test_init(t, name, 0, 0, NXT_OK); + if (ret != NXT_OK) { + return ret; + } + } + + ret = nxt_openssl_si_test_init(t, name, fail, 1, + fail == 0 ? NXT_OK : NXT_ERROR); + if (ret != NXT_OK) { + return ret; + } + + return nxt_openssl_si_test_free(t, name); +} + + +nxt_int_t +nxt_openssl_server_init_test(nxt_thread_t *thr) +{ + char path[256]; + const char *tmpdir; + nxt_int_t ret; + nxt_openssl_si_test_t t; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "openssl server init test started"); + + nxt_memzero(&t, sizeof(nxt_openssl_si_test_t)); + + t.thr = thr; + t.task = thr->task; + t.task->thread = thr; + + if (nxt_openssl_lib.library_init(t.task) != NXT_OK) { + return NXT_ERROR; + } + + tmpdir = getenv("TMPDIR"); + if (tmpdir == NULL || tmpdir[0] != '/') { + tmpdir = "/tmp"; + } + + (void) nxt_sprintf((u_char *) path, (u_char *) path + sizeof(path), + "%s/nxt_openssl_server_init_test.XXXXXX%Z", tmpdir); + + ret = mkstemp(path); + if (ret == -1) { + nxt_log_alert(thr->log, "openssl server init test: " + "mkstemp(\"%s\") failed %E", path, nxt_errno); + return NXT_ERROR; + } + + (void) close(ret); + + t.path = path; + + ret = NXT_ERROR; + + t.mp = nxt_mp_create(1024, 128, 256, 32); + if (t.mp == NULL) { + goto done; + } + + if (nxt_openssl_si_test_write_bundle(thr, path) != NXT_OK) { + goto done; + } + + ret = nxt_openssl_si_test_case(&t, "SSL_CTX_new() fails", 0, + NXT_OPENSSL_TEST_FAIL_CTX_NEW); + if (ret != NXT_OK) { + goto done; + } + + ret = nxt_openssl_si_test_case(&t, "SSL_CTX_new() fails, second bundle", + 1, NXT_OPENSSL_TEST_FAIL_CTX_NEW); + if (ret != NXT_OK) { + goto done; + } + + ret = nxt_openssl_si_test_case(&t, "BIO_new() fails", 0, + NXT_OPENSSL_TEST_FAIL_BIO_NEW); + if (ret != NXT_OK) { + goto done; + } + + ret = nxt_openssl_si_test_case(&t, "BIO_new() fails, second bundle", 1, + NXT_OPENSSL_TEST_FAIL_BIO_NEW); + if (ret != NXT_OK) { + goto done; + } + + ret = nxt_openssl_si_test_case(&t, "success", 1, 0); + +done: + + (void) unlink(path); + + if (t.mp != NULL) { + nxt_mp_destroy(t.mp); + } + + if (ret == NXT_OK) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "openssl server init test passed"); + } + + return ret; +} diff --git a/src/test/nxt_port_change_file_test.c b/src/test/nxt_port_change_file_test.c new file mode 100644 index 000000000..1b5e1b05d --- /dev/null +++ b/src/test/nxt_port_change_file_test.c @@ -0,0 +1,336 @@ +/* + * Copyright (C) F5, Inc. + */ + +/* + * Regression test for nxt_port_change_log_file_handler() (src/nxt_port.c). + * + * CHANGE_FILE carries a log file slot in its payload and the descriptor the + * slot is to be redirected to. Neither is authenticated, and the discovery + * and prototype processes, the router and the controller all install this + * handler. + * + * The handler read the slot without bounding it against the payload, passed + * it to nxt_list_elt() -- which returns NULL past the end of the list -- and + * dereferenced the result, so a forged slot crashed the receiver. It also + * never closed the second descriptor of the message, and lost the first one + * when the redirect failed; the dispatcher does not reclaim fds once the + * handler returns (see nxt_port_recv_msg_close_fds()), so a peer attaching + * descriptors to forged messages can exhaust the receiver's descriptor + * table. + * + * A message can carry two descriptors regardless of how many its type + * normally uses, so the rejected cases attach both and expect both back. + */ + +#include +#include +#include +#include "nxt_tests.h" + +#include + + +/* + * Feed the handler a message it has to refuse, and check that nothing the + * message carried outlives the call. + */ +static nxt_int_t +nxt_port_change_file_test_reject(nxt_thread_t *thr, nxt_task_t *task, + nxt_port_recv_msg_t *msg, nxt_bool_t with_fd0, const char *name) +{ + nxt_fd_t fd0, fd1; + + fd0 = -1; + + if (with_fd0) { + fd0 = open("/dev/null", O_RDONLY); + } + + fd1 = open("/dev/null", O_RDONLY); + + if ((with_fd0 && fd0 == -1) || fd1 == -1) { + nxt_log_alert(thr->log, "port change file test failed to open " + "/dev/null"); + goto fail; + } + + msg->fd[0] = fd0; + msg->fd[1] = fd1; + + nxt_port_change_log_file_handler(task, msg); + + if ((fd0 != -1 && nxt_test_fd_is_open(fd0)) + || nxt_test_fd_is_open(fd1)) + { + nxt_log_alert(thr->log, "port change file test: %s leaked a " + "descriptor", name); + goto fail; /* closes only what is still open */ + } + + if (msg->fd[0] != -1 || msg->fd[1] != -1) { + nxt_log_alert(thr->log, "port change file test: %s left fds in the " + "message", name); + return NXT_ERROR; + } + + return NXT_OK; + +fail: + + /* + * Close only what the handler left open: the descriptors it consumed + * are gone already, and closing them again could reach an unrelated + * descriptor that reused the number. + */ + if (fd0 != -1 && nxt_test_fd_is_open(fd0)) { + (void) close(fd0); + } + + if (fd1 != -1 && nxt_test_fd_is_open(fd1)) { + (void) close(fd1); + } + + return NXT_ERROR; +} + + +/* + * The accepted case: the slot exists, so the first descriptor is redirected + * onto the log file -- and consumed by dup2() doing so -- while the second + * one, which this message type has no use for, is closed rather than left + * behind. + */ +static nxt_int_t +nxt_port_change_file_test_accept(nxt_thread_t *thr, nxt_task_t *task, + nxt_port_recv_msg_t *msg, nxt_file_t *log_file, const char *name) +{ + nxt_fd_t fd0, fd1; + + fd0 = open("/dev/null", O_WRONLY); + fd1 = open("/dev/null", O_RDONLY); + + if (fd0 == -1 || fd1 == -1) { + nxt_log_alert(thr->log, "port change file test failed to open " + "/dev/null"); + goto fail; + } + + msg->fd[0] = fd0; + msg->fd[1] = fd1; + + nxt_port_change_log_file_handler(task, msg); + + if (nxt_test_fd_is_open(fd0) + || nxt_test_fd_is_open(fd1)) + { + nxt_log_alert(thr->log, "port change file test: %s leaked a " + "descriptor", name); + goto fail; + } + + if (msg->fd[0] != -1 || msg->fd[1] != -1) { + nxt_log_alert(thr->log, "port change file test: %s left fds in the " + "message", name); + return NXT_ERROR; + } + + if (!nxt_test_fd_is_open(log_file->fd)) { + nxt_log_alert(thr->log, "port change file test: %s closed the log " + "file", name); + return NXT_ERROR; + } + + return NXT_OK; + +fail: + + if (fd0 != -1 && nxt_test_fd_is_open(fd0)) { + (void) close(fd0); + } + + if (fd1 != -1 && nxt_test_fd_is_open(fd1)) { + (void) close(fd1); + } + + return NXT_ERROR; +} + + +nxt_int_t +nxt_port_change_file_test(nxt_thread_t *thr) +{ + nxt_mp_t *mp; + nxt_buf_t *b; + nxt_file_name_t *name; + nxt_uint_t slot; + nxt_task_t *task; + nxt_int_t ret; + nxt_file_t *log_file; + nxt_runtime_t *rt, *saved_rt; + nxt_port_recv_msg_t msg; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "port change file test started"); + + task = thr->task; + task->thread = thr; + + name = (nxt_file_name_t *) "port change file test"; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + rt = nxt_mp_zalloc(mp, sizeof(nxt_runtime_t)); + if (nxt_slow_path(rt == NULL)) { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + rt->mem_pool = mp; + + /* + * Two slots. Slot 0 stays unopened, so a redirect onto it fails, and + * the accepted case uses slot 1: a successful redirect of slot 0 makes + * the handler point stderr at the new file, which would follow the test + * process out of this function. + */ + rt->log_files = nxt_list_create(mp, 2, sizeof(nxt_file_t)); + if (nxt_slow_path(rt->log_files == NULL)) { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + log_file = nxt_list_add(rt->log_files); + if (nxt_slow_path(log_file == NULL)) { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + nxt_memzero(log_file, sizeof(nxt_file_t)); + + /* + * Never opened, so redirecting onto it fails -- and the failure path + * logs the name, which therefore cannot be left NULL. + */ + log_file->name = name; + log_file->fd = NXT_FILE_INVALID; + + log_file = nxt_list_add(rt->log_files); + if (nxt_slow_path(log_file == NULL)) { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + nxt_memzero(log_file, sizeof(nxt_file_t)); + + log_file->name = name; + log_file->fd = open("/dev/null", O_WRONLY); + if (nxt_slow_path(log_file->fd == -1)) { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + saved_rt = thr->runtime; + thr->runtime = rt; + + nxt_memzero(&msg, sizeof(nxt_port_recv_msg_t)); + + /* + * No payload at all. The dispatcher always hands the handler a buffer, + * but a NULL one costs a single test to rule out and would be a NULL + * dereference otherwise. + */ + msg.buf = NULL; + + ret = nxt_port_change_file_test_reject(thr, task, &msg, 1, "no payload"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + /* + * A buffer shorter than the slot it is supposed to carry: the sender + * declares the size, so a zero-length payload arrives the same way a + * full one does, and reading the slot out of it reads past its end. + */ + b = nxt_buf_mem_alloc(mp, sizeof(nxt_uint_t), 0); + if (nxt_slow_path(b == NULL)) { + ret = NXT_ERROR; + goto done; + } + + msg.buf = b; + + ret = nxt_port_change_file_test_reject(thr, task, &msg, 1, + "short payload"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + /* + * A slot past the end of rt->log_files. nxt_list_elt() returns NULL + * for it, and the handler used to dereference that straight away. + */ + slot = 4242; + b->mem.free = nxt_cpymem(b->mem.free, &slot, sizeof(nxt_uint_t)); + + ret = nxt_port_change_file_test_reject(thr, task, &msg, 1, + "out of range slot"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + /* An existing slot, but nothing to redirect it to. */ + b->mem.free = b->mem.pos; + slot = 1; + b->mem.free = nxt_cpymem(b->mem.free, &slot, sizeof(nxt_uint_t)); + + ret = nxt_port_change_file_test_reject(thr, task, &msg, 0, + "slot without a descriptor"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + /* + * A slot whose file was never opened: dup2() onto NXT_FILE_INVALID + * fails, and nxt_file_redirect() used to return without closing the + * descriptor it had been handed -- the caller does not close it on the + * success path, so nobody did. + */ + b->mem.free = b->mem.pos; + slot = 0; + b->mem.free = nxt_cpymem(b->mem.free, &slot, sizeof(nxt_uint_t)); + + ret = nxt_port_change_file_test_reject(thr, task, &msg, 1, + "slot that cannot be redirected"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + b->mem.free = b->mem.pos; + slot = 1; + b->mem.free = nxt_cpymem(b->mem.free, &slot, sizeof(nxt_uint_t)); + + ret = nxt_port_change_file_test_accept(thr, task, &msg, log_file, + "redirected slot"); + +done: + + thr->runtime = saved_rt; + + if (log_file->fd != -1) { + (void) close(log_file->fd); + } + + nxt_mp_destroy(mp); + + if (nxt_fast_path(ret == NXT_OK)) { + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port change file test passed"); + } + + return ret; +} diff --git a/src/test/nxt_port_ctrunc_test.c b/src/test/nxt_port_ctrunc_test.c new file mode 100644 index 000000000..16a032e87 --- /dev/null +++ b/src/test/nxt_port_ctrunc_test.c @@ -0,0 +1,1164 @@ +/* + * Copyright (C) F5, Inc. + */ + +/* + * Regression test for control-data truncation on a port socket + * (src/nxt_socket_msg.h, src/nxt_socket_msg.c). + * + * recvmsg() sets MSG_CTRUNC when it could not deliver all of the control + * data the sender attached. The case that matters here is an SCM_RIGHTS the + * kernel discards while delivering everything else -- the payload and, on + * Linux, the SCM_CREDENTIALS the receiver's SO_PASSCRED asks for. In + * production that happens under descriptor pressure: scm_detach_fds() cannot + * install the descriptors and drops the whole cmsg. + * + * Nothing used to look at msg_flags, so what reached a handler was a message + * that passed every check it makes -- right size, right type, kernel-validated + * sender -- and simply had fd[0] == -1. NEW_PORT, MMAP and CHANGE_FILE each + * take some fd-less path from there. + * + * The truncation is driven directly rather than through RLIMIT_NOFILE: the + * receive buffer's length is the other input to the same kernel decision, and + * unlike a descriptor limit it is deterministic and leaves the rest of the + * test process alone. What is asserted is the contract every receive path + * shares, that nxt_socket_msg_oob_get() and nxt_socket_msg_oob_get_fds() + * refuse a truncated control block -- and that they refuse it after reporting + * whatever descriptors did arrive, since the caller can only close what it + * has been told about. + */ + +#include +#include +#include +#include +#include +#include "nxt_tests.h" + +#include + + +#define NXT_CTRUNC_TEST_FD_MAX 1024 + +/* + * The control space the receiver has to keep for the credential cmsg before + * the SCM_RIGHTS can begin. Zero where the platform passes no credential. + */ +#if (NXT_CRED_USECMSG) +#define NXT_CTRUNC_TEST_CRED_SPACE CMSG_SPACE(sizeof(nxt_socket_cred_t)) +#else +#define NXT_CTRUNC_TEST_CRED_SPACE 0 +#endif + + +static nxt_uint_t +nxt_port_ctrunc_test_fd_count(void) +{ + int fd; + nxt_uint_t n; + + n = 0; + + for (fd = 0; fd < NXT_CTRUNC_TEST_FD_MAX; fd++) { + if (nxt_test_fd_is_open(fd)) { + n++; + } + } + + return n; +} + + +/* + * Send one message carrying two descriptors, and receive it with a control + * buffer of exactly "controllen" bytes, filling "oob" the way nxt_recvmsg() + * would. A short buffer is what makes the kernel drop the SCM_RIGHTS and + * raise MSG_CTRUNC. + */ +static nxt_int_t +nxt_port_ctrunc_test_exchange(nxt_thread_t *thr, nxt_socket_t *pair, + size_t controllen, nxt_recv_oob_t *oob, ssize_t *received) +{ + int fds[2]; + u_char payload; + ssize_t n; + nxt_send_oob_t send_oob; + struct iovec iov[1]; + struct msghdr msg; + + fds[0] = open("/dev/null", O_RDONLY); + fds[1] = open("/dev/null", O_RDONLY); + + if (fds[0] == -1 || fds[1] == -1) { + nxt_log_alert(thr->log, "port ctrunc test failed to open /dev/null"); + goto fail; + } + + nxt_socket_msg_oob_init(&send_oob, fds); + + payload = 0x5A; + + iov[0].iov_base = &payload; + iov[0].iov_len = sizeof(payload); + + n = nxt_sendmsg(pair[1], iov, 1, &send_oob); + + if (n != (ssize_t) sizeof(payload)) { + nxt_log_alert(thr->log, "port ctrunc test sendmsg failed %E", + nxt_errno); + goto fail; + } + + /* + * sendmsg() duplicated the descriptors into the socket; this end has no + * further use for them, and leaving them open would hide a leak on the + * receiving side behind a constant offset. + */ + (void) close(fds[0]); + (void) close(fds[1]); + + fds[0] = -1; + fds[1] = -1; + + payload = 0; + + iov[0].iov_base = &payload; + iov[0].iov_len = sizeof(payload); + + nxt_memzero(oob, sizeof(nxt_recv_oob_t)); + + msg.msg_name = NULL; + msg.msg_namelen = 0; + msg.msg_iov = iov; + msg.msg_iovlen = 1; + msg.msg_control = oob->buf; + msg.msg_controllen = controllen; + msg.msg_flags = 0; + + n = recvmsg(pair[0], &msg, 0); + + if (n != (ssize_t) sizeof(payload) || payload != 0x5A) { + nxt_log_alert(thr->log, "port ctrunc test recvmsg failed %z %E", + n, nxt_errno); + return NXT_ERROR; + } + + oob->size = msg.msg_controllen; + oob->truncated = (msg.msg_flags & MSG_CTRUNC) != 0; + + *received = n; + + return NXT_OK; + +fail: + + if (fds[0] != -1) { + (void) close(fds[0]); + } + + if (fds[1] != -1) { + (void) close(fds[1]); + } + + return NXT_ERROR; +} + + +static void +nxt_port_ctrunc_test_close_fds(nxt_fd_t *fd) +{ + nxt_uint_t i; + + for (i = 0; i < 2; i++) { + if (fd[i] != -1) { + (void) close(fd[i]); + fd[i] = -1; + } + } +} + + +/* + * Do "controllen" bytes of control buffer hold the two descriptors that were + * sent, and does the kernel say so? + * + * Returns NXT_OK when MSG_CTRUNC came back, NXT_DECLINED when the exchange + * worked and the flag did not, and NXT_ERROR when the probe itself could not + * run -- a caller that cannot tell those apart reports a kernel property it + * never observed. "used" and "flags" carry what was seen, for the log. + * + * Only SCM_RIGHTS is used, so this is the same on every platform that has any + * of it at all. + */ + +static nxt_int_t +nxt_port_ctrunc_test_probe(size_t controllen, size_t *used, int *flags) +{ + int fd[2], received; + char buf, *data, *limit; + size_t i; + ssize_t n; + nxt_int_t ret; + nxt_socket_t pair[2]; + struct iovec iov; + struct msghdr msg; + struct cmsghdr *cmsg; + union { + struct cmsghdr align; + char buf[CMSG_SPACE(2 * sizeof(int))]; + } send_control; + union { + struct cmsghdr align; + char buf[CMSG_SPACE(2 * sizeof(int))]; + } recv_control; + + *used = 0; + *flags = 0; + + if (nxt_slow_path(controllen > sizeof(recv_control.buf))) { + return NXT_ERROR; + } + + if (nxt_slow_path(socketpair(AF_UNIX, SOCK_STREAM, 0, pair) != 0)) { + return NXT_ERROR; + } + + if (nxt_slow_path(pipe(fd) != 0)) { + close(pair[0]); + close(pair[1]); + + return NXT_ERROR; + } + + buf = 'x'; + iov.iov_base = &buf; + iov.iov_len = 1; + + nxt_memzero(&msg, sizeof(struct msghdr)); + nxt_memzero(&send_control, sizeof(send_control)); + + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + msg.msg_control = send_control.buf; + msg.msg_controllen = sizeof(send_control.buf); + + cmsg = CMSG_FIRSTHDR(&msg); + cmsg->cmsg_level = SOL_SOCKET; + cmsg->cmsg_type = SCM_RIGHTS; + cmsg->cmsg_len = CMSG_LEN(2 * sizeof(int)); + nxt_memcpy(CMSG_DATA(cmsg), fd, 2 * sizeof(int)); + + ret = NXT_ERROR; + + if (nxt_fast_path(sendmsg(pair[0], &msg, 0) == 1)) { + + nxt_memzero(&msg, sizeof(struct msghdr)); + nxt_memzero(&recv_control, sizeof(recv_control)); + + iov.iov_base = &buf; + iov.iov_len = 1; + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + msg.msg_control = recv_control.buf; + msg.msg_controllen = controllen; + + n = recvmsg(pair[1], &msg, 0); + + if (nxt_fast_path(n == 1)) { + + /* + * Everything below walks what the kernel returned, and the reason + * this probe exists is that the layer underneath may not be one. + * Nothing here trusts a length it did not bound first: a + * msg_controllen larger than the buffer that was offered, or a + * cmsg_len reaching past it, would otherwise walk off the union + * and close whatever integer it found there. + */ + + if (msg.msg_controllen > controllen) { + msg.msg_controllen = controllen; + } + + *used = msg.msg_controllen; + *flags = msg.msg_flags; + + ret = (msg.msg_flags & MSG_CTRUNC) ? NXT_OK : NXT_DECLINED; + + limit = recv_control.buf + msg.msg_controllen; + + for (cmsg = CMSG_FIRSTHDR(&msg); + cmsg != NULL; + cmsg = NXT_CMSG_NXTHDR(&msg, cmsg)) + { + if (cmsg->cmsg_level != SOL_SOCKET + || cmsg->cmsg_type != SCM_RIGHTS) + { + continue; + } + + /* + * Whatever did fit was installed and belongs to this process + * now; leaving it open would move the descriptor count the + * cases this gates are measuring. Standard streams are never + * what was sent, so a 0, 1 or 2 here is a misread rather than + * a descriptor to close. + */ + + data = (char *) CMSG_DATA(cmsg); + + for (i = 0; data + i + sizeof(int) <= limit + && CMSG_LEN(i + sizeof(int)) <= cmsg->cmsg_len; + i += sizeof(int)) + { + nxt_memcpy(&received, data + i, sizeof(int)); + + if (received > STDERR_FILENO) { + close(received); + } + } + } + } + } + + close(fd[0]); + close(fd[1]); + close(pair[0]); + close(pair[1]); + + return ret; +} + + +/* + * The regression itself: a control buffer that holds the credential but not + * the SCM_RIGHTS. Everything a handler inspects is intact and the message is + * one descriptor short, which is exactly the shape the kernel delivers under + * descriptor pressure. Before the fix both accessors returned NXT_OK here. + */ +static nxt_int_t +nxt_port_ctrunc_test_truncated(nxt_thread_t *thr, nxt_socket_t *pair, + size_t controllen, const char *name) +{ + ssize_t n; + nxt_int_t ret; + nxt_pid_t pid; + nxt_uint_t before, after; + nxt_fd_t fd[2], fd2[2]; + nxt_recv_oob_t oob; + + before = nxt_port_ctrunc_test_fd_count(); + + if (nxt_port_ctrunc_test_exchange(thr, pair, controllen, &oob, &n) + != NXT_OK) + { + return NXT_ERROR; + } + + if (!oob.truncated) { + nxt_log_alert(thr->log, "port ctrunc test: %s did not truncate " + "(controllen %uz, received %uz)", + name, controllen, oob.size); + return NXT_ERROR; + } + + fd[0] = -1; + fd[1] = -1; + + /* + * The two accessors are checked against the same received control block, + * so both see the identical descriptor numbers; they are closed once, at + * the end, rather than after each call. + */ + ret = nxt_socket_msg_oob_get_fds(&oob, fd); + + if (ret != NXT_ERROR) { + nxt_log_alert(thr->log, "port ctrunc test: %s: oob_get_fds accepted " + "a truncated control block (fd %d)", name, fd[0]); + + nxt_port_ctrunc_test_close_fds(fd); + + return NXT_ERROR; + } + + fd2[0] = -1; + fd2[1] = -1; + pid = -1; + + ret = nxt_socket_msg_oob_get(&oob, fd2, &pid); + + if (ret != NXT_ERROR) { + nxt_log_alert(thr->log, "port ctrunc test: %s: oob_get accepted a " + "truncated control block (fd %d, pid %PI)", + name, fd2[0], pid); + + nxt_port_ctrunc_test_close_fds(fd); + + return NXT_ERROR; + } + + if (fd2[0] != fd[0] || fd2[1] != fd[1]) { + nxt_log_alert(thr->log, "port ctrunc test: %s: the two accessors " + "reported different descriptors (%d %d vs %d %d)", + name, fd[0], fd[1], fd2[0], fd2[1]); + + nxt_port_ctrunc_test_close_fds(fd); + + return NXT_ERROR; + } + + /* + * Whatever the kernel did install has to come back in fd[] even though + * the call failed, or nothing can close it. + */ + nxt_port_ctrunc_test_close_fds(fd); + + after = nxt_port_ctrunc_test_fd_count(); + + if (after != before) { + nxt_log_alert(thr->log, "port ctrunc test: %s leaked %d descriptors", + name, (int) (after - before)); + return NXT_ERROR; + } + + return NXT_OK; +} + + +/* + * The control case: a buffer of the size every production receive path uses. + * A message Unit can send never truncates against it, so the accessors accept + * it and hand back both descriptors -- which is what makes the case above an + * assertion about truncation rather than about the buffer being short. + */ +static nxt_int_t +nxt_port_ctrunc_test_intact(nxt_thread_t *thr, nxt_socket_t *pair) +{ + ssize_t n; + nxt_int_t ret; + nxt_pid_t pid; + nxt_uint_t before, after; + nxt_fd_t fd[2]; + nxt_recv_oob_t oob; + + before = nxt_port_ctrunc_test_fd_count(); + + if (nxt_port_ctrunc_test_exchange(thr, pair, NXT_OOB_RECV_SIZE, &oob, &n) + != NXT_OK) + { + return NXT_ERROR; + } + + if (oob.truncated) { + nxt_log_alert(thr->log, "port ctrunc test: a %uz byte control buffer " + "truncated a two descriptor message", + (size_t) NXT_OOB_RECV_SIZE); + return NXT_ERROR; + } + + fd[0] = -1; + fd[1] = -1; + pid = -1; + + ret = nxt_socket_msg_oob_get(&oob, fd, &pid); + + if (ret != NXT_OK || fd[0] == -1 || fd[1] == -1) { + nxt_log_alert(thr->log, "port ctrunc test: intact message refused " + "(ret %i, fd %d %d)", ret, fd[0], fd[1]); + + nxt_port_ctrunc_test_close_fds(fd); + + return NXT_ERROR; + } + +#if (NXT_CRED_USECMSG) + if (pid != getpid()) { + nxt_log_alert(thr->log, "port ctrunc test: intact message carried " + "pid %PI, expected %PI", pid, (nxt_pid_t) getpid()); + + nxt_port_ctrunc_test_close_fds(fd); + + return NXT_ERROR; + } +#endif + + nxt_port_ctrunc_test_close_fds(fd); + + after = nxt_port_ctrunc_test_fd_count(); + + if (after != before) { + nxt_log_alert(thr->log, "port ctrunc test: intact case leaked %d " + "descriptors", (int) (after - before)); + return NXT_ERROR; + } + + return NXT_OK; +} + + +/* + * The accessor contract on its own, with no kernel behaviour in it: take a + * control block the kernel delivered whole, mark it truncated, and require + * the same refusal. This is what holds everywhere -- the cases above depend + * on how a particular kernel divides a short control buffer, and xnu does not + * divide it the same way -- and it is also the exact shape the accessors see + * in production, since a real truncation can deliver every descriptor it was + * asked for and still raise MSG_CTRUNC. + */ +static nxt_int_t +nxt_port_ctrunc_test_marked(nxt_thread_t *thr, nxt_socket_t *pair) +{ + ssize_t n; + nxt_int_t ret; + nxt_pid_t pid; + nxt_uint_t before, after; + nxt_fd_t fd[2]; + nxt_recv_oob_t oob; + + before = nxt_port_ctrunc_test_fd_count(); + + if (nxt_port_ctrunc_test_exchange(thr, pair, NXT_OOB_RECV_SIZE, &oob, &n) + != NXT_OK) + { + return NXT_ERROR; + } + + if (oob.truncated) { + nxt_log_alert(thr->log, "port ctrunc test: a full size control " + "buffer truncated a two descriptor message"); + return NXT_ERROR; + } + + oob.truncated = 1; + + fd[0] = -1; + fd[1] = -1; + pid = -1; + + ret = nxt_socket_msg_oob_get(&oob, fd, &pid); + + if (ret != NXT_ERROR) { + nxt_log_alert(thr->log, "port ctrunc test: oob_get accepted a block " + "marked truncated (fd %d %d)", fd[0], fd[1]); + + nxt_port_ctrunc_test_close_fds(fd); + + return NXT_ERROR; + } + + /* + * Every descriptor the block carries still has to be reported, or the + * caller that is about to drop the message cannot close them. + */ + if (fd[0] == -1 || fd[1] == -1) { + nxt_log_alert(thr->log, "port ctrunc test: a block marked truncated " + "withheld its descriptors (fd %d %d)", fd[0], fd[1]); + + nxt_port_ctrunc_test_close_fds(fd); + + return NXT_ERROR; + } + + nxt_port_ctrunc_test_close_fds(fd); + + after = nxt_port_ctrunc_test_fd_count(); + + if (after != before) { + nxt_log_alert(thr->log, "port ctrunc test: marked case leaked %d " + "descriptors", (int) (after - before)); + return NXT_ERROR; + } + + return NXT_OK; +} + + +/* + * Linux only, deliberately narrower than NXT_CRED_USECMSG. This fixture + * manufactures a message the receiver must refuse by relying on the + * credential being *enforced but not attached*: nxt_socket_msg_oob_get() + * fails a control block that carries no credential, and with SO_PASSCRED off + * this socketpair the kernel adds none. Where the credential travels as + * SCM_CREDS instead (FreeBSD), nxt_socket_msg_oob_init() attaches one itself, + * so the same call produces a perfectly valid message and the refusal this + * fixture is built on never happens. The truncation cases above cover those + * platforms; this one needs Linux semantics to construct its input at all. + */ + +#if (NXT_HAVE_UCRED) + +static nxt_uint_t nxt_port_ctrunc_test_delivered; +static nxt_uint_t nxt_port_ctrunc_test_enables; + + +static void +nxt_port_ctrunc_test_quit_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) +{ + nxt_port_ctrunc_test_delivered++; +} + + +static const nxt_port_handlers_t nxt_port_ctrunc_test_handlers = { + .quit = nxt_port_ctrunc_test_quit_handler, +}; + + +static void +nxt_port_ctrunc_test_enable_read_stub(nxt_event_engine_t *engine, + nxt_fd_event_t *ev) +{ + /* + * The fixture engine polls nothing. Count the calls, and leave the + * event active as a real engine does. A drain that ends with EAGAIN + * must not re-arm an active read event: on epoll that costs an + * epoll_ctl() call per drain. + */ + + nxt_port_ctrunc_test_enables++; + + ev->read = NXT_EVENT_ACTIVE; +} + + +/* + * Send one message over "fd" with a control block the receiver will refuse. + * "with_fd" attaches a descriptor, so that the refusal happens after the + * descriptor has been delivered -- the shape a truncation has. + */ +static nxt_int_t +nxt_port_ctrunc_test_send_bad(nxt_thread_t *thr, nxt_socket_t fd, + nxt_port_msg_t *port_msg) +{ + int fds[2]; + ssize_t n; + nxt_send_oob_t oob; + struct iovec iov[1]; + + fds[0] = open("/dev/null", O_RDONLY); + fds[1] = -1; + + if (fds[0] == -1) { + nxt_log_alert(thr->log, "port ctrunc test failed to open /dev/null"); + return NXT_ERROR; + } + + nxt_socket_msg_oob_init(&oob, fds); + + iov[0].iov_base = port_msg; + iov[0].iov_len = sizeof(nxt_port_msg_t); + + n = nxt_sendmsg(fd, iov, 1, &oob); + + (void) close(fds[0]); + + if (n != (ssize_t) sizeof(nxt_port_msg_t)) { + nxt_log_alert(thr->log, "port ctrunc test sendmsg failed %E", + nxt_errno); + return NXT_ERROR; + } + + return NXT_OK; +} + + +/* + * The control message: no control data at all, which nxt_socket_msg_oob_get() + * accepts on any platform, so the handler dispatches it. + */ +static nxt_int_t +nxt_port_ctrunc_test_send_good(nxt_thread_t *thr, nxt_socket_t fd, + nxt_port_msg_t *port_msg) +{ + ssize_t n; + struct iovec iov[1]; + + iov[0].iov_base = port_msg; + iov[0].iov_len = sizeof(nxt_port_msg_t); + + n = nxt_sendmsg(fd, iov, 1, NULL); + + if (n != (ssize_t) sizeof(nxt_port_msg_t)) { + nxt_log_alert(thr->log, "port ctrunc test sendmsg failed %E", + nxt_errno); + return NXT_ERROR; + } + + return NXT_OK; +} + + +/* + * Announce a socket message on the queue the way nxt_port_socket_send() does. + * Without the READ_SOCKET marker the handler has port->from_socket == 0 and + * suspends the socket message instead of processing it, which is a different + * path from the one under test. + */ +static nxt_int_t +nxt_port_ctrunc_test_notify(nxt_thread_t *thr, nxt_port_queue_t *queue) +{ + int notify; + uint8_t qmsg; + + qmsg = _NXT_PORT_MSG_READ_SOCKET; + + if (nxt_port_queue_send(queue, &qmsg, 1, ¬ify) != NXT_OK) { + nxt_log_alert(thr->log, "port ctrunc test failed to queue a " + "read_socket marker"); + return NXT_ERROR; + } + + return NXT_OK; +} + + +/* + * Put a whole message on the queue, the way nxt_port_socket_write2() does for + * an enqueueable one, and report what it decided about the wakeup. "notify" + * is the crux of the stranding case: nxt_port_queue_send() clears it when + * nitems is already non-zero, on the understanding that a reader is awake and + * will come back round for this message. + */ +static nxt_int_t +nxt_port_ctrunc_test_enqueue(nxt_thread_t *thr, nxt_port_queue_t *queue, + nxt_port_msg_t *port_msg, int *notify) +{ + if (nxt_port_queue_send(queue, port_msg, sizeof(nxt_port_msg_t), notify) + != NXT_OK) + { + nxt_log_alert(thr->log, "port ctrunc test failed to enqueue a " + "queue message"); + return NXT_ERROR; + } + + return NXT_OK; +} + + +/* + * The queue-backed read path drops a refused message rather than failing the + * port, and has to give back everything the iteration took before it does. + * + * queue->nitems is the one that bites: it is incremented on entry and read by + * nxt_port_queue_send() in every *other* process holding this port, which + * skips the socket notification while it is non-zero because a reader is + * supposed to be awake. A bare return therefore left the counter raised + * forever, and the port was never woken for a queue-only message again -- + * one refused datagram costing the port every message after it. The buffer + * and the READ_SOCKET marker leaked on the same path. + * + * The refusal is driven by a control block the accessor rejects for a reason + * that needs no kernel cooperation: SO_PASSCRED is left off this socketpair, + * so the SCM_CREDENTIALS that nxt_socket_msg_oob_get() insists on never + * arrives, and the message is refused after its descriptor was delivered -- + * the same shape, and the same cleanup path, as a truncation. Driving an + * actual MSG_CTRUNC here would mean exhausting the test process's descriptor + * table, which is neither deterministic nor confined to this test. + */ +static nxt_int_t +nxt_port_ctrunc_test_queue(nxt_thread_t *thr) +{ + nxt_mp_t *mp; + int notify; + nxt_int_t ret; + nxt_port_t *port; + nxt_task_t *task; + nxt_runtime_t *rt, *saved_rt; + nxt_port_msg_t port_msg; + nxt_port_queue_t *queue; + nxt_event_engine_t *engine, *saved_engine; + nxt_uint_t enables; + nxt_nncq_atomic_t nitems; + + task = thr->task; + task->thread = thr; + + port = NULL; + queue = MAP_FAILED; + ret = NXT_ERROR; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + rt = nxt_mp_zalloc(mp, sizeof(nxt_runtime_t)); + engine = nxt_mp_zalloc(mp, sizeof(nxt_event_engine_t)); + + if (nxt_slow_path(rt == NULL || engine == NULL)) { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + rt->mem_pool = mp; + + if (nxt_slow_path(nxt_thread_mutex_create(&rt->processes_mutex) != NXT_OK)) + { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + saved_rt = thr->runtime; + saved_engine = thr->engine; + + thr->runtime = rt; + thr->engine = engine; + + engine->event.enable_read = nxt_port_ctrunc_test_enable_read_stub; + + port = nxt_port_new(task, 0, nxt_pid, NXT_PROCESS_APP); + if (nxt_slow_path(port == NULL)) { + goto done; + } + + /* + * Plain socketpair(), not nxt_socketpair_create(): the missing + * SO_PASSCRED is what makes the receiver refuse the message. The + * non-blocking mode that nxt_socketpair_create() also sets is not + * optional, though -- the read loop ends by receiving EAGAIN, so a + * blocking descriptor parks the handler in recvmsg() for good. + */ + if (nxt_slow_path(socketpair(AF_UNIX, SOCK_DGRAM, 0, port->pair) != 0)) { + nxt_log_alert(thr->log, "port ctrunc test: socketpair failed %E", + nxt_errno); + goto done; + } + + if (nxt_slow_path(nxt_socket_nonblocking(task, port->pair[0]) != NXT_OK + || nxt_socket_nonblocking(task, port->pair[1]) != NXT_OK)) + { + goto done; + } + + port->max_size = 1024; + + /* + * nxt_socketpair_recv() logs through ev->task; the event facility would + * have set it when the port was created against a real engine. + */ + port->socket.task = task; + + queue = mmap(NULL, sizeof(nxt_port_queue_t), PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_ANON, -1, 0); + + if (nxt_slow_path(queue == MAP_FAILED)) { + nxt_log_alert(thr->log, "port ctrunc test failed to map a queue"); + goto done; + } + + nxt_port_queue_init(queue); + + port->queue = queue; + + nxt_port_enable(task, port, &nxt_port_ctrunc_test_handlers); + nxt_port_read_enable(task, port); + + if (nxt_slow_path(port->socket.read_handler == NULL)) { + nxt_log_alert(thr->log, "port ctrunc test: no queue read handler"); + goto done; + } + + nitems = queue->nitems; + + nxt_memzero(&port_msg, sizeof(nxt_port_msg_t)); + port_msg.type = _NXT_PORT_MSG_QUIT; + port_msg.last = 1; + + if (nxt_slow_path(nxt_port_ctrunc_test_send_bad(thr, port->pair[1], + &port_msg) != NXT_OK)) + { + goto done; + } + + if (nxt_slow_path(nxt_port_ctrunc_test_notify(thr, queue) != NXT_OK)) { + goto done; + } + + /* What the event facility would have set. */ + port->socket.read_ready = 1; + + nxt_port_ctrunc_test_delivered = 0; + + enables = nxt_port_ctrunc_test_enables; + + port->socket.read_handler(task, &port->socket, NULL); + + if (nxt_port_ctrunc_test_enables != enables) { + nxt_log_alert(thr->log, "port ctrunc test: drain 1 re-armed " + "an active read event"); + goto done; + } + + if (nxt_port_ctrunc_test_delivered != 0) { + nxt_log_alert(thr->log, "port ctrunc test: a message with an " + "unusable control block was delivered"); + goto done; + } + + if (queue->nitems != nitems) { + nxt_log_alert(thr->log, "port ctrunc test: a refused message left " + "queue->nitems at %uD, was %uD; the port would never " + "be notified again", + (uint32_t) queue->nitems, (uint32_t) nitems); + goto done; + } + + if (port->from_socket != 0) { + nxt_log_alert(thr->log, "port ctrunc test: a refused message left " + "from_socket at %d", (int) port->from_socket); + goto done; + } + + /* + * The point of the counter: the very next message still has to arrive. + */ + if (nxt_slow_path(nxt_port_ctrunc_test_send_good(thr, port->pair[1], + &port_msg) != NXT_OK)) + { + goto done; + } + + if (nxt_slow_path(nxt_port_ctrunc_test_notify(thr, queue) != NXT_OK)) { + goto done; + } + + port->socket.read_ready = 1; + + enables = nxt_port_ctrunc_test_enables; + + port->socket.read_handler(task, &port->socket, NULL); + + if (nxt_port_ctrunc_test_enables != enables) { + nxt_log_alert(thr->log, "port ctrunc test: drain 2 re-armed " + "an active read event"); + goto done; + } + + if (nxt_port_ctrunc_test_delivered != 1) { + nxt_log_alert(thr->log, "port ctrunc test: the message after a " + "refused one was not delivered"); + goto done; + } + + /* + * The case the counter alone does not cover: a message queued inside the + * window, after the READ_SOCKET marker but before this handler gets to + * the datagram it announces. + * + * Its sender sees nitems already raised and so asks for no wakeup -- the + * assertion below -- because a reader is supposed to be awake and about to + * drain the queue. That reader is this invocation. If the refusal path + * returns instead of carrying on round the loop, nothing is left to + * collect the message and nothing will be scheduled to: it is stranded + * with the counter correct and the port quiet. Restoring nitems is + * necessary and not sufficient. + */ + + nxt_port_ctrunc_test_delivered = 0; + + if (nxt_slow_path(nxt_port_ctrunc_test_send_bad(thr, port->pair[1], + &port_msg) != NXT_OK)) + { + goto done; + } + + if (nxt_slow_path(nxt_port_ctrunc_test_notify(thr, queue) != NXT_OK)) { + goto done; + } + + if (nxt_slow_path(nxt_port_ctrunc_test_enqueue(thr, queue, &port_msg, + ¬ify) != NXT_OK)) + { + goto done; + } + + if (notify != 0) { + nxt_log_alert(thr->log, "port ctrunc test: the queued message asked " + "for a wakeup, so it is not the stranding case"); + goto done; + } + + port->socket.read_ready = 1; + + enables = nxt_port_ctrunc_test_enables; + + port->socket.read_handler(task, &port->socket, NULL); + + if (nxt_port_ctrunc_test_enables != enables) { + nxt_log_alert(thr->log, "port ctrunc test: drain 3 re-armed " + "an active read event"); + goto done; + } + + if (nxt_port_ctrunc_test_delivered != 1) { + nxt_log_alert(thr->log, "port ctrunc test: a message queued behind a " + "refused one was stranded"); + goto done; + } + + if (queue->nitems != nitems) { + nxt_log_alert(thr->log, "port ctrunc test: draining past a refused " + "message left queue->nitems at %uD, was %uD", + (uint32_t) queue->nitems, (uint32_t) nitems); + goto done; + } + + ret = NXT_OK; + +done: + + /* + * The port carries its own memory pool and a use count, and its pool + * cleanup asserts that both descriptors are already gone -- so it has to + * be taken down through nxt_port_close(), not with close() on the pair, + * and then released by dropping the reference nxt_port_new() took. The + * queue is detached first: it is this fixture's mapping, unmapped below, + * and nxt_port_close() would otherwise release it as its own. + */ + if (port != NULL) { + port->queue = NULL; + + nxt_port_close(task, port); + nxt_port_use(task, port, -1); + } + + if (queue != MAP_FAILED) { + (void) munmap(queue, sizeof(nxt_port_queue_t)); + } + + thr->runtime = saved_rt; + thr->engine = saved_engine; + + nxt_thread_mutex_destroy(&rt->processes_mutex); + nxt_mp_destroy(mp); + + return ret; +} + +#endif /* NXT_HAVE_UCRED */ + + +nxt_int_t +nxt_port_ctrunc_test(nxt_thread_t *thr) +{ + nxt_int_t ret; + nxt_task_t *task; + nxt_socket_t pair[2]; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "port ctrunc test started"); + + task = thr->task; + task->thread = thr; + + /* + * nxt_socketpair_create() rather than socketpair(): SO_PASSCRED is what + * puts a credential cmsg ahead of the SCM_RIGHTS, and so what lets a + * buffer sized for the credential alone reproduce the production shape -- + * a fully authenticated message with its descriptor missing. + */ + if (nxt_socketpair_create(task, pair) != NXT_OK) { + nxt_log_alert(thr->log, "port ctrunc test: socketpair failed"); + return NXT_ERROR; + } + + ret = nxt_port_ctrunc_test_intact(thr, pair); + + if (ret == NXT_OK) { + ret = nxt_port_ctrunc_test_marked(thr, pair); + } + + /* + * The two cases below assert how the kernel itself divides a control + * buffer that is too small. They are two different decisions: with no + * room for any descriptor it drops the whole SCM_RIGHTS cmsg, and with + * room for one of two it installs what fits. Both must raise MSG_CTRUNC + * for the cases to mean anything, and not every layer does -- xnu + * externalizes the rights before it measures, and qemu-user's cmsg + * translation reproduces neither kernel. Nothing in the code under test + * acts on a truncation that was not reported, so where the flag does not + * come back there is nothing to prove and asserting it anyway fails for + * the environment rather than for the code. + * + * Probe both shapes rather than one: a layer that mistranslates the + * budget for the second decision need not mistranslate the first, and a + * predicate narrower than what it gates would let a case through to fail + * for the environment again. + */ + if (ret == NXT_OK) { + nxt_int_t drop, partial; + size_t drop_used, partial_used; + int drop_flags, partial_flags; + + drop = nxt_port_ctrunc_test_probe(CMSG_LEN(0), &drop_used, + &drop_flags); + partial = nxt_port_ctrunc_test_probe(CMSG_LEN(0) + sizeof(int), + &partial_used, &partial_flags); + + if (drop == NXT_ERROR || partial == NXT_ERROR) { + nxt_log_alert(thr->log, "port ctrunc test: the MSG_CTRUNC probe " + "could not run, so the kernel driven cases cannot " + "be decided either way"); + ret = NXT_ERROR; + goto done; + } + + if (drop != NXT_OK || partial != NXT_OK) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, "port ctrunc test: " + "kernel driven truncation cases skipped -- this " + "kernel does not report MSG_CTRUNC (no room: used " + "%uz flags 0x%04Xd; one of two: used %uz flags " + "0x%04Xd)", drop_used, (unsigned) drop_flags, + partial_used, (unsigned) partial_flags); + goto done; + } + } + + if (ret == NXT_OK) { + ret = nxt_port_ctrunc_test_truncated(thr, pair, + NXT_CTRUNC_TEST_CRED_SPACE + CMSG_LEN(0), + "credential kept, SCM_RIGHTS dropped"); + } + + if (ret == NXT_OK) { + /* + * The other side of the same kernel decision: room for one descriptor + * where two were sent. scm_detach_fds() installs what fits and still + * raises MSG_CTRUNC, so the accessors have to report the partial + * descriptor and refuse the message, or it leaks. + * + * The length is CMSG_LEN(0) + sizeof(int), not CMSG_SPACE(sizeof(int)): + * cmsg padding rounds a one-descriptor payload up to the same space + * two occupy, so a CMSG_SPACE-derived budget delivers both and does + * not truncate at all. + */ + ret = nxt_port_ctrunc_test_truncated(thr, pair, + NXT_CTRUNC_TEST_CRED_SPACE + + CMSG_LEN(0) + sizeof(int), + "one of two descriptors delivered"); + } + +done: + + nxt_socket_close(task, pair[0]); + nxt_socket_close(task, pair[1]); + +#if (NXT_HAVE_UCRED) + + if (ret == NXT_OK) { + ret = nxt_port_ctrunc_test_queue(thr); + } + +#else + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "port ctrunc test: queue read " + "path case skipped without SCM_CREDENTIALS"); + +#endif + + if (ret != NXT_OK) { + return NXT_ERROR; + } + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "port ctrunc test passed"); + + return NXT_OK; +} diff --git a/src/test/nxt_port_fail_test.c b/src/test/nxt_port_fail_test.c index 1e1722ba3..d91e29fea 100644 --- a/src/test/nxt_port_fail_test.c +++ b/src/test/nxt_port_fail_test.c @@ -6,6 +6,7 @@ #include #include #include +#include #include "nxt_tests.h" #if (NXT_LINUX) @@ -15,9 +16,41 @@ static nxt_port_t *nxt_port_fail_test_port(nxt_task_t *task); static nxt_int_t nxt_port_fail_test_socket_write(nxt_thread_t *thr); +static nxt_int_t nxt_port_fail_test_inline_drop(nxt_thread_t *thr); +static void nxt_port_fail_test_enable_write(nxt_event_engine_t *engine, + nxt_fd_event_t *ev); +static nxt_int_t nxt_port_fail_test_wakeup_errno(nxt_thread_t *thr); +#if (NXT_HAVE_EPOLL_EDGE) +static nxt_int_t nxt_port_fail_test_enqueue(nxt_task_t *task, + nxt_port_t *port, nxt_mp_t *mp, nxt_err_t err, nxt_uint_t fails, + nxt_bool_t no_memory, nxt_bool_t *queued); +static void nxt_port_fail_test_turn(nxt_event_engine_t *engine, nxt_msec_t ms); +static nxt_uint_t nxt_port_fail_test_spin(nxt_task_t *task, + nxt_msec_t window, nxt_port_t *rearm); +#endif +static nxt_int_t nxt_port_fail_test_dead_peer(nxt_thread_t *thr); +static nxt_int_t nxt_port_fail_test_quit_log_level(nxt_thread_t *thr); +static nxt_int_t nxt_port_fail_test_send_to_dead_peer(nxt_thread_t *thr, + nxt_uint_t type, nxt_bool_t queued, nxt_bool_t shared, nxt_uint_t then, + nxt_uint_t *level); +static void nxt_cdecl nxt_port_fail_test_log_handler(nxt_uint_t level, + nxt_log_t *log, const char *fmt, ...); static nxt_int_t nxt_port_fail_test_rpc_register(nxt_thread_t *thr); static nxt_int_t nxt_port_fail_test_error_handler(nxt_thread_t *thr); static nxt_int_t nxt_port_fail_test_mp_baseline(nxt_thread_t *thr); +static nxt_int_t nxt_port_fail_test_cross_engine_release(nxt_thread_t *thr); +static nxt_int_t nxt_port_fail_test_cross_engine_acquire(nxt_thread_t *thr); +static nxt_int_t nxt_port_fail_test_cross_engine_race(nxt_thread_t *thr); +static nxt_int_t nxt_port_fail_test_cross_engine_batch(nxt_thread_t *thr); +static nxt_int_t nxt_port_fail_test_cross_engine_rearm(nxt_thread_t *thr); +static nxt_int_t nxt_port_fail_test_caller_owns_buf(nxt_thread_t *thr); +static void nxt_port_fail_test_racer(void *data); +static nxt_int_t nxt_port_fail_test_queued(nxt_locked_work_queue_t *lwq, + nxt_work_t *item); +static void nxt_port_fail_test_engine_signal(nxt_event_engine_t *engine, + nxt_uint_t signo); +static void nxt_port_fail_test_released(nxt_task_t *task, void *obj, + void *data); static nxt_int_t nxt_port_fail_test_sender_pattern(nxt_task_t *task, nxt_port_t *port, nxt_mp_t *mp); static void nxt_port_fail_test_mp_completion(nxt_task_t *task, void *obj, @@ -25,11 +58,13 @@ static void nxt_port_fail_test_mp_completion(nxt_task_t *task, void *obj, static void nxt_port_fail_test_completion(nxt_task_t *task, void *obj, void *data); static void nxt_port_fail_test_drain_wq(nxt_work_queue_t *wq); -static nxt_bool_t nxt_port_fail_test_fd_is_open(nxt_fd_t fd); static nxt_int_t nxt_port_fail_test_fd_count(void); static nxt_uint_t nxt_port_fail_test_completions; +static nxt_uint_t nxt_port_fail_test_rearms; +static nxt_uint_t nxt_port_fail_test_releases; +static nxt_uint_t nxt_port_fail_test_signals; nxt_int_t @@ -42,6 +77,22 @@ nxt_port_fail_test(nxt_thread_t *thr) return NXT_ERROR; } + if (nxt_port_fail_test_inline_drop(thr) != NXT_OK) { + return NXT_ERROR; + } + + if (nxt_port_fail_test_dead_peer(thr) != NXT_OK) { + return NXT_ERROR; + } + + if (nxt_port_fail_test_quit_log_level(thr) != NXT_OK) { + return NXT_ERROR; + } + + if (nxt_port_fail_test_wakeup_errno(thr) != NXT_OK) { + return NXT_ERROR; + } + if (nxt_port_fail_test_rpc_register(thr) != NXT_OK) { return NXT_ERROR; } @@ -54,6 +105,30 @@ nxt_port_fail_test(nxt_thread_t *thr) return NXT_ERROR; } + if (nxt_port_fail_test_cross_engine_release(thr) != NXT_OK) { + return NXT_ERROR; + } + + if (nxt_port_fail_test_cross_engine_acquire(thr) != NXT_OK) { + return NXT_ERROR; + } + + if (nxt_port_fail_test_cross_engine_race(thr) != NXT_OK) { + return NXT_ERROR; + } + + if (nxt_port_fail_test_cross_engine_batch(thr) != NXT_OK) { + return NXT_ERROR; + } + + if (nxt_port_fail_test_cross_engine_rearm(thr) != NXT_OK) { + return NXT_ERROR; + } + + if (nxt_port_fail_test_caller_owns_buf(thr) != NXT_OK) { + return NXT_ERROR; + } + nxt_thread_time_update(thr); nxt_log_error(NXT_LOG_NOTICE, thr->log, "port failure test passed"); @@ -147,7 +222,7 @@ nxt_port_fail_test_socket_write(nxt_thread_t *thr) nxt_port_test_msg_alloc_failures(0); - if (!nxt_port_fail_test_fd_is_open(fd)) { + if (!nxt_test_fd_is_open(fd)) { nxt_log_error(NXT_LOG_NOTICE, thr->log, "port failure test closed fd before ownership transfer"); goto fail_close_port; @@ -183,7 +258,7 @@ nxt_port_fail_test_socket_write(nxt_thread_t *thr) fail_close_fd: - if (fd != -1 && nxt_port_fail_test_fd_is_open(fd)) { + if (fd != -1 && nxt_test_fd_is_open(fd)) { nxt_fd_close(fd); } @@ -200,460 +275,3240 @@ nxt_port_fail_test_socket_write(nxt_thread_t *thr) } +static void +nxt_port_fail_test_enable_write(nxt_event_engine_t *engine, nxt_fd_event_t *ev) +{ + nxt_port_fail_test_rearms++; +} + + +/* + * The inline write that the port cannot take. + * + * The other leg above drives the allocation failure in + * nxt_port_msg_chk_insert(), before anything is sent. This one drives the + * other half: the port is write-ready with an empty queue, so chk_insert() + * declines and nxt_port_socket_write2() sends inline, and there the socket + * answers EAGAIN with the message unable to queue for later either. + * + * Nothing of the message was consumed on that path, so the answer must be + * NXT_ERROR, with the descriptor still open and no completion queued: the + * ownership contract in src/nxt_port.h, which the caller relies on to free + * its own payload. Answering NXT_OK there is what left an RPC reply lost, + * with the caller believing it had sent it (#335). + * + * The write event must also be re-armed. nxt_socketpair_send() cleared + * write_ready on the EAGAIN, and nothing else sets it back, so a port that + * skipped the re-arm would never drain what is already queued on it. The + * stub engine below exists to count that call. + */ + static nxt_int_t -nxt_port_fail_test_rpc_register(nxt_thread_t *thr) +nxt_port_fail_test_inline_drop(nxt_thread_t *thr) { - void *ex; - nxt_task_t *task; - nxt_port_t *port; + int sndbuf; + ssize_t n; + nxt_mp_t *mp; + nxt_fd_t fd, pair[2]; + nxt_buf_t *buf; + nxt_int_t ret; + nxt_task_t *task; + nxt_port_t *port; + nxt_event_engine_t engine, *saved_engine; + nxt_port_queue_t *queue; + nxt_event_interface_t stub; + u_char block[4096]; task = thr->task; task->thread = thr; - if (nxt_port_rpc_init() != NXT_OK) { + ret = NXT_ERROR; + fd = -1; + pair[0] = -1; + pair[1] = -1; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { return NXT_ERROR; } port = nxt_port_fail_test_port(task); if (nxt_slow_path(port == NULL)) { + nxt_mp_destroy(mp); return NXT_ERROR; } - port->pair[0] = 0; + /* + * Only fast_work_queue and the one event method this path can reach are + * real. The code under test never reaches anything else, so it stays + * unset. + */ - nxt_port_rpc_test_alloc_failures(1); + nxt_memzero(&engine, sizeof(engine)); + nxt_memzero(&stub, sizeof(stub)); - ex = nxt_port_rpc_register_handler_ex(task, port, NULL, NULL, 0); - if (ex != NULL) { - nxt_log_error(NXT_LOG_NOTICE, thr->log, - "port failure test expected rpc alloc failure"); - goto fail; - } + nxt_work_queue_cache_create(&engine.work_queue_cache, 1024); + engine.fast_work_queue.cache = &engine.work_queue_cache; + nxt_work_queue_name(&engine.fast_work_queue, "fast"); - if (port->use_count != 1 || !nxt_lvlhsh_is_empty(&port->rpc_streams)) { - nxt_log_error(NXT_LOG_NOTICE, thr->log, - "port failure test leaked failed rpc alloc"); - goto fail; - } + stub.enable_write = nxt_port_fail_test_enable_write; + engine.event = stub; - nxt_port_rpc_test_insert_failures(1); + saved_engine = thr->engine; + thr->engine = &engine; - ex = nxt_port_rpc_register_handler_ex(task, port, NULL, NULL, 0); - if (ex != NULL) { + /* + * port->engine matches the caller's, so nxt_port_post() is a direct + * call. The re-arm then lands on the stub above instead of on an event + * loop this fixture does not have. + */ + port->engine = &engine; + + /* + * SOCK_DGRAM: the type the port layer itself selects, since + * src/nxt_socketpair.c keeps SEQPACKET switched off. This fixture then + * builds a pair on any platform the product supports. It needs only a + * send buffer it can fill, not seqpacket semantics. + */ + + if (nxt_slow_path(socketpair(AF_UNIX, SOCK_DGRAM, 0, pair) != 0)) { nxt_log_error(NXT_LOG_NOTICE, thr->log, - "port failure test expected rpc insert failure"); - goto fail; + "port failure test: socketpair failed"); + goto done; } - if (port->use_count != 1 || !nxt_lvlhsh_is_empty(&port->rpc_streams)) { + sndbuf = 4096; + (void) setsockopt(pair[1], SOL_SOCKET, SO_SNDBUF, &sndbuf, sizeof(sndbuf)); + + if (nxt_slow_path(fcntl(pair[1], F_SETFL, O_NONBLOCK) == -1)) { nxt_log_error(NXT_LOG_NOTICE, thr->log, - "port failure test leaked failed rpc insert"); - goto fail; + "port failure test: O_NONBLOCK failed"); + goto done; } - port->pair[0] = -1; - nxt_port_use(task, port, -1); + port->pair[0] = pair[0]; + port->pair[1] = pair[1]; + port->socket.fd = pair[1]; + port->socket.task = task; + port->socket.log = thr->log; + port->max_size = 1024; + port->max_share = 1024; - return NXT_OK; + /* Nobody reads pair[0], so the send buffer fills and stays full. */ -fail: + nxt_memzero(block, sizeof(block)); - nxt_port_rpc_test_alloc_failures(0); - nxt_port_rpc_test_insert_failures(0); + while (send(pair[1], block, sizeof(block), 0) > 0) { + /* void */ + } - port->pair[0] = -1; - nxt_port_use(task, port, -1); + port->socket.write_ready = 1; + port->socket.write = NXT_EVENT_INACTIVE; - return NXT_ERROR; -} + fd = open("/dev/null", O_RDONLY); + if (nxt_slow_path(fd == -1)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: failed to open /dev/null"); + goto done; + } + buf = nxt_buf_mem_alloc(mp, 1, 0); + if (nxt_slow_path(buf == NULL)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: failed to allocate buf"); + goto done; + } -/* - * Verify the "queued, then write failed" cleanup path inside - * nxt_port_error_handler() — the reference behaviour that the - * cert/script/socket/access-log reply paths now mirror after the - * audit fix (close fd first, queue buffer completion second). A - * synthesised port + send_msg + buf are pushed into port->messages, - * then nxt_port_test_run_error_handler() is invoked and the - * resulting fast_work_queue is drained manually so the completion - * handler runs. Asserts: - * - * - the queued message is removed from port->messages, - * - the close_fd-marked fd is actually closed, - * - the buffer completion runs exactly once. - */ -static nxt_int_t -nxt_port_fail_test_error_handler(nxt_thread_t *thr) -{ - nxt_mp_t *mp; - nxt_fd_t fd; - nxt_buf_t *buf; - nxt_task_t *task; - nxt_port_t *port; - nxt_event_engine_t engine; - nxt_port_send_msg_t *msg; + buf->completion_handler = nxt_port_fail_test_completion; - task = thr->task; - task->thread = thr; + nxt_port_fail_test_completions = 0; + nxt_port_fail_test_rearms = 0; - /* - * The minimal engine the test injects so nxt_port_error_handler - * can deref task->thread->engine->fast_work_queue. Only the - * fast_work_queue + its cache need to be initialised. - * - * Invariant: the error path exercised here must touch nothing on - * the engine beyond fast_work_queue (and the port-owned - * write_mutex). Every other field is left zeroed, so any future - * code that dereferences another engine member would read garbage - * here -- extend this initialisation (or use a real engine) before - * relying on it. - */ - nxt_memzero(&engine, sizeof(engine)); - nxt_work_queue_cache_create(&engine.work_queue_cache, 1024); - engine.fast_work_queue.cache = &engine.work_queue_cache; - nxt_work_queue_name(&engine.fast_work_queue, "fast"); + nxt_port_test_msg_alloc_failures(1); - thr->engine = &engine; + ret = nxt_port_socket_write(task, port, NXT_PORT_MSG_DATA + | NXT_PORT_MSG_CLOSE_FD, fd, 1, 0, buf); - /* - * Test mp owns the buf so that the buffer outlives the test's - * stack frame even though the completion handler runs - * asynchronously via the work queue (see Gemini PR #57 review). - * Destroyed on every exit path below. - */ - mp = nxt_mp_create(1024, 128, 256, 32); - if (nxt_slow_path(mp == NULL)) { - goto fail_engine; + nxt_port_test_msg_alloc_failures(0); + + if (ret != NXT_ERROR) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: an inline write that was neither " + "sent nor queued answered %d, expected NXT_ERROR", + (int) ret); + ret = NXT_ERROR; + goto done; } - port = nxt_port_fail_test_port(task); - if (nxt_slow_path(port == NULL)) { - goto fail_mp; + ret = NXT_ERROR; + + if (!nxt_test_fd_is_open(fd)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: the inline drop closed a descriptor " + "it reported as not taken"); + fd = -1; + goto done; } /* - * Bump use_count so the per-msg use_delta-- inside - * nxt_port_error_handler() does not drive the port to zero - * before we have inspected its state. Released explicitly - * below. + * Drain first: both consuming paths queue the completion rather than run + * it, so a counter read before this would be 0 whether or not the port + * took the buffer -- an assertion that holds for the wrong reason. */ - port->use_count = 2; - fd = -1; - msg = NULL; + nxt_port_fail_test_drain_wq(&engine.fast_work_queue); - fd = open("/dev/null", O_RDONLY); - if (fd == -1) { + if (nxt_port_fail_test_completions != 0) { nxt_log_error(NXT_LOG_NOTICE, thr->log, - "port failure test failed to open /dev/null"); - goto fail_port; + "port failure test: the inline drop completed a buffer " + "the caller still owns"); + goto done; } - buf = nxt_buf_mem_alloc(mp, 1, 0); - if (nxt_slow_path(buf == NULL)) { + if (!nxt_queue_is_empty(&port->messages)) { nxt_log_error(NXT_LOG_NOTICE, thr->log, - "port failure test failed to allocate buf"); - goto fail_port; + "port failure test: the refused message was left on the " + "port"); + goto done; } - buf->completion_handler = nxt_port_fail_test_completion; + if (port->socket.write_ready != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: write_ready survived an EAGAIN"); + goto done; + } + + if (nxt_port_fail_test_rearms != 1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: the write event was re-armed %d " + "times, expected once -- without it nothing queued on " + "this port from here on would ever drain", + (int) nxt_port_fail_test_rearms); + goto done; + } /* - * nxt_port_release_send_msg() free()s msg only when ->allocated - * is set, so we mirror what nxt_port_msg_alloc() would do for a - * heap-allocated message: nxt_malloc + ->allocated = 1. + * The same failure, but on a port with a shared queue: the payload goes + * into the ring, and the inline write is only the wake-up that tells the + * reader to look. The buffer is already handed over, its completion + * already queued, so this one must answer NXT_OK however the wake-up + * ends. nxt_runtime_port_send_quit() is a caller that cleans up after a + * failed write; it would otherwise complete the same buffer twice. */ - msg = nxt_malloc(sizeof(nxt_port_send_msg_t)); - if (msg == NULL) { - goto fail_port; + + queue = nxt_mp_zalloc(mp, sizeof(nxt_port_queue_t)); + if (nxt_slow_path(queue == NULL)) { + ret = NXT_ERROR; + goto done; } - nxt_memzero(msg, sizeof(*msg)); - msg->allocated = 1; - msg->close_fd = 1; - msg->fd[0] = fd; - msg->fd[1] = -1; - msg->buf = buf; + nxt_port_queue_init(queue); + port->queue = queue; - nxt_queue_insert_tail(&port->messages, &msg->link); + buf = nxt_buf_mem_alloc(mp, 1, 0); + if (nxt_slow_path(buf == NULL)) { + ret = NXT_ERROR; + goto done; + } - nxt_port_fail_test_completions = 0; + buf->completion_handler = nxt_port_fail_test_completion; + buf->mem.free++; - nxt_port_test_run_error_handler(task, port); + nxt_port_fail_test_completions = 0; - /* - * msg was freed inside nxt_port_release_send_msg(); take the - * ownership pointer off so the failure label does not free it - * again. - */ - msg = NULL; + /* As the poller would leave it; the socket buffer is still full. */ + port->socket.write_ready = 1; - if (!nxt_queue_is_empty(&port->messages)) { - nxt_log_error(NXT_LOG_NOTICE, thr->log, - "port failure test left a queued message after " - "error_handler"); - goto fail_port; - } + nxt_port_test_msg_alloc_failures(1); - if (nxt_port_fail_test_fd_is_open(fd)) { - nxt_log_error(NXT_LOG_NOTICE, thr->log, - "port failure test did not close the queued fd"); - goto fail_port; - } + ret = nxt_port_socket_write(task, port, NXT_PORT_MSG_DATA, -1, 0, 0, buf); - /* fd is closed; clear so the failure label does not re-close. */ - fd = -1; + nxt_port_test_msg_alloc_failures(0); - /* Buffer completion was enqueued, not invoked synchronously. */ - if (nxt_port_fail_test_completions != 0) { + if (ret != NXT_OK) { nxt_log_error(NXT_LOG_NOTICE, thr->log, - "port failure test invoked completion synchronously"); - goto fail_port; + "port failure test: a payload that reached the shared " + "queue answered %d; the buffer is already the port's " + "and the caller must not be told to reclaim it", + (int) ret); + ret = NXT_ERROR; + goto done; } + ret = NXT_ERROR; + nxt_port_fail_test_drain_wq(&engine.fast_work_queue); if (nxt_port_fail_test_completions != 1) { nxt_log_error(NXT_LOG_NOTICE, thr->log, - "port failure test completion count: %d (want 1)", + "port failure test: the enqueued payload was completed " + "%d times, expected once", (int) nxt_port_fail_test_completions); - goto fail_port; + goto done; } - nxt_port_use(task, port, -1); - nxt_mp_destroy(mp); + /* + * The marker that should have announced it could not be written, so the + * port owes one. Nothing has been lost: once the socket takes writes + * again, the re-arm sends the marker from the stack -- no allocation, + * which is the point, since an allocation is what failed. + */ - nxt_work_queue_cache_destroy(&engine.work_queue_cache); - thr->engine = NULL; + if (port->announce != 1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: the lost queue marker was not " + "recorded (announce %d)", (int) port->announce); + goto done; + } - return NXT_OK; + while (recv(pair[0], block, sizeof(block), MSG_DONTWAIT) > 0) { + /* Make room, as a peer that reads its backlog would. */ + } -fail_port: + nxt_port_rearm(task, port); - if (msg != NULL) { - nxt_queue_remove(&msg->link); - nxt_free(msg); + if (port->announce != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: the queue marker was still owed " + "after the port became writable"); + goto done; } - if (fd != -1 && nxt_port_fail_test_fd_is_open(fd)) { + + /* + * Every re-arm in this leg ran on the port's own engine, so none of them + * posted anything and none may have touched the count of posts in flight. + * The flag is unsigned: one stray decrement wraps it to its maximum and + * every later cross-engine post is refused by the CAS, silently. + */ + + if (port->rearm_pending != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: a same-engine re-arm changed the " + "posted-item flag (rearm_pending %A)", + port->rearm_pending); + goto done; + } + + n = recv(pair[0], block, sizeof(block), MSG_DONTWAIT); + + if (n != (ssize_t) sizeof(nxt_port_msg_t)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: the announcement was %d bytes, " + "expected a %d byte header", (int) n, + (int) sizeof(nxt_port_msg_t)); + goto done; + } + + if (((nxt_port_msg_t *) block)->type != _NXT_PORT_MSG_READ_QUEUE) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: the announcement was of type %d, " + "expected READ_QUEUE", + (int) ((nxt_port_msg_t *) block)->type); + goto done; + } + + ret = NXT_OK; + +done: + + nxt_port_test_msg_alloc_failures(0); + + if (fd != -1 && nxt_test_fd_is_open(fd)) { nxt_fd_close(fd); } - nxt_port_use(task, port, -1); -fail_mp: + nxt_port_fail_test_drain_wq(&engine.fast_work_queue); - nxt_mp_destroy(mp); + /* + * Close through the port's own teardown, not by closing the pair here. + * nxt_port_mp_cleanup() asserts both descriptors are gone, and that + * assertion compiles out of a release build, so closing them by hand + * looks clean until someone runs the suite with --debug. + */ -fail_engine: + /* + * Take the queue back first. It came from the test's own mem pool, and + * nxt_port_close() munmap()s whatever port->queue points at -- 655380 + * bytes of live heap if the allocator ever returns a page-aligned + * pointer. Today the call fails EINVAL and only logs, which is luck, + * not a design. + */ + + port->queue = NULL; + + nxt_port_close(task, port); + + if (pair[0] != -1 && nxt_test_fd_is_open(pair[0])) { + nxt_fd_close(pair[0]); + } + + nxt_port_use(task, port, -1); + + thr->engine = saved_engine; nxt_work_queue_cache_destroy(&engine.work_queue_cache); - thr->engine = NULL; + nxt_mp_destroy(mp); - return NXT_ERROR; + return ret; } /* - * Verify the mp-refcount invariant from phpclub's #56 review: when - * nxt_port_socket_write() fails before the buffer is handed off to - * the port machinery, the temp config mp's retain count stays at - * its baseline (1). This regression-checks the audit fix in - * src/nxt_cert.c / src/nxt_script.c, which moved `nxt_mp_retain(mp)` - * to AFTER the successful socket_write. + * The same wake-up, to a peer that is gone. * - * The helper nxt_port_fail_test_sender_pattern() is a deliberate - * copy of the fixed sender shape: allocate the buf in `mp`, send, - * retain only on success. If a future change moves the retain back - * above socket_write (the bug), an injected msg_alloc failure - * leaves retain at 2 and this test fails. + * The payload reaches the shared queue exactly as above, so it is the port's + * whatever becomes of the wake-up and the answer is still NXT_OK. What + * differs is everything the port does afterwards. An EAGAIN leaves a socket + * that will take writes again, so the marker is owed and the write event is + * re-armed. A send that fails outright leaves a peer that will never read + * the ring, and nxt_port_write_msgs() has raised the error handler on it. + * + * Owing a marker there is not merely useless. The re-arm enables the write + * event, the next pass finds announce non-zero and sends the marker again, + * and each pass is another failed send on a port the error handler is tearing + * down -- until the event fires on a port that has been freed. That is what + * the application churn tests saw: repeated EPIPE, then pthread_mutex_lock() + * failing with EINVAL on the write mutex of a destroyed port. */ + static nxt_int_t -nxt_port_fail_test_mp_baseline(nxt_thread_t *thr) +nxt_port_fail_test_dead_peer(nxt_thread_t *thr) { - nxt_mp_t *mp; - nxt_int_t res; - nxt_task_t *task; - nxt_port_t *port; - uint32_t retain_before, retain_after; + nxt_mp_t *mp; + nxt_fd_t pair[2]; + nxt_buf_t *buf; + nxt_int_t ret; + nxt_task_t *task; + nxt_port_t *port; + nxt_event_engine_t engine, *saved_engine; + nxt_port_queue_t *queue; + nxt_event_interface_t stub; + u_char qbuf[NXT_PORT_QUEUE_MSG_SIZE]; task = thr->task; task->thread = thr; + ret = NXT_ERROR; + pair[0] = -1; + pair[1] = -1; + mp = nxt_mp_create(1024, 128, 256, 32); if (nxt_slow_path(mp == NULL)) { return NXT_ERROR; } - retain_before = nxt_mp_test_retain_count(mp); - port = nxt_port_fail_test_port(task); if (nxt_slow_path(port == NULL)) { nxt_mp_destroy(mp); return NXT_ERROR; } + nxt_memzero(&engine, sizeof(engine)); + nxt_memzero(&stub, sizeof(stub)); + + nxt_work_queue_cache_create(&engine.work_queue_cache, 1024); + engine.fast_work_queue.cache = &engine.work_queue_cache; + nxt_work_queue_name(&engine.fast_work_queue, "fast"); + + stub.enable_write = nxt_port_fail_test_enable_write; + engine.event = stub; + + saved_engine = thr->engine; + thr->engine = &engine; + + port->engine = &engine; + + if (nxt_slow_path(socketpair(AF_UNIX, SOCK_DGRAM, 0, pair) != 0)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: socketpair failed"); + goto done; + } + + port->pair[0] = pair[0]; + port->pair[1] = pair[1]; + port->socket.fd = pair[1]; + port->socket.task = task; + port->socket.log = thr->log; + port->max_size = 1024; + port->max_share = 1024; + + queue = nxt_mp_zalloc(mp, sizeof(nxt_port_queue_t)); + if (nxt_slow_path(queue == NULL)) { + goto done; + } + + nxt_port_queue_init(queue); + port->queue = queue; + + buf = nxt_buf_mem_alloc(mp, 1, 0); + if (nxt_slow_path(buf == NULL)) { + goto done; + } + + buf->completion_handler = nxt_port_fail_test_completion; + buf->mem.free++; + nxt_port_fail_test_completions = 0; - nxt_port_test_msg_alloc_failures(1); + nxt_port_fail_test_rearms = 0; - res = nxt_port_fail_test_sender_pattern(task, port, mp); + /* + * The reader end goes now, so the wake-up fails outright rather than + * with EAGAIN. Both copies of the number are cleared with it: the + * teardown below closes whatever port->pair[0] still holds, and closing + * a number twice would hand this fixture somebody else's descriptor. + */ - nxt_port_test_msg_alloc_failures(0); + nxt_fd_close(pair[0]); + pair[0] = -1; + port->pair[0] = -1; - if (res != NXT_ERROR) { + port->socket.write_ready = 1; + port->socket.write = NXT_EVENT_INACTIVE; + + ret = nxt_port_socket_write(task, port, NXT_PORT_MSG_DATA, -1, 0, 0, buf); + + if (ret != NXT_OK) { nxt_log_error(NXT_LOG_NOTICE, thr->log, - "mp baseline test: sender expected failure return"); - goto fail; + "port failure test: a payload that reached the shared " + "queue answered %d after the peer was gone; the buffer " + "is already the port's and the caller must not be told " + "to reclaim it", (int) ret); + ret = NXT_ERROR; + goto done; } - retain_after = nxt_mp_test_retain_count(mp); + ret = NXT_ERROR; - if (retain_after != retain_before) { + nxt_port_fail_test_drain_wq(&engine.fast_work_queue); + + if (nxt_port_fail_test_completions != 1) { nxt_log_error(NXT_LOG_NOTICE, thr->log, - "mp baseline test: retain before %uD, after %uD " - "(retain leaked across failed send)", - retain_before, retain_after); - goto fail; + "port failure test: the enqueued payload was completed " + "%d times, expected once", + (int) nxt_port_fail_test_completions); + goto done; + } + + if (port->announce != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: a queue marker was owed to a peer " + "that is gone (announce %d); every later pass resends " + "it on a port the error handler is tearing down", + (int) port->announce); + goto done; + } + + if (nxt_port_fail_test_rearms != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: the write event was re-armed %d " + "times on a failed port, expected none", + (int) nxt_port_fail_test_rearms); + goto done; } /* - * Buf-completion handler must not have run: the buf never - * entered the port queue. + * The same port, now owing a marker from before the peer died -- the + * state an earlier EAGAIN leaves, and the one the re-arm exists to + * settle. A failed pass must not settle it: the debt is read again on + * the way out of every write, so a port that keeps re-arming on it sends + * one more doomed marker per pass. + * + * The ring is drained first only so the next payload is again the item + * that raises "notify"; nxt_port_queue_send() raises it on the 0-to-1 + * transition alone. */ - if (nxt_port_fail_test_completions != 0) { + + while (nxt_port_queue_recv(queue, qbuf) >= 0) { + /* void */ + } + + nxt_atomic_fetch_add(&port->announce, 1); + + buf = nxt_buf_mem_alloc(mp, 1, 0); + if (nxt_slow_path(buf == NULL)) { + goto done; + } + + buf->completion_handler = nxt_port_fail_test_completion; + buf->mem.free++; + + nxt_port_fail_test_completions = 0; + + port->socket.write_ready = 1; + + ret = nxt_port_socket_write(task, port, NXT_PORT_MSG_DATA, -1, 0, 0, buf); + + if (ret != NXT_OK) { nxt_log_error(NXT_LOG_NOTICE, thr->log, - "mp baseline test: completion ran for unsent buf"); - goto fail; + "port failure test: the second enqueued payload " + "answered %d", (int) ret); + ret = NXT_ERROR; + goto done; } - nxt_port_use(task, port, -1); - nxt_mp_destroy(mp); + ret = NXT_ERROR; - return NXT_OK; + nxt_port_fail_test_drain_wq(&engine.fast_work_queue); -fail: + if (nxt_port_fail_test_rearms != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: a failed port re-armed to send a " + "marker it already owed (%d times)", + (int) nxt_port_fail_test_rearms); + goto done; + } + + if (port->announce != 1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: the owed marker was counted again " + "on a port that is going away (announce %d)", + (int) port->announce); + goto done; + } + + ret = NXT_OK; + +done: + + nxt_port_fail_test_drain_wq(&engine.fast_work_queue); + + port->queue = NULL; + + nxt_port_close(task, port); nxt_port_use(task, port, -1); + + thr->engine = saved_engine; + + nxt_work_queue_cache_destroy(&engine.work_queue_cache); nxt_mp_destroy(mp); - return NXT_ERROR; + return ret; } /* - * Mirror of the post-audit nxt_cert_store_get sender shape: - * - allocate buf in the caller-supplied temp_conf mp; - * - attempt socket_write; - * - retain mp ONLY after the write succeeded. - * - * The completion handler released by the buf — when it eventually - * runs — would call nxt_mp_release(mp); we do not exercise that - * branch here since the test forces a pre-queue failure. See - * nxt_port_fail_test_mp_baseline(). + * The level a failed send to a gone peer is logged at. A QUIT goes to a + * process that can have exited already. So EPIPE on a QUIT is info, at + * once or later from the port's own queue. On a SOCK_DGRAM pair the error + * is ECONNREFUSED, and that is info too. A failed send of any other + * message is an alert. A message to a port with a shared queue goes + * through that queue. The socket then carries only a READ_QUEUE wake-up. + * The wake-up for a QUIT keeps peer_may_be_gone, so it is info too. This + * also applies when the wake-up waits in port->messages first and fails + * from the write handler. The wake-up for a DATA is an alert. The peer's + * end is closed, so sendmsg() really fails. */ + +static nxt_uint_t nxt_port_fail_test_sendmsg_level; + + +static void nxt_cdecl +nxt_port_fail_test_log_handler(nxt_uint_t level, nxt_log_t *log, + const char *fmt, ...) +{ + if (nxt_strncmp(fmt, "sendmsg(", 8) == 0) { + nxt_port_fail_test_sendmsg_level = level; + } +} + + static nxt_int_t -nxt_port_fail_test_sender_pattern(nxt_task_t *task, nxt_port_t *port, - nxt_mp_t *mp) +nxt_port_fail_test_send_to_dead_peer(nxt_thread_t *thr, nxt_uint_t type, + nxt_bool_t queued, nxt_bool_t shared, nxt_uint_t then, nxt_uint_t *level) { - nxt_buf_t *b; - nxt_int_t res; + nxt_fd_t pair[2]; + nxt_int_t ret; + nxt_log_t log, *saved_log; + nxt_task_t *task; + nxt_port_t *port; + nxt_event_engine_t engine, *saved_engine; + nxt_port_queue_t *queue; + nxt_port_send_msg_t *msg; + nxt_event_interface_t stub; - b = nxt_buf_mem_alloc(mp, 16, 0); - if (b == NULL) { + task = thr->task; + task->thread = thr; + + port = nxt_port_fail_test_port(task); + if (nxt_slow_path(port == NULL)) { return NXT_ERROR; } - b->completion_handler = nxt_port_fail_test_mp_completion; - b->parent = mp; + nxt_memzero(&engine, sizeof(engine)); + nxt_memzero(&stub, sizeof(stub)); - res = nxt_port_socket_write(task, port, NXT_PORT_MSG_DATA, -1, 1, 0, b); - if (res != NXT_OK) { - return NXT_ERROR; - } + nxt_work_queue_cache_create(&engine.work_queue_cache, 1024); + engine.fast_work_queue.cache = &engine.work_queue_cache; + nxt_work_queue_name(&engine.fast_work_queue, "fast"); - /* - * Retain only after the buffer has been handed off to the port - * machinery — matches the fix in nxt_cert_store_get / - * nxt_script_store_get. - */ - nxt_mp_retain(mp); + stub.enable_write = nxt_port_fail_test_enable_write; + stub.block_write = nxt_port_fail_test_enable_write; + engine.event = stub; - return NXT_OK; -} + saved_engine = thr->engine; + thr->engine = &engine; + /* Only the "sendmsg() failed" level is of interest. */ -static void -nxt_port_fail_test_mp_completion(nxt_task_t *task, void *obj, void *data) -{ - nxt_mp_t *mp = data; + log = *task->log; + log.level = NXT_LOG_INFO; + log.handler = nxt_port_fail_test_log_handler; - nxt_port_fail_test_completions++; + saved_log = task->log; + task->log = &log; - if (mp != NULL) { - nxt_mp_release(mp); - } -} + nxt_port_fail_test_sendmsg_level = NXT_LOG_DEBUG; + ret = NXT_ERROR; + queue = NULL; -static void -nxt_port_fail_test_completion(nxt_task_t *task, void *obj, void *data) -{ - nxt_port_fail_test_completions++; -} + /* The production pair: SOCK_SEQPACKET where there is one, else DGRAM. */ + if (nxt_slow_path(nxt_socketpair_create(task, pair) != NXT_OK)) { + nxt_log_error(NXT_LOG_NOTICE, saved_log, + "port failure test: socketpair failed"); + goto done; + } -static void -nxt_port_fail_test_drain_wq(nxt_work_queue_t *wq) -{ - void *obj, *data; - nxt_task_t *t; - nxt_work_handler_t handler; + if (shared) { + queue = nxt_mp_zalloc(port->mem_pool, sizeof(nxt_port_queue_t)); + if (nxt_slow_path(queue == NULL)) { + goto done; + } - while (wq->head != NULL) { - handler = nxt_work_queue_pop(wq, &t, &obj, &data); - handler(t, obj, data); + nxt_port_queue_init(queue); + port->queue = queue; } -} + port->pair[0] = pair[0]; + port->pair[1] = pair[1]; + port->socket.task = task; + port->max_size = 1024; + port->max_share = 1024; -static nxt_bool_t -nxt_port_fail_test_fd_is_open(nxt_fd_t fd) -{ - return fcntl(fd, F_GETFD) != -1; -} + nxt_port_write_enable(task, port); + port->socket.log = &log; + port->socket.write = NXT_EVENT_INACTIVE; + port->socket.write_ready = !queued; -static nxt_int_t -nxt_port_fail_test_fd_count(void) -{ -#if (NXT_LINUX) - nxt_int_t count; - DIR *dir; - struct dirent *de; + /* The peer is gone before the send. */ - dir = opendir("/proc/self/fd"); - if (dir == NULL) { - return -1; - } + nxt_fd_close(pair[0]); + port->pair[0] = -1; - count = 0; + ret = nxt_port_socket_write(task, port, type, -1, 0, 0, NULL); - for ( ;; ) { - de = readdir(dir); + if (queue != NULL && queue->nitems != 1) { + nxt_log_error(NXT_LOG_NOTICE, saved_log, + "port failure test: the message did not go into the " + "shared queue"); + ret = NXT_ERROR; + goto done; + } - if (de == NULL) { - break; + /* + * Send a second message behind the first one. The shared queue is not + * empty any more. So the second message goes into the queue with no + * wake-up of its own (notify == 0), and the pending wake-up covers it. + */ + if (then != 0) { + ret = nxt_port_socket_write(task, port, then, -1, 0, 0, NULL); + + if (ret != NXT_OK || queue == NULL || queue->nitems != 2) { + nxt_log_error(NXT_LOG_NOTICE, saved_log, + "port failure test: the second message did not " + "join the shared queue (%d)", (int) ret); + ret = NXT_ERROR; + goto done; } + } - if (nxt_strcmp(de->d_name, ".") != 0 - && nxt_strcmp(de->d_name, "..") != 0) - { - count++; + if (queued) { + if (ret != NXT_OK || nxt_queue_is_empty(&port->messages)) { + nxt_log_error(NXT_LOG_NOTICE, saved_log, + "port failure test: the message was not queued " + "(%d)", (int) ret); + ret = NXT_ERROR; + goto done; } + + msg = nxt_queue_link_data(nxt_queue_first(&port->messages), + nxt_port_send_msg_t, link); + + /* What waits is the socket wake-up, not the message itself. */ + + if (shared && msg->port_msg.type != _NXT_PORT_MSG_READ_QUEUE) { + nxt_log_error(NXT_LOG_NOTICE, saved_log, + "port failure test: the queued message is not a " + "READ_QUEUE wake-up (%d)", + (int) msg->port_msg.type); + ret = NXT_ERROR; + goto done; + } + + port->socket.write_ready = 1; + + port->socket.write_handler(task, &port->socket, NULL); } - (void) closedir(dir); + *level = nxt_port_fail_test_sendmsg_level; + + ret = NXT_OK; + +done: + + nxt_port_fail_test_drain_wq(&engine.fast_work_queue); + + /* Not a mapping: nxt_port_close() must not munmap() it. */ + port->queue = NULL; + + nxt_port_close(task, port); + nxt_port_use(task, port, -1); + + task->log = saved_log; + thr->engine = saved_engine; + + nxt_work_queue_cache_destroy(&engine.work_queue_cache); + + return ret; +} + + +static nxt_int_t +nxt_port_fail_test_quit_log_level(nxt_thread_t *thr) +{ + nxt_uint_t i, level; + + static const struct { + nxt_uint_t type; + nxt_bool_t queued; + nxt_bool_t shared; + nxt_uint_t then; + nxt_uint_t level; + const char *name; + } legs[] = { + { NXT_PORT_MSG_QUIT, 0, 0, 0, NXT_LOG_INFO, "a QUIT sent at once" }, + { NXT_PORT_MSG_QUIT, 1, 0, 0, NXT_LOG_INFO, + "a QUIT sent from the queue" }, + { NXT_PORT_MSG_QUIT, 0, 1, 0, NXT_LOG_INFO, + "a QUIT to a port with a shared queue" }, + { NXT_PORT_MSG_QUIT, 1, 1, 0, NXT_LOG_INFO, + "a deferred QUIT wake-up to a port with a shared queue" }, + { NXT_PORT_MSG_DATA, 0, 0, 0, NXT_LOG_ALERT, "a DATA sent at once" }, + { NXT_PORT_MSG_DATA, 0, 1, 0, NXT_LOG_ALERT, + "a DATA to a port with a shared queue" }, + { NXT_PORT_MSG_DATA, 1, 1, 0, NXT_LOG_ALERT, + "a deferred DATA wake-up to a port with a shared queue" }, + { NXT_PORT_MSG_DATA, 1, 1, NXT_PORT_MSG_QUIT, NXT_LOG_INFO, + "a deferred wake-up that also covers a later QUIT" }, + }; + + for (i = 0; i < nxt_nitems(legs); i++) { + if (nxt_port_fail_test_send_to_dead_peer(thr, legs[i].type, + legs[i].queued, + legs[i].shared, + legs[i].then, &level) + != NXT_OK) + { + return NXT_ERROR; + } + + if (level != legs[i].level) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: %s to a peer that is gone was " + "logged at level %ui, expected %ui", legs[i].name, + level, legs[i].level); + return NXT_ERROR; + } + } + + return NXT_OK; +} + + +/* + * The counter wraps: the stream after UINT32_MAX must not be 0. The port + * is open, and the registrations are cancelled before the return. + */ + +static nxt_int_t +nxt_port_fail_test_rpc_stream_wrap(nxt_task_t *task, nxt_port_t *port) +{ + void *ex; + uint32_t stream[2], saved_stream; + + stream[0] = 0; + stream[1] = 0; + + saved_stream = nxt_port_rpc_test_set_stream_ident(UINT32_MAX); + + ex = nxt_port_rpc_register_handler_ex(task, port, NULL, NULL, 0); + if (ex != NULL) { + stream[0] = nxt_port_rpc_ex_stream(ex); + + ex = nxt_port_rpc_register_handler_ex(task, port, NULL, NULL, 0); + if (ex != NULL) { + stream[1] = nxt_port_rpc_ex_stream(ex); + nxt_port_rpc_cancel(task, port, stream[1]); + } + + nxt_port_rpc_cancel(task, port, stream[0]); + } + + nxt_port_rpc_test_set_stream_ident(saved_stream); + + if (stream[0] != UINT32_MAX || stream[1] == 0 + || port->use_count != 1 || !nxt_lvlhsh_is_empty(&port->rpc_streams)) + { + nxt_log_error(NXT_LOG_NOTICE, task->log, + "port failure test rpc stream wrap: %uD then %uD, " + "use_count %D", stream[0], stream[1], + (int32_t) port->use_count); + return NXT_ERROR; + } + + return NXT_OK; +} + + +static nxt_int_t +nxt_port_fail_test_rpc_register(nxt_thread_t *thr) +{ + void *ex; + nxt_task_t *task; + nxt_port_t *port; + + task = thr->task; + task->thread = thr; + + if (nxt_port_rpc_init() != NXT_OK) { + return NXT_ERROR; + } + + port = nxt_port_fail_test_port(task); + if (nxt_slow_path(port == NULL)) { + return NXT_ERROR; + } + + port->pair[0] = 0; + + if (nxt_port_fail_test_rpc_stream_wrap(task, port) != NXT_OK) { + goto fail; + } + + nxt_port_rpc_test_alloc_failures(1); + + ex = nxt_port_rpc_register_handler_ex(task, port, NULL, NULL, 0); + if (ex != NULL) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test expected rpc alloc failure"); + goto fail; + } + + if (port->use_count != 1 || !nxt_lvlhsh_is_empty(&port->rpc_streams)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test leaked failed rpc alloc"); + goto fail; + } + + nxt_port_rpc_test_insert_failures(1); + + ex = nxt_port_rpc_register_handler_ex(task, port, NULL, NULL, 0); + if (ex != NULL) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test expected rpc insert failure"); + goto fail; + } + + if (port->use_count != 1 || !nxt_lvlhsh_is_empty(&port->rpc_streams)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test leaked failed rpc insert"); + goto fail; + } + + port->pair[0] = -1; + nxt_port_use(task, port, -1); + + return NXT_OK; + +fail: + + nxt_port_rpc_test_alloc_failures(0); + nxt_port_rpc_test_insert_failures(0); + + port->pair[0] = -1; + nxt_port_use(task, port, -1); + + return NXT_ERROR; +} + + +/* + * Verify the "queued, then write failed" cleanup path inside + * nxt_port_error_handler() — the reference behaviour that the + * cert/script/socket/access-log reply paths now mirror after the + * audit fix (close fd first, queue buffer completion second). A + * synthesised port + send_msg + buf are pushed into port->messages, + * then nxt_port_test_run_error_handler() is invoked and the + * resulting fast_work_queue is drained manually so the completion + * handler runs. Asserts: + * + * - the queued message is removed from port->messages, + * - the close_fd-marked fd is actually closed, + * - the buffer completion runs exactly once. + */ +static nxt_int_t +nxt_port_fail_test_error_handler(nxt_thread_t *thr) +{ + nxt_mp_t *mp; + nxt_fd_t fd; + nxt_buf_t *buf; + nxt_task_t *task; + nxt_port_t *port; + nxt_event_engine_t engine; + nxt_port_send_msg_t *msg; + + task = thr->task; + task->thread = thr; + + /* + * The minimal engine the test injects so nxt_port_error_handler + * can deref task->thread->engine->fast_work_queue. Only the + * fast_work_queue + its cache need to be initialised. + * + * Invariant: the error path exercised here must touch nothing on + * the engine beyond fast_work_queue (and the port-owned + * write_mutex). Every other field is left zeroed, so any future + * code that dereferences another engine member would read garbage + * here -- extend this initialisation (or use a real engine) before + * relying on it. + */ + nxt_memzero(&engine, sizeof(engine)); + nxt_work_queue_cache_create(&engine.work_queue_cache, 1024); + engine.fast_work_queue.cache = &engine.work_queue_cache; + nxt_work_queue_name(&engine.fast_work_queue, "fast"); + + thr->engine = &engine; + + /* + * Test mp owns the buf so that the buffer outlives the test's + * stack frame even though the completion handler runs + * asynchronously via the work queue (see Gemini PR #57 review). + * Destroyed on every exit path below. + */ + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + goto fail_engine; + } + + port = nxt_port_fail_test_port(task); + if (nxt_slow_path(port == NULL)) { + goto fail_mp; + } + + /* + * Bump use_count so the per-msg use_delta-- inside + * nxt_port_error_handler() does not drive the port to zero + * before we have inspected its state. Released explicitly + * below. + */ + port->use_count = 2; + + fd = -1; + msg = NULL; + + fd = open("/dev/null", O_RDONLY); + if (fd == -1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test failed to open /dev/null"); + goto fail_port; + } + + buf = nxt_buf_mem_alloc(mp, 1, 0); + if (nxt_slow_path(buf == NULL)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test failed to allocate buf"); + goto fail_port; + } + + buf->completion_handler = nxt_port_fail_test_completion; + + /* + * nxt_port_release_send_msg() free()s msg only when ->allocated + * is set, so we mirror what nxt_port_msg_alloc() would do for a + * heap-allocated message: nxt_malloc + ->allocated = 1. + */ + msg = nxt_malloc(sizeof(nxt_port_send_msg_t)); + if (msg == NULL) { + goto fail_port; + } + + nxt_memzero(msg, sizeof(*msg)); + msg->allocated = 1; + msg->close_fd = 1; + msg->fd[0] = fd; + msg->fd[1] = -1; + msg->buf = buf; + + nxt_queue_insert_tail(&port->messages, &msg->link); + + nxt_port_fail_test_completions = 0; + + nxt_port_test_run_error_handler(task, port); + + /* + * msg was freed inside nxt_port_release_send_msg(); take the + * ownership pointer off so the failure label does not free it + * again. + */ + msg = NULL; + + if (!nxt_queue_is_empty(&port->messages)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test left a queued message after " + "error_handler"); + goto fail_port; + } + + if (nxt_test_fd_is_open(fd)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test did not close the queued fd"); + goto fail_port; + } + + /* fd is closed; clear so the failure label does not re-close. */ + fd = -1; + + /* Buffer completion was enqueued, not invoked synchronously. */ + if (nxt_port_fail_test_completions != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test invoked completion synchronously"); + goto fail_port; + } + + nxt_port_fail_test_drain_wq(&engine.fast_work_queue); + + if (nxt_port_fail_test_completions != 1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test completion count: %d (want 1)", + (int) nxt_port_fail_test_completions); + goto fail_port; + } + + nxt_port_use(task, port, -1); + nxt_mp_destroy(mp); + + nxt_work_queue_cache_destroy(&engine.work_queue_cache); + thr->engine = NULL; + + return NXT_OK; + +fail_port: + + if (msg != NULL) { + nxt_queue_remove(&msg->link); + nxt_free(msg); + } + if (fd != -1 && nxt_test_fd_is_open(fd)) { + nxt_fd_close(fd); + } + nxt_port_use(task, port, -1); + +fail_mp: + + nxt_mp_destroy(mp); + +fail_engine: + + nxt_work_queue_cache_destroy(&engine.work_queue_cache); + thr->engine = NULL; + + return NXT_ERROR; +} + + +/* + * Verify the mp-refcount invariant from phpclub's #56 review: when + * nxt_port_socket_write() fails before the buffer is handed off to + * the port machinery, the temp config mp's retain count stays at + * its baseline (1). This regression-checks the audit fix in + * src/nxt_cert.c / src/nxt_script.c, which moved `nxt_mp_retain(mp)` + * to AFTER the successful socket_write. + * + * The helper nxt_port_fail_test_sender_pattern() is a deliberate + * copy of the fixed sender shape: allocate the buf in `mp`, send, + * retain only on success. If a future change moves the retain back + * above socket_write (the bug), an injected msg_alloc failure + * leaves retain at 2 and this test fails. + */ +static nxt_int_t +nxt_port_fail_test_mp_baseline(nxt_thread_t *thr) +{ + nxt_mp_t *mp; + nxt_int_t res; + nxt_task_t *task; + nxt_port_t *port; + uint32_t retain_before, retain_after; + + task = thr->task; + task->thread = thr; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + retain_before = nxt_mp_test_retain_count(mp); + + port = nxt_port_fail_test_port(task); + if (nxt_slow_path(port == NULL)) { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + nxt_port_fail_test_completions = 0; + nxt_port_test_msg_alloc_failures(1); + + res = nxt_port_fail_test_sender_pattern(task, port, mp); + + nxt_port_test_msg_alloc_failures(0); + + if (res != NXT_ERROR) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "mp baseline test: sender expected failure return"); + goto fail; + } + + retain_after = nxt_mp_test_retain_count(mp); + + if (retain_after != retain_before) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "mp baseline test: retain before %uD, after %uD " + "(retain leaked across failed send)", + retain_before, retain_after); + goto fail; + } + + /* + * Buf-completion handler must not have run: the buf never + * entered the port queue. + */ + if (nxt_port_fail_test_completions != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "mp baseline test: completion ran for unsent buf"); + goto fail; + } + + nxt_port_use(task, port, -1); + nxt_mp_destroy(mp); + + return NXT_OK; + +fail: + + nxt_port_use(task, port, -1); + nxt_mp_destroy(mp); + + return NXT_ERROR; +} + + +/* + * Mirror of the post-audit nxt_cert_store_get sender shape: + * - allocate buf in the caller-supplied temp_conf mp; + * - attempt socket_write; + * - retain mp ONLY after the write succeeded. + * + * The completion handler released by the buf — when it eventually + * runs — would call nxt_mp_release(mp); we do not exercise that + * branch here since the test forces a pre-queue failure. See + * nxt_port_fail_test_mp_baseline(). + */ +static nxt_int_t +nxt_port_fail_test_sender_pattern(nxt_task_t *task, nxt_port_t *port, + nxt_mp_t *mp) +{ + nxt_buf_t *b; + nxt_int_t res; + + b = nxt_buf_mem_alloc(mp, 16, 0); + if (b == NULL) { + return NXT_ERROR; + } + + b->completion_handler = nxt_port_fail_test_mp_completion; + b->parent = mp; + + res = nxt_port_socket_write(task, port, NXT_PORT_MSG_DATA, -1, 1, 0, b); + if (res != NXT_OK) { + return NXT_ERROR; + } + + /* + * Retain only after the buffer has been handed off to the port + * machinery — matches the fix in nxt_cert_store_get / + * nxt_script_store_get. + */ + nxt_mp_retain(mp); + + return NXT_OK; +} + + +/* + * A last reference dropped from a thread other than port->engine's must be + * deferred to that engine, and the deferral must not be able to fail. + * + * nxt_port_use() used to route the drop through nxt_port_post(), which + * nxt_zalloc()s an nxt_port_work_t and returns NXT_ERROR when that fails. + * The return value was ignored: use_count was already zero, nothing was + * queued, and the port, its memory pool and the process reference it holds + * leaked (issue #187). + * + * There is no allocation-failure injection hook reachable from here, so the + * regression is pinned structurally instead: the item that lands on the + * foreign engine's locked work queue must be the one embedded in the port. + * That is false for any implementation that allocates it, whether or not the + * allocation succeeds. Draining then has to reach nxt_port_release(), which + * is observed through a cleanup on the port's own memory pool. + */ + +static nxt_int_t +nxt_port_fail_test_cross_engine_release(nxt_thread_t *thr) +{ + nxt_task_t *task; + nxt_port_t *port; + nxt_work_t *posted; + nxt_event_engine_t current, foreign; + + task = thr->task; + task->thread = thr; + + /* + * Two minimal engines: the one this thread runs on, and the port's. + * They only have to differ and to carry a work queue; the post path + * touches locked_work_queue, event.signal and task. + */ + nxt_memzero(¤t, sizeof(current)); + nxt_work_queue_cache_create(¤t.work_queue_cache, 1024); + current.fast_work_queue.cache = ¤t.work_queue_cache; + nxt_work_queue_name(¤t.fast_work_queue, "fast"); + + nxt_memzero(&foreign, sizeof(foreign)); + foreign.task.thread = thr; + foreign.task.log = thr->log; + + /* + * Without this stub nxt_event_engine_signal() would fall through to + * writing on foreign.pipe, which a bare engine does not have. + */ + foreign.event.signal = nxt_port_fail_test_engine_signal; + + thr->engine = ¤t; + + port = nxt_port_fail_test_port(task); + if (nxt_slow_path(port == NULL)) { + goto fail_engine; + } + + port->engine = &foreign; + + if (nxt_slow_path(nxt_mp_cleanup(port->mem_pool, + nxt_port_fail_test_released, + task, port, NULL) != NXT_OK)) + { + nxt_port_use(task, port, -1); + goto fail_engine; + } + + nxt_port_fail_test_releases = 0; + nxt_port_fail_test_signals = 0; + + posted = &port->release_work; + + nxt_port_use(task, port, -1); + + /* The release must be deferred, not run on this thread. */ + if (nxt_slow_path(nxt_port_fail_test_releases != 0)) { + nxt_log_alert(thr->log, "port fail test: cross-engine release ran " + "on the calling thread"); + goto fail_engine; + } + + if (nxt_slow_path(nxt_port_fail_test_signals != 1)) { + nxt_log_alert(thr->log, "port fail test: cross-engine release did " + "not signal the target engine (%ui)", + nxt_port_fail_test_signals); + goto fail_engine; + } + + /* + * The heart of the regression check: an allocated work item would make + * this a different pointer, and a failed allocation would leave the + * queue empty. + */ + if (nxt_slow_path(foreign.locked_work_queue.head != posted)) { + nxt_log_alert(thr->log, "port fail test: the posted item is not the " + "port's embedded release work"); + goto fail_engine; + } + + /* + * The reference is handed to the item, not given up: the count is back + * at 1 and the posted handler is what drops it. + */ + if (nxt_slow_path(port->use_count != 1)) { + nxt_log_alert(thr->log, "port fail test: the deferred release does " + "not hold a reference (use_count %A)", port->use_count); + goto fail_engine; + } + + /* The handler runs on the port's engine, so this thread becomes it. */ + thr->engine = &foreign; + + nxt_locked_work_queue_move(thr, &foreign.locked_work_queue, + ¤t.fast_work_queue); + + nxt_port_fail_test_drain_wq(¤t.fast_work_queue); + + if (nxt_slow_path(nxt_port_fail_test_releases != 1)) { + nxt_log_alert(thr->log, "port fail test: draining the target engine " + "did not release the port (%ui)", + nxt_port_fail_test_releases); + goto fail_engine; + } + + nxt_work_queue_cache_destroy(¤t.work_queue_cache); + thr->engine = NULL; + + return NXT_OK; + +fail_engine: + + nxt_work_queue_cache_destroy(¤t.work_queue_cache); + thr->engine = NULL; + + return NXT_ERROR; +} + + +/* + * A cross-engine last drop leaves the port reachable through + * process->ports until the deferred item runs, so another engine can still + * take a reference in that window -- nxt_process_broadcast_shm_ack() does. + * The deferral therefore has to hand its reference over and re-check at the + * far end, which is what routing through nxt_port_post() used to provide: + * it took a reference of its own and ended in nxt_port_use(port, -1). + * + * A deferred handler that released unconditionally would free a port + * somebody holds -- a use-after-free, and only an assertion in a --debug + * build. Here the acquisition is made between the post and the drain: the + * drain must not release, and the release must happen exactly once, when + * the other holder drops its reference. + */ + +static nxt_int_t +nxt_port_fail_test_cross_engine_acquire(nxt_thread_t *thr) +{ + nxt_task_t *task; + nxt_port_t *port; + nxt_event_engine_t current, foreign; + + task = thr->task; + task->thread = thr; + + nxt_memzero(¤t, sizeof(current)); + nxt_work_queue_cache_create(¤t.work_queue_cache, 1024); + current.fast_work_queue.cache = ¤t.work_queue_cache; + nxt_work_queue_name(¤t.fast_work_queue, "fast"); + + nxt_memzero(&foreign, sizeof(foreign)); + foreign.task.thread = thr; + foreign.task.log = thr->log; + foreign.event.signal = nxt_port_fail_test_engine_signal; + + thr->engine = ¤t; + + port = nxt_port_fail_test_port(task); + if (nxt_slow_path(port == NULL)) { + goto fail_engine; + } + + port->engine = &foreign; + + if (nxt_slow_path(nxt_mp_cleanup(port->mem_pool, + nxt_port_fail_test_released, + task, port, NULL) != NXT_OK)) + { + nxt_port_use(task, port, -1); + goto fail_engine; + } + + nxt_port_fail_test_releases = 0; + + nxt_port_use(task, port, -1); + + /* + * The concurrent acquisition, made while the item is queued. A real one + * comes from another engine; what matters here is that it happens after + * the drop that posted the item and before that item runs. + */ + nxt_port_use(task, port, 1); + + thr->engine = &foreign; + + nxt_locked_work_queue_move(thr, &foreign.locked_work_queue, + ¤t.fast_work_queue); + + nxt_port_fail_test_drain_wq(¤t.fast_work_queue); + + if (nxt_slow_path(nxt_port_fail_test_releases != 0)) { + nxt_log_alert(thr->log, "port fail test: the deferred release freed " + "a port another reference was taken on"); + goto fail_port; + } + + if (nxt_slow_path(port->use_count != 1)) { + nxt_log_alert(thr->log, "port fail test: use_count is %A after the " + "deferred release, expected 1", port->use_count); + goto fail_port; + } + + /* Now the other holder lets go, on the port's own engine. */ + nxt_port_use(task, port, -1); + + if (nxt_slow_path(nxt_port_fail_test_releases != 1)) { + nxt_log_alert(thr->log, "port fail test: dropping the last reference " + "released the port %ui times, expected 1", + nxt_port_fail_test_releases); + goto fail_engine; + } + + nxt_work_queue_cache_destroy(¤t.work_queue_cache); + thr->engine = NULL; + + return NXT_OK; + +fail_port: + + nxt_port_use(task, port, -1); + +fail_engine: + + nxt_work_queue_cache_destroy(¤t.work_queue_cache); + thr->engine = NULL; + + return NXT_ERROR; +} + + +/* + * The hand-over must not be able to post the embedded item twice. + * + * The item is a single object, so two threads that both believe they are + * dropping the last reference would post it twice, and + * nxt_locked_work_queue_add() links a work onto the queue tail: the second + * post makes tail->next point at the item itself, and the engine draining + * that queue then walks a one-element cycle forever. Routing through + * nxt_port_post() could not produce this, because it allocated a separate + * item for every post. + * + * The shape that would produce it is a foreign drop that publishes a zero + * use_count: another engine promoting a bare port pointer -- + * nxt_process_broadcast_shm_ack() walks a process's ports that way, and + * nxt_port_socket_write() takes a reference on each -- can then complete a + * 0 -> 1 -> 0 cycle of its own and reach the deferral as well. So the + * invariant this pins is the one that makes it impossible: use_count is + * never observable as zero off port->engine. + * + * A racer thread on a third engine watches use_count across a cross-engine + * last drop and reports every zero it sees; then, with the item already in + * flight, it takes a reference and drops it again -- the 1 -> 2 -> 1 that + * must not post anything. The locked queue is walked with a step limit, so + * a self-linked item fails the test instead of hanging it. + */ + +#define NXT_PORT_FAIL_TEST_RACE_TRIALS 128 +#define NXT_PORT_FAIL_TEST_RACE_SPINS 4000000 +#define NXT_PORT_FAIL_TEST_QUEUE_LIMIT 16 + + +typedef struct { + nxt_port_t *port; + nxt_event_engine_t *engine; + nxt_atomic_t ready; + nxt_atomic_t stop; + nxt_uint_t zeroes; +} nxt_port_fail_test_race_t; + + +static nxt_int_t +nxt_port_fail_test_cross_engine_race(nxt_thread_t *thr) +{ + nxt_int_t queued; + nxt_task_t *task; + nxt_uint_t trial; + nxt_port_t *port; + nxt_thread_link_t *link; + nxt_thread_handle_t handle; + nxt_event_engine_t current, foreign, other; + nxt_port_fail_test_race_t race; + + task = thr->task; + task->thread = thr; + + nxt_memzero(&other, sizeof(other)); + + for (trial = 0; trial < NXT_PORT_FAIL_TEST_RACE_TRIALS; trial++) { + + nxt_memzero(¤t, sizeof(current)); + nxt_work_queue_cache_create(¤t.work_queue_cache, 1024); + current.fast_work_queue.cache = ¤t.work_queue_cache; + nxt_work_queue_name(¤t.fast_work_queue, "fast"); + + nxt_memzero(&foreign, sizeof(foreign)); + foreign.task.thread = thr; + foreign.task.log = thr->log; + foreign.event.signal = nxt_port_fail_test_engine_signal; + + thr->engine = ¤t; + + port = nxt_port_fail_test_port(task); + if (nxt_slow_path(port == NULL)) { + goto fail_engine; + } + + port->engine = &foreign; + + if (nxt_slow_path(nxt_mp_cleanup(port->mem_pool, + nxt_port_fail_test_released, + task, port, NULL) != NXT_OK)) + { + nxt_port_use(task, port, -1); + goto fail_engine; + } + + nxt_port_fail_test_releases = 0; + nxt_port_fail_test_signals = 0; + + nxt_memzero(&race, sizeof(race)); + race.port = port; + race.engine = &other; + + link = nxt_zalloc(sizeof(nxt_thread_link_t)); + if (nxt_slow_path(link == NULL)) { + goto fail_port; + } + + link->start = nxt_port_fail_test_racer; + link->work.data = ∽̱ + + if (nxt_slow_path(nxt_thread_create(&handle, link) != NXT_OK)) { + goto fail_port; + } + + /* + * The racer has to be spinning before the drop, or the window it + * watches for is gone before it looks. + */ + + while (race.ready == 0) { + nxt_cpu_pause(); + } + + nxt_port_use(task, port, -1); + + race.stop = 1; + + nxt_thread_wait(handle); + + if (nxt_slow_path(race.zeroes != 0)) { + nxt_log_alert(thr->log, "port fail test: a foreign thread saw " + "use_count 0 on a live port %ui times", race.zeroes); + goto fail_port; + } + + queued = nxt_port_fail_test_queued(&foreign.locked_work_queue, + &port->release_work); + + if (nxt_slow_path(queued < 0)) { + nxt_log_alert(thr->log, "port fail test: the embedded release " + "work is linked into the target engine's queue " + "more than once"); + goto fail_engine; + } + + if (nxt_slow_path(queued != 1)) { + nxt_log_alert(thr->log, "port fail test: the embedded release " + "work is queued %i times, expected 1", queued); + goto fail_port; + } + + if (nxt_slow_path(nxt_port_fail_test_signals != 1)) { + nxt_log_alert(thr->log, "port fail test: the target engine was " + "signalled %ui times, expected 1", + nxt_port_fail_test_signals); + goto fail_port; + } + + if (nxt_slow_path(port->use_count != 1)) { + nxt_log_alert(thr->log, "port fail test: use_count is %A with " + "the deferral in flight, expected 1", + port->use_count); + goto fail_port; + } + + thr->engine = &foreign; + + nxt_locked_work_queue_move(thr, &foreign.locked_work_queue, + ¤t.fast_work_queue); + + nxt_port_fail_test_drain_wq(¤t.fast_work_queue); + + if (nxt_slow_path(nxt_port_fail_test_releases != 1)) { + nxt_log_alert(thr->log, "port fail test: draining the target " + "engine released the port %ui times, expected 1", + nxt_port_fail_test_releases); + goto fail_engine; + } + + nxt_work_queue_cache_destroy(¤t.work_queue_cache); + } + + thr->engine = NULL; + + return NXT_OK; + +fail_port: + + nxt_port_use(task, port, -1); + +fail_engine: + + nxt_work_queue_cache_destroy(¤t.work_queue_cache); + thr->engine = NULL; + + return NXT_ERROR; +} + + +static void +nxt_port_fail_test_racer(void *data) +{ + nxt_task_t task; + nxt_uint_t i; + nxt_thread_t *thr; + nxt_port_fail_test_race_t *race; + + race = data; + + thr = nxt_thread(); + thr->engine = race->engine; + + nxt_memzero(&task, sizeof(task)); + task.thread = thr; + task.log = thr->log; + + race->ready = 1; + + for (i = 0; i < NXT_PORT_FAIL_TEST_RACE_SPINS; i++) { + + if (nxt_slow_path(race->port->use_count == 0)) { + race->zeroes++; + break; + } + + if (race->stop != 0) { + break; + } + + nxt_cpu_pause(); + } + + /* + * The acquisition another engine can still make while the deferral is + * in flight, and the drop that follows it: 1 -> 2 -> 1, which must not + * reach the deferral and must not post anything. + */ + + nxt_port_use(&task, race->port, 1); + nxt_port_use(&task, race->port, -1); +} + + +/* + * Count the occurrences of one work item in a locked work queue, refusing to + * walk further than the queue can legitimately be. A work item posted twice + * becomes its own successor, so the walk would otherwise not terminate -- + * which is exactly what the draining engine does. + */ + +/* + * Re-arming a port that belongs to another engine. + * + * nxt_port_rearm() posts the port's own item rather than allocating one, so + * the re-arm survives the memory pressure that made the write fail in the + * first place. Three things have to hold: the item on the queue is the + * port's, a second attempt does not link it twice, and the reference the post + * takes is given back by the handler. + * + * The last phase covers a port closed while the item is in flight. The + * handler must not enable the event then: nxt_port_write_close() leaves + * socket.fd at the number it had, which by that point may belong to somebody + * else. + */ + +static nxt_int_t +nxt_port_fail_test_cross_engine_rearm(nxt_thread_t *thr) +{ + nxt_int_t ret; + nxt_task_t *task; + nxt_port_t *port; + nxt_work_t *posted; + nxt_event_engine_t current, foreign; + + task = thr->task; + task->thread = thr; + + ret = NXT_ERROR; + + nxt_memzero(¤t, sizeof(current)); + nxt_work_queue_cache_create(¤t.work_queue_cache, 1024); + current.fast_work_queue.cache = ¤t.work_queue_cache; + nxt_work_queue_name(¤t.fast_work_queue, "fast"); + + nxt_memzero(&foreign, sizeof(foreign)); + foreign.task.thread = thr; + foreign.task.log = thr->log; + foreign.event.signal = nxt_port_fail_test_engine_signal; + foreign.event.enable_write = nxt_port_fail_test_enable_write; + + thr->engine = ¤t; + + port = nxt_port_fail_test_port(task); + if (nxt_slow_path(port == NULL)) { + goto done; + } + + port->engine = &foreign; + + /* Any descriptor but -1: the stub engine never touches the number. */ + port->pair[1] = 0; + + nxt_port_fail_test_signals = 0; + nxt_port_fail_test_rearms = 0; + + posted = &port->rearm_work; + + nxt_port_rearm(task, port); + + if (nxt_slow_path(nxt_port_fail_test_rearms != 0)) { + nxt_log_alert(thr->log, "port fail test: the cross-engine re-arm ran " + "on the calling thread"); + goto done; + } + + if (nxt_slow_path(foreign.locked_work_queue.head != posted)) { + nxt_log_alert(thr->log, "port fail test: the posted item is not the " + "port's embedded re-arm work"); + goto done; + } + + if (nxt_slow_path(nxt_port_fail_test_signals != 1)) { + nxt_log_alert(thr->log, "port fail test: the cross-engine re-arm did " + "not signal the target engine (%ui)", + nxt_port_fail_test_signals); + goto done; + } + + if (nxt_slow_path(port->use_count != 2 || port->rearm_pending != 1)) { + nxt_log_alert(thr->log, "port fail test: the posted re-arm holds no " + "reference (use_count %A pending %A)", port->use_count, + port->rearm_pending); + goto done; + } + + /* A second attempt while the first is queued must be a no-op. */ + + nxt_port_rearm(task, port); + + if (nxt_slow_path(nxt_port_fail_test_signals != 1 + || port->use_count != 2)) + { + nxt_log_alert(thr->log, "port fail test: a second re-arm posted the " + "same item again (signals %ui use_count %A)", + nxt_port_fail_test_signals, port->use_count); + goto done; + } + + /* The handler runs on the port's engine, so this thread becomes it. */ + + thr->engine = &foreign; + + nxt_locked_work_queue_move(thr, &foreign.locked_work_queue, + ¤t.fast_work_queue); + + nxt_port_fail_test_drain_wq(¤t.fast_work_queue); + + if (nxt_slow_path(nxt_port_fail_test_rearms != 1)) { + nxt_log_alert(thr->log, "port fail test: the posted item re-armed the " + "write event %ui times, expected once", + nxt_port_fail_test_rearms); + goto done; + } + + if (nxt_slow_path(port->use_count != 1 || port->rearm_pending != 0)) { + nxt_log_alert(thr->log, "port fail test: the re-arm handler did not " + "give its reference back (use_count %A pending %A)", + port->use_count, port->rearm_pending); + goto done; + } + + /* + * A same-engine re-arm while a cross-engine post is still queued: it does + * its own work and leaves the queued item exactly as it was. Clearing the + * flag here would let a second poster link the same item again -- and + * severing its ->next would drop whatever the locked queue held behind it. + */ + + thr->engine = ¤t; + + nxt_port_rearm(task, port); + + thr->engine = &foreign; + + nxt_port_rearm(task, port); + + if (nxt_slow_path(port->rearm_pending != 1 + || foreign.locked_work_queue.head != posted + || posted->next != NULL)) + { + nxt_log_alert(thr->log, "port fail test: a same-engine re-arm " + "disturbed a post already in flight (pending %A)", + port->rearm_pending); + goto done; + } + + thr->engine = &foreign; + + nxt_locked_work_queue_move(thr, &foreign.locked_work_queue, + ¤t.fast_work_queue); + + nxt_port_fail_test_drain_wq(¤t.fast_work_queue); + + if (nxt_slow_path(port->rearm_pending != 0 || port->use_count != 1)) { + nxt_log_alert(thr->log, "port fail test: the queued post did not " + "settle (pending %A use_count %A)", port->rearm_pending, + port->use_count); + goto done; + } + + nxt_port_fail_test_rearms = 1; + + /* Closed while the next one is in flight: no enable, reference returned. */ + + thr->engine = ¤t; + + nxt_port_rearm(task, port); + + port->pair[1] = -1; + + thr->engine = &foreign; + + nxt_locked_work_queue_move(thr, &foreign.locked_work_queue, + ¤t.fast_work_queue); + + nxt_port_fail_test_drain_wq(¤t.fast_work_queue); + + if (nxt_slow_path(nxt_port_fail_test_rearms != 1)) { + nxt_log_alert(thr->log, "port fail test: the re-arm enabled the write " + "event of a closed port"); + goto done; + } + + if (nxt_slow_path(port->use_count != 1 || port->rearm_pending != 0)) { + nxt_log_alert(thr->log, "port fail test: a re-arm that found the port " + "closed kept its reference (use_count %A pending %A)", + port->use_count, port->rearm_pending); + goto done; + } + + ret = NXT_OK; + +done: + + if (port != NULL) { + /* nxt_port_mp_cleanup() asserts on this in a debug build. */ + port->pair[1] = -1; + + thr->engine = &foreign; + nxt_port_use(task, port, -1); + } + + thr->engine = ¤t; + + nxt_work_queue_cache_destroy(¤t.work_queue_cache); + + return ret; +} + + +/* + * A caller gets its buffer back when the port refuses the message. + * + * src/nxt_port.h says the port layer takes fd, fd2 and b only on NXT_OK, so on + * any other answer the caller still owns them. A caller that does not act on + * that leaks: nothing else completes the buffer, and for an engine pool that + * memory is never reclaimed while the process runs. + * + * nxt_port_send_port() stands in for the eight sites that hand a buffer over. + * It is the awkward one -- its buffer never leaves the function, so no caller + * of it could clean up even if it wanted to -- and it allocates from the + * engine pool, which makes nxt_mp_is_empty() a decisive check: the pool holds + * the buffer, and nothing else. + */ + +static nxt_int_t +nxt_port_fail_test_caller_owns_buf(nxt_thread_t *thr) +{ + nxt_mp_t *mp; + nxt_int_t ret; + nxt_task_t *task; + nxt_port_t *port, *new_port; + nxt_event_engine_t engine, *saved_engine; + + task = thr->task; + task->thread = thr; + + ret = NXT_ERROR; + port = NULL; + new_port = NULL; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + nxt_memzero(&engine, sizeof(engine)); + nxt_work_queue_cache_create(&engine.work_queue_cache, 1024); + engine.fast_work_queue.cache = &engine.work_queue_cache; + nxt_work_queue_name(&engine.fast_work_queue, "fast"); + + /* The buffer comes from here, which is what makes the check decisive. */ + engine.mem_pool = mp; + + saved_engine = thr->engine; + thr->engine = &engine; + + port = nxt_port_fail_test_port(task); + if (nxt_slow_path(port == NULL)) { + goto done; + } + + new_port = nxt_port_fail_test_port(task); + if (nxt_slow_path(new_port == NULL)) { + goto done; + } + + if (nxt_slow_path(!nxt_mp_is_empty(mp))) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: the fixture pool is not empty " + "before the write"); + goto done; + } + + nxt_port_test_msg_alloc_failures(1); + + ret = nxt_port_send_port(task, port, new_port, 0); + + nxt_port_test_msg_alloc_failures(0); + + if (ret == NXT_OK) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: the refused NEW_PORT answered " + "NXT_OK"); + ret = NXT_ERROR; + goto done; + } + + ret = NXT_ERROR; + + /* The completion is queued, as nxt_port_msg_drop() queues its own. */ + + nxt_port_fail_test_drain_wq(&engine.fast_work_queue); + + if (nxt_slow_path(!nxt_mp_is_empty(mp))) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: the buffer of a refused message was " + "left in the caller's pool"); + goto done; + } + + ret = NXT_OK; + +done: + + nxt_port_test_msg_alloc_failures(0); + + if (new_port != NULL) { + nxt_port_use(task, new_port, -1); + } + + if (port != NULL) { + nxt_port_use(task, port, -1); + } + + thr->engine = saved_engine; + + nxt_work_queue_cache_destroy(&engine.work_queue_cache); + nxt_mp_destroy(mp); + + return ret; +} + + +static nxt_int_t +nxt_port_fail_test_queued(nxt_locked_work_queue_t *lwq, nxt_work_t *item) +{ + nxt_int_t n, steps; + nxt_work_t *w; + + n = 0; + + for (w = lwq->head, steps = 0; w != NULL; w = w->next, steps++) { + + if (steps >= NXT_PORT_FAIL_TEST_QUEUE_LIMIT) { + return -1; + } + + if (w == item) { + n++; + } + } + + return n; +} + + +/* + * A drop can carry more than one reference at once: nxt_port_socket_write() + * and nxt_port_error_handler() batch theirs into use_delta + * (src/nxt_port_socket.c:601, src/nxt_port_socket.c:1500), so a cross-engine + * drop of -2 or more can be the last one. + * + * The deferral must then leave exactly one reference standing, whatever the + * size of the batch, because the posted handler drops exactly one. A + * hand-over that left the whole batch behind would strand the port, its + * memory pool and the process reference it holds -- the very leak this + * change exists to remove, reintroduced through the other door. + */ + +static nxt_int_t +nxt_port_fail_test_cross_engine_batch(nxt_thread_t *thr) +{ + nxt_task_t *task; + nxt_port_t *port; + nxt_event_engine_t current, foreign; + + task = thr->task; + task->thread = thr; + + nxt_memzero(¤t, sizeof(current)); + nxt_work_queue_cache_create(¤t.work_queue_cache, 1024); + current.fast_work_queue.cache = ¤t.work_queue_cache; + nxt_work_queue_name(¤t.fast_work_queue, "fast"); + + nxt_memzero(&foreign, sizeof(foreign)); + foreign.task.thread = thr; + foreign.task.log = thr->log; + foreign.event.signal = nxt_port_fail_test_engine_signal; + + thr->engine = ¤t; + + port = nxt_port_fail_test_port(task); + if (nxt_slow_path(port == NULL)) { + goto fail_engine; + } + + port->engine = &foreign; + + if (nxt_slow_path(nxt_mp_cleanup(port->mem_pool, + nxt_port_fail_test_released, + task, port, NULL) != NXT_OK)) + { + nxt_port_use(task, port, -1); + goto fail_engine; + } + + nxt_port_fail_test_releases = 0; + + /* + * Two references, given up by one drop, from a thread that is not the + * port's engine. + */ + + nxt_port_use(task, port, 1); + + nxt_port_use(task, port, -2); + + if (nxt_slow_path(nxt_port_fail_test_releases != 0)) { + nxt_log_alert(thr->log, "port fail test: a batched cross-engine " + "release ran on the calling thread"); + goto fail_engine; + } + + if (nxt_slow_path(foreign.locked_work_queue.head != &port->release_work)) { + nxt_log_alert(thr->log, "port fail test: a batched cross-engine last " + "drop did not post the port's embedded release work"); + goto fail_engine; + } + + if (nxt_slow_path(port->use_count != 1)) { + nxt_log_alert(thr->log, "port fail test: use_count is %A after a " + "batched cross-engine last drop, expected 1", + port->use_count); + goto fail_port; + } + + thr->engine = &foreign; + + nxt_locked_work_queue_move(thr, &foreign.locked_work_queue, + ¤t.fast_work_queue); + + nxt_port_fail_test_drain_wq(¤t.fast_work_queue); + + if (nxt_slow_path(nxt_port_fail_test_releases != 1)) { + nxt_log_alert(thr->log, "port fail test: draining the target engine " + "released the batched port %ui times, expected 1", + nxt_port_fail_test_releases); + goto fail_engine; + } + + nxt_work_queue_cache_destroy(¤t.work_queue_cache); + thr->engine = NULL; + + return NXT_OK; + +fail_port: + + nxt_port_use(task, port, -1); + +fail_engine: + + nxt_work_queue_cache_destroy(¤t.work_queue_cache); + thr->engine = NULL; + + return NXT_ERROR; +} + + +static void +nxt_port_fail_test_engine_signal(nxt_event_engine_t *engine, nxt_uint_t signo) +{ + nxt_port_fail_test_signals++; +} + + +static void +nxt_port_fail_test_released(nxt_task_t *task, void *obj, void *data) +{ + nxt_port_fail_test_releases++; +} + + +static void +nxt_port_fail_test_mp_completion(nxt_task_t *task, void *obj, void *data) +{ + nxt_mp_t *mp = data; + + nxt_port_fail_test_completions++; + + if (mp != NULL) { + nxt_mp_release(mp); + } +} + + +static void +nxt_port_fail_test_completion(nxt_task_t *task, void *obj, void *data) +{ + nxt_port_fail_test_completions++; +} + + +static void +nxt_port_fail_test_drain_wq(nxt_work_queue_t *wq) +{ + void *obj, *data; + nxt_task_t *t; + nxt_work_handler_t handler; + + while (wq->head != NULL) { + handler = nxt_work_queue_pop(wq, &t, &obj, &data); + handler(t, obj, data); + } +} + + +static nxt_int_t +nxt_port_fail_test_fd_count(void) +{ +#if (NXT_LINUX) + nxt_int_t count; + DIR *dir; + struct dirent *de; + + dir = opendir("/proc/self/fd"); + if (dir == NULL) { + return -1; + } + + count = 0; + + for ( ;; ) { + de = readdir(dir); + + if (de == NULL) { + break; + } + + if (nxt_strcmp(de->d_name, ".") != 0 + && nxt_strcmp(de->d_name, "..") != 0) + { + count++; + } + } + + (void) closedir(dir); + + return count; +#else + return -1; +#endif +} + + +/* + * One payload into the shared ring, with the wake-up that follows failing + * with "err" -- and with no memory to queue the marker when "no_memory" is + * set. The answer must be NXT_OK and the payload completed exactly once + * whatever became of the wake-up: it is in the ring, and the caller must + * not be told to reclaim it. The ring is drained first so that this + * enqueue is the 0-to-1 transition that raises the wake-up at all. + */ + +#if (NXT_HAVE_EPOLL_EDGE) + +static nxt_int_t +nxt_port_fail_test_enqueue(nxt_task_t *task, nxt_port_t *port, nxt_mp_t *mp, + nxt_err_t err, nxt_uint_t fails, nxt_bool_t no_memory, nxt_bool_t *queued) +{ + nxt_buf_t *buf; + nxt_int_t ret; + u_char block[NXT_PORT_QUEUE_MSG_SIZE]; + + while (nxt_port_queue_recv(port->queue, block) > 0) { + /* void */ + } + + buf = nxt_buf_mem_alloc(mp, 1, 0); + if (nxt_slow_path(buf == NULL)) { + return NXT_ERROR; + } + + buf->completion_handler = nxt_port_fail_test_completion; + buf->mem.free++; + + port->socket.write_ready = 1; + + nxt_socketpair_test_send_fail(err, fails); + + if (no_memory) { + nxt_port_test_msg_alloc_failures(1); + } + + ret = nxt_port_socket_write(task, port, NXT_PORT_MSG_DATA, -1, 0, 0, buf); + + nxt_port_test_msg_alloc_failures(0); + + /* + * The send hook clears itself once its count is spent, and the inline + * write above spends one. A leg that arms more than one clears what + * is left when it is done, so that a retry can be made to fail too. + */ + + if (ret != NXT_OK) { + nxt_log_error(NXT_LOG_NOTICE, task->log, + "port failure test: a payload that reached the shared " + "queue answered %d after sendmsg() error %d, expected " + "NXT_OK", (int) ret, (int) err); + return NXT_ERROR; + } + + /* + * Read before the drain, because the drain destroys the answer: an + * error raised by the write queues nxt_port_error_handler(), which + * empties port->messages whatever is in it. A caller that tested the + * queue after this returned would find it empty either way. + */ + + if (queued != NULL) { + *queued = !nxt_queue_is_empty(&port->messages); + } + + nxt_port_fail_test_drain_wq(&task->thread->engine->fast_work_queue); + + if (nxt_port_fail_test_completions != 1) { + nxt_log_error(NXT_LOG_NOTICE, task->log, + "port failure test: the enqueued payload was completed " + "%d times after sendmsg() error %d, expected once", + (int) nxt_port_fail_test_completions, (int) err); + return NXT_ERROR; + } + + return NXT_OK; +} + + +/* + * One turn of the event loop, as nxt_event_engine_start() runs it, except + * that the clock moves on by "ms" instead of being read from the machine. + * + * The backoff arms a timer on the engine, and a leg that measures pacing has + * to be able to tell a retry that came back because time passed from one + * that came back because nothing held it. A real clock cannot answer that + * without sleeping, and sleeping would make the numbers the scheduler's + * rather than the port's. In these tests, only nxt_timer_expire() + * advances engine->timers.now. The test functions control the timer clock + * directly by advancing it in discrete intervals. + */ + +static void +nxt_port_fail_test_turn(nxt_event_engine_t *engine, nxt_msec_t ms) +{ + /* + * Commits pending timer changes and sets timers->minimum, as the loop + * does before it polls; without it nxt_timer_expire() returns early. + */ + + (void) nxt_timer_find(engine); + + engine->event.poll(engine, 0); + + nxt_timer_expire(engine, engine->timers.now + ms); + + nxt_port_fail_test_drain_wq(&engine->fast_work_queue); +} + + +/* + * Poll hard for "window" virtual milliseconds and answer how many times the + * port's write handler was dispatched. + * + * The inner loop is the busy spin itself: on a socket that stayed writable, + * a retry that re-checks readiness on the spot has epoll_wait() return + * immediately every time, so the engine thread runs this loop as fast as it + * can rather than at any rate the clock sets. Sixteen polls per virtual + * millisecond stands in for that -- the real thing manages far more -- and + * makes the two outcomes differ by a factor, not by a margin: unpaced, the + * count follows the number of polls; paced, it follows the window. + * + * A non-NULL "rearm" is re-armed once every virtual millisecond, as a + * sender on this port would while the shortage lasts: each of its sends + * fails inline too and ends in nxt_port_rearm(). + */ + +static nxt_uint_t +nxt_port_fail_test_spin(nxt_task_t *task, nxt_msec_t window, + nxt_port_t *rearm) +{ + nxt_uint_t i, j; + nxt_event_engine_t *engine; + + engine = task->thread->engine; + + nxt_port_test_write_dispatches = 0; + + for (i = 0; i < window; i++) { + + if (rearm != NULL) { + nxt_port_rearm(task, rearm); + } + + for (j = 0; j < 16; j++) { + engine->event.poll(engine, 0); + + nxt_port_fail_test_drain_wq(&engine->fast_work_queue); + } + + nxt_port_fail_test_turn(engine, 1); + } + + return nxt_port_test_write_dispatches; +} + + +/* + * Spin for 64 virtual milliseconds with the failure held, report the count, + * and require it to follow the clock rather than the polls: sixteen is one + * dispatch per virtual millisecond, which no paced retry reaches and every + * unpaced one exceeds by the poll factor. + */ + +static nxt_int_t +nxt_port_fail_test_paced(nxt_task_t *task, nxt_uint_t leg, const char *what, + nxt_port_t *port, nxt_port_t *rearm) +{ + nxt_uint_t dispatches; + + dispatches = nxt_port_fail_test_spin(task, 64, rearm); + + /* + * Write dispatch counters do not record attempts made without an active + * event. Verify pacing directly: ensure the retry timer remains enabled + * at the end of the 64 ms window. + */ + + if (!port->retry_timer.enabled) { + nxt_log_error(NXT_LOG_NOTICE, task->log, + "port failure test: leg %ui: no retry timer is armed " + "after a 64ms window of held ENOMEM; the retry is not " + "paced", leg); + return NXT_ERROR; + } + + nxt_log_error(NXT_LOG_NOTICE, task->log, + "port failure test: leg %ui: %s, %ui write dispatches in a " + "64ms window of held ENOMEM", leg, what, dispatches); + + if (dispatches > 16) { + nxt_log_error(NXT_LOG_NOTICE, task->log, + "port failure test: leg %ui: the write handler was " + "dispatched %ui times in a 64ms window of held ENOMEM, " + "expected at most 16; the retry is not paced", + leg, dispatches); + return NXT_ERROR; + } + + return NXT_OK; +} + + +/* + * Let the injection go and require the marker to arrive anyway, once: + * pacing a retry must not turn into dropping one. The paced retry is + * waited for on the clock, since nothing else will bring it back. + */ + +static nxt_int_t +nxt_port_fail_test_recovers(nxt_task_t *task, nxt_uint_t leg, + nxt_port_t *port, nxt_fd_t peer) +{ + u_char block[sizeof(nxt_port_msg_t) + 1]; + ssize_t n; + nxt_uint_t i; + nxt_event_engine_t *engine; + + engine = task->thread->engine; + + nxt_socketpair_test_send_fail(0, 0); + + for (i = 0; i < 8; i++) { + if (port->announce == 0 && nxt_queue_is_empty(&port->messages)) { + break; + } + + nxt_port_fail_test_turn(engine, 64); + } + + n = recv(peer, block, sizeof(block), MSG_DONTWAIT); + + if (n != (ssize_t) sizeof(nxt_port_msg_t) + || ((nxt_port_msg_t *) block)->type != _NXT_PORT_MSG_READ_QUEUE) + { + nxt_log_error(NXT_LOG_NOTICE, task->log, + "port failure test: leg %ui: the peer read %z bytes " + "once the shortage was over, expected a %uz byte " + "READ_QUEUE header; the paced retry never came back " + "(write event state %d)", leg, n, + sizeof(nxt_port_msg_t), (int) port->socket.write); + return NXT_ERROR; + } + + if (!nxt_queue_is_empty(&port->messages) || port->announce != 0 + || nxt_port_fail_test_completions != 1) + { + nxt_log_error(NXT_LOG_NOTICE, task->log, + "port failure test: leg %ui: after the marker went out " + "announce is %d and the payload was completed %ui " + "times, expected 0 and once", leg, + (int) port->announce, nxt_port_fail_test_completions); + return NXT_ERROR; + } + + /* + * An active timer retains a port reference. If the timer is not disabled, + * the port leaks when the test frees the engine. + */ + + if (port->retry_timer.enabled) { + nxt_log_error(NXT_LOG_NOTICE, task->log, + "port failure test: leg %ui: the retry timer is still " + "armed after the marker went out", leg); + return NXT_ERROR; + } + + return NXT_OK; +} + +#endif + + +/* + * The wake-up that fails with an errno the socket recovers from (#392). + * + * A payload that fits the shared ring goes in there, and the inline write + * is only the READ_QUEUE marker that tells the peer to look. + * nxt_port_queue_send() raises "notify" on the 0-to-1 transition alone, so + * if that one marker is lost, no later enqueue on the port raises another: + * the peer stays unaware of this message and of everything after it. + * + * nxt_socketpair_send() answers NXT_AGAIN for EAGAIN and ENOBUFS, and + * nxt_port_write_msgs() then keeps the marker -- queued on the port, or + * owed in port->announce when even that allocation fails. A kernel ENOMEM + * is the same transient condition, and used to fall into the arm for + * errors the socket does not recover from: the marker was dropped, the + * error handler was raised against a live peer, and the write answered + * NXT_OK with nothing owed. + * + * NXT_AGAIN also clears write_ready, so keeping the marker is only half of + * it: the re-arm has to bring the write handler back on its own, with no + * later write from a caller to prompt it. So this runs on a real epoll + * engine, edge-triggered as the product's is, and the marker is read off + * the peer's end after one poll. Twice: once with the write event never + * armed, and once after the drain that sent the first marker has taken + * the event down again -- which used to leave it blocked rather than + * disabled, so the second re-arm touched no registration and the marker + * waited on an edge that never came. + * + * Then the same ENOMEM with no memory to queue the marker, which must be + * owed and announced from the stack; and EPIPE, where the peer is gone and + * nothing may be owed or armed. The errno is injected through + * nxt_socketpair_test_send_fail(), since a live pair does not produce + * either on demand. + */ + +static nxt_int_t +nxt_port_fail_test_wakeup_errno(nxt_thread_t *thr) +{ +#if (NXT_HAVE_EPOLL_EDGE) + ssize_t n; + nxt_mp_t *mp; + nxt_fd_t pair[2]; + nxt_int_t ret; + nxt_uint_t i; + nxt_bool_t queued; + nxt_task_t *task; + nxt_port_t *port; + nxt_event_engine_t *engine, *saved_engine; + nxt_port_queue_t *queue; + nxt_port_send_msg_t *msg; + u_char block[NXT_PORT_QUEUE_MSG_SIZE]; + + task = thr->task; + task->thread = thr; + + ret = NXT_ERROR; + pair[0] = -1; + pair[1] = -1; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + saved_engine = thr->engine; + + engine = nxt_event_engine_create(task, &nxt_epoll_edge_engine, NULL, 0, 0); + if (nxt_slow_path(engine == NULL)) { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + thr->engine = engine; + + port = nxt_port_fail_test_port(task); + if (nxt_slow_path(port == NULL)) { + goto fail; + } + + if (nxt_slow_path(socketpair(AF_UNIX, SOCK_DGRAM, 0, pair) != 0)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: socketpair failed"); + goto done; + } + + if (nxt_slow_path(fcntl(pair[1], F_SETFL, O_NONBLOCK) == -1)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: O_NONBLOCK failed"); + goto done; + } + + port->pair[0] = pair[0]; + port->pair[1] = pair[1]; + port->socket.task = task; + port->max_size = 1024; + port->max_share = 1024; + + /* The real thing: handlers, engine, write_ready, and no event armed. */ + + nxt_port_write_enable(task, port); + + port->socket.log = thr->log; + + queue = nxt_mp_zalloc(mp, sizeof(nxt_port_queue_t)); + if (nxt_slow_path(queue == NULL)) { + goto done; + } + + nxt_port_queue_init(queue); + port->queue = queue; + + /* + * Legs 1 and 2: ENOMEM on the wake-up, with memory to queue the marker. + * The first arms a write event that was never registered; the second + * finds it blocked by the drain that sent the first marker, and the + * enable then touches no epoll registration. Edge-triggered, the + * second is the one that would stall if the re-arm did not re-check. + */ + + for (i = 1; i <= 2; i++) { + + /* + * Quiescent first. The peer's read of the previous marker wakes + * this end's writers, and epoll keeps that as a ready entry for a + * registered descriptor; a poll here consumes it, so the leg + * measures the re-arm and not a wake-up left over from the last + * leg. + */ + + engine->event.poll(engine, 0); + + nxt_port_fail_test_drain_wq(&engine->fast_work_queue); + + nxt_port_fail_test_completions = 0; + + if (nxt_port_fail_test_enqueue(task, port, mp, NXT_ENOMEM, 1, 0, NULL) + != NXT_OK) + { + goto done; + } + + if (nxt_queue_is_empty(&port->messages)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: leg %ui: ENOMEM on the wake-up " + "dropped the queue marker (announce %d); the peer " + "is never told to read the ring", i, + (int) port->announce); + goto done; + } + + msg = nxt_queue_link_data(nxt_queue_first(&port->messages), + nxt_port_send_msg_t, link); + + if (msg->port_msg.type != _NXT_PORT_MSG_READ_QUEUE) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: leg %ui: the message kept " + "after ENOMEM is of type %d, expected READ_QUEUE", + i, (int) msg->port_msg.type); + goto done; + } + + if (port->socket.write_ready != 0 + || !nxt_fd_event_is_active(port->socket.write)) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: leg %ui: after ENOMEM " + "write_ready is %d and the write event state is " + "%d; expected 0 and an active event", i, + (int) port->socket.write_ready, + (int) port->socket.write); + goto done; + } + + /* One poll, nothing else: the re-arm alone must bring it back. */ + + engine->event.poll(engine, 0); + + nxt_port_fail_test_drain_wq(&engine->fast_work_queue); + + n = recv(pair[0], block, sizeof(block), MSG_DONTWAIT); + + if (n != (ssize_t) sizeof(nxt_port_msg_t) + || ((nxt_port_msg_t *) block)->type != _NXT_PORT_MSG_READ_QUEUE) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: leg %ui: the peer read %d bytes " + "after the re-arm and one poll, expected a %d byte " + "READ_QUEUE header (write event state %d)", i, + (int) n, (int) sizeof(nxt_port_msg_t), + (int) port->socket.write); + goto done; + } + + if (!nxt_queue_is_empty(&port->messages) || port->announce != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: leg %ui: the marker went out " + "but is still held (announce %d)", i, + (int) port->announce); + goto done; + } + + if (nxt_fd_event_is_active(port->socket.write)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: leg %ui: the drain left the " + "write event in state %d, expected it disabled", i, + (int) port->socket.write); + goto done; + } + } + + /* + * Leg 3: ENOMEM on the wake-up, and ENOMEM again when the write event + * retries the queued marker. The first failure is what legs 1 and 2 + * cover: the inline pass re-arms a disabled event and the retry runs + * on the next poll. The second finds the event already active, so + * nothing re-arms it, and the socket never filled, so no edge is + * coming either. The retry pass has to force the readiness re-check + * itself; without it the marker, and every message queued behind it, + * waits for ever. + * + * That re-check is paced (#407), so the marker does not go out on the + * next poll but on the first one after the timer the retry armed has + * expired. Which is why the legs from here on drive the clock as well + * as the poller. + */ + + engine->event.poll(engine, 0); + + nxt_port_fail_test_drain_wq(&engine->fast_work_queue); + + nxt_port_fail_test_completions = 0; + + if (nxt_port_fail_test_enqueue(task, port, mp, NXT_ENOMEM, 2, 0, NULL) + != NXT_OK) + { + goto done; + } + + if (nxt_queue_is_empty(&port->messages) || port->socket.write_ready != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: leg 3: after the first ENOMEM the " + "marker is not queued, or write_ready is %d", + (int) port->socket.write_ready); + goto done; + } + + /* The first poll runs the retry, which is made to fail again. */ + + engine->event.poll(engine, 0); + + nxt_port_fail_test_drain_wq(&engine->fast_work_queue); + + if (nxt_queue_is_empty(&port->messages)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: leg 3: the retry did not fail; " + "the marker left the queue after one poll"); + goto done; + } + + /* + * The re-check is all there is to bring the retry back, and it is armed + * on a timer: the first turn expires it and re-arms the write event, the + * second is the poll that then finds the socket writable. + */ + + nxt_port_fail_test_turn(engine, 64); + + nxt_port_fail_test_turn(engine, 0); + + nxt_socketpair_test_send_fail(0, 0); + + n = recv(pair[0], block, sizeof(block), MSG_DONTWAIT); + + if (n != (ssize_t) sizeof(nxt_port_msg_t) + || ((nxt_port_msg_t *) block)->type != _NXT_PORT_MSG_READ_QUEUE) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: leg 3: the peer read %d bytes after " + "a retry that ran out of memory and two polls, expected " + "a %d byte READ_QUEUE header; the queued marker is " + "stranded (write event state %d, write_ready %d)", + (int) n, (int) sizeof(nxt_port_msg_t), + (int) port->socket.write, + (int) port->socket.write_ready); + goto done; + } + + if (!nxt_queue_is_empty(&port->messages) || port->announce != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: leg 3: the marker went out but is " + "still held (announce %d)", (int) port->announce); + goto done; + } + + if (nxt_fd_event_is_active(port->socket.write)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: leg 3: the drain left the write " + "event in state %d, expected it disabled", + (int) port->socket.write); + goto done; + } + + /* Leg 4: ENOMEM on the wake-up, and no memory to queue the marker. */ + + nxt_port_fail_test_completions = 0; + + if (nxt_port_fail_test_enqueue(task, port, mp, NXT_ENOMEM, 1, 1, NULL) + != NXT_OK) + { + goto done; + } + + /* + * Owed, and sent from the stack by the re-arm inside the same write, + * before any poll: the socket is writable, so nxt_port_announce() runs + * on the spot and the count is back to 0 by the time the write answers. + */ + + if (!nxt_queue_is_empty(&port->messages)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: ENOMEM with no memory to queue the " + "marker queued one anyway"); + goto done; + } + + n = recv(pair[0], block, sizeof(block), MSG_DONTWAIT); + + if (port->announce != 0 || n != (ssize_t) sizeof(nxt_port_msg_t) + || ((nxt_port_msg_t *) block)->type != _NXT_PORT_MSG_READ_QUEUE) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: the owed marker was not announced " + "(announce %d, %d bytes read)", (int) port->announce, + (int) n); + goto done; + } + + /* Leg 5: EPIPE. The peer is gone; nothing is owed and nothing sent. */ + + nxt_port_fail_test_completions = 0; + + if (nxt_port_fail_test_enqueue(task, port, mp, NXT_EPIPE, 1, 0, &queued) + != NXT_OK) + { + goto done; + } + + if (queued || port->announce != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: EPIPE on the wake-up kept a marker " + "for a peer that is gone (queued %d, announce %d)", + (int) queued, (int) port->announce); + goto done; + } + + engine->event.poll(engine, 0); + + nxt_port_fail_test_drain_wq(&engine->fast_work_queue); + + n = recv(pair[0], block, sizeof(block), MSG_DONTWAIT); + + if (n != -1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: %d bytes reached the peer after " + "EPIPE", (int) n); + goto done; + } + + /* + * Leg 6: an ENOMEM that does not let up, on a queued message. + * + * The retry has to keep coming back -- that is #392 and #393 -- but a + * shortage lasts, and the socket it cannot send on stays writable, so a + * retry that re-checks readiness on the spot is dispatched again the + * moment it returns. The engine thread then spends a core on + * epoll_wait/sendmsg/epoll_ctl for as long as the machine is short of + * memory (#407). + * + * So: hold the failure, poll far harder than any clock would justify, + * and count the dispatches. Unpaced the count is the poll count; paced + * it is a handful, set by the window and the delays. Then let the + * injection go and require the marker to arrive anyway: pacing a retry + * must not turn into dropping one. + */ + + nxt_port_fail_test_turn(engine, 0); + + nxt_port_fail_test_completions = 0; + + if (nxt_port_fail_test_enqueue(task, port, mp, NXT_ENOMEM, 100000, 0, + NULL) + != NXT_OK) + { + goto done; + } + + if (nxt_port_fail_test_paced(task, 6, "queued marker", port, NULL) + != NXT_OK) + { + goto done; + } + + if (nxt_queue_is_empty(&port->messages)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: leg 6: the marker was dropped " + "while the retry was being paced"); + goto done; + } + + if (nxt_port_fail_test_recovers(task, 6, port, pair[0]) != NXT_OK) { + goto done; + } + + /* + * Leg 7: the same shortage on the owed marker, which is the other way + * this port can spin and does not come from #393 at all. + * nxt_port_announce() clears port->announce only on a send that fully + * succeeded, and the re-arm attempts the marker every pass, so a marker + * that keeps failing spins whether or not the queued path is paced. + * + * Here the marker is owed rather than queued, because the allocation + * that would have held it failed too -- leg 4's setup, with a failure + * that does not stop after one. + */ + + nxt_port_fail_test_completions = 0; + + if (nxt_port_fail_test_enqueue(task, port, mp, NXT_ENOMEM, 100000, 1, + NULL) + != NXT_OK) + { + goto done; + } + + if (port->announce == 0 || !nxt_queue_is_empty(&port->messages)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: leg 7: the marker is neither owed " + "nor queued (announce %d)", (int) port->announce); + goto done; + } + + if (nxt_port_fail_test_paced(task, 7, "owed marker", port, NULL) + != NXT_OK) + { + goto done; + } + + if (port->announce == 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: leg 7: the owed marker was " + "forgotten while the retry was being paced"); + goto done; + } + + if (nxt_port_fail_test_recovers(task, 7, port, pair[0]) != NXT_OK) { + goto done; + } + + /* + * The shortage is over, so the next one has to start from the short + * delay again. The marker went out through nxt_port_announce(), not + * through a queued send, and only a queued send used to reset it. + */ + + if (port->retry_delay != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: leg 7: the retry delay is %d ms " + "after the owed marker went out, expected 0", + (int) port->retry_delay); + goto done; + } + + /* + * Leg 8: leg 7's shortage, with senders that keep arriving. + * + * Each of their sends fails inline too and re-arms the port. A re-arm + * that enables the write event while the retry timer is armed brings + * the handler back on a socket that stayed writable, and the handler + * re-arms on the owed marker again: leg 7's spin, for as long as the + * senders keep coming. + */ + + nxt_port_fail_test_completions = 0; + + if (nxt_port_fail_test_enqueue(task, port, mp, NXT_ENOMEM, 100000, 1, + NULL) + != NXT_OK) + { + goto done; + } + + if (port->announce == 0 || !port->retry_timer.enabled) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: leg 8: the marker is not owed or " + "the retry is not armed (announce %d, armed %d)", + (int) port->announce, (int) port->retry_timer.enabled); + goto done; + } + + if (nxt_port_fail_test_paced(task, 8, "owed marker re-armed every ms", + port, port) + != NXT_OK) + { + goto done; + } + + if (nxt_port_fail_test_recovers(task, 8, port, pair[0]) != NXT_OK) { + goto done; + } + + /* + * Leg 9: A send fails with EPIPE while a retry is delayed and a marker + * is pending. + * + * When the retry timer fires, nxt_port_rearm_now() attempts to send + * the pending marker first. + * + * If the send returns EPIPE or another unrecoverable error, the error + * is permanent, not a transient memory shortage. Continuing to delay + * retries would leave the timer running indefinitely without re-enabling + * the write event. In that case, queued messages would never reach the + * error handler, leaking port references. + * + * Instead, nxt_port_announce() must forward queued messages to + * nxt_port_error_handler(), matching the write failure path. + * + * Sequence: + * 1. Trigger a pending marker (simulate ENOMEM when queuing). + * 2. Queue a second marker. + * 3. Hold ENOMEM until the retry timer is set. + * 4. Change simulated error to EPIPE and advance the timer clock. + */ + + nxt_port_fail_test_completions = 0; + + if (nxt_port_fail_test_enqueue(task, port, mp, NXT_ENOMEM, 100000, 1, + NULL) + != NXT_OK) + { + goto done; + } + + /* The helper counts one completion per enqueue. */ + + nxt_port_fail_test_completions = 0; + + if (nxt_port_fail_test_enqueue(task, port, mp, NXT_ENOMEM, 100000, 0, + NULL) + != NXT_OK) + { + goto done; + } + + if (port->announce == 0 || nxt_queue_is_empty(&port->messages)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: leg 9: expected an owed and a " + "queued marker (announce %d, queue empty %d)", + (int) port->announce, + (int) nxt_queue_is_empty(&port->messages)); + goto done; + } + + if (nxt_port_fail_test_paced(task, 9, "owed and queued marker", port, + NULL) + != NXT_OK) + { + goto done; + } + + nxt_socketpair_test_send_fail(NXT_EPIPE, 100000); + + for (i = 0; i < 4; i++) { + nxt_port_fail_test_turn(engine, 64); + } + + n = recv(pair[0], block, sizeof(block), MSG_DONTWAIT); + + if (n != -1 || nxt_errno != NXT_EAGAIN + || !nxt_queue_is_empty(&port->messages) + || port->announce != 0 || port->retry_timer.enabled + || nxt_fd_event_is_active(port->socket.write) + || port->use_count != 1 || nxt_port_fail_test_completions != 1) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: leg 9: after the send failed with " + "EPIPE, %d bytes read (errno %d), queue empty %d, " + "announce %d, timer armed %d, write active %d, " + "use_count %d, completions %ui; expected -1, %d, 1, 0, " + "0, 0, 1 and 1", (int) n, (int) nxt_errno, + (int) nxt_queue_is_empty(&port->messages), + (int) port->announce, (int) port->retry_timer.enabled, + (int) nxt_fd_event_is_active(port->socket.write), + (int) port->use_count, nxt_port_fail_test_completions, + (int) NXT_EAGAIN); + goto done; + } + + ret = NXT_OK; + +done: + + nxt_socketpair_test_send_fail(0, 0); + nxt_port_test_msg_alloc_failures(0); + + nxt_port_fail_test_drain_wq(&engine->fast_work_queue); + + /* See nxt_port_fail_test_inline_drop() for both of these. */ + + port->queue = NULL; + + nxt_port_close(task, port); + + /* + * If a test leg fails while the retry timer is enabled, the timer retains + * a port reference. + * + * Advance the event loop past the maximum delay. The expired timer detects + * that the port is closed and releases its reference, allowing clean + * engine teardown. + */ + + nxt_port_fail_test_turn(engine, 64); + + if (pair[0] != -1 && nxt_test_fd_is_open(pair[0])) { + nxt_fd_close(pair[0]); + } + + nxt_port_use(task, port, -1); + +fail: + + thr->engine = saved_engine; + + nxt_event_engine_free(engine); + nxt_mp_destroy(mp); + + return ret; - return count; #else - return -1; + + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: wake-up errno leg needs epoll, skipped"); + + return NXT_OK; + #endif } diff --git a/src/test/nxt_port_fd_test.c b/src/test/nxt_port_fd_test.c new file mode 100644 index 000000000..e5d2cc9ec --- /dev/null +++ b/src/test/nxt_port_fd_test.c @@ -0,0 +1,785 @@ +/* + * Copyright (C) F5, Inc. + */ + +/* + * Regression test for descriptor ownership in nxt_port_new_port_handler() + * and nxt_port_mmap_handler() (src/nxt_port.c). + * + * A port message can carry two descriptors whatever its type normally uses + * -- nxt_socket_msg_oob_get() accepts an SCM_RIGHTS payload of one or two -- + * and the dispatcher does not reclaim them once the handler returns. Every + * handler therefore has to release what it does not consume, and clear the + * slot of what it does. Both handlers here ignored fd[1] entirely, and + * nxt_port_mmap_handler() returned from its "invalid fd" guard without + * closing anything at all, so a peer attaching descriptors to forged + * messages could exhaust the descriptor table of the router or, through the + * NEW_PORT path, of the main process. + * + * This lives beside nxt_port_ready_test.c rather than inside it: that test + * builds a fixture for a single handler and walks its reject paths, while + * these two handlers need a port hash and a message body instead. The + * dispatcher (nxt_port_handler()) is covered here too: a type past the + * table, and an in-range type whose slot the handler table never filled, + * both have to close what the message carried without calling anything. + * + * The one path this cannot reach is nxt_runtime_process_port_create() + * returning NULL, which needs an allocation failure the test harness has no + * way to inject. + * + * nxt_port_fd_test_tail_close() and nxt_port_fd_test_frag_merge() go one + * level up, into the static nxt_port_read_msg_process() itself (via the + * NXT_TESTS wrapper nxt_port_test_run_read_msg_process()), for the two + * closes nothing above exercises: the tail close that reclaims whatever a + * dispatch left behind, and the close in the last-fragment merge that runs + * before msg->fd[0] is overwritten by the assembled message's own + * descriptors. nxt_router_response_ready_handler() stays uncovered here + * too, for the same reason as above. + */ + +#include +#include +#include +#include +#include "nxt_tests.h" + +#include +#include + + +static nxt_uint_t nxt_port_fd_test_dispatched; + + +static void +nxt_port_fd_test_enable_read_stub(nxt_event_engine_t *engine, + nxt_fd_event_t *ev) +{ + /* The fixture engine polls nothing. */ +} + + +/* + * A handler that honours the dispatch contract: consume or close what the + * message carries. Used to pin that the NULL-slot guard does not swallow + * types whose slot is set. + */ +static void +nxt_port_fd_test_dispatch_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) +{ + nxt_port_fd_test_dispatched++; + + nxt_port_recv_msg_close_fds(msg); +} + + +/* + * A handler that takes fd[0] -- blanking its slot without closing it, the + * way a handler that keeps a descriptor is required to -- and releases + * fd[1] as usual. Used to pin that nxt_port_read_msg_process() leaves a + * kept descriptor alone: the tail close and the fragment-merge close both + * have to treat "-1" as "already spoken for", not "nothing was here". + */ +static void +nxt_port_fd_test_keep_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) +{ + msg->fd[0] = -1; + + if (msg->fd[1] != -1) { + nxt_fd_close(msg->fd[1]); + msg->fd[1] = -1; + } +} + + +/* + * A handler that ignores the message entirely -- the shape GET_MMAP and + * OOSM have, and the shape every handler had before the fix. It never + * touches msg->fd[], so whatever nxt_port_read_msg_process() does with a + * dispatch's leftovers is the only thing standing between this and a leak. + */ +static void +nxt_port_fd_test_ignore_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) +{ +} + + +static const nxt_port_handlers_t nxt_port_fd_test_handlers = { + .quit = nxt_port_fd_test_dispatch_handler, + .status = nxt_port_fd_test_keep_handler, + .oosm = nxt_port_fd_test_ignore_handler, +}; + + +static void +nxt_port_fd_test_close(nxt_fd_t fd) +{ + if (fd != -1 && nxt_test_fd_is_open(fd)) { + (void) close(fd); + } +} + + +/* + * Attach two descriptors to the message, run the handler, and expect it to + * have released both: neither of these handlers keeps a descriptor of its + * own beyond the call. + */ +static nxt_int_t +nxt_port_fd_test_closed(nxt_thread_t *thr, nxt_task_t *task, + nxt_port_recv_msg_t *msg, + void (*handler)(nxt_task_t *task, nxt_port_recv_msg_t *msg), + nxt_bool_t first_fd, const char *name) +{ + nxt_fd_t fd0, fd1; + + fd0 = first_fd ? open("/dev/null", O_RDONLY) : -1; + fd1 = open("/dev/null", O_RDONLY); + + if ((first_fd && fd0 == -1) || fd1 == -1) { + nxt_log_alert(thr->log, "port fd test failed to open /dev/null"); + goto fail; + } + + msg->fd[0] = fd0; + msg->fd[1] = fd1; + + handler(task, msg); + + if ((first_fd && nxt_test_fd_is_open(fd0)) + || nxt_test_fd_is_open(fd1)) + { + nxt_log_alert(thr->log, "port fd test: %s leaked a descriptor", name); + goto fail; /* closes only what is still open */ + } + + if (msg->fd[0] != -1 || msg->fd[1] != -1) { + nxt_log_alert(thr->log, "port fd test: %s left fds in the message", + name); + return NXT_ERROR; + } + + return NXT_OK; + +fail: + + /* + * Close only what the handler left open: a descriptor it consumed is + * gone already, and closing the number again could reach an unrelated + * descriptor that has reused it. + */ + nxt_port_fd_test_close(fd0); + nxt_port_fd_test_close(fd1); + + return NXT_ERROR; +} + + +/* + * Runs a single, non-fragmented message through nxt_port_read_msg_process() + * and checks what the dispatcher's tail close left behind: this pins the + * close at the tail of the function, which runs after the handler and + * covers whatever it did not take -- distinct from nxt_port_fd_test_closed() + * above, which only exercises the dispatcher's own NULL-slot guard. + * + * "keep_fd0" selects the handler dispatched to: nxt_port_fd_test_handlers + * .oosm ignores the message outright (keep_fd0 false), .status blanks fd[0] + * without closing it (keep_fd0 true). Either way fd[1] is never taken, so + * it is always expected closed. + */ +static nxt_int_t +nxt_port_fd_test_tail_close(nxt_thread_t *thr, nxt_task_t *task, + nxt_port_t *port, nxt_bool_t keep_fd0, const char *name) +{ + int pipe0[2], pipe1[2]; + nxt_buf_t *b; + nxt_port_recv_msg_t msg; + + if (nxt_slow_path(pipe(pipe0) != 0 || pipe(pipe1) != 0)) { + nxt_log_alert(thr->log, "port fd test failed to create a pipe"); + return NXT_ERROR; + } + + /* Only the read end of each pipe travels in the message. */ + (void) close(pipe0[1]); + (void) close(pipe1[1]); + + b = nxt_mp_zalloc(port->mem_pool, sizeof(nxt_buf_t)); + if (nxt_slow_path(b == NULL)) { + (void) close(pipe0[0]); + (void) close(pipe1[0]); + return NXT_ERROR; + } + + nxt_memzero(&msg, sizeof(nxt_port_recv_msg_t)); + + msg.port = port; + msg.buf = b; + msg.size = sizeof(nxt_port_msg_t); + msg.fd[0] = pipe0[0]; + msg.fd[1] = pipe1[0]; + msg.port_msg.type = keep_fd0 ? _NXT_PORT_MSG_STATUS : _NXT_PORT_MSG_OOSM; + + nxt_port_test_run_read_msg_process(task, port, &msg); + + if (nxt_slow_path(keep_fd0 != nxt_test_fd_is_open(pipe0[0]))) { + nxt_log_alert(thr->log, "port fd test: %s left fd[0] %s", name, + keep_fd0 ? "closed" : "open"); + goto fail; + } + + if (nxt_slow_path(nxt_test_fd_is_open(pipe1[0]))) { + nxt_log_alert(thr->log, "port fd test: %s leaked fd[1]", name); + goto fail; + } + + if (nxt_slow_path(msg.fd[0] != -1 || msg.fd[1] != -1)) { + nxt_log_alert(thr->log, "port fd test: %s left fds in the message", + name); + goto fail; + } + + if (keep_fd0) { + /* Ours now: the dispatcher correctly left it alone. */ + (void) close(pipe0[0]); + } + + return NXT_OK; + +fail: + + nxt_port_fd_test_close(pipe0[0]); + nxt_port_fd_test_close(pipe1[0]); + + return NXT_ERROR; +} + + +/* + * Runs a first fragment carrying descriptor A and a last fragment carrying + * a different descriptor B through the same (stream, pid), and checks the + * close in the merge that runs just before msg->fd[0] is overwritten by the + * assembled message's own descriptors (fmsg->fd[0]). Without that close, + * B -- the last fragment's own descriptor, never seen by any handler -- is + * dropped on the floor when the overwrite happens. + * + * "keep_fd0" selects the same two handlers as nxt_port_fd_test_tail_close(): + * ignoring the message (A ends up closed by the tail close, since nothing + * took it) or blanking fd[0] (A is kept open). B is never handed to a + * handler at all -- it is always expected closed. + */ +static nxt_int_t +nxt_port_fd_test_frag_merge(nxt_thread_t *thr, nxt_task_t *task, + nxt_port_t *port, nxt_bool_t keep_fd0, const char *name) +{ + int pipe_a[2], pipe_b[2]; + nxt_buf_t *b; + nxt_port_recv_msg_t msg; + + if (nxt_slow_path(pipe(pipe_a) != 0 || pipe(pipe_b) != 0)) { + nxt_log_alert(thr->log, "port fd test failed to create a pipe"); + return NXT_ERROR; + } + + (void) close(pipe_a[1]); + (void) close(pipe_b[1]); + + /* First fragment: nf=0, mf=1, carries A. */ + + b = nxt_mp_zalloc(port->mem_pool, sizeof(nxt_buf_t)); + if (nxt_slow_path(b == NULL)) { + (void) close(pipe_a[0]); + (void) close(pipe_b[0]); + return NXT_ERROR; + } + + nxt_memzero(&msg, sizeof(nxt_port_recv_msg_t)); + + msg.port = port; + msg.buf = b; + msg.size = sizeof(nxt_port_msg_t); + msg.fd[0] = pipe_a[0]; + msg.fd[1] = -1; + msg.port_msg.stream = 0x46524147; /* "FRAG" -- distinct from other tests */ + msg.port_msg.pid = nxt_pid; + msg.port_msg.mf = 1; + msg.port_msg.type = keep_fd0 ? _NXT_PORT_MSG_STATUS : _NXT_PORT_MSG_OOSM; + + nxt_port_test_run_read_msg_process(task, port, &msg); + + /* Last fragment: nf=1, mf=0, carries a *different* descriptor, B. */ + + b = nxt_mp_zalloc(port->mem_pool, sizeof(nxt_buf_t)); + if (nxt_slow_path(b == NULL)) { + nxt_port_fd_test_close(pipe_a[0]); + (void) close(pipe_b[0]); + return NXT_ERROR; + } + + nxt_memzero(&msg, sizeof(nxt_port_recv_msg_t)); + + msg.port = port; + msg.buf = b; + msg.size = sizeof(nxt_port_msg_t); + msg.fd[0] = pipe_b[0]; + msg.fd[1] = -1; + msg.port_msg.stream = 0x46524147; + msg.port_msg.pid = nxt_pid; + msg.port_msg.nf = 1; + + nxt_port_test_run_read_msg_process(task, port, &msg); + + if (nxt_slow_path(keep_fd0 != nxt_test_fd_is_open(pipe_a[0]))) { + nxt_log_alert(thr->log, "port fd test: %s left A %s", name, + keep_fd0 ? "closed" : "open"); + goto fail; + } + + if (nxt_slow_path(nxt_test_fd_is_open(pipe_b[0]))) { + nxt_log_alert(thr->log, "port fd test: %s leaked B, the last " + "fragment's own descriptor", name); + goto fail; + } + + if (keep_fd0) { + /* Ours now: the handler kept it, the merge did not touch it. */ + (void) close(pipe_a[0]); + } + + return NXT_OK; + +fail: + + nxt_port_fd_test_close(pipe_a[0]); + nxt_port_fd_test_close(pipe_b[0]); + + return NXT_ERROR; +} + + +nxt_int_t +nxt_port_fd_test(nxt_thread_t *thr) +{ + nxt_mp_t *mp; + nxt_buf_t *b; + nxt_fd_t fd0, fd1; + nxt_task_t *task; + nxt_int_t ret; + nxt_pid_t port_pid; + nxt_port_t *port, *existing; + nxt_runtime_t *rt, *saved_rt; + nxt_port_recv_msg_t msg; + nxt_event_engine_t *engine, *saved_engine; + nxt_port_msg_new_port_t new_port_msg; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "port fd test started"); + + task = thr->task; + task->thread = thr; + + /* Defined before the first goto done: the cleanup there releases them. */ + port = NULL; + existing = NULL; + + /* + * Only read at "done" through port, and only ever set together with + * it (see the comment at its real assignment below); the initializer + * is just to satisfy -Wmaybe-uninitialized, which cannot see that + * pairing. + */ + port_pid = 0; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + rt = nxt_mp_zalloc(mp, sizeof(nxt_runtime_t)); + engine = nxt_mp_zalloc(mp, sizeof(nxt_event_engine_t)); + b = nxt_mp_zalloc(mp, sizeof(nxt_buf_t)); + + if (nxt_slow_path(rt == NULL || engine == NULL || b == NULL)) { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + rt->mem_pool = mp; + + if (nxt_slow_path(nxt_thread_mutex_create(&rt->processes_mutex) != NXT_OK)) + { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + saved_rt = thr->runtime; + saved_engine = thr->engine; + + thr->runtime = rt; + + /* + * nxt_port_write_enable() takes the address of a work queue inside the + * engine, so a zeroed one is enough to reach the end of the handler. + */ + thr->engine = engine; + + /* + * nxt_runtime_process_release() defers a process's teardown to + * rt->main_engine unless the calling thread is already on it, and + * compares it against thr->engine -- a mismatch posts to whatever + * rt->main_engine happens to hold, NULL here, and crashes. The fixture + * runs everything on the one engine above, so that is main_engine too. + */ + rt->main_engine = engine; + + nxt_memzero(&msg, sizeof(nxt_port_recv_msg_t)); + + /* + * The mmap handler refuses a message without a segment descriptor. The + * refusal used to return without closing the second descriptor, which + * the sender is free to attach regardless of the message type. + */ + ret = nxt_port_fd_test_closed(thr, task, &msg, nxt_port_mmap_handler, 0, + "mmap without a segment fd"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + /* + * A segment announced by a process the runtime does not know: the + * handler releases the segment descriptor on that path but used to keep + * the second one. + */ + msg.port_msg.pid = nxt_pid + 1; + + ret = nxt_port_fd_test_closed(thr, task, &msg, nxt_port_mmap_handler, 1, + "mmap from an unknown process"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + /* + * NEW_PORT naming a port that exists but has no queue yet. This is + * the normal path in the main process -- WHOAMI creates every + * application port before its NEW_PORT arrives -- so the socket is + * refused, but the queue descriptor has to reach the caller: closing + * it here would leave main's worker ports queueless and strand every + * CHANGE_FILE and QUIT main sends through them. + */ + nxt_memzero(&new_port_msg, sizeof(nxt_port_msg_new_port_t)); + + new_port_msg.pid = nxt_pid + 2; + new_port_msg.id = 7; + new_port_msg.type = NXT_PROCESS_APP; + + b->mem.pos = (u_char *) &new_port_msg; + msg.buf = b; + + /* + * A NEW_PORT body one byte short. The handler refuses it, closes both + * descriptors and creates no port. + */ + b->mem.free = b->mem.pos + sizeof(nxt_port_msg_new_port_t) - 1; + + ret = nxt_port_fd_test_closed(thr, task, &msg, nxt_port_new_port_handler, + 1, "short new port"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + if (nxt_slow_path(msg.u.new_port != NULL)) { + nxt_log_alert(thr->log, "port fd test: a short new port message " + "created a port"); + ret = NXT_ERROR; + goto done; + } + + b->mem.free = b->mem.pos + sizeof(nxt_port_msg_new_port_t); + + existing = nxt_runtime_process_port_create(task, rt, new_port_msg.pid, + new_port_msg.id, + new_port_msg.type); + if (nxt_slow_path(existing == NULL)) { + ret = NXT_ERROR; + goto done; + } + + existing->pair[0] = -1; + existing->pair[1] = -1; + existing->socket.fd = -1; + + fd0 = open("/dev/null", O_RDONLY); + fd1 = open("/dev/null", O_RDONLY); + + if (nxt_slow_path(fd0 == -1 || fd1 == -1)) { + nxt_log_alert(thr->log, "port fd test failed to open /dev/null"); + nxt_port_fd_test_close(fd0); + nxt_port_fd_test_close(fd1); + ret = NXT_ERROR; + goto done; + } + + msg.fd[0] = fd0; + msg.fd[1] = fd1; + msg.u.new_port = NULL; + + nxt_port_new_port_handler(task, &msg); + + if (nxt_slow_path(msg.u.new_port != existing)) { + nxt_log_alert(thr->log, "port fd test: an existing port was not " + "reported back to the caller"); + nxt_port_fd_test_close(fd0); + nxt_port_fd_test_close(fd1); + ret = NXT_ERROR; + goto done; + } + + if (nxt_slow_path(msg.fd[0] != -1 || nxt_test_fd_is_open(fd0))) { + nxt_log_alert(thr->log, "port fd test: an existing port did not " + "refuse the socket"); + nxt_port_fd_test_close(fd1); + ret = NXT_ERROR; + goto done; + } + + if (nxt_slow_path(msg.fd[1] != fd1 || !nxt_test_fd_is_open(fd1))) { + nxt_log_alert(thr->log, "port fd test: the queue descriptor of a " + "queueless existing port was not left to the caller"); + ret = NXT_ERROR; + goto done; + } + + nxt_port_fd_test_close(fd1); + msg.fd[1] = -1; + + /* + * NEW_PORT naming a port whose queue is already installed. Here the + * queue descriptor is refused with the socket: a caller that mapped + * fd[1] would re-point a live port's queue at memory this message + * chose. The mapping is a real one so that the cleanup below releases + * it the same way nxt_port_close() releases any port queue. + */ + existing->queue = mmap(NULL, sizeof(nxt_port_queue_t), + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANON, -1, 0); + if (nxt_slow_path(existing->queue == MAP_FAILED)) { + existing->queue = NULL; + nxt_log_alert(thr->log, "port fd test failed to map a queue"); + ret = NXT_ERROR; + goto done; + } + + msg.u.new_port = NULL; + + ret = nxt_port_fd_test_closed(thr, task, &msg, nxt_port_new_port_handler, + 1, "new port that already has a queue"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + if (nxt_slow_path(msg.u.new_port != existing)) { + nxt_log_alert(thr->log, "port fd test: an existing port was not " + "reported back to the caller"); + ret = NXT_ERROR; + goto done; + } + + /* + * The one path that keeps a descriptor: a port is created, it adopts + * the socket, and the queue descriptor is deliberately left in the + * message for the caller to map or discard. Pinned here because every + * caller now treats "fd[1] still set" as "this one is mine". + */ + new_port_msg.id = 8; + + fd0 = open("/dev/null", O_RDONLY); + fd1 = open("/dev/null", O_RDONLY); + + if (nxt_slow_path(fd0 == -1 || fd1 == -1)) { + nxt_log_alert(thr->log, "port fd test failed to open /dev/null"); + nxt_port_fd_test_close(fd0); + nxt_port_fd_test_close(fd1); + ret = NXT_ERROR; + goto done; + } + + msg.fd[0] = fd0; + msg.fd[1] = fd1; + msg.u.new_port = NULL; + + nxt_port_new_port_handler(task, &msg); + + port = msg.u.new_port; + + if (nxt_slow_path(port == NULL || port == existing)) { + nxt_log_alert(thr->log, "port fd test: a new port was not created"); + nxt_port_fd_test_close(fd0); + nxt_port_fd_test_close(fd1); + ret = NXT_ERROR; + goto done; + } + + /* + * nxt_port_enable() below, exercised by the dispatcher cases and the + * fragment tests, sets port->pid to this process's own pid -- the + * usual thing for a port a process enables for itself, but it retargets + * the pid this port was registered in rt->ports under. The teardown + * has to remove it by that original key, since a lookup by the mutated + * pid finds nothing and nxt_runtime_port_remove() would silently leak + * the port and its reference. + */ + port_pid = port->pid; + + if (nxt_slow_path(msg.fd[0] != -1 || port->pair[1] != fd0)) { + nxt_log_alert(thr->log, "port fd test: the new port did not take the " + "socket over"); + nxt_port_fd_test_close(fd1); + ret = NXT_ERROR; + goto done; + } + + if (nxt_slow_path(msg.fd[1] != fd1 || !nxt_test_fd_is_open(fd1))) { + nxt_log_alert(thr->log, "port fd test: the queue descriptor was not " + "left to the caller"); + ret = NXT_ERROR; + goto done; + } + + nxt_port_fd_test_close(fd1); + msg.fd[1] = -1; + + /* + * The dispatcher itself. The message type is a byte off the wire, so a + * peer can name one past the table -- or, just as easily, an in-range + * one whose slot this process never filled: the tables are designated + * initializers over a struct of named members, and most set only a few. + * Neither may call anything, and both have to close what the message + * carried, since no handler runs to take the descriptors. + * + * nxt_port_enable() installs the (static) dispatcher on the port; the + * fixture engine gets a no-op enable_read so that the read side of the + * enabling touches nothing real. + */ + engine->event.enable_read = nxt_port_fd_test_enable_read_stub; + + nxt_port_enable(task, port, &nxt_port_fd_test_handlers); + + msg.port = port; + + msg.port_msg.type = NXT_PORT_MSG_MAX; + + ret = nxt_port_fd_test_closed(thr, task, &msg, port->handler, 1, + "message type past the handler table"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + msg.port_msg.type = _NXT_PORT_MSG_NEW_PORT; /* slot not set above */ + + ret = nxt_port_fd_test_closed(thr, task, &msg, port->handler, 1, + "message type with an empty handler slot"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + if (nxt_slow_path(nxt_port_fd_test_dispatched != 0)) { + nxt_log_alert(thr->log, "port fd test: a refused type reached a " + "handler"); + ret = NXT_ERROR; + goto done; + } + + /* And a slot that is set still dispatches. */ + msg.port_msg.type = _NXT_PORT_MSG_QUIT; + + ret = nxt_port_fd_test_closed(thr, task, &msg, port->handler, 1, + "message type with its slot set"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + if (nxt_slow_path(nxt_port_fd_test_dispatched != 1)) { + nxt_log_alert(thr->log, "port fd test: a handled type did not reach " + "its handler"); + ret = NXT_ERROR; + goto done; + } + + /* + * nxt_port_read_msg_process() itself: the tail close that reclaims + * whatever a dispatch left in msg->fd[], and the close in the + * last-fragment merge that runs before msg->fd[0] is overwritten by the + * assembled message's own descriptors. + */ + + ret = nxt_port_fd_test_tail_close(thr, task, port, 0, + "dispatch to a handler that ignores " + "the message"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + ret = nxt_port_fd_test_tail_close(thr, task, port, 1, + "dispatch to a handler that keeps " + "fd[0]"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + ret = nxt_port_fd_test_frag_merge(thr, task, port, 0, + "fragment merge, handler ignores the " + "message"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + ret = nxt_port_fd_test_frag_merge(thr, task, port, 1, + "fragment merge, handler keeps fd[0]"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + ret = NXT_OK; + +done: + + /* + * The ports hold what their handlers adopted -- the new port its socket + * descriptor, the existing one its queue mapping -- and nothing else + * owns them here: leaving them to the pool teardown would leak both. + * + * close() releases those, but not the port: each one carries its own + * memory pool and a reference held by rt->ports, neither of which the + * nxt_mp_destroy() below can reach. remove() drops that reference, + * which is the last one, and frees the port -- so it has to happen + * while the fixture runtime is still installed. + */ + if (existing != NULL) { + nxt_port_close(task, existing); + nxt_runtime_port_remove(task, existing); + } + + if (port != NULL) { + /* + * Restore the pid it was registered under (see the comment at + * port_pid's assignment above) so the lookup inside remove() finds + * it. + */ + port->pid = port_pid; + + nxt_port_close(task, port); + nxt_runtime_port_remove(task, port); + } + + thr->runtime = saved_rt; + thr->engine = saved_engine; + + nxt_thread_mutex_destroy(&rt->processes_mutex); + nxt_mp_destroy(mp); + + if (nxt_fast_path(ret == NXT_OK)) { + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "port fd test passed"); + } + + return ret; +} diff --git a/src/test/nxt_port_frag_test.c b/src/test/nxt_port_frag_test.c new file mode 100644 index 000000000..2d2bf1a73 --- /dev/null +++ b/src/test/nxt_port_frag_test.c @@ -0,0 +1,640 @@ + +/* + * Copyright (C) FreeUnit contributors. + */ + +/* + * Fragment reassembly limits at a receiving port (#394): streams per port, + * bytes per stream and bytes per port. A stream past a limit is dropped + * whole, so its handler never runs. Fragment buffers point into a + * PROT_NONE reservation that nothing reads, so a 128 MB stream is free. + * A fragment counts for the buffer it came in, so empty fragments are + * bounded too. + * + * An mmap fragment that carries no shared memory record -- empty, a partial + * record, or one that names no segment -- is refused: its read buffer used + * to be kept in the stream and also given back to port->free_bufs. + * + * An mmap fragment of empty records keeps a buffer per record while its + * payload is next to nothing; the buffers count too, or a port's total + * admits millions of them. + */ + +#include +#include +#include +#include +#include +#include "nxt_tests.h" + +#include + + +#define NXT_FRAG_TEST_MB (1024 * 1024) +#define NXT_FRAG_TEST_STREAM 0x46524700 /* distinct from other tests */ + + +static u_char *nxt_frag_test_space; +static nxt_uint_t nxt_frag_test_calls; +static uint32_t nxt_frag_test_last_stream; +static size_t nxt_frag_test_last_size; +static nxt_bool_t nxt_frag_test_oom; + +static nxt_task_t *nxt_frag_test_task; +static nxt_port_t *nxt_frag_test_port; +static nxt_bool_t nxt_frag_test_aliased; + +static nxt_pid_t nxt_frag_test_sender; +static nxt_chunk_id_t nxt_frag_test_chunk; + + +typedef struct { + size_t size; /* of the mmap fragment's payload */ + nxt_uint_t pos; /* of the mmap fragment in a stream of three */ +} nxt_frag_test_mmap_case_t; + + +static const nxt_frag_test_mmap_case_t nxt_frag_test_mmap_cases[] = { + { 0, 0 }, { 0, 1 }, { 0, 2 }, + { 5, 0 }, { 5, 1 }, { 5, 2 }, + { sizeof(nxt_port_mmap_msg_t), 0 }, + { sizeof(nxt_port_mmap_msg_t), 1 }, + { sizeof(nxt_port_mmap_msg_t), 2 }, +}; + + +static void +nxt_frag_test_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) +{ + nxt_frag_test_calls++; + nxt_frag_test_last_stream = msg->port_msg.stream; + nxt_frag_test_last_size = msg->size; +} + + +static void +nxt_frag_test_send(nxt_task_t *task, nxt_port_t *port, uint32_t stream, + size_t size, nxt_bool_t first, nxt_bool_t last) +{ + nxt_buf_t *b; + nxt_port_recv_msg_t msg; + + b = nxt_mp_zalloc(port->mem_pool, sizeof(nxt_buf_t)); + if (nxt_slow_path(b == NULL)) { + nxt_frag_test_oom = 1; + return; + } + + b->mem.start = nxt_frag_test_space; + b->mem.pos = nxt_frag_test_space; + b->mem.free = nxt_frag_test_space; + b->mem.end = nxt_frag_test_space + size; + + nxt_memzero(&msg, sizeof(nxt_port_recv_msg_t)); + + msg.port = port; + msg.buf = b; + msg.size = sizeof(nxt_port_msg_t) + size; + msg.fd[0] = -1; + msg.fd[1] = -1; + msg.port_msg.stream = stream; + msg.port_msg.pid = nxt_pid; + msg.port_msg.type = _NXT_PORT_MSG_STATUS; + msg.port_msg.nf = !first; + msg.port_msg.mf = !last; + + nxt_port_test_run_read_msg_process(task, port, &msg); +} + + +/* A stream of "n" fragments of "mb" MB: delivered whole, or not at all. */ +static nxt_int_t +nxt_frag_test_expect(nxt_task_t *task, nxt_port_t *port, const char *name, + uint32_t stream, nxt_uint_t n, size_t mb, nxt_bool_t expect) +{ + nxt_uint_t i, calls; + + calls = nxt_frag_test_calls; + + for (i = 0; i < n; i++) { + nxt_frag_test_send(task, port, stream, mb * NXT_FRAG_TEST_MB, i == 0, + i == n - 1); + } + + NXT_TEST_CHECK(task->log, (nxt_frag_test_calls != calls) == expect, + "port frag test: %s: %s", name, + expect ? "dropped" : "delivered"); + NXT_TEST_CHECK(task->log, !expect + || (nxt_frag_test_last_stream == stream + && nxt_frag_test_last_size == n * mb * NXT_FRAG_TEST_MB), + "port frag test: %s: %uz bytes", name, + nxt_frag_test_last_size); + return NXT_OK; +} + + +static nxt_int_t +nxt_frag_test_run(nxt_task_t *task, nxt_port_t *port) +{ + size_t mb, half; + uint32_t s; + nxt_uint_t i, n, calls; + + s = NXT_FRAG_TEST_STREAM; + mb = NXT_PORT_FRAG_SIZE_MAX / NXT_FRAG_TEST_MB; + + /* Per stream: at the limit; past it on the last or a middle fragment. */ + + if (nxt_frag_test_expect(task, port, "at the size limit", s++, mb, 1, 1) + || nxt_frag_test_expect(task, port, "past it, last", s++, mb + 1, 1, 0) + || nxt_frag_test_expect(task, port, "past it, middle", s++, mb + 2, 1, + 0) + || nxt_frag_test_expect(task, port, "after a drop", s++, 3, 1, 1)) + { + return NXT_ERROR; + } + + /* + * Per port, streams: open the most there may be, never end them, then + * two more. Each new stream drops the oldest open one, so streams that + * are never ended cannot block the port. + */ + + for (i = 0; i < NXT_PORT_FRAG_STREAMS_MAX; i++) { + nxt_frag_test_send(task, port, s + i, 1, 1, 0); + } + + calls = nxt_frag_test_calls; + + nxt_frag_test_send(task, port, s + NXT_PORT_FRAG_STREAMS_MAX, 1, 1, 0); + nxt_frag_test_send(task, port, s + NXT_PORT_FRAG_STREAMS_MAX + 1, 1, 1, 0); + + NXT_TEST_CHECK(task->log, + port->frag_streams == NXT_PORT_FRAG_STREAMS_MAX, + "port frag test: %uD streams open after two evictions", + port->frag_streams); + + for (i = 0; i < 2; i++) { + nxt_frag_test_send(task, port, s + NXT_PORT_FRAG_STREAMS_MAX + i, 1, + 0, 1); + + NXT_TEST_CHECK(task->log, nxt_frag_test_calls == calls + i + 1 + && nxt_frag_test_last_stream + == s + NXT_PORT_FRAG_STREAMS_MAX + i, + "port frag test: a stream past the most was refused"); + } + + /* The two oldest are gone: their last fragments find no stream. */ + + calls = nxt_frag_test_calls; + + nxt_frag_test_send(task, port, s, 1, 0, 1); + nxt_frag_test_send(task, port, s + 1, 1, 0, 1); + + NXT_TEST_CHECK(task->log, nxt_frag_test_calls == calls, + "port frag test: an evicted stream completed"); + + /* The others still complete, in any order, and free their places. */ + + for (i = NXT_PORT_FRAG_STREAMS_MAX; i-- > 2; /* void */) { + nxt_frag_test_send(task, port, s + i, 1, 0, 1); + } + + NXT_TEST_CHECK(task->log, + nxt_frag_test_calls == calls + NXT_PORT_FRAG_STREAMS_MAX - 2 + && port->frag_streams == 0 && port->frag_size == 0, + "port frag test: %uD of the open streams completed, " + "%uD left", nxt_frag_test_calls - calls, + port->frag_streams); + + s += NXT_PORT_FRAG_STREAMS_MAX + 2; + + /* Per port, bytes: two open halves fill it; a third stream is dropped. */ + + half = NXT_PORT_FRAG_TOTAL_MAX / 2 / NXT_FRAG_TEST_MB * NXT_FRAG_TEST_MB; + + nxt_frag_test_send(task, port, s, half, 1, 0); + nxt_frag_test_send(task, port, s + 1, half, 1, 0); + + if (nxt_frag_test_expect(task, port, "past the port's total", s + 2, 2, + 1, 0)) + { + return NXT_ERROR; + } + + calls = nxt_frag_test_calls; + + nxt_frag_test_send(task, port, s, 0, 0, 1); + nxt_frag_test_send(task, port, s + 1, 0, 0, 1); + + NXT_TEST_CHECK(task->log, nxt_frag_test_calls == calls + 2, + "port frag test: streams within the total did not " + "complete"); + + /* Everything is accounted back: a fresh stream at the limit passes. */ + + if (nxt_frag_test_expect(task, port, "at the limit, again", s + 3, mb, 1, + 1)) + { + return NXT_ERROR; + } + + /* + * Empty fragments: each one holds a buffer of port->max_size, so the + * port's total admits NXT_PORT_FRAG_TOTAL_MAX / max_size of them. + */ + + port->max_size = 16 * 1024; + n = NXT_PORT_FRAG_TOTAL_MAX / port->max_size; + + if (nxt_frag_test_expect(task, port, "empty fragments up to the total", + s + 4, n + 1, 0, 1) + || nxt_frag_test_expect(task, port, "empty fragments past the total", + s + 5, n + 2, 0, 0)) + { + return NXT_ERROR; + } + + NXT_TEST_CHECK(task->log, !nxt_frag_test_oom, + "port frag test: out of memory"); + return NXT_OK; +} + + +/* Would the assembled message hand over a buffer the port already reuses? */ +static void +nxt_frag_test_mmap_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) +{ + nxt_buf_t *b, *f; + + nxt_frag_test_calls++; + + for (b = msg->buf; b != NULL; b = b->next) { + for (f = msg->port->free_bufs; f != NULL; f = f->next) { + if (f == b) { + nxt_frag_test_aliased = 1; + } + } + } +} + + +/* + * Sends one fragment the way nxt_port_read_handler() receives it: in a + * buffer of its own, given back to port->free_bufs afterwards unless the + * dispatch took it. + */ +static nxt_int_t +nxt_frag_test_mmap_send(nxt_port_t *port, const u_char *data, size_t size, + nxt_bool_t mmap, nxt_bool_t first, nxt_bool_t last) +{ + nxt_buf_t *b; + nxt_port_recv_msg_t msg; + + b = nxt_buf_mem_alloc(port->mem_pool, 64, 0); + if (b == NULL) { + return NXT_ERROR; + } + + nxt_memcpy(b->mem.start, data, size); + + nxt_memzero(&msg, sizeof(nxt_port_recv_msg_t)); + + msg.port = port; + msg.buf = b; + msg.size = sizeof(nxt_port_msg_t) + size; + msg.fd[0] = -1; + msg.fd[1] = -1; + msg.port_msg.stream = NXT_FRAG_TEST_STREAM - 1; + msg.port_msg.pid = nxt_pid; + msg.port_msg.type = _NXT_PORT_MSG_STATUS; + msg.port_msg.nf = !first; + msg.port_msg.mf = !last; + msg.port_msg.mmap = mmap; + + nxt_port_test_run_read_msg_process(nxt_frag_test_task, port, &msg); + + if (msg.buf == b) { + b->next = port->free_bufs; + port->free_bufs = b; + } + + return NXT_OK; +} + + +/* In a child: the last position used to complete plain buffers as mmap. */ +static int +nxt_frag_test_mmap_child(void *data) +{ + u_char payload[64]; + uint32_t streams, size; + nxt_uint_t i; + nxt_port_t *port; + nxt_port_mmap_msg_t rec; + const nxt_frag_test_mmap_case_t *tc; + + tc = data; + port = nxt_frag_test_port; + + /* A whole record, for a segment the sender never shared. */ + rec.mmap_id = 0; + rec.chunk_id = 0; + rec.size = 100; + + nxt_memset(payload, 'x', sizeof(payload)); + nxt_memcpy(payload, &rec, sizeof(rec)); + + port->handler = nxt_frag_test_mmap_handler; + nxt_frag_test_calls = 0; + nxt_frag_test_aliased = 0; + + streams = port->frag_streams; + size = port->frag_size; + + for (i = 0; i < 3; i++) { + if (nxt_frag_test_mmap_send(port, payload, + (i == tc->pos) ? tc->size : 16, + i == tc->pos, i == 0, i == 2) + != NXT_OK) + { + return 8; + } + } + + /* 1: delivered; 2: with a buffer the port reuses; 4: not given back. */ + return (nxt_frag_test_calls != 0) + | (nxt_frag_test_aliased << 1) + | ((port->frag_streams != streams || port->frag_size != size) << 2); +} + + +static nxt_int_t +nxt_frag_test_mmap_run(nxt_thread_t *thr, nxt_port_t *port) +{ + int status; + nxt_int_t ret; + nxt_uint_t i; + nxt_runtime_t *rt, *saved_rt; + + /* A runtime that knows no sender, so no record names a segment. */ + rt = nxt_mp_zalloc(port->mem_pool, sizeof(nxt_runtime_t)); + if (rt == NULL + || nxt_thread_mutex_create(&rt->processes_mutex) != NXT_OK) + { + return NXT_ERROR; + } + + saved_rt = thr->runtime; + thr->runtime = rt; + + nxt_frag_test_task = thr->task; + nxt_frag_test_port = port; + + ret = NXT_OK; + + for (i = 0; i < nxt_nitems(nxt_frag_test_mmap_cases); i++) { + status = nxt_test_in_child(thr, "port frag test", + nxt_frag_test_mmap_child, + (void *) &nxt_frag_test_mmap_cases[i]); + if (status != 0) { + nxt_log_alert(thr->log, "port frag test: an mmap fragment of " + "%uz bytes at position %ui was not refused (%d)", + nxt_frag_test_mmap_cases[i].size, + nxt_frag_test_mmap_cases[i].pos, status); + ret = NXT_ERROR; + } + } + + thr->runtime = saved_rt; + + nxt_thread_mutex_destroy(&rt->processes_mutex); + + return ret; +} + + +/* + * In a child: with the port's total all taken but "room", open a stream of + * mmap fragments, each a max_size read buffer full of empty records for a + * segment the sender did share, and send until the stream is dropped. + * Every record keeps a buffer of at least sizeof(nxt_buf_t), so what the + * stream held must fit in the room it had. + */ +static int +nxt_frag_test_records_child(void *data) +{ + size_t room, nrecs; + uint32_t streams; + nxt_buf_t *b; + nxt_uint_t i, j, n; + nxt_port_t *port; + nxt_work_queue_t wq; + nxt_port_mmap_msg_t *rec; + nxt_port_recv_msg_t msg; + nxt_work_queue_cache_t cache; + + port = nxt_frag_test_port; + port->handler = nxt_frag_test_handler; + port->max_size = 16 * 1024; + + /* Where a dropped stream's shared memory buffers complete. */ + nxt_memzero(&wq, sizeof(nxt_work_queue_t)); + nxt_work_queue_cache_create(&cache, 1024); + wq.cache = &cache; + port->socket.read_work_queue = &wq; + + nrecs = port->max_size / sizeof(nxt_port_mmap_msg_t); + room = 64 * port->max_size; + n = room / port->max_size + 2; + + streams = port->frag_streams; + port->frag_size = NXT_PORT_FRAG_TOTAL_MAX - room; + + for (i = 0; i < n; i++) { + b = nxt_buf_mem_alloc(port->mem_pool, port->max_size, 0); + if (b == NULL) { + return 8; + } + + rec = (nxt_port_mmap_msg_t *) b->mem.start; + + for (j = 0; j < nrecs; j++) { + rec[j].mmap_id = 0; + rec[j].chunk_id = nxt_frag_test_chunk; + rec[j].size = 0; + } + + nxt_memzero(&msg, sizeof(nxt_port_recv_msg_t)); + + msg.port = port; + msg.buf = b; + msg.size = sizeof(nxt_port_msg_t) + nrecs * sizeof(*rec); + msg.fd[0] = -1; + msg.fd[1] = -1; + msg.port_msg.stream = NXT_FRAG_TEST_STREAM - 2; + msg.port_msg.pid = nxt_frag_test_sender; + msg.port_msg.type = _NXT_PORT_MSG_STATUS; + msg.port_msg.nf = (i != 0); + msg.port_msg.mf = 1; + msg.port_msg.mmap = 1; + + nxt_port_test_run_read_msg_process(nxt_frag_test_task, port, &msg); + + if (msg.buf == b) { + b->next = port->free_bufs; + port->free_bufs = b; + } + + if (port->frag_streams == streams) { + break; + } + } + + /* 1: held more than its room; 2: never dropped; 4: not given back. */ + return (i * nrecs * sizeof(nxt_buf_t) > room) + | ((i == n) << 1) + | ((port->frag_size != NXT_PORT_FRAG_TOTAL_MAX - room) << 2); +} + + +/* + * The runtime knows one sender, with a segment in its incoming array, so + * nxt_port_mmap_read() turns each of its records into a buffer. + */ +static nxt_int_t +nxt_frag_test_records_run(nxt_thread_t *thr, nxt_port_t *port) +{ + int status; + nxt_buf_t *seg; + nxt_task_t *task; + nxt_process_t *process; + nxt_runtime_t *rt, *saved_rt; + nxt_event_engine_t engine, *saved_engine; + nxt_port_mmap_handler_t *mmap_handler; + + task = thr->task; + + rt = nxt_mp_zalloc(port->mem_pool, sizeof(nxt_runtime_t)); + process = nxt_mp_zalloc(port->mem_pool, sizeof(nxt_process_t)); + + if (rt == NULL || process == NULL + || nxt_thread_mutex_create(&rt->processes_mutex) != NXT_OK) + { + return NXT_ERROR; + } + + rt->mem_pool = port->mem_pool; + + nxt_memzero(&engine, sizeof(engine)); + engine.mem_pool = port->mem_pool; + engine.task.thread = thr; + engine.task.log = thr->log; + + saved_rt = thr->runtime; + saved_engine = thr->engine; + thr->runtime = rt; + thr->engine = &engine; + + status = -1; + + process->pid = nxt_pid + 43; + process->use_count = 1; + nxt_queue_init(&process->ports); + + if (nxt_thread_mutex_create(&process->incoming.mutex) != NXT_OK) { + goto done; + } + + nxt_runtime_process_add(task, process); + + seg = nxt_port_mmap_get_buf(task, &process->incoming, + PORT_MMAP_CHUNK_SIZE); + if (seg == NULL) { + goto done; + } + + mmap_handler = seg->parent; + + nxt_frag_test_task = task; + nxt_frag_test_port = port; + nxt_frag_test_sender = process->pid; + nxt_frag_test_chunk = nxt_port_mmap_chunk_id(mmap_handler->hdr, + seg->mem.pos); + + status = nxt_test_in_child(thr, "port frag test", + nxt_frag_test_records_child, NULL); + if (status != 0) { + nxt_log_alert(thr->log, "port frag test: mmap fragments of empty " + "records passed the port's total (%d)", status); + } + +done: + + /* The segment stays mapped, like other port fixtures. */ + + thr->runtime = saved_rt; + thr->engine = saved_engine; + + return (status == 0) ? NXT_OK : NXT_ERROR; +} + + +nxt_int_t +nxt_port_frag_test(nxt_thread_t *thr) +{ + size_t reserve; + nxt_int_t ret; + nxt_task_t *task; + nxt_port_t *port; + + nxt_thread_time_update(thr); + + task = thr->task; + task->thread = thr; + + reserve = NXT_PORT_FRAG_TOTAL_MAX + 16 * NXT_FRAG_TEST_MB; + + nxt_frag_test_space = mmap(NULL, reserve, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, + -1, 0); + if (nxt_frag_test_space == MAP_FAILED) { + nxt_log_alert(thr->log, "port frag test: mmap() failed %E", nxt_errno); + return NXT_ERROR; + } + + port = nxt_port_new(task, 1, nxt_pid, NXT_PROCESS_ROUTER); + if (nxt_slow_path(port == NULL)) { + (void) munmap(nxt_frag_test_space, reserve); + return NXT_ERROR; + } + + port->pair[0] = -1; + port->pair[1] = -1; + port->socket.fd = -1; + port->handler = nxt_frag_test_handler; + + ret = nxt_frag_test_run(task, port); + + if (ret == NXT_OK) { + ret = nxt_frag_test_mmap_run(thr, port); + } + + if (ret == NXT_OK) { + ret = nxt_frag_test_records_run(thr, port); + } + + /* + * The port's pool holds everything the streams allocated, the buffers + * on port->free_bufs included; nothing else refers to the port. + */ + nxt_port_use(task, port, -1); + + (void) munmap(nxt_frag_test_space, reserve); + + if (ret == NXT_OK) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, "port frag test passed"); + } + + return ret; +} diff --git a/src/test/nxt_port_mmap_read_test.c b/src/test/nxt_port_mmap_read_test.c new file mode 100644 index 000000000..1e16344ee --- /dev/null +++ b/src/test/nxt_port_mmap_read_test.c @@ -0,0 +1,327 @@ +/* + * Copyright (C) FreeUnit contributors. + */ + +/* + * nxt_port_mmap_read() (src/nxt_port_memory.c) turns an untrusted peer's + * array of nxt_port_mmap_msg_t into buffers over its shared memory. It + * used to read a partial last record past mem.free. The records sit at the + * end of a page followed by a PROT_NONE page, so an overrun faults in every + * build; each case runs in a child. The other rows pin the per-field checks. + */ + +#include +#include +#include +#include +#include +#include "nxt_tests.h" + +#include + + +typedef struct { + const char *name; + nxt_pid_t pid_off; /* 1: an unknown sender */ + nxt_uint_t nrecords; + size_t tail; + nxt_uint_t expect_bufs; + nxt_port_mmap_msg_t records[2]; +} nxt_mmap_read_case_t; + + +static const nxt_mmap_read_case_t nxt_mmap_read_cases[] = { + { "one record", 0, 1, 0, 1, { { 0, 0, 100 } } }, + { "two records", 0, 2, 0, 2, + { { 0, 0, 100 }, { 0, 1, PORT_MMAP_CHUNK_SIZE } } }, + { "partial tail only", 0, 0, 5, 0, { { 0 } } }, + { "unknown mmap_id", 0, 1, 0, 0, { { 1, 0, 100 } } }, + { "huge mmap_id", 0, 1, 0, 0, { { 0xFFFFFFFF, 0, 100 } } }, + { "chunk_id past the data area", 0, 1, 0, 0, + { { 0, PORT_MMAP_CHUNK_COUNT, 100 } } }, + { "size past the data area", 0, 1, 0, 0, + { { 0, PORT_MMAP_CHUNK_COUNT - 1, PORT_MMAP_CHUNK_SIZE + 1 } } }, + { "size near UINT32_MAX", 0, 1, 0, 0, { { 0, 0, 0xFFFFFFFF } } }, + { "unknown sender", 1, 1, 0, 0, { { 0, 0, 100 } } }, +}; + + +static nxt_task_t *nxt_mmap_read_task; +static nxt_port_t *nxt_mmap_read_port; + + +static int +nxt_mmap_read_child(void *data) +{ + u_char *page, *start, *end; + size_t psize, len; + nxt_uint_t i, nbufs; + nxt_buf_t *b, *in; + nxt_port_recv_msg_t msg; + const nxt_mmap_read_case_t *tc; + + tc = data; + psize = getpagesize(); + + page = mmap(NULL, 2 * psize, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (page == MAP_FAILED || mprotect(page + psize, psize, PROT_NONE) != 0) { + return 1; + } + + len = tc->nrecords * sizeof(nxt_port_mmap_msg_t); + end = page + psize; + start = end - len - tc->tail; + + /* A zero tail reads as mmap_id 0, which exists, so the overrun is real. */ + nxt_memcpy(start, tc->records, len); + nxt_memzero(start + len, tc->tail); + + in = nxt_buf_mem_alloc(nxt_mmap_read_port->mem_pool, 0, 0); + if (in == NULL) { + return 1; + } + + in->mem.start = start; + in->mem.pos = start; + in->mem.free = end; + in->mem.end = end; + + nxt_memzero(&msg, sizeof(nxt_port_recv_msg_t)); + msg.port = nxt_mmap_read_port; + msg.buf = in; + msg.port_msg.pid = nxt_pid + 41 + tc->pid_off; + msg.port_msg.mmap = 1; + + nxt_port_mmap_read(nxt_mmap_read_task, &msg); + + nbufs = 0; + + for (b = msg.buf; b != NULL && b != in; b = b->next) { + nbufs++; + } + + len = 0; + + for (i = 0; i < tc->expect_bufs; i++) { + len += tc->records[i].size; + } + + return nbufs != tc->expect_bufs || msg.size != len + || (size_t) (in->mem.pos - start) + != tc->expect_bufs * sizeof(nxt_port_mmap_msg_t); +} + + +nxt_int_t +nxt_port_mmap_read_test(nxt_thread_t *thr) +{ + nxt_mp_t *mp; + nxt_int_t ret; + nxt_buf_t *seg, *segs[3]; + nxt_uint_t i, nsegs; + nxt_bool_t incoming_mutex; + nxt_task_t *task; + nxt_process_t *process; + nxt_runtime_t *rt, *saved_rt; + nxt_chunk_id_t c; + nxt_event_engine_t engine, *saved_engine; + nxt_mmap_read_case_t tc; + nxt_port_mmap_handler_t *mmap_handler; + + nxt_thread_time_update(thr); + + task = thr->task; + task->thread = thr; + nxt_mmap_read_task = task; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + rt = nxt_mp_zalloc(mp, sizeof(nxt_runtime_t)); + process = nxt_mp_zalloc(mp, sizeof(nxt_process_t)); + + if (nxt_slow_path(rt == NULL || process == NULL + || nxt_thread_mutex_create(&rt->processes_mutex) + != NXT_OK)) + { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + rt->mem_pool = mp; + + nxt_memzero(&engine, sizeof(engine)); + engine.mem_pool = mp; + engine.task.thread = thr; + engine.task.log = thr->log; + + saved_rt = thr->runtime; + saved_engine = thr->engine; + thr->runtime = rt; + thr->engine = &engine; + + ret = NXT_ERROR; + nsegs = 0; + incoming_mutex = 0; + + process->pid = nxt_pid + 41; + process->use_count = 1; + nxt_queue_init(&process->ports); + + if (nxt_slow_path(nxt_thread_mutex_create(&process->incoming.mutex) + != NXT_OK)) + { + goto done; + } + + incoming_mutex = 1; + + nxt_runtime_process_add(task, process); + + /* The sender's segment 0, allocated through the same array. */ + seg = nxt_port_mmap_get_buf(task, &process->incoming, + 2 * PORT_MMAP_CHUNK_SIZE); + if (nxt_slow_path(seg == NULL)) { + goto done; + } + + segs[nsegs++] = seg; + + mmap_handler = seg->parent; + c = nxt_port_mmap_chunk_id(mmap_handler->hdr, seg->mem.pos); + + nxt_mmap_read_port = nxt_port_new(task, 1, nxt_pid, NXT_PROCESS_ROUTER); + if (nxt_slow_path(nxt_mmap_read_port == NULL)) { + goto done; + } + + nxt_mmap_read_port->pair[0] = -1; + nxt_mmap_read_port->pair[1] = -1; + nxt_mmap_read_port->socket.fd = -1; + + for (i = 0; i < nxt_nitems(nxt_mmap_read_cases); i++) { + if (nxt_test_in_child(thr, nxt_mmap_read_cases[i].name, + nxt_mmap_read_child, + (void *) &nxt_mmap_read_cases[i]) + != 0) + { + nxt_log_alert(thr->log, "port mmap read test \"%s\" failed", + nxt_mmap_read_cases[i].name); + goto done; + } + } + + /* A whole record followed by every possible partial tail. */ + + tc = nxt_mmap_read_cases[0]; + + for (tc.tail = 1; tc.tail < sizeof(nxt_port_mmap_msg_t); tc.tail++) { + if (nxt_test_in_child(thr, "partial tail", nxt_mmap_read_child, &tc) + != 0) + { + nxt_log_alert(thr->log, "port mmap read test: a %uz-byte tail " + "failed", tc.tail); + goto done; + } + } + + /* + * The peer maps the segment writable, so the busy sentinel past the + * last chunk may read as free: nxt_port_mmap_increase_buf() must still + * not grow a buffer past the data area. + */ + seg = nxt_port_mmap_get_buf(task, &process->incoming, + (PORT_MMAP_CHUNK_COUNT - c - 2) + * PORT_MMAP_CHUNK_SIZE); + if (seg != NULL) { + segs[nsegs++] = seg; + } + + if (nxt_slow_path(seg == NULL || seg->parent != mmap_handler)) { + goto done; + } + + for (i = 0; i < 4; i++) { + nxt_port_mmap_set_chunk_free(mmap_handler->hdr->free_map, + PORT_MMAP_CHUNK_COUNT + i); + } + + (void) nxt_port_mmap_increase_buf(task, seg, + nxt_buf_mem_free_size(&seg->mem) + + 4 * PORT_MMAP_CHUNK_SIZE, 1); + + if (seg->mem.end > nxt_port_mmap_chunk_start(mmap_handler->hdr, + PORT_MMAP_CHUNK_COUNT)) + { + nxt_log_alert(thr->log, "port mmap read test: increase_buf grew " + "past the data area"); + goto done; + } + + /* Nor may nxt_port_mmap_get() continue from the last chunk past it. */ + nxt_port_mmap_set_chunk_free(mmap_handler->hdr->free_map, + PORT_MMAP_CHUNK_COUNT - 1); + + seg = nxt_port_mmap_get_buf(task, &process->incoming, + 2 * PORT_MMAP_CHUNK_SIZE); + if (seg != NULL) { + segs[nsegs++] = seg; + } + + if (seg != NULL && seg->parent == mmap_handler + && seg->mem.end > nxt_port_mmap_chunk_start(mmap_handler->hdr, + PORT_MMAP_CHUNK_COUNT)) + { + nxt_log_alert(thr->log, "port mmap read test: get_buf ran past " + "the data area"); + goto done; + } + + ret = NXT_OK; + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "port mmap read test passed"); + +done: + + /* + * Each nxt_port_mmap_get_buf() took a reference on its segment's + * handler, and process->incoming holds one more. The buffers are + * never completed here, so their references are dropped by hand + * (nxt_port_mmap_handler_use() is private to nxt_port_memory.c); the + * array's reference, dropped last by nxt_port_mmaps_destroy(), then + * unmaps and frees each handler. The buffers themselves + * come from mp. The port has its own pool, freed by the last + * reference; it was never linked to a process. nxt_runtime_process_add() + * put the process in rt->processes, a malloc'd table outside mp. + */ + for (i = 0; i < nsegs; i++) { + mmap_handler = segs[i]->parent; + (void) nxt_atomic_fetch_add(&mmap_handler->use_count, -1); + } + + nxt_port_mmaps_destroy(&process->incoming, 1); + + if (nxt_mmap_read_port != NULL) { + nxt_port_use(task, nxt_mmap_read_port, -1); + nxt_mmap_read_port = NULL; + } + + if (process->registered) { + nxt_runtime_process_remove(rt, process); + } + + if (incoming_mutex) { + nxt_thread_mutex_destroy(&process->incoming.mutex); + } + + thr->runtime = saved_rt; + thr->engine = saved_engine; + + nxt_thread_mutex_destroy(&rt->processes_mutex); + nxt_mp_destroy(mp); + + return ret; +} diff --git a/src/test/nxt_port_mmaps_max_test.c b/src/test/nxt_port_mmaps_max_test.c new file mode 100644 index 000000000..61c65d7f0 --- /dev/null +++ b/src/test/nxt_port_mmaps_max_test.c @@ -0,0 +1,138 @@ + +/* + * Copyright (C) FreeUnit contributors. + */ + +/* + * NXT_PORT_MMAPS_MAX on the router side (src/nxt_port_memory.c). The + * array of segments is grown to hold the id it is asked for, and an + * incoming id is the peer's. nxt_port_mmap_at() used to grow the array for + * any id below UINT32_MAX, so one message with mmap_id 100000000 cost an + * allocation and initialisation of that many slots. Now the last id below + * the limit is accepted, the limit and everything past it are refused + * before any growth, and nxt_port_new_port_mmap() does not create a segment + * whose id would reach the limit. + * + * Cost: the boundary cases grow the array to NXT_PORT_MMAPS_MAX slots of + * one pointer, 512 KiB, and create one real segment of PORT_MMAP_SIZE + * (about 10 MiB) so that the last allowed id is really created. The + * segment is a sparse memfd or shm file and only its header page is + * touched, so the test takes well under a millisecond and no measurable + * RSS; it needs 10 MiB of /dev/shm on paper only. + */ + +#include +#include +#include +#include "nxt_tests.h" + + +nxt_int_t +nxt_port_mmaps_max_test(nxt_thread_t *thr) +{ + nxt_int_t ret; + nxt_task_t *task; + nxt_port_mmap_t *port_mmap; + nxt_port_mmaps_t mmaps; + nxt_port_mmap_handler_t *mmap_handler; + + nxt_thread_time_update(thr); + + task = thr->task; + task->thread = thr; + + nxt_memzero(&mmaps, sizeof(mmaps)); + + if (nxt_slow_path(nxt_thread_mutex_create(&mmaps.mutex) != NXT_OK)) { + return NXT_ERROR; + } + + ret = NXT_ERROR; + + /* A huge id is refused before the array is touched. */ + + port_mmap = nxt_port_test_mmap_at(&mmaps, 100000000); + + if (port_mmap != NULL || mmaps.elts != NULL || mmaps.cap != 0 + || mmaps.size != 0) + { + nxt_log_alert(thr->log, "port mmaps max test: id 100000000 was " + "accepted, or the array was grown for it"); + goto done; + } + + port_mmap = nxt_port_test_mmap_at(&mmaps, NXT_PORT_MMAPS_MAX); + + if (port_mmap != NULL || mmaps.elts != NULL || mmaps.cap != 0) { + nxt_log_alert(thr->log, "port mmaps max test: id %uD was accepted", + NXT_PORT_MMAPS_MAX); + goto done; + } + + port_mmap = nxt_port_test_mmap_at(&mmaps, UINT32_MAX); + + if (port_mmap != NULL || mmaps.elts != NULL || mmaps.cap != 0) { + nxt_log_alert(thr->log, "port mmaps max test: id UINT32_MAX was " + "accepted"); + goto done; + } + + /* An id just below the limit grows the array to MAX - 1 slots. */ + + port_mmap = nxt_port_test_mmap_at(&mmaps, NXT_PORT_MMAPS_MAX - 2); + + if (port_mmap == NULL || mmaps.size != NXT_PORT_MMAPS_MAX - 1 + || port_mmap != mmaps.elts + NXT_PORT_MMAPS_MAX - 2 + || port_mmap->mmap_handler != NULL) + { + nxt_log_alert(thr->log, "port mmaps max test: id %uD was refused", + NXT_PORT_MMAPS_MAX - 2); + goto done; + } + + /* + * The segment created now gets id NXT_PORT_MMAPS_MAX - 1, the last one + * allowed. + */ + + mmap_handler = nxt_port_test_new_port_mmap(task, &mmaps, 1); + + if (mmap_handler == NULL || mmaps.size != NXT_PORT_MMAPS_MAX + || mmap_handler->hdr->id != NXT_PORT_MMAPS_MAX - 1 + || mmaps.elts[NXT_PORT_MMAPS_MAX - 1].mmap_handler != mmap_handler) + { + nxt_log_alert(thr->log, "port mmaps max test: segment %uD was not " + "created", NXT_PORT_MMAPS_MAX - 1); + goto done; + } + + /* The next one would get the limit as its id: refused, nothing grows. */ + + mmap_handler = nxt_port_test_new_port_mmap(task, &mmaps, 1); + + if (mmap_handler != NULL || mmaps.size != NXT_PORT_MMAPS_MAX) { + nxt_log_alert(thr->log, "port mmaps max test: segment %uD was " + "created", NXT_PORT_MMAPS_MAX); + goto done; + } + + port_mmap = nxt_port_test_mmap_at(&mmaps, NXT_PORT_MMAPS_MAX); + + if (port_mmap != NULL || mmaps.size != NXT_PORT_MMAPS_MAX) { + nxt_log_alert(thr->log, "port mmaps max test: id %uD was accepted " + "into a full array", NXT_PORT_MMAPS_MAX); + goto done; + } + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "port mmaps max test passed"); + + ret = NXT_OK; + +done: + + nxt_port_mmaps_destroy(&mmaps, 1); + + nxt_thread_mutex_destroy(&mmaps.mutex); + + return ret; +} diff --git a/src/test/nxt_port_queued_fd_test.c b/src/test/nxt_port_queued_fd_test.c new file mode 100644 index 000000000..383a5b042 --- /dev/null +++ b/src/test/nxt_port_queued_fd_test.c @@ -0,0 +1,1077 @@ +/* + * Copyright (C) F5, Inc. + */ + +/* + * Regression test for descriptor ownership of a queued port message + * (nxt_port_msg_alloc() in src/nxt_port_socket.c, #388). + * + * nxt_port_send_port() (src/nxt_port.c) hands another port's pair[1] and + * queue_fd to nxt_port_socket_write2() as bare numbers, without + * NXT_PORT_MSG_CLOSE_FD: the message borrows them. When the socket is not + * writable the message is copied to the heap and queued, and the sendmsg() + * that puts the descriptors into SCM_RIGHTS runs later. If the owner closes + * them first, the deferred send names a closed number -- or, once the kernel + * has reused it, an unrelated descriptor of this process. + * + * Two legs. The first inspects the queued copy: it must own duplicates, + * not the caller's numbers, and dropping it through the error handler must + * close exactly those duplicates. The second drives the race the issue + * describes end to end over a real socketpair: queue, close the originals, + * reopen something else at the same number, then let the write handler + * send. The peer must receive the descriptor that was queued, not the + * one that took its number. + * + * A third leg covers the bound that ownership made necessary + * (NXT_PORT_MAX_FD_MSGS, freeunitorg/freeunit#394). A peer that stops + * reading holds two of this process's descriptors per queued message, so + * the number of descriptor-carrying entries is capped. The leg fills a + * port to the cap, checks that the next descriptor-carrying send is + * refused with nothing consumed, that a send with no descriptor is still + * accepted, and that nothing is leaked either way. + */ + +#include +#include +#include +#include +#include "nxt_tests.h" + +#include +#include +#include +#include + + +static nxt_int_t nxt_port_queued_fd_test_owned(nxt_thread_t *thr); +static nxt_int_t nxt_port_queued_fd_test_reused(nxt_thread_t *thr); +static nxt_int_t nxt_port_queued_fd_test_bounded(nxt_thread_t *thr); +static nxt_port_t *nxt_port_queued_fd_test_port(nxt_task_t *task, + nxt_event_engine_t *engine); +static void nxt_port_queued_fd_test_stub(nxt_event_engine_t *engine, + nxt_fd_event_t *ev); +static void nxt_port_queued_fd_test_completion(nxt_task_t *task, void *obj, + void *data); +static void nxt_port_queued_fd_test_drain_wq(nxt_work_queue_t *wq); +static nxt_bool_t nxt_port_queued_fd_test_same_file(nxt_fd_t a, + const struct stat *b); +static nxt_fd_t nxt_port_queued_fd_test_recv_fd(nxt_fd_t sock); +static nxt_uint_t nxt_port_queued_fd_test_open_fds(void); +static nxt_uint_t nxt_port_queued_fd_test_queued(nxt_port_t *port); +static void nxt_cdecl nxt_port_queued_fd_test_log(nxt_uint_t level, + nxt_log_t *log, const char *fmt, ...); + + +static nxt_uint_t nxt_port_queued_fd_test_completions; +static nxt_uint_t nxt_port_queued_fd_test_alerts; +static nxt_log_t *nxt_port_queued_fd_test_saved_log; + + +nxt_int_t +nxt_port_queued_fd_test(nxt_thread_t *thr) +{ + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "port queued fd test started"); + + if (nxt_port_queued_fd_test_owned(thr) != NXT_OK) { + return NXT_ERROR; + } + + if (nxt_port_queued_fd_test_reused(thr) != NXT_OK) { + return NXT_ERROR; + } + + if (nxt_port_queued_fd_test_bounded(thr) != NXT_OK) { + return NXT_ERROR; + } + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "port queued fd test passed"); + + return NXT_OK; +} + + +static void +nxt_port_queued_fd_test_stub(nxt_event_engine_t *engine, nxt_fd_event_t *ev) +{ + /* The fixture engine polls nothing. */ +} + + +/* + * A port on the caller's engine with a stub event interface, so that the + * write path's re-arm and block calls land on nothing. socket.write_ready + * is left clear: nxt_port_msg_chk_insert() then queues every message. + */ + +static nxt_port_t * +nxt_port_queued_fd_test_port(nxt_task_t *task, nxt_event_engine_t *engine) +{ + nxt_port_t *port; + + port = nxt_port_new(task, 1, nxt_pid, NXT_PROCESS_MAIN); + if (nxt_slow_path(port == NULL)) { + return NULL; + } + + port->pair[0] = -1; + port->pair[1] = -1; + port->socket.fd = -1; + port->socket.task = task; + port->socket.log = task->log; + port->socket.write_ready = 0; + port->socket.write = NXT_EVENT_INACTIVE; + port->engine = engine; + port->max_size = 1024; + port->max_share = 1024; + + port->socket.write_work_queue = &engine->fast_work_queue; + + return port; +} + + +static nxt_int_t +nxt_port_queued_fd_test_owned(nxt_thread_t *thr) +{ + nxt_mp_t *mp; + nxt_fd_t fd0, fd1; + nxt_buf_t *buf; + nxt_int_t ret; + nxt_task_t *task; + nxt_port_t *port; + struct stat st0, st1; + nxt_event_engine_t engine, *saved_engine; + nxt_port_send_msg_t *msg; + nxt_event_interface_t stub; + + task = thr->task; + task->thread = thr; + + ret = NXT_ERROR; + fd0 = -1; + fd1 = -1; + + nxt_memzero(&engine, sizeof(engine)); + nxt_memzero(&stub, sizeof(stub)); + + nxt_work_queue_cache_create(&engine.work_queue_cache, 1024); + engine.fast_work_queue.cache = &engine.work_queue_cache; + nxt_work_queue_name(&engine.fast_work_queue, "fast"); + + stub.enable_write = nxt_port_queued_fd_test_stub; + stub.block_write = nxt_port_queued_fd_test_stub; + engine.event = stub; + + saved_engine = thr->engine; + thr->engine = &engine; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + thr->engine = saved_engine; + return NXT_ERROR; + } + + port = nxt_port_queued_fd_test_port(task, &engine); + if (nxt_slow_path(port == NULL)) { + nxt_mp_destroy(mp); + thr->engine = saved_engine; + return NXT_ERROR; + } + + fd0 = open("/dev/null", O_RDONLY); + fd1 = open("/dev/zero", O_RDONLY); + + if (fd0 == -1 || fd1 == -1 || fstat(fd0, &st0) != 0 + || fstat(fd1, &st1) != 0) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: failed to open /dev/null"); + goto done; + } + + buf = nxt_buf_mem_alloc(mp, 1, 0); + if (nxt_slow_path(buf == NULL)) { + goto done; + } + + buf->completion_handler = nxt_port_queued_fd_test_completion; + nxt_port_queued_fd_test_completions = 0; + + /* The shape of nxt_port_send_port(): two borrowed descriptors. */ + + if (nxt_port_socket_write2(task, port, NXT_PORT_MSG_NEW_PORT, fd0, fd1, + 0, 0, buf) + != NXT_OK) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: write was not queued"); + goto done; + } + + if (nxt_queue_is_empty(&port->messages)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: nothing queued"); + goto done; + } + + msg = nxt_queue_link_data(nxt_queue_first(&port->messages), + nxt_port_send_msg_t, link); + + if (!msg->close_fd) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: the queued message does not own " + "its descriptors"); + goto done; + } + + if (msg->fd[0] == fd0 || msg->fd[1] == fd1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: the queued message names the " + "caller's descriptors %FD %FD", msg->fd[0], msg->fd[1]); + goto done; + } + + if (!nxt_port_queued_fd_test_same_file(msg->fd[0], &st0) + || !nxt_port_queued_fd_test_same_file(msg->fd[1], &st1)) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: the queued duplicates name " + "other files"); + goto done; + } + + /* The owner closes; the queued copy must be unaffected. */ + + nxt_fd_close(fd0); + nxt_fd_close(fd1); + + if (!nxt_test_fd_is_open(msg->fd[0]) || !nxt_test_fd_is_open(msg->fd[1])) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: closing the originals closed " + "the queued copy"); + fd0 = -1; + fd1 = -1; + goto done; + } + + fd0 = msg->fd[0]; + fd1 = msg->fd[1]; + + /* Drop the queue: the duplicates go with it, and nothing else does. */ + + nxt_port_test_run_error_handler(task, port); + + nxt_port_queued_fd_test_drain_wq(&engine.fast_work_queue); + + if (nxt_test_fd_is_open(fd0) || nxt_test_fd_is_open(fd1)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: dropping the queue leaked the " + "duplicates"); + goto done; + } + + fd0 = -1; + fd1 = -1; + + if (nxt_port_queued_fd_test_completions != 1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: %ui completions, expected 1", + nxt_port_queued_fd_test_completions); + goto done; + } + + ret = NXT_OK; + +done: + + if (fd0 != -1 && nxt_test_fd_is_open(fd0)) { + nxt_fd_close(fd0); + } + + if (fd1 != -1 && nxt_test_fd_is_open(fd1)) { + nxt_fd_close(fd1); + } + + if (!nxt_queue_is_empty(&port->messages)) { + nxt_port_test_run_error_handler(task, port); + nxt_port_queued_fd_test_drain_wq(&engine.fast_work_queue); + } + + nxt_port_use(task, port, -1); + nxt_mp_destroy(mp); + + nxt_work_queue_cache_destroy(&engine.work_queue_cache); + thr->engine = saved_engine; + + return ret; +} + + +/* + * The race itself: the owner closes after the message was queued, the + * number is taken by something else, then the send goes out. + */ + +static nxt_int_t +nxt_port_queued_fd_test_reused(nxt_thread_t *thr) +{ + nxt_mp_t *mp; + nxt_fd_t fd, decoy, got, pair[2]; + nxt_buf_t *buf; + nxt_int_t ret; + nxt_task_t *task; + nxt_port_t *port; + struct stat st, dst; + nxt_event_engine_t engine, *saved_engine; + nxt_event_interface_t stub; + + task = thr->task; + task->thread = thr; + + ret = NXT_ERROR; + fd = -1; + got = -1; + decoy = -1; + pair[0] = -1; + pair[1] = -1; + + nxt_memzero(&engine, sizeof(engine)); + nxt_memzero(&stub, sizeof(stub)); + + nxt_work_queue_cache_create(&engine.work_queue_cache, 1024); + engine.fast_work_queue.cache = &engine.work_queue_cache; + nxt_work_queue_name(&engine.fast_work_queue, "fast"); + + stub.enable_write = nxt_port_queued_fd_test_stub; + stub.block_write = nxt_port_queued_fd_test_stub; + engine.event = stub; + + saved_engine = thr->engine; + thr->engine = &engine; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + thr->engine = saved_engine; + return NXT_ERROR; + } + + port = nxt_port_queued_fd_test_port(task, &engine); + if (nxt_slow_path(port == NULL)) { + nxt_mp_destroy(mp); + thr->engine = saved_engine; + return NXT_ERROR; + } + + /* SOCK_DGRAM: the type src/nxt_socketpair.c selects. */ + + if (nxt_slow_path(socketpair(AF_UNIX, SOCK_DGRAM, 0, pair) != 0)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: socketpair failed"); + goto done; + } + + if (nxt_slow_path(fcntl(pair[1], F_SETFL, O_NONBLOCK) == -1)) { + goto done; + } + + port->pair[0] = pair[0]; + port->pair[1] = pair[1]; + + /* Installs the real write and error handlers; then not writable yet. */ + + nxt_port_write_enable(task, port); + + port->socket.write_ready = 0; + port->socket.write = NXT_EVENT_INACTIVE; + + fd = open("/dev/null", O_RDONLY); + if (fd == -1 || fstat(fd, &st) != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: failed to open /dev/null"); + goto done; + } + + buf = nxt_buf_mem_alloc(mp, 1, 0); + if (nxt_slow_path(buf == NULL)) { + goto done; + } + + buf->completion_handler = nxt_port_queued_fd_test_completion; + nxt_port_queued_fd_test_completions = 0; + + /* write_ready is clear: queued, not sent. */ + + if (nxt_port_socket_write2(task, port, NXT_PORT_MSG_NEW_PORT, fd, -1, + 0, 0, buf) + != NXT_OK + || nxt_queue_is_empty(&port->messages)) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: write was not queued"); + goto done; + } + + /* + * The owner goes away, and the lowest free number -- the one just + * closed -- is taken by an unrelated file before the queue drains. + */ + + nxt_fd_close(fd); + + decoy = open("/dev/zero", O_RDONLY); + if (decoy == -1 || fstat(decoy, &dst) != 0) { + fd = -1; + goto done; + } + + if (decoy != fd) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: the closed number %FD was not " + "reused (%FD), cannot stage the race", fd, decoy); + fd = -1; + goto done; + } + + fd = -1; + + /* Now the socket is writable and the queue drains. */ + + port->socket.write_ready = 1; + + port->socket.write_handler(task, &port->socket, NULL); + + nxt_port_queued_fd_test_drain_wq(&engine.fast_work_queue); + + got = nxt_port_queued_fd_test_recv_fd(pair[0]); + if (got == -1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: the peer received no descriptor"); + goto done; + } + + if (nxt_port_queued_fd_test_same_file(got, &dst)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: the peer received the descriptor " + "that reused the number, not the one that was queued"); + goto done; + } + + if (!nxt_port_queued_fd_test_same_file(got, &st)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: the peer received an unexpected " + "descriptor"); + goto done; + } + + if (!nxt_queue_is_empty(&port->messages)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: the message was sent but stayed " + "queued"); + goto done; + } + + if (nxt_port_queued_fd_test_completions != 1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: %ui completions, expected 1", + nxt_port_queued_fd_test_completions); + goto done; + } + + ret = NXT_OK; + +done: + + if (fd != -1) { + nxt_fd_close(fd); + } + + if (got != -1) { + nxt_fd_close(got); + } + + if (decoy != -1) { + nxt_fd_close(decoy); + } + + if (!nxt_queue_is_empty(&port->messages)) { + nxt_port_test_run_error_handler(task, port); + nxt_port_queued_fd_test_drain_wq(&engine.fast_work_queue); + } + + port->pair[0] = -1; + port->pair[1] = -1; + port->socket.fd = -1; + + if (pair[0] != -1) { + nxt_fd_close(pair[0]); + } + + if (pair[1] != -1) { + nxt_fd_close(pair[1]); + } + + nxt_port_use(task, port, -1); + nxt_mp_destroy(mp); + + nxt_work_queue_cache_destroy(&engine.work_queue_cache); + thr->engine = saved_engine; + + return ret; +} + + +/* + * The bound on descriptor-carrying entries. + * + * The port is not writable, so every send queues, and each queued copy dups + * what it was given: the open descriptor count rises by one per message and + * is the measurement the leg makes. Nothing here reaches inside the port to + * read a counter -- the bound is observed the way a caller observes it, from + * the answer to a send and from what the process still holds open. + */ + +static nxt_int_t +nxt_port_queued_fd_test_bounded(nxt_thread_t *thr) +{ + nxt_mp_t *mp; + nxt_uint_t i, base, filled; + nxt_fd_t fd, got, pair[2]; + nxt_buf_t *buf; + nxt_int_t ret; + nxt_log_t log, *saved_log; + struct stat st; + struct rlimit rlmt; + nxt_task_t *task; + nxt_port_t *port; + nxt_event_engine_t engine, *saved_engine; + nxt_event_interface_t stub; + + task = thr->task; + task->thread = thr; + + /* + * The fill holds NXT_PORT_MAX_FD_MSGS duplicates open at once, and the + * leg needs a few more of its own. Under a lower RLIMIT_NOFILE a dup + * fails before the bound is reached, which says nothing about the + * bound, so the leg is skipped rather than failed. + */ + + if (getrlimit(RLIMIT_NOFILE, &rlmt) == 0 + && rlmt.rlim_cur != RLIM_INFINITY + && rlmt.rlim_cur < nxt_port_queued_fd_test_open_fds() + + NXT_PORT_MAX_FD_MSGS + 16) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: bound leg skipped, " + "RLIMIT_NOFILE %d is too low for %d queued " + "descriptors", (int) rlmt.rlim_cur, + NXT_PORT_MAX_FD_MSGS); + return NXT_OK; + } + + saved_log = task->log; + + ret = NXT_ERROR; + fd = -1; + got = -1; + pair[0] = -1; + pair[1] = -1; + + nxt_memzero(&engine, sizeof(engine)); + nxt_memzero(&stub, sizeof(stub)); + + nxt_work_queue_cache_create(&engine.work_queue_cache, 1024); + engine.fast_work_queue.cache = &engine.work_queue_cache; + nxt_work_queue_name(&engine.fast_work_queue, "fast"); + + stub.enable_write = nxt_port_queued_fd_test_stub; + stub.block_write = nxt_port_queued_fd_test_stub; + engine.event = stub; + + saved_engine = thr->engine; + thr->engine = &engine; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + thr->engine = saved_engine; + return NXT_ERROR; + } + + port = nxt_port_queued_fd_test_port(task, &engine); + if (nxt_slow_path(port == NULL)) { + nxt_mp_destroy(mp); + thr->engine = saved_engine; + return NXT_ERROR; + } + + if (nxt_slow_path(socketpair(AF_UNIX, SOCK_DGRAM, 0, pair) != 0)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: socketpair failed"); + goto done; + } + + if (nxt_slow_path(fcntl(pair[1], F_SETFL, O_NONBLOCK) == -1)) { + goto done; + } + + port->pair[0] = pair[0]; + port->pair[1] = pair[1]; + + nxt_port_write_enable(task, port); + + port->socket.write_ready = 0; + port->socket.write = NXT_EVENT_INACTIVE; + + fd = open("/dev/null", O_RDONLY); + if (fd == -1 || fstat(fd, &st) != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: failed to open /dev/null"); + goto done; + } + + nxt_port_queued_fd_test_completions = 0; + + base = nxt_port_queued_fd_test_open_fds(); + + /* Fill the queue to the bound. Every one of these must be taken. */ + + for (i = 0; i < NXT_PORT_MAX_FD_MSGS; i++) { + if (nxt_port_socket_write2(task, port, NXT_PORT_MSG_NEW_PORT, fd, -1, + i, 0, NULL) + != NXT_OK) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: message %ui of %d was " + "refused below the bound", i, NXT_PORT_MAX_FD_MSGS); + goto done; + } + } + + filled = nxt_port_queued_fd_test_queued(port); + + if (filled != NXT_PORT_MAX_FD_MSGS) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: %ui messages queued, expected %d", + filled, NXT_PORT_MAX_FD_MSGS); + goto done; + } + + if (nxt_port_queued_fd_test_open_fds() != base + NXT_PORT_MAX_FD_MSGS) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: %ui descriptors open after the " + "fill, expected %ui", + nxt_port_queued_fd_test_open_fds(), + base + NXT_PORT_MAX_FD_MSGS); + goto done; + } + + /* + * One more, with a payload. It must be refused, and refused whole: the + * descriptor is still the caller's and open, the buffer is still the + * caller's and uncompleted, and the queue is as it was. + */ + + buf = nxt_buf_mem_alloc(mp, 1, 0); + if (nxt_slow_path(buf == NULL)) { + goto done; + } + + buf->completion_handler = nxt_port_queued_fd_test_completion; + + /* + * The refusals below are counted by the alerts they log: a stalled peer + * keeps the port at the bound, so only the first one may log. + */ + + log = *saved_log; + log.handler = nxt_port_queued_fd_test_log; + + nxt_port_queued_fd_test_saved_log = saved_log; + nxt_port_queued_fd_test_alerts = 0; + + task->log = &log; + + if (nxt_port_socket_write2(task, port, NXT_PORT_MSG_NEW_PORT, fd, -1, + NXT_PORT_MAX_FD_MSGS, 0, buf) + != NXT_ERROR) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: a message past the bound of %d " + "was accepted", NXT_PORT_MAX_FD_MSGS); + goto done; + } + + if (nxt_port_socket_write2(task, port, NXT_PORT_MSG_NEW_PORT, fd, -1, + NXT_PORT_MAX_FD_MSGS, 0, NULL) + != NXT_ERROR) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: a second message past the bound " + "was accepted"); + goto done; + } + + if (nxt_port_queued_fd_test_alerts != 1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: two refusals in a row logged %ui " + "alerts, expected 1", nxt_port_queued_fd_test_alerts); + goto done; + } + + nxt_port_queued_fd_test_drain_wq(&engine.fast_work_queue); + + if (nxt_port_queued_fd_test_completions != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: the refused message completed " + "%ui buffers, expected 0", + nxt_port_queued_fd_test_completions); + goto done; + } + + if (!nxt_test_fd_is_open(fd)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: the refused message closed the " + "caller's descriptor"); + fd = -1; + goto done; + } + + if (nxt_port_queued_fd_test_queued(port) != NXT_PORT_MAX_FD_MSGS + || nxt_port_queued_fd_test_open_fds() != base + NXT_PORT_MAX_FD_MSGS) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: the refused message changed the " + "queue"); + goto done; + } + + /* + * A message with no descriptor is not what the bound is about, and is + * still taken on the same full port. + */ + + if (nxt_port_socket_write2(task, port, NXT_PORT_MSG_DATA, -1, -1, + NXT_PORT_MAX_FD_MSGS + 1, 0, buf) + != NXT_OK) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: a message with no descriptor was " + "refused on a port at the bound"); + goto done; + } + + if (nxt_port_queued_fd_test_queued(port) != NXT_PORT_MAX_FD_MSGS + 1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: the message with no descriptor " + "was not queued"); + goto done; + } + + /* + * Cancelling a queued message releases its place under the bound as + * well: the port takes one more descriptor, and refusing the one after + * that is a new stall, so it logs again. + */ + + if (nxt_port_socket_cancel(task, port, NXT_PORT_MSG_NEW_PORT, 0, 0, NULL) + != NXT_PORT_MSG_CANCELLED) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: the first queued message could " + "not be cancelled"); + goto done; + } + + nxt_port_queued_fd_test_drain_wq(&engine.fast_work_queue); + + if (nxt_port_queued_fd_test_open_fds() != base + NXT_PORT_MAX_FD_MSGS - 1) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: %ui descriptors open after the " + "cancel, expected %ui", + nxt_port_queued_fd_test_open_fds(), + base + NXT_PORT_MAX_FD_MSGS - 1); + goto done; + } + + if (nxt_port_socket_write2(task, port, NXT_PORT_MSG_NEW_PORT, fd, -1, + NXT_PORT_MAX_FD_MSGS + 2, 0, NULL) + != NXT_OK) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: a cancelled message did not " + "release its place under the bound"); + goto done; + } + + if (nxt_port_socket_write2(task, port, NXT_PORT_MSG_NEW_PORT, fd, -1, + NXT_PORT_MAX_FD_MSGS + 3, 0, NULL) + != NXT_ERROR + || nxt_port_queued_fd_test_alerts != 2) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: the port at the bound again did " + "not refuse and log once more (%ui alerts)", + nxt_port_queued_fd_test_alerts); + goto done; + } + + task->log = saved_log; + + /* + * Draining is what releases the bound. The peer gets what was queued + * below it, and the duplicates go with the messages: the descriptor + * count comes back to where it started. + */ + + port->socket.write_ready = 1; + + port->socket.write_handler(task, &port->socket, NULL); + + nxt_port_queued_fd_test_drain_wq(&engine.fast_work_queue); + + got = nxt_port_queued_fd_test_recv_fd(pair[0]); + + if (got == -1 || !nxt_port_queued_fd_test_same_file(got, &st)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: the peer did not receive the " + "first message queued below the bound"); + goto done; + } + + nxt_fd_close(got); + got = -1; + + /* Whatever the socket would not take is dropped the ordinary way. */ + + if (!nxt_queue_is_empty(&port->messages)) { + nxt_port_test_run_error_handler(task, port); + nxt_port_queued_fd_test_drain_wq(&engine.fast_work_queue); + } + + while ((got = nxt_port_queued_fd_test_recv_fd(pair[0])) != -1) { + nxt_fd_close(got); + } + + if (nxt_port_queued_fd_test_open_fds() != base) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: %ui descriptors open after the " + "queue drained, expected the %ui it started with", + nxt_port_queued_fd_test_open_fds(), base); + goto done; + } + + /* And the port takes descriptor-carrying messages again. */ + + if (nxt_port_socket_write2(task, port, NXT_PORT_MSG_NEW_PORT, fd, -1, + 0, 0, NULL) + != NXT_OK) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: the drained port still refuses a " + "descriptor"); + goto done; + } + + ret = NXT_OK; + +done: + + task->log = saved_log; + + if (fd != -1 && nxt_test_fd_is_open(fd)) { + nxt_fd_close(fd); + } + + if (got != -1) { + nxt_fd_close(got); + } + + if (!nxt_queue_is_empty(&port->messages)) { + nxt_port_test_run_error_handler(task, port); + nxt_port_queued_fd_test_drain_wq(&engine.fast_work_queue); + } + + port->pair[0] = -1; + port->pair[1] = -1; + port->socket.fd = -1; + + if (pair[0] != -1) { + nxt_fd_close(pair[0]); + } + + if (pair[1] != -1) { + nxt_fd_close(pair[1]); + } + + nxt_port_use(task, port, -1); + nxt_mp_destroy(mp); + + nxt_work_queue_cache_destroy(&engine.work_queue_cache); + thr->engine = saved_engine; + + return ret; +} + + +/* + * How many descriptors this process holds open. Counted by probing numbers + * rather than by reading /proc, which is not there on every platform the C + * suite builds on. + */ + +static nxt_uint_t +nxt_port_queued_fd_test_open_fds(void) +{ + nxt_fd_t fd; + nxt_uint_t n, limit; + struct rlimit rlmt; + + limit = 4096; + + if (getrlimit(RLIMIT_NOFILE, &rlmt) == 0 + && rlmt.rlim_cur != RLIM_INFINITY + && rlmt.rlim_cur < limit) + { + limit = rlmt.rlim_cur; + } + + n = 0; + + for (fd = 0; fd < (nxt_fd_t) limit; fd++) { + if (nxt_test_fd_is_open(fd)) { + n++; + } + } + + return n; +} + + +/* + * Counts the alerts and passes every line on to the log the leg replaced, + * so the refusal still shows in the output. + */ + +static void nxt_cdecl +nxt_port_queued_fd_test_log(nxt_uint_t level, nxt_log_t *log, + const char *fmt, ...) +{ + u_char *p; + va_list args; + nxt_log_t *saved; + u_char msg[NXT_MAX_ERROR_STR]; + + if (level == NXT_LOG_ALERT) { + nxt_port_queued_fd_test_alerts++; + } + + va_start(args, fmt); + p = nxt_vsprintf(msg, msg + NXT_MAX_ERROR_STR - 1, fmt, args); + va_end(args); + + *p = '\0'; + + saved = nxt_port_queued_fd_test_saved_log; + + saved->handler(level, saved, "%s", msg); +} + + +static nxt_uint_t +nxt_port_queued_fd_test_queued(nxt_port_t *port) +{ + nxt_uint_t n; + nxt_queue_link_t *lnk; + + n = 0; + + for (lnk = nxt_queue_first(&port->messages); + lnk != nxt_queue_tail(&port->messages); + lnk = nxt_queue_next(lnk)) + { + n++; + } + + return n; +} + + +static void +nxt_port_queued_fd_test_completion(nxt_task_t *task, void *obj, void *data) +{ + nxt_port_queued_fd_test_completions++; +} + + +static void +nxt_port_queued_fd_test_drain_wq(nxt_work_queue_t *wq) +{ + void *obj, *data; + nxt_task_t *t; + nxt_work_handler_t handler; + + while (wq->head != NULL) { + handler = nxt_work_queue_pop(wq, &t, &obj, &data); + handler(t, obj, data); + } +} + + +static nxt_bool_t +nxt_port_queued_fd_test_same_file(nxt_fd_t a, const struct stat *b) +{ + struct stat st; + + if (a == -1 || fstat(a, &st) != 0) { + return 0; + } + + return st.st_dev == b->st_dev && st.st_ino == b->st_ino + && st.st_rdev == b->st_rdev; +} + + +/* One datagram with its SCM_RIGHTS payload; -1 if none arrived. */ + +static nxt_fd_t +nxt_port_queued_fd_test_recv_fd(nxt_fd_t sock) +{ + ssize_t n; + nxt_fd_t fd; + struct iovec iov; + struct msghdr mh; + struct cmsghdr *cm; + u_char payload[256]; + union { + struct cmsghdr align; + u_char space[CMSG_SPACE(sizeof(int) * 2)]; + } cbuf; + + iov.iov_base = payload; + iov.iov_len = sizeof(payload); + + nxt_memzero(&mh, sizeof(mh)); + mh.msg_iov = &iov; + mh.msg_iovlen = 1; + mh.msg_control = &cbuf; + mh.msg_controllen = sizeof(cbuf); + + n = recvmsg(sock, &mh, MSG_DONTWAIT); + if (n < (ssize_t) sizeof(nxt_port_msg_t)) { + return -1; + } + + fd = -1; + + for (cm = CMSG_FIRSTHDR(&mh); cm != NULL; cm = NXT_CMSG_NXTHDR(&mh, cm)) { + if (cm->cmsg_level == SOL_SOCKET && cm->cmsg_type == SCM_RIGHTS) { + nxt_memcpy(&fd, CMSG_DATA(cm), sizeof(int)); + break; + } + } + + return fd; +} diff --git a/src/test/nxt_port_ready_test.c b/src/test/nxt_port_ready_test.c new file mode 100644 index 000000000..618aefc9a --- /dev/null +++ b/src/test/nxt_port_ready_test.c @@ -0,0 +1,1760 @@ +/* + * Copyright (C) F5, Inc. + */ + +/* + * Regression test for descriptor ownership in + * nxt_port_process_ready_handler() (src/nxt_port.c). + * + * PROCESS_READY normally carries the queue descriptor of the process that + * announces itself, and the handler hands it to that process's port. A + * message naming a pid the runtime does not know took an early return that + * closed nothing, and a repeated READY re-ran the mapping and dropped the + * previous one -- both leak a descriptor of the receiver, which is the + * main or the prototype process. The dispatcher does not reclaim fds once + * the handler returns (see nxt_port_recv_msg_close_fds()), so a peer that + * attaches descriptors to forged messages can exhaust a privileged + * process's descriptor table. + * + * A message can carry two descriptors regardless of how many its type + * normally uses, so the test attaches both and expects both back. + * + * The handler also authenticates the sender against the isolated_pid of + * the process the message names, wherever the platform hands the receiver + * a kernel-validated pid (NXT_USE_CMSG_PID). The fixture models a + * pid-isolated process -- pid and isolated_pid deliberately differ -- so + * that the cases below can tell the two apart: a credential equal to the + * global pid has to be refused just like a foreign one. + * + * The last case covers what the handler does with process->stream, the start + * RPC the initiator armed: it is retired when the announcement that answers + * that RPC has gone out, and kept when it has not. See issue #271. + */ + +#include +#include +#include +#include +#include +#include "nxt_tests.h" + +#include +#include +#include +#include + +#if (NXT_HAVE_MEMFD_CREATE && NXT_HAVE_LINUX_MEMFD_H) +#include +#endif +#if (NXT_HAVE_MEMFD_CREATE) +#include +#endif + + +static nxt_int_t +nxt_port_ready_test_case(nxt_thread_t *thr, nxt_task_t *task, + nxt_port_recv_msg_t *msg, const char *name) +{ + nxt_fd_t fd0, fd1; + + fd0 = open("/dev/null", O_RDONLY); + fd1 = open("/dev/null", O_RDONLY); + + if (fd0 == -1 || fd1 == -1) { + nxt_log_alert(thr->log, "port ready test failed to open /dev/null"); + goto fail; + } + + msg->fd[0] = fd0; + msg->fd[1] = fd1; + + nxt_port_process_ready_handler(task, msg); + + if (nxt_test_fd_is_open(fd0) + || nxt_test_fd_is_open(fd1)) + { + nxt_log_alert(thr->log, "port ready test: %s leaked a descriptor", + name); + goto fail; /* closes only what is still open */ + } + + if (msg->fd[0] != -1 || msg->fd[1] != -1) { + nxt_log_alert(thr->log, "port ready test: %s left fds in the message", + name); + return NXT_ERROR; + } + + return NXT_OK; + +fail: + + /* + * Close only what the handler left open: the descriptors it consumed + * are gone already, and closing them again could reach an unrelated + * descriptor that reused the number. + */ + if (fd0 != -1 && nxt_test_fd_is_open(fd0)) { + (void) close(fd0); + } + + if (fd1 != -1 && nxt_test_fd_is_open(fd1)) { + (void) close(fd1); + } + + return NXT_ERROR; +} + + +#if (NXT_USE_CMSG_PID) + +/* + * A PROCESS_READY whose kernel-validated sender pid does not match the + * isolated_pid of the process it names has to be refused outright: the + * descriptors it carries are closed, the process keeps the state it had, + * and the port keeps the queue it had. + * + * The message carries a mappable queue descriptor wherever one can be + * made, so that a handler without the gate is caught installing it rather + * than merely caught failing to map it -- which is what a /dev/null + * descriptor would produce, and would pass these checks on a queue that is + * already installed. + */ +static nxt_int_t +nxt_port_ready_test_reject(nxt_thread_t *thr, nxt_task_t *task, + nxt_port_recv_msg_t *msg, nxt_process_t *process, nxt_port_t *port, + nxt_pid_t cmsg_pid, const char *name) +{ + void *prev_queue; + nxt_fd_t fd0, fd1, prev_fd; + nxt_pid_t saved_cmsg_pid; + nxt_process_state_t prev_state; + +#if (NXT_HAVE_MEMFD_CREATE) + fd0 = syscall(SYS_memfd_create, "nxt_port_ready_test", MFD_CLOEXEC); + + if (fd0 != -1 && ftruncate(fd0, sizeof(nxt_port_queue_t)) == -1) { + (void) close(fd0); + fd0 = -1; + } +#else + fd0 = open("/dev/null", O_RDONLY); +#endif + + fd1 = open("/dev/null", O_RDONLY); + + if (fd0 == -1 || fd1 == -1) { + nxt_log_alert(thr->log, "port ready test: %s failed to open the " + "descriptors", name); + goto fail; + } + + prev_state = process->state; + prev_queue = port->queue; + prev_fd = port->queue_fd; + + saved_cmsg_pid = msg->cmsg_pid; + msg->cmsg_pid = cmsg_pid; + + msg->fd[0] = fd0; + msg->fd[1] = fd1; + + nxt_port_process_ready_handler(task, msg); + + msg->cmsg_pid = saved_cmsg_pid; + + /* + * The state and the queue are checked before the descriptors: a + * handler that accepted the message owns fd0 through the port, and + * the cleanup below would close it a second time. + */ + if (process->state != prev_state) { + nxt_log_alert(thr->log, "port ready test: %s changed the process " + "state", name); + return NXT_ERROR; + } + + if (port->queue != prev_queue || port->queue_fd != prev_fd) { + nxt_log_alert(thr->log, "port ready test: %s replaced the queue", + name); + return NXT_ERROR; + } + + if (msg->fd[0] != -1 || msg->fd[1] != -1) { + nxt_log_alert(thr->log, "port ready test: %s left fds in the message", + name); + return NXT_ERROR; + } + + if (nxt_test_fd_is_open(fd0) + || nxt_test_fd_is_open(fd1)) + { + nxt_log_alert(thr->log, "port ready test: %s leaked a descriptor", + name); + goto fail; /* closes only what is still open */ + } + + return NXT_OK; + +fail: + + if (fd0 != -1 && nxt_test_fd_is_open(fd0)) { + (void) close(fd0); + } + + if (fd1 != -1 && nxt_test_fd_is_open(fd1)) { + (void) close(fd1); + } + + return NXT_ERROR; +} + +#endif + + +#if (NXT_HAVE_MEMFD_CREATE) + +/* + * The size check in nxt_port_queue_mmap() (src/nxt_port.c) refuses only an + * object shorter than the queue, and both edges of that rule are load + * bearing. + * + * An object of exactly sizeof(nxt_port_queue_t) is what both producers ask + * for -- nxt_shm_open() in the runtime and nxt_unit_shm_open() in libunit + * -- so the comparison has to be "<": a "<=" would refuse every queue. + * + * A longer object is accepted on purpose. A shm object is rounded up to a + * page on some systems and the queue is not a whole number of pages, so + * tightening the check to equality would refuse every legitimate queue + * there and stall application port setup. Only the first + * sizeof(nxt_port_queue_t) bytes are mapped either way. + * + * Both cases touch the last item of the mapping the way + * nxt_port_queue_send() would. That probe is not a SIGBUS check: every page + * of an accepted mapping is at least partially file-backed here, and a + * partial page at EOF is zero-filled, so no size the helper accepts can + * fault there. What it catches is the helper handing back a short or + * garbage mapping -- and it fixes the shape of the access, so that a future + * change to what gets mapped is exercised rather than merely returned. The + * teeth of these two cases are the NULL checks above it. + */ +static nxt_int_t +nxt_port_ready_test_queue_size(nxt_thread_t *thr, nxt_task_t *task, + size_t file_size, const char *name) +{ + void *mem; + nxt_fd_t fd; + volatile nxt_port_queue_t *queue; + + fd = syscall(SYS_memfd_create, "nxt_port_ready_test", MFD_CLOEXEC); + if (fd == -1) { + nxt_log_alert(thr->log, "port ready test failed to create the queue"); + return NXT_ERROR; + } + + if (ftruncate(fd, file_size) == -1) { + nxt_log_alert(thr->log, "port ready test failed to size the queue"); + (void) close(fd); + return NXT_ERROR; + } + + mem = nxt_port_queue_mmap(task, fd, sizeof(nxt_port_queue_t)); + + /* The mapping does not keep the descriptor. */ + (void) close(fd); + + if (mem == NULL) { + nxt_log_alert(thr->log, "port ready test: %s queue was refused", name); + return NXT_ERROR; + } + + queue = mem; + queue->items[NXT_PORT_QUEUE_SIZE - 1].size = 0; + + nxt_mem_munmap(mem, sizeof(nxt_port_queue_t)); + + return NXT_OK; +} + + +static nxt_int_t +nxt_port_ready_test_queue_sizes(nxt_thread_t *thr, nxt_task_t *task) +{ + size_t rounded; + + if (nxt_slow_path(nxt_port_ready_test_queue_size(thr, task, + sizeof(nxt_port_queue_t), + "an exact-size") + != NXT_OK)) + { + return NXT_ERROR; + } + + rounded = nxt_align_size(sizeof(nxt_port_queue_t), nxt_pagesize); + + return nxt_port_ready_test_queue_size(thr, task, rounded, + "a page-rounded"); +} + + +/* + * Hand the handler a mappable descriptor and check that the port takes it + * over: the queue is installed, whatever the port held before is released, + * and the second descriptor of the message is closed. + */ +static nxt_int_t +nxt_port_ready_test_queue(nxt_thread_t *thr, nxt_task_t *task, + nxt_port_recv_msg_t *msg, nxt_port_t *port, const char *name) +{ + void *prev_queue; + nxt_fd_t fd, spare, prev_fd; + + /* + * The syscall form, as nxt_unit_shm_open() and nxt_unit_port_recv_test.c + * use: auto/shmem probes SYS_memfd_create, so a libc without the wrapper + * still defines NXT_HAVE_MEMFD_CREATE and this file has to link. + */ + fd = syscall(SYS_memfd_create, "nxt_port_ready_test", MFD_CLOEXEC); + spare = open("/dev/null", O_RDONLY); + + if (fd == -1 || spare == -1) { + nxt_log_alert(thr->log, "port ready test failed to create the queue"); + goto fail; + } + + if (ftruncate(fd, sizeof(nxt_port_queue_t)) == -1) { + nxt_log_alert(thr->log, "port ready test failed to size the queue"); + goto fail; + } + + prev_fd = port->queue_fd; + prev_queue = port->queue; + + msg->fd[0] = fd; + msg->fd[1] = spare; + + nxt_port_process_ready_handler(task, msg); + + if (port->queue_fd != fd || port->queue == NULL + || port->queue == MAP_FAILED) + { + nxt_log_alert(thr->log, "port ready test: %s did not install the " + "queue", name); + return NXT_ERROR; + } + + if (msg->fd[0] != -1 || msg->fd[1] != -1) { + nxt_log_alert(thr->log, "port ready test: %s left fds in the message", + name); + return NXT_ERROR; + } + + if (nxt_test_fd_is_open(spare)) { + nxt_log_alert(thr->log, "port ready test: %s leaked the spare " + "descriptor", name); + return NXT_ERROR; + } + + if (prev_queue != NULL && nxt_test_fd_is_open(prev_fd)) { + nxt_log_alert(thr->log, "port ready test: %s did not release the " + "previous queue", name); + return NXT_ERROR; + } + + return NXT_OK; + +fail: + + if (fd != -1) { + (void) close(fd); + } + + if (spare != -1) { + (void) close(spare); + } + + return NXT_ERROR; +} + + +/* + * Hand the handler a descriptor too short for the queue. mmap() succeeds + * on it, so nothing fails until the first access past the object's last + * page, which raises SIGBUS in the process that mapped it -- main or the + * prototype. The handler has to refuse the descriptor and keep the queue + * it already had. + */ +static nxt_int_t +nxt_port_ready_test_short_queue(nxt_thread_t *thr, nxt_task_t *task, + nxt_port_recv_msg_t *msg, nxt_port_t *port) +{ + void *prev_queue; + nxt_fd_t fd, prev_fd; + volatile nxt_port_queue_t *queue; + + fd = syscall(SYS_memfd_create, "nxt_port_ready_test", MFD_CLOEXEC); + if (fd == -1) { + nxt_log_alert(thr->log, "port ready test failed to create the queue"); + return NXT_ERROR; + } + + /* + * One page: mmap() of the whole queue over it succeeds, and every byte + * beyond it faults. + */ + if (ftruncate(fd, nxt_pagesize) == -1) { + nxt_log_alert(thr->log, "port ready test failed to size the queue"); + (void) close(fd); + return NXT_ERROR; + } + + prev_fd = port->queue_fd; + prev_queue = port->queue; + + msg->fd[0] = fd; + msg->fd[1] = -1; + + nxt_port_test_broadcasts = 0; + + nxt_port_process_ready_handler(task, msg); + + /* + * The positive half of the broadcast assertion in the queueless case: + * refusing a replacement is not a failed start, so this message still + * reaches the announcement. Without this, "not broadcast" there would + * pass on a handler that never broadcasts at all. + */ + if (nxt_port_test_broadcasts == 0) { + nxt_log_alert(thr->log, "port ready test: a kept queue was not " + "broadcast"); + return NXT_ERROR; + } + + if (port->queue != prev_queue || port->queue_fd != prev_fd) { + nxt_log_alert(thr->log, "port ready test: short queue replaced the " + "installed one"); + return NXT_ERROR; + } + + if (msg->fd[0] != -1 || nxt_test_fd_is_open(fd)) { + nxt_log_alert(thr->log, "port ready test: short queue leaked its " + "descriptor"); + return NXT_ERROR; + } + + /* + * Touch the last item of the queue the port holds, the way + * nxt_port_queue_send() would, to prove the mapping the port kept is + * backed all the way to its end. The check above already catches a + * handler that installs the short mapping, and catches it before this + * access, which on such a handler takes SIGBUS and kills the test + * binary -- verified by moving this access ahead of the check. + */ + queue = port->queue; + queue->items[NXT_PORT_QUEUE_SIZE - 1].size = 0; + + return NXT_OK; +} + +/* + * The same wedge reached without any mmap failure: an authenticated + * PROCESS_READY for an application worker that carries no descriptor at all. + * + * At the receiver's RLIMIT_NOFILE the kernel delivers the payload and the + * SCM_CREDENTIALS while discarding SCM_RIGHTS and setting MSG_CTRUNC, which + * nothing in the receive path inspects -- so the handler sees a message that + * passes the sender gate with fd[0] == -1. Only libunit attaches a queue to + * PROCESS_READY and it always does, so for an NXT_PROCESS_APP port this can + * never be legitimate; the core processes that do send fd -1 announce to main + * and are not APP. + * + * The fd is injected directly rather than by exhausting descriptors: the + * kernel path is the motivation, and reproducing it would make the test + * depend on the receiver's rlimit rather than on the handler's decision. + */ +static nxt_int_t +nxt_port_ready_test_no_queue_fd(nxt_thread_t *thr, nxt_task_t *task, + nxt_port_recv_msg_t *msg, nxt_process_t *process, nxt_port_t *port) +{ + int status; + pid_t child, reaped; + + if (nxt_slow_path(port->queue != NULL + || port->type != NXT_PROCESS_APP)) + { + nxt_log_alert(thr->log, "port ready test: the no-queue-fd case needs " + "an app port with no queue"); + return NXT_ERROR; + } + + child = fork(); + if (child == -1) { + nxt_log_alert(thr->log, "port ready test failed to fork the kill " + "target"); + return NXT_ERROR; + } + + if (child == 0) { + for ( ;; ) { + pause(); + } + + _exit(0); + } + + process->isolated_pid = child; + process->start_failed = 0; + process->state = NXT_PROCESS_STATE_CREATING; + +#if (NXT_USE_CMSG_PID) + msg->cmsg_pid = child; +#endif + + /* What a truncated control block leaves behind. */ + msg->fd[0] = -1; + msg->fd[1] = -1; + + nxt_port_test_broadcasts = 0; + + nxt_port_process_ready_handler(task, msg); + +#if (NXT_USE_CMSG_PID) + + if (nxt_port_test_broadcasts != 0 || port->queue != NULL) { + nxt_log_alert(thr->log, "port ready test: a READY with no queue " + "descriptor was broadcast anyway"); + goto fail; + } + + if (process->state == NXT_PROCESS_STATE_READY || !process->start_failed) { + nxt_log_alert(thr->log, "port ready test: a READY with no queue " + "descriptor still marked the process ready"); + goto fail; + } + + if (waitpid(child, &status, 0) != child) { + nxt_log_alert(thr->log, "port ready test failed to reap the kill " + "target"); + return NXT_ERROR; + } + + if (!WIFSIGNALED(status) || WTERMSIG(status) != SIGKILL) { + nxt_log_alert(thr->log, "port ready test: the worker that announced " + "no queue did not die of SIGKILL (status %d)", status); + return NXT_ERROR; + } + +#else + + if (process->state != NXT_PROCESS_STATE_READY) { + nxt_log_alert(thr->log, "port ready test: a platform with no sender " + "credential did not keep the announcing behaviour"); + goto fail; + } + + (void) kill(child, SIGKILL); + + if (waitpid(child, &status, 0) != child) { + nxt_log_alert(thr->log, "port ready test failed to reap the kill " + "target"); + return NXT_ERROR; + } + +#endif + + (void) reaped; + + return NXT_OK; + +fail: + + (void) kill(child, SIGKILL); + (void) waitpid(child, &status, 0); + + return NXT_ERROR; +} + + +/* + * Issue #231: a first PROCESS_READY whose queue cannot be mapped. + * + * This is a different arm from nxt_port_ready_test_short_queue() above, + * and the distinction is the whole point. That case runs after a queue + * has been installed, so the handler keeps the working mapping and the + * process stays ready -- refusing a replacement is not a failed start. + * Here the port has no queue at all, so there is nothing to fall back to: + * the worker is unreachable on this port forever, and the start has to + * fail rather than be marked ready. + * + * The kill target is a real forked child, never nxt_pid + N. The suite + * runs as root in CI, so a made-up pid is a live process on the machine; + * only the pid fork() returned is known to be ours to signal. + */ +static nxt_int_t +nxt_port_ready_test_queueless(nxt_thread_t *thr, nxt_task_t *task, + nxt_port_recv_msg_t *msg, nxt_process_t *process, nxt_port_t *port) +{ + int status; + pid_t child, reaped; + nxt_fd_t fd; + nxt_uint_t i; + + if (nxt_slow_path(port->queue != NULL)) { + nxt_log_alert(thr->log, "port ready test: the queueless case needs a " + "port with no queue"); + return NXT_ERROR; + } + + /* + * A child that only waits to be killed. _exit() rather than return so + * that a child which somehow escapes pause() cannot run the rest of the + * test suite as a second process. + */ + child = fork(); + if (child == -1) { + nxt_log_alert(thr->log, "port ready test failed to fork the kill " + "target"); + return NXT_ERROR; + } + + if (child == 0) { + for ( ;; ) { + pause(); + } + + _exit(0); + } + + process->isolated_pid = child; + +#if (NXT_USE_CMSG_PID) + /* The sender gate compares against isolated_pid, which is now the child. */ + msg->cmsg_pid = child; +#endif + + fd = syscall(SYS_memfd_create, "nxt_port_ready_test", MFD_CLOEXEC); + if (fd == -1) { + nxt_log_alert(thr->log, "port ready test failed to create the queue"); + goto fail; + } + + /* One page: every byte past it faults, so the mapping must be refused. */ + if (ftruncate(fd, nxt_pagesize) == -1) { + nxt_log_alert(thr->log, "port ready test failed to size the queue"); + (void) close(fd); + goto fail; + } + + msg->fd[0] = fd; + msg->fd[1] = -1; + + nxt_port_test_broadcasts = 0; + + nxt_port_process_ready_handler(task, msg); + + if (msg->fd[0] != -1 || nxt_test_fd_is_open(fd)) { + nxt_log_alert(thr->log, "port ready test: the refused queue leaked " + "its descriptor"); + goto fail; + } + + if (port->queue != NULL || port->queue_fd != -1) { + nxt_log_alert(thr->log, "port ready test: a refused queue was " + "installed anyway"); + goto fail; + } + +#if (NXT_USE_CMSG_PID) + + /* + * Never announce a port nothing can be delivered on: every peer would + * hold the same unreachable copy, and a QUIT could never arrive. The + * kept-queue case above is what stops this passing vacuously. + */ + if (nxt_port_test_broadcasts != 0) { + nxt_log_alert(thr->log, "port ready test: a refused queue was " + "broadcast anyway"); + goto fail; + } + + /* + * Not ready: nothing about this worker ever became usable, and the + * prototype's SIGCHLD notifies the others for any state but CREATING, + * so leaving it at CREATED still reaches the start's failure. + */ + if (process->state == NXT_PROCESS_STATE_READY) { + nxt_log_alert(thr->log, "port ready test: a refused queue still " + "marked the process ready"); + goto fail; + } + + if (!process->start_failed) { + nxt_log_alert(thr->log, "port ready test: a refused queue did not " + "mark the start as failed"); + goto fail; + } + + /* + * The fixture enters at CREATING, as a worker whose WHOAMI has not been + * handled yet would. Killing it while it is still CREATING would leave + * the prototype's SIGCHLD silent (nxt_application.c:907) and the start + * pending -- the wedge, reached by a different road. + */ + if (process->state != NXT_PROCESS_STATE_CREATED) { + nxt_log_alert(thr->log, "port ready test: a refused queue left the " + "process at CREATING, so nothing will notify its death"); + goto fail; + } + + /* + * Polled rather than blocking: a handler that never signalled would + * otherwise hang the suite instead of failing it. + */ + for (i = 0; i < 100; i++) { + reaped = waitpid(child, &status, WNOHANG); + + if (reaped == child) { + break; + } + + if (reaped == -1) { + nxt_log_alert(thr->log, "port ready test: waitpid() failed %E", + nxt_errno); + goto fail; + } + + nxt_nanosleep(10 * 1000000); /* 10ms, so at most a second. */ + } + + if (reaped != child) { + nxt_log_alert(thr->log, "port ready test: the process whose queue " + "was refused was not killed"); + goto fail; + } + + if (!WIFSIGNALED(status) || WTERMSIG(status) != SIGKILL) { + nxt_log_alert(thr->log, "port ready test: the killed process did not " + "die of SIGKILL (status %d)", status); + return NXT_ERROR; + } + + /* + * The latch, tested for what it is actually for. A retransmitted + * PROCESS_READY must not signal a second time: the first pid has been + * reaped and may by now name an unrelated process. Pointing + * isolated_pid at a second, live child makes that concrete -- without + * the latch the handler re-enters the failure arm and kills it, so the + * child's survival is the assertion. Re-checking the first pid would + * prove nothing, since it is already dead. + */ + child = fork(); + if (child == -1) { + nxt_log_alert(thr->log, "port ready test failed to fork the second " + "kill target"); + return NXT_ERROR; + } + + if (child == 0) { + for ( ;; ) { + pause(); + } + + _exit(0); + } + + process->isolated_pid = child; + msg->cmsg_pid = child; + + fd = syscall(SYS_memfd_create, "nxt_port_ready_test", MFD_CLOEXEC); + if (fd == -1 || ftruncate(fd, nxt_pagesize) == -1) { + nxt_log_alert(thr->log, "port ready test failed to re-create the " + "queue"); + if (fd != -1) { + (void) close(fd); + } + goto fail; + } + + msg->fd[0] = fd; + msg->fd[1] = -1; + + nxt_port_process_ready_handler(task, msg); + + if (msg->fd[0] != -1 || nxt_test_fd_is_open(fd)) { + nxt_log_alert(thr->log, "port ready test: the retransmitted READY " + "leaked its descriptor"); + goto fail; + } + + if (process->state == NXT_PROCESS_STATE_READY) { + nxt_log_alert(thr->log, "port ready test: the retransmitted READY " + "marked the process ready"); + goto fail; + } + + /* + * Still running: the latch refused the arm before it reached the kill. + * + * SIGKILL is delivered asynchronously, so a single WNOHANG poll here + * would race a handler that did signal and read as "survived" -- the + * assertion has to give a kill time to land before concluding none + * happened. Verified by removing the latch check: without this window + * the mutation passes, with it the child is reaped and the test fails. + */ + for (i = 0; i < 20; i++) { + reaped = waitpid(child, &status, WNOHANG); + + if (reaped != 0) { + nxt_log_alert(thr->log, "port ready test: a retransmitted READY " + "killed again"); + return NXT_ERROR; + } + + nxt_nanosleep(10 * 1000000); /* 10ms, so 200ms in total. */ + } + + (void) kill(child, SIGKILL); + (void) waitpid(child, &status, 0); + +#else + + /* + * With no kernel-validated sender pid the start cannot be failed + * safely, so the handler keeps the pre-#231 behaviour and announces the + * process on a port that cannot be reached. Asserting that here is what + * keeps the guard honest: a change that quietly enabled the destructive + * arm on an unauthenticated platform fails this. + */ + (void) i; + (void) reaped; + + if (process->state != NXT_PROCESS_STATE_READY + || nxt_port_test_broadcasts == 0) + { + nxt_log_alert(thr->log, "port ready test: a platform with no sender " + "credential did not keep the announcing behaviour"); + goto fail; + } + + if (process->start_failed) { + nxt_log_alert(thr->log, "port ready test: a platform with no sender " + "credential latched a failed start"); + goto fail; + } + + /* Nothing signalled it, so the child is ours to clean up. */ + (void) kill(child, SIGKILL); + + if (waitpid(child, &status, 0) != child) { + nxt_log_alert(thr->log, "port ready test failed to reap the kill " + "target"); + return NXT_ERROR; + } + +#endif + + return NXT_OK; + +fail: + + (void) kill(child, SIGKILL); + (void) waitpid(child, &status, 0); + + return NXT_ERROR; +} + +#endif + + +/* + * Require that the announcement the handler just made is on the router's + * port and carries "expect" as its stream, then drain both peer ports so the + * next case starts from empty queues. + * + * The stream is what these cases are about: it is the number the router + * matches against its registered start RPCs, so announcing the wrong one + * answers nobody while looking, from every other angle, like a success. + * + * "expect" may be NXT_PORT_READY_TEST_NO_MSG to require that the router was + * not announced to at all, or NXT_PORT_READY_TEST_ANY to make no + * requirement and only report, through *got, whether it was -- which is what + * the mixed-outcome case below needs, since which of the two peers a single + * armed allocation failure lands on is hash order, not something this test + * fixes. + */ + +#define NXT_PORT_READY_TEST_NO_MSG ((uint32_t) -1) +#define NXT_PORT_READY_TEST_ANY ((uint32_t) -2) + +static nxt_int_t +nxt_port_ready_test_announced(nxt_thread_t *thr, nxt_task_t *task, + nxt_port_t *router_port, nxt_port_t *other_port, uint32_t expect, + nxt_bool_t *got, const char *name) +{ + nxt_int_t ret; + nxt_uint_t n; + nxt_queue_link_t *link; + nxt_port_send_msg_t *sent; + + ret = NXT_OK; + sent = NULL; + n = 0; + + for (link = nxt_queue_first(&router_port->messages); + link != nxt_queue_tail(&router_port->messages); + link = nxt_queue_next(link)) + { + n++; + sent = nxt_queue_link_data(link, nxt_port_send_msg_t, link); + } + + if (got != NULL) { + *got = (n != 0); + } + + if (expect == NXT_PORT_READY_TEST_ANY) { + /* Reported, not required. */ + + } else if (expect == NXT_PORT_READY_TEST_NO_MSG) { + if (nxt_slow_path(n != 0)) { + nxt_log_alert(thr->log, "port ready test: %s announced the port " + "to the router anyway", name); + ret = NXT_ERROR; + } + + } else if (nxt_slow_path(n != 1)) { + nxt_log_alert(thr->log, "port ready test: %s left %ui announcements " + "on the router port (expected 1)", name, n); + ret = NXT_ERROR; + + } else if (nxt_slow_path(sent->port_msg.stream != expect)) { + nxt_log_alert(thr->log, "port ready test: %s announced the port on " + "stream #%uD, but the start it answers is stream #%uD " + "-- the router matches its registrations on that " + "number, so this answers nobody", name, + sent->port_msg.stream, expect); + ret = NXT_ERROR; + } + + nxt_port_test_run_error_handler(task, router_port); + + if (other_port != NULL) { + nxt_port_test_run_error_handler(task, other_port); + } + + return ret; +} + + +/* + * Issue #271: process->stream is the start RPC the initiator armed, and the + * handler has to retire it exactly when that RPC has been answered -- which + * is when the NEW_PORT announcement carrying the stream has gone out, not + * when the READY state has been set. + * + * The two are not the same moment. The state is set first and the + * announcement is written afterwards, so a READY whose announcement fails + * leaves the initiator with no reply at all; the only thing that can still + * fail its RPC is the stream in the REMOVE_PID this process's death will + * send (src/nxt_router.c:1148-1153). Clearing on the state -- which is + * what nxt_main_process_sigchld_handler() used to do -- throws that away. + * Keeping it forever is the other error the issue names: stream numbers come + * from one counter that every request draws on, so after a wrap an ordinary + * exit retires somebody else's live RPC. + * + * The fixture is a process announcing itself with no queue descriptor, on a + * prototype port: that is a real shape (only libunit attaches a queue, and + * the core processes that announce to main do not), and it keeps the + * announcement free of descriptors, so the peer's queued message can be + * drained without closing a descriptor another port owns. + * + * The peer is a router, because nxt_proc_send_matrix[] pairs ROUTER with + * PROTOTYPE; without a peer the announcement would have nothing to fail on + * and both cases would pass vacuously. Its port has no socket and is not + * write-ready, so nxt_port_socket_write2() queues rather than writes, and + * nxt_port_test_msg_alloc_failures() can fail that queueing on demand. + */ +static nxt_int_t +nxt_port_ready_test_stream(nxt_thread_t *thr, nxt_task_t *task, + nxt_runtime_t *rt, nxt_mp_t *mp) +{ + uint32_t stream; + nxt_int_t ret; + nxt_bool_t router_got_it; + nxt_port_t *proto_port, *router_port, *main_port; + nxt_process_t *proto, *router, *mainp; + nxt_event_engine_t engine, *saved_engine; + nxt_port_recv_msg_t msg; + + ret = NXT_ERROR; + proto_port = NULL; + router_port = NULL; + main_port = NULL; + stream = 0x21212121; + + proto = nxt_mp_zalloc(mp, sizeof(nxt_process_t)); + router = nxt_mp_zalloc(mp, sizeof(nxt_process_t)); + mainp = nxt_mp_zalloc(mp, sizeof(nxt_process_t)); + + if (nxt_slow_path(proto == NULL || router == NULL || mainp == NULL)) { + return NXT_ERROR; + } + + proto->pid = nxt_pid + 7; + proto->isolated_pid = nxt_pid + 8; + proto->state = NXT_PROCESS_STATE_CREATING; + proto->stream = stream; + nxt_queue_init(&proto->ports); + + nxt_runtime_process_add(task, proto); + + proto_port = nxt_port_new(task, 0, proto->pid, NXT_PROCESS_PROTOTYPE); + if (nxt_slow_path(proto_port == NULL)) { + /* proto is in rt->processes already: take it out on the way. */ + goto done; + } + + proto_port->pair[0] = -1; + proto_port->pair[1] = -1; + proto_port->socket.fd = -1; + + /* The fixture's own hold on the record, as main's would be. */ + proto->use_count = 1; + + nxt_process_port_add(task, proto, proto_port); + + /* + * A second peer, of main's type. nxt_proc_send_matrix[] pairs MAIN with + * everything, so the prototype announcing a child writes to main as well + * as to the router -- and only the router's write is evidence about the + * start. Without this peer the two rules ("every peer succeeded" and + * "the router succeeded") cannot be told apart. + */ + mainp->pid = nxt_pid + 10; + mainp->isolated_pid = nxt_pid + 10; + mainp->state = NXT_PROCESS_STATE_READY; + nxt_queue_init(&mainp->ports); + + nxt_runtime_process_add(task, mainp); + + main_port = nxt_port_new(task, 0, mainp->pid, NXT_PROCESS_MAIN); + if (nxt_slow_path(main_port == NULL)) { + goto done; + } + + main_port->pair[0] = -1; + main_port->pair[1] = -1; + main_port->socket.fd = -1; + + /* The fixture's own hold on the record, as main's would be. */ + mainp->use_count = 1; + + nxt_process_port_add(task, mainp, main_port); + + router->pid = nxt_pid + 9; + router->isolated_pid = nxt_pid + 9; + router->state = NXT_PROCESS_STATE_READY; + nxt_queue_init(&router->ports); + + nxt_runtime_process_add(task, router); + + router_port = nxt_port_new(task, 0, router->pid, NXT_PROCESS_ROUTER); + if (nxt_slow_path(router_port == NULL)) { + goto done; + } + + router_port->pair[0] = -1; + router_port->pair[1] = -1; + router_port->socket.fd = -1; + + /* The fixture's own hold on the record, as main's would be. */ + router->use_count = 1; + + nxt_process_port_add(task, router, router_port); + + /* + * An engine only for the announcement: nxt_port_send_port() allocates + * its buffer from task->thread->engine->mem_pool, and the queued message + * is completed on the engine's work queue. The rest of this file needs + * none, so it is installed and taken away again here. + */ + nxt_memzero(&engine, sizeof(engine)); + nxt_work_queue_cache_create(&engine.work_queue_cache, 1024); + engine.fast_work_queue.cache = &engine.work_queue_cache; + nxt_work_queue_name(&engine.fast_work_queue, "fast"); + engine.mem_pool = mp; + + saved_engine = thr->engine; + thr->engine = &engine; + rt->main_engine = &engine; + + nxt_memzero(&msg, sizeof(nxt_port_recv_msg_t)); + + msg.port_msg.pid = proto->pid; + msg.port_msg.stream = stream; + msg.fd[0] = -1; + msg.fd[1] = -1; +#if (NXT_USE_CMSG_PID) + msg.cmsg_pid = proto->isolated_pid; +#endif + + /* + * The announcement cannot be queued. The process is ready as far as the + * state goes, and the initiator has still not been told anything, so the + * stream has to survive -- it is the only thing that will fail that RPC + * when this process dies. + */ + nxt_port_test_msg_alloc_failures(2); + + nxt_port_process_ready_handler(task, &msg); + + nxt_port_test_msg_alloc_failures(0); + + if (nxt_slow_path(nxt_port_ready_test_announced(thr, task, router_port, + main_port, + NXT_PORT_READY_TEST_NO_MSG, + NULL, + "an announcement whose " + "allocation was made to " + "fail") + != NXT_OK)) + { + goto drain; + } + + if (nxt_slow_path(proto->state != NXT_PROCESS_STATE_READY)) { + nxt_log_alert(thr->log, "port ready test: a queueless prototype was " + "not marked ready"); + goto drain; + } + + if (nxt_slow_path(proto->stream != stream)) { + nxt_log_alert(thr->log, "port ready test: the start stream was " + "cleared although the announcement never went out, so " + "nothing is left to fail that RPC"); + goto drain; + } + + /* + * The same message again, with the announcement allowed through. Now + * the initiator has its reply -- nxt_router_new_port_handler() feeds a + * stream-bearing NEW_PORT to nxt_port_rpc_handler() -- and the stream is + * spent: keeping it would put it into this process's REMOVE_PID and, + * once the counter has wrapped, fail an unrelated live RPC. + */ + nxt_port_process_ready_handler(task, &msg); + + if (nxt_slow_path(nxt_port_ready_test_announced(thr, task, router_port, + main_port, stream, NULL, + "a healthy READY") + != NXT_OK)) + { + goto drain; + } + + if (nxt_slow_path(proto->stream != 0)) { + nxt_log_alert(thr->log, "port ready test: the start stream survived " + "an announcement that answered it, so a later " + "REMOVE_PID would fail whatever RPC now holds that " + "number"); + goto drain; + } + + /* + * A child naming a stream of its own choosing. The gate above + * authenticates who sent a PROCESS_READY, not what it says, and the + * stream field is written by the sender: a child that echoes something + * other than the stream it was started for -- malformed, or forged by a + * compromised worker -- must not be able to redirect the answer. + * + * The announcement has to carry the stream this process registered when + * it forked the child, so the router retires the start that is really + * pending; and clearing is then correct, because that start has been + * answered. Taking the wire value instead would announce a number no + * registration waits on while clearing the one that does -- the start + * would never be retired by the reply, and the REMOVE_PID fallback would + * carry nothing. + */ + proto->stream = stream; + proto->state = NXT_PROCESS_STATE_CREATING; + msg.port_msg.stream = 0xdeadbeef; + + nxt_port_process_ready_handler(task, &msg); + + if (nxt_slow_path(nxt_port_ready_test_announced(thr, task, router_port, + main_port, stream, NULL, + "a READY naming a foreign " + "stream") + != NXT_OK)) + { + goto drain; + } + + if (nxt_slow_path(proto->stream != 0)) { + nxt_log_alert(thr->log, "port ready test: a READY naming a foreign " + "stream did not retire the start that was answered"); + goto drain; + } + + /* + * The same with a stream of 0, which is what a truncated or zeroed + * message carries. 0 is the "no RPC" value everywhere else + * (nxt_router_remove_pid_handler() returns on it), so a handler that + * passed the wire value straight through would announce nothing + * answerable and still count the start as replied to. + */ + proto->stream = stream; + proto->state = NXT_PROCESS_STATE_CREATING; + msg.port_msg.stream = 0; + + nxt_port_process_ready_handler(task, &msg); + + if (nxt_slow_path(nxt_port_ready_test_announced(thr, task, router_port, + main_port, stream, NULL, + "a READY naming stream 0") + != NXT_OK)) + { + goto drain; + } + + if (nxt_slow_path(proto->stream != 0)) { + nxt_log_alert(thr->log, "port ready test: a READY naming stream 0 did " + "not retire the start that was answered"); + goto drain; + } + + /* + * One peer's announcement fails and the other's succeeds. Only the + * router's is evidence about the start: it owns the registration for + * every start, whether main or the prototype did the forking, and + * nxt_router_new_port_handler() retires it on a stream-bearing NEW_PORT. + * A failure to any other peer is a delivery problem for that peer. + * + * One armed allocation failure lands on whichever peer the runtime walks + * first, and that is the order they were registered in, not the pids: + * main is added to the fixture before the router above so that the + * failure falls on main and the router's write goes through. That is + * the branch which discriminates the two implementations -- aggregating + * every peer keeps a stream the router has already retired -- and it is + * reached on every run; the log line below records which branch was + * taken, so a future change to the walk order shows up as a note rather + * than as a test that silently stopped testing anything. + * + * The assertion is still written as the rule for both branches, so it + * cannot fail spuriously if that order does change. The opposite branch + * -- the router itself missing out -- is covered deterministically by + * the failed-announcement case above, which arms a failure for every + * peer. + */ + proto->stream = stream; + proto->state = NXT_PROCESS_STATE_CREATING; + msg.port_msg.stream = stream; + + nxt_port_test_msg_alloc_failures(1); + + nxt_port_process_ready_handler(task, &msg); + + nxt_port_test_msg_alloc_failures(0); + + router_got_it = 0; + + if (nxt_slow_path(nxt_port_ready_test_announced(thr, task, router_port, + main_port, + NXT_PORT_READY_TEST_ANY, + &router_got_it, + "a partly failed " + "announcement") + != NXT_OK)) + { + goto drain; + } + + nxt_log_error(NXT_LOG_INFO, thr->log, "port ready test: the partly failed " + "announcement %s the router", + router_got_it ? "reached" : "missed"); + + if (nxt_slow_path(router_got_it && proto->stream != 0)) { + nxt_log_alert(thr->log, "port ready test: the router was announced to " + "and retired the start, but the stream was kept because " + "another peer's write failed -- after a counter wrap " + "that stream fails an unrelated RPC"); + goto drain; + } + + if (nxt_slow_path(!router_got_it && proto->stream != stream)) { + nxt_log_alert(thr->log, "port ready test: the router was not " + "announced to, so nothing retired the start, yet the " + "stream was cleared -- the REMOVE_PID fallback is gone"); + goto drain; + } + + msg.port_msg.stream = stream; + + ret = NXT_OK; + +drain: + + /* + * A queued message is malloc'd and holds a port reference; the error + * handler is what releases both, and it has to run while the engine it + * was queued on is still installed. The checker above drains after each + * case; this catches whatever a failed case left behind. + */ + nxt_port_test_run_error_handler(task, router_port); + + if (main_port != NULL) { + nxt_port_test_run_error_handler(task, main_port); + } + + thr->engine = saved_engine; + rt->main_engine = NULL; + + nxt_work_queue_cache_destroy(&engine.work_queue_cache); + +done: + + /* + * Each port was paired with its process by nxt_process_port_add(), + * which took a process reference. Dropping the port's last reference + * with nxt_port_use() lets nxt_port_release() unlink it, drop that + * process reference (2 -> 1, the fixture keeps its own hold) and free + * the port's own pool (see nxt_port_new()), which the caller's + * nxt_mp_destroy(mp) cannot reach. proto/mainp/router come from mp and + * are freed along with it, but nxt_runtime_process_add() put them in + * rt->processes, a malloc'd table outside mp, and + * nxt_runtime_process_remove() is what takes them back out. + * + * That add happens for each of the three only partway through the + * function -- main_port is created before router is registered -- so + * an early "goto done" can reach here with a process struct that is + * non-NULL but was never added. Removing it anyway would trip + * nxt_runtime_process_remove()'s "process->registered != 0" assert, so + * ->registered gates each removal alongside the NULL check. + */ + if (main_port != NULL) { + nxt_port_close(task, main_port); + nxt_port_use(task, main_port, -1); + } + + if (router_port != NULL) { + nxt_port_close(task, router_port); + nxt_port_use(task, router_port, -1); + } + + if (proto_port != NULL) { + nxt_port_close(task, proto_port); + nxt_port_use(task, proto_port, -1); + } + + if (mainp != NULL && mainp->registered) { + nxt_runtime_process_remove(rt, mainp); + } + + if (router != NULL && router->registered) { + nxt_runtime_process_remove(rt, router); + } + + if (proto != NULL && proto->registered) { + nxt_runtime_process_remove(rt, proto); + } + + return ret; +} + + +nxt_int_t +nxt_port_ready_test(nxt_thread_t *thr) +{ + nxt_mp_t *mp; + nxt_pid_t isolated_pid; + nxt_task_t *task; + nxt_int_t ret; + nxt_port_t *port, *queueless_port, *nofd_port; + nxt_runtime_t *rt, *saved_rt; + nxt_process_t *process; +#if (NXT_HAVE_MEMFD_CREATE) + nxt_process_t *queueless, *nofd; +#endif + nxt_port_recv_msg_t msg; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "port ready test started"); + + task = thr->task; + task->thread = thr; + +#if (NXT_HAVE_MEMFD_CREATE) + + /* + * The two accepted sizes, ahead of the handler fixtures and calling + * nxt_port_queue_mmap() directly: nothing is allocated yet, and a + * failure here names the rule that broke rather than the handler that + * relied on it. + */ + if (nxt_slow_path(nxt_port_ready_test_queue_sizes(thr, task) != NXT_OK)) { + return NXT_ERROR; + } + +#endif + + /* Defined before the first goto done: the cleanup there releases them. */ + port = NULL; + queueless_port = NULL; + nofd_port = NULL; + process = NULL; +#if (NXT_HAVE_MEMFD_CREATE) + queueless = NULL; + nofd = NULL; +#endif + + isolated_pid = nxt_pid + 3; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + rt = nxt_mp_zalloc(mp, sizeof(nxt_runtime_t)); + if (nxt_slow_path(rt == NULL)) { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + rt->mem_pool = mp; + + if (nxt_slow_path(nxt_thread_mutex_create(&rt->processes_mutex) != NXT_OK)) + { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + saved_rt = thr->runtime; + thr->runtime = rt; + + nxt_memzero(&msg, sizeof(nxt_port_recv_msg_t)); + + /* + * A pid the runtime does not know: the lookup fails and the handler + * has to close what the message carried. + */ + msg.port_msg.pid = nxt_pid + 1; + + ret = nxt_port_ready_test_case(thr, task, &msg, "unknown pid"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + /* + * A process that is ready already: the state check has to reject the + * message rather than map the descriptor a second time. The process + * carries no ports, so reaching the mapping at all would be a bug -- + * the empty-ports guard is the second reject path covered here. + */ + process = nxt_mp_zalloc(mp, sizeof(nxt_process_t)); + if (nxt_slow_path(process == NULL)) { + ret = NXT_ERROR; + goto done; + } + + /* + * A pid-isolated process: pid is the global pid the runtime hash is + * keyed on, isolated_pid is the namespace-local pid the receiver reads + * out of SCM_CREDENTIALS. They differ on purpose -- with one value + * every case would pass the sender gate whichever of the two it + * compared, and the collision the gate exists to refuse could not be + * expressed at all. + */ + process->pid = nxt_pid + 2; + process->isolated_pid = isolated_pid; + process->state = NXT_PROCESS_STATE_CREATING; + nxt_queue_init(&process->ports); + + nxt_runtime_process_add(task, process); + + msg.port_msg.pid = process->pid; +#if (NXT_USE_CMSG_PID) + msg.cmsg_pid = process->isolated_pid; +#endif + + ret = nxt_port_ready_test_case(thr, task, &msg, "ready with no ports"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + /* + * The guard has to reject before the state is set, so that a message + * the handler refuses leaves the process as it found it. + */ + if (nxt_slow_path(process->state == NXT_PROCESS_STATE_READY)) { + nxt_log_alert(thr->log, "port ready test: a rejected message marked " + "the process ready"); + ret = NXT_ERROR; + goto done; + } + + /* + * From here on the process has a port, so the empty-ports guard can no + * longer be what refuses a message, and a refusal has to come from the + * sender gate itself. + */ + port = nxt_port_new(task, 1, process->pid, NXT_PROCESS_APP); + if (nxt_slow_path(port == NULL)) { + ret = NXT_ERROR; + goto done; + } + + port->pair[0] = -1; + port->pair[1] = -1; + port->socket.fd = -1; + + /* The fixture's own hold on the record, as main's would be. */ + process->use_count = 1; + + nxt_process_port_add(task, process, port); + +#if (NXT_USE_CMSG_PID) + + /* An unrelated sender announcing somebody else's process. */ + ret = nxt_port_ready_test_reject(thr, task, &msg, process, port, + nxt_pid + 4, "a foreign sender"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + /* + * The global pid of the very process being announced. Inside a pid + * namespace it is not the pid the receiver sees for that process, and + * another task's namespace-local pid can be made to equal it, so + * accepting it as well would leave the gate bypassable. + */ + ret = nxt_port_ready_test_reject(thr, task, &msg, process, port, + process->pid, "the global pid"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + /* + * A record the receiver never forked keeps isolated_pid 0 -- + * nxt_runtime_process_get() fills only ->pid. There is nothing to + * authenticate against, so the credential of 0 that a plain equality + * test would accept has to be refused as well. + */ + process->isolated_pid = 0; + + ret = nxt_port_ready_test_reject(thr, task, &msg, process, port, 0, + "a process record nothing forked"); + + process->isolated_pid = isolated_pid; + + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + +#endif + +#if (NXT_HAVE_MEMFD_CREATE) + + /* + * The handler takes the descriptor over and maps it, and a second + * PROCESS_READY from the same, authenticated sender replaces that + * mapping instead of leaking it: a process may re-announce a new queue, + * and refusing the repeat would strand it on the mapping it has. + */ + ret = nxt_port_ready_test_queue(thr, task, &msg, port, "first ready"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + ret = nxt_port_ready_test_queue(thr, task, &msg, port, "replacing ready"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + ret = nxt_port_ready_test_short_queue(thr, task, &msg, port); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + /* + * Issue #231 needs its own fixture: the process above is ready and its + * port has a working queue, which is exactly the case that must NOT + * fail a start. A fresh record, never announced, is the only way to + * reach the first-mapping arm. + */ + queueless = nxt_mp_zalloc(mp, sizeof(nxt_process_t)); + if (nxt_slow_path(queueless == NULL)) { + ret = NXT_ERROR; + goto done; + } + + queueless->pid = nxt_pid + 5; + queueless->state = NXT_PROCESS_STATE_CREATING; + nxt_queue_init(&queueless->ports); + + nxt_runtime_process_add(task, queueless); + + queueless_port = nxt_port_new(task, 1, queueless->pid, NXT_PROCESS_APP); + if (nxt_slow_path(queueless_port == NULL)) { + ret = NXT_ERROR; + goto done; + } + + queueless_port->pair[0] = -1; + queueless_port->pair[1] = -1; + queueless_port->socket.fd = -1; + + /* The fixture's own hold on the record, as main's would be. */ + queueless->use_count = 1; + + nxt_process_port_add(task, queueless, queueless_port); + + msg.port_msg.pid = queueless->pid; + + ret = nxt_port_ready_test_queueless(thr, task, &msg, queueless, + queueless_port); + + /* + * Checked before the next case runs: its own assignment to ret would + * otherwise overwrite this failure and the suite would report a pass. + */ + if (nxt_slow_path(ret != NXT_OK)) { + msg.port_msg.pid = process->pid; + goto done; + } + + /* + * A second fresh record for the truncated-control case: the one above + * has its start_failed latch set, which would refuse the message before + * the arm under test is reached. + */ + nofd = nxt_mp_zalloc(mp, sizeof(nxt_process_t)); + if (nxt_slow_path(nofd == NULL)) { + ret = NXT_ERROR; + goto done; + } + + nofd->pid = nxt_pid + 6; + nofd->state = NXT_PROCESS_STATE_CREATING; + nxt_queue_init(&nofd->ports); + + nxt_runtime_process_add(task, nofd); + + nofd_port = nxt_port_new(task, 1, nofd->pid, NXT_PROCESS_APP); + if (nxt_slow_path(nofd_port == NULL)) { + ret = NXT_ERROR; + goto done; + } + + nofd_port->pair[0] = -1; + nofd_port->pair[1] = -1; + nofd_port->socket.fd = -1; + + /* The fixture's own hold on the record, as main's would be. */ + nofd->use_count = 1; + + nxt_process_port_add(task, nofd, nofd_port); + + msg.port_msg.pid = nofd->pid; + + ret = nxt_port_ready_test_no_queue_fd(thr, task, &msg, nofd, nofd_port); + + /* Restore the message for anything that runs after this case. */ + msg.port_msg.pid = process->pid; +#if (NXT_USE_CMSG_PID) + msg.cmsg_pid = process->isolated_pid; +#endif + + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + +#if (NXT_USE_CMSG_PID) + + /* + * The same forgery once a queue is installed and the process is ready: + * this is the one that matters, because it is where an accepted message + * would re-point a live port at shared memory of the sender's choosing. + */ + ret = nxt_port_ready_test_reject(thr, task, &msg, process, port, + nxt_pid + 4, "a foreign sender re-arming " + "an installed queue"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + +#endif + +#endif + + /* + * Last, because it registers two more processes in the fixture runtime + * and every case above walks that runtime when it announces a port. + */ + ret = nxt_port_ready_test_stream(thr, task, rt, mp); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + +done: + + /* + * The handler leaves the queue mapping and its descriptor on the port, + * and nothing else owns them here: releasing the pool would leak both. + * The fixture port never had a socket pair, so this only frees the queue. + * + * Each port was paired with its process by nxt_process_port_add(), + * which took a process reference. Dropping the port's last reference + * with nxt_port_use() lets nxt_port_release() unlink it, drop that + * process reference (2 -> 1, the fixture keeps its own hold) and free + * the port's own pool (see nxt_port_new()), which nxt_mp_destroy(mp) + * below cannot reach. The process itself came from mp and is freed + * along with it, but nxt_runtime_process_add() put it in rt->processes, + * a malloc'd table outside mp, and nxt_runtime_process_remove() is what + * takes it back out. + */ + if (port != NULL) { + nxt_port_close(task, port); + nxt_port_use(task, port, -1); + } + + if (queueless_port != NULL) { + nxt_port_close(task, queueless_port); + nxt_port_use(task, queueless_port, -1); + } + + if (nofd_port != NULL) { + nxt_port_close(task, nofd_port); + nxt_port_use(task, nofd_port, -1); + } + + /* + * nxt_runtime_process_add() can fail to allocate its hash node and leave + * a process unregistered, so ->registered gates each removal here too. + */ + if (process != NULL && process->registered) { + nxt_runtime_process_remove(rt, process); + } + +#if (NXT_HAVE_MEMFD_CREATE) + + if (queueless != NULL && queueless->registered) { + nxt_runtime_process_remove(rt, queueless); + } + + if (nofd != NULL && nofd->registered) { + nxt_runtime_process_remove(rt, nofd); + } + +#endif + + thr->runtime = saved_rt; + + nxt_thread_mutex_destroy(&rt->processes_mutex); + nxt_mp_destroy(mp); + + if (nxt_fast_path(ret == NXT_OK)) { + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "port ready test passed"); + } + + return ret; +} diff --git a/src/test/nxt_port_release_test.c b/src/test/nxt_port_release_test.c new file mode 100644 index 000000000..d4c4dcada --- /dev/null +++ b/src/test/nxt_port_release_test.c @@ -0,0 +1,148 @@ + +/* + * Copyright (C) FreeUnit contributors. + */ + +/* + * The port/process pairing (#425). Through nxt_process_port_add(), the only + * public path, a linked port always has ->process and one reference on it, + * a second add changes nothing, and nxt_port_release() takes all of it + * back. A port linked into a process's list by hand, with no ->process and + * no reference, used to abort a debug build and dereference NULL in a + * release build: it must be unlinked, and no reference it does not hold + * may be dropped. Runs in a child, so a crash is a failure. + */ + +#include +#include +#include +#include "nxt_tests.h" + + +static int +nxt_port_release_test_child(void *data) +{ + nxt_task_t *task; + nxt_port_t *port, *paired; + nxt_thread_t *thr; + nxt_runtime_t *rt; + nxt_process_t *process; + + rt = data; + thr = nxt_thread(); + thr->runtime = rt; + thr->engine = NULL; + task = thr->task; + + process = nxt_mp_zalloc(rt->mem_pool, sizeof(nxt_process_t)); + if (process == NULL) { + return 2; + } + + process->pid = nxt_pid + 53; + process->use_count = 1; + nxt_queue_init(&process->ports); + + nxt_runtime_process_add(task, process); + + /* A correctly paired port first: its release drops its reference. */ + + paired = nxt_port_new(task, 1, process->pid, NXT_PROCESS_APP); + if (paired == NULL) { + return 2; + } + + paired->pair[0] = -1; + paired->pair[1] = -1; + paired->socket.fd = -1; + + nxt_process_port_add(task, process, paired); + + if (paired->process != process + || nxt_queue_first(&process->ports) != &paired->link + || process->use_count != 2) + { + return 3; + } + +#if !(NXT_DEBUG) + + /* Already paired: refused in a release build; a debug build asserts. */ + + nxt_process_port_add(task, process, paired); + + if (paired->process != process + || nxt_queue_next(&paired->link) != nxt_queue_tail(&process->ports) + || nxt_queue_prev(&paired->link) != nxt_queue_head(&process->ports) + || process->use_count != 2) + { + return 7; + } + +#endif + + nxt_port_use(task, paired, -1); + + if (process->use_count != 1 || !nxt_queue_is_empty(&process->ports)) { + return 4; + } + + /* Linked by hand: no port->process, no reference. */ + + port = nxt_port_new(task, 2, process->pid, NXT_PROCESS_APP); + if (port == NULL) { + return 2; + } + + port->pair[0] = -1; + port->pair[1] = -1; + port->socket.fd = -1; + + nxt_queue_insert_tail(&process->ports, &port->link); + + nxt_port_use(task, port, -1); + + /* A port still on the list would point into the freed pool. */ + return !nxt_queue_is_empty(&process->ports) ? 5 + : process->use_count != 1 ? 6 : 0; +} + + +nxt_int_t +nxt_port_release_test(nxt_thread_t *thr) +{ + int rc; + nxt_mp_t *mp; + nxt_runtime_t *rt; + + thr->task->thread = thr; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + rt = nxt_mp_zalloc(mp, sizeof(nxt_runtime_t)); + + if (nxt_slow_path(rt == NULL + || nxt_thread_mutex_create(&rt->processes_mutex) + != NXT_OK)) + { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + rt->mem_pool = mp; + + rc = nxt_test_in_child(thr, "port release test", + nxt_port_release_test_child, rt); + + nxt_mp_destroy(mp); + + NXT_TEST_CHECK(thr->log, rc == 0, "port release test failed at step %d", + rc); + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "port release test passed"); + + return NXT_OK; +} diff --git a/src/test/nxt_port_rpc_fd_test.c b/src/test/nxt_port_rpc_fd_test.c new file mode 100644 index 000000000..922ee2ed2 --- /dev/null +++ b/src/test/nxt_port_rpc_fd_test.c @@ -0,0 +1,205 @@ +/* + * Copyright (C) F5, Inc. + */ + +/* + * Regression test for descriptor ownership in nxt_port_rpc_handler() + * (src/nxt_port_rpc.c). + * + * A stream that has no registered handler falls through nxt_port_rpc_handler() + * without running one, and until the fix nothing closed the descriptors the + * message carried. The stream number is a value off the wire -- any peer can + * name a stream this process never registered -- so a compromised application + * could exhaust the router's descriptor table one such message at a time. + * + * nxt_port_read_msg_process() now also closes what a dispatch leaves behind, + * which covers this path a second time; the close this test pins keeps the + * reject path correct on its own terms. + * + * This lives beside nxt_port_fd_test.c rather than inside it: that test + * drives nxt_port_new_port_handler(), nxt_port_mmap_handler() and the + * dispatcher itself, all of which need a port hash and NEW_PORT-shaped + * message bodies. This one only needs a port with an empty rpc_streams + * hash and a stream number nothing has registered -- the "find" and + * "delete" arms of nxt_port_rpc_handler() both take that path with a + * plain zeroed message. + */ + +#include +#include +#include "nxt_tests.h" + +#include + + +static void +nxt_port_rpc_fd_test_close(nxt_fd_t fd) +{ + if (fd != -1 && nxt_test_fd_is_open(fd)) { + (void) close(fd); + } +} + + +/* + * Runs nxt_port_rpc_handler() over an unregistered stream and checks that + * every real descriptor the message carried was closed, and that the slots + * were cleared the way a handler that consumes them would leave them. + * "last" selects between the two ways an unregistered stream is looked up: + * nxt_lvlhsh_delete() when the message is the last for its stream, + * nxt_lvlhsh_find() otherwise -- both reach the same early return. + */ +static nxt_int_t +nxt_port_rpc_fd_test_unregistered(nxt_thread_t *thr, nxt_task_t *task, + nxt_port_t *port, uint32_t stream, uint8_t last, nxt_bool_t second_fd, + const char *name) +{ + int pipe_fds[2]; + nxt_fd_t fd0, fd1; + nxt_port_recv_msg_t msg; + + if (nxt_slow_path(pipe(pipe_fds) != 0)) { + nxt_log_alert(thr->log, "port rpc fd test failed to create a pipe"); + return NXT_ERROR; + } + + fd0 = pipe_fds[0]; + fd1 = second_fd ? pipe_fds[1] : -1; + + if (!second_fd) { + /* Not needed for this case; close the write end right away. */ + (void) close(pipe_fds[1]); + } + + nxt_memzero(&msg, sizeof(nxt_port_recv_msg_t)); + + msg.port = port; + msg.fd[0] = fd0; + msg.fd[1] = fd1; + msg.port_msg.stream = stream; + msg.port_msg.last = last; + msg.port_msg.type = _NXT_PORT_MSG_RPC_READY; + + nxt_port_rpc_handler(task, &msg); + + if (nxt_slow_path(nxt_test_fd_is_open(fd0))) { + nxt_log_alert(thr->log, "port rpc fd test: %s leaked fd[0]", name); + goto fail; + } + + if (second_fd && nxt_slow_path(nxt_test_fd_is_open(fd1))) { + nxt_log_alert(thr->log, "port rpc fd test: %s leaked fd[1]", name); + goto fail; + } + + if (nxt_slow_path(msg.fd[0] != -1 || msg.fd[1] != -1)) { + nxt_log_alert(thr->log, "port rpc fd test: %s left fds in the " + "message", name); + return NXT_ERROR; + } + + return NXT_OK; + +fail: + + /* Close only what is still open; a closed number may already be reused. */ + nxt_port_rpc_fd_test_close(fd0); + nxt_port_rpc_fd_test_close(fd1); + + return NXT_ERROR; +} + + +nxt_int_t +nxt_port_rpc_fd_test(nxt_thread_t *thr) +{ + nxt_mp_t *mp; + nxt_int_t ret; + nxt_task_t *task; + nxt_port_t *port; + nxt_port_recv_msg_t msg; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "port rpc fd test started"); + + task = thr->task; + task->thread = thr; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + port = nxt_mp_zalloc(mp, sizeof(nxt_port_t)); + if (nxt_slow_path(port == NULL)) { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + /* + * port->rpc_streams starts zeroed, which is exactly "no stream + * registered" for nxt_lvlhsh: no insert has ever happened. + */ + port->mem_pool = mp; + + /* The stream number itself is arbitrary: nothing ever registers it. */ + ret = nxt_port_rpc_fd_test_unregistered(thr, task, port, 42, 0, 0, + "find, one fd"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + ret = nxt_port_rpc_fd_test_unregistered(thr, task, port, 42, 0, 1, + "find, two fds"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + ret = nxt_port_rpc_fd_test_unregistered(thr, task, port, 43, 1, 0, + "delete, one fd"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + ret = nxt_port_rpc_fd_test_unregistered(thr, task, port, 43, 1, 1, + "delete, two fds"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + /* + * A message with no descriptor attached (fd[0] == -1, as every real + * receive path sets it when the peer sent none) must not be mistaken + * for one that needs closing. + */ + nxt_memzero(&msg, sizeof(nxt_port_recv_msg_t)); + + msg.port = port; + msg.fd[0] = -1; + msg.fd[1] = -1; + msg.port_msg.stream = 44; + msg.port_msg.last = 0; + msg.port_msg.type = _NXT_PORT_MSG_RPC_READY; + + nxt_port_rpc_handler(task, &msg); + + if (nxt_slow_path(msg.fd[0] != -1 || msg.fd[1] != -1)) { + nxt_log_alert(thr->log, "port rpc fd test: a message without " + "descriptors was mishandled"); + ret = NXT_ERROR; + goto done; + } + + ret = NXT_OK; + +done: + + nxt_mp_destroy(mp); + + if (nxt_fast_path(ret == NXT_OK)) { + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "port rpc fd test passed"); + } + + return ret; +} diff --git a/src/test/nxt_port_use_unless_zero_test.c b/src/test/nxt_port_use_unless_zero_test.c new file mode 100644 index 000000000..72dc3f526 --- /dev/null +++ b/src/test/nxt_port_use_unless_zero_test.c @@ -0,0 +1,378 @@ +/* + * Copyright (C) NGINX, Inc. + */ + +#include +#include +#include "nxt_tests.h" + +#include + + +#define NXT_PORT_UUZ_RACERS 8 +#define NXT_PORT_UUZ_ROUNDS 2000 + + +typedef struct { + nxt_port_t *port; + nxt_atomic_t gen; + nxt_atomic_t done; + nxt_atomic_t released; + nxt_atomic_t wins; + nxt_atomic_t late_wins; + nxt_atomic_t stop; +} nxt_port_uuz_ctx_t; + + +static nxt_int_t nxt_port_uuz_test_semantics(nxt_thread_t *thr); +static nxt_int_t nxt_port_uuz_test_after_last_drop(nxt_thread_t *thr); +static nxt_int_t nxt_port_uuz_test_race(nxt_thread_t *thr); +static void *nxt_port_uuz_racer(void *data); +static nxt_port_t *nxt_port_uuz_port(nxt_task_t *task); +static void nxt_port_uuz_free(nxt_port_t *port); + + +nxt_int_t +nxt_port_use_unless_zero_test(nxt_thread_t *thr) +{ + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port use_unless_zero test started"); + + if (nxt_port_uuz_test_semantics(thr) != NXT_OK) { + return NXT_ERROR; + } + + if (nxt_port_uuz_test_after_last_drop(thr) != NXT_OK) { + return NXT_ERROR; + } + + if (nxt_port_uuz_test_race(thr) != NXT_OK) { + return NXT_ERROR; + } + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port use_unless_zero test passed"); + + return NXT_OK; +} + + +/* A live port hands out references, and the count follows them exactly. */ + +static nxt_int_t +nxt_port_uuz_test_semantics(nxt_thread_t *thr) +{ + nxt_port_t *port; + + port = nxt_port_uuz_port(thr->task); + if (nxt_slow_path(port == NULL)) { + return NXT_ERROR; + } + + if (nxt_slow_path(!nxt_port_use_unless_zero(port))) { + nxt_log_alert(thr->log, "port use_unless_zero test: a live port " + "(use_count %A) refused a reference", port->use_count); + goto fail; + } + + if (nxt_slow_path(port->use_count != 2)) { + nxt_log_alert(thr->log, "port use_unless_zero test: use_count is %A, " + "expected 2", port->use_count); + goto fail; + } + + if (nxt_slow_path(!nxt_port_use_unless_zero(port) + || port->use_count != 3)) + { + nxt_log_alert(thr->log, "port use_unless_zero test: a second " + "reference did not stack (use_count %A)", + port->use_count); + goto fail; + } + + nxt_atomic_fetch_add(&port->use_count, -2); + + if (nxt_slow_path(port->use_count != 1)) { + nxt_log_alert(thr->log, "port use_unless_zero test: use_count is %A " + "after two drops, expected 1", port->use_count); + goto fail; + } + + nxt_port_uuz_free(port); + + return NXT_OK; + +fail: + + nxt_port_uuz_free(port); + + return NXT_ERROR; +} + + +/* + * The regression itself (issue #195). + * + * nxt_process_broadcast_shm_ack() reaches an app port through + * process->ports, which is a lookup: the port may already have dropped its + * last reference and be inside nxt_port_release(), destroying its memory + * pool. The unconditional nxt_atomic_fetch_add() that nxt_port_post() does + * on that pointer resurrects a dead port -- the `use_count == 0` assertion + * in nxt_port_mp_cleanup() in a debug build, a use-after-free in a release + * one. + * + * The state reproduced here is exactly the window: use_count has reached + * zero and the port is still linked, because nxt_port_release() unlinks it + * only afterwards. A try-ref must fail, and must leave the count alone -- + * an implementation that increments first and inspects afterwards leaves 1 + * here and fails the same way the router did. + */ + +static nxt_int_t +nxt_port_uuz_test_after_last_drop(nxt_thread_t *thr) +{ + nxt_int_t ret; + nxt_uint_t i; + nxt_port_t *port; + + port = nxt_port_uuz_port(thr->task); + if (nxt_slow_path(port == NULL)) { + return NXT_ERROR; + } + + ret = NXT_OK; + + /* The last drop, without the release nxt_port_use() would run. */ + nxt_atomic_fetch_add(&port->use_count, -1); + + for (i = 0; i < 3; i++) { + if (nxt_slow_path(nxt_port_use_unless_zero(port))) { + nxt_log_alert(thr->log, "port use_unless_zero test: a port whose " + "last reference is gone handed out one anyway " + "(attempt %ui)", i + 1); + ret = NXT_ERROR; + break; + } + + if (nxt_slow_path(port->use_count != 0)) { + nxt_log_alert(thr->log, "port use_unless_zero test: a refused " + "try-ref changed use_count to %A", port->use_count); + ret = NXT_ERROR; + break; + } + } + + port->use_count = 0; + + nxt_port_uuz_free(port); + + return ret; +} + + +/* + * The same thing under contention, with the ordering the mutex in + * nxt_process_broadcast_shm_ack() and nxt_port_release() guarantees: the + * racers only look the port up after the owner has dropped the last + * reference. Every one of them must be refused. + * + * The rounds also exercise the CAS loop against concurrent increments, so a + * lost update would show up as a non-zero count at the end of a round. + */ + +static nxt_int_t +nxt_port_uuz_test_race(nxt_thread_t *thr) +{ + nxt_int_t ret; + nxt_uint_t i, round; + pthread_t racers[NXT_PORT_UUZ_RACERS]; + nxt_port_uuz_ctx_t ctx; + + nxt_memzero(&ctx, sizeof(ctx)); + + ctx.port = nxt_port_uuz_port(thr->task); + if (nxt_slow_path(ctx.port == NULL)) { + return NXT_ERROR; + } + + /* + * The port outlives every round: the count is put back by hand rather + * than letting the port be released, so that the racers never touch + * freed memory even when the implementation under test is wrong. + */ + ctx.port->use_count = 0; + + for (i = 0; i < NXT_PORT_UUZ_RACERS; i++) { + if (nxt_slow_path(pthread_create(&racers[i], NULL, nxt_port_uuz_racer, + &ctx) != 0)) + { + nxt_log_alert(thr->log, "port use_unless_zero test: " + "pthread_create() failed"); + + ctx.stop = 1; + nxt_atomic_fetch_add(&ctx.gen, 1); + + while (i > 0) { + pthread_join(racers[--i], NULL); + } + + nxt_port_uuz_free(ctx.port); + + return NXT_ERROR; + } + } + + ret = NXT_OK; + + for (round = 0; round < NXT_PORT_UUZ_ROUNDS; round++) { + ctx.port->use_count = 1; + ctx.released = 0; + ctx.late_wins = 0; + ctx.done = 0; + + nxt_atomic_fetch_add(&ctx.gen, 1); + + /* Contended try-refs, which may or may not win. */ + nxt_thread_yield(); + + if (nxt_atomic_fetch_add(&ctx.port->use_count, -1) != 1) { + /* + * A racer that won still holds a reference; wait for it. It + * cannot be held for long -- every winner drops immediately. + */ + while (ctx.port->use_count != 0) { + nxt_thread_yield(); + } + } + + /* From here on no lookup may produce a reference. */ + ctx.released = 1; + + while (ctx.done != NXT_PORT_UUZ_RACERS) { + nxt_thread_yield(); + } + + if (nxt_slow_path(ctx.port->use_count != 0)) { + nxt_log_alert(thr->log, "port use_unless_zero test: use_count is " + "%A after round %ui, expected 0", + ctx.port->use_count, round); + ret = NXT_ERROR; + break; + } + + if (nxt_slow_path(ctx.late_wins != 0)) { + nxt_log_alert(thr->log, "port use_unless_zero test: %A " + "reference(s) taken after the last drop, in round " + "%ui", ctx.late_wins, round); + ret = NXT_ERROR; + break; + } + } + + ctx.stop = 1; + nxt_atomic_fetch_add(&ctx.gen, 1); + + for (i = 0; i < NXT_PORT_UUZ_RACERS; i++) { + pthread_join(racers[i], NULL); + } + + if (ret == NXT_OK && ctx.wins == 0) { + /* + * Not a failure of the code under test, but nothing was contended, + * so say so rather than pass quietly. + */ + nxt_log_error(NXT_LOG_NOTICE, thr->log, "port use_unless_zero test: " + "no contended try-ref won; the race was not exercised"); + } + + ctx.port->use_count = 0; + + nxt_port_uuz_free(ctx.port); + + return ret; +} + + +static void * +nxt_port_uuz_racer(void *data) +{ + nxt_bool_t late; + nxt_atomic_int_t gen, last_gen; + nxt_port_uuz_ctx_t *ctx; + + ctx = data; + last_gen = 0; + + for ( ;; ) { + for ( ;; ) { + gen = ctx->gen; + + if (gen != last_gen) { + break; + } + + nxt_thread_yield(); + } + + last_gen = gen; + + if (ctx->stop) { + return NULL; + } + + /* + * Whether this lookup is racing the owner's last drop or is + * unambiguously after it decides what the outcome is allowed to be: + * a try-ref after the drop must always be refused. + */ + late = (ctx->released != 0); + + if (nxt_port_use_unless_zero(ctx->port)) { + if (late) { + nxt_atomic_fetch_add(&ctx->late_wins, 1); + } + + nxt_atomic_fetch_add(&ctx->wins, 1); + nxt_atomic_fetch_add(&ctx->port->use_count, -1); + } + + nxt_atomic_fetch_add(&ctx->done, 1); + } +} + + +static nxt_port_t * +nxt_port_uuz_port(nxt_task_t *task) +{ + nxt_port_t *port; + + port = nxt_port_new(task, 1, nxt_pid, NXT_PROCESS_APP); + + if (nxt_slow_path(port == NULL)) { + return NULL; + } + + port->pair[0] = -1; + port->pair[1] = -1; + port->socket.fd = -1; + + return port; +} + + +/* + * nxt_port_use() cannot be used here: the last drop goes to + * nxt_port_release(), which needs a runtime this test does not build. The + * pool cleanup that nxt_port_new() registered still runs, and still asserts + * the port is quiescent. + */ + +static void +nxt_port_uuz_free(nxt_port_t *port) +{ + port->use_count = 0; + + nxt_mp_destroy(port->mem_pool); +} diff --git a/src/test/nxt_proto_creating_wedge_test.c b/src/test/nxt_proto_creating_wedge_test.c new file mode 100644 index 000000000..10c861301 --- /dev/null +++ b/src/test/nxt_proto_creating_wedge_test.c @@ -0,0 +1,591 @@ +/* + * Copyright (C) F5, Inc. + */ + +/* + * Regression test for the CREATING-state start wedge: an application worker + * that died between fork() and PROCESS_CREATED was never reported to anyone, + * so the START_PROCESS RPC it was forked for stayed armed forever and the + * record the main process had already made for it was never retired. + * + * Every child of the prototype exists to satisfy a START_PROCESS request, and + * the router arms an RPC handler for each one. A child that got as far as + * PROCESS_CREATED is reported by REMOVE_PID, which carries ->stream, and + * nxt_router_remove_pid_handler() turns a stream-bearing REMOVE_PID into an + * RPC error. A child still in the CREATING state took neither path: + * nxt_proto_sigchld_handler() skipped the notification for it -- rightly, + * since REMOVE_PID says nothing but a pid and under pid isolation the + * prototype does not yet know a globally valid one -- and nothing else + * answered instead. + * + * The prototype's own death is covered: the router keys that RPC on the + * prototype (nxt_router_start_app_process_handler() calls + * nxt_port_rpc_ex_set_peer(), src/nxt_router.c), so REMOVE_PID for the + * prototype reaches nxt_port_rpc_remove_peer() and fails the start. A child + * that dies while CREATING has no such backstop: the armed handler owns an + * app->pending_processes slot, and a leaked slot makes + * nxt_router_app_can_start() false for good once the application's max is + * reached -- the application never starts another process, and a reload + * recovers only if its own config text changed. + * + * Answering the initiator is not the whole job. A worker that got as far as + * WHOAMI made main create a process record and a port holding main's end of + * the worker's port socket (nxt_main_process_whoami_handler()), and REMOVE_PID + * is the only thing that retires it -- nxt_proc_remove_notify_matrix pairs a + * dying APP with MAIN. Once the start RPC is retired and requests retry, a + * worker that keeps dying in that window costs main one record and one + * descriptor per attempt. So the CREATING case must both answer the initiator + * and send a streamless REMOVE_PID: streamless because the initiator has + * already been answered, and a stream-bearing one would fail the same RPC + * twice. + * + * That REMOVE_PID may only be sent when the pid in it is a usable global key, + * which is what rt->is_pid_isolated says -- the same test nxt_process_create() + * uses to decide whether a forked child may enter the runtime hash at all. + * The pid-isolated case is exercised here too: the namespace-local pid must + * not be broadcast, because it names an unrelated process everywhere else, so + * it goes to main alone as REMOVE_CHILD_PID. The router must still see + * nothing but the RPC error. + * + * The answer itself can fail to be written -- a full port queue or a failed + * allocation, neither of which leaves anything for the initiator -- and then + * ->stream must survive so that the REMOVE_PID carries it and the router turns + * it into the RPC error the way it always has for a CREATED child. Clearing + * it regardless would rebuild the wedge, silently. + * + * All four cases are driven against one router port and one main port, + * because the point is the split between them. + */ + +#include +#include +#include +#include +#include +#include "nxt_tests.h" + + +#define NXT_PROTO_WEDGE_CREATING_STREAM 0x0BADF00D +#define NXT_PROTO_WEDGE_ISOLATED_STREAM 0x0BADF00E +#define NXT_PROTO_WEDGE_UNSENT_STREAM 0x0BADF00F +#define NXT_PROTO_WEDGE_STUCK_STREAM 0x0BADF010 +#define NXT_PROTO_WEDGE_CREATED_STREAM 0x0C0FFEE1 + + +typedef struct { + nxt_uint_t type; + uint32_t stream; + nxt_pid_t pid; /* REMOVE_PID payload; 0 for a message without one */ +} nxt_proto_wedge_msg_t; + + +/* + * The ports are never write_ready here, so nxt_port_msg_chk_insert() queues + * whatever the handler writes instead of sending it -- which is exactly what + * this test wants to read back. + */ + +static nxt_int_t +nxt_proto_wedge_expect(nxt_thread_t *thr, nxt_port_t *port, + const nxt_proto_wedge_msg_t *expect, nxt_uint_t n, const char *label, + const char *whom) +{ + nxt_pid_t pid; + nxt_uint_t i; + nxt_queue_link_t *link; + nxt_port_send_msg_t *msg; + const nxt_proto_wedge_msg_t *e; + + link = nxt_queue_first(&port->messages); + + for (i = 0; i < n; i++) { + e = &expect[i]; + + if (link == nxt_queue_tail(&port->messages)) { + nxt_log_alert(thr->log, "proto creating wedge test: %s: nothing " + "more was written to the %s, expected message %d of " + "%d (type %d stream %uxD)", label, whom, (int) i + 1, + (int) n, (int) e->type, e->stream); + return NXT_ERROR; + } + + msg = nxt_queue_link_data(link, nxt_port_send_msg_t, link); + + if (msg->port_msg.type != e->type || msg->port_msg.stream != e->stream) + { + nxt_log_alert(thr->log, "proto creating wedge test: %s: the %s got " + "message %d of type %d stream %uxD (expected type %d " + "stream %uxD)", label, whom, (int) i + 1, + (int) msg->port_msg.type, msg->port_msg.stream, + (int) e->type, e->stream); + return NXT_ERROR; + } + + if (e->pid != 0) { + if (msg->buf == NULL + || nxt_buf_used_size(msg->buf) != sizeof(nxt_pid_t)) + { + nxt_log_alert(thr->log, "proto creating wedge test: %s: the %s " + "got a message carrying no pid", label, whom); + return NXT_ERROR; + } + + nxt_memcpy(&pid, msg->buf->mem.pos, sizeof(nxt_pid_t)); + + if (pid != e->pid) { + nxt_log_alert(thr->log, "proto creating wedge test: %s: the %s " + "was told to remove pid %PI (expected %PI)", + label, whom, pid, e->pid); + return NXT_ERROR; + } + } + + link = nxt_queue_next(link); + } + + if (link != nxt_queue_tail(&port->messages)) { + msg = nxt_queue_link_data(link, nxt_port_send_msg_t, link); + + nxt_log_alert(thr->log, "proto creating wedge test: %s: the %s got an " + "extra message of type %d stream %uxD after the %d " + "expected", label, whom, (int) msg->port_msg.type, + msg->port_msg.stream, (int) n); + return NXT_ERROR; + } + + return NXT_OK; +} + + +static void +nxt_proto_wedge_drain(nxt_task_t *task, nxt_port_t *port) +{ + /* + * Queued messages hold a port reference each; nxt_port_test_run_error + * _handler() is how the other port tests give them back, and it is what + * lets the pool cleanup assertions run under a debug build. + */ + nxt_port_test_run_error_handler(task, port); +} + + +nxt_int_t +nxt_proto_creating_wedge_test(nxt_thread_t *thr) +{ + nxt_mp_t *mp; + nxt_int_t ret; + nxt_task_t *task; + nxt_port_t *router_port, *main_port, *app_port; + nxt_runtime_t *rt, *saved_rt; + nxt_process_t *process; + nxt_event_engine_t engine, *saved_engine; + nxt_proto_wedge_msg_t expect[2]; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "proto creating wedge test started"); + + ret = NXT_ERROR; + app_port = NULL; + main_port = NULL; + router_port = NULL; + + task = thr->task; + task->thread = thr; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + rt = nxt_mp_zalloc(mp, sizeof(nxt_runtime_t)); + if (nxt_slow_path(rt == NULL)) { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + rt->mem_pool = mp; + + if (nxt_slow_path(nxt_thread_mutex_create(&rt->processes_mutex) != NXT_OK)) + { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + /* + * nxt_port_remove_notify_others() allocates its payload from + * task->thread->engine->mem_pool; only the members the reached code + * touches are set, as in src/test/nxt_router_new_port_test.c. + */ + nxt_memzero(&engine, sizeof(engine)); + nxt_work_queue_cache_create(&engine.work_queue_cache, 1024); + engine.fast_work_queue.cache = &engine.work_queue_cache; + nxt_work_queue_name(&engine.fast_work_queue, "fast"); + engine.mem_pool = mp; + + saved_rt = thr->runtime; + saved_engine = thr->engine; + thr->runtime = rt; + thr->engine = &engine; + rt->main_engine = &engine; + + /* + * The port the start request came from. Its pid must differ from + * nxt_pid: nxt_port_remove_notify_others() skips the sender's own + * process, so a router sharing this process's pid would make the + * CREATED case pass for the wrong reason. + */ + + router_port = nxt_runtime_process_port_create(task, rt, nxt_pid + 1, 0, + NXT_PROCESS_ROUTER); + if (nxt_slow_path(router_port == NULL)) { + goto done; + } + + router_port->pair[0] = -1; + router_port->pair[1] = -1; + router_port->socket.fd = -1; + + /* + * The process that answered the worker's WHOAMI and is holding a record + * and a descriptor for it. nxt_proc_remove_notify_matrix pairs a dying + * APP with MAIN, so this is the port the streamless REMOVE_PID has to + * reach. + */ + + main_port = nxt_runtime_process_port_create(task, rt, nxt_pid + 3, 0, + NXT_PROCESS_MAIN); + if (nxt_slow_path(main_port == NULL)) { + goto done; + } + + main_port->pair[0] = -1; + main_port->pair[1] = -1; + main_port->socket.fd = -1; + + /* The child being reaped. Deliberately not in rt->processes: a + CREATING child under pid isolation is kept out of the global pid hash, + and the CREATED case does not need to find itself there either. */ + + process = nxt_mp_zalloc(mp, sizeof(nxt_process_t)); + if (nxt_slow_path(process == NULL)) { + goto done; + } + + process->pid = nxt_pid + 2; + process->isolated_pid = nxt_pid + 2; + nxt_queue_init(&process->ports); + nxt_queue_init(&process->children); + + /* nxt_process_type() reads the first port, and the notify matrix pairs + NXT_PROCESS_APP with NXT_PROCESS_ROUTER and NXT_PROCESS_MAIN. */ + + app_port = nxt_port_new(task, 0, process->pid, NXT_PROCESS_APP); + if (nxt_slow_path(app_port == NULL)) { + goto done; + } + + app_port->pair[0] = -1; + app_port->pair[1] = -1; + app_port->socket.fd = -1; + + /* The fixture's own hold on the record, as main's would be. */ + process->use_count = 1; + + nxt_process_port_add(task, process, app_port); + + /* + * A child that died before PROCESS_CREATED, with the prototype in main's + * pid namespace. The old code left this silent, so the initiator's RPC + * stayed armed for the router's lifetime and main kept the record and the + * descriptor it made at WHOAMI time until the prototype exited. + */ + + process->state = NXT_PROCESS_STATE_CREATING; + process->stream = NXT_PROTO_WEDGE_CREATING_STREAM; + process->stream_pid = router_port->pid; + process->stream_port = router_port->id; + + nxt_proto_child_exited(task, process); + + /* + * The initiator is answered directly, and only then told to drop the pid + * -- streamless, or nxt_router_remove_pid_handler() would fail the same + * RPC a second time. + */ + + expect[0].type = _NXT_PORT_MSG_RPC_ERROR; + expect[0].stream = NXT_PROTO_WEDGE_CREATING_STREAM; + expect[0].pid = 0; + expect[1].type = _NXT_PORT_MSG_REMOVE_PID; + expect[1].stream = 0; + expect[1].pid = process->pid; + + ret = nxt_proto_wedge_expect(thr, router_port, expect, 2, + "a child that died while creating", + "start initiator"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + ret = nxt_proto_wedge_expect(thr, main_port, &expect[1], 1, + "a child that died while creating", + "main process"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + /* + * One answer per start request. Stream identifiers come from a shared + * counter and are reused, so a second error for a stream already retired + * could land on an unrelated RPC. + */ + + if (nxt_slow_path(process->stream != 0)) { + nxt_log_alert(thr->log, "proto creating wedge test: the answered " + "start request is still armed on the process"); + ret = NXT_ERROR; + goto done; + } + + nxt_proto_wedge_drain(task, router_port); + nxt_proto_wedge_drain(task, main_port); + + /* + * The same child under pid isolation. ->pid is then the namespace-local + * pid fork() returned, which REMOVE_PID must not broadcast. Main alone + * is told, with REMOVE_CHILD_PID; the router must see the RPC error and + * nothing else. + */ + + rt->is_pid_isolated = 1; + + process->state = NXT_PROCESS_STATE_CREATING; + process->stream = NXT_PROTO_WEDGE_ISOLATED_STREAM; + + nxt_proto_child_exited(task, process); + + expect[0].type = _NXT_PORT_MSG_RPC_ERROR; + expect[0].stream = NXT_PROTO_WEDGE_ISOLATED_STREAM; + expect[0].pid = 0; + + ret = nxt_proto_wedge_expect(thr, router_port, expect, 1, + "a pid-isolated child that died while " + "creating", "start initiator"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + expect[0].type = _NXT_PORT_MSG_REMOVE_CHILD_PID; + expect[0].stream = 0; + expect[0].pid = process->isolated_pid; + + ret = nxt_proto_wedge_expect(thr, main_port, expect, 1, + "a pid-isolated child that died while " + "creating", "main process"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + rt->is_pid_isolated = 0; + + nxt_proto_wedge_drain(task, router_port); + nxt_proto_wedge_drain(task, main_port); + + /* + * The same child, but the RPC error cannot be written. The prototype + * maps no queue for a router port, so nxt_port_socket_write() answers + * NXT_ERROR here -- nxt_port_msg_chk_insert() failing to allocate -- and + * leaves nothing behind for the initiator. Clearing ->stream regardless + * would send a streamless REMOVE_PID and restore the wedge with no + * diagnostic at all. The stream must survive the failed write instead, + * so that the REMOVE_PID carries it and nxt_router_remove_pid_handler() + * turns it into the RPC error -- the fallback the CREATED case below has + * always relied on. Only the allocation failure can be injected here; it + * exercises the same discarded return. + */ + + process->state = NXT_PROCESS_STATE_CREATING; + process->stream = NXT_PROTO_WEDGE_UNSENT_STREAM; + + nxt_port_test_msg_alloc_failures(1); + + nxt_proto_child_exited(task, process); + + nxt_port_test_msg_alloc_failures(0); + + expect[0].type = _NXT_PORT_MSG_REMOVE_PID; + expect[0].stream = NXT_PROTO_WEDGE_UNSENT_STREAM; + expect[0].pid = process->pid; + + ret = nxt_proto_wedge_expect(thr, router_port, expect, 1, + "a child whose start error could not be " + "written", "start initiator"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + ret = nxt_proto_wedge_expect(thr, main_port, expect, 1, + "a child whose start error could not be " + "written", "main process"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + nxt_proto_wedge_drain(task, router_port); + nxt_proto_wedge_drain(task, main_port); + + /* + * Both at once: pid-isolated, and the RPC error cannot be written. This + * is the one combination nothing can recover locally -- the direct answer + * is the only vehicle under isolation, and the REMOVE_PID that covers the + * failed write otherwise cannot be sent, because ->pid is the + * namespace-local one. + * + * What must not happen is a repair that looks like one: broadcasting that + * pid anyway would ask the router to remove whatever unrelated process + * holds the number. So the router must stay silent here, and main gets + * REMOVE_CHILD_PID -- the injected failure is spent on the RPC error, so + * the report itself is written. + * + * The start RPC is left armed, which the code says out loud. It is + * retired when the prototype itself dies -- the router keys that RPC on + * the prototype (nxt_port_rpc_ex_set_peer(), src/nxt_router.c), so the + * REMOVE_PID main sends for it reaches nxt_port_rpc_remove_peer() -- + * or sooner by "limits": {"start_timeout"} where the application sets + * one. ->stream is still cleared, since nothing may carry it later. + */ + + rt->is_pid_isolated = 1; + + process->state = NXT_PROCESS_STATE_CREATING; + process->stream = NXT_PROTO_WEDGE_STUCK_STREAM; + + nxt_port_test_msg_alloc_failures(1); + + nxt_proto_child_exited(task, process); + + nxt_port_test_msg_alloc_failures(0); + + ret = nxt_proto_wedge_expect(thr, router_port, expect, 0, + "a pid-isolated child whose start error " + "could not be written", "start initiator"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + expect[0].type = _NXT_PORT_MSG_REMOVE_CHILD_PID; + expect[0].stream = 0; + expect[0].pid = process->isolated_pid; + + ret = nxt_proto_wedge_expect(thr, main_port, expect, 1, + "a pid-isolated child whose start error " + "could not be written", "main process"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + if (nxt_slow_path(process->stream != 0)) { + nxt_log_alert(thr->log, "proto creating wedge test: a pid-isolated " + "child whose start error could not be written left the " + "start request armed on the process"); + ret = NXT_ERROR; + goto done; + } + + rt->is_pid_isolated = 0; + + nxt_proto_wedge_drain(task, router_port); + nxt_proto_wedge_drain(task, main_port); + + /* + * A child that reached PROCESS_CREATED keeps the path it always had: + * REMOVE_PID carrying the stream, which the router turns into an RPC + * error of its own, and which retires main's record as it always did. + */ + + process->state = NXT_PROCESS_STATE_CREATED; + process->stream = NXT_PROTO_WEDGE_CREATED_STREAM; + + nxt_proto_child_exited(task, process); + + expect[0].type = _NXT_PORT_MSG_REMOVE_PID; + expect[0].stream = NXT_PROTO_WEDGE_CREATED_STREAM; + expect[0].pid = process->pid; + + ret = nxt_proto_wedge_expect(thr, router_port, expect, 1, + "a child that died after it was created", + "start initiator"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + ret = nxt_proto_wedge_expect(thr, main_port, expect, 1, + "a child that died after it was created", + "main process"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + nxt_proto_wedge_drain(task, router_port); + nxt_proto_wedge_drain(task, main_port); + + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "proto creating wedge test passed"); + + ret = NXT_OK; + +done: + + /* + * Drain first, then hand each port back the way its owner would: these + * two were made by nxt_runtime_process_port_create() and carry pools of + * their own, which the fixture's pool does not cover. Through the + * ownership path rather than by destroying those pools directly -- + * nxt_port_mp_cleanup() asserts the descriptors are gone and the use + * count is zero, so nxt_mp_destroy() here would abort a debug build while + * looking clean in a release one. nxt_port_close() is a no-op on the + * -1 descriptors the fixture set; nxt_runtime_port_remove() drops the + * rt->ports reference, which is the last one, and frees the port. + */ + + if (router_port != NULL) { + nxt_proto_wedge_drain(task, router_port); + + if (!nxt_queue_is_empty(&router_port->messages)) { + nxt_log_alert(thr->log, "proto creating wedge test: the router " + "port still holds queued messages after teardown"); + ret = NXT_ERROR; + } + + nxt_port_close(task, router_port); + nxt_runtime_port_remove(task, router_port); + } + + if (main_port != NULL) { + nxt_proto_wedge_drain(task, main_port); + + if (!nxt_queue_is_empty(&main_port->messages)) { + nxt_log_alert(thr->log, "proto creating wedge test: the main " + "port still holds queued messages after teardown"); + ret = NXT_ERROR; + } + + nxt_port_close(task, main_port); + nxt_runtime_port_remove(task, main_port); + } + + /* Paired through nxt_process_port_add(), so its release unlinks it. */ + if (app_port != NULL) { + nxt_port_use(task, app_port, -1); + } + + thr->engine = saved_engine; + thr->runtime = saved_rt; + + nxt_work_queue_cache_destroy(&engine.work_queue_cache); + nxt_thread_mutex_destroy(&rt->processes_mutex); + nxt_mp_destroy(mp); + + return ret; +} diff --git a/src/test/nxt_regex_test.c b/src/test/nxt_regex_test.c new file mode 100644 index 000000000..0e2093de3 --- /dev/null +++ b/src/test/nxt_regex_test.c @@ -0,0 +1,165 @@ +/* + * Copyright (C) FreeUnit contributors. + */ + +/* + * A pattern with two greedy groups and a class that the subject lacks + * takes about 2M match steps on 2000 separators on PCRE2 10.4x and PCRE + * 8.39. The match must stop at NXT_REGEX_MATCH_LIMIT and report an error. + * + * A repeated group takes about 2 match steps per byte. On PCRE2 10.30 and + * later it must still match on an 8 KiB subject. PCRE 1 and PCRE2 before + * 10.30 also nest about 2 calls per byte on the stack. On an 8 KiB subject + * they must stop at the recursion limit and report an error, not overflow + * the stack. On a 512-byte subject they must match. + * + * The library counts match steps from zero again at each start position of + * a pattern that is not anchored. In PCRE2 10.42, "(?=a)(?:a|aa){0,12}[xy]" + * takes 8,193 steps and 28,659 callouts at each of the first 976 positions + * of 1000 "a" bytes and an "x". This is below the match limit at each + * position, but the whole match takes about 1 s. The callouts must stop it + * at NXT_REGEX_MATCH_LIMIT in total, with an error. The same pattern with + * "^" has one start position and must not match. + */ + +#include +#include +#include "nxt_tests.h" + +#if (NXT_HAVE_PCRE2) +#define PCRE2_CODE_UNIT_WIDTH 8 +#include +#endif + + +#define NXT_REGEX_TEST_SIZE 8192 +#define NXT_REGEX_TEST_SHORT 512 +#define NXT_REGEX_TEST_STARTS 1000 + +#if (NXT_HAVE_PCRE2 && (PCRE2_MAJOR > 10 || PCRE2_MINOR >= 30)) +#define NXT_REGEX_TEST_LONG_RESULT 1 +#else +#define NXT_REGEX_TEST_LONG_RESULT NXT_ERROR +#endif + + +/* + * Compile the pattern and match it against the subject. The match result + * goes to *result. The function returns NXT_ERROR if the setup fails. + */ + +static nxt_int_t +nxt_regex_test_run(nxt_thread_t *thr, nxt_mp_t *mp, const char *pattern, + u_char *subject, size_t length, nxt_int_t *result) +{ + nxt_str_t source; + nxt_regex_t *re; + nxt_regex_err_t err; + nxt_regex_match_t *match; + + source.start = (u_char *) pattern; + source.length = nxt_strlen(pattern); + + re = nxt_regex_compile(mp, &source, &err); + if (re == NULL) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, "regex test: compile of " + "\"%s\" failed: %s", pattern, err.msg); + return NXT_ERROR; + } + + match = nxt_regex_match_create(mp, 1); + if (match == NULL) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, "regex test: match data " + "for \"%s\" failed", pattern); + return NXT_ERROR; + } + + *result = nxt_regex_match(re, subject, length, match); + + return NXT_OK; +} + + +nxt_int_t +nxt_regex_test(nxt_thread_t *thr) +{ + u_char *subject; + nxt_mp_t *mp; + nxt_int_t ret, slow, deep, shallow, starts, anchored; + + nxt_thread_time_update(thr); + + mp = nxt_mp_create(1024, 128, 256, 32); + if (mp == NULL) { + return NXT_ERROR; + } + + subject = nxt_mp_alloc(mp, NXT_REGEX_TEST_SIZE); + if (subject == NULL) { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + subject[0] = '/'; + nxt_memset(subject + 1, '/', 2000); + + ret = nxt_regex_test_run(thr, mp, "^/(.*)/(.*)[xy]$", subject, 2001, + &slow); + + if (ret == NXT_OK) { + nxt_memset(subject + 1, 'a', NXT_REGEX_TEST_SIZE - 1); + + ret = nxt_regex_test_run(thr, mp, "^/(a|b)+$", subject, + NXT_REGEX_TEST_SIZE, &deep); + } + + if (ret == NXT_OK) { + ret = nxt_regex_test_run(thr, mp, "^/(a|b)+$", subject, + NXT_REGEX_TEST_SHORT, &shallow); + } + + if (ret == NXT_OK) { + nxt_memset(subject, 'a', NXT_REGEX_TEST_STARTS); + subject[NXT_REGEX_TEST_STARTS] = 'x'; + + ret = nxt_regex_test_run(thr, mp, "(?=a)(?:a|aa){0,12}[xy]", subject, + NXT_REGEX_TEST_STARTS + 1, &starts); + } + + if (ret == NXT_OK) { + ret = nxt_regex_test_run(thr, mp, "^(?=a)(?:a|aa){0,12}[xy]", + subject, NXT_REGEX_TEST_STARTS + 1, + &anchored); + } + + nxt_mp_destroy(mp); + + if (ret != NXT_OK) { + return NXT_ERROR; + } + + NXT_TEST_CHECK(thr->log, slow == NXT_ERROR, "regex test: backtracking " + "pattern returned %i, expected %i", slow, + (nxt_int_t) NXT_ERROR); + + NXT_TEST_CHECK(thr->log, deep == NXT_REGEX_TEST_LONG_RESULT, "regex test: " + "repeated group on %d bytes returned %i, expected %i", + NXT_REGEX_TEST_SIZE, deep, + (nxt_int_t) NXT_REGEX_TEST_LONG_RESULT); + + NXT_TEST_CHECK(thr->log, shallow == 1, "regex test: repeated group " + "on %d bytes returned %i, expected 1", + NXT_REGEX_TEST_SHORT, shallow); + + NXT_TEST_CHECK(thr->log, starts == NXT_ERROR, "regex test: pattern " + "without an anchor on %d bytes returned %i, expected %i", + NXT_REGEX_TEST_STARTS + 1, starts, (nxt_int_t) NXT_ERROR); + + NXT_TEST_CHECK(thr->log, anchored == 0, "regex test: anchored pattern " + "on %d bytes returned %i, expected 0", + NXT_REGEX_TEST_STARTS + 1, anchored); + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "regex test passed"); + + return NXT_OK; +} diff --git a/src/test/nxt_router_app_timeout_test.c b/src/test/nxt_router_app_timeout_test.c new file mode 100644 index 000000000..474d29331 --- /dev/null +++ b/src/test/nxt_router_app_timeout_test.c @@ -0,0 +1,546 @@ +/* + * Copyright (C) F5, Inc. + */ + +/* + * Regression test for the request deadline answering a request a worker is + * already running. + * + * nxt_router_app_timeout() serves two deadlines. After an acknowledgement a + * worker holds the request and 503 is the answer. Before one the request is + * still in the application's shared port queue, and whether it can be taken + * back is decided by a single CAS: nxt_app_queue_cancel() and the worker's + * nxt_unit_app_queue_recv() run the same compare-and-swap on the queue item's + * tracking word, so exactly one of them wins. The handler used to answer 503 + * and unlink regardless of who won, which failed a request that was executing + * -- and ran it a second time if the client retried. + * + * Nothing about that window can be reached by racing a real application: it is + * one CAS wide. So the CAS is driven here instead. The "worker" is this test + * calling nxt_app_queue_recv() and nxt_app_queue_cancel() exactly as + * nxt_unit_app_queue_recv() (src/nxt_unit.c) does, before or after the + * deadline fires, and the handler runs through the engine's real timer + * machinery. + * + * Three things are pinned down: + * + * - a claimed request is not answered and its message is left intact, since + * the tail of that chain is the body the acknowledgement still has to + * send; + * - a later expiry does not answer it either: until the acknowledgement + * arrives, an unlink would drop that body and cancel the RPC, and libunit + * would keep the request for ever; + * - a request nobody claimed is still answered at once, and its retraction + * really takes the message out of the queue. + * + * The request is a hand-built nxt_http_request_t with ->header_sent set and no + * protocol: nxt_http_request_error() then stops at its own guard and reduces + * to nxt_http_request_error_handler(), which sets ->error and nothing else. + * That makes ->error the mark of "this expiry answered the request" without + * dragging a connection into the fixture. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "nxt_tests.h" + + +/* File scope: the deadline hands the application to the router's + reference-counting helpers, so its storage must outlive the frame. */ +static nxt_app_t nxt_router_app_timeout_test_app; + +static nxt_uint_t nxt_router_app_timeout_test_completions; + + +typedef struct { + nxt_http_request_t *r; + nxt_request_rpc_data_t *req_rpc_data; + nxt_buf_t *header; + nxt_buf_t *body; +} nxt_router_app_timeout_test_req_t; + + +static void +nxt_router_app_timeout_test_completion(nxt_task_t *task, void *obj, void *data) +{ + nxt_router_app_timeout_test_completions++; +} + + +/* + * Commit the pending timer changes and run what is due, the way + * nxt_event_engine_start() does. nxt_timer_find() folds the change list into + * the rbtree, nxt_timer_expire() queues the handlers that are due, and the + * drain runs them -- including a handler that re-arms its own timer, which is + * what the claimed request does. + */ + +static void +nxt_router_app_timeout_test_tick(nxt_task_t *task, nxt_event_engine_t *engine, + nxt_msec_t advance) +{ + nxt_task_t *wq_task; + nxt_work_handler_t handler; + void *obj, *data; + + (void) nxt_timer_find(engine); + + nxt_timer_expire(engine, engine->timers.now + advance); + + /* nxt_work_queue_pop() dereferences the head unconditionally. */ + + while (engine->fast_work_queue.head != NULL) { + handler = nxt_work_queue_pop(&engine->fast_work_queue, &wq_task, &obj, + &data); + + if (handler == NULL) { + continue; + } + + handler(wq_task != NULL ? wq_task : task, obj, data); + } +} + + +/* + * What a worker does with a queue item, from nxt_unit_app_queue_recv(): + * dequeue it, then claim it with the same CAS the router's retraction uses. + * Answers whether this side won. + */ + +static nxt_bool_t +nxt_router_app_timeout_test_claim(nxt_app_queue_t *queue) +{ + uint32_t cookie; + ssize_t size; + uint8_t rbuf[NXT_APP_QUEUE_MSG_SIZE]; + nxt_port_msg_t *port_msg; + + size = nxt_app_queue_recv(queue, rbuf, &cookie); + + if (size < (ssize_t) sizeof(nxt_port_msg_t)) { + return 0; + } + + port_msg = (nxt_port_msg_t *) rbuf; + + return nxt_app_queue_cancel(queue, cookie, port_msg->stream); +} + + +/* + * A request in the state nxt_router_app_port_get() leaves behind: its message + * in the shared port queue, itself parked in ack_waiting_req with no worker, + * and the deadline armed. Each leg gets its own request, because the unlink + * that ends one leaves that request's timer armed for the pool release. + */ + +static nxt_int_t +nxt_router_app_timeout_test_arm(nxt_task_t *task, nxt_event_engine_t *engine, + nxt_mp_t *mp, nxt_app_t *app, uint32_t stream, + nxt_router_app_timeout_test_req_t *t) +{ + int notify; + nxt_buf_t *header, *body; + nxt_http_request_t *r; + nxt_request_rpc_data_t *req_rpc_data; + + struct { + nxt_port_msg_t pm; + nxt_port_mmap_msg_t mm; + } nxt_packed msg; + + r = nxt_mp_zalloc(mp, sizeof(nxt_http_request_t)); + req_rpc_data = nxt_mp_zalloc(mp, sizeof(nxt_request_rpc_data_t)); + header = nxt_mp_zalloc(mp, sizeof(nxt_buf_t)); + body = nxt_mp_zalloc(mp, sizeof(nxt_buf_t)); + + if (nxt_slow_path(r == NULL || req_rpc_data == NULL + || header == NULL || body == NULL)) + { + return NXT_ERROR; + } + + r->mem_pool = mp; + r->engine = engine; + + /* See the file comment: keeps the 503 out of the protocol layer. */ + r->header_sent = 1; + r->proto.any = NULL; + + r->timer.task = &engine->task; + r->timer.work_queue = &engine->fast_work_queue; + r->timer.log = engine->task.log; + r->timer.bias = NXT_TIMER_DEFAULT_BIAS; + r->timer.handler = nxt_router_test_app_timeout; + r->timer_data = req_rpc_data; + + r->req_rpc_data = req_rpc_data; + + req_rpc_data->stream = stream; + req_rpc_data->app = app; + req_rpc_data->app_port = app->shared_port; + req_rpc_data->apr_action = NXT_APR_REQUEST_FAILED; + req_rpc_data->request = r; + req_rpc_data->msg_info.body_fd = -1; + + /* The RPC registry holds nothing for this stream. */ + req_rpc_data->rpc_cancel = 0; + + nxt_port_inc_use(app->shared_port); + + /* + * The message chain nxt_router_app_prepare_request() leaves behind: the + * header buffer that went through the queue, marked sent, and the body + * behind it, which nxt_router_req_headers_ack_handler() sends only once a + * worker acknowledges. + */ + + header->completion_handler = nxt_router_app_timeout_test_completion; + header->is_port_mmap_sent = 1; + header->next = body; + + body->completion_handler = nxt_router_app_timeout_test_completion; + + req_rpc_data->msg_info.buf = header; + + nxt_memzero(&msg, sizeof(msg)); + msg.pm.stream = stream; + msg.pm.type = NXT_PORT_MSG_REQ_HEADERS; + msg.pm.mmap = 1; + + if (nxt_slow_path(nxt_app_queue_send(app->shared_port->queue, &msg, + sizeof(msg), stream, ¬ify, + &req_rpc_data->msg_info.tracking_cookie) != NXT_OK)) + { + return NXT_ERROR; + } + + nxt_thread_mutex_lock(&app->mutex); + + app->active_requests++; + nxt_queue_insert_tail(&app->ack_waiting_req, &r->app_link); + + nxt_thread_mutex_unlock(&app->mutex); + + nxt_mp_retain(mp); + + nxt_timer_add(engine, &r->timer, app->timeout); + + t->r = r; + t->req_rpc_data = req_rpc_data; + t->header = header; + t->body = body; + + return NXT_OK; +} + + +nxt_int_t +nxt_router_app_timeout_test(nxt_thread_t *thr) +{ + nxt_mp_t *mp, *req_mp; + nxt_app_t *app; + nxt_int_t ret; + nxt_task_t *task; + nxt_port_t *shared_port; + nxt_bool_t app_mutex, timers; + nxt_runtime_t *rt, *saved_rt; + nxt_app_queue_t *queue; + nxt_event_engine_t engine, *saved_engine; + nxt_router_app_timeout_test_req_t claimed, unclaimed; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "router app timeout test started"); + + ret = NXT_ERROR; + shared_port = NULL; + req_mp = NULL; + app_mutex = 0; + timers = 0; + app = &nxt_router_app_timeout_test_app; + + task = thr->task; + task->thread = thr; + + saved_rt = thr->runtime; + saved_engine = thr->engine; + + if (nxt_slow_path(nxt_port_rpc_init() != NXT_OK)) { + return NXT_ERROR; + } + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + rt = nxt_mp_zalloc(mp, sizeof(nxt_runtime_t)); + queue = nxt_mp_zalloc(mp, sizeof(nxt_app_queue_t)); + if (nxt_slow_path(rt == NULL || queue == NULL)) { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + rt->mem_pool = mp; + + if (nxt_slow_path(nxt_thread_mutex_create(&rt->processes_mutex) != NXT_OK)) + { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + nxt_memzero(&engine, sizeof(engine)); + nxt_work_queue_cache_create(&engine.work_queue_cache, 1024); + engine.fast_work_queue.cache = &engine.work_queue_cache; + nxt_work_queue_name(&engine.fast_work_queue, "fast"); + engine.mem_pool = mp; + + /* The deadline runs on the engine's own task, as it does in the router. */ + + engine.task.thread = thr; + engine.task.log = thr->log; + + /* + * Real timer machinery: the re-arm under test is an nxt_timer_add() from + * inside the timer's own handler, and a memzero'd nxt_timers_t would let a + * re-arm that never lands look correct. + */ + + if (nxt_slow_path(nxt_timers_init(&engine.timers, 64) != NXT_OK)) { + goto done; + } + + timers = 1; + + thr->runtime = rt; + thr->engine = &engine; + rt->main_engine = &engine; + + /* The port every worker of the application reads, and its queue. */ + + shared_port = nxt_port_new(task, NXT_SHARED_PORT_ID, nxt_pid, + NXT_PROCESS_APP); + if (nxt_slow_path(shared_port == NULL)) { + goto done; + } + + shared_port->pair[0] = -1; + shared_port->pair[1] = -1; + shared_port->socket.fd = -1; + shared_port->queue_fd = -1; + + nxt_app_queue_init(queue); + shared_port->queue = queue; + + nxt_memzero(app, sizeof(nxt_app_t)); + + if (nxt_slow_path(nxt_thread_mutex_create(&app->mutex) != NXT_OK)) { + goto done; + } + + app_mutex = 1; + + nxt_queue_init(&app->ports); + nxt_queue_init(&app->spare_ports); + nxt_queue_init(&app->idle_ports); + nxt_queue_init(&app->ack_waiting_req); + + nxt_str_set(&app->name, "app-timeout-test"); + + app->shared_port = shared_port; + app->timeout = 1000; + app->processes = 1; + + /* Never reached by the releases below, so the deadline cannot free it. */ + app->use_count = 8; + + /* + * The pool the requests live in. Retained beyond what the legs release, + * so nothing frees a request out from under the assertions that follow. + */ + + req_mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(req_mp == NULL)) { + goto done; + } + + nxt_mp_retain(req_mp); + nxt_mp_retain(req_mp); + + /* + * Leg 1: a worker claims the slot before the deadline fires. The + * retraction inside the handler must lose, and the handler must then leave + * the request alone. + */ + + if (nxt_slow_path(nxt_router_app_timeout_test_arm(task, &engine, req_mp, + app, 0x4321, &claimed) + != NXT_OK)) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router app timeout test: could not arm a request"); + goto done; + } + + if (nxt_slow_path(!nxt_router_app_timeout_test_claim(queue))) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router app timeout test: the worker could not claim " + "the queued message"); + goto done; + } + + nxt_router_app_timeout_test_completions = 0; + + nxt_router_app_timeout_test_tick(task, &engine, app->timeout); + + if (claimed.req_rpc_data->msg_info.cancel != NXT_MSG_CLAIMED) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router app timeout test: the deadline did not see the " + "claim (cancel %d)", + (int) claimed.req_rpc_data->msg_info.cancel); + goto done; + } + + if (claimed.r->error != 0 || claimed.r->req_rpc_data == NULL) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router app timeout test: a claimed request was " + "answered (error %d, rpc data %p)", + (int) claimed.r->error, claimed.r->req_rpc_data); + goto done; + } + + if (claimed.req_rpc_data->msg_info.buf != claimed.header + || claimed.header->next != claimed.body + || nxt_router_app_timeout_test_completions != 0) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router app timeout test: the message of a claimed " + "request was completed (%ui buffers), so the " + "acknowledgement has no body left to send", + nxt_router_app_timeout_test_completions); + goto done; + } + + if (claimed.r->timer.enabled) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router app timeout test: the deadline was re-armed " + "for a claimed request with no acknowledgement"); + goto done; + } + + /* + * Leg 2: the acknowledgement is late. Later expiries must not answer the + * request, because the acknowledgement still has to send the body. + */ + + nxt_router_app_timeout_test_tick(task, &engine, app->timeout); + nxt_router_app_timeout_test_tick(task, &engine, 2 * app->timeout); + + if (claimed.r->error != 0 || claimed.r->req_rpc_data == NULL + || claimed.req_rpc_data->msg_info.buf != claimed.header + || claimed.header->next != claimed.body + || nxt_router_app_timeout_test_completions != 0) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router app timeout test: a claimed request was " + "answered before its acknowledgement (error %d, " + "%ui buffers)", + (int) claimed.r->error, + nxt_router_app_timeout_test_completions); + goto done; + } + + /* + * Leg 3: the control. Nobody claims this one, so the retraction wins and + * the deadline answers at once -- the behaviour the fix must not change. + */ + + if (nxt_slow_path(nxt_router_app_timeout_test_arm(task, &engine, req_mp, + app, 0x4322, &unclaimed) + != NXT_OK)) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router app timeout test: could not arm a request"); + goto done; + } + + nxt_router_app_timeout_test_completions = 0; + + nxt_router_app_timeout_test_tick(task, &engine, app->timeout); + + if (unclaimed.req_rpc_data->msg_info.cancel != NXT_MSG_RETRACTED + || unclaimed.r->error == 0 || unclaimed.r->req_rpc_data != NULL) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router app timeout test: an unclaimed request was not " + "answered (cancel %d, error %d)", + (int) unclaimed.req_rpc_data->msg_info.cancel, + (int) unclaimed.r->error); + goto done; + } + + if (nxt_router_app_timeout_test_completions != 2 + || unclaimed.header->is_port_mmap_sent != 0) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router app timeout test: a retracted message did not " + "give its shared memory back (%ui buffers, sent %d)", + nxt_router_app_timeout_test_completions, + (int) unclaimed.header->is_port_mmap_sent); + goto done; + } + + /* + * The retraction has to be real, not just reported: a worker reaching the + * item now must find it cancelled and skip it, which is what + * nxt_unit_app_queue_recv() does with a CAS that answers false. + */ + + if (nxt_router_app_timeout_test_claim(queue)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router app timeout test: a worker still claimed a " + "retracted message"); + goto done; + } + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "router app timeout test passed"); + + ret = NXT_OK; + +done: + + if (shared_port != NULL) { + nxt_port_use(task, shared_port, -1); + } + + thr->engine = saved_engine; + thr->runtime = saved_rt; + + if (app_mutex) { + nxt_thread_mutex_destroy(&app->mutex); + } + + if (timers) { + nxt_free(engine.timers.changes); + } + + nxt_work_queue_cache_destroy(&engine.work_queue_cache); + nxt_thread_mutex_destroy(&rt->processes_mutex); + + if (req_mp != NULL) { + nxt_mp_destroy(req_mp); + } + + nxt_mp_destroy(mp); + + return ret; +} diff --git a/src/test/nxt_router_detached_test.c b/src/test/nxt_router_detached_test.c new file mode 100644 index 000000000..1153bd0be --- /dev/null +++ b/src/test/nxt_router_detached_test.c @@ -0,0 +1,360 @@ +/* + * Copyright (C) F5, Inc. + */ + +/* + * Regression test for the sender check in nxt_router_detached_handler() + * (src/nxt_router.c). + * + * Every application worker holds a copy of the router port, and the pid in + * a port message is chosen by the sender. A detached edge that named + * another worker's pid used to be applied to that worker: a forged start + * pinned it busy together with a reference to its application, and a forged + * finish let the reaper take a worker that was still running. The edge is + * now applied only when the kernel's credential names the same process. + * + * The fixture is one application with the main port of one worker. The port + * has no socket (pair[1] is -1) and is in no idle queue, so the edges change + * only ->detached, the count and the application reference, and those are + * what the test reads. Each forged edge is followed by the genuine one, which + * shows that the fixture reaches the accounting at all. + * + * It also covers the counting the accounting rests on: two starts and one + * finish leave the worker detached, the second finish releases it, and a + * finish that matches no start is refused without disturbing either count. + * The first such edge logs an alert, which is the expected output; later ones + * go to the debug log. Two presets reach the branches the application's + * edges alone do not: a start while the router holds its own mark, and a + * start at the maximum count. + */ + +#include +#include +#include +#include +#include +#include "nxt_tests.h" + + +#if (NXT_USE_CMSG_PID) + +/* Which credential the edge arrives with. */ + +#define NXT_ROUTER_DETACHED_TEST_SELF 0 +#define NXT_ROUTER_DETACHED_TEST_OTHER 1 +#define NXT_ROUTER_DETACHED_TEST_NONE 2 + + +/* + * The accounting an edge finds. An edge with a preset starts from it rather + * than from what the previous edge left, which is how the test reaches the + * states only the router's own path or a long run of edges would produce. + */ + +typedef struct { + nxt_atomic_uint_t use_count; + uint32_t detached_app; + uint32_t detached_router; + uint32_t processes; + uint8_t detached; +} nxt_router_detached_test_state_t; + + +typedef struct { + const nxt_router_detached_test_state_t *preset; + const char *name; + uint32_t detached_app; + uint32_t processes; + nxt_atomic_uint_t use_count; + uint8_t state; + uint8_t sender; + uint8_t detached; + uint8_t alerted; +} nxt_router_detached_test_edge_t; + + +/* + * What nxt_router_app_abandon() leaves on an idle port: the router's own + * mark, the detached state it set, and the application reference that state + * holds. + */ + +static const nxt_router_detached_test_state_t + nxt_router_detached_test_router_mark = { 9, 0, 1, 1, 1 }; + + +/* A worker whose count has reached the maximum. */ + +static const nxt_router_detached_test_state_t + nxt_router_detached_test_saturated = { 9, UINT32_MAX, 0, 1, 1 }; + + +#define NXT_RDT_START NXT_PORT_DETACHED_START +#define NXT_RDT_FINISH NXT_PORT_DETACHED_FINISH +#define NXT_RDT_SELF NXT_ROUTER_DETACHED_TEST_SELF +#define NXT_RDT_OTHER NXT_ROUTER_DETACHED_TEST_OTHER +#define NXT_RDT_NONE NXT_ROUTER_DETACHED_TEST_NONE + + +/* + * The edges, and the accounting each one must leave behind. The start and + * the finish are a counted pair: only the first start and the last finish + * move ->detached, app->detached_processes and the application reference, + * and a finish that matches no start moves none of them. A refused edge is + * an alert the first time on a port only. + * + * A start that finds the router's own mark already set changes no state, so + * the reference it took goes straight back: the state's reference belongs to + * nxt_router_app_abandoned_settle(). The finish that follows leaves the + * state to the router too. A start at the maximum count is refused rather + * than wrapped to "no detached work". + */ + +static const nxt_router_detached_test_edge_t + nxt_router_detached_test_edges[] = +{ + { NULL, "a forged start", + 0, 0, 8, NXT_RDT_START, NXT_RDT_OTHER, 0, 0 }, + { NULL, "a genuine start", + 1, 1, 9, NXT_RDT_START, NXT_RDT_SELF, 1, 0 }, + { NULL, "a forged finish", + 1, 1, 9, NXT_RDT_FINISH, NXT_RDT_OTHER, 1, 0 }, + { NULL, "a finish with no credential", + 1, 1, 9, NXT_RDT_FINISH, NXT_RDT_NONE, 1, 0 }, + { NULL, "a second start", + 2, 1, 9, NXT_RDT_START, NXT_RDT_SELF, 1, 0 }, + { NULL, "the first of two finishes", + 1, 1, 9, NXT_RDT_FINISH, NXT_RDT_SELF, 1, 0 }, + { NULL, "the last of two finishes", + 0, 0, 8, NXT_RDT_FINISH, NXT_RDT_SELF, 0, 0 }, + { NULL, "an unmatched finish", + 0, 0, 8, NXT_RDT_FINISH, NXT_RDT_SELF, 0, 1 }, + { NULL, "a second unmatched finish", + 0, 0, 8, NXT_RDT_FINISH, NXT_RDT_SELF, 0, 1 }, + { NULL, "a start after an unmatched finish", + 1, 1, 9, NXT_RDT_START, NXT_RDT_SELF, 1, 1 }, + { NULL, "the matching finish", + 0, 0, 8, NXT_RDT_FINISH, NXT_RDT_SELF, 0, 1 }, + { &nxt_router_detached_test_router_mark, + "a start while the router holds its own mark", + 1, 1, 9, NXT_RDT_START, NXT_RDT_SELF, 1, 1 }, + { NULL, "a finish while the router holds its own mark", + 0, 1, 9, NXT_RDT_FINISH, NXT_RDT_SELF, 1, 1 }, + { &nxt_router_detached_test_saturated, "a start at the maximum count", + UINT32_MAX, 1, 9, NXT_RDT_START, NXT_RDT_SELF, 1, 1 }, + { NULL, "a finish below the maximum count", + UINT32_MAX - 1, 1, 9, NXT_RDT_FINISH, NXT_RDT_SELF, 1, 1 }, +}; + + +static nxt_int_t +nxt_router_detached_test_edge(nxt_thread_t *thr, nxt_task_t *task, + nxt_port_t *port, nxt_app_t *app, + const nxt_router_detached_test_edge_t *e, nxt_pid_t sender) +{ + uint8_t state; + nxt_buf_t b; + nxt_port_recv_msg_t msg; + + state = e->state; + + nxt_memzero(&b, sizeof(nxt_buf_t)); + b.mem.start = &state; + b.mem.pos = &state; + b.mem.free = &state + 1; + b.mem.end = &state + 1; + + nxt_memzero(&msg, sizeof(nxt_port_recv_msg_t)); + msg.buf = &b; + msg.size = 1; + msg.fd[0] = -1; + msg.fd[1] = -1; + msg.port_msg.pid = port->pid; + msg.port_msg.type = _NXT_PORT_MSG_DETACHED; + msg.port_msg.last = 1; + msg.cmsg_pid = sender; + + if (e->preset != NULL) { + port->detached_app = e->preset->detached_app; + port->detached_router = e->preset->detached_router; + port->detached = e->preset->detached; + app->detached_processes = e->preset->processes; + app->use_count = e->preset->use_count; + } + + nxt_router_test_detached_handler(task, &msg); + + if (port->detached_app != e->detached_app + || port->detached != e->detached + || port->detached_alerted != e->alerted + || app->detached_processes != e->processes + || app->use_count != e->use_count) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router detached test: %s: detached_app %uD, " + "detached %d, alerted %d, count %uD, use count %d; " + "expected %uD, %d, %d, %uD, %d", + e->name, port->detached_app, (int) port->detached, + (int) port->detached_alerted, app->detached_processes, + (int) app->use_count, e->detached_app, + (int) e->detached, (int) e->alerted, e->processes, + (int) e->use_count); + return NXT_ERROR; + } + + return NXT_OK; +} + +#endif + + +nxt_int_t +nxt_router_detached_test(nxt_thread_t *thr) +{ +#if !(NXT_USE_CMSG_PID) + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router detached test skipped: no sender credentials"); + return NXT_OK; + +#else + + nxt_mp_t *mp; + nxt_app_t *app; + nxt_int_t ret; + nxt_uint_t i; + nxt_pid_t pid, other, sender; + nxt_task_t *task; + nxt_port_t *port; + nxt_bool_t app_mutex, hashed; + nxt_runtime_t *rt, *saved_rt; + nxt_event_engine_t engine, *saved_engine; + + const nxt_router_detached_test_edge_t *e; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "router detached test started"); + + ret = NXT_ERROR; + port = NULL; + app_mutex = 0; + hashed = 0; + + task = thr->task; + task->thread = thr; + + saved_rt = thr->runtime; + saved_engine = thr->engine; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + rt = nxt_mp_zalloc(mp, sizeof(nxt_runtime_t)); + app = nxt_mp_zalloc(mp, sizeof(nxt_app_t)); + if (nxt_slow_path(rt == NULL || app == NULL)) { + goto done; + } + + rt->mem_pool = mp; + + /* The handler runs on the main engine only. */ + + nxt_memzero(&engine, sizeof(nxt_event_engine_t)); + + thr->runtime = rt; + thr->engine = &engine; + rt->main_engine = &engine; + + if (nxt_slow_path(nxt_thread_mutex_create(&app->mutex) != NXT_OK)) { + goto done; + } + + app_mutex = 1; + + nxt_queue_init(&app->ports); + nxt_queue_init(&app->spare_ports); + nxt_queue_init(&app->idle_ports); + nxt_queue_init(&app->ack_waiting_req); + + nxt_str_set(&app->name, "detached-test"); + + /* Never reached by the edges below, so nothing frees the application. */ + app->use_count = 8; + + /* A pid that is not this process, so a sender of nxt_pid is foreign. */ + + pid = nxt_pid + 1; + other = nxt_pid; + + port = nxt_port_new(task, 0, pid, NXT_PROCESS_APP); + if (nxt_slow_path(port == NULL)) { + goto done; + } + + port->pair[0] = -1; + port->pair[1] = -1; + port->socket.fd = -1; + port->app = app; + + if (nxt_slow_path(nxt_port_hash_add(&rt->ports, port) != NXT_OK)) { + goto done; + } + + hashed = 1; + + for (i = 0; i < nxt_nitems(nxt_router_detached_test_edges); i++) { + e = &nxt_router_detached_test_edges[i]; + + switch (e->sender) { + case NXT_ROUTER_DETACHED_TEST_SELF: + sender = pid; + break; + + case NXT_ROUTER_DETACHED_TEST_OTHER: + sender = other; + break; + + default: + sender = -1; + break; + } + + if (nxt_slow_path(nxt_router_detached_test_edge(thr, task, port, app, + e, sender) + != NXT_OK)) + { + goto done; + } + } + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "router detached test passed"); + + ret = NXT_OK; + +done: + + if (hashed) { + (void) nxt_port_hash_remove(&rt->ports, port); + } + + if (port != NULL) { + port->app = NULL; + nxt_port_use(task, port, -1); + } + + thr->engine = saved_engine; + thr->runtime = saved_rt; + + if (app_mutex) { + nxt_thread_mutex_destroy(&app->mutex); + } + + nxt_mp_destroy(mp); + + return ret; + +#endif +} diff --git a/src/test/nxt_router_new_port_test.c b/src/test/nxt_router_new_port_test.c new file mode 100644 index 000000000..e26a62fb9 --- /dev/null +++ b/src/test/nxt_router_new_port_test.c @@ -0,0 +1,396 @@ +/* + * Copyright (C) F5, Inc. + */ + +/* + * Regression test for descriptor ownership in + * nxt_router_new_port_handler() (src/nxt_router.c). + * + * An application NEW_PORT carries the worker's socket in fd[0] and the + * queue it wants the router to map in fd[1]. The router maps the queue + * and closes fd[1] afterwards -- but the failure branch of that mapping + * used to return before the close, leaking the descriptor. + * + * That branch used to need the router to be out of address space, which is + * why it went unnoticed. Since the queue object is validated for size, a + * peer decides when it is taken: it hands over a short object and the + * router refuses it. The dispatcher reclaims nothing once a handler has + * returned (see nxt_port_recv_msg_close_fds()), so each refusal cost the + * router a descriptor, which is the exhaustion the validation exists to + * prevent. + * + * The port keeps fd[0]: it is the port's socket, and the port outlives the + * message. Only fd[1] is the message's to lose. + * + * The refusal itself is checked here too: the port must come back without a + * queue. A descriptor accounted for proves nothing about the mapping, and + * the size check is what keeps a short object off the port -- mmap() + * succeeds on one, and the router faults on the first access past its last + * page. + */ + +#include +#include +#include +#include +#include +#include +#include "nxt_tests.h" + +#include +#include + +#if (NXT_HAVE_MEMFD_CREATE && NXT_HAVE_LINUX_MEMFD_H) +#include +#endif +#if (NXT_HAVE_MEMFD_CREATE) +#include +#endif + + +nxt_int_t +nxt_router_new_port_test(nxt_thread_t *thr) +{ +#if !(NXT_HAVE_MEMFD_CREATE) + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router new port test skipped: no memfd_create()"); + return NXT_OK; + +#else + + nxt_mp_t *mp; + nxt_fd_t sock, queue; + nxt_buf_t buf; + nxt_task_t *task; + nxt_int_t ret; + nxt_port_t *port, *port2, *reply_port; + nxt_runtime_t *rt, *saved_rt; + nxt_event_engine_t engine; + nxt_port_recv_msg_t msg; + nxt_port_msg_new_port_t new_port; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "router new port test started"); + + ret = NXT_ERROR; + sock = -1; + queue = -1; + port = NULL; + port2 = NULL; + reply_port = NULL; + + task = thr->task; + task->thread = thr; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + rt = nxt_mp_zalloc(mp, sizeof(nxt_runtime_t)); + if (nxt_slow_path(rt == NULL)) { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + rt->mem_pool = mp; + + if (nxt_slow_path(nxt_thread_mutex_create(&rt->processes_mutex) != NXT_OK)) + { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + /* + * The minimal engine nxt_port_write_enable() needs, as + * nxt_port_fail_test.c injects one: only fast_work_queue is set up, so + * a path that reaches another member would read garbage. The handler + * under test does not get that far. + */ + nxt_memzero(&engine, sizeof(engine)); + nxt_work_queue_cache_create(&engine.work_queue_cache, 1024); + engine.fast_work_queue.cache = &engine.work_queue_cache; + nxt_work_queue_name(&engine.fast_work_queue, "fast"); + + saved_rt = thr->runtime; + thr->runtime = rt; + thr->engine = &engine; + + /* + * Releasing the port's last reference reaches nxt_runtime_process_release(), + * which frees inline when task->thread->engine is rt->main_engine and + * otherwise posts the free to that engine -- NULL here, and a posted work + * item would outlive this stack frame anyway. Pointing it at the fixture's + * own engine keeps the teardown synchronous. + */ + rt->main_engine = &engine; + + /* + * The port the message arrived on. An application NEW_PORT that answers + * a pending start carries a stream, and the handler dispatches such a + * message through nxt_port_rpc_handler(), which reads this port; the + * fixture registers no stream, so the dispatch is a lookup that finds + * nothing and returns. + * + * The stream is what keeps the path under test short. With stream 0 a + * handler that took the queue would run on into the main app port lookup + * -- an assertion the fixture cannot satisfy -- and then into a PORT_ACK + * written through the queue it just took, faulting on the short mapping. + * Either would kill the binary instead of failing the checks below. + */ + reply_port = nxt_runtime_process_port_create(task, rt, nxt_pid, 0, + NXT_PROCESS_ROUTER); + if (nxt_slow_path(reply_port == NULL)) { + goto done; + } + + /* + * nxt_port_new() zeroes the port, and 0 is a descriptor teardown must + * not close. + */ + reply_port->pair[0] = -1; + reply_port->pair[1] = -1; + reply_port->socket.fd = -1; + + /* + * A short queue: mmap() of the whole object over it would succeed and + * fault later, so the size check has to refuse it here. + */ + queue = syscall(SYS_memfd_create, "nxt_router_new_port_test", MFD_CLOEXEC); + sock = open("/dev/null", O_RDONLY); + + if (queue == -1 || sock == -1) { + nxt_log_alert(thr->log, "router new port test failed to open the " + "descriptors"); + goto done; + } + + if (ftruncate(queue, nxt_pagesize) == -1) { + nxt_log_alert(thr->log, "router new port test failed to size the " + "queue"); + goto done; + } + + nxt_memzero(&new_port, sizeof(nxt_port_msg_new_port_t)); + + new_port.pid = nxt_pid + 1; + new_port.id = 1; + new_port.type = NXT_PROCESS_APP; + new_port.max_size = 1024; + new_port.max_share = 1024; + + nxt_memzero(&buf, sizeof(nxt_buf_t)); + + buf.mem.pos = (u_char *) &new_port; + buf.mem.free = buf.mem.pos + sizeof(nxt_port_msg_new_port_t); + + nxt_memzero(&msg, sizeof(nxt_port_recv_msg_t)); + + msg.buf = &buf; + msg.fd[0] = sock; + msg.fd[1] = queue; + + msg.port = reply_port; + msg.port_msg.stream = 1; + msg.port_msg.pid = new_port.pid; + + nxt_router_new_port_handler(task, &msg); + + if (msg.fd[1] != -1 || nxt_test_fd_is_open(queue)) { + nxt_log_alert(thr->log, "router new port test: a refused queue " + "leaked its descriptor"); + goto done; + } + + queue = -1; + + if (msg.fd[0] != -1) { + nxt_log_alert(thr->log, "router new port test: the port did not take " + "the socket"); + goto done; + } + + /* + * The port owns the socket now, so the test must not close it directly + * -- but the port itself is the fixture's to release: it is registered + * in rt->ports and holds both the descriptor and its own pool, and + * destroying only the runtime pool below would leak them for the rest + * of the test binary. + */ + sock = -1; + port = msg.u.new_port; + + /* + * The point of the refusal: the short object must not be on the port. + * The descriptor checks above say only that nothing leaked, and would + * hold just as well for a handler that mapped the object and kept it. + */ + if (port->queue != NULL) { + nxt_log_alert(thr->log, "router new port test: a refused queue was " + "installed on the port"); + goto done; + } + + /* + * An application's main port (id 0) announced with no start stream. The + * handler has to tell the two ways that can happen apart, so both are + * exercised here. + * + * First the duplicate: register (pid + 2, 0) with a stream, then send + * that exact port again with none, as a worker that called + * nxt_unit_ready() twice does once its prototype has retired the start + * stream. The port must survive unchanged -- falling through would + * re-add it to the application hash and ack it a second time. + */ + + nxt_memzero(&new_port, sizeof(nxt_port_msg_new_port_t)); + + new_port.pid = nxt_pid + 2; + new_port.id = 0; + new_port.type = NXT_PROCESS_APP; + new_port.max_size = 1024; + new_port.max_share = 1024; + + nxt_memzero(&buf, sizeof(nxt_buf_t)); + + buf.mem.pos = (u_char *) &new_port; + buf.mem.free = buf.mem.pos + sizeof(nxt_port_msg_new_port_t); + + nxt_memzero(&msg, sizeof(nxt_port_recv_msg_t)); + + msg.buf = &buf; + msg.fd[0] = -1; + msg.fd[1] = -1; + msg.port = reply_port; + msg.port_msg.stream = 5; + msg.port_msg.pid = new_port.pid; + + nxt_router_new_port_handler(task, &msg); + + port2 = msg.u.new_port; + + if (port2 == NULL || !msg.new_port_created) { + nxt_log_alert(thr->log, "router new port test: the first main port " + "announcement did not create a port"); + goto done; + } + + buf.mem.pos = (u_char *) &new_port; + buf.mem.free = buf.mem.pos + sizeof(nxt_port_msg_new_port_t); + + nxt_memzero(&msg, sizeof(nxt_port_recv_msg_t)); + + msg.buf = &buf; + msg.fd[0] = -1; + msg.fd[1] = -1; + msg.port = reply_port; + msg.port_msg.stream = 0; + msg.port_msg.pid = new_port.pid; + + nxt_router_new_port_handler(task, &msg); + + if (msg.u.new_port != port2 || msg.new_port_created) { + nxt_log_alert(thr->log, "router new port test: the repeated " + "announcement did not resolve to the registered port"); + goto done; + } + + if (nxt_runtime_port_find(rt, new_port.pid, 0) != port2) { + nxt_log_alert(thr->log, "router new port test: the repeated " + "announcement disturbed the registered port"); + goto done; + } + + if (port2->app != NULL) { + nxt_log_alert(thr->log, "router new port test: the repeated " + "announcement enrolled the port in an application"); + goto done; + } + + /* + * Then the other case: a main port the router has never seen, announced + * with no stream. No start is waiting on it and nothing else will + * complete it, so the handler must undo the registration this message + * caused rather than leave an orphan in the runtime. + * + * What this pins, honestly: with the check removed both cases reach + * nxt_assert(port->id != 0) -- an abort in a debug build, compiled out in + * a release one, where the orphan check below is what reports it. So the + * case has teeth either way. What it does not cover is the other half of + * the duplicate: ->port_hash_count being incremented past a declined + * nxt_port_hash_add(), which needs an nxt_app_t the fixture does not + * build. + */ + + new_port.pid = nxt_pid + 3; + + buf.mem.pos = (u_char *) &new_port; + buf.mem.free = buf.mem.pos + sizeof(nxt_port_msg_new_port_t); + + nxt_memzero(&msg, sizeof(nxt_port_recv_msg_t)); + + msg.buf = &buf; + msg.fd[0] = -1; + msg.fd[1] = -1; + msg.port = reply_port; + msg.port_msg.stream = 0; + msg.port_msg.pid = new_port.pid; + + nxt_router_new_port_handler(task, &msg); + + if (nxt_runtime_port_find(rt, new_port.pid, 0) != NULL) { + nxt_log_alert(thr->log, "router new port test: a streamless first " + "announcement was left registered"); + goto done; + } + + ret = NXT_OK; + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "router new port test passed"); + +done: + + if (queue != -1) { + (void) close(queue); + } + + if (sock != -1) { + (void) close(sock); + } + + if (port != NULL) { + /* + * Through the ownership path, not by destroying the pool directly: + * nxt_port_mp_cleanup() asserts that the descriptors are gone and + * the use count is zero, so a direct nxt_mp_destroy() aborts a debug + * build even though it looks clean in a release one. close() clears + * pair[1]; remove() drops the rt->ports reference, which is the last + * one, and the port is freed there. + */ + nxt_port_close(task, port); + nxt_runtime_port_remove(task, port); + } + + if (port2 != NULL) { + nxt_port_close(task, port2); + nxt_runtime_port_remove(task, port2); + } + + if (reply_port != NULL) { + nxt_port_close(task, reply_port); + nxt_runtime_port_remove(task, reply_port); + } + + thr->runtime = saved_rt; + thr->engine = NULL; + + nxt_work_queue_cache_destroy(&engine.work_queue_cache); + nxt_thread_mutex_destroy(&rt->processes_mutex); + nxt_mp_destroy(mp); + + return ret; + +#endif +} diff --git a/src/test/nxt_router_prepare_msg_test.c b/src/test/nxt_router_prepare_msg_test.c new file mode 100644 index 000000000..02a0394f9 --- /dev/null +++ b/src/test/nxt_router_prepare_msg_test.c @@ -0,0 +1,310 @@ + +/* + * Copyright (C) FreeUnit contributors. + */ + +/* + * nxt_router_prepare_msg() (src/nxt_router.c) stores the method and header + * name lengths as uint8_t. A 251..255-byte name plus the "HTTP_" prefix, + * or a method over 255 bytes, used to wrap and be stored truncated; now they + * are refused with 431 and 501. The limits themselves are pinned, and so + * is nxt_app_msg_prefix[]: which app types get the prefix and which do not. + * + * The request is built by hand with one field, and in some cases a second + * one marked skip, which the router must leave out of the message and its + * size; the engine is a stub that only provides the memory pool the buffer + * is allocated from. The cases + * run in a child process: the accepted cases' buffers hold shared memory + * that only a real engine can release, and the child's exit frees it. + */ + +#include +#include +#include +#include +#include +#include +#include +#include "nxt_tests.h" + + +static u_char nxt_prepare_msg_test_name[300]; +static u_char nxt_prepare_msg_test_method[300]; + + +typedef struct { + const char *name; + nxt_app_type_t type; + size_t method_length; + size_t field_length; + size_t skipped_length; /* 0: no skipped field */ + nxt_http_status_t status; /* 0: accepted */ +} nxt_prepare_msg_test_case_t; + + +#define NXT_TEST_431 NXT_HTTP_REQUEST_HEADER_FIELDS_TOO_LARGE +#define NXT_TEST_501 NXT_HTTP_NOT_IMPLEMENTED + + +static const nxt_prepare_msg_test_case_t nxt_prepare_msg_test_cases[] = { + { "short name, php", NXT_APP_PHP, 3, 6, 0, 0 }, + { "250-byte name + prefix", NXT_APP_PHP, 3, 250, 0, 0 }, + { "251-byte name + prefix", NXT_APP_PHP, 3, 251, 0, + NXT_TEST_431 }, + { "251-byte name, perl", NXT_APP_PERL, 3, 251, 0, + NXT_TEST_431 }, + { "251-byte name, ruby", NXT_APP_RUBY, 3, 251, 0, + NXT_TEST_431 }, + { "253-byte name + prefix", NXT_APP_PHP, 3, 253, 0, + NXT_TEST_431 }, + { "255-byte name + prefix", NXT_APP_PHP, 3, 255, 0, + NXT_TEST_431 }, + { "255-byte name, external", NXT_APP_EXTERNAL, 3, 255, 0, 0 }, + { "255-byte name, python", NXT_APP_PYTHON, 3, 255, 0, 0 }, + { "255-byte name, java", NXT_APP_JAVA, 3, 255, 0, 0 }, + { "255-byte name, wasm", NXT_APP_WASM, 3, 255, 0, 0 }, + { "255-byte name, wasm-wc", NXT_APP_WASM_WC, 3, 255, 0, 0 }, + { "255-byte method", NXT_APP_PYTHON, 255, 6, 0, 0 }, + { "256-byte method", NXT_APP_PYTHON, 256, 6, 0, + NXT_TEST_501 }, + { "280-byte method", NXT_APP_PYTHON, 280, 6, 0, + NXT_TEST_501 }, + { "251-byte skipped name", NXT_APP_PHP, 3, 6, 251, 0 }, + { "255-byte skipped name", NXT_APP_PHP, 3, 6, 255, 0 }, +}; + + +/* The app types whose header names carry "HTTP_"; the rest get none. */ + +static nxt_bool_t +nxt_prepare_msg_test_prefixed(nxt_app_type_t type) +{ + return (type == NXT_APP_PHP || type == NXT_APP_PERL + || type == NXT_APP_RUBY); +} + + +/* On success *used is the size of the built message, 0 when refused. */ + +static nxt_int_t +nxt_prepare_msg_test_case(nxt_task_t *task, nxt_mp_t *mp, nxt_app_t *app, + nxt_sockaddr_t *sa, const nxt_prepare_msg_test_case_t *tc, size_t *used) +{ + u_char *name; + size_t expect; + nxt_buf_t *b; + nxt_str_t *method, *args; + nxt_uint_t status; + nxt_bool_t prefixed; + nxt_http_field_t *field; + nxt_unit_field_t *f; + nxt_unit_request_t *req; + nxt_http_request_t *r; + + r = nxt_mp_zalloc(mp, sizeof(nxt_http_request_t)); + method = nxt_mp_zalloc(mp, sizeof(nxt_str_t)); + args = nxt_mp_zalloc(mp, sizeof(nxt_str_t)); + + if (r == NULL || method == NULL || args == NULL) { + return NXT_ERROR; + } + + method->start = nxt_prepare_msg_test_method; + method->length = tc->method_length; + + r->method = method; + nxt_str_set(&r->version, "HTTP/1.1"); + nxt_str_set(&r->server_name, "localhost"); + nxt_str_set(&r->target, "/"); + r->path = &r->target; + r->args = args; + r->remote = sa; + r->local = sa; + r->content_length_n = -1; + + field = &r->inline_fields[0]; + r->num_inline_fields = 1; + + field->hash = 0; + field->name = nxt_prepare_msg_test_name; + field->name_length = tc->field_length; + field->value = (u_char *) "v"; + field->value_length = 1; + + if (tc->skipped_length != 0) { + field = &r->inline_fields[1]; + r->num_inline_fields = 2; + + field->hash = 0; + field->skip = 1; + field->name = nxt_prepare_msg_test_name; + field->name_length = tc->skipped_length; + field->value = (u_char *) "v"; + field->value_length = 1; + } + + app->type = tc->type; + *used = 0; + status = 0; + + b = nxt_router_test_prepare_msg(task, r, app, &status); + + if (b == NULL) { + if (status != (nxt_uint_t) tc->status) { + nxt_log_alert(task->log, "prepare msg test \"%s\": refused " + "with %d, expected %d", tc->name, (int) status, + (int) tc->status); + return NXT_ERROR; + } + + return NXT_OK; + } + + *used = nxt_buf_mem_used_size(&b->mem); + + req = (nxt_unit_request_t *) b->mem.pos; + f = &req->fields[0]; + name = nxt_unit_sptr_get(&f->name); + + prefixed = nxt_prepare_msg_test_prefixed(tc->type); + expect = tc->field_length + (prefixed ? nxt_length("HTTP_") : 0); + + if (tc->status != 0 + || req->method_length != tc->method_length + || req->fields_count != 1 + || f->name_length != expect + || nxt_strlen(name) != expect + || (nxt_strncmp(name, "HTTP_", nxt_length("HTTP_")) == 0) != prefixed) + { + nxt_log_alert(task->log, "prepare msg test \"%s\": accepted with " + "method_length %d, name_length %d; expected %d and " + "%uz, or status %d", tc->name, (int) req->method_length, + (int) f->name_length, (int) tc->method_length, expect, + (int) tc->status); + return NXT_ERROR; + } + + return NXT_OK; +} + + +static nxt_int_t +nxt_prepare_msg_test_run(nxt_thread_t *thr) +{ + size_t used, live_used; + nxt_mp_t *mp; + nxt_str_t addr; + nxt_int_t ret; + nxt_app_t *app; + nxt_uint_t i; + nxt_task_t *task; + nxt_sockaddr_t *sa; + nxt_event_engine_t engine, *saved_engine; + nxt_prepare_msg_test_case_t live; + const nxt_prepare_msg_test_case_t *tc; + + nxt_thread_time_update(thr); + + task = thr->task; + task->thread = thr; + + nxt_memset(nxt_prepare_msg_test_name, 'a', + sizeof(nxt_prepare_msg_test_name)); + nxt_memset(nxt_prepare_msg_test_method, 'A', + sizeof(nxt_prepare_msg_test_method)); + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + app = nxt_mp_zalloc(mp, sizeof(nxt_app_t)); + nxt_str_set(&addr, "127.0.0.1:8080"); + sa = nxt_sockaddr_parse(mp, &addr); + + if (nxt_slow_path(app == NULL || sa == NULL + || nxt_thread_mutex_create(&app->outgoing.mutex) + != NXT_OK)) + { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + nxt_memzero(&engine, sizeof(engine)); + engine.mem_pool = mp; + engine.task.thread = thr; + engine.task.log = thr->log; + + saved_engine = thr->engine; + thr->engine = &engine; + + ret = NXT_OK; + + for (i = 0; i < nxt_nitems(nxt_prepare_msg_test_cases); i++) { + tc = &nxt_prepare_msg_test_cases[i]; + + if (nxt_prepare_msg_test_case(task, mp, app, sa, tc, &used) + != NXT_OK) + { + ret = NXT_ERROR; + continue; + } + + if (tc->skipped_length == 0 || tc->status != 0) { + continue; + } + + /* The skipped field must add nothing to the message. */ + + live = *tc; + live.skipped_length = 0; + + if (nxt_prepare_msg_test_case(task, mp, app, sa, &live, &live_used) + != NXT_OK) + { + ret = NXT_ERROR; + continue; + } + + if (used != live_used) { + nxt_log_alert(task->log, "prepare msg test \"%s\": the message " + "is %uz bytes, %uz without the skipped field", + tc->name, used, live_used); + ret = NXT_ERROR; + } + } + + thr->engine = saved_engine; + + return ret; +} + + +nxt_int_t +nxt_router_prepare_msg_test(nxt_thread_t *thr) +{ + int status; + pid_t child; + + child = fork(); + + if (child == 0) { + _exit(nxt_prepare_msg_test_run(thr) == NXT_OK ? 0 : 1); + } + + if (nxt_slow_path(child == -1 || waitpid(child, &status, 0) != child)) { + nxt_log_alert(thr->log, "router prepare msg test: fork() failed %E", + nxt_errno); + return NXT_ERROR; + } + + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + nxt_log_alert(thr->log, "router prepare msg test failed, " + "status %d", status); + return NXT_ERROR; + } + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "router prepare msg test passed"); + + return NXT_OK; +} diff --git a/src/test/nxt_router_proto_death_test.c b/src/test/nxt_router_proto_death_test.c new file mode 100644 index 000000000..58649b25b --- /dev/null +++ b/src/test/nxt_router_proto_death_test.c @@ -0,0 +1,294 @@ +/* + * Copyright (C) F5, Inc. + */ + +/* + * Regression test for issue #269: a prototype that dies between forking a + * worker and that worker's PROCESS_READY strands the router's on-demand + * START_PROCESS registration. + * + * The router hears about the death as a REMOVE_PID for the prototype. That + * message carries no stream once the prototype itself has reached READY + * (src/nxt_main_process.c:1200), so the only thing left that can retire the + * armed RPC is nxt_port_rpc_remove_peer() (src/nxt_port.c:1067) -- and it + * retires registrations by peer. nxt_router_start_app_process_handler() + * used to arm its registration with no peer at all, so the sweep found + * nothing, the attempt's pending_processes slot was never given back, and + * with "processes: {spare: 0}" (max_pending_processes 1) + * nxt_router_app_can_start() stayed false for the router's lifetime: every + * later request parked in ack_waiting_req and no replacement prototype was + * ever asked for, because only the start handler asks for one and can_start + * is what gates it. + * + * This is the sibling of src/test/nxt_router_proto_wedge_test.c, which + * covers the prototype-start branch retired through nxt_port_rpc_close(). + * Here the application already has a prototype, so the handler takes the + * plain app-start branch, and the RPC is retired the way the real + * REMOVE_PID does it: by peer. Before the fix the sweep is a no-op and the + * checks below fail; after it the sweep runs nxt_router_app_port_error() + * exactly once, the slot comes back, and the application is startable again. + */ + +#include +#include +#include +#include +#include +#include +#include "nxt_tests.h" + + +/* File scope: the handler hands the application to the router's + reference-counting helpers, so its storage must outlive the frame. */ +static nxt_app_t nxt_router_proto_death_test_app; + + +nxt_int_t +nxt_router_proto_death_test(nxt_thread_t *thr) +{ + nxt_mp_t *mp; + nxt_app_t *app; + nxt_int_t ret; + nxt_bool_t app_mutex; + nxt_task_t *task; + nxt_port_t *router_port, *proto_port; + nxt_runtime_t *rt, *saved_rt; + nxt_app_joint_t *joint; + nxt_event_engine_t engine, *saved_engine; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router proto death test started"); + + ret = NXT_ERROR; + router_port = NULL; + proto_port = NULL; + joint = NULL; + app_mutex = 0; + app = &nxt_router_proto_death_test_app; + + task = thr->task; + task->thread = thr; + + if (nxt_slow_path(nxt_port_rpc_init() != NXT_OK)) { + return NXT_ERROR; + } + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + rt = nxt_mp_zalloc(mp, sizeof(nxt_runtime_t)); + joint = nxt_mp_zalloc(mp, sizeof(nxt_app_joint_t)); + if (nxt_slow_path(rt == NULL || joint == NULL)) { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + rt->mem_pool = mp; + + if (nxt_slow_path(nxt_thread_mutex_create(&rt->processes_mutex) != NXT_OK)) + { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + /* + * Only the members the reached code touches are set; see + * src/test/nxt_router_new_port_test.c. + */ + nxt_memzero(&engine, sizeof(engine)); + nxt_work_queue_cache_create(&engine.work_queue_cache, 1024); + engine.fast_work_queue.cache = &engine.work_queue_cache; + nxt_work_queue_name(&engine.fast_work_queue, "fast"); + engine.mem_pool = mp; + + saved_rt = thr->runtime; + saved_engine = thr->engine; + thr->runtime = rt; + thr->engine = &engine; + rt->main_engine = &engine; + + /* The port the handler registers its RPC on. */ + + router_port = nxt_port_new(task, 0, nxt_pid, NXT_PROCESS_ROUTER); + if (nxt_slow_path(router_port == NULL)) { + goto done; + } + + /* + * nxt_port_rpc_register_handler_ex() asserts on a live pair[0]; the + * descriptor is never touched here, so borrow the nxt_port_fail_test() + * convention of a placeholder that is reset before the port is freed. + */ + router_port->pair[0] = 0; + router_port->pair[1] = -1; + router_port->socket.fd = -1; + + /* + * The prototype the START_PROCESS is written to, and whose pid the + * REMOVE_PID sweep is driven with. No queue and write_ready unset, so + * nxt_port_socket_write2() takes the nxt_port_msg_chk_insert() path and + * the message is simply queued: the write succeeds and the RPC stays + * armed, which is what this test needs. + * + * The port's pid is the running process, as everywhere in these + * fixtures; the sweep only ever compares it with the value the handler + * recorded as the registration's peer. + */ + + proto_port = nxt_port_new(task, 1, nxt_pid, NXT_PROCESS_PROTOTYPE); + if (nxt_slow_path(proto_port == NULL)) { + goto done; + } + + proto_port->pair[0] = -1; + proto_port->pair[1] = -1; + proto_port->socket.fd = -1; + + nxt_memzero(app, sizeof(nxt_app_t)); + + if (nxt_slow_path(nxt_thread_mutex_create(&app->mutex) != NXT_OK)) { + goto done; + } + + app_mutex = 1; + + nxt_queue_init(&app->ports); + nxt_queue_init(&app->spare_ports); + nxt_queue_init(&app->idle_ports); + nxt_queue_init(&app->ack_waiting_req); + + nxt_str_set(&app->name, "proto-death-test"); + + app->joint = joint; + joint->app = app; + joint->use_count = 1; + + /* + * "processes: {spare: 0}" -- max_pending_processes is 1, so a single + * stranded slot is enough to wedge the application for good. + */ + app->max_processes = 8; + app->max_pending_processes = 1; + + /* A live prototype, so the handler takes the plain app-start branch. */ + app->proto_port = proto_port; + app->conf.length = 0; + + /* One process alive, so the failure path has no requests to fail. */ + app->processes = 1; + + /* What a real initiator leaves behind: the slot and the work's use. */ + app->pending_processes = 1; + app->use_count = 2; + + nxt_router_start_app_process_handler(task, router_port, app); + + if (app->pending_processes != 1 || joint->use_count != 2 + || app->use_count != 1) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router proto death test: successful start left " + "pending %uD joint %uD app use %d (expected 1, 2 and 1)", + app->pending_processes, joint->use_count, + (int) app->use_count); + goto done; + } + + /* + * The prototype dies with the worker it forked not yet ready. This is + * exactly what nxt_router_remove_pid_handler() does with the streamless + * REMOVE_PID that reports it. + */ + + nxt_port_rpc_remove_peer(task, router_port, proto_port->pid); + + if (app->pending_processes != 0 || joint->use_count != 1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router proto death test: prototype death left pending " + "%uD joint %uD (expected 0 and 1) -- the start RPC was " + "not retired", app->pending_processes, + joint->use_count); + goto done; + } + + /* + * The gate #269 wedges: with the slot back the application can ask for + * a replacement prototype on the next request. + */ + + app->processes = 0; + + if (!nxt_router_app_can_start(app)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router proto death test: app cannot start after the " + "prototype's death, processes %uD pending %uD", + app->processes, app->pending_processes); + goto done; + } + + /* + * And the death is reported exactly once: a second sweep for the same + * pid, which a REMOVE_PID for an already-forgotten process would drive, + * must find nothing left to fail rather than release a second slot. + */ + + app->pending_processes = 1; + + nxt_port_rpc_remove_peer(task, router_port, proto_port->pid); + + if (app->pending_processes != 1 || joint->use_count != 1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router proto death test: repeated sweep left pending " + "%uD joint %uD (expected 1 and 1)", + app->pending_processes, joint->use_count); + goto done; + } + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "router proto death test passed"); + + ret = NXT_OK; + +done: + + if (proto_port != NULL) { + /* + * The start above was queued rather than written, and + * nxt_port_msg_chk_insert() takes a port reference for every queued + * message. Left alone the port never reaches nxt_port_mp_cleanup(), + * so its pool and messages leak and the assertions there never run + * -- which would quietly hollow out this test under a debug build. + */ + nxt_port_test_run_error_handler(task, proto_port); + + if (!nxt_queue_is_empty(&proto_port->messages)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router proto death test: prototype port still " + "holds queued messages after teardown"); + ret = NXT_ERROR; + } + + nxt_port_use(task, proto_port, -1); + } + + if (router_port != NULL) { + router_port->pair[0] = -1; + + nxt_port_use(task, router_port, -1); + } + + thr->engine = saved_engine; + thr->runtime = saved_rt; + + if (app_mutex) { + nxt_thread_mutex_destroy(&app->mutex); + } + + nxt_work_queue_cache_destroy(&engine.work_queue_cache); + nxt_thread_mutex_destroy(&rt->processes_mutex); + nxt_mp_destroy(mp); + + return ret; +} diff --git a/src/test/nxt_router_proto_wedge_test.c b/src/test/nxt_router_proto_wedge_test.c new file mode 100644 index 000000000..afdf624fc --- /dev/null +++ b/src/test/nxt_router_proto_wedge_test.c @@ -0,0 +1,346 @@ +/* + * Copyright (C) F5, Inc. + */ + +/* + * Regression test for the prototype-start wedge: a prototype START_PROCESS + * RPC that resolves through nxt_router_app_port_error() instead of + * nxt_router_app_port_ready() used to leave app->proto_port_requests set. + * + * Only port_ready() cleared that counter, so once it was stranded every + * later nxt_router_start_app_process_handler() call for the application + * took the "wait for prototype process" branch at src/nxt_router.c:430 and + * parked without sending anything. Nothing was left to send a prototype + * request, so no port_ready or port_error could fire to clear it: the + * application could never start another process for the router's lifetime, + * and a reload that left its config block unchanged reused the wedged + * nxt_app_t rather than replacing it. + * + * The stranded counter also stranded pending_processes. Each parked caller + * holds one increment its initiator took, but port_error() released exactly + * one -- so a cohort of N parked callers leaked N-1 start slots on top of + * the wedge. + * + * The arrangement here is the one the router really uses: no prototype port + * yet, so the handler builds a START_PROCESS payload and arms an RPC with + * ->proto set; a second call then joins the wait. Tearing the port's + * registrations down with nxt_port_rpc_close() drives the router's own + * error handler for the armed RPC, which is how a start that never answers + * is reported. The test then requires what the wedge denied: the counter + * cleared, every slot in the cohort given back, and a later call actually + * sending a fresh prototype request instead of parking. + */ + +#include +#include +#include +#include +#include +#include +#include "nxt_tests.h" + + +/* File scope: the handler hands the application to the router's + reference-counting helpers, so its storage must outlive the frame. */ +static nxt_app_t nxt_router_proto_wedge_test_app; + + +nxt_int_t +nxt_router_proto_wedge_test(nxt_thread_t *thr) +{ + nxt_mp_t *mp; + nxt_app_t *app; + nxt_int_t ret; + nxt_bool_t app_mutex; + nxt_task_t *task; + nxt_port_t *router_port, *main_port, *shared_port; + nxt_runtime_t *rt, *saved_rt; + nxt_app_joint_t *joint; + nxt_event_engine_t engine, *saved_engine; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "router proto wedge test started"); + + ret = NXT_ERROR; + router_port = NULL; + main_port = NULL; + shared_port = NULL; + joint = NULL; + app_mutex = 0; + app = &nxt_router_proto_wedge_test_app; + + task = thr->task; + task->thread = thr; + + if (nxt_slow_path(nxt_port_rpc_init() != NXT_OK)) { + return NXT_ERROR; + } + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + rt = nxt_mp_zalloc(mp, sizeof(nxt_runtime_t)); + joint = nxt_mp_zalloc(mp, sizeof(nxt_app_joint_t)); + if (nxt_slow_path(rt == NULL || joint == NULL)) { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + rt->mem_pool = mp; + + if (nxt_slow_path(nxt_thread_mutex_create(&rt->processes_mutex) != NXT_OK)) + { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + /* + * The prototype branch allocates its payload from + * task->thread->engine->mem_pool. Only the members the reached code + * touches are set; see src/test/nxt_router_new_port_test.c. + */ + nxt_memzero(&engine, sizeof(engine)); + nxt_work_queue_cache_create(&engine.work_queue_cache, 1024); + engine.fast_work_queue.cache = &engine.work_queue_cache; + nxt_work_queue_name(&engine.fast_work_queue, "fast"); + engine.mem_pool = mp; + + saved_rt = thr->runtime; + saved_engine = thr->engine; + thr->runtime = rt; + thr->engine = &engine; + rt->main_engine = &engine; + + /* The port the handler registers its RPC on. */ + + router_port = nxt_port_new(task, 0, nxt_pid, NXT_PROCESS_ROUTER); + if (nxt_slow_path(router_port == NULL)) { + goto done; + } + + /* + * nxt_port_rpc_register_handler_ex() asserts on a live pair[0]; the + * descriptor is never touched here, so borrow the nxt_port_fail_test() + * convention of a placeholder that is reset before the port is freed. + */ + router_port->pair[0] = 0; + router_port->pair[1] = -1; + router_port->socket.fd = -1; + + /* + * A prototype start addresses the main process. No queue and + * write_ready unset, so nxt_port_socket_write2() takes the + * nxt_port_msg_chk_insert() path and the message is simply queued: + * the write succeeds and the RPC stays armed, which is what this test + * needs. + */ + + main_port = nxt_port_new(task, 1, nxt_pid, NXT_PROCESS_MAIN); + if (nxt_slow_path(main_port == NULL)) { + goto done; + } + + main_port->pair[0] = -1; + main_port->pair[1] = -1; + main_port->socket.fd = -1; + + rt->port_by_type[NXT_PROCESS_MAIN] = main_port; + + /* Only its two descriptors are read, and both travel as -1. */ + + shared_port = nxt_port_new(task, 2, nxt_pid, NXT_PROCESS_APP); + if (nxt_slow_path(shared_port == NULL)) { + goto done; + } + + shared_port->pair[0] = -1; + shared_port->pair[1] = -1; + shared_port->socket.fd = -1; + shared_port->queue_fd = -1; + + nxt_memzero(app, sizeof(nxt_app_t)); + + if (nxt_slow_path(nxt_thread_mutex_create(&app->mutex) != NXT_OK)) { + goto done; + } + + app_mutex = 1; + + nxt_queue_init(&app->ports); + nxt_queue_init(&app->spare_ports); + nxt_queue_init(&app->idle_ports); + nxt_queue_init(&app->ack_waiting_req); + + nxt_str_set(&app->name, "proto-wedge-test"); + + app->joint = joint; + joint->app = app; + joint->use_count = 1; + + app->shared_port = shared_port; + app->max_processes = 8; + app->max_pending_processes = 4; + + /* No prototype port yet: the next call starts one. */ + app->proto_port = NULL; + + /* One process alive, so the failure path has no requests to fail. */ + app->processes = 1; + + /* What a real initiator leaves behind: the slot and the work's use. */ + app->pending_processes = 1; + app->use_count = 2; + + nxt_router_start_app_process_handler(task, router_port, app); + + if (app->proto_port_requests != 1 || joint->use_count != 2 + || app->use_count != 1) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router proto wedge test: prototype start left " + "proto_port_requests %uD joint %uD app use %d " + "(expected 1, 2 and 1)", + app->proto_port_requests, joint->use_count, + (int) app->use_count); + goto done; + } + + /* + * A second caller finds the request already in flight and joins the + * wait, adding its own initiator increment to both counters. + */ + + app->pending_processes = 2; + app->use_count = 2; + + nxt_router_start_app_process_handler(task, router_port, app); + + if (app->proto_port_requests != 2 || joint->use_count != 2 + || app->use_count != 1) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router proto wedge test: parked caller left " + "proto_port_requests %uD joint %uD app use %d " + "(expected 2, 2 and 1)", + app->proto_port_requests, joint->use_count, + (int) app->use_count); + goto done; + } + + /* + * The prototype never answers. Closing the port's registrations drives + * nxt_router_app_port_error() for the armed RPC -- the wedge is what it + * used to leave behind. + */ + + nxt_port_rpc_close(task, router_port); + + if (app->proto_port_requests != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router proto wedge test: failed prototype left " + "proto_port_requests %uD (expected 0)", + app->proto_port_requests); + goto done; + } + + if (app->pending_processes != 0 || joint->use_count != 1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router proto wedge test: failed prototype left " + "pending %uD joint %uD (expected 0 and 1)", + app->pending_processes, joint->use_count); + goto done; + } + + /* + * The behaviour the wedge denied: with no prototype port and the + * counter clear, a later call must send a fresh prototype request -- + * arming an RPC and taking a joint reference -- rather than parking in + * the wait branch, which takes neither. + */ + + app->pending_processes = 1; + app->use_count = 2; + + nxt_router_start_app_process_handler(task, router_port, app); + + if (app->proto_port_requests != 1 || joint->use_count != 2 + || app->use_count != 1) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router proto wedge test: app wedged after a failed " + "prototype, proto_port_requests %uD joint %uD " + "app use %d (expected 1, 2 and 1)", + app->proto_port_requests, joint->use_count, + (int) app->use_count); + goto done; + } + + /* Release the second armed RPC the same way. */ + + nxt_port_rpc_close(task, router_port); + + if (app->pending_processes != 0 || joint->use_count != 1 + || app->proto_port_requests != 0) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router proto wedge test: second release left pending " + "%uD joint %uD proto_port_requests %uD " + "(expected 0, 1 and 0)", + app->pending_processes, joint->use_count, + app->proto_port_requests); + goto done; + } + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "router proto wedge test passed"); + + ret = NXT_OK; + +done: + + if (shared_port != NULL) { + nxt_port_use(task, shared_port, -1); + } + + if (main_port != NULL) { + rt->port_by_type[NXT_PROCESS_MAIN] = NULL; + + /* + * Both prototype requests were queued rather than written, and + * nxt_port_msg_chk_insert() takes a port reference for each. Left + * alone the port never reaches nxt_port_mp_cleanup(), so its pool + * and messages leak and the assertions there never run -- which + * would quietly hollow out this test under a debug build. + */ + nxt_port_test_run_error_handler(task, main_port); + + if (!nxt_queue_is_empty(&main_port->messages)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router proto wedge test: main port still holds " + "queued messages after teardown"); + ret = NXT_ERROR; + } + + nxt_port_use(task, main_port, -1); + } + + if (router_port != NULL) { + router_port->pair[0] = -1; + + nxt_port_use(task, router_port, -1); + } + + thr->engine = saved_engine; + thr->runtime = saved_rt; + + if (app_mutex) { + nxt_thread_mutex_destroy(&app->mutex); + } + + nxt_work_queue_cache_destroy(&engine.work_queue_cache); + nxt_thread_mutex_destroy(&rt->processes_mutex); + nxt_mp_destroy(mp); + + return ret; +} diff --git a/src/test/nxt_router_remove_pid_soak_test.c b/src/test/nxt_router_remove_pid_soak_test.c new file mode 100644 index 000000000..0f697d954 --- /dev/null +++ b/src/test/nxt_router_remove_pid_soak_test.c @@ -0,0 +1,452 @@ +/* + * Copyright (C) F5, Inc. + */ + +/* + * Accounting soak for the route a refused worker start really travels + * (issue #231). When a worker never becomes usable, nothing cancels its + * start RPC directly: the prototype reaps the child and sends REMOVE_PID + * carrying the stream the start was registered under, and the router turns + * that into the start failure. + * + * The chain this exercises, in the router's own order: + * + * nxt_router_remove_pid_handler() retypes a last-marked REMOVE_PID that + * carries a non-zero stream into + * _NXT_PORT_MSG_RPC_ERROR + * -> nxt_port_rpc_handler() finds and deletes the registration + * -> nxt_router_app_port_error() the RPC is a worker start, so ->proto + * is 0 and the count is 1 + * -> nxt_router_app_start_failed() gives that one slot back + * + * The handler is private to src/nxt_router.c; it is reached the way the + * port dispatcher reaches it, through the router's published handler table + * in nxt_router_process. + * + * Every iteration arms a real start through + * nxt_router_start_app_process_handler(), takes the stream from the + * START_PROCESS message that start actually queued for the prototype -- + * the very stream a prototype would echo back -- and requires + * pending_processes at its baseline afterwards. A leak leaves baseline+1 + * and a double release leaves baseline-1; both are caught without + * nxt_assert, so the check survives a non-debug build. + * + * The baselines alternate between 0 and a non-zero value because an + * over-release at baseline 0 is the case that hides: nxt_router.c clamps + * the count to pending_processes rather than letting it wrap, so the + * counter simply stays at 0 and an equality check against 0 cannot tell + * the difference. At a non-zero baseline the same bug shows up plainly. + * + * The counter app->proto_port_requests is required to stay 0 throughout. + * A worker start cannot touch it -- ->proto is set from the same condition + * that increments it, and this route never arms a prototype -- so the check + * is a guard against the fixture drifting into the prototype path, not a + * failure this route can produce. + * + * A stream nothing registered is delivered too, so the release above + * cannot be credited to anything other than the RPC registration. + */ + +#include +#include +#include +#include +#include +#include +#include +#include "nxt_tests.h" + + +#define NXT_ROUTER_REMOVE_PID_SOAK_N 1000 + + +/* File scope: the handler hands the application to the router's + reference-counting helpers, so its storage must outlive the frame. */ +static nxt_app_t nxt_router_remove_pid_soak_test_app; + + +/* + * The stream a start put on the wire. nxt_port_socket_write2() queues the + * START_PROCESS message at the tail of the destination port, so the newest + * one is this iteration's; returns 0, which the caller rejects, if the + * write did not queue anything. + */ + +static uint32_t +nxt_router_remove_pid_soak_stream(nxt_port_t *dport) +{ + nxt_queue_link_t *link; + nxt_port_send_msg_t *msg; + + if (nxt_slow_path(nxt_queue_is_empty(&dport->messages))) { + return 0; + } + + link = nxt_queue_last(&dport->messages); + msg = nxt_queue_link_data(link, nxt_port_send_msg_t, link); + + return msg->port_msg.stream; +} + + +/* + * A REMOVE_PID as the prototype sends it: the reaped worker's pid as the + * whole payload, the start stream, and last set so the registration is + * deleted rather than merely looked up. Modelled on the message + * nxt_port_rpc_remove_peer() synthesises for the same purpose. + */ + +static void +nxt_router_remove_pid_soak_msg(nxt_port_recv_msg_t *msg, nxt_buf_t *buf, + nxt_port_t *port, nxt_pid_t *pid, uint32_t stream) +{ + nxt_memzero(msg, sizeof(nxt_port_recv_msg_t)); + nxt_memzero(buf, sizeof(nxt_buf_t)); + + buf->mem.pos = (u_char *) pid; + buf->mem.free = (u_char *) (pid + 1); + + msg->fd[0] = -1; + msg->fd[1] = -1; + msg->buf = buf; + msg->port = port; + + msg->port_msg.stream = stream; + msg->port_msg.pid = nxt_pid; + msg->port_msg.type = _NXT_PORT_MSG_REMOVE_PID; + msg->port_msg.last = 1; + +#if (NXT_USE_CMSG_PID) + /* From main; see main_port in the test. */ + msg->cmsg_pid = nxt_pid; +#endif +} + + +nxt_int_t +nxt_router_remove_pid_soak_test(nxt_thread_t *thr) +{ + nxt_mp_t *mp; + nxt_app_t *app; + nxt_buf_t buf; + nxt_pid_t worker_pid; + nxt_int_t ret; + uint32_t baseline, stream; + nxt_uint_t i; + nxt_bool_t app_mutex; + nxt_task_t *task; + nxt_port_t *router_port, *dport, *main_port; + nxt_router_t router, *saved_router; + nxt_runtime_t *rt, *saved_rt; + nxt_app_joint_t *joint; + nxt_event_engine_t engine, *saved_engine; + nxt_port_recv_msg_t msg; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router remove pid soak test started"); + + ret = NXT_ERROR; + router_port = NULL; + dport = NULL; + main_port = NULL; + joint = NULL; + app_mutex = 0; + app = &nxt_router_remove_pid_soak_test_app; + + /* + * No process by this pid is registered, so nxt_port_remove_pid() finds + * nothing to tear down and the message carries the stream alone -- the + * part of REMOVE_PID this test is about. + */ + worker_pid = nxt_pid + 1; + + task = thr->task; + task->thread = thr; + + if (nxt_slow_path(nxt_port_rpc_init() != NXT_OK)) { + return NXT_ERROR; + } + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + rt = nxt_mp_zalloc(mp, sizeof(nxt_runtime_t)); + joint = nxt_mp_zalloc(mp, sizeof(nxt_app_joint_t)); + if (nxt_slow_path(rt == NULL || joint == NULL)) { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + rt->mem_pool = mp; + + if (nxt_slow_path(nxt_thread_mutex_create(&rt->processes_mutex) != NXT_OK)) + { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + /* + * Only the members the reached code touches are set; see + * src/test/nxt_router_new_port_test.c. + */ + nxt_memzero(&engine, sizeof(engine)); + nxt_work_queue_cache_create(&engine.work_queue_cache, 1024); + engine.fast_work_queue.cache = &engine.work_queue_cache; + nxt_work_queue_name(&engine.fast_work_queue, "fast"); + engine.mem_pool = mp; + + /* + * nxt_router_remove_pid_handler() forwards the removal to every router + * engine before it reaches the RPC, and dereferences the nxt_router + * global to find them. An empty list is the honest arrangement here: + * this fixture runs no worker engines, and what the forwarding does -- + * dropping the dead process from each engine's app port hash -- is not + * part of the pending_processes accounting under test. + */ + nxt_memzero(&router, sizeof(nxt_router_t)); + nxt_queue_init(&router.engines); + + saved_router = nxt_router; + saved_rt = thr->runtime; + saved_engine = thr->engine; + nxt_router = &router; + thr->runtime = rt; + thr->engine = &engine; + rt->main_engine = &engine; + + /* The port the handler registers its RPC on, and REMOVE_PID arrives on. */ + + router_port = nxt_port_new(task, 0, nxt_pid, NXT_PROCESS_ROUTER); + if (nxt_slow_path(router_port == NULL)) { + goto done; + } + + /* + * nxt_port_rpc_register_handler_ex() asserts on a live pair[0]; the + * descriptor is never touched here, so borrow the nxt_port_fail_test() + * convention of a placeholder that is reset before the port is freed. + */ + router_port->pair[0] = 0; + router_port->pair[1] = -1; + router_port->socket.fd = -1; + + /* + * The prototype the start addresses. No queue and write_ready unset, + * so nxt_port_socket_write2() takes the nxt_port_msg_chk_insert() path + * and the START_PROCESS message is simply queued: the write succeeds, + * the RPC stays armed, and the queued message is where the stream is + * read back from. + */ + + dport = nxt_port_new(task, 1, nxt_pid, NXT_PROCESS_APP); + if (nxt_slow_path(dport == NULL)) { + goto done; + } + + dport->pair[0] = -1; + dport->pair[1] = -1; + dport->socket.fd = -1; + + /* + * The router accepts REMOVE_PID only from main or a prototype (issue + * #341). Main sends it with a start stream when a prototype dies, so + * the message comes from main: a main port with the pid of this process. + */ + + main_port = nxt_port_new(task, 2, nxt_pid, NXT_PROCESS_MAIN); + if (nxt_slow_path(main_port == NULL)) { + goto done; + } + + main_port->pair[0] = -1; + main_port->pair[1] = -1; + main_port->socket.fd = -1; + + rt->port_by_type[NXT_PROCESS_MAIN] = main_port; + + nxt_memzero(app, sizeof(nxt_app_t)); + + if (nxt_slow_path(nxt_thread_mutex_create(&app->mutex) != NXT_OK)) { + goto done; + } + + app_mutex = 1; + + nxt_queue_init(&app->ports); + nxt_queue_init(&app->spare_ports); + nxt_queue_init(&app->idle_ports); + nxt_queue_init(&app->ack_waiting_req); + + nxt_str_set(&app->name, "remove-pid-soak-test"); + + app->joint = joint; + joint->app = app; + joint->use_count = 1; + + app->max_processes = 8; + app->max_pending_processes = 4; + + /* A prototype port is present, so every start is a worker start. */ + app->proto_port = dport; + + /* One process alive, so the failure path has no requests to fail. */ + app->processes = 1; + + /* Set once: a worker start must leave this alone. See the file head. */ + app->proto_port_requests = 0; + + for (i = 0; i < NXT_ROUTER_REMOVE_PID_SOAK_N; i++) { + baseline = (i % 2) ? 2 : 0; + + /* What a real initiator leaves behind: the slot and the work's use. */ + app->pending_processes = baseline + 1; + app->use_count = 2; + + nxt_router_start_app_process_handler(task, router_port, app); + + if (app->pending_processes != baseline + 1 || joint->use_count != 2 + || app->use_count != 1) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router remove pid soak test: iteration %ui armed " + "start left pending %uD joint %uD app use %d " + "(expected %uD, 2 and 1)", + i, app->pending_processes, joint->use_count, + (int) app->use_count, baseline + 1); + goto done; + } + + stream = nxt_router_remove_pid_soak_stream(dport); + + if (nxt_slow_path(stream == 0)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router remove pid soak test: iteration %ui start " + "queued no START_PROCESS message to read a stream " + "from", i); + goto done; + } + + /* + * A stream no start registered: the handler must find no + * registration and change nothing. Without this the release below + * could be credited to any bookkeeping REMOVE_PID happens to do. + */ + + nxt_router_remove_pid_soak_msg(&msg, &buf, router_port, &worker_pid, + stream + 1); + + nxt_router_process.port_handlers->remove_pid(task, &msg); + + if (app->pending_processes != baseline + 1 || joint->use_count != 2) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router remove pid soak test: iteration %ui " + "unregistered stream released pending %uD joint %uD " + "(expected %uD and 2)", + i, app->pending_processes, joint->use_count, + baseline + 1); + goto done; + } + + /* The real one: the worker died before it ever reported ready. */ + + nxt_router_remove_pid_soak_msg(&msg, &buf, router_port, &worker_pid, + stream); + + nxt_router_process.port_handlers->remove_pid(task, &msg); + + if (msg.port_msg.type != _NXT_PORT_MSG_RPC_ERROR + || msg.u.removed_pid != worker_pid) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router remove pid soak test: iteration %ui " + "REMOVE_PID was not turned into an RPC error for " + "pid %PI, type %d pid %PI", + i, worker_pid, (int) msg.port_msg.type, + msg.u.removed_pid); + goto done; + } + + if (app->pending_processes != baseline) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router remove pid soak test: iteration %ui " + "pending_processes %uD (expected %uD)", + i, app->pending_processes, baseline); + goto done; + } + + if (app->proto_port_requests != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router remove pid soak test: iteration %ui worker " + "start touched proto_port_requests %uD (expected 0)", + i, app->proto_port_requests); + goto done; + } + + if (joint->use_count != 1 || app->use_count != 1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router remove pid soak test: iteration %ui joint " + "use_count %uD app use_count %d (expected 1 and 1)", + i, joint->use_count, (int) app->use_count); + goto done; + } + } + + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router remove pid soak test passed: %d starts failed " + "through REMOVE_PID", NXT_ROUTER_REMOVE_PID_SOAK_N); + + ret = NXT_OK; + +done: + + if (dport != NULL) { + app->proto_port = NULL; + + /* + * Every start queued its START_PROCESS message rather than writing + * it, and nxt_port_msg_chk_insert() takes a port reference for each. + * Left alone the port never reaches nxt_port_mp_cleanup(), so its + * pool and messages leak and the assertions there never run -- which + * would quietly hollow out this test under a debug build. + */ + nxt_port_test_run_error_handler(task, dport); + + if (!nxt_queue_is_empty(&dport->messages)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router remove pid soak test: prototype port still " + "holds queued messages after teardown"); + ret = NXT_ERROR; + } + + nxt_port_use(task, dport, -1); + } + + if (router_port != NULL) { + router_port->pair[0] = -1; + + nxt_port_use(task, router_port, -1); + } + + if (main_port != NULL) { + rt->port_by_type[NXT_PROCESS_MAIN] = NULL; + + nxt_port_use(task, main_port, -1); + } + + thr->engine = saved_engine; + thr->runtime = saved_rt; + nxt_router = saved_router; + + if (app_mutex) { + nxt_thread_mutex_destroy(&app->mutex); + } + + nxt_work_queue_cache_destroy(&engine.work_queue_cache); + nxt_thread_mutex_destroy(&rt->processes_mutex); + nxt_mp_destroy(mp); + + return ret; +} diff --git a/src/test/nxt_router_response_parse_test.c b/src/test/nxt_router_response_parse_test.c new file mode 100644 index 000000000..73e3eb2b6 --- /dev/null +++ b/src/test/nxt_router_response_parse_test.c @@ -0,0 +1,197 @@ +/* + * Copyright (C) FreeUnit contributors. + */ + +/* + * nxt_router_response_header_parse() (src/nxt_router.c) decodes a response + * buffer the application wrote. Every count, length and sptr in it must be + * checked against the buffer, not followed. + */ + +#include +#include +#include +#include +#include "nxt_tests.h" + + +#define NXT_RRP_NAME "X-Test" +#define NXT_RRP_VALUE "value" +#define NXT_RRP_BODY "body" +#define NXT_RRP_SIZE \ + (sizeof(nxt_unit_response_t) + sizeof(nxt_unit_field_t) \ + + nxt_length(NXT_RRP_NAME) + nxt_length(NXT_RRP_VALUE) \ + + nxt_length(NXT_RRP_BODY)) + + +/* A well-formed response with one field and a piggybacked body. */ +static void +nxt_router_response_parse_test_fill(nxt_unit_response_t *resp) +{ + u_char *p; + nxt_unit_field_t *f; + + nxt_memzero(resp, NXT_RRP_SIZE); + + resp->status = 200; + resp->fields_count = 1; + resp->piggyback_content_length = nxt_length(NXT_RRP_BODY); + + f = &resp->fields[0]; + f->name_length = nxt_length(NXT_RRP_NAME); + f->value_length = nxt_length(NXT_RRP_VALUE); + + p = (u_char *) &resp->fields[1]; + + nxt_unit_sptr_set(&f->name, p); + p = nxt_cpymem(p, NXT_RRP_NAME, nxt_length(NXT_RRP_NAME)); + + nxt_unit_sptr_set(&f->value, p); + p = nxt_cpymem(p, NXT_RRP_VALUE, nxt_length(NXT_RRP_VALUE)); + + nxt_unit_sptr_set(&resp->piggyback_content, p); + nxt_memcpy(p, NXT_RRP_BODY, nxt_length(NXT_RRP_BODY)); +} + + +static void +nxt_rrp_edit_name(nxt_unit_response_t *resp) +{ + resp->fields[0].name.offset = 4096; +} + + +static void +nxt_rrp_edit_value(nxt_unit_response_t *resp) +{ + resp->fields[0].value.offset = 4096; +} + + +static void +nxt_rrp_edit_value_length(nxt_unit_response_t *resp) +{ + resp->fields[0].value_length = 4096; +} + + +static void +nxt_rrp_edit_count(nxt_unit_response_t *resp) +{ + resp->fields_count = 2; +} + + +static void +nxt_rrp_edit_piggyback(nxt_unit_response_t *resp) +{ + resp->piggyback_content.offset = 4096; +} + + +static void +nxt_rrp_edit_piggyback_length(nxt_unit_response_t *resp) +{ + resp->piggyback_content_length = 4096; +} + + +nxt_int_t +nxt_router_response_parse_test(nxt_thread_t *thr) +{ + nxt_mp_t *mp; + nxt_int_t ret; + nxt_uint_t i; + nxt_buf_t b; + nxt_task_t *task; + nxt_http_request_t *r; + + /* "resp" is in a page of its own, so a wrong offset stays mapped. */ + static union { + nxt_unit_response_t resp; + u_char page[8192]; + } u; + + static const struct { + const char *name; + void (*edit)(nxt_unit_response_t *resp); + size_t size; + nxt_int_t expect; + } tests[] = { + { "in-bounds response", NULL, NXT_RRP_SIZE, NXT_OK }, + { "short buffer", NULL, sizeof(nxt_unit_response_t) - 1, NXT_ERROR }, + { "fields_count past the buffer", nxt_rrp_edit_count, NXT_RRP_SIZE, + NXT_ERROR }, + { "name sptr out of the buffer", nxt_rrp_edit_name, NXT_RRP_SIZE, + NXT_ERROR }, + { "value sptr out of the buffer", nxt_rrp_edit_value, NXT_RRP_SIZE, + NXT_ERROR }, + { "value length past the buffer", nxt_rrp_edit_value_length, + NXT_RRP_SIZE, NXT_ERROR }, + { "piggyback sptr out of the buffer", nxt_rrp_edit_piggyback, + NXT_RRP_SIZE, NXT_ERROR }, + { "piggyback length past the buffer", nxt_rrp_edit_piggyback_length, + NXT_RRP_SIZE, NXT_ERROR }, + }; + + task = thr->task; + task->thread = thr; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + ret = NXT_OK; + + for (i = 0; ret == NXT_OK && i < nxt_nitems(tests); i++) { + r = nxt_mp_zalloc(mp, sizeof(nxt_http_request_t)); + if (r == NULL) { + ret = NXT_ERROR; + break; + } + + r->mem_pool = mp; + r->task = *task; + + nxt_router_response_parse_test_fill(&u.resp); + + if (tests[i].edit != NULL) { + tests[i].edit(&u.resp); + } + + nxt_memzero(&b, sizeof(nxt_buf_t)); + b.mem.start = u.page; + b.mem.pos = u.page; + b.mem.free = u.page + tests[i].size; + b.mem.end = b.mem.free; + + if (nxt_router_test_response_header_parse(task, r, &b) + != tests[i].expect) + { + nxt_log_alert(task->log, "response parse test \"%s\" failed", + tests[i].name); + ret = NXT_ERROR; + break; + } + + if (tests[i].expect == NXT_OK + && (r->status != 200 + || nxt_buf_mem_used_size(&b.mem) != nxt_length(NXT_RRP_BODY) + || memcmp(b.mem.pos, NXT_RRP_BODY, nxt_length(NXT_RRP_BODY)) + != 0)) + { + nxt_log_alert(task->log, "response parse test \"%s\": " + "wrong status or body", tests[i].name); + ret = NXT_ERROR; + } + } + + nxt_mp_destroy(mp); + + if (ret == NXT_OK) { + nxt_log(task, NXT_LOG_NOTICE, "router response parse test passed"); + } + + return ret; +} diff --git a/src/test/nxt_router_sender_test.c b/src/test/nxt_router_sender_test.c new file mode 100644 index 000000000..90f3f9116 --- /dev/null +++ b/src/test/nxt_router_sender_test.c @@ -0,0 +1,831 @@ +/* + * Copyright (C) FreeUnit contributors. + */ + +/* + * Test for the sender checks on the main port of the router + * (src/nxt_router.c, issue #341). The router accepts these message types + * only from these senders: + * + * QUIT, CHANGE_FILE, ACCESS_LOG main + * REMOVE_PID main or a registered prototype + * DATA, APP_RESTART, STATUS the controller + * RPC_READY, RPC_ERROR main, the controller or a prototype + * GET_PORT, GET_MMAP, MMAP, OOSM a registered worker, for itself + * NEW_PORT main; a prototype for an application + * port; a worker for its own + * application port, with no stream + * + * The test sends each type through the handler table of the router from + * each sender: main, the controller, a prototype, a worker, an unknown pid + * and no credential. A wrong sender sends twice: with its own pid in the + * header, and with the pid of the controller or of the worker. A refused + * message must add one to the refusal counter, close its two descriptors, + * and not run the handler. The fixture makes most handlers that run crash: + * nxt_router is NULL, and the reply ports that the messages name are + * registered. Thus the test first calls the check alone, which reports a + * failure as a message, and then sends the message through the table. + * + * An accepted message must not change the counter. The test calls the + * check of each type directly. It also sends CHANGE_FILE, DATA, + * APP_RESTART, STATUS and MMAP through the table: their bodies find no + * payload, no reply port or no segment, log that, and return. The other + * types are not sent from their sender, because their bodies stop the + * process, need a started router, or write to a port. + * + * A table of cases then sends a correct sender with a wrong body: a + * NEW_PORT that is not an application port, not the port of the sender, + * has a stream or is short; a GET_PORT for a port that is not a router + * port; a GET_MMAP that names a reply port that is not an application + * port. + * + * A second pass runs with no main port and no controller port, as in a + * restarted router before NEW_PORT. + */ + +#include +#include +#include +#include +#include +#include +#include "nxt_tests.h" + +#include + + +#if (NXT_USE_CMSG_PID) + +/* + * The senders. The first four have a port; NONE sends no credential. + * ROUTER is not a sender: it owns a port with the pid of this process. + */ + +enum { + NXT_ROUTER_SENDER_TEST_MAIN = 0, + NXT_ROUTER_SENDER_TEST_CONTROLLER, + NXT_ROUTER_SENDER_TEST_PROTO, + NXT_ROUTER_SENDER_TEST_WORKER, + NXT_ROUTER_SENDER_TEST_STRANGER, + NXT_ROUTER_SENDER_TEST_NONE, + NXT_ROUTER_SENDER_TEST_PORTS = NXT_ROUTER_SENDER_TEST_STRANGER, + NXT_ROUTER_SENDER_TEST_SENDERS = NXT_ROUTER_SENDER_TEST_NONE + 1, + NXT_ROUTER_SENDER_TEST_ROUTER = NXT_ROUTER_SENDER_TEST_SENDERS, +}; + + +/* The senders that a type accepts. */ + +enum { + NXT_ROUTER_SENDER_TEST_FROM_MAIN = 0, + NXT_ROUTER_SENDER_TEST_FROM_CONTROLLER, + NXT_ROUTER_SENDER_TEST_FROM_MAIN_OR_PROTO, + NXT_ROUTER_SENDER_TEST_FROM_NOT_APP, + NXT_ROUTER_SENDER_TEST_FROM_SELF, + NXT_ROUTER_SENDER_TEST_FROM_NEW_PORT, +}; + + +/* The changes to a correct message in the table of cases. */ + +enum { + NXT_ROUTER_SENDER_TEST_PORT_MAIN = 0, + NXT_ROUTER_SENDER_TEST_PORT_CONTROLLER, + NXT_ROUTER_SENDER_TEST_PORT_OF_PROTO, + NXT_ROUTER_SENDER_TEST_STREAM, + NXT_ROUTER_SENDER_TEST_SHORT, + NXT_ROUTER_SENDER_TEST_FOREIGN_PORT, + NXT_ROUTER_SENDER_TEST_FOREIGN_PORT_UNHASHED, + NXT_ROUTER_SENDER_TEST_REPLY_NOT_APP, +}; + + +static const char *nxt_router_sender_test_names[] = { + "main", "the controller", "a prototype", "a worker", "a stranger", + "no credential", +}; + + +static const nxt_process_type_t nxt_router_sender_test_port_types[] = { + NXT_PROCESS_MAIN, NXT_PROCESS_CONTROLLER, NXT_PROCESS_PROTOTYPE, + NXT_PROCESS_APP, +}; + + +/* The port ids in the port hash of the fixture. */ + +#define NXT_ROUTER_SENDER_TEST_REPLY_ID 1 +#define NXT_ROUTER_SENDER_TEST_ENGINE_ID 2 +#define NXT_ROUTER_SENDER_TEST_NOT_APP_ID 3 + + +typedef struct { + nxt_uint_t owner; + nxt_port_id_t id; + nxt_process_type_t type; +} nxt_router_sender_test_hashed_t; + + +static const nxt_router_sender_test_hashed_t + nxt_router_sender_test_hashed[] = +{ + /* The reply port that a forged DATA, APP_RESTART or STATUS names. */ + { NXT_ROUTER_SENDER_TEST_CONTROLLER, NXT_ROUTER_SENDER_TEST_REPLY_ID, + NXT_PROCESS_CONTROLLER }, + /* The reply port of the worker for GET_PORT and GET_MMAP. */ + { NXT_ROUTER_SENDER_TEST_WORKER, NXT_ROUTER_SENDER_TEST_REPLY_ID, + NXT_PROCESS_APP }, + /* A router engine port: the only kind that GET_PORT gives out. */ + { NXT_ROUTER_SENDER_TEST_ROUTER, NXT_ROUTER_SENDER_TEST_ENGINE_ID, + NXT_PROCESS_ROUTER }, + /* A port of another type with the pid of the worker, for GET_MMAP. */ + { NXT_ROUTER_SENDER_TEST_WORKER, NXT_ROUTER_SENDER_TEST_NOT_APP_ID, + NXT_PROCESS_ROUTER }, +}; + + +typedef struct { + const char *name; + nxt_uint_t type; + nxt_uint_t from; + /* The pid in the header of a message from a wrong sender. */ + nxt_uint_t claimed; + /* The body is safe to run from the correct sender. */ + nxt_bool_t run; +} nxt_router_sender_test_type_t; + + +static const nxt_router_sender_test_type_t nxt_router_sender_test_types[] = { + { "QUIT", _NXT_PORT_MSG_QUIT, + NXT_ROUTER_SENDER_TEST_FROM_MAIN, + NXT_ROUTER_SENDER_TEST_CONTROLLER, 0 }, + { "CHANGE_FILE", _NXT_PORT_MSG_CHANGE_FILE, + NXT_ROUTER_SENDER_TEST_FROM_MAIN, + NXT_ROUTER_SENDER_TEST_CONTROLLER, 1 }, + { "ACCESS_LOG", _NXT_PORT_MSG_ACCESS_LOG, + NXT_ROUTER_SENDER_TEST_FROM_MAIN, + NXT_ROUTER_SENDER_TEST_CONTROLLER, 0 }, + { "REMOVE_PID", _NXT_PORT_MSG_REMOVE_PID, + NXT_ROUTER_SENDER_TEST_FROM_MAIN_OR_PROTO, + NXT_ROUTER_SENDER_TEST_CONTROLLER, 0 }, + { "DATA", _NXT_PORT_MSG_DATA, + NXT_ROUTER_SENDER_TEST_FROM_CONTROLLER, + NXT_ROUTER_SENDER_TEST_CONTROLLER, 1 }, + { "APP_RESTART", _NXT_PORT_MSG_APP_RESTART, + NXT_ROUTER_SENDER_TEST_FROM_CONTROLLER, + NXT_ROUTER_SENDER_TEST_CONTROLLER, 1 }, + { "STATUS", _NXT_PORT_MSG_STATUS, + NXT_ROUTER_SENDER_TEST_FROM_CONTROLLER, + NXT_ROUTER_SENDER_TEST_CONTROLLER, 1 }, + { "RPC_READY", _NXT_PORT_MSG_RPC_READY, + NXT_ROUTER_SENDER_TEST_FROM_NOT_APP, + NXT_ROUTER_SENDER_TEST_CONTROLLER, 0 }, + { "RPC_ERROR", _NXT_PORT_MSG_RPC_ERROR, + NXT_ROUTER_SENDER_TEST_FROM_NOT_APP, + NXT_ROUTER_SENDER_TEST_CONTROLLER, 0 }, + { "GET_PORT", _NXT_PORT_MSG_GET_PORT, + NXT_ROUTER_SENDER_TEST_FROM_SELF, + NXT_ROUTER_SENDER_TEST_WORKER, 0 }, + { "GET_MMAP", _NXT_PORT_MSG_GET_MMAP, + NXT_ROUTER_SENDER_TEST_FROM_SELF, + NXT_ROUTER_SENDER_TEST_WORKER, 0 }, + { "MMAP", _NXT_PORT_MSG_MMAP, + NXT_ROUTER_SENDER_TEST_FROM_SELF, + NXT_ROUTER_SENDER_TEST_WORKER, 1 }, + { "OOSM", _NXT_PORT_MSG_OOSM, + NXT_ROUTER_SENDER_TEST_FROM_SELF, + NXT_ROUTER_SENDER_TEST_WORKER, 0 }, + { "NEW_PORT", _NXT_PORT_MSG_NEW_PORT, + NXT_ROUTER_SENDER_TEST_FROM_NEW_PORT, + NXT_ROUTER_SENDER_TEST_WORKER, 0 }, +}; + + +typedef struct { + const char *name; + nxt_uint_t type; + nxt_uint_t sender; + nxt_uint_t change; +} nxt_router_sender_test_case_t; + + +static const nxt_router_sender_test_case_t nxt_router_sender_test_cases[] = { + { "NEW_PORT of type MAIN", _NXT_PORT_MSG_NEW_PORT, + NXT_ROUTER_SENDER_TEST_WORKER, NXT_ROUTER_SENDER_TEST_PORT_MAIN }, + { "NEW_PORT of type CONTROLLER", _NXT_PORT_MSG_NEW_PORT, + NXT_ROUTER_SENDER_TEST_PROTO, NXT_ROUTER_SENDER_TEST_PORT_CONTROLLER }, + { "NEW_PORT of type MAIN", _NXT_PORT_MSG_NEW_PORT, + NXT_ROUTER_SENDER_TEST_PROTO, NXT_ROUTER_SENDER_TEST_PORT_MAIN }, + { "NEW_PORT for the prototype", _NXT_PORT_MSG_NEW_PORT, + NXT_ROUTER_SENDER_TEST_WORKER, NXT_ROUTER_SENDER_TEST_PORT_OF_PROTO }, + { "NEW_PORT with a stream", _NXT_PORT_MSG_NEW_PORT, + NXT_ROUTER_SENDER_TEST_WORKER, NXT_ROUTER_SENDER_TEST_STREAM }, + { "short NEW_PORT", _NXT_PORT_MSG_NEW_PORT, + NXT_ROUTER_SENDER_TEST_WORKER, NXT_ROUTER_SENDER_TEST_SHORT }, + { "short NEW_PORT", _NXT_PORT_MSG_NEW_PORT, + NXT_ROUTER_SENDER_TEST_PROTO, NXT_ROUTER_SENDER_TEST_SHORT }, + { "GET_PORT for the controller port", _NXT_PORT_MSG_GET_PORT, + NXT_ROUTER_SENDER_TEST_WORKER, NXT_ROUTER_SENDER_TEST_FOREIGN_PORT }, + { "GET_PORT for an unhashed foreign port", _NXT_PORT_MSG_GET_PORT, + NXT_ROUTER_SENDER_TEST_WORKER, + NXT_ROUTER_SENDER_TEST_FOREIGN_PORT_UNHASHED }, + { "short GET_PORT", _NXT_PORT_MSG_GET_PORT, + NXT_ROUTER_SENDER_TEST_WORKER, NXT_ROUTER_SENDER_TEST_SHORT }, + { "GET_MMAP to a port that is not an application port", + _NXT_PORT_MSG_GET_MMAP, + NXT_ROUTER_SENDER_TEST_WORKER, NXT_ROUTER_SENDER_TEST_REPLY_NOT_APP }, +}; + + +typedef struct { + nxt_port_recv_msg_t msg; + nxt_buf_t buf; + + union { + nxt_port_msg_new_port_t new_port; + nxt_port_msg_get_port_t get_port; + nxt_port_msg_get_mmap_t get_mmap; + } body; +} nxt_router_sender_test_msg_t; + + +static nxt_pid_t +nxt_router_sender_test_pid(nxt_uint_t sender) +{ + if (sender == NXT_ROUTER_SENDER_TEST_NONE) { + return -1; + } + + if (sender == NXT_ROUTER_SENDER_TEST_ROUTER) { + return nxt_pid; + } + + return nxt_pid + 1 + sender; +} + + +/* Whether the type accepts the sender; known: the router has both ports. */ + +static nxt_bool_t +nxt_router_sender_test_accepts(nxt_uint_t from, nxt_uint_t sender, + nxt_bool_t known) +{ + nxt_bool_t main, controller, proto, worker; + + main = (known && sender == NXT_ROUTER_SENDER_TEST_MAIN); + controller = (known && sender == NXT_ROUTER_SENDER_TEST_CONTROLLER); + proto = (sender == NXT_ROUTER_SENDER_TEST_PROTO); + worker = (sender == NXT_ROUTER_SENDER_TEST_WORKER); + + switch (from) { + + case NXT_ROUTER_SENDER_TEST_FROM_MAIN: + return main; + + case NXT_ROUTER_SENDER_TEST_FROM_CONTROLLER: + return controller; + + case NXT_ROUTER_SENDER_TEST_FROM_MAIN_OR_PROTO: + return (main || proto); + + case NXT_ROUTER_SENDER_TEST_FROM_NOT_APP: + return (main || controller || proto); + + case NXT_ROUTER_SENDER_TEST_FROM_SELF: + return worker; + + default: + return (main || proto || worker); + } +} + + +/* + * A message of the type from "sender", with the pid of "claimed" in the + * header. NEW_PORT announces an application port of "claimed", GET_PORT + * asks for the router engine port, and GET_MMAP asks for segment 0. The + * reply port of each message has the id NXT_ROUTER_SENDER_TEST_REPLY_ID. + */ + +static void +nxt_router_sender_test_msg(nxt_router_sender_test_msg_t *m, nxt_uint_t type, + nxt_uint_t claimed, nxt_uint_t sender) +{ + size_t size; + nxt_port_recv_msg_t *msg; + + nxt_memzero(m, sizeof(nxt_router_sender_test_msg_t)); + + msg = &m->msg; + + msg->fd[0] = -1; + msg->fd[1] = -1; + msg->port_msg.pid = nxt_router_sender_test_pid(claimed); + msg->port_msg.reply_port = NXT_ROUTER_SENDER_TEST_REPLY_ID; + msg->port_msg.type = type; + msg->port_msg.last = 1; + msg->cmsg_pid = nxt_router_sender_test_pid(sender); + + switch (type) { + + case _NXT_PORT_MSG_NEW_PORT: + m->body.new_port.pid = msg->port_msg.pid; + m->body.new_port.id = 5; + m->body.new_port.type = NXT_PROCESS_APP; + m->body.new_port.max_size = 1024; + m->body.new_port.max_share = 1024; + size = sizeof(nxt_port_msg_new_port_t); + break; + + case _NXT_PORT_MSG_GET_PORT: + m->body.get_port.pid = nxt_pid; + m->body.get_port.id = NXT_ROUTER_SENDER_TEST_ENGINE_ID; + size = sizeof(nxt_port_msg_get_port_t); + break; + + case _NXT_PORT_MSG_GET_MMAP: + m->body.get_mmap.id = 0; + size = sizeof(nxt_port_msg_get_mmap_t); + break; + + default: + /* The other types are sent with no payload. */ + return; + } + + m->buf.mem.start = (u_char *) &m->body; + m->buf.mem.pos = m->buf.mem.start; + m->buf.mem.free = m->buf.mem.start + size; + m->buf.mem.end = m->buf.mem.free; + + msg->buf = &m->buf; +} + + +static void +nxt_router_sender_test_change(nxt_router_sender_test_msg_t *m, + nxt_uint_t change) +{ + switch (change) { + + case NXT_ROUTER_SENDER_TEST_PORT_MAIN: + m->body.new_port.type = NXT_PROCESS_MAIN; + break; + + case NXT_ROUTER_SENDER_TEST_PORT_CONTROLLER: + m->body.new_port.type = NXT_PROCESS_CONTROLLER; + break; + + case NXT_ROUTER_SENDER_TEST_PORT_OF_PROTO: + m->body.new_port.pid = + nxt_router_sender_test_pid(NXT_ROUTER_SENDER_TEST_PROTO); + break; + + case NXT_ROUTER_SENDER_TEST_STREAM: + m->msg.port_msg.stream = 7; + break; + + case NXT_ROUTER_SENDER_TEST_SHORT: + m->buf.mem.free--; + break; + + case NXT_ROUTER_SENDER_TEST_FOREIGN_PORT: + m->body.get_port.pid = + nxt_router_sender_test_pid(NXT_ROUTER_SENDER_TEST_CONTROLLER); + m->body.get_port.id = NXT_ROUTER_SENDER_TEST_REPLY_ID; + break; + + case NXT_ROUTER_SENDER_TEST_FOREIGN_PORT_UNHASHED: + m->body.get_port.pid = + nxt_router_sender_test_pid(NXT_ROUTER_SENDER_TEST_CONTROLLER); + m->body.get_port.id = 999; + break; + + default: + m->msg.port_msg.reply_port = NXT_ROUTER_SENDER_TEST_NOT_APP_ID; + break; + } +} + + +/* As nxt_port_handler() does, through the table of the router. */ + +static void +nxt_router_sender_test_dispatch(nxt_task_t *task, nxt_port_recv_msg_t *msg) +{ + nxt_port_handler_t *handlers; + + handlers = (nxt_port_handler_t *) nxt_router_process.port_handlers; + + handlers[msg->port_msg.type](task, msg); +} + + +/* + * The check alone must refuse the message. Then the message goes through + * the table with two descriptors, and the router must refuse it again and + * close them. "m" is changed. + */ + +static nxt_int_t +nxt_router_sender_test_refused(nxt_thread_t *thr, nxt_task_t *task, + nxt_router_sender_test_msg_t *m, const char *name, const char *who) +{ + nxt_fd_t fd[2]; + nxt_bool_t res; + nxt_uint_t i, refused; + const char *err; + + refused = nxt_router_test_senders_refused; + + res = nxt_router_test_msg_sender_ok(task, &m->msg); + + NXT_TEST_CHECK(thr->log, !res, "router sender test: %s from %s: check " + "returned 1", name, who); + + NXT_TEST_CHECK(thr->log, nxt_router_test_senders_refused == refused + 1, + "router sender test: %s from %s: check did not count a " + "refusal", name, who); + + for (i = 0; i < 2; i++) { + fd[i] = open("/dev/null", O_RDONLY); + m->msg.fd[i] = fd[i]; + } + + err = NULL; + + if (fd[0] == -1 || fd[1] == -1) { + err = "failed to open /dev/null"; + goto done; + } + + refused = nxt_router_test_senders_refused; + + nxt_router_sender_test_dispatch(task, &m->msg); + + if (nxt_router_test_senders_refused != refused + 1) { + err = "was not refused"; + + } else if (nxt_test_fd_is_open(fd[0]) || nxt_test_fd_is_open(fd[1])) { + err = "left a descriptor open"; + + } else if (m->msg.fd[0] != -1 || m->msg.fd[1] != -1) { + err = "left fds in the message"; + } + +done: + + for (i = 0; i < 2; i++) { + if (fd[i] != -1 && nxt_test_fd_is_open(fd[i])) { + (void) close(fd[i]); + } + } + + if (err != NULL) { + nxt_log_alert(thr->log, "router sender test: %s from %s %s", name, + who, err); + return NXT_ERROR; + } + + return NXT_OK; +} + + +/* + * A wrong sender, twice: with its own pid in the header, and with the pid + * of the controller or of the worker. The body of a DATA, APP_RESTART or + * STATUS that names the controller finds the reply port and crashes on + * nxt_router. + */ + +static nxt_int_t +nxt_router_sender_test_wrong(nxt_thread_t *thr, nxt_task_t *task, + const nxt_router_sender_test_type_t *t, nxt_uint_t sender) +{ + nxt_int_t ret; + const char *who; + nxt_router_sender_test_msg_t m; + + who = nxt_router_sender_test_names[sender]; + + nxt_router_sender_test_msg(&m, t->type, sender, sender); + + ret = nxt_router_sender_test_refused(thr, task, &m, t->name, who); + + if (ret == NXT_OK && sender != t->claimed) { + nxt_router_sender_test_msg(&m, t->type, t->claimed, sender); + + ret = nxt_router_sender_test_refused(thr, task, &m, t->name, who); + } + + return ret; +} + + +static nxt_int_t +nxt_router_sender_test_accepted(nxt_thread_t *thr, nxt_task_t *task, + const nxt_router_sender_test_type_t *t, nxt_uint_t sender) +{ + nxt_bool_t res; + nxt_uint_t refused, claimed; + const char *who; + nxt_router_sender_test_msg_t m; + + who = nxt_router_sender_test_names[sender]; + + refused = nxt_router_test_senders_refused; + + nxt_router_sender_test_msg(&m, t->type, sender, sender); + + res = nxt_router_test_msg_sender_ok(task, &m.msg); + + NXT_TEST_CHECK(thr->log, res, "router sender test: %s from %s: check " + "returned 0", t->name, who); + + if (t->run) { + /* + * The header names a stranger: the body finds no reply port. A + * worker must name itself. + */ + claimed = (t->from == NXT_ROUTER_SENDER_TEST_FROM_SELF) + ? sender : NXT_ROUTER_SENDER_TEST_STRANGER; + + nxt_router_sender_test_msg(&m, t->type, claimed, sender); + + nxt_router_sender_test_dispatch(task, &m.msg); + } + + NXT_TEST_CHECK(thr->log, nxt_router_test_senders_refused == refused, + "router sender test: %s from %s was refused", t->name, + who); + + return NXT_OK; +} + + +static nxt_int_t +nxt_router_sender_test_run(nxt_thread_t *thr, nxt_task_t *task, + nxt_bool_t known) +{ + nxt_int_t ret; + nxt_uint_t i, s; + nxt_router_sender_test_msg_t m; + + const nxt_router_sender_test_case_t *c; + const nxt_router_sender_test_type_t *t; + + for (i = 0; i < nxt_nitems(nxt_router_sender_test_types); i++) { + t = &nxt_router_sender_test_types[i]; + + for (s = 0; s < NXT_ROUTER_SENDER_TEST_SENDERS; s++) { + if (nxt_router_sender_test_accepts(t->from, s, known)) { + ret = nxt_router_sender_test_accepted(thr, task, t, s); + + } else { + ret = nxt_router_sender_test_wrong(thr, task, t, s); + } + + if (ret != NXT_OK) { + return ret; + } + } + } + + for (i = 0; i < nxt_nitems(nxt_router_sender_test_cases); i++) { + c = &nxt_router_sender_test_cases[i]; + + nxt_router_sender_test_msg(&m, c->type, c->sender, c->sender); + nxt_router_sender_test_change(&m, c->change); + + ret = nxt_router_sender_test_refused(thr, task, &m, c->name, + nxt_router_sender_test_names[c->sender]); + if (ret != NXT_OK) { + return ret; + } + } + + return NXT_OK; +} + + +static nxt_port_t * +nxt_router_sender_test_port(nxt_task_t *task, nxt_port_id_t id, + nxt_uint_t sender, nxt_process_type_t type) +{ + nxt_port_t *port; + + port = nxt_port_new(task, id, nxt_router_sender_test_pid(sender), type); + + if (nxt_fast_path(port != NULL)) { + port->pair[0] = -1; + port->pair[1] = -1; + port->socket.fd = -1; + } + + return port; +} + + +/* A registered process with one port, as the NEW_PORT of main leaves it. */ + +static nxt_port_t * +nxt_router_sender_test_process(nxt_task_t *task, nxt_mp_t *mp, + nxt_uint_t sender, nxt_process_type_t type, nxt_process_t **processp) +{ + nxt_port_t *port; + nxt_process_t *process; + + process = nxt_mp_zalloc(mp, sizeof(nxt_process_t)); + if (nxt_slow_path(process == NULL)) { + return NULL; + } + + process->pid = nxt_router_sender_test_pid(sender); + process->isolated_pid = process->pid; + process->state = NXT_PROCESS_STATE_READY; + nxt_queue_init(&process->ports); + + nxt_runtime_process_add(task, process); + + *processp = process; + + port = nxt_router_sender_test_port(task, 0, sender, type); + if (nxt_slow_path(port == NULL)) { + return NULL; + } + + /* The hold of the fixture on the record. */ + process->use_count = 1; + + nxt_process_port_add(task, process, port); + + return port; +} + +#endif + + +nxt_int_t +nxt_router_sender_test(nxt_thread_t *thr) +{ +#if !(NXT_USE_CMSG_PID) + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router sender test skipped: no sender credentials"); + return NXT_OK; + +#else + + nxt_mp_t *mp; + nxt_int_t ret; + nxt_uint_t i; + nxt_task_t *task; + nxt_port_t *port, *ports[NXT_ROUTER_SENDER_TEST_PORTS]; + nxt_port_t *hashed[nxt_nitems(nxt_router_sender_test_hashed)]; + nxt_bool_t mutex; + nxt_router_t *saved_router; + nxt_runtime_t *rt, *saved_rt; + nxt_process_t *processes[NXT_ROUTER_SENDER_TEST_PORTS]; + nxt_event_engine_t engine, *saved_engine; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "router sender test started"); + + ret = NXT_ERROR; + mutex = 0; + + nxt_memzero(ports, sizeof(ports)); + nxt_memzero(hashed, sizeof(hashed)); + nxt_memzero(processes, sizeof(processes)); + + task = thr->task; + task->thread = thr; + + saved_rt = thr->runtime; + saved_engine = thr->engine; + saved_router = nxt_router; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + rt = nxt_mp_zalloc(mp, sizeof(nxt_runtime_t)); + if (nxt_slow_path(rt == NULL)) { + goto done; + } + + rt->mem_pool = mp; + + if (nxt_slow_path(nxt_thread_mutex_create(&rt->processes_mutex) != NXT_OK)) + { + goto done; + } + + mutex = 1; + + /* The handlers run on the main engine only. */ + + nxt_memzero(&engine, sizeof(nxt_event_engine_t)); + + thr->runtime = rt; + thr->engine = &engine; + rt->main_engine = &engine; + + /* A body that got past its check would dereference this. */ + + nxt_router = NULL; + + for (i = 0; i < NXT_ROUTER_SENDER_TEST_PORTS; i++) { + if (i < NXT_ROUTER_SENDER_TEST_PROTO) { + ports[i] = nxt_router_sender_test_port(task, 0, i, + nxt_router_sender_test_port_types[i]); + + } else { + ports[i] = nxt_router_sender_test_process(task, mp, i, + nxt_router_sender_test_port_types[i], + &processes[i]); + } + + if (nxt_slow_path(ports[i] == NULL)) { + goto done; + } + } + + /* The ports that nxt_runtime_port_find() finds. */ + + for (i = 0; i < nxt_nitems(nxt_router_sender_test_hashed); i++) { + port = nxt_router_sender_test_port(task, + nxt_router_sender_test_hashed[i].id, + nxt_router_sender_test_hashed[i].owner, + nxt_router_sender_test_hashed[i].type); + if (nxt_slow_path(port == NULL)) { + goto done; + } + + if (nxt_slow_path(nxt_port_hash_add(&rt->ports, port) != NXT_OK)) { + nxt_port_use(task, port, -1); + goto done; + } + + hashed[i] = port; + } + + rt->port_by_type[NXT_PROCESS_MAIN] = ports[NXT_ROUTER_SENDER_TEST_MAIN]; + rt->port_by_type[NXT_PROCESS_CONTROLLER] = + ports[NXT_ROUTER_SENDER_TEST_CONTROLLER]; + + ret = nxt_router_sender_test_run(thr, task, 1); + + if (ret == NXT_OK) { + rt->port_by_type[NXT_PROCESS_MAIN] = NULL; + rt->port_by_type[NXT_PROCESS_CONTROLLER] = NULL; + + ret = nxt_router_sender_test_run(thr, task, 0); + } + + if (ret == NXT_OK) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, "router sender test passed"); + } + +done: + + if (rt != NULL) { + rt->port_by_type[NXT_PROCESS_MAIN] = NULL; + rt->port_by_type[NXT_PROCESS_CONTROLLER] = NULL; + } + + for (i = 0; i < nxt_nitems(nxt_router_sender_test_hashed); i++) { + if (hashed[i] != NULL) { + (void) nxt_port_hash_remove(&rt->ports, hashed[i]); + nxt_port_use(task, hashed[i], -1); + } + } + + for (i = 0; i < NXT_ROUTER_SENDER_TEST_PORTS; i++) { + if (ports[i] != NULL) { + nxt_port_use(task, ports[i], -1); + } + } + + for (i = 0; i < NXT_ROUTER_SENDER_TEST_PORTS; i++) { + if (processes[i] != NULL && processes[i]->registered) { + nxt_runtime_process_remove(rt, processes[i]); + } + } + + nxt_router = saved_router; + thr->engine = saved_engine; + thr->runtime = saved_rt; + + if (mutex) { + nxt_thread_mutex_destroy(&rt->processes_mutex); + } + + nxt_mp_destroy(mp); + + return ret; + +#endif +} diff --git a/src/test/nxt_router_stale_joint_test.c b/src/test/nxt_router_stale_joint_test.c new file mode 100644 index 000000000..9a16e72e7 --- /dev/null +++ b/src/test/nxt_router_stale_joint_test.c @@ -0,0 +1,547 @@ +/* + * Copyright (C) FreeUnit contributors. + */ + +/* + * When nxt_listen_event() cannot allocate the listen event, the router + * create job acknowledges the configuration but leaves its joint in + * engine->joints, with the listen socket reference that the job took. A + * later update or delete of that listener must release both. Otherwise + * the old configurations stay referenced, engine->joints never becomes + * empty (so a worker thread that has to quit never exits), and a listen + * socket that no engine could use is never closed. + * + * The stale joints are matched by the listen socket: when an engine is + * removed, the delete job has the socket conf of the new configuration, + * and the stale joint has the old one. + * + * The fixture is a set of joints in one engine. The socket confs and the + * listen sockets keep more than one reference, so a release only unlinks + * the joint and drops one reference of each; the test reads the counts. + * + * nxt_router_stale_joint_jobs() then runs the real create and update jobs + * on a fixture engine: a create whose listen event allocation fails must + * keep its joint and take its listen socket reference, and the next update + * of that listener must release both. + */ + +#include +#include +#include +#include +#include +#include "nxt_tests.h" + + +#define NXT_STALE_JOINTS 5 + + +static nxt_bool_t +nxt_router_stale_joint_linked(nxt_event_engine_t *engine, + nxt_socket_conf_joint_t *joint) +{ + nxt_queue_link_t *qlk; + + for (qlk = nxt_queue_first(&engine->joints); + qlk != nxt_queue_tail(&engine->joints); + qlk = nxt_queue_next(qlk)) + { + if (qlk == &joint->link) { + return 1; + } + } + + return 0; +} + + +static nxt_int_t +nxt_router_stale_joint_check(nxt_thread_t *thr, const char *step, + nxt_event_engine_t *engine, nxt_socket_conf_joint_t *joint, + const nxt_bool_t *linked) +{ + nxt_uint_t i; + + for (i = 0; i < NXT_STALE_JOINTS; i++) { + if (nxt_router_stale_joint_linked(engine, &joint[i]) != linked[i]) { + nxt_log_alert(thr->log, "router stale joint test failed: %s: " + "joint %ui linked %d, expected %d", step, i, + !linked[i], linked[i]); + return NXT_ERROR; + } + + if (joint[i].socket_conf->count != (linked[i] ? 2 : 1)) { + nxt_log_alert(thr->log, "router stale joint test failed: %s: " + "socket conf %ui count %D", step, i, + joint[i].socket_conf->count); + return NXT_ERROR; + } + } + + return NXT_OK; +} + + +static nxt_int_t +nxt_router_stale_joint_last(nxt_thread_t *thr, nxt_router_conf_t *rtcf, + nxt_event_engine_t *engine) +{ + int fd; + nxt_sockaddr_t sa; + nxt_socket_conf_t skcf; + nxt_listen_socket_t *ls; + nxt_socket_conf_joint_t joint; + + fd = socket(AF_INET, SOCK_STREAM, 0); + if (fd == -1) { + nxt_log_alert(thr->log, "router stale joint test: socket() failed %E", + nxt_errno); + return NXT_ERROR; + } + + ls = nxt_zalloc(sizeof(nxt_listen_socket_t)); + if (ls == NULL) { + (void) close(fd); + return NXT_ERROR; + } + + nxt_memzero(&sa, sizeof(nxt_sockaddr_t)); + sa.u.sockaddr.sa_family = AF_INET; + + ls->socket = fd; + ls->sockaddr = &sa; + ls->count = 1; + + nxt_memzero(&skcf, sizeof(nxt_socket_conf_t)); + skcf.count = 2; + skcf.router_conf = rtcf; + skcf.listen = ls; + + nxt_memzero(&joint, sizeof(nxt_socket_conf_joint_t)); + joint.count = 1; + joint.stale = 1; + joint.socket_conf = &skcf; + joint.engine = engine; + + nxt_queue_insert_tail(&engine->joints, &joint.link); + + /* ls is freed here; only the saved fd is used after the call. */ + nxt_router_test_listen_socket_release_stale(thr->task, engine, ls, NULL); + + if (nxt_router_stale_joint_linked(engine, &joint)) { + nxt_log_alert(thr->log, "router stale joint test failed: last: " + "the joint is still linked"); + return NXT_ERROR; + } + + if (fcntl(fd, F_GETFD) != -1 || errno != EBADF) { + nxt_log_alert(thr->log, "router stale joint test failed: last: " + "the listen socket %d was not closed", fd); + (void) close(fd); + return NXT_ERROR; + } + + return NXT_OK; +} + + +static nxt_uint_t nxt_router_stale_joint_signals; + + +static void +nxt_router_stale_joint_signal(nxt_event_engine_t *engine, nxt_uint_t signo) +{ + nxt_router_stale_joint_signals++; +} + + +static void +nxt_router_stale_joint_enable_accept(nxt_event_engine_t *engine, + nxt_fd_event_t *ev) +{ + ev->read = NXT_EVENT_ACTIVE; +} + + +static void +nxt_router_stale_joint_accept(nxt_task_t *task, void *obj, void *data) +{ +} + + +static nxt_conn_io_t nxt_router_stale_joint_io = { + .accept = nxt_router_stale_joint_accept, +}; + + +/* + * Checks that the job was acknowledged: nxt_router_conf_wait_post() posts + * the work of the job to the engine of the temporary configuration. + */ + +static nxt_int_t +nxt_router_stale_joint_posted(nxt_thread_t *thr, const char *step, + nxt_event_engine_t *conf_engine, nxt_joint_job_t *job) +{ + if (conf_engine->locked_work_queue.head != &job->work + || nxt_router_stale_joint_signals != 1) + { + nxt_log_alert(thr->log, "router stale joint test failed: %s: the " + "job was not acknowledged", step); + return NXT_ERROR; + } + + conf_engine->locked_work_queue.head = NULL; + conf_engine->locked_work_queue.tail = NULL; + nxt_router_stale_joint_signals = 0; + + return NXT_OK; +} + + +static nxt_int_t +nxt_router_stale_joint_jobs(nxt_thread_t *thr, nxt_router_conf_t *rtcf) +{ + void *obj, *data; + nxt_int_t ret; + nxt_uint_t i; + nxt_task_t *task, *t; + nxt_sockaddr_t sa; + nxt_joint_job_t job[2]; + nxt_queue_link_t *qlk; + nxt_socket_conf_t skcf[2]; + nxt_event_engine_t engine, conf_engine, *saved_engine; + nxt_listen_event_t *lev; + nxt_work_handler_t handler; + nxt_listen_socket_t ls; + nxt_work_queue_cache_t cache; + nxt_router_temp_conf_t tmcf; + nxt_socket_conf_joint_t joint[2]; + + ret = NXT_ERROR; + lev = NULL; + task = thr->task; + + nxt_memzero(&engine, sizeof(nxt_event_engine_t)); + nxt_memzero(&conf_engine, sizeof(nxt_event_engine_t)); + nxt_memzero(&tmcf, sizeof(nxt_router_temp_conf_t)); + nxt_memzero(job, sizeof(job)); + nxt_memzero(skcf, sizeof(skcf)); + nxt_memzero(joint, sizeof(joint)); + nxt_memzero(&ls, sizeof(nxt_listen_socket_t)); + nxt_memzero(&sa, sizeof(nxt_sockaddr_t)); + + if (nxt_timers_init(&engine.timers, 4) != NXT_OK) { + return NXT_ERROR; + } + + engine.mem_pool = nxt_mp_create(1024, 128, 256, 32); + if (engine.mem_pool == NULL) { + nxt_free(engine.timers.changes); + return NXT_ERROR; + } + + engine.task = *task; + engine.event.io = &nxt_router_stale_joint_io; + engine.event.enable_accept = nxt_router_stale_joint_enable_accept; + + nxt_queue_init(&engine.joints); + nxt_queue_init(&engine.listen_connections); + nxt_queue_init(&engine.idle_connections); + + nxt_work_queue_cache_create(&cache, 0); + + engine.fast_work_queue.cache = &cache; + engine.close_work_queue.cache = &cache; + + /* The engine that gets the acknowledgements of the jobs. */ + conf_engine.task = *task; + conf_engine.event.signal = nxt_router_stale_joint_signal; + tmcf.engine = &conf_engine; + + /* + * The engine is at max_connections, so the listen event gets no spare + * conn: it stays unarmed, and its listen timer retries the spare. The + * listen socket is not used for I/O. + */ + sa.type = SOCK_STREAM; + sa.u.sockaddr.sa_family = AF_INET; + + ls.socket = -1; + ls.sockaddr = &sa; + ls.socklen = sizeof(struct sockaddr_in); + ls.count = 10; + + engine.max_connections = 0; + + for (i = 0; i < 2; i++) { + job[i].task = *task; + job[i].tmcf = &tmcf; + + skcf[i].count = 2; + skcf[i].router_conf = rtcf; + skcf[i].listen = &ls; + + joint[i].count = 1; + joint[i].socket_conf = &skcf[i]; + joint[i].engine = &engine; + } + + saved_engine = thr->engine; + thr->engine = &engine; + nxt_router_stale_joint_signals = 0; + + /* + * A create whose listen event cannot be allocated: the joint stays + * linked, the engine holds one listen socket reference, and the job is + * acknowledged. + */ + + nxt_listen_event_test_alloc_failures = 1; + + nxt_router_test_listen_socket_create(task, &job[0], &joint[0]); + + if (!nxt_router_stale_joint_linked(&engine, &joint[0]) + || !nxt_queue_is_empty(&engine.listen_connections)) + { + nxt_log_alert(thr->log, "router stale joint test failed: create: " + "the joint is not linked, or a listen event exists"); + goto done; + } + + if (ls.count != 11 || !joint[0].stale) { + nxt_log_alert(thr->log, "router stale joint test failed: create: " + "listen count %D (expected 11), stale flag %d", + ls.count, joint[0].stale); + goto done; + } + + if (nxt_router_stale_joint_posted(thr, "create", &conf_engine, &job[0]) + != NXT_OK) + { + goto done; + } + + /* + * The update of the same listener finds no listen event, so it creates + * one, and releases the stale joint of the failed create with its + * listen socket reference. + */ + + nxt_router_test_listen_socket_update(task, &job[1], &joint[1]); + + if (nxt_queue_is_empty(&engine.listen_connections)) { + nxt_log_alert(thr->log, "router stale joint test failed: update: " + "no listen event was created"); + goto done; + } + + qlk = nxt_queue_first(&engine.listen_connections); + lev = nxt_queue_link_data(qlk, nxt_listen_event_t, link); + + if (nxt_router_stale_joint_linked(&engine, &joint[0]) + || !nxt_router_stale_joint_linked(&engine, &joint[1])) + { + nxt_log_alert(thr->log, "router stale joint test failed: update: " + "the stale joint is still linked, or the new one is " + "not"); + goto done; + } + + if (ls.count != 11 || skcf[0].count != 1 || skcf[1].count != 2 + || joint[1].stale) + { + nxt_log_alert(thr->log, "router stale joint test failed: update: " + "listen count %D (expected 11), socket conf counts " + "%D %D (expected 1 2)", ls.count, skcf[0].count, + skcf[1].count); + goto done; + } + + if (nxt_router_stale_joint_posted(thr, "update", &conf_engine, &job[1]) + != NXT_OK) + { + goto done; + } + + ret = NXT_OK; + +done: + + nxt_listen_event_test_alloc_failures = 0; + + if (lev != NULL) { + nxt_timer_delete(&engine, &lev->timer); + nxt_queue_remove(&lev->link); + nxt_free(lev); + } + + /* The idle-close pass that the missing spare conn queued. */ + while (engine.close_work_queue.head != NULL) { + handler = nxt_work_queue_pop(&engine.close_work_queue, &t, &obj, + &data); + handler(t, obj, data); + } + + thr->engine = saved_engine; + + nxt_work_queue_cache_destroy(&cache); + nxt_mp_destroy(engine.mem_pool); + nxt_free(engine.timers.changes); + + return ret; +} + + +nxt_int_t +nxt_router_stale_joint_test(nxt_thread_t *thr) +{ + nxt_uint_t i; + nxt_router_t router; + nxt_router_conf_t rtcf; + nxt_event_engine_t engine; + nxt_listen_socket_t ls[3]; + nxt_socket_conf_t skcf[NXT_STALE_JOINTS]; + nxt_socket_conf_joint_t joint[NXT_STALE_JOINTS]; + + static const nxt_uint_t listen_of[] = { 0, 1, 0, 1, 1 }; + static const nxt_bool_t after_update[] = { 0, 1, 1, 1, 1 }; + static const nxt_bool_t after_delete[] = { 0, 0, 1, 0, 0 }; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "router stale joint test started"); + + nxt_memzero(&router, sizeof(nxt_router_t)); + nxt_memzero(&rtcf, sizeof(nxt_router_conf_t)); + nxt_memzero(&engine, sizeof(nxt_event_engine_t)); + nxt_memzero(ls, sizeof(ls)); + nxt_memzero(skcf, sizeof(skcf)); + nxt_memzero(joint, sizeof(joint)); + + rtcf.router = &router; + rtcf.count = 100; + + nxt_queue_init(&engine.joints); + + for (i = 0; i < nxt_nitems(ls); i++) { + ls[i].socket = -1; + ls[i].count = 10; + } + + /* + * 0: the stale joint of a failed create of listener A. + * 1: the stale joint of a failed create of listener B. + * 2: the new joint of the update of listener A (the job's own joint). + * 3, 4: more stale joints of listener B, each with its own socket conf, + * as after failed creates of B in older configurations. + */ + + for (i = 0; i < NXT_STALE_JOINTS; i++) { + skcf[i].count = 2; + skcf[i].router_conf = &rtcf; + skcf[i].listen = &ls[listen_of[i]]; + + joint[i].count = 1; + joint[i].stale = (i != 2); + joint[i].socket_conf = &skcf[i]; + joint[i].engine = &engine; + + nxt_queue_insert_tail(&engine.joints, &joint[i].link); + } + + /* + * An update of listener A keeps its own joint and releases only the + * stale joint 0 and its reference on A. + */ + + nxt_router_test_listen_socket_release_stale(thr->task, &engine, &ls[0], + &joint[2]); + + if (nxt_router_stale_joint_check(thr, "update", &engine, joint, + after_update) + != NXT_OK) + { + return NXT_ERROR; + } + + if (ls[0].count != 9 || ls[1].count != 10) { + nxt_log_alert(thr->log, "router stale joint test failed: update: " + "listen counts %D %D, expected 9 10", + ls[0].count, ls[1].count); + return NXT_ERROR; + } + + /* + * A delete of listener B releases all its joints, whatever socket + * conf they have, and one reference on B for each. + */ + + nxt_router_test_listen_socket_release_stale(thr->task, &engine, &ls[1], + NULL); + + if (nxt_router_stale_joint_check(thr, "delete", &engine, joint, + after_delete) + != NXT_OK) + { + return NXT_ERROR; + } + + if (ls[0].count != 9 || ls[1].count != 7) { + nxt_log_alert(thr->log, "router stale joint test failed: delete: " + "listen counts %D %D, expected 9 7", + ls[0].count, ls[1].count); + return NXT_ERROR; + } + + /* + * A joint without the stale flag stays, also when no joint is kept: + * it can be the joint of a closed listener that a request still holds. + */ + + nxt_router_test_listen_socket_release_stale(thr->task, &engine, &ls[0], + NULL); + + if (nxt_router_stale_joint_check(thr, "not stale", &engine, joint, + after_delete) + != NXT_OK) + { + return NXT_ERROR; + } + + /* A listen socket without stale joints: nothing changes. */ + + nxt_router_test_listen_socket_release_stale(thr->task, &engine, &ls[2], + NULL); + + if (nxt_router_stale_joint_check(thr, "no match", &engine, joint, + after_delete) + != NXT_OK) + { + return NXT_ERROR; + } + + if (ls[2].count != 10 || rtcf.count != 100) { + nxt_log_alert(thr->log, "router stale joint test failed: no match: " + "listen count %D, router conf count %D", + ls[2].count, rtcf.count); + return NXT_ERROR; + } + + /* + * The last reference closes the listen socket and frees it. This is + * the case of a listener whose listen event could not be allocated on + * any engine: before, nothing closed it, and the address stayed bound. + */ + + if (nxt_router_stale_joint_last(thr, &rtcf, &engine) != NXT_OK) { + return NXT_ERROR; + } + + if (nxt_router_stale_joint_jobs(thr, &rtcf) != NXT_OK) { + return NXT_ERROR; + } + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "router stale joint test passed"); + + return NXT_OK; +} diff --git a/src/test/nxt_router_start_fail_soak_test.c b/src/test/nxt_router_start_fail_soak_test.c new file mode 100644 index 000000000..8f2a1fd73 --- /dev/null +++ b/src/test/nxt_router_start_fail_soak_test.c @@ -0,0 +1,334 @@ +/* + * Copyright (C) F5, Inc. + */ + +/* + * Failure-injection soak for issue #214. The single-shot test proves that + * one early failure gives the pending_processes slot back; this one proves + * the accounting is exact under repetition and reaches all three "goto + * failed" sites in nxt_router_start_app_process_handler(): + * + * site 1 the prototype buffer allocation, forced by an app conf length + * no allocator can satisfy; + * site 2 nxt_port_rpc_register_handler_ex(), forced by the NXT_TESTS + * rpc allocation hook; + * site 3 nxt_port_socket_write2(), forced by the NXT_TESTS port message + * allocation hook (the site the single-shot test uses). + * + * Every iteration seeds the counter at a baseline, adds the one increment + * the initiator takes, and requires it back at the baseline afterwards -- + * a leak leaves baseline+1, a double release leaves baseline-1 (or wraps + * to UINT32_MAX at baseline 0). Both are caught without nxt_assert, so + * the check survives a non-debug build. Baselines alternate between 0 and + * a non-zero value so an underflow is caught as an ordinary miscount and + * not only as a wrap. + * + * After the loop the application must still be startable -- that is the + * damage #214 does -- and one successful start must still park exactly one + * increment on the RPC for nxt_router_app_port_error() to release. + */ + +#include +#include +#include +#include +#include +#include +#include "nxt_tests.h" + + +#define NXT_ROUTER_START_FAIL_SOAK_N 1000 + + +/* File scope: the handler hands the application to the router's + reference-counting helpers, so its storage must outlive the frame. */ +static nxt_app_t nxt_router_start_fail_soak_test_app; + + +nxt_int_t +nxt_router_start_fail_soak_test(nxt_thread_t *thr) +{ + uint32_t baseline; + nxt_mp_t *mp; + nxt_app_t *app; + nxt_int_t ret; + nxt_bool_t app_mutex; + nxt_uint_t i, site; + nxt_task_t *task; + nxt_port_t *router_port, *dport; + nxt_runtime_t *rt, *saved_rt; + nxt_app_joint_t *joint; + nxt_event_engine_t engine, *saved_engine; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start fail soak test started"); + + ret = NXT_ERROR; + router_port = NULL; + dport = NULL; + joint = NULL; + app_mutex = 0; + app = &nxt_router_start_fail_soak_test_app; + + task = thr->task; + task->thread = thr; + + if (nxt_slow_path(nxt_port_rpc_init() != NXT_OK)) { + return NXT_ERROR; + } + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + rt = nxt_mp_zalloc(mp, sizeof(nxt_runtime_t)); + joint = nxt_mp_zalloc(mp, sizeof(nxt_app_joint_t)); + if (nxt_slow_path(rt == NULL || joint == NULL)) { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + rt->mem_pool = mp; + + if (nxt_slow_path(nxt_thread_mutex_create(&rt->processes_mutex) != NXT_OK)) + { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + /* + * Site 1 allocates from task->thread->engine->mem_pool, so unlike the + * single-shot fixture this one needs an engine. Only the members the + * reached code touches are set; see src/test/nxt_router_new_port_test.c. + */ + nxt_memzero(&engine, sizeof(engine)); + nxt_work_queue_cache_create(&engine.work_queue_cache, 1024); + engine.fast_work_queue.cache = &engine.work_queue_cache; + nxt_work_queue_name(&engine.fast_work_queue, "fast"); + engine.mem_pool = mp; + + saved_rt = thr->runtime; + saved_engine = thr->engine; + thr->runtime = rt; + thr->engine = &engine; + rt->main_engine = &engine; + + /* The port the handler registers its RPC on. */ + + router_port = nxt_port_new(task, 0, nxt_pid, NXT_PROCESS_ROUTER); + if (nxt_slow_path(router_port == NULL)) { + goto done; + } + + /* + * nxt_port_rpc_register_handler_ex() asserts on a live pair[0]; the + * descriptor is never touched here, so borrow the nxt_port_fail_test() + * convention of a placeholder that is reset before the port is freed. + */ + router_port->pair[0] = 0; + router_port->pair[1] = -1; + router_port->socket.fd = -1; + + /* + * The destination port. No queue and write_ready unset, so + * nxt_port_socket_write2() takes the nxt_port_msg_chk_insert() path: + * the test hook fails its allocation for site 3, and without the hook + * the message is simply queued and the write succeeds. + */ + + dport = nxt_port_new(task, 1, nxt_pid, NXT_PROCESS_APP); + if (nxt_slow_path(dport == NULL)) { + goto done; + } + + dport->pair[0] = -1; + dport->pair[1] = -1; + dport->socket.fd = -1; + + nxt_memzero(app, sizeof(nxt_app_t)); + + if (nxt_slow_path(nxt_thread_mutex_create(&app->mutex) != NXT_OK)) { + goto done; + } + + app_mutex = 1; + + nxt_queue_init(&app->ports); + nxt_queue_init(&app->spare_ports); + nxt_queue_init(&app->idle_ports); + nxt_queue_init(&app->ack_waiting_req); + + nxt_str_set(&app->name, "start-fail-soak-test"); + + app->joint = joint; + joint->app = app; + joint->use_count = 1; + + app->max_processes = 8; + app->max_pending_processes = 1; + + for (i = 0; i < NXT_ROUTER_START_FAIL_SOAK_N; i++) { + site = i % 3; + baseline = (i % 2) ? 2 : 0; + + /* What a real initiator leaves behind: the slot and the work's use. */ + app->pending_processes = baseline + 1; + app->use_count = 2; + + /* One process alive, so the failure path has no requests to fail. */ + app->processes = 1; + app->proto_port_requests = 0; + + if (site == 0) { + /* + * Site 1: no prototype, so the handler builds the START_PROCESS + * payload first. A length no allocator can serve makes + * nxt_buf_mem_alloc() return NULL before the buffer is written + * to, so app->conf may stay empty. + */ + app->proto_port = NULL; + app->conf.length = (size_t) 1 << 46; + + } else { + app->proto_port = dport; + app->conf.length = 0; + + if (site == 1) { + nxt_port_rpc_test_alloc_failures(1); + + } else { + nxt_port_test_msg_alloc_failures(1); + } + } + + nxt_router_start_app_process_handler(task, router_port, app); + + nxt_port_rpc_test_alloc_failures(0); + nxt_port_test_msg_alloc_failures(0); + + if (app->pending_processes != baseline) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start fail soak test: iteration %ui site %ui " + "pending_processes %uD (expected %uD)", + i, site + 1, app->pending_processes, baseline); + goto done; + } + + if (app->use_count != 1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start fail soak test: iteration %ui site %ui " + "app use_count %d (expected 1)", + i, site + 1, (int) app->use_count); + goto done; + } + + if (joint->use_count != 1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start fail soak test: iteration %ui site %ui " + "joint use_count %uD (expected 1)", + i, site + 1, joint->use_count); + goto done; + } + } + + /* The gate #214 wedges: after the failures the app must still start. */ + + app->processes = 0; + app->pending_processes = 0; + app->conf.length = 0; + app->proto_port = dport; + + if (!nxt_router_app_can_start(app)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start fail soak test: app cannot start after " + "%d failures, processes %uD pending %uD", + NXT_ROUTER_START_FAIL_SOAK_N, app->processes, + app->pending_processes); + goto done; + } + + /* + * A start that succeeds parks the initiator's increment on the armed + * RPC and takes a joint reference; nothing is released here. + */ + + app->pending_processes = 1; + app->use_count = 2; + + nxt_router_start_app_process_handler(task, router_port, app); + + if (app->pending_processes != 1 || joint->use_count != 2) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start fail soak test: successful start left " + "pending %uD joint %uD (expected 1 and 2)", + app->pending_processes, joint->use_count); + goto done; + } + + /* + * Tearing the port's registrations down drives the router's own error + * handler for the armed RPC, which is how a start that never answers is + * reported; it must give the parked increment back. + */ + + nxt_port_rpc_close(task, router_port); + + if (app->pending_processes != 0 || joint->use_count != 1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start fail soak test: release left pending %uD " + "joint %uD (expected 0 and 1)", + app->pending_processes, joint->use_count); + goto done; + } + + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start fail soak test passed: %d iterations over " + "failure sites 1-3", + NXT_ROUTER_START_FAIL_SOAK_N); + + ret = NXT_OK; + +done: + + if (dport != NULL) { + /* + * The successful start above was queued rather than written, and + * nxt_port_msg_chk_insert() takes a port reference for every queued + * message. Releasing only the test's own reference leaves the port + * above zero, so it never reaches nxt_port_mp_cleanup() and the + * assertions there -- the ones a --debug build exists to run -- + * never execute for it. + */ + nxt_port_test_run_error_handler(task, dport); + + if (!nxt_queue_is_empty(&dport->messages)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start fail soak test: destination port " + "still holds queued messages after teardown"); + ret = NXT_ERROR; + } + + nxt_port_use(task, dport, -1); + } + + if (router_port != NULL) { + router_port->pair[0] = -1; + + nxt_port_use(task, router_port, -1); + } + + thr->engine = saved_engine; + thr->runtime = saved_rt; + + if (app_mutex) { + nxt_thread_mutex_destroy(&app->mutex); + } + + nxt_work_queue_cache_destroy(&engine.work_queue_cache); + nxt_thread_mutex_destroy(&rt->processes_mutex); + nxt_mp_destroy(mp); + + return ret; +} diff --git a/src/test/nxt_router_start_fail_test.c b/src/test/nxt_router_start_fail_test.c new file mode 100644 index 000000000..72c362982 --- /dev/null +++ b/src/test/nxt_router_start_fail_test.c @@ -0,0 +1,171 @@ +/* + * Copyright (C) F5, Inc. + */ + +/* + * Regression test for issue #214: nxt_router_start_app_process_handler() + * is entered owning one app->pending_processes increment (its initiator + * took it before posting the work), and its failure paths used to return + * without giving it back. The counter gates every future start + * (nxt_router_app_can_start()), so each leak permanently costs the + * application one start slot; with the default max_pending_processes of 1 + * a single failure wedges the application for the router's lifetime. + * + * The deterministic failure used here is the nxt_port_socket_write2() + * site: the NXT_TESTS allocation-failure hook makes the port layer's + * message allocation fail, the write returns NXT_ERROR, and the handler + * takes "goto failed" after cancelling the RPC registration it had just + * made. On entry the app is set up the way a real start attempt finds + * it: pending_processes already counted, one extra app use for the posted + * work item. + * + * This single-shot test reaches that third site only. All three failure + * sites, and the accounting across repeated failures, are covered by + * src/test/nxt_router_start_fail_soak_test.c. + */ + +#include +#include +#include +#include +#include +#include +#include "nxt_tests.h" + + +/* + * File scope rather than a stack object: the handler hands the application to + * the router's reference-counting helpers, so its storage must outlive the + * frame even though this test's arrangement never lets it be posted anywhere. + */ +static nxt_app_t nxt_router_start_fail_test_app; + + +nxt_int_t +nxt_router_start_fail_test(nxt_thread_t *thr) +{ + nxt_app_t *app; + nxt_int_t ret; + nxt_bool_t app_mutex; + nxt_task_t *task; + nxt_port_t *router_port, *dport; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "router start fail test started"); + + ret = NXT_ERROR; + router_port = NULL; + dport = NULL; + app_mutex = 0; + app = &nxt_router_start_fail_test_app; + + task = thr->task; + task->thread = thr; + + if (nxt_slow_path(nxt_port_rpc_init() != NXT_OK)) { + return NXT_ERROR; + } + + /* The port the handler registers its RPC on. */ + + router_port = nxt_port_new(task, 0, nxt_pid, NXT_PROCESS_ROUTER); + if (nxt_slow_path(router_port == NULL)) { + return NXT_ERROR; + } + + /* + * nxt_port_rpc_register_handler_ex() asserts on a live pair[0]; the + * descriptor is never touched here, so borrow the nxt_port_fail_test() + * convention of a placeholder that is reset before the port is freed. + */ + router_port->pair[0] = 0; + router_port->pair[1] = -1; + router_port->socket.fd = -1; + + /* + * The destination port the START_PROCESS message is written to. No + * queue and write_ready unset, so nxt_port_socket_write2() takes the + * nxt_port_msg_chk_insert() path, whose allocation the test hook + * fails. + */ + + dport = nxt_port_new(task, 1, nxt_pid, NXT_PROCESS_APP); + if (nxt_slow_path(dport == NULL)) { + goto done; + } + + dport->pair[0] = -1; + dport->pair[1] = -1; + dport->socket.fd = -1; + + nxt_memzero(app, sizeof(nxt_app_t)); + + if (nxt_slow_path(nxt_thread_mutex_create(&app->mutex) != NXT_OK)) { + goto done; + } + + app_mutex = 1; + + nxt_queue_init(&app->ports); + nxt_queue_init(&app->spare_ports); + nxt_queue_init(&app->idle_ports); + nxt_queue_init(&app->ack_waiting_req); + + nxt_str_set(&app->name, "start-fail-test"); + + /* + * A prototype port makes the handler take the plain app-start branch: + * no buffer allocation, straight to the RPC registration and the + * write. + */ + app->proto_port = dport; + + /* What a real initiator leaves behind: the slot and the work's use. */ + app->pending_processes = 1; + app->use_count = 2; + + /* One process alive, so the failure path has no requests to fail. */ + app->processes = 1; + + nxt_port_test_msg_alloc_failures(1); + + nxt_router_start_app_process_handler(task, router_port, app); + + nxt_port_test_msg_alloc_failures(0); + + if (app->pending_processes != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start fail test: pending_processes %uD leaked " + "(expected 0)", app->pending_processes); + goto done; + } + + if (app->use_count != 1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start fail test: app use_count %d (expected 1)", + (int) app->use_count); + goto done; + } + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "router start fail test passed"); + + ret = NXT_OK; + +done: + + if (dport != NULL) { + nxt_port_use(task, dport, -1); + } + + if (router_port != NULL) { + router_port->pair[0] = -1; + + nxt_port_use(task, router_port, -1); + } + + if (app_mutex) { + nxt_thread_mutex_destroy(&app->mutex); + } + + return ret; +} diff --git a/src/test/nxt_router_start_proto_gone_test.c b/src/test/nxt_router_start_proto_gone_test.c new file mode 100644 index 000000000..cf8d153bd --- /dev/null +++ b/src/test/nxt_router_start_proto_gone_test.c @@ -0,0 +1,326 @@ +/* + * Copyright (C) FreeUnit contributors. + */ + +/* + * Regression test: a worker start sent to a prototype that has already + * exited must not be logged as an alert. + * + * On shutdown main sends QUIT to every process directly. The prototype + * reports the exit of its workers to the router (REMOVE_PID) and then exits. + * The router can handle such a REMOVE_PID before its own QUIT. It then + * starts a replacement worker from nxt_router_app_port_close(), and writes + * START_PROCESS to the socket of the prototype. The prototype has closed + * that socket. sendmsg() fails with EPIPE (ECONNREFUSED on a SOCK_DGRAM + * pair), and nxt_router_start_app_process_handler() logged + * "app '...' failed to start a process" as an alert. The test suite fails + * on that alert (test_app_lifecycle_process_churn in the sanitize job). + * + * The first case sends to a real socket pair. The reader end of the pair is + * closed. The case checks that the slot is released and that the alert is + * not logged. The second case makes the send fail with an error that does + * not mean "the peer is gone". The case checks that the alert is logged. + */ + +#include +#include +#include +#include +#include +#include +#include "nxt_tests.h" + + +static void nxt_cdecl nxt_router_start_proto_gone_test_log_handler( + nxt_uint_t level, nxt_log_t *log, const char *fmt, ...); +static nxt_int_t nxt_router_start_proto_gone_test_run(nxt_thread_t *thr, + nxt_err_t inject, nxt_uint_t expected_alerts); +static void nxt_router_start_proto_gone_test_drain_wq(nxt_work_queue_t *wq); + + +/* + * The handler passes the application to the reference-counting helpers of + * the router. The storage must stay valid after the test function returns, + * so the application has file scope. + */ +static nxt_app_t nxt_router_start_proto_gone_test_app; + +static nxt_uint_t nxt_router_start_proto_gone_test_alerts; + + +nxt_int_t +nxt_router_start_proto_gone_test(nxt_thread_t *thr) +{ + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start proto gone test started"); + + /* The prototype has closed its socket: no alert. */ + + if (nxt_router_start_proto_gone_test_run(thr, 0, 0) != NXT_OK) { + return NXT_ERROR; + } + + /* Any other send failure is still an alert. */ + + if (nxt_router_start_proto_gone_test_run(thr, NXT_EINVAL, 1) != NXT_OK) { + return NXT_ERROR; + } + + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start proto gone test passed"); + + return NXT_OK; +} + + +/* + * Count the "failed to start a process" alerts. Pass every line on to the + * main log, so that the test output stays readable. + */ + +static void nxt_cdecl +nxt_router_start_proto_gone_test_log_handler(nxt_uint_t level, nxt_log_t *log, + const char *fmt, ...) +{ + u_char *p; + va_list args; + u_char msg[NXT_MAX_ERROR_STR]; + + va_start(args, fmt); + p = nxt_vsprintf(msg, msg + NXT_MAX_ERROR_STR - 1, fmt, args); + va_end(args); + + *p = '\0'; + + if (level == NXT_LOG_ALERT + && nxt_memstrn(msg, p, "failed to start a process", + nxt_length("failed to start a process")) != NULL) + { + nxt_router_start_proto_gone_test_alerts++; + } + + nxt_log_error(level, &nxt_main_log, "%s", msg); +} + + +static nxt_int_t +nxt_router_start_proto_gone_test_run(nxt_thread_t *thr, nxt_err_t inject, + nxt_uint_t expected_alerts) +{ + nxt_fd_t pair[2]; + nxt_app_t *app; + nxt_int_t ret; + nxt_log_t *saved_log; + nxt_bool_t app_mutex; + nxt_task_t *task; + nxt_port_t *router_port, *proto_port; + nxt_event_engine_t engine, *saved_engine; + + nxt_log_t log = { + NXT_LOG_INFO, + 0, + nxt_router_start_proto_gone_test_log_handler, + NULL, + NULL + }; + + ret = NXT_ERROR; + router_port = NULL; + proto_port = NULL; + app_mutex = 0; + pair[0] = -1; + pair[1] = -1; + app = &nxt_router_start_proto_gone_test_app; + + task = thr->task; + task->thread = thr; + + if (nxt_slow_path(nxt_port_rpc_init() != NXT_OK)) { + return NXT_ERROR; + } + + /* + * The failed send queues nxt_port_error_handler() on the fast work queue + * of the engine. That handler drops the reference that + * nxt_port_write_msgs() takes for it. + */ + nxt_memzero(&engine, sizeof(engine)); + nxt_work_queue_cache_create(&engine.work_queue_cache, 1024); + engine.fast_work_queue.cache = &engine.work_queue_cache; + nxt_work_queue_name(&engine.fast_work_queue, "fast"); + + saved_engine = thr->engine; + thr->engine = &engine; + + /* The handler registers its RPC on this port. */ + + router_port = nxt_port_new(task, 0, nxt_pid, NXT_PROCESS_ROUTER); + if (nxt_slow_path(router_port == NULL)) { + goto done; + } + + /* See src/test/nxt_router_start_fail_test.c. */ + router_port->pair[0] = 0; + router_port->pair[1] = -1; + router_port->socket.fd = -1; + + /* + * The prototype port. It is write-ready and its queue is empty, so + * nxt_port_socket_write2() sends inline and reports the failure to the + * handler. + */ + + proto_port = nxt_port_new(task, 1, nxt_pid, NXT_PROCESS_PROTOTYPE); + if (nxt_slow_path(proto_port == NULL)) { + goto done; + } + + proto_port->pair[0] = -1; + proto_port->pair[1] = -1; + proto_port->socket.fd = -1; + + if (nxt_slow_path(socketpair(AF_UNIX, SOCK_DGRAM, 0, pair) != 0)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start proto gone test: socketpair failed"); + goto done; + } + + if (inject == 0) { + /* The prototype has exited: its reader end is closed. */ + nxt_fd_close(pair[0]); + pair[0] = -1; + } + + proto_port->socket.fd = pair[1]; + proto_port->socket.task = task; + proto_port->socket.log = thr->log; + proto_port->socket.write_ready = 1; + proto_port->socket.write = NXT_EVENT_INACTIVE; + proto_port->max_size = 1024; + proto_port->max_share = 1024; + + nxt_memzero(app, sizeof(nxt_app_t)); + + if (nxt_slow_path(nxt_thread_mutex_create(&app->mutex) != NXT_OK)) { + goto done; + } + + app_mutex = 1; + + nxt_queue_init(&app->ports); + nxt_queue_init(&app->spare_ports); + nxt_queue_init(&app->idle_ports); + nxt_queue_init(&app->ack_waiting_req); + + nxt_str_set(&app->name, "start-proto-gone-test"); + + /* With a prototype port the handler takes the worker start branch. */ + app->proto_port = proto_port; + + /* A real initiator leaves these: the slot and the use of the work. */ + app->pending_processes = 1; + app->use_count = 2; + + /* One process is alive, so the failure path has no requests to fail. */ + app->processes = 1; + + nxt_router_start_proto_gone_test_alerts = 0; + + saved_log = task->log; + task->log = &log; + + if (inject != 0) { + nxt_socketpair_test_send_fail(inject, 1); + } + + nxt_router_start_app_process_handler(task, router_port, app); + + nxt_socketpair_test_send_fail(0, 0); + + task->log = saved_log; + + nxt_router_start_proto_gone_test_drain_wq(&engine.fast_work_queue); + + if (proto_port->socket.error == 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start proto gone test: the START_PROCESS send " + "did not fail (inject %d)", (int) inject); + goto done; + } + + if (app->pending_processes != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start proto gone test: pending_processes %uD " + "leaked (expected 0, inject %d)", + app->pending_processes, (int) inject); + goto done; + } + + if (app->use_count != 1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start proto gone test: app use_count %d " + "(expected 1, inject %d)", + (int) app->use_count, (int) inject); + goto done; + } + + if (nxt_router_start_proto_gone_test_alerts != expected_alerts) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start proto gone test: %ui \"failed to start a " + "process\" alerts (expected %ui, send error %d)", + nxt_router_start_proto_gone_test_alerts, + expected_alerts, (int) proto_port->socket.error); + goto done; + } + + ret = NXT_OK; + +done: + + nxt_router_start_proto_gone_test_drain_wq(&engine.fast_work_queue); + + if (proto_port != NULL) { + proto_port->socket.fd = -1; + + nxt_port_use(task, proto_port, -1); + } + + if (pair[0] != -1) { + nxt_fd_close(pair[0]); + } + + if (pair[1] != -1) { + nxt_fd_close(pair[1]); + } + + if (router_port != NULL) { + router_port->pair[0] = -1; + + nxt_port_use(task, router_port, -1); + } + + thr->engine = saved_engine; + + if (app_mutex) { + nxt_thread_mutex_destroy(&app->mutex); + } + + nxt_work_queue_cache_destroy(&engine.work_queue_cache); + + return ret; +} + + +static void +nxt_router_start_proto_gone_test_drain_wq(nxt_work_queue_t *wq) +{ + void *obj, *data; + nxt_task_t *t; + nxt_work_handler_t handler; + + while (wq->head != NULL) { + handler = nxt_work_queue_pop(wq, &t, &obj, &data); + handler(t, obj, data); + } +} diff --git a/src/test/nxt_router_start_timeout_test.c b/src/test/nxt_router_start_timeout_test.c new file mode 100644 index 000000000..f3d2cb84b --- /dev/null +++ b/src/test/nxt_router_start_timeout_test.c @@ -0,0 +1,1044 @@ +/* + * Copyright (C) F5, Inc. + */ + +/* + * Regression test for the silent-worker start wedge: an application process + * that is forked successfully but never calls nxt_unit_init() sends no + * PROCESS_READY, so the START_PROCESS RPC the router armed for it was never + * retired. Nothing else could retire it either -- the on-demand start + * registers that RPC with no peer (nxt_router_start_app_process_handler()), + * and REMOVE_PID only arrives if the process dies, which a `/bin/sleep` does + * not. The armed handler owned an app->pending_processes slot and, on the + * config-apply path, was the sole continuation of nxt_router_conf_apply(), so + * the configuration PUT never returned and the controller queued every later + * request behind it -- GET /status included. + * + * The fix is a deadline armed with the START_PROCESS write and cancelled by + * whichever handler answers first. What this test pins down is that the + * deadline resolves the RPC *through the existing failure path* rather than + * inventing accounting of its own: on expiry the application must get its + * pending_processes slot back and its proto_port_requests cohort cleared, + * exactly as nxt_router_app_port_error() does for a start that fails for any + * other reason, and the application must be able to start a process again + * afterwards. + * + * The other half is the race the deadline creates: a reply that lands after + * the timer has been queued must not free the timer struct out from under the + * expiry handler, and a cancelled timer must not leave a node in the engine's + * rbtree. Both are driven here against a real nxt_timers_t, because a + * memzero'd engine would let a broken cancel look correct. + * + * The arrangement is the router's own: no prototype port yet, so the handler + * builds a START_PROCESS payload and arms an RPC with ->proto set; the message + * only queues (main's port has no queue and write_ready unset), so the RPC + * stays armed and nothing ever answers it. + */ + +#include +#include +#include +#include +#include +#include +#include "nxt_tests.h" + + +/* File scope: the handler hands the application to the router's + reference-counting helpers, so its storage must outlive the frame. */ +static nxt_app_t nxt_router_start_timeout_test_app; + + +/* + * Commit the pending timer changes and move what is due onto the work queue, + * the way nxt_event_engine_start() does, without running any of it: + * nxt_timer_find() folds the change list into the rbtree and + * nxt_timer_expire() queues the handlers that are due. Split out from the + * tick below so a leg can queue work *behind* an already-queued timer handler, + * which is the ordering nxt_port_write_handler() creates for real. + */ + +static void +nxt_router_start_timeout_test_expire(nxt_event_engine_t *engine, + nxt_msec_t advance) +{ + (void) nxt_timer_find(engine); + + nxt_timer_expire(engine, engine->timers.now + advance); +} + + +/* Drain the work queue, returning the number of handlers that ran. */ + +static nxt_uint_t +nxt_router_start_timeout_test_drain(nxt_task_t *task, + nxt_event_engine_t *engine) +{ + nxt_uint_t ran; + nxt_task_t *wq_task; + nxt_work_handler_t handler; + void *obj, *data; + + ran = 0; + + /* nxt_work_queue_pop() dereferences the head unconditionally. */ + + while (engine->fast_work_queue.head != NULL) { + handler = nxt_work_queue_pop(&engine->fast_work_queue, &wq_task, &obj, + &data); + + if (handler == NULL) { + continue; + } + + ran++; + + handler(wq_task != NULL ? wq_task : task, obj, data); + } + + return ran; +} + + +static nxt_uint_t +nxt_router_start_timeout_test_tick(nxt_task_t *task, nxt_event_engine_t *engine, + nxt_msec_t advance) +{ + nxt_router_start_timeout_test_expire(engine, advance); + + return nxt_router_start_timeout_test_drain(task, engine); +} + + +/* The message a start left in a port's send queue, if it is still there. */ + +static nxt_port_send_msg_t * +nxt_router_start_timeout_test_msg(nxt_port_t *port) +{ + nxt_queue_link_t *lnk; + + lnk = nxt_queue_first(&port->messages); + + if (lnk == nxt_queue_tail(&port->messages)) { + return NULL; + } + + return nxt_queue_link_data(lnk, nxt_port_send_msg_t, link); +} + + +nxt_int_t +nxt_router_start_timeout_test(nxt_thread_t *thr) +{ + nxt_mp_t *mp; + nxt_app_t *app; + nxt_int_t ret; + nxt_uint_t ran; + uint32_t stream, joint_use; + nxt_bool_t app_mutex, timers; + nxt_task_t *task; + nxt_port_t *router_port, *main_port, *shared_port; + nxt_router_t test_router; + nxt_runtime_t *rt, *saved_rt; + nxt_atomic_t main_port_use; + nxt_app_joint_t *joint; + nxt_event_engine_t engine, *saved_engine; + nxt_port_send_msg_t *sent; + nxt_router_temp_conf_t *tmcf; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test started"); + + ret = NXT_ERROR; + router_port = NULL; + main_port = NULL; + shared_port = NULL; + joint = NULL; + app_mutex = 0; + timers = 0; + app = &nxt_router_start_timeout_test_app; + + task = thr->task; + task->thread = thr; + + saved_rt = thr->runtime; + saved_engine = thr->engine; + + if (nxt_slow_path(nxt_port_rpc_init() != NXT_OK)) { + return NXT_ERROR; + } + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + rt = nxt_mp_zalloc(mp, sizeof(nxt_runtime_t)); + joint = nxt_mp_zalloc(mp, sizeof(nxt_app_joint_t)); + if (nxt_slow_path(rt == NULL || joint == NULL)) { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + rt->mem_pool = mp; + + if (nxt_slow_path(nxt_thread_mutex_create(&rt->processes_mutex) != NXT_OK)) + { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + nxt_memzero(&engine, sizeof(engine)); + nxt_work_queue_cache_create(&engine.work_queue_cache, 1024); + engine.fast_work_queue.cache = &engine.work_queue_cache; + nxt_work_queue_name(&engine.fast_work_queue, "fast"); + engine.mem_pool = mp; + + /* + * The deadline runs on the engine's own task, as it does in the router, + * so that task must be usable: a memzero'd one has a NULL log and the + * first nxt_debug() from nxt_timer_add() dereferences it. + */ + + engine.task.thread = thr; + engine.task.log = thr->log; + + /* + * A real timer machinery, not the memzero'd one the other router tests + * get away with: nxt_timer_add() writes into timers->changes, and a + * cancel that leaves a node in this rbtree is precisely the defect the + * teardown check below looks for. + */ + + if (nxt_slow_path(nxt_timers_init(&engine.timers, 64) != NXT_OK)) { + goto done; + } + + timers = 1; + + thr->runtime = rt; + thr->engine = &engine; + rt->main_engine = &engine; + + /* The port the handler registers its RPC on. */ + + router_port = nxt_port_new(task, 0, nxt_pid, NXT_PROCESS_ROUTER); + if (nxt_slow_path(router_port == NULL)) { + goto done; + } + + router_port->pair[0] = 0; + router_port->pair[1] = -1; + router_port->socket.fd = -1; + + /* + * A prototype start addresses the main process. No queue and write_ready + * unset, so nxt_port_socket_write2() takes the nxt_port_msg_chk_insert() + * path: the write succeeds, the message is queued and the RPC stays armed + * -- a start that nobody will ever answer, which is the case under test. + */ + + main_port = nxt_port_new(task, 1, nxt_pid, NXT_PROCESS_MAIN); + if (nxt_slow_path(main_port == NULL)) { + goto done; + } + + main_port->pair[0] = -1; + main_port->pair[1] = -1; + main_port->socket.fd = -1; + + rt->port_by_type[NXT_PROCESS_MAIN] = main_port; + + shared_port = nxt_port_new(task, 2, nxt_pid, NXT_PROCESS_APP); + if (nxt_slow_path(shared_port == NULL)) { + goto done; + } + + shared_port->pair[0] = -1; + shared_port->pair[1] = -1; + shared_port->socket.fd = -1; + shared_port->queue_fd = -1; + + nxt_memzero(app, sizeof(nxt_app_t)); + + if (nxt_slow_path(nxt_thread_mutex_create(&app->mutex) != NXT_OK)) { + goto done; + } + + app_mutex = 1; + + nxt_queue_init(&app->ports); + nxt_queue_init(&app->spare_ports); + nxt_queue_init(&app->idle_ports); + nxt_queue_init(&app->ack_waiting_req); + + nxt_str_set(&app->name, "start-timeout-test"); + + app->joint = joint; + joint->app = app; + joint->use_count = 1; + + app->shared_port = shared_port; + app->max_processes = 8; + app->max_pending_processes = 4; + app->start_timeout = 1000; + + app->proto_port = NULL; + + /* One process alive, so the failure path has no requests to fail. */ + app->processes = 1; + + /* What a real initiator leaves behind: the slot and the work's use. */ + app->pending_processes = 1; + app->use_count = 2; + + /* Baseline for the reference-balance check after the deadline fires. */ + main_port_use = main_port->use_count; + + nxt_router_start_app_process_handler(task, router_port, app); + + if (app->proto_port_requests != 1 || joint->use_count != 2) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: start left " + "proto_port_requests %uD joint %uD (expected 1 and 2)", + app->proto_port_requests, joint->use_count); + goto done; + } + + /* + * Well before the deadline nothing may fire: a deadline that expires + * early would fail starts that are merely slow. + */ + + ran = nxt_router_start_timeout_test_tick(task, &engine, 100); + + if (ran != 0 || app->pending_processes != 1 + || app->proto_port_requests != 1) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: deadline fired early, " + "%ui handlers, pending %uD proto_port_requests %uD " + "(expected 0, 1 and 1)", + ran, app->pending_processes, app->proto_port_requests); + goto done; + } + + /* + * Past the deadline the RPC must resolve through the router's own error + * handler: the cohort cleared and every slot it owned given back. Before + * the fix nothing was armed at all, so this tick ran no handler and left + * both counters exactly as they were. + */ + + ran = nxt_router_start_timeout_test_tick(task, &engine, 2000); + + if (ran == 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: nothing bounded the start, " + "the RPC is armed forever"); + goto done; + } + + if (app->proto_port_requests != 0 || app->pending_processes != 0 + || joint->use_count != 1) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: expired start left " + "proto_port_requests %uD pending %uD joint %uD " + "(expected 0, 0 and 1)", + app->proto_port_requests, app->pending_processes, + joint->use_count); + goto done; + } + + /* + * And the start it gave up on is out of the send queue, with the reference + * nxt_port_msg_chk_insert() took for it discharged. This is the whole + * point of resolving through nxt_port_socket_cancel(): failing the RPC + * releases the payload and the application's shared-port descriptors, and + * a message left behind here would be dereferenced -- and its descriptors + * sent -- when the destination finally drained. + */ + + if (!nxt_queue_is_empty(&main_port->messages)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: an expired start left its " + "START_PROCESS in the destination's send queue"); + goto done; + } + + if (main_port->use_count != main_port_use) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: destination port use count " + "%uD after an expired start, expected %uD", + (uint32_t) main_port->use_count, + (uint32_t) main_port_use); + goto done; + } + + /* + * And the application is not wedged by its own deadline: with the cohort + * clear a later call must send a fresh prototype request -- arming an RPC + * and taking a joint reference -- rather than parking in the wait branch. + */ + + app->pending_processes = 1; + app->use_count = 2; + + nxt_router_start_app_process_handler(task, router_port, app); + + if (app->proto_port_requests != 1 || joint->use_count != 2) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: app wedged after an expired " + "start, proto_port_requests %uD joint %uD " + "(expected 1 and 2)", + app->proto_port_requests, joint->use_count); + goto done; + } + + /* + * The other order: an answer arrives first. Closing the port's + * registrations drives nxt_router_app_port_error(), which must cancel the + * armed deadline. A cancel that only nxt_timer_disable()d it would leave + * a node in the rbtree pointing at freed memory; the tick below walks + * that tree, and the teardown check requires it empty. + */ + + nxt_port_rpc_close(task, router_port); + + if (app->proto_port_requests != 0 || app->pending_processes != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: answered start left " + "proto_port_requests %uD pending %uD (expected 0 and 0)", + app->proto_port_requests, app->pending_processes); + goto done; + } + + /* + * Two ticks: the first lets a deferred release run, the second must find + * nothing left. A cancelled deadline that still fired would report the + * application as failing a start it never made. + */ + + (void) nxt_router_start_timeout_test_tick(task, &engine, 2000); + + ran = nxt_router_start_timeout_test_tick(task, &engine, 2000); + + if (ran != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: %ui handlers still pending " + "after a cancelled deadline (expected 0)", ran); + goto done; + } + + if (!nxt_rbtree_is_empty(&engine.timers.tree)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: a cancelled deadline left a " + "node in the engine's timer tree"); + goto done; + } + + /* + * The write/timer ordering inside one poll cycle. + * + * nxt_port_write_handler() removes a fully sent message from + * port->messages and only then queues its buffer completion, so the + * completion can land behind a timer handler that the same poll cycle + * queued ahead of it. The deadline then finds nothing in the send queue + * while the payload is still referenced by work that has not run: + * "not found" is not "safe to fail", and failing there would release the + * payload under the pending completion. + * + * Built here by expiring the timer first -- which only queues its handler + * -- and taking the message out afterwards, so its completion is queued + * behind that handler exactly as a real write would leave it. + */ + + app->pending_processes = 1; + app->use_count = 2; + + nxt_router_start_app_process_handler(task, router_port, app); + + sent = nxt_router_start_timeout_test_msg(main_port); + + if (sent == NULL) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: no message queued for the " + "same-poll ordering leg"); + goto done; + } + + stream = sent->port_msg.stream; + + nxt_router_start_timeout_test_expire(&engine, 2000); + + if (nxt_port_socket_cancel(task, main_port, NXT_PORT_MSG_START_PROCESS, + stream, router_port->id, NULL) + != NXT_PORT_MSG_CANCELLED) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: could not take the queued " + "start back for the same-poll ordering leg"); + goto done; + } + + if (app->proto_port_requests != 1 || app->pending_processes != 1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: taking the message back " + "failed the start by itself, proto_port_requests %uD " + "pending %uD (expected 1 and 1)", + app->proto_port_requests, app->pending_processes); + goto done; + } + + ran = nxt_router_start_timeout_test_drain(task, &engine); + + if (ran == 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: the deadline did not fire " + "for a sent-but-uncompleted start"); + goto done; + } + + if (app->proto_port_requests != 0 || app->pending_processes != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: a sent-but-uncompleted " + "start was not failed once its completion ran, " + "proto_port_requests %uD pending %uD (expected 0 and 0)", + app->proto_port_requests, app->pending_processes); + goto done; + } + + /* + * A start whose message is only partially transmitted. + * + * nxt_port_write_handler() sets port_msg.nf once a fragment has gone out, + * and clears msg->fd[] in the same breath: the peer is mid-stream and the + * descriptors are already with it, so nxt_port_socket_cancel() must refuse + * to take the message back. The deadline must then leave the RPC armed + * -- failing it would release the payload the remaining fragments point at + * -- and fail it from the completion of the last fragment instead. This + * is the documented limitation of a protocol without cancellation. + */ + + app->pending_processes = 1; + app->use_count = 2; + + nxt_router_start_app_process_handler(task, router_port, app); + + sent = nxt_router_start_timeout_test_msg(main_port); + + if (sent == NULL) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: no message queued for the " + "partial-send leg"); + goto done; + } + + sent->port_msg.nf = 1; + + ran = nxt_router_start_timeout_test_tick(task, &engine, 2000); + + if (ran == 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: the deadline did not fire " + "for a partially sent start"); + goto done; + } + + if (nxt_queue_is_empty(&main_port->messages)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: a partially sent message was " + "taken back out of the send queue"); + goto done; + } + + if (app->proto_port_requests != 1 || app->pending_processes != 1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: a partially sent start was " + "failed while its payload was still queued, " + "proto_port_requests %uD pending %uD (expected 1 and 1)", + app->proto_port_requests, app->pending_processes); + goto done; + } + + /* + * The remainder completes -- here by the destination dying, which is what + * nxt_port_error_handler() does to every message it still holds. The + * failure must arrive with it and not before. + */ + + nxt_port_test_run_error_handler(task, main_port); + + (void) nxt_router_start_timeout_test_drain(task, &engine); + + if (app->proto_port_requests != 0 || app->pending_processes != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: a partially sent start was " + "not failed by its final completion, " + "proto_port_requests %uD pending %uD (expected 0 and 0)", + app->proto_port_requests, app->pending_processes); + goto done; + } + + /* + * The other shape of START_PROCESS: a worker start, sent to a prototype + * that is already up. It carries no payload and no descriptors, so the + * deadline has nothing to wait on and fails the RPC itself -- but the + * message can still be sitting in the destination's send queue, and + * delivered after the stream is retired it starts a worker whose + * PROCESS_READY answers nobody, leaving a process and a port the router is + * not tracking. Recall it while it is still there. + * + * main_port stands in for the prototype: what matters is a destination + * that is not write-ready and has no shared-memory queue, so the message + * queues rather than going straight out. + */ + + app->proto_port = main_port; + + app->pending_processes = 1; + app->use_count = 2; + + /* Baseline for the reaper's inherited reference, checked below. */ + + joint_use = joint->use_count; + + nxt_router_start_app_process_handler(task, router_port, app); + + sent = nxt_router_start_timeout_test_msg(main_port); + + if (sent == NULL || sent->buf != NULL) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: the worker start queued no " + "header-only message"); + goto done; + } + + /* Read before the recall below frees the message. */ + + stream = sent->port_msg.stream; + + ran = nxt_router_start_timeout_test_tick(task, &engine, 2000); + + if (ran == 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: nothing bounded a worker " + "start"); + goto done; + } + + if (!nxt_queue_is_empty(&main_port->messages)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: an expired worker start left " + "its START_PROCESS queued, to start a worker for a " + "stream that is already retired"); + goto done; + } + + /* + * A recalled start forked nothing, so it is accounted for as any other + * failed attempt: the slot goes straight back. Moving it to + * unaccounted_processes instead would strand it -- no PROCESS_READY and + * no REMOVE_PID are coming for a process that was never created -- and + * that would make "processes": {"max"} a ratchet rather than a bound. + */ + + if (app->pending_processes != 0 || app->unaccounted_processes != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: a recalled worker start left " + "pending %uD unaccounted %uD (expected 0 and 0)", + app->pending_processes, app->unaccounted_processes); + goto done; + } + + if (joint->use_count != joint_use) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: a recalled worker start left " + "joint %uD (expected %uD)", joint->use_count, joint_use); + goto done; + } + + /* + * The start that does leave a process behind: one whose START_PROCESS + * reached the prototype, which forked a worker that never announced + * itself. Taking the message out of the send queue here is what the + * prototype's read does in the field, so the deadline below finds nothing + * to recall and knows the fork happened. + * + * The router has no pid for that worker -- a pid is what PROCESS_READY + * carries -- so it can neither kill it nor count it in ->processes. The + * slot moves to unaccounted_processes, where nxt_router_app_can_start() + * still sees it, and "processes": {"max"} keeps bounding OS children. + */ + + app->pending_processes = 1; + app->use_count = 2; + + joint_use = joint->use_count; + + nxt_router_start_app_process_handler(task, router_port, app); + + sent = nxt_router_start_timeout_test_msg(main_port); + + if (sent == NULL || sent->buf != NULL) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: the delivered worker start " + "queued no header-only message"); + goto done; + } + + stream = sent->port_msg.stream; + + if (nxt_port_socket_cancel(task, main_port, NXT_PORT_MSG_START_PROCESS, + stream, router_port->id, NULL) + != NXT_PORT_MSG_CANCELLED) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: could not stand in for the " + "prototype's read of the worker start"); + goto done; + } + + ran = nxt_router_start_timeout_test_tick(task, &engine, 2000); + + if (ran == 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: nothing bounded a delivered " + "worker start"); + goto done; + } + + if (app->pending_processes != 0 || app->unaccounted_processes != 1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: an expired worker start left " + "pending %uD unaccounted %uD (expected 0 and 1)", + app->pending_processes, app->unaccounted_processes); + goto done; + } + + /* + * And it is a bound rather than a leak because the expiry put a handler + * back on the stream. This is the REMOVE_PID the router gets when that + * worker finally dies: the prototype which forked it reaps it and + * notifies with the start's own stream still attached, and + * nxt_router_remove_pid_handler() retypes that into an RPC error. The + * slot comes back, and so does the joint reference the reaper inherited. + */ + + nxt_port_rpc_error(task, router_port, stream); + + if (app->unaccounted_processes != 0 || joint->use_count != joint_use) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: the death of the process an " + "expired start left behind gave back unaccounted %uD " + "joint %uD (expected 0 and %uD)", + app->unaccounted_processes, joint->use_count, joint_use); + goto done; + } + + app->proto_port = NULL; + + /* + * The other shape that leaves a process behind: a prototype start whose + * START_PROCESS reached main, which forked the prototype for it. A + * prototype that never announces itself forks no worker of its own, so + * the attempt leaves exactly one process however many requests parked on + * it -- and one slot has to account for that process, exactly as a + * worker's does. + * + * Two callers park here: the initiator, and one that joined the wait. + * The cohort stands for requests, so the joiner's slot goes straight + * back and the initiator's is the one that stays. Treating the whole + * cohort as starts that never happened, which is what the router did + * before, returned both: every later request then forked another + * prototype that would never answer either, and "processes": {"max"} + * bounded nothing at all. + */ + + app->pending_processes = 2; + app->unaccounted_processes = 0; + app->use_count = 3; + + joint_use = joint->use_count; + + nxt_router_start_app_process_handler(task, router_port, app); + nxt_router_start_app_process_handler(task, router_port, app); + + if (app->proto_port_requests != 2) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: the second caller did not " + "join the prototype wait, proto_port_requests %uD " + "(expected 2)", app->proto_port_requests); + goto done; + } + + sent = nxt_router_start_timeout_test_msg(main_port); + + if (sent == NULL || sent->buf == NULL) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: the prototype start queued " + "no payload message"); + goto done; + } + + stream = sent->port_msg.stream; + + /* + * Standing in for main's read. Consuming the payload is the half that + * matters: nxt_port_write_handler() advances the buffer as it sends, and + * that is how the completion below tells a message the destination has + * seen whole from one recalled or dropped with bytes still in it. + */ + + sent->buf->mem.pos = sent->buf->mem.free; + + if (nxt_port_socket_cancel(task, main_port, NXT_PORT_MSG_START_PROCESS, + stream, router_port->id, NULL) + != NXT_PORT_MSG_CANCELLED) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: could not stand in for " + "main's read of the prototype start"); + goto done; + } + + ran = nxt_router_start_timeout_test_tick(task, &engine, 2000); + + if (ran == 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: nothing bounded a delivered " + "prototype start"); + goto done; + } + + if (app->pending_processes != 0 || app->unaccounted_processes != 1 + || app->proto_port_requests != 0) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: an expired prototype start " + "left pending %uD unaccounted %uD proto_port_requests " + "%uD (expected 0, 1 and 0)", + app->pending_processes, app->unaccounted_processes, + app->proto_port_requests); + goto done; + } + + /* + * And the prototype's death gives the slot back, the same way a worker's + * does: main leaves the start's stream on a process that never reached + * READY, so the REMOVE_PID for it is retyped onto the reaper. + */ + + nxt_port_rpc_error(task, router_port, stream); + + if (app->unaccounted_processes != 0 || joint->use_count != joint_use) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: the death of the prototype " + "an expired start left behind gave back unaccounted %uD " + "joint %uD (expected 0 and %uD)", + app->unaccounted_processes, joint->use_count, joint_use); + goto done; + } + + /* + * A start whose write the port layer refuses. + * + * With the port write-ready and its queue empty, nxt_port_msg_chk_insert() + * declines to queue and nxt_port_socket_write2() sends inline. A send that + * fails there -- here because the descriptor is -1, in the field because + * the peer is gone -- leaves a message that was never in port->messages and + * was never consumed, so write2() answers NXT_ERROR. + * + * This start therefore needs no deadline: the caller takes its own + * "failed:" path, frees the payload it still owns, and gives the + * pending_processes slot back inside the call. + * + * The assertion that matters is the negative one below. The port layer + * must not have completed that payload on its way out, because the caller + * frees it with nxt_mp_free() -- a completion queued here would run + * against freed memory. + */ + + app->pending_processes = 1; + app->use_count = 2; + + /* + * The inline path really does call into the socket layer, which logs + * through the event's own task and sizes its iovec against the port's send + * limit. A port from nxt_port_new() has neither -- nxt_port_write_enable() + * is what fills them in for a real one -- and the descriptor stays -1, so + * the sendmsg() fails and the drop path is taken. + */ + + main_port->socket.task = task; + main_port->socket.log = thr->log; + main_port->max_size = 1024; + main_port->socket.write_ready = 1; + + nxt_router_start_app_process_handler(task, router_port, app); + + main_port->socket.write_ready = 0; + + if (!nxt_queue_is_empty(&main_port->messages)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: the dropped-write leg queued " + "its message instead of sending it inline"); + goto done; + } + + /* + * One work item: the error handler the dead socket raised. Not two -- + * a buffer completion queued here would run against memory the caller + * has already handed to nxt_mp_free(), which is the double free the + * ownership contract exists to prevent, and it is why this leg counts + * rather than merely draining. + */ + + ran = nxt_router_start_timeout_test_drain(task, &engine); + + if (ran != 1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: a refused write queued %d " + "work items, expected 1 -- the port layer completed a " + "payload the caller still owns and has freed", + (int) ran); + goto done; + } + + if (app->proto_port_requests != 0 || app->pending_processes != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: a start whose write was " + "refused stayed armed, proto_port_requests %uD pending " + "%uD (expected 0 and 0)", + app->proto_port_requests, app->pending_processes); + goto done; + } + + /* + * And no deadline was needed to get there: the refusal failed the start + * inside the call. A timer firing here would mean it had been left armed + * for one to find, which is the state this leg exists to rule out. + */ + + ran = nxt_router_start_timeout_test_tick(task, &engine, 2000); + + if (ran != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: a deadline fired for a start " + "the port layer had already refused (%d handlers)", + (int) ran); + goto done; + } + + /* + * And the same expiry on the config-apply path, which is where the payload + * has a lifetime of its own: nxt_router_app_rpc_create() allocates it from + * tmcf->mem_pool, and the failure it drives ends in + * nxt_router_conf_error(), which releases that pool. A deadline that + * failed the RPC with the message still queued would leave the send queue + * pointing into freed memory -- which is what this leg would report under + * ASan, from the teardown that drains the port below. + */ + + nxt_memzero(&test_router, sizeof(nxt_router_t)); + nxt_queue_init(&test_router.sockets); + nxt_queue_init(&test_router.apps); + + rt->port_by_type[NXT_PROCESS_ROUTER] = router_port; + + tmcf = nxt_router_test_temp_conf(task); + if (nxt_slow_path(tmcf == NULL)) { + goto done; + } + + tmcf->router_conf->router = &test_router; + + /* nxt_router_conf_send() answers on it and drops a reference for it. */ + tmcf->port = main_port; + nxt_port_inc_use(main_port); + + nxt_router_test_app_rpc_create(task, tmcf, app); + + if (nxt_router_start_timeout_test_msg(main_port) == NULL) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: the config-apply start " + "queued nothing"); + goto done; + } + + ran = nxt_router_start_timeout_test_tick(task, &engine, 2000); + + if (ran == 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: nothing bounded the " + "config-apply start"); + goto done; + } + + /* + * tmcf and its pool are gone from here on. What must not be gone with + * them is a queued reference to the payload they held: only the RPC_ERROR + * nxt_router_conf_send() wrote may remain. + */ + + sent = nxt_router_start_timeout_test_msg(main_port); + + if (sent != NULL + && sent->port_msg.type == (NXT_PORT_MSG_START_PROCESS + & NXT_PORT_MSG_MASK)) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: the config-apply start left " + "its START_PROCESS queued against a released pool"); + goto done; + } + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "router start timeout test passed"); + + ret = NXT_OK; + +done: + + if (shared_port != NULL) { + nxt_port_use(task, shared_port, -1); + } + + if (main_port != NULL) { + rt->port_by_type[NXT_PROCESS_MAIN] = NULL; + + /* + * Both prototype requests were queued rather than written, and + * nxt_port_msg_chk_insert() takes a port reference for each. Left + * alone the port never reaches nxt_port_mp_cleanup(), so its pool and + * messages leak and the assertions there never run. + */ + nxt_port_test_run_error_handler(task, main_port); + + nxt_port_use(task, main_port, -1); + } + + if (router_port != NULL) { + router_port->pair[0] = -1; + + nxt_port_use(task, router_port, -1); + } + + thr->engine = saved_engine; + thr->runtime = saved_rt; + + if (app_mutex) { + nxt_thread_mutex_destroy(&app->mutex); + } + + if (timers) { + nxt_free(engine.timers.changes); + } + + nxt_work_queue_cache_destroy(&engine.work_queue_cache); + nxt_thread_mutex_destroy(&rt->processes_mutex); + nxt_mp_destroy(mp); + + return ret; +} diff --git a/src/test/nxt_router_websocket_test.c b/src/test/nxt_router_websocket_test.c new file mode 100644 index 000000000..4c350158a --- /dev/null +++ b/src/test/nxt_router_websocket_test.c @@ -0,0 +1,250 @@ +/* + * Copyright (C) F5, Inc. + */ + +/* + * Regression test for the websocket accounting in + * nxt_router_app_port_release() (src/nxt_router.c). + * + * A websocket upgrade counts the session on the worker's main port, and + * nxt_router_app_port_idle() keeps a port with a session out of the idle + * queues. Nothing uncounted it, so a worker that had served one upgrade + * never rejoined the idle economy: the reaper never saw it and its slot + * counted against "processes": {"max"} for the life of the process. The + * upgrade now leaves NXT_APR_WEBSOCKET_CLOSE behind for the unlink, and + * that action uncounts the session. + * + * The fixture is one application with one worker port that is its own + * ->main_app_port. ->spare_processes is 1 and ->idle_processes starts at + * 0, so the idle transition inserts into spare_ports and never arms the + * reaper timer, which keeps the release confined to the queues and the + * counters this test reads. ->pair[1] is not -1, so the transition is + * reached at all. + * + * The table walks the life of a session: the release that follows the + * upgrade, a second session on the same worker, and the two closes. A + * close of a plain request is in it as well, to show that the decrement + * belongs to the new action and not to every close. + * + * The last session closes on a worker the router has also marked + * ->detached, because both are conditions of the one guard in + * nxt_router_app_port_idle(): uncounting the session must not hand back a + * port the detached state still holds, and clearing that state must. The + * step sets the flag directly, since app->detached_processes is settled by + * the edges that own it and is not read here. + */ + +#include +#include +#include +#include +#include +#include "nxt_tests.h" + + +typedef struct { + const char *name; + nxt_bool_t upgrade; + uint8_t detached; + nxt_apr_action_t action; + uint32_t active_websockets; + nxt_bool_t idle; + nxt_atomic_t use_count; +} nxt_router_websocket_test_step_t; + + +static const nxt_router_websocket_test_step_t + nxt_router_websocket_test_steps[] = +{ + { "the first upgrade", 1, 0, NXT_APR_UPGRADE, 1, 0, 5 }, + { "a second session on the same worker", 1, 0, NXT_APR_UPGRADE, 2, 0, 5 }, + { "the first session closes", 0, 0, NXT_APR_WEBSOCKET_CLOSE, 1, 0, 4 }, + { "a plain request closes", 0, 0, NXT_APR_CLOSE, 1, 0, 3 }, + { "the last session closes, detached", 0, 1, NXT_APR_WEBSOCKET_CLOSE, + 0, 0, 2 }, + { "the detached mark clears", 0, 0, NXT_APR_CLOSE, 0, 1, 1 }, +}; + + +static nxt_int_t +nxt_router_websocket_test_step(nxt_thread_t *thr, nxt_task_t *task, + nxt_port_t *port, nxt_app_t *app, + const nxt_router_websocket_test_step_t *step) +{ + nxt_bool_t idle; + + if (step->upgrade) { + /* + * What the two handlers ahead of the release do: the acknowledgement + * counts the request on the port, see + * nxt_router_req_headers_ack_handler(), and the response counts the + * session, see nxt_router_response_ready_handler(). NXT_APR_UPGRADE + * gives the request back and leaves the session behind. + */ + nxt_thread_mutex_lock(&app->mutex); + + port->active_requests++; + port->active_websockets++; + + nxt_thread_mutex_unlock(&app->mutex); + } + + nxt_thread_mutex_lock(&app->mutex); + + port->detached = step->detached; + + nxt_thread_mutex_unlock(&app->mutex); + + nxt_router_test_app_port_release(task, app, port, step->action); + + idle = (port->idle_link.next != NULL); + + if (port->active_requests != 0 + || port->active_websockets != step->active_websockets + || idle != step->idle + || app->idle_processes != (step->idle ? 1U : 0U) + || port->use_count != step->use_count) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router websocket test: %s: websockets %uD, idle %d, " + "idle processes %uD, use count %d; expected " + "%uD, %d, %uD, %d", + step->name, port->active_websockets, (int) idle, + app->idle_processes, (int) port->use_count, + step->active_websockets, (int) step->idle, + step->idle ? 1U : 0U, (int) step->use_count); + return NXT_ERROR; + } + + return NXT_OK; +} + + +nxt_int_t +nxt_router_websocket_test(nxt_thread_t *thr) +{ + nxt_mp_t *mp; + nxt_app_t *app; + nxt_uint_t i; + nxt_int_t ret; + nxt_task_t *task; + nxt_port_t *port; + nxt_bool_t app_mutex; + nxt_event_engine_t engine, *saved_engine; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "router websocket test started"); + + ret = NXT_ERROR; + port = NULL; + app_mutex = 0; + + task = thr->task; + task->thread = thr; + + saved_engine = thr->engine; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + app = nxt_mp_zalloc(mp, sizeof(nxt_app_t)); + if (nxt_slow_path(app == NULL)) { + goto done; + } + + nxt_memzero(&engine, sizeof(nxt_event_engine_t)); + + thr->engine = &engine; + + if (nxt_slow_path(nxt_thread_mutex_create(&app->mutex) != NXT_OK)) { + goto done; + } + + app_mutex = 1; + + nxt_queue_init(&app->ports); + nxt_queue_init(&app->spare_ports); + nxt_queue_init(&app->idle_ports); + nxt_queue_init(&app->ack_waiting_req); + + nxt_str_set(&app->name, "websocket-test"); + + /* Never reached by the releases below, so nothing frees the application. */ + app->use_count = 8; + + /* One spare, so the idle transition never arms the reaper timer. */ + app->spare_processes = 1; + + port = nxt_port_new(task, 0, nxt_pid, NXT_PROCESS_APP); + if (nxt_slow_path(port == NULL)) { + goto done; + } + + port->pair[0] = -1; + port->pair[1] = -1; + port->socket.fd = -1; + port->app = app; + port->main_app_port = port; + + /* + * The idle transition and the app->ports insertion both read ->pair[1], + * and nxt_port_close() must not act on the value, so set it after + * nxt_port_new() and clear it again before the port is released. + */ + port->pair[1] = 0; + + /* + * One reference for the fixture and one for each of the four releases + * below that drops one, so the port outlives the table. The first + * release takes one back when it puts the port into app->ports, which + * is why the table starts at 5. + */ + port->use_count = 4; + + for (i = 0; i < nxt_nitems(nxt_router_websocket_test_steps); i++) { + if (nxt_slow_path(nxt_router_websocket_test_step(thr, task, port, app, + &nxt_router_websocket_test_steps[i]) != NXT_OK)) + { + goto done; + } + } + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "router websocket test passed"); + + ret = NXT_OK; + +done: + + if (port != NULL) { + port->pair[1] = -1; + + if (port->app_link.next != NULL) { + nxt_queue_remove(&port->app_link); + port->app_link.next = NULL; + } + + if (port->idle_link.next != NULL) { + nxt_queue_remove(&port->idle_link); + port->idle_link.next = NULL; + } + + port->app = NULL; + port->main_app_port = NULL; + + port->use_count = 1; + + nxt_port_use(task, port, -1); + } + + thr->engine = saved_engine; + + if (app_mutex) { + nxt_thread_mutex_destroy(&app->mutex); + } + + nxt_mp_destroy(mp); + + return ret; +} diff --git a/src/test/nxt_runtime_idle_close_test.c b/src/test/nxt_runtime_idle_close_test.c new file mode 100644 index 000000000..59f86d663 --- /dev/null +++ b/src/test/nxt_runtime_idle_close_test.c @@ -0,0 +1,463 @@ +/* + * Copyright (C) FreeUnit contributors. + */ + +/* + * Regression test for M-11: nxt_runtime_close_idle_connections() + * (src/nxt_runtime.c) must close every idle conn through the conn's own + * protocol -- c->read_state->close_handler, the contract + * nxt_conn_accept_close_idle_handler() already uses under fd pressure -- + * and must leave a conn that has no protocol state yet alone. + * + * Before this fix, the idle-close pass called nxt_conn_close() directly: + * + * - a keep-alive h1p conn still carries nxt_h1p_request_send_state, + * whose ready_handler (nxt_h1p_conn_sent) is a no-op with c->write == + * NULL -- the conn was closed but never freed, a leak (CONN-INV-2); + * + * - a conn that never received a request has write_state == NULL, and + * nxt_conn_close_handler() dereferenced it, i.e. crashed; + * + * - a generic release (just nxt_conn_free()) is not enough either: the + * protocol free handlers also return c->remote to the engine's sockaddr + * cache (nxt_conn_accept_alloc() took it from there) and, for a router + * conn, drop the listener reference taken in nxt_conn_accept(), without + * which a draining listener never reaches its final release. + * + * The real protocol states (nxt_h1p_keepalive_state, + * nxt_controller_conn_read_state) are private to their modules and need a + * live listener configuration, so the protocol-owned conn below carries a + * stand-in read_state whose close_handler does what nxt_h1p_conn_close() + * ends up doing -- install a close state and nxt_conn_close() -- and whose + * close state's ready_handler does exactly what nxt_h1p_conn_free() does: + * nxt_sockaddr_cache_free(), nxt_conn_free() and the real + * nxt_router_listen_event_release(). What the test asserts is the + * dispatch contract of the idle-close pass: the protocol close handler is + * what runs, to completion, and the not-yet-initialized conn is untouched. + */ + +#include +#include +#include +#include +#include +#include +#include "nxt_tests.h" + +#include + + +static nxt_uint_t nxt_runtime_idle_close_test_stale_calls; +static nxt_uint_t nxt_runtime_idle_close_test_close_calls; +static nxt_uint_t nxt_runtime_idle_close_test_free_calls; +static nxt_conn_t *nxt_runtime_idle_close_test_closed_conn; + + +static void +nxt_runtime_idle_close_test_stale_handler(nxt_task_t *task, void *obj, + void *data) +{ + /* + * Stands in for nxt_h1p_conn_sent(): the leftover request-send + * ready_handler of a keep-alive conn, which does nothing to the conn + * itself. If this runs, the idle-close pass bypassed the protocol and + * closed the conn with its stale write_state. + */ + nxt_runtime_idle_close_test_stale_calls++; +} + + +static const nxt_conn_state_t nxt_runtime_idle_close_test_stale_state + nxt_aligned(64) = +{ + .ready_handler = nxt_runtime_idle_close_test_stale_handler, +}; + + +static void +nxt_runtime_idle_close_test_proto_free(nxt_task_t *task, void *obj, + void *data) +{ + nxt_conn_t *c; + nxt_listen_event_t *lev; + nxt_event_engine_t *engine; + + c = obj; + + /* The body of nxt_h1p_conn_free(). */ + + nxt_runtime_idle_close_test_free_calls++; + + engine = task->thread->engine; + + nxt_sockaddr_cache_free(engine, c); + + lev = c->listen; + + nxt_conn_free(task, c); + + nxt_router_listen_event_release(&engine->task, lev, NULL); +} + + +static const nxt_conn_state_t nxt_runtime_idle_close_test_proto_close_state + nxt_aligned(64) = +{ + .ready_handler = nxt_runtime_idle_close_test_proto_free, +}; + + +static void +nxt_runtime_idle_close_test_proto_close(nxt_task_t *task, void *obj, + void *data) +{ + nxt_conn_t *c; + + c = obj; + + /* What nxt_h1p_conn_close() ends up doing (nxt_h1p_conn_closing()). */ + + nxt_runtime_idle_close_test_close_calls++; + nxt_runtime_idle_close_test_closed_conn = c; + + c->write_state = &nxt_runtime_idle_close_test_proto_close_state; + + nxt_conn_close(task->thread->engine, c); +} + + +static const nxt_conn_state_t nxt_runtime_idle_close_test_proto_read_state + nxt_aligned(64) = +{ + .close_handler = nxt_runtime_idle_close_test_proto_close, +}; + + +/* + * A minimal event interface: only .close is exercised by the close path + * this test drives (c->socket.error is set to 1 below, so nxt_conn_close() + * takes the close_work_queue branch directly and never touches shutdown, + * enable/disable or poll). Returning 0 tells nxt_conn_close_handler() that + * nothing is pending, so it closes the fd and dispatches write_state's + * ready_handler on the same pass -- no timer round trip to wait out. + */ +static nxt_bool_t +nxt_runtime_idle_close_test_event_close(nxt_event_engine_t *engine, + nxt_fd_event_t *ev) +{ + return 0; +} + + +static void +nxt_runtime_idle_close_test_drain(nxt_work_queue_t *wq) +{ + void *obj, *data; + nxt_task_t *task; + nxt_work_handler_t handler; + + while (wq->head != NULL) { + handler = nxt_work_queue_pop(wq, &task, &obj, &data); + handler(task, obj, data); + } +} + + +nxt_int_t +nxt_runtime_idle_close_test(nxt_thread_t *thr) +{ + int fds[2]; + nxt_mp_t *mp; + nxt_int_t ret; + nxt_conn_t *proto, *fresh; + nxt_task_t *task; + nxt_sockaddr_t sa, *remote; + nxt_listen_event_t lev; + nxt_listen_socket_t ls; + nxt_event_engine_t engine, *saved_engine; + nxt_work_queue_cache_t cache; + + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "runtime idle close test started"); + + ret = NXT_ERROR; + fds[0] = -1; + fds[1] = -1; + proto = NULL; + fresh = NULL; + + task = thr->task; + task->thread = thr; + + nxt_memzero(&engine, sizeof(nxt_event_engine_t)); + nxt_memzero(&lev, sizeof(nxt_listen_event_t)); + nxt_memzero(&ls, sizeof(nxt_listen_socket_t)); + nxt_memzero(&sa, sizeof(nxt_sockaddr_t)); + + /* Backs the conn structs and the sockaddr cache. */ + engine.mem_pool = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(engine.mem_pool == NULL)) { + return NXT_ERROR; + } + + engine.task = *task; + engine.event.close = nxt_runtime_idle_close_test_event_close; + + nxt_queue_init(&engine.idle_connections); + nxt_queue_init(&engine.active_connections); + nxt_queue_init(&engine.listen_connections); + nxt_queue_init(&engine.joints); + + nxt_work_queue_cache_create(&cache, 1024); + + engine.fast_work_queue.cache = &cache; + nxt_work_queue_name(&engine.fast_work_queue, "fast"); + + engine.close_work_queue.cache = &cache; + nxt_work_queue_name(&engine.close_work_queue, "close"); + + engine.shutdown_work_queue.cache = &cache; + nxt_work_queue_name(&engine.shutdown_work_queue, "shutdown"); + + /* Enough of a listen socket to size and type cached sockaddrs from. */ + sa.type = SOCK_STREAM; + sa.u.sockaddr.sa_family = AF_INET; + + ls.socket = -1; + ls.sockaddr = &sa; + ls.socklen = sizeof(struct sockaddr_in); + ls.address_length = NXT_INET_ADDR_STR_LEN; + + /* + * The listener the conns were accepted on: its own reference plus the + * one nxt_conn_accept() takes for the protocol-owned conn below. Not + * draining and without a joint, so nxt_router_listen_event_release() + * only drops the count. + */ + lev.listen = &ls; + lev.count = 2; + + saved_engine = thr->engine; + thr->engine = &engine; + + /* + * Real, valid fds so nxt_socket_close() in the close handler closes + * something real rather than an already-invalid descriptor. + */ + if (nxt_slow_path(pipe(fds) != 0)) { + nxt_log_alert(thr->log, "runtime idle close test failed to open a " + "pipe"); + goto done; + } + + /* + * A keep-alive conn owned by its protocol: as nxt_conn_accept_alloc() + * and nxt_conn_accept() leave it, with a cached sockaddr, a listener + * reference, and a stale write_state from its last response. + */ + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + goto done; + } + + proto = nxt_conn_create(mp, task); + if (nxt_slow_path(proto == NULL)) { + nxt_mp_destroy(mp); + goto done; + } + + /* From here on the per-conn pool belongs to the conn. */ + + proto->socket.fd = fds[0]; + fds[0] = -1; + + proto->remote = nxt_sockaddr_cache_alloc(&engine, &ls); + if (nxt_slow_path(proto->remote == NULL)) { + goto done; + } + + remote = proto->remote; + + proto->listen = &lev; + + /* Bypasses the shutdown() syscall branch in nxt_conn_close(). */ + proto->socket.error = 1; + + proto->read_state = &nxt_runtime_idle_close_test_proto_read_state; + proto->write_state = &nxt_runtime_idle_close_test_stale_state; + + /* Idle, and eligible for the close pass (read_ready == 0). */ + proto->socket.read_ready = 0; + nxt_conn_idle(&engine, proto); + + /* + * A conn accepted a moment ago whose listen handler has not run yet: + * no protocol state at all. Before the fix the pass closed it with + * write_state == NULL, a crash in nxt_conn_close_handler(); the pass + * must leave it alone now. + */ + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + goto done; + } + + fresh = nxt_conn_create(mp, task); + if (nxt_slow_path(fresh == NULL)) { + nxt_mp_destroy(mp); + goto done; + } + + fresh->socket.fd = fds[1]; + fds[1] = -1; + + fresh->remote = nxt_sockaddr_cache_alloc(&engine, &ls); + if (nxt_slow_path(fresh->remote == NULL)) { + goto done; + } + + fresh->listen = &lev; + fresh->socket.error = 1; + fresh->socket.read_ready = 0; + nxt_conn_idle(&engine, fresh); + + nxt_runtime_idle_close_test_stale_calls = 0; + nxt_runtime_idle_close_test_close_calls = 0; + nxt_runtime_idle_close_test_free_calls = 0; + nxt_runtime_idle_close_test_closed_conn = NULL; + + nxt_runtime_test_close_idle_connections(&engine); + + /* Runs nxt_conn_close_handler(), which dispatches to fast_work_queue. */ + nxt_runtime_idle_close_test_drain(&engine.close_work_queue); + nxt_runtime_idle_close_test_drain(&engine.fast_work_queue); + + if (nxt_slow_path(nxt_runtime_idle_close_test_close_calls != 1 + || nxt_runtime_idle_close_test_closed_conn != proto)) + { + nxt_log_alert(thr->log, "runtime idle close test: the protocol " + "close handler did not run exactly once for the " + "protocol-owned conn (%ui calls)", + nxt_runtime_idle_close_test_close_calls); + goto done; + } + + if (nxt_slow_path(nxt_runtime_idle_close_test_stale_calls != 0)) { + nxt_log_alert(thr->log, "runtime idle close test: the stale " + "write_state ran -- the idle-close pass bypassed the " + "protocol close handler"); + goto done; + } + + if (nxt_slow_path(nxt_runtime_idle_close_test_free_calls != 1)) { + nxt_log_alert(thr->log, "runtime idle close test: the protocol " + "free handler did not run (%ui calls)", + nxt_runtime_idle_close_test_free_calls); + goto done; + } + + /* The protocol free path ran: proto is gone, only fresh is left. */ + proto = NULL; + + if (nxt_slow_path(lev.count != 1)) { + nxt_log_alert(thr->log, "runtime idle close test: the listener " + "reference was not released (count %uD, expected 1)", + lev.count); + goto done; + } + + /* + * The sockaddr went back to the engine cache: the cache hands out its + * free list first, so the next allocation for this listener must be + * that very block (it lives in the engine pool, destroyed below). + */ + if (nxt_slow_path(nxt_sockaddr_cache_alloc(&engine, &ls) != remote)) { + nxt_log_alert(thr->log, "runtime idle close test: the closed " + "conn's sockaddr was not returned to the engine " + "cache"); + goto done; + } + + if (nxt_slow_path(engine.connections != 1)) { + nxt_log_alert(thr->log, "runtime idle close test: engine accounts " + "for %uD connection(s), expected 1 -- either the " + "protocol-owned conn was not freed or the " + "uninitialized one was", engine.connections); + goto done; + } + + if (nxt_slow_path(nxt_queue_is_empty(&engine.idle_connections) + || nxt_queue_first(&engine.idle_connections) + != &fresh->link + || nxt_queue_next(&fresh->link) + != nxt_queue_tail(&engine.idle_connections) + || fresh->idle != NXT_CONN_TRACK_IDLE)) + { + nxt_log_alert(thr->log, "runtime idle close test: idle_connections " + "must hold exactly the not-yet-initialized conn"); + goto done; + } + + if (nxt_slow_path(fresh->write_state != NULL + || fresh->socket.fd == -1)) + { + nxt_log_alert(thr->log, "runtime idle close test: the " + "not-yet-initialized conn was touched by the pass"); + goto done; + } + + ret = NXT_OK; + +done: + + /* What the listen handler and the protocol would eventually do. */ + + if (fresh != NULL) { + if (fresh->socket.fd != -1) { + (void) close(fresh->socket.fd); + fresh->socket.fd = -1; + } + + nxt_conn_untrack(&engine, fresh); + + if (fresh->remote != NULL) { + nxt_sockaddr_cache_free(&engine, fresh); + } + + nxt_conn_free(task, fresh); + } + + if (proto != NULL) { + if (proto->socket.fd != -1) { + (void) close(proto->socket.fd); + proto->socket.fd = -1; + } + + nxt_conn_untrack(&engine, proto); + + if (proto->remote != NULL) { + nxt_sockaddr_cache_free(&engine, proto); + } + + nxt_conn_free(task, proto); + } + + if (fds[0] != -1) { + (void) close(fds[0]); + } + + if (fds[1] != -1) { + (void) close(fds[1]); + } + + thr->engine = saved_engine; + + nxt_work_queue_cache_destroy(&cache); + nxt_mp_destroy(engine.mem_pool); + + if (nxt_fast_path(ret == NXT_OK)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "runtime idle close test passed"); + } + + return ret; +} diff --git a/src/test/nxt_string_test.c b/src/test/nxt_string_test.c index f82d19be4..51400858b 100644 --- a/src/test/nxt_string_test.c +++ b/src/test/nxt_string_test.c @@ -11,6 +11,7 @@ nxt_int_t nxt_string_test(nxt_thread_t *thr) { u_char *p; + size_t n; nxt_uint_t i; nxt_bool_t enc; @@ -51,6 +52,20 @@ nxt_string_test(nxt_thread_t *thr) { nxt_string(" "), 0 }, /* space must be escaped */ }; + /* nxt_str_strip(): the length without trailing CR, LF, SP and HTAB. */ + static const struct { + nxt_str_t str; + size_t length; + } strip[] = { + { nxt_string("13700"), 5 }, + { nxt_string("13700\r\n"), 5 }, + { nxt_string("13700 \t\n"), 5 }, + { nxt_string("1 2\n"), 3 }, /* inner space stays */ + { nxt_string(" 13700"), 6 }, /* leading space stays */ + { nxt_string(" \t\r\n"), 0 }, + { nxt_string(""), 0 }, + }; + nxt_thread_time_update(thr); for (i = 0; i < nxt_nitems(rmem); i++) { @@ -94,6 +109,18 @@ nxt_string_test(nxt_thread_t *thr) } } + for (i = 0; i < nxt_nitems(strip); i++) { + n = nxt_str_strip(strip[i].str.start, + strip[i].str.start + strip[i].str.length); + + if (n != strip[i].length) { + nxt_log_alert(thr->log, "nxt_str_strip(\"%V\") failed: " + "got %uz, expected %uz", &strip[i].str, n, + strip[i].length); + return NXT_ERROR; + } + } + nxt_log_error(NXT_LOG_NOTICE, thr->log, "nxt_string helpers test passed"); return NXT_OK; diff --git a/src/test/nxt_tests.c b/src/test/nxt_tests.c index 9edd4c454..3602c0156 100644 --- a/src/test/nxt_tests.c +++ b/src/test/nxt_tests.c @@ -14,6 +14,72 @@ nxt_module_init_t nxt_init_modules[1]; nxt_uint_t nxt_init_modules_n; +/* + * Whether the descriptor is still open in this process. Several port tests + * assert on a descriptor's real state rather than on what a handler returned, + * because the bug they guard against is a close that did or did not happen. + */ + +nxt_bool_t +nxt_test_fd_is_open(nxt_fd_t fd) +{ + return fcntl(fd, F_GETFD) != -1; +} + + +/* + * Runs fn(data) in a child process. Returns the exit status of the child, + * or -1 when the child did not exit. + */ + +int +nxt_test_in_child(nxt_thread_t *thr, const char *name, int (*fn)(void *), + void *data) +{ + int status; + pid_t child; + + child = fork(); + + if (child == 0) { + _exit(fn(data)); + } + + if (child == -1) { + nxt_log_alert(thr->log, "%s: fork() failed %E", name, nxt_errno); + return -1; + } + + /* A signal may interrupt waitpid(). Then it is called again. */ + + while (waitpid(child, &status, 0) != child) { + if (nxt_errno != NXT_EINTR) { + nxt_log_alert(thr->log, "%s: waitpid() failed %E", name, + nxt_errno); + return -1; + } + } + + if (!WIFEXITED(status)) { + nxt_log_alert(thr->log, "%s: child killed by signal %d", name, + WTERMSIG(status)); + return -1; + } + + return WEXITSTATUS(status); +} + + +static nxt_int_t (*const nxt_security_tests[])(nxt_thread_t *) = { + nxt_checked_test, nxt_port_mmap_read_test, + nxt_router_response_parse_test, nxt_port_frag_test, + nxt_port_release_test, nxt_nncq_bound_test, +#if (NXT_HAVE_REGEX) + nxt_regex_test, +#endif +}; + + /* The function is defined here to prevent inline optimizations. */ static nxt_bool_t nxt_msec_less(nxt_msec_t first, nxt_msec_t second) @@ -25,6 +91,7 @@ nxt_msec_less(nxt_msec_t first, nxt_msec_t second) int nxt_cdecl main(int argc, char **argv) { + nxt_uint_t i; nxt_task_t task; nxt_thread_t *thr; @@ -114,18 +181,6 @@ main(int argc, char **argv) return 1; } - if (nxt_mem_zone_test(thr, 100, 20000, 128 - 1) != NXT_OK) { - return 1; - } - - if (nxt_mem_zone_test(thr, 100, 10000, 4096 - 1) != NXT_OK) { - return 1; - } - - if (nxt_mem_zone_test(thr, 1000, 40, 64 * 1024 - 1) != NXT_OK) { - return 1; - } - if (nxt_lvlhsh_test(thr, 2, 1) != NXT_OK) { return 1; } @@ -158,6 +213,10 @@ main(int argc, char **argv) return 1; } + if (nxt_utf8_sanitize_test(thr) != NXT_OK) { + return 1; + } + if (nxt_http_parse_test(thr) != NXT_OK) { return 1; } @@ -174,10 +233,22 @@ main(int argc, char **argv) return 1; } + if (nxt_buf_test(thr) != NXT_OK) { + return 1; + } + if (nxt_http_chunk_parse_test(thr) != NXT_OK) { return 1; } + if (nxt_http_validate_host_test(thr) != NXT_OK) { + return 1; + } + + if (nxt_http_request_body_alloc_test(thr) != NXT_OK) { + return 1; + } + if (nxt_conf_json_depth_test(thr) != NXT_OK) { return 1; } @@ -186,25 +257,191 @@ main(int argc, char **argv) return 1; } + if (nxt_conf_map_object_test(thr) != NXT_OK) { + return 1; + } + if (nxt_port_fail_test(thr) != NXT_OK) { return 1; } + if (nxt_fd_event_change_test(thr) != NXT_OK) { + return 1; + } + + if (nxt_port_use_unless_zero_test(thr) != NXT_OK) { + return 1; + } + if (nxt_port_mmap_range_test(thr) != NXT_OK) { return 1; } + if (nxt_port_mmaps_max_test(thr) != NXT_OK) { + return 1; + } + + if (nxt_port_ready_test(thr) != NXT_OK) { + return 1; + } + + if (nxt_conn_close_idle_test(thr) != NXT_OK) { + return 1; + } + + if (nxt_listen_event_test(thr) != NXT_OK) { + return 1; + } + + if (nxt_conn_io_accept_test(thr) != NXT_OK) { + return 1; + } + + if (nxt_runtime_idle_close_test(thr) != NXT_OK) { + return 1; + } + + if (nxt_router_new_port_test(thr) != NXT_OK) { + return 1; + } + + if (nxt_router_start_fail_test(thr) != NXT_OK) { + return 1; + } + + if (nxt_router_start_fail_soak_test(thr) != NXT_OK) { + return 1; + } + + if (nxt_router_start_proto_gone_test(thr) != NXT_OK) { + return 1; + } + + if (nxt_router_proto_wedge_test(thr) != NXT_OK) { + return 1; + } + + if (nxt_router_proto_death_test(thr) != NXT_OK) { + return 1; + } + + if (nxt_router_start_timeout_test(thr) != NXT_OK) { + return 1; + } + + if (nxt_router_app_timeout_test(thr) != NXT_OK) { + return 1; + } + + if (nxt_router_remove_pid_soak_test(thr) != NXT_OK) { + return 1; + } + + if (nxt_router_detached_test(thr) != NXT_OK) { + return 1; + } + + if (nxt_router_sender_test(thr) != NXT_OK) { + return 1; + } + + if (nxt_router_stale_joint_test(thr) != NXT_OK) { + return 1; + } + + if (nxt_router_websocket_test(thr) != NXT_OK) { + return 1; + } + + if (nxt_router_prepare_msg_test(thr) != NXT_OK) { + return 1; + } + + if (nxt_main_start_process_reply_test(thr) != NXT_OK) { + return 1; + } + + if (nxt_main_file_store_test(thr) != NXT_OK) { + return 1; + } + + if (nxt_main_store_test(thr) != NXT_OK) { + return 1; + } + + if (nxt_proto_creating_wedge_test(thr) != NXT_OK) { + return 1; + } + + if (nxt_main_whoami_test(thr) != NXT_OK) { + return 1; + } + + if (nxt_main_remove_child_pid_test(thr) != NXT_OK) { + return 1; + } + + if (nxt_port_change_file_test(thr) != NXT_OK) { + return 1; + } + + if (nxt_port_ctrunc_test(thr) != NXT_OK) { + return 1; + } + + if (nxt_port_fd_test(thr) != NXT_OK) { + return 1; + } + + if (nxt_port_rpc_fd_test(thr) != NXT_OK) { + return 1; + } + + if (nxt_port_queued_fd_test(thr) != NXT_OK) { + return 1; + } + + for (i = 0; i < nxt_nitems(nxt_security_tests); i++) { + if (nxt_security_tests[i](thr) != NXT_OK) { + return 1; + } + } + + if (nxt_conn_close_test(thr) != NXT_OK) { + return 1; + } + +#if (NXT_HAVE_OPENSSL) + if (nxt_openssl_server_init_test(thr) != NXT_OK) { + return 1; + } +#endif + #if (NXT_HAVE_CGROUP) if (nxt_cgroup_test(thr) != NXT_OK) { return 1; } #endif + if (nxt_controller_peer_test(thr) != NXT_OK) { + return 1; + } + + if (nxt_cpu_limit_test(thr) != NXT_OK) { + return 1; + } + #if (NXT_HAVE_CLONE_NEWUSER) if (nxt_clone_creds_test(thr) != NXT_OK) { return 1; } #endif +#if (NXT_HAVE_ISOLATION_ROOTFS) + if (nxt_isolation_mount_dst_test(thr) != NXT_OK) { + return 1; + } +#endif + return 0; } diff --git a/src/test/nxt_tests.h b/src/test/nxt_tests.h index 2f77c0706..a5aaaefeb 100644 --- a/src/test/nxt_tests.h +++ b/src/test/nxt_tests.h @@ -54,8 +54,6 @@ void nxt_rbtree1_mb_delete(nxt_thread_t *thr); nxt_int_t nxt_mp_test(nxt_thread_t *thr, nxt_uint_t runs, nxt_uint_t nblocks, size_t max_size); nxt_int_t nxt_mp_get_align_test(nxt_thread_t *thr); -nxt_int_t nxt_mem_zone_test(nxt_thread_t *thr, nxt_uint_t runs, - nxt_uint_t nblocks, size_t max_size); nxt_int_t nxt_lvlhsh_test(nxt_thread_t *thr, nxt_uint_t n, nxt_bool_t use_pool); @@ -63,17 +61,84 @@ nxt_int_t nxt_gmtime_test(nxt_thread_t *thr); nxt_int_t nxt_sprintf_test(nxt_thread_t *thr); nxt_int_t nxt_malloc_test(nxt_thread_t *thr); nxt_int_t nxt_utf8_test(nxt_thread_t *thr); +nxt_int_t nxt_utf8_sanitize_test(nxt_thread_t *thr); nxt_int_t nxt_http_parse_test(nxt_thread_t *thr); nxt_int_t nxt_strverscmp_test(nxt_thread_t *thr); nxt_int_t nxt_base64_test(nxt_thread_t *thr); nxt_int_t nxt_string_test(nxt_thread_t *thr); +nxt_int_t nxt_buf_test(nxt_thread_t *thr); nxt_int_t nxt_http_chunk_parse_test(nxt_thread_t *thr); +nxt_int_t nxt_http_validate_host_test(nxt_thread_t *thr); +nxt_int_t nxt_http_request_body_alloc_test(nxt_thread_t *thr); nxt_int_t nxt_conf_json_depth_test(nxt_thread_t *thr); nxt_int_t nxt_http_route_addr_test(nxt_thread_t *thr); +nxt_int_t nxt_conf_map_object_test(nxt_thread_t *thr); nxt_int_t nxt_port_fail_test(nxt_thread_t *thr); +nxt_int_t nxt_fd_event_change_test(nxt_thread_t *thr); +nxt_int_t nxt_port_use_unless_zero_test(nxt_thread_t *thr); nxt_int_t nxt_port_mmap_range_test(nxt_thread_t *thr); +nxt_int_t nxt_port_mmap_read_test(nxt_thread_t *thr); +nxt_int_t nxt_port_mmaps_max_test(nxt_thread_t *thr); +nxt_int_t nxt_port_ready_test(nxt_thread_t *thr); +nxt_int_t nxt_conn_close_idle_test(nxt_thread_t *thr); +nxt_int_t nxt_listen_event_test(nxt_thread_t *thr); +nxt_int_t nxt_conn_io_accept_test(nxt_thread_t *thr); +nxt_int_t nxt_runtime_idle_close_test(nxt_thread_t *thr); +nxt_int_t nxt_router_new_port_test(nxt_thread_t *thr); +nxt_int_t nxt_router_start_fail_test(nxt_thread_t *thr); +nxt_int_t nxt_router_start_fail_soak_test(nxt_thread_t *thr); +nxt_int_t nxt_router_start_proto_gone_test(nxt_thread_t *thr); +nxt_int_t nxt_router_proto_wedge_test(nxt_thread_t *thr); +nxt_int_t nxt_router_proto_death_test(nxt_thread_t *thr); +nxt_int_t nxt_router_start_timeout_test(nxt_thread_t *thr); +nxt_int_t nxt_router_app_timeout_test(nxt_thread_t *thr); +nxt_int_t nxt_router_remove_pid_soak_test(nxt_thread_t *thr); +nxt_int_t nxt_router_detached_test(nxt_thread_t *thr); +nxt_int_t nxt_router_sender_test(nxt_thread_t *thr); +nxt_int_t nxt_router_stale_joint_test(nxt_thread_t *thr); +nxt_int_t nxt_router_websocket_test(nxt_thread_t *thr); +nxt_int_t nxt_router_prepare_msg_test(nxt_thread_t *thr); +nxt_int_t nxt_main_start_process_reply_test(nxt_thread_t *thr); +nxt_int_t nxt_main_file_store_test(nxt_thread_t *thr); +nxt_int_t nxt_main_store_test(nxt_thread_t *thr); +nxt_int_t nxt_proto_creating_wedge_test(nxt_thread_t *thr); +nxt_int_t nxt_main_whoami_test(nxt_thread_t *thr); +nxt_int_t nxt_main_remove_child_pid_test(nxt_thread_t *thr); +nxt_int_t nxt_port_change_file_test(nxt_thread_t *thr); +nxt_int_t nxt_port_ctrunc_test(nxt_thread_t *thr); +nxt_int_t nxt_port_fd_test(nxt_thread_t *thr); +nxt_int_t nxt_port_rpc_fd_test(nxt_thread_t *thr); +nxt_int_t nxt_port_queued_fd_test(nxt_thread_t *thr); +nxt_int_t nxt_checked_test(nxt_thread_t *thr); +nxt_int_t nxt_conn_close_test(nxt_thread_t *thr); +nxt_int_t nxt_router_response_parse_test(nxt_thread_t *thr); +nxt_int_t nxt_port_frag_test(nxt_thread_t *thr); +nxt_int_t nxt_port_release_test(nxt_thread_t *thr); +nxt_int_t nxt_nncq_bound_test(nxt_thread_t *thr); +#if (NXT_HAVE_REGEX) +nxt_int_t nxt_regex_test(nxt_thread_t *thr); +#endif +nxt_int_t nxt_openssl_server_init_test(nxt_thread_t *thr); nxt_int_t nxt_cgroup_test(nxt_thread_t *thr); +nxt_int_t nxt_controller_peer_test(nxt_thread_t *thr); +nxt_int_t nxt_cpu_limit_test(nxt_thread_t *thr); nxt_int_t nxt_clone_creds_test(nxt_thread_t *thr); +#if (NXT_HAVE_ISOLATION_ROOTFS) +nxt_int_t nxt_isolation_mount_dst_test(nxt_thread_t *thr); +#endif + +nxt_bool_t nxt_test_fd_is_open(nxt_fd_t fd); +int nxt_test_in_child(nxt_thread_t *thr, const char *name, int (*fn)(void *), + void *data); + + +#define NXT_TEST_CHECK(log, cond, ...) \ + do { \ + if (!(cond)) { \ + nxt_log_alert(log, __VA_ARGS__); \ + return NXT_ERROR; \ + } \ + } while (0) #endif /* _NXT_TESTS_H_INCLUDED_ */ diff --git a/src/test/nxt_unit_msg_test.c b/src/test/nxt_unit_msg_test.c new file mode 100644 index 000000000..11bdc0dd5 --- /dev/null +++ b/src/test/nxt_unit_msg_test.c @@ -0,0 +1,1341 @@ + +/* + * Copyright (C) FreeUnit contributors. + */ + +/* + * Malformed port messages at libunit's receive side (src/nxt_unit.c), fed + * through nxt_unit_test_process_msg(). nxt_unit_mmap_read() used to read a + * partial last record past the message, mmap_id 0xFFFFFFFF wrapped the + * lib->incoming index, a segment id like 100000000 grew lib->incoming to + * that many slots, and a segment of any size was accepted. Cases that + * could crash the old code run in a child; a signal is a failure. + * + * The request cases put an nxt_unit_request_t into segment 0 the way the + * router does, and keep the test's own mapping of the segment: the segment + * is shared by every process of an application, so the test also plays a + * sibling process that writes the request after libunit checked it. + */ + +#include "nxt_main.h" +#include "nxt_port_memory_int.h" +#include "nxt_port_queue.h" +#include "nxt_app_queue.h" +#include "nxt_unit.h" +#include "nxt_unit_request.h" +#include "nxt_unit_websocket.h" + +#include +#include +#include +#include +#include +#include +#include +#include + + +/* What the request handler does with a request that arrived. */ +typedef enum { + NXT_UNIT_MSG_TEST_READ_BODY = 0, + NXT_UNIT_MSG_TEST_GROUP_DUP, + NXT_UNIT_MSG_TEST_RAISE_COUNT, + NXT_UNIT_MSG_TEST_UPGRADE, + NXT_UNIT_MSG_TEST_ORDER, +} nxt_unit_msg_test_mode_t; + + +#define NXT_UNIT_MSG_TEST_ROUTER_PORT 2 +#define NXT_UNIT_MSG_TEST_ENGINE_PORT 5 +#define NXT_UNIT_MSG_TEST_FIELDS 32 +#define NXT_UNIT_MSG_TEST_STREAM 7 +#define NXT_UNIT_MSG_TEST_BODY "ab" + + +static int nxt_unit_msg_test_failures; +static int nxt_unit_msg_test_send_fails; +static int nxt_unit_msg_test_quit_called; +static int nxt_unit_msg_test_handler_calls; +static int nxt_unit_msg_test_handler_ok; +static int nxt_unit_msg_test_ws_calls; +static int nxt_unit_msg_test_ws_ok; +static int nxt_unit_msg_test_close_calls; +static nxt_unit_msg_test_mode_t nxt_unit_msg_test_mode; +static pid_t nxt_unit_msg_test_pid; +static nxt_unit_ctx_t *nxt_unit_msg_test_ctx; +static nxt_unit_ctx_t *nxt_unit_msg_test_follower; +static nxt_port_mmap_header_t *nxt_unit_msg_test_seg0; +static uint16_t nxt_unit_msg_test_reply_port + = NXT_UNIT_MSG_TEST_ROUTER_PORT; + +/* What libunit sent to the socket of the engine port, in order. */ +static nxt_port_msg_t nxt_unit_msg_test_dgrams[16]; +static size_t nxt_unit_msg_test_ndgrams; + +static int nxt_unit_msg_test_respond(nxt_unit_request_info_t *req); + +static int nxt_unit_msg_test_send_records_to(nxt_unit_ctx_t *ctx, + const nxt_port_mmap_msg_t *records, size_t nrecords, size_t tail); + + +static void +nxt_unit_msg_test_assert(int cond, const char *name) +{ + if (cond) { + printf("unit msg test: %-50s passed\n", name); + + } else { + printf("unit msg test: %-50s FAILED\n", name); + + nxt_unit_msg_test_failures++; + } +} + + +static void +nxt_unit_msg_test_handler(nxt_unit_request_info_t *req) +{ + char buf[16]; + ssize_t n; + nxt_unit_field_t *f; + nxt_unit_request_t *r; + + nxt_unit_msg_test_handler_calls++; + + r = req->request; + + switch (nxt_unit_msg_test_mode) { + + case NXT_UNIT_MSG_TEST_READ_BODY: + n = nxt_unit_request_read(req, buf, sizeof(buf)); + + nxt_unit_msg_test_handler_ok = + n == (ssize_t) nxt_length(NXT_UNIT_MSG_TEST_BODY) + && memcmp(buf, NXT_UNIT_MSG_TEST_BODY, n) == 0; + + break; + + case NXT_UNIT_MSG_TEST_GROUP_DUP: + nxt_unit_request_group_dup_fields(req); + + /* "X-Dup" and "x-dup" are now neighbours, with one name. */ + f = r->fields; + + nxt_unit_msg_test_handler_ok = + r->fields_count == 3 + && f[1].name_length == nxt_length("X-Dup") + && nxt_unit_sptr_get(&f[1].name) == nxt_unit_sptr_get(&f[2].name) + && memcmp(nxt_unit_sptr_get(&f[1].value), "1", 1) == 0 + && memcmp(nxt_unit_sptr_get(&f[2].value), "2", 1) == 0 + && f[0].name_length == nxt_length("Host") + && memcmp(nxt_unit_sptr_get(&f[0].value), "localhost", 9) == 0; + + break; + + case NXT_UNIT_MSG_TEST_RAISE_COUNT: + /* + * A sibling process raises the count after libunit checked it. + * The request sits in the last chunk of the segment, so a walk + * over that many fields leaves the mapping. + */ + r->fields_count = 0x00FFFFFF; + + nxt_unit_request_group_dup_fields(req); + + nxt_unit_msg_test_handler_ok = 1; + + break; + + case NXT_UNIT_MSG_TEST_UPGRADE: + nxt_unit_msg_test_handler_ok = + nxt_unit_response_init(req, 101, 0, 0) == NXT_UNIT_OK + && nxt_unit_response_upgrade(req) == NXT_UNIT_OK + && nxt_unit_response_send(req) == NXT_UNIT_OK; + + /* The request now waits for frames; the close finishes it. */ + return; + + case NXT_UNIT_MSG_TEST_ORDER: + nxt_unit_msg_test_handler_ok = + nxt_unit_msg_test_respond(req) == NXT_UNIT_OK; + + nxt_unit_request_done(req, NXT_UNIT_OK); + + return; + } + + nxt_unit_request_done(req, NXT_UNIT_ERROR); +} + + +static void +nxt_unit_msg_test_websocket(nxt_unit_websocket_frame_t *ws) +{ + char buf[16]; + ssize_t n; + + nxt_unit_msg_test_ws_calls++; + + n = nxt_unit_websocket_read(ws, buf, sizeof(buf)); + + nxt_unit_msg_test_ws_ok = ws->payload_len == 3 && n == 3 + && memcmp(buf, "abc", 3) == 0; + + nxt_unit_websocket_done(ws); +} + + +static void +nxt_unit_msg_test_close(nxt_unit_request_info_t *req) +{ + nxt_unit_msg_test_close_calls++; + + nxt_unit_request_done(req, NXT_UNIT_ERROR); +} + + +static ssize_t +nxt_unit_msg_test_send(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, + const void *buf, size_t buf_size, const void *oob, size_t oob_size) +{ + const nxt_port_msg_t *msg; + nxt_port_mmap_msg_t rec; + + msg = buf; + + if (port->id.id == NXT_UNIT_MSG_TEST_ENGINE_PORT + && buf_size >= sizeof(nxt_port_msg_t) + && msg->type != _NXT_PORT_MSG_READ_QUEUE + && nxt_unit_msg_test_ndgrams < nxt_nitems(nxt_unit_msg_test_dgrams)) + { + memcpy(&nxt_unit_msg_test_dgrams[nxt_unit_msg_test_ndgrams++], msg, + sizeof(nxt_port_msg_t)); + } + + /* + * While one context asks the router for a segment, another one reads + * a record for the same segment and parks behind it. + */ + if (msg->type == _NXT_PORT_MSG_GET_MMAP + && nxt_unit_msg_test_follower != NULL) + { + rec.mmap_id = ((const nxt_port_msg_get_mmap_t *) (msg + 1))->id; + rec.chunk_id = 0; + rec.size = 100; + + (void) nxt_unit_msg_test_send_records_to(nxt_unit_msg_test_follower, + &rec, 1, 0); + + nxt_unit_msg_test_follower = NULL; + } + + return nxt_unit_msg_test_send_fails ? -1 : (ssize_t) buf_size; +} + + +static ssize_t +nxt_unit_msg_test_recv(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, void *buf, + size_t buf_size, void *oob, size_t *oob_size) +{ + *oob_size = 0; + + return 0; +} + + +static int +nxt_unit_msg_test_shm(size_t size) +{ + int fd; + +#if (NXT_HAVE_MEMFD_CREATE) + fd = syscall(SYS_memfd_create, "unit.msg.test", 0); +#else + char name[64]; + + snprintf(name, sizeof(name), "/unit.msg.test.%d", (int) getpid()); + + fd = shm_open(name, O_CREAT | O_EXCL | O_RDWR, 0600); + shm_unlink(name); +#endif + + if (fd == -1) { + perror("shm"); + return -1; + } + + if (ftruncate(fd, size) == -1) { + perror("ftruncate"); + close(fd); + return -1; + } + + return fd; +} + + +/* A segment the router would hand over, from this process to itself. */ +static int +nxt_unit_msg_test_send_segment(size_t size, uint32_t id) +{ + int fd; + nxt_port_msg_t msg; + nxt_port_mmap_header_t *hdr; + + fd = nxt_unit_msg_test_shm(size); + if (fd == -1) { + return -2; + } + + hdr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (hdr == MAP_FAILED) { + perror("mmap"); + close(fd); + return -2; + } + + hdr->id = id; + hdr->src_pid = getpid(); + hdr->dst_pid = getpid(); + + /* The test keeps segment 0 mapped: the request cases write into it. */ + if (id == 0 && nxt_unit_msg_test_seg0 == NULL && size == PORT_MMAP_SIZE) { + nxt_unit_msg_test_seg0 = hdr; + + } else { + munmap(hdr, size); + } + + memset(&msg, 0, sizeof(msg)); + + msg.pid = getpid(); + msg.type = _NXT_PORT_MSG_MMAP; + msg.last = 1; + + return nxt_unit_test_process_msg(nxt_unit_msg_test_ctx, &msg, sizeof(msg), + fd); +} + + +/* RPC_READY with the mmap bit: the result is the mmap read's verdict. */ +static int +nxt_unit_msg_test_send_records_to(nxt_unit_ctx_t *ctx, + const nxt_port_mmap_msg_t *records, size_t nrecords, size_t tail) +{ + u_char buf[256]; + size_t size; + nxt_port_msg_t msg; + + memset(&msg, 0, sizeof(msg)); + + msg.stream = 7; + msg.pid = getpid(); + msg.type = _NXT_PORT_MSG_RPC_READY; + msg.last = 1; + msg.mmap = 1; + + size = nrecords * sizeof(nxt_port_mmap_msg_t); + + memcpy(buf, &msg, sizeof(msg)); + memcpy(buf + sizeof(msg), records, size); + memset(buf + sizeof(msg) + size, 0, tail); + + return nxt_unit_test_process_msg(ctx, buf, sizeof(msg) + size + tail, -1); +} + + +static int +nxt_unit_msg_test_send_records(const nxt_port_mmap_msg_t *records, + size_t nrecords, size_t tail) +{ + return nxt_unit_msg_test_send_records_to(nxt_unit_msg_test_ctx, records, + nrecords, tail); +} + + +/* The child cases exit with the return code + 10, to tell it from 1, 2. */ + +#define NXT_UNIT_MSG_TEST_RC(rc) ((rc) + 10) + + +/* Runs "fn" in a child: asserts that it exits with RC(expect). */ +static void +nxt_unit_msg_test_in_child(const char *name, int (*fn)(void *), void *data, + int expect) +{ + int status; + pid_t child; + + status = -1; + fflush(stdout); + + child = fork(); + + if (child == 0) { + _exit(fn(data) & 0xFF); + } + + if (child != -1 && waitpid(child, &status, 0) == child + && WIFSIGNALED(status)) + { + printf("unit msg test: child killed by signal %d\n", WTERMSIG(status)); + } + + nxt_unit_msg_test_assert(child != -1 && WIFEXITED(status) + && WEXITSTATUS(status) + == NXT_UNIT_MSG_TEST_RC(expect), name); +} + + +static int +nxt_unit_msg_test_records_case(void *data) +{ + return NXT_UNIT_MSG_TEST_RC(nxt_unit_msg_test_send_records(data, 1, 0)); +} + + +typedef struct { + const char *name; + size_t size; + uint32_t id; + int expect; +} nxt_unit_msg_test_segment_t; + + +static const nxt_unit_msg_test_segment_t segments[] = { + { "segment id 0xFFFFFFFF is refused", PORT_MMAP_SIZE, 0xFFFFFFFF, + NXT_UNIT_ERROR }, + { "segment id 1000 is accepted", PORT_MMAP_SIZE, 1000, NXT_UNIT_OK }, + { "segment id NXT_PORT_MMAPS_MAX - 1 is accepted", PORT_MMAP_SIZE, + NXT_PORT_MMAPS_MAX - 1, NXT_UNIT_OK }, + { "segment id NXT_PORT_MMAPS_MAX is refused", PORT_MMAP_SIZE, + NXT_PORT_MMAPS_MAX, NXT_UNIT_ERROR }, + { "segment id 100000000 is refused", PORT_MMAP_SIZE, 100000000, + NXT_UNIT_ERROR }, + { "segment shorter than PORT_MMAP_SIZE is refused", + PORT_MMAP_HEADER_SIZE, 1, NXT_UNIT_ERROR }, + { "segment longer than PORT_MMAP_SIZE is refused", + 2 * PORT_MMAP_SIZE, 1, NXT_UNIT_ERROR }, +}; + + +static const struct { + const char *name; + nxt_port_mmap_msg_t rec; +} bad_records[] = { + { "mmap_id 0xFFFFFFFF is refused", { 0xFFFFFFFF, 0, 100 } }, + { "mmap_id NXT_PORT_MMAPS_MAX is refused", { NXT_PORT_MMAPS_MAX, 0, 100 } }, + { "mmap_id 100000000 is refused", { 100000000, 0, 100 } }, + { "chunk_id past the data area is refused", + { 0, PORT_MMAP_CHUNK_COUNT, 100 } }, + { "size past the data area is refused", + { 0, PORT_MMAP_CHUNK_COUNT - 1, PORT_MMAP_CHUNK_SIZE + 1 } }, +}; + + +static int +nxt_unit_msg_test_segment_case(void *data) +{ + const nxt_unit_msg_test_segment_t *seg = data; + + return NXT_UNIT_MSG_TEST_RC(nxt_unit_msg_test_send_segment(seg->size, + seg->id)); +} + + +static void +nxt_unit_msg_test_quit(nxt_unit_ctx_t *ctx) +{ + nxt_unit_msg_test_quit_called = 1; +} + + +/* + * A record for an unknown segment parks the read buffer and asks the router + * for the segment. When that cannot be sent, the buffer used to be freed + * still counted in wait_items, so a graceful quit never happened. + */ +static int +nxt_unit_msg_test_get_mmap_fail_case(void *data) +{ + int rc; + u_char buf[sizeof(nxt_port_msg_t) + 1]; + nxt_port_msg_t msg; + nxt_port_mmap_msg_t rec; + + rec.mmap_id = 5; + rec.chunk_id = 0; + rec.size = 100; + + nxt_unit_msg_test_send_fails = 1; + rc = nxt_unit_msg_test_send_records(&rec, 1, 0); + nxt_unit_msg_test_send_fails = 0; + + if (rc != NXT_UNIT_ERROR) { + return 1; + } + + memset(&msg, 0, sizeof(msg)); + + msg.pid = getpid(); + msg.type = _NXT_PORT_MSG_QUIT; + msg.last = 1; + + memcpy(buf, &msg, sizeof(msg)); + buf[sizeof(msg)] = NXT_PORT_QUIT_GRACEFUL; + + (void) nxt_unit_test_process_msg(nxt_unit_msg_test_ctx, buf, sizeof(buf), + -1); + + return nxt_unit_msg_test_quit_called ? NXT_UNIT_MSG_TEST_RC(NXT_UNIT_OK) + : 2; +} + + +/* + * Two contexts park on the same unknown segment; only the first asks the + * router for it. When that failed, the second buffer stayed parked with + * its wait counted, so the second context never quit gracefully. + */ +static int +nxt_unit_msg_test_get_mmap_follower_case(void *data) +{ + int rc; + u_char buf[sizeof(nxt_port_msg_t) + 1]; + nxt_port_msg_t msg; + nxt_unit_ctx_t *ctx2; + nxt_port_mmap_msg_t rec; + + ctx2 = nxt_unit_ctx_alloc(nxt_unit_msg_test_ctx, NULL); + if (ctx2 == NULL) { + return 1; + } + + rec.mmap_id = 6; + rec.chunk_id = 0; + rec.size = 100; + + nxt_unit_msg_test_follower = ctx2; + nxt_unit_msg_test_send_fails = 1; + + rc = nxt_unit_msg_test_send_records(&rec, 1, 0); + + if (rc != NXT_UNIT_ERROR || nxt_unit_msg_test_follower != NULL) { + return 2; + } + + memset(&msg, 0, sizeof(msg)); + + msg.pid = getpid(); + msg.type = _NXT_PORT_MSG_QUIT; + msg.last = 1; + + memcpy(buf, &msg, sizeof(msg)); + buf[sizeof(msg)] = NXT_PORT_QUIT_GRACEFUL; + + /* The follower reads its record again, asks, fails, and quits. */ + + (void) nxt_unit_test_process_msg(ctx2, buf, sizeof(buf), -1); + + return nxt_unit_msg_test_quit_called ? NXT_UNIT_MSG_TEST_RC(NXT_UNIT_OK) + : 3; +} + + +static int +nxt_unit_msg_test_count_maps(void) +{ + int n; + char line[512]; + FILE *f; + + f = fopen("/proc/self/maps", "r"); + if (f == NULL) { + return -1; + } + + n = 0; + + while (fgets(line, sizeof(line), f) != NULL) { + n += (strstr(line, "unit.msg.test") != NULL); + } + + fclose(f); + + return n; +} + + +/* A duplicate segment id must not leak a mapping; the first one stays. */ +static int +nxt_unit_msg_test_dup_id_case(void *data) +{ + int i, before, after; + nxt_port_mmap_msg_t rec; + + before = nxt_unit_msg_test_count_maps(); + + for (i = 0; i < 3; i++) { + if (nxt_unit_msg_test_send_segment(PORT_MMAP_SIZE, 0) != NXT_UNIT_OK) { + return 1; + } + } + + after = nxt_unit_msg_test_count_maps(); + + if (before < 0 || after != before) { + printf("unit msg test: %d segment mappings, was %d\n", after, before); + return 2; + } + + rec.mmap_id = 0; + rec.chunk_id = 0; + rec.size = 100; + + return NXT_UNIT_MSG_TEST_RC(nxt_unit_msg_test_send_records(&rec, 1, 0)); +} + + +/* + * A request as the router lays it out (nxt_router_prepare_msg()): the + * struct, fields[], the strings, then the preread body. Written at "p", + * with every sptr set; the size is returned. + */ +static uint32_t +nxt_unit_msg_test_write_request(u_char *p) +{ + u_char *s; + nxt_unit_field_t *f; + nxt_unit_request_t *r; + + static const struct { + const char *name; + const char *value; + } fields[] = { + { "Host", "localhost" }, + { "X-Dup", "1" }, + { "x-dup", "2" }, + }; + + r = (nxt_unit_request_t *) p; + + memset(r, 0, sizeof(nxt_unit_request_t)); + + r->fields_count = nxt_nitems(fields); + + r->content_length_field = NXT_UNIT_NONE_FIELD; + r->content_type_field = NXT_UNIT_NONE_FIELD; + r->cookie_field = NXT_UNIT_NONE_FIELD; + r->authorization_field = NXT_UNIT_NONE_FIELD; + + r->content_length = nxt_length(NXT_UNIT_MSG_TEST_BODY); + + s = (u_char *) &r->fields[r->fields_count]; + +#define NXT_UNIT_MSG_TEST_STR(sptr, len, str) \ + do { \ + nxt_unit_sptr_set(&r->sptr, s); \ + r->len = nxt_length(str); \ + s = nxt_cpymem(s, str, nxt_length(str)); \ + *s++ = '\0'; \ + } while (0) + + NXT_UNIT_MSG_TEST_STR(method, method_length, "GET"); + NXT_UNIT_MSG_TEST_STR(version, version_length, "HTTP/1.1"); + NXT_UNIT_MSG_TEST_STR(remote, remote_length, "127.0.0.1"); + NXT_UNIT_MSG_TEST_STR(local_addr, local_addr_length, "127.0.0.1"); + NXT_UNIT_MSG_TEST_STR(local_port, local_port_length, "8080"); + NXT_UNIT_MSG_TEST_STR(server_name, server_name_length, "localhost"); + NXT_UNIT_MSG_TEST_STR(target, target_length, "/a?b=c"); + NXT_UNIT_MSG_TEST_STR(path, path_length, "/a"); + NXT_UNIT_MSG_TEST_STR(query, query_length, "b=c"); + +#undef NXT_UNIT_MSG_TEST_STR + + for (f = r->fields; f < &r->fields[r->fields_count]; f++) { + f->hash = nxt_unit_field_hash(fields[f - r->fields].name, + strlen(fields[f - r->fields].name)); + f->skip = 0; + f->hopbyhop = 0; + + nxt_unit_sptr_set(&f->name, s); + f->name_length = strlen(fields[f - r->fields].name); + s = nxt_cpymem(s, fields[f - r->fields].name, f->name_length); + *s++ = '\0'; + + nxt_unit_sptr_set(&f->value, s); + f->value_length = strlen(fields[f - r->fields].value); + s = nxt_cpymem(s, fields[f - r->fields].value, f->value_length); + *s++ = '\0'; + } + + nxt_unit_sptr_set(&r->preread_content, s); + s = nxt_cpymem(s, NXT_UNIT_MSG_TEST_BODY, + nxt_length(NXT_UNIT_MSG_TEST_BODY)); + + return s - p; +} + + +/* One message about the request, with one record over "chunk". */ +static int +nxt_unit_msg_test_send_chunk(uint8_t type, uint8_t last, nxt_chunk_id_t chunk, + uint32_t size) +{ + u_char buf[sizeof(nxt_port_msg_t) + + sizeof(nxt_port_mmap_msg_t)]; + nxt_port_msg_t msg; + nxt_port_mmap_msg_t rec; + + memset(&msg, 0, sizeof(msg)); + + /* The router port was registered with the pid libunit started with. */ + msg.stream = NXT_UNIT_MSG_TEST_STREAM; + msg.pid = nxt_unit_msg_test_pid; + msg.reply_port = nxt_unit_msg_test_reply_port; + msg.type = type; + msg.last = last; + msg.mmap = 1; + + rec.mmap_id = 0; + rec.chunk_id = chunk; + rec.size = size; + + memcpy(buf, &msg, sizeof(msg)); + memcpy(buf + sizeof(msg), &rec, sizeof(rec)); + + return nxt_unit_test_process_msg(nxt_unit_msg_test_ctx, buf, sizeof(buf), + -1); +} + + +/* Writes a request into "chunk", lets "edit" damage it, and sends it. */ +static int +nxt_unit_msg_test_send_request(nxt_chunk_id_t chunk, + void (*edit)(nxt_unit_request_t *r)) +{ + u_char *p; + uint32_t size; + + p = nxt_port_mmap_chunk_start(nxt_unit_msg_test_seg0, chunk); + + size = nxt_unit_msg_test_write_request(p); + + if (edit != NULL) { + edit((nxt_unit_request_t *) p); + } + + nxt_unit_msg_test_handler_calls = 0; + nxt_unit_msg_test_handler_ok = 0; + + return nxt_unit_msg_test_send_chunk(_NXT_PORT_MSG_REQ_HEADERS, 1, chunk, + size); +} + + +static void +nxt_unit_msg_test_edit_target(nxt_unit_request_t *r) +{ + r->target.offset = 0x7FFFFFF0; +} + + +static void +nxt_unit_msg_test_edit_preread(nxt_unit_request_t *r) +{ + r->preread_content.offset = 0x7FFFFFF0; +} + + +static void +nxt_unit_msg_test_edit_count(nxt_unit_request_t *r) +{ + r->fields_count = 0x10000000; +} + + +static void +nxt_unit_msg_test_edit_index(nxt_unit_request_t *r) +{ + r->cookie_field = r->fields_count; +} + + +static void +nxt_unit_msg_test_edit_field_name(nxt_unit_request_t *r) +{ + r->fields[1].name.offset = 0x7FFFFFF0; +} + + +static void +nxt_unit_msg_test_edit_field_inside(nxt_unit_request_t *r) +{ + nxt_unit_sptr_set(&r->fields[1].name, &r->fields[0]); +} + + +static void +nxt_unit_msg_test_edit_handshake(nxt_unit_request_t *r) +{ + r->websocket_handshake = 1; +} + + +typedef struct { + const char *name; + void (*edit)(nxt_unit_request_t *r); + nxt_unit_msg_test_mode_t mode; + nxt_chunk_id_t chunk; + int expect; + int handler_calls; +} nxt_unit_msg_test_request_t; + + +static const nxt_unit_msg_test_request_t requests[] = { + { "well-formed request reaches the handler", NULL, + NXT_UNIT_MSG_TEST_READ_BODY, 2, NXT_UNIT_OK, 1 }, + { "target sptr out of buffer is refused", nxt_unit_msg_test_edit_target, + NXT_UNIT_MSG_TEST_READ_BODY, 2, NXT_UNIT_ERROR, 0 }, + { "preread sptr out of buffer is refused", nxt_unit_msg_test_edit_preread, + NXT_UNIT_MSG_TEST_READ_BODY, 2, NXT_UNIT_ERROR, 0 }, + { "fields_count past the buffer is refused", nxt_unit_msg_test_edit_count, + NXT_UNIT_MSG_TEST_READ_BODY, 2, NXT_UNIT_ERROR, 0 }, + { "cached field index out of range is refused", + nxt_unit_msg_test_edit_index, + NXT_UNIT_MSG_TEST_READ_BODY, 2, NXT_UNIT_ERROR, 0 }, + { "field name sptr out of buffer is refused", + nxt_unit_msg_test_edit_field_name, + NXT_UNIT_MSG_TEST_READ_BODY, 2, NXT_UNIT_ERROR, 0 }, + { "field name inside fields[] is refused", + nxt_unit_msg_test_edit_field_inside, + NXT_UNIT_MSG_TEST_READ_BODY, 2, NXT_UNIT_ERROR, 0 }, + { "duplicate fields are grouped", NULL, + NXT_UNIT_MSG_TEST_GROUP_DUP, 2, NXT_UNIT_OK, 1 }, + { "fields_count raised after the check stays in the buffer", NULL, + NXT_UNIT_MSG_TEST_RAISE_COUNT, PORT_MMAP_CHUNK_COUNT - 1, NXT_UNIT_OK, + 1 }, +}; + + +static int +nxt_unit_msg_test_request_case(void *data) +{ + int rc; + const nxt_unit_msg_test_request_t *t = data; + + nxt_unit_msg_test_mode = t->mode; + + rc = nxt_unit_msg_test_send_request(t->chunk, t->edit); + + if (nxt_unit_msg_test_handler_calls != t->handler_calls) { + printf("unit msg test: handler called %d times, expected %d\n", + nxt_unit_msg_test_handler_calls, t->handler_calls); + return 1; + } + + if (t->handler_calls != 0 && !nxt_unit_msg_test_handler_ok) { + printf("unit msg test: handler saw a wrong request\n"); + return 2; + } + + return NXT_UNIT_MSG_TEST_RC(rc); +} + + +/* + * A sibling process that keeps changing the preread offset while the + * request is checked and then used. libunit resolved the sptr once for + * the check and again for the body start, so a change in between put the + * body start 2 GB past the buffer, and the handler's read went there. + */ + +typedef struct { + volatile uint32_t *offset; + uint32_t good; + volatile int stop; +} nxt_unit_msg_test_racer_t; + + +static void * +nxt_unit_msg_test_racer(void *data) +{ + nxt_unit_msg_test_racer_t *racer = data; + + while (!racer->stop) { + *racer->offset = 0x7FFFFFF0; + *racer->offset = racer->good; + } + + return NULL; +} + + +static int +nxt_unit_msg_test_preread_race_case(void *data) +{ + int i, rc, accepted; + u_char *p; + uint32_t size; + pthread_t thread; + nxt_unit_request_t *r; + nxt_unit_msg_test_racer_t racer; + + nxt_unit_msg_test_mode = NXT_UNIT_MSG_TEST_READ_BODY; + + p = nxt_port_mmap_chunk_start(nxt_unit_msg_test_seg0, 2); + r = (nxt_unit_request_t *) p; + + size = nxt_unit_msg_test_write_request(p); + + racer.offset = &r->preread_content.offset; + racer.good = r->preread_content.offset; + racer.stop = 0; + + if (pthread_create(&thread, NULL, nxt_unit_msg_test_racer, &racer) != 0) { + return 1; + } + + accepted = 0; + + for (i = 0; i < 20000; i++) { + /* Releasing the request wipes the chunk; write it again. */ + (void) nxt_unit_msg_test_write_request(p); + + nxt_unit_msg_test_handler_calls = 0; + nxt_unit_msg_test_handler_ok = 0; + + rc = nxt_unit_msg_test_send_chunk(_NXT_PORT_MSG_REQ_HEADERS, 1, 2, + size); + + if (rc == NXT_UNIT_OK) { + if (nxt_unit_msg_test_handler_calls != 1 + || !nxt_unit_msg_test_handler_ok) + { + racer.stop = 1; + pthread_join(thread, NULL); + + printf("unit msg test: accepted request, wrong body\n"); + return 2; + } + + accepted++; + } + } + + racer.stop = 1; + pthread_join(thread, NULL); + + if (accepted == 0) { + printf("unit msg test: no request was accepted under the race\n"); + return 3; + } + + return NXT_UNIT_MSG_TEST_RC(NXT_UNIT_OK); +} + + +/* + * A Drupal REST error: a header of many fields, over 1024 bytes, goes into + * shared memory; a short JSON body goes as a plain message. + */ +static int +nxt_unit_msg_test_respond(nxt_unit_request_info_t *req) +{ + int i, rc; + char name[] = "X-Test-00"; + + static const char value[] = "config:field.storage.entity_test.field_test"; + static const char body[] = "{\"message\":\"No route found\"}"; + + rc = nxt_unit_response_init(req, 404, NXT_UNIT_MSG_TEST_FIELDS, + NXT_UNIT_MSG_TEST_FIELDS + * (sizeof(name) + sizeof(value))); + + for (i = 0; rc == NXT_UNIT_OK && i < NXT_UNIT_MSG_TEST_FIELDS; i++) { + name[7] = '0' + i / 10; + name[8] = '0' + i % 10; + + rc = nxt_unit_response_add_field(req, name, nxt_length(name), + value, nxt_length(value)); + } + + if (rc == NXT_UNIT_OK) { + rc = nxt_unit_response_send(req); + } + + if (rc == NXT_UNIT_OK) { + rc = nxt_unit_response_write(req, body, nxt_length(body)); + } + + return rc; +} + + +/* + * A router engine port has a shared memory queue. A message that goes to + * the socket puts only a READ_SOCKET marker into the queue, and for each + * marker the router reads the next datagram, from whichever process sent + * it (nxt_port_queue_read_handler()). Process "Y" puts its marker in + * before this request and sends its datagram after it, as if Y was + * preempted between the two. The router must still get the messages of + * the request in the order libunit sent them: ack, header, body, last. + * libunit put the header into the queue and the body on the socket, so the + * router read the body for Y's marker and parsed it as the header: + * "response buffer too small for fields count: 574235237" and a 503. + */ +static int +nxt_unit_msg_test_order_case(void *data) +{ + int rc, pair[2], notify; + char seq[16]; + void *queue; + size_t n, next; + ssize_t size; + uint8_t qmsg[NXT_PORT_QUEUE_MSG_SIZE]; + nxt_port_msg_t m; + + if (socketpair(AF_UNIX, SOCK_DGRAM, 0, pair) == -1) { + perror("socketpair"); + return 1; + } + + queue = mmap(NULL, sizeof(nxt_port_queue_t), PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_ANONYMOUS, -1, 0); + if (queue == MAP_FAILED) { + perror("mmap"); + return 1; + } + + nxt_port_queue_init(queue); + + rc = nxt_unit_test_add_queue_port(nxt_unit_msg_test_ctx, + nxt_unit_msg_test_pid, + NXT_UNIT_MSG_TEST_ENGINE_PORT, pair[0], + queue); + if (rc != NXT_UNIT_OK) { + printf("unit msg test: engine port not added\n"); + return 2; + } + + /* Y's marker. */ + qmsg[0] = _NXT_PORT_MSG_READ_SOCKET; + + if (nxt_port_queue_send(queue, qmsg, 1, ¬ify) != NXT_OK) { + return 3; + } + + nxt_unit_msg_test_mode = NXT_UNIT_MSG_TEST_ORDER; + nxt_unit_msg_test_reply_port = NXT_UNIT_MSG_TEST_ENGINE_PORT; + nxt_unit_msg_test_ndgrams = 0; + + rc = nxt_unit_msg_test_send_request(2, NULL); + + if (rc != NXT_UNIT_OK || nxt_unit_msg_test_handler_calls != 1 + || !nxt_unit_msg_test_handler_ok + || nxt_unit_msg_test_ndgrams == nxt_nitems(nxt_unit_msg_test_dgrams)) + { + printf("unit msg test: response failed: %d\n", rc); + return 4; + } + + /* Y's datagram. */ + memset(&m, 0, sizeof(m)); + + m.stream = NXT_UNIT_MSG_TEST_STREAM + 1; + m.type = _NXT_PORT_MSG_DATA; + + nxt_unit_msg_test_dgrams[nxt_unit_msg_test_ndgrams++] = m; + + /* The router reads the queue, and the next datagram for each marker. */ + n = 0; + next = 0; + + while (n < sizeof(seq) - 1) { + size = nxt_port_queue_recv(queue, qmsg); + if (size < 0) { + break; + } + + if (size == 1 && qmsg[0] == _NXT_PORT_MSG_READ_SOCKET) { + if (next == nxt_unit_msg_test_ndgrams) { + printf("unit msg test: a marker without a datagram\n"); + return 5; + } + + m = nxt_unit_msg_test_dgrams[next++]; + + } else if (size >= (ssize_t) sizeof(nxt_port_msg_t)) { + memcpy(&m, qmsg, sizeof(nxt_port_msg_t)); + + } else { + printf("unit msg test: a %d-byte queue message\n", (int) size); + return 6; + } + + if (m.stream != NXT_UNIT_MSG_TEST_STREAM) { + continue; + } + + seq[n++] = (m.type == _NXT_PORT_MSG_REQ_HEADERS_ACK) ? 'A' + : m.last ? 'L' + : m.mmap ? 'H' : 'B'; + } + + seq[n] = '\0'; + + if (strcmp(seq, "AHBL") != 0) { + printf("unit msg test: the router got \"%s\", expected \"AHBL\" " + "(ack, header, body, last)\n", seq); + + /* The child ends with _exit(), which does not flush stdout. */ + fflush(stdout); + + return 7; + } + + return NXT_UNIT_MSG_TEST_RC(NXT_UNIT_OK); +} + + +/* A request upgraded to a websocket, then one frame in shared memory. */ +static int +nxt_unit_msg_test_websocket_case(void *data) +{ + int rc, whole; + u_char *p; + + static const u_char mask[4] = { 0x10, 0x20, 0x30, 0x40 }; + + whole = *(const int *) data; + + nxt_unit_msg_test_mode = NXT_UNIT_MSG_TEST_UPGRADE; + + rc = nxt_unit_msg_test_send_request(2, nxt_unit_msg_test_edit_handshake); + + if (rc != NXT_UNIT_OK || nxt_unit_msg_test_handler_calls != 1 + || !nxt_unit_msg_test_handler_ok) + { + printf("unit msg test: upgrade failed: %d\n", rc); + return 1; + } + + /* A masked text frame with a 2-byte extended length: "abc". */ + p = nxt_port_mmap_chunk_start(nxt_unit_msg_test_seg0, 3); + + p[0] = 0x81; + p[1] = 0x80 | 126; + p[2] = 0; + p[3] = 3; + memcpy(p + 4, mask, 4); + p[8] = 'a' ^ mask[0]; + p[9] = 'b' ^ mask[1]; + p[10] = 'c' ^ mask[2]; + + nxt_unit_msg_test_ws_calls = 0; + nxt_unit_msg_test_ws_ok = 0; + + rc = nxt_unit_msg_test_send_chunk(_NXT_PORT_MSG_WEBSOCKET, 0, 3, + whole ? 11 : 2); + + if (whole) { + if (rc != NXT_UNIT_OK || nxt_unit_msg_test_ws_calls != 1 + || !nxt_unit_msg_test_ws_ok) + { + printf("unit msg test: frame: rc %d, handler %d ok %d\n", rc, + nxt_unit_msg_test_ws_calls, nxt_unit_msg_test_ws_ok); + return 2; + } + + } else { + /* Two bytes of a header that says it is eight. */ + if (rc != NXT_UNIT_ERROR || nxt_unit_msg_test_ws_calls != 0) { + printf("unit msg test: short frame: rc %d, handler %d\n", rc, + nxt_unit_msg_test_ws_calls); + return 3; + } + } + + nxt_unit_msg_test_close_calls = 0; + + rc = nxt_unit_msg_test_send_chunk(_NXT_PORT_MSG_WEBSOCKET, 1, 3, 0); + + if (rc != NXT_UNIT_OK || nxt_unit_msg_test_close_calls != 1) { + printf("unit msg test: close: rc %d, handler %d\n", rc, + nxt_unit_msg_test_close_calls); + return 4; + } + + return NXT_UNIT_MSG_TEST_RC(NXT_UNIT_OK); +} + + +int +main(void) +{ + int rc, ready[2], router[2], read[2], shared[2]; + char name[64]; + size_t tail, i; + nxt_unit_init_t init; + nxt_port_mmap_msg_t rec[2]; + + static const int whole_frame = 1, short_frame = 0; + + if (socketpair(AF_UNIX, SOCK_DGRAM, 0, ready) == -1 + || socketpair(AF_UNIX, SOCK_DGRAM, 0, router) == -1 + || socketpair(AF_UNIX, SOCK_DGRAM, 0, read) == -1 + || socketpair(AF_UNIX, SOCK_DGRAM, 0, shared) == -1) + { + perror("socketpair"); + return 1; + } + + nxt_unit_msg_test_pid = getpid(); + + memset(&init, 0, sizeof(init)); + + init.callbacks.request_handler = nxt_unit_msg_test_handler; + init.callbacks.websocket_handler = nxt_unit_msg_test_websocket; + init.callbacks.close_handler = nxt_unit_msg_test_close; + init.callbacks.port_send = nxt_unit_msg_test_send; + init.callbacks.port_recv = nxt_unit_msg_test_recv; + init.callbacks.quit = nxt_unit_msg_test_quit; + + init.ready_port.id.pid = getpid(); + init.ready_port.id.id = 1; + init.ready_port.in_fd = -1; + init.ready_port.out_fd = ready[0]; + + init.ready_stream = 1; + + init.router_port.id.pid = getpid(); + init.router_port.id.id = NXT_UNIT_MSG_TEST_ROUTER_PORT; + init.router_port.in_fd = -1; + init.router_port.out_fd = router[0]; + + init.read_port.id.pid = getpid(); + init.read_port.id.id = 3; + init.read_port.in_fd = read[0]; + init.read_port.out_fd = read[1]; + + init.shared_port_fd = shared[0]; + init.shared_queue_fd = nxt_unit_msg_test_shm(sizeof(nxt_app_queue_t)); + init.log_fd = STDERR_FILENO; + + if (init.shared_queue_fd == -1) { + return 1; + } + + nxt_unit_msg_test_ctx = nxt_unit_init(&init); + if (nxt_unit_msg_test_ctx == NULL) { + printf("unit msg test: nxt_unit_init() failed\n"); + return 1; + } + + /* Segment 0, the one every well-formed record below refers to. */ + + rc = nxt_unit_msg_test_send_segment(PORT_MMAP_SIZE, 0); + + nxt_unit_msg_test_assert(rc == NXT_UNIT_OK, "segment 0 is accepted"); + + if (rc != NXT_UNIT_OK) { + return 1; + } + + memset(rec, 0, sizeof(rec)); + + rec[0].size = 100; + rec[1].chunk_id = 1; + rec[1].size = PORT_MMAP_CHUNK_SIZE; + + nxt_unit_msg_test_assert( + nxt_unit_msg_test_send_records(rec, 1, 0) == NXT_UNIT_OK, + "one whole record is accepted"); + + nxt_unit_msg_test_assert( + nxt_unit_msg_test_send_records(rec, 2, 0) == NXT_UNIT_OK, + "two whole records are accepted"); + + for (tail = 1; tail < sizeof(nxt_port_mmap_msg_t); tail++) { + snprintf(name, sizeof(name), "record + %d-byte partial tail refused", + (int) tail); + + nxt_unit_msg_test_assert( + nxt_unit_msg_test_send_records(rec, 1, tail) == NXT_UNIT_ERROR, + name); + } + + nxt_unit_msg_test_assert( + nxt_unit_msg_test_send_records(rec, 0, 5) == NXT_UNIT_ERROR, + "a lone partial record is refused"); + + for (i = 0; i < nxt_nitems(bad_records); i++) { + nxt_unit_msg_test_in_child(bad_records[i].name, + nxt_unit_msg_test_records_case, + (void *) &bad_records[i].rec, + NXT_UNIT_ERROR); + } + + /* + * Capped at NXT_PORT_MMAPS_MAX, not by the shm limit: the router's + * outgoing segments are not bounded by it (#172). + */ + + for (i = 0; i < nxt_nitems(segments); i++) { + nxt_unit_msg_test_in_child(segments[i].name, + nxt_unit_msg_test_segment_case, + (void *) &segments[i], segments[i].expect); + } + + nxt_unit_msg_test_in_child("failed get_mmap does not block a graceful " + "quit", nxt_unit_msg_test_get_mmap_fail_case, + NULL, NXT_UNIT_OK); + + nxt_unit_msg_test_in_child("failed get_mmap unparks the other contexts", + nxt_unit_msg_test_get_mmap_follower_case, + NULL, NXT_UNIT_OK); + + nxt_unit_msg_test_in_child("duplicate segment id does not leak a mapping", + nxt_unit_msg_test_dup_id_case, NULL, + NXT_UNIT_OK); + + for (i = 0; i < nxt_nitems(requests); i++) { + nxt_unit_msg_test_in_child(requests[i].name, + nxt_unit_msg_test_request_case, + (void *) &requests[i], requests[i].expect); + } + + nxt_unit_msg_test_in_child("preread offset changed after the check is " + "not followed", + nxt_unit_msg_test_preread_race_case, NULL, + NXT_UNIT_OK); + + nxt_unit_msg_test_in_child("response messages keep their order through " + "the queue and the socket", + nxt_unit_msg_test_order_case, NULL, + NXT_UNIT_OK); + + nxt_unit_msg_test_in_child("masked websocket frame in shared memory", + nxt_unit_msg_test_websocket_case, + (void *) &whole_frame, NXT_UNIT_OK); + + nxt_unit_msg_test_in_child("truncated websocket frame is refused", + nxt_unit_msg_test_websocket_case, + (void *) &short_frame, NXT_UNIT_OK); + + /* + * Release the library. Otherwise the global context pointer keeps it + * reachable at exit, and LeakSanitizer does not see a leak of anything + * the library owns. + */ + nxt_unit_done(nxt_unit_msg_test_ctx); + + if (nxt_unit_msg_test_failures != 0) { + printf("unit msg test: %d failure(s)\n", nxt_unit_msg_test_failures); + return 1; + } + + printf("unit msg test: all passed\n"); + + return 0; +} diff --git a/src/test/nxt_unit_port_recv_test.c b/src/test/nxt_unit_port_recv_test.c index d93eb12cd..6870498c8 100644 --- a/src/test/nxt_unit_port_recv_test.c +++ b/src/test/nxt_unit_port_recv_test.c @@ -31,6 +31,7 @@ */ #include "nxt_main.h" +#include "nxt_socket_msg.h" #include "nxt_port.h" #include "nxt_port_queue.h" #include "nxt_app_queue.h" @@ -42,17 +43,23 @@ #include #include #include +#include -#if (NXT_HAVE_MEMFD_CREATE) +#if (NXT_HAVE_MEMFD_CREATE && NXT_HAVE_LINUX_MEMFD_H) #include #endif static int nxt_port_recv_test_failures; static int nxt_port_recv_test_step; +static int nxt_port_recv_test_block; +static int nxt_port_recv_test_quit_done; +static int nxt_port_recv_test_quit_calls; +static int nxt_port_recv_test_shared_queue_fd = -1; static int nxt_port_recv_test_sock_fd = -1; static int nxt_port_recv_test_queue_fd = -1; static int nxt_port_recv_test_ctx_queue_fd = -1; +static int nxt_port_recv_test_read_out_fd = -1; static void *nxt_port_recv_test_ctx_queue; static struct { @@ -81,6 +88,21 @@ nxt_port_recv_test_handler(nxt_unit_request_info_t *req) } +/* + * The quit callback of an embedder such as Node.js releases the context + * at once. That can free it when the libunit call returns. + */ +static void +nxt_port_recv_test_quit(nxt_unit_ctx_t *ctx) +{ + nxt_port_recv_test_quit_calls++; + + if (nxt_port_recv_test_quit_done) { + nxt_unit_done(ctx); + } +} + + /* * Captures the read port queue of the context being announced to the * router. libunit closes that descriptor as soon as the announcement is @@ -104,7 +126,7 @@ nxt_port_recv_test_send(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, for (cmsg = CMSG_FIRSTHDR(&msg); cmsg != NULL; - cmsg = CMSG_NXTHDR(&msg, cmsg)) + cmsg = NXT_CMSG_NXTHDR(&msg, cmsg)) { size = cmsg->cmsg_len - CMSG_LEN(0); @@ -135,6 +157,13 @@ nxt_port_recv_test_recv(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, void *buf, int fds[2]; struct cmsghdr *cmsg; + if (nxt_port_recv_test_block) { + /* A real blocking read, as libunit does without this callback. */ + *oob_size = 0; + + return read(port->in_fd, buf, buf_size); + } + if (nxt_port_recv_test_step++ > 0) { return 0; } @@ -248,6 +277,931 @@ nxt_port_recv_test_notify(void) } +#if (NXT_TESTS) +static void +nxt_port_recv_test_detached_single_fail(nxt_unit_ctx_t *ctx) +{ + int rc; + + /* + * Case A: FINISH send fails once, next retry in read loop succeeds, + * detached flag clears to 0, and worker keeps serving. + */ + nxt_unit_test_ctx_set_ready(ctx, 1); + nxt_unit_test_send_detached_failures(0); + nxt_unit_test_ctx_set_detached(ctx, 1); + + nxt_port_recv_test_assert( + nxt_unit_test_ctx_detached(ctx) == 1 + && nxt_unit_test_ctx_detached_retries(ctx) == 0, + "detached initial state is running"); + + /* Inject single FINISH send failure. */ + nxt_unit_test_send_detached_failures(1); + + /* Simulate request handler return calling nxt_unit_ctx_detached_done. */ + nxt_unit_test_ctx_detached_done(ctx); + + nxt_port_recv_test_assert( + nxt_unit_test_ctx_detached(ctx) == 1 + && nxt_unit_test_ctx_detached_retries(ctx) == 1, + "failed finish send schedules retry"); + + nxt_port_recv_test_assert( + nxt_unit_test_ctx_online(ctx) == 1 + && nxt_unit_test_ctx_ready(ctx) == 1, + "worker remains online and ready during retry"); + + /* Simulate read loop retry. Subsequent send succeeds. */ + rc = nxt_unit_test_ctx_detached_retry(ctx); + + nxt_port_recv_test_assert( + rc == NXT_UNIT_OK + && nxt_unit_test_ctx_detached(ctx) == 0 + && nxt_unit_test_ctx_detached_retries(ctx) == 0 + && nxt_unit_test_ctx_online(ctx) == 1, + "retry settles detached = 0 and keeps serving"); + + /* + * Case B: Graceful QUIT arrives while detached work is running. + * Quit is held off because detached == 1, and only honoured once + * the retry succeeds. + */ + nxt_unit_test_ctx_set_ready(ctx, 1); + nxt_unit_test_send_detached_failures(0); + nxt_unit_test_ctx_set_detached(ctx, 1); + + /* Graceful quit arrives while worker is detached. */ + nxt_unit_test_ctx_quit_graceful(ctx); + + nxt_port_recv_test_assert( + nxt_unit_test_ctx_ready(ctx) == 0 + && nxt_unit_test_ctx_online(ctx) == 1, + "graceful quit deferred while detached is running"); + + /* Inject single FINISH send failure on request handler return. */ + nxt_unit_test_send_detached_failures(1); + nxt_unit_test_ctx_detached_done(ctx); + + nxt_port_recv_test_assert( + nxt_unit_test_ctx_detached(ctx) == 1 + && nxt_unit_test_ctx_detached_retries(ctx) == 1 + && nxt_unit_test_ctx_online(ctx) == 1, + "graceful quit remains deferred while finish retry is pending"); + + /* + * Retry succeeds: detached clears and the deferred graceful quit is + * honoured. + */ + rc = nxt_unit_test_ctx_detached_retry(ctx); + + nxt_port_recv_test_assert( + rc == NXT_UNIT_OK + && nxt_unit_test_ctx_detached(ctx) == 0 + && nxt_unit_test_ctx_detached_retries(ctx) == 0 + && nxt_unit_test_ctx_online(ctx) == 0, + "retry settles detached = 0 and honours deferred graceful quit"); +} + + +static void +nxt_port_recv_test_detached_retry_budget(nxt_unit_ctx_t *ctx) +{ + int rc; + + /* + * A request handler that returns while the FINISH edge is pending sends + * it again, but must not reset the retry budget. Otherwise a worker + * that serves traffic never reaches the give-up. + * + * Three injected failures: the handler return, one read-loop retry, and + * a second handler return that must leave the count at 2, not 1. The + * fourth attempt succeeds and clears both. + */ + nxt_unit_test_ctx_set_ready(ctx, 1); + nxt_unit_test_ctx_set_detached(ctx, 1); + nxt_unit_test_ctx_set_detached_retries(ctx, 0); + nxt_unit_test_send_detached_failures(3); + + nxt_unit_test_ctx_detached_done(ctx); + + nxt_port_recv_test_assert( + nxt_unit_test_ctx_detached(ctx) == 1 + && nxt_unit_test_ctx_detached_retries(ctx) == 1, + "failed finish send arms the retry budget"); + + rc = nxt_unit_test_ctx_detached_retry(ctx); + + nxt_port_recv_test_assert( + rc == NXT_UNIT_OK + && nxt_unit_test_ctx_detached_retries(ctx) == 2, + "read loop spends one retry"); + + /* A request handler returns while the FINISH is still pending. */ + nxt_unit_test_ctx_detached_done(ctx); + + nxt_port_recv_test_assert( + nxt_unit_test_ctx_detached(ctx) == 1 + && nxt_unit_test_ctx_detached_retries(ctx) == 2, + "handler return does not reset the retry budget"); + + rc = nxt_unit_test_ctx_detached_retry(ctx); + + nxt_port_recv_test_assert( + rc == NXT_UNIT_OK + && nxt_unit_test_ctx_detached(ctx) == 0 + && nxt_unit_test_ctx_detached_retries(ctx) == 0 + && nxt_unit_test_ctx_online(ctx) == 1, + "the next successful retry clears the state"); +} + + +static void +nxt_port_recv_test_child_wait(pid_t pid, const char *name) +{ + int status; + + if (pid == -1) { + perror("fork"); + nxt_port_recv_test_failures++; + return; + } + + if (waitpid(pid, &status, 0) == -1) { + perror("waitpid"); + nxt_port_recv_test_failures++; + return; + } + + if (!WIFEXITED(status)) { + printf("port_recv test: child killed by signal %d\n", + WIFSIGNALED(status) ? WTERMSIG(status) : -1); + + } else if (WEXITSTATUS(status) != 0) { + printf("port_recv test: child exit %d\n", WEXITSTATUS(status)); + } + + nxt_port_recv_test_assert(WIFEXITED(status) && WEXITSTATUS(status) == 0, + name); +} + + +static void +nxt_port_recv_test_detached_start_give_up(nxt_unit_ctx_t *ctx) +{ + pid_t pid; + + /* + * The START edge is retried inside the request handler, because the read + * loop does not run until the handler returns. In a child process: the + * give-up quits the worker. + */ + pid = fork(); + + if (pid == 0) { + nxt_unit_test_ctx_set_ready(ctx, 1); + nxt_unit_test_ctx_set_detached(ctx, 0); + + /* Seven failures: the eighth and last attempt succeeds. */ + nxt_unit_test_send_detached_failures(7); + nxt_unit_test_ctx_detached_start(ctx); + + if (nxt_unit_test_ctx_detached(ctx) != 1 + || nxt_unit_test_ctx_detached_unreported(ctx) != 0) + { + _exit(1); + } + + nxt_unit_test_ctx_set_detached(ctx, 0); + + /* Eight failures: the worker is not marked detached. */ + nxt_unit_test_send_detached_failures(8); + nxt_unit_test_ctx_detached_start(ctx); + + if (nxt_unit_test_ctx_detached(ctx) != 0 + || nxt_unit_test_ctx_detached_unreported(ctx) != 1 + || nxt_unit_test_ctx_online(ctx) != 1) + { + _exit(2); + } + + /* The handler returns: the worker retires, without a FINISH. */ + nxt_unit_test_send_detached_failures(1); + nxt_unit_test_ctx_detached_done(ctx); + + if (nxt_unit_test_ctx_online(ctx) != 0 + || nxt_unit_test_ctx_detached_unreported(ctx) != 0) + { + _exit(3); + } + + _exit(0); + } + + nxt_port_recv_test_child_wait(pid, + "failed start send is retried in place, then retires the worker"); +} + + +static void +nxt_port_recv_test_detached_idle_retry(nxt_unit_ctx_t *ctx, uint8_t ready, + const char *name) +{ + pid_t pid; + nxt_app_queue_t *q; + + /* + * A FINISH retry must not wait for traffic. The read loop runs on an + * idle worker: its waits must expire, so the retries reach the give-up. + * Reads block, so an unbounded wait lasts until SIGALRM kills the child. + * A ready context waits in poll() on both ports; a context that is not + * ready waits on its read port only. + */ + pid = fork(); + + if (pid == 0) { + nxt_port_recv_test_block = 1; + + /* An empty shared queue, as the router leaves it. */ + q = mmap(NULL, sizeof(nxt_app_queue_t), PROT_READ | PROT_WRITE, + MAP_SHARED, nxt_port_recv_test_shared_queue_fd, 0); + if (q == MAP_FAILED) { + _exit(4); + } + + nxt_app_queue_init(q); + + nxt_unit_test_ctx_set_ready(ctx, ready); + nxt_unit_test_ctx_set_detached(ctx, 1); + nxt_unit_test_ctx_set_detached_retries(ctx, 0); + nxt_unit_test_send_detached_failures(20); + + nxt_unit_test_ctx_detached_done(ctx); + + if (nxt_unit_test_ctx_detached_retries(ctx) != 1) { + _exit(1); + } + + alarm(5); + + (void) nxt_unit_run(ctx); + + if (nxt_unit_test_ctx_online(ctx) != 0 + || nxt_unit_test_ctx_detached_retries(ctx) != 11) + { + _exit(2); + } + + _exit(0); + } + + nxt_port_recv_test_child_wait(pid, name); +} + + +static void +nxt_port_recv_test_deferred_quit_read(nxt_unit_ctx_t *ctx, int run_ctx, + const char *name) +{ + int rc; + pid_t pid; + + /* + * A graceful QUIT that arrives while the worker is detached is deferred. + * The retry that delivers the FINISH edge then completes it, which takes + * the context offline and removes its read port. The read loop must + * return at once: a receive on the removed port waits for a message the + * router will never send, so an unbounded wait lasts until SIGALRM kills + * the child. Both read loops run the retry, so check both. + */ + + pid = fork(); + + if (pid == 0) { + nxt_port_recv_test_block = 1; + + nxt_unit_test_ctx_set_ready(ctx, 1); + nxt_unit_test_ctx_set_detached(ctx, 1); + nxt_unit_test_ctx_set_detached_retries(ctx, 0); + + nxt_unit_test_ctx_quit_graceful(ctx); + + if (nxt_unit_test_ctx_ready(ctx) != 0 + || nxt_unit_test_ctx_online(ctx) != 1) + { + _exit(1); + } + + /* One failure: the handler return arms the retry, the retry sends. */ + nxt_unit_test_send_detached_failures(1); + nxt_unit_test_ctx_detached_done(ctx); + + if (nxt_unit_test_ctx_detached(ctx) != 1 + || nxt_unit_test_ctx_detached_retries(ctx) != 1) + { + _exit(2); + } + + alarm(5); + + rc = run_ctx ? nxt_unit_run_ctx(ctx) : nxt_unit_run(ctx); + + if (rc != NXT_UNIT_OK + || nxt_unit_test_ctx_online(ctx) != 0 + || nxt_unit_test_ctx_detached(ctx) != 0 + || nxt_unit_test_ctx_detached_retries(ctx) != 0) + { + _exit(3); + } + + _exit(0); + } + + nxt_port_recv_test_child_wait(pid, name); +} + + +static void +nxt_port_recv_test_deferred_quit_port_msg(nxt_unit_ctx_t *ctx, + const char *name) +{ + int rc; + pid_t pid; + + /* + * An embedder that drives its own event loop never enters a libunit + * read loop. So nxt_unit_process_port_msg() is the only wake-up that + * can run the retry. Without it, the FINISH edge is never sent again, + * and the router holds the worker detached. Here the retry completes + * a deferred graceful quit, which removes the read port. So the call + * must also report "no message" and must not receive. The alarm + * catches a wait for a message that the router will never send. + */ + + pid = fork(); + + if (pid == 0) { + nxt_port_recv_test_block = 1; + + nxt_unit_test_ctx_set_ready(ctx, 1); + nxt_unit_test_ctx_set_detached(ctx, 1); + nxt_unit_test_ctx_set_detached_retries(ctx, 0); + + nxt_unit_test_ctx_quit_graceful(ctx); + + /* One failure: the handler return arms the retry, the retry sends. */ + nxt_unit_test_send_detached_failures(1); + nxt_unit_test_ctx_detached_done(ctx); + + if (nxt_unit_test_ctx_detached(ctx) != 1 + || nxt_unit_test_ctx_detached_retries(ctx) != 1) + { + _exit(1); + } + + alarm(5); + + rc = nxt_unit_process_port_msg(ctx, + nxt_unit_test_ctx_read_port(ctx)); + + if (rc != NXT_UNIT_AGAIN + || nxt_unit_test_ctx_online(ctx) != 0 + || nxt_unit_test_ctx_detached(ctx) != 0 + || nxt_unit_test_ctx_detached_retries(ctx) != 0) + { + _exit(2); + } + + _exit(0); + } + + nxt_port_recv_test_child_wait(pid, name); +} + + +/* + * The clock of nxt_unit_detached_now(), in milliseconds, through a test + * hook. The retries are paced with a deadline on that clock, so the + * test measures them with the same clock. + */ + +static long +nxt_port_recv_test_ms(uint64_t from, uint64_t to) +{ + return (long) (to - from); +} + + +static void +nxt_port_recv_test_port_msg_retry_pending(nxt_unit_ctx_t *ctx, + const char *name) +{ + int rc; + long call_ms, max_ms; + pid_t pid; + uint64_t end, start, t0, t1; + unsigned int calls; + + /* + * A retry that fails again leaves the context online with the FINISH + * still pending. Nothing else wakes the embedder: its read port stays + * quiet until unrelated traffic arrives. The call must report + * NXT_UNIT_OK, because an embedder calls again on that code. It must + * not report NXT_UNIT_AGAIN, which stops the embedder and leaves the + * worker detached. The call must not wait for the backoff either, + * because that would freeze the embedder's event loop. Each call + * returns at once, and a deadline paces the retries. Reads block, so + * the alarm catches a wait for a message. The slowest call catches a + * bounded wait. The elapsed time catches retries that spin through the + * budget instead of pacing it. + */ + + pid = fork(); + + if (pid == 0) { + nxt_port_recv_test_block = 1; + + nxt_unit_test_ctx_set_ready(ctx, 1); + nxt_unit_test_ctx_set_detached(ctx, 1); + nxt_unit_test_ctx_set_detached_retries(ctx, 0); + nxt_unit_test_send_detached_failures(20); + + nxt_unit_test_ctx_detached_done(ctx); + + if (nxt_unit_test_ctx_detached_retries(ctx) != 1) { + _exit(1); + } + + alarm(10); + + max_ms = 0; + + start = nxt_unit_test_detached_now(); + + for (calls = 1; calls <= 10000000; calls++) { + t0 = nxt_unit_test_detached_now(); + + rc = nxt_unit_process_port_msg(ctx, + nxt_unit_test_ctx_read_port(ctx)); + + t1 = nxt_unit_test_detached_now(); + + call_ms = nxt_port_recv_test_ms(t0, t1); + max_ms = nxt_max(max_ms, call_ms); + + if (rc != NXT_UNIT_OK) { + break; + } + } + + end = nxt_unit_test_detached_now(); + + /* + * Each call reports "call me again" until the tenth retry reaches + * the give-up. The give-up closes the worker instead of leaving it + * detached for good. + */ + + if (rc != NXT_UNIT_ERROR || calls <= 10) { + _exit(2); + } + + if (nxt_unit_test_ctx_online(ctx) != 0 + || nxt_unit_test_ctx_detached_retries(ctx) != 11) + { + _exit(3); + } + + if (nxt_port_recv_test_ms(start, end) < 100) { + _exit(4); + } + + /* The last backoff steps are 256 ms. A wait that long is a block. */ + + if (max_ms >= 50) { + _exit(5); + } + + _exit(0); + } + + nxt_port_recv_test_child_wait(pid, name); +} + + +static void +nxt_port_recv_test_ms_sleep(int ms) +{ + struct timespec ts; + + ts.tv_sec = ms / 1000; + ts.tv_nsec = (ms % 1000) * 1000000L; + + (void) nanosleep(&ts, NULL); +} + + +static void +nxt_port_recv_test_port_msg_retry_timeout(nxt_unit_ctx_t *ctx, + const char *name) +{ + int rc, timeout; + pid_t pid; + uint64_t end, start; + unsigned int calls; + + /* + * An embedder calls again on NXT_UNIT_OK. If it calls at once each + * time, its loop spins through the backoff until the deadline. So + * nxt_unit_detached_retry_timeout() gives the delay to the deadline. + * An embedder that waits for that delay must still reach each retry + * and the give-up, with few calls. The alarm catches a wait for a + * message. + */ + + pid = fork(); + + if (pid == 0) { + nxt_port_recv_test_block = 1; + + nxt_unit_test_ctx_set_ready(ctx, 1); + nxt_unit_test_ctx_set_detached(ctx, 1); + nxt_unit_test_ctx_set_detached_retries(ctx, 0); + nxt_unit_test_send_detached_failures(20); + + /* No retry is pending. */ + + if (nxt_unit_detached_retry_timeout(ctx) != -1) { + _exit(1); + } + + nxt_unit_test_ctx_detached_done(ctx); + + /* The first retry is due at once. */ + + if (nxt_unit_test_ctx_detached_retries(ctx) != 1 + || nxt_unit_detached_retry_timeout(ctx) != 0) + { + _exit(2); + } + + alarm(10); + + /* The retry fails, and the call finds no message. */ + + rc = nxt_unit_process_port_msg(ctx, + nxt_unit_test_ctx_read_port(ctx)); + + timeout = nxt_unit_detached_retry_timeout(ctx); + + if (rc != NXT_UNIT_OK || timeout <= 0 || timeout > 256) { + _exit(3); + } + + /* After the deadline, the next retry is due at once. */ + + nxt_port_recv_test_ms_sleep(timeout + 1); + + if (nxt_unit_detached_retry_timeout(ctx) != 0) { + _exit(4); + } + + start = nxt_unit_test_detached_now(); + + for (calls = 1; calls <= 10000000; calls++) { + rc = nxt_unit_process_port_msg(ctx, + nxt_unit_test_ctx_read_port(ctx)); + if (rc != NXT_UNIT_OK) { + break; + } + + timeout = nxt_unit_detached_retry_timeout(ctx); + + if (timeout > 0) { + nxt_port_recv_test_ms_sleep(timeout); + } + } + + end = nxt_unit_test_detached_now(); + + if (rc != NXT_UNIT_ERROR || nxt_unit_test_ctx_online(ctx) != 0) { + _exit(5); + } + + /* The give-up leaves nothing pending. */ + + if (nxt_unit_detached_retry_timeout(ctx) != -1) { + _exit(6); + } + + /* The embedder waited: few calls, and the backoff still paced. */ + + if (calls > 30 || nxt_port_recv_test_ms(start, end) < 500) { + _exit(7); + } + + _exit(0); + } + + nxt_port_recv_test_child_wait(pid, name); +} + + +static void +nxt_port_recv_test_shared_quit_retry(nxt_unit_ctx_t *ctx, unsigned int fails, + int expect, const char *name) +{ + int rc; + pid_t pid; + uint64_t end, start; + + /* + * The shared loop ends when nxt_unit_chk_ready() turns false. That + * can happen with a finish retry pending: a graceful quit deferred on + * the detached state clears ->ready, and the request that ended the + * detached work can be the one that reached "request_limit". The loop + * must run the retry before it leaves. If not, the router holds the + * worker detached, and the deferred quit never completes. Nothing is + * ready, so no read reaches the shared port. The alarm catches a wait + * anyway. + */ + + pid = fork(); + + if (pid == 0) { + nxt_unit_test_ctx_set_ready(ctx, 1); + nxt_unit_test_ctx_set_detached(ctx, 1); + nxt_unit_test_ctx_set_detached_retries(ctx, 0); + + nxt_unit_test_ctx_quit_graceful(ctx); + + if (nxt_unit_test_ctx_ready(ctx) != 0 + || nxt_unit_test_ctx_online(ctx) != 1) + { + _exit(1); + } + + nxt_unit_test_send_detached_failures(fails); + nxt_unit_test_ctx_detached_done(ctx); + + if (nxt_unit_test_ctx_detached_retries(ctx) != 1) { + _exit(2); + } + + alarm(5); + + start = nxt_unit_test_detached_now(); + + rc = nxt_unit_run_shared(ctx); + + end = nxt_unit_test_detached_now(); + + if (rc != expect || nxt_unit_test_ctx_online(ctx) != 0) { + _exit(3); + } + + /* + * A retry that succeeds clears the state and completes the quit. + * A retry that never succeeds reaches the give-up. The give-up + * closes the worker instead of leaving it detached. + */ + + if (expect == NXT_UNIT_OK) { + if (nxt_unit_test_ctx_detached(ctx) != 0 + || nxt_unit_test_ctx_detached_retries(ctx) != 0) + { + _exit(4); + } + + } else if (nxt_unit_test_ctx_detached_retries(ctx) != 11) { + _exit(5); + + } else if (nxt_port_recv_test_ms(start, end) < 100) { + /* + * The retries use the same backoff as the read loops, about + * 0.77 s in all. A much shorter time means that the loop + * spins and burns the whole budget at once. + */ + + _exit(6); + } + + _exit(0); + } + + nxt_port_recv_test_child_wait(pid, name); +} + + +static void +nxt_port_recv_test_socket_quit(nxt_unit_ctx_t *ctx, uint8_t ready, + const char *name) +{ + pid_t pid; + ssize_t n; + nxt_app_queue_t *q; + + struct { + nxt_port_msg_t msg; + uint8_t quit_param; + } nxt_packed m; + + /* + * The prototype writes a QUIT to the bare socket of a worker that has + * not sent PROCESS_READY. No READ_SOCKET mark is queued for that QUIT. + * The worker must act on it. A suspended QUIT waits for a mark that + * never comes. The child then waits until SIGALRM stops it. + */ + pid = fork(); + + if (pid == 0) { + nxt_port_recv_test_block = 1; + + /* An empty shared queue, as the router leaves it. */ + q = mmap(NULL, sizeof(nxt_app_queue_t), PROT_READ | PROT_WRITE, + MAP_SHARED, nxt_port_recv_test_shared_queue_fd, 0); + if (q == MAP_FAILED) { + _exit(4); + } + + nxt_app_queue_init(q); + + nxt_unit_test_ctx_set_ready(ctx, ready); + nxt_unit_test_ctx_set_detached(ctx, 0); + nxt_unit_test_ctx_set_detached_retries(ctx, 0); + + /* The message nxt_runtime_port_send_quit() writes. */ + memset(&m, 0, sizeof(m)); + + m.msg.pid = getppid(); + m.msg.type = _NXT_PORT_MSG_QUIT; + m.quit_param = NXT_PORT_QUIT_GRACEFUL; + + n = write(nxt_port_recv_test_read_out_fd, &m, sizeof(m)); + if (n != (ssize_t) sizeof(m)) { + _exit(1); + } + + alarm(5); + + if (nxt_unit_run(ctx) != NXT_UNIT_OK) { + _exit(2); + } + + if (nxt_unit_test_ctx_online(ctx) != 0) { + _exit(3); + } + + _exit(0); + } + + nxt_port_recv_test_child_wait(pid, name); +} + + +static void +nxt_port_recv_test_port_msg_quit(nxt_unit_ctx_t *ctx, const char *name) +{ + int rc; + pid_t pid; + ssize_t n; + nxt_unit_port_t *port; + + struct { + nxt_port_msg_t msg; + uint8_t quit_param; + } nxt_packed m; + + /* + * A QUIT that nxt_unit_process_port_msg() processes runs the quit + * callback. The callback of Node.js calls nxt_unit_done(). Then the + * reference of the call is the last one, and the return frees the + * context. A caller that reads ctx after that, for example with + * nxt_unit_detached_retry_timeout(), touches freed memory. So the call + * must return NXT_UNIT_AGAIN, "do not call again", and the caller must + * not use ctx or port after that. The sanitizer catches a use after + * the free. The alarm catches a wait for a message. + */ + + pid = fork(); + + if (pid == 0) { + nxt_port_recv_test_block = 1; + nxt_port_recv_test_quit_done = 1; + nxt_port_recv_test_quit_calls = 0; + + nxt_unit_test_ctx_set_ready(ctx, 1); + nxt_unit_test_ctx_set_detached(ctx, 0); + nxt_unit_test_ctx_set_detached_retries(ctx, 0); + + port = nxt_unit_test_ctx_read_port(ctx); + + memset(&m, 0, sizeof(m)); + + m.msg.pid = getppid(); + m.msg.type = _NXT_PORT_MSG_QUIT; + m.quit_param = NXT_PORT_QUIT_GRACEFUL; + + n = write(nxt_port_recv_test_read_out_fd, &m, sizeof(m)); + if (n != (ssize_t) sizeof(m)) { + _exit(1); + } + + alarm(5); + + rc = nxt_unit_process_port_msg(ctx, port); + + /* ctx and port are freed here. Only rc and the counter are safe. */ + + if (nxt_port_recv_test_quit_calls != 1) { + _exit(2); + } + + if (rc != NXT_UNIT_AGAIN) { + _exit(3); + } + + _exit(0); + } + + nxt_port_recv_test_child_wait(pid, name); +} + + +static void +nxt_port_recv_test_detached_persistent_fail(nxt_unit_ctx_t *ctx) +{ + int i, rc, status; + pid_t pid; + + /* + * Persistent send failures (>10 retries): + * Test in a child process so that the worker teardown (via + * nxt_unit_quit(..., NXT_QUIT_NORMAL)) does not affect the main test. + */ + pid = fork(); + if (pid == -1) { + perror("fork"); + nxt_port_recv_test_failures++; + return; + } + + if (pid == 0) { + nxt_unit_test_ctx_set_detached(ctx, 1); + + /* Inject persistent failures for all attempts. */ + nxt_unit_test_send_detached_failures(20); + + /* Request handler return fails to send FINISH. */ + nxt_unit_test_ctx_detached_done(ctx); + + if (nxt_unit_test_ctx_detached(ctx) != 1 + || nxt_unit_test_ctx_detached_retries(ctx) != 1) + { + _exit(1); + } + + /* Retries 1..9 (attempts 2..10) return OK and increment retries. */ + for (i = 1; i <= 9; i++) { + rc = nxt_unit_test_ctx_detached_retry(ctx); + if (rc != NXT_UNIT_OK + || nxt_unit_test_ctx_detached(ctx) != 1 + || nxt_unit_test_ctx_detached_retries(ctx) != (i + 1) + || nxt_unit_test_ctx_online(ctx) != 1) + { + _exit(2); + } + } + + /* + * Retry 10 (attempt 11): retries exceed 10. + * Give-up path executes outside request processing: + * worker is closed (online == 0) and returns NXT_UNIT_ERROR. + */ + rc = nxt_unit_test_ctx_detached_retry(ctx); + if (rc != NXT_UNIT_ERROR + || nxt_unit_test_ctx_online(ctx) != 0 + || nxt_unit_test_ctx_detached_retries(ctx) != 11) + { + _exit(3); + } + + _exit(0); + } + + if (waitpid(pid, &status, 0) == -1) { + perror("waitpid"); + nxt_port_recv_test_failures++; + return; + } + + nxt_port_recv_test_assert( + WIFEXITED(status) && WEXITSTATUS(status) == 0, + "persistent send failures trigger give-up teardown outside request"); +} +#endif + + int main(void) { @@ -276,6 +1230,7 @@ main(void) } nxt_port_recv_test_sock_fd = sock[0]; + nxt_port_recv_test_read_out_fd = read[1]; nxt_port_recv_test_queue_fd = nxt_port_recv_test_shm(sizeof(nxt_port_queue_t)); @@ -289,6 +1244,7 @@ main(void) init.callbacks.request_handler = nxt_port_recv_test_handler; init.callbacks.port_send = nxt_port_recv_test_send; init.callbacks.port_recv = nxt_port_recv_test_recv; + init.callbacks.quit = nxt_port_recv_test_quit; init.ready_port.id.pid = getpid(); init.ready_port.id.id = 1; @@ -315,6 +1271,8 @@ main(void) return 1; } + nxt_port_recv_test_shared_queue_fd = dup(init.shared_queue_fd); + ctx = nxt_unit_init(&init); if (ctx == NULL) { printf("port_recv test: nxt_unit_init() failed\n"); @@ -378,6 +1336,49 @@ main(void) fcntl(unrelated, F_GETFD) != -1, "unrelated fd survives an empty read"); +#if (NXT_TESTS) + /* + * single_fail runs last: its Case B leaves the context offline, and + * every other case expects it online. + */ + nxt_port_recv_test_detached_retry_budget(ctx); + nxt_port_recv_test_detached_start_give_up(ctx); + nxt_port_recv_test_detached_idle_retry(ctx, 1, + "finish retry reaches the give-up, ready and idle"); + nxt_port_recv_test_detached_idle_retry(ctx, 0, + "finish retry reaches the give-up, not ready and idle"); + nxt_port_recv_test_detached_persistent_fail(ctx); + nxt_port_recv_test_socket_quit(ctx, 0, + "bare socket quit stops a context not ready"); + nxt_port_recv_test_socket_quit(ctx, 1, + "bare socket quit stops a ready context"); + nxt_port_recv_test_port_msg_quit(ctx, + "nxt_unit_process_port_msg() stops the caller on a quit that frees"); + nxt_port_recv_test_deferred_quit_port_msg(ctx, + "nxt_unit_process_port_msg() runs a pending finish retry"); + nxt_port_recv_test_port_msg_retry_pending(ctx, + "nxt_unit_process_port_msg() asks for a call back, no wait"); + nxt_port_recv_test_port_msg_retry_timeout(ctx, + "nxt_unit_detached_retry_timeout() paces the call back"); + nxt_port_recv_test_shared_quit_retry(ctx, 1, NXT_UNIT_OK, + "nxt_unit_run_shared() runs a pending finish retry"); + nxt_port_recv_test_shared_quit_retry(ctx, 20, NXT_UNIT_ERROR, + "nxt_unit_run_shared() reaches the give-up, not ready"); + nxt_port_recv_test_deferred_quit_read(ctx, 0, + "nxt_unit_run() returns on a retry that completes a quit"); + nxt_port_recv_test_deferred_quit_read(ctx, 1, + "nxt_unit_run_ctx() returns on a retry that completes a quit"); + nxt_port_recv_test_detached_single_fail(ctx); +#endif + + /* + * Release the library. After main() returns, no root keeps it + * reachable, and LeakSanitizer reports it or not depending on stale + * stack contents. + */ + nxt_unit_done(ctx2); + nxt_unit_done(ctx); + if (nxt_port_recv_test_failures != 0) { printf("port_recv test: %d failure(s)\n", nxt_port_recv_test_failures); diff --git a/src/test/nxt_utf8_file_name_test.c b/src/test/nxt_utf8_file_name_test.c deleted file mode 100644 index b77cd4ab0..000000000 --- a/src/test/nxt_utf8_file_name_test.c +++ /dev/null @@ -1,148 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) NGINX, Inc. - */ - -#include - - -extern char **environ; - -static nxt_int_t nxt_utf8_file_name_test(nxt_thread_t *thr); - - -nxt_module_init_t nxt_init_modules[1]; -nxt_uint_t nxt_init_modules_n; - - -int nxt_cdecl -main(int argc, char **argv) -{ - nxt_thread_t *thr; - - if (nxt_lib_start("utf8_file_name_test", argv, &environ) != NXT_OK) { - return 1; - } - - nxt_main_log.level = NXT_LOG_INFO; - - thr = nxt_thread(); - - if (nxt_utf8_file_name_test(thr) != NXT_OK) { - return 1; - } - - return 0; -} - - -static nxt_int_t -nxt_utf8_file_name_test(nxt_thread_t *thr) -{ - u_char *p, test[4], buf[32]; - ssize_t n; - uint32_t uc, lc; - nxt_int_t ret; - nxt_task_t task; - nxt_file_t uc_file, lc_file; - const u_char *pp; - nxt_file_name_t uc_name[10], lc_name[10]; - static const u_char utf8[4] NXT_NONSTRING = "UTF8"; - - nxt_thread_time_update(thr); - - uc_name[0] = 'u'; - uc_name[1] = 't'; - uc_name[2] = 'f'; - uc_name[3] = '8'; - uc_name[4] = '_'; - - lc_name[0] = 'u'; - lc_name[1] = 't'; - lc_name[2] = 'f'; - lc_name[3] = '8'; - lc_name[4] = '_'; - - nxt_memzero(&uc_file, sizeof(nxt_file_t)); - - uc_file.name = uc_name; - uc_file.log_level = NXT_LOG_ALERT; - - nxt_memzero(&lc_file, sizeof(nxt_file_t)); - - lc_file.name = lc_name; - - task.thread = thr; - task.log = thr->log; - - for (uc = 0x41; uc < 0x110000; uc++) { - - p = nxt_utf8_encode(&uc_name[5], uc); - - if (p == NULL) { - nxt_log_alert(thr->log, "nxt_utf8_encode(%05uxD) failed", uc); - return NXT_ERROR; - } - - *p = '\0'; - - pp = &uc_name[5]; - lc = nxt_utf8_lowcase(&pp, p); - - if (lc == 0xFFFFFFFF) { - nxt_log_alert(thr->log, "nxt_utf8_lowcase(%05uxD) failed: %05uxD", - uc, lc); - return NXT_ERROR; - } - - if (uc == lc) { - continue; - } - - p = nxt_utf8_encode(&lc_name[5], lc); - - if (p == NULL) { - nxt_log_alert(thr->log, "nxt_utf8_encode(%05uxD) failed", lc); - return NXT_ERROR; - } - - *p = '\0'; - - ret = nxt_file_open(&task, &uc_file, NXT_FILE_WRONLY, NXT_FILE_TRUNCATE, - NXT_FILE_DEFAULT_ACCESS); - if (ret != NXT_OK) { - return NXT_ERROR; - } - - if (nxt_file_write(&uc_file, utf8, 4, 0) != 4) { - return NXT_ERROR; - } - - nxt_file_close(&task, &uc_file); - - ret = nxt_file_open(&task, &lc_file, NXT_FILE_RDONLY, NXT_FILE_OPEN, - NXT_FILE_DEFAULT_ACCESS); - - if (ret == NXT_OK) { - n = nxt_file_read(&lc_file, test, 4, 0); - - nxt_file_close(&task, &lc_file); - - if (n != 4 || memcmp(utf8, test, 4) != 0) { - nxt_log_alert(thr->log, "nxt_file_read() mismatch"); - - nxt_file_delete(lc_file.name); - } - - p = nxt_sprintf(buf, buf + 32, "%04uXD; C; %04uXD;%n", uc, lc); - - nxt_fd_write(nxt_stdout, buf, p - buf); - } - - nxt_file_delete(uc_file.name); - } - - nxt_log_error(NXT_LOG_NOTICE, thr->log, "utf8 file name test passed"); - return NXT_OK; -} diff --git a/src/test/nxt_utf8_sanitize_test.c b/src/test/nxt_utf8_sanitize_test.c new file mode 100644 index 000000000..285b40fe2 --- /dev/null +++ b/src/test/nxt_utf8_sanitize_test.c @@ -0,0 +1,221 @@ + +/* + * Copyright (C) NGINX, Inc. + */ + +#include +#include "nxt_tests.h" + + +/* + * nxt_utf8_sanitize() is what keeps a JSON access log record readable when + * the value came from the request: a byte that begins no valid sequence has + * no JSON representation at all (RFC 8259 Sect. 8.1 makes JSON text UTF-8), + * so it is replaced with U+FFFD rather than escaped or copied. + * + * The strings below are the ones where a sizing pass and a copying pass can + * drift apart -- bytes kept, bytes replaced, and sequences that decode to + * nothing -- because the function walks the input twice and the second walk + * fills a buffer the first one sized. + */ + +/* + * A strict UTF-8 test written from the definition rather than from + * src/nxt_utf8.c, which is the decoder the escaper itself uses. Asserting + * with that decoder would only prove the escaper agrees with itself: its + * final gate is "overlong < u && u < 0x110000", so it accepts the surrogate + * range, and a surrogate copied into the output would satisfy an assertion + * built on it while a real consumer rejected the record. Unicode 15.0 + * Sect. 3.9, table 3-7. + */ + +static nxt_bool_t +nxt_utf8_sanitize_test_utf8(const u_char *p, size_t len) +{ + uint32_t u, min; + nxt_uint_t n; + const u_char *end; + + end = p + len; + + while (p < end) { + + if (*p < 0x80) { + p++; + continue; + } + + if (*p >= 0xF0) { + if (*p > 0xF4) { + return 0; + } + + u = *p & 0x07; + n = 3; + min = 0x10000; + + } else if (*p >= 0xE0) { + u = *p & 0x0F; + n = 2; + min = 0x800; + + } else if (*p >= 0xC2) { + u = *p & 0x1F; + n = 1; + min = 0x80; + + } else { + /* A continuation byte on its own, or an overlong C0/C1 lead. */ + return 0; + } + + if (p + n >= end) { + return 0; + } + + p++; + + while (n != 0) { + if ((*p & 0xC0) != 0x80) { + return 0; + } + + u = (u << 6) | (*p & 0x3F); + p++; + n--; + } + + /* Shortest form, in range, and not a surrogate. */ + + if (u < min || u > 0x10FFFF || (u >= 0xD800 && u <= 0xDFFF)) { + return 0; + } + } + + return 1; +} + + +static const struct { + const char *name; + const char *data; + size_t length; +} nxt_utf8_sanitize_cases[] = { + { "plain", "hello", 5 }, + { "quote", "a\"b", 3 }, + { "backslash", "a\\b", 3 }, + { "named control", "\n\r\t\b\f", 5 }, + { "numeric control", "\x01\x02\x1F", 3 }, + { "nul", "\x00", 1 }, + { "two-byte utf8", "\xC3\xA9", 2 }, + { "three-byte utf8", "\xE2\x82\xAC", 3 }, + { "four-byte utf8", "\xF0\x9F\x92\xA9", 4 }, + { "lone 0xFF", "\xFF", 1 }, + { "lone continuation", "\x80", 1 }, + { "truncated two-byte", "\xC3", 1 }, + { "truncated four-byte","\xF0\x9F", 2 }, + { "overlong", "\xC0\xAF", 2 }, + { "surrogate", "\xED\xA0\x80", 3 }, + { "above 0x10FFFF", "\xF5\x80\x80\x80", 4 }, + { "invalid then quote", "\xFF\"", 2 }, + { "invalid run", "\xFF\xFE\xFD", 3 }, + { "mixed", "a\xC3\xA9\xFF\n\"z", 7 }, +}; + + +nxt_int_t +nxt_utf8_sanitize_test(nxt_thread_t *thr) +{ + nxt_mp_t *mp; + nxt_str_t src, dst, alias; + nxt_int_t ret; + nxt_uint_t i; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "utf8 sanitize test started"); + + mp = nxt_mp_create(1024, 128, 256, 32); + if (mp == NULL) { + nxt_log_alert(thr->log, "utf8 sanitize test: nxt_mp_create() failed"); + return NXT_ERROR; + } + + for (i = 0; i < nxt_nitems(nxt_utf8_sanitize_cases); i++) { + + src.start = (u_char *) nxt_utf8_sanitize_cases[i].data; + src.length = nxt_utf8_sanitize_cases[i].length; + + dst.start = NULL; + dst.length = 0; + + ret = nxt_utf8_sanitize(mp, &dst, &src); + if (ret != NXT_OK) { + nxt_log_alert(thr->log, "utf8 sanitize test: \"%s\" failed", + nxt_utf8_sanitize_cases[i].name); + goto fail; + } + + if (!nxt_utf8_sanitize_test_utf8(dst.start, dst.length)) { + nxt_log_alert(thr->log, "utf8 sanitize test: \"%s\" produced " + "invalid UTF-8: \"%*s\"", + nxt_utf8_sanitize_cases[i].name, + (size_t) dst.length, dst.start); + goto fail; + } + + /* + * The access log sanitizes in place -- nxt_utf8_sanitize(mp, &s, &s) + * -- so the aliased call is the one that matters, and it is the one + * a two-pass implementation gets wrong by writing "dst" before the + * second pass has read "src". + */ + + alias = src; + + ret = nxt_utf8_sanitize(mp, &alias, &alias); + if (ret != NXT_OK) { + nxt_log_alert(thr->log, "utf8 sanitize test: \"%s\" failed " + "in place", nxt_utf8_sanitize_cases[i].name); + goto fail; + } + + if (alias.length != dst.length + || memcmp(alias.start, dst.start, dst.length) != 0) + { + nxt_log_alert(thr->log, "utf8 sanitize test: \"%s\" in place " + "gave \"%*s\", not the \"%*s\" of the separate " + "call", nxt_utf8_sanitize_cases[i].name, + (size_t) alias.length, alias.start, + (size_t) dst.length, dst.start); + goto fail; + } + + /* + * A string that needs nothing done to it must come back as itself, + * not as a copy: the ordinary request logs no invalid bytes, and + * allocating per value per request would be paid on every one. + */ + + if (nxt_utf8_sanitize_test_utf8(src.start, src.length) + && dst.start != src.start) + { + nxt_log_alert(thr->log, "utf8 sanitize test: \"%s\" copied a " + "string that was already valid", + nxt_utf8_sanitize_cases[i].name); + goto fail; + } + } + + nxt_mp_destroy(mp); + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "utf8 sanitize test passed"); + + return NXT_OK; + +fail: + + nxt_mp_destroy(mp); + + return NXT_ERROR; +} diff --git a/src/test/nxt_utf8_test.c b/src/test/nxt_utf8_test.c index 31e5bff94..bdfa2bc9b 100644 --- a/src/test/nxt_utf8_test.c +++ b/src/test/nxt_utf8_test.c @@ -83,7 +83,6 @@ nxt_utf8_test(nxt_thread_t *thr) { u_char *p, utf8[4]; size_t len; - int32_t n; uint32_t u, d; nxt_uint_t i, k, l, m; const u_char *pp; @@ -107,6 +106,24 @@ nxt_utf8_test(nxt_thread_t *thr) d = nxt_utf8_decode(&pp, p); + /* + * The surrogates are encodable as bytes but are not UTF-8, so the + * decoder refuses them; nxt_utf8_encode() still produces the + * three-byte form, which is what a WTF-8 producer would emit. + */ + + if (u >= 0xD800 && u <= 0xDFFF) { + + if (d != 0xFFFFFFFF) { + nxt_log_alert(thr->log, "nxt_utf8_decode(%05uxD) returned " + "%05uxD for a surrogate, expected a refusal", + u, d); + return NXT_ERROR; + } + + continue; + } + if (u != d) { nxt_log_alert(thr->log, "nxt_utf8_decode(%05uxD) failed: %05uxD", u, d); @@ -176,16 +193,6 @@ nxt_utf8_test(nxt_thread_t *thr) } } - n = nxt_utf8_casecmp((u_char *) "ABC АБВ ΑΒΓ", - (u_char *) "abc абв αβγ", - nxt_length("ABC АБВ ΑΒΓ"), - nxt_length("abc абв αβγ")); - - if (n != 0) { - nxt_log_alert(thr->log, "nxt_utf8_casecmp() failed"); - return NXT_ERROR; - } - nxt_log_error(NXT_LOG_NOTICE, thr->log, "utf8 test passed"); return NXT_OK; } diff --git a/src/wasm-wasi-component/Cargo.lock b/src/wasm-wasi-component/Cargo.lock index 818bb2077..41403cfbd 100644 --- a/src/wasm-wasi-component/Cargo.lock +++ b/src/wasm-wasi-component/Cargo.lock @@ -34,9 +34,9 @@ checksum = "e9d4ee0d472d1cd2e28c97dfa124b3d8d992e10eb0a035f33f5d12e3a177ba3b" [[package]] name = "anyhow" -version = "1.0.103" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "arbitrary" @@ -52,7 +52,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -63,32 +63,29 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "bindgen" -version = "0.68.1" +version = "0.72.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "726e4313eb6ec35d2730258ad4e15b547ee75d6afaa1361a922e78e59b7d8078" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" dependencies = [ "bitflags", "cexpr", "clang-sys", - "lazy_static", - "lazycell", + "itertools 0.13.0", "log", - "peeking_take_while", "prettyplease", "proc-macro2", "quote", "regex", - "rustc-hash 1.1.0", + "rustc-hash", "shlex 1.3.0", - "syn", - "which", + "syn 2.0.118", ] [[package]] name = "bitflags" -version = "2.13.0" +version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" [[package]] name = "block-buffer" @@ -110,27 +107,15 @@ dependencies = [ [[package]] name = "bytes" -version = "1.12.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" - -[[package]] -name = "cap-fs-ext" -version = "4.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d78e5a3368ae89b7cb68186411452b4b9fac8b41be9c19bf3f47c2d2c8e36e6b" -dependencies = [ - "cap-primitives", - "cap-std", - "io-lifetimes 3.0.1", - "windows-sys 0.61.2", -] +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "cap-primitives" -version = "4.0.2" +version = "4.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdadbd7c002d3a484b35243669abdae85a0ebaded5a61117169dc3400f9a7ff0" +checksum = "8b5f74729fd2f44701d1a8eb47e906cdb3ccd9ec0f02baad85a744b791940b18" dependencies = [ "ambient-authority", "fs-set-times", @@ -138,29 +123,17 @@ dependencies = [ "io-lifetimes 3.0.1", "ipnet", "maybe-owned", - "rustix 1.1.4", + "rustix", "rustix-linux-procfs", "windows-sys 0.61.2", "winx", ] -[[package]] -name = "cap-std" -version = "4.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7281235d6e96d3544ca18bba9049be92f4190f8d923e3caef1b5f66cfa752608" -dependencies = [ - "cap-primitives", - "io-extras", - "io-lifetimes 3.0.1", - "rustix 1.1.4", -] - [[package]] name = "cc" -version = "1.2.66" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5d6cac793997bd970000024b2934968efe83b382de4fdcf4fcb46b6ee4ad996" +checksum = "f360145194ee8e21db5ee7f3fcd4fe52210864c75c985dae33218202c8bbe040" dependencies = [ "find-msvc-tools", "shlex 2.0.1", @@ -183,9 +156,9 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "chacha20" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if", "cpufeatures 0.3.0", @@ -241,27 +214,27 @@ dependencies = [ [[package]] name = "cranelift-assembler-x64" -version = "0.134.2" +version = "0.135.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f0435c6939d6ef218bf0a3063094f2cc5c46f7a44a88f81cf932da8d3922ff8" +checksum = "3bfbb8f30b72ef5cedf2dbd85ee2f167a33c888d492a40e999250187138663c9" dependencies = [ "cranelift-assembler-x64-meta", ] [[package]] name = "cranelift-assembler-x64-meta" -version = "0.134.2" +version = "0.135.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aca52511cc21a2c9c927aef493d11c8087bdc5483249a3d93aee41e81b8bd4d1" +checksum = "5715fdd9d897798ba2bb834df5d6c1129319d51e82ddcf81812c23231e9b852b" dependencies = [ "cranelift-srcgen", ] [[package]] name = "cranelift-bforest" -version = "0.134.2" +version = "0.135.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6867e99f068454b0c81c50b807af88aa80aaf383105e2be26675d81fc4278466" +checksum = "1a266da98b248dd98dfc8a46d45d57810db04a55b81a6d8c7a9c6cf0bd2db7c3" dependencies = [ "cranelift-entity", "wasmtime-internal-core", @@ -269,9 +242,9 @@ dependencies = [ [[package]] name = "cranelift-bitset" -version = "0.134.2" +version = "0.135.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cf66017aed971344230199ad35ca01c955cf824a0e65fed5240452d2a302445" +checksum = "c67c1887ed2194ef3ecc283a9add4fe8a79d249a293063046f2adedc31ae7200" dependencies = [ "serde", "serde_derive", @@ -280,9 +253,9 @@ dependencies = [ [[package]] name = "cranelift-codegen" -version = "0.134.2" +version = "0.135.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "933b2e5237665bf9367a34ad2100eed70c94b2796586cb09d9ba11693e061667" +checksum = "90103abeafa22cc578c577999bdf1b51c32b8a59e15a09789841b4e9a854588d" dependencies = [ "bumpalo", "cranelift-assembler-x64", @@ -300,7 +273,7 @@ dependencies = [ "postcard", "pulley-interpreter", "regalloc2", - "rustc-hash 2.1.3", + "rustc-hash", "serde", "serde_derive", "sha2", @@ -311,9 +284,9 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.134.2" +version = "0.135.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "322e3ca0603683cc17f0c808559c0fb85ad52b0f20ace0478bdd4ba2fa1e9678" +checksum = "d73eaff808bc302e2899e42d822c0e64df23f38facb98ecdf695d1817d2063a3" dependencies = [ "cranelift-assembler-x64-meta", "cranelift-codegen-shared", @@ -324,24 +297,24 @@ dependencies = [ [[package]] name = "cranelift-codegen-shared" -version = "0.134.2" +version = "0.135.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "287ab88211d049178dc8dda243e2db176919d8b29db605c0efc7c6d76ebea0f7" +checksum = "de19c4fc55ac3cdeb8e153f35ef9f51db197da91a64c9811d0ddd21ed1b9d18d" [[package]] name = "cranelift-control" -version = "0.134.2" +version = "0.135.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c11438f9becf9a1e1e061bf76a33241b941a07ed01a489e9dad5fa79dc3624e9" +checksum = "9bbd919579d74a7212e6e849dc24ff015cb823e70051d01559eea828efaf773f" dependencies = [ "arbitrary", ] [[package]] name = "cranelift-entity" -version = "0.134.2" +version = "0.135.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "584ecabe1462593a9cba9efbf9022e0987421ddb548989b455ab8bdd444b743c" +checksum = "af201af93fa8a576ac4b04b901ac3f252545d7b703053c922526fce28f65a7be" dependencies = [ "cranelift-bitset", "serde", @@ -351,9 +324,9 @@ dependencies = [ [[package]] name = "cranelift-frontend" -version = "0.134.2" +version = "0.135.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34249873145213a12106d5fd8a98a9dc9e90e16525e2337b2907874d1e7e9ee7" +checksum = "69395ceee43706b93bf869b3a6631ab8d9dcb2bf19f9af4d1d2db0cb9e18af18" dependencies = [ "cranelift-codegen", "hashbrown 0.17.1", @@ -364,15 +337,15 @@ dependencies = [ [[package]] name = "cranelift-isle" -version = "0.134.2" +version = "0.135.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6f1c1dc4d6b4bc4f0be12b08cee0e59f7610287827aa62590f8e9ec6f45dff4" +checksum = "8e8d98a81f71e2ddc61849ca1454ff424fbf42a3ee5529365be8c0859ddf5b9a" [[package]] name = "cranelift-native" -version = "0.134.2" +version = "0.135.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31d9e4f6d9a3777b1ffa818000b4a6d7afd1b8787af71a9c58d94baef9076ced" +checksum = "273c79e0265b754f67ff76c42d6db4856dabf9101e3bb1b23c83255e2438dd02" dependencies = [ "cranelift-codegen", "libc", @@ -381,9 +354,9 @@ dependencies = [ [[package]] name = "cranelift-srcgen" -version = "0.134.2" +version = "0.135.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4155f8a1afdef96a0272c67804926eee4ef35f9708ab22ad6878d4c9543df596" +checksum = "2b73d7e686f3518831f88ffe88be02c6c92be09930fc5ab90a3dfd9a4de3815e" [[package]] name = "crc32fast" @@ -422,7 +395,7 @@ checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -470,9 +443,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" [[package]] name = "fnv" @@ -502,15 +475,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94e7099f6313ecacbe1256e8ff9d617b75d1bcb16a6fddef94866d225a01a14a" dependencies = [ "io-lifetimes 2.0.4", - "rustix 1.1.4", + "rustix", "windows-sys 0.59.0", ] [[package]] name = "futures" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" dependencies = [ "futures-channel", "futures-core", @@ -523,9 +496,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" dependencies = [ "futures-core", "futures-sink", @@ -533,15 +506,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" [[package]] name = "futures-executor" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" dependencies = [ "futures-core", "futures-task", @@ -550,38 +523,38 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" [[package]] name = "futures-macro" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.6", ] [[package]] name = "futures-sink" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" [[package]] name = "futures-task" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" [[package]] name = "futures-util" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ "futures-channel", "futures-core", @@ -636,9 +609,9 @@ checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" [[package]] name = "h2" -version = "0.4.15" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" dependencies = [ "atomic-waker", "bytes", @@ -676,20 +649,11 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" -[[package]] -name = "home" -version = "0.5.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" -dependencies = [ - "windows-sys 0.61.2", -] - [[package]] name = "http" -version = "1.4.2" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" dependencies = [ "bytes", "itoa", @@ -697,9 +661,9 @@ dependencies = [ [[package]] name = "http-body" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" dependencies = [ "bytes", "http", @@ -707,9 +671,9 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.3" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" dependencies = [ "bytes", "futures-core", @@ -732,9 +696,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hyper" -version = "1.10.1" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" dependencies = [ "atomic-waker", "bytes", @@ -901,6 +865,15 @@ version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.14.0" @@ -916,18 +889,6 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" -[[package]] -name = "lazy_static" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" - -[[package]] -name = "lazycell" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" - [[package]] name = "leb128" version = "0.2.7" @@ -962,12 +923,6 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" -[[package]] -name = "linux-raw-sys" -version = "0.4.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" - [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -1010,7 +965,7 @@ version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad38eb12aea514a0466ea40a80fd8cc83637065948eb4a426e4aa46261175227" dependencies = [ - "rustix 1.1.4", + "rustix", ] [[package]] @@ -1058,12 +1013,6 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" -[[package]] -name = "peeking_take_while" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" - [[package]] name = "percent-encoding" version = "2.3.2" @@ -1104,7 +1053,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn", + "syn 2.0.118", ] [[package]] @@ -1118,9 +1067,9 @@ dependencies = [ [[package]] name = "pulley-interpreter" -version = "47.0.2" +version = "48.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6064e52588328c1275d690baad99f8d4b3fdd870736966f1a63cfcbb20560b6" +checksum = "dfc9a7073f2530aab972ac0a7ce43624ff59de8ad06b76a4ef4021f7ddd1035a" dependencies = [ "cranelift-bitset", "log", @@ -1130,13 +1079,13 @@ dependencies = [ [[package]] name = "pulley-macros" -version = "47.0.2" +version = "48.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e832980317bfc7f8c8538ffc9c69b82912f3e33ec70f7c855d521654c70fa62" +checksum = "864f92c9538de48325a4f8709ec83e56471446798d48a29073ca094c5447b549" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1156,9 +1105,9 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rand" -version = "0.10.2" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" dependencies = [ "chacha20", "getrandom", @@ -1181,16 +1130,16 @@ dependencies = [ "bumpalo", "hashbrown 0.17.1", "log", - "rustc-hash 2.1.3", + "rustc-hash", "serde", "smallvec", ] [[package]] name = "regex" -version = "1.12.4" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" dependencies = [ "aho-corasick", "memchr", @@ -1200,9 +1149,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.14" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" dependencies = [ "aho-corasick", "memchr", @@ -1221,31 +1170,12 @@ version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b74b56ffa8bb2830709a538c2cbcae9aa062db0d2a42563bfb09bdaae44020eb" -[[package]] -name = "rustc-hash" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - [[package]] name = "rustc-hash" version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" -[[package]] -name = "rustix" -version = "0.38.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" -dependencies = [ - "bitflags", - "errno", - "libc", - "linux-raw-sys 0.4.15", - "windows-sys 0.59.0", -] - [[package]] name = "rustix" version = "1.1.4" @@ -1255,7 +1185,7 @@ dependencies = [ "bitflags", "errno", "libc", - "linux-raw-sys 0.12.1", + "linux-raw-sys", "windows-sys 0.61.2", ] @@ -1266,7 +1196,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2fc84bf7e9aa16c4f2c758f27412dc9841341e16aa682d9c7ac308fe3ee12056" dependencies = [ "once_cell", - "rustix 1.1.4", + "rustix", ] [[package]] @@ -1281,9 +1211,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", "serde_derive", @@ -1291,29 +1221,29 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.6", ] [[package]] name = "serde_json" -version = "1.0.150" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ "itoa", "memchr", @@ -1387,6 +1317,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "synstructure" version = "0.13.2" @@ -1395,7 +1336,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1439,7 +1380,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1450,7 +1391,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1465,9 +1406,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.52.3" +version = "1.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" dependencies = [ "bytes", "libc", @@ -1509,7 +1450,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1580,14 +1521,26 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasm-encoder" -version = "0.252.0" +version = "0.254.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8185ae345fa5687c054626ff9a50e7089797a343d9904d1dc9820eb4c4d3196f" +checksum = "ebaca0b540bdb15d195ff34fb23074255948708b8916a11a339ba4513dc61906" dependencies = [ "leb128fmt", "wasmparser", ] +[[package]] +name = "wasm-metadata" +version = "0.254.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d1880dd9a848aa68f387966256c34ad49abb4607e3cfa35720d2723d0591cd3" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + [[package]] name = "wasm-wasi-component" version = "0.1.0" @@ -1610,9 +1563,9 @@ dependencies = [ [[package]] name = "wasmparser" -version = "0.252.0" +version = "0.254.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3eb099dcadcde5be9eef55e3a337128efd4e44b4c93122487e4d2e4e1c6627c" +checksum = "c5ff8685b587722894b6f25958fc608d315b93b8d23647bfb567664e7006d757" dependencies = [ "bitflags", "hashbrown 0.17.1", @@ -1623,9 +1576,9 @@ dependencies = [ [[package]] name = "wasmprinter" -version = "0.252.0" +version = "0.254.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7142797de29b35ab8dbf15c00f55fda75d409da4c423a8ab8bd6b667a785824b" +checksum = "8b03c8f270f3f384f70f42869b5ff7f6253fe4f0aa8590f7edd332c1943e5f51" dependencies = [ "anyhow", "termcolor", @@ -1634,16 +1587,15 @@ dependencies = [ [[package]] name = "wasmtime" -version = "47.0.2" +version = "48.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06d9f263ccb212017dae1d109b9997f218a7675db1e9a7bedf07ae1643322bf3" +checksum = "b94eda6b1c1d8ade526a67e4ce50cc6234d1c7af68e5d7bf13f40bc07e70daba" dependencies = [ "addr2line", "async-trait", "bitflags", "bumpalo", "cc", - "cfg-if", "encoding_rs", "futures", "libc", @@ -1654,7 +1606,7 @@ dependencies = [ "once_cell", "postcard", "pulley-interpreter", - "rustix 1.1.4", + "rustix", "semver", "serde", "serde_derive", @@ -1676,9 +1628,9 @@ dependencies = [ [[package]] name = "wasmtime-environ" -version = "47.0.2" +version = "48.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9915e3d90c36a16e0fec8be746ee87e21cf0d4a017ecf09c49b7d1cc6f31b2b6" +checksum = "84ca4865ece3a37abb34c2df96c741ff4aed308d35744d84095cff79848752d4" dependencies = [ "anyhow", "cpp_demangle", @@ -1707,14 +1659,14 @@ dependencies = [ [[package]] name = "wasmtime-internal-component-macro" -version = "47.0.2" +version = "48.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b627ea698db36fb9110aa77868467f1d23791a76a88a433cb6a43e1242073b77" +checksum = "5ae080b05e5e0356a2167c478c264e26f3fe5dbaaf1b285f8f5501301b99facb" dependencies = [ "anyhow", "proc-macro2", "quote", - "syn", + "syn 2.0.118", "wasmtime-internal-component-util", "wasmtime-internal-wit-bindgen", "wit-parser", @@ -1722,15 +1674,15 @@ dependencies = [ [[package]] name = "wasmtime-internal-component-util" -version = "47.0.2" +version = "48.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f45bf943cff1d2f1976bac6a241ca4ecaf29a6efd5f1ad89e7722439fa5c392f" +checksum = "e8f014d9ad6f6acbc449e9d7c4e7b7757b4b6b1a84a3662f66ea6da2ebed788c" [[package]] name = "wasmtime-internal-core" -version = "47.0.2" +version = "48.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea33e1f9685be82f90223bebcf2ffefb6b596ea3d82a3c42384527b9c6f5b447" +checksum = "ecc0a0310fc4a3f545427c7880362657f72f838ffcfe4d3b8c864021df3b3488" dependencies = [ "anyhow", "hashbrown 0.17.1", @@ -1740,18 +1692,17 @@ dependencies = [ [[package]] name = "wasmtime-internal-cranelift" -version = "47.0.2" +version = "48.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b1ce104719932a48dbdfec0cc292c96494b0bfe764946e45fbfd3340e3a68f9" +checksum = "cb6c8d2b6f8ed3a784fcd127a1936589db8051febbd228230660175e41b5751d" dependencies = [ - "cfg-if", "cranelift-codegen", "cranelift-control", "cranelift-entity", "cranelift-frontend", "cranelift-native", "gimli", - "itertools", + "itertools 0.14.0", "log", "object", "pulley-interpreter", @@ -1767,14 +1718,13 @@ dependencies = [ [[package]] name = "wasmtime-internal-fiber" -version = "47.0.2" +version = "48.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "976e4b71b9c0da8a9a7bb8d5fa0abedc4da879b397111fd91ece73be524fe621" +checksum = "92b558b5892ca405508103641fa3a3ec2e986b1cc679a083ed47e6bcb08ecf31" dependencies = [ "cc", - "cfg-if", "libc", - "rustix 1.1.4", + "rustix", "wasmtime-environ", "wasmtime-internal-versioned-export-macros", "windows-sys 0.61.2", @@ -1782,9 +1732,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-jit-debug" -version = "47.0.2" +version = "48.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1f60ecba3497cfc26284903c2e79023fd5ea991b95b0e8dafe994fd9fca4c4e" +checksum = "c0669ce70a8a8c92eb8ad148c1d7b9b771aab37787e9462efd75b2e2b4575406" dependencies = [ "cc", "wasmtime-internal-versioned-export-macros", @@ -1792,11 +1742,10 @@ dependencies = [ [[package]] name = "wasmtime-internal-jit-icache-coherence" -version = "47.0.2" +version = "48.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fae6decb025ae6d12f3391b36deb62a6838d45b18864f796d84bced722ceac0d" +checksum = "6aa66405e95fc7b82adb195c14fa8b38f45046c3ad99a273b06262b8eb755e76" dependencies = [ - "cfg-if", "libc", "wasmtime-internal-core", "windows-sys 0.61.2", @@ -1804,11 +1753,10 @@ dependencies = [ [[package]] name = "wasmtime-internal-unwinder" -version = "47.0.2" +version = "48.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0982122d9470d3d72ed7aa9ed91e69538ae44c1494cd22d78fec7846b248a940" +checksum = "70555458e519d6fb8790e042b4274132ab58171ae0f2a6917c3dc29ded0184d0" dependencies = [ - "cfg-if", "cranelift-codegen", "log", "object", @@ -1817,44 +1765,43 @@ dependencies = [ [[package]] name = "wasmtime-internal-versioned-export-macros" -version = "47.0.2" +version = "48.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "363cf88a80e6ed13311d0d34d9bcd02a2e07d98113b994ff41ed386a206f498e" +checksum = "fca407e3e2809c0e6568be132065095a16f184c30c6bee4157dfdc497aee10a1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] name = "wasmtime-internal-wit-bindgen" -version = "47.0.2" +version = "48.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87ff41bb1656cbeb7befc42b8a7078b30a5f8393a0c8bca36a6cec94360c1a16" +checksum = "2aacdb42fb0b393353e402a1534e0b492a6b0fddf6528bfaec65c5f56b990c43" dependencies = [ "anyhow", "bitflags", "heck", "indexmap", + "wit-component", "wit-parser", ] [[package]] name = "wasmtime-wasi" -version = "47.0.2" +version = "48.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed1750b8e5e5d5b1ee3c0eb602d2c49d6a37a36e9996eaf0708b48ac92b62b06" +checksum = "1803e99b97392e59866a9f3178ed38f8addb74412facbcb6dc102b2e7420da76" dependencies = [ "async-trait", "bitflags", "bytes", - "cap-fs-ext", - "cap-std", - "cfg-if", + "cap-primitives", "futures", - "io-lifetimes 3.0.1", "rand", - "rustix 1.1.4", + "rustix", + "rustix-linux-procfs", "thiserror 2.0.18", "tokio", "tracing", @@ -1863,13 +1810,14 @@ dependencies = [ "wasmtime-wasi-io", "wiggle", "windows-sys 0.61.2", + "winx", ] [[package]] name = "wasmtime-wasi-http" -version = "47.0.2" +version = "48.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "014b8f05cacb7a5d665536e8a733b887bd192a9c179037580d48d0a6a82ecd34" +checksum = "1aeecf39d9003a7f3199ae7b9e70ad9115b3d73746ff9dce87a500fd95d572ea" dependencies = [ "async-trait", "bytes", @@ -1878,6 +1826,7 @@ dependencies = [ "http-body", "http-body-util", "hyper", + "pin-project-lite", "tokio", "tracing", "wasmtime", @@ -1887,9 +1836,9 @@ dependencies = [ [[package]] name = "wasmtime-wasi-io" -version = "47.0.2" +version = "48.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54777d3afeaf3f32cc444df661d9cb4fd57067971e183c2b1f5251f33b939e68" +checksum = "1af4069fb7fc9cdc72410be96fb7f83b772e500cf4d0758d7fe6b55cefb303b8" dependencies = [ "async-trait", "bytes", @@ -1907,23 +1856,11 @@ dependencies = [ "leb128", ] -[[package]] -name = "which" -version = "4.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" -dependencies = [ - "either", - "home", - "once_cell", - "rustix 0.38.44", -] - [[package]] name = "wiggle" -version = "47.0.2" +version = "48.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a23c2dc694dfafb35beb542e172d7c256488912cd84902828f8661723ad0eb7" +checksum = "314611904412397366e044ace02f657dc69db7fb419c630a319a843fe65afee4" dependencies = [ "bitflags", "thiserror 2.0.18", @@ -1935,27 +1872,27 @@ dependencies = [ [[package]] name = "wiggle-generate" -version = "47.0.2" +version = "48.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2fe48e5e455e827339d8e65de602cc7839d241bd7063be1fc235e35581788ad" +checksum = "d0c1d1d148dd7a0425eef52a5bc9e26bd35ebfe8a5f0fcfc5480a6d3cee56109" dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.118", "wasmtime-environ", "witx", ] [[package]] name = "wiggle-macro" -version = "47.0.2" +version = "48.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb520c20bed78e5a311d4c9ad883ac3b6d070617d07a19fbe5dc0b408c501b4b" +checksum = "488783149d44017c9807189b16cc09e02e84059740fe64178b5ab3cc19e9fc0b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", "wiggle-generate", ] @@ -2140,11 +2077,30 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "wit-component" +version = "0.254.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c96b3cea6838dbea55173baa56991fcb087896c321b0a770db201e5a0bd3506" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + [[package]] name = "wit-parser" -version = "0.252.0" +version = "0.254.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4266bea110371c620ccf3201c5023676046bc4556e5c7cfb5d500bda5ebc162d" +checksum = "22672d61bfb9df46a16b46ce38522293aee1cc4749fc84a3464ce4083b3d5fd6" dependencies = [ "anyhow", "hashbrown 0.17.1", @@ -2196,7 +2152,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", "synstructure", ] @@ -2217,7 +2173,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", "synstructure", ] @@ -2251,7 +2207,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] diff --git a/src/wasm-wasi-component/Cargo.toml b/src/wasm-wasi-component/Cargo.toml index 82de81083..f591bc11f 100644 --- a/src/wasm-wasi-component/Cargo.toml +++ b/src/wasm-wasi-component/Cargo.toml @@ -2,6 +2,7 @@ name = "wasm-wasi-component" version = "0.1.0" edition = "2021" +rust-version = "1.95" publish = false [lib] @@ -17,17 +18,17 @@ http-body = { version = "1.0.0", default-features = false } http-body-util = "0.1.0" hyper = "1.4.1" tokio = { version = "1.33.0", default-features = false } -wasmtime = { version = "47.0.2", default-features = false, features = ['anyhow', 'component-model', 'cranelift'] } -wasmtime-wasi = "47.0.2" +wasmtime = { version = "48.0.5", default-features = false, features = ['anyhow', 'component-model', 'cranelift'] } +wasmtime-wasi = "48.0.5" # default-features = false drops the "default-send-request" feature and # with it the whole rustls TLS stack (rustls, tokio-rustls, webpki-roots, # rustls-webpki) that exists only so a guest can dial out. "p2" is in the # crate's default set too, so it has to be named explicitly or the # wasmtime_wasi_http::p2 module this file uses disappears with it. -wasmtime-wasi-http = { version = "47.0.2", default-features = false, features = ['p2'] } +wasmtime-wasi-http = { version = "48.0.5", default-features = false, features = ['p2'] } [build-dependencies] -bindgen = "0.68.1" +bindgen = "0.72" cc = "1.0.83" [profile.dev] diff --git a/src/wasm-wasi-component/build.rs b/src/wasm-wasi-component/build.rs index 5ea74f172..de04ec9a5 100644 --- a/src/wasm-wasi-component/build.rs +++ b/src/wasm-wasi-component/build.rs @@ -15,7 +15,7 @@ fn main() { .blocklist_function("nxt_vsprintf") // Tell cargo to invalidate the built crate whenever any of the // included header files changed. - .parse_callbacks(Box::new(bindgen::CargoCallbacks)) + .parse_callbacks(Box::new(bindgen::CargoCallbacks::new())) // disable some features which aren't necessary .layout_tests(false) .derive_debug(false) diff --git a/src/wasm-wasi-component/src/lib.rs b/src/wasm-wasi-component/src/lib.rs index 7eb26f0bc..65b8ed455 100644 --- a/src/wasm-wasi-component/src/lib.rs +++ b/src/wasm-wasi-component/src/lib.rs @@ -3,25 +3,27 @@ use bytes::{Bytes, BytesMut}; use http_body_util::combinators::BoxBody; use http_body_util::{BodyExt, Full}; use hyper::Error; +use std::borrow::Cow; use std::ffi::{CStr, CString}; +use std::future::Future; use std::mem::MaybeUninit; +use std::os::raw::c_int; use std::process::exit; use std::ptr; -use std::sync::OnceLock; +use std::sync::{Arc, Condvar, Mutex, MutexGuard, OnceLock}; +use std::time::Duration; use tokio::sync::mpsc; use wasmtime::component::{Component, Linker, ResourceTable}; use wasmtime::{Config, Engine, Store}; use wasmtime_wasi::p2::add_to_linker_async; -use wasmtime_wasi::{DirPerms, FilePerms, WasiCtx, WasiCtxBuilder, - WasiCtxView, WasiView}; -use wasmtime_wasi_http::p2::bindings::http::types::{ErrorCode, Scheme}; +use wasmtime_wasi::{FsPerms, WasiCtx, WasiCtxBuilder, WasiCtxView, + WasiView}; +use wasmtime_wasi_http::p2::bindings::http::types::Scheme; use wasmtime_wasi_http::p2::bindings::ProxyPre; use wasmtime_wasi_http::p2::body::HyperOutgoingBody; -use wasmtime_wasi_http::p2::types::{HostFutureIncomingResponse, - OutgoingRequestConfig}; -use wasmtime_wasi_http::p2::{HttpResult, WasiHttpCtxView, WasiHttpHooks, - WasiHttpView}; -use wasmtime_wasi_http::WasiHttpCtx; +use wasmtime_wasi_http::{Error as HttpError, RequestOptions, WasiBody, + WasiHttpCtx, WasiHttpCtxView, WasiHttpHooks, + WasiHttpView}; #[allow( non_camel_case_types, @@ -100,6 +102,7 @@ unsafe extern "C" fn setup( let result = GLOBAL_CONFIG.set(GlobalConfig { component: component.to_string(), dirs, + execution_timeout: wasm_conf.execution_timeout.into(), }); assert!(result.is_ok()); Ok(()) @@ -178,12 +181,20 @@ unsafe extern "C" fn request_handler( // Enqueue this request to get processed by the Tokio event loop, and // otherwise immediately return. let state = GLOBAL_STATE.get().unwrap(); - state.sender.blocking_send(NxtRequestInfo { info }).unwrap(); + state + .sender + .blocking_send(NxtRequestInfo { + info, + response_started: false, + }) + .unwrap(); } struct GlobalConfig { component: String, dirs: Vec, + /// "execution_timeout" in milliseconds, 0 for unbounded. + execution_timeout: u64, } struct GlobalState { @@ -191,6 +202,32 @@ struct GlobalState { component: ProxyPre, global_config: &'static GlobalConfig, sender: mpsc::Sender, + /// `None` unless "execution_timeout" is configured, in which case the + /// engine has epoch interruption on and every store needs a deadline. + epoch: Option, +} + +/// What a store needs to be armed: who advances the epoch, and how far +/// ahead the deadline sits. +/// +/// The two travel together because neither is correct alone. With +/// `epoch_interruption(true)` a store that is never given a deadline traps +/// on its first instruction, and a deadline nothing advances never fires, +/// so a single `Option` is what keeps a later `Store::new` from getting +/// either. +struct Epoch { + ticker: Arc, + /// The deadline, in epoch ticks. See `epoch_deadline_ticks()`. + deadline: u64, +} + +impl Epoch { + /// Arms `store` and keeps the ticker awake until the guard is dropped. + fn arm(&self, store: &mut Store) -> Ticking { + store.set_epoch_deadline(self.deadline); + + self.ticker.ticking() + } } impl GlobalState { @@ -206,8 +243,36 @@ impl GlobalState { // * Memory limits/etc. let mut config = Config::new(); config.wasm_component_model(true); + + // A guest yields only at an async wasi import, so one that computes + // without calling any stays inside a single `call_handle()` poll for + // as long as it likes: `JoinHandle::abort()` is never observed, the + // store is held, and a runtime worker thread is pinned. Epoch + // interruption is what lets the host stop it. Fuel would too, but + // it counts every instruction to be deterministic, where this only + // reads a counter at function entries and loop headers and costs + // 2-3x less (wasmtime Config::epoch_interruption docs); a deadline + // does not need to be reproducible, only enforced. + // + // Off unless asked for: it changes the code generated for every + // guest, and a component that legitimately computes for a long time + // must keep working. + let bounded = global_config.execution_timeout != 0; + + if bounded { + config.epoch_interruption(true); + } + let engine = Engine::new(&config)?; + // One `bounded` for both: with epoch interruption on, a store that + // is never armed traps at once, so the engine setting and the + // `Epoch` that arms every store cannot be allowed to disagree. + let epoch = bounded.then(|| Epoch { + ticker: EpochTicker::start(engine.clone()), + deadline: epoch_deadline_ticks(global_config.execution_timeout), + }); + // Compile the binary component on disk in Wasmtime. This is then // pre-instantiated with host APIs defined by WASI. The result of // this is a "pre-instantiated instance" which can be used to @@ -242,6 +307,7 @@ impl GlobalState { component: proxy, sender, global_config, + epoch, }) } @@ -255,14 +321,26 @@ impl GlobalState { rt.block_on(async { while let Some(msg) = receiver.recv().await { let state = GLOBAL_STATE.get().unwrap(); - tokio::task::spawn(async move { - state.handle(msg).await.expect("failed to handle request") - }); + tokio::task::spawn(async move { state.handle(msg).await }); } }); } - async fn handle(&'static self, mut info: NxtRequestInfo) -> Result<()> { + /// Handle one request, and finish it whatever happens. + /// + /// Every error below used to reach `.expect()` in `run()`, which under + /// `panic = 'abort'` killed the worker and every other request on it. + async fn handle(&'static self, mut info: NxtRequestInfo) { + match self.handle_inner(&mut info).await { + Ok(()) => info.request_done(), + Err(e) => info.fail(&e), + } + } + + async fn handle_inner( + &'static self, + info: &mut NxtRequestInfo, + ) -> Result<()> { // Create a "Store" which is the unit of per-request isolation in // Wasmtime. let data = StoreState { @@ -274,12 +352,7 @@ impl GlobalState { cx.inherit_stderr(); cx.inherit_env(); for dir in self.global_config.dirs.iter() { - cx.preopened_dir( - dir, - dir, - DirPerms::all(), - FilePerms::all(), - )?; + cx.preopened_dir(dir, dir, FsPerms::ReadWrite)?; } cx.build() }, @@ -292,9 +365,22 @@ impl GlobalState { // Convert the `nxt_*` representation into the representation required // by Wasmtime's `wasi-http` implementation using the Rust `http` // crate. - let request = self.to_request_builder(&info)?; - let body = self.to_request_body(&mut info); - let request = request.body(body)?; + // A request that the `http` crate refuses to represent must fail on + // its own, not take the worker with it: `run()` turns an `Err` from + // here into `.expect()`, and both profiles set `panic = 'abort'`. + // Unit forwards bytes that `http` rejects -- a `Host` holding + // obs-text, a target holding `<`, `>` or a control byte -- so this is + // reachable from an unauthenticated request. + let builder = self.to_request_builder(info).context(BadRequest)?; + + // Not BadRequest: a negative read comes from read() on the spooled + // body failing (src/nxt_unit.c), which is our I/O, not the client's + // doing, so it answers 500. + let body = self.to_request_body(info)?; + let request = builder + .body(body) + .map_err(anyhow::Error::from) + .context(BadRequest)?; let (sender, receiver) = tokio::sync::oneshot::channel(); @@ -305,7 +391,18 @@ impl GlobalState { // writing the response when it's available. This enables wasm to // generate headers, write those below, and then compute the body // afterwards. - let task = tokio::spawn(async move { + let mut task = GuestTask(Some(tokio::spawn(async move { + // Arm the deadline here rather than beside `Store::new`, and + // hold the ticker awake for as long as the guest can run. The + // budget is spent by the epoch, which advances in wall clock + // while any request is in flight, so everything before this + // point would spend it without the guest running: the wait for + // a runtime worker -- under load, behind the very guests this + // bounds -- and the blocking `request_read()` inside + // `to_request_body()`. A task that is dropped before it is + // ever polled now also never wakes the ticker. + let _ticking = self.epoch.as_ref().map(|e| e.arm(&mut store)); + let req = store .data_mut() .http() @@ -319,7 +416,7 @@ impl GlobalState { .await .map_err(|e| e.context("failed to invoke wasm `handle`"))?; Ok::<_, anyhow::Error>(()) - }); + }))); // Wait for the wasm to produce the initial response. If this succeeds // then propagate that failure. If this fails then wait for the above @@ -328,27 +425,24 @@ impl GlobalState { let response = match receiver.await { Ok(response) => response.context("response generation failed")?, Err(_) => { - task.await.unwrap()?; - panic!("sender of response disappeared"); + task.join().await?; + bail!("the sender of the response disappeared"); } }; // Send the headers/status which will extract the body for the next // phase. - let body = self.send_response(&mut info, response); + let body = self.send_response(info, response)?; // Send the body, a blocking operation, over time as it becomes // available. - self.send_response_body(&mut info, body) + self.send_response_body(info, body) .await .context("failed to write response body")?; // Join on completion of the wasm task which should be done by this // point. - task.await.unwrap()?; - - // And finally signal that we're done. - info.request_done(); + task.join().await?; Ok(()) } @@ -359,8 +453,8 @@ impl GlobalState { ) -> Result { let mut request = http::Request::builder(); - request = request.method(info.method()); - request = match info.version() { + request = request.method(info.method().as_ref()); + request = match &*info.version() { "HTTP/0.9" => request.version(http::Version::HTTP_09), "HTTP/1.0" => request.version(http::Version::HTTP_10), "HTTP/1.1" => request.version(http::Version::HTTP_11), @@ -374,14 +468,14 @@ impl GlobalState { let uri = http::Uri::builder() .scheme(if info.tls() { "https" } else { "http" }) - .authority(info.server_name()) - .path_and_query(info.target()) + .authority(info.server_name().as_ref()) + .path_and_query(info.target().as_ref()) .build() .context("failed to build URI")?; request = request.uri(uri); for (name, value) in info.fields() { - request = request.header(name, value); + request = request.header(name.as_ref(), value.as_ref()); } Ok(request) } @@ -389,7 +483,7 @@ impl GlobalState { fn to_request_body( &self, info: &mut NxtRequestInfo, - ) -> BoxBody { + ) -> Result> { // TODO: should convert the body into a form of `Stream` to become an // async stream of frames. The return value can represent that here // but for now this slurps up the entire body into memory and puts it @@ -399,16 +493,16 @@ impl GlobalState { // TODO: can this perform a partial read? // TODO: how to make this async at the nxt level? - info.request_read(&mut body); + info.request_read(&mut body)?; - Full::new(body.freeze()).map_err(|e| match e {}).boxed() + Ok(Full::new(body.freeze()).map_err(|e| match e {}).boxed()) } fn send_response( &self, info: &mut NxtRequestInfo, response: http::Response, - ) -> T { + ) -> Result { info.init_response( response.status().as_u16(), response.headers().len().try_into().unwrap(), @@ -419,13 +513,13 @@ impl GlobalState { .sum::() .try_into() .unwrap(), - ); + )?; for (k, v) in response.headers() { - info.add_field(k.as_str().as_bytes(), v.as_bytes()); + info.add_field(k.as_str().as_bytes(), v.as_bytes())?; } - info.send_response(); + info.send_response()?; - response.into_body() + Ok(response.into_body()) } async fn send_response_body( @@ -444,7 +538,7 @@ impl GlobalState { }; match frame.data_ref() { Some(data) => { - info.response_write(&data); + info.response_write(&data)?; } None => { // TODO: what to do with trailers? @@ -454,8 +548,166 @@ impl GlobalState { } } +/// How often the epoch advances while a guest is running. +const EPOCH_TICK: Duration = Duration::from_millis(100); + +/// Turn a timeout in milliseconds into `Store::set_epoch_deadline()` ticks. +/// +/// A store is armed at an arbitrary point between two ticks, so with `n` +/// ticks the guest is stopped somewhere in `[(n - 1) * TICK, n * TICK]`. +/// The extra tick is what keeps the lower end at or after the configured +/// timeout: a request is never cut short of it. +/// +/// The far end is one tick past the timeout once the rounding up is a +/// no-op, which it always is here -- the config is whole seconds, so +/// `timeout_ms` is a multiple of `TICK` -- and two ticks otherwise. Add +/// to that whatever the guest takes to reach its next epoch check, which +/// wasmtime puts at function entries and loop headers. +fn epoch_deadline_ticks(timeout_ms: u64) -> u64 { + let tick = EPOCH_TICK.as_millis() as u64; + + timeout_ms.saturating_add(tick - 1) / tick + 1 +} + +/// Advances the engine's epoch so an armed deadline can fire. +/// +/// The ticker is its own OS thread rather than a task on the module's tokio +/// runtime. A guest that never yields occupies a runtime worker thread for +/// as long as it runs, so enough of them starve exactly the task that is +/// meant to preempt them -- on a one-core machine, the first one does. A +/// thread outside the runtime cannot be starved by the code it bounds. +/// +/// It also sleeps while nothing is running. A worker with no request in +/// flight has nothing to preempt, and a server can hold many idle workers, +/// so the ticker waits on `wake` until a request arms a deadline. +struct EpochTicker { + running: Mutex, + wake: Condvar, +} + +impl EpochTicker { + fn start(engine: Engine) -> Arc { + let ticker = Arc::new(EpochTicker { + running: Mutex::new(0), + wake: Condvar::new(), + }); + + let thread = ticker.clone(); + std::thread::spawn(move || thread.run(engine)); + + ticker + } + + fn run(&self, engine: Engine) { + loop { + { + let mut running = self.lock(); + + while *running == 0 { + running = match self.wake.wait(running) { + Ok(running) => running, + Err(e) => e.into_inner(), + }; + } + } + + // Sleeping outside the lock means a request that finishes + // meanwhile still gets this increment. That is one wasted + // atomic add, not a missed deadline. + std::thread::sleep(EPOCH_TICK); + engine.increment_epoch(); + } + } + + /// Keeps the ticker awake until the returned guard is dropped. + fn ticking(self: &Arc) -> Ticking { + *self.lock() += 1; + self.wake.notify_one(); + + Ticking(self.clone()) + } + + /// `panic = 'abort'` means a lock is never poisoned, but taking the + /// inner value rather than unwrapping keeps a second panic -- which + /// would end the worker -- off the table if that ever changes. + fn lock(&self) -> MutexGuard<'_, usize> { + match self.running.lock() { + Ok(running) => running, + Err(e) => e.into_inner(), + } + } +} + +/// Marks one request as running, so the ticker keeps the epoch moving. +struct Ticking(Arc); + +impl Drop for Ticking { + fn drop(&mut self) { + *self.0.lock() -= 1; + } +} + +/// Owns the wasm task so that leaving `handle_inner()` early stops it. +/// Dropping a `JoinHandle` only detaches the task, it does not cancel it: +/// a client that disconnects mid-response fails the write, the request is +/// finished without us, and the wasm invocation would keep its `Store` and +/// burn CPU in the background. `join()` takes the handle back out, so a +/// task that ran to completion is never aborted. +struct GuestTask(Option>>); + +impl GuestTask { + /// Wait for the wasm task, reporting a cancelled or panicked one as an + /// error. `unwrap()` here would panic a second time, and under + /// `panic = 'abort'` that ends the worker. A missing handle cannot + /// happen -- both callers return right after -- and is reported the + /// same way for the same reason. + async fn join(&mut self) -> Result<()> { + match self.0.take() { + Some(task) => match task.await { + Ok(result) => result, + Err(e) => bail!("the wasm task did not finish: {e}"), + }, + None => bail!("the wasm task was already joined"), + } + } +} + +impl Drop for GuestTask { + fn drop(&mut self) { + if let Some(task) = &self.0 { + task.abort(); + } + } +} + +/// Marks an error caused by what the client sent, so it is answered 400 +/// rather than 500. +#[derive(Debug)] +struct BadRequest; + +impl std::fmt::Display for BadRequest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("malformed request") + } +} + +impl std::error::Error for BadRequest {} + +/// libunit reports failure as a non-zero return. Turn that into an error +/// rather than a panic: under `panic = 'abort'` a panic here would take the +/// worker down and with it every other request it is serving. +fn check_rc(rc: c_int, what: &str) -> Result<()> { + if rc != 0 { + bail!("{what} failed with {rc}"); + } + Ok(()) +} + struct NxtRequestInfo { info: *mut bindings::nxt_unit_request_info_t, + // Once the status line is out, a failure can no longer be answered with + // a status -- all that is left is to end the request. + response_started: bool, } // TODO: is this actually safe? @@ -463,7 +715,7 @@ unsafe impl Send for NxtRequestInfo {} unsafe impl Sync for NxtRequestInfo {} impl NxtRequestInfo { - fn method(&self) -> &str { + fn method(&self) -> Cow<'_, str> { unsafe { let raw = (*self.info).request; self.get_str(&(*raw).method, (*raw).method_length.into()) @@ -474,21 +726,21 @@ impl NxtRequestInfo { unsafe { (*(*self.info).request).tls != 0 } } - fn version(&self) -> &str { + fn version(&self) -> Cow<'_, str> { unsafe { let raw = (*self.info).request; self.get_str(&(*raw).version, (*raw).version_length.into()) } } - fn server_name(&self) -> &str { + fn server_name(&self) -> Cow<'_, str> { unsafe { let raw = (*self.info).request; self.get_str(&(*raw).server_name, (*raw).server_name_length.into()) } } - fn target(&self) -> &str { + fn target(&self) -> Cow<'_, str> { unsafe { let raw = (*self.info).request; self.get_str(&(*raw).target, (*raw).target_length.into()) @@ -502,7 +754,7 @@ impl NxtRequestInfo { } } - fn fields(&self) -> impl Iterator { + fn fields(&self) -> impl Iterator, Cow<'_, str>)> { unsafe { let raw = (*self.info).request; (0..(*raw).fields_count).map(move |i| { @@ -516,39 +768,64 @@ impl NxtRequestInfo { } } - fn request_read(&mut self, dst: &mut BytesMut) { + fn request_read(&mut self, dst: &mut BytesMut) -> Result<()> { unsafe { let rest = dst.spare_capacity_mut(); let mut total_bytes_read = 0; - loop { + while total_bytes_read < rest.len() { let amt = bindings::nxt_unit_request_read( self.info, rest.as_mut_ptr().wrapping_add(total_bytes_read).cast(), - 32 * 1024 * 1024, + rest.len() - total_bytes_read, ); - total_bytes_read += amt as usize; - if total_bytes_read >= rest.len() { + + // A read returns a signed count. `amt as usize` on a + // negative one wraps to a huge number, which ends the loop + // and then moves `set_len()` past the allocation. + if amt < 0 { + bail!("failed to read the request body"); + } + + // Nothing left to read. Without this a steady 0 spins here + // forever, because 0 never reaches `rest.len()`. + if amt == 0 { break; } + + // libunit should never return more than it was asked for, + // but this count decides `set_len()` below, so do not take + // its word for it: a wrong one here is a buffer overrun, + // not a wrong answer. + let amt = amt as usize; + if amt > rest.len() - total_bytes_read { + bail!("the request body read returned more than it was given room for"); + } + + total_bytes_read += amt; } - // TODO: handle failure when `amt` is negative - let total_bytes_read: usize = total_bytes_read.try_into().unwrap(); dst.set_len(dst.len() + total_bytes_read); } + Ok(()) } - fn response_write(&mut self, data: &[u8]) { + fn response_write(&mut self, data: &[u8]) -> Result<()> { unsafe { let rc = bindings::nxt_unit_response_write( self.info, data.as_ptr().cast(), data.len(), ); - assert_eq!(rc, 0); + check_rc(rc, "nxt_unit_response_write")?; } + Ok(()) } - fn init_response(&mut self, status: u16, headers: u32, headers_size: u32) { + fn init_response( + &mut self, + status: u16, + headers: u32, + headers_size: u32, + ) -> Result<()> { unsafe { let rc = bindings::nxt_unit_response_init( self.info, @@ -556,27 +833,88 @@ impl NxtRequestInfo { headers, headers_size, ); - assert_eq!(rc, 0); + check_rc(rc, "nxt_unit_response_init")?; } + Ok(()) } - fn add_field(&mut self, key: &[u8], val: &[u8]) { + fn add_field(&mut self, key: &[u8], val: &[u8]) -> Result<()> { + // libunit takes the name length as a u8. A guest is free to emit a + // longer one, and `unwrap()` here would abort the worker over a + // header -- the failure this whole path exists to stop. + let Ok(key_len) = key.len().try_into() else { + bail!("a response header name is longer than 255 bytes"); + }; + let Ok(val_len) = val.len().try_into() else { + bail!("a response header value is too long"); + }; + unsafe { let rc = bindings::nxt_unit_response_add_field( self.info, key.as_ptr().cast(), - key.len().try_into().unwrap(), + key_len, val.as_ptr().cast(), - val.len().try_into().unwrap(), + val_len, ); - assert_eq!(rc, 0); + check_rc(rc, "nxt_unit_response_add_field")?; } + Ok(()) } - fn send_response(&mut self) { + fn send_response(&mut self) -> Result<()> { unsafe { let rc = bindings::nxt_unit_response_send(self.info); - assert_eq!(rc, 0); + check_rc(rc, "nxt_unit_response_send")?; + } + // Only now has a status line reached the router. `response_init` + // alone fills a local buffer, and libunit permits a second one, so + // a failure before this point can still be answered. + self.response_started = true; + Ok(()) + } + + /// Fail this one request, leaving the worker alive. + /// + /// Answers 500 if nothing has gone out yet. Once the status line is + /// sent there is no status left to send, so all this can do is end the + /// request and let the client see a truncated response. + fn fail(mut self, err: &anyhow::Error) { + let bad = err.downcast_ref::().is_some(); + let status = if bad { 400 } else { 500 }; + + self.log_err(&format!("failed to handle a request: {err:#}")); + + if !self.response_started + && self.init_response(status, 0, 0).is_ok() + && self.send_response().is_ok() + { + self.request_done(); + return; + } + + self.request_failed(); + } + + fn log_err(&self, msg: &str) { + unsafe { + if let Ok(msg) = CString::new(msg) { + bindings::nxt_unit_req_log( + self.info, + bindings::NXT_UNIT_LOG_ERR as i32, + "%s\0".as_ptr().cast(), + msg.as_ptr(), + ); + } + } + } + + fn request_failed(self) { + unsafe { + bindings::nxt_unit_request_done( + self.info, + bindings::NXT_UNIT_ERROR as i32, + ); } } @@ -593,10 +931,10 @@ impl NxtRequestInfo { &self, ptr: &bindings::nxt_unit_sptr_t, len: u32, - ) -> &str { + ) -> Cow<'_, str> { let ptr = bindings::nxt_unit_sptr_get(ptr); let slice = std::slice::from_raw_parts(ptr, len.try_into().unwrap()); - std::str::from_utf8(slice).unwrap() + String::from_utf8_lossy(slice) } } @@ -629,10 +967,21 @@ struct DenyOutboundHttp; impl WasiHttpHooks for DenyOutboundHttp { fn send_request( &mut self, - _request: hyper::Request, - _config: OutgoingRequestConfig, - ) -> HttpResult { - Err(ErrorCode::HttpRequestDenied.into()) + _request: hyper::Request, + _options: Option, + _fut: Box> + Send>, + ) -> Box< + dyn Future< + Output = Result< + ( + hyper::Response, + Box> + Send>, + ), + HttpError, + >, + > + Send, + > { + Box::new(async { Err(HttpError::HttpRequestDenied) }) } } diff --git a/src/wasm/nxt_rt_wasmtime.c b/src/wasm/nxt_rt_wasmtime.c index 64ef775de..6d9e401f1 100644 --- a/src/wasm/nxt_rt_wasmtime.c +++ b/src/wasm/nxt_rt_wasmtime.c @@ -58,6 +58,40 @@ nxt_wasmtime_err_msg(wasmtime_error_t *error, wasm_trap_t *trap, } +/* + * memory.grow can move the linear memory to a new address. Read the base + * again after the guest has run, and before an import function reads or + * writes the memory. The value is valid only until the guest runs again. + */ +static void +nxt_wasmtime_update_baddr(nxt_wasm_ctx_t *ctx, wasmtime_context_t *wctx) +{ + ctx->baddr = wasmtime_memory_data(wctx, &nxt_wasmtime_ctx.memory) + + ctx->baddr_off; +} + + +/* + * The guest can call an import before the host serves a request: from its + * start function or "_initialize" in wasmtime_linker_module(), from the + * malloc handler, or from the module_init hook. The host has no request to + * answer then. Before nxt_wasmtime_init_memory() it also has no handle for + * the memory. An import that needs a request traps in that case. The host + * serves a request only after nxt_wasmtime_init() succeeds, so a request + * implies a memory handle. The host does not clear ctx->req after a + * request, so this check does not show that a request is still live. + */ +static wasm_trap_t * +nxt_wasmtime_check_request(const nxt_wasm_ctx_t *ctx, const char *msg) +{ + if (ctx->req != NULL) { + return NULL; + } + + return wasmtime_trap_new(msg, strlen(msg)); +} + + static wasm_trap_t * nxt_wasm_get_init_mem_size(void *env, wasmtime_caller_t *caller, const wasmtime_val_t *args, size_t nargs, @@ -74,6 +108,14 @@ nxt_wasm_response_end(void *env, wasmtime_caller_t *caller, const wasmtime_val_t *args, size_t nargs, wasmtime_val_t *results, size_t nresults) { + wasm_trap_t *trap; + + trap = nxt_wasmtime_check_request(env, "nxt_wasm_response_end() called " + "outside a request"); + if (trap != NULL) { + return trap; + } + nxt_wasm_do_response_end(env); return NULL; @@ -85,6 +127,16 @@ nxt_wasm_send_response(void *env, wasmtime_caller_t *caller, const wasmtime_val_t *args, size_t nargs, wasmtime_val_t *results, size_t nresults) { + wasm_trap_t *trap; + + trap = nxt_wasmtime_check_request(env, "nxt_wasm_send_response() called " + "outside a request"); + if (trap != NULL) { + return trap; + } + + nxt_wasmtime_update_baddr(env, wasmtime_caller_context(caller)); + nxt_wasm_do_send_response(env, args[0].of.i32); return NULL; @@ -96,6 +148,16 @@ nxt_wasm_send_headers(void *env, wasmtime_caller_t *caller, const wasmtime_val_t *args, size_t nargs, wasmtime_val_t *results, size_t nresults) { + wasm_trap_t *trap; + + trap = nxt_wasmtime_check_request(env, "nxt_wasm_send_headers() called " + "outside a request"); + if (trap != NULL) { + return trap; + } + + nxt_wasmtime_update_baddr(env, wasmtime_caller_context(caller)); + nxt_wasm_do_send_headers(env, args[0].of.i32); return NULL; @@ -116,7 +178,7 @@ nxt_wasm_set_resp_status(void *env, wasmtime_caller_t *caller, static void -nxt_wasmtime_execute_hook(const nxt_wasm_ctx_t *ctx, nxt_wasm_fh_t hook) +nxt_wasmtime_execute_hook(nxt_wasm_ctx_t *ctx, nxt_wasm_fh_t hook) { const char *name = ctx->fh[hook].func_name; wasm_trap_t *trap = NULL; @@ -129,6 +191,9 @@ nxt_wasmtime_execute_hook(const nxt_wasm_ctx_t *ctx, nxt_wasm_fh_t hook) } error = wasmtime_func_call(rt_ctx->ctx, func, NULL, 0, NULL, 0, &trap); + + nxt_wasmtime_update_baddr(ctx, rt_ctx->ctx); + if (error != NULL || trap != NULL) { nxt_wasmtime_err_msg(error, trap, "failed to call hook function [%s]", name); @@ -137,7 +202,7 @@ nxt_wasmtime_execute_hook(const nxt_wasm_ctx_t *ctx, nxt_wasm_fh_t hook) static int -nxt_wasmtime_execute_request(const nxt_wasm_ctx_t *ctx) +nxt_wasmtime_execute_request(nxt_wasm_ctx_t *ctx) { int i = 0; wasm_trap_t *trap = NULL; @@ -151,6 +216,9 @@ nxt_wasmtime_execute_request(const nxt_wasm_ctx_t *ctx) args[i++].of.i32 = ctx->baddr_off; error = wasmtime_func_call(rt_ctx->ctx, func, args, i, results, 1, &trap); + + nxt_wasmtime_update_baddr(ctx, rt_ctx->ctx); + if (error != NULL || trap != NULL) { nxt_wasmtime_err_msg(error, trap, "failed to call function [->wasm_request_handler]" @@ -258,6 +326,14 @@ nxt_wasmtime_get_function_exports(nxt_wasm_ctx_t *ctx) ctx->fh[i].func_name); return -1; } + + if (item.kind != WASMTIME_EXTERN_FUNC) { + nxt_wasmtime_err_msg(NULL, NULL, + "module export (%s) is not a function", + ctx->fh[i].func_name); + return -1; + } + ctx->fh[i].func = item.of.func; } @@ -281,7 +357,9 @@ nxt_wasmtime_wasi_init(const nxt_wasm_ctx_t *ctx) wasi_config_inherit_stderr(wasi_config); for (dir = ctx->dirs; dir != NULL && *dir != NULL; dir++) { -#if defined(WASMTIME_VERSION_MAJOR) && (WASMTIME_VERSION_MAJOR >= 27) +#if defined(WASMTIME_VERSION_MAJOR) && (WASMTIME_VERSION_MAJOR >= 48) + wasi_config_preopen_dir(wasi_config, *dir, *dir, true); +#elif defined(WASMTIME_VERSION_MAJOR) && (WASMTIME_VERSION_MAJOR >= 27) wasi_config_preopen_dir(wasi_config, *dir, *dir, WASMTIME_WASI_DIR_PERMS_READ|WASMTIME_WASI_DIR_PERMS_WRITE, WASMTIME_WASI_FILE_PERMS_READ|WASMTIME_WASI_FILE_PERMS_WRITE); @@ -300,11 +378,50 @@ nxt_wasmtime_wasi_init(const nxt_wasm_ctx_t *ctx) } +/* + * The host passes and reads wasm32 offsets. A 64-bit memory has no 4 GiB + * limit, so it can grow past the address space that wasmtime reserved for + * it, and wasmtime then moves it. Refuse such a memory before the host + * calls the malloc handler. + */ +static int +nxt_wasmtime_check_memory(void) +{ + bool ok, is64; + wasmtime_extern_t item; + wasm_memorytype_t *type; + nxt_wasmtime_ctx_t *rt_ctx = &nxt_wasmtime_ctx; + + ok = wasmtime_linker_get(rt_ctx->linker, rt_ctx->ctx, "", 0, "memory", + strlen("memory"), &item); + if (!ok || item.kind != WASMTIME_EXTERN_MEMORY) { + nxt_wasmtime_err_msg(NULL, NULL, "couldn't get 'memory' from module"); + return -1; + } + + type = wasmtime_memory_type(rt_ctx->ctx, &item.of.memory); + is64 = wasmtime_memorytype_is64(type); + wasm_memorytype_delete(type); + + if (is64) { + nxt_wasmtime_err_msg(NULL, NULL, + "module memory is 64-bit; only a 32-bit memory " + "is supported"); + return -1; + } + + return 0; +} + + static int nxt_wasmtime_init_memory(nxt_wasm_ctx_t *ctx) { int i = 0; bool ok; + size_t need, size; + uint8_t *base; + uint32_t off; wasm_trap_t *trap = NULL; wasmtime_val_t args[1] = { }; wasmtime_val_t results[1] = { }; @@ -313,8 +430,10 @@ nxt_wasmtime_init_memory(nxt_wasm_ctx_t *ctx) nxt_wasmtime_ctx_t *rt_ctx = &nxt_wasmtime_ctx; const nxt_wasm_func_t *func = &ctx->fh[NXT_WASM_FH_MALLOC].func; + need = NXT_WASM_MEM_SIZE + NXT_WASM_PAGE_SIZE; + args[i].kind = WASMTIME_I32; - args[i++].of.i32 = NXT_WASM_MEM_SIZE + NXT_WASM_PAGE_SIZE; + args[i++].of.i32 = need; error = wasmtime_func_call(rt_ctx->ctx, func, args, i, results, 1, &trap); if (error != NULL || trap != NULL) { @@ -332,10 +451,36 @@ nxt_wasmtime_init_memory(nxt_wasm_ctx_t *ctx) } rt_ctx->memory = item.of.memory; - ctx->baddr_off = results[0].of.i32; - ctx->baddr = wasmtime_memory_data(rt_ctx->ctx, &rt_ctx->memory); + /* + * The guest returns a wasm32 pointer. Read it as unsigned, and check + * that the bytes the host asked for fit in the linear memory from it. + */ + off = (uint32_t) results[0].of.i32; + size = wasmtime_memory_data_size(rt_ctx->ctx, &rt_ctx->memory); - ctx->baddr += ctx->baddr_off; + if (size < need || off > size - need) { + nxt_wasmtime_err_msg(NULL, NULL, + "malloc handler returned offset %u outside " + "memory of %zu bytes", off, size); + return -1; + } + + base = wasmtime_memory_data(rt_ctx->ctx, &rt_ctx->memory); + + /* + * The host writes an nxt_wasm_request_t at this address. The structure + * has uint64_t members, so the address must be aligned for it. + */ + if ((uintptr_t) (base + off) % _Alignof(nxt_wasm_request_t) != 0) { + nxt_wasmtime_err_msg(NULL, NULL, + "malloc handler returned offset %u that is not " + "aligned to %zu bytes", off, + _Alignof(nxt_wasm_request_t)); + return -1; + } + + ctx->baddr_off = off; + ctx->baddr = base + off; return 0; } @@ -403,6 +548,11 @@ nxt_wasmtime_init(nxt_wasm_ctx_t *ctx) return -1; } + err = nxt_wasmtime_check_memory(); + if (err) { + return -1; + } + err = nxt_wasmtime_init_memory(ctx); if (err) { return -1; diff --git a/src/wasm/nxt_wasm.c b/src/wasm/nxt_wasm.c index 7a7689efa..8c320a5dc 100644 --- a/src/wasm/nxt_wasm.c +++ b/src/wasm/nxt_wasm.c @@ -312,6 +312,9 @@ nxt_wasm_request_handler(nxt_unit_request_info_t *req) offset = sizeof(nxt_wasm_request_t); do { + /* exec_request() read the base again; the memory may have moved. */ + wr = (nxt_wasm_request_t *) nxt_wasm_ctx.baddr; + read_bytes = nxt_min(content_len - content_sent, NXT_WASM_MEM_SIZE - offset); bytes_read = nxt_unit_request_read(req, (uint8_t *)wr + offset, diff --git a/src/wasm/nxt_wasm.h b/src/wasm/nxt_wasm.h index 6bc3ae35a..f301798a9 100644 --- a/src/wasm/nxt_wasm.h +++ b/src/wasm/nxt_wasm.h @@ -117,7 +117,7 @@ struct nxt_wasm_ctx_s { nxt_unit_request_info_t *req; uint8_t *baddr; - size_t baddr_off; + uint32_t baddr_off; size_t response_off; @@ -127,8 +127,8 @@ struct nxt_wasm_ctx_s { struct nxt_wasm_operations_s { int (*init)(nxt_wasm_ctx_t *ctx); void (*destroy)(const nxt_wasm_ctx_t *ctx); - int (*exec_request)(const nxt_wasm_ctx_t *ctx); - void (*exec_hook)(const nxt_wasm_ctx_t *ctx, nxt_wasm_fh_t hook); + int (*exec_request)(nxt_wasm_ctx_t *ctx); + void (*exec_hook)(nxt_wasm_ctx_t *ctx, nxt_wasm_fh_t hook); }; extern const nxt_wasm_operations_t nxt_wasm_ops; diff --git a/test/README.md b/test/README.md index 7dcb6259a..7460c88f9 100644 --- a/test/README.md +++ b/test/README.md @@ -53,7 +53,7 @@ build tools. ``` The script (`test/run-local.sh`) builds a `freeunit-test:local` image -mirroring `pkg/docker/template.Dockerfile` (Debian trixie, Rust 1.94.1, +mirroring `pkg/docker/template.Dockerfile` (Debian trixie, Rust 1.95.0, njs 0.9.6, system libssl-dev). Source code is mounted via Docker volume, so changes on the host are immediately reflected. The container then builds FreeUnit with `--tests --openssl --njs --zlib --zstd --brotli --otel`, @@ -70,12 +70,12 @@ docker rmi freeunit-test:local `run-local.sh` builds a full test image from scratch (downloads Rust, Go, njs) — slow for tight iteration. For prototyping a **proxy / TLS** test (no language runtime needed), reuse the pre-built builder image -`ghcr.io/freeunitorg/freeunit-builder:trixie-rust1.94.1` (Rust + all C build +`ghcr.io/freeunitorg/freeunit-builder:trixie-rust1.95.0` (Rust + all C build deps already baked in) and just mount the working tree. Build + run is ~30 s: ```bash docker run --rm --privileged -v "$(pwd):/unit" -w /unit \ - ghcr.io/freeunitorg/freeunit-builder:trixie-rust1.94.1 bash -c ' + ghcr.io/freeunitorg/freeunit-builder:trixie-rust1.95.0 bash -c ' apt-get update -qq && apt-get install -y -qq python3-pytest python3-openssl ./configure --openssl --tests make -j"$(nproc)" unitd @@ -106,6 +106,11 @@ core and `fake_upstream` rebuild incrementally. If you prefer to run tests natively (requires all dependencies installed): +The test tools come from the distribution (`python3-pytest`, `python3-openssl`) +or from `pip install -r test/requirements.txt`. That file needs Python 3.10 or +newer, because older pytest and pyOpenSSL releases have known vulnerabilities. +On a host with an older Python, use `./test/run-local.sh` instead. + ```bash # 1. Build FreeUnit with test support ./configure --openssl --njs --zlib --zstd --brotli --otel --tests diff --git a/test/capget_filter.c b/test/capget_filter.c new file mode 100644 index 000000000..ac1f709b7 --- /dev/null +++ b/test/capget_filter.c @@ -0,0 +1,296 @@ +/* + * LD_PRELOAD shim that makes capget(2) fail the way a syscall filter + * makes it fail, for test_capget_fallback.py. + * + * src/nxt_capability.c reaches the kernel through the nxt_capget() + * macro, which expands to glibc's syscall() wrapper -- an ordinary + * interposable symbol, not a compiler builtin or an inline "syscall" + * instruction. So a preloaded definition of syscall() shadows it and + * we can inject an errno without needing seccomp, a privileged + * helper, or a container. + * + * The interposition is total: every syscall() in the process, from + * every library, comes through here. Only SYS_capget is ever + * answered locally, and only when NXT_TEST_CAPGET_ERRNO names an + * errno; everything else is forwarded to the real symbol untouched. + * + * Set NXT_TEST_CAPGET_RECORD to a file path to have every pid whose + * capget() we failed appended to it, one per line. The test uses + * that to show which processes the filter actually reached -- the + * environment is inherited by unitd's children, so "no other process + * was affected" has to be demonstrated rather than assumed. + * + * That path is copied, not kept as the getenv() pointer, and the copy + * is what makes the demonstration mean anything. getenv() returns a + * pointer into the contiguous argv+environ block, and + * nxt_process_title() (src/nxt_process_title.c) writes the process + * title over that block and zero-pads the remainder, so in every + * process unitd forks the inherited pointer would read as "" and + * open("") would fail silently. The main process looked correct only + * by accident of ordering: nxt_capability_set() runs from + * nxt_runtime_conf_init(), before nxt_main_process_title() + * (src/nxt_main_process.c:96) rewrites the area. A record file that + * no child could have written to proves nothing about what the + * children did. + * + * NXT_TEST_CAPSET_RECORD does the same for capset(2). That is how the + * test observes nxt_capability_drop(): the drop has no visible effect + * in a suite that runs without any capability to drop, so "the app has + * no capabilities" would pass just as well without the code. Which + * *processes* called capset() is the part that cannot pass by + * accident: main must never appear (it keeps CAP_NET_BIND_SERVICE for + * bind()), the prototype must, and the application worker must not, + * because it inherits the prototype's already-emptied sets rather than + * dropping again. + * + * NXT_TEST_CAPGET_HOLD is the other direction: a hex mask that makes + * capget(2) *succeed* and report that mask in the permitted, effective + * and inheritable sets of the first data element. The suite runs as an + * ordinary user with no capabilities, so without it there is nothing to + * refuse over -- nxt_capability_still_held() would correctly answer + * "empty" and the refusal below would never fire. Faking the answer is + * what lets an unprivileged pytest reach the code the harness smoke test + * reaches with real ambient capabilities. + * + * The mask the test passes is CAP_NET_BIND_SERVICE, deliberately: main's + * own capget() sees it too, and nxt_capability_specific_set() only looks + * for CAP_SETUID, CAP_SETGID and CAP_SYS_CHROOT, so unitd still starts + * and behaves exactly like the unprivileged process it really is. A + * mask containing CAP_SETUID would make it attempt a setuid() it cannot + * perform. + * + * NXT_TEST_CAPSET_ERRNO makes capset(2) fail the same way, which is + * the other half of a "SystemCallFilter=" allowlist: a profile that + * omits capget() almost always omits capset() too. The call is still + * recorded before it is failed, so the record says who *tried*. Unit + * answers that differently per process -- a prototype refuses to start + * rather than fork workers that would inherit what it could not drop, + * while the router, controller and discovery warn and carry on -- so + * the test needs to be able to produce it. + * + * Build: + * cc -shared -fPIC -o capget_filter.so capget_filter.c -ldl + */ + +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +/* + * glibc declares syscall() variadic, but it is an assembly stub that + * only moves registers: it never walks a va_list, so calling it + * through a fixed six-argument prototype is what every syscall + * interposer does and is correct on every Linux ABI Unit builds for. + */ +typedef long (*nxt_syscall_fn)(long, long, long, long, long, long, long); + +static nxt_syscall_fn nxt_real_syscall; +static int nxt_capget_errno; /* 0: filter disabled */ +static int nxt_capset_errno; /* 0: filter disabled */ +static unsigned nxt_capget_hold; /* 0: report the real sets */ +static char *nxt_record_path; /* strdup()ed, see above */ +static char *nxt_capset_record_path; + + +static void +nxt_shim_init(void) +{ + const char *value; + + nxt_real_syscall = (nxt_syscall_fn) dlsym(RTLD_NEXT, "syscall"); + + value = getenv("NXT_TEST_CAPGET_ERRNO"); + + if (value != NULL && *value != '\0') { + nxt_capget_errno = (int) strtol(value, NULL, 10); + } + + value = getenv("NXT_TEST_CAPSET_ERRNO"); + + if (value != NULL && *value != '\0') { + nxt_capset_errno = (int) strtol(value, NULL, 10); + } + + value = getenv("NXT_TEST_CAPGET_HOLD"); + + if (value != NULL && *value != '\0') { + nxt_capget_hold = (unsigned) strtoul(value, NULL, 16); + } + + value = getenv("NXT_TEST_CAPGET_RECORD"); + + if (nxt_record_path == NULL && value != NULL && *value != '\0') { + nxt_record_path = strdup(value); + } + + value = getenv("NXT_TEST_CAPSET_RECORD"); + + if (nxt_capset_record_path == NULL && value != NULL && *value != '\0') { + nxt_capset_record_path = strdup(value); + } +} + + +__attribute__((constructor)) +static void +nxt_shim_ctor(void) +{ + nxt_shim_init(); +} + + +static void +nxt_shim_record(const char *path) +{ + int fd, len; + char buf[32]; + + if (path == NULL) { + return; + } + + /* + * open()/write()/close() are their own glibc stubs, not calls to + * syscall(), so this cannot recurse back into the shim. + */ + + fd = open(path, O_WRONLY | O_CREAT | O_APPEND, 0644); + + if (fd < 0) { + return; + } + + len = snprintf(buf, sizeof(buf), "%ld\n", (long) getpid()); + + if (len > 0) { + (void) !write(fd, buf, (size_t) len); + } + + close(fd); +} + + +long +syscall(long number, ...) +{ + long a0, a1, a2, a3, a4, a5; + va_list ap; + + if (nxt_real_syscall == NULL) { + /* + * A constructor in another preloaded object may run before + * ours and call syscall() from it. + */ + nxt_shim_init(); + + if (nxt_real_syscall == NULL) { + /* + * Nothing to forward to. Fail loudly: silently returning + * an error here would corrupt an unrelated syscall and + * make the test lie about what it exercised. + */ + _exit(127); + } + } + + /* + * Reading six arguments when the caller passed fewer yields + * garbage in the unused ones, which the kernel ignores for every + * syscall that does not declare them. + * + * ISO C calls that undefined (C17 7.16.1.1p2: va_arg with no + * actual next argument), and there is no portable fix -- a + * variadic forwarder cannot learn its own argument count, and + * re-declaring syscall() with a fixed prototype only trades this + * undefined behaviour for calling a function through an + * incompatible type. It is left as-is deliberately, because the + * concrete behaviour is the one every implementation already + * depends on: + * + * - glibc's syscall() is an assembly stub that unconditionally + * moves six argument registers, and musl's is a fixed + * seven-parameter C function. Neither ever walks a va_list, + * so both read exactly the slots this reads. + * - The slots read are always mapped memory, on every ABI Unit + * builds for. On x86-64 SysV the first five come from the + * register save area gcc/clang spill at entry and the sixth + * from the caller's own frame via overflow_arg_area; on + * aarch64 all seven fit the eight-register save area; on + * 32-bit ABIs the excess reads land in the caller's frame. + * None of them can fault, and none of them is used: only + * `number` decides what happens below. + * + * Whether the shim should install a real seccomp filter instead + * was weighed and rejected: a filter needs PR_SET_NO_NEW_PRIVS, + * after which it is inherited across both fork() and execve(), so + * every unitd child would see capget() fail too -- and the record + * file below, whose whole value is showing that only main was + * affected, could no longer say anything. It would also filter + * capset() in the children, which is the call the drop depends on. + */ + + va_start(ap, number); + a0 = va_arg(ap, long); + a1 = va_arg(ap, long); + a2 = va_arg(ap, long); + a3 = va_arg(ap, long); + a4 = va_arg(ap, long); + a5 = va_arg(ap, long); + va_end(ap); + +#ifdef SYS_capget + if (number == SYS_capget && nxt_capget_errno != 0) { + nxt_shim_record(nxt_record_path); + errno = nxt_capget_errno; + return -1; + } + + /* + * Only when a data pointer was passed: nxt_capability_linux_get_ + * version() calls capget() with NULL to have the kernel rewrite the + * header, and answering that one locally would leave the header + * untouched and change what Unit thinks the kernel supports. + */ + if (number == SYS_capget && nxt_capget_hold != 0 && (void *) a1 != NULL) { + struct __user_cap_data_struct *d = (void *) a1; + + memset(d, 0, 2 * sizeof(*d)); + + d[0].effective = nxt_capget_hold; + d[0].permitted = nxt_capget_hold; + d[0].inheritable = nxt_capget_hold; + + return 0; + } +#endif + +#ifdef SYS_capset + if (number == SYS_capset) { + /* + * Recorded first, then failed only if asked. The record is + * who *tried* to drop, which is what the unfiltered run + * asserts on; failing afterwards keeps that meaning intact + * for the filtered run, where the point is which processes + * survive a denial and which refuse to start. + */ + nxt_shim_record(nxt_capset_record_path); + + if (nxt_capset_errno != 0) { + errno = nxt_capset_errno; + return -1; + } + } +#endif + + return nxt_real_syscall(number, a0, a1, a2, a3, a4, a5); +} diff --git a/test/conftest.py b/test/conftest.py index 008125653..e8f9e13f2 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,6 +1,5 @@ import atexit import fcntl -import inspect import json import os import re @@ -11,6 +10,7 @@ import sys import tempfile import time +from collections import Counter from multiprocessing import Process from pathlib import Path @@ -83,12 +83,19 @@ def pytest_addoption(parser): # unitd tree. _pgids = {} _fds_info = { - 'main': {'fds': 0, 'skip': False}, - 'router': {'name': 'unit: router', 'pid': -1, 'fds': 0, 'skip': False}, + 'main': {'fds': 0, 'kinds': {}, 'skip': False}, + 'router': { + 'name': 'unit: router', + 'pid': -1, + 'fds': 0, + 'kinds': {}, + 'skip': False, + }, 'controller': { 'name': 'unit: controller', 'pid': -1, 'fds': 0, + 'kinds': {}, 'skip': False, }, } @@ -216,7 +223,6 @@ def run(request): r'sendmsg.+failed', r'recvmsg.+failed', ] - option.skip_sanitizer = False _fds_info['main']['skip'] = False _fds_info['router']['skip'] = False @@ -226,11 +232,11 @@ def run(request): # lazy initialization (e.g. OTel tokio runtime) that occurred after # unit_run() captured the initial baseline is not counted as a leak. if not option.restart: - _fds_info['main']['fds'] = _count_fds(unit_instance['pid']) + _fd_baseline(_fds_info['main'], unit_instance['pid']) router = _fds_info['router'] - router['fds'] = _count_fds(router['pid']) + _fd_baseline(router, router['pid']) controller = _fds_info['controller'] - controller['fds'] = _count_fds(controller['pid']) + _fd_baseline(controller, controller['pid']) yield @@ -253,6 +259,14 @@ def run(request): if not option.restart: _clear_conf(log=log) + + # Workers must be gone before their files are. _clear_conf() returns + # on the controller's ack, but the router quits workers asynchronously, + # and a java one still in scanClasses() keeps reading temp_dir. + # Only wait here: the identity asserts belong after _check_fds(), + # which is what refreshes the router/controller pids a respawn test + # has changed. + _wait_for_processes() _clear_temp_dir() # check descriptors @@ -263,6 +277,15 @@ def run(request): _check_processes() + # Teardown logs too, after the snapshot at the top of this fixture, so + # read again or those lines are charged to the next test. Not under + # --restart: it stops unitd and may have deleted the log with temp_dir. + + if not option.restart: + with Log.open() as f: + log += f.read() + Log.set_pos(f.tell()) + # print unit.log in case of error if hasattr(request.node, 'rep_call') and request.node.rep_call.failed: @@ -476,6 +499,17 @@ def unit_run(state_dir=None): option.temp_dir = temporary_dir public_dir(temporary_dir) + # Every start gets a fresh temp dir and therefore a fresh, empty + # unit.log, so the read offsets recorded against the previous one are + # meaningless here. Teardown saves that offset unconditionally and only + # resets it when it removes the temp dir, which --save-log stops it from + # doing: without this, a --save-log --restart run seeks each test's reads + # to the size of the PREVIOUS test's log. Everything log-based then sees + # a tail slice or nothing at all -- Log.wait_for_record misses records + # that are present, and, worse silently, the teardown Log.check_alerts() + # and _check_fds() stop examining most of what they are given. + Log.pos.clear() + if oct(stat.S_IMODE(Path(builddir).stat().st_mode)) != '0o777': public_dir(builddir) @@ -510,7 +544,22 @@ def unit_run(state_dir=None): # router, app workers) shares one group we can signal as a unit — and # which can never be confused with the pytest runner's own group or an # unrelated unitd already running on the box. - p = subprocess.Popen(unitd_args, stderr=log, start_new_session=True) + # UNIT_PYTHONHOME pins the embedded interpreter's stdlib to the + # prefix unit's python module was built against. It cannot be passed + # as PYTHONHOME from outside: unitd inherits this process's + # environment, and PYTHONHOME here would also rebind pytest's own + # interpreter, dropping dist-packages (where pytest itself lives) from + # its sys.path. Needed when unit is built against a non-system + # interpreter whose minor version matches the system one, because + # CPython then resolves its prefix by finding "python3" on PATH. + unitd_env = os.environ.copy() + pythonhome = unitd_env.pop('UNIT_PYTHONHOME', None) + if pythonhome: + unitd_env['PYTHONHOME'] = pythonhome + + p = subprocess.Popen( + unitd_args, stderr=log, start_new_session=True, env=unitd_env + ) unit_instance['process'] = p # Record the group id (== leader pid) immediately, on disk and in-memory, @@ -540,15 +589,15 @@ def unit_run(state_dir=None): if state_dir is None: _clear_conf() - _fds_info['main']['fds'] = _count_fds(unit_instance['pid']) + _fd_baseline(_fds_info['main'], unit_instance['pid']) router = _fds_info['router'] router['pid'] = pid_by_name(router['name']) - router['fds'] = _count_fds(router['pid']) + _fd_baseline(router, router['pid']) controller = _fds_info['controller'] controller['pid'] = pid_by_name(controller['name']) - controller['fds'] = _count_fds(controller['pid']) + _fd_baseline(controller, controller['pid']) Status._check_zeros() @@ -557,9 +606,6 @@ def unit_run(state_dir=None): def unit_stop(): if not option.restart: - if inspect.stack()[1].function.startswith('test_'): - pytest.skip('no restart mode') - return # Startup may have failed before the process/pid were recorded; nothing to @@ -730,9 +776,7 @@ def _clear_temp_dir(): time.sleep(1) -def _check_processes(): - router_pid = _fds_info['router']['pid'] - controller_pid = _fds_info['controller']['pid'] +def _wait_for_processes(): main_pid = unit_instance['pid'] for _ in range(600): @@ -753,6 +797,16 @@ def _check_processes(): time.sleep(0.1) + return out + + +def _check_processes(): + router_pid = _fds_info['router']['pid'] + controller_pid = _fds_info['controller']['pid'] + main_pid = unit_instance['pid'] + + out = _wait_for_processes() + if option.restart: assert len(out) == 0, 'all termimated' return @@ -797,11 +851,14 @@ def waitforfds(diff): lambda: _count_fds(unit_instance['pid']) - ps['fds'] ) ps['fds'] += fds_diff + before, ps['kinds'] = ps['kinds'], _fd_kinds(unit_instance['pid']) - assert fds_diff <= option.fds_threshold, 'descriptors leak main process' + assert fds_diff <= option.fds_threshold, _fd_leak( + 'main process', fds_diff, before, ps['kinds'] + ) else: - ps['fds'] = _count_fds(unit_instance['pid']) + _fd_baseline(ps, unit_instance['pid']) for name in ['controller', 'router']: ps = _fds_info[name] @@ -811,14 +868,136 @@ def waitforfds(diff): if not ps['skip']: fds_diff = waitforfds(lambda: _count_fds(ps['pid']) - ps['fds']) ps['fds'] += fds_diff + before, ps['kinds'] = ps['kinds'], _fd_kinds(ps['pid']) if not option.restart: assert ps['pid'] == ps_pid, f'same pid {name}' - assert fds_diff <= option.fds_threshold, f'descriptors leak {name}' + assert fds_diff <= option.fds_threshold, _fd_leak( + name, fds_diff, before, ps['kinds'] + ) else: - ps['fds'] = _count_fds(ps['pid']) + _fd_baseline(ps, ps['pid']) + + +def _fd_baseline(ps, pid): + ps['fds'] = _count_fds(pid) + ps['kinds'] = _fd_kinds(pid) + + +_TCP_STATES = { + '01': 'established', + '02': 'syn-sent', + '03': 'syn-recv', + '04': 'fin-wait1', + '05': 'fin-wait2', + '07': 'close', + '08': 'close-wait', + '09': 'last-ack', + '0A': 'listen', + '0B': 'closing', +} + + +def _socket_names(pid): + names = {} + + for proto in ('tcp', 'tcp6', 'udp', 'udp6', 'netlink', 'unix'): + try: + lines = ( + Path(f'/proc/{pid}/net/{proto}') + .read_bytes() + .decode('utf-8', 'backslashreplace') + .splitlines()[1:] + ) + except OSError: + continue + + for line in lines: + f = line.split(maxsplit=7 if proto == 'unix' else -1) + + if proto == 'unix': + if len(f) < 7: + continue + path = f[7] if len(f) > 7 else '' + if re.fullmatch(r'@[0-9a-f]{5}', path): + path = '@autobind' + names[f[6]] = f'unix:{path}' if path else 'unix' + + elif len(f) > 9: + if proto.startswith('tcp'): + name = f'{proto} {_TCP_STATES.get(f[3], f[3])}' + else: + name = proto + names[f[9]] = name + + return names + + +def _fd_kind(target, sockets): + if target.startswith('socket:'): + return sockets.get(target[len('socket:') :].strip('[]'), 'socket') + + for prefix, kind in ( + ('pipe:', 'pipe'), + ('/memfd:', 'memfd'), + ): + if target.startswith(prefix): + return kind + + if target.startswith('anon_inode:'): + return target[len('anon_inode:') :].strip('[]') + + return target + + +def _fd_kinds(pid): + kinds = Counter() + procfile = Path(f'/proc/{pid}/fd') + + try: + fds = list(procfile.iterdir()) + except OSError: + return kinds + + targets = [] + + for fd in fds: + try: + targets.append(os.readlink(fd)) + except OSError: + pass + + sockets = ( + _socket_names(pid) + if any(t.startswith('socket:') for t in targets) + else {} + ) + + for target in targets: + kinds[_fd_kind(target, sockets)] += 1 + + return kinds + + +def _fd_leak(name, fds_diff, before, after): + kinds = _fd_kinds_diff(before, after) + return f'descriptors leak {name}: {fds_diff:+d} fds, {kinds}' + + +def _fd_kinds_diff(before, after): + if not before and not after: + return 'no /proc fd detail' + + delta = Counter(after) + delta.subtract(before) + changed = sorted((k, n) for k, n in delta.items() if n) + + if not changed: + return 'same descriptor kinds as the baseline' + + return ', '.join(f'{k} {n:+d}' for k, n in changed) def _count_fds(pid): @@ -923,8 +1102,12 @@ def is_su(): @pytest.fixture -def is_unsafe(request): - return request.config.getoption("--unsafe") +def requires_restart(): + # unit_stop() does nothing without --restart, so a test that calls it in + # its body cannot check anything after the call. Skip such a test at + # setup. + if not option.restart: + pytest.skip('no restart mode') @pytest.fixture diff --git a/test/fake_otlp/Cargo.lock b/test/fake_otlp/Cargo.lock index 7a15315cd..07f80e752 100644 --- a/test/fake_otlp/Cargo.lock +++ b/test/fake_otlp/Cargo.lock @@ -4,9 +4,9 @@ version = 4 [[package]] name = "anyhow" -version = "1.0.102" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "async-trait" @@ -82,9 +82,9 @@ checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" [[package]] name = "bytes" -version = "1.11.1" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "cfg-if" @@ -197,9 +197,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.14" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" +checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" dependencies = [ "atomic-waker", "bytes", @@ -222,9 +222,9 @@ checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" [[package]] name = "http" -version = "1.4.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8be7462df143984c4598a256ef469b251d7d7f9e271135073e78fc535414f3d0" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" dependencies = [ "bytes", "itoa", @@ -232,9 +232,9 @@ dependencies = [ [[package]] name = "http-body" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" dependencies = [ "bytes", "http", @@ -242,9 +242,9 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.3" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" dependencies = [ "bytes", "futures-core", @@ -267,9 +267,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hyper" -version = "1.10.1" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" dependencies = [ "atomic-waker", "bytes", @@ -302,15 +302,16 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.20" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" dependencies = [ "bytes", "futures-channel", "futures-util", "http", "http-body", + "httparse", "hyper", "libc", "pin-project-lite", @@ -400,9 +401,9 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "opentelemetry" -version = "0.32.0" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0142c63252a9e054e68a4c61a5778f7b14f576274d593f8ce883d191a099682" +checksum = "6cdb0b1b267eb9db3331b434ed9ddab10d50e280a9adf9d13e5233e2002b61b5" dependencies = [ "futures-core", "futures-sink", @@ -413,9 +414,9 @@ dependencies = [ [[package]] name = "opentelemetry-proto" -version = "0.32.0" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56d658ba1faf63f7b9c492cfbe6e0ec365440a16132d3270c1065f7b33f1b638" +checksum = "25da1ac11a0aeccf38d7f77ee0348715adaf8340f65ad46c94a02c6b20e2f65d" dependencies = [ "opentelemetry", "opentelemetry_sdk", @@ -426,9 +427,9 @@ dependencies = [ [[package]] name = "opentelemetry_sdk" -version = "0.32.1" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b59f80e1ac4d5ff7a2db8fb6c80badb7f0f3f858211fba08dd9aaec750894f9" +checksum = "cb39533d9d1c912123efd7d41d7e0c29d16917b60ce15b4c8d87cb1af7f67520" dependencies = [ "futures-channel", "futures-executor", @@ -498,9 +499,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" dependencies = [ "bytes", "prost-derive", @@ -508,9 +509,9 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", "itertools", @@ -650,9 +651,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.52.3" +version = "1.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" dependencies = [ "bytes", "libc", diff --git a/test/fake_otlp/Cargo.toml b/test/fake_otlp/Cargo.toml index ed7b8dabb..6d8a5f2b4 100644 --- a/test/fake_otlp/Cargo.toml +++ b/test/fake_otlp/Cargo.toml @@ -2,6 +2,7 @@ name = "fake_otlp" version = "0.1.0" edition = "2021" +rust-version = "1.88" [[bin]] name = "fake_otlp" @@ -16,7 +17,7 @@ path = "src/main.rs" tonic = "0.14" prost = "0.14" tokio = { version = "1", features = ["rt-multi-thread", "macros"] } -opentelemetry-proto = { version = "0.32", default-features = false, features = [ +opentelemetry-proto = { version = "0.33", default-features = false, features = [ "gen-tonic", "trace", ] } diff --git a/test/fake_otlp/README.md b/test/fake_otlp/README.md index f8c0bfdcc..030ec47e3 100644 --- a/test/fake_otlp/README.md +++ b/test/fake_otlp/README.md @@ -27,6 +27,7 @@ built — mirroring FreeUnit's single `--otel` build. See PLAN.md. ``` fake_otlp --port [--protocol http|grpc] [--requests ] [--dump ] + [--status ] ``` | Flag | Meaning | @@ -35,6 +36,7 @@ fake_otlp --port [--protocol http|grpc] [--requests ] [--dump ] | `--protocol P` | transport (table below); default `http` | | `--requests N` | exit after N export requests (default: run forever) | | `--dump FILE` | append each received request (raw bytes) to FILE | +| `--status CODE` | answer each valid export with this HTTP status (default 200). For `grpc`, any other code answers `UNAVAILABLE`. The export is still counted and dumped. | ### Protocols diff --git a/test/fake_otlp/src/grpc.rs b/test/fake_otlp/src/grpc.rs index 6566ff25e..08c7a4acb 100644 --- a/test/fake_otlp/src/grpc.rs +++ b/test/fake_otlp/src/grpc.rs @@ -27,6 +27,7 @@ struct Collector { dump: Option, count: Arc, max_requests: Option, + status: u16, shutdown: Arc, } @@ -59,13 +60,22 @@ impl TraceService for Collector { self.shutdown.notify_one(); } + if self.status != 200 { + return Err(Status::unavailable("fake_otlp: --status failure")); + } + Ok(Response::new(ExportTraceServiceResponse::default())) } } /// Serve OTLP/gRPC until `max_requests` exports have arrived (or forever when /// `None`). Runs on a multi-thread tokio runtime owned by this thread. -pub fn serve_grpc(port: u16, max_requests: Option, dump: Option) { +pub fn serve_grpc( + port: u16, + max_requests: Option, + dump: Option, + status: u16, +) { let rt = tokio::runtime::Builder::new_multi_thread() .enable_all() .build() @@ -77,6 +87,7 @@ pub fn serve_grpc(port: u16, max_requests: Option, dump: Option) dump, count: Arc::new(AtomicUsize::new(0)), max_requests, + status, shutdown: shutdown.clone(), }; diff --git a/test/fake_otlp/src/main.rs b/test/fake_otlp/src/main.rs index bf3ef18ad..3d73651f0 100644 --- a/test/fake_otlp/src/main.rs +++ b/test/fake_otlp/src/main.rs @@ -5,12 +5,16 @@ /// /// Usage: /// fake_otlp --port [--protocol http|grpc] [--requests ] [--dump ] +/// [--status ] /// /// --port N TCP port to listen on (127.0.0.1) /// --protocol P transport: "http" (default) or "grpc" /// (HTTP/2 unary TraceService/Export) /// --requests N exit after receiving N export requests (default: forever) /// --dump FILE append each received request to FILE +/// --status CODE answer each valid export with this HTTP status (default +/// 200); for grpc, any code other than 200 answers UNAVAILABLE. +/// The export is still counted and dumped. /// /// HTTP behaviour: /// * Accepts POST /v1/traces with Content-Type: application/x-protobuf and a @@ -150,7 +154,7 @@ pub fn dump_request(dump: Option<&str>, bytes: &[u8]) { /// received, so a bare TCP probe (the test harness's readiness check, which /// connects and immediately closes) and malformed requests are not counted /// against --requests and are not written to the dump. -fn handle(mut stream: TcpStream, dump: Option<&str>) -> bool { +fn handle(mut stream: TcpStream, dump: Option<&str>, status: u16) -> bool { let req = match read_request(&mut stream) { Ok(r) => r, Err(_) => return false, @@ -185,13 +189,17 @@ fn handle(mut stream: TcpStream, dump: Option<&str>) -> bool { ); let _ = std::io::stdout().flush(); - respond(&mut stream, "200 OK"); + if status == 200 { + respond(&mut stream, "200 OK"); + } else { + respond(&mut stream, &format!("{status} Fake Failure")); + } true } /// Serve OTLP/HTTP until `max_requests` valid exports have been received (or /// forever when `None`). -fn serve_http(port: u16, max_requests: Option, dump: Option<&str>) { +fn serve_http(port: u16, max_requests: Option, dump: Option<&str>, status: u16) { let listener = TcpListener::bind((HOST, port)).unwrap_or_else(|e| { eprintln!("bind {HOST}:{port} — {e}"); process::exit(1); @@ -201,7 +209,7 @@ fn serve_http(port: u16, max_requests: Option, dump: Option<&str>) { for stream in listener.incoming() { match stream { Ok(s) => { - if handle(s, dump) { + if handle(s, dump, status) { count += 1; if max_requests.map_or(false, |n| count >= n) { break; @@ -221,7 +229,7 @@ fn serve_http(port: u16, max_requests: Option, dump: Option<&str>) { fn usage() -> ! { eprintln!( "Usage: fake_otlp --port [--protocol http|grpc] \ - [--requests ] [--dump ]" + [--requests ] [--dump ] [--status ]" ); process::exit(1); } @@ -233,6 +241,7 @@ fn main() { let mut protocol = String::from("http"); let mut max_requests: Option = None; let mut dump: Option = None; + let mut status: u16 = 200; let mut i = 1; while i < args.len() { @@ -255,6 +264,14 @@ fn main() { i += 1; dump = args.get(i).cloned(); } + "--status" => { + i += 1; + status = args + .get(i) + .and_then(|v| v.parse().ok()) + .filter(|v| (200..=599).contains(v)) + .unwrap_or_else(|| usage()); + } _ => {} } i += 1; @@ -263,8 +280,8 @@ fn main() { let port = port.unwrap_or_else(|| usage()); match protocol.as_str() { - "http" => serve_http(port, max_requests, dump.as_deref()), - "grpc" => grpc::serve_grpc(port, max_requests, dump), + "http" => serve_http(port, max_requests, dump.as_deref(), status), + "grpc" => grpc::serve_grpc(port, max_requests, dump, status), other => { eprintln!("fake_otlp: unknown --protocol {other:?} (use http or grpc)"); process::exit(1); diff --git a/test/fake_upstream/Cargo.toml b/test/fake_upstream/Cargo.toml index e7c04114d..d5e5cb509 100644 --- a/test/fake_upstream/Cargo.toml +++ b/test/fake_upstream/Cargo.toml @@ -2,6 +2,7 @@ name = "fake_upstream" version = "0.1.0" edition = "2021" +rust-version = "1.78" [[bin]] name = "fake_upstream" diff --git a/test/fake_upstream/README.md b/test/fake_upstream/README.md index 635d8b9f5..468fc9efe 100644 --- a/test/fake_upstream/README.md +++ b/test/fake_upstream/README.md @@ -76,6 +76,9 @@ single `grep` finds every side of a case. E.g. token `chunked_response`: | Port | Token | Mode (CLI) | Rust handler | pytest | |------|-------|-----------|--------------|--------| +| 7976 | — | — (plain-Python upstream in the test file) | — | `test_proxy_te.py` (upstream Transfer-Encoding framing: `chunked` in any letter case, lists, several field lines, Transfer-Encoding together with Content-Length) | +| 7977 | — | — (plain-Python upstream in the test file) | — | `test_proxy_interim.py` (the proxy drops upstream 1xx responses and sends the final one) | +| 7978 | — | — (plain-Python upstream in the test file) | — | `test_proxy_head.py` (bodyless upstream response: header block, with or without trailing bytes, then close — not a `fake_upstream` mode, reserved here so the 79xx block stays in one registry) | | 7979 | `dup_cl` | `dup-cl` | `respond_dup_cl` | `test_proxy_dup_cl_keepalive_disabled` (duplicate upstream Content-Length disables downstream keepalive for a keep-alive client; complete body keeps its terminal chunk) — out of the 7983–7999 block below, which is exhausted; 7980–7982 are `fake_otlp`, so 7974–7979 is the free gap | | 7983 | `dup_cl` | `dup-cl` | `respond_dup_cl` | `test_proxy_dup_cl_raw` (zero raw `Content-Length` occurrences on the wire, complete re-framed chunked body) | | 7984 | `dup_cl` | `dup-cl` | `respond_dup_cl` | `test_proxy_dup_cl` (duplicate upstream Content-Length → neither forwarded, body re-framed, #113) | @@ -96,7 +99,9 @@ single `grep` finds every side of a case. E.g. token `chunked_response`: > **7999 is not available here.** `test/test_proxy.py` and `test/test_proxy_chunked.py` > already bind `SERVER_PORT = 7999` for their own upstream, so this registry ends at -> 7998. New `fake_upstream` slots go to the 7974–7979 gap (7980–7982 are `fake_otlp`). +> 7998. New `fake_upstream` slots go to the 7974–7975 gap (7976 is taken by +> `test_proxy_te.py` and 7977 by `test_proxy_interim.py` above, 7978–7979 are +> taken above, 7980–7982 are `fake_otlp`). A test pins its port as a module constant referencing this table: diff --git a/test/java/server_name/app.java b/test/java/server_name/app.java new file mode 100644 index 000000000..1ce818a15 --- /dev/null +++ b/test/java/server_name/app.java @@ -0,0 +1,20 @@ +import java.io.IOException; + +import javax.servlet.ServletException; +import javax.servlet.annotation.WebServlet; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +@WebServlet("/") +public class app extends HttpServlet +{ + @Override + public void doGet(HttpServletRequest request, HttpServletResponse response) + throws IOException, ServletException + { + response.addHeader("X-Server-Name", request.getServerName()); + response.addHeader("X-Server-Port", + String.valueOf(request.getServerPort())); + } +} diff --git a/test/java/websockets_async/app.java b/test/java/websockets_async/app.java new file mode 100644 index 000000000..aa5c33520 --- /dev/null +++ b/test/java/websockets_async/app.java @@ -0,0 +1,287 @@ +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import javax.websocket.OnMessage; +import javax.websocket.OnOpen; +import javax.websocket.RemoteEndpoint; +import javax.websocket.SendHandler; +import javax.websocket.SendResult; +import javax.websocket.Session; +import javax.websocket.server.ServerEndpoint; + +/* + * A mirror app that echoes through the asynchronous remote (issue #434). + * + * A text message that starts with "handler:" is echoed with sendText(String, + * SendHandler); one that starts with "object:" with sendObject(Object), as an + * Integer when the rest of it is one and as a String otherwise; any other + * text message with the Future from sendText(String). A binary message + * whose first byte is 'h' is echoed with sendBinary(ByteBuffer, SendHandler), + * any other with the Future from sendBinary(ByteBuffer). + * + * After the echo the app reports what the asynchronous API told it, as a text + * frame sent through the blocking remote: "handler-ok" or "handler-fail: ..." + * once the SendHandler has run, "future-done" once Future.get() has returned, + * "future-timeout" when it has not returned within five seconds, or + * "future-fail: ..." when it threw. So a client sees exactly one of the two + * defects of #434 when the echo is missing, and the other when the report is. + * + * "batch:N" allows batching, sends N messages "b0".."b" through the + * asynchronous remote, then disallows batching again, which flushes them, and + * reports "batch-flushed" or "batch-fail: ...". "threads:N" sends "t0".."t" + * from N threads at once and reports "threads-done" once every future has + * completed, or the first failure or timeout. A binary message whose first + * byte is 'o' is sent batched from its fourth byte on; see offset(). + * "nested:..." waits on sends from inside a SendHandler; see nested(). + * "batchtext:..." is echoed batched and then flushed; see batchText(). + */ +@ServerEndpoint("/") +public class app { + + @OnOpen + public void onOpen(Session session) { + session.setMaxTextMessageBufferSize(16 * 1024 * 1024); + session.setMaxBinaryMessageBufferSize(16 * 1024 * 1024); + } + + @OnMessage + public void echoTextMessage(Session session, String msg) { + RemoteEndpoint.Async remote = session.getAsyncRemote(); + + if (msg.startsWith("batchtext:")) { + batchText(session, msg); + } else if (msg.startsWith("nested:")) { + nested(session, msg); + } else if (msg.startsWith("batch:")) { + batch(session, Integer.parseInt(msg.substring("batch:".length()))); + } else if (msg.startsWith("threads:")) { + threads(session, + Integer.parseInt(msg.substring("threads:".length()))); + } else if (msg.startsWith("handler:")) { + remote.sendText(msg, new Reporter(session)); + } else if (msg.startsWith("object:")) { + /* + * An Integer takes sendObject()'s primitive path and is echoed as + * its decimal string; anything else is sent as a String, which + * has no encoder, so the future must complete with a failure. + * That failure is this fork's behaviour (Util.isPrimitive() does + * not count String), not something the specification requires: + * the case is here to show a failed future completes. + */ + String arg = msg.substring("object:".length()); + Object obj; + try { + obj = Integer.valueOf(arg); + } catch (NumberFormatException e) { + obj = arg; + } + report(session, waitFor(remote.sendObject(obj))); + } else { + report(session, waitFor(remote.sendText(msg))); + } + } + + @OnMessage + public void echoBinaryMessage(Session session, ByteBuffer bb) { + RemoteEndpoint.Async remote = session.getAsyncRemote(); + + if (bb.remaining() > 3 && bb.get(bb.position()) == 'o') { + offset(session, bb); + } else if (bb.remaining() > 0 && bb.get(bb.position()) == 'h') { + remote.sendBinary(bb, new Reporter(session)); + } else { + report(session, waitFor(remote.sendBinary(bb))); + } + } + + private static void batch(Session session, int n) { + RemoteEndpoint.Async remote = session.getAsyncRemote(); + + try { + remote.setBatchingAllowed(true); + + for (int i = 0; i < n; i++) { + String status = waitFor(remote.sendText("b" + i)); + if (!status.equals("future-done")) { + report(session, "batch-fail: " + status); + return; + } + } + + remote.setBatchingAllowed(false); + } catch (Exception e) { + report(session, "batch-fail: " + e); + return; + } + + report(session, "batch-flushed"); + } + + /* + * With batching allowed, sends the message from its fourth byte on, as a + * buffer whose position() is not 0, then a text frame "after", flushes, + * and reports "offset-flushed". The header of the first frame has to + * give its real length, or the second one is read as part of it. + */ + private static void offset(Session session, ByteBuffer bb) { + RemoteEndpoint.Async remote = session.getAsyncRemote(); + + try { + remote.setBatchingAllowed(true); + + ByteBuffer copy = ByteBuffer.allocate(bb.remaining()); + copy.put(bb); + copy.flip(); + copy.position(3); + + String status = waitFor(remote.sendBinary(copy)); + if (status.equals("future-done")) { + status = waitFor(remote.sendText("after")); + } + if (!status.equals("future-done")) { + report(session, "offset-fail: " + status); + return; + } + + remote.setBatchingAllowed(false); + } catch (Exception e) { + report(session, "offset-fail: " + e); + return; + } + + report(session, "offset-flushed"); + } + + /* + * Echoes msg with a SendHandler whose onResult() waits on a send of its + * own, "nested-b", then batches "nested-c" and flushes it, and reports + * "nested-done", or the first failure. Both waits need the nested + * send's completion to run before onResult() returns. + */ + private static void nested(Session session, String msg) { + RemoteEndpoint.Async remote = session.getAsyncRemote(); + + remote.sendText(msg, result -> { + String status = waitFor(remote.sendText("nested-b")); + if (!status.equals("future-done")) { + report(session, "nested-fail: " + status); + return; + } + + try { + remote.setBatchingAllowed(true); + status = waitFor(remote.sendText("nested-c")); + remote.setBatchingAllowed(false); + } catch (Exception e) { + report(session, "nested-fail: " + e); + return; + } + + report(session, status.equals("future-done") + ? "nested-done" : "nested-fail: " + status); + }); + } + + /* + * Echoes msg with batching allowed, then disallows it, which flushes the + * echo, and reports "batchtext-flushed" or "batchtext-fail: ...". + */ + private static void batchText(Session session, String msg) { + RemoteEndpoint.Async remote = session.getAsyncRemote(); + + try { + remote.setBatchingAllowed(true); + String status = waitFor(remote.sendText(msg)); + remote.setBatchingAllowed(false); + + report(session, status.equals("future-done") + ? "batchtext-flushed" : "batchtext-fail: " + status); + } catch (Exception e) { + report(session, "batchtext-fail: " + e); + } + } + + private static void threads(Session session, int n) { + RemoteEndpoint.Async remote = session.getAsyncRemote(); + List threads = new ArrayList<>(); + String[] status = new String[n]; + + for (int i = 0; i < n; i++) { + final int k = i; + Thread t = new Thread(() -> { + /* + * A second send on the endpoint while the first is in flight + * is refused by the specification, so retry until this one + * gets its turn; what is under test is that every accepted + * send completes. + */ + for (;;) { + try { + status[k] = waitFor(remote.sendText("t" + k)); + return; + } catch (IllegalStateException e) { + Thread.yield(); + } + } + }); + threads.add(t); + t.start(); + } + + for (int i = 0; i < n; i++) { + try { + threads.get(i).join(10000); + } catch (InterruptedException e) { + report(session, "threads-fail: " + e); + return; + } + if (!"future-done".equals(status[i])) { + report(session, "threads-fail: " + i + " " + status[i]); + return; + } + } + + report(session, "threads-done"); + } + + private static String waitFor(Future future) { + try { + future.get(5, TimeUnit.SECONDS); + } catch (java.util.concurrent.TimeoutException e) { + return "future-timeout"; + } catch (Exception e) { + return "future-fail: " + e; + } + return "future-done"; + } + + private static void report(Session session, String status) { + try { + session.getBasicRemote().sendText(status); + } catch (IOException e) { + // Ignore + } + } + + private static class Reporter implements SendHandler { + private final Session session; + + Reporter(Session session) { + this.session = session; + } + + @Override + public void onResult(SendResult result) { + if (result.isOK()) { + report(session, "handler-ok"); + } else { + report(session, "handler-fail: " + result.getException()); + } + } + } +} diff --git a/test/java/websockets_binary_toobig/app.java b/test/java/websockets_binary_toobig/app.java new file mode 100644 index 000000000..7f94f0bc6 --- /dev/null +++ b/test/java/websockets_binary_toobig/app.java @@ -0,0 +1,58 @@ +import java.io.IOException; +import java.nio.ByteBuffer; + +import javax.websocket.OnMessage; +import javax.websocket.OnOpen; +import javax.websocket.PongMessage; +import javax.websocket.Session; +import javax.websocket.server.ServerEndpoint; + +/* + * A minimal mirror app with a small binary buffer, so a fragmented binary + * message that overflows it can be exercised with a handful of bytes + * instead of the 16 MiB the shared websockets_mirror app now requires + * (see issue #435). + */ +@ServerEndpoint("/") +public class app { + + @OnOpen + public void onOpen(Session session) { + session.setMaxBinaryMessageBufferSize(1024); + } + + @OnMessage + public void echoTextMessage(Session session, String msg) { + try { + if (session.isOpen()) { + session.getBasicRemote().sendText(msg, true); + } + } catch (IOException e) { + try { + session.close(); + } catch (IOException e1) { + // Ignore + } + } + } + + @OnMessage + public void echoBinaryMessage(Session session, ByteBuffer bb) { + try { + if (session.isOpen()) { + session.getBasicRemote().sendBinary(bb, true); + } + } catch (IOException e) { + try { + session.close(); + } catch (IOException e1) { + // Ignore + } + } + } + + @OnMessage + public void echoPongMessage(PongMessage pm) { + // NO-OP + } +} diff --git a/test/java/websockets_buffers/app.java b/test/java/websockets_buffers/app.java new file mode 100644 index 000000000..a5229b13e --- /dev/null +++ b/test/java/websockets_buffers/app.java @@ -0,0 +1,41 @@ +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.TimeUnit; + +import javax.websocket.OnMessage; +import javax.websocket.Session; +import javax.websocket.server.ServerEndpoint; + +/* + * Sends the bytes "456789ab" from a heap buffer that does not start at its + * array's first byte (issue #486): a slice for ":slice", a read-only + * buffer for ":readonly", through the async remote when is + * "async" and the blocking one otherwise. Then reports "done" or + * "fail: ...". + */ +@ServerEndpoint("/") +public class app { + + private static final byte[] DATA = + "0123456789abcdef".getBytes(StandardCharsets.US_ASCII); + + @OnMessage + public void onMessage(Session session, String msg) throws IOException { + ByteBuffer bb = ByteBuffer.wrap(DATA, 4, 8); + bb = msg.endsWith(":slice") ? bb.slice() : bb.asReadOnlyBuffer(); + + try { + if (msg.startsWith("async:")) { + session.getAsyncRemote().sendBinary(bb).get(5, TimeUnit.SECONDS); + } else { + session.getBasicRemote().sendBinary(bb); + } + } catch (Exception e) { + session.getBasicRemote().sendText("fail: " + e); + return; + } + + session.getBasicRemote().sendText("done"); + } +} diff --git a/test/java/websockets_mirror/app.java b/test/java/websockets_mirror/app.java index ada60231b..deeb0c9c8 100644 --- a/test/java/websockets_mirror/app.java +++ b/test/java/websockets_mirror/app.java @@ -12,7 +12,8 @@ public class app { @OnOpen public void onOpen(Session session) { - session.setMaxTextMessageBufferSize(8388608); + session.setMaxTextMessageBufferSize(16 * 1024 * 1024); + session.setMaxBinaryMessageBufferSize(16 * 1024 * 1024); } @OnMessage diff --git a/test/php/async_mirror/entrypoint.php b/test/php/async_mirror/entrypoint.php deleted file mode 100644 index 81103115c..000000000 --- a/test/php/async_mirror/entrypoint.php +++ /dev/null @@ -1,20 +0,0 @@ -body(); - $request->respond(200, [ - 'Content-Type' => 'text/plain', - 'Content-Length' => (string) strlen($body), - ], $body); -}); diff --git a/test/php/async_shutdown/entrypoint.php b/test/php/async_shutdown/entrypoint.php deleted file mode 100644 index 3365720bb..000000000 --- a/test/php/async_shutdown/entrypoint.php +++ /dev/null @@ -1,22 +0,0 @@ -respond(200, ['Content-Type' => 'text/plain'], "OK\n"); -}); diff --git a/test/php/async_slow/entrypoint.php b/test/php/async_slow/entrypoint.php deleted file mode 100644 index 554e7f8e7..000000000 --- a/test/php/async_slow/entrypoint.php +++ /dev/null @@ -1,41 +0,0 @@ -query(), $params); - - $delay = min((int) ($params['sleep'] ?? 3), 30); - - // Write a sentinel file before sleeping so the test process can - // detect that this handler is genuinely in-flight (not just queued - // in the kernel TCP buffer). The path is passed as ?signal=. - // Writing the worker PID lets the test double-check which process is - // handling the request. - if (!empty($params['signal'])) { - file_put_contents($params['signal'], (string) getmypid()); - } - - // Async sleep: suspends this coroutine without blocking the event - // loop, allowing other coroutines (and the quit handler) to run. - \Async\sleep($delay); - - $request->respond(200, ['Content-Type' => 'text/plain'], "done\n"); -}); diff --git a/test/php/big_response/index.php b/test/php/big_response/index.php new file mode 100644 index 000000000..6516a0fc1 --- /dev/null +++ b/test/php/big_response/index.php @@ -0,0 +1,20 @@ + mebibytes of 'x' in 64 KiB writes. Each write goes through + * the SAPI's ub_write -> nxt_unit_response_write(), which allocates the bytes + * out of the application's outgoing shared-memory segments, so a response + * larger than the application's "limits"/"shm" budget cannot be in flight all + * at once: the allocator runs the segment's free map dry and Unit's + * application library reports the out-of-shared-memory condition to the + * router (nxt_unit_send_oosm()). + */ +$mb = isset($_GET['mb']) ? (int) $_GET['mb'] : 32; + +header('Content-Type: text/plain'); +header('Content-Length: ' . ($mb * 1024 * 1024)); + +$chunk = str_repeat('x', 65536); + +for ($i = 0; $i < $mb * 16; $i++) { + echo $chunk; +} diff --git a/test/php/comp_coded_body/index.php b/test/php/comp_coded_body/index.php new file mode 100644 index 000000000..db8428181 --- /dev/null +++ b/test/php/comp_coded_body/index.php @@ -0,0 +1,15 @@ + 0) { + sleep($hold); + + $done = isset($_GET['done']) ? (string) $_GET['done'] : ''; + + if ($done !== '') { + file_put_contents($done, (string) getmypid()); + } + + echo "done"; + + return; +} + +$before = isset($_GET['before']) ? (int) $_GET['before'] : 0; + +if ($before > 0) { + sleep($before); +} + +echo "done"; + +fastcgi_finish_request(); + +$sleep = isset($_GET['sleep']) ? (int) $_GET['sleep'] : 0; + +if ($sleep > 0) { + sleep($sleep); +} + +$done = isset($_GET['done']) ? (string) $_GET['done'] : ''; + +if ($done !== '') { + file_put_contents($done, (string) getmypid()); +} + +$after = isset($_GET['after']) ? (string) $_GET['after'] : ''; + +if ($after === 'exit') { + exit(0); +} + +if ($after === 'fatal') { + /* An uncaught Error: a fatal path that runs after the response went out. */ + throw new Error('detached fatal'); +} diff --git a/test/php/flush/index.php b/test/php/flush/index.php new file mode 100644 index 000000000..60dff035b --- /dev/null +++ b/test/php/flush/index.php @@ -0,0 +1,20 @@ + diff --git a/test/php/flush_after_finish/index.php b/test/php/flush_after_finish/index.php new file mode 100644 index 000000000..429949f4c --- /dev/null +++ b/test/php/flush_after_finish/index.php @@ -0,0 +1,19 @@ + diff --git a/test/php/no_body_status/index.php b/test/php/no_body_status/index.php new file mode 100644 index 000000000..ef2179920 --- /dev/null +++ b/test/php/no_body_status/index.php @@ -0,0 +1,15 @@ +' % rid + part = (token * (size // len(token) + 1))[:size] + + start_response( + '200 OK', + [ + ('Content-Type', 'text/plain'), + ('Content-Length', str(size * parts)), + ], + ) + + def body(): + for i in range(parts): + if i: + time.sleep(delay) + + yield part + + return body() diff --git a/test/python/content_encoding/wsgi.py b/test/python/content_encoding/wsgi.py new file mode 100644 index 000000000..0a965571e --- /dev/null +++ b/test/python/content_encoding/wsgi.py @@ -0,0 +1,15 @@ +def application(environ, start_response): + # Echo the request body, labelled as gzip: the application says that it + # coded the body itself. + content_length = int(environ.get('CONTENT_LENGTH', 0)) + body = bytes(environ['wsgi.input'].read(content_length)) + + start_response( + '200', + [ + ('Content-Type', 'text/plain'), + ('Content-Length', str(len(body))), + ('Content-Encoding', 'gzip'), + ], + ) + return [body] diff --git a/test/python/header_then_body/asgi.py b/test/python/header_then_body/asgi.py new file mode 100644 index 000000000..3cf29b4e9 --- /dev/null +++ b/test/python/header_then_body/asgi.py @@ -0,0 +1,27 @@ +import asyncio +from urllib.parse import parse_qs + + +async def application(scope, receive, send): + assert scope['type'] == 'http' + + query = parse_qs(scope['query_string'].decode()) + + delay = int(query.get('delay', ['0'])[0]) + body = query.get('body', [''])[0].encode() + + await send( + { + 'type': 'http.response.start', + 'status': 200, + 'headers': [(b'x-header', b'1')], + } + ) + + # Unit sends the response header for a body message without "body". + # An empty "body" sends nothing. + await send({'type': 'http.response.body', 'more_body': True}) + + await asyncio.sleep(delay) + + await send({'type': 'http.response.body', 'body': body}) diff --git a/test/python/ns_inspect/wsgi.py b/test/python/ns_inspect/wsgi.py index fa1222e4f..d7798bfa8 100644 --- a/test/python/ns_inspect/wsgi.py +++ b/test/python/ns_inspect/wsgi.py @@ -16,7 +16,16 @@ def application(environ, start_response): d = parse_qs(environ['QUERY_STRING']) - ret['FileExists'] = os.path.exists(d.get('path')[0]) + if 'path' in d: + ret['FileExists'] = os.path.exists(d['path'][0]) + + if 'read' in d: + try: + with open(d['read'][0], 'r') as f: + ret['FileContent'] = f.read() + + except IOError: + ret['FileContent'] = None out = json.dumps(ret) diff --git a/test/python/send_bursts/wsgi.py b/test/python/send_bursts/wsgi.py new file mode 100644 index 000000000..8fd23960b --- /dev/null +++ b/test/python/send_bursts/wsgi.py @@ -0,0 +1,31 @@ +import time + + +def application(environ, start_response): + # Sends the body in bursts of parts. The first burst has X-First parts, + # each next burst has X-Parts parts. A burst starts each X-Interval + # seconds. The parts of a burst are 10 ms apart, so the router gets and + # writes each part alone. + first = int(environ.get('HTTP_X_FIRST', 1)) + parts = int(environ.get('HTTP_X_PARTS', 1)) + size = int(environ.get('HTTP_X_PART_SIZE', 1)) + bursts = int(environ.get('HTTP_X_BURSTS', 1)) + interval = float(environ.get('HTTP_X_INTERVAL', 0)) + + length = size * (first + parts * (bursts - 1)) + + write = start_response('200', [('Content-Length', str(length))]) + + part = b'x' * size + start = time.monotonic() + + for burst in range(bursts): + delay = start + burst * interval - time.monotonic() + if delay > 0: + time.sleep(delay) + + for _ in range(first if burst == 0 else parts): + write(part) + time.sleep(0.01) + + return [] diff --git a/test/python/slow_start/wsgi.py b/test/python/slow_start/wsgi.py new file mode 100644 index 000000000..be17d09d4 --- /dev/null +++ b/test/python/slow_start/wsgi.py @@ -0,0 +1,16 @@ +import os +import time + +# Widen the window between the worker's PROCESS_CREATED and its PROCESS_READY, +# which is what test_python_prototype_killed_mid_start() kills the prototype +# inside of. The delay is at import time, before `application` is even +# defined: nxt_python_start() imports the module and only then sends READY +# (src/python/nxt_python.c), so a sleep here lands squarely in that window, +# while a sleep inside application() would not -- the process is ready long +# before a request arrives. +time.sleep(float(os.environ.get('UNIT_SLOW_START', '4'))) + + +def application(environ, start_response): + start_response('200 OK', [('Content-Length', '0')]) + return [] diff --git a/test/python/traceparent/wsgi.py b/test/python/traceparent/wsgi.py new file mode 100644 index 000000000..2f628eb62 --- /dev/null +++ b/test/python/traceparent/wsgi.py @@ -0,0 +1,7 @@ +def application(environ, start_response): + traceparent = environ.get('HTTP_TRACEPARENT', '') + start_response( + '200', + [('Content-Length', '0'), ('X-Seen-Traceparent', traceparent)], + ) + return [] diff --git a/test/requirements.txt b/test/requirements.txt index 5f94fad23..3d63500cf 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,2 +1,4 @@ -pyOpenSSL>=20.0.1 -pytest>=6.0.1 +# Needs Python 3.10 or newer (pytest 9). On an older host, run the tests in +# a container with test/run-local.sh instead; see test/README.md. +pyOpenSSL>=26.0.0 +pytest>=9.0.3 diff --git a/test/run-local.sh b/test/run-local.sh index 1d9975cac..919937d18 100755 --- a/test/run-local.sh +++ b/test/run-local.sh @@ -180,7 +180,7 @@ RUN set -ex \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev \ zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf \ libclang-dev cmake python3-pytest python3-openssl sudo procps \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ diff --git a/test/security/docker/README.md b/test/security/docker/README.md new file mode 100644 index 000000000..189cb1f1c --- /dev/null +++ b/test/security/docker/README.md @@ -0,0 +1,92 @@ +# Docker hardening matrix + +`test-hardening.sh` starts the FreeUnit image once per hardening posture with +the same trivial Python application and asserts the outcome. It is +documentation that runs: the point is not that any of these postures is +required, but that the file cannot go stale about which ones work. + +```sh +./test/security/docker/test-hardening.sh # published image +./test/security/docker/test-hardening.sh freeunit:local # a locally built one +``` + +CI runs it as the last step of `.github/workflows/security-seccomp.yml`, +against the image that workflow has already built from the in-repo Dockerfile. +That workflow also triggers on `src/nxt_capability.*`, `src/nxt_credential.*`, +`src/nxt_isolation.*`, `src/nxt_process.*` and `src/nxt_runtime.*`, since +those are the files that decide what the table below says. + +The platform is pinned to `linux/amd64` -- what CI runs, and so that a +multi-arch tag cannot resolve to something else by ambient default. On an +arm64 host set `UNIT_HARDENING_PLATFORM=linux/arm64`. + +## What it establishes + +| posture | outcome | +|---|---| +| root, default capabilities | serves 200 | +| root, `--security-opt no-new-privileges` | serves 200 | +| root, `--cap-drop=ALL` | refused, never reaches readiness: `setgid(999) failed (1: Operation not permitted)` | +| `--user 1000:1000` + writable tmpfs on state/run/tmp | serves 200 | +| `--user 1000:1000 --cap-drop=ALL` + tmpfs | serves 200 | +| `--user 1000:1000 --cap-drop=ALL --security-opt no-new-privileges` + tmpfs | serves 200 | + +Every cell runs under Docker's **default seccomp profile**, unchanged. + +Cells assert a cause, not just an outcome. The refusal cell greps the +container log for `setgid(999) failed`, because a wholly broken image would +satisfy "no 200" too. The five serving cells check two things, both anchored to end of line so that +`gid=999` cannot be satisfied by `gid=9990`: + +- what the canary reports about **itself** -- `uid`, `gid` *and* + `os.getgroups()`, since a process that drops its primary gid while keeping + group 0 has root-group access under another name. It has to come from the + application: `docker exec ... id` reports the container's configured user, + a different process, and would look right even if Unit dropped the + application to the wrong credentials. +- the **router**'s uid and gid, read from `/proc` *inside* the container, + because a router still running as root would serve a correctly-dropped + application perfectly well. Not `docker top`: that reports host-side uids, + which under daemon user-namespace remapping are the remapped ones and would + turn every cell red. The scan matches the *start* of `cmdline` and skips + its own pid -- the probing shell's command line contains the string it is + searching for, and in a non-root cell it runs as 1000, so a substring match + would have reported the right answer whatever the router did. + +Root cells expect `uid=999 gid=999 groups=999` and a router at 999 -- the +image's `unit` user, so the drop is measured rather than assumed. Non-root +cells expect 1000 throughout. A 200 on its own says nothing about who served +it, and the flag list and the label describing it did in fact drift apart +once while this was being written. + +The wait is two-phase for the same reason. `docker-entrypoint.sh` runs a +temporary daemon to apply the initial configuration, which activates the +listener, and only then execs the real one; polling immediately can take a +200 from the throwaway and pass while production startup is broken. Each +cell waits for the entrypoint's "ready for start up" line first. + +Three things worth taking away: + +- **`--cap-drop=ALL` and root do not mix.** The daemon's first act is to drop + the router and applications to the unprivileged `unit` user, and that needs + CAP_SETGID/CAP_SETUID. Dropping capabilities is worth doing *after* the + process is already non-root, not instead of it. +- **`no-new-privileges` is free.** `setuid` from root is not a privilege gain, + so the bit never applies to what Unit does; it costs nothing in either + posture. +- **A bare `--tmpfs /var/lib/unit` is not enough.** Docker creates a tmpfs + root-owned and 0755, so a non-root container still gets EACCES on the state, + run and tmp directories. The ownership has to be spelled out: + `--tmpfs /var/lib/unit:uid=1000,gid=1000`. This is the step people miss. +- **`--user 1000` is not `--user 1000:1000`.** Uid 1000 has no passwd entry + in this image, so a bare uid resolves its primary group to GID 0 and the + container keeps root-group access. Spell out the group. + +## What it does not establish + +It does not test the shipped `pkg/docker/seccomp-no-af-alg.json` profile -- +that has its own test in `test/security/seccomp/` -- and it makes no claim +that Docker's default seccomp profile blocks `capget` or `mount`. Measured +against docker 29.7.2 it blocks neither: `capget` is allowed by name in +moby's default profile, and `mount` is gated on CAP_SYS_ADMIN plus the default +AppArmor profile, not on seccomp. Documentation that says otherwise is wrong. diff --git a/test/security/docker/app/config.json b/test/security/docker/app/config.json new file mode 100644 index 000000000..aff0109bb --- /dev/null +++ b/test/security/docker/app/config.json @@ -0,0 +1,15 @@ +{ + "listeners": { + "*:8080": { + "pass": "applications/canary" + } + }, + "applications": { + "canary": { + "type": "python", + "path": "/www/", + "module": "wsgi", + "processes": 1 + } + } +} diff --git a/test/security/docker/app/wsgi.py b/test/security/docker/app/wsgi.py new file mode 100644 index 000000000..3578792a1 --- /dev/null +++ b/test/security/docker/app/wsgi.py @@ -0,0 +1,24 @@ +"""Smallest application that proves the whole request path works. + +It reports the credentials of the process that served the request. That is +the only way the hardening matrix can check *who* answered: `docker exec ... +id` would report the container's configured user, which is a different +process and would look right even if Unit dropped to the wrong credentials. + +Supplementary groups are part of that: a process can drop its primary gid and +still hold group 0 if setgroups() was skipped, which is root-group access by +another name. +""" + +import os + + +def application(environ, start_response): + body = ("docker hardening canary uid=%d gid=%d groups=%s\n" + % (os.getuid(), os.getgid(), + ",".join(str(g) for g in sorted(os.getgroups())))).encode() + start_response( + "200 OK", + [("Content-Type", "text/plain"), ("Content-Length", str(len(body)))], + ) + return [body] diff --git a/test/security/docker/test-hardening.sh b/test/security/docker/test-hardening.sh new file mode 100755 index 000000000..f07f573d5 --- /dev/null +++ b/test/security/docker/test-hardening.sh @@ -0,0 +1,315 @@ +#!/bin/sh +# +# What the FreeUnit image can and cannot do under each common Docker +# hardening posture. Every cell starts the same image with the same trivial +# Python application and asserts the documented outcome -- a served 200, or a +# refusal at a named point. +# +# This is documentation that runs. It is not a claim that any of these +# postures is required, and it deliberately does not claim that Docker's +# default seccomp profile blocks capget or mount: measured against docker +# 29.7.2 it blocks neither (capget is allowed by name; mount is gated on +# CAP_SYS_ADMIN and AppArmor). Every cell below therefore runs with the +# default seccomp profile, unchanged; the profile FreeUnit does ship +# (pkg/docker/seccomp-no-af-alg.json) has its own test next door in +# test/security/seccomp/. +# +# Usage: +# ./test/security/docker/test-hardening.sh [IMAGE] +# +# IMAGE defaults to the published python-slim flavour. Requires docker and +# curl on the host. + +set -eu + +IMAGE="${1:-ghcr.io/freeunitorg/freeunit:latest-python-3.13-slim}" +HERE=$(cd "$(dirname "$0")" && pwd) +APP="$HERE/app" +PORT="${UNIT_HARDENING_PORT:-8099}" +# Pinned rather than left to the daemon's ambient default, so a multi-arch tag +# cannot silently resolve to something other than what CI runs. Override it +# on an arm64 host, where the arm64 manifest is the artifact worth testing: +# UNIT_HARDENING_PLATFORM=linux/arm64 ./test-hardening.sh +PLATFORM="${UNIT_HARDENING_PLATFORM:-linux/amd64}" + +PASS=0 +FAIL=0 +NAME= +RESULT= + +die() { echo "test-hardening.sh: $*" >&2; exit 1; } + +command -v docker >/dev/null 2>&1 || die "docker not found" +command -v curl >/dev/null 2>&1 || die "curl not found" + +LOGDIR=$(mktemp -d) + +cleanup() { + [ -n "$NAME" ] && { docker rm -f "$NAME" >/dev/null 2>&1 || :; } + rm -rf "$LOGDIR" + return 0 +} + +# A trapped signal does not end a POSIX shell: without an explicit exit the +# script would resume, with LOGDIR already deleted underneath it, and report +# an ordinary test error instead of stopping. The EXIT trap is cleared first +# so cleanup runs once. +trap cleanup EXIT +trap 'trap - EXIT; cleanup; exit 130' INT +trap 'trap - EXIT; cleanup; exit 143' TERM + +# Start the image with the given docker flags, wait for the application, and +# set RESULT to "200", "no-200" or "no-start". It sets a variable rather than +# echoing, because a $(command substitution) would run it in a subshell: NAME +# would never reach the parent, and the EXIT/INT/TERM trap could not remove a +# container left behind by a Ctrl-C -- leaking both the container and the +# published port. Never fails the script itself: a cell that is *expected* +# to be refused has to be able to report the refusal. +attempt() { + NAME="unit-hardening-$$" + docker rm -f "$NAME" >/dev/null 2>&1 || : + + # No --rm: a container that exits on its own would delete itself along + # with the logs, and the logs are the whole point of a refused cell. + # shellcheck disable=SC2086 # $1 is a deliberate list of docker flags + docker run -d --name "$NAME" --platform "$PLATFORM" \ + -v "$APP:/www:ro" \ + -v "$APP/config.json:/docker-entrypoint.d/config.json:ro" \ + -p "127.0.0.1:$PORT:8080" \ + $1 "$IMAGE" >/dev/null 2>"$LOGDIR/start.err" || { + cp "$LOGDIR/start.err" "$LOGDIR/container.log" + # docker run can fail *after* creating the container -- a port + # already allocated, for one -- leaving it in Created state. The + # trap cannot reach it once NAME is cleared, and the next run + # picks a different pid-based name, so remove it here. + docker rm -f "$NAME" >/dev/null 2>&1 || : + NAME= + RESULT=no-start + return 0 + } + + # docker-entrypoint.sh runs a *temporary* daemon to apply the initial + # configuration -- which activates this listener -- and only then execs + # the real one. Polling straight away can therefore get a 200 out of the + # throwaway daemon and pass while the production startup is broken. Wait + # for the entrypoint to say it is done first. + # Absolute deadlines, not iteration counts: with a per-request timeout an + # iteration bound is a bound on attempts, not on time, and a listener that + # accepts and never answers would spend a hundred five-second timeouts + # here -- eight minutes in one cell, against a twenty-minute job. + ready=no + deadline=$(( $(date +%s) + 45 )) + while [ "$(date +%s)" -lt "$deadline" ]; do + if docker logs "$NAME" 2>&1 | grep -q 'ready for start up'; then + ready=yes + break + fi + docker inspect -f '{{.State.Running}}' "$NAME" 2>/dev/null \ + | grep -q true || break + sleep 0.2 + done + + # Never probe an unready container. Exhausting the loop while it is still + # running means the entrypoint stalled after applying the configuration -- + # with the temporary daemon still answering, so a probe would take a 200 + # from it and pass the very failure this wait exists to catch. + if [ "$ready" = no ]; then + docker logs "$NAME" > "$LOGDIR/container.log" 2>&1 || : + running=$(docker inspect -f '{{.State.Running}}' "$NAME" 2>/dev/null || echo false) + docker rm -f "$NAME" >/dev/null 2>&1 || : + NAME= + if [ "$running" = true ]; then + RESULT=no-ready + else + RESULT=no-200 + fi + return 0 + fi + + deadline=$(( $(date +%s) + 30 )) + while [ "$(date +%s)" -lt "$deadline" ]; do + # --max-time as well as the deadline: without it one wedged request + # blocks past the deadline the loop can only check between attempts. + code=$(curl -s --max-time 5 -o "$LOGDIR/body.txt" -w '%{http_code}' \ + "http://127.0.0.1:$PORT/" 2>/dev/null || echo 000) + if [ "$code" = "200" ] && grep -q 'docker hardening canary' "$LOGDIR/body.txt"; then + docker logs "$NAME" > "$LOGDIR/container.log" 2>&1 || : + # The canary reports the credentials of the process that served + # the request, so a cell can assert who answered rather than + # trusting the flags in its own label. + sed 's/^/response: /' "$LOGDIR/body.txt" >> "$LOGDIR/container.log" + # ...and the router's own credentials, which the canary cannot + # speak for: a router still running as root would serve a + # correctly-dropped application just fine. Read from /proc inside + # the container rather than `docker top`, which reports host-side + # uids -- under daemon user-namespace remapping those are the + # remapped ones and would fail every cell. The image has no ps. + # + # Match on the *start* of cmdline, not a substring of it: this + # shell's own cmdline contains the pattern it is searching for, + # and in a non-root cell it runs as 1000 -- so a substring match + # would report the right answer no matter what the router did. + docker exec "$NAME" sh -c ' + for d in /proc/[0-9]*; do + [ "$d" = "/proc/$$" ] && continue + c=$(tr "\0" " " < "$d/cmdline" 2>/dev/null) || continue + case "$c" in "unit: router"*) ;; *) continue ;; esac + awk "/^Uid:/ {u=\$2} /^Gid:/ {g=\$2} + END {print \"router uid=\" u \" gid=\" g}" \ + "$d/status" + done' 2>/dev/null \ + | sed 's/^/proc: /' >> "$LOGDIR/container.log" || : + docker rm -f "$NAME" >/dev/null 2>&1 || : + NAME= + RESULT=200 + return 0 + fi + # A container that has already exited will never answer. + docker inspect -f '{{.State.Running}}' "$NAME" 2>/dev/null \ + | grep -q true || break + sleep 0.2 + done + + docker logs "$NAME" > "$LOGDIR/container.log" 2>&1 || : + docker rm -f "$NAME" >/dev/null 2>&1 || : + NAME= + RESULT=no-200 +} + +# cell DESCRIPTION 200|refused FLAGS [LOG-SUBSTRING] +# +# LOG-SUBSTRING is how a cell is pinned to its cause. "no-200" on its own is +# satisfied by a wholly broken image, so a cell documenting a refusal has to +# say which refusal; and a 200 says nothing about who served it, so every +# serving cell matches on the credentials the canary reports. Both the +# container log and the response body are searched. +cell() { + desc=$1 + expect=$2 + flags=$3 + want_log=${4:-} + + RESULT= + attempt "$flags" + got=$RESULT + + # "refused" is any of the ways a cell can fail to serve: the container + # never started, never reached readiness, or never answered. Which one a + # broken posture produces is not stable -- --cap-drop=ALL leaves the + # entrypoint's daemon alive but never ready -- and the cause is pinned by + # the log assertion below, not by which shade of failure it was. + case "$expect" in + refused) + case "$got" in + no-start|no-ready|no-200) got=refused ;; + esac + ;; + esac + + if [ "$got" != "$expect" ]; then + echo "FAIL: $desc -> $got, expected $expect" + [ -f "$LOGDIR/container.log" ] && sed 's/^/ | /' "$LOGDIR/container.log" + FAIL=$((FAIL + 1)) + return 0 + fi + + # want_log holds one pattern per line and every one has to match. + if [ -n "$want_log" ]; then + oldifs=$IFS + IFS=' +' + for pat in $want_log; do + IFS=$oldifs + if ! grep -q "$pat" "$LOGDIR/container.log"; then + echo "FAIL: $desc -> $got as expected, but the container log" \ + "does not match \"$pat\"" + [ -f "$LOGDIR/container.log" ] \ + && sed 's/^/ | /' "$LOGDIR/container.log" + FAIL=$((FAIL + 1)) + return 0 + fi + IFS=' +' + done + IFS=$oldifs + fi + + echo "PASS: $desc -> $got (as documented)" + PASS=$((PASS + 1)) +} + +echo "Image: $IMAGE" +echo "Platform: $PLATFORM" +echo "Every cell runs with Docker's DEFAULT seccomp profile." +echo "" + +# Started as root, the daemon drops the router and applications to the image's +# unprivileged "unit" user (uid 999). The canary reports what it actually ran +# as, so these cells assert the drop happened rather than assuming it. +# Two patterns, both anchored -- an unanchored "gid=999" also matches gid=9990. +# The first is the application's own view of itself, including supplementary +# groups: dropping the primary gid while keeping group 0 is root-group access +# under another name. The second is the router, which the application cannot +# speak for. +ROOT_ID="uid=999 gid=999 groups=999$ +^proc: router uid=999 gid=999$" + +# The shipped default: root in the container, full default capability set. +cell "root, default caps" 200 "" "$ROOT_ID" + +# no-new-privileges does not stop Unit dropping privileges: setuid/setgid from +# root is not a privilege *gain*, so the bit never applies to it. +cell "root, --security-opt no-new-privileges" 200 \ + "--security-opt no-new-privileges" "$ROOT_ID" + +# Root without capabilities cannot become the unprivileged 'unit' user, which +# is the first thing the daemon does: +# [alert] setgid(999) failed (1: Operation not permitted) +# CAP_SETGID and CAP_SETUID are what --cap-drop=ALL takes away here. Dropping +# capabilities is only useful once the process is already non-root, which is +# the cell below. Documented as a refusal, not a bug. +cell "root, --cap-drop=ALL" refused "--cap-drop=ALL" \ + "setgid(999) failed" + +# The non-root recipe. 1000:1000, not a bare 1000: uid 1000 has no passwd +# entry in this image, so `--user 1000` resolves the primary group to GID 0 -- +# +# $ docker run --rm --user 1000 --entrypoint sh IMAGE -c id +# uid=1000 gid=0(root) groups=0(root) +# +# -- and the container keeps root-group access, which is not what anyone means +# by "runs unprivileged". +# +# Nothing to drop to means nothing needs CAP_SETUID -- but the state, run and +# tmp directories are baked into the image root-owned, so they have to be +# replaced by writable mounts. A bare "--tmpfs /var/lib/unit" is not enough: +# a tmpfs is created root-owned 0755 and the daemon still gets EACCES, so the +# ownership has to be spelled out. This is the part of the non-root recipe +# people get wrong. +# +# One variable for all three cells, so a flag list can never drift from the +# label describing it -- which is exactly what happened while writing this. +# Each cell also asserts "uid=1000 gid=1000" from the canary's own +# os.getuid()/os.getgid(), so the identity of the process that served the +# request is measured rather than assumed. +NONROOT="--user 1000:1000" +NONROOT="$NONROOT --tmpfs /var/lib/unit:uid=1000,gid=1000" +NONROOT="$NONROOT --tmpfs /var/run:uid=1000,gid=1000" +NONROOT="$NONROOT --tmpfs /var/tmp:mode=1777" +NONROOT_ID="uid=1000 gid=1000 groups=1000$ +^proc: router uid=1000 gid=1000$" + +cell "--user 1000:1000, tmpfs state/run/tmp" 200 \ + "$NONROOT" "$NONROOT_ID" + +# The same, with every capability removed. Nothing above needed one. +cell "--user 1000:1000, --cap-drop=ALL, tmpfs" 200 \ + "$NONROOT --cap-drop=ALL" "$NONROOT_ID" + +# And with no-new-privileges on top: the posture to recommend. +cell "--user 1000:1000, --cap-drop=ALL, no-new-privileges, tmpfs" 200 \ + "$NONROOT --cap-drop=ALL --security-opt no-new-privileges" "$NONROOT_ID" + +echo "" +echo "Results: $PASS passed, $FAIL failed" +[ "$FAIL" -eq 0 ] diff --git a/test/syscalls/README.md b/test/syscalls/README.md new file mode 100644 index 000000000..6b8be43e3 --- /dev/null +++ b/test/syscalls/README.md @@ -0,0 +1,127 @@ +# Syscall drift gate + +A CI gate that notices when Unit starts using a kernel interface it did not use +before. It exists because Unit's portability story is a list of syscalls: the +kernel minimum, what a container's seccomp profile has to allow, and what a +`musl` build does differently from a `glibc` one are all statements about this +list, and nothing kept them honest. + + test/syscalls/capture.sh run unitd under strace, print its syscalls + test/syscalls/check.sh diff a capture against a baseline + test/syscalls/run-in-docker.sh do both inside a pinned container + test/syscalls/baseline-linux-glibc.txt + test/syscalls/baseline-linux-musl.txt + +`.github/workflows/security-syscalls.yml` runs the docker driver for both libcs +on every pull request that touches the build or `src/`. + +## The job is advisory for now + +The workflow is marked `continue-on-error: true` and its jobs are named +"(advisory)": a difference shows up in the checks list and in the log, but it +cannot block a merge. + +That is not timidity, it is honesty about where the baselines came from. They +were captured on a developer machine, and while the container pins the libc by +digest, the syscalls glibc picks still depend partly on the **host kernel** -- +so a runner on an older kernel can legitimately add a fallback syscall the +baseline has never seen. Until a baseline has been captured from a CI run and +committed, a red gate here would be more likely to be wrong than right. + +**Making it blocking** is one commit, and this is the sequence: + +1. Let the workflow run on a pull request (or dispatch it). +2. Download the `syscalls-glibc` and `syscalls-musl` artifacts -- they are + uploaded on every run, including failures, for exactly this. +3. Commit them as the baselines if they differ from what is here, noting in + the commit message which syscalls moved and why. +4. Drop `continue-on-error` and the "(advisory)" suffix. + +## The rule + +**A new syscall fails the job. A disappeared syscall only warns.** + +Reaching for a new kernel interface is a decision -- it moves the kernel +minimum, or needs a seccomp allowance, or does not exist on some platform -- so +it should be visible in review, with the baseline updated in the same commit. +Dropping one is never a hazard, and a libc or runner update drops one now and +then without anybody meaning it to. + +## Updating a baseline + +```sh +test/syscalls/run-in-docker.sh glibc --update +test/syscalls/run-in-docker.sh musl --update +git add test/syscalls/baseline-linux-*.txt +``` + +Commit the refreshed baseline together with the change that caused it, and say +in the commit message which syscall appeared and why. If the capture differs +from what CI sees, take CI's: the workflow uploads its captured list as the +`syscalls-` artifact on every run, exactly so a baseline can be +reconciled without guessing. + +Bumping either pinned image digest in `run-in-docker.sh` is also a baseline +refresh. + +## What is exercised, and what is not + +`capture.sh` starts `unitd`, PUTs a configuration with one TCP listener and one +`share` route, fetches a static file over it, asserts HTTP 200 and the expected +body, then stops the daemon with `SIGQUIT` -- all inside `strace -f`, so +startup, configuration, request handling and teardown are in one trace. The +served request is a static `share` route and nothing else: no application, no +language module, no prototype process. Run as root (which is how it runs in +the container) the router drops to the unprivileged default user, so `setgid` +and `setuid` are in the trace. + +`capget` is **not**, and it is worth saying why, because the audit that +prompted this gate assumed otherwise. `nxt_capability_set()` returns at +`src/nxt_capability.c:51` as soon as `geteuid() == 0` -- root already has what +it needs -- so the `capget` call below it is only reached by a non-root daemon. +Neither baseline contains it. A non-root capture leg would, and is listed as +a follow-up. + +No language application is started, on purpose. An app process is forked from +Unit and never `exec`s, so its syscalls are indistinguishable in the trace from +Unit's own -- and a CPython or PHP interpreter contributes far more of them +than Unit does, all of it moving with the runtime's own version. The gate +would then fail on a language patch release, which is noise, not drift. +Covering the module path is a separate job with a separate baseline; see the +follow-ups in the pull request that added this. + +The two lists are not directly comparable with each other: 10 syscalls appear +only in the glibc baseline (`access`, `clone3`, `epoll_create`, `epoll_wait`, +`lseek`, `newfstatat`, `openat`, `prlimit64`, `rseq`, `set_robust_list`) and 6 +only in the musl one (`epoll_create1`, `epoll_pwait`, `fork`, `futex`, +`membarrier`, `open`), because the libcs pick different syscalls for the same +operation (`epoll_create` vs `epoll_create1`, `clone3` vs `fork`, `openat` vs +`open`). `memfd_create` appears on both now that the probe finds it on musl +without the kernel UAPI header. That is the reason both are captured. + +## Guards + +Both scripts refuse a capture with fewer than 20 syscalls, and `capture.sh` +additionally requires a handful of syscalls Unit cannot possibly avoid +(`epoll_ctl`, `socket`, `bind`, `listen`, `read`, `write`, `close`). A missing +`strace`, a changed output format or a parse that quietly matched nothing all +produce an empty list, and an empty list compares clean against any baseline -- +a green check that verified nothing. + +`capture.sh` also fails on a nonzero exit from the traced daemon. `strace` +exits with its tracee's status, so a daemon that crashes during its `SIGQUIT` +teardown would otherwise still hand over a large, plausible list, and every +syscall the teardown never reached would be filed as a harmless warning. + +## Requirements + +`capture.sh` needs `strace` and `curl` and a built tree. `run-in-docker.sh` +needs only docker; it installs a toolchain, builds Unit and captures inside a +digest-pinned container. The digest pins the base image and therefore the +libc, but not the packages installed on top of it -- both are stable releases, +so that moves only on a point release, but it is a real residual. It passes `--cap-add=SYS_PTRACE`: on a stock daemon strace can +already trace its own descendants without it (verified on docker 29.7.2 with +the default seccomp and AppArmor profiles), but a host with a tightened +`ptrace_scope` or a custom profile needs it, and it costs nothing here. The +image is always pulled `--platform linux/amd64`: the baselines are per +architecture and an arm64 pull would silently produce a different list. diff --git a/test/syscalls/baseline-linux-glibc.txt b/test/syscalls/baseline-linux-glibc.txt new file mode 100644 index 000000000..6893deb69 --- /dev/null +++ b/test/syscalls/baseline-linux-glibc.txt @@ -0,0 +1,77 @@ +# Syscalls issued by unitd on linux/amd64, glibc. +# +# Regenerate: test/syscalls/run-in-docker.sh glibc --update +# Image: debian:trixie-slim@sha256:d7e12182ce18b85b93007c1dedf31f2d29e01ccf3182cc4017c709b6259bc132 +# See test/syscalls/README.md before editing this by hand. +accept4 +access +arch_prctl +bind +brk +capset +chmod +clone +clone3 +close +close_range +connect +dup2 +epoll_create +epoll_ctl +epoll_wait +eventfd2 +execve +exit_group +fchmod +fchown +fcntl +fstat +fsync +ftruncate +getdents64 +getegid +geteuid +getpid +getppid +getrandom +getsockopt +gettid +ioctl +kill +listen +lseek +memfd_create +mkdir +mmap +mprotect +munmap +newfstatat +openat +prctl +pread64 +prlimit64 +pwrite64 +read +recvfrom +recvmsg +rename +rseq +rt_sigaction +rt_sigprocmask +sched_getaffinity +sendmsg +set_robust_list +set_tid_address +setgid +setgroups +setsockopt +setuid +shutdown +signalfd4 +socket +socketpair +uname +unlink +wait4 +write +writev diff --git a/test/syscalls/baseline-linux-musl.txt b/test/syscalls/baseline-linux-musl.txt new file mode 100644 index 000000000..4defe5215 --- /dev/null +++ b/test/syscalls/baseline-linux-musl.txt @@ -0,0 +1,75 @@ +# Syscalls issued by unitd on linux/amd64, musl. +# +# Regenerate: test/syscalls/run-in-docker.sh musl --update +# Image: alpine:3.22@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce +# See test/syscalls/README.md before editing this by hand. +accept4 +arch_prctl +bind +brk +capset +chmod +clone +close +close_range +connect +dup2 +epoll_create1 +epoll_ctl +epoll_pwait +eventfd2 +execve +exit_group +fchmod +fchown +fcntl +fork +fstat +fsync +ftruncate +futex +getdents64 +getegid +geteuid +getpid +getppid +getrandom +getsockopt +gettid +ioctl +kill +listen +lseek +lstat +membarrier +memfd_create +mkdir +mmap +mprotect +munmap +open +prctl +pread64 +pwrite64 +read +recvfrom +recvmsg +rename +rt_sigaction +rt_sigprocmask +sched_getaffinity +sendmsg +set_tid_address +setgid +setgroups +setsockopt +setuid +shutdown +signalfd4 +socket +socketpair +uname +unlink +wait4 +write +writev diff --git a/test/syscalls/capture.sh b/test/syscalls/capture.sh new file mode 100755 index 000000000..0b074b9bc --- /dev/null +++ b/test/syscalls/capture.sh @@ -0,0 +1,238 @@ +#!/bin/sh +# +# Capture the set of syscalls unitd issues while starting up, accepting a +# configuration and serving one request, and print it as a sorted, unique list +# of syscall names -- one per line. +# +# The list is the input to the CI drift gate (test/syscalls/check.sh); see +# test/syscalls/README.md for the whole story, including why the exercised +# configuration deliberately does not start a language application. +# +# Usage: +# test/syscalls/capture.sh [-u UNITD] [-p PORT] [-o OUTFILE] +# +# -u path to the unitd binary (default: build/sbin/unitd) +# -p TCP port for the test listener (default: 8099) +# -o write the list here instead of stdout +# +# Requires: strace, curl. Run it from the top of a built source tree. + +set -eu + +UNITD=build/sbin/unitd +PORT=8099 +OUT= + +while getopts 'u:p:o:' opt; do + case "$opt" in + u) UNITD=$OPTARG ;; + p) PORT=$OPTARG ;; + o) OUT=$OPTARG ;; + *) echo "usage: $0 [-u UNITD] [-p PORT] [-o OUTFILE]" >&2; exit 2 ;; + esac +done + +die() { echo "capture.sh: $*" >&2; exit 1; } + +command -v strace >/dev/null 2>&1 || die "strace not found" +command -v curl >/dev/null 2>&1 || die "curl not found" +[ -x "$UNITD" ] || die "no unitd binary at $UNITD" + +WORK=$(mktemp -d) +STRACE_PID= + +# strace sets SIGINT/SIGQUIT/SIGTERM/SIGHUP to SIG_IGN in itself so that a +# terminal signal reaches the tracee instead, which means the daemon has to be +# signalled directly -- signalling strace is a no-op that hangs the script. +# unitd writes its main pid to --pid, so read it from there. +unit_pid() { + [ -s "$WORK/unit.pid" ] && cat "$WORK/unit.pid" +} + +# Every kill here is best-effort: by the time cleanup runs the daemon has +# usually exited already, and under `set -e` a failing kill would abort the +# EXIT trap before the rm and leak the temporary directory. +cleanup() { + up=$(unit_pid || :) + [ -n "$up" ] && { kill -QUIT "$up" 2>/dev/null || :; } + i=0 + while [ "$i" -lt 50 ] \ + && [ -n "$STRACE_PID" ] && kill -0 "$STRACE_PID" 2>/dev/null; do + i=$((i + 1)) + sleep 0.1 + done + [ -n "$up" ] && { kill -KILL "$up" 2>/dev/null || :; } + # SIGKILL on strace leaves its tracees behind by default -- hence + # --kill-on-exit below, so the controller and router go with it and do not + # sit on the test port. + [ -n "$STRACE_PID" ] && { kill -KILL "$STRACE_PID" 2>/dev/null || :; } + rm -rf "$WORK" + return 0 +} +# A trapped signal does not end a POSIX shell, so each signal handler exits +# explicitly; the EXIT trap is cleared first so the cleanup runs once. +trap cleanup EXIT +trap 'trap - EXIT; cleanup; exit 130' INT +trap 'trap - EXIT; cleanup; exit 143' TERM + +mkdir -p "$WORK/state" "$WORK/tmp" "$WORK/modules" "$WORK/share" +echo 'syscall drift canary' > "$WORK/share/index.txt" + +# Started as root, unitd drops the router to the configured unprivileged user +# (that is the point: setgid and setuid land in the trace). mktemp -d makes +# the directory 0700, so the router would then get EACCES and answer 403. +chmod 711 "$WORK" +chmod -R a+rX "$WORK/share" + +SOCK=$WORK/control.sock +TRACE=$WORK/trace.log + +cat > "$WORK/config.json" <"$WORK/stdout.log" 2>&1 & +STRACE_PID=$! + +# Wait for the control socket rather than sleeping a fixed amount: a slow +# runner must not silently produce a short trace. +i=0 +while [ ! -S "$SOCK" ]; do + i=$((i + 1)) + [ "$i" -lt 300 ] || { + cat "$WORK/stdout.log" >&2 || : + die "control socket did not appear within 30s" + } + kill -0 "$STRACE_PID" 2>/dev/null || { + cat "$WORK/stdout.log" >&2 || : + die "unitd exited before the control socket appeared" + } + sleep 0.1 +done + +curl -sS --fail --max-time 10 --unix-socket "$SOCK" \ + -X PUT --data-binary "@$WORK/config.json" \ + http://localhost/config >"$WORK/put.log" 2>&1 \ + || { cat "$WORK/put.log" >&2; die "configuration was rejected"; } + +# Poll the listener: PUT returns once the configuration is accepted, which is +# not the same instant the router has the socket bound. +i=0 +while :; do + code=$(curl -sS --max-time 5 -o "$WORK/body.txt" -w '%{http_code}' \ + "http://127.0.0.1:$PORT/index.txt" 2>>"$WORK/get.log") && break + i=$((i + 1)) + [ "$i" -lt 100 ] || { cat "$WORK/get.log" >&2; die "listener never answered"; } + sleep 0.1 +done + +[ "$code" = "200" ] || { cat "$WORK/body.txt" >&2; die "expected HTTP 200, got $code"; } +grep -q 'syscall drift canary' "$WORK/body.txt" \ + || die "served body is not the canary file" + +# Main stores the configuration in a child process. Wait for conf.json, so +# that the second PUT below stores over a file that exists. When SIGQUIT +# arrives while a store child runs and a newer store waits, main kills the +# child, and the kill can come before the child created conf.json. +i=0 +while [ ! -s "$WORK/state/conf.json" ]; do + i=$((i + 1)) + [ "$i" -lt 100 ] || die "conf.json was not stored within 10s" + sleep 0.1 +done + +# A second PUT, so the capture covers storing over state files that already +# exist. The first one writes conf.json and version into an empty statedir, +# where nxt_main_file_store_inherit() finds no destination and returns before +# it reads or copies anything -- so a single PUT never reaches the fchmod() +# and fchown() that carry a destination's mode and ownership onto its +# replacement, and the syscalls of the most privileged path in the store +# would be invisible to this gate. The settings block is here only to make +# the configuration differ from the one above; the listener and routes are +# untouched, so the canary served above still describes what is running. +sed -e 's/^{$/{\n "settings": {"http": {"header_read_timeout": 30}},/' \ + "$WORK/config.json" > "$WORK/config2.json" + +curl -sS --fail --max-time 10 --unix-socket "$SOCK" \ + -X PUT --data-binary "@$WORK/config2.json" \ + http://localhost/config >"$WORK/put2.log" 2>&1 \ + || { cat "$WORK/put2.log" >&2; die "the second configuration was rejected"; } + +# Stop the daemon and let it run its teardown path *inside* the trace, so +# shutdown syscalls are part of the captured set. Signal unitd, not strace +# (see unit_pid above), then wait for strace itself to reap and flush. +UPID=$(unit_pid) || die "unitd never wrote a pid file" +kill -QUIT "$UPID" +i=0 +while kill -0 "$STRACE_PID" 2>/dev/null; do + i=$((i + 1)) + [ "$i" -lt 200 ] || die "unitd did not exit within 20s of SIGQUIT" + sleep 0.1 +done +# strace exits with the tracee's status, so this is unitd's exit code from its +# SIGQUIT teardown. Ignoring it would let a daemon that crashes on shutdown +# still produce a large, plausible capture -- and every syscall its teardown +# never reached would only be a warning. +src=0 +wait "$STRACE_PID" || src=$? +STRACE_PID= +[ "$src" -eq 0 ] || { + tail -20 "$WORK/unit.log" >&2 2>/dev/null || : + die "unitd exited $src from its SIGQUIT teardown" +} + +[ -s "$TRACE" ] || die "strace produced an empty trace" + +# Two shapes carry a syscall name: a normal entry line, and the resumption of +# a call that was interrupted by another thread's output. +# 1234 epoll_pwait(4, ...) = 1 +# 1234 <... epoll_pwait resumed>[...], 1024, ...) = 1 +# Lines for signals (--- SIGCHLD ---) and exits (+++ exited +++) match neither. +SYSCALLS=$(sed -e 's/^[0-9][0-9]*[ \t][ \t]*//' "$TRACE" \ + | sed -n \ + -e 's/^\([a-zA-Z_][a-zA-Z0-9_]*\)(.*/\1/p' \ + -e 's/^<\.\.\. \([a-zA-Z_][a-zA-Z0-9_]*\) resumed.*/\1/p' \ + | LC_ALL=C sort -u) + +# A missing or renamed tool piped into sort -u yields an empty list, and an +# empty list compares clean against any baseline. Refuse to emit one. +COUNT=$(printf '%s\n' "$SYSCALLS" | grep -c '^[a-z]' || :) +[ "$COUNT" -ge 20 ] || die "only $COUNT syscalls extracted -- the trace did not parse" + +for required in epoll_ctl socket bind listen read write close; do + printf '%s\n' "$SYSCALLS" | grep -qx "$required" \ + || die "expected syscall '$required' is missing -- the trace did not parse" +done + +if [ -n "$OUT" ]; then + printf '%s\n' "$SYSCALLS" > "$OUT" + echo "capture.sh: wrote $COUNT syscalls to $OUT" >&2 +else + printf '%s\n' "$SYSCALLS" +fi diff --git a/test/syscalls/check.sh b/test/syscalls/check.sh new file mode 100755 index 000000000..3b89ed4f7 --- /dev/null +++ b/test/syscalls/check.sh @@ -0,0 +1,70 @@ +#!/bin/sh +# +# Compare a captured syscall list (test/syscalls/capture.sh) against a +# checked-in baseline. +# +# New syscalls -> failure. Unit reached for a kernel interface it did not +# use before; that is a deliberate change and the baseline +# has to be updated in the same commit. +# Gone syscalls -> warning. Losing a syscall is not a hazard, and a runner +# or libc update legitimately drops one now and then. +# +# Usage: test/syscalls/check.sh BASELINE CAPTURED +# +# See test/syscalls/README.md for how to refresh a baseline. + +set -eu + +BASELINE=${1:?usage: $0 BASELINE CAPTURED} +CAPTURED=${2:?usage: $0 BASELINE CAPTURED} + +die() { echo "check.sh: $*" >&2; exit 1; } + +[ -f "$BASELINE" ] || die "no baseline at $BASELINE" +[ -s "$CAPTURED" ] || die "captured list $CAPTURED is missing or empty" + +# An empty or unsorted input compares clean against anything, so the two +# failure modes that would make this gate silently pass are checked first. +n_captured=$(grep -c '^[a-z]' "$CAPTURED" || :) +[ "$n_captured" -ge 20 ] \ + || die "captured list has only $n_captured entries -- refusing to compare" + +work=$(mktemp -d) +# A trapped signal does not end a POSIX shell, so each signal handler exits +# explicitly; the EXIT trap is cleared first so the cleanup runs once. +trap 'rm -rf "$work"' EXIT +trap 'trap - EXIT; rm -rf "$work"; exit 130' INT +trap 'trap - EXIT; rm -rf "$work"; exit 143' TERM + +# Baselines carry '#' comments and blank lines; captures do not. +# LC_ALL=C throughout: comm rejects input sorted under a locale whose collation +# differs from the one that produced it, and '_' is exactly where they differ. +sed -e 's/#.*//' -e 's/[ \t]*$//' -e '/^$/d' "$BASELINE" | LC_ALL=C sort -u > "$work/base" +sed -e 's/#.*//' -e 's/[ \t]*$//' -e '/^$/d' "$CAPTURED" | LC_ALL=C sort -u > "$work/have" + +LC_ALL=C comm -13 "$work/base" "$work/have" > "$work/new" +LC_ALL=C comm -23 "$work/base" "$work/have" > "$work/gone" + +echo "baseline: $BASELINE ($(wc -l < "$work/base" | tr -d ' ') syscalls)" +echo "captured: $CAPTURED ($(wc -l < "$work/have" | tr -d ' ') syscalls)" + +if [ -s "$work/gone" ]; then + echo + echo "WARNING: syscalls in the baseline that were not observed:" + sed 's/^/ - /' "$work/gone" + echo " (not a failure; refresh the baseline when the drop is intended)" +fi + +if [ -s "$work/new" ]; then + echo + echo "FAIL: syscalls observed that are not in the baseline:" + sed 's/^/ + /' "$work/new" + echo + echo "If this is intended, refresh the baseline:" + echo " test/syscalls/run-in-docker.sh --update" + echo "and commit it with the change that introduced the syscall." + exit 1 +fi + +echo +echo "OK: no new syscalls." diff --git a/test/syscalls/run-in-docker.sh b/test/syscalls/run-in-docker.sh new file mode 100755 index 000000000..aee882022 --- /dev/null +++ b/test/syscalls/run-in-docker.sh @@ -0,0 +1,136 @@ +#!/bin/sh +# +# Build unitd and capture its syscall set inside a pinned container, so a +# developer's box and CI observe the same libc. +# +# Usage: +# test/syscalls/run-in-docker.sh glibc|musl [-o OUTFILE] [--update] [--check] +# +# -o OUTFILE write the captured list here +# (default: a temporary file, printed at the end) +# --check compare against the checked-in baseline (test/syscalls/check.sh) +# --update overwrite the checked-in baseline with what was captured +# +# Requires: docker. The image is pulled for linux/amd64 explicitly -- the +# baselines are per-architecture and a silent arm64 pull would produce a +# different, wrong list. + +set -eu + +die() { echo "run-in-docker.sh: $*" >&2; exit 1; } + +LIBC=${1:-} +shift 2>/dev/null || : +case "$LIBC" in + glibc|musl) ;; + *) die "usage: $0 glibc|musl [-o OUTFILE] [--update] [--check]" ;; +esac + +OUT= +UPDATE=no +CHECK=no +while [ $# -gt 0 ]; do + case "$1" in + -o) OUT=${2:?-o needs a path}; shift 2 ;; + --update) UPDATE=yes; shift ;; + --check) CHECK=yes; shift ;; + *) die "unknown argument: $1" ;; + esac +done + +REPO=$(cd "$(dirname "$0")/../.." && pwd) +BASELINE="$REPO/test/syscalls/baseline-linux-$LIBC.txt" + +# Pinned by digest, not by tag: the captured set depends on the libc, and +# debian:trixie-slim and alpine:3.22 are both mutable tags that would move a +# baseline with no change in this repository. Bumping either digest is a +# baseline refresh (README.md). +# +# The digest pins the base image, not the packages apt/apk install on top of +# it. Both are stable releases, so glibc and musl only move on a point +# release -- but that is a real residual, and one of the reasons the CI job +# is advisory rather than blocking. +if [ "$LIBC" = glibc ]; then + IMAGE=debian:trixie-slim@sha256:d7e12182ce18b85b93007c1dedf31f2d29e01ccf3182cc4017c709b6259bc132 + INSTALL='apt-get -qq update \ + && DEBIAN_FRONTEND=noninteractive apt-get -qq install -y --no-install-recommends \ + gcc make libc6-dev libpcre2-dev strace curl ca-certificates >/dev/null' +else + IMAGE=alpine:3.22@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce + # linux-headers, which musl-dev does not carry: is + # what auto/capability probes for, and without it NXT_HAVE_LINUX_CAPABILITY + # is off, nxt_capability_specific_set() and nxt_capability_drop() compile + # to stubs, and this leg reports "no new syscalls" about a binary that has + # no capability handling in it at all. + INSTALL='apk add --no-cache gcc make musl-dev linux-headers pcre2-dev \ + strace curl >/dev/null' +fi + +command -v docker >/dev/null 2>&1 || die "docker not found" + +work=$(mktemp -d) +# A trapped signal does not end a POSIX shell, so each signal handler exits +# explicitly; the EXIT trap is cleared first so the cleanup runs once. +trap 'rm -rf "$work"' EXIT +trap 'trap - EXIT; rm -rf "$work"; exit 130' INT +trap 'trap - EXIT; rm -rf "$work"; exit 143' TERM + +# With no -o the list goes to stdout, as the usage above says. Writing it to +# a temporary the EXIT trap then deletes would make the plainest invocation +# the one that throws its own result away. +TO_STDOUT=no +[ -n "$OUT" ] || { OUT="$work/captured.txt"; TO_STDOUT=yes; } + +# ptrace: Docker's default seccomp profile allows the ptrace syscall, but the +# default capability set has no CAP_SYS_PTRACE and the default AppArmor profile +# confines cross-process ptrace. strace only ever traces its own descendants +# here, so SYS_PTRACE alone is enough on a stock daemon; seccomp is left at the +# default on purpose, to keep this close to how the image really runs. +docker run --rm --platform linux/amd64 \ + --cap-add=SYS_PTRACE \ + -v "$REPO:/src:ro" \ + -w /work \ + "$IMAGE" \ + sh -eu -c " + $INSTALL + mkdir -p /work + # Copy out of the read-only mount: configure writes into the tree. + tar -C /src -cf - --exclude=./.git --exclude=./build . | tar -C /work -xf - + ./configure --prefix=/opt/unit >/dev/null + make -j\"\$(nproc)\" >/dev/null + exec test/syscalls/capture.sh + " > "$OUT.raw" 2>"$work/stderr.log" || { + cat "$work/stderr.log" >&2 + die "container run failed" + } +cat "$work/stderr.log" >&2 || : + +# The container writes progress to stderr and the list to stdout, but guard +# against a build that prints to stdout anyway: keep only syscall-shaped lines. +grep -E '^[a-z_][a-z0-9_]*$' "$OUT.raw" | LC_ALL=C sort -u > "$OUT" +rm -f "$OUT.raw" + +n=$(wc -l < "$OUT" | tr -d ' ') +[ "$n" -ge 20 ] || die "captured only $n syscalls from $IMAGE -- not trusting that" +if [ "$TO_STDOUT" = yes ]; then + echo "run-in-docker.sh: $LIBC ($IMAGE): $n syscalls" >&2 + cat "$OUT" +else + echo "run-in-docker.sh: $LIBC ($IMAGE): $n syscalls -> $OUT" >&2 +fi + +if [ "$UPDATE" = yes ]; then + { + echo "# Syscalls issued by unitd on linux/amd64, $LIBC." + echo "#" + echo "# Regenerate: test/syscalls/run-in-docker.sh $LIBC --update" + echo "# Image: $IMAGE" + echo "# See test/syscalls/README.md before editing this by hand." + cat "$OUT" + } > "$BASELINE" + echo "run-in-docker.sh: refreshed $BASELINE" >&2 +fi + +if [ "$CHECK" = yes ]; then + "$REPO/test/syscalls/check.sh" "$BASELINE" "$OUT" +fi diff --git a/test/test_access_log.py b/test/test_access_log.py index b5e1e3f9a..ce36c42a2 100644 --- a/test/test_access_log.py +++ b/test/test_access_log.py @@ -1,3 +1,4 @@ +import json import time import pytest @@ -307,6 +308,151 @@ def check_format(log_format, expect, url='/'): check_format(log_format, '{"status":"200","uri":"/"}') +def _read_log_lines(): + with open( + f'{option.temp_dir}/access.log', 'r', encoding='utf-8', errors='strict' + ) as f: + return [ln for ln in f.read().splitlines() if ln] + + +def test_access_log_text_escape(wait_for_record): + load('empty') + + # A percent-encoded CRLF in the target reaches "$uri" decoded, so without + # escaping these bytes end the record and start one the client wrote. + set_format('$remote_addr "$uri" $status') + + assert ( + client.get(url='/%0d%0a1.2.3.4 "GET /forged" 200')['status'] == 200 + ) + assert wait_for_record(r'\\x0D\\x0A', 'access.log') is not None + + lines = _read_log_lines() + + assert len(lines) == 1, 'one request, one record' + assert '\\x0D\\x0A' in lines[0], 'the CRLF is escaped, not written' + assert '\\x22GET /forged\\x22 200' in lines[0], 'the quotes are escaped too' + + +def test_access_log_text_escape_leaves_the_format_alone(wait_for_record): + load('empty') + + # The operator's own format text is not a value and is written as it is; + # only what a variable expands to is escaped. + set_format('start\t"$uri"\tend') + + assert client.get(url='/ok')['status'] == 200 + assert wait_for_record(r'start', 'access.log') is not None + + line = _read_log_lines()[-1] + + assert line == 'start\t"/ok"\tend', 'tabs in the format survive' + + +def test_access_log_text_escape_njs(require, wait_for_record): + require({'modules': {'njs': 'any'}}) + + load('empty') + + # An njs format produces the whole record, so all of it is escaped -- + # the quotes written here included. The client's CRLF must not reach the + # file, and the record must be exactly as long as it claims: the newline + # it ends with is appended inside the generated function, so a length that + # counted the flag bit instead of a 0/1 would report pool memory past the + # end of the record. + set_format('`${vars.remote_addr} "${vars.uri}"`') + + assert client.get(url='/%0d%0aFORGED')['status'] == 200 + assert wait_for_record(r'FORGED', 'access.log') is not None + + # On the raw bytes, not on split lines: three stray bytes that happened to + # be newlines would vanish into a line split and the record would still + # look right. + with open(f'{option.temp_dir}/access.log', 'rb') as f: + raw = f.read() + + assert ( + raw == b'127.0.0.1 \\x22/\\x0D\\x0AFORGED\\x22\n' + ), 'one record, one newline, nothing after' + + +def test_access_log_object_escape(wait_for_record): + load('empty') + + # Every character below is significant inside a JSON string: the first + # chunk closes the "ua" member and opens a forged one, then come a bare + # quote, a backslash, a control character (HTAB is legal in a field + # value) and a multi-byte UTF-8 sequence. + evil = '", "injected": "1 " \\ \t \u00e9' + + set_format({'status': '$status', 'ua': '$header_user_agent'}) + + assert ( + client.get( + headers={ + 'Host': 'localhost', + 'User-Agent': evil, + 'Connection': 'close', + } + )['status'] + == 200 + ) + assert wait_for_record(r'"status":"200"', 'access.log') is not None + + line = _read_log_lines()[-1] + + record = json.loads(line) + + assert list(record.keys()) == ['status', 'ua'], 'no injected members' + assert record['status'] == '200' + assert record['ua'] == evil, 'value round-trips' + + +def test_access_log_object_escape_is_not_the_text_escape(wait_for_record): + load('empty') + + # The other half of the split: the object format escapes as JSON and must + # not also take the text format's escaping, or a reader would get "\x0D" + # where JSON says "\r" and the member would no longer round-trip. + set_format({'uri': '$uri', 'status': '$status'}) + + assert client.get(url='/%0d%0aFORGED')['status'] == 200 + assert wait_for_record(r'FORGED', 'access.log') is not None + + record = json.loads(_read_log_lines()[-1]) + + assert record['uri'] == '/\r\nFORGED', 'the bytes round-trip through JSON' + assert '\\x0D' not in record['uri'], 'not escaped twice' + + +def test_access_log_object_escape_njs(require, wait_for_record): + require({'modules': {'njs': 'any'}}) + + load('empty') + + evil = '", "injected": "1 " \\ \t \u00e9' + + set_format({'uri': '`${vars.uri}`', 'ua': '$header_user_agent'}) + + assert ( + client.get( + headers={ + 'Host': 'localhost', + 'User-Agent': evil, + 'Connection': 'close', + } + )['status'] + == 200 + ) + assert wait_for_record(r'"uri":"/"', 'access.log') is not None + + record = json.loads(_read_log_lines()[-1]) + + assert list(record.keys()) == ['uri', 'ua'], 'no injected members' + assert record['uri'] == '/' + assert record['ua'] == evil, 'value round-trips' + + def test_access_log_variables(wait_for_record): load('mirror') @@ -320,6 +466,36 @@ def test_access_log_variables(wait_for_record): ), '$body_bytes_sent' +def test_access_log_body_bytes_sent_chunked(wait_for_record): + # The application sets no Content-Length, so the response is chunked. + # $body_bytes_sent counts the chunk framing. It does not count the + # empty line that ends the header. + load('chunked') + set_format('$uri $body_bytes_sent') + + body = '0123456789' + + resp = client.post(url='/data', body=body) + assert resp['status'] == 200 + assert resp['headers']['Transfer-Encoding'] == 'chunked' + assert resp['body'] == body + + # "a\r\n0123456789\r\n0\r\n\r\n" is 20 bytes. + record = wait_for_record(r'^/data \d+$', 'access.log') + assert record is not None, 'data record' + assert record.group(0) == '/data 20', 'chunked body' + + # An empty body has only the last chunk: "0\r\n\r\n" is 5 bytes. + resp = client.get(url='/empty') + assert resp['status'] == 200 + assert resp['headers']['Transfer-Encoding'] == 'chunked' + assert resp['body'] == '' + + record = wait_for_record(r'^/empty \d+$', 'access.log') + assert record is not None, 'empty record' + assert record.group(0) == '/empty 5', 'empty chunked body' + + def test_access_log_if(search_in_file, wait_for_record): load('empty') set_format('$uri') diff --git a/test/test_app_lifecycle.py b/test/test_app_lifecycle.py new file mode 100644 index 000000000..71003fc68 --- /dev/null +++ b/test/test_app_lifecycle.py @@ -0,0 +1,982 @@ +"""Application-process lifecycle churn, parametrized over every runtime. + +Rationale +--------- +The libunit port/fd lifecycle -- nxt_unit_add_port() and the add_port / +remove_port callbacks, nxt_unit_port_release()'s destroy path, and +nxt_unit_process_msg()'s `done:` descriptor cleanup -- lives in the +*application* process, and every runtime links the same src/nxt_unit.c. A bug +there (a double close(), a leaked descriptor, a use-after-free of a port +struct) surfaces as an "[alert]" line written by the app process into unit.log +during port teardown, or as a slowly growing /proc//fd. + +Before this module the suite exercised those paths deliberately in exactly one +runtime -- test_go_application.py's two churn tests, added alongside the Go +port-fd double-close fix. Every other runtime got only incidental coverage +from the conftest `run` fixture calling Log.check_alerts() after tests that do +little more than "send a request, check the body", so an app-side double close +could reach CI as nothing but an intermittent alert in one runtime's log. + +These tests deliberately churn the app-side port/fd lifecycle -- worker count +up and down, /control/.../restart storms, whole-application removal and re-add, +requests in flight across all of it -- and lean on the harness's existing +teardown assertions to fail the run: + + * conftest `run` fixture -> Log.check_alerts(): any "[alert]" from the app + process (e.g. "close(N) failed: Bad file descriptor") fails the test, and + any "Sanitizer" line fails it in the ASan/UBSan legs. + * conftest `_check_fds()`: descriptor leaks in main/router/controller + (with --fds-threshold=0, the CI default). + +plus one assertion the harness makes nowhere else: + + * test_app_lifecycle_app_fds_stable(): descriptor growth inside the + *application* process itself. + +Runtime coverage is table-driven (RUNTIMES below); a runtime that is not built +into this unitd self-skips, so the same file can run in every per-language CI +leg with only that leg's parameter executing. Adding a runtime is one entry. + +CI wiring note: .github/workflows/build-test.yml derives a per-module test glob +(test/test_*), so this file is named explicitly in that job's +`testpath` -- otherwise only the python/unit legs would ever run it. +""" + +import os +import subprocess +import time +from collections import namedtuple +from pathlib import Path + +import pytest + +from unit.applications.lang.go import ApplicationGo +from unit.applications.lang.java import ApplicationJava +from unit.applications.lang.node import ApplicationNode +from unit.applications.lang.perl import ApplicationPerl +from unit.applications.lang.php import ApplicationPHP +from unit.applications.lang.python import ApplicationPython +from unit.applications.lang.ruby import ApplicationRuby +from unit.applications.lang.wasm_component import ApplicationWasmComponent +from unit.applications.proto import ApplicationProto +from unit.option import option + +# Deliberately no module-level `client`: conftest.pytest_generate_tests() keys +# on it to fan a module out over language versions, which would collide with +# the per-runtime parametrization below. Deliberately no module-level +# `prerequisites` either: availability is per parameter, checked in `app`. + + +class ApplicationLibunit(ApplicationProto): + """The bare C libunit app, src/test/nxt_unit_app_test.c. + + Needs no language toolchain -- `./configure --tests && make tests` builds + it -- and links libunit directly, so it exercises the port/fd lifecycle + under test with nothing else in the way. It answers with a "Request data:" + dump that embeds the request body rather than mirroring it verbatim, hence + mirror=False in the table below. + """ + + application_type = 'external' + + @staticmethod + def executable(): + return f'{option.current_dir}/build/unit_app_test' + + def load(self, script, **kwargs): + self._load_conf( + { + "listeners": {"*:8080": {"pass": f"applications/{script}"}}, + "applications": { + script: { + "type": "external", + "processes": {"spare": 0}, + "working_directory": option.temp_dir, + "executable": ApplicationLibunit.executable(), + }, + }, + }, + **kwargs, + ) + + +# runtime -> (client factory, app name, required modules, required features, +# body is mirrored back verbatim, +# answers a body larger than one shared-memory segment) +# +# The last flag is false only for the libunit sample: it sizes its response +# buffer from the whole request buffer (src/test/nxt_unit_app_test.c), and +# nxt_unit_response_buf_alloc() refuses anything above PORT_MMAP_DATA_SIZE, so +# a request that fills a segment leaves it unable to build a response at all. +# That is a property of the sample app, not of the port handling under test. +# +# The legacy `wasm` runtime is absent: its client (test/test_wasm.py) is a +# WASI smoke guest, not a body-mirroring app, and sanitize.yml has no wasm +# leg. A row here would skip in every leg that exists. +RUNTIMES = { + 'libunit': (ApplicationLibunit, 'libunit', [], [], False, False), + 'go': (ApplicationGo, 'mirror', ['go'], [], True, True), + 'java': (ApplicationJava, 'mirror', ['java'], [], True, True), + 'node': (ApplicationNode, 'mirror', ['node'], [], True, True), + # there is no test/perl/mirror; body_array answers a fixed 10-byte body, + # which is enough to drive GET traffic through the churn + 'perl': (ApplicationPerl, 'body_array', ['perl'], [], False, True), + 'php': (ApplicationPHP, 'mirror', ['php'], [], True, True), + 'python': (ApplicationPython, 'mirror', ['python'], [], True, True), + 'ruby': (ApplicationRuby, 'mirror', ['ruby'], [], True, True), + 'wasm-wasi-component': ( + ApplicationWasmComponent, + 'hello_world', + ['wasm-wasi-component'], + ['cargo_component'], + False, + True, + ), +} + +BODY = '0123456789' * 200 + +# PORT_MMAP_DATA_SIZE on a non-NXT_MMAP_TINY_CHUNK build +# (src/nxt_port_memory_int.h): the payload area of one shared-memory segment, +# and therefore the amount of request data the router can hand an application +# before it has to create another one. +SEGMENT = 10 * 1024 * 1024 + +KEEPALIVE = {'Host': 'localhost', 'Connection': 'keep-alive'} + +# Requests per measurement window in test_app_lifecycle_app_fds_stable(): the +# size of the burst is what turns "one descriptor per request" into a number +# no lazy allocation can reach. +BURST = 40 + + +class LifecycleApp: + """Ties a runtime's client to its app name and request shape.""" + + def __init__(self, runtime, client, name, mirror, oversized): + self.runtime = runtime + self.client = client + self.name = name + self.mirror = mirror + self.oversized = oversized + + def load(self, **kwargs): + self.client.load(self.name, **kwargs) + + def conf_processes(self, conf): + assert 'success' in self.client.conf( + conf, f'applications/{self.name}/processes' + ), f'{self.runtime}: configure processes' + + def restart(self): + assert 'success' in self.client.conf_get( + f'/control/applications/{self.name}/restart' + ), f'{self.runtime}: restart' + + def request(self, sock=None, keepalive=False, **kwargs): + """One request that forces real request/response traffic. + + POST with a body where the runtime mirrors it back -- bodies travel + through the shared-memory path that carries descriptors to the app -- + plain GET otherwise. Returns (response, socket); the socket is None + unless the connection was kept alive. + + BODY is small on purpose, and so no request here spools: a body past + body_buffer_size reaches the application as a descriptor to a file + instead, and the runtime that would suffer most from one is the wasm + component, whose read loop does not terminate on a short read. + """ + if keepalive or sock is not None: + # A COPY. HTTP1.http() writes Content-Length into the dict it + # is handed (test/unit/http.py), so passing the module-level + # object would leave that header in it for good: the first + # mirrored POST poisons every later keep-alive GET, which then + # advertises a body it never sends and hangs until the read + # timeout. Only one language is built per CI leg, so today this + # needs two runtimes in one session to bite -- which is exactly + # what a local full run is. + kwargs['headers'] = dict(KEEPALIVE) + kwargs['start'] = True + # the server will not close a keep-alive connection, so recvall() + # cannot stop on EOF. framed=True stops the read when the + # response is complete by its Content-Length or its chunked + # framing. Every app in RUNTIMES sends one of the two. Before + # this, every keep-alive read waited for the full read_timeout: + # 15 reads in test_app_lifecycle_churn_under_keepalive, 30 s per + # runtime. + # + # read_timeout is now only the limit for an app that has not + # answered. Two seconds gives margin for an ASan worker on a + # loaded runner. A startup race that needs more than that is + # closed by waiting for the workers instead. + kwargs['framed'] = True + kwargs['read_timeout'] = 2 + + if sock is not None: + kwargs['sock'] = sock + + if self.mirror: + out = self.client.post(body=BODY, **kwargs) + else: + out = self.client.get(**kwargs) + + resp, sock = out if isinstance(out, tuple) else (out, None) + + # recvall() returns b'' rather than failing when a non-default + # read_timeout expires, and an empty response parses to {} -- which + # would surface as a KeyError instead of naming the runtime that went + # quiet. + assert 'status' in resp, f'{self.runtime}: no response before timeout' + + assert resp['status'] == 200, f'{self.runtime}: status' + + if self.mirror: + assert resp['body'] == BODY, f'{self.runtime}: mirror' + + return resp, sock + + def oversized_request(self, pool): + """One POST the router's `pool` existing segments cannot serve. + + nxt_router_prepare_msg() (src/nxt_router.c) copies the whole in-memory + body into the router's outgoing segments for this application, filling + a segment that has room and creating another when none has. A body of + `pool` whole segments is therefore always one chunk too large for the + `pool` segments already there -- the request line and fields took one + out of the first -- and forces exactly one to be created. + + The body has to reach the application through shared memory rather + than as a spool file, which is what the caller's body_buffer_size is + for; a spooled body travels as a descriptor and never touches the + segments. + + Nothing is asserted about the echoed body: the size comes from the + router's geometry, not from what a runtime is willing to hand back, + and the claim here is that the router created a segment, which the + caller checks directly. + """ + + resp = self.client.post( + body='x' * (pool * SEGMENT), read_buffer_size=1024 * 1024 + ) + + assert resp['status'] == 200, f'{self.runtime}: oversized status' + + def pids(self): + # Worker pids come from the PROTOTYPE's children, not from the + # workers' own process titles. + # + # `unit: "" application` is set by nxt_process_title() in the + # forked child (src/nxt_process.c), and for an embedded module that + # is the end of it. A `"type": "external"` application -- Go, Node, + # and the libunit test app -- then execve()s the target binary + # (src/nxt_external.c), which replaces argv[] wholesale, so the + # worker shows up in ps as its own executable and the title is gone. + # Matching on it finds nothing for exactly the runtimes this module + # most wants to cover. + # + # The prototype does not execve, so `unit: "" prototype` + # (built in src/nxt_main_process.c) survives for every runtime, and + # its children are the workers. ppid is read from ps rather than + # via `--ppid`, which is procps-specific. + time.sleep(0.2) + + output = subprocess.check_output( + ['ps', 'ax', '-o', 'pid=', '-o', 'ppid=', '-o', 'args='] + ).decode() + + rows = [] + for line in output.splitlines(): + parts = line.split(None, 2) + + if len(parts) == 3: + rows.append(parts) + + marker = f'unit: "{self.name}" prototype' + prototypes = {pid for pid, _, args in rows if marker in args} + + if not prototypes: + return set() + + return {pid for pid, ppid, _ in rows if ppid in prototypes} + + def wait_for_pids(self, count, timeout=15): + # `timeout` is nominal: pids() sleeps 0.2s of its own per sample, so + # the real ceiling is about twice this. Left as-is rather than + # tightened, because the margin is what absorbs a slow sanitizer + # teardown; the name is the thing that lies, so it is documented. + for _ in range(timeout * 5): + pids = self.pids() + + if len(pids) == count: + return pids + + time.sleep(0.2) + + return self.pids() + + def alive(self, pids): + """Which of `pids` still exist, prototype or no prototype. + + pids() derives workers from the prototype's children, so a teardown + that removes the prototype BEFORE its children leaves those children + reparented and invisible to it -- and an absent prototype would then + read as "no workers left", which is exactly the conclusion the + teardown assertions must not reach by default. Asking about + specific pids keeps them honest: a worker that outlives its + prototype still answers here. + + A pid the kernel recycled onto an unrelated process would count as + alive. Over the seconds these tests span that is remote, and it + would fail an assertion rather than pass one, which is the safe + direction to be wrong in. + """ + + if not pids: + return set() + + output = subprocess.check_output(['ps', 'ax', '-o', 'pid=']).decode() + running = { + line.strip() for line in output.splitlines() if line.strip() + } + + return set(pids) & running + + def wait_for_gone(self, pids, timeout=15): + """Wait for every pid in `pids` to leave the process table.""" + + for _ in range(timeout * 5): + left = self.alive(pids) + + if not left: + return set() + + time.sleep(0.2) + + return self.alive(pids) + + +def _skip_reason(runtime): + _, _, modules, features, _, _ = RUNTIMES[runtime] + + if runtime == 'libunit': + # built only by `./configure --tests && make tests` + if not Path(ApplicationLibunit.executable()).is_file(): + return 'build/unit_app_test not built (configure --tests)' + + return None + + available = option.available['modules'] + missed = [m for m in modules if not available.get(m)] + if missed: + return f'no {", ".join(missed)} module(s)' + + available = option.available['features'] + missed = [f for f in features if not available.get(f)] + if missed: + return f'{", ".join(missed)} feature(s) not supported' + + return None + + +@pytest.fixture(params=list(RUNTIMES)) +def app(request): + runtime = request.param + + reason = _skip_reason(runtime) + if reason is not None: + pytest.skip(f'{runtime}: {reason}') + + factory, name, _, _, mirror, oversized = RUNTIMES[runtime] + + return LifecycleApp(runtime, factory(), name, mirror, oversized) + + +FdCounts = namedtuple('FdCounts', ('sockets', 'shm', 'spool')) + + +def count_fds(pid): + """Descriptors of the kinds libunit hands an application process. + + A raw count of /proc//fd cannot tell a libunit leak from the host + runtime opening something of its own, and some runtimes legitimately do: + a JVM may open a jar, a JIT artefact or an eventfd part-way through a run + and keep it. Counting everything made this test fail on java-26 for a + single descriptor while java-17, 21 and 25 passed -- a difference in the + JVM, not in the port handling this test exists to watch. + + So count only what the docstring of the test actually claims: sockets + (ports) and shared memory segments (incoming mmap). Anything the + language runtime opens for its own purposes is deliberately out of scope, + and a real leak of a port or a segment still fails. + + Counted per class rather than as one total, because the classes do not + behave alike: an application acquires port sockets lazily, one per router + engine, so their count legitimately rises for a while, while a segment or + a spool descriptor is closed by the same message that delivered it and + must never rise at all. A single total would have to carry the socket + tolerance, and would then be blind to a per-segment leak. Separate + counts let the caller hold each class to its own rule. + + Segments are narrowed further, because "shared memory" is not + unit-specific -- php's opcache maps /memfd:opcache_lock, and a JVM heap + can appear as /memfd:java_heap -- so only names carrying `unit.` count. + + Spool files are narrowed by name, not by "regular file under the temp + dir". The temp dir is not unit's alone: it holds unit.log, which every + worker keeps open for its whole life, and ApplicationJava.prepare_env() + stages the whole webapp into {temp_dir}/java + (test/unit/applications/lang/java.py), so a directory rule counts the + JVM's classloader descriptors on WEB-INF/classes/*.class -- exactly the + runtime-specific noise the narrowing above exists to drop. A spool file + is created as mkstemp("/req-XXXXXXXX") and unlinked immediately + (nxt_h1p_request_body_read(), src/nxt_h1proto.c), so its link target is + `{temp_dir}/req-XXXXXXXX (deleted)`; match that basename. + + The class is kept as a guard, but no test here reaches it: bodies are + either 2000 bytes (well under the 16 KiB default body_buffer_size) or + sent by test_app_lifecycle_app_fds_stable, which raises that buffer above + the body precisely so the body does NOT spool. + """ + + fds = Path(f'/proc/{pid}/fd') + + if not fds.is_dir(): + return None + + sockets = 0 + shm = 0 + spool = 0 + + try: + for fd in fds.iterdir(): + try: + target = os.readlink(fd) + + except OSError: + # the descriptor closed underneath us; it is not a leak + continue + + if target.startswith('socket:'): + sockets += 1 + + elif target.startswith('/memfd:') or target.startswith( + '/dev/shm/' + ): + if 'unit.' in target: + shm += 1 + + elif target.startswith(f'{option.temp_dir}/'): + if _is_spool(target): + spool += 1 + + except OSError: + return None + + return FdCounts(sockets, shm, spool) + + +def _is_spool(target): + """Is this /proc//fd link target a request spool file?""" + + deleted = ' (deleted)' + + if target.endswith(deleted): + target = target[: -len(deleted)] + + return os.path.basename(target).startswith('req-') + + +def fd_targets(pid): + """What the descriptors of `pid` point at, for a failure message. + + A bare count says growth happened; this says what grew, which is the + difference between a report someone can act on and one that starts an + investigation from nothing. + + A list rather than a set: two descriptors on the same target are exactly + what a leak looks like, and deduplicating them would print fewer entries + than the count that failed. + + Labelled by the classes count_fds() counts, so the report and the number + that failed describe the same thing; everything else goes to `other`, + which is printed but never counted -- when growth is real it is useful to + see what the runtime is holding, and when it is not, its absence from the + counted classes is the answer. + """ + + targets = {'sockets': [], 'shm': [], 'spool': [], 'other': []} + + fds = Path(f'/proc/{pid}/fd') + + if not fds.is_dir(): + return targets + + try: + for fd in fds.iterdir(): + try: + target = os.readlink(fd) + + except OSError: + continue + + if target.startswith('socket:'): + targets['sockets'].append(target) + + elif ( + target.startswith('/memfd:') or target.startswith('/dev/shm/') + ) and 'unit.' in target: + targets['shm'].append(target) + + elif target.startswith(f'{option.temp_dir}/') and _is_spool( + target + ): + targets['spool'].append(target) + + else: + targets['other'].append(target) + + except OSError: + return targets + + return {name: sorted(items) for name, items in targets.items()} + + +def shm_segments(pid): + """Shared-memory segments this process was handed by another one. + + The descriptor that carries a segment is the one the fd test is blind to + by default: nxt_unit_incoming_mmap() (src/nxt_unit.c) maps it and + nxt_unit_process_msg()'s `done:` closes it, so a healthy application keeps + the mapping and no descriptor -- which is exactly why a *leak* of that + descriptor is only visible in a window where a segment actually arrived. + Counting the mappings is how the test knows a window contained one. + + Read from /proc//maps, and narrowed three ways, because a process + maps shared memory for several unrelated reasons. Segment-sized, which + drops the port queues (a few pages each). Named `unit.`, which drops + whatever the language runtime maps for itself. Not named `unit..`, + which drops the segments this process created for its own responses -- + it opens and closes those itself (nxt_unit_new_mmap()), so they carry no + incoming descriptor to leak. + + Returns the segment names rather than a count, so a failure can say which + ones were there. + """ + + maps = Path(f'/proc/{pid}/maps') + + segments = set() + + try: + lines = maps.read_text().splitlines() + + except OSError: + return segments + + for line in lines: + parts = line.split(None, 5) + + if len(parts) < 6: + continue + + bounds, path = parts[0], parts[5] + + if not path.startswith('/memfd:') and not path.startswith('/dev/shm/'): + continue + + if f'unit.{pid}.' in path or 'unit.' not in path: + continue + + start, _, end = bounds.partition('-') + + if int(end, 16) - int(start, 16) < SEGMENT: + continue + + segments.add(path) + + return segments + + +def test_app_lifecycle_process_churn(app): + """Scale the worker pool up and down with traffic in between. + + A scale-up spawns workers that run the full libunit port handshake + (NEW_PORT carrying a socket and a queue descriptor, the add_port callback, + PORT_ACK); a scale-down tears them down (QUIT, REMOVE_PID, port destroy). + Generalization of test_go_application.py's + test_go_application_port_fd_churn to every runtime. + + The loop does NOT wait for each pool size to be reached, which is + deliberate: conf_processes() returns on configuration success, not on + spawn, so an un-awaited rewrite races the next teardown against workers + that are still starting -- the overlap this test exists to put under a + sanitizer. That also means a given iteration may spawn fewer than four + workers, so the count is not asserted per round. One awaited cycle runs + first, to establish that scaling works at all before the racing starts; + without it a run in which no worker was ever spawned would look the same + as a run in which every one of them was. + """ + app.load() + + app.conf_processes('4') + assert len(app.wait_for_pids(4)) == 4, f'{app.runtime}: scaled up to 4' + + app.conf_processes('1') + assert len(app.wait_for_pids(1)) == 1, f'{app.runtime}: scaled down to 1' + + for _ in range(10): + app.conf_processes('4') + app.request() + + app.conf_processes('1') + app.request() + + +def test_app_lifecycle_restart_control(app): + """/control/applications//restart storm. + + A control restart replaces every worker: the outgoing generation is sent + QUIT and drops its ports while the incoming one announces fresh ports to + the same router. The pid-rotation assertion keeps a no-op restart from + making this pass vacuously. + """ + app.load(processes=2) + + pids = app.wait_for_pids(2) + assert len(pids) == 2, f'{app.runtime}: two workers' + + for _ in range(5): + app.restart() + + # Wait for the OLD generation to go before sampling the new one. + # nxt_router_app_restart_handler() (src/nxt_router.c) sends QUIT to + # the old prototype and replies RPC_READY_LAST in the same handler, + # so restart() returns while the old prototype and its workers are + # normally still alive. Sampling first would take the first set of + # two it sees -- which can be the two OLD workers, or one of each + # while both prototypes coexist -- and the disjointness assertion + # below would then fail a perfectly healthy restart. Under the + # sanitizer, where teardown is slow, that is the likely outcome + # rather than the unlucky one. + assert ( + app.wait_for_gone(pids) == set() + ), f'{app.runtime}: workers from the previous generation still alive' + + new_pids = app.wait_for_pids(2) + + assert len(new_pids) == 2, f'{app.runtime}: two workers after restart' + + # Kept as the pid-reuse guard: with the old generation already gone, + # an overlap here would mean the kernel handed a new worker a pid we + # had just watched die. + assert not new_pids & pids, f'{app.runtime}: all workers replaced' + + app.request() + + pids = new_pids + + +def test_app_lifecycle_reconfigure_storm(app): + """Remove and re-add the whole application repeatedly. + + Clearing `applications` makes the router QUIT every worker and drop the + app's ports; the app process then runs nxt_unit_quit() -> + ctx_free/lib_release, which is where the port destroy path closes the + descriptors libunit owns. Re-adding runs the whole spawn/handshake again. + + The configuration is captured once and replayed, because several runtime + clients stage their application into the temp dir on load() and cannot + stage it twice. + """ + app.load(processes=2) + app.request() + + conf = app.client.conf_get('/config') + + for _ in range(5): + # Not a bare pids() snapshot. Starting the pool is asynchronous and + # app.request() only proves ONE worker is ready, so an unvalidated + # sample can hold one of the two while the second is still spawning. + # A worker missing from `before` is one wait_for_gone() will never + # wait for, and wait_for_pids(0) would then be satisfied by the + # absent prototype again -- putting the hole straight back. + # + # "processes": 2 here is a fixed count, not a spare/max range, so the + # expected number is exact. + before = app.wait_for_pids(2) + + assert len(before) == 2, ( + f'{app.runtime}: expected 2 workers before removal, ' + f'saw {sorted(before)}' + ) + + assert 'success' in app.client.conf( + {"listeners": {}, "applications": {}} + ), f'{app.runtime}: clear conf' + + # Both checks are needed. wait_for_gone() names the processes that + # were serving a moment ago and waits for those to die, which an + # early prototype exit cannot fake; wait_for_pids(0) then confirms + # nothing was started to replace them. + assert ( + app.wait_for_gone(before) == set() + ), f'{app.runtime}: pre-removal workers still alive' + + assert app.wait_for_pids(0) == set(), f'{app.runtime}: workers gone' + + assert 'success' in app.client.conf(conf), f'{app.runtime}: restore' + + app.request() + + +def test_app_lifecycle_churn_under_keepalive(app): + """Churn the worker pool while keep-alive connections stay open. + + Requests arriving on connections that outlive a worker generation are the + case where a router-side response port and an app-side port lifecycle + overlap -- the interleaving in which a stale descriptor is most likely to + be closed a second time. + """ + app.load(processes=2) + + # not decoration: the first keep-alive read is the one racing application + # startup, and it gives up after read_timeout with whatever it has rather + # than failing, so a worker that is not up yet reads as an empty response. + app.wait_for_pids(2) + + socks = [] + try: + for _ in range(3): + _, sock = app.request(keepalive=True) + socks.append(sock) + + for i in range(4): + app.conf_processes('4' if i % 2 else '1') + + for j, sock in enumerate(socks): + _, socks[j] = app.request(sock=sock) + + finally: + for sock in socks: + if sock is not None: + sock.close() + + +def test_app_lifecycle_app_fds_stable(app): + """No per-request descriptor growth inside the application process. + + conftest's _check_fds() only watches main/router/controller. The libunit + paths that hand descriptors to an app -- incoming mmap segments and port + sockets -- are only observable here, so a leak of one descriptor per + request or per incoming segment is invisible to the rest of the suite. + + Per request and per incoming segment is the whole claim. A leak in the + one-time paths -- recv_msg->fd[0] in nxt_unit_process_new_port(), say -- + is NOT covered: this test loads the application with processes=1 and + never restarts or reconfigures it, so every NEW_PORT it sees is a first + discovery, lands in a single window, and is indistinguishable from the + legitimate lazy acquisition described below. Port-announcement churn is + covered by the other tests in this module, and only through the alerts a + bad close writes into unit.log (Log.check_alerts()). + + Request spool files are the third kind of descriptor an application is + handed, and this module deliberately does not exercise them: every body + it sends is either well under body_buffer_size or accompanied by a + buffer raised above it. count_fds() still counts them, as a guard for + whoever adds the coverage. + + Measured across THREE identical bursts rather than before-and-after one, + and per descriptor class rather than as one total. A single burst cannot + tell a leak from lazy one-time allocation, and the app process does + allocate lazily. Counting one window made this fail for a single + descriptor on java-26 and again on python, both times for growth that + never repeated. + + Only sockets allocate lazily, so only sockets pay for it: + + * shm and spool descriptors must be flat. Both are closed by the same + message that delivered them -- nxt_unit_process_msg()'s `done:` -- + so any growth at all, in either window, is a leak. Every burst + forces an incoming segment (below), which keeps a per-segment leak + visible in a single window rather than cancelling out. + + * sockets are bounded, not flat. The router runs nxt_ncpu engine + threads (rtcf->threads, src/nxt_router.c), an application does not + learn an engine's port until the first request routed through that + engine, and nxt_unit_process_new_port() keeps that NEW_PORT socket + (src/nxt_unit.c). Discovery is one-time PER ENGINE, not per + process: with several engines still undiscovered after the first + window, windows two and three can each add one, so "growth in both + windows" is a legitimate outcome and cannot be the rule. + + The socket rule is therefore one bound, and the discovery ceiling is + what makes it safe. There are at most nxt_ncpu engines, the test does + not touch `listen_threads`, and nxt_ncpu is at most CPU_COUNT() of the + affinity mask, capped by _SC_NPROCESSORS_ONLN (nxt_lib.c) -- which is + what os.sched_getaffinity() reports here. A cgroup v2 CPU limit can + only lower nxt_ncpu. So: total socket growth across the two windows may + not exceed that ceiling. Discovery can never exceed it, + however the first touches fall across the windows -- one window, both, + or a whole burst's worth of fresh engines on a machine with more CPUs + than the burst has requests -- so this bound cannot be tripped by the + engine count. A per-window bound was tried and dropped for exactly that + last case: on a host with BURST or more undiscovered engines, BURST fresh + connections can legitimately retain BURST new port sockets in one window. + + What the socket rule gives up is a socket leak that stays under the + ceiling: one socket per request adds 2 * BURST over the two windows and + is caught on any machine with fewer than 2 * BURST CPUs -- every CI + runner by a wide margin -- but on a larger host, or for a leak slower + than one per request, growth that sums to no more than nxt_ncpu reads as + discovery. That is unavoidable while discovery is live, and it is the + class where a leak is cheapest to spot elsewhere -- a port descriptor + mishandled in the churn tests above shows up as an alert. The classes + with no tolerance at all are the ones only this test watches. + + Measured on php with this workload reduced to six plain 40-request + windows, the app's socket count ran 8, 8, 9, 9, 9, 9: one discovery, + after the second window, then flat. The bound passes that; a + per-request leak (8, 48, 88) trips it on any host under 80 CPUs. + + Pinning the router to one engine would remove the variable at its source, + and is the more obvious fix -- but `listen_threads` is a live + reconfiguration, and restoring the default at teardown leaves conftest's + own _check_fds() looking at a router that has rebuilt its engines, which + it reports as a 21-descriptor leak in the router. + + Each burst ends with one oversized POST, and that is not decoration. The + router creates a segment only when the ones it has for this application + cannot serve a request, and 2000-byte requests release their chunks as + they complete, so a burst of them reuses the segment the warm-up caused + and never makes the router announce another. A window that received no + segment cannot show a descriptor leaked per segment -- the one leak this + test names in its own docstring would be invisible to it, and the flat + rule would pass a leaking build. The oversized request forces an + announcement; the burst asserts that it happened, so the probe cannot + quietly stop working. + + The POST grows by one segment per burst because the pool grows with it: + the segments an earlier burst forced are free again by the time the next + one runs, and would swallow a repeat of the same body. What stays equal + across the windows is the thing being counted -- one segment created, one + descriptor received -- which is what makes the differences between them + leaks rather than workload differences. + + Runtimes whose application cannot answer such a request keep the bursts + and lose only the segment part of the claim (see RUNTIMES); driving one + into a response it cannot build would test the sample app, not libunit. + """ + app.load(processes=1) + + # The oversized body must stay in memory to travel through shared memory + # at all: past body_buffer_size the router spools it to a file and sends + # a descriptor instead (nxt_h1p_request_body_read(), src/nxt_h1proto.c). + # + # The margin over the largest body sent below is deliberate. Sitting on + # the boundary would make the probe depend on which side of it a spooling + # test is written, and the wasm component's read loop + # (src/wasm-wasi-component/src/lib.rs) does not terminate on a short read, + # so a body that spooled would hang that leg rather than fail it. + assert 'success' in app.client.conf( + { + "http": { + "body_buffer_size": 4 * SEGMENT, + "max_body_size": 4 * SEGMENT, + } + }, + 'settings', + ), f'{app.runtime}: buffer oversized bodies in memory' + + pids = app.wait_for_pids(1) + assert len(pids) == 1, f'{app.runtime}: single worker' + + pid = pids.pop() + + # warm-up: the first requests create the outgoing mmap segments and, in + # threaded runtimes, the per-thread contexts + for _ in range(10): + app.request() + + def burst(pool): + for _ in range(BURST): + app.request() + + if app.oversized: + app.oversized_request(pool) + + # let anything the burst opened and is about to close settle, so a + # descriptor in flight is not counted as retained + settled = count_fds(pid) + + for _ in range(50): + n = count_fds(pid) + + if n is None or n == settled: + settled = n + break + + settled = n + time.sleep(0.1) + + # a disappearing worker is not a leak, but it is not a pass either: + # report it where it happened rather than as a count that is missing + assert settled is not None, ( + f'{app.runtime}: worker vanished mid-measurement' + ) + + return settled + + if count_fds(pid) is None: + pytest.skip('no /proc//fd on this platform') + + counts = [] + segments = [shm_segments(pid)] + + for pool in (1, 2, 3): + counts.append(burst(pool)) + segments.append(shm_segments(pid)) + + assert app.pids() == {pid}, f'{app.runtime}: same worker throughout' + + if app.oversized: + # Every window, not just the measured pair: a window that forced no + # segment would leave its own leak out of the baseline and make a + # per-segment leak read as growth that stopped. + for i in range(3): + assert len(segments[i + 1]) > len(segments[i]), ( + f'{app.runtime}: no incoming segment in burst {i + 1}, so a ' + f'leak of one cannot be seen: {sorted(segments[i])} -> ' + f'{sorted(segments[i + 1])}' + ) + + # the classes libunit closes as it goes: flat, in every window + for name in ('shm', 'spool'): + series = [getattr(n, name) for n in counts] + growth = [series[i + 1] - series[i] for i in range(2)] + + assert max(growth) <= option.fds_threshold, ( + f'{app.runtime}: {name} descriptor leak in application process: ' + f'{series} across three repeated identical bursts; open now: ' + f'{fd_targets(pid)}' + ) + + # sockets: bounded by the engine count, because port discovery is lazy + # and one-time per engine. nxt_ncpu is at most CPU_COUNT() of the + # affinity mask; a cgroup v2 CPU limit can only lower it (src/nxt_lib.c). + # The test does not touch `listen_threads`. + ncpu = max(len(os.sched_getaffinity(0)), 1) + + sockets = [n.sockets for n in counts] + growth = [sockets[i + 1] - sockets[i] for i in range(2)] + + # only the rises are summed: a window that lost a socket must not buy + # the next one room to leak in + total = sum(n for n in growth if n > 0) + + assert total <= ncpu + option.fds_threshold, ( + f'{app.runtime}: socket leak in application process: {sockets} ' + f'across three repeated identical bursts, growing by {total} in ' + f'total where lazy port discovery can account for at most {ncpu} ' + f'(one per router engine); open now: {fd_targets(pid)}' + ) diff --git a/test/test_app_start_timeout.py b/test/test_app_start_timeout.py new file mode 100644 index 000000000..3fb826cc8 --- /dev/null +++ b/test/test_app_start_timeout.py @@ -0,0 +1,708 @@ +"""A worker that starts but never announces itself must not strand the start. + +An application process is only ever announced by the process itself, from +nxt_unit_init() -> PROCESS_READY. A `"type": "external"` binary that never +gets that far -- /bin/sleep is the shortest example, but a wrapper script or a +runtime blocked in its own init reaches the same state -- answers nothing, and +because it stays alive nothing kills it either, so no REMOVE_PID converts the +wait into an error. + +Without a "limits": {"start_timeout"} that leaves the router's START_PROCESS +RPC armed for good. With the default "processes" that RPC is the only +continuation of nxt_router_conf_apply(), so: + + * the configuration PUT never returned, and + * the controller parks the in-flight request at the head of its queue, so + nxt_controller_check_postpone_request() queued every later control request + behind it -- GET /status included. + +The whole control plane was unavailable for as long as that process lived. + +The bound is opt-in: NXT_APP_START_TIMEOUT is 0, because the window it +measures is the application's own startup and a default would fail a slow but +honest one (see the comment on that macro). So these tests set it explicitly, +and test_app_start_timeout_default_is_unbounded() pins the default. + +Every test here would hang rather than fail on a build without the deadline, +which is why the module is worth running under `timeout` when comparing. +""" + +import re +import subprocess +import time + +import pytest + +from unit.control import Control +from unit.log import Log +from unit.option import option + +prerequisites = {} + +client = Control() + + +# Seconds, like every other "limits" member (NXT_CONF_MAP_MSEC). Long enough +# that a slow-but-honest start is not mistaken for a wedged one, short enough +# to keep the module quick. There is no product default to inherit: +# NXT_APP_START_TIMEOUT is 0, meaning unbounded. +START_TIMEOUT = 3 + +STUCK_ARG = 'unit-test-app-start-timeout' + + +def _stuck_conf(start_timeout=START_TIMEOUT): + app = { + 'type': 'external', + # `sh -c CMD NAME` runs CMD with $0 = NAME, which puts a marker in the + # process title that _stuck_pids() can grep for without matching a + # sleep that belongs to something else on the box. The trailing `:` + # keeps sh from exec'ing the sleep over itself, which would drop the + # marker again. + 'executable': '/bin/sh', + 'arguments': ['-c', 'sleep 600; :', STUCK_ARG], + 'limits': {'start_timeout': start_timeout}, + } + + return { + 'listeners': {'*:8080': {'pass': 'applications/stuck'}}, + 'applications': {'stuck': app}, + } + + +def _serving_conf(): + return { + 'listeners': {'*:8080': {'pass': 'routes'}}, + 'routes': [{'action': {'return': 200}}], + } + + +def _stuck_pids(): + """Pids of the workers this module started, if any are still alive.""" + + res = subprocess.run( + ['pgrep', '-f', STUCK_ARG], + capture_output=True, + text=True, + check=False, + ) + + return [int(p) for p in res.stdout.split()] + + +def _stuck_protos(): + """Pids of the prototypes started for the wedging application. + + A prototype cannot exit while a child of it is alive, so it is half of + what a leaked start leaves behind and has to be counted separately: the + marker in _stuck_pids() is the worker's, not the prototype's. + """ + + res = subprocess.run( + ['pgrep', '-f', 'unit: "stuck" prototype'], + capture_output=True, + text=True, + check=False, + ) + + return [int(p) for p in res.stdout.split()] + + +def _wait_gone(seconds=10): + """Wait for every worker and prototype of the wedging app to be gone.""" + + for _ in range(int(seconds * 10)): + if _stuck_pids() == [] and _stuck_protos() == []: + break + + time.sleep(0.1) + + return _stuck_pids(), _stuck_protos() + + +def _children(pid): + """Pids the kernel currently lists as children of pid.""" + + try: + with open(f'/proc/{pid}/task/{pid}/children', encoding='utf-8') as f: + return [int(p) for p in f.read().split()] + + except OSError: + return [] + + +def _reap_stuck(): + """Kill the silent workers, and the `sleep` each of them forked. + + The marker is the shell's $0, so _stuck_pids() finds the shell and never + the `sleep 600` it forked -- and killing only the shell reparents that + sleep to the nearest subreaper, where it keeps running. Nothing in this + module notices it afterwards: the pid no longer carries the marker and no + longer has a parent we know. Unit's own process group is the backstop + (conftest reaps it in unit_stop), but Unit is started once per session, so + until then one sleep per test survives into every module that follows. + + Collect the children before killing the shell: afterwards they are not its + children any more. + """ + + for pid in _stuck_pids(): + children = _children(pid) + + subprocess.run(['kill', '-9', str(pid)], check=False) + + for child in children: + subprocess.run(['kill', '-9', str(child)], check=False) + + +@pytest.fixture(autouse=True) +def _stuck_app(skip_alert): + """Every test here deliberately produces alerts; name them all once. + + * the deadline's own alert; + * "failed to apply new conf", which nxt_router_conf_error() logs for the + PUT the deadline makes fail; + * "exited on signal", which the prototype logs for a silent worker: the + router has no pid for one and cannot kill it, so either the test reaps + it or the prototype does, one "start_timeout" after it is told to quit + (see nxt_proto_quit_children(), and the three tests that pin it below). + """ + + skip_alert( + r'did not become ready in time', + r'failed to apply new conf', + r'exited on signal', + ) + + yield + + # A silent worker outlives the start that asked for it, and while its + # application is live nothing collects it -- the router never learns its + # pid. Clean up regardless, so a failure here does not leave sleeps + # behind for the next run. + _reap_stuck() + + +def test_app_start_timeout_put_fails(): + assert 'success' in client.conf(_serving_conf()), 'baseline configured' + + started = time.monotonic() + + resp = client.conf(_stuck_conf()) + + elapsed = time.monotonic() - started + + # The PUT must fail, not hang and not silently succeed. + assert 'error' in resp, f'stuck app rejected, got {resp}' + assert 'Failed to apply new configuration' in resp['error'], resp + + # Bounded by the deadline, with room for the fork and the round trips. + assert elapsed < START_TIMEOUT + 10, f'PUT returned in {elapsed:.1f}s' + + # And not *before* the deadline: a start that is merely slow must not be + # failed early, so the bound has to be the configured one. + assert elapsed >= START_TIMEOUT - 1, f'PUT returned in {elapsed:.1f}s' + + +def test_app_start_timeout_control_plane_survives(): + assert 'success' in client.conf(_serving_conf()), 'baseline configured' + + assert 'error' in client.conf(_stuck_conf()), 'stuck app rejected' + + # The symptom that made this severe: with the request parked at the head + # of the controller's queue, every later control request queued behind it. + started = time.monotonic() + + status = client.conf_get('/status') + + assert time.monotonic() - started < 10, 'GET /status answers promptly' + assert 'connections' in status, status + + # The failed PUT must not have installed anything. + assert 'stuck' not in status['applications'], status['applications'] + + # And the previous configuration is still the live one, still serving. + assert client.conf_get() == _serving_conf(), 'previous config retained' + + resp = client.get(url='/') + + assert resp['status'] == 200, 'previous config still serves' + + +def test_app_start_timeout_alert(wait_for_record): + assert 'success' in client.conf(_serving_conf()), 'baseline configured' + assert 'error' in client.conf(_stuck_conf()), 'stuck app rejected' + + assert ( + wait_for_record( + r'app "stuck" process did not become ready in time', wait=50 + ) + is not None + ), 'the alert names the application' + + assert re.search( + r'start_timeout', Log.read() + ), 'the alert points at the knob that raises the bound' + + +def test_app_start_timeout_knob_validated(): + """The knob is a real "limits" member, not something the parser drops. + + Asserted against an application that is actually in the configuration: a + PUT to /config/applications//limits on an app that was never applied + is rejected with "Value doesn't exist.", which would satisfy an assertion + that only looks for an error. /bin/true announces nothing either, but it + exits at once, and a start that fails by the worker dying is the + pre-existing path -- it needs no deadline and leaves nothing behind. + """ + + conf = { + 'listeners': {'*:8080': {'pass': 'routes'}}, + 'routes': [{'action': {'return': 200}}], + 'applications': { + 'stuck': { + 'type': 'external', + 'executable': '/bin/true', + 'processes': {'spare': 0}, + } + }, + } + + assert 'success' in client.conf(conf), 'application configured' + + path = 'applications/stuck/limits' + + resp = client.conf('{"start_timeout": "soon"}', path) + + assert 'error' in resp, 'a non-integer start_timeout is rejected' + + # Rejected for the right reason. A build without the knob rejects this + # with "Unknown parameter", and an application that is not in the + # configuration is rejected with "Value doesn't exist." -- either would + # satisfy the assertion above for nothing. + detail = resp.get('detail', '') + + assert 'Unknown parameter' not in detail, resp + assert "doesn't exist" not in detail, resp + assert 'integer' in detail, resp + + # And 0 -- the default, meaning unbounded -- is accepted and survives. + assert 'success' in client.conf( + '{"start_timeout": 0}', path + ), 'zero accepted' + + assert client.conf_get(f'{path}/start_timeout') == 0, 'zero round-trips' + + +def test_app_start_timeout_range_validated(): + """The knob is bounded by what a 32-bit millisecond clock can express. + + The value reaches the router through NXT_CONF_MAP_MSEC, which computes + (nxt_msec_t) seconds * 1000 and checks nothing. Above + NXT_INT32_T_MAX / 1000 the product lands past the sign bit, where + nxt_msec_diff() reads the deadline as already past -- so a configuration + asking for ~24.9 days would instead have fired immediately, silently + turning a bound meant to be generous into one that fails every start. + Negative seconds are an out-of-range conversion to an unsigned type. + """ + + conf = { + 'listeners': {'*:8080': {'pass': 'routes'}}, + 'routes': [{'action': {'return': 200}}], + 'applications': { + 'stuck': { + 'type': 'external', + 'executable': '/bin/true', + 'processes': {'spare': 0}, + } + }, + } + + assert 'success' in client.conf(conf), 'application configured' + + path = 'applications/stuck/limits' + + # Negative. + resp = client.conf('{"start_timeout": -1}', path) + + assert 'error' in resp, f'a negative start_timeout is rejected, got {resp}' + + detail = resp.get('detail', '') + + assert 'start_timeout' in detail, resp + assert 'negative' in detail, resp + + # Above NXT_INT32_T_MAX / 1000 == 2147483. + resp = client.conf('{"start_timeout": 2147484}', path) + + assert 'error' in resp, f'an out-of-range start_timeout is rejected, {resp}' + + detail = resp.get('detail', '') + + assert 'start_timeout' in detail, resp + assert '2147483' in detail, resp + + # And the bound itself is not off by one: the largest value the clock can + # express is still accepted. + assert 'success' in client.conf( + '{"start_timeout": 2147483}', path + ), 'the maximum is accepted' + + assert client.conf_get(f'{path}/start_timeout') == 2147483 + + +def test_app_start_timeout_retry_leaks_nothing(): + """Rejecting the same configuration N times must cost N times nothing. + + The deadline fails the PUT, and nxt_router_conf_error() discards the + temporary configuration -- with the nxt_app_t that + app->unaccounted_processes lives on. So there is no counter left to bound + the config-apply path with, and each retry forks a fresh prototype and a + fresh worker: ordinary control-plane retry automation exhausts host pids. + + What bounds it is not accounting but draining. Discarding the application + sends the prototype a QUIT, and the prototype is then able to act on it, + because nxt_proto_quit_children() signals a child that has never announced + itself instead of writing a port message nothing there reads. Before + that, both processes survived every retry -- and unitd's own exit. + """ + + retries = 5 + + assert 'success' in client.conf(_serving_conf()), 'baseline configured' + + for i in range(retries): + assert 'error' in client.conf(_stuck_conf()), f'PUT {i} rejected' + + workers, protos = _wait_gone() + + assert workers == [], ( + f'{len(workers)} silent workers survive {retries} rejected PUTs; ' + 'each retry is leaking the process it forked' + ) + + assert protos == [], ( + f'{len(protos)} prototypes survive {retries} rejected PUTs; a ' + 'prototype cannot exit while a child of it is alive' + ) + + # And none of it cost the control plane, which is the point of the bound. + assert 'connections' in client.conf_get('/status') + + assert client.conf_get() == _serving_conf(), 'previous config retained' + + +def test_app_start_timeout_default_is_unbounded(findall): + """With no "limits", a worker that is merely slow must still be waited for. + + NXT_APP_START_TIMEOUT is 0, so nothing is armed at all: the start is as + unbounded as it was before this change. Exercised with a worker that + really does take its time before announcing itself -- the libunit sample + app behind a `sleep`, which is the shape of every module's startup (user + code first, nxt_unit_init() after) -- so a default bound short enough to + matter would show up here as a failed PUT rather than as a served request. + """ + + delay = 4 + + conf = { + 'listeners': {'*:8080': {'pass': 'applications/slow'}}, + 'applications': { + 'slow': { + 'type': 'external', + 'working_directory': option.temp_dir, + # `sh -c 'sleep N; exec APP' -` announces only after the sleep; + # exec keeps the app in the process the prototype forked, and + # NXT_UNIT_INIT is inherited across it. + 'executable': '/bin/sh', + 'arguments': [ + '-c', + f'sleep {delay}; exec "$0"', + f'{option.current_dir}/build/unit_app_test', + ], + } + }, + } + + started = time.monotonic() + + assert 'success' in client.conf(conf), 'a slow start is still waited for' + + elapsed = time.monotonic() - started + + assert elapsed >= delay, f'the worker announced early ({elapsed:.1f}s)' + + assert client.get(url='/')['status'] == 200, 'the slow app serves' + + assert findall(r'did not become ready in time') == [], ( + 'no deadline fired without an explicit start_timeout' + ) + + +def _stuck_on_demand_conf(max_processes, start_timeout=START_TIMEOUT): + """The same silent worker, started by a request rather than by the PUT. + + "spare": 0 is what moves the start off the configuration-apply path: with + nothing to prefork the PUT returns at once, and the first request is what + asks for a process. That is the shape in which the deadline can fire more + than once for one application, which is what these tests are about. + """ + + conf = _stuck_conf(start_timeout) + conf['applications']['stuck']['processes'] = { + 'max': max_processes, + 'spare': 0, + } + + return conf + + +def _app_processes(name='stuck'): + return client.conf_get(f'/status/applications/{name}/processes') + + +def test_app_start_timeout_bounds_orphans(): + """"processes": {"max"} must bound the processes the deadline gives up on. + + The deadline fails the start, and the worker it gave up on keeps running: + the router has no pid for it and cannot kill it, and its application is + live, so nothing quits the prototype that could. Such a worker is in + neither app->processes, which only PROCESS_READY fills, nor + app->pending_processes, which the failure gives back -- so unless the + router counts it somewhere, every request forks another one and "max" + bounds nothing. Five requests against "max": 2 produced five `sleep` + processes. + + app->unaccounted_processes is that somewhere. Past the bound the + application stops forking, and a request that arrives then is answered + rather than parked: only a start can take a request back out of + ack_waiting_req, and no start is coming. + """ + + max_processes = 2 + requests = max_processes + 3 + + assert 'success' in client.conf( + _stuck_on_demand_conf(max_processes) + ), 'the on-demand config applies at once' + + assert _stuck_pids() == [], 'nothing forked before the first request' + + for i in range(requests): + started = time.monotonic() + + assert client.get(url='/')['status'] == 503, f'request {i} failed' + + elapsed = time.monotonic() - started + + if i < max_processes: + # A start really was attempted, so the deadline is what answered. + assert elapsed >= START_TIMEOUT - 1, ( + f'request {i} was failed before the deadline ({elapsed:.1f}s)' + ) + + else: + # Nothing left to start: the request must not wait for a process + # that will never be asked for, nor sit in ack_waiting_req until + # it times out. + assert elapsed < START_TIMEOUT, ( + f'request {i} waited {elapsed:.1f}s for a start that cannot ' + 'happen' + ) + + pids = _stuck_pids() + + assert len(pids) == max_processes, ( + f'{len(pids)} silent workers survive "max": {max_processes}; the ' + 'deadline is forking one per request again' + ) + + procs = _app_processes() + + assert procs['unaccounted'] == max_processes, procs + assert procs['running'] == 0 and procs['starting'] == 0, procs + + +def test_app_start_timeout_orphan_death_frees_the_slot(): + """The bound is a bound, not a one-way ratchet. + + The router does hear about the process it gave up on, once: the prototype + that forked it reaps it and notifies with the start's own stream still + attached, and nxt_router_remove_pid_handler() retypes that REMOVE_PID into + an RPC error on the handler the expiry left registered. So a slot comes + back when the process behind it dies, and the application can start again. + """ + + assert 'success' in client.conf(_stuck_on_demand_conf(1)) + + assert client.get(url='/')['status'] == 503, 'the start is bounded' + + pids = _stuck_pids() + + assert len(pids) == 1, pids + assert _app_processes()['unaccounted'] == 1 + + _reap_stuck() + + for _ in range(50): + if _app_processes()['unaccounted'] == 0: + break + + time.sleep(0.1) + + procs = _app_processes() + + assert procs['unaccounted'] == 0, ( + f'the dead process still holds its slot: {procs}' + ) + + # And the slot is usable again: this forks a second silent worker. + assert client.get(url='/')['status'] == 503, 'a start is possible again' + + assert len(_stuck_pids()) == 1, 'the freed slot started a process' + + +def test_app_start_timeout_slow_worker_adopted(): + """A worker that only just missed the deadline is kept, not thrown away. + + It announces itself on the stream the expiry left registered, and the + router moves its slot from unaccounted_processes to processes and adopts + the port. Discarding it instead would be the worse trade by far: the next + request would fork another worker that is equally slow, be failed by the + same deadline, and the application would never serve anything. + """ + + delay = START_TIMEOUT + 3 + + conf = { + 'listeners': {'*:8080': {'pass': 'applications/slow'}}, + 'applications': { + 'slow': { + 'type': 'external', + 'working_directory': option.temp_dir, + 'processes': {'max': 1, 'spare': 0}, + 'limits': {'start_timeout': START_TIMEOUT}, + 'executable': '/bin/sh', + 'arguments': [ + '-c', + f'sleep {delay}; exec "$0"', + f'{option.current_dir}/build/unit_app_test', + ], + } + }, + } + + assert 'success' in client.conf(conf), 'the slow app is configured' + + # The first request pays for the deadline: the worker is still sleeping. + assert client.get(url='/')['status'] == 503, 'the deadline fired' + + assert _app_processes('slow')['unaccounted'] == 1, _app_processes('slow') + + # It announces itself a few seconds later, and is taken on. + for _ in range(100): + if _app_processes('slow')['running'] == 1: + break + + time.sleep(0.2) + + assert _app_processes('slow') == { + 'running': 1, + 'starting': 0, + 'unaccounted': 0, + 'idle': 1, + 'detached': 0, + } + + assert client.get(url='/')['status'] == 200, 'the adopted worker serves' + + +def test_app_start_timeout_slow_worker_survives_a_quit(findall): + """A worker that is merely slow must ride out a quit, not be signalled. + + The prototype's deadline is for a worker that will never read the QUIT it + is sent (nxt_proto_quit_children()), and until one announces itself that + worker is indistinguishable from one that is simply still starting. The + deadline is how they are told apart: a slow one reads the QUIT out of its + port the moment nxt_unit_init() starts reading, which is well inside the + second "start_timeout" it is given here. + + A restart is the shape that catches it, and is what caught it: serving a + request replenishes the spare, so the application has a worker in the + middle of its start exactly when the old prototype is told to quit. + """ + + delay = 2 + + conf = { + 'listeners': {'*:8080': {'pass': 'applications/slow'}}, + 'applications': { + 'slow': { + 'type': 'external', + 'working_directory': option.temp_dir, + 'processes': {'max': 2, 'spare': 1, 'idle_timeout': 60}, + # Comfortably longer than the start below, so nothing here + # turns on a race between the two. + 'limits': {'start_timeout': delay * 10}, + 'executable': '/bin/sh', + 'arguments': [ + '-c', + f'sleep {delay}; exec "$0"', + f'{option.current_dir}/build/unit_app_test', + ], + } + }, + } + + assert 'success' in client.conf(conf), 'the slow app is configured' + + assert client.get(url='/')['status'] == 200, 'the slow app serves' + + # Serving that request is what asks for the second process, and it is + # still sleeping when the restart below quits the prototype that forked + # it. Nothing waits for it: waiting is what this test must not do. + assert 'success' in client.conf_get('/control/applications/slow/restart') + + # Long enough for both starts to finish and for a deadline that should + # not be armed at all to have fired if it were. + time.sleep(delay * 3) + + assert findall(r'killing it') == [], 'a slow worker was signalled' + + assert findall(r'exited on signal') == [], 'a slow worker was killed' + + assert client.get(url='/')['status'] == 200, 'the restarted app serves' + + +def test_app_start_timeout_reconfigure_collects_orphans(): + """Replacing the application must collect what its deadlines gave up on. + + This is the recovery the bound relies on: a worker wedged for good holds + its "processes" slot for the life of the application, and a + reconfiguration is what ends that life. It only works if the processes + actually die. They are children of the prototype, which is the one + process that knows their pids, and nxt_router_free_app() sends that + prototype a QUIT -- but a QUIT it cannot act on left the prototype waiting + on children that answer nothing, so the whole family outlived unitd. + """ + + max_processes = 2 + + assert 'success' in client.conf(_stuck_on_demand_conf(max_processes)) + + for i in range(max_processes): + assert client.get(url='/')['status'] == 503, f'request {i} failed' + + assert len(_stuck_pids()) == max_processes, 'the workers are there' + assert len(_stuck_protos()) == 1, 'and the prototype that forked them' + + assert _app_processes()['unaccounted'] == max_processes + + assert 'success' in client.conf(_serving_conf()), 'the app is replaced' + + workers, protos = _wait_gone() + + assert workers == [], f'{len(workers)} workers survive the reconfigure' + assert protos == [], f'{len(protos)} prototypes survive the reconfigure' + + assert client.get(url='/')['status'] == 200, 'the new config serves' diff --git a/test/test_asgi_application.py b/test/test_asgi_application.py index 226a1ed75..c7d4a52ac 100644 --- a/test/test_asgi_application.py +++ b/test/test_asgi_application.py @@ -196,6 +196,57 @@ def test_asgi_application_204_transfer_encoding(): ), '204 header transfer encoding' +def get_header_then_body(url): + # The application sleeps after it sends the header. Read for less + # time than the sleep: what arrives in that time came before the body. + # Then read the rest of the response. + sock = client.get(url=url, no_recv=True) + + early = client.recvall(sock, read_timeout=0.5).decode() + rest = client.recvall(sock).decode() + sock.close() + + return early, rest + + +def test_asgi_application_header_before_body(): + client.load('header_then_body') + + resp = client.get(url='/?body=data') + assert resp['status'] == 200, 'start the application' + + # The response has no Content-Length, so it is chunked. The empty + # line that ends the header must come with the header, not with the + # first chunk. + early, rest = get_header_then_body('/?delay=2&body=data') + + assert early.startswith('HTTP/1.1 200 OK\r\n'), 'status line' + assert 'x-header: 1\r\n' in early, 'header field' + assert 'Transfer-Encoding: chunked\r\n' in early, 'chunked' + assert early.endswith('\r\n\r\n'), 'header end before the body' + assert rest == '4\r\ndata\r\n0\r\n\r\n', 'chunks after the sleep' + + # An empty body: the last chunk is the first chunk. + early, rest = get_header_then_body('/?delay=2') + + assert early.endswith('\r\n\r\n'), 'empty body: header end' + assert rest == '0\r\n\r\n', 'empty body: last chunk only' + + # Two responses on one connection. The first chunk of the second + # response has no CRLF before it either. + one = 'Transfer-Encoding: chunked\r\n\r\n4\r\ndata\r\n0\r\n\r\n' + + raw = client.http( + b'GET /?body=data HTTP/1.1\r\nHost: localhost\r\n\r\n' + b'GET /?body=data HTTP/1.1\r\nHost: localhost\r\n' + b'Connection: close\r\n\r\n', + raw=True, + raw_resp=True, + ) + + assert raw.count(one) == 2, 'keep-alive' + + def test_asgi_application_shm_ack_handle(): # Minimum possible limit shm_limit = 10 * 1024 * 1024 @@ -394,7 +445,7 @@ def test_asgi_process_switch(): for _ in range(10): client.get(headers=headers_delay_1, no_recv=True) - client.get(headers=headers_delay_1) + assert client.get(headers=headers_delay_1)['status'] == 200 def test_asgi_application_loading_error(skip_alert): diff --git a/test/test_asgi_lifespan.py b/test/test_asgi_lifespan.py index e09ea1cc4..9f4409ef7 100644 --- a/test/test_asgi_lifespan.py +++ b/test/test_asgi_lifespan.py @@ -34,10 +34,13 @@ def setup_cookies(prefix): for name in ['startup', 'shutdown', 'version']: path = Path(f'{option.test_dir}/python/lifespan/empty/{prefix}{name}') - path.touch(0o777) + path.touch() + # touch() applies the umask; the application runs as "nobody" and + # must be able to write "version". + path.chmod(0o777) -def test_asgi_lifespan(): +def test_asgi_lifespan(requires_restart): client.load('lifespan/empty') setup_cookies('') @@ -49,7 +52,7 @@ def test_asgi_lifespan(): assert_cookies('') -def test_asgi_lifespan_targets(): +def test_asgi_lifespan_targets(requires_restart): path = f'{option.test_dir}/python/lifespan/empty' assert 'success' in client.conf( diff --git a/test/test_asgi_websockets.py b/test/test_asgi_websockets.py index f034195e1..553e864f1 100644 --- a/test/test_asgi_websockets.py +++ b/test/test_asgi_websockets.py @@ -1,3 +1,4 @@ +import socket import struct import time @@ -6,6 +7,7 @@ from unit.applications.lang.python import ApplicationPython from unit.applications.websockets import ApplicationWebsocket +from unit.option import option prerequisites = { 'modules': {'python': lambda v: version.parse(v) >= version.parse('3.5')} @@ -212,13 +214,21 @@ def test_asgi_websockets_large(): ws.frame_write(sock, ws.OP_TEXT, message) + # check_frame() asserts fin and opcode as well as the payload. This + # test used to read a second frame and concatenate it, which is how it + # sat for 30s on every run: frame_read() fills frame['data'] whatever + # the opcode is, the frame that eventually arrived was the zero-length + # keepalive PING (websocket_conf.keepalive_interval, + # src/nxt_router.c:2904), and appending b'' left the comparison true. + # + # The mirror echoes a message as one frame, so one read is right here. + # test_node_websockets_large reads twice on purpose: it loads + # websockets/mirror_fragmentation, whose app.js omits the + # fragmentOutgoingMessages and fragmentationThreshold overrides that + # websockets/mirror sets, so that server really does fragment. frame = ws.frame_read(sock) - data = frame['data'].decode('utf-8') - frame = ws.frame_read(sock) - data += frame['data'].decode('utf-8') - - assert message == data, 'large' + check_frame(frame, True, ws.OP_TEXT, message) sock.close() @@ -293,6 +303,7 @@ def test_asgi_websockets_handshake_connection_absent(): # FAIL 'Sec-WebSocket-Protocol': 'chat', 'Sec-WebSocket-Version': 13, }, + connection_close=False, ) assert resp['status'] == 400, 'status' @@ -424,9 +435,8 @@ def test_asgi_websockets_protocol_absent(): # autobahn-testsuite # -# Some following tests fail because of Unit does not support UTF-8 -# validation for websocket frames. It should be implemented -# by application, if necessary. +# The router validates the payload of a text message and the reason of a +# close frame as UTF-8 and fails the connection with 1007 when it is not. def test_asgi_websockets_1_1_1__1_1_8(): @@ -548,22 +558,53 @@ def test_asgi_websockets_2_10__2_11(): _, sock, _ = ws.upgrade() - for i in range(0, 10): - ws.frame_write(sock, ws.OP_PING, f'payload-{i}') + pings = [f'payload-{i}' for i in range(0, 10)] - for i in range(0, 10): - frame = ws.frame_read(sock) - check_frame(frame, True, ws.OP_PONG, f'payload-{i}') + for payload in pings: + ws.frame_write(sock, ws.OP_PING, payload) + + ws.pongs_read(sock, pings) # 2_11 - for i in range(0, 10): - opcode = ws.OP_PING - ws.frame_write(sock, opcode, f'payload-{i}', chopsize=1) + for payload in pings: + ws.frame_write(sock, ws.OP_PING, payload, chopsize=1) - for i in range(0, 10): - frame = ws.frame_read(sock) - check_frame(frame, True, ws.OP_PONG, f'payload-{i}') + ws.pongs_read(sock, pings) + + close_connection(sock) + + +def test_asgi_websockets_ping_pong_pending(): + client.load('websockets/mirror') + + # A client with a small receive buffer sends many PINGs and does not + # read. While a PONG is not sent, the router must keep only the PONG + # for the most recent PING (RFC 6455 Section 5.5.3). The connection + # must stay open, and the last PING must get its PONG. + + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 4096) + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + sock.connect(('127.0.0.1', 8080)) + + _, sock, _ = ws.upgrade(sock=sock) + + pings = [f'{i:05}'.encode() * 25 for i in range(2000)] + + sock.sendall(b''.join(ws.frame_to_send(ws.OP_PING, p) for p in pings)) + + answered = ws.pongs_read(sock, pings) + + assert answered[0] == 0, 'first ping answered' + assert len(answered) < len(pings), 'pings not answered' + + message = 'still open' + + ws.frame_write(sock, ws.OP_TEXT, message) + + frame = ws.frame_read(sock) + check_frame(frame, True, ws.OP_TEXT, message) close_connection(sock) @@ -1097,28 +1138,35 @@ def test_asgi_websockets_6_1_1__6_4_4(): close_connection(sock) + # 6_3_1 -# Unit does not support UTF-8 validation -# -# # 6_3_1 FAIL -# -# payload_1 = '\xce\xba\xe1\xbd\xb9\xcf\x83\xce\xbc\xce\xb5' -# payload_2 = '\xed\xa0\x80' -# payload_3 = '\x65\x64\x69\x74\x65\x64' -# -# payload = payload_1 + payload_2 + payload_3 -# -# ws.message(sock, ws.OP_TEXT, payload) -# check_close(sock, 1007) -# -# # 6_3_2 FAIL -# -# _, sock, _ = ws.upgrade() -# -# ws.message(sock, ws.OP_TEXT, payload, fragmention_size=1) -# check_close(sock, 1007) -# -# # 6_4_1 ... 6_4_4 FAIL + _, sock, _ = ws.upgrade() + + # Raw bytes: written as str these would be encoded to valid UTF-8. + payload_1 = b'\xce\xba\xe1\xbd\xb9\xcf\x83\xce\xbc\xce\xb5' + payload_2 = b'\xed\xa0\x80' + payload_3 = b'\x65\x64\x69\x74\x65\x64' + + payload = payload_1 + payload_2 + payload_3 + + ws.message(sock, ws.OP_TEXT, payload) + check_close(sock, 1007) + + # 6_3_2 + + _, sock, _ = ws.upgrade() + + ws.message(sock, ws.OP_TEXT, payload, fragmention_size=1) + check_close(sock, 1007) + + # A text message whose last frame ends inside a multi-byte sequence: the + # frame is complete, the message is not valid UTF-8. + + _, sock, _ = ws.upgrade() + + ws.frame_write(sock, ws.OP_TEXT, b'\xce') + + check_close(sock, 1007) def test_asgi_websockets_7_1_1__7_5_1(): @@ -1246,16 +1294,26 @@ def test_asgi_websockets_7_1_1__7_5_1(): ws.frame_write(sock, ws.OP_CLOSE, payload) check_close(sock, 1002) + # 7_5_1 -# # 7_5_1 FAIL Unit does not support UTF-8 validation -# -# _, sock, _ = ws.upgrade() -# -# payload = ws.serialize_close(reason = '\xce\xba\xe1\xbd\xb9\xcf' \ -# '\x83\xce\xbc\xce\xb5\xed\xa0\x80\x65\x64\x69\x74\x65\x64') -# -# ws.frame_write(sock, ws.OP_CLOSE, payload) -# check_close(sock, 1007) + _, sock, _ = ws.upgrade() + + # A raw reason: 0xed 0xa0 0x80 is a surrogate, which has no valid UTF-8 + # encoding. + payload = ws.serialize_close() + b'\xce\xba\xe1\xbd\xb9\xcf' \ + b'\x83\xce\xbc\xce\xb5\xed\xa0\x80\x65\x64\x69\x74\x65\x64' + + ws.frame_write(sock, ws.OP_CLOSE, payload) + check_close(sock, 1007) + + # A close reason that is valid UTF-8 but not ASCII is accepted. + + _, sock, _ = ws.upgrade() + + payload = ws.serialize_close() + b'\xce\xba' + + ws.frame_write(sock, ws.OP_CLOSE, payload) + check_close(sock) def test_asgi_websockets_7_7_X__7_9_X(): @@ -1318,10 +1376,8 @@ def test_asgi_websockets_7_13_1__7_13_2(): check_close(sock, 1002) -def test_asgi_websockets_9_1_1__9_6_6(is_unsafe, system): - if not is_unsafe: - pytest.skip('unsafe, long run') - +@pytest.mark.skipif(not option.unsafe, reason='unsafe, long run') +def test_asgi_websockets_9_1_1__9_6_6(system): client.load('websockets/mirror') assert 'success' in client.conf( @@ -1348,7 +1404,7 @@ def check_payload(opcode, length, chopsize=None): payload = b'*' * length ws.frame_write(sock, opcode, payload, chopsize=chopsize) - frame = ws.frame_read(sock, read_timeout=5) + frame = ws.frame_read(sock, read_timeout=ws.LARGE_MESSAGE_TIMEOUT) check_frame(frame, True, opcode, payload) def check_message(opcode, f_size): @@ -1358,7 +1414,7 @@ def check_message(opcode, f_size): payload = b'*' * 4 * 2**20 ws.message(sock, opcode, payload, fragmention_size=f_size) - frame = ws.frame_read(sock, read_timeout=5) + frame = ws.frame_read(sock, read_timeout=ws.LARGE_MESSAGE_TIMEOUT) check_frame(frame, True, opcode, payload) check_payload(op_text, 64 * 2**10) # 9_1_1 @@ -1428,6 +1484,191 @@ def test_asgi_websockets_10_1_1(): close_connection(sock) +def check_mirror(sock, opcode, payload): + """Read one message the mirror application echoed back. + + A refusal arrives as a CLOSE frame instead, and check_frame() decodes a + TEXT body as UTF-8, so a 2-byte close body would surface as a + UnicodeDecodeError rather than the close code. Name the code. + """ + frame = ws.frame_read(sock) + + assert frame['opcode'] != ws.OP_CLOSE, f"closed with {frame.get('code')}" + + check_frame(frame, True, opcode, payload) + + +def test_asgi_websockets_fragmented_message_over_1mb(): + client.load('websockets/mirror') + + _, sock, _ = ws.upgrade() + + # 2 MiB in 32 fragments of 64 KiB. Every frame is far below the default + # 1 MiB max_frame_size, so the router admits all of them; the module used + # to sum them against a private per-message 1 MiB of its own. + payload = '*' * 2 * 2**20 + + ws.message(sock, ws.OP_TEXT, payload, fragmention_size=64 * 2**10) + + check_mirror(sock, ws.OP_TEXT, payload) + + close_connection(sock) + + +def test_asgi_websockets_message_after_fragmented_message(): + client.load('websockets/mirror') + + _, sock, _ = ws.upgrade() + + # Three non-final 300 KiB fragments and a 100 KiB final one: under 1 MiB + # in total, so this message passed even the old cap. + payload = '*' * (3 * 300 * 2**10 + 100 * 2**10) + + ws.message(sock, ws.OP_TEXT, payload, fragmention_size=300 * 2**10) + + check_mirror(sock, ws.OP_TEXT, payload) + + # The old per-frame counter was never reset when the final fragment was + # delivered straight to a waiting receive(), so 900 KiB stayed on it and + # this 512 KiB frame was charged against 1 MiB - 900 KiB. That depends on + # the message above taking the direct-delivery branch, which the mirror + # application makes near-certain but does not guarantee: if this ever + # reports a 1009 on an unfixed build, rerun it, do not add a sleep. + payload = '*' * 512 * 2**10 + + ws.frame_write(sock, ws.OP_TEXT, payload) + + check_mirror(sock, ws.OP_TEXT, payload) + + close_connection(sock) + + +def test_asgi_websockets_frame_over_buffer_limit(): + client.load('websockets/mirror') + + assert 'success' in client.conf( + { + 'http': { + 'websocket': { + 'max_frame_size': 16777216, + 'keepalive_interval': 0, + } + } + }, + 'settings', + ), 'increase max_frame_size' + + _, sock, _ = ws.upgrade() + + # A single frame the configuration allows but which is larger than the + # module's own 10 MiB accumulation limit. The floor on that limit is what + # admits it; this is the only coverage of that branch outside --unsafe. + payload = '*' * 11 * 2**20 + + ws.frame_write(sock, ws.OP_TEXT, payload) + + check_mirror(sock, ws.OP_TEXT, payload) + + close_connection(sock) + + +def test_asgi_websockets_message_at_buffer_limit(): + client.load('websockets/mirror') + + _, sock, _ = ws.upgrade() + + # A masked frame with an 8-byte length carries 14 bytes of header, so this + # payload makes a frame of exactly the default 1 MiB max_frame_size. Ten + # of them are 10,485,620 bytes, just under the module's 10 MiB. + f_size = 2**20 - 14 + payload = '*' * (10 * f_size) + + ws.message(sock, ws.OP_TEXT, payload, fragmention_size=f_size) + + check_mirror(sock, ws.OP_TEXT, payload) + + close_connection(sock) + + +def test_asgi_websockets_message_over_buffer_limit(): + client.load('websockets/mirror') + + _, sock, _ = ws.upgrade() + + # One frame more than the test above: 11,534,182 bytes, over the 10 MiB. + # A property test, not a regression one -- an unfixed build also answers + # 1009 here, on the second frame rather than the eleventh. + f_size = 2**20 - 14 + payload = '*' * (11 * f_size) + + ws.message(sock, ws.OP_TEXT, payload, fragmention_size=f_size) + + check_close(sock, 1009) # 1009 - CLOSE_TOO_LARGE + + +def test_asgi_websockets_second_frame_over_buffer_limit(): + client.load('websockets/mirror') + + assert 'success' in client.conf( + { + 'http': { + 'websocket': { + 'max_frame_size': 33554432, + 'keepalive_interval': 0, + } + } + }, + 'settings', + ), 'increase max_frame_size' + + _, sock, _ = ws.upgrade() + + # The floor admits one over-limit frame into an empty queue and no more, + # so the connection holds at most one. Non-final, so it is suspended + # whatever the application is doing. Also a property test: an unfixed + # build refuses the 12 MiB frame itself. + ws.frame_write(sock, ws.OP_TEXT, '*' * 12 * 2**20, fin=False) + + # Without this pause the CONT frame can reach the application before the + # 12 MiB frame has been processed; on CI that failed with "Invalid state: + # attempt to process CONT frame". + time.sleep(2) + + ws.frame_write(sock, ws.OP_CONT, '*', fin=True) + + check_close(sock, 1009) # 1009 - CLOSE_TOO_LARGE + + +def test_asgi_websockets_close_after_frame_over_buffer_limit(): + client.load('websockets/mirror') + + assert 'success' in client.conf( + { + 'http': { + 'websocket': { + 'max_frame_size': 33554432, + 'keepalive_interval': 0, + } + } + }, + 'settings', + ), 'increase max_frame_size' + + _, sock, _ = ws.upgrade() + + ws.frame_write(sock, ws.OP_TEXT, '*' * 12 * 2**20, fin=False) + + # The same pause as above: let the 12 MiB frame be processed first. + time.sleep(2) + + # CLOSE is exempt from the accumulation limit: it is at most 125 bytes and + # refusing it would replace the disconnect the application is owed with a + # 1009. Drop the exemption and this returns 1009 instead of 1000. + ws.frame_write(sock, ws.OP_CLOSE, ws.serialize_close()) + + check_close(sock) + + # settings diff --git a/test/test_asgi_websockets_timeout.py b/test/test_asgi_websockets_timeout.py new file mode 100644 index 000000000..a78c8ad4e --- /dev/null +++ b/test/test_asgi_websockets_timeout.py @@ -0,0 +1,48 @@ +"""limits.timeout must not end a quiet, established WebSocket (#422).""" + +from packaging import version + +from unit.applications.lang.python import ApplicationPython +from unit.applications.websockets import ApplicationWebsocket + +prerequisites = { + 'modules': {'python': lambda v: version.parse(v) >= version.parse('3.5')} +} + +client = ApplicationPython(load_module='asgi') +ws = ApplicationWebsocket() + + +def test_asgi_websockets_timeout_quiet_session(skip_alert): + skip_alert(r'socket close\(\d+\) failed') + + assert 'success' in client.conf( + {'http': {'websocket': {'keepalive_interval': 0}}}, 'settings' + ), 'clear keepalive_interval' + + client.load('websockets/mirror', limits={'timeout': 1}) + + resp, sock, _ = ws.upgrade() + assert resp['status'] == 101, 'upgrade' + + # frame_read() would spin on EOF. + sock.settimeout(3) + + try: + early = sock.recv(4096) + except TimeoutError: + early = None + + assert early is None, f'nothing arrives on a quiet session: {early!r}' + + ws.frame_write(sock, ws.OP_TEXT, 'still here') + frame = ws.frame_read(sock, read_timeout=2) + + assert frame['opcode'] == ws.OP_TEXT, 'session survived the deadline' + assert frame['data'].decode('utf-8') == 'still here', 'mirror' + + ws.frame_write(sock, ws.OP_CLOSE, ws.serialize_close()) + frame = ws.frame_read(sock) + assert frame['opcode'] == ws.OP_CLOSE, 'close' + + sock.close() diff --git a/test/test_capget_fallback.py b/test/test_capget_fallback.py new file mode 100644 index 000000000..e4b505e56 --- /dev/null +++ b/test/test_capget_fallback.py @@ -0,0 +1,685 @@ +"""Unit keeps running when a syscall filter denies capget(). + +A hardened deployment -- a systemd "SystemCallFilter=" allowlist, a +hand-written seccomp profile -- can make capget(2) fail for a non-root +unitd. Since 'capability: keep running when capget() is filtered' that +is a warning rather than an outage for the two errnos such a filter +conventionally returns, EPERM and ENOSYS; every other errno still +aborts, because a privilege downgrade caused by a bug in Unit is worse +than a hard failure. + +The same shim also watches capset(2), which is how this file covers +nxt_capability_drop() -- the drop every forked process performs at the +tail of nxt_process_apply_creds(). Its effect cannot be tested here: +the suite runs as an ordinary user with no capabilities, so "the +application has none either" is true with or without the code. What +can be tested, and cannot pass by accident, is *which processes drop*. +Main must not, because it binds listening sockets on every +reconfiguration; the prototype must, because it is the ancestor of +every worker; and the worker itself must not, because it inherits. +A run with real capabilities to lose needs root, and lives in the +freeunit-harness smoke test instead. + +The shim can deny capset(2) as well, which is the deployment a +"SystemCallFilter=" allowlist actually produces -- a profile that omits +capget() rarely keeps capset(). Unit answers that per process: the +prototype refuses to start rather than fork workers that would inherit +what it could not drop, while the router, controller and discovery warn +and carry on, because they run no application code and killing them +would be a respawn loop or a silent death. Both halves are asserted in +test_capset_filtered_refuses_the_application(). + +The refusal is conditional on there being something to refuse over, and +that is the harder half to test here, because a suite running as an +ordinary user holds nothing. So the shim can also make capget() report +a set it does not have -- which is what lets the refusal be reached at +all -- and the opposite case, a denied capset() with genuinely empty +sets, gets its own test: it must serve, through either of the two +readers nxt_capability_still_held() can answer from. + +One branch stays out of reach here, and is named rather than left to be +found: the /proc/self/status reader reporting a *non-empty* set. The +refusal tests get their non-empty answer from the shim's fake capget(), +because the procfs reader reads the kernel and the kernel will not lie +for a suite that holds nothing; faking it would mean interposing open(), +read() and fstatfs() as well, and that fstatfs() check exists precisely +to stop a file being substituted for the real one. What is covered +unprivileged is that reader answering "empty" rather than "cannot tell" +-- the parsing, the streaming and the fstatfs() check, all of which fail +closed into a refusal if they break. Only the final comparison needs a +run with something genuinely held, under a filter that denies capget() +and capset() together, which is the harness smoke test's ground. + +Reproducing a seccomp filter in a test would need privileges the suite +does not have, so the errno is injected instead: src/nxt_capability.c +calls the kernel through the nxt_capget() macro, which expands to +glibc's syscall() wrapper, and that is an ordinary interposable symbol. +test/capget_filter.c preloads a definition of it -- see that file for +what it does and does not touch. + +Inheritance is accepted rather than papered over. Unit's language +modules are fork()ed, never exec()ed, so the application process +inherits the shim already mapped and already armed -- the errno was +read by the shim's constructor in the main process, long before the +fork. Clearing LD_PRELOAD from the application's "environment" would +therefore disarm nothing; it would only hide the fact. + +That inheritance costs nothing, and the test proves it rather than +asserting it: the shim appends the pid of every process whose capget() +it failed to a record file, and the test requires that set to be +exactly the unitd main process. It can be, because +nxt_capability_set() has a single caller, nxt_runtime_conf_init(), +which runs in the main process before anything is forked. For that +record to be evidence and not an empty file, the shim keeps a copy of +the record path rather than the getenv() pointer -- unitd overwrites +the argv+environ block with its process title, so the pointer alone +would leave every child unable to record anything at all. See +capget_filter.c. + +/proc//environ deliberately plays no part in that proof. It +cannot: nxt_process_arguments() takes the contiguous argv+environ +string area for the process title, so every Unit process has an +environ region full of title text and NUL padding while the real +environment lives on the heap. Measured here, the main process shows +the tail of its own argv and the application shows 4807 NUL bytes -- +LD_PRELOAD is invisible in both, in a run where the shim demonstrably +ran in the main process. What the application's /proc//maps +shows instead is the shim object itself, mapped, which is the thing +that would actually matter. +""" + +import errno +import json +import os +import re +import shutil +import signal +import stat +import subprocess +import tempfile +import time +from pathlib import Path + +import pytest + +from unit.http import HTTP1 +from unit.option import option +from unit.utils import public_dir, waitforfiles + +client = HTTP1() + +WARNING = r'\[warn\].+capget\(\) failed.+capabilities are unknown' +ALERT = r'\[alert\].+failed to get process capabilities' + +# nxt_capability_drop() when capset(2) is denied. Deliberately matched on +# the syscall alone and not on the sentence around it: this is the test's +# proof that the injected failure reached Unit at all, and it has to hold on +# both sides of the change it is a control for. Wording that only the fixed +# build emits would make the control fail for the wrong reason. +CAPSET_WARNING = r'\[warn\].+capset\(\) failed' + +# The two lines the refusal writes: one from the prototype that would have +# forked the workers, one from the router that was waiting for it. +REFUSED = r'\[alert\].+refused to start: capset\(\) is denied' +START_FAILED = r"\[error\].+app '.+' start attempt produced no process" + +# nxt_capability_still_held() giving up -- neither capget() nor +# /proc/self/status could say what the process holds. Never expected in +# this suite; asserted absent so that a test which reaches the "serves" +# outcome cannot do so by way of a reader that silently broke. +UNDETERMINED = r'capset\(\) failed.+could not be determined' + +# CAP_NET_BIND_SERVICE. Handed to the shim so capget() reports a +# non-empty set: the suite runs with no capabilities of its own, so +# without this there is genuinely nothing to refuse over. This +# particular bit because nxt_capability_specific_set() ignores it -- +# unitd goes on behaving as the unprivileged process it really is. +HELD = 1 << 10 + +# The two errnos a syscall filter conventionally returns for a denied +# call. nxt_capability.c accepts both, so both belong in the test. +FILTERED = [errno.EPERM, errno.ENOSYS] + + +@pytest.fixture(scope='module', autouse=True) +def requirements(): + if option.system != 'Linux': + pytest.skip('capget() is Linux only') + + if option.is_privileged: + pytest.skip('a root unitd never calls capget()') + + # Everything below reaches unitd through LD_PRELOAD, and everything + # below reasons about capabilities this process can see. A set-id bit + # or a file capability on the binary breaks both at once: glibc treats + # the exec as secure and drops LD_PRELOAD, so the shim never arms, and + # the spawned unitd gets capabilities the pytest process does not have, + # which would make the "nothing is held" premise a statement about the + # wrong process. Neither is true of an ordinary build, but "ordinary" + # is the sort of thing worth checking rather than assuming. + unitd = Path(f'{option.current_dir}/build/sbin/unitd') + + if unitd.is_file(): + if unitd.stat().st_mode & (stat.S_ISUID | stat.S_ISGID): + pytest.skip('unitd is set-id; LD_PRELOAD would be dropped') + + try: + os.getxattr(unitd, 'security.capability') + + except OSError: + pass # The ordinary case: no file capabilities. + + else: + pytest.skip( + 'unitd has file capabilities; LD_PRELOAD would be dropped' + ) + + +@pytest.fixture(scope='module') +def shim(requirements): + """Build test/capget_filter.c into a preloadable object. + + The suite has no convention for compiling C helpers -- nothing under + test/ builds any -- so rather than invent a build system this + compiles at test time with cc(1) and skips when there is none. + + Depends on requirements() rather than trusting fixture ordering: + both are module-scoped, and the link needs -ldl, so on a platform + that should skip the build would otherwise fail first and turn the + skip into an error. + """ + compiler = shutil.which('cc') or shutil.which('gcc') + + if compiler is None: + pytest.skip('requires a C compiler') + + source = f'{option.test_dir}/capget_filter.c' + outdir = tempfile.mkdtemp(prefix='unit-test-capget-') + library = f'{outdir}/capget_filter.so' + + build = subprocess.run( + [compiler, '-shared', '-fPIC', '-O1', '-o', library, source, '-ldl'], + check=False, + capture_output=True, + ) + + assert build.returncode == 0, ( + f'failed to build the capget shim:\n{build.stderr.decode()}' + ) + + yield library + + shutil.rmtree(outdir, ignore_errors=True) + + +class Unitd: + """A unitd of our own, since conftest's shared one has no way to + take an environment.""" + + def __init__(self, shim, errno, capset_errno=0, capget_hold=0): + builddir = f'{option.current_dir}/build' + self.unitd = f'{builddir}/sbin/unitd' + + if not Path(self.unitd).is_file(): + pytest.skip('could not find unitd') + + self.dir = tempfile.mkdtemp(prefix='unit-test-capget-') + public_dir(self.dir) + + Path(f'{self.dir}/state').mkdir() + + self.log = f'{self.dir}/unit.log' + self.stderr = f'{self.dir}/stderr.log' + self.record = f'{self.dir}/capget.pids' + self.capset_record = f'{self.dir}/capset.pids' + self.control = f'{self.dir}/control.unit.sock' + self.listener = f'{self.dir}/app.sock' + self.pidfile = f'{self.dir}/unit.pid' + + env = os.environ.copy() + + # Same UNIT_PYTHONHOME dance as conftest.unit_run(): PYTHONHOME + # must not be set for the pytest interpreter itself. + pythonhome = env.pop('UNIT_PYTHONHOME', None) + if pythonhome: + env['PYTHONHOME'] = pythonhome + + env['LD_PRELOAD'] = shim + env['NXT_TEST_CAPGET_ERRNO'] = str(errno) + env['NXT_TEST_CAPGET_RECORD'] = self.record + env['NXT_TEST_CAPSET_RECORD'] = self.capset_record + + # Only when asked, so the tests that do not deny capset() run + # with exactly the environment they ran with before. + if capset_errno: + env['NXT_TEST_CAPSET_ERRNO'] = str(capset_errno) + + if capget_hold: + env['NXT_TEST_CAPGET_HOLD'] = f'{capget_hold:x}' + + # unitd's stderr is kept apart from unit.log on purpose. The + # first warning is written from nxt_runtime_conf_init(), before + # nxt_runtime_log_files_create() dup2()s the log file over fd 2, + # so it can only land here; unit.log then holds exactly the copy + # nxt_runtime_start() repeats once the file exists. Merging the + # two -- which is what conftest does -- would make it impossible + # to tell which of the two emissions a match came from. + self.stderr_file = open(self.stderr, 'w', encoding='utf-8') + + self.process = subprocess.Popen( + [ + self.unitd, + '--no-daemon', + '--modulesdir', + f'{builddir}/lib/unit/modules', + '--statedir', + f'{self.dir}/state', + '--pid', + self.pidfile, + '--log', + self.log, + '--control', + f'unix:{self.control}', + '--tmpdir', + self.dir, + ], + stdout=subprocess.DEVNULL, + stderr=self.stderr_file, + start_new_session=True, + env=env, + ) + + def stop(self): + if self.process.poll() is None: + try: + os.killpg(self.process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + + self.process.wait(timeout=30) + self.stderr_file.close() + shutil.rmtree(self.dir, ignore_errors=True) + + def read(self, path): + return Path(path).read_text(encoding='utf-8', errors='ignore') + + def filtered_pids(self): + return self.recorded_pids(self.record) + + def capset_pids(self): + return self.recorded_pids(self.capset_record) + + def recorded_pids(self, path): + if not Path(path).exists(): + return set() + + return {int(p) for p in self.read(path).split()} + + +@pytest.fixture() +def unitd_factory(shim): + started = [] + + def factory(errno, capset_errno=0, capget_hold=0): + instance = Unitd(shim, errno, capset_errno, capget_hold) + started.append(instance) + return instance + + yield factory + + for instance in started: + instance.stop() + + +def app_conf(appdir): + """A one-line application that answers with its own pid, so the + /proc lookups below cannot land on the wrong process.""" + modules = option.available['modules'] + + if 'python' in modules: + Path(f'{appdir}/wsgi_pid.py').write_text( + 'import os\n' + '\n' + '\n' + 'def application(environ, start_response):\n' + ' body = str(os.getpid()).encode()\n' + " start_response('200 OK'," + " [('Content-Length', str(len(body)))])\n" + ' return [body]\n', + encoding='utf-8', + ) + + return { + "type": "python", + "processes": {"spare": 0}, + "path": appdir, + "module": "wsgi_pid", + } + + if 'php' in modules: + Path(f'{appdir}/index.php').write_text( + 'start directly + # without going through nxt_process_apply_creds(), so it inherits + # sets the prototype already emptied. If that ever changes this + # assertion is the thing that says so. + assert app_pid not in dropped, 'the worker inherits rather than drops' + + # The application inherited the shim -- fork() carries the mapping, + # armed, whatever the environment says afterwards ... + maps = Path(f'/proc/{app_pid}/maps').read_text(encoding='utf-8') + assert shim in maps, 'shim inherited by the application' + + # ... and still only the main process ever had a capget() filtered, + # which is what makes the inheritance harmless. Two entries: the + # version probe and the call that matters. + assert unitd.filtered_pids() == {main_pid}, 'only unitd asked' + + +def test_capget_einval_still_fatal(unitd_factory): + unitd = unitd_factory(errno.EINVAL) + + unitd.process.wait(timeout=30) + assert unitd.process.returncode != 0, 'unitd exits' + + # nxt_runtime_conf_init() fails before the log file is created, so + # stderr is the only place this can be recorded. + assert not Path(unitd.log).exists(), 'no unit.log' + assert re.search(ALERT, unitd.read(unitd.stderr), re.M), 'alert on stderr' + + +def configure_app(unitd): + """Load a one-process application and return the config response.""" + appdir = f'{unitd.dir}/app' + Path(appdir).mkdir() + public_dir(appdir) + + conf = { + "listeners": {f'unix:{unitd.listener}': {"pass": "applications/app"}}, + "applications": {"app": app_conf(appdir)}, + } + + return client.put( + url='/config', + sock_type='unix', + addr=unitd.control, + body=json.dumps(conf), + ) + + +@pytest.mark.parametrize('filtered_errno', FILTERED, ids=errno.errorcode.get) +def test_capset_filtered_refuses_the_application( + shim, unitd_factory, filtered_errno +): + """A denied capset() with capabilities still held must refuse. + + This is the deployment the whole file is about, taken one step + further: a "SystemCallFilter=" allowlist that omits capget() rarely + keeps capset(), so the fallback that lets unitd start lands in a + prototype that then cannot give up what it holds. Warning and + continuing there would hand unitd's capabilities to application + code -- silently, since the operator asked for neither. + + The refusal is deliberately not made inside nxt_capability_drop(). + That function is also on the router's, the controller's and + discovery's path through nxt_process_core_setup(), and those three + run no application code: router and controller are re-forked + immediately and without backoff, so refusing there would be a + respawn loop, and discovery notifies nobody when it dies. Both + halves are asserted below -- unitd comes up and stays up, and only + the application start fails. + """ + unitd = unitd_factory(0, capset_errno=filtered_errno, capget_hold=HELD) + + assert waitforfiles(unitd.control), ( + f'unitd started; its stderr was:\n{unitd.read(unitd.stderr)}' + ) + + # The injected failure reached Unit, and nxt_capability_still_held() + # found something to refuse over -- without both, the rest of this + # test could pass against a shim that never fired. And the core + # processes met the same denial in nxt_process_core_setup() and + # carried on: the controller is answering on the socket above and the + # router logged its own start line. + assert wait_for(unitd.log, CAPSET_WARNING) is not None, 'capset denied' + assert wait_for(unitd.log, r'router started') is not None, 'router started' + + assert 'success' in configure_app(unitd)['body'], 'configure' + + # The application does not serve. 503 rather than a hang: the + # prototype exits, main's SIGCHLD reaper notifies the router with the + # start's stream still attached, and the router fails the requests + # parked on the application instead of letting them time out. + resp = client.get(sock_type='unix', addr=unitd.listener) + assert resp['status'] == 503, 'application refused' + + # Both ends of that route say so above debug level, which is what an + # operator sees in a default configuration: the prototype names the + # syscall and the consequence, the router names the application. + assert wait_for(unitd.log, REFUSED) is not None, 'prototype refuses' + assert wait_for(unitd.log, START_FAILED) is not None, 'router reports' + + # No respawn loop. Nothing in Unit re-arms a start on its own -- + # every nxt_router_start_app_process() call site is driven by a + # request, a port becoming ready or a configuration change -- so + # after the one attempt above the count must stay where it is. + refusals = count_matches(unitd.log, REFUSED) + assert refusals == 1, 'exactly one attempt' + + time.sleep(2) + assert count_matches(unitd.log, REFUSED) == refusals, 'no retry on its own' + + # unitd itself is unharmed, which is the half that would have been + # lost by refusing inside nxt_capability_drop(). + assert unitd.process.poll() is None, 'unitd still running' + resp = client.get(url='/config', sock_type='unix', addr=unitd.control) + assert resp['status'] == 200, 'controller still answering' + + # The failure is stable rather than latched: a second request makes a + # second attempt, and that attempt fails the same way. Only the 503 + # is asserted -- whether Unit remembers the refusal or repeats it is + # an implementation choice this test does not fix. + resp = client.get(sock_type='unix', addr=unitd.listener) + assert resp['status'] == 503, 'still refused' + + # Main must still never appear among the processes that tried to + # drop, filter or no filter. + main_pid = int(unitd.read(unitd.pidfile).strip()) + tried = unitd.capset_pids() + assert tried, 'somebody tried to drop' + assert main_pid not in tried, 'main never tries' + + +# Which reader nxt_capability_still_held() has to answer from: capget() +# when the filter left it alone, /proc/self/status when it did not. +READERS = {'capget': 0, 'proc': errno.EPERM} + + +@pytest.mark.parametrize('reader', READERS) +def test_capset_filtered_serves_when_nothing_is_held( + shim, unitd_factory, reader +): + """A denied capset() with nothing to drop must not refuse anything. + + The refusal above is conditional on actually holding something, and + that condition is not decoration. Both of the commonest deployments + reach nxt_capability_drop() with every set already empty: a root + unitd, because switching away from uid 0 clears them, and an + ordinary unprivileged unitd, because it was never granted anything. + A filter that denies capset() in either of those denies a call that + had nothing left to do, and refusing over it would take every + application down for no security benefit whatsoever. + + Both readers are exercised, because they cover different filters and + a broken one fails the same way in each: silently, by returning "I + cannot tell", which is treated as a refusal. That is why the + absence of the "could not be determined" warning is asserted rather + than assumed -- without it this test would still pass with both + readers removed, on the strength of a refusal that never happened. + """ + # The premise, stated rather than inherited from whatever the host + # happens to be. unitd is about to be forked from this process, so + # "nothing is held" is a property of this process, and the reader is + # going to read it back through whichever of the two legs is under + # test. Without this the test says only "the host agreed with us + # today": on a runner with empty sets it passes for the right reason, + # and on a host that does hold something it would fail with a 503 and + # no clue as to why. + caps = caps_of() + held = { + name: caps[name] + for name in ('CapPrm', 'CapEff', 'CapAmb') + if caps[name] != 0 + } + + if held: + pytest.skip(f'the test process holds capabilities: {held}') + + # CapInh is deliberately not in that list, and this is the test that + # says so. An inheritable bit confers nothing without an execve() of + # a file carrying a matching one, so nxt_capability_still_held() does + # not count it -- and a desktop with pam_cap or a container runtime in + # the picture commonly has one, which is exactly the host where + # counting it would refuse every application for nothing. This run is + # only evidence of that when the bit is actually set, so record which + # kind of host produced the result. + print(f'\nCapInh of the test process: {caps["CapInh"]:016x}') + + unitd = unitd_factory(READERS[reader], capset_errno=errno.EPERM) + + assert waitforfiles(unitd.control), ( + f'unitd started; its stderr was:\n{unitd.read(unitd.stderr)}' + ) + assert wait_for(unitd.log, r'router started') is not None, 'router started' + + assert 'success' in configure_app(unitd)['body'], 'configure' + + resp = client.get(sock_type='unix', addr=unitd.listener) + assert resp['status'] == 200, 'application serves' + + log = unitd.read(unitd.log) + assert re.search(UNDETERMINED, log, re.M) is None, 'the reader answered' + assert re.search(REFUSED, log, re.M) is None, 'nothing was refused' + + # And the drop still happened where it always does, so the reader is + # not being consulted in place of the capset() itself. + proto_pid = int( + re.search( + r'^PPid:\s+(\d+)', + Path(f'/proc/{int(resp["body"].strip())}/status').read_text( + encoding='utf-8' + ), + re.M, + ).group(1) + ) + assert proto_pid in unitd.capset_pids(), 'the prototype still tried' diff --git a/test/test_chunked.py b/test/test_chunked.py index ef449fab3..894a9808f 100644 --- a/test/test_chunked.py +++ b/test/test_chunked.py @@ -56,6 +56,7 @@ def test_chunked_pipeline(): 'Host': 'localhost', 'Transfer-Encoding': 'chunked', }, + connection_close=False, body='1\r\n$\r\n0\r\n\r\n', ) @@ -96,6 +97,117 @@ def test_chunked_max_body_size(): ) +def test_chunked_max_body_size_later_read(): + # The body crosses the limit in a read after the header read. That + # check closed the connection with no status line. + # https://github.com/freeunitorg/freeunit/issues/542 + assert 'success' in client.conf( + {'max_body_size': 8, 'chunked_transform': True}, 'settings/http' + ) + + def send(writes, connection_close): + head = ( + b'POST / HTTP/1.1\r\nHost: localhost\r\n' + b'Transfer-Encoding: chunked\r\n' + ) + + if connection_close: + head += b'Connection: close\r\n' + + sock = client.http(head + b'\r\n', raw=True, no_recv=True) + + # Each write ends with the CRLF after the chunk data, and goes in + # its own read. + for w in writes: + time.sleep(0.3) + sock.sendall(w) + + resp = client.recvall(sock, read_timeout=10) + sock.close() + + return resp + + # The request does not ask to close, so "Connection: close" in the + # response shows that keep-alive was turned off. The rest of the body + # is never read. + resp = send([b'5\r\nhello\r\n', b'6\r\nworld!\r\n'], False) + assert resp.startswith(b'HTTP/1.1 413 '), resp + assert b'\r\nConnection: close\r\n' in resp, resp + + # Control: exactly 8 bytes is not over the limit. + resp = send([b'5\r\nhello\r\n', b'3\r\nabc\r\n', b'0\r\n\r\n'], True) + assert resp.startswith(b'HTTP/1.1 200 '), resp + assert resp.endswith(b'\r\n\r\nhelloabc'), resp + + +def test_chunked_malformed_later_read(): + # A malformed chunk in a read after the header read got 400, but the + # connection stayed open for a next request. + sock = client.http( + b'POST / HTTP/1.1\r\nHost: localhost\r\n' + b'Transfer-Encoding: chunked\r\n\r\n', + raw=True, + no_recv=True, + ) + + # "X" stands where the CRLF after the chunk data must be. + time.sleep(0.3) + sock.sendall(b'5\r\nhelloXXXXXXXX') + + start = time.monotonic() + resp = client.recvall(sock, read_timeout=5) + elapsed = time.monotonic() - start + sock.close() + + assert resp.startswith(b'HTTP/1.1 400 '), resp + assert b'\r\nConnection: close\r\n' in resp, resp + + # The server closes the connection. recvall() returns at the end of + # the stream, not at its read timeout. + assert elapsed < 3, elapsed + + +def test_chunked_read_ends_after_chunk_data(): + # A read that ended right after the chunk data, before its CRLF, left + # that data in the read buffer. The next read was parsed after it, so + # the parser found "h" where the CRLF must be, and the request got 400. + # https://github.com/freeunitorg/freeunit/issues/543 + head = ( + b'POST / HTTP/1.1\r\nHost: localhost\r\n' + b'Transfer-Encoding: chunked\r\nConnection: close\r\n\r\n' + ) + + def send(writes, expect_body): + sock = client.http(writes[0], raw=True, no_recv=True) + + # The pause puts each write in its own read. + for w in writes[1:]: + time.sleep(0.3) + sock.sendall(w) + + resp = client.recvall(sock, read_timeout=10) + sock.close() + + assert resp.startswith(b'HTTP/1.1 200 '), (writes, resp) + assert resp.endswith(b'\r\n\r\n' + expect_body), (writes, resp) + + # The read after the header read ends after the chunk data. + send([head, b'5\r\nhello', b'\r\n0\r\n\r\n'], b'hello') + + # The same, with a second chunk after the CRLF. + send( + [head, b'5\r\nhello', b'\r\n6\r\nworld!', b'\r\n0\r\n\r\n'], + b'helloworld!', + ) + + # The header read itself ends after the chunk data. + send([head + b'5\r\nhello', b'\r\n0\r\n\r\n'], b'hello') + + # Controls: the read ends inside the data, or after its CRLF. + send([head, b'5\r\nhel', b'lo\r\n0\r\n\r\n'], b'hello') + send([head, b'5\r\nhello\r\n', b'0\r\n\r\n'], b'hello') + + def test_chunked_after_last(): resp = client.get( headers={ diff --git a/test/test_conf_compressors_object.py b/test/test_conf_compressors_object.py new file mode 100644 index 000000000..c4677a51f --- /dev/null +++ b/test/test_conf_compressors_object.py @@ -0,0 +1,27 @@ +"""#431: the object form of "compressors" crashed the router's validator.""" + +import pytest + +from unit.control import Control + +client = Control() + + +@pytest.mark.parametrize( + 'compressor, result', + [ + ({"encoding": "identity", "min_length": 10}, 'success'), + ({"encoding": "not-a-real-encoding"}, 'error'), + ], +) +def test_compressors_object_form(compressor, result): + base = { + "listeners": {"*:8080": {"pass": "routes"}}, + "routes": [{"action": {"return": 200}}], + } + compression = {"types": ["text/css"], "compressors": compressor} + + assert result in client.conf( + {"settings": {"http": {"compression": compression}}, **base} + ) + assert 'success' in client.conf(base), 'the router is still alive' diff --git a/test/test_config_validation_errors.py b/test/test_config_validation_errors.py new file mode 100644 index 000000000..56651486f --- /dev/null +++ b/test/test_config_validation_errors.py @@ -0,0 +1,212 @@ +"""Tests for debuggable config validation errors. + +Covers the additive response fields introduced by roadmap item D5: + - ``location.path`` (RFC 6901 JSON Pointer to the offending member). + - ``suggestion`` (best-match member name when the input was a close typo). + +Legacy response fields (``error``, ``detail``, ``location.offset/line/column``) +must remain unchanged; several tests guard that. + +All cases are expressed using listeners + routes so they run without any +language SAPI module configured. +""" + +from unit.control import Control + +client = Control() + + +def _put(conf, url='/config'): + return client.conf(conf, url) + + +def test_unknown_top_level_key_has_root_path(): + r = _put({"foo": 1}) + assert 'error' in r + assert 'foo' in r['detail'] + assert r['location']['path'] == '' + + +def test_misspelled_listeners_suggests_listeners(): + r = _put({"listners": {}}) + assert 'error' in r + assert r.get('suggestion') == 'listeners' + + +def test_misspelled_applications_suggests_applications(): + r = _put({"aplications": {}}) + assert 'error' in r + assert r.get('suggestion') == 'applications' + + +def test_nested_unknown_key_has_container_path(): + r = _put( + { + "listeners": { + "*:8080": {"pass": "routes", "unknownkey": 1} + }, + "routes": [{"action": {"return": 200}}], + } + ) + assert 'error' in r + assert r['location']['path'] == '/listeners/*:8080' + + +def test_array_element_path_points_into_routes(): + r = _put( + { + "listeners": {"*:8080": {"pass": "routes"}}, + "routes": [ + {"action": {"return": 200}}, + {"action": {"return": 200}}, + {"action": {"return": 200}, "unknownkey": 1}, + ], + } + ) + assert 'error' in r + assert r['location']['path'].startswith('/routes/2') + + +def test_type_error_carries_path(): + r = _put( + { + "listeners": {"*:8080": {"pass": 123}}, + "routes": [{"action": {"return": 200}}], + } + ) + assert 'error' in r + assert '/listeners' in r['location']['path'] + + +def test_rfc6901_escapes_tilde(): + """Route name ``a~b`` must encode as ``a~0b`` in the JSON Pointer.""" + r = _put( + { + "listeners": {"*:8080": {"pass": "routes/a~b"}}, + "routes": { + "a~b": [{"action": {"return": 200}, "unknownkey": 1}] + }, + } + ) + assert 'error' in r + assert r['location']['path'] == '/routes/a~0b/0' + + +def test_rfc6901_escapes_slash(): + """Route name ``a/b`` must encode as ``a~1b``, not split the pointer.""" + r = _put( + {"routes": {"a/b": [{"action": {"return": 200}, "unknownkey": 1}]}} + ) + assert 'error' in r + assert r['location']['path'] == '/routes/a~1b/0' + + +def test_parse_error_keeps_legacy_location_shape(): + # A malformed document never reaches validation, so it has no pointer to + # report; the location must stay exactly what it was before this change. + r = client.conf('{"listeners"', '/config') + assert 'error' in r + assert 'offset' in r['location'] + assert 'path' not in r['location'] + + +def test_no_suggestion_when_distance_exceeds_threshold(): + r = _put({"zzzzzzz": 1}) + assert 'error' in r + assert 'suggestion' not in r + + +def test_no_suggestion_for_short_unrelated_name(): + # The distance threshold alone admits 2 edits for a two-byte name, which + # is every two-letter member in the schema. A suggestion must also cover + # at most half the shorter name, so "zz" gets none rather than "if". + r = _put({"access_log": {"zz": 1}}) + assert 'error' in r + assert 'zz' in r['detail'] + assert 'suggestion' not in r + + +def test_app_type_errors_point_at_the_type_member(): + # nxt_conf_vldt_app() reaches into the "type" member itself rather than + # descending through the instrumented iterator, so all three of its + # checks have to report against "type" and not against the application + # object that contains it. + r = _put({"applications": {"demo": {"type": 1}}}) + assert 'error' in r + assert r['location']['path'] == '/applications/demo/type' + + r = _put({"applications": {"demo": {}}}) + assert 'error' in r + assert r['location']['path'] == '/applications/demo/type' + + r = _put({"applications": {"demo": {"type": "nosuch"}}}) + assert 'error' in r + assert r['location']['path'] == '/applications/demo/type' + + +def test_upstream_missing_servers_points_at_the_member(): + # Same convention as a missing NXT_CONF_VLDT_REQUIRED member: report + # against the absent member, not its container. + r = _put({"upstreams": {"u1": {}}}) + assert 'error' in r + assert r['location']['path'] == '/upstreams/u1/servers' + + +def _app(processes): + return { + "applications": { + "d": { + "type": "external", + "executable": "x", + "processes": processes, + } + } + } + + +def test_post_traversal_checks_point_at_the_member(): + # Semantic checks that run after nxt_conf_vldt_object() has finished the + # traversal see a path with the member's segment already popped, so they + # have to re-attribute explicitly or they name the container instead. + r = _put({"access_log": {"path": ""}}) + assert 'error' in r + assert r['location']['path'] == '/access_log/path' + + r = _put({"access_log": {"path": "/tmp/a", "format": "`{"}}) + assert 'error' in r + assert r['location']['path'] == '/access_log/format' + + for processes, member in [ + ({"spare": -1}, 'spare'), + ({"max": 0}, 'max'), + ({"spare": 5, "max": 2}, 'spare'), + ({"idle_timeout": -1}, 'idle_timeout'), + ]: + r = _put(_app(processes)) + assert 'error' in r, processes + assert ( + r['location']['path'] == f'/applications/d/processes/{member}' + ), processes + + +def test_backward_compat_success_has_no_new_fields(): + r = _put( + { + "listeners": {"*:8080": {"pass": "routes"}}, + "routes": [{"action": {"return": 200}}], + "applications": {}, + } + ) + assert 'success' in r + assert 'error' not in r + assert 'suggestion' not in r + + +def test_backward_compat_error_shape_preserved(): + r = _put({"foo": 1}) + assert 'error' in r + assert 'detail' in r + # 'success' key must never coexist with an error. + assert 'success' not in r + # Legacy message wording preserved verbatim. + assert r['detail'] == 'Unknown parameter "foo".' diff --git a/test/test_configuration.py b/test/test_configuration.py index 477cbeca7..b48ef9bb8 100644 --- a/test/test_configuration.py +++ b/test/test_configuration.py @@ -1,8 +1,11 @@ +import re import socket +import time import pytest from unit.control import Control +from unit.log import Log prerequisites = {'modules': {'python': 'any'}} @@ -52,6 +55,62 @@ def test_json_unicode(): }, 'unicode get' +def test_json_utf8_invalid_value(): + # A configuration string is JSON text and JSON text is UTF-8 (RFC 8259 + # Sect. 8.1). Bytes that begin no valid sequence used to be stored and + # echoed back, which made GET /config undecodable. + resp = client.conf( + b'{"listeners": {"*:8080": {"pass": "routes"}},' + b' "routes": [{"match": {"headers": {"X-T": "caf\xff\xfe"}},' + b' "action": {"return": 200}}]}' + ) + + assert 'error' in resp, 'invalid utf-8 value rejected' + assert 'UTF-8' in resp['detail'], 'reason given' + assert ( + resp['location']['path'] == '/routes/0/match/headers/X-T' + ), 'pointer names the value' + + +def test_json_utf8_invalid_member_name(): + resp = client.conf( + b'{"listeners": {"*:8080": {"pass": "routes"}},' + b' "routes": [{"match": {"headers": {"X-\xff": "v"}},' + b' "action": {"return": 200}}]}' + ) + + assert 'error' in resp, 'invalid utf-8 member name rejected' + assert 'member name' in resp['detail'], 'name named as the culprit' + + # The pointer is echoed in this very response, so it must point at the + # object holding the bad name rather than embed the name itself -- + # otherwise the error report is unreadable for the reason it reports. + assert ( + resp['location']['path'] == '/routes/0/match/headers' + ), 'pointer stops at the parent' + + +def test_json_utf8_valid(): + # Multi-byte UTF-8 is not affected: it must round-trip byte for byte. + value = 'caf\u00e9-\U0001f600-\u043d' + + assert 'success' in client.conf( + { + "listeners": {"*:8080": {"pass": "routes"}}, + "routes": [ + { + "match": {"headers": {"X-T": value}}, + "action": {"return": 200}, + } + ], + } + ), 'valid utf-8 accepted' + + assert ( + client.conf_get('routes/0/match/headers/X-T') == value + ), 'valid utf-8 round-trips unchanged' + + def test_json_unicode_2(): assert 'success' in client.conf( { @@ -160,7 +219,6 @@ def test_applications_string(): assert 'error' in client.conf('"{}"', 'applications'), 'string' -@pytest.mark.skip('not yet, unsafe') def test_applications_type_only(): assert 'error' in client.conf( {"app": {"type": "python"}}, 'applications' @@ -294,7 +352,10 @@ def conf_log(value): ), 'path valid' -@pytest.mark.skip('not yet, unsafe') +@pytest.mark.xfail( + reason='validation accepts an empty listener; the router then fails', + strict=False, +) def test_listeners_empty(): assert 'error' in client.conf({"*:8080": {}}, 'listeners'), 'listener empty' @@ -306,13 +367,10 @@ def test_listeners_no_app(): def test_listeners_unix_abstract(system): - if system != 'Linux': - assert 'error' in try_addr("unix:@sock"), 'abstract at' - - pytest.skip('not yet') + if system == 'Linux': + pytest.skip('not yet') - assert 'error' in try_addr("unix:\0soc"), 'abstract \0' - assert 'error' in try_addr("unix:\u0000soc"), 'abstract \0 unicode' + assert 'error' in try_addr("unix:@sock"), 'abstract at' def test_listeners_addr(): @@ -333,7 +391,6 @@ def test_listeners_addr_error_2(skip_alert): def test_listeners_port_release(): for _ in range(10): - fail = False with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) @@ -346,17 +403,114 @@ def test_listeners_port_release(): resp = client.conf({"listeners": {}, "applications": {}}) - try: - s.bind(('127.0.0.1', 8080)) - s.listen() + # The router closes the listening descriptor before it + # acknowledges the removal, but close(2) only drops the closing + # thread's reference to the open file. Every other router + # thread that had the descriptor armed for accept(2) may still + # hold a transient kernel reference to it, and the address + # leaves the bind hash only when the last one is dropped, which + # happens asynchronously. A client therefore has to retry, so + # the test retries too; a listener that is genuinely never + # closed still fails here. + deadline = time.monotonic() + 5 + + while True: + try: + s.bind(('127.0.0.1', 8080)) + s.listen() + break + + except OSError: + if time.monotonic() >= deadline: + pytest.fail('cannot bind or listen to the address') + + time.sleep(0.01) - except OSError: - fail = True + assert 'success' in resp, 'port release' - if fail: - pytest.fail('cannot bind or listen to the address') - assert 'success' in resp, 'port release' +def test_listeners_close_before_reply(findall): + assert 'success' in client.conf( + { + "listeners": {"127.0.0.1:8080": {"pass": "routes"}}, + "routes": [], + } + ), 'listener added' + + # The markers below are all nxt_debug(), so a release build cannot + # observe this at all. There is no build flag in option.available, + # so ask the log: skipping is the only honest outcome here, passing + # would be a test that cannot fail. + if not findall(r'\[debug\]'): + pytest.skip('needs a build configured with --debug') + + # Remember where the log ends before the removal. The tail already + # holds a "controller conn write" for the request above, and the + # ordering is read off positions in a single read of what follows: + # two independent waits could each succeed with the order between + # them wrong. + pos = len(Log.read()) + + assert 'success' in client.conf( + {"listeners": {}, "applications": {}} + ), 'listener removed' + + # The reply is the barrier that makes the tail complete rather than + # something the assertions below compare against: it is written only + # after every engine has acknowledged, and every engine writes its + # own records before it acknowledges. + deadline = time.monotonic() + 15 + + while True: + tail = Log.read()[pos:] + + if 'controller conn write complete' in tail: + break + + if time.monotonic() >= deadline: + pytest.fail('the controller did not reply') + + time.sleep(0.01) + + # One record per engine, so with listen_threads > 1 there are + # several of each. Comparing the last of one with the last of the + # other is what makes the assertion independent of how the engines + # interleave: an engine acknowledges after it has finished closing, + # so the latest acknowledgement is later than every close; an engine + # that acknowledged first would put the latest close after every + # acknowledgement instead. + finish = list( + re.finditer( + r'\[debug\] (\d+)#\d+ .*listen socket close finish: (\d+)', + tail, + ) + ) + ack = list(re.finditer(r'listen socket close acknowledged', tail)) + + assert finish, 'listener closed' + assert ack, 'removal acknowledged' + + assert finish[-1].start() < ack[-1].start(), 'phase 2 before ack' + + # The one that has to hold: "close finish" is written on entry to + # nxt_router_listen_socket_close_finish(), before the descriptor is + # released, so the check above cannot see an acknowledgement moved + # back above nxt_router_listen_socket_release() inside that function + # -- the position #276 moved it out of. The close(2) of the + # descriptor is written by the same engine that then acknowledges, + # so anchoring on it covers both positions. + # + # Match the router's pid: the controller writes "socket close(N)" + # for its own connection after every reply, and fd numbers collide + # across processes. + pid, fd = finish[-1].group(1), finish[-1].group(2) + closed = re.search( + rf'\[debug\] {pid}#\d+ (\*\d+ )?socket close\({fd}\)', tail + ) + + assert closed is not None, 'descriptor closed' + + assert closed.start() < ack[-1].start(), 'closed before ack' def test_json_application_name_large(): diff --git a/test/test_conn_recycle.py b/test/test_conn_recycle.py new file mode 100644 index 000000000..5ab9368ac --- /dev/null +++ b/test/test_conn_recycle.py @@ -0,0 +1,118 @@ +import socket +import time +from pathlib import Path + +import pytest + +from unit.applications.proto import ApplicationProto +from unit.status import Status + +client = ApplicationProto() + +BODY = '0123456789' + + +@pytest.fixture(autouse=True) +def setup_method_fixture(temp_dir): + Path(f'{temp_dir}/assets').mkdir(parents=True) + Path(f'{temp_dir}/assets/index.html').write_text(BODY, encoding='utf-8') + + assert 'success' in client.conf( + { + "listeners": {"*:8080": {"pass": "routes"}}, + "routes": [{"action": {"share": f'{temp_dir}/assets$uri'}}], + "settings": {"listen_threads": 4}, + } + ) + + yield + + # The no-restart suite preserves /settings between tests. + assert 'success' in client.conf_delete('settings/listen_threads') + + +def _churn(n): + # Each short-lived connection is a nxt_conn_create (freelist pop) + + # nxt_conn_free (freelist push) cycle; spread across the listen_threads + # engines this fills each engine's connection freelist. A corrupted + # recycle returns a wrong/truncated body or, under the sanitizer, crashes + # the worker. + for i in range(n): + sock = socket.create_connection(('127.0.0.1', 8080)) + sock.settimeout(5) + try: + sock.sendall( + b'GET / HTTP/1.1\r\nHost: localhost\r\n' + b'Connection: close\r\n\r\n' + ) + data = b'' + while BODY.encode() not in data: + chunk = sock.recv(4096) + if not chunk: + break + data += chunk + assert BODY.encode() in data, f'body on connection {i}' + finally: + sock.close() + + +def _wait_conns(expected, timeout=100): + # A client-observed EOF is not a server-side barrier. nxt_conn_close() + # enqueues nxt_conn_shutdown_handler(), which performs the SHUT_RDWR the + # client sees as EOF and only then enqueues nxt_conn_close_handler() on + # the engine's close_work_queue; nxt_conn_untrack(), which moves a + # connection out of the active/idle queues and into closed_conns_cnt, + # runs in that later handler (src/nxt_conn_close.c). So the last + # connection of a churn can still be counted active after the client has + # read the whole body and closed -- reading through to EOF does not fix + # this, it only narrows the window. + # + # Poll rather than sleep: the close queue drains immediately in the + # common case, while a fixed delay would be dead time on every run and + # still too short under the sanitizer. Return the last reading either + # way, so that a real accounting bug fails as the counter mismatch it is + # rather than as an opaque timeout. + for _ in range(timeout): + conns = Status.get('/connections') + + if conns == expected: + break + + time.sleep(0.1) + + return conns + + +def test_conn_recycle_across_thread_churn(): + # Populate the per-engine connection freelists. + _churn(120) + + # Lowering listen_threads destroys worker engines, freeing their populated + # connection freelists (engine->mem_pool teardown); raising it recreates + # them. Repeat so structs are recycled, then torn down with the engine. + for _ in range(2): + assert 'success' in client.conf('1', 'settings/listen_threads') + _churn(60) + assert 'success' in client.conf('4', 'settings/listen_threads') + _churn(60) + + +def test_conn_recycle_connection_accounting(): + # The body-integrity churn above passes identically whether structs are + # recycled, parked forever, or never pushed at all, so it cannot see the + # recycler stop working -- notably pending_connections growing without + # bound because some struct never settles. Pin the counters the recycler + # now sits beside instead: every churned connection must be accounted for + # and none may be left behind once the churn is over. + Status.init() + + _churn(60) + + conns = _wait_conns( + {'accepted': 60, 'active': 0, 'idle': 0, 'closed': 60} + ) + + assert conns['accepted'] == 60, 'all accepted' + assert conns['active'] == 0, 'none left active' + assert conns['idle'] == 0, 'none left idle' + assert conns['closed'] == 60, 'all closed' diff --git a/test/test_control_socket_peer.py b/test/test_control_socket_peer.py new file mode 100644 index 000000000..26001eee9 --- /dev/null +++ b/test/test_control_socket_peer.py @@ -0,0 +1,225 @@ +import grp +import os +import pwd +import shutil +import signal +import socket +import subprocess +import tempfile + +import pytest + +from unit.option import option +from unit.utils import waitforfiles + +prerequisites = {'privileged_user': True} + + +def _peer_ids(): + try: + user = pwd.getpwnam('nobody') + except KeyError: + pytest.skip('requires the "nobody" user') + + for name in ('nogroup', 'nobody'): + try: + return user, grp.getgrnam(name) + except KeyError: + pass + + pytest.skip('requires the "nogroup" or "nobody" group') + + +def _get(sock_path, uid=None, gid=None, groups=None): + """GET / on the control socket from a child with the given credentials. + + Returns the status line, '' when the server closed the connection + without a response, or the connect() error name. + """ + r, w = os.pipe() + pid = os.fork() + + if pid == 0: + os.close(r) + result = b'' + + try: + if groups is not None: + os.setgroups(groups) + if gid is not None: + os.setgid(gid) + if uid is not None: + os.setuid(uid) + + s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + s.settimeout(5) + + try: + s.connect(sock_path) + except OSError as e: + result = f'connect: {e.strerror}'.encode() + else: + # A rejected peer is closed right after accept(), so the + # send can fail with EPIPE or ECONNRESET. That is the same + # result as a close without a response. + try: + s.sendall( + b'GET / HTTP/1.1\r\nHost: localhost\r\n' + b'Connection: close\r\n\r\n' + ) + except OSError: + pass + + data = b'' + + try: + while True: + chunk = s.recv(4096) + if not chunk: + break + data += chunk + except OSError: + pass + + result = data.split(b'\r\n', 1)[0] + + s.close() + + except Exception as e: # pylint: disable=broad-except + result = f'error: {e}'.encode() + + os.write(w, result) + os._exit(0) + + os.close(w) + data = b'' + + while True: + chunk = os.read(r, 4096) + if not chunk: + break + data += chunk + + os.close(r) + os.waitpid(pid, 0) + + return data.decode() + + +@pytest.fixture +def control_unitd(request): + instances = [] + + def run(*args): + tmp = tempfile.mkdtemp(prefix='unit-ctl-test-') + os.chmod(tmp, 0o755) + + sock = f'{tmp}/control.unit.sock' + unitd = f'{option.current_dir}/build/sbin/unitd' + + with open(f'{tmp}/unit.log', 'w', encoding='utf-8') as log: + proc = subprocess.Popen( + [ + unitd, + '--no-daemon', + '--modulesdir', + f'{tmp}/modules', + '--statedir', + f'{tmp}/state', + '--pid', + f'{tmp}/unit.pid', + '--log', + f'{tmp}/unit.log', + '--tmpdir', + tmp, + '--control', + f'unix:{sock}', + *args, + ], + stderr=log, + start_new_session=True, + ) + + instances.append((proc, tmp)) + + assert waitforfiles(sock), 'control socket' + + return sock + + yield run + + for proc, tmp in instances: + proc.send_signal(signal.SIGQUIT) + + try: + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + # unitd has child processes: kill the whole group. + try: + os.killpg(proc.pid, signal.SIGKILL) + except ProcessLookupError: + pass + + proc.wait() + + # Keep the log of this unitd when the test failed. + rep = getattr(request.node, 'rep_call', None) + + if rep is not None and rep.failed: + try: + with open(f'{tmp}/unit.log', encoding='utf-8') as log: + print(f'--- {tmp}/unit.log ---\n{log.read()}') + except OSError: + pass + + shutil.rmtree(tmp, ignore_errors=True) + + +def test_control_socket_peer_default(control_unitd): + user, _ = _peer_ids() + + # World-writable socket without delegation: the kernel allows the + # connect(), but the peer credential check must still reject it. + sock = control_unitd('--control-mode', '0666') + + assert _get(sock).startswith('HTTP/1.1 200'), 'root' + assert ( + _get(sock, uid=user.pw_uid, gid=user.pw_gid, groups=[]) == '' + ), 'unrelated user rejected' + + +def test_control_socket_peer_user(control_unitd): + user, _ = _peer_ids() + + sock = control_unitd( + '--control-user', user.pw_name, '--control-mode', '0600' + ) + + assert _get(sock).startswith('HTTP/1.1 200'), 'root' + assert _get( + sock, uid=user.pw_uid, gid=user.pw_gid, groups=[] + ).startswith('HTTP/1.1 200'), 'control user' + + +def test_control_socket_peer_group(control_unitd): + user, group = _peer_ids() + + sock = control_unitd( + '--control-group', group.gr_name, '--control-mode', '0660' + ) + + other_gid = 12345 if group.gr_gid != 12345 else 12346 + + assert _get( + sock, uid=user.pw_uid, gid=group.gr_gid, groups=[] + ).startswith('HTTP/1.1 200'), 'primary group' + + # Supplementary groups come from SO_PEERGROUPS (Linux 4.13+). + if os.uname().sysname == 'Linux': + assert _get( + sock, uid=user.pw_uid, gid=other_gid, groups=[group.gr_gid] + ).startswith('HTTP/1.1 200'), 'supplementary group' + + assert _get( + sock, uid=user.pw_uid, gid=other_gid, groups=[] + ).startswith('connect:'), 'not in group' diff --git a/test/test_http_client_framing.py b/test/test_http_client_framing.py new file mode 100644 index 000000000..ecb4f6282 --- /dev/null +++ b/test/test_http_client_framing.py @@ -0,0 +1,45 @@ +import pytest + +from unit.http import HTTP1 + +# HTTP1._response_complete() tells recvall(framed=True) when a keep-alive +# response is complete. These cases pin its framing rules. They need no +# application. + +HEAD = b'HTTP/1.1 200 OK\r\n' + + +@pytest.mark.parametrize( + 'data, complete', + [ + # Content-Length. + (HEAD + b'Content-Length: 5\r\n\r\nhello', True), + (HEAD + b'Content-Length: 5\r\n\r\nhel', False), + (HEAD + b'Content-Length: 0\r\n\r\n', True), + (HEAD + b'content-length: 3 \r\n\r\nabc', True), + (HEAD + b'Content-Length: x\r\n\r\nabc', False), + # The headers are not complete yet. + (HEAD + b'Content-Length: 0\r\n', False), + # Chunked. + ( + HEAD + + b'Transfer-Encoding: chunked\r\n\r\n5\r\nhello\r\n0\r\n\r\n', + True, + ), + (HEAD + b'Transfer-Encoding: chunked\r\n\r\n5\r\nhello\r\n', False), + (HEAD + b'Transfer-Encoding: chunked\r\n\r\n5\r\nhel', False), + (HEAD + b'Transfer-Encoding: chunked\r\n\r\n0\r\n', False), + (HEAD + b'Transfer-Encoding: chunked\r\n\r\n0\r\n\r\n', True), + # A chunk extension is not part of the size. + ( + HEAD + b'Transfer-Encoding: chunked\r\n\r\n' + b'5;name=value\r\nhello\r\n0\r\n\r\n', + True, + ), + (HEAD + b'Transfer-Encoding: chunked\r\n\r\nzz\r\n', False), + # No framing: the read ends on the timeout. + (HEAD + b'Connection: close\r\n\r\nbody', False), + ], +) +def test_http_client_response_complete(data, complete): + assert HTTP1._response_complete(data) is complete diff --git a/test/test_http_expect.py b/test/test_http_expect.py new file mode 100644 index 000000000..ab03965c3 --- /dev/null +++ b/test/test_http_expect.py @@ -0,0 +1,400 @@ +"""Expect: 100-continue (RFC 9110, 10.1.1). + +The router sends "100 Continue" when it is about to wait for more of the body. +A part of the body that came with the header does not stop the 100, as in +nginx. A request that the router answers from its header alone gets its final +status with no 100. The tests use raw sockets, because the client must wait +for the 100 before it sends the body. +""" + +import socket +import ssl +import time + +import pytest + +from unit.applications.lang.python import ApplicationPython +from unit.applications.tls import ApplicationTLS +from unit.option import option + +prerequisites = {'modules': {'python': 'any'}} + +client = ApplicationPython() + +CONTINUE = b'HTTP/1.1 100 Continue\r\n\r\n' + +# The longest wait for a 100. The wait ends when the 100 arrives, so a long +# value costs nothing. It covers a slow sanitizer build. +CONTINUE_WAIT = 2 + +# A test that expects nothing to arrive waits this long. +SILENCE_WAIT = 0.5 + + +def app(name): + python_dir = f'{option.test_dir}/python' + + return { + "type": client.get_application_type(), + "processes": {"spare": 0}, + "path": f'{python_dir}/{name}', + "working_directory": f'{python_dir}/{name}', + "module": "wsgi", + } + + +@pytest.fixture(autouse=True) +def setup_method_fixture(): + assert 'success' in client.conf( + { + "listeners": { + "*:8080": {"pass": "applications/mirror"}, + "*:8081": {"pass": "routes"}, + "*:8082": {"pass": "applications/header_fields"}, + }, + "routes": [{"action": {"proxy": "http://127.0.0.1:8082"}}], + "applications": { + "mirror": app('mirror'), + "header_fields": app('header_fields'), + }, + "settings": {"http": {"chunked_transform": True}}, + } + ) + + +def connect(port=8080, wrap=None): + sock = socket.create_connection(('127.0.0.1', port)) + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + + if wrap is not None: + sock = wrap(sock) + + return sock + + +def head(expect='100-continue', length=5, version='1.1', extra=''): + """A POST header section with no body.""" + + request = f'POST / HTTP/{version}\r\nHost: localhost\r\n' + + if expect is not None: + request += f'Expect: {expect}\r\n' + + if length is not None: + request += f'Content-Length: {length}\r\n' + + return (request + extra + '\r\n').encode() + + +def recv_for(sock, timeout, size=None): + """What arrives within timeout seconds, up to size bytes or EOF.""" + + data = b'' + end = time.monotonic() + timeout + + while size is None or len(data) < size: + left = end - time.monotonic() + if left <= 0: + break + + sock.settimeout(left) + + try: + part = sock.recv(4096) + except socket.timeout: + break + + if not part: + break + + data += part + + return data + + +def recv_continue(sock): + return recv_for(sock, CONTINUE_WAIT, len(CONTINUE)) + + +def recv_response(sock): + """One response framed by Content-Length: status line, fields, body.""" + + data = b'' + sock.settimeout(10) + + while b'\r\n\r\n' not in data: + part = sock.recv(4096) + assert part, f'closed before the header: {data!r}' + data += part + + header, _, body = data.partition(b'\r\n\r\n') + lines = header.decode().split('\r\n') + + fields = {} + for line in lines[1:]: + name, _, value = line.partition(':') + fields[name.strip().lower()] = value.strip() + + length = int(fields.get('content-length', 0)) + + while len(body) < length: + part = sock.recv(4096) + assert part, f'closed in the body: {body!r}' + body += part + + return lines[0], fields, body + + +def recv_all(sock): + return recv_for(sock, 10) + + +@pytest.mark.parametrize('expect', ['100-continue', '100-Continue']) +def test_expect_continue(expect): + with connect() as sock: + sock.sendall(head(expect, extra='Connection: close\r\n')) + + assert recv_continue(sock) == CONTINUE, 'no 100 within CONTINUE_WAIT' + + sock.sendall(b'hello') + + status, _, body = recv_response(sock) + assert status.startswith('HTTP/1.1 200'), status + assert body == b'hello' + + +def test_expect_continue_chunked(): + with connect() as sock: + sock.sendall( + head( + length=None, + extra='Transfer-Encoding: chunked\r\nConnection: close\r\n', + ) + ) + + assert recv_continue(sock) == CONTINUE, 'no 100 within CONTINUE_WAIT' + + sock.sendall(b'5\r\nhello\r\n0\r\n\r\n') + + status, _, body = recv_response(sock) + assert status.startswith('HTTP/1.1 200'), status + assert body == b'hello' + + +def test_expect_continue_keepalive(): + """Each request on a connection gets its own 100. The expectation of one + request does not carry over to the next one.""" + + with connect() as sock: + + for body in (b'hello', b'second'): + sock.sendall(head(length=len(body))) + + assert recv_continue(sock) == CONTINUE, f'no 100 before {body!r}' + + sock.sendall(body) + + status, _, got = recv_response(sock) + assert status.startswith('HTTP/1.1 200'), status + assert got == body + + sock.sendall(head(expect=None, extra='Connection: close\r\n')) + + assert recv_for(sock, SILENCE_WAIT) == b'', '100 without Expect' + + sock.sendall(b'third') + + data = recv_all(sock) + assert data.startswith(b'HTTP/1.1 200'), data + assert data.endswith(b'\r\n\r\nthird'), data + + +def test_expect_continue_body_bytes_sent(wait_for_record): + """$body_bytes_sent does not count the 100.""" + + assert 'success' in client.conf( + { + 'path': f'{option.temp_dir}/access.log', + 'format': '$uri $status $body_bytes_sent', + }, + 'access_log', + ) + + with connect() as sock: + sock.sendall( + b'POST /bbs HTTP/1.1\r\nHost: localhost\r\n' + b'Expect: 100-continue\r\nContent-Length: 5\r\n' + b'Connection: close\r\n\r\n' + ) + + assert recv_continue(sock) == CONTINUE, 'no 100 within CONTINUE_WAIT' + + sock.sendall(b'hello') + recv_all(sock) + + found = wait_for_record(r'^/bbs 200 (\d+)$', 'access.log') + assert found is not None, 'no access log record' + assert found.group(1) == '5', '$body_bytes_sent' + + +def test_expect_continue_proxy(): + """The router meets the expectation, so a proxied upstream does not get + the field. An application gets it.""" + + for port, seen in ((8082, True), (8081, False)): + with connect(port) as sock: + sock.sendall(head(extra='Connection: close\r\n')) + + assert recv_continue(sock) == CONTINUE, f'no 100 on {port}' + + sock.sendall(b'hello') + + status, fields, _ = recv_response(sock) + assert status.startswith('HTTP/1.1 200'), status + + names = fields['all-headers'].split(',') + assert ('HTTP_EXPECT' in names) == seen, f'{port}: {names}' + + +def test_expect_continue_tls(): + if not option.available['modules'].get('openssl'): + pytest.skip('requires openssl') + + tls = ApplicationTLS() + tls.certificate() + + assert 'success' in client.conf( + { + "pass": "applications/mirror", + "tls": {"certificate": "default"}, + }, + 'listeners/*:8080', + ) + + context = ssl.create_default_context() + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + + with connect(wrap=context.wrap_socket) as sock: + sock.sendall(head(extra='Connection: close\r\n')) + + assert recv_continue(sock) == CONTINUE, 'no 100 within CONTINUE_WAIT' + + sock.sendall(b'hello') + + status, _, body = recv_response(sock) + assert status.startswith('HTTP/1.1 200'), status + assert body == b'hello' + + +@pytest.mark.parametrize( + 'request_head', + [ + # RFC 9110: an HTTP/1.0 expectation is ignored. + head(version='1.0', extra='Connection: close\r\n'), + # Only "100-continue" is known; the others are ignored, as nginx + # does. + head(expect='100-continue-please', extra='Connection: close\r\n'), + head(expect='foo', extra='Connection: close\r\n'), + ], + ids=['http10', 'other-value', 'unknown'], +) +def test_expect_continue_ignored(request_head): + with connect() as sock: + sock.sendall(request_head) + + assert recv_for(sock, SILENCE_WAIT) == b'', 'answered before the body' + + sock.sendall(b'hello') + + data = recv_all(sock) + assert data.startswith(b'HTTP/1.1 200'), data + assert CONTINUE not in data, data + + +@pytest.mark.parametrize( + 'request_head, status', + [ + # No body. + (head(length=None, extra='Connection: close\r\n'), 200), + (head(length=0, extra='Connection: close\r\n'), 200), + # Rejected from the header alone. + (head(length=None, extra='Transfer-Encoding: gzip\r\n'), 501), + (head(extra='Transfer-Encoding: chunked\r\n'), 400), + ], + ids=['no-length', 'zero-length', 'te-unsupported', 'te-and-length'], +) +def test_expect_continue_final_status(request_head, status): + """The router answers from the header alone, with no 100.""" + + with connect() as sock: + sock.sendall(request_head) + + data = recv_all(sock) + assert data.startswith(f'HTTP/1.1 {status}'.encode()), data + assert CONTINUE not in data, data + + +def test_expect_continue_max_body_size(): + """A body over max_body_size gets 413 at once, with no 100.""" + + assert 'success' in client.conf('10', 'settings/http/max_body_size') + + with connect() as sock: + sock.sendall(head(length=11)) + + data = recv_all(sock) + assert data.startswith(b'HTTP/1.1 413'), data + assert CONTINUE not in data, data + + +def test_expect_continue_length_required(): + """With chunked_transform off, a chunked body gets 411 at once, with no + 100.""" + + assert 'success' in client.conf('false', 'settings/http/chunked_transform') + + with connect() as sock: + sock.sendall(head(length=None, extra='Transfer-Encoding: chunked\r\n')) + + data = recv_all(sock) + assert data.startswith(b'HTTP/1.1 411'), data + assert CONTINUE not in data, data + + +def test_expect_continue_part_of_body(): + """A part of the body comes with the header. The router sends the 100 + while it waits for the rest, as nginx does.""" + + with connect() as sock: + sock.sendall(head(length=10, extra='Connection: close\r\n') + b'hello') + + assert recv_continue(sock) == CONTINUE, 'no 100 after part of the body' + + sock.sendall(b'world') + + status, _, body = recv_response(sock) + assert status.startswith('HTTP/1.1 200'), status + assert body == b'helloworld' + + +def test_expect_continue_chunk_framing(): + """Only the framing of the first chunk comes with the header. The client + waits for the 100 before it sends the chunk data.""" + + with connect() as sock: + sock.sendall( + head( + length=None, + extra='Transfer-Encoding: chunked\r\nConnection: close\r\n', + ) + + b'5\r\n' + ) + + assert recv_continue(sock) == CONTINUE, 'no 100 after chunk framing' + + sock.sendall(b'hello\r\n0\r\n\r\n') + + status, _, body = recv_response(sock) + assert status.startswith('HTTP/1.1 200'), status + assert body == b'hello' diff --git a/test/test_http_header.py b/test/test_http_header.py index fce5d6f60..286e492b5 100644 --- a/test/test_http_header.py +++ b/test/test_http_header.py @@ -549,3 +549,34 @@ def test_http_header_fields_keepalive_spill(): assert f'HTTP_X_TEST_{i}' in got, f'keep-alive field {i} of {count}' sock.close() + + +def test_http_header_large_buffers_above_255(): + # The parser rewinds only to the start of an incomplete field, so a + # header of many short fields takes one large buffer for every few + # fields. The first buffer has the default size of 2048 bytes and holds + # about 68 fields. With 128-byte large buffers and 30-byte fields, 4 + # fields fit in one buffer, so 1200 fields take about 283 large buffers + # and 1350 fields take about 320. The count of + # buffers was 8 bits wide, so a limit of 256 or more never fired. + client.load('empty') + + assert 'success' in client.conf( + { + 'http': { + 'large_header_buffer_size': 128, + 'large_header_buffers': 300, + } + }, + 'settings', + ) + + def fields(count, expect): + headers = {'Host': 'localhost', 'Connection': 'close'} + for i in range(count): + headers[f'X-{i:04}'] = 'a' * 20 + + assert client.get(headers=headers)['status'] == expect, count + + fields(1200, 200) + fields(1350, 431) diff --git a/test/test_http_no_body_status.py b/test/test_http_no_body_status.py new file mode 100644 index 000000000..6b4d8dad8 --- /dev/null +++ b/test/test_http_no_body_status.py @@ -0,0 +1,289 @@ +"""Responses that RFC 9112 Sect. 6.3 defines as having no message body. + +An application that writes a body on 204/304 -- or on any response to HEAD -- +used to have those bytes forwarded verbatim, with no Content-Length and no +chunked framing, so a downstream parser read them as the start of the next +response. See freeunitorg/freeunit#164. +""" + +import base64 +import os + +import pytest + +from unit.applications.lang.php import ApplicationPHP +from unit.option import option + +prerequisites = {'modules': {'php': 'all'}} + +client = ApplicationPHP() + +# The application writes exactly these 10 bytes on every response. +SMUGGLED = 'SMUGGLEDXX' + +KEEPALIVE = {'Host': 'localhost', 'Connection': 'keep-alive'} +CLOSE = {'Host': 'localhost', 'Connection': 'close'} + + +@pytest.fixture(autouse=True) +def setup_method_fixture(): + client.load('no_body_status') + + +def split_response(raw): + """Split a raw response into (header block, everything after it).""" + + assert '\r\n\r\n' in raw, f'response has no header terminator: {raw!r}' + + head, _, rest = raw.partition('\r\n\r\n') + + return head, rest + + +def request_raw(url, method='GET', sock=None, headers=None): + kwargs = { + 'url': url, + 'headers': KEEPALIVE if headers is None else headers, + 'raw_resp': True, + 'read_timeout': 1, + 'start': True, + } + + if sock is not None: + kwargs['sock'] = sock + + return client.http(method, **kwargs) + + +@pytest.mark.parametrize('status', [204, 304]) +@pytest.mark.parametrize('content_length', [False, True]) +def test_no_body_status_drops_app_body(status, content_length): + url = f'/?status={status}' + if content_length: + url += '&cl=1' + + raw, sock = request_raw(url) + + try: + head, rest = split_response(raw) + + assert f' {status} ' in head.split('\r\n')[0], 'status line' + assert ( + SMUGGLED not in raw + ), f'body forwarded on {status}: {raw!r}' + assert rest == '', f'trailing bytes after header: {rest!r}' + assert ( + 'Transfer-Encoding' not in head + ), 'no chunked framing on a bodyless status' + + if status == 204: + # RFC 9110 Sect. 8.6: no Content-Length on 204. + assert 'Content-Length' not in head, 'no Content-Length on 204' + + # The connection must still be usable: the next request has to parse + # as a complete response of its own, not as a continuation. + resp = client.get(sock=sock, url='/?status=200', headers=CLOSE) + + assert resp['status'] == 200, 'keep-alive: second request status' + assert resp['body'] == SMUGGLED, 'keep-alive: second request body' + + finally: + sock.close() + + +def test_no_body_status_head_drops_app_body(): + raw, sock = request_raw('/?status=200&cl=1', method='HEAD') + + try: + head, rest = split_response(raw) + + assert ' 200 ' in head.split('\r\n')[0], 'status line' + assert SMUGGLED not in raw, f'body forwarded on HEAD: {raw!r}' + assert rest == '', f'trailing bytes after header: {rest!r}' + # A HEAD response keeps the length the equivalent GET would report. + assert 'Content-Length: 10' in head, 'HEAD keeps Content-Length' + + resp = client.get(sock=sock, url='/?status=200', headers=CLOSE) + + assert resp['status'] == 200, 'keep-alive after HEAD: status' + assert resp['body'] == SMUGGLED, 'keep-alive after HEAD: body' + + finally: + sock.close() + + +def test_no_body_status_get_still_has_body(): + """Control: an ordinary status keeps the application's body.""" + + resp = client.get(url='/?status=200') + + assert resp['status'] == 200, 'status' + assert resp['body'] == SMUGGLED, 'body' + + +def test_no_body_status_proxy(): + """The proxy path must not re-introduce the body either.""" + + php_dir = f'{option.test_dir}/php' + + assert 'success' in client.conf( + { + "listeners": { + "*:8080": {"pass": "routes"}, + "*:8081": {"pass": "applications/no_body_status"}, + }, + "routes": [{"action": {"proxy": "http://127.0.0.1:8081"}}], + "applications": { + "no_body_status": { + "type": client.get_application_type(), + "processes": {"spare": 0}, + "root": f'{php_dir}/no_body_status', + "working_directory": f'{php_dir}/no_body_status', + "index": "index.php", + } + }, + } + ), 'proxy configuration' + + raw, sock = request_raw('/?status=204') + + try: + _, rest = split_response(raw) + + assert SMUGGLED not in raw, f'proxy forwarded a 204 body: {raw!r}' + assert rest == '', f'proxy trailing bytes: {rest!r}' + + resp = client.get(sock=sock, url='/?status=200', headers=CLOSE) + + assert resp['status'] == 200, 'proxy keep-alive: status' + assert resp['body'] == SMUGGLED, 'proxy keep-alive: body' + + finally: + sock.close() + + +def test_no_body_status_websocket_upgrade_request(): + """A WebSocket-upgrade request answered with 204 is still bodyless. + + r->websocket_handshake is set while the *request* headers are parsed, long + before the application picks a status, so the upgrade exemption has to test + the status too -- otherwise this request keeps the unframed path. + """ + + key = base64.b64encode(os.urandom(16)).decode() + + raw, sock = request_raw( + '/?status=204', + headers={ + 'Host': 'localhost', + 'Connection': 'Upgrade', + 'Upgrade': 'websocket', + 'Sec-WebSocket-Key': key, + 'Sec-WebSocket-Version': 13, + }, + ) + + try: + head, rest = split_response(raw) + + assert ' 204 ' in head.split('\r\n')[0], 'status line' + assert SMUGGLED not in raw, f'body forwarded on upgrade+204: {raw!r}' + assert rest == '', f'trailing bytes after header: {rest!r}' + + resp = client.get(sock=sock, url='/?status=200', headers=CLOSE) + + assert resp['status'] == 200, 'keep-alive after upgrade+204: status' + assert resp['body'] == SMUGGLED, 'keep-alive after upgrade+204: body' + + finally: + sock.close() + + +def test_no_body_status_app_transfer_encoding(): + """An application-sent Transfer-Encoding is a generic field; drop it.""" + + raw, sock = request_raw('/?status=204&te=1') + + try: + head, rest = split_response(raw) + + assert ' 204 ' in head.split('\r\n')[0], 'status line' + assert ( + 'Transfer-Encoding' not in head + ), f'Transfer-Encoding kept on 204: {head!r}' + assert SMUGGLED not in raw, f'body forwarded on 204: {raw!r}' + assert rest == '', f'trailing bytes after header: {rest!r}' + + resp = client.get(sock=sock, url='/?status=200', headers=CLOSE) + + assert resp['status'] == 200, 'keep-alive: status' + assert resp['body'] == SMUGGLED, 'keep-alive: body' + + finally: + sock.close() + + +def test_no_body_status_response_headers_content_length(): + """Framing headers added by "response_headers" must go too. + + They are added as generic fields, without setting r->resp.content_length, + so skipping that one pointer is not enough. Note the lowercase name: the + config validator rejects "Content-Length" here, but with a case-sensitive + memcmp (nxt_conf_vldt_response_header()), so "content-length" is accepted + and reaches the response. Transfer-Encoding is not guarded at all. + """ + + php_dir = f'{option.test_dir}/php' + + assert 'success' in client.conf( + { + "listeners": {"*:8080": {"pass": "routes"}}, + "routes": [ + { + # Scoped to the 204 so the follow-up 200 on the same + # connection is framed by the application alone. + "match": {"arguments": {"status": "204"}}, + "action": { + "pass": "applications/no_body_status", + "response_headers": { + "content-length": "10", + "Transfer-Encoding": "chunked", + }, + }, + }, + {"action": {"pass": "applications/no_body_status"}}, + ], + "applications": { + "no_body_status": { + "type": client.get_application_type(), + "processes": {"spare": 0}, + "root": f'{php_dir}/no_body_status', + "working_directory": f'{php_dir}/no_body_status', + "index": "index.php", + } + }, + } + ), 'response_headers configuration' + + raw, sock = request_raw('/?status=204') + + try: + head, rest = split_response(raw) + + assert ' 204 ' in head.split('\r\n')[0], 'status line' + assert ( + 'content-length' not in head.lower() + ), f'response_headers Content-Length kept on 204: {head!r}' + assert ( + 'transfer-encoding' not in head.lower() + ), f'response_headers Transfer-Encoding kept on 204: {head!r}' + assert SMUGGLED not in raw, f'body forwarded on 204: {raw!r}' + assert rest == '', f'trailing bytes after header: {rest!r}' + + resp = client.get(sock=sock, url='/?status=200', headers=CLOSE) + + assert resp['status'] == 200, 'keep-alive: status' + assert resp['body'] == SMUGGLED, 'keep-alive: body' + + finally: + sock.close() diff --git a/test/test_java_application.py b/test/test_java_application.py index 28d692af6..fe457fbdd 100644 --- a/test/test_java_application.py +++ b/test/test_java_application.py @@ -838,6 +838,38 @@ def test_java_application_no_method(): assert client.post()['status'] == 405, 'no method' +def test_java_application_server_name(): + client.load('server_name') + + def headers(host): + return client.get(headers={'Host': host, 'Connection': 'close'})[ + 'headers' + ] + + def server_name(host): + return headers(host)['X-Server-Name'] + + assert server_name('localhost') == 'localhost', 'plain host' + assert server_name('localhost:8080') == 'localhost', 'port stripped' + assert server_name('LocalHost') == 'localhost', 'lowercased' + assert server_name('localhost.') == 'localhost', 'trailing dot stripped' + assert server_name('[::1]') == '[::1]', 'ipv6 literal' + assert server_name('[::1]:8080') == '[::1]', 'ipv6 literal with port' + + # the port comes from the listener, never from the Host field + assert headers('[::1]:9999')['X-Server-Port'] == '8080', 'server port' + + # HTTP/1.0 may omit Host. The router substitutes "localhost" rather than + # leaving the name empty, so that is what the servlet sees -- previously + # this module fell back to the listener address instead. Aligning with + # every other module is the point of the change, not a side effect. + resp = client.http( + b'GET / HTTP/1.0\r\nConnection: close\r\n\r\n', raw=True + ) + assert resp['status'] == 200, 'no host' + assert resp['headers']['X-Server-Name'] == 'localhost', 'no host name' + + def test_java_application_get_header(): client.load('get_header') diff --git a/test/test_java_websockets.py b/test/test_java_websockets.py index 94ac6f864..78823471e 100644 --- a/test/test_java_websockets.py +++ b/test/test_java_websockets.py @@ -5,6 +5,7 @@ from unit.applications.lang.java import ApplicationJava from unit.applications.websockets import ApplicationWebsocket +from unit.option import option prerequisites = {'modules': {'java': 'any'}} @@ -121,6 +122,29 @@ def test_java_websockets_fragmentation(): sock.close() +def check_text_one_frame(app, *after): + # A text message larger than the module's 8 KiB encoder buffer is sent + # as one frame. frame_read(), not message_read(), which would join the + # fragments and hide them. + client.load(app) + + _, sock, _ = ws.upgrade() + + payload = '*' * 64 * 2**10 + + ws.frame_write(sock, ws.OP_TEXT, payload) + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, payload) + + for text in after: + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, text) + + close_connection(sock) + + +def test_java_websockets_text_one_frame(): + check_text_one_frame('websockets_mirror') + + def test_java_websockets_frame_fragmentation_invalid(): client.load('websockets_mirror') @@ -217,6 +241,7 @@ def test_java_websockets_handshake_connection_absent(): # FAIL 'Sec-WebSocket-Protocol': 'chat', 'Sec-WebSocket-Version': 13, }, + connection_close=False, ) assert resp['status'] == 400, 'status' @@ -348,9 +373,8 @@ def test_java_websockets_protocol_absent(): # autobahn-testsuite # -# Some following tests fail because of Unit does not support UTF-8 -# validation for websocket frames. It should be implemented -# by application, if necessary. +# The router validates the payload of a text message and the reason of a +# close frame as UTF-8 and fails the connection with 1007 when it is not. def test_java_websockets_1_1_1__1_1_8(): @@ -472,22 +496,19 @@ def test_java_websockets_2_10__2_11(): _, sock, _ = ws.upgrade() - for i in range(0, 10): - ws.frame_write(sock, ws.OP_PING, f'payload-{i}') + pings = [f'payload-{i}' for i in range(0, 10)] - for i in range(0, 10): - frame = ws.frame_read(sock) - check_frame(frame, True, ws.OP_PONG, f'payload-{i}') + for payload in pings: + ws.frame_write(sock, ws.OP_PING, payload) + + ws.pongs_read(sock, pings) # 2_11 - for i in range(0, 10): - opcode = ws.OP_PING - ws.frame_write(sock, opcode, f'payload-{i}', chopsize=1) + for payload in pings: + ws.frame_write(sock, ws.OP_PING, payload, chopsize=1) - for i in range(0, 10): - frame = ws.frame_read(sock) - check_frame(frame, True, ws.OP_PONG, f'payload-{i}') + ws.pongs_read(sock, pings) close_connection(sock) @@ -1021,28 +1042,26 @@ def test_java_websockets_6_1_1__6_4_4(): close_connection(sock) + # 6_3_1 -# Unit does not support UTF-8 validation -# -# # 6_3_1 FAIL -# -# payload_1 = '\xce\xba\xe1\xbd\xb9\xcf\x83\xce\xbc\xce\xb5' -# payload_2 = '\xed\xa0\x80' -# payload_3 = '\x65\x64\x69\x74\x65\x64' -# -# payload = payload_1 + payload_2 + payload_3 -# -# ws.message(sock, ws.OP_TEXT, payload) -# check_close(sock, 1007) -# -# # 6_3_2 FAIL -# -# _, sock, _ = ws.upgrade() -# -# ws.message(sock, ws.OP_TEXT, payload, fragmention_size=1) -# check_close(sock, 1007) -# -# # 6_4_1 ... 6_4_4 FAIL + _, sock, _ = ws.upgrade() + + # Raw bytes: written as str these would be encoded to valid UTF-8. + payload_1 = b'\xce\xba\xe1\xbd\xb9\xcf\x83\xce\xbc\xce\xb5' + payload_2 = b'\xed\xa0\x80' + payload_3 = b'\x65\x64\x69\x74\x65\x64' + + payload = payload_1 + payload_2 + payload_3 + + ws.message(sock, ws.OP_TEXT, payload) + check_close(sock, 1007) + + # 6_3_2 + + _, sock, _ = ws.upgrade() + + ws.message(sock, ws.OP_TEXT, payload, fragmention_size=1) + check_close(sock, 1007) def test_java_websockets_7_1_1__7_5_1(): @@ -1170,16 +1189,17 @@ def test_java_websockets_7_1_1__7_5_1(): ws.frame_write(sock, ws.OP_CLOSE, payload) check_close(sock, 1002) + # 7_5_1 -# # 7_5_1 FAIL Unit does not support UTF-8 validation -# -# _, sock, _ = ws.upgrade() -# -# payload = ws.serialize_close(reason = '\xce\xba\xe1\xbd\xb9\xcf' \ -# '\x83\xce\xbc\xce\xb5\xed\xa0\x80\x65\x64\x69\x74\x65\x64') -# -# ws.frame_write(sock, ws.OP_CLOSE, payload) -# check_close(sock, 1007) + _, sock, _ = ws.upgrade() + + # A raw reason: 0xed 0xa0 0x80 is a surrogate, which has no valid UTF-8 + # encoding. + payload = ws.serialize_close() + b'\xce\xba\xe1\xbd\xb9\xcf' \ + b'\x83\xce\xbc\xce\xb5\xed\xa0\x80\x65\x64\x69\x74\x65\x64' + + ws.frame_write(sock, ws.OP_CLOSE, payload) + check_close(sock, 1007) def test_java_websockets_7_7_X__7_9_X(): @@ -1242,10 +1262,8 @@ def test_java_websockets_7_13_1__7_13_2(): check_close(sock, 1002) -def test_java_websockets_9_1_1__9_6_6(is_unsafe, system): - if not is_unsafe: - pytest.skip('unsafe, long run') - +@pytest.mark.skipif(not option.unsafe, reason='unsafe, long run') +def test_java_websockets_9_1_1__9_6_6(system): client.load('websockets_mirror') assert 'success' in client.conf( @@ -1282,7 +1300,7 @@ def check_message(opcode, f_size): payload = b'*' * 4 * 2**20 ws.message(sock, opcode, payload, fragmention_size=f_size) - frame = ws.frame_read(sock, read_timeout=5) + frame = ws.frame_read(sock, read_timeout=ws.LARGE_MESSAGE_TIMEOUT) check_frame(frame, True, opcode, payload) check_payload(op_text, 64 * 2**10) # 9_1_1 @@ -1413,3 +1431,343 @@ def test_java_websockets_keepalive_interval(): check_frame(frame, True, ws.OP_PING, '') # PING frame sock.close() + + +def test_java_websockets_binary_message_too_big_reason(): + # A fragmented binary message over the session's binary buffer used to + # report the text-message close reason (issue #435); an unfragmented + # binary message never reaches the check, so this needs two frames. + client.load('websockets_binary_toobig') + + _, sock, _ = ws.upgrade() + + ws.frame_write(sock, ws.OP_BINARY, '*' * 600, fin=False) + ws.frame_write(sock, ws.OP_CONT, '*' * 600, fin=True) + + frame = ws.frame_read(sock) + + assert frame['opcode'] == ws.OP_CLOSE, 'close opcode' + assert frame['code'] == 1009, 'close code' + assert 'text message' not in frame['reason'], 'reason names the wrong kind' + assert 'binary' in frame['reason'], 'reason names the binary message' + + sock.close() + + +# async remote (issue #434) + + +def test_java_websockets_async_text(): + # The asynchronous remote used to send nothing and never call the + # SendHandler, so the future from sendText(String) never completed. + client.load('websockets_async') + + _, sock, _ = ws.upgrade() + + # SendHandler form + + ws.frame_write(sock, ws.OP_TEXT, 'handler:blah') + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'handler:blah') + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'handler-ok') + + # Future form + + ws.frame_write(sock, ws.OP_TEXT, 'blah') + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'blah') + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'future-done') + + # sendObject(Object) with an Integer, which is sent as text + + ws.frame_write(sock, ws.OP_TEXT, 'object:123') + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, '123') + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'future-done') + + # sendObject(Object) with a String, which has no encoder: the future + # completes with the failure instead of hanging + + ws.frame_write(sock, ws.OP_TEXT, 'object:blah') + frame = ws.frame_read(sock) + assert frame['data'].startswith(b'future-fail: '), 'failed future' + assert b'EncodeException' in frame['data'], 'failed future reason' + + # the session is usable for a further message after each of them + + ws.frame_write(sock, ws.OP_TEXT, 'handler:again') + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'handler:again') + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'handler-ok') + + close_connection(sock) + + +def test_java_websockets_async_binary(): + client.load('websockets_async') + + _, sock, _ = ws.upgrade() + + # SendHandler form + + ws.frame_write(sock, ws.OP_BINARY, b'h\x00\x01\x02\xff') + check_frame(ws.frame_read(sock), True, ws.OP_BINARY, b'h\x00\x01\x02\xff') + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'handler-ok') + + # Future form + + ws.frame_write(sock, ws.OP_BINARY, b'\x00\x01\x02\xff') + check_frame(ws.frame_read(sock), True, ws.OP_BINARY, b'\x00\x01\x02\xff') + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'future-done') + + close_connection(sock) + + +def test_java_websockets_async_large(): + # A message larger than the module's 8 KiB buffers is sent whole through + # the asynchronous remote. test_java_websockets_async_text_one_frame + # checks the text message on the wire, frame by frame. + client.load('websockets_async') + + _, sock, _ = ws.upgrade() + + payload = '*' * 65536 + + ws.frame_write(sock, ws.OP_TEXT, payload) + check_frame(ws.message_read(sock), True, ws.OP_TEXT, payload) + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'future-done') + + payload = b'*' * 65536 + + ws.frame_write(sock, ws.OP_BINARY, payload) + check_frame(ws.frame_read(sock), True, ws.OP_BINARY, payload) + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'future-done') + + payload = b'h' + b'*' * 65535 + + ws.frame_write(sock, ws.OP_BINARY, payload) + check_frame(ws.frame_read(sock), True, ws.OP_BINARY, payload) + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'handler-ok') + + close_connection(sock) + + +def test_java_websockets_async_text_one_frame(): + check_text_one_frame('websockets_async', 'future-done') + + +def test_java_websockets_async_text_one_frame_forms(): + # One frame also through the SendHandler form, for characters of every + # UTF-8 width, and with batching on, where the frame reaches the wire + # through the 8 KiB output buffer in parts. + client.load('websockets_async') + + _, sock, _ = ws.upgrade() + + wide = 'a\u00e9\u20ac\U0001f600' * 4096 # 1, 2, 3 and 4 bytes, 40 KiB + + for payload, report in ( + ('handler:' + '*' * 64 * 2**10, 'handler-ok'), + (wide, 'future-done'), + ('handler:' + wide, 'handler-ok'), + ('batchtext:' + '*' * 64 * 2**10, 'batchtext-flushed'), + ): + ws.frame_write(sock, ws.OP_TEXT, payload) + + frame = ws.frame_read(sock) + assert frame['fin'], f'one frame, got {len(frame["data"])} bytes' + check_frame(frame, True, ws.OP_TEXT, payload) + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, report) + + close_connection(sock) + + +def test_java_websockets_async_text_above_cap(): + # Above nginx.unit.websocket.MAX_SEND_BUFFER_SIZE a text message still + # fragments, 8 KiB a frame; with the cap lowered to 8 KiB a 32 KiB + # message is four frames. + client.load('websockets_async') + + assert 'success' in client.conf( + ['-Dnginx.unit.websocket.MAX_SEND_BUFFER_SIZE=8192'], + 'applications/websockets_async/options', + ), 'lower the send buffer cap' + + _, sock, _ = ws.upgrade() + + payload = '*' * 32 * 2**10 + + ws.frame_write(sock, ws.OP_TEXT, payload) + + frames = [ws.frame_read(sock) for _ in range(4)] + assert [(f['opcode'], f['fin'], len(f['data'])) for f in frames] == [ + (ws.OP_TEXT, False, 8192), + (ws.OP_CONT, False, 8192), + (ws.OP_CONT, False, 8192), + (ws.OP_CONT, True, 8192), + ], 'four parts' + assert b''.join(f['data'] for f in frames).decode() == payload, 'payload' + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'future-done') + + close_connection(sock) + + +def test_java_websockets_async_batch(): + # Batched messages are held until the batch is flushed. flushBatch() + # used to hand the request a null payload and throw, so disallowing + # batching failed and the held messages were never sent. + client.load('websockets_async') + + _, sock, _ = ws.upgrade() + + ws.frame_write(sock, ws.OP_TEXT, 'batch:3') + + for i in range(3): + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, f'b{i}') + + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'batch-flushed') + + # the session is usable after the flush + + ws.frame_write(sock, ws.OP_TEXT, 'handler:again') + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'handler:again') + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'handler-ok') + + close_connection(sock) + + +def test_java_websockets_async_batch_offset(): + # A batched buffer whose position() is not 0: its frame header must give + # the bytes actually sent, or the frame after it is read as its payload. + client.load('websockets_async') + + _, sock, _ = ws.upgrade() + + payload = b'o\x00\x01\x02\x03\x04\x05' + + ws.frame_write(sock, ws.OP_BINARY, payload) + check_frame(ws.frame_read(sock), True, ws.OP_BINARY, payload[3:]) + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'after') + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'offset-flushed') + + close_connection(sock) + + +def test_java_websockets_async_nested(): + # A SendHandler that waits on a send of its own, and one that flushes a + # batch: the nested completion has to run before the handler returns. + client.load('websockets_async') + + _, sock, _ = ws.upgrade() + + ws.frame_write(sock, ws.OP_TEXT, 'nested:a') + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'nested:a') + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'nested-b') + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'nested-c') + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'nested-done') + + close_connection(sock) + + +def test_java_websockets_async_threads(): + # Sends from several threads at once: every one of them is echoed and + # every future completes. A completion is queued behind a running one + # only on the same thread; a queue shared by the endpoint could take a + # second thread's completion after the first had stopped draining it. + client.load('websockets_async') + + _, sock, _ = ws.upgrade() + + n = 8 + + for _ in range(10): + ws.frame_write(sock, ws.OP_TEXT, f'threads:{n}') + + echoes = set() + for _ in range(n): + frame = ws.frame_read(sock) + assert frame['opcode'] == ws.OP_TEXT, 'echo opcode' + echoes.add(frame['data'].decode()) + + assert echoes == {f't{i}' for i in range(n)}, 'every send echoed' + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'threads-done') + + close_connection(sock) + + +@pytest.mark.skipif(not option.unsafe, reason='unsafe, long run') +def test_java_websockets_async_16m(): + # 16 MiB is exactly the default send cap, so it is one frame. Above the + # cap a message is sent 8 KiB at a time, each part started from the + # previous one's completion; lowering the cap to 8 KiB makes 2048 parts, + # which must not nest on the stack. + client.load('websockets_async') + + assert 'success' in client.conf( + { + 'http': { + 'websocket': { + 'max_frame_size': 33554432, + 'keepalive_interval': 0, + } + } + }, + 'settings', + ), 'increase max_frame_size and keepalive_interval' + + _, sock, _ = ws.upgrade() + + payload = '*' * 16 * 2**20 + + ws.frame_write(sock, ws.OP_TEXT, payload) + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, payload) + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'future-done') + + payload = b'*' * 16 * 2**20 + + ws.frame_write(sock, ws.OP_BINARY, payload) + check_frame(ws.frame_read(sock), True, ws.OP_BINARY, payload) + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'future-done') + + close_connection(sock) + + assert 'success' in client.conf( + ['-Dnginx.unit.websocket.MAX_SEND_BUFFER_SIZE=8192'], + 'applications/websockets_async/options', + ), 'lower the send buffer cap' + + _, sock, _ = ws.upgrade() + + payload = '*' * 16 * 2**20 + + ws.frame_write(sock, ws.OP_TEXT, payload) + + sizes = [] + frame = {'fin': False} + while not frame['fin']: + frame = ws.frame_read(sock) + sizes.append(len(frame['data'])) + + assert sizes == [8192] * 2048, 'parts above the cap' + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'future-done') + + close_connection(sock) + +def check_buffers(remote): + # Heap buffers whose bytes do not start at their array's first one: a + # slice with a non-zero arrayOffset() used to be sent from the start of + # the array, and a read-only buffer failed with ReadOnlyBufferException. + client.load('websockets_buffers') + + _, sock, _ = ws.upgrade() + + for kind in ('slice', 'readonly'): + ws.frame_write(sock, ws.OP_TEXT, f'{remote}:{kind}') + check_frame(ws.frame_read(sock), True, ws.OP_BINARY, b'456789ab') + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'done') + + close_connection(sock) + + +def test_java_websockets_buffers(): + check_buffers('basic') + + +def test_java_websockets_async_buffers(): + check_buffers('async') diff --git a/test/test_listen_threads.py b/test/test_listen_threads.py new file mode 100644 index 000000000..63730c3b0 --- /dev/null +++ b/test/test_listen_threads.py @@ -0,0 +1,146 @@ +import time +from pathlib import Path + +import pytest +from conftest import _count_fds, pid_by_name + +from unit.applications.proto import ApplicationProto + +client = ApplicationProto() + +BODY = '0123456789' + +# Enough engines that shrinking destroys several of them on any host. The +# leak is per destroyed engine, so a cycle that starts from the default +# (nxt_ncpu) would destroy nothing on a single-CPU machine; pinning the high +# water mark here makes the delta the same everywhere. +THREADS = 4 + + +@pytest.fixture(autouse=True) +def setup_method_fixture(temp_dir): + Path(f'{temp_dir}/assets').mkdir(parents=True) + Path(f'{temp_dir}/assets/index.html').write_text(BODY, encoding='utf-8') + + assert 'success' in client.conf( + { + "listeners": {"*:8080": {"pass": "routes"}}, + "routes": [{"action": {"share": f'{temp_dir}/assets$uri'}}], + } + ) + + yield + + # The no-restart suite preserves /settings between tests. + assert 'success' in client.conf_delete('settings/listen_threads') + + +def _router_fds(): + return _count_fds(pid_by_name('unit: router')) + + +def _settle_fds(stable=5, timeout=100): + # Engines are created and destroyed asynchronously with respect to the + # configuration reply, so a reading taken right after it can catch the + # router mid-transition. Wait for the count to hold still instead of + # sleeping a fixed amount. + fds = _router_fds() + same = 0 + + for _ in range(timeout): + time.sleep(0.1) + + cur = _router_fds() + + same = same + 1 if cur == fds else 0 + fds = cur + + if same >= stable: + break + + return fds + + +def _wait_fds(expected, timeout=100): + # Return the last reading either way, so a leak fails as the count it is + # rather than as an opaque timeout. + for _ in range(timeout): + fds = _router_fds() + + if fds <= expected: + break + + time.sleep(0.1) + + return fds + + +def _set_threads(threads): + assert 'success' in client.conf( + {"listen_threads": threads}, 'settings' + ), f'listen_threads {threads}' + + assert client.get()['status'] == 200, f'request with {threads} engines' + + +def test_listen_threads_engine_fds(): + # A destroyed router engine used to keep its port socketpair and queue + # memfd open for the router's lifetime (3 descriptors each), because + # nxt_router_thread_exit_handler() only dropped one of the port's two + # references and nxt_port_release() closes nothing. Every shrink of + # "listen_threads" therefore cost descriptors that no later grow + # returned. + _set_threads(THREADS) + + before = _settle_fds() + + _set_threads(1) + _set_threads(THREADS) + + after = _wait_fds(before) + + assert after <= before, ( + f'router leaked {after - before} descriptors over a listen_threads' + f' {THREADS} -> 1 -> {THREADS} cycle ({before} before, {after} after)' + ) + + +def test_listen_threads_shrink_grow_no_listeners(): + # nxt_router_engine_quit() allocates its nxt_joint_job_t (the nxt_work_t + # *and* the nxt_task_t the work item points at) from tmcf->mem_pool but + # used to take no tmcf->count reference, so nxt_router_conf_ready() could + # release the pool while the job was still sitting on the dying engine's + # locked work queue. The window is widest with no listeners configured: + # engine->joints is then empty, so nxt_router_worker_thread_quit() is the + # only thing standing between the post and nxt_thread_exit(), and there is + # no listen-socket delete job on the same queue to serialise behind. + # + # Each PUT is asserted, so a "Value doesn't exist." reply is never counted + # as a clean cycle, and the router pid is compared every cycle because the + # control API only reports the failed apply of the crashed generation, not + # the crash itself. + # + # 50 cycles, not fewer: the per-cycle crash probability is only ~3%, so the + # count is what buys the detection rate. Measured on master, 10 runs each: + # 50 cycles caught it 8 times in 1.2 s per run, 20 cycles 4 times in 0.8 s. + # Halving the detection to save 0.4 s is not a trade worth making; the run + # time here is dominated by pytest and unitd start-up either way. + assert 'success' in client.conf( + { + "listeners": {}, + "routes": [{"action": {"return": 200}}], + } + ) + + pid = pid_by_name('unit: router') + + for i in range(50): + for threads in (8, 4, 8): + assert 'success' in client.conf( + {"listen_threads": threads}, 'settings' + ), f'cycle {i}: listen_threads {threads}' + + assert pid_by_name('unit: router') == pid, ( + f'router restarted during cycle {i}: the engine-quit job was' + ' freed with tmcf->mem_pool before the dying engine drained it' + ) diff --git a/test/test_njs_modules.py b/test/test_njs_modules.py index 79e113054..a42394e0c 100644 --- a/test/test_njs_modules.py +++ b/test/test_njs_modules.py @@ -1,3 +1,15 @@ +import ctypes +import os +import shutil +import signal +import tempfile +import threading +import time +from pathlib import Path + +import pytest + +from conftest import pid_by_name, unit_run, unit_stop from unit.applications.proto import ApplicationProto from unit.option import option @@ -125,3 +137,394 @@ def test_njs_settings_js_module_nul(): resp = client.conf({"js_module": ["next\0x"]}, 'settings') assert 'null character' in resp.get('detail', ''), 'array nul' + + +SHORT_MODULE = b"""export default { + "route": function() {return 'next'} +} +""" + +LONG_MODULE = b"""export default { + "route": function() {return 'next'}, + "other": function() {return '%s'} +} +""" % (b'x' * 4096) + + +def route_conf(name): + return { + "settings": {"js_module": name}, + "listeners": {"*:8080": {"pass": "routes/first"}}, + "routes": { + "first": [{"action": {"pass": f"`routes/${{{name}.route()}}`"}}], + "next": [{"action": {"return": 200}}], + }, + } + + +def start_on(statedir): + # main writes here as root; the directory only has to be traversable. + os.chmod(statedir, 0o755) + unit_run(state_dir=str(statedir)) + + +def test_njs_modules_replace_restart(requires_restart): + """A module deleted and uploaded again with a shorter text is stored + and loaded as the short text. Main deletes and stores in its store + child, in the order the requests came.""" + + unit_stop() + + statedir = Path(tempfile.mkdtemp(prefix='unit-state-')) + stored = statedir / 'scripts' / 'm' + + try: + start_on(statedir) + + assert 'success' in client.conf(LONG_MODULE, '/js_modules/m') + assert stored.read_bytes() == LONG_MODULE, 'long stored' + + assert 'success' in client.conf_delete('/js_modules/m') + assert 'success' in client.conf(SHORT_MODULE, '/js_modules/m') + assert stored.read_bytes() == SHORT_MODULE, 'short stored' + + unit_stop() + start_on(statedir) + + assert 'm' in client.conf_get('/js_modules'), 'loaded on restart' + assert 'success' in client.conf(route_conf('m')) + assert client.get()['status'] == 200 + + finally: + unit_stop() + shutil.rmtree(statedir, ignore_errors=True) + + +def test_njs_modules_replace_torn_file(requires_restart, skip_alert): + """A module file left torn on disk, as a write cut short leaves it, is + not loaded at startup. Uploading the module again must replace the + whole file, not overwrite its head and keep the old tail.""" + + skip_alert(r'.*JS compile module.*failed.*') + + unit_stop() + + statedir = Path(tempfile.mkdtemp(prefix='unit-state-')) + scripts = statedir / 'scripts' + stored = scripts / 'm' + + try: + # The first half of LONG_MODULE: a write cut short. + scripts.mkdir() + stored.write_bytes(LONG_MODULE[: len(LONG_MODULE) // 2]) + + start_on(statedir) + + assert 'm' not in client.conf_get('/js_modules'), 'torn not loaded' + + assert 'success' in client.conf(SHORT_MODULE, '/js_modules/m') + + on_disk = stored.read_bytes() + assert on_disk == SHORT_MODULE, f'stored {len(on_disk)} bytes' + + # The router reads the module from the file, not from the upload. + assert 'success' in client.conf(route_conf('m')), 'module compiles' + assert client.get()['status'] == 200 + + finally: + unit_stop() + shutil.rmtree(statedir, ignore_errors=True) + + +def test_njs_modules_load_after_broken(requires_restart, skip_alert): + """One stored module that does not compile must not hide the modules + read after it at startup.""" + + skip_alert(r'.*JS compile module.*failed.*') + + unit_stop() + + statedir = Path(tempfile.mkdtemp(prefix='unit-state-')) + scripts = statedir / 'scripts' + + try: + scripts.mkdir() + + good = [f'good{i}' for i in range(8)] + + for name in good: + (scripts / name).write_bytes(SHORT_MODULE) + + # Pick a name that readdir() returns before at least one good module; + # startup reads the directory in that order. + for i in range(32): + broken = f'broken{i}' + (scripts / broken).write_bytes(b'export default {') + + if os.listdir(scripts)[-1] != broken: + break + + (scripts / broken).unlink() + else: + assert False, 'no name is read before a good module' + + start_on(statedir) + + loaded = client.conf_get('/js_modules') + + assert sorted(n for n in loaded if n in good) == good, 'all loaded' + assert broken not in loaded + + finally: + unit_stop() + shutil.rmtree(statedir, ignore_errors=True) + + +def test_njs_modules_name_dot(): + """Main keeps its temporary file in the scripts directory under a name + that starts with ".", so no module name may start with ".".""" + + for name in ['.x', '.store.tmp']: + resp = client.conf(SHORT_MODULE, f'/js_modules/{name}') + assert resp.get('error') == 'Invalid JS module name.', name + + assert client.conf_get('/js_modules') == {} + + +def test_njs_modules_skip_store_tmp(requires_restart): + """A temporary file that a store left behind is not loaded as a + module.""" + + unit_stop() + + statedir = Path(tempfile.mkdtemp(prefix='unit-state-')) + scripts = statedir / 'scripts' + + try: + scripts.mkdir() + (scripts / '.store.tmp').write_bytes(SHORT_MODULE) + + start_on(statedir) + + assert client.conf_get('/js_modules') == {} + + finally: + unit_stop() + shutil.rmtree(statedir, ignore_errors=True) + + +PTRACE_CONT = 7 +PTRACE_DETACH = 17 +PTRACE_GETEVENTMSG = 0x4201 +PTRACE_SEIZE = 0x4206 +PTRACE_INTERRUPT = 0x4207 +PTRACE_O_TRACEFORK = 0x2 +PTRACE_O_TRACEVFORK = 0x4 +PTRACE_O_TRACECLONE = 0x8 +PTRACE_EVENT_STOP = 128 +WALL = 0x40000000 + + +class ForkFreezer(threading.Thread): + """Trace the main process with ptrace(2) and keep the first process it + forks stopped until release(). Every later child runs untouched. + unitd has no test hooks, so this is how a store child is held.""" + + def __init__(self, pid): + super().__init__(daemon=True) + + self.pid = pid + self.libc = ctypes.CDLL(None, use_errno=True) + self.libc.ptrace.restype = ctypes.c_long + self.libc.ptrace.argtypes = [ + ctypes.c_long, + ctypes.c_long, + ctypes.c_void_p, + ctypes.c_void_p, + ] + + self.error = None + self.frozen = None + self.attached = threading.Event() + self.holding = threading.Event() + self.release_req = threading.Event() + self.stop_req = threading.Event() + self.done = threading.Event() + + def ptrace(self, req, pid, addr=0, data=0): + return self.libc.ptrace(req, pid, addr, data) + + def run(self): + try: + self.trace() + finally: + self.done.set() + + def trace(self): + opts = PTRACE_O_TRACEFORK | PTRACE_O_TRACEVFORK | PTRACE_O_TRACECLONE + + if self.ptrace(PTRACE_SEIZE, self.pid, 0, opts) != 0: + self.error = os.strerror(ctypes.get_errno()) + self.attached.set() + return + + self.attached.set() + + released = False + interrupted = False + + while True: + if self.release_req.is_set() and self.holding.is_set(): + if not released: + self.ptrace(PTRACE_DETACH, self.frozen) + released = True + + if self.stop_req.is_set() and not interrupted: + self.ptrace(PTRACE_INTERRUPT, self.pid) + interrupted = True + + pid, status = os.waitpid(self.pid, WALL | os.WNOHANG) + + if pid == 0: + time.sleep(0.002) + continue + + if not os.WIFSTOPPED(status): + return + + sig = os.WSTOPSIG(status) + event = status >> 16 + + if event == 0: + # A signal for main: deliver it. + if interrupted: + self.ptrace(PTRACE_DETACH, self.pid, 0, sig) + return + + self.ptrace(PTRACE_CONT, self.pid, 0, sig) + continue + + if event == PTRACE_EVENT_STOP: + if interrupted: + self.ptrace(PTRACE_DETACH, self.pid) + return + + self.ptrace(PTRACE_CONT, self.pid) + continue + + # A fork, vfork or clone event: a new traced child. + msg = ctypes.c_ulong() + self.ptrace( + PTRACE_GETEVENTMSG, self.pid, 0, ctypes.addressof(msg) + ) + child = msg.value + + # The child reports its first stop; take it before main runs on. + os.waitpid(child, WALL) + + if self.frozen is None: + self.frozen = child + self.holding.set() + + else: + self.ptrace(PTRACE_DETACH, child) + + self.ptrace(PTRACE_CONT, self.pid) + + def release(self): + self.release_req.set() + + def stop(self): + self.release_req.set() + self.stop_req.set() + self.done.wait(10) + + +def test_njs_modules_store_router_restart( + requires_restart, unit_pid, skip_alert, skip_fds_check +): + """The router restarts while main stores a module. The controller must + answer the upload once, when the module is on disk, and must run a + configuration sent meanwhile only after the store.""" + + skip_alert(r'process \d+ exited on signal 9') + skip_fds_check(router=True) + + # The store of the configuration that the start of the test sent must + # not be the child that is held. + time.sleep(0.5) + + controller = pid_by_name('unit: controller') + router = pid_by_name('unit: router') + + freezer = ForkFreezer(unit_pid) + freezer.start() + freezer.attached.wait(5) + + if freezer.error is not None: + freezer.stop() + pytest.skip(f'ptrace: {freezer.error}') + + answers = {} + + def put(key, body, path='/config'): + answers[key] = client.conf(body, path) + answers[f'{key}_time'] = time.monotonic() + + upload = threading.Thread( + target=put, args=('module', SHORT_MODULE, '/js_modules/m') + ) + reconf = threading.Thread( + target=put, + args=( + 'conf', + { + "listeners": {"*:8080": {"pass": "routes"}}, + "routes": [{"action": {"return": 204}}], + }, + ), + ) + + try: + upload.start() + + assert freezer.holding.wait(10), 'the store child is held' + + os.kill(int(router), signal.SIGKILL) + + for _ in range(100): + new_router = pid_by_name('unit: router') + if new_router not in (None, router): + break + + time.sleep(0.05) + + else: + assert False, 'the router restarted' + + # Let the controller see the new router and send it the config. + time.sleep(1) + + reconf.start() + time.sleep(0.5) + + assert 'module' not in answers, 'no answer before the store' + assert 'conf' not in answers, 'no reconfiguration during the store' + + released = time.monotonic() + freezer.release() + + upload.join(10) + reconf.join(10) + + finally: + freezer.stop() + + assert answers['module'].get('success') == 'JS module uploaded.' + assert answers['module_time'] >= released + assert 'success' in answers['conf'], 'reconfiguration' + assert answers['conf_time'] >= released, 'after the store' + + assert pid_by_name('unit: controller') == controller, 'controller alive' + assert 'm' in client.conf_get('/js_modules') + assert client.get()['status'] == 204 diff --git a/test/test_node_websockets.py b/test/test_node_websockets.py index 68bdd578f..8dfbd126d 100644 --- a/test/test_node_websockets.py +++ b/test/test_node_websockets.py @@ -5,10 +5,13 @@ from unit.applications.lang.node import ApplicationNode from unit.applications.websockets import ApplicationWebsocket +from unit.control import Control +from unit.option import option prerequisites = {'modules': {'node': 'any'}} client = ApplicationNode() +control = Control() ws = ApplicationWebsocket() @@ -237,6 +240,7 @@ def test_node_websockets_handshake_connection_absent(): # FAIL 'Sec-WebSocket-Protocol': 'chat', 'Sec-WebSocket-Version': 13, }, + connection_close=False, ) assert resp['status'] == 400, 'status' @@ -368,9 +372,8 @@ def test_node_websockets_protocol_absent(): # autobahn-testsuite # -# Some following tests fail because of Unit does not support UTF-8 -# validation for websocket frames. It should be implemented -# by application, if necessary. +# The router validates the payload of a text message and the reason of a +# close frame as UTF-8 and fails the connection with 1007 when it is not. def test_node_websockets_1_1_1__1_1_8(): @@ -492,22 +495,19 @@ def test_node_websockets_2_10__2_11(): _, sock, _ = ws.upgrade() - for i in range(0, 10): - ws.frame_write(sock, ws.OP_PING, f'payload-{i}') + pings = [f'payload-{i}' for i in range(0, 10)] - for i in range(0, 10): - frame = ws.frame_read(sock) - check_frame(frame, True, ws.OP_PONG, f'payload-{i}') + for payload in pings: + ws.frame_write(sock, ws.OP_PING, payload) + + ws.pongs_read(sock, pings) # 2_11 - for i in range(0, 10): - opcode = ws.OP_PING - ws.frame_write(sock, opcode, f'payload-{i}', chopsize=1) + for payload in pings: + ws.frame_write(sock, ws.OP_PING, payload, chopsize=1) - for i in range(0, 10): - frame = ws.frame_read(sock) - check_frame(frame, True, ws.OP_PONG, f'payload-{i}') + ws.pongs_read(sock, pings) close_connection(sock) @@ -1041,28 +1041,26 @@ def test_node_websockets_6_1_1__6_4_4(): close_connection(sock) + # 6_3_1 -# Unit does not support UTF-8 validation -# -# # 6_3_1 FAIL -# -# payload_1 = '\xce\xba\xe1\xbd\xb9\xcf\x83\xce\xbc\xce\xb5' -# payload_2 = '\xed\xa0\x80' -# payload_3 = '\x65\x64\x69\x74\x65\x64' -# -# payload = payload_1 + payload_2 + payload_3 -# -# ws.message(sock, ws.OP_TEXT, payload) -# check_close(sock, 1007) -# -# # 6_3_2 FAIL -# -# _, sock, _ = ws.upgrade() -# -# ws.message(sock, ws.OP_TEXT, payload, fragmention_size=1) -# check_close(sock, 1007) -# -# # 6_4_1 ... 6_4_4 FAIL + _, sock, _ = ws.upgrade() + + # Raw bytes: written as str these would be encoded to valid UTF-8. + payload_1 = b'\xce\xba\xe1\xbd\xb9\xcf\x83\xce\xbc\xce\xb5' + payload_2 = b'\xed\xa0\x80' + payload_3 = b'\x65\x64\x69\x74\x65\x64' + + payload = payload_1 + payload_2 + payload_3 + + ws.message(sock, ws.OP_TEXT, payload) + check_close(sock, 1007) + + # 6_3_2 + + _, sock, _ = ws.upgrade() + + ws.message(sock, ws.OP_TEXT, payload, fragmention_size=1) + check_close(sock, 1007) def test_node_websockets_7_1_1__7_5_1(): @@ -1190,16 +1188,17 @@ def test_node_websockets_7_1_1__7_5_1(): ws.frame_write(sock, ws.OP_CLOSE, payload) check_close(sock, 1002) + # 7_5_1 -# # 7_5_1 FAIL Unit does not support UTF-8 validation -# -# _, sock, _ = ws.upgrade() -# -# payload = ws.serialize_close(reason = '\xce\xba\xe1\xbd\xb9\xcf' \ -# '\x83\xce\xbc\xce\xb5\xed\xa0\x80\x65\x64\x69\x74\x65\x64') -# -# ws.frame_write(sock, ws.OP_CLOSE, payload) -# check_close(sock, 1007) + _, sock, _ = ws.upgrade() + + # A raw reason: 0xed 0xa0 0x80 is a surrogate, which has no valid UTF-8 + # encoding. + payload = ws.serialize_close() + b'\xce\xba\xe1\xbd\xb9\xcf' \ + b'\x83\xce\xbc\xce\xb5\xed\xa0\x80\x65\x64\x69\x74\x65\x64' + + ws.frame_write(sock, ws.OP_CLOSE, payload) + check_close(sock, 1007) def test_node_websockets_7_7_X__7_9_X(): @@ -1262,10 +1261,8 @@ def test_node_websockets_7_13_1__7_13_2(): check_close(sock, 1002) -def test_node_websockets_9_1_1__9_6_6(is_unsafe, system): - if not is_unsafe: - pytest.skip('unsafe, long run') - +@pytest.mark.skipif(not option.unsafe, reason='unsafe, long run') +def test_node_websockets_9_1_1__9_6_6(system): client.load('websockets/mirror') assert 'success' in client.conf( @@ -1292,7 +1289,7 @@ def check_payload(opcode, length, chopsize=None): payload = b'*' * length ws.frame_write(sock, opcode, payload, chopsize=chopsize) - frame = ws.frame_read(sock, read_timeout=5) + frame = ws.frame_read(sock, read_timeout=ws.LARGE_MESSAGE_TIMEOUT) check_frame(frame, True, opcode, payload) def check_message(opcode, f_size): @@ -1302,7 +1299,7 @@ def check_message(opcode, f_size): payload = b'*' * 4 * 2**20 ws.message(sock, opcode, payload, fragmention_size=f_size) - frame = ws.frame_read(sock, read_timeout=5) + frame = ws.frame_read(sock, read_timeout=ws.LARGE_MESSAGE_TIMEOUT) check_frame(frame, True, opcode, payload) check_payload(op_text, 64 * 2**10) # 9_1_1 @@ -1433,3 +1430,90 @@ def test_node_websockets_keepalive_interval(): check_frame(frame, True, ws.OP_PING, '') # PING frame sock.close() + + +def test_node_websockets_timeout_does_not_detach(): + """An upgraded stream is not a request the deadline gave up on. + + "limits": {"timeout"} re-arms nxt_router_app_timeout() for every non-last + message from the application, and the 101 is one, so an idle websocket + reaches that handler with the deadline passed. The request's accounting + was already returned by the upgrade, so there is no worker still running + it: the router must not report the process "detached", which is its view + of a worker holding a request whose deadline passed. + + Only the accounting is asserted. The handler also answers 503 and drops + the stream, which is wrong for a websocket and is issue #422. + """ + + client.load('websockets/mirror') + + assert 'success' in client.conf( + {'timeout': 1}, 'applications/websockets%2Fmirror/limits' + ), 'configure timeout' + + _, sock, _ = ws.upgrade() + + # No traffic either way -- keepalive_interval is 0 from the fixture -- so + # nothing re-arms the timer and it expires. + time.sleep(3) + + processes = ( + control.conf_get('/status') + .get('applications', {}) + .get('websockets/mirror', {}) + .get('processes', {}) + ) + + assert processes.get('detached', 0) == 0, ( + f'an upgraded stream marked the worker detached: {processes}' + ) + + sock.close() + + +def wait_idle(name, idle, timeout=10): + """The idle count of an application, once it settles. + + Read by name rather than compared whole: /status grows keys, and this + is about one of them. + """ + + for _ in range(timeout * 10): + processes = control.conf_get('/status')['applications'][name][ + 'processes' + ] + + if processes['idle'] == idle: + break + + time.sleep(0.1) + + return processes + + +def test_node_websockets_worker_idle_after_close(): + client.load('websockets/mirror') + + name = 'websockets/mirror' + + _, sock, _ = ws.upgrade() + + ws.frame_write(sock, ws.OP_TEXT, 'blah') + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'blah') + + # The worker is out of the idle economy while the session is open. + + processes = wait_idle(name, 0) + + assert processes['running'] == 1, 'running while open' + assert processes['idle'] == 0, 'busy while open' + + close_connection(sock) + + # And rejoins it when the session ends. + + processes = wait_idle(name, 1) + + assert processes['running'] == 1, 'running after close' + assert processes['idle'] == 1, 'idle after close' diff --git a/test/test_otel.py b/test/test_otel.py index 3667e89bd..4e4eb3730 100644 --- a/test/test_otel.py +++ b/test/test_otel.py @@ -17,7 +17,9 @@ # and OTLP/gRPC, so the transport under test is chosen by config (--protocol), # never by how the mock was built. CI installs it via the "Build fake_otlp" step # in ci.yml, mirroring fake_upstream. Skip gracefully when it is not built. -FAKE_OTLP_BIN = '/usr/local/bin/fake_otlp' +# Overridable so the suite can be run against a locally built fake_otlp +# without root (installing into /usr/local/bin needs it). +FAKE_OTLP_BIN = os.environ.get('FAKE_OTLP_BIN', '/usr/local/bin/fake_otlp') _skipif_no_fake_otlp = pytest.mark.skipif( not os.path.exists(FAKE_OTLP_BIN), @@ -43,8 +45,12 @@ def _get_free_port(): return s.getsockname()[1] -def _run_fake_otlp(port, requests=None, dump=None, protocol='http'): +def _run_fake_otlp( + port, requests=None, dump=None, protocol='http', status=None +): cmd = [FAKE_OTLP_BIN, '--port', str(port), '--protocol', protocol] + if status is not None: + cmd += ['--status', str(status)] if requests is not None: cmd += ['--requests', str(requests)] if dump is not None: @@ -116,6 +122,41 @@ def _require_otel(): _configure_or_skip(1) +STATUS_CODE_KEY = b'http.response.status_code' + + +def _otlp_int_value(value): + """Encode an OTLP AnyValue int_value as it appears on the wire. + + int_value is field 3 of AnyValue with varint wire type, so the tag byte is + (3 << 3) | 0 == 0x18, followed by the value as a base-128 varint. + """ + out = bytearray(b'\x18') + while True: + byte = value & 0x7F + value >>= 7 + out.append(byte | 0x80 if value else byte) + if not value: + return bytes(out) + + +def _has_keyed_int_attr(body, key, value): + """Whether `key` is followed by an int_value attribute equal to `value`. + + The dump holds raw trace ids, span ids and timestamps, so the encoded value + can occur anywhere by coincidence. Anchoring the search to the bytes just + after each occurrence of the key is what makes this a statement about the + attribute rather than about the payload. + """ + want = _otlp_int_value(value) + at = body.find(key) + while at != -1: + if want in body[at + len(key):at + len(key) + 16]: + return True + at = body.find(key, at + 1) + return False + + def _response_headers_lower(resp): return {k.lower(): v for k, v in resp['headers'].items()} @@ -165,9 +206,11 @@ def test_otel_span_exported_with_service_name(tmp_path, protocol): with open(dump, 'rb') as f: body = f.read() assert b'FreeUnit' in body, 'exported span must carry service.name=FreeUnit' - # Semconv span attributes (1.35.6): recorded via nxt_otel_rs_add_attr, - # not as the old free-form span events. The attribute *keys* travel as - # literal strings in the OTLP protobuf payload. + # Semconv span attributes: recorded via nxt_otel_rs_add_attrs, not as + # the old free-form span events. The attribute *keys* still travel as + # literal strings in the OTLP protobuf payload -- keying them by id + # across the FFI changed where they live in the process, not the wire + # format. assert b'http.request.method' in body, 'span must carry semconv method attr' assert b'url.path' in body, 'span must carry semconv url.path attr' assert b'http.response.status_code' in body, 'span must carry status attr' @@ -180,11 +223,15 @@ def test_otel_span_exported_with_service_name(tmp_path, protocol): def test_otel_span_5xx_status(tmp_path, protocol): """A 5xx response is still traced and the span records the error status. - The status_code attribute is emitted as a string (sprintf "%d"), so the - literal `503` travels in the OTLP payload. Per the 1.35.6 fix, 503 >= 500 - also marks the span Status::Error (nxt_otel_rs_set_error); that enum is not - asserted here (it is not a literal in the protobuf), but the status_code - value is the concrete, reliable signal. + http.response.status_code is an OTLP *integer* attribute, so 503 travels as + a varint rather than the ASCII bytes `503`: in AnyValue, int_value is field + 3, giving tag 0x18 followed by varint 503 (0xf7 0x03). Asserting on those + bytes is what distinguishes a correctly typed integer attribute from the + stringified one the semantic conventions do not ask for. + + Per the 1.35.6 fix, 503 >= 500 also marks the span Status::Error + (nxt_otel_rs_set_error); that enum is not asserted here (it is a protobuf + enum, not a literal). """ port = _get_free_port() dump = str(tmp_path / 'otlp_dump.bin') @@ -206,11 +253,11 @@ def test_otel_span_5xx_status(tmp_path, protocol): '503', 'routes/0/action/return' ), 'switch route to return 503' - # Fire 503s until the error span reaches the collector. status_code is - # emitted as the string "503" (sprintf "%d"); per the 1.35.6 fix, - # 503 >= 500 also marks the span Status::Error (not asserted here -- it - # is a protobuf enum, not a literal). Only the 503 requests can add the - # "503" bytes, so the readiness 200 spans don't false-positive. + # Fire 503s until the error span reaches the collector. The wait + # condition is the same keyed check as the assertion below: a bare + # substring search would also match a trace id or timestamp that + # happens to contain these bytes, ending the poll before the 503 span + # was actually exported. body = b'' found = False for _ in range(int(EXPORT_TIMEOUT * 10)): @@ -222,14 +269,24 @@ def test_otel_span_5xx_status(tmp_path, protocol): with open(dump, 'rb') as f: body = f.read() - if b'503' in body: + if _has_keyed_int_attr(body, STATUS_CODE_KEY, 503): found = True break time.sleep(0.1) - assert found, 'span with status_code=503 was not exported' - assert b'http.response.status_code' in body, 'status_code attr missing' + assert STATUS_CODE_KEY in body, 'status_code attr missing' + assert found, ( + 'http.response.status_code must carry 503 as an int_value varint; ' + 'a span with that attribute was never exported' + ) + + # The readiness spans are still in the same dump and carry int_value + # 200, which confirms the keyed search discriminates by value rather + # than merely finding the key. + assert _has_keyed_int_attr(body, STATUS_CODE_KEY, 200), ( + 'readiness spans should still carry status_code=200' + ) finally: _kill(proc) @@ -255,7 +312,11 @@ def test_otel_traceparent_in_response(protocol): @_skipif_no_fake_otlp @pytest.mark.parametrize('protocol', ['http', 'grpc']) def test_otel_traceparent_inherited(tmp_path, protocol): - """An incoming traceparent is continued: the exported span keeps its trace id.""" + """An incoming traceparent is continued: the exported span keeps its + trace id, and the traceparent handed onward (echoed in the response, + and by extension forwarded to any peer/app) carries FreeUnit's own span + id as parent-id -- not the client's original parent-id -- so that a + downstream span is a child of FreeUnit's span, not a sibling of it.""" port = _get_free_port() dump = str(tmp_path / 'otlp_dump.bin') proc = _run_fake_otlp(port, requests=1, dump=dump, protocol=protocol) @@ -272,7 +333,13 @@ def test_otel_traceparent_inherited(tmp_path, protocol): ) assert resp['status'] == 200 # FreeUnit echoes the inherited traceparent back in the response. - assert TRACE_ID in _response_headers_lower(resp).get('traceparent', '') + echoed = _response_headers_lower(resp).get('traceparent', '') + assert TRACE_ID in echoed + assert PARENT_ID not in echoed, ( + 'the echoed/forwarded traceparent must not carry the client\'s ' + 'original parent-id' + ) + echoed_parent_id = echoed.split('-')[2] try: proc.wait(timeout=EXPORT_TIMEOUT) @@ -285,6 +352,13 @@ def test_otel_traceparent_inherited(tmp_path, protocol): assert bytes.fromhex(TRACE_ID) in body, ( 'exported span must keep the inherited trace id' ) + # And the span id FreeUnit exported for itself is the same span id it + # handed onward as parent-id -- proof that the forwarded traceparent + # names FreeUnit's own span, not a copy of the inbound one. + assert bytes.fromhex(echoed_parent_id) in body, ( + 'the parent-id handed onward must be the id of the span FreeUnit ' + 'itself exported' + ) finally: _kill(proc) @@ -310,6 +384,31 @@ def test_otel_sampling_zero_exports_nothing(protocol): _kill(proc) +@_skipif_no_fake_otlp +@pytest.mark.parametrize('protocol', ['http', 'grpc']) +def test_otel_sampling_zero_still_propagates(protocol): + """sampling_ratio=0 → nothing is exported, but context still propagates. + + The request path skips the attribute work for a span the sampler dropped + (nxt_otel_span_add_headers, src/nxt_otel.c). Propagation sits outside that + gate on purpose: W3C Trace Context requires the traceparent to reach the + peer and the application whatever the sampling decision was, so a + downstream service can join -- or knowingly decline to join -- the trace. + """ + port = _get_free_port() + proc = _run_fake_otlp(port, protocol=protocol) # run forever, absorb exports + try: + _configure_or_skip(port, sampling_ratio=0.0, protocol=protocol) + + resp = _get_until_header('traceparent') + assert resp['status'] == 200 + assert 'traceparent' in _response_headers_lower(resp), ( + 'an unsampled request must still carry a traceparent header' + ) + finally: + _kill(proc) + + @_skipif_no_fake_otlp @pytest.mark.parametrize('protocol', ['http', 'grpc']) def test_otel_traceparent_malformed(protocol): @@ -763,3 +862,108 @@ def test_otel_sampling_ratio_nonfinite_rejected(): conf = client.conf(body) assert 'error' in conf, f'non-finite sampling_ratio must be rejected: {conf}' + + +def _status_telemetry(): + """The /status "telemetry" object, or None when it is absent.""" + return client.conf_get('/status').get('telemetry') + + +def _wait_for_spans(field, timeout=EXPORT_TIMEOUT, delay=0.1): + """Poll /status until telemetry/spans/ is non-zero; return the object.""" + deadline = time.time() + timeout + telemetry = _status_telemetry() + while time.time() < deadline: + if telemetry is not None and telemetry['spans'][field] > 0: + return telemetry + time.sleep(delay) + telemetry = _status_telemetry() + return telemetry + + +def test_otel_status_absent_without_telemetry(): + """/status carries no "telemetry" object when telemetry is not configured. + + True of a build without --otel as well, which is why this one does not + probe for OTel support first: the absence is the contract in both cases. + """ + assert 'success' in client.conf( + { + "listeners": {"*:8080": {"pass": "routes"}}, + "routes": [{"action": {"return": 200}}], + "applications": {}, + } + ) + + assert _status_telemetry() is None + + +@_skipif_no_fake_otlp +@pytest.mark.parametrize('protocol', ['http', 'grpc']) +def test_otel_status_counts_exported_spans(protocol): + """A span that reaches the collector is counted in /status (issue #219).""" + port = _get_free_port() + proc = _run_fake_otlp(port, protocol=protocol) + try: + _configure_or_skip(port, protocol=protocol) + + assert _get_until_header('traceparent')['status'] == 200 + + telemetry = _wait_for_spans('exported') + assert telemetry is not None, '/status must report telemetry when configured' + assert telemetry['spans']['exported'] > 0, 'exported spans must be counted' + assert telemetry['spans']['failed'] == 0, 'a live collector must not fail' + finally: + _kill(proc) + + +def test_otel_status_counts_failed_spans(): + """An export the collector refuses is counted as failed, while the process + is still running -- the whole point of #219: before this the only signal + was a log line at exit. + """ + # Nothing is listening here: the export is refused, not merely slow. + port = _get_free_port() + _configure_or_skip(port) + + assert _get_until_header('traceparent')['status'] == 200 + + telemetry = _wait_for_spans('failed') + assert telemetry is not None, '/status must report telemetry when configured' + assert telemetry['spans']['failed'] > 0, 'a refused export must be counted' + assert telemetry['spans']['exported'] == 0, 'nothing can have been exported' + + +@_skipif_no_fake_otlp +@pytest.mark.parametrize('protocol', ['http', 'grpc']) +def test_otel_export_not_retried(tmp_path, protocol): + """A batch the collector refuses with a retryable error is sent once. + + opentelemetry-otlp 0.33 retries such an export up to 3 times by default. + The batch worker blocks during the retries, so FreeUnit turns them off. + With batch_size 1 each failed span is one batch, so the collector must + see exactly one request per failed span. + """ + port = _get_free_port() + dump = tmp_path / 'otlp_dump.bin' + proc = _run_fake_otlp(port, dump=str(dump), protocol=protocol, status=503) + try: + _configure_or_skip(port, protocol=protocol) + + assert _get_until_header('traceparent')['status'] == 200 + + telemetry = _wait_for_spans('failed') + assert telemetry is not None, '/status must report telemetry' + assert telemetry['spans']['failed'] > 0, 'a 503 export must fail' + + # With retries on, the last attempt of a batch comes about 1 s after + # the first. Wait longer than that, then compare the counts. + time.sleep(4) + + failed = _status_telemetry()['spans']['failed'] + requests = dump.read_bytes().count(b'--fake_otlp-request-boundary--') + assert requests == failed, ( + f'{requests} export requests for {failed} failed batches' + ) + finally: + _kill(proc) diff --git a/test/test_otel_traceparent_app.py b/test/test_otel_traceparent_app.py new file mode 100644 index 000000000..8744aaacf --- /dev/null +++ b/test/test_otel_traceparent_app.py @@ -0,0 +1,89 @@ +"""The traceparent an application sees; test_otel_traceparent_app_php.py +runs the same checks against PHP.""" + +import re +import time + +import pytest + +from test_otel import PARENT_ID, TRACE_ID, _get_free_port +from unit.applications.lang.python import ApplicationPython + +prerequisites = {'modules': {'python': 'all'}} + +client = ApplicationPython() + +INBOUND = f'00-{TRACE_ID}-{PARENT_ID}-01' +VALID = re.compile(r'^[0-9a-f]{2}-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$') + + +def seen(app, traceparent=None, otel=True): + app.load('traceparent') + + if otel: + conf = app.conf_get() + conf.setdefault('settings', {})['telemetry'] = { + 'endpoint': f'http://127.0.0.1:{_get_free_port()}/v1/traces', + 'protocol': 'http', + 'sampling_ratio': 1.0, + 'batch_size': 1, + } + resp = app.conf(conf) + + if 'telemetry' in str(resp).lower(): + pytest.skip('unit built without --otel') + + assert 'success' in resp, resp + + headers = {'Host': 'localhost', 'Connection': 'close'} + if traceparent is not None: + headers['traceparent'] = traceparent + + # otel (re)init races the first requests. + for _ in range(150): + resp = app.get(headers=headers) + value = resp['headers'].get('X-Seen-Traceparent', '') + + if resp['status'] == 200 and value: + return value + + time.sleep(0.1) + + return value + + +def check_forwarded(app): + assert seen(app, INBOUND, otel=False) == INBOUND + + +def check_generated(app): + value = seen(app) + assert VALID.match(value) and TRACE_ID not in value, value + + +def check_inherited(app): + # The parent-id must be FreeUnit's own span, so app spans are children. + value = seen(app, INBOUND) + assert VALID.match(value) and TRACE_ID in value, value + assert PARENT_ID not in value, value + + +def check_malformed(app): + value = seen(app, 'not-a-traceparent') + assert VALID.match(value), value + + +def test_traceparent_forwarded_without_otel(): + check_forwarded(client) + + +def test_traceparent_generated_when_missing_with_otel(): + check_generated(client) + + +def test_traceparent_inherited_with_otel(): + check_inherited(client) + + +def test_traceparent_malformed_replaced_with_otel(): + check_malformed(client) diff --git a/test/test_otel_traceparent_app_php.py b/test/test_otel_traceparent_app_php.py new file mode 100644 index 000000000..d460d3cef --- /dev/null +++ b/test/test_otel_traceparent_app_php.py @@ -0,0 +1,29 @@ +"""test_otel_traceparent_app.py against $_SERVER['HTTP_TRACEPARENT'].""" + +from test_otel_traceparent_app import ( + check_forwarded, + check_generated, + check_inherited, + check_malformed, +) +from unit.applications.lang.php import ApplicationPHP + +prerequisites = {'modules': {'php': 'any'}} + +client = ApplicationPHP() + + +def test_traceparent_forwarded_without_otel(): + check_forwarded(client) + + +def test_traceparent_generated_when_missing_with_otel(): + check_generated(client) + + +def test_traceparent_inherited_with_otel(): + check_inherited(client) + + +def test_traceparent_malformed_replaced_with_otel(): + check_malformed(client) diff --git a/test/test_php_application.py b/test/test_php_application.py index 6f2fc812d..5966f02a4 100644 --- a/test/test_php_application.py +++ b/test/test_php_application.py @@ -238,6 +238,109 @@ def test_php_application_fastcgi_finish_request_2(findall, unit_pid): assert len(errs) == 0, 'no error' +def get_flushed(method, url, **kwargs): + # The "flush" app sleeps after flush(). Read for less time than the + # sleep: what arrives in that time is what flush() sent. Then read + # the rest of the response. + sock = client.http(method, url=url, no_recv=True, **kwargs) + + early = client.recvall(sock, read_timeout=0.5).decode() + rest = client.recvall(sock).decode() + sock.close() + + resp = client._resp_to_dict(early + rest) + + if resp['headers'].get('Transfer-Encoding') == 'chunked': + resp['body'] = client._parse_chunked_body(resp['body']).decode() + + return early, resp + + +def test_php_application_flush_headers(): + client.load('flush', options={'admin': {'output_buffering': '0'}}) + + resp = client.get() + assert resp['status'] == 200, 'start the application' + assert resp['headers']['X-OB-Level'] == '0', 'no output buffer' + + early, resp = get_flushed('GET', '/?sleep=2') + + assert resp['status'] == 200 + assert resp['body'] == 'part2;headers_sent=true;part3;', 'headers_sent()' + assert early.startswith('HTTP/1.1 200 OK\r\n'), 'status line before sleep' + assert 'X-Flush: 1\r\n' in early, 'headers before sleep' + assert 'Transfer-Encoding: chunked\r\n' in early, 'chunked' + assert early.endswith('\r\n\r\n'), 'header end before sleep' + assert 'part2' not in early, 'no body before sleep' + + # A response to HTTP/1.0 is not chunked. The connection close ends + # its body. + early, resp = get_flushed('GET', '/?sleep=2', http_10=True) + + assert resp['body'] == 'part2;headers_sent=true;part3;', 'HTTP/1.0 body' + assert early.startswith('HTTP/1.1 200 OK\r\n'), 'HTTP/1.0 status line' + assert 'X-Flush: 1\r\n' in early, 'HTTP/1.0 headers' + assert early.endswith('\r\n\r\n'), 'HTTP/1.0 header end' + + early, resp = get_flushed('HEAD', '/?sleep=2') + + assert resp['status'] == 200 + assert resp['headers']['X-Flush'] == '1' + assert resp['body'] == '', 'HEAD body' + assert early.startswith('HTTP/1.1 200 OK\r\n'), 'HEAD status line' + assert early.endswith('\r\n\r\n'), 'HEAD header end' + + +def test_php_application_flush_output_buffering(): + client.load('flush', options={'admin': {'output_buffering': '4096'}}) + + resp = client.get() + assert resp['status'] == 200, 'start the application' + assert resp['headers']['X-OB-Level'] == '1', 'output buffer' + + early, resp = get_flushed('GET', '/?sleep=2&part1') + + assert resp['status'] == 200 + assert ( + resp['body'] == 'part1;part2;headers_sent=true;part3;' + ), 'headers_sent()' + assert early.startswith('HTTP/1.1 200 OK\r\n'), 'status line before sleep' + assert 'X-Flush: 1\r\n' in early, 'headers before sleep' + + # flush() does not empty the output buffers of PHP, as in mod_php. + # ob_flush() does that. So only the header comes before the sleep. + assert early.endswith('\r\n\r\n'), 'header end before sleep' + assert 'part1' not in early, 'part1 stays in the output buffer' + + +def test_php_application_flush_after_finish(findall, wait_for_record): + # fastcgi_finish_request() ends the response and leaves the context + # without a request. The script then calls flush() twice. + client.load('flush_after_finish') + + resp = client.get() + assert resp['status'] == 200 + # The client parses the chunked body, so a cut body fails here. + assert resp['headers']['Transfer-Encoding'] == 'chunked' + assert resp['body'] == 'before;', 'only the output before the finish' + + pid = resp['headers']['X-Pid'] + + # The worker is busy until the script ends. Then it serves the next + # request. + resp = client.get() + assert resp['status'] == 200 + assert resp['body'] == 'before;', 'second request' + + assert findall(r'.+\[alert\].+') == [], 'no alert' + assert resp['headers']['X-Pid'] == pid, 'the same process' + + assert ( + wait_for_record(r'php message: flush after finish: done') is not None + ), 'the script ran past flush()' + assert findall(r'Error in fastcgi_finish_request') == [] + + def test_php_application_query_string_absent(): client.load('query_string') @@ -705,6 +808,84 @@ def test_php_application_script(): assert resp['body'] != '', 'body not empty' +def php_script_conf(root, script): + return client.conf( + { + "listeners": {"*:8080": {"pass": "applications/script"}}, + "applications": { + "script": { + "type": client.get_application_type(), + "processes": {"spare": 0}, + "root": root, + "script": script, + } + }, + } + ) + + +def php_script_tree(temp_dir): + Path(f'{temp_dir}/app/sub').mkdir(parents=True) + Path(f'{temp_dir}/app2').mkdir() + + for path in ('app/sub', 'app2'): + Path(f'{temp_dir}/{path}/index.php').write_text( + '', encoding='utf-8' + ) + + +def test_php_application_script_under_root(temp_dir): + php_script_tree(temp_dir) + + assert 'success' in php_script_conf(f'{temp_dir}/app', 'sub/index.php') + + resp = client.get() + assert resp['status'] == 200, 'script in a subdirectory of the root' + assert resp['body'] == '/sub/index.php', 'SCRIPT_NAME' + + # The root "/" is trimmed to an empty string, and every absolute path is + # under it. The script is appended to the root, so it can be absolute. + script = os.path.realpath(f'{temp_dir}/app/sub/index.php') + + assert 'success' in php_script_conf('/', script) + + resp = client.get() + assert resp['status'] == 200, 'script under the root "/"' + assert resp['body'] == script, 'SCRIPT_NAME under the root "/"' + + +def test_php_application_script_outside_root( + temp_dir, skip_alert, wait_for_record +): + skip_alert(r'script is not under php root') + php_script_tree(temp_dir) + + # "/app2" starts with the bytes of "/app", but it is + # a sibling directory, not a subdirectory. + assert 'success' in php_script_conf( + f'{temp_dir}/app', '../app2/index.php' + ) + + assert client.get()['status'] == 503, 'script in a sibling directory' + assert ( + wait_for_record(r'script is not under php root') is not None + ), 'sibling directory alert' + + +def test_php_application_script_is_root( + temp_dir, skip_alert, wait_for_record +): + skip_alert(r'script is not under php root') + php_script_tree(temp_dir) + + assert 'success' in php_script_conf(f'{temp_dir}/app', '.') + + assert client.get()['status'] == 503, 'script is the root directory' + assert ( + wait_for_record(r'script is not under php root') is not None + ), 'root directory alert' + + def test_php_application_index_default(): assert 'success' in client.conf( { diff --git a/test/test_php_compression.py b/test/test_php_compression.py index 00c943154..0487db3a6 100644 --- a/test/test_php_compression.py +++ b/test/test_php_compression.py @@ -1,41 +1,28 @@ import gzip +import hashlib +import socket import pytest from unit.applications.lang.php import ApplicationPHP -from unit.option import option prerequisites = {'modules': {'php': 'any'}} client = ApplicationPHP() -@pytest.fixture(autouse=True) -def requires_restart_mode(): - """ - Configuring compression is not reversible within one unitd: the module - keeps its state in process globals pointing into the router configuration - that was current when they were parsed, and a later configuration without - a compression block neither reinitialises nor clears them, so the next - request dereferences a freed pool and the router dies (#167). - - Without --restart the whole session shares one unitd, so these tests would - crash whichever test runs next. With it, each test gets its own unitd and - the state cannot escape. Drop this fixture once #167 is fixed. - """ - if not option.restart: - pytest.skip('needs --restart until #167 is fixed') - -COMPRESSION_CONF = { - "compression": { - "types": ["text/html*"], - "compressors": [{"encoding": "gzip", "level": 1}], +def configure_compression(encoding='gzip'): + conf = { + "compression": { + "types": ["text/html*"], + "compressors": [{"encoding": encoding}], + } } -} + if encoding == 'gzip': + conf["compression"]["compressors"][0]["level"] = 1 -def configure_compression(): - resp = client.conf({"http": COMPRESSION_CONF}, 'settings') + resp = client.conf({"http": conf}, 'settings') if 'success' in resp: return @@ -43,7 +30,7 @@ def configure_compression(): # Compressors are build options; skip only for that, so a broken config # here fails loudly instead of quietly turning into a skipped test. if 'supported compressor' in resp.get('detail', ''): - pytest.skip('unit built without gzip compression support') + pytest.skip(f'unit built without {encoding} compression support') pytest.fail(f'could not configure compression: {resp}') @@ -68,7 +55,7 @@ def dechunk(data): data = data[end + 2 + size + 2 :] -def get_gzip(): +def get_gzip(encoding='gzip'): """ The shared client decodes bodies as text and splits chunked bodies on CRLF, both of which destroy a compressed payload, so read the response @@ -77,7 +64,7 @@ def get_gzip(): resp = client.get( headers={ 'Host': 'localhost', - 'Accept-Encoding': 'gzip', + 'Accept-Encoding': encoding, 'Connection': 'close', }, raw_resp=True, @@ -150,3 +137,309 @@ def test_php_compression_small_body(): # Declining to compress is an equally valid fix, as long as the # encoding is not advertised. assert body == b'A' * 64, 'uncompressed body delivered verbatim' + + +RANDOM_BODY_SIZE = 9 * 1024 * 1024 +LEAK_REQUESTS = 15 # > 100 MB in total: more shared memory than an app may hold + + +def raw_get(path, encoding, timeout=30, method='GET'): + """ + A plain socket request. The shared client cannot express "fail rather + than block forever", which is exactly the failure these tests look for. + """ + sock = socket.create_connection(('127.0.0.1', 8080), timeout) + sock.settimeout(timeout) + + try: + sock.sendall( + f'{method} {path} HTTP/1.1\r\n' + f'Host: localhost\r\n' + f'Accept-Encoding: {encoding}\r\n' + f'Connection: close\r\n\r\n'.encode() + ) + + chunks = [] + while True: + data = sock.recv(256 * 1024) + if not data: + break + chunks.append(data) + + finally: + sock.close() + + resp = b''.join(chunks) + head, _, body = resp.partition(b'\r\n\r\n') + lines = head.split(b'\r\n') + + status = int(lines[0].split()[1]) + + headers = {} + for line in lines[1:]: + name, _, value = line.partition(b':') + headers[name.strip().decode()] = value.strip().decode() + + if headers.get('Transfer-Encoding') == 'chunked': + body = dechunk(body) + + return status, headers, body + + +def decompress(encoding, data): + if encoding == 'gzip': + return gzip.decompress(data) + + if encoding == 'br': + brotli = pytest.importorskip( + 'brotli', reason='python brotli bindings not installed' + ) + return brotli.decompress(data) + + if encoding == 'zstd': + try: + from compression import zstd # Python >= 3.14 + + return zstd.decompress(data) + except ImportError: + zstandard = pytest.importorskip( + 'zstandard', reason='python zstd bindings not installed' + ) + return zstandard.ZstdDecompressor().decompressobj().decompress(data) + + pytest.fail(f'no decompressor for {encoding}') + + +@pytest.mark.parametrize('encoding', ['gzip', 'br', 'zstd']) +def test_php_compression_large_random_body(encoding): + """ + Coverage for the chain walk in nxt_http_comp_compress_app_response(): + a multi-megabyte incompressible body arrives as a long sequence of + port-mmap buffers, so every compressor is driven across many calls with + the finishing one arriving last. Check the body still round-trips. + """ + client.load('comp_random_body') + configure_compression(encoding) + + status, headers, body = raw_get( + f'/?n={RANDOM_BODY_SIZE}', encoding + ) + + assert status == 200, 'status' + assert headers['Content-Encoding'] == encoding, 'encoding advertised' + + # The compressed body is framed either by Content-Length or by chunked + # transfer encoding -- never by both, and never by neither, or the client + # cannot tell where it ends. + has_clen = 'Content-Length' in headers + is_chunked = headers.get('Transfer-Encoding') == 'chunked' + + assert has_clen != is_chunked, f'exactly one framing: {headers}' + + if has_clen: + assert len(body) == int(headers['Content-Length']), 'framing consistent' + + plain = decompress(encoding, body) + + assert len(plain) == RANDOM_BODY_SIZE, 'full body length' + assert ( + hashlib.sha256(plain).hexdigest() == headers['X-Body-Sha256'] + ), 'round-trips byte for byte' + + +def test_php_compression_shm_not_leaked(): + """ + Regression test for #177. + + nxt_http_comp_compress_app_response() released the application's buffer + with nxt_buf_free(), a plain pool free that never runs the buffer's + completion handler. For a port-mmap buffer that handler, + nxt_port_mmap_buf_completion(), is what marks the shared memory chunks + free again, so every compressed response leaked its chunks. + + An application may hold only so much shared memory (shm_limit, 100 MB by + default), so the leak is fatal rather than merely wasteful: after enough + compressed bytes the worker can no longer allocate a buffer and the next + request never gets a response. Before the fix this wedges partway + through the loop and the request below times out. + """ + client.load('comp_random_body') + configure_compression() + + for i in range(LEAK_REQUESTS): + status, headers, body = raw_get( + f'/?n={RANDOM_BODY_SIZE}', 'gzip' + ) + + assert status == 200, f'status of request {i}' + assert headers['Content-Encoding'] == 'gzip', f'encoding, request {i}' + assert ( + hashlib.sha256(gzip.decompress(body)).hexdigest() + == headers['X-Body-Sha256'] + ), f'body of request {i}' + + +def test_php_compression_identity_refused(): + # An application response is negotiated like a static one. The client + # refuses identity and names no coding Unit can apply, so no acceptable + # representation is left and the answer is 406. + # + # nxt_router_response_ready_handler() used to read every non-NXT_OK + # result from nxt_http_comp_check_acceptable() as a server error and take + # the generic "fail:" path, which answers 503 -- a negotiation failure + # reported as the server being unable to serve anybody. + client.load('comp_large_body') + configure_compression() + + status, headers, _ = raw_get('/', 'identity;q=0') + + assert status == 406, 'no representation left for the application body' + assert headers.get('Vary') == 'Accept-Encoding', 'the 406 varies' + assert 'Content-Encoding' not in headers + + +def test_php_compression_wildcard_offers_a_coding(): + # The wildcard reaches the application path too. Sect. 12.5.3: "*" + # matches any available content coding not explicitly listed. So this + # client refuses the own bytes of the response and accepts everything + # else, gzip included. Before, "*" was read as identity only. Then + # nothing was acceptable, and the router answered 406 to a request that + # it could serve. + client.load('comp_large_body') + configure_compression() + + status, headers, body = raw_get('/', 'identity;q=0, *;q=1') + + assert status == 200, 'the wildcard offers gzip' + assert headers.get('Content-Encoding') == 'gzip', 'encoding advertised' + assert gzip.decompress(body) == b'A' * 100000, 'round-trips' + + # A coding that the field names keeps its own weight, and the wildcard + # does not match it. So gzip is worth 0.1 here, and gzip is still sent. + status, headers, body = raw_get('/', 'identity;q=0, *;q=0.5, gzip;q=0.1') + + assert status == 200, 'an explicitly named gzip is still acceptable' + assert headers.get('Content-Encoding') == 'gzip', 'at its own weight' + assert gzip.decompress(body) == b'A' * 100000 + + +def test_php_compression_identity_refused_without_compressor(): + # The same request with compression off: identity is then the only + # coding, and this client refused it. This is the path that reaches the + # check with a NULL configuration. + client.load('comp_large_body') + + status, headers, _ = raw_get('/', 'identity;q=0') + + assert status == 406, 'identity is the only available coding' + assert headers.get('Vary') == 'Accept-Encoding', 'the 406 varies' + assert 'Content-Encoding' not in headers + + +def test_php_compression_identity_refused_below_min_length(): + # gzip is acceptable but never applied: "min_length" is above this body. + # So the response would fall back to the identity that the client + # refused. The answer is 406, and still not 503. + client.load('comp_small_body') + + # client.load() installs a configuration without a "settings" object. + # So a PUT to settings/http/compression is rejected. The whole + # "settings" object is put, and the result is checked. Before, the PUT + # failed silently, and this test passed through the no-compressor path. + assert 'success' in client.conf( + { + "http": { + "compression": { + "types": ["text/html*"], + "compressors": [ + {"encoding": "gzip", "min_length": 1000} + ], + } + } + }, + 'settings', + ), 'gzip with min_length 1000' + + status, headers, _ = raw_get('/', 'gzip, identity;q=0') + + assert status == 406, 'a below-minimum gzip is not available' + assert 'Content-Encoding' not in headers + + # The same response to a client that takes identity is unchanged. + status, headers, body = raw_get('/', 'gzip') + + assert status == 200, 'identity is acceptable here' + assert 'Content-Encoding' not in headers + assert body == b'A' * 64, 'the body, uncompressed' + + +def test_php_compression_identity_refused_no_representation(): + # A 1xx, 204 or 304 describes no representation. So there is nothing + # that the client could refuse, and no 406 to give. + # + # The length exits at the top of nxt_http_comp_check_acceptable() do not + # cover this. nxt_http_response_content_length() stores the + # Content-Length field of an application without setting + # r->resp.content_length_n, which stays -1. So a 204 or 304 that + # carries the field reached the compressor selection and came back 406. + # The status and the framing that nxt_http_request_header_send() would + # correct never got there. + client.load('no_body_status') + configure_compression() + + status, headers, body = raw_get('/?status=204&cl', 'identity;q=0') + + assert status == 204, 'a 204 has no representation to refuse' + assert 'Content-Encoding' not in headers + assert body == b'', 'no body on a 204' + + status, headers, body = raw_get('/?status=304&cl', 'identity;q=0') + + assert status == 304, 'a 304 describes the body the client already has' + assert 'Content-Encoding' not in headers + assert body == b'', 'no body on a 304' + + # The same application, the same Content-Length, and a status that + # describes a representation: still 406. + status, headers, _ = raw_get('/?status=200&cl', 'identity;q=0') + + assert status == 406, 'a 200 still has a representation to refuse' + assert 'Content-Encoding' not in headers + + +def test_php_compression_identity_refused_head(): + # A HEAD carries no body either. But it still describes the + # representation that the equivalent GET would return (RFC 9110 + # Sect. 9.3.2). So it is negotiated like that GET, and a refused + # identity is a 406. That is why the exemption above tests the status + # only, and never the method, and why + # nxt_http_request_is_bodyless_final() is the wrong question here. + client.load('comp_large_body') + configure_compression() + + status, headers, _ = raw_get('/', 'identity;q=0', method='HEAD') + + assert status == 406, 'a HEAD is negotiated like the GET it stands for' + assert 'Content-Encoding' not in headers + + status, headers, _ = raw_get('/', 'gzip', method='HEAD') + + assert status == 200, 'the same HEAD from a client that takes gzip' + + +def test_php_compression_identity_refused_already_coded(): + # The application coded its response itself. That response is not + # identity, so the refusal of identity does not apply to it, and Unit + # does not negotiate it again: 200 in the coding of the application, + # without a compressor configured and with one. + client.load('comp_coded_body') + + for configured in (False, True): + if configured: + configure_compression() + + status, headers, body = raw_get('/', 'identity;q=0') + + assert status == 200, f'a coded response is not identity: {configured}' + assert headers.get('Content-Encoding') == 'gzip', 'the coding kept' + assert gzip.decompress(body) == b'A' * 100000, 'coded once' diff --git a/test/test_php_detached_max.py b/test/test_php_detached_max.py new file mode 100644 index 000000000..dd018e651 --- /dev/null +++ b/test/test_php_detached_max.py @@ -0,0 +1,598 @@ +"""processes.max must bound live workers, detached post-response work included. + +fastcgi_finish_request() reports the request done while the script keeps +running. The router takes that as the response, so the worker is counted +idle and becomes reapable while it is in fact still executing PHP. + +These tests assert the property that follows from "max" being a cap on +processes rather than on requests: however the router accounts for a worker +that is finishing detached work, the number of live PHP children of the +prototype must never exceed "max". +""" + +import os +import signal +import subprocess +import threading +import time + +import pytest + +from unit.applications.lang.php import ApplicationPHP +from unit.control import Control +from unit.option import option + +prerequisites = {'modules': {'php': 'any'}} + +client = ApplicationPHP() + +IDLE_TIMEOUT = 2 + +# Comfortably past IDLE_TIMEOUT, so "the reaper did not act" is unambiguous, +# and short enough that every test can wait for the work to finish before it +# ends. Draining matters now: a detached worker is deliberately not reaped, +# so one left running would be counted by the next test and its port +# descriptors flagged as a leak by conftest. +SLEEP_SECONDS = 6 + + +def worker_pids(name='detached_worker'): + """Live worker pids, as the children of the application's prototype. + + Same derivation as test_app_lifecycle.py: the prototype's title survives + for every runtime, and its children are the workers. A worker that has + been sent QUIT but has not exited is still a child, which is the whole + point here -- it is still a live PHP process holding memory and a slot. + """ + + output = subprocess.check_output( + ['ps', 'ax', '-o', 'pid=', '-o', 'ppid=', '-o', 'args='] + ).decode() + + rows = [p for p in (l.split(None, 2) for l in output.splitlines()) + if len(p) == 3] + + marker = f'unit: "{name}" prototype' + prototypes = {pid for pid, _, args in rows if marker in args} + + if not prototypes: + return set() + + return {pid for pid, ppid, _ in rows if ppid in prototypes} + + +def app_processes(name='detached_worker'): + """The application's "processes" object from /status.""" + + status = Control().conf_get('/status') + apps = status.get('applications', {}) + + return apps.get(name, {}).get('processes', {}) + + +def drain(timeout=30): + """Wait for detached work to finish and the workers to settle. + + Polls the router's own view rather than sleeping a fixed time: the + "detached" count is the thing under test, so waiting on it also asserts + it reaches zero on every ordinary path. + """ + + deadline = time.time() + timeout + + while time.time() < deadline: + if app_processes().get('detached', 0) == 0: + return True + + time.sleep(0.2) + + return False + + +def marker(name): + """An absolute path under the test temp dir, removed if it exists.""" + + path = f'{option.temp_dir}/{name}' + + if os.path.exists(path): + os.remove(path) + + return path + + +def test_php_detached_response_overlaps_the_script(): + """The response really does arrive while the script is still running. + + Without this the other tests could pass against a build where + fastcgi_finish_request() did nothing, because every request would simply + be serialised. + """ + + client.load( + 'detached_worker', + processes={'max': 1, 'spare': 0, 'idle_timeout': IDLE_TIMEOUT}, + ) + + start = time.time() + assert client.get(url=f'/?sleep={SLEEP_SECONDS}')['status'] == 200 + elapsed = time.time() - start + + assert elapsed < SLEEP_SECONDS / 2, ( + f'the response took {elapsed:.1f}s, so it waited for the script ' + f'rather than overlapping it' + ) + + assert len(worker_pids()) == 1, 'one worker serves the detached request' + + assert drain(), 'the detached work finished and the count cleared' + + +def test_php_detached_work_does_not_exceed_max(): + """A parked request must not add a second live PHP child under max: 1. + + The first script detaches and sleeps well past idle_timeout. A second + request arrives while it runs, so it parks. Whatever the router does + about that parked request, "max": 1 forbids a second live worker while + the first is still executing. + """ + + client.load( + 'detached_worker', + processes={'max': 1, 'spare': 0, 'idle_timeout': IDLE_TIMEOUT}, + ) + + assert client.get(url=f'/?sleep={SLEEP_SECONDS}')['status'] == 200 + + first = worker_pids() + assert len(first) == 1, 'one worker after the detached request' + + # Park a second request against the still-busy worker. It is issued from + # a thread because it will not be answered until capacity exists. + threading.Thread( + target=lambda: client.get(read_timeout=SLEEP_SECONDS + 5), + daemon=True, + ).start() + + # Past idle_start + idle_timeout, where the reaper acts on the worker it + # believes is idle. + deadline = time.time() + IDLE_TIMEOUT * 3 + 2 + peak = set() + + while time.time() < deadline: + peak |= worker_pids() + + assert len(worker_pids()) <= 1, ( + f'{len(worker_pids())} live PHP children under "max": 1 -- ' + f'a worker running detached work was replaced rather than ' + f'counted' + ) + + time.sleep(0.2) + + assert first <= peak, 'the original worker stayed alive throughout' + + assert drain(), 'the detached work finished and the count cleared' + + +def test_php_detached_completion_releases_the_worker(): + """When the detached work ends, the worker rejoins the idle economy. + + The parked request must then be served -- and served without the router + having started a second process while the first was still running. + """ + + client.load( + 'detached_worker', + processes={'max': 1, 'spare': 0, 'idle_timeout': IDLE_TIMEOUT}, + ) + + done = marker('detached_done') + short = 6 + + assert client.get(url=f'/?sleep={short}&done={done}')['status'] == 200 + + served = {} + + def parked(): + served['resp'] = client.get(read_timeout=short + 15) + + thread = threading.Thread(target=parked, daemon=True) + thread.start() + + # While the detached work runs the worker is neither idle nor reapable. + time.sleep(IDLE_TIMEOUT * 2) + + procs = app_processes() + assert procs.get('detached', 0) == 1, ( + f'the worker is not reported detached: {procs}' + ) + assert procs.get('idle', 0) == 0, f'a busy worker is reported idle: {procs}' + assert len(worker_pids()) == 1, 'still one live worker' + + thread.join(timeout=short + 20) + + assert served.get('resp', {}).get('status') == 200, ( + 'the parked request was served once capacity existed' + ) + assert os.path.exists(done), 'the detached work ran to completion' + + # The detached state clears once the script returns. + deadline = time.time() + 10 + while time.time() < deadline and app_processes().get('detached', 0) != 0: + time.sleep(0.2) + + assert app_processes().get('detached', 0) == 0, 'detached count cleared' + + +def test_php_detached_parked_request_times_out(): + """limits.timeout bounds the wait for capacity, and the request never runs. + + The second request cannot be served until the detached work ends, so a + timeout below the detached duration must answer 503 -- and the script for + that request must never execute, before or after the first one returns. + """ + + ran = marker('timed_out_ran') + timeout = 3 + + client.load( + 'detached_worker', + processes={'max': 1, 'spare': 0, 'idle_timeout': IDLE_TIMEOUT}, + limits={'timeout': timeout}, + ) + + assert client.get(url=f'/?sleep={SLEEP_SECONDS}')['status'] == 200 + + start = time.time() + resp = client.get(url=f'/?ran={ran}', read_timeout=timeout + 10) + elapsed = time.time() - start + + assert resp['status'] == 503, f'parked request timed out, got {resp}' + assert elapsed < timeout + 5, f'answered at the deadline, took {elapsed:.1f}s' + # The end-to-end half of the claim. What it cannot show is the race + # underneath it -- a worker claiming the slot in the same instant the + # timer fires -- because that window is a single CAS. That half is + # driven deterministically in src/test/nxt_router_app_timeout_test.c, + # where the test plays the worker and chooses the CAS outcome. + assert not os.path.exists(ran), 'the timed-out request did not execute' + + # And it must still not run once the detached worker becomes free. + time.sleep(3) + assert not os.path.exists(ran), ( + 'the timed-out request executed after the worker freed up' + ) + + assert drain(), 'the detached work finished and the count cleared' + + +def test_php_detached_start_after_the_worker_went_idle(): + """A detached start for a worker the router has already parked as idle. + + The start edge is sent before the last response message, but that only + orders the two on the wire. A request that is failed rather than + answered has no start edge in sight at all: "limits": {"timeout"} answers + 503 while the script is still running, and the port is held out of the + idle economy until the report arrives -- by the release, and by the + router's own mark once the acknowledgement has moved the request to the + worker. + + When the mark is in place, the edge finds the port already out and has + nothing to unwind; it records the application's own reason, and the + FINISH ends the state. The unwind is still load-bearing for the other + order: the answer's settle runs on the request engine and this edge on the + main thread, nothing orders their readers, and a settle that wins clears + the router's mark, lets the release park the port, and leaves + nxt_router_app_port_busy(..., "detached") to take it back out. Both + orders are reachable as this test runs; the assertions hold either way. + + Without that, the reaper QUITs a process that is still executing PHP -- + the exact failure this feature exists to prevent -- and clears the port's + application on the way, after which nothing settles the detached state: + nxt_port_close() skips nxt_router_app_port_close() for a port with no + application, the finish edge is ignored for the same reason, and the count + sticks for the life of the application. + """ + + timeout = 2 + idle_timeout = 6 + + # Answer after the deadline has failed the request, and well before the + # reaper's own deadline, so the start edge really does land on a port + # sitting in idle_ports. + before = timeout + 2 + + # And keep running past the reap, so a reaped worker is one that was + # killed mid-script. + after = idle_timeout + 4 + + client.load( + 'detached_worker', + processes={'max': 1, 'spare': 0, 'idle_timeout': idle_timeout}, + limits={'timeout': timeout}, + ) + + done = marker('detached_late_start') + + start = time.time() + resp = client.get( + url=f'/?before={before}&sleep={after}&done={done}', + read_timeout=before + 10, + ) + + assert resp['status'] == 503, f'the request outlived limits.timeout: {resp}' + + first = worker_pids() + assert len(first) == 1, 'one worker is running the script' + + # After the start edge, before the reaper's deadline. + while time.time() - start < before + 2: + time.sleep(0.2) + + procs = app_processes() + + assert procs.get('detached', 0) == 1, ( + f'the worker did not report itself detached: {procs}' + ) + assert procs.get('idle', 0) == 0, ( + f'a worker running detached work is still counted idle and is ' + f'reapable: {procs}' + ) + + # Past the reaper's deadline, with the script still running. + while time.time() - start < before + idle_timeout + 2: + assert len(worker_pids()) <= 1, '"max": 1 still bounds live workers' + time.sleep(0.2) + + assert worker_pids() == first, 'the worker was not replaced' + + procs = app_processes() + assert procs.get('running', 0) == 1, ( + f'the worker was reaped while it was still running: {procs}' + ) + + assert drain(timeout=after + 20), ( + 'the detached state cleared once the script returned' + ) + assert os.path.exists(done), 'the detached work ran to completion' + + +def test_php_timed_out_request_keeps_the_worker_busy(): + """A request the router gave up on must not free the worker running it. + + The script holds the worker past limits.timeout and then answers normally: + no fastcgi_finish_request(), so no detached edge ever reports that it is + still running. Nothing else tells the router either -- the deadline + answers 503 while the worker keeps executing -- so the router's own + accounting has to keep the port out of the idle economy until the answer + arrives. Released, the port is parked as idle, the reaper QUITs a worker + in the middle of a script, and the slot counts as free under "max": 1. + + The counters asserted are the router's own view of that: "detached", for a + worker it may not hand out, and "idle", which must stay empty. The "ran" + marker is appended once per execution, so it pins that the request ran + exactly once as well. + """ + + timeout = 2 + hold = 12 + + client.load( + 'detached_worker', + processes={'max': 1, 'spare': 0, 'idle_timeout': IDLE_TIMEOUT}, + limits={'timeout': timeout}, + ) + + ran = marker('timed_out_ran') + done = marker('timed_out_done') + + start = time.time() + resp = client.get( + url=f'/?ran={ran}&hold={hold}&done={done}', read_timeout=timeout + 10 + ) + elapsed = time.time() - start + + assert resp['status'] == 503, f'the request outlived limits.timeout: {resp}' + assert elapsed < timeout + 5, f'answered at the deadline, took {elapsed:.1f}s' + assert os.path.exists(ran), 'the request executed' + + # Well past idle_start + idle_timeout with the script still running: a + # port the router had released would have been reaped by now. The + # marker says the window really is open, so a slow machine fails here + # loudly instead of reading counters after the script has answered. + time.sleep(IDLE_TIMEOUT * 2) + + assert not os.path.exists(done), ( + f'the script finished within {IDLE_TIMEOUT * 2}s of the deadline; ' + f'raise hold above {hold}' + ) + + procs = app_processes() + + assert procs.get('detached', 0) == 1, ( + f'the worker running the failed request is not held out of the idle ' + f'economy: {procs}' + ) + assert procs.get('idle', 0) == 0, ( + f'a worker still running a request the router gave up on is counted ' + f'idle and is reapable: {procs}' + ) + assert procs.get('running', 0) == 1, ( + f'the worker was reaped while the script ran: {procs}' + ) + assert len(worker_pids()) == 1, '"max": 1 still bounds live workers' + + # The script answers at "hold"; the worker is free from that message on. + deadline = time.time() + hold + 20 + while time.time() < deadline and ( + app_processes().get('detached', 0) != 0 or not os.path.exists(done) + ): + time.sleep(0.2) + + assert os.path.exists(done), 'the script ran to completion' + assert app_processes().get('detached', 0) == 0, ( + f"the port was not settled by the worker's answer: {app_processes()}" + ) + + with open(ran) as f: + pids = f.read().split() + + assert len(pids) == 1, f'the request executed {len(pids)} times: {pids}' + + assert client.get()['status'] == 200, 'the worker serves again' + + +def test_php_detached_repeated_requests_hold_the_bound(): + """Repeated detached requests must not accumulate live workers. + + This is the shape that turns a brief overshoot into an unbounded one: if + each reap starts a replacement while the previous worker is still running + PHP, one extra child appears per idle_timeout for as long as the scripts + run. Spans several intervals for that reason. + """ + + client.load( + 'detached_worker', + processes={'max': 1, 'spare': 0, 'idle_timeout': IDLE_TIMEOUT}, + ) + + peak = 0 + rounds = 6 + + for _ in range(rounds): + client.get(url=f'/?sleep={SLEEP_SECONDS}', read_timeout=10) + + for _ in range(int(IDLE_TIMEOUT * 5)): + live = len(worker_pids()) + peak = max(peak, live) + + assert live <= 1, ( + f'{live} live PHP children under "max": 1 after repeated ' + f'detached requests' + ) + + time.sleep(0.2) + + assert peak == 1, f'the application kept exactly one worker, saw {peak}' + + assert drain(), 'the detached work finished and the count cleared' + + +def test_php_detached_exit_and_fatal_still_report_finished(): + """exit() and a fatal error after the response still end the detached state. + + libunit reports the work finished when the request handler returns, and + PHP reaches that return through its bailout as well as through an + ordinary return. If it did not, the worker would stay pinned out of the + idle economy for good -- a worse failure than the one this feature + exists to fix, because nothing would ever clear it. + """ + + for after in ('exit', 'fatal'): + client.load( + 'detached_worker', + processes={'max': 1, 'spare': 0, 'idle_timeout': IDLE_TIMEOUT}, + ) + + done = marker(f'detached_{after}') + + assert ( + client.get(url=f'/?sleep=2&done={done}&after={after}')['status'] + == 200 + ), f'{after}: the response went out' + + assert drain(), f'{after}: the detached state cleared' + assert os.path.exists(done), f'{after}: the detached work ran' + + # And the application still works afterwards. + assert client.get(url='/')['status'] == 200, f'{after}: still serving' + + assert drain(), f'{after}: cleared again' + + +def test_php_detached_survives_configuration_removal(): + """Removing the application while a worker runs detached work is safe. + + A detached worker holds no request, and a request is what otherwise + keeps the application alive, so the detached state has to hold its own + reference. Without it the application is freed under a running process. + The assertion the router itself makes is in nxt_router_free_app(), which + is a debug build; here the observable is that the control plane survives + and the work still completes. + """ + + client.load( + 'detached_worker', + processes={'max': 1, 'spare': 0, 'idle_timeout': IDLE_TIMEOUT}, + ) + + done = marker('detached_removed') + + assert client.get(url=f'/?sleep=4&done={done}')['status'] == 200 + + # Remove the whole configuration while the script is still running. The + # application cannot be deleted on its own while a listener still routes + # to it, so the listener goes with it. + assert 'success' in client.conf( + {'listeners': {}, 'applications': {}}, '/config' + ) + + # The control plane is still answering, and the router did not die with it. + assert 'applications' in Control().conf_get('/status') + + deadline = time.time() + 20 + while time.time() < deadline and not os.path.exists(done): + time.sleep(0.2) + + assert os.path.exists(done), 'the detached work finished after removal' + + +@pytest.mark.parametrize('spare', [0, 1]) +@pytest.mark.parametrize('remove', [False, True]) +def test_php_detached_worker_dies(skip_alert, spare, remove): + """A detached worker's last app reference survives replacement dispatch.""" + + client.load('detached_worker', processes={'max': 1, 'spare': spare}) + assert client.get(url='/?sleep=30')['status'] == 200 + + deadline = time.monotonic() + 10 + while app_processes().get('detached', 0) != 1: + assert time.monotonic() < deadline, 'the worker reported detached' + time.sleep(0.05) + + pids = worker_pids() + assert len(pids) == 1 + pid = int(pids.pop()) + skip_alert(fr'app process {pid} exited on signal 9') + + if remove: + assert 'success' in client.conf( + {'listeners': {}, 'applications': {}}, '/config' + ) + + os.kill(pid, signal.SIGKILL) + + deadline = time.monotonic() + 10 + while True: + status = Control().conf_get('/status') + apps = status.get('applications', {}) + procs = apps.get('detached_worker', {}).get('processes', {}) + if remove: + settled = 'detached_worker' not in apps and not worker_pids() + else: + settled = ( + procs.get('detached') == 0 and str(pid) not in worker_pids() + ) + if settled: + break + assert time.monotonic() < deadline, 'the dead worker was settled' + time.sleep(0.05) + + assert 'applications' in status + if remove: + assert 'detached_worker' not in status['applications'] + else: + assert client.get()['status'] == 200, 'replacement serves requests' + assert drain(), 'the replacement finished' + + # The standard fixture also checks that the router PID did not change: + # a restarted router answering /status must not hide a crash here. diff --git a/test/test_php_isolation.py b/test/test_php_isolation.py index 4d9eb21c1..e88fb0da3 100644 --- a/test/test_php_isolation.py +++ b/test/test_php_isolation.py @@ -1,7 +1,4 @@ -import pytest - from unit.applications.lang.php import ApplicationPHP -from unit.option import option prerequisites = {'modules': {'php': 'any'}, 'features': {'isolation': True}} @@ -9,9 +6,6 @@ def test_php_isolation_rootfs(is_su, require, temp_dir): - versions = option.available['modules'].get('php', []) - if versions and versions[0].split('.')[:2] >= ['8', '5']: - pytest.skip('PHP 8.5 rootfs isolation: SIGSEGV under investigation') isolation = {'rootfs': temp_dir} if not is_su: @@ -89,3 +83,174 @@ def test_php_isolation_rootfs_extensions(is_su, require, temp_dir): assert 'json' in extensions, 'json in extensions list' assert 'unit' in extensions, 'unit in extensions list' + + +def test_php_isolation_rootfs_credential_without_mount(is_su, require, + temp_dir): + if not is_su: + require( + {'features': {'isolation': ['unprivileged_userns_clone', 'user']}} + ) + else: + require({'features': {'isolation': ['user']}}) + + client.load('phpinfo') + + # A new user namespace without a new mount namespace cannot mount + # anything: the child stays in the parent's mount namespace, owned by + # the initial user namespace. The automounts default to on, so this + # config could never start; it must be refused at configuration time. + resp = client.conf( + {'rootfs': temp_dir, 'namespaces': {'credential': True}}, + 'applications/phpinfo/isolation', + ) + + assert 'error' in resp, 'credential without mount rejected' + + detail = resp.get('detail', '') + + assert 'rootfs' in detail, 'detail names rootfs' + assert 'credential' in detail, 'detail names credential' + assert 'mount' in detail, 'detail names mount' + assert 'automount' in detail, 'detail names automount' + + # The remedy must name the knobs to turn off, not just "the automount + # options". "procfs" and "tmpfs" are unconditional builtins, so both + # arms fire here whatever the module declares. + assert '"procfs": false' in detail, 'detail names the procfs knob' + assert '"tmpfs": false' in detail, 'detail names the tmpfs knob' + + # An explicit "mount": false is the same thing. + resp = client.conf( + { + 'rootfs': temp_dir, + 'namespaces': {'credential': True, 'mount': False}, + }, + 'applications/phpinfo/isolation', + ) + + assert 'error' in resp, 'explicit mount false rejected' + + # Without "credential" the config is fine -- the prototype keeps the + # main process' privileges and mounts in the parent mount namespace. + assert 'success' in client.conf( + {'rootfs': temp_dir, 'namespaces': {'credential': False}}, + 'applications/phpinfo/isolation', + ), 'no credential accepted' + + +def test_php_isolation_rootfs_credential_with_mount(is_su, require, temp_dir): + require( + { + 'features': { + 'isolation': ( + ['user', 'mnt', 'pid'] + if is_su + else ['unprivileged_userns_clone', 'user', 'mnt', 'pid'] + ) + } + } + ) + + client.load( + 'phpinfo', + isolation={ + 'rootfs': temp_dir, + 'namespaces': {'credential': True, 'mount': True, 'pid': True}, + }, + ) + + assert 'success' in client.conf( + '"/app/php/phpinfo"', 'applications/phpinfo/root' + ) + assert 'success' in client.conf( + '"/app/php/phpinfo"', 'applications/phpinfo/working_directory' + ) + + assert client.get()['status'] == 200, 'credential + mount serves' + + +def test_php_isolation_rootfs_credential_no_automount(is_su, require, + temp_dir): + require( + { + 'features': { + 'isolation': ( + ['user'] + if is_su + else ['unprivileged_userns_clone', 'user'] + ) + } + } + ) + + # With every automount disabled nothing is mounted: the rootfs switch + # falls back to chroot(2), which needs only CAP_SYS_CHROOT in the new + # user namespace. This works, so the validator must keep accepting it. + # + # "language_deps" is off here too, so the case stays valid whether or + # not the PHP module declares dependency mounts of its own. + client.load( + 'phpinfo', + isolation={ + 'rootfs': temp_dir, + 'namespaces': {'credential': True}, + 'automount': { + 'procfs': False, + 'tmpfs': False, + 'language_deps': False, + }, + }, + ) + + assert 'success' in client.conf( + '"/app/php/phpinfo"', 'applications/phpinfo/root' + ) + assert 'success' in client.conf( + '"/app/php/phpinfo"', 'applications/phpinfo/working_directory' + ) + + assert client.get()['status'] == 200, 'no automount serves' + + +def test_php_isolation_pid_namespace(is_su, require): + """A prototype in its own pid namespace still starts workers. + + The router's START_PROCESS reaches such a prototype from an ancestor pid + namespace, so the credential the kernel attaches to it is 0 rather than + the router's pid: the prototype has no pid for the router at all. The + sender check in nxt_proto_start_process_handler() (issue #270) has to + know that, and a version of it that simply compared against the router's + global pid would refuse every start here -- with the application never + coming up, which is what this asks about. + + Unlike the rootfs cases above, the pid namespace is requested whether or + not the suite runs as root: as root is exactly where the other tests do + not ask for one. Unprivileged, unshare(CLONE_NEWPID) needs CAP_SYS_ADMIN, + so a user namespace has to come with it -- the same shape as + test_go_isolation.py::test_isolation_pid. + """ + require({'features': {'isolation': ['pid']}}) + + if not is_su: + require( + { + 'features': { + 'isolation': [ + 'unprivileged_userns_clone', + 'user', + 'mnt', + ] + } + } + ) + + isolation = {'namespaces': {'pid': True}} + + if not is_su: + isolation['namespaces']['mount'] = True + isolation['namespaces']['credential'] = True + + client.load('phpinfo', isolation=isolation) + + assert client.get()['status'] == 200, 'pid-isolated prototype' diff --git a/test/test_php_protocol_lengths.py b/test/test_php_protocol_lengths.py new file mode 100644 index 000000000..6c0ac040a --- /dev/null +++ b/test/test_php_protocol_lengths.py @@ -0,0 +1,29 @@ +"""Field names and methods too long for the libunit uint8_t lengths are +refused with 431 and 501 instead of reaching PHP truncated.""" + +from unit.applications.lang.php import ApplicationPHP + +prerequisites = {'modules': {'php': 'any'}} + +client = ApplicationPHP() + + +def test_php_protocol_field_name_length(): + client.load('variables') + + # "HTTP_" + name must fit in 255 bytes. + for length, status in ((250, 200), (251, 431), (253, 431), (255, 431)): + resp = client.get(headers={'Host': 'localhost', 'X' * length: 'v'}) + assert resp['status'] == status, length + + assert client.get()['status'] == 200, 'still serving' + + +def test_php_protocol_method_length(): + client.load('variables') + + for length, status in ((255, 200), (256, 501), (280, 501)): + resp = client.http('A' * length, headers={'Host': 'localhost'}) + assert resp['status'] == status, length + + assert client.get()['status'] == 200, 'still serving' diff --git a/test/test_php_targets.py b/test/test_php_targets.py index 6085db404..ec30cf0dc 100644 --- a/test/test_php_targets.py +++ b/test/test_php_targets.py @@ -1,5 +1,8 @@ +from pathlib import Path + from unit.applications.lang.php import ApplicationPHP from unit.option import option +from unit.utils import waitforfiles prerequisites = {'modules': {'php': 'any'}} @@ -62,6 +65,145 @@ def test_php_application_targets(): assert client.get(url='/')['body'] == 'index' +def test_php_application_targets_cwd(temp_dir): + # libunit fills a released request with 0xA5 (nxt_unit_mmap_release()). + # The module read the target index after the request ended, so it kept + # 0xA5 (165) as the last target. The target at index 165 then skipped + # chdir() and ran in the directory of the previous target. The index + # follows the hash order of the names, so every target gets its own + # directory, and the test requests each of them. libunit fills released + # memory only in --debug builds, so this test fails on the old code only + # there. The test below does not depend on the fill. + routes = [] + targets = {} + + for i in range(166): + name = f't{i}' + root = f'{temp_dir}/{name}' + + Path(root).mkdir() + Path(f'{root}/index.php').write_text( + '= ['8', '5'], - } -} - -client = ApplicationPHP() - -# ────────────────────────────────────────────────────────────────────────────── -# Constants -# ────────────────────────────────────────────────────────────────────────────── - -# Seconds a worker is given to exit cleanly after the quit signal. -# Keep it generous on slow CI runners, but short enough to surface hangs. -SHUTDOWN_TIMEOUT = 15 - -# Seconds to poll before declaring a worker "failed to start". -WORKER_START_TIMEOUT = 10 - -# ────────────────────────────────────────────────────────────────────────────── -# Module-level autouse fixture — replaces per-test runtime probes -# ────────────────────────────────────────────────────────────────────────────── - -@pytest.fixture(autouse=True) -def require_trueasync(): - """ - Skip every test in this module if the TrueAsync infrastructure is absent. - - Runs before each test body (after the conftest 'run' fixture starts Unit). - Three distinct failure modes are distinguished so the skip message is - actionable and unambiguous: - - • PHP version < 8.5 → TrueAsync C code was compiled out via - #if NXT_PHP_TRUEASYNC (which configure sets to 0 when zend_async_event_t - is absent from the PHP headers). The skip message points to the version - guard, not to a missing extension. This catches any edge case where an - older PHP version slips past the prerequisites filter. - - • Config rejected with "error" → "async"/"entrypoint" fields are missing - from nxt_php_app_conf_t (TODO.md #1). The fix is in nxt_application.h. - - • Config accepted but status != 200 → nxt_php_extension.c is not built, - so the PHP worker starts but \\Unit\\Server is undefined and the entrypoint - fails to register a request callback (TODO.md #2). - - The probe config is intentionally minimal (async_shutdown fixture, one - process) to minimise side-effects. The test body is responsible for - loading whatever config it actually needs — client.conf() in the test body - will replace the probe config before any real assertions run. - """ - # ── Guard 1: PHP version ──────────────────────────────────────────────── - # TrueAsync C code is compiled only when configure detects - # zend_async_event_t in the PHP headers (NXT_PHP_TRUEASYNC=1), which - # requires PHP 8.5+. A misleading "extension not available" message - # would appear on older PHP if we went straight to the HTTP probe. - php_type = client.get_application_type() # e.g. "php 8.5.0" - match = re.search(r'(\d+)\.(\d+)', php_type) - if not match or [match.group(1), match.group(2)] < ['8', '5']: - pytest.skip( - f'TrueAsync requires PHP 8.5+ with zend_async_event_t in headers ' - f'(got {php_type!r}); C code excluded by #if NXT_PHP_TRUEASYNC' - ) - - # ── Guard 2 & 3: runtime probe ────────────────────────────────────────── - probe_conf = { - 'listeners': {'*:8080': {'pass': 'applications/_trueasync_probe'}}, - 'applications': { - '_trueasync_probe': { - 'type': client.get_application_type(), - 'processes': {'spare': 0}, - 'root': f'{option.test_dir}/php/async_shutdown', - 'async': True, - 'entrypoint': 'entrypoint.php', - } - }, - } - - r = client.conf(probe_conf) - if 'error' in r: - pytest.skip( - 'PHP async config rejected — "async"/"entrypoint" fields are ' - 'missing from nxt_php_app_conf_t (see TODO.md #1)' - ) - - if client.get()['status'] != 200: - pytest.skip( - 'PHP TrueAsync extension not available — ' - 'build nxt_php_extension.c and relink the PHP module ' - '(see TODO.md #2)' - ) - - yield # test body runs here; probe config is replaced by the test itself - - -# ────────────────────────────────────────────────────────────────────────────── -# Helpers -# ────────────────────────────────────────────────────────────────────────────── - -def _async_app_conf(script: str) -> dict: - """ - Return a Unit application config dict for TrueAsync mode. - - "async": true + "entrypoint" is the combination that activates - nxt_php_quit_handler registration inside nxt_php_start(). - """ - return { - 'type': client.get_application_type(), - 'processes': {'spare': 0}, - 'root': f'{option.test_dir}/php/{script}', - 'async': True, - 'entrypoint': 'entrypoint.php', - } - - -def _worker_pids(app_name: str, unit_pid: int) -> list: - """ - Return PIDs of live worker processes for *app_name*. - - Unit names application workers after the application name, so we - match the Unit master PID as ppid and the app name in the command. - """ - output = subprocess.check_output(['ps', 'ax', '-O', 'ppid']).decode() - return re.findall( - fr'\s*(\d+)\s+{unit_pid}\s+.*{re.escape(app_name)}', - output, - ) - - -def _wait_for_worker(app_name: str, unit_pid: int, - timeout: float = WORKER_START_TIMEOUT) -> list: - """ - Poll until at least one worker for *app_name* appears in /proc. - Raises AssertionError if no worker appears within *timeout* seconds. - """ - deadline = time.monotonic() + timeout - while time.monotonic() < deadline: - pids = _worker_pids(app_name, unit_pid) - if pids: - return pids - time.sleep(0.1) - raise AssertionError( - f'Worker for {app_name!r} did not start within {timeout}s' - ) - - -def _workers_gone(pids: list, timeout: float = SHUTDOWN_TIMEOUT) -> bool: - """ - Return True once every PID in *pids* has vanished from /proc. - Return False if any remain alive after *timeout* seconds. - - Checking /proc/{pid} is more reliable than parsing 'ps' output: - the entry disappears exactly when the kernel reaps the process, - with no race between the read and the process exiting. - """ - deadline = time.monotonic() + timeout - while time.monotonic() < deadline: - if all(not os.path.exists(f'/proc/{p}') for p in pids): - return True - time.sleep(0.1) - return False - - -def _wait_for_file(path: str, timeout: float = WORKER_START_TIMEOUT) -> bool: - """ - Poll until *path* exists on disk (written by the PHP handler as a - start sentinel) or *timeout* elapses. Return True if found in time. - """ - deadline = time.monotonic() + timeout - while time.monotonic() < deadline: - if os.path.exists(path): - return True - time.sleep(0.05) - return False - - -# ────────────────────────────────────────────────────────────────────────────── -# Tests -# ────────────────────────────────────────────────────────────────────────────── - - -def test_php_trueasync_worker_starts(unit_pid): - """ - Smoke test: a PHP worker in TrueAsync mode starts and serves requests. - - This is the minimal sanity check for every other test. If it fails - after the require_trueasync fixture passed, the most likely cause is - a config field being silently ignored rather than raising an error - (e.g. "async" accepted but not wired to c->async in nxt_php_start). - """ - assert 'success' in client.conf( - { - 'listeners': {'*:8080': {'pass': 'applications/async_shutdown'}}, - 'applications': {'async_shutdown': _async_app_conf('async_shutdown')}, - } - ), 'configuration accepted' - - pids = _wait_for_worker('async_shutdown', unit_pid) - assert pids, 'at least one worker process is running' - - resp = client.get() - assert resp['status'] == 200, 'worker responds with 200' - assert 'OK' in resp['body'], 'response body from async handler' - - -def test_php_trueasync_graceful_shutdown_on_reconfigure(unit_pid, wait_for_record): - """ - Core regression test for commit 58ffa236. - - Scenario - -------- - 1. Start a PHP worker in TrueAsync mode. - 2. Confirm it is alive and serving. - 3. Remove the application — Unit's router sends the quit signal to - the worker, exercising nxt_php_quit_handler() → ZEND_ASYNC_SHUTDOWN(). - 4. Assert the worker exits within SHUTDOWN_TIMEOUT seconds. - - Before the fix the worker blocked in nxt_unit_run() indefinitely - because the scheduler never received a shutdown trigger. - """ - assert 'success' in client.conf( - { - 'listeners': {'*:8080': {'pass': 'applications/async_app'}}, - 'applications': {'async_app': _async_app_conf('async_shutdown')}, - } - ), 'load async_shutdown configuration' - - # Capture PIDs before reconfiguration so we can track these specific - # processes — not just any worker that might appear later. - pids = _wait_for_worker('async_app', unit_pid) - - assert client.get()['status'] == 200, 'worker alive before reconfigure' - - # Remove the application; Unit will deliver the quit signal. - assert 'success' in client.conf( - {'listeners': {}, 'applications': {}} - ), 'empty configuration accepted' - - assert _workers_gone(pids), ( - f'TrueAsync worker(s) {pids} did not exit within {SHUTDOWN_TIMEOUT}s ' - f'— nxt_php_quit_handler may not have fired, or ' - f'ZEND_ASYNC_SHUTDOWN() did not stop the scheduler' - ) - - # Informational: verify Unit logged the quit handler invocation. - # Only present when Unit is built with debug logging; not a hard failure. - wait_for_record(r'TrueAsync: quit handler called', wait=5) - - -def test_php_trueasync_graceful_shutdown_on_sigquit(unit_pid, skip_alert): - """ - Graceful shutdown when SIGQUIT is delivered to the Unit master. - - This is the normal production shutdown path: systemd ExecStop, - container runtime "stop", or NGINX-style graceful reload all use SIGQUIT. - The signal path is: SIGQUIT → master → router → quit callback → ZEND_ASYNC_SHUTDOWN. - - skip_alert suppresses the "exited on signal" log alert that would - otherwise cause the test-teardown checker to flag a failure. - """ - assert 'success' in client.conf( - { - 'listeners': {'*:8080': {'pass': 'applications/async_app'}}, - 'applications': {'async_app': _async_app_conf('async_shutdown')}, - } - ), 'load configuration' - - pids = _wait_for_worker('async_app', unit_pid) - - assert client.get()['status'] == 200, 'worker alive' - - for pid in pids: - skip_alert(fr'process {pid} exited on signal') - - os.kill(unit_pid, signal.SIGQUIT) - - assert _workers_gone(pids), ( - f'TrueAsync worker(s) did not exit within {SHUTDOWN_TIMEOUT}s ' - f'after SIGQUIT → Unit master (pid={unit_pid})' - ) - - -def test_php_trueasync_no_zombie_after_shutdown(unit_pid, skip_alert): - """ - Exited TrueAsync workers must be reaped — no zombies. - - A zombie means the master is not calling waitpid() after the quit - handler returns. This is a process-management regression independent - of TrueAsync, but worth asserting here alongside the shutdown tests. - """ - assert 'success' in client.conf( - { - 'listeners': {'*:8080': {'pass': 'applications/async_app'}}, - 'applications': {'async_app': _async_app_conf('async_shutdown')}, - } - ), 'load configuration' - - pids = _wait_for_worker('async_app', unit_pid) - for pid in pids: - skip_alert(fr'process {pid} exited on signal') - - assert 'success' in client.conf( - {'listeners': {}, 'applications': {}} - ), 'empty configuration to trigger quit' - - assert _workers_gone(pids), 'workers exited before zombie check' - - out = subprocess.check_output( - ['ps', 'ax', '-o', 'state', '-o', 'ppid'] - ).decode() - zombie_ppids = re.findall(r'Z\s+(\d+)', out) - assert str(unit_pid) not in zombie_ppids, ( - 'No zombie processes parented to Unit master after async worker shutdown' - ) - - -def test_php_trueasync_standard_mode_unaffected(): - """ - Regression guard: nxt_php_sapi.c changes must not break standard PHP. - - Standard mode uses nxt_php_request_handler (not the async variant). - nxt_php_quit_handler is NOT registered (php_init.callbacks.quit = NULL) - and the pre-existing shutdown path is used unchanged. - """ - # Load the plain mirror fixture — no async config involved. - client.load('mirror') - - resp = client.post(body='hello', headers={'Content-Length': '5'}) - assert resp['status'] == 200, 'standard mode: POST succeeds' - assert resp['body'] == 'hello', 'standard mode: body echoed' - - assert client.get()['status'] == 200, 'standard mode: GET succeeds' - - -def test_php_trueasync_mirror_request_body(unit_pid): - """ - Functional test: async handler reads and echoes the request body. - - Exercises nxt_php_request_handler_async(), - nxt_php_scope_populate_superglobals(), and the response-write path - including nxt_php_drain_queue under backpressure. - """ - assert 'success' in client.conf( - { - 'listeners': {'*:8080': {'pass': 'applications/async_mirror'}}, - 'applications': {'async_mirror': _async_app_conf('async_mirror')}, - } - ), 'load async_mirror configuration' - - _wait_for_worker('async_mirror', unit_pid) - - payload = 'TrueAsync echo test payload' - resp = client.post( - body=payload, - headers={'Content-Length': str(len(payload))}, - ) - assert resp['status'] == 200, 'async mirror: status 200' - assert resp['body'] == payload, 'async mirror: body echoed correctly' - - -def test_php_trueasync_multiple_workers_all_exit(unit_pid, skip_alert): - """ - All worker processes must exit gracefully on shutdown, not just one. - - Each worker runs nxt_php_start() independently, so nxt_php_quit_handler - must be registered per-worker (not once globally). ZEND_ASYNC_SHUTDOWN() - must stop each worker's own scheduler instance. - """ - conf = _async_app_conf('async_shutdown') - conf['processes'] = 2 # request two workers explicitly - - assert 'success' in client.conf( - { - 'listeners': {'*:8080': {'pass': 'applications/async_multi'}}, - 'applications': {'async_multi': conf}, - } - ), 'load multi-worker configuration' - - # Wait until both workers are visible in the process table. - deadline = time.monotonic() + WORKER_START_TIMEOUT - pids = [] - while time.monotonic() < deadline: - pids = _worker_pids('async_multi', unit_pid) - if len(pids) >= 2: - break - time.sleep(0.1) - - assert len(pids) >= 2, ( - f'Expected ≥2 TrueAsync workers, got {len(pids)}: {pids}' - ) - - for pid in pids: - skip_alert(fr'process {pid} exited on signal') - - assert 'success' in client.conf( - {'listeners': {}, 'applications': {}} - ), 'empty config to stop all workers' - - still_alive = [p for p in pids if os.path.exists(f'/proc/{p}')] - assert _workers_gone(pids), ( - f'Not all TrueAsync workers exited within {SHUTDOWN_TIMEOUT}s. ' - f'Still alive: {still_alive}' - ) - - -def test_php_trueasync_inflight_request_completes(unit_pid, skip_alert, tmp_path): - """ - In-flight requests must complete before the worker exits on shutdown. - - Approach - -------- - 1. Start the slow entrypoint (?sleep=2). The handler writes a sentinel - file to *signal_file* the moment it begins executing — before sleeping. - 2. Poll *signal_file* instead of using time.sleep(): once the file exists - the handler is definitively in-flight inside the PHP coroutine. - 3. Issue the reconfigure (removes the app, triggers the quit signal). - 4. Wait for curl to finish and assert the response is 200 "done". - - The slow request is made via curl in a subprocess rather than a thread: - subprocess.Popen avoids GIL contention and gives an independent TCP - connection not shared with the client used for conf() calls. - - NOTE: This test asserts cooperative-shutdown semantics — that - ZEND_ASYNC_SHUTDOWN() lets active coroutines run to completion. - If the TrueAsync scheduler cancels coroutines on shutdown instead, - this test will fail with a connection error or non-200 status; see - TODO.md #4 for the discussion and resolution path. - """ - assert 'success' in client.conf( - { - 'listeners': {'*:8080': {'pass': 'applications/async_slow'}}, - 'applications': {'async_slow': _async_app_conf('async_slow')}, - } - ), 'load async_slow configuration' - - pids = _wait_for_worker('async_slow', unit_pid) - for pid in pids: - skip_alert(fr'process {pid} exited on signal') - - # Build the sentinel file path in /tmp rather than in pytest's tmp_path. - # - # tmp_path is owned by the user running pytest (typically root in CI), - # but the Unit PHP worker runs under its own user (e.g. "unit"). The - # worker process would get EACCES writing to a 0o700 directory it does - # not own. /tmp is world-writable (mode 1777) on every POSIX system, - # so no chmod dance is needed. - # - # We include the pytest tmp_path basename (unique per-test) to avoid - # collisions when multiple test runs overlap, and clean up explicitly - # at the end of the test. - signal_file = os.path.join( - tempfile.gettempdir(), - f'unit_async_{os.path.basename(str(tmp_path))}', - ) - - # Launch curl in a subprocess: one independent TCP connection, - # no GIL contention, clean resource lifetime via context manager. - url = f'http://localhost:8080/?sleep=2&signal={signal_file}' - with subprocess.Popen( - ['curl', '-s', '--max-time', '25', url], - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, - ) as proc: - - # Wait for the PHP handler to create the sentinel file. - # This is the authoritative signal that the coroutine is sleeping - # inside the handler — not just enqueued in the kernel or router. - if not _wait_for_file(signal_file): - proc.terminate() - pytest.fail( - f'Slow request did not reach the PHP handler within ' - f'{WORKER_START_TIMEOUT}s (sentinel file not created). ' - f'Check that async_slow/entrypoint.php writes ?signal= path.' - ) - - # The request is now definitively in-flight. Trigger shutdown. - assert 'success' in client.conf( - {'listeners': {}, 'applications': {}} - ), 'reconfigure while request is in flight' - - # Wait for curl to finish; generous timeout covers sleep=2 plus - # scheduler shutdown overhead. - try: - stdout, _ = proc.communicate(timeout=30) - except subprocess.TimeoutExpired: - proc.kill() - pytest.fail('curl did not finish within 30s after shutdown trigger') - - body = stdout.decode() - - assert proc.returncode == 0, ( - f'curl exited with code {proc.returncode} — connection was dropped ' - f'before the response arrived. ZEND_ASYNC_SHUTDOWN() may cancel ' - f'in-flight coroutines rather than letting them complete (TODO.md #4).' - ) - assert 'done' in body, ( - f'Unexpected response body: {body!r}. ' - f'Expected "done" from async_slow handler.' - ) - - assert _workers_gone(pids), ( - f'Worker(s) did not exit within {SHUTDOWN_TIMEOUT}s after the ' - f'in-flight request completed.' - ) - - # Remove the sentinel file written by the PHP worker. It lives in /tmp - # (not in pytest's tmp_path) so pytest cannot clean it up automatically. - try: - os.unlink(signal_file) - except FileNotFoundError: - pass # handler may not have written it if the test failed early diff --git a/test/test_process_abrupt_teardown.py b/test/test_process_abrupt_teardown.py new file mode 100644 index 000000000..c7f92f369 --- /dev/null +++ b/test/test_process_abrupt_teardown.py @@ -0,0 +1,318 @@ +"""Abrupt application death while large requests are in flight. + +What this drives: router worker engines on several event engines +(``settings/listen_threads``) moving 1 MiB request and response bodies through +an application's shared memory, while the test repeatedly ``kill -9``s the one +process serving that application. The application is configured with a single +process, so every kill leaves the router with no worker for the application and +forces a respawn, and every kill lands on a process that is mid-transfer. + +Why that matters here: this is the *abrupt* teardown path, and it is a +different one from the graceful, configuration-driven teardown that +test_process_teardown_churn.py drives. A killed worker is noticed by the main +process' SIGCHLD handler, which broadcasts NXT_PORT_MSG_REMOVE_PID; the router +handles it in ``nxt_router_remove_pid_handler()`` -> ``nxt_port_remove_pid()`` +(src/nxt_port.c). That function was converted from ``nxt_runtime_process_find()`` +to ``nxt_runtime_process_ref()``, and it is the site where review found a double +free: it hands its result to ``nxt_process_close_ports()``, which takes its own ++1/-1 around the port loop, so on a process a router worker had already dropped +to zero that pair is a second 0 -> 1 -> 0 transition and so a second teardown, +racing the one already posted. Nothing else in the suite reaches that site with +shared-memory traffic in flight: test_respawn.py kills an idle worker, and the +churn test never kills anything. + +What this proves and what it does not: a green run is *weak* evidence for the +absence of the double free. The window is a handful of instructions wide and +an uninstrumented build will usually miss it. The value of this test is as a +crash and ASan/UBSan *driver* -- run under ``.github/workflows/sanitize.yml`` +instrumentation, a second teardown of the same process becomes a hard failure. +Read a pass as "workers died hard under load, the router noticed each one, and +the daemon kept serving correctly", nothing stronger. + +Correctness assertions are written so that a request legitimately failed by a +kill is not a failure, but a corrupt or hung one is. About two seconds of +workload: six kills, at least 0.3 s apart, against four request threads. Each +round waits for a worker the run has not killed before, and then for the signal +to have actually removed it, so all six are distinct abrupt teardowns rather +than repeat shots at one corpse. +""" + +import subprocess +import threading +import time + +import pytest + +from unit.applications.lang.php import ApplicationPHP +from unit.log import Log +from unit.shm_body import describe_mismatch, make_body + +prerequisites = {'modules': {'php': 'all'}} + +client = ApplicationPHP() + +APP = 'mirror' + +# Same reasoning as test_process_teardown_churn.py: 1 MiB spans 64 chunks of +# shared memory in each direction (PORT_MMAP_CHUNK_SIZE is 16 KiB, +# src/nxt_port_memory_int.h), so a request is inside the chunk allocator, and +# the process/port lookups that reach it, for long enough that a kill on +# another thread lands in the middle of one. +# +# The bytes are not uniform: every request carries a fresh body stamped, every +# 64 bytes, with that request's tag and the byte offset of the record (see +# unit/shm_body.py). WORKERS requests are in shared memory at once here, so a +# chunk recycled out from under one of them most likely holds another one's +# payload; against 'x' * BODY_SIZE that chunk is byte-identical to what +# belongs there and the check below cannot see it. +BODY_SIZE = 1024 * 1024 + +# > 1 so more than one router worker engine is live; this is a top-level +# "settings" member (src/nxt_conf_validation.c: nxt_conf_vldt_setting_members). +LISTEN_THREADS = 4 + +WORKERS = 4 +KILLS = 6 +KILL_INTERVAL = 0.3 + +# Not the suite default (60s), so that a stalled request returns instead of +# calling pytest.fail() on a helper thread; the stall is detected by timing it +# and reported as a hard failure below. +READ_TIMEOUT = 10 + + +def _app_pids(): + """PIDs currently serving the application, via ps (as conftest does).""" + out = subprocess.check_output( + ['ps', '-ax', '-o', 'pid', '-o', 'command'] + ).decode() + + marker = f'unit: "{APP}" application' + + return {line.split()[0] for line in out.splitlines() if marker in line} + + +def _wait_for_fresh_app_pids(spent, timeout=5): + """Wait for a worker that is not one already killed, so every kill in the + loop below lands on a live process instead of on the gap left by the + previous one -- or, worse, on the corpse of it. + + Excluding `spent` rather than just waiting for a non-empty set is what + makes each iteration a distinct abrupt teardown. A SIGKILLed worker + leaves `_app_pids()` promptly, because the marker is the setproctitle + string in /proc//cmdline and a zombie has no cmdline, so ps falls + back to "[comm] " -- but "promptly" is a scheduling property, + and this test exists to be run on loaded sanitizer machines. Making the + loop wait for a *new* pid turns that timing assumption into something the + test checks.""" + + deadline = time.time() + timeout + + while time.time() < deadline: + pids = _app_pids() - spent + + if pids: + return pids + + time.sleep(0.05) + + return set() + + +def _wait_for_pids_gone(pids, timeout=5): + """Wait until none of `pids` serves the application any more, i.e. until + the SIGKILL has actually taken a worker away rather than merely being + queued on one.""" + + deadline = time.time() + timeout + + while time.time() < deadline: + if not (_app_pids() & pids): + return True + + time.sleep(0.05) + + return False + + +def test_process_abrupt_teardown(skip_alert, skip_fds_check): + # Killing workers leaves descriptor accounting unsettled in all three + # long-lived processes for as long as the respawn takes, which is not what + # this test is about. + skip_fds_check(main=True, router=True, controller=True) + + # Expected, and in fact asserted on below: this is how the main process + # reports each kill. + skip_alert(r'app process \d+ exited on signal 9') + + # Exactly one process, so a kill always removes the only worker and the + # router has to tear the process down rather than route around it, and so + # every request thread contends on the same incoming.mutex. + client.load(APP, processes={"spare": 1, "max": 1, "idle_timeout": 30}) + + assert 'success' in client.conf( + {"listen_threads": LISTEN_THREADS}, 'settings' + ), 'listen_threads' + + assert client.post(body='ping', read_timeout=READ_TIMEOUT)['status'] == 200 + + stop = threading.Event() + lock = threading.Lock() + failures = [] + outcomes = [] + killed = [] + + def record(bucket, item): + with lock: + bucket.append(item) + + def request_worker(): + while not stop.is_set(): + expected = make_body(BODY_SIZE) + started = time.time() + + try: + resp = client.post( + body=expected, + read_timeout=READ_TIMEOUT, + read_buffer_size=65536, + ) + + except KeyboardInterrupt: + raise + + # pytest.fail() inside the helpers raises BaseException + except BaseException as exc: # noqa: BLE001 + # The listener stays open across a worker death, so a refused + # or reset connection is still not expected; record it and let + # the assertions below decide. + record(failures, f'request raised {exc!r}') + continue + + elapsed = time.time() - started + status = resp.get('status') + body = resp.get('body', '') + + if status == 200: + # A short body is legitimate: the worker can die after its + # headers are on the wire. Wrong *bytes* are not - that is + # what reading freed or recycled shared memory looks like. + if body != expected[: len(body)]: + # describe_mismatch() reports the first wrong byte and the + # record stamp found there, so a failure says whether the + # bytes came from another offset of this request or from + # another request entirely. It keeps the fail-open guard + # too: the overlong-but-correct-prefix shape has no + # mismatching byte, and a bare next() would raise + # StopIteration here, outside the try that guards the + # request, killing this thread instead of recording the + # corruption. + record( + failures, + f'corrupt body: {len(body)} bytes, ' + + describe_mismatch(body, expected), + ) + continue + + record(outcomes, 200 if len(body) == BODY_SIZE else 'truncated') + + elif status is None: + # No parsable response: connection closed as the worker died. + # Distinguish that from a hang, which is a real defect. + if elapsed >= READ_TIMEOUT: + record(failures, f'no response in {elapsed:.1f}s') + else: + record(outcomes, 'closed') + + elif status in (500, 503): + record(outcomes, status) + + else: + record(failures, f'unexpected status {status}') + + workers = [threading.Thread(target=request_worker) for _ in range(WORKERS)] + + for worker in workers: + worker.start() + + for round_ in range(KILLS): + pids = _wait_for_fresh_app_pids(set(killed)) + + if not pids: + record( + failures, f'no fresh application worker for kill {round_ + 1}' + ) + break + + subprocess.call(['kill', '-9', *pids]) + + # Only count the round once the signal has demonstrably removed those + # workers. Otherwise a kill that has not landed yet would leave the + # next iteration looking at the same process, and the run would lose + # abrupt-teardown coverage without losing a single assertion. + if not _wait_for_pids_gone(pids): + record(failures, f'kill -9 did not remove {sorted(pids)}') + break + + killed.extend(pids) + + time.sleep(KILL_INTERVAL) + + stop.set() + + for worker in workers: + worker.join(timeout=120) + assert not worker.is_alive(), 'request thread hung' + + assert not failures, f'{len(failures)} failure(s), first: {failures[0]}' + + # The daemon is still alive and still serves the whole payload correctly. + expected = make_body(BODY_SIZE) + + resp = client.post( + body=expected, read_timeout=READ_TIMEOUT, read_buffer_size=65536 + ) + + assert resp['status'] == 200, 'still serving' + assert len(resp['body']) == BODY_SIZE, 'still serving whole body' + assert resp['body'] == expected, 'still serving intact body' + + # Requests must not have been shut out entirely, or nothing was in flight + # when a worker died. + assert outcomes.count(200) > 0, f'no request completed: {outcomes[:20]}' + + # Every round killed a worker that had never been killed before, and the + # loop ran all KILLS of them -- so the router really did tear a process + # down and build a new one, repeatedly, and not two or three times with + # the rest of the rounds shooting at the same corpse. The loop enforces + # the first property; assert it anyway, since it is the coverage claim. + assert len(set(killed)) == len(killed), ( + f'a worker was killed twice: {killed}' + ) + assert len(killed) >= KILLS, ( + f'only {len(killed)} worker(s) killed: {killed}' + ) + + log = Log.read() + + # Available at any log level - this is the alert the main process emits + # from its SIGCHLD handler, and the same handler is what broadcasts + # NXT_PORT_MSG_REMOVE_PID to the router. One per kill. + for pid in set(killed): + assert f'app process {pid} exited on signal 9' in log, ( + f'main process did not report the death of {pid}' + ) + + # The direct evidence - "port remove pid handler" is logged by + # nxt_port_remove_pid() itself - is nxt_debug(), compiled out unless Unit + # was configured with --debug. Skip loudly rather than let the test + # degrade into a smoke test that never checks the site it names. + if '[debug]' not in log: + pytest.skip('remove_pid evidence needs a --debug build') + + for pid in set(killed): + assert f'port remove pid {pid} handler' in log, ( + f'nxt_port_remove_pid() did not run for {pid}' + ) + + Log.check_alerts() diff --git a/test/test_process_shm_oosm.py b/test/test_process_shm_oosm.py new file mode 100644 index 000000000..b9642f458 --- /dev/null +++ b/test/test_process_shm_oosm.py @@ -0,0 +1,337 @@ +"""Router handling of an application that runs out of shared memory. + +What this drives: a PHP application whose shared-memory budget +(``applications//limits/shm``) is one segment, writing a response several +times larger than that segment. Every write goes through +``nxt_unit_response_write()``, which allocates out of the application's +outgoing segments, and those chunks stay busy until the router has written them +to the client and completed the buffers. The client stalls before reading, so +the queued response keeps the segment's free map full while the application +still has output left; ``nxt_unit_mmap_get()`` (src/nxt_unit.c) then reports +the out-of-shared-memory condition to the router and blocks until it is +acknowledged. + +Why that matters here: the OOSM message is the only thing that reaches +``nxt_router_oosm_handler()`` (src/nxt_router.c), and the ACK the application +waits for is the only thing that reaches ``nxt_process_broadcast_shm_ack()`` +(src/nxt_port_memory.c), which the router sends from the ``hdr->oosm`` branch +of ``nxt_port_mmap_buf_completion()`` as it frees the chunks. All three look +the peer process up by pid, and all three were converted from +``nxt_runtime_process_find()``, which returns a process without a reference, to +``nxt_runtime_process_ref()``. Nothing else in this suite sends an OOSM +message, so before this test those conversions had no coverage at all. + +Which engine runs what: ``nxt_unit_send_oosm()`` always sends to +``lib->router_port``, so in practice ``nxt_router_oosm_handler()`` runs on the +router's *main* engine -- confirmed by the thread id in its log record -- even +though ``.oosm`` is in the worker port handler table too. The reference is +load-bearing anyway, because the engine that runs the handler is not the only +one manipulating the process refcount: with ``listen_threads`` above one, the +worker engines take and drop references of their own on every shared-memory +message, and one of those drops can be the last. ``nxt_port_mmap_buf_completion()`` +does run on a worker engine, since that is where the response was written out. + +What this proves and what it does not: it proves the code ran -- the assertions +are on log records emitted from inside those handlers, not on a response that +would look identical either way. It does *not* prove the absence of the +use-after-free they were converted to fix; reaching the handler is necessary +for that race, not sufficient, and an uninstrumented build will usually survive +a live one. Read a pass as "the OOSM/ACK round trip happened and the daemon +stayed correct". The value of the test is as a driver for the sanitiser legs, +where a stale process pointer becomes a hard failure instead of a coin flip. + +Cheap on purpose: one request and one deliberate client stall, about a second +of work once the PHP worker is warm. +""" + +import subprocess +import threading +import time + +import pytest + +from unit.applications.lang.php import ApplicationPHP +from unit.log import Log + +prerequisites = {'modules': {'php': 'all'}} + +client = ApplicationPHP() + +APP = 'big_response' + +# One segment is PORT_MMAP_DATA_SIZE == 10 MiB (src/nxt_port_memory_int.h), and +# nxt_unit_init() rounds "limits"/"shm" up to whole segments with a floor of +# one, so any value at or below 10 MiB leaves the application a single segment. +SHM_LIMIT = 1 + +# Comfortably past that 10 MiB even after the kernel socket buffers have +# absorbed their share of the response. Measured on this tree: 12 MiB produces +# no OOSM at all, because the socket buffers alone hold the overflow; 16 MiB +# produces 1-9 messages depending on timing; 24 MiB produced 12-14 on every +# run. Sized for the margin, not for the minimum. +RESPONSE_MB = 24 +RESPONSE_SIZE = RESPONSE_MB * 1024 * 1024 + +# The client sends, then does not read for this long, so the router's queued +# response buffers - which are the application's shared memory, busy until the +# write to the client completes - pile up instead of draining. +STALL = 0.3 + +# > 1 so router worker engines exist and are taking and dropping their own +# references on the same process while the OOSM round trip runs. +LISTEN_THREADS = 4 + +READ_TIMEOUT = 30 + + +def _body(resp, size): + """Trailing `size` bytes of a raw response, i.e. the body: the application + sets Content-Length and the connection is not chunked. Kept as bytes so a + 24 MiB response is not decoded into a second copy.""" + + return resp[-size:] + + +def test_process_shm_oosm(): + client.load(APP, limits={"shm": SHM_LIMIT}, processes=1) + + assert 'success' in client.conf( + {"listen_threads": LISTEN_THREADS}, 'settings' + ), 'listen_threads' + + sock = client.get(url=f'/?mb={RESPONSE_MB}', no_recv=True) + + time.sleep(STALL) + + resp = client.recvall(sock, read_timeout=READ_TIMEOUT, buff_size=65536) + sock.close() + + body = _body(resp, RESPONSE_SIZE) + + assert len(body) == RESPONSE_SIZE, 'whole response' + assert body.count(b'x') == RESPONSE_SIZE, 'intact response' + + log = Log.read() + + # The records asserted on below are nxt_debug()/nxt_unit_debug(), compiled + # out unless Unit was configured with --debug. Skip loudly rather than + # let the test degrade into a smoke test that proves nothing about the + # path it claims to cover. The sanitiser workflow builds --debug, which + # is the leg this test exists for. + if '[debug]' not in log: + pytest.skip('OOSM evidence needs a --debug build') + + # nxt_router_oosm_handler() logs this on entry, immediately before the + # reference it now takes; its presence is the proof that the converted + # call site ran. + assert 'oosm in ' in log, 'router handled an OOSM message' + + # The application logs "oosm: retry" only after nxt_unit_wait_shm_ack() + # has returned OK, and that happens only on an NXT_PORT_MSG_SHM_ACK + # message (src/nxt_unit.c). The router's only sender of that message is + # nxt_port_broadcast_shm_ack(), reached through the converted + # nxt_process_broadcast_shm_ack() -- so this record is proof that the + # cross-engine post, and the reference it now carries, ran too. + # + # Which of the two callers broadcast is deliberately not asserted: with + # the segment this full, nxt_router_oosm_handler() almost never finds a + # free chunk to acknowledge with ("oosm: already free" is absent from + # these runs) and the ACK comes from nxt_port_mmap_buf_completion(). + assert 'oosm: retry' in log, 'application got a SHM_ACK back' + + # Still serving, from the same worker, after the round trip. + sock = client.get(url='/?mb=1', no_recv=True) + resp = client.recvall(sock, read_timeout=READ_TIMEOUT, buff_size=65536) + sock.close() + + assert _body(resp, 1024 * 1024).count(b'x') == 1024 * 1024, ( + 'still serving' + ) + + Log.check_alerts() + + +# --------------------------------------------------------------------------- +# The OOSM ack path against an application that is being torn down (issue #195) +# --------------------------------------------------------------------------- + +# Kills, and the gap between them. Each one has to land while the segment is +# still full and clients are still stalled, which is what puts a SHM_ACK +# broadcast and the port teardown of the process it is aimed at on two engines +# at once. +KILLS = 5 +KILL_INTERVAL = 0.4 + +# Stalled readers, so the response buffers -- the application's shared memory +# -- stay busy for the whole run instead of draining between kills. +STALLERS = 3 + +# Bounded on purpose: this is a race driver, not a soak. Five kills at 0.4 s +# is about three seconds of workload, cheap enough for every leg including the +# sanitiser ones. It is timing-dependent by nature: a green run is evidence +# only in the same weak sense as test_process_abrupt_teardown.py, and the +# deterministic check of the primitive this test exercises lives in the C +# suite (src/test/nxt_port_use_unless_zero_test.c). + + +def _oosm_app_pids(): + """PIDs currently serving the application, via ps (as conftest does).""" + out = subprocess.check_output( + ['ps', '-ax', '-o', 'pid', '-o', 'command'] + ).decode() + + marker = f'unit: "{APP}" application' + + return {line.split()[0] for line in out.splitlines() if marker in line} + + +def _wait_for_fresh_pids(spent, timeout=5): + """A worker this run has not killed yet, so every kill is a distinct + abrupt teardown rather than a repeat shot at a corpse. Same reasoning as + test_process_abrupt_teardown.py.""" + + deadline = time.time() + timeout + + while time.time() < deadline: + pids = _oosm_app_pids() - spent + + if pids: + return pids + + time.sleep(0.05) + + return set() + + +def test_process_shm_oosm_kill(skip_alert): + """Kill the OOSM-blocked worker while the router is acknowledging it. + + What this drives: the same one-segment application as above, but with + several clients stalled at once so the segment stays full, and the worker + SIGKILLed underneath them. The router then has two things happening on + two engines: on a worker engine, + ``nxt_port_mmap_buf_completion()`` (src/nxt_port_memory.c) drains a + completed response buffer, wins ``cmp_set(&hdr->oosm, 1, 0)`` and calls + ``nxt_process_broadcast_shm_ack()``; on the router's main thread, the + SIGCHLD-driven ``nxt_port_remove_pid()`` -> ``nxt_process_close_ports()`` + has dropped the app port's references and ``nxt_port_release()`` + (src/nxt_port.c) is destroying its memory pool. + + ``nxt_process_broadcast_shm_ack()`` read the app port out of + ``process->ports`` as a bare pointer and handed it to ``nxt_port_post()``, + whose unconditional ``nxt_atomic_fetch_add(&port->use_count, 1)`` lands on + a port whose count is already zero: the ``port->use_count == 0`` assertion + in ``nxt_port_mp_cleanup()`` in a --debug build, and a use-after-free of + the port and of the posted work item in a release one. The fix takes the + port with ``nxt_port_use_unless_zero()`` under ``rt->processes_mutex``, + which ``nxt_port_release()`` now unlinks under. + + What this proves and what it does not: on a --debug build the assertion + fires as an alert and this test catches it directly -- that is the + reproduction from the issue. On a non-debug build it is only a UAF + driver, worth its runtime under ASan and little without it. A green run + is weak evidence, as in test_process_abrupt_teardown.py; read it as "the + OOSM ack raced repeated abrupt teardowns and the router stayed correct". + """ + + # SIGKILLing a worker is the point of the test, so the main process' + # report of it is expected, not a finding. Nothing else is waived -- + # in particular the assertion alert this test exists to catch is not. + skip_alert(r'process \d+ exited on signal 9') + + client.load(APP, limits={"shm": SHM_LIMIT}, processes=1) + + assert 'success' in client.conf( + {"listen_threads": LISTEN_THREADS}, 'settings' + ), 'listen_threads' + + stop = threading.Event() + failures = [] + + def staller(): + """Ask for a response far larger than the segment and then read it + slowly, so the application blocks in nxt_unit_wait_shm_ack() and the + router keeps completing buffers for it.""" + + while not stop.is_set(): + try: + sock = client.get(url=f'/?mb={RESPONSE_MB}', no_recv=True) + time.sleep(STALL) + client.recvall(sock, read_timeout=READ_TIMEOUT, + buff_size=65536) + sock.close() + + except Exception: + # A request cut short by a kill is the point of the test, not + # a failure. Only a hang or a crashed daemon is, and those + # are caught by the join and the assertions below. + pass + + stallers = [threading.Thread(target=staller) for _ in range(STALLERS)] + + for thread in stallers: + thread.start() + + # Let the first responses reach the OOSM state before killing anything. + time.sleep(STALL * 2) + + killed = [] + + for round_ in range(KILLS): + pids = _wait_for_fresh_pids(set(killed)) + + if not pids: + failures.append(f'no fresh worker for kill {round_ + 1}') + break + + subprocess.call(['kill', '-9', *pids]) + + killed.extend(pids) + + time.sleep(KILL_INTERVAL) + + stop.set() + + for thread in stallers: + thread.join(timeout=120) + assert not thread.is_alive(), 'stalled request thread hung' + + assert not failures, f'{len(failures)} failure(s), first: {failures[0]}' + assert len(killed) >= KILLS, ( + f'only {len(killed)} worker(s) killed: {killed}' + ) + + log = Log.read() + + # The direct hit. nxt_assert() aborts the router in a --debug build, and + # the record it prints names the file and line -- src/nxt_port.c, the + # use_count assertion in nxt_port_mp_cleanup(). Checked before anything + # else, because a dead router makes every other assertion here fail for + # the wrong reason. + assert 'assertion failed' not in log, ( + 'a port was referenced after its last drop: ' + + next( + line + for line in log.splitlines() + if 'assertion failed' in line + ) + ) + + # Still serving, on a worker started after the last kill. + sock = client.get(url='/?mb=1', no_recv=True) + resp = client.recvall(sock, read_timeout=READ_TIMEOUT, buff_size=65536) + sock.close() + + assert _body(resp, 1024 * 1024).count(b'x') == 1024 * 1024, ( + 'still serving' + ) + + if '[debug]' not in log: + pytest.skip('OOSM evidence needs a --debug build') + + # Same proof as the test above that the OOSM round trip really ran; here + # it also establishes that the kills landed on a blocked worker rather + # than an idle one. + assert 'oosm in ' in log, 'router handled an OOSM message' + + Log.check_alerts() diff --git a/test/test_process_teardown_churn.py b/test/test_process_teardown_churn.py new file mode 100644 index 000000000..f4cfc9f9a --- /dev/null +++ b/test/test_process_teardown_churn.py @@ -0,0 +1,292 @@ +"""Application teardown churn under concurrent shared-memory traffic. + +What this drives: several router worker engines (``settings/listen_threads``) +serving large POSTs to a PHP application while a second thread repeatedly +rewrites that application's configuration. Every configuration rewrite makes +the router build a *new* ``nxt_app_t`` and release the previous one +(``src/nxt_router.c``: an application is reused only when the printed JSON of +its configuration is byte-identical), which tears the old application's +processes down and destroys its outgoing port-mmap segments +(``nxt_port_mmaps_destroy()``) — while router workers on other engines are +still copying request bodies into, and reading responses out of, shared memory +for that same application. + +What this proves and what it does not: a green run is *weak* evidence for the +absence of a use-after-free on the process/port lookup path. The window is a +handful of instructions wide and an unsanitised build will almost always miss +it. The value of this test is as a crash and ASan/UBSan *driver*: run it under +``.github/workflows/sanitize.yml``-style instrumentation and a live +use-after-free becomes a hard failure instead of a coin flip. Read a pass here +as "the workload ran and the daemon stayed up", nothing stronger. + +Deliberately kept short - 64 requests of 1 MiB, about three seconds on an idle +8-core box - so it is cheap enough to run on every CI leg, including the much +slower sanitiser legs. Correctness assertions are written so that a +legitimately failed request during teardown is not a failure, but a corrupt or +hung one is. +""" + +import subprocess +import threading +import time + +from unit.applications.lang.php import ApplicationPHP +from unit.log import Log +from unit.shm_body import describe_mismatch, make_body + +prerequisites = {'modules': {'php': 'all'}} + +client = ApplicationPHP() + +# src/nxt_port_memory_int.h: PORT_MMAP_CHUNK_SIZE is 16 KiB and one shared +# memory segment holds PORT_MMAP_DATA_SIZE (10 MiB) of payload. The router +# always hands requests to an application through those segments +# (nxt_router_prepare_msg() allocates with nxt_port_mmap_get_buf()), but a +# small request fits in a single chunk of a single segment. 1 MiB spans 64 +# chunks in each direction and keeps the worker inside the chunk allocator +# (nxt_port_mmap_get_buf()/nxt_port_mmap_increase_buf()) long enough for a +# teardown on another thread to overlap it. +# +# The bytes are not uniform: every request carries a fresh body stamped, every +# 64 bytes, with that request's tag and the byte offset of the record (see +# unit/shm_body.py). WORKERS requests are in shared memory at once here, so a +# chunk recycled out from under one of them most likely holds another one's +# payload; against 'x' * BODY_SIZE that chunk is byte-identical to what +# belongs there and the check below cannot see it. +BODY_SIZE = 1024 * 1024 + +# > 1 so more than one router worker engine is live; this is a top-level +# "settings" member (src/nxt_conf_validation.c: nxt_conf_vldt_setting_members). +LISTEN_THREADS = 4 + +# Prestarted workers. Named because the baseline below has to wait for +# exactly this many before it means anything. +SPARE = 2 + +WORKERS = 4 +REQUESTS_PER_WORKER = 16 +CHURN_INTERVAL = 0.05 + +# Not the suite default (60s), so that a stalled request returns instead of +# calling pytest.fail() on a helper thread; the stall is detected by timing it +# and reported as a hard failure below. +READ_TIMEOUT = 10 + +APP = 'mirror' + + +def _app_pids(): + """PIDs currently serving the application, via ps (as conftest does).""" + out = subprocess.check_output( + ['ps', '-ax', '-o', 'pid', '-o', 'command'] + ).decode() + + marker = f'unit: "{APP}" application' + + return {line.split()[0] for line in out.splitlines() if marker in line} + + +def _wait_for_pids(count, timeout=100): + """Poll until at least `count` application workers are visible. + + Returns the last sample either way, so a shortfall is reported by the + caller's assertion as the pid set it actually saw rather than as a bare + timeout. + """ + + for _ in range(timeout): + pids = _app_pids() + + if len(pids) >= count: + break + + time.sleep(0.1) + + return pids + + +def test_process_teardown_churn(): + client.load(APP, processes={"spare": SPARE, "max": 4, "idle_timeout": 5}) + + assert 'success' in client.conf( + {"listen_threads": LISTEN_THREADS}, 'settings' + ), 'listen_threads' + + assert client.post(body='ping', read_timeout=READ_TIMEOUT)['status'] == 200 + + # The set of workers that exist *before* any configuration rewrite. The + # final assertion needs it to tell "the churn replaced processes" from + # "the churn changed nothing and these are the ones we started with". + # + # Wait for the configured spares rather than assuming the warm-up request + # brought them up. Unit starts spare processes asynchronously as the + # first one becomes idle, so an immediate sample can hold one of the two + # -- and the second would then appear in a later sample and be counted as + # a replacement, restoring the exact false positive the `fresh` check + # below exists to rule out. + baseline = _wait_for_pids(SPARE) + + # >=, not ==: "spare" is a floor, not a cap ("max" is 4), so the pool is + # allowed to be larger by the time the warm-up request has been served. + # What matters is that it is not still filling. + assert len(baseline) >= SPARE, ( + f'expected at least {SPARE} spare workers before the churn, ' + f'saw {sorted(baseline)}' + ) + + lock = threading.Lock() + stop = threading.Event() + failures = [] + outcomes = [] + pids = set() + + def record(bucket, item): + with lock: + bucket.append(item) + + def request_worker(): + for _ in range(REQUESTS_PER_WORKER): + expected = make_body(BODY_SIZE) + started = time.time() + + try: + resp = client.post( + body=expected, + read_timeout=READ_TIMEOUT, + read_buffer_size=65536, + ) + + except KeyboardInterrupt: + raise + + # pytest.fail() inside the helpers raises BaseException + except BaseException as exc: # noqa: BLE001 + record(failures, f'request raised {exc!r}') + continue + + elapsed = time.time() - started + status = resp.get('status') + body = resp.get('body', '') + + if status == 200: + # A short body is legitimate: the application can die after its + # headers are on the wire. Wrong *bytes* are not - that is + # what reading freed or recycled shared memory looks like. + if body != expected[: len(body)]: + # describe_mismatch() reports the first wrong byte and the + # record stamp found there, so a failure says whether the + # bytes came from another offset of this request or from + # another request entirely. It keeps the fail-open guard + # too: the overlong-but-correct-prefix shape has no + # mismatching byte, and a bare next() would raise + # StopIteration here, outside the try that guards the + # request, killing this thread instead of recording the + # corruption. + record( + failures, + f'corrupt body: {len(body)} bytes, ' + + describe_mismatch(body, expected), + ) + continue + + full = len(body) == BODY_SIZE + + record(outcomes, 200 if full else 'truncated') + + elif status is None: + # No parsable response: connection closed during teardown. + # Distinguish that from a hang, which is a real defect. + if elapsed >= READ_TIMEOUT: + record(failures, f'no response in {elapsed:.1f}s') + else: + record(outcomes, 'closed') + + elif status in (500, 503): + record(outcomes, status) + + else: + record(failures, f'unexpected status {status}') + + def churn_worker(): + generation = 0 + + while not stop.is_set(): + generation += 1 + + try: + resp = client.conf( + {"CHURN": str(generation)}, + f'applications/{APP}/environment', + ) + + except KeyboardInterrupt: + raise + + except BaseException as exc: # noqa: BLE001 + record(failures, f'config update {generation} raised {exc!r}') + return + + if 'success' not in resp: + record(failures, f'config update {generation} failed: {resp}') + return + + with lock: + pids.update(_app_pids()) + + time.sleep(CHURN_INTERVAL) + + churn = threading.Thread(target=churn_worker) + workers = [threading.Thread(target=request_worker) for _ in range(WORKERS)] + + churn.start() + for worker in workers: + worker.start() + + for worker in workers: + worker.join(timeout=120) + assert not worker.is_alive(), 'request thread hung' + + stop.set() + churn.join(timeout=60) + assert not churn.is_alive(), 'churn thread hung' + + assert not failures, f'{len(failures)} failure(s), first: {failures[0]}' + + # The daemon is still alive and still serves the whole payload correctly. + expected = make_body(BODY_SIZE) + + resp = client.post(body=expected, read_timeout=READ_TIMEOUT, + read_buffer_size=65536) + + assert resp['status'] == 200, 'still serving' + assert len(resp['body']) == BODY_SIZE, 'still serving whole body' + assert resp['body'] == expected, 'still serving intact body' + + # The workload has to have actually torn processes down, otherwise it is + # only a concurrency test. A count is not enough to show that: "spare": 2 + # above means `baseline` already holds two PIDs, so `len(pids) > 1` passes + # on a run where no process was ever replaced. What proves a teardown and + # respawn is a PID that was not serving the application before the churn + # started. + fresh = pids - baseline + gone = baseline - _app_pids() + + assert fresh, ( + f'no new application process appeared: saw {sorted(pids)}, ' + f'baseline {sorted(baseline)}' + ) + + # A new pid on its own only proves the pool grew, which "max": 4 permits + # without any teardown at all. A pre-churn worker that is no longer + # serving is the direct evidence that the configuration rewrites actually + # tore an application down -- which is the whole point of the workload. + assert gone, ( + f'no pre-churn application process was torn down: ' + f'baseline {sorted(baseline)} all still serving' + ) + + # Requests must not have been shut out entirely, or nothing reached shared + # memory concurrently with a teardown. + assert outcomes.count(200) > 0, f'no request completed: {outcomes}' + + Log.check_alerts() diff --git a/test/test_proxy_chunked.py b/test/test_proxy_chunked.py index 0f349ddb8..75e1df2d1 100644 --- a/test/test_proxy_chunked.py +++ b/test/test_proxy_chunked.py @@ -648,3 +648,32 @@ def worker(body_bytes): finally: proc.terminate() proc.wait() + + +def _handler_counts_content_length(data): + headers = data.split('\r\n\r\n', 1)[0].split('\r\n')[1:] + count = sum(1 for h in headers if h.lower().startswith('content-length:')) + body = str(count) + return ( + "HTTP/1.1 200 OK\r\nConnection: close\r\n" + f"Content-Length: {len(body)}\r\n\r\n{body}" + ) + + +def test_proxy_chunked_request_one_content_length(): + """A chunked request body goes upstream with exactly one Content-Length.""" + port = _get_free_port() + _run_proxy_server(_handler_counts_content_length, port) + _configure_proxy(port, 'counting backend') + + resp = client.post( + headers={ + 'Host': 'localhost', + 'Transfer-Encoding': 'chunked', + 'Connection': 'close', + }, + body=_chunked_encode(b'0123456789').decode('latin-1'), + ) + + assert resp['status'] == 200, 'status' + assert resp['body'] == '1', 'one Content-Length upstream' diff --git a/test/test_proxy_head.py b/test/test_proxy_head.py new file mode 100644 index 000000000..a5d233cca --- /dev/null +++ b/test/test_proxy_head.py @@ -0,0 +1,288 @@ +"""Proxied responses that RFC 9112 Sect. 6.3 defines as having no body. + +nxt_h1p_peer_header_read_done() used to arm the upstream body framing from the +upstream headers alone: Content-Length became h1p->remainder, Transfer-Encoding +armed the chunked parser. A response to HEAD, and any 204 or 304 response, +carries no body no matter what those headers say, so the upstream -- which +nxt_h1p_peer_header_send() always asks to "Connection: close" -- closed with the +remainder still outstanding. nxt_h1p_peer_closed() read that as a truncated +body and set r->truncated/r->inconsistent, and nxt_h1p_request_close() then +dropped the client keep-alive, losing any pipelined follow-up request. + +Each case pipelines two requests in a single write: the bodyless one, then a +plain GET. The second response arriving at all is the regression assertion. + +1xx is tested in test_proxy_interim.py. + +These cases also drive both shapes the header read can take -- with and +without body bytes alongside the header -- but they cannot assert that the +unrelayed header buffer is released: a stranded request pool is invisible from +the client side, and a stock build exposes no pool counter -- nxt_debug is +never assigned, so even a --debug build emits no "mp ... release" lines. That +was measured instead with a throwaway build that forces nxt_debug on and counts +pools reaching a zero retain per request; see the commit message. + +The same hold covers a third path nothing here reaches: an ordinary proxied +response whose body arrives in a read after the header, which stranded the pool +on master as well. It is unobservable from the client for the same reason, and +is covered by that measurement rather than by an assertion. + +See freeunitorg/freeunit#283 for the downstream half of the same rule. +""" + +import select +import socket + +import pytest + +from conftest import run_process +from unit.applications.proto import ApplicationProto +from unit.option import option +from unit.utils import waitforsocket + +client = ApplicationProto() + +# Reserved in test/fake_upstream/README.md's port registry; 7980-7982 +# belong to test/fake_otlp. +UPSTREAM_PORT = 7978 + +# What the upstream answers, keyed by the request target Unit forwards. +# Every one of these closes the connection right after writing what is here: +# none of them ever sends the body its own headers advertise. +UPSTREAM_RESPONSES = { + '/cl10': 'HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\n', + '/chunked': 'HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n', + '/204': 'HTTP/1.1 204 No Content\r\nContent-Length: 10\r\n\r\n', + '/304': 'HTTP/1.1 304 Not Modified\r\nContent-Length: 10\r\n\r\n', + # Header block and 10 junk bytes in a single write, so the bytes land in + # the same read as the header. A response to HEAD has no body, so they + # must be dropped rather than relayed -- and the buffer holding them + # reaches the bodyless branch of nxt_h1p_peer_header_read_done() with + # bytes still in it, unlike the cases above, which is where an earlier + # revision of this change stranded the request pool. + '/cl10junk': ( + 'HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\nJUNKJUNKJU' + ), + # The trailing request of every pipeline; this one does have a body. + '/ok': 'HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nOK', +} + + +def run_server(server_port): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + sock.bind(('127.0.0.1', server_port)) + sock.listen(10) + + def recv_request(conn): + data = b'' + + while b'\r\n\r\n' not in data: + if not select.select([conn], [], [], 5)[0]: + break + + part = conn.recv(4096) + if not part: + break + + data += part + + return data.decode('utf-8', errors='ignore') + + while True: + conn, _ = sock.accept() + + request = recv_request(conn) + target = request.split(' ')[1] if ' ' in request else '' + + conn.sendall( + UPSTREAM_RESPONSES.get( + target, 'HTTP/1.1 500 Internal Server Error\r\n\r\n' + ).encode() + ) + conn.close() + + +@pytest.fixture(autouse=True) +def setup_method_fixture(): + run_process(run_server, UPSTREAM_PORT) + waitforsocket(UPSTREAM_PORT) + + assert 'success' in client.conf( + { + "listeners": {"*:8080": {"pass": "routes"}}, + "routes": [ + {"action": {"proxy": f'http://127.0.0.1:{UPSTREAM_PORT}'}} + ], + } + ), 'bodyless proxy configuration' + + +def pipeline(method, target): + """Send " " and a GET /ok back to back, read to EOF. + + Both requests leave in one write, so the second is already sitting in the + listener's read buffer when the first response is generated: it can only be + answered if the connection survives that response. + """ + + request = ( + f'{method} {target} HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: keep-alive\r\n' + '\r\n' + 'GET /ok HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + '\r\n' + ) + + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + sock.settimeout(10) + + try: + sock.connect(('127.0.0.1', 8080)) + sock.sendall(request.encode()) + + data = b'' + while True: + try: + part = sock.recv(4096) + except socket.timeout: + break + + if not part: + break + + data += part + + finally: + sock.close() + + return data.decode('utf-8', errors='ignore') + + +def check_pipeline(raw, status): + """Assert the exact wire shape: bodyless header, then the whole GET /ok. + + Splitting on the header terminator pins both halves at once -- a body + leaking out of the first response, a chunked terminator appended to it, or + a missing second response all change the number of parts or their content. + """ + + parts = raw.split('\r\n\r\n') + + assert len(parts) == 3, f'expected two responses, got {raw!r}' + + first, second, body = parts + + assert first.startswith( + f'HTTP/1.1 {status}' + ), f'first response status line: {first!r}' + assert ( + 'close' not in first.lower() + ), f'keep-alive dropped on a bodyless response: {first!r}' + assert ( + 'transfer-encoding' not in first.lower() + ), f'chunked framing on a bodyless response: {first!r}' + + assert second.startswith( + 'HTTP/1.1 200' + ), f'pipelined request lost: {second!r}' + assert body == 'OK', f'pipelined response body: {body!r}' + + return first + + +def test_proxy_head_content_length(): + """HEAD: the upstream advertises a body and sends none, then closes.""" + + first = check_pipeline(pipeline('HEAD', '/cl10'), 200) + + # RFC 9110 Sect. 9.3.2: a HEAD response keeps the length the equivalent + # GET would report. + assert 'Content-Length: 10' in first, f'HEAD keeps Content-Length: {first!r}' + + +def test_proxy_head_chunked(): + """HEAD: the upstream announces chunked and sends no terminating chunk.""" + + check_pipeline(pipeline('HEAD', '/chunked'), 200) + + +def test_proxy_head_trailing_bytes(): + """HEAD: the upstream writes the header and 10 junk bytes together. + + A response to HEAD has no body, so those bytes are not one: they must not + reach the client and must not be counted as a body. They also put the + header buffer on the branch that used to hand it to + nxt_h1p_peer_body_process(), whose completion released it -- so this is the + case where simply returning early strands the buffer and, with it, the + request pool. That leak is not observable from here (see the module + docstring); what is checked is that the bytes are dropped and the + connection survives. + """ + + first = check_pipeline(pipeline('HEAD', '/cl10junk'), 200) + + assert 'Content-Length: 10' in first, f'HEAD keeps Content-Length: {first!r}' + + +def test_proxy_get_204(): + """204: Content-Length is advertised, no body follows, upstream closes.""" + + first = check_pipeline(pipeline('GET', '/204'), 204) + + # RFC 9110 Sect. 8.6 forbids Content-Length on a 204; #283 drops it. + assert ( + 'content-length' not in first.lower() + ), f'Content-Length kept on 204: {first!r}' + + +def test_proxy_get_304(): + """304: Content-Length describes the cached body and is kept.""" + + first = check_pipeline(pipeline('GET', '/304'), 304) + + assert 'Content-Length: 10' in first, f'304 keeps Content-Length: {first!r}' + + +def test_proxy_head_response_header_variable(wait_for_record): + """The relayed header bytes stay readable after the response completes. + + peer->fields point their name/value into the buffer the upstream header was + parsed from, and nxt_http_proxy_header_read() shallow-copies those field + structs into r->resp; $response_header_* reads them at access-log time. + This exercises that path on the early-complete branch, where the peer + connection is already closed by the time the record is written. + + It is coverage, not a use-after-free detector: returning that buffer to the + engine cache early leaves the pointers dangling but does not reliably + corrupt them, so this passes either way. The buffer must simply not be + freed there -- which is also what the normal path does. + """ + + assert 'success' in client.conf( + { + 'path': f'{option.temp_dir}/access.log', + 'format': '$request_line "$response_header_content_length"\n', + }, + 'access_log', + ), 'access_log format' + + check_pipeline(pipeline('HEAD', '/cl10'), 200) + + assert ( + wait_for_record(r'HEAD /cl10 HTTP/1.1 "10"', 'access.log') is not None + ), 'Content-Length readable from the access log after the response' + + +def test_proxy_get_body_still_relayed(): + """Control: a status that does define a body still gets one.""" + + raw = pipeline('GET', '/ok') + + assert raw.count('HTTP/1.1 200') == 2, f'expected two responses: {raw!r}' + assert raw.count('\r\n\r\nOK') == 2, f'both bodies relayed: {raw!r}' diff --git a/test/test_proxy_interim.py b/test/test_proxy_interim.py new file mode 100644 index 000000000..6ac82fb6a --- /dev/null +++ b/test/test_proxy_interim.py @@ -0,0 +1,358 @@ +"""Proxied upstream 1xx interim responses (RFC 9110 Sect. 15.2). + +The proxy drops every 1xx except 101 and sends only the final response. +More than ten 1xx responses, or one bigger than the header buffer, give 502. + +Most cases send a second request (GET /ok) on the same connection. If its +response is correct, the first final response was not lost. Cases that end +with an error or 101 send one request only. +""" + +import select +import socket +import time + +import pytest + +from conftest import run_process +from unit.applications.proto import ApplicationProto +from unit.utils import waitforsocket + +client = ApplicationProto() + +# Reserved in test/fake_upstream/README.md's port registry. +UPSTREAM_PORT = 7977 + +FINAL = 'HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nOK' + +# A final response with its own field. +MARKED_FINAL = ( + 'HTTP/1.1 200 OK\r\nContent-Length: 2\r\nX-Final: yes\r\n\r\nOK' +) + +# A typical 103. +INTERIM = 'HTTP/1.1 103 Early Hints\r\nLink: ; rel=preload\r\n\r\n' + +# More than the 16 inline fields of the parser. +BIG_FIELD_COUNT = 20 + +# The only fields the final response may have. +FINAL_FIELDS = { + 'Content-Length', 'X-Final', 'Server', 'Date', 'Connection', + 'Transfer-Encoding', +} + +# A 103 bigger than half of the 64 KiB header buffer. Two of them fit only +# if the proxy frees the bytes of each dropped 1xx. +BIG_HINTS = 'HTTP/1.1 103 Early Hints\r\n' + ( + ''.join( + f'Link: ; rel=preload; as=style\r\n' + for i in range(700) + ) +) + '\r\n' + +assert 32 * 1024 < len(BIG_HINTS) < 64 * 1024, len(BIG_HINTS) + +# A 103 bigger than the header buffer. +OVERSIZED_HINTS = 'HTTP/1.1 103 Early Hints\r\n' + ( + ''.join(f'X-Pad-{i:02d}: {"a" * 1000}\r\n' for i in range(70)) +) + '\r\n' + +assert len(OVERSIZED_HINTS) > 64 * 1024, len(OVERSIZED_HINTS) + +# Upstream response for each target. A string is sent in one write. Each +# item of a list is sent separately, with a pause, so Unit reads it separately. +UPSTREAM_RESPONSES = { + # 1xx and final response in one read. + '/continue': 'HTTP/1.1 100 Continue\r\n\r\n' + FINAL, + # Final response in a later read. + '/hints': [ + 'HTTP/1.1 103 Early Hints\r\n' + 'Link: ; rel=preload; as=style\r\n\r\n', + FINAL, + ], + # Several 1xx responses in several reads. + '/multi': [ + 'HTTP/1.1 100 Continue\r\n\r\n', + 'HTTP/1.1 103 Early Hints\r\nLink: ; rel=preload\r\n\r\n', + 'HTTP/1.1 103 Early Hints\r\nLink: ; rel=preload\r\n\r\n' + + FINAL, + ], + '/big': BIG_HINTS + BIG_HINTS + FINAL, + # The empty line of the 1xx is split between two reads. + '/split': [INTERIM[:-1], '\n' + FINAL], + # Fields of the 1xx must not appear in the final response. + '/fields': ( + 'HTTP/1.1 103 Early Hints\r\n' + + ''.join(f'X-Early-{i:02d}: v{i}\r\n' for i in range(BIG_FIELD_COUNT)) + + 'Link: ; rel=preload\r\n' + + '\r\n' + + MARKED_FINAL + ), + # Framing fields on a 1xx (not allowed, RFC 9112, 6.1) are dropped too. + '/framing': ( + 'HTTP/1.1 103 Early Hints\r\nContent-Length: 3\r\n' + 'Connection: close\r\nLink: ; rel=preload\r\n\r\n' + FINAL + ), + # Exactly the cap, and one past it. + '/ten': INTERIM * 10 + FINAL, + '/eleven': INTERIM * 11 + FINAL, + # 101 changes the protocol, so it is the response. + '/switch': 'HTTP/1.1 101 Switching Protocols\r\n\r\n', + # A 1xx without a final response. + '/truncated': 'HTTP/1.1 100 Continue\r\n\r\n', + # A 1xx bigger than the header buffer. + '/oversized': OVERSIZED_HINTS + FINAL, + # The second request. + '/ok': FINAL, +} + + +def run_server(server_port): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + sock.bind(('127.0.0.1', server_port)) + sock.listen(10) + + def recv_until(conn, data, done): + while not done(data): + if not select.select([conn], [], [], 5)[0]: + break + + part = conn.recv(65536) + if not part: + break + + data += part + + return data + + while True: + conn, _ = sock.accept() + + data = recv_until(conn, b'', lambda d: b'\r\n\r\n' in d) + request = data.decode('utf-8', errors='ignore') + target = request.split(' ')[1] if ' ' in request else '' + + if target == '/expect': + # Read the body, send 100, then send how many bytes were read. + header, _, body = data.partition(b'\r\n\r\n') + length = 0 + + for line in header.split(b'\r\n'): + if line.lower().startswith(b'content-length:'): + length = int(line.split(b':', 1)[1]) + + body = recv_until(conn, body, lambda d: len(d) >= length) + + conn.sendall(b'HTTP/1.1 100 Continue\r\n\r\n') + time.sleep(0.1) + + reply = f'got {len(body)}'.encode() + conn.sendall( + b'HTTP/1.1 200 OK\r\nContent-Length: %d\r\n\r\n%s' + % (len(reply), reply) + ) + + else: + response = UPSTREAM_RESPONSES.get( + target, 'HTTP/1.1 500 Internal Server Error\r\n\r\n' + ) + + if isinstance(response, str): + response = [response] + + try: + for i, part in enumerate(response): + if i > 0: + time.sleep(0.1) + + conn.sendall(part.encode()) + + except OSError: + # Unit closed the connection before reading it all. + pass + + conn.close() + + +@pytest.fixture(autouse=True) +def setup_method_fixture(): + run_process(run_server, UPSTREAM_PORT) + waitforsocket(UPSTREAM_PORT) + + assert 'success' in client.conf( + { + "listeners": {"*:8080": {"pass": "routes"}}, + "routes": [ + {"action": {"proxy": f'http://127.0.0.1:{UPSTREAM_PORT}'}} + ], + } + ), 'interim proxy configuration' + + +def pipeline(first_request, sentinel=True): + """Send first_request, then GET /ok if sentinel is true. Read to EOF.""" + + request = first_request + + if sentinel: + request += ( + 'GET /ok HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + '\r\n' + ) + + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + sock.settimeout(10) + + try: + sock.connect(('127.0.0.1', 8080)) + sock.sendall(request.encode()) + + data = b'' + while True: + try: + part = sock.recv(4096) + except socket.timeout: + break + + if not part: + break + + data += part + + finally: + sock.close() + + return data.decode('utf-8', errors='ignore') + + +def get(target): + return pipeline( + f'GET {target} HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: keep-alive\r\n' + '\r\n' + ) + + +def check(raw, body='OK'): + """Exactly two 200 responses, with the expected bodies, and no 1xx.""" + + assert 'HTTP/1.1 1' not in raw, f'1xx relayed to the client: {raw!r}' + + parts = raw.split('\r\n\r\n') + + assert len(parts) == 3, f'expected two responses, got {raw!r}' + + first, first_body_and_second, second_body = parts + + assert first.startswith('HTTP/1.1 200'), f'first status: {first!r}' + assert first_body_and_second.startswith( + body + ), f'first body: {first_body_and_second!r}' + assert ( + 'HTTP/1.1 200' in first_body_and_second + ), f'pipelined request lost: {first_body_and_second!r}' + assert second_body == 'OK', f'pipelined body: {second_body!r}' + + +def get_alone(target): + return pipeline( + f'GET {target} HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + '\r\n', + sentinel=False, + ) + + +@pytest.mark.parametrize( + 'target', + [ + '/ok', # control + '/continue', # interim and final in one read + '/hints', # final in a later read + '/multi', # several, split across reads + '/big', # fits only if the 1xx bytes are freed + '/split', # empty line of the 1xx in two reads + '/framing', # framing fields of the 1xx are dropped + '/ten', # exactly the cap + ], +) +def test_proxy_interim_consumed(target): + raw = get(target) + + check(raw) + + assert 'Content-Length: 3' not in raw, f'1xx framing relayed: {raw!r}' + + +def test_proxy_interim_expect_continue(): + """The upstream sends a 100 for /expect, and the proxy drops it. The + client gets one 100 only: the router sends it before it reads the rest + of the body.""" + + body = 'x' * 4096 + + raw = pipeline( + 'POST /expect HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: keep-alive\r\n' + 'Expect: 100-continue\r\n' + f'Content-Length: {len(body)}\r\n' + '\r\n' + body + ) + + # The first read holds at most header_buffer_size (2048) bytes, so the + # router always waits for the rest of the body. + router_continue = 'HTTP/1.1 100 Continue\r\n\r\n' + + assert raw.startswith(router_continue), f'no 100 from the router: {raw!r}' + + check(raw[len(router_continue) :], body=f'got {len(body)}') + + +def test_proxy_interim_fields_not_leaked(): + """Fields of a 1xx, also those after the 16 inline fields, are dropped.""" + + raw = get('/fields') + + check(raw) + + lines = raw.split('\r\n\r\n')[0].split('\r\n') + names = [line.split(':', 1)[0] for line in lines[1:]] + + assert 'X-Final: yes' in raw, f'final field lost: {raw!r}' + assert not [ + name for name in names if name not in FINAL_FIELDS + ], f'interim fields leaked into the final response: {lines!r}' + + +def test_proxy_interim_101_is_the_response(): + """101 changes the protocol, so it is the response.""" + + raw = get_alone('/switch') + + assert raw.startswith( + 'HTTP/1.1 101 Switching Protocols' + ), f'101 not relayed: {raw!r}' + + +@pytest.mark.parametrize( + 'target', + [ + '/truncated', # upstream closes after a 1xx + '/eleven', # one past the cap + '/oversized', # a 1xx bigger than the buffer + ], +) +def test_proxy_interim_bad_gateway(target): + raw = get_alone(target) + + assert raw.startswith('HTTP/1.1 502'), f'expected 502: {raw[:200]!r}' + assert 'HTTP/1.1 200' not in raw, f'final relayed: {raw[:200]!r}' diff --git a/test/test_proxy_te.py b/test/test_proxy_te.py new file mode 100644 index 000000000..4e3d3b3c8 --- /dev/null +++ b/test/test_proxy_te.py @@ -0,0 +1,336 @@ +"""How the proxy frames an upstream response by its Transfer-Encoding. + +nxt_h1p_peer_transfer_encoding() used to accept only the exact value +"chunked". RFC 9112 Sect. 6.1 makes coding names case-insensitive and the +value a comma-separated list. So "Chunked" or "gzip, chunked" was not seen +as chunked. The body was then framed by Content-Length or by connection +close, and the chunk framing reached the client as body bytes. + +The proxy decodes only chunked, and it never forwards Transfer-Encoding to +the client. So the whole list must be one "chunked", in any letter case. +Whitespace and empty list elements around it are allowed. Every other +Transfer-Encoding gives 502. Chunked together with Content-Length also +gives 502 (RFC 9112 Sect. 6.3). Content-Length never frames a body that +has a Transfer-Encoding. + +Neither 502 applies to a response the same Sect. 6.3 says has no body: a +response to HEAD, and any 204 or 304, ends at the first empty line whatever +Transfer-Encoding and Content-Length say. There is no body to decode there, +so those responses are completed as bodyless and relayed normally. The +keep-alive half of that path is pinned in test_proxy_head.py, whose pipeline +idiom the bodyless cases below reuse. + +The upstream is a plain Python socket server. The request target selects +the case. +""" + +import select +import socket + +import pytest + +from conftest import run_process +from unit.applications.proto import ApplicationProto +from unit.utils import waitforsocket + +client = ApplicationProto() + +# Reserved in test/fake_upstream/README.md's port registry. +UPSTREAM_PORT = 7976 + +BODY = 'hello, world' + +# A chunked body whose chunk-size lines and CRLFs would leak into the relayed +# body if the proxy did not decode it. +CHUNKED_BODY = '5\r\nhello\r\n7\r\n, world\r\n0\r\n\r\n' + + +def resp(*fields, body=CHUNKED_BODY): + head = ''.join(f'{f}\r\n' for f in fields) + return f'HTTP/1.1 200 OK\r\n{head}Connection: close\r\n\r\n{body}' + + +def bodyless(status, *fields): + """A response with no body at all, whatever its header fields claim.""" + + head = ''.join(f'{f}\r\n' for f in fields) + return f'HTTP/1.1 {status}\r\n{head}Connection: close\r\n\r\n' + + +UPSTREAM_RESPONSES = { + '/lower': resp('Transfer-Encoding: chunked'), + '/upper': resp('Transfer-Encoding: CHUNKED'), + '/mixed': resp('Transfer-Encoding: Chunked'), + '/ows': resp('Transfer-Encoding: ,\t chunked ,'), + # Content-Length would cut the body after the first chunk-size line. + '/mixed-cl': resp('Transfer-Encoding: Chunked', 'Content-Length: 3'), + '/lower-cl': resp('Transfer-Encoding: chunked', 'Content-Length: 3'), + '/gzip-chunked': resp('Transfer-Encoding: gzip, chunked'), + '/gzip-chunked-cl': resp( + 'Transfer-Encoding: gzip, Chunked', 'Content-Length: 3' + ), + '/chunked-gzip': resp('Transfer-Encoding: chunked, gzip'), + '/two-lines': resp( + 'Transfer-Encoding: gzip', 'Transfer-Encoding: chunked' + ), + '/chunked-twice': resp( + 'Transfer-Encoding: chunked', 'Transfer-Encoding: chunked' + ), + '/chunked-param': resp('Transfer-Encoding: chunked;a=b'), + '/empty': resp('Transfer-Encoding: ,'), + '/identity': resp( + 'Transfer-Encoding: identity', 'Content-Length: 3', body=BODY + ), + # Bodyless final responses carrying a Transfer-Encoding the proxy cannot + # decode. Every one of them writes only what is here and closes: none + # sends the body its own header names. + '/204-gzip': bodyless('204 No Content', 'Transfer-Encoding: gzip'), + '/304-identity': bodyless('304 Not Modified', 'Transfer-Encoding: identity'), + '/head-param': bodyless('200 OK', 'Transfer-Encoding: chunked;a=b'), + # Exact "chunked" reached the bodyless path before this change too, so this + # one guards against the move breaking it. + '/204-chunked': bodyless('204 No Content', 'Transfer-Encoding: chunked'), + # The other 502: chunked and Content-Length together. A 304 keeps its + # Content-Length (RFC 9110 Sect. 9.3.2 for HEAD, the cached length here). + '/304-chunked-cl': bodyless( + '304 Not Modified', 'Transfer-Encoding: chunked', 'Content-Length: 10' + ), + # The trailing request of every pipeline; this one does have a body. + '/ok': 'HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nOK', +} + + +def run_server(server_port): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + sock.bind(('127.0.0.1', server_port)) + sock.listen(10) + + def recv_request(conn): + data = b'' + + while b'\r\n\r\n' not in data: + if not select.select([conn], [], [], 5)[0]: + break + + part = conn.recv(4096) + if not part: + break + + data += part + + return data.decode('utf-8', errors='ignore') + + while True: + conn, _ = sock.accept() + + request = recv_request(conn) + target = request.split(' ')[1] if ' ' in request else '' + + conn.sendall( + UPSTREAM_RESPONSES.get( + target, 'HTTP/1.1 500 Internal Server Error\r\n\r\n' + ).encode() + ) + conn.close() + + +@pytest.fixture(autouse=True) +def setup_method_fixture(): + run_process(run_server, UPSTREAM_PORT) + waitforsocket(UPSTREAM_PORT) + + assert 'success' in client.conf( + { + "listeners": {"*:8080": {"pass": "routes"}}, + "routes": [ + {"action": {"proxy": f'http://127.0.0.1:{UPSTREAM_PORT}'}} + ], + } + ), 'proxy configuration' + + +def get(target): + return client.get(url=target, headers={'Host': 'localhost', + 'Connection': 'close'}) + + +@pytest.mark.parametrize('target', ['/lower', '/upper', '/mixed', '/ows']) +def test_proxy_te_chunked(target): + resp = get(target) + + assert resp['status'] == 200, 'status' + assert resp['body'] == BODY, 'chunked body decoded' + assert 'Content-Length' not in resp['headers'], 'no Content-Length' + + +@pytest.mark.parametrize( + 'target', + [ + '/mixed-cl', + '/lower-cl', + '/gzip-chunked', + '/gzip-chunked-cl', + '/chunked-gzip', + '/two-lines', + '/chunked-twice', + '/chunked-param', + '/empty', + '/identity', + ], +) +def test_proxy_te_bad_gateway(target): + resp = get(target) + + assert resp['status'] == 502, 'status' + assert 'hello' not in resp['body'], 'upstream body not relayed' + + +def pipeline(method, target): + """Send " " and a GET /ok back to back, read to EOF. + + Both requests leave in one write, so the second is already sitting in the + listener's read buffer when the first response is generated: it can only be + answered if the connection survives that response. That is how a + mis-framed bodyless response shows itself -- nxt_h1p_peer_closed() sets + r->truncated and r->inconsistent, and nxt_h1p_request_close() then drops + the client keep-alive. Copied from test_proxy_head.py, which pins the same + path without a Transfer-Encoding in play. + """ + + request = ( + f'{method} {target} HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: keep-alive\r\n' + '\r\n' + 'GET /ok HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + '\r\n' + ) + + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + sock.settimeout(10) + + try: + sock.connect(('127.0.0.1', 8080)) + sock.sendall(request.encode()) + + data = b'' + while True: + try: + part = sock.recv(4096) + except socket.timeout: + break + + if not part: + break + + data += part + + finally: + sock.close() + + return data.decode('utf-8', errors='ignore') + + +def check_bodyless(raw, status): + """Assert the exact wire shape: bodyless header, then the whole GET /ok. + + Splitting on the header terminator pins both halves at once -- a 502 in + place of the first response, a body leaking out of it, or a missing second + response all change the number of parts or their content. + """ + + parts = raw.split('\r\n\r\n') + + assert len(parts) == 3, f'expected two responses, got {raw!r}' + + first, second, body = parts + + assert first.startswith( + f'HTTP/1.1 {status}' + ), f'first response status line: {first!r}' + assert ( + 'transfer-encoding' not in first.lower() + ), f'Transfer-Encoding relayed to the client: {first!r}' + assert ( + 'close' not in first.lower() + ), f'keep-alive dropped on a bodyless response: {first!r}' + + assert second.startswith( + 'HTTP/1.1 200' + ), f'pipelined request lost: {second!r}' + assert body == 'OK', f'pipelined response body: {body!r}' + + return first + + +def test_proxy_te_204_undecodable(): + """204 with "Transfer-Encoding: gzip": bodyless, not 502.""" + + first = check_bodyless(pipeline('GET', '/204-gzip'), 204) + + assert ( + 'content-length' not in first.lower() + ), f'Content-Length added to a 204: {first!r}' + + +def test_proxy_te_304_undecodable(): + """304 with "Transfer-Encoding: identity": bodyless, not 502.""" + + check_bodyless(pipeline('GET', '/304-identity'), 304) + + +def test_proxy_te_head_undecodable(): + """HEAD answered with "Transfer-Encoding: chunked;a=b": bodyless, not 502. + + A coding with parameters is not the bare "chunked" the proxy decodes, so + on any other status this is a 502 -- see test_proxy_te_bad_gateway's + /chunked-param. + """ + + check_bodyless(pipeline('HEAD', '/head-param'), 200) + + +def test_proxy_te_204_chunked(): + """204 with exact "chunked": unchanged by the check ordering. + + This case took the normal branch before the 502 checks moved and still + completes as bodyless, so it guards the move itself. + """ + + check_bodyless(pipeline('GET', '/204-chunked'), 204) + + +def test_proxy_te_304_chunked_content_length(): + """304 with chunked and Content-Length: bodyless, not 502. + + The chunked-plus-Content-Length 502 predates this change, and it fired + above the bodyless branch. RFC 9112 Sect. 6.3 ends this response at the + first empty line, so the two fields frame nothing and cannot disagree. + """ + + first = check_bodyless(pipeline('GET', '/304-chunked-cl'), 304) + + # RFC 9110 Sect. 9.3.2 / Sect. 15.4.5: a 304 reports the length the + # cached representation has. + assert ( + 'Content-Length: 10' in first + ), f'304 keeps Content-Length: {first!r}' + + +def test_proxy_te_bad_gateway_still_fires(): + """Control: an undecodable Transfer-Encoding on a status that has a body. + + test_proxy_te_bad_gateway covers the whole matrix; this pins the one shape + the bodyless cases above are the exemption from, over the same pipeline, + so the 502 and the exemption are read side by side. + """ + + raw = pipeline('GET', '/gzip-chunked') + + assert raw.startswith('HTTP/1.1 502'), f'status: {raw!r}' + assert 'hello' not in raw, f'upstream body not relayed: {raw!r}' diff --git a/test/test_python_application.py b/test/test_python_application.py index 6563efd2d..b87a914dd 100644 --- a/test/test_python_application.py +++ b/test/test_python_application.py @@ -4,6 +4,7 @@ import pwd import re import subprocess +import threading import time import venv @@ -386,7 +387,7 @@ def test_python_process_switch(): for _ in range(10): client.get(headers=headers_delay_1, no_recv=True) - client.get(headers=headers_delay_1) + assert client.get(headers=headers_delay_1)['status'] == 200 @pytest.mark.skip('not yet') @@ -1052,3 +1053,197 @@ def test_python_application_upload_multiple_files(): ) assert resp['status'] == 200, 'multiple files status' assert resp['body'] == 'first.txtfirst data', 'multiple files body' + + +def _ps_rows(): + out = subprocess.check_output( + ['ps', 'ax', '-o', 'pid=', '-o', 'ppid=', '-o', 'args='] + ).decode() + + return [ + parts + for parts in (line.split(None, 2) for line in out.splitlines()) + if len(parts) == 3 + ] + + +def _proto_pid(name): + """The pid of the application's prototype process, or None. + + `unit: "" prototype` is the title nxt_main_process.c gives it, and + unlike a worker's title it survives for every runtime -- the prototype + never execve()s. Matched on the whole ps line for the same reason + test_app_lifecycle.py does. + """ + + marker = f'unit: "{name}" prototype' + + for pid, _, args in _ps_rows(): + if marker in args: + return pid + + return None + + +def _child_pids(ppid): + return [pid for pid, parent, _ in _ps_rows() if parent == ppid] + + +def test_python_prototype_killed_mid_start(skip_alert, findall): + """Issue #269: a prototype killed between forking a worker and that + worker's PROCESS_READY must not strand the router's START_PROCESS. + + The router takes an app->pending_processes slot before posting + nxt_router_start_app_process_handler(), which arms an RPC on the router + port and sends START_PROCESS to the prototype. The reply that would + retire it -- the worker's READY, forwarded by the prototype -- can never + arrive once the prototype is dead, and the REMOVE_PID that reports the + death carries no stream for a prototype that had itself reached READY. + Only a peer-keyed registration is retired by that message. Without one + the slot is never given back; with "spare": 0 that is the whole of + max_pending_processes, so nxt_router_app_can_start() is false for good, + every later request parks in ack_waiting_req with no timeout, and no + replacement prototype is ever requested -- the start handler is the only + thing that asks, and it is what can_start gates. The wedge is permanent + and survives a reload that leaves the app's config text unchanged. + + So the assertions are: the in-flight request is answered rather than + hung, /status shows the slot back at zero, and the next request gets a + fresh prototype and a 200. On the unfixed router the first of those + never returns, which is why every wait here is bounded. + + The app module sleeps in its *import* (test/python/slow_start/wsgi.py), + which is what widens the PROCESS_CREATED -> READY window enough to aim + at. Landing inside that window is a precondition, not an assertion: + the worker has to be discovered well before the window closes, and + /status has to still count the start as pending right before the kill. + A runner loaded enough to miss either skips rather than reporting a + failure the test did not drive. + """ + + client.load( + 'slow_start', + processes={'max': 2, 'spare': 0, 'idle_timeout': 1}, + environment={'UNIT_SLOW_START': '4'}, + ) + + resp = [] + + def _request(): + try: + # Bounded: on the unfixed router this request is never answered, + # and a thread stuck in recv() would outlive the test. + resp.append(client.get(read_timeout=20)['status']) + + except Exception as exc: # noqa: BLE001 - reported as a failure below + resp.append(repr(exc)) + + thread = threading.Thread(target=_request) + thread.start() + + try: + # The prototype appears as soon as main forks it; the worker it then + # forks spends the next 4s importing the module. + deadline = time.time() + 10 + pid = None + + while time.time() < deadline: + pid = _proto_pid('slow_start') + + if pid is not None: + break + + time.sleep(0.1) + + assert pid is not None, 'prototype never started' + + # Wait for the worker to exist: killing the prototype before it has + # forked would carry the original start stream and let an unfixed + # router pass this test by the other route. Bounded at 2s, inside + # the 4s import window: exhausting it means the fork stalled past + # the window, which leaves nothing for the kill to aim at. Then a + # short pause puts the worker mid-import: past PROCESS_CREATED and + # before READY. + workers = [] + for _ in range(40): + workers = _child_pids(pid) + if workers: + break + time.sleep(0.05) + + if not workers: + pytest.skip('the prototype forked no worker within the window') + + time.sleep(0.3) + + # The worker must still be importing for the kill to land where this + # test aims it. "starting" is app->pending_processes, which the + # router gives back only once the worker's port arrives ready, so + # anything but the one pending start means the window has closed -- + # the request would then be served with 200 and the assertions below + # would report the wrong thing. + starting = client.conf_get('/status')['applications']['slow_start'][ + 'processes' + ]['starting'] + + if starting != 1: + pytest.skip( + f'the start left the import window: starting == {starting}' + ) + + # Its own teardown noise is scoped to the worker's pid below rather + # than skipped by shape, so an alert from any other process still + # fails the test. + + subprocess.call(['kill', '-9', pid]) + skip_alert(fr'process {pid} exited on signal 9') + + for worker in workers: + # Orphaned by the kill, the worker finishes its import and then + # writes READY into the dead prototype's socket, which is EPIPE, + # and unwinds closing descriptors it inherited from the + # prototype -- which the kill already closed. All of that is + # the pre-existing consequence of killing a prototype, not + # something this test's subject produces; it happens on the + # unfixed router too. Both log prefixes are pid#tid + # (src/nxt_unit.c:7319, src/nxt_app_log.c:63), and the two are + # equal only for a main thread on Linux -- scope by the pid, + # which is the point of this skip, and let the tid be. + skip_alert(fr'\b{worker}#\d+ ') + + # The kill retires the start RPC, which releases the slot and -- no + # process being left -- fails the waiting request instead of letting + # it sit forever. + thread.join(timeout=25) + + finally: + if thread.is_alive(): + # Do not leave a thread blocked in recv() behind; the assertion + # below reports the wedge. + thread.join(timeout=10) + + assert not thread.is_alive(), ( + 'the in-flight request never completed: the start RPC was stranded' + ) + assert resp == [503], f'in-flight request: {resp}' + + # The slot is back, so the application is startable again. + deadline = time.time() + 10 + starting = None + + while time.time() < deadline: + starting = client.conf_get('/status')['applications']['slow_start'][ + 'processes' + ]['starting'] + + if starting == 0: + break + + time.sleep(0.2) + + assert starting == 0, f'processes.starting stuck at {starting}' + + # And the next request gets a brand new prototype and is served. + assert client.get(read_timeout=30)['status'] == 200, 'recovered' + assert _proto_pid('slow_start') not in (None, pid), 'fresh prototype' + assert findall(fr'process {pid} exited on signal 9'), 'kill logged' diff --git a/test/test_python_compression.py b/test/test_python_compression.py new file mode 100644 index 000000000..c61df2260 --- /dev/null +++ b/test/test_python_compression.py @@ -0,0 +1,475 @@ +import gzip +import re +import socket +import struct +import threading +import time +import zlib + +import pytest + +from unit.applications.lang.python import ApplicationPython + +prerequisites = {'modules': {'python': 'any'}} + +client = ApplicationPython() + +ENCODINGS = ['gzip', 'deflate', 'zstd', 'br'] + +# Concurrent responses per test. The bug needs two responses on one router +# thread, and the router has one thread per CPU. On 8 CPUs, 24 responses +# made the test fail in each of 10 runs before the fix. +CONCURRENCY = 24 +PARTS = 8 +SIZE = 4096 + +TOKEN = re.compile(rb'<(\d{6})>') + + +def decoder(encoding): + """Returns a function that decodes a whole body, or skips the test.""" + + if encoding == 'gzip': + return gzip.decompress + + if encoding == 'deflate': + return zlib.decompress + + if encoding == 'zstd': + try: + from compression import zstd # Python 3.14 and later. + + return zstd.decompress + except ImportError: + pass + + zstandard = pytest.importorskip('zstandard') + + def decode(data): + d = zstandard.ZstdDecompressor().decompressobj() + out = d.decompress(data) + + if not d.eof: + raise ValueError('zstd frame is not complete') + + return out + + return decode + + return pytest.importorskip('brotli').decompress + + +def setup_compression(encoding, **kwargs): + client.load('compression_stream', threads=32, **kwargs) + + resp = client.conf( + { + "http": { + "compression": { + "types": ["text/plain"], + "compressors": [{"encoding": encoding}], + } + } + }, + 'settings', + ) + + if 'success' in resp: + return + + if 'supported compressor' in resp.get('detail', ''): + pytest.skip(f'unit built without {encoding} compression support') + + pytest.fail(f'could not configure compression: {resp}') + + +# The app sleeps "delay" seconds between writes, so the responses on one +# router thread take turns. The stream tests depend on that: a machine that +# finishes a response in one turn can pass them without the fix. Before the +# fix, 10 of 10 runs failed, with 21 to 24 of 24 bodies bad. +def request_headers(rid, encoding, parts, delay=0.02): + return { + 'Host': 'localhost', + 'Accept-Encoding': encoding, + 'X-Id': str(rid), + 'X-Parts': str(parts), + 'X-Size': str(SIZE), + 'X-Delay': str(delay), + 'Connection': 'close', + } + + +def expected(rid, parts=PARTS): + token = b'<%06d>' % rid + + return (token * (SIZE // len(token) + 1))[:SIZE] * parts + + +def dechunk(data): + body = b'' + + while True: + end = data.find(b'\r\n') + if end == -1: + raise ValueError('the response ends before its last chunk') + + size = int(data[:end], 16) + if size == 0: + return body + + chunk = data[end + 2 : end + 2 + size] + if len(chunk) != size: + raise ValueError('the response ends inside a chunk') + + body += chunk + data = data[end + 2 + size + 2 :] + + +def fetch(rid, encoding): + # latin1 maps each byte to one character, so encoding back gives the + # compressed bytes exactly. + resp = client.get( + headers=request_headers(rid, encoding, PARTS), + raw_resp=True, + encoding='latin1', + read_buffer_size=65536, + ) + + head, _, body = resp.partition('\r\n\r\n') + lines = head.split('\r\n') + + headers = {} + for line in lines[1:]: + name, _, value = line.partition(':') + headers[name.strip()] = value.strip() + + return int(lines[0].split()[1]), headers, body.encode('latin1') + + +def check(rid, encoding, decode): + """Returns None for a correct response, else what is wrong with it.""" + + status, headers, body = fetch(rid, encoding) + + if status != 200: + return f'status {status}' + + if headers.get('Content-Encoding') != encoding: + return f'Content-Encoding {headers.get("Content-Encoding")}' + + # The length of the coded body is not known in advance, so the router + # sends it chunked. + try: + if headers.get('Transfer-Encoding') == 'chunked': + body = dechunk(body) + + data = decode(body) + except Exception as e: # pylint: disable=broad-except + return f'not valid {encoding}: {e!r}' + + foreign = {int(m) for m in TOKEN.findall(data)} - {rid} + if foreign: + return f'holds bytes of requests {sorted(foreign)[:4]}' + + if data != expected(rid): + return f'{len(data)} bytes, expected {len(expected(rid))}' + + return None + + +def burst(encoding, decode, first_rid): + """Sends CONCURRENCY requests at once; returns the bad responses.""" + + results = {} + barrier = threading.Barrier(CONCURRENCY) + + def worker(rid): + try: + barrier.wait(timeout=30) + results[rid] = check(rid, encoding, decode) + except BaseException as e: # pylint: disable=broad-except + results[rid] = repr(e) + + threads = [ + threading.Thread(target=worker, args=(rid,)) + for rid in range(first_rid, first_rid + CONCURRENCY) + ] + + for t in threads: + t.start() + + for t in threads: + t.join(timeout=120) + + assert len(results) == CONCURRENCY, 'every request finished' + + return {rid: why for rid, why in sorted(results.items()) if why} + + +def describe(bad): + return f'{len(bad)} of {CONCURRENCY} bad, first: {list(bad.items())[:3]}' + + +def start_streams(encoding, parts, delay): + """Starts 4 compressed responses and waits for the header of each.""" + + socks = [] + + for rid in range(1001, 1005): + sock = client.get( + headers=request_headers(rid, encoding, parts, delay), + no_recv=True, + ) + sock.settimeout(30) + socks.append(sock) + + # The router initialises the stream before it sends the header. The + # empty line at the end of the header goes out with the first chunk, + # which for zstd can be the last one, so do not wait for it. + coded = b'\r\nContent-Encoding: ' + encoding.encode() + b'\r\n' + + for sock in socks: + data = b'' + while coded not in data: + part = sock.recv(4096) + assert part, 'the response starts' + data += part + + return socks + + +def burst_with(encoding, decode, action): + """Runs action() while a burst of responses streams.""" + + result = {} + + def run_burst(): + result['bad'] = burst(encoding, decode, 1) + + t = threading.Thread(target=run_burst) + t.start() + + # Let the responses of the burst start to stream. + time.sleep(0.05) + + action() + + t.join(timeout=120) + + assert 'bad' in result, 'the burst finished' + + return result['bad'] + + +@pytest.mark.parametrize('encoding', ENCODINGS) +def test_python_compression_concurrent_streams(encoding): + # The compressor state was one object per router thread. A response that + # is compressed in several event loop turns shared it with each other + # response on that thread. A client then got invalid data, the bytes of + # another response, or a router crash. + decode = decoder(encoding) + setup_compression(encoding) + + bad = burst(encoding, decode, 1) + + assert not bad, describe(bad) + + +@pytest.mark.parametrize('encoding', ENCODINGS) +def test_python_compression_client_close(encoding): + # Clients that close the connection in the middle of a compressed body, + # while other compressed responses stream. The router must free the + # stream of each closed response (an ASan build reports a leak if it does + # not), and must not free or use the stream of another response. + decode = decoder(encoding) + setup_compression(encoding) + + socks = start_streams(encoding, 40, 0.02) + + def reset(): + for sock in socks: + sock.setsockopt( + socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0) + ) + sock.close() + + bad = burst_with(encoding, decode, reset) + + assert not bad, describe(bad) + + # The application ends the closed responses after 40 parts. Wait for + # that, then check that the router still serves a correct response. + time.sleep(1) + + assert check(2001, encoding, decode) is None, 'response after the close' + + +@pytest.mark.parametrize('encoding', ENCODINGS) +def test_python_compression_app_timeout(encoding): + # The application stops for longer than "limits": {"timeout"} in the + # middle of a compressed body. The router then closes the connection, + # and must free the stream of the response. A client close does not do + # this for zstd: the router writes no zstd data until the last part, so + # it does not see the close before the stream ends. + decode = decoder(encoding) + setup_compression(encoding, limits={"timeout": 1}) + + socks = start_streams(encoding, 2, 2) + + bad = burst_with(encoding, decode, lambda: None) + + assert not bad, describe(bad) + + for sock in socks: + data = b'' + while True: + part = sock.recv(65536) + if not part: + break + data += part + + sock.close() + + assert not data.endswith(b'0\r\n\r\n'), 'the response is cut' + + # The application ends the slow responses 2 seconds after it starts + # them. Wait for that, then check a response again. + time.sleep(1) + + assert check(2001, encoding, decode) is None, 'response after the timeout' + + +# Large enough to reach the router through shared memory, so the compressor +# takes it (see test_php_compression.py and issue 162 for small bodies). +BODY = gzip.compress(b'A' * 100000, compresslevel=0) + + +def post(body, url='/', accept='gzip', timeout=30): + # The "variables" application echoes the request body with the request's + # Content-Type. It needs a Custom-Header to build its response. Raw + # socket: the response body must not be decoded. + sock = socket.create_connection(('127.0.0.1', 8080), timeout) + sock.settimeout(timeout) + + try: + sock.sendall( + f'POST {url} HTTP/1.1\r\n' + f'Host: localhost\r\n' + f'Content-Type: text/plain\r\n' + f'Custom-Header: 557\r\n' + f'Content-Length: {len(body)}\r\n' + f'Accept-Encoding: {accept}\r\n' + f'Connection: close\r\n\r\n'.encode() + + body + ) + + chunks = [] + while True: + data = sock.recv(256 * 1024) + if not data: + break + chunks.append(data) + + finally: + sock.close() + + head, _, data = b''.join(chunks).partition(b'\r\n\r\n') + lines = head.split(b'\r\n') + + headers = {} + for line in lines[1:]: + name, _, value = line.partition(b':') + headers[name.strip().decode()] = value.strip().decode() + + if headers.get('Transfer-Encoding') == 'chunked': + data = dechunk(data) + + return int(lines[0].split()[1]), headers, data + + +def configure(response_headers, app='variables'): + client.load(app) + + action = {"pass": f"applications/{app}"} + + if response_headers is not None: + action["response_headers"] = response_headers + + assert 'success' in client.conf( + { + "http": { + "compression": { + "compressors": [{"encoding": "gzip", "level": 1}] + } + } + }, + 'settings', + ), 'compression' + assert 'success' in client.conf([{"action": action}], 'routes'), 'routes' + assert 'success' in client.conf( + {"*:8080": {"pass": "routes"}}, 'listeners' + ), 'listeners' + + +def test_python_compression_control(): + # Without a configured Content-Encoding, the echoed body is compressed. + # So the test below passes because of the configured field. + configure(None) + + status, headers, body = post(BODY) + + assert status == 200, 'status' + assert headers.get('Content-Encoding') == 'gzip', 'compressed' + assert gzip.decompress(body) == BODY, 'gzip of the echoed bytes' + + +def test_python_compression_response_headers_content_encoding(): + # https://github.com/freeunitorg/freeunit/issues/557, application path. + # The application sends bytes that are already gzip, and the action + # names their coding in "response_headers". The compressor must not + # code them again: that Content-Encoding replaces the compressor's. + configure({"Content-Encoding": "gzip"}) + + status, headers, body = post(BODY) + + assert status == 200, 'status' + assert headers.get('Content-Encoding') == 'gzip', 'the configured coding' + assert len(body) == len(BODY), 'the application length' + assert body == BODY, 'the application bytes, coded once' + + +def test_python_compression_response_headers_content_encoding_template(): + # A template value that is not safe in a field is never sent. So it + # must not keep the compressor out. Otherwise the body goes out with no + # Content-Encoding to a client that refused identity. + configure({"Content-Encoding": "$arg_c"}) + + status, headers, body = post(BODY, url='/?c=gzip') + + assert status == 200, 'status' + assert headers.get('Content-Encoding') == 'gzip', 'the configured coding' + assert body == BODY, 'the application bytes, coded once' + + status, headers, body = post( + BODY, url='/?c=gzip%0d%0a', accept='gzip, identity;q=0' + ) + + assert status == 200, 'unsafe value: status' + assert headers.get('Content-Encoding') == 'gzip', 'the compressor coding' + assert gzip.decompress(body) == BODY, 'gzip of the echoed bytes' + + +def test_python_compression_app_coding_removed_is_not_coded_again(): + # The application codes its own body and sends its own Content-Encoding. + # A null Content-Encoding in "response_headers" then removes only the + # field. The removal rule is for a coding that Unit would apply, so it + # does not apply here: the bytes are not coded again, and a client that + # refused identity gets no 406. + configure({"Content-Encoding": None}, app='content_encoding') + + for accept in ('gzip', 'gzip, identity;q=0'): + status, headers, body = post(BODY, accept=accept) + + assert status == 200, f'{accept}: status' + assert 'Content-Encoding' not in headers, f'{accept}: removed' + assert body == BODY, f'{accept}: the application bytes' diff --git a/test/test_python_isolation.py b/test/test_python_isolation.py index 4e27282a2..d0b222819 100644 --- a/test/test_python_isolation.py +++ b/test/test_python_isolation.py @@ -8,8 +8,6 @@ from unit.applications.lang.python import ApplicationPython from unit.option import option from unit.utils import findmnt -from unit.utils import waitformount -from unit.utils import waitforunmount prerequisites = {'modules': {'python': 'any'}, 'features': {'isolation': True}} @@ -78,29 +76,271 @@ def test_python_isolation_rootfs(is_su, require, temp_dir): assert ret['body']['FileExists'], 'application exists in rootfs' +def language_deps_mounts(temp_dir): + # The stdlib bind mount lands at , and auto/modules/ + # python picks that directory as the sys.path entry whose basename is + # "pythonX.Y". It is /usr/lib/pythonX.Y only when unit was built against + # the system interpreter; a toolcache, venv or /usr/local build mounts a + # different prefix, so match the stdlib directory rather than assuming one. + return [ + line + for line in findmnt().splitlines() + if line.startswith(f'{temp_dir}/') + and re.search(r'/python\d+\.\d+', line.split(' ')[0]) + ] + + +def waitfor(predicate, timeout=50): + for _ in range(timeout): + if predicate(): + return True + + time.sleep(0.1) + + return False + + def test_python_isolation_rootfs_no_language_deps(require, temp_dir): require({'privileged_user': True}) isolation = {'rootfs': temp_dir, 'automount': {'language_deps': False}} client.load('empty', isolation=isolation) - python_path = f'{temp_dir}/usr' - - assert findmnt().find(python_path) == -1 + assert not language_deps_mounts(temp_dir) assert client.get()['status'] != 200, 'disabled language_deps' - assert findmnt().find(python_path) == -1 + assert not language_deps_mounts(temp_dir) isolation['automount']['language_deps'] = True client.load('empty', isolation=isolation) - assert findmnt().find(python_path) == -1 + assert not language_deps_mounts(temp_dir) assert client.get()['status'] == 200, 'enabled language_deps' - assert waitformount(python_path), 'language_deps mount' + assert waitfor( + lambda: bool(language_deps_mounts(temp_dir)) + ), 'language_deps mount' client.conf({"listeners": {}, "applications": {}}) - assert waitforunmount(python_path), 'language_deps unmount' + assert waitfor( + lambda: not language_deps_mounts(temp_dir) + ), 'language_deps unmount' + + +# The first component of a language_deps destination: "usr" for a distro +# Python, "opt" for a toolcache Python (as in CI). +LANG_DEPS_TOP = ('usr', 'opt') + + +def lang_deps_tops(temp_dir): + # Return the members of LANG_DEPS_TOP that start a language_deps + # destination. The destinations come from the module build. To find + # them, load the application once with a rootfs that has no symlink, + # and read the mount table. + rootfs = Path(temp_dir) / 'plain' + rootfs.mkdir() + + client.load('empty', isolation={'rootfs': str(rootfs)}) + + assert client.get()['status'] == 200, 'plain rootfs' + assert waitfor( + lambda: bool(language_deps_mounts(str(rootfs))) + ), 'plain rootfs language_deps mount' + + tops = { + Path(line.split(' ')[0]).relative_to(rootfs).parts[0] + for line in language_deps_mounts(str(rootfs)) + } + + client.conf({"listeners": {}, "applications": {}}) + + assert waitfor( + lambda: not language_deps_mounts(str(rootfs)) + ), 'plain rootfs language_deps unmount' + + return sorted(tops.intersection(LANG_DEPS_TOP)) + + +def symlinked_rootfs(temp_dir): + # Make each LANG_DEPS_TOP in the rootfs a symlink out of the rootfs. + # mkdir -p follows the symlink and creates decoy/lib/... or + # decoy/hostedtoolcache/... + rootfs = Path(temp_dir) / 'r' + decoy = Path(temp_dir) / 'decoy' + + rootfs.mkdir() + decoy.mkdir() + + for top in LANG_DEPS_TOP: + (rootfs / top).symlink_to('../decoy') + + return rootfs, decoy + + +def wait_for_refusal(rootfs, tops, findall, wait_for_record): + # Wait for the record of the refused destination. Return the pid of + # the prototype and the destination. + tops_re = '|'.join(re.escape(top) for top in tops) + record = ( + fr'\[alert\] (\d+)#\d+ mount destination ' + fr'({re.escape(str(rootfs))}/(?:{tops_re})/\S+): ' + fr'component "(?:{tops_re})" (.*)$' + ) + + found = wait_for_record(record, wait=50) + assert found is not None, 'refusal logged' + + # The walk logs one reason with openat2() and another without it. + # unitd uses openat2() when the build has it, unless the kernel refuses + # the call. Then unitd logs a warning and takes the fallback. + config = Path(option.current_dir) / 'build/include/nxt_auto_config.h' + have_openat2 = re.search( + r'^#define NXT_HAVE_OPENAT2\b', config.read_text(), re.M + ) + no_openat2 = findall(r'openat2\(\) is not supported by the running kernel') + + if have_openat2 and not no_openat2: + reason = 'escapes rootfs or is not a directory: ' + else: + reason = 'is a symlink or not a directory' + + assert found.group(3).startswith(reason), 'refusal reason' + + return found.group(1), found.group(2) + + +def test_python_isolation_rootfs_symlinked_mount_dst( + findall, require, skip_alert, temp_dir, wait_for_record +): + require({'privileged_user': True}) + + # A destination whose path goes through a symlink out of the rootfs + # is refused, and no directory is created behind the symlink. + tops = lang_deps_tops(temp_dir) + if not tops: + pytest.skip('no language_deps destination under /usr or /opt') + + rootfs, decoy = symlinked_rootfs(temp_dir) + + skip_alert(r'mount destination .*', r'failed to apply', r'process .* exited') + + client.load('empty', isolation={'rootfs': str(rootfs)}) + + assert client.get()['status'] != 200, 'symlinked destination refused' + + wait_for_refusal(rootfs, tops, findall, wait_for_record) + + assert sorted(p.name for p in decoy.iterdir()) == [], 'decoy is empty' + + +def test_python_isolation_rootfs_symlinked_mount_dst_no_unmount( + findall, require, skip_alert, temp_dir, wait_for_record +): + require({'privileged_user': True}) + + # When the prototype refuses a destination because of a symlink, it + # must not unmount that destination. The path goes through the + # symlink, and without a mount namespace umount2() runs in the mount + # namespace of unitd. + tops = lang_deps_tops(temp_dir) + if not tops: + pytest.skip('no language_deps destination under /usr or /opt') + + rootfs, _ = symlinked_rootfs(temp_dir) + + skip_alert(r'mount destination .*', r'failed to apply', r'process .* exited') + + client.load('empty', isolation={'rootfs': str(rootfs)}) + client.get() + + pid, dst = wait_for_refusal(rootfs, tops, findall, wait_for_record) + dst = re.escape(dst) + + assert wait_for_record(fr'process {pid} exited'), 'prototype exited' + + assert not findall( + fr'\] {pid}#\d+ umount2\({dst},' + ), 'refused destination is not unmounted' + + +def test_python_isolation_rootfs_missing(require, skip_alert, temp_dir): + require({'privileged_user': True}) + + # A missing rootfs is refused. It is not created as root. The conf + # is sent directly: client.load() would create the rootfs. + missing = Path(temp_dir) / 'missing' + script_path = f'{option.test_dir}/python/empty' + + skip_alert(r'open rootfs', r'failed to apply', r'process .* exited') + + assert 'success' in client.conf( + { + "listeners": {"*:8080": {"pass": "applications/empty"}}, + "applications": { + "empty": { + "type": "python", + "processes": {"spare": 0}, + "path": script_path, + "working_directory": script_path, + "module": "wsgi", + "isolation": {"rootfs": str(missing / 'r')}, + } + }, + } + ) + + assert client.get()['status'] != 200, 'missing rootfs refused' + assert not missing.exists(), 'missing rootfs not created' + + +def test_python_isolation_rootfs_credential_language_deps( + is_su, require, temp_dir +): + if not is_su: + require( + {'features': {'isolation': ['unprivileged_userns_clone', 'user']}} + ) + else: + require({'features': {'isolation': ['user']}}) + + client.load('empty') + + # A new user namespace without a new mount namespace cannot mount + # anything. With "procfs" and "tmpfs" off the only mounts left are the + # language dependencies, which this module always declares (auto/modules/ + # python emits at least the stdlib directory), so the config still could + # never start and must be refused -- naming "language_deps", and only + # "language_deps", as the automount to switch off. + resp = client.conf( + { + 'rootfs': temp_dir, + 'namespaces': {'credential': True}, + 'automount': {'procfs': False, 'tmpfs': False}, + }, + 'applications/empty/isolation', + ) + + assert 'error' in resp, 'language_deps automount rejected' + + detail = resp.get('detail', '') + + assert '"language_deps": false' in detail, 'detail names the knob' + assert 'procfs' not in detail, 'disabled procfs not named' + assert 'tmpfs' not in detail, 'disabled tmpfs not named' + + # Nothing left to mount, so the very same config becomes valid. + assert 'success' in client.conf( + { + 'rootfs': temp_dir, + 'namespaces': {'credential': True}, + 'automount': { + 'procfs': False, + 'tmpfs': False, + 'language_deps': False, + }, + }, + 'applications/empty/isolation', + ), 'every automount off accepted' def test_python_isolation_procfs(require, temp_dir): @@ -155,6 +395,35 @@ def set_cgroup_path(path): assert len(cgroup_rel.parts) >= len(cgroup_abs.parts) +@pytest.mark.parametrize( + 'cgroup', [{'path': 'scope/python'}, None], ids=['path', 'no_path'] +) +def test_python_isolation_cgroup_namespace(require, cgroup): + require({'privileged_user': True, 'features': {'isolation': ['cgroup']}}) + + isolation = {'namespaces': {'cgroup': True}} + + if cgroup is not None: + isolation['cgroup'] = cgroup + + client.load('ns_inspect', isolation=isolation) + + ret = client.getjson(url='/?read=/proc/self/cgroup') + + content = ret['body']['FileContent'] + assert content is not None, 'read /proc/self/cgroup' + assert [ + l for l in content.splitlines() if l.startswith('0::') + ] == ['0::/'], 'cgroup namespace is rooted at the app cgroup' + + if cgroup is not None: + # The view inside is "/" also when the process is not moved. + assert Path(get_cgroup('ns_inspect')).parts[-2:] == ( + 'scope', + 'python', + ), 'app process is in the configured cgroup' + + def test_python_isolation_cgroup_two(require): require({'privileged_user': True, 'features': {'isolation': ['cgroup']}}) diff --git a/test/test_reconfigure_tls.py b/test/test_reconfigure_tls.py index 4f7d344ae..291000ca0 100644 --- a/test/test_reconfigure_tls.py +++ b/test/test_reconfigure_tls.py @@ -120,3 +120,52 @@ def test_reconfigure_tls_3(): clear_conf() assert client.get(sock=ssl_sock)['status'] == 408, 'request timeout' + + +@pytest.mark.parametrize('close', ['client', 'idle_timeout']) +def test_reconfigure_tls_keepalive_close(close): + # A keep-alive TLS connection that is closed after a reconfiguration. + # + # The listener stays, so the reconfiguration replaces its joint at once + # and releases the old one. Only the request held the old router + # configuration, and that reference went with the request. The idle + # connection's TLS state still points at the listener's nxt_tls_conf_t + # in that configuration, and the TLS shutdown at close read it: a + # heap-use-after-free under ASan, which the fixture's log check reports. + if close == 'idle_timeout': + assert 'success' in client.conf( + {'http': {'idle_timeout': 1}}, 'settings' + ) + + (resp, sock) = client.get_ssl( + headers={'Host': 'localhost', 'Connection': 'keep-alive'}, + start=True, + read_timeout=1, + ) + + assert resp['status'] == 200, 'keep-alive request' + + assert 'success' in client.conf({'return': 201}, 'routes/0/action') + + if close == 'client': + sock.close() + + else: + # The router closes the idle connection itself. It may write a + # 408 response first (nxt_h1p_idle_response()), so read up to EOF. + sock.settimeout(5) + + try: + while sock.recv(4096) != b'': + pass + except TimeoutError: + pytest.fail('the router did not close the idle connection') + except OSError: + # An SSL error or a reset: the router closed it. + pass + + # Let the router run the TLS shutdown of the closed connection; it must + # still be alive and serve the new configuration. + time.sleep(0.5) + + assert client.get_ssl()['status'] == 201, 'router alive' diff --git a/test/test_routing.py b/test/test_routing.py index 0d54094a1..39ed01de5 100644 --- a/test/test_routing.py +++ b/test/test_routing.py @@ -297,6 +297,60 @@ def test_routes_bad_regex(require): assert client.get(url='/nothing_z')['status'] == 500, '/nothing_z' +def test_routes_regex_match_limit(require, wait_for_record, findall): + require({'modules': {'regex': True}}) + + # PCRE2 10.4x and PCRE 8.x try each split of the commas between the two + # groups, about 2M steps. They do not see that no "x" or "y" is in the + # subject. + route_match({"headers": {"x-blah": "~^(.*),(.*)[xy]$"}}) + + assert ( + client.get(headers={'Host': 'localhost', 'X-Blah': ',' * 2000, + 'Connection': 'close'})['status'] + == 500 + ), 'match limit' + + assert ( + wait_for_record(r'\[warn\].+reached the match limit on 2000 bytes') + is not None + ), 'match limit log' + assert not findall(r',{64}'), 'no subject in the log' + + assert ( + client.get(headers={'Host': 'localhost', 'X-Blah': 'a,bx', + 'Connection': 'close'})['status'] + == 200 + ), 'short subject' + + +def test_routes_regex_match_limit_unanchored(require, wait_for_record): + require({'modules': {'regex': True}}) + + # The library counts the steps from zero again at each start position. + # This pattern takes about 8,000 steps at each of the first 976 + # positions, below the limit at each one. The limit must stop the + # whole match. + route_match({"headers": {"x-blah": "~(?=a)(?:a|aa){0,12}[xy]"}}) + + assert ( + client.get(headers={'Host': 'localhost', 'X-Blah': 'a' * 1000 + 'x', + 'Connection': 'close'})['status'] + == 500 + ), 'match limit' + + assert ( + wait_for_record(r'\[warn\].+reached the match limit on 1001 bytes') + is not None + ), 'match limit log' + + assert ( + client.get(headers={'Host': 'localhost', 'X-Blah': 'aax', + 'Connection': 'close'})['status'] + == 200 + ), 'short subject' + + def test_routes_match_regex_captures(require): require({'modules': {'regex': True}}) diff --git a/test/test_settings.py b/test/test_settings.py index 9d37d6cae..44a13e72a 100644 --- a/test/test_settings.py +++ b/test/test_settings.py @@ -1,11 +1,14 @@ import re import socket +import ssl import subprocess import time import pytest from unit.applications.lang.python import ApplicationPython +from unit.applications.tls import ApplicationTLS +from unit.option import option prerequisites = {'modules': {'python': 'any'}} @@ -309,6 +312,826 @@ def req(addr, data_len): assert len(data) > data_len, 'send timeout data 2' +def test_settings_send_timeout_recovery(temp_dir): + # test_settings_send_timeout checks the framing of a single aborted + # response. Nothing asserted that the router is still usable afterwards, + # so a request that fails to complete on the abort path -- e.g. its last + # buffer completed twice, or never -- would surface only indirectly. Abort + # several streamed responses in a row over a unix listener, then serve a + # normal request on the same listener. + client.load('body_generate') + + def req(addr, data_len): + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + + # A router that stops serving after an abort is exactly what this test + # looks for; without a timeout that failure would hang the run instead + # of failing it. + sock.settimeout(30) + sock.connect(addr) + + req = f"""GET / HTTP/1.1 +Host: localhost +X-Length: {data_len} +Connection: close + +""" + + sock.sendall(req.encode()) + + data = sock.recv(16).decode() + + time.sleep(3) + + data += client.recvall(sock).decode() + + sock.close() + + return data + + sysctl_out = sysctl() + values = re.findall(r'net.core.[rw]mem_(?:max|default).*?(\d+)', sysctl_out) + values = [int(v) for v in values] + + data_len = 1048576 if len(values) == 0 else 10 * max(values) + + addr = f'{temp_dir}/sock' + + assert 'success' in client.conf( + {f'unix:{addr}': {'application': 'body_generate'}}, 'listeners' + ) + + assert 'success' in client.conf({'http': {'send_timeout': 1}}, 'settings') + + try: + for i in range(3): + data = req(addr, data_len) + assert re.search(r'200 OK', data), f'aborted status {i}' + assert len(data) < data_len, f'aborted data {i}' + + data = req(addr, 10) + assert re.search(r'200 OK', data), 'recovery status' + assert re.search(r'XXXXXXXXXX$', data), 'recovery data' + + finally: + # The default no-restart suite preserves /settings between tests. + client.conf_delete('settings/http/send_timeout') + + +def min_rate_conf(conf): + assert 'success' in client.conf({'http': conf}, 'settings') + + +def min_rate_reset(): + # The default no-restart suite preserves /settings between tests. + for name in ( + 'body_read_timeout', + 'body_min_rate', + 'send_timeout', + 'send_min_rate', + 'chunked_transform', + ): + client.conf_delete(f'settings/http/{name}') + + +BODY_RATE_RECORD = r'client body rate is less than body_min_rate' + + +def min_rate_slow_body(sock, part, limit): + # Sends "part" about once each second for at most "limit" seconds, and + # reads what the router sends back. Returns (closed, data, elapsed). + sock.settimeout(1) + + start = time.monotonic() + data = b'' + closed = False + + while time.monotonic() - start < limit: + try: + sock.sendall(part) + except OSError: + closed = True + break + + try: + chunk = sock.recv(4096) + except socket.timeout: + continue + except OSError: + closed = True + break + + if not chunk: + closed = True + break + + data += chunk + + return closed, data, time.monotonic() - start + + +@pytest.mark.parametrize('timeout', [2, 0]) +def test_settings_body_min_rate_slow_body(timeout, wait_for_record): + # A client sends the body 1 byte each second. Each byte starts the + # body_read_timeout gap timer again, so only the rate floor can stop + # this client. The floor check starts after the grace time (equal to + # body_read_timeout, 2 s). The first body read after the grace time + # must give 408. With body_read_timeout 0 there is no gap timer and + # no grace time, and each read is checked against the time since the + # previous read. The first slow read must give 408. + client.load('empty') + + min_rate_conf({'body_read_timeout': timeout, 'body_min_rate': 256}) + + try: + sock = socket.create_connection(('127.0.0.1', 8080)) + sock.settimeout(1) + sock.sendall( + b'POST / HTTP/1.1\r\n' + b'Host: localhost\r\n' + b'Content-Length: 10000\r\n' + b'Connection: close\r\n\r\n' + ) + + closed, data, elapsed = min_rate_slow_body(sock, b'x', 12) + sock.close() + + # The router closes the connection after the 408 response. The + # client can get a reset before it reads the response, because + # the router does not read the rest of the body. + assert closed or data, 'slow body stopped' + assert data == b'' or data.startswith(b'HTTP/1.1 408'), 'slow body 408' + assert elapsed < 8, 'slow body stopped in time' + assert wait_for_record(BODY_RATE_RECORD) is not None, 'slow body log' + + finally: + min_rate_reset() + + +def test_settings_body_min_rate_chunked(wait_for_record): + # The same slow client as in the slow body test, with a chunked body. + # The client sends a chunk of 1 byte each second. The chunked body is + # read in the same function as a body with Content-Length, but the + # chunk parser runs there first. The floor counts the chunk framing + # too: 6 bytes each second, which is much less than 256. + client.load('empty') + + min_rate_conf( + { + 'body_read_timeout': 2, + 'body_min_rate': 256, + 'chunked_transform': True, + } + ) + + try: + sock = socket.create_connection(('127.0.0.1', 8080)) + sock.sendall( + b'POST / HTTP/1.1\r\n' + b'Host: localhost\r\n' + b'Transfer-Encoding: chunked\r\n' + b'Connection: close\r\n\r\n' + ) + + closed, data, elapsed = min_rate_slow_body(sock, b'1\r\nx\r\n', 12) + sock.close() + + assert closed or data, 'chunked body stopped' + assert data == b'' or data.startswith( + b'HTTP/1.1 408' + ), 'chunked body 408' + assert elapsed < 8, 'chunked body stopped in time' + assert wait_for_record(BODY_RATE_RECORD) is not None, 'chunked log' + + finally: + min_rate_reset() + + +def test_settings_body_min_rate_tls(wait_for_record): + # The same slow client as in the slow body test, on a TLS listener. + # The floor counts the bytes that the TLS layer gives to the router, + # so 1 byte each second is still below 256. + if not option.available['modules'].get('openssl'): + pytest.skip('requires openssl') + + client.load('empty') + + ApplicationTLS().certificate() + + assert 'success' in client.conf( + { + "pass": "applications/empty", + "tls": {"certificate": "default"}, + }, + 'listeners/*:8080', + ) + + min_rate_conf({'body_read_timeout': 2, 'body_min_rate': 256}) + + context = ssl.create_default_context() + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + + try: + sock = context.wrap_socket( + socket.create_connection(('127.0.0.1', 8080)) + ) + sock.sendall( + b'POST / HTTP/1.1\r\n' + b'Host: localhost\r\n' + b'Content-Length: 10000\r\n' + b'Connection: close\r\n\r\n' + ) + + closed, data, elapsed = min_rate_slow_body(sock, b'x', 12) + sock.close() + + assert closed or data, 'TLS slow body stopped' + assert data == b'' or data.startswith( + b'HTTP/1.1 408' + ), 'TLS slow body 408' + assert elapsed < 8, 'TLS slow body stopped in time' + assert wait_for_record(BODY_RATE_RECORD) is not None, 'TLS log' + + finally: + min_rate_reset() + + +def test_settings_body_min_rate_burst_then_slow(wait_for_record): + # A client sends half of the body at once, then 1 byte each second. + # With one average over the whole body, the burst would give credit + # for about 195 s at 256 bytes per second. The rate is checked for + # each window of at least body_read_timeout, so the first slow window + # must give 408. + client.load('empty') + + min_rate_conf({'body_read_timeout': 2, 'body_min_rate': 256}) + + try: + sock = socket.create_connection(('127.0.0.1', 8080)) + sock.settimeout(1) + sock.sendall( + b'POST / HTTP/1.1\r\n' + b'Host: localhost\r\n' + b'Content-Length: 100000\r\n' + b'Connection: close\r\n\r\n' + ) + sock.sendall(b'x' * 50000) + + closed, data, elapsed = min_rate_slow_body(sock, b'x', 20) + sock.close() + + assert closed or data, 'burst then slow body stopped' + assert data == b'' or data.startswith( + b'HTTP/1.1 408' + ), 'burst then slow body 408' + assert elapsed < 12, 'burst then slow body stopped in time' + assert ( + wait_for_record(BODY_RATE_RECORD) is not None + ), 'burst then slow body log' + + finally: + min_rate_reset() + + +@pytest.mark.parametrize('timeout', [2, 0]) +def test_settings_body_min_rate_normal_body(timeout): + # A body that is sent faster than the floor must not be stopped, also + # with body_read_timeout 0, when each read is checked. The second + # request on the same keep-alive connection comes after an idle time + # that is longer than the grace time. It must start with new rate + # state. + client.load('empty') + + min_rate_conf({'body_read_timeout': timeout, 'body_min_rate': 256}) + + def req(sock, close): + sock.sendall( + b'POST / HTTP/1.1\r\n' + b'Host: localhost\r\n' + b'Content-Length: 100000\r\n' + + (b'Connection: close\r\n' if close else b'') + + b'\r\n' + ) + + for _ in range(10): + time.sleep(0.2) + sock.sendall(b'x' * 10000) + + return sock.recv(4096) + + try: + sock = socket.create_connection(('127.0.0.1', 8080)) + sock.settimeout(10) + + assert req(sock, False).startswith(b'HTTP/1.1 200'), 'first' + + time.sleep(3) + + assert req(sock, True).startswith(b'HTTP/1.1 200'), 'second' + + sock.close() + + finally: + min_rate_reset() + + +CONTINUE = b'HTTP/1.1 100 Continue\r\n\r\n' + + +def min_rate_expect(length): + # Sends a header with "Expect: 100-continue" and reads the 100. The + # client sends no body byte before it gets the 100. + sock = socket.create_connection(('127.0.0.1', 8080)) + sock.sendall( + b'POST / HTTP/1.1\r\n' + b'Host: localhost\r\n' + b'Expect: 100-continue\r\n' + + f'Content-Length: {length}\r\n'.encode() + + b'Connection: close\r\n\r\n' + ) + + data = b'' + sock.settimeout(5) + + while len(data) < len(CONTINUE): + try: + part = sock.recv(len(CONTINUE) - len(data)) + except socket.timeout: + break + + if not part: + break + + data += part + + assert data == CONTINUE, '100 before the body' + + return sock + + +def test_settings_body_min_rate_expect_slow_body(wait_for_record): + # After a 100 (Continue), the body read state starts in another + # function. The floor must be active there too. Otherwise a client + # can avoid the floor with "Expect: 100-continue". The client sends + # 1 byte each second after the 100 and must get 408. + client.load('empty') + + min_rate_conf({'body_read_timeout': 2, 'body_min_rate': 256}) + + try: + sock = min_rate_expect(10000) + sock.settimeout(1) + + closed, data, elapsed = min_rate_slow_body(sock, b'x', 12) + sock.close() + + assert closed or data, 'slow body after 100 stopped' + assert data == b'' or data.startswith( + b'HTTP/1.1 408' + ), 'slow body after 100 408' + assert elapsed < 8, 'slow body after 100 stopped in time' + assert ( + wait_for_record(BODY_RATE_RECORD) is not None + ), 'slow body after 100 log' + + finally: + min_rate_reset() + + +def test_settings_body_min_rate_expect_wait(): + # The client reads the 100, waits 1 s, and then sends the body much + # faster than the floor. The wait is shorter than body_read_timeout. + # Like the gap timer, the floor counts the wait in the first window. + # The client sends enough bytes before the first check, at 2 s, so + # it must get 200. + client.load('empty') + + min_rate_conf({'body_read_timeout': 2, 'body_min_rate': 256}) + + try: + sock = min_rate_expect(20000) + + time.sleep(1) + + try: + for _ in range(20): + sock.sendall(b'x' * 1000) + time.sleep(0.1) + + except OSError: + # The router answered early. The status shows why. + pass + + sock.settimeout(10) + + try: + data = sock.recv(4096) + except OSError: + data = b'' + + sock.close() + + assert data.startswith(b'HTTP/1.1 200'), 'wait after 100' + + finally: + min_rate_reset() + + +@pytest.mark.parametrize('timeout', [3, 0]) +def test_settings_send_min_rate_slow_read(timeout, system, wait_for_record): + # A client reads a large response slower than send_min_rate, but fast + # enough that the router can write some data before each send_timeout. + # Each write starts the send_timeout gap timer again, so only the rate + # floor can stop this client. With send_timeout 0 there is no gap + # timer and no grace time, and each write is checked against the time + # since the previous write. + # + # The router counts the bytes that the kernel accepts. On loopback the + # kernel send buffer grows to some MiB, and the kernel signals a write + # event only when about one third of the buffer is free. A client that + # reads only some bytes each second thus gets no write events, and the + # gap timer stops it. To test the floor, this test uses larger numbers: + # the floor is 8 MiB/s and the client reads about 1 MiB/s. The logic + # is the same as for a floor of 256 B/s on a real network. + # + # After the router closes the connection, the kernel still gives the + # client the data in the socket buffers. Thus the test finds the close + # in the router log. The time limits are generous because the kernel + # buffer sizes and the time of the write events are not exact. + # + # The numbers above are for the Linux loopback buffers. Other systems + # size and signal the send buffer in a different way. + if system != 'Linux': + pytest.skip('Linux loopback buffer sizes only') + + client.load('body_generate') + + min_rate_conf({'send_timeout': timeout, 'send_min_rate': 8388608}) + + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 262144) + sock.settimeout(1) + sock.connect(('127.0.0.1', 8080)) + sock.sendall( + b'GET / HTTP/1.1\r\n' + b'Host: localhost\r\n' + b'X-Length: 67108864\r\n' + b'Connection: close\r\n\r\n' + ) + + start = time.monotonic() + total = 0 + closed = False + + while time.monotonic() - start < 12: + time.sleep(0.05) + + try: + chunk = sock.recv(65536) + except socket.timeout: + continue + except OSError: + closed = True + break + + if not chunk: + closed = True + break + + total += len(chunk) + + assert ( + wait_for_record( + r'client send rate is less than send_min_rate', wait=50 + ) + is not None + ), 'slow read stopped by the floor' + + # Read the data that the kernel keeps. Then the close must come. + sock.settimeout(5) + + try: + while not closed: + chunk = sock.recv(65536) + if not chunk: + closed = True + total += len(chunk) + except OSError: + closed = True + + sock.close() + + assert closed, 'slow read connection closed' + assert total < 67108864, 'slow read response cut' + + finally: + min_rate_reset() + + +@pytest.mark.parametrize('timeout', [2, 0]) +def test_settings_send_min_rate_fast_read(timeout): + # A client that reads at full speed must get all the data, also with + # send_timeout 0, when each write is checked. Two responses on one + # keep-alive connection check that the second response starts with + # new rate state. + client.load('body_generate') + + min_rate_conf({'send_timeout': timeout, 'send_min_rate': 256}) + + try: + for _ in range(2): + resp = client.get( + headers={ + 'Host': 'localhost', + 'X-Length': '4194304', + 'Connection': 'close', + }, + ) + assert resp['status'] == 200 + assert len(resp['body']) == 4194304 + + sock = socket.create_connection(('127.0.0.1', 8080)) + sock.settimeout(10) + + for close in (False, True): + time.sleep(3) + sock.sendall( + b'GET / HTTP/1.1\r\n' + b'Host: localhost\r\n' + b'X-Length: 1048576\r\n' + + (b'Connection: close\r\n' if close else b'') + + b'\r\n' + ) + + data = b'' + while b'\r\n\r\n' not in data: + data += sock.recv(65536) + + head, body = data.split(b'\r\n\r\n', 1) + assert head.startswith(b'HTTP/1.1 200'), 'keep-alive status' + + while len(body) < 1048576: + chunk = sock.recv(65536) + assert chunk, 'keep-alive body' + body += chunk + + assert len(body) == 1048576, 'keep-alive length' + + sock.close() + + finally: + min_rate_reset() + + +def test_settings_send_min_rate_slow_application(): + # The send floor counts only the time when response data waits for the + # client. The time when the router waits for the application is not + # counted. Thus a slow application (for example a stream of events) + # is not stopped, also with a high floor. The application sends 4 + # parts with 1 s between them, which is longer than the grace time + # (send_timeout 2). The client reads at full speed. + client.load('delayed') + + min_rate_conf({'send_timeout': 2, 'send_min_rate': 8388608}) + + try: + body = '0123456789' * 400 + + resp = client.post( + headers={ + 'Host': 'localhost', + 'X-Parts': '4', + 'X-Delay': '1', + 'Connection': 'close', + }, + body=body, + read_timeout=30, + ) + + assert resp['status'] == 200, 'slow application status' + assert resp['body'] == body, 'slow application body' + + finally: + min_rate_reset() + + +def test_settings_send_min_rate_gap_longer_than_grace(search_in_file): + # The time between send periods is not counted, also when it is longer + # than the grace time. The application sends 1000 bytes, waits 2.5 s, + # and sends 1000 bytes more. The grace time is 2 s. The client reads + # at full speed, so the first period stops after about 1 ms. The second + # period starts again from its own write, and its first check sees about + # 2 ms, not 2.5 s. If the gap was counted, the check after the second + # write would see about 2000 bytes in 2.5 s, below the floor. That + # write sends the last data, so the client still gets the full body, + # but the router closes the connection. Thus the test sends a second + # request on the same connection. On a router that counts the gap, + # that request finds the connection closed ('gap header'), and this + # assertion fails before the log search. + client.load('send_bursts') + + min_rate_conf({'send_timeout': 2, 'send_min_rate': 1048576}) + + sock = socket.create_connection(('127.0.0.1', 8080)) + sock.settimeout(10) + + def response(length): + data = b'' + while b'\r\n\r\n' not in data: + chunk = sock.recv(65536) + assert chunk, 'gap header' + data += chunk + + head, body = data.split(b'\r\n\r\n', 1) + assert head.startswith(b'HTTP/1.1 200'), 'gap status' + + while len(body) < length: + chunk = sock.recv(65536) + assert chunk, 'gap body' + body += chunk + + return body + + try: + start = time.monotonic() + + sock.sendall( + b'GET / HTTP/1.1\r\n' + b'Host: localhost\r\n' + b'X-Part-Size: 1000\r\n' + b'X-Bursts: 2\r\n' + b'X-Interval: 2.5\r\n\r\n' + ) + + assert response(2000) == b'x' * 2000, 'gap body' + assert time.monotonic() - start >= 2.5, 'gap time' + + sock.sendall( + b'GET / HTTP/1.1\r\n' + b'Host: localhost\r\n' + b'X-Part-Size: 10\r\n' + b'Connection: close\r\n\r\n' + ) + + assert response(10) == b'x' * 10, 'gap keep-alive body' + + assert ( + search_in_file(r'client send rate is less than send_min_rate') + is None + ), 'gap not stopped by the floor' + + finally: + sock.close() + min_rate_reset() + + +def test_settings_send_min_rate_stream_slow_read( + system, temp_dir, wait_for_record +): + # An application sends a stream in bursts, 1.5 s apart. The client + # keeps the socket send buffer full. Each 1.5 s it reads only one burst. + # Then the parts of the burst that wait can go out before send_timeout. + # Thus each write event empties the router write queue, and each send + # period stops there. The router must check the rate at that point + # too. Otherwise it never checks this client, and the client reads + # below the floor for as long as the stream lasts. + # + # The test uses a unix socket. On Linux, the kernel takes a unix socket + # write of 32 KiB whole or not at all, and the send buffer has the fixed + # size net.core.wmem_default. A write event comes when 3/4 of the + # buffer is free. Thus a part that does not fit waits whole, and the + # next write event sends all the parts that wait. No write sends only + # some of the queued data. + if system != 'Linux': + pytest.skip('Linux unix socket buffers only') + + # A new unix socket gets this size, the router socket too. A container + # can have no /proc/sys/net/core/wmem_default, so ask a socket. + pair = socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM) + wmem = pair[0].getsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF) + pair[0].close() + pair[1].close() + + # The first burst fills the send buffer, and 2 or 3 parts wait. Each + # next burst is a little more than the space that a write event frees. + # The client reads one burst each time, so 2 or 3 parts wait in each + # cycle, for about 1 s. With wmem_default 212992: 9, then 6 parts. + part = 32768 + first = wmem // part + 3 + parts = wmem * 3 // 4 // part + 2 + bursts = 5 + length = part * (first + parts * (bursts - 1)) + + client.load('send_bursts') + + addr = f'{temp_dir}/sock' + + assert 'success' in client.conf( + {f'unix:{addr}': {'application': 'send_bursts'}}, 'listeners' + ) + + # The router measures about one burst for each 1 s of waiting. The + # floor is four bursts each second. + min_rate_conf({'send_timeout': 2, 'send_min_rate': 4 * parts * part}) + + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + start = time.monotonic() + + def read(size): + # Returns "size" bytes, or less if the connection closes. + data = b'' + + while len(data) < size: + try: + chunk = sock.recv(min(65536, size - len(data))) + except OSError: + break + + if not chunk: + break + + data += chunk + + return data + + try: + sock.settimeout(5) + sock.connect(addr) + sock.sendall( + b'GET / HTTP/1.1\r\n' + b'Host: localhost\r\n' + + f'X-First: {first}\r\n'.encode() + + f'X-Parts: {parts}\r\n'.encode() + + f'X-Part-Size: {part}\r\n'.encode() + + f'X-Bursts: {bursts}\r\n'.encode() + + b'X-Interval: 1.5\r\n' + b'Connection: close\r\n\r\n' + ) + start = time.monotonic() + + head = b'' + while not head.endswith(b'\r\n\r\n'): + byte = read(1) + if not byte: + break + + head += byte + + assert head.startswith(b'HTTP/1.1 200'), 'stream status' + + total = 0 + + for burst in range(bursts): + time.sleep(max(0, start + 1.5 * burst + 1 - time.monotonic())) + + size = parts * part if burst < bursts - 1 else length - total + got = len(read(size)) + total += got + + if got < size: + break + + assert total < length, 'slow stream reader stopped' + + assert ( + wait_for_record( + r'client send rate is less than send_min_rate', wait=50 + ) + is not None + ), 'slow stream reader stopped by the floor' + + finally: + sock.close() + min_rate_reset() + + # The application still runs its schedule. Let it end before the + # next test changes the configuration. + time.sleep(max(0, start + 7 - time.monotonic())) + + +def test_settings_min_rate_validation(): + client.load('empty') + + try: + for name in ('body_min_rate', 'send_min_rate'): + assert 'error' in client.conf({'http': {name: -1}}, 'settings') + assert 'error' in client.conf({'http': {name: 1.5}}, 'settings') + assert 'error' in client.conf({'http': {name: '256'}}, 'settings') + assert 'error' in client.conf( + {'http': {name: 2147483648}}, 'settings' + ) + assert 'success' in client.conf({'http': {name: 0}}, 'settings') + assert 'success' in client.conf( + {'http': {name: 2147483647}}, 'settings' + ) + assert 'success' in client.conf({'http': {name: 256}}, 'settings') + + finally: + min_rate_reset() + + def test_settings_idle_timeout(): client.load('empty') diff --git a/test/test_state_store.py b/test/test_state_store.py new file mode 100644 index 000000000..96b179e86 --- /dev/null +++ b/test/test_state_store.py @@ -0,0 +1,412 @@ +"""Persistence of the state directory (issue #215). + +The controller hands each accepted configuration to the main process, which +writes it to /conf.json. That write used to go straight into +conf.json with O_TRUNC, so a store that could not finish left the file +truncated -- and the short-write path unlinked it outright, which is what +this test observes on the unfixed code: a configuration too large for the +filesystem leaves no conf.json at all. + +The failure is arranged by putting the state directory on a 64 KiB tmpfs, +which needs root and CAP_SYS_ADMIN (pytest as root in a --privileged +container) as well as --restart; the test skips otherwise. + +Main runs each store in a short-lived child process, one at a time, and waits +for it before it exits (https://github.com/freeunitorg/freeunit/issues/516). +The other tests here check that a configuration answered with 200 is on disk +after an immediate SIGTERM, and that stores in quick succession leave the +last configuration. They need --restart only. They catch a regression only +when a store takes longer than the few milliseconds main needs to exit, so on +a tmpfs they pass either way. + +The version tests seed a state directory by hand. unitd writes the version +file with no line end, but a file written by hand often has one. +""" + +import json +import os +import re +import shutil +import signal +import subprocess +import tempfile +import time +from pathlib import Path + +import pytest + +from conftest import unit_run, unit_stop +from unit.applications.proto import ApplicationProto +from unit.log import Log +from unit.utils import waitforsocket + +client = ApplicationProto() + + +SMALL_CONF = { + "listeners": {"*:8080": {"pass": "routes"}}, + "routes": [{"action": {"return": 200}}], +} + + +def big_conf(routes): + """Far more than the 64 KiB the filesystem below has.""" + + return { + "listeners": {"*:8080": {"pass": "routes"}}, + "routes": [ + {"match": {"uri": f"/{'p' * 200}{i}"}, "action": {"return": 200}} + for i in range(routes) + ], + } + + +def wait_for_stored(statedir, expected, wait=100): + """The store is asynchronous: the controller answers the PUT and only + then asks the main process to persist it.""" + + conf_json = statedir / 'conf.json' + + for _ in range(wait): + try: + conf = json.loads(conf_json.read_text(encoding='utf-8')) + if conf.get('listeners') == expected['listeners']: + return conf + except (OSError, ValueError): + pass + + time.sleep(0.1) + + return None + + +def test_state_store_full_filesystem(requires_restart, skip_alert): + """A store that runs out of space must not damage the stored config.""" + + if os.geteuid() != 0: + pytest.skip('requires root to mount a tmpfs') + + unit_stop() + + statedir = Path(tempfile.mkdtemp(prefix='unit-state-')) + + mounted = subprocess.run( + ['mount', '-t', 'tmpfs', '-o', 'size=64k', 'none', str(statedir)], + check=False, + capture_output=True, + ) + + if mounted.returncode != 0: + pytest.skip(f'could not mount a tmpfs: {mounted.stderr.decode()}') + + # The two alerts of the failed store, and the alert of main about it. + skip_alert( + r'failed to store current configuration', + r'write\(.*conf\.json\.tmp', + r'state store child \d+ failed', + ) + + try: + # main creates and writes everything here as root, so this only has + # to be traversable. 0777 would be the very condition the store is + # hardened against -- a state directory another user can plant a + # symbolic link in. + os.chmod(statedir, 0o755) + + unit_run(state_dir=str(statedir)) + + # The configuration that must survive. + assert 'success' in client.conf(SMALL_CONF), 'the small store' + assert wait_for_stored(statedir, SMALL_CONF) is not None, 'stored' + + stored = (statedir / 'conf.json').read_bytes() + + assert 'success' in client.conf(big_conf(400)), 'the large PUT' + + # The store is attempted asynchronously; wait for it to give up. + # The store child logs the first alert, and main logs the second + # when it reaps the child. + assert Log.wait_for_record( + r'failed to store current configuration' + ), 'the store child failed' + assert Log.wait_for_record( + r'state store child \d+ failed' + ), 'main logged the failed store' + + # The store failed; the previously stored configuration is intact, + # byte for byte, and still parses. + after = (statedir / 'conf.json').read_bytes() + + assert after == stored, ( + 'conf.json was damaged by a store that could not complete' + ) + assert json.loads(after)['listeners'] == SMALL_CONF['listeners'] + + # And nothing was left half-written next to it. + assert [p.name for p in statedir.iterdir() if '.tmp' in p.name] == [] + + # "version" goes through the same function and is stored first, so a + # store that damaged it would leave the state directory unloadable + # even with conf.json intact. + version = statedir / 'version' + + assert version.is_file(), 'version survived the failed store' + assert version.read_bytes().strip() != b'', 'version is not empty' + + finally: + unit_stop() + subprocess.run( + ['umount', str(statedir)], check=False, capture_output=True + ) + # A failed umount would make rmdir() raise out of the finally and + # mask whichever assertion above actually failed. + shutil.rmtree(statedir, ignore_errors=True) + + +def numbered_conf(i): + """SMALL_CONF, made different per store by a setting.""" + + return { + "settings": {"http": {"header_read_timeout": 30 + i}}, + **SMALL_CONF, + } + + +def stored_timeout(statedir): + """The setting in conf.json, or None when there is no conf.json.""" + + try: + text = (statedir / 'conf.json').read_text(encoding='utf-8') + except FileNotFoundError: + return None + + return json.loads(text)['settings']['http']['header_read_timeout'] + + +def group_alive(pgid): + try: + os.killpg(pgid, 0) + except ProcessLookupError: + return False + + return True + + +def put_and_terminate(statedir, count): + """Start unitd on "statedir", PUT "count" configurations, and send + SIGTERM to main as soon as the last one is answered. Return the + temporary directory of that unitd once main has exited.""" + + unit = unit_run(state_dir=str(statedir)) + temp_dir = unit['temp_dir'] + + for i in range(count): + assert 'success' in client.conf(numbered_conf(i)), f'PUT {i}' + + os.kill(int(unit['pid']), signal.SIGTERM) + + assert unit['process'].wait(15) == 0, 'main exited with 0' + + # Main exits only after its store child: no process of its group is + # left to finish the store. In a container main is PID 1, and anything + # left would be killed with it. + assert not group_alive(unit['pgid']), 'no store child outlives main' + + log = Path(temp_dir, 'unit.log').read_text(encoding='utf-8') + + unit_stop() + + # The teardown checks only the log of the last unitd; check this one. + Log.check_alerts(log=log) + + return temp_dir + + +def restart_and_get_timeout(statedir): + unit_run(state_dir=str(statedir)) + + return client.conf_get('settings/http/header_read_timeout') + + +def test_state_store_survives_exit(requires_restart): + """A configuration answered with 200 is stored even when SIGTERM + reaches main while the store runs.""" + + unit_stop() + + statedir = Path(tempfile.mkdtemp(prefix='unit-state-')) + + try: + temp_dir = put_and_terminate(statedir, 1) + + stored = stored_timeout(statedir) + loaded = restart_and_get_timeout(statedir) + + shutil.rmtree(temp_dir, ignore_errors=True) + + assert stored == 30, 'stored before exit' + assert loaded == 30, 'loaded on restart' + + finally: + unit_stop() + shutil.rmtree(statedir, ignore_errors=True) + + +def test_state_store_pending_at_exit(requires_restart): + """Stores that wait behind a running store child are not dropped at + exit: the last configuration is on disk.""" + + unit_stop() + + statedir = Path(tempfile.mkdtemp(prefix='unit-state-')) + + try: + temp_dir = put_and_terminate(statedir, 4) + + stored = stored_timeout(statedir) + loaded = restart_and_get_timeout(statedir) + + shutil.rmtree(temp_dir, ignore_errors=True) + + assert stored == 33, 'the last store won' + assert loaded == 33, 'loaded on restart' + + finally: + unit_stop() + shutil.rmtree(statedir, ignore_errors=True) + + +def test_state_store_serialised(requires_restart): + """Stores in quick succession run one after another and leave the last + configuration, with no temporary file and no alert.""" + + unit_stop() + + statedir = Path(tempfile.mkdtemp(prefix='unit-state-')) + + try: + unit_run(state_dir=str(statedir)) + + for i in range(6): + assert 'success' in client.conf(numbered_conf(i)), f'PUT {i}' + + last = numbered_conf(5) + + for _ in range(100): + try: + if stored_timeout(statedir) == 35: + break + except (ValueError, KeyError): + pass + + time.sleep(0.1) + + # A --debug build logs the start of each store child. A store + # child must not start before the previous one exits. + running = None + + events = re.findall( + r'#\d+ state store child (\d+)$|process (\d+) exited', + Log.read(), + re.M, + ) + + for start, exited in events: + if start: + assert running is None, f'child {start} ran with {running}' + running = start + + elif exited == running: + running = None + + assert stored_timeout(statedir) == 35, 'the last store won' + + # A store that ran after the last one would overwrite it; give any + # such store the time to finish before the second look. + time.sleep(1) + + conf = json.loads((statedir / 'conf.json').read_text(encoding='utf-8')) + + assert conf == last, 'conf.json is the last configuration' + assert [p.name for p in statedir.iterdir() if '.tmp' in p.name] == [] + + # Two stores that run at the same time race on the temporary name, + # and one of them logs a failed rename() or a failed store. + alerts = re.findall(r'.+\[alert\].+', Log.read()) + + assert alerts == [], 'no alert' + + finally: + unit_stop() + shutil.rmtree(statedir, ignore_errors=True) + + +VERSION_CONF = { + "listeners": {"*:8080": {"pass": "routes"}}, + "routes": [{"action": {"return": 204}}], +} + + +def run_with_version(version): + """Start unitd on a state directory with this version file.""" + + unit_stop() + + statedir = Path(tempfile.mkdtemp(prefix='unit-state-')) + + (statedir / 'conf.json').write_text(json.dumps(VERSION_CONF)) + (statedir / 'version').write_bytes(version) + + unit_run(state_dir=str(statedir)) + + return statedir + + +@pytest.mark.parametrize( + 'version', + [b'13700\n', b'13700\r\n', b'13700 \t\n'], + ids=['lf', 'crlf', 'ws'], +) +def test_state_store_version_line_end(requires_restart, version): + """A line end after the number does not drop the stored configuration.""" + + statedir = run_with_version(version) + + try: + assert ( + client.conf_get('listeners') == VERSION_CONF['listeners'] + ), 'stored configuration loaded' + + # GET /config can answer before the router has bound the listener. + waitforsocket(8080) + + assert client.get()['status'] == 204, 'stored configuration runs' + assert not Log.findall(r'invalid version string'), 'no alert' + + finally: + unit_stop() + shutil.rmtree(statedir, ignore_errors=True) + + +@pytest.mark.parametrize( + 'version', + [b'13700x\n', b'', b'\n', b' 13700', b'137\n00'], + ids=['suffix', 'empty', 'blank', 'leading', 'inner'], +) +def test_state_store_version_invalid(requires_restart, skip_alert, version): + """Other content is still an error, and nothing is restored.""" + + skip_alert(r'failed to restore previous configuration') + + statedir = run_with_version(version) + + try: + assert Log.wait_for_record( + r'failed to restore previous configuration: ' + r'invalid version string' + ), 'alert' + assert client.conf_get('listeners') == {}, 'nothing restored' + + finally: + unit_stop() + shutil.rmtree(statedir, ignore_errors=True) diff --git a/test/test_static.py b/test/test_static.py index 94fa0852c..b9968754c 100644 --- a/test/test_static.py +++ b/test/test_static.py @@ -1,5 +1,8 @@ +import gzip import os import socket +import time +from email.utils import formatdate, parsedate_to_datetime from pathlib import Path import pytest @@ -11,6 +14,24 @@ client = ApplicationProto() +def age_file(path, seconds=2): + """ + Backdates a file so its validators are strong. + + Both validators are derived from the whole-second mtime and the size, so + Unit marks them weak while the clock is still inside the second the file + was written -- another write could land in that second and change neither. + A test that writes a file and fetches it immediately is inside that window, + where a strong comparison (If-Match, If-Range) cannot match by design. + + Most tests here are about something else, so they get a file that looks + like a deployed one rather than one written microseconds ago. The window + itself is covered by test_static_conditional_etag_weak_within_mtime_second. + """ + when = os.stat(path).st_mtime - seconds + os.utime(path, (when, when)) + + @pytest.fixture(autouse=True) def setup_method_fixture(temp_dir): assets_dir = f'{temp_dir}/assets' @@ -21,6 +42,9 @@ def setup_method_fixture(temp_dir): Path(f'{assets_dir}/log.log').write_text('[debug]', encoding='utf-8') Path(f'{assets_dir}/dir/file').write_text('blah', encoding='utf-8') + for f in Path(assets_dir).rglob('*'): + age_file(f) + assert 'success' in client.conf( { "listeners": {"*:8080": {"pass": "routes"}}, @@ -51,6 +75,25 @@ def set_index(index): assert client.get()['status'] == 404, 'index empty' +def test_static_index_nul(temp_dir): + # "index" is appended to "share" and handed to open() as a NUL-terminated + # C string, so an embedded NUL truncated the name at the sink: an index of + # "README\0x" opened "README". "share" has been guarded against this since + # it is templated; this is the same sink reached by the other half. + assert 'error' in client.conf( + {"share": f'{temp_dir}/assets$uri', "index": "README\u0000x"}, + 'routes/0/action', + ), 'index with null character' + + # An empty index stays valid -- see test_static_index. + assert 'success' in client.conf( + {"share": f'{temp_dir}/assets$uri', "index": "README"}, + 'routes/0/action', + ), 'clean index still accepted' + + assert client.get()['body'] == 'readme', 'clean index still served' + + def test_static_index_default(): assert client.get(url='/index.html')['body'] == '0123456789', 'index' assert client.get(url='/')['body'] == '0123456789', 'index 2' @@ -106,49 +149,521 @@ def test_static_etag(temp_dir): assert etag != client.get(url='/')['headers']['ETag'], 'new ETag' -def test_static_range_ignored(): - # Unit does not implement byte-range requests for static files. Per - # RFC 7233 a server that does not support Range MUST ignore it and - # return the full 200 representation -- never a malformed 206. +def test_static_accept_ranges(): + resp = client.get(url='/index.html') + assert resp['status'] == 200, 'plain 200' + assert resp['headers']['Accept-Ranges'] == 'bytes', 'Accept-Ranges on 200' + + +def test_static_range_identity_refused_without_compressor(temp_dir): + # A range is served as identity. With no compressor configured, there + # is no other representation to offer a client that refused identity. + # So the request is not serveable. Do not give it the 206 of exactly the + # bytes that it refused. Nothing here configures compression, so this is + # the path that skipped the Accept-Encoding parse before. resp = client.get( url='/index.html', headers={ 'Host': 'localhost', - 'Range': 'bytes=0-4', 'Connection': 'close', + 'Accept-Encoding': 'identity;q=0', + 'Range': 'bytes=0-4', }, ) - assert resp['status'] == 200, 'range ignored -> full 200' - assert resp['body'] == '0123456789', 'full body returned' - assert 'Content-Range' not in resp['headers'], 'no Content-Range' + assert resp['status'] == 406, 'identity is the only available coding' + assert 'Content-Range' not in resp['headers'] -def test_static_conditional_ignored(): - # Conditional requests are not implemented; a matching validator must - # not produce a 304 (which would imply the body was withheld). - etag = client.get(url='/index.html')['headers']['ETag'] + # A zero-length file transfers no representation bytes. So there is + # nothing that the client refused, and the file is not negotiated: 200, + # and a Range against it keeps its 416. + Path(f'{temp_dir}/assets/empty').write_bytes(b'') resp = client.get( - url='/index.html', + url='/empty', headers={ 'Host': 'localhost', - 'If-None-Match': etag, 'Connection': 'close', + 'Accept-Encoding': 'identity;q=0', }, ) - assert resp['status'] == 200, 'if-none-match ignored' - assert resp['body'] == '0123456789', 'full body on matching etag' + + assert resp['status'] == 200, 'no bytes to refuse' + assert resp['body'] == '' resp = client.get( - url='/index.html', + url='/empty', headers={ 'Host': 'localhost', - 'If-Modified-Since': 'Thu, 01 Jan 2100 00:00:00 GMT', 'Connection': 'close', + 'Accept-Encoding': 'identity;q=0', + 'Range': 'bytes=0-4', }, ) - assert resp['status'] == 200, 'if-modified-since ignored' - assert resp['body'] == '0123456789', 'full body on future date' + + assert resp['status'] == 416, 'a range against no bytes' + assert resp['headers']['Content-Range'] == 'bytes */0' + + +def unit_second(resp): + """ + The second Unit believes it is in, taken from the response it just sent. + + Unit stamps "Date" from the same cached coarse clock that decides whether + a validator is weak, and stamps it after the validator is chosen, so it + is the only clock a test can compare against without racing. + """ + return int(parsedate_to_datetime(resp['headers']['Date']).timestamp()) + + +def wait_for_unit_second(second, timeout=5): + """ + Returns a response Unit sent during exactly `second`. + + Used to put a request inside the second a file is dated to, which is + where both validators are weak. The file is dated far enough ahead that + the whole of that second is available once it arrives. + """ + deadline = time.time() + timeout + + while time.time() < deadline: + resp = client.get(url='/index.html') + + if unit_second(resp) == second: + return resp + + pytest.fail(f'Unit never reported second {second}') + + +def range_get(**headers): + return client.get( + url='/index.html', + headers={'Host': 'localhost', 'Connection': 'close', **headers}, + ) + + +def test_static_range_satisfiable(): + # index.html is the 10-byte "0123456789". + resp = range_get(Range='bytes=0-4') + assert resp['status'] == 206, '0-4 is 206' + assert resp['body'] == '01234', '0-4 body slice' + assert resp['headers']['Content-Range'] == 'bytes 0-4/10' + assert resp['headers']['Content-Length'] == '5' + assert resp['headers']['Accept-Ranges'] == 'bytes' + + resp = range_get(Range='bytes=5-') + assert resp['status'] == 206, '5- is 206' + assert resp['body'] == '56789', '5- body slice' + assert resp['headers']['Content-Range'] == 'bytes 5-9/10' + + resp = range_get(Range='bytes=-5') + assert resp['status'] == 206, '-5 is 206' + assert resp['body'] == '56789', '-5 (suffix) body slice' + assert resp['headers']['Content-Range'] == 'bytes 5-9/10' + + # "0-" (whole file) is a valid, satisfiable single range -> 206. + resp = range_get(Range='bytes=0-') + assert resp['status'] == 206, '0- is 206' + assert resp['body'] == '0123456789', '0- body is the whole file' + assert resp['headers']['Content-Range'] == 'bytes 0-9/10' + + # Past EOF but still satisfiable: the end clamps to the last byte. + resp = range_get(Range='bytes=5-9999') + assert resp['status'] == 206, '5-9999 clamps and is 206' + assert resp['body'] == '56789', '5-9999 body slice' + assert resp['headers']['Content-Range'] == 'bytes 5-9/10' + + +def test_static_range_unsatisfiable(): + resp = range_get(Range='bytes=10-') + assert resp['status'] == 416, 'start at EOF is 416' + assert resp['body'] == '', 'no body on 416' + assert resp['headers']['Content-Range'] == 'bytes */10' + + # "-0" (a zero-length suffix) is unsatisfiable, unlike "0-". + resp = range_get(Range='bytes=-0') + assert resp['status'] == 416, '-0 is unsatisfiable' + assert resp['headers']['Content-Range'] == 'bytes */10' + + +def test_static_range_empty_file(temp_dir): + # RFC 9110 Sect. 14.4: no range is satisfiable against a zero-length + # representation. The suffix form is the trap: "size - suffix" clamps to + # 0 while "size - 1" is -1, which produced a 206 carrying the malformed + # "Content-Range: bytes 0--1/0". + Path(f'{temp_dir}/assets/empty.txt').write_text('', encoding='utf-8') + + def get(**headers): + return client.get( + url='/empty.txt', + headers={'Host': 'localhost', 'Connection': 'close', **headers}, + ) + + for value in ['bytes=-5', 'bytes=0-', 'bytes=0-4']: + resp = get(**{'Range': value}) + assert resp['status'] == 416, f'416 for {value} on an empty file' + assert ( + resp['headers']['Content-Range'] == 'bytes */0' + ), f'unsatisfiable Content-Range for {value}' + + resp = get() + assert resp['status'] == 200, 'no Range still serves the empty file' + assert resp['body'] == '', 'empty body' + + +def test_static_range_malformed(): + for value in ['bytes=abc', 'bytes=', 'bytes=5-1', 'notbytes=0-4']: + resp = range_get(Range=value) + assert resp['status'] == 200, f'malformed {value!r} -> full 200' + assert resp['body'] == '0123456789', f'full body for {value!r}' + assert 'Content-Range' not in resp['headers'], f'no Content-Range for {value!r}' + + +def test_static_range_multi_ignored(): + # A server may legally ignore a multi-range request; only single ranges + # are implemented, so this must fall back to a full 200. + resp = range_get(Range='bytes=0-4,5-9') + assert resp['status'] == 200, 'multi-range falls back to 200' + assert resp['body'] == '0123456789', 'full body' + assert 'Content-Range' not in resp['headers'], 'no Content-Range' + + +def test_static_range_if_range(): + r = client.get(url='/index.html') + etag = r['headers']['ETag'] + last_modified = r['headers']['Last-Modified'] + + # Matching tag: apply the range. + resp = range_get(Range='bytes=0-4', **{'If-Range': etag}) + assert resp['status'] == 206, 'If-Range matching tag applies range' + assert resp['body'] == '01234' + + # Non-matching tag: ignore Range, serve full 200. + resp = range_get(Range='bytes=0-4', **{'If-Range': '"nope"'}) + assert resp['status'] == 200, 'If-Range mismatched tag ignores range' + assert resp['body'] == '0123456789' + + # Matching date: apply the range. + resp = range_get(Range='bytes=0-4', **{'If-Range': last_modified}) + assert resp['status'] == 206, 'If-Range matching date applies range' + assert resp['body'] == '01234' + + # Non-matching (older) date: ignore Range, serve full 200. + resp = range_get( + Range='bytes=0-4', + **{'If-Range': 'Thu, 01 Jan 2000 00:00:00 GMT'}, + ) + assert resp['status'] == 200, 'If-Range mismatched date ignores range' + assert resp['body'] == '0123456789' + + # If-Range without Range is meaningless and must not affect the result. + resp = range_get(**{'If-Range': etag}) + assert resp['status'] == 200, 'If-Range without Range is a no-op' + assert resp['body'] == '0123456789' + + +def test_static_range_conditional_wins(): + etag = client.get(url='/index.html')['headers']['ETag'] + + # RFC 9110 Sect. 13.2.2: preconditions are evaluated before Range is + # considered at all -- a 304 (or 412) must win over a Range request. + resp = range_get(Range='bytes=0-4', **{'If-None-Match': etag}) + assert resp['status'] == 304, '304 wins over Range' + assert resp['body'] == '', 'no body on 304' + assert 'Content-Range' not in resp['headers'], 'no Content-Range on 304' + + resp = range_get(Range='bytes=0-4', **{'If-Match': '"nope"'}) + assert resp['status'] == 412, '412 wins over Range' + assert 'Content-Range' not in resp['headers'], 'no Content-Range on 412' + + +def test_static_conditional_etag(): + etag = client.get(url='/index.html')['headers']['ETag'] + + def get(**headers): + return client.get( + url='/index.html', + headers={'Host': 'localhost', 'Connection': 'close', **headers}, + ) + + for value in [etag, f'W/{etag}', '*', f'"nope", {etag}']: + resp = get(**{'If-None-Match': value}) + assert resp['status'] == 304, f'304 for {value}' + assert resp['body'] == '', f'no body for {value}' + assert resp['headers']['ETag'] == etag, f'ETag echoed for {value}' + assert ( + 'Content-Length' not in resp['headers'] + ), f'no Content-Length for {value}' + + resp = get(**{'If-None-Match': '"nope"'}) + assert resp['status'] == 200, 'mismatch sends the body' + assert resp['body'] == '0123456789', 'full body on mismatch' + + +def test_static_conditional_duplicate_field_lines(): + # RFC 9110 Sect. 5.3: repeated field lines are equivalent to one line + # holding the comma-separated concatenation. Keeping only the last line + # seen refused a legitimate request with 412 when an earlier If-Match + # line matched. + etag = client.get(url='/index.html')['headers']['ETag'] + + def get(header, values): + return client.get( + url='/index.html', + headers={ + 'Host': 'localhost', + 'Connection': 'close', + header: values, + }, + ) + + for values in [[etag, '"other"'], ['"other"', etag]]: + assert ( + get('If-None-Match', values)['status'] == 304 + ), f'If-None-Match matches in any line: {values}' + assert ( + get('If-Match', values)['status'] == 200 + ), f'If-Match matches in any line: {values}' + + assert ( + get('If-None-Match', ['"a"', '"b"'])['status'] == 200 + ), 'no If-None-Match line matches' + assert ( + get('If-Match', ['"a"', '"b"'])['status'] == 412 + ), 'no If-Match line matches' + + +def test_static_conditional_if_match(): + etag = client.get(url='/index.html')['headers']['ETag'] + + def get(**headers): + return client.get( + url='/index.html', + headers={'Host': 'localhost', 'Connection': 'close', **headers}, + ) + + resp = get(**{'If-Match': etag}) + assert resp['status'] == 200, 'matching If-Match serves the file' + assert resp['body'] == '0123456789', 'full body' + + assert get(**{'If-Match': '*'})['status'] == 200, '* matches' + assert get(**{'If-Match': '"nope"'})['status'] == 412, 'mismatch is 412' + + # RFC 9110 Sect. 13.1.1: If-Match uses the strong comparison function, + # so a weak tag never matches even when the opaque tags are equal. + assert get(**{'If-Match': f'W/{etag}'})['status'] == 412, 'weak tag' + + # Sect. 13.2.2 evaluation order: If-Match is checked before + # If-None-Match, so a failed If-Match is a 412, not a 304. + resp = get(**{'If-Match': '"nope"', 'If-None-Match': etag}) + assert resp['status'] == 412, 'If-Match evaluated first' + + +def test_static_conditional_unmodified_since(): + def get(value, **extra): + return client.get( + url='/index.html', + headers={ + 'Host': 'localhost', + 'Connection': 'close', + 'If-Unmodified-Since': value, + **extra, + }, + ) + + assert get('Thu, 01 Jan 2100 00:00:00 GMT')['status'] == 200, 'unmodified' + assert get('Thu, 01 Jan 2000 00:00:00 GMT')['status'] == 412, 'modified' + assert get('not-a-date')['status'] == 200, 'unparsable date is ignored' + + etag = client.get(url='/index.html')['headers']['ETag'] + + # If-Match takes precedence over If-Unmodified-Since. + resp = get('Thu, 01 Jan 2000 00:00:00 GMT', **{'If-Match': etag}) + assert resp['status'] == 200, 'If-Match wins' + + +def test_static_conditional_validator_override(temp_dir): + # A conditional request is judged against the validator the client was + # given. When "response_headers" replaces ETag, the tag Unit derives + # from the file is not what went out, so preconditions are not evaluated + # at all -- previously an If-Match carrying the advertised tag was + # refused with 412, which is a legitimate request denied. + assets_dir = f'{temp_dir}/assets' + + assert 'success' in client.conf( + { + "listeners": {"*:8080": {"pass": "routes"}}, + "routes": [ + { + "action": { + "share": f'{assets_dir}$uri', + "response_headers": {"ETag": '"release-42"'}, + } + } + ], + } + ), 'override configure' + + def get(**headers): + return client.get( + url='/index.html', + headers={'Host': 'localhost', 'Connection': 'close', **headers}, + ) + + assert get()['headers']['ETag'] == '"release-42"', 'override advertised' + + assert ( + get(**{'If-Match': '"release-42"'})['status'] == 200 + ), 'the advertised tag is not refused' + assert ( + get(**{'If-None-Match': '"release-42"'})['status'] == 200 + ), 'no 304 against a tag Unit did not generate' + assert ( + get(**{'If-Match': '"nope"'})['status'] == 200 + ), 'and no 412 either, rather than a wrong one' + + +def test_static_conditional_keepalive(): + # The other conditional tests all send "Connection: close", so none of + # them establishes that a 304 leaves the connection usable. A 304 is + # bodyless with no Content-Length and no Transfer-Encoding, so a client + # that miscounts the framing would desynchronise here rather than fail + # outright. + etag = client.get(url='/index.html')['headers']['ETag'] + + resp, sock = client.get( + url='/index.html', + headers={'Host': 'localhost', 'Connection': 'keep-alive', + 'If-None-Match': etag}, + start=True, + read_timeout=1, + ) + + assert resp['status'] == 304, '304 on a kept-alive connection' + assert resp['body'] == '', 'no body' + assert 'Content-Length' not in resp['headers'], 'no Content-Length' + assert 'Transfer-Encoding' not in resp['headers'], 'not chunked' + + resp = client.get( + url='/index.html', + headers={'Host': 'localhost', 'Connection': 'close'}, + sock=sock, + ) + + assert resp['status'] == 200, 'the same connection still serves' + assert resp['body'] == '0123456789', 'and the body is intact' + + +def test_static_conditional_head(): + etag = client.get(url='/index.html')['headers']['ETag'] + + def head(**headers): + return client.head( + url='/index.html', + headers={'Host': 'localhost', 'Connection': 'close', **headers}, + ) + + resp = head(**{'If-None-Match': etag}) + assert resp['status'] == 304, 'HEAD honours a matching validator' + assert resp['body'] == '', 'no body' + + resp = head(**{'If-None-Match': '"nope"'}) + assert resp['status'] == 200, 'HEAD with a stale validator' + assert resp['body'] == '', 'still no body on a HEAD' + assert ( + resp['headers']['Content-Length'] == '10' + ), 'HEAD reports the length it would have sent' + + assert head(**{'If-Match': '"nope"'})['status'] == 412, 'HEAD 412' + + +def test_static_conditional_modified_since(temp_dir): + last_modified = client.get(url='/index.html')['headers']['Last-Modified'] + + def get(value, **extra): + return client.get( + url='/index.html', + headers={ + 'Host': 'localhost', + 'Connection': 'close', + 'If-Modified-Since': value, + **extra, + }, + ) + + resp = get('Thu, 01 Jan 2100 00:00:00 GMT') + assert resp['status'] == 304, 'not modified since a future date' + assert resp['body'] == '', 'no body' + + resp = get(last_modified) + assert resp['status'] == 304, 'not modified since its own mtime' + + resp = get('Thu, 01 Jan 2000 00:00:00 GMT') + assert resp['status'] == 200, 'modified since an old date' + assert resp['body'] == '0123456789', 'full body' + + resp = get('not-a-date') + assert resp['status'] == 200, 'unparsable date is ignored' + + # RFC 9110 Sect. 13.2.2: a present If-None-Match wins outright, even when + # it does not match, so this must send the body despite the future date. + resp = get('Thu, 01 Jan 2100 00:00:00 GMT', **{'If-None-Match': '"nope"'}) + assert resp['status'] == 200, 'If-None-Match takes precedence' + assert resp['body'] == '0123456789', 'full body' + + +def test_static_last_modified_is_gmt(temp_dir): + # RFC 9110 Sect. 5.6.7: an HTTP-date is GMT. Last-Modified used to be + # rendered from localtime() and then labelled "GMT", so on a host not set + # to UTC the value was wrong by the UTC offset while still looking valid. + # Only a downstream cache comparing it would ever notice. + mtime = 1600000000 # Sun, 13 Sep 2020 12:26:40 GMT + + # Under UTC the two renderings are identical and this proves nothing, so + # say so out loud rather than passing. glibc also treats an unknown zone + # as UTC, which is how a runner image without tzdata would go green + # against a broken build; the workflow pins a zone for this reason. + if time.localtime(mtime).tm_gmtoff == 0: + pytest.skip('TZ is UTC: localtime() and gmtime() cannot be told apart') + + os.utime(f'{temp_dir}/assets/index.html', (mtime, mtime)) + + last_modified = client.get(url='/index.html')['headers']['Last-Modified'] + + assert last_modified == formatdate( + mtime, usegmt=True + ), 'Last-Modified is the GMT rendering of mtime' + + +def test_static_last_modified_pre_epoch(temp_dir): + # A pre-epoch mtime is legal ("touch -d 1969-07-20"). nxt_gmtime() used + # to take the time of day from an unsigned "s % 86400", which wraps for a + # negative time: the header came out as "Thu, 01 Jan 1970 1193046:28:1", + # malformed, with the " GMT" truncated off the end of the buffer. + path = f'{temp_dir}/assets/index.html' + mtime = -14182940 # Sun, 20 Jul 1969 20:17:40 GMT + + try: + os.utime(path, (mtime, mtime)) + except (OSError, OverflowError): + pytest.skip('filesystem rejects a pre-epoch mtime') + + if os.stat(path).st_mtime != mtime: + pytest.skip('filesystem does not keep a pre-epoch mtime') + + last_modified = client.get(url='/index.html')['headers']['Last-Modified'] + + # Checked separately from the equality below: this is the truncation, + # and it is worth failing on its own terms. + assert last_modified.endswith(' GMT'), 'well-formed pre-epoch date' + + # The exact string, not a parsed timestamp: parsedate_to_datetime() + # discards the day name, so a wrong weekday would parse to the right + # instant. This mtime is 164 days pre-epoch, far enough that a weekday + # taken from an unsigned day number comes out wrong. + assert last_modified == formatdate( + mtime, usegmt=True + ), 'pre-epoch mtime is reported as itself, not clamped or wrapped' def test_static_redirect(): @@ -396,6 +911,213 @@ def test_static_mime_types_correct(): ), 'mime_types same extensions case insensitive' +SVG = b'' + + +def svgz_write(path): + # mtime=0 keeps the gzip bytes the same on every run. + data = gzip.compress(SVG, mtime=0) + Path(path).write_bytes(data) + age_file(path) + return data + + +def raw_request(method='GET', url='/a.svgz', **headers): + # Raw bytes: the body is gzip data, so it must not be decoded as UTF-8. + raw = client.http( + method, + url=url, + headers={'Host': 'localhost', 'Connection': 'close', **headers}, + encoding='latin-1', + raw_resp=True, + ) + head, _, body = raw.partition('\r\n\r\n') + lines = head.split('\r\n') + status = int(lines[0].split(' ')[1]) + hdrs = dict(line.split(': ', 1) for line in lines[1:]) + return status, hdrs, body.encode('latin-1') + + +def test_static_svgz(temp_dir): + # A ".svgz" file is served as image/svg+xml with "Content-Encoding: gzip", + # and its stored bytes are sent unchanged. Before, it had no Content-Type + # and no Content-Encoding, so a browser did not display it. + data = svgz_write(f'{temp_dir}/assets/a.svgz') + + status, headers, body = raw_request() + assert status == 200 + assert headers.get('Content-Type') == 'image/svg+xml' + assert headers.get('Content-Encoding') == 'gzip' + assert headers['Content-Length'] == str(len(data)) + assert body == data, 'the stored bytes' + assert gzip.decompress(body) == SVG + assert 'Vary' not in headers, 'the coding is not negotiated' + + # A client that does not accept gzip gets the same bytes. No other + # representation exists. + status, headers, body = raw_request(**{'Accept-Encoding': 'identity'}) + assert status == 200 + assert headers.get('Content-Encoding') == 'gzip' + assert body == data + + # HEAD sends the same header fields and no body. + status, headers, body = raw_request('HEAD') + assert status == 200 + assert headers.get('Content-Type') == 'image/svg+xml' + assert headers.get('Content-Encoding') == 'gzip' + assert headers['Content-Length'] == str(len(data)) + assert body == b'', 'no body on HEAD' + + # A 304 carries no Content-Encoding. + status, headers, body = raw_request(**{'If-None-Match': headers['ETag']}) + assert status == 304 + assert 'Content-Encoding' not in headers + assert body == b'' + + +def test_static_svgz_case_insensitive(temp_dir): + # The extension matches without regard to case, as the type lookup does. + data = svgz_write(f'{temp_dir}/assets/B.SVGZ') + + status, headers, body = raw_request(url='/B.SVGZ') + assert status == 200 + assert headers.get('Content-Type') == 'image/svg+xml' + assert headers.get('Content-Encoding') == 'gzip' + assert body == data + + +def test_static_svgz_range(temp_dir): + # A range applies to the stored gzip bytes, and the 206 keeps + # "Content-Encoding: gzip": the slice is part of the gzip data. + data = svgz_write(f'{temp_dir}/assets/a.svgz') + + status, headers, body = raw_request(Range='bytes=0-9') + assert status == 206 + assert headers.get('Content-Encoding') == 'gzip' + assert headers['Content-Range'] == f'bytes 0-9/{len(data)}' + assert body == data[:10] + + # The ETag is strong, so a matching If-Range resumes the range. + etag = raw_request('HEAD')[1]['ETag'] + assert not etag.startswith('W/'), 'strong ETag' + + status, headers, body = raw_request( + Range='bytes=10-', **{'If-Range': etag} + ) + assert status == 206 + assert headers.get('Content-Encoding') == 'gzip' + assert body == data[10:] + + +def test_static_svgz_etag(temp_dir): + # The tag of a ".svgz" file has a "-gzip" suffix. Before, the file had + # the plain tag, and a cache stored it with no coding or with a second + # gzip under W/. That tag must no longer match, so the cache gets a full + # 200 with the coding instead of a 304 that keeps the old entry. + data = svgz_write(f'{temp_dir}/assets/a.svgz') + + # The same bytes and mtime under another name get the plain tag. + Path(f'{temp_dir}/assets/a.bin').write_bytes(data) + mtime = os.stat(f'{temp_dir}/assets/a.svgz').st_mtime + os.utime(f'{temp_dir}/assets/a.bin', (mtime, mtime)) + + etag = raw_request('HEAD')[1]['ETag'] + old = raw_request('HEAD', url='/a.bin')[1]['ETag'] + assert not old.startswith('W/'), 'strong ETag' + assert etag == f'{old[:-1]}-gzip"', 'the tag has the suffix' + + for tag in (old, f'W/{old}'): + status, headers, body = raw_request(**{'If-None-Match': tag}) + assert status == 200, f'old tag {tag} does not match' + assert headers['ETag'] == etag + assert headers.get('Content-Encoding') == 'gzip' + assert body == data + + status, headers, body = raw_request(**{'If-None-Match': etag}) + assert status == 304, 'the new tag matches' + assert 'Content-Encoding' not in headers + assert body == b'' + + assert raw_request(**{'If-Match': old})[0] == 412, 'If-Match old tag' + assert raw_request(**{'If-Match': etag})[0] == 200, 'If-Match new tag' + + # A range resumed against the old tag gets the whole file. + status, headers, body = raw_request(Range='bytes=10-', **{'If-Range': old}) + assert status == 200, 'If-Range old tag' + assert body == data + + +def test_static_svgz_date_validators(temp_dir): + # The Last-Modified date of a ".svgz" file did not change when its + # response gained the coding. So a date must not give a 304 or a range: + # a cache that stored the old response would keep it. + data = svgz_write(f'{temp_dir}/assets/a.svgz') + + # Control: the same bytes and mtime under another name. + Path(f'{temp_dir}/assets/a.bin').write_bytes(data) + mtime = os.stat(f'{temp_dir}/assets/a.svgz').st_mtime + os.utime(f'{temp_dir}/assets/a.bin', (mtime, mtime)) + + date = raw_request('HEAD')[1]['Last-Modified'] + later = formatdate(time.time() + 3600, usegmt=True) + + for ims in (date, later): + assert ( + raw_request(url='/a.bin', **{'If-Modified-Since': ims})[0] == 304 + ), f'control If-Modified-Since {ims}' + + status, headers, body = raw_request(**{'If-Modified-Since': ims}) + assert status == 200, f'If-Modified-Since {ims}' + assert headers['ETag'].endswith('-gzip"') + assert headers.get('Content-Encoding') == 'gzip' + assert body == data + + status, _, body = raw_request( + url='/a.bin', Range='bytes=10-', **{'If-Range': date} + ) + assert status == 206, 'control If-Range date' + assert body == data[10:] + + status, headers, body = raw_request(Range='bytes=10-', **{'If-Range': date}) + assert status == 200, 'If-Range date' + assert headers.get('Content-Encoding') == 'gzip' + assert body == data + + # If-Unmodified-Since still works: a 412 from a date stays correct. + earlier = formatdate(mtime - 3600, usegmt=True) + assert raw_request(**{'If-Unmodified-Since': earlier})[0] == 412 + assert raw_request(**{'If-Unmodified-Since': date})[0] == 200 + + +def test_static_svgz_range_not_satisfiable(temp_dir): + # A 416 has no body, so it carries no Content-Encoding. + data = svgz_write(f'{temp_dir}/assets/a.svgz') + + status, headers, body = raw_request(Range=f'bytes={len(data)}-') + assert status == 416 + assert headers['Content-Range'] == f'bytes */{len(data)}' + assert 'Content-Encoding' not in headers + assert body == b'' + + +def test_static_svgz_mime_types_remap(temp_dir): + # The coding is added only while ".svgz" resolves to image/svg+xml. An + # operator who maps it to another type gets that type and no coding. + svgz_write(f'{temp_dir}/assets/a.svgz') + + assert raw_request()[1].get('Content-Encoding') == 'gzip' + + assert 'success' in client.conf( + {"text/plain": [".log", "README"], "application/x-blob": ".svgz"}, + 'settings/http/static/mime_types', + ) + + status, headers, _ = raw_request() + assert status == 200 + assert headers.get('Content-Type') == 'application/x-blob' + assert 'Content-Encoding' not in headers + + @pytest.mark.skip('not yet') def test_static_mime_types_invalid(temp_dir): assert 'error' in client.http( @@ -446,3 +1168,1011 @@ def one_request(): assert one_request() == b'0123456789', f'keep-alive request {i}' finally: sock.close() + + +# --------------------------------------------------------------------------- +# Adversarial probes for the conditional-request and byte-range features. +# Every 206 asserts its Content-Range, every 416 too: the earlier suite passed +# while a suffix range against an empty file emitted "bytes 0--1/0". +# --------------------------------------------------------------------------- + + +def _raw(url, header_lines, method='GET'): + """Send a request built from literal header lines (so duplicated header + fields and odd bytes survive) and return the parsed response.""" + req = f'{method} {url} HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n' + req = req.encode() + for line in header_lines: + if isinstance(line, str): + line = line.encode() + req += line + b'\r\n' + req += b'\r\n' + + return client.http(req, raw=True, encoding='latin-1') + + +def test_static_range_header_syntax(): + # index.html is the 10-byte "0123456789". + # + # RFC 9110 Sect. 14.2: a server MAY ignore Range entirely, so a 200 is + # always legal; what is not legal is a 206 whose Content-Range and body + # disagree, or a 416 for a satisfiable range. + def check(value, body=None, crange=None): + resp = range_get(Range=value) + + if body is None: + assert resp['status'] == 200, f'{value!r} -> 200' + assert resp['body'] == '0123456789', f'{value!r} full body' + assert 'Content-Range' not in resp['headers'], value + return + + assert resp['status'] == 206, f'{value!r} -> 206' + assert resp['body'] == body, f'{value!r} body' + assert resp['headers']['Content-Range'] == crange, f'{value!r} range' + assert resp['headers']['Content-Length'] == str(len(body)), value + + # Sect. 14.1: range-unit names are case-insensitive. + check('BYTES=0-4', '01234', 'bytes 0-4/10') + check('Bytes=0-4', '01234', 'bytes 0-4/10') + + # Exactly one byte, first and last positions. + check('bytes=0-0', '0', 'bytes 0-0/10') + check('bytes=9-9', '9', 'bytes 9-9/10') + check('bytes=9-', '9', 'bytes 9-9/10') + check('bytes=-1', '9', 'bytes 9-9/10') + + # Whole file, several spellings; the end clamps to size-1 (Sect. 14.1.2). + check('bytes=0-9', '0123456789', 'bytes 0-9/10') + check('bytes=0-10', '0123456789', 'bytes 0-9/10') + check('bytes=-10', '0123456789', 'bytes 0-9/10') + check('bytes=-11', '0123456789', 'bytes 0-9/10') + check('bytes=-99999', '0123456789', 'bytes 0-9/10') + + # Leading zeros are plain decimal digits (1*DIGIT). + check('bytes=000-004', '01234', 'bytes 0-4/10') + check('bytes=-05', '56789', 'bytes 5-9/10') + + # The grammar has no OWS after "=" or around "-": ignoring is the + # RFC-blessed way out (Sect. 14.2), so a 200 is expected and a 206 must + # be self-consistent. + for value in ['bytes= 0-4', 'bytes=0 - 4', 'bytes=0-4 ', ' bytes=0-4', + 'bytes =0-4']: + resp = range_get(Range=value) + assert resp['status'] in (200, 206), f'{value!r} status' + if resp['status'] == 206: + assert resp['body'] == '01234', f'{value!r} body' + assert resp['headers']['Content-Range'] == 'bytes 0-4/10', value + else: + assert resp['body'] == '0123456789', f'{value!r} full body' + + # No unit, unknown unit, empty value, garbage. + for value in ['0-4', '=0-4', 'items=0-4', '', 'bytes', 'bytes=-', + 'bytes=--5', 'bytes=0--4', 'bytes=0-4-', 'bytes=+0-4', + 'bytes=-+4', 'bytes=0x0-4', 'bytes=,0-4', 'bytes=0-4,']: + check(value) + + +def test_static_range_last_byte_boundaries(): + # start == size-1 is the last satisfiable start, start == size is not. + resp = range_get(Range='bytes=9-100') + assert resp['status'] == 206 + assert resp['body'] == '9' + assert resp['headers']['Content-Range'] == 'bytes 9-9/10' + + resp = range_get(Range='bytes=10-10') + assert resp['status'] == 416, 'start == size is unsatisfiable' + assert resp['headers']['Content-Range'] == 'bytes */10' + assert resp['headers']['Content-Length'] == '0' + + resp = range_get(Range='bytes=11-') + assert resp['status'] == 416 + assert resp['headers']['Content-Range'] == 'bytes */10' + + +def test_static_range_one_byte_file(temp_dir): + Path(f'{temp_dir}/assets/one').write_text('x', encoding='utf-8') + + def get(**headers): + return client.get( + url='/one', + headers={'Host': 'localhost', 'Connection': 'close', **headers}, + ) + + for value in ['bytes=0-0', 'bytes=0-', 'bytes=-1', 'bytes=-2', + 'bytes=0-5']: + resp = get(Range=value) + assert resp['status'] == 206, f'{value} on 1-byte file' + assert resp['body'] == 'x', value + assert resp['headers']['Content-Range'] == 'bytes 0-0/1', value + assert resp['headers']['Content-Length'] == '1', value + + for value in ['bytes=1-', 'bytes=1-1', 'bytes=-0']: + resp = get(Range=value) + assert resp['status'] == 416, f'{value} on 1-byte file' + assert resp['headers']['Content-Range'] == 'bytes */1', value + assert resp['body'] == '' + + +def _check_range_honest(value, resp, size=10, content='0123456789'): + """A Range request may be answered 200 (ignored), 416 (unsatisfiable) or + 206 -- but a 206 must carry a well-formed Content-Range whose slice is + exactly the body.""" + status = resp['status'] + assert status in (200, 206, 416), f'{value!r} -> {status}' + + if status == 200: + assert resp['body'] == content, value + assert 'Content-Range' not in resp['headers'], value + + elif status == 416: + assert resp['headers']['Content-Range'] == f'bytes */{size}', value + assert resp['body'] == '' + + else: + crange = resp['headers']['Content-Range'] + assert crange.startswith('bytes ') and '--' not in crange, ( + f'{value!r}: malformed Content-Range {crange!r}' + ) + first, last = crange[6:].split('/')[0].split('-') + first, last = int(first), int(last) + assert 0 <= first <= last < size, f'{value!r}: {crange}' + assert resp['body'] == content[first : last + 1], value + assert resp['headers']['Content-Length'] == str(last - first + 1), ( + value + ) + + +def test_static_range_odd_bytes(): + # Control bytes inside the value: either the parser rejects the message + # (400) or the range is ignored/served honestly. Nothing may hang or + # emit a 206 whose slice does not match its Content-Range. + for value in [b'bytes=0-4\x00', b'bytes=\x000-4', b'bytes=0\x00-4', + b'bytes=0-4\x01', b'bytes=0-\x7f4', b'bytes=0-4\x0b', + b'bytes=0-4\x0c', b'bytes=\xff0-4', b'bytes=0-4\r', + b'bytes=0-4\t', b'\tbytes=0-4']: + resp = _raw('/index.html', [b'Range: ' + value]) + status = resp['status'] + assert status in (200, 206, 400), f'{value!r} -> {status}' + if status == 206: + assert resp['headers']['Content-Range'] == 'bytes 0-4/10', value + assert resp['body'] == '01234', value + elif status == 200: + assert resp['body'] == '0123456789', value + + # obs-fold continuation: either 400 or spliced as a single space. + resp = _raw('/index.html', ['Range: bytes=0-4', ' ,5-9']) + assert resp['status'] in (200, 206, 400) + if resp['status'] == 206: + assert resp['headers']['Content-Range'] == 'bytes 0-4/10' + assert resp['body'] == '01234' + + +def test_static_range_duplicate_headers(): + # Range is not a list-based field (RFC 9110 Sect. 14.2), so two Range + # lines make an invalid message: reject (400) or ignore (200). Serving a + # 206 for one of them is tolerated as long as it is self-consistent. + resp = _raw('/index.html', ['Range: bytes=0-4', 'Range: bytes=5-9']) + assert resp['status'] in (200, 206, 400) + if resp['status'] == 206: + crange = resp['headers']['Content-Range'] + assert crange in ('bytes 0-4/10', 'bytes 5-9/10'), crange + expected = '01234' if crange == 'bytes 0-4/10' else '56789' + assert resp['body'] == expected + + # Two If-Range lines, one matching, one not: the field is a single + # validator, so this is invalid; whichever is picked, the outcome must be + # a coherent 200 or 206. + etag = client.get(url='/index.html')['headers']['ETag'] + resp = _raw('/index.html', ['Range: bytes=0-4', f'If-Range: {etag}', + 'If-Range: "nope"']) + assert resp['status'] in (200, 206, 400) + if resp['status'] == 206: + assert resp['headers']['Content-Range'] == 'bytes 0-4/10' + assert resp['body'] == '01234' + + +def test_static_conditional_duplicate_list_headers(): + # If-None-Match and If-Match are list-based (#entity-tag), and RFC 9110 + # Sect. 5.3 lets a sender split a list across field lines; the recipient + # MUST treat them as the one combined list. The matching tag on the + # FIRST line must therefore count. + etag = client.get(url='/index.html')['headers']['ETag'] + + # Regression: only the last If-None-Match line used to be consulted. + resp = _raw('/index.html', [f'If-None-Match: {etag}', + 'If-None-Match: "other"']) + assert resp['status'] == 304, 'matching tag on the first INM line' + + resp = _raw('/index.html', ['If-None-Match: "other"', + f'If-None-Match: {etag}']) + assert resp['status'] == 304, 'matching tag on the last INM line' + + # Regression: only the last If-Match line used to be consulted, so a + # matching tag on the first line was lost and the request was refused. + resp = _raw('/index.html', [f'If-Match: {etag}', 'If-Match: "other"']) + assert resp['status'] == 200, 'matching tag on the first IM line' + assert resp['body'] == '0123456789' + + resp = _raw('/index.html', ['If-Match: "other"', f'If-Match: {etag}']) + assert resp['status'] == 200, 'matching tag on the last IM line' + + +def test_static_conditional_header_syntax(): + etag = client.get(url='/index.html')['headers']['ETag'] + opaque = etag.strip('"') + + def get(**headers): + return client.get( + url='/index.html', + headers={'Host': 'localhost', 'Connection': 'close', **headers}, + ) + + # Field names are case-insensitive. + assert get(**{'if-none-match': etag})['status'] == 304 + assert get(**{'IF-NONE-MATCH': etag})['status'] == 304 + assert get(**{'if-match': '"nope"'})['status'] == 412 + assert get(**{'RANGE': 'bytes=0-4'})['status'] == 206 + + # OWS and list punctuation around the tag (Sect. 5.6.1). + for value in [f' {etag}', f'{etag} ', f'\t{etag}', f'"x",{etag}', + f'"x", {etag}', f'{etag},"x"', f'"x" , {etag} , "y"', + f',{etag}', f'{etag},', f',,{etag},,']: + assert get(**{'If-None-Match': value})['status'] == 304, repr(value) + + # Weak tag vs. weak comparison (INM) and strong comparison (IM). + assert get(**{'If-None-Match': f'W/{etag}'})['status'] == 304 + assert get(**{'If-None-Match': f'"x", W/{etag}'})['status'] == 304 + assert get(**{'If-Match': f'W/{etag}'})['status'] == 412 + assert get(**{'If-Match': f'{etag}, W/"x"'})['status'] == 200 + + # "*" only as the entire field value. + assert get(**{'If-None-Match': '*'})['status'] == 304 + assert get(**{'If-None-Match': ' * '})['status'] == 304 + assert get(**{'If-Match': '*'})['status'] == 200 + assert get(**{'If-Match': '*, "x"'})['status'] == 412 + assert get(**{'If-None-Match': '"x", *'})['status'] == 200 + assert get(**{'If-None-Match': '**'})['status'] == 200 + assert get(**{'If-None-Match': '"*"'})['status'] == 200 + + # Tags that merely contain or are contained by ours must not match. + for value in [opaque, f'"{opaque}x"', f'"x{opaque}"', f'"{opaque[:-1]}"', + f'"{opaque.upper()}"', f'"{etag}']: + assert get(**{'If-None-Match': value})['status'] == 200, repr(value) + + # Unterminated quote must not run into a following list element. + assert get(**{'If-None-Match': f'"x, {etag}'})['status'] == 200 + + # Empty values: If-None-Match "" matches nothing -> 200. + assert get(**{'If-None-Match': ''})['status'] == 200 + assert get(**{'If-Range': ''})['status'] == 200 + + # A long list (just under the 8 KiB header limit) still finds the tag + # at the end / never matches. + long_list = ', '.join(['"%08d"' % i for i in range(500)]) + assert get(**{'If-None-Match': long_list + ', ' + etag})['status'] == 304 + assert get(**{'If-None-Match': long_list})['status'] == 200 + + +def test_static_conditional_dates(temp_dir): + mtime = int(time.time()) - 86400 * 3 + os.utime(f'{temp_dir}/assets/index.html', (mtime, mtime)) + + def get(**headers): + return client.get( + url='/index.html', + headers={'Host': 'localhost', 'Connection': 'close', **headers}, + ) + + last_modified = get()['headers']['Last-Modified'] + dt = parsedate_to_datetime(last_modified) + + # All three HTTP-date formats (Sect. 5.6.7) name the same instant. + rfc850 = dt.strftime('%A, %d-%b-%y %H:%M:%S GMT') + asctime = dt.strftime('%a %b ') + f'{dt.day:2d}' + dt.strftime( + ' %H:%M:%S %Y' + ) + + for value in [last_modified, rfc850, asctime]: + assert get(**{'If-Modified-Since': value})['status'] == 304, value + assert get(**{'If-Unmodified-Since': value})['status'] == 200, value + r = get(Range='bytes=0-4', **{'If-Range': value}) + assert r['status'] == 206, f'If-Range {value!r} exact' + assert r['headers']['Content-Range'] == 'bytes 0-4/10', value + + before = formatdate(mtime - 1, usegmt=True) + after = formatdate(mtime + 1, usegmt=True) + + assert get(**{'If-Modified-Since': before})['status'] == 200 + assert get(**{'If-Modified-Since': after})['status'] == 304 + assert get(**{'If-Unmodified-Since': before})['status'] == 412 + assert get(**{'If-Unmodified-Since': after})['status'] == 200 + + # Not an HTTP-date: ignored (Sect. 13.1.3 / 13.1.4), never a 412; as an + # If-Range it is a mismatch, so Range is ignored (200). + # (nxt_time_parse() is lenient about the zone token -- " GMT" missing, + # " XYZ", or trailing junk after a complete date all parse; not probed.) + for value in ['yesterday', '0', '1700000000', last_modified[:-8], '']: + assert get(**{'If-Modified-Since': value})['status'] == 200, value + assert get(**{'If-Unmodified-Since': value})['status'] == 200, value + r = get(Range='bytes=0-4', **{'If-Range': value}) + assert r['status'] == 200, f'If-Range {value!r} is a mismatch' + assert 'Content-Range' not in r['headers'] + + # If-Range wants an exact date; newer or older is a mismatch. + for value in [before, after]: + r = get(Range='bytes=0-4', **{'If-Range': value}) + assert r['status'] == 200, value + assert r['body'] == '0123456789' + + # A 304 carries the validators and no framing headers. + r = get(**{'If-Modified-Since': last_modified}) + assert r['status'] == 304 + assert r['headers']['Last-Modified'] == last_modified + assert 'ETag' in r['headers'] + assert 'Content-Length' not in r['headers'] + assert 'Transfer-Encoding' not in r['headers'] + + +def test_static_conditional_precedence(): + r = client.get(url='/index.html') + etag = r['headers']['ETag'] + last_modified = r['headers']['Last-Modified'] + old = 'Thu, 01 Jan 2000 00:00:00 GMT' + future = 'Fri, 01 Jan 2100 00:00:00 GMT' + + def get(**headers): + return client.get( + url='/index.html', + headers={'Host': 'localhost', 'Connection': 'close', **headers}, + ) + + # Sect. 13.2.2 step 1 -> 2: If-Unmodified-Since is ignored when If-Match + # is present, whichever way If-Match goes. + assert get(**{'If-Match': etag, 'If-Unmodified-Since': old})['status'] == 200 + assert get(**{'If-Match': '"x"', 'If-Unmodified-Since': future})[ + 'status' + ] == 412 + + # Step 3 -> 4: If-Modified-Since is ignored when If-None-Match is present. + assert get(**{'If-None-Match': '"x"', 'If-Modified-Since': future})[ + 'status' + ] == 200 + assert get(**{'If-None-Match': etag, 'If-Modified-Since': old})[ + 'status' + ] == 304 + + # A failed step 1/2 wins over a would-be 304 from step 3/4. + assert get(**{'If-Match': '"x"', 'If-None-Match': etag})['status'] == 412 + assert get(**{'If-Unmodified-Since': old, 'If-None-Match': etag})[ + 'status' + ] == 412 + assert get(**{'If-Unmodified-Since': old, 'If-Modified-Since': future})[ + 'status' + ] == 412 + + # A passed step 1 followed by a matching step 3 is a 304. + assert get(**{'If-Match': etag, 'If-None-Match': etag})['status'] == 304 + assert get(**{'If-Match': '*', 'If-None-Match': '*'})['status'] == 304 + + # Step 5: If-Range is consulted only after 1-4 pass. + r = get(Range='bytes=0-4', **{'If-Range': etag, 'If-None-Match': etag}) + assert r['status'] == 304 + assert 'Content-Range' not in r['headers'] + + r = get(Range='bytes=0-4', **{'If-Range': etag, 'If-Match': '"x"'}) + assert r['status'] == 412 + assert 'Content-Range' not in r['headers'] + + r = get(Range='bytes=0-4', **{'If-Range': last_modified, + 'If-Unmodified-Since': old}) + assert r['status'] == 412 + + r = get(Range='bytes=0-4', **{'If-Range': last_modified, + 'If-Modified-Since': last_modified}) + assert r['status'] == 304 + + # Preconditions pass but If-Range fails: full 200, not 206 and not 416. + r = get(Range='bytes=50-', **{'If-Range': '"x"', 'If-Match': etag}) + assert r['status'] == 200 + assert r['body'] == '0123456789' + assert 'Content-Range' not in r['headers'] + + # Preconditions pass, If-Range matches, range unsatisfiable: 416. + r = get(Range='bytes=50-', **{'If-Range': etag, 'If-None-Match': '"x"'}) + assert r['status'] == 416 + assert r['headers']['Content-Range'] == 'bytes */10' + + # A weak If-Range tag never matches (strong comparison, Sect. 13.1.5). + r = get(Range='bytes=0-4', **{'If-Range': f'W/{etag}'}) + assert r['status'] == 200 + assert r['body'] == '0123456789' + + # If-Range that is a list of tags is not the grammar; the only sane + # readings are "mismatch" (200) or a self-consistent 206. + r = get(Range='bytes=0-4', **{'If-Range': f'"x", {etag}'}) + assert r['status'] in (200, 206) + if r['status'] == 206: + assert r['headers']['Content-Range'] == 'bytes 0-4/10' + + # Every precondition + Range on a 412: no Content-Range. + for headers in [{'If-Match': '"x"'}, {'If-Unmodified-Since': old}]: + r = get(Range='bytes=0-4', **headers) + assert r['status'] == 412, headers + assert 'Content-Range' not in r['headers'], headers + + # Every precondition + Range on a 304: no Content-Range, no body, and a + # 304 wins over what would otherwise be a 416. + for headers in [{'If-None-Match': etag}, + {'If-Modified-Since': last_modified}]: + r = get(Range='bytes=0-4', **headers) + assert r['status'] == 304, headers + assert 'Content-Range' not in r['headers'], headers + assert r['body'] == '' + r = get(Range='bytes=50-', **headers) + assert r['status'] == 304, f'304 wins over 416 {headers}' + assert 'Content-Range' not in r['headers'], headers + + +def test_static_range_head(): + def head(**headers): + return client.head( + url='/index.html', + headers={'Host': 'localhost', 'Connection': 'close', **headers}, + ) + + resp = head(Range='bytes=2-6') + assert resp['status'] == 206, 'HEAD honours Range' + assert resp['body'] == '', 'no body on HEAD' + assert resp['headers']['Content-Range'] == 'bytes 2-6/10' + assert resp['headers']['Content-Length'] == '5', 'partial length on HEAD' + assert resp['headers']['Accept-Ranges'] == 'bytes' + + resp = head(Range='bytes=10-') + assert resp['status'] == 416 + assert resp['headers']['Content-Range'] == 'bytes */10' + assert resp['body'] == '' + + etag = client.get(url='/index.html')['headers']['ETag'] + resp = head(Range='bytes=2-6', **{'If-None-Match': etag}) + assert resp['status'] == 304 + assert 'Content-Range' not in resp['headers'] + + resp = head(Range='bytes=2-6', **{'If-Match': '"x"'}) + assert resp['status'] == 412 + + +def test_static_range_post_ignored(): + # Range is defined for GET only (Sect. 14.2); static serves GET/HEAD and + # answers 405 for the rest regardless of any Range header. + resp = client.post( + url='/index.html', + headers={'Host': 'localhost', 'Connection': 'close', + 'Range': 'bytes=0-4'}, + ) + assert resp['status'] == 405 + assert 'Content-Range' not in resp['headers'] + + +def _pattern(size): + # Byte i is (i * 7 + i // 251) & 0xff: aperiodic over any 128 KiB + # window, so a slice that is offset by even one byte, or fetched from + # the wrong buffer, compares unequal. + return bytes(((i * 7) + (i // 251)) & 0xFF for i in range(size)) + + +def _range_bytes(url, first, last=None, suffix=None): + if suffix is not None: + value = f'bytes=-{suffix}' + elif last is None: + value = f'bytes={first}-' + else: + value = f'bytes={first}-{last}' + + resp = client.get( + url=url, + headers={'Host': 'localhost', 'Connection': 'close', 'Range': value}, + encoding='latin-1', + read_buffer_size=1024 * 1024, + ) + resp['body'] = resp['body'].encode('latin-1') + return resp + + +def test_static_range_spans_buffers(temp_dir): + # NXT_HTTP_STATIC_BUF_SIZE is 128 KiB and NXT_HTTP_STATIC_BUF_COUNT + # buffers are issued per body-handler round; a 300 KiB file makes a + # mid-buffer start cross two boundaries and a second read round. + buf = 128 * 1024 + size = 300 * 1024 + data = _pattern(size) + Path(f'{temp_dir}/assets/big.bin').write_bytes(data) + + cases = [ + (0, size - 1), # whole file as a range + (0, buf - 1), # exactly the first buffer + (0, buf), # one past the first buffer + (1, buf), # off by one at both ends + (buf - 1, buf), # straddles the first boundary + (buf, 2 * buf - 1), # exactly the second buffer + (buf + 1, 2 * buf + 1), # mid-buffer start, two crossings + (100_000, 200_000), # arbitrary mid-buffer slice + (2 * buf + 5, size - 1), # tail past the second boundary + (size - 1, size - 1), # last byte + (12345, None), # open-ended from mid-buffer + (2 * buf + 17, None), # open-ended from the last buffer + ] + + for first, last in cases: + resp = _range_bytes('/big.bin', first, last) + end = size - 1 if last is None else min(last, size - 1) + assert resp['status'] == 206, (first, last) + assert resp['headers']['Content-Range'] == f'bytes {first}-{end}/{size}' + assert resp['headers']['Content-Length'] == str(end - first + 1) + assert len(resp['body']) == end - first + 1, (first, last) + assert resp['body'] == data[first : end + 1], ( + f'wrong bytes for {first}-{last}' + ) + + for suffix in [1, buf, buf + 1, 2 * buf + 3, size, size + 1]: + resp = _range_bytes('/big.bin', None, suffix=suffix) + first = max(size - suffix, 0) + assert resp['status'] == 206, suffix + assert resp['headers']['Content-Range'] == ( + f'bytes {first}-{size - 1}/{size}' + ) + assert resp['body'] == data[first:], f'wrong bytes for -{suffix}' + + # And the plain GET still returns everything, so the comparison base is + # not itself suspect. + resp = client.get(url='/big.bin', encoding='latin-1', + read_buffer_size=1024 * 1024) + assert resp['status'] == 200 + assert resp['body'].encode('latin-1') == data + + +def test_static_range_keepalive_mix(temp_dir): + # One connection, many requests: 206 / 200 / 416 / 304 in a row recycle + # the same per-thread buffer descriptor, so a stale file_pos or file_end + # would show up as a shifted or truncated body. The 304 and 416 also + # prove their framing (no body, connection stays usable). + size = 200 * 1024 + data = _pattern(size) + Path(f'{temp_dir}/assets/big.bin').write_bytes(data) + etag = client.get(url='/big.bin')['headers']['ETag'] + + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + sock.connect(('127.0.0.1', 8080)) + sock.settimeout(5) + + def request(extra=''): + sock.sendall( + ( + f'GET /big.bin HTTP/1.1\r\nHost: localhost\r\n' + f'Connection: keep-alive\r\n{extra}\r\n' + ).encode() + ) + raw = b'' + while b'\r\n\r\n' not in raw: + raw += sock.recv(65536) + head, _, rest = raw.partition(b'\r\n\r\n') + lines = head.decode().split('\r\n') + status = int(lines[0].split(' ')[1]) + headers = dict(line.split(': ', 1) for line in lines[1:]) + length = int(headers.get('Content-Length', 0)) + while len(rest) < length: + rest += sock.recv(65536) + assert len(rest) == length, 'no bytes after the declared body' + return status, headers, rest + + try: + steps = [ + ('Range: bytes=150000-180000\r\n', 206, data[150000:180001], + f'bytes 150000-180000/{size}'), + ('', 200, data, None), + ('Range: bytes=-10\r\n', 206, data[-10:], + f'bytes {size - 10}-{size - 1}/{size}'), + (f'Range: bytes={size}-\r\n', 416, b'', f'bytes */{size}'), + ('Range: bytes=0-0\r\n', 206, data[:1], f'bytes 0-0/{size}'), + (f'If-None-Match: {etag}\r\n', 304, b'', None), + ('Range: bytes=131071-131072\r\n', 206, data[131071:131073], + f'bytes 131071-131072/{size}'), + (f'Range: bytes=5-\r\nIf-None-Match: {etag}\r\n', 304, b'', None), + ('', 200, data, None), + ('Range: bytes=1-\r\n', 206, data[1:], f'bytes 1-{size - 1}/{size}'), + ] + + for i in range(3): + for extra, want_status, want_body, want_range in steps: + status, headers, body = request(extra) + assert status == want_status, (i, extra) + assert body == want_body, (i, extra, len(body)) + if want_range is None: + assert 'Content-Range' not in headers, (i, extra) + else: + assert headers['Content-Range'] == want_range, (i, extra) + finally: + sock.close() + + +def test_static_range_concurrent(temp_dir): + # Many clients pulling different slices of one file at the same time, + # each connection interleaving its reads with the others so the + # per-thread descriptor freelist is churned while bodies are in flight. + # A leaked file_pos would hand one client another client's bytes. + import threading + + size = 320 * 1024 + data = _pattern(size) + Path(f'{temp_dir}/assets/big.bin').write_bytes(data) + + ranges = [ + (0, 65535), (65536, 131071), (131072, 196607), (196608, 262143), + (262144, size - 1), (1, 300000), (131071, 131073), (100, 100), + (200000, None), (0, None), (size - 1, None), (77777, 233333), + ] + + errors = [] + barrier = threading.Barrier(len(ranges) * 2) + + def worker(first, last): + try: + for _ in range(4): + barrier.wait(timeout=10) + resp = _range_bytes('/big.bin', first, last) + end = size - 1 if last is None else last + if resp['status'] != 206: + errors.append((first, last, resp['status'])) + continue + if resp['headers']['Content-Range'] != ( + f'bytes {first}-{end}/{size}' + ): + errors.append((first, last, resp['headers']['Content-Range'])) + if resp['body'] != data[first : end + 1]: + got = resp['body'] + off = next( + (k for k in range(min(len(got), end - first + 1)) + if got[k] != data[first + k]), + min(len(got), end - first + 1), + ) + errors.append((first, last, 'body', len(got), off)) + except Exception as e: # pylint: disable=broad-except + errors.append((first, last, repr(e))) + + threads = [threading.Thread(target=worker, args=r) for r in ranges] + [ + threading.Thread(target=worker, args=r) for r in ranges + ] + for t in threads: + t.start() + for t in threads: + t.join(timeout=60) + + assert not errors, errors + + +def test_static_range_share_features(temp_dir): + assets = f'{temp_dir}/assets' + etag = client.get(url='/index.html')['headers']['ETag'] + + def get(url, **headers): + return client.get( + url=url, + headers={'Host': 'localhost', 'Connection': 'close', **headers}, + ) + + # Index: "/" resolves to index.html; Range and validators apply to it. + r = get('/', Range='bytes=2-4') + assert r['status'] == 206, 'range on an index' + assert r['body'] == '234' + assert r['headers']['Content-Range'] == 'bytes 2-4/10' + assert get('/', **{'If-None-Match': etag})['status'] == 304 + assert get('/', **{'If-Match': '"x"'})['status'] == 412 + + # Directory redirect: a 301 is neither conditional nor partial. + for headers in [{'Range': 'bytes=0-1'}, {'If-None-Match': '*'}, + {'If-Match': '"x"'}, {'If-None-Match': etag}, + {'If-Modified-Since': 'Fri, 01 Jan 2100 00:00:00 GMT'}, + {'If-Unmodified-Since': 'Thu, 01 Jan 2000 00:00:00 GMT'}]: + r = get('/dir', **headers) + assert r['status'] == 301, headers + assert r['headers']['Location'] == '/dir/', headers + assert 'Content-Range' not in r['headers'], headers + assert 'Accept-Ranges' not in r['headers'], headers + + # 404 is not conditional either (Sect. 13.2.1: preconditions apply only + # when the unconditional response would be 2xx or 412). + for headers in [{'Range': 'bytes=0-1'}, {'If-None-Match': '*'}, + {'If-Match': '"x"'}, {'If-Match': '*'}]: + r = get('/nope', **headers) + assert r['status'] == 404, headers + assert 'Content-Range' not in r['headers'], headers + + # types: index.html is text/html; a share restricted to text/plain + # refuses it (403, as test_static_types.py documents), so Range and + # validators never get evaluated and the outcome is the same as without. + assert 'success' in client.conf( + {"share": f'{assets}$uri', "types": ["text/plain"]}, + 'routes/0/action', + ) + for headers in [{}, {'Range': 'bytes=0-1'}, {'If-None-Match': etag}, + {'If-Match': '"x"'}]: + r = get('/index.html', **headers) + assert r['status'] == 403, headers + assert 'Content-Range' not in r['headers'], headers + assert 'Accept-Ranges' not in r['headers'], headers + + r = get('/README', Range='bytes=0-2') + assert r['status'] == 206, 'type-matched file still supports ranges' + assert r['body'] == 'rea' + assert r['headers']['Content-Range'] == 'bytes 0-2/6' + assert r['headers']['Content-Type'] == 'text/plain' + + # fallback: an unreadable share falls through and the fallback answers, + # whatever the conditional/range headers were. + assert 'success' in client.conf( + { + "share": f'{assets}/missing$uri', + "fallback": {"return": 200}, + }, + 'routes/0/action', + ) + for headers in [{'Range': 'bytes=0-1'}, {'If-None-Match': '*'}, + {'If-Match': '"x"'}]: + r = get('/index.html', **headers) + assert r['status'] == 200, headers + assert 'Content-Range' not in r['headers'], headers + + # fallback to a second share: the first share's 404 carries no + # partially-built response into the second share's 206. + assert 'success' in client.conf( + { + "share": [f'{assets}/missing$uri', f'{assets}$uri'], + }, + 'routes/0/action', + ) + r = get('/index.html', Range='bytes=3-5') + assert r['status'] == 206 + assert r['body'] == '345' + assert r['headers']['Content-Range'] == 'bytes 3-5/10' + assert get('/index.html', **{'If-None-Match': etag})['status'] == 304 + + +def test_static_range_file_changes_between_requests(temp_dir): + # The ETag encodes mtime and size, so If-Range must stop applying once + # the file is rewritten (Sect. 13.1.5 exists for resumed downloads). + path = Path(f'{temp_dir}/assets/index.html') + r = client.get(url='/index.html') + etag = r['headers']['ETag'] + last_modified = r['headers']['Last-Modified'] + + path.write_text('abcdefghijklmnop', encoding='utf-8') + + # A distinct mtime, and in the past: a file dated in the future is inside + # the weak window too, since the clock has not left that second yet, and + # the tail of this test needs the rewritten file to validate strongly. + new_mtime = int(time.time()) - 5 + os.utime(path, (new_mtime, new_mtime)) + + r = range_get(Range='bytes=10-', **{'If-Range': etag}) + assert r['status'] == 200, 'stale If-Range tag ignores Range' + assert r['body'] == 'abcdefghijklmnop' + + r = range_get(Range='bytes=10-', **{'If-Range': last_modified}) + assert r['status'] == 200, 'stale If-Range date ignores Range' + + r = range_get(**{'If-None-Match': etag}) + assert r['status'] == 200, 'stale INM is a miss' + + r = range_get(**{'If-Match': etag}) + assert r['status'] == 412, 'stale If-Match fails' + + new_etag = client.get(url='/index.html')['headers']['ETag'] + r = range_get(Range='bytes=10-', **{'If-Range': new_etag}) + assert r['status'] == 206 + assert r['body'] == 'klmnop' + assert r['headers']['Content-Range'] == 'bytes 10-15/16' + + +def test_static_range_gzip(temp_dir): + # Compression must be skipped for a 206 (a coded slice would be + # meaningless) but still work for the plain 200 alongside it. + import gzip + + resp = client.conf( + { + "http": { + "compression": { + "types": ["text/html*"], + "compressors": [{"encoding": "gzip", "level": 1}], + } + } + }, + 'settings', + ) + if 'supported compressor' in resp.get('detail', ''): + pytest.skip('unit built without gzip support') + assert 'success' in resp, resp + + size = 200 * 1024 + data = ('0123456789abcdef' * (size // 16)).encode() + Path(f'{temp_dir}/assets/big.html').write_bytes(data) + + def get(**headers): + raw = client.get( + url='/big.html', + headers={'Host': 'localhost', 'Connection': 'close', + 'Accept-Encoding': 'gzip', **headers}, + encoding='latin-1', + read_buffer_size=1024 * 1024, + raw_resp=True, + ) + head, _, body = raw.partition('\r\n\r\n') + lines = head.split('\r\n') + status = int(lines[0].split(' ')[1]) + headers = dict(line.split(': ', 1) for line in lines[1:]) + body = body.encode('latin-1') + if headers.get('Transfer-Encoding') == 'chunked': + body = client._parse_chunked_body(body) + return status, headers, body + + status, headers, body = get() + assert status == 200 + assert headers.get('Content-Encoding') == 'gzip', 'plain GET compresses' + assert gzip.decompress(body) == data + + status, headers, body = get(Range='bytes=100000-200000') + assert status == 206 + assert 'Content-Encoding' not in headers, 'no coding on a 206' + assert headers['Content-Range'] == f'bytes 100000-200000/{size}' + assert headers['Content-Length'] == '100001' + assert body == data[100000:200001] + + status, headers, body = get(Range=f'bytes={size}-') + assert status == 416 + assert 'Content-Encoding' not in headers + assert headers['Content-Range'] == f'bytes */{size}' + + # A 200 after the 206 still compresses: the skip is per request. + status, headers, body = get() + assert status == 200 + assert headers.get('Content-Encoding') == 'gzip' + + +def test_static_range_numeric_overflow(): + # Kept last in the file: the sign-flipped inputs below hang their + # connection and leak an open descriptor in the router, which the + # harness's teardown fd check reports and which then breaks every test + # that would run after it in the same unitd. + # RFC 9110 Sect. 14.1.2: a first-pos greater than or equal to the current + # length is unsatisfiable (416), whatever its magnitude. A value that + # does not fit nxt_off_t must not wrap into a "valid" offset. + # + # 2**63 flips the sign of a 64-bit signed accumulator, 2**64-1 wraps to + # -1, 2**64 wraps to 0. Any of these as the first-pos is >= size, so + # 416 (or 200) is right; a 206 for them is a wrap. + firsts = ['9223372036854775807', '9223372036854775808', + '9223372036854775809', '18446744073709551615', + '18446744073709551616', '18446744073709551617', + '99999999999999999999', '9' * 40, '9' * 4000] + + for first in firsts: + for value in [f'bytes={first}-', f'bytes={first}-{first}']: + resp = range_get(Range=value) + # Regression: a wrapped negative first-pos used to pass the + # "a >= size" test and be served as a 206 with a negative + # Content-Range start, leaking the open file descriptor. + assert resp['status'] != 206, ( + f'{value[:60]!r}: 206 {resp["headers"].get("Content-Range")}' + ) + _check_range_honest(value, resp) + + # The same magnitudes as last-pos: satisfiable, must clamp to size-1. + for last in firsts: + for first in ['0', '5']: + value = f'bytes={first}-{last}' + _check_range_honest(value, range_get(Range=value)) + + # And as a suffix-length (Sect. 14.1.2: longer than the representation + # means the whole representation). + for suffix in firsts: + value = f'bytes=-{suffix}' + _check_range_honest(value, range_get(Range=value)) + + # Negative-looking inputs are simply not the grammar. + for value in ['bytes=-5-9', 'bytes=-1-', 'bytes=5--9', 'bytes=-0-']: + _check_range_honest(value, range_get(Range=value)) + + +def test_static_conditional_etag_weak_within_mtime_second(temp_dir): + # RFC 9110 Sect. 8.8.1: a validator is strong only when the + # representation cannot change again without the validator changing too. + # Both of Unit's are derived from the whole-second mtime and the size, so + # a rewrite to the same size inside the second the file was written is + # invisible to both. While the clock is still inside that second the tag + # is marked weak; once the second has passed it is strong for good. + path = Path(f'{temp_dir}/assets/index.html') + + # Unit reads a coarse cached clock, so Python's clock cannot say which + # second Unit thinks it is in -- it leads Unit's by up to a jiffy, which + # made an earlier version of this test fail intermittently right at the + # boundary. Unit's own "Date" is rendered from that same cached second, + # so it is the oracle used throughout here. + # + # The file is dated two seconds ahead and the test then waits for Unit to + # report that exact second. That lands the request in the "now == mtime" + # case, which is the hazard itself: a rewrite landing in the second the + # file was written. Entering only the "now < mtime" case would leave the + # boundary untested -- a build with "<" in place of "<=" passes that. + mtime = int(time.time()) + 2 + os.utime(path, (mtime, mtime)) + + resp = wait_for_unit_second(mtime) + weak = resp['headers']['ETag'] + assert weak.startswith('W/'), 'weak in the second the file is dated to' + + # Weakness costs nothing else: a client caching on it still revalidates. + assert range_get(**{'If-None-Match': weak})['status'] == 304, 'weak 304' + + # A strong comparison cannot match a weak validator (Sect. 8.8.3.2), so + # If-Match fails and If-Range declines to serve a partial response. + assert range_get(**{'If-Match': weak})['status'] == 412, 'weak If-Match' + + resp = range_get(Range='bytes=0-4', **{'If-Range': weak}) + assert resp['status'] == 200, 'weak If-Range serves the whole file' + assert resp['body'] == '0123456789', 'and all of it' + + # A client that sends the opaque tag back without the "W/" is the case + # that tests Unit's own weakness rather than the client's: the list entry + # is strong, so only our side can fail the comparison. A cache holding + # the strong tag from before a same-second rewrite sends exactly this. + bare = weak[len('W/') :] + assert range_get(**{'If-Match': bare})['status'] == 412, 'bare If-Match' + + resp = range_get(Range='bytes=0-4', **{'If-Range': bare}) + assert resp['status'] == 200, 'bare weak If-Range serves the whole file' + + # Sect. 13.1.3 compares If-None-Match weakly, so the bare tag still hits. + assert range_get(**{'If-None-Match': bare})['status'] == 304, 'bare 304' + + # The date form of If-Range is the same validator and just as weak here. + # Getting this wrong splices two versions together in the client's file, + # which is worse than the stale copy a bad conditional GET leaves. + last_modified = client.get(url='/index.html')['headers']['Last-Modified'] + resp = range_get(Range='bytes=0-4', **{'If-Range': last_modified}) + assert resp['status'] == 200, 'weak If-Range date serves the whole file' + + # "*" asks whether a representation exists at all, not whether a + # validator matches, so the weakness does not bear on it. + assert range_get(**{'If-Match': '*'})['status'] == 200, 'If-Match *' + + # An unconditional Range is untouched. Only a range conditioned on a + # validator has anything to be wrong about, so weakening must not cost + # every range request in the window a partial response. + resp = range_get(Range='bytes=0-4') + assert resp['status'] == 206, 'plain Range still partial in the window' + assert resp['body'] == '01234', 'and the right slice' + + # Wait for the tag to go strong, then check it did not go strong early. + # Asserting on Unit's own "Date" rather than on a sleep is what keeps + # this off the boundary race: the tag may only harden once Unit has left + # the second the file is dated to. + deadline = time.time() + 5 + + while True: + resp = client.get(url='/index.html') + strong = resp['headers']['ETag'] + + if not strong.startswith('W/'): + break + + assert time.time() < deadline, 'tag never hardened' + + assert unit_second(resp) > mtime, 'hardened only after the second passed' + + # Only a prefix separates them: the tag's format does not change, so + # nothing already held in a cache is invalidated by the weakening. + assert weak == f'W/{strong}', 'same tag, weak prefix only' + + # And the strong comparisons work again on the very same bytes. + assert range_get(**{'If-Match': strong})['status'] == 200, 'strong match' + assert range_get(Range='bytes=0-4', **{'If-Range': strong})['status'] == 206 diff --git a/test/test_static_compression.py b/test/test_static_compression.py new file mode 100644 index 000000000..e9bd4e972 --- /dev/null +++ b/test/test_static_compression.py @@ -0,0 +1,1095 @@ +import gzip +import os +import zlib +from pathlib import Path + +import pytest + +from unit.applications.proto import ApplicationProto + +client = ApplicationProto() + + +@pytest.fixture(autouse=True) +def setup_method_fixture(temp_dir): + assets_dir = f'{temp_dir}/assets' + Path(assets_dir).mkdir(parents=True, exist_ok=True) + Path(f'{assets_dir}/big.css').write_text( + 'body{color:red}' * 500, encoding='utf-8' + ) + Path(f'{assets_dir}/tiny.css').write_text('ok', encoding='utf-8') + # Above the 10-byte "min_length" below. So a 406 on this file can come + # only from the media type, never from the length. + Path(f'{assets_dir}/raw').write_text('raw' * 10, encoding='utf-8') + + assert 'success' in client.conf( + { + "settings": { + "http": { + "compression": { + "types": ["text/css"], + "compressors": [ + {"encoding": "gzip", "level": 5, "min_length": 10} + ], + } + } + }, + "listeners": {"*:8080": {"pass": "routes"}}, + "routes": [{"action": {"share": f'{assets_dir}$uri'}}], + } + ), 'compression configure' + + +def test_static_compression_baseline(): + resp = client.get( + url='/big.css', + headers={ + 'Host': 'localhost', + 'Accept-Encoding': 'gzip', + 'Connection': 'close', + }, + ) + assert resp['status'] == 200, 'compressed 200' + assert resp['headers']['Content-Encoding'] == 'gzip', 'gzip applied' + + +def test_static_compression_regex_match_limit( + require, temp_dir, wait_for_record +): + # A "types" regex that reaches its match limit is not a match. So the + # response is not compressed. + require({'modules': {'regex': True}}) + + assets_dir = f'{temp_dir}/assets' + Path(f'{assets_dir}/a.slow').write_text('slow' * 10, encoding='utf-8') + Path(f'{assets_dir}/a.fast').write_text('fast' * 10, encoding='utf-8') + + assert 'success' in client.conf( + { + "static": { + "mime_types": {',' * 2000: ".slow", "a,bx": ".fast"} + }, + "compression": { + "types": ["~^(.*),(.*)[xy]$"], + "compressors": [ + {"encoding": "gzip", "level": 5, "min_length": 10} + ], + }, + }, + 'settings/http', + ), 'regex types configure' + + headers = { + 'Host': 'localhost', + 'Accept-Encoding': 'gzip', + 'Connection': 'close', + } + + resp = client.get(url='/a.fast', headers=headers) + assert resp['status'] == 200, 'fast status' + assert resp['headers'].get('Content-Encoding') == 'gzip', 'fast gzip' + + # gzip is selected, but the "types" regex reaches its match limit, and + # that is not a match. So a client that refuses identity gets a 406, + # not the bytes that it refused. + resp = client.get( + url='/a.slow', + headers={ + 'Host': 'localhost', + 'Accept-Encoding': 'gzip, identity;q=0', + 'Connection': 'close', + }, + ) + assert resp['status'] == 406, 'slow identity refused' + + resp = client.get(url='/a.slow', headers=headers) + assert resp['status'] == 200, 'slow status' + assert 'Content-Encoding' not in resp['headers'], 'slow identity' + assert resp['body'] == 'slow' * 10, 'slow body' + + assert ( + wait_for_record(r'\[warn\].+reached the match limit on 2000 bytes') + is not None + ), 'match limit log' + + +def test_static_compression_removed_between_requests(): + # #167: the compression state used to be process-global and allocated + # from the router configuration that parsed it, so a configuration + # without a "compression" block left it pointing into a freed pool and + # the next request killed the router. Pin the transition itself: every + # other test in this file only ever configures compression, so the bug + # reproduced through the file order of a whole suite run rather than + # through any one file. + headers = { + 'Host': 'localhost', + 'Accept-Encoding': 'gzip', + 'Connection': 'close', + } + + resp = client.get(url='/big.css', headers=headers) + assert resp['headers']['Content-Encoding'] == 'gzip', 'gzip before' + + assert 'success' in client.conf_delete( + 'settings/http/compression' + ), 'compression removed' + + resp = client.get(url='/big.css', headers=headers) + assert resp['status'] == 200, 'identity after the block is removed' + assert 'Content-Encoding' not in resp['headers'], 'no coding after' + + # The second request is the one that used to find a freed pool: the + # first may be answered before the old configuration is released. + resp = client.get(url='/big.css', headers=headers) + assert resp['status'] == 200, 'router still serving' + + +def test_static_compression_precondition_does_not_mask_406(): + # RFC 9110 Sect. 13.2.1: an ordinary failure outranks a precondition. A + # client that refuses every encoding cannot be served at all, so the + # answer is 406 -- a validator must not turn that into 304 or 412. + etag = client.get(url='/big.css')['headers']['ETag'] + + def get(**headers): + return client.get( + url='/big.css', + headers={ + 'Host': 'localhost', + 'Connection': 'close', + 'Accept-Encoding': 'identity;q=0, *;q=0', + **headers, + }, + ) + + assert get()['status'] == 406, 'unacceptable without a validator' + assert get(**{'If-None-Match': '*'})['status'] == 406, '406 outranks 304' + assert get(**{'If-None-Match': etag})['status'] == 406, '406 over 304' + assert get(**{'If-Match': '"nope"'})['status'] == 406, '406 outranks 412' + + +def test_static_compression_304_carries_no_encoding(): + # A 304 sends no body, so it must not claim one is encoded, and the + # compressor must not have been initialised for it either. + etag = client.get(url='/big.css')['headers']['ETag'] + + resp = client.get( + url='/big.css', + headers={ + 'Host': 'localhost', + 'Connection': 'close', + 'Accept-Encoding': 'gzip', + 'If-None-Match': etag, + }, + ) + + assert resp['status'] == 304, 'not modified' + assert resp['body'] == '', 'no body' + assert 'Content-Encoding' not in resp['headers'], 'no Content-Encoding' + assert 'Content-Length' not in resp['headers'], 'no Content-Length' + + +def test_static_compression_vary(): + # RFC 9110 Sect. 12.5.5: a response subject to proactive negotiation must + # say what it varied on, or a shared cache may serve gzip bytes to a + # client that cannot decode them. This is the companion of the weak + # entity-tag: that makes revalidation distinguish the codings, this makes + # the cache key distinguish them. + def get(**headers): + return client.get( + url='/big.css', + headers={'Host': 'localhost', 'Connection': 'close', **headers}, + ) + + resp = get(**{'Accept-Encoding': 'gzip'}) + assert resp['status'] == 200 + assert resp['headers']['Content-Encoding'] == 'gzip', 'compressed' + assert ( + resp['headers']['Vary'] == 'Accept-Encoding' + ), 'Vary on the coded response' + + # The identity response is the one a cache must not reuse for a + # gzip-capable client, so it needs the header just as much. + resp = get() + assert resp['status'] == 200 + assert 'Content-Encoding' not in resp['headers'], 'not compressed' + assert ( + resp['headers']['Vary'] == 'Accept-Encoding' + ), 'Vary on the identity response too' + + +@pytest.mark.parametrize( + ('configured', 'expected'), + [ + # No "Vary" of its own: the generated field stands unchanged. + (None, 'Accept-Encoding'), + # An empty value has no tokens to append to, so it is replaced + # rather than appended to -- otherwise ", Accept-Encoding". + ('', 'Accept-Encoding'), + # A different header still needs the coding added: the response + # varies on both. An operator writing "Origin" is adding CORS and + # is almost never aware Unit generates the field at all. + ('Origin', 'Origin, Accept-Encoding'), + ('a, b, c', 'a, b, c, Accept-Encoding'), + # Already named: left exactly as written, and compared per token so + # that "X-Accept-Encoding" does not count as a match. + ('Accept-Encoding', 'Accept-Encoding'), + ('a, Accept-Encoding', 'a, Accept-Encoding'), + ('X-Accept-Encoding', 'X-Accept-Encoding, Accept-Encoding'), + # The token scan compares case-insensitively, as a field name must + # be. With nxt_strncasecmp() swapped for nxt_strncmp() every other + # case here still passes; this one appends and fails. + ('accept-encoding', 'accept-encoding'), + ('a, ACCEPT-ENCODING', 'a, ACCEPT-ENCODING'), + # "*" varies on everything already; adding to it would say less. + ('*', '*'), + # A trailing separator is appended after the last real token, not + # after the comma: "Origin,," is legal but pointless. + ('Origin,', 'Origin, Accept-Encoding'), + ('Origin, ', 'Origin, Accept-Encoding'), + # Optional whitespace is trimmed before the comparisons, and the + # value itself is left alone. "response_headers" is the one source + # measured to deliver an untrimmed value: it passes the configured + # string through verbatim, where PHP normalises trailing whitespace + # before libunit sees it. Whether any other language module also + # delivers one is untested, so treat the trim's wider rationale as + # unverified and this case as the reason it stays. + # + # Without the trim around the wildcard test, "* " appends and emits + # "*, Accept-Encoding", narrowing a header that varied on + # everything. The last case is trimmed per token instead, by the + # "already listed?" scan. + # + # The leading-space case also depends on the test client preserving + # leading OWS in a field value, which some parsers strip; it is + # asserting Unit's behaviour through a client that happens not to. + ('* ', '* '), + (' * ', ' * '), + ('Accept-Encoding ', 'Accept-Encoding '), + ], +) +def test_static_compression_vary_merge(temp_dir, configured, expected): + # "response_headers" runs after the field is generated and replaces or + # removes a field it names outright. A response chosen by negotiation + # varies on Accept-Encoding whatever an operator wrote there, so the + # merge is re-asserted afterwards; without that, a "Vary: Origin" in the + # configuration silently drops the coding from every shared cache key. + action = {"share": f'{temp_dir}/assets$uri'} + + if configured is not None: + action["response_headers"] = {"Vary": configured} + + assert 'success' in client.conf(action, 'routes/0/action'), 'configure' + + resp = client.get( + url='/big.css', + headers={ + 'Host': 'localhost', + 'Connection': 'close', + 'Accept-Encoding': 'gzip', + }, + ) + + assert resp['status'] == 200, 'status 200' + assert resp['headers']['Content-Encoding'] == 'gzip', 'compressed' + assert resp['headers']['Vary'] == expected, 'merged Vary' + + +@pytest.mark.parametrize('coding', ['gzip', None]) +def test_static_compression_vary_merge_removed(temp_dir, coding): + # "response_headers" can remove a field outright by setting it to null, + # and removal is f->skip = 1 rather than an erasure. The merge scan + # filters on !f->skip, so a removed Vary is invisible to it: "vary" stays + # NULL and control reaches the tail that adds a fresh field. The response + # therefore holds two Vary fields, one skipped and one new. + # + # Assert the emitted header is a single string, not a list. The test + # client collapses one occurrence to a str and two to a list, so this + # fails if the skipped field is ever serialised -- a class of bug this + # code has hit before, where a skipped response field's slot is reused. + assert 'success' in client.conf( + { + "share": f'{temp_dir}/assets$uri', + "response_headers": {"Vary": None}, + }, + 'routes/0/action', + ), 'configure' + + headers = {'Host': 'localhost', 'Connection': 'close'} + + if coding is not None: + headers['Accept-Encoding'] = coding + + resp = client.get(url='/big.css', headers=headers) + + assert resp['status'] == 200, 'status 200' + assert resp['headers']['Vary'] == 'Accept-Encoding', 're-added' + assert isinstance(resp['headers']['Vary'], str), 'exactly one Vary' + + +def test_static_compression_vary_merge_field_name_case(temp_dir): + # A configured key of "vary" is still merged rather than emitted beside + # the generated field. + # + # Measured limit: this does NOT pin the nxt_strncasecmp(f->name, "Vary") + # comparison inside nxt_http_comp_merge_vary(). Making that comparison + # case-sensitive leaves this test passing, because "response_headers" + # matches the generated field case-insensitively itself and replaces its + # value, so the field reaching the merge is still named "Vary" whatever + # the configuration wrote. Pinning the merge's own field-name comparison + # needs a header authored by an application, which no test here can do -- + # see the note on the uncovered application path below. + assert 'success' in client.conf( + { + "share": f'{temp_dir}/assets$uri', + "response_headers": {"vary": "Origin"}, + }, + 'routes/0/action', + ), 'configure' + + resp = client.get( + url='/big.css', + headers={ + 'Host': 'localhost', + 'Connection': 'close', + 'Accept-Encoding': 'gzip', + }, + ) + + assert resp['status'] == 200, 'status 200' + assert resp['headers']['Vary'] == 'Origin, Accept-Encoding', 'merged' + + +@pytest.mark.parametrize( + ('configured', 'expected'), + [ + ('Origin', 'Origin, Accept-Encoding'), + ('*', '*'), + ('Accept-Encoding', 'Accept-Encoding'), + ], +) +def test_static_compression_vary_merge_identity(temp_dir, configured, expected): + # r->resp.vary_accept_encoding is set in nxt_http_comp_check_acceptable() + # before any coding is chosen, so the merge and the re-assert both run for + # an identity response too. The identity response is precisely the one a + # cache must not reuse for a gzip-capable client, so a merge that only + # worked on the coded path would leave the hole open from the other side. + assert 'success' in client.conf( + { + "share": f'{temp_dir}/assets$uri', + "response_headers": {"Vary": configured}, + }, + 'routes/0/action', + ), 'configure' + + resp = client.get( + url='/big.css', + headers={'Host': 'localhost', 'Connection': 'close'}, + ) + + assert resp['status'] == 200, 'status 200' + assert 'Content-Encoding' not in resp['headers'], 'not compressed' + assert resp['headers']['Vary'] == expected, 'merged on identity too' + + +# Uncovered by design, recorded so the gap is not mistaken for coverage: +# every test above drives the merge through "response_headers" against a +# "share", so the pre-existing Vary is always one Unit itself generated and +# response_headers then replaced. A Vary authored by an *application* reaches +# nxt_http_comp_merge_vary() by the other caller, +# nxt_router_response_ready_handler(), which copies the app's fields into +# r->resp before nxt_http_comp_check_acceptable() runs. Any CORS application +# emitting "Vary: Origin" takes that path, and nothing here exercises it -- +# including the merge's own case-insensitive field-name match. Covering it +# needs a language-module test, not a static one. + + +def _raw_get(url='/big.css', method='GET', **headers): + # Raw bytes: the body has to be decompressed, so it must not be decoded. + raw = client.http( + method, + url=url, + headers={'Host': 'localhost', 'Connection': 'close', **headers}, + encoding='latin-1', + read_buffer_size=1024 * 1024, + raw_resp=True, + ) + head, _, body = raw.partition('\r\n\r\n') + lines = head.split('\r\n') + status = int(lines[0].split(' ')[1]) + hdrs = dict(line.split(': ', 1) for line in lines[1:]) + body = body.encode('latin-1') + + if hdrs.get('Transfer-Encoding') == 'chunked': + body = client._parse_chunked_body(body) + + return status, hdrs, body + + +def _age(path): + # The ETag of a file written in the current second is weak by design + # (see test_static.py). Move the mtime back, so that a weak ETag can + # come only from compression. + when = os.stat(path).st_mtime - 10 + os.utime(path, (when, when)) + + +def test_static_compression_range_identity_refused(temp_dir): + # A Range is served as identity -- coding a byte slice would compress the + # wrong bytes -- so a client that sent "identity;q=0" must not be given + # one: it asked not to receive the file's own bytes and a 206 is exactly + # those. The request is still serveable, because it named a coding Unit + # has, so the Range is dropped and the full 200 is sent in that coding. + # + # Without the fix this is a 206 carrying identity bytes to a client that + # refused identity, which is what #355 reports. + data = Path(f'{temp_dir}/assets/big.css').read_bytes() + + status, headers, body = _raw_get( + **{'Accept-Encoding': 'gzip, identity;q=0', 'Range': 'bytes=0-9'} + ) + + assert status == 200, 'the Range is dropped, not the request' + assert headers.get('Content-Encoding') == 'gzip', 'served as gzip' + assert 'Content-Range' not in headers, 'no Content-Range on the full 200' + assert gzip.decompress(body) == data, 'the whole file, correctly coded' + + +def test_static_compression_svgz_not_compressed_twice(temp_dir): + # A ".svgz" file already has gzip coding. With every type compressed, + # it must still go out as its stored bytes: one gzip layer, a strong + # ETag, and no Vary, because the coding is not negotiated. Before, it + # had no type, so it was compressed again and labelled with one "gzip". + svg = b'' * 10 + data = gzip.compress(svg, mtime=0) + path = f'{temp_dir}/assets/a.svgz' + Path(path).write_bytes(data) + when = os.stat(path).st_mtime - 2 + os.utime(path, (when, when)) + + assert 'success' in client.conf_delete( + 'settings/http/compression/types' + ), 'compress every type' + + status, headers, body = _raw_get( + url='/a.svgz', **{'Accept-Encoding': 'gzip'} + ) + assert status == 200 + assert headers.get('Content-Type') == 'image/svg+xml' + assert headers.get('Content-Encoding') == 'gzip' + assert body == data, 'the stored bytes, not compressed again' + assert gzip.decompress(body) == svg + assert not headers['ETag'].startswith('W/'), 'strong ETag' + assert 'Vary' not in headers + + # "identity;q=0" does not drop the Range: the file is not sent as + # identity, and the slice is of the gzip bytes. + status, headers, body = _raw_get( + url='/a.svgz', + **{'Accept-Encoding': 'gzip, identity;q=0', 'Range': 'bytes=0-9'}, + ) + assert status == 206 + assert headers.get('Content-Encoding') == 'gzip' + assert headers['Content-Range'] == f'bytes 0-9/{len(data)}' + assert body == data[:10] + + # A client that accepts no coding still gets the gzip bytes, with 200. + # This is intended: no other representation of the file exists. For a + # file that can be sent as identity, the second header gives 406. + for accept in ('identity;q=0', 'identity;q=0, *;q=0'): + status, headers, body = _raw_get( + url='/a.svgz', **{'Accept-Encoding': accept} + ) + assert status == 200, accept + assert headers.get('Content-Encoding') == 'gzip', accept + assert body == data, accept + + assert ( + _raw_get(**{'Accept-Encoding': 'identity;q=0, *;q=0'})[0] == 406 + ), 'big.css: nothing acceptable' + + +def test_static_compression_identity_refused_below_min_length(): + # "min_length" is 10 and tiny.css is two bytes. So the gzip that made + # this request serveable is never applied, and the fallback is the + # identity that the client refused. The answer is 406, with a Range and + # without one: the Range is not what makes the request unserveable. + for extra in ({}, {'Range': 'bytes=0-1'}): + status, headers, _ = _raw_get( + '/tiny.css', + **{'Accept-Encoding': 'gzip, identity;q=0', **extra}, + ) + assert status == 406, 'a below-minimum gzip is not available' + assert 'Content-Encoding' not in headers + assert 'Content-Range' not in headers + + +def test_static_compression_identity_refused_without_eligible_coding(): + # The configured compressor applies only to text/css. This extensionless + # file has no media type. So no coding is applied to it, and identity is + # again all that is left. This client refused identity. + for extra in ({}, {'Range': 'bytes=0-1'}): + status, headers, _ = _raw_get( + '/raw', + **{'Accept-Encoding': 'gzip, identity;q=0', **extra}, + ) + assert status == 406, 'no configured coding can serve this response' + assert 'Content-Encoding' not in headers + assert 'Content-Range' not in headers + + +def test_static_compression_identity_refused_zero_length(temp_dir): + # A zero-length response transfers no representation bytes. So there is + # nothing that the client refused, and the response is not negotiated. + # It stays 200, and a Range against it keeps its 416. + Path(f'{temp_dir}/assets/empty.css').write_bytes(b'') + + status, headers, body = _raw_get( + '/empty.css', **{'Accept-Encoding': 'gzip, identity;q=0'} + ) + + assert status == 200, 'no bytes to refuse' + assert 'Content-Encoding' not in headers + assert body == b'' + + for accept in ('gzip, identity;q=0', 'gzip'): + status, headers, _ = _raw_get( + '/empty.css', **{'Accept-Encoding': accept, 'Range': 'bytes=0-4'} + ) + + assert status == 416, f'a range against no bytes: {accept!r}' + assert headers['Content-Range'] == 'bytes */0' + + +def test_static_compression_406_varies_on_accept_encoding(): + # A 406 from negotiation depends on Accept-Encoding, like each negotiated + # response. Without "Vary: Accept-Encoding", a cache that stores error + # responses can give this 406 to a later client that accepts identity. + for url, accept in ( + ('/big.css', 'identity;q=0, *;q=0'), # nothing acceptable + ('/tiny.css', 'gzip, identity;q=0'), # gzip below "min_length" + ('/raw', 'gzip, identity;q=0'), # media type outside "types" + ): + status, headers, _ = _raw_get(url, **{'Accept-Encoding': accept}) + assert status == 406, f'{url} {accept!r}' + assert headers.get('Vary') == 'Accept-Encoding', f'{url} {accept!r}' + + +def test_static_compression_min_length_is_per_compressor(): + # "min_length" belongs to each compressor. When the coding with the + # highest weight is below its own minimum, the next coding can still be + # above its minimum. gzip is at 1000 and deflate at 0, and tiny.css is + # two bytes: it is serveable as deflate. Before, gzip was selected on + # its weight alone, and the response was identity. For a client that + # refused identity, that was a 406 for a request that can be satisfied. + assert 'success' in client.conf( + { + "types": ["text/css"], + "compressors": [ + {"encoding": "gzip", "min_length": 1000}, + {"encoding": "deflate", "min_length": 0}, + ], + }, + 'settings/http/compression', + ), 'two compressors, different minimums' + + for extra in ({}, {'Accept-Encoding': 'gzip, deflate;q=0.5, identity;q=0'}): + headers = {'Accept-Encoding': 'gzip, deflate;q=0.5', **extra} + + status, hdrs, body = _raw_get('/tiny.css', **headers) + + assert status == 200, f'deflate can serve it: {headers}' + assert hdrs.get('Content-Encoding') == 'deflate', 'the eligible coding' + assert zlib.decompress(body) == b'ok', 'round-trips' + + # The weight order still decides between codings that can all be applied. + status, hdrs, _ = _raw_get( + '/big.css', **{'Accept-Encoding': 'gzip, deflate;q=0.5'} + ) + + assert status == 200, 'both are above their minimum here' + assert hdrs.get('Content-Encoding') == 'gzip', 'the highest weight wins' + + +def test_static_compression_wildcard_stands_for_every_coding(temp_dir): + # Sect. 12.5.3: "The asterisk '*' symbol in an Accept-Encoding field + # matches any available content coding not explicitly listed in the + # field." So "*" is not a synonym for identity. It offers each enabled + # coding that the client did not name, at its own weight. + # + # Before, "*" was read as identity only. Then it could not select a + # compressor. This request refuses identity and accepts everything else, + # and it was answered 406 although gzip was enabled, above its + # "min_length" and inside "types". + data = Path(f'{temp_dir}/assets/big.css').read_bytes() + + status, headers, body = _raw_get( + **{'Accept-Encoding': 'identity;q=0, *;q=1'} + ) + + assert status == 200, 'the wildcard offers gzip' + assert headers.get('Content-Encoding') == 'gzip', 'served as gzip' + assert gzip.decompress(body) == data, 'the whole file, correctly coded' + + # The wildcard does not match a coding that the field names. That coding + # keeps its own weight, however low. gzip is listed at 0.1, so gzip is + # worth 0.1. gzip is the only compressor, so it is still the only coding + # left when identity is refused. + status, headers, body = _raw_get( + **{'Accept-Encoding': 'identity;q=0, *;q=0.5, gzip;q=0.1'} + ) + + assert status == 200, 'an explicitly named gzip is still acceptable' + assert headers.get('Content-Encoding') == 'gzip', 'at its own weight' + assert gzip.decompress(body) == data + + # The wildcard does not bring back a coding that the field refused. + # gzip is out, and "*" offers identity here. + status, headers, body = _raw_get(**{'Accept-Encoding': 'gzip;q=0, *;q=1'}) + + assert status == 200, 'the wildcard still offers identity' + assert 'Content-Encoding' not in headers, 'gzip was refused by name' + assert body == data + + # A named coding wins a tie with the wildcard. The wildcard is the + # weaker statement, about a coding that the client did not name. + for spelling in ('gzip, *', 'gzip;q=0.5, *;q=0.5'): + status, headers, _ = _raw_get(**{'Accept-Encoding': spelling}) + + assert status == 200, f'tie with the wildcard: {spelling!r}' + assert headers.get('Content-Encoding') == 'gzip', 'the named coding' + + # When the field names no coding, the wildcard covers identity too, and + # identity takes the tie: nothing has to be applied to send it. + for spelling in ('*', '*;q=1'): + status, headers, body = _raw_get(**{'Accept-Encoding': spelling}) + + assert status == 200, f'the bare wildcard: {spelling!r}' + assert 'Content-Encoding' not in headers, 'identity takes the tie' + assert body == data + + +def test_static_compression_wildcard_skips_inapplicable_coding(temp_dir): + # The wildcard can stand only for a coding that is really applied. gzip + # is the only compressor, and tiny.css is below its "min_length". So "*" + # can offer only the identity that this client refused: 406, not a 200 of + # the bytes it refused. + status, headers, _ = _raw_get( + '/tiny.css', **{'Accept-Encoding': 'identity;q=0, *;q=1'} + ) + + assert status == 406, 'a below-minimum gzip is not available to "*"' + assert 'Content-Encoding' not in headers + + # "min_length" is set per compressor, so the wildcard must look past the + # first one. gzip is at 1000 and deflate at 0, and tiny.css is two + # bytes: "*" offers deflate, and the request is serveable. + assert 'success' in client.conf( + { + "types": ["text/css"], + "compressors": [ + {"encoding": "gzip", "min_length": 1000}, + {"encoding": "deflate", "min_length": 0}, + ], + }, + 'settings/http/compression', + ), 'two compressors, different minimums' + + status, headers, body = _raw_get( + '/tiny.css', **{'Accept-Encoding': 'identity;q=0, *;q=1'} + ) + + assert status == 200, 'the wildcard reaches the eligible coding' + assert headers.get('Content-Encoding') == 'deflate', 'the one that applies' + assert zlib.decompress(body) == b'ok', 'round-trips' + + # A coding that the wildcard offers at a higher weight wins against a + # coding that the field names at a lower weight: deflate at the wildcard + # weight 0.5 beats the named gzip at 0.1. + status, headers, body = _raw_get( + **{'Accept-Encoding': 'identity;q=0, *;q=0.5, gzip;q=0.1'} + ) + + assert status == 200 + assert headers.get('Content-Encoding') == 'deflate', 'the higher weight' + assert zlib.decompress(body) == Path( + f'{temp_dir}/assets/big.css' + ).read_bytes() + + +def test_static_compression_wildcard_without_compressors(temp_dir): + # With no compressor enabled, identity is the only available coding. So + # identity is all that "*" can stand for, and this client refused it. + assert 'success' in client.conf_delete( + 'settings/http/compression' + ), 'compression off' + + for spelling in ( + 'identity;q=0, *;q=1', + 'identity;q=0, *;q=0.5, gzip;q=0.1', + ): + status, headers, _ = _raw_get(**{'Accept-Encoding': spelling}) + + assert status == 406, f'"*" has only identity to offer: {spelling!r}' + assert 'Content-Encoding' not in headers + + # The wildcard still accepts identity when no token named it. + status, headers, body = _raw_get(**{'Accept-Encoding': 'gzip;q=0, *;q=1'}) + + assert status == 200, 'identity is what the wildcard offers' + assert 'Content-Encoding' not in headers + assert body == Path(f'{temp_dir}/assets/big.css').read_bytes() + + +def test_static_compression_range_identity_refused_guards(temp_dir): + # The cases either side of it, which must not move. + size = Path(f'{temp_dir}/assets/big.css').stat().st_size + + # Identity acceptable: a Range is still a 206 of identity bytes, which is + # deliberate and is what every other server does. + status, headers, body = _raw_get( + **{'Accept-Encoding': 'gzip', 'Range': 'bytes=0-9'} + ) + assert status == 206, 'gzip alone still gets its partial content' + assert 'Content-Encoding' not in headers, 'a 206 carries no coding' + assert headers['Content-Range'] == f'bytes 0-9/{size}' + assert body == b'body{color' + + # Nothing acceptable at all is 406, whether or not a Range is present. + # That outranks the Range and is unchanged by this fix. + for extra in ({}, {'Range': 'bytes=0-9'}): + status, _, _ = _raw_get( + **{'Accept-Encoding': 'identity;q=0, *;q=0', **extra} + ) + assert status == 406, 'unacceptable outranks any Range' + + # Refusing identity without a Range was already correct: full gzip 200. + status, headers, _ = _raw_get(**{'Accept-Encoding': 'gzip, identity;q=0'}) + assert status == 200 + assert headers.get('Content-Encoding') == 'gzip' + + # An explicitly named identity outranks the wildcard. The client refused + # everything it did not name and then named identity as acceptable, so a + # 206 of identity bytes is exactly what it asked for. Reading the + # wildcard as a veto here drops a range the client could take. + status, headers, body = _raw_get( + **{ + 'Accept-Encoding': 'gzip, identity;q=0.5, *;q=0', + 'Range': 'bytes=0-9', + } + ) + assert status == 206, 'an explicit identity;q>0 keeps its range' + assert 'Content-Encoding' not in headers + assert headers['Content-Range'] == f'bytes 0-9/{size}' + assert body == b'body{color' + + # A content coding is a token and tokens are case-insensitive + # (Sect. 8.4.1), so "Identity;q=0" refuses identity just as "identity;q=0" + # does. A case-sensitive compare drops the refusal on the floor and + # serves the 206 this whole test exists to prevent. + status, headers, _ = _raw_get( + **{'Accept-Encoding': 'gzip, Identity;q=0', 'Range': 'bytes=0-9'} + ) + assert status == 200, 'a mixed-case identity token still refuses' + assert headers.get('Content-Encoding') == 'gzip' + + # The weight is "('q' / 'Q') '=' qvalue" (Sect. 12.4.2), and an ABNF + # literal is case-insensitive anyway. A strstr() for ";q=" alone misses + # ";Q=", and for identity a missed weight is a missed refusal -- the + # request gets the 206 of identity bytes it asked not to receive. + status, headers, _ = _raw_get( + **{'Accept-Encoding': 'gzip, identity;Q=0', 'Range': 'bytes=0-9'} + ) + assert status == 200, 'an uppercase Q still refuses' + assert headers.get('Content-Encoding') == 'gzip' + + # "*" is a tchar, so "*foo" is a legal coding name that Unit does not + # have -- not the wildcard. Matching the wildcard on the first byte + # alone made an unknown coding refuse identity and cost the client a + # range it could have taken. + status, headers, body = _raw_get( + **{'Accept-Encoding': 'gzip, *foo;q=0', 'Range': 'bytes=0-9'} + ) + assert status == 206, 'an unknown coding is not the wildcard' + assert 'Content-Encoding' not in headers + assert headers['Content-Range'] == f'bytes 0-9/{size}' + assert body == b'body{color' + + # OWS is SP or HTAB (Sect. 5.6.3) and is legal either side of the + # weight's semicolon. Stripping only the space left the tab forms + # unparsed, so the element read as an unknown coding and took its + # refusal with it. + for spelling in ( + 'gzip, identity; q=0', + 'gzip, identity ;q=0', + 'gzip, identity; q=0', + ): + status, headers, _ = _raw_get( + **{'Accept-Encoding': spelling, 'Range': 'bytes=0-9'} + ) + assert status == 200, f'whitespace in the weight: {spelling!r}' + assert headers.get('Content-Encoding') == 'gzip' + + # Sect. 5.3: a list-valued field may arrive as several lines and must be + # read as one value joined by commas. A variable query answers with the + # first matching field only, so the second line went unread: in this order + # the refusal was lost and the 206 went out anyway, and in the other order + # the gzip the client would have taken was never seen and it drew a 406. + for lines_ae in (['gzip', 'identity;q=0'], ['identity;q=0', 'gzip']): + status, headers, _ = _raw_get( + **{'Accept-Encoding': lines_ae, 'Range': 'bytes=0-9'} + ) + assert status == 200, f'repeated field: {lines_ae!r}' + assert headers.get('Content-Encoding') == 'gzip' + + # Ignoring the Range is all or nothing. An unsatisfiable range from a + # client that refused identity would otherwise draw a 416 whose + # "Content-Range: bytes */size" reports the size of the very + # representation it refused. + status, headers, _ = _raw_get( + **{'Accept-Encoding': 'gzip, identity;q=0', 'Range': 'bytes=99999-'} + ) + assert status == 200, 'unsatisfiable range is ignored too' + assert headers.get('Content-Encoding') == 'gzip' + assert 'Content-Range' not in headers + + # A client that accepts identity still gets its 416. + status, headers, _ = _raw_get( + **{'Accept-Encoding': 'gzip', 'Range': 'bytes=99999-'} + ) + assert status == 416, 'an ordinary unsatisfiable range is still 416' + assert headers['Content-Range'] == f'bytes */{size}' + + +def test_static_compression_malformed_weight(temp_dir): + # A qvalue is "('0' ['.' 0*3DIGIT]) / ('1' ['.' 0*3('0')])" (Sect. + # 12.4.2). strtod() takes no digits at all from "q=" and "q=abc" and + # reports 0, reads "q=0x0" as hexadecimal and "q=0e0" as an exponent, so + # every one of these was read as an explicit refusal and answered 406 to + # a client that refused nothing. + size = Path(f'{temp_dir}/assets/big.css').stat().st_size + + for weight in ('', 'abc', '0x0', '0e0', 'nan', '.5', '2', '1.5'): + status, headers, body = _raw_get( + **{'Accept-Encoding': f'identity;q={weight}'} + ) + assert status == 200, f'a malformed weight is not a refusal: {weight!r}' + assert 'Content-Encoding' not in headers + assert len(body) == size + + # "q=nan" was worse than a refusal on any other coding: a NaN compares + # false against both range bounds, so the element survived the check, and + # false against the running best weight, so it was then selected. + status, headers, body = _raw_get(**{'Accept-Encoding': 'gzip;q=nan'}) + assert status == 200 + assert 'Content-Encoding' not in headers, 'a NaN weight selects nothing' + assert len(body) == size + + # A well-formed zero still refuses, including a fraction longer than the + # grammar's three digits: being strict about the digit count would read a + # refusal as an acceptance, which is the wrong way to be strict. + for weight in ('0', '0.0', '0.000', '0.0000'): + status, _, _ = _raw_get(**{'Accept-Encoding': f'identity;q={weight}'}) + assert status == 406, f'a well-formed zero still refuses: {weight!r}' + + # A parameter after the weight does not make the weight malformed. + status, headers, _ = _raw_get(**{'Accept-Encoding': 'gzip;q=0.5;ext=1'}) + assert status == 200 + assert headers.get('Content-Encoding') == 'gzip', 'q=0.5 then an extension' + + +def _precompressed(temp_dir, response_headers): + # A precompressed copy served as "$uri.gz", the usual way to serve such + # files with Unit. No "types": the ".gz" extension has no media type. + # So every coding in "compressors" applies to it, as in issue 557. + stored = gzip.compress(b'body{color:red}' * 500) + Path(f'{temp_dir}/assets/pre.css.gz').write_bytes(stored) + _age(f'{temp_dir}/assets/pre.css.gz') + + assert 'success' in client.conf( + {"compressors": [{"encoding": "gzip", "level": 5}]}, + 'settings/http/compression', + ), 'compression without types' + + action = {"share": f'{temp_dir}/assets$uri.gz'} + + if response_headers is not None: + action["response_headers"] = response_headers + + assert 'success' in client.conf(action, 'routes/0/action'), 'action' + + return stored + + +@pytest.mark.parametrize('name', ['Content-Encoding', 'content-encoding']) +def test_static_compression_response_headers_content_encoding(temp_dir, name): + # https://github.com/freeunitorg/freeunit/issues/557: "response_headers" + # are applied after the compressor ran. Their Content-Encoding replaced + # the one the compressor added, so the stored gzip bytes went out coded + # a second time under a single "gzip", with a weakened ETag. + stored = _precompressed( + temp_dir, {"Content-Type": "text/css", name: "gzip"} + ) + + for method in ('GET', 'HEAD'): + status, headers, body = _raw_get( + url='/pre.css', method=method, **{'Accept-Encoding': 'gzip'} + ) + + assert status == 200, f'{method} status' + assert headers[name] == 'gzip', f'{method} the configured coding' + assert headers['Content-Length'] == str(len(stored)), ( + f'{method} the stored length' + ) + assert not headers['ETag'].startswith('W/'), f'{method} strong ETag' + + if method == 'GET': + assert body == stored, 'the stored bytes, coded once' + assert gzip.decompress(body) == b'body{color:red}' * 500 + + +def test_static_compression_precompressed_control(temp_dir): + # The same share without a configured Content-Encoding is still + # compressed. So the test above passes because of the configured + # field, not because this share is never compressed. + stored = _precompressed(temp_dir, {"Content-Type": "text/css"}) + + status, headers, body = _raw_get( + url='/pre.css', **{'Accept-Encoding': 'gzip'} + ) + + assert status == 200, 'status' + assert headers['Content-Encoding'] == 'gzip', 'compressed' + assert headers['ETag'].startswith('W/'), 'weak ETag for the coded copy' + assert gzip.decompress(body) == stored, 'gzip of the stored bytes' + + +def test_static_compression_response_headers_content_encoding_removed( + temp_dir, +): + # A null value removes the field. The compressor must then not run, + # or the gzip bytes go out with no Content-Encoding at all. The body is + # identity, so a client that refused identity still gets a 406. + _age(f'{temp_dir}/assets/big.css') + + assert 'success' in client.conf( + { + "share": f'{temp_dir}/assets$uri', + "response_headers": {"Content-Encoding": None}, + }, + 'routes/0/action', + ), 'configure' + + status, headers, body = _raw_get(**{'Accept-Encoding': 'gzip'}) + + assert status == 200, 'status' + assert 'Content-Encoding' not in headers, 'removed' + assert headers['Content-Length'] == '7500', 'the identity length' + assert body == b'body{color:red}' * 500, 'identity bytes' + assert not headers['ETag'].startswith('W/'), 'strong ETag' + + status, _, _ = _raw_get(**{'Accept-Encoding': 'gzip, identity;q=0'}) + + assert status == 406, 'identity refused' + + +def test_static_compression_response_headers_content_encoding_template( + temp_dir, +): + # A template value is resolved when compression is decided, with the + # same code that later sets the field. A value that is not safe in a + # field is never sent. So it must not keep the compressor out either, + # or the body goes out with no Content-Encoding to a client that + # refused identity. + stored = _precompressed( + temp_dir, {"Content-Type": "text/css", "Content-Encoding": "$arg_c"} + ) + + status, headers, body = _raw_get( + url='/pre.css?c=gzip', **{'Accept-Encoding': 'gzip'} + ) + + assert status == 200, 'status' + assert headers['Content-Encoding'] == 'gzip', 'the configured coding' + assert body == stored, 'the stored bytes, coded once' + + status, headers, body = _raw_get( + url='/pre.css?c=gzip%0d%0a', + **{'Accept-Encoding': 'gzip, identity;q=0'}, + ) + + assert status == 200, 'unsafe value: status' + assert headers.get('Content-Encoding') == 'gzip', 'the compressor coding' + assert gzip.decompress(body) == stored, 'gzip of the stored bytes' + + +def test_static_compression_response_headers_content_encoding_skipped_key( + temp_dir, +): + # Keys are applied in order, and an unsafe value skips its key. So the + # result comes from the last key that is not skipped: here the null, + # which removes the field. The body is identity, and a client that + # refused identity gets 406. + _precompressed( + temp_dir, + { + "Content-Type": "text/css", + "Content-Encoding": None, + "content-encoding": "$arg_c", + }, + ) + + status, _, _ = _raw_get( + url='/pre.css?c=gzip%0d%0a', + **{'Accept-Encoding': 'gzip, identity;q=0'}, + ) + + assert status == 406, 'identity refused' + + +def test_static_compression_response_headers_content_encoding_empty( + temp_dir, +): + # "$arg_c" with no "c" argument expands to an empty string, as "?c=" + # does. An empty string is a value, not a removal: the field is sent + # empty, nothing is compressed, and a client that refused identity gets + # no 406. + stored = _precompressed( + temp_dir, {"Content-Type": "text/css", "Content-Encoding": "$arg_c"} + ) + + for url, accept in ( + ('/pre.css', 'gzip, identity;q=0'), + ('/pre.css?c=', 'gzip'), + ): + status, headers, body = _raw_get( + url=url, **{'Accept-Encoding': accept} + ) + + assert status == 200, f'{url} status' + assert headers['Content-Encoding'] == '', f'{url} an empty field' + assert body == stored, f'{url} the stored bytes' + + +def test_static_compression_response_headers_content_encoding_resolved_once( + temp_dir, +): + # The check resolves the value, and the header gets that same value. + # "$response_header_*" is not cached, and the static handler adds + # Accept-Ranges only after the check. A second resolution at send + # would give "bytes" here. + stored = _precompressed( + temp_dir, + { + "Content-Type": "text/css", + "Content-Encoding": "$response_header_accept_ranges", + }, + ) + + status, headers, body = _raw_get( + url='/pre.css', **{'Accept-Encoding': 'gzip'} + ) + + assert status == 200, 'status' + assert headers['Accept-Ranges'] == 'bytes', 'Accept-Ranges is sent' + assert headers['Content-Encoding'] == '', 'the value of the check' + assert body == stored, 'the stored bytes' diff --git a/test/test_static_types.py b/test/test_static_types.py index e931d949a..9f2f893ba 100644 --- a/test/test_static_types.py +++ b/test/test_static_types.py @@ -1,3 +1,4 @@ +import gzip from pathlib import Path import pytest @@ -172,3 +173,20 @@ def test_static_types_custom_mime(temp_dir): {"share": f'{temp_dir}/assets$uri', "types": ["test/mime-type"]} ) check_body('/file', '') + + +def test_static_types_svgz(temp_dir): + # ".svgz" is in the default MIME table as image/svg+xml. So a "types" + # rule that names image/* serves it, with its stored gzip coding. + # Before, the file had no type and the rule refused it with 403. + data = gzip.compress(b'', mtime=0) + Path(f'{temp_dir}/assets/a.svgz').write_bytes(data) + + action_update({"share": f'{temp_dir}/assets$uri', "types": ["image/*"]}) + + resp = client.get(url='/a.svgz', encoding='latin-1') + assert resp['status'] == 200, 'image/* serves .svgz' + assert resp['headers']['Content-Type'] == 'image/svg+xml' + assert resp['headers']['Content-Encoding'] == 'gzip' + assert resp['headers']['ETag'].endswith('-gzip"') + assert resp['body'].encode('latin-1') == data diff --git a/test/test_status.py b/test/test_status.py index 2f58f5ccd..b6ef7ae3e 100644 --- a/test/test_status.py +++ b/test/test_status.py @@ -195,7 +195,9 @@ def check_application(name, running, starting, idle, active): 'processes': { 'running': running, 'starting': starting, + 'unaccounted': 0, 'idle': idle, + 'detached': 0, }, 'requests': {'active': active}, } diff --git a/test/test_tls.py b/test/test_tls.py index ce605684c..c61b6fe6b 100644 --- a/test/test_tls.py +++ b/test/test_tls.py @@ -1,11 +1,16 @@ import io +import os +import shutil +import socket import ssl import subprocess import time +from concurrent.futures import ThreadPoolExecutor from pathlib import Path import pytest +from conftest import unit_run, unit_stop from unit.applications.tls import ApplicationTLS from unit.option import option @@ -83,6 +88,12 @@ def generate_ca_conf(): Path(f'{option.temp_dir}/certindex.attr').touch() +def replace_cert(name='default'): + # Make a new key pair under the same name and store it over the old one. + client.certificate(name, False) + return client.certificate_load(name) + + def remove_tls(application='empty', port=8080): assert 'success' in client.conf( {"pass": f"applications/{application}"}, f'listeners/*:{port}' @@ -168,7 +179,6 @@ def test_tls_certificate_remove_nonexisting(): ), 'remove nonexistings certificate' -@pytest.mark.skip('not yet') def test_tls_certificate_update(): client.load('empty') @@ -178,36 +188,829 @@ def test_tls_certificate_update(): cert_old = ssl.get_server_certificate(('127.0.0.1', 8080)) - client.certificate() + # The listener names the bundle, so Unit applies the configuration + # again before it answers. + assert ( + replace_cert().get('success') == 'Certificate chain updated.' + ), 'replaced' assert cert_old != ssl.get_server_certificate( ('127.0.0.1', 8080) ), 'update certificate' + assert 'chain' in client.conf_get('/certificates/default'), 'listed' + + +def test_tls_certificate_update_unused(): + client.load('empty') + + client.certificate() + client.certificate('unused') + + add_tls() + + # No listener names the bundle, so there is no reconfiguration. + assert ( + replace_cert('unused').get('success') == 'Certificate chain uploaded.' + ), 'stored' + + +def test_tls_certificate_update_unchanged(): + client.load('empty') + + client.certificate() + + # Tickets are off by default. With "tickets": true and no key in the + # configuration, the key is a random one of the SSL_CTX that issued the + # ticket. A reconfiguration builds a new context with a new key, so a + # resumed session proves that the listener kept its context. + assert 'success' in client.conf( + { + "pass": "applications/empty", + "tls": {"certificate": "default", "session": {"tickets": True}}, + }, + 'listeners/*:8080', + ) + + path = f'{option.temp_dir}/state/certs/default' + stored = os.stat(path) + + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + context.maximum_version = ssl.TLSVersion.TLSv1_2 + + def connect(session=None): + with socket.create_connection(('127.0.0.1', 8080)) as sock: + with context.wrap_socket(sock, session=session) as ssock: + ssock.sendall(b'GET / HTTP/1.1\r\nHost: localhost\r\n\r\n') + + resp = b'' + while b'\r\n\r\n' not in resp: + resp += ssock.recv(4096) + + assert resp.startswith(b'HTTP/1.1 200'), 'request' + + return ssock.session, ssock.session_reused + + try: + session, reused = connect() + except ssl.SSLError as e: + # Ticket resumption needs TLS 1.2. A system crypto policy can + # turn it off. + pytest.skip(f'TLS 1.2 is not available: {e}') + + assert not reused, 'new session' + + _, reused = connect(session) + assert reused, 'session resumed' + + # The same bundle again. The answer is the one a store gives, but + # main writes nothing, and the router keeps its contexts. + assert ( + client.certificate_load('default').get('success') + == 'Certificate chain updated.' + ), 'unchanged in use' + + stored_default = os.stat(path) + assert (stored.st_ino, stored.st_mtime_ns) == ( + stored_default.st_ino, + stored_default.st_mtime_ns, + ), 'not stored again' + + _, reused = connect(session) + assert reused, 'no reconfiguration' + + # The same bundle under a name no listener uses. + client.certificate('unused') + + path_unused = f'{option.temp_dir}/state/certs/unused' + stored = os.stat(path_unused) + + assert ( + client.certificate_load('unused').get('success') + == 'Certificate chain uploaded.' + ), 'unchanged unused' + + after = os.stat(path_unused) + assert stored.st_ino == after.st_ino, 'unused not stored again' + + # A new certificate is stored and applied, as before. + assert ( + replace_cert().get('success') == 'Certificate chain updated.' + ), 'replaced' + + assert os.stat(path).st_ino != stored_default.st_ino, 'new bundle stored' + + _, reused = connect(session) + assert not reused, 'reconfigured' + + +def test_tls_certificate_update_keepalive(): + client.load('empty') + + client.certificate() + + add_tls() + + (resp, sock) = client.get_ssl( + headers={'Host': 'localhost', 'Connection': 'keep-alive'}, + start=True, + read_timeout=1, + ) + + assert resp['status'] == 200, 'keepalive 1' + + cert_old = sock.getpeercert(True) + + assert 'success' in replace_cert(), 'replaced' + + # The accepted connection keeps its old TLS context. + (resp, sock) = client.get_ssl( + headers={'Host': 'localhost', 'Connection': 'close'}, + sock=sock, + start=True, + ) + + assert resp['status'] == 200, 'keepalive 2' + assert sock.getpeercert(True) == cert_old, 'old connection, old cert' + sock.close() + + (resp, sock) = client.get_ssl( + headers={'Host': 'localhost', 'Connection': 'close'}, start=True + ) + + assert resp['status'] == 200, 'new connection' + assert sock.getpeercert(True) != cert_old, 'new connection, new cert' + + +def test_tls_certificate_update_inflight(): + client.load('empty') + + client.certificate() + + add_tls() + + # Send half of the request before the replacement and the rest after. + sock = client.http( + b'GET / HTTP/1.1\r\nHost: localhost\r\n', + raw=True, + no_recv=True, + wrapper=client._default_context.wrap_socket, + ) + + cert_old = sock.getpeercert(True) + + assert 'success' in replace_cert(), 'replaced' + + resp = client.http(b'Connection: close\r\n\r\n', raw=True, sock=sock) + + assert resp['status'] == 200, 'in-flight request' + assert cert_old != ssl.get_server_certificate( + ('127.0.0.1', 8080) + ), 'new handshake, new cert' + + +def test_tls_certificate_update_mismatch(skip_alert): + skip_alert(r'certificate and private key do not match') + + client.load('empty') + + client.certificate() + + add_tls() + + cert_old = ssl.get_server_certificate(('127.0.0.1', 8080)) + + # Unit refuses a bundle with the wrong key. Nothing changes. + client.certificate('other', False) + + assert 'error' in client.certificate_load('default', 'other'), 'refused' + + assert cert_old == ssl.get_server_certificate( + ('127.0.0.1', 8080) + ), 'old certificate still served' + + assert 'chain' in client.conf_get('/certificates/default'), 'still listed' + + +def test_tls_certificate_update_store_fail(skip_alert): + skip_alert(r'unlink.*failed', r'failed to store certificate') + + client.load('empty') + + client.certificate() + + add_tls() + + cert_old = ssl.get_server_certificate(('127.0.0.1', 8080)) + info_old = client.conf_get('/certificates/default') + + # A directory with a file in it blocks the temporary file of main. + tmp = f'{option.temp_dir}/state/certs/.store.tmp' + os.makedirs(tmp) + Path(f'{tmp}/block').touch() + + assert ( + replace_cert()['error'] == 'Failed to store certificate.' + ), 'store failed' + + # The old metadata is back, and the old bundle stays in use. + assert client.conf_get('/certificates/default') == info_old, 'old info' + assert cert_old == ssl.get_server_certificate( + ('127.0.0.1', 8080) + ), 'old certificate still served' + + # A new name is removed from the metadata. + assert 'error' in replace_cert('new'), 'new store failed' + assert 'new' not in client.conf_get('/certificates'), 'no new info' + assert 'new' not in client.conf_get('/')['certificates'], 'not in root' + + shutil.rmtree(tmp) + + +def test_tls_certificate_upload_stored(): + """The 200 of an upload comes only when main has stored the bundle. + Main stores it in a child process + (https://github.com/freeunitorg/freeunit/issues/516) and answers when + the child exits. Two uploads sent at the same time both land.""" + + names = ('first', 'second', 'third') + + bundles = {} + + for name in names: + client.certificate(name, False) + + with open(f'{option.temp_dir}/{name}.key', 'rb') as k, open( + f'{option.temp_dir}/{name}.crt', 'rb' + ) as c: + bundles[name] = k.read() + c.read() + + certs = Path(f'{option.temp_dir}/state/certs') + + # One upload. The file is complete when the answer arrives. + assert 'success' in client.conf( + bundles['first'], '/certificates/first' + ), 'first uploaded' + assert (certs / 'first').read_bytes() == bundles['first'], 'first stored' + + # Two uploads at the same time. The controller runs one after the + # other, and main stores each one. + def upload(name): + return ApplicationTLS().conf(bundles[name], f'/certificates/{name}') + + with ThreadPoolExecutor(max_workers=2) as pool: + answers = list(pool.map(upload, ('second', 'third'))) + + for name, answer in zip(('second', 'third'), answers): + assert 'success' in answer, f'{name} uploaded' + assert (certs / name).read_bytes() == bundles[name], f'{name} stored' + + assert set(names) <= set(client.conf_get('/certificates')), 'listed' + assert not (certs / '.store.tmp').exists(), 'no temporary file' + + +def unit_children(unit_pid, name): + out = subprocess.check_output( + ['ps', 'ax', '-o', 'pid=,ppid=,args='] + ).decode() + return [ + f[0] + for f in (line.split(None, 2) for line in out.splitlines()) + if len(f) == 3 and f[1] == str(unit_pid) and name in f[2] + ] + + +def trace_fsync(unit_pid): + """Delay each fsync(2) of main and its children by 2 s with strace. + Skip the test when strace is not installed or cannot attach.""" + + strace = shutil.which('strace') + if strace is None: + pytest.skip('strace is not installed') + + tracer = subprocess.Popen( + [ + strace, + '-f', + '-p', + str(unit_pid), + '-o', + '/dev/null', + '-e', + 'trace=fsync', + '-e', + 'inject=fsync:delay_exit=2000000', + ], + stderr=subprocess.PIPE, + ) + + # strace says "Process N attached" on stderr once it traces main. + line = tracer.stderr.readline().decode() + if 'attached' not in line: + tracer.kill() + tracer.wait() + pytest.skip(f'strace cannot attach: {line.strip()}') + + return tracer + + +def test_tls_certificate_upload_router_restart( + skip_alert, skip_fds_check, unit_pid +): + """A router that restarts while main stores a bundle does not make the + controller run the upload again. The upload gets one 200, a change + sent during the store runs after it, and the controller survives. + + strace delays each fsync(2) of main and its children by 2 s, so the + store child of the upload runs for about 4 s.""" + + def children(name): + return unit_children(unit_pid, name) + + client.certificate('slow', False) + + with open(f'{option.temp_dir}/slow.key', 'rb') as k, open( + f'{option.temp_dir}/slow.crt', 'rb' + ) as c: + bundle = k.read() + c.read() + + controller = children('unit: controller') + router = children('unit: router') + assert len(controller) == 1 and len(router) == 1, 'processes' + + tracer = trace_fsync(unit_pid) + + done = {} + stored = Path(f'{option.temp_dir}/state/certs/slow') + + def upload(): + body = ApplicationTLS().conf(bundle, '/certificates/slow') + on_disk = stored.exists() and stored.read_bytes() == bundle + done['upload'] = (body, on_disk) + + def change(): + start = time.monotonic() + body = ApplicationTLS().conf( + {"http": {"idle_timeout": 77}}, 'settings' + ) + done['change'] = (time.monotonic() - start, body) + + try: + with ThreadPoolExecutor(max_workers=2) as pool: + first = pool.submit(upload) + + # The store child is in its first fsync(2) now. + time.sleep(1) + + skip_fds_check(router=True) + skip_alert(fr'process {router[0]} exited on signal 9') + subprocess.call(['kill', '-9', router[0]]) + + for _ in range(50): + new = children('unit: router') + if new and new != router: + break + time.sleep(0.1) + + # Let the new router get the configuration. + time.sleep(0.5) + + second = pool.submit(change) + + first.result(timeout=30) + second.result(timeout=30) + + finally: + tracer.terminate() + tracer.wait(10) + + assert 'success' in done['upload'][0], 'one 200 for the upload' + assert done['upload'][1], 'the bundle is on disk at the 200' + assert 'success' in done['change'][1], 'the change is applied' + + # The store runs about 4 s, and it started about 1.5 s before the + # change was sent. A change that did not wait takes milliseconds. + assert done['change'][0] > 1, 'the change waited for the store' + + # Main has answered the upload; the controller must still run. + time.sleep(0.5) + + assert children('unit: controller') == controller, 'controller survived' + assert client.conf_get('settings/http/idle_timeout') == 77 + + +def test_tls_certificate_update_router_restart( + skip_alert, skip_fds_check, unit_pid +): + """A listener uses the bundle, and the router restarts while main + stores it. The store ends while the new router still applies its + first configuration, because the application takes 8 s to start. The + controller sends the configuration again only after that apply ends. + Before, it sent it at once. The two applies ran at the same time in + the router: the upload got 500, the listener stopped answering, and the + next change crashed the router. + + strace delays each fsync(2) of main and its children by 2 s, so the + store child of the upload runs for about 4 s.""" + + def children(name): + return unit_children(unit_pid, name) + + bundles = {} + + for name in ('old', 'new'): + client.certificate(name, False) + + with open(f'{option.temp_dir}/{name}.key', 'rb') as k, open( + f'{option.temp_dir}/{name}.crt', 'rb' + ) as c: + bundles[name] = k.read() + c.read() + + def served(): + pem = ssl.get_server_certificate(('127.0.0.1', 8080), timeout=10) + return ssl.PEM_cert_to_DER_cert(pem) + + def der(name): + pem = Path(f'{option.temp_dir}/{name}.crt').read_text(encoding='utf-8') + return ssl.PEM_cert_to_DER_cert(pem) + + assert 'success' in client.conf(bundles['old'], '/certificates/c') + + app = f'{option.test_dir}/python/slow_start' + + assert 'success' in client.conf( + { + "listeners": { + "*:8080": { + "pass": "applications/slow_start", + "tls": {"certificate": "c"}, + } + }, + "applications": { + "slow_start": { + "type": "python", + "processes": 1, + "path": app, + "working_directory": app, + "module": "wsgi", + "environment": {"UNIT_SLOW_START": "8"}, + } + }, + } + ) + + assert served() == der('old'), 'the old certificate before' + + router = children('unit: router') + assert len(router) == 1, 'one router' + + stored = Path(f'{option.temp_dir}/state/certs/c') + + tracer = trace_fsync(unit_pid) + + done = {} + + def upload(): + start = time.monotonic() + body = ApplicationTLS().conf(bundles['new'], '/certificates/c') + done['upload'] = (time.monotonic() - start, body) + + try: + with ThreadPoolExecutor(max_workers=1) as pool: + first = pool.submit(upload) + + # The store child is in its first fsync(2) now. + time.sleep(1) + + assert stored.read_bytes() == bundles['old'], 'store not done' + + skip_fds_check(router=True) + skip_alert(fr'process {router[0]} exited on signal 9') + subprocess.call(['kill', '-9', router[0]]) + + first.result(timeout=60) + + finally: + tracer.terminate() + tracer.wait(10) + + elapsed, body = done['upload'] + + assert body.get('success') == 'Certificate chain updated.', 'applied' + + # The store ends after about 4 s. The first configuration of the new + # router ends after the application starts, 8 s after the kill. + assert elapsed > 6, 'the upload waited for the first configuration' + + assert served() == der('new'), 'the new certificate' + + router = children('unit: router') + assert len(router) == 1, 'one new router' + + assert 'success' in client.conf( + {"http": {"idle_timeout": 77}}, 'settings' + ), 'a later change' + + assert served() == der('new'), 'the new certificate after the change' + assert client.get_ssl()['status'] == 200, 'the listener answers' + assert children('unit: router') == router, 'the router survived' + + +def test_tls_certificate_update_controller_restart( + skip_alert, skip_fds_check, unit_pid +): + """A listener uses the bundle, and the controller exits while main + stores a new one. Main starts the new controller only when the store + ends, so the new controller reads the new bundle from disk. It lists + the new certificate, and the router serves it. When main started the + controller at once, the new controller read the old bundle. The answer + of the store went to the controller that had gone, so the router served + the old certificate after the store ended. + + strace delays each fsync(2) of main and its children by 2 s, so the + store child of the upload runs for about 4 s.""" + + def children(name): + return unit_children(unit_pid, name) + + bundles = {} + + for name in ('old', 'new'): + client.certificate(name, False) + + with open(f'{option.temp_dir}/{name}.key', 'rb') as k, open( + f'{option.temp_dir}/{name}.crt', 'rb' + ) as c: + bundles[name] = k.read() + c.read() + + def served(): + pem = ssl.get_server_certificate(('127.0.0.1', 8080), timeout=10) + return ssl.PEM_cert_to_DER_cert(pem) + + def der(name): + pem = Path(f'{option.temp_dir}/{name}.crt').read_text(encoding='utf-8') + return ssl.PEM_cert_to_DER_cert(pem) + + client.load('empty') + + assert 'success' in client.conf(bundles['old'], '/certificates/c') + + add_tls(cert='c') + + assert served() == der('old'), 'the old certificate before' + + controller = children('unit: controller') + assert len(controller) == 1, 'one controller' + + stored = Path(f'{option.temp_dir}/state/certs/c') + + tracer = trace_fsync(unit_pid) + + done = {} + + def upload(): + done['upload'] = ApplicationTLS().put( + url='/certificates/c', + sock_type='unix', + addr=f'{option.temp_dir}/control.unit.sock', + body=bundles['new'], + ) + + try: + with ThreadPoolExecutor(max_workers=1) as pool: + first = pool.submit(upload) + + # The store child is in its first fsync(2) now. + time.sleep(1) + + assert stored.read_bytes() == bundles['old'], 'store not done' + + skip_fds_check(controller=True) + skip_alert(fr'process {controller[0]} exited on signal 9') + subprocess.call(['kill', '-9', controller[0]]) + + first.result(timeout=30) + + # The store child is a fork of main, with the title of main. + for _ in range(100): + if not children('unit: main'): + break + time.sleep(0.1) + + assert not children('unit: main'), 'the store ended' + + for _ in range(100): + new = children('unit: controller') + if new and new != controller: + break + time.sleep(0.1) + + assert new and new != controller, 'a new controller' + + finally: + tracer.terminate() + tracer.wait(10) + + assert done['upload'] == {}, 'the controller exited before it answered' + assert stored.read_bytes() == bundles['new'], 'the new bundle on disk' + + # The new controller listens after the router applies its configuration. + assert ( + client.conf_get('/certificates/c/chain/0/subject/common_name') + == 'new' + ), 'the controller lists the new certificate' + + assert served() == der('new'), 'the router serves the new certificate' + assert client.get_ssl()['status'] == 200, 'the listener answers' + + +def test_tls_certificate_update_long_name(): + client.certificate('default', False) + + with open(f'{option.temp_dir}/default.key', 'rb') as k, open( + f'{option.temp_dir}/default.crt', 'rb' + ) as c: + bundle = k.read() + c.read() + + # The longest file name on most file systems. The temporary file of + # main has a fixed name, so the store does not need a longer name. + name = 'a' * 255 + + for _ in range(2): + assert 'success' in client.conf( + bundle, f'/certificates/{name}' + ), 'stored' + + assert 'chain' in client.conf_get(f'/certificates/{name}'), 'listed' + + # One byte more cannot be a file name. The controller refuses it before + # main sees it, so the answer is 400 and not 500. + assert ( + client.conf(bundle, f'/certificates/{name}a').get('error') + == 'Invalid certificate name.' + ), 'name too long' + + assert 'error' in client.conf_get(f'/certificates/{name}a'), 'not stored' + + +def test_tls_certificate_fingerprint(): + client.certificate() + + # The same value that the OpenSSL command line prints, after the "=". + out = subprocess.check_output( + [ + 'openssl', + 'x509', + '-noout', + '-fingerprint', + '-sha256', + '-in', + f'{option.temp_dir}/default.crt', + ] + ) + + expected = out.decode().split('=', 1)[1].strip() + + assert len(expected) == 95, 'openssl fingerprint' + + assert ( + client.conf_get('/certificates/default/fingerprint') == expected + ), 'fingerprint' + + assert ( + client.conf_get('/certificates')['default']['fingerprint'] == expected + ), 'fingerprint listed' + + # A new certificate under the same name gets a new fingerprint. + assert 'success' in replace_cert(), 'replaced' + + assert ( + client.conf_get('/certificates/default/fingerprint') != expected + ), 'fingerprint changed' + + +def test_tls_certificate_update_sni(): + client.load('empty') + + client.certificate('default') + client.certificate('localhost') + + add_tls(cert=['default', 'localhost']) + + def peer_cert(host): + (resp, sock) = client.get_ssl( + headers={'Host': host, 'Connection': 'close'}, start=True + ) + + assert resp['status'] == 200, host + return sock.getpeercert(True) + + default_old = peer_cert('default') + localhost_old = peer_cert('localhost') + + assert default_old != localhost_old, 'sni selects the bundle' + + # Replace one bundle of the array. The other bundle does not change. + assert ( + replace_cert('localhost').get('success') + == 'Certificate chain updated.' + ), 'replaced' + + assert peer_cert('default') == default_old, 'other element untouched' + assert peer_cert('localhost') != localhost_old, 'element replaced' + + +def test_tls_certificate_update_restart(requires_restart): + client.certificate() + + # Use no application. unit_run() expects a restarted instance to have + # no applications. + assert 'success' in client.conf( + { + "listeners": { + "*:8080": {"pass": "routes", "tls": {"certificate": "default"}} + }, + "routes": [{"action": {"return": 200}}], + "applications": {}, + } + ) + + assert 'success' in replace_cert(), 'replaced' + + cert_new = ssl.get_server_certificate(('127.0.0.1', 8080)) + + temp_dir_old = option.temp_dir + statedir = f'{temp_dir_old}/state' + + # Main renamed the bundle into place. No temporary file is left. + assert [ + name for name in os.listdir(f'{statedir}/certs') if name[0] == '.' + ] == [], 'no temporary left' + + unit_stop() + + try: + # The fixture owns the new instance. The fixture stops it, checks + # its log, and removes its temp dir. This test removes the old one. + unit_run(state_dir=statedir) + + assert cert_new == ssl.get_server_certificate( + ('127.0.0.1', 8080) + ), 'replaced bundle survives a restart' + + assert 'chain' in client.conf_get('/certificates/default'), 'listed' + + finally: + unit_stop() + shutil.rmtree(temp_dir_old, ignore_errors=True) + + +def test_tls_certificate_too_large(): + client.load('empty') + + # The bundle is over the 1 MiB limit. The controller refuses it, not main. + resp = client.put( + **client._get_args('/certificates/big', b'-' * (1024 * 1024 + 1)) + ) + + assert resp['status'] == 413, resp['body'] + assert 'too large' in resp['body'], 'too large message' + + assert 'error' in client.conf_get('/certificates/big'), 'not stored' + def test_tls_certificate_key_incorrect(skip_alert): - skip_alert(r'nxt_openssl_chain_file\(\) failed', r'failed to apply new conf') + skip_alert(r'certificate and private key do not match') client.load('empty') client.certificate('first', False) client.certificate('second', False) - # A structurally valid bundle (a private key + a certificate) is stored - # fine -- the cert store only checks PEM structure. The key/cert - # *mismatch* is caught later, when the router builds the listener's - # SSL_CTX (SSL_CTX_check_private_key), rather than being left to fail at - # the first handshake. - assert 'success' in client.certificate_load( + # The bundle is structurally valid (a private key + a certificate), but + # the key belongs to another certificate. The store refuses it at + # upload, so a listener can never name it. + assert 'error' in client.certificate_load( 'first', 'second' - ), 'mismatched bundle stored' + ), 'mismatched bundle refused' - # pass at the existing app so the mismatched cert is the only reason - # the listener config can be rejected. - assert 'error' in client.conf( - {"pass": "applications/empty", "tls": {"certificate": 'first'}}, - 'listeners/*:8080', - ), 'mismatched key/cert rejected at listener config' + assert 'error' in client.conf_get('/certificates/first'), 'not stored' + + +def test_tls_certificate_dot_name(): + client.load('empty') + + client.certificate('default', False) + + # Names starting with "." are reserved for the store's own files. + for name in ['.', '..', '.default', '.default.tmp']: + assert 'error' in client.conf( + b'', f'/certificates/{name}' + ), f'dot name {name}' + + assert 'success' in client.certificate_load('default'), 'plain name' def test_tls_certificate_change(): diff --git a/test/test_tls_conf_command.py b/test/test_tls_conf_command.py index 5a9a3f32c..555ec91d7 100644 --- a/test/test_tls_conf_command.py +++ b/test/test_tls_conf_command.py @@ -117,3 +117,26 @@ def check_conf_commands(conf_commands): check_conf_commands({"protocol": {}}) check_conf_commands({"protocol": "blah"}) check_conf_commands({"protocol": "TLSv1.2", "blah": ""}) + + +def test_tls_conf_command_invalid_path(skip_alert): + skip_alert(r'SSL_CONF_cmd', r'failed to apply new conf') + + # nxt_conf_vldt_object_conf_commands() iterates the object itself instead + # of going through the instrumented iterator, so the command name has to + # be pushed onto the validation path explicitly; otherwise the reported + # location stops at conf_commands and does not name the failing command. + r = client.conf( + {"certificate": "default", "conf_commands": {"protocol": {}}}, + 'listeners/*:8080/tls', + ) + + assert 'error' in r + + # builds without SSL_CONF_cmd reject the member itself, so there is no + # command name to attribute the error to. The validator's text is in + # "detail"; "error" is the generic "Invalid configuration." title. + if 'built without' in r['detail']: + pytest.skip('built without conf_commands support') + + assert r['location']['path'].endswith('/tls/conf_commands/protocol') diff --git a/test/test_tls_session.py b/test/test_tls_session.py index ea7f81bfd..b6f71b2aa 100644 --- a/test/test_tls_session.py +++ b/test/test_tls_session.py @@ -63,6 +63,28 @@ def connect(ctx=None, session=None): conn.set_session(session) conn.do_handshake() + + # Complete one request before the session is used again. OpenSSL + # inserts a server session into the shared SSL_CTX cache from + # tls_finish_handshake(), which runs after the Finished message it + # flushed has already reached the client. A handshake that has + # returned on the client therefore proves nothing about the insert: + # with more than one router engine the immediate reconnect can be + # accepted by an engine that has not seen it yet, and a full + # handshake results -- the flake tracked in issue #51. + # + # SSL_do_handshake() does not return on the server until the insert + # has happened, and Unit does not read application data before it + # returns (nxt_openssl_conn_handshake(), src/nxt_openssl.c), so a + # response the client has read proves the session is in the cache. + conn.sendall(b'GET / HTTP/1.1\r\nHost: localhost\r\n\r\n') + + response = b'' + while b'\r\n\r\n' not in response: + response += conn.recv(4096) + + assert response.startswith(b'HTTP/1.1 200'), 'request completed' + conn.shutdown() return ( @@ -119,11 +141,14 @@ def test_tls_session(): reason='session reuse is not supported', ) def test_tls_session_timeout(): - # OpenSSL evaluates session expiry in whole seconds (time_t), so a 1s - # timeout leaves barely over one second of real slack for the resume - # below and evicts the session early under CI load, making the "no - # timeout" assertion flaky. Use a comfortable window and sleep past it. - assert 'success' in add_session(cache_size=5, timeout=4) + # The two halves are configured separately rather than fitted into one + # window. "Still cached" only needs a timeout no run can outlast, and + # "evicted" only needs one every run outlasts; sharing a window made + # each assertion depend on the other's margin, and a slow machine + # broke whichever half it reached first. Configured apart, a slow + # machine makes the eviction half more certain rather than less. + + assert 'success' in add_session(cache_size=5, timeout=300) _, sess, ctx, reused = connect() assert not reused, 'new connection' @@ -131,7 +156,24 @@ def test_tls_session_timeout(): _, _, _, reused = connect(ctx, sess) assert reused, 'no timeout' - time.sleep(6) + # Three seconds, not one: the cache proof below must reconnect before + # the session expires, and a scheduling stall or an OpenSSL that counts + # this timeout in whole seconds (before 3.2) can eat a full second. + assert 'success' in add_session(cache_size=5, timeout=3) + + # The reconfiguration above rebuilds the TLS context and with it the + # session cache, so the session that has to outlive the timeout is + # established after it, not before. + _, sess, ctx, reused = connect() + assert not reused, 'new connection timeout' + + # Prove the short-timeout context cached the session at all, so the + # eviction below is attributable to the timeout and not to a session + # that was never resumable. + _, _, _, reused = connect(ctx, sess) + assert reused, 'cached before timeout' + + time.sleep(5) _, _, _, reused = connect(ctx, sess) assert not reused, 'timeout' diff --git a/test/test_variables.py b/test/test_variables.py index e20a3cd79..31cff538a 100644 --- a/test/test_variables.py +++ b/test/test_variables.py @@ -1,5 +1,7 @@ +import os import re import time +from email.utils import formatdate from pathlib import Path import pytest @@ -544,6 +546,11 @@ def test_variables_response_header(temp_dir, wait_for_record): Path(f'{temp_dir}/foo').mkdir() Path(f'{temp_dir}/foo/index.html').write_text('index', encoding='utf-8') + # A fixed mtime, so the logged $response_header_last_modified can be + # checked against a value computed here instead of against a shape. + mtime = 1600000000 # Sun, 13 Sep 2020 12:26:40 GMT + os.utime(f'{temp_dir}/foo/index.html', (mtime, mtime)) + assert 'success' in client.conf( { "listeners": {"*:8080": {"pass": "routes"}}, @@ -564,10 +571,21 @@ def test_variables_response_header(temp_dir, wait_for_record): '$response_header_connection' ) + # The exact string, not r'.*GMT': matching only the trailing "GMT" + # accepts any instant at all, which is how a Last-Modified rendered from + # localtime() and labelled GMT stayed green here for years. $date is the + # response clock, not a file's, so it stays a shape. + last_modified = re.escape(formatdate(mtime, usegmt=True)) + + # An entity-tag is a quoted string, and a text access log escapes the + # quote inside a value as \x22, the way nginx escapes every byte it + # escapes. That is what keeps a quoted field parseable when the value + # carries a quote of its own. assert client.get(url='/foo/index.html')['status'] == 200 assert ( wait_for_record( - r'share@.*GMT@".*"@text/html@Unit/.*@.*GMT@5@close', 'access.log' + rf'share@{last_modified}@\\x22.*\\x22@text/html@Unit/.*@.*GMT@5@close', + 'access.log', ) is not None ) diff --git a/test/test_wasm-wasi-component.py b/test/test_wasm-wasi-component.py index b1538287a..54e13930e 100644 --- a/test/test_wasm-wasi-component.py +++ b/test/test_wasm-wasi-component.py @@ -1,5 +1,9 @@ +import subprocess +import time + import pytest from unit.applications.lang.wasm_component import ApplicationWasmComponent +from unit.log import Log prerequisites = { 'modules': {'wasm-wasi-component': 'any'}, @@ -39,3 +43,209 @@ def conf_component(value): resp = conf_component("") assert 'must not be empty' in resp.get('detail', ''), 'empty' + + +def test_wasm_component_execution_timeout_invalid(): + # "execution_timeout" is seconds and reaches the module as a 32-bit + # millisecond value, so both ends are bounded at validation. + def conf_timeout(value): + return client.conf( + { + "app": { + "type": "wasm-wasi-component", + "processes": {"spare": 0}, + "component": "/app.wasm", + "execution_timeout": value, + } + }, + 'applications', + ) + + resp = conf_timeout(-1) + assert 'must not be negative' in resp.get('detail', ''), 'negative' + + resp = conf_timeout(2147483 + 1) + assert 'must not exceed' in resp.get('detail', ''), 'too large' + + assert 'success' in conf_timeout(2147483), 'largest' + assert 'success' in conf_timeout(0), 'unbounded' + + +def test_wasm_component_obs_text(): + client.load('hello_world') + + # Non-UTF-8 / obs-text bytes in target must be lossily replaced with U+FFFD + # rather than panicking/aborting the wasm worker. + resp = client.http( + b'GET /caf\xff\xfe HTTP/1.1\r\n' + b'Host: localhost\r\n' + b'Connection: close\r\n\r\n', + raw=True, + ) + assert resp['status'] == 200 + assert resp['body'] == 'Hello' + + # Non-UTF-8 / obs-text bytes in headers must also not crash the wasm worker. + resp = client.http( + b'GET / HTTP/1.1\r\n' + b'Host: localhost\r\n' + b'Custom: caf\xff\xfe\r\n' + b'Connection: close\r\n\r\n', + raw=True, + ) + assert resp['status'] == 200 + assert resp['body'] == 'Hello' + + +def test_wasm_component_unrepresentable_request(): + client.load('hello_world') + + # Unit forwards bytes that the Rust "http" crate will not put in a URI: + # obs-text in Host, and "<" or a control byte in the target. Building the + # request then fails, and that error must fail this request alone. The + # worker used to abort on it (SIGABRT), which conftest's Log.check_alerts() + # would also catch on teardown. + for req in ( + b'GET / HTTP/1.1\r\nHost: caf\xff\r\nConnection: close\r\n\r\n', + b'GET /a= 1, elapsed + assert elapsed < 5, elapsed + + # Assert the mechanism, not just the status: a guest that trapped for its + # own reasons would answer 500 through the same path. The epoch deadline + # renders as an "interrupt" trap. + assert Log.findall(r'wasm trap: interrupt') != [], 'stopped by the deadline' + + assert Log.findall(r'signal 6') == [], 'no abort' + assert Log.findall(r'panicked at') == [], 'no panic' + + +def test_wasm_component_execution_timeout_body(skip_fds_check): + # A request with a body still gets its full deadline. The deadline is + # armed inside the guest task rather than beside Store::new(), so the + # body read no longer comes out of the guest's budget. + # + # This pins the property, not the regression: the smallest configurable + # timeout is one second and the router hands the module a body that is + # already complete, so reading a few megabytes costs milliseconds and + # this passes under the old arming too. Catching that directly would + # need a body large enough to take about a second to read. + skip_fds_check(router=True) + + client.load('spin_loop', execution_timeout=1) + + start = time.time() + assert client.post(body='x' * (4 * 1024 * 1024))['status'] == 500, 'preempted' + elapsed = time.time() - start + + # The guest still gets its full second after the body has been read. + assert elapsed >= 1, elapsed + + assert Log.findall(r'signal 6') == [], 'no abort' + + + +def _reap_spinning(app): + """Kill a wasm worker that will not stop on its own. + + Without a deadline the guest never yields, so nothing in Unit can reclaim + the worker: the router answers the client and unlinks the rpc data, but + the process keeps running. Reap it here so it does not outlive the test + -- the same shape as test_app_start_timeout.py's _reap_stuck(). + """ + + out = subprocess.run( + ['ps', '-eo', 'pid,args'], capture_output=True, text=True, check=False + ).stdout + + for line in out.splitlines(): + if f'unit: "{app}" application' in line: + subprocess.run( + ['kill', '-9', line.split()[0]], check=False + ) + + +def test_wasm_component_execution_timeout_absent(skip_alert, skip_fds_check): + # The control for test_wasm_component_execution_timeout: the same guest + # with no deadline is never stopped. "limits": {"timeout"} bounds what + # the CLIENT waits for -- the router answers 503 and unlinks the request + # -- but it does not reach the application, so the worker spins on and + # this test reaps it. + # + # That difference is the point: 503-from-the-router is not + # 500-from-the-deadline, and "execution_timeout" is not a request timeout. + skip_alert(r'exited on signal') + skip_fds_check(router=True) + + try: + client.load('spin_loop', limits={"timeout": 3}) + + start = time.time() + assert client.get(read_timeout=30)['status'] == 503, 'router gave up' + elapsed = time.time() - start + + assert elapsed >= 3, elapsed + + assert Log.findall(r'wasm trap: interrupt') == [], 'no deadline fired' + + finally: + _reap_spinning('spin_loop') + + +def test_wasm_component_execution_timeout_unreached(): + # A deadline a normal guest never approaches stays out of the way. The + # extra tick in the tick computation is there so a request is never cut + # short of its timeout; this is the other half of that -- arming the + # deadline must not cost a request that answers immediately. + client.load('hello_world', execution_timeout=1) + + resp = client.get() + + assert resp['status'] == 200, 'answered' + assert resp['body'] == 'Hello', 'body' + + # And again on a warm worker, where the store is armed per request. + assert client.get()['status'] == 200, 'warm' diff --git a/test/test_wasm.py b/test/test_wasm.py new file mode 100644 index 000000000..783630866 --- /dev/null +++ b/test/test_wasm.py @@ -0,0 +1,117 @@ +from pathlib import Path + +import pytest + +from unit.applications.lang.wasm import ApplicationWasm +from unit.check.check_prerequisites import check_prerequisites +from unit.option import option + +prerequisites = {'modules': {'wasm': 'any'}, 'features': {'clang_wasm': True}} + +check_prerequisites(prerequisites) + +client = ApplicationWasm() + + +def test_wasm_hello(): + client.load('hello') + + resp = client.get() + + assert resp['status'] == 200 + assert resp['body'] == 'Hello from wasm\n' + + +def test_wasm_access_filesystem(): + data = Path(option.temp_dir) / 'wasm_data' + data.mkdir() + (data / 'hello.txt').write_text('from the preopen\n') + + client.load('hello', access_filesystem=[f'{data}/']) + + resp = client.get(url=f'{data}/hello.txt') + + assert resp['status'] == 200 + assert resp['body'] == 'from the preopen\n' + + +def test_wasm_access_filesystem_denied(): + data = Path(option.temp_dir) / 'wasm_data' + data.mkdir() + (data / 'inside.txt').write_text('inside the preopen\n') + outside = Path(option.temp_dir) / 'outside.txt' + outside.write_text('outside the preopen\n') + + # A symlink inside the preopen pointing outside it does not escape, with + # or without the trailing slash RUSTSEC-2026-0269 abused. + (data / 'link').symlink_to(outside) + + client.load('hello', access_filesystem=[f'{data}/']) + + # The guest answers 404 for any fopen() failure, so a broken preopen would + # pass every assertion below. Read a real file first. + resp = client.get(url=f'{data}/inside.txt') + + assert resp['status'] == 200 + assert resp['body'] == 'inside the preopen\n' + + resp = client.get(url=str(outside)) + + assert resp['status'] == 404 + + for url in (f'{data}/link', f'{data}/link/'): + resp = client.get(url=url) + + assert resp['status'] == 404, url + + +def test_wasm_malloc_handler_real_malloc(): + client.load('hello', malloc_handler='malloc') + + resp = client.get() + + assert resp['status'] == 200 + assert resp['body'] == 'Hello from wasm\n' + + +def test_wasm_malloc_at_memory_end(): + client.load('badmalloc', malloc_handler='edge_malloc') + + resp = client.get() + + assert resp['status'] == 200 + assert resp['body'] == 'accepted\n' + + +@pytest.mark.parametrize( + 'handler', ['bad_malloc_neg', 'bad_malloc_big', 'bad_malloc_end'] +) +def test_wasm_malloc_out_of_range(handler, wait_for_record): + client.load('badmalloc', malloc_handler=handler) + + assert client.get()['status'] == 503 + + assert ( + wait_for_record(r'malloc handler returned offset \d+ outside memory') + is not None + ) + + +def test_wasm_malloc_unaligned(wait_for_record): + client.load('badmalloc', malloc_handler='bad_malloc_odd') + + assert client.get()['status'] == 503 + + record = r'malloc handler returned offset \d+ that is not aligned' + assert wait_for_record(record) is not None + + +# The guest exports "__heap_base" as a global and "memory" as a memory. +@pytest.mark.parametrize('handler', ['__heap_base', 'memory']) +def test_wasm_handler_not_function(handler, wait_for_record): + client.load('hello', malloc_handler=handler) + + assert client.get()['status'] == 503 + + record = rf'module export \({handler}\) is not a function' + assert wait_for_record(record) is not None diff --git a/test/test_wasm_imports.py b/test/test_wasm_imports.py new file mode 100644 index 000000000..300a6e641 --- /dev/null +++ b/test/test_wasm_imports.py @@ -0,0 +1,107 @@ +from pathlib import Path +import shutil +import subprocess + +import pytest + +from unit.applications.lang.wasm import ApplicationWasm +from unit.check.check_prerequisites import check_prerequisites +from unit.option import option + +prerequisites = {'modules': {'wasm': 'any'}, 'features': {'clang_wasm': True}} + +check_prerequisites(prerequisites) + +client = ApplicationWasm() + +# The request buffer starts at 1 MiB. The host asks for 32 MiB + 64 KiB +# from there, so the memory needs at least 33 MiB + 64 KiB. +INITIAL_MEMORY = 36 * 1024 * 1024 + + +def build_noreq_guest(call, initialize=False): + source = Path(option.test_dir) / 'wasm' / 'noreq' / 'noreq.c' + suffix = '-initialize' if initialize else '' + output = Path(option.temp_dir) / f'noreq-{call}{suffix}.wasm' + + defines = [f'-DCALL_{call.upper()}'] + + if initialize: + defines.append('-DCALL_IN_INITIALIZE') + + subprocess.check_output( + [ + shutil.which('clang'), + '--target=wasm32-unknown-unknown', + '-O2', + '-nostdlib', + *defines, + '-Wl,--no-entry,--stack-first,-z,stack-size=65536,' + f'--initial-memory={INITIAL_MEMORY}', + str(source), + '-o', + str(output), + ], + stderr=subprocess.STDOUT, + ) + + return output + + +def load_noreq(module, **handlers): + app = { + 'type': 'wasm', + 'processes': {'spare': 0}, + 'module': str(module), + 'request_handler': 'request_handler', + 'malloc_handler': 'malloc_handler', + 'free_handler': 'free_handler', + } + app.update(handlers) + + assert 'success' in client.conf( + { + 'listeners': {'*:8080': {'pass': 'applications/noreq'}}, + 'applications': {'noreq': app}, + } + ) + + +@pytest.mark.parametrize( + 'call', ['send_response', 'send_headers', 'response_end'] +) +def test_wasm_import_in_initialize(call, wait_for_record): + load_noreq(build_noreq_guest(call, initialize=True)) + + assert client.get()['status'] == 503 + + record = rf'nxt_wasm_{call}\(\) called outside a request' + assert wait_for_record(record) is not None + + +def test_wasm_import_in_malloc_handler(wait_for_record): + load_noreq( + build_noreq_guest('send_headers'), malloc_handler='malloc_import' + ) + + assert client.get()['status'] == 503 + + record = r'nxt_wasm_send_headers\(\) called outside a request' + assert wait_for_record(record) is not None + + +def test_wasm_import_in_module_init(wait_for_record): + load_noreq( + build_noreq_guest('response_end'), module_init_handler='init_import' + ) + + resp = client.get() + + assert resp['status'] == 200 + assert resp['body'] == 'ok\n' + + record = r'failed to call hook function \[init_import\]' + assert wait_for_record(record) is not None + + record = r'nxt_wasm_response_end\(\) called outside a request' + assert wait_for_record(record) is not None diff --git a/test/test_wasm_memory64.py b/test/test_wasm_memory64.py new file mode 100644 index 000000000..b3e72a631 --- /dev/null +++ b/test/test_wasm_memory64.py @@ -0,0 +1,60 @@ +from pathlib import Path +import shutil +import subprocess + +from unit.applications.lang.wasm import ApplicationWasm +from unit.check.check_prerequisites import check_prerequisites +from unit.option import option + +prerequisites = {'modules': {'wasm': 'any'}, 'features': {'clang_wasm': True}} + +check_prerequisites(prerequisites) + +client = ApplicationWasm() + +# 1 MiB below the request buffer, then the 32 MiB + 64 KiB that the host +# asks for, rounded up to 36 MiB. +INITIAL_MEMORY = 36 * 1024 * 1024 + + +def build_memory64_guest(): + source = Path(option.test_dir) / 'wasm' / 'memory64' / 'memory64.c' + output = Path(option.temp_dir) / 'memory64.wasm' + + subprocess.check_output( + [ + shutil.which('clang'), + '--target=wasm64-unknown-unknown', + '-O2', + '-nostdlib', + f'-Wl,--no-entry,--initial-memory={INITIAL_MEMORY}', + str(source), + '-o', + str(output), + ], + stderr=subprocess.STDOUT, + ) + + return output + + +def test_wasm_memory64_refused(wait_for_record): + assert 'success' in client.conf( + { + 'listeners': {'*:8080': {'pass': 'applications/memory64'}}, + 'applications': { + 'memory64': { + 'type': 'wasm', + 'processes': {'spare': 0}, + 'module': str(build_memory64_guest()), + 'request_handler': 'request_handler', + 'malloc_handler': 'malloc_handler', + 'free_handler': 'free_handler', + } + }, + } + ) + + assert client.get()['status'] == 503 + + assert wait_for_record(r'module memory is 64-bit') is not None diff --git a/test/unit/applications/lang/java.py b/test/unit/applications/lang/java.py index bd510fd47..5fcabcc76 100644 --- a/test/unit/applications/lang/java.py +++ b/test/unit/applications/lang/java.py @@ -53,7 +53,7 @@ def prepare_env(self, script): os.makedirs(classes_path) classpath = ( - f'{option.current_dir}/build/tomcat-servlet-api-9.0.110.jar' + f'{option.current_dir}/build/tomcat-servlet-api-9.0.122.jar' ) ws_jars = glob.glob( @@ -63,8 +63,19 @@ def prepare_env(self, script): if not ws_jars: pytest.fail('websocket api jar not found.') + # The suite runs under sudo, whose secure_path drops the + # toolcache directory that actions/setup-java prepends, so a bare + # "javac" resolves to the system JDK rather than the one this + # matrix leg installed and built the jars with. + javac_bin = 'javac' + java_home = os.environ.get('JAVA_HOME') + if java_home: + javac_home = os.path.join(java_home, 'bin', 'javac') + if os.path.isfile(javac_home): + javac_bin = javac_home + javac = [ - 'javac', + javac_bin, '-target', '8', '-source', diff --git a/test/unit/applications/lang/php.py b/test/unit/applications/lang/php.py index ac59ec1b4..568260eb3 100644 --- a/test/unit/applications/lang/php.py +++ b/test/unit/applications/lang/php.py @@ -1,10 +1,26 @@ +import os from pathlib import Path import shutil +import pytest + from unit.applications.proto import ApplicationProto from unit.option import option +def _absolute_symlinks(path, names): + # A shutil.copytree() filter. Relative symlinks stay inside the copy, + # but an absolute one -- /usr/share/zoneinfo/localtime -> /etc/localtime + # on Debian -- points back at the host, and the conftest teardown chmods + # every file it walks in the temp dir, following it out of the rootfs. + return [ + name + for name in names + if os.path.islink(f'{path}/{name}') + and os.path.isabs(os.readlink(f'{path}/{name}')) + ] + + class ApplicationPHP(ApplicationProto): def __init__(self, application_type='php'): self.application_type = application_type @@ -20,6 +36,27 @@ def load(self, script, index='index.php', **kwargs): if not Path(f'{rootfs}/app/php/{script}').exists(): shutil.copytree(script_path, f'{rootfs}/app/php/{script}') + # PHP built with --with-system-tzdata -- every Debian, Ubuntu and + # Fedora package -- opens the timezone database lazily, at request + # time, which is after Unit has chroot()ed the app worker into + # "rootfs". Such builds segfault instead of raising an error when + # the database is not there, so provision it and keep these tests + # about Unit rather than about PHP packaging. + tzdir = '/usr/share/zoneinfo' + + if not Path(tzdir).is_dir(): + pytest.skip( + f'no timezone database at {tzdir} to copy into the rootfs' + ) + + if not Path(f'{rootfs}{tzdir}').exists(): + shutil.copytree( + tzdir, + f'{rootfs}{tzdir}', + symlinks=True, + ignore=_absolute_symlinks, + ) + script_path = f'/app/php/{script}' app = { diff --git a/test/unit/applications/lang/wasm.py b/test/unit/applications/lang/wasm.py new file mode 100644 index 000000000..b05d0264c --- /dev/null +++ b/test/unit/applications/lang/wasm.py @@ -0,0 +1,108 @@ +from pathlib import Path +import shutil +import subprocess +from urllib.parse import quote + +from unit.applications.proto import ApplicationProto +from unit.option import option + + +class ApplicationWasm(ApplicationProto): + @staticmethod + def prepare_env(script): + clang = shutil.which('clang') + root = Path(option.current_dir) + sysroot = root / 'pkg/contrib/wasi-sysroot' + include = root / 'pkg/contrib/libunit-wasm/src/c/include' + library = root / 'pkg/contrib/libunit-wasm/src/c/libunit-wasm.c' + source = Path(option.test_dir) / 'wasm' / script / f'{script}.c' + + if ( + clang is None + or not sysroot.is_dir() + or not include.is_dir() + or not library.is_file() + or not source.is_file() + ): + return None + + temp_dir = Path(option.temp_dir) / 'wasm' + temp_dir.mkdir(parents=True, exist_ok=True) + + output = temp_dir / f'{script}.wasm' + + # One -Wl, argument: clang splits it on commas and hands each piece to + # the linker, so "-z stack-size=..." stays a linker option. + link_flags = '-Wl,' + ','.join( + [ + '--no-entry', + '--export=__heap_base', + '--export=__data_end', + '--export=malloc', + '--export=free', + '--stack-first', + '-z', + 'stack-size=8388608', + ] + ) + + command = [ + clang, + f'-I{include}', + f'--sysroot={sysroot}', + '--target=wasm32-wasi', + '-O2', + '-g', + '-std=gnu11', + '-fno-common', + link_flags, + '-mexec-model=reactor', + '--rtlib=compiler-rt', + str(source), + str(library), + '-o', + str(output), + ] + + try: + subprocess.check_output(command, stderr=subprocess.STDOUT) + except (subprocess.CalledProcessError, FileNotFoundError): + return None + + return output + + def load( + self, + script, + access_filesystem=None, + malloc_handler='luw_malloc_handler', + **kwargs, + ): + module = self.prepare_env(script) + + app = { + 'type': 'wasm', + 'processes': {'spare': 0}, + 'module': str(module), + 'request_handler': 'luw_request_handler', + 'malloc_handler': malloc_handler, + 'free_handler': 'luw_free_handler', + # Without these two the guest's init hook never runs, so + # request_buf stays NULL and luw_set_req_buf() memcpy()s to + # address 0 of the linear memory instead of the allocation. + 'module_init_handler': 'luw_module_init_handler', + 'module_end_handler': 'luw_module_end_handler', + } + + if access_filesystem is not None: + app['access'] = {'filesystem': access_filesystem} + + self._load_conf( + { + 'listeners': { + '*:8080': {'pass': f'applications/{quote(script, "")}'} + }, + 'applications': {script: app}, + }, + **kwargs, + ) diff --git a/test/unit/applications/lang/wasm_component.py b/test/unit/applications/lang/wasm_component.py index a6c8dd140..f0eb5b1ad 100644 --- a/test/unit/applications/lang/wasm_component.py +++ b/test/unit/applications/lang/wasm_component.py @@ -41,23 +41,29 @@ def prepare_env(script): return output - def load(self, script, **kwargs): + def load(self, script, execution_timeout=None, limits=None, **kwargs): self.prepare_env(script) component_path = f'{option.temp_dir}/wasm_component/{script}/target/wasm32-wasip1/release/test_wasi_component.wasm' + app = { + "type": "wasm-wasi-component", + "processes": {"spare": 0}, + "component": component_path, + } + + if execution_timeout is not None: + app["execution_timeout"] = execution_timeout + + if limits is not None: + app["limits"] = limits + self._load_conf( { "listeners": { "*:8080": {"pass": f"applications/{quote(script, '')}"} }, - "applications": { - script: { - "type": "wasm-wasi-component", - "processes": {"spare": 0}, - "component": component_path, - } - }, + "applications": {script: app}, }, **kwargs, ) diff --git a/test/unit/applications/websockets.py b/test/unit/applications/websockets.py index a8e563d0f..4922bbfb8 100644 --- a/test/unit/applications/websockets.py +++ b/test/unit/applications/websockets.py @@ -14,6 +14,10 @@ class ApplicationWebsocket(ApplicationProto): + # Seconds to wait for the echo of a large message (the 9_x tests). + # A debug build needs 3.6-4.4 s to echo 65536 fragments of 64 bytes. + LARGE_MESSAGE_TIMEOUT = 30 + OP_CONT = 0x00 OP_TEXT = 0x01 OP_BINARY = 0x02 @@ -30,7 +34,7 @@ def accept(self, key): sha1 = hashlib.sha1((key + GUID).encode()).digest() return base64.b64encode(sha1).decode() - def upgrade(self, headers=None): + def upgrade(self, headers=None, sock=None): key = None if headers is None: @@ -44,10 +48,9 @@ def upgrade(self, headers=None): 'Sec-WebSocket-Version': 13, } - sock = self.get( - headers=headers, - no_recv=True, - ) + kwargs = {} if sock is None else {'sock': sock} + + sock = self.get(headers=headers, no_recv=True, **kwargs) resp = '' while True: @@ -140,6 +143,27 @@ def recv_bytes(sock, bytes_len): return frame + def pongs_read(self, sock, pings, read_timeout=60): + # RFC 6455 Section 5.5.3: while a PONG is not sent, the server can + # answer only the most recent PING. Read PONGs until the one for the + # last PING. They must answer PINGs in the order they were sent. + + pings = [p.encode() if isinstance(p, str) else p for p in pings] + answered = [] + + while not answered or answered[-1] != len(pings) - 1: + frame = self.frame_read(sock, read_timeout) + + assert frame['opcode'] == self.OP_PONG, 'pong opcode' + assert frame['data'] in pings, 'pong payload' + + i = pings.index(frame['data']) + assert not answered or i > answered[-1], 'pong order' + + answered.append(i) + + return answered + def frame_to_send( self, opcode, diff --git a/test/unit/check/clang_wasm.py b/test/unit/check/clang_wasm.py new file mode 100644 index 000000000..fb8db6749 --- /dev/null +++ b/test/unit/check/clang_wasm.py @@ -0,0 +1,5 @@ +from unit.applications.lang.wasm import ApplicationWasm + + +def check_clang_wasm(): + return ApplicationWasm.prepare_env('hello') is not None diff --git a/test/unit/check/discover_available.py b/test/unit/check/discover_available.py index 99e636047..c78dde1b1 100644 --- a/test/unit/check/discover_available.py +++ b/test/unit/check/discover_available.py @@ -3,6 +3,7 @@ from unit.check.cargo_component import check_cargo_component from unit.check.chroot import check_chroot +from unit.check.clang_wasm import check_clang_wasm from unit.check.go import check_go from unit.check.isolation import check_isolation from unit.check.njs import check_njs @@ -47,5 +48,6 @@ def discover_available(unit): option.available['features']['cargo_component'] = check_cargo_component() option.available['features']['chroot'] = check_chroot() + option.available['features']['clang_wasm'] = check_clang_wasm() option.available['features']['isolation'] = check_isolation() option.available['features']['unix_abstract'] = check_unix_abstract() diff --git a/test/unit/http.py b/test/unit/http.py index e449c8b54..446e3ff68 100644 --- a/test/unit/http.py +++ b/test/unit/http.py @@ -11,6 +11,34 @@ from unit.option import option +def request_headers(headers=None, connection_close=True): + """Return request headers with "Connection: close" merged in. + + HTTP/1.1 keeps a connection open by default and recvall() reads a + response to EOF, so a one-shot request that omits "Connection: close" + waits out the server's idle timeout. A "Connection" the caller set, in + any case, is left alone; connection_close=False drops the merge for a + request that means to keep the connection -- a handshake, a pipeline, a + socket that is reused. + + Only a caller that passes no dict at all gets "Host: localhost"; an + explicit dict is given "Connection: close" and nothing else, and a + zero-header request is not expressible through this helper. + """ + if headers is None: + headers = {'Host': 'localhost'} + + else: + headers = dict(headers) + + if connection_close and not any( + name.lower() == 'connection' for name in headers + ): + headers['Connection'] = 'close' + + return headers + + class HTTP1: def http(self, start_str, **kwargs): sock_type = kwargs.get('sock_type', 'ipv4') @@ -18,8 +46,8 @@ def http(self, start_str, **kwargs): url = kwargs.get('url', '/') http = 'HTTP/1.0' if 'http_10' in kwargs else 'HTTP/1.1' - headers = kwargs.get( - 'headers', {'Host': 'localhost', 'Connection': 'close'} + headers = request_headers( + kwargs.get('headers'), kwargs.get('connection_close', True) ) body = kwargs.get('body', b'') @@ -100,6 +128,9 @@ def http(self, start_str, **kwargs): if 'read_buffer_size' in kwargs: recvall_kwargs['buff_size'] = kwargs['read_buffer_size'] + if kwargs.get('framed'): + recvall_kwargs['framed'] = True + resp = self.recvall(sock, **recvall_kwargs).decode( encoding, errors='ignore' ) @@ -131,10 +162,14 @@ def log_out(self, log, encoding): if option.detailed: print('>>>') log = self.log_truncate(log) + + if isinstance(log, bytes): + log = log.decode(encoding, 'ignore') + try: - print(log.decode(encoding, 'ignore')) - except UnicodeEncodeError: print(log) + except UnicodeEncodeError: + print(log.encode()) def log_in(self, log): if option.detailed: @@ -148,13 +183,15 @@ def log_in(self, log): def log_truncate(self, log, limit=1024): len_log = len(log) if len_log > limit: - log = log[:limit] appendix = f'(...logged {limit} of {len_log} bytes)' + # Concatenate, do not format: an f-string turns bytes into its + # repr, so the result was a str holding "b'...'" and the caller + # that decodes it got an AttributeError instead of a log. if isinstance(log, bytes): - appendix = appendix.encode() - - log = f'{log}{appendix}' + log = log[:limit] + appendix.encode() + else: + log = log[:limit] + appendix return log @@ -178,6 +215,7 @@ def recvall(self, sock, **kwargs): timeout = kwargs.get('read_timeout', timeout_default) buff_size = kwargs.get('buff_size', 4096) + framed = kwargs.get('framed', False) data = b'' while True: @@ -203,8 +241,71 @@ def recvall(self, sock, **kwargs): if not part: break + if framed and self._response_complete(data): + break + return data + @staticmethod + def _response_complete(data): + """Return True when data holds one complete response by its framing. + + recvall(framed=True) uses it on a keep-alive socket. The server + does not close that socket, so without this the read ends only on + read_timeout. The function knows Content-Length and chunked. For + a response with neither, or one that does not parse, it returns + False and the read ends on the timeout, as before. Do not use it + for HEAD, 1xx, 204 or 304 responses: they have no body, whatever + their headers say. + """ + end = data.find(b'\r\n\r\n') + if end < 0: + return False + + body = data[end + 4 :] + length = None + chunked = False + + for line in data[:end].split(b'\r\n')[1:]: + name, _, value = line.partition(b':') + name = name.strip().lower() + value = value.strip() + + if name == b'content-length': + try: + length = int(value) + except ValueError: + return False + + elif name == b'transfer-encoding': + chunked = value.lower().endswith(b'chunked') + + if chunked: + while True: + eol = body.find(b'\r\n') + if eol < 0: + return False + + try: + size = int(body[:eol].split(b';')[0], 16) + except ValueError: + return False + + if size == 0: + # Unit sends no trailers: the last chunk is "0" CRLF CRLF. + return body[eol + 2 : eol + 4] == b'\r\n' + + need = eol + 2 + size + 2 + if len(body) < need: + return False + + body = body[need:] + + if length is not None: + return len(body) >= length + + return False + def _resp_to_dict(self, resp): m = re.search(r'(.*?\x0d\x0a?)\x0d\x0a?(.*)', resp, re.M | re.S) diff --git a/test/unit/log.py b/test/unit/log.py index 7d7e355aa..c5812f0f9 100644 --- a/test/unit/log.py +++ b/test/unit/log.py @@ -44,10 +44,9 @@ def check_alerts(log=None): assert not alerts, 'alert(s)' - if not option.skip_sanitizer: - sanitizer_errors = re.findall('.+Sanitizer.+', log) + sanitizer_errors = re.findall('.+Sanitizer.+', log) - assert not sanitizer_errors, 'sanitizer error(s)' + assert not sanitizer_errors, 'sanitizer error(s)' if found and option.detailed: print('skipped.') diff --git a/test/unit/option.py b/test/unit/option.py index 7c66c6193..ab9ff65eb 100644 --- a/test/unit/option.py +++ b/test/unit/option.py @@ -9,8 +9,8 @@ class Options: 'configure_flag': {}, 'is_privileged': os.geteuid() == 0, 'skip_alerts': [], - 'skip_sanitizer': False, 'system': platform.system(), + 'unsafe': False, } def __setattr__(self, name, value): diff --git a/test/unit/shm_body.py b/test/unit/shm_body.py new file mode 100644 index 000000000..bd71c6207 --- /dev/null +++ b/test/unit/shm_body.py @@ -0,0 +1,124 @@ +"""Self-identifying request bodies for the shared-memory teardown tests. + +Why not ``'x' * SIZE``: test_process_teardown_churn.py and +test_process_abrupt_teardown.py exist to notice the router serving freed or +recycled shared memory, and the most likely shape of that is a 16 KiB chunk +(PORT_MMAP_CHUNK_SIZE, src/nxt_port_memory_int.h) belonging to a *different* +concurrent request turning up in this one. Against a uniform body such a +chunk is byte-identical to the bytes that belong there, so the integrity +check cannot see it at all: it only ever catches garbage that happens not to +be 'x'. Both tests run several request threads on purpose, so a cross-request +chunk is exactly the corruption they are best placed to catch and exactly the +one a uniform body hides. + +The body is a run of fixed-width records, each stamped with the request's own +tag and with its own byte offset: + + | tag: 8 hex digits | offset: 8 hex digits | 48 filler letters | + +so every 64 bytes of the payload say both *which* request the bytes came from +and *where in it* they belong. A chunk swapped in from another request +mismatches on the tag; a chunk from another offset of the same request +mismatches on the offset field; a shift of less than one record mismatches on +the filler letter, which advances with the record index. Byte-wise, that +means any splice at a chunk boundary is wrong within at most 64 bytes of the +splice, and the record header at the first wrong byte names the culprit. + +Cost matters - these tests send 64 to 100 bodies of 1 MiB each and are meant +to stay at a few seconds - so the offset skeleton is built once per size at +import time and a per-request body is a single str.replace() of the tag +placeholder: one C-speed pass over 1 MiB instead of a Python loop. +""" + +import itertools + +# One record per 64 bytes: short enough that a corrupt region is localised to +# within 64 bytes of where it starts, long enough that a 1 MiB body is 16384 +# records rather than ten times that. +RECORD_SIZE = 64 + +TAG_LEN = 8 +OFFSET_LEN = 8 + +_FILLER_LEN = RECORD_SIZE - TAG_LEN - OFFSET_LEN + +_LETTERS = 'abcdefghijklmnopqrstuvwxyz' + +# Tags are hex digits, so '~' cannot occur in a finished body and replacing +# the placeholder cannot corrupt payload that happens to look like it. +_PLACEHOLDER = '~' * TAG_LEN + +_counter = itertools.count(1) + +_skeletons = {} + + +def _skeleton(size): + if size % RECORD_SIZE: + raise ValueError(f'size must be a multiple of {RECORD_SIZE}') + + if size not in _skeletons: + _skeletons[size] = ''.join( + _PLACEHOLDER + + f'{offset:0{OFFSET_LEN}X}' + + _LETTERS[(offset // RECORD_SIZE) % len(_LETTERS)] * _FILLER_LEN + for offset in range(0, size, RECORD_SIZE) + ) + + return _skeletons[size] + + +def make_body(size): + """A body of `size` bytes, distinguishable from every other body this + process has handed out. + + next() on an itertools.count is a single bytecode under the GIL, so + request threads can call this without a lock, and consecutive tags always + differ in at least one hex digit - which is all the byte-wise comparison + in the tests needs to tell two in-flight requests apart. + """ + + return _skeleton(size).replace(_PLACEHOLDER, f'{next(_counter):08X}') + + +def describe_mismatch(body, expected): + """One line saying how `body` differs from `expected`, for a failure + message. + + Reports the first wrong byte and the record header found there against + the header that belongs there, which is what separates "a chunk from + another request" (tag differs) from "a chunk from another offset of this + request" (offset differs) from plain garbage (neither parses). + """ + + # next() needs a default. zip() stops at the shorter of the two, so a + # body that is a correct prefix but LONGER than the request - which still + # fails the caller's `body != expected[:len(body)]` test, since that + # slice clamps to `expected` - has no mismatching byte to find here. A + # bare next() would raise StopIteration out of the caller's request loop + # and kill the thread instead of recording the corruption: the detector + # would fail open on one of the shapes it exists to catch. + at = next( + (i for i, (c, e) in enumerate(zip(body, expected)) if c != e), None + ) + + if at is None: + return f'correct prefix but longer than {len(expected)} bytes' + + return ( + f'first mismatch at {at} ({body[at]!r} != {expected[at]!r}): ' + f'got {_record_header(body, at)}, ' + f'want {_record_header(expected, at)}' + ) + + +def _record_header(text, at): + """The tag/offset stamp of the record containing byte `at`.""" + + start = at - at % RECORD_SIZE + head = text[start : start + TAG_LEN + OFFSET_LEN] + + if len(head) < TAG_LEN + OFFSET_LEN: + return f'' + + return f'tag={head[:TAG_LEN]} offset={head[TAG_LEN:]}' diff --git a/test/wasm/badmalloc/badmalloc.c b/test/wasm/badmalloc/badmalloc.c new file mode 100644 index 000000000..56a721dd3 --- /dev/null +++ b/test/wasm/badmalloc/badmalloc.c @@ -0,0 +1,80 @@ +/* SPDX-License-Identifier: Apache-2.0 */ +/* Copyright (C) FreeUnit contributors. */ + +/* + * A guest whose malloc handlers return offsets that the host must check. + * The host refuses a bad offset when the worker starts, so the request + * handler runs only for "edge_malloc". + */ + +#include "unit/unit-wasm.h" + +#define WASM_PAGE_SIZE 65536u +#define RESPONSE_OFFSET 4096 + +/* + * Adds at least "bytes" of new pages to the linear memory and returns its + * new size. The handlers below return offsets in the new pages, so the + * host never writes over the data or the stack of the guest. + */ +static u32 +grow_by(u32 bytes) +{ + __builtin_wasm_memory_grow(0, (bytes + WASM_PAGE_SIZE - 1) + / WASM_PAGE_SIZE); + + return (u32) __builtin_wasm_memory_size(0) * WASM_PAGE_SIZE; +} + +__luw_export_name("bad_malloc_neg") +u32 bad_malloc_neg(size_t size) +{ + (void) size; + + return 0x80000000u; +} + +__luw_export_name("bad_malloc_big") +u32 bad_malloc_big(size_t size) +{ + (void) size; + + return 0x7fff0000u; +} + +/* The last offset that leaves "size" bytes. The host accepts it. */ +__luw_export_name("edge_malloc") +u32 edge_malloc(size_t size) +{ + return grow_by(size) - size; +} + +/* + * The memory is larger than "size", but the offset leaves 8 bytes too few. + * The offset is aligned, so only the offset check refuses it. + */ +__luw_export_name("bad_malloc_end") +u32 bad_malloc_end(size_t size) +{ + return grow_by(size) - size + 8; +} + +/* The offset leaves enough bytes, but it is odd. */ +__luw_export_name("bad_malloc_odd") +u32 bad_malloc_odd(size_t size) +{ + return grow_by(size + WASM_PAGE_SIZE) - size - WASM_PAGE_SIZE + 1; +} + +__luw_export_name("luw_request_handler") +int luw_request_handler(u8 *addr) +{ + luw_ctx_t ctx; + + luw_init_ctx(&ctx, addr, RESPONSE_OFFSET); + luw_mem_writep(&ctx, "accepted\n"); + luw_http_send_response(&ctx); + luw_http_response_end(); + + return 0; +} diff --git a/test/wasm/hello/hello.c b/test/wasm/hello/hello.c new file mode 100644 index 000000000..e30080ec0 --- /dev/null +++ b/test/wasm/hello/hello.c @@ -0,0 +1,128 @@ +/* SPDX-License-Identifier: Apache-2.0 */ + +/* + * A small libunit-wasm guest for the legacy wasm runtime's smoke tests. + * + * "/" answers with a fixed body. Any other path names a file the guest + * tries to open; the host preopens every "access.filesystem" entry with the + * same path in the guest and hard-coded read+write permissions, so a path + * inside a preopen reads and anything else fails. The relative-offset + * constants below come from the upstream luw-echo-request example. + */ + +#include +#include +#include + +#include "unit/unit-wasm.h" + +#define RESPONSE_OFFSET 4096 + +static u8 *request_buf; + +__luw_export_name("luw_module_init_handler") +void luw_module_init_handler(void) +{ + request_buf = malloc(luw_mem_get_init_size()); +} + +__luw_export_name("luw_module_end_handler") +void luw_module_end_handler(void) +{ + free(request_buf); +} + +static void +send_simple(luw_ctx_t *ctx, luw_http_status_t status, const char *body) +{ + char clen[32]; + + snprintf(clen, sizeof(clen), "%zu", strlen(body)); + + luw_http_set_response_status(status); + luw_http_init_headers(ctx, 2, 0); + luw_http_add_header(ctx, "Content-Type", "text/plain"); + luw_http_add_header(ctx, "Content-Length", clen); + luw_http_send_headers(ctx); + luw_mem_writep_data(ctx, (const u8 *)body, strlen(body)); + luw_http_send_response(ctx); + luw_http_response_end(); +} + +static void +send_file(luw_ctx_t *ctx, const char *path) +{ + char clen[32]; + u8 *data; + long size; + FILE *file; + + file = fopen(path, "rb"); + if (file == NULL) { + send_simple(ctx, LUW_HTTP_NOT_FOUND, "not found\n"); + return; + } + + if (fseek(file, 0, SEEK_END) != 0) { + fclose(file); + send_simple(ctx, LUW_HTTP_INTERNAL_SERVER_ERROR, "seek failed\n"); + return; + } + + size = ftell(file); + rewind(file); + + if (size < 0) { + fclose(file); + send_simple(ctx, LUW_HTTP_INTERNAL_SERVER_ERROR, "tell failed\n"); + return; + } + + data = malloc(size); + if (data == NULL) { + fclose(file); + send_simple(ctx, LUW_HTTP_INTERNAL_SERVER_ERROR, "out of memory\n"); + return; + } + + if (fread(data, 1, size, file) != (size_t) size) { + free(data); + fclose(file); + send_simple(ctx, LUW_HTTP_INTERNAL_SERVER_ERROR, "read failed\n"); + return; + } + + fclose(file); + + snprintf(clen, sizeof(clen), "%ld", size); + + luw_http_init_headers(ctx, 2, 0); + luw_http_add_header(ctx, "Content-Type", "application/octet-stream"); + luw_http_add_header(ctx, "Content-Length", clen); + luw_http_send_headers(ctx); + luw_mem_writep_data(ctx, data, size); + luw_http_send_response(ctx); + luw_http_response_end(); + + free(data); +} + +__luw_export_name("luw_request_handler") +int luw_request_handler(u8 *addr) +{ + luw_ctx_t ctx; + const char *path; + + luw_init_ctx(&ctx, addr, RESPONSE_OFFSET); + luw_set_req_buf(&ctx, &request_buf, LUW_SRB_NONE); + + path = luw_get_http_path(&ctx); + + if (strcmp(path, "/") == 0) { + send_simple(&ctx, LUW_HTTP_OK, "Hello from wasm\n"); + } else { + send_file(&ctx, path); + } + + return 0; +} diff --git a/test/wasm/memory64/memory64.c b/test/wasm/memory64/memory64.c new file mode 100644 index 000000000..e2b10702a --- /dev/null +++ b/test/wasm/memory64/memory64.c @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: Apache-2.0 */ +/* Copyright (C) FreeUnit contributors. */ + +/* + * A guest with a 64-bit linear memory, built without libc or libunit-wasm. + * The host must refuse it when the worker starts. The handlers work, so a + * host that accepts the module answers 200 with "ok". + */ + +typedef unsigned int u32; + +#define IMPORT(name) __attribute__((import_module("env"), import_name(name))) +#define EXPORT(name) __attribute__((export_name(name))) + +/* The request buffer starts above the data and the stack of the guest. */ +#define BUFFER_OFFSET (1024 * 1024) +#define RESPONSE_OFFSET 4096 + +IMPORT("nxt_wasm_send_response") void send_response(u32 offset); +IMPORT("nxt_wasm_response_end") void response_end(void); + +EXPORT("malloc_handler") u32 +malloc_handler(u32 size) +{ + (void) size; + + return BUFFER_OFFSET; +} + +EXPORT("free_handler") void +free_handler(u32 addr) +{ + (void) addr; +} + +EXPORT("request_handler") int +request_handler(u32 addr) +{ + unsigned char *resp; + + resp = (unsigned char *) (unsigned long) addr + RESPONSE_OFFSET; + + /* The layout of nxt_wasm_response_t: a u32 size, then the data. */ + *(u32 *) resp = 3; + resp[4] = 'o'; + resp[5] = 'k'; + resp[6] = '\n'; + + send_response(RESPONSE_OFFSET); + response_end(); + + return 0; +} diff --git a/test/wasm/noreq/noreq.c b/test/wasm/noreq/noreq.c new file mode 100644 index 000000000..6ddd4452f --- /dev/null +++ b/test/wasm/noreq/noreq.c @@ -0,0 +1,97 @@ +/* SPDX-License-Identifier: Apache-2.0 */ +/* Copyright (C) FreeUnit contributors. */ + +/* + * A guest that calls a host import before the host serves a request. It is + * built without libc or libunit-wasm. -DCALL_SEND_RESPONSE, + * -DCALL_SEND_HEADERS or -DCALL_RESPONSE_END selects the import. With + * -DCALL_IN_INITIALIZE, the guest exports "_initialize", and wasmtime calls + * it when it instantiates the module. "malloc_import" and "init_import" + * call the import from the malloc handler and from the module_init hook. + * The other handlers work, so a host that serves a request answers 200 with + * "ok". + */ + +typedef unsigned int u32; + +#define IMPORT(name) __attribute__((import_module("env"), import_name(name))) +#define EXPORT(name) __attribute__((export_name(name))) + +/* The request buffer starts above the data and the stack of the guest. */ +#define BUFFER_OFFSET (1024 * 1024) +#define RESPONSE_OFFSET 4096 + +IMPORT("nxt_wasm_send_response") void send_response(u32 offset); +IMPORT("nxt_wasm_send_headers") void send_headers(u32 offset); +IMPORT("nxt_wasm_response_end") void response_end(void); + +static void +call_import(void) +{ +#if defined(CALL_SEND_RESPONSE) + send_response(RESPONSE_OFFSET); +#elif defined(CALL_SEND_HEADERS) + send_headers(RESPONSE_OFFSET); +#elif defined(CALL_RESPONSE_END) + response_end(); +#else +#error "define CALL_SEND_RESPONSE, CALL_SEND_HEADERS or CALL_RESPONSE_END" +#endif +} + +#if defined(CALL_IN_INITIALIZE) +EXPORT("_initialize") void +initialize(void) +{ + call_import(); +} +#endif + +EXPORT("malloc_import") u32 +malloc_import(u32 size) +{ + (void) size; + + call_import(); + + return BUFFER_OFFSET; +} + +EXPORT("init_import") void +init_import(void) +{ + call_import(); +} + +EXPORT("malloc_handler") u32 +malloc_handler(u32 size) +{ + (void) size; + + return BUFFER_OFFSET; +} + +EXPORT("free_handler") void +free_handler(u32 addr) +{ + (void) addr; +} + +EXPORT("request_handler") int +request_handler(u32 addr) +{ + unsigned char *resp; + + resp = (unsigned char *) addr + RESPONSE_OFFSET; + + /* The layout of nxt_wasm_response_t: a u32 size, then the data. */ + *(u32 *) resp = 3; + resp[4] = 'o'; + resp[5] = 'k'; + resp[6] = '\n'; + + send_response(RESPONSE_OFFSET); + response_end(); + + return 0; +} diff --git a/test/wasm_component/hello_world/Cargo.lock b/test/wasm_component/hello_world/Cargo.lock index 2daeb73de..3bf1dd23b 100644 --- a/test/wasm_component/hello_world/Cargo.lock +++ b/test/wasm_component/hello_world/Cargo.lock @@ -1,12 +1,12 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "bitflags" -version = "2.5.0" +version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" [[package]] name = "test-wasi-component" diff --git a/test/wasm_component/hello_world/Cargo.toml b/test/wasm_component/hello_world/Cargo.toml index a87fbeb5a..46760c8bf 100644 --- a/test/wasm_component/hello_world/Cargo.toml +++ b/test/wasm_component/hello_world/Cargo.toml @@ -2,6 +2,7 @@ name = "test-wasi-component" version = "0.1.0" edition = "2021" +rust-version = "1.78" [dependencies] bitflags = "2.4.2" diff --git a/test/wasm_component/spin_loop/Cargo.lock b/test/wasm_component/spin_loop/Cargo.lock new file mode 100644 index 000000000..3bf1dd23b --- /dev/null +++ b/test/wasm_component/spin_loop/Cargo.lock @@ -0,0 +1,34 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "test-wasi-component" +version = "0.1.0" +dependencies = [ + "bitflags", + "wasi", + "wit-bindgen-rt", +] + +[[package]] +name = "wasi" +version = "0.13.0+wasi-0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "652cd73449d0b957a2743b70c72d79d34a5fa505696488f4ca90b46f6da94118" +dependencies = [ + "bitflags", + "wit-bindgen-rt", +] + +[[package]] +name = "wit-bindgen-rt" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "026d24a27f6712541fa534f2954bd9e0eb66172f033c2157c0f31d106255c497" diff --git a/test/wasm_component/spin_loop/Cargo.toml b/test/wasm_component/spin_loop/Cargo.toml new file mode 100644 index 000000000..46760c8bf --- /dev/null +++ b/test/wasm_component/spin_loop/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "test-wasi-component" +version = "0.1.0" +edition = "2021" +rust-version = "1.78" + +[dependencies] +bitflags = "2.4.2" +wit-bindgen-rt = "0.21.0" +wasi = "0.13.0" + +[lib] +crate-type = ["cdylib"] + +[package.metadata.component] +package = "component:test-wasi-component" +proxy = true + +[package.metadata.component.dependencies] diff --git a/test/wasm_component/spin_loop/src/lib.rs b/test/wasm_component/spin_loop/src/lib.rs new file mode 100644 index 000000000..66210332b --- /dev/null +++ b/test/wasm_component/spin_loop/src/lib.rs @@ -0,0 +1,22 @@ +use wasi::http::types::{IncomingRequest, ResponseOutparam}; + +wasi::http::proxy::export!(Component); + +struct Component; + +// A guest that never yields and never answers. wasi-http is linked async, +// but a component only yields at an async wasi import, so this loop keeps +// the host inside a single call_handle() poll for as long as it runs. It +// exists to prove the host can preempt such a guest; without a deadline the +// request never completes and the worker thread stays pinned. +// +// black_box() keeps the loop from being optimised away. +impl wasi::exports::http::incoming_handler::Guest for Component { + fn handle(_request: IncomingRequest, _response_out: ResponseOutparam) { + let mut n: u64 = 0; + + loop { + n = core::hint::black_box(n).wrapping_add(1); + } + } +} diff --git a/test/wasm_component/spin_loop/wit/world.wit b/test/wasm_component/spin_loop/wit/world.wit new file mode 100644 index 000000000..82c810ef1 --- /dev/null +++ b/test/wasm_component/spin_loop/wit/world.wit @@ -0,0 +1,6 @@ +package component:test-wasi-component; + +/// An example world for the component to target. +world example { + export hello-world: func() -> string; +} diff --git a/tools/unitctl/CHANGELOG.md b/tools/unitctl/CHANGELOG.md index 7bf491ed6..60cbee592 100644 --- a/tools/unitctl/CHANGELOG.md +++ b/tools/unitctl/CHANGELOG.md @@ -7,6 +7,45 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.37.0] - 2026-10-04 + +### Removed +- JSON5 input. A `.json5` configuration file is refused by name, as hjson and YAML already + were, and the message names a converter. `edit` reads its temporary file as JSON, so a + comment written in the editor is now a syntax error. + +### Fixed +- `execute -m PUT -p /js_modules/.js -f .js` sends the module + (`Content-Type: application/javascript`, body verbatim) instead of panicking with + "Unknown input file type". A file whose type cannot be placed is refused with an error in + both `execute` and `import` rather than aborting. +- A JSON configuration file is sent as it was written. It was parsed into a map and + re-serialized, which silently merged duplicate member names and re-spelled numbers; the + server now sees the file and reports a duplicate member with its line and column. +- `save` no longer tells the operator to restore a configuration that holds non-UTF-8 bytes + with curl; the control API refuses such a configuration whichever client sends it, so the + message now names the value that has to be corrected. + +### Security +- rustls 0.23.42 → 0.23.45 (RUSTSEC-2026-0285, GHSA-2mjx-qc3c-rqvc), with aws-lc-rs 1.18.1, + aws-lc-sys 0.45.0 and rustls-webpki 0.103.15. + +### Changed +- Version bump to track the FreeUnit 1.37.0 release. + +## [1.36.1] - 2026-08-28 + +### Fixed +- Control API error responses that carry a `location` object (byte offset, line and column for + parse errors; the JSON Pointer `path` for validation errors) no longer fail to deserialise; + `unitctl` now prints the server's `detail`, `suggestion` and location instead of a generic JSON + decoding error. + +### Changed +- Version bump to track the FreeUnit 1.36.1 release. +- The `/status` counters in the OpenAPI spec are declared `format: int64`, so the generated + client models them as 64-bit instead of `i32`. + ## [1.36.0] - 2026-07-16 ### Added @@ -55,6 +94,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Previous releases See earlier versions for additional change history. -[Unreleased]: https://github.com/freeunitorg/freeunit/compare/unitctl/1.36.0...HEAD +[Unreleased]: https://github.com/freeunitorg/freeunit/compare/unitctl/1.37.0...HEAD +[1.37.0]: https://github.com/freeunitorg/freeunit/releases/tag/unitctl/1.37.0 +[1.36.1]: https://github.com/freeunitorg/freeunit/releases/tag/unitctl/1.36.1 [1.36.0]: https://github.com/freeunitorg/freeunit/releases/tag/unitctl/1.36.0 [1.35.5]: https://github.com/freeunitorg/freeunit/releases/tag/unitctl/1.35.5 diff --git a/tools/unitctl/Cargo.lock b/tools/unitctl/Cargo.lock index 354c2d17d..118041a3a 100644 --- a/tools/unitctl/Cargo.lock +++ b/tools/unitctl/Cargo.lock @@ -61,29 +61,17 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "arrayvec" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" - [[package]] name = "atomic-waker" version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" -[[package]] -name = "autocfg" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" - [[package]] name = "aws-lc-rs" -version = "1.17.1" +version = "1.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4342d8937fc7e5dd9b1c60292261c0670c882a2cd1719cfc11b1af41731e32ad" +checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e" dependencies = [ "aws-lc-sys", "zeroize", @@ -91,9 +79,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.42.0" +version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d9ceb1da931507a12f4fccea479dccd00da1943e1b4ae72d8e502d707361444" +checksum = "9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27" dependencies = [ "cc", "cmake", @@ -108,32 +96,17 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" -[[package]] -name = "bit-set" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" -dependencies = [ - "bit-vec", -] - -[[package]] -name = "bit-vec" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" - [[package]] name = "bitflags" -version = "2.13.1" +version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" [[package]] name = "bollard" -version = "0.21.0" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9d0a013e3d3ee4edd61e779adf117944c08902d375f18630a0c5b8f95659734" +checksum = "dbe8358268799ebb3e4df23cb9d47f4c72bbc4f5247e2fa6a1bf7b6c0baea220" dependencies = [ "base64", "bollard-stubs", @@ -172,56 +145,17 @@ dependencies = [ "serde_repr", ] -[[package]] -name = "buf-trait" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21eaafc770e8c073d6c3facafe7617e774305d4954aa6351b9c452eb37ee17b4" -dependencies = [ - "zerocopy 0.7.35", -] - -[[package]] -name = "bumpalo" -version = "3.20.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" - -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - [[package]] name = "bytes" version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" -[[package]] -name = "byteyarn" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b93e51d26468a15ea59f8525e0c13dc405db43e644a0b1e6d44346c72cf4cf7b" -dependencies = [ - "buf-trait", -] - -[[package]] -name = "castaway" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" -dependencies = [ - "rustversion", -] - [[package]] name = "cc" -version = "1.2.67" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +checksum = "f74872d07caf508b30a21f6836e7d7016a2eaf7d9ff4f48deaa58cd8a0407630" dependencies = [ "find-msvc-tools", "jobserver", @@ -235,17 +169,11 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" -[[package]] -name = "cfg_aliases" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" - [[package]] name = "clap" -version = "4.6.2" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd059f9da4f5c36b3787f65d38ccaab1cc315f07b01f89abc8359ee6a8205011" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" dependencies = [ "clap_builder", "clap_derive", @@ -253,9 +181,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.2" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" dependencies = [ "anstream", "anstyle", @@ -265,14 +193,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.1" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 3.0.6", ] [[package]] @@ -338,43 +266,12 @@ version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" -[[package]] -name = "crossterm_winapi" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" -dependencies = [ - "winapi", -] - [[package]] name = "custom_error" version = "1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f8a51dd197fa6ba5b4dc98a990a43cc13693c23eb0089ebb0fcc1f04152bca6" -[[package]] -name = "derive_more" -version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" -dependencies = [ - "derive_more-impl", -] - -[[package]] -name = "derive_more-impl" -version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" -dependencies = [ - "proc-macro2", - "quote", - "rustc_version", - "syn", - "unicode-xid", -] - [[package]] name = "dispatch2" version = "0.3.1" @@ -393,7 +290,7 @@ checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -418,17 +315,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "fancy-regex" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1e1dacd0d2082dfcf1351c4bdd566bbe89a2b263235a2b50058f1e130a47277" -dependencies = [ - "bit-set", - "regex-automata", - "regex-syntax", -] - [[package]] name = "fastrand" version = "2.4.1" @@ -447,15 +333,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" - -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" [[package]] name = "form_urlencoded" @@ -474,9 +354,9 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "futures" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" dependencies = [ "futures-channel", "futures-core", @@ -489,9 +369,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" dependencies = [ "futures-core", "futures-sink", @@ -499,15 +379,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" [[package]] name = "futures-executor" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" dependencies = [ "futures-core", "futures-task", @@ -516,38 +396,38 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" [[package]] name = "futures-macro" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.6", ] [[package]] name = "futures-sink" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" [[package]] name = "futures-task" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" [[package]] name = "futures-util" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ "futures-channel", "futures-core", @@ -594,25 +474,6 @@ dependencies = [ "r-efi 6.0.0", ] -[[package]] -name = "h2" -version = "0.4.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" -dependencies = [ - "atomic-waker", - "bytes", - "fnv", - "futures-core", - "futures-sink", - "http", - "indexmap", - "slab", - "tokio", - "tokio-util", - "tracing", -] - [[package]] name = "hashbrown" version = "0.17.1" @@ -633,9 +494,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "http" -version = "1.4.2" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" dependencies = [ "bytes", "itoa", @@ -653,9 +514,9 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" dependencies = [ "bytes", "futures-core", @@ -678,15 +539,14 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hyper" -version = "1.10.1" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" dependencies = [ "atomic-waker", "bytes", "futures-channel", "futures-core", - "h2", "http", "http-body", "httparse", @@ -715,9 +575,9 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.9" +version = "0.27.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +checksum = "dfa8e654703247911e29c23fbeaa261834bd9bb74efba2f9acddc37bfb127f53" dependencies = [ "http", "hyper", @@ -733,15 +593,16 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.20" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" dependencies = [ "bytes", "futures-channel", "futures-util", "http", "http-body", + "httparse", "hyper", "libc", "pin-project-lite", @@ -901,54 +762,12 @@ dependencies = [ "libc", ] -[[package]] -name = "js-sys" -version = "0.3.103" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" -dependencies = [ - "cfg-if", - "futures-util", - "wasm-bindgen", -] - -[[package]] -name = "json5" -version = "1.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "733a844dbd6fef128e98cb4487b887cb55454d92cd9994b1bafe004fabbe670c" -dependencies = [ - "serde", - "ucd-trie", -] - -[[package]] -name = "lean_string" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dff9d7a64297827f9ef2cc8a111ca1d87bead398b7f9fe76ba0eb9f50d175e8d" -dependencies = [ - "castaway", - "itoa", - "ryu", - "serde_core", -] - [[package]] name = "libc" version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" -[[package]] -name = "linked-hash-map" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" -dependencies = [ - "serde", -] - [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -961,31 +780,12 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" -[[package]] -name = "lock_api" -version = "0.4.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" -dependencies = [ - "scopeguard", -] - [[package]] name = "log" version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" -[[package]] -name = "lscolors" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d60e266dfb1426eb2d24792602e041131fdc0236bb7007abc0e589acafd60929" -dependencies = [ - "aho-corasick", - "nu-ansi-term", -] - [[package]] name = "memchr" version = "2.8.3" @@ -1003,18 +803,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "nix" -version = "0.31.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" -dependencies = [ - "bitflags", - "cfg-if", - "cfg_aliases", - "libc", -] - [[package]] name = "ntapi" version = "0.4.3" @@ -1024,72 +812,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "nu-ansi-term" -version = "0.50.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" -dependencies = [ - "windows-sys 0.61.2", -] - -[[package]] -name = "nu-json" -version = "0.113.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d664096864c9fc825f7da6185bcc3b288869aedfe8d70580d41fa96f90661d71" -dependencies = [ - "linked-hash-map", - "nu-utils", - "num-traits", - "serde", - "serde_json", -] - -[[package]] -name = "nu-utils" -version = "0.113.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec685be6fcc963735a023948a38e7e39e03a32fedc09649c30019ce8f85fbfc8" -dependencies = [ - "byteyarn", - "crossterm_winapi", - "derive_more", - "fancy-regex", - "lean_string", - "log", - "lscolors", - "memchr", - "nix", - "num-format", - "parking_lot", - "serde", - "serde_json", - "strip-ansi-escapes", - "sys-locale", - "unicase", - "web-time", -] - -[[package]] -name = "num-format" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a652d9771a63711fd3c3deb670acfbe5c30a4072e664d7a3bf5a9e1056ac72c3" -dependencies = [ - "arrayvec", - "itoa", -] - -[[package]] -name = "num-traits" -version = "0.2.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" -dependencies = [ - "autocfg", -] - [[package]] name = "objc2" version = "0.6.4" @@ -1165,29 +887,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" -[[package]] -name = "parking_lot" -version = "0.12.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec", - "windows-link", -] - [[package]] name = "pbr" version = "1.1.1" @@ -1232,7 +931,7 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" dependencies = [ - "zerocopy 0.8.54", + "zerocopy", ] [[package]] @@ -1294,15 +993,6 @@ dependencies = [ "getrandom 0.3.4", ] -[[package]] -name = "redox_syscall" -version = "0.5.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" -dependencies = [ - "bitflags", -] - [[package]] name = "regex" version = "1.13.1" @@ -1346,15 +1036,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "rustc_version" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" -dependencies = [ - "semver", -] - [[package]] name = "rustix" version = "1.1.4" @@ -1370,9 +1051,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.42" +version = "0.23.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138" +checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" dependencies = [ "aws-lc-rs", "log", @@ -1395,29 +1076,20 @@ dependencies = [ "security-framework", ] -[[package]] -name = "rustls-pemfile" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" -dependencies = [ - "rustls-pki-types", -] - [[package]] name = "rustls-pki-types" -version = "1.15.0" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" dependencies = [ "zeroize", ] [[package]] name = "rustls-webpki" -version = "0.103.13" +version = "0.103.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" dependencies = [ "aws-lc-rs", "ring", @@ -1425,12 +1097,6 @@ dependencies = [ "untrusted", ] -[[package]] -name = "rustversion" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" - [[package]] name = "ryu" version = "1.0.23" @@ -1455,12 +1121,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "scopeguard" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" - [[package]] name = "security-framework" version = "3.7.0" @@ -1484,17 +1144,11 @@ dependencies = [ "libc", ] -[[package]] -name = "semver" -version = "1.0.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" - [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", "serde_derive", @@ -1502,29 +1156,29 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.6", ] [[package]] name = "serde_json" -version = "1.0.150" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ "indexmap", "itoa", @@ -1542,7 +1196,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -1557,19 +1211,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_yaml" -version = "0.9.34+deprecated" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" -dependencies = [ - "indexmap", - "itoa", - "ryu", - "serde", - "unsafe-libyaml", -] - [[package]] name = "shlex" version = "2.0.1" @@ -1604,15 +1245,6 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" -[[package]] -name = "strip-ansi-escapes" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a8f8038e7e7969abb3f1b7c2a811225e9296da208539e0f79c5251d6cac0025" -dependencies = [ - "vte", -] - [[package]] name = "strsim" version = "0.11.1" @@ -1637,23 +1269,25 @@ dependencies = [ ] [[package]] -name = "synstructure" -version = "0.13.2" +name = "syn" +version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" dependencies = [ "proc-macro2", "quote", - "syn", + "unicode-ident", ] [[package]] -name = "sys-locale" -version = "0.3.2" +name = "synstructure" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8eab9a99a024a169fe8a903cf9d4a3b3601109bcc13bd9e3c6fff259138626c4" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ - "libc", + "proc-macro2", + "quote", + "syn 2.0.119", ] [[package]] @@ -1712,7 +1346,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -1727,9 +1361,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.52.4" +version = "1.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "317fafbbe3f02fc663dad00ea6186197de963cd4190e86a26d8d0fae095539af" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" dependencies = [ "bytes", "libc", @@ -1748,7 +1382,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -1805,33 +1439,15 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" -[[package]] -name = "ucd-trie" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" - -[[package]] -name = "unicase" -version = "2.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" - [[package]] name = "unicode-ident" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" -[[package]] -name = "unicode-xid" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" - [[package]] name = "unit-client-rs" -version = "1.36.0" +version = "1.37.0" dependencies = [ "bollard", "bytes", @@ -1857,7 +1473,7 @@ dependencies = [ [[package]] name = "unit-openapi" -version = "1.36.0" +version = "1.37.0" dependencies = [ "base64", "bytes", @@ -1873,7 +1489,7 @@ dependencies = [ [[package]] name = "unitctl" -version = "1.36.0" +version = "1.37.0" dependencies = [ "bytes", "clap", @@ -1882,12 +1498,9 @@ dependencies = [ "futures", "http-body-util", "hyper", - "json5", - "nu-json", - "rustls-pemfile", + "rustls-pki-types", "serde", "serde_json", - "serde_yaml", "tar", "tempfile", "tokio", @@ -1896,12 +1509,6 @@ dependencies = [ "which", ] -[[package]] -name = "unsafe-libyaml" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" - [[package]] name = "untrusted" version = "0.9.0" @@ -1932,15 +1539,6 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" -[[package]] -name = "vte" -version = "0.14.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "231fdcd7ef3037e8330d8e17e61011a2c244126acc0a982f4040ac3f9f0bc077" -dependencies = [ - "memchr", -] - [[package]] name = "walkdir" version = "2.5.0" @@ -1975,61 +1573,6 @@ dependencies = [ "wit-bindgen", ] -[[package]] -name = "wasm-bindgen" -version = "0.2.126" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" -dependencies = [ - "cfg-if", - "once_cell", - "rustversion", - "wasm-bindgen-macro", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.126" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.126" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" -dependencies = [ - "bumpalo", - "proc-macro2", - "quote", - "syn", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.126" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "web-time" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - [[package]] name = "webpki-roots" version = "1.0.8" @@ -2041,9 +1584,9 @@ dependencies = [ [[package]] name = "which" -version = "8.0.5" +version = "8.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f3ef584124b911bcc3875c2f1472e80f24361ceb789bd1c62b3e9a3df9ff43c" +checksum = "bae2f2b2b816647a1cab1acc91f5bd20812d53cb344382635ec2181940c8034f" dependencies = [ "libc", ] @@ -2132,7 +1675,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -2143,7 +1686,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -2318,38 +1861,17 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", "synstructure", ] -[[package]] -name = "zerocopy" -version = "0.7.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" -dependencies = [ - "byteorder", - "zerocopy-derive 0.7.35", -] - [[package]] name = "zerocopy" version = "0.8.54" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7cbbc0a705a0fd05cc3676525980d2bf5a9bc4adac6d6475209a7887cf59d19" dependencies = [ - "zerocopy-derive 0.8.54", -] - -[[package]] -name = "zerocopy-derive" -version = "0.7.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" -dependencies = [ - "proc-macro2", - "quote", - "syn", + "zerocopy-derive", ] [[package]] @@ -2360,7 +1882,7 @@ checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -2380,7 +1902,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", "synstructure", ] @@ -2420,7 +1942,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] diff --git a/tools/unitctl/GNUmakefile b/tools/unitctl/GNUmakefile index 975f50dfe..dd1f2fe87 100644 --- a/tools/unitctl/GNUmakefile +++ b/tools/unitctl/GNUmakefile @@ -44,6 +44,19 @@ endif RELEASE_BUILD_FLAGS ?= --quiet --release --bin $(OUTPUT_BINARY) +# The musl artifact is built without the "tls" feature. TLS pulls in +# aws-lc-sys, a C library built by cmake, and that is the only thing making +# this build need a C toolchain. +target_feature_flags = $(if $(findstring musl,$(1)),--no-default-features,) + +# The rule above keys on a target name, so it only reaches the cross builds that +# name one. A native build on a musl host -- "make release" on Alpine, which is +# how a distro packages this -- passes no --target and would otherwise keep TLS +# and compile aws-lc-sys. Ask rustc for the host triple and apply the same +# rule. Empty if rustc is absent, which leaves the flags unchanged. +HOST_TARGET := $(shell rustc -vV 2>/dev/null | sed -n 's/^host: //p') +HOST_FEATURE_FLAGS := $(call target_feature_flags,$(HOST_TARGET)) + Q = $(if $(filter 1,$V),,@) M = $(shell printf "\033[34;1m▶\033[0m") @@ -68,8 +81,8 @@ all: $(TARGETS) ## Build all available platform targets [see: list-targets] .ONESHELL: $(TARGETS) $(TARGETS): openapi-generate $Q if [ ! -f "$(CURDIR)/target/$(@)/release/$(OUTPUT_BINARY)" ]; then - echo "$(M) building $(OUTPUT_BINARY) with flags [$(RELEASE_BUILD_FLAGS) --target $(@)]" - $(CARGO) build $(RELEASE_BUILD_FLAGS) --target $@ + echo "$(M) building $(OUTPUT_BINARY) with flags [$(RELEASE_BUILD_FLAGS) $(call target_feature_flags,$@) --target $(@)]" + $(CARGO) build $(RELEASE_BUILD_FLAGS) $(call target_feature_flags,$@) --target $@ fi target target/debug: @@ -80,18 +93,18 @@ debug: target/debug/$(OUTPUT_BINARY) target/debug/$(OUTPUT_BINARY): openapi-generate $Q echo "$(M) building $(OUTPUT_BINARY) in debug mode for the current platform" - $Q $(CARGO) build --bin $(OUTPUT_BINARY) + $Q $(CARGO) build --bin $(OUTPUT_BINARY) $(HOST_FEATURE_FLAGS) .PHONY: release release: target/release/$(OUTPUT_BINARY) target/release/$(OUTPUT_BINARY): openapi-generate $Q echo "$(M) building $(OUTPUT_BINARY) in release mode for the current platform" - $Q $(CARGO) build $(RELEASE_BUILD_FLAGS) + $Q $(CARGO) build $(RELEASE_BUILD_FLAGS) $(HOST_FEATURE_FLAGS) .PHONY: test test: openapi-generate ## Run tests - $Q $(CARGO) test + $Q $(CARGO) test $(HOST_FEATURE_FLAGS) .ONESHELL: target/man/$(OUTPUT_BINARY).1.gz target/man/$(OUTPUT_BINARY).1.gz: diff --git a/tools/unitctl/README.md b/tools/unitctl/README.md index bfb40bf70..c0c6379fe 100644 --- a/tools/unitctl/README.md +++ b/tools/unitctl/README.md @@ -80,8 +80,7 @@ Options: Print version ``` -- Consumes alternative configuration formats Like YAML and converts them -- Can convert output to multiple different formats (YAML, plain JSON, highlighted JSON) +- Can convert output to multiple different formats (plain JSON, highlighted JSON) - Syntactic highlighting of JSON output - Interpretation of Unit errors with (arguably more) useful error messages @@ -203,16 +202,9 @@ Unitctl can query the control API to provide the status of the running Unit daemon. ``` -$ unitctl status -t yaml +$ unitctl status -t json No socket path provided - attempting to detect from running instance -connections: - accepted: 0 - active: 0 - idle: 0 - closed: 0 -requests: - total: 0 -applications: {} +{"connections":{"accepted":0,"active":0,"idle":0,"closed":0},"requests":{"total":0},"applications":{}} ``` *Note:* This command supports operating on multiple instances of Unit at once. diff --git a/tools/unitctl/openapi-config.json b/tools/unitctl/openapi-config.json index b8b428147..723adbb67 100644 --- a/tools/unitctl/openapi-config.json +++ b/tools/unitctl/openapi-config.json @@ -1,6 +1,6 @@ { "packageName": "unit-openapi", - "packageVersion": "1.36.0", + "packageVersion": "1.37.0", "library": "hyper", "preferUnsignedInt": true } diff --git a/tools/unitctl/unit-client-rs/Cargo.toml b/tools/unitctl/unit-client-rs/Cargo.toml index 28c879633..b4e0ae788 100644 --- a/tools/unitctl/unit-client-rs/Cargo.toml +++ b/tools/unitctl/unit-client-rs/Cargo.toml @@ -1,27 +1,37 @@ [package] name = "unit-client-rs" -version = "1.36.0" +version = "1.37.0" authors = ["Elijah Zupancic"] edition = "2021" +rust-version = "1.95" license = "Apache-2.0" [lib] name = "unit_client_rs" [features] -# this preserves the ordering of json -default = ["serde_json/preserve_order"] +default = ["tls"] +# "https://" control sockets. The control API has no TLS listener of its own, +# so this is for a TLS-terminating proxy in front of a remote control API. +# Turning it off drops aws-lc-sys, and with it the C toolchain the build needs; +# that is what the musl artifact does. hyper-rustls and rustls must be gated +# together: the direct rustls dependency is the only thing that turns on +# rustls's own default features, and losing one of those is silent. +tls = ["dep:hyper-rustls", "dep:rustls"] [dependencies] custom_error = "1.9" -hyper = { version = "1", features = ["http1", "http2"] } -hyper-rustls = { version = "0.27", features = ["rustls-native-certs", "webpki-roots", "http2"] } -hyper-util = { version = "0.1", features = ["client-legacy", "client", "http1", "http2", "tokio"] } +hyper = { version = "1", features = ["http1"] } +hyper-rustls = { version = "0.27", features = ["rustls-native-certs", "webpki-roots"], optional = true } +hyper-util = { version = "0.1", features = ["client-legacy", "client", "http1", "tokio"] } hyperlocal = "0.9" http-body-util = "0.1" bytes = "1" serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0" +# preserve_order is required, not optional. json_body compares two decodes of +# the same body member by member, in order, and a sorted map would pair them +# wrongly. It also keeps the server's member order in what unitctl prints. +serde_json = { version = "1.0", features = ["preserve_order"] } sysinfo = "0.39" tokio = { version = "1.34", features = ["macros"] } futures = "0.3" @@ -29,7 +39,10 @@ hex = "0.4" which = "8.0" unit-openapi = { path = "../unit-openapi" } -rustls = "0.23.5" +# Not used in code. It is here so that rustls gets its own default features, +# which hyper-rustls does not turn on for us: "prefer-post-quantum" comes from +# this line alone. +rustls = { version = "0.23.5", optional = true } bollard = "0.21" regex = "1.10.4" pbr = "1.1.1" diff --git a/tools/unitctl/unit-client-rs/src/control_socket_address.rs b/tools/unitctl/unit-client-rs/src/control_socket_address.rs index a1077a4e1..9fda6ac09 100644 --- a/tools/unitctl/unit-client-rs/src/control_socket_address.rs +++ b/tools/unitctl/unit-client-rs/src/control_socket_address.rs @@ -106,7 +106,7 @@ impl TryFrom for ControlSocket { let path = String::from_utf8_lossy(&bytes); ControlSocket::parse_address(path) } - Some("http") | Some("https") => Ok(TcpSocket(socket_uri)), + Some("http") | Some("https") => ControlSocket::tcp_socket(socket_uri), Some(unknown) => Err(UnitClientError::TcpSocketAddressParseError { message: format!("Unsupported scheme found in socket address: {}", unknown).to_string(), control_socket_address: socket_uri.to_string(), @@ -120,6 +120,26 @@ impl TryFrom for ControlSocket { } impl ControlSocket { + /// Builds a TCP control socket from an already parsed URI. + #[cfg(feature = "tls")] + fn tcp_socket(uri: Uri) -> Result { + Ok(TcpSocket(uri)) + } + + /// Without the "tls" feature there is no HTTPS connector, so refuse the + /// address here. A plain HTTP connector would otherwise be handed an + /// "https://" URI and fail later with an unrelated message. + #[cfg(not(feature = "tls"))] + fn tcp_socket(uri: Uri) -> Result { + if uri.scheme_str() == Some("https") { + return Err(UnitClientError::TcpSocketAddressParseError { + message: "https needs the \"tls\" feature, which this build does not have".to_string(), + control_socket_address: uri.to_string(), + }); + } + Ok(TcpSocket(uri)) + } + pub fn socket_scheme(&self) -> ControlSocketScheme { match self { UnixLocalAbstractSocket(_) => ControlSocketScheme::HTTP, @@ -302,7 +322,7 @@ impl ControlSocket { } let uri = Self::normalize_and_parse_http_address(buf)?; - Ok(TcpSocket(uri)) + Self::tcp_socket(uri) } pub fn is_local_socket(&self) -> bool { @@ -458,6 +478,12 @@ mod tests { "[0000:0000:0000:0000:0000:0000:0000:0000]:8080", ]; for socket_address in valid_socket_addresses { + // A build without the "tls" feature refuses https; see + // refuses_https_without_the_tls_feature below. + if cfg!(not(feature = "tls")) && socket_address.starts_with("https://") { + continue; + } + let mut expected = if socket_address.starts_with("http") { socket_address.to_string().trim_end_matches('/').to_string() } else { @@ -506,6 +532,24 @@ mod tests { assert_eq!(normalized, expected); } + #[cfg(feature = "tls")] + #[test] + fn accepts_https_with_the_tls_feature() { + let control_socket = ControlSocket::try_from("https://localhost:8443").expect("https should be accepted"); + assert_eq!(control_socket.to_string(), "https://localhost:8443/"); + } + + #[cfg(not(feature = "tls"))] + #[test] + fn refuses_https_without_the_tls_feature() { + let error = ControlSocket::try_from("https://localhost:8443").expect_err("https should be refused"); + assert_eq!( + error.to_string(), + "Invalid TCP socket address [control_socket_address=https://localhost:8443/]: \ + https needs the \"tls\" feature, which this build does not have" + ); + } + #[test] fn can_parse_http_addresses() { let valid_socket_addresses = vec![ diff --git a/tools/unitctl/unit-client-rs/src/json_body.rs b/tools/unitctl/unit-client-rs/src/json_body.rs new file mode 100644 index 000000000..4ff2eb9a0 --- /dev/null +++ b/tools/unitctl/unit-client-rs/src/json_body.rs @@ -0,0 +1,1358 @@ +//! Decoding control-API response bodies that are not guaranteed to be UTF-8. +//! +//! `nxt_conf_json_parse_string()` keeps every byte from 0x20 upwards that it +//! finds inside a JSON string, so a configuration stored by any released +//! version -- 1.34 LTS and every 1.36.x -- can hold bytes that are not valid +//! UTF-8, and `GET /config` hands them back verbatim. `serde_json` rejects +//! such a body whole, which would leave unitctl unable to look at the one +//! configuration it is most needed for. +//! +//! Everything here is a fallback. A body that is valid UTF-8 takes the path +//! it always took, byte for byte; only a body that cannot be decoded strictly +//! is decoded again with the undecodable bytes replaced, and that never +//! happens silently. + +use serde::de::{DeserializeOwned, MapAccess, SeqAccess, Visitor}; +use serde::{Deserialize, Deserializer}; +use serde_json::Value; +use std::fmt; + +/// The character `String::from_utf8_lossy` leaves behind. +const REPLACEMENT: char = '\u{fffd}'; + +/// How many members a warning names before it stops listing them. +const MAX_LISTED: usize = 8; + +/// How much of the server's answer survived decoding. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub enum Fidelity { + /// The body was valid UTF-8: the value is exactly what the server sent. + #[default] + Exact, + /// The body was not valid UTF-8. Every byte that could not be decoded was + /// replaced with U+FFFD, so the value is readable but is *not* what the + /// server stores. `members` holds the RFC 6901 pointers of the members + /// this altered, and `names` the ones among them whose *name* was altered + /// rather than its value -- a pointer in `names` cannot be PUT to, because + /// the path itself holds U+FFFD where the server holds a byte. + Replaced { members: Vec, names: Vec }, +} + +impl Fidelity { + pub fn is_exact(&self) -> bool { + matches!(self, Fidelity::Exact) + } + + /// The affected members, as a comma-separated list of RFC 6901 pointers. + pub fn members(&self) -> String { + let members = match self { + Fidelity::Exact => return String::new(), + Fidelity::Replaced { members, .. } => members, + }; + + if members.is_empty() { + // The bytes did not survive as a string value, so there is nothing + // to point at. Say that rather than printing an empty list. + return "an unlocatable part of the document".to_string(); + } + + list(members) + } + + /// The affected members whose *name* was altered, as a comma-separated list + /// of RFC 6901 pointers, or empty when none were. + /// + /// These are the ones no PUT can repair: the pointer holds U+FFFD where the + /// server holds a byte, so the path names a member that does not exist. + pub fn replaced_names(&self) -> String { + match self { + Fidelity::Exact => String::new(), + Fidelity::Replaced { names, .. } => list(names), + } + } + + /// The affected members whose *value* was altered, as a comma-separated list + /// of RFC 6901 pointers, or empty when none were. + /// + /// These are the ones a PUT to the path shown does repair. + pub fn replaced_values(&self) -> String { + let (members, names) = match self { + Fidelity::Exact => return String::new(), + Fidelity::Replaced { members, names } => (members, names), + }; + + let values: Vec = members + .iter() + .filter(|member| !names.contains(member)) + .cloned() + .collect(); + + list(&values) + } + + /// The warning to print when a body had to be decoded lossily, or `None` + /// when it did not. + pub fn warning(&self, path: &str) -> Option { + if self.is_exact() { + return None; + } + + Some(format!( + "Warning: {} returned bytes that are not valid UTF-8.\n\ + Warning: they are shown as U+FFFD and are NOT what the server stores: {}", + path, + self.members() + )) + } +} + +/// How much of a response body that would not parse is shown. +const MAX_SHOWN_BODY: usize = 2048; + +/// Pointers as one line for a person to read: capped, and with the control +/// characters a name can hold written out. +fn list(members: &[String]) -> String { + let listed = members + .iter() + .take(MAX_LISTED) + .map(|member| { + if member.is_empty() { + "the whole document".to_string() + } else { + one_line(member) + } + }) + .collect::>() + .join(", "); + + match members.len().checked_sub(MAX_LISTED) { + Some(rest) if rest > 0 => format!("{} and {} more", listed, rest), + _ => listed, + } +} + +/// How a control character is written when text from the server is printed. +/// +/// Deliberately not U+FFFD. That character is what [`Fidelity::warning`] tells +/// the operator is a byte the server stores and UTF-8 cannot carry, and a +/// control character in a name is not one: it is valid UTF-8 that arrived +/// intact, so showing it as U+FFFD would send the operator hunting a byte that +/// does not exist -- and `unitctl_error.rs`'s advice on such a member would be +/// wrong. The escape JSON itself would use keeps the two apart and still says +/// what was really there, which a `?` would throw away. +fn escaped(character: char) -> String { + format!("\\u{:04x}", character as u32) +} + +/// One line of text from the server, made safe to print. +/// +/// A member name can hold any character JSON can spell, and `serde_json` decodes +/// the escape for ESC into a real ESC: a name can carry an ANSI escape sequence +/// that the terminal would act on, or a newline or carriage return, which would +/// let it forge a line of unitctl's own output. Only a raw control *byte* is +/// refused by the parser, and a name reaches stderr through every warning about +/// replaced bytes, so every control character is written out by [`escaped`]. +/// +/// Every one of them, newline and tab included, because a pointer is one line. +/// [`body_for_display`] keeps those two, because a page is many lines. +pub(crate) fn one_line(text: &str) -> String { + let mut shown = String::with_capacity(text.len()); + + for character in text.chars() { + if character.is_control() { + shown.push_str(&escaped(character)); + } else { + shown.push(character); + } + } + + shown +} + +/// Render a response body that would not parse, for a terminal. +/// +/// Whatever answered the request wrote this: an intermediary's page, a truncated +/// answer, a captive portal's sign-in form. It is neither bounded -- an error +/// page can be megabytes -- nor trusted, and the control characters in it are +/// the ones a terminal acts on, so an escape sequence in a 502 page would +/// otherwise reach the user's screen. +/// +/// Newline and tab are kept, because a page is many lines. Every other control +/// character is written out by [`escaped`], and the U+FFFD that can appear here +/// is the real thing: this decodes lossily, so those are bytes that could not be +/// decoded. +pub fn body_for_display(bytes: &[u8]) -> String { + // Rendering never shortens: a character of n bytes comes back as n bytes, or + // as the 3 of U+FFFD, or as the 6 of an escape. So this many input bytes + // always suffice to fill the budget, and stopping there is what keeps a + // multi-megabyte page from being copied whole to show 2 KiB of it. The few + // extra bytes cover a character the cut lands inside. + let head = &bytes[..bytes.len().min(MAX_SHOWN_BODY + 4)]; + let mut shown = String::new(); + + for character in String::from_utf8_lossy(head).chars() { + let keep = match character { + '\n' | '\t' => character.to_string(), + control if control.is_control() => escaped(control), + other => other.to_string(), + }; + + if shown.len() + keep.len() > MAX_SHOWN_BODY { + // Said of the output, because that is what was measured: a page of + // control or undecodable bytes renders to several bytes per byte, so + // this is not a count of the body. + shown.push_str(&format!("\n[... truncated after {} bytes of output]", MAX_SHOWN_BODY)); + return shown; + } + + shown.push_str(&keep); + } + + shown +} + +/// Deserialize a control-API response body, tolerating bytes that are not +/// valid UTF-8. +/// +/// The strict decode is tried first and is what almost every body takes, so a +/// server that answers in UTF-8 is unaffected. Only when the bytes themselves +/// are not UTF-8 is the body decoded again with the undecodable bytes +/// replaced; a body that is valid UTF-8 but malformed JSON, or that does not +/// match `T`, still fails with the error it always failed with. +/// +/// A replacing decode that would lose a member fails instead. See +/// [`Unambiguous`]: the point of the fallback is to show the operator what is +/// there, so showing less than what is there without saying so is the one +/// outcome it must not have. +pub fn decode(bytes: &[u8]) -> Result<(T, Fidelity), serde_json::Error> { + match serde_json::from_slice::(bytes) { + Ok(value) => Ok((value, Fidelity::Exact)), + Err(strict) => { + if std::str::from_utf8(bytes).is_ok() { + return Err(strict); + } + + let text = String::from_utf8_lossy(bytes); + let Unambiguous(value) = serde_json::from_str(&text)?; + + // A second decode, so that a U+FFFD the operator stored is not + // reported as one this decode wrote. + let marked = serde_json::from_str::(&marked_decode(bytes)).ok(); + let altered = replaced_members(&value, marked.as_ref()); + let typed = serde_json::from_value(value)?; + + Ok(( + typed, + Fidelity::Replaced { + members: altered.all, + names: altered.names, + }, + )) + } + } +} + +/// What `decode` says when it will not show a document it knows is incomplete. +const AMBIGUOUS_NAME: &str = "two members of one object share a name, so one of them would be \ + dropped; the usual reason is names that differ only in bytes that are \ + not valid UTF-8, which decode alike. unitctl will not show a document \ + it knows is incomplete: read these bytes with 'unitctl export' or curl"; + +/// A JSON document whose member names are known to be distinct. +/// +/// `String::from_utf8_lossy` maps every undecodable byte to the same character, +/// so `"app-\xFF"` and `"app-\xFE"` arrive as one name twice. A +/// `serde_json::Map` keeps the last of two equal names, so the other member's +/// whole value -- an application, a listener -- would be missing from what is +/// shown, and the warning about replaced bytes says nothing about the loss. +/// Deserializing through this refuses instead, at every depth. +/// +/// The question is asked of the names the parser decoded, which is where the +/// answer lives. Asking it of any rewriting of the JSON *source* cannot work, +/// because JSON's own escapes forge whatever the rewriting uses: a `%` written +/// as the escape `\u0025` is not a `%` in the source and is one after parsing. +struct Unambiguous(Value); + +impl<'de> Deserialize<'de> for Unambiguous { + fn deserialize>(deserializer: D) -> Result { + deserializer.deserialize_any(UnambiguousVisitor) + } +} + +struct UnambiguousVisitor; + +impl<'de> Visitor<'de> for UnambiguousVisitor { + type Value = Unambiguous; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("any JSON value") + } + + fn visit_map>(self, mut access: A) -> Result { + let mut object = serde_json::Map::new(); + + while let Some(name) = access.next_key::()? { + // Recursing through the same wrapper is what makes this hold inside + // nested objects and arrays, not only at the top level. + let Unambiguous(member) = access.next_value()?; + + if object.insert(name, member).is_some() { + return Err(serde::de::Error::custom(AMBIGUOUS_NAME)); + } + } + + Ok(Unambiguous(Value::Object(object))) + } + + fn visit_seq>(self, mut access: A) -> Result { + let mut items = Vec::new(); + + while let Some(Unambiguous(item)) = access.next_element()? { + items.push(item); + } + + Ok(Unambiguous(Value::Array(items))) + } + + fn visit_bool(self, value: bool) -> Result { + Ok(Unambiguous(Value::Bool(value))) + } + + fn visit_i64(self, value: i64) -> Result { + Ok(Unambiguous(Value::from(value))) + } + + fn visit_u64(self, value: u64) -> Result { + Ok(Unambiguous(Value::from(value))) + } + + fn visit_f64(self, value: f64) -> Result { + Ok(Unambiguous(Value::from(value))) + } + + fn visit_str(self, value: &str) -> Result { + Ok(Unambiguous(Value::String(value.to_owned()))) + } + + fn visit_string(self, value: String) -> Result { + Ok(Unambiguous(Value::String(value))) + } + + fn visit_unit(self) -> Result { + Ok(Unambiguous(Value::Null)) + } + + fn visit_none(self) -> Result { + Ok(Unambiguous(Value::Null)) + } + + fn visit_some>(self, deserializer: D) -> Result { + Deserialize::deserialize(deserializer) + } +} + +/// What the second decode writes where bytes could not be decoded. +/// +/// Nothing ever searches for this text, and nothing about it needs to be +/// special: a string is reported because its two decodings *differ*, not because +/// it contains a marker. The only requirements are that it is not U+FFFD, so +/// that a byte which could not be decoded reads differently in the two +/// decodings, and that it is plain ASCII, so that decoding cannot produce it. +/// +/// Searching for it instead would be forgeable, which is why this does not: a +/// value that is literally this text defeats a search, and so does JSON's own +/// escaping spelling it out -- and checking that the text is absent from the raw +/// bytes does not see the escaped spelling, so that check would not help either. +const MARKER: &str = "?"; + +/// The same bytes decoded a second time, with every subsequence that is not +/// valid UTF-8 written as [`MARKER`] instead of U+FFFD. +/// +/// Grouped as `String::from_utf8_lossy` groups: one replacement per maximal +/// ill-formed subsequence. So the two decodings of one string agree on how many +/// replacements it has and where they fall, and two names equal under one are +/// equal under the other -- which is what lets the two trees be walked together. +fn marked_decode(bytes: &[u8]) -> String { + let mut text = String::new(); + let mut rest = bytes; + + loop { + let error = match std::str::from_utf8(rest) { + Ok(valid) => { + text.push_str(valid); + return text; + } + Err(error) => error, + }; + + let (valid, invalid) = rest.split_at(error.valid_up_to()); + text.push_str(std::str::from_utf8(valid).unwrap_or_default()); + text.push_str(MARKER); + + // `error_len()` is None when the bytes end mid-sequence: the rest of the + // input is that one incomplete sequence. + let undecodable = error.error_len().unwrap_or(invalid.len()); + rest = &invalid[undecodable..]; + } +} + +/// The second decode's tree, keeping every member of every object, in order. +/// +/// Deliberately not a `serde_json::Map`. The marker is ordinary text, so a +/// document can hold a member whose name *is* the marker beside one whose name +/// decodes to it -- a literal `"?"` key and a key holding a byte that cannot be +/// decoded -- and a map would keep one of the two. The two sides would then +/// disagree on length, the comparison would fall back for that whole object, and +/// every U+FFFD in it would be named: exactly the false positive the comparison +/// exists to prevent, reachable by writing the marker in the document. +/// +/// This is the third in-band token this check has had, and the first two were +/// each defeated by content that could contain them. Keeping the pairs ends +/// that: the marker's text stops mattering to the *structure*, so no document +/// can make the two sides fail to line up. +#[derive(Debug)] +enum Marked { + Object(Vec<(String, Marked)>), + Array(Vec), + String(String), + /// A number, a boolean or null: nothing that can hold a replaced byte. + Scalar, +} + +impl<'de> Deserialize<'de> for Marked { + fn deserialize>(deserializer: D) -> Result { + deserializer.deserialize_any(MarkedVisitor) + } +} + +struct MarkedVisitor; + +impl<'de> Visitor<'de> for MarkedVisitor { + type Value = Marked; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("any JSON value") + } + + fn visit_map>(self, mut access: A) -> Result { + let mut members = Vec::new(); + + while let Some(name) = access.next_key::()? { + members.push((name, access.next_value()?)); + } + + Ok(Marked::Object(members)) + } + + fn visit_seq>(self, mut access: A) -> Result { + let mut items = Vec::new(); + + while let Some(item) = access.next_element()? { + items.push(item); + } + + Ok(Marked::Array(items)) + } + + fn visit_str(self, value: &str) -> Result { + Ok(Marked::String(value.to_owned())) + } + + fn visit_string(self, value: String) -> Result { + Ok(Marked::String(value)) + } + + fn visit_bool(self, _value: bool) -> Result { + Ok(Marked::Scalar) + } + + fn visit_i64(self, _value: i64) -> Result { + Ok(Marked::Scalar) + } + + fn visit_u64(self, _value: u64) -> Result { + Ok(Marked::Scalar) + } + + fn visit_f64(self, _value: f64) -> Result { + Ok(Marked::Scalar) + } + + fn visit_unit(self) -> Result { + Ok(Marked::Scalar) + } + + fn visit_none(self) -> Result { + Ok(Marked::Scalar) + } + + fn visit_some>(self, deserializer: D) -> Result { + Deserialize::deserialize(deserializer) + } +} + +/// Where a replacing decode altered the document. +/// +/// `values` and `names` are RFC 6901 pointers. They are kept apart because the +/// advice differs: a pointer whose *value* was altered can be PUT to, and one +/// whose *name* was altered cannot -- the path itself holds U+FFFD where the +/// server holds a byte, so a PUT would address a different member. +#[derive(Debug, Default)] +struct Altered { + all: Vec, + names: Vec, +} + +/// The members this decode actually altered, and which of them are member names. +/// +/// Not every U+FFFD in the decoded tree is one of ours. `EF BF BD` is valid +/// UTF-8, so a U+FFFD an operator stored arrives intact -- and naming it would +/// send them to `edit`'s recovery advice for a member that was never damaged, +/// while the point of this warning is to say where the damage is. +/// +/// A string is affected exactly when its two decodings differ: a stored U+FFFD +/// reads the same both times, and a byte that could not be decoded reads U+FFFD +/// in one and [`MARKER`] in the other. Member names are compared the same way. +/// The pointers are built from the lossy tree, so they read as what the operator +/// will see everywhere else. +/// +/// `marked` is `None` -- and every U+FFFD is named, as before -- when the second +/// decode will not parse, and the walk falls back the same way for any subtree +/// whose two sides do not line up. A comparison that could not be made is not +/// grounds for saying less. With [`Marked`] keeping duplicate names, objects +/// always line up; what remains is defence, not a path a document can steer into. +fn replaced_members(lossy: &Value, marked: Option<&Marked>) -> Altered { + let mut found = Altered::default(); + walk(lossy, marked, String::new(), &mut found); + // A member whose name and value were both replaced pushes the same pointer + // twice, and the two pushes are adjacent. + found.all.dedup(); + found.names.dedup(); + found +} + +fn walk(lossy: &Value, marked: Option<&Marked>, pointer: String, found: &mut Altered) { + match lossy { + Value::String(string) => { + let other = match marked { + Some(Marked::String(other)) => Some(other.as_str()), + _ => None, + }; + + if differs(string, other) { + found.all.push(pointer); + } + } + Value::Array(items) => { + let others = match marked { + Some(Marked::Array(others)) if others.len() == items.len() => Some(others), + _ => None, + }; + + for (index, item) in items.iter().enumerate() { + let other = others.map(|others| &others[index]); + walk(item, other, format!("{}/{}", pointer, index), found); + } + } + Value::Object(members) => { + // Both decodes keep the server's member order, and `Unambiguous` has + // already refused any document whose names collide under the lossy + // decode, so the two objects line up by position. By position and + // not by name, because the names are what differ. + let others = match marked { + Some(Marked::Object(others)) if others.len() == members.len() => Some(others), + _ => None, + }; + let mut others = others.map(|others| others.iter()); + + for (name, member) in members { + let other = others.as_mut().and_then(|others| others.next()); + let child = format!("{}/{}", pointer, escape(name)); + + if differs(name, other.map(|(other_name, _)| other_name.as_str())) { + found.all.push(child.clone()); + found.names.push(child.clone()); + } + + walk(member, other.map(|(_, value)| value), child, found); + } + } + _ => {} + } +} + +/// Whether a decoded string holds a byte this decode replaced: it differs from +/// the same string decoded the other way, or -- with nothing to compare against +/// -- it holds a U+FFFD, which is what was reported before the comparison +/// existed. +fn differs(lossy: &str, marked: Option<&str>) -> bool { + match marked { + Some(marked) => lossy != marked, + None => lossy.contains(REPLACEMENT), + } +} + +/// RFC 6901 escaping for a member name used as one pointer segment. +fn escape(name: &str) -> String { + name.replace('~', "~0").replace('/', "~1") +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashMap; + + /// A configuration Unit will happily store and hand back: the value of + /// /routes/0/match/uri holds a raw 0xFF, which no UTF-8 decoder accepts. + fn body_with_raw_byte() -> Vec { + let mut bytes = br#"{"routes":[{"match":{"uri":"/caf"#.to_vec(); + bytes.extend_from_slice(&[0xc3, 0xa9, b'-', 0xff, b'-', b'r', b'a', b'w']); + bytes.extend_from_slice(br#""},"action":{"return":204}}]}"#); + bytes + } + + #[test] + fn valid_utf8_is_decoded_exactly() { + let (value, fidelity) = decode::(r#"{"a":"café"}"#.as_bytes()).expect("valid body must decode"); + assert_eq!(value, serde_json::json!({"a": "café"})); + assert_eq!(fidelity, Fidelity::Exact); + assert!(fidelity.warning("/config").is_none()); + } + + #[test] + fn undecodable_bytes_are_replaced_and_reported() { + let (value, fidelity) = decode::(&body_with_raw_byte()).expect("legacy body must still be readable"); + + // The document is intact apart from the one byte that cannot be + // carried by a Rust string. + assert_eq!(value["routes"][0]["match"]["uri"], "/café-\u{fffd}-raw"); + assert_eq!(value["routes"][0]["action"]["return"], 204); + + assert_eq!( + fidelity, + Fidelity::Replaced { + members: vec!["/routes/0/match/uri".to_string()], + names: Vec::new(), + } + ); + + let warning = fidelity.warning("/config").expect("a lossy decode must warn"); + assert!(warning.contains("/config")); + assert!(warning.contains("/routes/0/match/uri")); + assert!(warning.contains("not valid UTF-8")); + } + + /// The typed callers must get the same treatment as `Value` ones: the + /// error-body path deserializes into `Value`, but a caller asking for a + /// map or a `String` goes through `from_value`. + #[test] + fn a_typed_response_is_decoded_lossily_too() { + let mut bytes = br#"{"error":"Unknown parameter \""#.to_vec(); + bytes.push(0xff); + bytes.extend_from_slice(br#"\"."}"#); + + let (map, fidelity) = decode::>(&bytes).expect("error body must survive"); + assert_eq!(map["error"], "Unknown parameter \"\u{fffd}\"."); + assert_eq!( + fidelity, + Fidelity::Replaced { + members: vec!["/error".to_string()], + names: Vec::new(), + } + ); + } + + /// Two application names that differ only in their undecodable byte decode + /// to one name, and `serde_json` keeps the last of two equal names: the + /// first application's whole definition would be gone from what is shown, + /// under a warning that mentions only replaced bytes. Fail instead. + #[test] + fn names_that_collide_when_decoded_are_refused() { + let mut bytes = br#"{"applications":{"app-"#.to_vec(); + bytes.push(0xff); + bytes.extend_from_slice(br#"":{"type":"external","keep":"first"},"app-"#); + bytes.push(0xfe); + bytes.extend_from_slice(br#"":{"type":"python","keep":"second"}}}"#); + + // What the replacing decode on its own produces, and why it cannot be + // shown: one application, and it is not the first one. + let lossy: Value = serde_json::from_str(&String::from_utf8_lossy(&bytes)).expect("the lossy view parses"); + assert_eq!(lossy["applications"].as_object().expect("an object").len(), 1); + + let error = decode::(&bytes).expect_err("a decode that drops a member must fail"); + let message = error.to_string(); + assert!( + message.contains("differ only in bytes that are not valid UTF-8"), + "{}", + message + ); + assert!(message.contains("incomplete"), "{}", message); + } + + /// The collision check must not cost the bodies it is not about: one bad + /// byte in each of two *different* objects collides with nothing. + #[test] + fn replaced_names_in_different_objects_are_not_a_collision() { + let mut bytes = br#"{"applications":{"app-"#.to_vec(); + bytes.push(0xff); + bytes.extend_from_slice(br#"":{"type":"external"}},"upstreams":{"up-"#); + bytes.push(0xff); + bytes.extend_from_slice(br#"":{"servers":{}}}}"#); + + let (value, fidelity) = decode::(&bytes).expect("two separate objects must still decode"); + assert!(value["applications"]["app-\u{fffd}"].is_object()); + assert!(value["upstreams"]["up-\u{fffd}"].is_object()); + assert_eq!( + fidelity, + Fidelity::Replaced { + members: vec![ + "/applications/app-\u{fffd}".to_string(), + "/upstreams/up-\u{fffd}".to_string() + ], + names: vec![ + "/applications/app-\u{fffd}".to_string(), + "/upstreams/up-\u{fffd}".to_string() + ], + } + ); + } + + /// Two equal *values* are not a collision either: only names are keyed. + #[test] + fn replaced_values_are_never_a_collision() { + let mut bytes = br#"{"routes":[{"match":{"uri":"/"#.to_vec(); + bytes.push(0xff); + bytes.extend_from_slice(br#""}},{"match":{"uri":"/"#); + bytes.push(0xfe); + bytes.extend_from_slice(br#""}}]}"#); + + let (value, _) = decode::(&bytes).expect("equal values are not equal names"); + assert_eq!(value["routes"].as_array().expect("an array").len(), 2); + } + + /// A body that will not parse as JSON is shown, and a body shown on a + /// terminal is neither unbounded nor trusted. + #[test] + fn an_unparseable_body_is_bounded_and_rendered() { + // A proxy's error page, in the shape that matters: big. + let huge = body_for_display("A".repeat(5 * 1024 * 1024).as_bytes()); + assert!( + huge.len() < MAX_SHOWN_BODY + 128, + "the body must be bounded, got {} bytes", + huge.len() + ); + assert!(huge.starts_with("AAA")); + // The note counts what was measured, which is the output. + assert!(huge.ends_with("[... truncated after 2048 bytes of output]"), "{}", huge); + + // An escape sequence in that page must not reach the terminal, and is + // written out as the escape rather than as U+FFFD, which here would + // claim a byte could not be decoded. Newlines and tabs stay: a page is + // many lines. + let hostile = body_for_display(b"

502\x1b[2J\x1b]0;pwned\x07

\n\tat proxy\r\n"); + assert_eq!( + hostile, + "

502\\u001b[2J\\u001b]0;pwned\\u0007

\n\tat proxy\\u000d\n" + ); + + // Bytes that are not UTF-8 are shown as U+FFFD, which is what U+FFFD + // means here: a byte that could not be decoded. + assert_eq!(body_for_display(&[b'o', b'k', 0xff]), "ok\u{fffd}"); + + // A body that needs neither is passed through unchanged. + assert_eq!(body_for_display(b"502"), "502"); + } + + /// Reading no more of the body than can be shown is the point of the bound, + /// so the rendering must not depend on how much came after it. + #[test] + fn only_as_much_of_a_body_as_can_be_shown_is_read() { + let mut page = b"

502

".to_vec(); + page.resize(8 * 1024 * 1024, b'x'); + + let shown = body_for_display(&page); + assert!(shown.starts_with("

502

xxx")); + assert!(shown.len() < MAX_SHOWN_BODY + 128); + + // A character the cut lands inside must not break the rendering. + let mut multibyte = "caf\u{e9}".repeat(2000).into_bytes(); + multibyte.truncate(MAX_SHOWN_BODY + 1); + let shown = body_for_display(&multibyte); + assert!(shown.starts_with("caf\u{e9}caf\u{e9}"), "{}", &shown[..16]); + assert!(shown.len() < MAX_SHOWN_BODY + 128); + } + + /// A member name can hold any character JSON can spell, including the + /// escape for ESC, and every warning interpolates the pointer into stderr. + /// An application named with an ANSI sequence must not be able to clear the + /// operator's screen, and one named with a newline must not be able to forge + /// a line that looks like unitctl's own. + /// + /// The rendering is asserted, not merely the absence of control characters: + /// an absence passes for U+FFFD too, and U+FFFD here would claim the name + /// holds a byte the server could not hand over, which it does not. + #[test] + fn a_pointer_cannot_carry_control_characters_to_the_terminal() { + // The name is the six characters backslash-u-0-0-1-b and so on: what a + // server would send, not a raw control byte, which the parser refuses. + let mut bytes = Vec::new(); + bytes.extend_from_slice(b"{\"app"); + bytes.extend_from_slice(b"\\u001b[2J"); + bytes.extend_from_slice(b"\\u0007"); + bytes.extend_from_slice(b"\\nWarning: all is well"); + bytes.extend_from_slice(b"\\r\\t\":\"caf"); + bytes.push(0xff); + bytes.extend_from_slice(b"\"}"); + + // serde_json really does decode those into control characters: without + // that this test would be proving nothing. + let decoded: Value = serde_json::from_str(&String::from_utf8_lossy(&bytes)).expect("the body parses"); + let name = decoded.as_object().expect("an object").keys().next().expect("a name"); + assert!(name.contains('\u{1b}'), "serde must decode the escape: {:?}", name); + assert!(name.contains('\n')); + + let (_, fidelity) = decode::(&bytes).expect("the body must still be readable"); + + // Every control character is written out, as the escape and not as + // U+FFFD, and the ordinary text around them still identifies the member. + let listed = fidelity.members(); + assert_eq!(listed, "/app\\u001b[2J\\u0007\\u000aWarning: all is well\\u000d\\u0009"); + assert!( + !listed.contains(REPLACEMENT), + "a name that is valid UTF-8 must not be shown as a replaced byte: {:?}", + listed + ); + + let warning = fidelity.warning("/config").expect("a lossy decode must warn"); + assert!(warning.ends_with(&listed), "{}", warning); + // The warning's own two lines are the only newlines in it. + assert_eq!(warning.matches('\n').count(), 1, "a newline survived: {:?}", warning); + } + + /// The percent sign was once how this check told undecodable bytes apart, + /// and JSON can spell a percent as an escape, so neither a literal `%FF` in + /// a name nor an escaped one may hide a collision. Asking the question of + /// the decoded names is what makes the spelling irrelevant. + #[test] + fn no_spelling_of_a_name_hides_a_collision() { + let mut literal = br#"{"applications":{"app-"#.to_vec(); + literal.push(0xff); + literal.extend_from_slice(br#"":{"keep":"raw-ff"},"app-"#); + literal.push(0xfe); + literal.extend_from_slice(br#"":{"keep":"raw-fe"},"app-%FF":{"keep":"literal"}}}"#); + + let mut escaped = br#"{"applications":{"app-"#.to_vec(); + escaped.push(0xff); + escaped.extend_from_slice(br#"":{"keep":"raw-ff"},"app-"#); + escaped.push(0xfe); + escaped.extend_from_slice(br#"":{"keep":"raw-fe"},"app-"#); + // An escaped percent: the six characters backslash-u-0-0-2-5. + escaped.extend_from_slice(b"\\u0025"); + escaped.extend_from_slice(br#"FF":{"keep":"escaped"}}}"#); + + for bytes in [&literal, &escaped] { + // Three names went in and the replacing decode keeps two, so the + // body really is one a decode would silently truncate. + let lossy: Value = serde_json::from_str(&String::from_utf8_lossy(bytes)).expect("the lossy view parses"); + assert_eq!(lossy["applications"].as_object().expect("an object").len(), 2); + + let error = decode::(bytes).expect_err("a decode that drops a member must fail"); + assert!( + error + .to_string() + .contains("differ only in bytes that are not valid UTF-8"), + "{}", + error + ); + } + } + + /// The one collision where neither name was damaged by the decode: an + /// operator can legitimately store a name holding a real U+FFFD (`EF BF BD`), + /// and a sibling holding a raw undecodable byte decodes to the same thing. + #[test] + fn a_stored_replacement_character_collides_with_a_replaced_byte() { + let mut bytes = br#"{"applications":{"app-"#.to_vec(); + bytes.extend_from_slice("\u{fffd}".as_bytes()); + bytes.extend_from_slice(br#"":{"keep":"stored"},"app-"#); + bytes.push(0xff); + bytes.extend_from_slice(br#"":{"keep":"raw"}}}"#); + + // The stored name is valid UTF-8 and arrives untouched; it is the raw + // byte that decodes onto it. + let lossy: Value = serde_json::from_str(&String::from_utf8_lossy(&bytes)).expect("the lossy view parses"); + assert_eq!(lossy["applications"].as_object().expect("an object").len(), 1); + + let error = decode::(&bytes).expect_err("one of the two would be dropped"); + assert!(error.to_string().contains("share a name"), "{}", error); + } + + /// Duplicate names are refused whatever made them duplicate, so the refusal + /// must not blame undecodable bytes as though it had checked. + /// + /// A body that is valid UTF-8 never reaches the wrapper at all: the strict + /// decode succeeds first, dropping a member as `serde_json` always has. + /// Only a body that also holds an undecodable byte somewhere gets here. + #[test] + fn the_refusal_does_not_blame_a_cause_it_did_not_check() { + let clean = br#"{"applications":{"a":1,"a":2},"x":"ok"}"#; + let (value, fidelity) = decode::(clean).expect("the strict decode accepts duplicates"); + assert!(fidelity.is_exact(), "a valid-UTF-8 body must not reach the wrapper"); + assert_eq!(value["applications"]["a"], 2); + + let mut dirty = br#"{"applications":{"a":1,"a":2},"x":"caf"#.to_vec(); + dirty.push(0xff); + dirty.extend_from_slice(br#""}"#); + + let message = decode::(&dirty) + .expect_err("a duplicate name is still a dropped member") + .to_string(); + assert!( + message.contains("two members of one object share a name"), + "{}", + message + ); + // The mechanism is offered as the usual reason, not asserted as the cause. + assert!(message.contains("the usual reason is"), "{}", message); + assert!( + message.contains("differ only in bytes that are not valid UTF-8"), + "{}", + message + ); + assert!(message.contains("unitctl export"), "{}", message); + } + + /// The refusal is about names, at every depth, and nothing else: a duplicate + /// inside a nested object or inside an array element is the same loss. + #[test] + fn a_collision_is_caught_at_every_depth() { + let mut nested = br#"{"routes":{"a":{"b":{"app-"#.to_vec(); + nested.push(0xff); + nested.extend_from_slice(br#"":1,"app-"#); + nested.push(0xfe); + nested.extend_from_slice(br#"":2}}}}"#); + assert!(decode::(&nested).is_err(), "a nested collision must be caught"); + + let mut in_array = br#"{"routes":[{"app-"#.to_vec(); + in_array.push(0xff); + in_array.extend_from_slice(br#"":1,"app-"#); + in_array.push(0xfe); + in_array.extend_from_slice(br#"":2}]}"#); + assert!( + decode::(&in_array).is_err(), + "a collision inside an array element must be caught" + ); + } + + /// A sequence truncated mid-character is one undecodable run, not a reason + /// to refuse: there is one name, so there is nothing ambiguous about it. + #[test] + fn bytes_that_end_mid_character_still_decode() { + let mut bytes = br#"{"routes":[{"match":{"uri":"/caf"#.to_vec(); + bytes.extend_from_slice(&[0xe2, 0x82]); + bytes.extend_from_slice(br#""}}]}"#); + + let (value, fidelity) = decode::(&bytes).expect("one truncated character is not a collision"); + assert_eq!(value["routes"][0]["match"]["uri"], "/caf\u{fffd}"); + assert!(!fidelity.is_exact()); + } + + /// `decode` is generic, and the callers that read a configuration ask for a + /// map, which is what refuses a 200 carrying `null`, an array or a bare + /// string. That has to hold on the lossy path too, since a body with an + /// undecodable byte reaches `T` through `from_value` instead of directly. + #[test] + fn a_configuration_shape_is_required_on_both_paths() { + type Config = std::collections::BTreeMap; + + let (config, fidelity) = + decode::(br#"{"listeners":{"*:8080":{"pass":"routes"}}}"#).expect("an object is a configuration"); + assert_eq!(config.keys().collect::>(), vec!["listeners"]); + assert!(fidelity.is_exact()); + + for body in [&b"null"[..], b"[]", b"[{\"listeners\":{}}]", b"\"a string\"", b"204"] { + assert!( + decode::(body).is_err(), + "{} is not a configuration", + String::from_utf8_lossy(body) + ); + // And why the type the caller asks for is what does the refusing: + // this function cannot, because `Value` accepts every shape. + assert!(decode::(body).is_ok()); + } + + // The same bodies with an undecodable byte in them take the lossy path, + // and must be refused there as well. + let mut array = br#"["caf"#.to_vec(); + array.push(0xff); + array.extend_from_slice(br#""]"#); + assert!(std::str::from_utf8(&array).is_err()); + assert!(decode::(&array).is_err(), "an array is not a configuration"); + + let mut string = br#""caf"#.to_vec(); + string.push(0xff); + string.push(b'"'); + assert!( + decode::(&string).is_err(), + "a bare string is not a configuration" + ); + + // And an object with such a byte still decodes, lossily. + let mut object = br#"{"caf"#.to_vec(); + object.push(0xff); + object.extend_from_slice(br#"":{}}"#); + let (_, fidelity) = decode::(&object).expect("an object is still a configuration"); + assert!(!fidelity.is_exact()); + } + + /// `EF BF BD` is valid UTF-8, so a U+FFFD an operator stored arrives intact + /// and was never replaced. Naming it sends them to `edit`'s recovery advice + /// for a member that is not damaged, and the point of the warning is to say + /// which member is. + #[test] + fn a_stored_replacement_character_is_not_reported_as_replaced() { + let mut bytes = br#"{"valid":"stored: "#.to_vec(); + bytes.extend_from_slice(&[0xef, 0xbf, 0xbd]); + bytes.extend_from_slice(br#"","legacy":"raw: "#); + bytes.push(0xff); + bytes.extend_from_slice(br#""}"#); + + let (value, fidelity) = decode::(&bytes).expect("the body must still be readable"); + + // Both values read as U+FFFD, which is exactly why they cannot be told + // apart by looking at the decoded text. + assert_eq!(value["valid"], "stored: \u{fffd}"); + assert_eq!(value["legacy"], "raw: \u{fffd}"); + + assert_eq!( + fidelity, + Fidelity::Replaced { + members: vec!["/legacy".to_string()], + names: Vec::new(), + } + ); + + let warning = fidelity.warning("/config").expect("a lossy decode must warn"); + assert!(warning.contains("/legacy"), "{}", warning); + assert!( + !warning.contains("/valid"), + "an intact member must not be named: {}", + warning + ); + } + + /// The same distinction for a member *name*: one name holds a stored U+FFFD + /// and its sibling holds a byte that could not be decoded. They are + /// different names, so nothing collides, and only one was altered. + #[test] + fn a_stored_replacement_character_in_a_name_is_not_reported() { + let mut bytes = br#"{"applications":{"app-"#.to_vec(); + bytes.extend_from_slice(&[0xef, 0xbf, 0xbd]); + bytes.extend_from_slice(br#"-stored":{"type":"external"},"other-"#); + bytes.push(0xff); + bytes.extend_from_slice(br#"-raw":{"type":"python"}}}"#); + + let (_, fidelity) = decode::(&bytes).expect("two different names do not collide"); + assert_eq!( + fidelity, + Fidelity::Replaced { + members: vec!["/applications/other-\u{fffd}-raw".to_string()], + names: vec!["/applications/other-\u{fffd}-raw".to_string()], + } + ); + } + + /// With nothing to compare against, every U+FFFD is named, which is what was + /// reported before the second decode existed. A comparison that could not be + /// made is not grounds for saying less. + #[test] + fn without_a_second_decode_every_replacement_is_named() { + let lossy = serde_json::json!({ + "valid": "stored: \u{fffd}", + "legacy": "raw: \u{fffd}", + "clean": "nothing here", + }); + + let altered = replaced_members(&lossy, None); + assert_eq!(altered.all, vec!["/valid".to_string(), "/legacy".to_string()]); + assert!(altered.names.is_empty(), "no member *name* holds one here"); + } + + /// Nothing searches the second decode for its marker, so content that is + /// the marker -- or JSON's escaping spelling it out, which no check of the + /// raw bytes would see -- cannot make an intact member look replaced. + #[test] + fn content_that_looks_like_the_marker_is_not_reported() { + let mut literal = br#"{"innocent":"?","legacy":"raw: "#.to_vec(); + literal.push(0xff); + literal.extend_from_slice(br#""}"#); + + let mut escaped = br#"{"innocent":""#.to_vec(); + escaped.extend_from_slice(b"\\u003f"); + escaped.extend_from_slice(br#"","legacy":"raw: "#); + escaped.push(0xff); + escaped.extend_from_slice(br#""}"#); + + for bytes in [&literal, &escaped] { + let (value, fidelity) = decode::(bytes).expect("the body must decode"); + assert_eq!(value["innocent"], "?", "the marker text decodes to itself"); + assert_eq!( + fidelity, + Fidelity::Replaced { + members: vec!["/legacy".to_string()], + names: Vec::new(), + }, + "only the member with an undecodable byte is named" + ); + } + } + + /// The marker is ordinary text, so a document can hold a member whose name + /// *is* it beside one whose name decodes to it. Keeping duplicate names on + /// the marked side is what stops that from collapsing the two sides into + /// different lengths and falling back to naming every U+FFFD in the object. + #[test] + fn a_name_that_is_the_marker_does_not_disturb_the_comparison() { + let mut bytes = br#"{"applications":{"?":{"note":"literal question mark"},""#.to_vec(); + bytes.push(0xff); + bytes.extend_from_slice(br#"":{"note":"raw byte"},"valid":{"note":"stored: "#); + bytes.extend_from_slice(&[0xef, 0xbf, 0xbd]); + bytes.extend_from_slice(br#""}}}"#); + + // Three names go in, and two of them read as "?" once the bytes that + // could not be decoded are written as the marker. + let lossy: Value = serde_json::from_str(&String::from_utf8_lossy(&bytes)).expect("the lossy view parses"); + assert_eq!(lossy["applications"].as_object().expect("an object").len(), 3); + + let (_, fidelity) = decode::(&bytes).expect("the body must decode"); + assert_eq!( + fidelity, + Fidelity::Replaced { + members: vec!["/applications/\u{fffd}".to_string()], + names: vec!["/applications/\u{fffd}".to_string()], + }, + "only the name holding a byte that could not be decoded was altered" + ); + + // The two members that were not touched: the one literally named "?", + // and the one whose value holds a U+FFFD the operator stored. + let warning = fidelity.warning("/config").expect("a lossy decode must warn"); + assert!(!warning.contains("/valid"), "an intact member was named: {}", warning); + } + + /// A pointer whose *name* was altered cannot be PUT to: it shows U+FFFD + /// where the server holds a byte, so it names a member that does not exist. + /// The two kinds are kept apart so the advice can say so. + #[test] + fn a_replaced_name_is_reported_apart_from_a_replaced_value() { + let mut in_a_name = br#"{"applications":{"app-"#.to_vec(); + in_a_name.push(0xff); + in_a_name.extend_from_slice(br#"":{"type":"external"}}}"#); + + let (_, fidelity) = decode::(&in_a_name).expect("the body must decode"); + assert_eq!(fidelity.members(), "/applications/app-\u{fffd}"); + assert_eq!( + fidelity.replaced_names(), + "/applications/app-\u{fffd}", + "a replaced name must be reported as one" + ); + + let mut in_a_value = br#"{"routes":[{"match":{"uri":"/caf"#.to_vec(); + in_a_value.push(0xff); + in_a_value.extend_from_slice(br#""}}]}"#); + + let (_, fidelity) = decode::(&in_a_value).expect("the body must decode"); + assert_eq!(fidelity.members(), "/routes/0/match/uri"); + assert!( + fidelity.replaced_names().is_empty(), + "a replaced value is repairable by a PUT and must not be listed as a name" + ); + + // An exact body has neither. + let (_, fidelity) = decode::(br#"{"routes":[]}"#).expect("valid UTF-8 decodes exactly"); + assert!(fidelity.replaced_names().is_empty()); + } + + /// The two lists are what the advice branches on, so each must hold only its + /// own kind. + #[test] + fn values_and_names_are_listed_apart() { + let mut bytes = br#"{"applications":{"app-"#.to_vec(); + bytes.push(0xff); + bytes.extend_from_slice(br#"":{"type":"external"}},"routes":[{"uri":"/caf"#); + bytes.push(0xfe); + bytes.extend_from_slice(br#""}]}"#); + + let (_, fidelity) = decode::(&bytes).expect("the body decodes"); + assert_eq!(fidelity.replaced_values(), "/routes/0/uri"); + assert_eq!(fidelity.replaced_names(), "/applications/app-\u{fffd}"); + assert_eq!( + fidelity.members(), + "/applications/app-\u{fffd}, /routes/0/uri", + "the headline still lists both" + ); + + // A body with only a replaced value has no names, and the reverse. + let mut value_only = br#"{"routes":[{"uri":"/caf"#.to_vec(); + value_only.push(0xff); + value_only.extend_from_slice(br#""}]}"#); + let (_, fidelity) = decode::(&value_only).expect("the body decodes"); + assert_eq!(fidelity.replaced_values(), "/routes/0/uri"); + assert!(fidelity.replaced_names().is_empty()); + + // And an exact body has neither. + let (_, fidelity) = decode::(br#"{"routes":[]}"#).expect("valid UTF-8 decodes exactly"); + assert!(fidelity.replaced_values().is_empty()); + assert!(fidelity.replaced_names().is_empty()); + } + + /// Both at once: one member's name was altered and another's value was, and + /// each is reported in its own list. + #[test] + fn a_name_and_a_value_are_reported_separately() { + let mut bytes = br#"{"applications":{"app-"#.to_vec(); + bytes.push(0xff); + bytes.extend_from_slice(br#"":{"type":"external"}},"routes":[{"uri":"/caf"#); + bytes.push(0xfe); + bytes.extend_from_slice(br#""}]}"#); + + let (_, fidelity) = decode::(&bytes).expect("the body must decode"); + assert_eq!( + fidelity, + Fidelity::Replaced { + members: vec!["/applications/app-\u{fffd}".to_string(), "/routes/0/uri".to_string()], + names: vec!["/applications/app-\u{fffd}".to_string()], + } + ); + } + + /// serde_json must keep the order the server sent. The comparison in + /// `replaced_members` pairs the two decodes member by member, in order, so a + /// sorted map would pair them wrongly. The names here are not in sorted + /// order, so this fails if the preserve_order feature is ever turned off. + #[test] + fn member_order_survives_a_round_trip() { + let source = r#"{"z":1,"a":2,"m":3}"#; + let parsed: Value = serde_json::from_str(source).expect("the body parses"); + + let names: Vec<&str> = parsed + .as_object() + .expect("an object") + .keys() + .map(String::as_str) + .collect(); + assert_eq!(names, vec!["z", "a", "m"], "serde_json must keep member order"); + + assert_eq!(serde_json::to_string(&parsed).expect("it serializes"), source); + } + + /// The same precondition, at the place that depends on it: the two decodes + /// are paired by position. With a sorted map the damaged value and the + /// undamaged one swap places, and both are reported. + #[test] + fn members_out_of_sorted_order_are_paired_correctly() { + let mut bytes = br#"{"z":"ok","a":"raw: "#.to_vec(); + bytes.push(0xff); + bytes.extend_from_slice(br#""}"#); + + let (_, fidelity) = decode::(&bytes).expect("the body decodes"); + assert_eq!( + fidelity, + Fidelity::Replaced { + members: vec!["/a".to_string()], + names: Vec::new(), + }, + "only /a holds a byte that could not be decoded, and it is a value" + ); + } + + /// A member *name* can hold the raw bytes just as a value can, and an + /// application name is exactly such a member. + #[test] + fn a_replaced_member_name_is_pointed_at_once() { + let mut bytes = br#"{"applications":{"app-"#.to_vec(); + bytes.push(0xff); + bytes.extend_from_slice(br#"":{"type":"external"}}}"#); + + let (_, fidelity) = decode::(&bytes).expect("body must survive"); + assert_eq!( + fidelity, + Fidelity::Replaced { + members: vec!["/applications/app-\u{fffd}".to_string()], + names: vec!["/applications/app-\u{fffd}".to_string()], + } + ); + } + + /// `/` and `~` in a member name are pointer syntax and must be escaped, or + /// the pointer names a member that does not exist. + #[test] + fn pointer_segments_are_escaped() { + let mut bytes = br#"{"listeners":{"*:8080":{"pass":"routes/a~b/"#.to_vec(); + bytes.push(0xff); + bytes.extend_from_slice(br#""}}}"#); + + let (_, fidelity) = decode::(&bytes).expect("body must survive"); + assert_eq!( + fidelity, + Fidelity::Replaced { + members: vec!["/listeners/*:8080/pass".to_string()], + names: Vec::new(), + } + ); + + assert_eq!(escape("a/b~c"), "a~1b~0c"); + } + + /// Tolerating undecodable bytes must not turn malformed JSON into a + /// success: a body that is valid UTF-8 fails exactly as it did before. + #[test] + fn malformed_utf8_json_still_fails() { + assert!(decode::(br#"{"a":"#).is_err()); + assert!(decode::(b"not json at all").is_err()); + // Undecodable *and* malformed is still an error. + assert!(decode::(&[b'{', 0xff]).is_err()); + } + + /// A body that decodes but does not match the type asked for keeps failing + /// too, rather than being silently coerced. + #[test] + fn a_type_mismatch_is_not_hidden() { + assert!(decode::>(br#"{"a":1}"#).is_err()); + let mut bytes = br#"{"a":1,"b":""#.to_vec(); + bytes.push(0xff); + bytes.extend_from_slice(br#""}"#); + assert!(decode::>(&bytes).is_err()); + } + + /// The defensive arm: a lossy decode that located nothing still has to say + /// something, rather than printing an empty list. + #[test] + fn a_lossy_decode_that_located_nothing_still_says_so() { + let listed = Fidelity::Replaced { + members: Vec::new(), + names: Vec::new(), + } + .members(); + assert_eq!(listed, "an unlocatable part of the document"); + assert!(Fidelity::Replaced { + members: Vec::new(), + names: Vec::new() + } + .warning("/config") + .expect("a lossy decode must warn") + .contains("an unlocatable part of the document")); + } + + #[test] + fn a_long_list_of_members_is_capped() { + let members: Vec = (0..12).map(|i| format!("/m{}", i)).collect(); + let listed = Fidelity::Replaced { + members, + names: Vec::new(), + } + .members(); + assert!(listed.starts_with("/m0, /m1,")); + assert!(listed.ends_with("and 4 more")); + } +} diff --git a/tools/unitctl/unit-client-rs/src/lib.rs b/tools/unitctl/unit-client-rs/src/lib.rs index 59dd26813..6022463c7 100644 --- a/tools/unitctl/unit-client-rs/src/lib.rs +++ b/tools/unitctl/unit-client-rs/src/lib.rs @@ -5,6 +5,7 @@ extern crate hyperlocal; extern crate serde; extern crate serde_json; pub mod control_socket_address; +pub mod json_body; mod runtime_flags; pub mod unit_client; mod unitd_cmd; diff --git a/tools/unitctl/unit-client-rs/src/unit_client.rs b/tools/unitctl/unit-client-rs/src/unit_client.rs index 55ff72ad8..ecdcd7262 100644 --- a/tools/unitctl/unit-client-rs/src/unit_client.rs +++ b/tools/unitctl/unit-client-rs/src/unit_client.rs @@ -4,10 +4,11 @@ use std::fmt::Debug; use std::rc::Rc; use std::{fmt, io}; -use bytes::{Buf, Bytes}; +use bytes::Bytes; use custom_error::custom_error; use http_body_util::{BodyExt, Full}; use hyper::{http, Request}; +#[cfg(feature = "tls")] use hyper_rustls::{HttpsConnector, HttpsConnectorBuilder}; use hyper_util::client::legacy::connect::HttpConnector; use hyper_util::client::legacy::{Client, ResponseFuture}; @@ -16,6 +17,7 @@ use hyperlocal::UnixConnector; use serde::{Deserialize, Serialize}; use crate::control_socket_address::ControlSocket; +use crate::json_body::{self, Fidelity}; use unit_openapi::apis::configuration::Configuration; use unit_openapi::apis::{ ApplicationsApi, ApplicationsApiClient, AppsApi, AppsApiClient, Error as OpenAPIError, ListenersApi, @@ -28,7 +30,10 @@ const USER_AGENT: &str = concat!("Unit CLI/", env!("CARGO_PKG_VERSION"), "/rust" custom_error! {pub UnitClientError OpenAPIError { source: OpenAPIError } = "OpenAPI error", JsonError { source: serde_json::Error, - path: String} = "JSON error [path={path}]", + path: String} = "JSON error [path={path}]: {source}", + UnparseableBodyError { source: serde_json::Error, + path: String, + body: String} = "JSON error [path={path}]: {source}\n{body}", HyperError { source: hyper_util::client::legacy::Error, control_socket_address: String, path: String} = "Communications error [control_socket_address={control_socket_address}, path={path}]: {source}", @@ -43,7 +48,9 @@ custom_error! {pub UnitClientError HttpResponseJsonBodyError { status: http::StatusCode, path: String, error: String, - detail: String} = "HTTP response error [path={path}, status={status}]:\n Error: {error}\n Detail: {detail}", + detail: String, + location: String, + suggestion: String} = "HTTP response error [path={path}, status={status}]:\n Error: {error}\n Detail: {detail}{location}{suggestion}", IoError { source: io::Error, socket: String } = "IO error [socket={socket}]", UnixSocketAddressError { source: io::Error, @@ -126,21 +133,29 @@ macro_rules! new_openapi_client { }; } +/// Connector for a TCP control socket. Without the "tls" feature there is no +/// HTTPS connector, and an "https://" address is refused while it is parsed. +#[cfg(feature = "tls")] +type TcpConnector = HttpsConnector; +#[cfg(not(feature = "tls"))] +type TcpConnector = HttpConnector; + +#[cfg(feature = "tls")] +const TCP_CLIENT_NAME: &str = "Client, Full>"; +#[cfg(not(feature = "tls"))] +const TCP_CLIENT_NAME: &str = "Client>"; + #[derive(Clone)] pub enum RemoteClient { - Unix { - client: Client>, - }, - Tcp { - client: Client, Full>, - }, + Unix { client: Client> }, + Tcp { client: Client> }, } impl RemoteClient { fn client_name(&self) -> &str { match self { RemoteClient::Unix { .. } => "Client>", - RemoteClient::Tcp { .. } => "Client, Full>", + RemoteClient::Tcp { .. } => TCP_CLIENT_NAME, } } @@ -165,6 +180,62 @@ pub struct UnitClient { client: Box, } +/// Say out loud when a body had to be decoded with bytes replaced, so that +/// nobody mistakes what is printed for what the server stores. +fn warn_if_replaced(fidelity: &Fidelity, path: &str) { + if let Some(warning) = fidelity.warning(path) { + eprintln!("{}", warning); + } +} + +/// Pulls the parts of a control API error body out for display: the message, the +/// detail, where the error is, and the name a typo probably meant. The last two +/// come back pre-formatted, empty when they do not apply, so a response carrying +/// neither renders exactly as it did before they existed. +fn describe_error_body(body: &serde_json::Value) -> (String, String, String, String) { + // Whatever answered wrote these strings, and they are printed to a + // terminal. A genuine unitd echoes only the requester's own member name + // here (src/nxt_conf_validation.c), so the writer to guard against is the + // same intermediary the unparseable-body path already guards against: an + // escape sequence in a 400 would otherwise reach the screen, and a newline + // would let it forge a line of unitctl's own output. + let member = + |name: &str| json_body::one_line(body.get(name).and_then(serde_json::Value::as_str).unwrap_or_default()); + + let mut error = member("error"); + if error.is_empty() { + error = "Unknown error".to_string(); + } + + // A validation error places itself with an RFC 6901 pointer, where the empty + // string is the document root; an error found while parsing the request has + // no pointer and places itself by position instead. + let place = body.get("location").and_then(|location| { + let number = |name: &str| location.get(name).and_then(serde_json::Value::as_i64); + + match location.get("path").and_then(serde_json::Value::as_str) { + Some("") => Some("the document root".to_string()), + Some(pointer) => Some(json_body::one_line(pointer)), + None => match (number("line"), number("column"), number("offset")) { + (Some(line), Some(column), _) => Some(format!("line {}, column {}", line, column)), + (_, _, Some(offset)) => Some(format!("byte offset {}", offset)), + _ => None, + }, + } + }); + + let location = place + .map(|place| format!("\n Location: {}", place)) + .unwrap_or_default(); + + let suggestion = match member("suggestion").as_str() { + "" => String::new(), + name => format!("\n Did you mean: {}", name), + }; + + (error, member("detail"), location, suggestion) +} + impl UnitClient { pub fn new(control_socket: ControlSocket) -> Self { if control_socket.is_local_socket() { @@ -174,15 +245,26 @@ impl UnitClient { } } - pub fn new_http(control_socket: ControlSocket) -> Self { - let connector = HttpsConnectorBuilder::new() + #[cfg(feature = "tls")] + fn tcp_connector() -> TcpConnector { + HttpsConnectorBuilder::new() .with_native_roots() .unwrap_or_else(|_| HttpsConnectorBuilder::new().with_webpki_roots()) .https_or_http() - .enable_all_versions() - .build(); - let remote_client: Client, Full> = - Client::builder(TokioExecutor::new()).build(connector); + // The control API speaks HTTP/1.1 only. This also keeps "h2" out + // of the ALPN list we offer. + .enable_http1() + .build() + } + + #[cfg(not(feature = "tls"))] + fn tcp_connector() -> TcpConnector { + HttpConnector::new() + } + + pub fn new_http(control_socket: ControlSocket) -> Self { + let remote_client: Client> = + Client::builder(TokioExecutor::new()).build(Self::tcp_connector()); Self { control_socket, client: Box::from(RemoteClient::Tcp { client: remote_client }), @@ -198,11 +280,17 @@ impl UnitClient { } } - /// Sends a request to Unit and deserializes the JSON response body into the value of type `RESPONSE`. - pub async fn send_request_and_deserialize_response serde::Deserialize<'de>>( + /// Sends a request to Unit and returns the response body as it arrived. + /// + /// A caller that will write the document back needs the bytes the server + /// sent and not a re-encoding of them, because a configuration can hold + /// bytes no Rust `String` can carry. Turning a non-success status into an + /// error happens here so that those callers share it with the + /// deserializing variant below. + pub async fn send_request_and_collect_body( &self, mut request: Request>, - ) -> Result { + ) -> Result { let uri = request.uri().clone(); let path: &str = uri.path(); @@ -226,25 +314,67 @@ impl UnitClient { })? .to_bytes(); - let mut reader = body_bytes.reader(); if !status.is_success() { - let error: HashMap = - serde_json::from_reader(&mut reader).map_err(|error| UnitClientError::JsonError { - source: error, - path: path.to_string(), - })?; + // The control API answers an error with "error" and, where they + // apply, "detail", a "location" object placing the error, and a + // "suggestion" naming the parameter an unknown one was probably a + // typo of. "location" is an object, so the body is not a map of + // strings and must not be deserialized as one: that fails the + // whole body and hides the message it carries. + let (body, fidelity) = match json_body::decode::(&body_bytes) { + Ok(decoded) => decoded, + // Not JSON at all -- an intermediary's page, or a truncated + // answer. A body that will not parse is exactly when its text + // matters most, so show it instead of a parser complaint, but + // show it bounded and with the control characters taken out. + Err(_) => { + return Err(UnitClientError::HttpResponseError { + status, + path: path.to_string(), + body: json_body::body_for_display(&body_bytes), + }) + } + }; + + warn_if_replaced(&fidelity, path); + + let (error, detail, location, suggestion) = describe_error_body(&body); return Err(UnitClientError::HttpResponseJsonBodyError { status, path: path.to_string(), - error: error.get("error").unwrap_or(&"Unknown error".into()).to_string(), - detail: error.get("detail").unwrap_or(&"".into()).to_string(), + error, + detail, + location, + suggestion, }); } - serde_json::from_reader(&mut reader).map_err(|error| UnitClientError::JsonError { - source: error, - path: path.to_string(), - }) + + Ok(body_bytes) + } + + /// Sends a request to Unit and deserializes the JSON response body into the value of type `RESPONSE`. + pub async fn send_request_and_deserialize_response serde::Deserialize<'de>>( + &self, + request: Request>, + ) -> Result { + let path = request.uri().path().to_string(); + let body_bytes = self.send_request_and_collect_body(request).await?; + + // A success status is no guarantee that Unit answered: a proxy or a + // captive portal can return 200 with a page of its own. Show it, for + // the reason the non-success arm above shows its body -- a body that + // will not parse is exactly when its text matters most. + let (response, fidelity) = + json_body::decode(&body_bytes).map_err(|error| UnitClientError::UnparseableBodyError { + source: error, + path: path.clone(), + body: json_body::body_for_display(&body_bytes), + })?; + + warn_if_replaced(&fidelity, &path); + + Ok(response) } pub fn listeners_api(&self) -> Box { @@ -328,7 +458,14 @@ impl UnitClient { } } -pub type UnitSerializableMap = HashMap; +/// A JSON object as it travels between unitctl and the control API. +/// +/// `serde_json::Map` and not `HashMap`: with the crate's `preserve_order` +/// feature this keeps the server's member order, and a `HashMap` does not. A +/// `HashMap` here reordered the members of `/config` on every single run, so +/// two `unitctl export`s of one unchanged configuration disagreed, and +/// `unitctl edit` presented a document shuffled differently each time. +pub type UnitSerializableMap = serde_json::Map; #[derive(Clone, Debug, Deserialize, Serialize)] pub struct UnitStatus { @@ -370,6 +507,158 @@ mod tests { use crate::unitd_instance::UnitdInstance; use super::*; + + /// The four strings a control API error body carries are the server's text, + /// and they print straight to stderr. The rendering is asserted rather than + /// the absence of control characters: an absence is also satisfied by U+FFFD, + /// which would say these strings hold bytes that could not be decoded. + #[test] + fn an_error_body_cannot_carry_control_characters_to_the_terminal() { + let esc = String::from_utf8(vec![0x1b]).unwrap(); + let bell = String::from_utf8(vec![0x07]).unwrap(); + + let (error, detail, location, suggestion) = describe_error_body(&serde_json::json!({ + "error": format!("{}[2Jcleared", esc), + "detail": format!("{}pwned\nWarning: forged line", bell), + "location": {"path": format!("/routes/{}[31m", esc)}, + "suggestion": format!("pass{}", esc), + })); + + assert_eq!(error, "\\u001b[2Jcleared"); + assert_eq!(detail, "\\u0007pwned\\u000aWarning: forged line"); + assert_eq!(location, "\n Location: /routes/\\u001b[31m"); + assert_eq!(suggestion, "\n Did you mean: pass\\u001b"); + + for part in [&error, &detail, &location, &suggestion] { + assert!( + !part.contains('\u{fffd}'), + "text that is valid UTF-8 must not be shown as a replaced byte: {:?}", + part + ); + } + + // Location and suggestion are built with a deliberate leading newline, + // and it is the only one either may have. + assert_eq!(location.matches('\n').count(), 1); + assert_eq!(suggestion.matches('\n').count(), 1); + assert!(!error.contains('\n')); + assert!(!detail.contains('\n'), "detail forged a line: {:?}", detail); + } + + /// The shapes nxt_controller_response() can send, and how each renders. + #[test] + fn describes_every_error_body_shape() { + let case = |body: serde_json::Value| { + let (error, detail, location, suggestion) = describe_error_body(&body); + format!("{}|{}|{}|{}", error, detail, location, suggestion) + }; + + // A plain error, as before location and suggestion existed: both parts + // must be empty so the rendered message is unchanged. + assert_eq!( + case(serde_json::json!({"error": "Value doesn't exist."})), + "Value doesn't exist.|||" + ); + + // A validation error: an RFC 6901 pointer at the member at fault. + assert_eq!( + case(serde_json::json!({ + "error": "Invalid configuration.", + "detail": "Unknown parameter \"pas\".", + "location": {"path": "/routes/0/action"}, + "suggestion": "pass" + })), + "Invalid configuration.|Unknown parameter \"pas\".|\n Location: /routes/0/action|\n Did you mean: pass" + ); + + // The empty pointer is the document root, not a missing value. + assert_eq!( + case(serde_json::json!({"error": "Invalid configuration.", + "location": {"path": ""}})), + "Invalid configuration.||\n Location: the document root|" + ); + + // An error found while parsing places itself by position: no pointer. + assert_eq!( + case(serde_json::json!({"error": "Invalid JSON.", + "location": {"offset": 12, "line": 2, "column": 5}})), + "Invalid JSON.||\n Location: line 2, column 5|" + ); + + // Offset without line/column, which the controller sends when the + // position is known but not the line. + assert_eq!( + case(serde_json::json!({"error": "Invalid JSON.", "location": {"offset": 12}})), + "Invalid JSON.||\n Location: byte offset 12|" + ); + + // Nothing usable in location, and a body that is not an object at all: + // neither may panic or invent a location. + assert_eq!( + case(serde_json::json!({"error": "Invalid JSON.", "location": {}})), + "Invalid JSON.|||" + ); + assert_eq!(case(serde_json::json!("not an object")), "Unknown error|||"); + } + + /// A 200 is no guarantee that Unit answered: a proxy or a captive portal can + /// return one with a page of its own. The parser's complaint about that page + /// says nothing about it, so the page is shown as well -- bounded and with + /// its control characters written out, since it is a stranger's text. + #[test] + fn an_unparseable_success_body_is_shown_not_just_complained_about() { + let page = b"captive portal: sign in at http://portal.example/\x1b[2J"; + let source = serde_json::from_slice::(page).expect_err("not JSON"); + + let error = UnitClientError::UnparseableBodyError { + path: "/config".to_string(), + source, + body: json_body::body_for_display(page), + }; + let rendered = error.to_string(); + + assert!(rendered.contains("/config"), "{}", rendered); + assert!( + rendered.contains("captive portal: sign in"), + "the page must be shown: {}", + rendered + ); + assert!( + rendered.contains("expected value"), + "the parser's reason stays: {}", + rendered + ); + assert!( + !rendered.contains('\u{1b}'), + "an escape reached the message: {:?}", + rendered + ); + assert!( + rendered.contains("\\u001b[2J"), + "the escape is written out: {}", + rendered + ); + } + + /// The error that carries such a body must carry it as shown: the body is + /// embedded in the message the user reads. + #[test] + fn the_http_response_error_shows_the_body_it_was_given() { + let error = UnitClientError::HttpResponseError { + status: http::StatusCode::BAD_GATEWAY, + path: "/config".to_string(), + body: json_body::body_for_display(&[0x1b, b'[', b'2', b'J']), + }; + let rendered = error.to_string(); + assert!(rendered.contains("/config")); + assert!(rendered.contains("502")); + assert!( + !rendered.contains('\u{1b}'), + "an escape reached the message: {}", + rendered + ); + } + // Integration tests #[tokio::test] diff --git a/tools/unitctl/unit-openapi/Cargo.toml b/tools/unitctl/unit-openapi/Cargo.toml index b529a5e24..d371a86f3 100644 --- a/tools/unitctl/unit-openapi/Cargo.toml +++ b/tools/unitctl/unit-openapi/Cargo.toml @@ -1,10 +1,11 @@ [package] name = "unit-openapi" -version = "1.36.0" +version = "1.37.0" authors = ["team@freeunit.org"] description = "FreeUnit (ex NGINX Unit) is a lightweight and versatile application runtime that provides the essential components for your web application as a single open-source server: running application code, serving static assets, handling TLS and request routing. **Important**: Unit's API is designed to expose any part of its configuration as an addressable endpoint. Suppose a JSON object is stored at `/config/listeners/`: ```json { \"*:8080\": { \"pass\": \"applications/wp_emea_dev\" } } ``` Here, `/config/listeners/_*:8080` and `/config/listeners/_*:8080/pass` are also endpoints. Generally, object options are addressable by their names, array items—by their indexes (`/array/0/`). **Note**: By default, Unit is configured through a UNIX domain socket. To use this specification with OpenAPI tools interactively, start Unit with a TCP port as the control socket." license = "Apache 2.0" edition = "2018" +rust-version = "1.86" [dependencies] serde = { version = "1.0", features = ["derive"] } diff --git a/tools/unitctl/unit-openapi/README.md b/tools/unitctl/unit-openapi/README.md index bb576e910..ba1f1a1f0 100644 --- a/tools/unitctl/unit-openapi/README.md +++ b/tools/unitctl/unit-openapi/README.md @@ -21,7 +21,7 @@ For more information, please visit [https://github.com/freeunitorg/freeunit](htt This API client was generated by the [OpenAPI Generator](https://openapi-generator.tech) project. By using the [openapi-spec](https://openapis.org) from a remote server, you can easily generate an API client. - API version: 0.2.0 -- Package version: 1.36.0 +- Package version: 1.37.0 - Generator version: 7.6.0 - Build package: `org.openapitools.codegen.languages.RustClientCodegen` @@ -364,6 +364,10 @@ Class | Method | HTTP request | Description *StatusApi* | [**get_status_modules_lang_version**](docs/StatusApi.md#get_status_modules_lang_version) | **Get** /status/modules/{langMod}/version | Retrieve the language module version object *StatusApi* | [**get_status_requests**](docs/StatusApi.md#get_status_requests) | **Get** /status/requests | Retrieve the requests status object *StatusApi* | [**get_status_requests_total**](docs/StatusApi.md#get_status_requests_total) | **Get** /status/requests/total | Retrieve the total requests number +*StatusApi* | [**get_status_telemetry**](docs/StatusApi.md#get_status_telemetry) | **Get** /status/telemetry | Retrieve the telemetry status object +*StatusApi* | [**get_status_telemetry_spans**](docs/StatusApi.md#get_status_telemetry_spans) | **Get** /status/telemetry/spans | Retrieve the telemetry spans object +*StatusApi* | [**get_status_telemetry_spans_exported**](docs/StatusApi.md#get_status_telemetry_spans_exported) | **Get** /status/telemetry/spans/exported | Retrieve the exported spans number +*StatusApi* | [**get_status_telemetry_spans_failed**](docs/StatusApi.md#get_status_telemetry_spans_failed) | **Get** /status/telemetry/spans/failed | Retrieve the failed spans number *TlsApi* | [**delete_listener_tls**](docs/TlsApi.md#delete_listener_tls) | **Delete** /config/listeners/{listenerName}/tls | Delete the tls object in a listener *TlsApi* | [**delete_listener_tls_certificate**](docs/TlsApi.md#delete_listener_tls_certificate) | **Delete** /config/listeners/{listenerName}/tls/certificate/{arrayIndex} | Delete a certificate array item in a listener *TlsApi* | [**delete_listener_tls_certificates**](docs/TlsApi.md#delete_listener_tls_certificates) | **Delete** /config/listeners/{listenerName}/tls/certificate | Delete the certificate option in a listener @@ -465,6 +469,8 @@ Class | Method | HTTP request | Description - [ConfigSettingsHttpStatic](docs/ConfigSettingsHttpStatic.md) - [ConfigSettingsHttpStaticMimeType](docs/ConfigSettingsHttpStaticMimeType.md) - [ConfigSettingsTelemetry](docs/ConfigSettingsTelemetry.md) + - [ErrorLocation](docs/ErrorLocation.md) + - [JsonErrorMessage](docs/JsonErrorMessage.md) - [Status](docs/Status.md) - [StatusApplicationsApp](docs/StatusApplicationsApp.md) - [StatusApplicationsAppProcesses](docs/StatusApplicationsAppProcesses.md) @@ -472,6 +478,8 @@ Class | Method | HTTP request | Description - [StatusConnections](docs/StatusConnections.md) - [StatusModulesLang](docs/StatusModulesLang.md) - [StatusRequests](docs/StatusRequests.md) + - [StatusTelemetry](docs/StatusTelemetry.md) + - [StatusTelemetrySpans](docs/StatusTelemetrySpans.md) - [StringOrStringArray](docs/StringOrStringArray.md) diff --git a/tools/unitctl/unit-openapi/openapi-templates/Cargo.mustache b/tools/unitctl/unit-openapi/openapi-templates/Cargo.mustache index c95a9625e..864f60084 100644 --- a/tools/unitctl/unit-openapi/openapi-templates/Cargo.mustache +++ b/tools/unitctl/unit-openapi/openapi-templates/Cargo.mustache @@ -18,6 +18,7 @@ license = "{{.}}" license = "Unlicense" {{/licenseInfo}} edition = "2018" +rust-version = "1.86" {{#publishRustRegistry}} publish = ["{{.}}"] {{/publishRustRegistry}} diff --git a/tools/unitctl/unitctl/Cargo.toml b/tools/unitctl/unitctl/Cargo.toml index 676992e9c..b4048b5ca 100644 --- a/tools/unitctl/unitctl/Cargo.toml +++ b/tools/unitctl/unitctl/Cargo.toml @@ -1,9 +1,10 @@ [package] name = "unitctl" description = "CLI interface to the FreeUnit Control API" -version = "1.36.0" +version = "1.37.0" authors = ["Elijah Zupancic"] edition = "2021" +rust-version = "1.95" license = "Apache-2.0" [[bin]] @@ -11,23 +12,26 @@ name = "unitctl" path = "src/main.rs" [features] +default = ["tls"] +# "https://" control sockets. The musl build passes --no-default-features. +tls = ["unit-client-rs/tls"] [dependencies] clap = { version = "4.4", features = ["default", "derive", "cargo"] } custom_error = "1.9" serde = "1.0" -json5 = "1.3.1" -nu-json = "0.113.0" serde_json = { version = "1.0", optional = false } -serde_yaml = "0.9" -rustls-pemfile = "2.0.0" -unit-client-rs = { path = "../unit-client-rs" } +# The "std" feature is not on by default and pem::from_buf needs it. +rustls-pki-types = { version = "1.15", features = ["std"] } +# default-features = false, or unitctl's own --no-default-features would not +# reach unit-client-rs and TLS would stay on. +unit-client-rs = { path = "../unit-client-rs", default-features = false } colored_json = "5.0.0" tempfile = "3.8" which = "8.0.2" walkdir = "2.4" -hyper = { version = "1.9.0", features = ["http2", "server", "client"] } +hyper = { version = "1.9.0", features = ["server", "client"] } http-body-util = "0.1" bytes = "1" tokio = { version = "1.35", features = ["macros", "rt-multi-thread"] } diff --git a/tools/unitctl/unitctl/src/cmd/edit.rs b/tools/unitctl/unitctl/src/cmd/edit.rs index 64f2806d7..7ae946865 100644 --- a/tools/unitctl/unitctl/src/cmd/edit.rs +++ b/tools/unitctl/unitctl/src/cmd/edit.rs @@ -1,10 +1,11 @@ use crate::inputfile::{InputFile, InputFormat}; -use crate::requests::{send_and_validate_config_deserialize_response, send_empty_body_deserialize_response}; +use crate::requests::{send_config_deserialize_response, send_empty_body_read_bytes}; use crate::unitctl::UnitCtl; use crate::unitctl_error::ControlSocketErrorKind; use crate::{wait, OutputFormat, UnitctlError}; use std::path::{Path, PathBuf}; -use unit_client_rs::unit_client::UnitClient; +use unit_client_rs::json_body; +use unit_client_rs::unit_client::{UnitClient, UnitSerializableMap}; use which::which; const EDITOR_ENV_VARS: [&str; 2] = ["EDITOR", "VISUAL"]; @@ -20,7 +21,11 @@ const EDITOR_KNOWN_LIST: [&str; 8] = [ ]; pub(crate) async fn cmd(cli: &UnitCtl, output_format: OutputFormat) -> Result<(), UnitctlError> { - if cli.control_socket_addresses.as_ref().map_or(false, |addrs| addrs.len() > 1) { + if cli + .control_socket_addresses + .as_ref() + .map_or(false, |addrs| addrs.len() > 1) + { return Err(UnitctlError::ControlSocketError { kind: ControlSocketErrorKind::General, message: "too many control sockets. specify at most one.".to_string(), @@ -29,8 +34,36 @@ pub(crate) async fn cmd(cli: &UnitCtl, output_format: OutputFormat) -> Result<() let mut control_sockets = wait::wait_for_sockets(cli).await?; let client = UnitClient::new(control_sockets.pop().unwrap()); - // Get latest configuration - let current_config = send_empty_body_deserialize_response(&client, "GET", "/config").await?; + // Get latest configuration. Editing means handing it back, and the way + // back runs the file through serde_json, whose strings are Rust strings: + // bytes Unit stores but UTF-8 cannot carry would be replaced by the mere + // act of opening the editor. Refuse rather than rewrite a configuration + // the user only meant to look at. + // UnitSerializableMap and not serde_json::Value: a configuration is a JSON + // object, and the type is what keeps a 200 carrying "null", an array or a + // bare string -- an intermediary talking -- from being opened in the editor + // and handed back as the whole configuration. + let raw_config = send_empty_body_read_bytes(&client, "GET", "/config").await?; + let current_config: UnitSerializableMap = match json_body::decode(&raw_config) { + Ok((config, fidelity)) if fidelity.is_exact() => config, + Ok((_, fidelity)) => { + return Err(UnitctlError::UndecodableConfiguration { + path: "/config".to_string(), + members: fidelity.members(), + values: fidelity.replaced_values(), + names: fidelity.replaced_names(), + }) + } + // Name the path, as the JsonError this replaced did: "expected value at + // line 1 column 1" on its own does not say what was being read -- and + // show the body, because a 200 carrying an intermediary's page is the + // ordinary way this arm is reached. + Err(error) => { + return Err(UnitctlError::DeserializationError { + message: format!("/config: {}\n{}", error, json_body::body_for_display(&raw_config)), + }) + } + }; // Write JSON to temporary file - this file will automatically be deleted by the OS when // the last file handle to it is removed. @@ -48,7 +81,11 @@ pub(crate) async fn cmd(cli: &UnitCtl, output_format: OutputFormat) -> Result<() let temp_file_path = temp_file.path(); let before_edit_mod_time = temp_file_path.metadata().ok().map(|m| m.modified().ok()); - let inputfile = InputFile::FileWithFormat(temp_file_path.into(), InputFormat::Json5); + // The temporary file holds what Unit returned, pretty-printed above, and + // whatever the editor leaves in it is sent as it is. Reading it as JSON5 + // would let a comment be written here, but it would also parse and + // re-serialize the file, which is what this path no longer does. + let inputfile = InputFile::FileWithFormat(temp_file_path.into(), InputFormat::Json); open_editor(temp_file_path)?; let after_edit_mod_time = temp_file_path.metadata().ok().map(|m| m.modified().ok()); @@ -61,7 +98,7 @@ pub(crate) async fn cmd(cli: &UnitCtl, output_format: OutputFormat) -> Result<() }; // Send edited file to Unit to overwrite current configuration - send_and_validate_config_deserialize_response(&client, "PUT", "/config", Some(&inputfile)) + send_config_deserialize_response(&client, "PUT", "/config", &inputfile) .await .and_then(|status| output_format.write_to_stdout(&status)) } diff --git a/tools/unitctl/unitctl/src/cmd/execute.rs b/tools/unitctl/unitctl/src/cmd/execute.rs index 3a4a8b451..02438310a 100644 --- a/tools/unitctl/unitctl/src/cmd/execute.rs +++ b/tools/unitctl/unitctl/src/cmd/execute.rs @@ -1,6 +1,6 @@ use crate::inputfile::InputFile; use crate::requests::{ - send_and_validate_config_deserialize_response, send_and_validate_pem_data_deserialize_response, + send_and_validate_pem_data_deserialize_response, send_config_deserialize_response, send_body_deserialize_response, send_empty_body_deserialize_response, }; use crate::unitctl::UnitCtl; @@ -71,13 +71,21 @@ async fn send_and_deserialize( match input_file { Some(input_file) => { if input_file.is_config() { - send_and_validate_config_deserialize_response(&client, method.as_str(), path, Some(&input_file)).await + send_config_deserialize_response(&client, method.as_str(), path, &input_file).await // TLS certificate data } else if input_file.is_pem_bundle() { send_and_validate_pem_data_deserialize_response(&client, method.as_str(), path, &input_file).await - // This is unknown data + // A JS module: the bytes go up as they are, the way "import" sends them + } else if input_file.is_javascript() { + send_body_deserialize_response(&client, method.as_str(), path, Some(&input_file)).await + // A file this command cannot send anywhere } else { - panic!("Unknown input file type") + Err(UnitctlError::UnknownInputFileType { + path: input_file.to_path().map_or_else( + |_| "-".to_string(), + |path| path.to_string_lossy().into_owned(), + ), + }) } } // A none value for an input file can be considered a request to send an empty body diff --git a/tools/unitctl/unitctl/src/cmd/import.rs b/tools/unitctl/unitctl/src/cmd/import.rs index 956832f32..7106dc3be 100644 --- a/tools/unitctl/unitctl/src/cmd/import.rs +++ b/tools/unitctl/unitctl/src/cmd/import.rs @@ -12,16 +12,22 @@ enum UploadFormat { Javascript, } -impl From<&InputFile> for UploadFormat { - fn from(input_file: &InputFile) -> Self { +impl TryFrom<&InputFile> for UploadFormat { + type Error = UnitctlError; + + /// A file this command cannot place is refused, not fatal. This used to + /// panic on a format that reached it without a branch here. + fn try_from(input_file: &InputFile) -> Result { if input_file.is_config() { - UploadFormat::Config + Ok(UploadFormat::Config) } else if input_file.is_pem_bundle() { - UploadFormat::PemBundle + Ok(UploadFormat::PemBundle) } else if input_file.is_javascript() { - UploadFormat::Javascript + Ok(UploadFormat::Javascript) } else { - panic!("Unknown input file type"); + Err(UnitctlError::UnknownInputFileType { + path: input_file.to_path()?.to_string_lossy().into(), + }) } } } @@ -88,7 +94,7 @@ async fn process_entry(entry: DirEntry, client: &UnitClient) -> Result<(), Unitc path: input_file.to_path()?.to_string_lossy().into(), }); } - let upload_format = UploadFormat::from(&input_file); + let upload_format = UploadFormat::try_from(&input_file)?; let upload_path = upload_format.upload_path(entry.path()); // We can't overwrite JS or PEM files, so we delete them first @@ -100,13 +106,7 @@ async fn process_entry(entry: DirEntry, client: &UnitClient) -> Result<(), Unitc let result = match upload_format { UploadFormat::Config => { - requests::send_and_validate_config_deserialize_response( - client, - "PUT", - upload_path.as_str(), - Some(&input_file), - ) - .await + requests::send_config_deserialize_response(client, "PUT", upload_path.as_str(), &input_file).await } UploadFormat::PemBundle => { requests::send_and_validate_pem_data_deserialize_response(client, "PUT", upload_path.as_str(), &input_file) @@ -138,3 +138,45 @@ async fn process_entry(entry: DirEntry, client: &UnitClient) -> Result<(), Unitc } } } + +#[cfg(test)] +mod tests { + use super::*; + + fn input(name: &str) -> InputFile { + InputFile::from(Path::new(name)) + } + + /// Each format this command can place has its own destination, and the + /// ".js" case is the one `execute` used to abort on. + #[test] + fn every_known_format_has_an_upload_path() { + let cases = [ + ("conf.json", "/config"), + ("bundle.pem", "/certificates/bundle.pem"), + ("mod.js", "/js_modules/mod.js"), + ]; + + for (name, expected) in cases { + let file = input(name); + let format = UploadFormat::try_from(&file).expect("a known format"); + + assert_eq!(format.upload_path(Path::new(name)), expected); + } + } + + /// A file this command cannot place is refused by name. It used to + /// panic, which gave the operator a backtrace instead of the file name. + #[test] + fn an_unplaceable_file_is_refused_not_fatal() { + let Err(error) = UploadFormat::try_from(&input("notes.txt")) else { + panic!("an unplaceable file must be refused"); + }; + + assert!( + matches!(error, UnitctlError::UnknownInputFileType { .. }), + "unexpected error: {}", + error + ); + } +} diff --git a/tools/unitctl/unitctl/src/cmd/save.rs b/tools/unitctl/unitctl/src/cmd/save.rs index 075f0cbe2..6ee89b4cb 100644 --- a/tools/unitctl/unitctl/src/cmd/save.rs +++ b/tools/unitctl/unitctl/src/cmd/save.rs @@ -1,4 +1,4 @@ -use crate::requests::send_empty_body_deserialize_response; +use crate::requests::send_empty_body_read_bytes; use crate::unitctl::UnitCtl; use crate::unitctl_error::ControlSocketErrorKind; use crate::wait; @@ -6,10 +6,72 @@ use crate::UnitctlError; use std::fs::File; use std::io::stdout; use tar::{Builder, Header}; -use unit_client_rs::unit_client::UnitClient; +use unit_client_rs::json_body; +use unit_client_rs::unit_client::{UnitClient, UnitSerializableMap}; + +/// Pretty-print the saved document, and keep the server's own bytes when a Rust +/// string cannot carry them. +/// +/// A configuration can hold bytes that are not valid UTF-8, and re-encoding +/// those through `serde_json` would replace them. A backup that differs from +/// the configuration it backs up is worse than an unformatted one. +/// +/// Whichever way the body is read, it has to be a configuration to be stored as +/// one, and `UnitSerializableMap` is what says so: a configuration is a JSON +/// object, and `null`, an array or a bare string answered with a success status +/// is an intermediary talking, not a configuration. Neither is a truncated +/// answer, and either can arrive with bytes that are not valid UTF-8 -- a +/// Latin-1 error page does -- or without, so both arms ask the same question. +/// Only storing differs: bytes a Rust string can carry are pretty-printed, and +/// bytes it cannot are written exactly as they arrived. +fn prettify_or_keep(raw: &[u8], path: &str) -> Result, UnitctlError> { + // Show the body along with the parser's complaint: "expected value at line 1 + // column 1" says nothing about the captive portal's sign-in page that + // provoked it, and the page is what tells the operator what happened. + // json_body::body_for_display bounds it and takes its control characters out. + let refuse = |error: serde_json::Error| UnitctlError::DeserializationError { + message: format!("{}: {}\n{}", path, error, json_body::body_for_display(raw)), + }; + + match std::str::from_utf8(raw) { + Ok(text) => serde_json::from_str::(text) + .and_then(|config| serde_json::to_string_pretty(&config)) + .map(String::into_bytes) + .map_err(refuse), + Err(_) => { + // The lossy view answers whether this is a configuration without + // being what gets written: the bytes stored below are the server's + // own. Replacing a byte cannot turn valid JSON into invalid JSON, + // nor an object into something else, so a document that passes this + // way passed before the replacement too. + serde_json::from_str::(&String::from_utf8_lossy(raw)).map_err(refuse)?; + + eprintln!( + "Warning: {} holds bytes that are not valid UTF-8; saving what the server sent, verbatim", + path + ); + // Not a route around it: curl reaches the same control API, which + // refuses a configuration holding these bytes on POST and on PUT + // alike (nxt_conf_validate_encoding() in nxt_controller.c, before + // the schema is even looked at). A value like this only survives + // because it was loaded from the state file, and the server names + // it in the warning it logged then. + eprintln!( + "Warning: the control API refuses a configuration holding these bytes, so this \ + file cannot be sent back by any route; correct the value the server named in \ + its warning, then import" + ); + Ok(raw.to_vec()) + } + } +} pub async fn cmd(cli: &UnitCtl, filename: &String) -> Result<(), UnitctlError> { - if cli.control_socket_addresses.as_ref().map_or(false, |addrs| addrs.len() > 1) { + if cli + .control_socket_addresses + .as_ref() + .map_or(false, |addrs| addrs.len() > 1) + { return Err(UnitctlError::ControlSocketError { kind: ControlSocketErrorKind::General, message: "too many control sockets. specify at most one.".to_string(), @@ -23,13 +85,7 @@ pub async fn cmd(cli: &UnitCtl, filename: &String) -> Result<(), UnitctlError> { eprintln!("Warning: writing uncompressed tarball to {}", filename); } - let config_res = - serde_json::to_string_pretty(&send_empty_body_deserialize_response(&client, "GET", "/config").await?); - if let Err(e) = config_res { - return Err(UnitctlError::DeserializationError { message: e.to_string() }); - } - - let current_config = config_res.unwrap().into_bytes(); + let current_config = prettify_or_keep(&send_empty_body_read_bytes(&client, "GET", "/config").await?, "/config")?; //let current_js_modules = send_empty_body_deserialize_response(&client, "GET", "/js_modules") // .await?; @@ -53,3 +109,148 @@ pub async fn cmd(cli: &UnitCtl, filename: &String) -> Result<(), UnitctlError> { Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_configuration_is_pretty_printed() { + let saved = prettify_or_keep(br#"{"listeners":{"*:8080":{"pass":"routes"}}}"#, "/config") + .expect("a JSON body must be saved"); + let text = String::from_utf8(saved).expect("a UTF-8 body stays UTF-8"); + assert!(text.starts_with("{\n"), "expected pretty output, got {:?}", text); + assert!(text.contains("\"*:8080\"")); + } + + /// The one case that keeps the server's bytes: no Rust string can carry + /// them, so re-encoding would change the configuration being backed up. + #[test] + fn bytes_that_are_not_utf8_are_kept_verbatim() { + let mut raw = br#"{"routes":[{"match":{"uri":"/"#.to_vec(); + raw.push(0xff); + raw.extend_from_slice(br#""},"action":{"return":204}}]}"#); + + let saved = prettify_or_keep(&raw, "/config").expect("a legacy body must still be saved"); + assert_eq!(saved, raw, "the backup must be the bytes the server sent"); + } + + /// A 200 carrying something that is not a configuration must not be written + /// into the tarball as if it were one. + #[test] + fn a_valid_utf8_body_that_is_not_json_fails() { + let error = prettify_or_keep(b"

502 Bad Gateway

", "/config") + .expect_err("a non-JSON body is not a configuration"); + match error { + UnitctlError::DeserializationError { message } => { + assert!( + message.contains("/config"), + "the message must name the path: {}", + message + ) + } + other => panic!("expected a deserialization error, got {:?}", other), + } + + // Truncated, which is the same failure for a different reason. + assert!(prettify_or_keep(br#"{"listeners":"#, "/config").is_err()); + } + + /// Being unreadable as UTF-8 is not a pass either. A proxy's Latin-1 error + /// page fails both tests at once, and it must fail on the one that matters: + /// it is not a configuration. + #[test] + fn a_body_that_is_neither_utf8_nor_json_fails() { + // "Erreur de passerelle" with the accents written the Latin-1 way, which + // is not valid UTF-8. + let mut page = b"

502 Erreur de passerell".to_vec(); + page.extend_from_slice(&[0xe9, 0xe8]); + page.extend_from_slice(b"

"); + assert!(std::str::from_utf8(&page).is_err(), "the page must not be valid UTF-8"); + + let error = prettify_or_keep(&page, "/config").expect_err("a Latin-1 error page is not a configuration"); + match error { + UnitctlError::DeserializationError { message } => { + assert!( + message.contains("/config"), + "the message must name the path: {}", + message + ) + } + other => panic!("expected a deserialization error, got {:?}", other), + } + + // A document cut short mid-string, holding an undecodable byte as well. + let mut truncated = br#"{"routes":[{"match":{"uri":"/"#.to_vec(); + truncated.push(0xff); + assert!(prettify_or_keep(&truncated, "/config").is_err()); + } + + /// Valid JSON is not enough either: a configuration is an object, and a 200 + /// carrying `null` or an array is an intermediary talking. Archiving one + /// would write a `config.json` that cannot be a configuration, and `edit` + /// would offer it for editing and then PUT it back as the whole thing. + #[test] + fn a_body_that_is_not_an_object_fails() { + for body in [&b"null"[..], b"[]", b"[{\"listeners\":{}}]", b"\"a string\"", b"204"] { + let error = prettify_or_keep(body, "/config").expect_err("only an object is a configuration"); + match error { + UnitctlError::DeserializationError { message } => assert!( + message.contains("/config"), + "the message must name the path: {}", + message + ), + other => panic!("expected a deserialization error, got {:?}", other), + } + } + } + + /// The same question on the other arm: being unreadable as UTF-8 must not + /// let a non-object through to the tarball. + #[test] + fn a_non_object_holding_an_undecodable_byte_fails() { + let mut array = br#"["caf"#.to_vec(); + array.push(0xff); + array.extend_from_slice(br#""]"#); + assert!(std::str::from_utf8(&array).is_err(), "the body must not be valid UTF-8"); + assert!(serde_json::from_str::(&String::from_utf8_lossy(&array)).is_ok()); + + assert!( + prettify_or_keep(&array, "/config").is_err(), + "an array is not a configuration, whatever its bytes" + ); + } + + /// The page is what tells the operator what happened; the parser's complaint + /// about its first character does not. Both are shown. + #[test] + fn a_refusal_shows_the_body_it_refused() { + let page = b"captive portal: sign in at http://portal.example/"; + let message = prettify_or_keep(page, "/config") + .expect_err("not a configuration") + .to_string(); + + assert!(message.contains("/config"), "{}", message); + assert!(message.contains("expected value"), "the reason stays: {}", message); + assert!( + message.contains("captive portal: sign in at http://portal.example/"), + "the page must be shown: {}", + message + ); + + // The same holds on the other arm, where the body is not valid UTF-8: + // a Latin-1 page is shown too, with the undecodable bytes as U+FFFD. + let mut latin1 = b"

502 Erreur de passerell".to_vec(); + latin1.extend_from_slice(&[0xe9, 0xe8]); + latin1.extend_from_slice(b"

"); + let message = prettify_or_keep(&latin1, "/config") + .expect_err("not a configuration") + .to_string(); + assert!(message.contains("502 Erreur de passerell"), "{}", message); + assert!( + message.contains('\u{fffd}'), + "undecodable bytes are shown as U+FFFD: {}", + message + ); + } +} diff --git a/tools/unitctl/unitctl/src/inputfile.rs b/tools/unitctl/unitctl/src/inputfile.rs index b2479d505..e0cffaa50 100644 --- a/tools/unitctl/unitctl/src/inputfile.rs +++ b/tools/unitctl/unitctl/src/inputfile.rs @@ -1,4 +1,3 @@ -use std::collections::HashMap; use std::io; use std::io::{BufRead, BufReader, Error as IoError, Read}; use std::path::{Path, PathBuf}; @@ -6,15 +5,20 @@ use std::path::{Path, PathBuf}; use crate::known_size::KnownSize; use clap::ValueEnum; -use super::UnitSerializableMap; use super::UnitctlError; /// Input file data format #[derive(ValueEnum, Copy, Clone, Debug, PartialEq, Eq)] pub enum InputFormat { + /// Recognised, but not parsed. unitctl no longer reads YAML. The format + /// stays in this list so that a ".yaml" file gets a message saying that, + /// instead of a JSON parse error. Yaml, Json, Json5, + /// Recognised, but not parsed. unitctl no longer reads hjson. The format + /// stays in this list so that an ".hjson" file gets a message saying that, + /// instead of a JSON parse error. Hjson, Pem, JavaScript, @@ -52,7 +56,7 @@ impl InputFormat { .split_once('/') .map_or(lead_slash_removed, |(first, _)| first); match first_path { - "config" => InputFormat::Hjson, + "config" => InputFormat::Json, "certificates" => InputFormat::Pem, "js_modules" => InputFormat::JavaScript, _ => InputFormat::Json, @@ -139,44 +143,38 @@ impl InputFile { } } - /// Converts a HJSON Value type to a JSON Value type - fn hjson_value_to_json_value(value: nu_json::Value) -> serde_json::Value { - serde_json::to_value(value).expect("Failed to convert HJSON value to JSON value") - } + /// The media type and the bytes to PUT for a configuration input. + /// + /// JSON is sent as it was written. unitctl does not parse it, so an + /// operator's duplicate member reaches the server, which refuses it + /// (src/nxt_conf.c:1616), and a number keeps the spelling the file used. + /// Parsing here collapsed both silently. + pub fn to_config_body(&self) -> Result<(String, KnownSize), UnitctlError> { + let json = "application/json".to_string(); - pub fn to_unit_serializable_map(&self) -> Result { - let reader: Box = self.try_into()?; - let body_data: UnitSerializableMap = match self.format() { - InputFormat::Yaml => serde_yaml::from_reader(reader) - .map_err(|e| UnitctlError::DeserializationError { message: e.to_string() })?, - InputFormat::Json => serde_json::from_reader(reader) - .map_err(|e| UnitctlError::DeserializationError { message: e.to_string() })?, - InputFormat::Json5 => { - let mut reader = BufReader::new(reader); - let mut json5_string: String = String::new(); - reader - .read_to_string(&mut json5_string) - .map_err(|e| UnitctlError::DeserializationError { message: e.to_string() })?; - json5::from_str(&json5_string) - .map_err(|e| UnitctlError::DeserializationError { message: e.to_string() })? - } - InputFormat::Hjson => { - let hjson_value: HashMap = nu_json::from_reader(reader) - .map_err(|e| UnitctlError::DeserializationError { message: e.to_string() })?; - - hjson_value - .iter() - .map(|(k, v)| { - let json_value = Self::hjson_value_to_json_value(v.clone()); - (k.clone(), json_value) - }) - .collect() - } + match self.format() { + InputFormat::Json => Ok((json, self.try_into()?)), + // Refuse the formats Unit cannot read by name. Sending such a + // file as it is would make the server report a syntax error on + // the first comment or unquoted key, and that error does not tell + // the user what to do. + InputFormat::Hjson => Err(UnitctlError::DeserializationError { + message: "hjson is no longer supported: convert the file to JSON first".to_string(), + }), + InputFormat::Json5 => Err(UnitctlError::DeserializationError { + message: "JSON5 is no longer supported: convert the file to JSON first, for example \ + with \"json5 -o config.json config.json5\"" + .to_string(), + }), + InputFormat::Yaml => Err(UnitctlError::DeserializationError { + message: "YAML is no longer supported: convert the file to JSON first, for example with \ + \"yq -o=json\"" + .to_string(), + }), _ => Err(UnitctlError::DeserializationError { - message: format!("Unsupported input format for serialization: {:?}", self), - })?, - }; - Ok(body_data) + message: format!("Unsupported input format for a configuration: {:?}", self), + }), + } } } @@ -243,6 +241,7 @@ impl TryInto for &InputFile { #[cfg(test)] mod tests { use super::*; + use crate::UnitSerializableMap; #[test] fn can_parse_file_extensions() { @@ -250,6 +249,8 @@ mod tests { assert_eq!(InputFormat::from_file_extension("yml"), InputFormat::Yaml); assert_eq!(InputFormat::from_file_extension("json"), InputFormat::Json); assert_eq!(InputFormat::from_file_extension("json5"), InputFormat::Json5); + assert_eq!(InputFormat::from_file_extension("hjson"), InputFormat::Hjson); + assert_eq!(InputFormat::from_file_extension("cjson"), InputFormat::Hjson); assert_eq!(InputFormat::from_file_extension("pem"), InputFormat::Pem); assert_eq!(InputFormat::from_file_extension("js"), InputFormat::JavaScript); assert_eq!(InputFormat::from_file_extension("njs"), InputFormat::JavaScript); @@ -258,14 +259,14 @@ mod tests { #[test] fn can_parse_remote_paths() { - assert_eq!(InputFormat::from_remote_path("//config"), InputFormat::Hjson); - assert_eq!(InputFormat::from_remote_path("/config"), InputFormat::Hjson); - assert_eq!(InputFormat::from_remote_path("/config/"), InputFormat::Hjson); - assert_eq!(InputFormat::from_remote_path("config/"), InputFormat::Hjson); - assert_eq!(InputFormat::from_remote_path("config"), InputFormat::Hjson); - assert_eq!(InputFormat::from_remote_path("/config/something/"), InputFormat::Hjson); - assert_eq!(InputFormat::from_remote_path("config/something/"), InputFormat::Hjson); - assert_eq!(InputFormat::from_remote_path("config/something"), InputFormat::Hjson); + assert_eq!(InputFormat::from_remote_path("//config"), InputFormat::Json); + assert_eq!(InputFormat::from_remote_path("/config"), InputFormat::Json); + assert_eq!(InputFormat::from_remote_path("/config/"), InputFormat::Json); + assert_eq!(InputFormat::from_remote_path("config/"), InputFormat::Json); + assert_eq!(InputFormat::from_remote_path("config"), InputFormat::Json); + assert_eq!(InputFormat::from_remote_path("/config/something/"), InputFormat::Json); + assert_eq!(InputFormat::from_remote_path("config/something/"), InputFormat::Json); + assert_eq!(InputFormat::from_remote_path("config/something"), InputFormat::Json); assert_eq!(InputFormat::from_remote_path("/certificates"), InputFormat::Pem); assert_eq!(InputFormat::from_remote_path("/certificates/"), InputFormat::Pem); assert_eq!(InputFormat::from_remote_path("certificates/"), InputFormat::Pem); @@ -286,4 +287,180 @@ mod tests { InputFormat::Pem ); } + + /// Member names in an order no map sorts into by accident, so a map that + /// does not keep insertion order cannot pass by luck. + const MEMBERS: [&str; 6] = [ + "settings", + "listeners", + "routes", + "applications", + "access_log", + "upstreams", + ]; + + fn write_input(body: &str, suffix: &str) -> tempfile::NamedTempFile { + let mut file = tempfile::Builder::new() + .prefix("unitctl-inputfile-") + .suffix(suffix) + .tempfile() + .expect("a temporary file"); + std::io::Write::write_all(file.as_file_mut(), body.as_bytes()).expect("writing the input"); + file + } + + /// Read a body back without going through `into_full_body`, so the check + /// does not depend on the same code it is checking. + fn body_bytes(known_size: KnownSize) -> Vec { + match known_size { + KnownSize::Vec(bytes) => bytes, + KnownSize::String(text) => text.into_bytes(), + KnownSize::Read(mut reader, _) => { + let mut bytes = Vec::new(); + reader.read_to_end(&mut bytes).expect("reading the body"); + bytes + } + KnownSize::Empty => Vec::new(), + } + } + + fn config_body(file: &tempfile::NamedTempFile, format: InputFormat) -> Vec { + let (mime_type, body) = InputFile::FileWithFormat(file.path().into(), format) + .to_config_body() + .expect("the input must produce a body"); + + // Every configuration body is JSON by the time it leaves, whatever the + // file it came from. + assert_eq!(mime_type, "application/json"); + + body_bytes(body) + } + + fn members_of(file: &tempfile::NamedTempFile, format: InputFormat) -> Vec { + let body = config_body(file, format); + let map: UnitSerializableMap = serde_json::from_slice(&body).expect("the body must be JSON"); + map.keys().cloned().collect() + } + + /// The body Unit receives keeps member order. JSON is sent as it was + /// written, so it can only keep it. The check stays because the response + /// side still parses into `UnitSerializableMap`, which keeps order only + /// while `serde_json`'s "preserve_order" feature is on. + #[test] + fn a_json_body_keeps_member_order() { + let json = MEMBERS + .iter() + .map(|name| format!(" \"{}\": {{}}", name)) + .collect::>() + .join(",\n"); + let json = format!("{{\n{}\n}}\n", json); + + assert_eq!(members_of(&write_input(&json, ".json"), InputFormat::Json), MEMBERS); + } + + /// A JSON file is sent byte for byte. Parsing it here kept the last of two + /// members with the same name and re-spelled every number, so the server + /// never saw what the operator wrote. The server refuses the duplicate + /// itself (src/nxt_conf.c:1616). + #[test] + fn a_json_input_is_sent_unchanged() { + let written = "{\n \"listeners\": {},\n \"listeners\": {\"*:8080\": {}},\n \"settings\": 1.50\n}\n"; + let file = write_input(written, ".json"); + + let body = config_body(&file, InputFormat::Json); + + assert_eq!(String::from_utf8(body).expect("the body is UTF-8"), written); + } + + /// Bytes that are not valid UTF-8 now reach the server, which refuses them + /// and names the member (src/nxt_conf_validation.c:1840). unitctl refused + /// them first, with a parse error that said nothing about the encoding. + #[test] + fn a_file_that_is_not_valid_utf8_is_sent_to_the_server() { + let mut file = tempfile::Builder::new() + .suffix(".json") + .tempfile() + .expect("a temporary file"); + let mut raw = br#"{"routes":[{"match":{"uri":"/"#.to_vec(); + raw.push(0xff); + raw.extend_from_slice(br#""},"action":{"return":204}}]}"#); + std::io::Write::write_all(file.as_file_mut(), &raw).expect("writing the input"); + + let (_, body) = InputFile::FileWithFormat(file.path().into(), InputFormat::Json) + .to_config_body() + .expect("the bytes must be sent, not judged here"); + let body = body_bytes(body); + + assert_eq!(body, raw); + } + + /// hjson is refused by name. Handing the file to the JSON parser instead + /// would report a syntax error on the first comment, which does not tell + /// the user what to do. + #[test] + fn an_hjson_input_is_refused_with_a_message_about_hjson() { + let file = write_input("{\n # a comment\n routes: []\n}\n", ".hjson"); + let Err(error) = InputFile::from(file.path()).to_config_body() else { + panic!("hjson must be refused"); + }; + + assert!( + error.to_string().contains("hjson is no longer supported"), + "unexpected message: {}", + error + ); + } + + /// JSON5 is refused by name too. unitctl parsed it until the PUT path + /// stopped parsing anything; keeping the variant means a ".json5" file + /// gets told what to do instead of reaching the JSON parser. + #[test] + fn a_json5_input_is_refused_with_a_message_about_json5() { + let file = write_input("{\n // a comment\n routes: [],\n}\n", ".json5"); + let input = InputFile::from(file.path()); + + assert!(input.is_config(), ".json5 must stay a config format"); + + let Err(error) = input.to_config_body() else { + panic!("JSON5 must be refused"); + }; + + assert!( + error.to_string().contains("JSON5 is no longer supported"), + "unexpected message: {}", + error + ); + } + + /// YAML is refused by name for the same reason, and the ".yaml" extension + /// still has to reach that refusal. Were the variant dropped instead, the + /// file would become `InputFormat::Unknown`, which `execute` and `import` + /// report as an unknown input type -- the file name, but not what to do + /// about it. + #[test] + fn a_yaml_input_is_refused_with_a_message_about_yaml() { + for suffix in [".yaml", ".yml"] { + let file = write_input("routes: []\n", suffix); + let input = InputFile::from(file.path()); + + assert!(input.is_config(), "{} must stay a config format", suffix); + + let Err(error) = input.to_config_body() else { + panic!("{} must be refused", suffix); + }; + assert!( + error.to_string().contains("YAML is no longer supported"), + "unexpected message: {}", + error + ); + } + } + + #[test] + fn an_unsupported_input_format_is_refused() { + let file = write_input("not a configuration", ".txt"); + assert!(InputFile::FileWithFormat(file.path().into(), InputFormat::Pem) + .to_config_body() + .is_err()); + } } diff --git a/tools/unitctl/unitctl/src/known_size.rs b/tools/unitctl/unitctl/src/known_size.rs index 0de9ec814..c6e817c5b 100644 --- a/tools/unitctl/unitctl/src/known_size.rs +++ b/tools/unitctl/unitctl/src/known_size.rs @@ -126,8 +126,14 @@ mod tests { let data = b"hello, world!"; let reader = io::Cursor::new(data.to_vec()); let known = KnownSize::Read(Box::new(reader), data.len() as u64); - let (_, len) = known.into_full_body().expect("should succeed"); + let (body, len) = known.into_full_body().expect("should succeed"); assert_eq!(len, data.len() as u64); + // The length alone would pass for any thirteen bytes, and an ordinary + // JSON configuration file now travels this variant. + let collected = futures::executor::block_on(http_body_util::BodyExt::collect(body)) + .expect("collecting the body") + .to_bytes(); + assert_eq!(collected.as_ref(), data.as_slice()); } #[test] diff --git a/tools/unitctl/unitctl/src/main.rs b/tools/unitctl/unitctl/src/main.rs index 7e054d015..71e719bf6 100644 --- a/tools/unitctl/unitctl/src/main.rs +++ b/tools/unitctl/unitctl/src/main.rs @@ -1,8 +1,6 @@ extern crate clap; extern crate colored_json; extern crate custom_error; -extern crate nu_json; -extern crate rustls_pemfile; extern crate serde; extern crate unit_client_rs; diff --git a/tools/unitctl/unitctl/src/output_format.rs b/tools/unitctl/unitctl/src/output_format.rs index eb7f954e2..d5abbe226 100644 --- a/tools/unitctl/unitctl/src/output_format.rs +++ b/tools/unitctl/unitctl/src/output_format.rs @@ -6,7 +6,6 @@ use std::io::{stdout, BufWriter, Write}; #[derive(ValueEnum, Copy, Clone, Debug, PartialEq, Eq)] pub(crate) enum OutputFormat { - Yaml, Json, #[value(id = "json-pretty")] JsonPretty, @@ -24,8 +23,6 @@ impl OutputFormat { serde_json::to_value(object).map_err(|e| UnitctlError::SerializationError { message: e.to_string() })?; match (self, no_color) { - (OutputFormat::Yaml, _) => serde_yaml::to_writer(BufWriter::new(out), &value) - .map_err(|e| UnitctlError::SerializationError { message: e.to_string() }), (OutputFormat::Json, _) => serde_json::to_writer(BufWriter::new(out), &value) .map_err(|e| UnitctlError::SerializationError { message: e.to_string() }), (OutputFormat::JsonPretty, true) => serde_json::to_writer_pretty(BufWriter::new(out), &value) diff --git a/tools/unitctl/unitctl/src/requests.rs b/tools/unitctl/unitctl/src/requests.rs index d56efbd0b..116df0a0b 100644 --- a/tools/unitctl/unitctl/src/requests.rs +++ b/tools/unitctl/unitctl/src/requests.rs @@ -3,38 +3,34 @@ use super::UnitClient; use super::UnitSerializableMap; use super::UnitctlError; use crate::known_size::KnownSize; +use bytes::Bytes; +use http_body_util::Full; use hyper::Request; -use rustls_pemfile::Item; +use rustls_pki_types::pem; +use rustls_pki_types::pem::SectionKind; use std::collections::HashMap; +use std::io::BufRead; use std::io::Cursor; +use std::io::ErrorKind; use std::sync::atomic::AtomicUsize; use unit_client_rs::unit_client::UnitClientError; -/// Send the contents of a file to the unit server -/// We assume that the file is valid and can be sent to the server -pub async fn send_and_validate_config_deserialize_response( +/// Send a configuration file to the unit server. +/// +/// The bytes go out as they were written. unitctl used to parse the file +/// into a `serde_json::Map` and re-serialize it. That +/// dropped an operator's duplicate member before the server could refuse it +/// (src/nxt_conf.c:1616) and re-spelled every number. The server validates: +/// UTF-8 (src/nxt_conf_validation.c:1840), duplicates, and the schema. +pub async fn send_config_deserialize_response( client: &UnitClient, method: &str, path: &str, - input_file: Option<&InputFile>, + input_file: &InputFile, ) -> Result { - let body_data = match input_file { - Some(input) => Some(input.to_unit_serializable_map()?), - None => None, - }; - - /* Unfortunately, we have load the json text into memory before sending it to the server. - * This allows for validation of the json content before sending to the server. There may be - * a better way of doing this and it is worth investigating. */ - let json = serde_json::to_value(&body_data).map_err(|error| UnitClientError::JsonError { - source: error, - path: path.into(), - })?; - - let mime_type = input_file.map(|f| f.mime_type()); - let reader = KnownSize::String(json.to_string()); + let (mime_type, body) = input_file.to_config_body()?; - streaming_upload_deserialize_response(client, method, path, mime_type, reader) + streaming_upload_deserialize_response(client, method, path, Some(mime_type), body) .await .map_err(|e| UnitctlError::UnitClientError { source: e }) } @@ -58,10 +54,7 @@ pub async fn send_and_validate_pem_data_deserialize_response( let bytes: Vec = input_file.try_into()?; { let mut cursor = Cursor::new(&bytes); - let items = rustls_pemfile::read_all(&mut cursor) - .map(|item| item.map_err(|e| UnitctlError::IoError { source: e })) - .collect(); - validate_pem_items(items)?; + validate_pem_items(read_pem_sections(&mut cursor))?; } let known_size = KnownSize::Vec((*bytes).to_owned()); @@ -70,21 +63,61 @@ pub async fn send_and_validate_pem_data_deserialize_response( .map_err(|e| UnitctlError::UnitClientError { source: e }) } -/// Validate the contents of a PEM file -fn validate_pem_items(pem_items: Vec>) -> Result<(), UnitctlError> { - fn item_name(item: Item) -> String { - match item { - Item::X509Certificate(_) => "X509Certificate", - Item::Sec1Key(_) => "Sec1Key", - Item::Crl(_) => "Crl", - Item::Pkcs1Key(_) => "Pkcs1Key", - Item::Pkcs8Key(_) => "Pkcs8Key", - // Note: this is not a valid PEM item, but rustls_pemfile library defines the enum as non-exhaustive - _ => "Unknown", +/// Name a PEM section for the tally below. +/// +/// `None` means the section is not one this check knows, and the caller drops +/// it. Only the names ending in "Key" or "Certificate" are counted, so the +/// other names are there to keep the section in the tally. +fn section_name(kind: SectionKind) -> Option<&'static str> { + match kind { + SectionKind::Certificate => Some("X509Certificate"), + SectionKind::EcPrivateKey => Some("Sec1Key"), + SectionKind::Crl => Some("Crl"), + SectionKind::RsaPrivateKey => Some("Pkcs1Key"), + SectionKind::PrivateKey => Some("Pkcs8Key"), + SectionKind::PublicKey => Some("Unknown"), + SectionKind::Csr => Some("Unknown"), + // SectionKind is non-exhaustive. A kind we do not name is dropped, so + // a file that carries one is judged on its other sections alone. + _ => None, + } +} + +/// Read every PEM section this check knows about. +/// +/// Reading stops at the first error. The caller reports that error, so +/// anything after it would not be looked at. +fn read_pem_sections(rd: &mut dyn BufRead) -> Vec> { + let mut sections = Vec::new(); + + loop { + match pem::from_buf(rd) { + Ok(Some((kind, _data))) => { + if section_name(kind).is_some() { + sections.push(Ok(kind)); + } + } + Ok(None) => return sections, + Err(error) => { + sections.push(Err(pem_error(error))); + return sections; + } } - .to_string() } +} + +/// Turn a PEM parse failure into the error unitctl reports. +fn pem_error(error: pem::Error) -> UnitctlError { + match error { + pem::Error::Io(source) => UnitctlError::IoError { source }, + other => UnitctlError::IoError { + source: std::io::Error::new(ErrorKind::InvalidData, other.to_string()), + }, + } +} +/// Validate the contents of a PEM file +fn validate_pem_items(pem_items: Vec>) -> Result<(), UnitctlError> { if pem_items.is_empty() { let error = UnitctlError::CertificateError { message: "No certificates found in file".to_string(), @@ -96,7 +129,7 @@ fn validate_pem_items(pem_items: Vec>) -> Result<(), for pem_item_result in pem_items { let pem_item = pem_item_result?; - let key = item_name(pem_item); + let key = section_name(pem_item).unwrap_or("Unknown").to_string(); if let Some(count) = items_tally.get_mut(key.clone().as_str()) { count.fetch_add(1, std::sync::atomic::Ordering::Relaxed); } else { @@ -149,6 +182,21 @@ pub async fn send_body_deserialize_response serde::Deserializ .map_err(|e| UnitctlError::UnitClientError { source: e }) } +/// Read a document without decoding it. +/// +/// A caller that will hand the document back to Unit needs the bytes Unit +/// sent: a configuration can hold bytes that no Rust `String` carries, so +/// anything routed through `serde_json` on the way out would return something +/// other than what it was shown. +pub async fn send_empty_body_read_bytes(client: &UnitClient, method: &str, path: &str) -> Result { + let request = build_request(client, method, path, None, KnownSize::Empty) + .map_err(|e| UnitctlError::UnitClientError { source: e })?; + client + .send_request_and_collect_body(request) + .await + .map_err(|e| UnitctlError::UnitClientError { source: e }) +} + async fn streaming_upload_deserialize_response serde::Deserialize<'de>>( client: &UnitClient, method: &str, @@ -156,6 +204,17 @@ async fn streaming_upload_deserialize_response serde::Deseria mime_type: Option, read: KnownSize, ) -> Result { + let request = build_request(client, method, path, mime_type, read)?; + client.send_request_and_deserialize_response(request).await +} + +fn build_request( + client: &UnitClient, + method: &str, + path: &str, + mime_type: Option, + read: KnownSize, +) -> Result>, UnitClientError> { let uri = client.control_socket.create_uri_with_path(path); // Materialize the body first so that Content-Length reflects actual bytes produced, @@ -178,5 +237,86 @@ async fn streaming_upload_deserialize_response serde::Deseria .insert("Content-Type", content_type.parse().unwrap()); } - client.send_request_and_deserialize_response(request).await + Ok(request) +} + +#[cfg(test)] +mod tests { + use super::*; + + const CERT: &str = "-----BEGIN CERTIFICATE-----\nAQID\n-----END CERTIFICATE-----\n"; + const PKCS8_KEY: &str = "-----BEGIN PRIVATE KEY-----\nAQID\n-----END PRIVATE KEY-----\n"; + const RSA_KEY: &str = "-----BEGIN RSA PRIVATE KEY-----\nAQID\n-----END RSA PRIVATE KEY-----\n"; + const EC_KEY: &str = "-----BEGIN EC PRIVATE KEY-----\nAQID\n-----END EC PRIVATE KEY-----\n"; + const PUBLIC_KEY: &str = "-----BEGIN PUBLIC KEY-----\nAQID\n-----END PUBLIC KEY-----\n"; + const ECH_CONFIG: &str = "-----BEGIN ECHCONFIG-----\nAQID\n-----END ECHCONFIG-----\n"; + + /// Run the check a PEM upload goes through before it is sent. + fn validate(pem: &str) -> Result<(), UnitctlError> { + let mut cursor = Cursor::new(pem.as_bytes()); + validate_pem_items(read_pem_sections(&mut cursor)) + } + + fn error_message(pem: &str) -> String { + match validate(pem) { + Err(UnitctlError::CertificateError { message }) => message, + other => panic!("expected a certificate error, got {:?}", other.is_ok()), + } + } + + #[test] + fn accepts_a_certificate_with_a_key() { + validate(&format!("{}{}", CERT, PKCS8_KEY)).unwrap(); + validate(&format!("{}{}", CERT, RSA_KEY)).unwrap(); + validate(&format!("{}{}", CERT, EC_KEY)).unwrap(); + } + + #[test] + fn accepts_a_chain_of_certificates_with_one_key() { + validate(&format!("{}{}{}", CERT, CERT, PKCS8_KEY)).unwrap(); + } + + #[test] + fn rejects_a_certificate_with_no_key() { + assert_eq!(error_message(CERT), "No private keys found in file"); + } + + #[test] + fn rejects_a_key_with_no_certificate() { + assert_eq!(error_message(PKCS8_KEY), "No certificates found in file"); + } + + #[test] + fn rejects_a_file_with_no_pem_section() { + assert_eq!(error_message(""), "No certificates found in file"); + assert_eq!(error_message("not a pem file\n"), "No certificates found in file"); + } + + /// A public key is not a private key, so it cannot stand in for one. + #[test] + fn rejects_a_public_key_as_the_key() { + assert_eq!( + error_message(&format!("{}{}", CERT, PUBLIC_KEY)), + "No private keys found in file" + ); + } + + /// A section this check does not name is dropped, so the file is judged on + /// its other sections. + #[test] + fn ignores_a_section_it_does_not_name() { + validate(&format!("{}{}{}", CERT, ECH_CONFIG, PKCS8_KEY)).unwrap(); + assert_eq!(error_message(ECH_CONFIG), "No certificates found in file"); + } + + #[test] + fn rejects_a_section_with_no_end_marker() { + let truncated = "-----BEGIN CERTIFICATE-----\nAQID\n"; + match validate(truncated) { + Err(UnitctlError::IoError { source }) => { + assert_eq!(source.kind(), ErrorKind::InvalidData); + } + other => panic!("expected an IO error, got ok={}", other.is_ok()), + } + } } diff --git a/tools/unitctl/unitctl/src/unitctl.rs b/tools/unitctl/unitctl/src/unitctl.rs index 43f2b7776..ab0f6d48e 100644 --- a/tools/unitctl/unitctl/src/unitctl.rs +++ b/tools/unitctl/unitctl/src/unitctl.rs @@ -84,7 +84,7 @@ pub(crate) enum Commands { global = true, short = 'f', long = "file", - help = "Input file (json, json5, cjson, hjson yaml, pem) to send to unit when applicable use - for stdin" + help = "Input file (json, pem) to send to unit when applicable use - for stdin" )] input_file: Option, diff --git a/tools/unitctl/unitctl/src/unitctl_error.rs b/tools/unitctl/unitctl/src/unitctl_error.rs index 83b2da465..ea37b2b06 100644 --- a/tools/unitctl/unitctl/src/unitctl_error.rs +++ b/tools/unitctl/unitctl/src/unitctl_error.rs @@ -36,6 +36,11 @@ custom_error! {pub UnitctlError PathNotFound { path: String } = "Path not found: {path}", UnknownInputFileType { path: String } = "Unknown input type for file: {path}", NoFilesImported = "All imports failed", + UndecodableConfiguration { + path: String, + members: String, + values: String, + names: String } = "The configuration at {path} holds bytes that are not valid UTF-8: {members}", WaitTimeoutError = "Timeout waiting for unit to start has been exceeded", } @@ -118,6 +123,32 @@ pub fn eprint_error(error: &UnitctlError) { UnitctlError::UiServerError { ref message } => { eprintln!("UI server error: {}", message); } + UnitctlError::UndecodableConfiguration { + path, values, names, .. + } => { + // The sentence itself is the one custom_error! already defines, so + // that the two cannot drift apart. + eprintln!("{}", error); + eprintln!("Editing it would replace them, so unitctl will not open it."); + eprintln!("Read it with 'unitctl execute -m GET -p {}'.", path); + eprintln!("The bytes appear there as U+FFFD."); + + // A replaced value and a replaced name need different advice. + if !values.is_empty() { + eprintln!(); + eprintln!("These members hold a replaced byte in their value: {}", values); + eprintln!("Repair each one with a PUT to its own path."); + } + + if !names.is_empty() { + eprintln!(); + eprintln!("These are member names, not values: {}", names); + eprintln!("A PUT cannot repair them. The path shown holds U+FFFD where the server"); + eprintln!("holds a byte, so it names a member that does not exist. Get the name as"); + eprintln!("stored from 'unitctl export', which keeps the server's bytes. Then"); + eprintln!("rewrite the nearest parent that decodes: its object, or {}.", path); + } + } _ => { eprintln!("{}", error); } diff --git a/version b/version index 5fb12c9d5..cf548338b 100644 --- a/version +++ b/version @@ -1,5 +1,5 @@ # Copyright (C) FreeUnit Community -NXT_VERSION=1.36.0 -NXT_VERNUM=13600 +NXT_VERSION=1.37.0 +NXT_VERNUM=13700