From 06ad09252baa7361d47f71797cdff21ce1ae0a34 Mon Sep 17 00:00:00 2001 From: Andrew Tierney Date: Mon, 16 Nov 2026 11:03:01 +1100 Subject: [PATCH 1/5] Validate the output with independent readers and the input with fuzzing and corpora A writer that only reads its own output back cannot see the mistakes it shares with itself. validation.yml now puts the example output and a validation corpus (written by tools/SharpCorePDF.Validation, with a manifest of the text, passwords, permissions, revisions and signature coverage each document must have) through qpdf, pypdf, pdfminer.six, MuPDF, pdf.js, fontTools, OTS, PDFium and pyHanko, checks the incremental-update invariants, and replays the fuzz seeds and four hostile documents. robustness.yml runs libFuzzer over the parser and the DCT, CCITT, LZW, Flate and RunLength decoders, reads the veraPDF and pdf.js corpora one process per file, and judges the PDF/A writer with veraPDF over the veraPDF corpus. compliance.yml cross-validates PDF/A-4. The checks found defects that are not fixed here. Each is pinned by a skipped test (PdfSignatureAttackTests, PdfIncrementalInvariantTests) or a strict --known-defect flag that fails the run once the defect is fixed, and docs/validation.md lists them with their evidence. No shipped package changes. The constitution's statement that no Python validator runs in CI is corrected as a proposed PATCH amendment (2.0.1) for the maintainer to ratify. Local AI assistant state (.claude/, CLAUDE.local.md) is ignored. --- .config/dotnet-tools.json | 7 + .github/workflows/compliance.yml | 6 +- .github/workflows/robustness.yml | 282 +++++++++++++++ .github/workflows/validation.yml | 232 ++++++++++++ .gitignore | 4 + .specify/memory/constitution.md | 18 +- CHANGELOG.md | 28 ++ CONTRIBUTING.md | 6 +- README.md | 7 + SharpCorePDF.slnx | 4 +- docs/README.md | 2 + docs/tools/corpus-robustness.py | 207 +++++++++++ docs/tools/render-compare.py | 158 ++++++++ docs/tools/validate-arlington.py | 149 ++++++++ docs/tools/validate-encryption-pdfjs.mjs | 95 +++++ docs/tools/validate-encryption.py | 164 +++++++++ docs/tools/validate-fonts.py | 263 ++++++++++++++ docs/tools/validate-incremental.py | 145 ++++++++ docs/tools/validate-pdf.py | 30 +- docs/tools/validate-pdfa-cross.py | 11 +- docs/tools/validate-preflight-corpus.py | 148 ++++++++ docs/tools/validate-signatures.py | 77 ++++ docs/tools/validate-text.py | 147 ++++++++ docs/tools/validation-requirements.txt | 20 ++ docs/validation.md | 139 +++++++ .../PdfIncrementalInvariantTests.cs | 142 ++++++++ .../PdfSignatureAttackTests.cs | 218 +++++++++++ tools/SharpCorePDF.Validation/CorpusWriter.cs | 338 ++++++++++++++++++ tools/SharpCorePDF.Validation/FuzzTargets.cs | 254 +++++++++++++ tools/SharpCorePDF.Validation/Probe.cs | 251 +++++++++++++ tools/SharpCorePDF.Validation/Program.cs | 138 +++++++ .../SharpCorePDF.Validation.csproj | 52 +++ 32 files changed, 3730 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/robustness.yml create mode 100644 .github/workflows/validation.yml create mode 100644 docs/tools/corpus-robustness.py create mode 100644 docs/tools/render-compare.py create mode 100644 docs/tools/validate-arlington.py create mode 100644 docs/tools/validate-encryption-pdfjs.mjs create mode 100644 docs/tools/validate-encryption.py create mode 100644 docs/tools/validate-fonts.py create mode 100644 docs/tools/validate-incremental.py create mode 100644 docs/tools/validate-preflight-corpus.py create mode 100644 docs/tools/validate-text.py create mode 100644 docs/tools/validation-requirements.txt create mode 100644 docs/validation.md create mode 100644 tests/SharpCorePDF.Tests/PdfIncrementalInvariantTests.cs create mode 100644 tests/SharpCorePDF.Tests/PdfSignatureAttackTests.cs create mode 100644 tools/SharpCorePDF.Validation/CorpusWriter.cs create mode 100644 tools/SharpCorePDF.Validation/FuzzTargets.cs create mode 100644 tools/SharpCorePDF.Validation/Probe.cs create mode 100644 tools/SharpCorePDF.Validation/Program.cs create mode 100644 tools/SharpCorePDF.Validation/SharpCorePDF.Validation.csproj diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index 5ccda36..eee80fa 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -8,6 +8,13 @@ "dotnet-stryker" ], "rollForward": false + }, + "sharpfuzz.commandline": { + "version": "2.3.0", + "commands": [ + "sharpfuzz" + ], + "rollForward": false } } } \ No newline at end of file diff --git a/.github/workflows/compliance.yml b/.github/workflows/compliance.yml index 2b323f5..2f8af7e 100644 --- a/.github/workflows/compliance.yml +++ b/.github/workflows/compliance.yml @@ -214,7 +214,9 @@ jobs: # The same documents, read by two more implementations. --strict-tools makes a missing tool a failure # rather than a note, because a green run here is supposed to mean that all three agreed. PDFBox 3.0.8 - # and Ghostscript have no PDF/A-4 mode, so veraPDF is the only reading of that profile. + # and Ghostscript have no PDF/A-4 mode, so veraPDF stays the only verdict on that profile; PDFBox still + # parses the PDF 2.0 file and Ghostscript still rewrites it (as part 2), which is the reader test the + # other profiles get too. - name: Cross-validate with a second and a third implementation env: VERAPDF_HOME: ${{ env.VERAPDF_INSTALL }} @@ -223,7 +225,7 @@ jobs: output="$RUNNER_TEMP/SharpCorePDF.Examples/output" jar="$HOME/.cache/pdfbox/preflight-app-${PDFBOX_PREFLIGHT_VERSION}.jar" - for profile in 1b 2b 2u 3b 3u; do + for profile in 1b 2b 2u 3b 3u 4; do python3 docs/tools/validate-pdfa-cross.py --strict-tools \ --require-version "$VERAPDF_VERSION" \ --pdfbox "$jar" \ diff --git a/.github/workflows/robustness.yml b/.github/workflows/robustness.yml new file mode 100644 index 0000000..93de31b --- /dev/null +++ b/.github/workflows/robustness.yml @@ -0,0 +1,282 @@ +# Robustness: what the engine does with documents nobody here wrote, and with documents built to break it. +# +# validation.yml checks what the library writes, on every push. This workflow checks what it reads, weekly, +# because its inputs are large and slow: coverage-guided fuzzing of every decoder and of the document parser, +# two public corpora of real and deliberately broken files, and the PDF/A conversion measured against the +# veraPDF test corpus with veraPDF as the judge. None of it gates a pull request; a red run is a defect report. +# +# fuzz SharpFuzz instruments the three engine assemblies and libFuzzer drives one target per job +# (tools/SharpCorePDF.Validation, FuzzTargets.cs). A PdfException is the contract's refusal and +# not a finding; any other exception, a stack overflow or the memory limit is, and the input +# that caused it is uploaded. Reproduce one with: +# dotnet run --project tools/SharpCorePDF.Validation -- replay +# corpora corpus-robustness.py over the veraPDF corpus and the pdf.js test corpus, one process per +# file, under a time limit and a hard managed-heap limit. Fault, crash and timeout fail the run. +# preflight validate-preflight-corpus.py: every PDF/A file of the veraPDF corpus converted for its profile +# and the result judged by veraPDF. Report-only until its first run sets the baseline. +# +# Every external input is pinned: the libFuzzer driver by tag and SHA-256, SharpFuzz in +# .config/dotnet-tools.json, the corpora by commit, veraPDF by version (the same one compliance.yml pins). + +name: Robustness + +on: + schedule: + - cron: '0 3 * * 2' + workflow_dispatch: + inputs: + fuzz-minutes: + description: 'Minutes of fuzzing per target' + required: false + default: '30' + +permissions: + contents: read + +env: + LIBFUZZER_DOTNET_TAG: 'v2025.05.02.0904' + LIBFUZZER_DOTNET_SHA256: 'c2c2a90d94c409a4af339a0d4f244e0442c5a5d249be0f1252ba07871f285958' + VERAPDF_CORPUS_COMMIT: 'bb75f4f0073d9350dfd058c0162a367e6fadf25e' + PDFJS_TAG: 'v6.4.299' + PDFJS_COMMIT: 'd0991a0d53f3bb3a7307e3b83f279c1d117aafb3' + VERAPDF_VERSION: '1.30.4' + VERAPDF_INSTALL: ${{ github.workspace }}/.verapdf + +jobs: + fuzz: + name: Fuzz ${{ matrix.target }} + runs-on: ubuntu-latest + timeout-minutes: 60 + + strategy: + fail-fast: false + matrix: + target: [ document, dct, ccitt, lzw, flate, runlength ] + + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: Install the .NET SDKs + uses: actions/setup-dotnet@0f4a354dd5547a28c70b7853dbd95d5b67886e73 # v6 + with: + dotnet-version: | + 11.0.x + 10.0.x + + - name: Install SharpFuzz and the libFuzzer driver + run: | + set -euo pipefail + dotnet tool restore + curl --fail --silent --show-error --location --proto '=https' --tlsv1.2 \ + --output "$RUNNER_TEMP/libfuzzer-dotnet" \ + "https://github.com/Metalnem/libfuzzer-dotnet/releases/download/${LIBFUZZER_DOTNET_TAG}/libfuzzer-dotnet-ubuntu" + echo "${LIBFUZZER_DOTNET_SHA256} $RUNNER_TEMP/libfuzzer-dotnet" | sha256sum --check + chmod +x "$RUNNER_TEMP/libfuzzer-dotnet" + + # Two builds: an instrumented one for libFuzzer, and a plain one for the seeds and for replaying what + # libFuzzer finds, because instrumented code throws outside the fuzzer (see Program.cs). + - name: Build, then instrument the engine + run: | + set -euo pipefail + dotnet publish ./tools/SharpCorePDF.Validation -c Release -o "$RUNNER_TEMP/plain" + dotnet publish ./tools/SharpCorePDF.Validation -c Release -o "$RUNNER_TEMP/fuzz" + for layer in Domain Application Infrastructure; do + dotnet sharpfuzz "$RUNNER_TEMP/fuzz/SharpCorePDF.$layer.dll" + done + + - name: Seed the corpus + run: | + set -euo pipefail + tool="$RUNNER_TEMP/plain/SharpCorePDF.Validation.dll" + seeds="$RUNNER_TEMP/seeds" + assets=tests/SharpCorePDF.Tests/Assets + case "${{ matrix.target }}" in + document) dotnet "$tool" seed document "$seeds" "$assets/parity/corpus" "$assets/parity/labelled" "$assets/parity/forms" ;; + dct) dotnet "$tool" seed dct "$seeds" "$assets"/jpeg/*.jpg ;; + ccitt) dotnet "$tool" seed ccitt "$seeds" "$assets/ccitt/ccitt-g3.bin" "$assets/ccitt/ccitt-g4.bin" ;; + *) mkdir -p "$seeds" && printf '\x03\x00\x00\x3f\x3f' > "$seeds/empty" ;; + esac + dotnet "$tool" replay "${{ matrix.target }}" "$seeds" + + - name: Fuzz + env: + SCPDF_FUZZ_TARGET: ${{ matrix.target }} + MINUTES: ${{ github.event.inputs.fuzz-minutes || '30' }} + run: | + set -uo pipefail + mkdir -p artifacts/crashes + "$RUNNER_TEMP/libfuzzer-dotnet" \ + --target_path=dotnet --target_arg="$RUNNER_TEMP/fuzz/SharpCorePDF.Validation.dll" \ + -max_total_time=$(( MINUTES * 60 )) -timeout=30 -rss_limit_mb=4096 -max_len=262144 \ + -artifact_prefix=artifacts/crashes/ -print_final_stats=1 \ + "$RUNNER_TEMP/seeds" + status=$? + ls -la artifacts/crashes + if [ -n "$(ls -A artifacts/crashes)" ]; then + echo "::error::libFuzzer saved $(ls artifacts/crashes | wc -l) input(s) that broke the ${{ matrix.target }} target" + exit 1 + fi + exit $status + + - name: Upload what broke the engine + if: failure() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: fuzz-crashes-${{ matrix.target }} + path: artifacts/crashes + if-no-files-found: ignore + retention-days: 30 + + corpora: + name: Public corpora + runs-on: ubuntu-latest + timeout-minutes: 120 + + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: Install the .NET SDKs + uses: actions/setup-dotnet@0f4a354dd5547a28c70b7853dbd95d5b67886e73 # v6 + with: + dotnet-version: | + 11.0.x + 10.0.x + + - name: Fetch the corpora + run: | + set -euo pipefail + # The veraPDF corpus is small enough to clone without blobs and check out at the pinned commit; pdf.js + # is not, so it is cloned one commit deep at its tag and the commit is checked against the pin. + git clone -q --filter=blob:none --sparse https://github.com/veraPDF/veraPDF-corpus.git "$RUNNER_TEMP/verapdf-corpus" + git -C "$RUNNER_TEMP/verapdf-corpus" sparse-checkout set PDF_A-1b PDF_A-2b PDF_A-4 "ISO 32000-1" "ISO 32000-2" "Isartor test files" + git -C "$RUNNER_TEMP/verapdf-corpus" -c advice.detachedHead=false checkout -q "$VERAPDF_CORPUS_COMMIT" + + git clone -q --depth 1 --branch "$PDFJS_TAG" --filter=blob:none --sparse https://github.com/mozilla/pdf.js.git "$RUNNER_TEMP/pdfjs" + git -C "$RUNNER_TEMP/pdfjs" sparse-checkout set test/pdfs + test "$(git -C "$RUNNER_TEMP/pdfjs" rev-parse HEAD)" = "$PDFJS_COMMIT" + + find "$RUNNER_TEMP/verapdf-corpus" "$RUNNER_TEMP/pdfjs" -name '*.pdf' | wc -l + + - name: Build the probe + run: dotnet build ./tools/SharpCorePDF.Validation -c Release + + # The six known files fault on three recorded defects (docs/validation.md): an empty name as a + # dictionary key (issue3188, issue5896, pdfjs_wikipedia, poppler-742-0-fuzzed), a glyph whose box is + # upside down (issue14415) and a date whose offset is beyond 14 hours (TAMReview). + - name: Read every document + run: | + python3 docs/tools/corpus-robustness.py \ + --probe ./tools/SharpCorePDF.Validation/bin/Release/net11.0/SharpCorePDF.Validation.dll \ + --timeout 60 --heap-limit-mb 1024 --report artifacts/corpora.json \ + --known issue3188.pdf --known issue5896.pdf --known pdfjs_wikipedia.pdf \ + --known poppler-742-0-fuzzed.pdf --known issue14415.pdf --known TAMReview.pdf \ + "$RUNNER_TEMP/verapdf-corpus" "$RUNNER_TEMP/pdfjs/test/pdfs" + + - name: Upload the report + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: corpora + path: artifacts/corpora.json + if-no-files-found: warn + retention-days: 30 + + preflight: + name: PDF/A conversion against the veraPDF corpus + runs-on: ubuntu-latest + timeout-minutes: 180 + + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: Install the .NET SDKs + uses: actions/setup-dotnet@0f4a354dd5547a28c70b7853dbd95d5b67886e73 # v6 + with: + dotnet-version: | + 11.0.x + 10.0.x + + - name: Set up the JDK veraPDF runs on + uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6.1.0 + with: + distribution: 'zulu' + java-version: '17' + + - name: Cache the veraPDF installer + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/.cache/verapdf + key: verapdf-${{ env.VERAPDF_VERSION }}-installer + + # The same installation compliance.yml performs, so both workflows judge with the same validator. + - name: Install veraPDF + run: | + set -euo pipefail + band="${VERAPDF_VERSION%.*}" + archive="$HOME/.cache/verapdf/verapdf-greenfield-${VERAPDF_VERSION}-installer.zip" + mkdir -p "$(dirname "$archive")" + if [ ! -f "$archive" ]; then + curl --fail --silent --show-error --location \ + --proto '=https' --proto-redir '=https' --tlsv1.2 \ + --output "$archive" \ + "https://software.verapdf.org/releases/${band}/verapdf-greenfield-${VERAPDF_VERSION}-installer.zip" + fi + rm -rf "$VERAPDF_INSTALL" "$RUNNER_TEMP/verapdf" + mkdir -p "$VERAPDF_INSTALL" + unzip -q "$archive" -d "$RUNNER_TEMP/verapdf" + printf '%s\n' \ + '' \ + '' \ + ' ' \ + ' ' \ + " ${VERAPDF_INSTALL}" \ + ' ' \ + ' ' \ + ' ' \ + ' ' \ + ' ' \ + ' ' \ + ' ' \ + ' ' \ + ' ' \ + '' \ + > "$RUNNER_TEMP/auto-install.xml" + rm -f "/tmp/iz-veraPDF Software.tmp" "${TMPDIR:-/tmp}/iz-veraPDF Software.tmp" + bash "$RUNNER_TEMP/verapdf/verapdf-greenfield-${VERAPDF_VERSION}/verapdf-install" "$RUNNER_TEMP/auto-install.xml" + if [ ! -x "$VERAPDF_INSTALL/verapdf" ]; then + printf '%s\n' '#!/usr/bin/env bash' \ + "exec java -jar \"$VERAPDF_INSTALL\"/bin/cli-*.jar \"\$@\"" \ + > "$VERAPDF_INSTALL/verapdf" + chmod +x "$VERAPDF_INSTALL/verapdf" + fi + "$VERAPDF_INSTALL/verapdf" --version + + - name: Fetch the veraPDF corpus + run: | + set -euo pipefail + git clone -q --filter=blob:none --sparse https://github.com/veraPDF/veraPDF-corpus.git "$RUNNER_TEMP/verapdf-corpus" + git -C "$RUNNER_TEMP/verapdf-corpus" sparse-checkout set PDF_A-1b PDF_A-2b PDF_A-2u PDF_A-3b PDF_A-4 + git -C "$RUNNER_TEMP/verapdf-corpus" -c advice.detachedHead=false checkout -q "$VERAPDF_CORPUS_COMMIT" + + - name: Build the probe + run: dotnet build ./tools/SharpCorePDF.Validation -c Release + + - name: Convert and judge + run: | + python3 docs/tools/validate-preflight-corpus.py \ + --corpus "$RUNNER_TEMP/verapdf-corpus" \ + --probe ./tools/SharpCorePDF.Validation/bin/Release/net11.0/SharpCorePDF.Validation.dll \ + --verapdf "$VERAPDF_INSTALL/verapdf" \ + --report artifacts/preflight.json + + - name: Upload the report + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: preflight + path: artifacts/preflight.json + if-no-files-found: warn + retention-days: 30 diff --git a/.github/workflows/validation.yml b/.github/workflows/validation.yml new file mode 100644 index 0000000..6bb7cb0 --- /dev/null +++ b/.github/workflows/validation.yml @@ -0,0 +1,232 @@ +# Independent validation: what readers that share no code with this library make of what it writes. +# +# ci.yml answers whether the code builds and the suite passes; compliance.yml whether the archival output +# conforms. This workflow answers a third question -- whether other implementations read the documents the +# way the library meant them -- because a writer that only ever reads its own output back cannot see the +# mistakes it shares with itself. Every check here runs against the example output and against the validation +# corpus that tools/SharpCorePDF.Validation writes, whose manifest states what each document has to say. +# +# qpdf validate-pdf.py: the file parses, every page reads, every stream decodes +# text validate-text.py: pypdf, pdfminer.six and MuPDF extract the lines that were drawn +# encryption validate-encryption.py and the pdf.js leg: four readers open the RC4/AES matrix with the right +# password, refuse the wrong one, and read the revision and the permissions that were asked for +# fonts validate-fonts.py: fontTools and OTS read every embedded program, and the widths agree +# rendering render-compare.py: PDFium and MuPDF paint the same pages +# incremental validate-incremental.py: every revision stands alone, /ID[0] holds, an update only appends +# signatures validate-signatures.py --manifest: pyHanko agrees on integrity, trust and coverage +# fuzz seeds every fuzz target runs over its seeds in process, so a harness that rots is noticed here and +# not in the weekly fuzzing run (robustness.yml) +# hostile corpus-robustness.py over the documents it synthesizes for the failure classes the review named +# arlington validate-arlington.py against the veraPDF Arlington service (report-only until a baseline exists) +# +# --known-defect / --known names a failure that is recorded and not yet fixed. Each is strict: when the +# defect is fixed the check passes, the flag no longer matches, and the run fails until the flag is removed +# in the same change as the fix. The in-suite counterparts are the skipped tests in +# PdfSignatureAttackTests and PdfIncrementalInvariantTests. +# +# The validator versions are pinned in docs/tools/validation-requirements.txt, pdfjs-dist below. + +name: Independent validation + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + workflow_dispatch: + +permissions: + contents: read + +env: + PDFJS_VERSION: '6.4.299' + +jobs: + readers: + name: Read by other implementations + runs-on: ubuntu-latest + timeout-minutes: 40 + + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: Install the .NET SDKs + uses: actions/setup-dotnet@0f4a354dd5547a28c70b7853dbd95d5b67886e73 # v6 + with: + dotnet-version: | + 11.0.x + 10.0.x + + - name: Install the validators + run: | + set -euo pipefail + python3 -m venv "$RUNNER_TEMP/venv" + "$RUNNER_TEMP/venv/bin/pip" install --quiet -r docs/tools/validation-requirements.txt + echo "$RUNNER_TEMP/venv/bin" >> "$GITHUB_PATH" + npm install --silent --no-save --prefix "$RUNNER_TEMP/pdfjs" "pdfjs-dist@${PDFJS_VERSION}" + + - name: Build the examples and the validation harness + run: | + dotnet build ./examples/SharpCorePDF.Examples -c Release + dotnet build ./tools/SharpCorePDF.Validation -c Release + + - name: Write the documents + env: + TMPDIR: ${{ runner.temp }} + run: | + set -euo pipefail + dotnet run --project ./examples/SharpCorePDF.Examples -c Release --no-build + dotnet run --project ./tools/SharpCorePDF.Validation -c Release --no-build -- corpus "$RUNNER_TEMP/corpus" + + # protected.pdf is encrypted with a password the example makes for the run and prints; the encrypted + # documents qpdf does check are the corpus's, whose passwords the manifest carries + # (validate-encryption.py runs qpdf --check on each of them with its password). + - name: qpdf + run: | + set -euo pipefail + shopt -s nullglob + files=() + for file in "$RUNNER_TEMP"/SharpCorePDF.Examples/output/*.pdf "$RUNNER_TEMP"/corpus/*.pdf; do + case "$(basename "$file")" in protected.pdf|encrypted-*) continue ;; esac + files+=("$file") + done + # Both merge a page of NamedDests.pdf, which states no /MediaBox; the merge copies it without one, + # so the result breaks the rule that every page has a media box, and qpdf assumes Letter. + python docs/tools/validate-pdf.py --known-defect inspected.pdf --known-defect with-warnings.pdf "${files[@]}" + + - name: Text round trip + run: python docs/tools/validate-text.py "$RUNNER_TEMP/corpus" + + - name: Encryption matrix + run: | + set -euo pipefail + python docs/tools/validate-encryption.py "$RUNNER_TEMP/corpus" + node docs/tools/validate-encryption-pdfjs.mjs "$RUNNER_TEMP/corpus" \ + "$RUNNER_TEMP/pdfjs/node_modules/pdfjs-dist/legacy/build/pdf.mjs" + + - name: Embedded fonts + run: | + python docs/tools/validate-fonts.py \ + "$RUNNER_TEMP/SharpCorePDF.Examples/output" \ + "$RUNNER_TEMP/corpus/text-embedded.pdf" "$RUNNER_TEMP/corpus/text-merged.pdf" + + - name: Rendering, PDFium against MuPDF + run: | + python docs/tools/render-compare.py --report-dir artifacts/rendering --exclude protected.pdf \ + "$RUNNER_TEMP/SharpCorePDF.Examples/output" \ + "$RUNNER_TEMP/corpus/text-standard.pdf" "$RUNNER_TEMP/corpus/text-embedded.pdf" \ + "$RUNNER_TEMP/corpus/text-merged.pdf" "$RUNNER_TEMP/corpus/incremental.pdf" \ + "$RUNNER_TEMP/corpus/signed.pdf" + + # id-first: the editor writes a new identifier into both elements of /ID on an appended revision. + # appends-only: the editor's incremental update appends a renumbered copy of the whole document + # rather than the objects it changed. + - name: Incremental update invariants + run: | + python docs/tools/validate-incremental.py "$RUNNER_TEMP/corpus" \ + --known-defect id-first --known-defect appends-only + + # The same copy rewrites the signature dictionary after signing, which is why pyHanko cannot place the + # first signature of signed-then-edited.pdf at the end of a revision. + - name: Signatures, pyHanko + run: | + python docs/tools/validate-signatures.py --manifest "$RUNNER_TEMP/corpus/manifest.json" \ + --known-defect signed-then-edited.pdf:FirstSignature + + - name: Fuzz targets over their seeds + run: | + set -euo pipefail + tool=./tools/SharpCorePDF.Validation/bin/Release/net11.0/SharpCorePDF.Validation.dll + seeds="$RUNNER_TEMP/seeds" + dotnet "$tool" seed document "$seeds/document" tests/SharpCorePDF.Tests/Assets/parity/corpus "$RUNNER_TEMP/corpus" + dotnet "$tool" seed dct "$seeds/dct" tests/SharpCorePDF.Tests/Assets/jpeg/*.jpg + dotnet "$tool" seed ccitt "$seeds/ccitt" tests/SharpCorePDF.Tests/Assets/ccitt/ccitt-g3.bin tests/SharpCorePDF.Tests/Assets/ccitt/ccitt-g4.bin + for target in document dct ccitt; do + dotnet "$tool" replay "$target" "$seeds/$target" + done + + # Each of the four documents reaches a failure class the review named; each is a known defect until the + # change that fixes it removes its --known line. + - name: Hostile documents + run: | + set -euo pipefail + python docs/tools/corpus-robustness.py --synthesize "$RUNNER_TEMP/hostile" + python docs/tools/corpus-robustness.py \ + --probe ./tools/SharpCorePDF.Validation/bin/Release/net11.0/SharpCorePDF.Validation.dll \ + --timeout 120 --report artifacts/hostile.json \ + --known flate-bomb.pdf --known length-chain.pdf --known objstm-count.pdf --known jpeg-selector.pdf \ + "$RUNNER_TEMP/hostile" + + - name: Upload the documents and the reports + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: independent-validation + path: | + artifacts + ${{ runner.temp }}/corpus + ${{ runner.temp }}/SharpCorePDF.Examples/output/*.pdf + if-no-files-found: warn + retention-days: 14 + + arlington: + name: Arlington PDF Model + runs-on: ubuntu-latest + timeout-minutes: 30 + # The service's report format and the writer's baseline are confirmed by the first run on a runner; until + # then the job reports without blocking. Remove continue-on-error and pass --strict with the --accept list + # that run establishes. + continue-on-error: true + + services: + arlington: + image: verapdf/arlington:v1.30.4 + ports: + - 8080:8080 + + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: Install the .NET SDKs + uses: actions/setup-dotnet@0f4a354dd5547a28c70b7853dbd95d5b67886e73 # v6 + with: + dotnet-version: | + 11.0.x + 10.0.x + + - name: Write the documents + env: + TMPDIR: ${{ runner.temp }} + run: | + set -euo pipefail + dotnet run --project ./examples/SharpCorePDF.Examples -c Release + dotnet run --project ./tools/SharpCorePDF.Validation -c Release -- corpus "$RUNNER_TEMP/corpus" + + - name: Wait for the service + run: | + for attempt in $(seq 1 60); do + curl -sf http://localhost:8080/api/info > /dev/null && exit 0 + sleep 2 + done + echo "::error::the Arlington service did not answer on port 8080" + exit 1 + + - name: Check against the model + run: | + python3 docs/tools/validate-arlington.py --report-dir artifacts/arlington \ + --exclude protected.pdf \ + "$RUNNER_TEMP/SharpCorePDF.Examples/output" \ + "$RUNNER_TEMP/corpus/text-standard.pdf" "$RUNNER_TEMP/corpus/text-embedded.pdf" \ + "$RUNNER_TEMP/corpus/text-merged.pdf" "$RUNNER_TEMP/corpus/incremental.pdf" \ + "$RUNNER_TEMP/corpus/signed.pdf" + + - name: Upload the reports + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: arlington + path: artifacts/arlington + if-no-files-found: warn + retention-days: 14 diff --git a/.gitignore b/.gitignore index 4ca788b..a7b1b5e 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,10 @@ StrykerOutput/ *.sln.docstates *.DotSettings.user +# AI assistant state: prompts, transcripts and local settings stay on the machine that wrote them +.claude/ +CLAUDE.local.md + # Build logs *.binlog msbuild.log diff --git a/.specify/memory/constitution.md b/.specify/memory/constitution.md index af536c8..e45abe8 100644 --- a/.specify/memory/constitution.md +++ b/.specify/memory/constitution.md @@ -1,6 +1,13 @@ + + Exe + net11.0 + preview + SharpCorePDF.Validation + SharpCorePDF.Validation + enable + enable + nullable + true + latest-recommended + false + false + true + + + + + + + + + + + + + + + + + + + + From 8ad4c9e54663e4a4d467365154e25fbe4d360725 Mon Sep 17 00:00:00 2001 From: Andrew Tierney Date: Mon, 16 Nov 2026 11:55:24 +1100 Subject: [PATCH 2/5] Release pdf.js documents through their loading task pdf.js 6 has no PDFDocumentProxy.destroy(), so the encryption check's pdf.js leg reported every document as unopenable: the call threw inside the try that was meant to catch a wrong password. The loading task owns the document and destroying it releases both. The fix was made before the previous commit but an editor saved a stale copy of the file over it. --- docs/tools/validate-encryption-pdfjs.mjs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/tools/validate-encryption-pdfjs.mjs b/docs/tools/validate-encryption-pdfjs.mjs index 8ee5cd1..57a929c 100644 --- a/docs/tools/validate-encryption-pdfjs.mjs +++ b/docs/tools/validate-encryption-pdfjs.mjs @@ -26,8 +26,10 @@ async function open(file, password) { const data = new Uint8Array(await readFile(path.join(corpus, file))); const task = pdfjs.getDocument({data, password, isEvalSupported: false, verbosity: 0}); + // The loading task owns the document in pdf.js 6, which has no PDFDocumentProxy.destroy(): destroying + // the task is what releases both. try { - return await task.promise; + return {task, document: await task.promise}; } catch (error) { await task.destroy(); throw error; @@ -36,8 +38,8 @@ async function open(file, password) { async function refuses(file, password) { try { - const document = await open(file, password); - await document.destroy(); + const {task} = await open(file, password); + await task.destroy(); return false; } catch (error) { return error?.name === "PasswordException"; @@ -51,7 +53,7 @@ for (const entry of manifest.encryption) { const problems = []; try { - const document = await open(entry.file, entry.userPassword); + const {task, document} = await open(entry.file, entry.userPassword); const page = await document.getPage(1); const content = await page.getTextContent(); const text = normalise(content.items.map((item) => item.str + (item.hasEOL ? "\n" : "")).join("")); @@ -61,14 +63,14 @@ for (const entry of manifest.encryption) { problems.push(`page 1 text does not contain ${JSON.stringify(first)}`); } - await document.destroy(); + await task.destroy(); } catch (error) { problems.push(`the user password did not open it: ${error?.name}: ${error?.message}`); } try { - const document = await open(entry.file, entry.ownerPassword); - await document.destroy(); + const {task} = await open(entry.file, entry.ownerPassword); + await task.destroy(); } catch (error) { problems.push(`the owner password did not open it: ${error?.name}: ${error?.message}`); } From 7020f3bed853cb3fc0b1c4c12b4aef8242d74c73 Mon Sep 17 00:00:00 2001 From: Andrew Tierney Date: Mon, 16 Nov 2026 12:10:46 +1100 Subject: [PATCH 3/5] Refuse signature wrapping and bound what hostile input can allocate Signature wrapping: the validator required only that the byte range start at 0 and end at the end of the file, so bytes written into the unsigned gap could close the signature value early and add entries while the digest still matched, and the signature was reported valid and covering the whole document. The gap must now be exactly one hexadecimal string equal to the dictionary's /Contents; anything else is MalformedByteRange, integrity Unknown and CoversWholeDocument false. Decompression bombs: Flate, LZW and RunLength decoded without a limit, so a 1.1 MB document took the process past 1 GiB. PdfEngineOptions.MaxDecodedStreamBytes (128 MiB) now stops them; decode buffers above 16 MiB no longer stay in the shared array pool, and buffer growth no longer overflows. Stack overflow: a chain of streams whose /Length is the next stream was resolved by recursion, and 60,000 links ended the process. The chain is followed no deeper than MaxNestingDepth, then the stream is found by its endstream keyword, as for a /Length cycle. Allocations sized by the file: an object stream's /N is bounded by what the stream can hold, and a JPEG whose output is above the caller's budget is measured from its frame header instead of being decoded first; the working sizes are computed in 64 bits. Tests that fail without the fixes: PdfSignatureAttackTests (gap wrapping, gap shape), PdfFilterTests (limits per filter, content stream past the limit), PdfReaderRecoveryTests (length chain, indirect lengths, object stream /N), PdfJpegFilterTests (oversized frames, working buffers). The three hostile documents of validation.yml are read under a 1 GiB heap, and their --known flags are removed. --- .github/workflows/validation.yml | 7 +- CHANGELOG.md | 28 +++- docs/manual/src/10-dependency-injection.md | 14 +- docs/validation.md | 15 +- .../Pdf/JpegDecoder.cs | 78 ++++++++++ .../Pdf/PdfCrossReferenceReader.Streams.cs | 2 +- .../Pdf/PdfFileReader.ObjectStreams.cs | 22 +-- .../Pdf/PdfFileReader.Objects.cs | 14 +- .../Pdf/PdfFileReader.cs | 1 + .../Pdf/PdfFilters.Inflation.cs | 102 +++++++++++--- .../Pdf/PdfFilters.Lzw.cs | 21 ++- .../Pdf/PdfFilters.RunLength.cs | 24 +++- .../Pdf/PdfFilters.cs | 16 ++- .../Pdf/PdfFontDecoder.cs | 2 +- .../Pdf/PdfImageFilters.cs | 4 +- .../Pdf/PdfTextCollector.cs | 7 +- .../Pdf/PdfType1Encoding.cs | 2 +- .../Pdf/PooledByteBuffer.cs | 51 ++++++- .../PdfEditEngine.Overlays.cs | 2 +- .../PdfEngineOptions.cs | 23 +++ .../PdfPageContentReader.Content.cs | 7 +- .../PdfPageContentReader.Images.cs | 2 +- .../PdfPageContentReader.Metadata.cs | 2 +- .../PublicAPI.Unshipped.txt | 2 + .../Signatures/SignatureValidator.cs | 75 +++++++++- .../PdfEngineOptionsTests.cs | 10 ++ tests/SharpCorePDF.Tests/PdfFilterTests.cs | 96 +++++++++++++ .../SharpCorePDF.Tests/PdfJpegFilterTests.cs | 90 ++++++++++++ .../PdfReaderRecoveryTests.cs | 133 ++++++++++++++++++ .../PdfSignatureAttackTests.cs | 75 +++++++++- 30 files changed, 840 insertions(+), 87 deletions(-) diff --git a/.github/workflows/validation.yml b/.github/workflows/validation.yml index 6bb7cb0..de42cbe 100644 --- a/.github/workflows/validation.yml +++ b/.github/workflows/validation.yml @@ -146,8 +146,9 @@ jobs: dotnet "$tool" replay "$target" "$seeds/$target" done - # Each of the four documents reaches a failure class the review named; each is a known defect until the - # change that fixes it removes its --known line. + # Each of the four documents reaches a failure class the review named. flate-bomb, length-chain and + # objstm-count are refused or read since the decode limit, the length-chain bound and the /N bound; + # jpeg-selector is a known defect until the change that fixes it removes its --known line. - name: Hostile documents run: | set -euo pipefail @@ -155,7 +156,7 @@ jobs: python docs/tools/corpus-robustness.py \ --probe ./tools/SharpCorePDF.Validation/bin/Release/net11.0/SharpCorePDF.Validation.dll \ --timeout 120 --report artifacts/hostile.json \ - --known flate-bomb.pdf --known length-chain.pdf --known objstm-count.pdf --known jpeg-selector.pdf \ + --known jpeg-selector.pdf \ "$RUNNER_TEMP/hostile" - name: Upload the documents and the reports diff --git a/CHANGELOG.md b/CHANGELOG.md index 99ac86d..abe4bd9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,19 +33,37 @@ against an empty feed. reaches a consumer. [`docs/validation.md`](docs/validation.md) records each check, how it was shown to fail before it was trusted, and the defects below. +- **`PdfEngineOptions.MaxDecodedStreamBytes`** (128 MiB by default) bounds what one stream may decode to. + `MaxSourceBytes` bounds the input only, and Flate reaches about a thousand to one. + +### Fixed + +- **Security: signature wrapping.** The validator required only that the byte range start at 0 and end at the + end of the file, so bytes written into the unsigned gap could close the signature value early and add + entries while the digest still matched, and the signature was reported valid and covering the whole + document. The gap must now be exactly one hexadecimal string equal to the dictionary's `/Contents`; + anything else is `MalformedByteRange`, integrity `Unknown`, and `CoversWholeDocument` false. +- **Security: decompression bombs.** Flate, LZW and RunLength streams decoded without a limit, so a 1.1 MB + document took the process past 1 GiB. They now stop at `MaxDecodedStreamBytes`; buffer growth no longer + overflows past 1 GiB. +- **Security: stack overflow.** A chain of streams whose `/Length` is the next stream was resolved by + recursion, and 60,000 links ended the process. The chain is followed no deeper than `MaxNestingDepth`; + past it the stream is found by its `endstream` keyword, as for a `/Length` cycle. +- **Security: allocations sized by the file.** An object stream's `/N` sized two arrays before anything checked + it (513 bytes asked for 16 GiB), and a JPEG frame's 16-bit width and height sized the sample planes before + the caller's budget was consulted. `/N` is now bounded by what the stream can hold, a JPEG above the budget + is measured from its header without being decoded — the same answer as before, without the allocation — + and the working sizes are computed in 64 bits. + ### Known defects, found by the new checks and not yet fixed Each is reproduced, pinned by a skipped test or a strict `--known-defect` flag that fails the run once the defect is fixed, and listed in [`docs/validation.md`](docs/validation.md#known-defects). -- The signature validator does not require the gap between the byte ranges to be exactly the `/Contents` - string, so a gap carrying extra entries is still reported as covering the whole document. - An incremental update rewrites both elements of `/ID`, and appends a renumbered copy of the whole document rather than the objects it changed — which also rewrites the signature dictionary after signing. - A merge copies a page that states no `/MediaBox` without one. -- Hostile input: a Flate stream has no decoded-size limit, a chain of indirect `/Length` references overflows - the stack, an object stream's `/N` sizes an allocation unchecked, and a JPEG scan's Huffman table selector - is not range-checked. +- Hostile input: a JPEG scan's Huffman table selector is not range-checked. - Real-world input (the pdf.js test corpus): an empty name as a dictionary key stops a document opening, a glyph with an upside-down box faults text extraction, and `PdfDate.TryParse` throws on an out-of-range time-zone offset instead of answering false. diff --git a/docs/manual/src/10-dependency-injection.md b/docs/manual/src/10-dependency-injection.md index 953b10f..d46b5aa 100644 --- a/docs/manual/src/10-dependency-injection.md +++ b/docs/manual/src/10-dependency-injection.md @@ -144,6 +144,7 @@ defaults are deliberately permissive: | Option | Default | Effect | | --- | --- | --- | | `MaxSourceBytes` | 512 MiB | Largest source read into memory; larger sources are rejected | +| `MaxDecodedStreamBytes` | 128 MiB | Largest size one stream may decode to; a stream that would decode past it is refused | | `MaxNestingDepth` | 64 | Maximum dictionary/array nesting, page-tree depth and form field tree depth | | `MaxPagesPerSource` | 100 000 | Declared page limit; see the note below | | `ReconstructCrossReferenceOnFailure` | `true` | Rebuild a damaged or missing cross-reference table by scanning | @@ -154,8 +155,15 @@ The source limit is enforced by the reader, and exceeding it throws `PdfSourceFo detail that names the limit. It is not a streaming cap: the whole source is held in memory while it is processed, so the practical ceiling is your process's memory budget, not the number on the option. -`MaxNestingDepth` has four effects. It bounds how deep an object graph may be parsed, it bounds the -page-tree walk, it bounds the form field tree, and it bounds the tree the signature search descends. A +`MaxDecodedStreamBytes` bounds what a stream decodes to, which the source limit does not: Flate reaches about +a thousand to one, so a one-megabyte document can ask for a gigabyte. A content stream, form or font program +past the limit is reported as unreadable through the warning list, as other undecodable content is; an object +or cross-reference stream past it refuses the document with `PdfSourceFormatException`. Images are not refused +by it — their samples are carried only within `MaxInlineContentBytes` already. + +`MaxNestingDepth` has five effects. It bounds how deep an object graph may be parsed, it bounds how many +streams deep a chain of indirect `/Length` references is followed (past it, the stream is found by its +`endstream` keyword), it bounds the page-tree walk, it bounds the form field tree, and it bounds the tree the signature search descends. A page tree that is cut off contributes a `PageTreeTooDeep` warning rather than failing the merge; a field tree that is cut off contributes `FormFieldDepthExceeded` together with the fields above the cut, and the search for signatures reports the same warning and adds a `PdfSignatureProblemKind.FormFieldTreeTooDeep` @@ -176,7 +184,7 @@ file is binary; the marker costs a few bytes and prevents a class of corruption hard to diagnose. > `MaxPagesPerSource` is part of the options record but this version does not enforce it. Do not rely on -> it as a security boundary. For untrusted input, set `MaxSourceBytes`, inspect the document with +> it as a security boundary. For untrusted input, set `MaxSourceBytes` and `MaxDecodedStreamBytes`, inspect the document with > `IPdfInspector` and reject it if `PageCount` exceeds your own limit, before building a plan. ## Configuration file diff --git a/docs/validation.md b/docs/validation.md index 7986dba..6ce4585 100644 --- a/docs/validation.md +++ b/docs/validation.md @@ -83,25 +83,30 @@ fix. This is the list on 2026-11-16, every entry reproduced, not inferred: | Defect | Found by | Evidence | | --- | --- | --- | -| The signature validator does not check that the gap between the two byte ranges is exactly the `/Contents` string, so a gap that carries extra entries is still reported as covering the whole document (the signature-wrapping class) | #3 | `PdfSignatureAttackTests.Does_not_report_a_gap_that_holds_more_than_the_signature_value_as_covering_the_document` (skipped; fails when run) | | An incremental update writes a new identifier into both elements of `/ID`; ISO 32000-1 §14.4 keeps the first | #10 | `PdfIncrementalInvariantTests.Keeps_the_first_identifier_element_across_an_incremental_update` (skipped); `validate-incremental.py` `id-first` | | `UseIncrementalUpdate` appends a renumbered copy of the whole document rather than the objects it changed: every byte of the earlier revision is kept, but the file grows by its own size per edit, and the signature dictionary is written again after signing, so pyHanko places a signature followed by an edit as `CONTIGUOUS_BLOCK_FROM_START` with modification level `OTHER` instead of `ENTIRE_REVISION` | #10, #3 | `validate-incremental.py` `appends-only`; `validate-signatures.py` on `signed-then-edited.pdf` | | A merge copies a page that states no `/MediaBox` (and inherits none) without one, so the result breaks the rule that every page has a media box; qpdf repairs it by assuming Letter | #2 | `validate-pdf.py` on `inspected.pdf` and `with-warnings.pdf` (both merge a page of `NamedDests.pdf`) | -| A Flate stream has no limit on its decoded size: a 1.1 MB document inflates past 1 GiB | #12 | `flate-bomb.pdf`: `OutOfMemoryException` under a 1 GiB heap limit | -| A chain of indirect `/Length` references is resolved by recursion without a depth limit | #12 | `length-chain.pdf`: the process dies with a stack overflow, which no caller can catch | -| An object stream's `/N` sizes an allocation before anything checks it against the stream | #12 | `objstm-count.pdf` (513 bytes): `OutOfMemoryException` | | The JPEG decoder indexes its four Huffman tables with a scan's 4-bit selector unchecked | #12, #1 | `jpeg-selector.pdf`: `IndexOutOfRangeException` from `JpegDecoder.DecodeBlock` | | A dictionary key that is the empty name (`/` alone, which PDF syntax allows) makes `PdfDictionary.Set` throw `ArgumentException` while parsing, so the document does not open at all | #12 | pdf.js corpus: `issue3188.pdf`, `issue5896.pdf`, `pdfjs_wikipedia.pdf`, `poppler-742-0-fuzzed.pdf` | | A glyph whose box comes out upside down (a negative font size or a flipped font matrix) makes `PdfGlyph.BoundingBox` construct a `PdfRectangle` that throws, so text extraction faults | #12 | pdf.js corpus: `issue14415.pdf` | | `PdfDate.TryParse` throws `ArgumentOutOfRangeException` for a time-zone offset beyond 14 hours instead of answering false, so one malformed `/CreationDate` faults the whole inspection | #12 | pdf.js corpus: `TAMReview.pdf` | +Fixed since the list was first drawn up, each with the test that fails without its fix: + +| Defect | Fixed by | Test | +| --- | --- | --- | +| The signature validator accepted a `/ByteRange` gap holding more than the `/Contents` string (signature wrapping) | the gap must be exactly one hex string, equal to the dictionary's `/Contents` | `PdfSignatureAttackTests.Does_not_report_a_gap_that_holds_more_than_the_signature_value_as_covering_the_document` | +| No limit on the decoded size of a Flate, LZW or RunLength stream | `PdfEngineOptions.MaxDecodedStreamBytes` (128 MiB) | `PdfFilterTests.Refuses_a_flate_stream_that_decodes_past_the_limit` and its LZW, RunLength and content-stream companions | +| A chain of indirect `/Length` references overflowed the stack | the chain is followed no deeper than `MaxNestingDepth`, then the stream is found by its `endstream` | `PdfReaderRecoveryTests.Reads_a_stream_whose_length_is_a_chain_of_sixty_thousand_streams` | +| An object stream's `/N`, and a JPEG frame's width and height, sized allocations unchecked | `/N` is bounded by what the stream can hold; a JPEG above the caller's budget is measured from its header, and working sizes are computed in 64 bits | `PdfReaderRecoveryTests.Reads_an_object_stream_whose_count_its_header_cannot_hold`, `PdfJpegFilterTests.Measures_a_frame_too_large_for_the_budget_without_allocating_it` | + ## Corpus readings, 2026-11-16 | Corpus | Files | Read | Refused | Fault | Crash | Timeout | | --- | --- | --- | --- | --- | --- | --- | | veraPDF corpus, `PDF_A-2b` and `PDF_A-4` at `bb75f4f` | 1,471 | 1,471 | 0 | 0 | 0 | 0 | | pdf.js `test/pdfs` at `v6.4.299` | 987 | 977 | 4 | 6 | 0 | 0 | -| synthesized hostile documents | 4 | 0 | 0 | 3 | 1 | 0 | +| synthesized hostile documents, before the fixes | 4 | 0 | 0 | 3 | 1 | 0 | The six pdf.js faults are the three defects above. A sample of the pre-flight measurement (the first 60 files of `PDF_A-2b` and of `PDF_A-4`) wrote 108 results, refused 9 non-conforming inputs and refused 3 that diff --git a/src/SharpCorePDF.Infrastructure/Pdf/JpegDecoder.cs b/src/SharpCorePDF.Infrastructure/Pdf/JpegDecoder.cs index 2619fd8..528cf69 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/JpegDecoder.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/JpegDecoder.cs @@ -154,6 +154,16 @@ private bool Run(int maxBytes, out byte[]? pixels, out string? format, out int w return false; } + if (IsAboveBudget(maxBytes, out format, out width, out height)) + { + return format is not null; + } + + if (!WorkingBuffersFit()) + { + return false; + } + break; case 0xC2: progressive = true; @@ -163,6 +173,16 @@ private bool Run(int maxBytes, out byte[]? pixels, out string? format, out int w return false; } + if (IsAboveBudget(maxBytes, out format, out width, out height)) + { + return format is not null; + } + + if (!WorkingBuffersFit()) + { + return false; + } + break; case 0xDD: if (segment.Length < 2) @@ -320,6 +340,64 @@ private bool ParseHuffman(ReadOnlySpan segment) return true; } + /// + /// Answers, from the frame header alone, for an image whose samples the caller would not keep: the + /// format and the size are measured and nothing is decoded. + /// + /// The largest pixel buffer the caller keeps. + /// The pixel format the samples would state, or null for a component count the decoder has none for. + /// The width the frame states. + /// The height the frame states. + /// when the output is above the budget and the answer is final. + /// + /// The width and the height are two 16-bit numbers in the file: 65,535 squared is four gigasamples per + /// component, and the sample planes — and, for a progressive image, the coefficient grids at four bytes a + /// coefficient — were allocated and filled before the output was found too large to keep. The contract is + /// unchanged: such an image is measured and no pixels are returned. It is only no longer decoded first. + /// + private bool IsAboveBudget(int maxBytes, out string? format, out int measuredWidth, out int measuredHeight) + { + format = null; + measuredWidth = 0; + measuredHeight = 0; + + if ((long)width * height * components.Length <= maxBytes) + { + return false; + } + + format = components.Length switch + { + 1 => "Gray8", + 3 => "Rgb24", + 4 => "Cmyk32", + _ => null, + }; + measuredWidth = width; + measuredHeight = height; + return true; + } + + /// Determines whether the working buffers the frame asks for fit an array at all. + /// when every sample plane or coefficient grid fits. + /// The sizes are computed in 64 bits: in 32 they wrap, and a wrapped size is a small allocation that the decoder then writes past. + private bool WorkingBuffersFit() + { + var mcuColumns = (width + (8L * maxH) - 1) / (8L * maxH); + var mcuRows = (height + (8L * maxV) - 1) / (8L * maxV); + var perSample = progressive ? sizeof(int) : sizeof(byte); + + foreach (var component in components) + { + if (mcuColumns * 8 * component.H * mcuRows * 8 * component.V * perSample > Array.MaxLength) + { + return false; + } + } + + return true; + } + private bool ParseFrame(ReadOnlySpan segment) { if (segment.Length < 6 || segment[0] != 8) diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.Streams.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.Streams.cs index cec798b..7245f97 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.Streams.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.Streams.cs @@ -33,7 +33,7 @@ internal sealed partial class PdfCrossReferenceReader table.Add(new XrefEntry(objectNumber, offset, generation, XrefEntryType.Uncompressed)); } - var decoded = PdfFilters.Decode(stream, sourceName); + var decoded = PdfFilters.Decode(stream, sourceName, options.MaxDecodedStreamBytes); var widthType = 1; var widthField1 = 4; var widthField2 = 2; diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.ObjectStreams.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.ObjectStreams.cs index a208657..0a5df17 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.ObjectStreams.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.ObjectStreams.cs @@ -43,18 +43,24 @@ private void EnsureObjectStreamLoaded(long containerObjectNumber) } var dictionary = stream.Dictionary; - var count = (int)(dictionary.GetInteger("N") ?? 0); - var first = (int)(dictionary.GetInteger("First") ?? 0); + var declared = dictionary.GetInteger("N") ?? 0; + var first = dictionary.GetInteger("First") ?? 0; - if (count <= 0 || first < 0) + if (declared <= 0 || first < 0) { return; } - var decoded = PdfFilters.Decode(stream, SourceName); + var decoded = PdfFilters.Decode(stream, SourceName, options.MaxDecodedStreamBytes); + + // /N sizes two arrays, and it is a number the file states: 2^31 - 1 of them would ask for + // gigabytes from a stream of a few hundred bytes. Every pair of the header takes at least four + // bytes ("1 0 "), so the stream itself bounds how many objects it can describe, and the arrays + // are sized by that bound; a header that ends before /N says so below. + var count = (int)Math.Min(declared, (decoded.Length / 4L) + 1); var header = new PdfObjectParser(decoded, SourceName, 0, null, options.MaxNestingDepth); var numbers = new long[count]; - var offsets = new int[count]; + var offsets = new long[count]; for (var index = 0; index < count; index++) { @@ -62,12 +68,12 @@ private void EnsureObjectStreamLoaded(long containerObjectNumber) { warnings.Add(new( PdfWarningKind.ObjectStreamTruncated, - $"'{SourceName}': object stream {containerObjectNumber} declares {count} objects but its header ends early.")); + $"'{SourceName}': object stream {containerObjectNumber} declares {declared} objects but its header ends early.")); break; } numbers[index] = number; - offsets[index] = (int)offset; + offsets[index] = offset; } for (var index = 0; index < count; index++) @@ -79,7 +85,7 @@ private void EnsureObjectStreamLoaded(long containerObjectNumber) continue; } - var parser = new PdfObjectParser(decoded, SourceName, position, null, options.MaxNestingDepth); + var parser = new PdfObjectParser(decoded, SourceName, (int)position, null, options.MaxNestingDepth); fromObjectStreams[numbers[index]] = parser.ParseObject(); } } diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.Objects.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.Objects.cs index 1a39e97..4eeaa67 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.Objects.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.Objects.cs @@ -72,23 +72,33 @@ private PdfVersion ReadVersion() /// /// Resolves a stream /Length that is itself an indirect reference. The guard set breaks - /// the cycle that such a reference creates when the length object is the stream itself. + /// the cycle that such a reference creates when the length object is the stream itself, and the + /// depth guard breaks a chain: a length that is a stream whose length is a stream, and so on. /// /// The object number of the length object. /// The length, or when it cannot be resolved safely. + /// + /// Each link of a chain is resolved inside the one before it, so a chain of tens of thousands of + /// streams would overflow the stack — which no caller can catch: the process ends. A well-formed + /// length needs one level; past the length is answered + /// as unknown, exactly as a cycle is, and the parser finds the end of the stream by its keyword. + /// private long? ResolveIndirectLength(long objectNumber) { - if (!resolvingLengths.Add(objectNumber)) + if (lengthResolutionDepth >= options.MaxNestingDepth || !resolvingLengths.Add(objectNumber)) { return null; } + lengthResolutionDepth++; + try { return Resolve(objectNumber) is PdfInteger integer ? integer.Value : null; } finally { + lengthResolutionDepth--; resolvingLengths.Remove(objectNumber); } } diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.cs index 0c1d647..fcec760 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.cs @@ -20,6 +20,7 @@ internal sealed partial class PdfFileReader private readonly HashSet loadedObjectStreams = []; private readonly HashSet resolvingObjectStreams = []; private readonly HashSet resolvingLengths = []; + private int lengthResolutionDepth; private PageTreeWalk? pageTree; private PdfDictionary? catalog; diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.Inflation.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.Inflation.cs index e3c2f47..8fe69f2 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.Inflation.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.Inflation.cs @@ -12,30 +12,79 @@ internal static partial class PdfFilters private const int InflateChunkSize = 16 * 1024; private const int MaximumInitialInflateCapacity = 1024 * 1024; - /// Attempts to decompress Flate data, tolerating the raw deflate streams some producers write. - /// The compressed bytes. - /// The decompressed bytes when decompression succeeded. - /// when the data could be decompressed. - public static bool TryInflate(ReadOnlyMemory data, out byte[] result) + /// + /// The decoded size a stream is held to when the caller has no to read it + /// from; the same value as 's default. + /// + internal const long DefaultMaxDecodedBytes = 128L * 1024 * 1024; + + /// What one attempt at inflating a stream came to. + private enum InflateOutcome { - result = []; + /// The stream inflated within the limit. + Inflated, + + /// The bytes are not a stream this wrapper can inflate. + Invalid, - return !data.IsEmpty - && (TryInflateCore(data, zlibWrapper: true, out result) - || TryInflateCore(data, zlibWrapper: false, out result)); + /// The stream inflates past the limit; it was stopped there. + TooLarge, } - /// Decompresses Flate data. + /// Attempts to decompress Flate data, tolerating the raw deflate streams some producers write. + /// The compressed bytes. + /// The decompressed bytes when decompression succeeded. + /// when the data could be decompressed within the default limit. + public static bool TryInflate(ReadOnlyMemory data, out byte[] result) => + TryInflate(data, DefaultMaxDecodedBytes, out result) == InflateOutcome.Inflated; + + /// Decompresses Flate data under the default limit on its decoded size. /// The compressed bytes. /// The name of the source, used in diagnostics. /// The decompressed bytes. - /// The data could not be decompressed. + /// The data could not be decompressed, or decodes past the limit. public static byte[] Inflate(ReadOnlyMemory data, string sourceName) => - TryInflate(data, out var result) - ? result - : throw new PdfSourceFormatException(sourceName, "a Flate compressed stream could not be decompressed."); + Inflate(data, sourceName, DefaultMaxDecodedBytes); - private static bool TryInflateCore(ReadOnlyMemory data, bool zlibWrapper, out byte[] result) + /// Decompresses Flate data, refusing a stream that decodes to more than a limit. + /// The compressed bytes. + /// The name of the source, used in diagnostics. + /// The largest number of bytes the stream may decode to. + /// The decompressed bytes. + /// The data could not be decompressed, or decodes past the limit. + public static byte[] Inflate(ReadOnlyMemory data, string sourceName, long maxDecodedBytes) => + TryInflate(data, maxDecodedBytes, out var result) switch + { + InflateOutcome.Inflated => result, + InflateOutcome.TooLarge => throw DecodedTooLarge(sourceName, "a Flate", maxDecodedBytes), + _ => throw new PdfSourceFormatException(sourceName, "a Flate compressed stream could not be decompressed."), + }; + + /// Builds the refusal of a stream that decodes past the limit. + /// The name of the source. + /// The kind of stream, for the message. + /// The limit that was reached. + /// The exception to throw. + internal static PdfSourceFormatException DecodedTooLarge(string sourceName, string what, long maxDecodedBytes) => + new(sourceName, $"{what} stream decodes to more than {maxDecodedBytes} bytes, the limit PdfEngineOptions.MaxDecodedStreamBytes sets, so it was not decoded."); + + private static InflateOutcome TryInflate(ReadOnlyMemory data, long maxDecodedBytes, out byte[] result) + { + result = []; + + if (data.IsEmpty) + { + return InflateOutcome.Invalid; + } + + var outcome = TryInflateCore(data, zlibWrapper: true, maxDecodedBytes, out result); + + return outcome == InflateOutcome.Invalid + ? TryInflateCore(data, zlibWrapper: false, maxDecodedBytes, out result) + : outcome; + } + + private static InflateOutcome TryInflateCore(ReadOnlyMemory data, bool zlibWrapper, long maxDecodedBytes, out byte[] result) { result = []; @@ -51,7 +100,8 @@ private static bool TryInflateCore(ReadOnlyMemory data, bool zlibWrapper, // stream surprises us. Growth is progressive instead of one guess, so a small stream never // reserves the memory of a large one. using var output = new PooledByteBuffer( - (int)Math.Clamp((long)data.Length * 4, InflateChunkSize, MaximumInitialInflateCapacity)); + (int)Math.Clamp((long)data.Length * 4, InflateChunkSize, MaximumInitialInflateCapacity), + poolLargeArrays: false); while (true) { @@ -63,19 +113,27 @@ private static bool TryInflateCore(ReadOnlyMemory data, bool zlibWrapper, break; } + // Flate reaches about a thousand to one, so the limit on the input does not bound this + // buffer: a megabyte of zeros is a gigabyte here. The stream is stopped at the limit + // instead of being allowed to take the process with it. + if (output.Length + (long)read > maxDecodedBytes) + { + return InflateOutcome.TooLarge; + } + output.Advance(read); } result = output.ToExactArray(); - return true; + return InflateOutcome.Inflated; } catch (InvalidDataException) { - return false; + return InflateOutcome.Invalid; } catch (IOException) { - return false; + return InflateOutcome.Invalid; } } @@ -91,9 +149,9 @@ private static MemoryStream OpenInput(ReadOnlyMemory data) => ? new MemoryStream(array, segment.Offset, segment.Count, writable: false) : new MemoryStream(data.ToArray(), writable: false); - private static byte[] ApplyFlate(ReadOnlyMemory data, PdfDictionary? parameters, string sourceName) + private static byte[] ApplyFlate(ReadOnlyMemory data, PdfDictionary? parameters, string sourceName, long maxDecodedBytes) { - var inflated = Inflate(data, sourceName); + var inflated = Inflate(data, sourceName, maxDecodedBytes); // Every parameter of the filter is optional and every one of them is absent when the stream has no // filter dictionary at all, so the dictionary is established once here instead of being tested @@ -134,5 +192,5 @@ private static byte[] ApplyFlate(ReadOnlyMemory data, PdfDictionary? param public static byte[] InflateAndUnpredict( ReadOnlyMemory data, PdfDictionary? parameters, - string sourceName) => ApplyFlate(data, parameters, sourceName); + string sourceName) => ApplyFlate(data, parameters, sourceName, DefaultMaxDecodedBytes); } diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.Lzw.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.Lzw.cs index ee2e587..fd2162a 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.Lzw.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.Lzw.cs @@ -143,18 +143,31 @@ internal static int ReadEarlyChange(PdfDictionary? parameters) => /// The compressed bytes. /// The decode parameters of the filter, which state EarlyChange. /// The name of the source, used in diagnostics. + /// The largest number of bytes the stream may decode to. /// The decoded bytes. - /// The stream is not a valid LZW stream. + /// The stream is not a valid LZW stream, or decodes past the limit. internal static byte[] ApplyLzw( ReadOnlyMemory data, PdfDictionary? parameters, - string sourceName) + string sourceName, + long maxDecodedBytes) { - var output = new PooledByteBuffer(); + var output = new PooledByteBuffer(poolLargeArrays: false); try { - if (!TryDecodeLzw(data.Span, ReadEarlyChange(parameters), output.Write, out _)) + // A 12-bit code can stand for 4,096 bytes, so the input limit does not bound this buffer either. + void Write(ReadOnlySpan bytes) + { + if (output.Length + (long)bytes.Length > maxDecodedBytes) + { + throw DecodedTooLarge(sourceName, "an LZW", maxDecodedBytes); + } + + output.Write(bytes); + } + + if (!TryDecodeLzw(data.Span, ReadEarlyChange(parameters), Write, out _)) { throw new PdfSourceFormatException(sourceName, "an LZW stream could not be decompressed."); } diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.RunLength.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.RunLength.cs index 2acb163..2b029b1 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.RunLength.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.RunLength.cs @@ -24,11 +24,21 @@ internal static partial class PdfFilters /// reaches past the end contributes the bytes that are there, and a repeated run with no value byte /// contributes nothing. /// - public static byte[] DecodeRunLength(ReadOnlyMemory data) + public static byte[] DecodeRunLength(ReadOnlyMemory data) => + DecodeRunLength(data, "a stream", DefaultMaxDecodedBytes); + + /// Decodes a run-length encoded stream, refusing one that decodes past a limit. + /// The encoded bytes. + /// The name of the source, used in diagnostics. + /// The largest number of bytes the stream may decode to. + /// The decoded bytes. + /// The stream decodes past the limit. + /// Two bytes can stand for 128, so the limit on the input does not bound the output. + public static byte[] DecodeRunLength(ReadOnlyMemory data, string sourceName, long maxDecodedBytes) { // The output is at least as large as the input for the literal case, and can be 128 times larger // for the repeat case, so the buffer starts at the input length and grows only when it must. - using var result = new PooledByteBuffer(data.Length); + using var result = new PooledByteBuffer(data.Length, poolLargeArrays: false); var span = data.Span; var index = 0; @@ -48,6 +58,11 @@ public static byte[] DecodeRunLength(ReadOnlyMemory data) // end, which ends the loop. The explicit guard a survivor asked about is this comment. var count = Math.Min(control + 1, span.Length - index); + if (result.Length + (long)count > maxDecodedBytes) + { + throw DecodedTooLarge(sourceName, "a RunLength", maxDecodedBytes); + } + result.Write(span.Slice(index, count)); index += count; continue; @@ -60,6 +75,11 @@ public static byte[] DecodeRunLength(ReadOnlyMemory data) break; } + if (result.Length + (long)repeat > maxDecodedBytes) + { + throw DecodedTooLarge(sourceName, "a RunLength", maxDecodedBytes); + } + var value = span[index++]; for (var count = 0; count < repeat; count++) diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.cs index 261f5a8..5552260 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.cs @@ -27,7 +27,17 @@ public static bool HasFlateCompression(PdfDictionary dictionary) /// The decoded bytes. /// The stream uses a filter the engine does not implement. /// A compressed stream could not be decompressed. - public static byte[] Decode(PdfStream stream, string sourceName) + public static byte[] Decode(PdfStream stream, string sourceName) => + Decode(stream, sourceName, DefaultMaxDecodedBytes); + + /// Decodes a stream, applying every filter it declares in order, under a limit on its decoded size. + /// The stream to decode. + /// The name of the source, used in diagnostics. + /// The largest number of bytes any one filter may decode to. + /// The decoded bytes. + /// The stream uses a filter the engine does not implement. + /// A compressed stream could not be decompressed, or decodes past the limit. + public static byte[] Decode(PdfStream stream, string sourceName, long maxDecodedBytes) { ArgumentNullException.ThrowIfNull(stream); @@ -51,8 +61,8 @@ public static byte[] Decode(PdfStream stream, string sourceName) decoded = name switch { - "FlateDecode" or "Fl" => ApplyFlate(current, parametersForFilter, sourceName), - "LZWDecode" or "LZW" => ApplyLzw(current, parametersForFilter, sourceName), + "FlateDecode" or "Fl" => ApplyFlate(current, parametersForFilter, sourceName, maxDecodedBytes), + "LZWDecode" or "LZW" => ApplyLzw(current, parametersForFilter, sourceName, maxDecodedBytes), "ASCIIHexDecode" or "AHx" => DecodeAsciiHex(current), "ASCII85Decode" or "A85" => DecodeAscii85(current), _ => throw new PdfUnsupportedFeatureException( diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfFontDecoder.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfFontDecoder.cs index 7a31da3..33e41d8 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfFontDecoder.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfFontDecoder.cs @@ -313,7 +313,7 @@ private static PdfFont Create(PdfFileReader reader, string resourceName, PdfDict try { - data = PdfFilters.Decode(stream, reader.SourceName); + data = PdfFilters.Decode(stream, reader.SourceName, reader.Options.MaxDecodedStreamBytes); } catch (PdfUnsupportedFeatureException) { diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfImageFilters.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfImageFilters.cs index afa4917..15951b7 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfImageFilters.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfImageFilters.cs @@ -23,7 +23,7 @@ internal readonly record struct PdfPayload( /// /// /// -/// answers a different question and answers it correctly: an object +/// answers a different question and answers it correctly: an object /// stream that cannot be inflated means a broken document, so it raises. An image is different — a /// JPXDecode scan is a perfectly valid document this library simply does not decode — so this /// class reports what it could not do and hands back the raw bytes (NFR-7.6, NFR-7.8). @@ -361,7 +361,7 @@ private static bool TryDecodeOne( case "RunLengthDecode": { - var decoded = PdfFilters.DecodeRunLength(data); + var decoded = PdfFilters.DecodeRunLength(data, sourceName, PdfFilters.DefaultMaxDecodedBytes); length = decoded.Length; bytes = Keep(decoded, cap); diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfTextCollector.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfTextCollector.cs index 5a68441..b99551d 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfTextCollector.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfTextCollector.cs @@ -382,7 +382,7 @@ private void PaintForm(string name, PdfDictionary? resources, TextState state, i try { - if (!TryDecodeContent(stream, out var formContent)) + if (!TryDecodeContent(stream, reader.Options.MaxDecodedStreamBytes, out var formContent)) { warnings.Add(new( PdfWarningKind.ContentStreamUnreadable, @@ -432,13 +432,14 @@ private string DescribeUnmapped(PdfFont font) => string.Create( /// Decodes the content stream of a form. /// The stream. + /// The largest number of bytes the stream may decode to. /// The decoded bytes when decoding succeeded. /// when the stream could be decoded. - private static bool TryDecodeContent(PdfStream stream, out byte[] content) + private static bool TryDecodeContent(PdfStream stream, long maxDecodedBytes, out byte[] content) { try { - content = PdfFilters.Decode(stream, "form content stream"); + content = PdfFilters.Decode(stream, "form content stream", maxDecodedBytes); return true; } catch (Domain.Exceptions.PdfUnsupportedFeatureException) diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfType1Encoding.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfType1Encoding.cs index 2a19346..84eddd9 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfType1Encoding.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfType1Encoding.cs @@ -52,7 +52,7 @@ internal static class PdfType1Encoding try { - bytes = PdfFilters.Decode(program, reader.SourceName); + bytes = PdfFilters.Decode(program, reader.SourceName, reader.Options.MaxDecodedStreamBytes); } catch (PdfUnsupportedFeatureException) { diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PooledByteBuffer.cs b/src/SharpCorePDF.Infrastructure/Pdf/PooledByteBuffer.cs index 12615ac..5a9f02c 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PooledByteBuffer.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PooledByteBuffer.cs @@ -24,17 +24,32 @@ internal sealed class PooledByteBuffer : IDisposable { private const int MinimumCapacity = 16; + /// + /// The largest array a decode buffer takes from the pool. Above it the array is allocated and left to the + /// collector, because the shared pool keeps what is returned to it: a document that inflates a stream to + /// the decode limit in every read pass would otherwise leave hundreds of megabytes parked in the pool, + /// and the next pass would allocate beside them. + /// + private const int MaximumPooledDecodeLength = 16 * 1024 * 1024; + private byte[] array; private int length; + private readonly bool poolLargeArrays; private bool returned; /// Initializes a new instance of the class. /// The number of bytes to rent up front. - public PooledByteBuffer(int initialCapacity = 256) + /// + /// Whether arrays above 16 MiB go through the pool too. The read of a source keeps them pooled, because + /// reusing the one buffer a document is read into is what keeps a large read at one allocation; a decode + /// buffer does not, because its size is whatever a stream inflates to and the pool would keep it. + /// + public PooledByteBuffer(int initialCapacity = 256, bool poolLargeArrays = true) { ArgumentOutOfRangeException.ThrowIfNegative(initialCapacity); - array = ArrayPool.Shared.Rent(Math.Max(initialCapacity, MinimumCapacity)); + this.poolLargeArrays = poolLargeArrays; + array = Rent(Math.Max(initialCapacity, MinimumCapacity)); } /// Gets the number of bytes that have been written. @@ -112,7 +127,7 @@ public void Dispose() } returned = true; - ArrayPool.Shared.Return(array); + Return(array); array = []; length = 0; } @@ -124,14 +139,36 @@ private void EnsureFree(int sizeHint) return; } - Grow(length + sizeHint); + Grow((long)length + sizeHint); } - private void Grow(int minimumCapacity) + private void Grow(long minimumCapacity) { - var grown = ArrayPool.Shared.Rent(Math.Max(minimumCapacity, array.Length * 2)); + // Doubling is computed in 64 bits and clamped to the largest array the runtime allows, so a buffer + // past a gigabyte grows to the limit instead of to a negative size; a request beyond the limit is + // refused. The decode limits keep every caller below it, so this is a guard and not a path. + if (minimumCapacity > Array.MaxLength) + { + throw new InvalidOperationException($"A buffer of {minimumCapacity} bytes is larger than the runtime allows."); + } + + var capacity = (int)Math.Min(Math.Max(minimumCapacity, 2L * array.Length), Array.MaxLength); + var grown = Rent(capacity); array.AsSpan(0, length).CopyTo(grown); - ArrayPool.Shared.Return(array); + Return(array); array = grown; } + + private byte[] Rent(int length) => + poolLargeArrays || length <= MaximumPooledDecodeLength + ? ArrayPool.Shared.Rent(length) + : GC.AllocateUninitializedArray(length); + + private void Return(byte[] rented) + { + if (poolLargeArrays || rented.Length <= MaximumPooledDecodeLength) + { + ArrayPool.Shared.Return(rented); + } + } } diff --git a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Overlays.cs b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Overlays.cs index 7994f4e..23e7ab5 100644 --- a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Overlays.cs +++ b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Overlays.cs @@ -252,7 +252,7 @@ private static void AppendOverlayContent(PdfFileReader reader, PdfObject? value, return; } - content.Write(PdfFilters.Decode(stream, reader.SourceName)); + content.Write(PdfFilters.Decode(stream, reader.SourceName, reader.Options.MaxDecodedStreamBytes)); content.WriteByte((byte)'\n'); } diff --git a/src/SharpCorePDF.Infrastructure/PdfEngineOptions.cs b/src/SharpCorePDF.Infrastructure/PdfEngineOptions.cs index 3cdff45..8af71bd 100644 --- a/src/SharpCorePDF.Infrastructure/PdfEngineOptions.cs +++ b/src/SharpCorePDF.Infrastructure/PdfEngineOptions.cs @@ -18,6 +18,24 @@ public sealed record PdfEngineOptions /// public long MaxSourceBytes { get; init; } = 512L * 1024 * 1024; + /// + /// Gets the largest number of bytes one filter of one stream may decode to. A stream that would decode + /// past it is refused with a instead of + /// exhausting the process. + /// + /// + /// bounds what is read, not what it decodes to: Flate reaches about a + /// thousand to one, LZW four thousand and RunLength 128, so a one-megabyte document can ask for a + /// gigabyte. The default of 128 MiB is far above any real content stream, object stream or font program, + /// and low enough that refusing a stream at the limit — whose buffer doubles on the way there, on the + /// large-object heap, which the collector does not compact by default — stays reliably inside a 1 GiB + /// process when every read pass meets the same stream; the largest value accepted is the largest array + /// the runtime allows. An image is not refused by this limit — its samples are kept only + /// within already — but every other stream the engine has to + /// materialise is: content, forms, object streams, cross-reference streams and font programs. + /// + public long MaxDecodedStreamBytes { get; init; } = 128L * 1024 * 1024; + /// Gets the maximum nesting depth of dictionaries and arrays. public int MaxNestingDepth { get; init; } = 64; @@ -74,6 +92,11 @@ public IReadOnlyList Validate() errors.Add($"MaxSourceBytes must be at least 1024, but is {MaxSourceBytes}."); } + if (MaxDecodedStreamBytes < 1024 || MaxDecodedStreamBytes > Array.MaxLength) + { + errors.Add($"MaxDecodedStreamBytes must be between 1024 and {Array.MaxLength}, but is {MaxDecodedStreamBytes}."); + } + if (MaxNestingDepth is < 4 or > 1024) { errors.Add($"MaxNestingDepth must be between 4 and 1024, but is {MaxNestingDepth}."); diff --git a/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Content.cs b/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Content.cs index ae14104..6eaa240 100644 --- a/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Content.cs +++ b/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Content.cs @@ -64,7 +64,7 @@ public sealed partial class PdfPageContentReader foreach (var stream in streams) { - if (TryDecodeStream(stream, out var decoded)) + if (TryDecodeStream(stream, reader.Options.MaxDecodedStreamBytes, out var decoded)) { // The separator goes between the streams, not after the last one: a single stream is its // own sequence, and an inline image that runs to the end of it must not gain a byte. @@ -114,13 +114,14 @@ private static void CollectStreams(PdfFileReader reader, PdfArray array, ListDecodes a stream whose filters are the ones a content stream may use. /// The stream. + /// The largest number of bytes the stream may decode to. /// The decoded bytes when decoding succeeded. /// when the stream could be decoded. - private static bool TryDecodeStream(PdfStream stream, out byte[] decoded) + private static bool TryDecodeStream(PdfStream stream, long maxDecodedBytes, out byte[] decoded) { try { - decoded = PdfFilters.Decode(stream, "content stream"); + decoded = PdfFilters.Decode(stream, "content stream", maxDecodedBytes); return true; } catch (Domain.Exceptions.PdfUnsupportedFeatureException) diff --git a/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Images.cs b/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Images.cs index 9f9c48d..e56a77b 100644 --- a/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Images.cs +++ b/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Images.cs @@ -358,7 +358,7 @@ private void Paint( return; } - if (TryDecodeStream(stream, out var formContent)) + if (TryDecodeStream(stream, reader.Options.MaxDecodedStreamBytes, out var formContent)) { var formResources = reader.ResolveDictionary(stream.Dictionary.Get("Resources")) ?? resources; var formMatrix = ReadMatrix(reader, stream.Dictionary.Get("Matrix")) ?? PdfMatrix.Identity; diff --git a/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Metadata.cs b/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Metadata.cs index 027f05b..3681e29 100644 --- a/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Metadata.cs +++ b/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Metadata.cs @@ -35,7 +35,7 @@ public async Task ReadMetadataAsync( return new PdfXmpResult { Warnings = [.. reader.Warnings, .. warnings] }; } - if (!TryDecodeStream(stream, out var packet)) + if (!TryDecodeStream(stream, reader.Options.MaxDecodedStreamBytes, out var packet)) { warnings.Add(new( PdfWarningKind.XmpPacketMalformed, diff --git a/src/SharpCorePDF.Infrastructure/PublicAPI.Unshipped.txt b/src/SharpCorePDF.Infrastructure/PublicAPI.Unshipped.txt index 7dc5c58..f73aab7 100644 --- a/src/SharpCorePDF.Infrastructure/PublicAPI.Unshipped.txt +++ b/src/SharpCorePDF.Infrastructure/PublicAPI.Unshipped.txt @@ -1 +1,3 @@ #nullable enable +SharpCorePDF.Infrastructure.PdfEngineOptions.MaxDecodedStreamBytes.get -> long +SharpCorePDF.Infrastructure.PdfEngineOptions.MaxDecodedStreamBytes.init -> void diff --git a/src/SharpCorePDF.Infrastructure/Signatures/SignatureValidator.cs b/src/SharpCorePDF.Infrastructure/Signatures/SignatureValidator.cs index d0254eb..74bb5f0 100644 --- a/src/SharpCorePDF.Infrastructure/Signatures/SignatureValidator.cs +++ b/src/SharpCorePDF.Infrastructure/Signatures/SignatureValidator.cs @@ -258,7 +258,7 @@ public static PdfSignatureValidation Validate(PdfFileReader reader, SignatureFie $"The timestamp of '{field.FieldName}' is present but could not be read, so the moment it claims cannot be relied on.")); } - if (reading.Usable && !reading.CoversWhole) + if (reading.Usable && reading.GapHoldsValue && !reading.CoversWhole) { var revisions = CountRevisions(reader.Data, reader.Data.LongLength); @@ -278,6 +278,7 @@ public static PdfSignatureValidation Validate(PdfFileReader reader, SignatureFie /// The covered ranges. /// Whether the dictionary carries four usable numbers. /// Whether the ranges are ordered and inside the document. + /// Whether the bytes the ranges leave out are exactly the signature value. /// The DER structure of the signature value, without its padding. /// The parsed signature, when it parsed. /// Why the signature value could not be parsed, when it could not. @@ -287,6 +288,7 @@ private sealed record Reading( PdfByteRange Range, bool HasRange, bool Usable, + bool GapHoldsValue, byte[]? Contents, CmsSignature? Cms, Exception? Failure, @@ -314,6 +316,7 @@ private static Reading Read(PdfFileReader reader, PdfDictionary dictionary) && range.IsWellFormed && range.Length1 > 0 && range.Offset2 + range.Length2 <= reader.Data.LongLength; + var gapHoldsValue = usable && GapHoldsContents(reader, dictionary, range); var contents = ReadContents(reader, dictionary); CmsSignature? cms = null; Exception? failure = null; @@ -334,13 +337,72 @@ private static Reading Read(PdfFileReader reader, PdfDictionary dictionary) range, hasRange, usable, + gapHoldsValue, contents, cms, failure, - usable && range.Covers(reader.Data.LongLength), + usable && gapHoldsValue && range.Covers(reader.Data.LongLength), usable ? CountRevisions(reader.Data, range.Offset2 + range.Length2) : 0); } + /// + /// Determines whether the bytes the byte range leaves uncovered are exactly the signature value: one + /// hexadecimal string, delimiters included, and nothing else. + /// + /// The opened document. + /// The signature dictionary. + /// The byte range, already known to be ordered and inside the document. + /// when the gap holds the value and nothing beside it. + /// + /// The gap is never hashed, so anything in it is unsigned. A gap that closes the value early and carries + /// entries or objects of its own still leaves every signed byte intact and the digest matching — the + /// signature-wrapping attack — so the digest cannot be what catches it. A gap that is a single hex string + /// can only carry the value itself, and changing that breaks the digest. The value is also compared with + /// the /Contents the dictionary resolves to, so a dictionary that points its value somewhere else + /// cannot leave an unrelated string in the gap; a document whose strings were decrypted on the way in is + /// compared by shape only, because the parser hands over the decrypted form. + /// + private static bool GapHoldsContents(PdfFileReader reader, PdfDictionary dictionary, PdfByteRange range) + { + var data = reader.Data; + var start = checked((int)(range.Offset1 + range.Length1)); + var end = checked((int)range.Offset2); + + if (end - start < 2 || data[start] != (byte)'<' || data[end - 1] != (byte)'>') + { + return false; + } + + var inner = data.AsSpan(start + 1, end - start - 2); + var digits = new List(inner.Length); + + foreach (var value in inner) + { + if (Uri.IsHexDigit((char)value)) + { + digits.Add(value); + } + else if (!PdfLexer.IsWhitespace(value)) + { + return false; + } + } + + if (reader.Security is not null) + { + return true; + } + + // An odd number of digits is completed with a zero, as the specification reads a hex string. + if (digits.Count % 2 == 1) + { + digits.Add((byte)'0'); + } + + return reader.Resolve(dictionary.Get("Contents")) is PdfString contents + && Convert.FromHexString(System.Text.Encoding.ASCII.GetString(digits.ToArray())).AsSpan().SequenceEqual(contents.Bytes); + } + /// Reads the four numbers of the byte range. /// The opened document. /// The signature dictionary. @@ -469,6 +531,15 @@ private static PdfSignatureIntegrity Check( return PdfSignatureIntegrity.Unknown; } + if (!reading.GapHoldsValue) + { + problems.Add(new PdfSignatureProblem( + PdfSignatureProblemKind.MalformedByteRange, + $"The signature '{fieldName}' leaves bytes {reading.Range.Offset1 + reading.Range.Length1} to {reading.Range.Offset2} uncovered, and they are not exactly its /Contents value: bytes no signature covers would be read as part of the document.")); + + return PdfSignatureIntegrity.Unknown; + } + if (reading.Contents is not { Length: > 0 }) { problems.Add(new PdfSignatureProblem( diff --git a/tests/SharpCorePDF.Tests/PdfEngineOptionsTests.cs b/tests/SharpCorePDF.Tests/PdfEngineOptionsTests.cs index 9252147..5d58175 100644 --- a/tests/SharpCorePDF.Tests/PdfEngineOptionsTests.cs +++ b/tests/SharpCorePDF.Tests/PdfEngineOptionsTests.cs @@ -19,6 +19,7 @@ public void Has_the_documented_defaults() var options = PdfEngineOptions.Default; Assert.Equal(512L * 1024 * 1024, options.MaxSourceBytes); + Assert.Equal(128L * 1024 * 1024, options.MaxDecodedStreamBytes); Assert.Equal(64, options.MaxNestingDepth); Assert.Equal(100_000, options.MaxPagesPerSource); Assert.True(options.ReconstructCrossReferenceOnFailure); @@ -44,7 +45,16 @@ public void Reports_a_limit_that_cannot_be_honoured() (PdfEngineOptions.Default with { MaxPagesPerSource = 0 }).Validate(), error => error.Contains("MaxPagesPerSource", StringComparison.Ordinal)); + Assert.Contains( + (PdfEngineOptions.Default with { MaxDecodedStreamBytes = 1023 }).Validate(), + error => error.Contains("MaxDecodedStreamBytes", StringComparison.Ordinal)); + Assert.Contains( + (PdfEngineOptions.Default with { MaxDecodedStreamBytes = Array.MaxLength + 1L }).Validate(), + error => error.Contains("MaxDecodedStreamBytes", StringComparison.Ordinal)); + Assert.Empty((PdfEngineOptions.Default with { MaxSourceBytes = 1024 }).Validate()); + Assert.Empty((PdfEngineOptions.Default with { MaxDecodedStreamBytes = 1024 }).Validate()); + Assert.Empty((PdfEngineOptions.Default with { MaxDecodedStreamBytes = Array.MaxLength }).Validate()); Assert.Empty((PdfEngineOptions.Default with { MaxNestingDepth = 1024 }).Validate()); Assert.Empty((PdfEngineOptions.Default with { MaxPagesPerSource = 1 }).Validate()); } diff --git a/tests/SharpCorePDF.Tests/PdfFilterTests.cs b/tests/SharpCorePDF.Tests/PdfFilterTests.cs index 65cf9f2..99e6525 100644 --- a/tests/SharpCorePDF.Tests/PdfFilterTests.cs +++ b/tests/SharpCorePDF.Tests/PdfFilterTests.cs @@ -1,7 +1,11 @@ using System.IO.Compression; using System.Text; using SharpCorePDF.Domain.Exceptions; +using SharpCorePDF.Application.Services; +using SharpCorePDF.Domain.ValueObjects; +using SharpCorePDF.Infrastructure; using SharpCorePDF.Infrastructure.Pdf; +using SharpCorePDF.Infrastructure.Sources; namespace SharpCorePDF.Tests; @@ -144,6 +148,98 @@ public void Inflates_flate_data_however_it_was_written() Assert.Equal(text, raw); } + /// + /// A Flate stream that decodes past the limit is refused rather than inflated: zeros deflate about a + /// thousand to one, so the input limit alone lets a small document ask for a gigabyte. + /// + [Fact] + public void Refuses_a_flate_stream_that_decodes_past_the_limit() + { + var bomb = Deflate(new byte[2 * 1024 * 1024]); + + Assert.True(bomb.Length < 16 * 1024, "the fixture has to be small for the test to say anything"); + + var error = Assert.Throws(() => PdfFilters.Inflate(bomb, "bomb.pdf", 1024 * 1024)); + + Assert.Contains("MaxDecodedStreamBytes", error.Message, StringComparison.Ordinal); + Assert.Equal(2 * 1024 * 1024, PdfFilters.Inflate(bomb, "bomb.pdf", 2 * 1024 * 1024).Length); + Assert.True(PdfFilters.TryInflate(Deflate(new byte[1024]), out var small)); + Assert.Equal(1024, small.Length); + } + + /// An LZW stream is held to the same limit: one 12-bit code can stand for 4,096 bytes. + [Fact] + public void Refuses_an_lzw_stream_that_decodes_past_the_limit() + { + var encoded = TestLzw.Encode(new byte[100_000]); + + var error = Assert.Throws(() => PdfFilters.ApplyLzw(encoded, null, "lzw.pdf", 99_999)); + + Assert.Contains("MaxDecodedStreamBytes", error.Message, StringComparison.Ordinal); + Assert.Contains("LZW", error.Message, StringComparison.Ordinal); + Assert.Equal(100_000, PdfFilters.ApplyLzw(encoded, null, "lzw.pdf", 100_000).Length); + } + + /// A RunLength stream is held to the limit too: two bytes stand for 128. + [Fact] + public void Refuses_a_run_length_stream_that_decodes_past_the_limit() + { + var runs = Enumerable.Repeat(new byte[] { 0x81, 0x00 }, 1000).SelectMany(run => run).ToArray(); + + var error = Assert.Throws(() => PdfFilters.DecodeRunLength(runs, "runs.pdf", 127_999)); + + Assert.Contains("MaxDecodedStreamBytes", error.Message, StringComparison.Ordinal); + Assert.Contains("RunLength", error.Message, StringComparison.Ordinal); + Assert.Equal(128_000, PdfFilters.DecodeRunLength(runs, "runs.pdf", 128_000).Length); + } + + /// Literal runs are held to the limit on the same exact boundary as repeated ones. + [Fact] + public void Refuses_literal_runs_that_decode_past_the_limit() + { + byte[] run = [0x7F, .. Enumerable.Range(0, 128).Select(value => (byte)value)]; + var runs = Enumerable.Repeat(run, 100).SelectMany(bytes => bytes).ToArray(); + + var error = Assert.Throws(() => PdfFilters.DecodeRunLength(runs, "runs.pdf", 12_799)); + + Assert.Contains("RunLength", error.Message, StringComparison.Ordinal); + Assert.Equal(12_800, PdfFilters.DecodeRunLength(runs, "runs.pdf", 12_800).Length); + } + + /// + /// A page whose content stream decodes past the limit is reported as unreadable, the way any other + /// undecodable content is, and the read finishes: the limit turns a bomb into a warning. + /// + [Fact] + public async Task Reports_a_content_stream_past_the_limit_as_unreadable_instead_of_inflating_it() + { + var builder = new PdfBuilder(); + var catalog = builder.Reserve(); + var pages = builder.Reserve(); + + builder.Add("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Contents 4 0 R >>"); + builder.AddStream("/Filter /FlateDecode", PdfBuilder.Compress(new byte[1024 * 1024])); + builder.Set(catalog, "<< /Type /Catalog /Pages 2 0 R >>"); + builder.Set(pages, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + var document = builder.Build(catalog); + + var limited = new PdfContentReaderService( + new PdfPageContentReader(PdfEngineOptions.Default with { MaxDecodedStreamBytes = 64 * 1024 })); + var text = await limited.ReadTextAsync( + new PdfByteArraySource(document, "bomb.pdf"), + PageRange.All, + cancellationToken: TestContext.Current.CancellationToken); + + Assert.Contains(text.Warnings, warning => warning.Kind == PdfWarningKind.ContentStreamUnreadable); + + var unlimited = await new PdfContentReaderService(new PdfPageContentReader()).ReadTextAsync( + new PdfByteArraySource(document, "bomb.pdf"), + PageRange.All, + cancellationToken: TestContext.Current.CancellationToken); + + Assert.DoesNotContain(unlimited.Warnings, warning => warning.Kind == PdfWarningKind.ContentStreamUnreadable); + } + /// Flate data that cannot be decompressed is refused and reported as a damaged source. [Fact] public void Reports_data_that_cannot_be_decompressed() diff --git a/tests/SharpCorePDF.Tests/PdfJpegFilterTests.cs b/tests/SharpCorePDF.Tests/PdfJpegFilterTests.cs index 10c77a8..6db9f08 100644 --- a/tests/SharpCorePDF.Tests/PdfJpegFilterTests.cs +++ b/tests/SharpCorePDF.Tests/PdfJpegFilterTests.cs @@ -35,6 +35,96 @@ public sealed class PdfJpegFilterTests private static string Fixture(string name) => Path.Combine(TestAssets.Folder, "jpeg", name); + /// A budget of exactly the output's size keeps the pixels; one byte less measures them. + [Fact] + public void Keeps_pixels_that_fit_the_budget_exactly() + { + Assert.True(JpegDecoder.TryDecode(Rgb, 1 << 20, out var all, out _, out var width, out var height)); + + var exact = width * height * 3; + + Assert.Equal(exact, all!.Length); + Assert.True(JpegDecoder.TryDecode(Rgb, exact, out var kept, out _, out _, out _)); + Assert.Equal(all, kept); + Assert.True(JpegDecoder.TryDecode(Rgb, exact - 1, out var measured, out _, out _, out _)); + Assert.Null(measured); + } + + /// An image above the budget is measured with the format its components state. + /// The fixture. + /// The format it decodes to. + [Theory] + [InlineData("jpeg-progressive-gray.jpg", "Gray8")] + [InlineData("jpeg-cmyk.jpg", "Cmyk32")] + [InlineData("jpeg-444.jpg", "Rgb24")] + public void Measures_the_format_of_an_image_above_the_budget(string fixture, string expected) + { + Assert.True(JpegDecoder.TryDecode(File.ReadAllBytes(Fixture(fixture)), 16, out var pixels, out var format, out _, out _)); + Assert.Null(pixels); + Assert.Equal(expected, format); + } + + /// + /// A frame whose output fits a budget of while its coefficient grid does not fit + /// an array — 46,340 squared samples is just under the budget, at four bytes a coefficient it is eight + /// gigabytes — is refused before anything is allocated, instead of overflowing the size in 32 bits. + /// + [Fact] + public void Refuses_a_frame_whose_working_buffers_cannot_fit_an_array() + { + var jpeg = (byte[])ProgressiveGray.Clone(); + var frame = jpeg.AsSpan().IndexOf(new byte[] { 0xFF, 0xC2 }); + + Assert.True(frame > 0, "the fixture has to carry the frame header the test rewrites"); + + jpeg[frame + 5] = 46_340 >> 8; + jpeg[frame + 6] = 46_340 & 0xFF; + jpeg[frame + 7] = 46_340 >> 8; + jpeg[frame + 8] = 46_340 & 0xFF; + + var before = GC.GetAllocatedBytesForCurrentThread(); + var decoded = JpegDecoder.TryDecode(jpeg, int.MaxValue, out var pixels, out _, out _, out _); + var allocated = GC.GetAllocatedBytesForCurrentThread() - before; + + Assert.False(decoded); + Assert.Null(pixels); + Assert.True(allocated < 1 << 20, $"refusing the frame allocated {allocated} bytes"); + } + + /// + /// A frame header that claims 65,535 x 65,535 samples is measured from the header alone, with no pixels, + /// as any image above the budget is. The width and the height are two numbers in the file, and the + /// sample planes — four bytes a coefficient for a progressive image — were otherwise allocated before + /// the output was found too large to keep. + /// + /// Whether the progressive fixture is used rather than the baseline one. + [Theory] + [InlineData(false)] + [InlineData(true)] + public void Measures_a_frame_too_large_for_the_budget_without_allocating_it(bool progressive) + { + var jpeg = (byte[])(progressive ? Progressive : Rgb).Clone(); + var frame = jpeg.AsSpan().IndexOf(progressive ? [0xFF, 0xC2] : [0xFF, 0xC0]); + + Assert.True(frame > 0, "the fixture has to carry the frame header the test rewrites"); + + // Marker, length (two bytes), precision, then height and width, each a 16-bit big-endian number. + jpeg[frame + 5] = 0xFF; + jpeg[frame + 6] = 0xFF; + jpeg[frame + 7] = 0xFF; + jpeg[frame + 8] = 0xFF; + + var before = GC.GetAllocatedBytesForCurrentThread(); + var measured = JpegDecoder.TryDecode(jpeg, 1 << 20, out var pixels, out var format, out var width, out var height); + var allocated = GC.GetAllocatedBytesForCurrentThread() - before; + + Assert.True(measured); + Assert.Null(pixels); + Assert.Equal("Rgb24", format); + Assert.Equal((65_535, 65_535), (width, height)); + Assert.True(allocated < 1 << 20, $"refusing the frame allocated {allocated} bytes"); + } + /// A 4:4:4 RGB JPEG decodes to the bytes Pillow decodes from the same file. [Fact] public void Decodes_a_baseline_rgb_jpeg_within_two_levels() diff --git a/tests/SharpCorePDF.Tests/PdfReaderRecoveryTests.cs b/tests/SharpCorePDF.Tests/PdfReaderRecoveryTests.cs index fba32c8..da00411 100644 --- a/tests/SharpCorePDF.Tests/PdfReaderRecoveryTests.cs +++ b/tests/SharpCorePDF.Tests/PdfReaderRecoveryTests.cs @@ -18,6 +18,139 @@ namespace SharpCorePDF.Tests; /// public sealed class PdfReaderRecoveryTests { + /// + /// The bound on a chain of lengths leaves an ordinary indirect length alone, for every stream of a + /// document and not only the first ones: each stream's data holds an endstream line of its own, + /// so a length that was not resolved would cut the stream there instead of at its real end. + /// + [Fact] + public void Resolves_the_indirect_length_of_every_stream_of_a_document() + { + const int Streams = 100; + var data = "q\nendstream\nQ"u8.ToArray(); + var builder = new PdfBuilder(); + var catalog = builder.Reserve(); + var pages = builder.Reserve(); + + builder.Add("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Contents 4 0 R >>"); + + // Objects 4, 6, 8 ... are the streams; 5, 7, 9 ... are their lengths. + for (var stream = 0; stream < Streams; stream++) + { + var body = Encoding.Latin1.GetBytes($"<< /Length {5 + (2 * stream)} 0 R >>\nstream\n") + .Concat(data) + .Concat("\nendstream"u8.ToArray()) + .ToArray(); + + builder.Add(body); + builder.Add(data.Length.ToString(System.Globalization.CultureInfo.InvariantCulture)); + } + + builder.Set(catalog, "<< /Type /Catalog /Pages 2 0 R >>"); + builder.Set(pages, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + + var reader = new PdfFileReader(builder.Build(catalog), "lengths.pdf", PdfEngineOptions.Default); + + for (var stream = 0; stream < Streams; stream++) + { + var resolved = Assert.IsType(reader.Resolve(4 + (2 * stream))); + + Assert.Equal(data, resolved.Raw.ToArray()); + } + } + + /// + /// An object stream that declares 2^31 - 1 objects in a header of one pair is read for the one pair it + /// holds. /N sized two arrays before anything compared it with the stream, so a few hundred bytes asked + /// for gigabytes; the stream now bounds the count, and the page inside it is found. + /// + [Fact] + public void Reads_an_object_stream_whose_count_its_header_cannot_hold() + { + var page = "3 0 << /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Contents 4 0 R >>"u8.ToArray(); + var packed = PdfBuilder.Compress(page); + var output = new MemoryStream(); + var offsets = new Dictionary(); + + void Write(string text) => output.Write(Encoding.Latin1.GetBytes(text)); + + void Object(int number, string dictionary, byte[]? stream = null) + { + offsets[number] = output.Position; + Write($"{number} 0 obj\n{dictionary}\n"); + + if (stream is not null) + { + Write("stream\n"); + output.Write(stream); + Write("\nendstream\n"); + } + + Write("endobj\n"); + } + + Write("%PDF-1.7\n"); + Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); + Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + Object(4, "<< /Length 3 >>", "q Q"u8.ToArray()); + Object(5, $"<< /Type /ObjStm /N 2147483647 /First 4 /Filter /FlateDecode /Length {packed.Length} >>", packed); + + offsets[6] = output.Position; + (int Kind, long Field, int Index)[] entries = + [ + (0, 0, 65535), (1, offsets[1], 0), (1, offsets[2], 0), (2, 5, 0), + (1, offsets[4], 0), (1, offsets[5], 0), (1, offsets[6], 0), + ]; + var table = entries + .SelectMany(entry => new[] + { + (byte)entry.Kind, + (byte)(entry.Field >> 24), (byte)(entry.Field >> 16), (byte)(entry.Field >> 8), (byte)entry.Field, + (byte)(entry.Index >> 8), (byte)entry.Index, + }) + .ToArray(); + + Write($"6 0 obj\n<< /Type /XRef /Size 7 /W [1 4 2] /Root 1 0 R /Length {table.Length} >>\nstream\n"); + output.Write(table); + Write($"\nendstream\nendobj\nstartxref\n{offsets[6]}\n%%EOF\n"); + + var reader = new PdfFileReader(output.ToArray(), "objstm.pdf", PdfEngineOptions.Default); + + Assert.Single(reader.Pages); + Assert.Equal("Page", Assert.IsType(reader.Resolve(3)).GetName("Type")); + } + + /// + /// A stream whose /Length is another stream, whose length is another, sixty thousand deep, is + /// read by finding its endstream. Resolved link by link, the chain would overflow the stack, and a + /// stack overflow ends the process rather than throwing anything a caller could catch. + /// + [Fact] + public void Reads_a_stream_whose_length_is_a_chain_of_sixty_thousand_streams() + { + const int Chain = 60_000; + var builder = new PdfBuilder(); + var catalog = builder.Reserve(); + var pages = builder.Reserve(); + + builder.Add("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Contents 4 0 R >>"); + + for (var link = 0; link < Chain; link++) + { + builder.Add($"<< /Length {5 + link} 0 R >>\nstream\nq Q\nendstream"); + } + + builder.Add("3"); + builder.Set(catalog, "<< /Type /Catalog /Pages 2 0 R >>"); + builder.Set(pages, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + + var reader = new PdfFileReader(builder.Build(catalog), "chain.pdf", PdfEngineOptions.Default); + var content = Assert.IsType(reader.Resolve(4)); + + Assert.Equal("q Q", Encoding.Latin1.GetString(content.Raw.Span)); + Assert.Single(reader.Pages); + } + /// A table that no longer parses is rebuilt, and the document is read completely. [Fact] public async Task Rebuilds_a_cross_reference_table_that_does_not_parse() diff --git a/tests/SharpCorePDF.Tests/PdfSignatureAttackTests.cs b/tests/SharpCorePDF.Tests/PdfSignatureAttackTests.cs index 52957e9..bbedc1f 100644 --- a/tests/SharpCorePDF.Tests/PdfSignatureAttackTests.cs +++ b/tests/SharpCorePDF.Tests/PdfSignatureAttackTests.cs @@ -104,8 +104,7 @@ public async Task Never_reports_a_signature_whose_value_was_emptied_as_valid() /// what must fail is the claim that the signature covers the whole document, because part of the /// document is now something other than the signature value. /// - [Fact(Skip = - "Known defect: SignatureValidator does not check that the gap between the two ranges is exactly the /Contents string, so this wrapped document is reported as covering the whole file. Remove the skip with the fix.")] + [Fact] public async Task Does_not_report_a_gap_that_holds_more_than_the_signature_value_as_covering_the_document() { var signed = await SignAsync(); @@ -114,14 +113,76 @@ public async Task Does_not_report_a_gap_that_holds_more_than_the_signature_value // The attack only works while the signed bytes are unchanged; this is what makes it an attack. var validation = Assert.Single(await SigningFixtures.ValidateAsync(wrapped)); - Assert.False( - validation.Signer.CoversWholeDocument && validation.IsValid, - "a document whose uncovered gap carries more than the signature value must not be reported as covered"); + Assert.False(validation.Signer.CoversWholeDocument); + Assert.False(validation.IsValid); + Assert.NotEqual(PdfSignatureIntegrity.Valid, validation.Integrity); + + var (gapStart, _) = Gap(signed); + var problem = Assert.Single(validation.Problems, problem => problem.Kind == PdfSignatureProblemKind.MalformedByteRange); + + Assert.Contains(gapStart.ToString(CultureInfo.InvariantCulture), problem.Message, StringComparison.Ordinal); + + // The wrapped gap is the finding; the revision count would be a second, misleading one. + Assert.DoesNotContain(validation.Problems, problem => problem.Kind == PdfSignatureProblemKind.RevisionNotCovered); + } + + /// + /// A gap that is not exactly one hexadecimal string is refused whichever way it fails to be one: no + /// opening delimiter, no closing delimiter, or a byte inside it that is neither a digit nor whitespace. + /// + /// Which byte of the gap is changed. + [Theory] + [InlineData("opening-delimiter")] + [InlineData("closing-delimiter")] + [InlineData("not-a-hex-digit")] + public async Task Refuses_a_gap_that_is_not_one_hexadecimal_string(string variant) + { + var signed = await SignAsync(); + var (gapStart, gapEnd) = Gap(signed); + var changed = (byte[])signed.Clone(); + var (index, value) = variant switch + { + "opening-delimiter" => (gapStart, (byte)'('), + "closing-delimiter" => (gapEnd - 1, (byte)')'), + "not-a-hex-digit" => (gapEnd - 2, (byte)'g'), + _ => throw new ArgumentOutOfRangeException(nameof(variant), variant, "no such variant"), + }; + + changed[index] = value; + + var validation = Assert.Single(await SigningFixtures.ValidateAsync(changed)); + + Assert.False(validation.IsValid); + Assert.False(validation.Signer.CoversWholeDocument); + Assert.Contains(validation.Problems, problem => problem.Kind == PdfSignatureProblemKind.MalformedByteRange); + } + + /// + /// Whitespace inside the value is part of PDF's hexadecimal string syntax, and an odd number of digits is + /// completed with a zero; a gap written that way is still exactly the signature value and stays valid. + /// + [Fact] + public async Task Accepts_whitespace_and_an_odd_digit_count_inside_the_value() + { + var signed = await SignAsync(); + var (_, gapEnd) = Gap(signed); + var spaced = (byte[])signed.Clone(); + + // The last digit of the padding becomes a space: the value now has an odd number of digits, which a + // reader completes with the zero that was there. + Assert.Equal((byte)'0', spaced[gapEnd - 2]); + spaced[gapEnd - 2] = (byte)' '; + + var validation = Assert.Single(await SigningFixtures.ValidateAsync(spaced)); + + Assert.True(validation.IsValid); + Assert.True(validation.Signer.CoversWholeDocument); + Assert.Equal(PdfSignatureIntegrity.Valid, validation.Integrity); } /// - /// The precondition the skipped wrapping test depends on, asserted on its own so it keeps running: the - /// injection leaves every covered byte as it was, which is why only a gap check can catch it. + /// The precondition the wrapping test depends on: the injection leaves every covered byte as it was, which + /// is why only a gap check can catch it. /// [Fact] public async Task Leaves_every_covered_byte_unchanged_when_the_gap_is_wrapped() From 9e2ab83a9616915a9746852b7f1ec3aa7a3b6871 Mon Sep 17 00:00:00 2001 From: Andrew Tierney Date: Mon, 16 Nov 2026 12:26:01 +1100 Subject: [PATCH 4/5] Read the documents the JPEG, empty-name, glyph-box and date faults refused Four defects turned documents into exceptions that were not PdfExceptions, so a caller that handles the contract's refusal still saw the read fail: - A JPEG scan selected Huffman tables with 4-bit selectors and a progressive scan named its spectral selection, and both were used as indexes unchecked (IndexOutOfRangeException). ParseScan now refuses a selector above 3, and a progressive scan unless Ss <= Se <= 63 and Ah, Al <= 13. - The empty name, a solidus on its own, is a name PDF allows, and four documents of the pdf.js test corpus use it as a key; PdfDictionary.Set refused it, so they did not open at all. - A glyph drawn upside down (a negative font size) gave PdfGlyph.BoundingBox a bottom above its top, and the rectangle refused it, failing text extraction for the page. - PdfDate.TryParse threw for a time-zone offset beyond fourteen hours, so one malformed /CreationDate failed the inspection of the whole document; such an offset is now read as no offset. PdfContainerTests.Refuses_an_entry_without_a_key asserted the second defect and is rewritten: a null key is still refused, the empty name is kept. Tests that fail without the fixes: PdfJpegFilterTests (table selectors, progressive scan bounds), PdfObjectModelTests (empty-name key, in a dictionary and in a document), PdfTextScannerTests (upside-down glyph), PdfSerializationTests (offsets past fourteen hours). The pdf.js corpus and the hostile documents now read with no fault, crash or timeout, and their --known flags are removed. --- .github/workflows/robustness.yml | 5 -- .github/workflows/validation.yml | 7 +- CHANGELOG.md | 14 +++- docs/validation.md | 11 +-- .../ReadModels/PdfGlyph.cs | 8 +- .../Pdf/JpegDecoder.cs | 13 ++- .../Pdf/PdfDate.cs | 8 +- .../Pdf/PdfDictionary.cs | 4 +- tests/SharpCorePDF.Tests/PdfContainerTests.cs | 15 +++- .../SharpCorePDF.Tests/PdfJpegFilterTests.cs | 80 +++++++++++++++++++ .../SharpCorePDF.Tests/PdfObjectModelTests.cs | 40 ++++++++++ .../PdfSerializationTests.cs | 22 +++++ .../SharpCorePDF.Tests/PdfTextScannerTests.cs | 25 ++++++ 13 files changed, 224 insertions(+), 28 deletions(-) diff --git a/.github/workflows/robustness.yml b/.github/workflows/robustness.yml index 93de31b..50854f4 100644 --- a/.github/workflows/robustness.yml +++ b/.github/workflows/robustness.yml @@ -162,16 +162,11 @@ jobs: - name: Build the probe run: dotnet build ./tools/SharpCorePDF.Validation -c Release - # The six known files fault on three recorded defects (docs/validation.md): an empty name as a - # dictionary key (issue3188, issue5896, pdfjs_wikipedia, poppler-742-0-fuzzed), a glyph whose box is - # upside down (issue14415) and a date whose offset is beyond 14 hours (TAMReview). - name: Read every document run: | python3 docs/tools/corpus-robustness.py \ --probe ./tools/SharpCorePDF.Validation/bin/Release/net11.0/SharpCorePDF.Validation.dll \ --timeout 60 --heap-limit-mb 1024 --report artifacts/corpora.json \ - --known issue3188.pdf --known issue5896.pdf --known pdfjs_wikipedia.pdf \ - --known poppler-742-0-fuzzed.pdf --known issue14415.pdf --known TAMReview.pdf \ "$RUNNER_TEMP/verapdf-corpus" "$RUNNER_TEMP/pdfjs/test/pdfs" - name: Upload the report diff --git a/.github/workflows/validation.yml b/.github/workflows/validation.yml index de42cbe..a632011 100644 --- a/.github/workflows/validation.yml +++ b/.github/workflows/validation.yml @@ -146,9 +146,9 @@ jobs: dotnet "$tool" replay "$target" "$seeds/$target" done - # Each of the four documents reaches a failure class the review named. flate-bomb, length-chain and - # objstm-count are refused or read since the decode limit, the length-chain bound and the /N bound; - # jpeg-selector is a known defect until the change that fixes it removes its --known line. + # Each of the four documents reaches a failure class a review named, and each is now read: the decode + # limit, the length-chain bound, the /N bound and the JPEG scan-header checks closed them. A document + # that faults here again is a regression, so none is listed as known. - name: Hostile documents run: | set -euo pipefail @@ -156,7 +156,6 @@ jobs: python docs/tools/corpus-robustness.py \ --probe ./tools/SharpCorePDF.Validation/bin/Release/net11.0/SharpCorePDF.Validation.dll \ --timeout 120 --report artifacts/hostile.json \ - --known jpeg-selector.pdf \ "$RUNNER_TEMP/hostile" - name: Upload the documents and the reports diff --git a/CHANGELOG.md b/CHANGELOG.md index abe4bd9..01e5662 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,6 +55,16 @@ against an empty feed. is measured from its header without being decoded — the same answer as before, without the allocation — and the working sizes are computed in 64 bits. +- **A JPEG scan header is checked before it is used.** A Huffman table selector above 3, and a progressive + scan whose spectral selection runs past coefficient 63 or starts after it ends, indexed past the decoder's + tables (`IndexOutOfRangeException`); such a scan is now refused and the image reported undecodable. +- **The empty name is a dictionary key like any other.** `/` on its own is a name PDF allows; four documents + of the pdf.js test corpus use it as a key and did not open at all (`ArgumentException`). +- **A glyph drawn upside down has a box.** A negative font size made `PdfGlyph.BoundingBox` throw, so text + extraction faulted on the page. +- **`PdfDate.TryParse` answers instead of throwing** for a time-zone offset beyond fourteen hours, which it + now reads as no offset; one malformed `/CreationDate` no longer fails the inspection of the document. + ### Known defects, found by the new checks and not yet fixed Each is reproduced, pinned by a skipped test or a strict `--known-defect` flag that fails the run once the @@ -63,10 +73,6 @@ defect is fixed, and listed in [`docs/validation.md`](docs/validation.md#known-d - An incremental update rewrites both elements of `/ID`, and appends a renumbered copy of the whole document rather than the objects it changed — which also rewrites the signature dictionary after signing. - A merge copies a page that states no `/MediaBox` without one. -- Hostile input: a JPEG scan's Huffman table selector is not range-checked. -- Real-world input (the pdf.js test corpus): an empty name as a dictionary key stops a document opening, a - glyph with an upside-down box faults text extraction, and `PdfDate.TryParse` throws on an out-of-range - time-zone offset instead of answering false. - **Run-length images are decoded.** `RunLengthDecode` joins Flate, LZW and the two ASCII filters: a bilevel or scan-style image that declares it now yields its samples — and `Gray8`, `Rgb24` or `Cmyk32` diff --git a/docs/validation.md b/docs/validation.md index 6ce4585..1d79368 100644 --- a/docs/validation.md +++ b/docs/validation.md @@ -86,10 +86,6 @@ fix. This is the list on 2026-11-16, every entry reproduced, not inferred: | An incremental update writes a new identifier into both elements of `/ID`; ISO 32000-1 §14.4 keeps the first | #10 | `PdfIncrementalInvariantTests.Keeps_the_first_identifier_element_across_an_incremental_update` (skipped); `validate-incremental.py` `id-first` | | `UseIncrementalUpdate` appends a renumbered copy of the whole document rather than the objects it changed: every byte of the earlier revision is kept, but the file grows by its own size per edit, and the signature dictionary is written again after signing, so pyHanko places a signature followed by an edit as `CONTIGUOUS_BLOCK_FROM_START` with modification level `OTHER` instead of `ENTIRE_REVISION` | #10, #3 | `validate-incremental.py` `appends-only`; `validate-signatures.py` on `signed-then-edited.pdf` | | A merge copies a page that states no `/MediaBox` (and inherits none) without one, so the result breaks the rule that every page has a media box; qpdf repairs it by assuming Letter | #2 | `validate-pdf.py` on `inspected.pdf` and `with-warnings.pdf` (both merge a page of `NamedDests.pdf`) | -| The JPEG decoder indexes its four Huffman tables with a scan's 4-bit selector unchecked | #12, #1 | `jpeg-selector.pdf`: `IndexOutOfRangeException` from `JpegDecoder.DecodeBlock` | -| A dictionary key that is the empty name (`/` alone, which PDF syntax allows) makes `PdfDictionary.Set` throw `ArgumentException` while parsing, so the document does not open at all | #12 | pdf.js corpus: `issue3188.pdf`, `issue5896.pdf`, `pdfjs_wikipedia.pdf`, `poppler-742-0-fuzzed.pdf` | -| A glyph whose box comes out upside down (a negative font size or a flipped font matrix) makes `PdfGlyph.BoundingBox` construct a `PdfRectangle` that throws, so text extraction faults | #12 | pdf.js corpus: `issue14415.pdf` | -| `PdfDate.TryParse` throws `ArgumentOutOfRangeException` for a time-zone offset beyond 14 hours instead of answering false, so one malformed `/CreationDate` faults the whole inspection | #12 | pdf.js corpus: `TAMReview.pdf` | Fixed since the list was first drawn up, each with the test that fails without its fix: @@ -99,6 +95,10 @@ Fixed since the list was first drawn up, each with the test that fails without i | No limit on the decoded size of a Flate, LZW or RunLength stream | `PdfEngineOptions.MaxDecodedStreamBytes` (128 MiB) | `PdfFilterTests.Refuses_a_flate_stream_that_decodes_past_the_limit` and its LZW, RunLength and content-stream companions | | A chain of indirect `/Length` references overflowed the stack | the chain is followed no deeper than `MaxNestingDepth`, then the stream is found by its `endstream` | `PdfReaderRecoveryTests.Reads_a_stream_whose_length_is_a_chain_of_sixty_thousand_streams` | | An object stream's `/N`, and a JPEG frame's width and height, sized allocations unchecked | `/N` is bounded by what the stream can hold; a JPEG above the caller's budget is measured from its header, and working sizes are computed in 64 bits | `PdfReaderRecoveryTests.Reads_an_object_stream_whose_count_its_header_cannot_hold`, `PdfJpegFilterTests.Measures_a_frame_too_large_for_the_budget_without_allocating_it` | +| The JPEG decoder used a scan's 4-bit Huffman table selectors, and a progressive scan's spectral end, as indexes unchecked | `ParseScan` refuses a selector above 3, and a progressive scan whose selection is not `Ss ≤ Se ≤ 63` or whose approximation is above 13 | `PdfJpegFilterTests.Refuses_a_scan_that_selects_a_table_beyond_the_four_there_are`, `Refuses_a_progressive_scan_outside_the_block` | +| The empty name as a dictionary key (`/` alone, which PDF allows) stopped the document opening | `PdfDictionary.Set` accepts it; the entry is kept and written back as `/` | `PdfObjectModelTests.Keeps_an_entry_whose_key_is_the_empty_name`, `Opens_a_document_that_uses_the_empty_name_as_a_key` | +| A glyph drawn upside down (negative font size) faulted text extraction | `PdfGlyph.BoundingBox` takes the box between descender and ascender whichever way round | `PdfTextScannerTests.Gives_a_glyph_drawn_upside_down_a_box` | +| `PdfDate.TryParse` threw for a time-zone offset beyond 14 hours | an offset no time zone has is read as no offset | `PdfSerializationTests.Reads_an_offset_no_time_zone_has_as_no_offset` | ## Corpus readings, 2026-11-16 @@ -108,7 +108,8 @@ Fixed since the list was first drawn up, each with the test that fails without i | pdf.js `test/pdfs` at `v6.4.299` | 987 | 977 | 4 | 6 | 0 | 0 | | synthesized hostile documents, before the fixes | 4 | 0 | 0 | 3 | 1 | 0 | -The six pdf.js faults are the three defects above. A sample of the pre-flight measurement (the first 60 +The six pdf.js faults were the empty-name, upside-down glyph and date-offset defects in the fixed table; +after those fixes all six are read, as is `jpeg-selector.pdf`. A sample of the pre-flight measurement (the first 60 files of `PDF_A-2b` and of `PDF_A-4`) wrote 108 results, refused 9 non-conforming inputs and refused 3 that already conformed; whether the 108 conform is veraPDF's to say, which `robustness.yml` asks. diff --git a/src/SharpCorePDF.Domain/ReadModels/PdfGlyph.cs b/src/SharpCorePDF.Domain/ReadModels/PdfGlyph.cs index eb0ea41..f3106d9 100644 --- a/src/SharpCorePDF.Domain/ReadModels/PdfGlyph.cs +++ b/src/SharpCorePDF.Domain/ReadModels/PdfGlyph.cs @@ -70,13 +70,15 @@ public sealed record PdfGlyph /// /// Gets the area the glyph occupies, derived from the origin, the advance and the em box. The /// descender is taken as a fifth of the em and the ascender as four fifths, which is the proportion - /// most Latin fonts share and is stated here rather than read from the font program. + /// most Latin fonts share and is stated here rather than read from the font program. A negative font + /// size — a glyph drawn upside down, which a flipped text matrix or font matrix produces — puts the + /// descender above the ascender, so the box is taken between the two whichever way round they are. /// public PdfRectangle BoundingBox => new( X, - Y - (FontSize * 0.2d), + Math.Min(Y - (FontSize * 0.2d), Y + (FontSize * 0.8d)), X + Math.Max(0d, Width), - Y + (FontSize * 0.8d)); + Math.Max(Y - (FontSize * 0.2d), Y + (FontSize * 0.8d))); /// Gets a value indicating whether the glyph paints no ink, such as a space or a tab. public bool IsWhitespace => string.IsNullOrWhiteSpace(Text); diff --git a/src/SharpCorePDF.Infrastructure/Pdf/JpegDecoder.cs b/src/SharpCorePDF.Infrastructure/Pdf/JpegDecoder.cs index 528cf69..8919615 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/JpegDecoder.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/JpegDecoder.cs @@ -473,6 +473,13 @@ private bool ParseScan(ReadOnlySpan segment) return false; } + // A selector is four bits in the file and the decoder has four tables of each class: a selector + // above 3 would index past them, so it is a header this decoder cannot honour. + if (tables >> 4 > 3 || (tables & 15) > 3) + { + return false; + } + component.DcTable = tables >> 4; component.AcTable = tables & 15; named[index] = component; @@ -486,7 +493,11 @@ private bool ParseScan(ReadOnlySpan segment) scanHigh = segment[tail + 2] >> 4; scanLow = segment[tail + 2] & 15; - return true; + // The spectral selection names coefficients 0 to 63 of a block, start before end, and the successive + // approximation shifts no further than a 16-bit coefficient allows. A progressive scan indexes the + // zig-zag table with them, so a scan outside these bounds is refused rather than read past the + // table; a sequential scan states them too but decodes the whole block whatever they say. + return !progressive || (scanStart <= scanEnd && scanEnd <= 63 && scanHigh <= 13 && scanLow <= 13); } private void ParseAdobe(ReadOnlySpan segment) diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfDate.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfDate.cs index f240e68..80d8ce7 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfDate.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfDate.cs @@ -133,8 +133,12 @@ private static TimeSpan ReadOffset(ReadOnlySpan rest) field++; } - var sign = rest[0] == '-' ? -1 : 1; - return sign * new TimeSpan(Math.Clamp(hours, 0, 23), Math.Clamp(minutes, 0, 59), 0); + // DateTimeOffset accepts offsets up to fourteen hours either way and throws beyond them, so an offset + // no time zone has is read as no offset — UTC, which is what the specification assumes when none is + // given — instead of failing the inspection of the whole document over one metadata string. + var offset = (rest[0] == '-' ? -1 : 1) * new TimeSpan(Math.Clamp(hours, 0, 23), Math.Clamp(minutes, 0, 59), 0); + + return offset.Duration() > TimeSpan.FromHours(14) ? TimeSpan.Zero : offset; } /// Reads the next run of digits of a time zone offset, skipping what is not a digit. diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfDictionary.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfDictionary.cs index b877b2b..72643b5 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfDictionary.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfDictionary.cs @@ -54,7 +54,9 @@ public IEnumerable> Entries /// This dictionary, so calls can be chained. public PdfDictionary Set(string key, PdfObject? value) { - ArgumentException.ThrowIfNullOrEmpty(key); + // The empty name — a solidus on its own — is a name PDF allows (ISO 32000-1 §7.3.5), and documents + // use it as a key; refusing it here refused the whole document while it was being parsed. + ArgumentNullException.ThrowIfNull(key); if (value is null) { diff --git a/tests/SharpCorePDF.Tests/PdfContainerTests.cs b/tests/SharpCorePDF.Tests/PdfContainerTests.cs index b402acd..e562066 100644 --- a/tests/SharpCorePDF.Tests/PdfContainerTests.cs +++ b/tests/SharpCorePDF.Tests/PdfContainerTests.cs @@ -72,10 +72,19 @@ public void Copies_a_set_of_entries() Assert.Same(PdfNull.Instance, dictionary.Get("B")); } - /// An entry without a key is a programmer error. + /// + /// An entry without a key is a programmer error, but the empty name is a key: a solidus on its own is a + /// name PDF allows (ISO 32000-1 §7.3.5), and documents in the wild use it. + /// [Fact] - public void Refuses_an_entry_without_a_key() => - Assert.Throws(() => new PdfDictionary().Set(string.Empty, PdfNull.Instance)); + public void Refuses_an_entry_without_a_key_and_keeps_one_whose_key_is_the_empty_name() + { + Assert.Throws(() => new PdfDictionary().Set(null!, PdfNull.Instance)); + + var dictionary = new PdfDictionary().Set(string.Empty, new PdfInteger(1)); + + Assert.Equal(1L, dictionary.GetInteger(string.Empty)); + } /// The readers of a dictionary answer with the type they were asked for, or with nothing. [Fact] diff --git a/tests/SharpCorePDF.Tests/PdfJpegFilterTests.cs b/tests/SharpCorePDF.Tests/PdfJpegFilterTests.cs index 6db9f08..4a6644c 100644 --- a/tests/SharpCorePDF.Tests/PdfJpegFilterTests.cs +++ b/tests/SharpCorePDF.Tests/PdfJpegFilterTests.cs @@ -35,6 +35,86 @@ public sealed class PdfJpegFilterTests private static string Fixture(string name) => Path.Combine(TestAssets.Folder, "jpeg", name); + /// + /// A scan that selects a Huffman table the decoder does not have — the selector is four bits, the tables + /// number four of each class — is refused, not used as an index past them. + /// + /// The table selector byte of the first scan component: DC in the high nibble, AC in the low. + [Theory] + [InlineData(0x44)] + [InlineData(0x40)] + [InlineData(0x04)] + [InlineData(0xFF)] + public void Refuses_a_scan_that_selects_a_table_beyond_the_four_there_are(byte selectors) + { + var jpeg = (byte[])Rgb.Clone(); + var scan = jpeg.AsSpan().IndexOf(new byte[] { 0xFF, 0xDA }); + + Assert.True(scan > 0, "the fixture has to carry a scan header"); + + // Marker, length (two bytes), component count, then the first component's id and its selectors. + jpeg[scan + 6] = selectors; + + Assert.False(JpegDecoder.TryDecode(jpeg, 1 << 20, out var pixels, out _, out _, out _)); + Assert.Null(pixels); + } + + /// + /// A progressive scan whose spectral selection runs past coefficient 63, starts after it ends, or states a + /// successive approximation no 16-bit coefficient has is refused before it indexes the zig-zag table. + /// + /// Which field of the AC scan header is rewritten. + /// The value it is rewritten to. + [Theory] + [InlineData("end", 0xFF)] + [InlineData("end", 64)] + [InlineData("start-after-end", 0)] + [InlineData("approximation", 0xE0)] + [InlineData("approximation", 0x0E)] + public void Refuses_a_progressive_scan_outside_the_block(string field, int value) + { + var jpeg = (byte[])Progressive.Clone(); + var position = 0; + var tail = -1; + + // The first AC scan: a scan header whose spectral start is above zero. + while (tail < 0) + { + var scan = jpeg.AsSpan(position).IndexOf(new byte[] { 0xFF, 0xDA }); + + Assert.True(scan >= 0, "the fixture has to carry an AC scan"); + scan += position; + + var candidate = scan + 5 + (2 * jpeg[scan + 4]); + + if (jpeg[candidate] > 0) + { + tail = candidate; + } + + position = scan + 2; + } + + switch (field) + { + case "end": + jpeg[tail + 1] = (byte)value; + break; + + case "start-after-end": + jpeg[tail + 1] = (byte)value; + Assert.True(jpeg[tail] > value, "the scan has to start after the end it is given"); + break; + + default: + jpeg[tail + 2] = (byte)value; + break; + } + + Assert.False(JpegDecoder.TryDecode(jpeg, 1 << 20, out var pixels, out _, out _, out _)); + Assert.Null(pixels); + } + /// A budget of exactly the output's size keeps the pixels; one byte less measures them. [Fact] public void Keeps_pixels_that_fit_the_budget_exactly() diff --git a/tests/SharpCorePDF.Tests/PdfObjectModelTests.cs b/tests/SharpCorePDF.Tests/PdfObjectModelTests.cs index a520044..d59b80e 100644 --- a/tests/SharpCorePDF.Tests/PdfObjectModelTests.cs +++ b/tests/SharpCorePDF.Tests/PdfObjectModelTests.cs @@ -13,6 +13,46 @@ namespace SharpCorePDF.Tests; /// public sealed class PdfObjectModelTests { + /// + /// The empty name — a solidus on its own — is a key like any other: the dictionary keeps it and every + /// entry after it, and writing the dictionary back states it as a solidus again. + /// + [Fact] + public void Keeps_an_entry_whose_key_is_the_empty_name() + { + var parser = new PdfObjectParser(Encoding.Latin1.GetBytes("<< /A 1 / 2 /B 3 >>"), "empty-name.pdf", 0, null, 64); + var dictionary = Assert.IsType(parser.ParseObject()); + + Assert.Equal(1L, dictionary.GetInteger("A")); + Assert.Equal(2L, dictionary.GetInteger(string.Empty)); + Assert.Equal(3L, dictionary.GetInteger("B")); + + using var stream = new MemoryStream(); + var output = new PdfOutputBuffer(stream); + PdfSerializer.WriteDictionary(dictionary, output); + output.Flush(); + var written = Encoding.Latin1.GetString(stream.ToArray()); + + Assert.Contains("/ 2", written, StringComparison.Ordinal); + } + + /// A document whose page dictionary carries an empty-named entry opens and lists its page. + [Fact] + public void Opens_a_document_that_uses_the_empty_name_as_a_key() + { + var builder = new PdfBuilder(); + var catalog = builder.Reserve(); + var pages = builder.Reserve(); + + builder.Add("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] / (empty) >>"); + builder.Set(catalog, "<< /Type /Catalog /Pages 2 0 R / 1 >>"); + builder.Set(pages, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + + var reader = new PdfFileReader(builder.Build(catalog), "empty-name.pdf", SharpCorePDF.Infrastructure.PdfEngineOptions.Default); + + Assert.Single(reader.Pages); + } + /// The null object is one instance and writes itself as its keyword. [Fact] public void Exposes_the_null_object() diff --git a/tests/SharpCorePDF.Tests/PdfSerializationTests.cs b/tests/SharpCorePDF.Tests/PdfSerializationTests.cs index a3bf372..3ae3690 100644 --- a/tests/SharpCorePDF.Tests/PdfSerializationTests.cs +++ b/tests/SharpCorePDF.Tests/PdfSerializationTests.cs @@ -78,6 +78,28 @@ public void Writes_a_date_that_round_trips() Assert.Equal("D:20260401120000Z", PdfDate.Format(new DateTimeOffset(2026, 4, 1, 12, 0, 0, TimeSpan.Zero))); } + /// + /// An offset no time zone has — past fourteen hours either way, which refuses + /// to hold — is read as no offset rather than thrown, so one malformed metadata date does not fail the + /// inspection of the document it is in. The offsets at the edge of the range are kept. + /// + /// The date as a document states it. + /// The offset that is read. + [Theory] + [InlineData("D:20260101000000+99'00'", 0)] + [InlineData("D:20260101000000+23'59'", 0)] + [InlineData("D:20260101000000+14'30'", 0)] + [InlineData("D:20260101000000-15'00'", 0)] + [InlineData("D:20260101000000+14'00'", 14 * 60)] + [InlineData("D:20260101000000-14'00'", -14 * 60)] + [InlineData("D:20260101000000+05'30'", 5 * 60 + 30)] + public void Reads_an_offset_no_time_zone_has_as_no_offset(string text, int offsetMinutes) + { + Assert.True(PdfDate.TryParse(text, out var value)); + Assert.Equal(TimeSpan.FromMinutes(offsetMinutes), value.Offset); + Assert.Equal(new DateTime(2026, 1, 1, 0, 0, 0), value.DateTime); + } + /// Text that is not a date at all is rejected instead of being invented. [Theory] [InlineData("")] diff --git a/tests/SharpCorePDF.Tests/PdfTextScannerTests.cs b/tests/SharpCorePDF.Tests/PdfTextScannerTests.cs index 1811665..802c05a 100644 --- a/tests/SharpCorePDF.Tests/PdfTextScannerTests.cs +++ b/tests/SharpCorePDF.Tests/PdfTextScannerTests.cs @@ -15,6 +15,31 @@ namespace SharpCorePDF.Tests; /// public sealed class PdfTextScannerTests { + /// + /// A glyph drawn upside down — a negative font size, which a flipped text or font matrix produces — has a + /// box all the same: its descender is above its ascender, and the box is taken between them. + /// + [Fact] + public void Gives_a_glyph_drawn_upside_down_a_box() + { + var glyph = new SharpCorePDF.Domain.ReadModels.PdfGlyph + { + Text = "A", + PageNumber = 1, + X = 10, + Y = 100, + Width = 6, + FontSize = -10, + }; + + var box = glyph.BoundingBox; + + Assert.Equal(92d, box.Bottom, 6); + Assert.Equal(102d, box.Top, 6); + Assert.Equal(10d, box.Left, 6); + Assert.Equal(16d, box.Right, 6); + } + /// A string the scanner can take as a slice of the stream. /// The content stream. /// The text the shown string must decode to. From 74a4d7397d0bfc73fccf2c084be11bd79b6e5198 Mon Sep 17 00:00:00 2001 From: Andrew Tierney Date: Mon, 16 Nov 2026 12:39:33 +1100 Subject: [PATCH 5/5] Append only what an incremental update changes, and keep the permanent identifier UseIncrementalUpdate ran the full rewrite into an appended writer, so every object of the document, the signature dictionary of a signed source included, was written again under a new number. Every earlier byte was kept, but the file grew by its own size per edit, and validators that look at what changed after a signature (pyHanko, Acrobat) saw the signed field as rewritten: pyHanko placed a signature followed by an edit as CONTIGUOUS_BLOCK_FROM_START instead of ENTIRE_REVISION. In an appended revision the edited document's copier now keeps the numbers of its objects, the catalog, the page tree, the pages and the information dictionary are written under the numbers they already have, and the writer leaves out an object that would be written exactly as the source states it. Rotating a page of a signed document now appends 716 bytes and three objects instead of 20,969 bytes and twelve; the signature dictionary is not in the revision. The trailer also kept neither half of the source's /ID. ISO 32000-1 14.4 makes the first element permanent across incremental updates; it is now the source's, and only the second element is new. Tests that fail without the change: PdfIncrementalInvariantTests (the identifier, an update that appends only what it changed, a signature dictionary left out of the revision); the corpus checks validate-incremental.py and validate-signatures.py now pass with no --known-defect flag. The manual's editing chapter described every edit as a complete rewrite and now says what the option does. --- .github/workflows/validation.yml | 16 +--- CHANGELOG.md | 11 ++- docs/manual/src/14-editing.md | 12 ++- docs/validation.md | 8 +- .../Pdf/PdfObjectCopier.cs | 10 +- .../Pdf/PdfWriter.cs | 20 ++++ .../PdfEditEngine.Document.cs | 15 ++- .../PdfEditEngine.Incremental.cs | 46 +++++++++ .../PdfEditEngine.Pages.cs | 11 ++- .../PdfEditEngine.Sources.cs | 10 +- .../PdfEditEngine.cs | 50 ++++++++-- .../PdfIncrementalInvariantTests.cs | 95 ++++++++++++++++++- 12 files changed, 267 insertions(+), 37 deletions(-) diff --git a/.github/workflows/validation.yml b/.github/workflows/validation.yml index a632011..681f1fd 100644 --- a/.github/workflows/validation.yml +++ b/.github/workflows/validation.yml @@ -21,8 +21,7 @@ # # --known-defect / --known names a failure that is recorded and not yet fixed. Each is strict: when the # defect is fixed the check passes, the flag no longer matches, and the run fails until the flag is removed -# in the same change as the fix. The in-suite counterparts are the skipped tests in -# PdfSignatureAttackTests and PdfIncrementalInvariantTests. +# in the same change as the fix. # # The validator versions are pinned in docs/tools/validation-requirements.txt, pdfjs-dist below. @@ -119,20 +118,11 @@ jobs: "$RUNNER_TEMP/corpus/text-merged.pdf" "$RUNNER_TEMP/corpus/incremental.pdf" \ "$RUNNER_TEMP/corpus/signed.pdf" - # id-first: the editor writes a new identifier into both elements of /ID on an appended revision. - # appends-only: the editor's incremental update appends a renumbered copy of the whole document - # rather than the objects it changed. - name: Incremental update invariants - run: | - python docs/tools/validate-incremental.py "$RUNNER_TEMP/corpus" \ - --known-defect id-first --known-defect appends-only + run: python docs/tools/validate-incremental.py "$RUNNER_TEMP/corpus" - # The same copy rewrites the signature dictionary after signing, which is why pyHanko cannot place the - # first signature of signed-then-edited.pdf at the end of a revision. - name: Signatures, pyHanko - run: | - python docs/tools/validate-signatures.py --manifest "$RUNNER_TEMP/corpus/manifest.json" \ - --known-defect signed-then-edited.pdf:FirstSignature + run: python docs/tools/validate-signatures.py --manifest "$RUNNER_TEMP/corpus/manifest.json" - name: Fuzz targets over their seeds run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 01e5662..caf759e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -65,13 +65,20 @@ against an empty feed. - **`PdfDate.TryParse` answers instead of throwing** for a time-zone offset beyond fourteen hours, which it now reads as no offset; one malformed `/CreationDate` no longer fails the inspection of the document. +- **An incremental update appends what changed, and only that.** `UseIncrementalUpdate` appended a copy of + the whole document under new object numbers: the file grew by its own size per edit, and the signature + dictionary of a signed source was written again after signing, so validators that look at what changed + after a signature (pyHanko, Acrobat) saw the signed field as modified. The edited document's objects now + keep their numbers and an object written exactly as the source already states it is left out — rotating a + page of a signed document appends 716 bytes and three objects instead of 20,969 bytes and twelve. +- **An incremental update keeps the permanent identifier.** The first element of `/ID` is the source's in + every revision, as ISO 32000-1 §14.4 requires; only the second element is new. + ### Known defects, found by the new checks and not yet fixed Each is reproduced, pinned by a skipped test or a strict `--known-defect` flag that fails the run once the defect is fixed, and listed in [`docs/validation.md`](docs/validation.md#known-defects). -- An incremental update rewrites both elements of `/ID`, and appends a renumbered copy of the whole document - rather than the objects it changed — which also rewrites the signature dictionary after signing. - A merge copies a page that states no `/MediaBox` without one. - **Run-length images are decoded.** `RunLengthDecode` joins Flate, LZW and the two ASCII filters: a diff --git a/docs/manual/src/14-editing.md b/docs/manual/src/14-editing.md index 13f9bb8..11d036a 100644 --- a/docs/manual/src/14-editing.md +++ b/docs/manual/src/14-editing.md @@ -251,10 +251,18 @@ change the bytes of the content streams they show. The source can be encrypted a the `password` argument of `ApplyAsync`; the result is written in the clear, because output encryption is not part of this editing release. -The writer produces a complete new document rather than an incremental update. That is what keeps -earlier revisions and signatures out of a document whose pages have changed; it also means a copied +By default the writer produces a complete new document rather than an incremental update. That is what +keeps earlier revisions and signatures out of a document whose pages have changed; it also means a copied signature is not a signature of the edited document. +With `PdfEditOutputOptions.UseIncrementalUpdate` the edit is appended to the source instead: every byte of +the source stays where it was, and the revision carries only the objects the edit changed, under the numbers +they already had — rotating one page redeclares that page and the few objects around it, and an object that +would be written exactly as the source states it is left out. A signature over the source therefore still +covers the revision it was made over, and a validator that looks at what changed after signing sees the +edit and not a second copy of the document. The first element of the trailer `/ID` is kept, as ISO 32000-1 +§14.4 requires. + ## Reading the result `ApplyAsync` returns a `PdfEditResult`: diff --git a/docs/validation.md b/docs/validation.md index 1d79368..47882c2 100644 --- a/docs/validation.md +++ b/docs/validation.md @@ -56,8 +56,8 @@ dotnet run --project .\tools\SharpCorePDF.Validation -c Release -- corpus $env:T python docs/tools/validate-text.py $env:TEMP\corpus python docs/tools/validate-encryption.py $env:TEMP\corpus -python docs/tools/validate-incremental.py $env:TEMP\corpus --known-defect id-first --known-defect appends-only -python docs/tools/validate-signatures.py --manifest $env:TEMP\corpus\manifest.json --known-defect signed-then-edited.pdf:FirstSignature +python docs/tools/validate-incremental.py $env:TEMP\corpus +python docs/tools/validate-signatures.py --manifest $env:TEMP\corpus\manifest.json python docs/tools/validate-fonts.py $env:TEMP\SharpCorePDF.Examples\output python docs/tools/render-compare.py --exclude protected.pdf $env:TEMP\SharpCorePDF.Examples\output @@ -83,8 +83,6 @@ fix. This is the list on 2026-11-16, every entry reproduced, not inferred: | Defect | Found by | Evidence | | --- | --- | --- | -| An incremental update writes a new identifier into both elements of `/ID`; ISO 32000-1 §14.4 keeps the first | #10 | `PdfIncrementalInvariantTests.Keeps_the_first_identifier_element_across_an_incremental_update` (skipped); `validate-incremental.py` `id-first` | -| `UseIncrementalUpdate` appends a renumbered copy of the whole document rather than the objects it changed: every byte of the earlier revision is kept, but the file grows by its own size per edit, and the signature dictionary is written again after signing, so pyHanko places a signature followed by an edit as `CONTIGUOUS_BLOCK_FROM_START` with modification level `OTHER` instead of `ENTIRE_REVISION` | #10, #3 | `validate-incremental.py` `appends-only`; `validate-signatures.py` on `signed-then-edited.pdf` | | A merge copies a page that states no `/MediaBox` (and inherits none) without one, so the result breaks the rule that every page has a media box; qpdf repairs it by assuming Letter | #2 | `validate-pdf.py` on `inspected.pdf` and `with-warnings.pdf` (both merge a page of `NamedDests.pdf`) | Fixed since the list was first drawn up, each with the test that fails without its fix: @@ -99,6 +97,8 @@ Fixed since the list was first drawn up, each with the test that fails without i | The empty name as a dictionary key (`/` alone, which PDF allows) stopped the document opening | `PdfDictionary.Set` accepts it; the entry is kept and written back as `/` | `PdfObjectModelTests.Keeps_an_entry_whose_key_is_the_empty_name`, `Opens_a_document_that_uses_the_empty_name_as_a_key` | | A glyph drawn upside down (negative font size) faulted text extraction | `PdfGlyph.BoundingBox` takes the box between descender and ascender whichever way round | `PdfTextScannerTests.Gives_a_glyph_drawn_upside_down_a_box` | | `PdfDate.TryParse` threw for a time-zone offset beyond 14 hours | an offset no time zone has is read as no offset | `PdfSerializationTests.Reads_an_offset_no_time_zone_has_as_no_offset` | +| An incremental update wrote a new identifier into both elements of `/ID` | the first element of the source's `/ID` is kept; only the second is new | `PdfIncrementalInvariantTests.Keeps_the_first_identifier_element_across_an_incremental_update` | +| `UseIncrementalUpdate` appended a renumbered copy of the whole document, rewriting the signature dictionary after signing | the edited document's objects keep their numbers, and an object written exactly as the source states it is left out: rotating one page of a signed document appends 716 bytes and three objects instead of 20,969 bytes and twelve, and pyHanko places the signature at `ENTIRE_REVISION` | `PdfIncrementalInvariantTests.Appends_only_the_objects_an_edit_changes`, `Leaves_the_signature_dictionary_out_of_an_appended_revision` | ## Corpus readings, 2026-11-16 diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfObjectCopier.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfObjectCopier.cs index a9bc19d..a1bda02 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfObjectCopier.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfObjectCopier.cs @@ -27,6 +27,7 @@ internal sealed partial class PdfObjectCopier private readonly HashSet redirected = []; private readonly Queue pending = []; private readonly HashSet queued = []; + private bool keepNumbers; /// Initializes a new instance of the class. /// The document to copy from. @@ -65,6 +66,13 @@ public void Override(long sourceObjectNumber, Func rewrite overrides[sourceObjectNumber] = rewrite; } + /// + /// Makes every object this copier copies keep the number it has in its source, instead of being given a + /// new one: the copier of the document an incremental update is appended to, whose objects are the same + /// objects in both revisions. + /// + public void KeepObjectNumbers() => keepNumbers = true; + /// Reserves a target object number for an object of the source and queues its body. /// The object number in the source document. /// The object number in the merged document. @@ -75,7 +83,7 @@ public int Reserve(long sourceObjectNumber) return existing; } - var target = writer.AllocateObject(); + var target = keepNumbers ? checked((int)sourceObjectNumber) : writer.AllocateObject(); targets[sourceObjectNumber] = target; Queue(sourceObjectNumber); return target; diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfWriter.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfWriter.cs index 55e4da0..d3cd0b7 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfWriter.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfWriter.cs @@ -16,6 +16,7 @@ internal sealed class PdfWriter private readonly PdfSecurityWriter? security; private readonly Dictionary offsets = []; private readonly long origin; + private Func? isUnchanged; private int lastObjectNumber; /// Initializes a new instance of the class. @@ -95,6 +96,20 @@ private PdfWriter(Stream destination, PdfEngineOptions options, int highestObjec /// The reserved object number. public int AllocateObject() => ++lastObjectNumber; + /// + /// Makes the writer leave out an object that would be written exactly as the revision it is appended to + /// already holds it under the same number. + /// + /// Answers whether an object number and the body about to be written under it are + /// what the earlier revision already states. + /// + /// An incremental update is meant to carry what the edit changed. An object that is written again unchanged + /// costs its bytes and, worse, tells every reader that looks at what changed after a signature — pyHanko's + /// difference analysis, Acrobat's — that the object was modified after signing. + /// + public void SkipUnchanged(Func unchanged) => + isUnchanged = unchanged ?? throw new ArgumentNullException(nameof(unchanged)); + /// Gets the highest object number that has been allocated. public int HighestObjectNumber => lastObjectNumber; @@ -103,6 +118,11 @@ private PdfWriter(Stream destination, PdfEngineOptions options, int highestObjec /// The body of the object. public void WriteObject(int objectNumber, PdfObject body) { + if (isUnchanged?.Invoke(objectNumber, body) == true) + { + return; + } + offsets[objectNumber] = Length; output.WriteInteger(objectNumber); output.Write(" 0 obj\n"u8); diff --git a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Document.cs b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Document.cs index 539f7b3..93a0682 100644 --- a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Document.cs +++ b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Document.cs @@ -693,10 +693,11 @@ private static PdfMetadata ResolveMetadata( /// Writes the document information dictionary and returns its object number. /// The writer of the result. /// The document information of the result. + /// The number the dictionary already has in the revision an update is appended to, or to number it anew. /// The object number of the document information dictionary. - private static int WriteInformation(PdfWriter writer, PdfMetadata metadata) + private static int WriteInformation(PdfWriter writer, PdfMetadata metadata, int? existingNumber = null) { - var number = writer.AllocateObject(); + var number = existingNumber ?? writer.AllocateObject(); var information = new PdfDictionary(); information.Set("Title", Text(metadata.Title)); @@ -728,16 +729,18 @@ private static int WriteInformation(PdfWriter writer, PdfMetadata metadata) /// The writer of the result. /// The object number of the catalog. /// The object number of the document information dictionary, or when the profile forbids it. - /// The identifier written as both elements of the ID entry. + /// The identifier of this revision: the second element of the ID entry, and the first too unless a permanent one is given. /// The encryption of the result, or when it is written in the clear. /// The cross-reference section this revision chains to, or for a complete document. + /// The first element of the source's ID, kept by an appended revision, or . private static void WriteTrailer( PdfWriter writer, int catalogNumber, int? infoNumber, PdfString identifier, PdfSecurityWriter? security, - long? previousXref) + long? previousXref, + PdfString? permanentIdentifier = null) { var trailer = new PdfDictionary(); trailer.Set("Root", new PdfReference(catalogNumber, 0)); @@ -746,7 +749,9 @@ private static void WriteTrailer( trailer.Set("Info", new PdfReference(info, 0)); } - trailer.Set("ID", new PdfArray(new PdfObject[] { identifier, identifier })); + // The first element is permanent: an appended revision keeps the one the document was created with, + // and only the second, which identifies this revision, is new. + trailer.Set("ID", new PdfArray(new PdfObject[] { permanentIdentifier ?? identifier, identifier })); if (previousXref is { } previous) { diff --git a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Incremental.cs b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Incremental.cs index d9a8e3b..10625a4 100644 --- a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Incremental.cs +++ b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Incremental.cs @@ -108,6 +108,52 @@ private static bool FieldsCarrySignature(PdfFileReader reader, PdfObject? fields return false; } + /// + /// Determines whether an object is exactly what the revision an update is appended to already states under + /// the same number, by writing both the way the writer would and comparing the bytes. + /// + /// The document the revision is appended to. + /// The number the object is about to be written under. + /// The body about to be written. + /// when writing it would restate the source unchanged. + private static bool IsUnchangedInSource(PdfFileReader source, int objectNumber, PdfObject body) + { + if (objectNumber > source.CrossReferenceTable.HighestObjectNumber || source.Resolve(objectNumber) is not { } original) + { + return false; + } + + return Serialize(original).AsSpan().SequenceEqual(Serialize(body)); + } + + private static byte[] Serialize(PdfObject value) + { + using var stream = new MemoryStream(); + var output = new PdfOutputBuffer(stream); + + PdfSerializer.Write(value, output); + output.Flush(); + + return stream.ToArray(); + } + + /// Gets the number of the source's document information dictionary, when it is an object of its own. + /// The document the revision is appended to. + /// The object number, or . + private static int? SourceInformationNumber(PdfFileReader source) => + source.CrossReferenceTable.Trailer.Get("Info") is PdfReference information + ? checked((int)information.ObjectNumber) + : null; + + /// + /// Gets the first element of the source's /ID, which ISO 32000-1 §14.4 makes permanent: it "shall + /// not change when the file is incrementally updated". + /// + /// The document the revision is appended to. + /// The identifier, or when the source states none. + private static PdfString? PermanentIdentifier(PdfFileReader source) => + source.CrossReferenceTable.Trailer.Get("ID") is PdfArray { Items: [PdfString permanent, ..] } ? permanent : null; + /// Copies the source into the destination and opens the writer of the appended revision. /// The stream the result is written to. /// The document being edited. diff --git a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Pages.cs b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Pages.cs index 1fa27f2..dbda7f5 100644 --- a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Pages.cs +++ b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Pages.cs @@ -290,11 +290,18 @@ private static List BuildLayout( /// Reserves the object number of every page that is written. /// The writer of the result. /// The pages in output order. - private static void AllocatePageTargets(PdfWriter writer, List layout) + /// The document an incremental update is appended to, whose pages keep their numbers, or . + private static void AllocatePageTargets(PdfWriter writer, List layout, PdfFileReader? keepNumbersOf) { + var kept = new HashSet(); + foreach (var page in layout) { - page.TargetNumber = writer.AllocateObject(); + // In an appended revision a page of the source stays the object it was; a second occurrence of the + // same page — a duplicate — is a new page and gets a number of its own. + page.TargetNumber = page.Reader == keepNumbersOf && kept.Add(page.Source.ObjectNumber) + ? checked((int)page.Source.ObjectNumber) + : writer.AllocateObject(); } } diff --git a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Sources.cs b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Sources.cs index 77ca7b6..dc479e4 100644 --- a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Sources.cs +++ b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Sources.cs @@ -221,16 +221,24 @@ public EditSources( /// Creates the copier of every document. /// The writer of the result. + /// The document an incremental update is appended to, whose copier keeps the numbers of its objects, or . /// /// Object numbers are only unique inside one document, so every document gets a copier of its own. /// All of them write into the same result, which is what makes a copied page indistinguishable /// from a page that was already there. /// - public void Attach(PdfWriter writer) + public void Attach(PdfWriter writer, PdfFileReader? keepNumbersOf = null) { foreach (var reader in readers) { copiers[reader] = new PdfObjectCopier(reader, writer); + + // Only the document a revision is appended to shares its numbers with the result; a document + // the edit imports from has numbers of its own, which would collide with them. + if (reader == keepNumbersOf) + { + copiers[reader].KeepObjectNumbers(); + } } } diff --git a/src/SharpCorePDF.Infrastructure/PdfEditEngine.cs b/src/SharpCorePDF.Infrastructure/PdfEditEngine.cs index 7500422..c6d4751 100644 --- a/src/SharpCorePDF.Infrastructure/PdfEditEngine.cs +++ b/src/SharpCorePDF.Infrastructure/PdfEditEngine.cs @@ -216,20 +216,48 @@ public async Task ApplyAsync( : null; var writer = revision?.Writer ?? new PdfWriter(capture ?? destination, version, options, security); - var catalogNumber = writer.AllocateObject(); - var pagesNodeNumber = writer.AllocateObject(); + + // An appended revision updates the objects of the revision before it rather than writing a second + // copy of them: the catalog, the page tree, the pages and every object copied from the source keep + // their numbers, and an object that comes out exactly as the source already states it is left out. + var primary = revision is null ? null : reader; + + if (primary is not null) + { + writer.SkipUnchanged((number, body) => IsUnchangedInSource(primary, number, body)); + } + + var catalogNumber = primary is not null && primary.RootObjectNumber > 0 + ? checked((int)primary.RootObjectNumber) + : writer.AllocateObject(); + var pagesNodeNumber = primary?.Catalog?.Get("Pages") is PdfReference pagesRoot + ? checked((int)pagesRoot.ObjectNumber) + : writer.AllocateObject(); // Every document the plan reads has its own copier, because object numbers are only unique // inside one document. All copiers write into the same result, which is what makes an // imported page indistinguishable from a page that was already there. - sources.Attach(writer); + sources.Attach(writer, keepNumbersOf: primary); + + if (primary is not null) + { + // The catalog and the information dictionary are written by the engine under their own numbers; + // a copy of the old ones, reached through a reference, would be a second object of the same + // number in the same revision. + sources.CopierOf(primary).Redirect(catalogNumber, catalogNumber); + + if (SourceInformationNumber(primary) is { } informationNumber) + { + sources.CopierOf(primary).Redirect(informationNumber, informationNumber); + } + } foreach (var (source, copier) in sources.Copiers) { RedirectPageTree(source, copier, pagesNodeNumber); } - AllocatePageTargets(writer, layout); + AllocatePageTargets(writer, layout, primary); foreach (var (source, copier) in sources.Copiers) { @@ -279,8 +307,18 @@ [.. layout.Select(page => page.TargetNumber)], // PDF/A-4 forbids the document information dictionary (a trailer /Info is allowed only beside a // catalog /PieceInfo, which this writer does not produce), so the packet is the only copy there. - var infoNumber = profile == PdfAProfile.PdfA4 ? (int?)null : WriteInformation(writer, metadata); - WriteTrailer(writer, catalogNumber, infoNumber, identifier, security, revision?.PreviousXref); + var existingInformation = primary is null ? null : SourceInformationNumber(primary); + var infoNumber = profile == PdfAProfile.PdfA4 + ? (int?)null + : WriteInformation(writer, metadata, existingInformation); + WriteTrailer( + writer, + catalogNumber, + infoNumber, + identifier, + security, + revision?.PreviousXref, + primary is null ? null : PermanentIdentifier(primary)); Verify(capture, layout, plan.Output.Encryption?.UserPassword, warnings); diff --git a/tests/SharpCorePDF.Tests/PdfIncrementalInvariantTests.cs b/tests/SharpCorePDF.Tests/PdfIncrementalInvariantTests.cs index fc5ad8e..95fe3ef 100644 --- a/tests/SharpCorePDF.Tests/PdfIncrementalInvariantTests.cs +++ b/tests/SharpCorePDF.Tests/PdfIncrementalInvariantTests.cs @@ -2,6 +2,7 @@ using SharpCorePDF.Domain.Aggregates; using SharpCorePDF.Domain.ValueObjects; using SharpCorePDF.Infrastructure; +using SharpCorePDF.Infrastructure.Extensions; using SharpCorePDF.Infrastructure.Pdf; using SharpCorePDF.Infrastructure.Sources; @@ -75,7 +76,7 @@ public async Task States_a_size_that_covers_every_revision() } /// The permanent half of the identifier survives an update; only the second half may change. - [Fact(Skip = "Known defect: PdfEditEngine.WriteTrailer writes a new identifier into both elements of /ID even when the revision is appended (PdfEditEngine.Document.cs). Remove the skip with the fix.")] + [Fact] public async Task Keeps_the_first_identifier_element_across_an_incremental_update() { var source = Source(); @@ -85,6 +86,98 @@ public async Task Keeps_the_first_identifier_element_across_an_incremental_updat Assert.Equal(Identifier(source, 0), Identifier(updated, 0)); } + /// + /// An appended revision carries what the edit changed under the numbers those objects already had: rotating + /// one page redeclares that page and the few objects around it, and adds no copy of the document under new + /// numbers. + /// + [Fact] + public async Task Appends_only_the_objects_an_edit_changes() + { + var source = Source(); + var updated = await AppendAsync(source, PdfEditPlan.Create().RotatePage(1, PageRotation.Clockwise90)); + + var before = ObjectNumbers(source); + var appended = ObjectNumbers(updated[source.Length..]); + + Assert.NotEmpty(appended); + Assert.Subset(before, appended); + Assert.True(appended.Count <= 4, $"the revision redeclares {appended.Count} objects: {string.Join(", ", appended)}"); + } + + /// + /// A signature dictionary is not written again by an update appended after signing: it stays the object of + /// the signed revision, which is what lets a validator place the signature at the end of that revision. + /// + [Fact] + public async Task Leaves_the_signature_dictionary_out_of_an_appended_revision() + { + var signed = await SigningFixtures.SignAsync(Source(), SigningFixtures.Options(SigningFixtures.Certificate)); + var updated = await AppendAsync(signed, PdfEditPlan.Create().RotatePage(1, PageRotation.Clockwise90)); + var revision = System.Text.Encoding.Latin1.GetString(updated, signed.Length, updated.Length - signed.Length); + + Assert.DoesNotContain("/ByteRange", revision, StringComparison.Ordinal); + Assert.DoesNotContain("/Type /Sig", revision, StringComparison.Ordinal); + + var validation = Assert.Single(await SigningFixtures.ValidateAsync(updated)); + + Assert.True(validation.IsValid); + Assert.False(validation.Signer.CoversWholeDocument); + } + + /// + /// The objects an edit creates are written into the revision under numbers of their own: an inserted page + /// is a new object, and a reader of the result finds it between the pages that were there. + /// + [Fact] + public async Task Appends_the_objects_an_edit_creates() + { + var source = Source(); + var updated = await AppendAsync( + source, + PdfEditPlan.Create().InsertPage(2, PdfPageSize.A5, page => page.DrawText("inserted", 20, 40, PdfTypeface.Standard(StandardFont.Helvetica), 10))); + + var reader = new PdfFileReader(updated, "updated.pdf", Options); + var fresh = ObjectNumbers(updated[source.Length..]).Except(ObjectNumbers(source)).ToList(); + + Assert.Equal(3, reader.Pages.Count); + Assert.NotEmpty(fresh); + Assert.All(fresh, number => Assert.True(number > ObjectNumbers(source).Max)); + } + + /// + /// A source whose objects refer to its own catalog and information dictionary is updated with one + /// definition of each in the revision: the engine writes them, and the copier, which reaches them through + /// those references, must not write a second copy under the same number. + /// + [Fact] + public async Task Writes_each_object_once_when_the_source_refers_to_its_own_catalog() + { + var builder = new PdfBuilder(); + var catalog = builder.Reserve(); + var pages = builder.Reserve(); + + builder.Add("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Back 1 0 R /About 4 0 R >>"); + builder.Add("<< /Title (Refers to its own catalog) >>"); + builder.Set(catalog, "<< /Type /Catalog /Pages 2 0 R >>"); + builder.Set(pages, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + + var source = builder.Build(catalog, "/Info 4 0 R /ID [<00112233445566778899AABBCCDDEEFF> <00112233445566778899AABBCCDDEEFF>]"); + var updated = await AppendAsync(source, PdfEditPlan.Create().RotatePage(1, PageRotation.Clockwise90)); + var revision = System.Text.Encoding.Latin1.GetString(updated, source.Length, updated.Length - source.Length); + var declared = System.Text.RegularExpressions.Regex.Matches(revision, @"(?m)^(\d+) 0 obj\b") + .Select(match => match.Groups[1].Value) + .ToList(); + + Assert.Equal(declared.Distinct().Count(), declared.Count); + Assert.Equal(90, (int)(new PdfFileReader(updated, "updated.pdf", Options).Pages[0].Dictionary.GetInteger("Rotate") ?? 0L)); + } + + private static SortedSet ObjectNumbers(byte[] bytes) => + [.. System.Text.RegularExpressions.Regex + .Matches(System.Text.Encoding.Latin1.GetString(bytes), @"(?m)^(\d+) 0 obj\b") + .Select(match => int.Parse(match.Groups[1].Value, System.Globalization.CultureInfo.InvariantCulture))]; + private static byte[] Source() => PdfDocumentBuilder.Create(PdfAuthoringOptions.Default with { DocumentId = SourceId }) .AddPage(PdfPageSize.A5, page => page.DrawText("one", 72, 100, PdfTypeface.Standard(StandardFont.Helvetica), 12)) .AddPage(PdfPageSize.A4, page => page.DrawText("two", 72, 100, PdfTypeface.Standard(StandardFont.Helvetica), 12))