From 06ad09252baa7361d47f71797cdff21ce1ae0a34 Mon Sep 17 00:00:00 2001 From: Andrew Tierney Date: Mon, 16 Nov 2026 11:03:01 +1100 Subject: [PATCH 1/4] Validate the output with independent readers and the input with fuzzing and corpora A writer that only reads its own output back cannot see the mistakes it shares with itself. validation.yml now puts the example output and a validation corpus (written by tools/SharpCorePDF.Validation, with a manifest of the text, passwords, permissions, revisions and signature coverage each document must have) through qpdf, pypdf, pdfminer.six, MuPDF, pdf.js, fontTools, OTS, PDFium and pyHanko, checks the incremental-update invariants, and replays the fuzz seeds and four hostile documents. robustness.yml runs libFuzzer over the parser and the DCT, CCITT, LZW, Flate and RunLength decoders, reads the veraPDF and pdf.js corpora one process per file, and judges the PDF/A writer with veraPDF over the veraPDF corpus. compliance.yml cross-validates PDF/A-4. The checks found defects that are not fixed here. Each is pinned by a skipped test (PdfSignatureAttackTests, PdfIncrementalInvariantTests) or a strict --known-defect flag that fails the run once the defect is fixed, and docs/validation.md lists them with their evidence. No shipped package changes. The constitution's statement that no Python validator runs in CI is corrected as a proposed PATCH amendment (2.0.1) for the maintainer to ratify. Local AI assistant state (.claude/, CLAUDE.local.md) is ignored. --- .config/dotnet-tools.json | 7 + .github/workflows/compliance.yml | 6 +- .github/workflows/robustness.yml | 282 +++++++++++++++ .github/workflows/validation.yml | 232 ++++++++++++ .gitignore | 4 + .specify/memory/constitution.md | 18 +- CHANGELOG.md | 28 ++ CONTRIBUTING.md | 6 +- README.md | 7 + SharpCorePDF.slnx | 4 +- docs/README.md | 2 + docs/tools/corpus-robustness.py | 207 +++++++++++ docs/tools/render-compare.py | 158 ++++++++ docs/tools/validate-arlington.py | 149 ++++++++ docs/tools/validate-encryption-pdfjs.mjs | 95 +++++ docs/tools/validate-encryption.py | 164 +++++++++ docs/tools/validate-fonts.py | 263 ++++++++++++++ docs/tools/validate-incremental.py | 145 ++++++++ docs/tools/validate-pdf.py | 30 +- docs/tools/validate-pdfa-cross.py | 11 +- docs/tools/validate-preflight-corpus.py | 148 ++++++++ docs/tools/validate-signatures.py | 77 ++++ docs/tools/validate-text.py | 147 ++++++++ docs/tools/validation-requirements.txt | 20 ++ docs/validation.md | 139 +++++++ .../PdfIncrementalInvariantTests.cs | 142 ++++++++ .../PdfSignatureAttackTests.cs | 218 +++++++++++ tools/SharpCorePDF.Validation/CorpusWriter.cs | 338 ++++++++++++++++++ tools/SharpCorePDF.Validation/FuzzTargets.cs | 254 +++++++++++++ tools/SharpCorePDF.Validation/Probe.cs | 251 +++++++++++++ tools/SharpCorePDF.Validation/Program.cs | 138 +++++++ .../SharpCorePDF.Validation.csproj | 52 +++ 32 files changed, 3730 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/robustness.yml create mode 100644 .github/workflows/validation.yml create mode 100644 docs/tools/corpus-robustness.py create mode 100644 docs/tools/render-compare.py create mode 100644 docs/tools/validate-arlington.py create mode 100644 docs/tools/validate-encryption-pdfjs.mjs create mode 100644 docs/tools/validate-encryption.py create mode 100644 docs/tools/validate-fonts.py create mode 100644 docs/tools/validate-incremental.py create mode 100644 docs/tools/validate-preflight-corpus.py create mode 100644 docs/tools/validate-text.py create mode 100644 docs/tools/validation-requirements.txt create mode 100644 docs/validation.md create mode 100644 tests/SharpCorePDF.Tests/PdfIncrementalInvariantTests.cs create mode 100644 tests/SharpCorePDF.Tests/PdfSignatureAttackTests.cs create mode 100644 tools/SharpCorePDF.Validation/CorpusWriter.cs create mode 100644 tools/SharpCorePDF.Validation/FuzzTargets.cs create mode 100644 tools/SharpCorePDF.Validation/Probe.cs create mode 100644 tools/SharpCorePDF.Validation/Program.cs create mode 100644 tools/SharpCorePDF.Validation/SharpCorePDF.Validation.csproj diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index 5ccda36..eee80fa 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -8,6 +8,13 @@ "dotnet-stryker" ], "rollForward": false + }, + "sharpfuzz.commandline": { + "version": "2.3.0", + "commands": [ + "sharpfuzz" + ], + "rollForward": false } } } \ No newline at end of file diff --git a/.github/workflows/compliance.yml b/.github/workflows/compliance.yml index 2b323f5..2f8af7e 100644 --- a/.github/workflows/compliance.yml +++ b/.github/workflows/compliance.yml @@ -214,7 +214,9 @@ jobs: # The same documents, read by two more implementations. --strict-tools makes a missing tool a failure # rather than a note, because a green run here is supposed to mean that all three agreed. PDFBox 3.0.8 - # and Ghostscript have no PDF/A-4 mode, so veraPDF is the only reading of that profile. + # and Ghostscript have no PDF/A-4 mode, so veraPDF stays the only verdict on that profile; PDFBox still + # parses the PDF 2.0 file and Ghostscript still rewrites it (as part 2), which is the reader test the + # other profiles get too. - name: Cross-validate with a second and a third implementation env: VERAPDF_HOME: ${{ env.VERAPDF_INSTALL }} @@ -223,7 +225,7 @@ jobs: output="$RUNNER_TEMP/SharpCorePDF.Examples/output" jar="$HOME/.cache/pdfbox/preflight-app-${PDFBOX_PREFLIGHT_VERSION}.jar" - for profile in 1b 2b 2u 3b 3u; do + for profile in 1b 2b 2u 3b 3u 4; do python3 docs/tools/validate-pdfa-cross.py --strict-tools \ --require-version "$VERAPDF_VERSION" \ --pdfbox "$jar" \ diff --git a/.github/workflows/robustness.yml b/.github/workflows/robustness.yml new file mode 100644 index 0000000..93de31b --- /dev/null +++ b/.github/workflows/robustness.yml @@ -0,0 +1,282 @@ +# Robustness: what the engine does with documents nobody here wrote, and with documents built to break it. +# +# validation.yml checks what the library writes, on every push. This workflow checks what it reads, weekly, +# because its inputs are large and slow: coverage-guided fuzzing of every decoder and of the document parser, +# two public corpora of real and deliberately broken files, and the PDF/A conversion measured against the +# veraPDF test corpus with veraPDF as the judge. None of it gates a pull request; a red run is a defect report. +# +# fuzz SharpFuzz instruments the three engine assemblies and libFuzzer drives one target per job +# (tools/SharpCorePDF.Validation, FuzzTargets.cs). A PdfException is the contract's refusal and +# not a finding; any other exception, a stack overflow or the memory limit is, and the input +# that caused it is uploaded. Reproduce one with: +# dotnet run --project tools/SharpCorePDF.Validation -- replay +# corpora corpus-robustness.py over the veraPDF corpus and the pdf.js test corpus, one process per +# file, under a time limit and a hard managed-heap limit. Fault, crash and timeout fail the run. +# preflight validate-preflight-corpus.py: every PDF/A file of the veraPDF corpus converted for its profile +# and the result judged by veraPDF. Report-only until its first run sets the baseline. +# +# Every external input is pinned: the libFuzzer driver by tag and SHA-256, SharpFuzz in +# .config/dotnet-tools.json, the corpora by commit, veraPDF by version (the same one compliance.yml pins). + +name: Robustness + +on: + schedule: + - cron: '0 3 * * 2' + workflow_dispatch: + inputs: + fuzz-minutes: + description: 'Minutes of fuzzing per target' + required: false + default: '30' + +permissions: + contents: read + +env: + LIBFUZZER_DOTNET_TAG: 'v2025.05.02.0904' + LIBFUZZER_DOTNET_SHA256: 'c2c2a90d94c409a4af339a0d4f244e0442c5a5d249be0f1252ba07871f285958' + VERAPDF_CORPUS_COMMIT: 'bb75f4f0073d9350dfd058c0162a367e6fadf25e' + PDFJS_TAG: 'v6.4.299' + PDFJS_COMMIT: 'd0991a0d53f3bb3a7307e3b83f279c1d117aafb3' + VERAPDF_VERSION: '1.30.4' + VERAPDF_INSTALL: ${{ github.workspace }}/.verapdf + +jobs: + fuzz: + name: Fuzz ${{ matrix.target }} + runs-on: ubuntu-latest + timeout-minutes: 60 + + strategy: + fail-fast: false + matrix: + target: [ document, dct, ccitt, lzw, flate, runlength ] + + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: Install the .NET SDKs + uses: actions/setup-dotnet@0f4a354dd5547a28c70b7853dbd95d5b67886e73 # v6 + with: + dotnet-version: | + 11.0.x + 10.0.x + + - name: Install SharpFuzz and the libFuzzer driver + run: | + set -euo pipefail + dotnet tool restore + curl --fail --silent --show-error --location --proto '=https' --tlsv1.2 \ + --output "$RUNNER_TEMP/libfuzzer-dotnet" \ + "https://github.com/Metalnem/libfuzzer-dotnet/releases/download/${LIBFUZZER_DOTNET_TAG}/libfuzzer-dotnet-ubuntu" + echo "${LIBFUZZER_DOTNET_SHA256} $RUNNER_TEMP/libfuzzer-dotnet" | sha256sum --check + chmod +x "$RUNNER_TEMP/libfuzzer-dotnet" + + # Two builds: an instrumented one for libFuzzer, and a plain one for the seeds and for replaying what + # libFuzzer finds, because instrumented code throws outside the fuzzer (see Program.cs). + - name: Build, then instrument the engine + run: | + set -euo pipefail + dotnet publish ./tools/SharpCorePDF.Validation -c Release -o "$RUNNER_TEMP/plain" + dotnet publish ./tools/SharpCorePDF.Validation -c Release -o "$RUNNER_TEMP/fuzz" + for layer in Domain Application Infrastructure; do + dotnet sharpfuzz "$RUNNER_TEMP/fuzz/SharpCorePDF.$layer.dll" + done + + - name: Seed the corpus + run: | + set -euo pipefail + tool="$RUNNER_TEMP/plain/SharpCorePDF.Validation.dll" + seeds="$RUNNER_TEMP/seeds" + assets=tests/SharpCorePDF.Tests/Assets + case "${{ matrix.target }}" in + document) dotnet "$tool" seed document "$seeds" "$assets/parity/corpus" "$assets/parity/labelled" "$assets/parity/forms" ;; + dct) dotnet "$tool" seed dct "$seeds" "$assets"/jpeg/*.jpg ;; + ccitt) dotnet "$tool" seed ccitt "$seeds" "$assets/ccitt/ccitt-g3.bin" "$assets/ccitt/ccitt-g4.bin" ;; + *) mkdir -p "$seeds" && printf '\x03\x00\x00\x3f\x3f' > "$seeds/empty" ;; + esac + dotnet "$tool" replay "${{ matrix.target }}" "$seeds" + + - name: Fuzz + env: + SCPDF_FUZZ_TARGET: ${{ matrix.target }} + MINUTES: ${{ github.event.inputs.fuzz-minutes || '30' }} + run: | + set -uo pipefail + mkdir -p artifacts/crashes + "$RUNNER_TEMP/libfuzzer-dotnet" \ + --target_path=dotnet --target_arg="$RUNNER_TEMP/fuzz/SharpCorePDF.Validation.dll" \ + -max_total_time=$(( MINUTES * 60 )) -timeout=30 -rss_limit_mb=4096 -max_len=262144 \ + -artifact_prefix=artifacts/crashes/ -print_final_stats=1 \ + "$RUNNER_TEMP/seeds" + status=$? + ls -la artifacts/crashes + if [ -n "$(ls -A artifacts/crashes)" ]; then + echo "::error::libFuzzer saved $(ls artifacts/crashes | wc -l) input(s) that broke the ${{ matrix.target }} target" + exit 1 + fi + exit $status + + - name: Upload what broke the engine + if: failure() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: fuzz-crashes-${{ matrix.target }} + path: artifacts/crashes + if-no-files-found: ignore + retention-days: 30 + + corpora: + name: Public corpora + runs-on: ubuntu-latest + timeout-minutes: 120 + + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: Install the .NET SDKs + uses: actions/setup-dotnet@0f4a354dd5547a28c70b7853dbd95d5b67886e73 # v6 + with: + dotnet-version: | + 11.0.x + 10.0.x + + - name: Fetch the corpora + run: | + set -euo pipefail + # The veraPDF corpus is small enough to clone without blobs and check out at the pinned commit; pdf.js + # is not, so it is cloned one commit deep at its tag and the commit is checked against the pin. + git clone -q --filter=blob:none --sparse https://github.com/veraPDF/veraPDF-corpus.git "$RUNNER_TEMP/verapdf-corpus" + git -C "$RUNNER_TEMP/verapdf-corpus" sparse-checkout set PDF_A-1b PDF_A-2b PDF_A-4 "ISO 32000-1" "ISO 32000-2" "Isartor test files" + git -C "$RUNNER_TEMP/verapdf-corpus" -c advice.detachedHead=false checkout -q "$VERAPDF_CORPUS_COMMIT" + + git clone -q --depth 1 --branch "$PDFJS_TAG" --filter=blob:none --sparse https://github.com/mozilla/pdf.js.git "$RUNNER_TEMP/pdfjs" + git -C "$RUNNER_TEMP/pdfjs" sparse-checkout set test/pdfs + test "$(git -C "$RUNNER_TEMP/pdfjs" rev-parse HEAD)" = "$PDFJS_COMMIT" + + find "$RUNNER_TEMP/verapdf-corpus" "$RUNNER_TEMP/pdfjs" -name '*.pdf' | wc -l + + - name: Build the probe + run: dotnet build ./tools/SharpCorePDF.Validation -c Release + + # The six known files fault on three recorded defects (docs/validation.md): an empty name as a + # dictionary key (issue3188, issue5896, pdfjs_wikipedia, poppler-742-0-fuzzed), a glyph whose box is + # upside down (issue14415) and a date whose offset is beyond 14 hours (TAMReview). + - name: Read every document + run: | + python3 docs/tools/corpus-robustness.py \ + --probe ./tools/SharpCorePDF.Validation/bin/Release/net11.0/SharpCorePDF.Validation.dll \ + --timeout 60 --heap-limit-mb 1024 --report artifacts/corpora.json \ + --known issue3188.pdf --known issue5896.pdf --known pdfjs_wikipedia.pdf \ + --known poppler-742-0-fuzzed.pdf --known issue14415.pdf --known TAMReview.pdf \ + "$RUNNER_TEMP/verapdf-corpus" "$RUNNER_TEMP/pdfjs/test/pdfs" + + - name: Upload the report + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: corpora + path: artifacts/corpora.json + if-no-files-found: warn + retention-days: 30 + + preflight: + name: PDF/A conversion against the veraPDF corpus + runs-on: ubuntu-latest + timeout-minutes: 180 + + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: Install the .NET SDKs + uses: actions/setup-dotnet@0f4a354dd5547a28c70b7853dbd95d5b67886e73 # v6 + with: + dotnet-version: | + 11.0.x + 10.0.x + + - name: Set up the JDK veraPDF runs on + uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6.1.0 + with: + distribution: 'zulu' + java-version: '17' + + - name: Cache the veraPDF installer + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/.cache/verapdf + key: verapdf-${{ env.VERAPDF_VERSION }}-installer + + # The same installation compliance.yml performs, so both workflows judge with the same validator. + - name: Install veraPDF + run: | + set -euo pipefail + band="${VERAPDF_VERSION%.*}" + archive="$HOME/.cache/verapdf/verapdf-greenfield-${VERAPDF_VERSION}-installer.zip" + mkdir -p "$(dirname "$archive")" + if [ ! -f "$archive" ]; then + curl --fail --silent --show-error --location \ + --proto '=https' --proto-redir '=https' --tlsv1.2 \ + --output "$archive" \ + "https://software.verapdf.org/releases/${band}/verapdf-greenfield-${VERAPDF_VERSION}-installer.zip" + fi + rm -rf "$VERAPDF_INSTALL" "$RUNNER_TEMP/verapdf" + mkdir -p "$VERAPDF_INSTALL" + unzip -q "$archive" -d "$RUNNER_TEMP/verapdf" + printf '%s\n' \ + '' \ + '' \ + ' ' \ + ' ' \ + " ${VERAPDF_INSTALL}" \ + ' ' \ + ' ' \ + ' ' \ + ' ' \ + ' ' \ + ' ' \ + ' ' \ + ' ' \ + ' ' \ + '' \ + > "$RUNNER_TEMP/auto-install.xml" + rm -f "/tmp/iz-veraPDF Software.tmp" "${TMPDIR:-/tmp}/iz-veraPDF Software.tmp" + bash "$RUNNER_TEMP/verapdf/verapdf-greenfield-${VERAPDF_VERSION}/verapdf-install" "$RUNNER_TEMP/auto-install.xml" + if [ ! -x "$VERAPDF_INSTALL/verapdf" ]; then + printf '%s\n' '#!/usr/bin/env bash' \ + "exec java -jar \"$VERAPDF_INSTALL\"/bin/cli-*.jar \"\$@\"" \ + > "$VERAPDF_INSTALL/verapdf" + chmod +x "$VERAPDF_INSTALL/verapdf" + fi + "$VERAPDF_INSTALL/verapdf" --version + + - name: Fetch the veraPDF corpus + run: | + set -euo pipefail + git clone -q --filter=blob:none --sparse https://github.com/veraPDF/veraPDF-corpus.git "$RUNNER_TEMP/verapdf-corpus" + git -C "$RUNNER_TEMP/verapdf-corpus" sparse-checkout set PDF_A-1b PDF_A-2b PDF_A-2u PDF_A-3b PDF_A-4 + git -C "$RUNNER_TEMP/verapdf-corpus" -c advice.detachedHead=false checkout -q "$VERAPDF_CORPUS_COMMIT" + + - name: Build the probe + run: dotnet build ./tools/SharpCorePDF.Validation -c Release + + - name: Convert and judge + run: | + python3 docs/tools/validate-preflight-corpus.py \ + --corpus "$RUNNER_TEMP/verapdf-corpus" \ + --probe ./tools/SharpCorePDF.Validation/bin/Release/net11.0/SharpCorePDF.Validation.dll \ + --verapdf "$VERAPDF_INSTALL/verapdf" \ + --report artifacts/preflight.json + + - name: Upload the report + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: preflight + path: artifacts/preflight.json + if-no-files-found: warn + retention-days: 30 diff --git a/.github/workflows/validation.yml b/.github/workflows/validation.yml new file mode 100644 index 0000000..6bb7cb0 --- /dev/null +++ b/.github/workflows/validation.yml @@ -0,0 +1,232 @@ +# Independent validation: what readers that share no code with this library make of what it writes. +# +# ci.yml answers whether the code builds and the suite passes; compliance.yml whether the archival output +# conforms. This workflow answers a third question -- whether other implementations read the documents the +# way the library meant them -- because a writer that only ever reads its own output back cannot see the +# mistakes it shares with itself. Every check here runs against the example output and against the validation +# corpus that tools/SharpCorePDF.Validation writes, whose manifest states what each document has to say. +# +# qpdf validate-pdf.py: the file parses, every page reads, every stream decodes +# text validate-text.py: pypdf, pdfminer.six and MuPDF extract the lines that were drawn +# encryption validate-encryption.py and the pdf.js leg: four readers open the RC4/AES matrix with the right +# password, refuse the wrong one, and read the revision and the permissions that were asked for +# fonts validate-fonts.py: fontTools and OTS read every embedded program, and the widths agree +# rendering render-compare.py: PDFium and MuPDF paint the same pages +# incremental validate-incremental.py: every revision stands alone, /ID[0] holds, an update only appends +# signatures validate-signatures.py --manifest: pyHanko agrees on integrity, trust and coverage +# fuzz seeds every fuzz target runs over its seeds in process, so a harness that rots is noticed here and +# not in the weekly fuzzing run (robustness.yml) +# hostile corpus-robustness.py over the documents it synthesizes for the failure classes the review named +# arlington validate-arlington.py against the veraPDF Arlington service (report-only until a baseline exists) +# +# --known-defect / --known names a failure that is recorded and not yet fixed. Each is strict: when the +# defect is fixed the check passes, the flag no longer matches, and the run fails until the flag is removed +# in the same change as the fix. The in-suite counterparts are the skipped tests in +# PdfSignatureAttackTests and PdfIncrementalInvariantTests. +# +# The validator versions are pinned in docs/tools/validation-requirements.txt, pdfjs-dist below. + +name: Independent validation + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + workflow_dispatch: + +permissions: + contents: read + +env: + PDFJS_VERSION: '6.4.299' + +jobs: + readers: + name: Read by other implementations + runs-on: ubuntu-latest + timeout-minutes: 40 + + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: Install the .NET SDKs + uses: actions/setup-dotnet@0f4a354dd5547a28c70b7853dbd95d5b67886e73 # v6 + with: + dotnet-version: | + 11.0.x + 10.0.x + + - name: Install the validators + run: | + set -euo pipefail + python3 -m venv "$RUNNER_TEMP/venv" + "$RUNNER_TEMP/venv/bin/pip" install --quiet -r docs/tools/validation-requirements.txt + echo "$RUNNER_TEMP/venv/bin" >> "$GITHUB_PATH" + npm install --silent --no-save --prefix "$RUNNER_TEMP/pdfjs" "pdfjs-dist@${PDFJS_VERSION}" + + - name: Build the examples and the validation harness + run: | + dotnet build ./examples/SharpCorePDF.Examples -c Release + dotnet build ./tools/SharpCorePDF.Validation -c Release + + - name: Write the documents + env: + TMPDIR: ${{ runner.temp }} + run: | + set -euo pipefail + dotnet run --project ./examples/SharpCorePDF.Examples -c Release --no-build + dotnet run --project ./tools/SharpCorePDF.Validation -c Release --no-build -- corpus "$RUNNER_TEMP/corpus" + + # protected.pdf is encrypted with a password the example makes for the run and prints; the encrypted + # documents qpdf does check are the corpus's, whose passwords the manifest carries + # (validate-encryption.py runs qpdf --check on each of them with its password). + - name: qpdf + run: | + set -euo pipefail + shopt -s nullglob + files=() + for file in "$RUNNER_TEMP"/SharpCorePDF.Examples/output/*.pdf "$RUNNER_TEMP"/corpus/*.pdf; do + case "$(basename "$file")" in protected.pdf|encrypted-*) continue ;; esac + files+=("$file") + done + # Both merge a page of NamedDests.pdf, which states no /MediaBox; the merge copies it without one, + # so the result breaks the rule that every page has a media box, and qpdf assumes Letter. + python docs/tools/validate-pdf.py --known-defect inspected.pdf --known-defect with-warnings.pdf "${files[@]}" + + - name: Text round trip + run: python docs/tools/validate-text.py "$RUNNER_TEMP/corpus" + + - name: Encryption matrix + run: | + set -euo pipefail + python docs/tools/validate-encryption.py "$RUNNER_TEMP/corpus" + node docs/tools/validate-encryption-pdfjs.mjs "$RUNNER_TEMP/corpus" \ + "$RUNNER_TEMP/pdfjs/node_modules/pdfjs-dist/legacy/build/pdf.mjs" + + - name: Embedded fonts + run: | + python docs/tools/validate-fonts.py \ + "$RUNNER_TEMP/SharpCorePDF.Examples/output" \ + "$RUNNER_TEMP/corpus/text-embedded.pdf" "$RUNNER_TEMP/corpus/text-merged.pdf" + + - name: Rendering, PDFium against MuPDF + run: | + python docs/tools/render-compare.py --report-dir artifacts/rendering --exclude protected.pdf \ + "$RUNNER_TEMP/SharpCorePDF.Examples/output" \ + "$RUNNER_TEMP/corpus/text-standard.pdf" "$RUNNER_TEMP/corpus/text-embedded.pdf" \ + "$RUNNER_TEMP/corpus/text-merged.pdf" "$RUNNER_TEMP/corpus/incremental.pdf" \ + "$RUNNER_TEMP/corpus/signed.pdf" + + # id-first: the editor writes a new identifier into both elements of /ID on an appended revision. + # appends-only: the editor's incremental update appends a renumbered copy of the whole document + # rather than the objects it changed. + - name: Incremental update invariants + run: | + python docs/tools/validate-incremental.py "$RUNNER_TEMP/corpus" \ + --known-defect id-first --known-defect appends-only + + # The same copy rewrites the signature dictionary after signing, which is why pyHanko cannot place the + # first signature of signed-then-edited.pdf at the end of a revision. + - name: Signatures, pyHanko + run: | + python docs/tools/validate-signatures.py --manifest "$RUNNER_TEMP/corpus/manifest.json" \ + --known-defect signed-then-edited.pdf:FirstSignature + + - name: Fuzz targets over their seeds + run: | + set -euo pipefail + tool=./tools/SharpCorePDF.Validation/bin/Release/net11.0/SharpCorePDF.Validation.dll + seeds="$RUNNER_TEMP/seeds" + dotnet "$tool" seed document "$seeds/document" tests/SharpCorePDF.Tests/Assets/parity/corpus "$RUNNER_TEMP/corpus" + dotnet "$tool" seed dct "$seeds/dct" tests/SharpCorePDF.Tests/Assets/jpeg/*.jpg + dotnet "$tool" seed ccitt "$seeds/ccitt" tests/SharpCorePDF.Tests/Assets/ccitt/ccitt-g3.bin tests/SharpCorePDF.Tests/Assets/ccitt/ccitt-g4.bin + for target in document dct ccitt; do + dotnet "$tool" replay "$target" "$seeds/$target" + done + + # Each of the four documents reaches a failure class the review named; each is a known defect until the + # change that fixes it removes its --known line. + - name: Hostile documents + run: | + set -euo pipefail + python docs/tools/corpus-robustness.py --synthesize "$RUNNER_TEMP/hostile" + python docs/tools/corpus-robustness.py \ + --probe ./tools/SharpCorePDF.Validation/bin/Release/net11.0/SharpCorePDF.Validation.dll \ + --timeout 120 --report artifacts/hostile.json \ + --known flate-bomb.pdf --known length-chain.pdf --known objstm-count.pdf --known jpeg-selector.pdf \ + "$RUNNER_TEMP/hostile" + + - name: Upload the documents and the reports + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: independent-validation + path: | + artifacts + ${{ runner.temp }}/corpus + ${{ runner.temp }}/SharpCorePDF.Examples/output/*.pdf + if-no-files-found: warn + retention-days: 14 + + arlington: + name: Arlington PDF Model + runs-on: ubuntu-latest + timeout-minutes: 30 + # The service's report format and the writer's baseline are confirmed by the first run on a runner; until + # then the job reports without blocking. Remove continue-on-error and pass --strict with the --accept list + # that run establishes. + continue-on-error: true + + services: + arlington: + image: verapdf/arlington:v1.30.4 + ports: + - 8080:8080 + + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: Install the .NET SDKs + uses: actions/setup-dotnet@0f4a354dd5547a28c70b7853dbd95d5b67886e73 # v6 + with: + dotnet-version: | + 11.0.x + 10.0.x + + - name: Write the documents + env: + TMPDIR: ${{ runner.temp }} + run: | + set -euo pipefail + dotnet run --project ./examples/SharpCorePDF.Examples -c Release + dotnet run --project ./tools/SharpCorePDF.Validation -c Release -- corpus "$RUNNER_TEMP/corpus" + + - name: Wait for the service + run: | + for attempt in $(seq 1 60); do + curl -sf http://localhost:8080/api/info > /dev/null && exit 0 + sleep 2 + done + echo "::error::the Arlington service did not answer on port 8080" + exit 1 + + - name: Check against the model + run: | + python3 docs/tools/validate-arlington.py --report-dir artifacts/arlington \ + --exclude protected.pdf \ + "$RUNNER_TEMP/SharpCorePDF.Examples/output" \ + "$RUNNER_TEMP/corpus/text-standard.pdf" "$RUNNER_TEMP/corpus/text-embedded.pdf" \ + "$RUNNER_TEMP/corpus/text-merged.pdf" "$RUNNER_TEMP/corpus/incremental.pdf" \ + "$RUNNER_TEMP/corpus/signed.pdf" + + - name: Upload the reports + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: arlington + path: artifacts/arlington + if-no-files-found: warn + retention-days: 14 diff --git a/.gitignore b/.gitignore index 4ca788b..a7b1b5e 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,10 @@ StrykerOutput/ *.sln.docstates *.DotSettings.user +# AI assistant state: prompts, transcripts and local settings stay on the machine that wrote them +.claude/ +CLAUDE.local.md + # Build logs *.binlog msbuild.log diff --git a/.specify/memory/constitution.md b/.specify/memory/constitution.md index af536c8..e45abe8 100644 --- a/.specify/memory/constitution.md +++ b/.specify/memory/constitution.md @@ -1,6 +1,13 @@ + + Exe + net11.0 + preview + SharpCorePDF.Validation + SharpCorePDF.Validation + enable + enable + nullable + true + latest-recommended + false + false + true + + + + + + + + + + + + + + + + + + + + From 8ad4c9e54663e4a4d467365154e25fbe4d360725 Mon Sep 17 00:00:00 2001 From: Andrew Tierney Date: Mon, 16 Nov 2026 11:55:24 +1100 Subject: [PATCH 2/4] Release pdf.js documents through their loading task pdf.js 6 has no PDFDocumentProxy.destroy(), so the encryption check's pdf.js leg reported every document as unopenable: the call threw inside the try that was meant to catch a wrong password. The loading task owns the document and destroying it releases both. The fix was made before the previous commit but an editor saved a stale copy of the file over it. --- docs/tools/validate-encryption-pdfjs.mjs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/tools/validate-encryption-pdfjs.mjs b/docs/tools/validate-encryption-pdfjs.mjs index 8ee5cd1..57a929c 100644 --- a/docs/tools/validate-encryption-pdfjs.mjs +++ b/docs/tools/validate-encryption-pdfjs.mjs @@ -26,8 +26,10 @@ async function open(file, password) { const data = new Uint8Array(await readFile(path.join(corpus, file))); const task = pdfjs.getDocument({data, password, isEvalSupported: false, verbosity: 0}); + // The loading task owns the document in pdf.js 6, which has no PDFDocumentProxy.destroy(): destroying + // the task is what releases both. try { - return await task.promise; + return {task, document: await task.promise}; } catch (error) { await task.destroy(); throw error; @@ -36,8 +38,8 @@ async function open(file, password) { async function refuses(file, password) { try { - const document = await open(file, password); - await document.destroy(); + const {task} = await open(file, password); + await task.destroy(); return false; } catch (error) { return error?.name === "PasswordException"; @@ -51,7 +53,7 @@ for (const entry of manifest.encryption) { const problems = []; try { - const document = await open(entry.file, entry.userPassword); + const {task, document} = await open(entry.file, entry.userPassword); const page = await document.getPage(1); const content = await page.getTextContent(); const text = normalise(content.items.map((item) => item.str + (item.hasEOL ? "\n" : "")).join("")); @@ -61,14 +63,14 @@ for (const entry of manifest.encryption) { problems.push(`page 1 text does not contain ${JSON.stringify(first)}`); } - await document.destroy(); + await task.destroy(); } catch (error) { problems.push(`the user password did not open it: ${error?.name}: ${error?.message}`); } try { - const document = await open(entry.file, entry.ownerPassword); - await document.destroy(); + const {task} = await open(entry.file, entry.ownerPassword); + await task.destroy(); } catch (error) { problems.push(`the owner password did not open it: ${error?.name}: ${error?.message}`); } From 7020f3bed853cb3fc0b1c4c12b4aef8242d74c73 Mon Sep 17 00:00:00 2001 From: Andrew Tierney Date: Mon, 16 Nov 2026 12:10:46 +1100 Subject: [PATCH 3/4] Refuse signature wrapping and bound what hostile input can allocate Signature wrapping: the validator required only that the byte range start at 0 and end at the end of the file, so bytes written into the unsigned gap could close the signature value early and add entries while the digest still matched, and the signature was reported valid and covering the whole document. The gap must now be exactly one hexadecimal string equal to the dictionary's /Contents; anything else is MalformedByteRange, integrity Unknown and CoversWholeDocument false. Decompression bombs: Flate, LZW and RunLength decoded without a limit, so a 1.1 MB document took the process past 1 GiB. PdfEngineOptions.MaxDecodedStreamBytes (128 MiB) now stops them; decode buffers above 16 MiB no longer stay in the shared array pool, and buffer growth no longer overflows. Stack overflow: a chain of streams whose /Length is the next stream was resolved by recursion, and 60,000 links ended the process. The chain is followed no deeper than MaxNestingDepth, then the stream is found by its endstream keyword, as for a /Length cycle. Allocations sized by the file: an object stream's /N is bounded by what the stream can hold, and a JPEG whose output is above the caller's budget is measured from its frame header instead of being decoded first; the working sizes are computed in 64 bits. Tests that fail without the fixes: PdfSignatureAttackTests (gap wrapping, gap shape), PdfFilterTests (limits per filter, content stream past the limit), PdfReaderRecoveryTests (length chain, indirect lengths, object stream /N), PdfJpegFilterTests (oversized frames, working buffers). The three hostile documents of validation.yml are read under a 1 GiB heap, and their --known flags are removed. --- .github/workflows/validation.yml | 7 +- CHANGELOG.md | 28 +++- docs/manual/src/10-dependency-injection.md | 14 +- docs/validation.md | 15 +- .../Pdf/JpegDecoder.cs | 78 ++++++++++ .../Pdf/PdfCrossReferenceReader.Streams.cs | 2 +- .../Pdf/PdfFileReader.ObjectStreams.cs | 22 +-- .../Pdf/PdfFileReader.Objects.cs | 14 +- .../Pdf/PdfFileReader.cs | 1 + .../Pdf/PdfFilters.Inflation.cs | 102 +++++++++++--- .../Pdf/PdfFilters.Lzw.cs | 21 ++- .../Pdf/PdfFilters.RunLength.cs | 24 +++- .../Pdf/PdfFilters.cs | 16 ++- .../Pdf/PdfFontDecoder.cs | 2 +- .../Pdf/PdfImageFilters.cs | 4 +- .../Pdf/PdfTextCollector.cs | 7 +- .../Pdf/PdfType1Encoding.cs | 2 +- .../Pdf/PooledByteBuffer.cs | 51 ++++++- .../PdfEditEngine.Overlays.cs | 2 +- .../PdfEngineOptions.cs | 23 +++ .../PdfPageContentReader.Content.cs | 7 +- .../PdfPageContentReader.Images.cs | 2 +- .../PdfPageContentReader.Metadata.cs | 2 +- .../PublicAPI.Unshipped.txt | 2 + .../Signatures/SignatureValidator.cs | 75 +++++++++- .../PdfEngineOptionsTests.cs | 10 ++ tests/SharpCorePDF.Tests/PdfFilterTests.cs | 96 +++++++++++++ .../SharpCorePDF.Tests/PdfJpegFilterTests.cs | 90 ++++++++++++ .../PdfReaderRecoveryTests.cs | 133 ++++++++++++++++++ .../PdfSignatureAttackTests.cs | 75 +++++++++- 30 files changed, 840 insertions(+), 87 deletions(-) diff --git a/.github/workflows/validation.yml b/.github/workflows/validation.yml index 6bb7cb0..de42cbe 100644 --- a/.github/workflows/validation.yml +++ b/.github/workflows/validation.yml @@ -146,8 +146,9 @@ jobs: dotnet "$tool" replay "$target" "$seeds/$target" done - # Each of the four documents reaches a failure class the review named; each is a known defect until the - # change that fixes it removes its --known line. + # Each of the four documents reaches a failure class the review named. flate-bomb, length-chain and + # objstm-count are refused or read since the decode limit, the length-chain bound and the /N bound; + # jpeg-selector is a known defect until the change that fixes it removes its --known line. - name: Hostile documents run: | set -euo pipefail @@ -155,7 +156,7 @@ jobs: python docs/tools/corpus-robustness.py \ --probe ./tools/SharpCorePDF.Validation/bin/Release/net11.0/SharpCorePDF.Validation.dll \ --timeout 120 --report artifacts/hostile.json \ - --known flate-bomb.pdf --known length-chain.pdf --known objstm-count.pdf --known jpeg-selector.pdf \ + --known jpeg-selector.pdf \ "$RUNNER_TEMP/hostile" - name: Upload the documents and the reports diff --git a/CHANGELOG.md b/CHANGELOG.md index 99ac86d..abe4bd9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,19 +33,37 @@ against an empty feed. reaches a consumer. [`docs/validation.md`](docs/validation.md) records each check, how it was shown to fail before it was trusted, and the defects below. +- **`PdfEngineOptions.MaxDecodedStreamBytes`** (128 MiB by default) bounds what one stream may decode to. + `MaxSourceBytes` bounds the input only, and Flate reaches about a thousand to one. + +### Fixed + +- **Security: signature wrapping.** The validator required only that the byte range start at 0 and end at the + end of the file, so bytes written into the unsigned gap could close the signature value early and add + entries while the digest still matched, and the signature was reported valid and covering the whole + document. The gap must now be exactly one hexadecimal string equal to the dictionary's `/Contents`; + anything else is `MalformedByteRange`, integrity `Unknown`, and `CoversWholeDocument` false. +- **Security: decompression bombs.** Flate, LZW and RunLength streams decoded without a limit, so a 1.1 MB + document took the process past 1 GiB. They now stop at `MaxDecodedStreamBytes`; buffer growth no longer + overflows past 1 GiB. +- **Security: stack overflow.** A chain of streams whose `/Length` is the next stream was resolved by + recursion, and 60,000 links ended the process. The chain is followed no deeper than `MaxNestingDepth`; + past it the stream is found by its `endstream` keyword, as for a `/Length` cycle. +- **Security: allocations sized by the file.** An object stream's `/N` sized two arrays before anything checked + it (513 bytes asked for 16 GiB), and a JPEG frame's 16-bit width and height sized the sample planes before + the caller's budget was consulted. `/N` is now bounded by what the stream can hold, a JPEG above the budget + is measured from its header without being decoded — the same answer as before, without the allocation — + and the working sizes are computed in 64 bits. + ### Known defects, found by the new checks and not yet fixed Each is reproduced, pinned by a skipped test or a strict `--known-defect` flag that fails the run once the defect is fixed, and listed in [`docs/validation.md`](docs/validation.md#known-defects). -- The signature validator does not require the gap between the byte ranges to be exactly the `/Contents` - string, so a gap carrying extra entries is still reported as covering the whole document. - An incremental update rewrites both elements of `/ID`, and appends a renumbered copy of the whole document rather than the objects it changed — which also rewrites the signature dictionary after signing. - A merge copies a page that states no `/MediaBox` without one. -- Hostile input: a Flate stream has no decoded-size limit, a chain of indirect `/Length` references overflows - the stack, an object stream's `/N` sizes an allocation unchecked, and a JPEG scan's Huffman table selector - is not range-checked. +- Hostile input: a JPEG scan's Huffman table selector is not range-checked. - Real-world input (the pdf.js test corpus): an empty name as a dictionary key stops a document opening, a glyph with an upside-down box faults text extraction, and `PdfDate.TryParse` throws on an out-of-range time-zone offset instead of answering false. diff --git a/docs/manual/src/10-dependency-injection.md b/docs/manual/src/10-dependency-injection.md index 953b10f..d46b5aa 100644 --- a/docs/manual/src/10-dependency-injection.md +++ b/docs/manual/src/10-dependency-injection.md @@ -144,6 +144,7 @@ defaults are deliberately permissive: | Option | Default | Effect | | --- | --- | --- | | `MaxSourceBytes` | 512 MiB | Largest source read into memory; larger sources are rejected | +| `MaxDecodedStreamBytes` | 128 MiB | Largest size one stream may decode to; a stream that would decode past it is refused | | `MaxNestingDepth` | 64 | Maximum dictionary/array nesting, page-tree depth and form field tree depth | | `MaxPagesPerSource` | 100 000 | Declared page limit; see the note below | | `ReconstructCrossReferenceOnFailure` | `true` | Rebuild a damaged or missing cross-reference table by scanning | @@ -154,8 +155,15 @@ The source limit is enforced by the reader, and exceeding it throws `PdfSourceFo detail that names the limit. It is not a streaming cap: the whole source is held in memory while it is processed, so the practical ceiling is your process's memory budget, not the number on the option. -`MaxNestingDepth` has four effects. It bounds how deep an object graph may be parsed, it bounds the -page-tree walk, it bounds the form field tree, and it bounds the tree the signature search descends. A +`MaxDecodedStreamBytes` bounds what a stream decodes to, which the source limit does not: Flate reaches about +a thousand to one, so a one-megabyte document can ask for a gigabyte. A content stream, form or font program +past the limit is reported as unreadable through the warning list, as other undecodable content is; an object +or cross-reference stream past it refuses the document with `PdfSourceFormatException`. Images are not refused +by it — their samples are carried only within `MaxInlineContentBytes` already. + +`MaxNestingDepth` has five effects. It bounds how deep an object graph may be parsed, it bounds how many +streams deep a chain of indirect `/Length` references is followed (past it, the stream is found by its +`endstream` keyword), it bounds the page-tree walk, it bounds the form field tree, and it bounds the tree the signature search descends. A page tree that is cut off contributes a `PageTreeTooDeep` warning rather than failing the merge; a field tree that is cut off contributes `FormFieldDepthExceeded` together with the fields above the cut, and the search for signatures reports the same warning and adds a `PdfSignatureProblemKind.FormFieldTreeTooDeep` @@ -176,7 +184,7 @@ file is binary; the marker costs a few bytes and prevents a class of corruption hard to diagnose. > `MaxPagesPerSource` is part of the options record but this version does not enforce it. Do not rely on -> it as a security boundary. For untrusted input, set `MaxSourceBytes`, inspect the document with +> it as a security boundary. For untrusted input, set `MaxSourceBytes` and `MaxDecodedStreamBytes`, inspect the document with > `IPdfInspector` and reject it if `PageCount` exceeds your own limit, before building a plan. ## Configuration file diff --git a/docs/validation.md b/docs/validation.md index 7986dba..6ce4585 100644 --- a/docs/validation.md +++ b/docs/validation.md @@ -83,25 +83,30 @@ fix. This is the list on 2026-11-16, every entry reproduced, not inferred: | Defect | Found by | Evidence | | --- | --- | --- | -| The signature validator does not check that the gap between the two byte ranges is exactly the `/Contents` string, so a gap that carries extra entries is still reported as covering the whole document (the signature-wrapping class) | #3 | `PdfSignatureAttackTests.Does_not_report_a_gap_that_holds_more_than_the_signature_value_as_covering_the_document` (skipped; fails when run) | | An incremental update writes a new identifier into both elements of `/ID`; ISO 32000-1 §14.4 keeps the first | #10 | `PdfIncrementalInvariantTests.Keeps_the_first_identifier_element_across_an_incremental_update` (skipped); `validate-incremental.py` `id-first` | | `UseIncrementalUpdate` appends a renumbered copy of the whole document rather than the objects it changed: every byte of the earlier revision is kept, but the file grows by its own size per edit, and the signature dictionary is written again after signing, so pyHanko places a signature followed by an edit as `CONTIGUOUS_BLOCK_FROM_START` with modification level `OTHER` instead of `ENTIRE_REVISION` | #10, #3 | `validate-incremental.py` `appends-only`; `validate-signatures.py` on `signed-then-edited.pdf` | | A merge copies a page that states no `/MediaBox` (and inherits none) without one, so the result breaks the rule that every page has a media box; qpdf repairs it by assuming Letter | #2 | `validate-pdf.py` on `inspected.pdf` and `with-warnings.pdf` (both merge a page of `NamedDests.pdf`) | -| A Flate stream has no limit on its decoded size: a 1.1 MB document inflates past 1 GiB | #12 | `flate-bomb.pdf`: `OutOfMemoryException` under a 1 GiB heap limit | -| A chain of indirect `/Length` references is resolved by recursion without a depth limit | #12 | `length-chain.pdf`: the process dies with a stack overflow, which no caller can catch | -| An object stream's `/N` sizes an allocation before anything checks it against the stream | #12 | `objstm-count.pdf` (513 bytes): `OutOfMemoryException` | | The JPEG decoder indexes its four Huffman tables with a scan's 4-bit selector unchecked | #12, #1 | `jpeg-selector.pdf`: `IndexOutOfRangeException` from `JpegDecoder.DecodeBlock` | | A dictionary key that is the empty name (`/` alone, which PDF syntax allows) makes `PdfDictionary.Set` throw `ArgumentException` while parsing, so the document does not open at all | #12 | pdf.js corpus: `issue3188.pdf`, `issue5896.pdf`, `pdfjs_wikipedia.pdf`, `poppler-742-0-fuzzed.pdf` | | A glyph whose box comes out upside down (a negative font size or a flipped font matrix) makes `PdfGlyph.BoundingBox` construct a `PdfRectangle` that throws, so text extraction faults | #12 | pdf.js corpus: `issue14415.pdf` | | `PdfDate.TryParse` throws `ArgumentOutOfRangeException` for a time-zone offset beyond 14 hours instead of answering false, so one malformed `/CreationDate` faults the whole inspection | #12 | pdf.js corpus: `TAMReview.pdf` | +Fixed since the list was first drawn up, each with the test that fails without its fix: + +| Defect | Fixed by | Test | +| --- | --- | --- | +| The signature validator accepted a `/ByteRange` gap holding more than the `/Contents` string (signature wrapping) | the gap must be exactly one hex string, equal to the dictionary's `/Contents` | `PdfSignatureAttackTests.Does_not_report_a_gap_that_holds_more_than_the_signature_value_as_covering_the_document` | +| No limit on the decoded size of a Flate, LZW or RunLength stream | `PdfEngineOptions.MaxDecodedStreamBytes` (128 MiB) | `PdfFilterTests.Refuses_a_flate_stream_that_decodes_past_the_limit` and its LZW, RunLength and content-stream companions | +| A chain of indirect `/Length` references overflowed the stack | the chain is followed no deeper than `MaxNestingDepth`, then the stream is found by its `endstream` | `PdfReaderRecoveryTests.Reads_a_stream_whose_length_is_a_chain_of_sixty_thousand_streams` | +| An object stream's `/N`, and a JPEG frame's width and height, sized allocations unchecked | `/N` is bounded by what the stream can hold; a JPEG above the caller's budget is measured from its header, and working sizes are computed in 64 bits | `PdfReaderRecoveryTests.Reads_an_object_stream_whose_count_its_header_cannot_hold`, `PdfJpegFilterTests.Measures_a_frame_too_large_for_the_budget_without_allocating_it` | + ## Corpus readings, 2026-11-16 | Corpus | Files | Read | Refused | Fault | Crash | Timeout | | --- | --- | --- | --- | --- | --- | --- | | veraPDF corpus, `PDF_A-2b` and `PDF_A-4` at `bb75f4f` | 1,471 | 1,471 | 0 | 0 | 0 | 0 | | pdf.js `test/pdfs` at `v6.4.299` | 987 | 977 | 4 | 6 | 0 | 0 | -| synthesized hostile documents | 4 | 0 | 0 | 3 | 1 | 0 | +| synthesized hostile documents, before the fixes | 4 | 0 | 0 | 3 | 1 | 0 | The six pdf.js faults are the three defects above. A sample of the pre-flight measurement (the first 60 files of `PDF_A-2b` and of `PDF_A-4`) wrote 108 results, refused 9 non-conforming inputs and refused 3 that diff --git a/src/SharpCorePDF.Infrastructure/Pdf/JpegDecoder.cs b/src/SharpCorePDF.Infrastructure/Pdf/JpegDecoder.cs index 2619fd8..528cf69 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/JpegDecoder.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/JpegDecoder.cs @@ -154,6 +154,16 @@ private bool Run(int maxBytes, out byte[]? pixels, out string? format, out int w return false; } + if (IsAboveBudget(maxBytes, out format, out width, out height)) + { + return format is not null; + } + + if (!WorkingBuffersFit()) + { + return false; + } + break; case 0xC2: progressive = true; @@ -163,6 +173,16 @@ private bool Run(int maxBytes, out byte[]? pixels, out string? format, out int w return false; } + if (IsAboveBudget(maxBytes, out format, out width, out height)) + { + return format is not null; + } + + if (!WorkingBuffersFit()) + { + return false; + } + break; case 0xDD: if (segment.Length < 2) @@ -320,6 +340,64 @@ private bool ParseHuffman(ReadOnlySpan segment) return true; } + /// + /// Answers, from the frame header alone, for an image whose samples the caller would not keep: the + /// format and the size are measured and nothing is decoded. + /// + /// The largest pixel buffer the caller keeps. + /// The pixel format the samples would state, or null for a component count the decoder has none for. + /// The width the frame states. + /// The height the frame states. + /// when the output is above the budget and the answer is final. + /// + /// The width and the height are two 16-bit numbers in the file: 65,535 squared is four gigasamples per + /// component, and the sample planes — and, for a progressive image, the coefficient grids at four bytes a + /// coefficient — were allocated and filled before the output was found too large to keep. The contract is + /// unchanged: such an image is measured and no pixels are returned. It is only no longer decoded first. + /// + private bool IsAboveBudget(int maxBytes, out string? format, out int measuredWidth, out int measuredHeight) + { + format = null; + measuredWidth = 0; + measuredHeight = 0; + + if ((long)width * height * components.Length <= maxBytes) + { + return false; + } + + format = components.Length switch + { + 1 => "Gray8", + 3 => "Rgb24", + 4 => "Cmyk32", + _ => null, + }; + measuredWidth = width; + measuredHeight = height; + return true; + } + + /// Determines whether the working buffers the frame asks for fit an array at all. + /// when every sample plane or coefficient grid fits. + /// The sizes are computed in 64 bits: in 32 they wrap, and a wrapped size is a small allocation that the decoder then writes past. + private bool WorkingBuffersFit() + { + var mcuColumns = (width + (8L * maxH) - 1) / (8L * maxH); + var mcuRows = (height + (8L * maxV) - 1) / (8L * maxV); + var perSample = progressive ? sizeof(int) : sizeof(byte); + + foreach (var component in components) + { + if (mcuColumns * 8 * component.H * mcuRows * 8 * component.V * perSample > Array.MaxLength) + { + return false; + } + } + + return true; + } + private bool ParseFrame(ReadOnlySpan segment) { if (segment.Length < 6 || segment[0] != 8) diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.Streams.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.Streams.cs index cec798b..7245f97 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.Streams.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.Streams.cs @@ -33,7 +33,7 @@ internal sealed partial class PdfCrossReferenceReader table.Add(new XrefEntry(objectNumber, offset, generation, XrefEntryType.Uncompressed)); } - var decoded = PdfFilters.Decode(stream, sourceName); + var decoded = PdfFilters.Decode(stream, sourceName, options.MaxDecodedStreamBytes); var widthType = 1; var widthField1 = 4; var widthField2 = 2; diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.ObjectStreams.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.ObjectStreams.cs index a208657..0a5df17 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.ObjectStreams.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.ObjectStreams.cs @@ -43,18 +43,24 @@ private void EnsureObjectStreamLoaded(long containerObjectNumber) } var dictionary = stream.Dictionary; - var count = (int)(dictionary.GetInteger("N") ?? 0); - var first = (int)(dictionary.GetInteger("First") ?? 0); + var declared = dictionary.GetInteger("N") ?? 0; + var first = dictionary.GetInteger("First") ?? 0; - if (count <= 0 || first < 0) + if (declared <= 0 || first < 0) { return; } - var decoded = PdfFilters.Decode(stream, SourceName); + var decoded = PdfFilters.Decode(stream, SourceName, options.MaxDecodedStreamBytes); + + // /N sizes two arrays, and it is a number the file states: 2^31 - 1 of them would ask for + // gigabytes from a stream of a few hundred bytes. Every pair of the header takes at least four + // bytes ("1 0 "), so the stream itself bounds how many objects it can describe, and the arrays + // are sized by that bound; a header that ends before /N says so below. + var count = (int)Math.Min(declared, (decoded.Length / 4L) + 1); var header = new PdfObjectParser(decoded, SourceName, 0, null, options.MaxNestingDepth); var numbers = new long[count]; - var offsets = new int[count]; + var offsets = new long[count]; for (var index = 0; index < count; index++) { @@ -62,12 +68,12 @@ private void EnsureObjectStreamLoaded(long containerObjectNumber) { warnings.Add(new( PdfWarningKind.ObjectStreamTruncated, - $"'{SourceName}': object stream {containerObjectNumber} declares {count} objects but its header ends early.")); + $"'{SourceName}': object stream {containerObjectNumber} declares {declared} objects but its header ends early.")); break; } numbers[index] = number; - offsets[index] = (int)offset; + offsets[index] = offset; } for (var index = 0; index < count; index++) @@ -79,7 +85,7 @@ private void EnsureObjectStreamLoaded(long containerObjectNumber) continue; } - var parser = new PdfObjectParser(decoded, SourceName, position, null, options.MaxNestingDepth); + var parser = new PdfObjectParser(decoded, SourceName, (int)position, null, options.MaxNestingDepth); fromObjectStreams[numbers[index]] = parser.ParseObject(); } } diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.Objects.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.Objects.cs index 1a39e97..4eeaa67 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.Objects.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.Objects.cs @@ -72,23 +72,33 @@ private PdfVersion ReadVersion() /// /// Resolves a stream /Length that is itself an indirect reference. The guard set breaks - /// the cycle that such a reference creates when the length object is the stream itself. + /// the cycle that such a reference creates when the length object is the stream itself, and the + /// depth guard breaks a chain: a length that is a stream whose length is a stream, and so on. /// /// The object number of the length object. /// The length, or when it cannot be resolved safely. + /// + /// Each link of a chain is resolved inside the one before it, so a chain of tens of thousands of + /// streams would overflow the stack — which no caller can catch: the process ends. A well-formed + /// length needs one level; past the length is answered + /// as unknown, exactly as a cycle is, and the parser finds the end of the stream by its keyword. + /// private long? ResolveIndirectLength(long objectNumber) { - if (!resolvingLengths.Add(objectNumber)) + if (lengthResolutionDepth >= options.MaxNestingDepth || !resolvingLengths.Add(objectNumber)) { return null; } + lengthResolutionDepth++; + try { return Resolve(objectNumber) is PdfInteger integer ? integer.Value : null; } finally { + lengthResolutionDepth--; resolvingLengths.Remove(objectNumber); } } diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.cs index 0c1d647..fcec760 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.cs @@ -20,6 +20,7 @@ internal sealed partial class PdfFileReader private readonly HashSet loadedObjectStreams = []; private readonly HashSet resolvingObjectStreams = []; private readonly HashSet resolvingLengths = []; + private int lengthResolutionDepth; private PageTreeWalk? pageTree; private PdfDictionary? catalog; diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.Inflation.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.Inflation.cs index e3c2f47..8fe69f2 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.Inflation.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.Inflation.cs @@ -12,30 +12,79 @@ internal static partial class PdfFilters private const int InflateChunkSize = 16 * 1024; private const int MaximumInitialInflateCapacity = 1024 * 1024; - /// Attempts to decompress Flate data, tolerating the raw deflate streams some producers write. - /// The compressed bytes. - /// The decompressed bytes when decompression succeeded. - /// when the data could be decompressed. - public static bool TryInflate(ReadOnlyMemory data, out byte[] result) + /// + /// The decoded size a stream is held to when the caller has no to read it + /// from; the same value as 's default. + /// + internal const long DefaultMaxDecodedBytes = 128L * 1024 * 1024; + + /// What one attempt at inflating a stream came to. + private enum InflateOutcome { - result = []; + /// The stream inflated within the limit. + Inflated, + + /// The bytes are not a stream this wrapper can inflate. + Invalid, - return !data.IsEmpty - && (TryInflateCore(data, zlibWrapper: true, out result) - || TryInflateCore(data, zlibWrapper: false, out result)); + /// The stream inflates past the limit; it was stopped there. + TooLarge, } - /// Decompresses Flate data. + /// Attempts to decompress Flate data, tolerating the raw deflate streams some producers write. + /// The compressed bytes. + /// The decompressed bytes when decompression succeeded. + /// when the data could be decompressed within the default limit. + public static bool TryInflate(ReadOnlyMemory data, out byte[] result) => + TryInflate(data, DefaultMaxDecodedBytes, out result) == InflateOutcome.Inflated; + + /// Decompresses Flate data under the default limit on its decoded size. /// The compressed bytes. /// The name of the source, used in diagnostics. /// The decompressed bytes. - /// The data could not be decompressed. + /// The data could not be decompressed, or decodes past the limit. public static byte[] Inflate(ReadOnlyMemory data, string sourceName) => - TryInflate(data, out var result) - ? result - : throw new PdfSourceFormatException(sourceName, "a Flate compressed stream could not be decompressed."); + Inflate(data, sourceName, DefaultMaxDecodedBytes); - private static bool TryInflateCore(ReadOnlyMemory data, bool zlibWrapper, out byte[] result) + /// Decompresses Flate data, refusing a stream that decodes to more than a limit. + /// The compressed bytes. + /// The name of the source, used in diagnostics. + /// The largest number of bytes the stream may decode to. + /// The decompressed bytes. + /// The data could not be decompressed, or decodes past the limit. + public static byte[] Inflate(ReadOnlyMemory data, string sourceName, long maxDecodedBytes) => + TryInflate(data, maxDecodedBytes, out var result) switch + { + InflateOutcome.Inflated => result, + InflateOutcome.TooLarge => throw DecodedTooLarge(sourceName, "a Flate", maxDecodedBytes), + _ => throw new PdfSourceFormatException(sourceName, "a Flate compressed stream could not be decompressed."), + }; + + /// Builds the refusal of a stream that decodes past the limit. + /// The name of the source. + /// The kind of stream, for the message. + /// The limit that was reached. + /// The exception to throw. + internal static PdfSourceFormatException DecodedTooLarge(string sourceName, string what, long maxDecodedBytes) => + new(sourceName, $"{what} stream decodes to more than {maxDecodedBytes} bytes, the limit PdfEngineOptions.MaxDecodedStreamBytes sets, so it was not decoded."); + + private static InflateOutcome TryInflate(ReadOnlyMemory data, long maxDecodedBytes, out byte[] result) + { + result = []; + + if (data.IsEmpty) + { + return InflateOutcome.Invalid; + } + + var outcome = TryInflateCore(data, zlibWrapper: true, maxDecodedBytes, out result); + + return outcome == InflateOutcome.Invalid + ? TryInflateCore(data, zlibWrapper: false, maxDecodedBytes, out result) + : outcome; + } + + private static InflateOutcome TryInflateCore(ReadOnlyMemory data, bool zlibWrapper, long maxDecodedBytes, out byte[] result) { result = []; @@ -51,7 +100,8 @@ private static bool TryInflateCore(ReadOnlyMemory data, bool zlibWrapper, // stream surprises us. Growth is progressive instead of one guess, so a small stream never // reserves the memory of a large one. using var output = new PooledByteBuffer( - (int)Math.Clamp((long)data.Length * 4, InflateChunkSize, MaximumInitialInflateCapacity)); + (int)Math.Clamp((long)data.Length * 4, InflateChunkSize, MaximumInitialInflateCapacity), + poolLargeArrays: false); while (true) { @@ -63,19 +113,27 @@ private static bool TryInflateCore(ReadOnlyMemory data, bool zlibWrapper, break; } + // Flate reaches about a thousand to one, so the limit on the input does not bound this + // buffer: a megabyte of zeros is a gigabyte here. The stream is stopped at the limit + // instead of being allowed to take the process with it. + if (output.Length + (long)read > maxDecodedBytes) + { + return InflateOutcome.TooLarge; + } + output.Advance(read); } result = output.ToExactArray(); - return true; + return InflateOutcome.Inflated; } catch (InvalidDataException) { - return false; + return InflateOutcome.Invalid; } catch (IOException) { - return false; + return InflateOutcome.Invalid; } } @@ -91,9 +149,9 @@ private static MemoryStream OpenInput(ReadOnlyMemory data) => ? new MemoryStream(array, segment.Offset, segment.Count, writable: false) : new MemoryStream(data.ToArray(), writable: false); - private static byte[] ApplyFlate(ReadOnlyMemory data, PdfDictionary? parameters, string sourceName) + private static byte[] ApplyFlate(ReadOnlyMemory data, PdfDictionary? parameters, string sourceName, long maxDecodedBytes) { - var inflated = Inflate(data, sourceName); + var inflated = Inflate(data, sourceName, maxDecodedBytes); // Every parameter of the filter is optional and every one of them is absent when the stream has no // filter dictionary at all, so the dictionary is established once here instead of being tested @@ -134,5 +192,5 @@ private static byte[] ApplyFlate(ReadOnlyMemory data, PdfDictionary? param public static byte[] InflateAndUnpredict( ReadOnlyMemory data, PdfDictionary? parameters, - string sourceName) => ApplyFlate(data, parameters, sourceName); + string sourceName) => ApplyFlate(data, parameters, sourceName, DefaultMaxDecodedBytes); } diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.Lzw.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.Lzw.cs index ee2e587..fd2162a 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.Lzw.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.Lzw.cs @@ -143,18 +143,31 @@ internal static int ReadEarlyChange(PdfDictionary? parameters) => /// The compressed bytes. /// The decode parameters of the filter, which state EarlyChange. /// The name of the source, used in diagnostics. + /// The largest number of bytes the stream may decode to. /// The decoded bytes. - /// The stream is not a valid LZW stream. + /// The stream is not a valid LZW stream, or decodes past the limit. internal static byte[] ApplyLzw( ReadOnlyMemory data, PdfDictionary? parameters, - string sourceName) + string sourceName, + long maxDecodedBytes) { - var output = new PooledByteBuffer(); + var output = new PooledByteBuffer(poolLargeArrays: false); try { - if (!TryDecodeLzw(data.Span, ReadEarlyChange(parameters), output.Write, out _)) + // A 12-bit code can stand for 4,096 bytes, so the input limit does not bound this buffer either. + void Write(ReadOnlySpan bytes) + { + if (output.Length + (long)bytes.Length > maxDecodedBytes) + { + throw DecodedTooLarge(sourceName, "an LZW", maxDecodedBytes); + } + + output.Write(bytes); + } + + if (!TryDecodeLzw(data.Span, ReadEarlyChange(parameters), Write, out _)) { throw new PdfSourceFormatException(sourceName, "an LZW stream could not be decompressed."); } diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.RunLength.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.RunLength.cs index 2acb163..2b029b1 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.RunLength.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.RunLength.cs @@ -24,11 +24,21 @@ internal static partial class PdfFilters /// reaches past the end contributes the bytes that are there, and a repeated run with no value byte /// contributes nothing. /// - public static byte[] DecodeRunLength(ReadOnlyMemory data) + public static byte[] DecodeRunLength(ReadOnlyMemory data) => + DecodeRunLength(data, "a stream", DefaultMaxDecodedBytes); + + /// Decodes a run-length encoded stream, refusing one that decodes past a limit. + /// The encoded bytes. + /// The name of the source, used in diagnostics. + /// The largest number of bytes the stream may decode to. + /// The decoded bytes. + /// The stream decodes past the limit. + /// Two bytes can stand for 128, so the limit on the input does not bound the output. + public static byte[] DecodeRunLength(ReadOnlyMemory data, string sourceName, long maxDecodedBytes) { // The output is at least as large as the input for the literal case, and can be 128 times larger // for the repeat case, so the buffer starts at the input length and grows only when it must. - using var result = new PooledByteBuffer(data.Length); + using var result = new PooledByteBuffer(data.Length, poolLargeArrays: false); var span = data.Span; var index = 0; @@ -48,6 +58,11 @@ public static byte[] DecodeRunLength(ReadOnlyMemory data) // end, which ends the loop. The explicit guard a survivor asked about is this comment. var count = Math.Min(control + 1, span.Length - index); + if (result.Length + (long)count > maxDecodedBytes) + { + throw DecodedTooLarge(sourceName, "a RunLength", maxDecodedBytes); + } + result.Write(span.Slice(index, count)); index += count; continue; @@ -60,6 +75,11 @@ public static byte[] DecodeRunLength(ReadOnlyMemory data) break; } + if (result.Length + (long)repeat > maxDecodedBytes) + { + throw DecodedTooLarge(sourceName, "a RunLength", maxDecodedBytes); + } + var value = span[index++]; for (var count = 0; count < repeat; count++) diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.cs index 261f5a8..5552260 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.cs @@ -27,7 +27,17 @@ public static bool HasFlateCompression(PdfDictionary dictionary) /// The decoded bytes. /// The stream uses a filter the engine does not implement. /// A compressed stream could not be decompressed. - public static byte[] Decode(PdfStream stream, string sourceName) + public static byte[] Decode(PdfStream stream, string sourceName) => + Decode(stream, sourceName, DefaultMaxDecodedBytes); + + /// Decodes a stream, applying every filter it declares in order, under a limit on its decoded size. + /// The stream to decode. + /// The name of the source, used in diagnostics. + /// The largest number of bytes any one filter may decode to. + /// The decoded bytes. + /// The stream uses a filter the engine does not implement. + /// A compressed stream could not be decompressed, or decodes past the limit. + public static byte[] Decode(PdfStream stream, string sourceName, long maxDecodedBytes) { ArgumentNullException.ThrowIfNull(stream); @@ -51,8 +61,8 @@ public static byte[] Decode(PdfStream stream, string sourceName) decoded = name switch { - "FlateDecode" or "Fl" => ApplyFlate(current, parametersForFilter, sourceName), - "LZWDecode" or "LZW" => ApplyLzw(current, parametersForFilter, sourceName), + "FlateDecode" or "Fl" => ApplyFlate(current, parametersForFilter, sourceName, maxDecodedBytes), + "LZWDecode" or "LZW" => ApplyLzw(current, parametersForFilter, sourceName, maxDecodedBytes), "ASCIIHexDecode" or "AHx" => DecodeAsciiHex(current), "ASCII85Decode" or "A85" => DecodeAscii85(current), _ => throw new PdfUnsupportedFeatureException( diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfFontDecoder.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfFontDecoder.cs index 7a31da3..33e41d8 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfFontDecoder.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfFontDecoder.cs @@ -313,7 +313,7 @@ private static PdfFont Create(PdfFileReader reader, string resourceName, PdfDict try { - data = PdfFilters.Decode(stream, reader.SourceName); + data = PdfFilters.Decode(stream, reader.SourceName, reader.Options.MaxDecodedStreamBytes); } catch (PdfUnsupportedFeatureException) { diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfImageFilters.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfImageFilters.cs index afa4917..15951b7 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfImageFilters.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfImageFilters.cs @@ -23,7 +23,7 @@ internal readonly record struct PdfPayload( /// /// /// -/// answers a different question and answers it correctly: an object +/// answers a different question and answers it correctly: an object /// stream that cannot be inflated means a broken document, so it raises. An image is different — a /// JPXDecode scan is a perfectly valid document this library simply does not decode — so this /// class reports what it could not do and hands back the raw bytes (NFR-7.6, NFR-7.8). @@ -361,7 +361,7 @@ private static bool TryDecodeOne( case "RunLengthDecode": { - var decoded = PdfFilters.DecodeRunLength(data); + var decoded = PdfFilters.DecodeRunLength(data, sourceName, PdfFilters.DefaultMaxDecodedBytes); length = decoded.Length; bytes = Keep(decoded, cap); diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfTextCollector.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfTextCollector.cs index 5a68441..b99551d 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfTextCollector.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfTextCollector.cs @@ -382,7 +382,7 @@ private void PaintForm(string name, PdfDictionary? resources, TextState state, i try { - if (!TryDecodeContent(stream, out var formContent)) + if (!TryDecodeContent(stream, reader.Options.MaxDecodedStreamBytes, out var formContent)) { warnings.Add(new( PdfWarningKind.ContentStreamUnreadable, @@ -432,13 +432,14 @@ private string DescribeUnmapped(PdfFont font) => string.Create( /// Decodes the content stream of a form. /// The stream. + /// The largest number of bytes the stream may decode to. /// The decoded bytes when decoding succeeded. /// when the stream could be decoded. - private static bool TryDecodeContent(PdfStream stream, out byte[] content) + private static bool TryDecodeContent(PdfStream stream, long maxDecodedBytes, out byte[] content) { try { - content = PdfFilters.Decode(stream, "form content stream"); + content = PdfFilters.Decode(stream, "form content stream", maxDecodedBytes); return true; } catch (Domain.Exceptions.PdfUnsupportedFeatureException) diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfType1Encoding.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfType1Encoding.cs index 2a19346..84eddd9 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfType1Encoding.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfType1Encoding.cs @@ -52,7 +52,7 @@ internal static class PdfType1Encoding try { - bytes = PdfFilters.Decode(program, reader.SourceName); + bytes = PdfFilters.Decode(program, reader.SourceName, reader.Options.MaxDecodedStreamBytes); } catch (PdfUnsupportedFeatureException) { diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PooledByteBuffer.cs b/src/SharpCorePDF.Infrastructure/Pdf/PooledByteBuffer.cs index 12615ac..5a9f02c 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PooledByteBuffer.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PooledByteBuffer.cs @@ -24,17 +24,32 @@ internal sealed class PooledByteBuffer : IDisposable { private const int MinimumCapacity = 16; + /// + /// The largest array a decode buffer takes from the pool. Above it the array is allocated and left to the + /// collector, because the shared pool keeps what is returned to it: a document that inflates a stream to + /// the decode limit in every read pass would otherwise leave hundreds of megabytes parked in the pool, + /// and the next pass would allocate beside them. + /// + private const int MaximumPooledDecodeLength = 16 * 1024 * 1024; + private byte[] array; private int length; + private readonly bool poolLargeArrays; private bool returned; /// Initializes a new instance of the class. /// The number of bytes to rent up front. - public PooledByteBuffer(int initialCapacity = 256) + /// + /// Whether arrays above 16 MiB go through the pool too. The read of a source keeps them pooled, because + /// reusing the one buffer a document is read into is what keeps a large read at one allocation; a decode + /// buffer does not, because its size is whatever a stream inflates to and the pool would keep it. + /// + public PooledByteBuffer(int initialCapacity = 256, bool poolLargeArrays = true) { ArgumentOutOfRangeException.ThrowIfNegative(initialCapacity); - array = ArrayPool.Shared.Rent(Math.Max(initialCapacity, MinimumCapacity)); + this.poolLargeArrays = poolLargeArrays; + array = Rent(Math.Max(initialCapacity, MinimumCapacity)); } /// Gets the number of bytes that have been written. @@ -112,7 +127,7 @@ public void Dispose() } returned = true; - ArrayPool.Shared.Return(array); + Return(array); array = []; length = 0; } @@ -124,14 +139,36 @@ private void EnsureFree(int sizeHint) return; } - Grow(length + sizeHint); + Grow((long)length + sizeHint); } - private void Grow(int minimumCapacity) + private void Grow(long minimumCapacity) { - var grown = ArrayPool.Shared.Rent(Math.Max(minimumCapacity, array.Length * 2)); + // Doubling is computed in 64 bits and clamped to the largest array the runtime allows, so a buffer + // past a gigabyte grows to the limit instead of to a negative size; a request beyond the limit is + // refused. The decode limits keep every caller below it, so this is a guard and not a path. + if (minimumCapacity > Array.MaxLength) + { + throw new InvalidOperationException($"A buffer of {minimumCapacity} bytes is larger than the runtime allows."); + } + + var capacity = (int)Math.Min(Math.Max(minimumCapacity, 2L * array.Length), Array.MaxLength); + var grown = Rent(capacity); array.AsSpan(0, length).CopyTo(grown); - ArrayPool.Shared.Return(array); + Return(array); array = grown; } + + private byte[] Rent(int length) => + poolLargeArrays || length <= MaximumPooledDecodeLength + ? ArrayPool.Shared.Rent(length) + : GC.AllocateUninitializedArray(length); + + private void Return(byte[] rented) + { + if (poolLargeArrays || rented.Length <= MaximumPooledDecodeLength) + { + ArrayPool.Shared.Return(rented); + } + } } diff --git a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Overlays.cs b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Overlays.cs index 7994f4e..23e7ab5 100644 --- a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Overlays.cs +++ b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Overlays.cs @@ -252,7 +252,7 @@ private static void AppendOverlayContent(PdfFileReader reader, PdfObject? value, return; } - content.Write(PdfFilters.Decode(stream, reader.SourceName)); + content.Write(PdfFilters.Decode(stream, reader.SourceName, reader.Options.MaxDecodedStreamBytes)); content.WriteByte((byte)'\n'); } diff --git a/src/SharpCorePDF.Infrastructure/PdfEngineOptions.cs b/src/SharpCorePDF.Infrastructure/PdfEngineOptions.cs index 3cdff45..8af71bd 100644 --- a/src/SharpCorePDF.Infrastructure/PdfEngineOptions.cs +++ b/src/SharpCorePDF.Infrastructure/PdfEngineOptions.cs @@ -18,6 +18,24 @@ public sealed record PdfEngineOptions /// public long MaxSourceBytes { get; init; } = 512L * 1024 * 1024; + /// + /// Gets the largest number of bytes one filter of one stream may decode to. A stream that would decode + /// past it is refused with a instead of + /// exhausting the process. + /// + /// + /// bounds what is read, not what it decodes to: Flate reaches about a + /// thousand to one, LZW four thousand and RunLength 128, so a one-megabyte document can ask for a + /// gigabyte. The default of 128 MiB is far above any real content stream, object stream or font program, + /// and low enough that refusing a stream at the limit — whose buffer doubles on the way there, on the + /// large-object heap, which the collector does not compact by default — stays reliably inside a 1 GiB + /// process when every read pass meets the same stream; the largest value accepted is the largest array + /// the runtime allows. An image is not refused by this limit — its samples are kept only + /// within already — but every other stream the engine has to + /// materialise is: content, forms, object streams, cross-reference streams and font programs. + /// + public long MaxDecodedStreamBytes { get; init; } = 128L * 1024 * 1024; + /// Gets the maximum nesting depth of dictionaries and arrays. public int MaxNestingDepth { get; init; } = 64; @@ -74,6 +92,11 @@ public IReadOnlyList Validate() errors.Add($"MaxSourceBytes must be at least 1024, but is {MaxSourceBytes}."); } + if (MaxDecodedStreamBytes < 1024 || MaxDecodedStreamBytes > Array.MaxLength) + { + errors.Add($"MaxDecodedStreamBytes must be between 1024 and {Array.MaxLength}, but is {MaxDecodedStreamBytes}."); + } + if (MaxNestingDepth is < 4 or > 1024) { errors.Add($"MaxNestingDepth must be between 4 and 1024, but is {MaxNestingDepth}."); diff --git a/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Content.cs b/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Content.cs index ae14104..6eaa240 100644 --- a/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Content.cs +++ b/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Content.cs @@ -64,7 +64,7 @@ public sealed partial class PdfPageContentReader foreach (var stream in streams) { - if (TryDecodeStream(stream, out var decoded)) + if (TryDecodeStream(stream, reader.Options.MaxDecodedStreamBytes, out var decoded)) { // The separator goes between the streams, not after the last one: a single stream is its // own sequence, and an inline image that runs to the end of it must not gain a byte. @@ -114,13 +114,14 @@ private static void CollectStreams(PdfFileReader reader, PdfArray array, ListDecodes a stream whose filters are the ones a content stream may use. /// The stream. + /// The largest number of bytes the stream may decode to. /// The decoded bytes when decoding succeeded. /// when the stream could be decoded. - private static bool TryDecodeStream(PdfStream stream, out byte[] decoded) + private static bool TryDecodeStream(PdfStream stream, long maxDecodedBytes, out byte[] decoded) { try { - decoded = PdfFilters.Decode(stream, "content stream"); + decoded = PdfFilters.Decode(stream, "content stream", maxDecodedBytes); return true; } catch (Domain.Exceptions.PdfUnsupportedFeatureException) diff --git a/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Images.cs b/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Images.cs index 9f9c48d..e56a77b 100644 --- a/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Images.cs +++ b/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Images.cs @@ -358,7 +358,7 @@ private void Paint( return; } - if (TryDecodeStream(stream, out var formContent)) + if (TryDecodeStream(stream, reader.Options.MaxDecodedStreamBytes, out var formContent)) { var formResources = reader.ResolveDictionary(stream.Dictionary.Get("Resources")) ?? resources; var formMatrix = ReadMatrix(reader, stream.Dictionary.Get("Matrix")) ?? PdfMatrix.Identity; diff --git a/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Metadata.cs b/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Metadata.cs index 027f05b..3681e29 100644 --- a/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Metadata.cs +++ b/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Metadata.cs @@ -35,7 +35,7 @@ public async Task ReadMetadataAsync( return new PdfXmpResult { Warnings = [.. reader.Warnings, .. warnings] }; } - if (!TryDecodeStream(stream, out var packet)) + if (!TryDecodeStream(stream, reader.Options.MaxDecodedStreamBytes, out var packet)) { warnings.Add(new( PdfWarningKind.XmpPacketMalformed, diff --git a/src/SharpCorePDF.Infrastructure/PublicAPI.Unshipped.txt b/src/SharpCorePDF.Infrastructure/PublicAPI.Unshipped.txt index 7dc5c58..f73aab7 100644 --- a/src/SharpCorePDF.Infrastructure/PublicAPI.Unshipped.txt +++ b/src/SharpCorePDF.Infrastructure/PublicAPI.Unshipped.txt @@ -1 +1,3 @@ #nullable enable +SharpCorePDF.Infrastructure.PdfEngineOptions.MaxDecodedStreamBytes.get -> long +SharpCorePDF.Infrastructure.PdfEngineOptions.MaxDecodedStreamBytes.init -> void diff --git a/src/SharpCorePDF.Infrastructure/Signatures/SignatureValidator.cs b/src/SharpCorePDF.Infrastructure/Signatures/SignatureValidator.cs index d0254eb..74bb5f0 100644 --- a/src/SharpCorePDF.Infrastructure/Signatures/SignatureValidator.cs +++ b/src/SharpCorePDF.Infrastructure/Signatures/SignatureValidator.cs @@ -258,7 +258,7 @@ public static PdfSignatureValidation Validate(PdfFileReader reader, SignatureFie $"The timestamp of '{field.FieldName}' is present but could not be read, so the moment it claims cannot be relied on.")); } - if (reading.Usable && !reading.CoversWhole) + if (reading.Usable && reading.GapHoldsValue && !reading.CoversWhole) { var revisions = CountRevisions(reader.Data, reader.Data.LongLength); @@ -278,6 +278,7 @@ public static PdfSignatureValidation Validate(PdfFileReader reader, SignatureFie /// The covered ranges. /// Whether the dictionary carries four usable numbers. /// Whether the ranges are ordered and inside the document. + /// Whether the bytes the ranges leave out are exactly the signature value. /// The DER structure of the signature value, without its padding. /// The parsed signature, when it parsed. /// Why the signature value could not be parsed, when it could not. @@ -287,6 +288,7 @@ private sealed record Reading( PdfByteRange Range, bool HasRange, bool Usable, + bool GapHoldsValue, byte[]? Contents, CmsSignature? Cms, Exception? Failure, @@ -314,6 +316,7 @@ private static Reading Read(PdfFileReader reader, PdfDictionary dictionary) && range.IsWellFormed && range.Length1 > 0 && range.Offset2 + range.Length2 <= reader.Data.LongLength; + var gapHoldsValue = usable && GapHoldsContents(reader, dictionary, range); var contents = ReadContents(reader, dictionary); CmsSignature? cms = null; Exception? failure = null; @@ -334,13 +337,72 @@ private static Reading Read(PdfFileReader reader, PdfDictionary dictionary) range, hasRange, usable, + gapHoldsValue, contents, cms, failure, - usable && range.Covers(reader.Data.LongLength), + usable && gapHoldsValue && range.Covers(reader.Data.LongLength), usable ? CountRevisions(reader.Data, range.Offset2 + range.Length2) : 0); } + /// + /// Determines whether the bytes the byte range leaves uncovered are exactly the signature value: one + /// hexadecimal string, delimiters included, and nothing else. + /// + /// The opened document. + /// The signature dictionary. + /// The byte range, already known to be ordered and inside the document. + /// when the gap holds the value and nothing beside it. + /// + /// The gap is never hashed, so anything in it is unsigned. A gap that closes the value early and carries + /// entries or objects of its own still leaves every signed byte intact and the digest matching — the + /// signature-wrapping attack — so the digest cannot be what catches it. A gap that is a single hex string + /// can only carry the value itself, and changing that breaks the digest. The value is also compared with + /// the /Contents the dictionary resolves to, so a dictionary that points its value somewhere else + /// cannot leave an unrelated string in the gap; a document whose strings were decrypted on the way in is + /// compared by shape only, because the parser hands over the decrypted form. + /// + private static bool GapHoldsContents(PdfFileReader reader, PdfDictionary dictionary, PdfByteRange range) + { + var data = reader.Data; + var start = checked((int)(range.Offset1 + range.Length1)); + var end = checked((int)range.Offset2); + + if (end - start < 2 || data[start] != (byte)'<' || data[end - 1] != (byte)'>') + { + return false; + } + + var inner = data.AsSpan(start + 1, end - start - 2); + var digits = new List(inner.Length); + + foreach (var value in inner) + { + if (Uri.IsHexDigit((char)value)) + { + digits.Add(value); + } + else if (!PdfLexer.IsWhitespace(value)) + { + return false; + } + } + + if (reader.Security is not null) + { + return true; + } + + // An odd number of digits is completed with a zero, as the specification reads a hex string. + if (digits.Count % 2 == 1) + { + digits.Add((byte)'0'); + } + + return reader.Resolve(dictionary.Get("Contents")) is PdfString contents + && Convert.FromHexString(System.Text.Encoding.ASCII.GetString(digits.ToArray())).AsSpan().SequenceEqual(contents.Bytes); + } + /// Reads the four numbers of the byte range. /// The opened document. /// The signature dictionary. @@ -469,6 +531,15 @@ private static PdfSignatureIntegrity Check( return PdfSignatureIntegrity.Unknown; } + if (!reading.GapHoldsValue) + { + problems.Add(new PdfSignatureProblem( + PdfSignatureProblemKind.MalformedByteRange, + $"The signature '{fieldName}' leaves bytes {reading.Range.Offset1 + reading.Range.Length1} to {reading.Range.Offset2} uncovered, and they are not exactly its /Contents value: bytes no signature covers would be read as part of the document.")); + + return PdfSignatureIntegrity.Unknown; + } + if (reading.Contents is not { Length: > 0 }) { problems.Add(new PdfSignatureProblem( diff --git a/tests/SharpCorePDF.Tests/PdfEngineOptionsTests.cs b/tests/SharpCorePDF.Tests/PdfEngineOptionsTests.cs index 9252147..5d58175 100644 --- a/tests/SharpCorePDF.Tests/PdfEngineOptionsTests.cs +++ b/tests/SharpCorePDF.Tests/PdfEngineOptionsTests.cs @@ -19,6 +19,7 @@ public void Has_the_documented_defaults() var options = PdfEngineOptions.Default; Assert.Equal(512L * 1024 * 1024, options.MaxSourceBytes); + Assert.Equal(128L * 1024 * 1024, options.MaxDecodedStreamBytes); Assert.Equal(64, options.MaxNestingDepth); Assert.Equal(100_000, options.MaxPagesPerSource); Assert.True(options.ReconstructCrossReferenceOnFailure); @@ -44,7 +45,16 @@ public void Reports_a_limit_that_cannot_be_honoured() (PdfEngineOptions.Default with { MaxPagesPerSource = 0 }).Validate(), error => error.Contains("MaxPagesPerSource", StringComparison.Ordinal)); + Assert.Contains( + (PdfEngineOptions.Default with { MaxDecodedStreamBytes = 1023 }).Validate(), + error => error.Contains("MaxDecodedStreamBytes", StringComparison.Ordinal)); + Assert.Contains( + (PdfEngineOptions.Default with { MaxDecodedStreamBytes = Array.MaxLength + 1L }).Validate(), + error => error.Contains("MaxDecodedStreamBytes", StringComparison.Ordinal)); + Assert.Empty((PdfEngineOptions.Default with { MaxSourceBytes = 1024 }).Validate()); + Assert.Empty((PdfEngineOptions.Default with { MaxDecodedStreamBytes = 1024 }).Validate()); + Assert.Empty((PdfEngineOptions.Default with { MaxDecodedStreamBytes = Array.MaxLength }).Validate()); Assert.Empty((PdfEngineOptions.Default with { MaxNestingDepth = 1024 }).Validate()); Assert.Empty((PdfEngineOptions.Default with { MaxPagesPerSource = 1 }).Validate()); } diff --git a/tests/SharpCorePDF.Tests/PdfFilterTests.cs b/tests/SharpCorePDF.Tests/PdfFilterTests.cs index 65cf9f2..99e6525 100644 --- a/tests/SharpCorePDF.Tests/PdfFilterTests.cs +++ b/tests/SharpCorePDF.Tests/PdfFilterTests.cs @@ -1,7 +1,11 @@ using System.IO.Compression; using System.Text; using SharpCorePDF.Domain.Exceptions; +using SharpCorePDF.Application.Services; +using SharpCorePDF.Domain.ValueObjects; +using SharpCorePDF.Infrastructure; using SharpCorePDF.Infrastructure.Pdf; +using SharpCorePDF.Infrastructure.Sources; namespace SharpCorePDF.Tests; @@ -144,6 +148,98 @@ public void Inflates_flate_data_however_it_was_written() Assert.Equal(text, raw); } + /// + /// A Flate stream that decodes past the limit is refused rather than inflated: zeros deflate about a + /// thousand to one, so the input limit alone lets a small document ask for a gigabyte. + /// + [Fact] + public void Refuses_a_flate_stream_that_decodes_past_the_limit() + { + var bomb = Deflate(new byte[2 * 1024 * 1024]); + + Assert.True(bomb.Length < 16 * 1024, "the fixture has to be small for the test to say anything"); + + var error = Assert.Throws(() => PdfFilters.Inflate(bomb, "bomb.pdf", 1024 * 1024)); + + Assert.Contains("MaxDecodedStreamBytes", error.Message, StringComparison.Ordinal); + Assert.Equal(2 * 1024 * 1024, PdfFilters.Inflate(bomb, "bomb.pdf", 2 * 1024 * 1024).Length); + Assert.True(PdfFilters.TryInflate(Deflate(new byte[1024]), out var small)); + Assert.Equal(1024, small.Length); + } + + /// An LZW stream is held to the same limit: one 12-bit code can stand for 4,096 bytes. + [Fact] + public void Refuses_an_lzw_stream_that_decodes_past_the_limit() + { + var encoded = TestLzw.Encode(new byte[100_000]); + + var error = Assert.Throws(() => PdfFilters.ApplyLzw(encoded, null, "lzw.pdf", 99_999)); + + Assert.Contains("MaxDecodedStreamBytes", error.Message, StringComparison.Ordinal); + Assert.Contains("LZW", error.Message, StringComparison.Ordinal); + Assert.Equal(100_000, PdfFilters.ApplyLzw(encoded, null, "lzw.pdf", 100_000).Length); + } + + /// A RunLength stream is held to the limit too: two bytes stand for 128. + [Fact] + public void Refuses_a_run_length_stream_that_decodes_past_the_limit() + { + var runs = Enumerable.Repeat(new byte[] { 0x81, 0x00 }, 1000).SelectMany(run => run).ToArray(); + + var error = Assert.Throws(() => PdfFilters.DecodeRunLength(runs, "runs.pdf", 127_999)); + + Assert.Contains("MaxDecodedStreamBytes", error.Message, StringComparison.Ordinal); + Assert.Contains("RunLength", error.Message, StringComparison.Ordinal); + Assert.Equal(128_000, PdfFilters.DecodeRunLength(runs, "runs.pdf", 128_000).Length); + } + + /// Literal runs are held to the limit on the same exact boundary as repeated ones. + [Fact] + public void Refuses_literal_runs_that_decode_past_the_limit() + { + byte[] run = [0x7F, .. Enumerable.Range(0, 128).Select(value => (byte)value)]; + var runs = Enumerable.Repeat(run, 100).SelectMany(bytes => bytes).ToArray(); + + var error = Assert.Throws(() => PdfFilters.DecodeRunLength(runs, "runs.pdf", 12_799)); + + Assert.Contains("RunLength", error.Message, StringComparison.Ordinal); + Assert.Equal(12_800, PdfFilters.DecodeRunLength(runs, "runs.pdf", 12_800).Length); + } + + /// + /// A page whose content stream decodes past the limit is reported as unreadable, the way any other + /// undecodable content is, and the read finishes: the limit turns a bomb into a warning. + /// + [Fact] + public async Task Reports_a_content_stream_past_the_limit_as_unreadable_instead_of_inflating_it() + { + var builder = new PdfBuilder(); + var catalog = builder.Reserve(); + var pages = builder.Reserve(); + + builder.Add("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Contents 4 0 R >>"); + builder.AddStream("/Filter /FlateDecode", PdfBuilder.Compress(new byte[1024 * 1024])); + builder.Set(catalog, "<< /Type /Catalog /Pages 2 0 R >>"); + builder.Set(pages, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + var document = builder.Build(catalog); + + var limited = new PdfContentReaderService( + new PdfPageContentReader(PdfEngineOptions.Default with { MaxDecodedStreamBytes = 64 * 1024 })); + var text = await limited.ReadTextAsync( + new PdfByteArraySource(document, "bomb.pdf"), + PageRange.All, + cancellationToken: TestContext.Current.CancellationToken); + + Assert.Contains(text.Warnings, warning => warning.Kind == PdfWarningKind.ContentStreamUnreadable); + + var unlimited = await new PdfContentReaderService(new PdfPageContentReader()).ReadTextAsync( + new PdfByteArraySource(document, "bomb.pdf"), + PageRange.All, + cancellationToken: TestContext.Current.CancellationToken); + + Assert.DoesNotContain(unlimited.Warnings, warning => warning.Kind == PdfWarningKind.ContentStreamUnreadable); + } + /// Flate data that cannot be decompressed is refused and reported as a damaged source. [Fact] public void Reports_data_that_cannot_be_decompressed() diff --git a/tests/SharpCorePDF.Tests/PdfJpegFilterTests.cs b/tests/SharpCorePDF.Tests/PdfJpegFilterTests.cs index 10c77a8..6db9f08 100644 --- a/tests/SharpCorePDF.Tests/PdfJpegFilterTests.cs +++ b/tests/SharpCorePDF.Tests/PdfJpegFilterTests.cs @@ -35,6 +35,96 @@ public sealed class PdfJpegFilterTests private static string Fixture(string name) => Path.Combine(TestAssets.Folder, "jpeg", name); + /// A budget of exactly the output's size keeps the pixels; one byte less measures them. + [Fact] + public void Keeps_pixels_that_fit_the_budget_exactly() + { + Assert.True(JpegDecoder.TryDecode(Rgb, 1 << 20, out var all, out _, out var width, out var height)); + + var exact = width * height * 3; + + Assert.Equal(exact, all!.Length); + Assert.True(JpegDecoder.TryDecode(Rgb, exact, out var kept, out _, out _, out _)); + Assert.Equal(all, kept); + Assert.True(JpegDecoder.TryDecode(Rgb, exact - 1, out var measured, out _, out _, out _)); + Assert.Null(measured); + } + + /// An image above the budget is measured with the format its components state. + /// The fixture. + /// The format it decodes to. + [Theory] + [InlineData("jpeg-progressive-gray.jpg", "Gray8")] + [InlineData("jpeg-cmyk.jpg", "Cmyk32")] + [InlineData("jpeg-444.jpg", "Rgb24")] + public void Measures_the_format_of_an_image_above_the_budget(string fixture, string expected) + { + Assert.True(JpegDecoder.TryDecode(File.ReadAllBytes(Fixture(fixture)), 16, out var pixels, out var format, out _, out _)); + Assert.Null(pixels); + Assert.Equal(expected, format); + } + + /// + /// A frame whose output fits a budget of while its coefficient grid does not fit + /// an array — 46,340 squared samples is just under the budget, at four bytes a coefficient it is eight + /// gigabytes — is refused before anything is allocated, instead of overflowing the size in 32 bits. + /// + [Fact] + public void Refuses_a_frame_whose_working_buffers_cannot_fit_an_array() + { + var jpeg = (byte[])ProgressiveGray.Clone(); + var frame = jpeg.AsSpan().IndexOf(new byte[] { 0xFF, 0xC2 }); + + Assert.True(frame > 0, "the fixture has to carry the frame header the test rewrites"); + + jpeg[frame + 5] = 46_340 >> 8; + jpeg[frame + 6] = 46_340 & 0xFF; + jpeg[frame + 7] = 46_340 >> 8; + jpeg[frame + 8] = 46_340 & 0xFF; + + var before = GC.GetAllocatedBytesForCurrentThread(); + var decoded = JpegDecoder.TryDecode(jpeg, int.MaxValue, out var pixels, out _, out _, out _); + var allocated = GC.GetAllocatedBytesForCurrentThread() - before; + + Assert.False(decoded); + Assert.Null(pixels); + Assert.True(allocated < 1 << 20, $"refusing the frame allocated {allocated} bytes"); + } + + /// + /// A frame header that claims 65,535 x 65,535 samples is measured from the header alone, with no pixels, + /// as any image above the budget is. The width and the height are two numbers in the file, and the + /// sample planes — four bytes a coefficient for a progressive image — were otherwise allocated before + /// the output was found too large to keep. + /// + /// Whether the progressive fixture is used rather than the baseline one. + [Theory] + [InlineData(false)] + [InlineData(true)] + public void Measures_a_frame_too_large_for_the_budget_without_allocating_it(bool progressive) + { + var jpeg = (byte[])(progressive ? Progressive : Rgb).Clone(); + var frame = jpeg.AsSpan().IndexOf(progressive ? [0xFF, 0xC2] : [0xFF, 0xC0]); + + Assert.True(frame > 0, "the fixture has to carry the frame header the test rewrites"); + + // Marker, length (two bytes), precision, then height and width, each a 16-bit big-endian number. + jpeg[frame + 5] = 0xFF; + jpeg[frame + 6] = 0xFF; + jpeg[frame + 7] = 0xFF; + jpeg[frame + 8] = 0xFF; + + var before = GC.GetAllocatedBytesForCurrentThread(); + var measured = JpegDecoder.TryDecode(jpeg, 1 << 20, out var pixels, out var format, out var width, out var height); + var allocated = GC.GetAllocatedBytesForCurrentThread() - before; + + Assert.True(measured); + Assert.Null(pixels); + Assert.Equal("Rgb24", format); + Assert.Equal((65_535, 65_535), (width, height)); + Assert.True(allocated < 1 << 20, $"refusing the frame allocated {allocated} bytes"); + } + /// A 4:4:4 RGB JPEG decodes to the bytes Pillow decodes from the same file. [Fact] public void Decodes_a_baseline_rgb_jpeg_within_two_levels() diff --git a/tests/SharpCorePDF.Tests/PdfReaderRecoveryTests.cs b/tests/SharpCorePDF.Tests/PdfReaderRecoveryTests.cs index fba32c8..da00411 100644 --- a/tests/SharpCorePDF.Tests/PdfReaderRecoveryTests.cs +++ b/tests/SharpCorePDF.Tests/PdfReaderRecoveryTests.cs @@ -18,6 +18,139 @@ namespace SharpCorePDF.Tests; /// public sealed class PdfReaderRecoveryTests { + /// + /// The bound on a chain of lengths leaves an ordinary indirect length alone, for every stream of a + /// document and not only the first ones: each stream's data holds an endstream line of its own, + /// so a length that was not resolved would cut the stream there instead of at its real end. + /// + [Fact] + public void Resolves_the_indirect_length_of_every_stream_of_a_document() + { + const int Streams = 100; + var data = "q\nendstream\nQ"u8.ToArray(); + var builder = new PdfBuilder(); + var catalog = builder.Reserve(); + var pages = builder.Reserve(); + + builder.Add("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Contents 4 0 R >>"); + + // Objects 4, 6, 8 ... are the streams; 5, 7, 9 ... are their lengths. + for (var stream = 0; stream < Streams; stream++) + { + var body = Encoding.Latin1.GetBytes($"<< /Length {5 + (2 * stream)} 0 R >>\nstream\n") + .Concat(data) + .Concat("\nendstream"u8.ToArray()) + .ToArray(); + + builder.Add(body); + builder.Add(data.Length.ToString(System.Globalization.CultureInfo.InvariantCulture)); + } + + builder.Set(catalog, "<< /Type /Catalog /Pages 2 0 R >>"); + builder.Set(pages, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + + var reader = new PdfFileReader(builder.Build(catalog), "lengths.pdf", PdfEngineOptions.Default); + + for (var stream = 0; stream < Streams; stream++) + { + var resolved = Assert.IsType(reader.Resolve(4 + (2 * stream))); + + Assert.Equal(data, resolved.Raw.ToArray()); + } + } + + /// + /// An object stream that declares 2^31 - 1 objects in a header of one pair is read for the one pair it + /// holds. /N sized two arrays before anything compared it with the stream, so a few hundred bytes asked + /// for gigabytes; the stream now bounds the count, and the page inside it is found. + /// + [Fact] + public void Reads_an_object_stream_whose_count_its_header_cannot_hold() + { + var page = "3 0 << /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Contents 4 0 R >>"u8.ToArray(); + var packed = PdfBuilder.Compress(page); + var output = new MemoryStream(); + var offsets = new Dictionary(); + + void Write(string text) => output.Write(Encoding.Latin1.GetBytes(text)); + + void Object(int number, string dictionary, byte[]? stream = null) + { + offsets[number] = output.Position; + Write($"{number} 0 obj\n{dictionary}\n"); + + if (stream is not null) + { + Write("stream\n"); + output.Write(stream); + Write("\nendstream\n"); + } + + Write("endobj\n"); + } + + Write("%PDF-1.7\n"); + Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); + Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + Object(4, "<< /Length 3 >>", "q Q"u8.ToArray()); + Object(5, $"<< /Type /ObjStm /N 2147483647 /First 4 /Filter /FlateDecode /Length {packed.Length} >>", packed); + + offsets[6] = output.Position; + (int Kind, long Field, int Index)[] entries = + [ + (0, 0, 65535), (1, offsets[1], 0), (1, offsets[2], 0), (2, 5, 0), + (1, offsets[4], 0), (1, offsets[5], 0), (1, offsets[6], 0), + ]; + var table = entries + .SelectMany(entry => new[] + { + (byte)entry.Kind, + (byte)(entry.Field >> 24), (byte)(entry.Field >> 16), (byte)(entry.Field >> 8), (byte)entry.Field, + (byte)(entry.Index >> 8), (byte)entry.Index, + }) + .ToArray(); + + Write($"6 0 obj\n<< /Type /XRef /Size 7 /W [1 4 2] /Root 1 0 R /Length {table.Length} >>\nstream\n"); + output.Write(table); + Write($"\nendstream\nendobj\nstartxref\n{offsets[6]}\n%%EOF\n"); + + var reader = new PdfFileReader(output.ToArray(), "objstm.pdf", PdfEngineOptions.Default); + + Assert.Single(reader.Pages); + Assert.Equal("Page", Assert.IsType(reader.Resolve(3)).GetName("Type")); + } + + /// + /// A stream whose /Length is another stream, whose length is another, sixty thousand deep, is + /// read by finding its endstream. Resolved link by link, the chain would overflow the stack, and a + /// stack overflow ends the process rather than throwing anything a caller could catch. + /// + [Fact] + public void Reads_a_stream_whose_length_is_a_chain_of_sixty_thousand_streams() + { + const int Chain = 60_000; + var builder = new PdfBuilder(); + var catalog = builder.Reserve(); + var pages = builder.Reserve(); + + builder.Add("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Contents 4 0 R >>"); + + for (var link = 0; link < Chain; link++) + { + builder.Add($"<< /Length {5 + link} 0 R >>\nstream\nq Q\nendstream"); + } + + builder.Add("3"); + builder.Set(catalog, "<< /Type /Catalog /Pages 2 0 R >>"); + builder.Set(pages, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + + var reader = new PdfFileReader(builder.Build(catalog), "chain.pdf", PdfEngineOptions.Default); + var content = Assert.IsType(reader.Resolve(4)); + + Assert.Equal("q Q", Encoding.Latin1.GetString(content.Raw.Span)); + Assert.Single(reader.Pages); + } + /// A table that no longer parses is rebuilt, and the document is read completely. [Fact] public async Task Rebuilds_a_cross_reference_table_that_does_not_parse() diff --git a/tests/SharpCorePDF.Tests/PdfSignatureAttackTests.cs b/tests/SharpCorePDF.Tests/PdfSignatureAttackTests.cs index 52957e9..bbedc1f 100644 --- a/tests/SharpCorePDF.Tests/PdfSignatureAttackTests.cs +++ b/tests/SharpCorePDF.Tests/PdfSignatureAttackTests.cs @@ -104,8 +104,7 @@ public async Task Never_reports_a_signature_whose_value_was_emptied_as_valid() /// what must fail is the claim that the signature covers the whole document, because part of the /// document is now something other than the signature value. /// - [Fact(Skip = - "Known defect: SignatureValidator does not check that the gap between the two ranges is exactly the /Contents string, so this wrapped document is reported as covering the whole file. Remove the skip with the fix.")] + [Fact] public async Task Does_not_report_a_gap_that_holds_more_than_the_signature_value_as_covering_the_document() { var signed = await SignAsync(); @@ -114,14 +113,76 @@ public async Task Does_not_report_a_gap_that_holds_more_than_the_signature_value // The attack only works while the signed bytes are unchanged; this is what makes it an attack. var validation = Assert.Single(await SigningFixtures.ValidateAsync(wrapped)); - Assert.False( - validation.Signer.CoversWholeDocument && validation.IsValid, - "a document whose uncovered gap carries more than the signature value must not be reported as covered"); + Assert.False(validation.Signer.CoversWholeDocument); + Assert.False(validation.IsValid); + Assert.NotEqual(PdfSignatureIntegrity.Valid, validation.Integrity); + + var (gapStart, _) = Gap(signed); + var problem = Assert.Single(validation.Problems, problem => problem.Kind == PdfSignatureProblemKind.MalformedByteRange); + + Assert.Contains(gapStart.ToString(CultureInfo.InvariantCulture), problem.Message, StringComparison.Ordinal); + + // The wrapped gap is the finding; the revision count would be a second, misleading one. + Assert.DoesNotContain(validation.Problems, problem => problem.Kind == PdfSignatureProblemKind.RevisionNotCovered); + } + + /// + /// A gap that is not exactly one hexadecimal string is refused whichever way it fails to be one: no + /// opening delimiter, no closing delimiter, or a byte inside it that is neither a digit nor whitespace. + /// + /// Which byte of the gap is changed. + [Theory] + [InlineData("opening-delimiter")] + [InlineData("closing-delimiter")] + [InlineData("not-a-hex-digit")] + public async Task Refuses_a_gap_that_is_not_one_hexadecimal_string(string variant) + { + var signed = await SignAsync(); + var (gapStart, gapEnd) = Gap(signed); + var changed = (byte[])signed.Clone(); + var (index, value) = variant switch + { + "opening-delimiter" => (gapStart, (byte)'('), + "closing-delimiter" => (gapEnd - 1, (byte)')'), + "not-a-hex-digit" => (gapEnd - 2, (byte)'g'), + _ => throw new ArgumentOutOfRangeException(nameof(variant), variant, "no such variant"), + }; + + changed[index] = value; + + var validation = Assert.Single(await SigningFixtures.ValidateAsync(changed)); + + Assert.False(validation.IsValid); + Assert.False(validation.Signer.CoversWholeDocument); + Assert.Contains(validation.Problems, problem => problem.Kind == PdfSignatureProblemKind.MalformedByteRange); + } + + /// + /// Whitespace inside the value is part of PDF's hexadecimal string syntax, and an odd number of digits is + /// completed with a zero; a gap written that way is still exactly the signature value and stays valid. + /// + [Fact] + public async Task Accepts_whitespace_and_an_odd_digit_count_inside_the_value() + { + var signed = await SignAsync(); + var (_, gapEnd) = Gap(signed); + var spaced = (byte[])signed.Clone(); + + // The last digit of the padding becomes a space: the value now has an odd number of digits, which a + // reader completes with the zero that was there. + Assert.Equal((byte)'0', spaced[gapEnd - 2]); + spaced[gapEnd - 2] = (byte)' '; + + var validation = Assert.Single(await SigningFixtures.ValidateAsync(spaced)); + + Assert.True(validation.IsValid); + Assert.True(validation.Signer.CoversWholeDocument); + Assert.Equal(PdfSignatureIntegrity.Valid, validation.Integrity); } /// - /// The precondition the skipped wrapping test depends on, asserted on its own so it keeps running: the - /// injection leaves every covered byte as it was, which is why only a gap check can catch it. + /// The precondition the wrapping test depends on: the injection leaves every covered byte as it was, which + /// is why only a gap check can catch it. /// [Fact] public async Task Leaves_every_covered_byte_unchanged_when_the_gap_is_wrapped() From 9e2ab83a9616915a9746852b7f1ec3aa7a3b6871 Mon Sep 17 00:00:00 2001 From: Andrew Tierney Date: Mon, 16 Nov 2026 12:26:01 +1100 Subject: [PATCH 4/4] Read the documents the JPEG, empty-name, glyph-box and date faults refused Four defects turned documents into exceptions that were not PdfExceptions, so a caller that handles the contract's refusal still saw the read fail: - A JPEG scan selected Huffman tables with 4-bit selectors and a progressive scan named its spectral selection, and both were used as indexes unchecked (IndexOutOfRangeException). ParseScan now refuses a selector above 3, and a progressive scan unless Ss <= Se <= 63 and Ah, Al <= 13. - The empty name, a solidus on its own, is a name PDF allows, and four documents of the pdf.js test corpus use it as a key; PdfDictionary.Set refused it, so they did not open at all. - A glyph drawn upside down (a negative font size) gave PdfGlyph.BoundingBox a bottom above its top, and the rectangle refused it, failing text extraction for the page. - PdfDate.TryParse threw for a time-zone offset beyond fourteen hours, so one malformed /CreationDate failed the inspection of the whole document; such an offset is now read as no offset. PdfContainerTests.Refuses_an_entry_without_a_key asserted the second defect and is rewritten: a null key is still refused, the empty name is kept. Tests that fail without the fixes: PdfJpegFilterTests (table selectors, progressive scan bounds), PdfObjectModelTests (empty-name key, in a dictionary and in a document), PdfTextScannerTests (upside-down glyph), PdfSerializationTests (offsets past fourteen hours). The pdf.js corpus and the hostile documents now read with no fault, crash or timeout, and their --known flags are removed. --- .github/workflows/robustness.yml | 5 -- .github/workflows/validation.yml | 7 +- CHANGELOG.md | 14 +++- docs/validation.md | 11 +-- .../ReadModels/PdfGlyph.cs | 8 +- .../Pdf/JpegDecoder.cs | 13 ++- .../Pdf/PdfDate.cs | 8 +- .../Pdf/PdfDictionary.cs | 4 +- tests/SharpCorePDF.Tests/PdfContainerTests.cs | 15 +++- .../SharpCorePDF.Tests/PdfJpegFilterTests.cs | 80 +++++++++++++++++++ .../SharpCorePDF.Tests/PdfObjectModelTests.cs | 40 ++++++++++ .../PdfSerializationTests.cs | 22 +++++ .../SharpCorePDF.Tests/PdfTextScannerTests.cs | 25 ++++++ 13 files changed, 224 insertions(+), 28 deletions(-) diff --git a/.github/workflows/robustness.yml b/.github/workflows/robustness.yml index 93de31b..50854f4 100644 --- a/.github/workflows/robustness.yml +++ b/.github/workflows/robustness.yml @@ -162,16 +162,11 @@ jobs: - name: Build the probe run: dotnet build ./tools/SharpCorePDF.Validation -c Release - # The six known files fault on three recorded defects (docs/validation.md): an empty name as a - # dictionary key (issue3188, issue5896, pdfjs_wikipedia, poppler-742-0-fuzzed), a glyph whose box is - # upside down (issue14415) and a date whose offset is beyond 14 hours (TAMReview). - name: Read every document run: | python3 docs/tools/corpus-robustness.py \ --probe ./tools/SharpCorePDF.Validation/bin/Release/net11.0/SharpCorePDF.Validation.dll \ --timeout 60 --heap-limit-mb 1024 --report artifacts/corpora.json \ - --known issue3188.pdf --known issue5896.pdf --known pdfjs_wikipedia.pdf \ - --known poppler-742-0-fuzzed.pdf --known issue14415.pdf --known TAMReview.pdf \ "$RUNNER_TEMP/verapdf-corpus" "$RUNNER_TEMP/pdfjs/test/pdfs" - name: Upload the report diff --git a/.github/workflows/validation.yml b/.github/workflows/validation.yml index de42cbe..a632011 100644 --- a/.github/workflows/validation.yml +++ b/.github/workflows/validation.yml @@ -146,9 +146,9 @@ jobs: dotnet "$tool" replay "$target" "$seeds/$target" done - # Each of the four documents reaches a failure class the review named. flate-bomb, length-chain and - # objstm-count are refused or read since the decode limit, the length-chain bound and the /N bound; - # jpeg-selector is a known defect until the change that fixes it removes its --known line. + # Each of the four documents reaches a failure class a review named, and each is now read: the decode + # limit, the length-chain bound, the /N bound and the JPEG scan-header checks closed them. A document + # that faults here again is a regression, so none is listed as known. - name: Hostile documents run: | set -euo pipefail @@ -156,7 +156,6 @@ jobs: python docs/tools/corpus-robustness.py \ --probe ./tools/SharpCorePDF.Validation/bin/Release/net11.0/SharpCorePDF.Validation.dll \ --timeout 120 --report artifacts/hostile.json \ - --known jpeg-selector.pdf \ "$RUNNER_TEMP/hostile" - name: Upload the documents and the reports diff --git a/CHANGELOG.md b/CHANGELOG.md index abe4bd9..01e5662 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,6 +55,16 @@ against an empty feed. is measured from its header without being decoded — the same answer as before, without the allocation — and the working sizes are computed in 64 bits. +- **A JPEG scan header is checked before it is used.** A Huffman table selector above 3, and a progressive + scan whose spectral selection runs past coefficient 63 or starts after it ends, indexed past the decoder's + tables (`IndexOutOfRangeException`); such a scan is now refused and the image reported undecodable. +- **The empty name is a dictionary key like any other.** `/` on its own is a name PDF allows; four documents + of the pdf.js test corpus use it as a key and did not open at all (`ArgumentException`). +- **A glyph drawn upside down has a box.** A negative font size made `PdfGlyph.BoundingBox` throw, so text + extraction faulted on the page. +- **`PdfDate.TryParse` answers instead of throwing** for a time-zone offset beyond fourteen hours, which it + now reads as no offset; one malformed `/CreationDate` no longer fails the inspection of the document. + ### Known defects, found by the new checks and not yet fixed Each is reproduced, pinned by a skipped test or a strict `--known-defect` flag that fails the run once the @@ -63,10 +73,6 @@ defect is fixed, and listed in [`docs/validation.md`](docs/validation.md#known-d - An incremental update rewrites both elements of `/ID`, and appends a renumbered copy of the whole document rather than the objects it changed — which also rewrites the signature dictionary after signing. - A merge copies a page that states no `/MediaBox` without one. -- Hostile input: a JPEG scan's Huffman table selector is not range-checked. -- Real-world input (the pdf.js test corpus): an empty name as a dictionary key stops a document opening, a - glyph with an upside-down box faults text extraction, and `PdfDate.TryParse` throws on an out-of-range - time-zone offset instead of answering false. - **Run-length images are decoded.** `RunLengthDecode` joins Flate, LZW and the two ASCII filters: a bilevel or scan-style image that declares it now yields its samples — and `Gray8`, `Rgb24` or `Cmyk32` diff --git a/docs/validation.md b/docs/validation.md index 6ce4585..1d79368 100644 --- a/docs/validation.md +++ b/docs/validation.md @@ -86,10 +86,6 @@ fix. This is the list on 2026-11-16, every entry reproduced, not inferred: | An incremental update writes a new identifier into both elements of `/ID`; ISO 32000-1 §14.4 keeps the first | #10 | `PdfIncrementalInvariantTests.Keeps_the_first_identifier_element_across_an_incremental_update` (skipped); `validate-incremental.py` `id-first` | | `UseIncrementalUpdate` appends a renumbered copy of the whole document rather than the objects it changed: every byte of the earlier revision is kept, but the file grows by its own size per edit, and the signature dictionary is written again after signing, so pyHanko places a signature followed by an edit as `CONTIGUOUS_BLOCK_FROM_START` with modification level `OTHER` instead of `ENTIRE_REVISION` | #10, #3 | `validate-incremental.py` `appends-only`; `validate-signatures.py` on `signed-then-edited.pdf` | | A merge copies a page that states no `/MediaBox` (and inherits none) without one, so the result breaks the rule that every page has a media box; qpdf repairs it by assuming Letter | #2 | `validate-pdf.py` on `inspected.pdf` and `with-warnings.pdf` (both merge a page of `NamedDests.pdf`) | -| The JPEG decoder indexes its four Huffman tables with a scan's 4-bit selector unchecked | #12, #1 | `jpeg-selector.pdf`: `IndexOutOfRangeException` from `JpegDecoder.DecodeBlock` | -| A dictionary key that is the empty name (`/` alone, which PDF syntax allows) makes `PdfDictionary.Set` throw `ArgumentException` while parsing, so the document does not open at all | #12 | pdf.js corpus: `issue3188.pdf`, `issue5896.pdf`, `pdfjs_wikipedia.pdf`, `poppler-742-0-fuzzed.pdf` | -| A glyph whose box comes out upside down (a negative font size or a flipped font matrix) makes `PdfGlyph.BoundingBox` construct a `PdfRectangle` that throws, so text extraction faults | #12 | pdf.js corpus: `issue14415.pdf` | -| `PdfDate.TryParse` throws `ArgumentOutOfRangeException` for a time-zone offset beyond 14 hours instead of answering false, so one malformed `/CreationDate` faults the whole inspection | #12 | pdf.js corpus: `TAMReview.pdf` | Fixed since the list was first drawn up, each with the test that fails without its fix: @@ -99,6 +95,10 @@ Fixed since the list was first drawn up, each with the test that fails without i | No limit on the decoded size of a Flate, LZW or RunLength stream | `PdfEngineOptions.MaxDecodedStreamBytes` (128 MiB) | `PdfFilterTests.Refuses_a_flate_stream_that_decodes_past_the_limit` and its LZW, RunLength and content-stream companions | | A chain of indirect `/Length` references overflowed the stack | the chain is followed no deeper than `MaxNestingDepth`, then the stream is found by its `endstream` | `PdfReaderRecoveryTests.Reads_a_stream_whose_length_is_a_chain_of_sixty_thousand_streams` | | An object stream's `/N`, and a JPEG frame's width and height, sized allocations unchecked | `/N` is bounded by what the stream can hold; a JPEG above the caller's budget is measured from its header, and working sizes are computed in 64 bits | `PdfReaderRecoveryTests.Reads_an_object_stream_whose_count_its_header_cannot_hold`, `PdfJpegFilterTests.Measures_a_frame_too_large_for_the_budget_without_allocating_it` | +| The JPEG decoder used a scan's 4-bit Huffman table selectors, and a progressive scan's spectral end, as indexes unchecked | `ParseScan` refuses a selector above 3, and a progressive scan whose selection is not `Ss ≤ Se ≤ 63` or whose approximation is above 13 | `PdfJpegFilterTests.Refuses_a_scan_that_selects_a_table_beyond_the_four_there_are`, `Refuses_a_progressive_scan_outside_the_block` | +| The empty name as a dictionary key (`/` alone, which PDF allows) stopped the document opening | `PdfDictionary.Set` accepts it; the entry is kept and written back as `/` | `PdfObjectModelTests.Keeps_an_entry_whose_key_is_the_empty_name`, `Opens_a_document_that_uses_the_empty_name_as_a_key` | +| A glyph drawn upside down (negative font size) faulted text extraction | `PdfGlyph.BoundingBox` takes the box between descender and ascender whichever way round | `PdfTextScannerTests.Gives_a_glyph_drawn_upside_down_a_box` | +| `PdfDate.TryParse` threw for a time-zone offset beyond 14 hours | an offset no time zone has is read as no offset | `PdfSerializationTests.Reads_an_offset_no_time_zone_has_as_no_offset` | ## Corpus readings, 2026-11-16 @@ -108,7 +108,8 @@ Fixed since the list was first drawn up, each with the test that fails without i | pdf.js `test/pdfs` at `v6.4.299` | 987 | 977 | 4 | 6 | 0 | 0 | | synthesized hostile documents, before the fixes | 4 | 0 | 0 | 3 | 1 | 0 | -The six pdf.js faults are the three defects above. A sample of the pre-flight measurement (the first 60 +The six pdf.js faults were the empty-name, upside-down glyph and date-offset defects in the fixed table; +after those fixes all six are read, as is `jpeg-selector.pdf`. A sample of the pre-flight measurement (the first 60 files of `PDF_A-2b` and of `PDF_A-4`) wrote 108 results, refused 9 non-conforming inputs and refused 3 that already conformed; whether the 108 conform is veraPDF's to say, which `robustness.yml` asks. diff --git a/src/SharpCorePDF.Domain/ReadModels/PdfGlyph.cs b/src/SharpCorePDF.Domain/ReadModels/PdfGlyph.cs index eb0ea41..f3106d9 100644 --- a/src/SharpCorePDF.Domain/ReadModels/PdfGlyph.cs +++ b/src/SharpCorePDF.Domain/ReadModels/PdfGlyph.cs @@ -70,13 +70,15 @@ public sealed record PdfGlyph /// /// Gets the area the glyph occupies, derived from the origin, the advance and the em box. The /// descender is taken as a fifth of the em and the ascender as four fifths, which is the proportion - /// most Latin fonts share and is stated here rather than read from the font program. + /// most Latin fonts share and is stated here rather than read from the font program. A negative font + /// size — a glyph drawn upside down, which a flipped text matrix or font matrix produces — puts the + /// descender above the ascender, so the box is taken between the two whichever way round they are. /// public PdfRectangle BoundingBox => new( X, - Y - (FontSize * 0.2d), + Math.Min(Y - (FontSize * 0.2d), Y + (FontSize * 0.8d)), X + Math.Max(0d, Width), - Y + (FontSize * 0.8d)); + Math.Max(Y - (FontSize * 0.2d), Y + (FontSize * 0.8d))); /// Gets a value indicating whether the glyph paints no ink, such as a space or a tab. public bool IsWhitespace => string.IsNullOrWhiteSpace(Text); diff --git a/src/SharpCorePDF.Infrastructure/Pdf/JpegDecoder.cs b/src/SharpCorePDF.Infrastructure/Pdf/JpegDecoder.cs index 528cf69..8919615 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/JpegDecoder.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/JpegDecoder.cs @@ -473,6 +473,13 @@ private bool ParseScan(ReadOnlySpan segment) return false; } + // A selector is four bits in the file and the decoder has four tables of each class: a selector + // above 3 would index past them, so it is a header this decoder cannot honour. + if (tables >> 4 > 3 || (tables & 15) > 3) + { + return false; + } + component.DcTable = tables >> 4; component.AcTable = tables & 15; named[index] = component; @@ -486,7 +493,11 @@ private bool ParseScan(ReadOnlySpan segment) scanHigh = segment[tail + 2] >> 4; scanLow = segment[tail + 2] & 15; - return true; + // The spectral selection names coefficients 0 to 63 of a block, start before end, and the successive + // approximation shifts no further than a 16-bit coefficient allows. A progressive scan indexes the + // zig-zag table with them, so a scan outside these bounds is refused rather than read past the + // table; a sequential scan states them too but decodes the whole block whatever they say. + return !progressive || (scanStart <= scanEnd && scanEnd <= 63 && scanHigh <= 13 && scanLow <= 13); } private void ParseAdobe(ReadOnlySpan segment) diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfDate.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfDate.cs index f240e68..80d8ce7 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfDate.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfDate.cs @@ -133,8 +133,12 @@ private static TimeSpan ReadOffset(ReadOnlySpan rest) field++; } - var sign = rest[0] == '-' ? -1 : 1; - return sign * new TimeSpan(Math.Clamp(hours, 0, 23), Math.Clamp(minutes, 0, 59), 0); + // DateTimeOffset accepts offsets up to fourteen hours either way and throws beyond them, so an offset + // no time zone has is read as no offset — UTC, which is what the specification assumes when none is + // given — instead of failing the inspection of the whole document over one metadata string. + var offset = (rest[0] == '-' ? -1 : 1) * new TimeSpan(Math.Clamp(hours, 0, 23), Math.Clamp(minutes, 0, 59), 0); + + return offset.Duration() > TimeSpan.FromHours(14) ? TimeSpan.Zero : offset; } /// Reads the next run of digits of a time zone offset, skipping what is not a digit. diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfDictionary.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfDictionary.cs index b877b2b..72643b5 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfDictionary.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfDictionary.cs @@ -54,7 +54,9 @@ public IEnumerable> Entries /// This dictionary, so calls can be chained. public PdfDictionary Set(string key, PdfObject? value) { - ArgumentException.ThrowIfNullOrEmpty(key); + // The empty name — a solidus on its own — is a name PDF allows (ISO 32000-1 §7.3.5), and documents + // use it as a key; refusing it here refused the whole document while it was being parsed. + ArgumentNullException.ThrowIfNull(key); if (value is null) { diff --git a/tests/SharpCorePDF.Tests/PdfContainerTests.cs b/tests/SharpCorePDF.Tests/PdfContainerTests.cs index b402acd..e562066 100644 --- a/tests/SharpCorePDF.Tests/PdfContainerTests.cs +++ b/tests/SharpCorePDF.Tests/PdfContainerTests.cs @@ -72,10 +72,19 @@ public void Copies_a_set_of_entries() Assert.Same(PdfNull.Instance, dictionary.Get("B")); } - /// An entry without a key is a programmer error. + /// + /// An entry without a key is a programmer error, but the empty name is a key: a solidus on its own is a + /// name PDF allows (ISO 32000-1 §7.3.5), and documents in the wild use it. + /// [Fact] - public void Refuses_an_entry_without_a_key() => - Assert.Throws(() => new PdfDictionary().Set(string.Empty, PdfNull.Instance)); + public void Refuses_an_entry_without_a_key_and_keeps_one_whose_key_is_the_empty_name() + { + Assert.Throws(() => new PdfDictionary().Set(null!, PdfNull.Instance)); + + var dictionary = new PdfDictionary().Set(string.Empty, new PdfInteger(1)); + + Assert.Equal(1L, dictionary.GetInteger(string.Empty)); + } /// The readers of a dictionary answer with the type they were asked for, or with nothing. [Fact] diff --git a/tests/SharpCorePDF.Tests/PdfJpegFilterTests.cs b/tests/SharpCorePDF.Tests/PdfJpegFilterTests.cs index 6db9f08..4a6644c 100644 --- a/tests/SharpCorePDF.Tests/PdfJpegFilterTests.cs +++ b/tests/SharpCorePDF.Tests/PdfJpegFilterTests.cs @@ -35,6 +35,86 @@ public sealed class PdfJpegFilterTests private static string Fixture(string name) => Path.Combine(TestAssets.Folder, "jpeg", name); + /// + /// A scan that selects a Huffman table the decoder does not have — the selector is four bits, the tables + /// number four of each class — is refused, not used as an index past them. + /// + /// The table selector byte of the first scan component: DC in the high nibble, AC in the low. + [Theory] + [InlineData(0x44)] + [InlineData(0x40)] + [InlineData(0x04)] + [InlineData(0xFF)] + public void Refuses_a_scan_that_selects_a_table_beyond_the_four_there_are(byte selectors) + { + var jpeg = (byte[])Rgb.Clone(); + var scan = jpeg.AsSpan().IndexOf(new byte[] { 0xFF, 0xDA }); + + Assert.True(scan > 0, "the fixture has to carry a scan header"); + + // Marker, length (two bytes), component count, then the first component's id and its selectors. + jpeg[scan + 6] = selectors; + + Assert.False(JpegDecoder.TryDecode(jpeg, 1 << 20, out var pixels, out _, out _, out _)); + Assert.Null(pixels); + } + + /// + /// A progressive scan whose spectral selection runs past coefficient 63, starts after it ends, or states a + /// successive approximation no 16-bit coefficient has is refused before it indexes the zig-zag table. + /// + /// Which field of the AC scan header is rewritten. + /// The value it is rewritten to. + [Theory] + [InlineData("end", 0xFF)] + [InlineData("end", 64)] + [InlineData("start-after-end", 0)] + [InlineData("approximation", 0xE0)] + [InlineData("approximation", 0x0E)] + public void Refuses_a_progressive_scan_outside_the_block(string field, int value) + { + var jpeg = (byte[])Progressive.Clone(); + var position = 0; + var tail = -1; + + // The first AC scan: a scan header whose spectral start is above zero. + while (tail < 0) + { + var scan = jpeg.AsSpan(position).IndexOf(new byte[] { 0xFF, 0xDA }); + + Assert.True(scan >= 0, "the fixture has to carry an AC scan"); + scan += position; + + var candidate = scan + 5 + (2 * jpeg[scan + 4]); + + if (jpeg[candidate] > 0) + { + tail = candidate; + } + + position = scan + 2; + } + + switch (field) + { + case "end": + jpeg[tail + 1] = (byte)value; + break; + + case "start-after-end": + jpeg[tail + 1] = (byte)value; + Assert.True(jpeg[tail] > value, "the scan has to start after the end it is given"); + break; + + default: + jpeg[tail + 2] = (byte)value; + break; + } + + Assert.False(JpegDecoder.TryDecode(jpeg, 1 << 20, out var pixels, out _, out _, out _)); + Assert.Null(pixels); + } + /// A budget of exactly the output's size keeps the pixels; one byte less measures them. [Fact] public void Keeps_pixels_that_fit_the_budget_exactly() diff --git a/tests/SharpCorePDF.Tests/PdfObjectModelTests.cs b/tests/SharpCorePDF.Tests/PdfObjectModelTests.cs index a520044..d59b80e 100644 --- a/tests/SharpCorePDF.Tests/PdfObjectModelTests.cs +++ b/tests/SharpCorePDF.Tests/PdfObjectModelTests.cs @@ -13,6 +13,46 @@ namespace SharpCorePDF.Tests; /// public sealed class PdfObjectModelTests { + /// + /// The empty name — a solidus on its own — is a key like any other: the dictionary keeps it and every + /// entry after it, and writing the dictionary back states it as a solidus again. + /// + [Fact] + public void Keeps_an_entry_whose_key_is_the_empty_name() + { + var parser = new PdfObjectParser(Encoding.Latin1.GetBytes("<< /A 1 / 2 /B 3 >>"), "empty-name.pdf", 0, null, 64); + var dictionary = Assert.IsType(parser.ParseObject()); + + Assert.Equal(1L, dictionary.GetInteger("A")); + Assert.Equal(2L, dictionary.GetInteger(string.Empty)); + Assert.Equal(3L, dictionary.GetInteger("B")); + + using var stream = new MemoryStream(); + var output = new PdfOutputBuffer(stream); + PdfSerializer.WriteDictionary(dictionary, output); + output.Flush(); + var written = Encoding.Latin1.GetString(stream.ToArray()); + + Assert.Contains("/ 2", written, StringComparison.Ordinal); + } + + /// A document whose page dictionary carries an empty-named entry opens and lists its page. + [Fact] + public void Opens_a_document_that_uses_the_empty_name_as_a_key() + { + var builder = new PdfBuilder(); + var catalog = builder.Reserve(); + var pages = builder.Reserve(); + + builder.Add("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] / (empty) >>"); + builder.Set(catalog, "<< /Type /Catalog /Pages 2 0 R / 1 >>"); + builder.Set(pages, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + + var reader = new PdfFileReader(builder.Build(catalog), "empty-name.pdf", SharpCorePDF.Infrastructure.PdfEngineOptions.Default); + + Assert.Single(reader.Pages); + } + /// The null object is one instance and writes itself as its keyword. [Fact] public void Exposes_the_null_object() diff --git a/tests/SharpCorePDF.Tests/PdfSerializationTests.cs b/tests/SharpCorePDF.Tests/PdfSerializationTests.cs index a3bf372..3ae3690 100644 --- a/tests/SharpCorePDF.Tests/PdfSerializationTests.cs +++ b/tests/SharpCorePDF.Tests/PdfSerializationTests.cs @@ -78,6 +78,28 @@ public void Writes_a_date_that_round_trips() Assert.Equal("D:20260401120000Z", PdfDate.Format(new DateTimeOffset(2026, 4, 1, 12, 0, 0, TimeSpan.Zero))); } + /// + /// An offset no time zone has — past fourteen hours either way, which refuses + /// to hold — is read as no offset rather than thrown, so one malformed metadata date does not fail the + /// inspection of the document it is in. The offsets at the edge of the range are kept. + /// + /// The date as a document states it. + /// The offset that is read. + [Theory] + [InlineData("D:20260101000000+99'00'", 0)] + [InlineData("D:20260101000000+23'59'", 0)] + [InlineData("D:20260101000000+14'30'", 0)] + [InlineData("D:20260101000000-15'00'", 0)] + [InlineData("D:20260101000000+14'00'", 14 * 60)] + [InlineData("D:20260101000000-14'00'", -14 * 60)] + [InlineData("D:20260101000000+05'30'", 5 * 60 + 30)] + public void Reads_an_offset_no_time_zone_has_as_no_offset(string text, int offsetMinutes) + { + Assert.True(PdfDate.TryParse(text, out var value)); + Assert.Equal(TimeSpan.FromMinutes(offsetMinutes), value.Offset); + Assert.Equal(new DateTime(2026, 1, 1, 0, 0, 0), value.DateTime); + } + /// Text that is not a date at all is rejected instead of being invented. [Theory] [InlineData("")] diff --git a/tests/SharpCorePDF.Tests/PdfTextScannerTests.cs b/tests/SharpCorePDF.Tests/PdfTextScannerTests.cs index 1811665..802c05a 100644 --- a/tests/SharpCorePDF.Tests/PdfTextScannerTests.cs +++ b/tests/SharpCorePDF.Tests/PdfTextScannerTests.cs @@ -15,6 +15,31 @@ namespace SharpCorePDF.Tests; /// public sealed class PdfTextScannerTests { + /// + /// A glyph drawn upside down — a negative font size, which a flipped text or font matrix produces — has a + /// box all the same: its descender is above its ascender, and the box is taken between them. + /// + [Fact] + public void Gives_a_glyph_drawn_upside_down_a_box() + { + var glyph = new SharpCorePDF.Domain.ReadModels.PdfGlyph + { + Text = "A", + PageNumber = 1, + X = 10, + Y = 100, + Width = 6, + FontSize = -10, + }; + + var box = glyph.BoundingBox; + + Assert.Equal(92d, box.Bottom, 6); + Assert.Equal(102d, box.Top, 6); + Assert.Equal(10d, box.Left, 6); + Assert.Equal(16d, box.Right, 6); + } + /// A string the scanner can take as a slice of the stream. /// The content stream. /// The text the shown string must decode to.