From 06ad09252baa7361d47f71797cdff21ce1ae0a34 Mon Sep 17 00:00:00 2001 From: Andrew Tierney Date: Mon, 16 Nov 2026 11:03:01 +1100 Subject: [PATCH 01/11] Validate the output with independent readers and the input with fuzzing and corpora A writer that only reads its own output back cannot see the mistakes it shares with itself. validation.yml now puts the example output and a validation corpus (written by tools/SharpCorePDF.Validation, with a manifest of the text, passwords, permissions, revisions and signature coverage each document must have) through qpdf, pypdf, pdfminer.six, MuPDF, pdf.js, fontTools, OTS, PDFium and pyHanko, checks the incremental-update invariants, and replays the fuzz seeds and four hostile documents. robustness.yml runs libFuzzer over the parser and the DCT, CCITT, LZW, Flate and RunLength decoders, reads the veraPDF and pdf.js corpora one process per file, and judges the PDF/A writer with veraPDF over the veraPDF corpus. compliance.yml cross-validates PDF/A-4. The checks found defects that are not fixed here. Each is pinned by a skipped test (PdfSignatureAttackTests, PdfIncrementalInvariantTests) or a strict --known-defect flag that fails the run once the defect is fixed, and docs/validation.md lists them with their evidence. No shipped package changes. The constitution's statement that no Python validator runs in CI is corrected as a proposed PATCH amendment (2.0.1) for the maintainer to ratify. Local AI assistant state (.claude/, CLAUDE.local.md) is ignored. --- .config/dotnet-tools.json | 7 + .github/workflows/compliance.yml | 6 +- .github/workflows/robustness.yml | 282 +++++++++++++++ .github/workflows/validation.yml | 232 ++++++++++++ .gitignore | 4 + .specify/memory/constitution.md | 18 +- CHANGELOG.md | 28 ++ CONTRIBUTING.md | 6 +- README.md | 7 + SharpCorePDF.slnx | 4 +- docs/README.md | 2 + docs/tools/corpus-robustness.py | 207 +++++++++++ docs/tools/render-compare.py | 158 ++++++++ docs/tools/validate-arlington.py | 149 ++++++++ docs/tools/validate-encryption-pdfjs.mjs | 95 +++++ docs/tools/validate-encryption.py | 164 +++++++++ docs/tools/validate-fonts.py | 263 ++++++++++++++ docs/tools/validate-incremental.py | 145 ++++++++ docs/tools/validate-pdf.py | 30 +- docs/tools/validate-pdfa-cross.py | 11 +- docs/tools/validate-preflight-corpus.py | 148 ++++++++ docs/tools/validate-signatures.py | 77 ++++ docs/tools/validate-text.py | 147 ++++++++ docs/tools/validation-requirements.txt | 20 ++ docs/validation.md | 139 +++++++ .../PdfIncrementalInvariantTests.cs | 142 ++++++++ .../PdfSignatureAttackTests.cs | 218 +++++++++++ tools/SharpCorePDF.Validation/CorpusWriter.cs | 338 ++++++++++++++++++ tools/SharpCorePDF.Validation/FuzzTargets.cs | 254 +++++++++++++ tools/SharpCorePDF.Validation/Probe.cs | 251 +++++++++++++ tools/SharpCorePDF.Validation/Program.cs | 138 +++++++ .../SharpCorePDF.Validation.csproj | 52 +++ 32 files changed, 3730 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/robustness.yml create mode 100644 .github/workflows/validation.yml create mode 100644 docs/tools/corpus-robustness.py create mode 100644 docs/tools/render-compare.py create mode 100644 docs/tools/validate-arlington.py create mode 100644 docs/tools/validate-encryption-pdfjs.mjs create mode 100644 docs/tools/validate-encryption.py create mode 100644 docs/tools/validate-fonts.py create mode 100644 docs/tools/validate-incremental.py create mode 100644 docs/tools/validate-preflight-corpus.py create mode 100644 docs/tools/validate-text.py create mode 100644 docs/tools/validation-requirements.txt create mode 100644 docs/validation.md create mode 100644 tests/SharpCorePDF.Tests/PdfIncrementalInvariantTests.cs create mode 100644 tests/SharpCorePDF.Tests/PdfSignatureAttackTests.cs create mode 100644 tools/SharpCorePDF.Validation/CorpusWriter.cs create mode 100644 tools/SharpCorePDF.Validation/FuzzTargets.cs create mode 100644 tools/SharpCorePDF.Validation/Probe.cs create mode 100644 tools/SharpCorePDF.Validation/Program.cs create mode 100644 tools/SharpCorePDF.Validation/SharpCorePDF.Validation.csproj diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index 5ccda36..eee80fa 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -8,6 +8,13 @@ "dotnet-stryker" ], "rollForward": false + }, + "sharpfuzz.commandline": { + "version": "2.3.0", + "commands": [ + "sharpfuzz" + ], + "rollForward": false } } } \ No newline at end of file diff --git a/.github/workflows/compliance.yml b/.github/workflows/compliance.yml index 2b323f5..2f8af7e 100644 --- a/.github/workflows/compliance.yml +++ b/.github/workflows/compliance.yml @@ -214,7 +214,9 @@ jobs: # The same documents, read by two more implementations. --strict-tools makes a missing tool a failure # rather than a note, because a green run here is supposed to mean that all three agreed. PDFBox 3.0.8 - # and Ghostscript have no PDF/A-4 mode, so veraPDF is the only reading of that profile. + # and Ghostscript have no PDF/A-4 mode, so veraPDF stays the only verdict on that profile; PDFBox still + # parses the PDF 2.0 file and Ghostscript still rewrites it (as part 2), which is the reader test the + # other profiles get too. - name: Cross-validate with a second and a third implementation env: VERAPDF_HOME: ${{ env.VERAPDF_INSTALL }} @@ -223,7 +225,7 @@ jobs: output="$RUNNER_TEMP/SharpCorePDF.Examples/output" jar="$HOME/.cache/pdfbox/preflight-app-${PDFBOX_PREFLIGHT_VERSION}.jar" - for profile in 1b 2b 2u 3b 3u; do + for profile in 1b 2b 2u 3b 3u 4; do python3 docs/tools/validate-pdfa-cross.py --strict-tools \ --require-version "$VERAPDF_VERSION" \ --pdfbox "$jar" \ diff --git a/.github/workflows/robustness.yml b/.github/workflows/robustness.yml new file mode 100644 index 0000000..93de31b --- /dev/null +++ b/.github/workflows/robustness.yml @@ -0,0 +1,282 @@ +# Robustness: what the engine does with documents nobody here wrote, and with documents built to break it. +# +# validation.yml checks what the library writes, on every push. This workflow checks what it reads, weekly, +# because its inputs are large and slow: coverage-guided fuzzing of every decoder and of the document parser, +# two public corpora of real and deliberately broken files, and the PDF/A conversion measured against the +# veraPDF test corpus with veraPDF as the judge. None of it gates a pull request; a red run is a defect report. +# +# fuzz SharpFuzz instruments the three engine assemblies and libFuzzer drives one target per job +# (tools/SharpCorePDF.Validation, FuzzTargets.cs). A PdfException is the contract's refusal and +# not a finding; any other exception, a stack overflow or the memory limit is, and the input +# that caused it is uploaded. Reproduce one with: +# dotnet run --project tools/SharpCorePDF.Validation -- replay +# corpora corpus-robustness.py over the veraPDF corpus and the pdf.js test corpus, one process per +# file, under a time limit and a hard managed-heap limit. Fault, crash and timeout fail the run. +# preflight validate-preflight-corpus.py: every PDF/A file of the veraPDF corpus converted for its profile +# and the result judged by veraPDF. Report-only until its first run sets the baseline. +# +# Every external input is pinned: the libFuzzer driver by tag and SHA-256, SharpFuzz in +# .config/dotnet-tools.json, the corpora by commit, veraPDF by version (the same one compliance.yml pins). + +name: Robustness + +on: + schedule: + - cron: '0 3 * * 2' + workflow_dispatch: + inputs: + fuzz-minutes: + description: 'Minutes of fuzzing per target' + required: false + default: '30' + +permissions: + contents: read + +env: + LIBFUZZER_DOTNET_TAG: 'v2025.05.02.0904' + LIBFUZZER_DOTNET_SHA256: 'c2c2a90d94c409a4af339a0d4f244e0442c5a5d249be0f1252ba07871f285958' + VERAPDF_CORPUS_COMMIT: 'bb75f4f0073d9350dfd058c0162a367e6fadf25e' + PDFJS_TAG: 'v6.4.299' + PDFJS_COMMIT: 'd0991a0d53f3bb3a7307e3b83f279c1d117aafb3' + VERAPDF_VERSION: '1.30.4' + VERAPDF_INSTALL: ${{ github.workspace }}/.verapdf + +jobs: + fuzz: + name: Fuzz ${{ matrix.target }} + runs-on: ubuntu-latest + timeout-minutes: 60 + + strategy: + fail-fast: false + matrix: + target: [ document, dct, ccitt, lzw, flate, runlength ] + + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: Install the .NET SDKs + uses: actions/setup-dotnet@0f4a354dd5547a28c70b7853dbd95d5b67886e73 # v6 + with: + dotnet-version: | + 11.0.x + 10.0.x + + - name: Install SharpFuzz and the libFuzzer driver + run: | + set -euo pipefail + dotnet tool restore + curl --fail --silent --show-error --location --proto '=https' --tlsv1.2 \ + --output "$RUNNER_TEMP/libfuzzer-dotnet" \ + "https://github.com/Metalnem/libfuzzer-dotnet/releases/download/${LIBFUZZER_DOTNET_TAG}/libfuzzer-dotnet-ubuntu" + echo "${LIBFUZZER_DOTNET_SHA256} $RUNNER_TEMP/libfuzzer-dotnet" | sha256sum --check + chmod +x "$RUNNER_TEMP/libfuzzer-dotnet" + + # Two builds: an instrumented one for libFuzzer, and a plain one for the seeds and for replaying what + # libFuzzer finds, because instrumented code throws outside the fuzzer (see Program.cs). + - name: Build, then instrument the engine + run: | + set -euo pipefail + dotnet publish ./tools/SharpCorePDF.Validation -c Release -o "$RUNNER_TEMP/plain" + dotnet publish ./tools/SharpCorePDF.Validation -c Release -o "$RUNNER_TEMP/fuzz" + for layer in Domain Application Infrastructure; do + dotnet sharpfuzz "$RUNNER_TEMP/fuzz/SharpCorePDF.$layer.dll" + done + + - name: Seed the corpus + run: | + set -euo pipefail + tool="$RUNNER_TEMP/plain/SharpCorePDF.Validation.dll" + seeds="$RUNNER_TEMP/seeds" + assets=tests/SharpCorePDF.Tests/Assets + case "${{ matrix.target }}" in + document) dotnet "$tool" seed document "$seeds" "$assets/parity/corpus" "$assets/parity/labelled" "$assets/parity/forms" ;; + dct) dotnet "$tool" seed dct "$seeds" "$assets"/jpeg/*.jpg ;; + ccitt) dotnet "$tool" seed ccitt "$seeds" "$assets/ccitt/ccitt-g3.bin" "$assets/ccitt/ccitt-g4.bin" ;; + *) mkdir -p "$seeds" && printf '\x03\x00\x00\x3f\x3f' > "$seeds/empty" ;; + esac + dotnet "$tool" replay "${{ matrix.target }}" "$seeds" + + - name: Fuzz + env: + SCPDF_FUZZ_TARGET: ${{ matrix.target }} + MINUTES: ${{ github.event.inputs.fuzz-minutes || '30' }} + run: | + set -uo pipefail + mkdir -p artifacts/crashes + "$RUNNER_TEMP/libfuzzer-dotnet" \ + --target_path=dotnet --target_arg="$RUNNER_TEMP/fuzz/SharpCorePDF.Validation.dll" \ + -max_total_time=$(( MINUTES * 60 )) -timeout=30 -rss_limit_mb=4096 -max_len=262144 \ + -artifact_prefix=artifacts/crashes/ -print_final_stats=1 \ + "$RUNNER_TEMP/seeds" + status=$? + ls -la artifacts/crashes + if [ -n "$(ls -A artifacts/crashes)" ]; then + echo "::error::libFuzzer saved $(ls artifacts/crashes | wc -l) input(s) that broke the ${{ matrix.target }} target" + exit 1 + fi + exit $status + + - name: Upload what broke the engine + if: failure() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: fuzz-crashes-${{ matrix.target }} + path: artifacts/crashes + if-no-files-found: ignore + retention-days: 30 + + corpora: + name: Public corpora + runs-on: ubuntu-latest + timeout-minutes: 120 + + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: Install the .NET SDKs + uses: actions/setup-dotnet@0f4a354dd5547a28c70b7853dbd95d5b67886e73 # v6 + with: + dotnet-version: | + 11.0.x + 10.0.x + + - name: Fetch the corpora + run: | + set -euo pipefail + # The veraPDF corpus is small enough to clone without blobs and check out at the pinned commit; pdf.js + # is not, so it is cloned one commit deep at its tag and the commit is checked against the pin. + git clone -q --filter=blob:none --sparse https://github.com/veraPDF/veraPDF-corpus.git "$RUNNER_TEMP/verapdf-corpus" + git -C "$RUNNER_TEMP/verapdf-corpus" sparse-checkout set PDF_A-1b PDF_A-2b PDF_A-4 "ISO 32000-1" "ISO 32000-2" "Isartor test files" + git -C "$RUNNER_TEMP/verapdf-corpus" -c advice.detachedHead=false checkout -q "$VERAPDF_CORPUS_COMMIT" + + git clone -q --depth 1 --branch "$PDFJS_TAG" --filter=blob:none --sparse https://github.com/mozilla/pdf.js.git "$RUNNER_TEMP/pdfjs" + git -C "$RUNNER_TEMP/pdfjs" sparse-checkout set test/pdfs + test "$(git -C "$RUNNER_TEMP/pdfjs" rev-parse HEAD)" = "$PDFJS_COMMIT" + + find "$RUNNER_TEMP/verapdf-corpus" "$RUNNER_TEMP/pdfjs" -name '*.pdf' | wc -l + + - name: Build the probe + run: dotnet build ./tools/SharpCorePDF.Validation -c Release + + # The six known files fault on three recorded defects (docs/validation.md): an empty name as a + # dictionary key (issue3188, issue5896, pdfjs_wikipedia, poppler-742-0-fuzzed), a glyph whose box is + # upside down (issue14415) and a date whose offset is beyond 14 hours (TAMReview). + - name: Read every document + run: | + python3 docs/tools/corpus-robustness.py \ + --probe ./tools/SharpCorePDF.Validation/bin/Release/net11.0/SharpCorePDF.Validation.dll \ + --timeout 60 --heap-limit-mb 1024 --report artifacts/corpora.json \ + --known issue3188.pdf --known issue5896.pdf --known pdfjs_wikipedia.pdf \ + --known poppler-742-0-fuzzed.pdf --known issue14415.pdf --known TAMReview.pdf \ + "$RUNNER_TEMP/verapdf-corpus" "$RUNNER_TEMP/pdfjs/test/pdfs" + + - name: Upload the report + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: corpora + path: artifacts/corpora.json + if-no-files-found: warn + retention-days: 30 + + preflight: + name: PDF/A conversion against the veraPDF corpus + runs-on: ubuntu-latest + timeout-minutes: 180 + + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: Install the .NET SDKs + uses: actions/setup-dotnet@0f4a354dd5547a28c70b7853dbd95d5b67886e73 # v6 + with: + dotnet-version: | + 11.0.x + 10.0.x + + - name: Set up the JDK veraPDF runs on + uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6.1.0 + with: + distribution: 'zulu' + java-version: '17' + + - name: Cache the veraPDF installer + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/.cache/verapdf + key: verapdf-${{ env.VERAPDF_VERSION }}-installer + + # The same installation compliance.yml performs, so both workflows judge with the same validator. + - name: Install veraPDF + run: | + set -euo pipefail + band="${VERAPDF_VERSION%.*}" + archive="$HOME/.cache/verapdf/verapdf-greenfield-${VERAPDF_VERSION}-installer.zip" + mkdir -p "$(dirname "$archive")" + if [ ! -f "$archive" ]; then + curl --fail --silent --show-error --location \ + --proto '=https' --proto-redir '=https' --tlsv1.2 \ + --output "$archive" \ + "https://software.verapdf.org/releases/${band}/verapdf-greenfield-${VERAPDF_VERSION}-installer.zip" + fi + rm -rf "$VERAPDF_INSTALL" "$RUNNER_TEMP/verapdf" + mkdir -p "$VERAPDF_INSTALL" + unzip -q "$archive" -d "$RUNNER_TEMP/verapdf" + printf '%s\n' \ + '' \ + '' \ + ' ' \ + ' ' \ + " ${VERAPDF_INSTALL}" \ + ' ' \ + ' ' \ + ' ' \ + ' ' \ + ' ' \ + ' ' \ + ' ' \ + ' ' \ + ' ' \ + '' \ + > "$RUNNER_TEMP/auto-install.xml" + rm -f "/tmp/iz-veraPDF Software.tmp" "${TMPDIR:-/tmp}/iz-veraPDF Software.tmp" + bash "$RUNNER_TEMP/verapdf/verapdf-greenfield-${VERAPDF_VERSION}/verapdf-install" "$RUNNER_TEMP/auto-install.xml" + if [ ! -x "$VERAPDF_INSTALL/verapdf" ]; then + printf '%s\n' '#!/usr/bin/env bash' \ + "exec java -jar \"$VERAPDF_INSTALL\"/bin/cli-*.jar \"\$@\"" \ + > "$VERAPDF_INSTALL/verapdf" + chmod +x "$VERAPDF_INSTALL/verapdf" + fi + "$VERAPDF_INSTALL/verapdf" --version + + - name: Fetch the veraPDF corpus + run: | + set -euo pipefail + git clone -q --filter=blob:none --sparse https://github.com/veraPDF/veraPDF-corpus.git "$RUNNER_TEMP/verapdf-corpus" + git -C "$RUNNER_TEMP/verapdf-corpus" sparse-checkout set PDF_A-1b PDF_A-2b PDF_A-2u PDF_A-3b PDF_A-4 + git -C "$RUNNER_TEMP/verapdf-corpus" -c advice.detachedHead=false checkout -q "$VERAPDF_CORPUS_COMMIT" + + - name: Build the probe + run: dotnet build ./tools/SharpCorePDF.Validation -c Release + + - name: Convert and judge + run: | + python3 docs/tools/validate-preflight-corpus.py \ + --corpus "$RUNNER_TEMP/verapdf-corpus" \ + --probe ./tools/SharpCorePDF.Validation/bin/Release/net11.0/SharpCorePDF.Validation.dll \ + --verapdf "$VERAPDF_INSTALL/verapdf" \ + --report artifacts/preflight.json + + - name: Upload the report + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: preflight + path: artifacts/preflight.json + if-no-files-found: warn + retention-days: 30 diff --git a/.github/workflows/validation.yml b/.github/workflows/validation.yml new file mode 100644 index 0000000..6bb7cb0 --- /dev/null +++ b/.github/workflows/validation.yml @@ -0,0 +1,232 @@ +# Independent validation: what readers that share no code with this library make of what it writes. +# +# ci.yml answers whether the code builds and the suite passes; compliance.yml whether the archival output +# conforms. This workflow answers a third question -- whether other implementations read the documents the +# way the library meant them -- because a writer that only ever reads its own output back cannot see the +# mistakes it shares with itself. Every check here runs against the example output and against the validation +# corpus that tools/SharpCorePDF.Validation writes, whose manifest states what each document has to say. +# +# qpdf validate-pdf.py: the file parses, every page reads, every stream decodes +# text validate-text.py: pypdf, pdfminer.six and MuPDF extract the lines that were drawn +# encryption validate-encryption.py and the pdf.js leg: four readers open the RC4/AES matrix with the right +# password, refuse the wrong one, and read the revision and the permissions that were asked for +# fonts validate-fonts.py: fontTools and OTS read every embedded program, and the widths agree +# rendering render-compare.py: PDFium and MuPDF paint the same pages +# incremental validate-incremental.py: every revision stands alone, /ID[0] holds, an update only appends +# signatures validate-signatures.py --manifest: pyHanko agrees on integrity, trust and coverage +# fuzz seeds every fuzz target runs over its seeds in process, so a harness that rots is noticed here and +# not in the weekly fuzzing run (robustness.yml) +# hostile corpus-robustness.py over the documents it synthesizes for the failure classes the review named +# arlington validate-arlington.py against the veraPDF Arlington service (report-only until a baseline exists) +# +# --known-defect / --known names a failure that is recorded and not yet fixed. Each is strict: when the +# defect is fixed the check passes, the flag no longer matches, and the run fails until the flag is removed +# in the same change as the fix. The in-suite counterparts are the skipped tests in +# PdfSignatureAttackTests and PdfIncrementalInvariantTests. +# +# The validator versions are pinned in docs/tools/validation-requirements.txt, pdfjs-dist below. + +name: Independent validation + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + workflow_dispatch: + +permissions: + contents: read + +env: + PDFJS_VERSION: '6.4.299' + +jobs: + readers: + name: Read by other implementations + runs-on: ubuntu-latest + timeout-minutes: 40 + + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: Install the .NET SDKs + uses: actions/setup-dotnet@0f4a354dd5547a28c70b7853dbd95d5b67886e73 # v6 + with: + dotnet-version: | + 11.0.x + 10.0.x + + - name: Install the validators + run: | + set -euo pipefail + python3 -m venv "$RUNNER_TEMP/venv" + "$RUNNER_TEMP/venv/bin/pip" install --quiet -r docs/tools/validation-requirements.txt + echo "$RUNNER_TEMP/venv/bin" >> "$GITHUB_PATH" + npm install --silent --no-save --prefix "$RUNNER_TEMP/pdfjs" "pdfjs-dist@${PDFJS_VERSION}" + + - name: Build the examples and the validation harness + run: | + dotnet build ./examples/SharpCorePDF.Examples -c Release + dotnet build ./tools/SharpCorePDF.Validation -c Release + + - name: Write the documents + env: + TMPDIR: ${{ runner.temp }} + run: | + set -euo pipefail + dotnet run --project ./examples/SharpCorePDF.Examples -c Release --no-build + dotnet run --project ./tools/SharpCorePDF.Validation -c Release --no-build -- corpus "$RUNNER_TEMP/corpus" + + # protected.pdf is encrypted with a password the example makes for the run and prints; the encrypted + # documents qpdf does check are the corpus's, whose passwords the manifest carries + # (validate-encryption.py runs qpdf --check on each of them with its password). + - name: qpdf + run: | + set -euo pipefail + shopt -s nullglob + files=() + for file in "$RUNNER_TEMP"/SharpCorePDF.Examples/output/*.pdf "$RUNNER_TEMP"/corpus/*.pdf; do + case "$(basename "$file")" in protected.pdf|encrypted-*) continue ;; esac + files+=("$file") + done + # Both merge a page of NamedDests.pdf, which states no /MediaBox; the merge copies it without one, + # so the result breaks the rule that every page has a media box, and qpdf assumes Letter. + python docs/tools/validate-pdf.py --known-defect inspected.pdf --known-defect with-warnings.pdf "${files[@]}" + + - name: Text round trip + run: python docs/tools/validate-text.py "$RUNNER_TEMP/corpus" + + - name: Encryption matrix + run: | + set -euo pipefail + python docs/tools/validate-encryption.py "$RUNNER_TEMP/corpus" + node docs/tools/validate-encryption-pdfjs.mjs "$RUNNER_TEMP/corpus" \ + "$RUNNER_TEMP/pdfjs/node_modules/pdfjs-dist/legacy/build/pdf.mjs" + + - name: Embedded fonts + run: | + python docs/tools/validate-fonts.py \ + "$RUNNER_TEMP/SharpCorePDF.Examples/output" \ + "$RUNNER_TEMP/corpus/text-embedded.pdf" "$RUNNER_TEMP/corpus/text-merged.pdf" + + - name: Rendering, PDFium against MuPDF + run: | + python docs/tools/render-compare.py --report-dir artifacts/rendering --exclude protected.pdf \ + "$RUNNER_TEMP/SharpCorePDF.Examples/output" \ + "$RUNNER_TEMP/corpus/text-standard.pdf" "$RUNNER_TEMP/corpus/text-embedded.pdf" \ + "$RUNNER_TEMP/corpus/text-merged.pdf" "$RUNNER_TEMP/corpus/incremental.pdf" \ + "$RUNNER_TEMP/corpus/signed.pdf" + + # id-first: the editor writes a new identifier into both elements of /ID on an appended revision. + # appends-only: the editor's incremental update appends a renumbered copy of the whole document + # rather than the objects it changed. + - name: Incremental update invariants + run: | + python docs/tools/validate-incremental.py "$RUNNER_TEMP/corpus" \ + --known-defect id-first --known-defect appends-only + + # The same copy rewrites the signature dictionary after signing, which is why pyHanko cannot place the + # first signature of signed-then-edited.pdf at the end of a revision. + - name: Signatures, pyHanko + run: | + python docs/tools/validate-signatures.py --manifest "$RUNNER_TEMP/corpus/manifest.json" \ + --known-defect signed-then-edited.pdf:FirstSignature + + - name: Fuzz targets over their seeds + run: | + set -euo pipefail + tool=./tools/SharpCorePDF.Validation/bin/Release/net11.0/SharpCorePDF.Validation.dll + seeds="$RUNNER_TEMP/seeds" + dotnet "$tool" seed document "$seeds/document" tests/SharpCorePDF.Tests/Assets/parity/corpus "$RUNNER_TEMP/corpus" + dotnet "$tool" seed dct "$seeds/dct" tests/SharpCorePDF.Tests/Assets/jpeg/*.jpg + dotnet "$tool" seed ccitt "$seeds/ccitt" tests/SharpCorePDF.Tests/Assets/ccitt/ccitt-g3.bin tests/SharpCorePDF.Tests/Assets/ccitt/ccitt-g4.bin + for target in document dct ccitt; do + dotnet "$tool" replay "$target" "$seeds/$target" + done + + # Each of the four documents reaches a failure class the review named; each is a known defect until the + # change that fixes it removes its --known line. + - name: Hostile documents + run: | + set -euo pipefail + python docs/tools/corpus-robustness.py --synthesize "$RUNNER_TEMP/hostile" + python docs/tools/corpus-robustness.py \ + --probe ./tools/SharpCorePDF.Validation/bin/Release/net11.0/SharpCorePDF.Validation.dll \ + --timeout 120 --report artifacts/hostile.json \ + --known flate-bomb.pdf --known length-chain.pdf --known objstm-count.pdf --known jpeg-selector.pdf \ + "$RUNNER_TEMP/hostile" + + - name: Upload the documents and the reports + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: independent-validation + path: | + artifacts + ${{ runner.temp }}/corpus + ${{ runner.temp }}/SharpCorePDF.Examples/output/*.pdf + if-no-files-found: warn + retention-days: 14 + + arlington: + name: Arlington PDF Model + runs-on: ubuntu-latest + timeout-minutes: 30 + # The service's report format and the writer's baseline are confirmed by the first run on a runner; until + # then the job reports without blocking. Remove continue-on-error and pass --strict with the --accept list + # that run establishes. + continue-on-error: true + + services: + arlington: + image: verapdf/arlington:v1.30.4 + ports: + - 8080:8080 + + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: Install the .NET SDKs + uses: actions/setup-dotnet@0f4a354dd5547a28c70b7853dbd95d5b67886e73 # v6 + with: + dotnet-version: | + 11.0.x + 10.0.x + + - name: Write the documents + env: + TMPDIR: ${{ runner.temp }} + run: | + set -euo pipefail + dotnet run --project ./examples/SharpCorePDF.Examples -c Release + dotnet run --project ./tools/SharpCorePDF.Validation -c Release -- corpus "$RUNNER_TEMP/corpus" + + - name: Wait for the service + run: | + for attempt in $(seq 1 60); do + curl -sf http://localhost:8080/api/info > /dev/null && exit 0 + sleep 2 + done + echo "::error::the Arlington service did not answer on port 8080" + exit 1 + + - name: Check against the model + run: | + python3 docs/tools/validate-arlington.py --report-dir artifacts/arlington \ + --exclude protected.pdf \ + "$RUNNER_TEMP/SharpCorePDF.Examples/output" \ + "$RUNNER_TEMP/corpus/text-standard.pdf" "$RUNNER_TEMP/corpus/text-embedded.pdf" \ + "$RUNNER_TEMP/corpus/text-merged.pdf" "$RUNNER_TEMP/corpus/incremental.pdf" \ + "$RUNNER_TEMP/corpus/signed.pdf" + + - name: Upload the reports + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: arlington + path: artifacts/arlington + if-no-files-found: warn + retention-days: 14 diff --git a/.gitignore b/.gitignore index 4ca788b..a7b1b5e 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,10 @@ StrykerOutput/ *.sln.docstates *.DotSettings.user +# AI assistant state: prompts, transcripts and local settings stay on the machine that wrote them +.claude/ +CLAUDE.local.md + # Build logs *.binlog msbuild.log diff --git a/.specify/memory/constitution.md b/.specify/memory/constitution.md index af536c8..e45abe8 100644 --- a/.specify/memory/constitution.md +++ b/.specify/memory/constitution.md @@ -1,6 +1,13 @@ + + Exe + net11.0 + preview + SharpCorePDF.Validation + SharpCorePDF.Validation + enable + enable + nullable + true + latest-recommended + false + false + true + + + + + + + + + + + + + + + + + + + + From 8ad4c9e54663e4a4d467365154e25fbe4d360725 Mon Sep 17 00:00:00 2001 From: Andrew Tierney Date: Mon, 16 Nov 2026 11:55:24 +1100 Subject: [PATCH 02/11] Release pdf.js documents through their loading task pdf.js 6 has no PDFDocumentProxy.destroy(), so the encryption check's pdf.js leg reported every document as unopenable: the call threw inside the try that was meant to catch a wrong password. The loading task owns the document and destroying it releases both. The fix was made before the previous commit but an editor saved a stale copy of the file over it. --- docs/tools/validate-encryption-pdfjs.mjs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/tools/validate-encryption-pdfjs.mjs b/docs/tools/validate-encryption-pdfjs.mjs index 8ee5cd1..57a929c 100644 --- a/docs/tools/validate-encryption-pdfjs.mjs +++ b/docs/tools/validate-encryption-pdfjs.mjs @@ -26,8 +26,10 @@ async function open(file, password) { const data = new Uint8Array(await readFile(path.join(corpus, file))); const task = pdfjs.getDocument({data, password, isEvalSupported: false, verbosity: 0}); + // The loading task owns the document in pdf.js 6, which has no PDFDocumentProxy.destroy(): destroying + // the task is what releases both. try { - return await task.promise; + return {task, document: await task.promise}; } catch (error) { await task.destroy(); throw error; @@ -36,8 +38,8 @@ async function open(file, password) { async function refuses(file, password) { try { - const document = await open(file, password); - await document.destroy(); + const {task} = await open(file, password); + await task.destroy(); return false; } catch (error) { return error?.name === "PasswordException"; @@ -51,7 +53,7 @@ for (const entry of manifest.encryption) { const problems = []; try { - const document = await open(entry.file, entry.userPassword); + const {task, document} = await open(entry.file, entry.userPassword); const page = await document.getPage(1); const content = await page.getTextContent(); const text = normalise(content.items.map((item) => item.str + (item.hasEOL ? "\n" : "")).join("")); @@ -61,14 +63,14 @@ for (const entry of manifest.encryption) { problems.push(`page 1 text does not contain ${JSON.stringify(first)}`); } - await document.destroy(); + await task.destroy(); } catch (error) { problems.push(`the user password did not open it: ${error?.name}: ${error?.message}`); } try { - const document = await open(entry.file, entry.ownerPassword); - await document.destroy(); + const {task} = await open(entry.file, entry.ownerPassword); + await task.destroy(); } catch (error) { problems.push(`the owner password did not open it: ${error?.name}: ${error?.message}`); } From 7020f3bed853cb3fc0b1c4c12b4aef8242d74c73 Mon Sep 17 00:00:00 2001 From: Andrew Tierney Date: Mon, 16 Nov 2026 12:10:46 +1100 Subject: [PATCH 03/11] Refuse signature wrapping and bound what hostile input can allocate Signature wrapping: the validator required only that the byte range start at 0 and end at the end of the file, so bytes written into the unsigned gap could close the signature value early and add entries while the digest still matched, and the signature was reported valid and covering the whole document. The gap must now be exactly one hexadecimal string equal to the dictionary's /Contents; anything else is MalformedByteRange, integrity Unknown and CoversWholeDocument false. Decompression bombs: Flate, LZW and RunLength decoded without a limit, so a 1.1 MB document took the process past 1 GiB. PdfEngineOptions.MaxDecodedStreamBytes (128 MiB) now stops them; decode buffers above 16 MiB no longer stay in the shared array pool, and buffer growth no longer overflows. Stack overflow: a chain of streams whose /Length is the next stream was resolved by recursion, and 60,000 links ended the process. The chain is followed no deeper than MaxNestingDepth, then the stream is found by its endstream keyword, as for a /Length cycle. Allocations sized by the file: an object stream's /N is bounded by what the stream can hold, and a JPEG whose output is above the caller's budget is measured from its frame header instead of being decoded first; the working sizes are computed in 64 bits. Tests that fail without the fixes: PdfSignatureAttackTests (gap wrapping, gap shape), PdfFilterTests (limits per filter, content stream past the limit), PdfReaderRecoveryTests (length chain, indirect lengths, object stream /N), PdfJpegFilterTests (oversized frames, working buffers). The three hostile documents of validation.yml are read under a 1 GiB heap, and their --known flags are removed. --- .github/workflows/validation.yml | 7 +- CHANGELOG.md | 28 +++- docs/manual/src/10-dependency-injection.md | 14 +- docs/validation.md | 15 +- .../Pdf/JpegDecoder.cs | 78 ++++++++++ .../Pdf/PdfCrossReferenceReader.Streams.cs | 2 +- .../Pdf/PdfFileReader.ObjectStreams.cs | 22 +-- .../Pdf/PdfFileReader.Objects.cs | 14 +- .../Pdf/PdfFileReader.cs | 1 + .../Pdf/PdfFilters.Inflation.cs | 102 +++++++++++--- .../Pdf/PdfFilters.Lzw.cs | 21 ++- .../Pdf/PdfFilters.RunLength.cs | 24 +++- .../Pdf/PdfFilters.cs | 16 ++- .../Pdf/PdfFontDecoder.cs | 2 +- .../Pdf/PdfImageFilters.cs | 4 +- .../Pdf/PdfTextCollector.cs | 7 +- .../Pdf/PdfType1Encoding.cs | 2 +- .../Pdf/PooledByteBuffer.cs | 51 ++++++- .../PdfEditEngine.Overlays.cs | 2 +- .../PdfEngineOptions.cs | 23 +++ .../PdfPageContentReader.Content.cs | 7 +- .../PdfPageContentReader.Images.cs | 2 +- .../PdfPageContentReader.Metadata.cs | 2 +- .../PublicAPI.Unshipped.txt | 2 + .../Signatures/SignatureValidator.cs | 75 +++++++++- .../PdfEngineOptionsTests.cs | 10 ++ tests/SharpCorePDF.Tests/PdfFilterTests.cs | 96 +++++++++++++ .../SharpCorePDF.Tests/PdfJpegFilterTests.cs | 90 ++++++++++++ .../PdfReaderRecoveryTests.cs | 133 ++++++++++++++++++ .../PdfSignatureAttackTests.cs | 75 +++++++++- 30 files changed, 840 insertions(+), 87 deletions(-) diff --git a/.github/workflows/validation.yml b/.github/workflows/validation.yml index 6bb7cb0..de42cbe 100644 --- a/.github/workflows/validation.yml +++ b/.github/workflows/validation.yml @@ -146,8 +146,9 @@ jobs: dotnet "$tool" replay "$target" "$seeds/$target" done - # Each of the four documents reaches a failure class the review named; each is a known defect until the - # change that fixes it removes its --known line. + # Each of the four documents reaches a failure class the review named. flate-bomb, length-chain and + # objstm-count are refused or read since the decode limit, the length-chain bound and the /N bound; + # jpeg-selector is a known defect until the change that fixes it removes its --known line. - name: Hostile documents run: | set -euo pipefail @@ -155,7 +156,7 @@ jobs: python docs/tools/corpus-robustness.py \ --probe ./tools/SharpCorePDF.Validation/bin/Release/net11.0/SharpCorePDF.Validation.dll \ --timeout 120 --report artifacts/hostile.json \ - --known flate-bomb.pdf --known length-chain.pdf --known objstm-count.pdf --known jpeg-selector.pdf \ + --known jpeg-selector.pdf \ "$RUNNER_TEMP/hostile" - name: Upload the documents and the reports diff --git a/CHANGELOG.md b/CHANGELOG.md index 99ac86d..abe4bd9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,19 +33,37 @@ against an empty feed. reaches a consumer. [`docs/validation.md`](docs/validation.md) records each check, how it was shown to fail before it was trusted, and the defects below. +- **`PdfEngineOptions.MaxDecodedStreamBytes`** (128 MiB by default) bounds what one stream may decode to. + `MaxSourceBytes` bounds the input only, and Flate reaches about a thousand to one. + +### Fixed + +- **Security: signature wrapping.** The validator required only that the byte range start at 0 and end at the + end of the file, so bytes written into the unsigned gap could close the signature value early and add + entries while the digest still matched, and the signature was reported valid and covering the whole + document. The gap must now be exactly one hexadecimal string equal to the dictionary's `/Contents`; + anything else is `MalformedByteRange`, integrity `Unknown`, and `CoversWholeDocument` false. +- **Security: decompression bombs.** Flate, LZW and RunLength streams decoded without a limit, so a 1.1 MB + document took the process past 1 GiB. They now stop at `MaxDecodedStreamBytes`; buffer growth no longer + overflows past 1 GiB. +- **Security: stack overflow.** A chain of streams whose `/Length` is the next stream was resolved by + recursion, and 60,000 links ended the process. The chain is followed no deeper than `MaxNestingDepth`; + past it the stream is found by its `endstream` keyword, as for a `/Length` cycle. +- **Security: allocations sized by the file.** An object stream's `/N` sized two arrays before anything checked + it (513 bytes asked for 16 GiB), and a JPEG frame's 16-bit width and height sized the sample planes before + the caller's budget was consulted. `/N` is now bounded by what the stream can hold, a JPEG above the budget + is measured from its header without being decoded — the same answer as before, without the allocation — + and the working sizes are computed in 64 bits. + ### Known defects, found by the new checks and not yet fixed Each is reproduced, pinned by a skipped test or a strict `--known-defect` flag that fails the run once the defect is fixed, and listed in [`docs/validation.md`](docs/validation.md#known-defects). -- The signature validator does not require the gap between the byte ranges to be exactly the `/Contents` - string, so a gap carrying extra entries is still reported as covering the whole document. - An incremental update rewrites both elements of `/ID`, and appends a renumbered copy of the whole document rather than the objects it changed — which also rewrites the signature dictionary after signing. - A merge copies a page that states no `/MediaBox` without one. -- Hostile input: a Flate stream has no decoded-size limit, a chain of indirect `/Length` references overflows - the stack, an object stream's `/N` sizes an allocation unchecked, and a JPEG scan's Huffman table selector - is not range-checked. +- Hostile input: a JPEG scan's Huffman table selector is not range-checked. - Real-world input (the pdf.js test corpus): an empty name as a dictionary key stops a document opening, a glyph with an upside-down box faults text extraction, and `PdfDate.TryParse` throws on an out-of-range time-zone offset instead of answering false. diff --git a/docs/manual/src/10-dependency-injection.md b/docs/manual/src/10-dependency-injection.md index 953b10f..d46b5aa 100644 --- a/docs/manual/src/10-dependency-injection.md +++ b/docs/manual/src/10-dependency-injection.md @@ -144,6 +144,7 @@ defaults are deliberately permissive: | Option | Default | Effect | | --- | --- | --- | | `MaxSourceBytes` | 512 MiB | Largest source read into memory; larger sources are rejected | +| `MaxDecodedStreamBytes` | 128 MiB | Largest size one stream may decode to; a stream that would decode past it is refused | | `MaxNestingDepth` | 64 | Maximum dictionary/array nesting, page-tree depth and form field tree depth | | `MaxPagesPerSource` | 100 000 | Declared page limit; see the note below | | `ReconstructCrossReferenceOnFailure` | `true` | Rebuild a damaged or missing cross-reference table by scanning | @@ -154,8 +155,15 @@ The source limit is enforced by the reader, and exceeding it throws `PdfSourceFo detail that names the limit. It is not a streaming cap: the whole source is held in memory while it is processed, so the practical ceiling is your process's memory budget, not the number on the option. -`MaxNestingDepth` has four effects. It bounds how deep an object graph may be parsed, it bounds the -page-tree walk, it bounds the form field tree, and it bounds the tree the signature search descends. A +`MaxDecodedStreamBytes` bounds what a stream decodes to, which the source limit does not: Flate reaches about +a thousand to one, so a one-megabyte document can ask for a gigabyte. A content stream, form or font program +past the limit is reported as unreadable through the warning list, as other undecodable content is; an object +or cross-reference stream past it refuses the document with `PdfSourceFormatException`. Images are not refused +by it — their samples are carried only within `MaxInlineContentBytes` already. + +`MaxNestingDepth` has five effects. It bounds how deep an object graph may be parsed, it bounds how many +streams deep a chain of indirect `/Length` references is followed (past it, the stream is found by its +`endstream` keyword), it bounds the page-tree walk, it bounds the form field tree, and it bounds the tree the signature search descends. A page tree that is cut off contributes a `PageTreeTooDeep` warning rather than failing the merge; a field tree that is cut off contributes `FormFieldDepthExceeded` together with the fields above the cut, and the search for signatures reports the same warning and adds a `PdfSignatureProblemKind.FormFieldTreeTooDeep` @@ -176,7 +184,7 @@ file is binary; the marker costs a few bytes and prevents a class of corruption hard to diagnose. > `MaxPagesPerSource` is part of the options record but this version does not enforce it. Do not rely on -> it as a security boundary. For untrusted input, set `MaxSourceBytes`, inspect the document with +> it as a security boundary. For untrusted input, set `MaxSourceBytes` and `MaxDecodedStreamBytes`, inspect the document with > `IPdfInspector` and reject it if `PageCount` exceeds your own limit, before building a plan. ## Configuration file diff --git a/docs/validation.md b/docs/validation.md index 7986dba..6ce4585 100644 --- a/docs/validation.md +++ b/docs/validation.md @@ -83,25 +83,30 @@ fix. This is the list on 2026-11-16, every entry reproduced, not inferred: | Defect | Found by | Evidence | | --- | --- | --- | -| The signature validator does not check that the gap between the two byte ranges is exactly the `/Contents` string, so a gap that carries extra entries is still reported as covering the whole document (the signature-wrapping class) | #3 | `PdfSignatureAttackTests.Does_not_report_a_gap_that_holds_more_than_the_signature_value_as_covering_the_document` (skipped; fails when run) | | An incremental update writes a new identifier into both elements of `/ID`; ISO 32000-1 §14.4 keeps the first | #10 | `PdfIncrementalInvariantTests.Keeps_the_first_identifier_element_across_an_incremental_update` (skipped); `validate-incremental.py` `id-first` | | `UseIncrementalUpdate` appends a renumbered copy of the whole document rather than the objects it changed: every byte of the earlier revision is kept, but the file grows by its own size per edit, and the signature dictionary is written again after signing, so pyHanko places a signature followed by an edit as `CONTIGUOUS_BLOCK_FROM_START` with modification level `OTHER` instead of `ENTIRE_REVISION` | #10, #3 | `validate-incremental.py` `appends-only`; `validate-signatures.py` on `signed-then-edited.pdf` | | A merge copies a page that states no `/MediaBox` (and inherits none) without one, so the result breaks the rule that every page has a media box; qpdf repairs it by assuming Letter | #2 | `validate-pdf.py` on `inspected.pdf` and `with-warnings.pdf` (both merge a page of `NamedDests.pdf`) | -| A Flate stream has no limit on its decoded size: a 1.1 MB document inflates past 1 GiB | #12 | `flate-bomb.pdf`: `OutOfMemoryException` under a 1 GiB heap limit | -| A chain of indirect `/Length` references is resolved by recursion without a depth limit | #12 | `length-chain.pdf`: the process dies with a stack overflow, which no caller can catch | -| An object stream's `/N` sizes an allocation before anything checks it against the stream | #12 | `objstm-count.pdf` (513 bytes): `OutOfMemoryException` | | The JPEG decoder indexes its four Huffman tables with a scan's 4-bit selector unchecked | #12, #1 | `jpeg-selector.pdf`: `IndexOutOfRangeException` from `JpegDecoder.DecodeBlock` | | A dictionary key that is the empty name (`/` alone, which PDF syntax allows) makes `PdfDictionary.Set` throw `ArgumentException` while parsing, so the document does not open at all | #12 | pdf.js corpus: `issue3188.pdf`, `issue5896.pdf`, `pdfjs_wikipedia.pdf`, `poppler-742-0-fuzzed.pdf` | | A glyph whose box comes out upside down (a negative font size or a flipped font matrix) makes `PdfGlyph.BoundingBox` construct a `PdfRectangle` that throws, so text extraction faults | #12 | pdf.js corpus: `issue14415.pdf` | | `PdfDate.TryParse` throws `ArgumentOutOfRangeException` for a time-zone offset beyond 14 hours instead of answering false, so one malformed `/CreationDate` faults the whole inspection | #12 | pdf.js corpus: `TAMReview.pdf` | +Fixed since the list was first drawn up, each with the test that fails without its fix: + +| Defect | Fixed by | Test | +| --- | --- | --- | +| The signature validator accepted a `/ByteRange` gap holding more than the `/Contents` string (signature wrapping) | the gap must be exactly one hex string, equal to the dictionary's `/Contents` | `PdfSignatureAttackTests.Does_not_report_a_gap_that_holds_more_than_the_signature_value_as_covering_the_document` | +| No limit on the decoded size of a Flate, LZW or RunLength stream | `PdfEngineOptions.MaxDecodedStreamBytes` (128 MiB) | `PdfFilterTests.Refuses_a_flate_stream_that_decodes_past_the_limit` and its LZW, RunLength and content-stream companions | +| A chain of indirect `/Length` references overflowed the stack | the chain is followed no deeper than `MaxNestingDepth`, then the stream is found by its `endstream` | `PdfReaderRecoveryTests.Reads_a_stream_whose_length_is_a_chain_of_sixty_thousand_streams` | +| An object stream's `/N`, and a JPEG frame's width and height, sized allocations unchecked | `/N` is bounded by what the stream can hold; a JPEG above the caller's budget is measured from its header, and working sizes are computed in 64 bits | `PdfReaderRecoveryTests.Reads_an_object_stream_whose_count_its_header_cannot_hold`, `PdfJpegFilterTests.Measures_a_frame_too_large_for_the_budget_without_allocating_it` | + ## Corpus readings, 2026-11-16 | Corpus | Files | Read | Refused | Fault | Crash | Timeout | | --- | --- | --- | --- | --- | --- | --- | | veraPDF corpus, `PDF_A-2b` and `PDF_A-4` at `bb75f4f` | 1,471 | 1,471 | 0 | 0 | 0 | 0 | | pdf.js `test/pdfs` at `v6.4.299` | 987 | 977 | 4 | 6 | 0 | 0 | -| synthesized hostile documents | 4 | 0 | 0 | 3 | 1 | 0 | +| synthesized hostile documents, before the fixes | 4 | 0 | 0 | 3 | 1 | 0 | The six pdf.js faults are the three defects above. A sample of the pre-flight measurement (the first 60 files of `PDF_A-2b` and of `PDF_A-4`) wrote 108 results, refused 9 non-conforming inputs and refused 3 that diff --git a/src/SharpCorePDF.Infrastructure/Pdf/JpegDecoder.cs b/src/SharpCorePDF.Infrastructure/Pdf/JpegDecoder.cs index 2619fd8..528cf69 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/JpegDecoder.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/JpegDecoder.cs @@ -154,6 +154,16 @@ private bool Run(int maxBytes, out byte[]? pixels, out string? format, out int w return false; } + if (IsAboveBudget(maxBytes, out format, out width, out height)) + { + return format is not null; + } + + if (!WorkingBuffersFit()) + { + return false; + } + break; case 0xC2: progressive = true; @@ -163,6 +173,16 @@ private bool Run(int maxBytes, out byte[]? pixels, out string? format, out int w return false; } + if (IsAboveBudget(maxBytes, out format, out width, out height)) + { + return format is not null; + } + + if (!WorkingBuffersFit()) + { + return false; + } + break; case 0xDD: if (segment.Length < 2) @@ -320,6 +340,64 @@ private bool ParseHuffman(ReadOnlySpan segment) return true; } + /// + /// Answers, from the frame header alone, for an image whose samples the caller would not keep: the + /// format and the size are measured and nothing is decoded. + /// + /// The largest pixel buffer the caller keeps. + /// The pixel format the samples would state, or null for a component count the decoder has none for. + /// The width the frame states. + /// The height the frame states. + /// when the output is above the budget and the answer is final. + /// + /// The width and the height are two 16-bit numbers in the file: 65,535 squared is four gigasamples per + /// component, and the sample planes — and, for a progressive image, the coefficient grids at four bytes a + /// coefficient — were allocated and filled before the output was found too large to keep. The contract is + /// unchanged: such an image is measured and no pixels are returned. It is only no longer decoded first. + /// + private bool IsAboveBudget(int maxBytes, out string? format, out int measuredWidth, out int measuredHeight) + { + format = null; + measuredWidth = 0; + measuredHeight = 0; + + if ((long)width * height * components.Length <= maxBytes) + { + return false; + } + + format = components.Length switch + { + 1 => "Gray8", + 3 => "Rgb24", + 4 => "Cmyk32", + _ => null, + }; + measuredWidth = width; + measuredHeight = height; + return true; + } + + /// Determines whether the working buffers the frame asks for fit an array at all. + /// when every sample plane or coefficient grid fits. + /// The sizes are computed in 64 bits: in 32 they wrap, and a wrapped size is a small allocation that the decoder then writes past. + private bool WorkingBuffersFit() + { + var mcuColumns = (width + (8L * maxH) - 1) / (8L * maxH); + var mcuRows = (height + (8L * maxV) - 1) / (8L * maxV); + var perSample = progressive ? sizeof(int) : sizeof(byte); + + foreach (var component in components) + { + if (mcuColumns * 8 * component.H * mcuRows * 8 * component.V * perSample > Array.MaxLength) + { + return false; + } + } + + return true; + } + private bool ParseFrame(ReadOnlySpan segment) { if (segment.Length < 6 || segment[0] != 8) diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.Streams.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.Streams.cs index cec798b..7245f97 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.Streams.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.Streams.cs @@ -33,7 +33,7 @@ internal sealed partial class PdfCrossReferenceReader table.Add(new XrefEntry(objectNumber, offset, generation, XrefEntryType.Uncompressed)); } - var decoded = PdfFilters.Decode(stream, sourceName); + var decoded = PdfFilters.Decode(stream, sourceName, options.MaxDecodedStreamBytes); var widthType = 1; var widthField1 = 4; var widthField2 = 2; diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.ObjectStreams.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.ObjectStreams.cs index a208657..0a5df17 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.ObjectStreams.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.ObjectStreams.cs @@ -43,18 +43,24 @@ private void EnsureObjectStreamLoaded(long containerObjectNumber) } var dictionary = stream.Dictionary; - var count = (int)(dictionary.GetInteger("N") ?? 0); - var first = (int)(dictionary.GetInteger("First") ?? 0); + var declared = dictionary.GetInteger("N") ?? 0; + var first = dictionary.GetInteger("First") ?? 0; - if (count <= 0 || first < 0) + if (declared <= 0 || first < 0) { return; } - var decoded = PdfFilters.Decode(stream, SourceName); + var decoded = PdfFilters.Decode(stream, SourceName, options.MaxDecodedStreamBytes); + + // /N sizes two arrays, and it is a number the file states: 2^31 - 1 of them would ask for + // gigabytes from a stream of a few hundred bytes. Every pair of the header takes at least four + // bytes ("1 0 "), so the stream itself bounds how many objects it can describe, and the arrays + // are sized by that bound; a header that ends before /N says so below. + var count = (int)Math.Min(declared, (decoded.Length / 4L) + 1); var header = new PdfObjectParser(decoded, SourceName, 0, null, options.MaxNestingDepth); var numbers = new long[count]; - var offsets = new int[count]; + var offsets = new long[count]; for (var index = 0; index < count; index++) { @@ -62,12 +68,12 @@ private void EnsureObjectStreamLoaded(long containerObjectNumber) { warnings.Add(new( PdfWarningKind.ObjectStreamTruncated, - $"'{SourceName}': object stream {containerObjectNumber} declares {count} objects but its header ends early.")); + $"'{SourceName}': object stream {containerObjectNumber} declares {declared} objects but its header ends early.")); break; } numbers[index] = number; - offsets[index] = (int)offset; + offsets[index] = offset; } for (var index = 0; index < count; index++) @@ -79,7 +85,7 @@ private void EnsureObjectStreamLoaded(long containerObjectNumber) continue; } - var parser = new PdfObjectParser(decoded, SourceName, position, null, options.MaxNestingDepth); + var parser = new PdfObjectParser(decoded, SourceName, (int)position, null, options.MaxNestingDepth); fromObjectStreams[numbers[index]] = parser.ParseObject(); } } diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.Objects.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.Objects.cs index 1a39e97..4eeaa67 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.Objects.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.Objects.cs @@ -72,23 +72,33 @@ private PdfVersion ReadVersion() /// /// Resolves a stream /Length that is itself an indirect reference. The guard set breaks - /// the cycle that such a reference creates when the length object is the stream itself. + /// the cycle that such a reference creates when the length object is the stream itself, and the + /// depth guard breaks a chain: a length that is a stream whose length is a stream, and so on. /// /// The object number of the length object. /// The length, or when it cannot be resolved safely. + /// + /// Each link of a chain is resolved inside the one before it, so a chain of tens of thousands of + /// streams would overflow the stack — which no caller can catch: the process ends. A well-formed + /// length needs one level; past the length is answered + /// as unknown, exactly as a cycle is, and the parser finds the end of the stream by its keyword. + /// private long? ResolveIndirectLength(long objectNumber) { - if (!resolvingLengths.Add(objectNumber)) + if (lengthResolutionDepth >= options.MaxNestingDepth || !resolvingLengths.Add(objectNumber)) { return null; } + lengthResolutionDepth++; + try { return Resolve(objectNumber) is PdfInteger integer ? integer.Value : null; } finally { + lengthResolutionDepth--; resolvingLengths.Remove(objectNumber); } } diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.cs index 0c1d647..fcec760 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.cs @@ -20,6 +20,7 @@ internal sealed partial class PdfFileReader private readonly HashSet loadedObjectStreams = []; private readonly HashSet resolvingObjectStreams = []; private readonly HashSet resolvingLengths = []; + private int lengthResolutionDepth; private PageTreeWalk? pageTree; private PdfDictionary? catalog; diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.Inflation.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.Inflation.cs index e3c2f47..8fe69f2 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.Inflation.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.Inflation.cs @@ -12,30 +12,79 @@ internal static partial class PdfFilters private const int InflateChunkSize = 16 * 1024; private const int MaximumInitialInflateCapacity = 1024 * 1024; - /// Attempts to decompress Flate data, tolerating the raw deflate streams some producers write. - /// The compressed bytes. - /// The decompressed bytes when decompression succeeded. - /// when the data could be decompressed. - public static bool TryInflate(ReadOnlyMemory data, out byte[] result) + /// + /// The decoded size a stream is held to when the caller has no to read it + /// from; the same value as 's default. + /// + internal const long DefaultMaxDecodedBytes = 128L * 1024 * 1024; + + /// What one attempt at inflating a stream came to. + private enum InflateOutcome { - result = []; + /// The stream inflated within the limit. + Inflated, + + /// The bytes are not a stream this wrapper can inflate. + Invalid, - return !data.IsEmpty - && (TryInflateCore(data, zlibWrapper: true, out result) - || TryInflateCore(data, zlibWrapper: false, out result)); + /// The stream inflates past the limit; it was stopped there. + TooLarge, } - /// Decompresses Flate data. + /// Attempts to decompress Flate data, tolerating the raw deflate streams some producers write. + /// The compressed bytes. + /// The decompressed bytes when decompression succeeded. + /// when the data could be decompressed within the default limit. + public static bool TryInflate(ReadOnlyMemory data, out byte[] result) => + TryInflate(data, DefaultMaxDecodedBytes, out result) == InflateOutcome.Inflated; + + /// Decompresses Flate data under the default limit on its decoded size. /// The compressed bytes. /// The name of the source, used in diagnostics. /// The decompressed bytes. - /// The data could not be decompressed. + /// The data could not be decompressed, or decodes past the limit. public static byte[] Inflate(ReadOnlyMemory data, string sourceName) => - TryInflate(data, out var result) - ? result - : throw new PdfSourceFormatException(sourceName, "a Flate compressed stream could not be decompressed."); + Inflate(data, sourceName, DefaultMaxDecodedBytes); - private static bool TryInflateCore(ReadOnlyMemory data, bool zlibWrapper, out byte[] result) + /// Decompresses Flate data, refusing a stream that decodes to more than a limit. + /// The compressed bytes. + /// The name of the source, used in diagnostics. + /// The largest number of bytes the stream may decode to. + /// The decompressed bytes. + /// The data could not be decompressed, or decodes past the limit. + public static byte[] Inflate(ReadOnlyMemory data, string sourceName, long maxDecodedBytes) => + TryInflate(data, maxDecodedBytes, out var result) switch + { + InflateOutcome.Inflated => result, + InflateOutcome.TooLarge => throw DecodedTooLarge(sourceName, "a Flate", maxDecodedBytes), + _ => throw new PdfSourceFormatException(sourceName, "a Flate compressed stream could not be decompressed."), + }; + + /// Builds the refusal of a stream that decodes past the limit. + /// The name of the source. + /// The kind of stream, for the message. + /// The limit that was reached. + /// The exception to throw. + internal static PdfSourceFormatException DecodedTooLarge(string sourceName, string what, long maxDecodedBytes) => + new(sourceName, $"{what} stream decodes to more than {maxDecodedBytes} bytes, the limit PdfEngineOptions.MaxDecodedStreamBytes sets, so it was not decoded."); + + private static InflateOutcome TryInflate(ReadOnlyMemory data, long maxDecodedBytes, out byte[] result) + { + result = []; + + if (data.IsEmpty) + { + return InflateOutcome.Invalid; + } + + var outcome = TryInflateCore(data, zlibWrapper: true, maxDecodedBytes, out result); + + return outcome == InflateOutcome.Invalid + ? TryInflateCore(data, zlibWrapper: false, maxDecodedBytes, out result) + : outcome; + } + + private static InflateOutcome TryInflateCore(ReadOnlyMemory data, bool zlibWrapper, long maxDecodedBytes, out byte[] result) { result = []; @@ -51,7 +100,8 @@ private static bool TryInflateCore(ReadOnlyMemory data, bool zlibWrapper, // stream surprises us. Growth is progressive instead of one guess, so a small stream never // reserves the memory of a large one. using var output = new PooledByteBuffer( - (int)Math.Clamp((long)data.Length * 4, InflateChunkSize, MaximumInitialInflateCapacity)); + (int)Math.Clamp((long)data.Length * 4, InflateChunkSize, MaximumInitialInflateCapacity), + poolLargeArrays: false); while (true) { @@ -63,19 +113,27 @@ private static bool TryInflateCore(ReadOnlyMemory data, bool zlibWrapper, break; } + // Flate reaches about a thousand to one, so the limit on the input does not bound this + // buffer: a megabyte of zeros is a gigabyte here. The stream is stopped at the limit + // instead of being allowed to take the process with it. + if (output.Length + (long)read > maxDecodedBytes) + { + return InflateOutcome.TooLarge; + } + output.Advance(read); } result = output.ToExactArray(); - return true; + return InflateOutcome.Inflated; } catch (InvalidDataException) { - return false; + return InflateOutcome.Invalid; } catch (IOException) { - return false; + return InflateOutcome.Invalid; } } @@ -91,9 +149,9 @@ private static MemoryStream OpenInput(ReadOnlyMemory data) => ? new MemoryStream(array, segment.Offset, segment.Count, writable: false) : new MemoryStream(data.ToArray(), writable: false); - private static byte[] ApplyFlate(ReadOnlyMemory data, PdfDictionary? parameters, string sourceName) + private static byte[] ApplyFlate(ReadOnlyMemory data, PdfDictionary? parameters, string sourceName, long maxDecodedBytes) { - var inflated = Inflate(data, sourceName); + var inflated = Inflate(data, sourceName, maxDecodedBytes); // Every parameter of the filter is optional and every one of them is absent when the stream has no // filter dictionary at all, so the dictionary is established once here instead of being tested @@ -134,5 +192,5 @@ private static byte[] ApplyFlate(ReadOnlyMemory data, PdfDictionary? param public static byte[] InflateAndUnpredict( ReadOnlyMemory data, PdfDictionary? parameters, - string sourceName) => ApplyFlate(data, parameters, sourceName); + string sourceName) => ApplyFlate(data, parameters, sourceName, DefaultMaxDecodedBytes); } diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.Lzw.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.Lzw.cs index ee2e587..fd2162a 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.Lzw.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.Lzw.cs @@ -143,18 +143,31 @@ internal static int ReadEarlyChange(PdfDictionary? parameters) => /// The compressed bytes. /// The decode parameters of the filter, which state EarlyChange. /// The name of the source, used in diagnostics. + /// The largest number of bytes the stream may decode to. /// The decoded bytes. - /// The stream is not a valid LZW stream. + /// The stream is not a valid LZW stream, or decodes past the limit. internal static byte[] ApplyLzw( ReadOnlyMemory data, PdfDictionary? parameters, - string sourceName) + string sourceName, + long maxDecodedBytes) { - var output = new PooledByteBuffer(); + var output = new PooledByteBuffer(poolLargeArrays: false); try { - if (!TryDecodeLzw(data.Span, ReadEarlyChange(parameters), output.Write, out _)) + // A 12-bit code can stand for 4,096 bytes, so the input limit does not bound this buffer either. + void Write(ReadOnlySpan bytes) + { + if (output.Length + (long)bytes.Length > maxDecodedBytes) + { + throw DecodedTooLarge(sourceName, "an LZW", maxDecodedBytes); + } + + output.Write(bytes); + } + + if (!TryDecodeLzw(data.Span, ReadEarlyChange(parameters), Write, out _)) { throw new PdfSourceFormatException(sourceName, "an LZW stream could not be decompressed."); } diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.RunLength.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.RunLength.cs index 2acb163..2b029b1 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.RunLength.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.RunLength.cs @@ -24,11 +24,21 @@ internal static partial class PdfFilters /// reaches past the end contributes the bytes that are there, and a repeated run with no value byte /// contributes nothing. /// - public static byte[] DecodeRunLength(ReadOnlyMemory data) + public static byte[] DecodeRunLength(ReadOnlyMemory data) => + DecodeRunLength(data, "a stream", DefaultMaxDecodedBytes); + + /// Decodes a run-length encoded stream, refusing one that decodes past a limit. + /// The encoded bytes. + /// The name of the source, used in diagnostics. + /// The largest number of bytes the stream may decode to. + /// The decoded bytes. + /// The stream decodes past the limit. + /// Two bytes can stand for 128, so the limit on the input does not bound the output. + public static byte[] DecodeRunLength(ReadOnlyMemory data, string sourceName, long maxDecodedBytes) { // The output is at least as large as the input for the literal case, and can be 128 times larger // for the repeat case, so the buffer starts at the input length and grows only when it must. - using var result = new PooledByteBuffer(data.Length); + using var result = new PooledByteBuffer(data.Length, poolLargeArrays: false); var span = data.Span; var index = 0; @@ -48,6 +58,11 @@ public static byte[] DecodeRunLength(ReadOnlyMemory data) // end, which ends the loop. The explicit guard a survivor asked about is this comment. var count = Math.Min(control + 1, span.Length - index); + if (result.Length + (long)count > maxDecodedBytes) + { + throw DecodedTooLarge(sourceName, "a RunLength", maxDecodedBytes); + } + result.Write(span.Slice(index, count)); index += count; continue; @@ -60,6 +75,11 @@ public static byte[] DecodeRunLength(ReadOnlyMemory data) break; } + if (result.Length + (long)repeat > maxDecodedBytes) + { + throw DecodedTooLarge(sourceName, "a RunLength", maxDecodedBytes); + } + var value = span[index++]; for (var count = 0; count < repeat; count++) diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.cs index 261f5a8..5552260 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.cs @@ -27,7 +27,17 @@ public static bool HasFlateCompression(PdfDictionary dictionary) /// The decoded bytes. /// The stream uses a filter the engine does not implement. /// A compressed stream could not be decompressed. - public static byte[] Decode(PdfStream stream, string sourceName) + public static byte[] Decode(PdfStream stream, string sourceName) => + Decode(stream, sourceName, DefaultMaxDecodedBytes); + + /// Decodes a stream, applying every filter it declares in order, under a limit on its decoded size. + /// The stream to decode. + /// The name of the source, used in diagnostics. + /// The largest number of bytes any one filter may decode to. + /// The decoded bytes. + /// The stream uses a filter the engine does not implement. + /// A compressed stream could not be decompressed, or decodes past the limit. + public static byte[] Decode(PdfStream stream, string sourceName, long maxDecodedBytes) { ArgumentNullException.ThrowIfNull(stream); @@ -51,8 +61,8 @@ public static byte[] Decode(PdfStream stream, string sourceName) decoded = name switch { - "FlateDecode" or "Fl" => ApplyFlate(current, parametersForFilter, sourceName), - "LZWDecode" or "LZW" => ApplyLzw(current, parametersForFilter, sourceName), + "FlateDecode" or "Fl" => ApplyFlate(current, parametersForFilter, sourceName, maxDecodedBytes), + "LZWDecode" or "LZW" => ApplyLzw(current, parametersForFilter, sourceName, maxDecodedBytes), "ASCIIHexDecode" or "AHx" => DecodeAsciiHex(current), "ASCII85Decode" or "A85" => DecodeAscii85(current), _ => throw new PdfUnsupportedFeatureException( diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfFontDecoder.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfFontDecoder.cs index 7a31da3..33e41d8 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfFontDecoder.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfFontDecoder.cs @@ -313,7 +313,7 @@ private static PdfFont Create(PdfFileReader reader, string resourceName, PdfDict try { - data = PdfFilters.Decode(stream, reader.SourceName); + data = PdfFilters.Decode(stream, reader.SourceName, reader.Options.MaxDecodedStreamBytes); } catch (PdfUnsupportedFeatureException) { diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfImageFilters.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfImageFilters.cs index afa4917..15951b7 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfImageFilters.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfImageFilters.cs @@ -23,7 +23,7 @@ internal readonly record struct PdfPayload( /// /// /// -/// answers a different question and answers it correctly: an object +/// answers a different question and answers it correctly: an object /// stream that cannot be inflated means a broken document, so it raises. An image is different — a /// JPXDecode scan is a perfectly valid document this library simply does not decode — so this /// class reports what it could not do and hands back the raw bytes (NFR-7.6, NFR-7.8). @@ -361,7 +361,7 @@ private static bool TryDecodeOne( case "RunLengthDecode": { - var decoded = PdfFilters.DecodeRunLength(data); + var decoded = PdfFilters.DecodeRunLength(data, sourceName, PdfFilters.DefaultMaxDecodedBytes); length = decoded.Length; bytes = Keep(decoded, cap); diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfTextCollector.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfTextCollector.cs index 5a68441..b99551d 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfTextCollector.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfTextCollector.cs @@ -382,7 +382,7 @@ private void PaintForm(string name, PdfDictionary? resources, TextState state, i try { - if (!TryDecodeContent(stream, out var formContent)) + if (!TryDecodeContent(stream, reader.Options.MaxDecodedStreamBytes, out var formContent)) { warnings.Add(new( PdfWarningKind.ContentStreamUnreadable, @@ -432,13 +432,14 @@ private string DescribeUnmapped(PdfFont font) => string.Create( /// Decodes the content stream of a form. /// The stream. + /// The largest number of bytes the stream may decode to. /// The decoded bytes when decoding succeeded. /// when the stream could be decoded. - private static bool TryDecodeContent(PdfStream stream, out byte[] content) + private static bool TryDecodeContent(PdfStream stream, long maxDecodedBytes, out byte[] content) { try { - content = PdfFilters.Decode(stream, "form content stream"); + content = PdfFilters.Decode(stream, "form content stream", maxDecodedBytes); return true; } catch (Domain.Exceptions.PdfUnsupportedFeatureException) diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfType1Encoding.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfType1Encoding.cs index 2a19346..84eddd9 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfType1Encoding.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfType1Encoding.cs @@ -52,7 +52,7 @@ internal static class PdfType1Encoding try { - bytes = PdfFilters.Decode(program, reader.SourceName); + bytes = PdfFilters.Decode(program, reader.SourceName, reader.Options.MaxDecodedStreamBytes); } catch (PdfUnsupportedFeatureException) { diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PooledByteBuffer.cs b/src/SharpCorePDF.Infrastructure/Pdf/PooledByteBuffer.cs index 12615ac..5a9f02c 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PooledByteBuffer.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PooledByteBuffer.cs @@ -24,17 +24,32 @@ internal sealed class PooledByteBuffer : IDisposable { private const int MinimumCapacity = 16; + /// + /// The largest array a decode buffer takes from the pool. Above it the array is allocated and left to the + /// collector, because the shared pool keeps what is returned to it: a document that inflates a stream to + /// the decode limit in every read pass would otherwise leave hundreds of megabytes parked in the pool, + /// and the next pass would allocate beside them. + /// + private const int MaximumPooledDecodeLength = 16 * 1024 * 1024; + private byte[] array; private int length; + private readonly bool poolLargeArrays; private bool returned; /// Initializes a new instance of the class. /// The number of bytes to rent up front. - public PooledByteBuffer(int initialCapacity = 256) + /// + /// Whether arrays above 16 MiB go through the pool too. The read of a source keeps them pooled, because + /// reusing the one buffer a document is read into is what keeps a large read at one allocation; a decode + /// buffer does not, because its size is whatever a stream inflates to and the pool would keep it. + /// + public PooledByteBuffer(int initialCapacity = 256, bool poolLargeArrays = true) { ArgumentOutOfRangeException.ThrowIfNegative(initialCapacity); - array = ArrayPool.Shared.Rent(Math.Max(initialCapacity, MinimumCapacity)); + this.poolLargeArrays = poolLargeArrays; + array = Rent(Math.Max(initialCapacity, MinimumCapacity)); } /// Gets the number of bytes that have been written. @@ -112,7 +127,7 @@ public void Dispose() } returned = true; - ArrayPool.Shared.Return(array); + Return(array); array = []; length = 0; } @@ -124,14 +139,36 @@ private void EnsureFree(int sizeHint) return; } - Grow(length + sizeHint); + Grow((long)length + sizeHint); } - private void Grow(int minimumCapacity) + private void Grow(long minimumCapacity) { - var grown = ArrayPool.Shared.Rent(Math.Max(minimumCapacity, array.Length * 2)); + // Doubling is computed in 64 bits and clamped to the largest array the runtime allows, so a buffer + // past a gigabyte grows to the limit instead of to a negative size; a request beyond the limit is + // refused. The decode limits keep every caller below it, so this is a guard and not a path. + if (minimumCapacity > Array.MaxLength) + { + throw new InvalidOperationException($"A buffer of {minimumCapacity} bytes is larger than the runtime allows."); + } + + var capacity = (int)Math.Min(Math.Max(minimumCapacity, 2L * array.Length), Array.MaxLength); + var grown = Rent(capacity); array.AsSpan(0, length).CopyTo(grown); - ArrayPool.Shared.Return(array); + Return(array); array = grown; } + + private byte[] Rent(int length) => + poolLargeArrays || length <= MaximumPooledDecodeLength + ? ArrayPool.Shared.Rent(length) + : GC.AllocateUninitializedArray(length); + + private void Return(byte[] rented) + { + if (poolLargeArrays || rented.Length <= MaximumPooledDecodeLength) + { + ArrayPool.Shared.Return(rented); + } + } } diff --git a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Overlays.cs b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Overlays.cs index 7994f4e..23e7ab5 100644 --- a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Overlays.cs +++ b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Overlays.cs @@ -252,7 +252,7 @@ private static void AppendOverlayContent(PdfFileReader reader, PdfObject? value, return; } - content.Write(PdfFilters.Decode(stream, reader.SourceName)); + content.Write(PdfFilters.Decode(stream, reader.SourceName, reader.Options.MaxDecodedStreamBytes)); content.WriteByte((byte)'\n'); } diff --git a/src/SharpCorePDF.Infrastructure/PdfEngineOptions.cs b/src/SharpCorePDF.Infrastructure/PdfEngineOptions.cs index 3cdff45..8af71bd 100644 --- a/src/SharpCorePDF.Infrastructure/PdfEngineOptions.cs +++ b/src/SharpCorePDF.Infrastructure/PdfEngineOptions.cs @@ -18,6 +18,24 @@ public sealed record PdfEngineOptions /// public long MaxSourceBytes { get; init; } = 512L * 1024 * 1024; + /// + /// Gets the largest number of bytes one filter of one stream may decode to. A stream that would decode + /// past it is refused with a instead of + /// exhausting the process. + /// + /// + /// bounds what is read, not what it decodes to: Flate reaches about a + /// thousand to one, LZW four thousand and RunLength 128, so a one-megabyte document can ask for a + /// gigabyte. The default of 128 MiB is far above any real content stream, object stream or font program, + /// and low enough that refusing a stream at the limit — whose buffer doubles on the way there, on the + /// large-object heap, which the collector does not compact by default — stays reliably inside a 1 GiB + /// process when every read pass meets the same stream; the largest value accepted is the largest array + /// the runtime allows. An image is not refused by this limit — its samples are kept only + /// within already — but every other stream the engine has to + /// materialise is: content, forms, object streams, cross-reference streams and font programs. + /// + public long MaxDecodedStreamBytes { get; init; } = 128L * 1024 * 1024; + /// Gets the maximum nesting depth of dictionaries and arrays. public int MaxNestingDepth { get; init; } = 64; @@ -74,6 +92,11 @@ public IReadOnlyList Validate() errors.Add($"MaxSourceBytes must be at least 1024, but is {MaxSourceBytes}."); } + if (MaxDecodedStreamBytes < 1024 || MaxDecodedStreamBytes > Array.MaxLength) + { + errors.Add($"MaxDecodedStreamBytes must be between 1024 and {Array.MaxLength}, but is {MaxDecodedStreamBytes}."); + } + if (MaxNestingDepth is < 4 or > 1024) { errors.Add($"MaxNestingDepth must be between 4 and 1024, but is {MaxNestingDepth}."); diff --git a/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Content.cs b/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Content.cs index ae14104..6eaa240 100644 --- a/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Content.cs +++ b/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Content.cs @@ -64,7 +64,7 @@ public sealed partial class PdfPageContentReader foreach (var stream in streams) { - if (TryDecodeStream(stream, out var decoded)) + if (TryDecodeStream(stream, reader.Options.MaxDecodedStreamBytes, out var decoded)) { // The separator goes between the streams, not after the last one: a single stream is its // own sequence, and an inline image that runs to the end of it must not gain a byte. @@ -114,13 +114,14 @@ private static void CollectStreams(PdfFileReader reader, PdfArray array, ListDecodes a stream whose filters are the ones a content stream may use. /// The stream. + /// The largest number of bytes the stream may decode to. /// The decoded bytes when decoding succeeded. /// when the stream could be decoded. - private static bool TryDecodeStream(PdfStream stream, out byte[] decoded) + private static bool TryDecodeStream(PdfStream stream, long maxDecodedBytes, out byte[] decoded) { try { - decoded = PdfFilters.Decode(stream, "content stream"); + decoded = PdfFilters.Decode(stream, "content stream", maxDecodedBytes); return true; } catch (Domain.Exceptions.PdfUnsupportedFeatureException) diff --git a/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Images.cs b/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Images.cs index 9f9c48d..e56a77b 100644 --- a/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Images.cs +++ b/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Images.cs @@ -358,7 +358,7 @@ private void Paint( return; } - if (TryDecodeStream(stream, out var formContent)) + if (TryDecodeStream(stream, reader.Options.MaxDecodedStreamBytes, out var formContent)) { var formResources = reader.ResolveDictionary(stream.Dictionary.Get("Resources")) ?? resources; var formMatrix = ReadMatrix(reader, stream.Dictionary.Get("Matrix")) ?? PdfMatrix.Identity; diff --git a/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Metadata.cs b/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Metadata.cs index 027f05b..3681e29 100644 --- a/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Metadata.cs +++ b/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Metadata.cs @@ -35,7 +35,7 @@ public async Task ReadMetadataAsync( return new PdfXmpResult { Warnings = [.. reader.Warnings, .. warnings] }; } - if (!TryDecodeStream(stream, out var packet)) + if (!TryDecodeStream(stream, reader.Options.MaxDecodedStreamBytes, out var packet)) { warnings.Add(new( PdfWarningKind.XmpPacketMalformed, diff --git a/src/SharpCorePDF.Infrastructure/PublicAPI.Unshipped.txt b/src/SharpCorePDF.Infrastructure/PublicAPI.Unshipped.txt index 7dc5c58..f73aab7 100644 --- a/src/SharpCorePDF.Infrastructure/PublicAPI.Unshipped.txt +++ b/src/SharpCorePDF.Infrastructure/PublicAPI.Unshipped.txt @@ -1 +1,3 @@ #nullable enable +SharpCorePDF.Infrastructure.PdfEngineOptions.MaxDecodedStreamBytes.get -> long +SharpCorePDF.Infrastructure.PdfEngineOptions.MaxDecodedStreamBytes.init -> void diff --git a/src/SharpCorePDF.Infrastructure/Signatures/SignatureValidator.cs b/src/SharpCorePDF.Infrastructure/Signatures/SignatureValidator.cs index d0254eb..74bb5f0 100644 --- a/src/SharpCorePDF.Infrastructure/Signatures/SignatureValidator.cs +++ b/src/SharpCorePDF.Infrastructure/Signatures/SignatureValidator.cs @@ -258,7 +258,7 @@ public static PdfSignatureValidation Validate(PdfFileReader reader, SignatureFie $"The timestamp of '{field.FieldName}' is present but could not be read, so the moment it claims cannot be relied on.")); } - if (reading.Usable && !reading.CoversWhole) + if (reading.Usable && reading.GapHoldsValue && !reading.CoversWhole) { var revisions = CountRevisions(reader.Data, reader.Data.LongLength); @@ -278,6 +278,7 @@ public static PdfSignatureValidation Validate(PdfFileReader reader, SignatureFie /// The covered ranges. /// Whether the dictionary carries four usable numbers. /// Whether the ranges are ordered and inside the document. + /// Whether the bytes the ranges leave out are exactly the signature value. /// The DER structure of the signature value, without its padding. /// The parsed signature, when it parsed. /// Why the signature value could not be parsed, when it could not. @@ -287,6 +288,7 @@ private sealed record Reading( PdfByteRange Range, bool HasRange, bool Usable, + bool GapHoldsValue, byte[]? Contents, CmsSignature? Cms, Exception? Failure, @@ -314,6 +316,7 @@ private static Reading Read(PdfFileReader reader, PdfDictionary dictionary) && range.IsWellFormed && range.Length1 > 0 && range.Offset2 + range.Length2 <= reader.Data.LongLength; + var gapHoldsValue = usable && GapHoldsContents(reader, dictionary, range); var contents = ReadContents(reader, dictionary); CmsSignature? cms = null; Exception? failure = null; @@ -334,13 +337,72 @@ private static Reading Read(PdfFileReader reader, PdfDictionary dictionary) range, hasRange, usable, + gapHoldsValue, contents, cms, failure, - usable && range.Covers(reader.Data.LongLength), + usable && gapHoldsValue && range.Covers(reader.Data.LongLength), usable ? CountRevisions(reader.Data, range.Offset2 + range.Length2) : 0); } + /// + /// Determines whether the bytes the byte range leaves uncovered are exactly the signature value: one + /// hexadecimal string, delimiters included, and nothing else. + /// + /// The opened document. + /// The signature dictionary. + /// The byte range, already known to be ordered and inside the document. + /// when the gap holds the value and nothing beside it. + /// + /// The gap is never hashed, so anything in it is unsigned. A gap that closes the value early and carries + /// entries or objects of its own still leaves every signed byte intact and the digest matching — the + /// signature-wrapping attack — so the digest cannot be what catches it. A gap that is a single hex string + /// can only carry the value itself, and changing that breaks the digest. The value is also compared with + /// the /Contents the dictionary resolves to, so a dictionary that points its value somewhere else + /// cannot leave an unrelated string in the gap; a document whose strings were decrypted on the way in is + /// compared by shape only, because the parser hands over the decrypted form. + /// + private static bool GapHoldsContents(PdfFileReader reader, PdfDictionary dictionary, PdfByteRange range) + { + var data = reader.Data; + var start = checked((int)(range.Offset1 + range.Length1)); + var end = checked((int)range.Offset2); + + if (end - start < 2 || data[start] != (byte)'<' || data[end - 1] != (byte)'>') + { + return false; + } + + var inner = data.AsSpan(start + 1, end - start - 2); + var digits = new List(inner.Length); + + foreach (var value in inner) + { + if (Uri.IsHexDigit((char)value)) + { + digits.Add(value); + } + else if (!PdfLexer.IsWhitespace(value)) + { + return false; + } + } + + if (reader.Security is not null) + { + return true; + } + + // An odd number of digits is completed with a zero, as the specification reads a hex string. + if (digits.Count % 2 == 1) + { + digits.Add((byte)'0'); + } + + return reader.Resolve(dictionary.Get("Contents")) is PdfString contents + && Convert.FromHexString(System.Text.Encoding.ASCII.GetString(digits.ToArray())).AsSpan().SequenceEqual(contents.Bytes); + } + /// Reads the four numbers of the byte range. /// The opened document. /// The signature dictionary. @@ -469,6 +531,15 @@ private static PdfSignatureIntegrity Check( return PdfSignatureIntegrity.Unknown; } + if (!reading.GapHoldsValue) + { + problems.Add(new PdfSignatureProblem( + PdfSignatureProblemKind.MalformedByteRange, + $"The signature '{fieldName}' leaves bytes {reading.Range.Offset1 + reading.Range.Length1} to {reading.Range.Offset2} uncovered, and they are not exactly its /Contents value: bytes no signature covers would be read as part of the document.")); + + return PdfSignatureIntegrity.Unknown; + } + if (reading.Contents is not { Length: > 0 }) { problems.Add(new PdfSignatureProblem( diff --git a/tests/SharpCorePDF.Tests/PdfEngineOptionsTests.cs b/tests/SharpCorePDF.Tests/PdfEngineOptionsTests.cs index 9252147..5d58175 100644 --- a/tests/SharpCorePDF.Tests/PdfEngineOptionsTests.cs +++ b/tests/SharpCorePDF.Tests/PdfEngineOptionsTests.cs @@ -19,6 +19,7 @@ public void Has_the_documented_defaults() var options = PdfEngineOptions.Default; Assert.Equal(512L * 1024 * 1024, options.MaxSourceBytes); + Assert.Equal(128L * 1024 * 1024, options.MaxDecodedStreamBytes); Assert.Equal(64, options.MaxNestingDepth); Assert.Equal(100_000, options.MaxPagesPerSource); Assert.True(options.ReconstructCrossReferenceOnFailure); @@ -44,7 +45,16 @@ public void Reports_a_limit_that_cannot_be_honoured() (PdfEngineOptions.Default with { MaxPagesPerSource = 0 }).Validate(), error => error.Contains("MaxPagesPerSource", StringComparison.Ordinal)); + Assert.Contains( + (PdfEngineOptions.Default with { MaxDecodedStreamBytes = 1023 }).Validate(), + error => error.Contains("MaxDecodedStreamBytes", StringComparison.Ordinal)); + Assert.Contains( + (PdfEngineOptions.Default with { MaxDecodedStreamBytes = Array.MaxLength + 1L }).Validate(), + error => error.Contains("MaxDecodedStreamBytes", StringComparison.Ordinal)); + Assert.Empty((PdfEngineOptions.Default with { MaxSourceBytes = 1024 }).Validate()); + Assert.Empty((PdfEngineOptions.Default with { MaxDecodedStreamBytes = 1024 }).Validate()); + Assert.Empty((PdfEngineOptions.Default with { MaxDecodedStreamBytes = Array.MaxLength }).Validate()); Assert.Empty((PdfEngineOptions.Default with { MaxNestingDepth = 1024 }).Validate()); Assert.Empty((PdfEngineOptions.Default with { MaxPagesPerSource = 1 }).Validate()); } diff --git a/tests/SharpCorePDF.Tests/PdfFilterTests.cs b/tests/SharpCorePDF.Tests/PdfFilterTests.cs index 65cf9f2..99e6525 100644 --- a/tests/SharpCorePDF.Tests/PdfFilterTests.cs +++ b/tests/SharpCorePDF.Tests/PdfFilterTests.cs @@ -1,7 +1,11 @@ using System.IO.Compression; using System.Text; using SharpCorePDF.Domain.Exceptions; +using SharpCorePDF.Application.Services; +using SharpCorePDF.Domain.ValueObjects; +using SharpCorePDF.Infrastructure; using SharpCorePDF.Infrastructure.Pdf; +using SharpCorePDF.Infrastructure.Sources; namespace SharpCorePDF.Tests; @@ -144,6 +148,98 @@ public void Inflates_flate_data_however_it_was_written() Assert.Equal(text, raw); } + /// + /// A Flate stream that decodes past the limit is refused rather than inflated: zeros deflate about a + /// thousand to one, so the input limit alone lets a small document ask for a gigabyte. + /// + [Fact] + public void Refuses_a_flate_stream_that_decodes_past_the_limit() + { + var bomb = Deflate(new byte[2 * 1024 * 1024]); + + Assert.True(bomb.Length < 16 * 1024, "the fixture has to be small for the test to say anything"); + + var error = Assert.Throws(() => PdfFilters.Inflate(bomb, "bomb.pdf", 1024 * 1024)); + + Assert.Contains("MaxDecodedStreamBytes", error.Message, StringComparison.Ordinal); + Assert.Equal(2 * 1024 * 1024, PdfFilters.Inflate(bomb, "bomb.pdf", 2 * 1024 * 1024).Length); + Assert.True(PdfFilters.TryInflate(Deflate(new byte[1024]), out var small)); + Assert.Equal(1024, small.Length); + } + + /// An LZW stream is held to the same limit: one 12-bit code can stand for 4,096 bytes. + [Fact] + public void Refuses_an_lzw_stream_that_decodes_past_the_limit() + { + var encoded = TestLzw.Encode(new byte[100_000]); + + var error = Assert.Throws(() => PdfFilters.ApplyLzw(encoded, null, "lzw.pdf", 99_999)); + + Assert.Contains("MaxDecodedStreamBytes", error.Message, StringComparison.Ordinal); + Assert.Contains("LZW", error.Message, StringComparison.Ordinal); + Assert.Equal(100_000, PdfFilters.ApplyLzw(encoded, null, "lzw.pdf", 100_000).Length); + } + + /// A RunLength stream is held to the limit too: two bytes stand for 128. + [Fact] + public void Refuses_a_run_length_stream_that_decodes_past_the_limit() + { + var runs = Enumerable.Repeat(new byte[] { 0x81, 0x00 }, 1000).SelectMany(run => run).ToArray(); + + var error = Assert.Throws(() => PdfFilters.DecodeRunLength(runs, "runs.pdf", 127_999)); + + Assert.Contains("MaxDecodedStreamBytes", error.Message, StringComparison.Ordinal); + Assert.Contains("RunLength", error.Message, StringComparison.Ordinal); + Assert.Equal(128_000, PdfFilters.DecodeRunLength(runs, "runs.pdf", 128_000).Length); + } + + /// Literal runs are held to the limit on the same exact boundary as repeated ones. + [Fact] + public void Refuses_literal_runs_that_decode_past_the_limit() + { + byte[] run = [0x7F, .. Enumerable.Range(0, 128).Select(value => (byte)value)]; + var runs = Enumerable.Repeat(run, 100).SelectMany(bytes => bytes).ToArray(); + + var error = Assert.Throws(() => PdfFilters.DecodeRunLength(runs, "runs.pdf", 12_799)); + + Assert.Contains("RunLength", error.Message, StringComparison.Ordinal); + Assert.Equal(12_800, PdfFilters.DecodeRunLength(runs, "runs.pdf", 12_800).Length); + } + + /// + /// A page whose content stream decodes past the limit is reported as unreadable, the way any other + /// undecodable content is, and the read finishes: the limit turns a bomb into a warning. + /// + [Fact] + public async Task Reports_a_content_stream_past_the_limit_as_unreadable_instead_of_inflating_it() + { + var builder = new PdfBuilder(); + var catalog = builder.Reserve(); + var pages = builder.Reserve(); + + builder.Add("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Contents 4 0 R >>"); + builder.AddStream("/Filter /FlateDecode", PdfBuilder.Compress(new byte[1024 * 1024])); + builder.Set(catalog, "<< /Type /Catalog /Pages 2 0 R >>"); + builder.Set(pages, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + var document = builder.Build(catalog); + + var limited = new PdfContentReaderService( + new PdfPageContentReader(PdfEngineOptions.Default with { MaxDecodedStreamBytes = 64 * 1024 })); + var text = await limited.ReadTextAsync( + new PdfByteArraySource(document, "bomb.pdf"), + PageRange.All, + cancellationToken: TestContext.Current.CancellationToken); + + Assert.Contains(text.Warnings, warning => warning.Kind == PdfWarningKind.ContentStreamUnreadable); + + var unlimited = await new PdfContentReaderService(new PdfPageContentReader()).ReadTextAsync( + new PdfByteArraySource(document, "bomb.pdf"), + PageRange.All, + cancellationToken: TestContext.Current.CancellationToken); + + Assert.DoesNotContain(unlimited.Warnings, warning => warning.Kind == PdfWarningKind.ContentStreamUnreadable); + } + /// Flate data that cannot be decompressed is refused and reported as a damaged source. [Fact] public void Reports_data_that_cannot_be_decompressed() diff --git a/tests/SharpCorePDF.Tests/PdfJpegFilterTests.cs b/tests/SharpCorePDF.Tests/PdfJpegFilterTests.cs index 10c77a8..6db9f08 100644 --- a/tests/SharpCorePDF.Tests/PdfJpegFilterTests.cs +++ b/tests/SharpCorePDF.Tests/PdfJpegFilterTests.cs @@ -35,6 +35,96 @@ public sealed class PdfJpegFilterTests private static string Fixture(string name) => Path.Combine(TestAssets.Folder, "jpeg", name); + /// A budget of exactly the output's size keeps the pixels; one byte less measures them. + [Fact] + public void Keeps_pixels_that_fit_the_budget_exactly() + { + Assert.True(JpegDecoder.TryDecode(Rgb, 1 << 20, out var all, out _, out var width, out var height)); + + var exact = width * height * 3; + + Assert.Equal(exact, all!.Length); + Assert.True(JpegDecoder.TryDecode(Rgb, exact, out var kept, out _, out _, out _)); + Assert.Equal(all, kept); + Assert.True(JpegDecoder.TryDecode(Rgb, exact - 1, out var measured, out _, out _, out _)); + Assert.Null(measured); + } + + /// An image above the budget is measured with the format its components state. + /// The fixture. + /// The format it decodes to. + [Theory] + [InlineData("jpeg-progressive-gray.jpg", "Gray8")] + [InlineData("jpeg-cmyk.jpg", "Cmyk32")] + [InlineData("jpeg-444.jpg", "Rgb24")] + public void Measures_the_format_of_an_image_above_the_budget(string fixture, string expected) + { + Assert.True(JpegDecoder.TryDecode(File.ReadAllBytes(Fixture(fixture)), 16, out var pixels, out var format, out _, out _)); + Assert.Null(pixels); + Assert.Equal(expected, format); + } + + /// + /// A frame whose output fits a budget of while its coefficient grid does not fit + /// an array — 46,340 squared samples is just under the budget, at four bytes a coefficient it is eight + /// gigabytes — is refused before anything is allocated, instead of overflowing the size in 32 bits. + /// + [Fact] + public void Refuses_a_frame_whose_working_buffers_cannot_fit_an_array() + { + var jpeg = (byte[])ProgressiveGray.Clone(); + var frame = jpeg.AsSpan().IndexOf(new byte[] { 0xFF, 0xC2 }); + + Assert.True(frame > 0, "the fixture has to carry the frame header the test rewrites"); + + jpeg[frame + 5] = 46_340 >> 8; + jpeg[frame + 6] = 46_340 & 0xFF; + jpeg[frame + 7] = 46_340 >> 8; + jpeg[frame + 8] = 46_340 & 0xFF; + + var before = GC.GetAllocatedBytesForCurrentThread(); + var decoded = JpegDecoder.TryDecode(jpeg, int.MaxValue, out var pixels, out _, out _, out _); + var allocated = GC.GetAllocatedBytesForCurrentThread() - before; + + Assert.False(decoded); + Assert.Null(pixels); + Assert.True(allocated < 1 << 20, $"refusing the frame allocated {allocated} bytes"); + } + + /// + /// A frame header that claims 65,535 x 65,535 samples is measured from the header alone, with no pixels, + /// as any image above the budget is. The width and the height are two numbers in the file, and the + /// sample planes — four bytes a coefficient for a progressive image — were otherwise allocated before + /// the output was found too large to keep. + /// + /// Whether the progressive fixture is used rather than the baseline one. + [Theory] + [InlineData(false)] + [InlineData(true)] + public void Measures_a_frame_too_large_for_the_budget_without_allocating_it(bool progressive) + { + var jpeg = (byte[])(progressive ? Progressive : Rgb).Clone(); + var frame = jpeg.AsSpan().IndexOf(progressive ? [0xFF, 0xC2] : [0xFF, 0xC0]); + + Assert.True(frame > 0, "the fixture has to carry the frame header the test rewrites"); + + // Marker, length (two bytes), precision, then height and width, each a 16-bit big-endian number. + jpeg[frame + 5] = 0xFF; + jpeg[frame + 6] = 0xFF; + jpeg[frame + 7] = 0xFF; + jpeg[frame + 8] = 0xFF; + + var before = GC.GetAllocatedBytesForCurrentThread(); + var measured = JpegDecoder.TryDecode(jpeg, 1 << 20, out var pixels, out var format, out var width, out var height); + var allocated = GC.GetAllocatedBytesForCurrentThread() - before; + + Assert.True(measured); + Assert.Null(pixels); + Assert.Equal("Rgb24", format); + Assert.Equal((65_535, 65_535), (width, height)); + Assert.True(allocated < 1 << 20, $"refusing the frame allocated {allocated} bytes"); + } + /// A 4:4:4 RGB JPEG decodes to the bytes Pillow decodes from the same file. [Fact] public void Decodes_a_baseline_rgb_jpeg_within_two_levels() diff --git a/tests/SharpCorePDF.Tests/PdfReaderRecoveryTests.cs b/tests/SharpCorePDF.Tests/PdfReaderRecoveryTests.cs index fba32c8..da00411 100644 --- a/tests/SharpCorePDF.Tests/PdfReaderRecoveryTests.cs +++ b/tests/SharpCorePDF.Tests/PdfReaderRecoveryTests.cs @@ -18,6 +18,139 @@ namespace SharpCorePDF.Tests; /// public sealed class PdfReaderRecoveryTests { + /// + /// The bound on a chain of lengths leaves an ordinary indirect length alone, for every stream of a + /// document and not only the first ones: each stream's data holds an endstream line of its own, + /// so a length that was not resolved would cut the stream there instead of at its real end. + /// + [Fact] + public void Resolves_the_indirect_length_of_every_stream_of_a_document() + { + const int Streams = 100; + var data = "q\nendstream\nQ"u8.ToArray(); + var builder = new PdfBuilder(); + var catalog = builder.Reserve(); + var pages = builder.Reserve(); + + builder.Add("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Contents 4 0 R >>"); + + // Objects 4, 6, 8 ... are the streams; 5, 7, 9 ... are their lengths. + for (var stream = 0; stream < Streams; stream++) + { + var body = Encoding.Latin1.GetBytes($"<< /Length {5 + (2 * stream)} 0 R >>\nstream\n") + .Concat(data) + .Concat("\nendstream"u8.ToArray()) + .ToArray(); + + builder.Add(body); + builder.Add(data.Length.ToString(System.Globalization.CultureInfo.InvariantCulture)); + } + + builder.Set(catalog, "<< /Type /Catalog /Pages 2 0 R >>"); + builder.Set(pages, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + + var reader = new PdfFileReader(builder.Build(catalog), "lengths.pdf", PdfEngineOptions.Default); + + for (var stream = 0; stream < Streams; stream++) + { + var resolved = Assert.IsType(reader.Resolve(4 + (2 * stream))); + + Assert.Equal(data, resolved.Raw.ToArray()); + } + } + + /// + /// An object stream that declares 2^31 - 1 objects in a header of one pair is read for the one pair it + /// holds. /N sized two arrays before anything compared it with the stream, so a few hundred bytes asked + /// for gigabytes; the stream now bounds the count, and the page inside it is found. + /// + [Fact] + public void Reads_an_object_stream_whose_count_its_header_cannot_hold() + { + var page = "3 0 << /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Contents 4 0 R >>"u8.ToArray(); + var packed = PdfBuilder.Compress(page); + var output = new MemoryStream(); + var offsets = new Dictionary(); + + void Write(string text) => output.Write(Encoding.Latin1.GetBytes(text)); + + void Object(int number, string dictionary, byte[]? stream = null) + { + offsets[number] = output.Position; + Write($"{number} 0 obj\n{dictionary}\n"); + + if (stream is not null) + { + Write("stream\n"); + output.Write(stream); + Write("\nendstream\n"); + } + + Write("endobj\n"); + } + + Write("%PDF-1.7\n"); + Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); + Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + Object(4, "<< /Length 3 >>", "q Q"u8.ToArray()); + Object(5, $"<< /Type /ObjStm /N 2147483647 /First 4 /Filter /FlateDecode /Length {packed.Length} >>", packed); + + offsets[6] = output.Position; + (int Kind, long Field, int Index)[] entries = + [ + (0, 0, 65535), (1, offsets[1], 0), (1, offsets[2], 0), (2, 5, 0), + (1, offsets[4], 0), (1, offsets[5], 0), (1, offsets[6], 0), + ]; + var table = entries + .SelectMany(entry => new[] + { + (byte)entry.Kind, + (byte)(entry.Field >> 24), (byte)(entry.Field >> 16), (byte)(entry.Field >> 8), (byte)entry.Field, + (byte)(entry.Index >> 8), (byte)entry.Index, + }) + .ToArray(); + + Write($"6 0 obj\n<< /Type /XRef /Size 7 /W [1 4 2] /Root 1 0 R /Length {table.Length} >>\nstream\n"); + output.Write(table); + Write($"\nendstream\nendobj\nstartxref\n{offsets[6]}\n%%EOF\n"); + + var reader = new PdfFileReader(output.ToArray(), "objstm.pdf", PdfEngineOptions.Default); + + Assert.Single(reader.Pages); + Assert.Equal("Page", Assert.IsType(reader.Resolve(3)).GetName("Type")); + } + + /// + /// A stream whose /Length is another stream, whose length is another, sixty thousand deep, is + /// read by finding its endstream. Resolved link by link, the chain would overflow the stack, and a + /// stack overflow ends the process rather than throwing anything a caller could catch. + /// + [Fact] + public void Reads_a_stream_whose_length_is_a_chain_of_sixty_thousand_streams() + { + const int Chain = 60_000; + var builder = new PdfBuilder(); + var catalog = builder.Reserve(); + var pages = builder.Reserve(); + + builder.Add("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Contents 4 0 R >>"); + + for (var link = 0; link < Chain; link++) + { + builder.Add($"<< /Length {5 + link} 0 R >>\nstream\nq Q\nendstream"); + } + + builder.Add("3"); + builder.Set(catalog, "<< /Type /Catalog /Pages 2 0 R >>"); + builder.Set(pages, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + + var reader = new PdfFileReader(builder.Build(catalog), "chain.pdf", PdfEngineOptions.Default); + var content = Assert.IsType(reader.Resolve(4)); + + Assert.Equal("q Q", Encoding.Latin1.GetString(content.Raw.Span)); + Assert.Single(reader.Pages); + } + /// A table that no longer parses is rebuilt, and the document is read completely. [Fact] public async Task Rebuilds_a_cross_reference_table_that_does_not_parse() diff --git a/tests/SharpCorePDF.Tests/PdfSignatureAttackTests.cs b/tests/SharpCorePDF.Tests/PdfSignatureAttackTests.cs index 52957e9..bbedc1f 100644 --- a/tests/SharpCorePDF.Tests/PdfSignatureAttackTests.cs +++ b/tests/SharpCorePDF.Tests/PdfSignatureAttackTests.cs @@ -104,8 +104,7 @@ public async Task Never_reports_a_signature_whose_value_was_emptied_as_valid() /// what must fail is the claim that the signature covers the whole document, because part of the /// document is now something other than the signature value. /// - [Fact(Skip = - "Known defect: SignatureValidator does not check that the gap between the two ranges is exactly the /Contents string, so this wrapped document is reported as covering the whole file. Remove the skip with the fix.")] + [Fact] public async Task Does_not_report_a_gap_that_holds_more_than_the_signature_value_as_covering_the_document() { var signed = await SignAsync(); @@ -114,14 +113,76 @@ public async Task Does_not_report_a_gap_that_holds_more_than_the_signature_value // The attack only works while the signed bytes are unchanged; this is what makes it an attack. var validation = Assert.Single(await SigningFixtures.ValidateAsync(wrapped)); - Assert.False( - validation.Signer.CoversWholeDocument && validation.IsValid, - "a document whose uncovered gap carries more than the signature value must not be reported as covered"); + Assert.False(validation.Signer.CoversWholeDocument); + Assert.False(validation.IsValid); + Assert.NotEqual(PdfSignatureIntegrity.Valid, validation.Integrity); + + var (gapStart, _) = Gap(signed); + var problem = Assert.Single(validation.Problems, problem => problem.Kind == PdfSignatureProblemKind.MalformedByteRange); + + Assert.Contains(gapStart.ToString(CultureInfo.InvariantCulture), problem.Message, StringComparison.Ordinal); + + // The wrapped gap is the finding; the revision count would be a second, misleading one. + Assert.DoesNotContain(validation.Problems, problem => problem.Kind == PdfSignatureProblemKind.RevisionNotCovered); + } + + /// + /// A gap that is not exactly one hexadecimal string is refused whichever way it fails to be one: no + /// opening delimiter, no closing delimiter, or a byte inside it that is neither a digit nor whitespace. + /// + /// Which byte of the gap is changed. + [Theory] + [InlineData("opening-delimiter")] + [InlineData("closing-delimiter")] + [InlineData("not-a-hex-digit")] + public async Task Refuses_a_gap_that_is_not_one_hexadecimal_string(string variant) + { + var signed = await SignAsync(); + var (gapStart, gapEnd) = Gap(signed); + var changed = (byte[])signed.Clone(); + var (index, value) = variant switch + { + "opening-delimiter" => (gapStart, (byte)'('), + "closing-delimiter" => (gapEnd - 1, (byte)')'), + "not-a-hex-digit" => (gapEnd - 2, (byte)'g'), + _ => throw new ArgumentOutOfRangeException(nameof(variant), variant, "no such variant"), + }; + + changed[index] = value; + + var validation = Assert.Single(await SigningFixtures.ValidateAsync(changed)); + + Assert.False(validation.IsValid); + Assert.False(validation.Signer.CoversWholeDocument); + Assert.Contains(validation.Problems, problem => problem.Kind == PdfSignatureProblemKind.MalformedByteRange); + } + + /// + /// Whitespace inside the value is part of PDF's hexadecimal string syntax, and an odd number of digits is + /// completed with a zero; a gap written that way is still exactly the signature value and stays valid. + /// + [Fact] + public async Task Accepts_whitespace_and_an_odd_digit_count_inside_the_value() + { + var signed = await SignAsync(); + var (_, gapEnd) = Gap(signed); + var spaced = (byte[])signed.Clone(); + + // The last digit of the padding becomes a space: the value now has an odd number of digits, which a + // reader completes with the zero that was there. + Assert.Equal((byte)'0', spaced[gapEnd - 2]); + spaced[gapEnd - 2] = (byte)' '; + + var validation = Assert.Single(await SigningFixtures.ValidateAsync(spaced)); + + Assert.True(validation.IsValid); + Assert.True(validation.Signer.CoversWholeDocument); + Assert.Equal(PdfSignatureIntegrity.Valid, validation.Integrity); } /// - /// The precondition the skipped wrapping test depends on, asserted on its own so it keeps running: the - /// injection leaves every covered byte as it was, which is why only a gap check can catch it. + /// The precondition the wrapping test depends on: the injection leaves every covered byte as it was, which + /// is why only a gap check can catch it. /// [Fact] public async Task Leaves_every_covered_byte_unchanged_when_the_gap_is_wrapped() From 9e2ab83a9616915a9746852b7f1ec3aa7a3b6871 Mon Sep 17 00:00:00 2001 From: Andrew Tierney Date: Mon, 16 Nov 2026 12:26:01 +1100 Subject: [PATCH 04/11] Read the documents the JPEG, empty-name, glyph-box and date faults refused Four defects turned documents into exceptions that were not PdfExceptions, so a caller that handles the contract's refusal still saw the read fail: - A JPEG scan selected Huffman tables with 4-bit selectors and a progressive scan named its spectral selection, and both were used as indexes unchecked (IndexOutOfRangeException). ParseScan now refuses a selector above 3, and a progressive scan unless Ss <= Se <= 63 and Ah, Al <= 13. - The empty name, a solidus on its own, is a name PDF allows, and four documents of the pdf.js test corpus use it as a key; PdfDictionary.Set refused it, so they did not open at all. - A glyph drawn upside down (a negative font size) gave PdfGlyph.BoundingBox a bottom above its top, and the rectangle refused it, failing text extraction for the page. - PdfDate.TryParse threw for a time-zone offset beyond fourteen hours, so one malformed /CreationDate failed the inspection of the whole document; such an offset is now read as no offset. PdfContainerTests.Refuses_an_entry_without_a_key asserted the second defect and is rewritten: a null key is still refused, the empty name is kept. Tests that fail without the fixes: PdfJpegFilterTests (table selectors, progressive scan bounds), PdfObjectModelTests (empty-name key, in a dictionary and in a document), PdfTextScannerTests (upside-down glyph), PdfSerializationTests (offsets past fourteen hours). The pdf.js corpus and the hostile documents now read with no fault, crash or timeout, and their --known flags are removed. --- .github/workflows/robustness.yml | 5 -- .github/workflows/validation.yml | 7 +- CHANGELOG.md | 14 +++- docs/validation.md | 11 +-- .../ReadModels/PdfGlyph.cs | 8 +- .../Pdf/JpegDecoder.cs | 13 ++- .../Pdf/PdfDate.cs | 8 +- .../Pdf/PdfDictionary.cs | 4 +- tests/SharpCorePDF.Tests/PdfContainerTests.cs | 15 +++- .../SharpCorePDF.Tests/PdfJpegFilterTests.cs | 80 +++++++++++++++++++ .../SharpCorePDF.Tests/PdfObjectModelTests.cs | 40 ++++++++++ .../PdfSerializationTests.cs | 22 +++++ .../SharpCorePDF.Tests/PdfTextScannerTests.cs | 25 ++++++ 13 files changed, 224 insertions(+), 28 deletions(-) diff --git a/.github/workflows/robustness.yml b/.github/workflows/robustness.yml index 93de31b..50854f4 100644 --- a/.github/workflows/robustness.yml +++ b/.github/workflows/robustness.yml @@ -162,16 +162,11 @@ jobs: - name: Build the probe run: dotnet build ./tools/SharpCorePDF.Validation -c Release - # The six known files fault on three recorded defects (docs/validation.md): an empty name as a - # dictionary key (issue3188, issue5896, pdfjs_wikipedia, poppler-742-0-fuzzed), a glyph whose box is - # upside down (issue14415) and a date whose offset is beyond 14 hours (TAMReview). - name: Read every document run: | python3 docs/tools/corpus-robustness.py \ --probe ./tools/SharpCorePDF.Validation/bin/Release/net11.0/SharpCorePDF.Validation.dll \ --timeout 60 --heap-limit-mb 1024 --report artifacts/corpora.json \ - --known issue3188.pdf --known issue5896.pdf --known pdfjs_wikipedia.pdf \ - --known poppler-742-0-fuzzed.pdf --known issue14415.pdf --known TAMReview.pdf \ "$RUNNER_TEMP/verapdf-corpus" "$RUNNER_TEMP/pdfjs/test/pdfs" - name: Upload the report diff --git a/.github/workflows/validation.yml b/.github/workflows/validation.yml index de42cbe..a632011 100644 --- a/.github/workflows/validation.yml +++ b/.github/workflows/validation.yml @@ -146,9 +146,9 @@ jobs: dotnet "$tool" replay "$target" "$seeds/$target" done - # Each of the four documents reaches a failure class the review named. flate-bomb, length-chain and - # objstm-count are refused or read since the decode limit, the length-chain bound and the /N bound; - # jpeg-selector is a known defect until the change that fixes it removes its --known line. + # Each of the four documents reaches a failure class a review named, and each is now read: the decode + # limit, the length-chain bound, the /N bound and the JPEG scan-header checks closed them. A document + # that faults here again is a regression, so none is listed as known. - name: Hostile documents run: | set -euo pipefail @@ -156,7 +156,6 @@ jobs: python docs/tools/corpus-robustness.py \ --probe ./tools/SharpCorePDF.Validation/bin/Release/net11.0/SharpCorePDF.Validation.dll \ --timeout 120 --report artifacts/hostile.json \ - --known jpeg-selector.pdf \ "$RUNNER_TEMP/hostile" - name: Upload the documents and the reports diff --git a/CHANGELOG.md b/CHANGELOG.md index abe4bd9..01e5662 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,6 +55,16 @@ against an empty feed. is measured from its header without being decoded — the same answer as before, without the allocation — and the working sizes are computed in 64 bits. +- **A JPEG scan header is checked before it is used.** A Huffman table selector above 3, and a progressive + scan whose spectral selection runs past coefficient 63 or starts after it ends, indexed past the decoder's + tables (`IndexOutOfRangeException`); such a scan is now refused and the image reported undecodable. +- **The empty name is a dictionary key like any other.** `/` on its own is a name PDF allows; four documents + of the pdf.js test corpus use it as a key and did not open at all (`ArgumentException`). +- **A glyph drawn upside down has a box.** A negative font size made `PdfGlyph.BoundingBox` throw, so text + extraction faulted on the page. +- **`PdfDate.TryParse` answers instead of throwing** for a time-zone offset beyond fourteen hours, which it + now reads as no offset; one malformed `/CreationDate` no longer fails the inspection of the document. + ### Known defects, found by the new checks and not yet fixed Each is reproduced, pinned by a skipped test or a strict `--known-defect` flag that fails the run once the @@ -63,10 +73,6 @@ defect is fixed, and listed in [`docs/validation.md`](docs/validation.md#known-d - An incremental update rewrites both elements of `/ID`, and appends a renumbered copy of the whole document rather than the objects it changed — which also rewrites the signature dictionary after signing. - A merge copies a page that states no `/MediaBox` without one. -- Hostile input: a JPEG scan's Huffman table selector is not range-checked. -- Real-world input (the pdf.js test corpus): an empty name as a dictionary key stops a document opening, a - glyph with an upside-down box faults text extraction, and `PdfDate.TryParse` throws on an out-of-range - time-zone offset instead of answering false. - **Run-length images are decoded.** `RunLengthDecode` joins Flate, LZW and the two ASCII filters: a bilevel or scan-style image that declares it now yields its samples — and `Gray8`, `Rgb24` or `Cmyk32` diff --git a/docs/validation.md b/docs/validation.md index 6ce4585..1d79368 100644 --- a/docs/validation.md +++ b/docs/validation.md @@ -86,10 +86,6 @@ fix. This is the list on 2026-11-16, every entry reproduced, not inferred: | An incremental update writes a new identifier into both elements of `/ID`; ISO 32000-1 §14.4 keeps the first | #10 | `PdfIncrementalInvariantTests.Keeps_the_first_identifier_element_across_an_incremental_update` (skipped); `validate-incremental.py` `id-first` | | `UseIncrementalUpdate` appends a renumbered copy of the whole document rather than the objects it changed: every byte of the earlier revision is kept, but the file grows by its own size per edit, and the signature dictionary is written again after signing, so pyHanko places a signature followed by an edit as `CONTIGUOUS_BLOCK_FROM_START` with modification level `OTHER` instead of `ENTIRE_REVISION` | #10, #3 | `validate-incremental.py` `appends-only`; `validate-signatures.py` on `signed-then-edited.pdf` | | A merge copies a page that states no `/MediaBox` (and inherits none) without one, so the result breaks the rule that every page has a media box; qpdf repairs it by assuming Letter | #2 | `validate-pdf.py` on `inspected.pdf` and `with-warnings.pdf` (both merge a page of `NamedDests.pdf`) | -| The JPEG decoder indexes its four Huffman tables with a scan's 4-bit selector unchecked | #12, #1 | `jpeg-selector.pdf`: `IndexOutOfRangeException` from `JpegDecoder.DecodeBlock` | -| A dictionary key that is the empty name (`/` alone, which PDF syntax allows) makes `PdfDictionary.Set` throw `ArgumentException` while parsing, so the document does not open at all | #12 | pdf.js corpus: `issue3188.pdf`, `issue5896.pdf`, `pdfjs_wikipedia.pdf`, `poppler-742-0-fuzzed.pdf` | -| A glyph whose box comes out upside down (a negative font size or a flipped font matrix) makes `PdfGlyph.BoundingBox` construct a `PdfRectangle` that throws, so text extraction faults | #12 | pdf.js corpus: `issue14415.pdf` | -| `PdfDate.TryParse` throws `ArgumentOutOfRangeException` for a time-zone offset beyond 14 hours instead of answering false, so one malformed `/CreationDate` faults the whole inspection | #12 | pdf.js corpus: `TAMReview.pdf` | Fixed since the list was first drawn up, each with the test that fails without its fix: @@ -99,6 +95,10 @@ Fixed since the list was first drawn up, each with the test that fails without i | No limit on the decoded size of a Flate, LZW or RunLength stream | `PdfEngineOptions.MaxDecodedStreamBytes` (128 MiB) | `PdfFilterTests.Refuses_a_flate_stream_that_decodes_past_the_limit` and its LZW, RunLength and content-stream companions | | A chain of indirect `/Length` references overflowed the stack | the chain is followed no deeper than `MaxNestingDepth`, then the stream is found by its `endstream` | `PdfReaderRecoveryTests.Reads_a_stream_whose_length_is_a_chain_of_sixty_thousand_streams` | | An object stream's `/N`, and a JPEG frame's width and height, sized allocations unchecked | `/N` is bounded by what the stream can hold; a JPEG above the caller's budget is measured from its header, and working sizes are computed in 64 bits | `PdfReaderRecoveryTests.Reads_an_object_stream_whose_count_its_header_cannot_hold`, `PdfJpegFilterTests.Measures_a_frame_too_large_for_the_budget_without_allocating_it` | +| The JPEG decoder used a scan's 4-bit Huffman table selectors, and a progressive scan's spectral end, as indexes unchecked | `ParseScan` refuses a selector above 3, and a progressive scan whose selection is not `Ss ≤ Se ≤ 63` or whose approximation is above 13 | `PdfJpegFilterTests.Refuses_a_scan_that_selects_a_table_beyond_the_four_there_are`, `Refuses_a_progressive_scan_outside_the_block` | +| The empty name as a dictionary key (`/` alone, which PDF allows) stopped the document opening | `PdfDictionary.Set` accepts it; the entry is kept and written back as `/` | `PdfObjectModelTests.Keeps_an_entry_whose_key_is_the_empty_name`, `Opens_a_document_that_uses_the_empty_name_as_a_key` | +| A glyph drawn upside down (negative font size) faulted text extraction | `PdfGlyph.BoundingBox` takes the box between descender and ascender whichever way round | `PdfTextScannerTests.Gives_a_glyph_drawn_upside_down_a_box` | +| `PdfDate.TryParse` threw for a time-zone offset beyond 14 hours | an offset no time zone has is read as no offset | `PdfSerializationTests.Reads_an_offset_no_time_zone_has_as_no_offset` | ## Corpus readings, 2026-11-16 @@ -108,7 +108,8 @@ Fixed since the list was first drawn up, each with the test that fails without i | pdf.js `test/pdfs` at `v6.4.299` | 987 | 977 | 4 | 6 | 0 | 0 | | synthesized hostile documents, before the fixes | 4 | 0 | 0 | 3 | 1 | 0 | -The six pdf.js faults are the three defects above. A sample of the pre-flight measurement (the first 60 +The six pdf.js faults were the empty-name, upside-down glyph and date-offset defects in the fixed table; +after those fixes all six are read, as is `jpeg-selector.pdf`. A sample of the pre-flight measurement (the first 60 files of `PDF_A-2b` and of `PDF_A-4`) wrote 108 results, refused 9 non-conforming inputs and refused 3 that already conformed; whether the 108 conform is veraPDF's to say, which `robustness.yml` asks. diff --git a/src/SharpCorePDF.Domain/ReadModels/PdfGlyph.cs b/src/SharpCorePDF.Domain/ReadModels/PdfGlyph.cs index eb0ea41..f3106d9 100644 --- a/src/SharpCorePDF.Domain/ReadModels/PdfGlyph.cs +++ b/src/SharpCorePDF.Domain/ReadModels/PdfGlyph.cs @@ -70,13 +70,15 @@ public sealed record PdfGlyph /// /// Gets the area the glyph occupies, derived from the origin, the advance and the em box. The /// descender is taken as a fifth of the em and the ascender as four fifths, which is the proportion - /// most Latin fonts share and is stated here rather than read from the font program. + /// most Latin fonts share and is stated here rather than read from the font program. A negative font + /// size — a glyph drawn upside down, which a flipped text matrix or font matrix produces — puts the + /// descender above the ascender, so the box is taken between the two whichever way round they are. /// public PdfRectangle BoundingBox => new( X, - Y - (FontSize * 0.2d), + Math.Min(Y - (FontSize * 0.2d), Y + (FontSize * 0.8d)), X + Math.Max(0d, Width), - Y + (FontSize * 0.8d)); + Math.Max(Y - (FontSize * 0.2d), Y + (FontSize * 0.8d))); /// Gets a value indicating whether the glyph paints no ink, such as a space or a tab. public bool IsWhitespace => string.IsNullOrWhiteSpace(Text); diff --git a/src/SharpCorePDF.Infrastructure/Pdf/JpegDecoder.cs b/src/SharpCorePDF.Infrastructure/Pdf/JpegDecoder.cs index 528cf69..8919615 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/JpegDecoder.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/JpegDecoder.cs @@ -473,6 +473,13 @@ private bool ParseScan(ReadOnlySpan segment) return false; } + // A selector is four bits in the file and the decoder has four tables of each class: a selector + // above 3 would index past them, so it is a header this decoder cannot honour. + if (tables >> 4 > 3 || (tables & 15) > 3) + { + return false; + } + component.DcTable = tables >> 4; component.AcTable = tables & 15; named[index] = component; @@ -486,7 +493,11 @@ private bool ParseScan(ReadOnlySpan segment) scanHigh = segment[tail + 2] >> 4; scanLow = segment[tail + 2] & 15; - return true; + // The spectral selection names coefficients 0 to 63 of a block, start before end, and the successive + // approximation shifts no further than a 16-bit coefficient allows. A progressive scan indexes the + // zig-zag table with them, so a scan outside these bounds is refused rather than read past the + // table; a sequential scan states them too but decodes the whole block whatever they say. + return !progressive || (scanStart <= scanEnd && scanEnd <= 63 && scanHigh <= 13 && scanLow <= 13); } private void ParseAdobe(ReadOnlySpan segment) diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfDate.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfDate.cs index f240e68..80d8ce7 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfDate.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfDate.cs @@ -133,8 +133,12 @@ private static TimeSpan ReadOffset(ReadOnlySpan rest) field++; } - var sign = rest[0] == '-' ? -1 : 1; - return sign * new TimeSpan(Math.Clamp(hours, 0, 23), Math.Clamp(minutes, 0, 59), 0); + // DateTimeOffset accepts offsets up to fourteen hours either way and throws beyond them, so an offset + // no time zone has is read as no offset — UTC, which is what the specification assumes when none is + // given — instead of failing the inspection of the whole document over one metadata string. + var offset = (rest[0] == '-' ? -1 : 1) * new TimeSpan(Math.Clamp(hours, 0, 23), Math.Clamp(minutes, 0, 59), 0); + + return offset.Duration() > TimeSpan.FromHours(14) ? TimeSpan.Zero : offset; } /// Reads the next run of digits of a time zone offset, skipping what is not a digit. diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfDictionary.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfDictionary.cs index b877b2b..72643b5 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfDictionary.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfDictionary.cs @@ -54,7 +54,9 @@ public IEnumerable> Entries /// This dictionary, so calls can be chained. public PdfDictionary Set(string key, PdfObject? value) { - ArgumentException.ThrowIfNullOrEmpty(key); + // The empty name — a solidus on its own — is a name PDF allows (ISO 32000-1 §7.3.5), and documents + // use it as a key; refusing it here refused the whole document while it was being parsed. + ArgumentNullException.ThrowIfNull(key); if (value is null) { diff --git a/tests/SharpCorePDF.Tests/PdfContainerTests.cs b/tests/SharpCorePDF.Tests/PdfContainerTests.cs index b402acd..e562066 100644 --- a/tests/SharpCorePDF.Tests/PdfContainerTests.cs +++ b/tests/SharpCorePDF.Tests/PdfContainerTests.cs @@ -72,10 +72,19 @@ public void Copies_a_set_of_entries() Assert.Same(PdfNull.Instance, dictionary.Get("B")); } - /// An entry without a key is a programmer error. + /// + /// An entry without a key is a programmer error, but the empty name is a key: a solidus on its own is a + /// name PDF allows (ISO 32000-1 §7.3.5), and documents in the wild use it. + /// [Fact] - public void Refuses_an_entry_without_a_key() => - Assert.Throws(() => new PdfDictionary().Set(string.Empty, PdfNull.Instance)); + public void Refuses_an_entry_without_a_key_and_keeps_one_whose_key_is_the_empty_name() + { + Assert.Throws(() => new PdfDictionary().Set(null!, PdfNull.Instance)); + + var dictionary = new PdfDictionary().Set(string.Empty, new PdfInteger(1)); + + Assert.Equal(1L, dictionary.GetInteger(string.Empty)); + } /// The readers of a dictionary answer with the type they were asked for, or with nothing. [Fact] diff --git a/tests/SharpCorePDF.Tests/PdfJpegFilterTests.cs b/tests/SharpCorePDF.Tests/PdfJpegFilterTests.cs index 6db9f08..4a6644c 100644 --- a/tests/SharpCorePDF.Tests/PdfJpegFilterTests.cs +++ b/tests/SharpCorePDF.Tests/PdfJpegFilterTests.cs @@ -35,6 +35,86 @@ public sealed class PdfJpegFilterTests private static string Fixture(string name) => Path.Combine(TestAssets.Folder, "jpeg", name); + /// + /// A scan that selects a Huffman table the decoder does not have — the selector is four bits, the tables + /// number four of each class — is refused, not used as an index past them. + /// + /// The table selector byte of the first scan component: DC in the high nibble, AC in the low. + [Theory] + [InlineData(0x44)] + [InlineData(0x40)] + [InlineData(0x04)] + [InlineData(0xFF)] + public void Refuses_a_scan_that_selects_a_table_beyond_the_four_there_are(byte selectors) + { + var jpeg = (byte[])Rgb.Clone(); + var scan = jpeg.AsSpan().IndexOf(new byte[] { 0xFF, 0xDA }); + + Assert.True(scan > 0, "the fixture has to carry a scan header"); + + // Marker, length (two bytes), component count, then the first component's id and its selectors. + jpeg[scan + 6] = selectors; + + Assert.False(JpegDecoder.TryDecode(jpeg, 1 << 20, out var pixels, out _, out _, out _)); + Assert.Null(pixels); + } + + /// + /// A progressive scan whose spectral selection runs past coefficient 63, starts after it ends, or states a + /// successive approximation no 16-bit coefficient has is refused before it indexes the zig-zag table. + /// + /// Which field of the AC scan header is rewritten. + /// The value it is rewritten to. + [Theory] + [InlineData("end", 0xFF)] + [InlineData("end", 64)] + [InlineData("start-after-end", 0)] + [InlineData("approximation", 0xE0)] + [InlineData("approximation", 0x0E)] + public void Refuses_a_progressive_scan_outside_the_block(string field, int value) + { + var jpeg = (byte[])Progressive.Clone(); + var position = 0; + var tail = -1; + + // The first AC scan: a scan header whose spectral start is above zero. + while (tail < 0) + { + var scan = jpeg.AsSpan(position).IndexOf(new byte[] { 0xFF, 0xDA }); + + Assert.True(scan >= 0, "the fixture has to carry an AC scan"); + scan += position; + + var candidate = scan + 5 + (2 * jpeg[scan + 4]); + + if (jpeg[candidate] > 0) + { + tail = candidate; + } + + position = scan + 2; + } + + switch (field) + { + case "end": + jpeg[tail + 1] = (byte)value; + break; + + case "start-after-end": + jpeg[tail + 1] = (byte)value; + Assert.True(jpeg[tail] > value, "the scan has to start after the end it is given"); + break; + + default: + jpeg[tail + 2] = (byte)value; + break; + } + + Assert.False(JpegDecoder.TryDecode(jpeg, 1 << 20, out var pixels, out _, out _, out _)); + Assert.Null(pixels); + } + /// A budget of exactly the output's size keeps the pixels; one byte less measures them. [Fact] public void Keeps_pixels_that_fit_the_budget_exactly() diff --git a/tests/SharpCorePDF.Tests/PdfObjectModelTests.cs b/tests/SharpCorePDF.Tests/PdfObjectModelTests.cs index a520044..d59b80e 100644 --- a/tests/SharpCorePDF.Tests/PdfObjectModelTests.cs +++ b/tests/SharpCorePDF.Tests/PdfObjectModelTests.cs @@ -13,6 +13,46 @@ namespace SharpCorePDF.Tests; /// public sealed class PdfObjectModelTests { + /// + /// The empty name — a solidus on its own — is a key like any other: the dictionary keeps it and every + /// entry after it, and writing the dictionary back states it as a solidus again. + /// + [Fact] + public void Keeps_an_entry_whose_key_is_the_empty_name() + { + var parser = new PdfObjectParser(Encoding.Latin1.GetBytes("<< /A 1 / 2 /B 3 >>"), "empty-name.pdf", 0, null, 64); + var dictionary = Assert.IsType(parser.ParseObject()); + + Assert.Equal(1L, dictionary.GetInteger("A")); + Assert.Equal(2L, dictionary.GetInteger(string.Empty)); + Assert.Equal(3L, dictionary.GetInteger("B")); + + using var stream = new MemoryStream(); + var output = new PdfOutputBuffer(stream); + PdfSerializer.WriteDictionary(dictionary, output); + output.Flush(); + var written = Encoding.Latin1.GetString(stream.ToArray()); + + Assert.Contains("/ 2", written, StringComparison.Ordinal); + } + + /// A document whose page dictionary carries an empty-named entry opens and lists its page. + [Fact] + public void Opens_a_document_that_uses_the_empty_name_as_a_key() + { + var builder = new PdfBuilder(); + var catalog = builder.Reserve(); + var pages = builder.Reserve(); + + builder.Add("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] / (empty) >>"); + builder.Set(catalog, "<< /Type /Catalog /Pages 2 0 R / 1 >>"); + builder.Set(pages, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + + var reader = new PdfFileReader(builder.Build(catalog), "empty-name.pdf", SharpCorePDF.Infrastructure.PdfEngineOptions.Default); + + Assert.Single(reader.Pages); + } + /// The null object is one instance and writes itself as its keyword. [Fact] public void Exposes_the_null_object() diff --git a/tests/SharpCorePDF.Tests/PdfSerializationTests.cs b/tests/SharpCorePDF.Tests/PdfSerializationTests.cs index a3bf372..3ae3690 100644 --- a/tests/SharpCorePDF.Tests/PdfSerializationTests.cs +++ b/tests/SharpCorePDF.Tests/PdfSerializationTests.cs @@ -78,6 +78,28 @@ public void Writes_a_date_that_round_trips() Assert.Equal("D:20260401120000Z", PdfDate.Format(new DateTimeOffset(2026, 4, 1, 12, 0, 0, TimeSpan.Zero))); } + /// + /// An offset no time zone has — past fourteen hours either way, which refuses + /// to hold — is read as no offset rather than thrown, so one malformed metadata date does not fail the + /// inspection of the document it is in. The offsets at the edge of the range are kept. + /// + /// The date as a document states it. + /// The offset that is read. + [Theory] + [InlineData("D:20260101000000+99'00'", 0)] + [InlineData("D:20260101000000+23'59'", 0)] + [InlineData("D:20260101000000+14'30'", 0)] + [InlineData("D:20260101000000-15'00'", 0)] + [InlineData("D:20260101000000+14'00'", 14 * 60)] + [InlineData("D:20260101000000-14'00'", -14 * 60)] + [InlineData("D:20260101000000+05'30'", 5 * 60 + 30)] + public void Reads_an_offset_no_time_zone_has_as_no_offset(string text, int offsetMinutes) + { + Assert.True(PdfDate.TryParse(text, out var value)); + Assert.Equal(TimeSpan.FromMinutes(offsetMinutes), value.Offset); + Assert.Equal(new DateTime(2026, 1, 1, 0, 0, 0), value.DateTime); + } + /// Text that is not a date at all is rejected instead of being invented. [Theory] [InlineData("")] diff --git a/tests/SharpCorePDF.Tests/PdfTextScannerTests.cs b/tests/SharpCorePDF.Tests/PdfTextScannerTests.cs index 1811665..802c05a 100644 --- a/tests/SharpCorePDF.Tests/PdfTextScannerTests.cs +++ b/tests/SharpCorePDF.Tests/PdfTextScannerTests.cs @@ -15,6 +15,31 @@ namespace SharpCorePDF.Tests; /// public sealed class PdfTextScannerTests { + /// + /// A glyph drawn upside down — a negative font size, which a flipped text or font matrix produces — has a + /// box all the same: its descender is above its ascender, and the box is taken between them. + /// + [Fact] + public void Gives_a_glyph_drawn_upside_down_a_box() + { + var glyph = new SharpCorePDF.Domain.ReadModels.PdfGlyph + { + Text = "A", + PageNumber = 1, + X = 10, + Y = 100, + Width = 6, + FontSize = -10, + }; + + var box = glyph.BoundingBox; + + Assert.Equal(92d, box.Bottom, 6); + Assert.Equal(102d, box.Top, 6); + Assert.Equal(10d, box.Left, 6); + Assert.Equal(16d, box.Right, 6); + } + /// A string the scanner can take as a slice of the stream. /// The content stream. /// The text the shown string must decode to. From 74a4d7397d0bfc73fccf2c084be11bd79b6e5198 Mon Sep 17 00:00:00 2001 From: Andrew Tierney Date: Mon, 16 Nov 2026 12:39:33 +1100 Subject: [PATCH 05/11] Append only what an incremental update changes, and keep the permanent identifier UseIncrementalUpdate ran the full rewrite into an appended writer, so every object of the document, the signature dictionary of a signed source included, was written again under a new number. Every earlier byte was kept, but the file grew by its own size per edit, and validators that look at what changed after a signature (pyHanko, Acrobat) saw the signed field as rewritten: pyHanko placed a signature followed by an edit as CONTIGUOUS_BLOCK_FROM_START instead of ENTIRE_REVISION. In an appended revision the edited document's copier now keeps the numbers of its objects, the catalog, the page tree, the pages and the information dictionary are written under the numbers they already have, and the writer leaves out an object that would be written exactly as the source states it. Rotating a page of a signed document now appends 716 bytes and three objects instead of 20,969 bytes and twelve; the signature dictionary is not in the revision. The trailer also kept neither half of the source's /ID. ISO 32000-1 14.4 makes the first element permanent across incremental updates; it is now the source's, and only the second element is new. Tests that fail without the change: PdfIncrementalInvariantTests (the identifier, an update that appends only what it changed, a signature dictionary left out of the revision); the corpus checks validate-incremental.py and validate-signatures.py now pass with no --known-defect flag. The manual's editing chapter described every edit as a complete rewrite and now says what the option does. --- .github/workflows/validation.yml | 16 +--- CHANGELOG.md | 11 ++- docs/manual/src/14-editing.md | 12 ++- docs/validation.md | 8 +- .../Pdf/PdfObjectCopier.cs | 10 +- .../Pdf/PdfWriter.cs | 20 ++++ .../PdfEditEngine.Document.cs | 15 ++- .../PdfEditEngine.Incremental.cs | 46 +++++++++ .../PdfEditEngine.Pages.cs | 11 ++- .../PdfEditEngine.Sources.cs | 10 +- .../PdfEditEngine.cs | 50 ++++++++-- .../PdfIncrementalInvariantTests.cs | 95 ++++++++++++++++++- 12 files changed, 267 insertions(+), 37 deletions(-) diff --git a/.github/workflows/validation.yml b/.github/workflows/validation.yml index a632011..681f1fd 100644 --- a/.github/workflows/validation.yml +++ b/.github/workflows/validation.yml @@ -21,8 +21,7 @@ # # --known-defect / --known names a failure that is recorded and not yet fixed. Each is strict: when the # defect is fixed the check passes, the flag no longer matches, and the run fails until the flag is removed -# in the same change as the fix. The in-suite counterparts are the skipped tests in -# PdfSignatureAttackTests and PdfIncrementalInvariantTests. +# in the same change as the fix. # # The validator versions are pinned in docs/tools/validation-requirements.txt, pdfjs-dist below. @@ -119,20 +118,11 @@ jobs: "$RUNNER_TEMP/corpus/text-merged.pdf" "$RUNNER_TEMP/corpus/incremental.pdf" \ "$RUNNER_TEMP/corpus/signed.pdf" - # id-first: the editor writes a new identifier into both elements of /ID on an appended revision. - # appends-only: the editor's incremental update appends a renumbered copy of the whole document - # rather than the objects it changed. - name: Incremental update invariants - run: | - python docs/tools/validate-incremental.py "$RUNNER_TEMP/corpus" \ - --known-defect id-first --known-defect appends-only + run: python docs/tools/validate-incremental.py "$RUNNER_TEMP/corpus" - # The same copy rewrites the signature dictionary after signing, which is why pyHanko cannot place the - # first signature of signed-then-edited.pdf at the end of a revision. - name: Signatures, pyHanko - run: | - python docs/tools/validate-signatures.py --manifest "$RUNNER_TEMP/corpus/manifest.json" \ - --known-defect signed-then-edited.pdf:FirstSignature + run: python docs/tools/validate-signatures.py --manifest "$RUNNER_TEMP/corpus/manifest.json" - name: Fuzz targets over their seeds run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 01e5662..caf759e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -65,13 +65,20 @@ against an empty feed. - **`PdfDate.TryParse` answers instead of throwing** for a time-zone offset beyond fourteen hours, which it now reads as no offset; one malformed `/CreationDate` no longer fails the inspection of the document. +- **An incremental update appends what changed, and only that.** `UseIncrementalUpdate` appended a copy of + the whole document under new object numbers: the file grew by its own size per edit, and the signature + dictionary of a signed source was written again after signing, so validators that look at what changed + after a signature (pyHanko, Acrobat) saw the signed field as modified. The edited document's objects now + keep their numbers and an object written exactly as the source already states it is left out — rotating a + page of a signed document appends 716 bytes and three objects instead of 20,969 bytes and twelve. +- **An incremental update keeps the permanent identifier.** The first element of `/ID` is the source's in + every revision, as ISO 32000-1 §14.4 requires; only the second element is new. + ### Known defects, found by the new checks and not yet fixed Each is reproduced, pinned by a skipped test or a strict `--known-defect` flag that fails the run once the defect is fixed, and listed in [`docs/validation.md`](docs/validation.md#known-defects). -- An incremental update rewrites both elements of `/ID`, and appends a renumbered copy of the whole document - rather than the objects it changed — which also rewrites the signature dictionary after signing. - A merge copies a page that states no `/MediaBox` without one. - **Run-length images are decoded.** `RunLengthDecode` joins Flate, LZW and the two ASCII filters: a diff --git a/docs/manual/src/14-editing.md b/docs/manual/src/14-editing.md index 13f9bb8..11d036a 100644 --- a/docs/manual/src/14-editing.md +++ b/docs/manual/src/14-editing.md @@ -251,10 +251,18 @@ change the bytes of the content streams they show. The source can be encrypted a the `password` argument of `ApplyAsync`; the result is written in the clear, because output encryption is not part of this editing release. -The writer produces a complete new document rather than an incremental update. That is what keeps -earlier revisions and signatures out of a document whose pages have changed; it also means a copied +By default the writer produces a complete new document rather than an incremental update. That is what +keeps earlier revisions and signatures out of a document whose pages have changed; it also means a copied signature is not a signature of the edited document. +With `PdfEditOutputOptions.UseIncrementalUpdate` the edit is appended to the source instead: every byte of +the source stays where it was, and the revision carries only the objects the edit changed, under the numbers +they already had — rotating one page redeclares that page and the few objects around it, and an object that +would be written exactly as the source states it is left out. A signature over the source therefore still +covers the revision it was made over, and a validator that looks at what changed after signing sees the +edit and not a second copy of the document. The first element of the trailer `/ID` is kept, as ISO 32000-1 +§14.4 requires. + ## Reading the result `ApplyAsync` returns a `PdfEditResult`: diff --git a/docs/validation.md b/docs/validation.md index 1d79368..47882c2 100644 --- a/docs/validation.md +++ b/docs/validation.md @@ -56,8 +56,8 @@ dotnet run --project .\tools\SharpCorePDF.Validation -c Release -- corpus $env:T python docs/tools/validate-text.py $env:TEMP\corpus python docs/tools/validate-encryption.py $env:TEMP\corpus -python docs/tools/validate-incremental.py $env:TEMP\corpus --known-defect id-first --known-defect appends-only -python docs/tools/validate-signatures.py --manifest $env:TEMP\corpus\manifest.json --known-defect signed-then-edited.pdf:FirstSignature +python docs/tools/validate-incremental.py $env:TEMP\corpus +python docs/tools/validate-signatures.py --manifest $env:TEMP\corpus\manifest.json python docs/tools/validate-fonts.py $env:TEMP\SharpCorePDF.Examples\output python docs/tools/render-compare.py --exclude protected.pdf $env:TEMP\SharpCorePDF.Examples\output @@ -83,8 +83,6 @@ fix. This is the list on 2026-11-16, every entry reproduced, not inferred: | Defect | Found by | Evidence | | --- | --- | --- | -| An incremental update writes a new identifier into both elements of `/ID`; ISO 32000-1 §14.4 keeps the first | #10 | `PdfIncrementalInvariantTests.Keeps_the_first_identifier_element_across_an_incremental_update` (skipped); `validate-incremental.py` `id-first` | -| `UseIncrementalUpdate` appends a renumbered copy of the whole document rather than the objects it changed: every byte of the earlier revision is kept, but the file grows by its own size per edit, and the signature dictionary is written again after signing, so pyHanko places a signature followed by an edit as `CONTIGUOUS_BLOCK_FROM_START` with modification level `OTHER` instead of `ENTIRE_REVISION` | #10, #3 | `validate-incremental.py` `appends-only`; `validate-signatures.py` on `signed-then-edited.pdf` | | A merge copies a page that states no `/MediaBox` (and inherits none) without one, so the result breaks the rule that every page has a media box; qpdf repairs it by assuming Letter | #2 | `validate-pdf.py` on `inspected.pdf` and `with-warnings.pdf` (both merge a page of `NamedDests.pdf`) | Fixed since the list was first drawn up, each with the test that fails without its fix: @@ -99,6 +97,8 @@ Fixed since the list was first drawn up, each with the test that fails without i | The empty name as a dictionary key (`/` alone, which PDF allows) stopped the document opening | `PdfDictionary.Set` accepts it; the entry is kept and written back as `/` | `PdfObjectModelTests.Keeps_an_entry_whose_key_is_the_empty_name`, `Opens_a_document_that_uses_the_empty_name_as_a_key` | | A glyph drawn upside down (negative font size) faulted text extraction | `PdfGlyph.BoundingBox` takes the box between descender and ascender whichever way round | `PdfTextScannerTests.Gives_a_glyph_drawn_upside_down_a_box` | | `PdfDate.TryParse` threw for a time-zone offset beyond 14 hours | an offset no time zone has is read as no offset | `PdfSerializationTests.Reads_an_offset_no_time_zone_has_as_no_offset` | +| An incremental update wrote a new identifier into both elements of `/ID` | the first element of the source's `/ID` is kept; only the second is new | `PdfIncrementalInvariantTests.Keeps_the_first_identifier_element_across_an_incremental_update` | +| `UseIncrementalUpdate` appended a renumbered copy of the whole document, rewriting the signature dictionary after signing | the edited document's objects keep their numbers, and an object written exactly as the source states it is left out: rotating one page of a signed document appends 716 bytes and three objects instead of 20,969 bytes and twelve, and pyHanko places the signature at `ENTIRE_REVISION` | `PdfIncrementalInvariantTests.Appends_only_the_objects_an_edit_changes`, `Leaves_the_signature_dictionary_out_of_an_appended_revision` | ## Corpus readings, 2026-11-16 diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfObjectCopier.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfObjectCopier.cs index a9bc19d..a1bda02 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfObjectCopier.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfObjectCopier.cs @@ -27,6 +27,7 @@ internal sealed partial class PdfObjectCopier private readonly HashSet redirected = []; private readonly Queue pending = []; private readonly HashSet queued = []; + private bool keepNumbers; /// Initializes a new instance of the class. /// The document to copy from. @@ -65,6 +66,13 @@ public void Override(long sourceObjectNumber, Func rewrite overrides[sourceObjectNumber] = rewrite; } + /// + /// Makes every object this copier copies keep the number it has in its source, instead of being given a + /// new one: the copier of the document an incremental update is appended to, whose objects are the same + /// objects in both revisions. + /// + public void KeepObjectNumbers() => keepNumbers = true; + /// Reserves a target object number for an object of the source and queues its body. /// The object number in the source document. /// The object number in the merged document. @@ -75,7 +83,7 @@ public int Reserve(long sourceObjectNumber) return existing; } - var target = writer.AllocateObject(); + var target = keepNumbers ? checked((int)sourceObjectNumber) : writer.AllocateObject(); targets[sourceObjectNumber] = target; Queue(sourceObjectNumber); return target; diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfWriter.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfWriter.cs index 55e4da0..d3cd0b7 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfWriter.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfWriter.cs @@ -16,6 +16,7 @@ internal sealed class PdfWriter private readonly PdfSecurityWriter? security; private readonly Dictionary offsets = []; private readonly long origin; + private Func? isUnchanged; private int lastObjectNumber; /// Initializes a new instance of the class. @@ -95,6 +96,20 @@ private PdfWriter(Stream destination, PdfEngineOptions options, int highestObjec /// The reserved object number. public int AllocateObject() => ++lastObjectNumber; + /// + /// Makes the writer leave out an object that would be written exactly as the revision it is appended to + /// already holds it under the same number. + /// + /// Answers whether an object number and the body about to be written under it are + /// what the earlier revision already states. + /// + /// An incremental update is meant to carry what the edit changed. An object that is written again unchanged + /// costs its bytes and, worse, tells every reader that looks at what changed after a signature — pyHanko's + /// difference analysis, Acrobat's — that the object was modified after signing. + /// + public void SkipUnchanged(Func unchanged) => + isUnchanged = unchanged ?? throw new ArgumentNullException(nameof(unchanged)); + /// Gets the highest object number that has been allocated. public int HighestObjectNumber => lastObjectNumber; @@ -103,6 +118,11 @@ private PdfWriter(Stream destination, PdfEngineOptions options, int highestObjec /// The body of the object. public void WriteObject(int objectNumber, PdfObject body) { + if (isUnchanged?.Invoke(objectNumber, body) == true) + { + return; + } + offsets[objectNumber] = Length; output.WriteInteger(objectNumber); output.Write(" 0 obj\n"u8); diff --git a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Document.cs b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Document.cs index 539f7b3..93a0682 100644 --- a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Document.cs +++ b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Document.cs @@ -693,10 +693,11 @@ private static PdfMetadata ResolveMetadata( /// Writes the document information dictionary and returns its object number. /// The writer of the result. /// The document information of the result. + /// The number the dictionary already has in the revision an update is appended to, or to number it anew. /// The object number of the document information dictionary. - private static int WriteInformation(PdfWriter writer, PdfMetadata metadata) + private static int WriteInformation(PdfWriter writer, PdfMetadata metadata, int? existingNumber = null) { - var number = writer.AllocateObject(); + var number = existingNumber ?? writer.AllocateObject(); var information = new PdfDictionary(); information.Set("Title", Text(metadata.Title)); @@ -728,16 +729,18 @@ private static int WriteInformation(PdfWriter writer, PdfMetadata metadata) /// The writer of the result. /// The object number of the catalog. /// The object number of the document information dictionary, or when the profile forbids it. - /// The identifier written as both elements of the ID entry. + /// The identifier of this revision: the second element of the ID entry, and the first too unless a permanent one is given. /// The encryption of the result, or when it is written in the clear. /// The cross-reference section this revision chains to, or for a complete document. + /// The first element of the source's ID, kept by an appended revision, or . private static void WriteTrailer( PdfWriter writer, int catalogNumber, int? infoNumber, PdfString identifier, PdfSecurityWriter? security, - long? previousXref) + long? previousXref, + PdfString? permanentIdentifier = null) { var trailer = new PdfDictionary(); trailer.Set("Root", new PdfReference(catalogNumber, 0)); @@ -746,7 +749,9 @@ private static void WriteTrailer( trailer.Set("Info", new PdfReference(info, 0)); } - trailer.Set("ID", new PdfArray(new PdfObject[] { identifier, identifier })); + // The first element is permanent: an appended revision keeps the one the document was created with, + // and only the second, which identifies this revision, is new. + trailer.Set("ID", new PdfArray(new PdfObject[] { permanentIdentifier ?? identifier, identifier })); if (previousXref is { } previous) { diff --git a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Incremental.cs b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Incremental.cs index d9a8e3b..10625a4 100644 --- a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Incremental.cs +++ b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Incremental.cs @@ -108,6 +108,52 @@ private static bool FieldsCarrySignature(PdfFileReader reader, PdfObject? fields return false; } + /// + /// Determines whether an object is exactly what the revision an update is appended to already states under + /// the same number, by writing both the way the writer would and comparing the bytes. + /// + /// The document the revision is appended to. + /// The number the object is about to be written under. + /// The body about to be written. + /// when writing it would restate the source unchanged. + private static bool IsUnchangedInSource(PdfFileReader source, int objectNumber, PdfObject body) + { + if (objectNumber > source.CrossReferenceTable.HighestObjectNumber || source.Resolve(objectNumber) is not { } original) + { + return false; + } + + return Serialize(original).AsSpan().SequenceEqual(Serialize(body)); + } + + private static byte[] Serialize(PdfObject value) + { + using var stream = new MemoryStream(); + var output = new PdfOutputBuffer(stream); + + PdfSerializer.Write(value, output); + output.Flush(); + + return stream.ToArray(); + } + + /// Gets the number of the source's document information dictionary, when it is an object of its own. + /// The document the revision is appended to. + /// The object number, or . + private static int? SourceInformationNumber(PdfFileReader source) => + source.CrossReferenceTable.Trailer.Get("Info") is PdfReference information + ? checked((int)information.ObjectNumber) + : null; + + /// + /// Gets the first element of the source's /ID, which ISO 32000-1 §14.4 makes permanent: it "shall + /// not change when the file is incrementally updated". + /// + /// The document the revision is appended to. + /// The identifier, or when the source states none. + private static PdfString? PermanentIdentifier(PdfFileReader source) => + source.CrossReferenceTable.Trailer.Get("ID") is PdfArray { Items: [PdfString permanent, ..] } ? permanent : null; + /// Copies the source into the destination and opens the writer of the appended revision. /// The stream the result is written to. /// The document being edited. diff --git a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Pages.cs b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Pages.cs index 1fa27f2..dbda7f5 100644 --- a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Pages.cs +++ b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Pages.cs @@ -290,11 +290,18 @@ private static List BuildLayout( /// Reserves the object number of every page that is written. /// The writer of the result. /// The pages in output order. - private static void AllocatePageTargets(PdfWriter writer, List layout) + /// The document an incremental update is appended to, whose pages keep their numbers, or . + private static void AllocatePageTargets(PdfWriter writer, List layout, PdfFileReader? keepNumbersOf) { + var kept = new HashSet(); + foreach (var page in layout) { - page.TargetNumber = writer.AllocateObject(); + // In an appended revision a page of the source stays the object it was; a second occurrence of the + // same page — a duplicate — is a new page and gets a number of its own. + page.TargetNumber = page.Reader == keepNumbersOf && kept.Add(page.Source.ObjectNumber) + ? checked((int)page.Source.ObjectNumber) + : writer.AllocateObject(); } } diff --git a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Sources.cs b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Sources.cs index 77ca7b6..dc479e4 100644 --- a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Sources.cs +++ b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Sources.cs @@ -221,16 +221,24 @@ public EditSources( /// Creates the copier of every document. /// The writer of the result. + /// The document an incremental update is appended to, whose copier keeps the numbers of its objects, or . /// /// Object numbers are only unique inside one document, so every document gets a copier of its own. /// All of them write into the same result, which is what makes a copied page indistinguishable /// from a page that was already there. /// - public void Attach(PdfWriter writer) + public void Attach(PdfWriter writer, PdfFileReader? keepNumbersOf = null) { foreach (var reader in readers) { copiers[reader] = new PdfObjectCopier(reader, writer); + + // Only the document a revision is appended to shares its numbers with the result; a document + // the edit imports from has numbers of its own, which would collide with them. + if (reader == keepNumbersOf) + { + copiers[reader].KeepObjectNumbers(); + } } } diff --git a/src/SharpCorePDF.Infrastructure/PdfEditEngine.cs b/src/SharpCorePDF.Infrastructure/PdfEditEngine.cs index 7500422..c6d4751 100644 --- a/src/SharpCorePDF.Infrastructure/PdfEditEngine.cs +++ b/src/SharpCorePDF.Infrastructure/PdfEditEngine.cs @@ -216,20 +216,48 @@ public async Task ApplyAsync( : null; var writer = revision?.Writer ?? new PdfWriter(capture ?? destination, version, options, security); - var catalogNumber = writer.AllocateObject(); - var pagesNodeNumber = writer.AllocateObject(); + + // An appended revision updates the objects of the revision before it rather than writing a second + // copy of them: the catalog, the page tree, the pages and every object copied from the source keep + // their numbers, and an object that comes out exactly as the source already states it is left out. + var primary = revision is null ? null : reader; + + if (primary is not null) + { + writer.SkipUnchanged((number, body) => IsUnchangedInSource(primary, number, body)); + } + + var catalogNumber = primary is not null && primary.RootObjectNumber > 0 + ? checked((int)primary.RootObjectNumber) + : writer.AllocateObject(); + var pagesNodeNumber = primary?.Catalog?.Get("Pages") is PdfReference pagesRoot + ? checked((int)pagesRoot.ObjectNumber) + : writer.AllocateObject(); // Every document the plan reads has its own copier, because object numbers are only unique // inside one document. All copiers write into the same result, which is what makes an // imported page indistinguishable from a page that was already there. - sources.Attach(writer); + sources.Attach(writer, keepNumbersOf: primary); + + if (primary is not null) + { + // The catalog and the information dictionary are written by the engine under their own numbers; + // a copy of the old ones, reached through a reference, would be a second object of the same + // number in the same revision. + sources.CopierOf(primary).Redirect(catalogNumber, catalogNumber); + + if (SourceInformationNumber(primary) is { } informationNumber) + { + sources.CopierOf(primary).Redirect(informationNumber, informationNumber); + } + } foreach (var (source, copier) in sources.Copiers) { RedirectPageTree(source, copier, pagesNodeNumber); } - AllocatePageTargets(writer, layout); + AllocatePageTargets(writer, layout, primary); foreach (var (source, copier) in sources.Copiers) { @@ -279,8 +307,18 @@ [.. layout.Select(page => page.TargetNumber)], // PDF/A-4 forbids the document information dictionary (a trailer /Info is allowed only beside a // catalog /PieceInfo, which this writer does not produce), so the packet is the only copy there. - var infoNumber = profile == PdfAProfile.PdfA4 ? (int?)null : WriteInformation(writer, metadata); - WriteTrailer(writer, catalogNumber, infoNumber, identifier, security, revision?.PreviousXref); + var existingInformation = primary is null ? null : SourceInformationNumber(primary); + var infoNumber = profile == PdfAProfile.PdfA4 + ? (int?)null + : WriteInformation(writer, metadata, existingInformation); + WriteTrailer( + writer, + catalogNumber, + infoNumber, + identifier, + security, + revision?.PreviousXref, + primary is null ? null : PermanentIdentifier(primary)); Verify(capture, layout, plan.Output.Encryption?.UserPassword, warnings); diff --git a/tests/SharpCorePDF.Tests/PdfIncrementalInvariantTests.cs b/tests/SharpCorePDF.Tests/PdfIncrementalInvariantTests.cs index fc5ad8e..95fe3ef 100644 --- a/tests/SharpCorePDF.Tests/PdfIncrementalInvariantTests.cs +++ b/tests/SharpCorePDF.Tests/PdfIncrementalInvariantTests.cs @@ -2,6 +2,7 @@ using SharpCorePDF.Domain.Aggregates; using SharpCorePDF.Domain.ValueObjects; using SharpCorePDF.Infrastructure; +using SharpCorePDF.Infrastructure.Extensions; using SharpCorePDF.Infrastructure.Pdf; using SharpCorePDF.Infrastructure.Sources; @@ -75,7 +76,7 @@ public async Task States_a_size_that_covers_every_revision() } /// The permanent half of the identifier survives an update; only the second half may change. - [Fact(Skip = "Known defect: PdfEditEngine.WriteTrailer writes a new identifier into both elements of /ID even when the revision is appended (PdfEditEngine.Document.cs). Remove the skip with the fix.")] + [Fact] public async Task Keeps_the_first_identifier_element_across_an_incremental_update() { var source = Source(); @@ -85,6 +86,98 @@ public async Task Keeps_the_first_identifier_element_across_an_incremental_updat Assert.Equal(Identifier(source, 0), Identifier(updated, 0)); } + /// + /// An appended revision carries what the edit changed under the numbers those objects already had: rotating + /// one page redeclares that page and the few objects around it, and adds no copy of the document under new + /// numbers. + /// + [Fact] + public async Task Appends_only_the_objects_an_edit_changes() + { + var source = Source(); + var updated = await AppendAsync(source, PdfEditPlan.Create().RotatePage(1, PageRotation.Clockwise90)); + + var before = ObjectNumbers(source); + var appended = ObjectNumbers(updated[source.Length..]); + + Assert.NotEmpty(appended); + Assert.Subset(before, appended); + Assert.True(appended.Count <= 4, $"the revision redeclares {appended.Count} objects: {string.Join(", ", appended)}"); + } + + /// + /// A signature dictionary is not written again by an update appended after signing: it stays the object of + /// the signed revision, which is what lets a validator place the signature at the end of that revision. + /// + [Fact] + public async Task Leaves_the_signature_dictionary_out_of_an_appended_revision() + { + var signed = await SigningFixtures.SignAsync(Source(), SigningFixtures.Options(SigningFixtures.Certificate)); + var updated = await AppendAsync(signed, PdfEditPlan.Create().RotatePage(1, PageRotation.Clockwise90)); + var revision = System.Text.Encoding.Latin1.GetString(updated, signed.Length, updated.Length - signed.Length); + + Assert.DoesNotContain("/ByteRange", revision, StringComparison.Ordinal); + Assert.DoesNotContain("/Type /Sig", revision, StringComparison.Ordinal); + + var validation = Assert.Single(await SigningFixtures.ValidateAsync(updated)); + + Assert.True(validation.IsValid); + Assert.False(validation.Signer.CoversWholeDocument); + } + + /// + /// The objects an edit creates are written into the revision under numbers of their own: an inserted page + /// is a new object, and a reader of the result finds it between the pages that were there. + /// + [Fact] + public async Task Appends_the_objects_an_edit_creates() + { + var source = Source(); + var updated = await AppendAsync( + source, + PdfEditPlan.Create().InsertPage(2, PdfPageSize.A5, page => page.DrawText("inserted", 20, 40, PdfTypeface.Standard(StandardFont.Helvetica), 10))); + + var reader = new PdfFileReader(updated, "updated.pdf", Options); + var fresh = ObjectNumbers(updated[source.Length..]).Except(ObjectNumbers(source)).ToList(); + + Assert.Equal(3, reader.Pages.Count); + Assert.NotEmpty(fresh); + Assert.All(fresh, number => Assert.True(number > ObjectNumbers(source).Max)); + } + + /// + /// A source whose objects refer to its own catalog and information dictionary is updated with one + /// definition of each in the revision: the engine writes them, and the copier, which reaches them through + /// those references, must not write a second copy under the same number. + /// + [Fact] + public async Task Writes_each_object_once_when_the_source_refers_to_its_own_catalog() + { + var builder = new PdfBuilder(); + var catalog = builder.Reserve(); + var pages = builder.Reserve(); + + builder.Add("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Back 1 0 R /About 4 0 R >>"); + builder.Add("<< /Title (Refers to its own catalog) >>"); + builder.Set(catalog, "<< /Type /Catalog /Pages 2 0 R >>"); + builder.Set(pages, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + + var source = builder.Build(catalog, "/Info 4 0 R /ID [<00112233445566778899AABBCCDDEEFF> <00112233445566778899AABBCCDDEEFF>]"); + var updated = await AppendAsync(source, PdfEditPlan.Create().RotatePage(1, PageRotation.Clockwise90)); + var revision = System.Text.Encoding.Latin1.GetString(updated, source.Length, updated.Length - source.Length); + var declared = System.Text.RegularExpressions.Regex.Matches(revision, @"(?m)^(\d+) 0 obj\b") + .Select(match => match.Groups[1].Value) + .ToList(); + + Assert.Equal(declared.Distinct().Count(), declared.Count); + Assert.Equal(90, (int)(new PdfFileReader(updated, "updated.pdf", Options).Pages[0].Dictionary.GetInteger("Rotate") ?? 0L)); + } + + private static SortedSet ObjectNumbers(byte[] bytes) => + [.. System.Text.RegularExpressions.Regex + .Matches(System.Text.Encoding.Latin1.GetString(bytes), @"(?m)^(\d+) 0 obj\b") + .Select(match => int.Parse(match.Groups[1].Value, System.Globalization.CultureInfo.InvariantCulture))]; + private static byte[] Source() => PdfDocumentBuilder.Create(PdfAuthoringOptions.Default with { DocumentId = SourceId }) .AddPage(PdfPageSize.A5, page => page.DrawText("one", 72, 100, PdfTypeface.Standard(StandardFont.Helvetica), 12)) .AddPage(PdfPageSize.A4, page => page.DrawText("two", 72, 100, PdfTypeface.Standard(StandardFont.Helvetica), 12)) From 5fbfc925ad5b94491be121fae272fa455a005d1d Mon Sep 17 00:00:00 2001 From: Andrew Tierney Date: Mon, 16 Nov 2026 12:54:47 +1100 Subject: [PATCH 06/11] Write a missing media box, keep copied reals, and fix what the first CI runs found - A merge copied a page that states no /MediaBox, and inherits none, without one, so the result broke the rule that every page has a media box and each reader assumed its own size. The page is written with the A4 box the inspection already describes it with, and PageSizeAssumed says so. - A merge or an edit rewrote every real it copied to five decimals: a Type 3 font matrix of 0.000012 came out as 0.00001. A real read from a document now keeps its text when that is a well-formed PDF real and is written back from it. A real too large for the formatter's 32-byte buffer failed the write of the whole document with InvalidOperationException; the first run of the document fuzz target found it, and such a value is now formatted on the allocating path. - The image copy route let InvalidDataException escape for damaged Flate data, with or without a predictor (the flate fuzz target's first find); it now refuses it with PdfSourceFormatException. - The Arlington service names its verdict element arlingtonReport, which validate-arlington.py now reads. Its first run found that a free text annotation lacked the /DA ISO 32000-1 requires and annotation appearances lacked /Resources; both are now written. Tests that fail without the changes: PdfMergeFidelityTests (media box, copied reals, any magnitude), PdfContentReaderTests.Refuses_to_copy_an_image_whose_flate_data_is_damaged, and PdfDocumentEditorTests.States_the_default_appearance_and_resources_of_a_free_text_annotation and States_resources_on_an_appearance_that_needs_none. qpdf now passes every example output with no --known-defect flag, so the qpdf step lists none. --- .github/workflows/validation.yml | 4 +- CHANGELOG.md | 17 ++- docs/tools/validate-arlington.py | 7 +- docs/validation.md | 5 +- .../Annotations/AppearanceStreams.cs | 9 +- .../Pdf/PdfDocumentInspection.Pages.cs | 6 +- .../Pdf/PdfImageFilters.cs | 23 +++- .../Pdf/PdfObject.cs | 25 +++- .../Pdf/PdfObjectCopier.Clone.cs | 2 +- .../Pdf/PdfObjectParser.cs | 36 +++++- .../Pdf/PdfOutputBuffer.cs | 6 +- .../Pdf/PdfSerializer.cs | 4 + .../PdfEditEngine.Annotations.cs | 12 ++ .../PdfMergeEngine.Pages.cs | 26 +++- .../SharpCorePDF.Tests/PdfAnnotationTests.cs | 50 ++++++++ .../PdfContentReaderTests.Images.cs | 30 +++++ .../PdfMergeFidelityTests.cs | 113 ++++++++++++++++++ 17 files changed, 345 insertions(+), 30 deletions(-) create mode 100644 tests/SharpCorePDF.Tests/PdfMergeFidelityTests.cs diff --git a/.github/workflows/validation.yml b/.github/workflows/validation.yml index 681f1fd..b25e731 100644 --- a/.github/workflows/validation.yml +++ b/.github/workflows/validation.yml @@ -90,9 +90,7 @@ jobs: case "$(basename "$file")" in protected.pdf|encrypted-*) continue ;; esac files+=("$file") done - # Both merge a page of NamedDests.pdf, which states no /MediaBox; the merge copies it without one, - # so the result breaks the rule that every page has a media box, and qpdf assumes Letter. - python docs/tools/validate-pdf.py --known-defect inspected.pdf --known-defect with-warnings.pdf "${files[@]}" + python docs/tools/validate-pdf.py "${files[@]}" - name: Text round trip run: python docs/tools/validate-text.py "$RUNNER_TEMP/corpus" diff --git a/CHANGELOG.md b/CHANGELOG.md index caf759e..7aa74cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -74,12 +74,17 @@ against an empty feed. - **An incremental update keeps the permanent identifier.** The first element of `/ID` is the source's in every revision, as ISO 32000-1 §14.4 requires; only the second element is new. -### Known defects, found by the new checks and not yet fixed - -Each is reproduced, pinned by a skipped test or a strict `--known-defect` flag that fails the run once the -defect is fixed, and listed in [`docs/validation.md`](docs/validation.md#known-defects). - -- A merge copies a page that states no `/MediaBox` without one. +- **A merge writes a media box for a page that states none.** Every page of a PDF needs one; a source page + with none, and none to inherit, was copied without it, and each reader assumed its own size. It is now + written with the A4 box the inspection already describes it with, and `PageSizeAssumed` says so. +- **Copied real numbers keep their value.** A merge or an edit rewrote every real it copied to five decimals — + a Type 3 font matrix of `0.000012` became `0.00001`, seventeen percent off, and `0.000004` became `0`. A real + read from a document is now written back from its own text. A real too large for the formatter's buffer no + longer fails the write of the whole document (found by the `document` fuzz target in its first run). +- **The image copy route refuses damaged Flate data with `PdfSourceFormatException`** instead of letting the + decompressor's `InvalidDataException` reach the caller (found by the `flate` fuzz target). +- **Annotations satisfy the Arlington model.** A free text annotation states the `/DA` ISO 32000-1 requires, + and every annotation appearance states its `/Resources` (found by the Arlington check in its first run). - **Run-length images are decoded.** `RunLengthDecode` joins Flate, LZW and the two ASCII filters: a bilevel or scan-style image that declares it now yields its samples — and `Gray8`, `Rgb24` or `Cmyk32` diff --git a/docs/tools/validate-arlington.py b/docs/tools/validate-arlington.py index 3d99086..35bcd24 100644 --- a/docs/tools/validate-arlington.py +++ b/docs/tools/validate-arlington.py @@ -71,10 +71,13 @@ def read_report(payload: bytes) -> tuple[bool, list[str]]: return compliant.group(1) == "true", re.findall(r'"clause"\s*:\s*"([^"]+)"', text) - reports = [element for element in root.iter() if element.tag.endswith("validationReport")] + # The Arlington service names its verdict element arlingtonReport (veraPDF's PDF/A reports name theirs + # validationReport); both carry isCompliant, which is what the verdict is read from. + reports = [element for element in root.iter() + if element.tag.endswith(("arlingtonReport", "validationReport")) and element.get("isCompliant")] if not reports: - raise ValueError("the XML report holds no validationReport element") + raise ValueError("the XML report holds no arlingtonReport or validationReport element") failed = [] diff --git a/docs/validation.md b/docs/validation.md index 47882c2..a06e569 100644 --- a/docs/validation.md +++ b/docs/validation.md @@ -83,7 +83,6 @@ fix. This is the list on 2026-11-16, every entry reproduced, not inferred: | Defect | Found by | Evidence | | --- | --- | --- | -| A merge copies a page that states no `/MediaBox` (and inherits none) without one, so the result breaks the rule that every page has a media box; qpdf repairs it by assuming Letter | #2 | `validate-pdf.py` on `inspected.pdf` and `with-warnings.pdf` (both merge a page of `NamedDests.pdf`) | Fixed since the list was first drawn up, each with the test that fails without its fix: @@ -99,6 +98,10 @@ Fixed since the list was first drawn up, each with the test that fails without i | `PdfDate.TryParse` threw for a time-zone offset beyond 14 hours | an offset no time zone has is read as no offset | `PdfSerializationTests.Reads_an_offset_no_time_zone_has_as_no_offset` | | An incremental update wrote a new identifier into both elements of `/ID` | the first element of the source's `/ID` is kept; only the second is new | `PdfIncrementalInvariantTests.Keeps_the_first_identifier_element_across_an_incremental_update` | | `UseIncrementalUpdate` appended a renumbered copy of the whole document, rewriting the signature dictionary after signing | the edited document's objects keep their numbers, and an object written exactly as the source states it is left out: rotating one page of a signed document appends 716 bytes and three objects instead of 20,969 bytes and twelve, and pyHanko places the signature at `ENTIRE_REVISION` | `PdfIncrementalInvariantTests.Appends_only_the_objects_an_edit_changes`, `Leaves_the_signature_dictionary_out_of_an_appended_revision` | +| A merge copied a page that states no `/MediaBox` (and inherits none) without one | the page is written with the A4 box the inspection describes it with, and `PageSizeAssumed` says so | `PdfMergeFidelityTests.Writes_a_media_box_for_a_page_that_has_none` | +| Reals a merge or an edit copied were re-formatted to five decimals (`0.000012` became `0.00001`), and a real too large for the formatter's buffer failed the write of the whole document — the first crash the `document` fuzz target found | a real read from a document is written from its own text when that text is a well-formed PDF real; other values are formatted at any magnitude | `PdfMergeFidelityTests.Writes_the_reals_it_copies_as_the_source_states_them`, `Writes_a_real_of_any_magnitude_in_the_form_pdf_defines` | +| The copy route of an image let the decompressor's `InvalidDataException` escape for damaged Flate data — the crash the `flate` fuzz target found | the copy route refuses damaged data with `PdfSourceFormatException`, with or without a predictor | `PdfContentReaderTests.Refuses_to_copy_an_image_whose_flate_data_is_damaged` | +| A free text annotation was written without the `/DA` ISO 32000-1 requires, and annotation appearances without `/Resources` — both reported by the Arlington check's first run | `/DA` states the font, size and colour the appearance draws with; every appearance states its resources, empty if it needs none | `PdfDocumentEditorTests.States_the_default_appearance_and_resources_of_a_free_text_annotation`, `States_resources_on_an_appearance_that_needs_none` | ## Corpus readings, 2026-11-16 diff --git a/src/SharpCorePDF.Infrastructure/Annotations/AppearanceStreams.cs b/src/SharpCorePDF.Infrastructure/Annotations/AppearanceStreams.cs index c38fd87..ac2adfe 100644 --- a/src/SharpCorePDF.Infrastructure/Annotations/AppearanceStreams.cs +++ b/src/SharpCorePDF.Infrastructure/Annotations/AppearanceStreams.cs @@ -150,12 +150,11 @@ private static PdfStream Wrap( .SetName("Type", "XObject") .SetName("Subtype", "Form") .Set("BBox", ToArray(bounds)) - .SetName("Filter", "FlateDecode"); + .SetName("Filter", "FlateDecode") - if (resources.Count > 0) - { - dictionary.Set("Resources", resources); - } + // Stated even when empty: a form XObject without /Resources leaves a reader to borrow the page's, + // a PDF 1.1 fallback the Arlington model of PDF 1.7 and later no longer allows. + .Set("Resources", resources); return new PdfStream(dictionary, EmbeddedImage.Compress(operators)); } diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfDocumentInspection.Pages.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfDocumentInspection.Pages.cs index 13deea1..86f20a6 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfDocumentInspection.Pages.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfDocumentInspection.Pages.cs @@ -39,7 +39,11 @@ private static List ReadPages(PdfFileReader reader, ListReads the size a page's own or inherited /MediaBox states. + /// The document. + /// The page. + /// The size, or when the page has no readable media box. + internal static PdfPageSize? TryReadPageSize(PdfFileReader reader, SourcePage page) { var box = reader.ResolveArray(page.Dictionary.Get("MediaBox")) ?? reader.ResolveArray(page.Inherited.Get("MediaBox")); diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfImageFilters.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfImageFilters.cs index 15951b7..dc7d8f2 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfImageFilters.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfImageFilters.cs @@ -154,14 +154,27 @@ public static async Task CopyAsync( // reach, because the two tests name the same predictor values. using var predictor = PdfFilters.CreatePredictorDecoder(decodeParameters, sourceName); - if (predictor is not null) + // A damaged stream is the document's fault, not the caller's, and the contract names it with a + // PdfSourceFormatException: the decompressor's own exception would reach the caller as a failure + // of the library. What was written before the damage stays written; the caller learns it is + // incomplete from the exception. + try { - await InflatePredictorToAsync(stream.Raw, predictor, destination, cancellationToken) - .ConfigureAwait(false); + if (predictor is not null) + { + await InflatePredictorToAsync(stream.Raw, predictor, destination, cancellationToken) + .ConfigureAwait(false); + } + else + { + await InflateToAsync(stream.Raw, destination, cancellationToken).ConfigureAwait(false); + } } - else + catch (InvalidDataException exception) { - await InflateToAsync(stream.Raw, destination, cancellationToken).ConfigureAwait(false); + throw new PdfSourceFormatException( + sourceName, + $"the Flate data of the payload is damaged and could not be decompressed for copying: {exception.Message}"); } return; diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfObject.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfObject.cs index 7ae7d94..dfcee85 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfObject.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfObject.cs @@ -109,9 +109,29 @@ internal sealed class PdfReal : PdfObject /// The value. public PdfReal(double value) => Value = value; + /// Initializes a new instance of the class with the text a document wrote it as. + /// The value. + /// The bytes of the number as the document states it, or . + public PdfReal(double value, byte[]? original) + { + Value = value; + Original = original; + } + /// Gets the value. public double Value { get; } + /// + /// Gets the bytes the number was read from, when it was read from a document and they are a well-formed + /// PDF real; for a number the engine computed. + /// + /// + /// A merge or an edit copies a dictionary by writing it again, so a real it did not change is written from + /// these bytes: formatting the parsed value instead rounded it to five decimals, and a Type 3 font matrix of + /// 0.000012 came out as 0.00001. + /// + public byte[]? Original { get; } + /// public override PdfObjectKind Kind => PdfObjectKind.Number; @@ -119,7 +139,10 @@ internal sealed class PdfReal : PdfObject public override string ToString() => PdfNumberFormat.Format(Value); } -/// Formats numbers the way PDF expects them: invariant, shortest round-trippable form. +/// +/// Formats a number the engine computed the way PDF expects it: invariant, without an exponent, and to five +/// decimal places. A number read from a document is written from its own text instead (). +/// internal static class PdfNumberFormat { /// Formats a real number. diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfObjectCopier.Clone.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfObjectCopier.Clone.cs index 2693c65..d162fa7 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfObjectCopier.Clone.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfObjectCopier.Clone.cs @@ -13,7 +13,7 @@ internal sealed partial class PdfObjectCopier null or PdfNull => PdfNull.Instance, PdfBoolean boolean => PdfBoolean.From(boolean.Value), PdfInteger integer => new PdfInteger(integer.Value), - PdfReal real => new PdfReal(real.Value), + PdfReal real => new PdfReal(real.Value, real.Original), PdfName name => new PdfName(name.Value), PdfString text => new PdfString(text.Bytes, text.IsHexadecimal), PdfReference reference => new PdfReference(Reserve(reference.ObjectNumber), 0), diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfObjectParser.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfObjectParser.cs index c6e12a9..b7bffbf 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfObjectParser.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfObjectParser.cs @@ -128,7 +128,7 @@ public PdfObject ParseObject(int depth = 0) PdfTokenType.ArrayOpen => ParseArray(depth), PdfTokenType.DictionaryOpen => ParseDictionaryOrStream(depth), PdfTokenType.Number when token.IsInteger => ParseIntegerOrReference(token), - PdfTokenType.Number => new PdfReal(token.RealValue), + PdfTokenType.Number => new PdfReal(token.RealValue, OriginalReal(token)), PdfTokenType.Name => new PdfName(token.Text), PdfTokenType.LiteralString => new PdfString(DecryptString(token.Bytes ?? []), false), PdfTokenType.HexString => new PdfString(DecryptString(token.Bytes ?? []), true), @@ -137,6 +137,40 @@ public PdfObject ParseObject(int depth = 0) }; } + /// Keeps the bytes of a real when they are a well-formed PDF real, so a copy writes it back unchanged. + /// The number token. + /// The bytes, or for a form PDF does not define, which is written normalised. + /// + /// PDF defines a real as an optional sign, digits and at most one period (ISO 32000-1 §7.3.3); it has no + /// exponent. A token outside that form — an exponent, a second sign — was read leniently, and is written in + /// the form PDF defines rather than repeated. + /// + private byte[]? OriginalReal(PdfToken token) + { + var text = lexer.Data.AsSpan(token.Start, token.End - token.Start); + var body = text.Length > 0 && text[0] is (byte)'+' or (byte)'-' ? text[1..] : text; + var periods = 0; + var digits = 0; + + foreach (var value in body) + { + if (value == (byte)'.') + { + periods++; + } + else if (value is >= (byte)'0' and <= (byte)'9') + { + digits++; + } + else + { + return null; + } + } + + return digits > 0 && periods <= 1 && body.Length <= 64 ? text.ToArray() : null; + } + /// Decrypts the bytes of a string, when the parser was told how to. /// The bytes as they were stored. /// The bytes to expose. diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfOutputBuffer.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfOutputBuffer.cs index f284774..b79914c 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfOutputBuffer.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfOutputBuffer.cs @@ -80,9 +80,13 @@ public void WriteReal(double value) { Span scratch = stackalloc byte[32]; + // Thirty-two bytes hold every value a page states; a larger magnitude — which only a document can + // supply, PDF readers being limited to about 3.4e38 — takes the allocating path instead of failing + // the write of the whole document. if (!PdfNumberFormat.TryFormat(value, scratch, out var written)) { - throw new InvalidOperationException("The real number could not be formatted."); + Write(System.Text.Encoding.ASCII.GetBytes(PdfNumberFormat.Format(value))); + return; } Write(scratch[..written]); diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfSerializer.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfSerializer.cs index 5e974bf..efc3623 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfSerializer.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfSerializer.cs @@ -36,6 +36,10 @@ public static void Write( output.WriteInteger(integer.Value); break; + case PdfReal { Original: { } original }: + output.Write(original); + break; + case PdfReal real: output.WriteReal(real.Value); break; diff --git a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Annotations.cs b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Annotations.cs index c90b709..89b7d1c 100644 --- a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Annotations.cs +++ b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Annotations.cs @@ -318,6 +318,18 @@ private static PdfDictionary BuildAnnotationDictionary( dictionary.Set("Contents", PdfString.FromText(contents)); } + // A free text annotation must state its default appearance (ISO 32000-1, table 174): the font, size + // and colour a reader regenerating the appearance draws with, which are the ones the appearance stream + // this writer draws already uses. + if (annotation.Kind == PdfAnnotationKind.FreeText) + { + var style = annotation.TextStyle ?? PdfStandardTextStyle.Default; + + dictionary.Set("DA", PdfString.FromText(string.Create( + System.Globalization.CultureInfo.InvariantCulture, + $"/{AppearanceStreams.FontName} {PdfNumberFormat.Format(style.Size)} Tf {PdfNumberFormat.Format(style.Color.Red / 255.0)} {PdfNumberFormat.Format(style.Color.Green / 255.0)} {PdfNumberFormat.Format(style.Color.Blue / 255.0)} rg"))); + } + if (annotation.Color is { } color) { dictionary.Set("C", ColorArray(color)); diff --git a/src/SharpCorePDF.Infrastructure/PdfMergeEngine.Pages.cs b/src/SharpCorePDF.Infrastructure/PdfMergeEngine.Pages.cs index a09aa60..9af4d3c 100644 --- a/src/SharpCorePDF.Infrastructure/PdfMergeEngine.Pages.cs +++ b/src/SharpCorePDF.Infrastructure/PdfMergeEngine.Pages.cs @@ -38,7 +38,7 @@ private static void PreparePages(PdfMergeContext context, MergeSource source) source.PagesInOrder.Add(target); source.Copier.Override( page.ObjectNumber, - body => BuildMergedPage(context, source, body, page.Inherited)); + body => BuildMergedPage(context, source, body, page)); } if (source.PagesInOrder.Count == 0) @@ -66,14 +66,15 @@ private static void PreparePages(PdfMergeContext context, MergeSource source) /// The merge context. /// The source the page came from. /// The copied page dictionary. - /// The attributes the page inherited in its own document. + /// The page as its own document states it, with the attributes it inherited there. /// The page dictionary of the merged document. private static PdfDictionary BuildMergedPage( PdfMergeContext context, MergeSource source, PdfObject body, - PdfDictionary inherited) + SourcePage sourcePage) { + var inherited = sourcePage.Inherited; var copy = body as PdfDictionary ?? new PdfDictionary(); var page = new PdfDictionary(); @@ -92,6 +93,25 @@ private static PdfDictionary BuildMergedPage( page.Set("Type", new PdfName("Page")); page.Set("Parent", new PdfReference(context.PagesNodeNumber, 0)); + // Every page must have a media box (ISO 32000-1, table 30), on itself or inherited. A source page that + // has neither is described as A4 by the inspection, and is written with that box, so the result is a + // valid document whose page is the size the library reported instead of whatever each reader assumes. + if (Pdf.PdfDocumentInspection.TryReadPageSize(source.Reader, sourcePage) is null) + { + var assumed = Domain.ValueObjects.PdfPageSize.A4; + + page.Set("MediaBox", new PdfArray( + [ + new PdfInteger(0), + new PdfInteger(0), + new PdfReal(assumed.WidthPoints), + new PdfReal(assumed.HeightPoints), + ])); + context.Warnings.Add(new( + Domain.ValueObjects.PdfWarningKind.PageSizeAssumed, + $"'{source.DisplayName}': page {sourcePage.Index + 1} has no readable /MediaBox; it was written with the A4 box it is described with, because every page of the result needs one.")); + } + if (!context.Output.PreserveAnnotations) { page.Remove("Annots"); diff --git a/tests/SharpCorePDF.Tests/PdfAnnotationTests.cs b/tests/SharpCorePDF.Tests/PdfAnnotationTests.cs index 6da205b..fc0ae3e 100644 --- a/tests/SharpCorePDF.Tests/PdfAnnotationTests.cs +++ b/tests/SharpCorePDF.Tests/PdfAnnotationTests.cs @@ -240,6 +240,56 @@ public async Task Draws_the_text_of_a_free_text_annotation_into_its_appearance() Assert.Contains(" Tf", operators, StringComparison.Ordinal); } + /// + /// A free text annotation states the default appearance ISO 32000-1 requires of it — the font, the size and + /// the colour its appearance stream draws with — and that stream states resources of its own, as a form + /// XObject must under the Arlington model; neither was written before. + /// + [Fact] + public async Task States_the_default_appearance_and_resources_of_a_free_text_annotation() + { + var edited = await Editor.ApplyToArrayAsync( + new PdfByteArraySource(ThreePages(), "three-pages.pdf"), + PdfEditPlan.Create().AddAnnotation( + 1, + PdfAnnotations.FreeText( + new PdfRectangle(72, 500, 320, 530), + "Reviewed", + new PdfStandardTextStyle(StandardFont.Helvetica, 10, PdfColor.FromRgb(255, 0, 0)))), + cancellationToken: Token); + + var reader = Read(edited); + var annots = reader.ResolveArray(reader.Pages[0].Dictionary.Get("Annots")); + var annotation = reader.ResolveDictionary(annots?.Items[0]); + var appearance = Assert.IsType(reader.Resolve(reader.ResolveDictionary(annotation?.Get("AP"))?.Get("N"))); + var defaultAppearance = Assert.IsType(annotation?.Get("DA")); + + Assert.Equal("/Helv 10 Tf 1 0 0 rg", Encoding.ASCII.GetString(defaultAppearance.Bytes)); + Assert.IsType(reader.Resolve(appearance.Dictionary.Get("Resources"))); + } + + /// + /// An appearance that needs no font and no graphics state still states its resources, empty, so a reader + /// never borrows the page's. + /// + [Fact] + public async Task States_resources_on_an_appearance_that_needs_none() + { + var edited = await Editor.ApplyToArrayAsync( + new PdfByteArraySource(ThreePages(), "three-pages.pdf"), + PdfEditPlan.Create().AddAnnotation( + 1, + PdfAnnotations.Highlight([new PdfRectangle(72, 300, 200, 312)], PdfColor.FromRgb(255, 224, 130))), + cancellationToken: Token); + + var reader = Read(edited); + var annots = reader.ResolveArray(reader.Pages[0].Dictionary.Get("Annots")); + var annotation = reader.ResolveDictionary(annots?.Items[0]); + var appearance = Assert.IsType(reader.Resolve(reader.ResolveDictionary(annotation?.Get("AP"))?.Get("N"))); + + Assert.IsType(reader.Resolve(appearance.Dictionary.Get("Resources"))); + } + /// An annotation that states its own appearance is drawn with those operators and no others. [Fact] public async Task Writes_the_operators_of_a_supplied_annotation_appearance() diff --git a/tests/SharpCorePDF.Tests/PdfContentReaderTests.Images.cs b/tests/SharpCorePDF.Tests/PdfContentReaderTests.Images.cs index d4daed0..6388014 100644 --- a/tests/SharpCorePDF.Tests/PdfContentReaderTests.Images.cs +++ b/tests/SharpCorePDF.Tests/PdfContentReaderTests.Images.cs @@ -14,6 +14,36 @@ namespace SharpCorePDF.Tests; /// public sealed partial class PdfContentReaderTests { + /// + /// The copy route of an image whose Flate data is damaged refuses it with the contract's exception, whether a + /// predictor is declared or not: the decompressor's own exception used to reach the caller as if the library + /// had failed. + /// + /// The decode parameters the image declares. + [Theory] + [InlineData("/DecodeParms << /Predictor 12 /Colors 1 /BitsPerComponent 8 /Columns 4 >>")] + [InlineData("")] + public async Task Refuses_to_copy_an_image_whose_flate_data_is_damaged(string parameters) + { + var builder = new PdfBuilder(); + var catalog = builder.Reserve(); + var pages = builder.Reserve(); + var damaged = new byte[] { 0x78, 0x9C, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05 }; + + builder.Add("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 100 100] /Resources << /XObject << /Im0 5 0 R >> >> /Contents 4 0 R >>"); + builder.AddStream(string.Empty, "q 50 0 0 50 0 0 cm /Im0 Do Q"u8.ToArray()); + builder.AddStream($"/Type /XObject /Subtype /Image /Width 4 /Height 4 /BitsPerComponent 8 /ColorSpace /DeviceGray /Filter /FlateDecode {parameters}", damaged); + builder.Set(catalog, "<< /Type /Catalog /Pages 2 0 R >>"); + builder.Set(pages, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + + var source = Source(builder.Build(catalog)); + var reader = Reader(); + var image = Assert.Single((await reader.ReadImagesAsync(source, PageRange.All, cancellationToken: Token)).Items); + + await Assert.ThrowsAsync( + () => reader.CopyImageDataAsync(source, image, Stream.Null, cancellationToken: Token)); + } + /// One object drawn twice and shared by two pages is one entry carrying both pages. [Fact] public async Task Returns_each_image_once_with_every_page_that_draws_it() diff --git a/tests/SharpCorePDF.Tests/PdfMergeFidelityTests.cs b/tests/SharpCorePDF.Tests/PdfMergeFidelityTests.cs new file mode 100644 index 0000000..710d4c2 --- /dev/null +++ b/tests/SharpCorePDF.Tests/PdfMergeFidelityTests.cs @@ -0,0 +1,113 @@ +using System.Text; +using SharpCorePDF.Application.Extensions; +using SharpCorePDF.Application.Services; +using SharpCorePDF.Domain.Aggregates; +using SharpCorePDF.Domain.Entities; +using SharpCorePDF.Domain.ValueObjects; +using SharpCorePDF.Infrastructure; +using SharpCorePDF.Infrastructure.Pdf; +using SharpCorePDF.Infrastructure.Sources; + +namespace SharpCorePDF.Tests; + +/// +/// Tests of what a merge writes for values it does not change: the numbers a source states come back as the +/// source states them, and a page the source leaves without a size comes back with the size the library +/// describes it with. +/// +public sealed class PdfMergeFidelityTests +{ + private static CancellationToken Token => TestContext.Current.CancellationToken; + + /// + /// A real a merge copies is written from the text the source used, not from a value formatted to five + /// decimals: a Type 3 font matrix of 0.000012 used to come out as 0.00001, seventeen percent + /// off, and 0.000004 as 0. + /// + [Fact] + public async Task Writes_the_reals_it_copies_as_the_source_states_them() + { + var merged = await MergeAsync(Page("/Scale [0.000012 -0.00048828125 0.000004 12.5 +.5]")); + var text = Encoding.Latin1.GetString(merged); + + Assert.Contains("[0.000012 -0.00048828125 0.000004 12.5 +.5]", text, StringComparison.Ordinal); + } + + /// + /// A real outside the form PDF defines — here with an exponent, which PDF does not have — is written in the + /// form PDF defines, whatever its magnitude, instead of failing the write of the whole document. + /// + [Fact] + public async Task Writes_a_real_of_any_magnitude_in_the_form_pdf_defines() + { + var merged = await MergeAsync(Page("/Huge 1e300 /Small 2.5e-3")); + var text = Encoding.Latin1.GetString(merged); + + Assert.Contains("/Huge 1" + new string('0', 300), text, StringComparison.Ordinal); + Assert.Contains("/Small 0.0025", text, StringComparison.Ordinal); + } + + /// + /// A page that states no media box and inherits none is written with the A4 box the inspection describes it + /// with, because every page of a PDF needs one; the merge says so with . + /// + [Fact] + public async Task Writes_a_media_box_for_a_page_that_has_none() + { + var builder = new PdfBuilder(); + var catalog = builder.Reserve(); + var pages = builder.Reserve(); + + builder.Add("<< /Type /Page /Parent 2 0 R >>"); + builder.Set(catalog, "<< /Type /Catalog /Pages 2 0 R >>"); + builder.Set(pages, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + + using var destination = new MemoryStream(); + var result = await Merger().MergeAsync( + MergePlan.Create().AddSource(PdfSource.Create(new PdfByteArraySource(builder.Build(catalog), "sizeless.pdf"))), + destination, + Token); + + var reader = new PdfFileReader(destination.ToArray(), "merged.pdf", PdfEngineOptions.Default); + var box = Assert.IsType(reader.Pages[0].Dictionary.Get("MediaBox")); + + Assert.Equal(4, box.Count); + Assert.Equal(PdfPageSize.A4.WidthPoints, ((PdfReal)box.Items[2]).Value, 3); + Assert.Equal(PdfPageSize.A4.HeightPoints, ((PdfReal)box.Items[3]).Value, 3); + Assert.Contains(result.Warnings, warning => warning.Kind == PdfWarningKind.PageSizeAssumed); + } + + /// A page that states its own media box keeps it, and no size is assumed for it. + [Fact] + public async Task Keeps_the_media_box_a_page_states() + { + using var destination = new MemoryStream(); + var result = await Merger().MergeAsync( + MergePlan.Create().AddSource(PdfSource.Create(new PdfByteArraySource(Page(string.Empty), "sized.pdf"))), + destination, + Token); + + Assert.Contains("/MediaBox [0 0 200 300]", Encoding.Latin1.GetString(destination.ToArray()), StringComparison.Ordinal); + Assert.DoesNotContain(result.Warnings, warning => warning.Kind == PdfWarningKind.PageSizeAssumed); + } + + private static PdfMergerService Merger() => new(new PdfDocumentReader(), new PdfMergeEngine()); + + private static byte[] Page(string entries) + { + var builder = new PdfBuilder(); + var catalog = builder.Reserve(); + var pages = builder.Reserve(); + + builder.Add($"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 300] {entries} >>"); + builder.Set(catalog, "<< /Type /Catalog /Pages 2 0 R >>"); + builder.Set(pages, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + + return builder.Build(catalog); + } + + private static Task MergeAsync(byte[] document) => + Merger().MergeToBytesAsync( + MergePlan.Create().AddSource(PdfSource.Create(new PdfByteArraySource(document, "source.pdf"))), + Token); +} From 3a8b5784790b4177b9a78e2f8c301c83f5811799 Mon Sep 17 00:00:00 2001 From: Andrew Tierney Date: Mon, 16 Nov 2026 13:12:40 +1100 Subject: [PATCH 07/11] Rebuild moved tables, read object streams in recovery, and harden five edges Recovery - A cross-reference table that parses can still be wrong: bytes inserted or removed after it was written move every object. Reconstruction now also runs when the catalog's offset, or most of a sample of offsets, does not land on the n g obj header of its own object. - Reconstruction indexes the objects of every object stream the scan finds, and takes /Root from the dictionary of the last cross-reference stream, so a PDF 1.5 file whose cross-reference stream is lost reads instead of being refused with "no document catalog". The test that pinned the refusal now pins the rebuild. Hardening - AESV2 always uses a sixteen-byte key; a crypt filter that declares fewer made the file unopenable. - The whole-buffer predictor path sizes a row in 64 bits like the streaming decoder: a declared Columns x Colors that wrapped to zero looped forever under TIFF prediction, and to a negative number threw ArgumentOutOfRangeException. - A stream with a wrong /Length is cut where its zlib data is complete, by its Adler-32, instead of at the first endstream whose truncated bytes inflate; at most sixteen candidates are inflated. - A reference to a generation the table does not list resolves to null (ISO 32000-1, 7.3.10). - A revision appended to a file whose newest section is a cross-reference stream, by an edit or a signature, ends in a cross-reference stream instead of a classic table. qpdf and pyHanko accept an edit and two signatures appended this way. Corpus fault - The public-corpora job faulted on isartor-6-1-11-t01-fail-a.pdf: a file specification that only names an external file is listed as an attachment with ByteLength 0, and copying it threw ArgumentException. It now writes no bytes; an id the document does not have is still refused. Each change has a test that fails without it. Full suite passes on net11.0 and net10.0; the pdf.js corpus run reports no fault, crash or timeout. --- CHANGELOG.md | 20 +++ docs/validation.md | 4 + .../Ports/IPdfContentReader.cs | 8 +- .../Pdf/PdfCrossReference.cs | 6 + .../Pdf/PdfCrossReferenceReader.Scan.cs | 125 ++++++++++++++++- .../Pdf/PdfCrossReferenceReader.cs | 68 ++++++++- .../Pdf/PdfCrossReferenceStreamWriter.cs | 128 +++++++++++++++++ .../Pdf/PdfFileReader.cs | 25 +++- .../Pdf/PdfFilters.Inflation.cs | 37 +++++ .../Pdf/PdfFilters.Predictors.cs | 49 ++++--- .../Pdf/PdfIncrementalWriter.cs | 15 +- .../Pdf/PdfObjectParser.Streams.cs | 50 ++++++- .../Pdf/PdfSecurityHandler.cs | 5 +- .../Pdf/PdfWriter.cs | 16 ++- .../PdfEditEngine.Document.cs | 6 +- .../PdfEditEngine.cs | 3 +- .../PdfPageContentReader.Metadata.cs | 24 +++- .../Signatures/SignatureFieldWriter.cs | 2 +- .../PdfContentReaderTests.Metadata.cs | 36 +++++ .../PdfDocumentSignerTests.cs | 20 +++ .../PdfIncrementalWriterTests.cs | 38 ++++++ .../PdfMergeEngineTests.OutputEncryption.cs | 22 +++ tests/SharpCorePDF.Tests/PdfPredictorTests.cs | 24 ++++ .../PdfReaderRecoveryTests.cs | 129 ++++++++++++++++-- .../PdfStreamParsingTests.cs | 29 ++++ 25 files changed, 831 insertions(+), 58 deletions(-) create mode 100644 src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceStreamWriter.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 7aa74cc..29cdcc1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -85,6 +85,26 @@ against an empty feed. decompressor's `InvalidDataException` reach the caller (found by the `flate` fuzz target). - **Annotations satisfy the Arlington model.** A free text annotation states the `/DA` ISO 32000-1 requires, and every annotation appearance states its `/Resources` (found by the Arlington check in its first run). +- **A cross-reference table whose offsets moved is rebuilt.** Bytes inserted or removed after a table was + written — a converted line ending, a prepended byte order mark — leave a table that parses and offsets that + land inside other objects. Reconstruction used to run only when no table or no catalog was found; it now + also runs when the catalog's offset, or most of a sample of offsets, does not land on its own object. +- **Reconstruction looks inside object streams.** A PDF 1.5 file whose cross-reference stream is lost keeps + its catalog and pages in object streams, which a scan for object headers cannot see; it was refused with + "no document catalog". The objects of every object stream the scan finds are now indexed, and `/Root` is + taken from the dictionary of the last cross-reference stream. +- **Hardening.** AESV2 always uses a sixteen-byte key, as AES-128 requires, so a file that declares a shorter + one opens. A predicted row is sized in 64 bits on the whole-buffer path too: declared columns and colours + whose product wrapped to zero looped forever, and to a negative number threw `ArgumentOutOfRangeException`. + A stream whose length is wrong is cut where its compressed data is complete — where its Adler-32 matches — + rather than at the first `endstream` whose bytes inflate, and at most sixteen candidates are inflated. A + reference to a generation the table does not list resolves to null, as ISO 32000-1 reads it. +- **A revision appended to a file that uses cross-reference streams ends in one.** Edits and signatures + appended a classic table chained to a stream; they now write the form the document uses. +- **Copying a listed attachment that embeds nothing writes nothing.** A file specification in the embedded + files tree that only names an external file is listed, with a warning and a length of zero; copying it + threw `ArgumentException` as if the caller had passed a file of another document (found by the veraPDF + corpus run, `isartor-6-1-11-t01-fail-a.pdf`). - **Run-length images are decoded.** `RunLengthDecode` joins Flate, LZW and the two ASCII filters: a bilevel or scan-style image that declares it now yields its samples — and `Gray8`, `Rgb24` or `Cmyk32` diff --git a/docs/validation.md b/docs/validation.md index a06e569..3491b07 100644 --- a/docs/validation.md +++ b/docs/validation.md @@ -102,6 +102,10 @@ Fixed since the list was first drawn up, each with the test that fails without i | Reals a merge or an edit copied were re-formatted to five decimals (`0.000012` became `0.00001`), and a real too large for the formatter's buffer failed the write of the whole document — the first crash the `document` fuzz target found | a real read from a document is written from its own text when that text is a well-formed PDF real; other values are formatted at any magnitude | `PdfMergeFidelityTests.Writes_the_reals_it_copies_as_the_source_states_them`, `Writes_a_real_of_any_magnitude_in_the_form_pdf_defines` | | The copy route of an image let the decompressor's `InvalidDataException` escape for damaged Flate data — the crash the `flate` fuzz target found | the copy route refuses damaged data with `PdfSourceFormatException`, with or without a predictor | `PdfContentReaderTests.Refuses_to_copy_an_image_whose_flate_data_is_damaged` | | A free text annotation was written without the `/DA` ISO 32000-1 requires, and annotation appearances without `/Resources` — both reported by the Arlington check's first run | `/DA` states the font, size and colour the appearance draws with; every appearance states its resources, empty if it needs none | `PdfDocumentEditorTests.States_the_default_appearance_and_resources_of_a_free_text_annotation`, `States_resources_on_an_appearance_that_needs_none` | +| A table that parsed but whose offsets had moved was used as it was, and reconstruction could not see into object streams | the table is rebuilt when its offsets miss their objects; reconstruction indexes object streams and takes `/Root` from a cross-reference stream's dictionary | `PdfReaderRecoveryTests.Rebuilds_a_table_whose_offsets_no_longer_point_at_their_objects`, `Rebuilds_a_table_from_object_streams_when_the_cross_reference_stream_is_lost`, `Rebuilds_a_document_whose_catalog_lives_in_an_object_stream` | +| AESV2 with a short declared key, wrapped predictor row sizes, truncated `endstream` candidates, ignored generations | sixteen-byte AESV2 keys; 64-bit row sizes on both predictor paths; Adler-32-checked candidates, at most sixteen; a mismatched generation resolves to null | `PdfMergeEngineTests.Opens_an_aesv2_document_whose_crypt_filter_declares_a_short_key`, `PdfPredictorTests.Refuses_a_predicted_row_larger_than_the_engine_decodes`, `PdfStreamParsingTests.Scans_past_a_candidate_whose_compressed_data_is_cut_short`, `PdfReaderRecoveryTests.Resolves_a_reference_to_a_generation_the_table_does_not_list_as_null` | +| A revision appended to a file that uses cross-reference streams ended in a classic table | the revision ends in a cross-reference stream; qpdf and pyHanko accept both an edit and two signatures appended this way | `PdfIncrementalWriterTests.Appends_a_cross_reference_stream_to_a_document_that_uses_them`, `PdfDocumentSignerTests.Signs_a_document_that_uses_cross_reference_streams_with_one` | +| Copying an attachment the listing reported, whose specification embeds no stream, threw `ArgumentException` — the public-corpora job's fault on `isartor-6-1-11-t01-fail-a.pdf` | it writes no bytes, matching the `ByteLength` of zero the listing reports | `PdfContentReaderTests.Copies_nothing_for_a_listed_file_that_embeds_no_stream` | ## Corpus readings, 2026-11-16 diff --git a/src/SharpCorePDF.Domain/Ports/IPdfContentReader.cs b/src/SharpCorePDF.Domain/Ports/IPdfContentReader.cs index 3d0f975..d0a273a 100644 --- a/src/SharpCorePDF.Domain/Ports/IPdfContentReader.cs +++ b/src/SharpCorePDF.Domain/Ports/IPdfContentReader.cs @@ -190,10 +190,14 @@ Task> ReadEmbeddedFilesAsync( /// A task that completes when the bytes have been written. /// /// The bounded route, with the same contract as : a pooled scratch - /// buffer, the destination flushed and left open. + /// buffer, the destination flushed and left open. A file whose specification embeds no stream — one that + /// only names a file outside the document, listed with a warning and a ByteLength of zero — writes + /// no bytes. /// /// An argument is . - /// is not writable. + /// + /// is not writable, or is not a file of the document. + /// /// The content is not a readable PDF document. Task CopyEmbeddedFileDataAsync( IPdfContentSource source, diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReference.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReference.cs index 9230afb..e1067bf 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReference.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReference.cs @@ -48,6 +48,12 @@ internal sealed class PdfCrossReferenceTable /// Gets a value indicating whether the table was rebuilt by scanning the file. public bool WasReconstructed { get; internal set; } + /// + /// Gets a value indicating whether the section startxref points at is a cross-reference stream, so + /// a revision appended to the document writes its own section in that form. + /// + public bool NewestSectionIsStream { get; internal set; } + /// Gets the number of entries. public int Count => Entries.Count; diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.Scan.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.Scan.cs index f923423..d0218ed 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.Scan.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.Scan.cs @@ -25,6 +25,7 @@ private void Reconstruct() } var found = ScanForObjects(); + IndexObjectStreams(); var trailer = FindLastTrailer(); if (trailer is not null) @@ -136,7 +137,129 @@ private bool TryReadBackwardsInteger(ref int cursor, out long value) index = index == 0 ? -1 : data.AsSpan(0, index).LastIndexOf("trailer"u8); } - return FindCatalogTrailer(); + return FindCrossReferenceStreamTrailer() ?? FindCatalogTrailer(); + } + + /// + /// Registers the objects the object streams found by the scan hold. A PDF 1.5 file keeps most of its + /// objects — often its catalog and its pages — inside object streams, which a scan for n g obj + /// headers cannot see into, so without this a file whose cross-reference stream is lost has nothing left. + /// + /// + /// An object the scan found on its own stays where the scan found it: entries are only added, never + /// replaced, because an object written uncompressed after an object stream is the newer of the two. + /// + private void IndexObjectStreams() + { + foreach (var container in table.Entries.Values.Where(entry => entry.Type == XrefEntryType.Uncompressed).ToList()) + { + if (!MentionsObjectStream(container.Offset) + || ParseAt(container.Offset) is not PdfStream { Dictionary: var dictionary } stream + || dictionary.GetName("Type") != "ObjStm") + { + continue; + } + + byte[] decoded; + + try + { + decoded = PdfFilters.Decode(stream, sourceName, options.MaxDecodedStreamBytes); + } + catch (PdfException) + { + continue; + } + + var count = Math.Min(dictionary.GetInteger("N") ?? 0, (decoded.Length / 4L) + 1); + var header = new PdfObjectParser(decoded, sourceName, 0, null, options.MaxNestingDepth); + + for (var index = 0; index < count; index++) + { + if (!header.TryReadInteger(out var objectNumber) || !header.TryReadInteger(out _)) + { + break; + } + + if (objectNumber > 0) + { + table.Add(new XrefEntry(objectNumber, container.ObjectNumber, index, XrefEntryType.Compressed)); + } + } + } + } + + /// Determines cheaply whether the object at an offset names itself an object stream. + /// The offset of the object header. + /// when /ObjStm appears in the first bytes of the object. + private bool MentionsObjectStream(long offset) + { + if (offset < 0 || offset >= data.Length) + { + return false; + } + + var window = data.AsSpan((int)offset, (int)Math.Min(512, data.Length - offset)); + + return window.IndexOf("/ObjStm"u8) >= 0; + } + + /// Parses the object whose header is at an offset. + /// The offset. + /// The object, or . + private PdfObject? ParseAt(long offset) + { + var parser = new PdfObjectParser(data, sourceName, (int)offset, null, options.MaxNestingDepth); + + return parser.TryReadIndirectHeader(out _, out _) ? parser.ParseObject() : null; + } + + /// + /// Takes the trailer entries from the last cross-reference stream the scan found: a PDF 1.5 file has no + /// trailer keyword, and its /Root lives in the stream's dictionary, which survives when the + /// stream's data does not. + /// + /// A trailer carrying /Root, or . + private PdfDictionary? FindCrossReferenceStreamTrailer() + { + foreach (var offset in table.Entries.Values + .Where(entry => entry.Type == XrefEntryType.Uncompressed) + .Select(entry => entry.Offset) + .OrderByDescending(offset => offset)) + { + var window = data.AsSpan((int)offset, (int)Math.Min(512, data.Length - offset)); + + if (window.IndexOf("/XRef"u8) < 0) + { + continue; + } + + var dictionary = ParseAt(offset) switch + { + PdfStream stream => stream.Dictionary, + PdfDictionary plain => plain, + _ => null, + }; + + if (dictionary is null || dictionary.GetName("Type") != "XRef" || dictionary.GetReference("Root") is null) + { + continue; + } + + var trailer = new PdfDictionary(); + + foreach (var key in new[] { "Root", "Info", "ID", "Encrypt" }) + { + if (dictionary.Get(key) is { } value) + { + trailer.Set(key, value); + } + } + + return trailer; + } + + return null; } /// Looks for a document catalog and uses it as the root of a synthetic trailer. diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.cs index 9a591d2..7d04ddd 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.cs @@ -66,7 +66,12 @@ public PdfCrossReferenceTable Read() } } - if (table.Count == 0 || !table.Trailer.ContainsKey("Root")) + // A table that reads cleanly can still be wrong: bytes inserted or removed after it was written — a + // converted line ending, a prepended byte order mark — move every object, and each offset then lands + // in the middle of something else. Such a table is rebuilt like a table that did not parse. + if (table.Count == 0 + || !table.Trailer.ContainsKey("Root") + || (options.ReconstructCrossReferenceOnFailure && !OffsetsLandOnObjects())) { Reconstruct(); } @@ -81,6 +86,56 @@ public PdfCrossReferenceTable Read() return table; } + /// + /// Determines whether the table's offsets land on the objects they name: the catalog's, and a sample of + /// the others, each has to point at the n g obj header of its own number. + /// + /// when the catalog's offset misses, or most of the sample does. + private bool OffsetsLandOnObjects() + { + const int Sample = 32; + + if (table.Trailer.GetReference("Root") is { } root + && table.Get(root.ObjectNumber) is { Type: XrefEntryType.Uncompressed } rootEntry + && !HeaderNames(rootEntry)) + { + return false; + } + + var sampled = 0; + var missed = 0; + + foreach (var entry in table.Entries.Values + .Where(entry => entry.Type == XrefEntryType.Uncompressed) + .OrderBy(entry => entry.ObjectNumber) + .Take(Sample)) + { + sampled++; + + if (!HeaderNames(entry)) + { + missed++; + } + } + + return missed * 2 <= sampled; + } + + /// Determines whether an entry's offset is the header of the object it names. + /// The entry. + /// when the bytes at the offset are n g obj for its own number. + private bool HeaderNames(XrefEntry entry) + { + if (entry.Offset < 0 || entry.Offset >= data.Length) + { + return false; + } + + var parser = new PdfObjectParser(data, sourceName, (int)entry.Offset, null, options.MaxNestingDepth); + + return parser.TryReadIndirectHeader(out var objectNumber, out _) && objectNumber == entry.ObjectNumber; + } + private long? FindStartXref() { var searchStart = Math.Max(0, data.Length - 4096); @@ -106,8 +161,13 @@ public PdfCrossReferenceTable Read() { var probe = new PdfLexer(data, (int)offset); - return probe.TryReadKeyword("xref") - ? ReadClassicTable((int)offset, isNewest) - : ReadCrossReferenceStream((int)offset, isNewest); + if (probe.TryReadKeyword("xref")) + { + return ReadClassicTable((int)offset, isNewest); + } + + table.NewestSectionIsStream |= isNewest; + + return ReadCrossReferenceStream((int)offset, isNewest); } } diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceStreamWriter.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceStreamWriter.cs new file mode 100644 index 0000000..c9f086c --- /dev/null +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceStreamWriter.cs @@ -0,0 +1,128 @@ +namespace SharpCorePDF.Infrastructure.Pdf; + +/// +/// Writes the cross-reference section of an appended revision as a cross-reference stream, for a document +/// whose newest section is one. +/// +/// +/// A revision appended to a file that uses cross-reference streams used to end in a classic table whose +/// /Prev pointed at a stream. Most readers follow such a chain, but ISO 32000-1 only describes a +/// classic table chained to a classic table, a stream to a stream, and the hybrid form of 7.5.8.4, so the +/// revision is written in the form the document already uses. The stream is written uncompressed and in the +/// clear: a cross-reference stream is never encrypted, and a revision holds a few entries at most. +/// +internal static class PdfCrossReferenceStreamWriter +{ + /// The width of the generation field, which holds up to 65535. + private const int GenerationWidth = 2; + + /// Writes the stream, the startxref that points at it and the closing %%EOF. + /// The buffer the revision is written to. + /// The absolute offset the stream object is written at. + /// The objects of the revision: number, generation and absolute offset. + /// The trailer entries, which become entries of the stream's dictionary; its Size is corrected here. + /// Whether the section lists the head of the free list, object zero, as a classic table of a complete file does. + public static void Write( + PdfOutputBuffer output, + long startXref, + IReadOnlyCollection<(int Number, int Generation, long Offset)> entries, + PdfDictionary trailer, + bool freeHead) + { + ArgumentNullException.ThrowIfNull(output); + ArgumentNullException.ThrowIfNull(entries); + ArgumentNullException.ThrowIfNull(trailer); + + var size = (int)Math.Max(trailer.GetInteger("Size") ?? 0, 1); + + foreach (var entry in entries) + { + size = Math.Max(size, entry.Number + 1); + } + + // The stream is an object of the revision too, numbered above every other, and lists itself. + var streamNumber = size; + var rows = entries + .Select(entry => (entry.Number, Type: (byte)1, Field2: entry.Offset, Field3: entry.Generation)) + .Append((Number: streamNumber, Type: (byte)1, Field2: startXref, Field3: 0)); + + if (freeHead) + { + rows = rows.Append((Number: 0, Type: (byte)0, Field2: 0L, Field3: 65535)); + } + + var sorted = rows.OrderBy(row => row.Number).ToArray(); + var offsetWidth = BytesFor(startXref); + var data = new byte[sorted.Length * (1 + offsetWidth + GenerationWidth)]; + var index = new List(); + var position = 0; + + for (var row = 0; row < sorted.Length; row++) + { + if (row == 0 || sorted[row].Number != sorted[row - 1].Number + 1) + { + index.Add(new PdfInteger(sorted[row].Number)); + index.Add(new PdfInteger(0)); + } + + var count = (PdfInteger)index[^1]; + index[^1] = new PdfInteger(count.Value + 1); + + data[position++] = sorted[row].Type; + WriteBigEndian(data.AsSpan(position, offsetWidth), sorted[row].Field2); + position += offsetWidth; + WriteBigEndian(data.AsSpan(position, GenerationWidth), sorted[row].Field3); + position += GenerationWidth; + } + + var dictionary = new PdfDictionary().SetName("Type", "XRef"); + + foreach (var entry in trailer.Entries.Where(entry => entry.Key is not ("Size" or "XRefStm"))) + { + dictionary.Set(entry.Key, entry.Value); + } + + dictionary + .SetInteger("Size", streamNumber + 1) + .Set("W", new PdfArray([new PdfInteger(1), new PdfInteger(offsetWidth), new PdfInteger(GenerationWidth)])) + .Set("Index", new PdfArray(index)) + .SetInteger("Length", data.Length); + + output.WriteInteger(streamNumber); + output.Write(" 0 obj\n"u8); + PdfSerializer.WriteDictionary(dictionary, output); + output.Write("\nstream\n"u8); + output.Write(data); + output.Write("\nendstream\nendobj\nstartxref\n"u8); + output.WriteInteger(startXref); + output.Write("\n%%EOF\n"u8); + output.Flush(); + } + + /// Gets the number of bytes a big-endian field needs to hold a value. + /// The largest value of the field. + /// The width, at least one byte. + private static int BytesFor(long value) + { + var width = 1; + + while (width < 8 && value >> (8 * width) != 0) + { + width++; + } + + return width; + } + + /// Writes a value into a field, most significant byte first. + /// The field. + /// The value, which fits the field. + private static void WriteBigEndian(Span field, long value) + { + for (var index = field.Length - 1; index >= 0; index--) + { + field[index] = (byte)value; + value >>= 8; + } + } +} diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.cs index fcec760..11ee20e 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.cs @@ -187,8 +187,29 @@ public PdfDictionary? Catalog /// Resolves an indirect reference. /// The reference. - /// The object, or . - public PdfObject? Resolve(PdfReference? reference) => reference is null ? null : Resolve(reference.ObjectNumber); + /// + /// The object, or — also for a generation other than the one the table lists for + /// the number, which names an object that no longer exists (ISO 32000-1, 7.3.10). + /// + /// + /// Only an entry of the table states a generation: an object in an object stream has generation zero by + /// definition, and the entry of a compressed object holds its index there instead, so it is not compared. + /// + public PdfObject? Resolve(PdfReference? reference) + { + if (reference is null) + { + return null; + } + + if (CrossReferenceTable.Get(reference.ObjectNumber) is { Type: XrefEntryType.Uncompressed } entry + && entry.Generation != reference.Generation) + { + return null; + } + + return Resolve(reference.ObjectNumber); + } /// Resolves a value that may be a reference, leaving anything else as it is. /// The value. diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.Inflation.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.Inflation.cs index 8fe69f2..7672c78 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.Inflation.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.Inflation.cs @@ -38,6 +38,43 @@ private enum InflateOutcome public static bool TryInflate(ReadOnlyMemory data, out byte[] result) => TryInflate(data, DefaultMaxDecodedBytes, out result) == InflateOutcome.Inflated; + /// + /// Determines whether zlib data ends with the Adler-32 checksum of what it inflated to, which only data + /// that is complete does: a deflate stream cut short inflates without complaint to the bytes before the + /// cut, and the four bytes in front of the cut are then not its checksum. + /// + /// The zlib data, possibly followed by the end of line a writer put behind it. + /// The bytes it inflated to. + /// when the checksum is found at the end of the data. + internal static bool EndsWithAdler32Of(ReadOnlySpan compressed, ReadOnlySpan inflated) + { + const uint Modulus = 65521; + + uint low = 1; + uint high = 0; + + // 5552 bytes is the longest run the two sums can take before they have to be reduced to stay in 32 bits. + while (!inflated.IsEmpty) + { + var run = inflated[..Math.Min(5552, inflated.Length)]; + + foreach (var value in run) + { + low += value; + high += low; + } + + low %= Modulus; + high %= Modulus; + inflated = inflated[run.Length..]; + } + + Span checksum = [(byte)(high >> 8), (byte)high, (byte)(low >> 8), (byte)low]; + var trimmed = compressed.TrimEnd("\r\n \t\0"u8); + + return trimmed.EndsWith(checksum); + } + /// Decompresses Flate data under the default limit on its decoded size. /// The compressed bytes. /// The name of the source, used in diagnostics. diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.Predictors.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.Predictors.cs index 859db4d..eb6ef64 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.Predictors.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.Predictors.cs @@ -33,15 +33,16 @@ private static byte[] ApplyPredictor( ValidatePredictor(predictor, bitsPerComponent, sourceName); + var rowLength = PredictedRowLength(columns, colors, bitsPerComponent, sourceName); + if (predictor == 2) { - return ApplyTiffPredictor(data, colors, columns); + return ApplyTiffPredictor(data, colors, rowLength); } var bytesPerPixel = Math.Max(1, colors * bitsPerComponent / 8); - var rowLength = (columns * colors * bitsPerComponent + 7) / 8; - if (rowLength <= 0 || data.Length <= rowLength) + if (data.Length <= rowLength) { return data; } @@ -192,10 +193,8 @@ private static void UndoPaeth(Span current, ReadOnlySpan previous, i } } - private static byte[] ApplyTiffPredictor(byte[] data, int colors, int columns) + private static byte[] ApplyTiffPredictor(byte[] data, int colors, int rowLength) { - var rowLength = columns * colors; - for (var row = 0; row + rowLength <= data.Length; row += rowLength) { UndoTiffRow(data.AsSpan(row, rowLength), colors); @@ -292,6 +291,32 @@ private static void ValidatePredictor(int predictor, int bitsPerComponent, strin /// private const int MaximumPredictedRowBytes = 8 * 1024 * 1024; + /// Gets the length of a predicted row, refusing one the engine does not decode. + /// The samples in a row, at least one. + /// The components of a pixel, at least one. + /// The bits of each sample. + /// The name of the source, used in diagnostics. + /// The bytes of a row, without the filter type byte PNG prediction adds. + /// The row is empty, negative or larger than the engine decodes. + /// + /// The product is taken in 64 bits: in 32 the three declared values wrap, and a row of 65536 columns of + /// 65536 colours came out as zero bytes, on which a row loop never advances. + /// + private static int PredictedRowLength(int columns, int colors, int bitsPerComponent, string sourceName) + { + var bits = (long)columns * colors * bitsPerComponent; + + if (bits <= 0 || bits > MaximumPredictedRowBytes * 8L) + { + throw new PdfUnsupportedFeatureException( + sourceName, + "Predictor", + "the predicted row is larger than the engine decodes."); + } + + return (int)((bits + 7) / 8); + } + /// Receives one undone row of a predicted payload. /// The row, which is pooled and valid only for the duration of the call. internal delegate void RowHandler(ReadOnlySpan row); @@ -337,17 +362,7 @@ public PredictorRowDecoder( this.colors = colors; this.sourceName = sourceName; - var bits = (long)columns * colors * bitsPerComponent; - - if (bits <= 0 || bits > MaximumPredictedRowBytes * 8L) - { - throw new PdfUnsupportedFeatureException( - sourceName, - "Predictor", - "the predicted row is larger than the engine decodes."); - } - - rowLength = (int)((bits + 7) / 8); + rowLength = PredictedRowLength(columns, colors, bitsPerComponent, sourceName); bytesPerPixel = Math.Max(1, colors * bitsPerComponent / 8); stride = tiff ? rowLength : rowLength + 1; diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfIncrementalWriter.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfIncrementalWriter.cs index 13778e7..70fc140 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfIncrementalWriter.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfIncrementalWriter.cs @@ -92,11 +92,24 @@ public long WriteObject(int objectNumber, int generation, PdfObject body) /// Writes the cross-reference section, the trailer and the closing %%EOF. /// The trailer dictionary, which must already carry /Root and /Prev. - public void Complete(PdfDictionary trailer) + /// Whether the section is written as a cross-reference stream, the form the document it is appended to uses. + public void Complete(PdfDictionary trailer, bool asStream = false) { ArgumentNullException.ThrowIfNull(trailer); var startXref = Length; + + if (asStream) + { + PdfCrossReferenceStreamWriter.Write( + output, + startXref, + entries.Select(entry => (entry.Number, entry.Generation, entry.Offset)).ToArray(), + trailer, + freeHead: false); + return; + } + output.Write("xref\n"u8); WriteSubsections(); diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfObjectParser.Streams.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfObjectParser.Streams.cs index ece8c88..c154761 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfObjectParser.Streams.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfObjectParser.Streams.cs @@ -91,13 +91,23 @@ private bool EndStreamFollows(int position) _ => null, }; + /// + /// The most endstream candidates a compressed stream is inflated for. Each is inflated from the + /// start of the data, so the work grows with the square of the candidates, and a hostile file can hold + /// one every ten bytes. + /// + private const int MaximumInflatedCandidates = 16; + private PdfStream ReadStreamByScanning(PdfDictionary dictionary, int dataStart) { var data = lexer.Data; var isCompressed = PdfFilters.HasFlateCompression(dictionary); var searchFrom = dataStart; + var inflated = 0; ReadOnlyMemory? firstCandidate = null; var firstCandidateEnd = dataStart; + ReadOnlyMemory? inflatingCandidate = null; + var inflatingCandidateEnd = dataStart; while (searchFrom < data.Length) { @@ -118,20 +128,46 @@ private PdfStream ReadStreamByScanning(PdfDictionary dictionary, int dataStart) firstCandidateEnd = index + 9; } + if (!isCompressed) + { + return EndScannedStream(dictionary, candidate, index + 9); + } + // The candidate has to be decrypted before it can be decompressed, because the stream of - // an encrypted document is deflated first and encrypted second. - if (!isCompressed || PdfFilters.TryInflate(candidate, out _)) + // an encrypted document is deflated first and encrypted second. A deflate stream cut short + // inflates without complaint to the bytes in front of the cut, so a candidate is taken at once + // only when its data is complete; the first that merely inflates is kept for when none is. + if (inflated++ < MaximumInflatedCandidates && PdfFilters.TryInflate(candidate, out var decoded)) { - Position = index + 9; - TryReadKeyword("endstream"); - return new PdfStream(dictionary, candidate); + if (PdfFilters.EndsWithAdler32Of(candidate.Span, decoded)) + { + return EndScannedStream(dictionary, candidate, index + 9); + } + + if (inflatingCandidate is null) + { + inflatingCandidate = candidate; + inflatingCandidateEnd = index + 9; + } } searchFrom = index + 9; } - Position = firstCandidateEnd; - return new PdfStream(dictionary, firstCandidate ?? ReadOnlyMemory.Empty); + return inflatingCandidate is { } inflating + ? EndScannedStream(dictionary, inflating, inflatingCandidateEnd) + : EndScannedStream(dictionary, firstCandidate ?? ReadOnlyMemory.Empty, firstCandidateEnd); + } + + /// Ends a stream the scan delimited, behind the endstream keyword it chose. + /// The stream dictionary. + /// The body the scan chose. + /// The offset just behind the keyword that ends the body. + /// The stream. + private PdfStream EndScannedStream(PdfDictionary dictionary, ReadOnlyMemory body, int keywordEnd) + { + Position = keywordEnd; + return new PdfStream(dictionary, body); } /// Removes the end of line that a writer puts between the data and the keyword. diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfSecurityHandler.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfSecurityHandler.cs index 2a02185..65425db 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfSecurityHandler.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfSecurityHandler.cs @@ -201,7 +201,10 @@ public static PdfDecryptionResult TryCreate( // The encryption dictionary states the key length in bits; a crypt filter of revision 4 and 5 // states its own, in bytes, and it is the one the file key is actually derived with. var declaredBits = (int)(encryption.GetInteger("Length") ?? 40); - var keyBits = version >= 4 && cryptFilterBits is { } filterBits ? filterBits : declaredBits; + // AESV2 is AES-128 (ISO 32000-1, table 25): its key is sixteen bytes whatever length is declared, and a + // file that declares fewer was encrypted with sixteen by every producer that writes AESV2. + var statedBits = version >= 4 && cryptFilterBits is { } filterBits ? filterBits : declaredBits; + var keyBits = stringMethod == PdfCryptMethod.AesV2 || streamMethod == PdfCryptMethod.AesV2 ? 128 : statedBits; var encryptsMetadata = ReadEncryptsMetadata(encryption); byte[]? fileKey; diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfWriter.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfWriter.cs index d3cd0b7..f108a11 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfWriter.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfWriter.cs @@ -157,11 +157,25 @@ public int WriteEncryptionDictionary(PdfSecurityWriter security) /// Writes the cross-reference table, trailer and the closing startxref. /// The trailer dictionary; its Size entry is corrected here. - public void Complete(PdfDictionary trailer) + /// Whether the section is written as a cross-reference stream, the form a revision appended to a document that uses them takes. + public void Complete(PdfDictionary trailer, bool asStream = false) { ArgumentNullException.ThrowIfNull(trailer); var startXref = Length; + + if (asStream) + { + trailer.SetInteger("Size", Math.Max(trailer.GetInteger("Size") ?? 0, lastObjectNumber + 1)); + PdfCrossReferenceStreamWriter.Write( + output, + startXref, + offsets.Select(entry => (entry.Key, 0, entry.Value)).ToArray(), + trailer, + freeHead: true); + return; + } + output.Write("xref\n"u8); WriteSubsections(); diff --git a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Document.cs b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Document.cs index 93a0682..2117fb1 100644 --- a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Document.cs +++ b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Document.cs @@ -733,6 +733,7 @@ private static int WriteInformation(PdfWriter writer, PdfMetadata metadata, int? /// The encryption of the result, or when it is written in the clear. /// The cross-reference section this revision chains to, or for a complete document. /// The first element of the source's ID, kept by an appended revision, or . + /// Whether the section is a cross-reference stream, because the document the revision is appended to uses them. private static void WriteTrailer( PdfWriter writer, int catalogNumber, @@ -740,7 +741,8 @@ private static void WriteTrailer( PdfString identifier, PdfSecurityWriter? security, long? previousXref, - PdfString? permanentIdentifier = null) + PdfString? permanentIdentifier = null, + bool asStream = false) { var trailer = new PdfDictionary(); trailer.Set("Root", new PdfReference(catalogNumber, 0)); @@ -767,7 +769,7 @@ private static void WriteTrailer( trailer.Set("Encrypt", new PdfReference(writer.WriteEncryptionDictionary(security), 0)); } - writer.Complete(trailer); + writer.Complete(trailer, asStream); } /// Builds the identifier written twice in the trailer of the result. diff --git a/src/SharpCorePDF.Infrastructure/PdfEditEngine.cs b/src/SharpCorePDF.Infrastructure/PdfEditEngine.cs index c6d4751..7fb4d3b 100644 --- a/src/SharpCorePDF.Infrastructure/PdfEditEngine.cs +++ b/src/SharpCorePDF.Infrastructure/PdfEditEngine.cs @@ -318,7 +318,8 @@ [.. layout.Select(page => page.TargetNumber)], identifier, security, revision?.PreviousXref, - primary is null ? null : PermanentIdentifier(primary)); + primary is null ? null : PermanentIdentifier(primary), + primary?.CrossReferenceTable.NewestSectionIsStream == true); Verify(capture, layout, plan.Output.Encryption?.UserPassword, warnings); diff --git a/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Metadata.cs b/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Metadata.cs index 3681e29..4525962 100644 --- a/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Metadata.cs +++ b/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Metadata.cs @@ -179,15 +179,23 @@ public async Task CopyEmbeddedFileDataAsync( } var reader = await OpenAsync(sourceName, content, password, cancellationToken).ConfigureAwait(false); - var stream = FindEmbeddedStream(reader, file.Id); - if (stream is null) + if (!TryFindEmbeddedStream(reader, file.Id, out var stream)) { throw new ArgumentException( $"'{file.Id}' is not an embedded file of '{sourceName}'.", nameof(file)); } + // A specification that only names a file outside the document is listed, with a warning and a length + // of zero, and copies as what it holds: nothing. Refusing it as a caller's mistake contradicted the + // listing the caller took it from. + if (stream is null) + { + await destination.FlushAsync(cancellationToken).ConfigureAwait(false); + return; + } + await PdfImageFilters .CopyAsync(stream, destination, reader.SourceName, cancellationToken) .ConfigureAwait(false); @@ -250,9 +258,12 @@ private static void CollectNameTree( /// Finds the embedded stream whose identifier is the one asked for. /// The document. /// The identifier of the file. - /// The stream, or when the document has no such file. - private static PdfStream? FindEmbeddedStream(PdfFileReader reader, string id) + /// The stream, or when the file's specification embeds none. + /// when the document lists a file with that identifier. + private static bool TryFindEmbeddedStream(PdfFileReader reader, string id, out PdfStream? stream) { + stream = null; + var entries = new List>(); var names = reader.ResolveDictionary(reader.Catalog?.Get("Names")); @@ -282,11 +293,12 @@ private static void CollectNameTree( if (string.Equals(identifier, id, StringComparison.Ordinal)) { - return FindEmbeddedStream(reader, specification); + stream = FindEmbeddedStream(reader, specification); + return true; } } - return null; + return false; } /// Reads the name a file specification gives the file. diff --git a/src/SharpCorePDF.Infrastructure/Signatures/SignatureFieldWriter.cs b/src/SharpCorePDF.Infrastructure/Signatures/SignatureFieldWriter.cs index df28413..9d98ca9 100644 --- a/src/SharpCorePDF.Infrastructure/Signatures/SignatureFieldWriter.cs +++ b/src/SharpCorePDF.Infrastructure/Signatures/SignatureFieldWriter.cs @@ -155,7 +155,7 @@ public static async Task WriteAsync( writer.WriteObject(widgetNumber, 0, Widget(options, pageNumber, signatureReference, appearanceNumber)); writer.WriteObject(pageNumber, 0, PageWithWidget(reader, page, widgetReference)); writer.WriteObject(catalogNumber, 0, new PdfDictionary(catalog.Entries).Set("AcroForm", AcroForm(reader, catalog, widgetReference))); - writer.Complete(Trailer(reader, catalogNumber)); + writer.Complete(Trailer(reader, catalogNumber), asStream: reader.CrossReferenceTable.NewestSectionIsStream); var digest = SealRevision(buffer, byteRangeOffset, contentsStart, hexLength, options.HashAlgorithm); var value = await CmsBuilder.BuildAsync(digest, options, certificates, signingTime, cancellationToken).ConfigureAwait(false); diff --git a/tests/SharpCorePDF.Tests/PdfContentReaderTests.Metadata.cs b/tests/SharpCorePDF.Tests/PdfContentReaderTests.Metadata.cs index 151ae8d..6444f85 100644 --- a/tests/SharpCorePDF.Tests/PdfContentReaderTests.Metadata.cs +++ b/tests/SharpCorePDF.Tests/PdfContentReaderTests.Metadata.cs @@ -103,6 +103,42 @@ public async Task Copies_an_embedded_file_through_the_copy_route() Assert.Equal("second", Encoding.Latin1.GetString(materialised.Span)); } + /// + /// A file specification in the embedded files tree that only names a file outside the document — the shape + /// of the veraPDF corpus file isartor-6-1-11-t01-fail-a.pdf — is listed with a warning and a length + /// of zero, and copying it writes no bytes; it used to be refused as if the caller had passed a file of + /// another document. A file the document does not have is still refused. + /// + [Fact] + public async Task Copies_nothing_for_a_listed_file_that_embeds_no_stream() + { + var builder = new PdfBuilder(); + var catalog = builder.Reserve(); + var pages = builder.Reserve(); + var page = builder.Add("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] >>"); + var specification = builder.Add("<< /Type /Filespec /F (img/pageA4.pdf) >>"); + + builder.Set(catalog, $"<< /Type /Catalog /Pages {pages} 0 R /Names << /EmbeddedFiles << /Names [(pageA4.pdf) {specification} 0 R] >> >> >>"); + builder.Set(pages, $"<< /Type /Pages /Kids [{page} 0 R] /Count 1 >>"); + + var source = Source(builder.Build(catalog), "external.pdf"); + var reader = Reader(); + var result = await reader.ReadEmbeddedFilesAsync(source, cancellationToken: Token); + var file = Assert.Single(result.Items); + + using var destination = new MemoryStream(); + + await reader.CopyEmbeddedFileDataAsync(source, file, destination, cancellationToken: Token); + + Assert.Equal("img/pageA4.pdf", file.Id); + Assert.Equal(0, file.ByteLength); + Assert.Empty(destination.ToArray()); + AssertWarning(result.Warnings, PdfWarningKind.EmbeddedFileUnreadable); + + await Assert.ThrowsAsync( + () => reader.CopyEmbeddedFileDataAsync(source, file with { Id = "other.pdf" }, Stream.Null, cancellationToken: Token)); + } + /// A document without attachments yields an empty result rather than an exception. [Fact] public async Task Returns_an_empty_result_for_a_document_without_embedded_files() diff --git a/tests/SharpCorePDF.Tests/PdfDocumentSignerTests.cs b/tests/SharpCorePDF.Tests/PdfDocumentSignerTests.cs index 828cfe3..a8a6c63 100644 --- a/tests/SharpCorePDF.Tests/PdfDocumentSignerTests.cs +++ b/tests/SharpCorePDF.Tests/PdfDocumentSignerTests.cs @@ -3,6 +3,7 @@ using SharpCorePDF.Domain.Ports; using SharpCorePDF.Domain.ValueObjects; using SharpCorePDF.Infrastructure; +using SharpCorePDF.Infrastructure.Pdf; namespace SharpCorePDF.Tests; @@ -86,6 +87,25 @@ public async Task Appends_a_revision_and_leaves_the_source_bytes_untouched() Assert.Contains(" /SigFlags 3", text, StringComparison.Ordinal); } + /// + /// A signature appended to a document whose newest section is a cross-reference stream ends its revision + /// in a stream too, and the signature is valid over the whole file; a classic table used to be appended. + /// + [Fact] + public async Task Signs_a_document_that_uses_cross_reference_streams_with_one() + { + var source = File.ReadAllBytes(TestAssets.Modern); + var signed = await SigningFixtures.SignAsync(source, SigningFixtures.Options(Certificate)); + + var reader = new PdfFileReader(signed, "signed.pdf", PdfEngineOptions.Default); + var validation = Assert.Single(await SigningFixtures.ValidateAsync(signed)); + + Assert.True(reader.CrossReferenceTable.NewestSectionIsStream); + Assert.False(reader.CrossReferenceTable.WasReconstructed); + Assert.Equal(PdfSignatureIntegrity.Valid, validation.Integrity); + Assert.True(validation.Signer.CoversWholeDocument); + } + /// /// The second signature appends a revision of its own, and the first one still covers exactly the bytes it /// signed: intact, and no longer the whole file. diff --git a/tests/SharpCorePDF.Tests/PdfIncrementalWriterTests.cs b/tests/SharpCorePDF.Tests/PdfIncrementalWriterTests.cs index e535b94..eff4bf2 100644 --- a/tests/SharpCorePDF.Tests/PdfIncrementalWriterTests.cs +++ b/tests/SharpCorePDF.Tests/PdfIncrementalWriterTests.cs @@ -136,6 +136,44 @@ public async Task Chains_the_revision_to_the_cross_reference_section_of_the_sour Assert.Contains(ending, Encoding.Latin1.GetString(edited), StringComparison.Ordinal); } + /// + /// A revision appended to a document whose newest section is a cross-reference stream ends in a stream + /// too, chained to the source's, listing itself, and every page of the result reads; a classic table used + /// to be appended to such a document, a chain ISO 32000-1 does not describe. + /// + [Fact] + public async Task Appends_a_cross_reference_stream_to_a_document_that_uses_them() + { + var source = File.ReadAllBytes(TestAssets.Modern); + var (_, edited) = await ApplyAsync(source, Incremental(PdfEditPlan.Create().SetMetadata(new PdfMetadata { Title = "streamed" }))); + + var offset = StartXref(edited); + var revision = Encoding.Latin1.GetString(edited, source.Length, edited.Length - source.Length); + var reader = new PdfFileReader(edited, "edited.pdf", Options); + var section = Assert.IsType(reader.Resolve(reader.CrossReferenceTable.Trailer.GetInteger("Size")!.Value - 1)); + + Assert.Equal(source, edited[..source.Length]); + Assert.True(offset > source.Length); + Assert.DoesNotContain("\nxref\n", revision, StringComparison.Ordinal); + Assert.DoesNotContain("trailer", revision, StringComparison.Ordinal); + Assert.Equal("XRef", section.Dictionary.GetName("Type")); + Assert.Equal(StartXref(source), section.Dictionary.GetInteger("Prev")); + Assert.True(reader.CrossReferenceTable.NewestSectionIsStream); + Assert.False(reader.CrossReferenceTable.WasReconstructed); + Assert.Equal("streamed", (await new PdfDocumentReader(Options).ReadAsync("edited.pdf", new MemoryStream(edited), null, Token)).Metadata.Title); + } + + /// A revision appended to a document with a classic table keeps writing a classic table. + [Fact] + public async Task Appends_a_classic_table_to_a_document_that_uses_one() + { + var source = ThreePages(); + var (_, edited) = await ApplyAsync(source, Incremental(PdfEditPlan.Create().DeletePage(2))); + + Assert.False(new PdfFileReader(source, "source.pdf", Options).CrossReferenceTable.NewestSectionIsStream); + Assert.False(new PdfFileReader(edited, "edited.pdf", Options).CrossReferenceTable.NewestSectionIsStream); + } + /// The offsets of the revision are absolute in a destination that can seek. [Fact] public async Task Counts_offsets_from_the_start_of_a_seekable_destination() diff --git a/tests/SharpCorePDF.Tests/PdfMergeEngineTests.OutputEncryption.cs b/tests/SharpCorePDF.Tests/PdfMergeEngineTests.OutputEncryption.cs index 3c48f8c..79fdd06 100644 --- a/tests/SharpCorePDF.Tests/PdfMergeEngineTests.OutputEncryption.cs +++ b/tests/SharpCorePDF.Tests/PdfMergeEngineTests.OutputEncryption.cs @@ -260,6 +260,28 @@ await MergeEncryptedAsync(rc4, moment), Assert.Equal(3, (await ReadAsync(second, MergedUserPassword)).PageCount); } + /// + /// AESV2 is AES-128 whatever key length its crypt filter declares, so a file that declares five bytes still + /// opens with its password: the key used to be cut to the declared length, which no AES-128 producer used. + /// + [Fact] + public async Task Opens_an_aesv2_document_whose_crypt_filter_declares_a_short_key() + { + var bytes = await MergeEncryptedAsync(Encryption(PdfEncryptionAlgorithm.AesWith128BitKey)); + var text = Encoding.Latin1.GetString(bytes); + var declared = text.IndexOf("/AESV2 /Length 16", StringComparison.Ordinal); + + Assert.True(declared >= 0); + + // The same number of bytes, so every offset of the file stays where it was. + Encoding.Latin1.GetBytes("/AESV2 /Length 5 ").CopyTo(bytes, declared); + + var info = await ReadAsync(bytes, MergedUserPassword); + + Assert.True(info.IsReadable); + Assert.Equal(3, info.PageCount); + } + /// Merges an encrypted source and a plain one into a result with the encryption asked for. /// The encryption to write the result with. /// The moment to stamp the result with, when it has to be reproducible. diff --git a/tests/SharpCorePDF.Tests/PdfPredictorTests.cs b/tests/SharpCorePDF.Tests/PdfPredictorTests.cs index f15d42e..dadd829 100644 --- a/tests/SharpCorePDF.Tests/PdfPredictorTests.cs +++ b/tests/SharpCorePDF.Tests/PdfPredictorTests.cs @@ -150,6 +150,30 @@ public void Reports_tiff_prediction_that_is_not_eight_bit() /// The predicted rows, as hexadecimal. /// The decode parameters of the stream. /// The decoded rows. + /// + /// A row size the declared columns, colours and bit depth multiply out to beyond what the engine decodes is + /// refused, whatever the product wraps to in 32 bits: 65536 × 65536 used to wrap to a row of zero bytes, + /// on which TIFF prediction looped forever, and 65536 × 32769 to a negative one, which escaped as an + /// . The streaming decoder already refused both. + /// + /// The predictor. + /// The declared columns. + /// The declared colours. + /// The declared bit depth. + [Theory(Timeout = 10_000)] + [InlineData(2, 65536, 65536, 8)] + [InlineData(2, 65536, 32769, 8)] + [InlineData(12, 65536, 65536, 8)] + [InlineData(12, 1 << 30, 1, 16)] + [InlineData(12, 4, 1, -8)] + public async Task Refuses_a_predicted_row_larger_than_the_engine_decodes(int predictor, int columns, int colors, int bitsPerComponent) + { + var parameters = Parameters(predictor, columns: columns, colors: colors, bitsPerComponent: bitsPerComponent); + + await Assert.ThrowsAnyAsync( + () => Task.Run(() => Decode("0001020304050607", parameters), TestContext.Current.CancellationToken)); + } + private static byte[] Decode(string encoded, PdfDictionary parameters) => PdfFilters.Decode( Stream(Compress(Convert.FromHexString(encoded)), parameters), diff --git a/tests/SharpCorePDF.Tests/PdfReaderRecoveryTests.cs b/tests/SharpCorePDF.Tests/PdfReaderRecoveryTests.cs index da00411..8666a25 100644 --- a/tests/SharpCorePDF.Tests/PdfReaderRecoveryTests.cs +++ b/tests/SharpCorePDF.Tests/PdfReaderRecoveryTests.cs @@ -1,3 +1,4 @@ +using System.Globalization; using System.Text; using SharpCorePDF.Domain.Exceptions; using SharpCorePDF.Domain.ReadModels; @@ -18,6 +19,108 @@ namespace SharpCorePDF.Tests; /// public sealed class PdfReaderRecoveryTests { + /// + /// A file whose bytes moved after its table was written — here three bytes inserted behind the header, the + /// way a converted line ending or a prepended byte order mark moves them — has a table that parses and + /// offsets that land in the middle of objects. The table is rebuilt instead of the objects going missing. + /// + [Fact] + public void Rebuilds_a_table_whose_offsets_no_longer_point_at_their_objects() + { + var original = TwoPageDocument(); + var header = "%PDF-1.7\n"u8.Length; + var shifted = original[..header].Concat(" "u8.ToArray()).Concat(original[header..]).ToArray(); + + // The startxref is corrected, so only the entries of the table are wrong: the case reconstruction + // used to skip, because the table read cleanly and named a catalog. + var text = Encoding.Latin1.GetString(shifted); + var keyword = text.LastIndexOf("startxref", StringComparison.Ordinal); + var digits = text.IndexOf('\n', keyword) + 1; + var end = text.IndexOf('\n', digits); + var corrected = (long.Parse(text[digits..end], CultureInfo.InvariantCulture) + 3).ToString(CultureInfo.InvariantCulture); + var repaired = Encoding.Latin1.GetBytes(text[..digits] + corrected + text[end..]); + + var reader = new PdfFileReader(repaired, "shifted.pdf", PdfEngineOptions.Default); + + Assert.Equal(2, reader.Pages.Count); + Assert.Contains(reader.Warnings, warning => warning.Kind == PdfWarningKind.CrossReferenceRebuilt); + } + + /// + /// A PDF 1.5 file whose cross-reference stream is lost keeps its page inside an object stream and its + /// catalog reference in the cross-reference stream's dictionary. Rebuilding finds both: the objects an + /// object stream holds, and the /Root of the last cross-reference stream. + /// + [Fact] + public void Rebuilds_a_table_from_object_streams_when_the_cross_reference_stream_is_lost() + { + var page = "3 0 << /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] >>"u8.ToArray(); + var packed = PdfBuilder.Compress(page); + var output = new MemoryStream(); + var offsets = new Dictionary(); + + void Write(string value) => output.Write(Encoding.Latin1.GetBytes(value)); + + void Object(int number, string dictionary, byte[]? stream = null) + { + offsets[number] = output.Position; + Write($"{number} 0 obj\n{dictionary}\n"); + + if (stream is not null) + { + Write("stream\n"); + output.Write(stream); + Write("\nendstream\n"); + } + + Write("endobj\n"); + } + + Write("%PDF-1.7\n"); + Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); + Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + Object(5, $"<< /Type /ObjStm /N 1 /First 4 /Filter /FlateDecode /Length {packed.Length} >>", packed); + + // The cross-reference stream is damaged beyond reading: its data is gone, its dictionary survives. + Object(6, "<< /Type /XRef /Size 7 /W [1 4 2] /Root 1 0 R /Length 0 >>", []); + Write($"startxref\n{offsets[6] + 7}\n%%EOF\n"); + + var reader = new PdfFileReader(output.ToArray(), "lost-xref.pdf", PdfEngineOptions.Default); + + Assert.Single(reader.Pages); + Assert.Equal("Page", Assert.IsType(reader.Resolve(3)).GetName("Type")); + Assert.Contains(reader.Warnings, warning => warning.Kind == PdfWarningKind.CrossReferenceRebuilt); + } + + /// + /// A reference names an object by number and generation, and one whose generation is not the one the + /// table lists for that number refers to an object that no longer exists, which ISO 32000-1 reads as + /// null; the generation used to be ignored, so 3 1 R resolved to 3 0 obj. + /// + [Fact] + public void Resolves_a_reference_to_a_generation_the_table_does_not_list_as_null() + { + var reader = new PdfFileReader(TwoPageDocument(), "generations.pdf", PdfEngineOptions.Default); + + Assert.IsType(reader.Resolve(new PdfReference(3, 0))); + Assert.Null(reader.Resolve(new PdfReference(3, 1))); + Assert.IsType(reader.Resolve(new PdfReference(3, 0))); + } + + private static byte[] TwoPageDocument() + { + var builder = new PdfBuilder(); + var catalog = builder.Reserve(); + var pages = builder.Reserve(); + + builder.Add("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] >>"); + builder.Add("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] >>"); + builder.Set(catalog, "<< /Type /Catalog /Pages 2 0 R >>"); + builder.Set(pages, "<< /Type /Pages /Kids [3 0 R 4 0 R] /Count 2 >>"); + + return builder.Build(catalog); + } + /// /// The bound on a chain of lengths leaves an ordinary indirect length alone, for every stream of a /// document and not only the first ones: each stream's data holds an endstream line of its own, @@ -175,27 +278,29 @@ public async Task Rebuilds_a_table_that_is_not_where_startxref_announces_it() warning => warning.Kind == PdfWarningKind.CrossReferenceRebuilt); } - /// A document whose catalog lives in an object stream cannot be rebuilt by scanning. + /// + /// A document whose catalog lives in an object stream is rebuilt from the object streams the scan finds and + /// the /Root of its cross-reference stream's dictionary, and reads the pages the undamaged file has. + /// /// /// The scanner finds object headers in the bytes of the file, and the catalog of a document that keeps its - /// objects in object streams is inside a compressed stream, where no scan can see it. Refusing the - /// document is the honest answer; the alternative is a merge that silently contributes nothing. + /// objects in object streams is inside a compressed stream, where a scan for headers cannot see it. This + /// test used to assert that such a document is refused; reconstruction now looks inside the object streams + /// it finds, which is what other readers do. /// [Fact] - public async Task Refuses_a_document_whose_catalog_cannot_be_found_by_scanning() + public async Task Rebuilds_a_document_whose_catalog_lives_in_an_object_stream() { var reader = new PdfDocumentReader(PdfEngineOptions.Default); - using var content = new MemoryStream(PointStartxrefAtTheHeader(TestAssets.Modern)); + using var intact = new MemoryStream(File.ReadAllBytes(TestAssets.Modern)); + using var damaged = new MemoryStream(PointStartxrefAtTheHeader(TestAssets.Modern)); - var exception = await Assert.ThrowsAsync( - async () => await reader.ReadAsync( - "damaged.pdf", - content, - password: null, - TestContext.Current.CancellationToken)); + var expected = await reader.ReadAsync("intact.pdf", intact, password: null, TestContext.Current.CancellationToken); + var rebuilt = await reader.ReadAsync("damaged.pdf", damaged, password: null, TestContext.Current.CancellationToken); - Assert.Contains("no document catalog", exception.Message, StringComparison.Ordinal); + Assert.Equal(expected.PageCount, rebuilt.PageCount); + Assert.Contains(rebuilt.Warnings, warning => warning.Kind == PdfWarningKind.CrossReferenceRebuilt); } /// A file that was cut behind its last object keeps working. diff --git a/tests/SharpCorePDF.Tests/PdfStreamParsingTests.cs b/tests/SharpCorePDF.Tests/PdfStreamParsingTests.cs index 6404682..f0c1e52 100644 --- a/tests/SharpCorePDF.Tests/PdfStreamParsingTests.cs +++ b/tests/SharpCorePDF.Tests/PdfStreamParsingTests.cs @@ -68,6 +68,35 @@ public void Returns_no_bytes_when_the_keyword_is_not_where_it_must_be() => public void Keeps_the_bytes_it_found_when_a_compressed_candidate_does_not_inflate() => Assert.Equal("not compressed", Read("<< /Filter /FlateDecode >>\nstream\nnot compressed\nendstream")); + /// + /// A compressed stream whose data holds the word endstream is cut at the candidate where the + /// compressed data is complete — where its Adler-32 checksum matches — not at the first one that inflates: + /// a deflate stream cut short inflates without complaint to the bytes in front of the cut. + /// + [Fact] + public void Scans_past_a_candidate_whose_compressed_data_is_cut_short() + { + var content = Encoding.Latin1.GetBytes("BT (before)\nendstream\n(after) Tj ET"); + using var buffer = new MemoryStream(); + + // Stored blocks keep the word readable in the compressed bytes, where a writer that compresses + // its content can leave it too. + using (var zlib = new System.IO.Compression.ZLibStream(buffer, System.IO.Compression.CompressionLevel.NoCompression, leaveOpen: true)) + { + zlib.Write(content); + } + + var bytes = Encoding.Latin1.GetBytes("<< /Filter /FlateDecode >>\nstream\n") + .Concat(buffer.ToArray()) + .Concat(Encoding.Latin1.GetBytes("\nendstream")) + .ToArray(); + + var parser = new PdfObjectParser(bytes, "damaged.pdf", 0, null, 64); + var stream = Assert.IsType(parser.ParseObject()); + + Assert.Equal(content, PdfFilters.Decode(stream, "damaged.pdf")); + } + /// A stream is read behind an indirect object header. [Fact] public void Reads_a_stream_behind_an_object_header() From 54c08d57e94a0e7fcf8603e1163008c35bba6285 Mon Sep 17 00:00:00 2001 From: Andrew Tierney Date: Mon, 16 Nov 2026 13:31:06 +1100 Subject: [PATCH 08/11] Pin what the scoped Stryker run found unasserted The scoped run over the files of the last two commits scored 70.8% on their changed lines. The survivors that were missing assertions now have them: - PdfCrossReferenceStreamWriterTests compares every row of an appended cross-reference stream byte for byte, its /Index subsections, the width of each /W field at the byte boundaries, its /Size and the trailer entries it carries or drops. - The free text /DA test uses three different colour channels; the merge warning names the page as a person counts it; the damaged-Flate refusal says why. - The predictor limit is pinned at the largest row accepted and one column more, and a zero-bit row is refused. - The endstream scan leaves the parser behind the keyword it chose, and the sixteen-candidate cap is pinned at its boundary. - A signature on a classic-table document keeps a classic table, and only an edit's stream lists the head of the free list. PdfWriter's stream path no longer sets /Size before the stream writer, which derives it itself; the statement was the survivor nothing could kill. The remaining survivors are equivalent mutants (ConfigureAwait flips, bounds checks on offsets the scan already validated) or fallbacks for damaged files noted in the pull request. --- .../Pdf/PdfWriter.cs | 1 - .../SharpCorePDF.Tests/PdfAnnotationTests.cs | 4 +- .../PdfContentReaderTests.Images.cs | 4 +- .../PdfCrossReferenceStreamWriterTests.cs | 98 +++++++++++++++++++ .../PdfDocumentSignerTests.cs | 16 +++ .../PdfIncrementalWriterTests.cs | 1 + .../PdfMergeFidelityTests.cs | 4 +- tests/SharpCorePDF.Tests/PdfPredictorTests.cs | 9 ++ .../PdfStreamParsingTests.cs | 39 +++++++- 9 files changed, 170 insertions(+), 6 deletions(-) create mode 100644 tests/SharpCorePDF.Tests/PdfCrossReferenceStreamWriterTests.cs diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfWriter.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfWriter.cs index f108a11..38f7ac7 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfWriter.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfWriter.cs @@ -166,7 +166,6 @@ public void Complete(PdfDictionary trailer, bool asStream = false) if (asStream) { - trailer.SetInteger("Size", Math.Max(trailer.GetInteger("Size") ?? 0, lastObjectNumber + 1)); PdfCrossReferenceStreamWriter.Write( output, startXref, diff --git a/tests/SharpCorePDF.Tests/PdfAnnotationTests.cs b/tests/SharpCorePDF.Tests/PdfAnnotationTests.cs index fc0ae3e..2f3ab84 100644 --- a/tests/SharpCorePDF.Tests/PdfAnnotationTests.cs +++ b/tests/SharpCorePDF.Tests/PdfAnnotationTests.cs @@ -255,7 +255,7 @@ public async Task States_the_default_appearance_and_resources_of_a_free_text_ann PdfAnnotations.FreeText( new PdfRectangle(72, 500, 320, 530), "Reviewed", - new PdfStandardTextStyle(StandardFont.Helvetica, 10, PdfColor.FromRgb(255, 0, 0)))), + new PdfStandardTextStyle(StandardFont.Helvetica, 10, PdfColor.FromRgb(255, 128, 51)))), cancellationToken: Token); var reader = Read(edited); @@ -264,7 +264,7 @@ public async Task States_the_default_appearance_and_resources_of_a_free_text_ann var appearance = Assert.IsType(reader.Resolve(reader.ResolveDictionary(annotation?.Get("AP"))?.Get("N"))); var defaultAppearance = Assert.IsType(annotation?.Get("DA")); - Assert.Equal("/Helv 10 Tf 1 0 0 rg", Encoding.ASCII.GetString(defaultAppearance.Bytes)); + Assert.Equal("/Helv 10 Tf 1 0.50196 0.2 rg", Encoding.ASCII.GetString(defaultAppearance.Bytes)); Assert.IsType(reader.Resolve(appearance.Dictionary.Get("Resources"))); } diff --git a/tests/SharpCorePDF.Tests/PdfContentReaderTests.Images.cs b/tests/SharpCorePDF.Tests/PdfContentReaderTests.Images.cs index 6388014..8224248 100644 --- a/tests/SharpCorePDF.Tests/PdfContentReaderTests.Images.cs +++ b/tests/SharpCorePDF.Tests/PdfContentReaderTests.Images.cs @@ -40,8 +40,10 @@ public async Task Refuses_to_copy_an_image_whose_flate_data_is_damaged(string pa var reader = Reader(); var image = Assert.Single((await reader.ReadImagesAsync(source, PageRange.All, cancellationToken: Token)).Items); - await Assert.ThrowsAsync( + var exception = await Assert.ThrowsAsync( () => reader.CopyImageDataAsync(source, image, Stream.Null, cancellationToken: Token)); + + Assert.Contains("Flate data of the payload is damaged", exception.Message, StringComparison.Ordinal); } /// One object drawn twice and shared by two pages is one entry carrying both pages. diff --git a/tests/SharpCorePDF.Tests/PdfCrossReferenceStreamWriterTests.cs b/tests/SharpCorePDF.Tests/PdfCrossReferenceStreamWriterTests.cs new file mode 100644 index 0000000..deba1e3 --- /dev/null +++ b/tests/SharpCorePDF.Tests/PdfCrossReferenceStreamWriterTests.cs @@ -0,0 +1,98 @@ +using System.Text; +using SharpCorePDF.Infrastructure.Pdf; + +namespace SharpCorePDF.Tests; + +/// +/// Tests of the cross-reference stream an appended revision ends in: the bytes of every row, the subsections +/// /Index names, the field widths of /W, and the trailer entries the stream's dictionary carries. +/// +/// +/// The rows are compared as hexadecimal so that a wrong width, a wrong byte order or a missing row shows in +/// the expectation itself. +/// +public sealed class PdfCrossReferenceStreamWriterTests +{ + /// + /// Every row is a type byte, an offset as wide as the largest offset needs and a two-byte generation, in + /// object number order; the stream lists itself, numbered above every other object, and the head of the free + /// list when it is asked to. + /// + [Fact] + public void Writes_every_row_of_the_section_in_object_number_order() + { + var (dictionary, rows, text) = Write( + [(7, 2, 70000), (3, 0, 100), (4, 0, 200)], + 300000, + new PdfDictionary().SetInteger("Size", 5).SetInteger("XRefStm", 9).Set("Root", new PdfReference(1, 0)), + freeHead: true); + + Assert.Equal("XRef", dictionary.GetName("Type")); + Assert.Equal(9, dictionary.GetInteger("Size")); + Assert.Null(dictionary.Get("XRefStm")); + Assert.Equal(1, Assert.IsType(dictionary.Get("Root")).ObjectNumber); + Assert.Equal([1L, 3L, 2L], Integers(dictionary.Get("W"))); + Assert.Equal([0L, 1L, 3L, 2L, 7L, 2L], Integers(dictionary.Get("Index"))); + Assert.Equal( + "00000000FFFF" + "010000640000" + "010000C80000" + "010111700002" + "010493E00000", + Convert.ToHexString(rows)); + Assert.StartsWith("8 0 obj\n", text, StringComparison.Ordinal); + Assert.EndsWith("\nendstream\nendobj\nstartxref\n300000\n%%EOF\n", text, StringComparison.Ordinal); + } + + /// + /// The stream is numbered at the /Size the trailer states when that is above every object of the + /// revision, and a revision appended by a signature lists no free head. + /// + [Fact] + public void Numbers_the_stream_at_the_size_the_trailer_states() + { + var (dictionary, rows, text) = Write([(3, 0, 16)], 40, new PdfDictionary().SetInteger("Size", 20), freeHead: false); + + Assert.Equal(21, dictionary.GetInteger("Size")); + Assert.Equal([3L, 1L, 20L, 1L], Integers(dictionary.Get("Index"))); + Assert.Equal("0110000001280000", Convert.ToHexString(rows)); + Assert.StartsWith("20 0 obj\n", text, StringComparison.Ordinal); + } + + /// The offset field is as narrow as the largest offset allows, and no narrower. + /// The offset of the stream, the largest of the section. + /// The width the offset field must have. + [Theory] + [InlineData(255L, 1)] + [InlineData(256L, 2)] + [InlineData(65535L, 2)] + [InlineData(65536L, 3)] + [InlineData(16777216L, 4)] + [InlineData(4294967296L, 5)] + public void Sizes_the_offset_field_to_the_largest_offset(long startXref, int width) + { + var (dictionary, _, _) = Write([], startXref, new PdfDictionary(), freeHead: false); + + Assert.Equal(width, Integers(dictionary.Get("W"))[1]); + } + + private static (PdfDictionary Dictionary, byte[] Rows, string Text) Write( + (int Number, int Generation, long Offset)[] entries, + long startXref, + PdfDictionary trailer, + bool freeHead) + { + using var buffer = new MemoryStream(); + var output = new PdfOutputBuffer(buffer); + + PdfCrossReferenceStreamWriter.Write(output, startXref, entries, trailer, freeHead); + + var bytes = buffer.ToArray(); + var parser = new PdfObjectParser(bytes, "section.pdf", 0, null, 64); + + Assert.True(parser.TryReadIndirectHeader(out _, out _)); + + var stream = Assert.IsType(parser.ParseObject()); + + return (stream.Dictionary, stream.Raw.ToArray(), Encoding.Latin1.GetString(bytes)); + } + + private static long[] Integers(PdfObject? array) => + Assert.IsType(array).Items.Select(item => Assert.IsType(item).Value).ToArray(); +} diff --git a/tests/SharpCorePDF.Tests/PdfDocumentSignerTests.cs b/tests/SharpCorePDF.Tests/PdfDocumentSignerTests.cs index a8a6c63..5aeb628 100644 --- a/tests/SharpCorePDF.Tests/PdfDocumentSignerTests.cs +++ b/tests/SharpCorePDF.Tests/PdfDocumentSignerTests.cs @@ -104,6 +104,22 @@ public async Task Signs_a_document_that_uses_cross_reference_streams_with_one() Assert.False(reader.CrossReferenceTable.WasReconstructed); Assert.Equal(PdfSignatureIntegrity.Valid, validation.Integrity); Assert.True(validation.Signer.CoversWholeDocument); + + // The revision lists the objects it wrote and the stream itself, and not the head of the free list, + // which the first revision already lists. + var section = Assert.IsType(reader.Resolve(reader.CrossReferenceTable.Trailer.GetInteger("Size")!.Value - 1)); + var index = Assert.IsType(section.Dictionary.Get("Index")); + + Assert.NotEqual(0, Assert.IsType(index.Items[0]).Value); + } + + /// A signature appended to a document with a classic table ends in a classic table. + [Fact] + public async Task Signs_a_document_that_uses_a_classic_table_with_one() + { + var signed = await SigningFixtures.SignAsync(SigningFixtures.BuildDocument(), SigningFixtures.Options(Certificate)); + + Assert.False(new PdfFileReader(signed, "signed.pdf", PdfEngineOptions.Default).CrossReferenceTable.NewestSectionIsStream); } /// diff --git a/tests/SharpCorePDF.Tests/PdfIncrementalWriterTests.cs b/tests/SharpCorePDF.Tests/PdfIncrementalWriterTests.cs index eff4bf2..4a00c7a 100644 --- a/tests/SharpCorePDF.Tests/PdfIncrementalWriterTests.cs +++ b/tests/SharpCorePDF.Tests/PdfIncrementalWriterTests.cs @@ -158,6 +158,7 @@ public async Task Appends_a_cross_reference_stream_to_a_document_that_uses_them( Assert.DoesNotContain("trailer", revision, StringComparison.Ordinal); Assert.Equal("XRef", section.Dictionary.GetName("Type")); Assert.Equal(StartXref(source), section.Dictionary.GetInteger("Prev")); + Assert.Equal(0, Assert.IsType(Assert.IsType(section.Dictionary.Get("Index")).Items[0]).Value); Assert.True(reader.CrossReferenceTable.NewestSectionIsStream); Assert.False(reader.CrossReferenceTable.WasReconstructed); Assert.Equal("streamed", (await new PdfDocumentReader(Options).ReadAsync("edited.pdf", new MemoryStream(edited), null, Token)).Metadata.Title); diff --git a/tests/SharpCorePDF.Tests/PdfMergeFidelityTests.cs b/tests/SharpCorePDF.Tests/PdfMergeFidelityTests.cs index 710d4c2..5309569 100644 --- a/tests/SharpCorePDF.Tests/PdfMergeFidelityTests.cs +++ b/tests/SharpCorePDF.Tests/PdfMergeFidelityTests.cs @@ -74,7 +74,9 @@ public async Task Writes_a_media_box_for_a_page_that_has_none() Assert.Equal(4, box.Count); Assert.Equal(PdfPageSize.A4.WidthPoints, ((PdfReal)box.Items[2]).Value, 3); Assert.Equal(PdfPageSize.A4.HeightPoints, ((PdfReal)box.Items[3]).Value, 3); - Assert.Contains(result.Warnings, warning => warning.Kind == PdfWarningKind.PageSizeAssumed); + Assert.Contains( + result.Warnings, + warning => warning.Kind == PdfWarningKind.PageSizeAssumed && warning.Message.Contains("page 1 ", StringComparison.Ordinal)); } /// A page that states its own media box keeps it, and no size is assumed for it. diff --git a/tests/SharpCorePDF.Tests/PdfPredictorTests.cs b/tests/SharpCorePDF.Tests/PdfPredictorTests.cs index dadd829..61cc6c4 100644 --- a/tests/SharpCorePDF.Tests/PdfPredictorTests.cs +++ b/tests/SharpCorePDF.Tests/PdfPredictorTests.cs @@ -166,6 +166,8 @@ public void Reports_tiff_prediction_that_is_not_eight_bit() [InlineData(12, 65536, 65536, 8)] [InlineData(12, 1 << 30, 1, 16)] [InlineData(12, 4, 1, -8)] + [InlineData(12, 4, 1, 0)] + [InlineData(12, (8 * 1024 * 1024) + 1, 1, 8)] public async Task Refuses_a_predicted_row_larger_than_the_engine_decodes(int predictor, int columns, int colors, int bitsPerComponent) { var parameters = Parameters(predictor, columns: columns, colors: colors, bitsPerComponent: bitsPerComponent); @@ -174,6 +176,13 @@ await Assert.ThrowsAnyAsync( () => Task.Run(() => Decode("0001020304050607", parameters), TestContext.Current.CancellationToken)); } + /// A row of exactly the largest size the engine decodes is accepted. + [Fact] + public void Accepts_a_predicted_row_of_the_largest_size_the_engine_decodes() => + Assert.Equal( + Convert.FromHexString("0001"), + Decode("0001", Parameters(12, columns: 8 * 1024 * 1024, colors: 1, bitsPerComponent: 8))); + private static byte[] Decode(string encoded, PdfDictionary parameters) => PdfFilters.Decode( Stream(Compress(Convert.FromHexString(encoded)), parameters), diff --git a/tests/SharpCorePDF.Tests/PdfStreamParsingTests.cs b/tests/SharpCorePDF.Tests/PdfStreamParsingTests.cs index f0c1e52..8eda724 100644 --- a/tests/SharpCorePDF.Tests/PdfStreamParsingTests.cs +++ b/tests/SharpCorePDF.Tests/PdfStreamParsingTests.cs @@ -91,10 +91,47 @@ public void Scans_past_a_candidate_whose_compressed_data_is_cut_short() .Concat(Encoding.Latin1.GetBytes("\nendstream")) .ToArray(); - var parser = new PdfObjectParser(bytes, "damaged.pdf", 0, null, 64); + var parser = new PdfObjectParser(bytes.Concat(Encoding.Latin1.GetBytes("\n42")).ToArray(), "damaged.pdf", 0, null, 64); var stream = Assert.IsType(parser.ParseObject()); Assert.Equal(content, PdfFilters.Decode(stream, "damaged.pdf")); + + // The parser stands behind the keyword that ended the body, so the next object reads. + Assert.Equal(42, Assert.IsType(parser.ParseObject()).Value); + } + + /// + /// At most sixteen candidates are inflated: each is inflated from the start of the data, and a hostile + /// file can hold one every ten bytes. A body whose complete end is the sixteenth candidate is found; one + /// whose end is the seventeenth is not, and the first candidate that inflated is kept instead. + /// + /// The endstream words inside the compressed data. + /// Whether the complete body is expected. + [Theory] + [InlineData(15, true)] + [InlineData(16, false)] + public void Inflates_at_most_sixteen_candidates(int falseEnds, bool complete) + { + var text = string.Concat(Enumerable.Range(0, falseEnds).Select(index => $"({index})\nendstream\n")) + "(end)"; + var content = Encoding.Latin1.GetBytes(text); + using var buffer = new MemoryStream(); + + using (var zlib = new System.IO.Compression.ZLibStream(buffer, System.IO.Compression.CompressionLevel.NoCompression, leaveOpen: true)) + { + zlib.Write(content); + } + + var bytes = Encoding.Latin1.GetBytes("<< /Filter /FlateDecode >>\nstream\n") + .Concat(buffer.ToArray()) + .Concat(Encoding.Latin1.GetBytes("\nendstream\n42")) + .ToArray(); + + var parser = new PdfObjectParser(bytes, "damaged.pdf", 0, null, 64); + var stream = Assert.IsType(parser.ParseObject()); + var decoded = PdfFilters.Decode(stream, "damaged.pdf"); + + Assert.Equal(complete, decoded.SequenceEqual(content)); + Assert.Equal(complete ? 42 : 0, complete ? Assert.IsType(parser.ParseObject()).Value : 0); } /// A stream is read behind an indirect object header. From 7c97a3898087a9b0f4375f4319733a5814808070 Mon Sep 17 00:00:00 2001 From: Andrew Tierney Date: Mon, 16 Nov 2026 13:48:35 +1100 Subject: [PATCH 09/11] Refuse ZLibException on the copy route and read rotations as the inspection does The second robustness run, on 54c08d5, found two more faults now that the first ones are fixed: - flate: the image copy route caught the decompressor's InvalidDataException but not the ZLibException it throws for other damaged input (crash-0b3d19d6). Both are now refused with PdfSourceFormatException. ZLibException is not public, so it is recognised by name rather than by catching every IOException, which would also swallow the destination's own failures. - document: a page declaring /Rotate 91 made the edit engine throw ArgumentOutOfRangeException (crash-7201f51f). The edit engine read /Rotate as an integer and required a multiple of 90, while the inspection reads any number and rounds to the nearest quarter turn, so a real /Rotate was also read as none. The edit engine now uses the inspection's reading, and the quarter turns are reduced before they become an integer, so a declared value beyond int's range no longer wraps. Tests that fail without the changes: PdfContentReaderTests.Refuses_to_copy_an_image_whose_flate_data_is_damaged (the fuzz input's payload) and PdfMergeFidelityTests.Edits_a_page_whose_rotation_is_not_a_quarter_turn. Both crash inputs and the two from the first run replay with no fault. Scoped Stryker over the two changed files kills every mutant on the changed lines. --- CHANGELOG.md | 7 ++++- docs/validation.md | 3 ++- .../Pdf/PdfDocumentInspection.Pages.cs | 16 ++++++++++-- .../Pdf/PdfImageFilters.cs | 6 ++++- .../PdfEditEngine.Overlays.cs | 2 +- .../PdfEditEngine.Pages.cs | 13 +++------- .../PdfEditEngine.cs | 2 +- .../PdfContentReaderTests.Images.cs | 15 +++++++---- .../PdfMergeFidelityTests.cs | 26 +++++++++++++++++++ 9 files changed, 69 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 29cdcc1..bf5ff64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -82,7 +82,12 @@ against an empty feed. read from a document is now written back from its own text. A real too large for the formatter's buffer no longer fails the write of the whole document (found by the `document` fuzz target in its first run). - **The image copy route refuses damaged Flate data with `PdfSourceFormatException`** instead of letting the - decompressor's `InvalidDataException` reach the caller (found by the `flate` fuzz target). + decompressor's `InvalidDataException`, or the `ZLibException` it throws for some inputs, reach the caller + (both found by the `flate` fuzz target). +- **An edit reads a page's rotation the way the inspection does.** A `/Rotate` that is not a multiple of 90 + made the edit engine throw `ArgumentOutOfRangeException` (found by the `document` fuzz target), and a real + or indirect `/Rotate` was read as no rotation at all. Both now take the nearest quarter turn, the rotation + the inspection already reported, and a value beyond the range of `int` is reduced instead of wrapped. - **Annotations satisfy the Arlington model.** A free text annotation states the `/DA` ISO 32000-1 requires, and every annotation appearance states its `/Resources` (found by the Arlington check in its first run). - **A cross-reference table whose offsets moved is rebuilt.** Bytes inserted or removed after a table was diff --git a/docs/validation.md b/docs/validation.md index 3491b07..7a2a36f 100644 --- a/docs/validation.md +++ b/docs/validation.md @@ -100,7 +100,8 @@ Fixed since the list was first drawn up, each with the test that fails without i | `UseIncrementalUpdate` appended a renumbered copy of the whole document, rewriting the signature dictionary after signing | the edited document's objects keep their numbers, and an object written exactly as the source states it is left out: rotating one page of a signed document appends 716 bytes and three objects instead of 20,969 bytes and twelve, and pyHanko places the signature at `ENTIRE_REVISION` | `PdfIncrementalInvariantTests.Appends_only_the_objects_an_edit_changes`, `Leaves_the_signature_dictionary_out_of_an_appended_revision` | | A merge copied a page that states no `/MediaBox` (and inherits none) without one | the page is written with the A4 box the inspection describes it with, and `PageSizeAssumed` says so | `PdfMergeFidelityTests.Writes_a_media_box_for_a_page_that_has_none` | | Reals a merge or an edit copied were re-formatted to five decimals (`0.000012` became `0.00001`), and a real too large for the formatter's buffer failed the write of the whole document — the first crash the `document` fuzz target found | a real read from a document is written from its own text when that text is a well-formed PDF real; other values are formatted at any magnitude | `PdfMergeFidelityTests.Writes_the_reals_it_copies_as_the_source_states_them`, `Writes_a_real_of_any_magnitude_in_the_form_pdf_defines` | -| The copy route of an image let the decompressor's `InvalidDataException` escape for damaged Flate data — the crash the `flate` fuzz target found | the copy route refuses damaged data with `PdfSourceFormatException`, with or without a predictor | `PdfContentReaderTests.Refuses_to_copy_an_image_whose_flate_data_is_damaged` | +| The copy route of an image let the decompressor's `InvalidDataException`, and for other inputs its `ZLibException`, escape for damaged Flate data — the two crashes the `flate` fuzz target found | the copy route refuses damaged data with `PdfSourceFormatException`, with or without a predictor | `PdfContentReaderTests.Refuses_to_copy_an_image_whose_flate_data_is_damaged` | +| The edit engine threw `ArgumentOutOfRangeException` for a `/Rotate` that is not a multiple of 90, and read a real or indirect one as none — the second crash the `document` fuzz target found | the edit reads the rotation as the inspection does, to the nearest quarter turn | `PdfMergeFidelityTests.Edits_a_page_whose_rotation_is_not_a_quarter_turn` | | A free text annotation was written without the `/DA` ISO 32000-1 requires, and annotation appearances without `/Resources` — both reported by the Arlington check's first run | `/DA` states the font, size and colour the appearance draws with; every appearance states its resources, empty if it needs none | `PdfDocumentEditorTests.States_the_default_appearance_and_resources_of_a_free_text_annotation`, `States_resources_on_an_appearance_that_needs_none` | | A table that parsed but whose offsets had moved was used as it was, and reconstruction could not see into object streams | the table is rebuilt when its offsets miss their objects; reconstruction indexes object streams and takes `/Root` from a cross-reference stream's dictionary | `PdfReaderRecoveryTests.Rebuilds_a_table_whose_offsets_no_longer_point_at_their_objects`, `Rebuilds_a_table_from_object_streams_when_the_cross_reference_stream_is_lost`, `Rebuilds_a_document_whose_catalog_lives_in_an_object_stream` | | AESV2 with a short declared key, wrapped predictor row sizes, truncated `endstream` candidates, ignored generations | sixteen-byte AESV2 keys; 64-bit row sizes on both predictor paths; Adler-32-checked candidates, at most sixteen; a mismatched generation resolves to null | `PdfMergeEngineTests.Opens_an_aesv2_document_whose_crypt_filter_declares_a_short_key`, `PdfPredictorTests.Refuses_a_predicted_row_larger_than_the_engine_decodes`, `PdfStreamParsingTests.Scans_past_a_candidate_whose_compressed_data_is_cut_short`, `PdfReaderRecoveryTests.Resolves_a_reference_to_a_generation_the_table_does_not_list_as_null` | diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfDocumentInspection.Pages.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfDocumentInspection.Pages.cs index 86f20a6..10e6bfd 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfDocumentInspection.Pages.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfDocumentInspection.Pages.cs @@ -72,7 +72,16 @@ private static List ReadPages(PdfFileReader reader, List 0d && height > 0d ? PdfPageSize.FromPoints(width, height) : null; } - private static PageRotation ReadRotation(PdfFileReader reader, SourcePage page) + /// Reads the rotation a page declares or inherits, as the nearest quarter turn. + /// The document. + /// The page. + /// The rotation; when none is declared. + /// + /// ISO 32000-1 requires a multiple of 90; a value that is not one, or that is a real or an indirect + /// object, is read the way other readers read it rather than refused. The inspection and the edit engine + /// both read it here, so an edit keeps the rotation the inspection reported. + /// + internal static PageRotation ReadRotation(PdfFileReader reader, SourcePage page) { var value = ReadNumber(reader, page.Dictionary.Get("Rotate")) ?? ReadNumber(reader, page.Inherited.Get("Rotate")); @@ -82,7 +91,10 @@ private static PageRotation ReadRotation(PdfFileReader reader, SourcePage page) return PageRotation.None; } - var normalized = (((int)Math.Round(value.Value / 90d) * 90) % 360 + 360) % 360; + // The quarter turns are counted in floating point and reduced before they become an integer, so a + // declared value beyond the range of int is reduced rather than wrapped. + var quarterTurns = Math.Round(value.Value / 90d) % 4d; + var normalized = (((int)quarterTurns * 90) % 360 + 360) % 360; return normalized switch { diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfImageFilters.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfImageFilters.cs index dc7d8f2..db1eed9 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfImageFilters.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfImageFilters.cs @@ -170,7 +170,11 @@ await InflatePredictorToAsync(stream.Raw, predictor, destination, cancellationTo await InflateToAsync(stream.Raw, destination, cancellationToken).ConfigureAwait(false); } } - catch (InvalidDataException exception) + // The decompressor reports most damage as InvalidDataException and some as ZLibException, an + // IOException the runtime does not make public, so it is recognised by name: the destination's own + // IOException is the caller's, and is not turned into a verdict on the document. + catch (Exception exception) when (exception is InvalidDataException + || exception.GetType().FullName == "System.IO.Compression.ZLibException") { throw new PdfSourceFormatException( sourceName, diff --git a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Overlays.cs b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Overlays.cs index 23e7ab5..84f8af6 100644 --- a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Overlays.cs +++ b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Overlays.cs @@ -439,7 +439,7 @@ private sealed class OverlayLayer( public PdfOverlayLayer Layer { get; } = layer; /// Gets the rotation the layer page declares, which the form is placed with. - public PageRotation Rotation { get; } = ReadRotation(page); + public PageRotation Rotation { get; } = ReadRotation(reader, page); /// Gets or sets the object number of the form XObject, or zero before it is written. public int FormNumber { get; set; } diff --git a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Pages.cs b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Pages.cs index dbda7f5..0098155 100644 --- a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Pages.cs +++ b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Pages.cs @@ -490,16 +490,11 @@ private static void WritePagesNode(PdfWriter writer, int pagesNodeNumber, ListReads the rotation a page declares, before any edit is applied. + /// The document the page belongs to. /// The page. /// The normalised rotation. - private static PageRotation ReadRotation(SourcePage page) - { - var declared = page.Dictionary.GetInteger("Rotate") - ?? page.Inherited.GetInteger("Rotate") - ?? 0; - - return PageRotationExtensions.Normalize((int)declared); - } + private static PageRotation ReadRotation(PdfFileReader reader, SourcePage page) => + Pdf.PdfDocumentInspection.ReadRotation(reader, page); private static void EnsurePage(List layout, int pageNumber, int index, string sourceName) { @@ -1033,7 +1028,7 @@ private sealed class EditedPage(PdfFileReader reader, SourcePage source, PdfRect public int TargetNumber { get; set; } public PageRotation EffectiveRotation => - PageRotationExtensions.Normalize(ReadRotation(Source).Degrees + AddedRotation.Degrees); + PageRotationExtensions.Normalize(ReadRotation(Reader, Source).Degrees + AddedRotation.Degrees); public EditedPage Copy() { diff --git a/src/SharpCorePDF.Infrastructure/PdfEditEngine.cs b/src/SharpCorePDF.Infrastructure/PdfEditEngine.cs index 7fb4d3b..117e20f 100644 --- a/src/SharpCorePDF.Infrastructure/PdfEditEngine.cs +++ b/src/SharpCorePDF.Infrastructure/PdfEditEngine.cs @@ -366,7 +366,7 @@ private void Verify(PdfCaptureStream? capture, List layout, string? for (var index = 0; index < layout.Count; index++) { var expected = layout[index].EffectiveRotation; - var actual = ReadRotation(reader.Pages[index]); + var actual = ReadRotation(reader, reader.Pages[index]); if (actual != expected) { diff --git a/tests/SharpCorePDF.Tests/PdfContentReaderTests.Images.cs b/tests/SharpCorePDF.Tests/PdfContentReaderTests.Images.cs index 8224248..b1ecde4 100644 --- a/tests/SharpCorePDF.Tests/PdfContentReaderTests.Images.cs +++ b/tests/SharpCorePDF.Tests/PdfContentReaderTests.Images.cs @@ -17,18 +17,23 @@ public sealed partial class PdfContentReaderTests /// /// The copy route of an image whose Flate data is damaged refuses it with the contract's exception, whether a /// predictor is declared or not: the decompressor's own exception used to reach the caller as if the library - /// had failed. + /// had failed. The decompressor reports damage as or, for some inputs — + /// the second data, found by the flate fuzz target — as a ZLibException, an + /// ; both are refused. /// /// The decode parameters the image declares. + /// The damaged Flate data, as hexadecimal. [Theory] - [InlineData("/DecodeParms << /Predictor 12 /Colors 1 /BitsPerComponent 8 /Columns 4 >>")] - [InlineData("")] - public async Task Refuses_to_copy_an_image_whose_flate_data_is_damaged(string parameters) + [InlineData("/DecodeParms << /Predictor 12 /Colors 1 /BitsPerComponent 8 /Columns 4 >>", "789CFFFFFFFF000102030405")] + [InlineData("", "789CFFFFFFFF000102030405")] + [InlineData("/DecodeParms << /Predictor 12 /Colors 1 /BitsPerComponent 8 /Columns 4 >>", "08B85B00C3000040")] + [InlineData("", "08B85B00C3000040")] + public async Task Refuses_to_copy_an_image_whose_flate_data_is_damaged(string parameters, string data) { var builder = new PdfBuilder(); var catalog = builder.Reserve(); var pages = builder.Reserve(); - var damaged = new byte[] { 0x78, 0x9C, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05 }; + var damaged = Convert.FromHexString(data); builder.Add("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 100 100] /Resources << /XObject << /Im0 5 0 R >> >> /Contents 4 0 R >>"); builder.AddStream(string.Empty, "q 50 0 0 50 0 0 cm /Im0 Do Q"u8.ToArray()); diff --git a/tests/SharpCorePDF.Tests/PdfMergeFidelityTests.cs b/tests/SharpCorePDF.Tests/PdfMergeFidelityTests.cs index 5309569..ec81fd6 100644 --- a/tests/SharpCorePDF.Tests/PdfMergeFidelityTests.cs +++ b/tests/SharpCorePDF.Tests/PdfMergeFidelityTests.cs @@ -93,6 +93,32 @@ public async Task Keeps_the_media_box_a_page_states() Assert.DoesNotContain(result.Warnings, warning => warning.Kind == PdfWarningKind.PageSizeAssumed); } + /// + /// A page whose /Rotate is not a multiple of 90 — which ISO 32000-1 requires it to be — is edited + /// with the nearest quarter turn, the rotation the inspection already reports for it; the edit engine used + /// to throw for it (found by the document fuzz target). + /// + /// The rotation the page declares. + /// The rotation the edited page carries. + [Theory] + [InlineData("91", 90)] + [InlineData("-91", 270)] + [InlineData("44", 0)] + [InlineData("180.4", 180)] + [InlineData("99999999999", 270)] + public async Task Edits_a_page_whose_rotation_is_not_a_quarter_turn(string declared, int expected) + { + var editor = new PdfDocumentEditorService(new PdfDocumentReader(), new PdfEditEngine()); + var edited = await editor.ApplyToArrayAsync( + new PdfByteArraySource(Page($"/Rotate {declared}"), "rotated.pdf"), + PdfEditPlan.Create().SetMetadata(new PdfMetadata { Title = "rotated" }), + cancellationToken: Token); + + var rotate = new PdfFileReader(edited, "edited.pdf", PdfEngineOptions.Default).Pages[0].Dictionary.GetInteger("Rotate") ?? 0; + + Assert.Equal(expected, rotate); + } + private static PdfMergerService Merger() => new(new PdfDocumentReader(), new PdfMergeEngine()); private static byte[] Page(string entries) From f529b02cf089c0056a6c6998b90517c150f88bf6 Mon Sep 17 00:00:00 2001 From: Andrew Tierney Date: Mon, 16 Nov 2026 14:09:45 +1100 Subject: [PATCH 10/11] Bound the outline walk of a merge by depth and copy each bookmark once The third robustness run, on 7c97a38, found a stack overflow (crash-d34275a3): a merge walks the outline recursively, one frame per level, and a source nested about fifty thousand levels deep ended the process. The walk now stops at 256 levels and reports BookmarksTruncated, and a bookmark reached a second time - a /First or /Next pointing back up the tree, or a child two parents share - is copied once and reported with BookmarkSkipped, which also ends cycles and the repeated walks of shared subtrees. Tests: PdfMergeOutlineLimitsTests (depth limit, shared child, self cycle). The crash input replays with no fault; the full suite passes. --- .../PdfMergeEngine.Outlines.cs | 72 ++++++++++++--- .../PdfMergeOutlineLimitsTests.cs | 90 +++++++++++++++++++ 2 files changed, 152 insertions(+), 10 deletions(-) create mode 100644 tests/SharpCorePDF.Tests/PdfMergeOutlineLimitsTests.cs diff --git a/src/SharpCorePDF.Infrastructure/PdfMergeEngine.Outlines.cs b/src/SharpCorePDF.Infrastructure/PdfMergeEngine.Outlines.cs index 6011ea8..4c8d7fc 100644 --- a/src/SharpCorePDF.Infrastructure/PdfMergeEngine.Outlines.cs +++ b/src/SharpCorePDF.Infrastructure/PdfMergeEngine.Outlines.cs @@ -17,6 +17,13 @@ public sealed partial class PdfMergeEngine /// The largest number of outline items that is copied, as a guard against damaged trees. private const int MaxOutlineItems = 100_000; + /// + /// The deepest level of bookmarks that is copied. The outline is walked recursively, one frame per level, + /// and a hostile file nested fifty thousand levels deep overflowed the stack and ended the process; a real + /// outline is a handful of levels deep. + /// + private const int MaxOutlineDepth = 256; + /// The name of the entry that holds the first child of a bookmark or of the outline root. private const string FirstEntry = "First"; @@ -54,8 +61,9 @@ public sealed partial class PdfMergeEngine } var linkage = new OutlineLinkage(); - var nodes = ReadChain(source.Reader, outline.Get(FirstEntry), context, source); - var chain = CopyChain(context, source, nodes, parent, linkage); + var walk = new OutlineWalk(); + var nodes = ReadChain(source.Reader, outline.Get(FirstEntry), context, source, walk); + var chain = CopyChain(context, source, nodes, parent, linkage, walk, depth: 0); if (!chain.IsEmpty) { @@ -102,12 +110,14 @@ public sealed partial class PdfMergeEngine /// The /First entry of the parent, or the /Next chain start. /// The merge context. /// The source being read. + /// The bookmarks of the source read so far. /// The items of the chain, in document order. private static List ReadChain( PdfFileReader reader, PdfObject? first, PdfMergeContext context, - MergeSource source) + MergeSource source, + OutlineWalk walk) { var nodes = new List(); @@ -135,6 +145,17 @@ private static List ReadChain( break; } + // A bookmark reached a second time — through a /Next or /First that points back up the tree, or + // a child two parents share — is not copied again: a cycle would never end, and a shared child + // is walked once for each of its parents, which grows with the product of the levels. + if (!walk.Visited.Add(reference.ObjectNumber)) + { + context.Warnings.Add(new( + PdfWarningKind.BookmarkSkipped, + $"'{source.DisplayName}': a bookmark is reached twice in the outline, which a tree does not allow; it was copied once.")); + break; + } + nodes.Add(new OutlineNode(reference.ObjectNumber, dictionary)); reference = dictionary.GetReference("Next"); } @@ -155,13 +176,17 @@ private static List ReadChain( /// The bookmarks of this level, in document order. /// The object number the bookmarks must point at as their parent. /// The chain this level belongs to, used for links to the sibling level. + /// The bookmarks of the source read so far. + /// The level of the chain, zero for the top level. /// What was copied. private static OutlineChain CopyChain( PdfMergeContext context, MergeSource source, List nodes, OutlineParent parent, - OutlineLinkage linkage) + OutlineLinkage linkage, + OutlineWalk walk, + int depth) { if (nodes.Count == 0) { @@ -182,12 +207,10 @@ private static OutlineChain CopyChain( var position = index; var node = nodes[position]; var itemParent = new OutlineParent { Target = targets[position] }; - var children = CopyChain( - context, - source, - ReadChain(source.Reader, node.Dictionary.Get(FirstEntry), context, source), - itemParent, - new OutlineLinkage()); + var below = depth + 1 < MaxOutlineDepth + ? ReadChain(source.Reader, node.Dictionary.Get(FirstEntry), context, source, walk) + : Truncate(node, context, source, walk); + var children = CopyChain(context, source, below, itemParent, new OutlineLinkage(), walk, depth + 1); descendants += children.TotalItems; @@ -202,6 +225,35 @@ private static OutlineChain CopyChain( return new OutlineChain(targets[0], targets[^1], nodes.Count, descendants); } + /// Stops the walk at the deepest level copied, saying so once per source. + /// The bookmark whose children are not copied. + /// The merge context. + /// The source being read. + /// The bookmarks of the source read so far. + /// No children. + private static List Truncate(OutlineNode node, PdfMergeContext context, MergeSource source, OutlineWalk walk) + { + if (node.Dictionary.Get(FirstEntry) is not null && !walk.WasTruncated) + { + walk.WasTruncated = true; + context.Warnings.Add(new( + PdfWarningKind.BookmarksTruncated, + $"'{source.DisplayName}': the bookmarks are nested more than {MaxOutlineDepth} levels deep; the levels below were skipped.")); + } + + return []; + } + + /// What the walk of one source's outline has seen. + private sealed class OutlineWalk + { + /// Gets the object numbers of the bookmarks read so far. + public HashSet Visited { get; } = []; + + /// Gets or sets a value indicating whether a level was cut at the depth limit. + public bool WasTruncated { get; set; } + } + /// A parent object number that is filled in after the outline has been walked. private sealed class OutlineParent { diff --git a/tests/SharpCorePDF.Tests/PdfMergeOutlineLimitsTests.cs b/tests/SharpCorePDF.Tests/PdfMergeOutlineLimitsTests.cs new file mode 100644 index 0000000..fa45b61 --- /dev/null +++ b/tests/SharpCorePDF.Tests/PdfMergeOutlineLimitsTests.cs @@ -0,0 +1,90 @@ +using SharpCorePDF.Application.Services; +using SharpCorePDF.Domain.Aggregates; +using SharpCorePDF.Domain.Entities; +using SharpCorePDF.Domain.ValueObjects; +using SharpCorePDF.Infrastructure; +using SharpCorePDF.Infrastructure.Sources; + +namespace SharpCorePDF.Tests; + +/// +/// Tests of how a merge copies an outline that is not a tree: nested deeper than it walks, or reaching a +/// bookmark twice. The first crash of this kind the document fuzz target found overflowed the stack. +/// +public sealed class PdfMergeOutlineLimitsTests +{ + private static CancellationToken Token => TestContext.Current.CancellationToken; + + /// An outline nested deeper than 256 levels is copied to that depth, and the merge says so. + [Fact] + public async Task Copies_an_outline_nested_deeper_than_it_walks_to_its_limit() + { + const int Levels = 300; + var bookmarks = new List(); + + // Bookmark i is object 4 + i, its parent object 3 + i (object 3 is the outline root). + for (var level = 0; level < Levels; level++) + { + var first = level + 1 < Levels ? $"/First {5 + level} 0 R /Last {5 + level} 0 R /Count 1" : string.Empty; + bookmarks.Add($"<< /Title (level {level}) /Parent {3 + level} 0 R {first} >>"); + } + + var result = await MergeAsync(bookmarks); + + Assert.Contains(result.Warnings, warning => warning.Kind == PdfWarningKind.BookmarksTruncated); + } + + /// + /// A bookmark two parents share is copied once, and the merge says so; it was walked again under every + /// parent, which grows with the product of the levels, and a bookmark that points back up the tree never + /// ended. + /// + [Fact] + public async Task Copies_a_bookmark_reached_twice_once() + { + var result = await MergeAsync( + [ + "<< /Title (a) /Parent 3 0 R /Next 6 0 R /First 5 0 R /Last 5 0 R /Count 1 >>", + "<< /Title (shared) /Parent 4 0 R >>", + "<< /Title (b) /Parent 3 0 R /Prev 4 0 R /First 5 0 R /Last 5 0 R /Count 1 >>", + ]); + + Assert.Contains(result.Warnings, warning => warning.Kind == PdfWarningKind.BookmarkSkipped); + } + + /// A bookmark whose child is its own parent ends instead of recursing until the stack runs out. + [Fact] + public async Task Copies_a_bookmark_that_is_its_own_child_once() + { + var result = await MergeAsync(["<< /Title (loop) /Parent 3 0 R /First 4 0 R /Last 4 0 R /Count 1 >>"]); + + Assert.Contains(result.Warnings, warning => warning.Kind == PdfWarningKind.BookmarkSkipped); + } + + /// Merges a one page document whose outline root (object 3) holds the given bookmarks, objects 4 on. + private static async Task MergeAsync(IReadOnlyList bookmarks) + { + var builder = new PdfBuilder(); + var catalog = builder.Reserve(); + var pages = builder.Reserve(); + var outline = builder.Reserve(); + + foreach (var bookmark in bookmarks) + { + builder.Add(bookmark); + } + + var page = builder.Add($"<< /Type /Page /Parent {pages} 0 R /MediaBox [0 0 200 200] >>"); + + builder.Set(catalog, $"<< /Type /Catalog /Pages {pages} 0 R /Outlines {outline} 0 R >>"); + builder.Set(pages, $"<< /Type /Pages /Kids [{page} 0 R] /Count 1 >>"); + builder.Set(outline, "<< /Type /Outlines /First 4 0 R /Last 4 0 R >>"); + + using var destination = new MemoryStream(); + + return await new PdfMergerService(new PdfDocumentReader(), new PdfMergeEngine()).MergeAsync( + MergePlan.Create().AddSource(PdfSource.Create(new PdfByteArraySource(builder.Build(catalog), "outline.pdf"))), + destination, + Token); + } +} From 71485c034881dcf1a503143851d185fa70f623a3 Mon Sep 17 00:00:00 2001 From: Andrew Tierney Date: Mon, 16 Nov 2026 19:08:45 +1100 Subject: [PATCH 11/11] Pass over the offsets a rebuilt table cannot read The robustness run on f529b02 found a crash in the document fuzz target (crash-e154541f). Reconstruction keeps the damaged table's entries beside the ones its scan finds, and the search for a cross-reference stream's trailer sliced the document at each of their offsets unchecked. An offset past the end of the file, or one beyond the range of an index that wraps negative when cast, threw ArgumentOutOfRangeException instead of letting the rebuild go on to find the catalog. Such an entry is now passed over. The catalog search needs no such check: PdfLexer clamps the position it starts at, which a scoped Stryker run confirmed by leaving the same guard there unkillable, so it was not added. Test that fails without the change: PdfReaderRecoveryTests.Rebuilds_a_table_that_states_an_offset_past_the_end_of_the_file, for an offset past the end and one that wraps negative. The crash input replays with no fault. On the changed lines the two surviving mutants are equivalent: offset 0 is the header and an offset equal to the length gives an empty window, so neither boundary changes what is read. The outline-depth fix of f529b02 is now in CHANGELOG.md and the fixed-defects table of docs/validation.md, which it had not been. --- CHANGELOG.md | 10 +++++ docs/validation.md | 2 + .../Pdf/PdfCrossReferenceReader.Scan.cs | 13 +++++- .../PdfReaderRecoveryTests.cs | 40 +++++++++++++++++++ 4 files changed, 64 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bf5ff64..7551519 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -110,6 +110,16 @@ against an empty feed. files tree that only names an external file is listed, with a warning and a length of zero; copying it threw `ArgumentException` as if the caller had passed a file of another document (found by the veraPDF corpus run, `isartor-6-1-11-t01-fail-a.pdf`). +- **A merge walks a deep or looping outline without ending the process.** The outline was copied + recursively, one stack frame per level, and a source nested about fifty thousand levels deep overflowed + the stack. The walk stops at 256 levels with `BookmarksTruncated`, and a bookmark reached a second time + — a `/First` or `/Next` that points back up the tree, or a child two parents share — is copied once and + reported with `BookmarkSkipped` (found by the `document` fuzz target). +- **Rebuilding a table passes over the offsets it cannot read.** Reconstruction keeps the damaged table's + entries beside the ones its scan finds, and the search for a cross-reference stream's trailer sliced the + document at each of their offsets unchecked: one past the end of the file, or beyond the range of an + index, threw `ArgumentOutOfRangeException` instead of letting the rebuild find the catalog (found by the + `document` fuzz target). - **Run-length images are decoded.** `RunLengthDecode` joins Flate, LZW and the two ASCII filters: a bilevel or scan-style image that declares it now yields its samples — and `Gray8`, `Rgb24` or `Cmyk32` diff --git a/docs/validation.md b/docs/validation.md index 7a2a36f..83233fe 100644 --- a/docs/validation.md +++ b/docs/validation.md @@ -106,6 +106,8 @@ Fixed since the list was first drawn up, each with the test that fails without i | A table that parsed but whose offsets had moved was used as it was, and reconstruction could not see into object streams | the table is rebuilt when its offsets miss their objects; reconstruction indexes object streams and takes `/Root` from a cross-reference stream's dictionary | `PdfReaderRecoveryTests.Rebuilds_a_table_whose_offsets_no_longer_point_at_their_objects`, `Rebuilds_a_table_from_object_streams_when_the_cross_reference_stream_is_lost`, `Rebuilds_a_document_whose_catalog_lives_in_an_object_stream` | | AESV2 with a short declared key, wrapped predictor row sizes, truncated `endstream` candidates, ignored generations | sixteen-byte AESV2 keys; 64-bit row sizes on both predictor paths; Adler-32-checked candidates, at most sixteen; a mismatched generation resolves to null | `PdfMergeEngineTests.Opens_an_aesv2_document_whose_crypt_filter_declares_a_short_key`, `PdfPredictorTests.Refuses_a_predicted_row_larger_than_the_engine_decodes`, `PdfStreamParsingTests.Scans_past_a_candidate_whose_compressed_data_is_cut_short`, `PdfReaderRecoveryTests.Resolves_a_reference_to_a_generation_the_table_does_not_list_as_null` | | A revision appended to a file that uses cross-reference streams ended in a classic table | the revision ends in a cross-reference stream; qpdf and pyHanko accept both an edit and two signatures appended this way | `PdfIncrementalWriterTests.Appends_a_cross_reference_stream_to_a_document_that_uses_them`, `PdfDocumentSignerTests.Signs_a_document_that_uses_cross_reference_streams_with_one` | +| A merge copied the outline recursively, so a source nested about fifty thousand levels deep overflowed the stack; a looping or shared outline was walked again at every visit — a crash the `document` fuzz target found | the walk stops at 256 levels with `BookmarksTruncated`; a bookmark reached twice is copied once with `BookmarkSkipped` | `PdfMergeOutlineLimitsTests` | +| Rebuilding a table sliced the document at offsets the damaged table stated past its end, or beyond an index's range, and threw `ArgumentOutOfRangeException` — a crash the `document` fuzz target found | an offset outside the document is passed over, and the rebuild finds the catalog | `PdfReaderRecoveryTests.Rebuilds_a_table_that_states_an_offset_past_the_end_of_the_file` | | Copying an attachment the listing reported, whose specification embeds no stream, threw `ArgumentException` — the public-corpora job's fault on `isartor-6-1-11-t01-fail-a.pdf` | it writes no bytes, matching the `ByteLength` of zero the listing reports | `PdfContentReaderTests.Copies_nothing_for_a_listed_file_that_embeds_no_stream` | ## Corpus readings, 2026-11-16 diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.Scan.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.Scan.cs index d0218ed..41b31d9 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.Scan.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.Scan.cs @@ -204,6 +204,17 @@ private bool MentionsObjectStream(long offset) return window.IndexOf("/ObjStm"u8) >= 0; } + /// Determines whether an offset names a byte of the document. + /// The offset. + /// when the offset is inside the data. + /// + /// Reconstruction keeps the entries of the table it is repairing beside the ones the scan found, and those + /// offsets are whatever the damaged table stated: past the end of the file, negative, or beyond the range + /// of an , which wraps to a negative index when it is cast. A slice of the data needs + /// this check; a parser does not, because clamps the position it starts at. + /// + private bool IsInside(long offset) => offset >= 0 && offset < data.Length; + /// Parses the object whose header is at an offset. /// The offset. /// The object, or . @@ -223,7 +234,7 @@ private bool MentionsObjectStream(long offset) private PdfDictionary? FindCrossReferenceStreamTrailer() { foreach (var offset in table.Entries.Values - .Where(entry => entry.Type == XrefEntryType.Uncompressed) + .Where(entry => entry.Type == XrefEntryType.Uncompressed && IsInside(entry.Offset)) .Select(entry => entry.Offset) .OrderByDescending(offset => offset)) { diff --git a/tests/SharpCorePDF.Tests/PdfReaderRecoveryTests.cs b/tests/SharpCorePDF.Tests/PdfReaderRecoveryTests.cs index 8666a25..7929ee8 100644 --- a/tests/SharpCorePDF.Tests/PdfReaderRecoveryTests.cs +++ b/tests/SharpCorePDF.Tests/PdfReaderRecoveryTests.cs @@ -19,6 +19,46 @@ namespace SharpCorePDF.Tests; /// public sealed class PdfReaderRecoveryTests { + /// + /// A table whose trailer names no catalog is rebuilt, and an entry the damaged table states past the end + /// of the file is passed over rather than read: reconstruction keeps the table's own entries beside the + /// ones it scans, and the offset of such an entry is whatever the damage left — the document fuzz target + /// found it sliced the data at one and threw. Cast to an index, 9,999,999,999 lands past the end and + /// 3,000,000,000 wraps to a negative one; the search for a cross-reference stream's trailer sliced the data + /// at either. The entry is object 1, ahead of the catalog, so every search meets it before the catalog. + /// + /// The ten-digit offset the damaged table states. + [Theory] + [InlineData(9_999_999_999L)] + [InlineData(3_000_000_000L)] + public void Rebuilds_a_table_that_states_an_offset_past_the_end_of_the_file(long offset) + { + var output = new MemoryStream(); + var offsets = new long[5]; + + void Write(string text) => output.Write(Encoding.Latin1.GetBytes(text)); + + Write("%PDF-1.7\n"); + offsets[2] = output.Position; + Write("2 0 obj\n<< /Type /Catalog /Pages 3 0 R >>\nendobj\n"); + offsets[3] = output.Position; + Write("3 0 obj\n<< /Type /Pages /Kids [4 0 R] /Count 1 >>\nendobj\n"); + offsets[4] = output.Position; + Write("4 0 obj\n<< /Type /Page /Parent 3 0 R /MediaBox [0 0 200 200] >>\nendobj\n"); + + var xref = output.Position; + Write("xref\n0 5\n0000000000 65535 f \n"); + Write($"{offset:D10} 00000 n \n"); + Write($"{offsets[2]:D10} 00000 n \n{offsets[3]:D10} 00000 n \n{offsets[4]:D10} 00000 n \n"); + Write($"trailer\n<< /Size 5 >>\nstartxref\n{xref}\n%%EOF\n"); + + var reader = new PdfFileReader(output.ToArray(), "past-the-end.pdf", PdfEngineOptions.Default); + + Assert.True(reader.CrossReferenceTable.WasReconstructed); + Assert.Single(reader.Pages); + Assert.Null(reader.Resolve(1)); + } + /// /// A file whose bytes moved after its table was written — here three bytes inserted behind the header, the /// way a converted line ending or a prepended byte order mark moves them — has a table that parses and