From 06ad09252baa7361d47f71797cdff21ce1ae0a34 Mon Sep 17 00:00:00 2001 From: Andrew Tierney Date: Mon, 16 Nov 2026 11:03:01 +1100 Subject: [PATCH 01/15] Validate the output with independent readers and the input with fuzzing and corpora A writer that only reads its own output back cannot see the mistakes it shares with itself. validation.yml now puts the example output and a validation corpus (written by tools/SharpCorePDF.Validation, with a manifest of the text, passwords, permissions, revisions and signature coverage each document must have) through qpdf, pypdf, pdfminer.six, MuPDF, pdf.js, fontTools, OTS, PDFium and pyHanko, checks the incremental-update invariants, and replays the fuzz seeds and four hostile documents. robustness.yml runs libFuzzer over the parser and the DCT, CCITT, LZW, Flate and RunLength decoders, reads the veraPDF and pdf.js corpora one process per file, and judges the PDF/A writer with veraPDF over the veraPDF corpus. compliance.yml cross-validates PDF/A-4. The checks found defects that are not fixed here. Each is pinned by a skipped test (PdfSignatureAttackTests, PdfIncrementalInvariantTests) or a strict --known-defect flag that fails the run once the defect is fixed, and docs/validation.md lists them with their evidence. No shipped package changes. The constitution's statement that no Python validator runs in CI is corrected as a proposed PATCH amendment (2.0.1) for the maintainer to ratify. Local AI assistant state (.claude/, CLAUDE.local.md) is ignored. --- .config/dotnet-tools.json | 7 + .github/workflows/compliance.yml | 6 +- .github/workflows/robustness.yml | 282 +++++++++++++++ .github/workflows/validation.yml | 232 ++++++++++++ .gitignore | 4 + .specify/memory/constitution.md | 18 +- CHANGELOG.md | 28 ++ CONTRIBUTING.md | 6 +- README.md | 7 + SharpCorePDF.slnx | 4 +- docs/README.md | 2 + docs/tools/corpus-robustness.py | 207 +++++++++++ docs/tools/render-compare.py | 158 ++++++++ docs/tools/validate-arlington.py | 149 ++++++++ docs/tools/validate-encryption-pdfjs.mjs | 95 +++++ docs/tools/validate-encryption.py | 164 +++++++++ docs/tools/validate-fonts.py | 263 ++++++++++++++ docs/tools/validate-incremental.py | 145 ++++++++ docs/tools/validate-pdf.py | 30 +- docs/tools/validate-pdfa-cross.py | 11 +- docs/tools/validate-preflight-corpus.py | 148 ++++++++ docs/tools/validate-signatures.py | 77 ++++ docs/tools/validate-text.py | 147 ++++++++ docs/tools/validation-requirements.txt | 20 ++ docs/validation.md | 139 +++++++ .../PdfIncrementalInvariantTests.cs | 142 ++++++++ .../PdfSignatureAttackTests.cs | 218 +++++++++++ tools/SharpCorePDF.Validation/CorpusWriter.cs | 338 ++++++++++++++++++ tools/SharpCorePDF.Validation/FuzzTargets.cs | 254 +++++++++++++ tools/SharpCorePDF.Validation/Probe.cs | 251 +++++++++++++ tools/SharpCorePDF.Validation/Program.cs | 138 +++++++ .../SharpCorePDF.Validation.csproj | 52 +++ 32 files changed, 3730 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/robustness.yml create mode 100644 .github/workflows/validation.yml create mode 100644 docs/tools/corpus-robustness.py create mode 100644 docs/tools/render-compare.py create mode 100644 docs/tools/validate-arlington.py create mode 100644 docs/tools/validate-encryption-pdfjs.mjs create mode 100644 docs/tools/validate-encryption.py create mode 100644 docs/tools/validate-fonts.py create mode 100644 docs/tools/validate-incremental.py create mode 100644 docs/tools/validate-preflight-corpus.py create mode 100644 docs/tools/validate-text.py create mode 100644 docs/tools/validation-requirements.txt create mode 100644 docs/validation.md create mode 100644 tests/SharpCorePDF.Tests/PdfIncrementalInvariantTests.cs create mode 100644 tests/SharpCorePDF.Tests/PdfSignatureAttackTests.cs create mode 100644 tools/SharpCorePDF.Validation/CorpusWriter.cs create mode 100644 tools/SharpCorePDF.Validation/FuzzTargets.cs create mode 100644 tools/SharpCorePDF.Validation/Probe.cs create mode 100644 tools/SharpCorePDF.Validation/Program.cs create mode 100644 tools/SharpCorePDF.Validation/SharpCorePDF.Validation.csproj diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index 5ccda36..eee80fa 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -8,6 +8,13 @@ "dotnet-stryker" ], "rollForward": false + }, + "sharpfuzz.commandline": { + "version": "2.3.0", + "commands": [ + "sharpfuzz" + ], + "rollForward": false } } } \ No newline at end of file diff --git a/.github/workflows/compliance.yml b/.github/workflows/compliance.yml index 2b323f5..2f8af7e 100644 --- a/.github/workflows/compliance.yml +++ b/.github/workflows/compliance.yml @@ -214,7 +214,9 @@ jobs: # The same documents, read by two more implementations. --strict-tools makes a missing tool a failure # rather than a note, because a green run here is supposed to mean that all three agreed. PDFBox 3.0.8 - # and Ghostscript have no PDF/A-4 mode, so veraPDF is the only reading of that profile. + # and Ghostscript have no PDF/A-4 mode, so veraPDF stays the only verdict on that profile; PDFBox still + # parses the PDF 2.0 file and Ghostscript still rewrites it (as part 2), which is the reader test the + # other profiles get too. - name: Cross-validate with a second and a third implementation env: VERAPDF_HOME: ${{ env.VERAPDF_INSTALL }} @@ -223,7 +225,7 @@ jobs: output="$RUNNER_TEMP/SharpCorePDF.Examples/output" jar="$HOME/.cache/pdfbox/preflight-app-${PDFBOX_PREFLIGHT_VERSION}.jar" - for profile in 1b 2b 2u 3b 3u; do + for profile in 1b 2b 2u 3b 3u 4; do python3 docs/tools/validate-pdfa-cross.py --strict-tools \ --require-version "$VERAPDF_VERSION" \ --pdfbox "$jar" \ diff --git a/.github/workflows/robustness.yml b/.github/workflows/robustness.yml new file mode 100644 index 0000000..93de31b --- /dev/null +++ b/.github/workflows/robustness.yml @@ -0,0 +1,282 @@ +# Robustness: what the engine does with documents nobody here wrote, and with documents built to break it. +# +# validation.yml checks what the library writes, on every push. This workflow checks what it reads, weekly, +# because its inputs are large and slow: coverage-guided fuzzing of every decoder and of the document parser, +# two public corpora of real and deliberately broken files, and the PDF/A conversion measured against the +# veraPDF test corpus with veraPDF as the judge. None of it gates a pull request; a red run is a defect report. +# +# fuzz SharpFuzz instruments the three engine assemblies and libFuzzer drives one target per job +# (tools/SharpCorePDF.Validation, FuzzTargets.cs). A PdfException is the contract's refusal and +# not a finding; any other exception, a stack overflow or the memory limit is, and the input +# that caused it is uploaded. Reproduce one with: +# dotnet run --project tools/SharpCorePDF.Validation -- replay +# corpora corpus-robustness.py over the veraPDF corpus and the pdf.js test corpus, one process per +# file, under a time limit and a hard managed-heap limit. Fault, crash and timeout fail the run. +# preflight validate-preflight-corpus.py: every PDF/A file of the veraPDF corpus converted for its profile +# and the result judged by veraPDF. Report-only until its first run sets the baseline. +# +# Every external input is pinned: the libFuzzer driver by tag and SHA-256, SharpFuzz in +# .config/dotnet-tools.json, the corpora by commit, veraPDF by version (the same one compliance.yml pins). + +name: Robustness + +on: + schedule: + - cron: '0 3 * * 2' + workflow_dispatch: + inputs: + fuzz-minutes: + description: 'Minutes of fuzzing per target' + required: false + default: '30' + +permissions: + contents: read + +env: + LIBFUZZER_DOTNET_TAG: 'v2025.05.02.0904' + LIBFUZZER_DOTNET_SHA256: 'c2c2a90d94c409a4af339a0d4f244e0442c5a5d249be0f1252ba07871f285958' + VERAPDF_CORPUS_COMMIT: 'bb75f4f0073d9350dfd058c0162a367e6fadf25e' + PDFJS_TAG: 'v6.4.299' + PDFJS_COMMIT: 'd0991a0d53f3bb3a7307e3b83f279c1d117aafb3' + VERAPDF_VERSION: '1.30.4' + VERAPDF_INSTALL: ${{ github.workspace }}/.verapdf + +jobs: + fuzz: + name: Fuzz ${{ matrix.target }} + runs-on: ubuntu-latest + timeout-minutes: 60 + + strategy: + fail-fast: false + matrix: + target: [ document, dct, ccitt, lzw, flate, runlength ] + + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: Install the .NET SDKs + uses: actions/setup-dotnet@0f4a354dd5547a28c70b7853dbd95d5b67886e73 # v6 + with: + dotnet-version: | + 11.0.x + 10.0.x + + - name: Install SharpFuzz and the libFuzzer driver + run: | + set -euo pipefail + dotnet tool restore + curl --fail --silent --show-error --location --proto '=https' --tlsv1.2 \ + --output "$RUNNER_TEMP/libfuzzer-dotnet" \ + "https://github.com/Metalnem/libfuzzer-dotnet/releases/download/${LIBFUZZER_DOTNET_TAG}/libfuzzer-dotnet-ubuntu" + echo "${LIBFUZZER_DOTNET_SHA256} $RUNNER_TEMP/libfuzzer-dotnet" | sha256sum --check + chmod +x "$RUNNER_TEMP/libfuzzer-dotnet" + + # Two builds: an instrumented one for libFuzzer, and a plain one for the seeds and for replaying what + # libFuzzer finds, because instrumented code throws outside the fuzzer (see Program.cs). + - name: Build, then instrument the engine + run: | + set -euo pipefail + dotnet publish ./tools/SharpCorePDF.Validation -c Release -o "$RUNNER_TEMP/plain" + dotnet publish ./tools/SharpCorePDF.Validation -c Release -o "$RUNNER_TEMP/fuzz" + for layer in Domain Application Infrastructure; do + dotnet sharpfuzz "$RUNNER_TEMP/fuzz/SharpCorePDF.$layer.dll" + done + + - name: Seed the corpus + run: | + set -euo pipefail + tool="$RUNNER_TEMP/plain/SharpCorePDF.Validation.dll" + seeds="$RUNNER_TEMP/seeds" + assets=tests/SharpCorePDF.Tests/Assets + case "${{ matrix.target }}" in + document) dotnet "$tool" seed document "$seeds" "$assets/parity/corpus" "$assets/parity/labelled" "$assets/parity/forms" ;; + dct) dotnet "$tool" seed dct "$seeds" "$assets"/jpeg/*.jpg ;; + ccitt) dotnet "$tool" seed ccitt "$seeds" "$assets/ccitt/ccitt-g3.bin" "$assets/ccitt/ccitt-g4.bin" ;; + *) mkdir -p "$seeds" && printf '\x03\x00\x00\x3f\x3f' > "$seeds/empty" ;; + esac + dotnet "$tool" replay "${{ matrix.target }}" "$seeds" + + - name: Fuzz + env: + SCPDF_FUZZ_TARGET: ${{ matrix.target }} + MINUTES: ${{ github.event.inputs.fuzz-minutes || '30' }} + run: | + set -uo pipefail + mkdir -p artifacts/crashes + "$RUNNER_TEMP/libfuzzer-dotnet" \ + --target_path=dotnet --target_arg="$RUNNER_TEMP/fuzz/SharpCorePDF.Validation.dll" \ + -max_total_time=$(( MINUTES * 60 )) -timeout=30 -rss_limit_mb=4096 -max_len=262144 \ + -artifact_prefix=artifacts/crashes/ -print_final_stats=1 \ + "$RUNNER_TEMP/seeds" + status=$? + ls -la artifacts/crashes + if [ -n "$(ls -A artifacts/crashes)" ]; then + echo "::error::libFuzzer saved $(ls artifacts/crashes | wc -l) input(s) that broke the ${{ matrix.target }} target" + exit 1 + fi + exit $status + + - name: Upload what broke the engine + if: failure() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: fuzz-crashes-${{ matrix.target }} + path: artifacts/crashes + if-no-files-found: ignore + retention-days: 30 + + corpora: + name: Public corpora + runs-on: ubuntu-latest + timeout-minutes: 120 + + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: Install the .NET SDKs + uses: actions/setup-dotnet@0f4a354dd5547a28c70b7853dbd95d5b67886e73 # v6 + with: + dotnet-version: | + 11.0.x + 10.0.x + + - name: Fetch the corpora + run: | + set -euo pipefail + # The veraPDF corpus is small enough to clone without blobs and check out at the pinned commit; pdf.js + # is not, so it is cloned one commit deep at its tag and the commit is checked against the pin. + git clone -q --filter=blob:none --sparse https://github.com/veraPDF/veraPDF-corpus.git "$RUNNER_TEMP/verapdf-corpus" + git -C "$RUNNER_TEMP/verapdf-corpus" sparse-checkout set PDF_A-1b PDF_A-2b PDF_A-4 "ISO 32000-1" "ISO 32000-2" "Isartor test files" + git -C "$RUNNER_TEMP/verapdf-corpus" -c advice.detachedHead=false checkout -q "$VERAPDF_CORPUS_COMMIT" + + git clone -q --depth 1 --branch "$PDFJS_TAG" --filter=blob:none --sparse https://github.com/mozilla/pdf.js.git "$RUNNER_TEMP/pdfjs" + git -C "$RUNNER_TEMP/pdfjs" sparse-checkout set test/pdfs + test "$(git -C "$RUNNER_TEMP/pdfjs" rev-parse HEAD)" = "$PDFJS_COMMIT" + + find "$RUNNER_TEMP/verapdf-corpus" "$RUNNER_TEMP/pdfjs" -name '*.pdf' | wc -l + + - name: Build the probe + run: dotnet build ./tools/SharpCorePDF.Validation -c Release + + # The six known files fault on three recorded defects (docs/validation.md): an empty name as a + # dictionary key (issue3188, issue5896, pdfjs_wikipedia, poppler-742-0-fuzzed), a glyph whose box is + # upside down (issue14415) and a date whose offset is beyond 14 hours (TAMReview). + - name: Read every document + run: | + python3 docs/tools/corpus-robustness.py \ + --probe ./tools/SharpCorePDF.Validation/bin/Release/net11.0/SharpCorePDF.Validation.dll \ + --timeout 60 --heap-limit-mb 1024 --report artifacts/corpora.json \ + --known issue3188.pdf --known issue5896.pdf --known pdfjs_wikipedia.pdf \ + --known poppler-742-0-fuzzed.pdf --known issue14415.pdf --known TAMReview.pdf \ + "$RUNNER_TEMP/verapdf-corpus" "$RUNNER_TEMP/pdfjs/test/pdfs" + + - name: Upload the report + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: corpora + path: artifacts/corpora.json + if-no-files-found: warn + retention-days: 30 + + preflight: + name: PDF/A conversion against the veraPDF corpus + runs-on: ubuntu-latest + timeout-minutes: 180 + + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: Install the .NET SDKs + uses: actions/setup-dotnet@0f4a354dd5547a28c70b7853dbd95d5b67886e73 # v6 + with: + dotnet-version: | + 11.0.x + 10.0.x + + - name: Set up the JDK veraPDF runs on + uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6.1.0 + with: + distribution: 'zulu' + java-version: '17' + + - name: Cache the veraPDF installer + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/.cache/verapdf + key: verapdf-${{ env.VERAPDF_VERSION }}-installer + + # The same installation compliance.yml performs, so both workflows judge with the same validator. + - name: Install veraPDF + run: | + set -euo pipefail + band="${VERAPDF_VERSION%.*}" + archive="$HOME/.cache/verapdf/verapdf-greenfield-${VERAPDF_VERSION}-installer.zip" + mkdir -p "$(dirname "$archive")" + if [ ! -f "$archive" ]; then + curl --fail --silent --show-error --location \ + --proto '=https' --proto-redir '=https' --tlsv1.2 \ + --output "$archive" \ + "https://software.verapdf.org/releases/${band}/verapdf-greenfield-${VERAPDF_VERSION}-installer.zip" + fi + rm -rf "$VERAPDF_INSTALL" "$RUNNER_TEMP/verapdf" + mkdir -p "$VERAPDF_INSTALL" + unzip -q "$archive" -d "$RUNNER_TEMP/verapdf" + printf '%s\n' \ + '' \ + '' \ + ' ' \ + ' ' \ + " ${VERAPDF_INSTALL}" \ + ' ' \ + ' ' \ + ' ' \ + ' ' \ + ' ' \ + ' ' \ + ' ' \ + ' ' \ + ' ' \ + '' \ + > "$RUNNER_TEMP/auto-install.xml" + rm -f "/tmp/iz-veraPDF Software.tmp" "${TMPDIR:-/tmp}/iz-veraPDF Software.tmp" + bash "$RUNNER_TEMP/verapdf/verapdf-greenfield-${VERAPDF_VERSION}/verapdf-install" "$RUNNER_TEMP/auto-install.xml" + if [ ! -x "$VERAPDF_INSTALL/verapdf" ]; then + printf '%s\n' '#!/usr/bin/env bash' \ + "exec java -jar \"$VERAPDF_INSTALL\"/bin/cli-*.jar \"\$@\"" \ + > "$VERAPDF_INSTALL/verapdf" + chmod +x "$VERAPDF_INSTALL/verapdf" + fi + "$VERAPDF_INSTALL/verapdf" --version + + - name: Fetch the veraPDF corpus + run: | + set -euo pipefail + git clone -q --filter=blob:none --sparse https://github.com/veraPDF/veraPDF-corpus.git "$RUNNER_TEMP/verapdf-corpus" + git -C "$RUNNER_TEMP/verapdf-corpus" sparse-checkout set PDF_A-1b PDF_A-2b PDF_A-2u PDF_A-3b PDF_A-4 + git -C "$RUNNER_TEMP/verapdf-corpus" -c advice.detachedHead=false checkout -q "$VERAPDF_CORPUS_COMMIT" + + - name: Build the probe + run: dotnet build ./tools/SharpCorePDF.Validation -c Release + + - name: Convert and judge + run: | + python3 docs/tools/validate-preflight-corpus.py \ + --corpus "$RUNNER_TEMP/verapdf-corpus" \ + --probe ./tools/SharpCorePDF.Validation/bin/Release/net11.0/SharpCorePDF.Validation.dll \ + --verapdf "$VERAPDF_INSTALL/verapdf" \ + --report artifacts/preflight.json + + - name: Upload the report + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: preflight + path: artifacts/preflight.json + if-no-files-found: warn + retention-days: 30 diff --git a/.github/workflows/validation.yml b/.github/workflows/validation.yml new file mode 100644 index 0000000..6bb7cb0 --- /dev/null +++ b/.github/workflows/validation.yml @@ -0,0 +1,232 @@ +# Independent validation: what readers that share no code with this library make of what it writes. +# +# ci.yml answers whether the code builds and the suite passes; compliance.yml whether the archival output +# conforms. This workflow answers a third question -- whether other implementations read the documents the +# way the library meant them -- because a writer that only ever reads its own output back cannot see the +# mistakes it shares with itself. Every check here runs against the example output and against the validation +# corpus that tools/SharpCorePDF.Validation writes, whose manifest states what each document has to say. +# +# qpdf validate-pdf.py: the file parses, every page reads, every stream decodes +# text validate-text.py: pypdf, pdfminer.six and MuPDF extract the lines that were drawn +# encryption validate-encryption.py and the pdf.js leg: four readers open the RC4/AES matrix with the right +# password, refuse the wrong one, and read the revision and the permissions that were asked for +# fonts validate-fonts.py: fontTools and OTS read every embedded program, and the widths agree +# rendering render-compare.py: PDFium and MuPDF paint the same pages +# incremental validate-incremental.py: every revision stands alone, /ID[0] holds, an update only appends +# signatures validate-signatures.py --manifest: pyHanko agrees on integrity, trust and coverage +# fuzz seeds every fuzz target runs over its seeds in process, so a harness that rots is noticed here and +# not in the weekly fuzzing run (robustness.yml) +# hostile corpus-robustness.py over the documents it synthesizes for the failure classes the review named +# arlington validate-arlington.py against the veraPDF Arlington service (report-only until a baseline exists) +# +# --known-defect / --known names a failure that is recorded and not yet fixed. Each is strict: when the +# defect is fixed the check passes, the flag no longer matches, and the run fails until the flag is removed +# in the same change as the fix. The in-suite counterparts are the skipped tests in +# PdfSignatureAttackTests and PdfIncrementalInvariantTests. +# +# The validator versions are pinned in docs/tools/validation-requirements.txt, pdfjs-dist below. + +name: Independent validation + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + workflow_dispatch: + +permissions: + contents: read + +env: + PDFJS_VERSION: '6.4.299' + +jobs: + readers: + name: Read by other implementations + runs-on: ubuntu-latest + timeout-minutes: 40 + + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: Install the .NET SDKs + uses: actions/setup-dotnet@0f4a354dd5547a28c70b7853dbd95d5b67886e73 # v6 + with: + dotnet-version: | + 11.0.x + 10.0.x + + - name: Install the validators + run: | + set -euo pipefail + python3 -m venv "$RUNNER_TEMP/venv" + "$RUNNER_TEMP/venv/bin/pip" install --quiet -r docs/tools/validation-requirements.txt + echo "$RUNNER_TEMP/venv/bin" >> "$GITHUB_PATH" + npm install --silent --no-save --prefix "$RUNNER_TEMP/pdfjs" "pdfjs-dist@${PDFJS_VERSION}" + + - name: Build the examples and the validation harness + run: | + dotnet build ./examples/SharpCorePDF.Examples -c Release + dotnet build ./tools/SharpCorePDF.Validation -c Release + + - name: Write the documents + env: + TMPDIR: ${{ runner.temp }} + run: | + set -euo pipefail + dotnet run --project ./examples/SharpCorePDF.Examples -c Release --no-build + dotnet run --project ./tools/SharpCorePDF.Validation -c Release --no-build -- corpus "$RUNNER_TEMP/corpus" + + # protected.pdf is encrypted with a password the example makes for the run and prints; the encrypted + # documents qpdf does check are the corpus's, whose passwords the manifest carries + # (validate-encryption.py runs qpdf --check on each of them with its password). + - name: qpdf + run: | + set -euo pipefail + shopt -s nullglob + files=() + for file in "$RUNNER_TEMP"/SharpCorePDF.Examples/output/*.pdf "$RUNNER_TEMP"/corpus/*.pdf; do + case "$(basename "$file")" in protected.pdf|encrypted-*) continue ;; esac + files+=("$file") + done + # Both merge a page of NamedDests.pdf, which states no /MediaBox; the merge copies it without one, + # so the result breaks the rule that every page has a media box, and qpdf assumes Letter. + python docs/tools/validate-pdf.py --known-defect inspected.pdf --known-defect with-warnings.pdf "${files[@]}" + + - name: Text round trip + run: python docs/tools/validate-text.py "$RUNNER_TEMP/corpus" + + - name: Encryption matrix + run: | + set -euo pipefail + python docs/tools/validate-encryption.py "$RUNNER_TEMP/corpus" + node docs/tools/validate-encryption-pdfjs.mjs "$RUNNER_TEMP/corpus" \ + "$RUNNER_TEMP/pdfjs/node_modules/pdfjs-dist/legacy/build/pdf.mjs" + + - name: Embedded fonts + run: | + python docs/tools/validate-fonts.py \ + "$RUNNER_TEMP/SharpCorePDF.Examples/output" \ + "$RUNNER_TEMP/corpus/text-embedded.pdf" "$RUNNER_TEMP/corpus/text-merged.pdf" + + - name: Rendering, PDFium against MuPDF + run: | + python docs/tools/render-compare.py --report-dir artifacts/rendering --exclude protected.pdf \ + "$RUNNER_TEMP/SharpCorePDF.Examples/output" \ + "$RUNNER_TEMP/corpus/text-standard.pdf" "$RUNNER_TEMP/corpus/text-embedded.pdf" \ + "$RUNNER_TEMP/corpus/text-merged.pdf" "$RUNNER_TEMP/corpus/incremental.pdf" \ + "$RUNNER_TEMP/corpus/signed.pdf" + + # id-first: the editor writes a new identifier into both elements of /ID on an appended revision. + # appends-only: the editor's incremental update appends a renumbered copy of the whole document + # rather than the objects it changed. + - name: Incremental update invariants + run: | + python docs/tools/validate-incremental.py "$RUNNER_TEMP/corpus" \ + --known-defect id-first --known-defect appends-only + + # The same copy rewrites the signature dictionary after signing, which is why pyHanko cannot place the + # first signature of signed-then-edited.pdf at the end of a revision. + - name: Signatures, pyHanko + run: | + python docs/tools/validate-signatures.py --manifest "$RUNNER_TEMP/corpus/manifest.json" \ + --known-defect signed-then-edited.pdf:FirstSignature + + - name: Fuzz targets over their seeds + run: | + set -euo pipefail + tool=./tools/SharpCorePDF.Validation/bin/Release/net11.0/SharpCorePDF.Validation.dll + seeds="$RUNNER_TEMP/seeds" + dotnet "$tool" seed document "$seeds/document" tests/SharpCorePDF.Tests/Assets/parity/corpus "$RUNNER_TEMP/corpus" + dotnet "$tool" seed dct "$seeds/dct" tests/SharpCorePDF.Tests/Assets/jpeg/*.jpg + dotnet "$tool" seed ccitt "$seeds/ccitt" tests/SharpCorePDF.Tests/Assets/ccitt/ccitt-g3.bin tests/SharpCorePDF.Tests/Assets/ccitt/ccitt-g4.bin + for target in document dct ccitt; do + dotnet "$tool" replay "$target" "$seeds/$target" + done + + # Each of the four documents reaches a failure class the review named; each is a known defect until the + # change that fixes it removes its --known line. + - name: Hostile documents + run: | + set -euo pipefail + python docs/tools/corpus-robustness.py --synthesize "$RUNNER_TEMP/hostile" + python docs/tools/corpus-robustness.py \ + --probe ./tools/SharpCorePDF.Validation/bin/Release/net11.0/SharpCorePDF.Validation.dll \ + --timeout 120 --report artifacts/hostile.json \ + --known flate-bomb.pdf --known length-chain.pdf --known objstm-count.pdf --known jpeg-selector.pdf \ + "$RUNNER_TEMP/hostile" + + - name: Upload the documents and the reports + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: independent-validation + path: | + artifacts + ${{ runner.temp }}/corpus + ${{ runner.temp }}/SharpCorePDF.Examples/output/*.pdf + if-no-files-found: warn + retention-days: 14 + + arlington: + name: Arlington PDF Model + runs-on: ubuntu-latest + timeout-minutes: 30 + # The service's report format and the writer's baseline are confirmed by the first run on a runner; until + # then the job reports without blocking. Remove continue-on-error and pass --strict with the --accept list + # that run establishes. + continue-on-error: true + + services: + arlington: + image: verapdf/arlington:v1.30.4 + ports: + - 8080:8080 + + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: Install the .NET SDKs + uses: actions/setup-dotnet@0f4a354dd5547a28c70b7853dbd95d5b67886e73 # v6 + with: + dotnet-version: | + 11.0.x + 10.0.x + + - name: Write the documents + env: + TMPDIR: ${{ runner.temp }} + run: | + set -euo pipefail + dotnet run --project ./examples/SharpCorePDF.Examples -c Release + dotnet run --project ./tools/SharpCorePDF.Validation -c Release -- corpus "$RUNNER_TEMP/corpus" + + - name: Wait for the service + run: | + for attempt in $(seq 1 60); do + curl -sf http://localhost:8080/api/info > /dev/null && exit 0 + sleep 2 + done + echo "::error::the Arlington service did not answer on port 8080" + exit 1 + + - name: Check against the model + run: | + python3 docs/tools/validate-arlington.py --report-dir artifacts/arlington \ + --exclude protected.pdf \ + "$RUNNER_TEMP/SharpCorePDF.Examples/output" \ + "$RUNNER_TEMP/corpus/text-standard.pdf" "$RUNNER_TEMP/corpus/text-embedded.pdf" \ + "$RUNNER_TEMP/corpus/text-merged.pdf" "$RUNNER_TEMP/corpus/incremental.pdf" \ + "$RUNNER_TEMP/corpus/signed.pdf" + + - name: Upload the reports + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: arlington + path: artifacts/arlington + if-no-files-found: warn + retention-days: 14 diff --git a/.gitignore b/.gitignore index 4ca788b..a7b1b5e 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,10 @@ StrykerOutput/ *.sln.docstates *.DotSettings.user +# AI assistant state: prompts, transcripts and local settings stay on the machine that wrote them +.claude/ +CLAUDE.local.md + # Build logs *.binlog msbuild.log diff --git a/.specify/memory/constitution.md b/.specify/memory/constitution.md index af536c8..e45abe8 100644 --- a/.specify/memory/constitution.md +++ b/.specify/memory/constitution.md @@ -1,6 +1,13 @@ + + Exe + net11.0 + preview + SharpCorePDF.Validation + SharpCorePDF.Validation + enable + enable + nullable + true + latest-recommended + false + false + true + + + + + + + + + + + + + + + + + + + + From 8ad4c9e54663e4a4d467365154e25fbe4d360725 Mon Sep 17 00:00:00 2001 From: Andrew Tierney Date: Mon, 16 Nov 2026 11:55:24 +1100 Subject: [PATCH 02/15] Release pdf.js documents through their loading task pdf.js 6 has no PDFDocumentProxy.destroy(), so the encryption check's pdf.js leg reported every document as unopenable: the call threw inside the try that was meant to catch a wrong password. The loading task owns the document and destroying it releases both. The fix was made before the previous commit but an editor saved a stale copy of the file over it. --- docs/tools/validate-encryption-pdfjs.mjs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/tools/validate-encryption-pdfjs.mjs b/docs/tools/validate-encryption-pdfjs.mjs index 8ee5cd1..57a929c 100644 --- a/docs/tools/validate-encryption-pdfjs.mjs +++ b/docs/tools/validate-encryption-pdfjs.mjs @@ -26,8 +26,10 @@ async function open(file, password) { const data = new Uint8Array(await readFile(path.join(corpus, file))); const task = pdfjs.getDocument({data, password, isEvalSupported: false, verbosity: 0}); + // The loading task owns the document in pdf.js 6, which has no PDFDocumentProxy.destroy(): destroying + // the task is what releases both. try { - return await task.promise; + return {task, document: await task.promise}; } catch (error) { await task.destroy(); throw error; @@ -36,8 +38,8 @@ async function open(file, password) { async function refuses(file, password) { try { - const document = await open(file, password); - await document.destroy(); + const {task} = await open(file, password); + await task.destroy(); return false; } catch (error) { return error?.name === "PasswordException"; @@ -51,7 +53,7 @@ for (const entry of manifest.encryption) { const problems = []; try { - const document = await open(entry.file, entry.userPassword); + const {task, document} = await open(entry.file, entry.userPassword); const page = await document.getPage(1); const content = await page.getTextContent(); const text = normalise(content.items.map((item) => item.str + (item.hasEOL ? "\n" : "")).join("")); @@ -61,14 +63,14 @@ for (const entry of manifest.encryption) { problems.push(`page 1 text does not contain ${JSON.stringify(first)}`); } - await document.destroy(); + await task.destroy(); } catch (error) { problems.push(`the user password did not open it: ${error?.name}: ${error?.message}`); } try { - const document = await open(entry.file, entry.ownerPassword); - await document.destroy(); + const {task} = await open(entry.file, entry.ownerPassword); + await task.destroy(); } catch (error) { problems.push(`the owner password did not open it: ${error?.name}: ${error?.message}`); } From 7020f3bed853cb3fc0b1c4c12b4aef8242d74c73 Mon Sep 17 00:00:00 2001 From: Andrew Tierney Date: Mon, 16 Nov 2026 12:10:46 +1100 Subject: [PATCH 03/15] Refuse signature wrapping and bound what hostile input can allocate Signature wrapping: the validator required only that the byte range start at 0 and end at the end of the file, so bytes written into the unsigned gap could close the signature value early and add entries while the digest still matched, and the signature was reported valid and covering the whole document. The gap must now be exactly one hexadecimal string equal to the dictionary's /Contents; anything else is MalformedByteRange, integrity Unknown and CoversWholeDocument false. Decompression bombs: Flate, LZW and RunLength decoded without a limit, so a 1.1 MB document took the process past 1 GiB. PdfEngineOptions.MaxDecodedStreamBytes (128 MiB) now stops them; decode buffers above 16 MiB no longer stay in the shared array pool, and buffer growth no longer overflows. Stack overflow: a chain of streams whose /Length is the next stream was resolved by recursion, and 60,000 links ended the process. The chain is followed no deeper than MaxNestingDepth, then the stream is found by its endstream keyword, as for a /Length cycle. Allocations sized by the file: an object stream's /N is bounded by what the stream can hold, and a JPEG whose output is above the caller's budget is measured from its frame header instead of being decoded first; the working sizes are computed in 64 bits. Tests that fail without the fixes: PdfSignatureAttackTests (gap wrapping, gap shape), PdfFilterTests (limits per filter, content stream past the limit), PdfReaderRecoveryTests (length chain, indirect lengths, object stream /N), PdfJpegFilterTests (oversized frames, working buffers). The three hostile documents of validation.yml are read under a 1 GiB heap, and their --known flags are removed. --- .github/workflows/validation.yml | 7 +- CHANGELOG.md | 28 +++- docs/manual/src/10-dependency-injection.md | 14 +- docs/validation.md | 15 +- .../Pdf/JpegDecoder.cs | 78 ++++++++++ .../Pdf/PdfCrossReferenceReader.Streams.cs | 2 +- .../Pdf/PdfFileReader.ObjectStreams.cs | 22 +-- .../Pdf/PdfFileReader.Objects.cs | 14 +- .../Pdf/PdfFileReader.cs | 1 + .../Pdf/PdfFilters.Inflation.cs | 102 +++++++++++--- .../Pdf/PdfFilters.Lzw.cs | 21 ++- .../Pdf/PdfFilters.RunLength.cs | 24 +++- .../Pdf/PdfFilters.cs | 16 ++- .../Pdf/PdfFontDecoder.cs | 2 +- .../Pdf/PdfImageFilters.cs | 4 +- .../Pdf/PdfTextCollector.cs | 7 +- .../Pdf/PdfType1Encoding.cs | 2 +- .../Pdf/PooledByteBuffer.cs | 51 ++++++- .../PdfEditEngine.Overlays.cs | 2 +- .../PdfEngineOptions.cs | 23 +++ .../PdfPageContentReader.Content.cs | 7 +- .../PdfPageContentReader.Images.cs | 2 +- .../PdfPageContentReader.Metadata.cs | 2 +- .../PublicAPI.Unshipped.txt | 2 + .../Signatures/SignatureValidator.cs | 75 +++++++++- .../PdfEngineOptionsTests.cs | 10 ++ tests/SharpCorePDF.Tests/PdfFilterTests.cs | 96 +++++++++++++ .../SharpCorePDF.Tests/PdfJpegFilterTests.cs | 90 ++++++++++++ .../PdfReaderRecoveryTests.cs | 133 ++++++++++++++++++ .../PdfSignatureAttackTests.cs | 75 +++++++++- 30 files changed, 840 insertions(+), 87 deletions(-) diff --git a/.github/workflows/validation.yml b/.github/workflows/validation.yml index 6bb7cb0..de42cbe 100644 --- a/.github/workflows/validation.yml +++ b/.github/workflows/validation.yml @@ -146,8 +146,9 @@ jobs: dotnet "$tool" replay "$target" "$seeds/$target" done - # Each of the four documents reaches a failure class the review named; each is a known defect until the - # change that fixes it removes its --known line. + # Each of the four documents reaches a failure class the review named. flate-bomb, length-chain and + # objstm-count are refused or read since the decode limit, the length-chain bound and the /N bound; + # jpeg-selector is a known defect until the change that fixes it removes its --known line. - name: Hostile documents run: | set -euo pipefail @@ -155,7 +156,7 @@ jobs: python docs/tools/corpus-robustness.py \ --probe ./tools/SharpCorePDF.Validation/bin/Release/net11.0/SharpCorePDF.Validation.dll \ --timeout 120 --report artifacts/hostile.json \ - --known flate-bomb.pdf --known length-chain.pdf --known objstm-count.pdf --known jpeg-selector.pdf \ + --known jpeg-selector.pdf \ "$RUNNER_TEMP/hostile" - name: Upload the documents and the reports diff --git a/CHANGELOG.md b/CHANGELOG.md index 99ac86d..abe4bd9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,19 +33,37 @@ against an empty feed. reaches a consumer. [`docs/validation.md`](docs/validation.md) records each check, how it was shown to fail before it was trusted, and the defects below. +- **`PdfEngineOptions.MaxDecodedStreamBytes`** (128 MiB by default) bounds what one stream may decode to. + `MaxSourceBytes` bounds the input only, and Flate reaches about a thousand to one. + +### Fixed + +- **Security: signature wrapping.** The validator required only that the byte range start at 0 and end at the + end of the file, so bytes written into the unsigned gap could close the signature value early and add + entries while the digest still matched, and the signature was reported valid and covering the whole + document. The gap must now be exactly one hexadecimal string equal to the dictionary's `/Contents`; + anything else is `MalformedByteRange`, integrity `Unknown`, and `CoversWholeDocument` false. +- **Security: decompression bombs.** Flate, LZW and RunLength streams decoded without a limit, so a 1.1 MB + document took the process past 1 GiB. They now stop at `MaxDecodedStreamBytes`; buffer growth no longer + overflows past 1 GiB. +- **Security: stack overflow.** A chain of streams whose `/Length` is the next stream was resolved by + recursion, and 60,000 links ended the process. The chain is followed no deeper than `MaxNestingDepth`; + past it the stream is found by its `endstream` keyword, as for a `/Length` cycle. +- **Security: allocations sized by the file.** An object stream's `/N` sized two arrays before anything checked + it (513 bytes asked for 16 GiB), and a JPEG frame's 16-bit width and height sized the sample planes before + the caller's budget was consulted. `/N` is now bounded by what the stream can hold, a JPEG above the budget + is measured from its header without being decoded — the same answer as before, without the allocation — + and the working sizes are computed in 64 bits. + ### Known defects, found by the new checks and not yet fixed Each is reproduced, pinned by a skipped test or a strict `--known-defect` flag that fails the run once the defect is fixed, and listed in [`docs/validation.md`](docs/validation.md#known-defects). -- The signature validator does not require the gap between the byte ranges to be exactly the `/Contents` - string, so a gap carrying extra entries is still reported as covering the whole document. - An incremental update rewrites both elements of `/ID`, and appends a renumbered copy of the whole document rather than the objects it changed — which also rewrites the signature dictionary after signing. - A merge copies a page that states no `/MediaBox` without one. -- Hostile input: a Flate stream has no decoded-size limit, a chain of indirect `/Length` references overflows - the stack, an object stream's `/N` sizes an allocation unchecked, and a JPEG scan's Huffman table selector - is not range-checked. +- Hostile input: a JPEG scan's Huffman table selector is not range-checked. - Real-world input (the pdf.js test corpus): an empty name as a dictionary key stops a document opening, a glyph with an upside-down box faults text extraction, and `PdfDate.TryParse` throws on an out-of-range time-zone offset instead of answering false. diff --git a/docs/manual/src/10-dependency-injection.md b/docs/manual/src/10-dependency-injection.md index 953b10f..d46b5aa 100644 --- a/docs/manual/src/10-dependency-injection.md +++ b/docs/manual/src/10-dependency-injection.md @@ -144,6 +144,7 @@ defaults are deliberately permissive: | Option | Default | Effect | | --- | --- | --- | | `MaxSourceBytes` | 512 MiB | Largest source read into memory; larger sources are rejected | +| `MaxDecodedStreamBytes` | 128 MiB | Largest size one stream may decode to; a stream that would decode past it is refused | | `MaxNestingDepth` | 64 | Maximum dictionary/array nesting, page-tree depth and form field tree depth | | `MaxPagesPerSource` | 100 000 | Declared page limit; see the note below | | `ReconstructCrossReferenceOnFailure` | `true` | Rebuild a damaged or missing cross-reference table by scanning | @@ -154,8 +155,15 @@ The source limit is enforced by the reader, and exceeding it throws `PdfSourceFo detail that names the limit. It is not a streaming cap: the whole source is held in memory while it is processed, so the practical ceiling is your process's memory budget, not the number on the option. -`MaxNestingDepth` has four effects. It bounds how deep an object graph may be parsed, it bounds the -page-tree walk, it bounds the form field tree, and it bounds the tree the signature search descends. A +`MaxDecodedStreamBytes` bounds what a stream decodes to, which the source limit does not: Flate reaches about +a thousand to one, so a one-megabyte document can ask for a gigabyte. A content stream, form or font program +past the limit is reported as unreadable through the warning list, as other undecodable content is; an object +or cross-reference stream past it refuses the document with `PdfSourceFormatException`. Images are not refused +by it — their samples are carried only within `MaxInlineContentBytes` already. + +`MaxNestingDepth` has five effects. It bounds how deep an object graph may be parsed, it bounds how many +streams deep a chain of indirect `/Length` references is followed (past it, the stream is found by its +`endstream` keyword), it bounds the page-tree walk, it bounds the form field tree, and it bounds the tree the signature search descends. A page tree that is cut off contributes a `PageTreeTooDeep` warning rather than failing the merge; a field tree that is cut off contributes `FormFieldDepthExceeded` together with the fields above the cut, and the search for signatures reports the same warning and adds a `PdfSignatureProblemKind.FormFieldTreeTooDeep` @@ -176,7 +184,7 @@ file is binary; the marker costs a few bytes and prevents a class of corruption hard to diagnose. > `MaxPagesPerSource` is part of the options record but this version does not enforce it. Do not rely on -> it as a security boundary. For untrusted input, set `MaxSourceBytes`, inspect the document with +> it as a security boundary. For untrusted input, set `MaxSourceBytes` and `MaxDecodedStreamBytes`, inspect the document with > `IPdfInspector` and reject it if `PageCount` exceeds your own limit, before building a plan. ## Configuration file diff --git a/docs/validation.md b/docs/validation.md index 7986dba..6ce4585 100644 --- a/docs/validation.md +++ b/docs/validation.md @@ -83,25 +83,30 @@ fix. This is the list on 2026-11-16, every entry reproduced, not inferred: | Defect | Found by | Evidence | | --- | --- | --- | -| The signature validator does not check that the gap between the two byte ranges is exactly the `/Contents` string, so a gap that carries extra entries is still reported as covering the whole document (the signature-wrapping class) | #3 | `PdfSignatureAttackTests.Does_not_report_a_gap_that_holds_more_than_the_signature_value_as_covering_the_document` (skipped; fails when run) | | An incremental update writes a new identifier into both elements of `/ID`; ISO 32000-1 §14.4 keeps the first | #10 | `PdfIncrementalInvariantTests.Keeps_the_first_identifier_element_across_an_incremental_update` (skipped); `validate-incremental.py` `id-first` | | `UseIncrementalUpdate` appends a renumbered copy of the whole document rather than the objects it changed: every byte of the earlier revision is kept, but the file grows by its own size per edit, and the signature dictionary is written again after signing, so pyHanko places a signature followed by an edit as `CONTIGUOUS_BLOCK_FROM_START` with modification level `OTHER` instead of `ENTIRE_REVISION` | #10, #3 | `validate-incremental.py` `appends-only`; `validate-signatures.py` on `signed-then-edited.pdf` | | A merge copies a page that states no `/MediaBox` (and inherits none) without one, so the result breaks the rule that every page has a media box; qpdf repairs it by assuming Letter | #2 | `validate-pdf.py` on `inspected.pdf` and `with-warnings.pdf` (both merge a page of `NamedDests.pdf`) | -| A Flate stream has no limit on its decoded size: a 1.1 MB document inflates past 1 GiB | #12 | `flate-bomb.pdf`: `OutOfMemoryException` under a 1 GiB heap limit | -| A chain of indirect `/Length` references is resolved by recursion without a depth limit | #12 | `length-chain.pdf`: the process dies with a stack overflow, which no caller can catch | -| An object stream's `/N` sizes an allocation before anything checks it against the stream | #12 | `objstm-count.pdf` (513 bytes): `OutOfMemoryException` | | The JPEG decoder indexes its four Huffman tables with a scan's 4-bit selector unchecked | #12, #1 | `jpeg-selector.pdf`: `IndexOutOfRangeException` from `JpegDecoder.DecodeBlock` | | A dictionary key that is the empty name (`/` alone, which PDF syntax allows) makes `PdfDictionary.Set` throw `ArgumentException` while parsing, so the document does not open at all | #12 | pdf.js corpus: `issue3188.pdf`, `issue5896.pdf`, `pdfjs_wikipedia.pdf`, `poppler-742-0-fuzzed.pdf` | | A glyph whose box comes out upside down (a negative font size or a flipped font matrix) makes `PdfGlyph.BoundingBox` construct a `PdfRectangle` that throws, so text extraction faults | #12 | pdf.js corpus: `issue14415.pdf` | | `PdfDate.TryParse` throws `ArgumentOutOfRangeException` for a time-zone offset beyond 14 hours instead of answering false, so one malformed `/CreationDate` faults the whole inspection | #12 | pdf.js corpus: `TAMReview.pdf` | +Fixed since the list was first drawn up, each with the test that fails without its fix: + +| Defect | Fixed by | Test | +| --- | --- | --- | +| The signature validator accepted a `/ByteRange` gap holding more than the `/Contents` string (signature wrapping) | the gap must be exactly one hex string, equal to the dictionary's `/Contents` | `PdfSignatureAttackTests.Does_not_report_a_gap_that_holds_more_than_the_signature_value_as_covering_the_document` | +| No limit on the decoded size of a Flate, LZW or RunLength stream | `PdfEngineOptions.MaxDecodedStreamBytes` (128 MiB) | `PdfFilterTests.Refuses_a_flate_stream_that_decodes_past_the_limit` and its LZW, RunLength and content-stream companions | +| A chain of indirect `/Length` references overflowed the stack | the chain is followed no deeper than `MaxNestingDepth`, then the stream is found by its `endstream` | `PdfReaderRecoveryTests.Reads_a_stream_whose_length_is_a_chain_of_sixty_thousand_streams` | +| An object stream's `/N`, and a JPEG frame's width and height, sized allocations unchecked | `/N` is bounded by what the stream can hold; a JPEG above the caller's budget is measured from its header, and working sizes are computed in 64 bits | `PdfReaderRecoveryTests.Reads_an_object_stream_whose_count_its_header_cannot_hold`, `PdfJpegFilterTests.Measures_a_frame_too_large_for_the_budget_without_allocating_it` | + ## Corpus readings, 2026-11-16 | Corpus | Files | Read | Refused | Fault | Crash | Timeout | | --- | --- | --- | --- | --- | --- | --- | | veraPDF corpus, `PDF_A-2b` and `PDF_A-4` at `bb75f4f` | 1,471 | 1,471 | 0 | 0 | 0 | 0 | | pdf.js `test/pdfs` at `v6.4.299` | 987 | 977 | 4 | 6 | 0 | 0 | -| synthesized hostile documents | 4 | 0 | 0 | 3 | 1 | 0 | +| synthesized hostile documents, before the fixes | 4 | 0 | 0 | 3 | 1 | 0 | The six pdf.js faults are the three defects above. A sample of the pre-flight measurement (the first 60 files of `PDF_A-2b` and of `PDF_A-4`) wrote 108 results, refused 9 non-conforming inputs and refused 3 that diff --git a/src/SharpCorePDF.Infrastructure/Pdf/JpegDecoder.cs b/src/SharpCorePDF.Infrastructure/Pdf/JpegDecoder.cs index 2619fd8..528cf69 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/JpegDecoder.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/JpegDecoder.cs @@ -154,6 +154,16 @@ private bool Run(int maxBytes, out byte[]? pixels, out string? format, out int w return false; } + if (IsAboveBudget(maxBytes, out format, out width, out height)) + { + return format is not null; + } + + if (!WorkingBuffersFit()) + { + return false; + } + break; case 0xC2: progressive = true; @@ -163,6 +173,16 @@ private bool Run(int maxBytes, out byte[]? pixels, out string? format, out int w return false; } + if (IsAboveBudget(maxBytes, out format, out width, out height)) + { + return format is not null; + } + + if (!WorkingBuffersFit()) + { + return false; + } + break; case 0xDD: if (segment.Length < 2) @@ -320,6 +340,64 @@ private bool ParseHuffman(ReadOnlySpan segment) return true; } + /// + /// Answers, from the frame header alone, for an image whose samples the caller would not keep: the + /// format and the size are measured and nothing is decoded. + /// + /// The largest pixel buffer the caller keeps. + /// The pixel format the samples would state, or null for a component count the decoder has none for. + /// The width the frame states. + /// The height the frame states. + /// when the output is above the budget and the answer is final. + /// + /// The width and the height are two 16-bit numbers in the file: 65,535 squared is four gigasamples per + /// component, and the sample planes — and, for a progressive image, the coefficient grids at four bytes a + /// coefficient — were allocated and filled before the output was found too large to keep. The contract is + /// unchanged: such an image is measured and no pixels are returned. It is only no longer decoded first. + /// + private bool IsAboveBudget(int maxBytes, out string? format, out int measuredWidth, out int measuredHeight) + { + format = null; + measuredWidth = 0; + measuredHeight = 0; + + if ((long)width * height * components.Length <= maxBytes) + { + return false; + } + + format = components.Length switch + { + 1 => "Gray8", + 3 => "Rgb24", + 4 => "Cmyk32", + _ => null, + }; + measuredWidth = width; + measuredHeight = height; + return true; + } + + /// Determines whether the working buffers the frame asks for fit an array at all. + /// when every sample plane or coefficient grid fits. + /// The sizes are computed in 64 bits: in 32 they wrap, and a wrapped size is a small allocation that the decoder then writes past. + private bool WorkingBuffersFit() + { + var mcuColumns = (width + (8L * maxH) - 1) / (8L * maxH); + var mcuRows = (height + (8L * maxV) - 1) / (8L * maxV); + var perSample = progressive ? sizeof(int) : sizeof(byte); + + foreach (var component in components) + { + if (mcuColumns * 8 * component.H * mcuRows * 8 * component.V * perSample > Array.MaxLength) + { + return false; + } + } + + return true; + } + private bool ParseFrame(ReadOnlySpan segment) { if (segment.Length < 6 || segment[0] != 8) diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.Streams.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.Streams.cs index cec798b..7245f97 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.Streams.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.Streams.cs @@ -33,7 +33,7 @@ internal sealed partial class PdfCrossReferenceReader table.Add(new XrefEntry(objectNumber, offset, generation, XrefEntryType.Uncompressed)); } - var decoded = PdfFilters.Decode(stream, sourceName); + var decoded = PdfFilters.Decode(stream, sourceName, options.MaxDecodedStreamBytes); var widthType = 1; var widthField1 = 4; var widthField2 = 2; diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.ObjectStreams.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.ObjectStreams.cs index a208657..0a5df17 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.ObjectStreams.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.ObjectStreams.cs @@ -43,18 +43,24 @@ private void EnsureObjectStreamLoaded(long containerObjectNumber) } var dictionary = stream.Dictionary; - var count = (int)(dictionary.GetInteger("N") ?? 0); - var first = (int)(dictionary.GetInteger("First") ?? 0); + var declared = dictionary.GetInteger("N") ?? 0; + var first = dictionary.GetInteger("First") ?? 0; - if (count <= 0 || first < 0) + if (declared <= 0 || first < 0) { return; } - var decoded = PdfFilters.Decode(stream, SourceName); + var decoded = PdfFilters.Decode(stream, SourceName, options.MaxDecodedStreamBytes); + + // /N sizes two arrays, and it is a number the file states: 2^31 - 1 of them would ask for + // gigabytes from a stream of a few hundred bytes. Every pair of the header takes at least four + // bytes ("1 0 "), so the stream itself bounds how many objects it can describe, and the arrays + // are sized by that bound; a header that ends before /N says so below. + var count = (int)Math.Min(declared, (decoded.Length / 4L) + 1); var header = new PdfObjectParser(decoded, SourceName, 0, null, options.MaxNestingDepth); var numbers = new long[count]; - var offsets = new int[count]; + var offsets = new long[count]; for (var index = 0; index < count; index++) { @@ -62,12 +68,12 @@ private void EnsureObjectStreamLoaded(long containerObjectNumber) { warnings.Add(new( PdfWarningKind.ObjectStreamTruncated, - $"'{SourceName}': object stream {containerObjectNumber} declares {count} objects but its header ends early.")); + $"'{SourceName}': object stream {containerObjectNumber} declares {declared} objects but its header ends early.")); break; } numbers[index] = number; - offsets[index] = (int)offset; + offsets[index] = offset; } for (var index = 0; index < count; index++) @@ -79,7 +85,7 @@ private void EnsureObjectStreamLoaded(long containerObjectNumber) continue; } - var parser = new PdfObjectParser(decoded, SourceName, position, null, options.MaxNestingDepth); + var parser = new PdfObjectParser(decoded, SourceName, (int)position, null, options.MaxNestingDepth); fromObjectStreams[numbers[index]] = parser.ParseObject(); } } diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.Objects.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.Objects.cs index 1a39e97..4eeaa67 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.Objects.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.Objects.cs @@ -72,23 +72,33 @@ private PdfVersion ReadVersion() /// /// Resolves a stream /Length that is itself an indirect reference. The guard set breaks - /// the cycle that such a reference creates when the length object is the stream itself. + /// the cycle that such a reference creates when the length object is the stream itself, and the + /// depth guard breaks a chain: a length that is a stream whose length is a stream, and so on. /// /// The object number of the length object. /// The length, or when it cannot be resolved safely. + /// + /// Each link of a chain is resolved inside the one before it, so a chain of tens of thousands of + /// streams would overflow the stack — which no caller can catch: the process ends. A well-formed + /// length needs one level; past the length is answered + /// as unknown, exactly as a cycle is, and the parser finds the end of the stream by its keyword. + /// private long? ResolveIndirectLength(long objectNumber) { - if (!resolvingLengths.Add(objectNumber)) + if (lengthResolutionDepth >= options.MaxNestingDepth || !resolvingLengths.Add(objectNumber)) { return null; } + lengthResolutionDepth++; + try { return Resolve(objectNumber) is PdfInteger integer ? integer.Value : null; } finally { + lengthResolutionDepth--; resolvingLengths.Remove(objectNumber); } } diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.cs index 0c1d647..fcec760 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.cs @@ -20,6 +20,7 @@ internal sealed partial class PdfFileReader private readonly HashSet loadedObjectStreams = []; private readonly HashSet resolvingObjectStreams = []; private readonly HashSet resolvingLengths = []; + private int lengthResolutionDepth; private PageTreeWalk? pageTree; private PdfDictionary? catalog; diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.Inflation.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.Inflation.cs index e3c2f47..8fe69f2 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.Inflation.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.Inflation.cs @@ -12,30 +12,79 @@ internal static partial class PdfFilters private const int InflateChunkSize = 16 * 1024; private const int MaximumInitialInflateCapacity = 1024 * 1024; - /// Attempts to decompress Flate data, tolerating the raw deflate streams some producers write. - /// The compressed bytes. - /// The decompressed bytes when decompression succeeded. - /// when the data could be decompressed. - public static bool TryInflate(ReadOnlyMemory data, out byte[] result) + /// + /// The decoded size a stream is held to when the caller has no to read it + /// from; the same value as 's default. + /// + internal const long DefaultMaxDecodedBytes = 128L * 1024 * 1024; + + /// What one attempt at inflating a stream came to. + private enum InflateOutcome { - result = []; + /// The stream inflated within the limit. + Inflated, + + /// The bytes are not a stream this wrapper can inflate. + Invalid, - return !data.IsEmpty - && (TryInflateCore(data, zlibWrapper: true, out result) - || TryInflateCore(data, zlibWrapper: false, out result)); + /// The stream inflates past the limit; it was stopped there. + TooLarge, } - /// Decompresses Flate data. + /// Attempts to decompress Flate data, tolerating the raw deflate streams some producers write. + /// The compressed bytes. + /// The decompressed bytes when decompression succeeded. + /// when the data could be decompressed within the default limit. + public static bool TryInflate(ReadOnlyMemory data, out byte[] result) => + TryInflate(data, DefaultMaxDecodedBytes, out result) == InflateOutcome.Inflated; + + /// Decompresses Flate data under the default limit on its decoded size. /// The compressed bytes. /// The name of the source, used in diagnostics. /// The decompressed bytes. - /// The data could not be decompressed. + /// The data could not be decompressed, or decodes past the limit. public static byte[] Inflate(ReadOnlyMemory data, string sourceName) => - TryInflate(data, out var result) - ? result - : throw new PdfSourceFormatException(sourceName, "a Flate compressed stream could not be decompressed."); + Inflate(data, sourceName, DefaultMaxDecodedBytes); - private static bool TryInflateCore(ReadOnlyMemory data, bool zlibWrapper, out byte[] result) + /// Decompresses Flate data, refusing a stream that decodes to more than a limit. + /// The compressed bytes. + /// The name of the source, used in diagnostics. + /// The largest number of bytes the stream may decode to. + /// The decompressed bytes. + /// The data could not be decompressed, or decodes past the limit. + public static byte[] Inflate(ReadOnlyMemory data, string sourceName, long maxDecodedBytes) => + TryInflate(data, maxDecodedBytes, out var result) switch + { + InflateOutcome.Inflated => result, + InflateOutcome.TooLarge => throw DecodedTooLarge(sourceName, "a Flate", maxDecodedBytes), + _ => throw new PdfSourceFormatException(sourceName, "a Flate compressed stream could not be decompressed."), + }; + + /// Builds the refusal of a stream that decodes past the limit. + /// The name of the source. + /// The kind of stream, for the message. + /// The limit that was reached. + /// The exception to throw. + internal static PdfSourceFormatException DecodedTooLarge(string sourceName, string what, long maxDecodedBytes) => + new(sourceName, $"{what} stream decodes to more than {maxDecodedBytes} bytes, the limit PdfEngineOptions.MaxDecodedStreamBytes sets, so it was not decoded."); + + private static InflateOutcome TryInflate(ReadOnlyMemory data, long maxDecodedBytes, out byte[] result) + { + result = []; + + if (data.IsEmpty) + { + return InflateOutcome.Invalid; + } + + var outcome = TryInflateCore(data, zlibWrapper: true, maxDecodedBytes, out result); + + return outcome == InflateOutcome.Invalid + ? TryInflateCore(data, zlibWrapper: false, maxDecodedBytes, out result) + : outcome; + } + + private static InflateOutcome TryInflateCore(ReadOnlyMemory data, bool zlibWrapper, long maxDecodedBytes, out byte[] result) { result = []; @@ -51,7 +100,8 @@ private static bool TryInflateCore(ReadOnlyMemory data, bool zlibWrapper, // stream surprises us. Growth is progressive instead of one guess, so a small stream never // reserves the memory of a large one. using var output = new PooledByteBuffer( - (int)Math.Clamp((long)data.Length * 4, InflateChunkSize, MaximumInitialInflateCapacity)); + (int)Math.Clamp((long)data.Length * 4, InflateChunkSize, MaximumInitialInflateCapacity), + poolLargeArrays: false); while (true) { @@ -63,19 +113,27 @@ private static bool TryInflateCore(ReadOnlyMemory data, bool zlibWrapper, break; } + // Flate reaches about a thousand to one, so the limit on the input does not bound this + // buffer: a megabyte of zeros is a gigabyte here. The stream is stopped at the limit + // instead of being allowed to take the process with it. + if (output.Length + (long)read > maxDecodedBytes) + { + return InflateOutcome.TooLarge; + } + output.Advance(read); } result = output.ToExactArray(); - return true; + return InflateOutcome.Inflated; } catch (InvalidDataException) { - return false; + return InflateOutcome.Invalid; } catch (IOException) { - return false; + return InflateOutcome.Invalid; } } @@ -91,9 +149,9 @@ private static MemoryStream OpenInput(ReadOnlyMemory data) => ? new MemoryStream(array, segment.Offset, segment.Count, writable: false) : new MemoryStream(data.ToArray(), writable: false); - private static byte[] ApplyFlate(ReadOnlyMemory data, PdfDictionary? parameters, string sourceName) + private static byte[] ApplyFlate(ReadOnlyMemory data, PdfDictionary? parameters, string sourceName, long maxDecodedBytes) { - var inflated = Inflate(data, sourceName); + var inflated = Inflate(data, sourceName, maxDecodedBytes); // Every parameter of the filter is optional and every one of them is absent when the stream has no // filter dictionary at all, so the dictionary is established once here instead of being tested @@ -134,5 +192,5 @@ private static byte[] ApplyFlate(ReadOnlyMemory data, PdfDictionary? param public static byte[] InflateAndUnpredict( ReadOnlyMemory data, PdfDictionary? parameters, - string sourceName) => ApplyFlate(data, parameters, sourceName); + string sourceName) => ApplyFlate(data, parameters, sourceName, DefaultMaxDecodedBytes); } diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.Lzw.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.Lzw.cs index ee2e587..fd2162a 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.Lzw.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.Lzw.cs @@ -143,18 +143,31 @@ internal static int ReadEarlyChange(PdfDictionary? parameters) => /// The compressed bytes. /// The decode parameters of the filter, which state EarlyChange. /// The name of the source, used in diagnostics. + /// The largest number of bytes the stream may decode to. /// The decoded bytes. - /// The stream is not a valid LZW stream. + /// The stream is not a valid LZW stream, or decodes past the limit. internal static byte[] ApplyLzw( ReadOnlyMemory data, PdfDictionary? parameters, - string sourceName) + string sourceName, + long maxDecodedBytes) { - var output = new PooledByteBuffer(); + var output = new PooledByteBuffer(poolLargeArrays: false); try { - if (!TryDecodeLzw(data.Span, ReadEarlyChange(parameters), output.Write, out _)) + // A 12-bit code can stand for 4,096 bytes, so the input limit does not bound this buffer either. + void Write(ReadOnlySpan bytes) + { + if (output.Length + (long)bytes.Length > maxDecodedBytes) + { + throw DecodedTooLarge(sourceName, "an LZW", maxDecodedBytes); + } + + output.Write(bytes); + } + + if (!TryDecodeLzw(data.Span, ReadEarlyChange(parameters), Write, out _)) { throw new PdfSourceFormatException(sourceName, "an LZW stream could not be decompressed."); } diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.RunLength.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.RunLength.cs index 2acb163..2b029b1 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.RunLength.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.RunLength.cs @@ -24,11 +24,21 @@ internal static partial class PdfFilters /// reaches past the end contributes the bytes that are there, and a repeated run with no value byte /// contributes nothing. /// - public static byte[] DecodeRunLength(ReadOnlyMemory data) + public static byte[] DecodeRunLength(ReadOnlyMemory data) => + DecodeRunLength(data, "a stream", DefaultMaxDecodedBytes); + + /// Decodes a run-length encoded stream, refusing one that decodes past a limit. + /// The encoded bytes. + /// The name of the source, used in diagnostics. + /// The largest number of bytes the stream may decode to. + /// The decoded bytes. + /// The stream decodes past the limit. + /// Two bytes can stand for 128, so the limit on the input does not bound the output. + public static byte[] DecodeRunLength(ReadOnlyMemory data, string sourceName, long maxDecodedBytes) { // The output is at least as large as the input for the literal case, and can be 128 times larger // for the repeat case, so the buffer starts at the input length and grows only when it must. - using var result = new PooledByteBuffer(data.Length); + using var result = new PooledByteBuffer(data.Length, poolLargeArrays: false); var span = data.Span; var index = 0; @@ -48,6 +58,11 @@ public static byte[] DecodeRunLength(ReadOnlyMemory data) // end, which ends the loop. The explicit guard a survivor asked about is this comment. var count = Math.Min(control + 1, span.Length - index); + if (result.Length + (long)count > maxDecodedBytes) + { + throw DecodedTooLarge(sourceName, "a RunLength", maxDecodedBytes); + } + result.Write(span.Slice(index, count)); index += count; continue; @@ -60,6 +75,11 @@ public static byte[] DecodeRunLength(ReadOnlyMemory data) break; } + if (result.Length + (long)repeat > maxDecodedBytes) + { + throw DecodedTooLarge(sourceName, "a RunLength", maxDecodedBytes); + } + var value = span[index++]; for (var count = 0; count < repeat; count++) diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.cs index 261f5a8..5552260 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.cs @@ -27,7 +27,17 @@ public static bool HasFlateCompression(PdfDictionary dictionary) /// The decoded bytes. /// The stream uses a filter the engine does not implement. /// A compressed stream could not be decompressed. - public static byte[] Decode(PdfStream stream, string sourceName) + public static byte[] Decode(PdfStream stream, string sourceName) => + Decode(stream, sourceName, DefaultMaxDecodedBytes); + + /// Decodes a stream, applying every filter it declares in order, under a limit on its decoded size. + /// The stream to decode. + /// The name of the source, used in diagnostics. + /// The largest number of bytes any one filter may decode to. + /// The decoded bytes. + /// The stream uses a filter the engine does not implement. + /// A compressed stream could not be decompressed, or decodes past the limit. + public static byte[] Decode(PdfStream stream, string sourceName, long maxDecodedBytes) { ArgumentNullException.ThrowIfNull(stream); @@ -51,8 +61,8 @@ public static byte[] Decode(PdfStream stream, string sourceName) decoded = name switch { - "FlateDecode" or "Fl" => ApplyFlate(current, parametersForFilter, sourceName), - "LZWDecode" or "LZW" => ApplyLzw(current, parametersForFilter, sourceName), + "FlateDecode" or "Fl" => ApplyFlate(current, parametersForFilter, sourceName, maxDecodedBytes), + "LZWDecode" or "LZW" => ApplyLzw(current, parametersForFilter, sourceName, maxDecodedBytes), "ASCIIHexDecode" or "AHx" => DecodeAsciiHex(current), "ASCII85Decode" or "A85" => DecodeAscii85(current), _ => throw new PdfUnsupportedFeatureException( diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfFontDecoder.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfFontDecoder.cs index 7a31da3..33e41d8 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfFontDecoder.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfFontDecoder.cs @@ -313,7 +313,7 @@ private static PdfFont Create(PdfFileReader reader, string resourceName, PdfDict try { - data = PdfFilters.Decode(stream, reader.SourceName); + data = PdfFilters.Decode(stream, reader.SourceName, reader.Options.MaxDecodedStreamBytes); } catch (PdfUnsupportedFeatureException) { diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfImageFilters.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfImageFilters.cs index afa4917..15951b7 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfImageFilters.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfImageFilters.cs @@ -23,7 +23,7 @@ internal readonly record struct PdfPayload( /// /// /// -/// answers a different question and answers it correctly: an object +/// answers a different question and answers it correctly: an object /// stream that cannot be inflated means a broken document, so it raises. An image is different — a /// JPXDecode scan is a perfectly valid document this library simply does not decode — so this /// class reports what it could not do and hands back the raw bytes (NFR-7.6, NFR-7.8). @@ -361,7 +361,7 @@ private static bool TryDecodeOne( case "RunLengthDecode": { - var decoded = PdfFilters.DecodeRunLength(data); + var decoded = PdfFilters.DecodeRunLength(data, sourceName, PdfFilters.DefaultMaxDecodedBytes); length = decoded.Length; bytes = Keep(decoded, cap); diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfTextCollector.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfTextCollector.cs index 5a68441..b99551d 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfTextCollector.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfTextCollector.cs @@ -382,7 +382,7 @@ private void PaintForm(string name, PdfDictionary? resources, TextState state, i try { - if (!TryDecodeContent(stream, out var formContent)) + if (!TryDecodeContent(stream, reader.Options.MaxDecodedStreamBytes, out var formContent)) { warnings.Add(new( PdfWarningKind.ContentStreamUnreadable, @@ -432,13 +432,14 @@ private string DescribeUnmapped(PdfFont font) => string.Create( /// Decodes the content stream of a form. /// The stream. + /// The largest number of bytes the stream may decode to. /// The decoded bytes when decoding succeeded. /// when the stream could be decoded. - private static bool TryDecodeContent(PdfStream stream, out byte[] content) + private static bool TryDecodeContent(PdfStream stream, long maxDecodedBytes, out byte[] content) { try { - content = PdfFilters.Decode(stream, "form content stream"); + content = PdfFilters.Decode(stream, "form content stream", maxDecodedBytes); return true; } catch (Domain.Exceptions.PdfUnsupportedFeatureException) diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfType1Encoding.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfType1Encoding.cs index 2a19346..84eddd9 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfType1Encoding.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfType1Encoding.cs @@ -52,7 +52,7 @@ internal static class PdfType1Encoding try { - bytes = PdfFilters.Decode(program, reader.SourceName); + bytes = PdfFilters.Decode(program, reader.SourceName, reader.Options.MaxDecodedStreamBytes); } catch (PdfUnsupportedFeatureException) { diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PooledByteBuffer.cs b/src/SharpCorePDF.Infrastructure/Pdf/PooledByteBuffer.cs index 12615ac..5a9f02c 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PooledByteBuffer.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PooledByteBuffer.cs @@ -24,17 +24,32 @@ internal sealed class PooledByteBuffer : IDisposable { private const int MinimumCapacity = 16; + /// + /// The largest array a decode buffer takes from the pool. Above it the array is allocated and left to the + /// collector, because the shared pool keeps what is returned to it: a document that inflates a stream to + /// the decode limit in every read pass would otherwise leave hundreds of megabytes parked in the pool, + /// and the next pass would allocate beside them. + /// + private const int MaximumPooledDecodeLength = 16 * 1024 * 1024; + private byte[] array; private int length; + private readonly bool poolLargeArrays; private bool returned; /// Initializes a new instance of the class. /// The number of bytes to rent up front. - public PooledByteBuffer(int initialCapacity = 256) + /// + /// Whether arrays above 16 MiB go through the pool too. The read of a source keeps them pooled, because + /// reusing the one buffer a document is read into is what keeps a large read at one allocation; a decode + /// buffer does not, because its size is whatever a stream inflates to and the pool would keep it. + /// + public PooledByteBuffer(int initialCapacity = 256, bool poolLargeArrays = true) { ArgumentOutOfRangeException.ThrowIfNegative(initialCapacity); - array = ArrayPool.Shared.Rent(Math.Max(initialCapacity, MinimumCapacity)); + this.poolLargeArrays = poolLargeArrays; + array = Rent(Math.Max(initialCapacity, MinimumCapacity)); } /// Gets the number of bytes that have been written. @@ -112,7 +127,7 @@ public void Dispose() } returned = true; - ArrayPool.Shared.Return(array); + Return(array); array = []; length = 0; } @@ -124,14 +139,36 @@ private void EnsureFree(int sizeHint) return; } - Grow(length + sizeHint); + Grow((long)length + sizeHint); } - private void Grow(int minimumCapacity) + private void Grow(long minimumCapacity) { - var grown = ArrayPool.Shared.Rent(Math.Max(minimumCapacity, array.Length * 2)); + // Doubling is computed in 64 bits and clamped to the largest array the runtime allows, so a buffer + // past a gigabyte grows to the limit instead of to a negative size; a request beyond the limit is + // refused. The decode limits keep every caller below it, so this is a guard and not a path. + if (minimumCapacity > Array.MaxLength) + { + throw new InvalidOperationException($"A buffer of {minimumCapacity} bytes is larger than the runtime allows."); + } + + var capacity = (int)Math.Min(Math.Max(minimumCapacity, 2L * array.Length), Array.MaxLength); + var grown = Rent(capacity); array.AsSpan(0, length).CopyTo(grown); - ArrayPool.Shared.Return(array); + Return(array); array = grown; } + + private byte[] Rent(int length) => + poolLargeArrays || length <= MaximumPooledDecodeLength + ? ArrayPool.Shared.Rent(length) + : GC.AllocateUninitializedArray(length); + + private void Return(byte[] rented) + { + if (poolLargeArrays || rented.Length <= MaximumPooledDecodeLength) + { + ArrayPool.Shared.Return(rented); + } + } } diff --git a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Overlays.cs b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Overlays.cs index 7994f4e..23e7ab5 100644 --- a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Overlays.cs +++ b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Overlays.cs @@ -252,7 +252,7 @@ private static void AppendOverlayContent(PdfFileReader reader, PdfObject? value, return; } - content.Write(PdfFilters.Decode(stream, reader.SourceName)); + content.Write(PdfFilters.Decode(stream, reader.SourceName, reader.Options.MaxDecodedStreamBytes)); content.WriteByte((byte)'\n'); } diff --git a/src/SharpCorePDF.Infrastructure/PdfEngineOptions.cs b/src/SharpCorePDF.Infrastructure/PdfEngineOptions.cs index 3cdff45..8af71bd 100644 --- a/src/SharpCorePDF.Infrastructure/PdfEngineOptions.cs +++ b/src/SharpCorePDF.Infrastructure/PdfEngineOptions.cs @@ -18,6 +18,24 @@ public sealed record PdfEngineOptions /// public long MaxSourceBytes { get; init; } = 512L * 1024 * 1024; + /// + /// Gets the largest number of bytes one filter of one stream may decode to. A stream that would decode + /// past it is refused with a instead of + /// exhausting the process. + /// + /// + /// bounds what is read, not what it decodes to: Flate reaches about a + /// thousand to one, LZW four thousand and RunLength 128, so a one-megabyte document can ask for a + /// gigabyte. The default of 128 MiB is far above any real content stream, object stream or font program, + /// and low enough that refusing a stream at the limit — whose buffer doubles on the way there, on the + /// large-object heap, which the collector does not compact by default — stays reliably inside a 1 GiB + /// process when every read pass meets the same stream; the largest value accepted is the largest array + /// the runtime allows. An image is not refused by this limit — its samples are kept only + /// within already — but every other stream the engine has to + /// materialise is: content, forms, object streams, cross-reference streams and font programs. + /// + public long MaxDecodedStreamBytes { get; init; } = 128L * 1024 * 1024; + /// Gets the maximum nesting depth of dictionaries and arrays. public int MaxNestingDepth { get; init; } = 64; @@ -74,6 +92,11 @@ public IReadOnlyList Validate() errors.Add($"MaxSourceBytes must be at least 1024, but is {MaxSourceBytes}."); } + if (MaxDecodedStreamBytes < 1024 || MaxDecodedStreamBytes > Array.MaxLength) + { + errors.Add($"MaxDecodedStreamBytes must be between 1024 and {Array.MaxLength}, but is {MaxDecodedStreamBytes}."); + } + if (MaxNestingDepth is < 4 or > 1024) { errors.Add($"MaxNestingDepth must be between 4 and 1024, but is {MaxNestingDepth}."); diff --git a/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Content.cs b/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Content.cs index ae14104..6eaa240 100644 --- a/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Content.cs +++ b/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Content.cs @@ -64,7 +64,7 @@ public sealed partial class PdfPageContentReader foreach (var stream in streams) { - if (TryDecodeStream(stream, out var decoded)) + if (TryDecodeStream(stream, reader.Options.MaxDecodedStreamBytes, out var decoded)) { // The separator goes between the streams, not after the last one: a single stream is its // own sequence, and an inline image that runs to the end of it must not gain a byte. @@ -114,13 +114,14 @@ private static void CollectStreams(PdfFileReader reader, PdfArray array, ListDecodes a stream whose filters are the ones a content stream may use. /// The stream. + /// The largest number of bytes the stream may decode to. /// The decoded bytes when decoding succeeded. /// when the stream could be decoded. - private static bool TryDecodeStream(PdfStream stream, out byte[] decoded) + private static bool TryDecodeStream(PdfStream stream, long maxDecodedBytes, out byte[] decoded) { try { - decoded = PdfFilters.Decode(stream, "content stream"); + decoded = PdfFilters.Decode(stream, "content stream", maxDecodedBytes); return true; } catch (Domain.Exceptions.PdfUnsupportedFeatureException) diff --git a/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Images.cs b/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Images.cs index 9f9c48d..e56a77b 100644 --- a/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Images.cs +++ b/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Images.cs @@ -358,7 +358,7 @@ private void Paint( return; } - if (TryDecodeStream(stream, out var formContent)) + if (TryDecodeStream(stream, reader.Options.MaxDecodedStreamBytes, out var formContent)) { var formResources = reader.ResolveDictionary(stream.Dictionary.Get("Resources")) ?? resources; var formMatrix = ReadMatrix(reader, stream.Dictionary.Get("Matrix")) ?? PdfMatrix.Identity; diff --git a/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Metadata.cs b/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Metadata.cs index 027f05b..3681e29 100644 --- a/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Metadata.cs +++ b/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Metadata.cs @@ -35,7 +35,7 @@ public async Task ReadMetadataAsync( return new PdfXmpResult { Warnings = [.. reader.Warnings, .. warnings] }; } - if (!TryDecodeStream(stream, out var packet)) + if (!TryDecodeStream(stream, reader.Options.MaxDecodedStreamBytes, out var packet)) { warnings.Add(new( PdfWarningKind.XmpPacketMalformed, diff --git a/src/SharpCorePDF.Infrastructure/PublicAPI.Unshipped.txt b/src/SharpCorePDF.Infrastructure/PublicAPI.Unshipped.txt index 7dc5c58..f73aab7 100644 --- a/src/SharpCorePDF.Infrastructure/PublicAPI.Unshipped.txt +++ b/src/SharpCorePDF.Infrastructure/PublicAPI.Unshipped.txt @@ -1 +1,3 @@ #nullable enable +SharpCorePDF.Infrastructure.PdfEngineOptions.MaxDecodedStreamBytes.get -> long +SharpCorePDF.Infrastructure.PdfEngineOptions.MaxDecodedStreamBytes.init -> void diff --git a/src/SharpCorePDF.Infrastructure/Signatures/SignatureValidator.cs b/src/SharpCorePDF.Infrastructure/Signatures/SignatureValidator.cs index d0254eb..74bb5f0 100644 --- a/src/SharpCorePDF.Infrastructure/Signatures/SignatureValidator.cs +++ b/src/SharpCorePDF.Infrastructure/Signatures/SignatureValidator.cs @@ -258,7 +258,7 @@ public static PdfSignatureValidation Validate(PdfFileReader reader, SignatureFie $"The timestamp of '{field.FieldName}' is present but could not be read, so the moment it claims cannot be relied on.")); } - if (reading.Usable && !reading.CoversWhole) + if (reading.Usable && reading.GapHoldsValue && !reading.CoversWhole) { var revisions = CountRevisions(reader.Data, reader.Data.LongLength); @@ -278,6 +278,7 @@ public static PdfSignatureValidation Validate(PdfFileReader reader, SignatureFie /// The covered ranges. /// Whether the dictionary carries four usable numbers. /// Whether the ranges are ordered and inside the document. + /// Whether the bytes the ranges leave out are exactly the signature value. /// The DER structure of the signature value, without its padding. /// The parsed signature, when it parsed. /// Why the signature value could not be parsed, when it could not. @@ -287,6 +288,7 @@ private sealed record Reading( PdfByteRange Range, bool HasRange, bool Usable, + bool GapHoldsValue, byte[]? Contents, CmsSignature? Cms, Exception? Failure, @@ -314,6 +316,7 @@ private static Reading Read(PdfFileReader reader, PdfDictionary dictionary) && range.IsWellFormed && range.Length1 > 0 && range.Offset2 + range.Length2 <= reader.Data.LongLength; + var gapHoldsValue = usable && GapHoldsContents(reader, dictionary, range); var contents = ReadContents(reader, dictionary); CmsSignature? cms = null; Exception? failure = null; @@ -334,13 +337,72 @@ private static Reading Read(PdfFileReader reader, PdfDictionary dictionary) range, hasRange, usable, + gapHoldsValue, contents, cms, failure, - usable && range.Covers(reader.Data.LongLength), + usable && gapHoldsValue && range.Covers(reader.Data.LongLength), usable ? CountRevisions(reader.Data, range.Offset2 + range.Length2) : 0); } + /// + /// Determines whether the bytes the byte range leaves uncovered are exactly the signature value: one + /// hexadecimal string, delimiters included, and nothing else. + /// + /// The opened document. + /// The signature dictionary. + /// The byte range, already known to be ordered and inside the document. + /// when the gap holds the value and nothing beside it. + /// + /// The gap is never hashed, so anything in it is unsigned. A gap that closes the value early and carries + /// entries or objects of its own still leaves every signed byte intact and the digest matching — the + /// signature-wrapping attack — so the digest cannot be what catches it. A gap that is a single hex string + /// can only carry the value itself, and changing that breaks the digest. The value is also compared with + /// the /Contents the dictionary resolves to, so a dictionary that points its value somewhere else + /// cannot leave an unrelated string in the gap; a document whose strings were decrypted on the way in is + /// compared by shape only, because the parser hands over the decrypted form. + /// + private static bool GapHoldsContents(PdfFileReader reader, PdfDictionary dictionary, PdfByteRange range) + { + var data = reader.Data; + var start = checked((int)(range.Offset1 + range.Length1)); + var end = checked((int)range.Offset2); + + if (end - start < 2 || data[start] != (byte)'<' || data[end - 1] != (byte)'>') + { + return false; + } + + var inner = data.AsSpan(start + 1, end - start - 2); + var digits = new List(inner.Length); + + foreach (var value in inner) + { + if (Uri.IsHexDigit((char)value)) + { + digits.Add(value); + } + else if (!PdfLexer.IsWhitespace(value)) + { + return false; + } + } + + if (reader.Security is not null) + { + return true; + } + + // An odd number of digits is completed with a zero, as the specification reads a hex string. + if (digits.Count % 2 == 1) + { + digits.Add((byte)'0'); + } + + return reader.Resolve(dictionary.Get("Contents")) is PdfString contents + && Convert.FromHexString(System.Text.Encoding.ASCII.GetString(digits.ToArray())).AsSpan().SequenceEqual(contents.Bytes); + } + /// Reads the four numbers of the byte range. /// The opened document. /// The signature dictionary. @@ -469,6 +531,15 @@ private static PdfSignatureIntegrity Check( return PdfSignatureIntegrity.Unknown; } + if (!reading.GapHoldsValue) + { + problems.Add(new PdfSignatureProblem( + PdfSignatureProblemKind.MalformedByteRange, + $"The signature '{fieldName}' leaves bytes {reading.Range.Offset1 + reading.Range.Length1} to {reading.Range.Offset2} uncovered, and they are not exactly its /Contents value: bytes no signature covers would be read as part of the document.")); + + return PdfSignatureIntegrity.Unknown; + } + if (reading.Contents is not { Length: > 0 }) { problems.Add(new PdfSignatureProblem( diff --git a/tests/SharpCorePDF.Tests/PdfEngineOptionsTests.cs b/tests/SharpCorePDF.Tests/PdfEngineOptionsTests.cs index 9252147..5d58175 100644 --- a/tests/SharpCorePDF.Tests/PdfEngineOptionsTests.cs +++ b/tests/SharpCorePDF.Tests/PdfEngineOptionsTests.cs @@ -19,6 +19,7 @@ public void Has_the_documented_defaults() var options = PdfEngineOptions.Default; Assert.Equal(512L * 1024 * 1024, options.MaxSourceBytes); + Assert.Equal(128L * 1024 * 1024, options.MaxDecodedStreamBytes); Assert.Equal(64, options.MaxNestingDepth); Assert.Equal(100_000, options.MaxPagesPerSource); Assert.True(options.ReconstructCrossReferenceOnFailure); @@ -44,7 +45,16 @@ public void Reports_a_limit_that_cannot_be_honoured() (PdfEngineOptions.Default with { MaxPagesPerSource = 0 }).Validate(), error => error.Contains("MaxPagesPerSource", StringComparison.Ordinal)); + Assert.Contains( + (PdfEngineOptions.Default with { MaxDecodedStreamBytes = 1023 }).Validate(), + error => error.Contains("MaxDecodedStreamBytes", StringComparison.Ordinal)); + Assert.Contains( + (PdfEngineOptions.Default with { MaxDecodedStreamBytes = Array.MaxLength + 1L }).Validate(), + error => error.Contains("MaxDecodedStreamBytes", StringComparison.Ordinal)); + Assert.Empty((PdfEngineOptions.Default with { MaxSourceBytes = 1024 }).Validate()); + Assert.Empty((PdfEngineOptions.Default with { MaxDecodedStreamBytes = 1024 }).Validate()); + Assert.Empty((PdfEngineOptions.Default with { MaxDecodedStreamBytes = Array.MaxLength }).Validate()); Assert.Empty((PdfEngineOptions.Default with { MaxNestingDepth = 1024 }).Validate()); Assert.Empty((PdfEngineOptions.Default with { MaxPagesPerSource = 1 }).Validate()); } diff --git a/tests/SharpCorePDF.Tests/PdfFilterTests.cs b/tests/SharpCorePDF.Tests/PdfFilterTests.cs index 65cf9f2..99e6525 100644 --- a/tests/SharpCorePDF.Tests/PdfFilterTests.cs +++ b/tests/SharpCorePDF.Tests/PdfFilterTests.cs @@ -1,7 +1,11 @@ using System.IO.Compression; using System.Text; using SharpCorePDF.Domain.Exceptions; +using SharpCorePDF.Application.Services; +using SharpCorePDF.Domain.ValueObjects; +using SharpCorePDF.Infrastructure; using SharpCorePDF.Infrastructure.Pdf; +using SharpCorePDF.Infrastructure.Sources; namespace SharpCorePDF.Tests; @@ -144,6 +148,98 @@ public void Inflates_flate_data_however_it_was_written() Assert.Equal(text, raw); } + /// + /// A Flate stream that decodes past the limit is refused rather than inflated: zeros deflate about a + /// thousand to one, so the input limit alone lets a small document ask for a gigabyte. + /// + [Fact] + public void Refuses_a_flate_stream_that_decodes_past_the_limit() + { + var bomb = Deflate(new byte[2 * 1024 * 1024]); + + Assert.True(bomb.Length < 16 * 1024, "the fixture has to be small for the test to say anything"); + + var error = Assert.Throws(() => PdfFilters.Inflate(bomb, "bomb.pdf", 1024 * 1024)); + + Assert.Contains("MaxDecodedStreamBytes", error.Message, StringComparison.Ordinal); + Assert.Equal(2 * 1024 * 1024, PdfFilters.Inflate(bomb, "bomb.pdf", 2 * 1024 * 1024).Length); + Assert.True(PdfFilters.TryInflate(Deflate(new byte[1024]), out var small)); + Assert.Equal(1024, small.Length); + } + + /// An LZW stream is held to the same limit: one 12-bit code can stand for 4,096 bytes. + [Fact] + public void Refuses_an_lzw_stream_that_decodes_past_the_limit() + { + var encoded = TestLzw.Encode(new byte[100_000]); + + var error = Assert.Throws(() => PdfFilters.ApplyLzw(encoded, null, "lzw.pdf", 99_999)); + + Assert.Contains("MaxDecodedStreamBytes", error.Message, StringComparison.Ordinal); + Assert.Contains("LZW", error.Message, StringComparison.Ordinal); + Assert.Equal(100_000, PdfFilters.ApplyLzw(encoded, null, "lzw.pdf", 100_000).Length); + } + + /// A RunLength stream is held to the limit too: two bytes stand for 128. + [Fact] + public void Refuses_a_run_length_stream_that_decodes_past_the_limit() + { + var runs = Enumerable.Repeat(new byte[] { 0x81, 0x00 }, 1000).SelectMany(run => run).ToArray(); + + var error = Assert.Throws(() => PdfFilters.DecodeRunLength(runs, "runs.pdf", 127_999)); + + Assert.Contains("MaxDecodedStreamBytes", error.Message, StringComparison.Ordinal); + Assert.Contains("RunLength", error.Message, StringComparison.Ordinal); + Assert.Equal(128_000, PdfFilters.DecodeRunLength(runs, "runs.pdf", 128_000).Length); + } + + /// Literal runs are held to the limit on the same exact boundary as repeated ones. + [Fact] + public void Refuses_literal_runs_that_decode_past_the_limit() + { + byte[] run = [0x7F, .. Enumerable.Range(0, 128).Select(value => (byte)value)]; + var runs = Enumerable.Repeat(run, 100).SelectMany(bytes => bytes).ToArray(); + + var error = Assert.Throws(() => PdfFilters.DecodeRunLength(runs, "runs.pdf", 12_799)); + + Assert.Contains("RunLength", error.Message, StringComparison.Ordinal); + Assert.Equal(12_800, PdfFilters.DecodeRunLength(runs, "runs.pdf", 12_800).Length); + } + + /// + /// A page whose content stream decodes past the limit is reported as unreadable, the way any other + /// undecodable content is, and the read finishes: the limit turns a bomb into a warning. + /// + [Fact] + public async Task Reports_a_content_stream_past_the_limit_as_unreadable_instead_of_inflating_it() + { + var builder = new PdfBuilder(); + var catalog = builder.Reserve(); + var pages = builder.Reserve(); + + builder.Add("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Contents 4 0 R >>"); + builder.AddStream("/Filter /FlateDecode", PdfBuilder.Compress(new byte[1024 * 1024])); + builder.Set(catalog, "<< /Type /Catalog /Pages 2 0 R >>"); + builder.Set(pages, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + var document = builder.Build(catalog); + + var limited = new PdfContentReaderService( + new PdfPageContentReader(PdfEngineOptions.Default with { MaxDecodedStreamBytes = 64 * 1024 })); + var text = await limited.ReadTextAsync( + new PdfByteArraySource(document, "bomb.pdf"), + PageRange.All, + cancellationToken: TestContext.Current.CancellationToken); + + Assert.Contains(text.Warnings, warning => warning.Kind == PdfWarningKind.ContentStreamUnreadable); + + var unlimited = await new PdfContentReaderService(new PdfPageContentReader()).ReadTextAsync( + new PdfByteArraySource(document, "bomb.pdf"), + PageRange.All, + cancellationToken: TestContext.Current.CancellationToken); + + Assert.DoesNotContain(unlimited.Warnings, warning => warning.Kind == PdfWarningKind.ContentStreamUnreadable); + } + /// Flate data that cannot be decompressed is refused and reported as a damaged source. [Fact] public void Reports_data_that_cannot_be_decompressed() diff --git a/tests/SharpCorePDF.Tests/PdfJpegFilterTests.cs b/tests/SharpCorePDF.Tests/PdfJpegFilterTests.cs index 10c77a8..6db9f08 100644 --- a/tests/SharpCorePDF.Tests/PdfJpegFilterTests.cs +++ b/tests/SharpCorePDF.Tests/PdfJpegFilterTests.cs @@ -35,6 +35,96 @@ public sealed class PdfJpegFilterTests private static string Fixture(string name) => Path.Combine(TestAssets.Folder, "jpeg", name); + /// A budget of exactly the output's size keeps the pixels; one byte less measures them. + [Fact] + public void Keeps_pixels_that_fit_the_budget_exactly() + { + Assert.True(JpegDecoder.TryDecode(Rgb, 1 << 20, out var all, out _, out var width, out var height)); + + var exact = width * height * 3; + + Assert.Equal(exact, all!.Length); + Assert.True(JpegDecoder.TryDecode(Rgb, exact, out var kept, out _, out _, out _)); + Assert.Equal(all, kept); + Assert.True(JpegDecoder.TryDecode(Rgb, exact - 1, out var measured, out _, out _, out _)); + Assert.Null(measured); + } + + /// An image above the budget is measured with the format its components state. + /// The fixture. + /// The format it decodes to. + [Theory] + [InlineData("jpeg-progressive-gray.jpg", "Gray8")] + [InlineData("jpeg-cmyk.jpg", "Cmyk32")] + [InlineData("jpeg-444.jpg", "Rgb24")] + public void Measures_the_format_of_an_image_above_the_budget(string fixture, string expected) + { + Assert.True(JpegDecoder.TryDecode(File.ReadAllBytes(Fixture(fixture)), 16, out var pixels, out var format, out _, out _)); + Assert.Null(pixels); + Assert.Equal(expected, format); + } + + /// + /// A frame whose output fits a budget of while its coefficient grid does not fit + /// an array — 46,340 squared samples is just under the budget, at four bytes a coefficient it is eight + /// gigabytes — is refused before anything is allocated, instead of overflowing the size in 32 bits. + /// + [Fact] + public void Refuses_a_frame_whose_working_buffers_cannot_fit_an_array() + { + var jpeg = (byte[])ProgressiveGray.Clone(); + var frame = jpeg.AsSpan().IndexOf(new byte[] { 0xFF, 0xC2 }); + + Assert.True(frame > 0, "the fixture has to carry the frame header the test rewrites"); + + jpeg[frame + 5] = 46_340 >> 8; + jpeg[frame + 6] = 46_340 & 0xFF; + jpeg[frame + 7] = 46_340 >> 8; + jpeg[frame + 8] = 46_340 & 0xFF; + + var before = GC.GetAllocatedBytesForCurrentThread(); + var decoded = JpegDecoder.TryDecode(jpeg, int.MaxValue, out var pixels, out _, out _, out _); + var allocated = GC.GetAllocatedBytesForCurrentThread() - before; + + Assert.False(decoded); + Assert.Null(pixels); + Assert.True(allocated < 1 << 20, $"refusing the frame allocated {allocated} bytes"); + } + + /// + /// A frame header that claims 65,535 x 65,535 samples is measured from the header alone, with no pixels, + /// as any image above the budget is. The width and the height are two numbers in the file, and the + /// sample planes — four bytes a coefficient for a progressive image — were otherwise allocated before + /// the output was found too large to keep. + /// + /// Whether the progressive fixture is used rather than the baseline one. + [Theory] + [InlineData(false)] + [InlineData(true)] + public void Measures_a_frame_too_large_for_the_budget_without_allocating_it(bool progressive) + { + var jpeg = (byte[])(progressive ? Progressive : Rgb).Clone(); + var frame = jpeg.AsSpan().IndexOf(progressive ? [0xFF, 0xC2] : [0xFF, 0xC0]); + + Assert.True(frame > 0, "the fixture has to carry the frame header the test rewrites"); + + // Marker, length (two bytes), precision, then height and width, each a 16-bit big-endian number. + jpeg[frame + 5] = 0xFF; + jpeg[frame + 6] = 0xFF; + jpeg[frame + 7] = 0xFF; + jpeg[frame + 8] = 0xFF; + + var before = GC.GetAllocatedBytesForCurrentThread(); + var measured = JpegDecoder.TryDecode(jpeg, 1 << 20, out var pixels, out var format, out var width, out var height); + var allocated = GC.GetAllocatedBytesForCurrentThread() - before; + + Assert.True(measured); + Assert.Null(pixels); + Assert.Equal("Rgb24", format); + Assert.Equal((65_535, 65_535), (width, height)); + Assert.True(allocated < 1 << 20, $"refusing the frame allocated {allocated} bytes"); + } + /// A 4:4:4 RGB JPEG decodes to the bytes Pillow decodes from the same file. [Fact] public void Decodes_a_baseline_rgb_jpeg_within_two_levels() diff --git a/tests/SharpCorePDF.Tests/PdfReaderRecoveryTests.cs b/tests/SharpCorePDF.Tests/PdfReaderRecoveryTests.cs index fba32c8..da00411 100644 --- a/tests/SharpCorePDF.Tests/PdfReaderRecoveryTests.cs +++ b/tests/SharpCorePDF.Tests/PdfReaderRecoveryTests.cs @@ -18,6 +18,139 @@ namespace SharpCorePDF.Tests; /// public sealed class PdfReaderRecoveryTests { + /// + /// The bound on a chain of lengths leaves an ordinary indirect length alone, for every stream of a + /// document and not only the first ones: each stream's data holds an endstream line of its own, + /// so a length that was not resolved would cut the stream there instead of at its real end. + /// + [Fact] + public void Resolves_the_indirect_length_of_every_stream_of_a_document() + { + const int Streams = 100; + var data = "q\nendstream\nQ"u8.ToArray(); + var builder = new PdfBuilder(); + var catalog = builder.Reserve(); + var pages = builder.Reserve(); + + builder.Add("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Contents 4 0 R >>"); + + // Objects 4, 6, 8 ... are the streams; 5, 7, 9 ... are their lengths. + for (var stream = 0; stream < Streams; stream++) + { + var body = Encoding.Latin1.GetBytes($"<< /Length {5 + (2 * stream)} 0 R >>\nstream\n") + .Concat(data) + .Concat("\nendstream"u8.ToArray()) + .ToArray(); + + builder.Add(body); + builder.Add(data.Length.ToString(System.Globalization.CultureInfo.InvariantCulture)); + } + + builder.Set(catalog, "<< /Type /Catalog /Pages 2 0 R >>"); + builder.Set(pages, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + + var reader = new PdfFileReader(builder.Build(catalog), "lengths.pdf", PdfEngineOptions.Default); + + for (var stream = 0; stream < Streams; stream++) + { + var resolved = Assert.IsType(reader.Resolve(4 + (2 * stream))); + + Assert.Equal(data, resolved.Raw.ToArray()); + } + } + + /// + /// An object stream that declares 2^31 - 1 objects in a header of one pair is read for the one pair it + /// holds. /N sized two arrays before anything compared it with the stream, so a few hundred bytes asked + /// for gigabytes; the stream now bounds the count, and the page inside it is found. + /// + [Fact] + public void Reads_an_object_stream_whose_count_its_header_cannot_hold() + { + var page = "3 0 << /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Contents 4 0 R >>"u8.ToArray(); + var packed = PdfBuilder.Compress(page); + var output = new MemoryStream(); + var offsets = new Dictionary(); + + void Write(string text) => output.Write(Encoding.Latin1.GetBytes(text)); + + void Object(int number, string dictionary, byte[]? stream = null) + { + offsets[number] = output.Position; + Write($"{number} 0 obj\n{dictionary}\n"); + + if (stream is not null) + { + Write("stream\n"); + output.Write(stream); + Write("\nendstream\n"); + } + + Write("endobj\n"); + } + + Write("%PDF-1.7\n"); + Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); + Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + Object(4, "<< /Length 3 >>", "q Q"u8.ToArray()); + Object(5, $"<< /Type /ObjStm /N 2147483647 /First 4 /Filter /FlateDecode /Length {packed.Length} >>", packed); + + offsets[6] = output.Position; + (int Kind, long Field, int Index)[] entries = + [ + (0, 0, 65535), (1, offsets[1], 0), (1, offsets[2], 0), (2, 5, 0), + (1, offsets[4], 0), (1, offsets[5], 0), (1, offsets[6], 0), + ]; + var table = entries + .SelectMany(entry => new[] + { + (byte)entry.Kind, + (byte)(entry.Field >> 24), (byte)(entry.Field >> 16), (byte)(entry.Field >> 8), (byte)entry.Field, + (byte)(entry.Index >> 8), (byte)entry.Index, + }) + .ToArray(); + + Write($"6 0 obj\n<< /Type /XRef /Size 7 /W [1 4 2] /Root 1 0 R /Length {table.Length} >>\nstream\n"); + output.Write(table); + Write($"\nendstream\nendobj\nstartxref\n{offsets[6]}\n%%EOF\n"); + + var reader = new PdfFileReader(output.ToArray(), "objstm.pdf", PdfEngineOptions.Default); + + Assert.Single(reader.Pages); + Assert.Equal("Page", Assert.IsType(reader.Resolve(3)).GetName("Type")); + } + + /// + /// A stream whose /Length is another stream, whose length is another, sixty thousand deep, is + /// read by finding its endstream. Resolved link by link, the chain would overflow the stack, and a + /// stack overflow ends the process rather than throwing anything a caller could catch. + /// + [Fact] + public void Reads_a_stream_whose_length_is_a_chain_of_sixty_thousand_streams() + { + const int Chain = 60_000; + var builder = new PdfBuilder(); + var catalog = builder.Reserve(); + var pages = builder.Reserve(); + + builder.Add("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Contents 4 0 R >>"); + + for (var link = 0; link < Chain; link++) + { + builder.Add($"<< /Length {5 + link} 0 R >>\nstream\nq Q\nendstream"); + } + + builder.Add("3"); + builder.Set(catalog, "<< /Type /Catalog /Pages 2 0 R >>"); + builder.Set(pages, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + + var reader = new PdfFileReader(builder.Build(catalog), "chain.pdf", PdfEngineOptions.Default); + var content = Assert.IsType(reader.Resolve(4)); + + Assert.Equal("q Q", Encoding.Latin1.GetString(content.Raw.Span)); + Assert.Single(reader.Pages); + } + /// A table that no longer parses is rebuilt, and the document is read completely. [Fact] public async Task Rebuilds_a_cross_reference_table_that_does_not_parse() diff --git a/tests/SharpCorePDF.Tests/PdfSignatureAttackTests.cs b/tests/SharpCorePDF.Tests/PdfSignatureAttackTests.cs index 52957e9..bbedc1f 100644 --- a/tests/SharpCorePDF.Tests/PdfSignatureAttackTests.cs +++ b/tests/SharpCorePDF.Tests/PdfSignatureAttackTests.cs @@ -104,8 +104,7 @@ public async Task Never_reports_a_signature_whose_value_was_emptied_as_valid() /// what must fail is the claim that the signature covers the whole document, because part of the /// document is now something other than the signature value. /// - [Fact(Skip = - "Known defect: SignatureValidator does not check that the gap between the two ranges is exactly the /Contents string, so this wrapped document is reported as covering the whole file. Remove the skip with the fix.")] + [Fact] public async Task Does_not_report_a_gap_that_holds_more_than_the_signature_value_as_covering_the_document() { var signed = await SignAsync(); @@ -114,14 +113,76 @@ public async Task Does_not_report_a_gap_that_holds_more_than_the_signature_value // The attack only works while the signed bytes are unchanged; this is what makes it an attack. var validation = Assert.Single(await SigningFixtures.ValidateAsync(wrapped)); - Assert.False( - validation.Signer.CoversWholeDocument && validation.IsValid, - "a document whose uncovered gap carries more than the signature value must not be reported as covered"); + Assert.False(validation.Signer.CoversWholeDocument); + Assert.False(validation.IsValid); + Assert.NotEqual(PdfSignatureIntegrity.Valid, validation.Integrity); + + var (gapStart, _) = Gap(signed); + var problem = Assert.Single(validation.Problems, problem => problem.Kind == PdfSignatureProblemKind.MalformedByteRange); + + Assert.Contains(gapStart.ToString(CultureInfo.InvariantCulture), problem.Message, StringComparison.Ordinal); + + // The wrapped gap is the finding; the revision count would be a second, misleading one. + Assert.DoesNotContain(validation.Problems, problem => problem.Kind == PdfSignatureProblemKind.RevisionNotCovered); + } + + /// + /// A gap that is not exactly one hexadecimal string is refused whichever way it fails to be one: no + /// opening delimiter, no closing delimiter, or a byte inside it that is neither a digit nor whitespace. + /// + /// Which byte of the gap is changed. + [Theory] + [InlineData("opening-delimiter")] + [InlineData("closing-delimiter")] + [InlineData("not-a-hex-digit")] + public async Task Refuses_a_gap_that_is_not_one_hexadecimal_string(string variant) + { + var signed = await SignAsync(); + var (gapStart, gapEnd) = Gap(signed); + var changed = (byte[])signed.Clone(); + var (index, value) = variant switch + { + "opening-delimiter" => (gapStart, (byte)'('), + "closing-delimiter" => (gapEnd - 1, (byte)')'), + "not-a-hex-digit" => (gapEnd - 2, (byte)'g'), + _ => throw new ArgumentOutOfRangeException(nameof(variant), variant, "no such variant"), + }; + + changed[index] = value; + + var validation = Assert.Single(await SigningFixtures.ValidateAsync(changed)); + + Assert.False(validation.IsValid); + Assert.False(validation.Signer.CoversWholeDocument); + Assert.Contains(validation.Problems, problem => problem.Kind == PdfSignatureProblemKind.MalformedByteRange); + } + + /// + /// Whitespace inside the value is part of PDF's hexadecimal string syntax, and an odd number of digits is + /// completed with a zero; a gap written that way is still exactly the signature value and stays valid. + /// + [Fact] + public async Task Accepts_whitespace_and_an_odd_digit_count_inside_the_value() + { + var signed = await SignAsync(); + var (_, gapEnd) = Gap(signed); + var spaced = (byte[])signed.Clone(); + + // The last digit of the padding becomes a space: the value now has an odd number of digits, which a + // reader completes with the zero that was there. + Assert.Equal((byte)'0', spaced[gapEnd - 2]); + spaced[gapEnd - 2] = (byte)' '; + + var validation = Assert.Single(await SigningFixtures.ValidateAsync(spaced)); + + Assert.True(validation.IsValid); + Assert.True(validation.Signer.CoversWholeDocument); + Assert.Equal(PdfSignatureIntegrity.Valid, validation.Integrity); } /// - /// The precondition the skipped wrapping test depends on, asserted on its own so it keeps running: the - /// injection leaves every covered byte as it was, which is why only a gap check can catch it. + /// The precondition the wrapping test depends on: the injection leaves every covered byte as it was, which + /// is why only a gap check can catch it. /// [Fact] public async Task Leaves_every_covered_byte_unchanged_when_the_gap_is_wrapped() From 9e2ab83a9616915a9746852b7f1ec3aa7a3b6871 Mon Sep 17 00:00:00 2001 From: Andrew Tierney Date: Mon, 16 Nov 2026 12:26:01 +1100 Subject: [PATCH 04/15] Read the documents the JPEG, empty-name, glyph-box and date faults refused Four defects turned documents into exceptions that were not PdfExceptions, so a caller that handles the contract's refusal still saw the read fail: - A JPEG scan selected Huffman tables with 4-bit selectors and a progressive scan named its spectral selection, and both were used as indexes unchecked (IndexOutOfRangeException). ParseScan now refuses a selector above 3, and a progressive scan unless Ss <= Se <= 63 and Ah, Al <= 13. - The empty name, a solidus on its own, is a name PDF allows, and four documents of the pdf.js test corpus use it as a key; PdfDictionary.Set refused it, so they did not open at all. - A glyph drawn upside down (a negative font size) gave PdfGlyph.BoundingBox a bottom above its top, and the rectangle refused it, failing text extraction for the page. - PdfDate.TryParse threw for a time-zone offset beyond fourteen hours, so one malformed /CreationDate failed the inspection of the whole document; such an offset is now read as no offset. PdfContainerTests.Refuses_an_entry_without_a_key asserted the second defect and is rewritten: a null key is still refused, the empty name is kept. Tests that fail without the fixes: PdfJpegFilterTests (table selectors, progressive scan bounds), PdfObjectModelTests (empty-name key, in a dictionary and in a document), PdfTextScannerTests (upside-down glyph), PdfSerializationTests (offsets past fourteen hours). The pdf.js corpus and the hostile documents now read with no fault, crash or timeout, and their --known flags are removed. --- .github/workflows/robustness.yml | 5 -- .github/workflows/validation.yml | 7 +- CHANGELOG.md | 14 +++- docs/validation.md | 11 +-- .../ReadModels/PdfGlyph.cs | 8 +- .../Pdf/JpegDecoder.cs | 13 ++- .../Pdf/PdfDate.cs | 8 +- .../Pdf/PdfDictionary.cs | 4 +- tests/SharpCorePDF.Tests/PdfContainerTests.cs | 15 +++- .../SharpCorePDF.Tests/PdfJpegFilterTests.cs | 80 +++++++++++++++++++ .../SharpCorePDF.Tests/PdfObjectModelTests.cs | 40 ++++++++++ .../PdfSerializationTests.cs | 22 +++++ .../SharpCorePDF.Tests/PdfTextScannerTests.cs | 25 ++++++ 13 files changed, 224 insertions(+), 28 deletions(-) diff --git a/.github/workflows/robustness.yml b/.github/workflows/robustness.yml index 93de31b..50854f4 100644 --- a/.github/workflows/robustness.yml +++ b/.github/workflows/robustness.yml @@ -162,16 +162,11 @@ jobs: - name: Build the probe run: dotnet build ./tools/SharpCorePDF.Validation -c Release - # The six known files fault on three recorded defects (docs/validation.md): an empty name as a - # dictionary key (issue3188, issue5896, pdfjs_wikipedia, poppler-742-0-fuzzed), a glyph whose box is - # upside down (issue14415) and a date whose offset is beyond 14 hours (TAMReview). - name: Read every document run: | python3 docs/tools/corpus-robustness.py \ --probe ./tools/SharpCorePDF.Validation/bin/Release/net11.0/SharpCorePDF.Validation.dll \ --timeout 60 --heap-limit-mb 1024 --report artifacts/corpora.json \ - --known issue3188.pdf --known issue5896.pdf --known pdfjs_wikipedia.pdf \ - --known poppler-742-0-fuzzed.pdf --known issue14415.pdf --known TAMReview.pdf \ "$RUNNER_TEMP/verapdf-corpus" "$RUNNER_TEMP/pdfjs/test/pdfs" - name: Upload the report diff --git a/.github/workflows/validation.yml b/.github/workflows/validation.yml index de42cbe..a632011 100644 --- a/.github/workflows/validation.yml +++ b/.github/workflows/validation.yml @@ -146,9 +146,9 @@ jobs: dotnet "$tool" replay "$target" "$seeds/$target" done - # Each of the four documents reaches a failure class the review named. flate-bomb, length-chain and - # objstm-count are refused or read since the decode limit, the length-chain bound and the /N bound; - # jpeg-selector is a known defect until the change that fixes it removes its --known line. + # Each of the four documents reaches a failure class a review named, and each is now read: the decode + # limit, the length-chain bound, the /N bound and the JPEG scan-header checks closed them. A document + # that faults here again is a regression, so none is listed as known. - name: Hostile documents run: | set -euo pipefail @@ -156,7 +156,6 @@ jobs: python docs/tools/corpus-robustness.py \ --probe ./tools/SharpCorePDF.Validation/bin/Release/net11.0/SharpCorePDF.Validation.dll \ --timeout 120 --report artifacts/hostile.json \ - --known jpeg-selector.pdf \ "$RUNNER_TEMP/hostile" - name: Upload the documents and the reports diff --git a/CHANGELOG.md b/CHANGELOG.md index abe4bd9..01e5662 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,6 +55,16 @@ against an empty feed. is measured from its header without being decoded — the same answer as before, without the allocation — and the working sizes are computed in 64 bits. +- **A JPEG scan header is checked before it is used.** A Huffman table selector above 3, and a progressive + scan whose spectral selection runs past coefficient 63 or starts after it ends, indexed past the decoder's + tables (`IndexOutOfRangeException`); such a scan is now refused and the image reported undecodable. +- **The empty name is a dictionary key like any other.** `/` on its own is a name PDF allows; four documents + of the pdf.js test corpus use it as a key and did not open at all (`ArgumentException`). +- **A glyph drawn upside down has a box.** A negative font size made `PdfGlyph.BoundingBox` throw, so text + extraction faulted on the page. +- **`PdfDate.TryParse` answers instead of throwing** for a time-zone offset beyond fourteen hours, which it + now reads as no offset; one malformed `/CreationDate` no longer fails the inspection of the document. + ### Known defects, found by the new checks and not yet fixed Each is reproduced, pinned by a skipped test or a strict `--known-defect` flag that fails the run once the @@ -63,10 +73,6 @@ defect is fixed, and listed in [`docs/validation.md`](docs/validation.md#known-d - An incremental update rewrites both elements of `/ID`, and appends a renumbered copy of the whole document rather than the objects it changed — which also rewrites the signature dictionary after signing. - A merge copies a page that states no `/MediaBox` without one. -- Hostile input: a JPEG scan's Huffman table selector is not range-checked. -- Real-world input (the pdf.js test corpus): an empty name as a dictionary key stops a document opening, a - glyph with an upside-down box faults text extraction, and `PdfDate.TryParse` throws on an out-of-range - time-zone offset instead of answering false. - **Run-length images are decoded.** `RunLengthDecode` joins Flate, LZW and the two ASCII filters: a bilevel or scan-style image that declares it now yields its samples — and `Gray8`, `Rgb24` or `Cmyk32` diff --git a/docs/validation.md b/docs/validation.md index 6ce4585..1d79368 100644 --- a/docs/validation.md +++ b/docs/validation.md @@ -86,10 +86,6 @@ fix. This is the list on 2026-11-16, every entry reproduced, not inferred: | An incremental update writes a new identifier into both elements of `/ID`; ISO 32000-1 §14.4 keeps the first | #10 | `PdfIncrementalInvariantTests.Keeps_the_first_identifier_element_across_an_incremental_update` (skipped); `validate-incremental.py` `id-first` | | `UseIncrementalUpdate` appends a renumbered copy of the whole document rather than the objects it changed: every byte of the earlier revision is kept, but the file grows by its own size per edit, and the signature dictionary is written again after signing, so pyHanko places a signature followed by an edit as `CONTIGUOUS_BLOCK_FROM_START` with modification level `OTHER` instead of `ENTIRE_REVISION` | #10, #3 | `validate-incremental.py` `appends-only`; `validate-signatures.py` on `signed-then-edited.pdf` | | A merge copies a page that states no `/MediaBox` (and inherits none) without one, so the result breaks the rule that every page has a media box; qpdf repairs it by assuming Letter | #2 | `validate-pdf.py` on `inspected.pdf` and `with-warnings.pdf` (both merge a page of `NamedDests.pdf`) | -| The JPEG decoder indexes its four Huffman tables with a scan's 4-bit selector unchecked | #12, #1 | `jpeg-selector.pdf`: `IndexOutOfRangeException` from `JpegDecoder.DecodeBlock` | -| A dictionary key that is the empty name (`/` alone, which PDF syntax allows) makes `PdfDictionary.Set` throw `ArgumentException` while parsing, so the document does not open at all | #12 | pdf.js corpus: `issue3188.pdf`, `issue5896.pdf`, `pdfjs_wikipedia.pdf`, `poppler-742-0-fuzzed.pdf` | -| A glyph whose box comes out upside down (a negative font size or a flipped font matrix) makes `PdfGlyph.BoundingBox` construct a `PdfRectangle` that throws, so text extraction faults | #12 | pdf.js corpus: `issue14415.pdf` | -| `PdfDate.TryParse` throws `ArgumentOutOfRangeException` for a time-zone offset beyond 14 hours instead of answering false, so one malformed `/CreationDate` faults the whole inspection | #12 | pdf.js corpus: `TAMReview.pdf` | Fixed since the list was first drawn up, each with the test that fails without its fix: @@ -99,6 +95,10 @@ Fixed since the list was first drawn up, each with the test that fails without i | No limit on the decoded size of a Flate, LZW or RunLength stream | `PdfEngineOptions.MaxDecodedStreamBytes` (128 MiB) | `PdfFilterTests.Refuses_a_flate_stream_that_decodes_past_the_limit` and its LZW, RunLength and content-stream companions | | A chain of indirect `/Length` references overflowed the stack | the chain is followed no deeper than `MaxNestingDepth`, then the stream is found by its `endstream` | `PdfReaderRecoveryTests.Reads_a_stream_whose_length_is_a_chain_of_sixty_thousand_streams` | | An object stream's `/N`, and a JPEG frame's width and height, sized allocations unchecked | `/N` is bounded by what the stream can hold; a JPEG above the caller's budget is measured from its header, and working sizes are computed in 64 bits | `PdfReaderRecoveryTests.Reads_an_object_stream_whose_count_its_header_cannot_hold`, `PdfJpegFilterTests.Measures_a_frame_too_large_for_the_budget_without_allocating_it` | +| The JPEG decoder used a scan's 4-bit Huffman table selectors, and a progressive scan's spectral end, as indexes unchecked | `ParseScan` refuses a selector above 3, and a progressive scan whose selection is not `Ss ≤ Se ≤ 63` or whose approximation is above 13 | `PdfJpegFilterTests.Refuses_a_scan_that_selects_a_table_beyond_the_four_there_are`, `Refuses_a_progressive_scan_outside_the_block` | +| The empty name as a dictionary key (`/` alone, which PDF allows) stopped the document opening | `PdfDictionary.Set` accepts it; the entry is kept and written back as `/` | `PdfObjectModelTests.Keeps_an_entry_whose_key_is_the_empty_name`, `Opens_a_document_that_uses_the_empty_name_as_a_key` | +| A glyph drawn upside down (negative font size) faulted text extraction | `PdfGlyph.BoundingBox` takes the box between descender and ascender whichever way round | `PdfTextScannerTests.Gives_a_glyph_drawn_upside_down_a_box` | +| `PdfDate.TryParse` threw for a time-zone offset beyond 14 hours | an offset no time zone has is read as no offset | `PdfSerializationTests.Reads_an_offset_no_time_zone_has_as_no_offset` | ## Corpus readings, 2026-11-16 @@ -108,7 +108,8 @@ Fixed since the list was first drawn up, each with the test that fails without i | pdf.js `test/pdfs` at `v6.4.299` | 987 | 977 | 4 | 6 | 0 | 0 | | synthesized hostile documents, before the fixes | 4 | 0 | 0 | 3 | 1 | 0 | -The six pdf.js faults are the three defects above. A sample of the pre-flight measurement (the first 60 +The six pdf.js faults were the empty-name, upside-down glyph and date-offset defects in the fixed table; +after those fixes all six are read, as is `jpeg-selector.pdf`. A sample of the pre-flight measurement (the first 60 files of `PDF_A-2b` and of `PDF_A-4`) wrote 108 results, refused 9 non-conforming inputs and refused 3 that already conformed; whether the 108 conform is veraPDF's to say, which `robustness.yml` asks. diff --git a/src/SharpCorePDF.Domain/ReadModels/PdfGlyph.cs b/src/SharpCorePDF.Domain/ReadModels/PdfGlyph.cs index eb0ea41..f3106d9 100644 --- a/src/SharpCorePDF.Domain/ReadModels/PdfGlyph.cs +++ b/src/SharpCorePDF.Domain/ReadModels/PdfGlyph.cs @@ -70,13 +70,15 @@ public sealed record PdfGlyph /// /// Gets the area the glyph occupies, derived from the origin, the advance and the em box. The /// descender is taken as a fifth of the em and the ascender as four fifths, which is the proportion - /// most Latin fonts share and is stated here rather than read from the font program. + /// most Latin fonts share and is stated here rather than read from the font program. A negative font + /// size — a glyph drawn upside down, which a flipped text matrix or font matrix produces — puts the + /// descender above the ascender, so the box is taken between the two whichever way round they are. /// public PdfRectangle BoundingBox => new( X, - Y - (FontSize * 0.2d), + Math.Min(Y - (FontSize * 0.2d), Y + (FontSize * 0.8d)), X + Math.Max(0d, Width), - Y + (FontSize * 0.8d)); + Math.Max(Y - (FontSize * 0.2d), Y + (FontSize * 0.8d))); /// Gets a value indicating whether the glyph paints no ink, such as a space or a tab. public bool IsWhitespace => string.IsNullOrWhiteSpace(Text); diff --git a/src/SharpCorePDF.Infrastructure/Pdf/JpegDecoder.cs b/src/SharpCorePDF.Infrastructure/Pdf/JpegDecoder.cs index 528cf69..8919615 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/JpegDecoder.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/JpegDecoder.cs @@ -473,6 +473,13 @@ private bool ParseScan(ReadOnlySpan segment) return false; } + // A selector is four bits in the file and the decoder has four tables of each class: a selector + // above 3 would index past them, so it is a header this decoder cannot honour. + if (tables >> 4 > 3 || (tables & 15) > 3) + { + return false; + } + component.DcTable = tables >> 4; component.AcTable = tables & 15; named[index] = component; @@ -486,7 +493,11 @@ private bool ParseScan(ReadOnlySpan segment) scanHigh = segment[tail + 2] >> 4; scanLow = segment[tail + 2] & 15; - return true; + // The spectral selection names coefficients 0 to 63 of a block, start before end, and the successive + // approximation shifts no further than a 16-bit coefficient allows. A progressive scan indexes the + // zig-zag table with them, so a scan outside these bounds is refused rather than read past the + // table; a sequential scan states them too but decodes the whole block whatever they say. + return !progressive || (scanStart <= scanEnd && scanEnd <= 63 && scanHigh <= 13 && scanLow <= 13); } private void ParseAdobe(ReadOnlySpan segment) diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfDate.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfDate.cs index f240e68..80d8ce7 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfDate.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfDate.cs @@ -133,8 +133,12 @@ private static TimeSpan ReadOffset(ReadOnlySpan rest) field++; } - var sign = rest[0] == '-' ? -1 : 1; - return sign * new TimeSpan(Math.Clamp(hours, 0, 23), Math.Clamp(minutes, 0, 59), 0); + // DateTimeOffset accepts offsets up to fourteen hours either way and throws beyond them, so an offset + // no time zone has is read as no offset — UTC, which is what the specification assumes when none is + // given — instead of failing the inspection of the whole document over one metadata string. + var offset = (rest[0] == '-' ? -1 : 1) * new TimeSpan(Math.Clamp(hours, 0, 23), Math.Clamp(minutes, 0, 59), 0); + + return offset.Duration() > TimeSpan.FromHours(14) ? TimeSpan.Zero : offset; } /// Reads the next run of digits of a time zone offset, skipping what is not a digit. diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfDictionary.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfDictionary.cs index b877b2b..72643b5 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfDictionary.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfDictionary.cs @@ -54,7 +54,9 @@ public IEnumerable> Entries /// This dictionary, so calls can be chained. public PdfDictionary Set(string key, PdfObject? value) { - ArgumentException.ThrowIfNullOrEmpty(key); + // The empty name — a solidus on its own — is a name PDF allows (ISO 32000-1 §7.3.5), and documents + // use it as a key; refusing it here refused the whole document while it was being parsed. + ArgumentNullException.ThrowIfNull(key); if (value is null) { diff --git a/tests/SharpCorePDF.Tests/PdfContainerTests.cs b/tests/SharpCorePDF.Tests/PdfContainerTests.cs index b402acd..e562066 100644 --- a/tests/SharpCorePDF.Tests/PdfContainerTests.cs +++ b/tests/SharpCorePDF.Tests/PdfContainerTests.cs @@ -72,10 +72,19 @@ public void Copies_a_set_of_entries() Assert.Same(PdfNull.Instance, dictionary.Get("B")); } - /// An entry without a key is a programmer error. + /// + /// An entry without a key is a programmer error, but the empty name is a key: a solidus on its own is a + /// name PDF allows (ISO 32000-1 §7.3.5), and documents in the wild use it. + /// [Fact] - public void Refuses_an_entry_without_a_key() => - Assert.Throws(() => new PdfDictionary().Set(string.Empty, PdfNull.Instance)); + public void Refuses_an_entry_without_a_key_and_keeps_one_whose_key_is_the_empty_name() + { + Assert.Throws(() => new PdfDictionary().Set(null!, PdfNull.Instance)); + + var dictionary = new PdfDictionary().Set(string.Empty, new PdfInteger(1)); + + Assert.Equal(1L, dictionary.GetInteger(string.Empty)); + } /// The readers of a dictionary answer with the type they were asked for, or with nothing. [Fact] diff --git a/tests/SharpCorePDF.Tests/PdfJpegFilterTests.cs b/tests/SharpCorePDF.Tests/PdfJpegFilterTests.cs index 6db9f08..4a6644c 100644 --- a/tests/SharpCorePDF.Tests/PdfJpegFilterTests.cs +++ b/tests/SharpCorePDF.Tests/PdfJpegFilterTests.cs @@ -35,6 +35,86 @@ public sealed class PdfJpegFilterTests private static string Fixture(string name) => Path.Combine(TestAssets.Folder, "jpeg", name); + /// + /// A scan that selects a Huffman table the decoder does not have — the selector is four bits, the tables + /// number four of each class — is refused, not used as an index past them. + /// + /// The table selector byte of the first scan component: DC in the high nibble, AC in the low. + [Theory] + [InlineData(0x44)] + [InlineData(0x40)] + [InlineData(0x04)] + [InlineData(0xFF)] + public void Refuses_a_scan_that_selects_a_table_beyond_the_four_there_are(byte selectors) + { + var jpeg = (byte[])Rgb.Clone(); + var scan = jpeg.AsSpan().IndexOf(new byte[] { 0xFF, 0xDA }); + + Assert.True(scan > 0, "the fixture has to carry a scan header"); + + // Marker, length (two bytes), component count, then the first component's id and its selectors. + jpeg[scan + 6] = selectors; + + Assert.False(JpegDecoder.TryDecode(jpeg, 1 << 20, out var pixels, out _, out _, out _)); + Assert.Null(pixels); + } + + /// + /// A progressive scan whose spectral selection runs past coefficient 63, starts after it ends, or states a + /// successive approximation no 16-bit coefficient has is refused before it indexes the zig-zag table. + /// + /// Which field of the AC scan header is rewritten. + /// The value it is rewritten to. + [Theory] + [InlineData("end", 0xFF)] + [InlineData("end", 64)] + [InlineData("start-after-end", 0)] + [InlineData("approximation", 0xE0)] + [InlineData("approximation", 0x0E)] + public void Refuses_a_progressive_scan_outside_the_block(string field, int value) + { + var jpeg = (byte[])Progressive.Clone(); + var position = 0; + var tail = -1; + + // The first AC scan: a scan header whose spectral start is above zero. + while (tail < 0) + { + var scan = jpeg.AsSpan(position).IndexOf(new byte[] { 0xFF, 0xDA }); + + Assert.True(scan >= 0, "the fixture has to carry an AC scan"); + scan += position; + + var candidate = scan + 5 + (2 * jpeg[scan + 4]); + + if (jpeg[candidate] > 0) + { + tail = candidate; + } + + position = scan + 2; + } + + switch (field) + { + case "end": + jpeg[tail + 1] = (byte)value; + break; + + case "start-after-end": + jpeg[tail + 1] = (byte)value; + Assert.True(jpeg[tail] > value, "the scan has to start after the end it is given"); + break; + + default: + jpeg[tail + 2] = (byte)value; + break; + } + + Assert.False(JpegDecoder.TryDecode(jpeg, 1 << 20, out var pixels, out _, out _, out _)); + Assert.Null(pixels); + } + /// A budget of exactly the output's size keeps the pixels; one byte less measures them. [Fact] public void Keeps_pixels_that_fit_the_budget_exactly() diff --git a/tests/SharpCorePDF.Tests/PdfObjectModelTests.cs b/tests/SharpCorePDF.Tests/PdfObjectModelTests.cs index a520044..d59b80e 100644 --- a/tests/SharpCorePDF.Tests/PdfObjectModelTests.cs +++ b/tests/SharpCorePDF.Tests/PdfObjectModelTests.cs @@ -13,6 +13,46 @@ namespace SharpCorePDF.Tests; /// public sealed class PdfObjectModelTests { + /// + /// The empty name — a solidus on its own — is a key like any other: the dictionary keeps it and every + /// entry after it, and writing the dictionary back states it as a solidus again. + /// + [Fact] + public void Keeps_an_entry_whose_key_is_the_empty_name() + { + var parser = new PdfObjectParser(Encoding.Latin1.GetBytes("<< /A 1 / 2 /B 3 >>"), "empty-name.pdf", 0, null, 64); + var dictionary = Assert.IsType(parser.ParseObject()); + + Assert.Equal(1L, dictionary.GetInteger("A")); + Assert.Equal(2L, dictionary.GetInteger(string.Empty)); + Assert.Equal(3L, dictionary.GetInteger("B")); + + using var stream = new MemoryStream(); + var output = new PdfOutputBuffer(stream); + PdfSerializer.WriteDictionary(dictionary, output); + output.Flush(); + var written = Encoding.Latin1.GetString(stream.ToArray()); + + Assert.Contains("/ 2", written, StringComparison.Ordinal); + } + + /// A document whose page dictionary carries an empty-named entry opens and lists its page. + [Fact] + public void Opens_a_document_that_uses_the_empty_name_as_a_key() + { + var builder = new PdfBuilder(); + var catalog = builder.Reserve(); + var pages = builder.Reserve(); + + builder.Add("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] / (empty) >>"); + builder.Set(catalog, "<< /Type /Catalog /Pages 2 0 R / 1 >>"); + builder.Set(pages, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + + var reader = new PdfFileReader(builder.Build(catalog), "empty-name.pdf", SharpCorePDF.Infrastructure.PdfEngineOptions.Default); + + Assert.Single(reader.Pages); + } + /// The null object is one instance and writes itself as its keyword. [Fact] public void Exposes_the_null_object() diff --git a/tests/SharpCorePDF.Tests/PdfSerializationTests.cs b/tests/SharpCorePDF.Tests/PdfSerializationTests.cs index a3bf372..3ae3690 100644 --- a/tests/SharpCorePDF.Tests/PdfSerializationTests.cs +++ b/tests/SharpCorePDF.Tests/PdfSerializationTests.cs @@ -78,6 +78,28 @@ public void Writes_a_date_that_round_trips() Assert.Equal("D:20260401120000Z", PdfDate.Format(new DateTimeOffset(2026, 4, 1, 12, 0, 0, TimeSpan.Zero))); } + /// + /// An offset no time zone has — past fourteen hours either way, which refuses + /// to hold — is read as no offset rather than thrown, so one malformed metadata date does not fail the + /// inspection of the document it is in. The offsets at the edge of the range are kept. + /// + /// The date as a document states it. + /// The offset that is read. + [Theory] + [InlineData("D:20260101000000+99'00'", 0)] + [InlineData("D:20260101000000+23'59'", 0)] + [InlineData("D:20260101000000+14'30'", 0)] + [InlineData("D:20260101000000-15'00'", 0)] + [InlineData("D:20260101000000+14'00'", 14 * 60)] + [InlineData("D:20260101000000-14'00'", -14 * 60)] + [InlineData("D:20260101000000+05'30'", 5 * 60 + 30)] + public void Reads_an_offset_no_time_zone_has_as_no_offset(string text, int offsetMinutes) + { + Assert.True(PdfDate.TryParse(text, out var value)); + Assert.Equal(TimeSpan.FromMinutes(offsetMinutes), value.Offset); + Assert.Equal(new DateTime(2026, 1, 1, 0, 0, 0), value.DateTime); + } + /// Text that is not a date at all is rejected instead of being invented. [Theory] [InlineData("")] diff --git a/tests/SharpCorePDF.Tests/PdfTextScannerTests.cs b/tests/SharpCorePDF.Tests/PdfTextScannerTests.cs index 1811665..802c05a 100644 --- a/tests/SharpCorePDF.Tests/PdfTextScannerTests.cs +++ b/tests/SharpCorePDF.Tests/PdfTextScannerTests.cs @@ -15,6 +15,31 @@ namespace SharpCorePDF.Tests; /// public sealed class PdfTextScannerTests { + /// + /// A glyph drawn upside down — a negative font size, which a flipped text or font matrix produces — has a + /// box all the same: its descender is above its ascender, and the box is taken between them. + /// + [Fact] + public void Gives_a_glyph_drawn_upside_down_a_box() + { + var glyph = new SharpCorePDF.Domain.ReadModels.PdfGlyph + { + Text = "A", + PageNumber = 1, + X = 10, + Y = 100, + Width = 6, + FontSize = -10, + }; + + var box = glyph.BoundingBox; + + Assert.Equal(92d, box.Bottom, 6); + Assert.Equal(102d, box.Top, 6); + Assert.Equal(10d, box.Left, 6); + Assert.Equal(16d, box.Right, 6); + } + /// A string the scanner can take as a slice of the stream. /// The content stream. /// The text the shown string must decode to. From 74a4d7397d0bfc73fccf2c084be11bd79b6e5198 Mon Sep 17 00:00:00 2001 From: Andrew Tierney Date: Mon, 16 Nov 2026 12:39:33 +1100 Subject: [PATCH 05/15] Append only what an incremental update changes, and keep the permanent identifier UseIncrementalUpdate ran the full rewrite into an appended writer, so every object of the document, the signature dictionary of a signed source included, was written again under a new number. Every earlier byte was kept, but the file grew by its own size per edit, and validators that look at what changed after a signature (pyHanko, Acrobat) saw the signed field as rewritten: pyHanko placed a signature followed by an edit as CONTIGUOUS_BLOCK_FROM_START instead of ENTIRE_REVISION. In an appended revision the edited document's copier now keeps the numbers of its objects, the catalog, the page tree, the pages and the information dictionary are written under the numbers they already have, and the writer leaves out an object that would be written exactly as the source states it. Rotating a page of a signed document now appends 716 bytes and three objects instead of 20,969 bytes and twelve; the signature dictionary is not in the revision. The trailer also kept neither half of the source's /ID. ISO 32000-1 14.4 makes the first element permanent across incremental updates; it is now the source's, and only the second element is new. Tests that fail without the change: PdfIncrementalInvariantTests (the identifier, an update that appends only what it changed, a signature dictionary left out of the revision); the corpus checks validate-incremental.py and validate-signatures.py now pass with no --known-defect flag. The manual's editing chapter described every edit as a complete rewrite and now says what the option does. --- .github/workflows/validation.yml | 16 +--- CHANGELOG.md | 11 ++- docs/manual/src/14-editing.md | 12 ++- docs/validation.md | 8 +- .../Pdf/PdfObjectCopier.cs | 10 +- .../Pdf/PdfWriter.cs | 20 ++++ .../PdfEditEngine.Document.cs | 15 ++- .../PdfEditEngine.Incremental.cs | 46 +++++++++ .../PdfEditEngine.Pages.cs | 11 ++- .../PdfEditEngine.Sources.cs | 10 +- .../PdfEditEngine.cs | 50 ++++++++-- .../PdfIncrementalInvariantTests.cs | 95 ++++++++++++++++++- 12 files changed, 267 insertions(+), 37 deletions(-) diff --git a/.github/workflows/validation.yml b/.github/workflows/validation.yml index a632011..681f1fd 100644 --- a/.github/workflows/validation.yml +++ b/.github/workflows/validation.yml @@ -21,8 +21,7 @@ # # --known-defect / --known names a failure that is recorded and not yet fixed. Each is strict: when the # defect is fixed the check passes, the flag no longer matches, and the run fails until the flag is removed -# in the same change as the fix. The in-suite counterparts are the skipped tests in -# PdfSignatureAttackTests and PdfIncrementalInvariantTests. +# in the same change as the fix. # # The validator versions are pinned in docs/tools/validation-requirements.txt, pdfjs-dist below. @@ -119,20 +118,11 @@ jobs: "$RUNNER_TEMP/corpus/text-merged.pdf" "$RUNNER_TEMP/corpus/incremental.pdf" \ "$RUNNER_TEMP/corpus/signed.pdf" - # id-first: the editor writes a new identifier into both elements of /ID on an appended revision. - # appends-only: the editor's incremental update appends a renumbered copy of the whole document - # rather than the objects it changed. - name: Incremental update invariants - run: | - python docs/tools/validate-incremental.py "$RUNNER_TEMP/corpus" \ - --known-defect id-first --known-defect appends-only + run: python docs/tools/validate-incremental.py "$RUNNER_TEMP/corpus" - # The same copy rewrites the signature dictionary after signing, which is why pyHanko cannot place the - # first signature of signed-then-edited.pdf at the end of a revision. - name: Signatures, pyHanko - run: | - python docs/tools/validate-signatures.py --manifest "$RUNNER_TEMP/corpus/manifest.json" \ - --known-defect signed-then-edited.pdf:FirstSignature + run: python docs/tools/validate-signatures.py --manifest "$RUNNER_TEMP/corpus/manifest.json" - name: Fuzz targets over their seeds run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 01e5662..caf759e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -65,13 +65,20 @@ against an empty feed. - **`PdfDate.TryParse` answers instead of throwing** for a time-zone offset beyond fourteen hours, which it now reads as no offset; one malformed `/CreationDate` no longer fails the inspection of the document. +- **An incremental update appends what changed, and only that.** `UseIncrementalUpdate` appended a copy of + the whole document under new object numbers: the file grew by its own size per edit, and the signature + dictionary of a signed source was written again after signing, so validators that look at what changed + after a signature (pyHanko, Acrobat) saw the signed field as modified. The edited document's objects now + keep their numbers and an object written exactly as the source already states it is left out — rotating a + page of a signed document appends 716 bytes and three objects instead of 20,969 bytes and twelve. +- **An incremental update keeps the permanent identifier.** The first element of `/ID` is the source's in + every revision, as ISO 32000-1 §14.4 requires; only the second element is new. + ### Known defects, found by the new checks and not yet fixed Each is reproduced, pinned by a skipped test or a strict `--known-defect` flag that fails the run once the defect is fixed, and listed in [`docs/validation.md`](docs/validation.md#known-defects). -- An incremental update rewrites both elements of `/ID`, and appends a renumbered copy of the whole document - rather than the objects it changed — which also rewrites the signature dictionary after signing. - A merge copies a page that states no `/MediaBox` without one. - **Run-length images are decoded.** `RunLengthDecode` joins Flate, LZW and the two ASCII filters: a diff --git a/docs/manual/src/14-editing.md b/docs/manual/src/14-editing.md index 13f9bb8..11d036a 100644 --- a/docs/manual/src/14-editing.md +++ b/docs/manual/src/14-editing.md @@ -251,10 +251,18 @@ change the bytes of the content streams they show. The source can be encrypted a the `password` argument of `ApplyAsync`; the result is written in the clear, because output encryption is not part of this editing release. -The writer produces a complete new document rather than an incremental update. That is what keeps -earlier revisions and signatures out of a document whose pages have changed; it also means a copied +By default the writer produces a complete new document rather than an incremental update. That is what +keeps earlier revisions and signatures out of a document whose pages have changed; it also means a copied signature is not a signature of the edited document. +With `PdfEditOutputOptions.UseIncrementalUpdate` the edit is appended to the source instead: every byte of +the source stays where it was, and the revision carries only the objects the edit changed, under the numbers +they already had — rotating one page redeclares that page and the few objects around it, and an object that +would be written exactly as the source states it is left out. A signature over the source therefore still +covers the revision it was made over, and a validator that looks at what changed after signing sees the +edit and not a second copy of the document. The first element of the trailer `/ID` is kept, as ISO 32000-1 +§14.4 requires. + ## Reading the result `ApplyAsync` returns a `PdfEditResult`: diff --git a/docs/validation.md b/docs/validation.md index 1d79368..47882c2 100644 --- a/docs/validation.md +++ b/docs/validation.md @@ -56,8 +56,8 @@ dotnet run --project .\tools\SharpCorePDF.Validation -c Release -- corpus $env:T python docs/tools/validate-text.py $env:TEMP\corpus python docs/tools/validate-encryption.py $env:TEMP\corpus -python docs/tools/validate-incremental.py $env:TEMP\corpus --known-defect id-first --known-defect appends-only -python docs/tools/validate-signatures.py --manifest $env:TEMP\corpus\manifest.json --known-defect signed-then-edited.pdf:FirstSignature +python docs/tools/validate-incremental.py $env:TEMP\corpus +python docs/tools/validate-signatures.py --manifest $env:TEMP\corpus\manifest.json python docs/tools/validate-fonts.py $env:TEMP\SharpCorePDF.Examples\output python docs/tools/render-compare.py --exclude protected.pdf $env:TEMP\SharpCorePDF.Examples\output @@ -83,8 +83,6 @@ fix. This is the list on 2026-11-16, every entry reproduced, not inferred: | Defect | Found by | Evidence | | --- | --- | --- | -| An incremental update writes a new identifier into both elements of `/ID`; ISO 32000-1 §14.4 keeps the first | #10 | `PdfIncrementalInvariantTests.Keeps_the_first_identifier_element_across_an_incremental_update` (skipped); `validate-incremental.py` `id-first` | -| `UseIncrementalUpdate` appends a renumbered copy of the whole document rather than the objects it changed: every byte of the earlier revision is kept, but the file grows by its own size per edit, and the signature dictionary is written again after signing, so pyHanko places a signature followed by an edit as `CONTIGUOUS_BLOCK_FROM_START` with modification level `OTHER` instead of `ENTIRE_REVISION` | #10, #3 | `validate-incremental.py` `appends-only`; `validate-signatures.py` on `signed-then-edited.pdf` | | A merge copies a page that states no `/MediaBox` (and inherits none) without one, so the result breaks the rule that every page has a media box; qpdf repairs it by assuming Letter | #2 | `validate-pdf.py` on `inspected.pdf` and `with-warnings.pdf` (both merge a page of `NamedDests.pdf`) | Fixed since the list was first drawn up, each with the test that fails without its fix: @@ -99,6 +97,8 @@ Fixed since the list was first drawn up, each with the test that fails without i | The empty name as a dictionary key (`/` alone, which PDF allows) stopped the document opening | `PdfDictionary.Set` accepts it; the entry is kept and written back as `/` | `PdfObjectModelTests.Keeps_an_entry_whose_key_is_the_empty_name`, `Opens_a_document_that_uses_the_empty_name_as_a_key` | | A glyph drawn upside down (negative font size) faulted text extraction | `PdfGlyph.BoundingBox` takes the box between descender and ascender whichever way round | `PdfTextScannerTests.Gives_a_glyph_drawn_upside_down_a_box` | | `PdfDate.TryParse` threw for a time-zone offset beyond 14 hours | an offset no time zone has is read as no offset | `PdfSerializationTests.Reads_an_offset_no_time_zone_has_as_no_offset` | +| An incremental update wrote a new identifier into both elements of `/ID` | the first element of the source's `/ID` is kept; only the second is new | `PdfIncrementalInvariantTests.Keeps_the_first_identifier_element_across_an_incremental_update` | +| `UseIncrementalUpdate` appended a renumbered copy of the whole document, rewriting the signature dictionary after signing | the edited document's objects keep their numbers, and an object written exactly as the source states it is left out: rotating one page of a signed document appends 716 bytes and three objects instead of 20,969 bytes and twelve, and pyHanko places the signature at `ENTIRE_REVISION` | `PdfIncrementalInvariantTests.Appends_only_the_objects_an_edit_changes`, `Leaves_the_signature_dictionary_out_of_an_appended_revision` | ## Corpus readings, 2026-11-16 diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfObjectCopier.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfObjectCopier.cs index a9bc19d..a1bda02 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfObjectCopier.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfObjectCopier.cs @@ -27,6 +27,7 @@ internal sealed partial class PdfObjectCopier private readonly HashSet redirected = []; private readonly Queue pending = []; private readonly HashSet queued = []; + private bool keepNumbers; /// Initializes a new instance of the class. /// The document to copy from. @@ -65,6 +66,13 @@ public void Override(long sourceObjectNumber, Func rewrite overrides[sourceObjectNumber] = rewrite; } + /// + /// Makes every object this copier copies keep the number it has in its source, instead of being given a + /// new one: the copier of the document an incremental update is appended to, whose objects are the same + /// objects in both revisions. + /// + public void KeepObjectNumbers() => keepNumbers = true; + /// Reserves a target object number for an object of the source and queues its body. /// The object number in the source document. /// The object number in the merged document. @@ -75,7 +83,7 @@ public int Reserve(long sourceObjectNumber) return existing; } - var target = writer.AllocateObject(); + var target = keepNumbers ? checked((int)sourceObjectNumber) : writer.AllocateObject(); targets[sourceObjectNumber] = target; Queue(sourceObjectNumber); return target; diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfWriter.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfWriter.cs index 55e4da0..d3cd0b7 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfWriter.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfWriter.cs @@ -16,6 +16,7 @@ internal sealed class PdfWriter private readonly PdfSecurityWriter? security; private readonly Dictionary offsets = []; private readonly long origin; + private Func? isUnchanged; private int lastObjectNumber; /// Initializes a new instance of the class. @@ -95,6 +96,20 @@ private PdfWriter(Stream destination, PdfEngineOptions options, int highestObjec /// The reserved object number. public int AllocateObject() => ++lastObjectNumber; + /// + /// Makes the writer leave out an object that would be written exactly as the revision it is appended to + /// already holds it under the same number. + /// + /// Answers whether an object number and the body about to be written under it are + /// what the earlier revision already states. + /// + /// An incremental update is meant to carry what the edit changed. An object that is written again unchanged + /// costs its bytes and, worse, tells every reader that looks at what changed after a signature — pyHanko's + /// difference analysis, Acrobat's — that the object was modified after signing. + /// + public void SkipUnchanged(Func unchanged) => + isUnchanged = unchanged ?? throw new ArgumentNullException(nameof(unchanged)); + /// Gets the highest object number that has been allocated. public int HighestObjectNumber => lastObjectNumber; @@ -103,6 +118,11 @@ private PdfWriter(Stream destination, PdfEngineOptions options, int highestObjec /// The body of the object. public void WriteObject(int objectNumber, PdfObject body) { + if (isUnchanged?.Invoke(objectNumber, body) == true) + { + return; + } + offsets[objectNumber] = Length; output.WriteInteger(objectNumber); output.Write(" 0 obj\n"u8); diff --git a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Document.cs b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Document.cs index 539f7b3..93a0682 100644 --- a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Document.cs +++ b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Document.cs @@ -693,10 +693,11 @@ private static PdfMetadata ResolveMetadata( /// Writes the document information dictionary and returns its object number. /// The writer of the result. /// The document information of the result. + /// The number the dictionary already has in the revision an update is appended to, or to number it anew. /// The object number of the document information dictionary. - private static int WriteInformation(PdfWriter writer, PdfMetadata metadata) + private static int WriteInformation(PdfWriter writer, PdfMetadata metadata, int? existingNumber = null) { - var number = writer.AllocateObject(); + var number = existingNumber ?? writer.AllocateObject(); var information = new PdfDictionary(); information.Set("Title", Text(metadata.Title)); @@ -728,16 +729,18 @@ private static int WriteInformation(PdfWriter writer, PdfMetadata metadata) /// The writer of the result. /// The object number of the catalog. /// The object number of the document information dictionary, or when the profile forbids it. - /// The identifier written as both elements of the ID entry. + /// The identifier of this revision: the second element of the ID entry, and the first too unless a permanent one is given. /// The encryption of the result, or when it is written in the clear. /// The cross-reference section this revision chains to, or for a complete document. + /// The first element of the source's ID, kept by an appended revision, or . private static void WriteTrailer( PdfWriter writer, int catalogNumber, int? infoNumber, PdfString identifier, PdfSecurityWriter? security, - long? previousXref) + long? previousXref, + PdfString? permanentIdentifier = null) { var trailer = new PdfDictionary(); trailer.Set("Root", new PdfReference(catalogNumber, 0)); @@ -746,7 +749,9 @@ private static void WriteTrailer( trailer.Set("Info", new PdfReference(info, 0)); } - trailer.Set("ID", new PdfArray(new PdfObject[] { identifier, identifier })); + // The first element is permanent: an appended revision keeps the one the document was created with, + // and only the second, which identifies this revision, is new. + trailer.Set("ID", new PdfArray(new PdfObject[] { permanentIdentifier ?? identifier, identifier })); if (previousXref is { } previous) { diff --git a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Incremental.cs b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Incremental.cs index d9a8e3b..10625a4 100644 --- a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Incremental.cs +++ b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Incremental.cs @@ -108,6 +108,52 @@ private static bool FieldsCarrySignature(PdfFileReader reader, PdfObject? fields return false; } + /// + /// Determines whether an object is exactly what the revision an update is appended to already states under + /// the same number, by writing both the way the writer would and comparing the bytes. + /// + /// The document the revision is appended to. + /// The number the object is about to be written under. + /// The body about to be written. + /// when writing it would restate the source unchanged. + private static bool IsUnchangedInSource(PdfFileReader source, int objectNumber, PdfObject body) + { + if (objectNumber > source.CrossReferenceTable.HighestObjectNumber || source.Resolve(objectNumber) is not { } original) + { + return false; + } + + return Serialize(original).AsSpan().SequenceEqual(Serialize(body)); + } + + private static byte[] Serialize(PdfObject value) + { + using var stream = new MemoryStream(); + var output = new PdfOutputBuffer(stream); + + PdfSerializer.Write(value, output); + output.Flush(); + + return stream.ToArray(); + } + + /// Gets the number of the source's document information dictionary, when it is an object of its own. + /// The document the revision is appended to. + /// The object number, or . + private static int? SourceInformationNumber(PdfFileReader source) => + source.CrossReferenceTable.Trailer.Get("Info") is PdfReference information + ? checked((int)information.ObjectNumber) + : null; + + /// + /// Gets the first element of the source's /ID, which ISO 32000-1 §14.4 makes permanent: it "shall + /// not change when the file is incrementally updated". + /// + /// The document the revision is appended to. + /// The identifier, or when the source states none. + private static PdfString? PermanentIdentifier(PdfFileReader source) => + source.CrossReferenceTable.Trailer.Get("ID") is PdfArray { Items: [PdfString permanent, ..] } ? permanent : null; + /// Copies the source into the destination and opens the writer of the appended revision. /// The stream the result is written to. /// The document being edited. diff --git a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Pages.cs b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Pages.cs index 1fa27f2..dbda7f5 100644 --- a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Pages.cs +++ b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Pages.cs @@ -290,11 +290,18 @@ private static List BuildLayout( /// Reserves the object number of every page that is written. /// The writer of the result. /// The pages in output order. - private static void AllocatePageTargets(PdfWriter writer, List layout) + /// The document an incremental update is appended to, whose pages keep their numbers, or . + private static void AllocatePageTargets(PdfWriter writer, List layout, PdfFileReader? keepNumbersOf) { + var kept = new HashSet(); + foreach (var page in layout) { - page.TargetNumber = writer.AllocateObject(); + // In an appended revision a page of the source stays the object it was; a second occurrence of the + // same page — a duplicate — is a new page and gets a number of its own. + page.TargetNumber = page.Reader == keepNumbersOf && kept.Add(page.Source.ObjectNumber) + ? checked((int)page.Source.ObjectNumber) + : writer.AllocateObject(); } } diff --git a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Sources.cs b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Sources.cs index 77ca7b6..dc479e4 100644 --- a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Sources.cs +++ b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Sources.cs @@ -221,16 +221,24 @@ public EditSources( /// Creates the copier of every document. /// The writer of the result. + /// The document an incremental update is appended to, whose copier keeps the numbers of its objects, or . /// /// Object numbers are only unique inside one document, so every document gets a copier of its own. /// All of them write into the same result, which is what makes a copied page indistinguishable /// from a page that was already there. /// - public void Attach(PdfWriter writer) + public void Attach(PdfWriter writer, PdfFileReader? keepNumbersOf = null) { foreach (var reader in readers) { copiers[reader] = new PdfObjectCopier(reader, writer); + + // Only the document a revision is appended to shares its numbers with the result; a document + // the edit imports from has numbers of its own, which would collide with them. + if (reader == keepNumbersOf) + { + copiers[reader].KeepObjectNumbers(); + } } } diff --git a/src/SharpCorePDF.Infrastructure/PdfEditEngine.cs b/src/SharpCorePDF.Infrastructure/PdfEditEngine.cs index 7500422..c6d4751 100644 --- a/src/SharpCorePDF.Infrastructure/PdfEditEngine.cs +++ b/src/SharpCorePDF.Infrastructure/PdfEditEngine.cs @@ -216,20 +216,48 @@ public async Task ApplyAsync( : null; var writer = revision?.Writer ?? new PdfWriter(capture ?? destination, version, options, security); - var catalogNumber = writer.AllocateObject(); - var pagesNodeNumber = writer.AllocateObject(); + + // An appended revision updates the objects of the revision before it rather than writing a second + // copy of them: the catalog, the page tree, the pages and every object copied from the source keep + // their numbers, and an object that comes out exactly as the source already states it is left out. + var primary = revision is null ? null : reader; + + if (primary is not null) + { + writer.SkipUnchanged((number, body) => IsUnchangedInSource(primary, number, body)); + } + + var catalogNumber = primary is not null && primary.RootObjectNumber > 0 + ? checked((int)primary.RootObjectNumber) + : writer.AllocateObject(); + var pagesNodeNumber = primary?.Catalog?.Get("Pages") is PdfReference pagesRoot + ? checked((int)pagesRoot.ObjectNumber) + : writer.AllocateObject(); // Every document the plan reads has its own copier, because object numbers are only unique // inside one document. All copiers write into the same result, which is what makes an // imported page indistinguishable from a page that was already there. - sources.Attach(writer); + sources.Attach(writer, keepNumbersOf: primary); + + if (primary is not null) + { + // The catalog and the information dictionary are written by the engine under their own numbers; + // a copy of the old ones, reached through a reference, would be a second object of the same + // number in the same revision. + sources.CopierOf(primary).Redirect(catalogNumber, catalogNumber); + + if (SourceInformationNumber(primary) is { } informationNumber) + { + sources.CopierOf(primary).Redirect(informationNumber, informationNumber); + } + } foreach (var (source, copier) in sources.Copiers) { RedirectPageTree(source, copier, pagesNodeNumber); } - AllocatePageTargets(writer, layout); + AllocatePageTargets(writer, layout, primary); foreach (var (source, copier) in sources.Copiers) { @@ -279,8 +307,18 @@ [.. layout.Select(page => page.TargetNumber)], // PDF/A-4 forbids the document information dictionary (a trailer /Info is allowed only beside a // catalog /PieceInfo, which this writer does not produce), so the packet is the only copy there. - var infoNumber = profile == PdfAProfile.PdfA4 ? (int?)null : WriteInformation(writer, metadata); - WriteTrailer(writer, catalogNumber, infoNumber, identifier, security, revision?.PreviousXref); + var existingInformation = primary is null ? null : SourceInformationNumber(primary); + var infoNumber = profile == PdfAProfile.PdfA4 + ? (int?)null + : WriteInformation(writer, metadata, existingInformation); + WriteTrailer( + writer, + catalogNumber, + infoNumber, + identifier, + security, + revision?.PreviousXref, + primary is null ? null : PermanentIdentifier(primary)); Verify(capture, layout, plan.Output.Encryption?.UserPassword, warnings); diff --git a/tests/SharpCorePDF.Tests/PdfIncrementalInvariantTests.cs b/tests/SharpCorePDF.Tests/PdfIncrementalInvariantTests.cs index fc5ad8e..95fe3ef 100644 --- a/tests/SharpCorePDF.Tests/PdfIncrementalInvariantTests.cs +++ b/tests/SharpCorePDF.Tests/PdfIncrementalInvariantTests.cs @@ -2,6 +2,7 @@ using SharpCorePDF.Domain.Aggregates; using SharpCorePDF.Domain.ValueObjects; using SharpCorePDF.Infrastructure; +using SharpCorePDF.Infrastructure.Extensions; using SharpCorePDF.Infrastructure.Pdf; using SharpCorePDF.Infrastructure.Sources; @@ -75,7 +76,7 @@ public async Task States_a_size_that_covers_every_revision() } /// The permanent half of the identifier survives an update; only the second half may change. - [Fact(Skip = "Known defect: PdfEditEngine.WriteTrailer writes a new identifier into both elements of /ID even when the revision is appended (PdfEditEngine.Document.cs). Remove the skip with the fix.")] + [Fact] public async Task Keeps_the_first_identifier_element_across_an_incremental_update() { var source = Source(); @@ -85,6 +86,98 @@ public async Task Keeps_the_first_identifier_element_across_an_incremental_updat Assert.Equal(Identifier(source, 0), Identifier(updated, 0)); } + /// + /// An appended revision carries what the edit changed under the numbers those objects already had: rotating + /// one page redeclares that page and the few objects around it, and adds no copy of the document under new + /// numbers. + /// + [Fact] + public async Task Appends_only_the_objects_an_edit_changes() + { + var source = Source(); + var updated = await AppendAsync(source, PdfEditPlan.Create().RotatePage(1, PageRotation.Clockwise90)); + + var before = ObjectNumbers(source); + var appended = ObjectNumbers(updated[source.Length..]); + + Assert.NotEmpty(appended); + Assert.Subset(before, appended); + Assert.True(appended.Count <= 4, $"the revision redeclares {appended.Count} objects: {string.Join(", ", appended)}"); + } + + /// + /// A signature dictionary is not written again by an update appended after signing: it stays the object of + /// the signed revision, which is what lets a validator place the signature at the end of that revision. + /// + [Fact] + public async Task Leaves_the_signature_dictionary_out_of_an_appended_revision() + { + var signed = await SigningFixtures.SignAsync(Source(), SigningFixtures.Options(SigningFixtures.Certificate)); + var updated = await AppendAsync(signed, PdfEditPlan.Create().RotatePage(1, PageRotation.Clockwise90)); + var revision = System.Text.Encoding.Latin1.GetString(updated, signed.Length, updated.Length - signed.Length); + + Assert.DoesNotContain("/ByteRange", revision, StringComparison.Ordinal); + Assert.DoesNotContain("/Type /Sig", revision, StringComparison.Ordinal); + + var validation = Assert.Single(await SigningFixtures.ValidateAsync(updated)); + + Assert.True(validation.IsValid); + Assert.False(validation.Signer.CoversWholeDocument); + } + + /// + /// The objects an edit creates are written into the revision under numbers of their own: an inserted page + /// is a new object, and a reader of the result finds it between the pages that were there. + /// + [Fact] + public async Task Appends_the_objects_an_edit_creates() + { + var source = Source(); + var updated = await AppendAsync( + source, + PdfEditPlan.Create().InsertPage(2, PdfPageSize.A5, page => page.DrawText("inserted", 20, 40, PdfTypeface.Standard(StandardFont.Helvetica), 10))); + + var reader = new PdfFileReader(updated, "updated.pdf", Options); + var fresh = ObjectNumbers(updated[source.Length..]).Except(ObjectNumbers(source)).ToList(); + + Assert.Equal(3, reader.Pages.Count); + Assert.NotEmpty(fresh); + Assert.All(fresh, number => Assert.True(number > ObjectNumbers(source).Max)); + } + + /// + /// A source whose objects refer to its own catalog and information dictionary is updated with one + /// definition of each in the revision: the engine writes them, and the copier, which reaches them through + /// those references, must not write a second copy under the same number. + /// + [Fact] + public async Task Writes_each_object_once_when_the_source_refers_to_its_own_catalog() + { + var builder = new PdfBuilder(); + var catalog = builder.Reserve(); + var pages = builder.Reserve(); + + builder.Add("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Back 1 0 R /About 4 0 R >>"); + builder.Add("<< /Title (Refers to its own catalog) >>"); + builder.Set(catalog, "<< /Type /Catalog /Pages 2 0 R >>"); + builder.Set(pages, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + + var source = builder.Build(catalog, "/Info 4 0 R /ID [<00112233445566778899AABBCCDDEEFF> <00112233445566778899AABBCCDDEEFF>]"); + var updated = await AppendAsync(source, PdfEditPlan.Create().RotatePage(1, PageRotation.Clockwise90)); + var revision = System.Text.Encoding.Latin1.GetString(updated, source.Length, updated.Length - source.Length); + var declared = System.Text.RegularExpressions.Regex.Matches(revision, @"(?m)^(\d+) 0 obj\b") + .Select(match => match.Groups[1].Value) + .ToList(); + + Assert.Equal(declared.Distinct().Count(), declared.Count); + Assert.Equal(90, (int)(new PdfFileReader(updated, "updated.pdf", Options).Pages[0].Dictionary.GetInteger("Rotate") ?? 0L)); + } + + private static SortedSet ObjectNumbers(byte[] bytes) => + [.. System.Text.RegularExpressions.Regex + .Matches(System.Text.Encoding.Latin1.GetString(bytes), @"(?m)^(\d+) 0 obj\b") + .Select(match => int.Parse(match.Groups[1].Value, System.Globalization.CultureInfo.InvariantCulture))]; + private static byte[] Source() => PdfDocumentBuilder.Create(PdfAuthoringOptions.Default with { DocumentId = SourceId }) .AddPage(PdfPageSize.A5, page => page.DrawText("one", 72, 100, PdfTypeface.Standard(StandardFont.Helvetica), 12)) .AddPage(PdfPageSize.A4, page => page.DrawText("two", 72, 100, PdfTypeface.Standard(StandardFont.Helvetica), 12)) From 5fbfc925ad5b94491be121fae272fa455a005d1d Mon Sep 17 00:00:00 2001 From: Andrew Tierney Date: Mon, 16 Nov 2026 12:54:47 +1100 Subject: [PATCH 06/15] Write a missing media box, keep copied reals, and fix what the first CI runs found - A merge copied a page that states no /MediaBox, and inherits none, without one, so the result broke the rule that every page has a media box and each reader assumed its own size. The page is written with the A4 box the inspection already describes it with, and PageSizeAssumed says so. - A merge or an edit rewrote every real it copied to five decimals: a Type 3 font matrix of 0.000012 came out as 0.00001. A real read from a document now keeps its text when that is a well-formed PDF real and is written back from it. A real too large for the formatter's 32-byte buffer failed the write of the whole document with InvalidOperationException; the first run of the document fuzz target found it, and such a value is now formatted on the allocating path. - The image copy route let InvalidDataException escape for damaged Flate data, with or without a predictor (the flate fuzz target's first find); it now refuses it with PdfSourceFormatException. - The Arlington service names its verdict element arlingtonReport, which validate-arlington.py now reads. Its first run found that a free text annotation lacked the /DA ISO 32000-1 requires and annotation appearances lacked /Resources; both are now written. Tests that fail without the changes: PdfMergeFidelityTests (media box, copied reals, any magnitude), PdfContentReaderTests.Refuses_to_copy_an_image_whose_flate_data_is_damaged, and PdfDocumentEditorTests.States_the_default_appearance_and_resources_of_a_free_text_annotation and States_resources_on_an_appearance_that_needs_none. qpdf now passes every example output with no --known-defect flag, so the qpdf step lists none. --- .github/workflows/validation.yml | 4 +- CHANGELOG.md | 17 ++- docs/tools/validate-arlington.py | 7 +- docs/validation.md | 5 +- .../Annotations/AppearanceStreams.cs | 9 +- .../Pdf/PdfDocumentInspection.Pages.cs | 6 +- .../Pdf/PdfImageFilters.cs | 23 +++- .../Pdf/PdfObject.cs | 25 +++- .../Pdf/PdfObjectCopier.Clone.cs | 2 +- .../Pdf/PdfObjectParser.cs | 36 +++++- .../Pdf/PdfOutputBuffer.cs | 6 +- .../Pdf/PdfSerializer.cs | 4 + .../PdfEditEngine.Annotations.cs | 12 ++ .../PdfMergeEngine.Pages.cs | 26 +++- .../SharpCorePDF.Tests/PdfAnnotationTests.cs | 50 ++++++++ .../PdfContentReaderTests.Images.cs | 30 +++++ .../PdfMergeFidelityTests.cs | 113 ++++++++++++++++++ 17 files changed, 345 insertions(+), 30 deletions(-) create mode 100644 tests/SharpCorePDF.Tests/PdfMergeFidelityTests.cs diff --git a/.github/workflows/validation.yml b/.github/workflows/validation.yml index 681f1fd..b25e731 100644 --- a/.github/workflows/validation.yml +++ b/.github/workflows/validation.yml @@ -90,9 +90,7 @@ jobs: case "$(basename "$file")" in protected.pdf|encrypted-*) continue ;; esac files+=("$file") done - # Both merge a page of NamedDests.pdf, which states no /MediaBox; the merge copies it without one, - # so the result breaks the rule that every page has a media box, and qpdf assumes Letter. - python docs/tools/validate-pdf.py --known-defect inspected.pdf --known-defect with-warnings.pdf "${files[@]}" + python docs/tools/validate-pdf.py "${files[@]}" - name: Text round trip run: python docs/tools/validate-text.py "$RUNNER_TEMP/corpus" diff --git a/CHANGELOG.md b/CHANGELOG.md index caf759e..7aa74cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -74,12 +74,17 @@ against an empty feed. - **An incremental update keeps the permanent identifier.** The first element of `/ID` is the source's in every revision, as ISO 32000-1 §14.4 requires; only the second element is new. -### Known defects, found by the new checks and not yet fixed - -Each is reproduced, pinned by a skipped test or a strict `--known-defect` flag that fails the run once the -defect is fixed, and listed in [`docs/validation.md`](docs/validation.md#known-defects). - -- A merge copies a page that states no `/MediaBox` without one. +- **A merge writes a media box for a page that states none.** Every page of a PDF needs one; a source page + with none, and none to inherit, was copied without it, and each reader assumed its own size. It is now + written with the A4 box the inspection already describes it with, and `PageSizeAssumed` says so. +- **Copied real numbers keep their value.** A merge or an edit rewrote every real it copied to five decimals — + a Type 3 font matrix of `0.000012` became `0.00001`, seventeen percent off, and `0.000004` became `0`. A real + read from a document is now written back from its own text. A real too large for the formatter's buffer no + longer fails the write of the whole document (found by the `document` fuzz target in its first run). +- **The image copy route refuses damaged Flate data with `PdfSourceFormatException`** instead of letting the + decompressor's `InvalidDataException` reach the caller (found by the `flate` fuzz target). +- **Annotations satisfy the Arlington model.** A free text annotation states the `/DA` ISO 32000-1 requires, + and every annotation appearance states its `/Resources` (found by the Arlington check in its first run). - **Run-length images are decoded.** `RunLengthDecode` joins Flate, LZW and the two ASCII filters: a bilevel or scan-style image that declares it now yields its samples — and `Gray8`, `Rgb24` or `Cmyk32` diff --git a/docs/tools/validate-arlington.py b/docs/tools/validate-arlington.py index 3d99086..35bcd24 100644 --- a/docs/tools/validate-arlington.py +++ b/docs/tools/validate-arlington.py @@ -71,10 +71,13 @@ def read_report(payload: bytes) -> tuple[bool, list[str]]: return compliant.group(1) == "true", re.findall(r'"clause"\s*:\s*"([^"]+)"', text) - reports = [element for element in root.iter() if element.tag.endswith("validationReport")] + # The Arlington service names its verdict element arlingtonReport (veraPDF's PDF/A reports name theirs + # validationReport); both carry isCompliant, which is what the verdict is read from. + reports = [element for element in root.iter() + if element.tag.endswith(("arlingtonReport", "validationReport")) and element.get("isCompliant")] if not reports: - raise ValueError("the XML report holds no validationReport element") + raise ValueError("the XML report holds no arlingtonReport or validationReport element") failed = [] diff --git a/docs/validation.md b/docs/validation.md index 47882c2..a06e569 100644 --- a/docs/validation.md +++ b/docs/validation.md @@ -83,7 +83,6 @@ fix. This is the list on 2026-11-16, every entry reproduced, not inferred: | Defect | Found by | Evidence | | --- | --- | --- | -| A merge copies a page that states no `/MediaBox` (and inherits none) without one, so the result breaks the rule that every page has a media box; qpdf repairs it by assuming Letter | #2 | `validate-pdf.py` on `inspected.pdf` and `with-warnings.pdf` (both merge a page of `NamedDests.pdf`) | Fixed since the list was first drawn up, each with the test that fails without its fix: @@ -99,6 +98,10 @@ Fixed since the list was first drawn up, each with the test that fails without i | `PdfDate.TryParse` threw for a time-zone offset beyond 14 hours | an offset no time zone has is read as no offset | `PdfSerializationTests.Reads_an_offset_no_time_zone_has_as_no_offset` | | An incremental update wrote a new identifier into both elements of `/ID` | the first element of the source's `/ID` is kept; only the second is new | `PdfIncrementalInvariantTests.Keeps_the_first_identifier_element_across_an_incremental_update` | | `UseIncrementalUpdate` appended a renumbered copy of the whole document, rewriting the signature dictionary after signing | the edited document's objects keep their numbers, and an object written exactly as the source states it is left out: rotating one page of a signed document appends 716 bytes and three objects instead of 20,969 bytes and twelve, and pyHanko places the signature at `ENTIRE_REVISION` | `PdfIncrementalInvariantTests.Appends_only_the_objects_an_edit_changes`, `Leaves_the_signature_dictionary_out_of_an_appended_revision` | +| A merge copied a page that states no `/MediaBox` (and inherits none) without one | the page is written with the A4 box the inspection describes it with, and `PageSizeAssumed` says so | `PdfMergeFidelityTests.Writes_a_media_box_for_a_page_that_has_none` | +| Reals a merge or an edit copied were re-formatted to five decimals (`0.000012` became `0.00001`), and a real too large for the formatter's buffer failed the write of the whole document — the first crash the `document` fuzz target found | a real read from a document is written from its own text when that text is a well-formed PDF real; other values are formatted at any magnitude | `PdfMergeFidelityTests.Writes_the_reals_it_copies_as_the_source_states_them`, `Writes_a_real_of_any_magnitude_in_the_form_pdf_defines` | +| The copy route of an image let the decompressor's `InvalidDataException` escape for damaged Flate data — the crash the `flate` fuzz target found | the copy route refuses damaged data with `PdfSourceFormatException`, with or without a predictor | `PdfContentReaderTests.Refuses_to_copy_an_image_whose_flate_data_is_damaged` | +| A free text annotation was written without the `/DA` ISO 32000-1 requires, and annotation appearances without `/Resources` — both reported by the Arlington check's first run | `/DA` states the font, size and colour the appearance draws with; every appearance states its resources, empty if it needs none | `PdfDocumentEditorTests.States_the_default_appearance_and_resources_of_a_free_text_annotation`, `States_resources_on_an_appearance_that_needs_none` | ## Corpus readings, 2026-11-16 diff --git a/src/SharpCorePDF.Infrastructure/Annotations/AppearanceStreams.cs b/src/SharpCorePDF.Infrastructure/Annotations/AppearanceStreams.cs index c38fd87..ac2adfe 100644 --- a/src/SharpCorePDF.Infrastructure/Annotations/AppearanceStreams.cs +++ b/src/SharpCorePDF.Infrastructure/Annotations/AppearanceStreams.cs @@ -150,12 +150,11 @@ private static PdfStream Wrap( .SetName("Type", "XObject") .SetName("Subtype", "Form") .Set("BBox", ToArray(bounds)) - .SetName("Filter", "FlateDecode"); + .SetName("Filter", "FlateDecode") - if (resources.Count > 0) - { - dictionary.Set("Resources", resources); - } + // Stated even when empty: a form XObject without /Resources leaves a reader to borrow the page's, + // a PDF 1.1 fallback the Arlington model of PDF 1.7 and later no longer allows. + .Set("Resources", resources); return new PdfStream(dictionary, EmbeddedImage.Compress(operators)); } diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfDocumentInspection.Pages.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfDocumentInspection.Pages.cs index 13deea1..86f20a6 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfDocumentInspection.Pages.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfDocumentInspection.Pages.cs @@ -39,7 +39,11 @@ private static List ReadPages(PdfFileReader reader, ListReads the size a page's own or inherited /MediaBox states. + /// The document. + /// The page. + /// The size, or when the page has no readable media box. + internal static PdfPageSize? TryReadPageSize(PdfFileReader reader, SourcePage page) { var box = reader.ResolveArray(page.Dictionary.Get("MediaBox")) ?? reader.ResolveArray(page.Inherited.Get("MediaBox")); diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfImageFilters.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfImageFilters.cs index 15951b7..dc7d8f2 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfImageFilters.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfImageFilters.cs @@ -154,14 +154,27 @@ public static async Task CopyAsync( // reach, because the two tests name the same predictor values. using var predictor = PdfFilters.CreatePredictorDecoder(decodeParameters, sourceName); - if (predictor is not null) + // A damaged stream is the document's fault, not the caller's, and the contract names it with a + // PdfSourceFormatException: the decompressor's own exception would reach the caller as a failure + // of the library. What was written before the damage stays written; the caller learns it is + // incomplete from the exception. + try { - await InflatePredictorToAsync(stream.Raw, predictor, destination, cancellationToken) - .ConfigureAwait(false); + if (predictor is not null) + { + await InflatePredictorToAsync(stream.Raw, predictor, destination, cancellationToken) + .ConfigureAwait(false); + } + else + { + await InflateToAsync(stream.Raw, destination, cancellationToken).ConfigureAwait(false); + } } - else + catch (InvalidDataException exception) { - await InflateToAsync(stream.Raw, destination, cancellationToken).ConfigureAwait(false); + throw new PdfSourceFormatException( + sourceName, + $"the Flate data of the payload is damaged and could not be decompressed for copying: {exception.Message}"); } return; diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfObject.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfObject.cs index 7ae7d94..dfcee85 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfObject.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfObject.cs @@ -109,9 +109,29 @@ internal sealed class PdfReal : PdfObject /// The value. public PdfReal(double value) => Value = value; + /// Initializes a new instance of the class with the text a document wrote it as. + /// The value. + /// The bytes of the number as the document states it, or . + public PdfReal(double value, byte[]? original) + { + Value = value; + Original = original; + } + /// Gets the value. public double Value { get; } + /// + /// Gets the bytes the number was read from, when it was read from a document and they are a well-formed + /// PDF real; for a number the engine computed. + /// + /// + /// A merge or an edit copies a dictionary by writing it again, so a real it did not change is written from + /// these bytes: formatting the parsed value instead rounded it to five decimals, and a Type 3 font matrix of + /// 0.000012 came out as 0.00001. + /// + public byte[]? Original { get; } + /// public override PdfObjectKind Kind => PdfObjectKind.Number; @@ -119,7 +139,10 @@ internal sealed class PdfReal : PdfObject public override string ToString() => PdfNumberFormat.Format(Value); } -/// Formats numbers the way PDF expects them: invariant, shortest round-trippable form. +/// +/// Formats a number the engine computed the way PDF expects it: invariant, without an exponent, and to five +/// decimal places. A number read from a document is written from its own text instead (). +/// internal static class PdfNumberFormat { /// Formats a real number. diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfObjectCopier.Clone.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfObjectCopier.Clone.cs index 2693c65..d162fa7 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfObjectCopier.Clone.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfObjectCopier.Clone.cs @@ -13,7 +13,7 @@ internal sealed partial class PdfObjectCopier null or PdfNull => PdfNull.Instance, PdfBoolean boolean => PdfBoolean.From(boolean.Value), PdfInteger integer => new PdfInteger(integer.Value), - PdfReal real => new PdfReal(real.Value), + PdfReal real => new PdfReal(real.Value, real.Original), PdfName name => new PdfName(name.Value), PdfString text => new PdfString(text.Bytes, text.IsHexadecimal), PdfReference reference => new PdfReference(Reserve(reference.ObjectNumber), 0), diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfObjectParser.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfObjectParser.cs index c6e12a9..b7bffbf 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfObjectParser.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfObjectParser.cs @@ -128,7 +128,7 @@ public PdfObject ParseObject(int depth = 0) PdfTokenType.ArrayOpen => ParseArray(depth), PdfTokenType.DictionaryOpen => ParseDictionaryOrStream(depth), PdfTokenType.Number when token.IsInteger => ParseIntegerOrReference(token), - PdfTokenType.Number => new PdfReal(token.RealValue), + PdfTokenType.Number => new PdfReal(token.RealValue, OriginalReal(token)), PdfTokenType.Name => new PdfName(token.Text), PdfTokenType.LiteralString => new PdfString(DecryptString(token.Bytes ?? []), false), PdfTokenType.HexString => new PdfString(DecryptString(token.Bytes ?? []), true), @@ -137,6 +137,40 @@ public PdfObject ParseObject(int depth = 0) }; } + /// Keeps the bytes of a real when they are a well-formed PDF real, so a copy writes it back unchanged. + /// The number token. + /// The bytes, or for a form PDF does not define, which is written normalised. + /// + /// PDF defines a real as an optional sign, digits and at most one period (ISO 32000-1 §7.3.3); it has no + /// exponent. A token outside that form — an exponent, a second sign — was read leniently, and is written in + /// the form PDF defines rather than repeated. + /// + private byte[]? OriginalReal(PdfToken token) + { + var text = lexer.Data.AsSpan(token.Start, token.End - token.Start); + var body = text.Length > 0 && text[0] is (byte)'+' or (byte)'-' ? text[1..] : text; + var periods = 0; + var digits = 0; + + foreach (var value in body) + { + if (value == (byte)'.') + { + periods++; + } + else if (value is >= (byte)'0' and <= (byte)'9') + { + digits++; + } + else + { + return null; + } + } + + return digits > 0 && periods <= 1 && body.Length <= 64 ? text.ToArray() : null; + } + /// Decrypts the bytes of a string, when the parser was told how to. /// The bytes as they were stored. /// The bytes to expose. diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfOutputBuffer.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfOutputBuffer.cs index f284774..b79914c 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfOutputBuffer.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfOutputBuffer.cs @@ -80,9 +80,13 @@ public void WriteReal(double value) { Span scratch = stackalloc byte[32]; + // Thirty-two bytes hold every value a page states; a larger magnitude — which only a document can + // supply, PDF readers being limited to about 3.4e38 — takes the allocating path instead of failing + // the write of the whole document. if (!PdfNumberFormat.TryFormat(value, scratch, out var written)) { - throw new InvalidOperationException("The real number could not be formatted."); + Write(System.Text.Encoding.ASCII.GetBytes(PdfNumberFormat.Format(value))); + return; } Write(scratch[..written]); diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfSerializer.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfSerializer.cs index 5e974bf..efc3623 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfSerializer.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfSerializer.cs @@ -36,6 +36,10 @@ public static void Write( output.WriteInteger(integer.Value); break; + case PdfReal { Original: { } original }: + output.Write(original); + break; + case PdfReal real: output.WriteReal(real.Value); break; diff --git a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Annotations.cs b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Annotations.cs index c90b709..89b7d1c 100644 --- a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Annotations.cs +++ b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Annotations.cs @@ -318,6 +318,18 @@ private static PdfDictionary BuildAnnotationDictionary( dictionary.Set("Contents", PdfString.FromText(contents)); } + // A free text annotation must state its default appearance (ISO 32000-1, table 174): the font, size + // and colour a reader regenerating the appearance draws with, which are the ones the appearance stream + // this writer draws already uses. + if (annotation.Kind == PdfAnnotationKind.FreeText) + { + var style = annotation.TextStyle ?? PdfStandardTextStyle.Default; + + dictionary.Set("DA", PdfString.FromText(string.Create( + System.Globalization.CultureInfo.InvariantCulture, + $"/{AppearanceStreams.FontName} {PdfNumberFormat.Format(style.Size)} Tf {PdfNumberFormat.Format(style.Color.Red / 255.0)} {PdfNumberFormat.Format(style.Color.Green / 255.0)} {PdfNumberFormat.Format(style.Color.Blue / 255.0)} rg"))); + } + if (annotation.Color is { } color) { dictionary.Set("C", ColorArray(color)); diff --git a/src/SharpCorePDF.Infrastructure/PdfMergeEngine.Pages.cs b/src/SharpCorePDF.Infrastructure/PdfMergeEngine.Pages.cs index a09aa60..9af4d3c 100644 --- a/src/SharpCorePDF.Infrastructure/PdfMergeEngine.Pages.cs +++ b/src/SharpCorePDF.Infrastructure/PdfMergeEngine.Pages.cs @@ -38,7 +38,7 @@ private static void PreparePages(PdfMergeContext context, MergeSource source) source.PagesInOrder.Add(target); source.Copier.Override( page.ObjectNumber, - body => BuildMergedPage(context, source, body, page.Inherited)); + body => BuildMergedPage(context, source, body, page)); } if (source.PagesInOrder.Count == 0) @@ -66,14 +66,15 @@ private static void PreparePages(PdfMergeContext context, MergeSource source) /// The merge context. /// The source the page came from. /// The copied page dictionary. - /// The attributes the page inherited in its own document. + /// The page as its own document states it, with the attributes it inherited there. /// The page dictionary of the merged document. private static PdfDictionary BuildMergedPage( PdfMergeContext context, MergeSource source, PdfObject body, - PdfDictionary inherited) + SourcePage sourcePage) { + var inherited = sourcePage.Inherited; var copy = body as PdfDictionary ?? new PdfDictionary(); var page = new PdfDictionary(); @@ -92,6 +93,25 @@ private static PdfDictionary BuildMergedPage( page.Set("Type", new PdfName("Page")); page.Set("Parent", new PdfReference(context.PagesNodeNumber, 0)); + // Every page must have a media box (ISO 32000-1, table 30), on itself or inherited. A source page that + // has neither is described as A4 by the inspection, and is written with that box, so the result is a + // valid document whose page is the size the library reported instead of whatever each reader assumes. + if (Pdf.PdfDocumentInspection.TryReadPageSize(source.Reader, sourcePage) is null) + { + var assumed = Domain.ValueObjects.PdfPageSize.A4; + + page.Set("MediaBox", new PdfArray( + [ + new PdfInteger(0), + new PdfInteger(0), + new PdfReal(assumed.WidthPoints), + new PdfReal(assumed.HeightPoints), + ])); + context.Warnings.Add(new( + Domain.ValueObjects.PdfWarningKind.PageSizeAssumed, + $"'{source.DisplayName}': page {sourcePage.Index + 1} has no readable /MediaBox; it was written with the A4 box it is described with, because every page of the result needs one.")); + } + if (!context.Output.PreserveAnnotations) { page.Remove("Annots"); diff --git a/tests/SharpCorePDF.Tests/PdfAnnotationTests.cs b/tests/SharpCorePDF.Tests/PdfAnnotationTests.cs index 6da205b..fc0ae3e 100644 --- a/tests/SharpCorePDF.Tests/PdfAnnotationTests.cs +++ b/tests/SharpCorePDF.Tests/PdfAnnotationTests.cs @@ -240,6 +240,56 @@ public async Task Draws_the_text_of_a_free_text_annotation_into_its_appearance() Assert.Contains(" Tf", operators, StringComparison.Ordinal); } + /// + /// A free text annotation states the default appearance ISO 32000-1 requires of it — the font, the size and + /// the colour its appearance stream draws with — and that stream states resources of its own, as a form + /// XObject must under the Arlington model; neither was written before. + /// + [Fact] + public async Task States_the_default_appearance_and_resources_of_a_free_text_annotation() + { + var edited = await Editor.ApplyToArrayAsync( + new PdfByteArraySource(ThreePages(), "three-pages.pdf"), + PdfEditPlan.Create().AddAnnotation( + 1, + PdfAnnotations.FreeText( + new PdfRectangle(72, 500, 320, 530), + "Reviewed", + new PdfStandardTextStyle(StandardFont.Helvetica, 10, PdfColor.FromRgb(255, 0, 0)))), + cancellationToken: Token); + + var reader = Read(edited); + var annots = reader.ResolveArray(reader.Pages[0].Dictionary.Get("Annots")); + var annotation = reader.ResolveDictionary(annots?.Items[0]); + var appearance = Assert.IsType(reader.Resolve(reader.ResolveDictionary(annotation?.Get("AP"))?.Get("N"))); + var defaultAppearance = Assert.IsType(annotation?.Get("DA")); + + Assert.Equal("/Helv 10 Tf 1 0 0 rg", Encoding.ASCII.GetString(defaultAppearance.Bytes)); + Assert.IsType(reader.Resolve(appearance.Dictionary.Get("Resources"))); + } + + /// + /// An appearance that needs no font and no graphics state still states its resources, empty, so a reader + /// never borrows the page's. + /// + [Fact] + public async Task States_resources_on_an_appearance_that_needs_none() + { + var edited = await Editor.ApplyToArrayAsync( + new PdfByteArraySource(ThreePages(), "three-pages.pdf"), + PdfEditPlan.Create().AddAnnotation( + 1, + PdfAnnotations.Highlight([new PdfRectangle(72, 300, 200, 312)], PdfColor.FromRgb(255, 224, 130))), + cancellationToken: Token); + + var reader = Read(edited); + var annots = reader.ResolveArray(reader.Pages[0].Dictionary.Get("Annots")); + var annotation = reader.ResolveDictionary(annots?.Items[0]); + var appearance = Assert.IsType(reader.Resolve(reader.ResolveDictionary(annotation?.Get("AP"))?.Get("N"))); + + Assert.IsType(reader.Resolve(appearance.Dictionary.Get("Resources"))); + } + /// An annotation that states its own appearance is drawn with those operators and no others. [Fact] public async Task Writes_the_operators_of_a_supplied_annotation_appearance() diff --git a/tests/SharpCorePDF.Tests/PdfContentReaderTests.Images.cs b/tests/SharpCorePDF.Tests/PdfContentReaderTests.Images.cs index d4daed0..6388014 100644 --- a/tests/SharpCorePDF.Tests/PdfContentReaderTests.Images.cs +++ b/tests/SharpCorePDF.Tests/PdfContentReaderTests.Images.cs @@ -14,6 +14,36 @@ namespace SharpCorePDF.Tests; /// public sealed partial class PdfContentReaderTests { + /// + /// The copy route of an image whose Flate data is damaged refuses it with the contract's exception, whether a + /// predictor is declared or not: the decompressor's own exception used to reach the caller as if the library + /// had failed. + /// + /// The decode parameters the image declares. + [Theory] + [InlineData("/DecodeParms << /Predictor 12 /Colors 1 /BitsPerComponent 8 /Columns 4 >>")] + [InlineData("")] + public async Task Refuses_to_copy_an_image_whose_flate_data_is_damaged(string parameters) + { + var builder = new PdfBuilder(); + var catalog = builder.Reserve(); + var pages = builder.Reserve(); + var damaged = new byte[] { 0x78, 0x9C, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05 }; + + builder.Add("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 100 100] /Resources << /XObject << /Im0 5 0 R >> >> /Contents 4 0 R >>"); + builder.AddStream(string.Empty, "q 50 0 0 50 0 0 cm /Im0 Do Q"u8.ToArray()); + builder.AddStream($"/Type /XObject /Subtype /Image /Width 4 /Height 4 /BitsPerComponent 8 /ColorSpace /DeviceGray /Filter /FlateDecode {parameters}", damaged); + builder.Set(catalog, "<< /Type /Catalog /Pages 2 0 R >>"); + builder.Set(pages, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + + var source = Source(builder.Build(catalog)); + var reader = Reader(); + var image = Assert.Single((await reader.ReadImagesAsync(source, PageRange.All, cancellationToken: Token)).Items); + + await Assert.ThrowsAsync( + () => reader.CopyImageDataAsync(source, image, Stream.Null, cancellationToken: Token)); + } + /// One object drawn twice and shared by two pages is one entry carrying both pages. [Fact] public async Task Returns_each_image_once_with_every_page_that_draws_it() diff --git a/tests/SharpCorePDF.Tests/PdfMergeFidelityTests.cs b/tests/SharpCorePDF.Tests/PdfMergeFidelityTests.cs new file mode 100644 index 0000000..710d4c2 --- /dev/null +++ b/tests/SharpCorePDF.Tests/PdfMergeFidelityTests.cs @@ -0,0 +1,113 @@ +using System.Text; +using SharpCorePDF.Application.Extensions; +using SharpCorePDF.Application.Services; +using SharpCorePDF.Domain.Aggregates; +using SharpCorePDF.Domain.Entities; +using SharpCorePDF.Domain.ValueObjects; +using SharpCorePDF.Infrastructure; +using SharpCorePDF.Infrastructure.Pdf; +using SharpCorePDF.Infrastructure.Sources; + +namespace SharpCorePDF.Tests; + +/// +/// Tests of what a merge writes for values it does not change: the numbers a source states come back as the +/// source states them, and a page the source leaves without a size comes back with the size the library +/// describes it with. +/// +public sealed class PdfMergeFidelityTests +{ + private static CancellationToken Token => TestContext.Current.CancellationToken; + + /// + /// A real a merge copies is written from the text the source used, not from a value formatted to five + /// decimals: a Type 3 font matrix of 0.000012 used to come out as 0.00001, seventeen percent + /// off, and 0.000004 as 0. + /// + [Fact] + public async Task Writes_the_reals_it_copies_as_the_source_states_them() + { + var merged = await MergeAsync(Page("/Scale [0.000012 -0.00048828125 0.000004 12.5 +.5]")); + var text = Encoding.Latin1.GetString(merged); + + Assert.Contains("[0.000012 -0.00048828125 0.000004 12.5 +.5]", text, StringComparison.Ordinal); + } + + /// + /// A real outside the form PDF defines — here with an exponent, which PDF does not have — is written in the + /// form PDF defines, whatever its magnitude, instead of failing the write of the whole document. + /// + [Fact] + public async Task Writes_a_real_of_any_magnitude_in_the_form_pdf_defines() + { + var merged = await MergeAsync(Page("/Huge 1e300 /Small 2.5e-3")); + var text = Encoding.Latin1.GetString(merged); + + Assert.Contains("/Huge 1" + new string('0', 300), text, StringComparison.Ordinal); + Assert.Contains("/Small 0.0025", text, StringComparison.Ordinal); + } + + /// + /// A page that states no media box and inherits none is written with the A4 box the inspection describes it + /// with, because every page of a PDF needs one; the merge says so with . + /// + [Fact] + public async Task Writes_a_media_box_for_a_page_that_has_none() + { + var builder = new PdfBuilder(); + var catalog = builder.Reserve(); + var pages = builder.Reserve(); + + builder.Add("<< /Type /Page /Parent 2 0 R >>"); + builder.Set(catalog, "<< /Type /Catalog /Pages 2 0 R >>"); + builder.Set(pages, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + + using var destination = new MemoryStream(); + var result = await Merger().MergeAsync( + MergePlan.Create().AddSource(PdfSource.Create(new PdfByteArraySource(builder.Build(catalog), "sizeless.pdf"))), + destination, + Token); + + var reader = new PdfFileReader(destination.ToArray(), "merged.pdf", PdfEngineOptions.Default); + var box = Assert.IsType(reader.Pages[0].Dictionary.Get("MediaBox")); + + Assert.Equal(4, box.Count); + Assert.Equal(PdfPageSize.A4.WidthPoints, ((PdfReal)box.Items[2]).Value, 3); + Assert.Equal(PdfPageSize.A4.HeightPoints, ((PdfReal)box.Items[3]).Value, 3); + Assert.Contains(result.Warnings, warning => warning.Kind == PdfWarningKind.PageSizeAssumed); + } + + /// A page that states its own media box keeps it, and no size is assumed for it. + [Fact] + public async Task Keeps_the_media_box_a_page_states() + { + using var destination = new MemoryStream(); + var result = await Merger().MergeAsync( + MergePlan.Create().AddSource(PdfSource.Create(new PdfByteArraySource(Page(string.Empty), "sized.pdf"))), + destination, + Token); + + Assert.Contains("/MediaBox [0 0 200 300]", Encoding.Latin1.GetString(destination.ToArray()), StringComparison.Ordinal); + Assert.DoesNotContain(result.Warnings, warning => warning.Kind == PdfWarningKind.PageSizeAssumed); + } + + private static PdfMergerService Merger() => new(new PdfDocumentReader(), new PdfMergeEngine()); + + private static byte[] Page(string entries) + { + var builder = new PdfBuilder(); + var catalog = builder.Reserve(); + var pages = builder.Reserve(); + + builder.Add($"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 300] {entries} >>"); + builder.Set(catalog, "<< /Type /Catalog /Pages 2 0 R >>"); + builder.Set(pages, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + + return builder.Build(catalog); + } + + private static Task MergeAsync(byte[] document) => + Merger().MergeToBytesAsync( + MergePlan.Create().AddSource(PdfSource.Create(new PdfByteArraySource(document, "source.pdf"))), + Token); +} From 3a8b5784790b4177b9a78e2f8c301c83f5811799 Mon Sep 17 00:00:00 2001 From: Andrew Tierney Date: Mon, 16 Nov 2026 13:12:40 +1100 Subject: [PATCH 07/15] Rebuild moved tables, read object streams in recovery, and harden five edges Recovery - A cross-reference table that parses can still be wrong: bytes inserted or removed after it was written move every object. Reconstruction now also runs when the catalog's offset, or most of a sample of offsets, does not land on the n g obj header of its own object. - Reconstruction indexes the objects of every object stream the scan finds, and takes /Root from the dictionary of the last cross-reference stream, so a PDF 1.5 file whose cross-reference stream is lost reads instead of being refused with "no document catalog". The test that pinned the refusal now pins the rebuild. Hardening - AESV2 always uses a sixteen-byte key; a crypt filter that declares fewer made the file unopenable. - The whole-buffer predictor path sizes a row in 64 bits like the streaming decoder: a declared Columns x Colors that wrapped to zero looped forever under TIFF prediction, and to a negative number threw ArgumentOutOfRangeException. - A stream with a wrong /Length is cut where its zlib data is complete, by its Adler-32, instead of at the first endstream whose truncated bytes inflate; at most sixteen candidates are inflated. - A reference to a generation the table does not list resolves to null (ISO 32000-1, 7.3.10). - A revision appended to a file whose newest section is a cross-reference stream, by an edit or a signature, ends in a cross-reference stream instead of a classic table. qpdf and pyHanko accept an edit and two signatures appended this way. Corpus fault - The public-corpora job faulted on isartor-6-1-11-t01-fail-a.pdf: a file specification that only names an external file is listed as an attachment with ByteLength 0, and copying it threw ArgumentException. It now writes no bytes; an id the document does not have is still refused. Each change has a test that fails without it. Full suite passes on net11.0 and net10.0; the pdf.js corpus run reports no fault, crash or timeout. --- CHANGELOG.md | 20 +++ docs/validation.md | 4 + .../Ports/IPdfContentReader.cs | 8 +- .../Pdf/PdfCrossReference.cs | 6 + .../Pdf/PdfCrossReferenceReader.Scan.cs | 125 ++++++++++++++++- .../Pdf/PdfCrossReferenceReader.cs | 68 ++++++++- .../Pdf/PdfCrossReferenceStreamWriter.cs | 128 +++++++++++++++++ .../Pdf/PdfFileReader.cs | 25 +++- .../Pdf/PdfFilters.Inflation.cs | 37 +++++ .../Pdf/PdfFilters.Predictors.cs | 49 ++++--- .../Pdf/PdfIncrementalWriter.cs | 15 +- .../Pdf/PdfObjectParser.Streams.cs | 50 ++++++- .../Pdf/PdfSecurityHandler.cs | 5 +- .../Pdf/PdfWriter.cs | 16 ++- .../PdfEditEngine.Document.cs | 6 +- .../PdfEditEngine.cs | 3 +- .../PdfPageContentReader.Metadata.cs | 24 +++- .../Signatures/SignatureFieldWriter.cs | 2 +- .../PdfContentReaderTests.Metadata.cs | 36 +++++ .../PdfDocumentSignerTests.cs | 20 +++ .../PdfIncrementalWriterTests.cs | 38 ++++++ .../PdfMergeEngineTests.OutputEncryption.cs | 22 +++ tests/SharpCorePDF.Tests/PdfPredictorTests.cs | 24 ++++ .../PdfReaderRecoveryTests.cs | 129 ++++++++++++++++-- .../PdfStreamParsingTests.cs | 29 ++++ 25 files changed, 831 insertions(+), 58 deletions(-) create mode 100644 src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceStreamWriter.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 7aa74cc..29cdcc1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -85,6 +85,26 @@ against an empty feed. decompressor's `InvalidDataException` reach the caller (found by the `flate` fuzz target). - **Annotations satisfy the Arlington model.** A free text annotation states the `/DA` ISO 32000-1 requires, and every annotation appearance states its `/Resources` (found by the Arlington check in its first run). +- **A cross-reference table whose offsets moved is rebuilt.** Bytes inserted or removed after a table was + written — a converted line ending, a prepended byte order mark — leave a table that parses and offsets that + land inside other objects. Reconstruction used to run only when no table or no catalog was found; it now + also runs when the catalog's offset, or most of a sample of offsets, does not land on its own object. +- **Reconstruction looks inside object streams.** A PDF 1.5 file whose cross-reference stream is lost keeps + its catalog and pages in object streams, which a scan for object headers cannot see; it was refused with + "no document catalog". The objects of every object stream the scan finds are now indexed, and `/Root` is + taken from the dictionary of the last cross-reference stream. +- **Hardening.** AESV2 always uses a sixteen-byte key, as AES-128 requires, so a file that declares a shorter + one opens. A predicted row is sized in 64 bits on the whole-buffer path too: declared columns and colours + whose product wrapped to zero looped forever, and to a negative number threw `ArgumentOutOfRangeException`. + A stream whose length is wrong is cut where its compressed data is complete — where its Adler-32 matches — + rather than at the first `endstream` whose bytes inflate, and at most sixteen candidates are inflated. A + reference to a generation the table does not list resolves to null, as ISO 32000-1 reads it. +- **A revision appended to a file that uses cross-reference streams ends in one.** Edits and signatures + appended a classic table chained to a stream; they now write the form the document uses. +- **Copying a listed attachment that embeds nothing writes nothing.** A file specification in the embedded + files tree that only names an external file is listed, with a warning and a length of zero; copying it + threw `ArgumentException` as if the caller had passed a file of another document (found by the veraPDF + corpus run, `isartor-6-1-11-t01-fail-a.pdf`). - **Run-length images are decoded.** `RunLengthDecode` joins Flate, LZW and the two ASCII filters: a bilevel or scan-style image that declares it now yields its samples — and `Gray8`, `Rgb24` or `Cmyk32` diff --git a/docs/validation.md b/docs/validation.md index a06e569..3491b07 100644 --- a/docs/validation.md +++ b/docs/validation.md @@ -102,6 +102,10 @@ Fixed since the list was first drawn up, each with the test that fails without i | Reals a merge or an edit copied were re-formatted to five decimals (`0.000012` became `0.00001`), and a real too large for the formatter's buffer failed the write of the whole document — the first crash the `document` fuzz target found | a real read from a document is written from its own text when that text is a well-formed PDF real; other values are formatted at any magnitude | `PdfMergeFidelityTests.Writes_the_reals_it_copies_as_the_source_states_them`, `Writes_a_real_of_any_magnitude_in_the_form_pdf_defines` | | The copy route of an image let the decompressor's `InvalidDataException` escape for damaged Flate data — the crash the `flate` fuzz target found | the copy route refuses damaged data with `PdfSourceFormatException`, with or without a predictor | `PdfContentReaderTests.Refuses_to_copy_an_image_whose_flate_data_is_damaged` | | A free text annotation was written without the `/DA` ISO 32000-1 requires, and annotation appearances without `/Resources` — both reported by the Arlington check's first run | `/DA` states the font, size and colour the appearance draws with; every appearance states its resources, empty if it needs none | `PdfDocumentEditorTests.States_the_default_appearance_and_resources_of_a_free_text_annotation`, `States_resources_on_an_appearance_that_needs_none` | +| A table that parsed but whose offsets had moved was used as it was, and reconstruction could not see into object streams | the table is rebuilt when its offsets miss their objects; reconstruction indexes object streams and takes `/Root` from a cross-reference stream's dictionary | `PdfReaderRecoveryTests.Rebuilds_a_table_whose_offsets_no_longer_point_at_their_objects`, `Rebuilds_a_table_from_object_streams_when_the_cross_reference_stream_is_lost`, `Rebuilds_a_document_whose_catalog_lives_in_an_object_stream` | +| AESV2 with a short declared key, wrapped predictor row sizes, truncated `endstream` candidates, ignored generations | sixteen-byte AESV2 keys; 64-bit row sizes on both predictor paths; Adler-32-checked candidates, at most sixteen; a mismatched generation resolves to null | `PdfMergeEngineTests.Opens_an_aesv2_document_whose_crypt_filter_declares_a_short_key`, `PdfPredictorTests.Refuses_a_predicted_row_larger_than_the_engine_decodes`, `PdfStreamParsingTests.Scans_past_a_candidate_whose_compressed_data_is_cut_short`, `PdfReaderRecoveryTests.Resolves_a_reference_to_a_generation_the_table_does_not_list_as_null` | +| A revision appended to a file that uses cross-reference streams ended in a classic table | the revision ends in a cross-reference stream; qpdf and pyHanko accept both an edit and two signatures appended this way | `PdfIncrementalWriterTests.Appends_a_cross_reference_stream_to_a_document_that_uses_them`, `PdfDocumentSignerTests.Signs_a_document_that_uses_cross_reference_streams_with_one` | +| Copying an attachment the listing reported, whose specification embeds no stream, threw `ArgumentException` — the public-corpora job's fault on `isartor-6-1-11-t01-fail-a.pdf` | it writes no bytes, matching the `ByteLength` of zero the listing reports | `PdfContentReaderTests.Copies_nothing_for_a_listed_file_that_embeds_no_stream` | ## Corpus readings, 2026-11-16 diff --git a/src/SharpCorePDF.Domain/Ports/IPdfContentReader.cs b/src/SharpCorePDF.Domain/Ports/IPdfContentReader.cs index 3d0f975..d0a273a 100644 --- a/src/SharpCorePDF.Domain/Ports/IPdfContentReader.cs +++ b/src/SharpCorePDF.Domain/Ports/IPdfContentReader.cs @@ -190,10 +190,14 @@ Task> ReadEmbeddedFilesAsync( /// A task that completes when the bytes have been written. /// /// The bounded route, with the same contract as : a pooled scratch - /// buffer, the destination flushed and left open. + /// buffer, the destination flushed and left open. A file whose specification embeds no stream — one that + /// only names a file outside the document, listed with a warning and a ByteLength of zero — writes + /// no bytes. /// /// An argument is . - /// is not writable. + /// + /// is not writable, or is not a file of the document. + /// /// The content is not a readable PDF document. Task CopyEmbeddedFileDataAsync( IPdfContentSource source, diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReference.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReference.cs index 9230afb..e1067bf 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReference.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReference.cs @@ -48,6 +48,12 @@ internal sealed class PdfCrossReferenceTable /// Gets a value indicating whether the table was rebuilt by scanning the file. public bool WasReconstructed { get; internal set; } + /// + /// Gets a value indicating whether the section startxref points at is a cross-reference stream, so + /// a revision appended to the document writes its own section in that form. + /// + public bool NewestSectionIsStream { get; internal set; } + /// Gets the number of entries. public int Count => Entries.Count; diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.Scan.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.Scan.cs index f923423..d0218ed 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.Scan.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.Scan.cs @@ -25,6 +25,7 @@ private void Reconstruct() } var found = ScanForObjects(); + IndexObjectStreams(); var trailer = FindLastTrailer(); if (trailer is not null) @@ -136,7 +137,129 @@ private bool TryReadBackwardsInteger(ref int cursor, out long value) index = index == 0 ? -1 : data.AsSpan(0, index).LastIndexOf("trailer"u8); } - return FindCatalogTrailer(); + return FindCrossReferenceStreamTrailer() ?? FindCatalogTrailer(); + } + + /// + /// Registers the objects the object streams found by the scan hold. A PDF 1.5 file keeps most of its + /// objects — often its catalog and its pages — inside object streams, which a scan for n g obj + /// headers cannot see into, so without this a file whose cross-reference stream is lost has nothing left. + /// + /// + /// An object the scan found on its own stays where the scan found it: entries are only added, never + /// replaced, because an object written uncompressed after an object stream is the newer of the two. + /// + private void IndexObjectStreams() + { + foreach (var container in table.Entries.Values.Where(entry => entry.Type == XrefEntryType.Uncompressed).ToList()) + { + if (!MentionsObjectStream(container.Offset) + || ParseAt(container.Offset) is not PdfStream { Dictionary: var dictionary } stream + || dictionary.GetName("Type") != "ObjStm") + { + continue; + } + + byte[] decoded; + + try + { + decoded = PdfFilters.Decode(stream, sourceName, options.MaxDecodedStreamBytes); + } + catch (PdfException) + { + continue; + } + + var count = Math.Min(dictionary.GetInteger("N") ?? 0, (decoded.Length / 4L) + 1); + var header = new PdfObjectParser(decoded, sourceName, 0, null, options.MaxNestingDepth); + + for (var index = 0; index < count; index++) + { + if (!header.TryReadInteger(out var objectNumber) || !header.TryReadInteger(out _)) + { + break; + } + + if (objectNumber > 0) + { + table.Add(new XrefEntry(objectNumber, container.ObjectNumber, index, XrefEntryType.Compressed)); + } + } + } + } + + /// Determines cheaply whether the object at an offset names itself an object stream. + /// The offset of the object header. + /// when /ObjStm appears in the first bytes of the object. + private bool MentionsObjectStream(long offset) + { + if (offset < 0 || offset >= data.Length) + { + return false; + } + + var window = data.AsSpan((int)offset, (int)Math.Min(512, data.Length - offset)); + + return window.IndexOf("/ObjStm"u8) >= 0; + } + + /// Parses the object whose header is at an offset. + /// The offset. + /// The object, or . + private PdfObject? ParseAt(long offset) + { + var parser = new PdfObjectParser(data, sourceName, (int)offset, null, options.MaxNestingDepth); + + return parser.TryReadIndirectHeader(out _, out _) ? parser.ParseObject() : null; + } + + /// + /// Takes the trailer entries from the last cross-reference stream the scan found: a PDF 1.5 file has no + /// trailer keyword, and its /Root lives in the stream's dictionary, which survives when the + /// stream's data does not. + /// + /// A trailer carrying /Root, or . + private PdfDictionary? FindCrossReferenceStreamTrailer() + { + foreach (var offset in table.Entries.Values + .Where(entry => entry.Type == XrefEntryType.Uncompressed) + .Select(entry => entry.Offset) + .OrderByDescending(offset => offset)) + { + var window = data.AsSpan((int)offset, (int)Math.Min(512, data.Length - offset)); + + if (window.IndexOf("/XRef"u8) < 0) + { + continue; + } + + var dictionary = ParseAt(offset) switch + { + PdfStream stream => stream.Dictionary, + PdfDictionary plain => plain, + _ => null, + }; + + if (dictionary is null || dictionary.GetName("Type") != "XRef" || dictionary.GetReference("Root") is null) + { + continue; + } + + var trailer = new PdfDictionary(); + + foreach (var key in new[] { "Root", "Info", "ID", "Encrypt" }) + { + if (dictionary.Get(key) is { } value) + { + trailer.Set(key, value); + } + } + + return trailer; + } + + return null; } /// Looks for a document catalog and uses it as the root of a synthetic trailer. diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.cs index 9a591d2..7d04ddd 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.cs @@ -66,7 +66,12 @@ public PdfCrossReferenceTable Read() } } - if (table.Count == 0 || !table.Trailer.ContainsKey("Root")) + // A table that reads cleanly can still be wrong: bytes inserted or removed after it was written — a + // converted line ending, a prepended byte order mark — move every object, and each offset then lands + // in the middle of something else. Such a table is rebuilt like a table that did not parse. + if (table.Count == 0 + || !table.Trailer.ContainsKey("Root") + || (options.ReconstructCrossReferenceOnFailure && !OffsetsLandOnObjects())) { Reconstruct(); } @@ -81,6 +86,56 @@ public PdfCrossReferenceTable Read() return table; } + /// + /// Determines whether the table's offsets land on the objects they name: the catalog's, and a sample of + /// the others, each has to point at the n g obj header of its own number. + /// + /// when the catalog's offset misses, or most of the sample does. + private bool OffsetsLandOnObjects() + { + const int Sample = 32; + + if (table.Trailer.GetReference("Root") is { } root + && table.Get(root.ObjectNumber) is { Type: XrefEntryType.Uncompressed } rootEntry + && !HeaderNames(rootEntry)) + { + return false; + } + + var sampled = 0; + var missed = 0; + + foreach (var entry in table.Entries.Values + .Where(entry => entry.Type == XrefEntryType.Uncompressed) + .OrderBy(entry => entry.ObjectNumber) + .Take(Sample)) + { + sampled++; + + if (!HeaderNames(entry)) + { + missed++; + } + } + + return missed * 2 <= sampled; + } + + /// Determines whether an entry's offset is the header of the object it names. + /// The entry. + /// when the bytes at the offset are n g obj for its own number. + private bool HeaderNames(XrefEntry entry) + { + if (entry.Offset < 0 || entry.Offset >= data.Length) + { + return false; + } + + var parser = new PdfObjectParser(data, sourceName, (int)entry.Offset, null, options.MaxNestingDepth); + + return parser.TryReadIndirectHeader(out var objectNumber, out _) && objectNumber == entry.ObjectNumber; + } + private long? FindStartXref() { var searchStart = Math.Max(0, data.Length - 4096); @@ -106,8 +161,13 @@ public PdfCrossReferenceTable Read() { var probe = new PdfLexer(data, (int)offset); - return probe.TryReadKeyword("xref") - ? ReadClassicTable((int)offset, isNewest) - : ReadCrossReferenceStream((int)offset, isNewest); + if (probe.TryReadKeyword("xref")) + { + return ReadClassicTable((int)offset, isNewest); + } + + table.NewestSectionIsStream |= isNewest; + + return ReadCrossReferenceStream((int)offset, isNewest); } } diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceStreamWriter.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceStreamWriter.cs new file mode 100644 index 0000000..c9f086c --- /dev/null +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceStreamWriter.cs @@ -0,0 +1,128 @@ +namespace SharpCorePDF.Infrastructure.Pdf; + +/// +/// Writes the cross-reference section of an appended revision as a cross-reference stream, for a document +/// whose newest section is one. +/// +/// +/// A revision appended to a file that uses cross-reference streams used to end in a classic table whose +/// /Prev pointed at a stream. Most readers follow such a chain, but ISO 32000-1 only describes a +/// classic table chained to a classic table, a stream to a stream, and the hybrid form of 7.5.8.4, so the +/// revision is written in the form the document already uses. The stream is written uncompressed and in the +/// clear: a cross-reference stream is never encrypted, and a revision holds a few entries at most. +/// +internal static class PdfCrossReferenceStreamWriter +{ + /// The width of the generation field, which holds up to 65535. + private const int GenerationWidth = 2; + + /// Writes the stream, the startxref that points at it and the closing %%EOF. + /// The buffer the revision is written to. + /// The absolute offset the stream object is written at. + /// The objects of the revision: number, generation and absolute offset. + /// The trailer entries, which become entries of the stream's dictionary; its Size is corrected here. + /// Whether the section lists the head of the free list, object zero, as a classic table of a complete file does. + public static void Write( + PdfOutputBuffer output, + long startXref, + IReadOnlyCollection<(int Number, int Generation, long Offset)> entries, + PdfDictionary trailer, + bool freeHead) + { + ArgumentNullException.ThrowIfNull(output); + ArgumentNullException.ThrowIfNull(entries); + ArgumentNullException.ThrowIfNull(trailer); + + var size = (int)Math.Max(trailer.GetInteger("Size") ?? 0, 1); + + foreach (var entry in entries) + { + size = Math.Max(size, entry.Number + 1); + } + + // The stream is an object of the revision too, numbered above every other, and lists itself. + var streamNumber = size; + var rows = entries + .Select(entry => (entry.Number, Type: (byte)1, Field2: entry.Offset, Field3: entry.Generation)) + .Append((Number: streamNumber, Type: (byte)1, Field2: startXref, Field3: 0)); + + if (freeHead) + { + rows = rows.Append((Number: 0, Type: (byte)0, Field2: 0L, Field3: 65535)); + } + + var sorted = rows.OrderBy(row => row.Number).ToArray(); + var offsetWidth = BytesFor(startXref); + var data = new byte[sorted.Length * (1 + offsetWidth + GenerationWidth)]; + var index = new List(); + var position = 0; + + for (var row = 0; row < sorted.Length; row++) + { + if (row == 0 || sorted[row].Number != sorted[row - 1].Number + 1) + { + index.Add(new PdfInteger(sorted[row].Number)); + index.Add(new PdfInteger(0)); + } + + var count = (PdfInteger)index[^1]; + index[^1] = new PdfInteger(count.Value + 1); + + data[position++] = sorted[row].Type; + WriteBigEndian(data.AsSpan(position, offsetWidth), sorted[row].Field2); + position += offsetWidth; + WriteBigEndian(data.AsSpan(position, GenerationWidth), sorted[row].Field3); + position += GenerationWidth; + } + + var dictionary = new PdfDictionary().SetName("Type", "XRef"); + + foreach (var entry in trailer.Entries.Where(entry => entry.Key is not ("Size" or "XRefStm"))) + { + dictionary.Set(entry.Key, entry.Value); + } + + dictionary + .SetInteger("Size", streamNumber + 1) + .Set("W", new PdfArray([new PdfInteger(1), new PdfInteger(offsetWidth), new PdfInteger(GenerationWidth)])) + .Set("Index", new PdfArray(index)) + .SetInteger("Length", data.Length); + + output.WriteInteger(streamNumber); + output.Write(" 0 obj\n"u8); + PdfSerializer.WriteDictionary(dictionary, output); + output.Write("\nstream\n"u8); + output.Write(data); + output.Write("\nendstream\nendobj\nstartxref\n"u8); + output.WriteInteger(startXref); + output.Write("\n%%EOF\n"u8); + output.Flush(); + } + + /// Gets the number of bytes a big-endian field needs to hold a value. + /// The largest value of the field. + /// The width, at least one byte. + private static int BytesFor(long value) + { + var width = 1; + + while (width < 8 && value >> (8 * width) != 0) + { + width++; + } + + return width; + } + + /// Writes a value into a field, most significant byte first. + /// The field. + /// The value, which fits the field. + private static void WriteBigEndian(Span field, long value) + { + for (var index = field.Length - 1; index >= 0; index--) + { + field[index] = (byte)value; + value >>= 8; + } + } +} diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.cs index fcec760..11ee20e 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.cs @@ -187,8 +187,29 @@ public PdfDictionary? Catalog /// Resolves an indirect reference. /// The reference. - /// The object, or . - public PdfObject? Resolve(PdfReference? reference) => reference is null ? null : Resolve(reference.ObjectNumber); + /// + /// The object, or — also for a generation other than the one the table lists for + /// the number, which names an object that no longer exists (ISO 32000-1, 7.3.10). + /// + /// + /// Only an entry of the table states a generation: an object in an object stream has generation zero by + /// definition, and the entry of a compressed object holds its index there instead, so it is not compared. + /// + public PdfObject? Resolve(PdfReference? reference) + { + if (reference is null) + { + return null; + } + + if (CrossReferenceTable.Get(reference.ObjectNumber) is { Type: XrefEntryType.Uncompressed } entry + && entry.Generation != reference.Generation) + { + return null; + } + + return Resolve(reference.ObjectNumber); + } /// Resolves a value that may be a reference, leaving anything else as it is. /// The value. diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.Inflation.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.Inflation.cs index 8fe69f2..7672c78 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.Inflation.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.Inflation.cs @@ -38,6 +38,43 @@ private enum InflateOutcome public static bool TryInflate(ReadOnlyMemory data, out byte[] result) => TryInflate(data, DefaultMaxDecodedBytes, out result) == InflateOutcome.Inflated; + /// + /// Determines whether zlib data ends with the Adler-32 checksum of what it inflated to, which only data + /// that is complete does: a deflate stream cut short inflates without complaint to the bytes before the + /// cut, and the four bytes in front of the cut are then not its checksum. + /// + /// The zlib data, possibly followed by the end of line a writer put behind it. + /// The bytes it inflated to. + /// when the checksum is found at the end of the data. + internal static bool EndsWithAdler32Of(ReadOnlySpan compressed, ReadOnlySpan inflated) + { + const uint Modulus = 65521; + + uint low = 1; + uint high = 0; + + // 5552 bytes is the longest run the two sums can take before they have to be reduced to stay in 32 bits. + while (!inflated.IsEmpty) + { + var run = inflated[..Math.Min(5552, inflated.Length)]; + + foreach (var value in run) + { + low += value; + high += low; + } + + low %= Modulus; + high %= Modulus; + inflated = inflated[run.Length..]; + } + + Span checksum = [(byte)(high >> 8), (byte)high, (byte)(low >> 8), (byte)low]; + var trimmed = compressed.TrimEnd("\r\n \t\0"u8); + + return trimmed.EndsWith(checksum); + } + /// Decompresses Flate data under the default limit on its decoded size. /// The compressed bytes. /// The name of the source, used in diagnostics. diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.Predictors.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.Predictors.cs index 859db4d..eb6ef64 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.Predictors.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfFilters.Predictors.cs @@ -33,15 +33,16 @@ private static byte[] ApplyPredictor( ValidatePredictor(predictor, bitsPerComponent, sourceName); + var rowLength = PredictedRowLength(columns, colors, bitsPerComponent, sourceName); + if (predictor == 2) { - return ApplyTiffPredictor(data, colors, columns); + return ApplyTiffPredictor(data, colors, rowLength); } var bytesPerPixel = Math.Max(1, colors * bitsPerComponent / 8); - var rowLength = (columns * colors * bitsPerComponent + 7) / 8; - if (rowLength <= 0 || data.Length <= rowLength) + if (data.Length <= rowLength) { return data; } @@ -192,10 +193,8 @@ private static void UndoPaeth(Span current, ReadOnlySpan previous, i } } - private static byte[] ApplyTiffPredictor(byte[] data, int colors, int columns) + private static byte[] ApplyTiffPredictor(byte[] data, int colors, int rowLength) { - var rowLength = columns * colors; - for (var row = 0; row + rowLength <= data.Length; row += rowLength) { UndoTiffRow(data.AsSpan(row, rowLength), colors); @@ -292,6 +291,32 @@ private static void ValidatePredictor(int predictor, int bitsPerComponent, strin /// private const int MaximumPredictedRowBytes = 8 * 1024 * 1024; + /// Gets the length of a predicted row, refusing one the engine does not decode. + /// The samples in a row, at least one. + /// The components of a pixel, at least one. + /// The bits of each sample. + /// The name of the source, used in diagnostics. + /// The bytes of a row, without the filter type byte PNG prediction adds. + /// The row is empty, negative or larger than the engine decodes. + /// + /// The product is taken in 64 bits: in 32 the three declared values wrap, and a row of 65536 columns of + /// 65536 colours came out as zero bytes, on which a row loop never advances. + /// + private static int PredictedRowLength(int columns, int colors, int bitsPerComponent, string sourceName) + { + var bits = (long)columns * colors * bitsPerComponent; + + if (bits <= 0 || bits > MaximumPredictedRowBytes * 8L) + { + throw new PdfUnsupportedFeatureException( + sourceName, + "Predictor", + "the predicted row is larger than the engine decodes."); + } + + return (int)((bits + 7) / 8); + } + /// Receives one undone row of a predicted payload. /// The row, which is pooled and valid only for the duration of the call. internal delegate void RowHandler(ReadOnlySpan row); @@ -337,17 +362,7 @@ public PredictorRowDecoder( this.colors = colors; this.sourceName = sourceName; - var bits = (long)columns * colors * bitsPerComponent; - - if (bits <= 0 || bits > MaximumPredictedRowBytes * 8L) - { - throw new PdfUnsupportedFeatureException( - sourceName, - "Predictor", - "the predicted row is larger than the engine decodes."); - } - - rowLength = (int)((bits + 7) / 8); + rowLength = PredictedRowLength(columns, colors, bitsPerComponent, sourceName); bytesPerPixel = Math.Max(1, colors * bitsPerComponent / 8); stride = tiff ? rowLength : rowLength + 1; diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfIncrementalWriter.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfIncrementalWriter.cs index 13778e7..70fc140 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfIncrementalWriter.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfIncrementalWriter.cs @@ -92,11 +92,24 @@ public long WriteObject(int objectNumber, int generation, PdfObject body) /// Writes the cross-reference section, the trailer and the closing %%EOF. /// The trailer dictionary, which must already carry /Root and /Prev. - public void Complete(PdfDictionary trailer) + /// Whether the section is written as a cross-reference stream, the form the document it is appended to uses. + public void Complete(PdfDictionary trailer, bool asStream = false) { ArgumentNullException.ThrowIfNull(trailer); var startXref = Length; + + if (asStream) + { + PdfCrossReferenceStreamWriter.Write( + output, + startXref, + entries.Select(entry => (entry.Number, entry.Generation, entry.Offset)).ToArray(), + trailer, + freeHead: false); + return; + } + output.Write("xref\n"u8); WriteSubsections(); diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfObjectParser.Streams.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfObjectParser.Streams.cs index ece8c88..c154761 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfObjectParser.Streams.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfObjectParser.Streams.cs @@ -91,13 +91,23 @@ private bool EndStreamFollows(int position) _ => null, }; + /// + /// The most endstream candidates a compressed stream is inflated for. Each is inflated from the + /// start of the data, so the work grows with the square of the candidates, and a hostile file can hold + /// one every ten bytes. + /// + private const int MaximumInflatedCandidates = 16; + private PdfStream ReadStreamByScanning(PdfDictionary dictionary, int dataStart) { var data = lexer.Data; var isCompressed = PdfFilters.HasFlateCompression(dictionary); var searchFrom = dataStart; + var inflated = 0; ReadOnlyMemory? firstCandidate = null; var firstCandidateEnd = dataStart; + ReadOnlyMemory? inflatingCandidate = null; + var inflatingCandidateEnd = dataStart; while (searchFrom < data.Length) { @@ -118,20 +128,46 @@ private PdfStream ReadStreamByScanning(PdfDictionary dictionary, int dataStart) firstCandidateEnd = index + 9; } + if (!isCompressed) + { + return EndScannedStream(dictionary, candidate, index + 9); + } + // The candidate has to be decrypted before it can be decompressed, because the stream of - // an encrypted document is deflated first and encrypted second. - if (!isCompressed || PdfFilters.TryInflate(candidate, out _)) + // an encrypted document is deflated first and encrypted second. A deflate stream cut short + // inflates without complaint to the bytes in front of the cut, so a candidate is taken at once + // only when its data is complete; the first that merely inflates is kept for when none is. + if (inflated++ < MaximumInflatedCandidates && PdfFilters.TryInflate(candidate, out var decoded)) { - Position = index + 9; - TryReadKeyword("endstream"); - return new PdfStream(dictionary, candidate); + if (PdfFilters.EndsWithAdler32Of(candidate.Span, decoded)) + { + return EndScannedStream(dictionary, candidate, index + 9); + } + + if (inflatingCandidate is null) + { + inflatingCandidate = candidate; + inflatingCandidateEnd = index + 9; + } } searchFrom = index + 9; } - Position = firstCandidateEnd; - return new PdfStream(dictionary, firstCandidate ?? ReadOnlyMemory.Empty); + return inflatingCandidate is { } inflating + ? EndScannedStream(dictionary, inflating, inflatingCandidateEnd) + : EndScannedStream(dictionary, firstCandidate ?? ReadOnlyMemory.Empty, firstCandidateEnd); + } + + /// Ends a stream the scan delimited, behind the endstream keyword it chose. + /// The stream dictionary. + /// The body the scan chose. + /// The offset just behind the keyword that ends the body. + /// The stream. + private PdfStream EndScannedStream(PdfDictionary dictionary, ReadOnlyMemory body, int keywordEnd) + { + Position = keywordEnd; + return new PdfStream(dictionary, body); } /// Removes the end of line that a writer puts between the data and the keyword. diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfSecurityHandler.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfSecurityHandler.cs index 2a02185..65425db 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfSecurityHandler.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfSecurityHandler.cs @@ -201,7 +201,10 @@ public static PdfDecryptionResult TryCreate( // The encryption dictionary states the key length in bits; a crypt filter of revision 4 and 5 // states its own, in bytes, and it is the one the file key is actually derived with. var declaredBits = (int)(encryption.GetInteger("Length") ?? 40); - var keyBits = version >= 4 && cryptFilterBits is { } filterBits ? filterBits : declaredBits; + // AESV2 is AES-128 (ISO 32000-1, table 25): its key is sixteen bytes whatever length is declared, and a + // file that declares fewer was encrypted with sixteen by every producer that writes AESV2. + var statedBits = version >= 4 && cryptFilterBits is { } filterBits ? filterBits : declaredBits; + var keyBits = stringMethod == PdfCryptMethod.AesV2 || streamMethod == PdfCryptMethod.AesV2 ? 128 : statedBits; var encryptsMetadata = ReadEncryptsMetadata(encryption); byte[]? fileKey; diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfWriter.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfWriter.cs index d3cd0b7..f108a11 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfWriter.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfWriter.cs @@ -157,11 +157,25 @@ public int WriteEncryptionDictionary(PdfSecurityWriter security) /// Writes the cross-reference table, trailer and the closing startxref. /// The trailer dictionary; its Size entry is corrected here. - public void Complete(PdfDictionary trailer) + /// Whether the section is written as a cross-reference stream, the form a revision appended to a document that uses them takes. + public void Complete(PdfDictionary trailer, bool asStream = false) { ArgumentNullException.ThrowIfNull(trailer); var startXref = Length; + + if (asStream) + { + trailer.SetInteger("Size", Math.Max(trailer.GetInteger("Size") ?? 0, lastObjectNumber + 1)); + PdfCrossReferenceStreamWriter.Write( + output, + startXref, + offsets.Select(entry => (entry.Key, 0, entry.Value)).ToArray(), + trailer, + freeHead: true); + return; + } + output.Write("xref\n"u8); WriteSubsections(); diff --git a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Document.cs b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Document.cs index 93a0682..2117fb1 100644 --- a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Document.cs +++ b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Document.cs @@ -733,6 +733,7 @@ private static int WriteInformation(PdfWriter writer, PdfMetadata metadata, int? /// The encryption of the result, or when it is written in the clear. /// The cross-reference section this revision chains to, or for a complete document. /// The first element of the source's ID, kept by an appended revision, or . + /// Whether the section is a cross-reference stream, because the document the revision is appended to uses them. private static void WriteTrailer( PdfWriter writer, int catalogNumber, @@ -740,7 +741,8 @@ private static void WriteTrailer( PdfString identifier, PdfSecurityWriter? security, long? previousXref, - PdfString? permanentIdentifier = null) + PdfString? permanentIdentifier = null, + bool asStream = false) { var trailer = new PdfDictionary(); trailer.Set("Root", new PdfReference(catalogNumber, 0)); @@ -767,7 +769,7 @@ private static void WriteTrailer( trailer.Set("Encrypt", new PdfReference(writer.WriteEncryptionDictionary(security), 0)); } - writer.Complete(trailer); + writer.Complete(trailer, asStream); } /// Builds the identifier written twice in the trailer of the result. diff --git a/src/SharpCorePDF.Infrastructure/PdfEditEngine.cs b/src/SharpCorePDF.Infrastructure/PdfEditEngine.cs index c6d4751..7fb4d3b 100644 --- a/src/SharpCorePDF.Infrastructure/PdfEditEngine.cs +++ b/src/SharpCorePDF.Infrastructure/PdfEditEngine.cs @@ -318,7 +318,8 @@ [.. layout.Select(page => page.TargetNumber)], identifier, security, revision?.PreviousXref, - primary is null ? null : PermanentIdentifier(primary)); + primary is null ? null : PermanentIdentifier(primary), + primary?.CrossReferenceTable.NewestSectionIsStream == true); Verify(capture, layout, plan.Output.Encryption?.UserPassword, warnings); diff --git a/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Metadata.cs b/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Metadata.cs index 3681e29..4525962 100644 --- a/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Metadata.cs +++ b/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Metadata.cs @@ -179,15 +179,23 @@ public async Task CopyEmbeddedFileDataAsync( } var reader = await OpenAsync(sourceName, content, password, cancellationToken).ConfigureAwait(false); - var stream = FindEmbeddedStream(reader, file.Id); - if (stream is null) + if (!TryFindEmbeddedStream(reader, file.Id, out var stream)) { throw new ArgumentException( $"'{file.Id}' is not an embedded file of '{sourceName}'.", nameof(file)); } + // A specification that only names a file outside the document is listed, with a warning and a length + // of zero, and copies as what it holds: nothing. Refusing it as a caller's mistake contradicted the + // listing the caller took it from. + if (stream is null) + { + await destination.FlushAsync(cancellationToken).ConfigureAwait(false); + return; + } + await PdfImageFilters .CopyAsync(stream, destination, reader.SourceName, cancellationToken) .ConfigureAwait(false); @@ -250,9 +258,12 @@ private static void CollectNameTree( /// Finds the embedded stream whose identifier is the one asked for. /// The document. /// The identifier of the file. - /// The stream, or when the document has no such file. - private static PdfStream? FindEmbeddedStream(PdfFileReader reader, string id) + /// The stream, or when the file's specification embeds none. + /// when the document lists a file with that identifier. + private static bool TryFindEmbeddedStream(PdfFileReader reader, string id, out PdfStream? stream) { + stream = null; + var entries = new List>(); var names = reader.ResolveDictionary(reader.Catalog?.Get("Names")); @@ -282,11 +293,12 @@ private static void CollectNameTree( if (string.Equals(identifier, id, StringComparison.Ordinal)) { - return FindEmbeddedStream(reader, specification); + stream = FindEmbeddedStream(reader, specification); + return true; } } - return null; + return false; } /// Reads the name a file specification gives the file. diff --git a/src/SharpCorePDF.Infrastructure/Signatures/SignatureFieldWriter.cs b/src/SharpCorePDF.Infrastructure/Signatures/SignatureFieldWriter.cs index df28413..9d98ca9 100644 --- a/src/SharpCorePDF.Infrastructure/Signatures/SignatureFieldWriter.cs +++ b/src/SharpCorePDF.Infrastructure/Signatures/SignatureFieldWriter.cs @@ -155,7 +155,7 @@ public static async Task WriteAsync( writer.WriteObject(widgetNumber, 0, Widget(options, pageNumber, signatureReference, appearanceNumber)); writer.WriteObject(pageNumber, 0, PageWithWidget(reader, page, widgetReference)); writer.WriteObject(catalogNumber, 0, new PdfDictionary(catalog.Entries).Set("AcroForm", AcroForm(reader, catalog, widgetReference))); - writer.Complete(Trailer(reader, catalogNumber)); + writer.Complete(Trailer(reader, catalogNumber), asStream: reader.CrossReferenceTable.NewestSectionIsStream); var digest = SealRevision(buffer, byteRangeOffset, contentsStart, hexLength, options.HashAlgorithm); var value = await CmsBuilder.BuildAsync(digest, options, certificates, signingTime, cancellationToken).ConfigureAwait(false); diff --git a/tests/SharpCorePDF.Tests/PdfContentReaderTests.Metadata.cs b/tests/SharpCorePDF.Tests/PdfContentReaderTests.Metadata.cs index 151ae8d..6444f85 100644 --- a/tests/SharpCorePDF.Tests/PdfContentReaderTests.Metadata.cs +++ b/tests/SharpCorePDF.Tests/PdfContentReaderTests.Metadata.cs @@ -103,6 +103,42 @@ public async Task Copies_an_embedded_file_through_the_copy_route() Assert.Equal("second", Encoding.Latin1.GetString(materialised.Span)); } + /// + /// A file specification in the embedded files tree that only names a file outside the document — the shape + /// of the veraPDF corpus file isartor-6-1-11-t01-fail-a.pdf — is listed with a warning and a length + /// of zero, and copying it writes no bytes; it used to be refused as if the caller had passed a file of + /// another document. A file the document does not have is still refused. + /// + [Fact] + public async Task Copies_nothing_for_a_listed_file_that_embeds_no_stream() + { + var builder = new PdfBuilder(); + var catalog = builder.Reserve(); + var pages = builder.Reserve(); + var page = builder.Add("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] >>"); + var specification = builder.Add("<< /Type /Filespec /F (img/pageA4.pdf) >>"); + + builder.Set(catalog, $"<< /Type /Catalog /Pages {pages} 0 R /Names << /EmbeddedFiles << /Names [(pageA4.pdf) {specification} 0 R] >> >> >>"); + builder.Set(pages, $"<< /Type /Pages /Kids [{page} 0 R] /Count 1 >>"); + + var source = Source(builder.Build(catalog), "external.pdf"); + var reader = Reader(); + var result = await reader.ReadEmbeddedFilesAsync(source, cancellationToken: Token); + var file = Assert.Single(result.Items); + + using var destination = new MemoryStream(); + + await reader.CopyEmbeddedFileDataAsync(source, file, destination, cancellationToken: Token); + + Assert.Equal("img/pageA4.pdf", file.Id); + Assert.Equal(0, file.ByteLength); + Assert.Empty(destination.ToArray()); + AssertWarning(result.Warnings, PdfWarningKind.EmbeddedFileUnreadable); + + await Assert.ThrowsAsync( + () => reader.CopyEmbeddedFileDataAsync(source, file with { Id = "other.pdf" }, Stream.Null, cancellationToken: Token)); + } + /// A document without attachments yields an empty result rather than an exception. [Fact] public async Task Returns_an_empty_result_for_a_document_without_embedded_files() diff --git a/tests/SharpCorePDF.Tests/PdfDocumentSignerTests.cs b/tests/SharpCorePDF.Tests/PdfDocumentSignerTests.cs index 828cfe3..a8a6c63 100644 --- a/tests/SharpCorePDF.Tests/PdfDocumentSignerTests.cs +++ b/tests/SharpCorePDF.Tests/PdfDocumentSignerTests.cs @@ -3,6 +3,7 @@ using SharpCorePDF.Domain.Ports; using SharpCorePDF.Domain.ValueObjects; using SharpCorePDF.Infrastructure; +using SharpCorePDF.Infrastructure.Pdf; namespace SharpCorePDF.Tests; @@ -86,6 +87,25 @@ public async Task Appends_a_revision_and_leaves_the_source_bytes_untouched() Assert.Contains(" /SigFlags 3", text, StringComparison.Ordinal); } + /// + /// A signature appended to a document whose newest section is a cross-reference stream ends its revision + /// in a stream too, and the signature is valid over the whole file; a classic table used to be appended. + /// + [Fact] + public async Task Signs_a_document_that_uses_cross_reference_streams_with_one() + { + var source = File.ReadAllBytes(TestAssets.Modern); + var signed = await SigningFixtures.SignAsync(source, SigningFixtures.Options(Certificate)); + + var reader = new PdfFileReader(signed, "signed.pdf", PdfEngineOptions.Default); + var validation = Assert.Single(await SigningFixtures.ValidateAsync(signed)); + + Assert.True(reader.CrossReferenceTable.NewestSectionIsStream); + Assert.False(reader.CrossReferenceTable.WasReconstructed); + Assert.Equal(PdfSignatureIntegrity.Valid, validation.Integrity); + Assert.True(validation.Signer.CoversWholeDocument); + } + /// /// The second signature appends a revision of its own, and the first one still covers exactly the bytes it /// signed: intact, and no longer the whole file. diff --git a/tests/SharpCorePDF.Tests/PdfIncrementalWriterTests.cs b/tests/SharpCorePDF.Tests/PdfIncrementalWriterTests.cs index e535b94..eff4bf2 100644 --- a/tests/SharpCorePDF.Tests/PdfIncrementalWriterTests.cs +++ b/tests/SharpCorePDF.Tests/PdfIncrementalWriterTests.cs @@ -136,6 +136,44 @@ public async Task Chains_the_revision_to_the_cross_reference_section_of_the_sour Assert.Contains(ending, Encoding.Latin1.GetString(edited), StringComparison.Ordinal); } + /// + /// A revision appended to a document whose newest section is a cross-reference stream ends in a stream + /// too, chained to the source's, listing itself, and every page of the result reads; a classic table used + /// to be appended to such a document, a chain ISO 32000-1 does not describe. + /// + [Fact] + public async Task Appends_a_cross_reference_stream_to_a_document_that_uses_them() + { + var source = File.ReadAllBytes(TestAssets.Modern); + var (_, edited) = await ApplyAsync(source, Incremental(PdfEditPlan.Create().SetMetadata(new PdfMetadata { Title = "streamed" }))); + + var offset = StartXref(edited); + var revision = Encoding.Latin1.GetString(edited, source.Length, edited.Length - source.Length); + var reader = new PdfFileReader(edited, "edited.pdf", Options); + var section = Assert.IsType(reader.Resolve(reader.CrossReferenceTable.Trailer.GetInteger("Size")!.Value - 1)); + + Assert.Equal(source, edited[..source.Length]); + Assert.True(offset > source.Length); + Assert.DoesNotContain("\nxref\n", revision, StringComparison.Ordinal); + Assert.DoesNotContain("trailer", revision, StringComparison.Ordinal); + Assert.Equal("XRef", section.Dictionary.GetName("Type")); + Assert.Equal(StartXref(source), section.Dictionary.GetInteger("Prev")); + Assert.True(reader.CrossReferenceTable.NewestSectionIsStream); + Assert.False(reader.CrossReferenceTable.WasReconstructed); + Assert.Equal("streamed", (await new PdfDocumentReader(Options).ReadAsync("edited.pdf", new MemoryStream(edited), null, Token)).Metadata.Title); + } + + /// A revision appended to a document with a classic table keeps writing a classic table. + [Fact] + public async Task Appends_a_classic_table_to_a_document_that_uses_one() + { + var source = ThreePages(); + var (_, edited) = await ApplyAsync(source, Incremental(PdfEditPlan.Create().DeletePage(2))); + + Assert.False(new PdfFileReader(source, "source.pdf", Options).CrossReferenceTable.NewestSectionIsStream); + Assert.False(new PdfFileReader(edited, "edited.pdf", Options).CrossReferenceTable.NewestSectionIsStream); + } + /// The offsets of the revision are absolute in a destination that can seek. [Fact] public async Task Counts_offsets_from_the_start_of_a_seekable_destination() diff --git a/tests/SharpCorePDF.Tests/PdfMergeEngineTests.OutputEncryption.cs b/tests/SharpCorePDF.Tests/PdfMergeEngineTests.OutputEncryption.cs index 3c48f8c..79fdd06 100644 --- a/tests/SharpCorePDF.Tests/PdfMergeEngineTests.OutputEncryption.cs +++ b/tests/SharpCorePDF.Tests/PdfMergeEngineTests.OutputEncryption.cs @@ -260,6 +260,28 @@ await MergeEncryptedAsync(rc4, moment), Assert.Equal(3, (await ReadAsync(second, MergedUserPassword)).PageCount); } + /// + /// AESV2 is AES-128 whatever key length its crypt filter declares, so a file that declares five bytes still + /// opens with its password: the key used to be cut to the declared length, which no AES-128 producer used. + /// + [Fact] + public async Task Opens_an_aesv2_document_whose_crypt_filter_declares_a_short_key() + { + var bytes = await MergeEncryptedAsync(Encryption(PdfEncryptionAlgorithm.AesWith128BitKey)); + var text = Encoding.Latin1.GetString(bytes); + var declared = text.IndexOf("/AESV2 /Length 16", StringComparison.Ordinal); + + Assert.True(declared >= 0); + + // The same number of bytes, so every offset of the file stays where it was. + Encoding.Latin1.GetBytes("/AESV2 /Length 5 ").CopyTo(bytes, declared); + + var info = await ReadAsync(bytes, MergedUserPassword); + + Assert.True(info.IsReadable); + Assert.Equal(3, info.PageCount); + } + /// Merges an encrypted source and a plain one into a result with the encryption asked for. /// The encryption to write the result with. /// The moment to stamp the result with, when it has to be reproducible. diff --git a/tests/SharpCorePDF.Tests/PdfPredictorTests.cs b/tests/SharpCorePDF.Tests/PdfPredictorTests.cs index f15d42e..dadd829 100644 --- a/tests/SharpCorePDF.Tests/PdfPredictorTests.cs +++ b/tests/SharpCorePDF.Tests/PdfPredictorTests.cs @@ -150,6 +150,30 @@ public void Reports_tiff_prediction_that_is_not_eight_bit() /// The predicted rows, as hexadecimal. /// The decode parameters of the stream. /// The decoded rows. + /// + /// A row size the declared columns, colours and bit depth multiply out to beyond what the engine decodes is + /// refused, whatever the product wraps to in 32 bits: 65536 × 65536 used to wrap to a row of zero bytes, + /// on which TIFF prediction looped forever, and 65536 × 32769 to a negative one, which escaped as an + /// . The streaming decoder already refused both. + /// + /// The predictor. + /// The declared columns. + /// The declared colours. + /// The declared bit depth. + [Theory(Timeout = 10_000)] + [InlineData(2, 65536, 65536, 8)] + [InlineData(2, 65536, 32769, 8)] + [InlineData(12, 65536, 65536, 8)] + [InlineData(12, 1 << 30, 1, 16)] + [InlineData(12, 4, 1, -8)] + public async Task Refuses_a_predicted_row_larger_than_the_engine_decodes(int predictor, int columns, int colors, int bitsPerComponent) + { + var parameters = Parameters(predictor, columns: columns, colors: colors, bitsPerComponent: bitsPerComponent); + + await Assert.ThrowsAnyAsync( + () => Task.Run(() => Decode("0001020304050607", parameters), TestContext.Current.CancellationToken)); + } + private static byte[] Decode(string encoded, PdfDictionary parameters) => PdfFilters.Decode( Stream(Compress(Convert.FromHexString(encoded)), parameters), diff --git a/tests/SharpCorePDF.Tests/PdfReaderRecoveryTests.cs b/tests/SharpCorePDF.Tests/PdfReaderRecoveryTests.cs index da00411..8666a25 100644 --- a/tests/SharpCorePDF.Tests/PdfReaderRecoveryTests.cs +++ b/tests/SharpCorePDF.Tests/PdfReaderRecoveryTests.cs @@ -1,3 +1,4 @@ +using System.Globalization; using System.Text; using SharpCorePDF.Domain.Exceptions; using SharpCorePDF.Domain.ReadModels; @@ -18,6 +19,108 @@ namespace SharpCorePDF.Tests; /// public sealed class PdfReaderRecoveryTests { + /// + /// A file whose bytes moved after its table was written — here three bytes inserted behind the header, the + /// way a converted line ending or a prepended byte order mark moves them — has a table that parses and + /// offsets that land in the middle of objects. The table is rebuilt instead of the objects going missing. + /// + [Fact] + public void Rebuilds_a_table_whose_offsets_no_longer_point_at_their_objects() + { + var original = TwoPageDocument(); + var header = "%PDF-1.7\n"u8.Length; + var shifted = original[..header].Concat(" "u8.ToArray()).Concat(original[header..]).ToArray(); + + // The startxref is corrected, so only the entries of the table are wrong: the case reconstruction + // used to skip, because the table read cleanly and named a catalog. + var text = Encoding.Latin1.GetString(shifted); + var keyword = text.LastIndexOf("startxref", StringComparison.Ordinal); + var digits = text.IndexOf('\n', keyword) + 1; + var end = text.IndexOf('\n', digits); + var corrected = (long.Parse(text[digits..end], CultureInfo.InvariantCulture) + 3).ToString(CultureInfo.InvariantCulture); + var repaired = Encoding.Latin1.GetBytes(text[..digits] + corrected + text[end..]); + + var reader = new PdfFileReader(repaired, "shifted.pdf", PdfEngineOptions.Default); + + Assert.Equal(2, reader.Pages.Count); + Assert.Contains(reader.Warnings, warning => warning.Kind == PdfWarningKind.CrossReferenceRebuilt); + } + + /// + /// A PDF 1.5 file whose cross-reference stream is lost keeps its page inside an object stream and its + /// catalog reference in the cross-reference stream's dictionary. Rebuilding finds both: the objects an + /// object stream holds, and the /Root of the last cross-reference stream. + /// + [Fact] + public void Rebuilds_a_table_from_object_streams_when_the_cross_reference_stream_is_lost() + { + var page = "3 0 << /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] >>"u8.ToArray(); + var packed = PdfBuilder.Compress(page); + var output = new MemoryStream(); + var offsets = new Dictionary(); + + void Write(string value) => output.Write(Encoding.Latin1.GetBytes(value)); + + void Object(int number, string dictionary, byte[]? stream = null) + { + offsets[number] = output.Position; + Write($"{number} 0 obj\n{dictionary}\n"); + + if (stream is not null) + { + Write("stream\n"); + output.Write(stream); + Write("\nendstream\n"); + } + + Write("endobj\n"); + } + + Write("%PDF-1.7\n"); + Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); + Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + Object(5, $"<< /Type /ObjStm /N 1 /First 4 /Filter /FlateDecode /Length {packed.Length} >>", packed); + + // The cross-reference stream is damaged beyond reading: its data is gone, its dictionary survives. + Object(6, "<< /Type /XRef /Size 7 /W [1 4 2] /Root 1 0 R /Length 0 >>", []); + Write($"startxref\n{offsets[6] + 7}\n%%EOF\n"); + + var reader = new PdfFileReader(output.ToArray(), "lost-xref.pdf", PdfEngineOptions.Default); + + Assert.Single(reader.Pages); + Assert.Equal("Page", Assert.IsType(reader.Resolve(3)).GetName("Type")); + Assert.Contains(reader.Warnings, warning => warning.Kind == PdfWarningKind.CrossReferenceRebuilt); + } + + /// + /// A reference names an object by number and generation, and one whose generation is not the one the + /// table lists for that number refers to an object that no longer exists, which ISO 32000-1 reads as + /// null; the generation used to be ignored, so 3 1 R resolved to 3 0 obj. + /// + [Fact] + public void Resolves_a_reference_to_a_generation_the_table_does_not_list_as_null() + { + var reader = new PdfFileReader(TwoPageDocument(), "generations.pdf", PdfEngineOptions.Default); + + Assert.IsType(reader.Resolve(new PdfReference(3, 0))); + Assert.Null(reader.Resolve(new PdfReference(3, 1))); + Assert.IsType(reader.Resolve(new PdfReference(3, 0))); + } + + private static byte[] TwoPageDocument() + { + var builder = new PdfBuilder(); + var catalog = builder.Reserve(); + var pages = builder.Reserve(); + + builder.Add("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] >>"); + builder.Add("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] >>"); + builder.Set(catalog, "<< /Type /Catalog /Pages 2 0 R >>"); + builder.Set(pages, "<< /Type /Pages /Kids [3 0 R 4 0 R] /Count 2 >>"); + + return builder.Build(catalog); + } + /// /// The bound on a chain of lengths leaves an ordinary indirect length alone, for every stream of a /// document and not only the first ones: each stream's data holds an endstream line of its own, @@ -175,27 +278,29 @@ public async Task Rebuilds_a_table_that_is_not_where_startxref_announces_it() warning => warning.Kind == PdfWarningKind.CrossReferenceRebuilt); } - /// A document whose catalog lives in an object stream cannot be rebuilt by scanning. + /// + /// A document whose catalog lives in an object stream is rebuilt from the object streams the scan finds and + /// the /Root of its cross-reference stream's dictionary, and reads the pages the undamaged file has. + /// /// /// The scanner finds object headers in the bytes of the file, and the catalog of a document that keeps its - /// objects in object streams is inside a compressed stream, where no scan can see it. Refusing the - /// document is the honest answer; the alternative is a merge that silently contributes nothing. + /// objects in object streams is inside a compressed stream, where a scan for headers cannot see it. This + /// test used to assert that such a document is refused; reconstruction now looks inside the object streams + /// it finds, which is what other readers do. /// [Fact] - public async Task Refuses_a_document_whose_catalog_cannot_be_found_by_scanning() + public async Task Rebuilds_a_document_whose_catalog_lives_in_an_object_stream() { var reader = new PdfDocumentReader(PdfEngineOptions.Default); - using var content = new MemoryStream(PointStartxrefAtTheHeader(TestAssets.Modern)); + using var intact = new MemoryStream(File.ReadAllBytes(TestAssets.Modern)); + using var damaged = new MemoryStream(PointStartxrefAtTheHeader(TestAssets.Modern)); - var exception = await Assert.ThrowsAsync( - async () => await reader.ReadAsync( - "damaged.pdf", - content, - password: null, - TestContext.Current.CancellationToken)); + var expected = await reader.ReadAsync("intact.pdf", intact, password: null, TestContext.Current.CancellationToken); + var rebuilt = await reader.ReadAsync("damaged.pdf", damaged, password: null, TestContext.Current.CancellationToken); - Assert.Contains("no document catalog", exception.Message, StringComparison.Ordinal); + Assert.Equal(expected.PageCount, rebuilt.PageCount); + Assert.Contains(rebuilt.Warnings, warning => warning.Kind == PdfWarningKind.CrossReferenceRebuilt); } /// A file that was cut behind its last object keeps working. diff --git a/tests/SharpCorePDF.Tests/PdfStreamParsingTests.cs b/tests/SharpCorePDF.Tests/PdfStreamParsingTests.cs index 6404682..f0c1e52 100644 --- a/tests/SharpCorePDF.Tests/PdfStreamParsingTests.cs +++ b/tests/SharpCorePDF.Tests/PdfStreamParsingTests.cs @@ -68,6 +68,35 @@ public void Returns_no_bytes_when_the_keyword_is_not_where_it_must_be() => public void Keeps_the_bytes_it_found_when_a_compressed_candidate_does_not_inflate() => Assert.Equal("not compressed", Read("<< /Filter /FlateDecode >>\nstream\nnot compressed\nendstream")); + /// + /// A compressed stream whose data holds the word endstream is cut at the candidate where the + /// compressed data is complete — where its Adler-32 checksum matches — not at the first one that inflates: + /// a deflate stream cut short inflates without complaint to the bytes in front of the cut. + /// + [Fact] + public void Scans_past_a_candidate_whose_compressed_data_is_cut_short() + { + var content = Encoding.Latin1.GetBytes("BT (before)\nendstream\n(after) Tj ET"); + using var buffer = new MemoryStream(); + + // Stored blocks keep the word readable in the compressed bytes, where a writer that compresses + // its content can leave it too. + using (var zlib = new System.IO.Compression.ZLibStream(buffer, System.IO.Compression.CompressionLevel.NoCompression, leaveOpen: true)) + { + zlib.Write(content); + } + + var bytes = Encoding.Latin1.GetBytes("<< /Filter /FlateDecode >>\nstream\n") + .Concat(buffer.ToArray()) + .Concat(Encoding.Latin1.GetBytes("\nendstream")) + .ToArray(); + + var parser = new PdfObjectParser(bytes, "damaged.pdf", 0, null, 64); + var stream = Assert.IsType(parser.ParseObject()); + + Assert.Equal(content, PdfFilters.Decode(stream, "damaged.pdf")); + } + /// A stream is read behind an indirect object header. [Fact] public void Reads_a_stream_behind_an_object_header() From 54c08d57e94a0e7fcf8603e1163008c35bba6285 Mon Sep 17 00:00:00 2001 From: Andrew Tierney Date: Mon, 16 Nov 2026 13:31:06 +1100 Subject: [PATCH 08/15] Pin what the scoped Stryker run found unasserted The scoped run over the files of the last two commits scored 70.8% on their changed lines. The survivors that were missing assertions now have them: - PdfCrossReferenceStreamWriterTests compares every row of an appended cross-reference stream byte for byte, its /Index subsections, the width of each /W field at the byte boundaries, its /Size and the trailer entries it carries or drops. - The free text /DA test uses three different colour channels; the merge warning names the page as a person counts it; the damaged-Flate refusal says why. - The predictor limit is pinned at the largest row accepted and one column more, and a zero-bit row is refused. - The endstream scan leaves the parser behind the keyword it chose, and the sixteen-candidate cap is pinned at its boundary. - A signature on a classic-table document keeps a classic table, and only an edit's stream lists the head of the free list. PdfWriter's stream path no longer sets /Size before the stream writer, which derives it itself; the statement was the survivor nothing could kill. The remaining survivors are equivalent mutants (ConfigureAwait flips, bounds checks on offsets the scan already validated) or fallbacks for damaged files noted in the pull request. --- .../Pdf/PdfWriter.cs | 1 - .../SharpCorePDF.Tests/PdfAnnotationTests.cs | 4 +- .../PdfContentReaderTests.Images.cs | 4 +- .../PdfCrossReferenceStreamWriterTests.cs | 98 +++++++++++++++++++ .../PdfDocumentSignerTests.cs | 16 +++ .../PdfIncrementalWriterTests.cs | 1 + .../PdfMergeFidelityTests.cs | 4 +- tests/SharpCorePDF.Tests/PdfPredictorTests.cs | 9 ++ .../PdfStreamParsingTests.cs | 39 +++++++- 9 files changed, 170 insertions(+), 6 deletions(-) create mode 100644 tests/SharpCorePDF.Tests/PdfCrossReferenceStreamWriterTests.cs diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfWriter.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfWriter.cs index f108a11..38f7ac7 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfWriter.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfWriter.cs @@ -166,7 +166,6 @@ public void Complete(PdfDictionary trailer, bool asStream = false) if (asStream) { - trailer.SetInteger("Size", Math.Max(trailer.GetInteger("Size") ?? 0, lastObjectNumber + 1)); PdfCrossReferenceStreamWriter.Write( output, startXref, diff --git a/tests/SharpCorePDF.Tests/PdfAnnotationTests.cs b/tests/SharpCorePDF.Tests/PdfAnnotationTests.cs index fc0ae3e..2f3ab84 100644 --- a/tests/SharpCorePDF.Tests/PdfAnnotationTests.cs +++ b/tests/SharpCorePDF.Tests/PdfAnnotationTests.cs @@ -255,7 +255,7 @@ public async Task States_the_default_appearance_and_resources_of_a_free_text_ann PdfAnnotations.FreeText( new PdfRectangle(72, 500, 320, 530), "Reviewed", - new PdfStandardTextStyle(StandardFont.Helvetica, 10, PdfColor.FromRgb(255, 0, 0)))), + new PdfStandardTextStyle(StandardFont.Helvetica, 10, PdfColor.FromRgb(255, 128, 51)))), cancellationToken: Token); var reader = Read(edited); @@ -264,7 +264,7 @@ public async Task States_the_default_appearance_and_resources_of_a_free_text_ann var appearance = Assert.IsType(reader.Resolve(reader.ResolveDictionary(annotation?.Get("AP"))?.Get("N"))); var defaultAppearance = Assert.IsType(annotation?.Get("DA")); - Assert.Equal("/Helv 10 Tf 1 0 0 rg", Encoding.ASCII.GetString(defaultAppearance.Bytes)); + Assert.Equal("/Helv 10 Tf 1 0.50196 0.2 rg", Encoding.ASCII.GetString(defaultAppearance.Bytes)); Assert.IsType(reader.Resolve(appearance.Dictionary.Get("Resources"))); } diff --git a/tests/SharpCorePDF.Tests/PdfContentReaderTests.Images.cs b/tests/SharpCorePDF.Tests/PdfContentReaderTests.Images.cs index 6388014..8224248 100644 --- a/tests/SharpCorePDF.Tests/PdfContentReaderTests.Images.cs +++ b/tests/SharpCorePDF.Tests/PdfContentReaderTests.Images.cs @@ -40,8 +40,10 @@ public async Task Refuses_to_copy_an_image_whose_flate_data_is_damaged(string pa var reader = Reader(); var image = Assert.Single((await reader.ReadImagesAsync(source, PageRange.All, cancellationToken: Token)).Items); - await Assert.ThrowsAsync( + var exception = await Assert.ThrowsAsync( () => reader.CopyImageDataAsync(source, image, Stream.Null, cancellationToken: Token)); + + Assert.Contains("Flate data of the payload is damaged", exception.Message, StringComparison.Ordinal); } /// One object drawn twice and shared by two pages is one entry carrying both pages. diff --git a/tests/SharpCorePDF.Tests/PdfCrossReferenceStreamWriterTests.cs b/tests/SharpCorePDF.Tests/PdfCrossReferenceStreamWriterTests.cs new file mode 100644 index 0000000..deba1e3 --- /dev/null +++ b/tests/SharpCorePDF.Tests/PdfCrossReferenceStreamWriterTests.cs @@ -0,0 +1,98 @@ +using System.Text; +using SharpCorePDF.Infrastructure.Pdf; + +namespace SharpCorePDF.Tests; + +/// +/// Tests of the cross-reference stream an appended revision ends in: the bytes of every row, the subsections +/// /Index names, the field widths of /W, and the trailer entries the stream's dictionary carries. +/// +/// +/// The rows are compared as hexadecimal so that a wrong width, a wrong byte order or a missing row shows in +/// the expectation itself. +/// +public sealed class PdfCrossReferenceStreamWriterTests +{ + /// + /// Every row is a type byte, an offset as wide as the largest offset needs and a two-byte generation, in + /// object number order; the stream lists itself, numbered above every other object, and the head of the free + /// list when it is asked to. + /// + [Fact] + public void Writes_every_row_of_the_section_in_object_number_order() + { + var (dictionary, rows, text) = Write( + [(7, 2, 70000), (3, 0, 100), (4, 0, 200)], + 300000, + new PdfDictionary().SetInteger("Size", 5).SetInteger("XRefStm", 9).Set("Root", new PdfReference(1, 0)), + freeHead: true); + + Assert.Equal("XRef", dictionary.GetName("Type")); + Assert.Equal(9, dictionary.GetInteger("Size")); + Assert.Null(dictionary.Get("XRefStm")); + Assert.Equal(1, Assert.IsType(dictionary.Get("Root")).ObjectNumber); + Assert.Equal([1L, 3L, 2L], Integers(dictionary.Get("W"))); + Assert.Equal([0L, 1L, 3L, 2L, 7L, 2L], Integers(dictionary.Get("Index"))); + Assert.Equal( + "00000000FFFF" + "010000640000" + "010000C80000" + "010111700002" + "010493E00000", + Convert.ToHexString(rows)); + Assert.StartsWith("8 0 obj\n", text, StringComparison.Ordinal); + Assert.EndsWith("\nendstream\nendobj\nstartxref\n300000\n%%EOF\n", text, StringComparison.Ordinal); + } + + /// + /// The stream is numbered at the /Size the trailer states when that is above every object of the + /// revision, and a revision appended by a signature lists no free head. + /// + [Fact] + public void Numbers_the_stream_at_the_size_the_trailer_states() + { + var (dictionary, rows, text) = Write([(3, 0, 16)], 40, new PdfDictionary().SetInteger("Size", 20), freeHead: false); + + Assert.Equal(21, dictionary.GetInteger("Size")); + Assert.Equal([3L, 1L, 20L, 1L], Integers(dictionary.Get("Index"))); + Assert.Equal("0110000001280000", Convert.ToHexString(rows)); + Assert.StartsWith("20 0 obj\n", text, StringComparison.Ordinal); + } + + /// The offset field is as narrow as the largest offset allows, and no narrower. + /// The offset of the stream, the largest of the section. + /// The width the offset field must have. + [Theory] + [InlineData(255L, 1)] + [InlineData(256L, 2)] + [InlineData(65535L, 2)] + [InlineData(65536L, 3)] + [InlineData(16777216L, 4)] + [InlineData(4294967296L, 5)] + public void Sizes_the_offset_field_to_the_largest_offset(long startXref, int width) + { + var (dictionary, _, _) = Write([], startXref, new PdfDictionary(), freeHead: false); + + Assert.Equal(width, Integers(dictionary.Get("W"))[1]); + } + + private static (PdfDictionary Dictionary, byte[] Rows, string Text) Write( + (int Number, int Generation, long Offset)[] entries, + long startXref, + PdfDictionary trailer, + bool freeHead) + { + using var buffer = new MemoryStream(); + var output = new PdfOutputBuffer(buffer); + + PdfCrossReferenceStreamWriter.Write(output, startXref, entries, trailer, freeHead); + + var bytes = buffer.ToArray(); + var parser = new PdfObjectParser(bytes, "section.pdf", 0, null, 64); + + Assert.True(parser.TryReadIndirectHeader(out _, out _)); + + var stream = Assert.IsType(parser.ParseObject()); + + return (stream.Dictionary, stream.Raw.ToArray(), Encoding.Latin1.GetString(bytes)); + } + + private static long[] Integers(PdfObject? array) => + Assert.IsType(array).Items.Select(item => Assert.IsType(item).Value).ToArray(); +} diff --git a/tests/SharpCorePDF.Tests/PdfDocumentSignerTests.cs b/tests/SharpCorePDF.Tests/PdfDocumentSignerTests.cs index a8a6c63..5aeb628 100644 --- a/tests/SharpCorePDF.Tests/PdfDocumentSignerTests.cs +++ b/tests/SharpCorePDF.Tests/PdfDocumentSignerTests.cs @@ -104,6 +104,22 @@ public async Task Signs_a_document_that_uses_cross_reference_streams_with_one() Assert.False(reader.CrossReferenceTable.WasReconstructed); Assert.Equal(PdfSignatureIntegrity.Valid, validation.Integrity); Assert.True(validation.Signer.CoversWholeDocument); + + // The revision lists the objects it wrote and the stream itself, and not the head of the free list, + // which the first revision already lists. + var section = Assert.IsType(reader.Resolve(reader.CrossReferenceTable.Trailer.GetInteger("Size")!.Value - 1)); + var index = Assert.IsType(section.Dictionary.Get("Index")); + + Assert.NotEqual(0, Assert.IsType(index.Items[0]).Value); + } + + /// A signature appended to a document with a classic table ends in a classic table. + [Fact] + public async Task Signs_a_document_that_uses_a_classic_table_with_one() + { + var signed = await SigningFixtures.SignAsync(SigningFixtures.BuildDocument(), SigningFixtures.Options(Certificate)); + + Assert.False(new PdfFileReader(signed, "signed.pdf", PdfEngineOptions.Default).CrossReferenceTable.NewestSectionIsStream); } /// diff --git a/tests/SharpCorePDF.Tests/PdfIncrementalWriterTests.cs b/tests/SharpCorePDF.Tests/PdfIncrementalWriterTests.cs index eff4bf2..4a00c7a 100644 --- a/tests/SharpCorePDF.Tests/PdfIncrementalWriterTests.cs +++ b/tests/SharpCorePDF.Tests/PdfIncrementalWriterTests.cs @@ -158,6 +158,7 @@ public async Task Appends_a_cross_reference_stream_to_a_document_that_uses_them( Assert.DoesNotContain("trailer", revision, StringComparison.Ordinal); Assert.Equal("XRef", section.Dictionary.GetName("Type")); Assert.Equal(StartXref(source), section.Dictionary.GetInteger("Prev")); + Assert.Equal(0, Assert.IsType(Assert.IsType(section.Dictionary.Get("Index")).Items[0]).Value); Assert.True(reader.CrossReferenceTable.NewestSectionIsStream); Assert.False(reader.CrossReferenceTable.WasReconstructed); Assert.Equal("streamed", (await new PdfDocumentReader(Options).ReadAsync("edited.pdf", new MemoryStream(edited), null, Token)).Metadata.Title); diff --git a/tests/SharpCorePDF.Tests/PdfMergeFidelityTests.cs b/tests/SharpCorePDF.Tests/PdfMergeFidelityTests.cs index 710d4c2..5309569 100644 --- a/tests/SharpCorePDF.Tests/PdfMergeFidelityTests.cs +++ b/tests/SharpCorePDF.Tests/PdfMergeFidelityTests.cs @@ -74,7 +74,9 @@ public async Task Writes_a_media_box_for_a_page_that_has_none() Assert.Equal(4, box.Count); Assert.Equal(PdfPageSize.A4.WidthPoints, ((PdfReal)box.Items[2]).Value, 3); Assert.Equal(PdfPageSize.A4.HeightPoints, ((PdfReal)box.Items[3]).Value, 3); - Assert.Contains(result.Warnings, warning => warning.Kind == PdfWarningKind.PageSizeAssumed); + Assert.Contains( + result.Warnings, + warning => warning.Kind == PdfWarningKind.PageSizeAssumed && warning.Message.Contains("page 1 ", StringComparison.Ordinal)); } /// A page that states its own media box keeps it, and no size is assumed for it. diff --git a/tests/SharpCorePDF.Tests/PdfPredictorTests.cs b/tests/SharpCorePDF.Tests/PdfPredictorTests.cs index dadd829..61cc6c4 100644 --- a/tests/SharpCorePDF.Tests/PdfPredictorTests.cs +++ b/tests/SharpCorePDF.Tests/PdfPredictorTests.cs @@ -166,6 +166,8 @@ public void Reports_tiff_prediction_that_is_not_eight_bit() [InlineData(12, 65536, 65536, 8)] [InlineData(12, 1 << 30, 1, 16)] [InlineData(12, 4, 1, -8)] + [InlineData(12, 4, 1, 0)] + [InlineData(12, (8 * 1024 * 1024) + 1, 1, 8)] public async Task Refuses_a_predicted_row_larger_than_the_engine_decodes(int predictor, int columns, int colors, int bitsPerComponent) { var parameters = Parameters(predictor, columns: columns, colors: colors, bitsPerComponent: bitsPerComponent); @@ -174,6 +176,13 @@ await Assert.ThrowsAnyAsync( () => Task.Run(() => Decode("0001020304050607", parameters), TestContext.Current.CancellationToken)); } + /// A row of exactly the largest size the engine decodes is accepted. + [Fact] + public void Accepts_a_predicted_row_of_the_largest_size_the_engine_decodes() => + Assert.Equal( + Convert.FromHexString("0001"), + Decode("0001", Parameters(12, columns: 8 * 1024 * 1024, colors: 1, bitsPerComponent: 8))); + private static byte[] Decode(string encoded, PdfDictionary parameters) => PdfFilters.Decode( Stream(Compress(Convert.FromHexString(encoded)), parameters), diff --git a/tests/SharpCorePDF.Tests/PdfStreamParsingTests.cs b/tests/SharpCorePDF.Tests/PdfStreamParsingTests.cs index f0c1e52..8eda724 100644 --- a/tests/SharpCorePDF.Tests/PdfStreamParsingTests.cs +++ b/tests/SharpCorePDF.Tests/PdfStreamParsingTests.cs @@ -91,10 +91,47 @@ public void Scans_past_a_candidate_whose_compressed_data_is_cut_short() .Concat(Encoding.Latin1.GetBytes("\nendstream")) .ToArray(); - var parser = new PdfObjectParser(bytes, "damaged.pdf", 0, null, 64); + var parser = new PdfObjectParser(bytes.Concat(Encoding.Latin1.GetBytes("\n42")).ToArray(), "damaged.pdf", 0, null, 64); var stream = Assert.IsType(parser.ParseObject()); Assert.Equal(content, PdfFilters.Decode(stream, "damaged.pdf")); + + // The parser stands behind the keyword that ended the body, so the next object reads. + Assert.Equal(42, Assert.IsType(parser.ParseObject()).Value); + } + + /// + /// At most sixteen candidates are inflated: each is inflated from the start of the data, and a hostile + /// file can hold one every ten bytes. A body whose complete end is the sixteenth candidate is found; one + /// whose end is the seventeenth is not, and the first candidate that inflated is kept instead. + /// + /// The endstream words inside the compressed data. + /// Whether the complete body is expected. + [Theory] + [InlineData(15, true)] + [InlineData(16, false)] + public void Inflates_at_most_sixteen_candidates(int falseEnds, bool complete) + { + var text = string.Concat(Enumerable.Range(0, falseEnds).Select(index => $"({index})\nendstream\n")) + "(end)"; + var content = Encoding.Latin1.GetBytes(text); + using var buffer = new MemoryStream(); + + using (var zlib = new System.IO.Compression.ZLibStream(buffer, System.IO.Compression.CompressionLevel.NoCompression, leaveOpen: true)) + { + zlib.Write(content); + } + + var bytes = Encoding.Latin1.GetBytes("<< /Filter /FlateDecode >>\nstream\n") + .Concat(buffer.ToArray()) + .Concat(Encoding.Latin1.GetBytes("\nendstream\n42")) + .ToArray(); + + var parser = new PdfObjectParser(bytes, "damaged.pdf", 0, null, 64); + var stream = Assert.IsType(parser.ParseObject()); + var decoded = PdfFilters.Decode(stream, "damaged.pdf"); + + Assert.Equal(complete, decoded.SequenceEqual(content)); + Assert.Equal(complete ? 42 : 0, complete ? Assert.IsType(parser.ParseObject()).Value : 0); } /// A stream is read behind an indirect object header. From 7c97a3898087a9b0f4375f4319733a5814808070 Mon Sep 17 00:00:00 2001 From: Andrew Tierney Date: Mon, 16 Nov 2026 13:48:35 +1100 Subject: [PATCH 09/15] Refuse ZLibException on the copy route and read rotations as the inspection does The second robustness run, on 54c08d5, found two more faults now that the first ones are fixed: - flate: the image copy route caught the decompressor's InvalidDataException but not the ZLibException it throws for other damaged input (crash-0b3d19d6). Both are now refused with PdfSourceFormatException. ZLibException is not public, so it is recognised by name rather than by catching every IOException, which would also swallow the destination's own failures. - document: a page declaring /Rotate 91 made the edit engine throw ArgumentOutOfRangeException (crash-7201f51f). The edit engine read /Rotate as an integer and required a multiple of 90, while the inspection reads any number and rounds to the nearest quarter turn, so a real /Rotate was also read as none. The edit engine now uses the inspection's reading, and the quarter turns are reduced before they become an integer, so a declared value beyond int's range no longer wraps. Tests that fail without the changes: PdfContentReaderTests.Refuses_to_copy_an_image_whose_flate_data_is_damaged (the fuzz input's payload) and PdfMergeFidelityTests.Edits_a_page_whose_rotation_is_not_a_quarter_turn. Both crash inputs and the two from the first run replay with no fault. Scoped Stryker over the two changed files kills every mutant on the changed lines. --- CHANGELOG.md | 7 ++++- docs/validation.md | 3 ++- .../Pdf/PdfDocumentInspection.Pages.cs | 16 ++++++++++-- .../Pdf/PdfImageFilters.cs | 6 ++++- .../PdfEditEngine.Overlays.cs | 2 +- .../PdfEditEngine.Pages.cs | 13 +++------- .../PdfEditEngine.cs | 2 +- .../PdfContentReaderTests.Images.cs | 15 +++++++---- .../PdfMergeFidelityTests.cs | 26 +++++++++++++++++++ 9 files changed, 69 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 29cdcc1..bf5ff64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -82,7 +82,12 @@ against an empty feed. read from a document is now written back from its own text. A real too large for the formatter's buffer no longer fails the write of the whole document (found by the `document` fuzz target in its first run). - **The image copy route refuses damaged Flate data with `PdfSourceFormatException`** instead of letting the - decompressor's `InvalidDataException` reach the caller (found by the `flate` fuzz target). + decompressor's `InvalidDataException`, or the `ZLibException` it throws for some inputs, reach the caller + (both found by the `flate` fuzz target). +- **An edit reads a page's rotation the way the inspection does.** A `/Rotate` that is not a multiple of 90 + made the edit engine throw `ArgumentOutOfRangeException` (found by the `document` fuzz target), and a real + or indirect `/Rotate` was read as no rotation at all. Both now take the nearest quarter turn, the rotation + the inspection already reported, and a value beyond the range of `int` is reduced instead of wrapped. - **Annotations satisfy the Arlington model.** A free text annotation states the `/DA` ISO 32000-1 requires, and every annotation appearance states its `/Resources` (found by the Arlington check in its first run). - **A cross-reference table whose offsets moved is rebuilt.** Bytes inserted or removed after a table was diff --git a/docs/validation.md b/docs/validation.md index 3491b07..7a2a36f 100644 --- a/docs/validation.md +++ b/docs/validation.md @@ -100,7 +100,8 @@ Fixed since the list was first drawn up, each with the test that fails without i | `UseIncrementalUpdate` appended a renumbered copy of the whole document, rewriting the signature dictionary after signing | the edited document's objects keep their numbers, and an object written exactly as the source states it is left out: rotating one page of a signed document appends 716 bytes and three objects instead of 20,969 bytes and twelve, and pyHanko places the signature at `ENTIRE_REVISION` | `PdfIncrementalInvariantTests.Appends_only_the_objects_an_edit_changes`, `Leaves_the_signature_dictionary_out_of_an_appended_revision` | | A merge copied a page that states no `/MediaBox` (and inherits none) without one | the page is written with the A4 box the inspection describes it with, and `PageSizeAssumed` says so | `PdfMergeFidelityTests.Writes_a_media_box_for_a_page_that_has_none` | | Reals a merge or an edit copied were re-formatted to five decimals (`0.000012` became `0.00001`), and a real too large for the formatter's buffer failed the write of the whole document — the first crash the `document` fuzz target found | a real read from a document is written from its own text when that text is a well-formed PDF real; other values are formatted at any magnitude | `PdfMergeFidelityTests.Writes_the_reals_it_copies_as_the_source_states_them`, `Writes_a_real_of_any_magnitude_in_the_form_pdf_defines` | -| The copy route of an image let the decompressor's `InvalidDataException` escape for damaged Flate data — the crash the `flate` fuzz target found | the copy route refuses damaged data with `PdfSourceFormatException`, with or without a predictor | `PdfContentReaderTests.Refuses_to_copy_an_image_whose_flate_data_is_damaged` | +| The copy route of an image let the decompressor's `InvalidDataException`, and for other inputs its `ZLibException`, escape for damaged Flate data — the two crashes the `flate` fuzz target found | the copy route refuses damaged data with `PdfSourceFormatException`, with or without a predictor | `PdfContentReaderTests.Refuses_to_copy_an_image_whose_flate_data_is_damaged` | +| The edit engine threw `ArgumentOutOfRangeException` for a `/Rotate` that is not a multiple of 90, and read a real or indirect one as none — the second crash the `document` fuzz target found | the edit reads the rotation as the inspection does, to the nearest quarter turn | `PdfMergeFidelityTests.Edits_a_page_whose_rotation_is_not_a_quarter_turn` | | A free text annotation was written without the `/DA` ISO 32000-1 requires, and annotation appearances without `/Resources` — both reported by the Arlington check's first run | `/DA` states the font, size and colour the appearance draws with; every appearance states its resources, empty if it needs none | `PdfDocumentEditorTests.States_the_default_appearance_and_resources_of_a_free_text_annotation`, `States_resources_on_an_appearance_that_needs_none` | | A table that parsed but whose offsets had moved was used as it was, and reconstruction could not see into object streams | the table is rebuilt when its offsets miss their objects; reconstruction indexes object streams and takes `/Root` from a cross-reference stream's dictionary | `PdfReaderRecoveryTests.Rebuilds_a_table_whose_offsets_no_longer_point_at_their_objects`, `Rebuilds_a_table_from_object_streams_when_the_cross_reference_stream_is_lost`, `Rebuilds_a_document_whose_catalog_lives_in_an_object_stream` | | AESV2 with a short declared key, wrapped predictor row sizes, truncated `endstream` candidates, ignored generations | sixteen-byte AESV2 keys; 64-bit row sizes on both predictor paths; Adler-32-checked candidates, at most sixteen; a mismatched generation resolves to null | `PdfMergeEngineTests.Opens_an_aesv2_document_whose_crypt_filter_declares_a_short_key`, `PdfPredictorTests.Refuses_a_predicted_row_larger_than_the_engine_decodes`, `PdfStreamParsingTests.Scans_past_a_candidate_whose_compressed_data_is_cut_short`, `PdfReaderRecoveryTests.Resolves_a_reference_to_a_generation_the_table_does_not_list_as_null` | diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfDocumentInspection.Pages.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfDocumentInspection.Pages.cs index 86f20a6..10e6bfd 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfDocumentInspection.Pages.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfDocumentInspection.Pages.cs @@ -72,7 +72,16 @@ private static List ReadPages(PdfFileReader reader, List 0d && height > 0d ? PdfPageSize.FromPoints(width, height) : null; } - private static PageRotation ReadRotation(PdfFileReader reader, SourcePage page) + /// Reads the rotation a page declares or inherits, as the nearest quarter turn. + /// The document. + /// The page. + /// The rotation; when none is declared. + /// + /// ISO 32000-1 requires a multiple of 90; a value that is not one, or that is a real or an indirect + /// object, is read the way other readers read it rather than refused. The inspection and the edit engine + /// both read it here, so an edit keeps the rotation the inspection reported. + /// + internal static PageRotation ReadRotation(PdfFileReader reader, SourcePage page) { var value = ReadNumber(reader, page.Dictionary.Get("Rotate")) ?? ReadNumber(reader, page.Inherited.Get("Rotate")); @@ -82,7 +91,10 @@ private static PageRotation ReadRotation(PdfFileReader reader, SourcePage page) return PageRotation.None; } - var normalized = (((int)Math.Round(value.Value / 90d) * 90) % 360 + 360) % 360; + // The quarter turns are counted in floating point and reduced before they become an integer, so a + // declared value beyond the range of int is reduced rather than wrapped. + var quarterTurns = Math.Round(value.Value / 90d) % 4d; + var normalized = (((int)quarterTurns * 90) % 360 + 360) % 360; return normalized switch { diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfImageFilters.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfImageFilters.cs index dc7d8f2..db1eed9 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfImageFilters.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfImageFilters.cs @@ -170,7 +170,11 @@ await InflatePredictorToAsync(stream.Raw, predictor, destination, cancellationTo await InflateToAsync(stream.Raw, destination, cancellationToken).ConfigureAwait(false); } } - catch (InvalidDataException exception) + // The decompressor reports most damage as InvalidDataException and some as ZLibException, an + // IOException the runtime does not make public, so it is recognised by name: the destination's own + // IOException is the caller's, and is not turned into a verdict on the document. + catch (Exception exception) when (exception is InvalidDataException + || exception.GetType().FullName == "System.IO.Compression.ZLibException") { throw new PdfSourceFormatException( sourceName, diff --git a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Overlays.cs b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Overlays.cs index 23e7ab5..84f8af6 100644 --- a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Overlays.cs +++ b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Overlays.cs @@ -439,7 +439,7 @@ private sealed class OverlayLayer( public PdfOverlayLayer Layer { get; } = layer; /// Gets the rotation the layer page declares, which the form is placed with. - public PageRotation Rotation { get; } = ReadRotation(page); + public PageRotation Rotation { get; } = ReadRotation(reader, page); /// Gets or sets the object number of the form XObject, or zero before it is written. public int FormNumber { get; set; } diff --git a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Pages.cs b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Pages.cs index dbda7f5..0098155 100644 --- a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Pages.cs +++ b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Pages.cs @@ -490,16 +490,11 @@ private static void WritePagesNode(PdfWriter writer, int pagesNodeNumber, ListReads the rotation a page declares, before any edit is applied. + /// The document the page belongs to. /// The page. /// The normalised rotation. - private static PageRotation ReadRotation(SourcePage page) - { - var declared = page.Dictionary.GetInteger("Rotate") - ?? page.Inherited.GetInteger("Rotate") - ?? 0; - - return PageRotationExtensions.Normalize((int)declared); - } + private static PageRotation ReadRotation(PdfFileReader reader, SourcePage page) => + Pdf.PdfDocumentInspection.ReadRotation(reader, page); private static void EnsurePage(List layout, int pageNumber, int index, string sourceName) { @@ -1033,7 +1028,7 @@ private sealed class EditedPage(PdfFileReader reader, SourcePage source, PdfRect public int TargetNumber { get; set; } public PageRotation EffectiveRotation => - PageRotationExtensions.Normalize(ReadRotation(Source).Degrees + AddedRotation.Degrees); + PageRotationExtensions.Normalize(ReadRotation(Reader, Source).Degrees + AddedRotation.Degrees); public EditedPage Copy() { diff --git a/src/SharpCorePDF.Infrastructure/PdfEditEngine.cs b/src/SharpCorePDF.Infrastructure/PdfEditEngine.cs index 7fb4d3b..117e20f 100644 --- a/src/SharpCorePDF.Infrastructure/PdfEditEngine.cs +++ b/src/SharpCorePDF.Infrastructure/PdfEditEngine.cs @@ -366,7 +366,7 @@ private void Verify(PdfCaptureStream? capture, List layout, string? for (var index = 0; index < layout.Count; index++) { var expected = layout[index].EffectiveRotation; - var actual = ReadRotation(reader.Pages[index]); + var actual = ReadRotation(reader, reader.Pages[index]); if (actual != expected) { diff --git a/tests/SharpCorePDF.Tests/PdfContentReaderTests.Images.cs b/tests/SharpCorePDF.Tests/PdfContentReaderTests.Images.cs index 8224248..b1ecde4 100644 --- a/tests/SharpCorePDF.Tests/PdfContentReaderTests.Images.cs +++ b/tests/SharpCorePDF.Tests/PdfContentReaderTests.Images.cs @@ -17,18 +17,23 @@ public sealed partial class PdfContentReaderTests /// /// The copy route of an image whose Flate data is damaged refuses it with the contract's exception, whether a /// predictor is declared or not: the decompressor's own exception used to reach the caller as if the library - /// had failed. + /// had failed. The decompressor reports damage as or, for some inputs — + /// the second data, found by the flate fuzz target — as a ZLibException, an + /// ; both are refused. /// /// The decode parameters the image declares. + /// The damaged Flate data, as hexadecimal. [Theory] - [InlineData("/DecodeParms << /Predictor 12 /Colors 1 /BitsPerComponent 8 /Columns 4 >>")] - [InlineData("")] - public async Task Refuses_to_copy_an_image_whose_flate_data_is_damaged(string parameters) + [InlineData("/DecodeParms << /Predictor 12 /Colors 1 /BitsPerComponent 8 /Columns 4 >>", "789CFFFFFFFF000102030405")] + [InlineData("", "789CFFFFFFFF000102030405")] + [InlineData("/DecodeParms << /Predictor 12 /Colors 1 /BitsPerComponent 8 /Columns 4 >>", "08B85B00C3000040")] + [InlineData("", "08B85B00C3000040")] + public async Task Refuses_to_copy_an_image_whose_flate_data_is_damaged(string parameters, string data) { var builder = new PdfBuilder(); var catalog = builder.Reserve(); var pages = builder.Reserve(); - var damaged = new byte[] { 0x78, 0x9C, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05 }; + var damaged = Convert.FromHexString(data); builder.Add("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 100 100] /Resources << /XObject << /Im0 5 0 R >> >> /Contents 4 0 R >>"); builder.AddStream(string.Empty, "q 50 0 0 50 0 0 cm /Im0 Do Q"u8.ToArray()); diff --git a/tests/SharpCorePDF.Tests/PdfMergeFidelityTests.cs b/tests/SharpCorePDF.Tests/PdfMergeFidelityTests.cs index 5309569..ec81fd6 100644 --- a/tests/SharpCorePDF.Tests/PdfMergeFidelityTests.cs +++ b/tests/SharpCorePDF.Tests/PdfMergeFidelityTests.cs @@ -93,6 +93,32 @@ public async Task Keeps_the_media_box_a_page_states() Assert.DoesNotContain(result.Warnings, warning => warning.Kind == PdfWarningKind.PageSizeAssumed); } + /// + /// A page whose /Rotate is not a multiple of 90 — which ISO 32000-1 requires it to be — is edited + /// with the nearest quarter turn, the rotation the inspection already reports for it; the edit engine used + /// to throw for it (found by the document fuzz target). + /// + /// The rotation the page declares. + /// The rotation the edited page carries. + [Theory] + [InlineData("91", 90)] + [InlineData("-91", 270)] + [InlineData("44", 0)] + [InlineData("180.4", 180)] + [InlineData("99999999999", 270)] + public async Task Edits_a_page_whose_rotation_is_not_a_quarter_turn(string declared, int expected) + { + var editor = new PdfDocumentEditorService(new PdfDocumentReader(), new PdfEditEngine()); + var edited = await editor.ApplyToArrayAsync( + new PdfByteArraySource(Page($"/Rotate {declared}"), "rotated.pdf"), + PdfEditPlan.Create().SetMetadata(new PdfMetadata { Title = "rotated" }), + cancellationToken: Token); + + var rotate = new PdfFileReader(edited, "edited.pdf", PdfEngineOptions.Default).Pages[0].Dictionary.GetInteger("Rotate") ?? 0; + + Assert.Equal(expected, rotate); + } + private static PdfMergerService Merger() => new(new PdfDocumentReader(), new PdfMergeEngine()); private static byte[] Page(string entries) From f529b02cf089c0056a6c6998b90517c150f88bf6 Mon Sep 17 00:00:00 2001 From: Andrew Tierney Date: Mon, 16 Nov 2026 14:09:45 +1100 Subject: [PATCH 10/15] Bound the outline walk of a merge by depth and copy each bookmark once The third robustness run, on 7c97a38, found a stack overflow (crash-d34275a3): a merge walks the outline recursively, one frame per level, and a source nested about fifty thousand levels deep ended the process. The walk now stops at 256 levels and reports BookmarksTruncated, and a bookmark reached a second time - a /First or /Next pointing back up the tree, or a child two parents share - is copied once and reported with BookmarkSkipped, which also ends cycles and the repeated walks of shared subtrees. Tests: PdfMergeOutlineLimitsTests (depth limit, shared child, self cycle). The crash input replays with no fault; the full suite passes. --- .../PdfMergeEngine.Outlines.cs | 72 ++++++++++++--- .../PdfMergeOutlineLimitsTests.cs | 90 +++++++++++++++++++ 2 files changed, 152 insertions(+), 10 deletions(-) create mode 100644 tests/SharpCorePDF.Tests/PdfMergeOutlineLimitsTests.cs diff --git a/src/SharpCorePDF.Infrastructure/PdfMergeEngine.Outlines.cs b/src/SharpCorePDF.Infrastructure/PdfMergeEngine.Outlines.cs index 6011ea8..4c8d7fc 100644 --- a/src/SharpCorePDF.Infrastructure/PdfMergeEngine.Outlines.cs +++ b/src/SharpCorePDF.Infrastructure/PdfMergeEngine.Outlines.cs @@ -17,6 +17,13 @@ public sealed partial class PdfMergeEngine /// The largest number of outline items that is copied, as a guard against damaged trees. private const int MaxOutlineItems = 100_000; + /// + /// The deepest level of bookmarks that is copied. The outline is walked recursively, one frame per level, + /// and a hostile file nested fifty thousand levels deep overflowed the stack and ended the process; a real + /// outline is a handful of levels deep. + /// + private const int MaxOutlineDepth = 256; + /// The name of the entry that holds the first child of a bookmark or of the outline root. private const string FirstEntry = "First"; @@ -54,8 +61,9 @@ public sealed partial class PdfMergeEngine } var linkage = new OutlineLinkage(); - var nodes = ReadChain(source.Reader, outline.Get(FirstEntry), context, source); - var chain = CopyChain(context, source, nodes, parent, linkage); + var walk = new OutlineWalk(); + var nodes = ReadChain(source.Reader, outline.Get(FirstEntry), context, source, walk); + var chain = CopyChain(context, source, nodes, parent, linkage, walk, depth: 0); if (!chain.IsEmpty) { @@ -102,12 +110,14 @@ public sealed partial class PdfMergeEngine /// The /First entry of the parent, or the /Next chain start. /// The merge context. /// The source being read. + /// The bookmarks of the source read so far. /// The items of the chain, in document order. private static List ReadChain( PdfFileReader reader, PdfObject? first, PdfMergeContext context, - MergeSource source) + MergeSource source, + OutlineWalk walk) { var nodes = new List(); @@ -135,6 +145,17 @@ private static List ReadChain( break; } + // A bookmark reached a second time — through a /Next or /First that points back up the tree, or + // a child two parents share — is not copied again: a cycle would never end, and a shared child + // is walked once for each of its parents, which grows with the product of the levels. + if (!walk.Visited.Add(reference.ObjectNumber)) + { + context.Warnings.Add(new( + PdfWarningKind.BookmarkSkipped, + $"'{source.DisplayName}': a bookmark is reached twice in the outline, which a tree does not allow; it was copied once.")); + break; + } + nodes.Add(new OutlineNode(reference.ObjectNumber, dictionary)); reference = dictionary.GetReference("Next"); } @@ -155,13 +176,17 @@ private static List ReadChain( /// The bookmarks of this level, in document order. /// The object number the bookmarks must point at as their parent. /// The chain this level belongs to, used for links to the sibling level. + /// The bookmarks of the source read so far. + /// The level of the chain, zero for the top level. /// What was copied. private static OutlineChain CopyChain( PdfMergeContext context, MergeSource source, List nodes, OutlineParent parent, - OutlineLinkage linkage) + OutlineLinkage linkage, + OutlineWalk walk, + int depth) { if (nodes.Count == 0) { @@ -182,12 +207,10 @@ private static OutlineChain CopyChain( var position = index; var node = nodes[position]; var itemParent = new OutlineParent { Target = targets[position] }; - var children = CopyChain( - context, - source, - ReadChain(source.Reader, node.Dictionary.Get(FirstEntry), context, source), - itemParent, - new OutlineLinkage()); + var below = depth + 1 < MaxOutlineDepth + ? ReadChain(source.Reader, node.Dictionary.Get(FirstEntry), context, source, walk) + : Truncate(node, context, source, walk); + var children = CopyChain(context, source, below, itemParent, new OutlineLinkage(), walk, depth + 1); descendants += children.TotalItems; @@ -202,6 +225,35 @@ private static OutlineChain CopyChain( return new OutlineChain(targets[0], targets[^1], nodes.Count, descendants); } + /// Stops the walk at the deepest level copied, saying so once per source. + /// The bookmark whose children are not copied. + /// The merge context. + /// The source being read. + /// The bookmarks of the source read so far. + /// No children. + private static List Truncate(OutlineNode node, PdfMergeContext context, MergeSource source, OutlineWalk walk) + { + if (node.Dictionary.Get(FirstEntry) is not null && !walk.WasTruncated) + { + walk.WasTruncated = true; + context.Warnings.Add(new( + PdfWarningKind.BookmarksTruncated, + $"'{source.DisplayName}': the bookmarks are nested more than {MaxOutlineDepth} levels deep; the levels below were skipped.")); + } + + return []; + } + + /// What the walk of one source's outline has seen. + private sealed class OutlineWalk + { + /// Gets the object numbers of the bookmarks read so far. + public HashSet Visited { get; } = []; + + /// Gets or sets a value indicating whether a level was cut at the depth limit. + public bool WasTruncated { get; set; } + } + /// A parent object number that is filled in after the outline has been walked. private sealed class OutlineParent { diff --git a/tests/SharpCorePDF.Tests/PdfMergeOutlineLimitsTests.cs b/tests/SharpCorePDF.Tests/PdfMergeOutlineLimitsTests.cs new file mode 100644 index 0000000..fa45b61 --- /dev/null +++ b/tests/SharpCorePDF.Tests/PdfMergeOutlineLimitsTests.cs @@ -0,0 +1,90 @@ +using SharpCorePDF.Application.Services; +using SharpCorePDF.Domain.Aggregates; +using SharpCorePDF.Domain.Entities; +using SharpCorePDF.Domain.ValueObjects; +using SharpCorePDF.Infrastructure; +using SharpCorePDF.Infrastructure.Sources; + +namespace SharpCorePDF.Tests; + +/// +/// Tests of how a merge copies an outline that is not a tree: nested deeper than it walks, or reaching a +/// bookmark twice. The first crash of this kind the document fuzz target found overflowed the stack. +/// +public sealed class PdfMergeOutlineLimitsTests +{ + private static CancellationToken Token => TestContext.Current.CancellationToken; + + /// An outline nested deeper than 256 levels is copied to that depth, and the merge says so. + [Fact] + public async Task Copies_an_outline_nested_deeper_than_it_walks_to_its_limit() + { + const int Levels = 300; + var bookmarks = new List(); + + // Bookmark i is object 4 + i, its parent object 3 + i (object 3 is the outline root). + for (var level = 0; level < Levels; level++) + { + var first = level + 1 < Levels ? $"/First {5 + level} 0 R /Last {5 + level} 0 R /Count 1" : string.Empty; + bookmarks.Add($"<< /Title (level {level}) /Parent {3 + level} 0 R {first} >>"); + } + + var result = await MergeAsync(bookmarks); + + Assert.Contains(result.Warnings, warning => warning.Kind == PdfWarningKind.BookmarksTruncated); + } + + /// + /// A bookmark two parents share is copied once, and the merge says so; it was walked again under every + /// parent, which grows with the product of the levels, and a bookmark that points back up the tree never + /// ended. + /// + [Fact] + public async Task Copies_a_bookmark_reached_twice_once() + { + var result = await MergeAsync( + [ + "<< /Title (a) /Parent 3 0 R /Next 6 0 R /First 5 0 R /Last 5 0 R /Count 1 >>", + "<< /Title (shared) /Parent 4 0 R >>", + "<< /Title (b) /Parent 3 0 R /Prev 4 0 R /First 5 0 R /Last 5 0 R /Count 1 >>", + ]); + + Assert.Contains(result.Warnings, warning => warning.Kind == PdfWarningKind.BookmarkSkipped); + } + + /// A bookmark whose child is its own parent ends instead of recursing until the stack runs out. + [Fact] + public async Task Copies_a_bookmark_that_is_its_own_child_once() + { + var result = await MergeAsync(["<< /Title (loop) /Parent 3 0 R /First 4 0 R /Last 4 0 R /Count 1 >>"]); + + Assert.Contains(result.Warnings, warning => warning.Kind == PdfWarningKind.BookmarkSkipped); + } + + /// Merges a one page document whose outline root (object 3) holds the given bookmarks, objects 4 on. + private static async Task MergeAsync(IReadOnlyList bookmarks) + { + var builder = new PdfBuilder(); + var catalog = builder.Reserve(); + var pages = builder.Reserve(); + var outline = builder.Reserve(); + + foreach (var bookmark in bookmarks) + { + builder.Add(bookmark); + } + + var page = builder.Add($"<< /Type /Page /Parent {pages} 0 R /MediaBox [0 0 200 200] >>"); + + builder.Set(catalog, $"<< /Type /Catalog /Pages {pages} 0 R /Outlines {outline} 0 R >>"); + builder.Set(pages, $"<< /Type /Pages /Kids [{page} 0 R] /Count 1 >>"); + builder.Set(outline, "<< /Type /Outlines /First 4 0 R /Last 4 0 R >>"); + + using var destination = new MemoryStream(); + + return await new PdfMergerService(new PdfDocumentReader(), new PdfMergeEngine()).MergeAsync( + MergePlan.Create().AddSource(PdfSource.Create(new PdfByteArraySource(builder.Build(catalog), "outline.pdf"))), + destination, + Token); + } +} From 71485c034881dcf1a503143851d185fa70f623a3 Mon Sep 17 00:00:00 2001 From: Andrew Tierney Date: Mon, 16 Nov 2026 19:08:45 +1100 Subject: [PATCH 11/15] Pass over the offsets a rebuilt table cannot read The robustness run on f529b02 found a crash in the document fuzz target (crash-e154541f). Reconstruction keeps the damaged table's entries beside the ones its scan finds, and the search for a cross-reference stream's trailer sliced the document at each of their offsets unchecked. An offset past the end of the file, or one beyond the range of an index that wraps negative when cast, threw ArgumentOutOfRangeException instead of letting the rebuild go on to find the catalog. Such an entry is now passed over. The catalog search needs no such check: PdfLexer clamps the position it starts at, which a scoped Stryker run confirmed by leaving the same guard there unkillable, so it was not added. Test that fails without the change: PdfReaderRecoveryTests.Rebuilds_a_table_that_states_an_offset_past_the_end_of_the_file, for an offset past the end and one that wraps negative. The crash input replays with no fault. On the changed lines the two surviving mutants are equivalent: offset 0 is the header and an offset equal to the length gives an empty window, so neither boundary changes what is read. The outline-depth fix of f529b02 is now in CHANGELOG.md and the fixed-defects table of docs/validation.md, which it had not been. --- CHANGELOG.md | 10 +++++ docs/validation.md | 2 + .../Pdf/PdfCrossReferenceReader.Scan.cs | 13 +++++- .../PdfReaderRecoveryTests.cs | 40 +++++++++++++++++++ 4 files changed, 64 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bf5ff64..7551519 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -110,6 +110,16 @@ against an empty feed. files tree that only names an external file is listed, with a warning and a length of zero; copying it threw `ArgumentException` as if the caller had passed a file of another document (found by the veraPDF corpus run, `isartor-6-1-11-t01-fail-a.pdf`). +- **A merge walks a deep or looping outline without ending the process.** The outline was copied + recursively, one stack frame per level, and a source nested about fifty thousand levels deep overflowed + the stack. The walk stops at 256 levels with `BookmarksTruncated`, and a bookmark reached a second time + — a `/First` or `/Next` that points back up the tree, or a child two parents share — is copied once and + reported with `BookmarkSkipped` (found by the `document` fuzz target). +- **Rebuilding a table passes over the offsets it cannot read.** Reconstruction keeps the damaged table's + entries beside the ones its scan finds, and the search for a cross-reference stream's trailer sliced the + document at each of their offsets unchecked: one past the end of the file, or beyond the range of an + index, threw `ArgumentOutOfRangeException` instead of letting the rebuild find the catalog (found by the + `document` fuzz target). - **Run-length images are decoded.** `RunLengthDecode` joins Flate, LZW and the two ASCII filters: a bilevel or scan-style image that declares it now yields its samples — and `Gray8`, `Rgb24` or `Cmyk32` diff --git a/docs/validation.md b/docs/validation.md index 7a2a36f..83233fe 100644 --- a/docs/validation.md +++ b/docs/validation.md @@ -106,6 +106,8 @@ Fixed since the list was first drawn up, each with the test that fails without i | A table that parsed but whose offsets had moved was used as it was, and reconstruction could not see into object streams | the table is rebuilt when its offsets miss their objects; reconstruction indexes object streams and takes `/Root` from a cross-reference stream's dictionary | `PdfReaderRecoveryTests.Rebuilds_a_table_whose_offsets_no_longer_point_at_their_objects`, `Rebuilds_a_table_from_object_streams_when_the_cross_reference_stream_is_lost`, `Rebuilds_a_document_whose_catalog_lives_in_an_object_stream` | | AESV2 with a short declared key, wrapped predictor row sizes, truncated `endstream` candidates, ignored generations | sixteen-byte AESV2 keys; 64-bit row sizes on both predictor paths; Adler-32-checked candidates, at most sixteen; a mismatched generation resolves to null | `PdfMergeEngineTests.Opens_an_aesv2_document_whose_crypt_filter_declares_a_short_key`, `PdfPredictorTests.Refuses_a_predicted_row_larger_than_the_engine_decodes`, `PdfStreamParsingTests.Scans_past_a_candidate_whose_compressed_data_is_cut_short`, `PdfReaderRecoveryTests.Resolves_a_reference_to_a_generation_the_table_does_not_list_as_null` | | A revision appended to a file that uses cross-reference streams ended in a classic table | the revision ends in a cross-reference stream; qpdf and pyHanko accept both an edit and two signatures appended this way | `PdfIncrementalWriterTests.Appends_a_cross_reference_stream_to_a_document_that_uses_them`, `PdfDocumentSignerTests.Signs_a_document_that_uses_cross_reference_streams_with_one` | +| A merge copied the outline recursively, so a source nested about fifty thousand levels deep overflowed the stack; a looping or shared outline was walked again at every visit — a crash the `document` fuzz target found | the walk stops at 256 levels with `BookmarksTruncated`; a bookmark reached twice is copied once with `BookmarkSkipped` | `PdfMergeOutlineLimitsTests` | +| Rebuilding a table sliced the document at offsets the damaged table stated past its end, or beyond an index's range, and threw `ArgumentOutOfRangeException` — a crash the `document` fuzz target found | an offset outside the document is passed over, and the rebuild finds the catalog | `PdfReaderRecoveryTests.Rebuilds_a_table_that_states_an_offset_past_the_end_of_the_file` | | Copying an attachment the listing reported, whose specification embeds no stream, threw `ArgumentException` — the public-corpora job's fault on `isartor-6-1-11-t01-fail-a.pdf` | it writes no bytes, matching the `ByteLength` of zero the listing reports | `PdfContentReaderTests.Copies_nothing_for_a_listed_file_that_embeds_no_stream` | ## Corpus readings, 2026-11-16 diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.Scan.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.Scan.cs index d0218ed..41b31d9 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.Scan.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.Scan.cs @@ -204,6 +204,17 @@ private bool MentionsObjectStream(long offset) return window.IndexOf("/ObjStm"u8) >= 0; } + /// Determines whether an offset names a byte of the document. + /// The offset. + /// when the offset is inside the data. + /// + /// Reconstruction keeps the entries of the table it is repairing beside the ones the scan found, and those + /// offsets are whatever the damaged table stated: past the end of the file, negative, or beyond the range + /// of an , which wraps to a negative index when it is cast. A slice of the data needs + /// this check; a parser does not, because clamps the position it starts at. + /// + private bool IsInside(long offset) => offset >= 0 && offset < data.Length; + /// Parses the object whose header is at an offset. /// The offset. /// The object, or . @@ -223,7 +234,7 @@ private bool MentionsObjectStream(long offset) private PdfDictionary? FindCrossReferenceStreamTrailer() { foreach (var offset in table.Entries.Values - .Where(entry => entry.Type == XrefEntryType.Uncompressed) + .Where(entry => entry.Type == XrefEntryType.Uncompressed && IsInside(entry.Offset)) .Select(entry => entry.Offset) .OrderByDescending(offset => offset)) { diff --git a/tests/SharpCorePDF.Tests/PdfReaderRecoveryTests.cs b/tests/SharpCorePDF.Tests/PdfReaderRecoveryTests.cs index 8666a25..7929ee8 100644 --- a/tests/SharpCorePDF.Tests/PdfReaderRecoveryTests.cs +++ b/tests/SharpCorePDF.Tests/PdfReaderRecoveryTests.cs @@ -19,6 +19,46 @@ namespace SharpCorePDF.Tests; /// public sealed class PdfReaderRecoveryTests { + /// + /// A table whose trailer names no catalog is rebuilt, and an entry the damaged table states past the end + /// of the file is passed over rather than read: reconstruction keeps the table's own entries beside the + /// ones it scans, and the offset of such an entry is whatever the damage left — the document fuzz target + /// found it sliced the data at one and threw. Cast to an index, 9,999,999,999 lands past the end and + /// 3,000,000,000 wraps to a negative one; the search for a cross-reference stream's trailer sliced the data + /// at either. The entry is object 1, ahead of the catalog, so every search meets it before the catalog. + /// + /// The ten-digit offset the damaged table states. + [Theory] + [InlineData(9_999_999_999L)] + [InlineData(3_000_000_000L)] + public void Rebuilds_a_table_that_states_an_offset_past_the_end_of_the_file(long offset) + { + var output = new MemoryStream(); + var offsets = new long[5]; + + void Write(string text) => output.Write(Encoding.Latin1.GetBytes(text)); + + Write("%PDF-1.7\n"); + offsets[2] = output.Position; + Write("2 0 obj\n<< /Type /Catalog /Pages 3 0 R >>\nendobj\n"); + offsets[3] = output.Position; + Write("3 0 obj\n<< /Type /Pages /Kids [4 0 R] /Count 1 >>\nendobj\n"); + offsets[4] = output.Position; + Write("4 0 obj\n<< /Type /Page /Parent 3 0 R /MediaBox [0 0 200 200] >>\nendobj\n"); + + var xref = output.Position; + Write("xref\n0 5\n0000000000 65535 f \n"); + Write($"{offset:D10} 00000 n \n"); + Write($"{offsets[2]:D10} 00000 n \n{offsets[3]:D10} 00000 n \n{offsets[4]:D10} 00000 n \n"); + Write($"trailer\n<< /Size 5 >>\nstartxref\n{xref}\n%%EOF\n"); + + var reader = new PdfFileReader(output.ToArray(), "past-the-end.pdf", PdfEngineOptions.Default); + + Assert.True(reader.CrossReferenceTable.WasReconstructed); + Assert.Single(reader.Pages); + Assert.Null(reader.Resolve(1)); + } + /// /// A file whose bytes moved after its table was written — here three bytes inserted behind the header, the /// way a converted line ending or a prepended byte order mark moves them — has a table that parses and From 1f301e4a9513cdb3452f1ab5719e376b9b192fca Mon Sep 17 00:00:00 2001 From: Andrew Tierney Date: Mon, 16 Nov 2026 20:29:27 +1100 Subject: [PATCH 12/15] Resolve objects the way the cross-reference data places them A hybrid-reference file lists the objects it keeps in object streams as free in its classic table, so a reader that knows no streams skips them, and describes them in the stream the trailer's /XRefStm names. The table kept the first entry it met, so the free entry hid the real one and such a file read as having no pages. The stream may now answer for the numbers its own section's table frees, and is read before the revision /Prev points at, which is the order ISO 32000-1 7.5.8.4 gives. Separately, loading an object stream cached every object inside it, and Resolve consulted that cache before the table. A stream still holds the copies it held when it was written, so an object a later revision rewrote in the file body came back stale, and one it freed came back from the dead, depending on which object was resolved first. Only the numbers the table places in a stream are taken from it now, and the table is consulted first. PdfCrossReferencePrecedenceTests builds each case as a document; all four fail on the previous reader. --- .../Pdf/PdfCrossReferenceReader.Streams.cs | 20 +- .../Pdf/PdfCrossReferenceReader.Tables.cs | 24 +- .../Pdf/PdfCrossReferenceReader.cs | 27 +- .../Pdf/PdfFileReader.ObjectStreams.cs | 10 + .../Pdf/PdfFileReader.cs | 5 - .../PdfCrossReferencePrecedenceTests.cs | 326 ++++++++++++++++++ 6 files changed, 386 insertions(+), 26 deletions(-) create mode 100644 tests/SharpCorePDF.Tests/PdfCrossReferencePrecedenceTests.cs diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.Streams.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.Streams.cs index 7245f97..97b0fc7 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.Streams.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.Streams.cs @@ -11,7 +11,7 @@ namespace SharpCorePDF.Infrastructure.Pdf; /// internal sealed partial class PdfCrossReferenceReader { - private (long? Previous, long? Hybrid) ReadCrossReferenceStream(int offset, bool isNewest) + private (long? Previous, long? Hybrid) ReadCrossReferenceStream(int offset, bool isNewest, HashSet? freedByTable = null) { var parser = new PdfObjectParser(data, sourceName, offset, null, options.MaxNestingDepth); @@ -48,7 +48,7 @@ internal sealed partial class PdfCrossReferenceReader var size = dictionary.GetInteger("Size") ?? 0; var ranges = ReadIndexRanges(dictionary, size); - ReadEntries(decoded, widthType, widthField1, widthField2, ranges); + ReadEntries(decoded, widthType, widthField1, widthField2, ranges, freedByTable); var previous = dictionary.GetInteger("Prev"); table.MergeTrailer(dictionary, onlyMissingKeys: !isNewest); @@ -81,7 +81,8 @@ private void ReadEntries( int widthType, int widthField1, int widthField2, - List<(long First, long Count)> ranges) + List<(long First, long Count)> ranges, + HashSet? freedByTable) { var entrySize = widthType + widthField1 + widthField2; @@ -112,7 +113,18 @@ private void ReadEntries( _ => XrefEntryType.Free, }; - table.Add(new XrefEntry(first + index, field1, (int)field2, kind)); + var entry = new XrefEntry(first + index, field1, (int)field2, kind); + + // The stream of a hybrid section describes what that section's table lists as free, so it may + // replace those entries — and only those: a number a newer section freed stays freed. + if (freedByTable?.Contains(entry.ObjectNumber) == true && table.Get(entry.ObjectNumber) is { Type: XrefEntryType.Free }) + { + table.AddOrReplace(entry); + } + else + { + table.Add(entry); + } } } } diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.Tables.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.Tables.cs index de8aab7..71e2396 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.Tables.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.Tables.cs @@ -10,27 +10,31 @@ namespace SharpCorePDF.Infrastructure.Pdf; /// internal sealed partial class PdfCrossReferenceReader { - private (long? Previous, long? Hybrid) ReadClassicTable(int offset, bool isNewest) + private (long? Previous, long? Hybrid, HashSet? Freed) ReadClassicTable(int offset, bool isNewest) { var parser = new PdfObjectParser(data, sourceName, offset, null, options.MaxNestingDepth); + var freed = new HashSet(); parser.TryReadKeyword("xref"); - ReadSubsections(parser); + ReadSubsections(parser, freed); if (!parser.TryReadKeyword("trailer") || parser.ParseObject() is not PdfDictionary trailer) { - return (null, null); + return (null, null, null); } var previous = trailer.GetInteger("Prev"); var hybrid = trailer.GetInteger("XRefStm"); table.MergeTrailer(trailer, onlyMissingKeys: !isNewest); - return (previous, hybrid); + // The numbers this table frees are the ones a hybrid section's stream may describe: a hybrid file lists + // the objects it keeps in object streams as free here, so a reader that knows no streams skips them. + return (previous, hybrid, hybrid is null ? null : freed); } /// Reads the subsections of the table, which are as many as the file wrote. /// The parser positioned at the first subsection. - private void ReadSubsections(PdfObjectParser parser) + /// The set that receives the numbers the table lists as free. + private void ReadSubsections(PdfObjectParser parser, HashSet freed) { while (true) { @@ -42,7 +46,7 @@ private void ReadSubsections(PdfObjectParser parser) return; } - if (!ReadSubsectionEntries(parser, first, count)) + if (!ReadSubsectionEntries(parser, first, count, freed)) { return; } @@ -76,8 +80,9 @@ private static bool TryReadSubsectionHeader(PdfObjectParser parser, out long fir /// The parser to read from. /// The object number the subsection starts at. /// The number of entries the subsection announces. + /// The set that receives the numbers the table lists as free. /// when every announced entry was read. - private bool ReadSubsectionEntries(PdfObjectParser parser, long first, int count) + private bool ReadSubsectionEntries(PdfObjectParser parser, long first, int count, HashSet freed) { for (var index = 0; index < count; index++) { @@ -93,6 +98,11 @@ private bool ReadSubsectionEntries(PdfObjectParser parser, long first, int count entryOffset, generation, inUse ? XrefEntryType.Uncompressed : XrefEntryType.Free)); + + if (!inUse) + { + freed.Add(first + index); + } } return true; diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.cs index 7d04ddd..efb89fd 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfCrossReferenceReader.cs @@ -35,34 +35,37 @@ public PdfCrossReferenceReader(byte[] data, string sourceName, PdfEngineOptions /// The table is damaged beyond repair. public PdfCrossReferenceTable Read() { - var pending = new Queue<(long Offset, bool IsNewest)>(); + var pending = new Queue<(long Offset, bool IsNewest, HashSet? FreedByTable)>(); var visited = new HashSet(); if (FindStartXref() is { } start) { table.StartXrefOffset = start; - pending.Enqueue((start, true)); + pending.Enqueue((start, true, null)); } while (pending.Count > 0) { - var (offset, isNewest) = pending.Dequeue(); + var (offset, isNewest, freedByTable) = pending.Dequeue(); if (offset < 0 || offset >= data.Length || !visited.Add(offset)) { continue; } - var (previous, hybrid) = TryReadSection(offset, isNewest); + var (previous, hybrid, freed) = TryReadSection(offset, isNewest, freedByTable); - if (previous is { } previousOffset && previousOffset > 0 && previousOffset < data.Length) + // ISO 32000-1 7.5.8.4 gives the order a lookup follows: the section's table, the stream its + // trailer names in /XRefStm, and only then the revision /Prev points at. The table keeps the + // entry it meets first, so the stream is queued ahead of the older revision. + if (hybrid is { } hybridOffset && hybridOffset > 0 && hybridOffset < data.Length) { - pending.Enqueue((previousOffset, false)); + pending.Enqueue((hybridOffset, false, freed ?? [])); } - if (hybrid is { } hybridOffset && hybridOffset > 0 && hybridOffset < data.Length) + if (previous is { } previousOffset && previousOffset > 0 && previousOffset < data.Length) { - pending.Enqueue((hybridOffset, false)); + pending.Enqueue((previousOffset, false, null)); } } @@ -157,7 +160,7 @@ private bool HeaderNames(XrefEntry entry) return lexer.TryReadInteger(out var offset) ? offset : null; } - private (long? Previous, long? Hybrid) TryReadSection(long offset, bool isNewest) + private (long? Previous, long? Hybrid, HashSet? Freed) TryReadSection(long offset, bool isNewest, HashSet? freedByTable) { var probe = new PdfLexer(data, (int)offset); @@ -168,6 +171,10 @@ private bool HeaderNames(XrefEntry entry) table.NewestSectionIsStream |= isNewest; - return ReadCrossReferenceStream((int)offset, isNewest); + var (previous, hybrid) = ReadCrossReferenceStream((int)offset, isNewest, freedByTable); + + // The stream a hybrid section names describes the objects of that section only; the chain of + // revisions runs through the table's /Prev, so the stream's own /Prev, when it states one, is not followed. + return freedByTable is null ? (previous, hybrid, null) : (null, null, null); } } diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.ObjectStreams.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.ObjectStreams.cs index 0a5df17..189aac2 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.ObjectStreams.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.ObjectStreams.cs @@ -85,6 +85,16 @@ private void EnsureObjectStreamLoaded(long containerObjectNumber) continue; } + // The table decides where an object is. A stream still holds the copies it held when it was + // written — an object a later revision rewrote in the file body, or deleted — and only the + // numbers the table places in this stream are taken from it, so neither answers for the + // newer state. + if (CrossReferenceTable.Get(numbers[index]) is not { Type: XrefEntryType.Compressed } placed + || placed.Offset != containerObjectNumber) + { + continue; + } + var parser = new PdfObjectParser(decoded, SourceName, (int)position, null, options.MaxNestingDepth); fromObjectStreams[numbers[index]] = parser.ParseObject(); } diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.cs index 11ee20e..8e156ac 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfFileReader.cs @@ -164,11 +164,6 @@ public PdfDictionary? Catalog return cached; } - if (fromObjectStreams.TryGetValue(objectNumber, out var streamed)) - { - return streamed; - } - var entry = CrossReferenceTable.Get(objectNumber); if (entry is not { } xref || !xref.IsInUse) diff --git a/tests/SharpCorePDF.Tests/PdfCrossReferencePrecedenceTests.cs b/tests/SharpCorePDF.Tests/PdfCrossReferencePrecedenceTests.cs new file mode 100644 index 0000000..1ee46b8 --- /dev/null +++ b/tests/SharpCorePDF.Tests/PdfCrossReferencePrecedenceTests.cs @@ -0,0 +1,326 @@ +using System.Text; +using SharpCorePDF.Domain.ValueObjects; +using SharpCorePDF.Infrastructure; +using SharpCorePDF.Infrastructure.Pdf; + +namespace SharpCorePDF.Tests; + +/// +/// Tests of which version of an object the reader resolves when a document can answer in more than one +/// way: an object that lives in an object stream and also in the file body, and a hybrid-reference file whose +/// classic table lists as free the objects its cross-reference stream describes. +/// +/// +/// ISO 32000-1 makes the cross-reference data the authority on where an object is. A reader that lets an +/// object stream, or an older section, answer for a number the newest section places elsewhere shows a +/// different document than every conforming viewer does, and the inspection, the text, a signature check and a +/// merge would all describe or copy content nobody sees. +/// +public sealed class PdfCrossReferencePrecedenceTests +{ + private static readonly PdfEngineOptions Options = PdfEngineOptions.Default with + { + ReconstructCrossReferenceOnFailure = false + }; + + /// + /// An object stream that still holds an old copy of a number the table places in the file body does not + /// answer for it, whichever object is resolved first: resolving a neighbour loads the stream, and loading + /// it must not leave its stale copy where the next lookup finds it before the table is consulted. + /// + [Fact] + public void Resolves_the_object_the_table_names_not_the_copy_an_object_stream_still_holds() + { + using var document = new RawDocument(); + + document.Add(1, "<< /Type /Catalog /Pages 2 0 R >>"); + document.Add(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + document.Add(3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] >>"); + document.Add(10, "<< /Version /Current >>"); + + // Object stream 6 holds object 10 as it stood in an earlier revision, and object 11, which only it has. + document.AddObjectStream(6, ("10", "<< /Version /Stale >>"), ("11", "<< /Neighbour true >>")); + + var reader = new PdfFileReader( + document.BuildWithXrefStream(root: 1, + compressed: new Dictionary { [11] = (6, 1) }), + "stale.pdf", + Options); + + // 11 is resolved first, which loads stream 6 and everything in it. + Assert.True(Assert.IsType(reader.Resolve(11)).ContainsKey("Neighbour")); + + var resolved = Assert.IsType(reader.Resolve(10)); + + Assert.Equal("Current", resolved.GetName("Version")); + } + + /// + /// A number the table lists as free is not resolved from an object stream that happens to hold it: the + /// object was deleted, and a deleted object that reappears is one no other reader shows. + /// + [Fact] + public void Does_not_bring_back_an_object_the_table_lists_as_free() + { + using var document = new RawDocument(); + + document.Add(1, "<< /Type /Catalog /Pages 2 0 R >>"); + document.Add(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + document.Add(3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] >>"); + document.AddObjectStream(6, ("10", "<< /Deleted true >>"), ("11", "<< /Neighbour true >>")); + + var reader = new PdfFileReader( + document.BuildWithXrefStream(root: 1, + compressed: new Dictionary { [11] = (6, 1) }, free: [10]), + "deleted.pdf", + Options); + + Assert.NotNull(reader.Resolve(11)); + Assert.Null(reader.Resolve(10)); + } + + /// + /// A hybrid-reference file lists the objects it keeps in object streams as free in its classic table, so + /// a reader that does not know cross-reference streams skips them, and describes them in the stream the + /// trailer's /XRefStm names. A reader that does know the stream has to let it answer for those + /// numbers: keeping the table's free entry hides the page tree from the one reader the file was written for. + /// + [Fact] + public void Reads_an_object_a_hybrid_file_lists_as_free_in_its_table_and_describes_in_its_stream() + { + using var document = new RawDocument(); + + document.Add(1, "<< /Type /Catalog /Pages 2 0 R >>"); + document.Add(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + document.AddObjectStream(5, ("3", "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] >>")); + + var reader = new PdfFileReader( + document.BuildHybrid(root: 1, hidden: new Dictionary { [3] = (5, 0) }), + "hybrid.pdf", + Options); + + Assert.Single(reader.Pages); + Assert.Equal("Page", Assert.IsType(reader.Resolve(3)).GetName("Type")); + } + + /// + /// What the stream a hybrid file names says about a number is read before what an older revision says, + /// because the stream belongs to the newer section: the table, then its stream, then /Prev. + /// + [Fact] + public void Reads_the_stream_of_a_hybrid_section_before_the_revision_it_follows() + { + using var older = new RawDocument(); + + older.Add(1, "<< /Type /Catalog /Pages 2 0 R >>"); + older.Add(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + older.Add(3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 100 100] >>"); + + using var newer = older.Append(); + + newer.AddObjectStream(5, ("3", "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] >>")); + + var reader = new PdfFileReader( + newer.BuildHybrid(root: 1, hidden: new Dictionary { [3] = (5, 0) }), + "hybrid-update.pdf", + Options); + + var page = Assert.Single(reader.Pages); + var box = Assert.IsType(page.Dictionary.Get("MediaBox")); + + Assert.Equal(300, Assert.IsType(box.Items[2]).Value); + } + + /// Builds the bytes of a document object by object, with the cross-reference data written by hand. + private sealed class RawDocument : IDisposable + { + private readonly MemoryStream output = new(); + private readonly SortedDictionary offsets = []; + private readonly byte[] previous; + private readonly long previousXref; + + /// Releases the buffer the document is built in. + public void Dispose() => output.Dispose(); + + public RawDocument() + { + previous = []; + Write("%PDF-1.7\n"); + } + + private RawDocument(byte[] earlier, long earlierXref) + { + previous = earlier; + previousXref = earlierXref; + output.Write(earlier); + } + + /// Starts a revision appended to this document, which must have been built first. + public RawDocument Append() + { + var bytes = BuildClassic(root: 1); + + return new RawDocument(bytes, LastStartXref(bytes)); + } + + public void Add(int number, string body) + { + offsets[number] = output.Position; + Write($"{number} 0 obj\n{body}\nendobj\n"); + } + + /// Adds an uncompressed object stream whose first object is listed first in its header. + public void AddObjectStream(int number, params (string Number, string Body)[] objects) + { + var header = new StringBuilder(); + var body = new StringBuilder(); + + foreach (var (objectNumber, text) in objects) + { + header.Append(System.Globalization.CultureInfo.InvariantCulture, $"{objectNumber} {body.Length} "); + body.Append(text).Append('\n'); + } + + var content = header.ToString() + body; + + Add(number, + $"<< /Type /ObjStm /N {objects.Length} /First {header.Length} /Length {content.Length} >>\nstream\n{content}\nendstream"); + } + + /// Writes a classic table and trailer for every object added. + public byte[] BuildClassic(int root) + { + var start = output.Position; + var size = (offsets.Keys.Max()) + 1; + + Write($"xref\n0 {size}\n0000000000 65535 f \n"); + + for (var number = 1; number < size; number++) + { + Write(offsets.TryGetValue(number, out var offset) + ? $"{offset:D10} 00000 n \n" + : "0000000000 00000 f \n"); + } + + Write($"trailer\n<< /Size {size} /Root {root} 0 R >>\nstartxref\n{start}\n%%EOF\n"); + + return output.ToArray(); + } + + /// Writes one cross-reference stream: uncompressed entries for the objects added, compressed ones as given. + public byte[] BuildWithXrefStream( + int root, + Dictionary compressed, + int[]? free = null) + { + var streamNumber = Math.Max(offsets.Keys.Max(), compressed.Keys.Max()) + 1; + var size = streamNumber + 1; + var start = output.Position; + + offsets[streamNumber] = start; + + var table = new MemoryStream(); + + for (var number = 0; number < size; number++) + { + if (number == 0 || (free?.Contains(number) ?? false)) + { + Entry(table, 0, 0, 65535); + } + else if (compressed.TryGetValue(number, out var place)) + { + Entry(table, 2, place.Container, place.Index); + } + else if (offsets.TryGetValue(number, out var offset)) + { + Entry(table, 1, offset, 0); + } + else + { + Entry(table, 0, 0, 0); + } + } + + Write( + $"{streamNumber} 0 obj\n<< /Type /XRef /Size {size} /W [1 4 2] /Root {root} 0 R /Length {table.Length} >>\nstream\n"); + output.Write(table.ToArray()); + Write($"\nendstream\nendobj\nstartxref\n{start}\n%%EOF\n"); + + return output.ToArray(); + } + + /// + /// Writes a hybrid section: a classic table that lists every hidden object as free, and a + /// cross-reference stream, named by the trailer's /XRefStm, that describes them. + /// + public byte[] BuildHybrid(int root, Dictionary hidden) + { + var streamNumber = Math.Max(offsets.Keys.Max(), hidden.Keys.Max()) + 1; + var size = streamNumber + 1; + var streamStart = output.Position; + + offsets[streamNumber] = streamStart; + + var table = new MemoryStream(); + + // The stream lists only what the table hides, as a hybrid file does. + foreach (var (number, place) in hidden.OrderBy(pair => pair.Key)) + { + Entry(table, 2, place.Container, place.Index); + } + + var index = string.Join(' ', hidden.OrderBy(pair => pair.Key).Select(pair => $"{pair.Key} 1")); + + Write( + $"{streamNumber} 0 obj\n<< /Type /XRef /Size {size} /W [1 4 2] /Index [{index}] /Length {table.Length} >>\nstream\n"); + output.Write(table.ToArray()); + Write("\nendstream\nendobj\n"); + + var start = output.Position; + + // One subsection per number this section states, as an incremental update writes it: a number it + // does not mention is not freed, it keeps what the revision it follows says. + Write("xref\n"); + + if (previousXref == 0) + { + Write("0 1\n0000000000 65535 f \n"); + } + + foreach (var number in offsets.Keys.Union(hidden.Keys).Order()) + { + Write(hidden.ContainsKey(number) + ? $"{number} 1\n0000000000 00000 f \n" + : $"{number} 1\n{offsets[number]:D10} 00000 n \n"); + } + + var prev = previousXref > 0 ? $" /Prev {previousXref}" : string.Empty; + + Write( + $"trailer\n<< /Size {size} /Root {root} 0 R /XRefStm {streamStart}{prev} >>\nstartxref\n{start}\n%%EOF\n"); + + return output.ToArray(); + } + + private static long LastStartXref(byte[] bytes) + { + var text = Encoding.Latin1.GetString(bytes); + var at = text.LastIndexOf("startxref", StringComparison.Ordinal) + "startxref".Length; + + return long.Parse(text[at..].Trim().Split('\n')[0], System.Globalization.CultureInfo.InvariantCulture); + } + + private static void Entry(MemoryStream table, int kind, long field, int index) + { + table.WriteByte((byte)kind); + table.WriteByte((byte)(field >> 24)); + table.WriteByte((byte)(field >> 16)); + table.WriteByte((byte)(field >> 8)); + table.WriteByte((byte)field); + table.WriteByte((byte)(index >> 8)); + table.WriteByte((byte)index); + } + + private void Write(string text) => output.Write(Encoding.Latin1.GetBytes(text)); + } +} From a23fdc3b2fc69ffda44729a1fff117d31c3b4814 Mon Sep 17 00:00:00 2001 From: Andrew Tierney Date: Mon, 16 Nov 2026 20:29:28 +1100 Subject: [PATCH 13/15] Bound the forms a page paints and make the PDF/A conversion read what the profiles name The checks of the previous pull requests found a page that never finishes reading, and a PDF/A conversion that writes files veraPDF rejects. Forms. A form that paints the next form forty times, six levels down, is under the depth limit and the cycle guard, and a 3.6 KB document asked for four billion paints. The text walk and the image walk now count a page's paints against PdfEngineOptions.MaxFormPaintsPerPage (100,000) and its decoded form content against 256 MiB, report FormPaintLimitExceeded, and cache a form painted again. Field appearances state /Resources even when empty, as annotation appearances already did; the Arlington model of PDF 1.7 requires it. PDF/A. robustness.yml converted the veraPDF corpus and asked veraPDF to judge each result: 395 of 2,074 were written and rejected, 17 of them from sources that already conformed. The analysis never decoded a content stream or read an annotation's flags, a bookmark, or an action's /Next chain. It now sets the Print flag right (PdfAFeatureRepaired), refuses hidden annotations and types a profile does not list, drops the actions of widgets and fields and an XFA form, finds forbidden and named actions along chains and on bookmarks, pages and the open action with the set the profiles list (SubmitForm is permitted), refuses DeviceCMYK or DeviceRGB that the output intent does not cover (through Separation and DeviceN alternates and transparency groups), undefined operators and forbidden graphics-state keys, and keeps a usable output intent the source already states instead of relabelling it sRGB. Measured locally with veraPDF 1.30.4 on the ten clause groups changed (PDF_A-2b, 150 inputs meant to fail, 51 meant to pass): failing inputs 48 refused, 76 conforming, 26 still failing; passing inputs 39 conforming, 3 refused, 0 damaged. The 26 are listed by clause in docs/validation.md. A page painted in DeviceRGB under an intent that is not RGB is now refused, which PdfATests used to call the validator's question. Tests that fail without the changes: PdfFormBudgetTests, PdfAArchivalChecksTests (22 of 33 fail on the previous analysis) and PdfFormEditorTests.States_resources_on_the_appearance_of_a_button_that_draws_no_text. Scoped Stryker over the touched files scores 99.6% on the changed lines; the two survivors, a content array, are now killed. The Arlington check is blocking, with an accept list that gives a reason for each rule. --- .github/workflows/validation.yml | 29 +- CHANGELOG.md | 33 ++ docs/manual/src/24-pdf-a.md | 36 +- docs/release-checklist.md | 24 +- docs/tools/corpus-robustness.py | 28 +- docs/validation.md | 68 ++- .../PublicAPI.Unshipped.txt | 2 + .../ReadModels/PdfAConversionResult.cs | 15 +- .../ValueObjects/PdfWarningKind.cs | 13 + .../Authoring/PdfAAnnotationRules.cs | 197 ++++++++ .../Authoring/PdfAContentChecks.cs | 366 ++++++++++++++ .../Authoring/PdfARequirements.cs | 282 ++++++++++- .../Authoring/PdfAWriter.cs | 56 ++- .../Forms/FieldAppearance.cs | 11 +- .../Pdf/PdfFormBudget.cs | 100 ++++ .../Pdf/PdfTextCollector.cs | 25 +- .../PdfEditEngine.Document.cs | 4 + .../PdfEditEngine.cs | 14 +- .../PdfEngineOptions.cs | 19 + .../PdfPageContentReader.Images.cs | 25 +- .../PublicAPI.Unshipped.txt | 2 + .../PdfAArchivalChecksTests.cs | 468 ++++++++++++++++++ tests/SharpCorePDF.Tests/PdfATests.cs | 45 +- .../SharpCorePDF.Tests/PdfFormBudgetTests.cs | 133 +++++ .../SharpCorePDF.Tests/PdfFormEditorTests.cs | 39 ++ 25 files changed, 1973 insertions(+), 61 deletions(-) create mode 100644 src/SharpCorePDF.Infrastructure/Authoring/PdfAAnnotationRules.cs create mode 100644 src/SharpCorePDF.Infrastructure/Authoring/PdfAContentChecks.cs create mode 100644 src/SharpCorePDF.Infrastructure/Pdf/PdfFormBudget.cs create mode 100644 tests/SharpCorePDF.Tests/PdfAArchivalChecksTests.cs create mode 100644 tests/SharpCorePDF.Tests/PdfFormBudgetTests.cs diff --git a/.github/workflows/validation.yml b/.github/workflows/validation.yml index b25e731..37580fe 100644 --- a/.github/workflows/validation.yml +++ b/.github/workflows/validation.yml @@ -134,9 +134,10 @@ jobs: dotnet "$tool" replay "$target" "$seeds/$target" done - # Each of the four documents reaches a failure class a review named, and each is now read: the decode - # limit, the length-chain bound, the /N bound and the JPEG scan-header checks closed them. A document - # that faults here again is a regression, so none is listed as known. + # Each of the five documents reaches a failure class a review named, and each is now read: the decode + # limit, the length-chain bound, the /N bound, the JPEG scan-header checks and the form-paint budget + # closed them. A document that faults or does not finish here again is a regression, so none is listed + # as known. - name: Hostile documents run: | set -euo pipefail @@ -162,10 +163,8 @@ jobs: name: Arlington PDF Model runs-on: ubuntu-latest timeout-minutes: 30 - # The service's report format and the writer's baseline are confirmed by the first run on a runner; until - # then the job reports without blocking. Remove continue-on-error and pass --strict with the --accept list - # that run establishes. - continue-on-error: true + # Blocking: a rule the model finds that is not in the accept list below fails the run. The list is the + # baseline the first runs established, and every entry says why it is accepted. services: arlington: @@ -201,9 +200,23 @@ jobs: echo "::error::the Arlington service did not answer on port 8080" exit 1 + # Accepted, and why: + # DocInfo-Title/Author/Producer-5, FontDescriptorCIDType2-CIDSet-5 + # deprecated in PDF 2.0, not forbidden: the library still writes the document information dictionary + # and a /CIDSet because readers and PDF/A-1 and -2 profiles use them; a 2.0 document that carries + # them is still readable and still conforms (veraPDF passes archive-4.pdf). + # FontType1-FirstChar/LastChar/Widths/FontDescriptor-11 + # PDF 2.0 withdrew the exemption of the 14 standard fonts, so a document that declares version 2.0 and + # draws with Helvetica has to describe it. The writer lets the caller choose the version and does not + # add descriptors for standard fonts; a 2.0 document that needs to satisfy the model embeds its font. + # Recorded in docs/validation.md as a limit, not hidden. - name: Check against the model run: | - python3 docs/tools/validate-arlington.py --report-dir artifacts/arlington \ + python3 docs/tools/validate-arlington.py --strict --report-dir artifacts/arlington \ + --accept DocInfo-Title-5 --accept DocInfo-Author-5 --accept DocInfo-Producer-5 \ + --accept FontDescriptorCIDType2-CIDSet-5 \ + --accept FontType1-FirstChar-11 --accept FontType1-LastChar-11 \ + --accept FontType1-Widths-11 --accept FontType1-FontDescriptor-11 \ --exclude protected.pdf \ "$RUNNER_TEMP/SharpCorePDF.Examples/output" \ "$RUNNER_TEMP/corpus/text-standard.pdf" "$RUNNER_TEMP/corpus/text-embedded.pdf" \ diff --git a/CHANGELOG.md b/CHANGELOG.md index 7551519..1e7e6f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,9 +35,42 @@ against an empty feed. - **`PdfEngineOptions.MaxDecodedStreamBytes`** (128 MiB by default) bounds what one stream may decode to. `MaxSourceBytes` bounds the input only, and Flate reaches about a thousand to one. +- **`PdfEngineOptions.MaxFormPaintsPerPage`** (100,000 by default) bounds how many times one page may paint a + form XObject, across every level of forms inside forms; a page that reaches it is read as far as it got, with a + `PdfWarningKind.FormPaintLimitExceeded` remark. `PdfWarningKind.PdfAFeatureRepaired` reports an entry a PDF/A + conversion set right in the copy. ### Fixed +- **A hybrid-reference file is read.** Its classic table lists the objects it keeps in object streams as free, so + a reader that knows no streams skips them, and describes them in the stream the trailer's `/XRefStm` names. The + table kept the first entry it met, so the free entry hid the one in the stream, and such a file read as having + no pages. The stream may now answer for the numbers its own section's table frees, and is read before the + revision `/Prev` points at, as ISO 32000-1 7.5.8.4 orders them. +- **An object stream no longer answers for an object the table places elsewhere.** A stream still holds the + copies it held when it was written, and loading it to resolve one object cached every object inside it: an + object a later revision rewrote in the file body came back stale, and one a later revision freed came back + from the dead, depending on which object was resolved first. Only the numbers the table places in a stream + are taken from it, and the table is consulted before any cache. +- **A page that paints forms without end is read up to a budget.** A form that paints the next form forty times, + six levels down, is under the depth limit and the guard against a form that paints itself, and asked a + 3.6 KB document for four billion paints: the read did not finish. Both the text walk and the image walk + now count the paints of a page against `MaxFormPaintsPerPage`, and the form content of a page against 256 MiB, and + keep a small per-page cache so a form painted again is not decoded again. +- **Form field appearances state `/Resources`.** A check box or radio button draws no text, names no font, and + its appearance stream had no `/Resources`; the Arlington model of PDF 1.7 and later requires the entry. +- **The PDF/A conversion sets right, removes, or refuses more of what the profiles name.** The analysis + never looked at a content stream, a bookmark, an annotation's flags or a form's actions; the veraPDF test + corpus, converted and judged by veraPDF 1.30.4, showed it writing documents that still broke the rule. Now: + an annotation's Print flag is set when it is missing (`PdfAFeatureRepaired`); a hidden annotation and a type the + profile does not list are refused; a widget's and a field's actions, and an XFA form, are dropped; a forbidden + action is found along `/Next` chains and on bookmarks, pages and the open action, and the forbidden set is the + one the profiles list (`SubmitForm` is permitted, `ResetForm`, `Hide`, `SetOCGState`, `Trans`, `GoTo3DView`, + `SetState` and `NoOp` are not; a named action must be one of four); content that paints in `DeviceCMYK`, through a + `Separation` or `DeviceN` alternate included, under an RGB intent, an operator ISO 32000 does not define, and + the graphics-state keys the profiles forbid are refused; and a source that already states a usable + `GTS_PDFA1` output intent keeps it instead of being re-labelled sRGB. `docs/validation.md` has the measurement. + - **Security: signature wrapping.** The validator required only that the byte range start at 0 and end at the end of the file, so bytes written into the unsigned gap could close the signature value early and add entries while the digest still matched, and the signature was reported valid and covering the whole diff --git a/docs/manual/src/24-pdf-a.md b/docs/manual/src/24-pdf-a.md index ffaacf0..1b9b849 100644 --- a/docs/manual/src/24-pdf-a.md +++ b/docs/manual/src/24-pdf-a.md @@ -84,7 +84,11 @@ byte-identical, which is what a test that compares files needs. `PdfAConversionOptions.OutputIntent` instead: the bytes, the component count and the condition it names are written as given, and the library does not convert the pages into that colour space. The intent is checked as an ICC profile first — declared length, the `acsp` signature, and a data colour space that agrees with - the component count — because a broken profile produces a file that looks archival and is not. + the component count — because a broken profile produces a file that looks archival and is not. When the + caller supplies none and the source already states a usable `GTS_PDFA1` output intent — one that passes the + same checks — that intent is kept: the document was prepared for that condition, and an sRGB intent written + over it would contradict content painted in its colour space. A source with none, or one whose profile fails + the checks, gets the sRGB intent. - **`/NeedAppearances false`** on the form, when the source has one: a conformance reader must be able to paint the form without building appearances itself. - **An association for every embedded file**, when the profile is PDF/A-3b: the catalog carries an `/AF` @@ -110,7 +114,16 @@ Two different things can stand between a source and a profile, and they are trea | A catalog open action that runs JavaScript | The document opens without running anything. | | The `/NeedAppearances` flag | The appearances must be in the file, not built by the reader. | | Custom document information entries | They have no XMP property without an extension schema. | -| The XMP packet and output intent of the source | Replaced by the ones the profile requires. | +| The XMP packet of the source, and its output intent when the checks above fail it | Replaced by the ones the profile requires. | +| The `/A` and `/AA` actions of a form widget, and the actions of a form field | The profile does not let a form carry an action; the form is read without running anything. | +| The XFA form of the document | A reader of an archival copy cannot be asked to run a program to build the form; the widgets carry their appearances. | + +**Set right, with a warning of kind `PdfAFeatureRepaired`**: an entry the profile requires that the source left +out, where setting it changes nothing a viewer shows. + +| Set right | Why it is safe | +| --- | --- | +| The Print bit of an annotation's `/F` flags, or the whole entry when it is missing | The annotation is shown as it was; it prints as well. Parts 2, 3 and 4 do not ask it of a popup, part 1 does. | **Refused, with `IsWritten = false` and a named requirement**: things that cannot be repaired without changing what a page looks like, or that are data. @@ -124,7 +137,13 @@ changing what a page looks like, or that are data. | Embedded files for 1b, 2b and 4 | Only PDF/A-3 allows attachments; PDF/A-4f would, and is not offered. | | Optional content for 1b | PDF 1.4 has no layers. | | An annotation without an appearance stream (except `Link` and `Popup`) | A viewer would have to invent the content. | -| A JavaScript, `Launch`, `Sound`, `Movie`, `Rendition`, `ImportData` or `SubmitForm` action | The profile forbids the action itself, not just its startup use. | +| A `JavaScript`, `Launch`, `Sound`, `Movie`, `Rendition`, `ImportData`, `ResetForm`, `Hide`, `SetOCGState`, `Trans`, `GoTo3DView`, `SetState` or `NoOp` action | The profile forbids the action itself, not just its startup use. A link, a bookmark, a page, the catalog and every action that one names in `/Next` are looked at; `SubmitForm` is permitted by every part. | +| A named action other than `NextPage`, `PrevPage`, `FirstPage` and `LastPage` | The other named actions are viewer commands. | +| An annotation hidden by its flags (`Invisible`, `Hidden`, `NoView`, and for parts 2 to 4 `ToggleNoView`) | Clearing the flag would put on the page something nobody saw. | +| An annotation of a type the profile does not list — `3D`, `Sound`, `Screen`, `Movie`, and for part 4 `RichMedia` and `FileAttachment` | The profile's list is the one the reference validator applies. | +| Content that paints in `DeviceCMYK`, or in `DeviceRGB` under a non-RGB intent, with no matching output intent and no `Default` colour space | A device space means nothing alone. The writer states one intent, sRGB unless the source's or the caller's is a different one; supply a CMYK `OutputIntent` for CMYK content. A `Separation` or `DeviceN` space counts through its alternate, and so does the colour space of a transparency group. A page this library authors paints in `DeviceRGB` unless the author states another colour, so authoring for a CMYK or gray intent means painting in that space. | +| An operator ISO 32000 does not define, even between `BX` and `EX` | The profile forbids it, and the writer does not know what it was meant to do. | +| An extended graphics state with `/HTP`, `/TR`, a `/TR2` other than `Default`, a halftone that is not type 1 or 5 or that is named, or a blend mode ISO 32000-1 does not define | Each is a rendering instruction the profile does not allow, and removing it changes the page. | | An external file stream (`/F`) | The document must contain what it displays. | A refusal writes nothing to the destination, so a caller that ignores `IsWritten` does not get a file it @@ -138,6 +157,17 @@ the font 'Helvetica' is not embedded; a PDF/A document must carry the program of Every conversion that does write adds a `PdfAProfileWritten` warning naming the profile and stating that conformance is established by an external validator. +### What this analysis is, and how far it reaches + +The analysis looks for the features this writer cannot repair. It is not a validator and it does not +read every rule of ISO 19005: a document with a violation it does not look for is written, and the +`PdfAProfileWritten` warning says conformance is established elsewhere. How far that reaches is measured, not +claimed. `docs/validation.md` records the result of converting the veraPDF test corpus — documents that each break +one clause, and documents that each pass one — and judging every result with veraPDF 1.30.4, with the counts +per profile and the clauses that remain: colour-space consistency between `Separation` arrays, font width and +encoding rules, and a few more. `robustness.yml` repeats the measurement weekly and fails when the number of +conversions veraPDF rejects rises. + ## Archival edits An edit can target a profile, which is what an archive wants when it has to fix one page of a master: diff --git a/docs/release-checklist.md b/docs/release-checklist.md index 6793c17..6bc5c8f 100644 --- a/docs/release-checklist.md +++ b/docs/release-checklist.md @@ -13,8 +13,10 @@ symbols story" — so the gate reads without it, and nothing here depends on a d ## Where the repository stands, 2026-10-17 -The table below is the `1.0.0` baseline, kept as it was written. The current state is the most recent -`G1–G9 at` section further down — `1.7.0` today, and `1.8.0` when it is cut. +The table below is the `1.0.0` baseline, kept as it was written. `1.7.0` is the last version on nuget.org. +`1.8.0` is prepared and not tagged, and the tree has moved on since: the section +[`Since 1.8.0 was prepared`](#since-180-was-prepared-2026-11-16) says what a tag on it would now carry, which +is not what the `1.8.0` section below describes. | Fact | State | | --- | --- | @@ -626,6 +628,24 @@ What is deliberately not in this version, and is tracked rather than implied: - **The image filters the reader passes through** — `DCTDecode`, `JPXDecode`, `JBIG2Decode`, `CCITTFaxDecode` and `RunLengthDecode` — remain `ImageFilterUnsupported`, the limit parity row 14 states. +## Since `1.8.0` was prepared (2026-11-16) + +`1.8.0` is still not tagged — `v1.7.0` is the newest tag on `origin`, and the feed ends at `1.7.0` — and `main` +has gained changes the `1.8.0` section above does not describe. Tagging the tree as it stands would publish +them under that number, so the owner has two choices: cut `1.8.0` at the commit the section above names, or +amend that section and the `1.8.0` changelog entry to say what the tree holds now. This record exists so the +second choice can be made on facts instead of on a memory of 14 November. + +| Gate | What changed | Consequence for the tag | +| --- | --- | --- | +| G1 — the surface | `PublicAPI.Unshipped.txt` lists four members: `PdfEngineOptions.MaxDecodedStreamBytes` and `.MaxFormPaintsPerPage` (Infrastructure) and `PdfWarningKind.PdfAFeatureRepaired` and `.FormPaintLimitExceeded` (Domain); the Application, DependencyInjection and entry-point files are empty | the `1.8.0` section says `Unshipped` is back to `#nullable enable`, which is no longer true, and new members are a minor-version change under the repository's SemVer rule: the next version is `1.9.0`, not a `1.8.x` patch, unless `1.8.0` is cut at the earlier commit. Two of the four are enum members, so a caller that switches over `PdfWarningKind` without a default sees two kinds it has not met | +| G3 — notes | `CHANGELOG.md` `[Unreleased]` holds the validation checks, the security and robustness fixes, and the four image-filter decoders (run-length, CCITT, baseline and progressive JPEG) | the `1.8.0` bullet that says those filters "remain `ImageFilterUnsupported`" is wrong for any commit after the decoders landed; move `[Unreleased]` under the version heading when it is cut, with the date | +| G9 — the gate | the suite is **1,413 tests on each leg** (`net10.0` and `net11.0`), up from 1,229 | the figure in the `1.8.0` table is a record of that day, not a current claim | +| CI | `validation.yml` and `robustness.yml` are new beside `ci.yml` and `compliance.yml`; `compliance.yml` cross-validates PDF/A-4 as well | a release commit should have all four green; `robustness.yml` runs weekly and on dispatch, not on a push, so its newest run is the evidence, not the run for the commit | +| Constitution | amendment `2.0.1` (a statement of fact about Python tools in CI) is proposed and not ratified | it needs the maintainer's approval in the pull request that carries it; nothing a release depends on | + +`docs/validation.md` lists what the new checks found and what is fixed, and what is still open. + ## `1.6.0` (2026-11-14, tagged and published) This version is about archival evidence rather than new pages: the PDF/A output is checked by a pinned diff --git a/docs/tools/corpus-robustness.py b/docs/tools/corpus-robustness.py index 24ceec9..8c94f61 100644 --- a/docs/tools/corpus-robustness.py +++ b/docs/tools/corpus-robustness.py @@ -25,6 +25,7 @@ flate-bomb.pdf a 1 MB content stream that inflates past 1 GiB: no limit on decoded size objstm-count.pdf an object stream that declares 2^31-1 objects: an allocation sized by /N jpeg-selector.pdf a JPEG scan that selects Huffman table 4 of 0-3: an unchecked table index + form-fanout.pdf six levels of forms, each painting the next 40 times: 40^6 paints from one page --report writes every outcome, with the first line of what the probe said, for the run's artefacts. """ @@ -137,7 +138,32 @@ def document(objects: list[bytes], root: int = 1) -> bytes: image, ])) - print(f"wrote 4 hostile documents to {directory}") + # Every form paints the next one forty times, six levels down: the forms are small and none is cyclic, so + # the depth limit and the cycle guard both pass, and the page asks for 40^6 paints of content that is + # decoded and scanned again on every one of them. + levels = 6 + forms = [] + + for level in range(levels): + if level < levels - 1: + body = (b"q /F%d Do Q\n" % (level + 1)) * 40 + resources = b"<< /XObject << /F%d %d 0 R >> >>" % (level + 1, 6 + level) + else: + body = b"0 0 m 1 1 l S\n" + resources = b"<< >>" + + forms.append(b"<< /Type /XObject /Subtype /Form /BBox [0 0 10 10] /Resources %s /Length %d >>\nstream\n%s\nendstream" + % (resources, len(body), body)) + + page_content = b"/F0 Do" + (directory / "form-fanout.pdf").write_bytes(document([ + head[0], head[1], + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Resources << /XObject << /F0 5 0 R >> >> /Contents 4 0 R >>", + b"<< /Length %d >>\nstream\n%s\nendstream" % (len(page_content), page_content), + *forms, + ])) + + print(f"wrote 5 hostile documents to {directory}") def main() -> None: diff --git a/docs/validation.md b/docs/validation.md index 83233fe..8cd8192 100644 --- a/docs/validation.md +++ b/docs/validation.md @@ -109,6 +109,21 @@ Fixed since the list was first drawn up, each with the test that fails without i | A merge copied the outline recursively, so a source nested about fifty thousand levels deep overflowed the stack; a looping or shared outline was walked again at every visit — a crash the `document` fuzz target found | the walk stops at 256 levels with `BookmarksTruncated`; a bookmark reached twice is copied once with `BookmarkSkipped` | `PdfMergeOutlineLimitsTests` | | Rebuilding a table sliced the document at offsets the damaged table stated past its end, or beyond an index's range, and threw `ArgumentOutOfRangeException` — a crash the `document` fuzz target found | an offset outside the document is passed over, and the rebuild finds the catalog | `PdfReaderRecoveryTests.Rebuilds_a_table_that_states_an_offset_past_the_end_of_the_file` | | Copying an attachment the listing reported, whose specification embeds no stream, threw `ArgumentException` — the public-corpora job's fault on `isartor-6-1-11-t01-fail-a.pdf` | it writes no bytes, matching the `ByteLength` of zero the listing reports | `PdfContentReaderTests.Copies_nothing_for_a_listed_file_that_embeds_no_stream` | +| A hybrid-reference file read as having no pages: its classic table lists the objects it keeps in object streams as free, and the free entry hid the real one in `/XRefStm` | the stream may answer for the numbers its own section's table frees, and is read before the revision `/Prev` points at | `PdfCrossReferencePrecedenceTests.Reads_an_object_a_hybrid_file_lists_as_free_in_its_table_and_describes_in_its_stream`, `.Reads_the_stream_of_a_hybrid_section_before_the_revision_it_follows` | +| An object stream answered for numbers the table places elsewhere: an object a later revision rewrote came back stale, and one it freed came back from the dead, depending on which object was resolved first | only the numbers the table places in a stream are taken from it, and the table is consulted before any cache | `.Resolves_the_object_the_table_names_not_the_copy_an_object_stream_still_holds`, `.Does_not_bring_back_an_object_the_table_lists_as_free` | +| A page that paints forms without end — six levels of forms, each painting the next 40 times, from a 3.6 KB document — never finished reading | `PdfEngineOptions.MaxFormPaintsPerPage` (100,000) and a per-page budget of 256 MiB of decoded form content, with a per-page cache | `PdfFormBudgetTests` | +| A check box or radio button's appearance stream had no `/Resources`, which the Arlington model of PDF 1.7 and later requires | the entry is stated, empty when the button draws no text | `PdfFormEditorTests.States_resources_on_the_appearance_of_a_button_that_draws_no_text` | +| The PDF/A conversion wrote documents that broke rules its analysis never looked at: annotation flags and types, the actions of widgets, fields, bookmarks and `/Next` chains, XFA, device colour under a mismatched output intent, undefined operators, graphics-state keys | see [the section below](#the-pdfa-conversion-against-the-verapdf-corpus) and chapter 24 | `PdfAArchivalChecksTests` (32 tests, 21 of which fail on the previous analysis) | + +## Known limits + +What the checks show and nothing here fixes, stated so that no claim goes further than the evidence: + +| Limit | Where it shows | What a caller does | +| --- | --- | --- | +| A document that declares PDF 2.0 and draws with one of the 14 standard fonts omits `/FirstChar`, `/LastChar`, `/Widths` and `/FontDescriptor`, which PDF 2.0 withdrew the exemption from | the Arlington check, `FontType1-*-11` on `report.pdf`; accepted in `validation.yml` with the reason beside it | embed a font, or write the document as PDF 1.7 | +| The PDF/A conversion does not read every rule of ISO 19005: `Separation` and `DeviceN` colorant consistency (`6.2.4.4`), the font rules (`6.2.10` and `6.2.11`: `CIDToGIDMap`, `CIDSystemInfo`, embedded CMaps, widths, encodings, `.notdef`), rendering intent values (`6.2.6`), a halftone's transfer function (`6.2.5-6`), undefined operators inside Type 3 glyph procedures, metadata extension schemas (`6.6.2.3`), odd-length hexadecimal strings and integers beyond 32 bits (`6.1.5`, `6.1.13`), and the `Alternates`, `OPI` and reference-XObject keys (`6.2.7` to `6.2.9`) | the preflight measurement below, which lists them by clause | judge the result with veraPDF; the library says so in `PdfAProfileWritten` | +| A source whose own output intent is not a usable ICC profile, or whose content paints in a space no intent covers, is refused rather than converted | `PdfAConversionResult.UnsatisfiedRequirements` | supply an `OutputIntent` the content was prepared for | ## Corpus readings, 2026-11-16 @@ -119,9 +134,44 @@ Fixed since the list was first drawn up, each with the test that fails without i | synthesized hostile documents, before the fixes | 4 | 0 | 0 | 3 | 1 | 0 | The six pdf.js faults were the empty-name, upside-down glyph and date-offset defects in the fixed table; -after those fixes all six are read, as is `jpeg-selector.pdf`. A sample of the pre-flight measurement (the first 60 -files of `PDF_A-2b` and of `PDF_A-4`) wrote 108 results, refused 9 non-conforming inputs and refused 3 that -already conformed; whether the 108 conform is veraPDF's to say, which `robustness.yml` asks. +after those fixes all six are read, as is `jpeg-selector.pdf`. + +## The PDF/A conversion against the veraPDF corpus + +`validate-preflight-corpus.py` converts every file of the veraPDF corpus for its profile and asks veraPDF 1.30.4 +whether the result conforms. Each file breaks one clause (`-fail-`) or passes one (`-pass-`), so the answer +separates three things: a conversion that writes a file veraPDF rejects (a *false success*), one that +refuses a source that already conformed (an *over-refusal*), and one that damages a conforming source. + +**Before** (`robustness.yml` on `f529b02`, 2,074 files, five profiles): 395 false successes, of which 378 were +sources meant to fail and 17 were sources that conformed — the converter damaged them, chiefly by replacing a CMYK +output intent with sRGB. By profile: 1b 48, 2b 169, 2u 9, 4 169, 3b 0. + +| Clause group | False successes | What veraPDF found | +| --- | --- | --- | +| Colour: `6.1.7`, `6.2.4.x`, `6.2.5`, `6.2.10` (transparency, part 2) | 88 | chiefly `6.2.4.3-3`: DeviceCMYK used with no CMYK output intent | +| Annotations and forms: `6.3.x`, `6.4.x` | 79 | `/F` missing (`6.3.2-1`), types the profile does not list, actions on widgets and fields, XFA | +| Actions: `6.5.x`, `6.6.1`, `6.6.3` | 28 | forbidden actions along `/Next` chains and on bookmarks; named actions | +| Fonts: `6.2.10.x` (part 4), `6.2.11.x` (part 2) | 88 | not looked at | +| The rest: output intent profile `6.2.3` (18), integers beyond 32 bits `6.1.13` (15), operators `6.2.2` (15), `6.1.12` (13), `Alternates`/`OPI`/reference XObjects `6.2.7` to `6.2.9` (26), strings, metadata | 112 | not looked at, except `6.2.2` | + +**After**, the ten clause groups that were changed, converted and judged locally with the same veraPDF +(150 fail-inputs and 51 pass-inputs of `PDF_A-2b`): + +| | Refused | Conforms | Still fails | +| --- | --- | --- | --- | +| Inputs meant to fail (150) | 48 | 76 | 26 | +| Inputs meant to pass (51) | 3 | 39 | **0** | + +No conforming source is damaged any more, which is the defect that mattered most: a caller who archives a +document that already conformed is told so by the result, not given a file that no longer does. The 26 inputs that +still fail break `6.2.4.4-1/-2` (colorant consistency, 7), the remaining `6.2.4.3` shapes (14), and a few single rules +the analysis does not read (`6.2.2-2`, `6.2.5-6`, `6.2.6-1`, `6.5.1-1`); they are the known limits above, by +clause. The three conforming sources refused are over-refusals: their colour use is allowed by an intent or a +`Default` space the analysis does not read. + +`robustness.yml` repeats the full measurement weekly. Its `--max-false-success` is the ratchet: the run fails when the +count of false successes rises above the figure measured on the commit that set it. ## How the checks were calibrated @@ -147,9 +197,11 @@ A check that never fails proves nothing, so each was shown to fail on purpose be path (`SCPDF_FUZZ_TARGET=dct dotnet SharpCorePDF.Validation.dll `), with the stack ending in `JpegDecoder.DecodeBlock`; the seeds of `document`, `dct` and `ccitt` run clean. -## What has not been run here +## What runs where -The Arlington check (#4) and the judged half of the pre-flight measurement (#7) need the Arlington service -and veraPDF, and the fuzzing (#1) needs Linux; they were written against the documented interfaces and run -first on a runner. Both report-only checks say so in their workflow, with what to change once the first run -has set the baseline. +On a Windows workstation the checks that need only Python, Node, .NET and a portable JRE run in a minute or +two, and veraPDF installs without a system change (`jdk-17` and `verapdf-greenfield-1.30.4` unpacked into a +scratch directory), which is how the per-clause measurement above was taken file by file. Two things do not run +there: libFuzzer, which needs Linux, and the Arlington service, which needs a Docker daemon. Both run in +`robustness.yml` and `validation.yml`, where the Arlington check is blocking and its accepted rules are listed +beside the step with the reason for each. diff --git a/src/SharpCorePDF.Domain/PublicAPI.Unshipped.txt b/src/SharpCorePDF.Domain/PublicAPI.Unshipped.txt index 7dc5c58..8b2a0fe 100644 --- a/src/SharpCorePDF.Domain/PublicAPI.Unshipped.txt +++ b/src/SharpCorePDF.Domain/PublicAPI.Unshipped.txt @@ -1 +1,3 @@ #nullable enable +SharpCorePDF.Domain.ValueObjects.PdfWarningKind.PdfAFeatureRepaired = 68 -> SharpCorePDF.Domain.ValueObjects.PdfWarningKind +SharpCorePDF.Domain.ValueObjects.PdfWarningKind.FormPaintLimitExceeded = 69 -> SharpCorePDF.Domain.ValueObjects.PdfWarningKind diff --git a/src/SharpCorePDF.Domain/ReadModels/PdfAConversionResult.cs b/src/SharpCorePDF.Domain/ReadModels/PdfAConversionResult.cs index 0606f34..a47da88 100644 --- a/src/SharpCorePDF.Domain/ReadModels/PdfAConversionResult.cs +++ b/src/SharpCorePDF.Domain/ReadModels/PdfAConversionResult.cs @@ -8,15 +8,18 @@ namespace SharpCorePDF.Domain.ReadModels; /// /// /// -/// is the honest answer to "may I ship this file as an archival master". It is -/// when the writer knows the profile is not satisfied — an unembedded font, a -/// transparency the profile forbids, an attachment — and in that case the caller's destination was -/// left untouched and names every reason. +/// is when the writer found something in the source that the +/// profile forbids and that it cannot make right — an unembedded font, a transparency the profile forbids, +/// an attachment, content in a colour space the output intent does not cover — and in that case the caller's +/// destination was left untouched and names every reason. /// /// /// being means the structures the profile requires were -/// written, not that a validator has accepted the file. veraPDF remains the external authority, and -/// the library makes no conformance claim of its own. +/// written and the analysis found none of what it looks for. It does not mean a validator has accepted the file: +/// the analysis is not a validator and does not read every rule of ISO 19005, so a source with a violation it does +/// not look for is written with that violation. veraPDF remains the external authority, and the library makes +/// no conformance claim of its own; how far the analysis reaches is measured against the veraPDF test corpus in +/// docs/validation.md. /// /// public sealed record PdfAConversionResult diff --git a/src/SharpCorePDF.Domain/ValueObjects/PdfWarningKind.cs b/src/SharpCorePDF.Domain/ValueObjects/PdfWarningKind.cs index 9c6d178..2ff4319 100644 --- a/src/SharpCorePDF.Domain/ValueObjects/PdfWarningKind.cs +++ b/src/SharpCorePDF.Domain/ValueObjects/PdfWarningKind.cs @@ -405,4 +405,17 @@ public enum PdfWarningKind /// the document and not a failure to read it. /// FontDescriptorMissing, + + /// + /// An entry the PDF/A profile requires was missing or incomplete and was set right in the written copy, for + /// example the Print bit of an annotation's flags, and the caller was told which one. + /// + PdfAFeatureRepaired, + + /// + /// The content of a page paints form XObjects more often than PdfEngineOptions.MaxFormPaintsPerPage + /// allows, or reads more decoded form content than the page's budget, so the rest of the page was not read. + /// What was read before the limit is reported. + /// + FormPaintLimitExceeded, } diff --git a/src/SharpCorePDF.Infrastructure/Authoring/PdfAAnnotationRules.cs b/src/SharpCorePDF.Infrastructure/Authoring/PdfAAnnotationRules.cs new file mode 100644 index 0000000..e79c53a --- /dev/null +++ b/src/SharpCorePDF.Infrastructure/Authoring/PdfAAnnotationRules.cs @@ -0,0 +1,197 @@ +using SharpCorePDF.Domain.ValueObjects; +using SharpCorePDF.Infrastructure.Pdf; + +namespace SharpCorePDF.Infrastructure.Authoring; + +/// +/// What ISO 19005 states about annotation dictionaries and the actions of a form, shared by the analysis that +/// decides whether a document can be written and the rewrite that makes the copy conform. +/// +/// +/// +/// The two halves read the same rules from here so that they cannot disagree: an annotation the analysis +/// accepts is one the rewrite leaves as it states it or repairs, and one the analysis refuses is never copied. +/// +/// +/// A repair is limited to what changes no page. A missing /F, or a Print bit that is not set, is made +/// right because the annotation is shown as it was and only prints as well; a flag that hides the annotation is +/// refused instead, because clearing it would put something on the page that nobody saw. The action of a widget +/// or a field is dropped because the form is read without running anything, as the profile requires. +/// +/// +internal static class PdfAAnnotationRules +{ + private const long Invisible = 1; + private const long Hidden = 2; + private const long Print = 4; + private const long NoView = 32; + private const long ToggleNoView = 256; + + private static readonly HashSet Part1Types = new(StringComparer.Ordinal) + { + "Circle", "FreeText", "Highlight", "Ink", "Line", "Link", "Popup", "PrinterMark", "Square", "Squiggly", + "Stamp", "StrikeOut", "Text", "TrapNet", "Underline", "Widget", + }; + + private static readonly HashSet Part2And3Types = new(StringComparer.Ordinal) + { + "Caret", "Circle", "FileAttachment", "FreeText", "Highlight", "Ink", "Line", "Link", "PolyLine", "Polygon", + "Popup", "PrinterMark", "Redact", "Square", "Squiggly", "Stamp", "StrikeOut", "Text", "TrapNet", + "Underline", "Watermark", "Widget", + }; + + private static readonly HashSet Part4Types = new(StringComparer.Ordinal) + { + "Caret", "Circle", "FreeText", "Highlight", "Ink", "Line", "Link", "PolyLine", "Polygon", "Popup", + "PrinterMark", "Projection", "Redact", "Square", "Squiggly", "Stamp", "StrikeOut", "Text", "TrapNet", + "Underline", "Watermark", "Widget", + }; + + /// Determines whether a profile permits an annotation type. + /// The profile. + /// The /Subtype of the annotation. + /// when the type is one the profile lists. + /// + /// The lists are the ones the profile's reference validator applies: the types the underlying ISO 32000 + /// edition defines, less the ones the part forbids (3D, Sound, Screen and Movie everywhere; in part 4 also + /// RichMedia and FileAttachment; in part 1 also the types PDF 1.4 does not have). + /// + public static bool PermitsType(PdfAProfile profile, string subtype) => profile switch + { + PdfAProfile.PdfA1B => Part1Types.Contains(subtype), + PdfAProfile.PdfA4 => Part4Types.Contains(subtype), + _ => Part2And3Types.Contains(subtype), + }; + + /// Determines whether a profile requires a flags entry on an annotation. + /// The profile. + /// The /Subtype of the annotation. + /// when the annotation has to state /F. + /// Parts 2, 3 and 4 exempt a popup, which a viewer places beside the annotation it belongs to; part 1 does not. + public static bool RequiresFlags(PdfAProfile profile, string? subtype) => + profile == PdfAProfile.PdfA1B || !string.Equals(subtype, "Popup", StringComparison.Ordinal); + + /// Gets the flag bits an annotation of a profile must not set. + /// The profile. + /// The bits. + /// Toggling the view is a PDF 1.5 flag, so part 1 does not name it. + public static long ForbiddenFlags(PdfAProfile profile) => + profile == PdfAProfile.PdfA1B ? Invisible | Hidden | NoView : Invisible | Hidden | NoView | ToggleNoView; + + /// Determines whether the flags of an annotation hide it, which the profile forbids. + /// The annotation dictionary. + /// The profile. + /// when a bit the profile forbids is set. + public static bool IsHidden(PdfDictionary annotation, PdfAProfile profile) => + ((annotation.GetInteger("F") ?? 0L) & ForbiddenFlags(profile)) != 0; + + /// Determines whether the flags of an annotation need the Print bit set. + /// The annotation dictionary. + /// The profile. + /// when the entry is missing or does not set Print. + public static bool NeedsPrintFlag(PdfDictionary annotation, PdfAProfile profile) => + RequiresFlags(profile, annotation.GetName("Subtype")) && ((annotation.GetInteger("F") ?? 0L) & Print) == 0; + + /// Determines whether an annotation is a widget, the annotation a form field stands on. + /// The annotation dictionary. + /// for a widget. + public static bool IsWidget(PdfDictionary annotation) => + string.Equals(annotation.GetName("Subtype"), "Widget", StringComparison.Ordinal); + + /// Rewrites the copy of an annotation so that it states what the profile requires. + /// The copy of the annotation dictionary, or any other object, which is returned as it is. + /// The profile. + /// The body to write. + public static PdfObject Repair(PdfObject body, PdfAProfile profile) + { + if (body is not PdfDictionary annotation || annotation.GetName("Subtype") is null) + { + return body; + } + + if (NeedsPrintFlag(annotation, profile)) + { + annotation.SetInteger("F", (annotation.GetInteger("F") ?? 0L) | Print); + } + + if (IsWidget(annotation)) + { + annotation.Remove("A"); + annotation.Remove("AA"); + } + + return body; + } + + /// + /// Registers, for every annotation and form field of a document, the rewrite that makes its copy state what the + /// profile requires. + /// + /// The source document. + /// The copier that writes the source objects into the result. + /// The profile. + /// + /// Only an object the document states as its own — a reference — can be rewritten when it is copied; an + /// annotation written directly inside its page's array is a rarity this registration does not reach, and the + /// analysis refuses what it cannot make right in that position. + /// + public static void Register(PdfFileReader reader, PdfObjectCopier copier, PdfAProfile profile) + { + // Fields first: a widget that is also a field is registered again as an annotation, and that rewrite, + // which also drops its actions, is the one that stays. + RegisterFields(reader, copier, reader.ResolveDictionary(reader.Catalog?.Get("AcroForm"))?.Get("Fields"), depth: 0, visited: []); + + foreach (var page in reader.Pages) + { + foreach (var annotation in reader.ResolveArray(page.Dictionary.Get("Annots"))?.Items ?? []) + { + if (annotation is PdfReference reference) + { + copier.Override(reference.ObjectNumber, body => Repair(body, profile)); + } + } + } + } + + private static void RegisterFields( + PdfFileReader reader, + PdfObjectCopier copier, + PdfObject? fields, + int depth, + HashSet visited) + { + if (depth > 32 || reader.ResolveArray(fields) is not { } array) + { + return; + } + + foreach (var item in array.Items) + { + if (reader.ResolveDictionary(item) is not { } field || !visited.Add(field)) + { + continue; + } + + if (item is PdfReference reference) + { + copier.Override(reference.ObjectNumber, RepairField); + } + + RegisterFields(reader, copier, field.Get("Kids"), depth + 1, visited); + } + } + + /// Rewrites the copy of a form field so that it carries no action. + /// The copy of the field dictionary, or any other object, which is returned as it is. + /// The body to write. + public static PdfObject RepairField(PdfObject body) + { + if (body is PdfDictionary field) + { + field.Remove("A"); + field.Remove("AA"); + } + + return body; + } +} diff --git a/src/SharpCorePDF.Infrastructure/Authoring/PdfAContentChecks.cs b/src/SharpCorePDF.Infrastructure/Authoring/PdfAContentChecks.cs new file mode 100644 index 0000000..6026c75 --- /dev/null +++ b/src/SharpCorePDF.Infrastructure/Authoring/PdfAContentChecks.cs @@ -0,0 +1,366 @@ +using SharpCorePDF.Domain.Exceptions; +using SharpCorePDF.Domain.ValueObjects; +using SharpCorePDF.Infrastructure.Pdf; + +namespace SharpCorePDF.Infrastructure.Authoring; + +/// +/// The rules of ISO 19005 that can only be seen in what a content stream says: the colour spaces it paints +/// in, and the operators it uses. +/// +/// +/// +/// A device colour space has no meaning of its own, so the standard allows one only where the document says +/// what it stands for: a matching output intent, or a Default space the resources define. This writer +/// states one output intent, sRGB unless the caller supplies another, so content that paints in DeviceCMYK is +/// refused instead of being written under an intent that contradicts it — the one rule four clauses of the +/// archival corpus reached. +/// +/// +/// A stream counts as painting in a space when it names it: an operator that selects it, a colour space entry +/// its operators select, an image or shading that states it, or an inline image. An image in the resources that no +/// operator paints is counted too, which can refuse a document whose unused resource would never have been seen; +/// refusing too much is the safe side of a check whose answer is "may this be archived". +/// +/// +internal static class PdfAContentChecks +{ + private const int MaxDepth = 8; + + private static readonly HashSet Operators = new(StringComparer.Ordinal) + { + "b", "B", "b*", "B*", "BDC", "BI", "BMC", "BT", "BX", "c", "cm", "CS", "cs", "d", "d0", "d1", "Do", "DP", + "EI", "EMC", "ET", "EX", "f", "F", "f*", "G", "g", "gs", "h", "i", "ID", "j", "J", "K", "k", "l", "m", "M", + "MP", "n", "q", "Q", "re", "RG", "rg", "ri", "s", "S", "SC", "sc", "SCN", "scn", "sh", "T*", "Tc", "Td", + "TD", "Tf", "Tj", "TJ", "TL", "Tm", "Tr", "Ts", "Tw", "Tz", "v", "w", "W", "W*", "y", "'", "\"", + }; + + private static readonly HashSet Literals = new(StringComparer.Ordinal) { "true", "false", "null" }; + + /// The device colour spaces a stream painted in. + [Flags] + private enum Families + { + None = 0, + Rgb = 1, + Cmyk = 2, + } + + /// Checks the content of every page, form, pattern and annotation appearance of a document. + /// The source document. + /// The profile the document would be written for. + /// The number of colour components of the output intent the result will state. + /// The list that receives the requirements that are not met. + public static void Check(PdfFileReader reader, PdfAProfile profile, int intentComponents, List unsatisfied) + { + var visited = new HashSet(); + + for (var index = 0; index < reader.Pages.Count; index++) + { + var page = reader.Pages[index]; + var where = $"page {index + 1}"; + var resources = reader.ResolveDictionary(page.Dictionary.Get("Resources")) + ?? reader.ResolveDictionary(page.Inherited.Get("Resources")); + + CheckContent(reader, page.Dictionary.Get("Contents"), resources, $"the content of {where}", profile, intentComponents, unsatisfied); + CheckResources(reader, resources, where, profile, intentComponents, unsatisfied, visited, depth: 0); + CheckGroup(reader, page.Dictionary, resources, $"the transparency group of {where}", profile, intentComponents, unsatisfied); + + foreach (var annotation in reader.ResolveArray(page.Dictionary.Get("Annots"))?.Items ?? []) + { + CheckAppearance(reader, annotation, where, profile, intentComponents, unsatisfied, visited); + } + } + } + + private static void CheckAppearance( + PdfFileReader reader, + PdfObject annotation, + string where, + PdfAProfile profile, + int intentComponents, + List unsatisfied, + HashSet visited) + { + if (reader.ResolveDictionary(reader.ResolveDictionary(annotation)?.Get("AP"))?.Get("N") is not { } normal) + { + return; + } + + // The normal appearance is a stream, or a dictionary of streams, one per state. + var streams = reader.Resolve(normal) is PdfDictionary states + ? states.Entries.Select(entry => entry.Value) + : [normal]; + + foreach (var stream in streams) + { + CheckForm(reader, stream, $"an annotation of {where}", profile, intentComponents, unsatisfied, visited, depth: 1); + } + } + + private static void CheckResources( + PdfFileReader reader, + PdfDictionary? resources, + string where, + PdfAProfile profile, + int intentComponents, + List unsatisfied, + HashSet visited, + int depth) + { + if (resources is null || depth > MaxDepth) + { + return; + } + + foreach (var entry in reader.ResolveDictionary(resources.Get("XObject"))?.Entries ?? []) + { + CheckForm(reader, entry.Value, $"the object '{entry.Key}' of {where}", profile, intentComponents, unsatisfied, visited, depth + 1); + } + + foreach (var entry in reader.ResolveDictionary(resources.Get("Pattern"))?.Entries ?? []) + { + CheckForm(reader, entry.Value, $"the pattern '{entry.Key}' of {where}", profile, intentComponents, unsatisfied, visited, depth + 1); + } + + foreach (var entry in reader.ResolveDictionary(resources.Get("Shading"))?.Entries ?? []) + { + Report( + FamiliesOf(reader, reader.ResolveDictionary(entry.Value)?.Get("ColorSpace"), resources), + resources, + $"the shading '{entry.Key}' of {where}", + profile, + intentComponents, + unsatisfied, + reader); + } + } + + /// Checks a stream a resource or an appearance names: an image states a space, a form has content of its own. + private static void CheckForm( + PdfFileReader reader, + PdfObject? value, + string where, + PdfAProfile profile, + int intentComponents, + List unsatisfied, + HashSet visited, + int depth) + { + if (depth > MaxDepth || reader.Resolve(value) is not PdfStream stream) + { + return; + } + + if (value is PdfReference reference && !visited.Add(reference.ObjectNumber)) + { + return; + } + + var dictionary = stream.Dictionary; + var resources = reader.ResolveDictionary(dictionary.Get("Resources")); + + if (string.Equals(dictionary.GetName("Subtype"), "Image", StringComparison.Ordinal)) + { + Report(FamiliesOf(reader, dictionary.Get("ColorSpace"), resources), resources, where, profile, intentComponents, unsatisfied, reader); + return; + } + + CheckContent(reader, value, resources, $"the content of {where}", profile, intentComponents, unsatisfied); + CheckResources(reader, resources, where, profile, intentComponents, unsatisfied, visited, depth); + CheckGroup(reader, dictionary, resources, $"the transparency group of {where}", profile, intentComponents, unsatisfied); + } + + /// + /// Checks the colour space a page or a form states for its transparency group, which the reference validator + /// counts as a use of that space: a group in DeviceRGB under a CMYK output intent breaks the same rule as a fill. + /// + private static void CheckGroup( + PdfFileReader reader, + PdfDictionary owner, + PdfDictionary? resources, + string where, + PdfAProfile profile, + int intentComponents, + List unsatisfied) + { + if (reader.ResolveDictionary(owner.Get("Group")) is { } group) + { + Report(FamiliesOf(reader, group.Get("CS"), resources), resources, where, profile, intentComponents, unsatisfied, reader); + } + } + + private static void CheckContent( + PdfFileReader reader, + PdfObject? contents, + PdfDictionary? resources, + string where, + PdfAProfile profile, + int intentComponents, + List unsatisfied) + { + if (!TryRead(reader, contents, out var data)) + { + return; + } + + var used = Families.None; + var undefined = new SortedSet(StringComparer.Ordinal); + var colourSpaces = reader.ResolveDictionary(resources?.Get("ColorSpace")); + + foreach (var operation in PdfContentScanner.Scan(data, reader.SourceName)) + { + var name = operation.Operator; + + if (!Operators.Contains(name) && !Literals.Contains(name)) + { + undefined.Add(name); + continue; + } + + switch (name) + { + case "rg" or "RG": + used |= Families.Rgb; + break; + + case "k" or "K": + used |= Families.Cmyk; + break; + + case "cs" or "CS" when operation.Names.Count > 0: + used |= FamiliesOfName(reader, operation.Names[0], colourSpaces); + break; + + case "BI" when operation.InlineImage is { } image: + used |= FamiliesOf(reader, image.Get("ColorSpace") ?? image.Get("CS"), resources); + break; + } + } + + foreach (var undefinedOperator in undefined) + { + unsatisfied.Add( + $"{where} uses the operator '{undefinedOperator}', which ISO 32000 does not define; {PdfAProfileRules.Name(profile)} forbids it even between BX and EX."); + } + + Report(used, resources, where, profile, intentComponents, unsatisfied, reader); + } + + /// Reports the device spaces a stream painted in that the output intent and the resources do not cover. + private static void Report( + Families used, + PdfDictionary? resources, + string where, + PdfAProfile profile, + int intentComponents, + List unsatisfied, + PdfFileReader reader) + { + var defaults = reader.ResolveDictionary(resources?.Get("ColorSpace")); + + if (used.HasFlag(Families.Cmyk) && intentComponents != 4 && defaults?.ContainsKey("DefaultCMYK") != true) + { + unsatisfied.Add( + $"{where} paints in DeviceCMYK, which {PdfAProfileRules.Name(profile)} allows only with a CMYK output intent or a DefaultCMYK colour space; the output intent of the result has {intentComponents} colour components. Supply a CMYK output intent, or convert the colours."); + } + + if (used.HasFlag(Families.Rgb) && intentComponents != 3 && defaults?.ContainsKey("DefaultRGB") != true) + { + unsatisfied.Add( + $"{where} paints in DeviceRGB, which {PdfAProfileRules.Name(profile)} allows only with an RGB output intent or a DefaultRGB colour space; the output intent of the result has {intentComponents} colour components."); + } + } + + private static Families FamiliesOfName(PdfFileReader reader, string name, PdfDictionary? colourSpaces) => name switch + { + "DeviceRGB" or "RGB" => Families.Rgb, + "DeviceCMYK" or "CMYK" => Families.Cmyk, + "DeviceGray" or "G" or "Pattern" => Families.None, + _ => colourSpaces?.Get(name) is { } entry ? FamiliesOf(reader, entry, resources: null) : Families.None, + }; + + /// Names the device spaces a colour space value is, or is built on. + private static Families FamiliesOf(PdfFileReader reader, PdfObject? value, PdfDictionary? resources) + { + switch (reader.Resolve(value)) + { + case PdfName name: + return FamiliesOfName(reader, name.Value, reader.ResolveDictionary(resources?.Get("ColorSpace"))); + + case PdfArray { Items.Count: > 1 } array when reader.Resolve(array.Items[0]) is PdfName { Value: "Indexed" or "I" }: + return FamiliesOf(reader, array.Items[1], resources); + + // A Separation or DeviceN space is painted through its alternate when the colorant is not available, + // and the reference validator counts the alternate as a use of that device space. + case PdfArray { Items.Count: > 2 } array when reader.Resolve(array.Items[0]) is PdfName { Value: "Separation" or "DeviceN" }: + return FamiliesOf(reader, array.Items[2], resources) | ColorantFamilies(reader, array, resources); + + default: + return Families.None; + } + } + + /// Names the device spaces the Separation colorants of a DeviceN space are built on. + private static Families ColorantFamilies(PdfFileReader reader, PdfArray deviceN, PdfDictionary? resources) + { + // The attributes dictionary is the fifth element; its /Colorants map each name to a Separation array. + if (deviceN.Items.Count < 5 + || reader.ResolveDictionary(reader.ResolveDictionary(deviceN.Items[4])?.Get("Colorants")) is not { } colorants) + { + return Families.None; + } + + var families = Families.None; + + foreach (var entry in colorants.Entries) + { + families |= FamiliesOf(reader, entry.Value, resources); + } + + return families; + } + + private static bool TryRead(PdfFileReader reader, PdfObject? value, out byte[] data) + { + data = []; + + try + { + switch (reader.Resolve(value)) + { + case PdfStream stream: + data = PdfFilters.Decode(stream, reader.SourceName, reader.Options.MaxDecodedStreamBytes); + return true; + + case PdfArray array: + using (var joined = new MemoryStream()) + { + foreach (var item in array.Items) + { + if (reader.Resolve(item) is PdfStream part) + { + joined.Write(PdfFilters.Decode(part, reader.SourceName, reader.Options.MaxDecodedStreamBytes)); + joined.WriteByte((byte)'\n'); + } + } + + data = joined.ToArray(); + } + + return true; + + default: + return false; + } + } + catch (PdfSourceFormatException) + { + return false; + } + catch (PdfUnsupportedFeatureException) + { + return false; + } + } +} diff --git a/src/SharpCorePDF.Infrastructure/Authoring/PdfARequirements.cs b/src/SharpCorePDF.Infrastructure/Authoring/PdfARequirements.cs index 8b04b2b..762da3b 100644 --- a/src/SharpCorePDF.Infrastructure/Authoring/PdfARequirements.cs +++ b/src/SharpCorePDF.Infrastructure/Authoring/PdfARequirements.cs @@ -3,12 +3,14 @@ namespace SharpCorePDF.Infrastructure.Authoring; -/// What one analysis of a source found: what must be refused, and what may be removed. +/// What one analysis of a source found: what must be refused, what may be removed, and what is set right. /// The requirements the profile states that the document does not meet. /// The features the writer removes, each with the reason it is allowed to. +/// The entries the writer sets right, each stated as the sentence the caller is told. internal sealed record PdfARequirementReport( IReadOnlyList Unsatisfied, - IReadOnlyList Removable); + IReadOnlyList Removable, + IReadOnlyList? Repairable = null); /// /// Decides whether a source document can be written for a PDF/A profile, before a single byte of the @@ -30,9 +32,26 @@ internal sealed record PdfARequirementReport( /// internal static class PdfARequirements { - /// The action types ISO 19005 forbids in every part. - private static readonly HashSet ForbiddenActions = - new(StringComparer.Ordinal) { "JavaScript", "Launch", "Sound", "Movie", "Rendition", "ImportData", "SubmitForm" }; + /// The action types ISO 19005 forbids, as the reference validator applies them to each part. + /// + /// Part 1 forbids Launch, Sound, Movie, ResetForm, ImportData, JavaScript and Hide; parts 2 and 3 add Rendition, + /// SetOCGState, Trans and GoTo3DView, which part 1 has no such action for; the deprecated SetState and NoOp are + /// forbidden throughout. SubmitForm is permitted by every part. + /// + private static readonly HashSet ForbiddenActions = new(StringComparer.Ordinal) + { + "JavaScript", "Launch", "Sound", "Movie", "Rendition", "ImportData", "ResetForm", "Hide", "SetOCGState", + "Trans", "GoTo3DView", "SetState", "NoOp", + }; + + /// The named actions every part permits; any other named action is forbidden. + private static readonly HashSet StandardNamedActions = new(StringComparer.Ordinal) + { + "NextPage", "PrevPage", "FirstPage", "LastPage", + }; + + /// The longest chain of /Next actions the analysis follows. + private const int MaxActionChain = 64; /// The deepest chain of form XObjects the analysis follows. private const int MaxFormDepth = 32; @@ -41,19 +60,22 @@ internal static class PdfARequirements /// The source document. /// The profile it would be written for. /// The requirements that are not met, and the features that will be removed. - public static PdfARequirementReport Analyze(PdfFileReader reader, PdfAProfile profile) + /// The number of colour components of the output intent the result will state: three for the built-in sRGB. + public static PdfARequirementReport Analyze(PdfFileReader reader, PdfAProfile profile, int outputIntentComponents = 3) { ArgumentNullException.ThrowIfNull(reader); var unsatisfied = new List(); var removable = new List(); + var repairable = new List(); var fonts = new Dictionary(StringComparer.Ordinal); var filterProblems = new HashSet(StringComparer.Ordinal); var catalog = reader.Catalog; CheckCatalog(reader, catalog, profile, unsatisfied, removable); - CheckPages(reader, profile, fonts, filterProblems, unsatisfied); + CheckPages(reader, profile, fonts, filterProblems, unsatisfied, removable, repairable); + PdfAContentChecks.Check(reader, profile, outputIntentComponents, unsatisfied); foreach (var entry in fonts.OrderBy(entry => entry.Key, StringComparer.Ordinal)) { @@ -71,7 +93,10 @@ public static PdfARequirementReport Analyze(PdfFileReader reader, PdfAProfile pr // The same font, form or annotation can be reachable from several pages; the caller is told // once per condition, not once per page that repeats it. - return new PdfARequirementReport([.. unsatisfied.Distinct(StringComparer.Ordinal)], removable); + return new PdfARequirementReport( + [.. unsatisfied.Distinct(StringComparer.Ordinal)], + [.. removable.Distinct(StringComparer.Ordinal)], + [.. repairable.Distinct(StringComparer.Ordinal)]); } /// Checks the catalog: its name trees, its open action and its optional content. @@ -118,13 +143,16 @@ private static void CheckCatalog( { removable.Add("the catalog open action, which runs JavaScript"); } - else if (kind is not null && ForbiddenActions.Contains(kind)) + else { - unsatisfied.Add( - $"the catalog open action is a '{kind}' action, which {PdfAProfileRules.Name(profile)} forbids."); + // The open action names the one that runs after it, and a forbidden action is as often the + // second link as the first. + CheckAction(reader, catalog.Get("OpenAction"), "the catalog open action", profile, unsatisfied); } } + CheckOutlineActions(reader, reader.ResolveDictionary(catalog.Get("Outlines")), profile, unsatisfied); + CheckActionDictionary( reader, reader.ResolveDictionary(catalog.Get("AA")), @@ -138,6 +166,87 @@ private static void CheckCatalog( { removable.Add("the /NeedAppearances flag of the form, which asks the reader to build appearances"); } + + if (form?.ContainsKey("XFA") == true) + { + removable.Add("the XFA form of the document, which a reader of the archival copy cannot be asked to run"); + } + + if (form is not null) + { + CheckFieldActions(reader, form.Get("Fields"), removable, depth: 0, visited: []); + } + } + + /// Checks the action of every bookmark, which is a place the profile's action rules reach. + private static void CheckOutlineActions( + PdfFileReader reader, + PdfDictionary? outlines, + PdfAProfile profile, + List unsatisfied) + { + // A bookmark names its first child and its next sibling; the walk is by identity so that an outline + // that leads back to itself ends, and it stops at a count no real document reaches. + const int MaxBookmarks = 100_000; + + var seen = new HashSet(ReferenceEqualityComparer.Instance); + var pending = new Stack(); + + if (reader.ResolveDictionary(outlines?.Get("First")) is { } first) + { + pending.Push(first); + } + + while (pending.Count > 0 && seen.Count < MaxBookmarks) + { + var node = pending.Pop(); + + if (!seen.Add(node)) + { + continue; + } + + CheckAction(reader, node.Get("A"), "a bookmark", profile, unsatisfied); + + if (reader.ResolveDictionary(node.Get("Next")) is { } next) + { + pending.Push(next); + } + + if (reader.ResolveDictionary(node.Get("First")) is { } child) + { + pending.Push(child); + } + } + } + + /// Finds the fields of a form that carry an action, which the profile does not allow a field to hold. + private static void CheckFieldActions( + PdfFileReader reader, + PdfObject? fields, + List removable, + int depth, + HashSet visited) + { + if (depth > MaxFormDepth) + { + return; + } + + foreach (var item in ReadArray(reader, fields)) + { + if (reader.ResolveDictionary(item) is not { } field || !visited.Add(field)) + { + continue; + } + + if (field.ContainsKey("A") || field.ContainsKey("AA")) + { + removable.Add($"the action of the form field '{field.GetText("T") ?? "(unnamed)"}', which an archival document does not let a field carry"); + } + + CheckFieldActions(reader, field.Get("Kids"), removable, depth + 1, visited); + } } /// Checks every page, its resources, its content and its annotations. @@ -146,12 +255,16 @@ private static void CheckCatalog( /// The fonts found so far, with whether each is embedded. /// The list that receives the filters the profile forbids. /// The list that receives the requirements that are not met. + /// The list that receives the features the writer removes. + /// The list that receives the entries the writer sets right. private static void CheckPages( PdfFileReader reader, PdfAProfile profile, Dictionary fonts, HashSet filterProblems, - List unsatisfied) + List unsatisfied, + List removable, + List repairable) { // A form XObject can be painted by several pages; its resources and its content are checked // once, because an object number is what a reference to it names. @@ -169,6 +282,8 @@ private static void CheckPages( CheckResources(reader, resources, where, profile, fonts, filterProblems, unsatisfied, visited, depth: 0); + CheckActionDictionary(reader, reader.ResolveDictionary(page.Dictionary.Get("AA")), where, profile, unsatisfied); + if (PdfAProfileRules.ForbidsTransparency(profile)) { CheckTransparencyGroup(reader, page.Dictionary, where, unsatisfied); @@ -176,7 +291,7 @@ private static void CheckPages( foreach (var annotation in ReadArray(reader, page.Dictionary.Get("Annots"))) { - CheckAnnotation(reader, annotation, where, profile, fonts, filterProblems, unsatisfied, visited); + CheckAnnotation(reader, annotation, where, profile, fonts, filterProblems, unsatisfied, removable, repairable, visited); } } } @@ -231,6 +346,14 @@ private static void CheckResources( } } + foreach (var entry in reader.ResolveDictionary(resources.Get("ExtGState"))?.Entries ?? []) + { + if (reader.ResolveDictionary(entry.Value) is { } state) + { + CheckGraphicsState(reader, state, $"the graphics state '{entry.Key}' used by {where}", profile, unsatisfied); + } + } + if (reader.ResolveDictionary(resources.Get("ExtGState")) is { } states && PdfAProfileRules.ForbidsTransparency(profile)) { @@ -267,6 +390,57 @@ private static void CheckResources( } } + /// The blend modes ISO 32000-1 names; any other value of /BM is forbidden. + private static readonly HashSet BlendModes = new(StringComparer.Ordinal) + { + "Normal", "Compatible", "Multiply", "Screen", "Overlay", "Darken", "Lighten", "ColorDodge", "ColorBurn", + "HardLight", "SoftLight", "Difference", "Exclusion", + }; + + /// Checks the keys of an extended graphics state that the profiles forbid: halftone phase, transfer functions, halftones, odd blend modes. + private static void CheckGraphicsState( + PdfFileReader reader, + PdfDictionary state, + string where, + PdfAProfile profile, + List unsatisfied) + { + var profileName = PdfAProfileRules.Name(profile); + + if (state.ContainsKey("HTP")) + { + unsatisfied.Add($"{where} states a halftone phase (/HTP), which {profileName} forbids."); + } + + if (state.ContainsKey("TR")) + { + unsatisfied.Add($"{where} states a transfer function (/TR), which {profileName} forbids."); + } + + if (state.Get("TR2") is { } transfer && transfer is not PdfName { Value: "Default" }) + { + unsatisfied.Add($"{where} states a transfer function (/TR2) other than Default, which {profileName} forbids."); + } + + if (reader.ResolveDictionary(state.Get("HT")) is { } halftone + && (halftone.GetInteger("HalftoneType") is not (1 or 5) || halftone.ContainsKey("HalftoneName"))) + { + unsatisfied.Add($"{where} states a halftone of a kind, or with a name, that {profileName} forbids: only types 1 and 5, without /HalftoneName, are allowed."); + } + + var modes = reader.Resolve(state.Get("BM")) switch + { + PdfName name => new[] { name.Value }, + PdfArray array => array.Items.Select(item => (reader.Resolve(item) as PdfName)?.Value ?? string.Empty).ToArray(), + _ => [], + }; + + if (modes.Any(mode => !BlendModes.Contains(mode))) + { + unsatisfied.Add($"{where} states a blend mode ISO 32000-1 does not define, which {profileName} forbids."); + } + } + /// Checks one annotation: its actions, its appearance and the resources of its appearance. /// The source document. /// The annotation, possibly as a reference. @@ -275,6 +449,8 @@ private static void CheckResources( /// The fonts found so far, with whether each is embedded. /// The list that receives the filters the profile forbids. /// The list that receives the requirements that are not met. + /// The list that receives the features the writer removes. + /// The list that receives the entries the writer sets right. /// The objects already inspected. private static void CheckAnnotation( PdfFileReader reader, @@ -284,6 +460,8 @@ private static void CheckAnnotation( Dictionary fonts, HashSet filterProblems, List unsatisfied, + List removable, + List repairable, HashSet visited) { if (reader.ResolveDictionary(value) is not { } annotation) @@ -293,9 +471,36 @@ private static void CheckAnnotation( var subtype = annotation.GetName("Subtype") ?? "annotation"; var description = $"the {subtype} of {where}"; + var profileName = PdfAProfileRules.Name(profile); - CheckAction(reader, annotation.Get("A"), description, profile, unsatisfied); - CheckActionDictionary(reader, reader.ResolveDictionary(annotation.Get("AA")), description, profile, unsatisfied); + if (annotation.GetName("Subtype") is { } type && !PdfAAnnotationRules.PermitsType(profile, type)) + { + unsatisfied.Add($"{description} is of a type {profileName} does not permit."); + } + + if (PdfAAnnotationRules.IsHidden(annotation, profile)) + { + // Clearing a flag that hides the annotation would put on the page something nobody saw. + unsatisfied.Add($"{description} is hidden by its flags, which {profileName} forbids; the writer does not change what a viewer shows."); + } + else if (PdfAAnnotationRules.NeedsPrintFlag(annotation, profile)) + { + repairable.Add($"the flags of {description} did not set the Print bit {profileName} requires; the writer set it."); + } + + if (PdfAAnnotationRules.IsWidget(annotation)) + { + // A widget carries no action in an archival document: the form is read without running anything. + if (annotation.ContainsKey("A") || annotation.ContainsKey("AA")) + { + removable.Add($"the action of {description}, which {profileName} does not let a widget carry"); + } + } + else + { + CheckAction(reader, annotation.Get("A"), description, profile, unsatisfied); + CheckActionDictionary(reader, reader.ResolveDictionary(annotation.Get("AA")), description, profile, unsatisfied); + } if (annotation.Get("AP") is null) { @@ -450,10 +655,51 @@ private static void CheckAction( PdfAProfile profile, List unsatisfied) { - if (reader.ResolveDictionary(value)?.GetName("S") is { } kind && ForbiddenActions.Contains(kind)) + // An action names the one that runs after it in /Next, as one action or an array of them, so a + // forbidden action is as often the second link of a chain as the first. The dictionaries are + // compared by identity: the reader hands back the same instance for the same object, which is what + // ends a chain that leads back to itself. + var seen = new HashSet(ReferenceEqualityComparer.Instance); + var pending = new Stack<(PdfObject? Action, int Depth)>(); + + pending.Push((value, 0)); + + while (pending.Count > 0) { - unsatisfied.Add( - $"{where} carries a '{kind}' action, which {PdfAProfileRules.Name(profile)} forbids."); + var (current, depth) = pending.Pop(); + + if (depth > MaxActionChain || reader.ResolveDictionary(current) is not { } action || !seen.Add(action)) + { + continue; + } + + var kind = action.GetName("S"); + + if (kind is not null && ForbiddenActions.Contains(kind)) + { + unsatisfied.Add($"{where} carries a '{kind}' action, which {PdfAProfileRules.Name(profile)} forbids."); + } + else if (string.Equals(kind, "Named", StringComparison.Ordinal) + && !StandardNamedActions.Contains(action.GetName("N") ?? string.Empty)) + { + unsatisfied.Add( + $"{where} carries the named action '{action.GetName("N") ?? "(none)"}', which {PdfAProfileRules.Name(profile)} forbids: only NextPage, PrevPage, FirstPage and LastPage are allowed."); + } + + switch (reader.Resolve(action.Get("Next"))) + { + case PdfDictionary next: + pending.Push((next, depth + 1)); + break; + + case PdfArray chain: + foreach (var item in chain.Items) + { + pending.Push((item, depth + 1)); + } + + break; + } } } diff --git a/src/SharpCorePDF.Infrastructure/Authoring/PdfAWriter.cs b/src/SharpCorePDF.Infrastructure/Authoring/PdfAWriter.cs index 06c995e..9c76113 100644 --- a/src/SharpCorePDF.Infrastructure/Authoring/PdfAWriter.cs +++ b/src/SharpCorePDF.Infrastructure/Authoring/PdfAWriter.cs @@ -110,7 +110,10 @@ public async Task ConvertAsync( $"the document has no page tree, so it cannot be written for {PdfAProfileRules.Name(profile)}."); } - var requirements = PdfARequirements.Analyze(reader, profile); + // The intent the source already states is kept when the caller supplies none: the document was prepared + // for that condition, and writing sRGB over it would contradict content painted in its colour space. + var outputIntent = effective.OutputIntent ?? ReadSourceOutputIntent(reader); + var requirements = PdfARequirements.Analyze(reader, profile, outputIntent?.NumberOfComponents ?? 3); if (requirements.Unsatisfied.Count > 0) { @@ -134,7 +137,7 @@ public async Task ConvertAsync( ModificationDate = effective.ModificationDate, Producer = effective.Producer, DocumentId = effective.DocumentId, - OutputIntent = effective.OutputIntent, + OutputIntent = outputIntent, AssociatedFileRelationship = effective.AssociatedFileRelationship, }); @@ -155,4 +158,53 @@ public async Task ConvertAsync( Warnings = edited.Warnings, }; } + + /// Reads the PDF/A output intent a source states, when it states a usable one. + /// The source document. + /// The intent, or when the source has none or its profile is not a usable ICC profile. + private static PdfOutputIntent? ReadSourceOutputIntent(PdfFileReader reader) + { + foreach (var item in reader.ResolveArray(reader.Catalog?.Get("OutputIntents"))?.Items ?? []) + { + if (reader.ResolveDictionary(item) is not { } intent + || !string.Equals(intent.GetName("S"), "GTS_PDFA1", StringComparison.Ordinal) + || reader.Resolve(intent.Get("DestOutputProfile")) is not PdfStream profile) + { + continue; + } + + byte[] icc; + + try + { + icc = PdfFilters.Decode(profile, reader.SourceName, reader.Options.MaxDecodedStreamBytes); + } + catch (PdfSourceFormatException) + { + continue; + } + catch (PdfUnsupportedFeatureException) + { + continue; + } + + var candidate = new PdfOutputIntent + { + IccProfile = icc, + NumberOfComponents = (int)(profile.Dictionary.GetInteger("N") ?? 0), + OutputConditionIdentifier = intent.GetText("OutputConditionIdentifier") ?? "Custom", + OutputCondition = intent.GetText("OutputCondition"), + Info = intent.GetText("Info"), + RegistryName = intent.GetText("RegistryName"), + }; + + // The same checks a caller's profile has to pass: the header, the signature and the component count. + if (candidate.Validate().Count == 0) + { + return candidate; + } + } + + return null; + } } diff --git a/src/SharpCorePDF.Infrastructure/Forms/FieldAppearance.cs b/src/SharpCorePDF.Infrastructure/Forms/FieldAppearance.cs index 70279d6..07e1145 100644 --- a/src/SharpCorePDF.Infrastructure/Forms/FieldAppearance.cs +++ b/src/SharpCorePDF.Infrastructure/Forms/FieldAppearance.cs @@ -85,13 +85,18 @@ public static PdfStream Wrap(byte[] operators, int fontObjectNumber, PdfRectangl .Set("BBox", AppearanceStreams.ToArray(box)) .SetName("Filter", "FlateDecode"); + // Stated even when empty, as an annotation's appearance does: a form XObject without /Resources leaves + // a reader to borrow the page's, a PDF 1.1 fallback the Arlington model of PDF 1.7 and later no longer + // allows. A button that draws no text has no font to name, and is the case that was left without. + var resources = new PdfDictionary(); + if (fontObjectNumber != 0) { - dictionary.Set( - "Resources", - new PdfDictionary().Set("Font", new PdfDictionary().Set(FontName, new PdfReference(fontObjectNumber, 0)))); + resources.Set("Font", new PdfDictionary().Set(FontName, new PdfReference(fontObjectNumber, 0))); } + dictionary.Set("Resources", resources); + return new PdfStream(dictionary, Authoring.EmbeddedImage.Compress(operators)); } diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfFormBudget.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfFormBudget.cs new file mode 100644 index 0000000..4bcd0bf --- /dev/null +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfFormBudget.cs @@ -0,0 +1,100 @@ +using SharpCorePDF.Domain.ValueObjects; + +namespace SharpCorePDF.Infrastructure.Pdf; + +/// +/// What one page may spend on the forms it paints: how many paints, how many decoded bytes of form content, and a +/// small cache so that the same form painted again is not decoded again. +/// +/// +/// The walkers that follow a page into its forms share this rule instead of each carrying a copy, so that the +/// limit one reports is the limit the other applies. A new page starts a new budget. +/// +internal sealed class PdfFormBudget +{ + /// The decoded form content one page may read, in bytes. + internal const long MaxBytesPerPage = 256L * 1024 * 1024; + + /// The decoded form content kept for a page so a repeated paint does not decode it again, in bytes. + private const long MaxCachedBytes = 16L * 1024 * 1024; + + private readonly int maxPaints; + private readonly Dictionary decoded = []; + private int page; + private int paints; + private long bytes; + private long cachedBytes; + private bool reported; + + /// Initializes a new instance of the class. + /// The largest number of form paints per page. + public PdfFormBudget(int maxPaints) => this.maxPaints = maxPaints; + + /// Starts the budget of a page, which discards what the previous page spent and cached. + /// The one-based page number. + public void StartPage(int pageNumber) + { + if (page == pageNumber) + { + return; + } + + page = pageNumber; + paints = 0; + bytes = 0; + cachedBytes = 0; + reported = false; + decoded.Clear(); + } + + /// Asks to paint one more form. + /// The list that receives the remark, once per page, when the limit is reached. + /// The name of the source, used in the remark. + /// when the paint is within the budget. + public bool TryPaint(List warnings, string sourceName) + { + if (++paints <= maxPaints && bytes <= MaxBytesPerPage) + { + return true; + } + + if (!reported) + { + reported = true; + warnings.Add(new( + PdfWarningKind.FormPaintLimitExceeded, + $"'{sourceName}': page {page} paints forms more than {maxPaints} times or reads more than {MaxBytesPerPage} bytes of form content; the rest of the page was not read.")); + } + + return false; + } + + /// Looks for the decoded content of a form painted before on this page. + /// The object number of the form, or 0 for a form that is not an indirect object. + /// The decoded content when it was cached. + /// when the content was cached, which is also counted against the page's bytes. + public bool TryGetCached(long objectNumber, out byte[] content) + { + if (objectNumber != 0 && decoded.TryGetValue(objectNumber, out content!)) + { + bytes += content.Length; + return true; + } + + content = []; + return false; + } + + /// Counts the decoded content of a form against the page, and keeps it when there is room. + /// The object number of the form, or 0 for a form that is not an indirect object. + /// The decoded content. + public void Spend(long objectNumber, byte[] content) + { + bytes += content.Length; + + if (objectNumber != 0 && cachedBytes + content.Length <= MaxCachedBytes && decoded.TryAdd(objectNumber, content)) + { + cachedBytes += content.Length; + } + } +} diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfTextCollector.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfTextCollector.cs index b99551d..65e6b77 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfTextCollector.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfTextCollector.cs @@ -42,6 +42,7 @@ internal sealed class PdfTextCollector private List sink = []; private readonly HashSet reported = new(StringComparer.Ordinal); private readonly HashSet formsInProgress = []; + private readonly PdfFormBudget formBudget; /// Initializes a new instance of the class. /// The document. @@ -63,6 +64,8 @@ public PdfTextCollector( this.pageNumber = pageNumber; this.warnings = warnings; this.fontCache = fontCache; + formBudget = new PdfFormBudget(reader.Options.MaxFormPaintsPerPage); + formBudget.StartPage(pageNumber); } /// Collects the glyphs a page's content draws. @@ -373,6 +376,11 @@ private void PaintForm(string name, PdfDictionary? resources, TextState state, i return; } + if (!formBudget.TryPaint(warnings, reader.SourceName)) + { + return; + } + var reference = value as PdfReference; if (reference is not null && !formsInProgress.Add(reference.ObjectNumber)) @@ -382,13 +390,20 @@ private void PaintForm(string name, PdfDictionary? resources, TextState state, i try { - if (!TryDecodeContent(stream, reader.Options.MaxDecodedStreamBytes, out var formContent)) + var objectNumber = reference?.ObjectNumber ?? 0; + + if (!formBudget.TryGetCached(objectNumber, out var formContent)) { - warnings.Add(new( - PdfWarningKind.ContentStreamUnreadable, - $"'{reader.SourceName}': the content stream of a form on page {pageNumber} could not be decoded, so its text is not reported.")); + if (!TryDecodeContent(stream, reader.Options.MaxDecodedStreamBytes, out formContent)) + { + warnings.Add(new( + PdfWarningKind.ContentStreamUnreadable, + $"'{reader.SourceName}': the content stream of a form on page {pageNumber} could not be decoded, so its text is not reported.")); + + return; + } - return; + formBudget.Spend(objectNumber, formContent); } var formResources = reader.ResolveDictionary(stream.Dictionary.Get("Resources")) ?? resources; diff --git a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Document.cs b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Document.cs index 2117fb1..d53e438 100644 --- a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Document.cs +++ b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Document.cs @@ -129,6 +129,10 @@ private static PdfDictionary BuildCatalog( // flag that asks it to build appearances itself is cleared rather than copied. var form = (PdfDictionary)copier.Clone(sourceForm); form.Set("NeedAppearances", PdfBoolean.False); + + // An XFA form is a program a reader runs to build the form; the archival copy has to be painted + // from the appearances the widgets already carry. + form.Remove("XFA"); catalog.Set("AcroForm", form); } diff --git a/src/SharpCorePDF.Infrastructure/PdfEditEngine.cs b/src/SharpCorePDF.Infrastructure/PdfEditEngine.cs index 117e20f..3253fd3 100644 --- a/src/SharpCorePDF.Infrastructure/PdfEditEngine.cs +++ b/src/SharpCorePDF.Infrastructure/PdfEditEngine.cs @@ -136,7 +136,7 @@ public async Task ApplyAsync( { // The analysis is what keeps a refusal honest: a document that cannot be made conformant is // stopped here, before the writer exists and before a byte reaches the destination. - var requirements = PdfARequirements.Analyze(reader, archival); + var requirements = PdfARequirements.Analyze(reader, archival, plan.Output.OutputIntent?.NumberOfComponents ?? 3); if (requirements.Unsatisfied.Count > 0) { @@ -153,6 +153,11 @@ public async Task ApplyAsync( PdfWarningKind.PdfAFeatureRemoved, $"'{sourceName}': {removal} was removed because {PdfAProfileRules.Name(archival)} forbids it.")); } + + foreach (var repair in requirements.Repairable ?? []) + { + warnings.Add(new(PdfWarningKind.PdfAFeatureRepaired, $"'{sourceName}': {repair}")); + } } var sources = await OpenSourcesAsync(reader, plan, warnings, cancellationToken).ConfigureAwait(false); @@ -275,6 +280,13 @@ public async Task ApplyAsync( // flattened field adds a stream and a resource the page dictionary has to carry. var acroForm = WriteForm(reader, sources.CopierOf(reader), writer, layout, form, warnings); + if (profile is { } archivalProfile) + { + // The rewrite that makes each annotation and field state what the profile requires has to be + // registered before the objects are copied: they are written when the pages name them. + PdfAAnnotationRules.Register(reader, sources.CopierOf(reader), archivalProfile); + } + WritePages(sources, writer, pagesNodeNumber, layout, new AnnotationContext(writer, modificationDate), warnings); var catalog = BuildCatalog( diff --git a/src/SharpCorePDF.Infrastructure/PdfEngineOptions.cs b/src/SharpCorePDF.Infrastructure/PdfEngineOptions.cs index 8af71bd..a35f86e 100644 --- a/src/SharpCorePDF.Infrastructure/PdfEngineOptions.cs +++ b/src/SharpCorePDF.Infrastructure/PdfEngineOptions.cs @@ -36,6 +36,20 @@ public sealed record PdfEngineOptions /// public long MaxDecodedStreamBytes { get; init; } = 128L * 1024 * 1024; + /// + /// Gets the largest number of times the content of one page may paint a form XObject, counted across every + /// level of forms inside forms. Past it the rest of the page is not read, and a + /// remark says so. + /// + /// + /// and the guard against a form that paints itself bound how deep a page goes + /// and not how much it does: a form that paints the next form forty times, six levels down, is under both and + /// asks for four billion paints from a document of a few kilobytes. The default of 100,000 is far above what a + /// page that draws something needs. Alongside it, the forms of one page are read for at most 256 MiB of + /// decoded content, which bounds a page whose few paints are of very large forms. + /// + public int MaxFormPaintsPerPage { get; init; } = 100_000; + /// Gets the maximum nesting depth of dictionaries and arrays. public int MaxNestingDepth { get; init; } = 64; @@ -97,6 +111,11 @@ public IReadOnlyList Validate() errors.Add($"MaxDecodedStreamBytes must be between 1024 and {Array.MaxLength}, but is {MaxDecodedStreamBytes}."); } + if (MaxFormPaintsPerPage < 1) + { + errors.Add($"MaxFormPaintsPerPage must be at least 1, but is {MaxFormPaintsPerPage}."); + } + if (MaxNestingDepth is < 4 or > 1024) { errors.Add($"MaxNestingDepth must be between 4 and 1024, but is {MaxNestingDepth}."); diff --git a/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Images.cs b/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Images.cs index e56a77b..62e34df 100644 --- a/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Images.cs +++ b/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Images.cs @@ -153,6 +153,7 @@ private sealed class ImageWalker private readonly List order = []; private readonly Dictionary entries = new(StringComparer.Ordinal); private readonly HashSet formsInProgress = []; + private readonly PdfFormBudget formBudget; private int anonymous; /// Initializes a new instance of the class. @@ -164,6 +165,7 @@ public ImageWalker(PdfFileReader reader, int maxInlineBytes, List wa this.reader = reader; this.maxInlineBytes = maxInlineBytes; this.warnings = warnings; + formBudget = new PdfFormBudget(reader.Options.MaxFormPaintsPerPage); } /// Gets the identifier of the single image the walk is looking for, if any. @@ -353,12 +355,33 @@ private void Paint( return; } + // A page that paints forms without end is read up to a budget, and the images found before it are kept. + formBudget.StartPage(pageNumber); + + if (!formBudget.TryPaint(warnings, reader.SourceName)) + { + return; + } + if (reference is not null && !formsInProgress.Add(reference.ObjectNumber)) { return; } - if (TryDecodeStream(stream, reader.Options.MaxDecodedStreamBytes, out var formContent)) + var objectNumber = reference?.ObjectNumber ?? 0; + byte[]? formContent = null; + + if (formBudget.TryGetCached(objectNumber, out var cached)) + { + formContent = cached; + } + else if (TryDecodeStream(stream, reader.Options.MaxDecodedStreamBytes, out var fresh)) + { + formBudget.Spend(objectNumber, fresh); + formContent = fresh; + } + + if (formContent is not null) { var formResources = reader.ResolveDictionary(stream.Dictionary.Get("Resources")) ?? resources; var formMatrix = ReadMatrix(reader, stream.Dictionary.Get("Matrix")) ?? PdfMatrix.Identity; diff --git a/src/SharpCorePDF.Infrastructure/PublicAPI.Unshipped.txt b/src/SharpCorePDF.Infrastructure/PublicAPI.Unshipped.txt index f73aab7..6a962d8 100644 --- a/src/SharpCorePDF.Infrastructure/PublicAPI.Unshipped.txt +++ b/src/SharpCorePDF.Infrastructure/PublicAPI.Unshipped.txt @@ -1,3 +1,5 @@ #nullable enable SharpCorePDF.Infrastructure.PdfEngineOptions.MaxDecodedStreamBytes.get -> long SharpCorePDF.Infrastructure.PdfEngineOptions.MaxDecodedStreamBytes.init -> void +SharpCorePDF.Infrastructure.PdfEngineOptions.MaxFormPaintsPerPage.get -> int +SharpCorePDF.Infrastructure.PdfEngineOptions.MaxFormPaintsPerPage.init -> void diff --git a/tests/SharpCorePDF.Tests/PdfAArchivalChecksTests.cs b/tests/SharpCorePDF.Tests/PdfAArchivalChecksTests.cs new file mode 100644 index 0000000..de87b1a --- /dev/null +++ b/tests/SharpCorePDF.Tests/PdfAArchivalChecksTests.cs @@ -0,0 +1,468 @@ +using System.Text; +using SharpCorePDF.Domain.ReadModels; +using SharpCorePDF.Domain.ValueObjects; +using SharpCorePDF.Infrastructure; +using SharpCorePDF.Infrastructure.Authoring; +using SharpCorePDF.Infrastructure.Pdf; + +namespace SharpCorePDF.Tests; + +/// +/// Tests of what the archival conversion does about the rules of ISO 19005 that live in annotations, actions, +/// forms and content streams: what it sets right, what it removes, and what it refuses. +/// +/// +/// Each test builds the smallest document that carries one feature and converts it for PDF/A-2b, so what is +/// asserted is the conversion's answer and the bytes it wrote, not what a validator makes of them. +/// docs/validation.md records the measurement against the veraPDF corpus, which is the other half of the +/// evidence: these tests pin each behaviour, and the corpus says how far the behaviours reach. +/// +public sealed class PdfAArchivalChecksTests +{ + private static readonly DateTimeOffset Moment = new(2026, 3, 1, 12, 0, 0, TimeSpan.Zero); + + private static CancellationToken Token => TestContext.Current.CancellationToken; + + /// An annotation with no flags prints as well as it shows once the profile's Print bit is set. + [Fact] + public async Task Sets_the_print_flag_of_an_annotation_that_states_none() + { + var (result, output) = await ConvertAsync(Annotated("/Subtype /Square /Rect [10 10 50 50]"), PdfAProfile.PdfA2B); + + Assert.True(result.IsWritten, string.Join(" ", result.UnsatisfiedRequirements)); + Assert.Equal(4, FlagsOf(output)); + Assert.Contains(result.Warnings, warning => warning.Kind == PdfWarningKind.PdfAFeatureRepaired); + } + + /// Flags the source already states are kept and the Print bit is added to them. + [Fact] + public async Task Adds_the_print_bit_to_flags_that_state_other_bits() + { + var (result, output) = await ConvertAsync(Annotated("/Subtype /Square /Rect [10 10 50 50] /F 8"), PdfAProfile.PdfA2B); + + Assert.True(result.IsWritten); + Assert.Equal(12, FlagsOf(output)); + } + + /// An annotation that already prints is left exactly as it is, and nothing is reported about it. + [Fact] + public async Task Leaves_the_flags_of_an_annotation_that_already_prints() + { + var (result, output) = await ConvertAsync(Annotated("/Subtype /Square /Rect [10 10 50 50] /F 4"), PdfAProfile.PdfA2B); + + Assert.True(result.IsWritten); + Assert.Equal(4, FlagsOf(output)); + Assert.DoesNotContain(result.Warnings, warning => warning.Kind == PdfWarningKind.PdfAFeatureRepaired); + } + + /// A flag that hides the annotation is refused: clearing it would show what nobody saw. + [Theory] + [InlineData(1)] + [InlineData(2)] + [InlineData(32)] + public async Task Refuses_an_annotation_that_its_flags_hide(int flags) + { + var (result, _) = await ConvertAsync(Annotated($"/Subtype /Square /Rect [10 10 50 50] /F {flags + 4}"), PdfAProfile.PdfA2B); + + Assert.False(result.IsWritten); + Assert.Contains(result.UnsatisfiedRequirements, requirement => requirement.Contains("hidden by its flags", StringComparison.Ordinal)); + } + + /// A popup is exempt from the flags in parts 2 to 4, and a Link carries none of the appearance a square does. + [Fact] + public async Task Does_not_ask_a_popup_for_flags_in_part_two() + { + var (result, output) = await ConvertAsync(Annotated("/Subtype /Popup /Rect [10 10 50 50]"), PdfAProfile.PdfA2B); + + Assert.True(result.IsWritten); + Assert.Null(AnnotationOf(output).Get("F")); + } + + /// The types the profile does not list — here a movie — are refused. + [Fact] + public async Task Refuses_an_annotation_of_a_type_the_profile_does_not_permit() + { + var (result, _) = await ConvertAsync(Annotated("/Subtype /Movie /Rect [10 10 50 50] /F 4"), PdfAProfile.PdfA2B); + + Assert.False(result.IsWritten); + Assert.Contains(result.UnsatisfiedRequirements, requirement => requirement.Contains("does not permit", StringComparison.Ordinal)); + } + + /// The first part lists fewer types than the second: a polygon is permitted by one and not the other. + [Fact] + public void Lists_the_annotation_types_of_each_part_as_the_reference_validator_does() + { + Assert.True(PdfAAnnotationRules.PermitsType(PdfAProfile.PdfA2B, "Polygon")); + Assert.False(PdfAAnnotationRules.PermitsType(PdfAProfile.PdfA1B, "Polygon")); + Assert.True(PdfAAnnotationRules.PermitsType(PdfAProfile.PdfA2B, "FileAttachment")); + Assert.False(PdfAAnnotationRules.PermitsType(PdfAProfile.PdfA4, "FileAttachment")); + Assert.False(PdfAAnnotationRules.PermitsType(PdfAProfile.PdfA2B, "3D")); + Assert.False(PdfAAnnotationRules.PermitsType(PdfAProfile.PdfA2B, "NotAType")); + } + + /// Part 1 asks a popup for flags too; the later parts do not. + [Fact] + public void Asks_a_popup_for_flags_only_in_part_one() + { + Assert.True(PdfAAnnotationRules.RequiresFlags(PdfAProfile.PdfA1B, "Popup")); + Assert.False(PdfAAnnotationRules.RequiresFlags(PdfAProfile.PdfA2B, "Popup")); + Assert.True(PdfAAnnotationRules.RequiresFlags(PdfAProfile.PdfA2B, "Square")); + } + + /// A widget carries no action in an archival document, and the copy does not. + [Fact] + public async Task Removes_the_action_of_a_widget() + { + const string Widget = "/Subtype /Widget /FT /Btn /T (ok) /Rect [10 10 50 50] /F 4 /A << /S /URI /URI (https://example.org) >>"; + + var (result, output) = await ConvertAsync(Annotated(Widget, catalogExtra: "/AcroForm << /Fields [5 0 R] >>"), PdfAProfile.PdfA2B); + + Assert.True(result.IsWritten, string.Join(" ", result.UnsatisfiedRequirements)); + Assert.Null(AnnotationOf(output).Get("A")); + Assert.Contains(result.Warnings, warning => warning.Kind == PdfWarningKind.PdfAFeatureRemoved + && warning.Message.Contains("widget", StringComparison.Ordinal)); + } + + /// The XFA form of a document is not carried into the archival copy. + [Fact] + public async Task Drops_the_xfa_form() + { + var (result, output) = await ConvertAsync( + Annotated("/Subtype /Square /Rect [10 10 50 50] /F 4", catalogExtra: "/AcroForm << /Fields [] /XFA [(template) 6 0 R] >>"), + PdfAProfile.PdfA2B); + + Assert.True(result.IsWritten, string.Join(" ", result.UnsatisfiedRequirements)); + Assert.False(Open(output).ResolveDictionary(Open(output).Catalog!.Get("AcroForm"))!.ContainsKey("XFA")); + Assert.Contains(result.Warnings, warning => warning.Kind == PdfWarningKind.PdfAFeatureRemoved + && warning.Message.Contains("XFA", StringComparison.Ordinal)); + } + + /// A forbidden action is as often the second link of a chain as the first; the chain is followed. + [Fact] + public async Task Refuses_a_forbidden_action_that_an_allowed_one_names_in_next() + { + const string Link = "/Subtype /Link /Rect [10 10 50 50] /F 4 /A << /S /URI /URI (https://example.org) /Next << /S /Launch /F (x.exe) >> >>"; + + var (result, _) = await ConvertAsync(Annotated(Link), PdfAProfile.PdfA2B); + + Assert.False(result.IsWritten); + Assert.Contains(result.UnsatisfiedRequirements, requirement => requirement.Contains("'Launch'", StringComparison.Ordinal)); + } + + /// The action of a bookmark is looked at too. + [Fact] + public async Task Refuses_a_forbidden_action_on_a_bookmark() + { + var builder = new PdfBuilder(); + var catalog = builder.Reserve(); + var pages = builder.Reserve(); + + builder.Add("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Contents 4 0 R >>"); + builder.AddStream(string.Empty, "0 0 m 1 1 l S"u8.ToArray()); + builder.Add("<< /Type /Outlines /First 6 0 R /Last 6 0 R /Count 1 >>"); + builder.Add("<< /Title (go) /Parent 5 0 R /A << /S /Launch /F (x.exe) >> >>"); + builder.Set(catalog, "<< /Type /Catalog /Pages 2 0 R /Outlines 5 0 R >>"); + builder.Set(pages, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + + var (result, _) = await ConvertAsync(builder.Build(catalog), PdfAProfile.PdfA2B); + + Assert.False(result.IsWritten); + Assert.Contains(result.UnsatisfiedRequirements, requirement => requirement.Contains("a bookmark", StringComparison.Ordinal)); + } + + /// A named action outside the four the profiles allow is refused, and one of the four is not. + [Theory] + [InlineData("Print", false)] + [InlineData("NextPage", true)] + public async Task Refuses_a_named_action_other_than_the_four_standard_ones(string name, bool written) + { + var (result, _) = await ConvertAsync( + Annotated($"/Subtype /Link /Rect [10 10 50 50] /F 4 /A << /S /Named /N /{name} >>"), + PdfAProfile.PdfA2B); + + Assert.Equal(written, result.IsWritten); + } + + /// The submission of a form is permitted by every part, which the analysis used to refuse. + [Fact] + public async Task Writes_a_document_whose_link_submits_a_form() + { + var (result, _) = await ConvertAsync( + Annotated("/Subtype /Link /Rect [10 10 50 50] /F 4 /A << /S /SubmitForm /F (https://example.org/submit) >>"), + PdfAProfile.PdfA2B); + + Assert.True(result.IsWritten, string.Join(" ", result.UnsatisfiedRequirements)); + } + + /// Content in DeviceCMYK under the writer's RGB output intent is refused: the two contradict. + [Fact] + public async Task Refuses_content_that_paints_in_devicecmyk_under_an_rgb_intent() + { + var (result, _) = await ConvertAsync(Plain("0 0 0 1 k 10 10 50 50 re f"), PdfAProfile.PdfA2B); + + Assert.False(result.IsWritten); + Assert.Contains(result.UnsatisfiedRequirements, requirement => requirement.Contains("DeviceCMYK", StringComparison.Ordinal)); + } + + /// The same content is written when the caller supplies the CMYK intent it was prepared for. + [Fact] + public async Task Writes_content_in_devicecmyk_under_a_cmyk_intent_the_caller_supplies() + { + var options = Options() with { OutputIntent = CmykIntent() }; + + var (result, output) = await ConvertAsync(Plain("0 0 0 1 k 10 10 50 50 re f"), PdfAProfile.PdfA2B, options); + + Assert.True(result.IsWritten, string.Join(" ", result.UnsatisfiedRequirements)); + Assert.Equal(4, IntentComponents(output)); + } + + /// + /// The colour space a transparency group states counts as a use of it: a group in DeviceRGB breaks the rule under a + /// CMYK output intent as a fill in DeviceRGB does, which the reference validator reports against the group. + /// + [Fact] + public async Task Refuses_a_transparency_group_in_devicergb_under_a_cmyk_intent() + { + var options = Options() with { OutputIntent = CmykIntent() }; + var source = Plain("0 0 m 1 1 l S", pageExtra: "/Group << /S /Transparency /CS /DeviceRGB >>"); + + var (result, _) = await ConvertAsync(source, PdfAProfile.PdfA2B, options); + + Assert.False(result.IsWritten); + Assert.Contains(result.UnsatisfiedRequirements, requirement => requirement.Contains("transparency group", StringComparison.Ordinal) + && requirement.Contains("DeviceRGB", StringComparison.Ordinal)); + } + + /// A page's content may be an array of streams, read as one: the operator in the second one is still seen. + [Fact] + public async Task Reads_a_content_array_as_one_stream_and_finds_what_the_second_part_paints() + { + var builder = new PdfBuilder(); + var catalog = builder.Reserve(); + var pages = builder.Reserve(); + + builder.Add("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Contents [4 0 R 5 0 R] >>"); + builder.AddStream(string.Empty, "q"u8.ToArray()); + builder.AddStream(string.Empty, "0 0 0 1 k 10 10 50 50 re f Q"u8.ToArray()); + builder.Set(catalog, "<< /Type /Catalog /Pages 2 0 R >>"); + builder.Set(pages, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + + var (result, _) = await ConvertAsync(builder.Build(catalog), PdfAProfile.PdfA2B); + + Assert.False(result.IsWritten); + Assert.Contains(result.UnsatisfiedRequirements, requirement => requirement.Contains("DeviceCMYK", StringComparison.Ordinal)); + } + + /// A DefaultCMYK space the page defines says what DeviceCMYK means, so no matching intent is needed. + [Fact] + public async Task Writes_content_in_devicecmyk_when_the_page_defines_a_default_space() + { + var source = Plain( + "0 0 0 1 k 10 10 50 50 re f", + pageExtra: "/Resources << /ColorSpace << /DefaultCMYK [/ICCBased 5 0 R] >> >>", + extra: ("/N 4", Encoding.Latin1.GetBytes("not read"))); + + var (result, _) = await ConvertAsync(source, PdfAProfile.PdfA2B); + + Assert.True(result.IsWritten, string.Join(" ", result.UnsatisfiedRequirements)); + } + + /// A Separation space counts through its alternate: it is painted in DeviceCMYK when its colorant is missing. + [Fact] + public async Task Refuses_a_separation_space_built_on_devicecmyk_under_an_rgb_intent() + { + var source = Plain( + "/CS0 cs 1 scn 10 10 50 50 re f", + pageExtra: "/Resources << /ColorSpace << /CS0 [/Separation /Spot /DeviceCMYK << /FunctionType 2 /Domain [0 1] /C0 [0 0 0 0] /C1 [0 0 0 1] /N 1 >>] >> >>"); + + var (result, _) = await ConvertAsync(source, PdfAProfile.PdfA2B); + + Assert.False(result.IsWritten); + Assert.Contains(result.UnsatisfiedRequirements, requirement => requirement.Contains("DeviceCMYK", StringComparison.Ordinal)); + } + + /// + /// A source that already states a usable output intent keeps it. Writing sRGB over a CMYK intent would leave + /// content painted in CMYK under an intent that contradicts it. + /// + [Fact] + public async Task Keeps_the_output_intent_a_source_already_states() + { + var profile = CmykIntent(); + var source = Plain( + "0 0 0 1 k 10 10 50 50 re f", + catalogExtra: "/OutputIntents [<< /Type /OutputIntent /S /GTS_PDFA1 /OutputConditionIdentifier (CMYK source) /DestOutputProfile 5 0 R >>]", + extra: ("/N 4", profile.IccProfile.ToArray())); + + var (result, output) = await ConvertAsync(source, PdfAProfile.PdfA2B); + + Assert.True(result.IsWritten, string.Join(" ", result.UnsatisfiedRequirements)); + Assert.Equal(4, IntentComponents(output)); + } + + /// An operator ISO 32000 does not define is refused, wherever it stands. + [Theory] + [InlineData("1 2 foo")] + [InlineData("BX 1 2 foo EX")] + public async Task Refuses_content_that_uses_an_operator_nothing_defines(string content) + { + var (result, _) = await ConvertAsync(Plain(content), PdfAProfile.PdfA2B); + + Assert.False(result.IsWritten); + Assert.Contains(result.UnsatisfiedRequirements, requirement => requirement.Contains("'foo'", StringComparison.Ordinal)); + } + + /// The operators of the specification, and the three literals a property list holds, are not refused. + [Fact] + public async Task Writes_content_that_uses_only_defined_operators() + { + var (result, _) = await ConvertAsync( + Plain("q 1 0 0 rg 10 10 50 50 re f Q /Span <> BDC EMC [1 2] 0 d 1 w 0 0 m 5 5 l S"), + PdfAProfile.PdfA2B); + + Assert.True(result.IsWritten, string.Join(" ", result.UnsatisfiedRequirements)); + } + + /// The graphics-state keys the profiles forbid are refused, each by name. + [Theory] + [InlineData("/HTP 1 0 R", "halftone phase")] + [InlineData("/TR 1 0 R", "transfer function")] + [InlineData("/TR2 /Custom", "transfer function")] + [InlineData("/BM /Weird", "blend mode")] + public async Task Refuses_an_extended_graphics_state_the_profile_forbids(string entry, string named) + { + var source = Plain( + "/GS0 gs 10 10 50 50 re f", + pageExtra: $"/Resources << /ExtGState << /GS0 << /Type /ExtGState {entry} >> >> >>"); + + var (result, _) = await ConvertAsync(source, PdfAProfile.PdfA2B); + + Assert.False(result.IsWritten); + Assert.Contains(result.UnsatisfiedRequirements, requirement => requirement.Contains(named, StringComparison.Ordinal)); + } + + /// A blend mode the specification names, in either spelling, is not refused. + [Theory] + [InlineData("/BM /Multiply")] + [InlineData("/BM [/Multiply /Screen]")] + public async Task Writes_an_extended_graphics_state_with_a_defined_blend_mode(string entry) + { + var source = Plain( + "/GS0 gs 10 10 50 50 re f", + pageExtra: $"/Resources << /ExtGState << /GS0 << /Type /ExtGState {entry} >> >> >>"); + + var (result, _) = await ConvertAsync(source, PdfAProfile.PdfA2B); + + Assert.True(result.IsWritten, string.Join(" ", result.UnsatisfiedRequirements)); + } + + private static PdfAConversionOptions Options() => new() + { + ModificationDate = Moment, + DocumentId = "00112233445566778899AABBCCDDEEFF", + }; + + private static PdfOutputIntent CmykIntent() + { + var bytes = new byte[132]; + + bytes[3] = 132; + Encoding.ASCII.GetBytes("CMYK").CopyTo(bytes, 16); + Encoding.ASCII.GetBytes("acsp").CopyTo(bytes, 36); + + return new PdfOutputIntent + { + IccProfile = bytes, + NumberOfComponents = 4, + OutputConditionIdentifier = "test CMYK", + }; + } + + private static async Task<(PdfAConversionResult Result, byte[] Output)> ConvertAsync( + byte[] source, + PdfAProfile profile, + PdfAConversionOptions? options = null) + { + using var destination = new MemoryStream(); + + var result = await new PdfAWriter().ConvertAsync( + "source.pdf", + new MemoryStream(source), + destination, + profile, + options ?? Options(), + password: null, + Token); + + return (result, destination.ToArray()); + } + + private static PdfFileReader Open(byte[] bytes) => new(bytes, "converted.pdf", PdfEngineOptions.Default); + + private static PdfDictionary AnnotationOf(byte[] output) + { + var reader = Open(output); + + return reader.ResolveDictionary(reader.ResolveArray(reader.Pages[0].Dictionary.Get("Annots"))!.Items[0])!; + } + + private static long FlagsOf(byte[] output) => AnnotationOf(output).GetInteger("F") ?? -1; + + private static long IntentComponents(byte[] output) + { + var reader = Open(output); + var intent = reader.ResolveDictionary(reader.ResolveArray(reader.Catalog!.Get("OutputIntents"))!.Items[0])!; + + return ((PdfStream)reader.Resolve(intent.Get("DestOutputProfile"))!).Dictionary.GetInteger("N") ?? -1; + } + + /// + /// Builds a one page document whose first annotation is object 5, with its form appearance as object 6. The + /// annotation carries an appearance stream, which the profile requires of every annotation but a link or a popup. + /// + private static byte[] Annotated(string annotation, string catalogExtra = "") + { + var builder = new PdfBuilder(); + var catalog = builder.Reserve(); + var pages = builder.Reserve(); + + builder.Add("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Contents 4 0 R /Annots [5 0 R] >>"); + builder.AddStream(string.Empty, "0 0 m 1 1 l S"u8.ToArray()); + builder.Add($"<< /Type /Annot {annotation} /AP << /N 6 0 R >> >>"); + + builder.AddStream("/Type /XObject /Subtype /Form /BBox [0 0 40 40] /Resources << >>", "0 0 m 1 1 l S"u8.ToArray()); + + builder.Set(catalog, $"<< /Type /Catalog /Pages 2 0 R {catalogExtra} >>"); + builder.Set(pages, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + + return builder.Build(catalog); + } + + /// + /// Builds a one page document that draws . The optional stream is object 5, which a + /// resource or an output intent can name. + /// + private static byte[] Plain( + string content, + string pageExtra = "", + string catalogExtra = "", + (string Entries, byte[] Data)? extra = null) + { + var builder = new PdfBuilder(); + var catalog = builder.Reserve(); + var pages = builder.Reserve(); + + builder.Add($"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Contents 4 0 R {pageExtra} >>"); + builder.AddStream(string.Empty, Encoding.Latin1.GetBytes(content)); + + if (extra is { } stream) + { + builder.AddStream(stream.Entries, stream.Data); + } + + builder.Set(catalog, $"<< /Type /Catalog /Pages 2 0 R {catalogExtra} >>"); + builder.Set(pages, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + + return builder.Build(catalog); + } +} diff --git a/tests/SharpCorePDF.Tests/PdfATests.cs b/tests/SharpCorePDF.Tests/PdfATests.cs index 2845314..3bef20b 100644 --- a/tests/SharpCorePDF.Tests/PdfATests.cs +++ b/tests/SharpCorePDF.Tests/PdfATests.cs @@ -245,8 +245,8 @@ public async Task Writes_the_callers_output_intent_instead_of_the_built_in_one() /// /// The header here is synthetic — a Gray profile with no colour conversion data — because what the test /// pins is that the writer takes the count and the registry from the intent instead of substituting its - /// own sRGB answers. Whether such a profile is a good idea for a page painted in device RGB is the - /// validator's question, and Chapter 24 of the manual says so. + /// own sRGB answers. The page is painted in gray, which a gray intent covers; a page painted in device RGB + /// under it is refused, as the next test pins. /// [Fact] public async Task Writes_the_registry_and_the_component_count_the_caller_states() @@ -260,7 +260,7 @@ public async Task Writes_the_registry_and_the_component_count_the_caller_states( }; var (result, output) = await ConvertAsync( - Authored(), + GrayPage(), PdfAProfile.PdfA2B, options: Options() with { OutputIntent = intent }); @@ -279,6 +279,45 @@ public async Task Writes_the_registry_and_the_component_count_the_caller_states( Assert.Equal(1, icc.Dictionary.GetInteger("N")); } + /// + /// A page painted in device RGB is refused under an intent that is not RGB. The two contradict — the standard + /// allows a device space only where the document says what it stands for — and the reference validator rejects + /// the file, so writing it would hand the caller an archival master that is not one. + /// + [Fact] + public async Task Refuses_a_page_painted_in_device_rgb_under_an_intent_that_is_not_rgb() + { + var intent = new PdfOutputIntent + { + IccProfile = IccHeader("GRAY"), + NumberOfComponents = 1, + OutputConditionIdentifier = "gray condition", + }; + + var (result, _) = await ConvertAsync( + Authored(), + PdfAProfile.PdfA2B, + options: Options() with { OutputIntent = intent }); + + Assert.False(result.IsWritten); + Assert.Contains(result.UnsatisfiedRequirements, requirement => requirement.Contains("DeviceRGB", StringComparison.Ordinal)); + } + + /// Builds a page painted only in gray, with no text and so no font to embed. + private static byte[] GrayPage() + { + var builder = new PdfBuilder(); + var catalog = builder.Reserve(); + var pages = builder.Reserve(); + + builder.Add("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Contents 4 0 R >>"); + builder.AddStream(string.Empty, "0.5 g 10 10 50 50 re f"u8.ToArray()); + builder.Set(catalog, "<< /Type /Catalog /Pages 2 0 R >>"); + builder.Set(pages, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + + return builder.Build(catalog); + } + /// Refuses data that is not an ICC profile before a byte of the result is written. [Fact] public async Task Refuses_an_output_intent_that_is_not_an_icc_profile() diff --git a/tests/SharpCorePDF.Tests/PdfFormBudgetTests.cs b/tests/SharpCorePDF.Tests/PdfFormBudgetTests.cs new file mode 100644 index 0000000..ebf7846 --- /dev/null +++ b/tests/SharpCorePDF.Tests/PdfFormBudgetTests.cs @@ -0,0 +1,133 @@ +using System.Text; +using SharpCorePDF.Application.Services; +using SharpCorePDF.Domain.ValueObjects; +using SharpCorePDF.Infrastructure; +using SharpCorePDF.Infrastructure.Sources; + +namespace SharpCorePDF.Tests; + +/// +/// Tests of the budget a page has for the forms it paints: how many paints, so that a small document cannot ask +/// for billions of them, and that the budget belongs to one page and not to the read. +/// +/// +/// The depth limit and the guard against a form that paints itself bound how deep a page goes, not how much it +/// does. A form that paints the next form twenty times, four levels down, is under both and asks for eight +/// thousand paints (1 + 20 + 400 + 8,000); six levels of forty is four billion, from a document of a few kilobytes. +/// +public sealed class PdfFormBudgetTests +{ + private static CancellationToken Token => TestContext.Current.CancellationToken; + + /// A page that paints forms past the limit is read as far as the limit and the caller is told. + [Fact] + public async Task Stops_reading_the_text_of_a_page_that_paints_more_forms_than_the_limit_and_says_so() + { + var document = FanOut(pages: 1, levels: 4, width: 20); + + var result = await Reader(maxPaints: 1_000).ReadTextAsync(Source(document), PageRange.All, cancellationToken: Token); + + var warning = Assert.Single(result.Warnings, w => w.Kind == PdfWarningKind.FormPaintLimitExceeded); + + Assert.Contains("page 1", warning.Message, StringComparison.Ordinal); + Assert.Contains("1000", warning.Message, StringComparison.Ordinal); + } + + /// The walk that looks for images keeps the same budget, and what it found before the limit is kept. + [Fact] + public async Task Stops_reading_the_images_of_a_page_that_paints_more_forms_than_the_limit_and_says_so() + { + var document = FanOut(pages: 1, levels: 4, width: 20); + + var result = await Reader(maxPaints: 1_000).ReadImagesAsync(Source(document), PageRange.All, cancellationToken: Token); + + Assert.Single(result.Warnings, w => w.Kind == PdfWarningKind.FormPaintLimitExceeded); + } + + /// A page that stays within the limit is read completely and nothing is reported. + [Fact] + public async Task Reads_a_page_that_paints_fewer_forms_than_the_limit_without_a_remark() + { + var document = FanOut(pages: 1, levels: 2, width: 10); + var reader = Reader(maxPaints: 1_000); + + var text = await reader.ReadTextAsync(Source(document), PageRange.All, cancellationToken: Token); + var images = await reader.ReadImagesAsync(Source(document), PageRange.All, cancellationToken: Token); + + Assert.DoesNotContain(text.Warnings, w => w.Kind == PdfWarningKind.FormPaintLimitExceeded); + Assert.DoesNotContain(images.Warnings, w => w.Kind == PdfWarningKind.FormPaintLimitExceeded); + } + + /// + /// The budget is a page's own: two pages that each stay within the limit are not charged together, so a long + /// document is not cut off for painting forms on every page. + /// + [Fact] + public async Task Gives_every_page_its_own_budget() + { + // 31 paints a page (the form, and the thirty paints it makes), 62 over the two: under 40 each, over 40 together. + var document = FanOut(pages: 2, levels: 2, width: 30); + var reader = Reader(maxPaints: 40); + + var text = await reader.ReadTextAsync(Source(document), PageRange.All, cancellationToken: Token); + var images = await reader.ReadImagesAsync(Source(document), PageRange.All, cancellationToken: Token); + + Assert.DoesNotContain(text.Warnings, w => w.Kind == PdfWarningKind.FormPaintLimitExceeded); + Assert.DoesNotContain(images.Warnings, w => w.Kind == PdfWarningKind.FormPaintLimitExceeded); + } + + /// The limit is an option with a default that real pages sit far below, and zero is not a limit. + [Fact] + public void States_the_limit_as_an_option_a_page_cannot_ask_to_be_without() + { + Assert.Equal(100_000, PdfEngineOptions.Default.MaxFormPaintsPerPage); + Assert.Contains( + (PdfEngineOptions.Default with { MaxFormPaintsPerPage = 0 }).Validate(), + error => error.Contains("MaxFormPaintsPerPage", StringComparison.Ordinal)); + Assert.Empty((PdfEngineOptions.Default with { MaxFormPaintsPerPage = 1 }).Validate()); + } + + private static PdfContentReaderService Reader(int maxPaints) => + new(new PdfPageContentReader(PdfEngineOptions.Default with { MaxFormPaintsPerPage = maxPaints })); + + private static PdfByteArraySource Source(byte[] bytes) => new(bytes, "fan-out.pdf"); + + /// + /// Builds a document whose every page paints one form, which paints the next times, + /// forms deep. No form paints itself, so no guard against a cycle applies. + /// + private static byte[] FanOut(int pages, int levels, int width) + { + var builder = new PdfBuilder(); + var catalog = builder.Reserve(); + var tree = builder.Reserve(); + var firstPage = 3; + var firstContent = firstPage + pages; + var firstForm = firstContent + pages; + var kids = string.Join(' ', Enumerable.Range(0, pages).Select(page => $"{firstPage + page} 0 R")); + + for (var page = 0; page < pages; page++) + { + builder.Add($"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Resources << /XObject << /F0 {firstForm} 0 R >> >> /Contents {firstContent + page} 0 R >>"); + } + + for (var page = 0; page < pages; page++) + { + builder.AddStream(string.Empty, Encoding.Latin1.GetBytes("/F0 Do")); + } + + for (var level = 0; level < levels; level++) + { + var last = level == levels - 1; + var content = last ? "0 0 m 1 1 l S\n" : string.Concat(Enumerable.Repeat($"q /F{level + 1} Do Q\n", width)); + var resources = last ? "<< >>" : $"<< /XObject << /F{level + 1} {firstForm + level + 1} 0 R >> >>"; + + builder.AddStream($"/Type /XObject /Subtype /Form /BBox [0 0 10 10] /Resources {resources}", Encoding.Latin1.GetBytes(content)); + } + + builder.Set(catalog, "<< /Type /Catalog /Pages 2 0 R >>"); + builder.Set(tree, $"<< /Type /Pages /Kids [{kids}] /Count {pages} >>"); + + return builder.Build(catalog); + } +} diff --git a/tests/SharpCorePDF.Tests/PdfFormEditorTests.cs b/tests/SharpCorePDF.Tests/PdfFormEditorTests.cs index 53db551..03e5615 100644 --- a/tests/SharpCorePDF.Tests/PdfFormEditorTests.cs +++ b/tests/SharpCorePDF.Tests/PdfFormEditorTests.cs @@ -97,6 +97,45 @@ public async Task Fills_and_reads_back_form_values() Assert.Equal(4, ((PdfArray)form.Get("Fields")!).Count); } + /// + /// Every appearance the engine draws for a field states /Resources, including the ones that draw no + /// text and so name no font: a check box has none to name, and a form XObject without the entry leaves a + /// reader to borrow the page's, a PDF 1.1 fallback the Arlington model of later versions no longer allows. + /// + [Fact] + public async Task States_resources_on_the_appearance_of_a_button_that_draws_no_text() + { + var (edited, _) = await ApplyAsync( + BlankPage(), + PdfEditPlan.Create().SetForm(PdfForm.Create( + PdfFormFields.CheckBox(1, "agree", new PdfRectangle(72, 600, 88, 616), isChecked: true), + PdfFormFields.Text(1, "person.name", new PdfRectangle(72, 640, 300, 664), "Ada Lovelace")))); + + var reader = Reader(edited); + var streams = new List(); + + // The widgets a viewer paints are the annotations of the page. + foreach (var item in reader.ResolveArray(reader.Pages[0].Dictionary.Get("Annots"))!.Items) + { + var appearance = reader.ResolveDictionary(reader.ResolveDictionary(item)!.Get("AP"))!; + + switch (reader.Resolve(appearance.Get("N"))) + { + case PdfStream stream: + streams.Add(stream); + break; + + case PdfDictionary states: + streams.AddRange(states.Entries.Select(entry => reader.Resolve(entry.Value)).OfType()); + break; + } + } + + // The check box states its on and off appearances, and the text field its one: three streams. + Assert.Equal(3, streams.Count); + Assert.All(streams, stream => Assert.NotNull(reader.ResolveDictionary(stream.Dictionary.Get("Resources")))); + } + /// A field the document carries is filled where it stands, and the rest is untouched. [Fact] public async Task Fills_a_field_the_document_already_carries() From e4c615e2581b74c92eb0f16a6a59508f086c5701 Mon Sep 17 00:00:00 2001 From: Andrew Tierney Date: Mon, 16 Nov 2026 21:40:20 +1100 Subject: [PATCH 14/15] Treat a page box that overflows as unreadable, and gate PDF/A false successes at the measured count The first Robustness run of this branch found a crash: a 378-byte file whose MediaBox digits overflow a double reads as infinite, and the inspection, the page reader and the edit engine each passed that to a constructor that refuses it, so an ArgumentOutOfRangeException escaped. A number that is not finite is now unreadable, so the page is described as A4 with a remark, as one with no box is. Assumes_a_size_for_a_page_whose_media_box_overflows and Edits_a_page_whose_media_box_overflows fail without the change. The same run measured the PDF/A conversion over the whole veraPDF corpus: 229 false successes, down from 395, and 2 conforming sources damaged, down from 17. robustness.yml now fails above 229, and docs/validation.md records the figures and the clauses they fall in. Scoped Stryker over the four files changed scores 100% (424 mutants). --- .github/workflows/robustness.yml | 2 +- CHANGELOG.md | 5 ++++ docs/release-checklist.md | 2 +- docs/validation.md | 16 ++++++++--- .../Pdf/PdfDocumentInspection.Pages.cs | 4 ++- .../PdfEditEngine.Pages.cs | 2 +- .../PdfPageContentReader.Pages.cs | 4 ++- .../PdfPageContentReader.cs | 2 +- .../PdfDocumentEditorTests.cs | 28 +++++++++++++++++++ .../PdfMergeFidelityTests.cs | 26 +++++++++++++++++ 10 files changed, 81 insertions(+), 10 deletions(-) diff --git a/.github/workflows/robustness.yml b/.github/workflows/robustness.yml index 50854f4..b5e18ed 100644 --- a/.github/workflows/robustness.yml +++ b/.github/workflows/robustness.yml @@ -265,7 +265,7 @@ jobs: --corpus "$RUNNER_TEMP/verapdf-corpus" \ --probe ./tools/SharpCorePDF.Validation/bin/Release/net11.0/SharpCorePDF.Validation.dll \ --verapdf "$VERAPDF_INSTALL/verapdf" \ - --report artifacts/preflight.json + --max-false-success 229 --report artifacts/preflight.json - name: Upload the report if: always() diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e7e6f2..8633125 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,11 @@ against an empty feed. ### Fixed +- **A media box whose digits overflow a double no longer crashes a read or an edit.** A number too large for a + double reads as infinite, and three readers passed it to a constructor that refuses it, so the `document` fuzz + target found a 378-byte file that threw `ArgumentOutOfRangeException` from the inspection, the page reader and + the edit engine. Such a box is now taken as unreadable, and the page is described as A4 with a remark, as one + with no box is. - **A hybrid-reference file is read.** Its classic table lists the objects it keeps in object streams as free, so a reader that knows no streams skips them, and describes them in the stream the trailer's `/XRefStm` names. The table kept the first entry it met, so the free entry hid the one in the stream, and such a file read as having diff --git a/docs/release-checklist.md b/docs/release-checklist.md index 6bc5c8f..b42efd0 100644 --- a/docs/release-checklist.md +++ b/docs/release-checklist.md @@ -640,7 +640,7 @@ second choice can be made on facts instead of on a memory of 14 November. | --- | --- | --- | | G1 — the surface | `PublicAPI.Unshipped.txt` lists four members: `PdfEngineOptions.MaxDecodedStreamBytes` and `.MaxFormPaintsPerPage` (Infrastructure) and `PdfWarningKind.PdfAFeatureRepaired` and `.FormPaintLimitExceeded` (Domain); the Application, DependencyInjection and entry-point files are empty | the `1.8.0` section says `Unshipped` is back to `#nullable enable`, which is no longer true, and new members are a minor-version change under the repository's SemVer rule: the next version is `1.9.0`, not a `1.8.x` patch, unless `1.8.0` is cut at the earlier commit. Two of the four are enum members, so a caller that switches over `PdfWarningKind` without a default sees two kinds it has not met | | G3 — notes | `CHANGELOG.md` `[Unreleased]` holds the validation checks, the security and robustness fixes, and the four image-filter decoders (run-length, CCITT, baseline and progressive JPEG) | the `1.8.0` bullet that says those filters "remain `ImageFilterUnsupported`" is wrong for any commit after the decoders landed; move `[Unreleased]` under the version heading when it is cut, with the date | -| G9 — the gate | the suite is **1,413 tests on each leg** (`net10.0` and `net11.0`), up from 1,229 | the figure in the `1.8.0` table is a record of that day, not a current claim | +| G9 — the gate | the suite is **1,415 tests on each leg** (`net10.0` and `net11.0`), up from 1,229 | the figure in the `1.8.0` table is a record of that day, not a current claim | | CI | `validation.yml` and `robustness.yml` are new beside `ci.yml` and `compliance.yml`; `compliance.yml` cross-validates PDF/A-4 as well | a release commit should have all four green; `robustness.yml` runs weekly and on dispatch, not on a push, so its newest run is the evidence, not the run for the commit | | Constitution | amendment `2.0.1` (a statement of fact about Python tools in CI) is proposed and not ratified | it needs the maintainer's approval in the pull request that carries it; nothing a release depends on | diff --git a/docs/validation.md b/docs/validation.md index 8cd8192..98f9f22 100644 --- a/docs/validation.md +++ b/docs/validation.md @@ -109,6 +109,7 @@ Fixed since the list was first drawn up, each with the test that fails without i | A merge copied the outline recursively, so a source nested about fifty thousand levels deep overflowed the stack; a looping or shared outline was walked again at every visit — a crash the `document` fuzz target found | the walk stops at 256 levels with `BookmarksTruncated`; a bookmark reached twice is copied once with `BookmarkSkipped` | `PdfMergeOutlineLimitsTests` | | Rebuilding a table sliced the document at offsets the damaged table stated past its end, or beyond an index's range, and threw `ArgumentOutOfRangeException` — a crash the `document` fuzz target found | an offset outside the document is passed over, and the rebuild finds the catalog | `PdfReaderRecoveryTests.Rebuilds_a_table_that_states_an_offset_past_the_end_of_the_file` | | Copying an attachment the listing reported, whose specification embeds no stream, threw `ArgumentException` — the public-corpora job's fault on `isartor-6-1-11-t01-fail-a.pdf` | it writes no bytes, matching the `ByteLength` of zero the listing reports | `PdfContentReaderTests.Copies_nothing_for_a_listed_file_that_embeds_no_stream` | +| A media box whose digits overflow a double read as infinite, and the inspection, the page reader and the edit engine passed it to a constructor that refuses it — a 378-byte crash the `document` fuzz target found on the first Robustness run of this branch | a number that is not finite is unreadable, so the page is described as A4 with a remark, as one with no box is | `PdfMergeFidelityTests.Assumes_a_size_for_a_page_whose_media_box_overflows`, `PdfDocumentEditorTests.Edits_a_page_whose_media_box_overflows` | | A hybrid-reference file read as having no pages: its classic table lists the objects it keeps in object streams as free, and the free entry hid the real one in `/XRefStm` | the stream may answer for the numbers its own section's table frees, and is read before the revision `/Prev` points at | `PdfCrossReferencePrecedenceTests.Reads_an_object_a_hybrid_file_lists_as_free_in_its_table_and_describes_in_its_stream`, `.Reads_the_stream_of_a_hybrid_section_before_the_revision_it_follows` | | An object stream answered for numbers the table places elsewhere: an object a later revision rewrote came back stale, and one it freed came back from the dead, depending on which object was resolved first | only the numbers the table places in a stream are taken from it, and the table is consulted before any cache | `.Resolves_the_object_the_table_names_not_the_copy_an_object_stream_still_holds`, `.Does_not_bring_back_an_object_the_table_lists_as_free` | | A page that paints forms without end — six levels of forms, each painting the next 40 times, from a 3.6 KB document — never finished reading | `PdfEngineOptions.MaxFormPaintsPerPage` (100,000) and a per-page budget of 256 MiB of decoded form content, with a per-page cache | `PdfFormBudgetTests` | @@ -155,8 +156,14 @@ output intent with sRGB. By profile: 1b 48, 2b 169, 2u 9, 4 169, 3b 0. | Fonts: `6.2.10.x` (part 4), `6.2.11.x` (part 2) | 88 | not looked at | | The rest: output intent profile `6.2.3` (18), integers beyond 32 bits `6.1.13` (15), operators `6.2.2` (15), `6.1.12` (13), `Alternates`/`OPI`/reference XObjects `6.2.7` to `6.2.9` (26), strings, metadata | 112 | not looked at, except `6.2.2` | -**After**, the ten clause groups that were changed, converted and judged locally with the same veraPDF -(150 fail-inputs and 51 pass-inputs of `PDF_A-2b`): +**After**, on the same corpus and the same veraPDF (`robustness.yml` run on `a23fdc3`, 2,074 files): 229 false +successes, of which 227 were sources meant to fail and 2 sources that conformed (`6-6-2-3-2-t01-pass-b` and +`-d`, whose metadata extension schemas the conversion rewrites). By profile: 1b 33, 2b 95, 2u 9, 4 92, 3b 0. The +remaining 229 are mostly rules the analysis does not read: fonts `6.2.11` (43) and `6.2.10` (38), colour `6.2.4` +(34), integers `6.1.13` (15), `6.1.12` (13), `6.2.8` (13). + +The ten clause groups that were changed, converted and judged locally file by file (150 fail-inputs and 51 +pass-inputs of `PDF_A-2b`): | | Refused | Conforms | Still fails | | --- | --- | --- | --- | @@ -170,8 +177,9 @@ the analysis does not read (`6.2.2-2`, `6.2.5-6`, `6.2.6-1`, `6.5.1-1`); they ar clause. The three conforming sources refused are over-refusals: their colour use is allowed by an intent or a `Default` space the analysis does not read. -`robustness.yml` repeats the full measurement weekly. Its `--max-false-success` is the ratchet: the run fails when the -count of false successes rises above the figure measured on the commit that set it. +`robustness.yml` repeats the full measurement weekly. Its `--max-false-success` is the ratchet, set to 229, the +figure above: the run fails when the count of false successes rises above it. Lower it in the change that lowers +the count. ## How the checks were calibrated diff --git a/src/SharpCorePDF.Infrastructure/Pdf/PdfDocumentInspection.Pages.cs b/src/SharpCorePDF.Infrastructure/Pdf/PdfDocumentInspection.Pages.cs index 10e6bfd..89ab13f 100644 --- a/src/SharpCorePDF.Infrastructure/Pdf/PdfDocumentInspection.Pages.cs +++ b/src/SharpCorePDF.Infrastructure/Pdf/PdfDocumentInspection.Pages.cs @@ -69,7 +69,9 @@ private static List ReadPages(PdfFileReader reader, List 0d && height > 0d ? PdfPageSize.FromPoints(width, height) : null; + return double.IsFinite(width) && width > 0d && double.IsFinite(height) && height > 0d + ? PdfPageSize.FromPoints(width, height) + : null; } /// Reads the rotation a page declares or inherits, as the nearest quarter turn. diff --git a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Pages.cs b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Pages.cs index 0098155..9ceb010 100644 --- a/src/SharpCorePDF.Infrastructure/PdfEditEngine.Pages.cs +++ b/src/SharpCorePDF.Infrastructure/PdfEditEngine.Pages.cs @@ -687,7 +687,7 @@ private static List Extract( private static double? ReadNumber(PdfFileReader reader, PdfObject? value) => reader.Resolve(value) switch { PdfInteger integer => integer.Value, - PdfReal real => real.Value, + PdfReal real when double.IsFinite(real.Value) => real.Value, _ => null, }; diff --git a/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Pages.cs b/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Pages.cs index 9755c3b..0643577 100644 --- a/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Pages.cs +++ b/src/SharpCorePDF.Infrastructure/PdfPageContentReader.Pages.cs @@ -80,7 +80,9 @@ private static PdfPageSize ReadPageSize( var box = reader.ResolveArray(page.Dictionary.Get("MediaBox")) ?? reader.ResolveArray(page.Inherited.Get("MediaBox")); - if (ReadRectangle(reader, box) is { } rectangle && rectangle.Width > 0d && rectangle.Height > 0d) + if (ReadRectangle(reader, box) is { } rectangle + && double.IsFinite(rectangle.Width) && rectangle.Width > 0d + && double.IsFinite(rectangle.Height) && rectangle.Height > 0d) { return PdfPageSize.FromPoints(rectangle.Width, rectangle.Height); } diff --git a/src/SharpCorePDF.Infrastructure/PdfPageContentReader.cs b/src/SharpCorePDF.Infrastructure/PdfPageContentReader.cs index c6153ee..737b5f8 100644 --- a/src/SharpCorePDF.Infrastructure/PdfPageContentReader.cs +++ b/src/SharpCorePDF.Infrastructure/PdfPageContentReader.cs @@ -142,7 +142,7 @@ internal static PdfReadResult ResultOwned( private static double? ReadNumber(PdfFileReader reader, PdfObject? value) => value switch { PdfInteger integer => integer.Value, - PdfReal real => real.Value, + PdfReal real when double.IsFinite(real.Value) => real.Value, PdfReference reference => ReadNumber(reader, reader.Resolve(reference)), _ => null, }; diff --git a/tests/SharpCorePDF.Tests/PdfDocumentEditorTests.cs b/tests/SharpCorePDF.Tests/PdfDocumentEditorTests.cs index 3d26494..1005396 100644 --- a/tests/SharpCorePDF.Tests/PdfDocumentEditorTests.cs +++ b/tests/SharpCorePDF.Tests/PdfDocumentEditorTests.cs @@ -36,6 +36,34 @@ private static byte[] ThreePages() => PdfDocumentBuilder.Create() .AddPage(PdfPageSize.Letter, page => page.DrawText("three", 72, 100, Helvetica, 12)) .ToArray(); + /// + /// A media box whose digits overflow a double reads as infinite, which is no size a page can have: the page is + /// read as having no box and edited as such, instead of the edit throwing — a crash the document fuzz + /// target found. + /// + [Fact] + public async Task Edits_a_page_whose_media_box_overflows() + { + var builder = new PdfBuilder(); + var catalog = builder.Reserve(); + var pages = builder.Reserve(); + var huge = "1" + new string('0', 400); + + builder.Add($"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 {huge} {huge}] >>"); + builder.Set(catalog, "<< /Type /Catalog /Pages 2 0 R >>"); + builder.Set(pages, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + + var edited = await Editor.ApplyToArrayAsync( + new PdfByteArraySource(builder.Build(catalog), "overflow.pdf"), + PdfEditPlan.Create().SetMetadata(new PdfMetadata { Title = "edited" }), + cancellationToken: Token); + + var info = await InspectAsync(edited); + + Assert.Equal(1, info.PageCount); + Assert.Equal("edited", info.Metadata.Title); + } + private static async Task InspectAsync(byte[] bytes) => await Inspector.InspectAsync(new PdfByteArraySource(bytes, "edited.pdf"), cancellationToken: Token); diff --git a/tests/SharpCorePDF.Tests/PdfMergeFidelityTests.cs b/tests/SharpCorePDF.Tests/PdfMergeFidelityTests.cs index ec81fd6..b005030 100644 --- a/tests/SharpCorePDF.Tests/PdfMergeFidelityTests.cs +++ b/tests/SharpCorePDF.Tests/PdfMergeFidelityTests.cs @@ -79,6 +79,32 @@ public async Task Writes_a_media_box_for_a_page_that_has_none() warning => warning.Kind == PdfWarningKind.PageSizeAssumed && warning.Message.Contains("page 1 ", StringComparison.Ordinal)); } + /// + /// A media box whose digits overflow a double reads as infinite, which is no size a page can have; the page is + /// described as A4 with a remark, as one with no media box is, instead of the read throwing — a crash the + /// document fuzz target found. + /// + [Fact] + public async Task Assumes_a_size_for_a_page_whose_media_box_overflows() + { + var builder = new PdfBuilder(); + var catalog = builder.Reserve(); + var pages = builder.Reserve(); + var huge = "1" + new string('0', 400); + + builder.Add($"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 {huge} {huge}] >>"); + builder.Set(catalog, "<< /Type /Catalog /Pages 2 0 R >>"); + builder.Set(pages, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + + using var destination = new MemoryStream(); + var result = await Merger().MergeAsync( + MergePlan.Create().AddSource(PdfSource.Create(new PdfByteArraySource(builder.Build(catalog), "overflow.pdf"))), + destination, + Token); + + Assert.Contains(result.Warnings, warning => warning.Kind == PdfWarningKind.PageSizeAssumed); + } + /// A page that states its own media box keeps it, and no size is assumed for it. [Fact] public async Task Keeps_the_media_box_a_page_states() From 618d2a287088a492df841687c8335503b96f7ed7 Mon Sep 17 00:00:00 2001 From: Andrew Tierney Date: Mon, 16 Nov 2026 22:03:08 +1100 Subject: [PATCH 15/15] Fuzz the document target only (throwaway, not for merge) --- .github/workflows/robustness.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/robustness.yml b/.github/workflows/robustness.yml index b5e18ed..ad2328c 100644 --- a/.github/workflows/robustness.yml +++ b/.github/workflows/robustness.yml @@ -51,7 +51,7 @@ jobs: strategy: fail-fast: false matrix: - target: [ document, dct, ccitt, lzw, flate, runlength ] + target: [ document ] steps: - name: Check out