From 689d137057fb3f54b1a0f6bdb5e6341f0eb74c32 Mon Sep 17 00:00:00 2001 From: James Dumay Date: Thu, 12 Nov 2026 13:03:59 +1100 Subject: [PATCH 1/6] ci: adopt example-plugin workflow standard Test on Linux, macOS and Windows, x86-64 and ARM64 (ubuntu-24.04-arm and windows-11-arm native runners), and publish matching release archives. Release builds now run the test suite before packaging. --- .github/workflows/ci.yml | 48 +++++++++++++ .github/workflows/release.yml | 131 ++++++++++++++++++++++++++++++++++ 2 files changed, 179 insertions(+) create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..e9490a9 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,48 @@ +name: CI + +on: + pull_request: + push: + branches: + - main + +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + rust: + name: Rust (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: + - ubuntu-latest + - ubuntu-24.04-arm + - macos-15 + - windows-latest + - windows-11-arm + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + with: + components: clippy, rustfmt + + - name: Cache Rust + uses: Swatinem/rust-cache@v2 + + - name: Check formatting + run: cargo fmt --all -- --check + + - name: Clippy + run: cargo clippy --all-targets --locked -- -D warnings + + - name: Test + run: cargo test --locked + + - name: Build + run: cargo build --locked diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..dcd43ac --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,131 @@ +name: Release + +on: + push: + tags: + - "v*" + workflow_dispatch: + inputs: + version: + description: "Existing release tag, for example v0.1.0" + required: true + type: string + +permissions: + contents: write + +jobs: + build: + name: Build ${{ matrix.name }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - name: linux-x86_64 + os: ubuntu-latest + binary: mesh-pdf-plugin + target: x86_64-unknown-linux-gnu + archive_ext: tar.gz + package: tar + - name: linux-aarch64 + os: ubuntu-24.04-arm + binary: mesh-pdf-plugin + target: aarch64-unknown-linux-gnu + archive_ext: tar.gz + package: tar + - name: macos-aarch64 + os: macos-15 + binary: mesh-pdf-plugin + target: aarch64-apple-darwin + archive_ext: tar.gz + package: tar + - name: windows-x86_64 + os: windows-latest + binary: mesh-pdf-plugin.exe + target: x86_64-pc-windows-msvc + archive_ext: zip + package: zip + - name: windows-aarch64 + os: windows-11-arm + binary: mesh-pdf-plugin.exe + target: aarch64-pc-windows-msvc + archive_ext: zip + package: zip + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ github.event_name == 'workflow_dispatch' && format('refs/tags/{0}', inputs.version) || github.ref }} + + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + + - name: Cache Rust + uses: Swatinem/rust-cache@v2 + + - name: Build release binary + run: cargo build --release --locked + + - name: Test release binary + run: cargo test --release --locked + + - name: Package Unix archive + if: matrix.package == 'tar' + shell: bash + run: | + set -euo pipefail + version="${{ github.event.inputs.version || github.ref_name }}" + version="${version#v}" + stable_archive="pdf-${{ matrix.target }}.${{ matrix.archive_ext }}" + versioned_archive="pdf-${version}-${{ matrix.target }}.${{ matrix.archive_ext }}" + mkdir -p dist/pdf + cp "target/release/${{ matrix.binary }}" dist/pdf/pdf + cp plugin.toml dist/pdf/plugin.toml + cp README.md dist/pdf/README.md + tar -czf "$stable_archive" -C dist pdf + cp "$stable_archive" "$versioned_archive" + + - name: Package Windows archive + if: matrix.package == 'zip' + shell: pwsh + run: | + $version = "${{ github.event.inputs.version || github.ref_name }}".TrimStart("v") + $stableArchive = "pdf-${{ matrix.target }}.${{ matrix.archive_ext }}" + $versionedArchive = "pdf-$version-${{ matrix.target }}.${{ matrix.archive_ext }}" + New-Item -ItemType Directory -Force -Path dist/pdf | Out-Null + Copy-Item "target/release/${{ matrix.binary }}" "dist/pdf/pdf.exe" + Copy-Item plugin.toml "dist/pdf/plugin.toml" + Copy-Item README.md "dist/pdf/README.md" + Compress-Archive -Path "dist/pdf" -DestinationPath $stableArchive + Copy-Item $stableArchive $versionedArchive + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: pdf-${{ matrix.target }} + path: | + pdf-*.tar.gz + pdf-*.zip + if-no-files-found: error + + publish: + name: Publish GitHub release + needs: build + runs-on: ubuntu-latest + steps: + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + path: artifacts + merge-multiple: true + + - name: Publish release + uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ github.event.inputs.version || github.ref_name }} + name: ${{ github.event.inputs.version || github.ref_name }} + files: artifacts/* + fail_on_unmatched_files: true + overwrite_files: true From 0d1d970a21dcfb6ea1542a5a53a70a8071afe421 Mon Sep 17 00:00:00 2001 From: Michael Neale <14976+michaelneale@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:06:11 +1100 Subject: [PATCH 2/6] ci: release archives with native Tika for Linux, macOS and Windows Package and smoke-test each platform's archive (libtika_native beside the executable via @loader_path, $ORIGIN, or the exe directory), build Linux on Ubuntu 22.04, build Windows with MSVC, and port the store and home lookup to Windows. No Windows on ARM: GraalVM has no native-image for it. Pin mesh-llm-plugin to #3012's current head. --- .github/workflows/ci.yml | 31 ++++++++--- .github/workflows/release.yml | 96 ++++++++++++---------------------- Cargo.lock | 26 ++++----- Cargo.toml | 4 +- README.md | 33 ++++++++---- scripts/package.sh | 62 ++++++++++++++++------ scripts/smoke.sh | 31 +++++++++++ src/settings.rs | 16 ++++-- src/store.rs | 31 +++++++---- src/tika.rs | 9 +++- tests/fixtures/smoke.pdf | Bin 0 -> 611 bytes 11 files changed, 211 insertions(+), 128 deletions(-) create mode 100755 scripts/smoke.sh create mode 100644 tests/fixtures/smoke.pdf diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e9490a9..8045234 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,12 +17,16 @@ jobs: strategy: fail-fast: false matrix: + # No Windows on ARM: GraalVM has no native-image for it. os: - - ubuntu-latest - - ubuntu-24.04-arm + - ubuntu-22.04 + - ubuntu-22.04-arm - macos-15 + - macos-15-intel - windows-latest - - windows-11-arm + defaults: + run: + shell: bash steps: - name: Checkout uses: actions/checkout@v4 @@ -32,17 +36,30 @@ jobs: with: components: clippy, rustfmt + - name: Install Linux build tools + if: runner.os == 'Linux' + run: sudo apt-get update && sudo apt-get install -y patchelf zlib1g-dev + + - name: MSVC environment (native-image) + if: runner.os == 'Windows' + uses: ilammy/msvc-dev-cmd@v1 + - name: Cache Rust uses: Swatinem/rust-cache@v2 - name: Check formatting run: cargo fmt --all -- --check + # Everything in release: the build script compiles Tika with GraalVM + # native-image once per profile, and that is the slow part. - name: Clippy - run: cargo clippy --all-targets --locked -- -D warnings + run: cargo clippy --release --all-targets --locked -- -D warnings - name: Test - run: cargo test --locked + run: cargo test --release --locked + + - name: Package + run: echo "archive=$(PDF_SKIP_BUILD=1 scripts/package.sh | tail -1)" >> "$GITHUB_ENV" - - name: Build - run: cargo build --locked + - name: Smoke-test the archive + run: scripts/smoke.sh "$archive" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index dcd43ac..fac406f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,45 +14,31 @@ on: permissions: contents: write +env: + VERSION: ${{ github.event.inputs.version || github.ref_name }} + jobs: build: - name: Build ${{ matrix.name }} + name: Build ${{ matrix.target }} runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: include: - - name: linux-x86_64 - os: ubuntu-latest - binary: mesh-pdf-plugin + # Ubuntu 22.04 keeps the glibc requirement at 2.35. + - os: ubuntu-22.04 target: x86_64-unknown-linux-gnu - archive_ext: tar.gz - package: tar - - name: linux-aarch64 - os: ubuntu-24.04-arm - binary: mesh-pdf-plugin + - os: ubuntu-22.04-arm target: aarch64-unknown-linux-gnu - archive_ext: tar.gz - package: tar - - name: macos-aarch64 - os: macos-15 - binary: mesh-pdf-plugin + - os: macos-15 target: aarch64-apple-darwin - archive_ext: tar.gz - package: tar - - name: windows-x86_64 - os: windows-latest - binary: mesh-pdf-plugin.exe + - os: macos-15-intel + target: x86_64-apple-darwin + - os: windows-latest target: x86_64-pc-windows-msvc - archive_ext: zip - package: zip - - name: windows-aarch64 - os: windows-11-arm - binary: mesh-pdf-plugin.exe - target: aarch64-pc-windows-msvc - archive_ext: zip - package: zip - + defaults: + run: + shell: bash steps: - name: Checkout uses: actions/checkout@v4 @@ -62,52 +48,34 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@stable + - name: Install Linux build tools + if: runner.os == 'Linux' + run: sudo apt-get update && sudo apt-get install -y patchelf zlib1g-dev + + - name: MSVC environment (native-image) + if: runner.os == 'Windows' + uses: ilammy/msvc-dev-cmd@v1 + - name: Cache Rust uses: Swatinem/rust-cache@v2 - - name: Build release binary - run: cargo build --release --locked - - - name: Test release binary + - name: Test run: cargo test --release --locked - - name: Package Unix archive - if: matrix.package == 'tar' - shell: bash + - name: Package run: | - set -euo pipefail - version="${{ github.event.inputs.version || github.ref_name }}" - version="${version#v}" - stable_archive="pdf-${{ matrix.target }}.${{ matrix.archive_ext }}" - versioned_archive="pdf-${version}-${{ matrix.target }}.${{ matrix.archive_ext }}" - mkdir -p dist/pdf - cp "target/release/${{ matrix.binary }}" dist/pdf/pdf - cp plugin.toml dist/pdf/plugin.toml - cp README.md dist/pdf/README.md - tar -czf "$stable_archive" -C dist pdf - cp "$stable_archive" "$versioned_archive" + archive=$(scripts/package.sh "$VERSION" | tail -1) + test "$(basename "$archive")" = "pdf-${VERSION#v}-${{ matrix.target }}.$([[ $archive == *.zip ]] && echo zip || echo tar.gz)" + echo "archive=$archive" >> "$GITHUB_ENV" - - name: Package Windows archive - if: matrix.package == 'zip' - shell: pwsh - run: | - $version = "${{ github.event.inputs.version || github.ref_name }}".TrimStart("v") - $stableArchive = "pdf-${{ matrix.target }}.${{ matrix.archive_ext }}" - $versionedArchive = "pdf-$version-${{ matrix.target }}.${{ matrix.archive_ext }}" - New-Item -ItemType Directory -Force -Path dist/pdf | Out-Null - Copy-Item "target/release/${{ matrix.binary }}" "dist/pdf/pdf.exe" - Copy-Item plugin.toml "dist/pdf/plugin.toml" - Copy-Item README.md "dist/pdf/README.md" - Compress-Archive -Path "dist/pdf" -DestinationPath $stableArchive - Copy-Item $stableArchive $versionedArchive + - name: Smoke-test the archive + run: scripts/smoke.sh "$archive" - name: Upload artifact uses: actions/upload-artifact@v4 with: name: pdf-${{ matrix.target }} - path: | - pdf-*.tar.gz - pdf-*.zip + path: dist/pdf-* if-no-files-found: error publish: @@ -124,8 +92,8 @@ jobs: - name: Publish release uses: softprops/action-gh-release@v2 with: - tag_name: ${{ github.event.inputs.version || github.ref_name }} - name: ${{ github.event.inputs.version || github.ref_name }} + tag_name: ${{ env.VERSION }} + name: ${{ env.VERSION }} files: artifacts/* fail_on_unmatched_files: true overwrite_files: true diff --git a/Cargo.lock b/Cargo.lock index 79993c8..5d2b4cc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -482,7 +482,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -1241,7 +1241,7 @@ checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "mesh-llm-config" version = "0.79.0" -source = "git+https://github.com/Mesh-LLM/mesh-llm.git?rev=0f30127b05ea1ebe87bd8a5dac69b2d019a7d913#0f30127b05ea1ebe87bd8a5dac69b2d019a7d913" +source = "git+https://github.com/Mesh-LLM/mesh-llm.git?rev=355d29c8c1d3f35c056493b22bf2d1af44c028cb#355d29c8c1d3f35c056493b22bf2d1af44c028cb" dependencies = [ "anyhow", "dirs", @@ -1261,7 +1261,7 @@ dependencies = [ [[package]] name = "mesh-llm-plugin" version = "0.79.0" -source = "git+https://github.com/Mesh-LLM/mesh-llm.git?rev=0f30127b05ea1ebe87bd8a5dac69b2d019a7d913#0f30127b05ea1ebe87bd8a5dac69b2d019a7d913" +source = "git+https://github.com/Mesh-LLM/mesh-llm.git?rev=355d29c8c1d3f35c056493b22bf2d1af44c028cb#355d29c8c1d3f35c056493b22bf2d1af44c028cb" dependencies = [ "anyhow", "async-trait", @@ -1279,7 +1279,7 @@ dependencies = [ [[package]] name = "mesh-llm-types" version = "0.79.0" -source = "git+https://github.com/Mesh-LLM/mesh-llm.git?rev=0f30127b05ea1ebe87bd8a5dac69b2d019a7d913#0f30127b05ea1ebe87bd8a5dac69b2d019a7d913" +source = "git+https://github.com/Mesh-LLM/mesh-llm.git?rev=355d29c8c1d3f35c056493b22bf2d1af44c028cb#355d29c8c1d3f35c056493b22bf2d1af44c028cb" dependencies = [ "hex", "serde", @@ -1858,7 +1858,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2113,7 +2113,7 @@ checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" [[package]] name = "skippy-model-artifact" version = "0.79.0" -source = "git+https://github.com/Mesh-LLM/mesh-llm.git?rev=0f30127b05ea1ebe87bd8a5dac69b2d019a7d913#0f30127b05ea1ebe87bd8a5dac69b2d019a7d913" +source = "git+https://github.com/Mesh-LLM/mesh-llm.git?rev=355d29c8c1d3f35c056493b22bf2d1af44c028cb#355d29c8c1d3f35c056493b22bf2d1af44c028cb" dependencies = [ "anyhow", "async-trait", @@ -2125,7 +2125,7 @@ dependencies = [ [[package]] name = "skippy-model-ref" version = "0.79.0" -source = "git+https://github.com/Mesh-LLM/mesh-llm.git?rev=0f30127b05ea1ebe87bd8a5dac69b2d019a7d913#0f30127b05ea1ebe87bd8a5dac69b2d019a7d913" +source = "git+https://github.com/Mesh-LLM/mesh-llm.git?rev=355d29c8c1d3f35c056493b22bf2d1af44c028cb#355d29c8c1d3f35c056493b22bf2d1af44c028cb" dependencies = [ "serde", ] @@ -2133,7 +2133,7 @@ dependencies = [ [[package]] name = "skippy-native-runtime" version = "0.79.0" -source = "git+https://github.com/Mesh-LLM/mesh-llm.git?rev=0f30127b05ea1ebe87bd8a5dac69b2d019a7d913#0f30127b05ea1ebe87bd8a5dac69b2d019a7d913" +source = "git+https://github.com/Mesh-LLM/mesh-llm.git?rev=355d29c8c1d3f35c056493b22bf2d1af44c028cb#355d29c8c1d3f35c056493b22bf2d1af44c028cb" dependencies = [ "anyhow", "hex", @@ -2145,7 +2145,7 @@ dependencies = [ [[package]] name = "skippy-protocol" version = "0.79.0" -source = "git+https://github.com/Mesh-LLM/mesh-llm.git?rev=0f30127b05ea1ebe87bd8a5dac69b2d019a7d913#0f30127b05ea1ebe87bd8a5dac69b2d019a7d913" +source = "git+https://github.com/Mesh-LLM/mesh-llm.git?rev=355d29c8c1d3f35c056493b22bf2d1af44c028cb#355d29c8c1d3f35c056493b22bf2d1af44c028cb" dependencies = [ "prost", "prost-build", @@ -2157,7 +2157,7 @@ dependencies = [ [[package]] name = "skippy-tokenizer" version = "0.79.0" -source = "git+https://github.com/Mesh-LLM/mesh-llm.git?rev=0f30127b05ea1ebe87bd8a5dac69b2d019a7d913#0f30127b05ea1ebe87bd8a5dac69b2d019a7d913" +source = "git+https://github.com/Mesh-LLM/mesh-llm.git?rev=355d29c8c1d3f35c056493b22bf2d1af44c028cb#355d29c8c1d3f35c056493b22bf2d1af44c028cb" dependencies = [ "serde", ] @@ -2302,10 +2302,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2753,7 +2753,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 9aee6f8..59fc2a0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,8 +11,8 @@ base64 = "0.22" # Tika compiled to a native library with GraalVM (no JVM); its build downloads GraalVM. extractous = "0.3.0" hex = "0.4" -# Needs request_transform.v1: draft https://github.com/Mesh-LLM/mesh-llm/pull/3012 -mesh-llm-plugin = { git = "https://github.com/Mesh-LLM/mesh-llm.git", rev = "0f30127b05ea1ebe87bd8a5dac69b2d019a7d913" } +# Needs request_transform.v1 (#3012 head until it merges): https://github.com/Mesh-LLM/mesh-llm/pull/3012 +mesh-llm-plugin = { git = "https://github.com/Mesh-LLM/mesh-llm.git", rev = "355d29c8c1d3f35c056493b22bf2d1af44c028cb" } rand = "0.10" schemars = "1" serde = { version = "1", features = ["derive"] } diff --git a/README.md b/README.md index 991b0e7..675d23c 100644 --- a/README.md +++ b/README.md @@ -40,14 +40,24 @@ disk: arguments `path` or `data_base64`, and `output` = `auto` | `text` | `file` ## Install -You need Tesseract for OCR (`brew install tesseract`) and a mesh-llm built from -#3012. No Java. Building downloads GraalVM once (extractous's build script) and -compiles Tika with native-image, which takes about 2 minutes. The archive -carries `libtika_native.dylib` (131 MB, 49 MB compressed) next to the `pdf` -executable. +Releases carry an archive per platform: Linux x86_64 and arm64 (glibc 2.35+, +built on Ubuntu 22.04), macOS arm64 and x86_64, and Windows x64. There is no +Windows on ARM build, because GraalVM has no native-image for it. Each archive +holds the `pdf` executable and `libtika_native` (Tika, about 130 MB, 50 MB +compressed) next to it. No Java. You need a mesh-llm built from #3012, and +Tesseract for OCR (`brew install tesseract`, `apt install tesseract-ocr`, or +the Windows installer at its default `C:\Program Files\Tesseract-OCR`). ```sh -scripts/package.sh # prints dist/pdf-0.1.0-.tar.gz +mesh-llm plugins install Mesh-LLM/pdf-plugin +``` + +To build an archive yourself (the build downloads GraalVM once and compiles +Tika with native-image, a few minutes; Linux needs `patchelf`): + +```sh +scripts/package.sh # prints dist/pdf-0.1.0-.tar.gz (.zip on Windows) +scripts/smoke.sh dist/pdf-0.1.0-.tar.gz mesh-llm plugins install --archive dist/pdf-0.1.0-.tar.gz --name pdf ``` @@ -98,8 +108,9 @@ same extracted text: - **Native Tika is 2.9.2,** the version extractous pins. The Tika 2.x line is end of life, and extractous hasn't had a release since Dec 2024. Moving to Tika 3 means building our own native-image of it. -- **Packaging is macOS only:** `scripts/package.sh` re-points the library to - `@loader_path`. Linux would need the same with `$ORIGIN`. Untested. +- **Only tested end to end on macOS.** CI builds every release archive and + checks it extracts text with native Tika outside the build tree, but OCR and + the live mesh tests have only been run on a Mac. - **Claude Code often sends PDFs as page images.** When its Read tool is given `pages` (at most 20 per call), Claude Code sends JPEG page images instead of a document. The 3B model asked for `pages` unprompted, including @@ -117,5 +128,9 @@ same extracted text: ## Build ```sh -cargo test && cargo clippy --all-targets -- -D warnings +cargo test --release && cargo clippy --release --all-targets -- -D warnings ``` + +Release: push a `v*` tag. The release workflow builds, tests and smoke-tests +each platform's archive and publishes them to the GitHub release, named +`pdf--` plus the stable alias `pdf-`. diff --git a/scripts/package.sh b/scripts/package.sh index c4a04ab..c2f219c 100755 --- a/scripts/package.sh +++ b/scripts/package.sh @@ -1,31 +1,61 @@ #!/bin/bash -# Build an installable archive: dist/pdf--.tar.gz +# Build an installable archive for this machine's target: +# dist/pdf--.tar.gz (.zip on Windows), plus the stable alias +# dist/pdf-. that `mesh-llm plugins install` also accepts. # mesh-llm plugins install --archive dist/pdf-....tar.gz --name pdf +# Set PDF_SKIP_BUILD=1 to package an existing release build. set -euo pipefail -version=${1:-$(sed -n 's/^version = "\(.*\)"/\1/p' "$(dirname "$0")/../plugin.toml")} root=$(cd "$(dirname "$0")/.." && pwd) cd "$root" -cargo build --locked --release ${CARGO_OFFLINE:+--offline} +version=${1:-$(sed -n 's/^version = "\(.*\)"/\1/p' plugin.toml)} +version=${version#v} target_dir=${CARGO_TARGET_DIR:-target} +target=$(rustc -vV | sed -n 's/^host: //p') +if [[ -z ${PDF_SKIP_BUILD:-} ]]; then + cargo build --locked --release ${CARGO_OFFLINE:+--offline} +fi + +exe="" +case "$target" in + *-windows-*) exe=".exe"; lib_glob="libtika_native.dll"; ext=zip ;; + *-apple-darwin) lib_glob="libtika_native.dylib"; ext=tar.gz ;; + *) lib_glob="libtika_native.so"; ext=tar.gz ;; +esac + stage=$(mktemp -d) trap 'rm -rf "$stage"' EXIT pkg="$stage/pdf" mkdir -p "$pkg" dist cp README.md "$pkg/" # The host expects the executable to carry the plugin name. -cp "$target_dir/release/mesh-pdf-plugin" "$pkg/pdf" +cp "$target_dir/release/mesh-pdf-plugin$exe" "$pkg/pdf$exe" # Tika, compiled to a native library (no JVM), ships next to the executable. -lib=$(ls -t "$target_dir"/release/build/extractous-*/out/libs/libtika_native.* | head -1) +lib=$(ls -t "$target_dir"/release/build/extractous-*/out/libs/$lib_glob | head -1) cp "$lib" "$pkg/" -if [[ "$(uname)" == Darwin ]]; then - # The build links the library by its absolute build path: load it from beside the executable. - old=$(otool -D "$lib" | tail -1) - install_name_tool -id @rpath/libtika_native.dylib "$pkg/libtika_native.dylib" - install_name_tool -change "$old" @rpath/libtika_native.dylib -add_rpath @loader_path "$pkg/pdf" - codesign -f -s - "$pkg/libtika_native.dylib" "$pkg/pdf" +case "$target" in + *-apple-darwin) + # The build links the library by its absolute build path: load it from beside the executable. + old=$(otool -D "$lib" | tail -1) + install_name_tool -id @rpath/libtika_native.dylib "$pkg/libtika_native.dylib" + install_name_tool -change "$old" @rpath/libtika_native.dylib -add_rpath @loader_path "$pkg/pdf" + codesign -f -s - "$pkg/libtika_native.dylib" "$pkg/pdf" + ;; + *-linux-*) + # Find the library beside the executable, not in the build tree. + patchelf --set-rpath '$ORIGIN' "$pkg/pdf" + ;; + # Windows loads DLLs from the executable's directory first. +esac +sed "s/^version = .*/version = \"$version\"/" plugin.toml > "$pkg/plugin.toml" +"$pkg/pdf$exe" --packaged-manifest > "$pkg/plugin-manifest.json" + +versioned="dist/pdf-$version-$target.$ext" +stable="dist/pdf-$target.$ext" +rm -f "$versioned" "$stable" +if [[ $ext == zip ]]; then + (cd "$stage" && 7z a -tzip -bso0 "$root/$versioned" pdf) +else + COPYFILE_DISABLE=1 tar -czf "$versioned" -C "$stage" pdf fi -sed "s/^version = .*/version = \"${version#v}\"/" plugin.toml > "$pkg/plugin.toml" -"$pkg/pdf" --packaged-manifest > "$pkg/plugin-manifest.json" -name="pdf-$version-$(rustc -vV | sed -n 's/^host: //p').tar.gz" -COPYFILE_DISABLE=1 tar -czf "dist/$name" -C "$stage" pdf -echo "dist/$name" +cp "$versioned" "$stable" +echo "$versioned" diff --git a/scripts/smoke.sh b/scripts/smoke.sh new file mode 100755 index 0000000..df8aa9f --- /dev/null +++ b/scripts/smoke.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Check a packaged archive works on its own: unpack it outside the build tree, +# then run its executable for the manifest and a native Tika extraction. +# scripts/smoke.sh dist/pdf--. +set -euo pipefail +archive=$(cd "$(dirname "$1")" && pwd)/$(basename "$1") +root=$(cd "$(dirname "$0")/.." && pwd) +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT +case "$archive" in + *.zip) + if command -v 7z >/dev/null; then (cd "$work" && 7z x -bso0 "$archive") + else powershell -NoProfile -Command "Expand-Archive -LiteralPath '$(cygpath -w "$archive")' -DestinationPath '$(cygpath -w "$work")'" + fi ;; + *) tar -xzf "$archive" -C "$work" ;; +esac +exe="$work/pdf/pdf" +[[ -f $exe.exe ]] && exe="$exe.exe" +for file in plugin.toml plugin-manifest.json README.md; do + [[ -s $work/pdf/$file ]] || { echo "missing $file" >&2; exit 1; } +done +ls -l "$work/pdf" +# No cargo-provided library paths: the archive has to find Tika by itself. +env -u LD_LIBRARY_PATH -u DYLD_LIBRARY_PATH -u DYLD_FALLBACK_LIBRARY_PATH \ + "$exe" --packaged-manifest > "$work/manifest.json" +grep -q '"pdf"' "$work/manifest.json" +env -u LD_LIBRARY_PATH -u DYLD_LIBRARY_PATH -u DYLD_FALLBACK_LIBRARY_PATH \ + "$exe" --extract-worker --deadline-secs 120 < "$root/tests/fixtures/smoke.pdf" > "$work/out.xhtml" +grep -q "Hello from the pdf-plugin smoke test" "$work/out.xhtml" \ + || { echo "extraction failed:" >&2; head -c 2000 "$work/out.xhtml" >&2; exit 1; } +echo "smoke ok: $(basename "$archive")" diff --git a/src/settings.rs b/src/settings.rs index 59710c8..23d2935 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -77,10 +77,9 @@ impl Settings { } return Ok(dir.clone()); } - let home = std::env::var_os("HOME") - .filter(|home| !home.is_empty()) - .context("HOME is not set; configure store_dir")?; - Ok(PathBuf::from(home).join(".mesh-llm").join("pdf")) + let home = + home_dir().context("no home directory (HOME or USERPROFILE); configure store_dir")?; + Ok(home.join(".mesh-llm").join("pdf")) } pub fn ttl(&self) -> Duration { @@ -100,6 +99,15 @@ impl Settings { } } +/// `HOME`, or on Windows `USERPROFILE` when `HOME` is unset. Empty counts as unset. +fn home_dir() -> Option { + let var = |name| std::env::var_os(name).filter(|value| !value.is_empty()); + let home = var("HOME"); + #[cfg(windows)] + let home = home.or_else(|| var("USERPROFILE")); + home.map(PathBuf::from) +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/store.rs b/src/store.rs index 1ce1bff..895cc26 100644 --- a/src/store.rs +++ b/src/store.rs @@ -10,6 +10,7 @@ use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use std::fs::{DirBuilder, OpenOptions}; use std::io::Write; +#[cfg(unix)] use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt}; use std::path::{Path, PathBuf}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; @@ -61,17 +62,22 @@ fn now_secs() -> u64 { .as_secs() } +/// Owner-only on Unix. On Windows the default ACL under the user's profile +/// already limits it to that user. fn private_dir(path: &Path) -> std::io::Result<()> { - DirBuilder::new().recursive(true).mode(0o700).create(path) + let mut builder = DirBuilder::new(); + builder.recursive(true); + #[cfg(unix)] + builder.mode(0o700); + builder.create(path) } fn write_private(path: &Path, bytes: &[u8]) -> std::io::Result<()> { - let mut file = OpenOptions::new() - .write(true) - .create(true) - .truncate(true) - .mode(0o600) - .open(path)?; + let mut options = OpenOptions::new(); + options.write(true).create(true).truncate(true); + #[cfg(unix)] + options.mode(0o600); + let mut file = options.open(path)?; file.write_all(bytes)?; file.sync_all() } @@ -230,7 +236,6 @@ fn dir_size(path: &Path) -> u64 { mod tests { use super::*; use crate::test_support::TempDir; - use std::os::unix::fs::PermissionsExt; fn pages() -> Vec { vec!["one".into(), "two".into()] @@ -245,9 +250,13 @@ mod tests { let id = document_id(&pdf); assert!(store.lookup(&id).is_none()); let entry = store.save(&id, &pdf, "a.pdf", &pages()).unwrap(); - let mode = |path: &Path| std::fs::metadata(path).unwrap().permissions().mode() & 0o777; - assert_eq!(mode(&entry.dir), 0o700); - assert_eq!(mode(&entry.text_path()), 0o600); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mode = |path: &Path| std::fs::metadata(path).unwrap().permissions().mode() & 0o777; + assert_eq!(mode(&entry.dir), 0o700); + assert_eq!(mode(&entry.text_path()), 0o600); + } assert_eq!(std::fs::read(entry.pdf_path()).unwrap(), pdf); let text = std::fs::read_to_string(entry.text_path()).unwrap(); assert!(text.contains("--- Page 2 ---\n\ntwo")); diff --git a/src/tika.rs b/src/tika.rs index 2e834d4..75bd72d 100644 --- a/src/tika.rs +++ b/src/tika.rs @@ -143,13 +143,18 @@ impl Tika { } /// PATH for the worker: `tesseract_dir` first, then the plugin's PATH, then -/// the usual Homebrew/local bins (a GUI-launched node has a bare PATH). +/// the usual Homebrew/local bins, or Tesseract's default Windows install +/// folder (a GUI-launched node has a bare PATH). fn worker_path(tesseract_dir: Option<&std::path::Path>) -> Option { let mut dirs: Vec = tesseract_dir.map(PathBuf::from).into_iter().collect(); dirs.extend(std::env::split_paths( &std::env::var_os("PATH").unwrap_or_default(), )); - for extra in ["/opt/homebrew/bin", "/usr/local/bin", "/usr/bin", "/bin"] { + #[cfg(windows)] + let extras = [r"C:\Program Files\Tesseract-OCR"]; + #[cfg(not(windows))] + let extras = ["/opt/homebrew/bin", "/usr/local/bin", "/usr/bin", "/bin"]; + for extra in extras { let extra = PathBuf::from(extra); if extra.is_dir() && !dirs.contains(&extra) { dirs.push(extra); diff --git a/tests/fixtures/smoke.pdf b/tests/fixtures/smoke.pdf new file mode 100644 index 0000000000000000000000000000000000000000..ae12fae1508b7c27eb36ab85e7802e670950a7a6 GIT binary patch literal 611 zcmZWnL2kk@5WM>pb3tkkG;v4*NQeV1s8U_oh~45HMki;ZRVSI-)aVzk=lw;!c!$=h$KUoRRbSMikn9&3XQDN zTi7*WmuBrR?gZ<^cZe!+?(wew$9pt65dD|I#yWj zlWe|J7r`AW$^>U2{NIH*iaHT8oUuOYD#|k!^{y}}*^^rryddOH@DJ-;_S#D21yyz6 ev;3P7^LuFRp11iM#;VHJKEt3VTCMTxMEn4jUZUFo literal 0 HcmV?d00001 From b67b86d5804d9f39231dfbabe45f4a3549d970fc Mon Sep 17 00:00:00 2001 From: Michael Neale <14976+michaelneale@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:15:39 +1100 Subject: [PATCH 3/6] Rename to documents-plugin: plugin id, binary, archives and data dir are now "documents" Co-authored-by: Gandalf <474797421bc8061b82a684ffc1438abc87e1355d67aa3cc59917808655971c9f@meshllm.communities.buzz.xyz> --- .github/workflows/ci.yml | 2 +- .github/workflows/release.yml | 6 +++--- Cargo.lock | 2 +- Cargo.toml | 4 ++-- README.md | 26 +++++++++++++------------- plugin.toml | 6 +++--- scripts/package.sh | 30 +++++++++++++++--------------- scripts/smoke.sh | 10 +++++----- src/lib.rs | 18 ++++++++++-------- src/main.rs | 8 ++++---- src/settings.rs | 4 ++-- src/transform.rs | 2 +- tests/worker.rs | 2 +- 13 files changed, 61 insertions(+), 59 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8045234..e183f85 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,7 +59,7 @@ jobs: run: cargo test --release --locked - name: Package - run: echo "archive=$(PDF_SKIP_BUILD=1 scripts/package.sh | tail -1)" >> "$GITHUB_ENV" + run: echo "archive=$(DOCUMENTS_SKIP_BUILD=1 scripts/package.sh | tail -1)" >> "$GITHUB_ENV" - name: Smoke-test the archive run: scripts/smoke.sh "$archive" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fac406f..b780019 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -65,7 +65,7 @@ jobs: - name: Package run: | archive=$(scripts/package.sh "$VERSION" | tail -1) - test "$(basename "$archive")" = "pdf-${VERSION#v}-${{ matrix.target }}.$([[ $archive == *.zip ]] && echo zip || echo tar.gz)" + test "$(basename "$archive")" = "documents-${VERSION#v}-${{ matrix.target }}.$([[ $archive == *.zip ]] && echo zip || echo tar.gz)" echo "archive=$archive" >> "$GITHUB_ENV" - name: Smoke-test the archive @@ -74,8 +74,8 @@ jobs: - name: Upload artifact uses: actions/upload-artifact@v4 with: - name: pdf-${{ matrix.target }} - path: dist/pdf-* + name: documents-${{ matrix.target }} + path: dist/documents-* if-no-files-found: error publish: diff --git a/Cargo.lock b/Cargo.lock index 5d2b4cc..6a79c8a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1289,7 +1289,7 @@ dependencies = [ ] [[package]] -name = "mesh-pdf-plugin" +name = "mesh-documents-plugin" version = "0.1.0" dependencies = [ "anyhow", diff --git a/Cargo.toml b/Cargo.toml index 59fc2a0..7c1ef76 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,9 +1,9 @@ [package] -name = "mesh-pdf-plugin" +name = "mesh-documents-plugin" version = "0.1.0" edition = "2024" license = "Apache-2.0" -description = "PDF text extraction for Mesh LLM with Apache Tika (and OCR)" +description = "Document (PDF) text extraction for Mesh LLM with Apache Tika (and OCR)" [dependencies] anyhow = "1" diff --git a/README.md b/README.md index 675d23c..8291053 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ -# pdf-plugin +# documents-plugin > **⚠️ DRAFT: experimental, not for production use.** It needs the draft > mesh-llm hook [`request_transform.v1` (#3012)](https://github.com/Mesh-LLM/mesh-llm/pull/3012), > which is not on `main` yet. It has been tested on one Mac. See [Gaps](#gaps). -This [mesh-llm](https://github.com/Mesh-LLM/mesh-llm) plugin reads PDFs that +This [mesh-llm](https://github.com/Mesh-LLM/mesh-llm) plugin reads documents (PDFs today) that arrive **inside inference requests** and turns them into text the model can use. It uses [Apache Tika](https://tika.apache.org/), with OCR (Tesseract) for scanned pages. There's no JVM: Tika comes compiled to a native library with @@ -31,11 +31,11 @@ Accepted PDF forms (base64 only): including inside `tool_result` content (Claude Code's Read tool) File ids are rejected with a clear error, since the mesh has no files API. -Extractions are cached by the PDF's SHA-256 in `~/.mesh-llm/pdf` (owner-only) +Extractions are cached by the PDF's SHA-256 in `~/.mesh-llm/documents` (owner-only) and removed 24 hours after last use. -It also keeps the `extract` tool (MCP `pdf.extract` on the console's `/mcp`, -and HTTP `POST /api/plugins/pdf/tools/extract`) for agents that have a PDF on +It also keeps the `extract` tool (MCP `documents.extract` on the console's `/mcp`, +and HTTP `POST /api/plugins/documents/tools/extract`) for agents that have a PDF on disk: arguments `path` or `data_base64`, and `output` = `auto` | `text` | `file`. ## Install @@ -43,35 +43,35 @@ disk: arguments `path` or `data_base64`, and `output` = `auto` | `text` | `file` Releases carry an archive per platform: Linux x86_64 and arm64 (glibc 2.35+, built on Ubuntu 22.04), macOS arm64 and x86_64, and Windows x64. There is no Windows on ARM build, because GraalVM has no native-image for it. Each archive -holds the `pdf` executable and `libtika_native` (Tika, about 130 MB, 50 MB +holds the `documents` executable and `libtika_native` (Tika, about 130 MB, 50 MB compressed) next to it. No Java. You need a mesh-llm built from #3012, and Tesseract for OCR (`brew install tesseract`, `apt install tesseract-ocr`, or the Windows installer at its default `C:\Program Files\Tesseract-OCR`). ```sh -mesh-llm plugins install Mesh-LLM/pdf-plugin +mesh-llm plugins install Mesh-LLM/documents-plugin ``` To build an archive yourself (the build downloads GraalVM once and compiles Tika with native-image, a few minutes; Linux needs `patchelf`): ```sh -scripts/package.sh # prints dist/pdf-0.1.0-.tar.gz (.zip on Windows) -scripts/smoke.sh dist/pdf-0.1.0-.tar.gz -mesh-llm plugins install --archive dist/pdf-0.1.0-.tar.gz --name pdf +scripts/package.sh # prints dist/documents-0.1.0-.tar.gz (.zip on Windows) +scripts/smoke.sh dist/documents-0.1.0-.tar.gz +mesh-llm plugins install --archive dist/documents-0.1.0-.tar.gz --name documents ``` `~/.mesh-llm/config.toml`: ```toml [[plugin]] -name = "pdf" +name = "documents" ``` Other settings, with their defaults: `inline_max_chars = 40000`, `ocr = true`, `ocr_language = "eng"`, `tesseract_dir`, `timeout_secs = 300`, `max_pdf_mb = 64`, `store_dir`, `ttl_hours = 24`, `max_store_mb = 2048`, -`tika_concurrency = 2`. Each extraction runs in a child process (the `pdf` +`tika_concurrency = 2`. Each extraction runs in a child process (the `documents` executable in worker mode), which is killed at `timeout_secs` and also stops itself then. It never fetches URLs. @@ -133,4 +133,4 @@ cargo test --release && cargo clippy --release --all-targets -- -D warnings Release: push a `v*` tag. The release workflow builds, tests and smoke-tests each platform's archive and publishes them to the GitHub release, named -`pdf--` plus the stable alias `pdf-`. +`documents--` plus the stable alias `documents-`. diff --git a/plugin.toml b/plugin.toml index edf80bb..0e8e7c1 100644 --- a/plugin.toml +++ b/plugin.toml @@ -1,4 +1,4 @@ -name = "pdf" +name = "documents" version = "0.1.0" -description = "PDF attachments in inference requests to text, with Apache Tika and OCR (DRAFT; needs mesh-llm request_transform.v1)." -command = "pdf" +description = "Document attachments in inference requests to text, with Apache Tika and OCR (PDF today; needs mesh-llm request_transform.v1)." +command = "documents" diff --git a/scripts/package.sh b/scripts/package.sh index c2f219c..42f67b1 100755 --- a/scripts/package.sh +++ b/scripts/package.sh @@ -1,9 +1,9 @@ #!/bin/bash # Build an installable archive for this machine's target: -# dist/pdf--.tar.gz (.zip on Windows), plus the stable alias -# dist/pdf-. that `mesh-llm plugins install` also accepts. -# mesh-llm plugins install --archive dist/pdf-....tar.gz --name pdf -# Set PDF_SKIP_BUILD=1 to package an existing release build. +# dist/documents--.tar.gz (.zip on Windows), plus the stable alias +# dist/documents-. that `mesh-llm plugins install` also accepts. +# mesh-llm plugins install --archive dist/documents-....tar.gz --name documents +# Set DOCUMENTS_SKIP_BUILD=1 to package an existing release build. set -euo pipefail root=$(cd "$(dirname "$0")/.." && pwd) cd "$root" @@ -11,7 +11,7 @@ version=${1:-$(sed -n 's/^version = "\(.*\)"/\1/p' plugin.toml)} version=${version#v} target_dir=${CARGO_TARGET_DIR:-target} target=$(rustc -vV | sed -n 's/^host: //p') -if [[ -z ${PDF_SKIP_BUILD:-} ]]; then +if [[ -z ${DOCUMENTS_SKIP_BUILD:-} ]]; then cargo build --locked --release ${CARGO_OFFLINE:+--offline} fi @@ -24,11 +24,11 @@ esac stage=$(mktemp -d) trap 'rm -rf "$stage"' EXIT -pkg="$stage/pdf" +pkg="$stage/documents" mkdir -p "$pkg" dist cp README.md "$pkg/" # The host expects the executable to carry the plugin name. -cp "$target_dir/release/mesh-pdf-plugin$exe" "$pkg/pdf$exe" +cp "$target_dir/release/mesh-documents-plugin$exe" "$pkg/documents$exe" # Tika, compiled to a native library (no JVM), ships next to the executable. lib=$(ls -t "$target_dir"/release/build/extractous-*/out/libs/$lib_glob | head -1) cp "$lib" "$pkg/" @@ -37,25 +37,25 @@ case "$target" in # The build links the library by its absolute build path: load it from beside the executable. old=$(otool -D "$lib" | tail -1) install_name_tool -id @rpath/libtika_native.dylib "$pkg/libtika_native.dylib" - install_name_tool -change "$old" @rpath/libtika_native.dylib -add_rpath @loader_path "$pkg/pdf" - codesign -f -s - "$pkg/libtika_native.dylib" "$pkg/pdf" + install_name_tool -change "$old" @rpath/libtika_native.dylib -add_rpath @loader_path "$pkg/documents" + codesign -f -s - "$pkg/libtika_native.dylib" "$pkg/documents" ;; *-linux-*) # Find the library beside the executable, not in the build tree. - patchelf --set-rpath '$ORIGIN' "$pkg/pdf" + patchelf --set-rpath '$ORIGIN' "$pkg/documents" ;; # Windows loads DLLs from the executable's directory first. esac sed "s/^version = .*/version = \"$version\"/" plugin.toml > "$pkg/plugin.toml" -"$pkg/pdf$exe" --packaged-manifest > "$pkg/plugin-manifest.json" +"$pkg/documents$exe" --packaged-manifest > "$pkg/plugin-manifest.json" -versioned="dist/pdf-$version-$target.$ext" -stable="dist/pdf-$target.$ext" +versioned="dist/documents-$version-$target.$ext" +stable="dist/documents-$target.$ext" rm -f "$versioned" "$stable" if [[ $ext == zip ]]; then - (cd "$stage" && 7z a -tzip -bso0 "$root/$versioned" pdf) + (cd "$stage" && 7z a -tzip -bso0 "$root/$versioned" documents) else - COPYFILE_DISABLE=1 tar -czf "$versioned" -C "$stage" pdf + COPYFILE_DISABLE=1 tar -czf "$versioned" -C "$stage" documents fi cp "$versioned" "$stable" echo "$versioned" diff --git a/scripts/smoke.sh b/scripts/smoke.sh index df8aa9f..bd52213 100755 --- a/scripts/smoke.sh +++ b/scripts/smoke.sh @@ -1,7 +1,7 @@ #!/bin/bash # Check a packaged archive works on its own: unpack it outside the build tree, # then run its executable for the manifest and a native Tika extraction. -# scripts/smoke.sh dist/pdf--. +# scripts/smoke.sh dist/documents--. set -euo pipefail archive=$(cd "$(dirname "$1")" && pwd)/$(basename "$1") root=$(cd "$(dirname "$0")/.." && pwd) @@ -14,16 +14,16 @@ case "$archive" in fi ;; *) tar -xzf "$archive" -C "$work" ;; esac -exe="$work/pdf/pdf" +exe="$work/documents/documents" [[ -f $exe.exe ]] && exe="$exe.exe" for file in plugin.toml plugin-manifest.json README.md; do - [[ -s $work/pdf/$file ]] || { echo "missing $file" >&2; exit 1; } + [[ -s $work/documents/$file ]] || { echo "missing $file" >&2; exit 1; } done -ls -l "$work/pdf" +ls -l "$work/documents" # No cargo-provided library paths: the archive has to find Tika by itself. env -u LD_LIBRARY_PATH -u DYLD_LIBRARY_PATH -u DYLD_FALLBACK_LIBRARY_PATH \ "$exe" --packaged-manifest > "$work/manifest.json" -grep -q '"pdf"' "$work/manifest.json" +grep -q '"documents"' "$work/manifest.json" env -u LD_LIBRARY_PATH -u DYLD_LIBRARY_PATH -u DYLD_FALLBACK_LIBRARY_PATH \ "$exe" --extract-worker --deadline-secs 120 < "$root/tests/fixtures/smoke.pdf" > "$work/out.xhtml" grep -q "Hello from the pdf-plugin smoke test" "$work/out.xhtml" \ diff --git a/src/lib.rs b/src/lib.rs index 4ae4436..8908ac6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,10 +1,10 @@ -//! `pdf`: PDF attachments for Mesh LLM, decoded with Apache Tika (and OCR). +//! `documents`: document (PDF) attachments for Mesh LLM, decoded with Apache Tika (and OCR). //! //! - `transform`: a request transform (`request_transform.v1`) that replaces //! PDF attachments in inference requests with their text, on the node the //! client called, before routing. Clients keep their usual base URL. -//! - `extract`: the same extraction as a tool, over MCP (`pdf.extract` on the -//! console's `/mcp`) and HTTP (`/api/plugins/pdf/tools/extract`). +//! - `extract`: the same extraction as a tool, over MCP (`documents.extract` on the +//! console's `/mcp`) and HTTP (`/api/plugins/documents/tools/extract`). mod attachments; mod error; @@ -101,14 +101,16 @@ fn current(shared: &Shared) -> Result, mesh_llm_plugin::PluginError> { .read() .unwrap_or_else(|e| e.into_inner()) .clone() - .ok_or_else(|| mesh_llm_plugin::PluginError::internal("pdf plugin is not initialized")) + .ok_or_else(|| { + mesh_llm_plugin::PluginError::internal("documents plugin is not initialized") + }) } #[derive(serde::Deserialize, schemars::JsonSchema)] struct NoArgs {} pub fn plugin() -> SimplePlugin { - let name = std::env::var("MESH_LLM_PLUGIN_NAME").unwrap_or_else(|_| "pdf".into()); + let name = std::env::var("MESH_LLM_PLUGIN_NAME").unwrap_or_else(|_| "documents".into()); let version = env!("CARGO_PKG_VERSION"); let shared: Shared = Arc::default(); SimplePlugin::new(PluginMetadata::new( @@ -117,8 +119,8 @@ pub fn plugin() -> SimplePlugin { plugin_server_info( name.clone(), version, - "PDF", - "PDF text extraction with Apache Tika, including OCR", + "Documents", + "Document (PDF) text extraction with Apache Tika, including OCR", None::, ), )) @@ -168,7 +170,7 @@ fn manifest(name: &str) -> mesh_llm_plugin::proto::PluginManifest { .setting(setting( "store_dir", config_path(), - "Where extracted documents are kept (default ~/.mesh-llm/pdf)." + "Where extracted documents are kept (default ~/.mesh-llm/documents)." )) .setting( setting("ttl_hours", config_integer(), "Remove extractions unused for this long.") diff --git a/src/main.rs b/src/main.rs index 9b04006..0c0e89b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,8 +2,8 @@ use mesh_llm_plugin::Plugin; fn main() -> anyhow::Result { let args: Vec = std::env::args().collect(); - if args.get(1).map(String::as_str) == Some(mesh_pdf_plugin::tika::WORKER_FLAG) { - return Ok(mesh_pdf_plugin::tika::run_worker(&args[2..])); + if args.get(1).map(String::as_str) == Some(mesh_documents_plugin::tika::WORKER_FLAG) { + return Ok(mesh_documents_plugin::tika::run_worker(&args[2..])); } tokio::runtime::Runtime::new()?.block_on(serve(&args))?; Ok(std::process::ExitCode::SUCCESS) @@ -13,12 +13,12 @@ async fn serve(args: &[String]) -> anyhow::Result<()> { if args.iter().any(|arg| arg == "--packaged-manifest") { // Written into the release archive as plugin-manifest.json, so the host // can validate `[plugin.settings]` before the plugin starts. - let manifest = mesh_pdf_plugin::plugin() + let manifest = mesh_documents_plugin::plugin() .manifest() .ok_or_else(|| anyhow::anyhow!("plugin has no manifest"))?; let json = mesh_llm_plugin::package_manifest_json(&manifest)?; std::io::Write::write_all(&mut std::io::stdout(), json.as_bytes())?; return Ok(()); } - mesh_pdf_plugin::run().await + mesh_documents_plugin::run().await } diff --git a/src/settings.rs b/src/settings.rs index 23d2935..a736437 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -47,7 +47,7 @@ impl Settings { let settings: Self = if settings_json.trim().is_empty() { Self::default() } else { - serde_json::from_str(settings_json).context("invalid pdf plugin settings")? + serde_json::from_str(settings_json).context("invalid documents plugin settings")? }; if settings.tika_concurrency == 0 { bail!("tika_concurrency must be at least 1"); @@ -79,7 +79,7 @@ impl Settings { } let home = home_dir().context("no home directory (HOME or USERPROFILE); configure store_dir")?; - Ok(home.join(".mesh-llm").join("pdf")) + Ok(home.join(".mesh-llm").join("documents")) } pub fn ttl(&self) -> Duration { diff --git a/src/transform.rs b/src/transform.rs index fe3bf88..18231e6 100644 --- a/src/transform.rs +++ b/src/transform.rs @@ -101,5 +101,5 @@ fn describe( /// The transform's error as the client will read it. pub fn plugin_error(error: DocError) -> mesh_llm_plugin::PluginError { - mesh_llm_plugin::PluginError::invalid_params(format!("pdf: {}", error.message)) + mesh_llm_plugin::PluginError::invalid_params(format!("documents: {}", error.message)) } diff --git a/tests/worker.rs b/tests/worker.rs index 50971bd..9ddfe3f 100644 --- a/tests/worker.rs +++ b/tests/worker.rs @@ -4,7 +4,7 @@ use std::io::Write; use std::process::{Command, Output, Stdio}; fn run_worker(input: &[u8]) -> Output { - let mut child = Command::new(env!("CARGO_BIN_EXE_mesh-pdf-plugin")) + let mut child = Command::new(env!("CARGO_BIN_EXE_mesh-documents-plugin")) .args(["--extract-worker", "--deadline-secs", "60"]) .stdin(Stdio::piped()) .stdout(Stdio::piped()) From 420a4a91fbd593f37e3bf2f37e03fe100fe3ea29 Mon Sep 17 00:00:00 2001 From: Michael Neale <14976+michaelneale@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:44:05 +1100 Subject: [PATCH 4/6] ci: ship Tika's JDK libraries (libawt) with the archive; use MSVC link.exe on Windows --- .github/workflows/ci.yml | 5 +++++ .github/workflows/release.yml | 5 +++++ scripts/package.sh | 13 +++++++------ 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e183f85..404e2f3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,6 +44,11 @@ jobs: if: runner.os == 'Windows' uses: ilammy/msvc-dev-cmd@v1 + # Git Bash's coreutils `link` shadows MSVC's link.exe on PATH. + - name: Remove Git's link.exe + if: runner.os == 'Windows' + run: rm -f /usr/bin/link.exe + - name: Cache Rust uses: Swatinem/rust-cache@v2 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b780019..c5b4079 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -56,6 +56,11 @@ jobs: if: runner.os == 'Windows' uses: ilammy/msvc-dev-cmd@v1 + # Git Bash's coreutils `link` shadows MSVC's link.exe on PATH. + - name: Remove Git's link.exe + if: runner.os == 'Windows' + run: rm -f /usr/bin/link.exe + - name: Cache Rust uses: Swatinem/rust-cache@v2 diff --git a/scripts/package.sh b/scripts/package.sh index 42f67b1..c9b92cc 100755 --- a/scripts/package.sh +++ b/scripts/package.sh @@ -17,9 +17,9 @@ fi exe="" case "$target" in - *-windows-*) exe=".exe"; lib_glob="libtika_native.dll"; ext=zip ;; - *-apple-darwin) lib_glob="libtika_native.dylib"; ext=tar.gz ;; - *) lib_glob="libtika_native.so"; ext=tar.gz ;; + *-windows-*) exe=".exe"; lib_ext=dll; ext=zip ;; + *-apple-darwin) lib_ext=dylib; ext=tar.gz ;; + *) lib_ext=so; ext=tar.gz ;; esac stage=$(mktemp -d) @@ -29,9 +29,10 @@ mkdir -p "$pkg" dist cp README.md "$pkg/" # The host expects the executable to carry the plugin name. cp "$target_dir/release/mesh-documents-plugin$exe" "$pkg/documents$exe" -# Tika, compiled to a native library (no JVM), ships next to the executable. -lib=$(ls -t "$target_dir"/release/build/extractous-*/out/libs/$lib_glob | head -1) -cp "$lib" "$pkg/" +# Tika, compiled to a native library (no JVM), ships next to the executable, +# with the JDK libraries it loads at run time from its own folder (libawt on Linux). +lib=$(ls -t "$target_dir"/release/build/extractous-*/out/libs/libtika_native.$lib_ext | head -1) +cp "$(dirname "$lib")"/*.$lib_ext "$pkg/" case "$target" in *-apple-darwin) # The build links the library by its absolute build path: load it from beside the executable. From b6d26e6fafa9c51b41bdf13755a0a1b891d0514a Mon Sep 17 00:00:00 2001 From: Michael Neale <14976+michaelneale@users.noreply.github.com> Date: Thu, 12 Nov 2026 17:00:33 +1100 Subject: [PATCH 5/6] test: make path and worker tests portable to Windows Use temp_dir paths instead of Unix-only absolute paths, and run the /bin/sh-based Tika worker tests on Unix only. --- Cargo.lock | 36 ++++++++++++++++++------------------ src/extract.rs | 9 +++++++-- src/tika.rs | 2 ++ 3 files changed, 27 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6a79c8a..f345a1a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1238,6 +1238,24 @@ version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +[[package]] +name = "mesh-documents-plugin" +version = "0.1.0" +dependencies = [ + "anyhow", + "base64 0.22.1", + "extractous", + "hex", + "mesh-llm-plugin", + "rand", + "schemars", + "serde", + "serde_json", + "sha2", + "tokio", + "url", +] + [[package]] name = "mesh-llm-config" version = "0.79.0" @@ -1288,24 +1306,6 @@ dependencies = [ "skippy-model-artifact", ] -[[package]] -name = "mesh-documents-plugin" -version = "0.1.0" -dependencies = [ - "anyhow", - "base64 0.22.1", - "extractous", - "hex", - "mesh-llm-plugin", - "rand", - "schemars", - "serde", - "serde_json", - "sha2", - "tokio", - "url", -] - [[package]] name = "mime" version = "0.3.18" diff --git a/src/extract.rs b/src/extract.rs index 12cd723..22e90f7 100644 --- a/src/extract.rs +++ b/src/extract.rs @@ -249,9 +249,14 @@ mod tests { #[test] fn paths_must_be_absolute_files() { assert_eq!(read_path("rel.pdf", 1024).unwrap_err().code, "invalid_path"); - assert_eq!(read_path("/", 1024).unwrap_err().code, "invalid_path"); + let dir = std::env::temp_dir(); assert_eq!( - read_path("/nonexistent/x.pdf", 1024).unwrap_err().code, + read_path(dir.to_str().unwrap(), 1024).unwrap_err().code, + "invalid_path" + ); + let missing = dir.join("nonexistent-documents-plugin").join("x.pdf"); + assert_eq!( + read_path(missing.to_str().unwrap(), 1024).unwrap_err().code, "file_not_found" ); } diff --git a/src/tika.rs b/src/tika.rs index 75bd72d..e10207b 100644 --- a/src/tika.rs +++ b/src/tika.rs @@ -229,6 +229,7 @@ mod tests { use super::*; /// A worker that never answers: the deadline must end the call, and kill it. + #[cfg(unix)] #[tokio::test] async fn a_hung_worker_is_killed_at_the_deadline() { let tika = Tika::with_program("/bin/sh", &["-c", "sleep 30", "sh"]); @@ -244,6 +245,7 @@ mod tests { assert!(started.elapsed() < Duration::from_secs(2)); } + #[cfg(unix)] #[tokio::test] async fn a_failing_worker_is_unavailable() { let tika = Tika::with_program("/bin/sh", &["-c", "echo boom >&2; exit 1", "sh"]); From 58b14b25319f8a0ad3a962818870ca7a7c7f69db Mon Sep 17 00:00:00 2001 From: Michael Neale <14976+michaelneale@users.noreply.github.com> Date: Thu, 12 Nov 2026 17:52:13 +1100 Subject: [PATCH 6/6] windows: warm PDFBox's font cache at startup; allow a cold first run in tests On Windows PDFBox scans every system font on its first text extraction (~70 s for 346 fonts on a GitHub runner), then reads ~/.pdfbox.cache (~1 s). The worker test's 60 s deadline killed that first run. --- scripts/smoke.sh | 2 +- src/tika.rs | 27 ++++++++++++++++++++++++++- tests/worker.rs | 3 ++- 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/scripts/smoke.sh b/scripts/smoke.sh index bd52213..7d72864 100755 --- a/scripts/smoke.sh +++ b/scripts/smoke.sh @@ -25,7 +25,7 @@ env -u LD_LIBRARY_PATH -u DYLD_LIBRARY_PATH -u DYLD_FALLBACK_LIBRARY_PATH \ "$exe" --packaged-manifest > "$work/manifest.json" grep -q '"documents"' "$work/manifest.json" env -u LD_LIBRARY_PATH -u DYLD_LIBRARY_PATH -u DYLD_FALLBACK_LIBRARY_PATH \ - "$exe" --extract-worker --deadline-secs 120 < "$root/tests/fixtures/smoke.pdf" > "$work/out.xhtml" + "$exe" --extract-worker --deadline-secs 300 < "$root/tests/fixtures/smoke.pdf" > "$work/out.xhtml" grep -q "Hello from the pdf-plugin smoke test" "$work/out.xhtml" \ || { echo "extraction failed:" >&2; head -c 2000 "$work/out.xhtml" >&2; exit 1; } echo "smoke ok: $(basename "$archive")" diff --git a/src/tika.rs b/src/tika.rs index e10207b..f07ab35 100644 --- a/src/tika.rs +++ b/src/tika.rs @@ -43,8 +43,11 @@ impl Tika { args.push("--ocr".into()); args.push(settings.ocr_language.clone().into()); } + let program = std::env::current_exe()?; + #[cfg(windows)] + warm_font_cache(program.clone()); Ok(Self { - program: std::env::current_exe()?, + program, args, path: worker_path(settings.tesseract_dir.as_deref()), permits: Semaphore::new(settings.tika_concurrency), @@ -142,6 +145,28 @@ impl Tika { } } +/// On Windows, PDFBox's first text extraction scans every system font (about +/// 70 s on a GitHub runner) and caches the result in `~/.pdfbox.cache`; later +/// ones take a second. Pay that once at startup, in the background, rather +/// than on the first request. +#[cfg(windows)] +fn warm_font_cache(program: PathBuf) { + std::thread::spawn(move || { + let child = std::process::Command::new(program) + .args([WORKER_FLAG, "--deadline-secs", "600"]) + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn(); + if let Ok(mut child) = child { + if let Some(mut stdin) = child.stdin.take() { + let _ = stdin.write_all(include_bytes!("../tests/fixtures/smoke.pdf")); + } + let _ = child.wait(); + } + }); +} + /// PATH for the worker: `tesseract_dir` first, then the plugin's PATH, then /// the usual Homebrew/local bins, or Tesseract's default Windows install /// folder (a GUI-launched node has a bare PATH). diff --git a/tests/worker.rs b/tests/worker.rs index 9ddfe3f..11ccc29 100644 --- a/tests/worker.rs +++ b/tests/worker.rs @@ -5,7 +5,8 @@ use std::process::{Command, Output, Stdio}; fn run_worker(input: &[u8]) -> Output { let mut child = Command::new(env!("CARGO_BIN_EXE_mesh-documents-plugin")) - .args(["--extract-worker", "--deadline-secs", "60"]) + // A cold Windows run spends about 70 s scanning system fonts first. + .args(["--extract-worker", "--deadline-secs", "300"]) .stdin(Stdio::piped()) .stdout(Stdio::piped()) .stderr(Stdio::piped())