diff --git a/.github/workflows/context-eval.yml b/.github/workflows/context-eval.yml index cbfea8e..9aca568 100644 --- a/.github/workflows/context-eval.yml +++ b/.github/workflows/context-eval.yml @@ -1,6 +1,7 @@ name: context-eval -# Quality gate for `helpbase context`. Not per-PR (each run costs real +# Quality gate for `helpbase ingest` (the canonical name — the `context` +# alias is deprecated but still aliases here). Not per-PR (each run costs real # LLM tokens against the AI Gateway). Two triggers: # # - workflow_dispatch: run on demand before a release or after a @@ -19,7 +20,7 @@ on: jobs: eval: - name: helpbase context — 5-Q self-dogfood eval + name: helpbase ingest — 5-Q self-dogfood eval runs-on: ubuntu-latest timeout-minutes: 45 steps: @@ -42,7 +43,7 @@ jobs: - name: Build CLI run: pnpm --filter helpbase build - - name: Run context eval (Sonnet 4.6) + - name: Run ingest eval (Sonnet 4.6) env: AI_GATEWAY_API_KEY: ${{ secrets.AI_GATEWAY_API_KEY }} HELPBASE_EVAL_MODEL: anthropic/claude-sonnet-4.6 diff --git a/CHANGELOG.md b/CHANGELOG.md index aabab6a..5e29f30 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed +- **`helpbase context` renamed to `helpbase ingest`.** `context` described + a domain concept, not an action — `ingest` names what the command does + (walk your code + markdown, synthesize cited how-tos, wire MCP). The + old name continues to work as a deprecation shim: same flags, same + behavior, one-line stderr warning, suppressed under `--json`/`--quiet`. + Help text + README Quick Start + scaffolder next-steps output now lead + with `ingest`. CI scripts pinned to `helpbase context` keep working + until v0.7, which is when the shim is removed. Flag surface is shared + via `applyIngestOptions()` so the two commands can't drift. First PR + of the CLI DX v2 plan. + ### Added - **Unit tests for the device-authorize state machine.** Extracted the 8-phase `Phase` union from `AuthorizeDeviceClient` into a pure diff --git a/README.md b/README.md index be47fdd..e26be79 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ monorepo. - **Docs site** — a full Next.js + shadcn/ui help center in your repo - **MCP server** — `@helpbase/mcp` runs next to your app, not in a vendor's cloud - **`llms.txt` + `llms-full.txt`** — generated at your build time, served from your origin -- **`helpbase context`** — point at any repo, get cited how-to guides + MCP + llms.txt in one command +- **`helpbase ingest`** — point at any repo, get cited how-to guides + MCP + llms.txt in one command - **Doc sync** — `helpbase sync` proposes codebase-grounded doc diffs with mandatory citations, opens them as PRs in your repo via your CI - **Workflows** — `shadcn add helpbase-workflow` drops a plain GitHub Action that runs `helpbase sync` in your Actions minutes, with your secrets @@ -60,16 +60,16 @@ log into. ## Quick start -Point `helpbase context` at any repo. Under 90 seconds from zero to +Point `helpbase ingest` at any repo. Under 90 seconds from zero to generated docs. Works on macOS, Linux, and Windows. Requires Node 20+. ```bash npx helpbase login # magic-link device flow, free 500k tokens/day, no card -npx helpbase context . # synthesize cited how-tos from your repo -npx helpbase context . --ask "how do I log in?" # answer in terminal, no MCP client required +npx helpbase ingest . # synthesize cited how-tos from your repo +npx helpbase ingest . --ask "how do I log in?" # answer in terminal, no MCP client required ``` -`helpbase context` walks your code + markdown, synthesizes task-oriented +`helpbase ingest` walks your code + markdown, synthesizes task-oriented how-to guides grounded in cited files, writes MDX to `.helpbase/docs/`, and emits `llms.txt` + an MCP config hint. Same content, two surfaces: humans read the MDX, agents query over MCP. @@ -265,6 +265,7 @@ Edit, delete, or regenerate any of them. You own them. ```bash helpbase dev # start the Next.js dev server +helpbase ingest [repo] # turn a repo into cited how-tos + llms.txt + MCP (.helpbase/) helpbase generate --url # generate articles from a URL helpbase audit # validate frontmatter, categories, schema helpbase new # create a new article from a template (interactive or --type/--title) diff --git a/docs/cli.md b/docs/cli.md index 3d18317..ad775ed 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -3,268 +3,479 @@ # helpbase CLI reference -> Generated from commander metadata. CLI version: `0.0.1`. +> Generated from commander metadata. CLI version: `0.5.0`. ## Commands -- [`helpbase Get`](#helpbase-Get) -- [`helpbase Ship`](#helpbase-Ship) -- [`helpbase Author`](#helpbase-Author) -- [`helpbase Account`](#helpbase-Account) -- [`helpbase Diagnose`](#helpbase-Diagnose) +- [`helpbase ingest`](#helpbase-ingest) +- [`helpbase preview`](#helpbase-preview) +- [`helpbase new`](#helpbase-new) +- [`helpbase dev`](#helpbase-dev) +- [`helpbase deploy`](#helpbase-deploy) +- [`helpbase link`](#helpbase-link) +- [`helpbase open`](#helpbase-open) +- [`helpbase generate`](#helpbase-generate) +- [`helpbase sync`](#helpbase-sync) +- [`helpbase mcp`](#helpbase-mcp) +- [`helpbase audit`](#helpbase-audit) +- [`helpbase login`](#helpbase-login) +- [`helpbase logout`](#helpbase-logout) +- [`helpbase whoami`](#helpbase-whoami) +- [`helpbase config`](#helpbase-config) +- [`helpbase doctor`](#helpbase-doctor) +- [`helpbase feedback`](#helpbase-feedback) +- [`helpbase completion`](#helpbase-completion) +- [`helpbase upgrade`](#helpbase-upgrade) +- [`helpbase context`](#helpbase-context) + +## `helpbase ingest` + +```text +Usage: helpbase ingest [options] [repoPath] + +Turn a repo into agent-ready docs: walks your code + markdown, synthesizes cited +how-to guides, wires up MCP. Your docs, always up to date. + +Arguments: + repoPath Path to the repo to ingest (default: ".") + +Options: + -o, --output Output directory for generated content (default: + ".helpbase") + --max-tokens Token budget for the LLM input (per run) (default: + "100000") + --chars-per-token Chars-per-token ratio used for the budget estimate (3.5 + = mid-range; 2.8 for code-heavy, 4.2 for prose) + (default: "3.5") + --model Override the model ID (e.g. + anthropic/claude-sonnet-4.6) + --test Use the cheap test model + (google/gemini-3.1-flash-lite-preview) and print model + info + --debug Write the raw assembled prompt to /_prompt.txt + before calling the LLM + --dry-run Walk the repo and print what would be sent to the LLM, + without spending tokens + --allow-dirty Explicitly allow generating from a dirty working tree + (default behavior; kept for clarity) + --require-clean Exit 1 if the working tree has uncommitted changes (CI + mode) + --overwrite Overwrite existing docs, including source:custom + (default preserves custom files) + --yes Skip interactive confirmations (for CI/scripted use) + --only Only (re)generate docs in one category slug + --prompt Override the default prompt with a file path (content + is still wrapped in untrusted-content delimiters) + --ask After generating, answer a question against the fresh + docs in-terminal (no MCP client required) + --reuse-existing With --ask: skip generation, answer against the + existing .helpbase/docs/ on disk (for eval/batch runs) + -h, --help display help for command + +Examples: + $ helpbase ingest . # ingest current repo + $ helpbase ingest ./path/to/repo # ingest a specific path + $ helpbase ingest . --dry-run # preview without spending tokens + $ helpbase ingest . --ask "how do I log in?" # ingest + answer in terminal + $ helpbase ingest . --only auth # regen just one category + $ helpbase ingest . --require-clean # fail if tree is dirty (CI mode) + +Auth: run helpbase login (free, 500k tokens/day, no card) + OR export one of ANTHROPIC_API_KEY / OPENAI_API_KEY / AI_GATEWAY_API_KEY + (BYOK — first key found wins; --model overrides) +``` + +## `helpbase preview` + +```text +Usage: helpbase preview [options] + +Open a browser-viewable help center from the docs `helpbase ingest` generated. -## `helpbase Get` +Options: + -p, --port Port to serve on (default: "3000") + --reset Wipe the cached preview renderer and re-scaffold + --setup-only Scaffold and install without starting the server + -h, --help display help for command +Examples: + $ helpbase preview # open http://localhost:3000 + $ helpbase preview --port 4001 # use a different port + $ helpbase preview --reset # wipe the cache and re-scaffold + $ helpbase preview --setup-only # warm the cache, don't start server + +First run takes ~45-60s (scaffold + install). Every run after is ~3s. + +Pair with: + $ helpbase ingest . # generate the docs first ``` -helpbase v0.0.1 -CLI for managing your Helpbase help center -Most common: - helpbase new · helpbase dev · helpbase deploy +## `helpbase new` -Usage: - helpbase [global-options] [command-options] +```text +Usage: helpbase new [options] -Global options: - -V, --version output the version number - --json Emit machine-readable JSON on stdout (suppresses decorative output) - --quiet Suppress decorative output (spinners, next-steps, summaries) - -h, --help display help for command +Create a new article from a template -Commands: +Options: + -t, --type Template type: getting-started, how-to, concept, + troubleshooting + -d, --dir Content directory (default: "content") + -c, --category Category slug (defaults to the template's + default) + --title Article title + --description <description> Short description for the article frontmatter + --slug <slug> Article slug (derived from title if omitted) + -h, --help display help for command - Get started - new Create a new article from a template - dev Start the development server - - Ship - deploy Deploy your help center to helpbase cloud - link Link this project to a helpbase cloud tenant - open Open this project's help center in the default browser - - Author - generate Generate help articles using AI - sync Propose MDX edits grounded in a code diff (no writes by default) - mcp Run the Helpbase MCP server (stdio or HTTP) - audit Check content health: missing fields, broken links, schema errors - - Account - login Log in to helpbase cloud - logout Log out of helpbase cloud - whoami Print the current login and linked tenant - config Read or write CLI preferences (~/.helpbase/config.json) - - Diagnose - doctor Print diagnostic info about your helpbase install and project - feedback Open a prefilled GitHub issue with your environment info - completion Print a shell completion script - upgrade Print the command to upgrade helpbase (does not execute it) - -Run `helpbase <command> --help` for command-specific options. -Docs: https://helpbase.dev +Examples: + $ helpbase new # fully interactive + $ helpbase new --type how-to --title "Reset your password" + $ helpbase new --type getting-started --title "Get started" --category intro ``` -## `helpbase Ship` +## `helpbase dev` +```text +Usage: helpbase dev [options] + +Start the development server + +Options: + -p, --port <port> Port to run on (default: "3000") + --no-lint Skip on-save article linting + --content-dir <path> Directory to watch for MDX changes (default: "content") + -h, --help display help for command +``` + +## `helpbase deploy` + +```text +Usage: helpbase deploy [options] + +Deploy your help center to helpbase cloud + +Options: + --slug <slug> Subdomain slug (e.g., my-product) + --rotate-mcp-token Rotate the tenant's MCP bearer token (invalidates all + currently-active clients). Does not publish content. + --delete <slug> Hard-delete a tenant and release its slug. Requires owner + match. Cascades to articles, categories, chunks, deploys, + and queries. + --yes Skip the confirmation prompt for --delete (required for + non-interactive). + -h, --help display help for command + +Examples: + $ helpbase deploy + $ helpbase deploy --slug acme-docs + $ HELPBASE_TOKEN=xxx helpbase deploy --slug acme-docs # CI / non-interactive + $ helpbase deploy --rotate-mcp-token # rotate MCP token only + $ helpbase deploy --delete acme-docs --yes # delete tenant + release slug ``` -helpbase v0.0.1 -CLI for managing your Helpbase help center -Most common: - helpbase new · helpbase dev · helpbase deploy +## `helpbase link` -Usage: - helpbase [global-options] <command> [command-options] +```text +Usage: helpbase link [options] -Global options: - -V, --version output the version number - --json Emit machine-readable JSON on stdout (suppresses decorative output) - --quiet Suppress decorative output (spinners, next-steps, summaries) +Link this project to a helpbase cloud tenant + +Options: + --slug <slug> Link directly to a tenant by slug (skips the picker) + --remove Remove the existing .helpbase/project.json binding -h, --help display help for command +``` -Commands: +## `helpbase open` - Get started - new Create a new article from a template - dev Start the development server - - Ship - deploy Deploy your help center to helpbase cloud - link Link this project to a helpbase cloud tenant - open Open this project's help center in the default browser - - Author - generate Generate help articles using AI - sync Propose MDX edits grounded in a code diff (no writes by default) - mcp Run the Helpbase MCP server (stdio or HTTP) - audit Check content health: missing fields, broken links, schema errors - - Account - login Log in to helpbase cloud - logout Log out of helpbase cloud - whoami Print the current login and linked tenant - config Read or write CLI preferences (~/.helpbase/config.json) - - Diagnose - doctor Print diagnostic info about your helpbase install and project - feedback Open a prefilled GitHub issue with your environment info - completion Print a shell completion script - upgrade Print the command to upgrade helpbase (does not execute it) - -Run `helpbase <command> --help` for command-specific options. -Docs: https://helpbase.dev +```text +Usage: helpbase open [options] + +Open this project's help center in the default browser + +Options: + --print Print the URL instead of opening it (useful for CI / scripts) + -h, --help display help for command ``` -## `helpbase Author` +## `helpbase generate` + +```text +Usage: helpbase generate [options] [repoPath] + +Generate help articles using AI + +Arguments: + repoPath Path to a local repository (alias for --repo; ignored if + --repo is set) + +Options: + --url <url> Scrape a website URL and generate articles + --repo <path> Read a local repository and generate articles + --screenshots <dir> Generate visual how-to articles from a folder of + screenshots + --title <title> Title for the generated how-to guide (required with + --screenshots when no --url) + -o, --output <dir> Output directory for generated articles (default: + "content") + --test Use the cheap test model + (google/gemini-3.1-flash-lite-preview) and print model + info + --model <id> Override the model ID (e.g. anthropic/claude-sonnet-4.6) + --debug Write the raw scraped text to <output>/_scrape.txt before + calling the LLM (useful when debugging bad article + quality) + --dry-run Scrape the URL and print what would be sent to the LLM, + without spending tokens + --yes Skip interactive confirmations (for CI/scripted use) + --no-overwrite Error instead of overwriting existing image assets + -h, --help display help for command + +Examples: + $ helpbase generate --url https://myproduct.com + $ helpbase generate --url https://myproduct.com --test # cheap model + $ helpbase generate . # read local repo markdown (positional) + $ helpbase generate --repo . # same, explicit flag + $ helpbase generate ./path/to/repo --dry-run # preview without spending tokens + $ helpbase generate --screenshots ./flow --title "How to invite a teammate" + $ helpbase generate --url https://myproduct.com --dry-run # preview without spending tokens + +Auth: run `helpbase login` (free, 500k tokens/day) OR export one of +ANTHROPIC_API_KEY / OPENAI_API_KEY / AI_GATEWAY_API_KEY (BYOK, first found wins). +``` +## `helpbase sync` + +```text +Usage: helpbase sync [options] + +Propose MDX edits grounded in a code diff (no writes by default) + +Options: + --since <rev> Git rev to diff against (default: origin/main or HEAD~10) + --content <dir> MDX content directory (default: "content") + -o, --output <file> Write the unified diff to this file instead of stdout + --model <id> Override the model ID (e.g. anthropic/claude-sonnet-4.6) + --test Use the cheap test model + (google/gemini-3.1-flash-lite-preview) + --dry-run Show what would be proposed without calling the LLM + --demo Skip the LLM and render the bundled demo fixture + (first-run magic) + --apply Apply accepted proposals to the MDX files in-place + --yes Skip interactive confirmations (for CI) + -h, --help display help for command + +Examples: + $ helpbase sync --demo # 30-second tour, no key required + $ helpbase sync --since HEAD~5 # diff last 5 commits + $ helpbase sync --since origin/main # diff against tracking branch + $ helpbase sync --apply # apply proposals in-place + +Set AI_GATEWAY_API_KEY first — https://vercel.com/ai-gateway ``` -helpbase v0.0.1 -CLI for managing your Helpbase help center -Most common: - helpbase new · helpbase dev · helpbase deploy +## `helpbase mcp` -Usage: - helpbase [global-options] <command> [command-options] +```text +Usage: helpbase mcp [options] [command] -Global options: - -V, --version output the version number - --json Emit machine-readable JSON on stdout (suppresses decorative output) - --quiet Suppress decorative output (spinners, next-steps, summaries) - -h, --help display help for command +Run the Helpbase MCP server (stdio or HTTP) + +Options: + -h, --help display help for command Commands: + start [options] Start the MCP server + help [command] display help for command + +Examples: + $ helpbase mcp start # stdio — Claude Desktop / Cursor / Zed + $ helpbase mcp start --http # HTTP — remote agents, internal KB + $ helpbase mcp start --http --port 4000 # custom port - Get started - new Create a new article from a template - dev Start the development server - - Ship - deploy Deploy your help center to helpbase cloud - link Link this project to a helpbase cloud tenant - open Open this project's help center in the default browser - - Author - generate Generate help articles using AI - sync Propose MDX edits grounded in a code diff (no writes by default) - mcp Run the Helpbase MCP server (stdio or HTTP) - audit Check content health: missing fields, broken links, schema errors - - Account - login Log in to helpbase cloud - logout Log out of helpbase cloud - whoami Print the current login and linked tenant - config Read or write CLI preferences (~/.helpbase/config.json) - - Diagnose - doctor Print diagnostic info about your helpbase install and project - feedback Open a prefilled GitHub issue with your environment info - completion Print a shell completion script - upgrade Print the command to upgrade helpbase (does not execute it) - -Run `helpbase <command> --help` for command-specific options. -Docs: https://helpbase.dev +HTTP mode reads two env vars: + HELPBASE_MCP_TOKEN required — bearer token for auth + HELPBASE_MCP_ALLOWED_ORIGINS optional — comma-separated CORS allowlist ``` -## `helpbase Account` +## `helpbase audit` + +```text +Usage: helpbase audit [options] +Check content health: missing fields, broken links, schema errors + +Options: + -d, --dir <dir> Content directory to audit (default: "content") + -f, --format <format> Output format: text or json (default: "text") + -h, --help display help for command ``` -helpbase v0.0.1 -CLI for managing your Helpbase help center -Most common: - helpbase new · helpbase dev · helpbase deploy +## `helpbase login` -Usage: - helpbase [global-options] <command> [command-options] +```text +Usage: helpbase login [options] -Global options: - -V, --version output the version number - --json Emit machine-readable JSON on stdout (suppresses decorative output) - --quiet Suppress decorative output (spinners, next-steps, summaries) - -h, --help display help for command +Log in to helpbase cloud -Commands: +Options: + -e, --email [email] Use magic-link email fallback (CI / sandboxed envs) + -h, --help display help for command +``` + +## `helpbase logout` + +```text +Usage: helpbase logout [options] + +Log out of helpbase cloud - Get started - new Create a new article from a template - dev Start the development server - - Ship - deploy Deploy your help center to helpbase cloud - link Link this project to a helpbase cloud tenant - open Open this project's help center in the default browser - - Author - generate Generate help articles using AI - sync Propose MDX edits grounded in a code diff (no writes by default) - mcp Run the Helpbase MCP server (stdio or HTTP) - audit Check content health: missing fields, broken links, schema errors - - Account - login Log in to helpbase cloud - logout Log out of helpbase cloud - whoami Print the current login and linked tenant - config Read or write CLI preferences (~/.helpbase/config.json) - - Diagnose - doctor Print diagnostic info about your helpbase install and project - feedback Open a prefilled GitHub issue with your environment info - completion Print a shell completion script - upgrade Print the command to upgrade helpbase (does not execute it) - -Run `helpbase <command> --help` for command-specific options. -Docs: https://helpbase.dev +Options: + -h, --help display help for command ``` -## `helpbase Diagnose` +## `helpbase whoami` +```text +Usage: helpbase whoami [options] + +Print the current login, linked tenant, and today's usage + +Options: + -f, --format <format> Output format: text or json (default: "text") + -h, --help display help for command ``` -helpbase v0.0.1 -CLI for managing your Helpbase help center -Most common: - helpbase new · helpbase dev · helpbase deploy +## `helpbase config` -Usage: - helpbase [global-options] <command> [command-options] +```text +Usage: helpbase config [options] [command] -Global options: - -V, --version output the version number - --json Emit machine-readable JSON on stdout (suppresses decorative output) - --quiet Suppress decorative output (spinners, next-steps, summaries) - -h, --help display help for command +Read or write CLI preferences (~/.helpbase/config.json) + +Options: + -h, --help display help for command Commands: + get <key> Print the current value of a config key + set <key> <value> Set a config key + unset <key> Clear a config key + list Print all known config keys and their current values + help [command] display help for command +``` + +## `helpbase doctor` + +```text +Usage: helpbase doctor [options] + +Print diagnostic info about your helpbase install and project + +Options: + -f, --format <format> Output format: text or json (default: "text") + --offline Skip network checks (useful on planes and behind corp + proxies) + -h, --help display help for command +``` + +## `helpbase feedback` + +```text +Usage: helpbase feedback [options] + +Open a prefilled GitHub issue with your environment info + +Options: + --print Print the URL instead of opening the browser + -t, --title <title> Issue title to prefill (default: "Bug in helpbase CLI") + -h, --help display help for command +``` + +## `helpbase completion` + +```text +Usage: helpbase completion [options] <shell> + +Print a shell completion script + +Arguments: + shell Target shell: bash, zsh, fish, powershell + +Options: + -h, --help display help for command + +Examples: + $ helpbase completion zsh > ~/.zsh/completions/_helpbase + $ eval "$(helpbase completion bash)" + $ helpbase completion fish > ~/.config/fish/completions/helpbase.fish + $ helpbase completion powershell | Out-String | Invoke-Expression +``` + +## `helpbase upgrade` + +```text +Usage: helpbase upgrade [options] + +Print the command to upgrade helpbase (does not execute it) + +Options: + -h, --help display help for command + +Examples: + $ helpbase upgrade # prints the detected upgrade command +``` - Get started - new Create a new article from a template - dev Start the development server - - Ship - deploy Deploy your help center to helpbase cloud - link Link this project to a helpbase cloud tenant - open Open this project's help center in the default browser - - Author - generate Generate help articles using AI - sync Propose MDX edits grounded in a code diff (no writes by default) - mcp Run the Helpbase MCP server (stdio or HTTP) - audit Check content health: missing fields, broken links, schema errors - - Account - login Log in to helpbase cloud - logout Log out of helpbase cloud - whoami Print the current login and linked tenant - config Read or write CLI preferences (~/.helpbase/config.json) - - Diagnose - doctor Print diagnostic info about your helpbase install and project - feedback Open a prefilled GitHub issue with your environment info - completion Print a shell completion script - upgrade Print the command to upgrade helpbase (does not execute it) - -Run `helpbase <command> --help` for command-specific options. -Docs: https://helpbase.dev +## `helpbase context` + +```text +Usage: helpbase context [options] [repoPath] + +[deprecated: use `helpbase ingest`] Turn a repo into agent-ready docs. + +Arguments: + repoPath Path to the repo to ingest (default: ".") + +Options: + -o, --output <dir> Output directory for generated content (default: + ".helpbase") + --max-tokens <n> Token budget for the LLM input (per run) (default: + "100000") + --chars-per-token <n> Chars-per-token ratio used for the budget estimate (3.5 + = mid-range; 2.8 for code-heavy, 4.2 for prose) + (default: "3.5") + --model <id> Override the model ID (e.g. + anthropic/claude-sonnet-4.6) + --test Use the cheap test model + (google/gemini-3.1-flash-lite-preview) and print model + info + --debug Write the raw assembled prompt to <output>/_prompt.txt + before calling the LLM + --dry-run Walk the repo and print what would be sent to the LLM, + without spending tokens + --allow-dirty Explicitly allow generating from a dirty working tree + (default behavior; kept for clarity) + --require-clean Exit 1 if the working tree has uncommitted changes (CI + mode) + --overwrite Overwrite existing docs, including source:custom + (default preserves custom files) + --yes Skip interactive confirmations (for CI/scripted use) + --only <category> Only (re)generate docs in one category slug + --prompt <file> Override the default prompt with a file path (content + is still wrapped in untrusted-content delimiters) + --ask <question> After generating, answer a question against the fresh + docs in-terminal (no MCP client required) + --reuse-existing With --ask: skip generation, answer against the + existing .helpbase/docs/ on disk (for eval/batch runs) + -h, --help display help for command + +This command is deprecated. Use helpbase ingest instead — same flags, +same behavior. Slated for removal in v0.7. + +Examples: + $ helpbase ingest . # ingest current repo + $ helpbase ingest . --ask "how do I log in?" # ingest + answer in terminal + +See: helpbase ingest --help ``` diff --git a/packages/cli/eval/README.md b/packages/cli/eval/README.md index 8278715..4095b2e 100644 --- a/packages/cli/eval/README.md +++ b/packages/cli/eval/README.md @@ -1,6 +1,6 @@ -# helpbase context — eval harness +# helpbase ingest — eval harness -Quantitative ship-block for `helpbase context`. An aggregate score ≥ 0.70 is the bar. +Quantitative ship-block for `helpbase ingest`. An aggregate score ≥ 0.70 is the bar. ## Usage @@ -13,8 +13,8 @@ The runner writes `packages/cli/eval/eval-report.json` and exits 0 on pass, 1 on ## What it does -1. For each repo in `questions.ts`, runs `helpbase context <repo>` to generate `.helpbase/docs`. -2. For each question, runs `helpbase context <repo> --ask <question>` to collect an in-terminal answer. +1. For each repo in `questions.ts`, runs `helpbase ingest <repo>` to generate `.helpbase/docs`. +2. For each question, runs `helpbase ingest <repo> --ask <question>` to collect an in-terminal answer. 3. Passes each (question, rubric, answer) to `grader.ts` — an LLM-as-judge that returns `{ score, citationCorrect, reasoning }`. 4. Aggregates per-repo + overall scores. Overall is the mean across repos. diff --git a/packages/cli/eval/grader.ts b/packages/cli/eval/grader.ts index 035b613..95024fa 100644 --- a/packages/cli/eval/grader.ts +++ b/packages/cli/eval/grader.ts @@ -1,7 +1,7 @@ /** * LLM-as-judge grader for eval. * - * Takes a question + rubric + the answer produced by `helpbase context --ask`, + * Takes a question + rubric + the answer produced by `helpbase ingest --ask`, * returns a numeric score (0–1) and structured reasoning. The grading * prompt is fixed so runs are reproducible across sessions. * diff --git a/packages/cli/eval/questions.ts b/packages/cli/eval/questions.ts index 9e3ac52..89b3a87 100644 --- a/packages/cli/eval/questions.ts +++ b/packages/cli/eval/questions.ts @@ -1,5 +1,5 @@ /** - * Eval question set for `helpbase context`. + * Eval question set for `helpbase ingest`. * * v1 scope: the helpbase repo itself. Each question has an expected * citation target (a path the correct doc should reference) and a @@ -51,8 +51,14 @@ export const HELPBASE_SELF_REPO: EvalRepo = { id: "context-citation-validator", question: "What happens when a generated doc's citations can't be validated against the repo?", + // Both paths accepted during the deprecation window: the canonical + // implementation is in ingest.ts, but a model analyzing the pre-v0.7 + // repo may still cite the deprecation shim context.ts. Either + // matches the rubric. TODO(v0.7): drop the context.ts entry when + // the shim is removed. expectedCitations: [ "packages/shared/src/citations.ts", + "packages/cli/src/commands/ingest.ts", "packages/cli/src/commands/context.ts", ], rubric: diff --git a/packages/cli/eval/runner.ts b/packages/cli/eval/runner.ts index b59bc6b..0a424e3 100644 --- a/packages/cli/eval/runner.ts +++ b/packages/cli/eval/runner.ts @@ -1,9 +1,9 @@ /** - * Eval runner for `helpbase context`. + * Eval runner for `helpbase ingest`. * * Per repo: - * 1. Run `helpbase context <repo>` to generate .helpbase/docs. - * 2. For each question, run `helpbase context <repo> --ask <q>` to + * 1. Run `helpbase ingest <repo>` to generate .helpbase/docs. + * 2. For each question, run `helpbase ingest <repo> --ask <q>` to * get the in-terminal answer. * 3. Grade each answer with grader.ts (LLM-as-judge). * 4. Aggregate per-repo + overall scores. Compare to EVAL_PASS_THRESHOLD. @@ -126,9 +126,9 @@ async function runRepoEval(repo: EvalRepo): Promise<RepoResult> { } // Generate docs once per repo — individual --ask calls re-use them. - console.log(` › helpbase context ${repoAbs} (model: ${EVAL_MODEL})`) + console.log(` › helpbase ingest ${repoAbs} (model: ${EVAL_MODEL})`) execSync( - `node ${CLI_PATH} context ${repoAbs} --max-tokens ${EVAL_MAX_TOKENS} --model ${EVAL_MODEL} --yes`, + `node ${CLI_PATH} ingest ${repoAbs} --max-tokens ${EVAL_MAX_TOKENS} --model ${EVAL_MODEL} --yes`, { stdio: "inherit", env: process.env }, ) @@ -174,7 +174,7 @@ function runAsk( // answer against the .helpbase/docs generated earlier in runRepoEval. // Cuts an N-question eval from N+1 full generations to 1. const out = execSync( - `node ${CLI_PATH} context ${repoAbs} --ask ${qArg} --reuse-existing --model ${EVAL_MODEL}`, + `node ${CLI_PATH} ingest ${repoAbs} --ask ${qArg} --reuse-existing --model ${EVAL_MODEL}`, { encoding: "utf8", env: process.env }, ) return { answer: out, durationMs: Date.now() - started } diff --git a/packages/cli/scripts/generate-cli-docs.mjs b/packages/cli/scripts/generate-cli-docs.mjs index 0d48e45..1726b1b 100644 --- a/packages/cli/scripts/generate-cli-docs.mjs +++ b/packages/cli/scripts/generate-cli-docs.mjs @@ -45,16 +45,32 @@ function help(args) { const topHelp = help("--help") -// Extract commands from the top-level help output — commander lists them in -// a "Commands:" block that runs to end of output. +// Extract commands from the top-level help output — commander (via our +// grouped help renderer in src/lib/help.ts) lists them in a "Commands:" +// block, with group labels at 2-space indent and command names at +// 4-space indent: +// +// Commands: +// +// Get started +// ingest Turn a repo into agent-ready docs... +// preview Open a browser-viewable help center... +// Ship +// deploy Deploy your help center to helpbase cloud +// ... +// +// Matching 2-space indent would pick up "Get started"/"Ship"/etc. as if +// they were commands, which is why docs/cli.md previously had bogus +// `## `helpbase Get`` entries. Match 4-space indent so only real +// subcommand names land in the list. const commands = [] { const idx = topHelp.indexOf("Commands:") if (idx !== -1) { const block = topHelp.slice(idx) for (const line of block.split("\n")) { - // " dev [options] Start the development server" - const m = line.match(/^ {2}(\w[\w-]*)(?:\s|$)/) + // " ingest Turn a repo into agent-ready docs..." + const m = line.match(/^ {4}(\w[\w-]*)(?:\s|$)/) if (m && m[1] !== "help" && !commands.includes(m[1])) { commands.push(m[1]) } @@ -93,7 +109,10 @@ lines.push("") for (const cmd of commands) { lines.push(`## \`helpbase ${cmd}\``) lines.push("") - lines.push("```") + // Language identifier `text` silences markdownlint MD040 on every + // command block. `text` is the right tag — this is literal `--help` + // output, not a shell transcript (no `$` prompts). + lines.push("```text") lines.push(help(`${cmd} --help`).trim()) lines.push("```") lines.push("") diff --git a/packages/cli/src/commands/context-errors.ts b/packages/cli/src/commands/context-errors.ts index e2391cb..4a0b431 100644 --- a/packages/cli/src/commands/context-errors.ts +++ b/packages/cli/src/commands/context-errors.ts @@ -1,5 +1,5 @@ /** - * Error catalog for `helpbase context`. + * Error catalog for `helpbase ingest` (and its deprecated `helpbase context` alias). * * Every error path surfaces the same {problem, cause, fix} shape so the * user always knows what happened and what to do next. This is the @@ -8,6 +8,14 @@ * * Thin layer over HelpbaseError — the doc URL is derived automatically * from the `code` (see errors.ts `docUrl()`). + * + * NOTE: the `E_CONTEXT_*` prefix is intentional and NOT tied to the + * deprecated `helpbase context` command. These codes name the + * documentation-synthesis domain; they remain stable even after the + * command alias is removed in v0.7. Doc URLs live under + * /errors/e-context-* and agents/scripts that grep for the code strings + * keep working. Do not rename to E_INGEST_* without a migration plan + * for existing users who've linked to the doc URLs. */ import { HelpbaseError, type ErrorCode } from "../lib/errors.js" @@ -24,7 +32,7 @@ export const CONTEXT_ERRORS: Record<ContextErrorCode, ContextErrorBase> = { E_CONTEXT_MISSING_KEY: { problem: "No LLM API key is set in your environment.", cause: - "helpbase context accepts any one of: AI_GATEWAY_API_KEY (any provider), " + + "helpbase ingest accepts any one of: AI_GATEWAY_API_KEY (any provider), " + "ANTHROPIC_API_KEY (--model anthropic/...), or OPENAI_API_KEY (--model openai/...). " + "Set the one you already have.", fix: [ @@ -36,11 +44,11 @@ export const CONTEXT_ERRORS: Record<ContextErrorCode, ContextErrorBase> = { E_CONTEXT_NO_SOURCES: { problem: "No source files found in the target repo.", cause: - "helpbase context walks markdown + selected code extensions and skips " + + "helpbase ingest walks markdown + selected code extensions and skips " + "secret files, lockfile dirs, build output, and .gitignore-style paths. " + "Either the directory is empty or everything got filtered out.", fix: [ - "Check the path you passed — `helpbase context .` uses the current directory.", + "Check the path you passed — `helpbase ingest .` uses the current directory.", "Add a README.md or source files with supported extensions (.md, .mdx, .ts, .tsx, .js, .py, .go, .rs, .rb, .java).", ], }, @@ -83,7 +91,7 @@ export const CONTEXT_ERRORS: Record<ContextErrorCode, ContextErrorBase> = { fix: [ "Check .helpbase/synthesis-report.json for per-citation drop reasons.", "If snippets were paraphrased, retry with --model anthropic/claude-sonnet-4.6.", - "If cited code files were missing from the reader, use --include-ext to widen it.", + "If cited files were skipped, make sure the relevant code uses one of the supported extensions (.md, .mdx, .ts, .tsx, .js, .py, .go, .rs, .rb, .java) — or add a README bridging them.", ], }, E_CONTEXT_SECRET: { @@ -99,7 +107,7 @@ export const CONTEXT_ERRORS: Record<ContextErrorCode, ContextErrorBase> = { }, E_CONTEXT_REPO_PATH: { problem: "The repo path you passed does not exist or is not a directory.", - cause: "helpbase context needs a local directory to walk.", + cause: "helpbase ingest needs a local directory to walk.", fix: [ "Check the path. Use `.` for the current directory. Absolute paths work too.", ], @@ -110,14 +118,14 @@ export const CONTEXT_ERRORS: Record<ContextErrorCode, ContextErrorBase> = { "--reuse-existing skips the walk + LLM generation so an existing .helpbase/docs/ " + "can be queried with --ask. On its own it would do nothing.", fix: [ - "Pair with --ask: `helpbase context . --reuse-existing --ask \"...\"`.", + "Pair with --ask: `helpbase ingest . --reuse-existing --ask \"...\"`.", "Or drop --reuse-existing to regenerate from scratch.", ], }, E_CONTEXT_REUSE_EMPTY: { problem: "--reuse-existing was set but .helpbase/docs/ has no MDX files to reuse.", cause: - "The output directory is empty. You need to have run `helpbase context` at " + + "The output directory is empty. You need to have run `helpbase ingest` at " + "least once before --reuse-existing has anything to answer from.", fix: [ "Run once without --reuse-existing to populate .helpbase/docs, then re-run with --reuse-existing.", @@ -127,9 +135,9 @@ export const CONTEXT_ERRORS: Record<ContextErrorCode, ContextErrorBase> = { E_CONTEXT_PREVIEW_NO_DOCS: { problem: "`helpbase preview` needs .helpbase/docs/ to exist in the current directory.", cause: - "Preview renders the MDX `helpbase context` produces. There's nothing to render yet.", + "Preview renders the MDX `helpbase ingest` produces. There's nothing to render yet.", fix: [ - "Run `helpbase context .` first — that generates the docs.", + "Run `helpbase ingest .` first — that generates the docs.", "Then `helpbase preview` to open them in the browser.", ], }, @@ -153,6 +161,41 @@ export const CONTEXT_ERRORS: Record<ContextErrorCode, ContextErrorBase> = { "Retry: `helpbase preview --reset` to re-scaffold + re-install.", ], }, + E_CONTEXT_INVALID_BUDGET: { + problem: "--max-tokens or --chars-per-token is not a valid positive number.", + cause: + "These flags must be finite positive numbers. --max-tokens accepts 0 " + + "(disables the gate); anything negative, NaN, or non-numeric used to " + + "silently fall back to defaults instead of failing loudly.", + fix: [ + "Check the value you passed. Typical: --max-tokens 100000 --chars-per-token 3.5.", + "To disable the token gate entirely, pass --max-tokens 0.", + ], + }, + E_CONTEXT_REFUSE_CLOBBER: { + problem: "--overwrite would replace custom-edited docs without --yes.", + cause: + "Some .helpbase/docs/ files have `source: custom` in their frontmatter. " + + "--overwrite together with --yes is the deliberate two-gate opt-in for " + + "clobbering them; without --yes we refuse to touch them.", + fix: [ + "Re-run with --yes to confirm: `helpbase ingest . --overwrite --yes`.", + "Or keep the custom files by dropping --overwrite.", + ], + }, + E_CONTEXT_SECRET_SOURCE: { + problem: "Secret-shaped content detected in a source file before the LLM call.", + cause: + "The source walker picked up a file whose content matched a known secret " + + "pattern (API key, PEM block, etc.). The pre-LLM scan aborts the run so " + + "the secret never leaves your machine — no LLM call, no --debug prompt dump.", + fix: [ + "Open the file and line shown above.", + "If it's a legitimate example, swap in a placeholder (sk-xxxxx) instead of a realistic-looking value.", + "If it's a real secret, rotate it and add the source file to .gitignore.", + "Check your .env*, *.pem, and *.key files — those should never leak into walked source content.", + ], + }, } export function contextError( diff --git a/packages/cli/src/commands/context.ts b/packages/cli/src/commands/context.ts index bf717c2..ab5fd63 100644 --- a/packages/cli/src/commands/context.ts +++ b/packages/cli/src/commands/context.ts @@ -1,732 +1,56 @@ import { Command } from "commander" import pc from "picocolors" -import fs from "node:fs" -import path from "node:path" -import { execSync } from "node:child_process" -import matter from "gray-matter" - -import { resolveModel, TEST_MODEL, GatewayError, MissingApiKeyError } from "@workspace/shared/ai" -import { - readContextSources, - totalChars, - type ContextSource, -} from "@workspace/shared/context-reader" -import { - buildContextPrompt, - buildLocalAskPrompt, - generateHowtosFromRepo, - articleToMdxWithCitations, - enrichCitationsFromDisk, - sanitizeMdx, - estimateTokens, - TokenBudgetExceededError, - SchemaGenerationError, -} from "@workspace/shared/ai-context" import { - createFileCache, - validateArticleCitations, -} from "@workspace/shared/citations" -import { scanForSecrets, formatSecretError } from "@workspace/shared/secrets" -import { - inventoryExistingDocs, - planContextWrites, - atomicWriteFileSync, - ensureGitignoreEntry, -} from "@workspace/shared/context-writer" -import { - generateLlmsTxt, - LLMS_FULL_MAX_BYTES, -} from "@workspace/shared/llms-txt" -import type { GeneratedContextDoc, ContextCitation } from "@workspace/shared/schemas" -import { slugify } from "@workspace/shared/slugify" + applyIngestOptions, + runIngestAction, + type IngestOpts, +} from "./ingest.js" +import { isJsonMode, isQuiet } from "../lib/tty.js" -import { HelpbaseError } from "../lib/errors.js" -import { contextError } from "./context-errors.js" -import { callLlmText } from "@workspace/shared/llm" -import { formatQuotaSuffix } from "@workspace/shared/llm-errors" -import { resolveAuthOrPromptLogin } from "../lib/inline-auth.js" -import { toCliLlmError } from "../lib/llm-errors-cli.js" +/** + * Deprecated `helpbase context` alias for `helpbase ingest`. + * + * Kept alive so CI scripts, blog posts, and user muscle memory don't + * break when we renamed the command. Every flag on `ingest` is mirrored + * here via `applyIngestOptions`, and the same `runIngestAction` handler + * runs — so behavior is identical. Only difference: a one-line + * deprecation warning on stderr before the real work starts. + * + * Suppressed under `--json` / `--quiet` so pipes and scripts don't get + * a stray non-JSON line polluting their output. + * + * Slated for removal in v0.7. Until then, this is the full-fidelity + * alias: same options, same exit codes, same next-steps output. + */ -export const contextCommand = new Command("context") - .description( - "Turn a repo into agent-ready docs: walks your code + markdown, synthesizes cited how-to guides, wires up MCP. Your docs, always up to date.", - ) - .argument("[repoPath]", "Path to the repo to ingest", ".") - .option("-o, --output <dir>", "Output directory for generated content", ".helpbase") - .option("--max-tokens <n>", "Token budget for the LLM input (per run)", "100000") - .option( - "--chars-per-token <n>", - "Chars-per-token ratio used for the budget estimate (3.5 = mid-range; 2.8 for code-heavy, 4.2 for prose)", - "3.5", - ) - .option("--model <id>", "Override the model ID (e.g. anthropic/claude-sonnet-4.6)") - .option("--test", `Use the cheap test model (${TEST_MODEL}) and print model info`) - .option( - "--debug", - "Write the raw assembled prompt to <output>/_prompt.txt before calling the LLM", - ) - .option("--dry-run", "Walk the repo and print what would be sent to the LLM, without spending tokens") - .option("--allow-dirty", "Explicitly allow generating from a dirty working tree (default behavior; kept for clarity)") - .option("--require-clean", "Exit 1 if the working tree has uncommitted changes (CI mode)") - .option("--overwrite", "Overwrite existing docs, including source:custom (default preserves custom files)") - .option("--yes", "Skip interactive confirmations (for CI/scripted use)") - .option("--only <category>", "Only (re)generate docs in one category slug") - .option("--prompt <file>", "Override the default prompt with a file path (content is still wrapped in untrusted-content delimiters)") - .option("--ask <question>", "After generating, answer a question against the fresh docs in-terminal (no MCP client required)") - .option("--reuse-existing", "With --ask: skip generation, answer against the existing .helpbase/docs/ on disk (for eval/batch runs)") +export const contextCommand = applyIngestOptions( + new Command("context").description( + "[deprecated: use `helpbase ingest`] Turn a repo into agent-ready docs.", + ), +) .addHelpText( "after", ` +${pc.yellow("This command is deprecated.")} Use ${pc.cyan("helpbase ingest")} instead — same flags, +same behavior. Slated for removal in v0.7. + Examples: - $ helpbase context . # ingest current repo - $ helpbase context ./path/to/repo # ingest a specific path - $ helpbase context . --dry-run # preview without spending tokens - $ helpbase context . --ask "how do I log in?" # ingest + answer in terminal - $ helpbase context . --only auth # regen just one category - $ helpbase context . --require-clean # fail if tree is dirty (CI mode) + $ helpbase ingest . # ingest current repo + $ helpbase ingest . --ask "how do I log in?" # ingest + answer in terminal -Auth: run ${pc.cyan("helpbase login")} (free, 500k tokens/day, no card) - OR export one of ${pc.cyan("ANTHROPIC_API_KEY")} / ${pc.cyan("OPENAI_API_KEY")} / ${pc.cyan("AI_GATEWAY_API_KEY")} - (BYOK — first key found wins; --model overrides) +See: ${pc.cyan("helpbase ingest --help")} `, ) - .action(async (repoPathArg: string, opts) => { - try { - await runContext(repoPathArg, opts) - } catch (err) { - if (err instanceof HelpbaseError) throw err - // Translate hosted-proxy errors (AuthRequired/Quota/GlobalCap/Network/Gateway) - // into HelpbaseError with per-code fix copy. Untranslated errors fall through. - const llmWrapped = toCliLlmError(err, { - retryCommand: `helpbase context ${repoPathArg}${opts.ask ? ` --ask ${JSON.stringify(opts.ask)}` : ""}`, - }) - if (llmWrapped instanceof HelpbaseError) throw llmWrapped - if (err instanceof MissingApiKeyError) throw contextError("E_CONTEXT_MISSING_KEY") - if (err instanceof TokenBudgetExceededError) { - const top = err.files - .slice() - .sort((a, b) => b.chars - a.chars) - .slice(0, 10) - .map((f) => ` ${f.path} ${formatChars(f.chars)}`) - .join("\n") - throw contextError("E_CONTEXT_OVER_BUDGET", { - cause: `Estimated ${err.estimatedTokens} tokens (cap ${err.maxTokens}). Biggest files:\n${top}`, - }) - } - if (err instanceof SchemaGenerationError) throw contextError("E_CONTEXT_SCHEMA") - if (err instanceof GatewayError) { - throw new HelpbaseError({ - code: "E_NETWORK", - problem: "The LLM gateway call failed.", - cause: err.message, - fix: [ - "Retry in a moment — transient gateway errors are common.", - "Check your AI_GATEWAY_API_KEY is valid.", - "Try --model anthropic/claude-sonnet-4.6 to switch providers.", - ], - }) - } - throw err - } + .action(async (repoPathArg: string, opts: IngestOpts) => { + emitDeprecationWarning() + await runIngestAction(repoPathArg, opts) }) -interface ContextOpts { - output: string - maxTokens: string - charsPerToken: string - model?: string - test?: boolean - debug?: boolean - dryRun?: boolean - allowDirty?: boolean - requireClean?: boolean - overwrite?: boolean - yes?: boolean - only?: string - prompt?: string - ask?: string - reuseExisting?: boolean -} - -async function runContext(repoPathArg: string, opts: ContextOpts): Promise<void> { - // ─── 1. Resolve repo path ─────────────────────────────────────── - const repoRoot = path.resolve(process.cwd(), repoPathArg || ".") - if (!fs.existsSync(repoRoot) || !fs.statSync(repoRoot).isDirectory()) { - throw contextError("E_CONTEXT_REPO_PATH", { - cause: `Resolved path: ${repoRoot}`, - }) - } - - // ─── 1.5. --reuse-existing fast path (eval / batch --ask runs) ── - // Skips the walk + LLM + validate + write pipeline entirely. Only - // legal when paired with --ask, because there's no point reusing docs - // if you're not asking a question. Reads existing .helpbase/docs and - // hands them straight to runLocalAsk. Net effect for the eval runner: - // one generate + N cheap --ask passes, instead of N+1 full regens. - if (opts.reuseExisting) { - if (!opts.ask) { - throw contextError("E_CONTEXT_REUSE_WITHOUT_ASK") - } - const outputDir = path.resolve(repoRoot, opts.output || ".helpbase") - const serialized = loadExistingDocs(outputDir) - const model = resolveModel({ test: opts.test, modelOverride: opts.model }) - const auth = await resolveAuthOrPromptLogin({ - verb: "ask", - retryCommand: `helpbase context --reuse-existing --ask ${JSON.stringify(opts.ask)}`, - }) - await runLocalAsk(opts.ask, serialized, model, auth.authToken) - return - } - - // ─── 2. Dirty-tree check (warn-by-default, block only on --require-clean) ── - checkDirtyTree(repoRoot, opts) - - // ─── 3. Walk sources ──────────────────────────────────────────── - const sources = readContextSources(repoRoot) - if (sources.length === 0) { - throw contextError("E_CONTEXT_NO_SOURCES", { - cause: `Walked ${repoRoot} and found no eligible files.`, - }) - } - - const outputDir = path.resolve(repoRoot, opts.output || ".helpbase") - const docsDir = path.join(outputDir, "docs") - const model = resolveModel({ test: opts.test, modelOverride: opts.model }) - const maxTokens = Number.parseInt(opts.maxTokens, 10) || 100000 - const charsPerToken = Number.parseFloat(opts.charsPerToken) || 3.5 - - // ─── 5. Dry-run summary — no LLM call, no writes ──────────────── - if (opts.dryRun) { - const estimated = estimateTokens(sources, charsPerToken) - const preview = sources - .slice(0, 10) - .map((s) => ` ${s.path} ${formatChars(s.content.length)}`) - .join("\n") - const tail = - sources.length > 10 ? `\n ...and ${sources.length - 10} more` : "" - console.log("") - console.log(`${pc.dim("›")} ${pc.bold("Dry run — no LLM call, no writes")}`) - console.log(` Repo: ${pc.cyan(repoRoot)}`) - console.log(` Output: ${pc.cyan(outputDir)}`) - console.log(` Model: ${pc.cyan(model)}`) - console.log(` Sources found: ${sources.length}`) - console.log(` Total chars: ${formatChars(totalChars(sources))}`) - console.log(` Estimated tokens: ${estimated.toLocaleString()} (budget ${maxTokens.toLocaleString()})`) - console.log(` Sample sources:`) - console.log(preview + tail) - console.log("") - console.log(` ${pc.dim("Remove --dry-run to synthesize docs.")}`) - return - } - - // Project-name resolution: prefer package.json `name`, fall back to the - // repo directory basename. Used for the LLM prompt ("You are documenting - // X") and for llms.txt — so both surfaces agree. Without this, a repo in - // a temp dir like /tmp/xyz123 tells the LLM it's documenting "xyz123" - // even when package.json clearly says "todo-app". - const repoLabel = resolveProjectName(repoRoot) - - // ─── 6. Debug prompt dump (optional) ──────────────────────────── - if (opts.debug) { - const prompt = buildContextPrompt({ sources, repoLabel }) - fs.mkdirSync(outputDir, { recursive: true }) - const debugPath = path.join(outputDir, "_prompt.txt") - fs.writeFileSync(debugPath, prompt) - console.log(`${pc.dim("›")} Wrote prompt to ${pc.cyan(debugPath)}`) - } - - // ─── 6.5. Auth resolution — BYOK env var OR helpbase session ────── - const auth = await resolveAuthOrPromptLogin({ - verb: "context", - retryCommand: `helpbase context ${repoPathArg}${opts.ask ? ` --ask ${JSON.stringify(opts.ask)}` : ""}`, - }) - - // ─── 7. LLM synthesis ─────────────────────────────────────────── - console.log( - `${pc.dim("›")} Synthesizing how-tos from ${pc.bold(String(sources.length))} source files...`, - ) - const startLlm = Date.now() - const rawDocs = await generateHowtosFromRepo({ - sources, - repoLabel, - model, - maxTokens, - charsPerToken, - authToken: auth.authToken, - }) - const llmMs = Date.now() - startLlm - console.log( - ` ${pc.dim("›")} ${pc.bold(String(rawDocs.length))} draft docs in ${(llmMs / 1000).toFixed(1)}s`, - ) - - // ─── 8. Sanitize + validate citations ─────────────────────────── - const cache = createFileCache() - const report: SynthesisReport = { - startedAt: new Date().toISOString(), - repoRoot, - model, - tokensEstimated: estimateTokens(sources, charsPerToken), - durationMs: 0, - kept: [], - droppedDocs: [], - preservedFiles: [], - deletedFiles: [], - } - - const kept: GeneratedContextDoc[] = [] - for (const rawDoc of rawDocs) { - const sanitized: GeneratedContextDoc = { - ...rawDoc, - content: sanitizeMdx(rawDoc.content), - } - const v = validateArticleCitations(sanitized, repoRoot, cache) - if (v.kept.length === 0) { - report.droppedDocs.push({ - title: rawDoc.title, - reason: "all citations failed validation", - citations: v.dropped.map((d) => ({ - file: d.citation.file, - startLine: d.citation.startLine, - endLine: d.citation.endLine, - reason: d.reason, - })), - }) - continue - } - if (v.dropped.length > 0) { - report.droppedDocs.push({ - title: rawDoc.title, - reason: `partial: ${v.dropped.length} of ${sanitized.citations.length} citations dropped; doc kept with remaining ${v.kept.length}`, - citations: v.dropped.map((d) => ({ - file: d.citation.file, - startLine: d.citation.startLine, - endLine: d.citation.endLine, - reason: d.reason, - })), - }) - } - // v2 citations omit `snippet` — enrich from disk so MDX carries real - // bytes, not model-paraphrased text. Reuses the validator's file cache. - const enriched = enrichCitationsFromDisk(v.kept, repoRoot, cache) - kept.push({ ...sanitized, citations: enriched }) - report.kept.push({ - title: sanitized.title, - category: sanitized.category, - citationCount: enriched.length, - }) - } - - // ─── 9. Zero-doc guard ────────────────────────────────────────── - if (kept.length === 0) { - fs.mkdirSync(outputDir, { recursive: true }) - const reportPath = path.join(outputDir, "synthesis-report.json") - report.durationMs = Date.now() - startLlm - fs.writeFileSync(reportPath, JSON.stringify(report, null, 2)) - throw contextError("E_CONTEXT_NO_VALID_CITATIONS", { - cause: `${rawDocs.length} docs emitted, 0 passed citation validation. See ${reportPath} for per-doc drop reasons.`, - }) - } - - // ─── 10. Serialize to MDX strings ─────────────────────────────── - const serialized: Array<{ relPath: string; content: string; doc: GeneratedContextDoc }> = [] - kept.forEach((doc, idx) => { - const catSlug = slugify(doc.category) - const docSlug = slugify(doc.title) - const relPath = `${catSlug}/${docSlug}.mdx` - const content = articleToMdxWithCitations(doc, idx + 1) - serialized.push({ relPath, content, doc }) - }) - - // ─── 11. Pre-write secret scan (on EVERY final MDX string) ────── - for (const s of serialized) { - const matches = scanForSecrets(s.content) - if (matches.length > 0) { - const msg = formatSecretError(matches, s.relPath) - process.stderr.write(msg + "\n") - throw contextError("E_CONTEXT_SECRET", { - cause: `Pattern "${matches[0]!.patternName}" matched in ${s.relPath}. No files were written.`, - }) - } - } - - // ─── 12. Inventory + plan writes ──────────────────────────────── - const existing = inventoryExistingDocs(docsDir) - const planned = planContextWrites({ - newDocs: serialized.map(({ relPath, content }) => ({ relPath, content })), - docsDir, - existing, - onlyCategory: opts.only ? slugify(opts.only) : undefined, - }) - // --overwrite removes custom-file preservation. - if (opts.overwrite && planned.preserves.length > 0) { - // Don't silently clobber custom edits — require --yes as a second gate. - if (!opts.yes) { - process.stderr.write( - `${pc.yellow("!")} --overwrite was set but --yes was not. Refusing to clobber ${planned.preserves.length} custom file(s). Pass --yes to confirm.\n`, - ) - process.exit(1) - } - // Move preserves into writes so they get overwritten. - // Build the serialized-by-rel index so we only overwrite custom files - // that have a matching generated doc (otherwise nothing to overwrite with). - const byRel = new Map(serialized.map((s) => [path.join(docsDir, s.relPath), s.content])) - for (const p of planned.preserves) { - const replacement = byRel.get(p) - if (replacement) planned.writes.push({ absPath: p, content: replacement }) - } - } - report.preservedFiles = planned.preserves - report.deletedFiles = planned.deletes - - // ─── 13. Execute writes atomically ────────────────────────────── - const written: string[] = [] - try { - for (const w of planned.writes) { - atomicWriteFileSync(w.absPath, w.content) - written.push(w.absPath) - } - } catch (err) { - // Best-effort rollback: remove anything we wrote this run. - for (const p of written) { - try { - fs.rmSync(p, { force: true }) - } catch { - // best-effort only - } - } - throw err - } - // Deletes come AFTER successful writes so a failed write doesn't lose stale generated files. - for (const p of planned.deletes) { - try { - fs.rmSync(p, { force: true }) - } catch { - // best-effort — a stale generated file left behind is not worth aborting - } - } - - // ─── 14. .gitignore safety ────────────────────────────────────── - ensureGitignoreEntry(repoRoot, path.relative(repoRoot, outputDir)) - - // ─── 14.5. Emit llms.txt + llms-full.txt ──────────────────────── - emitLlmsTxt(repoRoot, docsDir, outputDir) - - // ─── 14.7. Emit mcp.json config hint ──────────────────────────── - emitMcpJsonHint(outputDir, docsDir) - - // ─── 15. Synthesis report ─────────────────────────────────────── - fs.mkdirSync(outputDir, { recursive: true }) - report.durationMs = Date.now() - startLlm - const reportPath = path.join(outputDir, "synthesis-report.json") - fs.writeFileSync(reportPath, JSON.stringify(report, null, 2)) - - console.log( - ` ${pc.dim("›")} Wrote ${pc.bold(String(planned.writes.length))} MDX files → ${pc.cyan(path.relative(repoRoot, docsDir))}`, - ) - if (planned.preserves.length > 0) { - console.log( - ` ${pc.dim("›")} Preserved ${planned.preserves.length} custom file(s)`, - ) - } - if (planned.deletes.length > 0) { - console.log( - ` ${pc.dim("›")} Deleted ${planned.deletes.length} stale generated file(s)`, - ) - } - - // ─── 16. --ask (local RAG, no MCP client required) ───────────── - if (opts.ask) { - await runLocalAsk(opts.ask, serialized, model, auth.authToken) - } else { - printNextSteps(repoPathArg || ".", outputDir) - } -} - -function printNextSteps(repoArg: string, outputDir: string): void { - console.log("") - console.log(pc.bold("Next:")) - console.log( - ` ${pc.dim("1. Open it in a browser:")} ${pc.cyan("helpbase preview")}`, - ) - console.log( - ` ${pc.dim("2. Try it in-terminal:")} ${pc.cyan(`helpbase context ${repoArg} --ask "how do I log in?"`)}`, - ) - console.log(` ${pc.dim("3. Share with an MCP client — copy a block from:")}`) - console.log(` ${pc.cyan(path.join(outputDir, "mcp.json"))}`) - console.log(` ${pc.dim("into your client's config file:")}`) - for (const p of mcpClientConfigPaths()) { - console.log(` ${pc.dim("·")} ${p.client}: ${pc.cyan(p.path)}`) - } - console.log( - ` ${pc.dim("4. Or browse")} ${pc.cyan(path.join(outputDir, "llms.txt"))} / ${pc.cyan(path.join(outputDir, "llms-full.txt"))} ${pc.dim("— standard llms.txt format")}`, - ) - console.log("") -} - -// ── Helpers ────────────────────────────────────────────────────────── - -function checkDirtyTree(repoRoot: string, opts: ContextOpts): void { - let dirtyLines = "" - try { - dirtyLines = execSync("git status --porcelain", { - cwd: repoRoot, - encoding: "utf8", - stdio: ["ignore", "pipe", "ignore"], - }).trim() - } catch { - // Not a git repo — silently skip the check. - return - } - if (!dirtyLines) return - const count = dirtyLines.split("\n").length - if (opts.requireClean) { - throw contextError("E_CONTEXT_DIRTY_TREE", { - cause: `${count} uncommitted file(s) detected.`, - }) - } - // Default + --allow-dirty: warn and continue. +function emitDeprecationWarning(): void { + if (isJsonMode() || isQuiet()) return process.stderr.write( - `${pc.yellow("!")} Working tree has ${count} uncommitted change(s). Continuing — pass --require-clean for CI-mode.\n`, + `${pc.yellow("!")} ${pc.dim("helpbase context is deprecated. Use ")}${pc.cyan( + "helpbase ingest", + )}${pc.dim(" — same flags, same behavior. Removed in v0.7.")}\n`, ) } - -/** - * Resolve the project's display name. - * - * Preferred: `package.json` `name` field. Fallback: the repo directory's - * basename. A repo cloned into a temp path shouldn't leak "tmp-ax89z" into - * user-visible output — package.json is the user's intent, the path is an - * implementation detail. - */ -export function resolveProjectName(repoRoot: string): string { - const pkgPath = path.join(repoRoot, "package.json") - if (fs.existsSync(pkgPath)) { - try { - const pkg = JSON.parse(fs.readFileSync(pkgPath, "utf8")) - if (typeof pkg.name === "string" && pkg.name.trim().length > 0) { - return pkg.name.trim() - } - } catch { - // fall through to the basename - } - } - return path.basename(repoRoot) -} - -function emitLlmsTxt(repoRoot: string, docsDir: string, outputDir: string): void { - const projectName = resolveProjectName(repoRoot) - let projectSummary: string | undefined - let siteUrl: string | undefined = process.env.HELPBASE_SITE_URL - const pkgPath = path.join(repoRoot, "package.json") - if (fs.existsSync(pkgPath)) { - try { - const pkg = JSON.parse(fs.readFileSync(pkgPath, "utf8")) - if (typeof pkg.description === "string" && pkg.description) - projectSummary = pkg.description - if (!siteUrl && typeof pkg.homepage === "string" && pkg.homepage) - siteUrl = pkg.homepage - } catch { - // fall back to defaults - } - } - if (!projectSummary) { - projectSummary = `Docs for ${projectName}, generated by helpbase context.` - } - - const { llmsTxt, llmsFullTxt, docCount, fullBytes } = generateLlmsTxt({ - contentDir: docsDir, - projectName, - projectSummary, - siteUrl, - }) - - fs.writeFileSync(path.join(outputDir, "llms.txt"), llmsTxt, "utf-8") - fs.writeFileSync(path.join(outputDir, "llms-full.txt"), llmsFullTxt, "utf-8") - - if (!siteUrl) { - process.stderr.write( - `${pc.dim("›")} HELPBASE_SITE_URL not set; llms.txt uses relative URLs. Set the env var or add "homepage" to package.json for absolute links.\n`, - ) - } - if (fullBytes > LLMS_FULL_MAX_BYTES) { - process.stderr.write( - `${pc.yellow("!")} llms-full.txt is ${(fullBytes / 1024 / 1024).toFixed(1)}MB (over ${LLMS_FULL_MAX_BYTES / 1024 / 1024}MB sanity cap).\n`, - ) - } - // Intentionally silent on the happy path — the next-steps block - // surfaces the paths; no need for a per-file console log. - void docCount -} - -function emitMcpJsonHint(outputDir: string, docsDir: string): void { - // The mcp.json file is NOT a config the MCP server consumes — it's a - // hint for the user. They copy one of the blocks into their client's - // config file (paths listed after this function runs). - // - // HELPBASE_CONTENT_DIR = absolute path to the generated docs dir. Uses - // path.resolve(opts.output, "docs") so --output overrides work. - const mcpServerBlock = { - command: "npx", - args: ["@helpbase/mcp@latest"], - env: { - HELPBASE_CONTENT_DIR: docsDir, - }, - } - const hint = { - _comment: - "This file is a HINT — copy one of the blocks below into your MCP client's config. Do NOT commit: HELPBASE_CONTENT_DIR is an absolute machine-local path.", - helpbaseMcpVersion: "1", - claude_desktop: { mcpServers: { helpbase: mcpServerBlock } }, - cursor: { mcpServers: { helpbase: mcpServerBlock } }, - claude_code: { mcpServers: { helpbase: mcpServerBlock } }, - } - fs.writeFileSync( - path.join(outputDir, "mcp.json"), - JSON.stringify(hint, null, 2) + "\n", - "utf8", - ) -} - -function mcpClientConfigPaths(): Array<{ client: string; path: string }> { - const home = process.env.HOME ?? process.env.USERPROFILE ?? "~" - const isWindows = process.platform === "win32" - if (isWindows) { - const appData = process.env.APPDATA ?? `${home}\\AppData\\Roaming` - return [ - { client: "Claude Desktop", path: `${appData}\\Claude\\claude_desktop_config.json` }, - { client: "Cursor", path: `${home}\\.cursor\\mcp.json` }, - { client: "Claude Code", path: `${home}\\.claude\\mcp.json` }, - ] - } - const isMac = process.platform === "darwin" - const claudeDesktop = isMac - ? `${home}/Library/Application Support/Claude/claude_desktop_config.json` - : `${home}/.config/Claude/claude_desktop_config.json` - return [ - { client: "Claude Desktop", path: claudeDesktop }, - { client: "Cursor", path: `${home}/.cursor/mcp.json` }, - { client: "Claude Code", path: `${home}/.claude/mcp.json` }, - ] -} - -function formatChars(n: number): string { - if (n < 1000) return `${n}b` - if (n < 1_000_000) return `${(n / 1000).toFixed(1)}Kb` - return `${(n / 1_000_000).toFixed(1)}Mb` -} - -interface SynthesisReport { - startedAt: string - repoRoot: string - model: string - tokensEstimated: number - durationMs: number - kept: Array<{ title: string; category: string; citationCount: number }> - droppedDocs: Array<{ - title: string - reason: string - citations: Array<{ file: string; startLine: number; endLine: number; reason: string }> - }> - preservedFiles: string[] - deletedFiles: string[] -} - -// ── --reuse-existing: load docs from disk into runLocalAsk shape ───── - -/** - * Walk `.helpbase/docs/` for existing MDX files and reconstruct the - * `serialized`-shape array that `runLocalAsk` consumes. Used by the - * `--reuse-existing --ask` fast path so the eval runner can do one - * expensive generation and many cheap question passes instead of N+1 - * full regens. - */ -function loadExistingDocs( - outputDir: string, -): Array<{ relPath: string; content: string; doc: GeneratedContextDoc }> { - const docsDir = path.join(outputDir, "docs") - if (!fs.existsSync(docsDir)) { - throw contextError("E_CONTEXT_REUSE_EMPTY", { - cause: `No directory at ${docsDir}.`, - }) - } - const results: Array<{ relPath: string; content: string; doc: GeneratedContextDoc }> = [] - // docs/<category>/<slug>.mdx — two-level walk, not recursive globbing. - for (const categoryDir of fs.readdirSync(docsDir, { withFileTypes: true })) { - if (!categoryDir.isDirectory()) continue - const catPath = path.join(docsDir, categoryDir.name) - for (const entry of fs.readdirSync(catPath, { withFileTypes: true })) { - if (!entry.isFile() || !entry.name.endsWith(".mdx")) continue - const full = path.join(catPath, entry.name) - const content = fs.readFileSync(full, "utf8") - const parsed = matter(content) - // Minimal reconstruction — runLocalAsk uses title from doc, body - // from parsed content. Extra fields kept so the shape is valid. - const doc: GeneratedContextDoc = { - title: String(parsed.data.title ?? entry.name), - description: String(parsed.data.description ?? ""), - category: categoryDir.name, - tags: Array.isArray(parsed.data.tags) ? parsed.data.tags : [], - content: parsed.content, - citations: Array.isArray(parsed.data.citations) ? parsed.data.citations : [], - sourcePaths: [], - } - results.push({ - relPath: path.join(categoryDir.name, entry.name), - content, - doc, - }) - } - } - if (results.length === 0) { - throw contextError("E_CONTEXT_REUSE_EMPTY") - } - return results -} - -// ── Local --ask (the magical-moment fix) ───────────────────────────── - -async function runLocalAsk( - question: string, - serialized: Array<{ relPath: string; content: string; doc: GeneratedContextDoc }>, - model: string, - authToken?: string, -): Promise<void> { - // Ask against just the generated body (not frontmatter), since the model - // doesn't need to see YAML. Pass the slug path so it can cite correctly. - const docs = serialized.map((s) => { - const parsed = matter(s.content) - return { - title: s.doc.title, - path: s.relPath, - body: parsed.content, - } - }) - const prompt = buildLocalAskPrompt({ question, docs }) - console.log("") - console.log(`${pc.dim("›")} Answering: ${pc.bold(JSON.stringify(question))}`) - const started = Date.now() - let text: string - let quotaForWarning: ReturnType<typeof formatQuotaSuffix> | null = null - try { - const result = await callLlmText({ model, prompt, authToken }) - text = result.text - if (result.quota) { - const pct = Math.round((result.quota.usedToday / result.quota.dailyLimit) * 100) - if (pct >= 80) { - quotaForWarning = formatQuotaSuffix(result.quota) - } - } - } catch (err) { - throw toCliLlmError(err, { - retryCommand: `helpbase context --ask ${JSON.stringify(question)}`, - }) - } - const ms = Date.now() - started - console.log("") - console.log(text) - console.log("") - console.log(` ${pc.dim(`answered in ${(ms / 1000).toFixed(1)}s using ${model}`)}`) - if (quotaForWarning) { - console.log(` ${pc.yellow("heads-up:")} ${quotaForWarning}`) - } -} diff --git a/packages/cli/src/commands/ingest.ts b/packages/cli/src/commands/ingest.ts new file mode 100644 index 0000000..ea2feee --- /dev/null +++ b/packages/cli/src/commands/ingest.ts @@ -0,0 +1,820 @@ +import { Command } from "commander" +import pc from "picocolors" +import fs from "node:fs" +import path from "node:path" +import { execSync } from "node:child_process" +import matter from "gray-matter" + +import { resolveModel, TEST_MODEL, GatewayError, MissingApiKeyError } from "@workspace/shared/ai" +import { + readContextSources, + totalChars, +} from "@workspace/shared/context-reader" +import { + buildContextPrompt, + buildLocalAskPrompt, + generateHowtosFromRepo, + articleToMdxWithCitations, + enrichCitationsFromDisk, + sanitizeMdx, + estimateTokens, + TokenBudgetExceededError, + SchemaGenerationError, +} from "@workspace/shared/ai-context" +import { + createFileCache, + validateArticleCitations, +} from "@workspace/shared/citations" +import { scanForSecrets, formatSecretError } from "@workspace/shared/secrets" +import { + inventoryExistingDocs, + planContextWrites, + atomicWriteFileSync, + ensureGitignoreEntry, +} from "@workspace/shared/context-writer" +import { + generateLlmsTxt, + LLMS_FULL_MAX_BYTES, +} from "@workspace/shared/llms-txt" +import type { GeneratedContextDoc } from "@workspace/shared/schemas" +import { slugify } from "@workspace/shared/slugify" + +import { HelpbaseError } from "../lib/errors.js" +import { contextError } from "./context-errors.js" +import { callLlmText } from "@workspace/shared/llm" +import { formatQuotaSuffix } from "@workspace/shared/llm-errors" +import { resolveAuthOrPromptLogin } from "../lib/inline-auth.js" +import { toCliLlmError } from "../lib/llm-errors-cli.js" + +/** + * Apply the full ingest option surface to a Command. Shared between + * `ingestCommand` and the deprecated `contextCommand` (in context.ts) so + * any flag added here is automatically available on `helpbase context` + * while the shim is alive. Keeps the two surfaces in lockstep without + * option-by-option duplication. + */ +export function applyIngestOptions(cmd: Command): Command { + return cmd + .argument("[repoPath]", "Path to the repo to ingest", ".") + .option("-o, --output <dir>", "Output directory for generated content", ".helpbase") + .option("--max-tokens <n>", "Token budget for the LLM input (per run)", "100000") + .option( + "--chars-per-token <n>", + "Chars-per-token ratio used for the budget estimate (3.5 = mid-range; 2.8 for code-heavy, 4.2 for prose)", + "3.5", + ) + .option("--model <id>", "Override the model ID (e.g. anthropic/claude-sonnet-4.6)") + .option("--test", `Use the cheap test model (${TEST_MODEL}) and print model info`) + .option( + "--debug", + "Write the raw assembled prompt to <output>/_prompt.txt before calling the LLM", + ) + .option("--dry-run", "Walk the repo and print what would be sent to the LLM, without spending tokens") + .option("--allow-dirty", "Explicitly allow generating from a dirty working tree (default behavior; kept for clarity)") + .option("--require-clean", "Exit 1 if the working tree has uncommitted changes (CI mode)") + .option("--overwrite", "Overwrite existing docs, including source:custom (default preserves custom files)") + .option("--yes", "Skip interactive confirmations (for CI/scripted use)") + .option("--only <category>", "Only (re)generate docs in one category slug") + .option("--prompt <file>", "Override the default prompt with a file path (content is still wrapped in untrusted-content delimiters)") + .option("--ask <question>", "After generating, answer a question against the fresh docs in-terminal (no MCP client required)") + .option("--reuse-existing", "With --ask: skip generation, answer against the existing .helpbase/docs/ on disk (for eval/batch runs)") +} + +/** + * Shared action handler for `helpbase ingest` and the deprecated + * `helpbase context` alias. Wraps `runIngest` with the error-translation + * layer (TokenBudget / Schema / Gateway / MissingApiKey → HelpbaseError). + */ +export async function runIngestAction( + repoPathArg: string, + opts: IngestOpts, +): Promise<void> { + try { + await runIngest(repoPathArg, opts) + } catch (err) { + if (err instanceof HelpbaseError) throw err + // Translate hosted-proxy errors (AuthRequired/Quota/GlobalCap/Network/Gateway) + // into HelpbaseError with per-code fix copy. Untranslated errors fall through. + const llmWrapped = toCliLlmError(err, { + retryCommand: `helpbase ingest ${repoPathArg}${opts.ask ? ` --ask ${JSON.stringify(opts.ask)}` : ""}`, + }) + if (llmWrapped instanceof HelpbaseError) throw llmWrapped + if (err instanceof MissingApiKeyError) throw contextError("E_CONTEXT_MISSING_KEY") + if (err instanceof TokenBudgetExceededError) { + const top = err.files + .slice() + .sort((a, b) => b.chars - a.chars) + .slice(0, 10) + .map((f) => ` ${f.path} ${formatChars(f.chars)}`) + .join("\n") + throw contextError("E_CONTEXT_OVER_BUDGET", { + cause: `Estimated ${err.estimatedTokens} tokens (cap ${err.maxTokens}). Biggest files:\n${top}`, + }) + } + if (err instanceof SchemaGenerationError) throw contextError("E_CONTEXT_SCHEMA") + if (err instanceof GatewayError) { + throw new HelpbaseError({ + code: "E_NETWORK", + problem: "The LLM gateway call failed.", + cause: err.message, + fix: [ + "Retry in a moment — transient gateway errors are common.", + "Check your AI_GATEWAY_API_KEY is valid.", + "Try --model anthropic/claude-sonnet-4.6 to switch providers.", + ], + }) + } + throw err + } +} + +export const ingestCommand = applyIngestOptions( + new Command("ingest").description( + "Turn a repo into agent-ready docs: walks your code + markdown, synthesizes cited how-to guides, wires up MCP. Your docs, always up to date.", + ), +) + .addHelpText( + "after", + ` +Examples: + $ helpbase ingest . # ingest current repo + $ helpbase ingest ./path/to/repo # ingest a specific path + $ helpbase ingest . --dry-run # preview without spending tokens + $ helpbase ingest . --ask "how do I log in?" # ingest + answer in terminal + $ helpbase ingest . --only auth # regen just one category + $ helpbase ingest . --require-clean # fail if tree is dirty (CI mode) + +Auth: run ${pc.cyan("helpbase login")} (free, 500k tokens/day, no card) + OR export one of ${pc.cyan("ANTHROPIC_API_KEY")} / ${pc.cyan("OPENAI_API_KEY")} / ${pc.cyan("AI_GATEWAY_API_KEY")} + (BYOK — first key found wins; --model overrides) +`, + ) + .action(runIngestAction) + +export interface IngestOpts { + output: string + maxTokens: string + charsPerToken: string + model?: string + test?: boolean + debug?: boolean + dryRun?: boolean + allowDirty?: boolean + requireClean?: boolean + overwrite?: boolean + yes?: boolean + only?: string + prompt?: string + ask?: string + reuseExisting?: boolean +} + +async function runIngest(repoPathArg: string, opts: IngestOpts): Promise<void> { + // ─── 1. Resolve repo path ─────────────────────────────────────── + const repoRoot = path.resolve(process.cwd(), repoPathArg || ".") + if (!fs.existsSync(repoRoot) || !fs.statSync(repoRoot).isDirectory()) { + throw contextError("E_CONTEXT_REPO_PATH", { + cause: `Resolved path: ${repoRoot}`, + }) + } + + // ─── 1.5. --reuse-existing fast path (eval / batch --ask runs) ── + // Skips the walk + LLM + validate + write pipeline entirely. Only + // legal when paired with --ask, because there's no point reusing docs + // if you're not asking a question. Reads existing .helpbase/docs and + // hands them straight to runLocalAsk. Net effect for the eval runner: + // one generate + N cheap --ask passes, instead of N+1 full regens. + if (opts.reuseExisting) { + if (!opts.ask) { + throw contextError("E_CONTEXT_REUSE_WITHOUT_ASK") + } + const outputDir = path.resolve(repoRoot, opts.output || ".helpbase") + const serialized = loadExistingDocs(outputDir) + const model = resolveModel({ test: opts.test, modelOverride: opts.model }) + const auth = await resolveAuthOrPromptLogin({ + verb: "ask", + // Include the repo path so the retry hint reproduces the full + // invocation — otherwise a user who ran `helpbase ingest /some/repo + // --reuse-existing --ask ...` would see a retry command without the + // path and re-run against the wrong working directory. + retryCommand: `helpbase ingest ${JSON.stringify(repoPathArg || ".")} --reuse-existing --ask ${JSON.stringify(opts.ask)}`, + }) + await runLocalAsk(opts.ask, serialized, model, auth.authToken) + return + } + + // ─── 2. Dirty-tree check (warn-by-default, block only on --require-clean) ── + checkDirtyTree(repoRoot, opts) + + // ─── 3. Walk sources ──────────────────────────────────────────── + const sources = readContextSources(repoRoot) + if (sources.length === 0) { + throw contextError("E_CONTEXT_NO_SOURCES", { + cause: `Walked ${repoRoot} and found no eligible files.`, + }) + } + + // ─── 3.5. Pre-LLM secret scan on source content ───────────────── + // readContextSources already skips files whose NAMES look secret + // (.env*, *.pem, *.key). This gate catches secret CONTENT inside + // eligible files — e.g. a hardcoded API key in a .ts file. Without + // this, the secret would flow through buildContextPrompt → _prompt.txt + // (under --debug) and to the LLM before the post-MDX secret scan + // catches it. The post-MDX scan is the last gate; this is the first. + for (const s of sources) { + const matches = scanForSecrets(s.content) + if (matches.length > 0) { + const msg = formatSecretError(matches, s.path) + process.stderr.write(msg + "\n") + throw contextError("E_CONTEXT_SECRET_SOURCE", { + cause: `Pattern "${matches[0]!.patternName}" matched in ${s.path}. No LLM call was made; nothing was written.`, + }) + } + } + + const outputDir = path.resolve(repoRoot, opts.output || ".helpbase") + const docsDir = path.join(outputDir, "docs") + const model = resolveModel({ test: opts.test, modelOverride: opts.model }) + const maxTokens = parseBudgetInt(opts.maxTokens, "--max-tokens", 100_000) + const charsPerToken = parseBudgetFloat(opts.charsPerToken, "--chars-per-token", 3.5) + + // ─── 5. Dry-run summary — no LLM call, no writes ──────────────── + if (opts.dryRun) { + const estimated = estimateTokens(sources, charsPerToken) + const preview = sources + .slice(0, 10) + .map((s) => ` ${s.path} ${formatChars(s.content.length)}`) + .join("\n") + const tail = + sources.length > 10 ? `\n ...and ${sources.length - 10} more` : "" + console.log("") + console.log(`${pc.dim("›")} ${pc.bold("Dry run — no LLM call, no writes")}`) + console.log(` Repo: ${pc.cyan(repoRoot)}`) + console.log(` Output: ${pc.cyan(outputDir)}`) + console.log(` Model: ${pc.cyan(model)}`) + console.log(` Sources found: ${sources.length}`) + console.log(` Total chars: ${formatChars(totalChars(sources))}`) + console.log(` Estimated tokens: ${estimated.toLocaleString()} (budget ${maxTokens.toLocaleString()})`) + console.log(` Sample sources:`) + console.log(preview + tail) + console.log("") + console.log(` ${pc.dim("Remove --dry-run to synthesize docs.")}`) + return + } + + // Project-name resolution: prefer package.json `name`, fall back to the + // repo directory basename. Used for the LLM prompt ("You are documenting + // X") and for llms.txt — so both surfaces agree. Without this, a repo in + // a temp dir like /tmp/xyz123 tells the LLM it's documenting "xyz123" + // even when package.json clearly says "todo-app". + const repoLabel = resolveProjectName(repoRoot) + + // ─── 6. Debug prompt dump (optional) ──────────────────────────── + if (opts.debug) { + const prompt = buildContextPrompt({ sources, repoLabel }) + fs.mkdirSync(outputDir, { recursive: true }) + const debugPath = path.join(outputDir, "_prompt.txt") + fs.writeFileSync(debugPath, prompt) + console.log(`${pc.dim("›")} Wrote prompt to ${pc.cyan(debugPath)}`) + } + + // ─── 6.5. Auth resolution — BYOK env var OR helpbase session ────── + const auth = await resolveAuthOrPromptLogin({ + verb: "ingest", + retryCommand: `helpbase ingest ${repoPathArg}${opts.ask ? ` --ask ${JSON.stringify(opts.ask)}` : ""}`, + }) + + // ─── 7. LLM synthesis ─────────────────────────────────────────── + console.log( + `${pc.dim("›")} Synthesizing how-tos from ${pc.bold(String(sources.length))} source files...`, + ) + const startLlm = Date.now() + const rawDocs = await generateHowtosFromRepo({ + sources, + repoLabel, + model, + maxTokens, + charsPerToken, + authToken: auth.authToken, + }) + const llmMs = Date.now() - startLlm + console.log( + ` ${pc.dim("›")} ${pc.bold(String(rawDocs.length))} draft docs in ${(llmMs / 1000).toFixed(1)}s`, + ) + + // ─── 8. Sanitize + validate citations ─────────────────────────── + const cache = createFileCache() + const report: SynthesisReport = { + startedAt: new Date().toISOString(), + repoRoot, + model, + tokensEstimated: estimateTokens(sources, charsPerToken), + durationMs: 0, + kept: [], + droppedDocs: [], + preservedFiles: [], + deletedFiles: [], + } + + const kept: GeneratedContextDoc[] = [] + for (const rawDoc of rawDocs) { + const sanitized: GeneratedContextDoc = { + ...rawDoc, + content: sanitizeMdx(rawDoc.content), + } + const v = validateArticleCitations(sanitized, repoRoot, cache) + if (v.kept.length === 0) { + report.droppedDocs.push({ + title: rawDoc.title, + reason: "all citations failed validation", + citations: v.dropped.map((d) => ({ + file: d.citation.file, + startLine: d.citation.startLine, + endLine: d.citation.endLine, + reason: d.reason, + })), + }) + continue + } + if (v.dropped.length > 0) { + report.droppedDocs.push({ + title: rawDoc.title, + reason: `partial: ${v.dropped.length} of ${sanitized.citations.length} citations dropped; doc kept with remaining ${v.kept.length}`, + citations: v.dropped.map((d) => ({ + file: d.citation.file, + startLine: d.citation.startLine, + endLine: d.citation.endLine, + reason: d.reason, + })), + }) + } + // v2 citations omit `snippet` — enrich from disk so MDX carries real + // bytes, not model-paraphrased text. Reuses the validator's file cache. + const enriched = enrichCitationsFromDisk(v.kept, repoRoot, cache) + kept.push({ ...sanitized, citations: enriched }) + report.kept.push({ + title: sanitized.title, + category: sanitized.category, + citationCount: enriched.length, + }) + } + + // ─── 9. Zero-doc guard ────────────────────────────────────────── + if (kept.length === 0) { + fs.mkdirSync(outputDir, { recursive: true }) + const reportPath = path.join(outputDir, "synthesis-report.json") + report.durationMs = Date.now() - startLlm + fs.writeFileSync(reportPath, JSON.stringify(report, null, 2)) + throw contextError("E_CONTEXT_NO_VALID_CITATIONS", { + cause: `${rawDocs.length} docs emitted, 0 passed citation validation. See ${reportPath} for per-doc drop reasons.`, + }) + } + + // ─── 10. Serialize to MDX strings ─────────────────────────────── + const serialized: Array<{ relPath: string; content: string; doc: GeneratedContextDoc }> = [] + kept.forEach((doc, idx) => { + const catSlug = slugify(doc.category) + const docSlug = slugify(doc.title) + const relPath = `${catSlug}/${docSlug}.mdx` + const content = articleToMdxWithCitations(doc, idx + 1) + serialized.push({ relPath, content, doc }) + }) + + // ─── 11. Pre-write secret scan (on EVERY final MDX string) ────── + for (const s of serialized) { + const matches = scanForSecrets(s.content) + if (matches.length > 0) { + const msg = formatSecretError(matches, s.relPath) + process.stderr.write(msg + "\n") + throw contextError("E_CONTEXT_SECRET", { + cause: `Pattern "${matches[0]!.patternName}" matched in ${s.relPath}. No files were written.`, + }) + } + } + + // ─── 12. Inventory + plan writes ──────────────────────────────── + const existing = inventoryExistingDocs(docsDir) + const planned = planContextWrites({ + newDocs: serialized.map(({ relPath, content }) => ({ relPath, content })), + docsDir, + existing, + onlyCategory: opts.only ? slugify(opts.only) : undefined, + }) + // --overwrite removes custom-file preservation. + if (opts.overwrite && planned.preserves.length > 0) { + // Don't silently clobber custom edits — require --yes as a second gate. + // Throw a structured HelpbaseError instead of stderr.write + process.exit + // so this path goes through the same formatting layer as every other + // user error (JSON/quiet honored, doc URL emitted, exit code via the + // top-level handler in index.ts). + if (!opts.yes) { + throw contextError("E_CONTEXT_REFUSE_CLOBBER", { + cause: `${planned.preserves.length} custom file(s) in ${path.relative(repoRoot, docsDir) || "."} would be clobbered without --yes.`, + }) + } + // Move preserves into writes so they get overwritten. + // Build the serialized-by-rel index so we only overwrite custom files + // that have a matching generated doc (otherwise nothing to overwrite with). + const byRel = new Map(serialized.map((s) => [path.join(docsDir, s.relPath), s.content])) + for (const p of planned.preserves) { + const replacement = byRel.get(p) + if (replacement) planned.writes.push({ absPath: p, content: replacement }) + } + } + report.preservedFiles = planned.preserves + report.deletedFiles = planned.deletes + + // ─── 13. Execute writes atomically ────────────────────────────── + const written: string[] = [] + try { + for (const w of planned.writes) { + atomicWriteFileSync(w.absPath, w.content) + written.push(w.absPath) + } + } catch (err) { + // Best-effort rollback: remove anything we wrote this run. + for (const p of written) { + try { + fs.rmSync(p, { force: true }) + } catch { + // best-effort only + } + } + throw err + } + // Deletes come AFTER successful writes so a failed write doesn't lose stale generated files. + for (const p of planned.deletes) { + try { + fs.rmSync(p, { force: true }) + } catch { + // best-effort — a stale generated file left behind is not worth aborting + } + } + + // ─── 14. .gitignore safety ────────────────────────────────────── + ensureGitignoreEntry(repoRoot, path.relative(repoRoot, outputDir)) + + // ─── 14.5. Emit llms.txt + llms-full.txt ──────────────────────── + emitLlmsTxt(repoRoot, docsDir, outputDir) + + // ─── 14.7. Emit mcp.json config hint ──────────────────────────── + emitMcpJsonHint(outputDir, docsDir) + + // ─── 15. Synthesis report ─────────────────────────────────────── + fs.mkdirSync(outputDir, { recursive: true }) + report.durationMs = Date.now() - startLlm + const reportPath = path.join(outputDir, "synthesis-report.json") + fs.writeFileSync(reportPath, JSON.stringify(report, null, 2)) + + console.log( + ` ${pc.dim("›")} Wrote ${pc.bold(String(planned.writes.length))} MDX files → ${pc.cyan(path.relative(repoRoot, docsDir))}`, + ) + if (planned.preserves.length > 0) { + console.log( + ` ${pc.dim("›")} Preserved ${planned.preserves.length} custom file(s)`, + ) + } + if (planned.deletes.length > 0) { + console.log( + ` ${pc.dim("›")} Deleted ${planned.deletes.length} stale generated file(s)`, + ) + } + + // ─── 16. --ask (local RAG, no MCP client required) ───────────── + if (opts.ask) { + await runLocalAsk(opts.ask, serialized, model, auth.authToken) + } else { + printNextSteps(repoPathArg || ".", outputDir) + } +} + +function printNextSteps(repoArg: string, outputDir: string): void { + console.log("") + console.log(pc.bold("Next:")) + console.log( + ` ${pc.dim("1. Open it in a browser:")} ${pc.cyan("helpbase preview")}`, + ) + console.log( + ` ${pc.dim("2. Try it in-terminal:")} ${pc.cyan(`helpbase ingest ${repoArg} --ask "how do I log in?"`)}`, + ) + console.log(` ${pc.dim("3. Share with an MCP client — copy a block from:")}`) + console.log(` ${pc.cyan(path.join(outputDir, "mcp.json"))}`) + console.log(` ${pc.dim("into your client's config file:")}`) + for (const p of mcpClientConfigPaths()) { + console.log(` ${pc.dim("·")} ${p.client}: ${pc.cyan(p.path)}`) + } + console.log( + ` ${pc.dim("4. Or browse")} ${pc.cyan(path.join(outputDir, "llms.txt"))} / ${pc.cyan(path.join(outputDir, "llms-full.txt"))} ${pc.dim("— standard llms.txt format")}`, + ) + console.log("") +} + +// ── Helpers ────────────────────────────────────────────────────────── + +function checkDirtyTree(repoRoot: string, opts: IngestOpts): void { + let dirtyLines = "" + try { + dirtyLines = execSync("git status --porcelain", { + cwd: repoRoot, + encoding: "utf8", + stdio: ["ignore", "pipe", "ignore"], + }).trim() + } catch { + // Not a git repo — silently skip the check. + return + } + if (!dirtyLines) return + const count = dirtyLines.split("\n").length + if (opts.requireClean) { + throw contextError("E_CONTEXT_DIRTY_TREE", { + cause: `${count} uncommitted file(s) detected.`, + }) + } + // Default + --allow-dirty: warn and continue. + process.stderr.write( + `${pc.yellow("!")} Working tree has ${count} uncommitted change(s). Continuing — pass --require-clean for CI-mode.\n`, + ) +} + +/** + * Resolve the project's display name. + * + * Preferred: `package.json` `name` field. Fallback: the repo directory's + * basename. A repo cloned into a temp path shouldn't leak "tmp-ax89z" into + * user-visible output — package.json is the user's intent, the path is an + * implementation detail. + */ +export function resolveProjectName(repoRoot: string): string { + const pkgPath = path.join(repoRoot, "package.json") + if (fs.existsSync(pkgPath)) { + try { + const pkg = JSON.parse(fs.readFileSync(pkgPath, "utf8")) + if (typeof pkg.name === "string" && pkg.name.trim().length > 0) { + return pkg.name.trim() + } + } catch { + // fall through to the basename + } + } + return path.basename(repoRoot) +} + +function emitLlmsTxt(repoRoot: string, docsDir: string, outputDir: string): void { + const projectName = resolveProjectName(repoRoot) + let projectSummary: string | undefined + let siteUrl: string | undefined = process.env.HELPBASE_SITE_URL + const pkgPath = path.join(repoRoot, "package.json") + if (fs.existsSync(pkgPath)) { + try { + const pkg = JSON.parse(fs.readFileSync(pkgPath, "utf8")) + if (typeof pkg.description === "string" && pkg.description) + projectSummary = pkg.description + if (!siteUrl && typeof pkg.homepage === "string" && pkg.homepage) + siteUrl = pkg.homepage + } catch { + // fall back to defaults + } + } + if (!projectSummary) { + projectSummary = `Docs for ${projectName}, generated by helpbase ingest.` + } + + const { llmsTxt, llmsFullTxt, docCount, fullBytes } = generateLlmsTxt({ + contentDir: docsDir, + projectName, + projectSummary, + siteUrl, + }) + + fs.writeFileSync(path.join(outputDir, "llms.txt"), llmsTxt, "utf-8") + fs.writeFileSync(path.join(outputDir, "llms-full.txt"), llmsFullTxt, "utf-8") + + if (!siteUrl) { + process.stderr.write( + `${pc.dim("›")} HELPBASE_SITE_URL not set; llms.txt uses relative URLs. Set the env var or add "homepage" to package.json for absolute links.\n`, + ) + } + if (fullBytes > LLMS_FULL_MAX_BYTES) { + process.stderr.write( + `${pc.yellow("!")} llms-full.txt is ${(fullBytes / 1024 / 1024).toFixed(1)}MB (over ${LLMS_FULL_MAX_BYTES / 1024 / 1024}MB sanity cap).\n`, + ) + } + // Intentionally silent on the happy path — the next-steps block + // surfaces the paths; no need for a per-file console log. + void docCount +} + +function emitMcpJsonHint(outputDir: string, docsDir: string): void { + // The mcp.json file is NOT a config the MCP server consumes — it's a + // hint for the user. They copy one of the blocks into their client's + // config file (paths listed after this function runs). + // + // HELPBASE_CONTENT_DIR = absolute path to the generated docs dir. Uses + // path.resolve(opts.output, "docs") so --output overrides work. + const mcpServerBlock = { + command: "npx", + args: ["@helpbase/mcp@latest"], + env: { + HELPBASE_CONTENT_DIR: docsDir, + }, + } + const hint = { + _comment: + "This file is a HINT — copy one of the blocks below into your MCP client's config. Do NOT commit: HELPBASE_CONTENT_DIR is an absolute machine-local path.", + helpbaseMcpVersion: "1", + claude_desktop: { mcpServers: { helpbase: mcpServerBlock } }, + cursor: { mcpServers: { helpbase: mcpServerBlock } }, + claude_code: { mcpServers: { helpbase: mcpServerBlock } }, + } + fs.writeFileSync( + path.join(outputDir, "mcp.json"), + JSON.stringify(hint, null, 2) + "\n", + "utf8", + ) +} + +function mcpClientConfigPaths(): Array<{ client: string; path: string }> { + const home = process.env.HOME ?? process.env.USERPROFILE ?? "~" + const isWindows = process.platform === "win32" + if (isWindows) { + const appData = process.env.APPDATA ?? `${home}\\AppData\\Roaming` + return [ + { client: "Claude Desktop", path: `${appData}\\Claude\\claude_desktop_config.json` }, + { client: "Cursor", path: `${home}\\.cursor\\mcp.json` }, + { client: "Claude Code", path: `${home}\\.claude\\mcp.json` }, + ] + } + const isMac = process.platform === "darwin" + const claudeDesktop = isMac + ? `${home}/Library/Application Support/Claude/claude_desktop_config.json` + : `${home}/.config/Claude/claude_desktop_config.json` + return [ + { client: "Claude Desktop", path: claudeDesktop }, + { client: "Cursor", path: `${home}/.cursor/mcp.json` }, + { client: "Claude Code", path: `${home}/.claude/mcp.json` }, + ] +} + +/** + * Parse --max-tokens. Accepts 0 (disables the gate, per the flag's docs) and + * positive integers; rejects NaN, negative, and non-numeric. Previously + * `Number.parseInt(x) || default` silently swallowed garbage input — including + * negative values that then bypassed the `maxTokens > 0` gate check inside + * generateHowtosFromRepo, letting oversized runs hit the LLM gateway. + * + * @param raw the raw flag value (Commander always passes a string + * because we declared --max-tokens <n>) + * @param flagName user-facing flag label for the error message + * @param fallbackDefault returned when `raw` itself is empty/undefined (unused + * in normal Commander flow — Commander fills in the + * registered default — but kept for direct callers) + */ +function parseBudgetInt(raw: string, flagName: string, fallbackDefault: number): number { + if (raw === undefined || raw === null || raw === "") return fallbackDefault + const parsed = Number.parseInt(raw, 10) + if (!Number.isFinite(parsed) || parsed < 0 || String(parsed) !== raw.trim()) { + throw contextError("E_CONTEXT_INVALID_BUDGET", { + cause: `${flagName} received ${JSON.stringify(raw)} — expected a non-negative integer (0 disables the gate).`, + }) + } + return parsed +} + +/** + * Parse --chars-per-token. Must be a strictly positive finite float. Unlike + * --max-tokens, 0 is NOT a valid value here — a zero chars-per-token would + * make the token estimate trivially 0 and bypass the budget gate regardless + * of the --max-tokens value. + */ +function parseBudgetFloat(raw: string, flagName: string, fallbackDefault: number): number { + if (raw === undefined || raw === null || raw === "") return fallbackDefault + const parsed = Number.parseFloat(raw) + if (!Number.isFinite(parsed) || parsed <= 0) { + throw contextError("E_CONTEXT_INVALID_BUDGET", { + cause: `${flagName} received ${JSON.stringify(raw)} — expected a positive number (e.g. 3.5).`, + }) + } + return parsed +} + +function formatChars(n: number): string { + if (n < 1000) return `${n}b` + if (n < 1_000_000) return `${(n / 1000).toFixed(1)}Kb` + return `${(n / 1_000_000).toFixed(1)}Mb` +} + +interface SynthesisReport { + startedAt: string + repoRoot: string + model: string + tokensEstimated: number + durationMs: number + kept: Array<{ title: string; category: string; citationCount: number }> + droppedDocs: Array<{ + title: string + reason: string + citations: Array<{ file: string; startLine: number; endLine: number; reason: string }> + }> + preservedFiles: string[] + deletedFiles: string[] +} + +// ── --reuse-existing: load docs from disk into runLocalAsk shape ───── + +/** + * Walk `.helpbase/docs/` for existing MDX files and reconstruct the + * `serialized`-shape array that `runLocalAsk` consumes. Used by the + * `--reuse-existing --ask` fast path so the eval runner can do one + * expensive generation and many cheap question passes instead of N+1 + * full regens. + */ +function loadExistingDocs( + outputDir: string, +): Array<{ relPath: string; content: string; doc: GeneratedContextDoc }> { + const docsDir = path.join(outputDir, "docs") + if (!fs.existsSync(docsDir)) { + throw contextError("E_CONTEXT_REUSE_EMPTY", { + cause: `No directory at ${docsDir}.`, + }) + } + const results: Array<{ relPath: string; content: string; doc: GeneratedContextDoc }> = [] + // docs/<category>/<slug>.mdx — two-level walk, not recursive globbing. + for (const categoryDir of fs.readdirSync(docsDir, { withFileTypes: true })) { + if (!categoryDir.isDirectory()) continue + const catPath = path.join(docsDir, categoryDir.name) + for (const entry of fs.readdirSync(catPath, { withFileTypes: true })) { + if (!entry.isFile() || !entry.name.endsWith(".mdx")) continue + const full = path.join(catPath, entry.name) + const content = fs.readFileSync(full, "utf8") + const parsed = matter(content) + // Minimal reconstruction — runLocalAsk uses title from doc, body + // from parsed content. Extra fields kept so the shape is valid. + const doc: GeneratedContextDoc = { + title: String(parsed.data.title ?? entry.name), + description: String(parsed.data.description ?? ""), + category: categoryDir.name, + tags: Array.isArray(parsed.data.tags) ? parsed.data.tags : [], + content: parsed.content, + citations: Array.isArray(parsed.data.citations) ? parsed.data.citations : [], + sourcePaths: [], + } + results.push({ + relPath: path.join(categoryDir.name, entry.name), + content, + doc, + }) + } + } + if (results.length === 0) { + throw contextError("E_CONTEXT_REUSE_EMPTY") + } + return results +} + +// ── Local --ask (the magical-moment fix) ───────────────────────────── + +async function runLocalAsk( + question: string, + serialized: Array<{ relPath: string; content: string; doc: GeneratedContextDoc }>, + model: string, + authToken?: string, +): Promise<void> { + // Ask against just the generated body (not frontmatter), since the model + // doesn't need to see YAML. Pass the slug path so it can cite correctly. + const docs = serialized.map((s) => { + const parsed = matter(s.content) + return { + title: s.doc.title, + path: s.relPath, + body: parsed.content, + } + }) + const prompt = buildLocalAskPrompt({ question, docs }) + console.log("") + console.log(`${pc.dim("›")} Answering: ${pc.bold(JSON.stringify(question))}`) + const started = Date.now() + let text: string + let quotaForWarning: ReturnType<typeof formatQuotaSuffix> | null = null + try { + const result = await callLlmText({ model, prompt, authToken }) + text = result.text + if (result.quota) { + const pct = Math.round((result.quota.usedToday / result.quota.dailyLimit) * 100) + if (pct >= 80) { + quotaForWarning = formatQuotaSuffix(result.quota) + } + } + } catch (err) { + throw toCliLlmError(err, { + retryCommand: `helpbase ingest --ask ${JSON.stringify(question)}`, + }) + } + const ms = Date.now() - started + console.log("") + console.log(text) + console.log("") + console.log(` ${pc.dim(`answered in ${(ms / 1000).toFixed(1)}s using ${model}`)}`) + if (quotaForWarning) { + console.log(` ${pc.yellow("heads-up:")} ${quotaForWarning}`) + } +} diff --git a/packages/cli/src/commands/preview.ts b/packages/cli/src/commands/preview.ts index 4b748e1..7101906 100644 --- a/packages/cli/src/commands/preview.ts +++ b/packages/cli/src/commands/preview.ts @@ -10,7 +10,7 @@ import { contextError } from "./context-errors.js" /** * `helpbase preview` — materialize a human-facing help center from the - * `.helpbase/docs/` `helpbase context` just produced. + * `.helpbase/docs/` `helpbase ingest` just produced. * * The generated MDX is not a website on its own. This command bridges it * by caching a full Next.js renderer at `~/.helpbase/preview-<version>/`, @@ -44,7 +44,7 @@ interface PreviewOpts { export const previewCommand = new Command("preview") .description( - "Open a browser-viewable help center from the docs `helpbase context` generated.", + "Open a browser-viewable help center from the docs `helpbase ingest` generated.", ) .option("-p, --port <port>", "Port to serve on", "3000") .option("--reset", "Wipe the cached preview renderer and re-scaffold") @@ -61,7 +61,7 @@ Examples: First run takes ~45-60s (scaffold + install). Every run after is ~3s. Pair with: - $ helpbase context . # generate the docs first + $ helpbase ingest . # generate the docs first `, ) .action(async (opts: PreviewOpts) => { diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index e1c5f8c..22dff77 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -7,6 +7,7 @@ import { fileURLToPath } from "node:url" import { dirname, join } from "node:path" import { devCommand } from "./commands/dev.js" import { generateCommand } from "./commands/generate.js" +import { ingestCommand } from "./commands/ingest.js" import { contextCommand } from "./commands/context.js" import { previewCommand } from "./commands/preview.js" import { syncCommand } from "./commands/sync.js" @@ -73,6 +74,12 @@ program.helpInformation = () => renderGroupedHelp(program) program.addCommand(devCommand) program.addCommand(generateCommand) +program.addCommand(ingestCommand) +// Deprecated alias for `helpbase ingest`. Slated for removal in v0.7. +// TODO(v0.7): delete this addCommand + the context.ts file + the +// "helpbase context (deprecated alias)" describe block in +// packages/cli/test/ingest.test.ts together. `grep -rn 'v0.7'` should +// surface all deletion sites. program.addCommand(contextCommand) program.addCommand(previewCommand) program.addCommand(syncCommand) @@ -95,15 +102,26 @@ program.addCommand(upgradeCommand) // opted in. Records command name, duration, exit code, and flag names // (not values). Never slows the CLI — 2s timeout, swallows errors. const startedAt = Date.now() +// Map of deprecated aliases to their canonical command name. Keep telemetry +// dashboards continuous across renames: a user running `helpbase context foo` +// reports `command: "ingest"` + `alias: "context"` — total invocations of +// the pipeline stay accurate, and alias-adoption rate stays measurable. +// TODO(v0.7): remove the "context" entry when the shim is deleted. +const DEPRECATED_ALIASES: Record<string, string> = { + context: "ingest", +} program.hook("postAction", (_thisCommand, actionCommand) => { - const cmdName = actionCommand.name() + const invokedName = actionCommand.name() + const canonicalName = DEPRECATED_ALIASES[invokedName] ?? invokedName + const alias = canonicalName === invokedName ? undefined : invokedName const flags = actionCommand.opts() const flagNames = Object.keys(flags).filter( (k) => flags[k] !== undefined && flags[k] !== false, ) sendEvent( { - command: cmdName, + command: canonicalName, + ...(alias ? { alias } : {}), durationMs: Date.now() - startedAt, exitCode: 0, flags: flagNames, diff --git a/packages/cli/src/lib/errors.ts b/packages/cli/src/lib/errors.ts index 0ee06d3..de049db 100644 --- a/packages/cli/src/lib/errors.ts +++ b/packages/cli/src/lib/errors.ts @@ -48,6 +48,9 @@ export type ErrorCode = | "E_CONTEXT_PREVIEW_NO_DOCS" | "E_CONTEXT_PREVIEW_SCAFFOLD" | "E_CONTEXT_PREVIEW_INSTALL" + | "E_CONTEXT_INVALID_BUDGET" + | "E_CONTEXT_REFUSE_CLOBBER" + | "E_CONTEXT_SECRET_SOURCE" | "E_AUTH_REQUIRED" | "E_QUOTA_EXCEEDED" | "E_GLOBAL_CAP" diff --git a/packages/cli/src/lib/help.ts b/packages/cli/src/lib/help.ts index 6f2364a..bb4a697 100644 --- a/packages/cli/src/lib/help.ts +++ b/packages/cli/src/lib/help.ts @@ -28,14 +28,18 @@ interface GroupDef { } const GROUPS: GroupDef[] = [ - { label: "Get started", commands: ["context", "preview", "new", "dev"] }, + { label: "Get started", commands: ["ingest", "preview", "new", "dev"] }, { label: "Ship", commands: ["deploy", "link", "open"] }, { label: "Author", commands: ["generate", "sync", "mcp", "audit"] }, { label: "Account", commands: ["login", "logout", "whoami", "config"] }, { label: "Diagnose", commands: ["doctor", "feedback", "completion", "upgrade"] }, ] -const MOST_COMMON = ["helpbase context", "helpbase new", "helpbase dev"] +const MOST_COMMON = ["helpbase ingest", "helpbase new", "helpbase dev"] + +// `context` is intentionally NOT listed in GROUPS above — it's deprecated and +// falls into the auto-generated "Other" group at the bottom of --help. Users +// running old scripts still see it; new users aren't guided toward it. /** * Render grouped help text for the top-level program. Returns a string diff --git a/packages/cli/src/lib/telemetry.ts b/packages/cli/src/lib/telemetry.ts index a076e98..5bf4cd9 100644 --- a/packages/cli/src/lib/telemetry.ts +++ b/packages/cli/src/lib/telemetry.ts @@ -28,6 +28,15 @@ export interface TelemetryEvent { exitCode: number /** Flag names actually used (e.g. "--slug", "--yes"). Never values. */ flags: string[] + /** + * If the user invoked a deprecated alias (e.g. `helpbase context` for the + * `ingest` command), `command` is the canonical name and `alias` is what + * they actually typed. Preserves metric continuity across renames while + * still surfacing the alias-adoption rate. Optional so non-alias calls + * stay the same shape. Server-side consumers that don't know this field + * can ignore it safely. + */ + alias?: string } export function isTelemetryEnabled(): boolean { @@ -65,6 +74,9 @@ export function sendEvent(event: TelemetryEvent, cliVersion: string): void { nodeVersion: process.version, platform: process.platform, arch: process.arch, + // Only present when the user invoked via a deprecated alias. Lets + // dashboards track rename-migration progress without double-counting. + ...(event.alias ? { alias: event.alias } : {}), } // 2 second timeout — telemetry must never slow users down. diff --git a/packages/cli/test/context.test.ts b/packages/cli/test/context.test.ts deleted file mode 100644 index 3885d21..0000000 --- a/packages/cli/test/context.test.ts +++ /dev/null @@ -1,238 +0,0 @@ -import { describe, it, expect } from "vitest" -import path from "node:path" -import fs from "node:fs" -import os from "node:os" -import { execSync } from "node:child_process" - -const CLI_PATH = path.resolve(__dirname, "../dist/index.js") - -function run(args: string): { output: string; exitCode: number } { - return runWithEnv(args, {}) -} - -function runWithEnv( - args: string, - envOverrides: Record<string, string>, -): { output: string; exitCode: number } { - // Merge stdout + stderr on both success and failure — context writes - // diagnostic notices + error catalog output to stderr, and tests want to - // assert on the full user-visible output regardless of exit code. - try { - const stdout = execSync(`node ${CLI_PATH} ${args} 2>&1`, { - encoding: "utf-8", - stdio: ["pipe", "pipe", "pipe"], - env: { ...process.env, NO_COLOR: "1", ...envOverrides }, - }) - return { output: stdout, exitCode: 0 } - } catch (err: any) { - const output = (err.stdout ?? "") + (err.stderr ?? "") - return { output, exitCode: err.status ?? 1 } - } -} - -describe("helpbase context", () => { - it("advertises itself in --help", () => { - const result = run("context --help") - expect(result.exitCode).toBe(0) - expect(result.output).toContain("context") - expect(result.output).toContain("--max-tokens") - expect(result.output).toContain("--ask") - expect(result.output).toContain("--only") - expect(result.output).toContain("--prompt") - expect(result.output).toContain("--require-clean") - }) - - it("documents BYOK env vars in help examples", () => { - const result = run("context --help") - expect(result.output).toContain("ANTHROPIC_API_KEY") - expect(result.output).toContain("OPENAI_API_KEY") - expect(result.output).toContain("AI_GATEWAY_API_KEY") - }) - - it("--dry-run on a repo with no eligible files errors with E_CONTEXT_NO_SOURCES", () => { - // Empty temp dir has nothing to walk → helpful error, not a crash. - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "helpbase-ctx-empty-")) - try { - const result = run(`context ${tmp} --dry-run`) - // No sources available; even --dry-run goes through the walker. - expect(result.exitCode).toBe(1) - expect(result.output).toContain("E_CONTEXT_NO_SOURCES") - } finally { - fs.rmSync(tmp, { recursive: true, force: true }) - } - }) - - it("no auth + no BYOK on a real repo surfaces E_AUTH_REQUIRED", () => { - // Create a minimal repo the walker likes. - // With the hosted-proxy default, running context without a helpbase session - // AND without AI_GATEWAY_API_KEY in non-TTY mode returns E_AUTH_REQUIRED - // (the inline-login prompt can only fire on a real TTY). Interactive - // users on a TTY get the login prompt instead. - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "helpbase-ctx-nokey-")) - try { - fs.writeFileSync(path.join(tmp, "README.md"), "# Hello\n\nsample repo for test") - const result = runWithEnv(`context ${tmp}`, { - AI_GATEWAY_API_KEY: "", - ANTHROPIC_API_KEY: "", - OPENAI_API_KEY: "", - HELPBASE_TOKEN: "", - }) - expect(result.exitCode).toBe(1) - expect(result.output).toContain("E_AUTH_REQUIRED") - } finally { - fs.rmSync(tmp, { recursive: true, force: true }) - } - }) - - it("invalid repo path surfaces E_CONTEXT_REPO_PATH", () => { - const result = run("context /tmp/helpbase-does-not-exist-ctx-test") - expect(result.exitCode).toBe(1) - expect(result.output).toContain("E_CONTEXT_REPO_PATH") - }) - - it("--dry-run on a real repo prints a plan without an LLM call", () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "helpbase-ctx-dry-")) - try { - fs.writeFileSync(path.join(tmp, "README.md"), "# Hello\n\nsample repo for test") - fs.writeFileSync(path.join(tmp, "index.ts"), "export const one = 1") - // Dry-run does not need a key. - const result = runWithEnv(`context ${tmp} --dry-run`, { - AI_GATEWAY_API_KEY: "", - ANTHROPIC_API_KEY: "", - OPENAI_API_KEY: "", - }) - expect(result.exitCode).toBe(0) - expect(result.output).toContain("Dry run") - expect(result.output).toContain("Sources found:") - expect(result.output).toContain("Estimated tokens:") - } finally { - fs.rmSync(tmp, { recursive: true, force: true }) - } - }) - - it("--reuse-existing without --ask surfaces E_CONTEXT_REUSE_WITHOUT_ASK", () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "helpbase-ctx-reuse-")) - try { - fs.writeFileSync(path.join(tmp, "README.md"), "# Hello\n\nsample") - const result = run(`context ${tmp} --reuse-existing`) - expect(result.exitCode).toBe(1) - expect(result.output).toContain("E_CONTEXT_REUSE_WITHOUT_ASK") - } finally { - fs.rmSync(tmp, { recursive: true, force: true }) - } - }) - - it("--reuse-existing --ask with empty .helpbase surfaces E_CONTEXT_REUSE_EMPTY", () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "helpbase-ctx-reuse-")) - try { - fs.writeFileSync(path.join(tmp, "README.md"), "# Hello\n\nsample") - // No .helpbase/docs directory exists — reuse should fail loudly. - const result = run(`context ${tmp} --reuse-existing --ask "anything"`) - expect(result.exitCode).toBe(1) - expect(result.output).toContain("E_CONTEXT_REUSE_EMPTY") - } finally { - fs.rmSync(tmp, { recursive: true, force: true }) - } - }) - - it("resolves project name from package.json, not the containing directory (regression)", async () => { - // Temp dirs have names like `helpbase-user-test-znh2Rp`. Without this - // fix, that string ended up in the LLM prompt and in llms.txt as the - // project's display name — drowning out the `name: "todo-app"` the - // user set in their package.json. - const { resolveProjectName } = await import("../src/commands/context.js") - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "helpbase-name-regression-")) - try { - fs.writeFileSync( - path.join(tmp, "package.json"), - JSON.stringify({ name: "todo-app", version: "0.1.0" }), - ) - expect(resolveProjectName(tmp)).toBe("todo-app") - } finally { - fs.rmSync(tmp, { recursive: true, force: true }) - } - }) - - it("falls back to the directory basename when package.json has no name", async () => { - const { resolveProjectName } = await import("../src/commands/context.js") - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "helpbase-noname-")) - try { - fs.writeFileSync(path.join(tmp, "package.json"), JSON.stringify({ version: "0.1.0" })) - expect(resolveProjectName(tmp)).toBe(path.basename(tmp)) - } finally { - fs.rmSync(tmp, { recursive: true, force: true }) - } - }) - - it("falls back to the directory basename when there is no package.json at all", async () => { - const { resolveProjectName } = await import("../src/commands/context.js") - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "helpbase-nopkg-")) - try { - expect(resolveProjectName(tmp)).toBe(path.basename(tmp)) - } finally { - fs.rmSync(tmp, { recursive: true, force: true }) - } - }) - - it("--reuse-existing --ask with a populated .helpbase/docs reaches the LLM call (fast path, no walk)", () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "helpbase-ctx-reuse-")) - try { - // Seed a pre-generated doc on disk — the fast path reads these. - const docsDir = path.join(tmp, ".helpbase", "docs", "getting-started") - fs.mkdirSync(docsDir, { recursive: true }) - fs.writeFileSync( - path.join(docsDir, "how-to-log-in.mdx"), - [ - "---", - "schemaVersion: 1", - 'title: "How to log in"', - 'description: "Log in with a magic link."', - "tags: []", - "order: 1", - "---", - "", - "Call POST /api/auth/login with email and password.", - "", - ].join("\n"), - ) - // No AI key and no source walk — the fast path goes straight to - // runLocalAsk, which needs a key. Observable behavior: we get past - // the walker (no NO_SOURCES, no REPO_PATH error) and fail at the - // LLM call with a gateway/key error, NOT at the pipeline gate. - const result = runWithEnv( - `context ${tmp} --reuse-existing --ask "how do I log in?"`, - { - AI_GATEWAY_API_KEY: "vck_this_is_not_a_real_key_abcdefghijklmnop", - }, - ) - // The run should fail at the LLM boundary, not the reuse gate. - expect(result.output).not.toContain("E_CONTEXT_REUSE_EMPTY") - expect(result.output).not.toContain("E_CONTEXT_REUSE_WITHOUT_ASK") - expect(result.output).not.toContain("E_CONTEXT_NO_SOURCES") - // The Answering line proves we reached runLocalAsk's prompt stage. - expect(result.output).toContain("Answering:") - } finally { - fs.rmSync(tmp, { recursive: true, force: true }) - } - }) -}) - -describe("helpbase generate positional alias", () => { - it("accepts a positional path as an alias for --repo", () => { - // With neither --url, --repo, nor --screenshots, and no positional, - // generate must error. With a positional path but no --repo flag, the - // alias kicks in and the command proceeds to path-existence checks. - // We can observe the alias indirectly: a bogus path yields a repo - // error, not the "Provide a source" error. - const result = run("generate /tmp/definitely-does-not-exist-helpbase-alias-test") - expect(result.exitCode).toBe(1) - // The alias should have mapped to --repo; the error is about the repo path, not missing source. - expect(result.output).not.toContain("Provide a source") - }) - - it("still shows the source error when no positional and no flags", () => { - const result = run("generate") - expect(result.exitCode).toBe(1) - expect(result.output).toContain("Provide a source") - }) -}) diff --git a/packages/cli/test/doctor.test.ts b/packages/cli/test/doctor.test.ts index d60abaf..9266943 100644 --- a/packages/cli/test/doctor.test.ts +++ b/packages/cli/test/doctor.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect } from "vitest" +import { describe, it, expect, beforeAll, afterAll } from "vitest" import path from "node:path" import { execSync } from "node:child_process" import fs from "node:fs" @@ -6,12 +6,30 @@ import os from "node:os" const CLI_PATH = path.resolve(__dirname, "../dist/index.js") +// Isolated HOME so these tests don't consume the developer's real +// `~/.helpbase/auth.json`. Without this, the "fix: helpbase login" assertion +// flakes on any contributor laptop where `helpbase login` has been run: +// doctor reports "Logged in as ..." instead of emitting the login fix line. +let FAKE_HOME: string +beforeAll(() => { + FAKE_HOME = fs.mkdtempSync(path.join(os.tmpdir(), "helpbase-doctor-home-")) +}) +afterAll(() => { + fs.rmSync(FAKE_HOME, { recursive: true, force: true }) +}) + function run(args: string, cwd: string): string { // doctor prints to stdout (text) or via JSON. Capture both streams. return execSync(`node ${CLI_PATH} ${args}`, { encoding: "utf-8", cwd, - env: { ...process.env, NO_COLOR: "1" }, + env: { + ...process.env, + NO_COLOR: "1", + HOME: FAKE_HOME, + USERPROFILE: FAKE_HOME, + HELPBASE_TOKEN: "", + }, }) } diff --git a/packages/cli/test/help.test.ts b/packages/cli/test/help.test.ts index ca93ebe..0bc4e81 100644 --- a/packages/cli/test/help.test.ts +++ b/packages/cli/test/help.test.ts @@ -36,19 +36,34 @@ describe("helpbase --help", () => { expect(output).toContain("Diagnose") }) - it("surfaces the three canonical commands up top (context leads — flagship AI-native flow)", () => { + it("surfaces the three canonical commands up top (ingest leads — flagship AI-native flow)", () => { const output = runNoColor("--help") - expect(output).toMatch(/Most common:[\s\S]*helpbase context/) + expect(output).toMatch(/Most common:[\s\S]*helpbase ingest/) expect(output).toMatch(/Most common:[\s\S]*helpbase new/) expect(output).toMatch(/Most common:[\s\S]*helpbase dev/) }) - it("lists context under Get started, not Other", () => { + it("lists ingest under Get started (deprecated context falls into Other)", () => { const output = runNoColor("--help") - // Extract the "Get started" block — ends at the next group heading - // ("Ship") or EOL. - const getStarted = output.match(/Get started[\s\S]*?(?=Ship|$)/)?.[0] ?? "" - expect(getStarted).toContain("context") + // Extract the "Get started" block — anchor to the next group heading + // ("Ship") so a future group reorder can't make the regex swallow the + // Other section and flake the assertion below. + const getStartedMatch = output.match(/Get started\s*\n([\s\S]*?)\n\s*Ship/) + const getStarted = getStartedMatch?.[1] ?? "" + expect(getStarted).toContain("ingest") + // context still registered (deprecation shim) but NOT in the curated groups. + expect(getStarted).not.toContain("context") + }) + + it("lists deprecated context under Other so users still discover it", () => { + const output = runNoColor("--help") + // Other is the auto-generated group for any command not in GROUPS. + // Anchored between "Other" and the closing "Run `helpbase" help text + // so a future group addition doesn't break the capture. + const otherMatch = output.match(/Other\s*\n([\s\S]*?)\n\s*Run /) + const other = otherMatch?.[1] ?? "" + expect(other).toContain("context") + expect(other).toContain("deprecated") }) it("advertises --json and --quiet as global options", () => { diff --git a/packages/cli/test/ingest.test.ts b/packages/cli/test/ingest.test.ts new file mode 100644 index 0000000..4358436 --- /dev/null +++ b/packages/cli/test/ingest.test.ts @@ -0,0 +1,425 @@ +import { describe, it, expect, beforeAll, afterAll } from "vitest" +import path from "node:path" +import fs from "node:fs" +import os from "node:os" +import { execSync } from "node:child_process" + +const CLI_PATH = path.resolve(__dirname, "../dist/index.js") + +// Isolated HOME + HELPBASE_* scrub so tests never consume the developer's +// real `~/.helpbase/auth.json`. Without this, tests asserting "E_AUTH_REQUIRED" +// pass in CI (clean HOME) but fail on any contributor laptop that has run +// `helpbase login` — a stealth paid-LLM call path. Caught by /review: +// ingest.test.ts hermeticity finding, 2026-04-18. +let FAKE_HOME: string +beforeAll(() => { + FAKE_HOME = fs.mkdtempSync(path.join(os.tmpdir(), "helpbase-test-home-")) +}) +afterAll(() => { + fs.rmSync(FAKE_HOME, { recursive: true, force: true }) +}) + +function run(args: string): { output: string; exitCode: number } { + return runWithEnv(args, {}) +} + +function runWithEnv( + args: string, + envOverrides: Record<string, string>, +): { output: string; exitCode: number } { + // Merge stdout + stderr on both success and failure — ingest writes + // diagnostic notices + error catalog output to stderr, and tests want to + // assert on the full user-visible output regardless of exit code. + try { + const stdout = execSync(`node ${CLI_PATH} ${args} 2>&1`, { + encoding: "utf-8", + stdio: ["pipe", "pipe", "pipe"], + env: { + ...process.env, + NO_COLOR: "1", + // Hermetic auth isolation — see beforeAll note above. + HOME: FAKE_HOME, + USERPROFILE: FAKE_HOME, + HELPBASE_TOKEN: "", + ...envOverrides, + }, + }) + return { output: stdout, exitCode: 0 } + } catch (err: any) { + const output = (err.stdout ?? "") + (err.stderr ?? "") + return { output, exitCode: err.status ?? 1 } + } +} + +describe("helpbase ingest", () => { + it("advertises itself in --help", () => { + const result = run("ingest --help") + expect(result.exitCode).toBe(0) + expect(result.output).toContain("ingest") + expect(result.output).toContain("--max-tokens") + expect(result.output).toContain("--ask") + expect(result.output).toContain("--only") + expect(result.output).toContain("--prompt") + expect(result.output).toContain("--require-clean") + }) + + it("documents BYOK env vars in help examples", () => { + const result = run("ingest --help") + expect(result.output).toContain("ANTHROPIC_API_KEY") + expect(result.output).toContain("OPENAI_API_KEY") + expect(result.output).toContain("AI_GATEWAY_API_KEY") + }) + + it("--dry-run on a repo with no eligible files errors with E_CONTEXT_NO_SOURCES", () => { + // Empty temp dir has nothing to walk → helpful error, not a crash. + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "helpbase-ing-empty-")) + try { + const result = run(`ingest ${tmp} --dry-run`) + // No sources available; even --dry-run goes through the walker. + expect(result.exitCode).toBe(1) + expect(result.output).toContain("E_CONTEXT_NO_SOURCES") + } finally { + fs.rmSync(tmp, { recursive: true, force: true }) + } + }) + + it("no auth + no BYOK on a real repo surfaces E_AUTH_REQUIRED", () => { + // Create a minimal repo the walker likes. + // With the hosted-proxy default, running ingest without a helpbase session + // AND without AI_GATEWAY_API_KEY in non-TTY mode returns E_AUTH_REQUIRED + // (the inline-login prompt can only fire on a real TTY). Interactive + // users on a TTY get the login prompt instead. + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "helpbase-ing-nokey-")) + try { + fs.writeFileSync(path.join(tmp, "README.md"), "# Hello\n\nsample repo for test") + const result = runWithEnv(`ingest ${tmp}`, { + AI_GATEWAY_API_KEY: "", + ANTHROPIC_API_KEY: "", + OPENAI_API_KEY: "", + HELPBASE_TOKEN: "", + }) + expect(result.exitCode).toBe(1) + expect(result.output).toContain("E_AUTH_REQUIRED") + } finally { + fs.rmSync(tmp, { recursive: true, force: true }) + } + }) + + it("invalid repo path surfaces E_CONTEXT_REPO_PATH", () => { + const result = run("ingest /tmp/helpbase-does-not-exist-ing-test") + expect(result.exitCode).toBe(1) + expect(result.output).toContain("E_CONTEXT_REPO_PATH") + }) + + it("rejects negative --max-tokens instead of silently falling back", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "helpbase-ing-budget-")) + try { + fs.writeFileSync(path.join(tmp, "README.md"), "# Hello\n\nsample repo") + // Negative --max-tokens used to coerce to 100_000 via `parseInt(x) || default`, + // which also left the `maxTokens > 0` gate inside generateHowtosFromRepo + // bypassed if the caller actually passed a negative number at runtime. + const result = runWithEnv(`ingest ${tmp} --max-tokens -1 --dry-run`, { + AI_GATEWAY_API_KEY: "", + ANTHROPIC_API_KEY: "", + OPENAI_API_KEY: "", + }) + expect(result.exitCode).toBe(1) + expect(result.output).toContain("E_CONTEXT_INVALID_BUDGET") + } finally { + fs.rmSync(tmp, { recursive: true, force: true }) + } + }) + + it("rejects zero --chars-per-token (would trivially make the estimate 0)", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "helpbase-ing-chars-")) + try { + fs.writeFileSync(path.join(tmp, "README.md"), "# Hello\n\nsample repo") + const result = runWithEnv(`ingest ${tmp} --chars-per-token 0 --dry-run`, { + AI_GATEWAY_API_KEY: "", + ANTHROPIC_API_KEY: "", + OPENAI_API_KEY: "", + }) + expect(result.exitCode).toBe(1) + expect(result.output).toContain("E_CONTEXT_INVALID_BUDGET") + } finally { + fs.rmSync(tmp, { recursive: true, force: true }) + } + }) + + it("accepts --max-tokens 0 as a gate-disable (documented behavior)", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "helpbase-ing-zero-")) + try { + fs.writeFileSync(path.join(tmp, "README.md"), "# Hello\n\nsample repo") + // 0 disables the budget gate entirely per the flag's contract. + // --dry-run skips the LLM call so we can exercise parseBudgetInt + // without a real key. + const result = runWithEnv(`ingest ${tmp} --max-tokens 0 --dry-run`, { + AI_GATEWAY_API_KEY: "", + ANTHROPIC_API_KEY: "", + OPENAI_API_KEY: "", + }) + expect(result.exitCode).toBe(0) + expect(result.output).toContain("Dry run") + } finally { + fs.rmSync(tmp, { recursive: true, force: true }) + } + }) + + it("aborts before LLM call on secret-shaped content in source files", () => { + // Pre-LLM secret scan: a real key in a .ts file must abort the run + // before buildContextPrompt or any LLM call — otherwise the secret + // would leak into _prompt.txt under --debug or to the LLM gateway. + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "helpbase-ing-secret-")) + try { + fs.writeFileSync(path.join(tmp, "README.md"), "# Hello\n\nsample repo") + // Use a credential-assignment pattern that matches SECRET_CONTENT_PATTERNS + // without being a real key (the pattern only needs 10+ non-ws chars). + fs.writeFileSync( + path.join(tmp, "config.ts"), + 'export const ANTHROPIC_API_KEY = "sk-ant-fake1234567890abcdef"\n', + ) + const result = runWithEnv(`ingest ${tmp} --dry-run`, { + AI_GATEWAY_API_KEY: "", + ANTHROPIC_API_KEY: "", + OPENAI_API_KEY: "", + }) + expect(result.exitCode).toBe(1) + expect(result.output).toContain("E_CONTEXT_SECRET_SOURCE") + // Exact line number + pattern name, never the secret bytes. + expect(result.output).toContain("sk-api-key") + expect(result.output).not.toContain("sk-ant-fake1234567890abcdef") + } finally { + fs.rmSync(tmp, { recursive: true, force: true }) + } + }) + + it("--dry-run on a real repo prints a plan without an LLM call", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "helpbase-ing-dry-")) + try { + fs.writeFileSync(path.join(tmp, "README.md"), "# Hello\n\nsample repo for test") + fs.writeFileSync(path.join(tmp, "index.ts"), "export const one = 1") + // Dry-run does not need a key. + const result = runWithEnv(`ingest ${tmp} --dry-run`, { + AI_GATEWAY_API_KEY: "", + ANTHROPIC_API_KEY: "", + OPENAI_API_KEY: "", + }) + expect(result.exitCode).toBe(0) + expect(result.output).toContain("Dry run") + expect(result.output).toContain("Sources found:") + expect(result.output).toContain("Estimated tokens:") + } finally { + fs.rmSync(tmp, { recursive: true, force: true }) + } + }) + + it("--reuse-existing without --ask surfaces E_CONTEXT_REUSE_WITHOUT_ASK", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "helpbase-ing-reuse-")) + try { + fs.writeFileSync(path.join(tmp, "README.md"), "# Hello\n\nsample") + const result = run(`ingest ${tmp} --reuse-existing`) + expect(result.exitCode).toBe(1) + expect(result.output).toContain("E_CONTEXT_REUSE_WITHOUT_ASK") + } finally { + fs.rmSync(tmp, { recursive: true, force: true }) + } + }) + + it("--reuse-existing --ask with empty .helpbase surfaces E_CONTEXT_REUSE_EMPTY", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "helpbase-ing-reuse-")) + try { + fs.writeFileSync(path.join(tmp, "README.md"), "# Hello\n\nsample") + // No .helpbase/docs directory exists — reuse should fail loudly. + const result = run(`ingest ${tmp} --reuse-existing --ask "anything"`) + expect(result.exitCode).toBe(1) + expect(result.output).toContain("E_CONTEXT_REUSE_EMPTY") + } finally { + fs.rmSync(tmp, { recursive: true, force: true }) + } + }) + + it("resolves project name from package.json, not the containing directory (regression)", async () => { + // Temp dirs have names like `helpbase-user-test-znh2Rp`. Without this + // fix, that string ended up in the LLM prompt and in llms.txt as the + // project's display name — drowning out the `name: "todo-app"` the + // user set in their package.json. + const { resolveProjectName } = await import("../src/commands/ingest.js") + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "helpbase-name-regression-")) + try { + fs.writeFileSync( + path.join(tmp, "package.json"), + JSON.stringify({ name: "todo-app", version: "0.1.0" }), + ) + expect(resolveProjectName(tmp)).toBe("todo-app") + } finally { + fs.rmSync(tmp, { recursive: true, force: true }) + } + }) + + it("falls back to the directory basename when package.json has no name", async () => { + const { resolveProjectName } = await import("../src/commands/ingest.js") + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "helpbase-noname-")) + try { + fs.writeFileSync(path.join(tmp, "package.json"), JSON.stringify({ version: "0.1.0" })) + expect(resolveProjectName(tmp)).toBe(path.basename(tmp)) + } finally { + fs.rmSync(tmp, { recursive: true, force: true }) + } + }) + + it("falls back to the directory basename when there is no package.json at all", async () => { + const { resolveProjectName } = await import("../src/commands/ingest.js") + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "helpbase-nopkg-")) + try { + expect(resolveProjectName(tmp)).toBe(path.basename(tmp)) + } finally { + fs.rmSync(tmp, { recursive: true, force: true }) + } + }) + + it("--reuse-existing --ask with a populated .helpbase/docs reaches auth without walking sources (hermetic)", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "helpbase-ing-reuse-")) + try { + // Seed a pre-generated doc on disk — the fast path reads these. + const docsDir = path.join(tmp, ".helpbase", "docs", "getting-started") + fs.mkdirSync(docsDir, { recursive: true }) + fs.writeFileSync( + path.join(docsDir, "how-to-log-in.mdx"), + [ + "---", + "schemaVersion: 1", + 'title: "How to log in"', + 'description: "Log in with a magic link."', + "tags: []", + "order: 1", + "---", + "", + "Call POST /api/auth/login with email and password.", + "", + ].join("\n"), + ) + // Observable behavior: the run passes the reuse gate + the source-walker + // gate (no REUSE_EMPTY, no NO_SOURCES, no REPO_PATH) and then fails at + // the auth boundary — NEVER reaches the network. Previously this test + // used a fake `vck_...` AI_GATEWAY_API_KEY and asserted "Answering:" to + // prove runLocalAsk fired, but that let a contributor-laptop with a + // real ANTHROPIC_API_KEY or OPENAI_API_KEY burn real tokens on every + // test run. CodeRabbit called the hazard out — the hermetic replacement + // checks for E_AUTH_REQUIRED instead, which is a strictly stronger + // signal about where execution reached. + const result = runWithEnv( + `ingest ${tmp} --reuse-existing --ask "how do I log in?"`, + { + AI_GATEWAY_API_KEY: "", + ANTHROPIC_API_KEY: "", + OPENAI_API_KEY: "", + HELPBASE_TOKEN: "", + }, + ) + expect(result.exitCode).toBe(1) + expect(result.output).toContain("E_AUTH_REQUIRED") + expect(result.output).not.toContain("E_CONTEXT_REUSE_EMPTY") + expect(result.output).not.toContain("E_CONTEXT_REUSE_WITHOUT_ASK") + expect(result.output).not.toContain("E_CONTEXT_NO_SOURCES") + } finally { + fs.rmSync(tmp, { recursive: true, force: true }) + } + }) +}) + +describe("ingest/context option surface parity", () => { + it("applyIngestOptions produces byte-identical option flags on both commands", async () => { + // Drift guard: if a future dev adds a .option(...) directly to + // `ingestCommand` at the declaration site (instead of inside + // applyIngestOptions), the new flag silently missing on `context` would + // only surface as a user running `helpbase context --new-flag` and + // hitting Commander's "unknown option" error. Introspect both Command + // instances here so the drift is a test failure, not a user report. + const { ingestCommand } = await import("../src/commands/ingest.js") + const { contextCommand } = await import("../src/commands/context.js") + + const flagsOf = (cmd: { options: Array<{ flags: string }> }) => + cmd.options.map((o) => o.flags).sort() + const argsOf = (cmd: { _args: Array<{ _name: string; required: boolean; defaultValue: unknown }> }) => + cmd._args.map((a) => ({ name: a._name, required: a.required, def: a.defaultValue })) + + expect(flagsOf(ingestCommand)).toEqual(flagsOf(contextCommand)) + expect(argsOf(ingestCommand as unknown as { _args: Array<{ _name: string; required: boolean; defaultValue: unknown }> })).toEqual( + argsOf(contextCommand as unknown as { _args: Array<{ _name: string; required: boolean; defaultValue: unknown }> }), + ) + }) +}) + +// TODO(v0.7): delete this entire describe block together with +// packages/cli/src/commands/context.ts and the `addCommand(contextCommand)` +// line in packages/cli/src/index.ts. +describe("helpbase context (deprecated alias)", () => { + it("--help still works and flags the command as deprecated", () => { + const result = run("context --help") + expect(result.exitCode).toBe(0) + expect(result.output).toContain("deprecated") + expect(result.output).toContain("helpbase ingest") + // Option surface is mirrored via applyIngestOptions — spot-check a few. + expect(result.output).toContain("--max-tokens") + expect(result.output).toContain("--ask") + expect(result.output).toContain("--require-clean") + }) + + it("prints the deprecation warning to stderr on run", () => { + // An invalid repo path triggers the error path — we only care that the + // deprecation warning fires at all, which happens before any action. + const result = run(`context /tmp/definitely-not-a-real-path-ctx-depr-test`) + expect(result.output).toContain("helpbase context is deprecated") + expect(result.output).toContain("helpbase ingest") + }) + + it("deprecation warning suppressed under --quiet", () => { + const result = run("context --quiet /tmp/definitely-not-a-real-path-ctx-quiet-test") + expect(result.output).not.toContain("helpbase context is deprecated") + }) + + it("deprecation warning suppressed under --json (pipes should stay clean)", () => { + // Mirror of the --quiet test — the shim's emitDeprecationWarning honors + // both flags. A refactor that drops one check would pollute JSON stdin + // for downstream consumers. + const result = run("context --json /tmp/definitely-not-a-real-path-ctx-json-test") + expect(result.output).not.toContain("helpbase context is deprecated") + }) + + it("forwards to the same pipeline — --dry-run works identically", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "helpbase-ctx-dry-")) + try { + fs.writeFileSync(path.join(tmp, "README.md"), "# Hello\n\nsample repo for test") + const result = runWithEnv(`context ${tmp} --dry-run`, { + AI_GATEWAY_API_KEY: "", + ANTHROPIC_API_KEY: "", + OPENAI_API_KEY: "", + }) + expect(result.exitCode).toBe(0) + expect(result.output).toContain("Dry run") + expect(result.output).toContain("Sources found:") + } finally { + fs.rmSync(tmp, { recursive: true, force: true }) + } + }) +}) + +describe("helpbase generate positional alias", () => { + it("accepts a positional path as an alias for --repo", () => { + // With neither --url, --repo, nor --screenshots, and no positional, + // generate must error. With a positional path but no --repo flag, the + // alias kicks in and the command proceeds to path-existence checks. + // We can observe the alias indirectly: a bogus path yields a repo + // error, not the "Provide a source" error. + const result = run("generate /tmp/definitely-does-not-exist-helpbase-alias-test") + expect(result.exitCode).toBe(1) + // The alias should have mapped to --repo; the error is about the repo path, not missing source. + expect(result.output).not.toContain("Provide a source") + }) + + it("still shows the source error when no positional and no flags", () => { + const result = run("generate") + expect(result.exitCode).toBe(1) + expect(result.output).toContain("Provide a source") + }) +}) diff --git a/packages/create-helpbase/src/emit-mcp-json.ts b/packages/create-helpbase/src/emit-mcp-json.ts index 00ba4f0..3780cfc 100644 --- a/packages/create-helpbase/src/emit-mcp-json.ts +++ b/packages/create-helpbase/src/emit-mcp-json.ts @@ -7,11 +7,11 @@ import path from "node:path" * Claude Desktop / Cursor / Claude Code and their agent immediately * sees the cited help-center articles. * - * Shape matches `helpbase context`'s emit so returning users who already + * Shape matches `helpbase ingest`'s emit so returning users who already * know one layout find the other familiar. The key difference: the * content dir here is `<projectDir>/content/` (what the Next.js app * renders), NOT `.helpbase/docs/` (the agent-only layout helpbase - * context emits). + * ingest emits). * * The file is a hint, not a working MCP client config. The user copies * one of the per-client blocks into their real config file. Absolute diff --git a/packages/create-helpbase/src/generate-from-repo.ts b/packages/create-helpbase/src/generate-from-repo.ts index 5a42286..ac6b959 100644 --- a/packages/create-helpbase/src/generate-from-repo.ts +++ b/packages/create-helpbase/src/generate-from-repo.ts @@ -23,7 +23,7 @@ import type { GeneratedContextDoc } from "@workspace/shared/schemas" * Thrown when the LLM returns articles but every single one is dropped by * the citation validator. The caller (scaffolder) restores sample content * and shows an actionable error — retry on a stronger model, run - * `helpbase context .` manually, etc. + * `helpbase ingest .` manually, etc. * * Separate class so the outer try/catch can distinguish "nothing to write" * from "LLM errored" or "missing auth" without string-matching. @@ -76,7 +76,7 @@ export interface GenerateFromRepoResult { * `<projectDir>/content/<category>/<slug>.mdx` plus `_category.json` * scaffolds that the Next.js sidebar consumes. * - * Mirrors the `helpbase context` pipeline but targets the scaffolder's + * Mirrors the `helpbase ingest` pipeline but targets the scaffolder's * human-facing layout (`content/`) instead of the agent-facing * `.helpbase/docs/` layout. Every pre-write invariant the CLI enforces * (citation validation, secret deny-list, MDX sanitization) runs here too. @@ -159,7 +159,7 @@ export async function generateFromRepo( for (const s of serialized) { const matches = scanForSecrets(s.content) if (matches.length > 0) { - // Same policy as helpbase context: refuse to write any file if any + // Same policy as helpbase ingest: refuse to write any file if any // file would leak. Pattern name + line number only — never the secret. throw new Error(formatSecretError(matches, s.relPath)) } @@ -227,7 +227,7 @@ export async function generateFromRepo( } /** - * Prefer `package.json` name (matches helpbase context behavior) so the + * Prefer `package.json` name (matches helpbase ingest behavior) so the * LLM writes articles about "my-app" not "my-docs-test". Falls back to * the repo directory basename if package.json is missing or malformed. */ diff --git a/packages/create-helpbase/src/index.ts b/packages/create-helpbase/src/index.ts index 9d79e59..64d98ac 100644 --- a/packages/create-helpbase/src/index.ts +++ b/packages/create-helpbase/src/index.ts @@ -286,7 +286,7 @@ async function run(directory: string | undefined, opts: RunOptions) { note(bootstrap, "Run it locally") note( - `${pc.cyan("npx helpbase context .")} ${pc.dim("generate docs from your repo source")}\n` + + `${pc.cyan("npx helpbase ingest .")} ${pc.dim("generate docs from your repo source")}\n` + `${pc.cyan("npx helpbase deploy")} ${pc.dim("go live at <slug>.helpbase.dev")}\n` + `${pc.cyan("npx helpbase new")} ${pc.dim("add a new article interactively")}`, "What next", @@ -706,11 +706,28 @@ function printGenerationFallbackHint(err: unknown, url: string): void { ) } +/** + * Minimal POSIX shell-quote for user-facing copy/paste commands. Avoids + * adding an `shell-quote` dep for a single call site. Returns the string + * unchanged when it only contains safe characters; otherwise wraps it in + * single quotes and escapes embedded single quotes via the `'\''` idiom. + * + * Relevant for `printRepoGenerationFallbackHint` below: a user who runs + * `create-helpbase` from `/Users/foo/my docs` needs the retry hint to + * print `helpbase ingest '/Users/foo/my docs'` — unquoted, the copy/paste + * breaks on the space. + */ +function quoteShellArg(value: string): string { + if (/^[A-Za-z0-9_./:@%+=,-]+$/.test(value)) return value + return `'${value.replace(/'/g, `'\\''`)}'` +} + function printRepoGenerationFallbackHint(err: unknown, repoPath: string): void { + const quotedRepoPath = quoteShellArg(repoPath) if (err instanceof MissingApiKeyError) { note( `Run ${pc.cyan("helpbase login")} (free, no card) then:\n` + - ` ${pc.cyan(`helpbase context ${repoPath}`)}\n\n` + + ` ${pc.cyan(`helpbase ingest ${quotedRepoPath}`)}\n\n` + `Or bring your own key: ${pc.cyan("ANTHROPIC_API_KEY")}, ${pc.cyan("OPENAI_API_KEY")}, or ${pc.cyan("AI_GATEWAY_API_KEY")} ` + `(docs: ${pc.cyan("helpbase.dev/guides/byok")})`, "AI generation skipped", @@ -721,7 +738,7 @@ function printRepoGenerationFallbackHint(err: unknown, repoPath: string): void { note( `${err.message}\n` + `Point at a subdirectory with more focused content, or run:\n` + - ` ${pc.cyan(`helpbase context ${repoPath} --max-tokens 200000`)} after scaffold.`, + ` ${pc.cyan(`helpbase ingest ${quotedRepoPath} --max-tokens 200000`)} after scaffold.`, "Repo is too large", ) return @@ -738,7 +755,7 @@ function printRepoGenerationFallbackHint(err: unknown, repoPath: string): void { `The model returned ${err.rawDocCount} article(s) but citation validation dropped all of them. ` + `This is common on cheap models that paraphrase quoted code.\n\n` + `Retry with:\n` + - ` ${pc.cyan(`helpbase context ${repoPath} --model anthropic/claude-sonnet-4.6`)}\n` + + ` ${pc.cyan(`helpbase ingest ${quotedRepoPath} --model anthropic/claude-sonnet-4.6`)}\n` + `after scaffold to regenerate with a stronger model.`, "All articles dropped", ) @@ -747,13 +764,13 @@ function printRepoGenerationFallbackHint(err: unknown, repoPath: string): void { if (err instanceof GatewayError) { note( `Gateway error: ${err.message}\n` + - `Retry with ${pc.cyan(`helpbase context ${repoPath} --test`)} after scaffold to use a cheap fallback model.`, + `Retry with ${pc.cyan(`helpbase ingest ${quotedRepoPath} --test`)} after scaffold to use a cheap fallback model.`, "AI generation failed", ) return } note( - `Run ${pc.cyan(`helpbase context ${repoPath}`)} later to retry.`, + `Run ${pc.cyan(`helpbase ingest ${quotedRepoPath}`)} later to retry.`, "Tip", ) } diff --git a/packages/create-helpbase/templates/lib/schemas.ts b/packages/create-helpbase/templates/lib/schemas.ts index eeffbea..1913b60 100644 --- a/packages/create-helpbase/templates/lib/schemas.ts +++ b/packages/create-helpbase/templates/lib/schemas.ts @@ -33,7 +33,7 @@ function isAllowedEmbedHost(url: string): boolean { } /** - * A citation produced by `helpbase context` — points at the specific file + * A citation produced by `helpbase ingest` — points at the specific file * + line range that justifies a generated how-to guide. * * v2 contract: the LLM returns `{ file, startLine, endLine, reason? }` and @@ -86,7 +86,7 @@ export const frontmatterSchema = z.object({ }) .optional(), ogImage: z.string().optional(), - // Fields set by `helpbase context` on generated docs. Optional so hand- + // Fields set by `helpbase ingest` on generated docs. Optional so hand- // written articles and scaffolded content keep validating unchanged. citations: z.array(contextCitationSchema).optional(), source: z.enum(["generated", "custom"]).optional(), @@ -134,7 +134,7 @@ export const generatedArticlesSchema = z.object({ export type GeneratedArticle = z.infer<typeof generatedArticleSchema> /** - * Generated context doc — what the LLM returns for `helpbase context`. Extends + * Generated context doc — what the LLM returns for `helpbase ingest`. Extends * the article shape with citations (1–5 per doc, enforced; literal-text * validated before write) and a list of source file paths the model was * inspired by. Dropped docs (zero valid citations) never reach disk. diff --git a/packages/shared/src/ai-context.ts b/packages/shared/src/ai-context.ts index 6239837..bb09cc4 100644 --- a/packages/shared/src/ai-context.ts +++ b/packages/shared/src/ai-context.ts @@ -1,5 +1,5 @@ /** - * Prompt + generator + MDX wrapper for `helpbase context`. + * Prompt + generator + MDX wrapper for `helpbase ingest` (formerly `context`). * * Responsibilities: * - buildContextPrompt: wrap repo content in <untrusted-repo-content> @@ -418,7 +418,7 @@ export interface BuildAskPromptInput { question: string /** * Already-written MDX docs — pass the raw strings (title + body) - * that context just produced. The LLM answers against these. + * that `helpbase ingest` just produced. The LLM answers against these. */ docs: Array<{ title: string; path: string; body: string }> } @@ -429,7 +429,7 @@ export interface BuildAskPromptInput { * * Decision #8: the magical moment has to happen where the user already * is — their terminal — not after a 5-step Claude Desktop config paste. - * This prompt is what powers `helpbase context --ask "<question>"`. + * This prompt is what powers `helpbase ingest --ask "<question>"`. */ export function buildLocalAskPrompt({ question, diff --git a/packages/shared/src/ai-text.ts b/packages/shared/src/ai-text.ts index b912f30..634ec16 100644 --- a/packages/shared/src/ai-text.ts +++ b/packages/shared/src/ai-text.ts @@ -83,7 +83,7 @@ export const MAX_REPO_CONTENT_CHARS = 200_000 // Kept local to avoid a circular import between shared modules; these live // in a dedicated `./secrets.ts` module that both `ai-text` (walker) and the -// CLI `context` command import from. +// CLI `ingest` command (plus its deprecated `context` shim) import from. import { isSecretFile } from "./secrets.js" const MARKDOWN_EXTENSIONS = new Set([".md", ".mdx", ".markdown"]) @@ -125,7 +125,7 @@ function walkMarkdownFiles(rootDir: string): string[] { const ext = path.extname(entry.name).toLowerCase() if (MARKDOWN_EXTENSIONS.has(ext)) { // Drop files whose NAMES indicate a secret (.env*, *.pem, *.key, ...). - // Content-level secret scanning runs separately in helpbase context; + // Content-level secret scanning runs separately in helpbase ingest; // this gate protects even generate --repo from slurping secrets into // an LLM prompt. if (isSecretFile(entry.name)) continue diff --git a/packages/shared/src/citations.ts b/packages/shared/src/citations.ts index b264828..a6d952e 100644 --- a/packages/shared/src/citations.ts +++ b/packages/shared/src/citations.ts @@ -1,5 +1,5 @@ /** - * Citation validator for `helpbase context`. + * Citation validator for `helpbase ingest` (formerly `context`). * * Every generated how-to doc cites 1–5 specific file/line-range tuples. * This module verifies each citation against the repo on disk: @@ -184,7 +184,7 @@ export interface ArticleValidationResult { /** * Validate all citations on an article. Returns the subset that passed * ("kept") and the failures with reasons ("dropped"). Callers decide what - * to do with a 0-kept article (context.ts drops it entirely and writes the + * to do with a 0-kept article (ingest.ts drops it entirely and writes the * reason to `.helpbase/synthesis-report.json`). */ export function validateArticleCitations( diff --git a/packages/shared/src/context-reader.ts b/packages/shared/src/context-reader.ts index 41ed088..354a038 100644 --- a/packages/shared/src/context-reader.ts +++ b/packages/shared/src/context-reader.ts @@ -1,5 +1,5 @@ /** - * Context-specific repo reader for `helpbase context`. + * Context-specific repo reader for `helpbase ingest` (formerly `context`). * * Different from `readRepoContent` (which concatenates markdown into one * string and silently truncates at 200k chars): this one returns a list @@ -191,7 +191,7 @@ export function readContextSources( if (rawBuf.byteLength > maxBytes) { const placeholder = `// [file too large, skipped: ${rawBuf.byteLength} bytes > ${maxBytes}B cap]` process.stderr.write( - `[helpbase-context] Skipping ${rel} (${rawBuf.byteLength} bytes > ${maxBytes}B cap)\n`, + `[helpbase-ingest] Skipping ${rel} (${rawBuf.byteLength} bytes > ${maxBytes}B cap)\n`, ) sources.push({ path: rel, diff --git a/packages/shared/src/context-writer.ts b/packages/shared/src/context-writer.ts index 6d50512..ebd72e5 100644 --- a/packages/shared/src/context-writer.ts +++ b/packages/shared/src/context-writer.ts @@ -1,15 +1,15 @@ /** - * Inventory-aware write planner for `helpbase context`. + * Inventory-aware write planner for `helpbase ingest` (formerly `context`). * * `planArticleWrites` (in ai.ts) blindly maps articles to target paths and * knows nothing about previously generated content or user-edited custom - * files. For context, regen must: + * files. For ingest, regen must: * - preserve files whose frontmatter says `source: custom` * - replace files whose frontmatter says `source: generated` * - delete stale generated files (previous run had them; this run doesn't) * - leave anything else alone * - * This module does the inventory + diffing in pure code so the context + * This module does the inventory + diffing in pure code so the ingest * command's action handler reads top-to-bottom. */ diff --git a/packages/shared/src/llms-txt.ts b/packages/shared/src/llms-txt.ts index 782e4f4..bfe98b5 100644 --- a/packages/shared/src/llms-txt.ts +++ b/packages/shared/src/llms-txt.ts @@ -10,7 +10,7 @@ * * Extracted from apps/web/scripts/generate-llms.mjs and * packages/create-helpbase/template-assets/generate-llms.mjs so both - * scripts can become thin wrappers + the `helpbase context` command + * scripts can become thin wrappers + the `helpbase ingest` command * can emit the same format into `.helpbase/`. * * Pure: no fs writes. Caller decides where to persist. diff --git a/packages/shared/src/schemas.ts b/packages/shared/src/schemas.ts index eeffbea..1913b60 100644 --- a/packages/shared/src/schemas.ts +++ b/packages/shared/src/schemas.ts @@ -33,7 +33,7 @@ function isAllowedEmbedHost(url: string): boolean { } /** - * A citation produced by `helpbase context` — points at the specific file + * A citation produced by `helpbase ingest` — points at the specific file * + line range that justifies a generated how-to guide. * * v2 contract: the LLM returns `{ file, startLine, endLine, reason? }` and @@ -86,7 +86,7 @@ export const frontmatterSchema = z.object({ }) .optional(), ogImage: z.string().optional(), - // Fields set by `helpbase context` on generated docs. Optional so hand- + // Fields set by `helpbase ingest` on generated docs. Optional so hand- // written articles and scaffolded content keep validating unchanged. citations: z.array(contextCitationSchema).optional(), source: z.enum(["generated", "custom"]).optional(), @@ -134,7 +134,7 @@ export const generatedArticlesSchema = z.object({ export type GeneratedArticle = z.infer<typeof generatedArticleSchema> /** - * Generated context doc — what the LLM returns for `helpbase context`. Extends + * Generated context doc — what the LLM returns for `helpbase ingest`. Extends * the article shape with citations (1–5 per doc, enforced; literal-text * validated before write) and a list of source file paths the model was * inspired by. Dropped docs (zero valid citations) never reach disk. diff --git a/packages/shared/src/secrets.ts b/packages/shared/src/secrets.ts index 0534bc0..6445760 100644 --- a/packages/shared/src/secrets.ts +++ b/packages/shared/src/secrets.ts @@ -1,5 +1,5 @@ /** - * Secret deny-list — two-gate defense for helpbase context ingestion. + * Secret deny-list — two-gate defense for helpbase ingest (formerly context). * * Gate 1: `isSecretFile(relPath)` — called by walkers to skip files whose * NAMES indicate secrets (.env*, *.pem, *.key, etc.). These files are @@ -161,6 +161,6 @@ export function formatSecretError( return ( `Secret-shaped content detected in ${filePath}:\n${lines}\n` + `\nThe run was aborted before any .helpbase/ write. Inspect the file, ` + - `remove or gitignore the secret source, and re-run helpbase context.` + `remove or gitignore the secret source, and re-run helpbase ingest.` ) } diff --git a/registry/helpbase/lib/schemas.ts b/registry/helpbase/lib/schemas.ts index eeffbea..1913b60 100644 --- a/registry/helpbase/lib/schemas.ts +++ b/registry/helpbase/lib/schemas.ts @@ -33,7 +33,7 @@ function isAllowedEmbedHost(url: string): boolean { } /** - * A citation produced by `helpbase context` — points at the specific file + * A citation produced by `helpbase ingest` — points at the specific file * + line range that justifies a generated how-to guide. * * v2 contract: the LLM returns `{ file, startLine, endLine, reason? }` and @@ -86,7 +86,7 @@ export const frontmatterSchema = z.object({ }) .optional(), ogImage: z.string().optional(), - // Fields set by `helpbase context` on generated docs. Optional so hand- + // Fields set by `helpbase ingest` on generated docs. Optional so hand- // written articles and scaffolded content keep validating unchanged. citations: z.array(contextCitationSchema).optional(), source: z.enum(["generated", "custom"]).optional(), @@ -134,7 +134,7 @@ export const generatedArticlesSchema = z.object({ export type GeneratedArticle = z.infer<typeof generatedArticleSchema> /** - * Generated context doc — what the LLM returns for `helpbase context`. Extends + * Generated context doc — what the LLM returns for `helpbase ingest`. Extends * the article shape with citations (1–5 per doc, enforced; literal-text * validated before write) and a list of source file paths the model was * inspired by. Dropped docs (zero valid citations) never reach disk.