From 2ebe26227f147ff7713a5577893782ed711652cd Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Wed, 14 Oct 2026 08:07:11 -0400 Subject: [PATCH 1/4] feat(agent-context): trigger evaluation for the prompts and codegen skill descriptions Add should-trigger and should-not-trigger cases (12 each) for metaobjects-prompts and metaobjects-codegen, a runner that wraps the skills in a throwaway plugin for claude plugin eval, and a CI test that the cases parse and name existing skills. First run: codegen 36/36, prompts 32/36 (one run of an entities-only near-miss fired the skill). Narrow the prompts description to projects that declare template.prompt; codegen needed no change. Rerun: 36/36 and 36/36. --- agent-context/evals/README.md | 28 ++++++++ agent-context/evals/run-skill-triggers.sh | 23 +++++++ .../graders/trigger.md | 6 ++ .../01-pos-write-openapi-generator/prompt.md | 8 +++ .../graders/trigger.md | 6 ++ .../prompt.md | 8 +++ .../graders/trigger.md | 6 ++ .../03-pos-configure-generators/prompt.md | 8 +++ .../graders/trigger.md | 6 ++ .../04-pos-regen-hand-edits/prompt.md | 8 +++ .../graders/trigger.md | 6 ++ .../05-pos-json-schema-output/prompt.md | 8 +++ .../06-pos-dialect-config/graders/trigger.md | 6 ++ .../06-pos-dialect-config/prompt.md | 8 +++ .../graders/trigger.md | 8 +++ .../07-neg-add-entity-field/prompt.md | 8 +++ .../08-neg-verify-failure/graders/trigger.md | 8 +++ .../08-neg-verify-failure/prompt.md | 8 +++ .../graders/trigger.md | 8 +++ .../09-neg-openapi-no-metaobjects/prompt.md | 8 +++ .../10-neg-prisma-codegen/graders/trigger.md | 8 +++ .../10-neg-prisma-codegen/prompt.md | 8 +++ .../11-neg-prompt-template/graders/trigger.md | 8 +++ .../11-neg-prompt-template/prompt.md | 8 +++ .../12-neg-runtime-query/graders/trigger.md | 8 +++ .../12-neg-runtime-query/prompt.md | 8 +++ .../01-pos-fstring-prompt/graders/trigger.md | 6 ++ .../01-pos-fstring-prompt/prompt.md | 8 +++ .../graders/trigger.md | 6 ++ .../02-pos-declare-template-prompt/prompt.md | 8 +++ .../graders/trigger.md | 6 ++ .../03-pos-regex-reply-parse/prompt.md | 8 +++ .../graders/trigger.md | 6 ++ .../04-pos-test-harness-prompt/prompt.md | 8 +++ .../graders/trigger.md | 6 ++ .../05-pos-prompt-drift-verify/prompt.md | 8 +++ .../graders/trigger.md | 6 ++ .../06-pos-review-json-parse/prompt.md | 8 +++ .../graders/trigger.md | 8 +++ .../07-neg-fstring-no-metaobjects/prompt.md | 8 +++ .../graders/trigger.md | 8 +++ .../08-neg-langchain-system-prompt/prompt.md | 8 +++ .../graders/trigger.md | 8 +++ .../prompt.md | 8 +++ .../graders/trigger.md | 8 +++ .../prompt.md | 8 +++ .../graders/trigger.md | 8 +++ .../11-neg-add-entity-field/prompt.md | 8 +++ .../graders/trigger.md | 8 +++ .../12-neg-non-llm-string-building/prompt.md | 8 +++ .../skills/metaobjects-prompts/SKILL.md | 2 +- .../skills/metaobjects-prompts/SKILL.md | 2 +- .../skills/metaobjects-prompts/SKILL.md | 2 +- .../skills/metaobjects-prompts/SKILL.md | 2 +- .../skills/metaobjects-prompts/SKILL.md | 2 +- .../skills/metaobjects-prompts/SKILL.md | 2 +- .../agent-context/skill-trigger-evals.test.ts | 68 +++++++++++++++++++ 57 files changed, 485 insertions(+), 6 deletions(-) create mode 100644 agent-context/evals/README.md create mode 100755 agent-context/evals/run-skill-triggers.sh create mode 100644 agent-context/evals/skill-triggers/metaobjects-codegen/01-pos-write-openapi-generator/graders/trigger.md create mode 100644 agent-context/evals/skill-triggers/metaobjects-codegen/01-pos-write-openapi-generator/prompt.md create mode 100644 agent-context/evals/skill-triggers/metaobjects-codegen/02-pos-eject-reference-generator/graders/trigger.md create mode 100644 agent-context/evals/skill-triggers/metaobjects-codegen/02-pos-eject-reference-generator/prompt.md create mode 100644 agent-context/evals/skill-triggers/metaobjects-codegen/03-pos-configure-generators/graders/trigger.md create mode 100644 agent-context/evals/skill-triggers/metaobjects-codegen/03-pos-configure-generators/prompt.md create mode 100644 agent-context/evals/skill-triggers/metaobjects-codegen/04-pos-regen-hand-edits/graders/trigger.md create mode 100644 agent-context/evals/skill-triggers/metaobjects-codegen/04-pos-regen-hand-edits/prompt.md create mode 100644 agent-context/evals/skill-triggers/metaobjects-codegen/05-pos-json-schema-output/graders/trigger.md create mode 100644 agent-context/evals/skill-triggers/metaobjects-codegen/05-pos-json-schema-output/prompt.md create mode 100644 agent-context/evals/skill-triggers/metaobjects-codegen/06-pos-dialect-config/graders/trigger.md create mode 100644 agent-context/evals/skill-triggers/metaobjects-codegen/06-pos-dialect-config/prompt.md create mode 100644 agent-context/evals/skill-triggers/metaobjects-codegen/07-neg-add-entity-field/graders/trigger.md create mode 100644 agent-context/evals/skill-triggers/metaobjects-codegen/07-neg-add-entity-field/prompt.md create mode 100644 agent-context/evals/skill-triggers/metaobjects-codegen/08-neg-verify-failure/graders/trigger.md create mode 100644 agent-context/evals/skill-triggers/metaobjects-codegen/08-neg-verify-failure/prompt.md create mode 100644 agent-context/evals/skill-triggers/metaobjects-codegen/09-neg-openapi-no-metaobjects/graders/trigger.md create mode 100644 agent-context/evals/skill-triggers/metaobjects-codegen/09-neg-openapi-no-metaobjects/prompt.md create mode 100644 agent-context/evals/skill-triggers/metaobjects-codegen/10-neg-prisma-codegen/graders/trigger.md create mode 100644 agent-context/evals/skill-triggers/metaobjects-codegen/10-neg-prisma-codegen/prompt.md create mode 100644 agent-context/evals/skill-triggers/metaobjects-codegen/11-neg-prompt-template/graders/trigger.md create mode 100644 agent-context/evals/skill-triggers/metaobjects-codegen/11-neg-prompt-template/prompt.md create mode 100644 agent-context/evals/skill-triggers/metaobjects-codegen/12-neg-runtime-query/graders/trigger.md create mode 100644 agent-context/evals/skill-triggers/metaobjects-codegen/12-neg-runtime-query/prompt.md create mode 100644 agent-context/evals/skill-triggers/metaobjects-prompts/01-pos-fstring-prompt/graders/trigger.md create mode 100644 agent-context/evals/skill-triggers/metaobjects-prompts/01-pos-fstring-prompt/prompt.md create mode 100644 agent-context/evals/skill-triggers/metaobjects-prompts/02-pos-declare-template-prompt/graders/trigger.md create mode 100644 agent-context/evals/skill-triggers/metaobjects-prompts/02-pos-declare-template-prompt/prompt.md create mode 100644 agent-context/evals/skill-triggers/metaobjects-prompts/03-pos-regex-reply-parse/graders/trigger.md create mode 100644 agent-context/evals/skill-triggers/metaobjects-prompts/03-pos-regex-reply-parse/prompt.md create mode 100644 agent-context/evals/skill-triggers/metaobjects-prompts/04-pos-test-harness-prompt/graders/trigger.md create mode 100644 agent-context/evals/skill-triggers/metaobjects-prompts/04-pos-test-harness-prompt/prompt.md create mode 100644 agent-context/evals/skill-triggers/metaobjects-prompts/05-pos-prompt-drift-verify/graders/trigger.md create mode 100644 agent-context/evals/skill-triggers/metaobjects-prompts/05-pos-prompt-drift-verify/prompt.md create mode 100644 agent-context/evals/skill-triggers/metaobjects-prompts/06-pos-review-json-parse/graders/trigger.md create mode 100644 agent-context/evals/skill-triggers/metaobjects-prompts/06-pos-review-json-parse/prompt.md create mode 100644 agent-context/evals/skill-triggers/metaobjects-prompts/07-neg-fstring-no-metaobjects/graders/trigger.md create mode 100644 agent-context/evals/skill-triggers/metaobjects-prompts/07-neg-fstring-no-metaobjects/prompt.md create mode 100644 agent-context/evals/skill-triggers/metaobjects-prompts/08-neg-langchain-system-prompt/graders/trigger.md create mode 100644 agent-context/evals/skill-triggers/metaobjects-prompts/08-neg-langchain-system-prompt/prompt.md create mode 100644 agent-context/evals/skill-triggers/metaobjects-prompts/09-neg-json-reply-no-metaobjects/graders/trigger.md create mode 100644 agent-context/evals/skill-triggers/metaobjects-prompts/09-neg-json-reply-no-metaobjects/prompt.md create mode 100644 agent-context/evals/skill-triggers/metaobjects-prompts/10-neg-entities-only-project-prompt-string/graders/trigger.md create mode 100644 agent-context/evals/skill-triggers/metaobjects-prompts/10-neg-entities-only-project-prompt-string/prompt.md create mode 100644 agent-context/evals/skill-triggers/metaobjects-prompts/11-neg-add-entity-field/graders/trigger.md create mode 100644 agent-context/evals/skill-triggers/metaobjects-prompts/11-neg-add-entity-field/prompt.md create mode 100644 agent-context/evals/skill-triggers/metaobjects-prompts/12-neg-non-llm-string-building/graders/trigger.md create mode 100644 agent-context/evals/skill-triggers/metaobjects-prompts/12-neg-non-llm-string-building/prompt.md create mode 100644 server/typescript/packages/sdk/test/agent-context/skill-trigger-evals.test.ts diff --git a/agent-context/evals/README.md b/agent-context/evals/README.md new file mode 100644 index 000000000..2532f80a2 --- /dev/null +++ b/agent-context/evals/README.md @@ -0,0 +1,28 @@ +# Skill trigger evaluations + +Should-trigger and should-not-trigger cases for the `description` of two shipped skills, +`metaobjects-prompts` and `metaobjects-codegen`. A description decides when a skill's whole +body is loaded, so an over-eager one costs tokens on every unrelated edit. Each case is a +one-message prompt plus a `tool_used: Skill` grader: positive cases require the skill to +fire, negative cases (near-misses: LLM code in a project that does not use MetaObjects +prompts, model edits that need no generated output) forbid it. + +- `skill-triggers//-pos-*|neg-*/` — `prompt.md` and `graders/trigger.md`, in the + `claude plugin eval` case format. +- `run-skill-triggers.sh` — wraps `agent-context/skills` in a throwaway plugin (all seven + skills, so routing competes realistically) and runs the cases, single-arm. + +## Running it (manual) + +``` +agent-context/evals/run-skill-triggers.sh +``` + +It starts real Claude sessions on your own login (24 cases x 3 runs = 72 sessions, a few +minutes with `-j 4`) and writes `/result.json`. Use `--case ''` to run a +subset. It is not part of CI because model behaviour is not deterministic; CI only checks +that the cases parse and name existing skills +(`server/typescript/packages/sdk/test/agent-context/skill-trigger-evals.test.ts`). + +Re-run it after editing either description. A `Reached maximum number of turns` note on a +case is expected: the runs are capped at 3 turns because only the first tool call matters. diff --git a/agent-context/evals/run-skill-triggers.sh b/agent-context/evals/run-skill-triggers.sh new file mode 100755 index 000000000..30017848e --- /dev/null +++ b/agent-context/evals/run-skill-triggers.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Run the skill trigger evaluation (agent-context/evals/skill-triggers) with `claude plugin eval`. +# Manual, not CI: it drives real model sessions on your own Claude login (subscription quota). +# +# usage: agent-context/evals/run-skill-triggers.sh [OUT_DIR] [extra `claude plugin eval` flags] +# OUT_DIR defaults to a fresh temp dir; the scored result is written to OUT_DIR/result.json. +# e.g. agent-context/evals/run-skill-triggers.sh /tmp/triggers --case 'metaobjects-codegen/*' +set -euo pipefail +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +skills="$here/../skills" +out="${1:-$(mktemp -d)}"; [ $# -gt 0 ] && shift +mkdir -p "$out" +plugin="$out/plugin" +rm -rf "$plugin" +mkdir -p "$plugin/.claude-plugin" +# `claude plugin eval` refuses an evals/ link that points outside the plugin, so copy. +echo '{"name":"mo-skills","version":"0.0.0","description":"MetaObjects skills, wrapped for trigger evaluation"}' >"$plugin/.claude-plugin/plugin.json" +cp -R "$skills" "$plugin/skills" +cp -R "$here/skill-triggers" "$plugin/evals" +cd "$plugin" +# --ablation none: one arm, so the `tool_used: Skill` graders count toward the score. +exec claude plugin eval . --ablation none --no-publish --trust-plugin -j 4 --threshold 0 \ + --output-dir "$out/runs" --json "$out/result.json" "$@" diff --git a/agent-context/evals/skill-triggers/metaobjects-codegen/01-pos-write-openapi-generator/graders/trigger.md b/agent-context/evals/skill-triggers/metaobjects-codegen/01-pos-write-openapi-generator/graders/trigger.md new file mode 100644 index 000000000..b8e0fb847 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-codegen/01-pos-write-openapi-generator/graders/trigger.md @@ -0,0 +1,6 @@ +--- +type: tool_used +tool: Skill +input_match: metaobjects-codegen +min: 1 +--- diff --git a/agent-context/evals/skill-triggers/metaobjects-codegen/01-pos-write-openapi-generator/prompt.md b/agent-context/evals/skill-triggers/metaobjects-codegen/01-pos-write-openapi-generator/prompt.md new file mode 100644 index 000000000..32b6d7753 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-codegen/01-pos-write-openapi-generator/prompt.md @@ -0,0 +1,8 @@ +--- +max_turns: 3 +timeout_seconds: 180 +allowed_tools: [Skill] +runs: 3 +--- + +This project uses MetaObjects (it has .metaobjects/config.json and a metaobjects/ directory of meta.*.json files). I need an OpenAPI document generated from the model. Write a generator for it. diff --git a/agent-context/evals/skill-triggers/metaobjects-codegen/02-pos-eject-reference-generator/graders/trigger.md b/agent-context/evals/skill-triggers/metaobjects-codegen/02-pos-eject-reference-generator/graders/trigger.md new file mode 100644 index 000000000..b8e0fb847 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-codegen/02-pos-eject-reference-generator/graders/trigger.md @@ -0,0 +1,6 @@ +--- +type: tool_used +tool: Skill +input_match: metaobjects-codegen +min: 1 +--- diff --git a/agent-context/evals/skill-triggers/metaobjects-codegen/02-pos-eject-reference-generator/prompt.md b/agent-context/evals/skill-triggers/metaobjects-codegen/02-pos-eject-reference-generator/prompt.md new file mode 100644 index 000000000..241aeb71f --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-codegen/02-pos-eject-reference-generator/prompt.md @@ -0,0 +1,8 @@ +--- +max_turns: 3 +timeout_seconds: 180 +allowed_tools: [Skill] +runs: 3 +--- + +This project uses MetaObjects (it has .metaobjects/config.json and a metaobjects/ directory of meta.*.json files). The reference Zod generator does not emit what I need. Eject it so I can own and change it. diff --git a/agent-context/evals/skill-triggers/metaobjects-codegen/03-pos-configure-generators/graders/trigger.md b/agent-context/evals/skill-triggers/metaobjects-codegen/03-pos-configure-generators/graders/trigger.md new file mode 100644 index 000000000..b8e0fb847 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-codegen/03-pos-configure-generators/graders/trigger.md @@ -0,0 +1,6 @@ +--- +type: tool_used +tool: Skill +input_match: metaobjects-codegen +min: 1 +--- diff --git a/agent-context/evals/skill-triggers/metaobjects-codegen/03-pos-configure-generators/prompt.md b/agent-context/evals/skill-triggers/metaobjects-codegen/03-pos-configure-generators/prompt.md new file mode 100644 index 000000000..2279cdbb2 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-codegen/03-pos-configure-generators/prompt.md @@ -0,0 +1,8 @@ +--- +max_turns: 3 +timeout_seconds: 180 +allowed_tools: [Skill] +runs: 3 +--- + +This project uses MetaObjects (it has .metaobjects/config.json and a metaobjects/ directory of meta.*.json files). Configure which generators run and where `meta gen` writes each target in metaobjects.config.ts. diff --git a/agent-context/evals/skill-triggers/metaobjects-codegen/04-pos-regen-hand-edits/graders/trigger.md b/agent-context/evals/skill-triggers/metaobjects-codegen/04-pos-regen-hand-edits/graders/trigger.md new file mode 100644 index 000000000..b8e0fb847 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-codegen/04-pos-regen-hand-edits/graders/trigger.md @@ -0,0 +1,6 @@ +--- +type: tool_used +tool: Skill +input_match: metaobjects-codegen +min: 1 +--- diff --git a/agent-context/evals/skill-triggers/metaobjects-codegen/04-pos-regen-hand-edits/prompt.md b/agent-context/evals/skill-triggers/metaobjects-codegen/04-pos-regen-hand-edits/prompt.md new file mode 100644 index 000000000..b5bc67095 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-codegen/04-pos-regen-hand-edits/prompt.md @@ -0,0 +1,8 @@ +--- +max_turns: 3 +timeout_seconds: 180 +allowed_tools: [Skill] +runs: 3 +--- + +This project uses MetaObjects (it has .metaobjects/config.json and a metaobjects/ directory of meta.*.json files). I hand-edited a generated entity file and now `meta gen` reports a merge conflict. How do I keep my edit? diff --git a/agent-context/evals/skill-triggers/metaobjects-codegen/05-pos-json-schema-output/graders/trigger.md b/agent-context/evals/skill-triggers/metaobjects-codegen/05-pos-json-schema-output/graders/trigger.md new file mode 100644 index 000000000..b8e0fb847 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-codegen/05-pos-json-schema-output/graders/trigger.md @@ -0,0 +1,6 @@ +--- +type: tool_used +tool: Skill +input_match: metaobjects-codegen +min: 1 +--- diff --git a/agent-context/evals/skill-triggers/metaobjects-codegen/05-pos-json-schema-output/prompt.md b/agent-context/evals/skill-triggers/metaobjects-codegen/05-pos-json-schema-output/prompt.md new file mode 100644 index 000000000..44ff8cf61 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-codegen/05-pos-json-schema-output/prompt.md @@ -0,0 +1,8 @@ +--- +max_turns: 3 +timeout_seconds: 180 +allowed_tools: [Skill] +runs: 3 +--- + +This project uses MetaObjects (it has .metaobjects/config.json and a metaobjects/ directory of meta.*.json files). Our frontend team needs JSON Schema files for every entity, derived from the model and kept current. diff --git a/agent-context/evals/skill-triggers/metaobjects-codegen/06-pos-dialect-config/graders/trigger.md b/agent-context/evals/skill-triggers/metaobjects-codegen/06-pos-dialect-config/graders/trigger.md new file mode 100644 index 000000000..b8e0fb847 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-codegen/06-pos-dialect-config/graders/trigger.md @@ -0,0 +1,6 @@ +--- +type: tool_used +tool: Skill +input_match: metaobjects-codegen +min: 1 +--- diff --git a/agent-context/evals/skill-triggers/metaobjects-codegen/06-pos-dialect-config/prompt.md b/agent-context/evals/skill-triggers/metaobjects-codegen/06-pos-dialect-config/prompt.md new file mode 100644 index 000000000..4f8229929 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-codegen/06-pos-dialect-config/prompt.md @@ -0,0 +1,8 @@ +--- +max_turns: 3 +timeout_seconds: 180 +allowed_tools: [Skill] +runs: 3 +--- + +This project uses MetaObjects (it has .metaobjects/config.json and a metaobjects/ directory of meta.*.json files). Change the dialect that code generation uses from postgres to sqlite in the gen configuration. diff --git a/agent-context/evals/skill-triggers/metaobjects-codegen/07-neg-add-entity-field/graders/trigger.md b/agent-context/evals/skill-triggers/metaobjects-codegen/07-neg-add-entity-field/graders/trigger.md new file mode 100644 index 000000000..5a6f0d612 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-codegen/07-neg-add-entity-field/graders/trigger.md @@ -0,0 +1,8 @@ +--- +type: tool_used +tool: Skill +input_match: metaobjects-codegen +min: 0 +max: 0 +arm: both +--- diff --git a/agent-context/evals/skill-triggers/metaobjects-codegen/07-neg-add-entity-field/prompt.md b/agent-context/evals/skill-triggers/metaobjects-codegen/07-neg-add-entity-field/prompt.md new file mode 100644 index 000000000..ddec56d5f --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-codegen/07-neg-add-entity-field/prompt.md @@ -0,0 +1,8 @@ +--- +max_turns: 3 +timeout_seconds: 180 +allowed_tools: [Skill] +runs: 3 +--- + +This project uses MetaObjects (it has .metaobjects/config.json and a metaobjects/ directory of meta.*.json files). Add a nullable `nickname` string field to the User entity. Do not generate anything yet. diff --git a/agent-context/evals/skill-triggers/metaobjects-codegen/08-neg-verify-failure/graders/trigger.md b/agent-context/evals/skill-triggers/metaobjects-codegen/08-neg-verify-failure/graders/trigger.md new file mode 100644 index 000000000..5a6f0d612 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-codegen/08-neg-verify-failure/graders/trigger.md @@ -0,0 +1,8 @@ +--- +type: tool_used +tool: Skill +input_match: metaobjects-codegen +min: 0 +max: 0 +arm: both +--- diff --git a/agent-context/evals/skill-triggers/metaobjects-codegen/08-neg-verify-failure/prompt.md b/agent-context/evals/skill-triggers/metaobjects-codegen/08-neg-verify-failure/prompt.md new file mode 100644 index 000000000..b9fb40001 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-codegen/08-neg-verify-failure/prompt.md @@ -0,0 +1,8 @@ +--- +max_turns: 3 +timeout_seconds: 180 +allowed_tools: [Skill] +runs: 3 +--- + +This project uses MetaObjects (it has .metaobjects/config.json and a metaobjects/ directory of meta.*.json files). `meta verify --db` reports the live schema drifted from the metadata. Explain what that means. diff --git a/agent-context/evals/skill-triggers/metaobjects-codegen/09-neg-openapi-no-metaobjects/graders/trigger.md b/agent-context/evals/skill-triggers/metaobjects-codegen/09-neg-openapi-no-metaobjects/graders/trigger.md new file mode 100644 index 000000000..5a6f0d612 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-codegen/09-neg-openapi-no-metaobjects/graders/trigger.md @@ -0,0 +1,8 @@ +--- +type: tool_used +tool: Skill +input_match: metaobjects-codegen +min: 0 +max: 0 +arm: both +--- diff --git a/agent-context/evals/skill-triggers/metaobjects-codegen/09-neg-openapi-no-metaobjects/prompt.md b/agent-context/evals/skill-triggers/metaobjects-codegen/09-neg-openapi-no-metaobjects/prompt.md new file mode 100644 index 000000000..b3b2a240b --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-codegen/09-neg-openapi-no-metaobjects/prompt.md @@ -0,0 +1,8 @@ +--- +max_turns: 3 +timeout_seconds: 180 +allowed_tools: [Skill] +runs: 3 +--- + +This is a plain project; it does not use MetaObjects. Generate an OpenAPI document from our FastAPI routes. diff --git a/agent-context/evals/skill-triggers/metaobjects-codegen/10-neg-prisma-codegen/graders/trigger.md b/agent-context/evals/skill-triggers/metaobjects-codegen/10-neg-prisma-codegen/graders/trigger.md new file mode 100644 index 000000000..5a6f0d612 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-codegen/10-neg-prisma-codegen/graders/trigger.md @@ -0,0 +1,8 @@ +--- +type: tool_used +tool: Skill +input_match: metaobjects-codegen +min: 0 +max: 0 +arm: both +--- diff --git a/agent-context/evals/skill-triggers/metaobjects-codegen/10-neg-prisma-codegen/prompt.md b/agent-context/evals/skill-triggers/metaobjects-codegen/10-neg-prisma-codegen/prompt.md new file mode 100644 index 000000000..852dc29d7 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-codegen/10-neg-prisma-codegen/prompt.md @@ -0,0 +1,8 @@ +--- +max_turns: 3 +timeout_seconds: 180 +allowed_tools: [Skill] +runs: 3 +--- + +This is a plain project; it does not use MetaObjects. Our `prisma generate` step fails in CI. Fix the CI job. diff --git a/agent-context/evals/skill-triggers/metaobjects-codegen/11-neg-prompt-template/graders/trigger.md b/agent-context/evals/skill-triggers/metaobjects-codegen/11-neg-prompt-template/graders/trigger.md new file mode 100644 index 000000000..5a6f0d612 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-codegen/11-neg-prompt-template/graders/trigger.md @@ -0,0 +1,8 @@ +--- +type: tool_used +tool: Skill +input_match: metaobjects-codegen +min: 0 +max: 0 +arm: both +--- diff --git a/agent-context/evals/skill-triggers/metaobjects-codegen/11-neg-prompt-template/prompt.md b/agent-context/evals/skill-triggers/metaobjects-codegen/11-neg-prompt-template/prompt.md new file mode 100644 index 000000000..7f641a885 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-codegen/11-neg-prompt-template/prompt.md @@ -0,0 +1,8 @@ +--- +max_turns: 3 +timeout_seconds: 180 +allowed_tools: [Skill] +runs: 3 +--- + +This project uses MetaObjects (it has .metaobjects/config.json and a metaobjects/ directory of meta.*.json files). Declare a template.prompt for summarizing tickets with a typed payload. diff --git a/agent-context/evals/skill-triggers/metaobjects-codegen/12-neg-runtime-query/graders/trigger.md b/agent-context/evals/skill-triggers/metaobjects-codegen/12-neg-runtime-query/graders/trigger.md new file mode 100644 index 000000000..5a6f0d612 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-codegen/12-neg-runtime-query/graders/trigger.md @@ -0,0 +1,8 @@ +--- +type: tool_used +tool: Skill +input_match: metaobjects-codegen +min: 0 +max: 0 +arm: both +--- diff --git a/agent-context/evals/skill-triggers/metaobjects-codegen/12-neg-runtime-query/prompt.md b/agent-context/evals/skill-triggers/metaobjects-codegen/12-neg-runtime-query/prompt.md new file mode 100644 index 000000000..5c35bc114 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-codegen/12-neg-runtime-query/prompt.md @@ -0,0 +1,8 @@ +--- +max_turns: 3 +timeout_seconds: 180 +allowed_tools: [Skill] +runs: 3 +--- + +This project uses MetaObjects (it has .metaobjects/config.json and a metaobjects/ directory of meta.*.json files). Use the ObjectManager at runtime to list all active users, sorted by name. diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/01-pos-fstring-prompt/graders/trigger.md b/agent-context/evals/skill-triggers/metaobjects-prompts/01-pos-fstring-prompt/graders/trigger.md new file mode 100644 index 000000000..ba923f268 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/01-pos-fstring-prompt/graders/trigger.md @@ -0,0 +1,6 @@ +--- +type: tool_used +tool: Skill +input_match: metaobjects-prompts +min: 1 +--- diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/01-pos-fstring-prompt/prompt.md b/agent-context/evals/skill-triggers/metaobjects-prompts/01-pos-fstring-prompt/prompt.md new file mode 100644 index 000000000..fafc4da6c --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/01-pos-fstring-prompt/prompt.md @@ -0,0 +1,8 @@ +--- +max_turns: 3 +timeout_seconds: 180 +allowed_tools: [Skill] +runs: 3 +--- + +This project uses MetaObjects (it has .metaobjects/config.json and a metaobjects/ directory of meta.*.json files). It declares a template.prompt named SummarizeTicket. In src/summarize.py I build the LLM request with an f-string: f"Summarize this ticket: {ticket.title}\n{ticket.body}". Add the customer's plan tier to what the model sees. diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/02-pos-declare-template-prompt/graders/trigger.md b/agent-context/evals/skill-triggers/metaobjects-prompts/02-pos-declare-template-prompt/graders/trigger.md new file mode 100644 index 000000000..ba923f268 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/02-pos-declare-template-prompt/graders/trigger.md @@ -0,0 +1,6 @@ +--- +type: tool_used +tool: Skill +input_match: metaobjects-prompts +min: 1 +--- diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/02-pos-declare-template-prompt/prompt.md b/agent-context/evals/skill-triggers/metaobjects-prompts/02-pos-declare-template-prompt/prompt.md new file mode 100644 index 000000000..5ab891d5b --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/02-pos-declare-template-prompt/prompt.md @@ -0,0 +1,8 @@ +--- +max_turns: 3 +timeout_seconds: 180 +allowed_tools: [Skill] +runs: 3 +--- + +This project uses MetaObjects (it has .metaobjects/config.json and a metaobjects/ directory of meta.*.json files). Declare a new template.prompt for classifying support tickets, with a typed payload projection and a response model. diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/03-pos-regex-reply-parse/graders/trigger.md b/agent-context/evals/skill-triggers/metaobjects-prompts/03-pos-regex-reply-parse/graders/trigger.md new file mode 100644 index 000000000..ba923f268 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/03-pos-regex-reply-parse/graders/trigger.md @@ -0,0 +1,6 @@ +--- +type: tool_used +tool: Skill +input_match: metaobjects-prompts +min: 1 +--- diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/03-pos-regex-reply-parse/prompt.md b/agent-context/evals/skill-triggers/metaobjects-prompts/03-pos-regex-reply-parse/prompt.md new file mode 100644 index 000000000..c6cc56c7e --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/03-pos-regex-reply-parse/prompt.md @@ -0,0 +1,8 @@ +--- +max_turns: 3 +timeout_seconds: 180 +allowed_tools: [Skill] +runs: 3 +--- + +This project uses MetaObjects (it has .metaobjects/config.json and a metaobjects/ directory of meta.*.json files). It declares a template.prompt whose reply we read. src/triage.ts reads the model's answer with a regex like /Category: (\w+)/. The regex breaks when the model adds a preamble. Fix the parsing. diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/04-pos-test-harness-prompt/graders/trigger.md b/agent-context/evals/skill-triggers/metaobjects-prompts/04-pos-test-harness-prompt/graders/trigger.md new file mode 100644 index 000000000..ba923f268 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/04-pos-test-harness-prompt/graders/trigger.md @@ -0,0 +1,6 @@ +--- +type: tool_used +tool: Skill +input_match: metaobjects-prompts +min: 1 +--- diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/04-pos-test-harness-prompt/prompt.md b/agent-context/evals/skill-triggers/metaobjects-prompts/04-pos-test-harness-prompt/prompt.md new file mode 100644 index 000000000..aeac947f1 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/04-pos-test-harness-prompt/prompt.md @@ -0,0 +1,8 @@ +--- +max_turns: 3 +timeout_seconds: 180 +allowed_tools: [Skill] +runs: 3 +--- + +This project uses MetaObjects (it has .metaobjects/config.json and a metaobjects/ directory of meta.*.json files). It declares template.prompt nodes. A test script in tests/eval_harness.py concatenates strings to build the text it sends to the LLM. Add one more instruction line to that text. diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/05-pos-prompt-drift-verify/graders/trigger.md b/agent-context/evals/skill-triggers/metaobjects-prompts/05-pos-prompt-drift-verify/graders/trigger.md new file mode 100644 index 000000000..ba923f268 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/05-pos-prompt-drift-verify/graders/trigger.md @@ -0,0 +1,6 @@ +--- +type: tool_used +tool: Skill +input_match: metaobjects-prompts +min: 1 +--- diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/05-pos-prompt-drift-verify/prompt.md b/agent-context/evals/skill-triggers/metaobjects-prompts/05-pos-prompt-drift-verify/prompt.md new file mode 100644 index 000000000..0becb8f09 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/05-pos-prompt-drift-verify/prompt.md @@ -0,0 +1,8 @@ +--- +max_turns: 3 +timeout_seconds: 180 +allowed_tools: [Skill] +runs: 3 +--- + +This project uses MetaObjects (it has .metaobjects/config.json and a metaobjects/ directory of meta.*.json files). I renamed a field on the payload projection of the ReviewDraft template.prompt. Update the prompt template and its payload so rendering stays in sync. diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/06-pos-review-json-parse/graders/trigger.md b/agent-context/evals/skill-triggers/metaobjects-prompts/06-pos-review-json-parse/graders/trigger.md new file mode 100644 index 000000000..ba923f268 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/06-pos-review-json-parse/graders/trigger.md @@ -0,0 +1,6 @@ +--- +type: tool_used +tool: Skill +input_match: metaobjects-prompts +min: 1 +--- diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/06-pos-review-json-parse/prompt.md b/agent-context/evals/skill-triggers/metaobjects-prompts/06-pos-review-json-parse/prompt.md new file mode 100644 index 000000000..96760e12d --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/06-pos-review-json-parse/prompt.md @@ -0,0 +1,8 @@ +--- +max_turns: 3 +timeout_seconds: 180 +allowed_tools: [Skill] +runs: 3 +--- + +This project uses MetaObjects (it has .metaobjects/config.json and a metaobjects/ directory of meta.*.json files). It declares template.prompt nodes. Review this diff: it adds JSON.parse(completion.text) in src/agent.ts to read an LLM reply. diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/07-neg-fstring-no-metaobjects/graders/trigger.md b/agent-context/evals/skill-triggers/metaobjects-prompts/07-neg-fstring-no-metaobjects/graders/trigger.md new file mode 100644 index 000000000..7b6368ff0 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/07-neg-fstring-no-metaobjects/graders/trigger.md @@ -0,0 +1,8 @@ +--- +type: tool_used +tool: Skill +input_match: metaobjects-prompts +min: 0 +max: 0 +arm: both +--- diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/07-neg-fstring-no-metaobjects/prompt.md b/agent-context/evals/skill-triggers/metaobjects-prompts/07-neg-fstring-no-metaobjects/prompt.md new file mode 100644 index 000000000..c4c0238fb --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/07-neg-fstring-no-metaobjects/prompt.md @@ -0,0 +1,8 @@ +--- +max_turns: 3 +timeout_seconds: 180 +allowed_tools: [Skill] +runs: 3 +--- + +This is a plain project; it does not use MetaObjects. In src/summarize.py I build an OpenAI request with an f-string: f"Summarize this ticket: {ticket.title}". Add the customer's plan tier to the text. diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/08-neg-langchain-system-prompt/graders/trigger.md b/agent-context/evals/skill-triggers/metaobjects-prompts/08-neg-langchain-system-prompt/graders/trigger.md new file mode 100644 index 000000000..7b6368ff0 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/08-neg-langchain-system-prompt/graders/trigger.md @@ -0,0 +1,8 @@ +--- +type: tool_used +tool: Skill +input_match: metaobjects-prompts +min: 0 +max: 0 +arm: both +--- diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/08-neg-langchain-system-prompt/prompt.md b/agent-context/evals/skill-triggers/metaobjects-prompts/08-neg-langchain-system-prompt/prompt.md new file mode 100644 index 000000000..4f153983e --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/08-neg-langchain-system-prompt/prompt.md @@ -0,0 +1,8 @@ +--- +max_turns: 3 +timeout_seconds: 180 +allowed_tools: [Skill] +runs: 3 +--- + +This is a plain project; it does not use MetaObjects. It is a LangChain app. Tighten the wording of the system prompt in agent/prompts.py so the bot answers more briefly. diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/09-neg-json-reply-no-metaobjects/graders/trigger.md b/agent-context/evals/skill-triggers/metaobjects-prompts/09-neg-json-reply-no-metaobjects/graders/trigger.md new file mode 100644 index 000000000..7b6368ff0 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/09-neg-json-reply-no-metaobjects/graders/trigger.md @@ -0,0 +1,8 @@ +--- +type: tool_used +tool: Skill +input_match: metaobjects-prompts +min: 0 +max: 0 +arm: both +--- diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/09-neg-json-reply-no-metaobjects/prompt.md b/agent-context/evals/skill-triggers/metaobjects-prompts/09-neg-json-reply-no-metaobjects/prompt.md new file mode 100644 index 000000000..f36f5f39b --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/09-neg-json-reply-no-metaobjects/prompt.md @@ -0,0 +1,8 @@ +--- +max_turns: 3 +timeout_seconds: 180 +allowed_tools: [Skill] +runs: 3 +--- + +This is a plain project; it does not use MetaObjects. src/extract.ts calls an LLM and runs JSON.parse on the reply. Add zod validation to that parse. diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/10-neg-entities-only-project-prompt-string/graders/trigger.md b/agent-context/evals/skill-triggers/metaobjects-prompts/10-neg-entities-only-project-prompt-string/graders/trigger.md new file mode 100644 index 000000000..7b6368ff0 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/10-neg-entities-only-project-prompt-string/graders/trigger.md @@ -0,0 +1,8 @@ +--- +type: tool_used +tool: Skill +input_match: metaobjects-prompts +min: 0 +max: 0 +arm: both +--- diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/10-neg-entities-only-project-prompt-string/prompt.md b/agent-context/evals/skill-triggers/metaobjects-prompts/10-neg-entities-only-project-prompt-string/prompt.md new file mode 100644 index 000000000..7edf64c55 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/10-neg-entities-only-project-prompt-string/prompt.md @@ -0,0 +1,8 @@ +--- +max_turns: 3 +timeout_seconds: 180 +allowed_tools: [Skill] +runs: 3 +--- + +This project uses MetaObjects (it has .metaobjects/config.json and a metaobjects/ directory of meta.*.json files). It declares entities only; no template.prompt exists and none is wanted. In src/helpers/greeter.py there is a one-line prompt string passed to the LLM client. Change the greeting from formal to casual. diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/11-neg-add-entity-field/graders/trigger.md b/agent-context/evals/skill-triggers/metaobjects-prompts/11-neg-add-entity-field/graders/trigger.md new file mode 100644 index 000000000..7b6368ff0 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/11-neg-add-entity-field/graders/trigger.md @@ -0,0 +1,8 @@ +--- +type: tool_used +tool: Skill +input_match: metaobjects-prompts +min: 0 +max: 0 +arm: both +--- diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/11-neg-add-entity-field/prompt.md b/agent-context/evals/skill-triggers/metaobjects-prompts/11-neg-add-entity-field/prompt.md new file mode 100644 index 000000000..4e81d2e40 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/11-neg-add-entity-field/prompt.md @@ -0,0 +1,8 @@ +--- +max_turns: 3 +timeout_seconds: 180 +allowed_tools: [Skill] +runs: 3 +--- + +This project uses MetaObjects (it has .metaobjects/config.json and a metaobjects/ directory of meta.*.json files). Add a nullable `nickname` string field to the User entity. diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/12-neg-non-llm-string-building/graders/trigger.md b/agent-context/evals/skill-triggers/metaobjects-prompts/12-neg-non-llm-string-building/graders/trigger.md new file mode 100644 index 000000000..7b6368ff0 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/12-neg-non-llm-string-building/graders/trigger.md @@ -0,0 +1,8 @@ +--- +type: tool_used +tool: Skill +input_match: metaobjects-prompts +min: 0 +max: 0 +arm: both +--- diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/12-neg-non-llm-string-building/prompt.md b/agent-context/evals/skill-triggers/metaobjects-prompts/12-neg-non-llm-string-building/prompt.md new file mode 100644 index 000000000..bf7ec06bc --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/12-neg-non-llm-string-building/prompt.md @@ -0,0 +1,8 @@ +--- +max_turns: 3 +timeout_seconds: 180 +allowed_tools: [Skill] +runs: 3 +--- + +This project uses MetaObjects (it has .metaobjects/config.json and a metaobjects/ directory of meta.*.json files). In src/mail.ts the welcome e-mail body is built with template literals. Add the user's first name to the greeting. No LLM is involved. diff --git a/agent-context/skills/metaobjects-prompts/SKILL.md b/agent-context/skills/metaobjects-prompts/SKILL.md index 66a26ec73..425bb4707 100644 --- a/agent-context/skills/metaobjects-prompts/SKILL.md +++ b/agent-context/skills/metaobjects-prompts/SKILL.md @@ -1,6 +1,6 @@ --- name: metaobjects-prompts -description: Use when writing, editing or reviewing ANY code that builds text sent to an LLM (string formatting, f-strings, text blocks, template literals, string builders, concatenation) or reads an LLM reply (regex, string search, JSON or XML parsing), even in a test harness or script. In a project that declares a template.prompt, that code belongs in a template, a payload VO and a response model. Also for declaring template.prompt/template.output, typed payload projections, provider-resolved text, deterministic render, prompt-drift verify and parser-on-receipt. +description: Use when writing, editing or reviewing code in a project that declares template.prompt nodes and that builds text sent to an LLM (string formatting, f-strings, text blocks, template literals, string builders, concatenation) or reads an LLM reply (regex, string search, JSON or XML parsing); test harnesses and scripts count. That code belongs in a template, a payload VO and a response model. Also for declaring template.prompt/template.output, typed payload projections, provider-resolved text, deterministic render, prompt-drift verify and parser-on-receipt. Do not use for LLM code in a project that declares no template.prompt. --- # MetaObjects prompt construction diff --git a/fixtures/agent-context-conformance/java-kotlin-react-tanstack/expected/.claude/skills/metaobjects-prompts/SKILL.md b/fixtures/agent-context-conformance/java-kotlin-react-tanstack/expected/.claude/skills/metaobjects-prompts/SKILL.md index 66a26ec73..425bb4707 100644 --- a/fixtures/agent-context-conformance/java-kotlin-react-tanstack/expected/.claude/skills/metaobjects-prompts/SKILL.md +++ b/fixtures/agent-context-conformance/java-kotlin-react-tanstack/expected/.claude/skills/metaobjects-prompts/SKILL.md @@ -1,6 +1,6 @@ --- name: metaobjects-prompts -description: Use when writing, editing or reviewing ANY code that builds text sent to an LLM (string formatting, f-strings, text blocks, template literals, string builders, concatenation) or reads an LLM reply (regex, string search, JSON or XML parsing), even in a test harness or script. In a project that declares a template.prompt, that code belongs in a template, a payload VO and a response model. Also for declaring template.prompt/template.output, typed payload projections, provider-resolved text, deterministic render, prompt-drift verify and parser-on-receipt. +description: Use when writing, editing or reviewing code in a project that declares template.prompt nodes and that builds text sent to an LLM (string formatting, f-strings, text blocks, template literals, string builders, concatenation) or reads an LLM reply (regex, string search, JSON or XML parsing); test harnesses and scripts count. That code belongs in a template, a payload VO and a response model. Also for declaring template.prompt/template.output, typed payload projections, provider-resolved text, deterministic render, prompt-drift verify and parser-on-receipt. Do not use for LLM code in a project that declares no template.prompt. --- # MetaObjects prompt construction diff --git a/fixtures/agent-context-conformance/java-react/expected/.claude/skills/metaobjects-prompts/SKILL.md b/fixtures/agent-context-conformance/java-react/expected/.claude/skills/metaobjects-prompts/SKILL.md index 66a26ec73..425bb4707 100644 --- a/fixtures/agent-context-conformance/java-react/expected/.claude/skills/metaobjects-prompts/SKILL.md +++ b/fixtures/agent-context-conformance/java-react/expected/.claude/skills/metaobjects-prompts/SKILL.md @@ -1,6 +1,6 @@ --- name: metaobjects-prompts -description: Use when writing, editing or reviewing ANY code that builds text sent to an LLM (string formatting, f-strings, text blocks, template literals, string builders, concatenation) or reads an LLM reply (regex, string search, JSON or XML parsing), even in a test harness or script. In a project that declares a template.prompt, that code belongs in a template, a payload VO and a response model. Also for declaring template.prompt/template.output, typed payload projections, provider-resolved text, deterministic render, prompt-drift verify and parser-on-receipt. +description: Use when writing, editing or reviewing code in a project that declares template.prompt nodes and that builds text sent to an LLM (string formatting, f-strings, text blocks, template literals, string builders, concatenation) or reads an LLM reply (regex, string search, JSON or XML parsing); test harnesses and scripts count. That code belongs in a template, a payload VO and a response model. Also for declaring template.prompt/template.output, typed payload projections, provider-resolved text, deterministic render, prompt-drift verify and parser-on-receipt. Do not use for LLM code in a project that declares no template.prompt. --- # MetaObjects prompt construction diff --git a/fixtures/agent-context-conformance/python/expected/.claude/skills/metaobjects-prompts/SKILL.md b/fixtures/agent-context-conformance/python/expected/.claude/skills/metaobjects-prompts/SKILL.md index 66a26ec73..425bb4707 100644 --- a/fixtures/agent-context-conformance/python/expected/.claude/skills/metaobjects-prompts/SKILL.md +++ b/fixtures/agent-context-conformance/python/expected/.claude/skills/metaobjects-prompts/SKILL.md @@ -1,6 +1,6 @@ --- name: metaobjects-prompts -description: Use when writing, editing or reviewing ANY code that builds text sent to an LLM (string formatting, f-strings, text blocks, template literals, string builders, concatenation) or reads an LLM reply (regex, string search, JSON or XML parsing), even in a test harness or script. In a project that declares a template.prompt, that code belongs in a template, a payload VO and a response model. Also for declaring template.prompt/template.output, typed payload projections, provider-resolved text, deterministic render, prompt-drift verify and parser-on-receipt. +description: Use when writing, editing or reviewing code in a project that declares template.prompt nodes and that builds text sent to an LLM (string formatting, f-strings, text blocks, template literals, string builders, concatenation) or reads an LLM reply (regex, string search, JSON or XML parsing); test harnesses and scripts count. That code belongs in a template, a payload VO and a response model. Also for declaring template.prompt/template.output, typed payload projections, provider-resolved text, deterministic render, prompt-drift verify and parser-on-receipt. Do not use for LLM code in a project that declares no template.prompt. --- # MetaObjects prompt construction diff --git a/fixtures/agent-context-conformance/ts-react-tanstack/expected/.claude/skills/metaobjects-prompts/SKILL.md b/fixtures/agent-context-conformance/ts-react-tanstack/expected/.claude/skills/metaobjects-prompts/SKILL.md index 66a26ec73..425bb4707 100644 --- a/fixtures/agent-context-conformance/ts-react-tanstack/expected/.claude/skills/metaobjects-prompts/SKILL.md +++ b/fixtures/agent-context-conformance/ts-react-tanstack/expected/.claude/skills/metaobjects-prompts/SKILL.md @@ -1,6 +1,6 @@ --- name: metaobjects-prompts -description: Use when writing, editing or reviewing ANY code that builds text sent to an LLM (string formatting, f-strings, text blocks, template literals, string builders, concatenation) or reads an LLM reply (regex, string search, JSON or XML parsing), even in a test harness or script. In a project that declares a template.prompt, that code belongs in a template, a payload VO and a response model. Also for declaring template.prompt/template.output, typed payload projections, provider-resolved text, deterministic render, prompt-drift verify and parser-on-receipt. +description: Use when writing, editing or reviewing code in a project that declares template.prompt nodes and that builds text sent to an LLM (string formatting, f-strings, text blocks, template literals, string builders, concatenation) or reads an LLM reply (regex, string search, JSON or XML parsing); test harnesses and scripts count. That code belongs in a template, a payload VO and a response model. Also for declaring template.prompt/template.output, typed payload projections, provider-resolved text, deterministic render, prompt-drift verify and parser-on-receipt. Do not use for LLM code in a project that declares no template.prompt. --- # MetaObjects prompt construction diff --git a/fixtures/agent-context-conformance/ts-requirements/expected/.claude/skills/metaobjects-prompts/SKILL.md b/fixtures/agent-context-conformance/ts-requirements/expected/.claude/skills/metaobjects-prompts/SKILL.md index 66a26ec73..425bb4707 100644 --- a/fixtures/agent-context-conformance/ts-requirements/expected/.claude/skills/metaobjects-prompts/SKILL.md +++ b/fixtures/agent-context-conformance/ts-requirements/expected/.claude/skills/metaobjects-prompts/SKILL.md @@ -1,6 +1,6 @@ --- name: metaobjects-prompts -description: Use when writing, editing or reviewing ANY code that builds text sent to an LLM (string formatting, f-strings, text blocks, template literals, string builders, concatenation) or reads an LLM reply (regex, string search, JSON or XML parsing), even in a test harness or script. In a project that declares a template.prompt, that code belongs in a template, a payload VO and a response model. Also for declaring template.prompt/template.output, typed payload projections, provider-resolved text, deterministic render, prompt-drift verify and parser-on-receipt. +description: Use when writing, editing or reviewing code in a project that declares template.prompt nodes and that builds text sent to an LLM (string formatting, f-strings, text blocks, template literals, string builders, concatenation) or reads an LLM reply (regex, string search, JSON or XML parsing); test harnesses and scripts count. That code belongs in a template, a payload VO and a response model. Also for declaring template.prompt/template.output, typed payload projections, provider-resolved text, deterministic render, prompt-drift verify and parser-on-receipt. Do not use for LLM code in a project that declares no template.prompt. --- # MetaObjects prompt construction diff --git a/server/typescript/packages/sdk/test/agent-context/skill-trigger-evals.test.ts b/server/typescript/packages/sdk/test/agent-context/skill-trigger-evals.test.ts new file mode 100644 index 000000000..ecd799190 --- /dev/null +++ b/server/typescript/packages/sdk/test/agent-context/skill-trigger-evals.test.ts @@ -0,0 +1,68 @@ +// test/agent-context/skill-trigger-evals.test.ts +// +// The trigger evaluation itself drives live model sessions and cannot run in CI +// (agent-context/evals/README.md has the manual command). This keeps the part that is +// deterministic: every case parses, names a skill that exists, and each skill carries +// both should-trigger and should-not-trigger cases. +import { test, expect, describe } from "bun:test"; +import { readFileSync, readdirSync, existsSync } from "node:fs"; +import { join } from "node:path"; + +const AGENT_CONTEXT = join(import.meta.dir, "../../../../../../agent-context"); +const SKILLS = new Set(readdirSync(join(AGENT_CONTEXT, "skills"))); +const EVALS = join(AGENT_CONTEXT, "evals/skill-triggers"); + +const frontmatter = (text: string): Record => { + const m = /^---\n([\s\S]*?)\n---\n/.exec(text); + if (!m) throw new Error("no frontmatter"); + const out: Record = {}; + for (const line of m[1]!.split("\n")) { + const kv = /^([A-Za-z_]+):\s*(.*)$/.exec(line); + if (kv) out[kv[1]!] = kv[2]!; + } + return out; +}; + +const cases = readdirSync(EVALS).flatMap((skill) => + readdirSync(join(EVALS, skill)).map((name) => ({ skill, name, dir: join(EVALS, skill, name) })), +); + +describe("skill trigger eval cases", () => { + test("only the two evaluated skills have case sets", () => { + expect(readdirSync(EVALS).sort()).toEqual(["metaobjects-codegen", "metaobjects-prompts"]); + }); + + test("every case directory parses and names an existing skill", () => { + for (const c of cases) { + expect(SKILLS.has(c.skill)).toBe(true); + const prompt = join(c.dir, "prompt.md"); + expect(existsSync(prompt)).toBe(true); + const body = readFileSync(prompt, "utf8").replace(/^---\n[\s\S]*?\n---\n/, "").trim(); + expect(body.length).toBeGreaterThan(0); + const graders = readdirSync(join(c.dir, "graders")); + expect(graders.length).toBeGreaterThan(0); + for (const g of graders) { + const fm = frontmatter(readFileSync(join(c.dir, "graders", g), "utf8")); + expect(fm.type).toBe("tool_used"); + expect(fm.tool).toBe("Skill"); + expect(fm.input_match).toBe(c.skill); + } + } + }); + + test("each skill has should-trigger and should-not-trigger cases", () => { + for (const skill of readdirSync(EVALS)) { + const kinds = cases.filter((c) => c.skill === skill).map((c) => (c.name.includes("-pos-") ? "pos" : "neg")); + expect(kinds.filter((k) => k === "pos").length).toBeGreaterThanOrEqual(5); + expect(kinds.filter((k) => k === "neg").length).toBeGreaterThanOrEqual(5); + } + }); + + test("positive cases require a firing, negative cases forbid one", () => { + for (const c of cases) { + const fm = frontmatter(readFileSync(join(c.dir, "graders", "trigger.md"), "utf8")); + if (c.name.includes("-pos-")) expect(fm.min).toBe("1"); + else expect([fm.min, fm.max]).toEqual(["0", "0"]); + } + }); +}); From 578703cc8cf7204b166fd67c414411fc819614c6 Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Wed, 14 Oct 2026 08:11:25 -0400 Subject: [PATCH 2/4] no-mistakes(review): Scope prompts skill gate; add adoption eval case --- agent-context/evals/README.md | 2 +- .../13-pos-adopt-first-prompt/graders/trigger.md | 6 ++++++ .../13-pos-adopt-first-prompt/prompt.md | 8 ++++++++ agent-context/skills/metaobjects-prompts/SKILL.md | 2 +- .../expected/.claude/skills/metaobjects-prompts/SKILL.md | 2 +- .../expected/.claude/skills/metaobjects-prompts/SKILL.md | 2 +- .../expected/.claude/skills/metaobjects-prompts/SKILL.md | 2 +- .../expected/.claude/skills/metaobjects-prompts/SKILL.md | 2 +- .../expected/.claude/skills/metaobjects-prompts/SKILL.md | 2 +- 9 files changed, 21 insertions(+), 7 deletions(-) create mode 100644 agent-context/evals/skill-triggers/metaobjects-prompts/13-pos-adopt-first-prompt/graders/trigger.md create mode 100644 agent-context/evals/skill-triggers/metaobjects-prompts/13-pos-adopt-first-prompt/prompt.md diff --git a/agent-context/evals/README.md b/agent-context/evals/README.md index 2532f80a2..f28833144 100644 --- a/agent-context/evals/README.md +++ b/agent-context/evals/README.md @@ -18,7 +18,7 @@ prompts, model edits that need no generated output) forbid it. agent-context/evals/run-skill-triggers.sh ``` -It starts real Claude sessions on your own login (24 cases x 3 runs = 72 sessions, a few +It starts real Claude sessions on your own login (25 cases x 3 runs = 75 sessions, a few minutes with `-j 4`) and writes `/result.json`. Use `--case ''` to run a subset. It is not part of CI because model behaviour is not deterministic; CI only checks that the cases parse and name existing skills diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/13-pos-adopt-first-prompt/graders/trigger.md b/agent-context/evals/skill-triggers/metaobjects-prompts/13-pos-adopt-first-prompt/graders/trigger.md new file mode 100644 index 000000000..ba923f268 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/13-pos-adopt-first-prompt/graders/trigger.md @@ -0,0 +1,6 @@ +--- +type: tool_used +tool: Skill +input_match: metaobjects-prompts +min: 1 +--- diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/13-pos-adopt-first-prompt/prompt.md b/agent-context/evals/skill-triggers/metaobjects-prompts/13-pos-adopt-first-prompt/prompt.md new file mode 100644 index 000000000..6662ae869 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/13-pos-adopt-first-prompt/prompt.md @@ -0,0 +1,8 @@ +--- +max_turns: 3 +timeout_seconds: 180 +allowed_tools: [Skill] +runs: 3 +--- + +This project uses MetaObjects (it has .metaobjects/config.json and a metaobjects/ directory of meta.*.json files) but has not declared any template.prompt yet. Our ticket summarizer builds its LLM prompt with an f-string in summarize.py. I want to start declaring prompts the MetaObjects way, so move that prompt onto it. diff --git a/agent-context/skills/metaobjects-prompts/SKILL.md b/agent-context/skills/metaobjects-prompts/SKILL.md index 425bb4707..011c28dc8 100644 --- a/agent-context/skills/metaobjects-prompts/SKILL.md +++ b/agent-context/skills/metaobjects-prompts/SKILL.md @@ -1,6 +1,6 @@ --- name: metaobjects-prompts -description: Use when writing, editing or reviewing code in a project that declares template.prompt nodes and that builds text sent to an LLM (string formatting, f-strings, text blocks, template literals, string builders, concatenation) or reads an LLM reply (regex, string search, JSON or XML parsing); test harnesses and scripts count. That code belongs in a template, a payload VO and a response model. Also for declaring template.prompt/template.output, typed payload projections, provider-resolved text, deterministic render, prompt-drift verify and parser-on-receipt. Do not use for LLM code in a project that declares no template.prompt. +description: Use when writing, editing or reviewing code that builds text sent to an LLM (string formatting, f-strings, text blocks, template literals, string builders, concatenation) or reads an LLM reply (regex, string search, JSON or XML parsing), in a project that declares template.prompt nodes; test harnesses and scripts count. That code belongs in a template, a payload VO and a response model. Also for declaring template.prompt/template.output, typed payload projections, provider-resolved text, deterministic render, prompt-drift verify and parser-on-receipt, including a MetaObjects project that has no template.prompt yet and wants to move existing LLM code onto them. Do not use to edit existing LLM code in a project that declares no template.prompt when the task is not declaring prompts. --- # MetaObjects prompt construction diff --git a/fixtures/agent-context-conformance/java-kotlin-react-tanstack/expected/.claude/skills/metaobjects-prompts/SKILL.md b/fixtures/agent-context-conformance/java-kotlin-react-tanstack/expected/.claude/skills/metaobjects-prompts/SKILL.md index 425bb4707..011c28dc8 100644 --- a/fixtures/agent-context-conformance/java-kotlin-react-tanstack/expected/.claude/skills/metaobjects-prompts/SKILL.md +++ b/fixtures/agent-context-conformance/java-kotlin-react-tanstack/expected/.claude/skills/metaobjects-prompts/SKILL.md @@ -1,6 +1,6 @@ --- name: metaobjects-prompts -description: Use when writing, editing or reviewing code in a project that declares template.prompt nodes and that builds text sent to an LLM (string formatting, f-strings, text blocks, template literals, string builders, concatenation) or reads an LLM reply (regex, string search, JSON or XML parsing); test harnesses and scripts count. That code belongs in a template, a payload VO and a response model. Also for declaring template.prompt/template.output, typed payload projections, provider-resolved text, deterministic render, prompt-drift verify and parser-on-receipt. Do not use for LLM code in a project that declares no template.prompt. +description: Use when writing, editing or reviewing code that builds text sent to an LLM (string formatting, f-strings, text blocks, template literals, string builders, concatenation) or reads an LLM reply (regex, string search, JSON or XML parsing), in a project that declares template.prompt nodes; test harnesses and scripts count. That code belongs in a template, a payload VO and a response model. Also for declaring template.prompt/template.output, typed payload projections, provider-resolved text, deterministic render, prompt-drift verify and parser-on-receipt, including a MetaObjects project that has no template.prompt yet and wants to move existing LLM code onto them. Do not use to edit existing LLM code in a project that declares no template.prompt when the task is not declaring prompts. --- # MetaObjects prompt construction diff --git a/fixtures/agent-context-conformance/java-react/expected/.claude/skills/metaobjects-prompts/SKILL.md b/fixtures/agent-context-conformance/java-react/expected/.claude/skills/metaobjects-prompts/SKILL.md index 425bb4707..011c28dc8 100644 --- a/fixtures/agent-context-conformance/java-react/expected/.claude/skills/metaobjects-prompts/SKILL.md +++ b/fixtures/agent-context-conformance/java-react/expected/.claude/skills/metaobjects-prompts/SKILL.md @@ -1,6 +1,6 @@ --- name: metaobjects-prompts -description: Use when writing, editing or reviewing code in a project that declares template.prompt nodes and that builds text sent to an LLM (string formatting, f-strings, text blocks, template literals, string builders, concatenation) or reads an LLM reply (regex, string search, JSON or XML parsing); test harnesses and scripts count. That code belongs in a template, a payload VO and a response model. Also for declaring template.prompt/template.output, typed payload projections, provider-resolved text, deterministic render, prompt-drift verify and parser-on-receipt. Do not use for LLM code in a project that declares no template.prompt. +description: Use when writing, editing or reviewing code that builds text sent to an LLM (string formatting, f-strings, text blocks, template literals, string builders, concatenation) or reads an LLM reply (regex, string search, JSON or XML parsing), in a project that declares template.prompt nodes; test harnesses and scripts count. That code belongs in a template, a payload VO and a response model. Also for declaring template.prompt/template.output, typed payload projections, provider-resolved text, deterministic render, prompt-drift verify and parser-on-receipt, including a MetaObjects project that has no template.prompt yet and wants to move existing LLM code onto them. Do not use to edit existing LLM code in a project that declares no template.prompt when the task is not declaring prompts. --- # MetaObjects prompt construction diff --git a/fixtures/agent-context-conformance/python/expected/.claude/skills/metaobjects-prompts/SKILL.md b/fixtures/agent-context-conformance/python/expected/.claude/skills/metaobjects-prompts/SKILL.md index 425bb4707..011c28dc8 100644 --- a/fixtures/agent-context-conformance/python/expected/.claude/skills/metaobjects-prompts/SKILL.md +++ b/fixtures/agent-context-conformance/python/expected/.claude/skills/metaobjects-prompts/SKILL.md @@ -1,6 +1,6 @@ --- name: metaobjects-prompts -description: Use when writing, editing or reviewing code in a project that declares template.prompt nodes and that builds text sent to an LLM (string formatting, f-strings, text blocks, template literals, string builders, concatenation) or reads an LLM reply (regex, string search, JSON or XML parsing); test harnesses and scripts count. That code belongs in a template, a payload VO and a response model. Also for declaring template.prompt/template.output, typed payload projections, provider-resolved text, deterministic render, prompt-drift verify and parser-on-receipt. Do not use for LLM code in a project that declares no template.prompt. +description: Use when writing, editing or reviewing code that builds text sent to an LLM (string formatting, f-strings, text blocks, template literals, string builders, concatenation) or reads an LLM reply (regex, string search, JSON or XML parsing), in a project that declares template.prompt nodes; test harnesses and scripts count. That code belongs in a template, a payload VO and a response model. Also for declaring template.prompt/template.output, typed payload projections, provider-resolved text, deterministic render, prompt-drift verify and parser-on-receipt, including a MetaObjects project that has no template.prompt yet and wants to move existing LLM code onto them. Do not use to edit existing LLM code in a project that declares no template.prompt when the task is not declaring prompts. --- # MetaObjects prompt construction diff --git a/fixtures/agent-context-conformance/ts-react-tanstack/expected/.claude/skills/metaobjects-prompts/SKILL.md b/fixtures/agent-context-conformance/ts-react-tanstack/expected/.claude/skills/metaobjects-prompts/SKILL.md index 425bb4707..011c28dc8 100644 --- a/fixtures/agent-context-conformance/ts-react-tanstack/expected/.claude/skills/metaobjects-prompts/SKILL.md +++ b/fixtures/agent-context-conformance/ts-react-tanstack/expected/.claude/skills/metaobjects-prompts/SKILL.md @@ -1,6 +1,6 @@ --- name: metaobjects-prompts -description: Use when writing, editing or reviewing code in a project that declares template.prompt nodes and that builds text sent to an LLM (string formatting, f-strings, text blocks, template literals, string builders, concatenation) or reads an LLM reply (regex, string search, JSON or XML parsing); test harnesses and scripts count. That code belongs in a template, a payload VO and a response model. Also for declaring template.prompt/template.output, typed payload projections, provider-resolved text, deterministic render, prompt-drift verify and parser-on-receipt. Do not use for LLM code in a project that declares no template.prompt. +description: Use when writing, editing or reviewing code that builds text sent to an LLM (string formatting, f-strings, text blocks, template literals, string builders, concatenation) or reads an LLM reply (regex, string search, JSON or XML parsing), in a project that declares template.prompt nodes; test harnesses and scripts count. That code belongs in a template, a payload VO and a response model. Also for declaring template.prompt/template.output, typed payload projections, provider-resolved text, deterministic render, prompt-drift verify and parser-on-receipt, including a MetaObjects project that has no template.prompt yet and wants to move existing LLM code onto them. Do not use to edit existing LLM code in a project that declares no template.prompt when the task is not declaring prompts. --- # MetaObjects prompt construction diff --git a/fixtures/agent-context-conformance/ts-requirements/expected/.claude/skills/metaobjects-prompts/SKILL.md b/fixtures/agent-context-conformance/ts-requirements/expected/.claude/skills/metaobjects-prompts/SKILL.md index 425bb4707..011c28dc8 100644 --- a/fixtures/agent-context-conformance/ts-requirements/expected/.claude/skills/metaobjects-prompts/SKILL.md +++ b/fixtures/agent-context-conformance/ts-requirements/expected/.claude/skills/metaobjects-prompts/SKILL.md @@ -1,6 +1,6 @@ --- name: metaobjects-prompts -description: Use when writing, editing or reviewing code in a project that declares template.prompt nodes and that builds text sent to an LLM (string formatting, f-strings, text blocks, template literals, string builders, concatenation) or reads an LLM reply (regex, string search, JSON or XML parsing); test harnesses and scripts count. That code belongs in a template, a payload VO and a response model. Also for declaring template.prompt/template.output, typed payload projections, provider-resolved text, deterministic render, prompt-drift verify and parser-on-receipt. Do not use for LLM code in a project that declares no template.prompt. +description: Use when writing, editing or reviewing code that builds text sent to an LLM (string formatting, f-strings, text blocks, template literals, string builders, concatenation) or reads an LLM reply (regex, string search, JSON or XML parsing), in a project that declares template.prompt nodes; test harnesses and scripts count. That code belongs in a template, a payload VO and a response model. Also for declaring template.prompt/template.output, typed payload projections, provider-resolved text, deterministic render, prompt-drift verify and parser-on-receipt, including a MetaObjects project that has no template.prompt yet and wants to move existing LLM code onto them. Do not use to edit existing LLM code in a project that declares no template.prompt when the task is not declaring prompts. --- # MetaObjects prompt construction From ff963325cec30b76c55c35b95d5f46db95ef17e7 Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Wed, 14 Oct 2026 09:10:49 -0400 Subject: [PATCH 3/4] no-mistakes(test): Tune codegen skill description to trigger on dialect config --- agent-context/skills/metaobjects-codegen/SKILL.md | 2 +- .../expected/.claude/skills/metaobjects-codegen/SKILL.md | 2 +- .../expected/.claude/skills/metaobjects-codegen/SKILL.md | 2 +- .../python/expected/.claude/skills/metaobjects-codegen/SKILL.md | 2 +- .../expected/.claude/skills/metaobjects-codegen/SKILL.md | 2 +- .../expected/.claude/skills/metaobjects-codegen/SKILL.md | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/agent-context/skills/metaobjects-codegen/SKILL.md b/agent-context/skills/metaobjects-codegen/SKILL.md index c1f048a88..f4b4af165 100644 --- a/agent-context/skills/metaobjects-codegen/SKILL.md +++ b/agent-context/skills/metaobjects-codegen/SKILL.md @@ -1,6 +1,6 @@ --- name: metaobjects-codegen -description: Use when you need ANY output derived from the MetaObjects model — write your own generator (OpenAPI, JSON Schema, Zod, DTOs, a client, docs, anything) or eject a reference one — and when configuring or running code generation, generators/targets/dialect config, the gen command, and hand-edit-preserving regeneration. +description: Use when you need ANY output derived from the MetaObjects model — write your own generator (OpenAPI, JSON Schema, Zod, DTOs, a client, docs, anything) or eject a reference one — and when configuring or running code generation: editing the gen config (metaobjects.config.ts) to choose generators, targets or the SQL dialect (e.g. postgres to sqlite), the gen command, and hand-edit-preserving regeneration. --- # MetaObjects code generation diff --git a/fixtures/agent-context-conformance/java-kotlin-react-tanstack/expected/.claude/skills/metaobjects-codegen/SKILL.md b/fixtures/agent-context-conformance/java-kotlin-react-tanstack/expected/.claude/skills/metaobjects-codegen/SKILL.md index c1f048a88..f4b4af165 100644 --- a/fixtures/agent-context-conformance/java-kotlin-react-tanstack/expected/.claude/skills/metaobjects-codegen/SKILL.md +++ b/fixtures/agent-context-conformance/java-kotlin-react-tanstack/expected/.claude/skills/metaobjects-codegen/SKILL.md @@ -1,6 +1,6 @@ --- name: metaobjects-codegen -description: Use when you need ANY output derived from the MetaObjects model — write your own generator (OpenAPI, JSON Schema, Zod, DTOs, a client, docs, anything) or eject a reference one — and when configuring or running code generation, generators/targets/dialect config, the gen command, and hand-edit-preserving regeneration. +description: Use when you need ANY output derived from the MetaObjects model — write your own generator (OpenAPI, JSON Schema, Zod, DTOs, a client, docs, anything) or eject a reference one — and when configuring or running code generation: editing the gen config (metaobjects.config.ts) to choose generators, targets or the SQL dialect (e.g. postgres to sqlite), the gen command, and hand-edit-preserving regeneration. --- # MetaObjects code generation diff --git a/fixtures/agent-context-conformance/java-react/expected/.claude/skills/metaobjects-codegen/SKILL.md b/fixtures/agent-context-conformance/java-react/expected/.claude/skills/metaobjects-codegen/SKILL.md index c1f048a88..f4b4af165 100644 --- a/fixtures/agent-context-conformance/java-react/expected/.claude/skills/metaobjects-codegen/SKILL.md +++ b/fixtures/agent-context-conformance/java-react/expected/.claude/skills/metaobjects-codegen/SKILL.md @@ -1,6 +1,6 @@ --- name: metaobjects-codegen -description: Use when you need ANY output derived from the MetaObjects model — write your own generator (OpenAPI, JSON Schema, Zod, DTOs, a client, docs, anything) or eject a reference one — and when configuring or running code generation, generators/targets/dialect config, the gen command, and hand-edit-preserving regeneration. +description: Use when you need ANY output derived from the MetaObjects model — write your own generator (OpenAPI, JSON Schema, Zod, DTOs, a client, docs, anything) or eject a reference one — and when configuring or running code generation: editing the gen config (metaobjects.config.ts) to choose generators, targets or the SQL dialect (e.g. postgres to sqlite), the gen command, and hand-edit-preserving regeneration. --- # MetaObjects code generation diff --git a/fixtures/agent-context-conformance/python/expected/.claude/skills/metaobjects-codegen/SKILL.md b/fixtures/agent-context-conformance/python/expected/.claude/skills/metaobjects-codegen/SKILL.md index c1f048a88..f4b4af165 100644 --- a/fixtures/agent-context-conformance/python/expected/.claude/skills/metaobjects-codegen/SKILL.md +++ b/fixtures/agent-context-conformance/python/expected/.claude/skills/metaobjects-codegen/SKILL.md @@ -1,6 +1,6 @@ --- name: metaobjects-codegen -description: Use when you need ANY output derived from the MetaObjects model — write your own generator (OpenAPI, JSON Schema, Zod, DTOs, a client, docs, anything) or eject a reference one — and when configuring or running code generation, generators/targets/dialect config, the gen command, and hand-edit-preserving regeneration. +description: Use when you need ANY output derived from the MetaObjects model — write your own generator (OpenAPI, JSON Schema, Zod, DTOs, a client, docs, anything) or eject a reference one — and when configuring or running code generation: editing the gen config (metaobjects.config.ts) to choose generators, targets or the SQL dialect (e.g. postgres to sqlite), the gen command, and hand-edit-preserving regeneration. --- # MetaObjects code generation diff --git a/fixtures/agent-context-conformance/ts-react-tanstack/expected/.claude/skills/metaobjects-codegen/SKILL.md b/fixtures/agent-context-conformance/ts-react-tanstack/expected/.claude/skills/metaobjects-codegen/SKILL.md index c1f048a88..f4b4af165 100644 --- a/fixtures/agent-context-conformance/ts-react-tanstack/expected/.claude/skills/metaobjects-codegen/SKILL.md +++ b/fixtures/agent-context-conformance/ts-react-tanstack/expected/.claude/skills/metaobjects-codegen/SKILL.md @@ -1,6 +1,6 @@ --- name: metaobjects-codegen -description: Use when you need ANY output derived from the MetaObjects model — write your own generator (OpenAPI, JSON Schema, Zod, DTOs, a client, docs, anything) or eject a reference one — and when configuring or running code generation, generators/targets/dialect config, the gen command, and hand-edit-preserving regeneration. +description: Use when you need ANY output derived from the MetaObjects model — write your own generator (OpenAPI, JSON Schema, Zod, DTOs, a client, docs, anything) or eject a reference one — and when configuring or running code generation: editing the gen config (metaobjects.config.ts) to choose generators, targets or the SQL dialect (e.g. postgres to sqlite), the gen command, and hand-edit-preserving regeneration. --- # MetaObjects code generation diff --git a/fixtures/agent-context-conformance/ts-requirements/expected/.claude/skills/metaobjects-codegen/SKILL.md b/fixtures/agent-context-conformance/ts-requirements/expected/.claude/skills/metaobjects-codegen/SKILL.md index c1f048a88..f4b4af165 100644 --- a/fixtures/agent-context-conformance/ts-requirements/expected/.claude/skills/metaobjects-codegen/SKILL.md +++ b/fixtures/agent-context-conformance/ts-requirements/expected/.claude/skills/metaobjects-codegen/SKILL.md @@ -1,6 +1,6 @@ --- name: metaobjects-codegen -description: Use when you need ANY output derived from the MetaObjects model — write your own generator (OpenAPI, JSON Schema, Zod, DTOs, a client, docs, anything) or eject a reference one — and when configuring or running code generation, generators/targets/dialect config, the gen command, and hand-edit-preserving regeneration. +description: Use when you need ANY output derived from the MetaObjects model — write your own generator (OpenAPI, JSON Schema, Zod, DTOs, a client, docs, anything) or eject a reference one — and when configuring or running code generation: editing the gen config (metaobjects.config.ts) to choose generators, targets or the SQL dialect (e.g. postgres to sqlite), the gen command, and hand-edit-preserving regeneration. --- # MetaObjects code generation From 57fa8b1bbf44d70920e021f9554a348a5188daf9 Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Wed, 14 Oct 2026 12:38:01 -0400 Subject: [PATCH 4/4] chore(agent-context): drop the parse-only trigger-case test; the evaluation is manual-only A test that only parses the case files proves nothing about trigger behaviour, so the cases and the documented run command are the evidence. README no longer claims a CI check. --- agent-context/evals/README.md | 4 +- .../agent-context/skill-trigger-evals.test.ts | 68 ------------------- 2 files changed, 1 insertion(+), 71 deletions(-) delete mode 100644 server/typescript/packages/sdk/test/agent-context/skill-trigger-evals.test.ts diff --git a/agent-context/evals/README.md b/agent-context/evals/README.md index f28833144..f4f3cc589 100644 --- a/agent-context/evals/README.md +++ b/agent-context/evals/README.md @@ -20,9 +20,7 @@ agent-context/evals/run-skill-triggers.sh It starts real Claude sessions on your own login (25 cases x 3 runs = 75 sessions, a few minutes with `-j 4`) and writes `/result.json`. Use `--case ''` to run a -subset. It is not part of CI because model behaviour is not deterministic; CI only checks -that the cases parse and name existing skills -(`server/typescript/packages/sdk/test/agent-context/skill-trigger-evals.test.ts`). +subset. It is manual-only and not part of CI, because model behaviour is not deterministic. Re-run it after editing either description. A `Reached maximum number of turns` note on a case is expected: the runs are capped at 3 turns because only the first tool call matters. diff --git a/server/typescript/packages/sdk/test/agent-context/skill-trigger-evals.test.ts b/server/typescript/packages/sdk/test/agent-context/skill-trigger-evals.test.ts deleted file mode 100644 index ecd799190..000000000 --- a/server/typescript/packages/sdk/test/agent-context/skill-trigger-evals.test.ts +++ /dev/null @@ -1,68 +0,0 @@ -// test/agent-context/skill-trigger-evals.test.ts -// -// The trigger evaluation itself drives live model sessions and cannot run in CI -// (agent-context/evals/README.md has the manual command). This keeps the part that is -// deterministic: every case parses, names a skill that exists, and each skill carries -// both should-trigger and should-not-trigger cases. -import { test, expect, describe } from "bun:test"; -import { readFileSync, readdirSync, existsSync } from "node:fs"; -import { join } from "node:path"; - -const AGENT_CONTEXT = join(import.meta.dir, "../../../../../../agent-context"); -const SKILLS = new Set(readdirSync(join(AGENT_CONTEXT, "skills"))); -const EVALS = join(AGENT_CONTEXT, "evals/skill-triggers"); - -const frontmatter = (text: string): Record => { - const m = /^---\n([\s\S]*?)\n---\n/.exec(text); - if (!m) throw new Error("no frontmatter"); - const out: Record = {}; - for (const line of m[1]!.split("\n")) { - const kv = /^([A-Za-z_]+):\s*(.*)$/.exec(line); - if (kv) out[kv[1]!] = kv[2]!; - } - return out; -}; - -const cases = readdirSync(EVALS).flatMap((skill) => - readdirSync(join(EVALS, skill)).map((name) => ({ skill, name, dir: join(EVALS, skill, name) })), -); - -describe("skill trigger eval cases", () => { - test("only the two evaluated skills have case sets", () => { - expect(readdirSync(EVALS).sort()).toEqual(["metaobjects-codegen", "metaobjects-prompts"]); - }); - - test("every case directory parses and names an existing skill", () => { - for (const c of cases) { - expect(SKILLS.has(c.skill)).toBe(true); - const prompt = join(c.dir, "prompt.md"); - expect(existsSync(prompt)).toBe(true); - const body = readFileSync(prompt, "utf8").replace(/^---\n[\s\S]*?\n---\n/, "").trim(); - expect(body.length).toBeGreaterThan(0); - const graders = readdirSync(join(c.dir, "graders")); - expect(graders.length).toBeGreaterThan(0); - for (const g of graders) { - const fm = frontmatter(readFileSync(join(c.dir, "graders", g), "utf8")); - expect(fm.type).toBe("tool_used"); - expect(fm.tool).toBe("Skill"); - expect(fm.input_match).toBe(c.skill); - } - } - }); - - test("each skill has should-trigger and should-not-trigger cases", () => { - for (const skill of readdirSync(EVALS)) { - const kinds = cases.filter((c) => c.skill === skill).map((c) => (c.name.includes("-pos-") ? "pos" : "neg")); - expect(kinds.filter((k) => k === "pos").length).toBeGreaterThanOrEqual(5); - expect(kinds.filter((k) => k === "neg").length).toBeGreaterThanOrEqual(5); - } - }); - - test("positive cases require a firing, negative cases forbid one", () => { - for (const c of cases) { - const fm = frontmatter(readFileSync(join(c.dir, "graders", "trigger.md"), "utf8")); - if (c.name.includes("-pos-")) expect(fm.min).toBe("1"); - else expect([fm.min, fm.max]).toEqual(["0", "0"]); - } - }); -});