diff --git a/agent-context/evals/README.md b/agent-context/evals/README.md new file mode 100644 index 000000000..f4f3cc589 --- /dev/null +++ b/agent-context/evals/README.md @@ -0,0 +1,26 @@ +# Skill trigger evaluations + +Should-trigger and should-not-trigger cases for the `description` of two shipped skills, +`metaobjects-prompts` and `metaobjects-codegen`. A description decides when a skill's whole +body is loaded, so an over-eager one costs tokens on every unrelated edit. Each case is a +one-message prompt plus a `tool_used: Skill` grader: positive cases require the skill to +fire, negative cases (near-misses: LLM code in a project that does not use MetaObjects +prompts, model edits that need no generated output) forbid it. + +- `skill-triggers//-pos-*|neg-*/` — `prompt.md` and `graders/trigger.md`, in the + `claude plugin eval` case format. +- `run-skill-triggers.sh` — wraps `agent-context/skills` in a throwaway plugin (all seven + skills, so routing competes realistically) and runs the cases, single-arm. + +## Running it (manual) + +``` +agent-context/evals/run-skill-triggers.sh +``` + +It starts real Claude sessions on your own login (25 cases x 3 runs = 75 sessions, a few +minutes with `-j 4`) and writes `/result.json`. Use `--case ''` to run a +subset. It is manual-only and not part of CI, because model behaviour is not deterministic. + +Re-run it after editing either description. A `Reached maximum number of turns` note on a +case is expected: the runs are capped at 3 turns because only the first tool call matters. diff --git a/agent-context/evals/run-skill-triggers.sh b/agent-context/evals/run-skill-triggers.sh new file mode 100755 index 000000000..30017848e --- /dev/null +++ b/agent-context/evals/run-skill-triggers.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Run the skill trigger evaluation (agent-context/evals/skill-triggers) with `claude plugin eval`. +# Manual, not CI: it drives real model sessions on your own Claude login (subscription quota). +# +# usage: agent-context/evals/run-skill-triggers.sh [OUT_DIR] [extra `claude plugin eval` flags] +# OUT_DIR defaults to a fresh temp dir; the scored result is written to OUT_DIR/result.json. +# e.g. agent-context/evals/run-skill-triggers.sh /tmp/triggers --case 'metaobjects-codegen/*' +set -euo pipefail +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +skills="$here/../skills" +out="${1:-$(mktemp -d)}"; [ $# -gt 0 ] && shift +mkdir -p "$out" +plugin="$out/plugin" +rm -rf "$plugin" +mkdir -p "$plugin/.claude-plugin" +# `claude plugin eval` refuses an evals/ link that points outside the plugin, so copy. +echo '{"name":"mo-skills","version":"0.0.0","description":"MetaObjects skills, wrapped for trigger evaluation"}' >"$plugin/.claude-plugin/plugin.json" +cp -R "$skills" "$plugin/skills" +cp -R "$here/skill-triggers" "$plugin/evals" +cd "$plugin" +# --ablation none: one arm, so the `tool_used: Skill` graders count toward the score. +exec claude plugin eval . --ablation none --no-publish --trust-plugin -j 4 --threshold 0 \ + --output-dir "$out/runs" --json "$out/result.json" "$@" diff --git a/agent-context/evals/skill-triggers/metaobjects-codegen/01-pos-write-openapi-generator/graders/trigger.md b/agent-context/evals/skill-triggers/metaobjects-codegen/01-pos-write-openapi-generator/graders/trigger.md new file mode 100644 index 000000000..b8e0fb847 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-codegen/01-pos-write-openapi-generator/graders/trigger.md @@ -0,0 +1,6 @@ +--- +type: tool_used +tool: Skill +input_match: metaobjects-codegen +min: 1 +--- diff --git a/agent-context/evals/skill-triggers/metaobjects-codegen/01-pos-write-openapi-generator/prompt.md b/agent-context/evals/skill-triggers/metaobjects-codegen/01-pos-write-openapi-generator/prompt.md new file mode 100644 index 000000000..32b6d7753 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-codegen/01-pos-write-openapi-generator/prompt.md @@ -0,0 +1,8 @@ +--- +max_turns: 3 +timeout_seconds: 180 +allowed_tools: [Skill] +runs: 3 +--- + +This project uses MetaObjects (it has .metaobjects/config.json and a metaobjects/ directory of meta.*.json files). I need an OpenAPI document generated from the model. Write a generator for it. diff --git a/agent-context/evals/skill-triggers/metaobjects-codegen/02-pos-eject-reference-generator/graders/trigger.md b/agent-context/evals/skill-triggers/metaobjects-codegen/02-pos-eject-reference-generator/graders/trigger.md new file mode 100644 index 000000000..b8e0fb847 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-codegen/02-pos-eject-reference-generator/graders/trigger.md @@ -0,0 +1,6 @@ +--- +type: tool_used +tool: Skill +input_match: metaobjects-codegen +min: 1 +--- diff --git a/agent-context/evals/skill-triggers/metaobjects-codegen/02-pos-eject-reference-generator/prompt.md b/agent-context/evals/skill-triggers/metaobjects-codegen/02-pos-eject-reference-generator/prompt.md new file mode 100644 index 000000000..241aeb71f --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-codegen/02-pos-eject-reference-generator/prompt.md @@ -0,0 +1,8 @@ +--- +max_turns: 3 +timeout_seconds: 180 +allowed_tools: [Skill] +runs: 3 +--- + +This project uses MetaObjects (it has .metaobjects/config.json and a metaobjects/ directory of meta.*.json files). The reference Zod generator does not emit what I need. Eject it so I can own and change it. diff --git a/agent-context/evals/skill-triggers/metaobjects-codegen/03-pos-configure-generators/graders/trigger.md b/agent-context/evals/skill-triggers/metaobjects-codegen/03-pos-configure-generators/graders/trigger.md new file mode 100644 index 000000000..b8e0fb847 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-codegen/03-pos-configure-generators/graders/trigger.md @@ -0,0 +1,6 @@ +--- +type: tool_used +tool: Skill +input_match: metaobjects-codegen +min: 1 +--- diff --git a/agent-context/evals/skill-triggers/metaobjects-codegen/03-pos-configure-generators/prompt.md b/agent-context/evals/skill-triggers/metaobjects-codegen/03-pos-configure-generators/prompt.md new file mode 100644 index 000000000..2279cdbb2 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-codegen/03-pos-configure-generators/prompt.md @@ -0,0 +1,8 @@ +--- +max_turns: 3 +timeout_seconds: 180 +allowed_tools: [Skill] +runs: 3 +--- + +This project uses MetaObjects (it has .metaobjects/config.json and a metaobjects/ directory of meta.*.json files). Configure which generators run and where `meta gen` writes each target in metaobjects.config.ts. diff --git a/agent-context/evals/skill-triggers/metaobjects-codegen/04-pos-regen-hand-edits/graders/trigger.md b/agent-context/evals/skill-triggers/metaobjects-codegen/04-pos-regen-hand-edits/graders/trigger.md new file mode 100644 index 000000000..b8e0fb847 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-codegen/04-pos-regen-hand-edits/graders/trigger.md @@ -0,0 +1,6 @@ +--- +type: tool_used +tool: Skill +input_match: metaobjects-codegen +min: 1 +--- diff --git a/agent-context/evals/skill-triggers/metaobjects-codegen/04-pos-regen-hand-edits/prompt.md b/agent-context/evals/skill-triggers/metaobjects-codegen/04-pos-regen-hand-edits/prompt.md new file mode 100644 index 000000000..b5bc67095 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-codegen/04-pos-regen-hand-edits/prompt.md @@ -0,0 +1,8 @@ +--- +max_turns: 3 +timeout_seconds: 180 +allowed_tools: [Skill] +runs: 3 +--- + +This project uses MetaObjects (it has .metaobjects/config.json and a metaobjects/ directory of meta.*.json files). I hand-edited a generated entity file and now `meta gen` reports a merge conflict. How do I keep my edit? diff --git a/agent-context/evals/skill-triggers/metaobjects-codegen/05-pos-json-schema-output/graders/trigger.md b/agent-context/evals/skill-triggers/metaobjects-codegen/05-pos-json-schema-output/graders/trigger.md new file mode 100644 index 000000000..b8e0fb847 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-codegen/05-pos-json-schema-output/graders/trigger.md @@ -0,0 +1,6 @@ +--- +type: tool_used +tool: Skill +input_match: metaobjects-codegen +min: 1 +--- diff --git a/agent-context/evals/skill-triggers/metaobjects-codegen/05-pos-json-schema-output/prompt.md b/agent-context/evals/skill-triggers/metaobjects-codegen/05-pos-json-schema-output/prompt.md new file mode 100644 index 000000000..44ff8cf61 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-codegen/05-pos-json-schema-output/prompt.md @@ -0,0 +1,8 @@ +--- +max_turns: 3 +timeout_seconds: 180 +allowed_tools: [Skill] +runs: 3 +--- + +This project uses MetaObjects (it has .metaobjects/config.json and a metaobjects/ directory of meta.*.json files). Our frontend team needs JSON Schema files for every entity, derived from the model and kept current. diff --git a/agent-context/evals/skill-triggers/metaobjects-codegen/06-pos-dialect-config/graders/trigger.md b/agent-context/evals/skill-triggers/metaobjects-codegen/06-pos-dialect-config/graders/trigger.md new file mode 100644 index 000000000..b8e0fb847 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-codegen/06-pos-dialect-config/graders/trigger.md @@ -0,0 +1,6 @@ +--- +type: tool_used +tool: Skill +input_match: metaobjects-codegen +min: 1 +--- diff --git a/agent-context/evals/skill-triggers/metaobjects-codegen/06-pos-dialect-config/prompt.md b/agent-context/evals/skill-triggers/metaobjects-codegen/06-pos-dialect-config/prompt.md new file mode 100644 index 000000000..4f8229929 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-codegen/06-pos-dialect-config/prompt.md @@ -0,0 +1,8 @@ +--- +max_turns: 3 +timeout_seconds: 180 +allowed_tools: [Skill] +runs: 3 +--- + +This project uses MetaObjects (it has .metaobjects/config.json and a metaobjects/ directory of meta.*.json files). Change the dialect that code generation uses from postgres to sqlite in the gen configuration. diff --git a/agent-context/evals/skill-triggers/metaobjects-codegen/07-neg-add-entity-field/graders/trigger.md b/agent-context/evals/skill-triggers/metaobjects-codegen/07-neg-add-entity-field/graders/trigger.md new file mode 100644 index 000000000..5a6f0d612 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-codegen/07-neg-add-entity-field/graders/trigger.md @@ -0,0 +1,8 @@ +--- +type: tool_used +tool: Skill +input_match: metaobjects-codegen +min: 0 +max: 0 +arm: both +--- diff --git a/agent-context/evals/skill-triggers/metaobjects-codegen/07-neg-add-entity-field/prompt.md b/agent-context/evals/skill-triggers/metaobjects-codegen/07-neg-add-entity-field/prompt.md new file mode 100644 index 000000000..ddec56d5f --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-codegen/07-neg-add-entity-field/prompt.md @@ -0,0 +1,8 @@ +--- +max_turns: 3 +timeout_seconds: 180 +allowed_tools: [Skill] +runs: 3 +--- + +This project uses MetaObjects (it has .metaobjects/config.json and a metaobjects/ directory of meta.*.json files). Add a nullable `nickname` string field to the User entity. Do not generate anything yet. diff --git a/agent-context/evals/skill-triggers/metaobjects-codegen/08-neg-verify-failure/graders/trigger.md b/agent-context/evals/skill-triggers/metaobjects-codegen/08-neg-verify-failure/graders/trigger.md new file mode 100644 index 000000000..5a6f0d612 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-codegen/08-neg-verify-failure/graders/trigger.md @@ -0,0 +1,8 @@ +--- +type: tool_used +tool: Skill +input_match: metaobjects-codegen +min: 0 +max: 0 +arm: both +--- diff --git a/agent-context/evals/skill-triggers/metaobjects-codegen/08-neg-verify-failure/prompt.md b/agent-context/evals/skill-triggers/metaobjects-codegen/08-neg-verify-failure/prompt.md new file mode 100644 index 000000000..b9fb40001 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-codegen/08-neg-verify-failure/prompt.md @@ -0,0 +1,8 @@ +--- +max_turns: 3 +timeout_seconds: 180 +allowed_tools: [Skill] +runs: 3 +--- + +This project uses MetaObjects (it has .metaobjects/config.json and a metaobjects/ directory of meta.*.json files). `meta verify --db` reports the live schema drifted from the metadata. Explain what that means. diff --git a/agent-context/evals/skill-triggers/metaobjects-codegen/09-neg-openapi-no-metaobjects/graders/trigger.md b/agent-context/evals/skill-triggers/metaobjects-codegen/09-neg-openapi-no-metaobjects/graders/trigger.md new file mode 100644 index 000000000..5a6f0d612 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-codegen/09-neg-openapi-no-metaobjects/graders/trigger.md @@ -0,0 +1,8 @@ +--- +type: tool_used +tool: Skill +input_match: metaobjects-codegen +min: 0 +max: 0 +arm: both +--- diff --git a/agent-context/evals/skill-triggers/metaobjects-codegen/09-neg-openapi-no-metaobjects/prompt.md b/agent-context/evals/skill-triggers/metaobjects-codegen/09-neg-openapi-no-metaobjects/prompt.md new file mode 100644 index 000000000..b3b2a240b --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-codegen/09-neg-openapi-no-metaobjects/prompt.md @@ -0,0 +1,8 @@ +--- +max_turns: 3 +timeout_seconds: 180 +allowed_tools: [Skill] +runs: 3 +--- + +This is a plain project; it does not use MetaObjects. Generate an OpenAPI document from our FastAPI routes. diff --git a/agent-context/evals/skill-triggers/metaobjects-codegen/10-neg-prisma-codegen/graders/trigger.md b/agent-context/evals/skill-triggers/metaobjects-codegen/10-neg-prisma-codegen/graders/trigger.md new file mode 100644 index 000000000..5a6f0d612 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-codegen/10-neg-prisma-codegen/graders/trigger.md @@ -0,0 +1,8 @@ +--- +type: tool_used +tool: Skill +input_match: metaobjects-codegen +min: 0 +max: 0 +arm: both +--- diff --git a/agent-context/evals/skill-triggers/metaobjects-codegen/10-neg-prisma-codegen/prompt.md b/agent-context/evals/skill-triggers/metaobjects-codegen/10-neg-prisma-codegen/prompt.md new file mode 100644 index 000000000..852dc29d7 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-codegen/10-neg-prisma-codegen/prompt.md @@ -0,0 +1,8 @@ +--- +max_turns: 3 +timeout_seconds: 180 +allowed_tools: [Skill] +runs: 3 +--- + +This is a plain project; it does not use MetaObjects. Our `prisma generate` step fails in CI. Fix the CI job. diff --git a/agent-context/evals/skill-triggers/metaobjects-codegen/11-neg-prompt-template/graders/trigger.md b/agent-context/evals/skill-triggers/metaobjects-codegen/11-neg-prompt-template/graders/trigger.md new file mode 100644 index 000000000..5a6f0d612 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-codegen/11-neg-prompt-template/graders/trigger.md @@ -0,0 +1,8 @@ +--- +type: tool_used +tool: Skill +input_match: metaobjects-codegen +min: 0 +max: 0 +arm: both +--- diff --git a/agent-context/evals/skill-triggers/metaobjects-codegen/11-neg-prompt-template/prompt.md b/agent-context/evals/skill-triggers/metaobjects-codegen/11-neg-prompt-template/prompt.md new file mode 100644 index 000000000..7f641a885 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-codegen/11-neg-prompt-template/prompt.md @@ -0,0 +1,8 @@ +--- +max_turns: 3 +timeout_seconds: 180 +allowed_tools: [Skill] +runs: 3 +--- + +This project uses MetaObjects (it has .metaobjects/config.json and a metaobjects/ directory of meta.*.json files). Declare a template.prompt for summarizing tickets with a typed payload. diff --git a/agent-context/evals/skill-triggers/metaobjects-codegen/12-neg-runtime-query/graders/trigger.md b/agent-context/evals/skill-triggers/metaobjects-codegen/12-neg-runtime-query/graders/trigger.md new file mode 100644 index 000000000..5a6f0d612 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-codegen/12-neg-runtime-query/graders/trigger.md @@ -0,0 +1,8 @@ +--- +type: tool_used +tool: Skill +input_match: metaobjects-codegen +min: 0 +max: 0 +arm: both +--- diff --git a/agent-context/evals/skill-triggers/metaobjects-codegen/12-neg-runtime-query/prompt.md b/agent-context/evals/skill-triggers/metaobjects-codegen/12-neg-runtime-query/prompt.md new file mode 100644 index 000000000..5c35bc114 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-codegen/12-neg-runtime-query/prompt.md @@ -0,0 +1,8 @@ +--- +max_turns: 3 +timeout_seconds: 180 +allowed_tools: [Skill] +runs: 3 +--- + +This project uses MetaObjects (it has .metaobjects/config.json and a metaobjects/ directory of meta.*.json files). Use the ObjectManager at runtime to list all active users, sorted by name. diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/01-pos-fstring-prompt/graders/trigger.md b/agent-context/evals/skill-triggers/metaobjects-prompts/01-pos-fstring-prompt/graders/trigger.md new file mode 100644 index 000000000..ba923f268 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/01-pos-fstring-prompt/graders/trigger.md @@ -0,0 +1,6 @@ +--- +type: tool_used +tool: Skill +input_match: metaobjects-prompts +min: 1 +--- diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/01-pos-fstring-prompt/prompt.md b/agent-context/evals/skill-triggers/metaobjects-prompts/01-pos-fstring-prompt/prompt.md new file mode 100644 index 000000000..fafc4da6c --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/01-pos-fstring-prompt/prompt.md @@ -0,0 +1,8 @@ +--- +max_turns: 3 +timeout_seconds: 180 +allowed_tools: [Skill] +runs: 3 +--- + +This project uses MetaObjects (it has .metaobjects/config.json and a metaobjects/ directory of meta.*.json files). It declares a template.prompt named SummarizeTicket. In src/summarize.py I build the LLM request with an f-string: f"Summarize this ticket: {ticket.title}\n{ticket.body}". Add the customer's plan tier to what the model sees. diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/02-pos-declare-template-prompt/graders/trigger.md b/agent-context/evals/skill-triggers/metaobjects-prompts/02-pos-declare-template-prompt/graders/trigger.md new file mode 100644 index 000000000..ba923f268 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/02-pos-declare-template-prompt/graders/trigger.md @@ -0,0 +1,6 @@ +--- +type: tool_used +tool: Skill +input_match: metaobjects-prompts +min: 1 +--- diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/02-pos-declare-template-prompt/prompt.md b/agent-context/evals/skill-triggers/metaobjects-prompts/02-pos-declare-template-prompt/prompt.md new file mode 100644 index 000000000..5ab891d5b --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/02-pos-declare-template-prompt/prompt.md @@ -0,0 +1,8 @@ +--- +max_turns: 3 +timeout_seconds: 180 +allowed_tools: [Skill] +runs: 3 +--- + +This project uses MetaObjects (it has .metaobjects/config.json and a metaobjects/ directory of meta.*.json files). Declare a new template.prompt for classifying support tickets, with a typed payload projection and a response model. diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/03-pos-regex-reply-parse/graders/trigger.md b/agent-context/evals/skill-triggers/metaobjects-prompts/03-pos-regex-reply-parse/graders/trigger.md new file mode 100644 index 000000000..ba923f268 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/03-pos-regex-reply-parse/graders/trigger.md @@ -0,0 +1,6 @@ +--- +type: tool_used +tool: Skill +input_match: metaobjects-prompts +min: 1 +--- diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/03-pos-regex-reply-parse/prompt.md b/agent-context/evals/skill-triggers/metaobjects-prompts/03-pos-regex-reply-parse/prompt.md new file mode 100644 index 000000000..c6cc56c7e --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/03-pos-regex-reply-parse/prompt.md @@ -0,0 +1,8 @@ +--- +max_turns: 3 +timeout_seconds: 180 +allowed_tools: [Skill] +runs: 3 +--- + +This project uses MetaObjects (it has .metaobjects/config.json and a metaobjects/ directory of meta.*.json files). It declares a template.prompt whose reply we read. src/triage.ts reads the model's answer with a regex like /Category: (\w+)/. The regex breaks when the model adds a preamble. Fix the parsing. diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/04-pos-test-harness-prompt/graders/trigger.md b/agent-context/evals/skill-triggers/metaobjects-prompts/04-pos-test-harness-prompt/graders/trigger.md new file mode 100644 index 000000000..ba923f268 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/04-pos-test-harness-prompt/graders/trigger.md @@ -0,0 +1,6 @@ +--- +type: tool_used +tool: Skill +input_match: metaobjects-prompts +min: 1 +--- diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/04-pos-test-harness-prompt/prompt.md b/agent-context/evals/skill-triggers/metaobjects-prompts/04-pos-test-harness-prompt/prompt.md new file mode 100644 index 000000000..aeac947f1 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/04-pos-test-harness-prompt/prompt.md @@ -0,0 +1,8 @@ +--- +max_turns: 3 +timeout_seconds: 180 +allowed_tools: [Skill] +runs: 3 +--- + +This project uses MetaObjects (it has .metaobjects/config.json and a metaobjects/ directory of meta.*.json files). It declares template.prompt nodes. A test script in tests/eval_harness.py concatenates strings to build the text it sends to the LLM. Add one more instruction line to that text. diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/05-pos-prompt-drift-verify/graders/trigger.md b/agent-context/evals/skill-triggers/metaobjects-prompts/05-pos-prompt-drift-verify/graders/trigger.md new file mode 100644 index 000000000..ba923f268 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/05-pos-prompt-drift-verify/graders/trigger.md @@ -0,0 +1,6 @@ +--- +type: tool_used +tool: Skill +input_match: metaobjects-prompts +min: 1 +--- diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/05-pos-prompt-drift-verify/prompt.md b/agent-context/evals/skill-triggers/metaobjects-prompts/05-pos-prompt-drift-verify/prompt.md new file mode 100644 index 000000000..0becb8f09 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/05-pos-prompt-drift-verify/prompt.md @@ -0,0 +1,8 @@ +--- +max_turns: 3 +timeout_seconds: 180 +allowed_tools: [Skill] +runs: 3 +--- + +This project uses MetaObjects (it has .metaobjects/config.json and a metaobjects/ directory of meta.*.json files). I renamed a field on the payload projection of the ReviewDraft template.prompt. Update the prompt template and its payload so rendering stays in sync. diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/06-pos-review-json-parse/graders/trigger.md b/agent-context/evals/skill-triggers/metaobjects-prompts/06-pos-review-json-parse/graders/trigger.md new file mode 100644 index 000000000..ba923f268 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/06-pos-review-json-parse/graders/trigger.md @@ -0,0 +1,6 @@ +--- +type: tool_used +tool: Skill +input_match: metaobjects-prompts +min: 1 +--- diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/06-pos-review-json-parse/prompt.md b/agent-context/evals/skill-triggers/metaobjects-prompts/06-pos-review-json-parse/prompt.md new file mode 100644 index 000000000..96760e12d --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/06-pos-review-json-parse/prompt.md @@ -0,0 +1,8 @@ +--- +max_turns: 3 +timeout_seconds: 180 +allowed_tools: [Skill] +runs: 3 +--- + +This project uses MetaObjects (it has .metaobjects/config.json and a metaobjects/ directory of meta.*.json files). It declares template.prompt nodes. Review this diff: it adds JSON.parse(completion.text) in src/agent.ts to read an LLM reply. diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/07-neg-fstring-no-metaobjects/graders/trigger.md b/agent-context/evals/skill-triggers/metaobjects-prompts/07-neg-fstring-no-metaobjects/graders/trigger.md new file mode 100644 index 000000000..7b6368ff0 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/07-neg-fstring-no-metaobjects/graders/trigger.md @@ -0,0 +1,8 @@ +--- +type: tool_used +tool: Skill +input_match: metaobjects-prompts +min: 0 +max: 0 +arm: both +--- diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/07-neg-fstring-no-metaobjects/prompt.md b/agent-context/evals/skill-triggers/metaobjects-prompts/07-neg-fstring-no-metaobjects/prompt.md new file mode 100644 index 000000000..c4c0238fb --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/07-neg-fstring-no-metaobjects/prompt.md @@ -0,0 +1,8 @@ +--- +max_turns: 3 +timeout_seconds: 180 +allowed_tools: [Skill] +runs: 3 +--- + +This is a plain project; it does not use MetaObjects. In src/summarize.py I build an OpenAI request with an f-string: f"Summarize this ticket: {ticket.title}". Add the customer's plan tier to the text. diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/08-neg-langchain-system-prompt/graders/trigger.md b/agent-context/evals/skill-triggers/metaobjects-prompts/08-neg-langchain-system-prompt/graders/trigger.md new file mode 100644 index 000000000..7b6368ff0 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/08-neg-langchain-system-prompt/graders/trigger.md @@ -0,0 +1,8 @@ +--- +type: tool_used +tool: Skill +input_match: metaobjects-prompts +min: 0 +max: 0 +arm: both +--- diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/08-neg-langchain-system-prompt/prompt.md b/agent-context/evals/skill-triggers/metaobjects-prompts/08-neg-langchain-system-prompt/prompt.md new file mode 100644 index 000000000..4f153983e --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/08-neg-langchain-system-prompt/prompt.md @@ -0,0 +1,8 @@ +--- +max_turns: 3 +timeout_seconds: 180 +allowed_tools: [Skill] +runs: 3 +--- + +This is a plain project; it does not use MetaObjects. It is a LangChain app. Tighten the wording of the system prompt in agent/prompts.py so the bot answers more briefly. diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/09-neg-json-reply-no-metaobjects/graders/trigger.md b/agent-context/evals/skill-triggers/metaobjects-prompts/09-neg-json-reply-no-metaobjects/graders/trigger.md new file mode 100644 index 000000000..7b6368ff0 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/09-neg-json-reply-no-metaobjects/graders/trigger.md @@ -0,0 +1,8 @@ +--- +type: tool_used +tool: Skill +input_match: metaobjects-prompts +min: 0 +max: 0 +arm: both +--- diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/09-neg-json-reply-no-metaobjects/prompt.md b/agent-context/evals/skill-triggers/metaobjects-prompts/09-neg-json-reply-no-metaobjects/prompt.md new file mode 100644 index 000000000..f36f5f39b --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/09-neg-json-reply-no-metaobjects/prompt.md @@ -0,0 +1,8 @@ +--- +max_turns: 3 +timeout_seconds: 180 +allowed_tools: [Skill] +runs: 3 +--- + +This is a plain project; it does not use MetaObjects. src/extract.ts calls an LLM and runs JSON.parse on the reply. Add zod validation to that parse. diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/10-neg-entities-only-project-prompt-string/graders/trigger.md b/agent-context/evals/skill-triggers/metaobjects-prompts/10-neg-entities-only-project-prompt-string/graders/trigger.md new file mode 100644 index 000000000..7b6368ff0 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/10-neg-entities-only-project-prompt-string/graders/trigger.md @@ -0,0 +1,8 @@ +--- +type: tool_used +tool: Skill +input_match: metaobjects-prompts +min: 0 +max: 0 +arm: both +--- diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/10-neg-entities-only-project-prompt-string/prompt.md b/agent-context/evals/skill-triggers/metaobjects-prompts/10-neg-entities-only-project-prompt-string/prompt.md new file mode 100644 index 000000000..7edf64c55 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/10-neg-entities-only-project-prompt-string/prompt.md @@ -0,0 +1,8 @@ +--- +max_turns: 3 +timeout_seconds: 180 +allowed_tools: [Skill] +runs: 3 +--- + +This project uses MetaObjects (it has .metaobjects/config.json and a metaobjects/ directory of meta.*.json files). It declares entities only; no template.prompt exists and none is wanted. In src/helpers/greeter.py there is a one-line prompt string passed to the LLM client. Change the greeting from formal to casual. diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/11-neg-add-entity-field/graders/trigger.md b/agent-context/evals/skill-triggers/metaobjects-prompts/11-neg-add-entity-field/graders/trigger.md new file mode 100644 index 000000000..7b6368ff0 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/11-neg-add-entity-field/graders/trigger.md @@ -0,0 +1,8 @@ +--- +type: tool_used +tool: Skill +input_match: metaobjects-prompts +min: 0 +max: 0 +arm: both +--- diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/11-neg-add-entity-field/prompt.md b/agent-context/evals/skill-triggers/metaobjects-prompts/11-neg-add-entity-field/prompt.md new file mode 100644 index 000000000..4e81d2e40 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/11-neg-add-entity-field/prompt.md @@ -0,0 +1,8 @@ +--- +max_turns: 3 +timeout_seconds: 180 +allowed_tools: [Skill] +runs: 3 +--- + +This project uses MetaObjects (it has .metaobjects/config.json and a metaobjects/ directory of meta.*.json files). Add a nullable `nickname` string field to the User entity. diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/12-neg-non-llm-string-building/graders/trigger.md b/agent-context/evals/skill-triggers/metaobjects-prompts/12-neg-non-llm-string-building/graders/trigger.md new file mode 100644 index 000000000..7b6368ff0 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/12-neg-non-llm-string-building/graders/trigger.md @@ -0,0 +1,8 @@ +--- +type: tool_used +tool: Skill +input_match: metaobjects-prompts +min: 0 +max: 0 +arm: both +--- diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/12-neg-non-llm-string-building/prompt.md b/agent-context/evals/skill-triggers/metaobjects-prompts/12-neg-non-llm-string-building/prompt.md new file mode 100644 index 000000000..bf7ec06bc --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/12-neg-non-llm-string-building/prompt.md @@ -0,0 +1,8 @@ +--- +max_turns: 3 +timeout_seconds: 180 +allowed_tools: [Skill] +runs: 3 +--- + +This project uses MetaObjects (it has .metaobjects/config.json and a metaobjects/ directory of meta.*.json files). In src/mail.ts the welcome e-mail body is built with template literals. Add the user's first name to the greeting. No LLM is involved. diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/13-pos-adopt-first-prompt/graders/trigger.md b/agent-context/evals/skill-triggers/metaobjects-prompts/13-pos-adopt-first-prompt/graders/trigger.md new file mode 100644 index 000000000..ba923f268 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/13-pos-adopt-first-prompt/graders/trigger.md @@ -0,0 +1,6 @@ +--- +type: tool_used +tool: Skill +input_match: metaobjects-prompts +min: 1 +--- diff --git a/agent-context/evals/skill-triggers/metaobjects-prompts/13-pos-adopt-first-prompt/prompt.md b/agent-context/evals/skill-triggers/metaobjects-prompts/13-pos-adopt-first-prompt/prompt.md new file mode 100644 index 000000000..6662ae869 --- /dev/null +++ b/agent-context/evals/skill-triggers/metaobjects-prompts/13-pos-adopt-first-prompt/prompt.md @@ -0,0 +1,8 @@ +--- +max_turns: 3 +timeout_seconds: 180 +allowed_tools: [Skill] +runs: 3 +--- + +This project uses MetaObjects (it has .metaobjects/config.json and a metaobjects/ directory of meta.*.json files) but has not declared any template.prompt yet. Our ticket summarizer builds its LLM prompt with an f-string in summarize.py. I want to start declaring prompts the MetaObjects way, so move that prompt onto it. diff --git a/agent-context/skills/metaobjects-codegen/SKILL.md b/agent-context/skills/metaobjects-codegen/SKILL.md index c1f048a88..f4b4af165 100644 --- a/agent-context/skills/metaobjects-codegen/SKILL.md +++ b/agent-context/skills/metaobjects-codegen/SKILL.md @@ -1,6 +1,6 @@ --- name: metaobjects-codegen -description: Use when you need ANY output derived from the MetaObjects model — write your own generator (OpenAPI, JSON Schema, Zod, DTOs, a client, docs, anything) or eject a reference one — and when configuring or running code generation, generators/targets/dialect config, the gen command, and hand-edit-preserving regeneration. +description: Use when you need ANY output derived from the MetaObjects model — write your own generator (OpenAPI, JSON Schema, Zod, DTOs, a client, docs, anything) or eject a reference one — and when configuring or running code generation: editing the gen config (metaobjects.config.ts) to choose generators, targets or the SQL dialect (e.g. postgres to sqlite), the gen command, and hand-edit-preserving regeneration. --- # MetaObjects code generation diff --git a/agent-context/skills/metaobjects-prompts/SKILL.md b/agent-context/skills/metaobjects-prompts/SKILL.md index 66a26ec73..011c28dc8 100644 --- a/agent-context/skills/metaobjects-prompts/SKILL.md +++ b/agent-context/skills/metaobjects-prompts/SKILL.md @@ -1,6 +1,6 @@ --- name: metaobjects-prompts -description: Use when writing, editing or reviewing ANY code that builds text sent to an LLM (string formatting, f-strings, text blocks, template literals, string builders, concatenation) or reads an LLM reply (regex, string search, JSON or XML parsing), even in a test harness or script. In a project that declares a template.prompt, that code belongs in a template, a payload VO and a response model. Also for declaring template.prompt/template.output, typed payload projections, provider-resolved text, deterministic render, prompt-drift verify and parser-on-receipt. +description: Use when writing, editing or reviewing code that builds text sent to an LLM (string formatting, f-strings, text blocks, template literals, string builders, concatenation) or reads an LLM reply (regex, string search, JSON or XML parsing), in a project that declares template.prompt nodes; test harnesses and scripts count. That code belongs in a template, a payload VO and a response model. Also for declaring template.prompt/template.output, typed payload projections, provider-resolved text, deterministic render, prompt-drift verify and parser-on-receipt, including a MetaObjects project that has no template.prompt yet and wants to move existing LLM code onto them. Do not use to edit existing LLM code in a project that declares no template.prompt when the task is not declaring prompts. --- # MetaObjects prompt construction diff --git a/fixtures/agent-context-conformance/java-kotlin-react-tanstack/expected/.claude/skills/metaobjects-codegen/SKILL.md b/fixtures/agent-context-conformance/java-kotlin-react-tanstack/expected/.claude/skills/metaobjects-codegen/SKILL.md index c1f048a88..f4b4af165 100644 --- a/fixtures/agent-context-conformance/java-kotlin-react-tanstack/expected/.claude/skills/metaobjects-codegen/SKILL.md +++ b/fixtures/agent-context-conformance/java-kotlin-react-tanstack/expected/.claude/skills/metaobjects-codegen/SKILL.md @@ -1,6 +1,6 @@ --- name: metaobjects-codegen -description: Use when you need ANY output derived from the MetaObjects model — write your own generator (OpenAPI, JSON Schema, Zod, DTOs, a client, docs, anything) or eject a reference one — and when configuring or running code generation, generators/targets/dialect config, the gen command, and hand-edit-preserving regeneration. +description: Use when you need ANY output derived from the MetaObjects model — write your own generator (OpenAPI, JSON Schema, Zod, DTOs, a client, docs, anything) or eject a reference one — and when configuring or running code generation: editing the gen config (metaobjects.config.ts) to choose generators, targets or the SQL dialect (e.g. postgres to sqlite), the gen command, and hand-edit-preserving regeneration. --- # MetaObjects code generation diff --git a/fixtures/agent-context-conformance/java-kotlin-react-tanstack/expected/.claude/skills/metaobjects-prompts/SKILL.md b/fixtures/agent-context-conformance/java-kotlin-react-tanstack/expected/.claude/skills/metaobjects-prompts/SKILL.md index 66a26ec73..011c28dc8 100644 --- a/fixtures/agent-context-conformance/java-kotlin-react-tanstack/expected/.claude/skills/metaobjects-prompts/SKILL.md +++ b/fixtures/agent-context-conformance/java-kotlin-react-tanstack/expected/.claude/skills/metaobjects-prompts/SKILL.md @@ -1,6 +1,6 @@ --- name: metaobjects-prompts -description: Use when writing, editing or reviewing ANY code that builds text sent to an LLM (string formatting, f-strings, text blocks, template literals, string builders, concatenation) or reads an LLM reply (regex, string search, JSON or XML parsing), even in a test harness or script. In a project that declares a template.prompt, that code belongs in a template, a payload VO and a response model. Also for declaring template.prompt/template.output, typed payload projections, provider-resolved text, deterministic render, prompt-drift verify and parser-on-receipt. +description: Use when writing, editing or reviewing code that builds text sent to an LLM (string formatting, f-strings, text blocks, template literals, string builders, concatenation) or reads an LLM reply (regex, string search, JSON or XML parsing), in a project that declares template.prompt nodes; test harnesses and scripts count. That code belongs in a template, a payload VO and a response model. Also for declaring template.prompt/template.output, typed payload projections, provider-resolved text, deterministic render, prompt-drift verify and parser-on-receipt, including a MetaObjects project that has no template.prompt yet and wants to move existing LLM code onto them. Do not use to edit existing LLM code in a project that declares no template.prompt when the task is not declaring prompts. --- # MetaObjects prompt construction diff --git a/fixtures/agent-context-conformance/java-react/expected/.claude/skills/metaobjects-codegen/SKILL.md b/fixtures/agent-context-conformance/java-react/expected/.claude/skills/metaobjects-codegen/SKILL.md index c1f048a88..f4b4af165 100644 --- a/fixtures/agent-context-conformance/java-react/expected/.claude/skills/metaobjects-codegen/SKILL.md +++ b/fixtures/agent-context-conformance/java-react/expected/.claude/skills/metaobjects-codegen/SKILL.md @@ -1,6 +1,6 @@ --- name: metaobjects-codegen -description: Use when you need ANY output derived from the MetaObjects model — write your own generator (OpenAPI, JSON Schema, Zod, DTOs, a client, docs, anything) or eject a reference one — and when configuring or running code generation, generators/targets/dialect config, the gen command, and hand-edit-preserving regeneration. +description: Use when you need ANY output derived from the MetaObjects model — write your own generator (OpenAPI, JSON Schema, Zod, DTOs, a client, docs, anything) or eject a reference one — and when configuring or running code generation: editing the gen config (metaobjects.config.ts) to choose generators, targets or the SQL dialect (e.g. postgres to sqlite), the gen command, and hand-edit-preserving regeneration. --- # MetaObjects code generation diff --git a/fixtures/agent-context-conformance/java-react/expected/.claude/skills/metaobjects-prompts/SKILL.md b/fixtures/agent-context-conformance/java-react/expected/.claude/skills/metaobjects-prompts/SKILL.md index 66a26ec73..011c28dc8 100644 --- a/fixtures/agent-context-conformance/java-react/expected/.claude/skills/metaobjects-prompts/SKILL.md +++ b/fixtures/agent-context-conformance/java-react/expected/.claude/skills/metaobjects-prompts/SKILL.md @@ -1,6 +1,6 @@ --- name: metaobjects-prompts -description: Use when writing, editing or reviewing ANY code that builds text sent to an LLM (string formatting, f-strings, text blocks, template literals, string builders, concatenation) or reads an LLM reply (regex, string search, JSON or XML parsing), even in a test harness or script. In a project that declares a template.prompt, that code belongs in a template, a payload VO and a response model. Also for declaring template.prompt/template.output, typed payload projections, provider-resolved text, deterministic render, prompt-drift verify and parser-on-receipt. +description: Use when writing, editing or reviewing code that builds text sent to an LLM (string formatting, f-strings, text blocks, template literals, string builders, concatenation) or reads an LLM reply (regex, string search, JSON or XML parsing), in a project that declares template.prompt nodes; test harnesses and scripts count. That code belongs in a template, a payload VO and a response model. Also for declaring template.prompt/template.output, typed payload projections, provider-resolved text, deterministic render, prompt-drift verify and parser-on-receipt, including a MetaObjects project that has no template.prompt yet and wants to move existing LLM code onto them. Do not use to edit existing LLM code in a project that declares no template.prompt when the task is not declaring prompts. --- # MetaObjects prompt construction diff --git a/fixtures/agent-context-conformance/python/expected/.claude/skills/metaobjects-codegen/SKILL.md b/fixtures/agent-context-conformance/python/expected/.claude/skills/metaobjects-codegen/SKILL.md index c1f048a88..f4b4af165 100644 --- a/fixtures/agent-context-conformance/python/expected/.claude/skills/metaobjects-codegen/SKILL.md +++ b/fixtures/agent-context-conformance/python/expected/.claude/skills/metaobjects-codegen/SKILL.md @@ -1,6 +1,6 @@ --- name: metaobjects-codegen -description: Use when you need ANY output derived from the MetaObjects model — write your own generator (OpenAPI, JSON Schema, Zod, DTOs, a client, docs, anything) or eject a reference one — and when configuring or running code generation, generators/targets/dialect config, the gen command, and hand-edit-preserving regeneration. +description: Use when you need ANY output derived from the MetaObjects model — write your own generator (OpenAPI, JSON Schema, Zod, DTOs, a client, docs, anything) or eject a reference one — and when configuring or running code generation: editing the gen config (metaobjects.config.ts) to choose generators, targets or the SQL dialect (e.g. postgres to sqlite), the gen command, and hand-edit-preserving regeneration. --- # MetaObjects code generation diff --git a/fixtures/agent-context-conformance/python/expected/.claude/skills/metaobjects-prompts/SKILL.md b/fixtures/agent-context-conformance/python/expected/.claude/skills/metaobjects-prompts/SKILL.md index 66a26ec73..011c28dc8 100644 --- a/fixtures/agent-context-conformance/python/expected/.claude/skills/metaobjects-prompts/SKILL.md +++ b/fixtures/agent-context-conformance/python/expected/.claude/skills/metaobjects-prompts/SKILL.md @@ -1,6 +1,6 @@ --- name: metaobjects-prompts -description: Use when writing, editing or reviewing ANY code that builds text sent to an LLM (string formatting, f-strings, text blocks, template literals, string builders, concatenation) or reads an LLM reply (regex, string search, JSON or XML parsing), even in a test harness or script. In a project that declares a template.prompt, that code belongs in a template, a payload VO and a response model. Also for declaring template.prompt/template.output, typed payload projections, provider-resolved text, deterministic render, prompt-drift verify and parser-on-receipt. +description: Use when writing, editing or reviewing code that builds text sent to an LLM (string formatting, f-strings, text blocks, template literals, string builders, concatenation) or reads an LLM reply (regex, string search, JSON or XML parsing), in a project that declares template.prompt nodes; test harnesses and scripts count. That code belongs in a template, a payload VO and a response model. Also for declaring template.prompt/template.output, typed payload projections, provider-resolved text, deterministic render, prompt-drift verify and parser-on-receipt, including a MetaObjects project that has no template.prompt yet and wants to move existing LLM code onto them. Do not use to edit existing LLM code in a project that declares no template.prompt when the task is not declaring prompts. --- # MetaObjects prompt construction diff --git a/fixtures/agent-context-conformance/ts-react-tanstack/expected/.claude/skills/metaobjects-codegen/SKILL.md b/fixtures/agent-context-conformance/ts-react-tanstack/expected/.claude/skills/metaobjects-codegen/SKILL.md index c1f048a88..f4b4af165 100644 --- a/fixtures/agent-context-conformance/ts-react-tanstack/expected/.claude/skills/metaobjects-codegen/SKILL.md +++ b/fixtures/agent-context-conformance/ts-react-tanstack/expected/.claude/skills/metaobjects-codegen/SKILL.md @@ -1,6 +1,6 @@ --- name: metaobjects-codegen -description: Use when you need ANY output derived from the MetaObjects model — write your own generator (OpenAPI, JSON Schema, Zod, DTOs, a client, docs, anything) or eject a reference one — and when configuring or running code generation, generators/targets/dialect config, the gen command, and hand-edit-preserving regeneration. +description: Use when you need ANY output derived from the MetaObjects model — write your own generator (OpenAPI, JSON Schema, Zod, DTOs, a client, docs, anything) or eject a reference one — and when configuring or running code generation: editing the gen config (metaobjects.config.ts) to choose generators, targets or the SQL dialect (e.g. postgres to sqlite), the gen command, and hand-edit-preserving regeneration. --- # MetaObjects code generation diff --git a/fixtures/agent-context-conformance/ts-react-tanstack/expected/.claude/skills/metaobjects-prompts/SKILL.md b/fixtures/agent-context-conformance/ts-react-tanstack/expected/.claude/skills/metaobjects-prompts/SKILL.md index 66a26ec73..011c28dc8 100644 --- a/fixtures/agent-context-conformance/ts-react-tanstack/expected/.claude/skills/metaobjects-prompts/SKILL.md +++ b/fixtures/agent-context-conformance/ts-react-tanstack/expected/.claude/skills/metaobjects-prompts/SKILL.md @@ -1,6 +1,6 @@ --- name: metaobjects-prompts -description: Use when writing, editing or reviewing ANY code that builds text sent to an LLM (string formatting, f-strings, text blocks, template literals, string builders, concatenation) or reads an LLM reply (regex, string search, JSON or XML parsing), even in a test harness or script. In a project that declares a template.prompt, that code belongs in a template, a payload VO and a response model. Also for declaring template.prompt/template.output, typed payload projections, provider-resolved text, deterministic render, prompt-drift verify and parser-on-receipt. +description: Use when writing, editing or reviewing code that builds text sent to an LLM (string formatting, f-strings, text blocks, template literals, string builders, concatenation) or reads an LLM reply (regex, string search, JSON or XML parsing), in a project that declares template.prompt nodes; test harnesses and scripts count. That code belongs in a template, a payload VO and a response model. Also for declaring template.prompt/template.output, typed payload projections, provider-resolved text, deterministic render, prompt-drift verify and parser-on-receipt, including a MetaObjects project that has no template.prompt yet and wants to move existing LLM code onto them. Do not use to edit existing LLM code in a project that declares no template.prompt when the task is not declaring prompts. --- # MetaObjects prompt construction diff --git a/fixtures/agent-context-conformance/ts-requirements/expected/.claude/skills/metaobjects-codegen/SKILL.md b/fixtures/agent-context-conformance/ts-requirements/expected/.claude/skills/metaobjects-codegen/SKILL.md index c1f048a88..f4b4af165 100644 --- a/fixtures/agent-context-conformance/ts-requirements/expected/.claude/skills/metaobjects-codegen/SKILL.md +++ b/fixtures/agent-context-conformance/ts-requirements/expected/.claude/skills/metaobjects-codegen/SKILL.md @@ -1,6 +1,6 @@ --- name: metaobjects-codegen -description: Use when you need ANY output derived from the MetaObjects model — write your own generator (OpenAPI, JSON Schema, Zod, DTOs, a client, docs, anything) or eject a reference one — and when configuring or running code generation, generators/targets/dialect config, the gen command, and hand-edit-preserving regeneration. +description: Use when you need ANY output derived from the MetaObjects model — write your own generator (OpenAPI, JSON Schema, Zod, DTOs, a client, docs, anything) or eject a reference one — and when configuring or running code generation: editing the gen config (metaobjects.config.ts) to choose generators, targets or the SQL dialect (e.g. postgres to sqlite), the gen command, and hand-edit-preserving regeneration. --- # MetaObjects code generation diff --git a/fixtures/agent-context-conformance/ts-requirements/expected/.claude/skills/metaobjects-prompts/SKILL.md b/fixtures/agent-context-conformance/ts-requirements/expected/.claude/skills/metaobjects-prompts/SKILL.md index 66a26ec73..011c28dc8 100644 --- a/fixtures/agent-context-conformance/ts-requirements/expected/.claude/skills/metaobjects-prompts/SKILL.md +++ b/fixtures/agent-context-conformance/ts-requirements/expected/.claude/skills/metaobjects-prompts/SKILL.md @@ -1,6 +1,6 @@ --- name: metaobjects-prompts -description: Use when writing, editing or reviewing ANY code that builds text sent to an LLM (string formatting, f-strings, text blocks, template literals, string builders, concatenation) or reads an LLM reply (regex, string search, JSON or XML parsing), even in a test harness or script. In a project that declares a template.prompt, that code belongs in a template, a payload VO and a response model. Also for declaring template.prompt/template.output, typed payload projections, provider-resolved text, deterministic render, prompt-drift verify and parser-on-receipt. +description: Use when writing, editing or reviewing code that builds text sent to an LLM (string formatting, f-strings, text blocks, template literals, string builders, concatenation) or reads an LLM reply (regex, string search, JSON or XML parsing), in a project that declares template.prompt nodes; test harnesses and scripts count. That code belongs in a template, a payload VO and a response model. Also for declaring template.prompt/template.output, typed payload projections, provider-resolved text, deterministic render, prompt-drift verify and parser-on-receipt, including a MetaObjects project that has no template.prompt yet and wants to move existing LLM code onto them. Do not use to edit existing LLM code in a project that declares no template.prompt when the task is not declaring prompts. --- # MetaObjects prompt construction