From 2623fa9b90fb2ad1101410152e8b7517207d965c Mon Sep 17 00:00:00 2001 From: vostride <32242596+pranshuchittora@users.noreply.github.com> Date: Mon, 26 Oct 2026 00:34:10 +0530 Subject: [PATCH] feat(agent-qa): add testing and evidence skill --- .../agent-qa/.zcode-plugin/plugin.json | 23 ++ external_plugins/agent-qa/LICENSE | 202 ++++++++++++++++++ external_plugins/agent-qa/README.md | 41 ++++ external_plugins/agent-qa/README_CN.md | 41 ++++ .../agent-qa/skills/agent-qa/SKILL.md | 63 ++++++ .../references/evidence-and-memory.md | 126 +++++++++++ .../agent-qa/references/project-and-runs.md | 150 +++++++++++++ marketplace.json | 22 ++ 8 files changed, 668 insertions(+) create mode 100644 external_plugins/agent-qa/.zcode-plugin/plugin.json create mode 100644 external_plugins/agent-qa/LICENSE create mode 100644 external_plugins/agent-qa/README.md create mode 100644 external_plugins/agent-qa/README_CN.md create mode 100644 external_plugins/agent-qa/skills/agent-qa/SKILL.md create mode 100644 external_plugins/agent-qa/skills/agent-qa/references/evidence-and-memory.md create mode 100644 external_plugins/agent-qa/skills/agent-qa/references/project-and-runs.md diff --git a/external_plugins/agent-qa/.zcode-plugin/plugin.json b/external_plugins/agent-qa/.zcode-plugin/plugin.json new file mode 100644 index 00000000..4cdcd360 --- /dev/null +++ b/external_plugins/agent-qa/.zcode-plugin/plugin.json @@ -0,0 +1,23 @@ +{ + "name": "agent-qa", + "version": "0.1.0", + "description": "Natural-language web and mobile testing with agent-qa from vostride, recorded run evidence, and advanced memidex memory and self-improvement.", + "description_i18n": { + "en": "Natural-language web and mobile testing with agent-qa from vostride, recorded run evidence, and advanced memidex memory and self-improvement.", + "zh-CN": "使用 vostride 的 agent-qa 进行自然语言 Web 与移动端测试，查看运行证据，并利用先进的 memidex 记忆引擎和自我改进能力。" + }, + "author": { + "name": "vostride", + "url": "https://vostride.com" + }, + "homepage": "https://vostride.com/docs/agent-qa/quickstart", + "license": "Apache-2.0", + "keywords": [ + "testing", + "qa", + "web", + "mobile", + "memidex" + ], + "skills": "./skills" +} diff --git a/external_plugins/agent-qa/LICENSE b/external_plugins/agent-qa/LICENSE new file mode 100644 index 00000000..d6456956 --- /dev/null +++ b/external_plugins/agent-qa/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/external_plugins/agent-qa/README.md b/external_plugins/agent-qa/README.md new file mode 100644 index 00000000..8c99c82a --- /dev/null +++ b/external_plugins/agent-qa/README.md @@ -0,0 +1,41 @@ +# agent-qa + +[中文文档](./README_CN.md) + +Use [agent-qa](https://github.com/vostride/agent-qa) from vostride for natural-language Web, Android, and iOS tests, recorded run evidence, and advanced memidex memory and self-improvement. + +## Use in ZCode + +Install and enable **agent-qa** from the marketplace, then open a new session in the application project. The `agent-qa` skill is triggered by explicit agent-qa requests or its configuration and test files. For example: + +- “Use agent-qa to validate and run this project's checkout test.” +- “Review the recorded agent-qa checkout evidence only. Do not start a run or change knowledge.” +- “Explain what memidex knowledge this agent-qa run used and learned.” +- “Author an agent-qa test for the configured komazon target that checks the product catalog is visible.” + +The skill keeps preparation checks separate from execution results. A dry run or missing completed-run record is not a pass. Authoring uses the installed package's supported contract and can execute a test during verification. + +## Prerequisites + +The separately installed agent-qa runtime 0.2.3 requires Node.js 24+. Use the application's existing runtime version and configuration when present. For a new setup, follow the [quickstart](https://vostride.com/docs/agent-qa/quickstart) in the intended application project: + +```bash +npm install -D agent-qa +npx agent-qa init --help +``` + +Configure the chosen model provider through its supported credential flow. Claude or Codex subscription sign-in also requires `npm install -D @vostride/agent-qa-subscription-auth`. Browser execution requires the configured browser; Android/iOS execution requires Appium and the relevant device/driver setup. Saved-run reviews do not need a new browser session. + +The plugin supplies skill instructions; it does not bundle agent-qa, install dependencies, register MCP, or start services automatically. For connected authoring tools, configure agent-qa MCP separately using the [coding-agent guide](https://vostride.com/docs/agent-qa/coding-agents). Use the intended project configuration and a tool-call timeout appropriate for the test or suite. Without a connected tool, discover the installed capability interface rather than guessing an authoring command. + +## Dependencies and side effects + +- The skill can guide ZCode to execute the installed agent-qa CLI and use configured MCP tools within the requested task. It contains no executable hooks or bundled MCP server. +- Package installation downloads from npm. Test execution contacts the configured application and model provider; mobile execution also uses configured Appium/device services. Provider charges or subscription limits may apply. +- Initialization, authoring, and execution can create configuration, YAML tests, local run records, screenshots, logs, and knowledge in the application project. A read-only evidence review does not call authoring/execution or change outcome marks or knowledge. +- Runtime analytics, authentication, and data handling follow the installed agent-qa version and project settings. Only configured secret values are masked; inspect evidence before sharing it. Do not commit credentials or put them in chat. +- Outcome correction can change attributable knowledge or expectations. It requires a correction task and a reviewed effect; a corrected verdict is not evidence of a rerun. + +## Contents and license + +The package contains one skill and two references for setup/runs and evidence/memory. Instructions and manifests are contributed by vostride under Apache-2.0, the marketplace's license. The separately installed [agent-qa runtime uses FSL-1.1-ALv2](https://github.com/vostride/agent-qa/blob/main/LICENSE.md); it is not included in this plugin. diff --git a/external_plugins/agent-qa/README_CN.md b/external_plugins/agent-qa/README_CN.md new file mode 100644 index 00000000..d11900ff --- /dev/null +++ b/external_plugins/agent-qa/README_CN.md @@ -0,0 +1,41 @@ +# agent-qa + +[English documentation](./README.md) + +使用 vostride 的 [agent-qa](https://github.com/vostride/agent-qa) 进行自然语言 Web、Android 和 iOS 测试，查看运行证据，并利用先进的 memidex 记忆引擎和自我改进能力。 + +## 在 ZCode 中使用 + +从市场安装并启用 **agent-qa**，然后在应用项目中开启新会话。明确提到 agent-qa 的请求，或相关配置和测试文件，会触发 `agent-qa` 技能。例如： + +- “使用 agent-qa 验证并运行本项目的结账测试。” +- “只查看已有的 agent-qa 结账运行证据，不要启动测试或修改知识。” +- “解释这次 agent-qa 运行使用和学到了哪些 memidex 知识。” +- “为已配置的 komazon 目标编写 agent-qa 测试，检查商品目录是否可见。” + +技能会区分准备检查与实际执行结果。试运行或缺少已完成的运行记录都不代表测试通过。编写测试时使用已安装版本支持的接口，验证过程可能执行测试。 + +## 前置条件 + +单独安装的 agent-qa 0.2.3 运行时要求 Node.js 24+。项目已有运行时版本和配置时，应沿用现有设置。首次配置请在目标应用项目中按照[快速入门](https://vostride.com/docs/agent-qa/quickstart)操作： + +```bash +npm install -D agent-qa +npx agent-qa init --help +``` + +通过所选模型提供方支持的凭据流程完成配置。使用 Claude 或 Codex 订阅登录时，还需运行 `npm install -D @vostride/agent-qa-subscription-auth`。Web 测试需要配置相应浏览器；Android/iOS 测试需要 Appium 以及对应设备和驱动。查看已保存的运行记录无需新建浏览器会话。 + +插件提供技能指令，不打包 agent-qa，不会自动安装依赖、注册 MCP 或启动服务。如需连接测试编写工具，请按照[编码代理指南](https://vostride.com/docs/agent-qa/coding-agents)单独配置 agent-qa MCP，使用正确的项目配置，并根据测试或套件时长设置工具调用超时。未连接工具时，应查阅已安装版本的能力接口，而不是猜测测试编写命令。 + +## 依赖与副作用 + +- 技能可引导 ZCode 在用户请求范围内执行已安装的 agent-qa CLI，或使用已配置的 MCP 工具。插件不含可执行 Hook，也不捆绑 MCP 服务。 +- 安装软件包会访问 npm。测试执行会连接已配置的应用和模型提供方；移动端测试还会使用已配置的 Appium 与设备服务。可能产生模型费用或消耗订阅额度。 +- 初始化、编写测试和执行可能在应用项目中创建配置、YAML 测试、本地运行记录、截图、日志和知识。只读证据查看不会调用编写或执行操作，也不会修改结果标记或知识。 +- 运行时的分析统计、身份认证与数据处理遵循已安装的 agent-qa 版本及项目设置。只有配置的秘密值会被遮蔽；分享前应检查证据。不要提交凭据或将其粘贴到聊天中。 +- 修正结果可能修改可追溯的知识或预期，需要明确的修正任务并检查其影响。修正后的判定不等于重新执行测试的证据。 + +## 内容与许可证 + +插件包含一个技能以及两份参考文档，分别介绍项目设置与运行、证据与记忆。指令和清单由 vostride 按本市场的 Apache-2.0 许可证贡献。单独安装的 [agent-qa 运行时使用 FSL-1.1-ALv2](https://github.com/vostride/agent-qa/blob/main/LICENSE.md)，不包含在此插件中。 diff --git a/external_plugins/agent-qa/skills/agent-qa/SKILL.md b/external_plugins/agent-qa/skills/agent-qa/SKILL.md new file mode 100644 index 00000000..a65a72e6 --- /dev/null +++ b/external_plugins/agent-qa/skills/agent-qa/SKILL.md @@ -0,0 +1,63 @@ +--- +name: agent-qa +description: Operates agent-qa tests and interprets recorded run evidence and memidex knowledge. Use when working with agent-qa.config.yaml, agent-qa test or suite files, agent_qa MCP tools, or an explicit request to test web, Android, or iOS journeys with agent-qa. +license: Apache-2.0 +--- + +# agent-qa + +Use agent-qa from vostride for natural-language web and mobile testing with advanced memidex memory and self-improvement. Answer in the user's language. + +## Choose the requested work + +Locate the application's project instructions and `agent-qa.config.yaml`. +Distinguish authoring, execution, and review of existing evidence; a request to explain a saved run does not authorize a new run or a knowledge correction. +Preserve the configured provider, target, test identity, and acceptance criteria. +Do not introduce agent-qa into unrelated testing work solely because this skill is installed. + +Use the installed CLI version and its help output. These references were checked with agent-qa 0.2.3: + +| Work | Read | +| --- | --- | +| Setup, authoring, validation, execution, optional MCP | [Project and runs](references/project-and-runs.md) | +| Run verdicts, step evidence, artifacts, memory, corrections | [Evidence and memory](references/evidence-and-memory.md) | + +## Author or execute + +Read `agent-qa init --agent` for the installed capability catalog and `agent-qa skills --json` for version-matched package guidance. Discovery does not initialize the project. +Use the supported authoring contract; prefer connected MCP authoring tools when available. +The host creates canonical entity IDs. Never invent an ID or duplicate another test's ID for a new test. +Inspect the actual write or refusal receipt before reporting creation. A rejected draft is not a created test. +Authoring and verification can execute tests and call models; do not invoke them for an explicitly non-executing draft request. +Use configured `{{secret:NAME}}` references in steps. Recorded `[secret:NAME]` markers are masked output, not values to type. + +For an existing test, validation and `run --dry-run` check preparation and selection. They do not prove application behavior. +Execute only the requested journey or suite when execution is in scope; retain its actual run identity and inspect its terminal outcome and relevant evidence. +A setup failure is not an application failure. An acknowledgement or unfinished process is not a passing run. + +## Review recorded evidence + +Use supported CLI JSON readers or connected MCP tools. Start with the requested run ID, or list completed runs: + +```bash +npx agent-qa runs list --json +``` + +Match the run's target, test identity, completion time, and build to the request. +An empty list means matching completed evidence was not returned; it does not prove a pass or tell you whether another run is still active. +The digest is `agent-qa digest RUN_ID --json`, not a subcommand of `runs`. +Keep zero-based test and step indices attached to evidence and use IDs returned by the readers. +Inspect reader errors rather than treating them as empty successful results. +Never query or edit `.agent-qa/` databases, caches, or indexes directly. + +For memory work, read the `AGENTS.md` policy in the project's committed `agent-qa-knowledge` directory before changing it. +Distinguish what a run consumed from what it learned. A learned record or corrected verdict does not prove the application was rerun. +Preview and apply outcome corrections only within the user's authorized task, and report the actual receipt or refusal. +Do not weaken expected outcomes or erase failures to produce a passing result. + +## Report + +Name the target, test or suite, and actual run ID when available. +State the recorded outcome, supporting step evidence, and any missing coverage or setup limitation. +Separate preparation checks, execution results, and knowledge changes. +Review logs and screenshots before sharing: only configured secret values receive masking. diff --git a/external_plugins/agent-qa/skills/agent-qa/references/evidence-and-memory.md b/external_plugins/agent-qa/skills/agent-qa/references/evidence-and-memory.md new file mode 100644 index 00000000..a30f8fb8 --- /dev/null +++ b/external_plugins/agent-qa/skills/agent-qa/references/evidence-and-memory.md @@ -0,0 +1,126 @@ +# Recorded Evidence and memidex + +Use this reference when: Explaining recorded outcomes, inspecting artifacts, tracing knowledge, or reviewing an outcome correction. + +## Establish which run answers the question + +Start with the requested run ID, or list completed runs when it is unknown: + +```bash +npx agent-qa runs list --json +npx agent-qa runs list --test checkout --limit 20 --json +``` + +A title filter can return several tests. Match the target, test identity, completion time, and build information to the request. +Do not infer checkout coverage from a passing cart test or infer the current build's status from an older run. +An empty completed-run list means no matching completed record was returned. +It does not establish whether an asynchronous run is queued or active. +Check the relevant supported queue or returned stream when that distinction matters; do not start a replacement run by default. + +The following commands use `RUN_ID`, `RECORD_ID`, and `ARTIFACT_SHA` as placeholders. +Replace them with identifiers returned by the supported readers; do not invent identifiers. + +```bash +npx agent-qa digest RUN_ID --json +npx agent-qa runs overview RUN_ID --json +``` + +The digest is a top-level command. +The overview maps recorded tests, steps, and artifacts. +Use the overview's zero-based test and step indices when addressing evidence. +Read returned errors as errors, not as an empty successful result. + +## Inspect only the relevant evidence + +For a recorded first test and first step, these examples request structured documents: + +```bash +npx agent-qa runs evidence RUN_ID --test 0 --step 0 --kind console --json +npx agent-qa runs evidence RUN_ID --test 0 --step 0 --kind network --json +``` + +Choose indices from the actual overview rather than copying `0` automatically. +Other supported document kinds include accessibility, ARIA, and variables; consult `runs evidence --help` for exact flags. +A missing artifact narrows what can be concluded. It does not justify reconstructing a screenshot or inventing a browser observation. +Export a recorded screenshot or recording through the artifact reader: + +```bash +npx agent-qa runs evidence RUN_ID --screenshot ARTIFACT_SHA --out ./run-evidence.png +``` + +The output path must be new; the export command refuses to overwrite an existing file. +The `--screenshot` flag also accepts a recording's artifact SHA. Use an appropriate extension for the actual artifact. +Inspect the exported artifact before claiming what it shows. +Keep the artifact tied to its run, test, and step; a screenshot alone may not show the required final state. + +## Interpret results without extending coverage + +| Observation | Supported conclusion | What still needs evidence | +|-------------|----------------------|---------------------------| +| Test validates or dry run lists it | Configuration and selection checks succeeded | Actual application behavior | +| MCP returns a run acknowledgement | A run was acknowledged | Terminal status and test outcome | +| Matching completed test is recorded as passed | That recorded test passed in that run | Untested journeys or a different build | +| Test records a failed expected outcome | The run recorded that failure | Whether it is an application defect, stale expectation, or environment issue | +| No matching completed record is returned | Matching completed evidence is unavailable | Any claim that the requested journey passed | + +Do not silently relabel `healed`, `timeout`, or `cancelled` outcomes as `passed`. +Preserve the tool's recorded verdict and explain any uncertainty about its interpretation. +Report application defects separately from provider authentication, unavailable browsers, device setup, and other execution failures. +For a read-only request, end with evidence and the remaining gap rather than initiating a new test or correction. + +## Trace memory and self-improvement + +memidex stores reviewable application knowledge, including facts, procedures, cautions, and policies. +agent-qa can use and refine that knowledge across runs; a memory change is not proof of a passing test. +Read the `AGENTS.md` policy in the project’s committed `agent-qa-knowledge` directory before changing knowledge files. +Use the recorded knowledge readers to distinguish what a run consumed from what it learned: + +```bash +npx agent-qa runs knowledge RUN_ID --json +npx agent-qa knowledge inspect --json +npx agent-qa knowledge show RECORD_ID --json +``` + +Use the recorded brief trace when the question is why a particular step received a piece of knowledge: + +```bash +npx agent-qa knowledge brief-why RUN_ID --step 0 --json +``` + +For a multi-test run, add the actual test ID using `--test` when needed. +Read provenance and gate decisions before treating a record as accepted or current. +Use documented knowledge operations and the repository's knowledge policy for authorized corrections. +Do not edit `.agent-qa/` databases or derived indexes to manufacture a result, erase a failure, or bypass a gate. + +## Review an outcome correction + +An outcome mark is a mutation, not an annotation-only read: it can apply an attributable correction. +First compare the expectation and recorded evidence and identify the exact test and step. +When MCP is connected, `agent_qa_preview_run_mark` previews the effect without writing. +Inspect the current schema and preview for the requested mark, such as `false-fail`, `false-pass`, or `confirmed-bug`. +A preview may refuse an ambiguous or unsupported correction; resolve that ambiguity rather than choosing a nearby step. +Apply `agent_qa_mark_run` or the supported CLI mark only when that correction is within the user's authorized task. +Record its actual receipt and report any refusal or incomplete effect. +Do not claim that a correction or learned record reran the application or proved a previously untested behavior. + +## Use MCP equivalents when available + +| Need | MCP tool name before any client namespace | +|------|------------------------------------------| +| Completed runs | `agent_qa_list_runs` | +| Run verdict and decisions | `agent_qa_get_run_digest` | +| Test, step, and artifact map | `agent_qa_get_run_overview` | +| Step evidence | `agent_qa_get_run_evidence` | +| Knowledge used and learned | `agent_qa_get_run_knowledge` | +| Proposed correction effects | `agent_qa_preview_run_mark` | + +Discover actual tool availability and schemas on the connected server. +A tool error or missing reader is a limitation to report, not permission to inspect private storage internals. + +## Report only what the evidence supports + +Name the observed run, test, target, and relevant step when available. +Separate the recorded verdict, your interpretation, and any coverage gap. +Preserve configured secret placeholders; do not recover their original values for the report. +Review evidence before sharing it because arbitrary logs and screenshots can contain unmasked information. +Link local exported artifacts when useful, and describe any corrections actually applied with their receipts. diff --git a/external_plugins/agent-qa/skills/agent-qa/references/project-and-runs.md b/external_plugins/agent-qa/skills/agent-qa/references/project-and-runs.md new file mode 100644 index 00000000..3f667d7b --- /dev/null +++ b/external_plugins/agent-qa/skills/agent-qa/references/project-and-runs.md @@ -0,0 +1,150 @@ +# Project Setup and Runs + +Use this reference when: Configuring an agent-qa project, authoring tests, validating files, executing a journey, or connecting MCP. + +## Choose the existing project + +Read project instructions and locate `agent-qa.config.yaml` before installing or initializing anything. +Use the package manager and runtime version already selected by the repository. +For another configuration file, pass `--config` explicitly rather than relying on the shell's working directory. +A saved-run review only needs the existing project and supported data readers; do not reinitialize it. + +```bash +npx agent-qa --version +npx agent-qa --help +npx agent-qa --config ./agent-qa.config.yaml validate +``` + +The published 0.2.3 runtime requires Node.js 24 or newer. +If installation is part of the task, the npm setup is: + +```bash +npm install -D agent-qa +``` + +For Claude or Codex subscription authentication, also install its sign-in plugin: + +```bash +npm install -D @vostride/agent-qa-subscription-auth +``` + +API-key and compatible-endpoint setups do not require that subscription plugin. +Use the project's chosen provider; do not substitute a provider or invent an API key to get past a setup failure. +For a new project, inspect the current initialization choices: + +```bash +npx agent-qa init --help +npx agent-qa auth --help +``` + +Initialize only the intended test project. Preserve its existing configuration when it already exists. +Use the documented credential flow; do not ask the user to paste credentials into chat or commit them in test files. +The [quickstart](https://vostride.com/docs/agent-qa/quickstart) explains initialization and model sign-in. + +## Platform readiness + +For web execution, install the browser required by the project when needed: + +```bash +npx agent-qa install-browsers --chromium +``` + +Android and iOS execution use Appium with the appropriate device and driver setup. +Check the configured target and local device bindings before running a mobile test. +Do not invent device identifiers, app paths, or cloud-device credentials. +The [mobile guide](https://vostride.com/docs/agent-qa/guides/mobile-testing) covers UiAutomator2 and XCUITest setup. +Browser installation is not a prerequisite for reviewing already-recorded evidence. + +## Author through the installed contract + +Read the installed capability catalog and workspace posture before choosing an authoring operation: + +```bash +npx agent-qa init --agent +npx agent-qa skills --json +``` + +The first command reports discovery metadata without initializing or changing the workspace. +The second locates the package's version-matched skills and schema references. +Read the packaged authoring guidance when exact fields, ID rules, or tool migrations matter. +Use an existing configured target key; preserve the user's stated values and acceptance criteria. + +For a new test, call the connected `agent_qa_write_test` tool with its discovered schema. +For the stock example project's `komazon` target, an input is: + +```json +{ + "intent": "Create a storefront test that verifies the product catalog is visible.", + "scope": { "tier": "product", "key": "komazon" } +} +``` + +Use the actual target for a real application; the authoring scope is not an execution filter. +The host creates canonical IDs when it writes an entity. Do not invent an ID or copy another test's ID into a new file. +Read the written file, entity identity, verification disposition, and any refusal from the returned receipt. +A rejected draft is not an authored test. If a value-preservation check refuses the request, report the missing values rather than rephrasing around the check. +For an existing test, use `agent_qa_refine_test` with the current edit or verify action and the requested identity. +Its verify action evaluates the draft; it is not merely a text linter. +Authoring and verification can call models and execute tests. Respect an explicit request for a non-executing draft or read-only work. + +For files already present, CLI validation and dry-run selection are additional non-execution checks. +The stock initializer provides this example file: + +```bash +npx agent-qa validate tests/examples/komazon-checkout.yaml +npx agent-qa run tests/examples/komazon-checkout.yaml --dry-run +``` + +Neither command establishes that the browser journey passes, even if a dry run prints a run ID. +An unknown target or malformed existing test is a preparation problem to resolve through the supported authoring flow. +Do not weaken an expectation because the application fails it. + +## Execute and retain identity + +Prefer `agent_qa_start_run` when MCP is connected and execution is in scope. +Select files with `patterns`; the local 0.2.3 server rejects the advertised `scope` field for this tool. +When using the CLI, run the requested existing test: + +```bash +npx agent-qa run tests/examples/komazon-checkout.yaml +``` + +Use an existing suite path when the requested scope spans a suite. +Keep the actual outcome and run ID from the command or supported run readers. +A process that has not finished is still in progress; a setup error is not an application failure. +Read the completed run's digest and relevant step evidence before reporting a result. +A later rerun has a different evidentiary scope; retain the earlier failure instead of silently replacing it. +See `evidence-and-memory.md` for the supported readers and correction workflow. + +## Connect through MCP + +An MCP client may launch the server outside the test project's directory. +Supply an absolute configuration path and keep it as one argument, including when it contains spaces: + +```json +{ + "mcpServers": { + "agent-qa": { + "command": "npx", + "args": ["-y", "agent-qa@0.2.3", "--config", "/absolute/project/agent-qa.config.yaml", "mcp"] + } + } +} +``` + +Replace the example path with the intended project configuration. +Install and configure the project's model and optional authentication plugin before requesting execution. +For execution-capable tools, configure the client's tool-call timeout for the expected test or suite duration. +The [coding-agent guide](https://vostride.com/docs/agent-qa/coding-agents) uses 100 minutes for a default single test; larger suites may need more. +Discover tools from the connected server; use their returned schemas instead of guessing argument names. +`agent_qa_write_test` can author and verify a test, so treat it as execution-capable work, not a read-only query. +`agent_qa_start_run` acknowledges a run; inspect its returned identity and follow completion through supported readers or streams. +If only completed-run listing is available, an absent entry does not prove the run failed or never started. +Avoid starting duplicate runs while the original is still active or its state is unknown. + +## Finish gate + +Report preparation checks separately from execution results. +For an execution request, include the actual run outcome and evidence, or the concrete condition that prevented it. +For authoring work, report the actual written artifact and any verification performed by the authoring tool. +If the user explicitly forbids execution, do not invoke an authoring operation that runs tests; provide the proposed intent and explain that limitation. diff --git a/marketplace.json b/marketplace.json index 884bfc8c..d0bb19d4 100644 --- a/marketplace.json +++ b/marketplace.json @@ -1409,6 +1409,28 @@ "mcp", "verification" ] + }, + { + "name": "agent-qa", + "source": "./external_plugins/agent-qa", + "description": "Natural-language web and mobile testing with agent-qa from vostride, recorded run evidence, and advanced memidex memory and self-improvement.", + "description_i18n": { + "en": "Natural-language web and mobile testing with agent-qa from vostride, recorded run evidence, and advanced memidex memory and self-improvement.", + "zh-CN": "使用 vostride 的 agent-qa 进行自然语言 Web 与移动端测试，查看运行证据，并利用先进的 memidex 记忆引擎和自我改进能力。" + }, + "version": "0.1.0", + "author": { + "name": "vostride", + "url": "https://vostride.com" + }, + "keywords": [ + "testing", + "qa", + "web", + "mobile", + "memidex" + ], + "category": "developer-tools" } ] } \ No newline at end of file