From 6c28475473273fa87e535521e90149dffb481f95 Mon Sep 17 00:00:00 2001 From: dej611 Date: Fri, 31 Jul 2026 16:20:00 +0200 Subject: [PATCH 1/5] :sparkles: Add workflow validation script for CI --- .buildkite/default-pipeline.yml | 25 +- CONTRIBUTING.md | 18 +- package-lock.json | 175 +++++++++- package.json | 8 +- scripts/validate-workflows.mjs | 136 ++++++++ scripts/validate/README.md | 71 ++++ scripts/validate/compile-validators.mjs | 53 +++ scripts/validate/discover-files.mjs | 60 ++++ scripts/validate/index.mjs | 33 ++ scripts/validate/issues.mjs | 32 ++ scripts/validate/liquid/engine.mjs | 152 +++++++++ .../liquid/extract-error-position.mjs | 106 ++++++ .../validate/liquid/pick-object-fields.mjs | 103 ++++++ scripts/validate/liquid/regex.mjs | 45 +++ .../liquid/validate-liquid-template.mjs | 88 +++++ scripts/validate/load-schema.mjs | 109 ++++++ scripts/validate/report.mjs | 107 ++++++ scripts/validate/validate-file.mjs | 309 ++++++++++++++++++ scripts/validate/validate-liquid.mjs | 16 + scripts/validate/validate-schema.mjs | 87 +++++ scripts/validate/validate-semantics.mjs | 116 +++++++ scripts/validate/validate-worker.mjs | 52 +++ scripts/validate/variants.mjs | 2 + scripts/validate/yaml-parse.mjs | 25 ++ 24 files changed, 1911 insertions(+), 17 deletions(-) create mode 100644 scripts/validate-workflows.mjs create mode 100644 scripts/validate/README.md create mode 100644 scripts/validate/compile-validators.mjs create mode 100644 scripts/validate/discover-files.mjs create mode 100644 scripts/validate/index.mjs create mode 100644 scripts/validate/issues.mjs create mode 100644 scripts/validate/liquid/engine.mjs create mode 100644 scripts/validate/liquid/extract-error-position.mjs create mode 100644 scripts/validate/liquid/pick-object-fields.mjs create mode 100644 scripts/validate/liquid/regex.mjs create mode 100644 scripts/validate/liquid/validate-liquid-template.mjs create mode 100644 scripts/validate/load-schema.mjs create mode 100644 scripts/validate/report.mjs create mode 100644 scripts/validate/validate-file.mjs create mode 100644 scripts/validate/validate-liquid.mjs create mode 100644 scripts/validate/validate-schema.mjs create mode 100644 scripts/validate/validate-semantics.mjs create mode 100644 scripts/validate/validate-worker.mjs create mode 100644 scripts/validate/variants.mjs create mode 100644 scripts/validate/yaml-parse.mjs diff --git a/.buildkite/default-pipeline.yml b/.buildkite/default-pipeline.yml index 477c433..ea59dbd 100644 --- a/.buildkite/default-pipeline.yml +++ b/.buildkite/default-pipeline.yml @@ -4,8 +4,29 @@ steps: - label: "Build" command: "true" - - label: "Test" - command: "true" + # Validates workflow YAML against the workflow step JSON Schema (step-name + # uniqueness + LiquidJS syntax layered on top). Library templates are forced to + # the `template` variant and examples to the `strict` variant, so each tree is + # held to the schema it is meant to satisfy. The schema source is parameterized: + # set WORKFLOWS_SCHEMA_CDN_URL (a CDN base URL) or WORKFLOWS_SCHEMA (a local + # bundle directory) as a pipeline env var. Until a schema source is configured + # the step skips gracefully so CI stays green. + - label: "Validate workflows" + command: | + if [ -z "${WORKFLOWS_SCHEMA_CDN_URL:-}" ] && [ -z "${WORKFLOWS_SCHEMA:-}" ]; then + echo "Neither WORKFLOWS_SCHEMA_CDN_URL nor WORKFLOWS_SCHEMA is set; skipping workflow validation until the schema source is configured." + exit 0 + fi + npm ci + # A local bundle path takes precedence; otherwise the CLI reads + # WORKFLOWS_SCHEMA_CDN_URL from the environment automatically. + if [ -n "${WORKFLOWS_SCHEMA:-}" ]; then + npm run validate -- library/workflows --recursive --variant template --schema "${WORKFLOWS_SCHEMA}" + npm run validate -- examples --recursive --variant strict --schema "${WORKFLOWS_SCHEMA}" + else + npm run validate -- library/workflows --recursive --variant template + npm run validate -- examples --recursive --variant strict + fi - label: "Deploy" command: "true" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4683cfe..52844e8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -216,7 +216,23 @@ The generator enforces only: - `template-metadata` is present with all required fields (`slug`, `version`, `availability`, `name`, `description`, `categories`). - At least one template is discovered. -Deeper authoring invariants — slug ⇄ directory parity, valid `version` semver and `availability` range, `categories[]` membership in the vocab, `install.form` ⇄ `__install__` consistency, and step/connector type validity — are delegated to the separate validation step (planned to run in CI), not the generator. +Deeper authoring invariants — slug ⇄ directory parity, valid `version` semver and `availability` range, `categories[]` membership in the vocab, `install.form` ⇄ `__install__` consistency — remain the generator's / reviewer's responsibility. Step and connector **type** validity is enforced by the schema validator below. + +### Schema validation + +`npm run validate` checks workflow YAML (both `examples/` and `library/` templates) against the generated workflow **step JSON Schema**, layering step-name uniqueness and LiquidJS syntax checks on top. It is a plain-JS port of Kibana's `@kbn/workflow-yaml-validate-cli`; see [`scripts/validate/README.md`](./scripts/validate/README.md) for the full flag list and what each layer covers. + +The schema artifact is not bundled in this repo, so a source must be provided — a local bundle directory or a CDN base URL: + +```bash +# validate everything against a local schema bundle (a dir containing index.json) +npm run validate -- examples library/workflows --recursive --schema /path/to/workflow_step_schemas//release + +# or point at a CDN base URL (equivalently, set WORKFLOWS_SCHEMA_CDN_URL) +npm run validate -- examples library/workflows --recursive --schema-cdn-url https:/// +``` + +A non-zero exit means at least one file has an error. JSON-schema constraints on values that hold a LiquidJS expression (e.g. `{{ event.foo }}`) are reported as non-failing warnings, not errors. Execution-graph (DAG) validation is not part of this script yet. --- diff --git a/package-lock.json b/package-lock.json index 27d336a..5c11410 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,10 +14,14 @@ }, "devDependencies": { "@eslint/js": "9.39.5", + "ajv": "^8.20.0", + "ajv-formats": "^3.0.1", "eslint": "9.39.5", "eslint-config-prettier": "10.1.8", "globals": "17.7.0", - "prettier": "3.9.5" + "liquidjs": "^10.27.2", + "prettier": "3.9.5", + "yaml": "^2.9.0" }, "engines": { "node": ">=20" @@ -124,6 +128,23 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/@eslint/eslintrc/node_modules/ajv": { + "version": "6.15.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.15.0.tgz", + "integrity": "sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, "node_modules/@eslint/eslintrc/node_modules/globals": { "version": "14.0.0", "resolved": "https://registry.npmjs.org/globals/-/globals-14.0.0.tgz", @@ -136,6 +157,13 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/@eslint/eslintrc/node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true, + "license": "MIT" + }, "node_modules/@eslint/js": { "version": "9.39.5", "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.39.5.tgz", @@ -266,21 +294,40 @@ } }, "node_modules/ajv": { - "version": "6.15.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.15.0.tgz", - "integrity": "sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==", + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", "dev": true, + "license": "MIT", "dependencies": { - "fast-deep-equal": "^3.1.1", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.4.1", - "uri-js": "^4.2.2" + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" }, "funding": { "type": "github", "url": "https://github.com/sponsors/epoberezkin" } }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, "node_modules/ansi-styles": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", @@ -360,6 +407,16 @@ "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", "dev": true }, + "node_modules/commander": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-10.0.1.tgz", + "integrity": "sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14" + } + }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -519,6 +576,30 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/eslint/node_modules/ajv": { + "version": "6.15.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.15.0.tgz", + "integrity": "sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/eslint/node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true, + "license": "MIT" + }, "node_modules/espree": { "version": "10.4.0", "resolved": "https://registry.npmjs.org/espree/-/espree-10.4.0.tgz", @@ -588,7 +669,8 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/fast-levenshtein": { "version": "2.0.6", @@ -596,6 +678,23 @@ "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", "dev": true }, + "node_modules/fast-uri": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.4.tgz", + "integrity": "sha512-8JnbkQ4juDyvYs4mgFGQqg4yCYtFDtUtmp2QIQq11ZZe5CFQ5wcqm1rqDgAh/QdMySuBnPzMUiJUNZG5N/AiQw==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, "node_modules/file-entry-cache": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz", @@ -767,10 +866,11 @@ "dev": true }, "node_modules/json-schema-traverse": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", - "dev": true + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "dev": true, + "license": "MIT" }, "node_modules/json-stable-stringify-without-jsonify": { "version": "1.0.1", @@ -800,6 +900,27 @@ "node": ">= 0.8.0" } }, + "node_modules/liquidjs": { + "version": "10.27.2", + "resolved": "https://registry.npmjs.org/liquidjs/-/liquidjs-10.27.2.tgz", + "integrity": "sha512-kvknfAEtOHjHkAAv7GxLEJh8ghpMQm3Fc4uWVyF7hERSTsSRsdC7saWs0p5aDG7GDcWsu5o+T4232O+8KZO55w==", + "dev": true, + "license": "MIT", + "dependencies": { + "commander": "^10.0.0" + }, + "bin": { + "liquid": "bin/liquid.js", + "liquidjs": "bin/liquid.js" + }, + "engines": { + "node": ">=16" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/liquidjs" + } + }, "node_modules/locate-path": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", @@ -952,10 +1073,21 @@ "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", "dev": true, + "license": "MIT", "engines": { "node": ">=6" } }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/resolve-from": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", @@ -1039,6 +1171,7 @@ "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", "dev": true, + "license": "BSD-2-Clause", "dependencies": { "punycode": "^2.1.0" } @@ -1067,6 +1200,22 @@ "node": ">=0.10.0" } }, + "node_modules/yaml": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.0.tgz", + "integrity": "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==", + "dev": true, + "license": "ISC", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14.6" + }, + "funding": { + "url": "https://github.com/sponsors/eemeli" + } + }, "node_modules/yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", diff --git a/package.json b/package.json index 1bdcb73..e059e8d 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,8 @@ }, "scripts": { "build:catalog": "node scripts/build-catalog.mjs", + "validate": "node scripts/validate-workflows.mjs", + "validate:all": "node scripts/validate-workflows.mjs examples library/workflows --recursive", "lint": "eslint .", "lint:fix": "eslint . --fix", "format": "prettier --write .", @@ -21,9 +23,13 @@ }, "devDependencies": { "@eslint/js": "9.39.5", + "ajv": "^8.20.0", + "ajv-formats": "^3.0.1", "eslint": "9.39.5", "eslint-config-prettier": "10.1.8", "globals": "17.7.0", - "prettier": "3.9.5" + "liquidjs": "^10.27.2", + "prettier": "3.9.5", + "yaml": "^2.9.0" } } diff --git a/scripts/validate-workflows.mjs b/scripts/validate-workflows.mjs new file mode 100644 index 0000000..621910f --- /dev/null +++ b/scripts/validate-workflows.mjs @@ -0,0 +1,136 @@ +#!/usr/bin/env node +/** + * validate-workflows.mjs — Workflow YAML validator. + * + * Validates workflow YAML (files or folders) against the generated workflow + * step JSON Schema artifact, then layers step-name uniqueness and LiquidJS + * syntax checks. JSON-schema constraints on values that hold a LiquidJS + * expression are downgraded to non-failing warnings. + * + * This is the plain-JS port of Kibana's `@kbn/workflow-yaml-validate-cli` + * (elastic/kibana#281827), trimmed for this repo: no strict zod metadata + * validation and no DAG check (see scripts/validate/README.md). + * + * Usage: + * node scripts/validate-workflows.mjs [flags] + * + * Flags: + * --recursive, -r Descend into subdirectories (default: top-level only) + * --summary-only Suppress per-file streaming; print only failures + summary + * --variant Schema variant: auto | strict | template (default: auto) + * --schema Explicit schema source: a bundle directory or http(s):// base URL + * --schema-cdn-url CDN base URL fallback (or set WORKFLOWS_SCHEMA_CDN_URL) + * --json Write a structured JSON report to this path + */ + +import fs from 'node:fs'; +import { parseArgs } from 'node:util'; +import { discoverFiles } from './validate/discover-files.mjs'; +import { loadSchemaDocuments } from './validate/load-schema.mjs'; +import { createSchemaValidator } from './validate/validate-schema.mjs'; +import { validateWorkflowYaml } from './validate/index.mjs'; +import { printFileResult, printSummary, writeJsonReport } from './validate/report.mjs'; +import { VARIANTS } from './validate/variants.mjs'; + +// Minimal console logger mirroring the ToolingLog surface the report uses. +const log = { + info: (...args) => console.log('[validate]', ...args), + success: (...args) => console.log(...args), + warning: (...args) => console.warn(...args), + error: (...args) => console.error(...args), +}; + +const VARIANT_MODES = ['auto', ...VARIANTS]; + +const parseVariantMode = (value) => { + const variant = value && value.length > 0 ? value : 'auto'; + if (!VARIANT_MODES.includes(variant)) { + throw new Error( + `Invalid --variant "${variant}". Expected one of: ${VARIANT_MODES.join(', ')}.` + ); + } + return variant; +}; + +const emptyToUndefined = (value) => + typeof value === 'string' && value.length > 0 ? value : undefined; + +async function main() { + const { values, positionals } = parseArgs({ + allowPositionals: true, + options: { + recursive: { type: 'boolean', short: 'r', default: false }, + 'summary-only': { type: 'boolean', default: false }, + variant: { type: 'string' }, + schema: { type: 'string' }, + 'schema-cdn-url': { type: 'string' }, + json: { type: 'string' }, + }, + }); + + const targets = positionals.length > 0 ? positionals : ['.']; + const recursive = Boolean(values.recursive); + const summaryOnly = Boolean(values['summary-only']); + const variantMode = parseVariantMode(values.variant); + const schema = emptyToUndefined(values.schema); + const cdnUrl = + emptyToUndefined(values['schema-cdn-url']) ?? + emptyToUndefined(process.env.WORKFLOWS_SCHEMA_CDN_URL); + const jsonOutput = emptyToUndefined(values.json); + + // Discover files across every target, de-duped and sorted for determinism. + const files = [ + ...new Set(targets.flatMap((target) => discoverFiles(target, { recursive }))), + ].sort(); + if (files.length === 0) { + log.warning(`No workflow YAML files found in ${targets.join(', ')}. Nothing to validate.`); + return; + } + + const { schemas, source } = await loadSchemaDocuments({ schema, cdnUrl }); + log.info(`Validating ${files.length} file(s) against schema from ${source}`); + + // The workflow schema is deeply recursive, so compile + validate in a worker + // thread with an enlarged stack (see validate-schema.mjs). + const { validateSchema, close } = createSchemaValidator(schemas); + const outcomes = []; + + try { + // Stream each file's result as it completes. + for (const file of files) { + const yaml = await fs.promises.readFile(file, 'utf8'); + const outcome = await validateWorkflowYaml({ file, yaml, validateSchema, variantMode }); + outcomes.push(outcome); + if (!summaryOnly) { + printFileResult(log, outcome); + } + } + } finally { + await close(); + } + + if (summaryOnly) { + // Surface failures and pass-with-warnings files so skipped LiquidJS + // positions are not silently hidden in summary-only mode. + for (const outcome of outcomes) { + if (!outcome.ok || outcome.issues.length > 0) { + printFileResult(log, outcome); + } + } + } + printSummary(log, outcomes); + + if (jsonOutput) { + writeJsonReport(jsonOutput, source, outcomes); + log.info(`Wrote JSON report to ${jsonOutput}`); + } + + if (outcomes.some((outcome) => !outcome.ok)) { + process.exitCode = 1; + } +} + +main().catch((error) => { + log.error(`[validate] FAILED: ${error instanceof Error ? error.message : String(error)}`); + process.exitCode = 1; +}); diff --git a/scripts/validate/README.md b/scripts/validate/README.md new file mode 100644 index 0000000..d324321 --- /dev/null +++ b/scripts/validate/README.md @@ -0,0 +1,71 @@ +# Workflow YAML validator + +Validates workflow YAML (a single file or folders of files) against the +generated workflow step JSON Schema, layering step-name uniqueness and LiquidJS +syntax checks on top. It is the plain-JS (ESM) port of Kibana's +`@kbn/workflow-yaml-validate-cli` ([elastic/kibana#281827](https://github.com/elastic/kibana/pull/281827)), +trimmed for this repo. + +## Usage + +```bash +node scripts/validate-workflows.mjs [flags] + +# validate both trees against a local schema bundle +node scripts/validate-workflows.mjs examples library/workflows --recursive \ + --schema /path/to/workflow_step_schemas/9.6.0/release + +# or against a CDN base URL (equivalently: WORKFLOWS_SCHEMA_CDN_URL env var) +node scripts/validate-workflows.mjs examples library/workflows -r \ + --schema-cdn-url https:/// +``` + +The npm scripts `npm run validate` (bare) and `npm run validate:all` +(`examples library/workflows --recursive`) wrap this entry point. + +| Flag | Description | Default | +| --- | --- | --- | +| `--recursive`, `-r` | Descend into subdirectories | off (top-level only) | +| `--summary-only` | Suppress per-file streaming; print only failures + the summary | off | +| `--variant ` | Force a schema variant: `strict` or `template` | `auto` (per file) | +| `--schema ` | Explicit schema source: a bundle directory or an `http(s)://` base URL | — | +| `--schema-cdn-url ` | CDN base URL fallback (or set `WORKFLOWS_SCHEMA_CDN_URL`) | — | +| `--json ` | Write a structured JSON report to this path | — | + +## Schema source resolution + +The schema bundle (the directory/URL containing `index.json`) is resolved from, +in order: `--schema`, then `--schema-cdn-url` / `WORKFLOWS_SCHEMA_CDN_URL`. There +is no built-in default and no local `target/` lookup (this repo has no Kibana +build tree). Each variant's `schema.json` is integrity-checked against the +`sha256` recorded in `index.json` before use. + +## What it checks + +Three layers run per file: + +1. **JSON Schema** — the document is validated with `ajv` (draft-07) against the + `strict` or `template` variant. Plain workflows (`examples/`) use `strict`; + templates (files with a `template-metadata` block) have that block stripped + and the remaining body validated against `template`. The step/trigger unions + carry a `discriminator`, so each step is validated only against its `type`'s + branch — precise errors, no cross-branch explosion. Constraints on values + that hold a whole-value LiquidJS expression (`{{ … }}`, `${{ … }}`, or a + `{% … %}` tag) are downgraded to non-failing warnings. +2. **Step-name uniqueness** — duplicate step names (including nested steps) are + reported. Runs only when the schema layer passed for that file. +3. **LiquidJS syntax** — every `{{ }}` / `{% %}` scalar is parsed with the real + LiquidJS engine. Runs unconditionally. Scope is syntax only. + +The process exits non-zero if any file has an error-severity issue. Warnings +alone do not fail the run. An empty target logs a warning and exits 0. + +## Intentional differences from the Kibana CLI + +- **No DAG check.** Execution-graph (cycle) validation depends on Kibana's + ~1355-line execution-graph builder + its zod spec schema + `@dagrejs/dagre`, + which would drag a large, fast-churning slice of Kibana into this repo. The + hook is documented in `validate-semantics.mjs`. +- **No strict metadata schema.** `template-metadata` is detected and stripped, + but not validated against a zod schema — `scripts/build-catalog.mjs` already + enforces its required fields. diff --git a/scripts/validate/compile-validators.mjs b/scripts/validate/compile-validators.mjs new file mode 100644 index 0000000..fa2fa32 --- /dev/null +++ b/scripts/validate/compile-validators.mjs @@ -0,0 +1,53 @@ +/** + * ajv compilation of the schema variants — ported from Kibana's + * `compile_validators.ts`. Relies on ajv's native `discriminator` support so a + * step is validated only against its `type`'s branch (precise errors, no + * cross-branch explosion). + */ + +import Ajv from 'ajv'; +import addFormats from 'ajv-formats'; +import { VARIANTS } from './variants.mjs'; + +/** ajv options for the full-document validator. */ +export const AJV_OPTIONS = { + // The artifact is already generated as valid draft-07, so skip meta-schema + // self-validation (also avoids runtime meta-schema codegen). + validateSchema: false, + // The workflow schema is deeply recursive (nested steps); inlining recursive + // refs blows the call stack on real workflows. + inlineRefs: false, + // With the discriminator anchoring each step to its branch, we can safely + // collect all real issues at once. + allErrors: true, + // Enable native `discriminator` support (the artifact's step/trigger unions). + discriminator: true, + strict: false, +}; + +/** + * Compile a variant document into a validator. A fresh Ajv per variant keeps + * their (identically-named) internal refs isolated. + */ +export const compileVariant = (doc, variant) => { + const ajv = new Ajv(AJV_OPTIONS); + addFormats(ajv); + try { + return { validate: ajv.compile(doc) }; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new Error( + `Failed to compile the "${variant}" workflow schema. The artifact may be stale or ` + + `predate native discriminator support. Underlying ajv error: ${message}` + ); + } +}; + +/** Compile both schema variants into ajv validators. */ +export const compileValidators = (schemas) => { + const validators = {}; + for (const variant of VARIANTS) { + validators[variant] = compileVariant(schemas[variant], variant); + } + return validators; +}; diff --git a/scripts/validate/discover-files.mjs b/scripts/validate/discover-files.mjs new file mode 100644 index 0000000..47707c5 --- /dev/null +++ b/scripts/validate/discover-files.mjs @@ -0,0 +1,60 @@ +/** + * Resolve the set of workflow YAML files to validate — ported from Kibana's + * `discover_files.ts`. + * + * The target may be a single `.yml`/`.yaml` file (returned as-is) or a + * directory. Directories are scanned for `*.yml`/`*.yaml`, skipping dotfiles and + * dot-directories; `recursive` controls whether subdirectories are descended. + * The result is sorted for deterministic output. + */ + +import fs from 'node:fs'; +import path from 'node:path'; + +const YAML_EXT = /\.ya?ml$/i; + +const walk = (dir, recursive, out) => { + const entries = fs.readdirSync(dir, { withFileTypes: true }); + for (const entry of entries) { + // Skip dotfiles and dot-directories (e.g. `.git`, `.DS_Store`). + if (entry.name.startsWith('.')) { + continue; + } + const absolute = path.join(dir, entry.name); + if (entry.isDirectory()) { + if (recursive) { + walk(absolute, recursive, out); + } + continue; + } + if (entry.isFile() && YAML_EXT.test(entry.name)) { + out.push(absolute); + } + } +}; + +export const discoverFiles = (target, { recursive = false } = {}) => { + const absolute = path.resolve(target); + + let stat; + try { + stat = fs.statSync(absolute); + } catch { + throw new Error(`Path does not exist: ${absolute}`); + } + + if (stat.isFile()) { + if (!YAML_EXT.test(absolute)) { + throw new Error(`Not a YAML file (expected .yml/.yaml): ${absolute}`); + } + return [absolute]; + } + + if (!stat.isDirectory()) { + throw new Error(`Path is neither a file nor a directory: ${absolute}`); + } + + const found = []; + walk(absolute, recursive, found); + return found.sort(); +}; diff --git a/scripts/validate/index.mjs b/scripts/validate/index.mjs new file mode 100644 index 0000000..91deb12 --- /dev/null +++ b/scripts/validate/index.mjs @@ -0,0 +1,33 @@ +/** + * Per-file orchestration — ported from Kibana's `validate.ts`. + * + * Runs the three validation layers for one file and merges their issues: + * 1. JSON Schema (+ template detection/strip) — always. + * 2. Semantic (step-name uniqueness) — only when the schema layer passed. + * 3. LiquidJS syntax — always. + */ + +import { validateFile } from './validate-file.mjs'; +import { validateSemantics } from './validate-semantics.mjs'; +import { validateLiquid } from './validate-liquid.mjs'; +import { isErrorIssue } from './issues.mjs'; + +export const validateWorkflowYaml = async ({ file, yaml, validateSchema, variantMode }) => { + const schemaResult = await validateFile({ yaml, validateSchema, variantMode }); + const issues = [...schemaResult.issues]; + + if (schemaResult.schemaPassed && schemaResult.body) { + issues.push(...validateSemantics(schemaResult.body)); + } + + issues.push(...validateLiquid(yaml, schemaResult.document)); + + return { + file, + // Warnings (e.g. skipped LiquidJS positions) never fail the run. + ok: !issues.some(isErrorIssue), + isTemplate: schemaResult.isTemplate, + variant: schemaResult.variant, + issues, + }; +}; diff --git a/scripts/validate/issues.mjs b/scripts/validate/issues.mjs new file mode 100644 index 0000000..c980528 --- /dev/null +++ b/scripts/validate/issues.mjs @@ -0,0 +1,32 @@ +/** + * Issue/outcome shapes shared across the validation layers — ported from + * Kibana's `types.ts` (types stripped for plain JS; kept as JSDoc for clarity). + * + * An issue's `source` is one of: + * 'yaml-syntax' | 'schema' | 'metadata' | 'step-name' | 'graph' | + * 'liquid' | 'liquidjs-expression' + * + * `liquidjs-expression` is distinct from `liquid`: the latter is a LiquidJS + * *syntax* error, while the former marks a schema constraint intentionally + * skipped because the value is a whole-value LiquidJS expression. + * + * `severity` is 'error' | 'warning'; an absent severity is treated as an error. + * + * @typedef {Object} ValidationIssue + * @property {string} source + * @property {string} message + * @property {('error'|'warning')} [severity] + * @property {string} [path] Dotted instance path (schema/metadata/step-name/graph). + * @property {number} [line] 1-based source line (liquid issues). + * @property {number} [column] 1-based source column (liquid issues). + * + * @typedef {Object} ValidationOutcome + * @property {string} file + * @property {boolean} ok True when there are no error-severity issues. + * @property {boolean} isTemplate + * @property {(string|null)} variant + * @property {ValidationIssue[]} issues + */ + +/** An issue fails the run unless it is explicitly a warning. */ +export const isErrorIssue = (issue) => issue.severity !== 'warning'; diff --git a/scripts/validate/liquid/engine.mjs b/scripts/validate/liquid/engine.mjs new file mode 100644 index 0000000..71aa86f --- /dev/null +++ b/scripts/validate/liquid/engine.mjs @@ -0,0 +1,152 @@ +/** + * LiquidJS engine configured for workflow templates — ported from Kibana's + * `@kbn/workflows` `createWorkflowLiquidEngine`. Restricts tags to the supported + * set and registers the custom filters so `engine.parse()` does not throw on + * known filter/tag names during syntax validation. + */ + +import { Liquid } from 'liquidjs'; +import { pickObjectFields } from './pick-object-fields.mjs'; + +/** + * LiquidJS tags supported in workflow templates. Tags not in this set are + * removed from the engine, so LiquidJS treats them as unknown (parse error). + */ +export const LIQUID_ALLOWED_TAGS = new Set([ + 'assign', + 'for', + 'capture', + 'case', + 'comment', + 'decrement', + 'increment', + 'cycle', + 'if', + 'unless', + 'break', + 'continue', + 'raw', + 'echo', + 'liquid', + '#', +]); + +// A no-op filesystem: workflow templates do not support file operations. +const noopFs = { + exists: async () => false, + existsSync: () => false, + readFile: async (filepath) => { + throw new Error( + `File reading is not supported in workflow templates. Attempted to read: ${filepath}` + ); + }, + readFileSync: (filepath) => { + throw new Error( + `File reading is not supported in workflow templates. Attempted to read: ${filepath}` + ); + }, + resolve: (_dir, file) => file, + contains: async () => false, +}; + +const removeDisallowedLiquidTags = (engine) => { + for (const tagName of Object.keys(engine.tags)) { + if (!LIQUID_ALLOWED_TAGS.has(tagName)) { + delete engine.tags[tagName]; + } + } +}; + +/** + * Registers the custom filters required by workflow templates. Registering them + * centrally keeps every engine instance in step with the runtime. + */ +export const registerWorkflowLiquidFilters = (engine) => { + engine.registerFilter('json_parse', (value) => { + if (typeof value !== 'string') { + return value; + } + try { + return JSON.parse(value); + } catch { + return value; + } + }); + + engine.registerFilter('entries', (value) => { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + return value; + } + return Object.entries(value).map(([k, v]) => ({ key: k, value: v })); + }); + + engine.registerFilter('pick', (value, ...args) => { + const paths = args.length === 1 && Array.isArray(args[0]) ? args[0] : args; + return pickObjectFields( + value, + paths.filter((path) => typeof path === 'string') + ); + }); +}; + +/** + * Creates a LiquidJS engine configured for workflow templates. Extra options + * (e.g. `strictFilters`) are merged with the enforced defaults. + */ +export const createWorkflowLiquidEngine = (options) => { + const { parseLimit = 150_000, renderLimit = 1_000, memoryLimit = 15_000_000 } = options ?? {}; + + const engine = new Liquid({ + ...options, + ownPropertyOnly: true, + fs: noopFs, + relativeReference: false, + templates: {}, + parseLimit, + renderLimit, + memoryLimit, + }); + removeDisallowedLiquidTags(engine); + registerWorkflowLiquidFilters(engine); + return engine; +}; + +let liquidInstance = null; + +/** + * Returns the shared workflow Liquid engine instance (lazy-initialized). + * Registers stub filters so `parse()` does not throw on known filter names. + */ +export function getLiquidInstance() { + if (!liquidInstance) { + liquidInstance = createWorkflowLiquidEngine({ + strictFilters: true, + strictVariables: false, + }); + } + return liquidInstance; +} + +const MAX_PARSE_CACHE_SIZE = 64; +const parseCache = new Map(); + +/** + * Parses a Liquid template string with a bounded LRU cache. + */ +export function parseTemplateString(templateString) { + const cached = parseCache.get(templateString); + if (cached) { + parseCache.delete(templateString); + parseCache.set(templateString, cached); + return cached; + } + const result = getLiquidInstance().parse(templateString); + parseCache.set(templateString, result); + if (parseCache.size > MAX_PARSE_CACHE_SIZE) { + const firstKey = parseCache.keys().next().value; + if (firstKey !== undefined) { + parseCache.delete(firstKey); + } + } + return result; +} diff --git a/scripts/validate/liquid/extract-error-position.mjs b/scripts/validate/liquid/extract-error-position.mjs new file mode 100644 index 0000000..1db1370 --- /dev/null +++ b/scripts/validate/liquid/extract-error-position.mjs @@ -0,0 +1,106 @@ +/** + * Extracts position information from liquidjs error messages — ported from + * Kibana's `@kbn/workflows-yaml` `extractLiquidErrorPosition`. liquidjs reports + * the start of the error line; this pinpoints the specific problematic token. + */ + +const DEFAULT_ERROR_HIGHLIGHT_EXTENSION = 10; + +/** + * Tries to find the specific token causing the error based on message patterns. + */ +const findSpecificErrorToken = (text, errorMessage) => { + const filterMatch = errorMessage.match(/undefined filter:\s*([a-zA-Z_]\w*)/); + if (filterMatch) { + const filterName = filterMatch[1]; + const filterRegex = new RegExp(`\\|\\s*${filterName}\\b`, 'g'); + const match = filterRegex.exec(text); + if (match) { + const pipeIndex = match.index; + const filterStart = text.indexOf(filterName, pipeIndex); + return { start: filterStart, end: filterStart + filterName.length }; + } + } + + const tagMatch = errorMessage.match(/tag ['"](.*?)['"] not found/); + if (tagMatch) { + const tagName = tagMatch[1]; + const tagIndex = text.indexOf(tagName); + if (tagIndex !== -1) { + return { start: tagIndex, end: tagIndex + tagName.length }; + } + } + + const unclosedMatch = errorMessage.match(/(output|tag) ['"](\{\{.*?|\{%.*?)['"] not closed/); + if (unclosedMatch) { + const content = unclosedMatch[2]; + const contentIndex = text.indexOf(content); + if (contentIndex !== -1) { + return { + start: contentIndex, + end: Math.min( + contentIndex + content.length + DEFAULT_ERROR_HIGHLIGHT_EXTENSION, + text.length + ), + }; + } + } + + if (errorMessage.includes('invalid value expression')) { + const emptyExpressionMatch = text.match(/\{\{\s*\}\}/); + if (emptyExpressionMatch && emptyExpressionMatch.index !== undefined) { + return { + start: emptyExpressionMatch.index, + end: emptyExpressionMatch.index + emptyExpressionMatch[0].length, + }; + } + } + + return null; +}; + +export const extractLiquidErrorPosition = (text, errorMessage) => { + const specificTokenPosition = findSpecificErrorToken(text, errorMessage); + if (specificTokenPosition) { + return specificTokenPosition; + } + + const lineColMatch = errorMessage.match(/line:(\d+),\s*col:(\d+)/); + if (lineColMatch) { + const line = parseInt(lineColMatch[1], 10); + const col = parseInt(lineColMatch[2], 10); + + const lines = text.split('\n'); + let offset = 0; + for (let i = 0; i < line - 1 && i < lines.length; i++) { + offset += lines[i].length + 1; // +1 for newline character + } + offset += Math.max(0, col - 1); + + const remainingText = text.substring(offset); + let end = offset + 1; + if (remainingText.startsWith('{{')) { + const closeMatch = remainingText.indexOf('}}'); + end = offset + (closeMatch > -1 ? closeMatch + 2 : Math.min(50, remainingText.length)); + } else if (remainingText.startsWith('{%')) { + const closeMatch = remainingText.indexOf('%}'); + end = offset + (closeMatch > -1 ? closeMatch + 2 : Math.min(50, remainingText.length)); + } else { + const wordMatch = remainingText.match(/^\S+/); + end = offset + (wordMatch ? wordMatch[0].length : DEFAULT_ERROR_HIGHLIGHT_EXTENSION); + } + + return { start: Math.max(0, offset), end: Math.min(text.length, end) }; + } + + const liquidPattern = /\{\{|\{%/g; + let match; + while ((match = liquidPattern.exec(text)) !== null) { + const start = match.index; + const remaining = text.substring(start); + const end = remaining.search(/\}\}|%\}/) + start; + return { start, end: end > start ? end + 2 : Math.min(start + 20, text.length) }; + } + + return { start: 0, end: Math.min(1, text.length) }; +}; diff --git a/scripts/validate/liquid/pick-object-fields.mjs b/scripts/validate/liquid/pick-object-fields.mjs new file mode 100644 index 0000000..a9d311c --- /dev/null +++ b/scripts/validate/liquid/pick-object-fields.mjs @@ -0,0 +1,103 @@ +/** + * pickObjectFields — returns a new object containing only the given dotted-path + * fields from `source`, preserving the original nested structure and value types. + * + * Ported from Kibana's `@kbn/workflows` `pickObjectFields`. Only used by the + * `pick` Liquid filter (see engine.mjs); template validation parses but never + * renders, so this never actually runs — it exists so the filter is registered. + */ + +const isPlainObject = (value) => + typeof value === 'object' && value !== null && !Array.isArray(value); + +// A segment addresses an array element when it is a run of digits (e.g. `9`). +const isArrayIndex = (segment) => /^\d+$/.test(segment); + +// Keys that would let a crafted path walk into the prototype chain. +const UNSAFE_SEGMENTS = new Set(['__proto__', 'prototype', 'constructor']); + +const getChild = (container, key) => + Array.isArray(container) ? container[Number(key)] : container[key]; + +const setChild = (container, key, value) => { + if (Array.isArray(container)) { + container[Number(key)] = value; + } else { + container[key] = value; + } +}; + +// Deep clone limited to JSON-serializable values. +const cloneValue = (value) => { + if (value === null || typeof value !== 'object') { + return value; + } + if (Array.isArray(value)) { + return value.map(cloneValue); + } + const clone = {}; + for (const [key, val] of Object.entries(value)) { + clone[key] = cloneValue(val); + } + return clone; +}; + +const copyPath = (source, result, path) => { + if (path.length === 0) { + return; + } + + const segments = path.split('.'); + if (segments.some((segment) => UNSAFE_SEGMENTS.has(segment))) { + return; + } + + let cursor = source; + for (const segment of segments) { + if (isPlainObject(cursor) && Object.prototype.hasOwnProperty.call(cursor, segment)) { + cursor = cursor[segment]; + } else if ( + Array.isArray(cursor) && + isArrayIndex(segment) && + Object.prototype.hasOwnProperty.call(cursor, segment) + ) { + cursor = cursor[Number(segment)]; + } else { + return; + } + } + + if (cursor === undefined) { + return; + } + + let target = result; + for (let i = 0; i < segments.length - 1; i++) { + const segment = segments[i]; + const childShouldBeArray = isArrayIndex(segments[i + 1]); + const existing = getChild(target, segment); + const existingIsRightType = childShouldBeArray + ? Array.isArray(existing) + : isPlainObject(existing); + if (!existingIsRightType) { + setChild(target, segment, childShouldBeArray ? [] : {}); + } + target = getChild(target, segment); + } + setChild(target, segments[segments.length - 1], cloneValue(cursor)); +}; + +export const pickObjectFields = (source, paths) => { + if (!isPlainObject(source)) { + return source; + } + + const result = {}; + for (const path of paths) { + if (typeof path === 'string') { + copyPath(source, result, path); + } + } + + return result; +}; diff --git a/scripts/validate/liquid/regex.mjs b/scripts/validate/liquid/regex.mjs new file mode 100644 index 0000000..83269a1 --- /dev/null +++ b/scripts/validate/liquid/regex.mjs @@ -0,0 +1,45 @@ +/** + * LiquidJS value predicates — ported from Kibana's `@kbn/workflows-yaml` + * `common/regex.ts`. Kept in lock-step with the runtime so the CLI, runtime, and + * weaver share one notion of "this value is a dynamic/template expression". + */ + +export const DYNAMIC_VALUE_REGEX = /^\$\{\{\s*\S[\s\S]*\}\}$/; + +/** + * Whole-value dynamic/templated value: `${{ ... }}`. + * Examples: `${{env.USER}}`, `${{ref:myVar}}`, `${{someVariable}}`. + */ +export function isDynamicValue(value) { + if (typeof value !== 'string') { + return false; + } + return DYNAMIC_VALUE_REGEX.test(value); +} + +export const VARIABLE_VALUE_REGEX = /^\{\{\s*\S[\s\S]*\}\}$/; + +/** + * Whole-value variable expression: `{{ ... }}`. + * Examples: `{{ variable }}`, `{{ variable | filter }}`. + */ +export function isVariableValue(value) { + if (typeof value !== 'string') { + return false; + } + return VARIABLE_VALUE_REGEX.test(value); +} + +// Matches Liquid tags: `{% ... %}` or `{%- ... -%}`. The `s` flag lets `.` +// match newlines so multi-line tag blocks are matched too. +export const LIQUID_TAG_VALUE_REGEX = /\{%-?\s*[^%]*?\s*-?%\}/s; + +/** + * Contains a Liquid tag: `{% ... %}` / `{%- ... -%}` (single- or multi-line). + */ +export function isLiquidTagValue(value) { + if (typeof value !== 'string') { + return false; + } + return LIQUID_TAG_VALUE_REGEX.test(value); +} diff --git a/scripts/validate/liquid/validate-liquid-template.mjs b/scripts/validate/liquid/validate-liquid-template.mjs new file mode 100644 index 0000000..8b34c9b --- /dev/null +++ b/scripts/validate/liquid/validate-liquid-template.mjs @@ -0,0 +1,88 @@ +/** + * LiquidJS syntax validation — ported from Kibana's `@kbn/workflows-yaml` + * `validateLiquidTemplate`. Visits every scalar in the parsed YAML document, + * parses any `{{ }}` / `{% %}` content with the real LiquidJS engine, and maps + * parse errors back to absolute line/column positions in the source string. + */ + +import { Scalar, visit } from 'yaml'; +import { extractLiquidErrorPosition } from './extract-error-position.mjs'; +import { parseTemplateString } from './engine.mjs'; + +function convertOffsetToLineColumn(text, offset) { + let line = 1; + let lastNewline = -1; + for (let i = 0; i < offset; i++) { + if (text.charCodeAt(i) === 10) { + line++; + lastNewline = i; + } + } + return { line, column: offset - lastNewline }; +} + +/** + * Maps an error position (relative to `node.value`) back to an absolute offset + * range within the full YAML source string. Block scalars strip indentation, so + * a linear shift is wrong there and we search the node's source span instead. + */ +function mapToAbsolutePosition(yamlString, node, errorMessage, relativePosition) { + const range = node.range; + if (!range) return relativePosition; + + switch (node.type) { + case Scalar.BLOCK_FOLDED: + case Scalar.BLOCK_LITERAL: { + const nodeSource = yamlString.substring(range[0], range[2]); + const posInSource = extractLiquidErrorPosition(nodeSource, errorMessage); + return { start: range[0] + posInSource.start, end: range[0] + posInSource.end }; + } + case Scalar.QUOTE_DOUBLE: + case Scalar.QUOTE_SINGLE: { + const valueStart = range[0] + 1; + return { start: valueStart + relativePosition.start, end: valueStart + relativePosition.end }; + } + default: { + return { start: range[0] + relativePosition.start, end: range[0] + relativePosition.end }; + } + } +} + +const LIQUID_OUTPUT_PATTERN = '{{'; +const LIQUID_TAG_PATTERN = '{%'; + +export function validateLiquidTemplate(yamlString, yamlDocument) { + const errors = []; + + visit(yamlDocument, { + Scalar(key, node) { + if (key === 'key') return; + if (!node.range) return; + if (typeof node.value !== 'string') return; + if (!node.value.includes(LIQUID_OUTPUT_PATTERN) && !node.value.includes(LIQUID_TAG_PATTERN)) { + return; + } + + try { + parseTemplateString(node.value); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : 'Invalid Liquid syntax'; + const relativePosition = extractLiquidErrorPosition(node.value, errorMessage); + const absPosition = mapToAbsolutePosition(yamlString, node, errorMessage, relativePosition); + + const startPos = convertOffsetToLineColumn(yamlString, absPosition.start); + const endPos = convertOffsetToLineColumn(yamlString, absPosition.end); + + errors.push({ + message: errorMessage.replace(/, line:\d+, col:\d+/g, ''), + startLine: startPos.line, + startColumn: startPos.column, + endLine: endPos.line, + endColumn: endPos.column, + }); + } + }, + }); + + return errors; +} diff --git a/scripts/validate/load-schema.mjs b/scripts/validate/load-schema.mjs new file mode 100644 index 0000000..11c0bf8 --- /dev/null +++ b/scripts/validate/load-schema.mjs @@ -0,0 +1,109 @@ +/** + * Schema artifact loading — adapted from Kibana's `load_schema.ts`. + * + * Resolves the schema bundle (the directory/URL containing `index.json`) from, + * in order: an explicit `--schema `, then a `--schema-cdn-url` (or + * `WORKFLOWS_SCHEMA_CDN_URL` env) fallback. Each variant's `schema.json` is + * integrity-checked against the `sha256` recorded in `index.json` before use. + * + * Unlike the Kibana CLI, there is no `target//` local lookup: + * this repo has no Kibana build tree, so the source is always an explicit path + * or URL. + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import { createHash } from 'node:crypto'; +import { VARIANTS } from './variants.mjs'; + +const INDEX_FILE = 'index.json'; + +const isUrl = (value) => /^https?:\/\//i.test(value); + +const sha256Hex = (input) => createHash('sha256').update(input, 'utf8').digest('hex'); + +const createFsReader = (baseDir) => ({ + source: baseDir, + readText: async (relativePath) => fs.promises.readFile(path.join(baseDir, relativePath), 'utf8'), +}); + +const createHttpReader = (baseUrl) => { + let end = baseUrl.length; + while (end > 0 && baseUrl.charCodeAt(end - 1) === 47 /* '/' */) { + end--; + } + const normalizedBase = baseUrl.slice(0, end); + return { + source: normalizedBase, + readText: async (relativePath) => { + const url = `${normalizedBase}/${relativePath}`; + const response = await fetch(url); + if (!response.ok) { + throw new Error(`GET ${url} failed: ${response.status} ${response.statusText}`); + } + return response.text(); + }, + }; +}; + +/** + * Resolve the bundle reader in priority order: explicit `--schema` (path or + * URL), then the configured CDN. Throws a descriptive error when nothing + * resolves. + */ +const resolveReader = ({ schema, cdnUrl }) => { + if (schema) { + if (isUrl(schema)) { + return createHttpReader(schema); + } + const absolute = path.resolve(schema); + if (!fs.existsSync(path.join(absolute, INDEX_FILE))) { + throw new Error(`No ${INDEX_FILE} found in --schema directory: ${absolute}`); + } + return createFsReader(absolute); + } + + if (cdnUrl) { + return createHttpReader(cdnUrl); + } + + throw new Error( + `Could not resolve a schema artifact. Provide one with --schema , ` + + `or set --schema-cdn-url / the WORKFLOWS_SCHEMA_CDN_URL env var.` + ); +}; + +const readAndVerifyVariant = async (reader, manifest, variant) => { + const entry = manifest.variants?.[variant]; + if (!entry) { + throw new Error(`Manifest is missing the "${variant}" variant`); + } + + const bytes = await reader.readText(entry.path); + const actualSha = sha256Hex(bytes); + if (actualSha !== entry.sha256) { + throw new Error( + `Integrity check failed for ${variant} (${entry.path}): expected ${entry.sha256}, got ${actualSha}` + ); + } + + return JSON.parse(bytes); +}; + +/** + * Resolve the schema source, read + integrity-verify the manifest and each + * variant, and JSON-parse them. + */ +export const loadSchemaDocuments = async (options = {}) => { + const reader = resolveReader(options); + + const indexText = await reader.readText(INDEX_FILE); + const manifest = JSON.parse(indexText); + + const schemas = {}; + for (const variant of VARIANTS) { + schemas[variant] = await readAndVerifyVariant(reader, manifest, variant); + } + + return { manifest, schemas, source: reader.source }; +}; diff --git a/scripts/validate/report.mjs b/scripts/validate/report.mjs new file mode 100644 index 0000000..3c94653 --- /dev/null +++ b/scripts/validate/report.mjs @@ -0,0 +1,107 @@ +/** + * Human-readable + JSON reporting — ported from Kibana's `report.ts`, with the + * ToolingLog replaced by a small console logger. + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import { isErrorIssue } from './issues.mjs'; + +/** Display labels for issue sources whose token differs from what we print. */ +const SOURCE_LABELS = { + 'liquidjs-expression': 'liquidjs exp', +}; + +const sourceLabel = (source) => SOURCE_LABELS[source] ?? source; + +const countWarnings = (outcomes) => + outcomes.reduce( + (total, outcome) => total + outcome.issues.filter((issue) => !isErrorIssue(issue)).length, + 0 + ); + +const pluralize = (count, noun) => `${count} ${noun}${count === 1 ? '' : 's'}`; + +const relativize = (file) => { + const rel = path.relative(process.cwd(), file); + return rel === '' || rel.startsWith('..') ? file : rel; +}; + +const formatIssue = (issue) => { + if (issue.source === 'liquid') { + const location = issue.line != null ? `${issue.line}:${issue.column ?? 1}` : ''; + return `[liquid] ${location ? `${location} ` : ''}${issue.message}`; + } + const location = issue.path ? `${issue.path}: ` : ''; + return `[${sourceLabel(issue.source)}] ${location}${issue.message}`; +}; + +/** Print one issue, routing warnings to warn and errors to error. */ +const printIssue = (log, issue) => { + const line = ` ${formatIssue(issue)}`; + if (isErrorIssue(issue)) { + log.error(line); + } else { + log.warning(line); + } +}; + +/** Print a single file's human-readable result (streamed as validation progresses). */ +export const printFileResult = (log, outcome) => { + const label = relativize(outcome.file); + if (outcome.ok) { + const warnings = outcome.issues.filter((issue) => !isErrorIssue(issue)); + if (warnings.length === 0) { + log.success(`PASS ${label}`); + return; + } + log.warning(`PASS ${label} (${pluralize(warnings.length, 'warning')})`); + for (const issue of warnings) { + printIssue(log, issue); + } + return; + } + log.error(`FAIL ${label}`); + for (const issue of outcome.issues) { + printIssue(log, issue); + } +}; + +/** Print the closing summary line for a completed run. */ +export const printSummary = (log, outcomes) => { + const passed = outcomes.filter((outcome) => outcome.ok).length; + const failed = outcomes.length - passed; + const warnings = countWarnings(outcomes); + const warningNote = warnings > 0 ? ` (${pluralize(warnings, 'warning')})` : ''; + const summary = `Validated ${outcomes.length} file(s): ${passed} passed, ${failed} failed${warningNote}.`; + if (failed > 0) { + log.error(summary); + } else if (warnings > 0) { + log.warning(summary); + } else { + log.success(summary); + } +}; + +export const buildJsonReport = (source, outcomes) => { + const passed = outcomes.filter((outcome) => outcome.ok).length; + const issues = outcomes.reduce((total, outcome) => total + outcome.issues.length, 0); + return { + summary: { + total: outcomes.length, + passed, + failed: outcomes.length - passed, + warnings: countWarnings(outcomes), + issues, + }, + source, + files: outcomes, + }; +}; + +/** Write the structured JSON report to disk, creating parent dirs as needed. */ +export const writeJsonReport = (outputPath, source, outcomes) => { + const absolute = path.resolve(outputPath); + fs.mkdirSync(path.dirname(absolute), { recursive: true }); + fs.writeFileSync(absolute, `${JSON.stringify(buildJsonReport(source, outcomes), null, 2)}\n`); +}; diff --git a/scripts/validate/validate-file.mjs b/scripts/validate/validate-file.mjs new file mode 100644 index 0000000..0ea7d92 --- /dev/null +++ b/scripts/validate/validate-file.mjs @@ -0,0 +1,309 @@ +/** + * The JSON-Schema validation layer — ported from Kibana's `validate_file.ts`. + * + * Parses the YAML, detects installable templates (by the `template-metadata` + * root key), strips that block for templates, and validates the document/body + * against the selected ajv variant. ajv's raw errors are de-noised and the + * tolerant `anyOf`/`oneOf` template-value wrappers are pruned; positions whose + * value is a LiquidJS expression are downgraded from errors to warnings. + * + * Unlike the Kibana CLI, the `template-metadata` block is detected + stripped + * but not validated against a strict zod schema — the source repo's catalog + * generator (scripts/build-catalog.mjs) already enforces its required fields. + */ + +import { parseYamlToJSONWithoutValidation } from './yaml-parse.mjs'; +import { isVariableValue, isDynamicValue, isLiquidTagValue } from './liquid/regex.mjs'; +import { isErrorIssue } from './issues.mjs'; + +/** Root key that marks a file as an installable library template. */ +export const METADATA_KEY = 'template-metadata'; + +/** + * Even with the native discriminator anchoring each step to its own branch, a + * single step can still yield a handful of intra-branch errors. Dedupe by + * path+message and cap so the report stays readable. + */ +const MAX_SCHEMA_ISSUES = 20; + +const isRecord = (value) => value !== null && typeof value === 'object' && !Array.isArray(value); + +/** ajv `instancePath` (e.g. `/steps/0/type`) -> dotted path (`steps.0.type`). */ +const formatInstancePath = (instancePath) => + instancePath === '' ? '' : instancePath.replace(/^\//, '').replace(/\//g, '.'); + +/** Undo JSON Pointer escaping (`~1` -> `/`, `~0` -> `~`) for one path segment. */ +const decodePointerSegment = (segment) => segment.replace(/~1/g, '/').replace(/~0/g, '~'); + +/** Resolve the value the parsed document holds at an ajv JSON-Pointer `instancePath`. */ +const resolveInstanceValue = (target, instancePath) => { + if (instancePath === '') { + return target; + } + let value = target; + for (const segment of instancePath.split('/').slice(1).map(decodePointerSegment)) { + if (value == null || typeof value !== 'object') { + return undefined; + } + value = value[segment]; + } + return value; +}; + +/** + * A value is a LiquidJS expression when it is a whole-value `{{ }}` / `${{ }}`, + * or contains a `{% %}` tag — the exact predicates the runtime uses to suppress + * schema errors. + */ +const isLiquidjsValue = (value) => + isVariableValue(value) || isDynamicValue(value) || isLiquidTagValue(value); + +/** + * Instance paths whose *own* value is a LiquidJS expression. Scalar-only: a + * structural error anchored on a parent object resolves to an object, not a + * template, so it stays a failing error. + */ +const collectLiquidPaths = (errors, target) => { + const paths = new Set(); + for (const error of errors) { + if (isLiquidjsValue(resolveInstanceValue(target, error.instancePath))) { + paths.add(error.instancePath); + } + } + return paths; +}; + +/** Turn an ajv error into a readable issue, special-casing discriminator/additionalProperties. */ +const toIssue = (error) => { + if (error.keyword === 'discriminator') { + const { error: kind, tag = 'type', tagValue } = error.params ?? {}; + const base = formatInstancePath(error.instancePath); + if (kind === 'mapping') { + // The `type` value did not match any branch of the discriminated union. + return { + source: 'schema', + message: `unknown step type "${tagValue}"`, + path: base === '' ? tag : `${base}.${tag}`, + }; + } + // Missing/non-string discriminator tag. + return { source: 'schema', message: `must have a string "${tag}"`, path: base }; + } + + if (error.keyword === 'additionalProperties') { + const extra = error.params?.additionalProperty; + if (extra) { + return { + source: 'schema', + message: `must NOT have additional property '${extra}'`, + path: formatInstancePath(error.instancePath), + }; + } + } + + return { + source: 'schema', + message: error.message ?? 'Invalid value', + path: formatInstancePath(error.instancePath), + }; +}; + +/** Keywords that pinpoint the real problem (as opposed to branch-selection noise). */ +const SPECIFIC_KEYWORDS = new Set([ + 'additionalProperties', + 'required', + 'const', + 'enum', + 'discriminator', +]); +/** The `anyOf`/`oneOf` wrappers only say "no branch matched" — never actionable alone. */ +const isBranchWrapper = (error) => error.keyword === 'anyOf' || error.keyword === 'oneOf'; +/** Noise from the Liquid-template-value alternative that shadows most fields. */ +const isTemplateValueNoise = (error) => + error.keyword === 'pattern' || (error.keyword === 'type' && error.message === 'must be string'); +/** Wrapper/template noise that is only meaningful when nothing deeper explains the failure. */ +const isNoise = (error) => isBranchWrapper(error) || isTemplateValueNoise(error); + +/** + * Within a single location, a property is often `anyOf: [,