From 466d1860fff965c816f780377d880967c7ea181d Mon Sep 17 00:00:00 2001 From: psy_inf Date: Mon, 21 Sep 2026 09:28:02 +0200 Subject: [PATCH] feat: parameters shared by several graphs, from parameter files A stage argument can name a parameter, `Truncate(width=$text.width)`, whose value comes from a JSON parameter file that several graph files share. A graph file names its parameter files with `params "file.json"` lines, relative to the graph file; later files override earlier ones member by member (JSON merge patch). - dsl::loadGraphProgram(path, overrides) reads a graph file with its parameter files and binds them; parseGraphProgram(text, parameters), bindParameters and loadParameters bind values from the caller. - Dots walk into nested objects, and a parameter can hold any JSON value, so lists and objects can now reach a stage from the DSL. - An unknown name is a located diagnostic with a suggestion; an unreadable parameter file is one at its params line; unused parameters are not reported. Unbound parameters are reported as not set when a graph is built, and renderings show them as `$name`. - validateDslGraph also takes a parsed GraphProgram. - New sample apps/tunedPipeline and EXAMPLE.md section 11. --- CHANGELOG.md | 19 +- EXAMPLE.md | 144 ++++++++- README.md | 98 +++++- TODO.md | 7 +- apps/CMakeLists.txt | 1 + apps/tunedPipeline/CMakeLists.txt | 29 ++ apps/tunedPipeline/graphs/alerts.fg | 5 + apps/tunedPipeline/graphs/report.fg | 6 + apps/tunedPipeline/graphs/report.json | 9 + apps/tunedPipeline/graphs/tuning.json | 6 + apps/tunedPipeline/main.cpp | 201 ++++++++++++ libs/filterGraph/CMakeLists.txt | 1 + .../core/filterGraph/DslFilterGraph.hpp | 59 +++- .../core/filterGraph/GraphLang.hpp | 200 ++++++++++-- .../core/filterGraph/GraphLangHandwritten.hpp | 3 +- .../core/filterGraph/GraphParameters.hpp | 235 ++++++++++++++ tests/FilterGraphTests/ParameterTests.cpp | 294 ++++++++++++++++++ 17 files changed, 1261 insertions(+), 56 deletions(-) create mode 100644 apps/tunedPipeline/CMakeLists.txt create mode 100644 apps/tunedPipeline/graphs/alerts.fg create mode 100644 apps/tunedPipeline/graphs/report.fg create mode 100644 apps/tunedPipeline/graphs/report.json create mode 100644 apps/tunedPipeline/graphs/tuning.json create mode 100644 apps/tunedPipeline/main.cpp create mode 100644 libs/filterGraph/core/filterGraph/GraphParameters.hpp create mode 100644 tests/FilterGraphTests/ParameterTests.cpp diff --git a/CHANGELOG.md b/CHANGELOG.md index c4c0ecf..3d8aa95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -117,6 +117,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 edge it produces (or `end`), fan-out as siblings, and each merge under its (named) group once its inputs have been shown. `AsciiStyle::plain` (the default) draws with ASCII, `AsciiStyle::unicode` with box-drawing characters. +- **Parameters** (`GraphParameters.hpp`) — a stage argument can name a + parameter, `Truncate(width=$text.width)`, whose value comes from a parameter + file (a JSON object) that several graphs share. A graph file names its + parameter files with `params "tuning.json"` lines, relative to the graph + file; later files override earlier ones member by member. + **`dsl::loadGraphProgram(path, overrides)`** reads a graph file with its + parameter files and binds them; **`dsl::parseGraphProgram(text, + parameters)`**, **`dsl::bindParameters`** and **`dsl::loadParameters`** bind + values the caller supplies. Dots walk into nested objects, and a parameter + can hold any JSON value, so nested objects and lists can reach a stage from + the DSL. An unknown name is a located diagnostic with a suggestion; a + parameter file that cannot be read is one at its `params` line; parameters a + graph does not use are not reported. A graph built from a program whose + parameters were never bound reports each as not set, and the renderings show + unbound parameters as `$name`. New sample: `apps/tunedPipeline`. +- `validateDslGraph` also takes a parsed `dsl::GraphProgram`. ### Changed - A `.` on anything but `in` and `out` (e.g. `msg.x`) is now a diagnostic; @@ -153,7 +169,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - The hand-written DSL parser. It moved to `GraphLangHandwritten.hpp` as `dsl::parseGraphProgramHandwritten`, marked `[[deprecated]]`, and will be removed in a future release. It produces the same results as - `dsl::parseGraphProgram`; the tests check the two against each other. + `dsl::parseGraphProgram`, except that it does not support parameters; the + tests check the two against each other. ## [0.2.0] - 2026-09-15 diff --git a/EXAMPLE.md b/EXAMPLE.md index 7fbfa4f..779cf5c 100644 --- a/EXAMPLE.md +++ b/EXAMPLE.md @@ -1,6 +1,6 @@ # filterGraph by example -This walkthrough follows four runnable samples under [`apps/`](apps/), all of +This walkthrough follows five runnable samples under [`apps/`](apps/), all of them small text pipelines: | Sections | Sample | Covers | @@ -9,8 +9,9 @@ them small text pipelines: | 8 | [`statefulPipeline`](apps/statefulPipeline/main.cpp) | stages that **carry state**: `finish()`, the `GraphContext`, a merge with per-instance state | | 9 | [`compositePipeline`](apps/compositePipeline/main.cpp) | **composition**: `JoinFilter`, a graph nested as a stage, a `Void` sink, in-band ticks | | 10 | [`namedPipeline`](apps/namedPipeline/main.cpp) | **names in the wiring**: named merge slots, several named graph inputs (also read directly by a merge), and the checks both make possible | +| 11 | [`tunedPipeline`](apps/tunedPipeline/main.cpp) | **parameters**: graph files that share their tuning values through a parameter file, one-off overrides, and the checks for misspelled names | -Start at the top: sections 8–10 assume the vocabulary of 1–7. +Start at the top: sections 8–11 assume the vocabulary of 1–7. ## Core concept: a chain of stages @@ -968,6 +969,124 @@ in.greeting `-> Greet -> out ``` +## 11. Parameters shared by several graphs + +Two graphs that cut lines to a width should cut them to the *same* width. With +the width written into each graph, keeping them in step is a matter of care. +The fifth sample, [`apps/tunedPipeline/main.cpp`](apps/tunedPipeline/main.cpp), +keeps such values in a parameter file that the graphs share, under +[`apps/tunedPipeline/graphs/`](apps/tunedPipeline/graphs/): + +```text +# alerts.fg: the long lines, cut to the shared width. +params "tuning.json" + +in -> MinLength(minLength=$text.minLength) -> long +long -> Truncate(width=$text.width) -> out +``` + +```text +# report.fg: every line, cut to the same width as the alerts, and its size class. +params "tuning.json" # shared with alerts.fg +params "report.json" # this graph's own parameters + +in -> Truncate(width=$text.width) -> out.text +in -> Classify(classes=$report.classes) -> out.size +``` + +```jsonc +// tuning.json +{ "text": { "minLength": 12, "width": 16 } } + +// report.json +{ "report": { "classes": [ { "upTo": 10, "label": "short" }, + { "upTo": 30, "label": "medium" }, + { "upTo": 1000, "label": "long" } ] } } +``` + +```mermaid +flowchart LR + tuning[/"tuning.json
text.minLength, text.width"/] + report[/"report.json
report.classes"/] + alerts["alerts.fg"] + reportGraph["report.fg"] + tuning --> alerts + tuning --> reportGraph + report --> reportGraph +``` + +- An argument written `$text.width` names a parameter; the dot walks into the + `text` object. `$report.classes` is a list of objects, which an inline + argument cannot express. +- The `params` paths are relative to the graph file. When a graph names + several files, later ones override earlier ones member by member. + +### Loading the graphs + +```cpp +DslFilterGraph alerts(dsl::loadGraphProgram(graphs / "alerts.fg")); +DslFilterGraph report(dsl::loadGraphProgram(graphs / "report.fg")); +``` + +`loadGraphProgram` reads the graph file and its parameter files and binds the +parameters; the stages receive them in their config like any other argument. +Both graphs cut at 16 characters, and changing `text.width` in `tuning.json` +changes both: + +```text +[shared] short one short alert: - +[shared] a line of medium... medium alert: a line of medium... +[shared] and a considerab... long alert: and a considerab... +``` + +### One value changed for a run + +To try a value out without editing the file, pass overrides; they apply on top +of the files, the same way the files apply on top of each other: + +```cpp +DslFilterGraph alerts( + dsl::loadGraphProgram(graphs / "alerts.fg", {{"text", {{"width", 6}}}})); +``` + +```text +[override] and a ... +``` + +### Unbound and bound + +`dsl::parseGraphProgram(text)` reads no files, so a program parsed from text +has its parameters unbound, and the renderings show them by name. Binding them, +here from a parameter file read with `dsl::loadParameters`, puts in the values: + +```cpp +dsl::toAscii(dsl::parseGraphProgram("in -> Truncate(width=$text.width) -> out")); +dsl::toAscii(dsl::parseGraphProgram("in -> Truncate(width=$text.width) -> out", + dsl::loadParameters(graphs / "tuning.json"))); +``` + +```text +in +`-> Truncate(width=$text.width) -> out + +in +`-> Truncate(width=16) -> out +``` + +A graph built from the unbound program would report +`1:22: parameter '$text.width' is not set; ...`. + +### A misspelled parameter + +A name no parameter file defines is a located diagnostic, with a suggestion if +one is close. Parameters a graph does not use are fine: `alerts.fg` never reads +`report.classes`, and a shared file will always hold values that some graph +does not need. + +```text +[check] 1:22: unknown parameter '$text.widht' — did you mean '$text.width'? +``` + ## Build & run these examples ```powershell @@ -980,8 +1099,12 @@ cmake --build --preset windows-msvc-release-user-mode ./out/build/windows-msvc-release-user-mode/apps/statefulPipeline/statefulPipeline ./out/build/windows-msvc-release-user-mode/apps/compositePipeline/compositePipeline ./out/build/windows-msvc-release-user-mode/apps/namedPipeline/namedPipeline +./out/build/windows-msvc-release-user-mode/apps/tunedPipeline/tunedPipeline ``` +`tunedPipeline` finds its graph files in the source tree; pass another +directory as its first argument to load graphs from there. + On Linux/macOS, use a matching preset such as `unixlike-gcc-release` or `unixlike-clang-release`. @@ -1071,3 +1194,20 @@ And of `namedPipeline` (section 10): followed by the Mermaid flowchart and the `[direct]` block shown in section 10. (The `[feed check]` type name is shortened here too.) + +And of `tunedPipeline` (section 11): + +```text +[shared] short one short alert: - +[shared] a line of medium... medium alert: a line of medium... +[shared] and a considerab... long alert: and a considerab... +[override] and a ... + +in +`-> Truncate(width=$text.width) -> out + +in +`-> Truncate(width=16) -> out + +[check] 1:22: unknown parameter '$text.widht' — did you mean '$text.width'? +``` diff --git a/README.md b/README.md index 783d37a..1ff3372 100644 --- a/README.md +++ b/README.md @@ -11,13 +11,14 @@ paths, merge them back together, and drop/short-circuit messages — all without hard-coding the pipeline shape in source code. > **New here? Start with the [example walkthrough (EXAMPLE.md)](EXAMPLE.md)** — -> a step-by-step, diagrammed tour of four runnable samples: +> a step-by-step, diagrammed tour of five runnable samples: > [`apps/textPipeline`](apps/textPipeline/main.cpp) (the core building blocks), > [`apps/statefulPipeline`](apps/statefulPipeline/main.cpp) (stages that carry > state), [`apps/compositePipeline`](apps/compositePipeline/main.cpp) -> (composing graphs out of graphs) and +> (composing graphs out of graphs), > [`apps/namedPipeline`](apps/namedPipeline/main.cpp) (named merge slots and -> graph inputs). +> graph inputs) and [`apps/tunedPipeline`](apps/tunedPipeline/main.cpp) +> (parameter files shared by several graphs). > **A note on the word "filter".** Here "filter" follows the Unix-pipeline and > media-graph (DirectShow / GStreamer / FFmpeg) tradition: a stage that reads a @@ -101,9 +102,14 @@ drops the message, and everything downstream of that edge is skipped. **`registerTypedMergeFilter`** — fan-in stages that declare their slot types, so the edges of a group are checked when the graph is built and the stage receives typed `std::optional`s instead of `std::any`. -- **`validateDslGraph(text)`** — the same checks as construction, - returned as a list of `line:column` diagnostics instead of a thrown - **`GraphError`**. +- **Parameters** — `Stage(width=$text.width)` takes an argument from a + parameter file (JSON) that several graph files share, named in each with + `params "tuning.json"`; **`dsl::loadGraphProgram(path)`** loads a graph file + with its parameters, and **`dsl::bindParameters`** binds values the caller + supplies. +- **`validateDslGraph(text)`** (or a parsed `GraphProgram`) — the + same checks as construction, returned as a list of `line:column` diagnostics + instead of a thrown **`GraphError`**. - **`dsl::parseGraphProgram`** / **`dsl::toMermaid`** / **`dsl::toDot`** / **`dsl::toAscii`** — parse a graph into its node/edge form and render it as a Mermaid flowchart, a Graphviz DOT digraph or a console listing. The parser is @@ -420,6 +426,71 @@ matcher.filter(GraphInputs{}.set("orders", order).set("quotes", quote)); // one - A graph uses either `in` or named inputs: `in` with `GraphInputs`, or `in.` with any other input type, is a build-time diagnostic. +### Parameters shared by several graphs + +Tuning values that several graphs must agree on belong in one place. An +argument can name a **parameter**, `$name`, instead of giving a value, and a +**parameter file** — a JSON object — gives the values. A graph file names its +parameter files with `params` lines: + +```text +# alerts.fg +params "tuning.json" +in -> MinLength(minLength=$text.minLength) -> long +long -> Truncate(width=$text.width) -> out +``` + +```text +# report.fg +params "tuning.json" # shared with alerts.fg +params "report.json" # this graph's own parameters +in -> Truncate(width=$text.width) -> out.text +in -> Classify(classes=$report.classes) -> out.size +``` + +```json +{ "text": { "minLength": 12, "width": 16 } } +``` + +```cpp +#include // includes GraphParameters.hpp + +DslFilterGraph alerts(dsl::loadGraphProgram("graphs/alerts.fg")); +DslFilterGraph report(dsl::loadGraphProgram("graphs/report.fg")); + +// The same graph with one value changed for this run; the files stay as they are. +DslFilterGraph narrow( + dsl::loadGraphProgram("graphs/alerts.fg", {{"text", {{"width", 6}}}})); + +// A graph given as text, with parameters from the caller. +DslFilterGraph fromText( + dsl::parseGraphProgram("in -> Truncate(width=$text.width) -> out", dsl::loadParameters("graphs/tuning.json"))); +``` + +- **Names.** `$name` names a member of the parameter object; dots walk into + nested objects (`$text.width`). A parameter can hold any JSON value, so a + stage that needs a list or an object (`$report.classes`) can get one, which + inline arguments cannot express. Parameter files may contain `//` and + `/* */` comments. +- **Several files.** `params` paths are relative to the graph file. Files are + applied in the order written, each overriding the ones before it member by + member (JSON merge patch: nested objects merge, other values replace), and + `loadGraphProgram`'s second argument is applied last. +- **Checks.** A name that no file defines is a located diagnostic, with a + suggestion if one is close: + `1:22: unknown parameter '$text.widht' — did you mean '$text.width'?`. A file + that cannot be read, is not JSON or is not an object is a diagnostic at its + `params` line. Parameters a graph does not use are *not* reported, since a + file is meant to be shared. +- **Without a file.** `dsl::parseGraphProgram(text)` reads no files: it records + the `params` lines (`GraphProgram::parameterFiles`) and leaves the + parameters unbound, which the renderings show as `width=$text.width`. + Building a graph from it reports each unbound parameter as not set. + `dsl::parseGraphProgram(text, parameters)` and `dsl::bindParameters(program, + parameters)` bind values from the caller instead. +- A stage cannot tell a parameter from a literal argument: both arrive in its + config, and one stage's arguments can mix the two. + ### Checking and visualizing a graph `validateDslGraph` runs every construction check without running a message @@ -432,6 +503,9 @@ for (const auto& d : validateDslGraph(text)) } ``` +It takes a parsed `GraphProgram` as well, e.g. +`validateDslGraph(dsl::loadGraphProgram("graph.fg"))`. + ```text 1:7: unknown stage type 'Uppercas' — did you mean 'Uppercase'? 2:7: could not construct 'MinLength': [json.exception.out_of_range.403] key 'minLength' not found @@ -468,8 +542,9 @@ the UTF-8 code page (`chcp 65001`). ### Current limitations -- Stage arguments are flat `key=value` pairs; nested objects and lists are not - expressible yet. Stages that need them can be configured in JSON. +- Stage arguments are flat `key=value` pairs; nested objects and lists cannot + be written inline. A [parameter](#parameters-shared-by-several-graphs) can + hold them, or the stage can be configured in JSON. - Only the graph input type, the declared named input types and the single `out` type are checked against the C++ side; the types inside `GraphOutputs` are checked when they are read (`get` throws `std::bad_any_cast` on a @@ -679,9 +754,10 @@ Run the bundled examples directly after building: ./out/build/windows-msvc-release-user-mode/apps/statefulPipeline/statefulPipeline ./out/build/windows-msvc-release-user-mode/apps/compositePipeline/compositePipeline ./out/build/windows-msvc-release-user-mode/apps/namedPipeline/namedPipeline +./out/build/windows-msvc-release-user-mode/apps/tunedPipeline/tunedPipeline ``` -[EXAMPLE.md](EXAMPLE.md) walks through all four, with their output. +[EXAMPLE.md](EXAMPLE.md) walks through all five, with their output. ## Roadmap @@ -696,8 +772,8 @@ available in the meantime. The current list: - **Injectable registry with duplicate detection**, instead of a singleton that silently overwrites. - **Documentation** of 0..n outputs, fan-out copies and stateful stages. -- **Lifting the current limitations**: nested stage arguments in the DSL, and - type checking inside `GraphOutputs`. +- **Lifting the current limitations**: nested stage arguments written inline in + the DSL, and type checking inside `GraphOutputs`. ## License diff --git a/TODO.md b/TODO.md index 175c267..6c197b6 100644 --- a/TODO.md +++ b/TODO.md @@ -23,7 +23,7 @@ Examples use the generic stages of the [README](README.md) (`Parse`, | 3 | [Config-aware `registerMergeFilter`](#3-config-aware-registermergefilter) | low | additive | subclass + `FilterRegistrar` creator | | 4 | [Injectable registry, duplicate detection](#4-injectable-registry-duplicate-detection) | low | mostly additive | unique names | | 5 | [Documentation: 0..n outputs, large messages](#5-documentation-0n-outputs-large-messages) | doc only | — | — | -| 6 | [Known limitations to lift](#6-known-limitations-to-lift) | low–medium | additive | JSON config; read `GraphOutputs` carefully | +| 6 | [Known limitations to lift](#6-known-limitations-to-lift) | low–medium | additive | a parameter or JSON config; read `GraphOutputs` carefully | | 7 | [2D box layout for console rendering](#7-2d-box-layout-for-console-rendering) | low | additive | `toAscii` listing; `toDot` into `graph-easy --as=boxart` | Items of the same list that are done, and therefore not repeated here: @@ -172,8 +172,9 @@ The [current limitations](README.md#current-limitations) the README lists, as work items: - **Nested stage arguments in the DSL.** Arguments are flat `key=value` pairs; - nested objects and lists are not expressible, so a stage that needs them has - to be configured in JSON. Lifting this means a value grammar for objects and + nested objects and lists cannot be written inline, so a stage that needs them + takes them from a parameter (`classes=$report.classes`, see the README) or is + configured in JSON. Lifting this means a value grammar for objects and arrays, plus diagnostics for it. - **Type checking inside `GraphOutputs`.** Only the graph's input type and the single `out` type are checked against the C++ template parameters. The types diff --git a/apps/CMakeLists.txt b/apps/CMakeLists.txt index 9ab929c..d272f00 100644 --- a/apps/CMakeLists.txt +++ b/apps/CMakeLists.txt @@ -2,3 +2,4 @@ add_subdirectory(textPipeline) add_subdirectory(statefulPipeline) add_subdirectory(compositePipeline) add_subdirectory(namedPipeline) +add_subdirectory(tunedPipeline) diff --git a/apps/tunedPipeline/CMakeLists.txt b/apps/tunedPipeline/CMakeLists.txt new file mode 100644 index 0000000..9d2da50 --- /dev/null +++ b/apps/tunedPipeline/CMakeLists.txt @@ -0,0 +1,29 @@ +project(tunedPipeline) + +set(CMAKE_CXX_STANDARD 20) +file(GLOB_RECURSE HEADER_FILES CONFIGURE_DEPENDS "*.h*") +file(GLOB_RECURSE CPP_FILES CONFIGURE_DEPENDS "*.cpp") + +add_executable(${PROJECT_NAME} ${HEADER_FILES} ${CPP_FILES} ) + + + +target_link_libraries(${PROJECT_NAME} + PUBLIC + filterGraph::filterGraph +) +target_include_directories(${PROJECT_NAME} + PRIVATE + $ + $ +) +# Where the sample finds its graph and parameter files, unless given one on +# the command line. +target_compile_definitions(${PROJECT_NAME} + PRIVATE + TUNED_PIPELINE_GRAPHS="${CMAKE_CURRENT_SOURCE_DIR}/graphs" +) + +enable_coverage(${PROJECT_NAME}) + +install(TARGETS ${PROJECT_NAME}) \ No newline at end of file diff --git a/apps/tunedPipeline/graphs/alerts.fg b/apps/tunedPipeline/graphs/alerts.fg new file mode 100644 index 0000000..a9713da --- /dev/null +++ b/apps/tunedPipeline/graphs/alerts.fg @@ -0,0 +1,5 @@ +# Alerts: the long lines, cut to the shared width. +params "tuning.json" + +in -> MinLength(minLength=$text.minLength) -> long +long -> Truncate(width=$text.width) -> out diff --git a/apps/tunedPipeline/graphs/report.fg b/apps/tunedPipeline/graphs/report.fg new file mode 100644 index 0000000..b57e102 --- /dev/null +++ b/apps/tunedPipeline/graphs/report.fg @@ -0,0 +1,6 @@ +# Report: every line, cut to the same width as the alerts, and its size class. +params "tuning.json" # shared with alerts.fg +params "report.json" # this graph's own parameters + +in -> Truncate(width=$text.width) -> out.text +in -> Classify(classes=$report.classes) -> out.size diff --git a/apps/tunedPipeline/graphs/report.json b/apps/tunedPipeline/graphs/report.json new file mode 100644 index 0000000..caa4a97 --- /dev/null +++ b/apps/tunedPipeline/graphs/report.json @@ -0,0 +1,9 @@ +{ + "report": { + "classes": [ + { "upTo": 10, "label": "short" }, + { "upTo": 30, "label": "medium" }, + { "upTo": 1000, "label": "long" } + ] + } +} diff --git a/apps/tunedPipeline/graphs/tuning.json b/apps/tunedPipeline/graphs/tuning.json new file mode 100644 index 0000000..0150e32 --- /dev/null +++ b/apps/tunedPipeline/graphs/tuning.json @@ -0,0 +1,6 @@ +{ + "text": { + "minLength": 12, + "width": 16 + } +} diff --git a/apps/tunedPipeline/main.cpp b/apps/tunedPipeline/main.cpp new file mode 100644 index 0000000..672525d --- /dev/null +++ b/apps/tunedPipeline/main.cpp @@ -0,0 +1,201 @@ +// Tuning parameters kept out of the graph text, in files that several graphs +// share: +// - `$name` arguments, `Truncate(width=$text.width)`, and nested values such +// as a list of objects, which inline arguments cannot express +// - graph files (graphs/*.fg) that name their parameter files with +// `params "tuning.json"`, loaded with dsl::loadGraphProgram +// - one parameter changed for a run, without editing any file +// - renderings of a graph before and after its parameters are bound +// - the diagnostic for a misspelled parameter +// +// See EXAMPLE.md, "Parameters shared by several graphs". +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using filterGraph::DslFilterGraph; +using filterGraph::FilterRegistrar; +using filterGraph::GraphOutputs; +using filterGraph::MessageFilter; +using filterGraph::validateDslGraph; + +namespace dsl = filterGraph::dsl; + +namespace { + +// --- Stages ------------------------------------------------------------ + +// Drops lines shorter than `minLength`. +class MinLengthFilter : public MessageFilter +{ +public: + explicit MinLengthFilter(std::size_t minLength) + : mMinLength(minLength) + { + } + + std::optional filter(std::string&& text) override + { + if (text.size() < mMinLength) + { + return std::nullopt; + } + return std::move(text); + } + +private: + std::size_t mMinLength; +}; + +// Cuts lines to `width` characters, marking a cut with "...". +class TruncateFilter : public MessageFilter +{ +public: + explicit TruncateFilter(std::size_t width) + : mWidth(width) + { + } + + std::optional filter(std::string&& text) override + { + if (text.size() > mWidth) + { + text = text.substr(0, mWidth) + "..."; + } + return std::move(text); + } + +private: + std::size_t mWidth; +}; + +// Names a line's size class: the label of the first class it fits into. +class ClassifyFilter : public MessageFilter +{ +public: + struct Class + { + std::size_t upTo; + std::string label; + }; + + explicit ClassifyFilter(std::vector classes) + : mClasses(std::move(classes)) + { + } + + std::optional filter(std::string&& text) override + { + for (const auto& sizeClass : mClasses) + { + if (text.size() <= sizeClass.upTo) + { + return sizeClass.label; + } + } + return std::nullopt; + } + +private: + std::vector mClasses; +}; + +// --- Registration -------------------------------------------------------- + +// A parameter arrives in the stage's config like any other argument. +static FilterRegistrar registerMinLength("MinLength", [](const nlohmann::json& config) { + return std::make_shared(config.at("minLength").get()); +}); + +static FilterRegistrar registerTruncate("Truncate", [](const nlohmann::json& config) { + return std::make_shared(config.at("width").get()); +}); + +// `classes` is a list of objects: only a parameter can give it in the DSL. +static FilterRegistrar registerClassify("Classify", [](const nlohmann::json& config) { + std::vector classes; + for (const auto& sizeClass : config.at("classes")) + { + classes.push_back({sizeClass.at("upTo").get(), sizeClass.at("label").get()}); + } + return std::make_shared(std::move(classes)); +}); + +const std::vector kLines = { + "short one", + "a line of medium length", + "and a considerably longer line than the others", +}; + +} // namespace + +int main(int argc, char** argv) +{ + const std::filesystem::path graphs = argc > 1 ? argv[1] : TUNED_PIPELINE_GRAPHS; + + try + { + // 1) Two graphs, one parameter file. Both read `$text.width` from + // tuning.json, so they cut lines to the same width; report.fg also + // reads its size classes from report.json. + { + DslFilterGraph alerts(dsl::loadGraphProgram(graphs / "alerts.fg")); + DslFilterGraph report(dsl::loadGraphProgram(graphs / "report.fg")); + + for (auto line : kLines) + { + const auto alert = alerts.filter(std::string{line}); + auto row = report.filter(std::move(line)); + std::cout << std::format("[shared] {:<20} {:<7} alert: {}\n", *row->get("text"), + *row->get("size"), alert.value_or("-")); + } + } + + // 2) One run with a different width, e.g. to try a value out, without + // editing tuning.json: overrides apply on top of the files. + { + DslFilterGraph alerts( + dsl::loadGraphProgram(graphs / "alerts.fg", {{"text", {{"width", 6}}}})); + + std::cout << "[override] " << alerts.filter(std::string{kLines.back()}).value_or("-") << '\n'; + } + + // 3) Parsing reads no files: the parameters show by name. Once bound, + // the renderings show their values. + { + std::cout << '\n' + << dsl::toAscii(dsl::parseGraphProgram("in -> Truncate(width=$text.width) -> out")) << '\n' + << dsl::toAscii(dsl::parseGraphProgram("in -> Truncate(width=$text.width) -> out", + dsl::loadParameters(graphs / "tuning.json"))) + << '\n'; + } + + // 4) A misspelled parameter is a located diagnostic, with a suggestion. + { + const auto program = dsl::parseGraphProgram("in -> Truncate(width=$text.widht) -> out", + dsl::loadParameters(graphs / "tuning.json")); + for (const auto& diagnostic : validateDslGraph(program)) + { + std::cout << "[check] " << dsl::formatDiagnostic(diagnostic) << '\n'; + } + } + } + catch (const std::exception& error) + { + std::cerr << error.what() << '\n'; + return 1; + } + return 0; +} diff --git a/libs/filterGraph/CMakeLists.txt b/libs/filterGraph/CMakeLists.txt index e850971..90fa7e6 100644 --- a/libs/filterGraph/CMakeLists.txt +++ b/libs/filterGraph/CMakeLists.txt @@ -20,6 +20,7 @@ target_sources(${PROJECT_NAME} core/filterGraph/GraphLang.hpp core/filterGraph/GraphLangLexy.hpp core/filterGraph/GraphLangHandwritten.hpp + core/filterGraph/GraphParameters.hpp core/filterGraph/DslFilterGraph.hpp core/filterGraph/GraphContext.hpp ) diff --git a/libs/filterGraph/core/filterGraph/DslFilterGraph.hpp b/libs/filterGraph/core/filterGraph/DslFilterGraph.hpp index 693333b..c304a5a 100644 --- a/libs/filterGraph/core/filterGraph/DslFilterGraph.hpp +++ b/libs/filterGraph/core/filterGraph/DslFilterGraph.hpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -555,6 +556,10 @@ class GraphPlan { continue; // already reported by parseGraphProgram; its output stays untyped } + if (!hasAllParameters(program, node, diagnostics)) + { + continue; // its config is incomplete; its output stays untyped + } std::shared_ptr filter; try @@ -785,6 +790,32 @@ class GraphPlan // Registers the graph's inputs: the single `in`, or the named `in.` // of a GraphInputs graph, checked against the declared input types. + // Whether every `$name` argument of `node` has a value. If the program's + // parameters were never bound, reports each missing one; if they were, + // bindParameters has reported them already. + static bool hasAllParameters(const GraphProgram& program, const StageNode& node, + std::vector& diagnostics) + { + bool bound = true; + for (const auto& parameter : node.parameters) + { + if (parameter.bound) + { + continue; + } + bound = false; + if (!program.parametersBound) + { + diagnostics.push_back( + {parameter.loc, + std::format("parameter '${}' is not set; load the graph with dsl::loadGraphProgram, or bind " + "its parameters with dsl::bindParameters", + parameter.name)}); + } + } + return bound; + } + template void bindInputs(const GraphProgram& program, std::type_index inputType, @@ -1075,13 +1106,13 @@ class DslFilterGraph : public MessageFilter dsl::detail::GraphPlan mPlan; }; -// Checks DSL text for use as DslFilterGraph without -// running any messages, and returns every problem found (sorted by location) -// instead of throwing. Stages are instantiated to check their config and types. +// Checks a parsed graph for use as DslFilterGraph +// without running any messages, and returns every problem found (sorted by +// location) instead of throwing. Stages are instantiated to check their config +// and types. template -std::vector validateDslGraph(std::string_view text) +std::vector validateDslGraph(const dsl::GraphProgram& program) { - const dsl::GraphProgram program = dsl::parseGraphProgram(text); std::vector diagnostics = program.diagnostics; [[maybe_unused]] const dsl::detail::GraphPlan plan( program, typeid(InputType), dsl::detail::expectedOutputs(), nullptr, diagnostics); @@ -1092,9 +1123,8 @@ std::vector validateDslGraph(std::string_view text) // The same for a graph with named inputs whose types are declared. template requires std::is_same_v -std::vector validateDslGraph(std::string_view text, const GraphInputTypes& inputs) +std::vector validateDslGraph(const dsl::GraphProgram& program, const GraphInputTypes& inputs) { - const dsl::GraphProgram program = dsl::parseGraphProgram(text); std::vector diagnostics = program.diagnostics; [[maybe_unused]] const dsl::detail::GraphPlan plan( program, typeid(InputType), dsl::detail::expectedOutputs(), &inputs, diagnostics); @@ -1102,4 +1132,19 @@ std::vector validateDslGraph(std::string_view text, const G return diagnostics; } +// Checks DSL text, as validateDslGraph(dsl::parseGraphProgram(text)). +template +std::vector validateDslGraph(std::string_view text) +{ + return validateDslGraph(dsl::parseGraphProgram(text)); +} + +// The same for a graph with named inputs whose types are declared. +template + requires std::is_same_v +std::vector validateDslGraph(std::string_view text, const GraphInputTypes& inputs) +{ + return validateDslGraph(dsl::parseGraphProgram(text), inputs); +} + } // namespace filterGraph diff --git a/libs/filterGraph/core/filterGraph/GraphLang.hpp b/libs/filterGraph/core/filterGraph/GraphLang.hpp index 4f73b08..079d23d 100644 --- a/libs/filterGraph/core/filterGraph/GraphLang.hpp +++ b/libs/filterGraph/core/filterGraph/GraphLang.hpp @@ -47,6 +47,9 @@ // merge's slots, `(raw: a, checked: b) -> Merge -> c`, which matches them by // name instead of by position; a group may read `in` / `in.` directly, // `(in, a)` or `(plots: a, ticks: in.ticks)`; +// - an argument may name a parameter instead of giving a value, +// `Gate(max=$tracker.gate)`, and a line `params "tuning.json"` names a file +// of parameters shared by several graphs (see GraphParameters.hpp); // - `#` starts a comment that runs to the end of the line. // // This header parses (with a lexy-based parser) and structurally validates a @@ -68,19 +71,39 @@ struct TextDiagnostic std::string message; }; +// A stage argument that names a parameter, `key=$name` (or `$section.name`), +// instead of giving a value. bindParameters (GraphParameters.hpp) copies the +// parameter's value into the stage's config. +struct ParameterRef +{ + std::string argument; // the config key it sets + std::string name; // the parameter, without the '$'; dots address nested objects + SourceLoc loc; // of the '$' + bool bound = false; // the value is in the stage's config +}; + // A stage application: consumes one or more input edges (several = fan-in) and // either produces a named edge, or terminates the path at `out`/`end`. struct StageNode { std::string type; // registered filter/merge name - nlohmann::json config; // parsed config args + nlohmann::json config; // parsed config args; an unbound parameter is null std::vector inputs; // source edge names ("in" allowed) std::optional output; // produced edge name ("$outN" for `out`); nullopt => routed to end bool fanIn = false; // inputs came from a group `(a, b) -> Stage` std::vector slotNames; // per input, from `(name: edge, ...)`; empty for a positional group + std::vector parameters; // arguments given as `$name` SourceLoc loc; }; +// A parameter file the program names with `params "file.json"`. Parsing does +// not read it; loadGraphProgram (GraphParameters.hpp) does. +struct ParameterFile +{ + std::string path; // as written: relative to the graph file, or absolute + SourceLoc loc; // of the path +}; + // One graph input the program reads: `in`, or a named `in.`, located at // its first use. struct InputBinding @@ -105,6 +128,8 @@ struct GraphProgram std::vector inputs; // distinct graph inputs, in order of first use std::vector outputs; std::vector deadEnds; // edges routed to `end` + std::vector parameterFiles; // in written order; later files override earlier ones + bool parametersBound = false; // bindParameters ran (and reported unknown names) std::vector diagnostics; bool ok() const @@ -310,6 +335,20 @@ inline Found describeFound(std::string_view rest) return {true, unexpectedCharacter(c)}; } +// A line that starts (after blanks) with `params "`: a parameter file, not a +// statement. A statement cannot have a string literal as its second token, so +// an edge named `params` stays usable. +inline bool startsParameterFile(std::string_view rest) +{ + constexpr std::string_view keyword = "params"; + if (!rest.starts_with(keyword)) + { + return false; + } + const std::size_t next = rest.find_first_not_of(" \t", keyword.size()); + return next != std::string_view::npos && rest[next] == '"'; +} + // -------------------------------- terms ---------------------------------- // A single term in a statement, already classified by the parser. @@ -320,13 +359,15 @@ struct Term Edge, // a plain edge identifier, or `in` Boundary, // `out` / `out.` / `end` Stage, // a stage application (name + config) - Group // a fan-in group of edges + Group, // a fan-in group of edges + ParameterFile // a `params "file.json"` line; `name` is the path }; Kind kind = Kind::Edge; std::string name; // edge/stage/boundary name std::optional key; // for `in.` / `out.` nlohmann::json config; // for Stage + std::vector parameters; // for Stage: arguments given as `$name` std::vector edges; // for Group std::vector> edgeKeys; // for Group: per edge, from `edge.` std::vector edgeLocs; // for Group: per edge @@ -487,8 +528,9 @@ inline void buildStatement(std::vector& terms, StageNode node; node.type = terms[i].name; // A stage without `(...)` gets an empty object, as a JSON stage without "config" does. - node.config = terms[i].config.is_null() ? nlohmann::json::object() : terms[i].config; - node.loc = terms[i].loc; + node.config = terms[i].config.is_null() ? nlohmann::json::object() : terms[i].config; + node.parameters = terms[i].parameters; + node.loc = terms[i].loc; const Term& source = terms[i - 1]; if (source.kind == Term::Kind::Group) @@ -802,34 +844,64 @@ inline bool parseStatementLine(std::string_view lineText, return fail(sc.position(), found.invalid ? found.text : std::format("{} but found {}", expected, found.text)); }; + // A string literal at the current position. + auto parseString = [&](std::string& value) { + auto literal = scanStringLiteral(rest()); + if (!literal) + { + return fail(sc.position(), "unterminated string literal"); + } + // Consume chunk by chunk up to the closing quote; an escaped quote + // or backslash is consumed on its own so it cannot end a chunk. + sc.parse(ld::lit_c<'"'>); + while (sc) + { + sc.parse(ld::until(stringStop)); + if (!sc || sc.position()[-1] == '"') + { + break; + } + if (!sc.branch(ld::lit_c<'"'>)) + { + sc.branch(ld::lit_c<'\\'>); + } + } + value = std::move(literal->value); + return static_cast(sc); + }; + + // `$name` or `$section.name`, without blanks; `name` receives it without the '$'. + auto parseReference = [&](std::string& name) { + sc.parse(ld::lit_c<'$'>); + while (true) + { + if (!sc.peek(ld::ascii::alpha_underscore)) + { + return unexpected(name.empty() ? "expected a parameter name after '$'" + : std::format("expected a name after '${}'", name)); + } + name += captureIdent(); + if (!sc.branch(ld::lit_c<'.'>)) + { + return true; + } + name += '.'; + } + }; + auto parseValue = [&](nlohmann::json& value) { const char* const start = sc.position(); const std::string_view text = rest(); if (sc.peek(ld::lit_c<'"'>)) { - auto literal = scanStringLiteral(text); - if (!literal) + std::string string; + if (!parseString(string)) { - return fail(start, "unterminated string literal"); - } - // Consume chunk by chunk up to the closing quote; an escaped quote - // or backslash is consumed on its own so it cannot end a chunk. - sc.parse(ld::lit_c<'"'>); - while (sc) - { - sc.parse(ld::until(stringStop)); - if (!sc || sc.position()[-1] == '"') - { - break; - } - if (!sc.branch(ld::lit_c<'"'>)) - { - sc.branch(ld::lit_c<'\\'>); - } + return false; } - value = std::move(literal->value); - return static_cast(sc); + value = std::move(string); + return true; } if (const std::size_t length = numberLength(text)) @@ -858,8 +930,9 @@ inline bool parseStatementLine(std::string_view lineText, return unexpected("expected an argument value"); }; - auto parseArgs = [&](nlohmann::json& config) { - const char* const open = sc.position(); + auto parseArgs = [&](Term& stage) { + nlohmann::json& config = stage.config; + const char* const open = sc.position(); sc.parse(ld::lit_c<'('>); config = nlohmann::json::object(); while (true) @@ -888,6 +961,19 @@ inline bool parseStatementLine(std::string_view lineText, return unexpected(std::format("expected '=' after argument '{}'", key)); } skipBlank(); + // The last value given for a key wins, as in a JSON object. + std::erase_if(stage.parameters, [&](const ParameterRef& parameter) { return parameter.argument == key; }); + if (sc.peek(ld::lit_c<'$'>)) + { + ParameterRef parameter{key, {}, locOf(sc.position())}; + if (!parseReference(parameter.name)) + { + return false; + } + config[key] = nullptr; + stage.parameters.push_back(std::move(parameter)); + continue; + } nlohmann::json value; if (!parseValue(value)) { @@ -897,6 +983,26 @@ inline bool parseStatementLine(std::string_view lineText, } }; + // `params "file.json"`, a line of its own. + auto parseParameterFile = [&] { + sc.parse(LEXY_LIT("params")); + skipBlank(); + Term term; + term.kind = Term::Kind::ParameterFile; + term.loc = locOf(sc.position()); + if (!parseString(term.name)) + { + return false; + } + skipBlank(); + if (!atEndOfLine()) + { + return unexpected("expected end of line after the parameter file"); + } + terms.push_back(std::move(term)); + return true; + }; + auto parseGroup = [&] { const char* const open = sc.position(); Term term; @@ -978,7 +1084,7 @@ inline bool parseStatementLine(std::string_view lineText, if (sc.peek(ld::lit_c<'('>)) { term.kind = Term::Kind::Stage; - if (!parseArgs(term.config)) + if (!parseArgs(term)) { return false; } @@ -1014,6 +1120,10 @@ inline bool parseStatementLine(std::string_view lineText, { return true; // blank or comment-only line } + if (startsParameterFile(rest())) + { + return parseParameterFile(); + } if (!parseTerm()) { return false; @@ -1060,7 +1170,14 @@ inline GraphProgram parseGraphProgram(std::string_view text) std::vector terms; if (detail::lexy_impl::parseStatementLine(line, lineNo, terms, program.diagnostics) && !terms.empty()) { - detail::buildStatement(terms, program, outputIndex, program.diagnostics); + if (terms.front().kind == detail::Term::Kind::ParameterFile) + { + program.parameterFiles.push_back({std::move(terms.front().name), terms.front().loc}); + } + else + { + detail::buildStatement(terms, program, outputIndex, program.diagnostics); + } } if (newline == std::string_view::npos) @@ -1098,6 +1215,28 @@ inline std::string displayName(const std::unordered_mapsecond : edge; } +// The parameter an argument names, while it is not bound: its value is not +// known yet, so renderings show `$name` instead. +inline const ParameterRef* unboundParameter(const StageNode& stage, const std::string& argument) +{ + auto it = std::ranges::find_if(stage.parameters, [&](const ParameterRef& parameter) { + return !parameter.bound && parameter.argument == argument; + }); + return it != stage.parameters.end() ? &*it : nullptr; +} + +// An argument's value as written in a rendering: `$name` for an unbound +// parameter, strings unquoted or quoted as asked, anything else as JSON. +inline std::string argumentText(const StageNode& stage, const std::string& argument, const nlohmann::json& value, + bool quoteStrings) +{ + if (const ParameterRef* parameter = unboundParameter(stage, argument)) + { + return '$' + parameter->name; + } + return value.is_string() && !quoteStrings ? value.get() : value.dump(); +} + // A stage's config as "key=value" lines, strings unquoted, for diagram labels. inline std::vector configLines(const StageNode& stage) { @@ -1106,8 +1245,7 @@ inline std::vector configLines(const StageNode& stage) { for (const auto& item : stage.config.items()) { - const std::string value = item.value().is_string() ? item.value().get() : item.value().dump(); - lines.push_back(item.key() + '=' + value); + lines.push_back(item.key() + '=' + argumentText(stage, item.key(), item.value(), false)); } } return lines; @@ -1329,7 +1467,7 @@ inline std::string stageCall(const StageNode& stage) std::string args; for (const auto& item : stage.config.items()) { - args += (args.empty() ? "" : ", ") + item.key() + '=' + item.value().dump(); + args += (args.empty() ? "" : ", ") + item.key() + '=' + argumentText(stage, item.key(), item.value(), true); } text += '(' + args + ')'; } diff --git a/libs/filterGraph/core/filterGraph/GraphLangHandwritten.hpp b/libs/filterGraph/core/filterGraph/GraphLangHandwritten.hpp index 5927986..08f1586 100644 --- a/libs/filterGraph/core/filterGraph/GraphLangHandwritten.hpp +++ b/libs/filterGraph/core/filterGraph/GraphLangHandwritten.hpp @@ -490,7 +490,8 @@ class Parser } // namespace detail::handwritten // Parses a graph program with the deprecated hand-written parser. Produces the -// same result as parseGraphProgram. +// same result as parseGraphProgram, except that it does not know parameters +// (`$name` arguments and `params` lines), which came after its deprecation. [[deprecated("the hand-written DSL parser is deprecated; use dsl::parseGraphProgram, which gives the same result")]] inline GraphProgram parseGraphProgramHandwritten(std::string_view text) { diff --git a/libs/filterGraph/core/filterGraph/GraphParameters.hpp b/libs/filterGraph/core/filterGraph/GraphParameters.hpp new file mode 100644 index 0000000..35b4bda --- /dev/null +++ b/libs/filterGraph/core/filterGraph/GraphParameters.hpp @@ -0,0 +1,235 @@ +#pragma once + +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// Parameters: tuning values kept out of the graph text, so that several graphs +// can share them. A stage argument names a parameter instead of giving a value, +// and a parameter file (a JSON object) gives the values: +// +// # alerts.fg +// params "tuning.json" +// in -> MinLength(minLength=$text.minLength) -> long +// long -> Truncate(width=$text.width) -> out +// +// // tuning.json +// { "text": { "minLength": 5, "width": 12 } } +// +// - `$name` names a top-level member; dots walk into nested objects +// (`$text.width`). A parameter's value may be any JSON value, objects and +// arrays included. Parameter files may contain comments. +// - A graph may name several parameter files; later ones override earlier ones +// member by member (JSON merge patch: nested objects merge, other values +// replace, null removes). Paths are relative to the graph file. +// - A name that no parameter file defines is a diagnostic; parameters that the +// graph does not use are not, since a file is meant to be shared. +// +// parseGraphProgram(text) only records the references and files; it reads +// nothing. loadGraphProgram(path) reads a graph file with its parameter files, +// and parseGraphProgram(text, parameters) / bindParameters take the values +// from the caller. A graph built from a program whose parameters were never +// bound reports each of them as not set. +namespace filterGraph::dsl { + +namespace detail { + +// The parameter `name` ("a" or "a.b.c") in `parameters`, or nullptr. +inline const nlohmann::json* findParameter(const nlohmann::json& parameters, std::string_view name) +{ + const nlohmann::json* value = ¶meters; + while (true) + { + const std::size_t dot = name.find('.'); + const std::string_view segment = name.substr(0, dot); + if (!value->is_object()) + { + return nullptr; + } + auto it = value->find(std::string(segment)); + if (it == value->end()) + { + return nullptr; + } + value = &*it; + if (dot == std::string_view::npos) + { + return value; + } + name.remove_prefix(dot + 1); + } +} + +// Every name a reference could use: each member, nested ones joined by dots. +inline void parameterNames(const nlohmann::json& parameters, const std::string& prefix, std::vector& names) +{ + if (!parameters.is_object()) + { + return; + } + for (const auto& item : parameters.items()) + { + const std::string name = prefix.empty() ? item.key() : prefix + '.' + item.key(); + names.push_back(name); + parameterNames(item.value(), name, names); + } +} + +// Reads a parameter file: a JSON object. On failure returns std::nullopt and +// sets `error`. +inline std::optional readParameterFile(const std::filesystem::path& file, std::string& error) +{ + std::ifstream stream(file, std::ios::binary); + if (!stream) + { + error = std::format("cannot open parameter file '{}'", file.string()); + return std::nullopt; + } + nlohmann::json parameters = nlohmann::json::parse(stream, nullptr, false, true); + if (parameters.is_discarded()) + { + // Parse again for the message; it names the line and column in the file. + stream.clear(); + stream.seekg(0); + try + { + [[maybe_unused]] const auto ignored = nlohmann::json::parse(stream, nullptr, true, true); + } + catch (const nlohmann::json::parse_error& parseError) + { + error = std::format("parameter file '{}' is not valid JSON: {}", file.string(), parseError.what()); + return std::nullopt; + } + error = std::format("parameter file '{}' is not valid JSON", file.string()); + return std::nullopt; + } + if (!parameters.is_object()) + { + error = std::format("parameter file '{}' must hold a JSON object, not {}", file.string(), parameters.type_name()); + return std::nullopt; + } + return parameters; +} + +// bindParameters; `reportUnknown` is false when a parameter file could not be +// read, which is reported already and would explain every unknown name. +inline void bindParameters(GraphProgram& program, const nlohmann::json& parameters, bool reportUnknown) +{ + if (!parameters.is_object()) + { + throw std::invalid_argument(std::format("parameters must be a JSON object, not {}", parameters.type_name())); + } + + std::vector known; + parameterNames(parameters, "", known); + + for (auto& stage : program.stages) + { + for (auto& parameter : stage.parameters) + { + if (const nlohmann::json* value = findParameter(parameters, parameter.name)) + { + stage.config[parameter.argument] = *value; + parameter.bound = true; + continue; + } + if (!reportUnknown) + { + continue; + } + std::string message = std::format("unknown parameter '${}'", parameter.name); + if (auto suggestion = filterGraph::detail::closestName(parameter.name, known)) + { + message += std::format(" — did you mean '${}'?", *suggestion); + } + program.diagnostics.push_back({parameter.loc, std::move(message)}); + } + } + program.parametersBound = true; + sortDiagnostics(program.diagnostics); +} + +} // namespace detail + +// Binds every `$name` argument of `program` to its value in `parameters` (a +// JSON object; dots in a name walk into nested objects), which is copied into +// the stage's config. A name `parameters` does not define is a diagnostic, +// with a suggestion if one is close. Call it once per program. Throws +// std::invalid_argument if `parameters` is not a JSON object. +inline void bindParameters(GraphProgram& program, const nlohmann::json& parameters) +{ + detail::bindParameters(program, parameters, true); +} + +// Parses `text` and binds its parameters from `parameters`. A `params` line in +// the text is not read; use loadGraphProgram for a graph file. +inline GraphProgram parseGraphProgram(std::string_view text, const nlohmann::json& parameters) +{ + GraphProgram program = parseGraphProgram(text); + bindParameters(program, parameters); + return program; +} + +// Reads a parameter file, a JSON object, e.g. to bind a graph given as text. +// Throws std::runtime_error if it cannot be read or is not a JSON object. +inline nlohmann::json loadParameters(const std::filesystem::path& file) +{ + std::string error; + auto parameters = detail::readParameterFile(file, error); + if (!parameters) + { + throw std::runtime_error(error); + } + return std::move(*parameters); +} + +// Loads a graph file (e.g. `alerts.fg`): parses it, reads the parameter files +// it names with `params "file.json"` (relative to the graph file's directory), +// in order, each overriding the ones before it, then applies `overrides` the +// same way, and binds the result. A parameter file that cannot be read, or is +// not a JSON object, is a diagnostic at its `params` line. Throws +// std::runtime_error if the graph file itself cannot be read. +inline GraphProgram loadGraphProgram(const std::filesystem::path& file, + const nlohmann::json& overrides = nlohmann::json::object()) +{ + std::ifstream stream(file, std::ios::binary); + if (!stream) + { + throw std::runtime_error(std::format("cannot open graph file '{}'", file.string())); + } + const std::string text{std::istreambuf_iterator(stream), std::istreambuf_iterator()}; + + GraphProgram program = parseGraphProgram(text); + nlohmann::json parameters = nlohmann::json::object(); + bool allRead = true; + for (const auto& parameterFile : program.parameterFiles) + { + std::string error; + if (auto values = detail::readParameterFile(file.parent_path() / parameterFile.path, error)) + { + parameters.merge_patch(*values); + } + else + { + program.diagnostics.push_back({parameterFile.loc, std::move(error)}); + allRead = false; + } + } + parameters.merge_patch(overrides); + detail::bindParameters(program, parameters, allRead); + return program; +} + +} // namespace filterGraph::dsl diff --git a/tests/FilterGraphTests/ParameterTests.cpp b/tests/FilterGraphTests/ParameterTests.cpp new file mode 100644 index 0000000..7169d8b --- /dev/null +++ b/tests/FilterGraphTests/ParameterTests.cpp @@ -0,0 +1,294 @@ +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace filterGraph; + +namespace { + +// Adds `amount` to its input. +class AddFilter : public MessageFilter +{ +public: + explicit AddFilter(int amount) + : mAmount(amount) + { + } + + std::optional filter(int&& value) override + { + return value + mAmount; + } + +private: + int mAmount; +}; + +// Drops values outside [min, max]; the bounds come as one nested object, +// `range={"min": ..., "max": ...}`. +class ClampFilter : public MessageFilter +{ +public: + ClampFilter(int min, int max) + : mMin(min) + , mMax(max) + { + } + + std::optional filter(int&& value) override + { + return value < mMin || value > mMax ? std::nullopt : std::optional(value); + } + +private: + int mMin; + int mMax; +}; + +static FilterRegistrar registerAdd("Add", [](const nlohmann::json& config) { + return std::make_shared(config.at("amount").get()); +}); + +static FilterRegistrar registerClamp("Clamp", [](const nlohmann::json& config) { + const auto& range = config.at("range"); + return std::make_shared(range.at("min").get(), range.at("max").get()); +}); + +std::vector messages(const std::vector& diagnostics) +{ + std::vector texts; + for (const auto& diagnostic : diagnostics) + { + texts.push_back(dsl::formatDiagnostic(diagnostic)); + } + return texts; +} + +// A fresh directory for graph and parameter files, removed at the end of a test. +class TempDir +{ +public: + TempDir() + { + // Test cases may run in parallel processes, so the name is random. + std::random_device device; + mPath = std::filesystem::temp_directory_path() / + ("filterGraphParameterTests-" + std::to_string(device()) + "-" + std::to_string(device())); + std::filesystem::remove_all(mPath); + std::filesystem::create_directories(mPath); + } + + ~TempDir() + { + std::error_code ignored; + std::filesystem::remove_all(mPath, ignored); + } + + TempDir(const TempDir&) = delete; + TempDir& operator=(const TempDir&) = delete; + + std::filesystem::path write(const std::string& name, std::string_view content) const + { + const auto path = mPath / name; + std::filesystem::create_directories(path.parent_path()); + std::ofstream stream(path, std::ios::binary); + stream << content; + return path; + } + +private: + std::filesystem::path mPath; +}; + +} // namespace + +TEST_CASE("A $name argument takes its value from the parameters", "[Parameters]") +{ + DslFilterGraph graph(dsl::parseGraphProgram("in -> Add(amount=$step) -> out", + nlohmann::json{{"step", 5}})); + REQUIRE(graph.filter(1) == 6); +} + +TEST_CASE("Dots in a parameter name walk into nested objects", "[Parameters]") +{ + const nlohmann::json parameters = {{"tracker", {{"step", 2}, {"window", {{"min", 0}, {"max", 10}}}}}}; + DslFilterGraph graph(dsl::parseGraphProgram( + "in -> Add(amount=$tracker.step) -> stepped -> Clamp(range=$tracker.window) -> out", parameters)); + + REQUIRE(graph.filter(3) == 5); + REQUIRE(graph.filter(9) == std::nullopt); // 11 is out of range +} + +TEST_CASE("Parameters mix with literal arguments, and the last value for a key wins", "[Parameters]") +{ + const nlohmann::json parameters = {{"step", 5}}; + DslFilterGraph literalLast(dsl::parseGraphProgram("in -> Add(amount=$step, amount=1) -> out", parameters)); + DslFilterGraph parameterLast(dsl::parseGraphProgram("in -> Add(amount=1, amount=$step) -> out", parameters)); + + REQUIRE(literalLast.filter(0) == 1); + REQUIRE(parameterLast.filter(0) == 5); +} + +TEST_CASE("Parameters the graph does not use are not an error", "[Parameters]") +{ + const nlohmann::json parameters = {{"step", 1}, {"otherGraph", {{"threshold", 0.5}}}}; + REQUIRE(validateDslGraph(dsl::parseGraphProgram("in -> Add(amount=$step) -> out", parameters)).empty()); +} + +TEST_CASE("An unknown parameter is located and a close name is suggested", "[Parameters]") +{ + const auto program = + dsl::parseGraphProgram("in -> Add(amount=$tracker.stp) -> out", nlohmann::json{{"tracker", {{"step", 1}}}}); + + REQUIRE(messages(program.diagnostics) == + std::vector{"1:18: unknown parameter '$tracker.stp' — did you mean '$tracker.step'?"}); + // Reported once, not again as "not set" when the graph is built. + REQUIRE(messages(validateDslGraph(program)) == messages(program.diagnostics)); +} + +TEST_CASE("A graph whose parameters were never bound reports each as not set", "[Parameters]") +{ + const auto diagnostics = validateDslGraph("in -> Add(amount=$step) -> added -> Add(amount=$step) -> out"); + + REQUIRE(diagnostics.size() == 2); + REQUIRE(dsl::formatDiagnostic(diagnostics[0]).starts_with("1:18: parameter '$step' is not set;")); + REQUIRE(dsl::formatDiagnostic(diagnostics[1]).starts_with("1:48: parameter '$step' is not set;")); + REQUIRE_THROWS_AS((DslFilterGraph("in -> Add(amount=$step) -> out")), GraphError); +} + +TEST_CASE("Renderings show unbound parameters by name and bound ones by value", "[Parameters]") +{ + constexpr std::string_view text = "in -> Add(amount=$step) -> out"; + + const auto unbound = dsl::parseGraphProgram(text); + REQUIRE(dsl::toAscii(unbound) == "in\n`-> Add(amount=$step) -> out\n"); + REQUIRE(dsl::toMermaid(unbound).find("amount=$step") != std::string::npos); + REQUIRE(dsl::toDot(unbound).find("amount=$step") != std::string::npos); + + const auto bound = dsl::parseGraphProgram(text, nlohmann::json{{"step", 3}}); + REQUIRE(dsl::toAscii(bound) == "in\n`-> Add(amount=3) -> out\n"); +} + +TEST_CASE("Malformed parameter references are syntax errors", "[Parameters]") +{ + REQUIRE(messages(dsl::parseGraphProgram("in -> Add(amount=$) -> out").diagnostics) == + std::vector{"1:19: expected a parameter name after '$' but found ')'"}); + REQUIRE(messages(dsl::parseGraphProgram("in -> Add(amount=$a.) -> out").diagnostics) == + std::vector{"1:21: expected a name after '$a.' but found ')'"}); + REQUIRE(messages(dsl::parseGraphProgram("in -> Add(amount=$a b) -> out").diagnostics) == + std::vector{"1:22: expected '=' after argument 'b' but found ')'"}); +} + +TEST_CASE("A params line names a parameter file and is not a statement", "[Parameters]") +{ + const auto program = dsl::parseGraphProgram("params \"common.json\"\n" + " params \"graph.json\" # overrides common.json\n" + "in -> Add(amount=1) -> out\n"); + REQUIRE(program.ok()); + REQUIRE(program.stages.size() == 1); + REQUIRE(program.parameterFiles.size() == 2); + REQUIRE(program.parameterFiles[0].path == "common.json"); + REQUIRE(program.parameterFiles[1].path == "graph.json"); + REQUIRE(program.parameterFiles[1].loc.line == 2); + REQUIRE(program.parameterFiles[1].loc.column == 10); +} + +TEST_CASE("An edge named params still works", "[Parameters]") +{ + DslFilterGraph graph("in -> Add(amount=1) -> params\n" + "params -> Add(amount=2) -> out\n"); + REQUIRE(graph.filter(0) == 3); +} + +TEST_CASE("Malformed params lines are syntax errors", "[Parameters]") +{ + REQUIRE(messages(dsl::parseGraphProgram("params \"tuning.json").diagnostics) == + std::vector{"1:8: unterminated string literal"}); + REQUIRE(messages(dsl::parseGraphProgram("params \"tuning.json\" -> Add -> out").diagnostics) == + std::vector{"1:22: expected end of line after the parameter file but found '->'"}); +} + +TEST_CASE("Two graph files share one parameter file", "[Parameters]") +{ + TempDir dir; + dir.write("tuning.json", R"({ "step": 10, "window": { "min": 0, "max": 100 } })"); + const auto add = dir.write("add.fg", "params \"tuning.json\"\nin -> Add(amount=$step) -> out\n"); + const auto clamp = dir.write("clamp.fg", "params \"tuning.json\"\nin -> Clamp(range=$window) -> out\n"); + + DslFilterGraph addGraph(dsl::loadGraphProgram(add)); + DslFilterGraph clampGraph(dsl::loadGraphProgram(clamp)); + + REQUIRE(addGraph.filter(1) == 11); + REQUIRE(clampGraph.filter(50) == 50); + REQUIRE(clampGraph.filter(101) == std::nullopt); +} + +TEST_CASE("Parameter files are relative to the graph file and override in order", "[Parameters]") +{ + TempDir dir; + dir.write("shared/common.json", R"({ "step": 1, "window": { "min": 0, "max": 10 } })"); + dir.write("graphs/local.json", R"({ "window": { "max": 20 } })"); + const auto graph = dir.write("graphs/graph.fg", "params \"../shared/common.json\"\n" + "params \"local.json\"\n" + "in -> Add(amount=$step) -> added\n" + "added -> Clamp(range=$window) -> out\n"); + + DslFilterGraph merged(dsl::loadGraphProgram(graph)); + REQUIRE(merged.filter(14) == 15); // max 20 from local.json, min 0 kept from common.json + + DslFilterGraph overridden(dsl::loadGraphProgram(graph, nlohmann::json{{"step", 100}})); + REQUIRE(overridden.filter(0) == std::nullopt); // 100 > 20 +} + +TEST_CASE("A parameter file that cannot be read is reported at its params line", "[Parameters]") +{ + TempDir dir; + dir.write("broken.json", R"({ "step": )"); + dir.write("list.json", "[1, 2]"); + const auto graph = dir.write("graph.fg", "params \"missing.json\"\n" + "params \"broken.json\"\n" + "params \"list.json\"\n" + "in -> Add(amount=$step) -> out\n"); + + const auto program = dsl::loadGraphProgram(graph); + const auto texts = messages(program.diagnostics); + REQUIRE(texts.size() == 3); // no follow-on "unknown parameter '$step'" + REQUIRE(texts[0].starts_with("1:8: cannot open parameter file '")); + REQUIRE(texts[1].starts_with("2:8: parameter file '")); + REQUIRE(texts[1].find("is not valid JSON: ") != std::string::npos); + REQUIRE(texts[2].starts_with("3:8: parameter file '")); + REQUIRE(texts[2].ends_with("must hold a JSON object, not array")); + REQUIRE(messages(validateDslGraph(program)) == texts); +} + +TEST_CASE("A missing graph file throws", "[Parameters]") +{ + TempDir dir; + REQUIRE_THROWS_AS(dsl::loadGraphProgram(dir.write("tuning.json", "{}").parent_path() / "missing.fg"), + std::runtime_error); +} + +TEST_CASE("loadParameters binds a graph given as text", "[Parameters]") +{ + TempDir dir; + const auto tuning = dir.write("tuning.json", R"({ "step": 7 })"); + + DslFilterGraph graph(dsl::parseGraphProgram("in -> Add(amount=$step) -> out", dsl::loadParameters(tuning))); + REQUIRE(graph.filter(0) == 7); + REQUIRE_THROWS_AS(dsl::loadParameters(dir.write("list.json", "[]")), std::runtime_error); + dsl::GraphProgram program; + REQUIRE_THROWS_AS(dsl::bindParameters(program, nlohmann::json::array()), std::invalid_argument); +}