From 585d509c56e0dcf270d88827f245274190f11253 Mon Sep 17 00:00:00 2001 From: Steven McCanne Date: Wed, 4 Nov 2026 14:00:18 -0800 Subject: [PATCH] change pipe-level none handling and empty subquery This commit changes none handling at the output of pipe operators to drop singlton nones. If a none is created the query writer knows what they are doing, and if they didn't want to drop it, they should use ??. We also changed the behavior of pipe expression subqueries to return none instead of null for empty result. SQL subqueries still return null. --- book/src/super-sql/expressions/match.md | 3 +- book/src/super-sql/expressions/subqueries.md | 20 +++ .../super-sql/expressions/type-assertions.md | 3 +- book/src/super-sql/functions/generics/ok.md | 5 +- book/src/super-sql/operators/aggregate.md | 4 + book/src/super-sql/operators/debug.md | 2 + book/src/super-sql/operators/unnest.md | 3 +- book/src/super-sql/operators/values.md | 22 ++++ compiler/dag/expr.go | 6 +- compiler/dag/op.go | 2 +- compiler/rungen/vexpr.go | 4 +- compiler/rungen/vop.go | 2 +- compiler/semantic/analyzer.go | 3 - compiler/semantic/checker.go | 60 ++++++--- compiler/semantic/compare.go | 2 +- compiler/semantic/dagen.go | 1 + compiler/semantic/expr.go | 5 +- compiler/semantic/op.go | 24 +--- compiler/semantic/sem/expr.go | 6 +- compiler/semantic/ztests/agg-missing.yaml | 5 +- compiler/semantic/ztests/none-output.yaml | 50 +++---- compiler/semantic/ztests/none.yaml | 3 +- runtime/op/aggregate/aggregate.go | 4 + runtime/op/cachedsubquery.go | 10 +- runtime/op/debug.go | 3 +- runtime/op/nones.go | 123 ++++++++++++++++++ runtime/op/output.go | 103 ++------------- runtime/op/subquery.go | 18 ++- runtime/op/unnest.go | 12 +- runtime/op/values.go | 18 ++- runtime/ztests/expr/function/under.yaml | 1 - runtime/ztests/expr/index.yaml | 5 - runtime/ztests/expr/subquery-empty.yaml | 21 +++ runtime/ztests/op/aggregate/none-key.yaml | 28 ++++ runtime/ztests/op/debug-none.yaml | 11 ++ runtime/ztests/op/unnest.yaml | 27 +++- runtime/ztests/op/values/drop-none.yaml | 55 ++++++++ spq_test.go | 10 +- sup/ztests/decorate-none.yaml | 1 - vector/recordbuilder.go | 19 ++- 40 files changed, 504 insertions(+), 200 deletions(-) create mode 100644 runtime/op/nones.go create mode 100644 runtime/ztests/expr/subquery-empty.yaml create mode 100644 runtime/ztests/op/aggregate/none-key.yaml create mode 100644 runtime/ztests/op/debug-none.yaml create mode 100644 runtime/ztests/op/values/drop-none.yaml diff --git a/book/src/super-sql/expressions/match.md b/book/src/super-sql/expressions/match.md index 5688f660d0..664dfbabe0 100644 --- a/book/src/super-sql/expressions/match.md +++ b/book/src/super-sql/expressions/match.md @@ -19,7 +19,8 @@ match expression. A value matches a type by the same rules as a its underlying value and [named types](../types/named.md) are distinct. If no case matches, the result is `` or, without an -`else` clause, `none`, which disappears from a record, array, or map +`else` clause, `none`, which disappears from a record, array, or map, +is dropped from the output of the [values](../operators/values.md) operator, and is typically handled with the `??` operator. An [error](../types/error.md) or `missing` value is passed through unless a case has its type. diff --git a/book/src/super-sql/expressions/subqueries.md b/book/src/super-sql/expressions/subqueries.md index f30e1550a5..268e8ca6b2 100644 --- a/book/src/super-sql/expressions/subqueries.md +++ b/book/src/super-sql/expressions/subqueries.md @@ -19,6 +19,9 @@ results in the value `{s:"HELLO, WORLD"}`. Except for subqueries appearing as the right-hand side of an [in](containment.md) operator, the result of a subquery must be a single value. When multiple values are generated, an [error](../types/error.md) is produced. +When no values are generated, the result is `none`, which disappears from a +record, array, or map and is dropped from the output of the +[values](../operators/values.md) operator, or, in a SQL expression, `null`. For the [in](containment.md) operator, any subquery on the right-hand side is always treated as an [array subquery](#array-subqueries), thus @@ -278,3 +281,20 @@ values walk(this, &addOne) [2,3,4] [{x:[2,"foo"]},{y:3}] ``` + +--- + +_A subquery that produces no values is none_ + +```mdtest-spq +# spq +values {s:(unnest a | where this > 1)}, (unnest a | where this > 1) ?? 0 +# input +{a:[1,2]} +{a:[1]} +# expected output +{s:2} +2 +{} +0 +``` diff --git a/book/src/super-sql/expressions/type-assertions.md b/book/src/super-sql/expressions/type-assertions.md index a0456b4140..f4b1b2fc01 100644 --- a/book/src/super-sql/expressions/type-assertions.md +++ b/book/src/super-sql/expressions/type-assertions.md @@ -7,7 +7,8 @@ A type assertion has the form where `` is any [expression](intro.md) and `` is any [type](../types/intro.md). Its value is the value of `` when that value has type `` and is otherwise `none`, -which disappears from a record, array, or map and is typically handled with +which disappears from a record, array, or map, is dropped from the output of the +[values](../operators/values.md) operator, and is typically handled with the `??` operator, e.g., ``` v.(string) ?? "default" diff --git a/book/src/super-sql/functions/generics/ok.md b/book/src/super-sql/functions/generics/ok.md index 6dff8bf0cf..6ca4093c8f 100644 --- a/book/src/super-sql/functions/generics/ok.md +++ b/book/src/super-sql/functions/generics/ok.md @@ -16,8 +16,9 @@ unmodified. This function is useful when errors are expected and are to be explicitly managed with `none` operators. The `none` disappears from a record expression, an -array, or a map and otherwise must be handled with `??` since a query may not -output a `none`. +array, or a map and is dropped from the output of the +[values](../../operators/values.md) operator, so `??` is needed only to replace +it with another value. ## Examples diff --git a/book/src/super-sql/operators/aggregate.md b/book/src/super-sql/operators/aggregate.md index 6ef616fc09..aa80bfe586 100644 --- a/book/src/super-sql/operators/aggregate.md +++ b/book/src/super-sql/operators/aggregate.md @@ -55,6 +55,10 @@ The output values are records formed from the [field assignments](intro.md#field-assignment) first from the grouping expressions then from the aggregate functions in left-to-right order. +As in a record expression, a grouping key whose value +is a plain `none` (e.g., `ok(x)` when `x` is missing) disappears from its +output record, so the values whose key is `none` form a group whose record +has no field for that key. When the result of `aggregate` is a single value (e.g., a single aggregate function without grouping expressions or a single grouping expression without aggregates) diff --git a/book/src/super-sql/operators/debug.md b/book/src/super-sql/operators/debug.md index 33a2b3a0b5..f8bde42ad9 100644 --- a/book/src/super-sql/operators/debug.md +++ b/book/src/super-sql/operators/debug.md @@ -16,6 +16,8 @@ each value in the input, evaluates the optional and transmits each result to a _debugging output_. If `` is omitted, then each input value is passed unmodified to the debugging output. +As for the [values](values.md) operator, a result that is a plain `none` +(e.g., from [ok](../functions/generics/ok.md) or `?.`) is dropped. An optional filter may be applied to the values sent to the debugging output, which is specified with Boolean-valued expression ``. diff --git a/book/src/super-sql/operators/unnest.md b/book/src/super-sql/operators/unnest.md index aa06bf872b..acdbf43de5 100644 --- a/book/src/super-sql/operators/unnest.md +++ b/book/src/super-sql/operators/unnest.md @@ -41,7 +41,8 @@ produces ``` The value of `` must be an array (or a set), whose elements form the -derived sequence. +derived sequence. A `null` or a plain `none` value (e.g., from +[ok](../functions/generics/ok.md) or `?.`) produces no values. To carry other values along with each unnested element, give the unnested value a name with `:=` or list the values in a `with` clause. Then each diff --git a/book/src/super-sql/operators/values.md b/book/src/super-sql/operators/values.md index 87d0b073ad..316e99ad6e 100644 --- a/book/src/super-sql/operators/values.md +++ b/book/src/super-sql/operators/values.md @@ -19,6 +19,13 @@ in left-to-right order. Each `` may be any valid The input order convolved with left-to-right evaluation order is preserved at the output. +A result that is a plain `none`, e.g., from [ok](../functions/generics/ok.md), +`?.`, a [type assertion](../expressions/type-assertions.md) that fails, or a +[match](../expressions/match.md) with no matching case and no `else`, +is dropped, so a `none` is never passed to a downstream operator. +To output another value in its place, use the `??` operator. +An option's `none` (e.g., `none::int64`) is a value and is output as is. + The `values` operator name is optional since it can be used as a [shortcut](intro.md#shortcuts). When used as a shortcut, only one expression may be present. @@ -127,3 +134,18 @@ values [a,b],[b,a] | collect(this) # expected output [[1,2],[2,1]] ``` + +--- + +_A plain none is dropped unless it's replaced with ??_ +```mdtest-spq +# spq +values ok(a), ok(a) ?? 0 +# input +{a:1} +{b:2} +# expected output +1 +1 +0 +``` diff --git a/compiler/dag/expr.go b/compiler/dag/expr.go index 07e36f9613..ef701942a3 100644 --- a/compiler/dag/expr.go +++ b/compiler/dag/expr.go @@ -146,7 +146,11 @@ type ( SubqueryExpr struct { Kind string `json:"kind" unpack:""` Correlated bool `json:"correlated"` - Body Seq `json:"body"` + // SQL is true for a subquery in a SQL expression, whose value is + // null when the subquery produces no values. Otherwise, the value + // is none. + SQL bool `json:"sql,omitempty"` + Body Seq `json:"body"` } ThisExpr struct { Kind string `json:"kind" unpack:""` diff --git a/compiler/dag/op.go b/compiler/dag/op.go index 9bf11784ab..8154a99d1f 100644 --- a/compiler/dag/op.go +++ b/compiler/dag/op.go @@ -370,7 +370,7 @@ func NewFilterOp(e Expr) *FilterOp { } func NewValuesOp(exprs ...Expr) *ValuesOp { - return &ValuesOp{"ValuesOp", exprs} + return &ValuesOp{Kind: "ValuesOp", Exprs: exprs} } func CopySeq(seq Seq) Seq { diff --git a/compiler/rungen/vexpr.go b/compiler/rungen/vexpr.go index 6cd542c73f..f3776fc9ed 100644 --- a/compiler/rungen/vexpr.go +++ b/compiler/rungen/vexpr.go @@ -313,11 +313,11 @@ func (b *Builder) compileVamSubquery(query *dag.SubqueryExpr) (expr.Evaluator, e return nil, err } body := b.combineVam(exits) - return op.NewCachedSubquery(b.sctx(), body), nil + return op.NewCachedSubquery(b.sctx(), body, query.SQL), nil } var create func() *op.Subquery create = func() *op.Subquery { - subquery := op.NewSubquery(b.rctx.Context, b.sctx(), func() *op.Subquery { + subquery := op.NewSubquery(b.rctx.Context, b.sctx(), query.SQL, func() *op.Subquery { b.mu.Lock() defer b.mu.Unlock() return create() diff --git a/compiler/rungen/vop.go b/compiler/rungen/vop.go index 9549110bfb..c5b9b7c5b0 100644 --- a/compiler/rungen/vop.go +++ b/compiler/rungen/vop.go @@ -317,7 +317,7 @@ func (b *Builder) compileVamLeaf(o dag.Op, parent vio.Puller) (vio.Puller, error return vio.NewPuller(vector.NewNull(1)), nil case *dag.OutputOp: if parent != nil { - parent = op.NewOutput(b.sctx(), parent) + parent = op.NewOutput(parent) } b.channels[o.Name] = append(b.channels[o.Name], parent) return parent, nil diff --git a/compiler/semantic/analyzer.go b/compiler/semantic/analyzer.go index 5ebd59a897..43a51250bb 100644 --- a/compiler/semantic/analyzer.go +++ b/compiler/semantic/analyzer.go @@ -79,9 +79,6 @@ type translator struct { sctx *super.Context types *sup.Analyzer defs *super.Context - // subqueries is the depth of expression subqueries being translated. - // Their results may be none (see checker.output). - subqueries int } func newTranslator(ctx context.Context, r reporter, env *exec.Environment) *translator { diff --git a/compiler/semantic/checker.go b/compiler/semantic/checker.go index bb5c4d8f31..7d82149d2f 100644 --- a/compiler/semantic/checker.go +++ b/compiler/semantic/checker.go @@ -162,7 +162,7 @@ func (c *checker) op(typ super.Type, op sem.Op) super.Type { case *sem.UnnestOp: return c.seq(c.unnest(op.Expr, c.expr(typ, op.Expr), op.Field), op.Body) case *sem.ValuesOp: - return c.fuse(c.exprs(typ, op.Exprs)) + return c.values(c.exprs(typ, op.Exprs)) default: panic(op) } @@ -213,6 +213,10 @@ func (c *checker) unnest(loc ast.Node, typ super.Type, field string) super.Type if !ok { c.keepErrs(errs) } + if typ == nil { + // The value is always a plain none, so there are no values. + return c.unknown + } return typ } @@ -220,6 +224,10 @@ func (c *checker) unnestCheck(loc ast.Node, typ super.Type) (super.Type, bool) { if isUnknown(typ) { return c.unknown, true } + if typ == super.TypeNone { + // A plain none produces no values. + return nil, true + } switch typ := under(typ).(type) { case *super.TypeError: c.error(loc, errors.New("unnested record cannot be an error")) @@ -230,7 +238,9 @@ func (c *checker) unnestCheck(loc ast.Node, typ super.Type) (super.Type, bool) { for _, t := range typ.Types { typ, tok := c.unnestCheck(loc, t) if tok { - types = append(types, typ) + if typ != nil { + types = append(types, typ) + } ok = true } } @@ -258,6 +268,9 @@ func (c *checker) unnestField(loc ast.Node, typ super.Type, field string) (super return c.unknown, true } elemType, ok := c.unnestCheck(loc, rec.Fields[k].Type) + if elemType == nil { + return nil, ok + } fields := slices.Clone(rec.Fields) fields[k].Type = elemType out, err := c.t.sctx.LookupTypeRecord(fields) @@ -1734,23 +1747,40 @@ func (c *checker) result(loc ast.Node, typ super.Type) { } } -// output reports an error at loc if typ, the type of an output value, may -// be a plain none. (An option none or a none of a named type is a value -// and is allowed.) -func (c *checker) output(loc ast.Node, typ super.Type) { - typ = defuse(typ) - if typ != nil && !hasUnknown(typ) && hasPlainNone(typ) { - c.error(loc, errNone) +// values returns the type of the output of a values operator whose +// expressions have types. The operator drops plain nones, so none is removed +// from each type and an expression that's always a plain none produces no +// values. (An option none or a none of a named type is a value and is kept.) +func (c *checker) values(types []super.Type) super.Type { + var out []super.Type + for _, typ := range types { + if typ, ok := c.withoutPlainNone(typ); ok { + out = append(out, typ) + } } + return c.fuse(out) } -// hasPlainNone reports whether typ is none or a union with none as one of -// its types. Unlike hasNone, a named type whose type is none isn't none. -func hasPlainNone(typ super.Type) bool { - if u, ok := typ.(*super.TypeUnion); ok { - return slices.ContainsFunc(u.Types, hasPlainNone) +// withoutPlainNone returns typ with plain none removed from it (from its +// members if it's a union). It returns false if nothing is left. +func (c *checker) withoutPlainNone(typ super.Type) (super.Type, bool) { + if typ == nil || hasUnknown(typ) { + return typ, true + } + fusion, isFusion := typ.(*super.TypeFusion) + out, ok := c.withoutTypes(defuse(typ), func(typ super.Type) bool { + return typ == super.TypeNone + }) + if !ok { + return nil, false + } + if isFusion && out != fusion.Type { + if _, isUnion := out.(*super.TypeUnion); isUnion { + // The values are still those of a fusion. + return c.t.sctx.MustLookupTypeFusion(out), true + } } - return typ == super.TypeNone + return out, true } // notMissing reports an error at loc if typ may be missing. diff --git a/compiler/semantic/compare.go b/compiler/semantic/compare.go index 20a8be258f..f96b76ec73 100644 --- a/compiler/semantic/compare.go +++ b/compiler/semantic/compare.go @@ -281,7 +281,7 @@ func eqExpr(aexpr, bexpr sem.Expr) bool { return ok && eqExpr(a.Expr, b.Expr) && eqExpr(a.From, b.From) && eqExpr(a.To, b.To) case *sem.SubqueryExpr: b, ok := bexpr.(*sem.SubqueryExpr) - return ok && a.Correlated == b.Correlated && a.Array == b.Array && eqSeq(a.Body, b.Body) + return ok && a.Correlated == b.Correlated && a.Array == b.Array && a.SQL == b.SQL && eqSeq(a.Body, b.Body) case *sem.ThisExpr: b, ok := bexpr.(*sem.ThisExpr) if !ok || len(a.Chain) != len(b.Chain) { diff --git a/compiler/semantic/dagen.go b/compiler/semantic/dagen.go index b3a2b25812..b78e2c2b3c 100644 --- a/compiler/semantic/dagen.go +++ b/compiler/semantic/dagen.go @@ -587,6 +587,7 @@ func (d *dagen) subquery(e *sem.SubqueryExpr) *dag.SubqueryExpr { subquery := &dag.SubqueryExpr{ Kind: "SubqueryExpr", Correlated: e.Correlated, + SQL: e.SQL, Body: d.seq(e.Body), } if e.Array { diff --git a/compiler/semantic/expr.go b/compiler/semantic/expr.go index 07830e38fa..bfc00d1b30 100644 --- a/compiler/semantic/expr.go +++ b/compiler/semantic/expr.go @@ -730,6 +730,7 @@ func (t *translator) maybeSubquery(n ast.Node, name string, inType super.Type) ( return &sem.SubqueryExpr{ Node: n, Correlated: isCorrelated(seq), + SQL: t.scope.sql != nil, Body: seq, }, typ } @@ -884,6 +885,7 @@ func (t *translator) maybeSubqueryCall(call *ast.CallExpr, name string, inType s Node: call, Array: false, Correlated: isCorrelated(userOp), + SQL: t.scope.sql != nil, Body: userOp, }, typ } @@ -1327,14 +1329,13 @@ func (t *translator) arraySubquery(elems []sem.ArrayElem) *sem.SubqueryExpr { func (t *translator) subqueryExpr(astExpr ast.Expr, array bool, body ast.Seq, inType super.Type) (*sem.SubqueryExpr, super.Type) { // We pass inType in whether or not it's correlated since an uncorrelated // subquery will just ignore it. - t.subqueries++ seq, outType := t.seq(body, inType) - t.subqueries-- correlated := isCorrelated(seq) e := &sem.SubqueryExpr{ Node: astExpr, Array: array, Correlated: correlated, + SQL: t.scope.sql != nil, Body: seq, } // Add a nullscan only for uncorrelated queries that don't have a source. diff --git a/compiler/semantic/op.go b/compiler/semantic/op.go index 68389c94c9..e30c14740b 100644 --- a/compiler/semantic/op.go +++ b/compiler/semantic/op.go @@ -1117,9 +1117,8 @@ func (t *translator) semOp(o ast.Op, seq sem.Seq, inType super.Type) (sem.Seq, s exprs, types := t.exprs(o.Exprs, inType) for k, typ := range types { t.checker.result(o.Exprs[k], typ) - t.output(o.Exprs[k], typ) } - return append(seq, sem.NewValues(o, exprs...)), t.checker.fuse(types) + return append(seq, sem.NewValues(o, exprs...)), t.checker.values(types) case *ast.Script: t.error(o, errors.New("a script of multiple statements cannot be used here")) return append(seq, badOp), badType @@ -1655,10 +1654,10 @@ func (t *translator) exprOp(e ast.Expr, seq sem.Seq, inType super.Type) (sem.Seq if t.isBool(out) { return append(seq, &sem.FilterOp{Node: e, Expr: out}), inType } - // As with the values operator, the value must not be missing or none. + // As with the values operator, the value must not be missing and a + // plain none is dropped. t.checker.result(e, typ) - t.output(e, typ) - return append(seq, sem.NewValues(e, out)), typ + return append(seq, sem.NewValues(e, out)), t.checker.values([]super.Type{typ}) } func (t *translator) isBool(e sem.Expr) bool { @@ -1962,21 +1961,6 @@ func (t *translator) maybeEval(e sem.Expr) (super.Value, bool) { return newEvaluator(t, t.resolver.funcs).maybeEval(t.sctx, e) } -// output reports an error at loc if typ, the type of a value output by the -// values operator, may be a plain none (e.g., from ok() or ?.). A plain none -// disappears from a record, array, or map but otherwise must be handled with -// ?? before it leaves the operator that produced it, i.e., a plain none is -// never passed to a downstream operator, even one that would handle it. -// (A plain none that reaches the query output at runtime is an error.) The -// result of an expression subquery may be none since it's an expression -// value. -func (t *translator) output(loc ast.Node, typ super.Type) { - if t.subqueries > 0 { - return - } - t.checker.output(loc, typ) -} - // unnestExpr translates the expression of an unnest operator and returns it, // its type, and, if the unnest is named ("unnest name:=expr") or has a with // clause ("with assignments"), the name of the field to unnest. In that diff --git a/compiler/semantic/sem/expr.go b/compiler/semantic/sem/expr.go index 5632ef5379..ce66492f05 100644 --- a/compiler/semantic/sem/expr.go +++ b/compiler/semantic/sem/expr.go @@ -133,7 +133,10 @@ type ( ast.Node Correlated bool Array bool - Body Seq + // SQL is true for a subquery in a SQL expression (see + // dag.SubqueryExpr). + SQL bool + Body Seq } ThisExpr struct { ast.Node @@ -462,6 +465,7 @@ func CopyExpr(e Expr) Expr { Node: e.Node, Correlated: e.Correlated, Array: e.Array, + SQL: e.SQL, Body: CopySeq(e.Body), } case *ThisExpr: diff --git a/compiler/semantic/ztests/agg-missing.yaml b/compiler/semantic/ztests/agg-missing.yaml index ac8e642f35..9dae20952b 100644 --- a/compiler/semantic/ztests/agg-missing.yaml +++ b/compiler/semantic/ztests/agg-missing.yaml @@ -86,7 +86,8 @@ error: | --- -# With ok(), the absent values form the none group. +# With ok(), the absent values form the none group, whose record has no y +# since a plain none disappears from a record. spq: aggregate sum(x) by y.ok() | sort y input: | @@ -96,7 +97,7 @@ input: | output: | {y:"a",sum:4} - {y:none,sum:2} + {sum:2} --- diff --git a/compiler/semantic/ztests/none-output.yaml b/compiler/semantic/ztests/none-output.yaml index 4c3b0ac9ae..47d7d6c018 100644 --- a/compiler/semantic/ztests/none-output.yaml +++ b/compiler/semantic/ztests/none-output.yaml @@ -22,12 +22,20 @@ output: | --- -# Statically, a value that may be a plain none (e.g., from ?. or ok()) can't -# be output and must be handled with ??. +# A plain none (e.g., from ?. or ok()) output by the values operator is +# dropped, so it isn't passed downstream, unless it's handled +# with ??. script: | super -o in.bsup in.sup - ! super -s -c 'from in.bsup | values org?.id' - ! super -s -c 'from in.bsup | values y.ok()' + echo === + super -s -c 'from in.bsup | values org?.id' + echo === + super -s -c 'from in.bsup | values y.ok()' + echo === + super -s -c 'from in.bsup | values y.ok() | values typeof(this)' + echo === + super -s -c 'from in.bsup | y.ok()' + echo === super -s -c 'from in.bsup | values org?.id ?? 0, {y:y.ok()}' inputs: @@ -39,22 +47,23 @@ inputs: outputs: - name: stdout data: | + === + 1 + === + 2 + === + + === + 2 + === 0 {} 1 {y:2} - - name: stderr - data: | - none not handled (consider ?? or ?.) at line 1, column 23: - from in.bsup | values org?.id - ~~~~~~~ - none not handled (consider ?? or ?.) at line 1, column 23: - from in.bsup | values y.ok() - ~~~~~~ --- -# At runtime, a plain none output is an error. +# At runtime, a plain none output is dropped. spq: values org?.id flags: -dynamic @@ -64,18 +73,15 @@ input: | {org?:{id:1}} output: | - error("none not handled (consider ?? or ?.)") 1 --- -# A none literal can't be output. -spq: values none +# A none literal is a plain none, so it's dropped. +spq: values none, 1 -error: | - none not handled (consider ?? or ?.) at line 1, column 8: - values none - ~~~~ +output: | + 1 --- @@ -111,8 +117,8 @@ output: | --- -# At runtime, a union whose type has a none member keeps its type for values -# that aren't plain nones. (Statically, the type may be none so it's an error.) +# A union whose type has a none member keeps its type for values that +# aren't plain nones. spq: values this flags: -dynamic diff --git a/compiler/semantic/ztests/none.yaml b/compiler/semantic/ztests/none.yaml index d6d20e081a..ea9c184b3b 100644 --- a/compiler/semantic/ztests/none.yaml +++ b/compiler/semantic/ztests/none.yaml @@ -144,7 +144,7 @@ output: | --- -# The runtime ?. passes the none from ok() through, which is an error when +# The runtime ?. passes the none from ok() through, which is dropped when # it's output. spq: values r.ok()?.f @@ -156,7 +156,6 @@ input: | output: | 1 - error("none not handled (consider ?? or ?.)") --- diff --git a/runtime/op/aggregate/aggregate.go b/runtime/op/aggregate/aggregate.go index b85009f8c7..a9a1ceb913 100644 --- a/runtime/op/aggregate/aggregate.go +++ b/runtime/op/aggregate/aggregate.go @@ -32,6 +32,10 @@ func New(parent vio.Puller, sctx *super.Context, aggNames []field.Path, aggExprs if err != nil { return nil, err } + // A plain none group key (e.g., from ok() or ?.) disappears from an + // output record. It's kept in a partial so the downstream aggregate can + // group by it. + builder.DropNones = !partialsOut return &Aggregate{ parent: parent, sctx: sctx, diff --git a/runtime/op/cachedsubquery.go b/runtime/op/cachedsubquery.go index cfbcd1b67f..f47ce59ece 100644 --- a/runtime/op/cachedsubquery.go +++ b/runtime/op/cachedsubquery.go @@ -11,11 +11,15 @@ import ( type cachedSubquery struct { sctx *super.Context body vio.Puller + sql bool cached vector.Any } -func NewCachedSubquery(sctx *super.Context, body vio.Puller) expr.Evaluator { - return &cachedSubquery{sctx: sctx, body: body} +// NewCachedSubquery returns an uncorrelated subquery, which is evaluated once. +// If it produces no values, its value is null if sql is true (as in SQL) and +// otherwise none. +func NewCachedSubquery(sctx *super.Context, body vio.Puller, sql bool) expr.Evaluator { + return &cachedSubquery{sctx: sctx, body: body, sql: sql} } func (c *cachedSubquery) Eval(this vector.Any) vector.Any { @@ -40,7 +44,7 @@ func (c *cachedSubquery) exec() vector.Any { vecs = append(vecs, vec) } if len(vecs) == 0 { - return vector.NewNull(1) + return emptyResult(c.sql) } if len(vecs) == 1 && vecs[0].Len() == 1 { return vecs[0] diff --git a/runtime/op/debug.go b/runtime/op/debug.go index b5741089f5..c159b7e4a9 100644 --- a/runtime/op/debug.go +++ b/runtime/op/debug.go @@ -56,7 +56,8 @@ func (d *Debug) Pull(done bool) (vector.Any, error) { if d.filter != nil { filtered, _ = applyMask(val, d.filter.Eval(filtered)) } - if e := d.expr.Eval(filtered); e.Len() != 0 { + // A plain none is dropped, as for the values operator. + if e := dropPlainNones(d.expr.Eval(filtered)); e != nil && e.Len() != 0 { select { case local <- e: case <-d.ctx.Done(): diff --git a/runtime/op/nones.go b/runtime/op/nones.go new file mode 100644 index 0000000000..0de6eb5220 --- /dev/null +++ b/runtime/op/nones.go @@ -0,0 +1,123 @@ +package op + +import ( + "github.com/superdb/super" + "github.com/superdb/super/vector" +) + +// dropPlainNones returns vec without its plain none values or nil if all of +// its values are plain nones. A plain none comes from an expression like +// ok() or ?. that handles a none. It disappears from a record, array, or +// map and, as a value output by the values operator or the query, is +// dropped. A none of a named type and an option's none are values and +// are kept. +func dropPlainNones(vec vector.Any) vector.Any { + nones := plainNoneSlots(vec) + if len(nones) == 0 { + return vec + } + n := vec.Len() + if len(nones) == int(n) { + return nil + } + rest := make([]uint32, 0, int(n)-len(nones)) + var k int + for slot := range n { + if k < len(nones) && nones[k] == slot { + k++ + continue + } + rest = append(rest, slot) + } + return vector.Pick(vec, rest) +} + +// plainNoneSlots returns the slots of vec, in order, whose values are plain +// nones. +func plainNoneSlots(vec vector.Any) []uint32 { + switch v := vec.(type) { + case *vector.Dynamic: + // The values of a dynamic may themselves be dynamics. + marks := make([][]bool, len(v.Values)) + var found bool + for k, val := range v.Values { + if val == nil { + continue + } + if slots := plainNoneSlots(val); len(slots) != 0 { + marks[k] = markSlots(val.Len(), slots) + found = true + } + } + if !found { + return nil + } + var nones []uint32 + counts := make([]uint32, len(v.Values)) + for slot, tag := range v.Tags { + if m := marks[tag]; m != nil && m[counts[tag]] { + nones = append(nones, uint32(slot)) + } + counts[tag]++ + } + return nones + case *vector.Union: + tags := v.Tags() + values := v.Values() + var nones []uint32 + for slot, tag := range tags { + if isNoneVec(values[tag]) { + nones = append(nones, uint32(slot)) + } + } + return nones + case *vector.Fusion: + // A fusion value is a plain none if its subtype is none. + var nones []uint32 + for slot, typ := range v.Subtypes.Types() { + if typ == super.TypeNone { + nones = append(nones, uint32(slot)) + } + } + return nones + case *vector.View: + slots := plainNoneSlots(v.Any) + if len(slots) == 0 { + return nil + } + marks := markSlots(v.Any.Len(), slots) + var nones []uint32 + for slot, index := range v.Index { + if marks[index] { + nones = append(nones, uint32(slot)) + } + } + return nones + } + if isNoneVec(vec) { + return allSlots(vec.Len()) + } + return nil +} + +func allSlots(n uint32) []uint32 { + slots := make([]uint32, n) + for k := range slots { + slots[k] = uint32(k) + } + return slots +} + +func markSlots(n uint32, slots []uint32) []bool { + marks := make([]bool, n) + for _, slot := range slots { + marks[slot] = true + } + return marks +} + +// isNoneVec reports whether vec is a plain none. (A none of a named type +// is a value.) +func isNoneVec(vec vector.Any) bool { + return vec != nil && vec.Type() == super.TypeNone +} diff --git a/runtime/op/output.go b/runtime/op/output.go index 45094835c6..647c95269c 100644 --- a/runtime/op/output.go +++ b/runtime/op/output.go @@ -1,109 +1,30 @@ package op import ( - "slices" - - "github.com/superdb/super" "github.com/superdb/super/vector" "github.com/superdb/super/vector/vio" ) -// Output is the puller for an output of a query. It turns any plain none -// values into errors. A plain none comes from an operator like ok() or ?. -// that handles a none so it can disappear from a record, array, or map, but -// a query must not output one (it should be handled with ??). A none of a -// named type is a value and is output as is. +// Output is the puller for an output of a query. It drops any plain none +// values, which come from an expression like ok() or ?. that handles a none +// (see dropPlainNones). A none of a named type is a value and is output +// as is. type Output struct { - sctx *super.Context parent vio.Puller } -func NewOutput(sctx *super.Context, parent vio.Puller) *Output { - return &Output{sctx: sctx, parent: parent} +func NewOutput(parent vio.Puller) *Output { + return &Output{parent: parent} } func (o *Output) Pull(done bool) (vector.Any, error) { - vec, err := o.parent.Pull(done) - if vec == nil || err != nil { - return vec, err - } - return o.noneToError(vec), nil -} - -const noneErr = "none not handled (consider ?? or ?.)" - -func (o *Output) noneToError(vec vector.Any) vector.Any { - switch v := vec.(type) { - case *vector.None: - return vector.NewStringError(o.sctx, noneErr, v.Len()) - case *vector.Dynamic: - // The values of a dynamic may themselves be dynamics. - var vals []vector.Any - for k, val := range v.Values { - if val == nil { - continue - } - if out := o.noneToError(val); out != val { - if vals == nil { - vals = slices.Clone(v.Values) - } - vals[k] = out - } - } - if vals == nil { - return vec - } - return vector.NewDynamic(v.Tags, vals) - case *vector.Union: - // Only the slots that are plain nones become errors. The rest - // keep the union type. - if !slices.ContainsFunc(v.Values(), isNoneVec) { - return vec - } - values := v.Values() - var index, rest []uint32 - for i, tag := range v.Tags() { - if isNoneVec(values[tag]) { - index = append(index, uint32(i)) - } else { - rest = append(rest, uint32(i)) - } + for { + vec, err := o.parent.Pull(done) + if vec == nil || err != nil { + return vec, err } - return o.slotsToError(vec, index, rest) - case *vector.Fusion: - // A fusion value is a plain none if its subtype is none. - var index, rest []uint32 - for i, typ := range v.Subtypes.Types() { - if typ == super.TypeNone { - index = append(index, uint32(i)) - } else { - rest = append(rest, uint32(i)) - } + if vec = dropPlainNones(vec); vec != nil { + return vec, nil } - return o.slotsToError(vec, index, rest) - case *vector.View: - if isNoneVec(v.Any) { - return vector.NewStringError(o.sctx, noneErr, v.Len()) - } - } - return vec -} - -// slotsToError returns vec with the slots in index replaced by errors and the -// slots in rest left as is. -func (o *Output) slotsToError(vec vector.Any, index, rest []uint32) vector.Any { - if len(index) == 0 { - return vec } - errs := vector.NewStringError(o.sctx, noneErr, uint32(len(index))) - if len(rest) == 0 { - return errs - } - return vector.Combine(vector.Pick(vec, rest), index, errs) -} - -// isNoneVec reports whether vec is a plain none. (A none of a named type -// is a value.) -func isNoneVec(vec vector.Any) bool { - return vec != nil && vec.Type() == super.TypeNone } diff --git a/runtime/op/subquery.go b/runtime/op/subquery.go index a04fe279c3..2027bb1863 100644 --- a/runtime/op/subquery.go +++ b/runtime/op/subquery.go @@ -14,10 +14,12 @@ import ( // the expression. When Eval is called, it repeatedly constructs a single-slot // vector from the current slot of the this vector, sends it to the embedded // query, pulls from the query until EOS, appends the value to the result -// vector, and then advances to the next slot. +// vector, and then advances to the next slot. If the query produces no +// values, the result is null if sql is true (as in SQL) and otherwise none. type Subquery struct { ctx context.Context sctx *super.Context + sql bool create func() *Subquery body vio.Puller @@ -28,10 +30,11 @@ type Subquery struct { tos int } -func NewSubquery(ctx context.Context, sctx *super.Context, create func() *Subquery) *Subquery { +func NewSubquery(ctx context.Context, sctx *super.Context, sql bool, create func() *Subquery) *Subquery { return &Subquery{ ctx: ctx, sctx: sctx, + sql: sql, create: create, ch: make(chan vector.Any, 1), tos: -2, @@ -83,7 +86,7 @@ func (s *Subquery) bodyPull() vector.Any { return vector.NewStringError(s.sctx, err.Error(), 1) } if vec == nil { - return vector.NewNull(1) + return emptyResult(s.sql) } vec2, err := s.body.Pull(false) if err != nil { @@ -113,3 +116,12 @@ func (s *Subquery) Pull(done bool) (vector.Any, error) { return nil, s.ctx.Err() } } + +// emptyResult returns the value of a subquery that produces no values: null +// in a SQL expression and otherwise none. +func emptyResult(sql bool) vector.Any { + if sql { + return vector.NewNull(1) + } + return vector.NewNone(1) +} diff --git a/runtime/op/unnest.go b/runtime/op/unnest.go index 568f230f87..5dfed36610 100644 --- a/runtime/op/unnest.go +++ b/runtime/op/unnest.go @@ -50,12 +50,15 @@ func (u *Unnest) Pull(done bool) (vector.Any, error) { // flatten unnests vec, which is a record whose field named u.field is // unnested if record is true and otherwise an array or set, and returns the // unnested values and the offsets of the values of each slot of vec. It -// returns nil values for nulls and nil offsets for errors, which map -// one-to-one to the slots of vec. +// returns nil values for nulls and plain nones, which produce no values, and +// nil offsets for errors, which map one-to-one to the slots of vec. func (u *Unnest) flatten(vec vector.Any, record bool) (vector.Any, []uint32) { if dynamic, ok := vec.(*vector.Dynamic); ok { return u.flattenDynamic(dynamic, record) } + if isNoneVec(vec) { + return nil, nil + } vec = vector.Under(vec) switch vec.Kind() { case vector.KindNull: @@ -114,8 +117,9 @@ func (u *Unnest) flattenRecord(vec *vector.Record) (vector.Any, []uint32) { k, ok := vec.Typ.IndexOfField(u.field) if !ok { // The field is absent since its value is a plain none, which - // disappears from a record, so report it as for an unnamed unnest. - return vector.NewWrappedError(u.sctx, "unnest: encountered non-array value", vector.NewNone(vec.Len())), nil + // disappears from a record, so as for an unnamed unnest, there's + // nothing to unnest. + return nil, nil } if union, ok := vec.Fields[k].(*vector.Union); ok { dynamic := union.Dynamic() diff --git a/runtime/op/values.go b/runtime/op/values.go index c86c7509d7..468b5fcc3c 100644 --- a/runtime/op/values.go +++ b/runtime/op/values.go @@ -15,6 +15,8 @@ type Values struct { var _ vio.Puller = (*Values)(nil) +// NewValues returns a Values that outputs the values of exprs. Plain none +// values are dropped (see dropPlainNones). func NewValues(sctx *super.Context, parent vio.Puller, exprs []expr.Evaluator) *Values { return &Values{ sctx: sctx, @@ -33,12 +35,18 @@ func (v *Values) Pull(done bool) (vector.Any, error) { for _, e := range v.exprs { vals = append(vals, e.Eval(val)) } - if len(vals) == 1 { - return vals[0], nil - } else if len(vals) != 0 { - return interleave(vals), nil + var out vector.Any + switch len(vals) { + case 0: + continue + case 1: + out = vals[0] + default: + out = interleave(vals) + } + if out = dropPlainNones(out); out != nil { + return out, nil } - // If no vals, continue the loop. } } diff --git a/runtime/ztests/expr/function/under.yaml b/runtime/ztests/expr/function/under.yaml index 0b86d8dd47..40aa92545b 100644 --- a/runtime/ztests/expr/function/under.yaml +++ b/runtime/ztests/expr/function/under.yaml @@ -31,7 +31,6 @@ output: | 1 <{s:string}> 1 - error("none not handled (consider ?? or ?.)") --- diff --git a/runtime/ztests/expr/index.yaml b/runtime/ztests/expr/index.yaml index 8637deda99..f78384dd27 100644 --- a/runtime/ztests/expr/index.yaml +++ b/runtime/ztests/expr/index.yaml @@ -59,7 +59,6 @@ output: | 1 2 1 - error("none not handled (consider ?? or ?.)") 127.0.0.1 --- @@ -107,7 +106,6 @@ output: | 5 error("cannot access field 'x' of set") "foo" - error("none not handled (consider ?? or ?.)") error({message:"invalid record index",on:1}) 123 @@ -143,8 +141,5 @@ input: | output: | "foo" "bar" - error("none not handled (consider ?? or ?.)") - error("none not handled (consider ?? or ?.)") - error("none not handled (consider ?? or ?.)") "foo" "foo" diff --git a/runtime/ztests/expr/subquery-empty.yaml b/runtime/ztests/expr/subquery-empty.yaml new file mode 100644 index 0000000000..d69820307a --- /dev/null +++ b/runtime/ztests/expr/subquery-empty.yaml @@ -0,0 +1,21 @@ +# A subquery that produces no values is none in a pipe expression and null +# in a SQL expression, for both correlated and uncorrelated subqueries. +spq: | + values {c:(values this | where false)}, + {u:(values 1 | where false)}, + (values this | where false) ?? "none" + +input: | + 1 + +output: | + {} + {} + "none" + +--- + +spq: select (select 1 where false) as x + +output: | + {x:null} diff --git a/runtime/ztests/op/aggregate/none-key.yaml b/runtime/ztests/op/aggregate/none-key.yaml new file mode 100644 index 0000000000..9dd92586b1 --- /dev/null +++ b/runtime/ztests/op/aggregate/none-key.yaml @@ -0,0 +1,28 @@ +# A plain none group key (e.g., from ok()) disappears from its output record, +# including when the aggregate is split into partials (as for a file scan) +# and for a nested key. +script: | + super -s -c 'values {a:1},{b:2},{b:3} | count() by k:=ok(a) | sort count' + echo === + super -s -c 'from test.sup | count() by k:=ok(a) | sort count' + echo === + super -s -c 'from test.sup | count() by x.k:=ok(a) | sort count' + +inputs: + - name: test.sup + data: | + {a:1} + {b:2} + {b:3} + +outputs: + - name: stdout + data: | + {k:1,count:1} + {count:2} + === + {k:1,count:1} + {count:2} + === + {x:{k:1},count:1} + {x:{},count:2} diff --git a/runtime/ztests/op/debug-none.yaml b/runtime/ztests/op/debug-none.yaml new file mode 100644 index 0000000000..e9168d596c --- /dev/null +++ b/runtime/ztests/op/debug-none.yaml @@ -0,0 +1,11 @@ +# A plain none debug value is dropped, as for the values operator. +script: | + super -s -c 'values {a:1},{b:2} | debug ok(a) | count()' + +outputs: + - name: stdout + data: | + 2 + - name: stderr + data: | + 1 diff --git a/runtime/ztests/op/unnest.yaml b/runtime/ztests/op/unnest.yaml index cb0ce48780..55391aa031 100644 --- a/runtime/ztests/op/unnest.yaml +++ b/runtime/ztests/op/unnest.yaml @@ -92,7 +92,7 @@ output: | --- -# A plain none to unnest is an error as for an unnamed unnest. +# A plain none to unnest produces no values as for an unnamed unnest. spq: unnest x:=o?.a with s flags: -dynamic @@ -102,5 +102,28 @@ input: | {o:some({a:[5]}),s:2} output: | - error({message:"unnest: encountered non-array value",on:none}) {x:5,s:2} + +--- + +# A plain none to unnest (e.g., from ok()) produces no values. +spq: unnest ok(a) + +input: | + {a:[1,2]} + {b:3} + {a:[4]} + +output: | + 1 + 2 + 4 + +--- + +# Statically, a plain none to unnest isn't an error. +spq: values {a:[1,2]},{b:3} | unnest ok(a) | values typeof(this) + +output: | + + diff --git a/runtime/ztests/op/values/drop-none.yaml b/runtime/ztests/op/values/drop-none.yaml new file mode 100644 index 0000000000..e331512620 --- /dev/null +++ b/runtime/ztests/op/values/drop-none.yaml @@ -0,0 +1,55 @@ +# The values operator drops plain nones, including those interleaved with +# the values of other expressions and those in a union, but keeps an +# option's none. +spq: values ok(a), 0 + +input: | + {a:1} + {b:2} + {a:"s"} + +output: | + 1 + 0 + 0 + "s" + 0 + +--- + +spq: values this + +input: | + 1::(int64|none) + none::(int64|none) + 2::(int64|none) + +output: | + 1::(int64|none) + 2::(int64|none) + +--- + +spq: values ok(a) ?? none::int64 + +input: | + {a:1} + {b:2} + +output: | + 1 + none::int64 + +--- + +# A dropped none isn't passed downstream. +spq: "values this==2 ? none : this | values typeof(this)" + +input: | + 1 + 2 + 3 + +output: | + + diff --git a/spq_test.go b/spq_test.go index 2a8fdea5ee..4c851d055d 100644 --- a/spq_test.go +++ b/spq_test.go @@ -184,6 +184,11 @@ func runOneBoomerang(t *testing.T, format, data string) { } t.Fatalf("unexpected error writing %s baseline: %s", format, err) } + if baseline == "" { + // The data has no values (e.g., it's only plain nones, which are + // dropped). + t.Skip("skipping due to no values") + } readerFormat := format if readerFormat == "bsup -rows" { readerFormat = "bsup" @@ -305,8 +310,9 @@ func (f *fusionRemovingReader) Read() (*super.Value, error) { if f.h.hasFusion(val.Type()) { continue } - if val.Type() == super.TypeNone { - // A plain none can't be output by a query. + if val.Deunion().Type() == super.TypeNone { + // A plain none (even in a union) is dropped from the + // output of a query. continue } } diff --git a/sup/ztests/decorate-none.yaml b/sup/ztests/decorate-none.yaml index 8fcaf9f9b3..dac6223dd1 100644 --- a/sup/ztests/decorate-none.yaml +++ b/sup/ztests/decorate-none.yaml @@ -24,7 +24,6 @@ input: | missing output: | - error("none not handled (consider ?? or ?.)") [1,none] [some(1),none::int64] set[none::int64,some(1)] diff --git a/vector/recordbuilder.go b/vector/recordbuilder.go index d9164f6369..ea84133a2c 100644 --- a/vector/recordbuilder.go +++ b/vector/recordbuilder.go @@ -10,6 +10,10 @@ import ( type RecordBuilder struct { sctx *super.Context base *rec + // DropNones causes a field whose value is a plain none (e.g., an + // aggregate's group key from ok() or ?.) to disappear from a record, as + // for a record expression. + DropNones bool } func NewRecordBuilder(sctx *super.Context, fields field.List) (*RecordBuilder, error) { @@ -23,7 +27,7 @@ func NewRecordBuilder(sctx *super.Context, fields field.List) (*RecordBuilder, e } func (r *RecordBuilder) New(vecs []Any) *Record { - rec, _ := r.base.build(r.sctx, vecs) + rec, _ := r.base.build(r.sctx, vecs, r.DropNones) return rec } @@ -56,19 +60,26 @@ func addPath(r *rec, path field.Path) error { return nil } -func (r *rec) build(sctx *super.Context, leafs []Any) (*Record, []Any) { +// build returns a record whose leaf fields are the leading values of leafs +// and the rest of leafs. If dropNones is true, a leaf field whose value is +// a plain none disappears from the record. +func (r *rec) build(sctx *super.Context, leafs []Any, dropNones bool) (*Record, []Any) { + n := leafs[0].Len() var fields []super.Field var out []Any for i, name := range r.paths { var vec Any if r.recs[i] != nil { - vec, leafs = r.recs[i].build(sctx, leafs) + vec, leafs = r.recs[i].build(sctx, leafs, dropNones) } else { vec, leafs = leafs[0], leafs[1:] + if dropNones && vec.Type() == super.TypeNone { + continue + } } fields = append(fields, super.NewField(name, vec.Type())) out = append(out, vec) } typ := sctx.MustLookupTypeRecord(fields) - return NewRecord(typ, out, out[0].Len()), leafs + return NewRecord(typ, out, n), leafs }