Merge main into feat/47-G-join11: the node-engine's if-config, config-lines, config-json and stop at a failure (issue 1979)

The whole test's web manifest keeps G's entry form (no build artifact, the entry read by fingerprint) and takes main's resources under if-config and config-lines.

Builds: design 47 G §421; design 42 zfs-owns-zfs H1-H5

Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
This commit is contained in:
2026-11-13 09:10:44 +01:00
15 changed files with 582 additions and 116 deletions
+116
View File
@@ -0,0 +1,116 @@
package resolution
import (
"encoding/json"
"fmt"
"math"
"regexp"
"slices"
"strconv"
"strings"
mcat "git.novox.be/novox/mesh-sdk/go/catalogue"
)
// configLinesRef is `${config-lines:<key>:<prefix>}` and `${config-lines:<key>:<prefix>:mib-bytes}` in a file's
// content (design 42 zfs-owns-zfs §3, ADR 0679 §16 (c)): one line per entry of a list value, `<prefix><entry>`. The
// prefix is literal text with no `}`, `$` or newline. A placeholder that holds a whole line takes its newline with it.
var configLinesRef = regexp.MustCompile(`\$\{config-lines:([a-z0-9][a-z0-9_.-]*):([^}$\n]*?)(:mib-bytes)?\}(\n?)`)
// configJSONRef is `${config-json}` in a file's content (design 42 zfs-owns-zfs §9, answer A-15): the module's set
// config values as one JSON object.
var configJSONRef = regexp.MustCompile(`\$\{config-json\}`)
// configLinesInto writes each `${config-lines:…}` of a file's content from the layers: a line `<prefix><entry>` for each
// entry of a list, one for a set scalar, nothing for an unset value or an empty list. With `mib-bytes` an entry given in
// MiB is written in bytes. An entry holding a newline is refused, and the content is left as it was.
func configLinesInto(content string, layers []mcat.Layer, module string) (string, error) {
var refused error
filled := configLinesRef.ReplaceAllStringFunc(content, func(token string) string {
m := configLinesRef.FindStringSubmatch(token)
key, prefix, mib, newline := m[1], m[2], m[3] != "", m[4]
value, set := settingValue(layers, key)
if !set {
return ""
}
entries, isList := value.([]any)
if !isList {
entries = []any{value}
}
var lines []string
for _, e := range entries {
text := plainly(e)
if strings.ContainsAny(text, "\r\n") {
refused = fmt.Errorf("%s: an entry of the config value %q holds a newline, which would begin a line "+
"of its own in the file it is written into (design 42 §3)", module, key)
return token
}
if mib {
n, err := mibWhole(text)
if err != nil || n > 1<<44 {
refused = fmt.Errorf("%s: the config value %q holds %q, and mib-bytes writes a whole number of MiB "+
"in bytes (design 42 §3)", module, key, text)
return token
}
text = strconv.FormatUint(n<<20, 10)
}
lines = append(lines, prefix+text)
}
if len(lines) == 0 {
return ""
}
return strings.Join(lines, "\n") + newline
})
if refused != nil {
return content, refused
}
return filled, nil
}
// configJSONInto writes `${config-json}` in a file's content as the module's set config values in one JSON object
// (design 42 zfs-owns-zfs §9, answer A-15): only keys the module declares under `config` are written, and what no layer sets is absent, and no unset value refuses the module; a
// list and an object are written as JSON, a number, a boolean and a text as JSON values of their own type.
func configJSONInto(content string, layers []mcat.Layer, declared []string) (string, error) {
if !configJSONRef.MatchString(content) {
return content, nil
}
set := map[string]any{}
for _, layer := range layers {
for k, v := range layer.Values {
if slices.Contains(declared, k) {
set[k] = v
}
}
}
var out strings.Builder
enc := json.NewEncoder(&out)
enc.SetEscapeHTML(false)
enc.SetIndent("", " ")
if err := enc.Encode(set); err != nil {
return content, fmt.Errorf("the config values cannot be written as JSON: %w", err)
}
object := strings.TrimSuffix(out.String(), "\n")
return configJSONRef.ReplaceAllLiteralString(content, object), nil
}
// mibWhole is a size in MiB as a whole number: digits, or a whole number a program wrote in exponent form (`1e+06`),
// which is how a float of a million or more prints (design 42 §3).
func mibWhole(text string) (uint64, error) {
if n, err := strconv.ParseUint(text, 10, 64); err == nil {
return n, nil
}
f, err := strconv.ParseFloat(text, 64)
if err != nil || f < 0 || f != math.Trunc(f) || f > 1<<44 {
return 0, fmt.Errorf("not a whole number of MiB")
}
return uint64(f), nil
}
// declaredConfig is the keys a module's manifest declares under `config`.
func declaredConfig(m mcat.Manifest) []string {
keys := make([]string, 0, len(m.Settings))
for k := range m.Settings {
keys = append(keys, k)
}
return keys
}
+4 -1
View File
@@ -659,6 +659,9 @@ func resolutionCompose(r mcat.Resolution, with mcat.Rendering, owner map[string]
// parses strictly and knows no such field.
absent := AbsentFor(r, m.Module)
resources = withoutAbsent(resources, absent)
// **A resource declared only while a config value holds** (design 42 zfs-owns-zfs §2): left out while none of
// its `if-config` conditions holds, and the key dropped from the rest.
resources = withoutUnconfigured(resources, WithDefaults(m, with.Settings[m.Module]))
// No container is given the mesh's names (novox/hq ADR 0148). It used to be: every
// container got the whole roster as `--add-host` entries at creation, and a name that
@@ -906,7 +909,7 @@ func resolutionCompose(r mcat.Resolution, with mcat.Rendering, owner map[string]
// **An operator's value, from the assignment** (novox/hq ADR 0112, ADR 0155): what a
// definition may not carry because it is true of one installation only. Filled from
// the same layers a mergeable file takes, and refused when no layer set it.
if err := unsetInConfigWords(settingInto(copied, WithDefaults(m, with.Settings[m.Module]), m.Module), m); err != nil {
if err := unsetInConfigWords(settingInto(copied, WithDefaults(m, with.Settings[m.Module]), m.Module, declaredConfig(m)...), m); err != nil {
return nil, err
}
// **Placed before anything reads a path.** A pathless directory receives the path
+80
View File
@@ -0,0 +1,80 @@
package resolution
import (
"fmt"
"strings"
mcat "git.novox.be/novox/mesh-sdk/go/catalogue"
)
// withoutUnconfigured drops each resource whose `if-config` holds none of its conditions against the module's config
// values (design 42 zfs-owns-zfs §2, ADR 0679 §16 (b)), and the key itself from every resource that keeps it: the
// node-engine parses strictly and knows no such key. What leaves the declaration this way is removed as any resource
// that leaves it (ADR 0269 §2), by the node-engine's own orphan removal.
func withoutUnconfigured(resources []map[string]any, layers []mcat.Layer) []map[string]any {
out := make([]map[string]any, 0, len(resources))
for _, r := range resources {
conditions, has := r[mcat.IfConfigField]
if !has {
out = append(out, r)
continue
}
if !ifConfigHolds(conditions, layers) {
continue
}
kept := make(map[string]any, len(r))
for k, v := range r {
if k != mcat.IfConfigField {
kept[k] = v
}
}
out = append(out, kept)
}
return out
}
// ifConfigHolds is whether any condition of a list holds: `<key>` when the config value is set and is neither false,
// off, none, empty nor an empty list; `<key>=<value>` when the value equals it, or a list value holds it as an entry.
func ifConfigHolds(conditions any, layers []mcat.Layer) bool {
list, _ := conditions.([]any)
for _, c := range list {
key, want, equals := strings.Cut(fmt.Sprint(c), "=")
value, set := settingValue(layers, key)
if !set {
continue
}
if !equals {
if !isOff(value) {
return true
}
continue
}
if entries, isList := value.([]any); isList {
for _, e := range entries {
if plainly(e) == want {
return true
}
}
} else if plainly(value) == want {
return true
}
}
return false
}
// isOff is a config value that says "not here": false, off, none, an empty text or an empty list.
func isOff(value any) bool {
switch v := value.(type) {
case bool:
return !v
case []any:
return len(v) == 0
case nil:
return true
}
switch plainly(value) {
case "", "false", "off", "none":
return true
}
return false
}
+10 -2
View File
@@ -65,8 +65,9 @@ func unsetInConfigWords(err error, m mcat.Manifest) error {
// The last layer setting a key wins, which is the node's over the mesh's over the module's own
// default (novox/hq ADR 0262) — the same order settle applies to a mergeable file. The caller lays
// the defaults under the layers with WithDefaults. A value that is not a string is written the way a program would read
// it (a number without a trailing .000000, a boolean as true/false).
func settingInto(resource map[string]any, layers []mcat.Layer, module string) error {
// it (a number without a trailing .000000, a boolean as true/false). declared are the keys the module's manifest
// declares under `config`, the only ones `${config-json}` writes (design 42 §9, answer A-15).
func settingInto(resource map[string]any, layers []mcat.Layer, module string, declared ...string) error {
if fmt.Sprint(resource["type"]) == "service" {
return settingIntoUnit(resource, layers, module)
}
@@ -83,6 +84,13 @@ func settingInto(resource map[string]any, layers []mcat.Layer, module string) er
if !ok {
return nil
}
content, err := configLinesInto(content, layers, module)
if err != nil {
return err
}
if content, err = configJSONInto(content, layers, declared); err != nil {
return err
}
for _, key := range settingsUsed(content) {
value, set := settingValue(layers, key)
if !set {
+134 -25
View File
@@ -1,42 +1,151 @@
package resolution
import (
"errors"
"fmt"
"strings"
"testing"
mcat "git.novox.be/novox/mesh-sdk/go/catalogue"
)
// An operator's value reaches a file from the assignment's settings, the node's layer over the
// mesh's (novox/hq ADR 0112, ADR 0155), and a value nothing set is refused by name.
func TestASettingReachesAFileFromTheLayers(t *testing.T) {
file := map[string]any{"id": "env", "type": "file", "path": "/x/mail.env",
"content": "DOMAIN=${setting:domain}\nSITENAME=${setting:sitename}\nWORKERS=${setting:workers}\n"}
layers := []mcat.Layer{
{From: "the mesh", Values: map[string]any{"domain": "example.tld", "sitename": "Mesh", "workers": float64(4)}},
{From: "this node", Values: map[string]any{"sitename": "This one"}},
// A config value reaches a resource from the assignment's layers, the node's over the mesh's (novox/hq ADR 0112,
// ADR 0155), and a value nothing set is refused by name; and a resource under `if-config` is declared only while a
// config value holds (design 42 zfs-owns-zfs §2, ADR 0679 §16 (b)). One table of the two, each row its layers, what it
// does to a resource and what comes of it.
func TestAConfigValueReachesAResourceAndDecidesWhetherItIsDeclared(t *testing.T) {
filled := func(content string) func([]mcat.Layer) (string, error) {
return func(layers []mcat.Layer) (string, error) {
file := map[string]any{"id": "env", "type": "file", "path": "/x/mail.env", "content": content}
err := settingInto(file, layers, "mail", "n", "b", "s", "l", "o")
return file["content"].(string), err
}
}
if err := settingInto(file, layers, "mail"); err != nil {
t.Fatal(err)
// declared is the ids of what is left of a resource under each condition, and whether the key is gone from it.
declared := func(conditions ...any) func([]mcat.Layer) (string, error) {
return func(layers []mcat.Layer) (string, error) {
kept := withoutUnconfigured([]map[string]any{
{"id": "plain", "type": "file"}, {"id": "cond", "type": "file", mcat.IfConfigField: conditions}}, layers)
var ids []string
for _, r := range kept {
if _, has := r[mcat.IfConfigField]; has {
return "", errKeyKept
}
ids = append(ids, r["id"].(string))
}
return strings.Join(ids, ","), nil
}
}
if file["content"] != "DOMAIN=example.tld\nSITENAME=This one\nWORKERS=4\n" {
t.Fatalf("filled as %q", file["content"])
values := func(kv map[string]any) []mcat.Layer { return []mcat.Layer{{From: "the mesh", Values: kv}} }
for _, row := range []struct {
name string
layers []mcat.Layer
do func([]mcat.Layer) (string, error)
want string // what comes of it
errs []string // or what the refusal says
}{
{"the node's layer over the mesh's", []mcat.Layer{
{From: "the mesh", Values: map[string]any{"domain": "example.tld", "sitename": "Mesh", "workers": float64(4)}},
{From: "this node", Values: map[string]any{"sitename": "This one"}}},
filled("DOMAIN=${setting:domain}\nSITENAME=${setting:sitename}\nWORKERS=${setting:workers}\n"),
"DOMAIN=example.tld\nSITENAME=This one\nWORKERS=4\n", nil},
{"a value nothing set is refused, the file left as it was", values(map[string]any{"other": "x"}),
filled("DOMAIN=${setting:domain}\n"), "DOMAIN=${setting:domain}\n",
[]string{"${setting:domain}", "settings set mail", "set today: other"}},
{"config-lines: a list of two writes two lines in order", values(map[string]any{"k": []any{"a", "b"}}),
filled("h\n${config-lines:k:options zfs x=}\nt\n"), "h\noptions zfs x=a\noptions zfs x=b\nt\n", nil},
{"config-lines: mib-bytes writes a million MiB as a whole number", values(map[string]any{"k": float64(1000000)}),
filled("${config-lines:k:zfs_arc_max=:mib-bytes}\n"), "zfs_arc_max=1048576000000\n", nil},
{"config-lines: mib-bytes writes a million MiB given as text as a whole number", values(map[string]any{"k": "1000000"}),
filled("${config-lines:k:zfs_arc_max=:mib-bytes}\n"), "zfs_arc_max=1048576000000\n", nil},
{"config-lines: mib-bytes writes a size printed in exponent form as a whole number", values(map[string]any{"k": "1e+06"}),
filled("${config-lines:k:zfs_arc_max=:mib-bytes}\n"), "zfs_arc_max=1048576000000\n", nil},
{"config-lines: mib-bytes refuses a fraction", values(map[string]any{"k": "1.5"}),
filled("${config-lines:k:zfs_arc_max=:mib-bytes}\n"), "${config-lines:k:zfs_arc_max=:mib-bytes}\n", []string{"mib-bytes"}},
{"config-lines: an empty list writes nothing", values(map[string]any{"k": []any{}}),
filled("h\n${config-lines:k:p=}\nt\n"), "h\nt\n", nil},
{"config-lines: an unset value writes nothing", nil, filled("h\n${config-lines:k:p=}\nt\n"), "h\nt\n", nil},
{"config-lines: a set scalar writes one line", values(map[string]any{"k": "v"}),
filled("${config-lines:k:p=}\n"), "p=v\n", nil},
{"config-lines: mib-bytes writes MiB in bytes", values(map[string]any{"k": float64(64)}),
filled("${config-lines:k:zfs_arc_max=:mib-bytes}\n"), "zfs_arc_max=67108864\n", nil},
{"config-lines: an entry with a newline is refused, the file left as it was", values(map[string]any{"k": []any{"a\nb"}}),
filled("${config-lines:k:p=}\n"), "${config-lines:k:p=}\n", []string{"newline"}},
{"config-json: the set values only, each as its own type", values(map[string]any{"n": float64(64), "b": true, "s": "x",
"l": []any{"a"}, "o": map[string]any{"f": float64(1)}}), filled("${config-json}\n"),
"{\n \"b\": true,\n \"l\": [\n \"a\"\n ],\n \"n\": 64,\n \"o\": {\n \"f\": 1\n },\n \"s\": \"x\"\n}\n", nil},
{"config-json: a key the module does not declare is not written", values(map[string]any{"s": "x", "scrub-pool": "tank"}),
filled("${config-json}\n"), "{\n \"s\": \"x\"\n}\n", nil},
{"config-json: nothing set is an empty object, not a refusal", nil, filled("${config-json}\n"), "{}\n", nil},
{"if-config: a key never set leaves it out", nil, declared("k"), "plain", nil},
{"if-config: a key set to text declares it", values(map[string]any{"k": "x"}), declared("k"), "plain,cond", nil},
{"if-config: a key set to false, off, none, empty or an empty list leaves it out", values(map[string]any{"k": false}),
declared("k"), "plain", nil},
{"if-config: off", values(map[string]any{"k": "off"}), declared("k"), "plain", nil},
{"if-config: none", values(map[string]any{"k": "none"}), declared("k"), "plain", nil},
{"if-config: empty text", values(map[string]any{"k": ""}), declared("k"), "plain", nil},
{"if-config: an empty list", values(map[string]any{"k": []any{}}), declared("k"), "plain", nil},
{"if-config: a list of entries", values(map[string]any{"k": []any{"a"}}), declared("k"), "plain,cond", nil},
{"if-config: a value equal to the condition", values(map[string]any{"k": "on"}), declared("k=on"), "plain,cond", nil},
{"if-config: a value other than the condition", values(map[string]any{"k": "off"}), declared("k=on"), "plain", nil},
{"if-config: a list holding the entry", values(map[string]any{"k": []any{"a", "b"}}), declared("k=b"), "plain,cond", nil},
{"if-config: a list without the entry", values(map[string]any{"k": []any{"a"}}), declared("k=b"), "plain", nil},
{"if-config: any of the conditions", values(map[string]any{"j": "x"}), declared("k", "j"), "plain,cond", nil},
{"if-config: a value from a default layer", []mcat.Layer{{Default: true, Values: map[string]any{"k": "on"}}},
declared("k=on"), "plain,cond", nil},
{"if-config: a number equal to the condition", values(map[string]any{"k": float64(64)}), declared("k=64"), "plain,cond", nil},
} {
t.Run(row.name, func(t *testing.T) {
got, err := row.do(row.layers)
if len(row.errs) == 0 && err != nil {
t.Fatal(err)
}
if len(row.errs) > 0 && err == nil {
t.Fatalf("a refusal was expected, got %q", got)
}
for _, want := range row.errs {
if !strings.Contains(err.Error(), want) {
t.Fatalf("the refusal lacks %q: %v", want, err)
}
}
if got != row.want {
t.Fatalf("got %q, want %q", got, row.want)
}
})
}
}
func TestASettingNothingSetIsRefusedByName(t *testing.T) {
file := map[string]any{"id": "env", "type": "file", "content": "DOMAIN=${setting:domain}\n"}
err := settingInto(file, []mcat.Layer{{From: "the mesh", Values: map[string]any{"other": "x"}}}, "mail")
if err == nil {
t.Fatal("a setting nothing set was written as something")
}
for _, want := range []string{"${setting:domain}", "settings set mail", "set today: other"} {
if !strings.Contains(err.Error(), want) {
t.Fatalf("the refusal lacks %q: %v", want, err)
// errKeyKept is what a row of the table above says when `if-config` reached the node-engine, which parses strictly.
var errKeyKept = errors.New("the if-config key was kept in a declared resource")
// The composition takes a resource out while its condition does not hold, and a value going from set to unset takes it
// out of the next composition, which the node-engine removes as it removes any resource that left (design 42 §2).
func TestAConfigConditionedResourceIsComposedWhileItsValueHolds(t *testing.T) {
m := mustParse(t, `{"module":"zfs","version":"1","config":{"zed":{"type":"string","default":"off"},"mods":{"type":"list"}},"resources":[
{"id":"opts","type":"file","path":"/etc/modprobe.d/zfs.conf","content":"${config-lines:mods:options zfs }\n${config-json}\n"},
{"id":"conf","type":"file","path":"/etc/zfs/mesh.conf","content":"x\n"},
{"id":"zed","type":"file","path":"/etc/zfs/zed.rc","content":"y\n","if-config":["zed=on"]}]}`)
for _, row := range []struct {
layers []mcat.Layer
zed bool
}{{nil, false}, {[]mcat.Layer{{From: "this node", Values: map[string]any{"zed": "on", "mods": []any{"a=1"}}}}, true}} {
out, err := Declaration(mcat.Resolution{Node: "workstation", Modules: []mcat.Manifest{m}},
mcat.Rendering{Settings: mcat.SettingsBy{"zfs": row.layers}})
if err != nil {
t.Fatal(err)
}
if fileNamed(out, "zfs.conf") == nil {
t.Error("the unconditioned file was left out")
}
if opts := fileNamed(out, "zfs.opts"); opts == nil || (row.zed != strings.HasPrefix(fmt.Sprint(opts["content"]), "options zfs a=1\n")) {
t.Errorf("the template's lines do not follow the list: %v", opts)
}
zed := fileNamed(out, "zfs.zed")
if (zed != nil) != row.zed {
t.Errorf("with %v the conditioned file is declared: %v, want %v", row.layers, zed != nil, row.zed)
}
if _, has := zed[mcat.IfConfigField]; has {
t.Errorf("%s reached the node-engine, which parses strictly", mcat.IfConfigField)
}
}
// Left as it was: the literal placeholder must never reach a machine.
if file["content"] != "DOMAIN=${setting:domain}\n" {
t.Fatalf("content was changed on refusal: %q", file["content"])
}
}
+1 -1
View File
@@ -458,7 +458,7 @@ func JudgeSettings(m mcat.Manifest, layers []mcat.Layer, adopted bool) error {
for k, v := range settled {
copied[k] = v
}
if err := unsetInConfigWords(settingInto(copied, WithDefaults(m, layers), m.Module), m); err != nil {
if err := unsetInConfigWords(settingInto(copied, WithDefaults(m, layers), m.Module, declaredConfig(m)...), m); err != nil {
return err
}
}
@@ -25,6 +25,10 @@ type filler struct {
// or the sweep refuses the placeholder it would have filled.
var fillers = []filler{
{namespace: "setting", patterns: []*regexp.Regexp{settingRef}, fields: map[string][]string{"file": {"content"}, "service": {"unit"}, "directory": {"owner"}, "container": {"groups"}}},
// A file's content only: lines from a config value's entries, and the module's config as one JSON object
// (design 42 zfs-owns-zfs §3, §9; config_lines.go).
{namespace: "config-lines", patterns: []*regexp.Regexp{configLinesRef}, fields: map[string][]string{"file": {"content"}}},
{namespace: "config-json", patterns: []*regexp.Regexp{configJSONRef}, fields: map[string][]string{"file": {"content"}}},
{namespace: "dir", patterns: []*regexp.Regexp{dirRef}, fields: map[string][]string{"*": {"path", "content", "volumes", "env", "env-file"}}},
{namespace: "access", patterns: []*regexp.Regexp{accessRef}, fields: map[string][]string{"*": {"path", "content", "volumes", "env", "env-file"}}},
{namespace: "secret", patterns: []*regexp.Regexp{placeholder}, fields: map[string][]string{"file": {"content"}},