diff --git a/cmd/mesh-host/nodeagent_integration_test.go b/cmd/mesh-host/nodeagent_integration_test.go index be40a99a..597da0df 100644 --- a/cmd/mesh-host/nodeagent_integration_test.go +++ b/cmd/mesh-host/nodeagent_integration_test.go @@ -495,8 +495,16 @@ func h8(t *testing.T, b *itBed, checkBuild bool) (func(), h8state) { candidates := kvOf(buckets.ProposalCandidates) packages := kvOf(link.PackagesBucket) path := filepath.Join(b.dir, "web.conf") - man := `{"module":"web","version":"1","config":{"greeting":{"type":"string","description":"what the file says","default":"hello"}},` + + // Beside the greeting: a file under `if-config`, a template with `config-lines` from a list, and two files under a + // value that makes the first of them fail (a file below a file), the second being left (design 42 zfs-owns-zfs §5). + more := func(name string) string { return filepath.Join(b.dir, "web-"+name+".conf") } + man := `{"module":"web","version":"1","config":{"greeting":{"type":"string","description":"what the file says","default":"hello"},` + + `"mods":{"type":"list","description":"lines","default":["a","b"]}},` + `"resources":[{"id":"conf","type":"file","path":"` + path + `","mode":"0644","content":"greeting=${config:greeting}\n"},` + + `{"id":"mods","type":"file","path":"` + more("mods") + `","mode":"0644","content":"${config-lines:mods:m=}\n"},` + + `{"id":"salut","type":"file","path":"` + more("salut") + `","mode":"0644","content":"salut\n","if-config":["greeting=salut"]},` + + `{"id":"bad","type":"file","path":"` + path + `/x","mode":"0644","content":"x\n","if-config":["greeting=boom"]},` + + `{"id":"after","type":"file","path":"` + more("after") + `","mode":"0644","content":"after\n","if-config":["greeting=boom"]},` + `{"id":"tools","type":"package","package":"web-tools"}]}` digest := meshcatalogclient.ManifestDigest([]byte(man)) mraw, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: "web", Manifest: man}) @@ -606,6 +614,36 @@ func h8rest(t *testing.T, b *itBed, resolved, candidates nats.KeyValue, hkey str t.Fatal(err) } within(t, 3*time.Second, "a changed config value was not composed and applied again", func() bool { return b.file("web") == "greeting=salut\n" }) + // Design 42 zfs-owns-zfs §5: the file under `if-config` appears with its value and goes with it, the template's lines + // follow the list, and a failing resource stops its module's apply with the later resource left. + other := func(name string) string { + got, _ := os.ReadFile(filepath.Join(b.dir, "web-"+name+".conf")) + return string(got) + } + setGreeting := func(value string) { + if _, err := resolved.Put(itNode+".web.greeting", []byte(`{"value":"`+value+`","source":"module"}`)); err != nil { + t.Fatal(err) + } + } + within(t, 3*time.Second, "the file under if-config did not appear with its value, or the lines do not follow the list", func() bool { + return other("salut") == "salut\n" && other("mods") == "m=a\nm=b\n" + }) + setGreeting("bonjour") + within(t, 3*time.Second, "the file under if-config did not go with its value", func() bool { + _, err := os.Stat(filepath.Join(b.dir, "web-salut.conf")) + return b.file("web") == "greeting=bonjour\n" && errors.Is(err, os.ErrNotExist) + }) + setGreeting("boom") + within(t, 5*time.Second, "the failing resource did not fail its module", func() bool { + return b.have("web").Phase == nec.PhaseFailed + }) + if got := other("after"); got != "" { + t.Errorf("a resource after the failed one was applied: %q", got) + } + setGreeting("salut") + within(t, 5*time.Second, "the module did not run again once its value held no failing resource", func() bool { + return b.file("web") == "greeting=salut\n" && b.have("web").Phase == nec.PhaseRunning + }) within(t, 2*time.Second, "the have does not name the revision", func() bool { h := b.have("web") return h.Fp == fp && h.Composed != first && h.Phase == nec.PhaseRunning diff --git a/cmd/mesh-host/probe_test.go b/cmd/mesh-host/probe_test.go index 357aa1e0..be454719 100644 --- a/cmd/mesh-host/probe_test.go +++ b/cmd/mesh-host/probe_test.go @@ -88,11 +88,12 @@ func TestThePackagesGoInWhenTheApplyThatRestoresThePackageManagerWroteIt(t *test conf := filepath.Join(dir, "pacman.conf") pacmanAnsweringOnce(t, conf) opts := options{state: filepath.Join(dir, "state.json")} - // The package first: within one pass it is asked about before the file is written, which is the order that - // leaves it to a second pass. + // The file first, as the pacman module declares it: a module's apply stops at its first failed resource + // (design 42 zfs-owns-zfs §1, answer A-26), so the file that makes the package manager answer comes before the + // package that needs it. raw := []byte(`{"declaration":1,"resources":[` + - `{"id":"pacman.base","type":"package","package":"base"},` + - `{"id":"pacman.conf","type":"file","path":"` + conf + `","content":"[options]\n"}]}`) + `{"id":"pacman.conf","type":"file","path":"` + conf + `","content":"[options]\n"},` + + `{"id":"pacman.base","type":"package","package":"base"}]}`) report := applyAndKeep(context.Background(), opts, raw, nil, nil, nil) diff --git a/internal/apply/apply.go b/internal/apply/apply.go index a6c00f9a..a1ff719a 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -578,6 +578,17 @@ func applyScoped( } if declaration.Type(orphan.Type) == declaration.TypeService { if by, held := heldUnits[unitKey(orphan.Scope, orphan.User, orphan.Target)]; held { + // **What the going record found passes to the record that stays** (design 42 zfs-owns-zfs §2, answer + // A-21), as a file is handed to the resource still declaring its path (ADR 0223): the unit as the machine + // had it before the mesh is what the staying record gives back when it goes, not what the mesh made of + // it. Only where the staying record has no finding of its own, kept as an unfinished apply's is. + if orphan.Found != nil && declared[by] { + if rec, has := known.Find(by); !has || rec.Found == nil { + if _, pending := known.FoundFirst[by]; !pending { + known.KeepFound(by, origin, *orphan.Found) + } + } + } known.Forget(orphan.ID) detail := "no longer declared; " + by + " still holds the unit, so it was left as it is" report.Outcomes = append(report.Outcomes, Outcome{ @@ -1162,6 +1173,16 @@ func applyScoped( if p, ok := resource.(*declaration.Process); ok && p.RunOnce { gates = true } + // **A module's apply stops at its first failed resource** (design 42 zfs-owns-zfs §1, ADR 0679 §16 (a)): any + // other resource of a module that failed gates the rest of that module, which is reported skipped with this + // reason. What it applied before stays applied; other modules, and a resource of no module, go on as before. + if module, ours := d.ModuleOfID(resource.Identity()); ours && !gates { + failed.Gated = true + gated[module] = true + gatedWhy[module] = fmt.Sprintf("%s failed, so nothing more of this module was attempted", resource.Identity()) + log(fmt.Sprintf(" gated %s.*: %s failed, so the rest of it was not attempted", module, resource.Identity())) + continue + } if gates { failed.Gated = true // **A step that failed has not completed, whatever it did before** (novox/hq issue 467). A diff --git a/internal/apply/apply_test.go b/internal/apply/apply_test.go index b5b62294..506a81a4 100644 --- a/internal/apply/apply_test.go +++ b/internal/apply/apply_test.go @@ -182,77 +182,93 @@ func TestARenameToTheSamePathDoesNotDeleteTheNewFile(t *testing.T) { } } -func TestAFailedStepFailsTheApplyAndTheRestIsStillAttempted(t *testing.T) { - // The apply fails, names the resource, and carries what did happen — because the machine is - // in whatever state the apply reached and the only honest thing to hand back is that list. - // - // **And everything is attempted.** It used to stop at the first failure, which made one - // broken resource hold the whole machine hostage: a module declaring a package that does not - // exist meant every module after it was never applied, for ever - // (novox/hq 04-ISSUES/011). The case for stopping was that a later resource may depend on an - // earlier one — and it still may, and it then fails its own check and is reported, which is - // more information than not attempting it. +// A failed resource fails the apply and names itself, and what else is attempted depends on whose it is +// (novox/hq 04-ISSUES/011; design 42 zfs-owns-zfs §1, ADR 0679 §16 (a)): the resources of no module are all +// attempted, so one broken resource holds no machine hostage and every failure is counted, not just the first; +// a module's apply stops at its first failed resource, reports each one left, and no other module is touched. +func TestAFailedResourceFailsTheApplyAndWhatFollowsIsAttemptedByWhoseItIs(t *testing.T) { dir := t.TempDir() - blocker := filepath.Join(dir, "blocker") - if err := os.WriteFile(blocker, []byte("i am a file\n"), 0o644); err != nil { - t.Fatal(err) - } - - d := parse(t, `{"declaration":1,"resources":[ - {"id":"fine","type":"file","path":"`+filepath.Join(dir, "fine.conf")+`","content":"a\n"}, - {"id":"doomed","type":"directory","path":"`+blocker+`"}, - {"id":"after","type":"file","path":"`+filepath.Join(dir, "after.conf")+`","content":"b\n"} - ]}`) - - _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil, nil) - if err == nil { - t.Fatal("an impossible resource did not fail the apply") - } - - var applyErr *Error - if !errors.As(err, &applyErr) { - t.Fatalf("expected an apply error, got %T", err) - } - if applyErr.Resource != "doomed" { - t.Errorf("the failure names %q, not the resource that failed", applyErr.Resource) - } - // Everything that worked is in the report, before and after the failure. - if len(applyErr.Done.Outcomes) != 2 { - t.Errorf("the error does not carry what was applied: %+v", applyErr.Done.Outcomes) - } - if _, err := os.Stat(filepath.Join(dir, "after.conf")); err != nil { - t.Error("a resource after the failing one was never attempted, so one broken module " + - "still blocks every module after it") - } -} - -func TestEveryFailureIsCountedNotJustTheFirst(t *testing.T) { - // "One thing failed" and "eleven things failed" are different machines, and the first line is - // what somebody reads. - dir := t.TempDir() - for _, name := range []string{"one", "two"} { - if err := os.WriteFile(filepath.Join(dir, name), []byte("a file\n"), 0o644); err != nil { + for _, name := range []string{"blocker", "one", "two"} { + if err := os.WriteFile(filepath.Join(dir, name), []byte("i am a file\n"), 0o644); err != nil { t.Fatal(err) } } - d := parse(t, `{"declaration":1,"resources":[ - {"id":"first","type":"directory","path":"`+filepath.Join(dir, "one")+`"}, - {"id":"second","type":"directory","path":"`+filepath.Join(dir, "two")+`"} - ]}`) - - _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil, nil) - if err == nil { - t.Fatal("two impossible resources did not fail the apply") + file := func(id, name string) string { + return `{"id":"` + id + `","type":"file","path":"` + filepath.Join(dir, name) + `","content":"a\n"}` } - var applyErr *Error - if !errors.As(err, &applyErr) { - t.Fatalf("got %T", err) + unwritable := func(id, name string) string { // a file below a file: it cannot be made + return `{"id":"` + id + `","type":"file","path":"` + filepath.Join(dir, name, "x.conf") + `","content":"a\n"}` } - if applyErr.Others != 1 { - t.Errorf("the failure says %d others also failed, and one did", applyErr.Others) + doomed := func(id, name string) string { + return `{"id":"` + id + `","type":"directory","path":"` + filepath.Join(dir, name) + `"}` } - if !strings.Contains(err.Error(), "one other resource also failed") { - t.Errorf("the message does not say others failed: %v", err) + exists := func(name string) bool { + _, err := os.Stat(filepath.Join(dir, name)) + return err == nil + } + for _, tc := range []struct { + name string + resources []string + failed string + others int + applied []string // files that must exist + left []string // files that must not, reported skipped + detail string // what the module's result says, when it is one module's + }{ + {"of no module, the rest is attempted", []string{file("fine", "fine.conf"), doomed("doomed", "blocker"), file("after", "after.conf")}, + "doomed", 0, []string{"fine.conf", "after.conf"}, nil, ""}, + {"of no module, every failure is counted", []string{doomed("first", "one"), doomed("second", "two")}, + "first", 1, nil, nil, ""}, + {"of a module, it stops there and names each one left", + []string{file("m.first", "first.conf"), unwritable("m.second", "blocker"), file("m.third", "third.conf"), file("m.fourth", "fourth.conf"), file("o.other", "other.conf")}, + "m.second", 0, []string{"first.conf", "other.conf"}, []string{"third.conf", "fourth.conf"}, + "m.second failed"}, + } { + t.Run(tc.name, func(t *testing.T) { + d := parse(t, `{"declaration":1,"resources":[`+strings.Join(tc.resources, ",")+`]}`) + report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil, nil) + var applyErr *Error + if !errors.As(err, &applyErr) { + t.Fatalf("expected an apply error, got %v", err) + } + if applyErr.Resource != tc.failed || applyErr.Others != tc.others { + t.Errorf("the failure names %q with %d others, want %q with %d", applyErr.Resource, applyErr.Others, tc.failed, tc.others) + } + if tc.others == 1 && !strings.Contains(err.Error(), "one other resource also failed") { + t.Errorf("the message does not say others failed: %v", err) + } + for _, name := range tc.applied { + if !exists(name) { + t.Errorf("%s was not applied", name) + } + } + for _, name := range tc.left { + if exists(name) { + t.Errorf("%s was applied after its module failed", name) + } + } + if tc.detail == "" { + return + } + outcomes := applyErr.Done.Outcomes + skipped := 0 + for _, o := range outcomes { + if o.Action == "skipped" { + skipped++ + } + } + if skipped != len(tc.left) { + t.Errorf("%d resources reported skipped, want %d: %+v", skipped, len(tc.left), outcomes) + } + result := ModuleResults(d, report, err)["m"] + if !result.Failed || !strings.Contains(result.Detail, tc.detail) || + !strings.Contains(result.Detail, "m.third") || !strings.Contains(result.Detail, "m.fourth") { + t.Errorf("the module's result does not name the failed resource and each one left: %+v", result) + } + if other := ModuleResults(d, report, err)["o"]; other.Failed { + t.Errorf("another module was failed by it: %+v", other) + } + }) } } diff --git a/internal/apply/into_handover_test.go b/internal/apply/into_handover_test.go index 30283519..fc533904 100644 --- a/internal/apply/into_handover_test.go +++ b/internal/apply/into_handover_test.go @@ -112,18 +112,77 @@ func TestTheRegistryMovesToTheRuntimesModuleWithoutLeavingTheList(t *testing.T) } } -// And the preview says the same before it happens. -func TestThePlanForgetsARecordOfAUnitStillHeld(t *testing.T) { - path := filepath.Join(t.TempDir(), "daemon.json") - var commands []string - _, state, err := Apply(context.Background(), archHost(t), runtimeDecl(t, path, true), store.State{}, - store.OriginDeclared, recordingServices(&commands), nil, nil) - if err != nil { - t.Fatal(err) - } - for _, step := range Plan(runtimeDecl(t, path, false), state, store.OriginDeclared) { - if step.ID == "networking.registry-trust-reload" && step.Verb != "forget" { - t.Errorf("the plan would %s the network's record of a unit docker.runtime holds: %s", step.Verb, step.Why) +// A unit two records name, under opposite conditions, passes between them (design 42 zfs-owns-zfs §2, answer A-21): the +// going record is forgotten, the preview says so before it happens, and what it found of the unit before the mesh is what +// the staying record gives back when its module leaves, not what the mesh made of it. +func TestARecordOfAUnitStillHeldIsForgottenAndPassesWhatItFound(t *testing.T) { + t.Run("the preview forgets it", func(t *testing.T) { + path := filepath.Join(t.TempDir(), "daemon.json") + var commands []string + _, state, err := Apply(context.Background(), archHost(t), runtimeDecl(t, path, true), store.State{}, + store.OriginDeclared, recordingServices(&commands), nil, nil) + if err != nil { + t.Fatal(err) } - } + for _, step := range Plan(runtimeDecl(t, path, false), state, store.OriginDeclared) { + if step.ID == "networking.registry-trust-reload" && step.Verb != "forget" { + t.Errorf("the plan would %s the network's record of a unit docker.runtime holds: %s", step.Verb, step.Why) + } + } + }) + t.Run("what it found passes to the record that stays", func(t *testing.T) { + // A unit found stopped and disabled, so the mesh's first record of it started it. + active, enabled := false, false + var commands []string + run := func(_ context.Context, name string, args ...string) (string, error) { + args = oldShape(name, args) + line := name + " " + strings.Join(args, " ") + commands = append(commands, line) + switch { + case strings.Contains(line, " start "): + active = true + case strings.Contains(line, " stop "): + active = false + case strings.Contains(line, " enable "): + enabled = true + case strings.Contains(line, " disable "): + enabled = false + case strings.Contains(line, "is-enabled"): + if enabled { + return "enabled", nil + } + return "disabled", nil + case strings.Contains(line, "show") && strings.Contains(line, "ActiveState"): + if active { + return "LoadState=loaded\nActiveState=active\nSubState=running", nil + } + return "LoadState=loaded\nActiveState=inactive\nSubState=dead", nil + } + return "", nil + } + unit := func(id, state, boot string) *declaration.Declaration { + return parse(t, `{"declaration":1,"resources":[{"id":"`+id+`","type":"service","unit":"z.service","state":"`+ + state+`","boot":"`+boot+`"}]}`) + } + apply := func(d *declaration.Declaration, known store.State) store.State { + t.Helper() + _, known, err := Apply(context.Background(), archHost(t), d, known, store.OriginDeclared, run, nil, nil) + if err != nil { + t.Fatal(err) + } + return known + } + known := apply(unit("z.on", "running", "enabled"), store.State{}) + known = apply(unit("z.off", "stopped", "disabled"), known) // the value changed: one record goes, the other stays + if rec, _ := known.Find("z.off"); rec.Found == nil || rec.Found.State != "stopped" || rec.Found.Boot != "disabled" { + t.Fatalf("the staying record did not receive what the going one found: %+v", rec.Found) + } + commands = nil + apply(somethingElse(t), known) // the module leaves + for _, c := range commands { + if strings.Contains(c, " start ") || strings.Contains(c, " enable ") { + t.Errorf("the unit was given back as the mesh made it, not as it was found: %q", c) + } + } + }) } diff --git a/internal/apply/module_results.go b/internal/apply/module_results.go index affe5fbb..5156ecc7 100644 --- a/internal/apply/module_results.go +++ b/internal/apply/module_results.go @@ -41,6 +41,7 @@ func ModuleResults(d *declaration.Declaration, report Report, err error) map[str outcomes = append(append([]Outcome(nil), outcomes...), failed.Done.Outcomes...) } skipped := map[string]string{} + left := map[string][]string{} for _, o := range outcomes { module, ok := d.ModuleOfID(o.ID) if !ok { @@ -48,6 +49,7 @@ func ModuleResults(d *declaration.Declaration, report Report, err error) map[str } if o.Action == "skipped" { skipped[module] = o.Detail + left[module] = append(left[module], o.ID) continue } attempted[o.ID] = true @@ -92,6 +94,11 @@ func ModuleResults(d *declaration.Declaration, report Report, err error) map[str } } for module, why := range reasons { + // Each resource left unapplied is named (design 42 zfs-owns-zfs §1). + if ids := left[module]; len(ids) > 0 { + sort.Strings(ids) + why = append(why, "left unapplied: "+strings.Join(ids, ", ")) + } sort.Strings(why) out[module] = ModuleResult{Failed: true, Detail: strings.Join(why, "; ")} } diff --git a/internal/apply/refused_directory_test.go b/internal/apply/refused_directory_test.go index 997cf288..f6ae9d0e 100644 --- a/internal/apply/refused_directory_test.go +++ b/internal/apply/refused_directory_test.go @@ -118,10 +118,10 @@ func TestAFileIsJudgedByTheSameGuardAsADirectory(t *testing.T) { if _, statErr := os.Stat(filepath.Join(home, "data", "notes.conf")); statErr == nil { t.Error("a file was written below a directory the engine refused") } - // A refused file has no outcome: it is a failure. Declared first, it is the failure the apply reports, - // and the directory the other. + // A refused file has no outcome: it is a failure. Declared first, it is the failure the apply reports, and the + // directory after it is left unapplied, as the rest of its module is (design 42 zfs-owns-zfs §1). var applyErr *Error - if !errors.As(err, &applyErr) || applyErr.Resource != "notes.conf" || applyErr.Others != 1 { + if !errors.As(err, &applyErr) || applyErr.Resource != "notes.conf" || applyErr.Others != 0 { t.Errorf("the file below a refused directory is not among the failures: %v", err) } if o := outcomeOf(report, "notes.conf"); o.ID != "" { diff --git a/internal/compose/resolution/config_lines.go b/internal/compose/resolution/config_lines.go new file mode 100644 index 00000000..b6dc64e7 --- /dev/null +++ b/internal/compose/resolution/config_lines.go @@ -0,0 +1,116 @@ +package resolution + +import ( + "encoding/json" + "fmt" + "math" + "regexp" + "slices" + "strconv" + "strings" + + mcat "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// configLinesRef is `${config-lines::}` and `${config-lines:::mib-bytes}` in a file's +// content (design 42 zfs-owns-zfs §3, ADR 0679 §16 (c)): one line per entry of a list value, ``. The +// prefix is literal text with no `}`, `$` or newline. A placeholder that holds a whole line takes its newline with it. +var configLinesRef = regexp.MustCompile(`\$\{config-lines:([a-z0-9][a-z0-9_.-]*):([^}$\n]*?)(:mib-bytes)?\}(\n?)`) + +// configJSONRef is `${config-json}` in a file's content (design 42 zfs-owns-zfs §9, answer A-15): the module's set +// config values as one JSON object. +var configJSONRef = regexp.MustCompile(`\$\{config-json\}`) + +// configLinesInto writes each `${config-lines:…}` of a file's content from the layers: a line `` for each +// entry of a list, one for a set scalar, nothing for an unset value or an empty list. With `mib-bytes` an entry given in +// MiB is written in bytes. An entry holding a newline is refused, and the content is left as it was. +func configLinesInto(content string, layers []mcat.Layer, module string) (string, error) { + var refused error + filled := configLinesRef.ReplaceAllStringFunc(content, func(token string) string { + m := configLinesRef.FindStringSubmatch(token) + key, prefix, mib, newline := m[1], m[2], m[3] != "", m[4] + value, set := settingValue(layers, key) + if !set { + return "" + } + entries, isList := value.([]any) + if !isList { + entries = []any{value} + } + var lines []string + for _, e := range entries { + text := plainly(e) + if strings.ContainsAny(text, "\r\n") { + refused = fmt.Errorf("%s: an entry of the config value %q holds a newline, which would begin a line "+ + "of its own in the file it is written into (design 42 §3)", module, key) + return token + } + if mib { + n, err := mibWhole(text) + if err != nil || n > 1<<44 { + refused = fmt.Errorf("%s: the config value %q holds %q, and mib-bytes writes a whole number of MiB "+ + "in bytes (design 42 §3)", module, key, text) + return token + } + text = strconv.FormatUint(n<<20, 10) + } + lines = append(lines, prefix+text) + } + if len(lines) == 0 { + return "" + } + return strings.Join(lines, "\n") + newline + }) + if refused != nil { + return content, refused + } + return filled, nil +} + +// configJSONInto writes `${config-json}` in a file's content as the module's set config values in one JSON object +// (design 42 zfs-owns-zfs §9, answer A-15): only keys the module declares under `config` are written, and what no layer sets is absent, and no unset value refuses the module; a +// list and an object are written as JSON, a number, a boolean and a text as JSON values of their own type. +func configJSONInto(content string, layers []mcat.Layer, declared []string) (string, error) { + if !configJSONRef.MatchString(content) { + return content, nil + } + set := map[string]any{} + for _, layer := range layers { + for k, v := range layer.Values { + if slices.Contains(declared, k) { + set[k] = v + } + } + } + var out strings.Builder + enc := json.NewEncoder(&out) + enc.SetEscapeHTML(false) + enc.SetIndent("", " ") + if err := enc.Encode(set); err != nil { + return content, fmt.Errorf("the config values cannot be written as JSON: %w", err) + } + object := strings.TrimSuffix(out.String(), "\n") + return configJSONRef.ReplaceAllLiteralString(content, object), nil +} + +// mibWhole is a size in MiB as a whole number: digits, or a whole number a program wrote in exponent form (`1e+06`), +// which is how a float of a million or more prints (design 42 §3). +func mibWhole(text string) (uint64, error) { + if n, err := strconv.ParseUint(text, 10, 64); err == nil { + return n, nil + } + f, err := strconv.ParseFloat(text, 64) + if err != nil || f < 0 || f != math.Trunc(f) || f > 1<<44 { + return 0, fmt.Errorf("not a whole number of MiB") + } + return uint64(f), nil +} + +// declaredConfig is the keys a module's manifest declares under `config`. +func declaredConfig(m mcat.Manifest) []string { + keys := make([]string, 0, len(m.Settings)) + for k := range m.Settings { + keys = append(keys, k) + } + return keys +} diff --git a/internal/compose/resolution/declaration.go b/internal/compose/resolution/declaration.go index b3f5672c..faa2202d 100644 --- a/internal/compose/resolution/declaration.go +++ b/internal/compose/resolution/declaration.go @@ -659,6 +659,9 @@ func resolutionCompose(r mcat.Resolution, with mcat.Rendering, owner map[string] // parses strictly and knows no such field. absent := AbsentFor(r, m.Module) resources = withoutAbsent(resources, absent) + // **A resource declared only while a config value holds** (design 42 zfs-owns-zfs §2): left out while none of + // its `if-config` conditions holds, and the key dropped from the rest. + resources = withoutUnconfigured(resources, WithDefaults(m, with.Settings[m.Module])) // No container is given the mesh's names (novox/hq ADR 0148). It used to be: every // container got the whole roster as `--add-host` entries at creation, and a name that @@ -906,7 +909,7 @@ func resolutionCompose(r mcat.Resolution, with mcat.Rendering, owner map[string] // **An operator's value, from the assignment** (novox/hq ADR 0112, ADR 0155): what a // definition may not carry because it is true of one installation only. Filled from // the same layers a mergeable file takes, and refused when no layer set it. - if err := unsetInConfigWords(settingInto(copied, WithDefaults(m, with.Settings[m.Module]), m.Module), m); err != nil { + if err := unsetInConfigWords(settingInto(copied, WithDefaults(m, with.Settings[m.Module]), m.Module, declaredConfig(m)...), m); err != nil { return nil, err } // **Placed before anything reads a path.** A pathless directory receives the path diff --git a/internal/compose/resolution/if_config.go b/internal/compose/resolution/if_config.go new file mode 100644 index 00000000..1c1c5db9 --- /dev/null +++ b/internal/compose/resolution/if_config.go @@ -0,0 +1,80 @@ +package resolution + +import ( + "fmt" + "strings" + + mcat "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// withoutUnconfigured drops each resource whose `if-config` holds none of its conditions against the module's config +// values (design 42 zfs-owns-zfs §2, ADR 0679 §16 (b)), and the key itself from every resource that keeps it: the +// node-engine parses strictly and knows no such key. What leaves the declaration this way is removed as any resource +// that leaves it (ADR 0269 §2), by the node-engine's own orphan removal. +func withoutUnconfigured(resources []map[string]any, layers []mcat.Layer) []map[string]any { + out := make([]map[string]any, 0, len(resources)) + for _, r := range resources { + conditions, has := r[mcat.IfConfigField] + if !has { + out = append(out, r) + continue + } + if !ifConfigHolds(conditions, layers) { + continue + } + kept := make(map[string]any, len(r)) + for k, v := range r { + if k != mcat.IfConfigField { + kept[k] = v + } + } + out = append(out, kept) + } + return out +} + +// ifConfigHolds is whether any condition of a list holds: `` when the config value is set and is neither false, +// off, none, empty nor an empty list; `=` when the value equals it, or a list value holds it as an entry. +func ifConfigHolds(conditions any, layers []mcat.Layer) bool { + list, _ := conditions.([]any) + for _, c := range list { + key, want, equals := strings.Cut(fmt.Sprint(c), "=") + value, set := settingValue(layers, key) + if !set { + continue + } + if !equals { + if !isOff(value) { + return true + } + continue + } + if entries, isList := value.([]any); isList { + for _, e := range entries { + if plainly(e) == want { + return true + } + } + } else if plainly(value) == want { + return true + } + } + return false +} + +// isOff is a config value that says "not here": false, off, none, an empty text or an empty list. +func isOff(value any) bool { + switch v := value.(type) { + case bool: + return !v + case []any: + return len(v) == 0 + case nil: + return true + } + switch plainly(value) { + case "", "false", "off", "none": + return true + } + return false +} diff --git a/internal/compose/resolution/setting_into.go b/internal/compose/resolution/setting_into.go index df0abe2e..984621bc 100644 --- a/internal/compose/resolution/setting_into.go +++ b/internal/compose/resolution/setting_into.go @@ -65,8 +65,9 @@ func unsetInConfigWords(err error, m mcat.Manifest) error { // The last layer setting a key wins, which is the node's over the mesh's over the module's own // default (novox/hq ADR 0262) — the same order settle applies to a mergeable file. The caller lays // the defaults under the layers with WithDefaults. A value that is not a string is written the way a program would read -// it (a number without a trailing .000000, a boolean as true/false). -func settingInto(resource map[string]any, layers []mcat.Layer, module string) error { +// it (a number without a trailing .000000, a boolean as true/false). declared are the keys the module's manifest +// declares under `config`, the only ones `${config-json}` writes (design 42 §9, answer A-15). +func settingInto(resource map[string]any, layers []mcat.Layer, module string, declared ...string) error { if fmt.Sprint(resource["type"]) == "service" { return settingIntoUnit(resource, layers, module) } @@ -83,6 +84,13 @@ func settingInto(resource map[string]any, layers []mcat.Layer, module string) er if !ok { return nil } + content, err := configLinesInto(content, layers, module) + if err != nil { + return err + } + if content, err = configJSONInto(content, layers, declared); err != nil { + return err + } for _, key := range settingsUsed(content) { value, set := settingValue(layers, key) if !set { diff --git a/internal/compose/resolution/setting_into_test.go b/internal/compose/resolution/setting_into_test.go index 21d76477..9b7dd551 100644 --- a/internal/compose/resolution/setting_into_test.go +++ b/internal/compose/resolution/setting_into_test.go @@ -1,42 +1,151 @@ package resolution import ( + "errors" + "fmt" "strings" "testing" mcat "git.novox.be/novox/mesh-sdk/go/catalogue" ) -// An operator's value reaches a file from the assignment's settings, the node's layer over the -// mesh's (novox/hq ADR 0112, ADR 0155), and a value nothing set is refused by name. -func TestASettingReachesAFileFromTheLayers(t *testing.T) { - file := map[string]any{"id": "env", "type": "file", "path": "/x/mail.env", - "content": "DOMAIN=${setting:domain}\nSITENAME=${setting:sitename}\nWORKERS=${setting:workers}\n"} - layers := []mcat.Layer{ - {From: "the mesh", Values: map[string]any{"domain": "example.tld", "sitename": "Mesh", "workers": float64(4)}}, - {From: "this node", Values: map[string]any{"sitename": "This one"}}, +// A config value reaches a resource from the assignment's layers, the node's over the mesh's (novox/hq ADR 0112, +// ADR 0155), and a value nothing set is refused by name; and a resource under `if-config` is declared only while a +// config value holds (design 42 zfs-owns-zfs §2, ADR 0679 §16 (b)). One table of the two, each row its layers, what it +// does to a resource and what comes of it. +func TestAConfigValueReachesAResourceAndDecidesWhetherItIsDeclared(t *testing.T) { + filled := func(content string) func([]mcat.Layer) (string, error) { + return func(layers []mcat.Layer) (string, error) { + file := map[string]any{"id": "env", "type": "file", "path": "/x/mail.env", "content": content} + err := settingInto(file, layers, "mail", "n", "b", "s", "l", "o") + return file["content"].(string), err + } } - if err := settingInto(file, layers, "mail"); err != nil { - t.Fatal(err) + // declared is the ids of what is left of a resource under each condition, and whether the key is gone from it. + declared := func(conditions ...any) func([]mcat.Layer) (string, error) { + return func(layers []mcat.Layer) (string, error) { + kept := withoutUnconfigured([]map[string]any{ + {"id": "plain", "type": "file"}, {"id": "cond", "type": "file", mcat.IfConfigField: conditions}}, layers) + var ids []string + for _, r := range kept { + if _, has := r[mcat.IfConfigField]; has { + return "", errKeyKept + } + ids = append(ids, r["id"].(string)) + } + return strings.Join(ids, ","), nil + } } - if file["content"] != "DOMAIN=example.tld\nSITENAME=This one\nWORKERS=4\n" { - t.Fatalf("filled as %q", file["content"]) + values := func(kv map[string]any) []mcat.Layer { return []mcat.Layer{{From: "the mesh", Values: kv}} } + for _, row := range []struct { + name string + layers []mcat.Layer + do func([]mcat.Layer) (string, error) + want string // what comes of it + errs []string // or what the refusal says + }{ + {"the node's layer over the mesh's", []mcat.Layer{ + {From: "the mesh", Values: map[string]any{"domain": "example.tld", "sitename": "Mesh", "workers": float64(4)}}, + {From: "this node", Values: map[string]any{"sitename": "This one"}}}, + filled("DOMAIN=${setting:domain}\nSITENAME=${setting:sitename}\nWORKERS=${setting:workers}\n"), + "DOMAIN=example.tld\nSITENAME=This one\nWORKERS=4\n", nil}, + {"a value nothing set is refused, the file left as it was", values(map[string]any{"other": "x"}), + filled("DOMAIN=${setting:domain}\n"), "DOMAIN=${setting:domain}\n", + []string{"${setting:domain}", "settings set mail", "set today: other"}}, + {"config-lines: a list of two writes two lines in order", values(map[string]any{"k": []any{"a", "b"}}), + filled("h\n${config-lines:k:options zfs x=}\nt\n"), "h\noptions zfs x=a\noptions zfs x=b\nt\n", nil}, + {"config-lines: mib-bytes writes a million MiB as a whole number", values(map[string]any{"k": float64(1000000)}), + filled("${config-lines:k:zfs_arc_max=:mib-bytes}\n"), "zfs_arc_max=1048576000000\n", nil}, + {"config-lines: mib-bytes writes a million MiB given as text as a whole number", values(map[string]any{"k": "1000000"}), + filled("${config-lines:k:zfs_arc_max=:mib-bytes}\n"), "zfs_arc_max=1048576000000\n", nil}, + {"config-lines: mib-bytes writes a size printed in exponent form as a whole number", values(map[string]any{"k": "1e+06"}), + filled("${config-lines:k:zfs_arc_max=:mib-bytes}\n"), "zfs_arc_max=1048576000000\n", nil}, + {"config-lines: mib-bytes refuses a fraction", values(map[string]any{"k": "1.5"}), + filled("${config-lines:k:zfs_arc_max=:mib-bytes}\n"), "${config-lines:k:zfs_arc_max=:mib-bytes}\n", []string{"mib-bytes"}}, + {"config-lines: an empty list writes nothing", values(map[string]any{"k": []any{}}), + filled("h\n${config-lines:k:p=}\nt\n"), "h\nt\n", nil}, + {"config-lines: an unset value writes nothing", nil, filled("h\n${config-lines:k:p=}\nt\n"), "h\nt\n", nil}, + {"config-lines: a set scalar writes one line", values(map[string]any{"k": "v"}), + filled("${config-lines:k:p=}\n"), "p=v\n", nil}, + {"config-lines: mib-bytes writes MiB in bytes", values(map[string]any{"k": float64(64)}), + filled("${config-lines:k:zfs_arc_max=:mib-bytes}\n"), "zfs_arc_max=67108864\n", nil}, + {"config-lines: an entry with a newline is refused, the file left as it was", values(map[string]any{"k": []any{"a\nb"}}), + filled("${config-lines:k:p=}\n"), "${config-lines:k:p=}\n", []string{"newline"}}, + {"config-json: the set values only, each as its own type", values(map[string]any{"n": float64(64), "b": true, "s": "x", + "l": []any{"a"}, "o": map[string]any{"f": float64(1)}}), filled("${config-json}\n"), + "{\n \"b\": true,\n \"l\": [\n \"a\"\n ],\n \"n\": 64,\n \"o\": {\n \"f\": 1\n },\n \"s\": \"x\"\n}\n", nil}, + {"config-json: a key the module does not declare is not written", values(map[string]any{"s": "x", "scrub-pool": "tank"}), + filled("${config-json}\n"), "{\n \"s\": \"x\"\n}\n", nil}, + {"config-json: nothing set is an empty object, not a refusal", nil, filled("${config-json}\n"), "{}\n", nil}, + {"if-config: a key never set leaves it out", nil, declared("k"), "plain", nil}, + {"if-config: a key set to text declares it", values(map[string]any{"k": "x"}), declared("k"), "plain,cond", nil}, + {"if-config: a key set to false, off, none, empty or an empty list leaves it out", values(map[string]any{"k": false}), + declared("k"), "plain", nil}, + {"if-config: off", values(map[string]any{"k": "off"}), declared("k"), "plain", nil}, + {"if-config: none", values(map[string]any{"k": "none"}), declared("k"), "plain", nil}, + {"if-config: empty text", values(map[string]any{"k": ""}), declared("k"), "plain", nil}, + {"if-config: an empty list", values(map[string]any{"k": []any{}}), declared("k"), "plain", nil}, + {"if-config: a list of entries", values(map[string]any{"k": []any{"a"}}), declared("k"), "plain,cond", nil}, + {"if-config: a value equal to the condition", values(map[string]any{"k": "on"}), declared("k=on"), "plain,cond", nil}, + {"if-config: a value other than the condition", values(map[string]any{"k": "off"}), declared("k=on"), "plain", nil}, + {"if-config: a list holding the entry", values(map[string]any{"k": []any{"a", "b"}}), declared("k=b"), "plain,cond", nil}, + {"if-config: a list without the entry", values(map[string]any{"k": []any{"a"}}), declared("k=b"), "plain", nil}, + {"if-config: any of the conditions", values(map[string]any{"j": "x"}), declared("k", "j"), "plain,cond", nil}, + {"if-config: a value from a default layer", []mcat.Layer{{Default: true, Values: map[string]any{"k": "on"}}}, + declared("k=on"), "plain,cond", nil}, + {"if-config: a number equal to the condition", values(map[string]any{"k": float64(64)}), declared("k=64"), "plain,cond", nil}, + } { + t.Run(row.name, func(t *testing.T) { + got, err := row.do(row.layers) + if len(row.errs) == 0 && err != nil { + t.Fatal(err) + } + if len(row.errs) > 0 && err == nil { + t.Fatalf("a refusal was expected, got %q", got) + } + for _, want := range row.errs { + if !strings.Contains(err.Error(), want) { + t.Fatalf("the refusal lacks %q: %v", want, err) + } + } + if got != row.want { + t.Fatalf("got %q, want %q", got, row.want) + } + }) } } -func TestASettingNothingSetIsRefusedByName(t *testing.T) { - file := map[string]any{"id": "env", "type": "file", "content": "DOMAIN=${setting:domain}\n"} - err := settingInto(file, []mcat.Layer{{From: "the mesh", Values: map[string]any{"other": "x"}}}, "mail") - if err == nil { - t.Fatal("a setting nothing set was written as something") - } - for _, want := range []string{"${setting:domain}", "settings set mail", "set today: other"} { - if !strings.Contains(err.Error(), want) { - t.Fatalf("the refusal lacks %q: %v", want, err) +// errKeyKept is what a row of the table above says when `if-config` reached the node-engine, which parses strictly. +var errKeyKept = errors.New("the if-config key was kept in a declared resource") + +// The composition takes a resource out while its condition does not hold, and a value going from set to unset takes it +// out of the next composition, which the node-engine removes as it removes any resource that left (design 42 §2). +func TestAConfigConditionedResourceIsComposedWhileItsValueHolds(t *testing.T) { + m := mustParse(t, `{"module":"zfs","version":"1","config":{"zed":{"type":"string","default":"off"},"mods":{"type":"list"}},"resources":[ + {"id":"opts","type":"file","path":"/etc/modprobe.d/zfs.conf","content":"${config-lines:mods:options zfs }\n${config-json}\n"}, + {"id":"conf","type":"file","path":"/etc/zfs/mesh.conf","content":"x\n"}, + {"id":"zed","type":"file","path":"/etc/zfs/zed.rc","content":"y\n","if-config":["zed=on"]}]}`) + for _, row := range []struct { + layers []mcat.Layer + zed bool + }{{nil, false}, {[]mcat.Layer{{From: "this node", Values: map[string]any{"zed": "on", "mods": []any{"a=1"}}}}, true}} { + out, err := Declaration(mcat.Resolution{Node: "workstation", Modules: []mcat.Manifest{m}}, + mcat.Rendering{Settings: mcat.SettingsBy{"zfs": row.layers}}) + if err != nil { + t.Fatal(err) + } + if fileNamed(out, "zfs.conf") == nil { + t.Error("the unconditioned file was left out") + } + if opts := fileNamed(out, "zfs.opts"); opts == nil || (row.zed != strings.HasPrefix(fmt.Sprint(opts["content"]), "options zfs a=1\n")) { + t.Errorf("the template's lines do not follow the list: %v", opts) + } + zed := fileNamed(out, "zfs.zed") + if (zed != nil) != row.zed { + t.Errorf("with %v the conditioned file is declared: %v, want %v", row.layers, zed != nil, row.zed) + } + if _, has := zed[mcat.IfConfigField]; has { + t.Errorf("%s reached the node-engine, which parses strictly", mcat.IfConfigField) } } - // Left as it was: the literal placeholder must never reach a machine. - if file["content"] != "DOMAIN=${setting:domain}\n" { - t.Fatalf("content was changed on refusal: %q", file["content"]) - } } diff --git a/internal/compose/resolution/settings.go b/internal/compose/resolution/settings.go index 71c1864f..ccc3037a 100644 --- a/internal/compose/resolution/settings.go +++ b/internal/compose/resolution/settings.go @@ -458,7 +458,7 @@ func JudgeSettings(m mcat.Manifest, layers []mcat.Layer, adopted bool) error { for k, v := range settled { copied[k] = v } - if err := unsetInConfigWords(settingInto(copied, WithDefaults(m, layers), m.Module), m); err != nil { + if err := unsetInConfigWords(settingInto(copied, WithDefaults(m, layers), m.Module, declaredConfig(m)...), m); err != nil { return err } } diff --git a/internal/compose/resolution/unconsumed_placeholder.go b/internal/compose/resolution/unconsumed_placeholder.go index b83fe4cc..b46d65b5 100644 --- a/internal/compose/resolution/unconsumed_placeholder.go +++ b/internal/compose/resolution/unconsumed_placeholder.go @@ -25,6 +25,10 @@ type filler struct { // or the sweep refuses the placeholder it would have filled. var fillers = []filler{ {namespace: "setting", patterns: []*regexp.Regexp{settingRef}, fields: map[string][]string{"file": {"content"}, "service": {"unit"}, "directory": {"owner"}, "container": {"groups"}}}, + // A file's content only: lines from a config value's entries, and the module's config as one JSON object + // (design 42 zfs-owns-zfs §3, §9; config_lines.go). + {namespace: "config-lines", patterns: []*regexp.Regexp{configLinesRef}, fields: map[string][]string{"file": {"content"}}}, + {namespace: "config-json", patterns: []*regexp.Regexp{configJSONRef}, fields: map[string][]string{"file": {"content"}}}, {namespace: "dir", patterns: []*regexp.Regexp{dirRef}, fields: map[string][]string{"*": {"path", "content", "volumes", "env", "env-file"}}}, {namespace: "access", patterns: []*regexp.Regexp{accessRef}, fields: map[string][]string{"*": {"path", "content", "volumes", "env", "env-file"}}}, {namespace: "secret", patterns: []*regexp.Regexp{placeholder}, fields: map[string][]string{"file": {"content"}}, diff --git a/internal/link/have_test.go b/internal/link/have_test.go index ed29f89c..7c1ec527 100644 --- a/internal/link/have_test.go +++ b/internal/link/have_test.go @@ -376,8 +376,12 @@ func TestTheNodeKeyCarriesTheNodesFacts(t *testing.T) { t.Errorf("record = %+v, want %+v", *got.Record, wantRecord) } // Among them, that it fills a seat's reach from the seat holder's listen (ADR 0632 §1). - if !slices.Contains(got.Record.ManifestReads, catalogue.ReadForm+catalogue.ReferenceFormSeatReach) { - t.Errorf("the engine does not state it reads a seat's reach from its holder: %v", got.Record.ManifestReads) + // And the words and forms it builds (design 42 zfs-owns-zfs §4, answer A-13). + for _, read := range []string{catalogue.ReadForm + catalogue.ReferenceFormSeatReach, "word:resources.if-config", + "form:config-lines", "form:config-json"} { + if !slices.Contains(got.Record.ManifestReads, read) { + t.Errorf("the engine does not state it reads %s: %v", read, got.Record.ManifestReads) + } } at := l.kv["n1.node"].rev q.SayFacts(context.Background(), rep)