From 2fd5ffdd6263db7577cd222010c1cad106bff337 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 27 Oct 2026 23:17:05 +0100 Subject: [PATCH 1/9] A node's answer names every module it cannot compose, a folded module of an old-form build is left to the node as it runs, and no store reached says why (issue 1126, the seed's shadow) The seed's shadow answers named the first refusal only (ace: baserow's archive, novox: de-spiegel), so each round showed one more. A build that cannot be read or taken in, and a module the composition refuses, are now each composed as a stub and named. A module folded into the node's own composition whose manifest names a build (de-spiegel, nats, photos, systemd-resolved: haves naming a build-... id) runs a build only the controller holds: it is composed as a stub and its resources are carried as the node runs them. --- go/compose/composed.go | 126 +++++++++++++++++++++++++ go/compose/frombus/answer.go | 105 ++++++++++++++++++++- go/compose/frombus/answer_test.go | 152 ++++++++++++++++++++++++++++++ go/compose/frombus/frombus.go | 102 +++++++++++++++++--- go/compose/input.go | 6 ++ 5 files changed, 476 insertions(+), 15 deletions(-) create mode 100644 go/compose/frombus/answer_test.go diff --git a/go/compose/composed.go b/go/compose/composed.go index 2c6313c..5975c25 100644 --- a/go/compose/composed.go +++ b/go/compose/composed.go @@ -8,6 +8,7 @@ import ( "sort" "strings" + "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/declaration" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/overlay" @@ -76,6 +77,42 @@ func compositions(in Input, node string, fps map[string]string, mode Mode, appli if !found { return nil, nil, nil, fmt.Errorf("compose: the mesh has no node %q", node) } + // A folded module built by a build of the old form (Node.RunsOwn) is left to the node: composed as a stub, so what + // it provides and requires still shapes the rest, and its resources are then the ones the node runs. + carried := map[string]bool{} + var runsOwn *delivery.Composed + for i := range reading.Nodes { + n := &reading.Nodes[i] + if n.Name != node || n.RunsOwn == nil { + continue + } + runsOwn = n.RunsOwn + for m := range inOwn { + if !folds(m) { + continue + } + man, ok := n.Kept[m] + if !ok { + man, ok = reading.Catalogue[m] + } + if !ok || !NamesBuild(man) { + continue + } + carried[m] = true + kept := make(map[string]catalogue.Manifest, len(n.Kept)+1) + for k, v := range n.Kept { + kept[k] = v + } + kept[m] = Stub(man) + n.Kept = kept + } + } + isCarried := func(module, id string) bool { + if module == "" { + module, _ = declaration.ModuleGuess(id) + } + return carried[module] + } d, problems, err := ComposeIn(reading, node, mode) if err != nil { return nil, nil, nil, err @@ -118,6 +155,7 @@ func compositions(in Input, node string, fps map[string]string, mode Mode, appli left := map[string]bool{} for _, raw := range envelope.Resources { var head struct { + ID string `json:"id"` Module string `json:"module"` Type string `json:"type"` Path string `json:"path"` @@ -125,6 +163,9 @@ func compositions(in Input, node string, fps map[string]string, mode Mode, appli if err := json.Unmarshal(raw, &head); err != nil { return nil, nil, nil, err } + if isCarried(head.Module, head.ID) { + continue // what the stub composed of it: the node's own carries what runs of it instead + } if folds(head.Module) { // Into the node's own, in its place among the node's resources, its module taken off (mesh-controller's // compositionsOf): the node's own composition is then the controller's, byte for byte. @@ -151,6 +192,18 @@ func compositions(in Input, node string, fps map[string]string, mode Mode, appli } kept = append(kept, raw) } + // What runs of each carried module, in the node's own composition, each after the resource it runs after. + if len(carried) > 0 { + var err error + if kept, foldedPaths, err = carry(kept, foldedPaths, runsOwn, isCarried); err != nil { + return nil, nil, nil, err + } + for _, p := range runsOwn.Filled { + if foldedPaths[p] && !slices.Contains(envelope.Filled, p) { + envelope.Filled = append(envelope.Filled, p) + } + } + } // A filled file of a module's own composition goes with that module (SplitComposed); one of a folded module goes with // the node's own, after them, in the declaration's order; one of a module left out is dropped. var filled, ownFilled []string @@ -188,3 +241,76 @@ func compositions(in Input, node string, fps map[string]string, mode Mode, appli sort.Strings(skipped) return out, skipped, stood, nil } + +// NamesBuild says a manifest names what a build makes: its build, or a resource naming an artifact. A node cannot +// compose such a module folded into its own composition: the build of the old form it runs is held by the controller +// alone (issue 1126). +func NamesBuild(m catalogue.Manifest) bool { + if m.Build != nil { + return true + } + for _, r := range m.Resources { + if _, ok := r["artifact"]; ok { + return true + } + } + return false +} + +// Stub is m with nothing to build and no resource: what it provides, requires, serves and grants still shape the rest of +// the composition, and nothing of its own is composed (issue 1126). +func Stub(m catalogue.Manifest) catalogue.Manifest { + m.Build = nil + m.Resources = nil + return m +} + +// carry puts into the node's resources (kept, in the declaration's order) the resources of the node's own composition as +// it runs (runs) that belong to a carried module, each after the resource it runs after, or first when that one is +// not among them; a file among them is a folded path. +func carry(kept []json.RawMessage, foldedPaths map[string]bool, runs *delivery.Composed, carried func(module, id string) bool) ([]json.RawMessage, map[string]bool, error) { + idOf := func(raw json.RawMessage) (string, string, string, string, error) { + var head struct { + ID string `json:"id"` + Module string `json:"module"` + Type string `json:"type"` + Path string `json:"path"` + } + err := json.Unmarshal(raw, &head) + return head.ID, head.Module, head.Type, head.Path, err + } + present := map[string]bool{} + for _, raw := range kept { + if id, module, _, _, err := idOf(raw); err == nil && module == "" { + present[id] = true + } + } + // By the id of the resource of the node's own it runs after, "" for first: the last one before it that is composed. + after := map[string][]json.RawMessage{} + anchor := "" + for _, raw := range runs.Resources { + id, module, typ, path, err := idOf(raw) + if err != nil { + return nil, nil, fmt.Errorf("compose: a resource the node runs in its own composition: %w", err) + } + if !carried(module, id) { + if module == "" && present[id] { + anchor = id + } + continue + } + after[anchor] = append(after[anchor], raw) + if typ == string(declaration.TypeFile) { + foldedPaths[path] = true + } + } + out := make([]json.RawMessage, 0, len(kept)+len(runs.Resources)) + out = append(out, after[""]...) + for _, raw := range kept { + out = append(out, raw) + if id, module, _, _, err := idOf(raw); err == nil && module == "" { + out = append(out, after[id]...) + } + } + return out, foldedPaths, nil +} diff --git a/go/compose/frombus/answer.go b/go/compose/frombus/answer.go index 20115be..baa549e 100644 --- a/go/compose/frombus/answer.go +++ b/go/compose/frombus/answer.go @@ -4,9 +4,11 @@ import ( "context" "fmt" "regexp" + "slices" "sort" "strings" + "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/compose" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -32,7 +34,13 @@ func Answer(ctx context.Context, r Reader, mesh Mesh, s Scope, fps map[string]st if err != nil { return refuse(fmt.Errorf("frombus: placement: %w", err)) } - in, err := Input(ctx, r, mesh, s) + if s.Running == nil { + s.Running = running + } + // Every module it cannot compose is named, not the first only (issue 1126): a build it cannot read or take in, and + // below, a module the composition refuses, each composed as a stub so the rest is composed and compared. + failed := map[string]string{} + in, err := input(ctx, r, mesh, s, func(module string, err error) { failed[module] = err.Error() }) if err != nil { return refuse(err) } @@ -63,8 +71,20 @@ func Answer(ctx context.Context, r Reader, mesh Mesh, s Scope, fps map[string]st } } composed, skipped, stood, err := compose.CompositionsShadow(in, s.Node, fps) - if err != nil { - return refuse(err) + for err != nil { + module := culprit(in, s.Node, fps, failed, err) + if module == "" { + // Not one module's: the node's own (a value served elsewhere, the mesh's network), said once as the node's. + if len(failed) == 0 { + return refuse(err) + } + failed[delivery.HaveNodeWord] = err.Error() + composed, skipped, stood = nil, nil, nil + break + } + failed[module] = err.Error() + in = stubbed(in, s.Node, module) + composed, skipped, stood, err = compose.CompositionsShadow(in, s.Node, fps) } // The node's own composition is judged only where the node runs one: one that runs none applies it only once a want // names it (the node agent's own composer hands it on that want alone), so a candidate does not add it. @@ -79,8 +99,22 @@ func Answer(ctx context.Context, r Reader, mesh Mesh, s Scope, fps map[string]st } } } + if _, whole := failed[delivery.HaveNodeWord]; whole { + // Nothing composed to compare: every refusal is said, and nothing else. + for _, m := range sortedKeys(failed) { + a.Refusals = append(a.Refusals, delivery.ModuleRefused{Module: m, Why: failed[m]}) + } + a.Verdict = delivery.VerdictOf(a) + return a + } cmp := delivery.CompareComposed(composed, running) - a.Same, a.Differs, a.Removed = cmp.Same, cmp.Differs, cmp.Removed + // A refused module was composed as a stub: what it composed is no difference; its refusal is said. + notFailed := func(m string) bool { _, ok := failed[m]; return !ok } + a.Same = slices.DeleteFunc(cmp.Same, func(m string) bool { return !notFailed(m) }) + a.Removed = slices.DeleteFunc(cmp.Removed, func(m string) bool { return !notFailed(m) }) + a.Differs = slices.DeleteFunc(cmp.Differs, func(d delivery.ModuleDiffers) bool { return !notFailed(d.Module) }) + cmp.Added = slices.DeleteFunc(cmp.Added, func(d delivery.ModuleAdded) bool { return !notFailed(d.Module) }) + skipped = slices.DeleteFunc(skipped, func(m string) bool { return !notFailed(m) }) // What the group adds, by node and module: placed by a candidate and not by the trunk on that node. added := map[string]bool{} @@ -138,8 +172,14 @@ func Answer(ctx context.Context, r Reader, mesh Mesh, s Scope, fps map[string]st if _, isAdded := addedHere[m]; isAdded { continue // a module the candidate adds here: what it lacks is said under Added } + if _, isFailed := failed[m]; isFailed { + continue // said below, as why it could not be composed + } a.Refusals = append(a.Refusals, delivery.ModuleRefused{Module: m, Why: "stands in for " + strings.Join(refusals[m], ", ")}) } + for _, m := range sortedKeys(failed) { + a.Refusals = append(a.Refusals, delivery.ModuleRefused{Module: m, Why: failed[m]}) + } sort.Slice(a.Refusals, func(i, j int) bool { return a.Refusals[i].Module < a.Refusals[j].Module }) a.Verdict = delivery.VerdictOf(a) return a @@ -170,3 +210,60 @@ func standsInFor(x, node string) (string, string) { } return node, "" } + +// culprit is the module placed on node whose composition the error is of: the first, in placement order, whose stub +// (compose.Stub) makes the composition succeed or fail otherwise; empty when none does (the error is not one module's). +// Modules already refused are passed over. +func culprit(in compose.Input, node string, fps map[string]string, failed map[string]string, err error) string { + var assigned []string + for _, n := range in.Nodes { + if n.Name == node { + assigned = n.Assigned + } + } + // The modules the error names first: the words of a module's refusal open with its name. + order := make([]string, 0, len(assigned)) + for _, m := range assigned { + if strings.Contains(err.Error(), m) { + order = append(order, m) + } + } + for _, m := range assigned { + if !strings.Contains(err.Error(), m) { + order = append(order, m) + } + } + for _, m := range order { + if _, done := failed[m]; done { + continue + } + if _, _, _, again := compose.CompositionsShadow(stubbed(in, node, m), node, fps); again == nil || again.Error() != err.Error() { + return m + } + } + return "" +} + +// stubbed is in with module composed as a stub on node alone (compose.Stub, as node's Kept). +func stubbed(in compose.Input, node, module string) compose.Input { + out := in + out.Nodes = append([]compose.Node(nil), in.Nodes...) + for i := range out.Nodes { + if out.Nodes[i].Name != node { + continue + } + man, ok := out.Nodes[i].Kept[module] + if !ok { + if man, ok = in.Catalogue[module]; !ok { + return in + } + } + kept := make(map[string]catalogue.Manifest, len(out.Nodes[i].Kept)+1) + for k, v := range out.Nodes[i].Kept { + kept[k] = v + } + kept[module] = compose.Stub(man) + out.Nodes[i].Kept = kept + } + return out +} diff --git a/go/compose/frombus/answer_test.go b/go/compose/frombus/answer_test.go new file mode 100644 index 0000000..1f6ab25 --- /dev/null +++ b/go/compose/frombus/answer_test.go @@ -0,0 +1,152 @@ +package frombus + +import ( + "context" + "encoding/json" + "reflect" + "slices" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/compose" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/overlay" +) + +// Issue 1126 (the seed's shadow, round 2): a node's answer names every module it cannot compose, not the first only; +// and a module folded into its own composition that names a build of the old form (de-spiegel on novox, nats, photos, +// systemd-resolved: their haves name a build-... id and no fingerprint) is left to the node, carried as it runs. +func TestTheAnswerNamesEveryRefusalAndLeavesAnOldBuildToTheNode(t *testing.T) { + ctx := context.Background() + image := "alpine@sha256:" + strings.Repeat("a", 64) + manifest := func(module string) []byte { + return []byte(`{"module":"` + module + `","version":"1","capabilities":["container-runtime"],` + + `"resources":[{"id":"state","type":"directory","path":"/var/lib/` + module + `","mode":"0700"},` + + `{"id":"server","type":"container","name":"` + module + `","artifact":"site"}],` + + `"build":{"artifacts":[{"name":"site","kind":"upstream","from":"` + image + `"}]}}`) + } + lemurs := []byte(`{"module":"lemurs","version":"1","resources":[{"id":"service","type":"service","unit":"lemurs.service","state":"running","boot":"enabled"}]}`) + fp := func(c string) string { return "src2:" + strings.Repeat(c, 64) } + mk := func(placed ...string) *bus { + p := delivery.Placement{Node: "n1", Decided: delivery.NodeDecided{Hub: true, Address: "10.77.0.1"}} + for _, m := range placed { + p.Modules = append(p.Modules, delivery.PlacedModule{Module: m}) + } + b := &bus{placements: map[string]delivery.Placement{"n1": p}, + records: map[string]delivery.NodeRecord{"n1": {Node: "n1", TunnelKey: publicKey(t), SealingKey: "sk", Capabilities: []string{"systemd", "container-runtime"}}}, + modules: map[string]delivery.Module{"lemurs": {Manifest: delivery.ManifestDigest(lemurs)}}, + manifests: map[string][]byte{delivery.ManifestDigest(lemurs): lemurs}, + builds: map[string]*delivery.BuilderBuild{}} + for _, m := range []string{"web", "api", "spiegel"} { + raw := manifest(m) + b.manifests[delivery.ManifestDigest(raw)] = raw + b.modules[m] = delivery.Module{Manifest: delivery.ManifestDigest(raw)} + } + // web and api are built, their keys done and waiting for the reference of their image; spiegel has no build of + // the new form at all. + for i, m := range []string{"web", "api"} { + mod := b.modules[m] + mod.FP2 = fp(string(rune('1' + i))) + b.modules[m] = mod + b.builds[m+" "+mod.FP2] = &delivery.BuilderBuild{State: delivery.BuildDone, Manifest: mod.Manifest, + Artifacts: []delivery.BuiltArtifact{{Name: "site", Kind: "image", Digest: "sha256:" + strings.Repeat("b", 64)}}} + } + return b + } + given := func(b *bus) map[string]*delivery.Placement { + p := b.placements["n1"] + return map[string]*delivery.Placement{"n1": &p} + } + + // Every refusal: two builds taken in, and a module the composition refuses (its image never built), each named. + b := mk("lemurs", "web", "api", "spiegel") + a := Answer(ctx, b, Mesh{}, Scope{Node: "n1", Placements: given(b)}, + map[string]string{"lemurs": fp("e"), "web": fp("1"), "api": fp("2"), "spiegel": fp("3")}, nil, "h1", 3) + var refused []string + for _, r := range a.Refusals { + refused = append(refused, r.Module) + } + if a.Verdict != delivery.CandidateRefused || a.Refused != "" || !reflect.DeepEqual(refused, []string{"api", "spiegel", "web"}) { + t.Errorf("every refusal: %+v, want api, spiegel and web refused, each named", a) + } + for _, r := range a.Refusals { + want := map[string]string{"api": "waits for the reference of site", "web": "waits for the reference of site", + "spiegel": "spiegel has not been built for this mesh"}[r.Module] + if !strings.Contains(r.Why, want) { + t.Errorf("%s refused as %q, want %q", r.Module, r.Why, want) + } + } + if raw, err := delivery.EncodeHaveNode(delivery.HaveNode{Candidates: map[string]delivery.CandidateAnswer{"g": a}}); err != nil { + t.Errorf("the answer is not one a have holds: %v (%s)", err, raw) + } + if lemursOK := slices.ContainsFunc(a.Added, func(m delivery.ModuleAdded) bool { return m.Module == "lemurs" }); !lemursOK { + t.Errorf("lemurs, which composes, is not answered beside the refusals: %+v", a) + } + + // The fold: spiegel runs inside the node's own composition at a build of the old form. Its resources are carried as + // they run, each after the one it runs after: the node's own is the one it runs, and the node composes itself. + b = mk("lemurs", overlay.Name, "spiegel") + p := b.placements["n1"] + p.Decided.Endpoint = "n1.example:51820" + b.placements["n1"] = p + fps := map[string]string{"lemurs": fp("e")} + stub := Scope{Node: "n1"} + in, err := Input(ctx, b, Mesh{}, stub) + if err != nil { + t.Fatal(err) + } + in.Nodes[0].Kept = map[string]catalogue.Manifest{"spiegel": compose.Stub(in.Catalogue["spiegel"])} + base, _, err := compose.Compositions(in, "n1", fps, compose.Reading) + if err != nil { + t.Fatal(err) + } + ownKey, _ := delivery.ComposedNodeKey("n1") + own := base[ownKey] + carriedRes := []json.RawMessage{ + json.RawMessage(`{"id":"spiegel.state","type":"directory","path":"/var/lib/spiegel","mode":"0700"}`), + json.RawMessage(`{"id":"spiegel.server","type":"container","name":"spiegel","image":"registry.example/spiegel@sha256:` + strings.Repeat("c", 64) + `"}`)} + // They run after the node's first own resource. + own.Resources = append(append(append([]json.RawMessage{}, own.Resources[:1]...), carriedRes...), own.Resources[1:]...) + var running []delivery.Composed + for k, c := range base { + if k == ownKey { + c = own + } + running = append(running, c) + } + scope := Scope{Node: "n1", InOwn: []string{"spiegel"}, Running: running} + got, _, err := Compose(ctx, b, Mesh{}, scope, fps) + if err != nil { + t.Fatalf("the node does not compose itself with spiegel folded at a build of the old form: %v", err) + } + gotOwn, _ := json.Marshal(got[ownKey].Resources) + wantOwn, _ := json.Marshal(own.Resources) + if string(gotOwn) != string(wantOwn) { + t.Errorf("the node's own composition is\n%s\nwant what runs\n%s", gotOwn, wantOwn) + } + a = Answer(ctx, b, Mesh{}, Scope{Node: "n1", Placements: given(b), InOwn: []string{"spiegel"}}, fps, running, "h1", 3) + if a.Verdict != delivery.CandidateSame { + t.Errorf("the fold's answer: %+v, want the same", a) + } + // Without what runs, the fold composes spiegel from its manifest, as before, and the composition refuses it. + if _, _, err := Compose(ctx, b, Mesh{}, Scope{Node: "n1", InOwn: []string{"spiegel"}}, fps); err == nil || + !strings.Contains(err.Error(), "spiegel has not been built for this mesh") { + t.Errorf("composed spiegel without what runs: %v", err) + } +} + +// A node that reaches no artifact store says what its composition holds of one (issue 1126: ace and g14 answered +// "reaches no artifact store", and nothing said why). +func TestNoStoreReachedSaysWhy(t *testing.T) { + in := compose.Input{Catalogue: map[string]catalogue.Manifest{ + "distribution": {Module: "distribution", Serves: map[string]map[string]any{catalogue.ArtifactStoreProvision: {"port": 5000}}}}, + Nodes: []compose.Node{{Name: "g14", Assigned: []string{"adwaita"}}, {Name: "novox", Assigned: []string{"distribution"}}}} + if got := storeSought(in); got != "the composition holds g14, novox; artifact-store is served by distribution on novox, none of it on the private network with a port" { + t.Errorf("said %q", got) + } + in.Nodes = in.Nodes[:1] + if got := storeSought(in); got != "the composition holds g14; no module placed there serves artifact-store" { + t.Errorf("said %q", got) + } +} diff --git a/go/compose/frombus/frombus.go b/go/compose/frombus/frombus.go index 0c1c28f..7aef315 100644 --- a/go/compose/frombus/frombus.go +++ b/go/compose/frombus/frombus.go @@ -113,12 +113,17 @@ type Mesh struct { // // InOwn is the modules the composing node runs inside its own composition (InOwn of what it runs), composed there again // (compose.Node.InOwn) rather than cut out as modules no build is given for (issue 1126). +// +// Running is what the composing node runs; of it, its own composition (`.node`) is given to the composition +// (compose.Node.RunsOwn), so a module folded there that names a build of the old form is left to the node, carried +// as it runs (issue 1126). Nil composes every folded module from its manifest. type Scope struct { Node string Every bool KeptAt map[string]string Placements map[string]*delivery.Placement InOwn []string + Running []delivery.Composed } // InOwn is, of the modules p places on its node, those whose resources the node's own composition it runs @@ -154,6 +159,13 @@ func InOwn(running []delivery.Composed, p delivery.Placement) []string { // Input is the composition input for s, read through r. func Input(ctx context.Context, r Reader, mesh Mesh, s Scope) (compose.Input, error) { + return input(ctx, r, mesh, s, nil) +} + +// input is Input; with refuse set, a module placed on the composing node whose build cannot be read or taken in is +// handed to refuse and composed as a stub (compose.Stub, kept for that node alone) rather than ending the read, so a +// node's answer names every module it cannot compose (issue 1126). +func input(ctx context.Context, r Reader, mesh Mesh, s Scope, refuse func(module string, err error)) (compose.Input, error) { placements, err := r.Placements(ctx) if err != nil { return compose.Input{}, fmt.Errorf("frombus: placement: %w", err) @@ -280,6 +292,12 @@ func Input(ctx context.Context, r Reader, mesh Mesh, s Scope) (compose.Input, er } n.InOwn[module] = true } + for i := range s.Running { + if s.Running[i].Module == "" { + own := s.Running[i] + n.RunsOwn = &own + } + } } in.Nodes = append(in.Nodes, n) in.Holdings = append(in.Holdings, p.Held()...) @@ -309,7 +327,7 @@ func Input(ctx context.Context, r Reader, mesh Mesh, s Scope) (compose.Input, er if _, ok := records[s.Node]; !ok { return compose.Input{}, fmt.Errorf("frombus: %s has written no record of itself yet", s.Node) } - if err := fromBuilds(ctx, r, &in, s, placements[s.Node], modules, read, parse); err != nil { + if err := fromBuilds(ctx, r, &in, s, placements[s.Node], modules, read, parse, refuse); err != nil { return compose.Input{}, err } // Its own modules' secrets, sealed to its own key by the vault: a composition carries them sealed (ADR 0550 §5). @@ -358,7 +376,29 @@ func Compose(ctx context.Context, r Reader, mesh Mesh, s Scope, fps map[string]s // by the one take-in rule (takein.Manifest), a templates archive read in the artifact store as the node reaches it // (issue 1126). func fromBuilds(ctx context.Context, r Reader, in *compose.Input, s Scope, p delivery.Placement, modules map[string]delivery.Module, - read func(module, digest string) (catalogue.Manifest, error), parse func([]byte) (catalogue.Manifest, error)) error { + read func(module, digest string) (catalogue.Manifest, error), parse func([]byte) (catalogue.Manifest, error), + refuse func(module string, err error)) error { + // stub keeps a refused module in the composition as a stub for this node alone (compose.Stub): what it provides + // and requires still shapes the rest. + stub := func(module string, err error) { + refuse(module, err) + for i := range in.Nodes { + if in.Nodes[i].Name != s.Node { + continue + } + man, ok := in.Nodes[i].Kept[module] + if !ok { + man, ok = in.Catalogue[module] + } + if !ok { + return + } + if in.Nodes[i].Kept == nil { + in.Nodes[i].Kept = map[string]catalogue.Manifest{} + } + in.Nodes[i].Kept[module] = compose.Stub(man) + } + } address, addressErr := "", error(nil) addressed := false archive := func(ctx context.Context, reference, digest string) ([]byte, error) { @@ -370,7 +410,7 @@ func fromBuilds(ctx context.Context, r Reader, in *compose.Input, s Scope, p del case addressErr != nil: return nil, addressErr case address == "": - return nil, fmt.Errorf("%s reaches no artifact store", s.Node) + return nil, fmt.Errorf("%s reaches no artifact store (%s)", s.Node, storeSought(*in)) } return r.Archive(ctx, catalogue.Routed(reference, address), digest) } @@ -409,14 +449,23 @@ func fromBuilds(ctx context.Context, r Reader, in *compose.Input, s Scope, p del } b, err := r.Build(ctx, pm.Module, m.FP2) if err != nil { - return fmt.Errorf("frombus: the build of %s at %s: %w", pm.Module, m.FP2, err) + err = fmt.Errorf("frombus: the build of %s at %s: %w", pm.Module, m.FP2, err) + if refuse == nil { + return err + } + stub(pm.Module, err) + continue } if b == nil || b.State != delivery.BuildDone { continue // no done build at the trunk's: composed from the indexer's manifest, as before } resolved, err := built(pm.Module, m.FP2, b) if err != nil { - return err + if refuse == nil { + return err + } + stub(pm.Module, err) + continue } in.Catalogue[pm.Module] = resolved } @@ -427,15 +476,22 @@ func fromBuilds(ctx context.Context, r Reader, in *compose.Input, s Scope, p del for _, module := range sortedKeys(s.KeptAt) { fp := s.KeptAt[module] b, err := r.Build(ctx, module, fp) - if err != nil { - return fmt.Errorf("frombus: the build of %s at %s: %w", module, fp, err) + switch { + case err != nil: + err = fmt.Errorf("frombus: the build of %s at %s: %w", module, fp, err) + case b == nil || b.State != delivery.BuildDone: + err = fmt.Errorf("frombus: %s is wanted at %s, of which the builder holds no done build", module, fp) } - if b == nil || b.State != delivery.BuildDone { - return fmt.Errorf("frombus: %s is wanted at %s, of which the builder holds no done build", module, fp) + var resolved catalogue.Manifest + if err == nil { + resolved, err = built(module, fp, b) } - resolved, err := built(module, fp, b) if err != nil { - return err + if refuse == nil { + return err + } + stub(module, err) + continue } if in.Nodes[i].Kept == nil { in.Nodes[i].Kept = map[string]catalogue.Manifest{} @@ -613,3 +669,27 @@ func sortedKeys[V any](m map[string]V) []string { sort.Strings(out) return out } + +// storeSought says, for a node that reaches no artifact store, what the composition holds of one: the nodes it holds and +// where each module serving the artifact store is placed, so the refusal names why (issue 1126). +func storeSought(in compose.Input) string { + var nodes, where []string + for _, n := range in.Nodes { + nodes = append(nodes, n.Name) + for _, a := range n.Assigned { + if m, ok := in.Catalogue[a]; ok { + if _, serves := m.Serves[catalogue.ArtifactStoreProvision]; serves { + where = append(where, a+" on "+n.Name) + } + } + } + } + sort.Strings(nodes) + sort.Strings(where) + said := "the composition holds " + strings.Join(nodes, ", ") + if len(where) == 0 { + return said + "; no module placed there serves " + catalogue.ArtifactStoreProvision + } + return said + "; " + catalogue.ArtifactStoreProvision + " is served by " + strings.Join(where, ", ") + + ", none of it on the private network with a port" +} diff --git a/go/compose/input.go b/go/compose/input.go index bf06c3c..2ca4f25 100644 --- a/go/compose/input.go +++ b/go/compose/input.go @@ -23,6 +23,7 @@ import ( "strings" "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" ) // Input is everything the composition reads. Each field is what one read of the controller's inventory answers, named @@ -194,6 +195,11 @@ type Node struct { // resources into the node's own, as the controller's composed writer does, unless fps gives one of the new form. // The private network's module (overlay.Name), which nobody builds, is folded whether or not it is named here. InOwn map[string]bool + // RunsOwn is the node's own composition (`.node`) as it runs, when the node composes itself: a module folded + // there (InOwn) whose manifest names a build (NamesBuild) runs a build of the old form only the controller holds, + // so it is left to the node: Compositions composes it as a stub (Stub) and carries its resources as they run, each + // after the resource it runs after (issue 1126). Nil composes every folded module from its manifest, as before. + RunsOwn *delivery.Composed } // Place is where a node is, as its placement says (inv.Overlay). From 472d5f314466afa910117ac8ad033f4df9ea2439 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 27 Oct 2026 23:22:54 +0100 Subject: [PATCH 2/9] A node's have says whose compositions it applies, and its answer to a head deleting its placement says it returns to the controller's (issue 1126, the seed's revert) The check of the seed's revert (mesh-catalog #526) failed on 'would stop running node': the shadow answered a deleted placement as everything removed, while the node keeps what runs. HaveNode.Composes and ComposesWhy say whether a node applies its own compositions or the controller's; CandidateAnswer.Returns answers a deleted placement, verdict same, naming nothing composed. --- go/delivery/delivery.go | 20 +++++++++++++++++++- go/delivery/placement_candidate.go | 11 +++++++++++ go/delivery/placement_test.go | 20 +++++++++++++++++--- 3 files changed, 47 insertions(+), 4 deletions(-) diff --git a/go/delivery/delivery.go b/go/delivery/delivery.go index 1b2306f..6e51c60 100644 --- a/go/delivery/delivery.go +++ b/go/delivery/delivery.go @@ -410,8 +410,20 @@ type HaveNode struct { // its shadow composer whenever an answer changes, removed when the candidate key goes. The node applies nothing of a // candidate. Candidates map[string]CandidateAnswer `json:"candidates,omitempty"` + // Composes is whose compositions the node applies: its own (ComposesOwn, the catalogue places it) or the + // controller's (ComposesController); ComposesWhy says why in words. Written by the node agent at each start, so a + // node returning to the controller's compositions when its placement key is deleted says so (issue 1126). Empty + // from an engine older than the field. + Composes string `json:"composes,omitempty"` + ComposesWhy string `json:"composes-why,omitempty"` } +// Whose compositions a node applies (HaveNode.Composes). +const ( + ComposesOwn = "own" + ComposesController = "controller" +) + // Presence is whether a node or a part's instance is online; the entry's own time says since when. The watchman // writes it. type Presence struct { @@ -506,6 +518,11 @@ func DecodeHave(data []byte) (Have, error) { } func (n HaveNode) check() error { + switch n.Composes { + case "", ComposesOwn, ComposesController: + default: + return fmt.Errorf("delivery: a node composes %q, neither %s nor %s", n.Composes, ComposesOwn, ComposesController) + } for group, a := range n.Candidates { if group == "" { return fmt.Errorf("delivery: a node's candidate answer names no group") @@ -590,7 +607,8 @@ func SameHaveNode(a, b HaveNode) bool { return a.V == b.V && a.Composed == b.Composed && sameJSON(a.Windows, b.Windows) && sameJSON(a.Busy, b.Busy) && sameJSON(a.Strays, b.Strays) && sameJSON(a.Filters, b.Filters) && sameJSON(a.Profile, b.Profile) && (a.Record == nil) == (b.Record == nil) && (a.Record == nil || SameNodeRecord(*a.Record, *b.Record)) && - jsonEqual(a.Sealed, b.Sealed) && jsonEqual(a.Candidates, b.Candidates) + jsonEqual(a.Sealed, b.Sealed) && jsonEqual(a.Candidates, b.Candidates) && a.Composes == b.Composes && + a.ComposesWhy == b.ComposesWhy } // SamePresence says whether two presences are the same value. diff --git a/go/delivery/placement_candidate.go b/go/delivery/placement_candidate.go index edf257b..e3736bc 100644 --- a/go/delivery/placement_candidate.go +++ b/go/delivery/placement_candidate.go @@ -132,8 +132,16 @@ type CandidateAnswer struct { Refusals []ModuleRefused `json:"refusals,omitempty"` // Refused is why the node could not compose with the candidate at all; Refusals is then empty. Refused string `json:"refused,omitempty"` + // Returns is, for a head that deletes the node's placement file, what the node does then: it returns to the + // controller's compositions (ReturnsToController), as a node the catalogue does not place composes none of its own + // (issue 1126: the seed's revert). The verdict is then same, and nothing else is named: the node composes nothing + // with the candidate, and judges nothing of what the controller composes. + Returns string `json:"returns,omitempty"` } +// ReturnsToController is what a node answers a head that deletes its placement file. +const ReturnsToController = "the head deletes this node's placement: it returns to the controller's compositions" + // ModuleDiffers is one module the node runs and the candidate still places whose composition is not the one it runs: // both digests (Composed with the candidate, Running what runs), the first resource that differs by its id, or empty // when every resource is equal, and why in words. @@ -187,6 +195,9 @@ func (a CandidateAnswer) check() error { if a.Refused != "" && len(a.Same)+len(a.Differs)+len(a.Added)+len(a.Removed)+len(a.Refusals) > 0 { return fmt.Errorf("delivery: a candidate the node could not compose names modules") } + if a.Returns != "" && (a.Refused != "" || len(a.Same)+len(a.Differs)+len(a.Added)+len(a.Removed)+len(a.Refusals) > 0) { + return fmt.Errorf("delivery: a candidate the node returns to the controller's compositions on names what it composed") + } for _, d := range a.Differs { if d.Module == "" || d.Why == "" { return fmt.Errorf("delivery: a candidate difference names no module or no why") diff --git a/go/delivery/placement_test.go b/go/delivery/placement_test.go index ae77b5f..29376d0 100644 --- a/go/delivery/placement_test.go +++ b/go/delivery/placement_test.go @@ -229,19 +229,31 @@ func TestPlacement(t *testing.T) { "refused": {Head: "h1", Revision: 3, Verdict: CandidateRefused, Refused: "unbound postgres"}, "added": {Head: "h1", Verdict: CandidateDiffers, Same: []string{"node"}, Added: []ModuleAdded{{Module: "app", Unissued: []string{"app/token"}}}, Removed: []string{"old"}}, "a value": {Head: "h1", Verdict: CandidateRefused, Same: []string{"node"}, Refusals: []ModuleRefused{{Module: "db", Why: "stands in for db/password"}}}, + // A head that deletes the node's placement file: the node returns to the controller's compositions (issue 1126, + // the seed's revert, mesh-catalog #526). + "returns": {Head: "h1", Verdict: CandidateSame, Returns: ReturnsToController}, } - n := HaveNode{Composed: "sha256:" + strings.Repeat("a", 64), Sealed: []string{"box"}, Candidates: answers} + n := HaveNode{Composed: "sha256:" + strings.Repeat("a", 64), Sealed: []string{"box"}, Candidates: answers, + Composes: ComposesOwn, ComposesWhy: "the catalogue places it (placement.n1)"} raw, err := EncodeHaveNode(n) if err != nil { t.Fatal(err) } back, err := DecodeHaveNode(raw) - if err != nil || !SameHaveNode(back, HaveNode{V: Version, Composed: n.Composed, Sealed: n.Sealed, Candidates: answers}) { + if err != nil || !SameHaveNode(back, HaveNode{V: Version, Composed: n.Composed, Sealed: n.Sealed, Candidates: answers, + Composes: ComposesOwn, ComposesWhy: n.ComposesWhy}) { t.Fatalf("a node's answers changed on the way: %+v, %v", back, err) } - if SameHaveNode(back, HaveNode{V: Version, Composed: n.Composed, Sealed: n.Sealed}) { + if SameHaveNode(back, HaveNode{V: Version, Composed: n.Composed, Sealed: n.Sealed, Composes: ComposesOwn, ComposesWhy: n.ComposesWhy}) { t.Error("a node's have without its answers counts as the same") } + if SameHaveNode(back, HaveNode{V: Version, Composed: n.Composed, Sealed: n.Sealed, Candidates: answers, + Composes: ComposesController, ComposesWhy: n.ComposesWhy}) { + t.Error("a node's have composing otherwise counts as the same") + } + if _, err := EncodeHaveNode(HaveNode{Composes: "both"}); err == nil { + t.Error("a node composing neither its own nor the controller's was taken") + } // CompareComposed: the bus's user list is left out on both sides; anything else that differs is named. file := func(id, content string) json.RawMessage { return json.RawMessage(`{"id":"` + id + `","type":"file","module":"nats","path":"/etc/` + id + `","mode":"0600","content":"` + content + `"}`) @@ -285,6 +297,8 @@ func TestPlacement(t *testing.T) { "same with an added": {Head: "h", Verdict: CandidateSame, Added: []ModuleAdded{{Module: "a"}}}, "differs refusing": {Head: "h", Verdict: CandidateDiffers, Removed: []string{"a"}, Refusals: []ModuleRefused{{Module: "b", Why: "x"}}}, "refused and modules": {Head: "h", Verdict: CandidateRefused, Refused: "x", Same: []string{"a"}}, + "returns and removes": {Head: "h", Verdict: CandidateDiffers, Returns: ReturnsToController, Removed: []string{"a"}}, + "returns refused": {Head: "h", Verdict: CandidateRefused, Returns: ReturnsToController, Refused: "x"}, } { if _, err := EncodeHaveNode(HaveNode{Candidates: map[string]CandidateAnswer{"g": a}}); err == nil { t.Errorf("%s: taken", name) From e32d777530247a7f8fda33dcfd3920d7a3982eea Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 27 Oct 2026 23:32:31 +0100 Subject: [PATCH 3/9] A deleting head's answer compares the controller's compositions with what runs, and is refused only without returning; fold the shadow's tests into TestInput (issue 1126, review of the shadow, round 1) --- .../frombus/{answer_test.go => answer_rows_test.go} | 4 ++-- go/compose/frombus/frombus_test.go | 4 ++++ go/delivery/placement_candidate.go | 10 ++++++---- go/delivery/placement_test.go | 6 ++++-- 4 files changed, 16 insertions(+), 8 deletions(-) rename go/compose/frombus/{answer_test.go => answer_rows_test.go} (98%) diff --git a/go/compose/frombus/answer_test.go b/go/compose/frombus/answer_rows_test.go similarity index 98% rename from go/compose/frombus/answer_test.go rename to go/compose/frombus/answer_rows_test.go index 1f6ab25..f0aa574 100644 --- a/go/compose/frombus/answer_test.go +++ b/go/compose/frombus/answer_rows_test.go @@ -17,7 +17,7 @@ import ( // Issue 1126 (the seed's shadow, round 2): a node's answer names every module it cannot compose, not the first only; // and a module folded into its own composition that names a build of the old form (de-spiegel on novox, nats, photos, // systemd-resolved: their haves name a build-... id and no fingerprint) is left to the node, carried as it runs. -func TestTheAnswerNamesEveryRefusalAndLeavesAnOldBuildToTheNode(t *testing.T) { +func answerNamesEveryRefusalAndLeavesAnOldBuildToTheNode(t *testing.T) { ctx := context.Background() image := "alpine@sha256:" + strings.Repeat("a", 64) manifest := func(module string) []byte { @@ -138,7 +138,7 @@ func TestTheAnswerNamesEveryRefusalAndLeavesAnOldBuildToTheNode(t *testing.T) { // A node that reaches no artifact store says what its composition holds of one (issue 1126: ace and g14 answered // "reaches no artifact store", and nothing said why). -func TestNoStoreReachedSaysWhy(t *testing.T) { +func noStoreReachedSaysWhy(t *testing.T) { in := compose.Input{Catalogue: map[string]catalogue.Manifest{ "distribution": {Module: "distribution", Serves: map[string]map[string]any{catalogue.ArtifactStoreProvision: {"port": 5000}}}}, Nodes: []compose.Node{{Name: "g14", Assigned: []string{"adwaita"}}, {Name: "novox", Assigned: []string{"distribution"}}}} diff --git a/go/compose/frombus/frombus_test.go b/go/compose/frombus/frombus_test.go index 20a5ae5..ee7697d 100644 --- a/go/compose/frombus/frombus_test.go +++ b/go/compose/frombus/frombus_test.go @@ -119,6 +119,10 @@ func publicKey(t *testing.T) string { // controller's store fills for compose's own two-machine mesh (compose's aMesh: anchor the hub, laptop), and that Input // composes; a node reads only its own config values; what cannot be composed is refused, naming it. func TestInput(t *testing.T) { + // Issue 1126, the seed's shadow: every refusal named, an old-form module left to the node, and no store reached + // saying why (answer_rows_test.go). + t.Run("every refusal, and an old build left to the node", answerNamesEveryRefusalAndLeavesAnOldBuildToTheNode) + t.Run("no store reached says why", noStoreReachedSaysWhy) ctx := context.Background() xterm, err := os.ReadFile("../../catalogue/testdata/catalogue-835bf22b-xterm.module.json") if err != nil { diff --git a/go/delivery/placement_candidate.go b/go/delivery/placement_candidate.go index e3736bc..6b28569 100644 --- a/go/delivery/placement_candidate.go +++ b/go/delivery/placement_candidate.go @@ -134,8 +134,9 @@ type CandidateAnswer struct { Refused string `json:"refused,omitempty"` // Returns is, for a head that deletes the node's placement file, what the node does then: it returns to the // controller's compositions (ReturnsToController), as a node the catalogue does not place composes none of its own - // (issue 1126: the seed's revert). The verdict is then same, and nothing else is named: the node composes nothing - // with the candidate, and judges nothing of what the controller composes. + // (issue 1126: the seed's revert). Same, Differs, Added and Removed then compare the controller's compositions at + // the node's wants with what it runs, and the verdict is theirs; a want the controller holds no composition for is + // refused (Refused, without Returns). Returns string `json:"returns,omitempty"` } @@ -195,8 +196,9 @@ func (a CandidateAnswer) check() error { if a.Refused != "" && len(a.Same)+len(a.Differs)+len(a.Added)+len(a.Removed)+len(a.Refusals) > 0 { return fmt.Errorf("delivery: a candidate the node could not compose names modules") } - if a.Returns != "" && (a.Refused != "" || len(a.Same)+len(a.Differs)+len(a.Added)+len(a.Removed)+len(a.Refusals) > 0) { - return fmt.Errorf("delivery: a candidate the node returns to the controller's compositions on names what it composed") + if a.Returns != "" && (a.Refused != "" || len(a.Refusals) > 0) { + return fmt.Errorf("delivery: a candidate the node returns to the controller's compositions on is compared, never refused: " + + "a composition the controller holds none of is refused without returning") } for _, d := range a.Differs { if d.Module == "" || d.Why == "" { diff --git a/go/delivery/placement_test.go b/go/delivery/placement_test.go index 29376d0..e0b7810 100644 --- a/go/delivery/placement_test.go +++ b/go/delivery/placement_test.go @@ -231,7 +231,9 @@ func TestPlacement(t *testing.T) { "a value": {Head: "h1", Verdict: CandidateRefused, Same: []string{"node"}, Refusals: []ModuleRefused{{Module: "db", Why: "stands in for db/password"}}}, // A head that deletes the node's placement file: the node returns to the controller's compositions (issue 1126, // the seed's revert, mesh-catalog #526). - "returns": {Head: "h1", Verdict: CandidateSame, Returns: ReturnsToController}, + "returns": {Head: "h1", Verdict: CandidateSame, Returns: ReturnsToController, Same: []string{"app", "node"}}, + "returns, the controller's differs": {Head: "h1", Verdict: CandidateDiffers, Returns: ReturnsToController, Same: []string{"node"}, + Differs: []ModuleDiffers{{Module: "app", Resource: "file:/etc/app", Why: "the resource differs"}}, Removed: []string{"old"}}, } n := HaveNode{Composed: "sha256:" + strings.Repeat("a", 64), Sealed: []string{"box"}, Candidates: answers, Composes: ComposesOwn, ComposesWhy: "the catalogue places it (placement.n1)"} @@ -297,7 +299,7 @@ func TestPlacement(t *testing.T) { "same with an added": {Head: "h", Verdict: CandidateSame, Added: []ModuleAdded{{Module: "a"}}}, "differs refusing": {Head: "h", Verdict: CandidateDiffers, Removed: []string{"a"}, Refusals: []ModuleRefused{{Module: "b", Why: "x"}}}, "refused and modules": {Head: "h", Verdict: CandidateRefused, Refused: "x", Same: []string{"a"}}, - "returns and removes": {Head: "h", Verdict: CandidateDiffers, Returns: ReturnsToController, Removed: []string{"a"}}, + "returns refusing": {Head: "h", Verdict: CandidateRefused, Returns: ReturnsToController, Refusals: []ModuleRefused{{Module: "b", Why: "x"}}}, "returns refused": {Head: "h", Verdict: CandidateRefused, Returns: ReturnsToController, Refused: "x"}, } { if _, err := EncodeHaveNode(HaveNode{Candidates: map[string]CandidateAnswer{"g": a}}); err == nil { From 7359d187db8dcd50d13b3d620b951027c3401137 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 28 Oct 2026 00:21:45 +0100 Subject: [PATCH 4/9] A stub keeps its module's directories, so the places its own secrets and data name still resolve (issue 1126: novox refused nats's ${dir:mesh-state}) --- go/compose/composed.go | 13 ++++++++++--- go/compose/frombus/answer_rows_test.go | 6 ++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/go/compose/composed.go b/go/compose/composed.go index 5975c25..a6a8bea 100644 --- a/go/compose/composed.go +++ b/go/compose/composed.go @@ -257,11 +257,18 @@ func NamesBuild(m catalogue.Manifest) bool { return false } -// Stub is m with nothing to build and no resource: what it provides, requires, serves and grants still shape the rest of -// the composition, and nothing of its own is composed (issue 1126). +// Stub is m with nothing to build and no resource but its directories: what it provides, requires, serves and grants +// still shape the rest of the composition, the places its own secrets, data and grants name (${dir:…}) still resolve, +// and nothing else of its own is composed (issue 1126: nats's ${dir:mesh-state} on novox). func Stub(m catalogue.Manifest) catalogue.Manifest { m.Build = nil - m.Resources = nil + var dirs []map[string]any + for _, r := range m.Resources { + if r["type"] == "directory" { + dirs = append(dirs, r) + } + } + m.Resources = dirs return m } diff --git a/go/compose/frombus/answer_rows_test.go b/go/compose/frombus/answer_rows_test.go index f0aa574..9c191d0 100644 --- a/go/compose/frombus/answer_rows_test.go +++ b/go/compose/frombus/answer_rows_test.go @@ -21,7 +21,10 @@ func answerNamesEveryRefusalAndLeavesAnOldBuildToTheNode(t *testing.T) { ctx := context.Background() image := "alpine@sha256:" + strings.Repeat("a", 64) manifest := func(module string) []byte { + // Its data names its directory, as nats's names ${dir:mesh-state} (issue 1126: novox's own composition refused + // "nats says ${dir:mesh-state}, and nats declares no directory mesh-state" when its stub held no directory). return []byte(`{"module":"` + module + `","version":"1","capabilities":["container-runtime"],` + + `"own-secrets":{"token":"${dir:state}/token"},` + `"resources":[{"id":"state","type":"directory","path":"/var/lib/` + module + `","mode":"0700"},` + `{"id":"server","type":"container","name":"` + module + `","artifact":"site"}],` + `"build":{"artifacts":[{"name":"site","kind":"upstream","from":"` + image + `"}]}}`) @@ -38,6 +41,9 @@ func answerNamesEveryRefusalAndLeavesAnOldBuildToTheNode(t *testing.T) { modules: map[string]delivery.Module{"lemurs": {Manifest: delivery.ManifestDigest(lemurs)}}, manifests: map[string][]byte{delivery.ManifestDigest(lemurs): lemurs}, builds: map[string]*delivery.BuilderBuild{}} + for _, m := range []string{"web", "api", "spiegel"} { + b.own = append(b.own, compose.OwnSecretRow{Node: "n1", Module: m, Name: "token", Origin: "made", SealedTo: "sk", Sealed: `"box"`}) + } for _, m := range []string{"web", "api", "spiegel"} { raw := manifest(m) b.manifests[delivery.ManifestDigest(raw)] = raw From b31c4580aa93cea337b2e322b60a637e756f6710 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 28 Oct 2026 01:08:49 +0100 Subject: [PATCH 5/9] A candidate answer shows every differing resource, masked; a module folded in the node's own is judged there; the node's own carries its bus membership as it runs it (issue 1126, seed #531) novox's answer to seed #531 named 15 resources by id alone, so why each differed was guessed. ModuleDiffers.Bodies keeps every differing resource with both bodies, each value under a key a secret could be in masked (the same, another seal of the same value, another value). The five old-form modules read 'removed' because the agent also lists them by module from the declaration kept before the switch; they are judged inside the node's own. And the node's own lacked bus-membership, which only the controller composes from its store: the membership the node runs is carried, until the grants part hands it sealed. --- go/compose/frombus/answer.go | 13 +- go/compose/frombus/answer_rows_test.go | 19 +++ go/compose/frombus/frombus.go | 29 ++++ go/delivery/composed_compare.go | 194 ++++++++++++++++++++++++- go/delivery/placement_candidate.go | 21 +++ go/delivery/placement_test.go | 33 +++++ 6 files changed, 303 insertions(+), 6 deletions(-) diff --git a/go/compose/frombus/answer.go b/go/compose/frombus/answer.go index baa549e..52f3bbb 100644 --- a/go/compose/frombus/answer.go +++ b/go/compose/frombus/answer.go @@ -107,7 +107,18 @@ func Answer(ctx context.Context, r Reader, mesh Mesh, s Scope, fps map[string]st a.Verdict = delivery.VerdictOf(a) return a } - cmp := delivery.CompareComposed(composed, running) + // A module the composition folds inside the node's own is judged there: the agent also says it runs by module, from + // the declaration it kept before the switch (agent.Everything), which is no removal (issue 1126, seed #531). + if len(s.InOwn) > 0 { + var judged []delivery.Composed + for _, c := range running { + if c.Module == "" || !slices.Contains(s.InOwn, c.Module) { + judged = append(judged, c) + } + } + running = judged + } + cmp := delivery.CompareComposedWith(composed, running, s.SameValue) // A refused module was composed as a stub: what it composed is no difference; its refusal is said. notFailed := func(m string) bool { _, ok := failed[m]; return !ok } a.Same = slices.DeleteFunc(cmp.Same, func(m string) bool { return !notFailed(m) }) diff --git a/go/compose/frombus/answer_rows_test.go b/go/compose/frombus/answer_rows_test.go index 9c191d0..2a361af 100644 --- a/go/compose/frombus/answer_rows_test.go +++ b/go/compose/frombus/answer_rows_test.go @@ -135,6 +135,25 @@ func answerNamesEveryRefusalAndLeavesAnOldBuildToTheNode(t *testing.T) { if a.Verdict != delivery.CandidateSame { t.Errorf("the fold's answer: %+v, want the same", a) } + // (A) The agent also says spiegel runs by module, from the declaration kept before the switch (agent.Everything): a + // module the composition folds is judged inside the node's own, not as removed (issue 1126, seed #531's answer). + keptToo := append(append([]delivery.Composed{}, running...), delivery.Composed{Node: "n1", Module: "spiegel", Resources: carriedRes}) + if a = Answer(ctx, b, Mesh{}, Scope{Node: "n1", Placements: given(b), InOwn: []string{"spiegel"}}, fps, keptToo, "h1", 3); a.Verdict != delivery.CandidateSame || len(a.Removed) != 0 { + t.Errorf("a folded module running from the kept declaration is answered %+v, want the same", a) + } + // (B) The node's own composition carries its bus membership as it runs it (the controller's, from its store): composed + // the same, not 'in what runs, not composed' (seed #531, novox). + membership := json.RawMessage(`{"id":"bus-membership","type":"file","path":"` + catalogue.BusMembershipPath + `","sealed":"{\"box\":\"m\"}","mode":"0600"}`) + withMembership := make([]delivery.Composed, 0, len(running)) + for _, c := range running { + if c.Module == "" { + c.Resources = append(append([]json.RawMessage{}, c.Resources...), membership) + } + withMembership = append(withMembership, c) + } + if a = Answer(ctx, b, Mesh{}, Scope{Node: "n1", Placements: given(b), InOwn: []string{"spiegel"}}, fps, withMembership, "h1", 3); a.Verdict != delivery.CandidateSame { + t.Errorf("the node's own with its bus membership is answered %+v, want the same", a) + } // Without what runs, the fold composes spiegel from its manifest, as before, and the composition refuses it. if _, _, err := Compose(ctx, b, Mesh{}, Scope{Node: "n1", InOwn: []string{"spiegel"}}, fps); err == nil || !strings.Contains(err.Error(), "spiegel has not been built for this mesh") { diff --git a/go/compose/frombus/frombus.go b/go/compose/frombus/frombus.go index 7aef315..735a831 100644 --- a/go/compose/frombus/frombus.go +++ b/go/compose/frombus/frombus.go @@ -124,6 +124,10 @@ type Scope struct { Placements map[string]*delivery.Placement InOwn []string Running []delivery.Composed + // SameValue, in an answer, says whether two sealed values open to the same value (the node opens them with its own + // key), so a differing resource shows a value sealed again as such (delivery.CompareComposedWith). Nil shows them + // as differing. + SameValue func(a, b string) (same, known bool) } // InOwn is, of the modules p places on its node, those whose resources the node's own composition it runs @@ -296,6 +300,16 @@ func input(ctx context.Context, r Reader, mesh Mesh, s Scope, refuse func(module if s.Running[i].Module == "" { own := s.Running[i] n.RunsOwn = &own + // Its bus membership as it runs it: the controller composes it from its store, which the node does + // not read; until the grants part hands memberships sealed to the node, the node's own composition + // carries the one it runs (issue 1126, seed #531 on novox: "bus-membership in what runs, not + // composed", which applied would have taken the node's bus credential away). + if sealed := membershipOf(own); sealed != "" { + if in.BusMemberships == nil { + in.BusMemberships = map[string]string{} + } + in.BusMemberships[s.Node] = sealed + } } } } @@ -693,3 +707,18 @@ func storeSought(in compose.Input) string { return said + "; " + catalogue.ArtifactStoreProvision + " is served by " + strings.Join(where, ", ") + ", none of it on the private network with a port" } + +// membershipOf is the sealed value of the bus membership resource (catalogue.BusMembershipID) of a node's own +// composition, or "". +func membershipOf(own delivery.Composed) string { + for _, raw := range own.Resources { + var r struct { + ID string `json:"id"` + Sealed string `json:"sealed"` + } + if json.Unmarshal(raw, &r) == nil && r.ID == catalogue.BusMembershipID() { + return r.Sealed + } + } + return "" +} diff --git a/go/delivery/composed_compare.go b/go/delivery/composed_compare.go index ecb4ca0..4faee55 100644 --- a/go/delivery/composed_compare.go +++ b/go/delivery/composed_compare.go @@ -28,6 +28,13 @@ type Comparison struct { // resource is equal it says what else differs (their order, the files filled, the data, the node's own parts, a file // carried by digest). func CompareComposed(composed map[string]Composed, running []Composed) Comparison { + return CompareComposedWith(composed, running, nil) +} + +// CompareComposedWith is CompareComposed showing, for each module that differs, every differing resource with both +// bodies masked (MaskedPair); sameValue, when given, says whether two sealed values open to the same value (the node +// opens them with its own key), so a value sealed again reads as such. +func CompareComposedWith(composed map[string]Composed, running []Composed, sameValue func(a, b string) (same, known bool)) Comparison { name := func(c Composed) string { if c.Module == "" { return HaveNodeWord @@ -60,7 +67,7 @@ func CompareComposed(composed map[string]Composed, running []Composed) Compariso case !composedHere: out.Removed = append(out.Removed, m) default: - if d, ok := compareOne(m, a, b); ok { + if d, ok := compareOne(m, a, b, sameValue); ok { out.Same = append(out.Same, m) } else { out.Differs = append(out.Differs, d) @@ -71,7 +78,7 @@ func CompareComposed(composed map[string]Composed, running []Composed) Compariso } // compareOne is one module's comparison: true when the digests agree, else what differs first. -func compareOne(module string, composed, running Composed) (ModuleDiffers, bool) { +func compareOne(module string, composed, running Composed, sameValue func(a, b string) (bool, bool)) (ModuleDiffers, bool) { composed, running = withoutBusUsers(composed), withoutBusUsers(running) dc, errC := ComposedDigestOf(composed) dr, errR := ComposedDigestOf(running) @@ -99,17 +106,35 @@ func compareOne(module string, composed, running Composed) (ModuleDiffers, bool) for id := range rb { ids[id] = true } + var first ModuleDiffers + found := false + var bodies []ResourceBodies for _, id := range sortedKeys(ids) { x, inA := ra[id] y, inB := rb[id] + why := "" switch { case !inB: - return differs(id, "composed, and not in what runs") + why = "composed, and not in what runs" case !inA: - return differs(id, "in what runs, and not composed") + why = "in what runs, and not composed" case !bytes.Equal(x, y): - return differs(id, "the resource differs") + why = "the resource differs" + default: + continue } + if !found { + first, _ = differs(id, why) + found = true + } + if len(bodies) < MaxBodies { + c, r := MaskedPair(x, y, sameValue) + bodies = append(bodies, ResourceBodies{ID: id, Why: why, Composed: cut(c), Running: cut(r)}) + } + } + if found { + first.Bodies = bodies + return first, false } switch { case !jsonEqual(a.Filled, b.Filled): @@ -165,3 +190,162 @@ func resourcesByID(resources []json.RawMessage) map[string][]byte { } return out } + +// cut is s cut at MaxBody bytes, saying so. +func cut(s string) string { + if len(s) <= MaxBody { + return s + } + return s[:MaxBody] + "… (cut at " + strconv.Itoa(MaxBody) + " bytes)" +} + +// secretWords are the words of a key whose value may hold a secret or a sealed one: masked in a shown body. +var secretWords = []string{"sealed", "secret", "password", "passwd", "token", "private", "key", "credential", "jwt", "salt", "hash"} + +func secretKey(k string) bool { + k = strings.ToLower(k) + for _, w := range secretWords { + if strings.Contains(k, w) { + return true + } + } + return false +} + +// MaskedPair is two bodies of one resource (either may be nil) as JSON text with every value under a key a secret could +// be in (secretWords, at any depth) masked: "", "" or "" when sameValue says so, "" otherwise, "" on one side +// alone. A body that is not JSON is shown as its length only. +func MaskedPair(composed, running json.RawMessage, sameValue func(a, b string) (same, known bool)) (string, string) { + var a, b any + okA := composed != nil && json.Unmarshal(composed, &a) == nil + okB := running != nil && json.Unmarshal(running, &b) == nil + if okA && okB { + a, b = maskBoth(a, b, sameValue) + } else if okA { + a, _ = maskBoth(a, nil, nil) + } else if okB { + _, b = maskBoth(nil, b, nil) + } + show := func(v any, raw json.RawMessage, ok bool) string { + if raw == nil { + return "" + } + if !ok { + return "" + } + var out bytes.Buffer + enc := json.NewEncoder(&out) + enc.SetEscapeHTML(false) + _ = enc.Encode(v) + return strings.TrimSuffix(out.String(), "\n") + } + return show(a, composed, okA), show(b, running, okB) +} + +func maskBoth(a, b any, sameValue func(a, b string) (bool, bool)) (any, any) { + ma, isA := a.(map[string]any) + mb, isB := b.(map[string]any) + if isA || isB { + outA, outB := map[string]any{}, map[string]any{} + keys := map[string]bool{} + for k := range ma { + keys[k] = true + } + for k := range mb { + keys[k] = true + } + for k := range keys { + va, inA := ma[k] + vb, inB := mb[k] + if secretKey(k) { + word := maskWord(va, vb, inA && inB, sameValue) + if inA { + outA[k] = word + } + if inB { + outB[k] = word + } + continue + } + x, y := maskBoth(va, vb, sameValue) + if inA { + outA[k] = x + } + if inB { + outB[k] = y + } + } + var ra, rb any + if isA { + ra = outA + } else { + ra = a + } + if isB { + rb = outB + } else { + rb = b + } + return ra, rb + } + la, isLA := a.([]any) + lb, isLB := b.([]any) + if isLA || isLB { + outA := make([]any, len(la)) + outB := make([]any, len(lb)) + for i := range max(len(la), len(lb)) { + var x, y any + if i < len(la) { + x = la[i] + } + if i < len(lb) { + y = lb[i] + } + x, y = maskBoth(x, y, sameValue) + if i < len(la) { + outA[i] = x + } + if i < len(lb) { + outB[i] = y + } + } + var ra, rb any = a, b + if isLA { + ra = outA + } + if isLB { + rb = outB + } + return ra, rb + } + return a, b +} + +func maskWord(a, b any, both bool, sameValue func(a, b string) (bool, bool)) string { + if !both { + return "" + } + x, _ := json.Marshal(a) + y, _ := json.Marshal(b) + if bytes.Equal(x, y) { + return "" + } + if sameValue != nil { + sa, okA := a.(string) + sb, okB := b.(string) + if !okA { + sa = string(x) + } + if !okB { + sb = string(y) + } + if same, known := sameValue(sa, sb); known { + if same { + return "" + } + return "" + } + } + return "" +} diff --git a/go/delivery/placement_candidate.go b/go/delivery/placement_candidate.go index 6b28569..64a8d47 100644 --- a/go/delivery/placement_candidate.go +++ b/go/delivery/placement_candidate.go @@ -138,6 +138,9 @@ type CandidateAnswer struct { // the node's wants with what it runs, and the verdict is theirs; a want the controller holds no composition for is // refused (Refused, without Returns). Returns string `json:"returns,omitempty"` + // Note is what the node says of how it answered, in words, beside the verdict: a pass that ran over a minute, inputs + // it waited for (issue 1126: the shadow shows its work). It changes no verdict. + Note string `json:"note,omitempty"` } // ReturnsToController is what a node answers a head that deletes its placement file. @@ -152,8 +155,26 @@ type ModuleDiffers struct { Running string `json:"running,omitempty"` Resource string `json:"resource,omitempty"` Why string `json:"why"` + // Bodies is every resource that differs, both sides, each value a secret could be in masked (issue 1126: the shadow + // shows its work, so why a module differs is read, not guessed). At most MaxBodies, each side cut at MaxBody bytes. + Bodies []ResourceBodies `json:"bodies,omitempty"` } +// ResourceBodies is one differing resource of a module: its id, why, and its body composed and as it runs, masked +// (MaskedPair); a side that does not hold it is empty. +type ResourceBodies struct { + ID string `json:"id"` + Why string `json:"why"` + Composed string `json:"composed,omitempty"` + Running string `json:"running,omitempty"` +} + +// The bounds of a module's shown bodies. +const ( + MaxBodies = 20 + MaxBody = 8192 +) + // ModuleAdded is one module the candidate places on the node and the node does not run: its composition's digest when // it composed, and the values not yet issued for it (its own secrets, its grants, a host port), each in the words the // composition stood in for it with. diff --git a/go/delivery/placement_test.go b/go/delivery/placement_test.go index e0b7810..dd906b2 100644 --- a/go/delivery/placement_test.go +++ b/go/delivery/placement_test.go @@ -283,6 +283,39 @@ func TestPlacement(t *testing.T) { t.Errorf("%s: differs %+v, want %s: %s with both digests", name, c.Differs, tc.resource, tc.why) } } + // Issue 1126, the shadow shows its work: every differing resource with both bodies, each value a secret could be + // in masked (never a secret in an answer), saying whether the two sides hold the same value. + sealedFile := func(id, sealed, mode string) json.RawMessage { + return json.RawMessage(`{"id":"` + id + `","type":"file","module":"nats","path":"/etc/` + id + `","mode":"` + mode + `","sealed":"` + sealed + `"}`) + } + shown := CompareComposedWith(map[string]Composed{"k": comp(sealedFile("a", "box-1", "0600"), sealedFile("b", "box-2", "0600"), file("c", "x"))}, + []Composed{comp(sealedFile("a", "box-1", "0644"), sealedFile("b", "box-3", "0600"), file("c", "y"))}, + func(a, b string) (same, known bool) { return a == "box-2" && b == "box-3", true }) + if len(shown.Differs) != 1 || len(shown.Differs[0].Bodies) != 3 { + t.Fatalf("every differing resource is not shown: %+v", shown.Differs) + } + for _, row := range []struct{ id, has, hasNot string }{ + {"a", `"mode":"0600"`, "box-1"}, + {"b", "another seal of the same value", "box-2"}, + {"c", `"content":"x"`, ""}, + } { + var body ResourceBodies + for _, b := range shown.Differs[0].Bodies { + if b.ID == row.id { + body = b + } + } + both := body.Composed + body.Running + if !strings.Contains(both, row.has) || (row.hasNot != "" && strings.Contains(both, row.hasNot)) { + t.Errorf("%s shown as %+v", row.id, body) + } + } + if c, r := MaskedPair(json.RawMessage(`{"env":{"DB_PASSWORD":"hunter2","PORT":"80"},"list":[{"token":"t1"}]}`), + json.RawMessage(`{"env":{"DB_PASSWORD":"hunter3","PORT":"81"},"list":[{"token":"t1"}]}`), nil); strings.Contains(c+r, "hunter") || + strings.Contains(c+r, `"t1"`) || !strings.Contains(r, `"PORT":"81"`) || !strings.Contains(c, "") || + !strings.Contains(c, "") { + t.Errorf("masked as %s and %s", c, r) + } if c := CompareComposed(nil, []Composed{{Node: "novox", Resources: []json.RawMessage{}}}); len(c.Removed) != 1 || c.Removed[0] != "node" { t.Errorf("the node's own composition run and not composed is removed: %+v", c) } From 4e0c57771e8ce8b89987b0eaebdb8fc9e268c157 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 28 Oct 2026 01:09:02 +0100 Subject: [PATCH 6/9] Bound the bodies an answer shows, naming the rest (issue 1126) --- go/compose/frombus/answer.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/go/compose/frombus/answer.go b/go/compose/frombus/answer.go index 52f3bbb..cefd190 100644 --- a/go/compose/frombus/answer.go +++ b/go/compose/frombus/answer.go @@ -119,6 +119,22 @@ func Answer(ctx context.Context, r Reader, mesh Mesh, s Scope, fps map[string]st running = judged } cmp := delivery.CompareComposedWith(composed, running, s.SameValue) + // The answer rides in the node's have key: its shown bodies are bounded, the rest named only. + budget := MaxShown + for i := range cmp.Differs { + kept := cmp.Differs[i].Bodies[:0] + for _, b := range cmp.Differs[i].Bodies { + if n := len(b.Composed) + len(b.Running); n <= budget { + budget -= n + kept = append(kept, b) + } else { + kept = append(kept, delivery.ResourceBodies{ID: b.ID, Why: b.Why}) + a.Note = "some differing resources are named without their bodies: the answer holds at most " + + fmt.Sprint(MaxShown) + " bytes of them" + } + } + cmp.Differs[i].Bodies = kept + } // A refused module was composed as a stub: what it composed is no difference; its refusal is said. notFailed := func(m string) bool { _, ok := failed[m]; return !ok } a.Same = slices.DeleteFunc(cmp.Same, func(m string) bool { return !notFailed(m) }) @@ -196,6 +212,9 @@ func Answer(ctx context.Context, r Reader, mesh Mesh, s Scope, fps map[string]st return a } +// MaxShown bounds the bytes of shown resource bodies in one answer (delivery.ModuleDiffers.Bodies). +const MaxShown = 256 << 10 + var ( grantStandIn = regexp.MustCompile(`^the grant of \S+ for (\S+) on (\S+)$`) needsStandIn = regexp.MustCompile(`^(\S+) on (\S+) needs `) From 362a3f1378aace745b81b712e24f198f62c01ef5 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 28 Oct 2026 01:39:07 +0100 Subject: [PATCH 7/9] A refusal for want of a provider names the modules another node cannot run in this composition, and a refusal naming no placed module is answered at once (issue 1126, probe #532) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ace and g14 answered 'nothing in this mesh provides artifact-store … assign distribution to a node' while novox runs it: a module another node cannot run in this composition (catalogue Resolution.Unhostable) offers nothing, silently. The refusal now names each such module and what it lacks, and each node left out. And the culprit search composed the whole mesh once per placed module for a refusal no stub changes (ace's answer took 7m46s): only the modules the refusal names are tried, at most 8. --- go/compose/frombus/answer.go | 16 ++++++------- go/compose/frombus/answer_rows_test.go | 32 ++++++++++++++++++++++++++ go/compose/frombus/frombus_test.go | 1 + go/compose/network.go | 3 +++ go/compose/resolve.go | 16 ++++++++++++- go/compose/rest.go | 5 ++++ 6 files changed, 64 insertions(+), 9 deletions(-) diff --git a/go/compose/frombus/answer.go b/go/compose/frombus/answer.go index cefd190..078dbbc 100644 --- a/go/compose/frombus/answer.go +++ b/go/compose/frombus/answer.go @@ -212,6 +212,9 @@ func Answer(ctx context.Context, r Reader, mesh Mesh, s Scope, fps map[string]st return a } +// maxCulprits bounds the modules one refusal is tried on, each a whole composition. +const maxCulprits = 8 + // MaxShown bounds the bytes of shown resource bodies in one answer (delivery.ModuleDiffers.Bodies). const MaxShown = 256 << 10 @@ -251,15 +254,12 @@ func culprit(in compose.Input, node string, fps map[string]string, failed map[st assigned = n.Assigned } } - // The modules the error names first: the words of a module's refusal open with its name. - order := make([]string, 0, len(assigned)) + // Only the modules the error names: the words of a module's refusal name it. Each try is a whole composition, so a + // refusal naming none is no module's, answered at once rather than after a composition per module (issue 1126: ace's + // answer took 7m46s). + var order []string for _, m := range assigned { - if strings.Contains(err.Error(), m) { - order = append(order, m) - } - } - for _, m := range assigned { - if !strings.Contains(err.Error(), m) { + if strings.Contains(err.Error(), m) && len(order) < maxCulprits { order = append(order, m) } } diff --git a/go/compose/frombus/answer_rows_test.go b/go/compose/frombus/answer_rows_test.go index 2a361af..74b1601 100644 --- a/go/compose/frombus/answer_rows_test.go +++ b/go/compose/frombus/answer_rows_test.go @@ -3,6 +3,7 @@ package frombus import ( "context" "encoding/json" + "fmt" "reflect" "slices" "strings" @@ -107,6 +108,11 @@ func answerNamesEveryRefusalAndLeavesAnOldBuildToTheNode(t *testing.T) { if err != nil { t.Fatal(err) } + // A refusal naming no module placed here is no module's: no compose is spent stubbing each in turn (issue 1126: ace's + // answer took 7m46s, one compose per module), though here a stub would compose. + if m := culprit(in, "n1", fps, map[string]string{}, fmt.Errorf("nothing in this mesh provides \"artifact-store\"")); m != "" { + t.Errorf("a refusal naming no placed module was laid on %s", m) + } ownKey, _ := delivery.ComposedNodeKey("n1") own := base[ownKey] carriedRes := []json.RawMessage{ @@ -161,6 +167,32 @@ func answerNamesEveryRefusalAndLeavesAnOldBuildToTheNode(t *testing.T) { } } +// A module another node cannot run in this composition offers nothing in it: the refusal names it and why +// (issue 1126, probe #532: ace and g14 answered "nothing in this mesh provides artifact-store … assign distribution to a +// node" while novox runs it). And a refusal no module's stub changes is answered at once, not after a compose per module +// (ace took 7m46s). +func otherNodeNotResolvedIsNamed(t *testing.T) { + ctx := context.Background() + prov := []byte(`{"module":"prov","version":"1","capabilities":["container-runtime"],"provides":[{"name":"thing","scope":"mesh"}]}`) + cons := []byte(`{"module":"cons","version":"1","requires":["thing"]}`) + // n2 runs prov, which needs a container runtime n2's record does not say it has: n2's set does not resolve here. + b := &bus{placements: map[string]delivery.Placement{ + "n1": {Node: "n1", Decided: delivery.NodeDecided{Hub: true, Address: "10.77.0.1", Endpoint: "n1.example:51820"}, Modules: []delivery.PlacedModule{{Module: "cons"}, {Module: overlay.Name}}}, + "n2": {Node: "n2", Decided: delivery.NodeDecided{Address: "10.77.0.2"}, Modules: []delivery.PlacedModule{{Module: "prov"}, {Module: overlay.Name}}}}, + records: map[string]delivery.NodeRecord{"n1": {Node: "n1", TunnelKey: publicKey(t)}, "n2": {Node: "n2", TunnelKey: publicKey(t)}}, + modules: map[string]delivery.Module{"prov": {Manifest: delivery.ManifestDigest(prov)}, "cons": {Manifest: delivery.ManifestDigest(cons)}}, + manifests: map[string][]byte{delivery.ManifestDigest(prov): prov, delivery.ManifestDigest(cons): cons}} + given := map[string]*delivery.Placement{} + for n, p := range b.placements { + p := p + given[n] = &p + } + a := Answer(ctx, b, Mesh{}, Scope{Node: "n1", Placements: given}, map[string]string{"cons": "src2:" + strings.Repeat("1", 64)}, nil, "h", 1) + if a.Verdict != delivery.CandidateRefused || !strings.Contains(a.Refused, "prov on n2 lacks container-runtime") { + t.Errorf("a refusal for want of n2's provider = %+v", a) + } +} + // A node that reaches no artifact store says what its composition holds of one (issue 1126: ace and g14 answered // "reaches no artifact store", and nothing said why). func noStoreReachedSaysWhy(t *testing.T) { diff --git a/go/compose/frombus/frombus_test.go b/go/compose/frombus/frombus_test.go index ee7697d..e02fa56 100644 --- a/go/compose/frombus/frombus_test.go +++ b/go/compose/frombus/frombus_test.go @@ -123,6 +123,7 @@ func TestInput(t *testing.T) { // saying why (answer_rows_test.go). t.Run("every refusal, and an old build left to the node", answerNamesEveryRefusalAndLeavesAnOldBuildToTheNode) t.Run("no store reached says why", noStoreReachedSaysWhy) + t.Run("another node not resolved is named", otherNodeNotResolvedIsNamed) ctx := context.Background() xterm, err := os.ReadFile("../../catalogue/testdata/catalogue-835bf22b-xterm.module.json") if err != nil { diff --git a/go/compose/network.go b/go/compose/network.go index afc8363..f51d6d7 100644 --- a/go/compose/network.go +++ b/go/compose/network.go @@ -24,6 +24,9 @@ type composer struct { in Input said []string seen map[string]bool + // unhostable is, from the last look at the rest of the mesh, each module assigned to another node that this + // composition finds that node cannot run: what it would have offered is missing here (issue 1126, probe #532). + unhostable []string } func newComposer(in Input) *composer { return &composer{in: in, seen: map[string]bool{}} } diff --git a/go/compose/resolve.go b/go/compose/resolve.go index d2fffd8..a481909 100644 --- a/go/compose/resolve.go +++ b/go/compose/resolve.go @@ -3,6 +3,7 @@ package compose import ( "fmt" "slices" + "strings" "git.novox.be/novox/mesh-sdk/go/catalogue" ) @@ -203,7 +204,7 @@ func (c *composer) resolution(nodeName string) (catalogue.Resolution, error) { return catalogue.Resolution{}, err } shelf := c.shelfFor(node) - world, _, _, err := c.theRestOfTheMesh(nodeName) + world, _, left, err := c.theRestOfTheMesh(nodeName) if err != nil { return catalogue.Resolution{}, err } @@ -226,6 +227,19 @@ func (c *composer) resolution(nodeName string) (catalogue.Resolution, error) { Account: node.Account, AccountHome: node.AccountHome, AgentAccount: node.AgentAccount, AgentAccountHome: node.AgentAccountHome}, world) if err != nil { + // What another node's set would have offered is missing when that set does not resolve here: said with the + // refusal, so "nothing provides" is read beside why (issue 1126, probe #532). + if len(left) > 0 { + var why []string + for _, l := range left { + why = append(why, l.String()) + } + err = fmt.Errorf("%w\n\nand %d node(s) offer nothing here, as they could not be resolved: %s", err, len(left), strings.Join(why, "; ")) + } + if len(c.unhostable) > 0 { + err = fmt.Errorf("%w\n\nand these modules offer nothing here, as their node cannot run them in this composition: %s", + err, strings.Join(c.unhostable, "; ")) + } // The node's own set does not compose: the only failure a mesh-wide gatherer may pass over. return catalogue.Resolution{}, notResolvable{err} } diff --git a/go/compose/rest.go b/go/compose/rest.go index f7509e2..a5caf70 100644 --- a/go/compose/rest.go +++ b/go/compose/rest.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "sort" + "strings" "git.novox.be/novox/mesh-sdk/go/catalogue" ) @@ -100,6 +101,7 @@ func (c *composer) theRestOfTheMesh(exclude string) (catalogue.World, Passes, [] var left []LeftOut offered := map[string][]catalogue.Provider{} var firstHeld []catalogue.Held + c.unhostable = nil for _, o := range others { got, err := catalogue.Resolve(shelf, o.assigned, o.node, catalogue.World{Unchecked: true, Holdings: holdings}) if err != nil { @@ -107,6 +109,9 @@ func (c *composer) theRestOfTheMesh(exclude string) (catalogue.World, Passes, [] left = append(left, LeftOut{Node: o.node.Name, Err: err}) continue } + for _, u := range got.Unhostable { + c.unhostable = append(c.unhostable, fmt.Sprintf("%s on %s lacks %s", u.Module, o.node.Name, strings.Join(u.Missing, ", "))) + } firstHeld = append(firstHeld, got.Claims...) for _, m := range got.Modules { for _, name := range m.OffersAt(catalogue.ScopeMesh) { From de72cb9a11cc763dbab147ec57e891bcf55480bd Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 28 Oct 2026 09:26:12 +0100 Subject: [PATCH 8/9] =?UTF-8?q?The=20vault=20signs:=20the=20contract=20of?= =?UTF-8?q?=20its=20state=20signers,=20its=20verb=20sign=20and=20its=20eve?= =?UTF-8?q?nt=20signed,=20a=20node's=20grants=20to=20ask=20it=20and=20to?= =?UTF-8?q?=20read=20what=20it=20trusts,=20and=20the=20authority=20compose?= =?UTF-8?q?d=20as=20a=20bundle=20(issue=20774,=20ADR=200550=20=C2=A72)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go/buscompose/buscompose_test.go | 17 ++ go/buscompose/nats.go | 22 ++ go/buscompose/users.go | 2 +- go/catalogue/seats.go | 8 + go/compose/frombus/frombus.go | 17 ++ go/compose/frombus/frombus_test.go | 26 +- go/delivery/composed_signature_test.go | 49 ++++ go/delivery/noderecord.go | 4 + go/delivery/noderecord_test.go | 2 +- go/delivery/signers.go | 328 +++++++++++++++++++++++++ 10 files changed, 465 insertions(+), 10 deletions(-) create mode 100644 go/delivery/signers.go diff --git a/go/buscompose/buscompose_test.go b/go/buscompose/buscompose_test.go index 2489938..453966d 100644 --- a/go/buscompose/buscompose_test.go +++ b/go/buscompose/buscompose_test.go @@ -130,6 +130,8 @@ func grantsAsTheControllers(t *testing.T, got, want string) { } } +const SignersBucketName = "mesh-vault_signers" + func TestTheCopiedComposerComposesAsTheControllers(t *testing.T) { raw, err := os.ReadFile(filepath.Join("testdata", "controller-user-lists.json")) if err != nil { @@ -236,6 +238,21 @@ func TestTheCopiedComposerComposesAsTheControllers(t *testing.T) { if !slices.Contains(perms.Subscribe, VaultSecretChanged) { t.Errorf("node one does not hear the vault's secret-changed: %v", perms.Subscribe) } + // L3 s6 (ADR 0550 §2): while the vault serves `sign`, a node asks it in its own name alone, and reads the vault's + // `signers` and the lease key, never writing either. + signs, _ := PermissionsFor(Principal{Kind: KindNode, Node: "one", AsksTheVaultToSign: true}) + signersGet := "$JS.API.DIRECT.GET.KV_" + SignersBucketName + ".$KV." + SignersBucketName + ".key" + leaseGet := "$JS.API.DIRECT.GET.KV_" + LeaseBucket + ".$KV." + LeaseBucket + ".holder" + for subject, want := range map[string]bool{VaultSignCall(one): true, signersGet: true, leaseGet: true, + VaultSignCall(Principal{Kind: KindNode, Node: "two"}): false, "$KV." + SignersBucketName + ".key": false, + "$KV." + LeaseBucket + ".holder": false} { + if MayPublish(signs, subject) != want { + t.Errorf("a node asking sign may publish %s: %v, want %v", subject, !want, want) + } + } + if MayPublish(perms, VaultSignCall(one)) || MayPublish(perms, signersGet) { + t.Error("a node asks sign or reads signers while the vault does not serve sign") + } // And what a module on another node serves from its settings, that key alone (mailu's smtp domain, issue 1126). users, err := Users(Records{Nodes: []string{"one", "two"}, Assigned: map[string][]Declared{ "two": {{Module: "mailer", Served: []string{"domain", "not.a.token"}}}}}) diff --git a/go/buscompose/nats.go b/go/buscompose/nats.go index 682c37e..fb5f6a5 100644 --- a/go/buscompose/nats.go +++ b/go/buscompose/nats.go @@ -217,6 +217,11 @@ type Principal struct { // ADR 0547 §2, ADR 0550 §5, issue 1126): its modules' own secrets sealed to its key, for the compositions it makes // itself. Set while the vault's holder serves it. AsksItsOwnSecrets bool + // AsksTheVaultToSign is a machine principal whose node-engine may ask the vault's `sign` in its own name, read the + // vault's public signing halves (`signers`) and the controller's lease key (novox/hq ADR 0550 §2): its own node's + // certificate, the keys it trusts, and the lease write that drops the controller's key. Set while the vault's holder + // serves `sign`. + AsksTheVaultToSign bool // ServedReads are, for a machine principal, the keys of mesh-config's resolved values that other nodes' modules serve // from (`..`, catalogue.ServedSettings): what every consumer is told, and so what a node composing // its own modules reads of a provider elsewhere (issue 1126), each key alone. Sorted, none twice. @@ -360,6 +365,10 @@ var VaultNodeKey = SeatVerb{Seat: "mesh-vault", Verb: "node-key"} // key (issue 1126, ADR 0550 §5); granted as Principal.AsksItsOwnSecrets says. var VaultOwn = SeatVerb{Seat: "mesh-vault", Verb: "own"} +// VaultSign is the vault's verb a machine's node-engine asks in its own name for its own node's certificate (novox/hq +// ADR 0550 §2); granted as Principal.AsksTheVaultToSign says. +var VaultSign = SeatVerb{Seat: "mesh-vault", Verb: delivery.SignVerb} + // VerbsTheControllerActsOnAWarrant are the other seat verbs the controller calls when the operator's warrant // chooses them (novox/hq ADR 0259): a machine's service restarted, and a walk started or stopped through the // controller's own `plans`, a machine's held builds sent by its own `push` and the builds held after a failed release @@ -953,6 +962,14 @@ func PermissionsFor(p Principal) (Permissions, error) { // on it rather than at the next unrelated change (ADR 0547 §2). sub = append(sub, VaultSecretChanged) } + // And, while the vault serves `sign`, its own node's certificate in its own name, the vault's public signing + // halves, read and watched, and the controller's lease key, read and watched, which says when the controller's key + // is no longer trusted (ADR 0550 §2). Read only. + if p.AsksTheVaultToSign { + pub = append(pub, VaultSignCall(p)) + pub = append(pub, readGrants(delivery.SignersBucket, "")...) + pub = append(pub, readGrants(LeaseBucket, leaseKey)...) + } // And, on the node holding mesh-broker, the grants part's sealed user list it follows from take-over (novox/hq ADR // 0547 clause 7): one key, and an object by its name. if p.FollowsGrantsUserList { @@ -1845,6 +1862,11 @@ func VaultOwnCall(p Principal) string { return "mesh.seat." + VaultOwn.Seat + "." + CallKind + "." + VaultOwn.Verb + "." + CallerToken(p.Username()) } +// VaultSignCall is the subject a machine principal's node-engine asks the vault's `sign` on, in its own name. +func VaultSignCall(p Principal) string { + return "mesh.seat." + VaultSign.Seat + "." + CallKind + "." + VaultSign.Verb + "." + CallerToken(p.Username()) +} + // VaultSecretChanged is the vault's event that a secret changed (a name and its generation, never a value), which a // node composing its own hears to compose again (issue 1126, ADR 0547 §2). var VaultSecretChanged = "mesh.mod." + VaultOwn.Seat + ".event.secret-changed" diff --git a/go/buscompose/users.go b/go/buscompose/users.go index 15cbd14..2424fcf 100644 --- a/go/buscompose/users.go +++ b/go/buscompose/users.go @@ -165,7 +165,7 @@ func Users(r Records) ([]Principal, error) { } out = append(out, Principal{Kind: KindNode, Node: node, WitnessesController: witness, Checks: checks, MovesIntoTheVault: servesMoveIn(r), StatesItsKeyToTheVault: servesVerb(r, VaultNodeKey), - AsksItsOwnSecrets: servesVerb(r, VaultOwn), + AsksItsOwnSecrets: servesVerb(r, VaultOwn), AsksTheVaultToSign: servesVerb(r, VaultSign), FollowsGrantsUserList: bus, ServedReads: servedOn(served, node)}) // **And the machine's provisioner client, on an account of its own** (novox/hq ADR 0349 clause 11): one // per machine that runs it, delivered to the module running it. diff --git a/go/catalogue/seats.go b/go/catalogue/seats.go index 0417420..ba5ff8f 100644 --- a/go/catalogue/seats.go +++ b/go/catalogue/seats.go @@ -228,6 +228,14 @@ var defaultSeats = append([]Seat{ "ADR 0561 §9): only while the controller's lease is handed away, never a node's account or the " + "controller's, never anew for one the vault holds a value or an adopted hash for. Never the value.", Input: schema(map[string]string{"user": "the bus user, e.g. ace.mesh-build-go"}, []string{"user"})}, + {Name: "sign", Optional: true, Description: "Sign with the vault's key or authority (novox/hq ADR 0550 §2): " + + "kind node-certificate signs the certificate of the calling machine's internal name for the serving key " + + "its record states, built from bus state alone and asked by that machine's node-engine only; kind " + + "enrolment signs an enrolment reply under its own context, for a caller the adoption record names. " + + "Answers the certificate or the signature and the signer's fingerprint, and says signed; never a key.", + Input: schema(map[string]string{"kind": "node-certificate or enrolment", + "node": "the machine, for a node certificate: the caller's own", + "reply": "the enrolment reply's bytes in base64, for an enrolment"}, []string{"kind"})}, }}, // The owner of every config value (novox/hq ADR 0348 §9, design 57 K3): its holder alone writes the buckets // mesh-config_values, _resolved, _predefined and _proposals, and serves the one verb `config` (§1, §10). diff --git a/go/compose/frombus/frombus.go b/go/compose/frombus/frombus.go index 0c1c28f..9e3ae13 100644 --- a/go/compose/frombus/frombus.go +++ b/go/compose/frombus/frombus.go @@ -83,6 +83,16 @@ type Reader interface { Served(ctx context.Context, node, module string, keys []string) (map[string]any, error) } +// SignersReader is a Reader that also reads the vault's public authority from its state `signers` (delivery.SignersBucket, +// key delivery.SignersAuthorityKey; ADR 0550 §2). A Reader that has it composes the mesh's authority as a bundle, the +// controller's and the vault's; one that has not, or that answers none, composes the controller's alone, as before. +type SignersReader interface { + // VaultAuthority is the vault's authority certificate as PEM, checked (delivery.DecodeSignerAuthority); empty when the + // vault has published none. A read that fails is said by the reader and answered empty: a node that may not read the + // state yet composes as before rather than not at all. + VaultAuthority(ctx context.Context) string +} + // Mesh is what is true of the whole mesh and is no part's state on the bus yet: the private network's range and the // internal names' suffix as genesis was told them, and the mesh's authority certificate (public). Empty fields take // compose's defaults. @@ -203,6 +213,13 @@ func Input(ctx context.Context, r Reader, mesh Mesh, s Scope) (compose.Input, er } } } + // And the vault's authority beside it, once the vault publishes one (ADR 0550 §2): a certificate either signed checks + // against the bundle. The controller's stays while a certificate it issued is in a node's record. + if sr, ok := r.(SignersReader); ok { + if vault := sr.VaultAuthority(ctx); vault != "" { + in.Authority = delivery.AuthorityBundle(in.Authority, vault) + } + } parse := func(raw []byte) (catalogue.Manifest, error) { if mesh.Predefined != nil { return catalogue.ParseManifestWith(raw, mesh.Predefined) diff --git a/go/compose/frombus/frombus_test.go b/go/compose/frombus/frombus_test.go index 20a5ae5..cf77da5 100644 --- a/go/compose/frombus/frombus_test.go +++ b/go/compose/frombus/frombus_test.go @@ -25,14 +25,15 @@ import ( // bus is a Reader over values held in memory, counting whose config values were read. type bus struct { - placements map[string]delivery.Placement - records map[string]delivery.NodeRecord - modules map[string]delivery.Module - manifests map[string][]byte - values map[string]map[string]any // by node.module - valuesRead []string - own []compose.OwnSecretRow - ownErr error + vaultAuthority string + placements map[string]delivery.Placement + records map[string]delivery.NodeRecord + modules map[string]delivery.Module + manifests map[string][]byte + values map[string]map[string]any // by node.module + valuesRead []string + own []compose.OwnSecretRow + ownErr error // asked is every consumer credential asked; the fake refuses the whole ask, as the vault's own does, for a consuming // module its placement does not put on the node. asked []compose.CredentialKey @@ -74,6 +75,7 @@ func (b *bus) Served(_ context.Context, node, module string, keys []string) (map func (b *bus) Placements(context.Context) (map[string]delivery.Placement, error) { return b.placements, nil } +func (b *bus) VaultAuthority(context.Context) string { return b.vaultAuthority } func (b *bus) Records(context.Context) (map[string]delivery.NodeRecord, error) { return b.records, nil } func (b *bus) Modules(context.Context) (map[string]delivery.Module, error) { return b.modules, nil } func (b *bus) Manifest(_ context.Context, digest string) ([]byte, error) { @@ -211,6 +213,14 @@ func TestInput(t *testing.T) { wj, _ := json.MarshalIndent(want, "", " ") t.Fatalf("Input from the bus differs from the store's:\n got %s\nwant %s", gj, wj) } + // L3 s8 (ADR 0550 §2): with the vault's authority published, the mesh's authority is the bundle of the controller's + // and the vault's; with none, the controller's alone, as above. + signing := mk() + signing.vaultAuthority = "the vault's certificate" + if bundled, err := Input(ctx, signing, Mesh{}, Scope{Node: "laptop"}); err != nil || + bundled.Authority != "the mesh's certificate\nthe vault's certificate\n" { + t.Errorf("the authority with the vault's published: %q, %v", bundled.Authority, err) + } if !reflect.DeepEqual(b.valuesRead, []string{"laptop." + overlay.Name, "laptop.xterm", "laptop.media"}) { t.Errorf("laptop read config values %v; a node reads its own slice alone", b.valuesRead) } diff --git a/go/delivery/composed_signature_test.go b/go/delivery/composed_signature_test.go index 816e8a8..4d411bd 100644 --- a/go/delivery/composed_signature_test.go +++ b/go/delivery/composed_signature_test.go @@ -6,6 +6,7 @@ import ( "encoding/json" "strings" "testing" + "time" ) func aSigningKey(t *testing.T) (ed25519.PublicKey, ed25519.PrivateKey) { @@ -100,4 +101,52 @@ func TestACompositionIsSignedByAKeptKey(t *testing.T) { if _, err := SignComposedBy(c, pub, func(m []byte) ([]byte, error) { return ed25519.Sign(other, m), nil }); err == nil { t.Errorf("a signature by another key than the one named was accepted") } + + // L3 s1 (ADR 0550 §2): a message the vault or the controller signed under a kind's context verifies against the + // signers a node trusts and names which; an unknown key and another kind's context are refused. + vaultPub, vault := aSigningKey(t) + controllerPub, controller := aSigningKey(t) + trusted := []Signer{{Name: SignerController, Key: controllerPub}, {Name: SignerVault, Key: vaultPub}} + enrol, _ := SignContext(KindEnrolment) + certCtx, _ := SignContext(KindNodeCertificate) + msg := []byte("the reply") + for _, row := range []struct { + name string + key ed25519.PrivateKey + signed string + checked string + want string + }{ + {"the vault's key verifies", vault, enrol, enrol, SignerVault}, + {"the controller's key verifies", controller, enrol, enrol, SignerController}, + {"an unknown key is refused", other, enrol, enrol, ""}, + {"the wrong context is refused", vault, certCtx, enrol, ""}, + } { + got, err := VerifyBy(trusted, row.checked, msg, SignMessage(row.key, row.signed, msg)) + if row.want == "" && err == nil { + t.Errorf("%s: verified as %s", row.name, got.Name) + } + if row.want != "" && (err != nil || got.Name != row.want) { + t.Errorf("%s: %+v, %v", row.name, got.Name, err) + } + } + if _, err := SignContext("hand-over"); err == nil { + t.Error("a kind the vault does not sign has a context") + } + // The signers' key reads back only when its fingerprint is its key's, and what a node trusts names both holders. + k := NewSignerKey(vaultPub, time.Unix(0, 0)) + raw, _ := json.Marshal(k) + if back, err := DecodeSignerKey(raw); err != nil || back.Fingerprint != SignerFingerprint(vaultPub) { + t.Errorf("the signers' key: %+v, %v", back, err) + } + k.Fingerprint = SignerFingerprint(controllerPub) + if raw, _ := json.Marshal(k); true { + if _, err := DecodeSignerKey(raw); err == nil { + t.Error("a signers' key naming another fingerprint was taken") + } + } + if got := Trusts(trusted, nil); len(got) != 2 || !TrustsKey(got, SignerVault, SignerFingerprint(vaultPub)) || + !TrustsKey(got, SignerController, SignerFingerprint(controllerPub)) { + t.Errorf("trusts: %v", got) + } } diff --git a/go/delivery/noderecord.go b/go/delivery/noderecord.go index c22f8b2..423cf6b 100644 --- a/go/delivery/noderecord.go +++ b/go/delivery/noderecord.go @@ -55,6 +55,10 @@ type NodeRecord struct { // Authority is the mesh's own certificate as this machine holds it (public): the one its declaration last wrote where // a module asked for it. A node composing its own composes a certificate with it (compose.Input.Authority). Authority string `json:"authority,omitempty"` + // Trusts is what this machine trusts a mesh signature from now (Trusts in signers.go, ADR 0550 §2): the controller's + // key until the lease is handed away, and the vault's key and authority once it reads them from `signers`. The gate + // before the lease write reads every node's from the bus. + Trusts []string `json:"trusts,omitempty"` } func (r NodeRecord) check() error { diff --git a/go/delivery/noderecord_test.go b/go/delivery/noderecord_test.go index e49b327..1e985e9 100644 --- a/go/delivery/noderecord_test.go +++ b/go/delivery/noderecord_test.go @@ -18,7 +18,7 @@ var secretNamed = regexp.MustCompile(`(?i)(secret|passw|token|api[-_]?key|privat func TestNodeRecord(t *testing.T) { hub := NodeRecord{Node: "novox", TunnelKey: "k", Capabilities: []string{"docker"}, SealingKey: "age1x", ServingKey: "pub", Certificate: "cert", OutwardLinks: []string{"eth0"}, Networks: []catalogue.AttachedNetwork{{Link: "eth0", Network: "192.0.2.0/24"}}, - NetworksSaid: true, Contract: 3} + NetworksSaid: true, Contract: 3, Trusts: []string{"controller key sha256:aa", "vault key sha256:bb"}} raw, err := EncodeNodeRecord(hub) if err != nil { t.Fatal(err) diff --git a/go/delivery/signers.go b/go/delivery/signers.go new file mode 100644 index 0000000..52834a2 --- /dev/null +++ b/go/delivery/signers.go @@ -0,0 +1,328 @@ +package delivery + +import ( + "crypto/ed25519" + "crypto/sha256" + "crypto/x509" + "crypto/x509/pkix" + "encoding/base64" + "encoding/hex" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "math/big" + "slices" + "strings" + "time" +) + +// **The vault signs** (novox/hq ADR 0550 §2; design 47's L3 item d). The vault makes an ed25519 signing key and an x509 +// authority once, keeps both as its own secrets with their operator copies, and publishes their public halves in its +// state `signers`, which only the vault writes and every node principal and composing part reads. Its verb `sign` signs a +// node's certificate for that node's own node-engine, from bus state alone (the name from the node's placement, the +// serving key from its record), and says `signed` naming the caller, the kind, the node and the digest, never the key. +// +// A node trusts the vault's key through the bus grants alone (the lead's design limit, 2026-10-28): only the vault may +// write `signers`, the same trust the wants and the placement rest on. Until the controller's lease is handed away the +// node trusts the controller's key beside it; from then on the vault's alone (LeaseHandedAway). What a node trusts now +// it states in its record, NodeRecord.Trusts, so the gate before the lease write reads it from the bus. + +const ( + // VaultModule is the module holding the seat mesh-vault, whose state `signers` is. + VaultModule = "mesh-vault" + // SignersState is the vault's state of its public signing halves, and SignersBucket its bucket. + SignersState = "signers" + SignersBucket = VaultModule + "_" + SignersState + // SignersRead is the state as a manifest's `reads` names it. + SignersRead = VaultModule + "." + SignersState + // SignersKeyKey holds the signing key's public half (SignerKey); SignersAuthorityKey the authority (SignerAuthority). + SignersKeyKey = "key" + SignersAuthorityKey = "authority" + + // SignVerb is the vault's verb that signs. + SignVerb = "sign" + // SignedEvent is what the vault says when it signed (Signed), on SignedSubject. + SignedEvent = "signed" + SignedSubject = "mesh.mod." + VaultModule + ".event." + SignedEvent + + // KindNodeCertificate is a node's certificate for its internal name, asked by that node's node-engine alone. + KindNodeCertificate = "node-certificate" + // KindEnrolment is an enrolment reply, signed under its own context; no caller is named until adoption's record + // (ADR 0561 §5). + KindEnrolment = "enrolment" + // KindAuthority is the vault's own authority certificate, which the vault signs for itself. + KindAuthority = "authority" + + // SignerController and SignerVault name the two signers a node may trust. + SignerController = "controller" + SignerVault = "vault" + + // CertificateLife is how long a certificate the vault signs for a node lasts: the controller's ten years + // (identity.forever), renewed by asking again, never by a clock running out unseen. + CertificateLife = 10 * 365 * 24 * time.Hour +) + +// SignKinds are the kinds `sign` knows. +var SignKinds = []string{KindNodeCertificate, KindEnrolment, KindAuthority} + +// SignContext is the line a message of kind is signed after, so a signature made for one kind never verifies as +// another's (as the composition's and the asks' contexts are kept apart). +func SignContext(kind string) (string, error) { + if !slices.Contains(SignKinds, kind) { + return "", fmt.Errorf("delivery: %q is not a kind the vault signs (%s)", kind, strings.Join(SignKinds, ", ")) + } + return "novox " + kind + " v1\n", nil +} + +// SignerKey is the key `key` of `signers`: the vault's signing key's public half in standard base64, its fingerprint, +// and when it was made. +type SignerKey struct { + Key string `json:"key"` + Fingerprint string `json:"fingerprint"` + Made time.Time `json:"made"` +} + +// SignerAuthority is the key `authority` of `signers`: the vault's authority certificate as PEM, its fingerprint, and +// when it was made. +type SignerAuthority struct { + Certificate string `json:"certificate"` + Fingerprint string `json:"fingerprint"` + Made time.Time `json:"made"` +} + +// SignerFingerprint is an ed25519 public key's fingerprint, `sha256:` of its 32 bytes. +func SignerFingerprint(key ed25519.PublicKey) string { + sum := sha256.Sum256(key) + return "sha256:" + hex.EncodeToString(sum[:]) +} + +// AuthorityFingerprint is an authority certificate's fingerprint, `sha256:` of its DER; refused when certificate is +// not one PEM certificate. +func AuthorityFingerprint(certificate string) (string, error) { + c, err := ParseCertificate(certificate) + if err != nil { + return "", err + } + sum := sha256.Sum256(c.Raw) + return "sha256:" + hex.EncodeToString(sum[:]), nil +} + +// ParseCertificate is the first certificate of a PEM text. +func ParseCertificate(text string) (*x509.Certificate, error) { + block, _ := pem.Decode([]byte(strings.TrimSpace(text))) + if block == nil || block.Type != "CERTIFICATE" { + return nil, errors.New("delivery: not a PEM certificate") + } + return x509.ParseCertificate(block.Bytes) +} + +// NewSignerKey is the value of `key` for pub. +func NewSignerKey(pub ed25519.PublicKey, made time.Time) SignerKey { + return SignerKey{Key: base64.StdEncoding.EncodeToString(pub), Fingerprint: SignerFingerprint(pub), Made: made.UTC()} +} + +// PublicKey is k's key, checked: an ed25519 public key whose fingerprint is the one k names. +func (k SignerKey) PublicKey() (ed25519.PublicKey, error) { + raw, err := base64.StdEncoding.DecodeString(k.Key) + if err != nil || len(raw) != ed25519.PublicKeySize { + return nil, errors.New("delivery: the signers' key is not an ed25519 public key in base64") + } + if fp := SignerFingerprint(raw); fp != k.Fingerprint { + return nil, fmt.Errorf("delivery: the signers' key has the fingerprint %s, and names %s", fp, k.Fingerprint) + } + return ed25519.PublicKey(raw), nil +} + +// DecodeSignerKey reads the key `key` of `signers`, refused when it does not check. +func DecodeSignerKey(raw []byte) (SignerKey, error) { + var k SignerKey + if err := json.Unmarshal(raw, &k); err != nil { + return SignerKey{}, fmt.Errorf("delivery: %s %s: %w", SignersRead, SignersKeyKey, err) + } + if _, err := k.PublicKey(); err != nil { + return SignerKey{}, err + } + return k, nil +} + +// DecodeSignerAuthority reads the key `authority` of `signers`: one authority certificate (a CA that signs leaves +// alone) whose fingerprint is the one named. +func DecodeSignerAuthority(raw []byte) (SignerAuthority, error) { + var a SignerAuthority + if err := json.Unmarshal(raw, &a); err != nil { + return SignerAuthority{}, fmt.Errorf("delivery: %s %s: %w", SignersRead, SignersAuthorityKey, err) + } + c, err := ParseCertificate(a.Certificate) + if err != nil { + return SignerAuthority{}, err + } + if !c.IsCA { + return SignerAuthority{}, errors.New("delivery: the signers' authority is not an authority certificate") + } + if fp, _ := AuthorityFingerprint(a.Certificate); fp != a.Fingerprint { + return SignerAuthority{}, fmt.Errorf("delivery: the signers' authority has the fingerprint %s, and names %s", fp, a.Fingerprint) + } + return a, nil +} + +// Signer is one key a node trusts, by who holds it (SignerController, SignerVault, or the operator's). +type Signer struct { + Name string + Key ed25519.PublicKey +} + +// Fingerprint is the signer's key's fingerprint. +func (s Signer) Fingerprint() string { return SignerFingerprint(s.Key) } + +// VerifyBy is the trusted signer whose key verifies sig over context and msg: refused when no trusted key does, saying +// so, and when a key is not an ed25519 key. A signature made under another context never verifies. +func VerifyBy(trusted []Signer, context string, msg, sig []byte) (Signer, error) { + if len(sig) != ed25519.SignatureSize { + return Signer{}, errors.New("delivery: not an ed25519 signature") + } + message := append([]byte(context), msg...) + for _, s := range trusted { + if len(s.Key) != ed25519.PublicKeySize { + continue + } + if ed25519.Verify(s.Key, message, sig) { + return s, nil + } + } + names := make([]string, 0, len(trusted)) + for _, s := range trusted { + names = append(names, s.Name) + } + return Signer{}, fmt.Errorf("delivery: the signature verifies against no trusted key (%s)", strings.Join(names, ", ")) +} + +// SignMessage is sig over context and msg by key: what the vault's `sign` does for a kind whose bytes are given. +func SignMessage(key ed25519.PrivateKey, context string, msg []byte) []byte { + return ed25519.Sign(key, append([]byte(context), msg...)) +} + +// Signed is the vault's `signed`: who asked, the kind, the node, and the sha256 digest of what was signed. Never a key. +type Signed struct { + Caller string `json:"caller"` + Kind string `json:"kind"` + Node string `json:"node,omitempty"` + Digest string `json:"digest"` + Signer string `json:"signer"` + At time.Time `json:"at"` +} + +// SignAnswer is what `sign` answers: for a node certificate the certificate as PEM and the authority that signed it, +// for an enrolment the signature in base64; always the signer's fingerprint and the digest. Never a key's private half. +type SignAnswer struct { + Kind string `json:"kind"` + Node string `json:"node,omitempty"` + Certificate string `json:"certificate,omitempty"` + Authority string `json:"authority,omitempty"` + Signature string `json:"signature,omitempty"` + Signer string `json:"signer"` + Digest string `json:"digest"` +} + +// DigestOf is the digest a `signed` names: `sha256:` of the bytes signed. +func DigestOf(b []byte) string { + sum := sha256.Sum256(b) + return "sha256:" + hex.EncodeToString(sum[:]) +} + +// NodeCertificateTemplate is the certificate the vault signs for node's internal name, as the controller's authority +// shaped it (identity.Certify): the name in the subject and the subject alternative names, server and client use, the +// serial derived from the name and the key so the same ask gives the same certificate, and its life from notBefore. +func NodeCertificateTemplate(name string, servingKey ed25519.PublicKey, notBefore time.Time) *x509.Certificate { + sum := sha256.Sum256(append([]byte(name+"\n"), servingKey...)) + return &x509.Certificate{ + SerialNumber: new(big.Int).SetBytes(sum[:16]), + Subject: pkix.Name{CommonName: name}, + DNSNames: []string{name}, + NotBefore: notBefore.UTC().Truncate(time.Second), + NotAfter: notBefore.UTC().Truncate(time.Second).Add(CertificateLife), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + } +} + +// AuthorityTemplate is the vault's authority certificate for its key, made once: an authority that signs leaves and no +// other authority, as the controller's. +func AuthorityTemplate(key ed25519.PublicKey, made time.Time) *x509.Certificate { + sum := sha256.Sum256(append([]byte("authority\n"), key...)) + return &x509.Certificate{ + SerialNumber: new(big.Int).SetBytes(sum[:16]), + Subject: pkix.Name{CommonName: "the mesh's vault"}, + NotBefore: made.UTC().Truncate(time.Second), + NotAfter: made.UTC().Truncate(time.Second).Add(2 * CertificateLife), + IsCA: true, + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign, + BasicConstraintsValid: true, + MaxPathLen: 0, + MaxPathLenZero: true, + } +} + +// CheckNodeCertificate says why certificate is not a certificate of name for servingKey by one of the authorities, or +// nil: a node uses it to tell whether its record holds a certificate from an authority it trusts, and another node to +// check one the vault signed (ADR 0550 §2's live check). +func CheckNodeCertificate(certificate string, authorities []string, name string, servingKey ed25519.PublicKey, at time.Time) error { + c, err := ParseCertificate(certificate) + if err != nil { + return err + } + pool := x509.NewCertPool() + for _, a := range authorities { + if ac, err := ParseCertificate(a); err == nil { + pool.AddCert(ac) + } + } + if _, err := c.Verify(x509.VerifyOptions{Roots: pool, DNSName: name, CurrentTime: at, + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}}); err != nil { + return fmt.Errorf("delivery: the certificate of %s is not one a trusted authority signed for it: %w", name, err) + } + held, ok := c.PublicKey.(ed25519.PublicKey) + if !ok || !held.Equal(servingKey) { + return fmt.Errorf("delivery: the certificate of %s binds another key than the node serves with", name) + } + return nil +} + +// AuthorityBundle is the authorities given as one PEM text, each once, in the order given, blanks left out: what a +// module asking for the mesh's authority is given once the vault signs beside the controller (ADR 0550 §2). +func AuthorityBundle(authorities ...string) string { + var out []string + for _, a := range authorities { + a = strings.TrimSpace(a) + if a == "" || slices.Contains(out, a) { + continue + } + out = append(out, a) + } + if len(out) == 0 { + return "" + } + return strings.Join(out, "\n") + "\n" +} + +// Trusts is what a node states it trusts now (NodeRecord.Trusts): ` key ` for each signer and +// ` authority ` for each authority certificate, by who holds it, each once, sorted. The gate before the +// lease write reads `vault key ` in every node's record. +func Trusts(signers []Signer, authorities map[string]string) []string { + var out []string + for _, s := range signers { + if len(s.Key) == ed25519.PublicKeySize { + out = append(out, s.Name+" key "+s.Fingerprint()) + } + } + for who, a := range authorities { + if fp, err := AuthorityFingerprint(a); err == nil { + out = append(out, who+" authority "+fp) + } + } + slices.Sort(out) + return slices.Compact(out) +} + +// TrustsKey says whether trusts names the key with fingerprint fp held by who. +func TrustsKey(trusts []string, who, fp string) bool { return slices.Contains(trusts, who+" key "+fp) } From c07cdaf2f4b1fd55f25078fbc0b6668784bd03eb Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 28 Oct 2026 09:27:17 +0100 Subject: [PATCH 9/9] =?UTF-8?q?Bundle=20the=20mesh's=20authorities=20by=20?= =?UTF-8?q?certificate,=20and=20check=20a=20node's=20certificate=20against?= =?UTF-8?q?=20every=20one=20a=20bundle=20holds=20(issue=20774,=20ADR=20055?= =?UTF-8?q?0=20=C2=A72)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go/delivery/composed_signature_test.go | 38 ++++++++++++++++++++++++ go/delivery/signers.go | 40 ++++++++++++++++++++------ 2 files changed, 69 insertions(+), 9 deletions(-) diff --git a/go/delivery/composed_signature_test.go b/go/delivery/composed_signature_test.go index 4d411bd..3098e27 100644 --- a/go/delivery/composed_signature_test.go +++ b/go/delivery/composed_signature_test.go @@ -3,7 +3,9 @@ package delivery import ( "crypto/ed25519" "crypto/rand" + "crypto/x509" "encoding/json" + "encoding/pem" "strings" "testing" "time" @@ -149,4 +151,40 @@ func TestACompositionIsSignedByAKeptKey(t *testing.T) { !TrustsKey(got, SignerController, SignerFingerprint(controllerPub)) { t.Errorf("trusts: %v", got) } + // A certificate the vault's authority signs for a node's name and serving key checks against a bundle holding that + // authority, and not for another name, another key, or against the controller's alone; a bundle given again adds no + // certificate twice. + authority := func(pub ed25519.PublicKey, priv ed25519.PrivateKey) string { + tpl := AuthorityTemplate(pub, time.Now().Add(-time.Hour)) + der, err := x509.CreateCertificate(nil, tpl, tpl, pub, priv) + if err != nil { + t.Fatal(err) + } + return string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})) + } + vaultCA, controllerCA := authority(vaultPub, vault), authority(controllerPub, controller) + parent, _ := ParseCertificate(vaultCA) + serving, _ := aSigningKey(t) + der, err := x509.CreateCertificate(nil, NodeCertificateTemplate("ace.internal", serving, time.Now().Add(-time.Hour)), parent, serving, vault) + if err != nil { + t.Fatal(err) + } + leaf := string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})) + bundle := AuthorityBundle(controllerCA, AuthorityBundle(controllerCA, vaultCA)) + if n := strings.Count(bundle, "BEGIN CERTIFICATE"); n != 2 { + t.Errorf("the bundle holds %d certificates, want 2", n) + } + now := time.Now() + if err := CheckNodeCertificate(leaf, []string{bundle}, "ace.internal", serving, now); err != nil { + t.Errorf("the vault's certificate does not check: %v", err) + } + for name, err := range map[string]error{ + "another name": CheckNodeCertificate(leaf, []string{vaultCA}, "g14.internal", serving, now), + "another key": CheckNodeCertificate(leaf, []string{vaultCA}, "ace.internal", controllerPub, now), + "the controller's alone": CheckNodeCertificate(leaf, []string{controllerCA}, "ace.internal", serving, now), + } { + if err == nil { + t.Errorf("%s: checked", name) + } + } } diff --git a/go/delivery/signers.go b/go/delivery/signers.go index 52834a2..811ea13 100644 --- a/go/delivery/signers.go +++ b/go/delivery/signers.go @@ -273,9 +273,7 @@ func CheckNodeCertificate(certificate string, authorities []string, name string, } pool := x509.NewCertPool() for _, a := range authorities { - if ac, err := ParseCertificate(a); err == nil { - pool.AddCert(ac) - } + pool.AppendCertsFromPEM([]byte(a)) // a bundle is every certificate it holds } if _, err := c.Verify(x509.VerifyOptions{Roots: pool, DNSName: name, CurrentTime: at, KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}}); err != nil { @@ -288,16 +286,18 @@ func CheckNodeCertificate(certificate string, authorities []string, name string, return nil } -// AuthorityBundle is the authorities given as one PEM text, each once, in the order given, blanks left out: what a -// module asking for the mesh's authority is given once the vault signs beside the controller (ADR 0550 §2). +// AuthorityBundle is the authorities given as one PEM text, each certificate once, in the order given, blanks left out: +// what a module asking for the mesh's authority is given once the vault signs beside the controller (ADR 0550 §2). An +// authority given is split into its certificates, so a bundle given again (the one a node kept from its last +// composition) adds nothing twice; a text that holds no PEM block is taken whole. func AuthorityBundle(authorities ...string) string { var out []string for _, a := range authorities { - a = strings.TrimSpace(a) - if a == "" || slices.Contains(out, a) { - continue + for _, piece := range pemPieces(a) { + if !slices.Contains(out, piece) { + out = append(out, piece) + } } - out = append(out, a) } if len(out) == 0 { return "" @@ -305,6 +305,28 @@ func AuthorityBundle(authorities ...string) string { return strings.Join(out, "\n") + "\n" } +// pemPieces is each PEM block of text re-encoded, or the trimmed text whole when it holds none. +func pemPieces(text string) []string { + text = strings.TrimSpace(text) + if text == "" { + return nil + } + var out []string + rest := []byte(text) + for { + block, more := pem.Decode(rest) + if block == nil { + break + } + out = append(out, strings.TrimSpace(string(pem.EncodeToMemory(block)))) + rest = more + } + if len(out) == 0 { + return []string{text} + } + return out +} + // Trusts is what a node states it trusts now (NodeRecord.Trusts): ` key ` for each signer and // ` authority ` for each authority certificate, by who holds it, each once, sorted. The gate before the // lease write reads `vault key ` in every node's record.