From 2fd5ffdd6263db7577cd222010c1cad106bff337 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 27 Oct 2026 23:17:05 +0100 Subject: [PATCH 1/3] A node's answer names every module it cannot compose, a folded module of an old-form build is left to the node as it runs, and no store reached says why (issue 1126, the seed's shadow) The seed's shadow answers named the first refusal only (ace: baserow's archive, novox: de-spiegel), so each round showed one more. A build that cannot be read or taken in, and a module the composition refuses, are now each composed as a stub and named. A module folded into the node's own composition whose manifest names a build (de-spiegel, nats, photos, systemd-resolved: haves naming a build-... id) runs a build only the controller holds: it is composed as a stub and its resources are carried as the node runs them. --- go/compose/composed.go | 126 +++++++++++++++++++++++++ go/compose/frombus/answer.go | 105 ++++++++++++++++++++- go/compose/frombus/answer_test.go | 152 ++++++++++++++++++++++++++++++ go/compose/frombus/frombus.go | 102 +++++++++++++++++--- go/compose/input.go | 6 ++ 5 files changed, 476 insertions(+), 15 deletions(-) create mode 100644 go/compose/frombus/answer_test.go diff --git a/go/compose/composed.go b/go/compose/composed.go index 2c6313c..5975c25 100644 --- a/go/compose/composed.go +++ b/go/compose/composed.go @@ -8,6 +8,7 @@ import ( "sort" "strings" + "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/declaration" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/overlay" @@ -76,6 +77,42 @@ func compositions(in Input, node string, fps map[string]string, mode Mode, appli if !found { return nil, nil, nil, fmt.Errorf("compose: the mesh has no node %q", node) } + // A folded module built by a build of the old form (Node.RunsOwn) is left to the node: composed as a stub, so what + // it provides and requires still shapes the rest, and its resources are then the ones the node runs. + carried := map[string]bool{} + var runsOwn *delivery.Composed + for i := range reading.Nodes { + n := &reading.Nodes[i] + if n.Name != node || n.RunsOwn == nil { + continue + } + runsOwn = n.RunsOwn + for m := range inOwn { + if !folds(m) { + continue + } + man, ok := n.Kept[m] + if !ok { + man, ok = reading.Catalogue[m] + } + if !ok || !NamesBuild(man) { + continue + } + carried[m] = true + kept := make(map[string]catalogue.Manifest, len(n.Kept)+1) + for k, v := range n.Kept { + kept[k] = v + } + kept[m] = Stub(man) + n.Kept = kept + } + } + isCarried := func(module, id string) bool { + if module == "" { + module, _ = declaration.ModuleGuess(id) + } + return carried[module] + } d, problems, err := ComposeIn(reading, node, mode) if err != nil { return nil, nil, nil, err @@ -118,6 +155,7 @@ func compositions(in Input, node string, fps map[string]string, mode Mode, appli left := map[string]bool{} for _, raw := range envelope.Resources { var head struct { + ID string `json:"id"` Module string `json:"module"` Type string `json:"type"` Path string `json:"path"` @@ -125,6 +163,9 @@ func compositions(in Input, node string, fps map[string]string, mode Mode, appli if err := json.Unmarshal(raw, &head); err != nil { return nil, nil, nil, err } + if isCarried(head.Module, head.ID) { + continue // what the stub composed of it: the node's own carries what runs of it instead + } if folds(head.Module) { // Into the node's own, in its place among the node's resources, its module taken off (mesh-controller's // compositionsOf): the node's own composition is then the controller's, byte for byte. @@ -151,6 +192,18 @@ func compositions(in Input, node string, fps map[string]string, mode Mode, appli } kept = append(kept, raw) } + // What runs of each carried module, in the node's own composition, each after the resource it runs after. + if len(carried) > 0 { + var err error + if kept, foldedPaths, err = carry(kept, foldedPaths, runsOwn, isCarried); err != nil { + return nil, nil, nil, err + } + for _, p := range runsOwn.Filled { + if foldedPaths[p] && !slices.Contains(envelope.Filled, p) { + envelope.Filled = append(envelope.Filled, p) + } + } + } // A filled file of a module's own composition goes with that module (SplitComposed); one of a folded module goes with // the node's own, after them, in the declaration's order; one of a module left out is dropped. var filled, ownFilled []string @@ -188,3 +241,76 @@ func compositions(in Input, node string, fps map[string]string, mode Mode, appli sort.Strings(skipped) return out, skipped, stood, nil } + +// NamesBuild says a manifest names what a build makes: its build, or a resource naming an artifact. A node cannot +// compose such a module folded into its own composition: the build of the old form it runs is held by the controller +// alone (issue 1126). +func NamesBuild(m catalogue.Manifest) bool { + if m.Build != nil { + return true + } + for _, r := range m.Resources { + if _, ok := r["artifact"]; ok { + return true + } + } + return false +} + +// Stub is m with nothing to build and no resource: what it provides, requires, serves and grants still shape the rest of +// the composition, and nothing of its own is composed (issue 1126). +func Stub(m catalogue.Manifest) catalogue.Manifest { + m.Build = nil + m.Resources = nil + return m +} + +// carry puts into the node's resources (kept, in the declaration's order) the resources of the node's own composition as +// it runs (runs) that belong to a carried module, each after the resource it runs after, or first when that one is +// not among them; a file among them is a folded path. +func carry(kept []json.RawMessage, foldedPaths map[string]bool, runs *delivery.Composed, carried func(module, id string) bool) ([]json.RawMessage, map[string]bool, error) { + idOf := func(raw json.RawMessage) (string, string, string, string, error) { + var head struct { + ID string `json:"id"` + Module string `json:"module"` + Type string `json:"type"` + Path string `json:"path"` + } + err := json.Unmarshal(raw, &head) + return head.ID, head.Module, head.Type, head.Path, err + } + present := map[string]bool{} + for _, raw := range kept { + if id, module, _, _, err := idOf(raw); err == nil && module == "" { + present[id] = true + } + } + // By the id of the resource of the node's own it runs after, "" for first: the last one before it that is composed. + after := map[string][]json.RawMessage{} + anchor := "" + for _, raw := range runs.Resources { + id, module, typ, path, err := idOf(raw) + if err != nil { + return nil, nil, fmt.Errorf("compose: a resource the node runs in its own composition: %w", err) + } + if !carried(module, id) { + if module == "" && present[id] { + anchor = id + } + continue + } + after[anchor] = append(after[anchor], raw) + if typ == string(declaration.TypeFile) { + foldedPaths[path] = true + } + } + out := make([]json.RawMessage, 0, len(kept)+len(runs.Resources)) + out = append(out, after[""]...) + for _, raw := range kept { + out = append(out, raw) + if id, module, _, _, err := idOf(raw); err == nil && module == "" { + out = append(out, after[id]...) + } + } + return out, foldedPaths, nil +} diff --git a/go/compose/frombus/answer.go b/go/compose/frombus/answer.go index 20115be..baa549e 100644 --- a/go/compose/frombus/answer.go +++ b/go/compose/frombus/answer.go @@ -4,9 +4,11 @@ import ( "context" "fmt" "regexp" + "slices" "sort" "strings" + "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/compose" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -32,7 +34,13 @@ func Answer(ctx context.Context, r Reader, mesh Mesh, s Scope, fps map[string]st if err != nil { return refuse(fmt.Errorf("frombus: placement: %w", err)) } - in, err := Input(ctx, r, mesh, s) + if s.Running == nil { + s.Running = running + } + // Every module it cannot compose is named, not the first only (issue 1126): a build it cannot read or take in, and + // below, a module the composition refuses, each composed as a stub so the rest is composed and compared. + failed := map[string]string{} + in, err := input(ctx, r, mesh, s, func(module string, err error) { failed[module] = err.Error() }) if err != nil { return refuse(err) } @@ -63,8 +71,20 @@ func Answer(ctx context.Context, r Reader, mesh Mesh, s Scope, fps map[string]st } } composed, skipped, stood, err := compose.CompositionsShadow(in, s.Node, fps) - if err != nil { - return refuse(err) + for err != nil { + module := culprit(in, s.Node, fps, failed, err) + if module == "" { + // Not one module's: the node's own (a value served elsewhere, the mesh's network), said once as the node's. + if len(failed) == 0 { + return refuse(err) + } + failed[delivery.HaveNodeWord] = err.Error() + composed, skipped, stood = nil, nil, nil + break + } + failed[module] = err.Error() + in = stubbed(in, s.Node, module) + composed, skipped, stood, err = compose.CompositionsShadow(in, s.Node, fps) } // The node's own composition is judged only where the node runs one: one that runs none applies it only once a want // names it (the node agent's own composer hands it on that want alone), so a candidate does not add it. @@ -79,8 +99,22 @@ func Answer(ctx context.Context, r Reader, mesh Mesh, s Scope, fps map[string]st } } } + if _, whole := failed[delivery.HaveNodeWord]; whole { + // Nothing composed to compare: every refusal is said, and nothing else. + for _, m := range sortedKeys(failed) { + a.Refusals = append(a.Refusals, delivery.ModuleRefused{Module: m, Why: failed[m]}) + } + a.Verdict = delivery.VerdictOf(a) + return a + } cmp := delivery.CompareComposed(composed, running) - a.Same, a.Differs, a.Removed = cmp.Same, cmp.Differs, cmp.Removed + // A refused module was composed as a stub: what it composed is no difference; its refusal is said. + notFailed := func(m string) bool { _, ok := failed[m]; return !ok } + a.Same = slices.DeleteFunc(cmp.Same, func(m string) bool { return !notFailed(m) }) + a.Removed = slices.DeleteFunc(cmp.Removed, func(m string) bool { return !notFailed(m) }) + a.Differs = slices.DeleteFunc(cmp.Differs, func(d delivery.ModuleDiffers) bool { return !notFailed(d.Module) }) + cmp.Added = slices.DeleteFunc(cmp.Added, func(d delivery.ModuleAdded) bool { return !notFailed(d.Module) }) + skipped = slices.DeleteFunc(skipped, func(m string) bool { return !notFailed(m) }) // What the group adds, by node and module: placed by a candidate and not by the trunk on that node. added := map[string]bool{} @@ -138,8 +172,14 @@ func Answer(ctx context.Context, r Reader, mesh Mesh, s Scope, fps map[string]st if _, isAdded := addedHere[m]; isAdded { continue // a module the candidate adds here: what it lacks is said under Added } + if _, isFailed := failed[m]; isFailed { + continue // said below, as why it could not be composed + } a.Refusals = append(a.Refusals, delivery.ModuleRefused{Module: m, Why: "stands in for " + strings.Join(refusals[m], ", ")}) } + for _, m := range sortedKeys(failed) { + a.Refusals = append(a.Refusals, delivery.ModuleRefused{Module: m, Why: failed[m]}) + } sort.Slice(a.Refusals, func(i, j int) bool { return a.Refusals[i].Module < a.Refusals[j].Module }) a.Verdict = delivery.VerdictOf(a) return a @@ -170,3 +210,60 @@ func standsInFor(x, node string) (string, string) { } return node, "" } + +// culprit is the module placed on node whose composition the error is of: the first, in placement order, whose stub +// (compose.Stub) makes the composition succeed or fail otherwise; empty when none does (the error is not one module's). +// Modules already refused are passed over. +func culprit(in compose.Input, node string, fps map[string]string, failed map[string]string, err error) string { + var assigned []string + for _, n := range in.Nodes { + if n.Name == node { + assigned = n.Assigned + } + } + // The modules the error names first: the words of a module's refusal open with its name. + order := make([]string, 0, len(assigned)) + for _, m := range assigned { + if strings.Contains(err.Error(), m) { + order = append(order, m) + } + } + for _, m := range assigned { + if !strings.Contains(err.Error(), m) { + order = append(order, m) + } + } + for _, m := range order { + if _, done := failed[m]; done { + continue + } + if _, _, _, again := compose.CompositionsShadow(stubbed(in, node, m), node, fps); again == nil || again.Error() != err.Error() { + return m + } + } + return "" +} + +// stubbed is in with module composed as a stub on node alone (compose.Stub, as node's Kept). +func stubbed(in compose.Input, node, module string) compose.Input { + out := in + out.Nodes = append([]compose.Node(nil), in.Nodes...) + for i := range out.Nodes { + if out.Nodes[i].Name != node { + continue + } + man, ok := out.Nodes[i].Kept[module] + if !ok { + if man, ok = in.Catalogue[module]; !ok { + return in + } + } + kept := make(map[string]catalogue.Manifest, len(out.Nodes[i].Kept)+1) + for k, v := range out.Nodes[i].Kept { + kept[k] = v + } + kept[module] = compose.Stub(man) + out.Nodes[i].Kept = kept + } + return out +} diff --git a/go/compose/frombus/answer_test.go b/go/compose/frombus/answer_test.go new file mode 100644 index 0000000..1f6ab25 --- /dev/null +++ b/go/compose/frombus/answer_test.go @@ -0,0 +1,152 @@ +package frombus + +import ( + "context" + "encoding/json" + "reflect" + "slices" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/compose" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/overlay" +) + +// Issue 1126 (the seed's shadow, round 2): a node's answer names every module it cannot compose, not the first only; +// and a module folded into its own composition that names a build of the old form (de-spiegel on novox, nats, photos, +// systemd-resolved: their haves name a build-... id and no fingerprint) is left to the node, carried as it runs. +func TestTheAnswerNamesEveryRefusalAndLeavesAnOldBuildToTheNode(t *testing.T) { + ctx := context.Background() + image := "alpine@sha256:" + strings.Repeat("a", 64) + manifest := func(module string) []byte { + return []byte(`{"module":"` + module + `","version":"1","capabilities":["container-runtime"],` + + `"resources":[{"id":"state","type":"directory","path":"/var/lib/` + module + `","mode":"0700"},` + + `{"id":"server","type":"container","name":"` + module + `","artifact":"site"}],` + + `"build":{"artifacts":[{"name":"site","kind":"upstream","from":"` + image + `"}]}}`) + } + lemurs := []byte(`{"module":"lemurs","version":"1","resources":[{"id":"service","type":"service","unit":"lemurs.service","state":"running","boot":"enabled"}]}`) + fp := func(c string) string { return "src2:" + strings.Repeat(c, 64) } + mk := func(placed ...string) *bus { + p := delivery.Placement{Node: "n1", Decided: delivery.NodeDecided{Hub: true, Address: "10.77.0.1"}} + for _, m := range placed { + p.Modules = append(p.Modules, delivery.PlacedModule{Module: m}) + } + b := &bus{placements: map[string]delivery.Placement{"n1": p}, + records: map[string]delivery.NodeRecord{"n1": {Node: "n1", TunnelKey: publicKey(t), SealingKey: "sk", Capabilities: []string{"systemd", "container-runtime"}}}, + modules: map[string]delivery.Module{"lemurs": {Manifest: delivery.ManifestDigest(lemurs)}}, + manifests: map[string][]byte{delivery.ManifestDigest(lemurs): lemurs}, + builds: map[string]*delivery.BuilderBuild{}} + for _, m := range []string{"web", "api", "spiegel"} { + raw := manifest(m) + b.manifests[delivery.ManifestDigest(raw)] = raw + b.modules[m] = delivery.Module{Manifest: delivery.ManifestDigest(raw)} + } + // web and api are built, their keys done and waiting for the reference of their image; spiegel has no build of + // the new form at all. + for i, m := range []string{"web", "api"} { + mod := b.modules[m] + mod.FP2 = fp(string(rune('1' + i))) + b.modules[m] = mod + b.builds[m+" "+mod.FP2] = &delivery.BuilderBuild{State: delivery.BuildDone, Manifest: mod.Manifest, + Artifacts: []delivery.BuiltArtifact{{Name: "site", Kind: "image", Digest: "sha256:" + strings.Repeat("b", 64)}}} + } + return b + } + given := func(b *bus) map[string]*delivery.Placement { + p := b.placements["n1"] + return map[string]*delivery.Placement{"n1": &p} + } + + // Every refusal: two builds taken in, and a module the composition refuses (its image never built), each named. + b := mk("lemurs", "web", "api", "spiegel") + a := Answer(ctx, b, Mesh{}, Scope{Node: "n1", Placements: given(b)}, + map[string]string{"lemurs": fp("e"), "web": fp("1"), "api": fp("2"), "spiegel": fp("3")}, nil, "h1", 3) + var refused []string + for _, r := range a.Refusals { + refused = append(refused, r.Module) + } + if a.Verdict != delivery.CandidateRefused || a.Refused != "" || !reflect.DeepEqual(refused, []string{"api", "spiegel", "web"}) { + t.Errorf("every refusal: %+v, want api, spiegel and web refused, each named", a) + } + for _, r := range a.Refusals { + want := map[string]string{"api": "waits for the reference of site", "web": "waits for the reference of site", + "spiegel": "spiegel has not been built for this mesh"}[r.Module] + if !strings.Contains(r.Why, want) { + t.Errorf("%s refused as %q, want %q", r.Module, r.Why, want) + } + } + if raw, err := delivery.EncodeHaveNode(delivery.HaveNode{Candidates: map[string]delivery.CandidateAnswer{"g": a}}); err != nil { + t.Errorf("the answer is not one a have holds: %v (%s)", err, raw) + } + if lemursOK := slices.ContainsFunc(a.Added, func(m delivery.ModuleAdded) bool { return m.Module == "lemurs" }); !lemursOK { + t.Errorf("lemurs, which composes, is not answered beside the refusals: %+v", a) + } + + // The fold: spiegel runs inside the node's own composition at a build of the old form. Its resources are carried as + // they run, each after the one it runs after: the node's own is the one it runs, and the node composes itself. + b = mk("lemurs", overlay.Name, "spiegel") + p := b.placements["n1"] + p.Decided.Endpoint = "n1.example:51820" + b.placements["n1"] = p + fps := map[string]string{"lemurs": fp("e")} + stub := Scope{Node: "n1"} + in, err := Input(ctx, b, Mesh{}, stub) + if err != nil { + t.Fatal(err) + } + in.Nodes[0].Kept = map[string]catalogue.Manifest{"spiegel": compose.Stub(in.Catalogue["spiegel"])} + base, _, err := compose.Compositions(in, "n1", fps, compose.Reading) + if err != nil { + t.Fatal(err) + } + ownKey, _ := delivery.ComposedNodeKey("n1") + own := base[ownKey] + carriedRes := []json.RawMessage{ + json.RawMessage(`{"id":"spiegel.state","type":"directory","path":"/var/lib/spiegel","mode":"0700"}`), + json.RawMessage(`{"id":"spiegel.server","type":"container","name":"spiegel","image":"registry.example/spiegel@sha256:` + strings.Repeat("c", 64) + `"}`)} + // They run after the node's first own resource. + own.Resources = append(append(append([]json.RawMessage{}, own.Resources[:1]...), carriedRes...), own.Resources[1:]...) + var running []delivery.Composed + for k, c := range base { + if k == ownKey { + c = own + } + running = append(running, c) + } + scope := Scope{Node: "n1", InOwn: []string{"spiegel"}, Running: running} + got, _, err := Compose(ctx, b, Mesh{}, scope, fps) + if err != nil { + t.Fatalf("the node does not compose itself with spiegel folded at a build of the old form: %v", err) + } + gotOwn, _ := json.Marshal(got[ownKey].Resources) + wantOwn, _ := json.Marshal(own.Resources) + if string(gotOwn) != string(wantOwn) { + t.Errorf("the node's own composition is\n%s\nwant what runs\n%s", gotOwn, wantOwn) + } + a = Answer(ctx, b, Mesh{}, Scope{Node: "n1", Placements: given(b), InOwn: []string{"spiegel"}}, fps, running, "h1", 3) + if a.Verdict != delivery.CandidateSame { + t.Errorf("the fold's answer: %+v, want the same", a) + } + // Without what runs, the fold composes spiegel from its manifest, as before, and the composition refuses it. + if _, _, err := Compose(ctx, b, Mesh{}, Scope{Node: "n1", InOwn: []string{"spiegel"}}, fps); err == nil || + !strings.Contains(err.Error(), "spiegel has not been built for this mesh") { + t.Errorf("composed spiegel without what runs: %v", err) + } +} + +// A node that reaches no artifact store says what its composition holds of one (issue 1126: ace and g14 answered +// "reaches no artifact store", and nothing said why). +func TestNoStoreReachedSaysWhy(t *testing.T) { + in := compose.Input{Catalogue: map[string]catalogue.Manifest{ + "distribution": {Module: "distribution", Serves: map[string]map[string]any{catalogue.ArtifactStoreProvision: {"port": 5000}}}}, + Nodes: []compose.Node{{Name: "g14", Assigned: []string{"adwaita"}}, {Name: "novox", Assigned: []string{"distribution"}}}} + if got := storeSought(in); got != "the composition holds g14, novox; artifact-store is served by distribution on novox, none of it on the private network with a port" { + t.Errorf("said %q", got) + } + in.Nodes = in.Nodes[:1] + if got := storeSought(in); got != "the composition holds g14; no module placed there serves artifact-store" { + t.Errorf("said %q", got) + } +} diff --git a/go/compose/frombus/frombus.go b/go/compose/frombus/frombus.go index 0c1c28f..7aef315 100644 --- a/go/compose/frombus/frombus.go +++ b/go/compose/frombus/frombus.go @@ -113,12 +113,17 @@ type Mesh struct { // // InOwn is the modules the composing node runs inside its own composition (InOwn of what it runs), composed there again // (compose.Node.InOwn) rather than cut out as modules no build is given for (issue 1126). +// +// Running is what the composing node runs; of it, its own composition (`.node`) is given to the composition +// (compose.Node.RunsOwn), so a module folded there that names a build of the old form is left to the node, carried +// as it runs (issue 1126). Nil composes every folded module from its manifest. type Scope struct { Node string Every bool KeptAt map[string]string Placements map[string]*delivery.Placement InOwn []string + Running []delivery.Composed } // InOwn is, of the modules p places on its node, those whose resources the node's own composition it runs @@ -154,6 +159,13 @@ func InOwn(running []delivery.Composed, p delivery.Placement) []string { // Input is the composition input for s, read through r. func Input(ctx context.Context, r Reader, mesh Mesh, s Scope) (compose.Input, error) { + return input(ctx, r, mesh, s, nil) +} + +// input is Input; with refuse set, a module placed on the composing node whose build cannot be read or taken in is +// handed to refuse and composed as a stub (compose.Stub, kept for that node alone) rather than ending the read, so a +// node's answer names every module it cannot compose (issue 1126). +func input(ctx context.Context, r Reader, mesh Mesh, s Scope, refuse func(module string, err error)) (compose.Input, error) { placements, err := r.Placements(ctx) if err != nil { return compose.Input{}, fmt.Errorf("frombus: placement: %w", err) @@ -280,6 +292,12 @@ func Input(ctx context.Context, r Reader, mesh Mesh, s Scope) (compose.Input, er } n.InOwn[module] = true } + for i := range s.Running { + if s.Running[i].Module == "" { + own := s.Running[i] + n.RunsOwn = &own + } + } } in.Nodes = append(in.Nodes, n) in.Holdings = append(in.Holdings, p.Held()...) @@ -309,7 +327,7 @@ func Input(ctx context.Context, r Reader, mesh Mesh, s Scope) (compose.Input, er if _, ok := records[s.Node]; !ok { return compose.Input{}, fmt.Errorf("frombus: %s has written no record of itself yet", s.Node) } - if err := fromBuilds(ctx, r, &in, s, placements[s.Node], modules, read, parse); err != nil { + if err := fromBuilds(ctx, r, &in, s, placements[s.Node], modules, read, parse, refuse); err != nil { return compose.Input{}, err } // Its own modules' secrets, sealed to its own key by the vault: a composition carries them sealed (ADR 0550 ยง5). @@ -358,7 +376,29 @@ func Compose(ctx context.Context, r Reader, mesh Mesh, s Scope, fps map[string]s // by the one take-in rule (takein.Manifest), a templates archive read in the artifact store as the node reaches it // (issue 1126). func fromBuilds(ctx context.Context, r Reader, in *compose.Input, s Scope, p delivery.Placement, modules map[string]delivery.Module, - read func(module, digest string) (catalogue.Manifest, error), parse func([]byte) (catalogue.Manifest, error)) error { + read func(module, digest string) (catalogue.Manifest, error), parse func([]byte) (catalogue.Manifest, error), + refuse func(module string, err error)) error { + // stub keeps a refused module in the composition as a stub for this node alone (compose.Stub): what it provides + // and requires still shapes the rest. + stub := func(module string, err error) { + refuse(module, err) + for i := range in.Nodes { + if in.Nodes[i].Name != s.Node { + continue + } + man, ok := in.Nodes[i].Kept[module] + if !ok { + man, ok = in.Catalogue[module] + } + if !ok { + return + } + if in.Nodes[i].Kept == nil { + in.Nodes[i].Kept = map[string]catalogue.Manifest{} + } + in.Nodes[i].Kept[module] = compose.Stub(man) + } + } address, addressErr := "", error(nil) addressed := false archive := func(ctx context.Context, reference, digest string) ([]byte, error) { @@ -370,7 +410,7 @@ func fromBuilds(ctx context.Context, r Reader, in *compose.Input, s Scope, p del case addressErr != nil: return nil, addressErr case address == "": - return nil, fmt.Errorf("%s reaches no artifact store", s.Node) + return nil, fmt.Errorf("%s reaches no artifact store (%s)", s.Node, storeSought(*in)) } return r.Archive(ctx, catalogue.Routed(reference, address), digest) } @@ -409,14 +449,23 @@ func fromBuilds(ctx context.Context, r Reader, in *compose.Input, s Scope, p del } b, err := r.Build(ctx, pm.Module, m.FP2) if err != nil { - return fmt.Errorf("frombus: the build of %s at %s: %w", pm.Module, m.FP2, err) + err = fmt.Errorf("frombus: the build of %s at %s: %w", pm.Module, m.FP2, err) + if refuse == nil { + return err + } + stub(pm.Module, err) + continue } if b == nil || b.State != delivery.BuildDone { continue // no done build at the trunk's: composed from the indexer's manifest, as before } resolved, err := built(pm.Module, m.FP2, b) if err != nil { - return err + if refuse == nil { + return err + } + stub(pm.Module, err) + continue } in.Catalogue[pm.Module] = resolved } @@ -427,15 +476,22 @@ func fromBuilds(ctx context.Context, r Reader, in *compose.Input, s Scope, p del for _, module := range sortedKeys(s.KeptAt) { fp := s.KeptAt[module] b, err := r.Build(ctx, module, fp) - if err != nil { - return fmt.Errorf("frombus: the build of %s at %s: %w", module, fp, err) + switch { + case err != nil: + err = fmt.Errorf("frombus: the build of %s at %s: %w", module, fp, err) + case b == nil || b.State != delivery.BuildDone: + err = fmt.Errorf("frombus: %s is wanted at %s, of which the builder holds no done build", module, fp) } - if b == nil || b.State != delivery.BuildDone { - return fmt.Errorf("frombus: %s is wanted at %s, of which the builder holds no done build", module, fp) + var resolved catalogue.Manifest + if err == nil { + resolved, err = built(module, fp, b) } - resolved, err := built(module, fp, b) if err != nil { - return err + if refuse == nil { + return err + } + stub(module, err) + continue } if in.Nodes[i].Kept == nil { in.Nodes[i].Kept = map[string]catalogue.Manifest{} @@ -613,3 +669,27 @@ func sortedKeys[V any](m map[string]V) []string { sort.Strings(out) return out } + +// storeSought says, for a node that reaches no artifact store, what the composition holds of one: the nodes it holds and +// where each module serving the artifact store is placed, so the refusal names why (issue 1126). +func storeSought(in compose.Input) string { + var nodes, where []string + for _, n := range in.Nodes { + nodes = append(nodes, n.Name) + for _, a := range n.Assigned { + if m, ok := in.Catalogue[a]; ok { + if _, serves := m.Serves[catalogue.ArtifactStoreProvision]; serves { + where = append(where, a+" on "+n.Name) + } + } + } + } + sort.Strings(nodes) + sort.Strings(where) + said := "the composition holds " + strings.Join(nodes, ", ") + if len(where) == 0 { + return said + "; no module placed there serves " + catalogue.ArtifactStoreProvision + } + return said + "; " + catalogue.ArtifactStoreProvision + " is served by " + strings.Join(where, ", ") + + ", none of it on the private network with a port" +} diff --git a/go/compose/input.go b/go/compose/input.go index bf06c3c..2ca4f25 100644 --- a/go/compose/input.go +++ b/go/compose/input.go @@ -23,6 +23,7 @@ import ( "strings" "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" ) // Input is everything the composition reads. Each field is what one read of the controller's inventory answers, named @@ -194,6 +195,11 @@ type Node struct { // resources into the node's own, as the controller's composed writer does, unless fps gives one of the new form. // The private network's module (overlay.Name), which nobody builds, is folded whether or not it is named here. InOwn map[string]bool + // RunsOwn is the node's own composition (`.node`) as it runs, when the node composes itself: a module folded + // there (InOwn) whose manifest names a build (NamesBuild) runs a build of the old form only the controller holds, + // so it is left to the node: Compositions composes it as a stub (Stub) and carries its resources as they run, each + // after the resource it runs after (issue 1126). Nil composes every folded module from its manifest, as before. + RunsOwn *delivery.Composed } // Place is where a node is, as its placement says (inv.Overlay). From 472d5f314466afa910117ac8ad033f4df9ea2439 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 27 Oct 2026 23:22:54 +0100 Subject: [PATCH 2/3] A node's have says whose compositions it applies, and its answer to a head deleting its placement says it returns to the controller's (issue 1126, the seed's revert) The check of the seed's revert (mesh-catalog #526) failed on 'would stop running node': the shadow answered a deleted placement as everything removed, while the node keeps what runs. HaveNode.Composes and ComposesWhy say whether a node applies its own compositions or the controller's; CandidateAnswer.Returns answers a deleted placement, verdict same, naming nothing composed. --- go/delivery/delivery.go | 20 +++++++++++++++++++- go/delivery/placement_candidate.go | 11 +++++++++++ go/delivery/placement_test.go | 20 +++++++++++++++++--- 3 files changed, 47 insertions(+), 4 deletions(-) diff --git a/go/delivery/delivery.go b/go/delivery/delivery.go index 1b2306f..6e51c60 100644 --- a/go/delivery/delivery.go +++ b/go/delivery/delivery.go @@ -410,8 +410,20 @@ type HaveNode struct { // its shadow composer whenever an answer changes, removed when the candidate key goes. The node applies nothing of a // candidate. Candidates map[string]CandidateAnswer `json:"candidates,omitempty"` + // Composes is whose compositions the node applies: its own (ComposesOwn, the catalogue places it) or the + // controller's (ComposesController); ComposesWhy says why in words. Written by the node agent at each start, so a + // node returning to the controller's compositions when its placement key is deleted says so (issue 1126). Empty + // from an engine older than the field. + Composes string `json:"composes,omitempty"` + ComposesWhy string `json:"composes-why,omitempty"` } +// Whose compositions a node applies (HaveNode.Composes). +const ( + ComposesOwn = "own" + ComposesController = "controller" +) + // Presence is whether a node or a part's instance is online; the entry's own time says since when. The watchman // writes it. type Presence struct { @@ -506,6 +518,11 @@ func DecodeHave(data []byte) (Have, error) { } func (n HaveNode) check() error { + switch n.Composes { + case "", ComposesOwn, ComposesController: + default: + return fmt.Errorf("delivery: a node composes %q, neither %s nor %s", n.Composes, ComposesOwn, ComposesController) + } for group, a := range n.Candidates { if group == "" { return fmt.Errorf("delivery: a node's candidate answer names no group") @@ -590,7 +607,8 @@ func SameHaveNode(a, b HaveNode) bool { return a.V == b.V && a.Composed == b.Composed && sameJSON(a.Windows, b.Windows) && sameJSON(a.Busy, b.Busy) && sameJSON(a.Strays, b.Strays) && sameJSON(a.Filters, b.Filters) && sameJSON(a.Profile, b.Profile) && (a.Record == nil) == (b.Record == nil) && (a.Record == nil || SameNodeRecord(*a.Record, *b.Record)) && - jsonEqual(a.Sealed, b.Sealed) && jsonEqual(a.Candidates, b.Candidates) + jsonEqual(a.Sealed, b.Sealed) && jsonEqual(a.Candidates, b.Candidates) && a.Composes == b.Composes && + a.ComposesWhy == b.ComposesWhy } // SamePresence says whether two presences are the same value. diff --git a/go/delivery/placement_candidate.go b/go/delivery/placement_candidate.go index edf257b..e3736bc 100644 --- a/go/delivery/placement_candidate.go +++ b/go/delivery/placement_candidate.go @@ -132,8 +132,16 @@ type CandidateAnswer struct { Refusals []ModuleRefused `json:"refusals,omitempty"` // Refused is why the node could not compose with the candidate at all; Refusals is then empty. Refused string `json:"refused,omitempty"` + // Returns is, for a head that deletes the node's placement file, what the node does then: it returns to the + // controller's compositions (ReturnsToController), as a node the catalogue does not place composes none of its own + // (issue 1126: the seed's revert). The verdict is then same, and nothing else is named: the node composes nothing + // with the candidate, and judges nothing of what the controller composes. + Returns string `json:"returns,omitempty"` } +// ReturnsToController is what a node answers a head that deletes its placement file. +const ReturnsToController = "the head deletes this node's placement: it returns to the controller's compositions" + // ModuleDiffers is one module the node runs and the candidate still places whose composition is not the one it runs: // both digests (Composed with the candidate, Running what runs), the first resource that differs by its id, or empty // when every resource is equal, and why in words. @@ -187,6 +195,9 @@ func (a CandidateAnswer) check() error { if a.Refused != "" && len(a.Same)+len(a.Differs)+len(a.Added)+len(a.Removed)+len(a.Refusals) > 0 { return fmt.Errorf("delivery: a candidate the node could not compose names modules") } + if a.Returns != "" && (a.Refused != "" || len(a.Same)+len(a.Differs)+len(a.Added)+len(a.Removed)+len(a.Refusals) > 0) { + return fmt.Errorf("delivery: a candidate the node returns to the controller's compositions on names what it composed") + } for _, d := range a.Differs { if d.Module == "" || d.Why == "" { return fmt.Errorf("delivery: a candidate difference names no module or no why") diff --git a/go/delivery/placement_test.go b/go/delivery/placement_test.go index ae77b5f..29376d0 100644 --- a/go/delivery/placement_test.go +++ b/go/delivery/placement_test.go @@ -229,19 +229,31 @@ func TestPlacement(t *testing.T) { "refused": {Head: "h1", Revision: 3, Verdict: CandidateRefused, Refused: "unbound postgres"}, "added": {Head: "h1", Verdict: CandidateDiffers, Same: []string{"node"}, Added: []ModuleAdded{{Module: "app", Unissued: []string{"app/token"}}}, Removed: []string{"old"}}, "a value": {Head: "h1", Verdict: CandidateRefused, Same: []string{"node"}, Refusals: []ModuleRefused{{Module: "db", Why: "stands in for db/password"}}}, + // A head that deletes the node's placement file: the node returns to the controller's compositions (issue 1126, + // the seed's revert, mesh-catalog #526). + "returns": {Head: "h1", Verdict: CandidateSame, Returns: ReturnsToController}, } - n := HaveNode{Composed: "sha256:" + strings.Repeat("a", 64), Sealed: []string{"box"}, Candidates: answers} + n := HaveNode{Composed: "sha256:" + strings.Repeat("a", 64), Sealed: []string{"box"}, Candidates: answers, + Composes: ComposesOwn, ComposesWhy: "the catalogue places it (placement.n1)"} raw, err := EncodeHaveNode(n) if err != nil { t.Fatal(err) } back, err := DecodeHaveNode(raw) - if err != nil || !SameHaveNode(back, HaveNode{V: Version, Composed: n.Composed, Sealed: n.Sealed, Candidates: answers}) { + if err != nil || !SameHaveNode(back, HaveNode{V: Version, Composed: n.Composed, Sealed: n.Sealed, Candidates: answers, + Composes: ComposesOwn, ComposesWhy: n.ComposesWhy}) { t.Fatalf("a node's answers changed on the way: %+v, %v", back, err) } - if SameHaveNode(back, HaveNode{V: Version, Composed: n.Composed, Sealed: n.Sealed}) { + if SameHaveNode(back, HaveNode{V: Version, Composed: n.Composed, Sealed: n.Sealed, Composes: ComposesOwn, ComposesWhy: n.ComposesWhy}) { t.Error("a node's have without its answers counts as the same") } + if SameHaveNode(back, HaveNode{V: Version, Composed: n.Composed, Sealed: n.Sealed, Candidates: answers, + Composes: ComposesController, ComposesWhy: n.ComposesWhy}) { + t.Error("a node's have composing otherwise counts as the same") + } + if _, err := EncodeHaveNode(HaveNode{Composes: "both"}); err == nil { + t.Error("a node composing neither its own nor the controller's was taken") + } // CompareComposed: the bus's user list is left out on both sides; anything else that differs is named. file := func(id, content string) json.RawMessage { return json.RawMessage(`{"id":"` + id + `","type":"file","module":"nats","path":"/etc/` + id + `","mode":"0600","content":"` + content + `"}`) @@ -285,6 +297,8 @@ func TestPlacement(t *testing.T) { "same with an added": {Head: "h", Verdict: CandidateSame, Added: []ModuleAdded{{Module: "a"}}}, "differs refusing": {Head: "h", Verdict: CandidateDiffers, Removed: []string{"a"}, Refusals: []ModuleRefused{{Module: "b", Why: "x"}}}, "refused and modules": {Head: "h", Verdict: CandidateRefused, Refused: "x", Same: []string{"a"}}, + "returns and removes": {Head: "h", Verdict: CandidateDiffers, Returns: ReturnsToController, Removed: []string{"a"}}, + "returns refused": {Head: "h", Verdict: CandidateRefused, Returns: ReturnsToController, Refused: "x"}, } { if _, err := EncodeHaveNode(HaveNode{Candidates: map[string]CandidateAnswer{"g": a}}); err == nil { t.Errorf("%s: taken", name) From e32d777530247a7f8fda33dcfd3920d7a3982eea Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 27 Oct 2026 23:32:31 +0100 Subject: [PATCH 3/3] A deleting head's answer compares the controller's compositions with what runs, and is refused only without returning; fold the shadow's tests into TestInput (issue 1126, review of the shadow, round 1) --- .../frombus/{answer_test.go => answer_rows_test.go} | 4 ++-- go/compose/frombus/frombus_test.go | 4 ++++ go/delivery/placement_candidate.go | 10 ++++++---- go/delivery/placement_test.go | 6 ++++-- 4 files changed, 16 insertions(+), 8 deletions(-) rename go/compose/frombus/{answer_test.go => answer_rows_test.go} (98%) diff --git a/go/compose/frombus/answer_test.go b/go/compose/frombus/answer_rows_test.go similarity index 98% rename from go/compose/frombus/answer_test.go rename to go/compose/frombus/answer_rows_test.go index 1f6ab25..f0aa574 100644 --- a/go/compose/frombus/answer_test.go +++ b/go/compose/frombus/answer_rows_test.go @@ -17,7 +17,7 @@ import ( // Issue 1126 (the seed's shadow, round 2): a node's answer names every module it cannot compose, not the first only; // and a module folded into its own composition that names a build of the old form (de-spiegel on novox, nats, photos, // systemd-resolved: their haves name a build-... id and no fingerprint) is left to the node, carried as it runs. -func TestTheAnswerNamesEveryRefusalAndLeavesAnOldBuildToTheNode(t *testing.T) { +func answerNamesEveryRefusalAndLeavesAnOldBuildToTheNode(t *testing.T) { ctx := context.Background() image := "alpine@sha256:" + strings.Repeat("a", 64) manifest := func(module string) []byte { @@ -138,7 +138,7 @@ func TestTheAnswerNamesEveryRefusalAndLeavesAnOldBuildToTheNode(t *testing.T) { // A node that reaches no artifact store says what its composition holds of one (issue 1126: ace and g14 answered // "reaches no artifact store", and nothing said why). -func TestNoStoreReachedSaysWhy(t *testing.T) { +func noStoreReachedSaysWhy(t *testing.T) { in := compose.Input{Catalogue: map[string]catalogue.Manifest{ "distribution": {Module: "distribution", Serves: map[string]map[string]any{catalogue.ArtifactStoreProvision: {"port": 5000}}}}, Nodes: []compose.Node{{Name: "g14", Assigned: []string{"adwaita"}}, {Name: "novox", Assigned: []string{"distribution"}}}} diff --git a/go/compose/frombus/frombus_test.go b/go/compose/frombus/frombus_test.go index 20a5ae5..ee7697d 100644 --- a/go/compose/frombus/frombus_test.go +++ b/go/compose/frombus/frombus_test.go @@ -119,6 +119,10 @@ func publicKey(t *testing.T) string { // controller's store fills for compose's own two-machine mesh (compose's aMesh: anchor the hub, laptop), and that Input // composes; a node reads only its own config values; what cannot be composed is refused, naming it. func TestInput(t *testing.T) { + // Issue 1126, the seed's shadow: every refusal named, an old-form module left to the node, and no store reached + // saying why (answer_rows_test.go). + t.Run("every refusal, and an old build left to the node", answerNamesEveryRefusalAndLeavesAnOldBuildToTheNode) + t.Run("no store reached says why", noStoreReachedSaysWhy) ctx := context.Background() xterm, err := os.ReadFile("../../catalogue/testdata/catalogue-835bf22b-xterm.module.json") if err != nil { diff --git a/go/delivery/placement_candidate.go b/go/delivery/placement_candidate.go index e3736bc..6b28569 100644 --- a/go/delivery/placement_candidate.go +++ b/go/delivery/placement_candidate.go @@ -134,8 +134,9 @@ type CandidateAnswer struct { Refused string `json:"refused,omitempty"` // Returns is, for a head that deletes the node's placement file, what the node does then: it returns to the // controller's compositions (ReturnsToController), as a node the catalogue does not place composes none of its own - // (issue 1126: the seed's revert). The verdict is then same, and nothing else is named: the node composes nothing - // with the candidate, and judges nothing of what the controller composes. + // (issue 1126: the seed's revert). Same, Differs, Added and Removed then compare the controller's compositions at + // the node's wants with what it runs, and the verdict is theirs; a want the controller holds no composition for is + // refused (Refused, without Returns). Returns string `json:"returns,omitempty"` } @@ -195,8 +196,9 @@ func (a CandidateAnswer) check() error { if a.Refused != "" && len(a.Same)+len(a.Differs)+len(a.Added)+len(a.Removed)+len(a.Refusals) > 0 { return fmt.Errorf("delivery: a candidate the node could not compose names modules") } - if a.Returns != "" && (a.Refused != "" || len(a.Same)+len(a.Differs)+len(a.Added)+len(a.Removed)+len(a.Refusals) > 0) { - return fmt.Errorf("delivery: a candidate the node returns to the controller's compositions on names what it composed") + if a.Returns != "" && (a.Refused != "" || len(a.Refusals) > 0) { + return fmt.Errorf("delivery: a candidate the node returns to the controller's compositions on is compared, never refused: " + + "a composition the controller holds none of is refused without returning") } for _, d := range a.Differs { if d.Module == "" || d.Why == "" { diff --git a/go/delivery/placement_test.go b/go/delivery/placement_test.go index 29376d0..e0b7810 100644 --- a/go/delivery/placement_test.go +++ b/go/delivery/placement_test.go @@ -231,7 +231,9 @@ func TestPlacement(t *testing.T) { "a value": {Head: "h1", Verdict: CandidateRefused, Same: []string{"node"}, Refusals: []ModuleRefused{{Module: "db", Why: "stands in for db/password"}}}, // A head that deletes the node's placement file: the node returns to the controller's compositions (issue 1126, // the seed's revert, mesh-catalog #526). - "returns": {Head: "h1", Verdict: CandidateSame, Returns: ReturnsToController}, + "returns": {Head: "h1", Verdict: CandidateSame, Returns: ReturnsToController, Same: []string{"app", "node"}}, + "returns, the controller's differs": {Head: "h1", Verdict: CandidateDiffers, Returns: ReturnsToController, Same: []string{"node"}, + Differs: []ModuleDiffers{{Module: "app", Resource: "file:/etc/app", Why: "the resource differs"}}, Removed: []string{"old"}}, } n := HaveNode{Composed: "sha256:" + strings.Repeat("a", 64), Sealed: []string{"box"}, Candidates: answers, Composes: ComposesOwn, ComposesWhy: "the catalogue places it (placement.n1)"} @@ -297,7 +299,7 @@ func TestPlacement(t *testing.T) { "same with an added": {Head: "h", Verdict: CandidateSame, Added: []ModuleAdded{{Module: "a"}}}, "differs refusing": {Head: "h", Verdict: CandidateDiffers, Removed: []string{"a"}, Refusals: []ModuleRefused{{Module: "b", Why: "x"}}}, "refused and modules": {Head: "h", Verdict: CandidateRefused, Refused: "x", Same: []string{"a"}}, - "returns and removes": {Head: "h", Verdict: CandidateDiffers, Returns: ReturnsToController, Removed: []string{"a"}}, + "returns refusing": {Head: "h", Verdict: CandidateRefused, Returns: ReturnsToController, Refusals: []ModuleRefused{{Module: "b", Why: "x"}}}, "returns refused": {Head: "h", Verdict: CandidateRefused, Returns: ReturnsToController, Refused: "x"}, } { if _, err := EncodeHaveNode(HaveNode{Candidates: map[string]CandidateAnswer{"g": a}}); err == nil {