From 2fd5ffdd6263db7577cd222010c1cad106bff337 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 27 Oct 2026 23:17:05 +0100 Subject: [PATCH 1/7] A node's answer names every module it cannot compose, a folded module of an old-form build is left to the node as it runs, and no store reached says why (issue 1126, the seed's shadow) The seed's shadow answers named the first refusal only (ace: baserow's archive, novox: de-spiegel), so each round showed one more. A build that cannot be read or taken in, and a module the composition refuses, are now each composed as a stub and named. A module folded into the node's own composition whose manifest names a build (de-spiegel, nats, photos, systemd-resolved: haves naming a build-... id) runs a build only the controller holds: it is composed as a stub and its resources are carried as the node runs them. --- go/compose/composed.go | 126 +++++++++++++++++++++++++ go/compose/frombus/answer.go | 105 ++++++++++++++++++++- go/compose/frombus/answer_test.go | 152 ++++++++++++++++++++++++++++++ go/compose/frombus/frombus.go | 102 +++++++++++++++++--- go/compose/input.go | 6 ++ 5 files changed, 476 insertions(+), 15 deletions(-) create mode 100644 go/compose/frombus/answer_test.go diff --git a/go/compose/composed.go b/go/compose/composed.go index 2c6313c..5975c25 100644 --- a/go/compose/composed.go +++ b/go/compose/composed.go @@ -8,6 +8,7 @@ import ( "sort" "strings" + "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/declaration" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/overlay" @@ -76,6 +77,42 @@ func compositions(in Input, node string, fps map[string]string, mode Mode, appli if !found { return nil, nil, nil, fmt.Errorf("compose: the mesh has no node %q", node) } + // A folded module built by a build of the old form (Node.RunsOwn) is left to the node: composed as a stub, so what + // it provides and requires still shapes the rest, and its resources are then the ones the node runs. + carried := map[string]bool{} + var runsOwn *delivery.Composed + for i := range reading.Nodes { + n := &reading.Nodes[i] + if n.Name != node || n.RunsOwn == nil { + continue + } + runsOwn = n.RunsOwn + for m := range inOwn { + if !folds(m) { + continue + } + man, ok := n.Kept[m] + if !ok { + man, ok = reading.Catalogue[m] + } + if !ok || !NamesBuild(man) { + continue + } + carried[m] = true + kept := make(map[string]catalogue.Manifest, len(n.Kept)+1) + for k, v := range n.Kept { + kept[k] = v + } + kept[m] = Stub(man) + n.Kept = kept + } + } + isCarried := func(module, id string) bool { + if module == "" { + module, _ = declaration.ModuleGuess(id) + } + return carried[module] + } d, problems, err := ComposeIn(reading, node, mode) if err != nil { return nil, nil, nil, err @@ -118,6 +155,7 @@ func compositions(in Input, node string, fps map[string]string, mode Mode, appli left := map[string]bool{} for _, raw := range envelope.Resources { var head struct { + ID string `json:"id"` Module string `json:"module"` Type string `json:"type"` Path string `json:"path"` @@ -125,6 +163,9 @@ func compositions(in Input, node string, fps map[string]string, mode Mode, appli if err := json.Unmarshal(raw, &head); err != nil { return nil, nil, nil, err } + if isCarried(head.Module, head.ID) { + continue // what the stub composed of it: the node's own carries what runs of it instead + } if folds(head.Module) { // Into the node's own, in its place among the node's resources, its module taken off (mesh-controller's // compositionsOf): the node's own composition is then the controller's, byte for byte. @@ -151,6 +192,18 @@ func compositions(in Input, node string, fps map[string]string, mode Mode, appli } kept = append(kept, raw) } + // What runs of each carried module, in the node's own composition, each after the resource it runs after. + if len(carried) > 0 { + var err error + if kept, foldedPaths, err = carry(kept, foldedPaths, runsOwn, isCarried); err != nil { + return nil, nil, nil, err + } + for _, p := range runsOwn.Filled { + if foldedPaths[p] && !slices.Contains(envelope.Filled, p) { + envelope.Filled = append(envelope.Filled, p) + } + } + } // A filled file of a module's own composition goes with that module (SplitComposed); one of a folded module goes with // the node's own, after them, in the declaration's order; one of a module left out is dropped. var filled, ownFilled []string @@ -188,3 +241,76 @@ func compositions(in Input, node string, fps map[string]string, mode Mode, appli sort.Strings(skipped) return out, skipped, stood, nil } + +// NamesBuild says a manifest names what a build makes: its build, or a resource naming an artifact. A node cannot +// compose such a module folded into its own composition: the build of the old form it runs is held by the controller +// alone (issue 1126). +func NamesBuild(m catalogue.Manifest) bool { + if m.Build != nil { + return true + } + for _, r := range m.Resources { + if _, ok := r["artifact"]; ok { + return true + } + } + return false +} + +// Stub is m with nothing to build and no resource: what it provides, requires, serves and grants still shape the rest of +// the composition, and nothing of its own is composed (issue 1126). +func Stub(m catalogue.Manifest) catalogue.Manifest { + m.Build = nil + m.Resources = nil + return m +} + +// carry puts into the node's resources (kept, in the declaration's order) the resources of the node's own composition as +// it runs (runs) that belong to a carried module, each after the resource it runs after, or first when that one is +// not among them; a file among them is a folded path. +func carry(kept []json.RawMessage, foldedPaths map[string]bool, runs *delivery.Composed, carried func(module, id string) bool) ([]json.RawMessage, map[string]bool, error) { + idOf := func(raw json.RawMessage) (string, string, string, string, error) { + var head struct { + ID string `json:"id"` + Module string `json:"module"` + Type string `json:"type"` + Path string `json:"path"` + } + err := json.Unmarshal(raw, &head) + return head.ID, head.Module, head.Type, head.Path, err + } + present := map[string]bool{} + for _, raw := range kept { + if id, module, _, _, err := idOf(raw); err == nil && module == "" { + present[id] = true + } + } + // By the id of the resource of the node's own it runs after, "" for first: the last one before it that is composed. + after := map[string][]json.RawMessage{} + anchor := "" + for _, raw := range runs.Resources { + id, module, typ, path, err := idOf(raw) + if err != nil { + return nil, nil, fmt.Errorf("compose: a resource the node runs in its own composition: %w", err) + } + if !carried(module, id) { + if module == "" && present[id] { + anchor = id + } + continue + } + after[anchor] = append(after[anchor], raw) + if typ == string(declaration.TypeFile) { + foldedPaths[path] = true + } + } + out := make([]json.RawMessage, 0, len(kept)+len(runs.Resources)) + out = append(out, after[""]...) + for _, raw := range kept { + out = append(out, raw) + if id, module, _, _, err := idOf(raw); err == nil && module == "" { + out = append(out, after[id]...) + } + } + return out, foldedPaths, nil +} diff --git a/go/compose/frombus/answer.go b/go/compose/frombus/answer.go index 20115be..baa549e 100644 --- a/go/compose/frombus/answer.go +++ b/go/compose/frombus/answer.go @@ -4,9 +4,11 @@ import ( "context" "fmt" "regexp" + "slices" "sort" "strings" + "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/compose" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -32,7 +34,13 @@ func Answer(ctx context.Context, r Reader, mesh Mesh, s Scope, fps map[string]st if err != nil { return refuse(fmt.Errorf("frombus: placement: %w", err)) } - in, err := Input(ctx, r, mesh, s) + if s.Running == nil { + s.Running = running + } + // Every module it cannot compose is named, not the first only (issue 1126): a build it cannot read or take in, and + // below, a module the composition refuses, each composed as a stub so the rest is composed and compared. + failed := map[string]string{} + in, err := input(ctx, r, mesh, s, func(module string, err error) { failed[module] = err.Error() }) if err != nil { return refuse(err) } @@ -63,8 +71,20 @@ func Answer(ctx context.Context, r Reader, mesh Mesh, s Scope, fps map[string]st } } composed, skipped, stood, err := compose.CompositionsShadow(in, s.Node, fps) - if err != nil { - return refuse(err) + for err != nil { + module := culprit(in, s.Node, fps, failed, err) + if module == "" { + // Not one module's: the node's own (a value served elsewhere, the mesh's network), said once as the node's. + if len(failed) == 0 { + return refuse(err) + } + failed[delivery.HaveNodeWord] = err.Error() + composed, skipped, stood = nil, nil, nil + break + } + failed[module] = err.Error() + in = stubbed(in, s.Node, module) + composed, skipped, stood, err = compose.CompositionsShadow(in, s.Node, fps) } // The node's own composition is judged only where the node runs one: one that runs none applies it only once a want // names it (the node agent's own composer hands it on that want alone), so a candidate does not add it. @@ -79,8 +99,22 @@ func Answer(ctx context.Context, r Reader, mesh Mesh, s Scope, fps map[string]st } } } + if _, whole := failed[delivery.HaveNodeWord]; whole { + // Nothing composed to compare: every refusal is said, and nothing else. + for _, m := range sortedKeys(failed) { + a.Refusals = append(a.Refusals, delivery.ModuleRefused{Module: m, Why: failed[m]}) + } + a.Verdict = delivery.VerdictOf(a) + return a + } cmp := delivery.CompareComposed(composed, running) - a.Same, a.Differs, a.Removed = cmp.Same, cmp.Differs, cmp.Removed + // A refused module was composed as a stub: what it composed is no difference; its refusal is said. + notFailed := func(m string) bool { _, ok := failed[m]; return !ok } + a.Same = slices.DeleteFunc(cmp.Same, func(m string) bool { return !notFailed(m) }) + a.Removed = slices.DeleteFunc(cmp.Removed, func(m string) bool { return !notFailed(m) }) + a.Differs = slices.DeleteFunc(cmp.Differs, func(d delivery.ModuleDiffers) bool { return !notFailed(d.Module) }) + cmp.Added = slices.DeleteFunc(cmp.Added, func(d delivery.ModuleAdded) bool { return !notFailed(d.Module) }) + skipped = slices.DeleteFunc(skipped, func(m string) bool { return !notFailed(m) }) // What the group adds, by node and module: placed by a candidate and not by the trunk on that node. added := map[string]bool{} @@ -138,8 +172,14 @@ func Answer(ctx context.Context, r Reader, mesh Mesh, s Scope, fps map[string]st if _, isAdded := addedHere[m]; isAdded { continue // a module the candidate adds here: what it lacks is said under Added } + if _, isFailed := failed[m]; isFailed { + continue // said below, as why it could not be composed + } a.Refusals = append(a.Refusals, delivery.ModuleRefused{Module: m, Why: "stands in for " + strings.Join(refusals[m], ", ")}) } + for _, m := range sortedKeys(failed) { + a.Refusals = append(a.Refusals, delivery.ModuleRefused{Module: m, Why: failed[m]}) + } sort.Slice(a.Refusals, func(i, j int) bool { return a.Refusals[i].Module < a.Refusals[j].Module }) a.Verdict = delivery.VerdictOf(a) return a @@ -170,3 +210,60 @@ func standsInFor(x, node string) (string, string) { } return node, "" } + +// culprit is the module placed on node whose composition the error is of: the first, in placement order, whose stub +// (compose.Stub) makes the composition succeed or fail otherwise; empty when none does (the error is not one module's). +// Modules already refused are passed over. +func culprit(in compose.Input, node string, fps map[string]string, failed map[string]string, err error) string { + var assigned []string + for _, n := range in.Nodes { + if n.Name == node { + assigned = n.Assigned + } + } + // The modules the error names first: the words of a module's refusal open with its name. + order := make([]string, 0, len(assigned)) + for _, m := range assigned { + if strings.Contains(err.Error(), m) { + order = append(order, m) + } + } + for _, m := range assigned { + if !strings.Contains(err.Error(), m) { + order = append(order, m) + } + } + for _, m := range order { + if _, done := failed[m]; done { + continue + } + if _, _, _, again := compose.CompositionsShadow(stubbed(in, node, m), node, fps); again == nil || again.Error() != err.Error() { + return m + } + } + return "" +} + +// stubbed is in with module composed as a stub on node alone (compose.Stub, as node's Kept). +func stubbed(in compose.Input, node, module string) compose.Input { + out := in + out.Nodes = append([]compose.Node(nil), in.Nodes...) + for i := range out.Nodes { + if out.Nodes[i].Name != node { + continue + } + man, ok := out.Nodes[i].Kept[module] + if !ok { + if man, ok = in.Catalogue[module]; !ok { + return in + } + } + kept := make(map[string]catalogue.Manifest, len(out.Nodes[i].Kept)+1) + for k, v := range out.Nodes[i].Kept { + kept[k] = v + } + kept[module] = compose.Stub(man) + out.Nodes[i].Kept = kept + } + return out +} diff --git a/go/compose/frombus/answer_test.go b/go/compose/frombus/answer_test.go new file mode 100644 index 0000000..1f6ab25 --- /dev/null +++ b/go/compose/frombus/answer_test.go @@ -0,0 +1,152 @@ +package frombus + +import ( + "context" + "encoding/json" + "reflect" + "slices" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/compose" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/overlay" +) + +// Issue 1126 (the seed's shadow, round 2): a node's answer names every module it cannot compose, not the first only; +// and a module folded into its own composition that names a build of the old form (de-spiegel on novox, nats, photos, +// systemd-resolved: their haves name a build-... id and no fingerprint) is left to the node, carried as it runs. +func TestTheAnswerNamesEveryRefusalAndLeavesAnOldBuildToTheNode(t *testing.T) { + ctx := context.Background() + image := "alpine@sha256:" + strings.Repeat("a", 64) + manifest := func(module string) []byte { + return []byte(`{"module":"` + module + `","version":"1","capabilities":["container-runtime"],` + + `"resources":[{"id":"state","type":"directory","path":"/var/lib/` + module + `","mode":"0700"},` + + `{"id":"server","type":"container","name":"` + module + `","artifact":"site"}],` + + `"build":{"artifacts":[{"name":"site","kind":"upstream","from":"` + image + `"}]}}`) + } + lemurs := []byte(`{"module":"lemurs","version":"1","resources":[{"id":"service","type":"service","unit":"lemurs.service","state":"running","boot":"enabled"}]}`) + fp := func(c string) string { return "src2:" + strings.Repeat(c, 64) } + mk := func(placed ...string) *bus { + p := delivery.Placement{Node: "n1", Decided: delivery.NodeDecided{Hub: true, Address: "10.77.0.1"}} + for _, m := range placed { + p.Modules = append(p.Modules, delivery.PlacedModule{Module: m}) + } + b := &bus{placements: map[string]delivery.Placement{"n1": p}, + records: map[string]delivery.NodeRecord{"n1": {Node: "n1", TunnelKey: publicKey(t), SealingKey: "sk", Capabilities: []string{"systemd", "container-runtime"}}}, + modules: map[string]delivery.Module{"lemurs": {Manifest: delivery.ManifestDigest(lemurs)}}, + manifests: map[string][]byte{delivery.ManifestDigest(lemurs): lemurs}, + builds: map[string]*delivery.BuilderBuild{}} + for _, m := range []string{"web", "api", "spiegel"} { + raw := manifest(m) + b.manifests[delivery.ManifestDigest(raw)] = raw + b.modules[m] = delivery.Module{Manifest: delivery.ManifestDigest(raw)} + } + // web and api are built, their keys done and waiting for the reference of their image; spiegel has no build of + // the new form at all. + for i, m := range []string{"web", "api"} { + mod := b.modules[m] + mod.FP2 = fp(string(rune('1' + i))) + b.modules[m] = mod + b.builds[m+" "+mod.FP2] = &delivery.BuilderBuild{State: delivery.BuildDone, Manifest: mod.Manifest, + Artifacts: []delivery.BuiltArtifact{{Name: "site", Kind: "image", Digest: "sha256:" + strings.Repeat("b", 64)}}} + } + return b + } + given := func(b *bus) map[string]*delivery.Placement { + p := b.placements["n1"] + return map[string]*delivery.Placement{"n1": &p} + } + + // Every refusal: two builds taken in, and a module the composition refuses (its image never built), each named. + b := mk("lemurs", "web", "api", "spiegel") + a := Answer(ctx, b, Mesh{}, Scope{Node: "n1", Placements: given(b)}, + map[string]string{"lemurs": fp("e"), "web": fp("1"), "api": fp("2"), "spiegel": fp("3")}, nil, "h1", 3) + var refused []string + for _, r := range a.Refusals { + refused = append(refused, r.Module) + } + if a.Verdict != delivery.CandidateRefused || a.Refused != "" || !reflect.DeepEqual(refused, []string{"api", "spiegel", "web"}) { + t.Errorf("every refusal: %+v, want api, spiegel and web refused, each named", a) + } + for _, r := range a.Refusals { + want := map[string]string{"api": "waits for the reference of site", "web": "waits for the reference of site", + "spiegel": "spiegel has not been built for this mesh"}[r.Module] + if !strings.Contains(r.Why, want) { + t.Errorf("%s refused as %q, want %q", r.Module, r.Why, want) + } + } + if raw, err := delivery.EncodeHaveNode(delivery.HaveNode{Candidates: map[string]delivery.CandidateAnswer{"g": a}}); err != nil { + t.Errorf("the answer is not one a have holds: %v (%s)", err, raw) + } + if lemursOK := slices.ContainsFunc(a.Added, func(m delivery.ModuleAdded) bool { return m.Module == "lemurs" }); !lemursOK { + t.Errorf("lemurs, which composes, is not answered beside the refusals: %+v", a) + } + + // The fold: spiegel runs inside the node's own composition at a build of the old form. Its resources are carried as + // they run, each after the one it runs after: the node's own is the one it runs, and the node composes itself. + b = mk("lemurs", overlay.Name, "spiegel") + p := b.placements["n1"] + p.Decided.Endpoint = "n1.example:51820" + b.placements["n1"] = p + fps := map[string]string{"lemurs": fp("e")} + stub := Scope{Node: "n1"} + in, err := Input(ctx, b, Mesh{}, stub) + if err != nil { + t.Fatal(err) + } + in.Nodes[0].Kept = map[string]catalogue.Manifest{"spiegel": compose.Stub(in.Catalogue["spiegel"])} + base, _, err := compose.Compositions(in, "n1", fps, compose.Reading) + if err != nil { + t.Fatal(err) + } + ownKey, _ := delivery.ComposedNodeKey("n1") + own := base[ownKey] + carriedRes := []json.RawMessage{ + json.RawMessage(`{"id":"spiegel.state","type":"directory","path":"/var/lib/spiegel","mode":"0700"}`), + json.RawMessage(`{"id":"spiegel.server","type":"container","name":"spiegel","image":"registry.example/spiegel@sha256:` + strings.Repeat("c", 64) + `"}`)} + // They run after the node's first own resource. + own.Resources = append(append(append([]json.RawMessage{}, own.Resources[:1]...), carriedRes...), own.Resources[1:]...) + var running []delivery.Composed + for k, c := range base { + if k == ownKey { + c = own + } + running = append(running, c) + } + scope := Scope{Node: "n1", InOwn: []string{"spiegel"}, Running: running} + got, _, err := Compose(ctx, b, Mesh{}, scope, fps) + if err != nil { + t.Fatalf("the node does not compose itself with spiegel folded at a build of the old form: %v", err) + } + gotOwn, _ := json.Marshal(got[ownKey].Resources) + wantOwn, _ := json.Marshal(own.Resources) + if string(gotOwn) != string(wantOwn) { + t.Errorf("the node's own composition is\n%s\nwant what runs\n%s", gotOwn, wantOwn) + } + a = Answer(ctx, b, Mesh{}, Scope{Node: "n1", Placements: given(b), InOwn: []string{"spiegel"}}, fps, running, "h1", 3) + if a.Verdict != delivery.CandidateSame { + t.Errorf("the fold's answer: %+v, want the same", a) + } + // Without what runs, the fold composes spiegel from its manifest, as before, and the composition refuses it. + if _, _, err := Compose(ctx, b, Mesh{}, Scope{Node: "n1", InOwn: []string{"spiegel"}}, fps); err == nil || + !strings.Contains(err.Error(), "spiegel has not been built for this mesh") { + t.Errorf("composed spiegel without what runs: %v", err) + } +} + +// A node that reaches no artifact store says what its composition holds of one (issue 1126: ace and g14 answered +// "reaches no artifact store", and nothing said why). +func TestNoStoreReachedSaysWhy(t *testing.T) { + in := compose.Input{Catalogue: map[string]catalogue.Manifest{ + "distribution": {Module: "distribution", Serves: map[string]map[string]any{catalogue.ArtifactStoreProvision: {"port": 5000}}}}, + Nodes: []compose.Node{{Name: "g14", Assigned: []string{"adwaita"}}, {Name: "novox", Assigned: []string{"distribution"}}}} + if got := storeSought(in); got != "the composition holds g14, novox; artifact-store is served by distribution on novox, none of it on the private network with a port" { + t.Errorf("said %q", got) + } + in.Nodes = in.Nodes[:1] + if got := storeSought(in); got != "the composition holds g14; no module placed there serves artifact-store" { + t.Errorf("said %q", got) + } +} diff --git a/go/compose/frombus/frombus.go b/go/compose/frombus/frombus.go index 0c1c28f..7aef315 100644 --- a/go/compose/frombus/frombus.go +++ b/go/compose/frombus/frombus.go @@ -113,12 +113,17 @@ type Mesh struct { // // InOwn is the modules the composing node runs inside its own composition (InOwn of what it runs), composed there again // (compose.Node.InOwn) rather than cut out as modules no build is given for (issue 1126). +// +// Running is what the composing node runs; of it, its own composition (`.node`) is given to the composition +// (compose.Node.RunsOwn), so a module folded there that names a build of the old form is left to the node, carried +// as it runs (issue 1126). Nil composes every folded module from its manifest. type Scope struct { Node string Every bool KeptAt map[string]string Placements map[string]*delivery.Placement InOwn []string + Running []delivery.Composed } // InOwn is, of the modules p places on its node, those whose resources the node's own composition it runs @@ -154,6 +159,13 @@ func InOwn(running []delivery.Composed, p delivery.Placement) []string { // Input is the composition input for s, read through r. func Input(ctx context.Context, r Reader, mesh Mesh, s Scope) (compose.Input, error) { + return input(ctx, r, mesh, s, nil) +} + +// input is Input; with refuse set, a module placed on the composing node whose build cannot be read or taken in is +// handed to refuse and composed as a stub (compose.Stub, kept for that node alone) rather than ending the read, so a +// node's answer names every module it cannot compose (issue 1126). +func input(ctx context.Context, r Reader, mesh Mesh, s Scope, refuse func(module string, err error)) (compose.Input, error) { placements, err := r.Placements(ctx) if err != nil { return compose.Input{}, fmt.Errorf("frombus: placement: %w", err) @@ -280,6 +292,12 @@ func Input(ctx context.Context, r Reader, mesh Mesh, s Scope) (compose.Input, er } n.InOwn[module] = true } + for i := range s.Running { + if s.Running[i].Module == "" { + own := s.Running[i] + n.RunsOwn = &own + } + } } in.Nodes = append(in.Nodes, n) in.Holdings = append(in.Holdings, p.Held()...) @@ -309,7 +327,7 @@ func Input(ctx context.Context, r Reader, mesh Mesh, s Scope) (compose.Input, er if _, ok := records[s.Node]; !ok { return compose.Input{}, fmt.Errorf("frombus: %s has written no record of itself yet", s.Node) } - if err := fromBuilds(ctx, r, &in, s, placements[s.Node], modules, read, parse); err != nil { + if err := fromBuilds(ctx, r, &in, s, placements[s.Node], modules, read, parse, refuse); err != nil { return compose.Input{}, err } // Its own modules' secrets, sealed to its own key by the vault: a composition carries them sealed (ADR 0550 §5). @@ -358,7 +376,29 @@ func Compose(ctx context.Context, r Reader, mesh Mesh, s Scope, fps map[string]s // by the one take-in rule (takein.Manifest), a templates archive read in the artifact store as the node reaches it // (issue 1126). func fromBuilds(ctx context.Context, r Reader, in *compose.Input, s Scope, p delivery.Placement, modules map[string]delivery.Module, - read func(module, digest string) (catalogue.Manifest, error), parse func([]byte) (catalogue.Manifest, error)) error { + read func(module, digest string) (catalogue.Manifest, error), parse func([]byte) (catalogue.Manifest, error), + refuse func(module string, err error)) error { + // stub keeps a refused module in the composition as a stub for this node alone (compose.Stub): what it provides + // and requires still shapes the rest. + stub := func(module string, err error) { + refuse(module, err) + for i := range in.Nodes { + if in.Nodes[i].Name != s.Node { + continue + } + man, ok := in.Nodes[i].Kept[module] + if !ok { + man, ok = in.Catalogue[module] + } + if !ok { + return + } + if in.Nodes[i].Kept == nil { + in.Nodes[i].Kept = map[string]catalogue.Manifest{} + } + in.Nodes[i].Kept[module] = compose.Stub(man) + } + } address, addressErr := "", error(nil) addressed := false archive := func(ctx context.Context, reference, digest string) ([]byte, error) { @@ -370,7 +410,7 @@ func fromBuilds(ctx context.Context, r Reader, in *compose.Input, s Scope, p del case addressErr != nil: return nil, addressErr case address == "": - return nil, fmt.Errorf("%s reaches no artifact store", s.Node) + return nil, fmt.Errorf("%s reaches no artifact store (%s)", s.Node, storeSought(*in)) } return r.Archive(ctx, catalogue.Routed(reference, address), digest) } @@ -409,14 +449,23 @@ func fromBuilds(ctx context.Context, r Reader, in *compose.Input, s Scope, p del } b, err := r.Build(ctx, pm.Module, m.FP2) if err != nil { - return fmt.Errorf("frombus: the build of %s at %s: %w", pm.Module, m.FP2, err) + err = fmt.Errorf("frombus: the build of %s at %s: %w", pm.Module, m.FP2, err) + if refuse == nil { + return err + } + stub(pm.Module, err) + continue } if b == nil || b.State != delivery.BuildDone { continue // no done build at the trunk's: composed from the indexer's manifest, as before } resolved, err := built(pm.Module, m.FP2, b) if err != nil { - return err + if refuse == nil { + return err + } + stub(pm.Module, err) + continue } in.Catalogue[pm.Module] = resolved } @@ -427,15 +476,22 @@ func fromBuilds(ctx context.Context, r Reader, in *compose.Input, s Scope, p del for _, module := range sortedKeys(s.KeptAt) { fp := s.KeptAt[module] b, err := r.Build(ctx, module, fp) - if err != nil { - return fmt.Errorf("frombus: the build of %s at %s: %w", module, fp, err) + switch { + case err != nil: + err = fmt.Errorf("frombus: the build of %s at %s: %w", module, fp, err) + case b == nil || b.State != delivery.BuildDone: + err = fmt.Errorf("frombus: %s is wanted at %s, of which the builder holds no done build", module, fp) } - if b == nil || b.State != delivery.BuildDone { - return fmt.Errorf("frombus: %s is wanted at %s, of which the builder holds no done build", module, fp) + var resolved catalogue.Manifest + if err == nil { + resolved, err = built(module, fp, b) } - resolved, err := built(module, fp, b) if err != nil { - return err + if refuse == nil { + return err + } + stub(module, err) + continue } if in.Nodes[i].Kept == nil { in.Nodes[i].Kept = map[string]catalogue.Manifest{} @@ -613,3 +669,27 @@ func sortedKeys[V any](m map[string]V) []string { sort.Strings(out) return out } + +// storeSought says, for a node that reaches no artifact store, what the composition holds of one: the nodes it holds and +// where each module serving the artifact store is placed, so the refusal names why (issue 1126). +func storeSought(in compose.Input) string { + var nodes, where []string + for _, n := range in.Nodes { + nodes = append(nodes, n.Name) + for _, a := range n.Assigned { + if m, ok := in.Catalogue[a]; ok { + if _, serves := m.Serves[catalogue.ArtifactStoreProvision]; serves { + where = append(where, a+" on "+n.Name) + } + } + } + } + sort.Strings(nodes) + sort.Strings(where) + said := "the composition holds " + strings.Join(nodes, ", ") + if len(where) == 0 { + return said + "; no module placed there serves " + catalogue.ArtifactStoreProvision + } + return said + "; " + catalogue.ArtifactStoreProvision + " is served by " + strings.Join(where, ", ") + + ", none of it on the private network with a port" +} diff --git a/go/compose/input.go b/go/compose/input.go index bf06c3c..2ca4f25 100644 --- a/go/compose/input.go +++ b/go/compose/input.go @@ -23,6 +23,7 @@ import ( "strings" "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" ) // Input is everything the composition reads. Each field is what one read of the controller's inventory answers, named @@ -194,6 +195,11 @@ type Node struct { // resources into the node's own, as the controller's composed writer does, unless fps gives one of the new form. // The private network's module (overlay.Name), which nobody builds, is folded whether or not it is named here. InOwn map[string]bool + // RunsOwn is the node's own composition (`.node`) as it runs, when the node composes itself: a module folded + // there (InOwn) whose manifest names a build (NamesBuild) runs a build of the old form only the controller holds, + // so it is left to the node: Compositions composes it as a stub (Stub) and carries its resources as they run, each + // after the resource it runs after (issue 1126). Nil composes every folded module from its manifest, as before. + RunsOwn *delivery.Composed } // Place is where a node is, as its placement says (inv.Overlay). From 472d5f314466afa910117ac8ad033f4df9ea2439 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 27 Oct 2026 23:22:54 +0100 Subject: [PATCH 2/7] A node's have says whose compositions it applies, and its answer to a head deleting its placement says it returns to the controller's (issue 1126, the seed's revert) The check of the seed's revert (mesh-catalog #526) failed on 'would stop running node': the shadow answered a deleted placement as everything removed, while the node keeps what runs. HaveNode.Composes and ComposesWhy say whether a node applies its own compositions or the controller's; CandidateAnswer.Returns answers a deleted placement, verdict same, naming nothing composed. --- go/delivery/delivery.go | 20 +++++++++++++++++++- go/delivery/placement_candidate.go | 11 +++++++++++ go/delivery/placement_test.go | 20 +++++++++++++++++--- 3 files changed, 47 insertions(+), 4 deletions(-) diff --git a/go/delivery/delivery.go b/go/delivery/delivery.go index 1b2306f..6e51c60 100644 --- a/go/delivery/delivery.go +++ b/go/delivery/delivery.go @@ -410,8 +410,20 @@ type HaveNode struct { // its shadow composer whenever an answer changes, removed when the candidate key goes. The node applies nothing of a // candidate. Candidates map[string]CandidateAnswer `json:"candidates,omitempty"` + // Composes is whose compositions the node applies: its own (ComposesOwn, the catalogue places it) or the + // controller's (ComposesController); ComposesWhy says why in words. Written by the node agent at each start, so a + // node returning to the controller's compositions when its placement key is deleted says so (issue 1126). Empty + // from an engine older than the field. + Composes string `json:"composes,omitempty"` + ComposesWhy string `json:"composes-why,omitempty"` } +// Whose compositions a node applies (HaveNode.Composes). +const ( + ComposesOwn = "own" + ComposesController = "controller" +) + // Presence is whether a node or a part's instance is online; the entry's own time says since when. The watchman // writes it. type Presence struct { @@ -506,6 +518,11 @@ func DecodeHave(data []byte) (Have, error) { } func (n HaveNode) check() error { + switch n.Composes { + case "", ComposesOwn, ComposesController: + default: + return fmt.Errorf("delivery: a node composes %q, neither %s nor %s", n.Composes, ComposesOwn, ComposesController) + } for group, a := range n.Candidates { if group == "" { return fmt.Errorf("delivery: a node's candidate answer names no group") @@ -590,7 +607,8 @@ func SameHaveNode(a, b HaveNode) bool { return a.V == b.V && a.Composed == b.Composed && sameJSON(a.Windows, b.Windows) && sameJSON(a.Busy, b.Busy) && sameJSON(a.Strays, b.Strays) && sameJSON(a.Filters, b.Filters) && sameJSON(a.Profile, b.Profile) && (a.Record == nil) == (b.Record == nil) && (a.Record == nil || SameNodeRecord(*a.Record, *b.Record)) && - jsonEqual(a.Sealed, b.Sealed) && jsonEqual(a.Candidates, b.Candidates) + jsonEqual(a.Sealed, b.Sealed) && jsonEqual(a.Candidates, b.Candidates) && a.Composes == b.Composes && + a.ComposesWhy == b.ComposesWhy } // SamePresence says whether two presences are the same value. diff --git a/go/delivery/placement_candidate.go b/go/delivery/placement_candidate.go index edf257b..e3736bc 100644 --- a/go/delivery/placement_candidate.go +++ b/go/delivery/placement_candidate.go @@ -132,8 +132,16 @@ type CandidateAnswer struct { Refusals []ModuleRefused `json:"refusals,omitempty"` // Refused is why the node could not compose with the candidate at all; Refusals is then empty. Refused string `json:"refused,omitempty"` + // Returns is, for a head that deletes the node's placement file, what the node does then: it returns to the + // controller's compositions (ReturnsToController), as a node the catalogue does not place composes none of its own + // (issue 1126: the seed's revert). The verdict is then same, and nothing else is named: the node composes nothing + // with the candidate, and judges nothing of what the controller composes. + Returns string `json:"returns,omitempty"` } +// ReturnsToController is what a node answers a head that deletes its placement file. +const ReturnsToController = "the head deletes this node's placement: it returns to the controller's compositions" + // ModuleDiffers is one module the node runs and the candidate still places whose composition is not the one it runs: // both digests (Composed with the candidate, Running what runs), the first resource that differs by its id, or empty // when every resource is equal, and why in words. @@ -187,6 +195,9 @@ func (a CandidateAnswer) check() error { if a.Refused != "" && len(a.Same)+len(a.Differs)+len(a.Added)+len(a.Removed)+len(a.Refusals) > 0 { return fmt.Errorf("delivery: a candidate the node could not compose names modules") } + if a.Returns != "" && (a.Refused != "" || len(a.Same)+len(a.Differs)+len(a.Added)+len(a.Removed)+len(a.Refusals) > 0) { + return fmt.Errorf("delivery: a candidate the node returns to the controller's compositions on names what it composed") + } for _, d := range a.Differs { if d.Module == "" || d.Why == "" { return fmt.Errorf("delivery: a candidate difference names no module or no why") diff --git a/go/delivery/placement_test.go b/go/delivery/placement_test.go index ae77b5f..29376d0 100644 --- a/go/delivery/placement_test.go +++ b/go/delivery/placement_test.go @@ -229,19 +229,31 @@ func TestPlacement(t *testing.T) { "refused": {Head: "h1", Revision: 3, Verdict: CandidateRefused, Refused: "unbound postgres"}, "added": {Head: "h1", Verdict: CandidateDiffers, Same: []string{"node"}, Added: []ModuleAdded{{Module: "app", Unissued: []string{"app/token"}}}, Removed: []string{"old"}}, "a value": {Head: "h1", Verdict: CandidateRefused, Same: []string{"node"}, Refusals: []ModuleRefused{{Module: "db", Why: "stands in for db/password"}}}, + // A head that deletes the node's placement file: the node returns to the controller's compositions (issue 1126, + // the seed's revert, mesh-catalog #526). + "returns": {Head: "h1", Verdict: CandidateSame, Returns: ReturnsToController}, } - n := HaveNode{Composed: "sha256:" + strings.Repeat("a", 64), Sealed: []string{"box"}, Candidates: answers} + n := HaveNode{Composed: "sha256:" + strings.Repeat("a", 64), Sealed: []string{"box"}, Candidates: answers, + Composes: ComposesOwn, ComposesWhy: "the catalogue places it (placement.n1)"} raw, err := EncodeHaveNode(n) if err != nil { t.Fatal(err) } back, err := DecodeHaveNode(raw) - if err != nil || !SameHaveNode(back, HaveNode{V: Version, Composed: n.Composed, Sealed: n.Sealed, Candidates: answers}) { + if err != nil || !SameHaveNode(back, HaveNode{V: Version, Composed: n.Composed, Sealed: n.Sealed, Candidates: answers, + Composes: ComposesOwn, ComposesWhy: n.ComposesWhy}) { t.Fatalf("a node's answers changed on the way: %+v, %v", back, err) } - if SameHaveNode(back, HaveNode{V: Version, Composed: n.Composed, Sealed: n.Sealed}) { + if SameHaveNode(back, HaveNode{V: Version, Composed: n.Composed, Sealed: n.Sealed, Composes: ComposesOwn, ComposesWhy: n.ComposesWhy}) { t.Error("a node's have without its answers counts as the same") } + if SameHaveNode(back, HaveNode{V: Version, Composed: n.Composed, Sealed: n.Sealed, Candidates: answers, + Composes: ComposesController, ComposesWhy: n.ComposesWhy}) { + t.Error("a node's have composing otherwise counts as the same") + } + if _, err := EncodeHaveNode(HaveNode{Composes: "both"}); err == nil { + t.Error("a node composing neither its own nor the controller's was taken") + } // CompareComposed: the bus's user list is left out on both sides; anything else that differs is named. file := func(id, content string) json.RawMessage { return json.RawMessage(`{"id":"` + id + `","type":"file","module":"nats","path":"/etc/` + id + `","mode":"0600","content":"` + content + `"}`) @@ -285,6 +297,8 @@ func TestPlacement(t *testing.T) { "same with an added": {Head: "h", Verdict: CandidateSame, Added: []ModuleAdded{{Module: "a"}}}, "differs refusing": {Head: "h", Verdict: CandidateDiffers, Removed: []string{"a"}, Refusals: []ModuleRefused{{Module: "b", Why: "x"}}}, "refused and modules": {Head: "h", Verdict: CandidateRefused, Refused: "x", Same: []string{"a"}}, + "returns and removes": {Head: "h", Verdict: CandidateDiffers, Returns: ReturnsToController, Removed: []string{"a"}}, + "returns refused": {Head: "h", Verdict: CandidateRefused, Returns: ReturnsToController, Refused: "x"}, } { if _, err := EncodeHaveNode(HaveNode{Candidates: map[string]CandidateAnswer{"g": a}}); err == nil { t.Errorf("%s: taken", name) From e32d777530247a7f8fda33dcfd3920d7a3982eea Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 27 Oct 2026 23:32:31 +0100 Subject: [PATCH 3/7] A deleting head's answer compares the controller's compositions with what runs, and is refused only without returning; fold the shadow's tests into TestInput (issue 1126, review of the shadow, round 1) --- .../frombus/{answer_test.go => answer_rows_test.go} | 4 ++-- go/compose/frombus/frombus_test.go | 4 ++++ go/delivery/placement_candidate.go | 10 ++++++---- go/delivery/placement_test.go | 6 ++++-- 4 files changed, 16 insertions(+), 8 deletions(-) rename go/compose/frombus/{answer_test.go => answer_rows_test.go} (98%) diff --git a/go/compose/frombus/answer_test.go b/go/compose/frombus/answer_rows_test.go similarity index 98% rename from go/compose/frombus/answer_test.go rename to go/compose/frombus/answer_rows_test.go index 1f6ab25..f0aa574 100644 --- a/go/compose/frombus/answer_test.go +++ b/go/compose/frombus/answer_rows_test.go @@ -17,7 +17,7 @@ import ( // Issue 1126 (the seed's shadow, round 2): a node's answer names every module it cannot compose, not the first only; // and a module folded into its own composition that names a build of the old form (de-spiegel on novox, nats, photos, // systemd-resolved: their haves name a build-... id and no fingerprint) is left to the node, carried as it runs. -func TestTheAnswerNamesEveryRefusalAndLeavesAnOldBuildToTheNode(t *testing.T) { +func answerNamesEveryRefusalAndLeavesAnOldBuildToTheNode(t *testing.T) { ctx := context.Background() image := "alpine@sha256:" + strings.Repeat("a", 64) manifest := func(module string) []byte { @@ -138,7 +138,7 @@ func TestTheAnswerNamesEveryRefusalAndLeavesAnOldBuildToTheNode(t *testing.T) { // A node that reaches no artifact store says what its composition holds of one (issue 1126: ace and g14 answered // "reaches no artifact store", and nothing said why). -func TestNoStoreReachedSaysWhy(t *testing.T) { +func noStoreReachedSaysWhy(t *testing.T) { in := compose.Input{Catalogue: map[string]catalogue.Manifest{ "distribution": {Module: "distribution", Serves: map[string]map[string]any{catalogue.ArtifactStoreProvision: {"port": 5000}}}}, Nodes: []compose.Node{{Name: "g14", Assigned: []string{"adwaita"}}, {Name: "novox", Assigned: []string{"distribution"}}}} diff --git a/go/compose/frombus/frombus_test.go b/go/compose/frombus/frombus_test.go index 20a5ae5..ee7697d 100644 --- a/go/compose/frombus/frombus_test.go +++ b/go/compose/frombus/frombus_test.go @@ -119,6 +119,10 @@ func publicKey(t *testing.T) string { // controller's store fills for compose's own two-machine mesh (compose's aMesh: anchor the hub, laptop), and that Input // composes; a node reads only its own config values; what cannot be composed is refused, naming it. func TestInput(t *testing.T) { + // Issue 1126, the seed's shadow: every refusal named, an old-form module left to the node, and no store reached + // saying why (answer_rows_test.go). + t.Run("every refusal, and an old build left to the node", answerNamesEveryRefusalAndLeavesAnOldBuildToTheNode) + t.Run("no store reached says why", noStoreReachedSaysWhy) ctx := context.Background() xterm, err := os.ReadFile("../../catalogue/testdata/catalogue-835bf22b-xterm.module.json") if err != nil { diff --git a/go/delivery/placement_candidate.go b/go/delivery/placement_candidate.go index e3736bc..6b28569 100644 --- a/go/delivery/placement_candidate.go +++ b/go/delivery/placement_candidate.go @@ -134,8 +134,9 @@ type CandidateAnswer struct { Refused string `json:"refused,omitempty"` // Returns is, for a head that deletes the node's placement file, what the node does then: it returns to the // controller's compositions (ReturnsToController), as a node the catalogue does not place composes none of its own - // (issue 1126: the seed's revert). The verdict is then same, and nothing else is named: the node composes nothing - // with the candidate, and judges nothing of what the controller composes. + // (issue 1126: the seed's revert). Same, Differs, Added and Removed then compare the controller's compositions at + // the node's wants with what it runs, and the verdict is theirs; a want the controller holds no composition for is + // refused (Refused, without Returns). Returns string `json:"returns,omitempty"` } @@ -195,8 +196,9 @@ func (a CandidateAnswer) check() error { if a.Refused != "" && len(a.Same)+len(a.Differs)+len(a.Added)+len(a.Removed)+len(a.Refusals) > 0 { return fmt.Errorf("delivery: a candidate the node could not compose names modules") } - if a.Returns != "" && (a.Refused != "" || len(a.Same)+len(a.Differs)+len(a.Added)+len(a.Removed)+len(a.Refusals) > 0) { - return fmt.Errorf("delivery: a candidate the node returns to the controller's compositions on names what it composed") + if a.Returns != "" && (a.Refused != "" || len(a.Refusals) > 0) { + return fmt.Errorf("delivery: a candidate the node returns to the controller's compositions on is compared, never refused: " + + "a composition the controller holds none of is refused without returning") } for _, d := range a.Differs { if d.Module == "" || d.Why == "" { diff --git a/go/delivery/placement_test.go b/go/delivery/placement_test.go index 29376d0..e0b7810 100644 --- a/go/delivery/placement_test.go +++ b/go/delivery/placement_test.go @@ -231,7 +231,9 @@ func TestPlacement(t *testing.T) { "a value": {Head: "h1", Verdict: CandidateRefused, Same: []string{"node"}, Refusals: []ModuleRefused{{Module: "db", Why: "stands in for db/password"}}}, // A head that deletes the node's placement file: the node returns to the controller's compositions (issue 1126, // the seed's revert, mesh-catalog #526). - "returns": {Head: "h1", Verdict: CandidateSame, Returns: ReturnsToController}, + "returns": {Head: "h1", Verdict: CandidateSame, Returns: ReturnsToController, Same: []string{"app", "node"}}, + "returns, the controller's differs": {Head: "h1", Verdict: CandidateDiffers, Returns: ReturnsToController, Same: []string{"node"}, + Differs: []ModuleDiffers{{Module: "app", Resource: "file:/etc/app", Why: "the resource differs"}}, Removed: []string{"old"}}, } n := HaveNode{Composed: "sha256:" + strings.Repeat("a", 64), Sealed: []string{"box"}, Candidates: answers, Composes: ComposesOwn, ComposesWhy: "the catalogue places it (placement.n1)"} @@ -297,7 +299,7 @@ func TestPlacement(t *testing.T) { "same with an added": {Head: "h", Verdict: CandidateSame, Added: []ModuleAdded{{Module: "a"}}}, "differs refusing": {Head: "h", Verdict: CandidateDiffers, Removed: []string{"a"}, Refusals: []ModuleRefused{{Module: "b", Why: "x"}}}, "refused and modules": {Head: "h", Verdict: CandidateRefused, Refused: "x", Same: []string{"a"}}, - "returns and removes": {Head: "h", Verdict: CandidateDiffers, Returns: ReturnsToController, Removed: []string{"a"}}, + "returns refusing": {Head: "h", Verdict: CandidateRefused, Returns: ReturnsToController, Refusals: []ModuleRefused{{Module: "b", Why: "x"}}}, "returns refused": {Head: "h", Verdict: CandidateRefused, Returns: ReturnsToController, Refused: "x"}, } { if _, err := EncodeHaveNode(HaveNode{Candidates: map[string]CandidateAnswer{"g": a}}); err == nil { From 7359d187db8dcd50d13b3d620b951027c3401137 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 28 Oct 2026 00:21:45 +0100 Subject: [PATCH 4/7] A stub keeps its module's directories, so the places its own secrets and data name still resolve (issue 1126: novox refused nats's ${dir:mesh-state}) --- go/compose/composed.go | 13 ++++++++++--- go/compose/frombus/answer_rows_test.go | 6 ++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/go/compose/composed.go b/go/compose/composed.go index 5975c25..a6a8bea 100644 --- a/go/compose/composed.go +++ b/go/compose/composed.go @@ -257,11 +257,18 @@ func NamesBuild(m catalogue.Manifest) bool { return false } -// Stub is m with nothing to build and no resource: what it provides, requires, serves and grants still shape the rest of -// the composition, and nothing of its own is composed (issue 1126). +// Stub is m with nothing to build and no resource but its directories: what it provides, requires, serves and grants +// still shape the rest of the composition, the places its own secrets, data and grants name (${dir:…}) still resolve, +// and nothing else of its own is composed (issue 1126: nats's ${dir:mesh-state} on novox). func Stub(m catalogue.Manifest) catalogue.Manifest { m.Build = nil - m.Resources = nil + var dirs []map[string]any + for _, r := range m.Resources { + if r["type"] == "directory" { + dirs = append(dirs, r) + } + } + m.Resources = dirs return m } diff --git a/go/compose/frombus/answer_rows_test.go b/go/compose/frombus/answer_rows_test.go index f0aa574..9c191d0 100644 --- a/go/compose/frombus/answer_rows_test.go +++ b/go/compose/frombus/answer_rows_test.go @@ -21,7 +21,10 @@ func answerNamesEveryRefusalAndLeavesAnOldBuildToTheNode(t *testing.T) { ctx := context.Background() image := "alpine@sha256:" + strings.Repeat("a", 64) manifest := func(module string) []byte { + // Its data names its directory, as nats's names ${dir:mesh-state} (issue 1126: novox's own composition refused + // "nats says ${dir:mesh-state}, and nats declares no directory mesh-state" when its stub held no directory). return []byte(`{"module":"` + module + `","version":"1","capabilities":["container-runtime"],` + + `"own-secrets":{"token":"${dir:state}/token"},` + `"resources":[{"id":"state","type":"directory","path":"/var/lib/` + module + `","mode":"0700"},` + `{"id":"server","type":"container","name":"` + module + `","artifact":"site"}],` + `"build":{"artifacts":[{"name":"site","kind":"upstream","from":"` + image + `"}]}}`) @@ -38,6 +41,9 @@ func answerNamesEveryRefusalAndLeavesAnOldBuildToTheNode(t *testing.T) { modules: map[string]delivery.Module{"lemurs": {Manifest: delivery.ManifestDigest(lemurs)}}, manifests: map[string][]byte{delivery.ManifestDigest(lemurs): lemurs}, builds: map[string]*delivery.BuilderBuild{}} + for _, m := range []string{"web", "api", "spiegel"} { + b.own = append(b.own, compose.OwnSecretRow{Node: "n1", Module: m, Name: "token", Origin: "made", SealedTo: "sk", Sealed: `"box"`}) + } for _, m := range []string{"web", "api", "spiegel"} { raw := manifest(m) b.manifests[delivery.ManifestDigest(raw)] = raw From b31c4580aa93cea337b2e322b60a637e756f6710 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 28 Oct 2026 01:08:49 +0100 Subject: [PATCH 5/7] A candidate answer shows every differing resource, masked; a module folded in the node's own is judged there; the node's own carries its bus membership as it runs it (issue 1126, seed #531) novox's answer to seed #531 named 15 resources by id alone, so why each differed was guessed. ModuleDiffers.Bodies keeps every differing resource with both bodies, each value under a key a secret could be in masked (the same, another seal of the same value, another value). The five old-form modules read 'removed' because the agent also lists them by module from the declaration kept before the switch; they are judged inside the node's own. And the node's own lacked bus-membership, which only the controller composes from its store: the membership the node runs is carried, until the grants part hands it sealed. --- go/compose/frombus/answer.go | 13 +- go/compose/frombus/answer_rows_test.go | 19 +++ go/compose/frombus/frombus.go | 29 ++++ go/delivery/composed_compare.go | 194 ++++++++++++++++++++++++- go/delivery/placement_candidate.go | 21 +++ go/delivery/placement_test.go | 33 +++++ 6 files changed, 303 insertions(+), 6 deletions(-) diff --git a/go/compose/frombus/answer.go b/go/compose/frombus/answer.go index baa549e..52f3bbb 100644 --- a/go/compose/frombus/answer.go +++ b/go/compose/frombus/answer.go @@ -107,7 +107,18 @@ func Answer(ctx context.Context, r Reader, mesh Mesh, s Scope, fps map[string]st a.Verdict = delivery.VerdictOf(a) return a } - cmp := delivery.CompareComposed(composed, running) + // A module the composition folds inside the node's own is judged there: the agent also says it runs by module, from + // the declaration it kept before the switch (agent.Everything), which is no removal (issue 1126, seed #531). + if len(s.InOwn) > 0 { + var judged []delivery.Composed + for _, c := range running { + if c.Module == "" || !slices.Contains(s.InOwn, c.Module) { + judged = append(judged, c) + } + } + running = judged + } + cmp := delivery.CompareComposedWith(composed, running, s.SameValue) // A refused module was composed as a stub: what it composed is no difference; its refusal is said. notFailed := func(m string) bool { _, ok := failed[m]; return !ok } a.Same = slices.DeleteFunc(cmp.Same, func(m string) bool { return !notFailed(m) }) diff --git a/go/compose/frombus/answer_rows_test.go b/go/compose/frombus/answer_rows_test.go index 9c191d0..2a361af 100644 --- a/go/compose/frombus/answer_rows_test.go +++ b/go/compose/frombus/answer_rows_test.go @@ -135,6 +135,25 @@ func answerNamesEveryRefusalAndLeavesAnOldBuildToTheNode(t *testing.T) { if a.Verdict != delivery.CandidateSame { t.Errorf("the fold's answer: %+v, want the same", a) } + // (A) The agent also says spiegel runs by module, from the declaration kept before the switch (agent.Everything): a + // module the composition folds is judged inside the node's own, not as removed (issue 1126, seed #531's answer). + keptToo := append(append([]delivery.Composed{}, running...), delivery.Composed{Node: "n1", Module: "spiegel", Resources: carriedRes}) + if a = Answer(ctx, b, Mesh{}, Scope{Node: "n1", Placements: given(b), InOwn: []string{"spiegel"}}, fps, keptToo, "h1", 3); a.Verdict != delivery.CandidateSame || len(a.Removed) != 0 { + t.Errorf("a folded module running from the kept declaration is answered %+v, want the same", a) + } + // (B) The node's own composition carries its bus membership as it runs it (the controller's, from its store): composed + // the same, not 'in what runs, not composed' (seed #531, novox). + membership := json.RawMessage(`{"id":"bus-membership","type":"file","path":"` + catalogue.BusMembershipPath + `","sealed":"{\"box\":\"m\"}","mode":"0600"}`) + withMembership := make([]delivery.Composed, 0, len(running)) + for _, c := range running { + if c.Module == "" { + c.Resources = append(append([]json.RawMessage{}, c.Resources...), membership) + } + withMembership = append(withMembership, c) + } + if a = Answer(ctx, b, Mesh{}, Scope{Node: "n1", Placements: given(b), InOwn: []string{"spiegel"}}, fps, withMembership, "h1", 3); a.Verdict != delivery.CandidateSame { + t.Errorf("the node's own with its bus membership is answered %+v, want the same", a) + } // Without what runs, the fold composes spiegel from its manifest, as before, and the composition refuses it. if _, _, err := Compose(ctx, b, Mesh{}, Scope{Node: "n1", InOwn: []string{"spiegel"}}, fps); err == nil || !strings.Contains(err.Error(), "spiegel has not been built for this mesh") { diff --git a/go/compose/frombus/frombus.go b/go/compose/frombus/frombus.go index 7aef315..735a831 100644 --- a/go/compose/frombus/frombus.go +++ b/go/compose/frombus/frombus.go @@ -124,6 +124,10 @@ type Scope struct { Placements map[string]*delivery.Placement InOwn []string Running []delivery.Composed + // SameValue, in an answer, says whether two sealed values open to the same value (the node opens them with its own + // key), so a differing resource shows a value sealed again as such (delivery.CompareComposedWith). Nil shows them + // as differing. + SameValue func(a, b string) (same, known bool) } // InOwn is, of the modules p places on its node, those whose resources the node's own composition it runs @@ -296,6 +300,16 @@ func input(ctx context.Context, r Reader, mesh Mesh, s Scope, refuse func(module if s.Running[i].Module == "" { own := s.Running[i] n.RunsOwn = &own + // Its bus membership as it runs it: the controller composes it from its store, which the node does + // not read; until the grants part hands memberships sealed to the node, the node's own composition + // carries the one it runs (issue 1126, seed #531 on novox: "bus-membership in what runs, not + // composed", which applied would have taken the node's bus credential away). + if sealed := membershipOf(own); sealed != "" { + if in.BusMemberships == nil { + in.BusMemberships = map[string]string{} + } + in.BusMemberships[s.Node] = sealed + } } } } @@ -693,3 +707,18 @@ func storeSought(in compose.Input) string { return said + "; " + catalogue.ArtifactStoreProvision + " is served by " + strings.Join(where, ", ") + ", none of it on the private network with a port" } + +// membershipOf is the sealed value of the bus membership resource (catalogue.BusMembershipID) of a node's own +// composition, or "". +func membershipOf(own delivery.Composed) string { + for _, raw := range own.Resources { + var r struct { + ID string `json:"id"` + Sealed string `json:"sealed"` + } + if json.Unmarshal(raw, &r) == nil && r.ID == catalogue.BusMembershipID() { + return r.Sealed + } + } + return "" +} diff --git a/go/delivery/composed_compare.go b/go/delivery/composed_compare.go index ecb4ca0..4faee55 100644 --- a/go/delivery/composed_compare.go +++ b/go/delivery/composed_compare.go @@ -28,6 +28,13 @@ type Comparison struct { // resource is equal it says what else differs (their order, the files filled, the data, the node's own parts, a file // carried by digest). func CompareComposed(composed map[string]Composed, running []Composed) Comparison { + return CompareComposedWith(composed, running, nil) +} + +// CompareComposedWith is CompareComposed showing, for each module that differs, every differing resource with both +// bodies masked (MaskedPair); sameValue, when given, says whether two sealed values open to the same value (the node +// opens them with its own key), so a value sealed again reads as such. +func CompareComposedWith(composed map[string]Composed, running []Composed, sameValue func(a, b string) (same, known bool)) Comparison { name := func(c Composed) string { if c.Module == "" { return HaveNodeWord @@ -60,7 +67,7 @@ func CompareComposed(composed map[string]Composed, running []Composed) Compariso case !composedHere: out.Removed = append(out.Removed, m) default: - if d, ok := compareOne(m, a, b); ok { + if d, ok := compareOne(m, a, b, sameValue); ok { out.Same = append(out.Same, m) } else { out.Differs = append(out.Differs, d) @@ -71,7 +78,7 @@ func CompareComposed(composed map[string]Composed, running []Composed) Compariso } // compareOne is one module's comparison: true when the digests agree, else what differs first. -func compareOne(module string, composed, running Composed) (ModuleDiffers, bool) { +func compareOne(module string, composed, running Composed, sameValue func(a, b string) (bool, bool)) (ModuleDiffers, bool) { composed, running = withoutBusUsers(composed), withoutBusUsers(running) dc, errC := ComposedDigestOf(composed) dr, errR := ComposedDigestOf(running) @@ -99,17 +106,35 @@ func compareOne(module string, composed, running Composed) (ModuleDiffers, bool) for id := range rb { ids[id] = true } + var first ModuleDiffers + found := false + var bodies []ResourceBodies for _, id := range sortedKeys(ids) { x, inA := ra[id] y, inB := rb[id] + why := "" switch { case !inB: - return differs(id, "composed, and not in what runs") + why = "composed, and not in what runs" case !inA: - return differs(id, "in what runs, and not composed") + why = "in what runs, and not composed" case !bytes.Equal(x, y): - return differs(id, "the resource differs") + why = "the resource differs" + default: + continue } + if !found { + first, _ = differs(id, why) + found = true + } + if len(bodies) < MaxBodies { + c, r := MaskedPair(x, y, sameValue) + bodies = append(bodies, ResourceBodies{ID: id, Why: why, Composed: cut(c), Running: cut(r)}) + } + } + if found { + first.Bodies = bodies + return first, false } switch { case !jsonEqual(a.Filled, b.Filled): @@ -165,3 +190,162 @@ func resourcesByID(resources []json.RawMessage) map[string][]byte { } return out } + +// cut is s cut at MaxBody bytes, saying so. +func cut(s string) string { + if len(s) <= MaxBody { + return s + } + return s[:MaxBody] + "… (cut at " + strconv.Itoa(MaxBody) + " bytes)" +} + +// secretWords are the words of a key whose value may hold a secret or a sealed one: masked in a shown body. +var secretWords = []string{"sealed", "secret", "password", "passwd", "token", "private", "key", "credential", "jwt", "salt", "hash"} + +func secretKey(k string) bool { + k = strings.ToLower(k) + for _, w := range secretWords { + if strings.Contains(k, w) { + return true + } + } + return false +} + +// MaskedPair is two bodies of one resource (either may be nil) as JSON text with every value under a key a secret could +// be in (secretWords, at any depth) masked: "", "" or "" when sameValue says so, "" otherwise, "" on one side +// alone. A body that is not JSON is shown as its length only. +func MaskedPair(composed, running json.RawMessage, sameValue func(a, b string) (same, known bool)) (string, string) { + var a, b any + okA := composed != nil && json.Unmarshal(composed, &a) == nil + okB := running != nil && json.Unmarshal(running, &b) == nil + if okA && okB { + a, b = maskBoth(a, b, sameValue) + } else if okA { + a, _ = maskBoth(a, nil, nil) + } else if okB { + _, b = maskBoth(nil, b, nil) + } + show := func(v any, raw json.RawMessage, ok bool) string { + if raw == nil { + return "" + } + if !ok { + return "" + } + var out bytes.Buffer + enc := json.NewEncoder(&out) + enc.SetEscapeHTML(false) + _ = enc.Encode(v) + return strings.TrimSuffix(out.String(), "\n") + } + return show(a, composed, okA), show(b, running, okB) +} + +func maskBoth(a, b any, sameValue func(a, b string) (bool, bool)) (any, any) { + ma, isA := a.(map[string]any) + mb, isB := b.(map[string]any) + if isA || isB { + outA, outB := map[string]any{}, map[string]any{} + keys := map[string]bool{} + for k := range ma { + keys[k] = true + } + for k := range mb { + keys[k] = true + } + for k := range keys { + va, inA := ma[k] + vb, inB := mb[k] + if secretKey(k) { + word := maskWord(va, vb, inA && inB, sameValue) + if inA { + outA[k] = word + } + if inB { + outB[k] = word + } + continue + } + x, y := maskBoth(va, vb, sameValue) + if inA { + outA[k] = x + } + if inB { + outB[k] = y + } + } + var ra, rb any + if isA { + ra = outA + } else { + ra = a + } + if isB { + rb = outB + } else { + rb = b + } + return ra, rb + } + la, isLA := a.([]any) + lb, isLB := b.([]any) + if isLA || isLB { + outA := make([]any, len(la)) + outB := make([]any, len(lb)) + for i := range max(len(la), len(lb)) { + var x, y any + if i < len(la) { + x = la[i] + } + if i < len(lb) { + y = lb[i] + } + x, y = maskBoth(x, y, sameValue) + if i < len(la) { + outA[i] = x + } + if i < len(lb) { + outB[i] = y + } + } + var ra, rb any = a, b + if isLA { + ra = outA + } + if isLB { + rb = outB + } + return ra, rb + } + return a, b +} + +func maskWord(a, b any, both bool, sameValue func(a, b string) (bool, bool)) string { + if !both { + return "" + } + x, _ := json.Marshal(a) + y, _ := json.Marshal(b) + if bytes.Equal(x, y) { + return "" + } + if sameValue != nil { + sa, okA := a.(string) + sb, okB := b.(string) + if !okA { + sa = string(x) + } + if !okB { + sb = string(y) + } + if same, known := sameValue(sa, sb); known { + if same { + return "" + } + return "" + } + } + return "" +} diff --git a/go/delivery/placement_candidate.go b/go/delivery/placement_candidate.go index 6b28569..64a8d47 100644 --- a/go/delivery/placement_candidate.go +++ b/go/delivery/placement_candidate.go @@ -138,6 +138,9 @@ type CandidateAnswer struct { // the node's wants with what it runs, and the verdict is theirs; a want the controller holds no composition for is // refused (Refused, without Returns). Returns string `json:"returns,omitempty"` + // Note is what the node says of how it answered, in words, beside the verdict: a pass that ran over a minute, inputs + // it waited for (issue 1126: the shadow shows its work). It changes no verdict. + Note string `json:"note,omitempty"` } // ReturnsToController is what a node answers a head that deletes its placement file. @@ -152,8 +155,26 @@ type ModuleDiffers struct { Running string `json:"running,omitempty"` Resource string `json:"resource,omitempty"` Why string `json:"why"` + // Bodies is every resource that differs, both sides, each value a secret could be in masked (issue 1126: the shadow + // shows its work, so why a module differs is read, not guessed). At most MaxBodies, each side cut at MaxBody bytes. + Bodies []ResourceBodies `json:"bodies,omitempty"` } +// ResourceBodies is one differing resource of a module: its id, why, and its body composed and as it runs, masked +// (MaskedPair); a side that does not hold it is empty. +type ResourceBodies struct { + ID string `json:"id"` + Why string `json:"why"` + Composed string `json:"composed,omitempty"` + Running string `json:"running,omitempty"` +} + +// The bounds of a module's shown bodies. +const ( + MaxBodies = 20 + MaxBody = 8192 +) + // ModuleAdded is one module the candidate places on the node and the node does not run: its composition's digest when // it composed, and the values not yet issued for it (its own secrets, its grants, a host port), each in the words the // composition stood in for it with. diff --git a/go/delivery/placement_test.go b/go/delivery/placement_test.go index e0b7810..dd906b2 100644 --- a/go/delivery/placement_test.go +++ b/go/delivery/placement_test.go @@ -283,6 +283,39 @@ func TestPlacement(t *testing.T) { t.Errorf("%s: differs %+v, want %s: %s with both digests", name, c.Differs, tc.resource, tc.why) } } + // Issue 1126, the shadow shows its work: every differing resource with both bodies, each value a secret could be + // in masked (never a secret in an answer), saying whether the two sides hold the same value. + sealedFile := func(id, sealed, mode string) json.RawMessage { + return json.RawMessage(`{"id":"` + id + `","type":"file","module":"nats","path":"/etc/` + id + `","mode":"` + mode + `","sealed":"` + sealed + `"}`) + } + shown := CompareComposedWith(map[string]Composed{"k": comp(sealedFile("a", "box-1", "0600"), sealedFile("b", "box-2", "0600"), file("c", "x"))}, + []Composed{comp(sealedFile("a", "box-1", "0644"), sealedFile("b", "box-3", "0600"), file("c", "y"))}, + func(a, b string) (same, known bool) { return a == "box-2" && b == "box-3", true }) + if len(shown.Differs) != 1 || len(shown.Differs[0].Bodies) != 3 { + t.Fatalf("every differing resource is not shown: %+v", shown.Differs) + } + for _, row := range []struct{ id, has, hasNot string }{ + {"a", `"mode":"0600"`, "box-1"}, + {"b", "another seal of the same value", "box-2"}, + {"c", `"content":"x"`, ""}, + } { + var body ResourceBodies + for _, b := range shown.Differs[0].Bodies { + if b.ID == row.id { + body = b + } + } + both := body.Composed + body.Running + if !strings.Contains(both, row.has) || (row.hasNot != "" && strings.Contains(both, row.hasNot)) { + t.Errorf("%s shown as %+v", row.id, body) + } + } + if c, r := MaskedPair(json.RawMessage(`{"env":{"DB_PASSWORD":"hunter2","PORT":"80"},"list":[{"token":"t1"}]}`), + json.RawMessage(`{"env":{"DB_PASSWORD":"hunter3","PORT":"81"},"list":[{"token":"t1"}]}`), nil); strings.Contains(c+r, "hunter") || + strings.Contains(c+r, `"t1"`) || !strings.Contains(r, `"PORT":"81"`) || !strings.Contains(c, "") || + !strings.Contains(c, "") { + t.Errorf("masked as %s and %s", c, r) + } if c := CompareComposed(nil, []Composed{{Node: "novox", Resources: []json.RawMessage{}}}); len(c.Removed) != 1 || c.Removed[0] != "node" { t.Errorf("the node's own composition run and not composed is removed: %+v", c) } From 4e0c57771e8ce8b89987b0eaebdb8fc9e268c157 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 28 Oct 2026 01:09:02 +0100 Subject: [PATCH 6/7] Bound the bodies an answer shows, naming the rest (issue 1126) --- go/compose/frombus/answer.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/go/compose/frombus/answer.go b/go/compose/frombus/answer.go index 52f3bbb..cefd190 100644 --- a/go/compose/frombus/answer.go +++ b/go/compose/frombus/answer.go @@ -119,6 +119,22 @@ func Answer(ctx context.Context, r Reader, mesh Mesh, s Scope, fps map[string]st running = judged } cmp := delivery.CompareComposedWith(composed, running, s.SameValue) + // The answer rides in the node's have key: its shown bodies are bounded, the rest named only. + budget := MaxShown + for i := range cmp.Differs { + kept := cmp.Differs[i].Bodies[:0] + for _, b := range cmp.Differs[i].Bodies { + if n := len(b.Composed) + len(b.Running); n <= budget { + budget -= n + kept = append(kept, b) + } else { + kept = append(kept, delivery.ResourceBodies{ID: b.ID, Why: b.Why}) + a.Note = "some differing resources are named without their bodies: the answer holds at most " + + fmt.Sprint(MaxShown) + " bytes of them" + } + } + cmp.Differs[i].Bodies = kept + } // A refused module was composed as a stub: what it composed is no difference; its refusal is said. notFailed := func(m string) bool { _, ok := failed[m]; return !ok } a.Same = slices.DeleteFunc(cmp.Same, func(m string) bool { return !notFailed(m) }) @@ -196,6 +212,9 @@ func Answer(ctx context.Context, r Reader, mesh Mesh, s Scope, fps map[string]st return a } +// MaxShown bounds the bytes of shown resource bodies in one answer (delivery.ModuleDiffers.Bodies). +const MaxShown = 256 << 10 + var ( grantStandIn = regexp.MustCompile(`^the grant of \S+ for (\S+) on (\S+)$`) needsStandIn = regexp.MustCompile(`^(\S+) on (\S+) needs `) From 362a3f1378aace745b81b712e24f198f62c01ef5 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 28 Oct 2026 01:39:07 +0100 Subject: [PATCH 7/7] A refusal for want of a provider names the modules another node cannot run in this composition, and a refusal naming no placed module is answered at once (issue 1126, probe #532) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ace and g14 answered 'nothing in this mesh provides artifact-store … assign distribution to a node' while novox runs it: a module another node cannot run in this composition (catalogue Resolution.Unhostable) offers nothing, silently. The refusal now names each such module and what it lacks, and each node left out. And the culprit search composed the whole mesh once per placed module for a refusal no stub changes (ace's answer took 7m46s): only the modules the refusal names are tried, at most 8. --- go/compose/frombus/answer.go | 16 ++++++------- go/compose/frombus/answer_rows_test.go | 32 ++++++++++++++++++++++++++ go/compose/frombus/frombus_test.go | 1 + go/compose/network.go | 3 +++ go/compose/resolve.go | 16 ++++++++++++- go/compose/rest.go | 5 ++++ 6 files changed, 64 insertions(+), 9 deletions(-) diff --git a/go/compose/frombus/answer.go b/go/compose/frombus/answer.go index cefd190..078dbbc 100644 --- a/go/compose/frombus/answer.go +++ b/go/compose/frombus/answer.go @@ -212,6 +212,9 @@ func Answer(ctx context.Context, r Reader, mesh Mesh, s Scope, fps map[string]st return a } +// maxCulprits bounds the modules one refusal is tried on, each a whole composition. +const maxCulprits = 8 + // MaxShown bounds the bytes of shown resource bodies in one answer (delivery.ModuleDiffers.Bodies). const MaxShown = 256 << 10 @@ -251,15 +254,12 @@ func culprit(in compose.Input, node string, fps map[string]string, failed map[st assigned = n.Assigned } } - // The modules the error names first: the words of a module's refusal open with its name. - order := make([]string, 0, len(assigned)) + // Only the modules the error names: the words of a module's refusal name it. Each try is a whole composition, so a + // refusal naming none is no module's, answered at once rather than after a composition per module (issue 1126: ace's + // answer took 7m46s). + var order []string for _, m := range assigned { - if strings.Contains(err.Error(), m) { - order = append(order, m) - } - } - for _, m := range assigned { - if !strings.Contains(err.Error(), m) { + if strings.Contains(err.Error(), m) && len(order) < maxCulprits { order = append(order, m) } } diff --git a/go/compose/frombus/answer_rows_test.go b/go/compose/frombus/answer_rows_test.go index 2a361af..74b1601 100644 --- a/go/compose/frombus/answer_rows_test.go +++ b/go/compose/frombus/answer_rows_test.go @@ -3,6 +3,7 @@ package frombus import ( "context" "encoding/json" + "fmt" "reflect" "slices" "strings" @@ -107,6 +108,11 @@ func answerNamesEveryRefusalAndLeavesAnOldBuildToTheNode(t *testing.T) { if err != nil { t.Fatal(err) } + // A refusal naming no module placed here is no module's: no compose is spent stubbing each in turn (issue 1126: ace's + // answer took 7m46s, one compose per module), though here a stub would compose. + if m := culprit(in, "n1", fps, map[string]string{}, fmt.Errorf("nothing in this mesh provides \"artifact-store\"")); m != "" { + t.Errorf("a refusal naming no placed module was laid on %s", m) + } ownKey, _ := delivery.ComposedNodeKey("n1") own := base[ownKey] carriedRes := []json.RawMessage{ @@ -161,6 +167,32 @@ func answerNamesEveryRefusalAndLeavesAnOldBuildToTheNode(t *testing.T) { } } +// A module another node cannot run in this composition offers nothing in it: the refusal names it and why +// (issue 1126, probe #532: ace and g14 answered "nothing in this mesh provides artifact-store … assign distribution to a +// node" while novox runs it). And a refusal no module's stub changes is answered at once, not after a compose per module +// (ace took 7m46s). +func otherNodeNotResolvedIsNamed(t *testing.T) { + ctx := context.Background() + prov := []byte(`{"module":"prov","version":"1","capabilities":["container-runtime"],"provides":[{"name":"thing","scope":"mesh"}]}`) + cons := []byte(`{"module":"cons","version":"1","requires":["thing"]}`) + // n2 runs prov, which needs a container runtime n2's record does not say it has: n2's set does not resolve here. + b := &bus{placements: map[string]delivery.Placement{ + "n1": {Node: "n1", Decided: delivery.NodeDecided{Hub: true, Address: "10.77.0.1", Endpoint: "n1.example:51820"}, Modules: []delivery.PlacedModule{{Module: "cons"}, {Module: overlay.Name}}}, + "n2": {Node: "n2", Decided: delivery.NodeDecided{Address: "10.77.0.2"}, Modules: []delivery.PlacedModule{{Module: "prov"}, {Module: overlay.Name}}}}, + records: map[string]delivery.NodeRecord{"n1": {Node: "n1", TunnelKey: publicKey(t)}, "n2": {Node: "n2", TunnelKey: publicKey(t)}}, + modules: map[string]delivery.Module{"prov": {Manifest: delivery.ManifestDigest(prov)}, "cons": {Manifest: delivery.ManifestDigest(cons)}}, + manifests: map[string][]byte{delivery.ManifestDigest(prov): prov, delivery.ManifestDigest(cons): cons}} + given := map[string]*delivery.Placement{} + for n, p := range b.placements { + p := p + given[n] = &p + } + a := Answer(ctx, b, Mesh{}, Scope{Node: "n1", Placements: given}, map[string]string{"cons": "src2:" + strings.Repeat("1", 64)}, nil, "h", 1) + if a.Verdict != delivery.CandidateRefused || !strings.Contains(a.Refused, "prov on n2 lacks container-runtime") { + t.Errorf("a refusal for want of n2's provider = %+v", a) + } +} + // A node that reaches no artifact store says what its composition holds of one (issue 1126: ace and g14 answered // "reaches no artifact store", and nothing said why). func noStoreReachedSaysWhy(t *testing.T) { diff --git a/go/compose/frombus/frombus_test.go b/go/compose/frombus/frombus_test.go index ee7697d..e02fa56 100644 --- a/go/compose/frombus/frombus_test.go +++ b/go/compose/frombus/frombus_test.go @@ -123,6 +123,7 @@ func TestInput(t *testing.T) { // saying why (answer_rows_test.go). t.Run("every refusal, and an old build left to the node", answerNamesEveryRefusalAndLeavesAnOldBuildToTheNode) t.Run("no store reached says why", noStoreReachedSaysWhy) + t.Run("another node not resolved is named", otherNodeNotResolvedIsNamed) ctx := context.Background() xterm, err := os.ReadFile("../../catalogue/testdata/catalogue-835bf22b-xterm.module.json") if err != nil { diff --git a/go/compose/network.go b/go/compose/network.go index afc8363..f51d6d7 100644 --- a/go/compose/network.go +++ b/go/compose/network.go @@ -24,6 +24,9 @@ type composer struct { in Input said []string seen map[string]bool + // unhostable is, from the last look at the rest of the mesh, each module assigned to another node that this + // composition finds that node cannot run: what it would have offered is missing here (issue 1126, probe #532). + unhostable []string } func newComposer(in Input) *composer { return &composer{in: in, seen: map[string]bool{}} } diff --git a/go/compose/resolve.go b/go/compose/resolve.go index d2fffd8..a481909 100644 --- a/go/compose/resolve.go +++ b/go/compose/resolve.go @@ -3,6 +3,7 @@ package compose import ( "fmt" "slices" + "strings" "git.novox.be/novox/mesh-sdk/go/catalogue" ) @@ -203,7 +204,7 @@ func (c *composer) resolution(nodeName string) (catalogue.Resolution, error) { return catalogue.Resolution{}, err } shelf := c.shelfFor(node) - world, _, _, err := c.theRestOfTheMesh(nodeName) + world, _, left, err := c.theRestOfTheMesh(nodeName) if err != nil { return catalogue.Resolution{}, err } @@ -226,6 +227,19 @@ func (c *composer) resolution(nodeName string) (catalogue.Resolution, error) { Account: node.Account, AccountHome: node.AccountHome, AgentAccount: node.AgentAccount, AgentAccountHome: node.AgentAccountHome}, world) if err != nil { + // What another node's set would have offered is missing when that set does not resolve here: said with the + // refusal, so "nothing provides" is read beside why (issue 1126, probe #532). + if len(left) > 0 { + var why []string + for _, l := range left { + why = append(why, l.String()) + } + err = fmt.Errorf("%w\n\nand %d node(s) offer nothing here, as they could not be resolved: %s", err, len(left), strings.Join(why, "; ")) + } + if len(c.unhostable) > 0 { + err = fmt.Errorf("%w\n\nand these modules offer nothing here, as their node cannot run them in this composition: %s", + err, strings.Join(c.unhostable, "; ")) + } // The node's own set does not compose: the only failure a mesh-wide gatherer may pass over. return catalogue.Resolution{}, notResolvable{err} } diff --git a/go/compose/rest.go b/go/compose/rest.go index f7509e2..a5caf70 100644 --- a/go/compose/rest.go +++ b/go/compose/rest.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "sort" + "strings" "git.novox.be/novox/mesh-sdk/go/catalogue" ) @@ -100,6 +101,7 @@ func (c *composer) theRestOfTheMesh(exclude string) (catalogue.World, Passes, [] var left []LeftOut offered := map[string][]catalogue.Provider{} var firstHeld []catalogue.Held + c.unhostable = nil for _, o := range others { got, err := catalogue.Resolve(shelf, o.assigned, o.node, catalogue.World{Unchecked: true, Holdings: holdings}) if err != nil { @@ -107,6 +109,9 @@ func (c *composer) theRestOfTheMesh(exclude string) (catalogue.World, Passes, [] left = append(left, LeftOut{Node: o.node.Name, Err: err}) continue } + for _, u := range got.Unhostable { + c.unhostable = append(c.unhostable, fmt.Sprintf("%s on %s lacks %s", u.Module, o.node.Name, strings.Join(u.Missing, ", "))) + } firstHeld = append(firstHeld, got.Claims...) for _, m := range got.Modules { for _, name := range m.OffersAt(catalogue.ScopeMesh) {