From e67916bbd86c5551cc397207171ebc305630b5cf Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 25 Oct 2026 02:44:50 +0200 Subject: [PATCH 1/2] Take every catalogued module's done key once at the start, beside the updates-only watch, so a key written or interrupted across a restart is registered (issue 1945) --- cmd/mesh-controller/composed_keys.go | 54 ++++++++++++++++++- .../composed_keys_start_test.go | 32 +++++++++++ 2 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 cmd/mesh-controller/composed_keys_start_test.go diff --git a/cmd/mesh-controller/composed_keys.go b/cmd/mesh-controller/composed_keys.go index 2a95d9ea..eaf9f87b 100644 --- a/cmd/mesh-controller/composed_keys.go +++ b/cmd/mesh-controller/composed_keys.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "sort" "strings" "sync" "time" @@ -24,7 +25,7 @@ import ( // carries the module's permissions at the new fingerprint: a new revision of the bus module's composition alone, // written into its want at once by the assigner), and writes `grants..`. The assigner's sixth and seventh // rules then let a want name the fingerprint. **No timer**: woken by the keys, and a starting controller looks at every -// module once (B5's start pass). +// module once (doneKeysAtStart). // doneKeySteps are what taking a module's done key in does, each a step a test may stand in for. type doneKeySteps struct { @@ -261,4 +262,55 @@ func watchDoneKeys(ctx context.Context, open *stores) { <-ctx.Done() closeKeys() }() + // And every module once now, after the watch is open so no write falls between the two. + inBackground(func() { + doneKeysAtStart(ctx, catalogedModules(open), func(module string) { takeDoneKeyAtStart(ctx, open, module) }) + }) +} + +// doneKeysAtStart takes in the done key of every catalogued module once, one after another: the start's look the watch of +// new writes cannot give (it hears updates only). A done key written while no controller served, or whose take-in the +// controller this one replaced was stopped in, is taken here; without it the module stays in the node's own composition +// with no credential until its keys are written again (issue 1945, live on 2026-10-25 at 02:34). A take-in with nothing +// to do does nothing; a catalogue that cannot be read is said. +func doneKeysAtStart(ctx context.Context, modules func(context.Context) ([]string, error), take func(module string)) { + names, err := modules(ctx) + if err != nil { + fmt.Printf("done keys: the modules are not looked at once at the start: %v\n", err) + return + } + sort.Strings(names) + for _, m := range names { + if ctx.Err() != nil { + return + } + take(m) + } +} + +// takeDoneKeyAtStart is one take-in of the start's look, under the module's lock; one that fails is handed to takeDoneKey, +// which tries again every doneKeyRetry, loudly, so a failing module never holds up the look at the others. +func takeDoneKeyAtStart(ctx context.Context, open *stores, module string) { + mu, _ := doneKeysMu.LoadOrStore(module, &sync.Mutex{}) + mu.(*sync.Mutex).Lock() + err := takeDoneKeyOnce(ctx, open, module) + mu.(*sync.Mutex).Unlock() + if err != nil && !errors.Is(err, errKeyWaits) { + inBackground(func() { takeDoneKey(ctx, open, module) }) + } +} + +// catalogedModules are the names of every module the catalogue holds. +func catalogedModules(open *stores) func(context.Context) ([]string, error) { + return func(ctx context.Context) ([]string, error) { + entries, err := open.inventory.Catalogued(ctx) + if err != nil { + return nil, err + } + out := make([]string, 0, len(entries)) + for _, e := range entries { + out = append(out, e.Manifest.Module) + } + return out, nil + } } diff --git a/cmd/mesh-controller/composed_keys_start_test.go b/cmd/mesh-controller/composed_keys_start_test.go new file mode 100644 index 00000000..33f266cb --- /dev/null +++ b/cmd/mesh-controller/composed_keys_start_test.go @@ -0,0 +1,32 @@ +package main + +import ( + "context" + "errors" + "reflect" + "sort" + "testing" +) + +// A starting controller takes in the done key of every catalogued module once, beside the watch of new writes: a done key +// written while no controller served, or whose take-in the controller it replaced was stopped in, is otherwise never +// taken, and its module stays in the node's own composition with no credential (issue 1945, live on 2026-10-25 at 02:34: +// the controller replaced mid take-in left mesh-provisioner-client, gitea, postgres and many more unregistered at their +// new fingerprints, and the new one, watching updates only, never looked). +func TestAStartingControllerTakesEveryModulesDoneKeyOnce(t *testing.T) { + var took []string + doneKeysAtStart(context.Background(), func(context.Context) ([]string, error) { + return []string{"mesh-provisioner-client", "gitea", "postgres"}, nil + }, func(module string) { took = append(took, module) }) + sort.Strings(took) + if want := []string{"gitea", "mesh-provisioner-client", "postgres"}; !reflect.DeepEqual(took, want) { + t.Fatalf("took %v, want %v", took, want) + } + // A catalogue that cannot be read takes nothing and is said, never fatal. + took = nil + doneKeysAtStart(context.Background(), func(context.Context) ([]string, error) { return nil, errors.New("down") }, + func(module string) { took = append(took, module) }) + if len(took) != 0 { + t.Fatalf("took %v from a catalogue that could not be read", took) + } +} From eafd1883f65ca8c099d1ceac3f96148f5d5b4798 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 25 Oct 2026 02:57:24 +0200 Subject: [PATCH 2/2] Write a node's own composition last, and not in a pass where a module's composition failed (issue 1945) --- .../composed_node_last_test.go | 56 +++++++++++++++++++ cmd/mesh-controller/composed_writer.go | 20 ++++++- 2 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 cmd/mesh-controller/composed_node_last_test.go diff --git a/cmd/mesh-controller/composed_node_last_test.go b/cmd/mesh-controller/composed_node_last_test.go new file mode 100644 index 00000000..d0fafef0 --- /dev/null +++ b/cmd/mesh-controller/composed_node_last_test.go @@ -0,0 +1,56 @@ +package main + +import ( + "context" + "encoding/json" + "testing" + + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// The node's own composition is written last, and not at all in a pass where a module's composition failed: written +// without the resources that moved to that module, it would have the node-engine remove them while the module's own +// composition is not there to take them (issue 1945, the review's gap 1). +func TestTheNodesOwnIsWrittenLastAndNotAfterAModuleFailed(t *testing.T) { + own, _ := sdkdelivery.ComposedNodeKey("ace") + // "aaa" sorts before "ace.node"... and "zzz" after: the node's own must still come last. + a, _ := sdkdelivery.ComposedKey("ace", "aaa", fp2For('a')) + z, _ := sdkdelivery.ComposedKey("ace", "zzz", fp2For('b')) + file := func(module string) json.RawMessage { + return json.RawMessage(`{"id":"` + module + `.f","type":"file","module":"` + module + `","path":"/etc/` + module + + `","mode":"0644","content":"x"}`) + } + comps := map[string]sdkdelivery.Composed{ + own: {Node: "ace", Resources: []json.RawMessage{}}, + a: {Node: "ace", Module: "aaa", FP: fp2For('a'), Resources: []json.RawMessage{file("aaa")}}, + z: {Node: "ace", Module: "zzz", FP: fp2For('b'), Resources: []json.RawMessage{file("zzz")}}, + } + at, _, _, _ := aPlace(t, "ace") + wrote, err := writeNodeCompositions(context.Background(), at, comps) + if err != nil { + t.Fatal(err) + } + if len(wrote) != 3 || wrote[2] != own { + t.Fatalf("written in the order %v, the node's own last", wrote) + } + + // A module's composition that cannot be written: the node's own is not written in that pass. + at, _, _, _ = aPlace(t, "ace") + broken := comps[z] + broken.Resources = []json.RawMessage{file("other")} // a resource of another module: refused + comps[z] = broken + if _, err := writeNodeCompositions(context.Background(), at, comps); err == nil { + t.Fatal("a composition that cannot be written was not said") + } + if _, rev, _ := readComposed(context.Background(), at.bucket, own); rev != 0 { + t.Fatal("the node's own composition was written in a pass where a module's failed") + } +} + +func fp2For(c byte) string { + b := make([]byte, 64) + for i := range b { + b[i] = c + } + return sdkdelivery.SourcePrefix2 + string(b) +} diff --git a/cmd/mesh-controller/composed_writer.go b/cmd/mesh-controller/composed_writer.go index a6fec709..b5f54b0b 100644 --- a/cmd/mesh-controller/composed_writer.go +++ b/cmd/mesh-controller/composed_writer.go @@ -321,9 +321,18 @@ func casWrite(ctx context.Context, kv composedBucketsKV, key string, value []byt // writeNodeCompositions writes every composition of a node that differs, and prunes what is no longer kept. It answers // the keys it wrote. func writeNodeCompositions(ctx context.Context, at composedPlace, comps map[string]sdkdelivery.Composed) ([]string, error) { + // The modules' compositions first and the node's own last, and the node's own not at all in a pass where a module's + // failed: written without the resources that moved to that module, it would have the node-engine remove them while + // the module's own composition is not there to take them (issue 1945, the review's gap 1). + own, err := sdkdelivery.ComposedNodeKey(at.node) + if err != nil { + return nil, err + } keys := make([]string, 0, len(comps)) for k := range comps { - keys = append(keys, k) + if k != own { + keys = append(keys, k) + } } sort.Strings(keys) var wrote []string @@ -338,6 +347,15 @@ func writeNodeCompositions(ctx context.Context, at composedPlace, comps map[stri wrote = append(wrote, key) } } + if c, ok := comps[own]; ok { + if len(errs) > 0 { + errs = append(errs, fmt.Errorf("%s is not written: a module's composition of this pass was not", own)) + } else if w, err := writeComposed(ctx, at, own, c); err != nil { + errs = append(errs, err) + } else if w { + wrote = append(wrote, own) + } + } if err := pruneCompositions(ctx, at); err != nil { errs = append(errs, err) }