Merge pull request 'Register every done key once at the start, and write a node's own composition last (issue 1945)' (#459) from fix/r6-fp2-seen into main

cmd/mesh-controller/composed_keys.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"sort"
 	"strings"
 	"sync"
 	"time"
@@ -24,7 +25,7 @@ import (
 // carries the module's permissions at the new fingerprint: a new revision of the bus module's composition alone,
 // written into its want at once by the assigner), and writes `grants.<module>.<fp>`. The assigner's sixth and seventh
 // rules then let a want name the fingerprint. **No timer**: woken by the keys, and a starting controller looks at every
-// module once (B5's start pass).
+// module once (doneKeysAtStart).
 
 // doneKeySteps are what taking a module's done key in does, each a step a test may stand in for.
 type doneKeySteps struct {
@@ -261,4 +262,55 @@ func watchDoneKeys(ctx context.Context, open *stores) {
 		<-ctx.Done()
 		closeKeys()
 	}()
+	// And every module once now, after the watch is open so no write falls between the two.
+	inBackground(func() {
+		doneKeysAtStart(ctx, catalogedModules(open), func(module string) { takeDoneKeyAtStart(ctx, open, module) })
+	})
+}
+
+// doneKeysAtStart takes in the done key of every catalogued module once, one after another: the start's look the watch of
+// new writes cannot give (it hears updates only). A done key written while no controller served, or whose take-in the
+// controller this one replaced was stopped in, is taken here; without it the module stays in the node's own composition
+// with no credential until its keys are written again (issue 1945, live on 2026-10-25 at 02:34). A take-in with nothing
+// to do does nothing; a catalogue that cannot be read is said.
+func doneKeysAtStart(ctx context.Context, modules func(context.Context) ([]string, error), take func(module string)) {
+	names, err := modules(ctx)
+	if err != nil {
+		fmt.Printf("done keys: the modules are not looked at once at the start: %v\n", err)
+		return
+	}
+	sort.Strings(names)
+	for _, m := range names {
+		if ctx.Err() != nil {
+			return
+		}
+		take(m)
+	}
+}
+
+// takeDoneKeyAtStart is one take-in of the start's look, under the module's lock; one that fails is handed to takeDoneKey,
+// which tries again every doneKeyRetry, loudly, so a failing module never holds up the look at the others.
+func takeDoneKeyAtStart(ctx context.Context, open *stores, module string) {
+	mu, _ := doneKeysMu.LoadOrStore(module, &sync.Mutex{})
+	mu.(*sync.Mutex).Lock()
+	err := takeDoneKeyOnce(ctx, open, module)
+	mu.(*sync.Mutex).Unlock()
+	if err != nil && !errors.Is(err, errKeyWaits) {
+		inBackground(func() { takeDoneKey(ctx, open, module) })
+	}
+}
+
+// catalogedModules are the names of every module the catalogue holds.
+func catalogedModules(open *stores) func(context.Context) ([]string, error) {
+	return func(ctx context.Context) ([]string, error) {
+		entries, err := open.inventory.Catalogued(ctx)
+		if err != nil {
+			return nil, err
+		}
+		out := make([]string, 0, len(entries))
+		for _, e := range entries {
+			out = append(out, e.Manifest.Module)
+		}
+		return out, nil
+	}
 }

cmd/mesh-controller/composed_keys_start_test.go

@@ -0,0 +1,32 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"reflect"
+	"sort"
+	"testing"
+)
+
+// A starting controller takes in the done key of every catalogued module once, beside the watch of new writes: a done key
+// written while no controller served, or whose take-in the controller it replaced was stopped in, is otherwise never
+// taken, and its module stays in the node's own composition with no credential (issue 1945, live on 2026-10-25 at 02:34:
+// the controller replaced mid take-in left mesh-provisioner-client, gitea, postgres and many more unregistered at their
+// new fingerprints, and the new one, watching updates only, never looked).
+func TestAStartingControllerTakesEveryModulesDoneKeyOnce(t *testing.T) {
+	var took []string
+	doneKeysAtStart(context.Background(), func(context.Context) ([]string, error) {
+		return []string{"mesh-provisioner-client", "gitea", "postgres"}, nil
+	}, func(module string) { took = append(took, module) })
+	sort.Strings(took)
+	if want := []string{"gitea", "mesh-provisioner-client", "postgres"}; !reflect.DeepEqual(took, want) {
+		t.Fatalf("took %v, want %v", took, want)
+	}
+	// A catalogue that cannot be read takes nothing and is said, never fatal.
+	took = nil
+	doneKeysAtStart(context.Background(), func(context.Context) ([]string, error) { return nil, errors.New("down") },
+		func(module string) { took = append(took, module) })
+	if len(took) != 0 {
+		t.Fatalf("took %v from a catalogue that could not be read", took)
+	}
+}

cmd/mesh-controller/composed_node_last_test.go

@@ -0,0 +1,56 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+
+	sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
+)
+
+// The node's own composition is written last, and not at all in a pass where a module's composition failed: written
+// without the resources that moved to that module, it would have the node-engine remove them while the module's own
+// composition is not there to take them (issue 1945, the review's gap 1).
+func TestTheNodesOwnIsWrittenLastAndNotAfterAModuleFailed(t *testing.T) {
+	own, _ := sdkdelivery.ComposedNodeKey("ace")
+	// "aaa" sorts before "ace.node"... and "zzz" after: the node's own must still come last.
+	a, _ := sdkdelivery.ComposedKey("ace", "aaa", fp2For('a'))
+	z, _ := sdkdelivery.ComposedKey("ace", "zzz", fp2For('b'))
+	file := func(module string) json.RawMessage {
+		return json.RawMessage(`{"id":"` + module + `.f","type":"file","module":"` + module + `","path":"/etc/` + module +
+			`","mode":"0644","content":"x"}`)
+	}
+	comps := map[string]sdkdelivery.Composed{
+		own: {Node: "ace", Resources: []json.RawMessage{}},
+		a:   {Node: "ace", Module: "aaa", FP: fp2For('a'), Resources: []json.RawMessage{file("aaa")}},
+		z:   {Node: "ace", Module: "zzz", FP: fp2For('b'), Resources: []json.RawMessage{file("zzz")}},
+	}
+	at, _, _, _ := aPlace(t, "ace")
+	wrote, err := writeNodeCompositions(context.Background(), at, comps)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(wrote) != 3 || wrote[2] != own {
+		t.Fatalf("written in the order %v, the node's own last", wrote)
+	}
+
+	// A module's composition that cannot be written: the node's own is not written in that pass.
+	at, _, _, _ = aPlace(t, "ace")
+	broken := comps[z]
+	broken.Resources = []json.RawMessage{file("other")} // a resource of another module: refused
+	comps[z] = broken
+	if _, err := writeNodeCompositions(context.Background(), at, comps); err == nil {
+		t.Fatal("a composition that cannot be written was not said")
+	}
+	if _, rev, _ := readComposed(context.Background(), at.bucket, own); rev != 0 {
+		t.Fatal("the node's own composition was written in a pass where a module's failed")
+	}
+}
+
+func fp2For(c byte) string {
+	b := make([]byte, 64)
+	for i := range b {
+		b[i] = c
+	}
+	return sdkdelivery.SourcePrefix2 + string(b)
+}

cmd/mesh-controller/composed_writer.go

@@ -321,9 +321,18 @@ func casWrite(ctx context.Context, kv composedBucketsKV, key string, value []byt
 // writeNodeCompositions writes every composition of a node that differs, and prunes what is no longer kept. It answers
 // the keys it wrote.
 func writeNodeCompositions(ctx context.Context, at composedPlace, comps map[string]sdkdelivery.Composed) ([]string, error) {
+	// The modules' compositions first and the node's own last, and the node's own not at all in a pass where a module's
+	// failed: written without the resources that moved to that module, it would have the node-engine remove them while
+	// the module's own composition is not there to take them (issue 1945, the review's gap 1).
+	own, err := sdkdelivery.ComposedNodeKey(at.node)
+	if err != nil {
+		return nil, err
+	}
 	keys := make([]string, 0, len(comps))
 	for k := range comps {
-		keys = append(keys, k)
+		if k != own {
+			keys = append(keys, k)
+		}
 	}
 	sort.Strings(keys)
 	var wrote []string
@@ -338,6 +347,15 @@ func writeNodeCompositions(ctx context.Context, at composedPlace, comps map[stri
 			wrote = append(wrote, key)
 		}
 	}
+	if c, ok := comps[own]; ok {
+		if len(errs) > 0 {
+			errs = append(errs, fmt.Errorf("%s is not written: a module's composition of this pass was not", own))
+		} else if w, err := writeComposed(ctx, at, own, c); err != nil {
+			errs = append(errs, err)
+		} else if w {
+			wrote = append(wrote, own)
+		}
+	}
 	if err := pruneCompositions(ctx, at); err != nil {
 		errs = append(errs, err)
 	}