diff --git a/cmd/mesh-controller/composed_keys_test.go b/cmd/mesh-controller/composed_keys_test.go index d29be2c9..b336ba9f 100644 --- a/cmd/mesh-controller/composed_keys_test.go +++ b/cmd/mesh-controller/composed_keys_test.go @@ -99,7 +99,7 @@ func TestAChangedUserListIsANewRevisionOfTheBusModulesCompositionAlone(t *testin d["resources"] = append(d["resources"].([]any), map[string]any{"id": "nats.bus-users", "type": "file", "module": "nats", "path": "/var/lib/mesh/nats/users.conf", "content": users, "mode": "0600"}) raw, _ := json.Marshal(d) - comps, _, err := compositionsOf("g14", raw, fps, nil, nil) + comps, _, _, err := compositionsOf("g14", raw, fps, nil, nil, nil) if err != nil { t.Fatal(err) } diff --git a/cmd/mesh-controller/composed_live.go b/cmd/mesh-controller/composed_live.go index c4efa78d..d910c03e 100644 --- a/cmd/mesh-controller/composed_live.go +++ b/cmd/mesh-controller/composed_live.go @@ -119,7 +119,7 @@ func composeModulesByNode(ctx context.Context, open *stores, nodes []string) (ma if err := open.inventory.RecordComposedPlans(ctx, node, plansKept(declared)); err != nil { errsOf[node] = append(errsOf[node], fmt.Errorf("%s: its provisioner's plans: %w", node, err)) } - comps, moved, err := compositionsOf(node, body, fps, manifests, declared.modules) + comps, moved, left, err := compositionsOf(node, body, fps, manifests, declared.modules, namedByOwnComposition(ctx, at)) if err != nil { errsOf[node] = append(errsOf[node], fmt.Errorf("%s: %w", node, err)) return @@ -128,6 +128,11 @@ func composeModulesByNode(ctx context.Context, open *stores, nodes []string) (ma fmt.Printf("compositions: %s's %s compose with the node's own: no fingerprint of the new form names them\n", node, strings.Join(moved, ", ")) } + if len(left) > 0 { + fmt.Printf("compositions: %s's %s are left out of the node's own: the controller knows no fingerprint of the "+ + "new form for them, and the node's want or have names their own composition (issue 1945)\n", + node, strings.Join(left, ", ")) + } if composeTrace != nil { traceComposed(ctx, at, node, comps) } @@ -156,6 +161,32 @@ func composeModulesByNode(ctx context.Context, open *stores, nodes []string) (ma return out, byNode } +// namedByOwnComposition says a module's want or have on the node names a composition of its own: a fingerprint of the +// new form (issue 1945). +func namedByOwnComposition(ctx context.Context, at composedPlace) func(module string) (bool, error) { + return func(module string) (bool, error) { + if at.wanted != nil { + fp, err := at.wanted(ctx, module) + if err != nil { + return false, err + } + if strings.HasPrefix(fp, sdkdelivery.SourcePrefix2) { + return true, nil + } + } + if at.ran != nil { + h, found, err := at.ran(ctx, module) + if err != nil { + return false, err + } + if found && strings.HasPrefix(h.FP, sdkdelivery.SourcePrefix2) { + return true, nil + } + } + return false, nil + } +} + // composeAndSay is composeModules for an act that has nothing to do with its answer but say it: what each former send // of a verb or an event calls. func composeAndSay(ctx context.Context, open *stores, nodes []string) error { diff --git a/cmd/mesh-controller/composed_objects_test.go b/cmd/mesh-controller/composed_objects_test.go index 7fe9b8d9..69b75367 100644 --- a/cmd/mesh-controller/composed_objects_test.go +++ b/cmd/mesh-controller/composed_objects_test.go @@ -26,7 +26,7 @@ func TestALargeFileOfACompositionHasItsObjectInTheBucket(t *testing.T) { if err != nil { t.Fatal(err) } - comps, _, err := compositionsOf("novox", body, nil, nil, nil) + comps, _, _, err := compositionsOf("novox", body, nil, nil, nil, nil) if err != nil { t.Fatal(err) } @@ -50,7 +50,7 @@ func TestACompositionNamingAnObjectTheBucketLacksIsRefused(t *testing.T) { if err != nil { t.Fatal(err) } - comps, _, err := compositionsOf("novox", body, nil, nil, nil) + comps, _, _, err := compositionsOf("novox", body, nil, nil, nil, nil) if err != nil { t.Fatal(err) } @@ -78,7 +78,7 @@ func TestEveryFilledFileStaysFilledInTheCompositions(t *testing.T) { for _, m := range []string{"power", "claude-code", "node-env"} { delete(fps, m) } - comps, _, err := compositionsOf(node, body, fps, nil, nil) + comps, _, _, err := compositionsOf(node, body, fps, nil, nil, nil) if err != nil { t.Fatalf("%s: %v", node, err) } diff --git a/cmd/mesh-controller/composed_writer.go b/cmd/mesh-controller/composed_writer.go index fef48a36..4a7c7a10 100644 --- a/cmd/mesh-controller/composed_writer.go +++ b/cmd/mesh-controller/composed_writer.go @@ -67,8 +67,15 @@ type composedPlace struct { // fingerprint fps gives it, the node's own with its adoption and provisioner, and every resource of a module fps names // no fingerprint for in the node's own, its module taken off it and named in moved. Data says the module's manifest // declares data. -func compositionsOf(node string, body []byte, fps map[string]string, manifests map[string]catalogue.Manifest, runs []string) ( - out map[string]sdkdelivery.Composed, moved []string, err error) { +// +// **A module a want or a have names by its own composition is never in the node's own** (issue 1945): the node-engine +// runs that module's composition, and the same resources in the node's own were declared twice, which it refuses +// whole ("2 resources declare X", every node's apply failing from 2026-10-25 12:43). named says, for a module fps names +// no fingerprint of the new form for, that the node's want or have names a fingerprint of the new form; such a module's +// resources and filled paths are left out of this pass (its own composition, as written before, stays what the node +// runs) and it is named in left. +func compositionsOf(node string, body []byte, fps map[string]string, manifests map[string]catalogue.Manifest, runs []string, + named func(module string) (bool, error)) (out map[string]sdkdelivery.Composed, moved, left []string, err error) { var envelope struct { Resources []map[string]json.RawMessage `json:"resources"` @@ -77,9 +84,9 @@ func compositionsOf(node string, body []byte, fps map[string]string, manifests m Provisioner string `json:"provisioner_account"` } if err := json.Unmarshal(body, &envelope); err != nil { - return nil, nil, fmt.Errorf("%s's declaration: %w", node, err) + return nil, nil, nil, fmt.Errorf("%s's declaration: %w", node, err) } - inModule := map[string]bool{} + inModule, ownComposition := map[string]bool{}, map[string]bool{} resources := make([]json.RawMessage, 0, len(envelope.Resources)) for _, r := range envelope.Resources { var module string @@ -87,15 +94,30 @@ func compositionsOf(node string, body []byte, fps map[string]string, manifests m _ = json.Unmarshal(raw, &module) } if module != "" && !strings.HasPrefix(fps[module], sdkdelivery.SourcePrefix2) { - delete(r, "module") - if !inModule[module] { - inModule[module] = true - moved = append(moved, module) + if !inModule[module] && !ownComposition[module] { + isNamed := false + if named != nil { + if isNamed, err = named(module); err != nil { + return nil, nil, nil, fmt.Errorf("whether %s's want or have names %s's own composition: %w", + node, module, err) + } + } + if isNamed { + ownComposition[module] = true + left = append(left, module) + } else { + inModule[module] = true + moved = append(moved, module) + } } + if ownComposition[module] { + continue + } + delete(r, "module") } raw, err := json.Marshal(r) if err != nil { - return nil, nil, err + return nil, nil, nil, err } resources = append(resources, raw) } @@ -106,22 +128,26 @@ func compositionsOf(node string, body []byte, fps map[string]string, manifests m // and shanks). var filled, ownFilled []string for _, p := range envelope.Filled { - if filledByMoved(envelope.Resources, p, inModule) { + switch { + case filledByMoved(envelope.Resources, p, ownComposition): + // The file of a module whose own composition the node runs: that composition names it, not this pass. + case filledByMoved(envelope.Resources, p, inModule): ownFilled = append(ownFilled, p) - } else { + default: filled = append(filled, p) } } sort.Strings(moved) + sort.Strings(left) out, err = sdkdelivery.SplitComposed(node, resources, fps, filled, sdkdelivery.NodeOwn{Adoption: envelope.Adoption, Provisioner: envelope.Provisioner}) if err != nil { - return nil, nil, err + return nil, nil, nil, err } if len(ownFilled) > 0 { own, err := sdkdelivery.ComposedNodeKey(node) if err != nil { - return nil, nil, err + return nil, nil, nil, err } c := out[own] c.Filled = append(c.Filled, ownFilled...) @@ -149,7 +175,7 @@ func compositionsOf(node string, body []byte, fps map[string]string, manifests m out[key] = c } } - return out, moved, nil + return out, moved, left, nil } // filledByMoved says a filled path is a file of a module whose resources were moved into the node's own composition. diff --git a/cmd/mesh-controller/composed_writer_test.go b/cmd/mesh-controller/composed_writer_test.go index 65efa8df..d28d3a9a 100644 --- a/cmd/mesh-controller/composed_writer_test.go +++ b/cmd/mesh-controller/composed_writer_test.go @@ -218,7 +218,7 @@ func TestTheCompositionsOfANodeAreItsDeclaration(t *testing.T) { Resources []json.RawMessage `json:"resources"` } _ = json.Unmarshal(body, &d) - comps, moved, err := compositionsOf(node, body, fps, nil, nil) + comps, moved, _, err := compositionsOf(node, body, fps, nil, nil, nil) if err != nil { t.Fatalf("%s: %v", node, err) } @@ -253,7 +253,7 @@ func TestTheCompositionsOfANodeAreItsDeclaration(t *testing.T) { // unchanged it writes nothing; a changed input writes a new revision of that key alone, by compare-and-set. func TestACompositionIsWrittenOnceAndAgainOnlyWhenItChanges(t *testing.T) { body, fps := capturedBody(t, "g14") - comps, _, err := compositionsOf("g14", body, fps, nil, nil) + comps, _, _, err := compositionsOf("g14", body, fps, nil, nil, nil) if err != nil { t.Fatal(err) } @@ -342,7 +342,7 @@ func TestCompositionsOutsideTheFiveNewestAreDeleted(t *testing.T) { func TestAModuleWithNoResourceOfItsOwnStillHasItsComposition(t *testing.T) { body := []byte(`{"declaration":1,"resources":[]}`) fp := sdkdelivery.SourcePrefix2 + strings.Repeat("b", 64) - comps, _, err := compositionsOf("laptop", body, map[string]string{"board": fp}, nil, []string{"board", "legacy"}) + comps, _, _, err := compositionsOf("laptop", body, map[string]string{"board": fp}, nil, []string{"board", "legacy"}, nil) if err != nil { t.Fatal(err) } @@ -360,3 +360,54 @@ func TestAModuleWithNoResourceOfItsOwnStillHasItsComposition(t *testing.T) { } } } + +// 1945/AC16 (unit): a module the controller knows no fingerprint of the new form for, whose want or have on the node +// names its own composition, is never in the node's own composition, neither its resources nor its filled paths, and is +// named as left; a module neither names still goes with the node's own, its filled path with it. (Live from 2026-10-25 +// 12:43: zsh, power, restic and more were in both, and every node refused "2 resources declare X".) +func TestAModuleItsWantNamesIsNeverInTheNodesOwnComposition(t *testing.T) { + body := []byte(`{"declaration":1,"resources":[ + {"id":"zsh.rc","type":"file","module":"zsh","path":"/etc/zsh/zshrc"}, + {"id":"power.unit","type":"file","module":"power","path":"/etc/power.conf"}, + {"id":"mesh-wireguard.conf","type":"file","module":"mesh-wireguard","path":"/etc/wireguard/mesh0.conf"}], + "filled_files":["/etc/zsh/zshrc","/etc/power.conf","/etc/wireguard/mesh0.conf"]}`) + asked := map[string]int{} + named := func(module string) (bool, error) { + asked[module]++ + return module == "zsh" || module == "power", nil + } + comps, moved, left, err := compositionsOf("ace", body, map[string]string{}, nil, nil, named) + if err != nil { + t.Fatal(err) + } + if strings.Join(left, ",") != "power,zsh" || strings.Join(moved, ",") != "mesh-wireguard" { + t.Errorf("left %v, moved %v; want power,zsh left and mesh-wireguard moved", left, moved) + } + if asked["zsh"] != 1 || asked["power"] != 1 { + t.Errorf("asked each module once: %v", asked) + } + own, _ := sdkdelivery.ComposedNodeKey("ace") + for key, c := range comps { + for _, id := range resourceIDs(t, c.Resources) { + if strings.HasPrefix(id, "zsh.") || strings.HasPrefix(id, "power.") { + t.Errorf("%s holds %s, which the module's own composition declares", key, id) + } + } + for _, p := range c.Filled { + if p == "/etc/zsh/zshrc" || p == "/etc/power.conf" { + t.Errorf("%s names %s as filled, which the module's own composition names", key, p) + } + } + } + if ids := strings.Join(resourceIDs(t, comps[own].Resources), " "); ids != "mesh-wireguard.conf" { + t.Errorf("the node's own composition holds %q, want mesh-wireguard.conf alone", ids) + } + if f := strings.Join(comps[own].Filled, " "); f != "/etc/wireguard/mesh0.conf" { + t.Errorf("the node's own composition names %q as filled, want mesh-wireguard's file alone", f) + } + + if _, _, _, err := compositionsOf("ace", body, map[string]string{}, nil, nil, + func(string) (bool, error) { return false, fmt.Errorf("the wants bucket is gone") }); err == nil { + t.Errorf("a want that cannot be read composed the node anyway") + } +} diff --git a/cmd/mesh-controller/fingerprints_at_test.go b/cmd/mesh-controller/fingerprints_at_test.go new file mode 100644 index 00000000..f9fa255a --- /dev/null +++ b/cmd/mesh-controller/fingerprints_at_test.go @@ -0,0 +1,93 @@ +package main + +import ( + "context" + "errors" + "fmt" + "testing" + + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" + + "github.com/novox/mesh-controller/internal/builderkeys" +) + +// 1945/AC15 (unit): a module registered at a commit no build record points at (a catalogue merge re-indexed it with +// the fingerprint it had, and the take-in registered it from the done key whose record stands at its first commit) +// has, as its current fingerprint, the one the indexer's trunk value at that commit names; a module whose commit has a +// build keeps that build's; a module the indexer names nothing of the new form for has none; an indexer error is the +// answer's, never a module silently left out. +func TestACommitWithNoBuildOfItsOwnTakesTheIndexersFingerprint(t *testing.T) { + zsh, board := grantFP('a'), grantFP('b') + commits := map[string]string{"zsh": "68f7431d98d7", "board": "2167b95a1111", "legacy": "abcdef012345", "held": ""} + built := func(_ context.Context, module, commit string) (string, bool, error) { + if module == "board" { + return board, true, nil + } + return "", false, nil + } + indexed := func(_ context.Context, module, commit string) (string, error) { + switch module { + case "zsh": + return zsh, nil + case "board": + t.Errorf("the indexer was asked for board, whose commit has a build") + case "held": + t.Errorf("the indexer was asked for a module with no commit") + } + return "", nil + } + got, err := fingerprintsAt(context.Background(), commits, built, indexed) + if err != nil { + t.Fatal(err) + } + if got["zsh"] != zsh || got["board"] != board || len(got) != 2 { + t.Errorf("got %v, want zsh at %s and board at %s alone", got, zsh, board) + } + + broken := func(context.Context, string, string) (string, error) { return "", errors.New("the bus is gone") } + if _, err := fingerprintsAt(context.Background(), commits, built, broken); err == nil { + t.Errorf("an indexer error was not the answer's") + } +} + +// fakeTrunk is the indexer's trunk values of one module: now, and its history by commit. +type fakeTrunk struct { + now *sdkdelivery.Module + history map[string]sdkdelivery.Module + asked []string +} + +func (f *fakeTrunk) ModuleAt(_ context.Context, module, commit string) (sdkdelivery.Module, error) { + f.asked = append(f.asked, commit) + if commit == "" { + if f.now == nil { + return sdkdelivery.Module{}, fmt.Errorf("%w (%s)", builderkeys.ErrNoValue, module) + } + return *f.now, nil + } + m, ok := f.history[commit] + if !ok { + return sdkdelivery.Module{}, fmt.Errorf("%w (%s at %s)", builderkeys.ErrNoValue, module, commit) + } + return m, nil +} + +// 1945/AC15 (unit): the indexer's fingerprint at a commit is read from the value it holds now when that is the +// commit's (one read), else from the key's history at the commit; none held is no fingerprint. +func TestTheIndexersFingerprintAtACommit(t *testing.T) { + now := sdkdelivery.Module{Commit: "68f7431d98d7a1279cba3a615b6e9f22c3295783", FP2: grantFP('a')} + f := &fakeTrunk{now: &now} + if fp, err := indexedFingerprintAt(context.Background(), f, "zsh", "68f7431d98d7a1279cba3a615b6e9f22c3295783"); err != nil || fp != grantFP('a') || len(f.asked) != 1 { + t.Errorf("now's commit: %q (%v), asked %v", fp, err, f.asked) + } + f = &fakeTrunk{now: &now, history: map[string]sdkdelivery.Module{"2167b95a": {Commit: "2167b95a", FP2: grantFP('c')}}} + if fp, err := indexedFingerprintAt(context.Background(), f, "zsh", "2167b95a"); err != nil || fp != grantFP('c') { + t.Errorf("an older commit: %q (%v)", fp, err) + } + if fp, err := indexedFingerprintAt(context.Background(), f, "zsh", "99999999"); err != nil || fp != "" { + t.Errorf("a commit the indexer never wrote: %q (%v)", fp, err) + } + if fp, err := indexedFingerprintAt(context.Background(), &fakeTrunk{}, "zsh", "68f7431d"); err != nil || fp != "" { + t.Errorf("no value at all: %q (%v)", fp, err) + } +} diff --git a/cmd/mesh-controller/grants.go b/cmd/mesh-controller/grants.go index 802bc549..32310c53 100644 --- a/cmd/mesh-controller/grants.go +++ b/cmd/mesh-controller/grants.go @@ -11,6 +11,7 @@ import ( "github.com/nats-io/nats.go/jetstream" "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/builderkeys" "github.com/novox/mesh-controller/internal/inventory" ) @@ -55,29 +56,128 @@ func grantsFrom(principals []broker.Principal, hashes map[string]string, current // currentFingerprints is, by module, the fingerprint of the build each registered module is registered at: its // manifest's commit (built_from), that commit's newest successful build, and the fingerprint that build names. +// +// **A commit with no build of its own asks the indexer** (issue 1945, live from 2026-10-25 12:43). A catalogue merge +// that leaves a module's sources alone re-indexes it at the merge's commit with the fingerprint it had: the take-in +// registers the module at the new commit from the same done key, whose build record (one per key and revision) already +// stands at the commit it was first taken in at, so the new commit has no build pointing at it. Read only by the +// commit, such a module had no fingerprint of the new form: it was composed into the node's own composition while its +// want named its own, and every node declared its files twice. The indexer's trunk value at the commit names the +// fingerprint the module was registered from, so that is its fingerprint. func currentFingerprints(ctx context.Context, inv *inventory.Inventory) (map[string]string, error) { builds, err := inv.CurrentBuilds(ctx) if err != nil { return nil, err } - out := map[string]string{} + commits := make(map[string]string, len(builds)) for module, b := range builds { - if b.Commit == "" { + commits[module] = b.Commit + } + var indexed func(ctx context.Context, module, commit string) (string, error) + var closeIndexed func() + defer func() { + if closeIndexed != nil { + closeIndexed() + } + }() + return fingerprintsAt(ctx, commits, + func(ctx context.Context, module, commit string) (string, bool, error) { + id, err := inv.BuildOf(ctx, module, commit) + if err != nil || id == "" { + return "", false, nil + } + build, found, err := inv.BuildByID(ctx, id) + if err != nil || !found { + return "", false, nil + } + return build.SourceFingerprint, true, nil + }, + func(ctx context.Context, module, commit string) (string, error) { + if indexed == nil { + read, done, err := openIndexedFingerprints() + if err != nil { + return "", err + } + indexed, closeIndexed = read, done + } + return indexed(ctx, module, commit) + }) +} + +// fingerprintsAt is, by module, the fingerprint of the build registered at its commit (built), or else the fingerprint +// of the new form the indexer's trunk value at that commit names (indexed). A module with no commit (held without a +// source) has none, and a module neither knows has none. An error of the indexer's is the answer's: a module left out +// for it would be composed into the node's own composition beside its own (issue 1945). +func fingerprintsAt(ctx context.Context, commits map[string]string, + built func(ctx context.Context, module, commit string) (string, bool, error), + indexed func(ctx context.Context, module, commit string) (string, error)) (map[string]string, error) { + modules := make([]string, 0, len(commits)) + for module := range commits { + modules = append(modules, module) + } + sort.Strings(modules) + out := map[string]string{} + for _, module := range modules { + commit := commits[module] + if commit == "" { continue } - id, err := inv.BuildOf(ctx, module, b.Commit) - if err != nil || id == "" { + fp, found, err := built(ctx, module, commit) + if err != nil { + return nil, err + } + if found { + out[module] = fp continue } - build, found, err := inv.BuildByID(ctx, id) - if err != nil || !found { - continue + fp, err = indexed(ctx, module, commit) + if err != nil { + return nil, fmt.Errorf("the fingerprint %s is registered at (%.8s): %w", module, commit, err) + } + if strings.HasPrefix(fp, sdkdelivery.SourcePrefix2) { + out[module] = fp } - out[module] = build.SourceFingerprint } return out, nil } +// openIndexedFingerprints opens the indexer's trunk values: what reads the fingerprint of the new form a module's value +// at a commit names ("" when it holds none for that commit), and what closes the bus connection it opened. +var openIndexedFingerprints = func() (func(ctx context.Context, module, commit string) (string, error), func(), error) { + keys, closeKeys, err := openBuilderKeys() + if err != nil { + return nil, nil, fmt.Errorf("the indexer's trunk values: %w", err) + } + return func(ctx context.Context, module, commit string) (string, error) { + return indexedFingerprintAt(ctx, keys, module, commit) + }, closeKeys, nil +} + +// indexedFingerprintAt is the fingerprint of the new form the indexer's value of module at commit names: the value it +// holds now when that is the commit's (one read), else the newest of the key's history at the commit; "" when none. +func indexedFingerprintAt(ctx context.Context, keys interface { + ModuleAt(ctx context.Context, module, commit string) (sdkdelivery.Module, error) +}, module, commit string) (string, error) { + m, err := keys.ModuleAt(ctx, module, "") + if err == nil && sameCommitID(m.Commit, commit) { + return m.FP2, nil + } + if err != nil && !errors.Is(err, builderkeys.ErrNoValue) { + return "", err + } + if errors.Is(err, builderkeys.ErrNoValue) { + return "", nil + } + m, err = keys.ModuleAt(ctx, module, commit) + if errors.Is(err, builderkeys.ErrNoValue) { + return "", nil + } + if err != nil { + return "", err + } + return m.FP2, nil +} + // grantsKV is what writing the grants needs of their bucket. type grantsKV interface { Get(ctx context.Context, key string) (jetstream.KeyValueEntry, error)