package main import ( "context" "encoding/json" "errors" "flag" "fmt" "os" "reflect" "sort" "strings" "time" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) // **The move of config** (novox/hq ADR 0348 §9 and its consequences, design 57 K5, issue 664): every config value a // settings layer holds is set in mesh-config, one of the moves of data a release runs once after its delivery, after // the move of placement. A module's layer on a node becomes its own value on that node (`module...`). // A whole-mesh layer has no module value for the whole mesh (ADR 0348 §3): its value becomes mesh config for the // predefined key it falls back to where that module alone falls back to the key (claude-code's managed settings // become `agent-managed-settings`), and otherwise the module's own value on each node it is assigned to, laid under // that node's layer as composition lays it. A key a layer still says by its file's name (nodered's `timeZone`) is // set under its config key (`time-zone`, issue 751). // // **Every node composes byte for byte as before.** The move sets the values, reads every module's resolved values // back through mesh-config and compares them with what the layers lay today; only when they agree does it empty the // layers it moved (each kept whole in its history first) and turn on the switch (inventory.FinishConfigMove), after // which the controller reads mesh-config where it read the layers. A module still in the settings form keeps its // layers and is said so. Anything that cannot move as it composes refuses the whole move, by name, before anything // is set. Asked again, it sets nothing that is set already. // configLevel is where mesh-config keeps a value: mesh config, node config, or a module's own value on a node. type configLevel struct { Level string `json:"level"` Node string `json:"node,omitempty"` Module string `json:"module,omitempty"` } func (l configLevel) valueKey(key string) string { switch l.Level { case "node": return "node." + l.Node + "." + key case "module": return "module." + l.Node + "." + l.Module + "." + key } return "mesh." + key } func (l configLevel) String() string { switch l.Level { case "node": return "node config on " + l.Node case "module": return l.Module + "'s own value on " + l.Node } return "mesh config" } // configSet is one value the move sets in mesh-config, and the layers it comes from. type configSet struct { configLevel Key string `json:"key"` Value any `json:"value"` Trusted bool `json:"trusted,omitempty"` // From is the layers it comes from, as ` on : `. From []string `json:"from"` // Already is true when mesh-config holds this value at this level already: nothing is set. Already bool `json:"already,omitempty"` } // configMoveAnswer is what the move of config says it did, or would do. type configMoveAnswer struct { // DryRun is true when nothing was set: asked without --confirm. DryRun bool `json:"dry-run"` Sets []configSet `json:"sets"` // Moved is each layer whose values the move sets, emptied once they read back. Moved []string `json:"moved"` // Stays is each layer of a module still in the settings form: kept, and composed from as before. Stays []string `json:"stays,omitempty"` // Unread is each layer that composes nothing (its module is not assigned where it says): kept in the history, // emptied, and set nowhere. Unread []string `json:"unread,omitempty"` // Problems is why the move refuses, each by name; nothing is set while there is one. Problems []string `json:"problems,omitempty"` // Switched is true once the controller reads mesh-config where it read the layers. Switched bool `json:"switched"` } // configMovePlan is the move worked out from the layers, before anything is asked of mesh-config. type configMovePlan struct { answer configMoveAnswer // moved is the layers the move empties once the values read back. moved []inventory.LayerToMove // expected is, for each module the move reaches, keyed `.`, every value composition lays over // its files today: what mesh-config must resolve for it from mesh config, node config or its own value. expected map[string]map[string]any // modules is each module in the config form, by the nodes it is assigned to: whose resolved values are read back. modules map[string][]string } func whereOf(node string) string { if node == "" { return "the whole mesh" } return node } // planConfigMove works out the move from every layer that holds a value, the catalogue, and where each module is // assigned. func planConfigMove(layers []inventory.LayerToMove, shelf map[string]catalogue.Manifest, on map[string][]string) configMovePlan { p := configMovePlan{answer: configMoveAnswer{Sets: []configSet{}, Moved: []string{}}, expected: map[string]map[string]any{}, modules: map[string][]string{}} problem := func(format string, a ...any) { p.answer.Problems = append(p.answer.Problems, fmt.Sprintf(format, a...)) } for name, m := range shelf { if m.DeclaresConfig() { p.modules[name] = on[name] } } byModule := map[string]*layersOfModule{} var names []string for _, l := range layers { said := fmt.Sprintf("%s on %s", l.Module, whereOf(l.Node)) m, known := shelf[l.Module] if !known || !m.DeclaresConfig() { p.answer.Stays = append(p.answer.Stays, said+" (its module is in the settings form)") continue } if _, placement := catalogue.SplitPlacement(l.Values); len(placement) > 0 { problem("%s still holds placement (%s): run move-placement first", said, strings.Join(movedKeys(placement), ", ")) continue } assigned := on[l.Module] if (l.Node == "" && len(assigned) == 0) || (l.Node != "" && !containsString(assigned, l.Node)) { p.answer.Unread = append(p.answer.Unread, said+" (the module is not assigned there, so it composes nothing)") p.moved = append(p.moved, l) continue } // The layer's keys as config keys: a key the file names otherwise is set under its config key. as := catalogue.TakenAs(m) values := map[string]any{} from := map[string]string{} for _, k := range movedKeys(l.Values) { key := k if _, declared := m.Settings[k]; !declared { if renamed, ok := as[k]; ok { key = renamed } else { problem("%s says %q, which %s does not declare: declare it, or clear it from the layer", said, k, l.Module) continue } } if _, both := values[key]; both { // The layer says the key and the file's name for it: the config key wins, as composition has it (issue 751). if key != k { continue } } values[key] = l.Values[k] from[key] = said + ": " + k } om := byModule[l.Module] if om == nil { om = &layersOfModule{nodes: map[string]map[string]any{}, saidAt: map[string]map[string]string{}} byModule[l.Module] = om names = append(names, l.Module) } om.saidAt[l.Node] = from if l.Node == "" { om.mesh = values } else { om.nodes[l.Node] = values } p.moved = append(p.moved, l) p.answer.Moved = append(p.answer.Moved, said) } sort.Strings(names) // Which predefined key each module falls back to, so a whole-mesh value goes to mesh config only where no other // module's value moves with it. fallingBack := map[string][]string{} for name, m := range shelf { for _, d := range m.Settings { if d.Predefined != "" && !containsString(fallingBack[d.Predefined], name) { fallingBack[d.Predefined] = append(fallingBack[d.Predefined], name) } } } for _, name := range names { m, om := shelf[name], byModule[name] asked := catalogue.AskedKeys(m) keys := map[string]bool{} for k := range om.mesh { keys[k] = true } for _, values := range om.nodes { for k := range values { keys[k] = true } } for _, key := range movedKeys(keys) { d := m.Settings[key] meshValue, meshSays := om.mesh[key] alone := d.Predefined != "" && len(fallingBack[d.Predefined]) == 1 if meshSays && alone { p.set(configLevel{Level: "mesh"}, d.Predefined, meshValue, d, om.said("", key)) } for _, node := range on[name] { nodeValue, nodeSays := om.nodes[node][key] value := meshValue switch { case nodeSays && meshSays: _, under := meshValue.(map[string]any) _, over := nodeValue.(map[string]any) if under && over && asked[key] { problem("%s on %s lays an object for %q over the whole mesh's, and a placeholder takes it whole "+ "where its file merges the two: say the value on %s whole, then move", name, node, key, node) continue } value = catalogue.LaidOver(meshValue, nodeValue) case nodeSays: value = nodeValue case !meshSays: continue } p.expect(node, name, key, value) if nodeSays || !alone { saidBy := om.said(node, key) if meshSays { saidBy = append(om.said("", key), saidBy...) } p.set(configLevel{Level: "module", Node: node, Module: name}, key, value, d, saidBy) } } } } sort.Strings(p.answer.Problems) return p } // layersOfModule is one module's layers as the move reads them: its whole-mesh layer's config values, each node's, // and where each key was said, by node ("" for the whole mesh). type layersOfModule struct { mesh map[string]any nodes map[string]map[string]any saidAt map[string]map[string]string } func (om *layersOfModule) said(node, key string) []string { if s, ok := om.saidAt[node][key]; ok { return []string{s} } return nil } // set is one value the move sets, judged against its declaration as mesh-config will judge it. func (p *configMovePlan) set(level configLevel, key string, value any, d catalogue.SettingDeclaration, from []string) { p.answer.Sets = append(p.answer.Sets, configSet{configLevel: level, Key: key, Value: value, Trusted: d.Trusted, From: from}) for _, wrong := range catalogue.ValueProblems(d, value) { p.answer.Problems = append(p.answer.Problems, fmt.Sprintf("%s at %s: %s", key, level, wrong)) } } func (p *configMovePlan) expect(node, module, key string, value any) { k := node + "." + module if p.expected[k] == nil { p.expected[k] = map[string]any{} } p.expected[k][key] = value } func movedKeys[V any](m map[string]V) []string { out := make([]string, 0, len(m)) for k := range m { out = append(out, k) } sort.Strings(out) return out } // configService is mesh-config as the move asks it: what it holds at each level, what it resolved, and a value set // as the controller carries it from its terminal. type configService interface { // Values is every value set, by its key in mesh-config_values (`mesh.`, `node..`, // `module...`). Values(ctx context.Context) (map[string]json.RawMessage, error) // Resolved is every resolved value. Resolved(ctx context.Context) ([]resolvedEntry, error) // Set sets one value at its level, by whom and why. Set(ctx context.Context, level configLevel, key string, value any, by, why string) error } // runConfigMove is the move against a store and mesh-config: a dry run says what it would set. func runConfigMove(ctx context.Context, inv *inventory.Inventory, config configService, apply bool, by, why string) (configMoveAnswer, error) { layers, holding, err := inv.LayersToMove(ctx) if err != nil { return configMoveAnswer{}, err } entries, err := inv.Catalogued(ctx) if err != nil { return configMoveAnswer{}, err } shelf := map[string]catalogue.Manifest{} on := map[string][]string{} for _, e := range entries { shelf[e.Manifest.Module] = e.Manifest on[e.Manifest.Module] = e.On } p := planConfigMove(layers, shelf, on) if len(holding) > 0 && len(p.answer.Problems) == 0 { p.answer.Problems = append(p.answer.Problems, "a layer still holds placement ("+strings.Join(holding, "; ")+ "): run move-placement first") } moved, err := inv.ConfigMoved(ctx) if err != nil { return configMoveAnswer{}, err } p.answer.Switched = moved.Moved if config == nil { return p.answer, errors.New("mesh-config cannot be asked, so nothing was moved") } held, err := config.Values(ctx) if err != nil { return p.answer, fmt.Errorf("what mesh-config holds cannot be read, so nothing was moved: %w", err) } for i := range p.answer.Sets { s := &p.answer.Sets[i] if was, ok := held[s.valueKey(s.Key)]; ok && sameJSON(was, s.Value) { s.Already = true } } p.answer.DryRun = !apply if !apply || len(p.answer.Problems) > 0 { if apply { return p.answer, fmt.Errorf("the move of config refuses, and nothing was set: %s", strings.Join(p.answer.Problems, "; ")) } return p.answer, nil } if len(p.moved) == 0 && moved.Moved { return p.answer, nil } for _, s := range p.answer.Sets { if s.Already { continue } said := "moved from the settings layer of " + strings.Join(s.From, " and ") + " (issue 664): " + why if err := config.Set(ctx, s.configLevel, s.Key, s.Value, by, said); err != nil { return p.answer, fmt.Errorf("%s at %s was not set, and no layer was emptied; ask the move again: %w", s.Key, s.configLevel, err) } } // Read back through mesh-config before anything points at it (design 57): every module the move reaches, and // every other in the config form, must resolve exactly what its layers lay today. if differs, err := readsBack(ctx, config, p); err != nil { return p.answer, err } else if len(differs) > 0 { return p.answer, fmt.Errorf("mesh-config does not resolve what the layers lay today, so no layer was emptied and "+ "the controller still composes from the layers: %s", strings.Join(differs, "; ")) } if err := inv.FinishConfigMove(ctx, p.moved, by, why); err != nil { return p.answer, err } p.answer.Switched = true return p.answer, nil } // readsBack compares what mesh-config resolved with what the layers lay, for every module in the config form on // every node it is assigned to, and says each difference. A value the layers do not lay and mesh-config resolves from // a level would be laid once the switch is on: a difference too. func readsBack(ctx context.Context, config configService, p configMovePlan) ([]string, error) { var resolved []resolvedEntry var err error // mesh-config writes the resolved values with each set; a pass that missed one is put right by its next // resolution, so a difference is read again for a little while before it is said. for wait := 0; ; wait++ { if resolved, err = config.Resolved(ctx); err != nil { return nil, fmt.Errorf("mesh-config's resolved values cannot be read back, so no layer was emptied: %w", err) } differs, err := differences(resolved, p) if err != nil || len(differs) == 0 || wait >= readBackTries { return differs, err } select { case <-ctx.Done(): return nil, ctx.Err() case <-time.After(readBackEvery): } } } // How often, and how many times, the move reads mesh-config's resolved values back before it says they differ. var ( readBackEvery = 2 * time.Second readBackTries = 15 ) func differences(resolved []resolvedEntry, p configMovePlan) ([]string, error) { var differs []string for _, module := range movedKeys(p.modules) { for _, node := range p.modules[module] { laid, err := configLaid(resolved, node, module) if err != nil { return nil, err } want := p.expected[node+"."+module] if want == nil { want = map[string]any{} } for _, k := range movedKeys(mergedKeys(laid, want)) { got, has := laid[k] w, wants := want[k] switch { case !wants: differs = append(differs, fmt.Sprintf("%s on %s resolves %s from a level no layer says; clear it "+ "there, or set it in a layer first", module, node, k)) case !has: differs = append(differs, fmt.Sprintf("%s on %s does not resolve %s from the value moved", module, node, k)) case !reflect.DeepEqual(normalJSON(got), normalJSON(w)): differs = append(differs, fmt.Sprintf("%s on %s resolves %s as %s, and its layers lay %s", module, node, k, compactJSON(got), compactJSON(w))) } } } } return differs, nil } func mergedKeys(a, b map[string]any) map[string]bool { out := map[string]bool{} for k := range a { out[k] = true } for k := range b { out[k] = true } return out } func normalJSON(v any) any { raw, _ := json.Marshal(v) var out any _ = json.Unmarshal(raw, &out) return out } func compactJSON(v any) string { raw, _ := json.Marshal(v) return string(raw) } func sameJSON(raw json.RawMessage, v any) bool { var held any if err := json.Unmarshal(raw, &held); err != nil { return false } return reflect.DeepEqual(held, normalJSON(v)) } // busConfig is mesh-config on this controller's bus: its buckets read directly, a value set through its verb in // the controller's own name, so mesh-config takes a trusted value as given at the controller's terminal (ADR 0348 §8). type busConfig struct{} func (busConfig) Values(ctx context.Context) (map[string]json.RawMessage, error) { out := map[string]json.RawMessage{} err := onTheBus(func(conn *nats.Conn) error { js, err := jetstream.New(conn) if err != nil { return err } ctx, cancel := context.WithTimeout(ctx, 20*time.Second) defer cancel() kv, err := js.KeyValue(ctx, broker.BucketName("mesh-config", "values")) if err != nil { return err } w, err := kv.WatchAll(ctx, jetstream.IgnoreDeletes()) if err != nil { return err } defer func() { _ = w.Stop() }() for e := range w.Updates() { if e == nil { return nil } var v struct { Value json.RawMessage `json:"value"` } if err := json.Unmarshal(e.Value(), &v); err != nil { return fmt.Errorf("mesh-config holds %q, which is not a value: %w", e.Key(), err) } out[e.Key()] = v.Value } return errors.New("mesh-config's values stopped answering") }) return out, err } func (busConfig) Resolved(ctx context.Context) ([]resolvedEntry, error) { return readResolved(ctx) } func (busConfig) Set(ctx context.Context, level configLevel, key string, value any, by, why string) error { args := map[string]any{"action": "set", "key": key, "value": value, "why": why, "by": by} if level.Node != "" { args["node"] = level.Node } if level.Module != "" { args["module"] = level.Module } return onTheBus(func(conn *nats.Conn) error { answer, err := link.AskMeshSeatToolAs(ctx, conn, broker.ControllerName, "mesh-config", "config", args, 30*time.Second) if err != nil { return err } if answer.Error != "" { return errors.New(answer.Error) } return nil }) } // moveConfigCommand is `move-config`: the move of config (issue 664). func moveConfigCommand(ctx context.Context, args []string) error { set := flag.NewFlagSet("move-config", flag.ContinueOnError) asJSON := set.Bool("json", false, "answer as JSON") confirm := set.Bool("confirm", false, "set the values and switch, rather than only say what would move") f := addHandActFlags(set) positionals, err := parseAround(set, args) if err != nil { return err } if len(positionals) != 0 { return errors.New("move-config [--json] [--confirm --why ]") } if !*confirm && f.given() { return errors.New("move-config takes --why only with --confirm: without it nothing is moved. Nothing was done") } if *confirm { if err := f.require("move-config"); err != nil { return err } } open, err := openStores(ctx) if err != nil { return err } defer open.Close() if *confirm { f.record(ctx, "move-config", nil) } why := "" if *confirm { why = strings.TrimSpace(*f.why) } answer, err := runConfigMove(ctx, open.inventory, busConfig{}, *confirm, "the move of config, run by "+link.Caller(), why) if *asJSON { encoder := json.NewEncoder(os.Stdout) encoder.SetIndent("", " ") if encErr := encoder.Encode(answer); encErr != nil { return encErr } return err } fmt.Print(describeConfigMove(answer)) return err } // describeConfigMove says the answer as a person reads it. func describeConfigMove(a configMoveAnswer) string { var b strings.Builder verb := "set" if a.DryRun { verb = "would set" } for _, s := range a.Sets { state := verb if s.Already { state = "holds already" } fmt.Fprintf(&b, "%s %s = %s at %s (from %s)\n", state, s.Key, compactJSON(s.Value), s.configLevel, strings.Join(s.From, " and ")) } for _, s := range a.Unread { fmt.Fprintf(&b, "emptied, set nowhere: %s\n", s) } for _, s := range a.Stays { fmt.Fprintf(&b, "stays: %s\n", s) } for _, s := range a.Problems { fmt.Fprintf(&b, "refused: %s\n", s) } switch { case len(a.Problems) > 0: b.WriteString("nothing was set: the move refuses until each is put right\n") case a.DryRun: fmt.Fprintf(&b, "%d layer(s), %d value(s); nothing was set — with --confirm --why they are\n", len(a.Moved), len(a.Sets)) case a.Switched: fmt.Fprintf(&b, "%d layer(s) moved into mesh-config, each kept whole in its history first; the controller "+ "reads mesh-config where it read them\n", len(a.Moved)) } return b.String() }