mesh-controller/cmd/mesh-controller/config_move.go

package main

import (
	"context"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"os"
	"reflect"
	"sort"
	"strings"
	"time"

	"github.com/nats-io/nats.go"
	"github.com/nats-io/nats.go/jetstream"

	"github.com/novox/mesh-controller/internal/broker"
	"github.com/novox/mesh-controller/internal/catalogue"
	"github.com/novox/mesh-controller/internal/inventory"
	"github.com/novox/mesh-controller/internal/link"
)

// **The move of config** (novox/hq ADR 0348 §9 and its consequences, design 57 K5, issue 664): every config value a
// settings layer holds is set in mesh-config, one of the moves of data a release runs once after its delivery, after
// the move of placement. A module's layer on a node becomes its own value on that node (`module.<node>.<module>.<key>`).
// A whole-mesh layer has no module value for the whole mesh (ADR 0348 §3): its value becomes mesh config for the
// predefined key it falls back to where that module alone falls back to the key (claude-code's managed settings
// become `agent-managed-settings`), and otherwise the module's own value on each node it is assigned to, laid under
// that node's layer as composition lays it. A key a layer still says by its file's name (nodered's `timeZone`) is
// set under its config key (`time-zone`, issue 751).
//
// **Every node composes byte for byte as before.** The move sets the values, reads every module's resolved values
// back through mesh-config and compares them with what the layers lay today; only when they agree does it empty the
// layers it moved (each kept whole in its history first) and turn on the switch (inventory.FinishConfigMove), after
// which the controller reads mesh-config where it read the layers. A module still in the settings form keeps its
// layers and is said so. Anything that cannot move as it composes refuses the whole move, by name, before anything
// is set. Asked again, it sets nothing that is set already.

// configLevel is where mesh-config keeps a value: mesh config, node config, or a module's own value on a node.
type configLevel struct {
	Level  string `json:"level"`
	Node   string `json:"node,omitempty"`
	Module string `json:"module,omitempty"`
}

func (l configLevel) valueKey(key string) string {
	switch l.Level {
	case "node":
		return "node." + l.Node + "." + key
	case "module":
		return "module." + l.Node + "." + l.Module + "." + key
	}
	return "mesh." + key
}

func (l configLevel) String() string {
	switch l.Level {
	case "node":
		return "node config on " + l.Node
	case "module":
		return l.Module + "'s own value on " + l.Node
	}
	return "mesh config"
}

// configSet is one value the move sets in mesh-config, and the layers it comes from.
type configSet struct {
	configLevel
	Key     string `json:"key"`
	Value   any    `json:"value"`
	Trusted bool   `json:"trusted,omitempty"`
	// From is the layers it comes from, as `<module> on <where>: <key as the layer says it>`.
	From []string `json:"from"`
	// Already is true when mesh-config holds this value at this level already: nothing is set.
	Already bool `json:"already,omitempty"`
}

// configMoveAnswer is what the move of config says it did, or would do.
type configMoveAnswer struct {
	// DryRun is true when nothing was set: asked without --confirm.
	DryRun bool        `json:"dry-run"`
	Sets   []configSet `json:"sets"`
	// Moved is each layer whose values the move sets, emptied once they read back.
	Moved []string `json:"moved"`
	// Stays is each layer of a module still in the settings form: kept, and composed from as before.
	Stays []string `json:"stays,omitempty"`
	// Unread is each layer that composes nothing (its module is not assigned where it says): kept in the history,
	// emptied, and set nowhere.
	Unread []string `json:"unread,omitempty"`
	// Problems is why the move refuses, each by name; nothing is set while there is one.
	Problems []string `json:"problems,omitempty"`
	// Switched is true once the controller reads mesh-config where it read the layers.
	Switched bool `json:"switched"`
}

// configMovePlan is the move worked out from the layers, before anything is asked of mesh-config.
type configMovePlan struct {
	answer configMoveAnswer
	// moved is the layers the move empties once the values read back.
	moved []inventory.LayerToMove
	// expected is, for each module the move reaches, keyed `<node>.<module>`, every value composition lays over
	// its files today: what mesh-config must resolve for it from mesh config, node config or its own value.
	expected map[string]map[string]any
	// modules is each module in the config form, by the nodes it is assigned to: whose resolved values are read back.
	modules map[string][]string
}

func whereOf(node string) string {
	if node == "" {
		return "the whole mesh"
	}
	return node
}

// planConfigMove works out the move from every layer that holds a value, the catalogue, and where each module is
// assigned.
func planConfigMove(layers []inventory.LayerToMove, shelf map[string]catalogue.Manifest,
	on map[string][]string) configMovePlan {
	p := configMovePlan{answer: configMoveAnswer{Sets: []configSet{}, Moved: []string{}},
		expected: map[string]map[string]any{}, modules: map[string][]string{}}
	problem := func(format string, a ...any) {
		p.answer.Problems = append(p.answer.Problems, fmt.Sprintf(format, a...))
	}
	for name, m := range shelf {
		if m.DeclaresConfig() {
			p.modules[name] = on[name]
		}
	}

	byModule := map[string]*layersOfModule{}
	var names []string
	for _, l := range layers {
		said := fmt.Sprintf("%s on %s", l.Module, whereOf(l.Node))
		m, known := shelf[l.Module]
		if !known || !m.DeclaresConfig() {
			p.answer.Stays = append(p.answer.Stays, said+" (its module is in the settings form)")
			continue
		}
		if _, placement := catalogue.SplitPlacement(l.Values); len(placement) > 0 {
			problem("%s still holds placement (%s): run move-placement first", said, strings.Join(movedKeys(placement), ", "))
			continue
		}
		assigned := on[l.Module]
		if (l.Node == "" && len(assigned) == 0) || (l.Node != "" && !containsString(assigned, l.Node)) {
			p.answer.Unread = append(p.answer.Unread, said+" (the module is not assigned there, so it composes nothing)")
			p.moved = append(p.moved, l)
			continue
		}
		// The layer's keys as config keys: a key the file names otherwise is set under its config key.
		as := catalogue.TakenAs(m)
		values := map[string]any{}
		from := map[string]string{}
		for _, k := range movedKeys(l.Values) {
			key := k
			if _, declared := m.Settings[k]; !declared {
				if renamed, ok := as[k]; ok {
					key = renamed
				} else {
					problem("%s says %q, which %s does not declare: declare it, or clear it from the layer", said, k, l.Module)
					continue
				}
			}
			if _, both := values[key]; both {
				// The layer says the key and the file's name for it: the config key wins, as composition has it (issue 751).
				if key != k {
					continue
				}
			}
			values[key] = l.Values[k]
			from[key] = said + ": " + k
		}
		om := byModule[l.Module]
		if om == nil {
			om = &layersOfModule{nodes: map[string]map[string]any{}, saidAt: map[string]map[string]string{}}
			byModule[l.Module] = om
			names = append(names, l.Module)
		}
		om.saidAt[l.Node] = from
		if l.Node == "" {
			om.mesh = values
		} else {
			om.nodes[l.Node] = values
		}
		p.moved = append(p.moved, l)
		p.answer.Moved = append(p.answer.Moved, said)
	}
	sort.Strings(names)

	// Which predefined key each module falls back to, so a whole-mesh value goes to mesh config only where no other
	// module's value moves with it.
	fallingBack := map[string][]string{}
	for name, m := range shelf {
		for _, d := range m.Settings {
			if d.Predefined != "" && !containsString(fallingBack[d.Predefined], name) {
				fallingBack[d.Predefined] = append(fallingBack[d.Predefined], name)
			}
		}
	}

	for _, name := range names {
		m, om := shelf[name], byModule[name]
		asked := catalogue.AskedKeys(m)
		keys := map[string]bool{}
		for k := range om.mesh {
			keys[k] = true
		}
		for _, values := range om.nodes {
			for k := range values {
				keys[k] = true
			}
		}
		for _, key := range movedKeys(keys) {
			d := m.Settings[key]
			meshValue, meshSays := om.mesh[key]
			alone := d.Predefined != "" && len(fallingBack[d.Predefined]) == 1
			if meshSays && alone {
				p.set(configLevel{Level: "mesh"}, d.Predefined, meshValue, d, om.said("", key))
			}
			for _, node := range on[name] {
				nodeValue, nodeSays := om.nodes[node][key]
				value := meshValue
				switch {
				case nodeSays && meshSays:
					_, under := meshValue.(map[string]any)
					_, over := nodeValue.(map[string]any)
					if under && over && asked[key] {
						problem("%s on %s lays an object for %q over the whole mesh's, and a placeholder takes it whole "+
							"where its file merges the two: say the value on %s whole, then move", name, node, key, node)
						continue
					}
					value = catalogue.LaidOver(meshValue, nodeValue)
				case nodeSays:
					value = nodeValue
				case !meshSays:
					continue
				}
				p.expect(node, name, key, value)
				if nodeSays || !alone {
					saidBy := om.said(node, key)
					if meshSays {
						saidBy = append(om.said("", key), saidBy...)
					}
					p.set(configLevel{Level: "module", Node: node, Module: name}, key, value, d, saidBy)
				}
			}
		}
	}
	sort.Strings(p.answer.Problems)
	return p
}

// layersOfModule is one module's layers as the move reads them: its whole-mesh layer's config values, each node's,
// and where each key was said, by node ("" for the whole mesh).
type layersOfModule struct {
	mesh   map[string]any
	nodes  map[string]map[string]any
	saidAt map[string]map[string]string
}

func (om *layersOfModule) said(node, key string) []string {
	if s, ok := om.saidAt[node][key]; ok {
		return []string{s}
	}
	return nil
}

// set is one value the move sets, judged against its declaration as mesh-config will judge it.
func (p *configMovePlan) set(level configLevel, key string, value any, d catalogue.SettingDeclaration, from []string) {
	p.answer.Sets = append(p.answer.Sets, configSet{configLevel: level, Key: key, Value: value, Trusted: d.Trusted,
		From: from})
	for _, wrong := range catalogue.ValueProblems(d, value) {
		p.answer.Problems = append(p.answer.Problems, fmt.Sprintf("%s at %s: %s", key, level, wrong))
	}
}

func (p *configMovePlan) expect(node, module, key string, value any) {
	k := node + "." + module
	if p.expected[k] == nil {
		p.expected[k] = map[string]any{}
	}
	p.expected[k][key] = value
}

func movedKeys[V any](m map[string]V) []string {
	out := make([]string, 0, len(m))
	for k := range m {
		out = append(out, k)
	}
	sort.Strings(out)
	return out
}

// configService is mesh-config as the move asks it: what it holds at each level, what it resolved, and a value set
// as the controller carries it from its terminal.
type configService interface {
	// Values is every value set, by its key in mesh-config_values (`mesh.<key>`, `node.<node>.<key>`,
	// `module.<node>.<module>.<key>`).
	Values(ctx context.Context) (map[string]json.RawMessage, error)
	// Resolved is every resolved value.
	Resolved(ctx context.Context) ([]resolvedEntry, error)
	// Set sets one value at its level, by whom and why.
	Set(ctx context.Context, level configLevel, key string, value any, by, why string) error
}

// runConfigMove is the move against a store and mesh-config: a dry run says what it would set.
func runConfigMove(ctx context.Context, inv *inventory.Inventory, config configService, apply bool, by,
	why string) (configMoveAnswer, error) {
	layers, holding, err := inv.LayersToMove(ctx)
	if err != nil {
		return configMoveAnswer{}, err
	}
	entries, err := inv.Catalogued(ctx)
	if err != nil {
		return configMoveAnswer{}, err
	}
	shelf := map[string]catalogue.Manifest{}
	on := map[string][]string{}
	for _, e := range entries {
		shelf[e.Manifest.Module] = e.Manifest
		on[e.Manifest.Module] = e.On
	}
	p := planConfigMove(layers, shelf, on)
	if len(holding) > 0 && len(p.answer.Problems) == 0 {
		p.answer.Problems = append(p.answer.Problems, "a layer still holds placement ("+strings.Join(holding, "; ")+
			"): run move-placement first")
	}
	moved, err := inv.ConfigMoved(ctx)
	if err != nil {
		return configMoveAnswer{}, err
	}
	p.answer.Switched = moved.Moved
	if config == nil {
		return p.answer, errors.New("mesh-config cannot be asked, so nothing was moved")
	}
	held, err := config.Values(ctx)
	if err != nil {
		return p.answer, fmt.Errorf("what mesh-config holds cannot be read, so nothing was moved: %w", err)
	}
	for i := range p.answer.Sets {
		s := &p.answer.Sets[i]
		if was, ok := held[s.valueKey(s.Key)]; ok && sameJSON(was, s.Value) {
			s.Already = true
		}
	}
	p.answer.DryRun = !apply
	if !apply || len(p.answer.Problems) > 0 {
		if apply {
			return p.answer, fmt.Errorf("the move of config refuses, and nothing was set: %s",
				strings.Join(p.answer.Problems, "; "))
		}
		return p.answer, nil
	}
	if len(p.moved) == 0 && moved.Moved {
		return p.answer, nil
	}
	for _, s := range p.answer.Sets {
		if s.Already {
			continue
		}
		said := "moved from the settings layer of " + strings.Join(s.From, " and ") + " (issue 664): " + why
		if err := config.Set(ctx, s.configLevel, s.Key, s.Value, by, said); err != nil {
			return p.answer, fmt.Errorf("%s at %s was not set, and no layer was emptied; ask the move again: %w",
				s.Key, s.configLevel, err)
		}
	}
	// Read back through mesh-config before anything points at it (design 57): every module the move reaches, and
	// every other in the config form, must resolve exactly what its layers lay today.
	if differs, err := readsBack(ctx, config, p); err != nil {
		return p.answer, err
	} else if len(differs) > 0 {
		return p.answer, fmt.Errorf("mesh-config does not resolve what the layers lay today, so no layer was emptied and "+
			"the controller still composes from the layers: %s", strings.Join(differs, "; "))
	}
	if err := inv.FinishConfigMove(ctx, p.moved, by, why); err != nil {
		return p.answer, err
	}
	p.answer.Switched = true
	return p.answer, nil
}

// readsBack compares what mesh-config resolved with what the layers lay, for every module in the config form on
// every node it is assigned to, and says each difference. A value the layers do not lay and mesh-config resolves from
// a level would be laid once the switch is on: a difference too.
func readsBack(ctx context.Context, config configService, p configMovePlan) ([]string, error) {
	var resolved []resolvedEntry
	var err error
	// mesh-config writes the resolved values with each set; a pass that missed one is put right by its next
	// resolution, so a difference is read again for a little while before it is said.
	for wait := 0; ; wait++ {
		if resolved, err = config.Resolved(ctx); err != nil {
			return nil, fmt.Errorf("mesh-config's resolved values cannot be read back, so no layer was emptied: %w", err)
		}
		differs, err := differences(resolved, p)
		if err != nil || len(differs) == 0 || wait >= readBackTries {
			return differs, err
		}
		select {
		case <-ctx.Done():
			return nil, ctx.Err()
		case <-time.After(readBackEvery):
		}
	}
}

// How often, and how many times, the move reads mesh-config's resolved values back before it says they differ.
var (
	readBackEvery = 2 * time.Second
	readBackTries = 15
)

func differences(resolved []resolvedEntry, p configMovePlan) ([]string, error) {
	var differs []string
	for _, module := range movedKeys(p.modules) {
		for _, node := range p.modules[module] {
			laid, err := configLaid(resolved, node, module)
			if err != nil {
				return nil, err
			}
			want := p.expected[node+"."+module]
			if want == nil {
				want = map[string]any{}
			}
			for _, k := range movedKeys(mergedKeys(laid, want)) {
				got, has := laid[k]
				w, wants := want[k]
				switch {
				case !wants:
					differs = append(differs, fmt.Sprintf("%s on %s resolves %s from a level no layer says; clear it "+
						"there, or set it in a layer first", module, node, k))
				case !has:
					differs = append(differs, fmt.Sprintf("%s on %s does not resolve %s from the value moved", module, node, k))
				case !reflect.DeepEqual(normalJSON(got), normalJSON(w)):
					differs = append(differs, fmt.Sprintf("%s on %s resolves %s as %s, and its layers lay %s", module,
						node, k, compactJSON(got), compactJSON(w)))
				}
			}
		}
	}
	return differs, nil
}

func mergedKeys(a, b map[string]any) map[string]bool {
	out := map[string]bool{}
	for k := range a {
		out[k] = true
	}
	for k := range b {
		out[k] = true
	}
	return out
}

func normalJSON(v any) any {
	raw, _ := json.Marshal(v)
	var out any
	_ = json.Unmarshal(raw, &out)
	return out
}

func compactJSON(v any) string {
	raw, _ := json.Marshal(v)
	return string(raw)
}

func sameJSON(raw json.RawMessage, v any) bool {
	var held any
	if err := json.Unmarshal(raw, &held); err != nil {
		return false
	}
	return reflect.DeepEqual(held, normalJSON(v))
}

// busConfig is mesh-config on this controller's bus: its buckets read directly, a value set through its verb in
// the controller's own name, so mesh-config takes a trusted value as given at the controller's terminal (ADR 0348 §8).
type busConfig struct{}

func (busConfig) Values(ctx context.Context) (map[string]json.RawMessage, error) {
	out := map[string]json.RawMessage{}
	err := onTheBus(func(conn *nats.Conn) error {
		js, err := jetstream.New(conn)
		if err != nil {
			return err
		}
		ctx, cancel := context.WithTimeout(ctx, 20*time.Second)
		defer cancel()
		kv, err := js.KeyValue(ctx, broker.BucketName("mesh-config", "values"))
		if err != nil {
			return err
		}
		w, err := kv.WatchAll(ctx, jetstream.IgnoreDeletes())
		if err != nil {
			return err
		}
		defer func() { _ = w.Stop() }()
		for e := range w.Updates() {
			if e == nil {
				return nil
			}
			var v struct {
				Value json.RawMessage `json:"value"`
			}
			if err := json.Unmarshal(e.Value(), &v); err != nil {
				return fmt.Errorf("mesh-config holds %q, which is not a value: %w", e.Key(), err)
			}
			out[e.Key()] = v.Value
		}
		return errors.New("mesh-config's values stopped answering")
	})
	return out, err
}

func (busConfig) Resolved(ctx context.Context) ([]resolvedEntry, error) { return readResolved(ctx) }

func (busConfig) Set(ctx context.Context, level configLevel, key string, value any, by, why string) error {
	args := map[string]any{"action": "set", "key": key, "value": value, "why": why, "by": by}
	if level.Node != "" {
		args["node"] = level.Node
	}
	if level.Module != "" {
		args["module"] = level.Module
	}
	return onTheBus(func(conn *nats.Conn) error {
		answer, err := link.AskMeshSeatToolAs(ctx, conn, broker.ControllerName, "mesh-config", "config", args, 30*time.Second)
		if err != nil {
			return err
		}
		if answer.Error != "" {
			return errors.New(answer.Error)
		}
		return nil
	})
}

// moveConfigCommand is `move-config`: the move of config (issue 664).
func moveConfigCommand(ctx context.Context, args []string) error {
	set := flag.NewFlagSet("move-config", flag.ContinueOnError)
	asJSON := set.Bool("json", false, "answer as JSON")
	confirm := set.Bool("confirm", false, "set the values and switch, rather than only say what would move")
	f := addHandActFlags(set)
	positionals, err := parseAround(set, args)
	if err != nil {
		return err
	}
	if len(positionals) != 0 {
		return errors.New("move-config [--json] [--confirm --why <text>]")
	}
	if !*confirm && f.given() {
		return errors.New("move-config takes --why only with --confirm: without it nothing is moved. Nothing was done")
	}
	if *confirm {
		if err := f.require("move-config"); err != nil {
			return err
		}
	}
	open, err := openStores(ctx)
	if err != nil {
		return err
	}
	defer open.Close()
	if *confirm {
		f.record(ctx, "move-config", nil)
	}
	why := ""
	if *confirm {
		why = strings.TrimSpace(*f.why)
	}
	answer, err := runConfigMove(ctx, open.inventory, busConfig{}, *confirm, "the move of config, run by "+link.Caller(), why)
	if *asJSON {
		encoder := json.NewEncoder(os.Stdout)
		encoder.SetIndent("", " ")
		if encErr := encoder.Encode(answer); encErr != nil {
			return encErr
		}
		return err
	}
	fmt.Print(describeConfigMove(answer))
	return err
}

// describeConfigMove says the answer as a person reads it.
func describeConfigMove(a configMoveAnswer) string {
	var b strings.Builder
	verb := "set"
	if a.DryRun {
		verb = "would set"
	}
	for _, s := range a.Sets {
		state := verb
		if s.Already {
			state = "holds already"
		}
		fmt.Fprintf(&b, "%s %s = %s at %s (from %s)\n", state, s.Key, compactJSON(s.Value), s.configLevel,
			strings.Join(s.From, " and "))
	}
	for _, s := range a.Unread {
		fmt.Fprintf(&b, "emptied, set nowhere: %s\n", s)
	}
	for _, s := range a.Stays {
		fmt.Fprintf(&b, "stays: %s\n", s)
	}
	for _, s := range a.Problems {
		fmt.Fprintf(&b, "refused: %s\n", s)
	}
	switch {
	case len(a.Problems) > 0:
		b.WriteString("nothing was set: the move refuses until each is put right\n")
	case a.DryRun:
		fmt.Fprintf(&b, "%d layer(s), %d value(s); nothing was set — with --confirm --why <text> they are\n",
			len(a.Moved), len(a.Sets))
	case a.Switched:
		fmt.Fprintf(&b, "%d layer(s) moved into mesh-config, each kept whole in its history first; the controller "+
			"reads mesh-config where it read them\n", len(a.Moved))
	}
	return b.String()
}