diff --git a/cmd/mesh-controller/epoch_send_test.go b/cmd/mesh-controller/epoch_send_test.go index 2c16c9a..ae6adbd 100644 --- a/cmd/mesh-controller/epoch_send_test.go +++ b/cmd/mesh-controller/epoch_send_test.go @@ -45,7 +45,7 @@ func TestAMachineIsSentTheEpochOnlyOnceItSaysItReadsOne(t *testing.T) { t.Fatal(err) } d := &bodiesDelivery{recordedDelivery: recordedDelivery{inv: inv}, bodies: map[string][]byte{}} - if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, composeForPush(open, gens), d, ""); err != nil { + if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, composeForPush(open, gens), d, "", nil); err != nil { t.Fatal(err) } carried := func(node string) (epoch float64, has bool) { @@ -92,7 +92,7 @@ func TestNothingIsComposedOrSentWithoutTheLease(t *testing.T) { t.Fatal(err) } d := &bodiesDelivery{recordedDelivery: recordedDelivery{inv: open.inventory}, bodies: map[string][]byte{}} - refused, err := sendRound(ctx, open, []string{"anchor"}, composeForPush(open, gens), d, "") + refused, err := sendRound(ctx, open, []string{"anchor"}, composeForPush(open, gens), d, "", nil) if err != nil { t.Fatal(err) } diff --git a/cmd/mesh-controller/held_back.go b/cmd/mesh-controller/held_back.go index fc22083..0d90cc1 100644 --- a/cmd/mesh-controller/held_back.go +++ b/cmd/mesh-controller/held_back.go @@ -179,7 +179,7 @@ func sayHeld(w io.Writer, node string, why []string) { // machines that could not be composed, as refusals. func flushBehind(ctx context.Context, open *stores, nodes []inventory.Node, handled map[string]bool, compose func(held context.Context, node string) (sendable, error), d delivery, holder string, - w io.Writer) ([]string, error) { + keep keeping, w io.Writer) ([]string, error) { inv := open.inventory var refusals []string // Bounded by the node count: a node is marked handled the round it is considered and is never @@ -237,7 +237,7 @@ func flushBehind(ctx context.Context, open *stores, nodes []inventory.Node, hand // a refusal and reported at the end, and the others are still sent (novox/hq ADR 0066). // Held for this round only, and after the last round's were given back, so two pushes // cascading into each other's machines never each wait on the other. - refused, err := sendRound(ctx, open, sending, compose, d, holder) + refused, err := sendRound(ctx, open, sending, compose, d, holder, keep) refusals = append(refusals, refused...) if err != nil { return refusals, err diff --git a/cmd/mesh-controller/held_back_test.go b/cmd/mesh-controller/held_back_test.go index 0507b64..bcb0fe9 100644 --- a/cmd/mesh-controller/held_back_test.go +++ b/cmd/mesh-controller/held_back_test.go @@ -203,7 +203,7 @@ func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) { } compose := composeForPush(open, gens) d := &recordedDelivery{inv: inv} - if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, ""); err != nil { + if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, "", nil); err != nil { t.Fatal(err) } if builds, known, err := inv.SentBuilds(ctx, "laptop"); err != nil || !known || builds["resolver"] != "c1c1c1c1c1" { @@ -221,13 +221,13 @@ func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) { // The change merges; the policy is record. `push anchor` sends the anchor... aResolver(t, open, "c2c2c2c2c2", asked.Add(time.Minute)) d.declared = nil - if _, err := sendRound(ctx, open, []string{"anchor"}, compose, d, ""); err != nil { + if _, err := sendRound(ctx, open, []string{"anchor"}, compose, d, "", nil); err != nil { t.Fatal(err) } // ...and its cascade leaves the laptop, saying so. var said bytes.Buffer d.declared = nil - refused, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true}, compose, d, "", &said) + refused, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true}, compose, d, "", nil, &said) if err != nil || len(refused) != 0 { t.Fatalf("the cascade failed: %v %v", refused, err) } @@ -248,13 +248,13 @@ func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) { // is held, and that what else it is owed waits with it. aThirdMachine(t, open) d.declared = nil - if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, ""); err != nil { + if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, "", nil); err != nil { t.Fatal(err) } d.declared = nil said.Reset() if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true}, - compose, d, "", &said); err != nil { + compose, d, "", nil, &said); err != nil { t.Fatal(err) } if len(d.declared) != 0 || digestOfLaptop() != before { @@ -271,7 +271,7 @@ func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) { } said.Reset() if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true}, - compose, d, "", &said); err != nil { + compose, d, "", nil, &said); err != nil { t.Fatal(err) } if len(d.declared) != 0 || !strings.Contains(said.String(), "has passed no gate yet") { @@ -284,7 +284,7 @@ func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) { } said.Reset() if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true}, - compose, d, "", &said); err != nil { + compose, d, "", nil, &said); err != nil { t.Fatal(err) } if !reflect.DeepEqual(d.declared, []string{"laptop"}) || digestOfLaptop() == before { @@ -311,18 +311,18 @@ func TestANamedPushStillSendsAConsequenceNothingHolds(t *testing.T) { } compose := composeForPush(open, gens) d := &recordedDelivery{inv: inv} - if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, ""); err != nil { + if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, "", nil); err != nil { t.Fatal(err) } // `push spare`, the machine just placed: the others' peers change with it. aThirdMachine(t, open) - if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, ""); err != nil { + if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, "", nil); err != nil { t.Fatal(err) } d.declared = nil var said bytes.Buffer - if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", &said); err != nil { + if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", nil, &said); err != nil { t.Fatal(err) } if !reflect.DeepEqual(d.declared, []string{"anchor", "laptop"}) { @@ -342,7 +342,7 @@ func TestANamedPushStillSendsAConsequenceNothingHolds(t *testing.T) { } d.declared = nil said.Reset() - if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", &said); err != nil { + if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", nil, &said); err != nil { t.Fatal(err) } // The anchor, the hub, may still be settling from the machine placed above; the laptop is the diff --git a/cmd/mesh-controller/hold_test.go b/cmd/mesh-controller/hold_test.go index c66d769..8301b42 100644 --- a/cmd/mesh-controller/hold_test.go +++ b/cmd/mesh-controller/hold_test.go @@ -23,11 +23,11 @@ func TestASendRoundGivesItsHoldBackOnEveryWayOut(t *testing.T) { for name, round := range map[string]func() error{ "a body that cannot be marshalled": func() error { - _, err := sendRound(ctx, open, []string{"anchor"}, unmarshallable, fine, "") + _, err := sendRound(ctx, open, []string{"anchor"}, unmarshallable, fine, "", nil) return err }, "a send that fails": func() error { - _, err := sendRound(ctx, open, []string{"anchor"}, plain, failing, "") + _, err := sendRound(ctx, open, []string{"anchor"}, plain, failing, "", nil) return err }, } { diff --git a/cmd/mesh-controller/licence.go b/cmd/mesh-controller/licence.go index 25d12ef..0b93e9f 100644 --- a/cmd/mesh-controller/licence.go +++ b/cmd/mesh-controller/licence.go @@ -232,7 +232,7 @@ func licenceKey(ctx context.Context, args []string) error { // and printing the value here would put the one copy that matters on a terminal. fmt.Printf("sealed to %d holder(s). The mesh has discarded the key and cannot read it back\n", sealed) - fmt.Printf(" run `push` to deliver it\n") + fmt.Printf(" run `push --behind` to deliver it\n") return nil } @@ -340,7 +340,7 @@ func licenceSetGrant(ctx context.Context, args []string) error { return err } fmt.Printf("%s now holds a refresh token for %s, sealed to that node's key and readable by it "+ - "alone.\n the control plane stored the box without opening it; run `push` to deliver it\n", + "alone.\n the control plane stored the box without opening it; run `push --behind` to deliver it\n", "the manager", name) return nil } @@ -423,7 +423,7 @@ func licenceSubmitRefresh(ctx context.Context, args []string) error { "manager node alone" } fmt.Printf("submitted a refresh for %s: a new access token sealed to %d holder(s), and %s.\n"+ - " run `push` to deliver it\n", name, sealed, rotatedNote) + " run `push --behind` to deliver it\n", name, sealed, rotatedNote) return nil } @@ -456,7 +456,7 @@ func licenceRefresh(ctx context.Context, args []string) error { return err } fmt.Printf("refreshed %s: a new access token sealed to %d holder(s), and the refresh token left "+ - "with its manager.\n run `push` to deliver it\n", name, sealed) + "with its manager.\n run `push --behind` to deliver it\n", name, sealed) return nil } diff --git a/cmd/mesh-controller/modules.go b/cmd/mesh-controller/modules.go index 950bc76..38b8b1b 100644 --- a/cmd/mesh-controller/modules.go +++ b/cmd/mesh-controller/modules.go @@ -8,7 +8,6 @@ import ( "fmt" "net" "os" - "sort" "strings" "github.com/novox/mesh-controller/internal/broker" @@ -383,7 +382,8 @@ func assignCommand(ctx context.Context, verb string, args []string) error { func settingsCommand(ctx context.Context, args []string) error { if len(args) == 0 { - return errors.New("settings set [--node ], or settings clear [--node ]") + return errors.New("settings show [--node ] [--history], settings set " + + "[--node ] [--replace], or settings clear [--node ]") } open, err := openStores(ctx) if err != nil { @@ -394,6 +394,11 @@ func settingsCommand(ctx context.Context, args []string) error { set := flag.NewFlagSet("settings", flag.ContinueOnError) node := set.String("node", "", "one machine, rather than the whole mesh") + // **What a set removes is refused unless meant** (novox/hq ADR 0217). A layer is replaced whole, + // and on 2026-10-05 setting one placement dropped a machine's whole layer for a module without a + // word (novox/hq issue 304). Adding and changing keys needs nothing; removing one needs this. + replace := set.Bool("replace", false, "for set: remove the keys the new layer does not name") + history := set.Bool("history", false, "for show: the layers this one replaced, the latest first") positionals, err := parseAround(set, args[1:]) if err != nil { return err @@ -421,17 +426,76 @@ func settingsCommand(ctx context.Context, args []string) error { if err := json.Unmarshal(raw, &values); err != nil { return fmt.Errorf("%s is not a settings file: %w", positionals[1], err) } + before, _, err := inv.Layer(ctx, *node, positionals[0]) + if err != nil { + return err + } + added, changed, removed := settingsChange(before, values) + if len(removed) > 0 && !*replace { + return fmt.Errorf("%s on %s: this layer would no longer set %s. A layer is replaced whole; "+ + "read it with `settings show %s%s` and include what should stay, or add --replace if the "+ + "removal is meant (novox/hq ADR 0217). Nothing was changed", + positionals[0], where, strings.Join(removed, ", "), positionals[0], nodeFlag(*node)) + } if err := inv.SetSettings(ctx, *node, positionals[0], values); err != nil { return err } - - var keys []string - for k := range values { - keys = append(keys, k) + fmt.Printf("%s on %s:\n", positionals[0], where) + if len(added)+len(changed)+len(removed) == 0 { + fmt.Println(" nothing changed") } - sort.Strings(keys) - fmt.Printf("%s on %s: %s\n", positionals[0], where, strings.Join(keys, ", ")) - fmt.Println(" run `push` to send it") + for _, p := range added { + fmt.Printf(" + %s\n", p) + } + for _, p := range changed { + fmt.Printf(" ~ %s\n", p) + } + for _, p := range removed { + fmt.Printf(" - %s\n", p) + } + if before != nil { + fmt.Printf(" the layer it replaced is kept: settings show %s%s --history\n", positionals[0], nodeFlag(*node)) + } + if *node != "" { + fmt.Printf(" run `plan %s --diff` to see what it changes, `push %s` to send it\n", *node, *node) + } else { + fmt.Println(" run `push --behind` to send it to the machines running it") + } + return nil + + case "show": + if len(positionals) != 1 { + return errors.New("settings show [--node ] [--history]") + } + if *history { + past, err := inv.SettingsHistory(ctx, *node, positionals[0]) + if err != nil { + return err + } + if len(past) == 0 { + fmt.Printf("%s on %s: no layer has been replaced\n", positionals[0], where) + return nil + } + for _, p := range past { + shown, _ := json.MarshalIndent(p.Values, " ", " ") + fmt.Printf("%s on %s, until %s (%s):\n %s\n", positionals[0], where, + p.ReplacedAt.Local().Format("2006-01-02 15:04:05"), p.ReplacedBy, shown) + } + return nil + } + values, has, err := inv.Layer(ctx, *node, positionals[0]) + if err != nil { + return err + } + if !has { + fmt.Printf("%s on %s: no layer — the module's definition says\n", positionals[0], where) + return nil + } + shown, err := json.MarshalIndent(values, "", " ") + if err != nil { + return err + } + fmt.Println(string(shown)) return nil case "clear": @@ -445,10 +509,18 @@ func settingsCommand(ctx context.Context, args []string) error { return nil default: - return fmt.Errorf("settings has no %q; it has set and clear", args[0]) + return fmt.Errorf("settings has no %q; it has show, set and clear", args[0]) } } +// nodeFlag is ` --node ` for a machine's layer, nothing for the whole mesh's. +func nodeFlag(node string) string { + if node == "" { + return "" + } + return " --node " + node +} + // describeOffers says what a module provides, and marks the ones answered from anywhere in the // mesh — because "provides a database" and "provides a shell" are read the same way and mean // entirely different things about where the answer has to be. @@ -683,7 +755,7 @@ func issueWith(ctx context.Context, inv *inventory.Inventory, m catalogue.Manife Emits: m.EmitsAll(), Consumes: m.Consumes, Serves: m.Tools, }); needed { if busAddress == "" { - fmt.Printf(" %s consumes; its consumer is created when the bus is reachable (`push`, then "+ + fmt.Printf(" %s consumes; its consumer is created when the bus is reachable (`push --behind`, then "+ "`rollout mint` again is harmless)\n", m.Module) } else { js, err := broker.Dial(busAddress) diff --git a/cmd/mesh-controller/plan.go b/cmd/mesh-controller/plan.go index be9dc8f..648b682 100644 --- a/cmd/mesh-controller/plan.go +++ b/cmd/mesh-controller/plan.go @@ -1185,12 +1185,15 @@ func planCommand(ctx context.Context, args []string) error { // checking it against the host's own parser, most usefully, which is the only way to know // that what the control plane emits is what the host accepts. asJSON := set.Bool("json", false, "print the declaration this node would be sent") + // What a push would change, against what the machine was last sent (novox/hq ADR 0217): read + // before sending, so a change that removes, moves or recreates something is seen first. + asDiff := set.Bool("diff", false, "what a push would change on this node, against what it was last sent") positionals, err := parseAround(set, args) if err != nil { return err } if len(positionals) != 1 { - return errors.New("plan [--files] [--json]") + return errors.New("plan [--files] [--json] [--diff]") } args = positionals open, err := openStores(ctx) @@ -1207,6 +1210,17 @@ func planCommand(ctx context.Context, args []string) error { fmt.Printf("%s is assigned nothing\n", args[0]) return nil } + if *asDiff { + declared, err := declarationFor(ctx, open, args[0], plan, settings) + if err != nil { + return err + } + body, err := declared.Body() + if err != nil { + return err + } + return writePlanDiff(ctx, open.inventory, args[0], body) + } if *asJSON { declared, err := declarationFor(ctx, open, args[0], plan, settings) if err != nil { diff --git a/cmd/mesh-controller/push.go b/cmd/mesh-controller/push.go index 0996ace..5b7a934 100644 --- a/cmd/mesh-controller/push.go +++ b/cmd/mesh-controller/push.go @@ -359,14 +359,31 @@ func pushCommand(ctx context.Context, args []string) error { // A push by hand is a repair, and says why (novox/hq to-be 45 §7): required through the seat, // recorded when given at a shell — see handacts.go for why a shell is not refused. why := addHandActFlags(set) + // Every machine at once is said, not defaulted to (novox/hq ADR 0217): a bare `push` sent the + // whole mesh on 2026-10-05 when its usage was wanted. + all := set.Bool("all", false, "every machine") + // A running module's data moving is sent only when said, per module (novox/hq ADR 0217). + move := set.String("move", "", "modules whose data may move with this push, comma-separated") positionals, err := parseAround(set, args) if err != nil { return err } args = positionals if len(args) > 1 { - return errors.New("push [] [--behind] — one node, or all of them") + return errors.New("push | push --behind | push --all — one machine, the ones behind, or all of them") } + if len(args) == 0 && !*behind && !*all { + return errors.New("push names a machine, or says --behind (the ones not running what they " + + "should) or --all (every machine) — a push to the whole mesh is not the default (novox/hq ADR 0217)") + } + if *all && (*behind || len(args) == 1) { + return errors.New("push --all is every machine; it takes no machine and no --behind") + } + movable := map[string]bool{} + for _, m := range splitModules(*move) { + movable[m] = true + } + var heldBack []string if len(args) == 1 && *behind { // Naming a machine and asking for the ones that need it are two different requests, and // guessing which was meant would sometimes push to a machine somebody did not name. @@ -377,6 +394,12 @@ func pushCommand(ctx context.Context, args []string) error { if *behind { recorded = append(recorded, "--behind") } + if *all { + recorded = append(recorded, "--all") + } + if *move != "" { + recorded = append(recorded, "--move", *move) + } why.record(ctx, "push", recorded) open, err := openStores(ctx) if err != nil { @@ -587,22 +610,28 @@ func pushCommand(ctx context.Context, args []string) error { }) defer release() + // A machine whose declaration would move a running module's data is held, and only it (novox/hq + // ADR 0217): said, with the command that sends it, and left out of everything below. + toSend, err := holdingBack(ctx, inv, sending, movable, &heldBack) + if err != nil { + return err + } // **What it recreates, said before it is sent** (novox/hq ADR 0245): a person's push moves every // module whose build changed, and recreates what changed in it — a gated send said so, a push did not. - sayWhatAPushRecreates(ctx, open, sending, os.Stdout) + sayWhatAPushRecreates(ctx, open, toSend, os.Stdout) // Each machine's memberships first, then the declarations (novox/hq issue 249, ADR 0160): a push // is the one most operators run, and on 2026-10-01 it was the one path that issued none. bus := overTheBus{open: open, server: server, signer: ident} - sentDigest, err := deliver(ctx, bus, holder, sending) + sentDigest, err := deliver(ctx, bus, holder, toSend) if err != nil { return err } release() - told := make([]string, 0, len(sending)) - for _, r := range sending { + told := make([]string, 0, len(toSend)) + for _, r := range toSend { told = append(told, r.node) } - fmt.Printf("\n%d node(s) told: %s\n", len(sending), strings.Join(told, ", ")) + fmt.Printf("\n%d node(s) told: %s\n", len(toSend), strings.Join(told, ", ")) reportUnheldPushed(os.Stdout, len(args) == 1, asked, unheld) // **A named push leaves the mesh consistent, not just the machine it named** (novox/hq @@ -624,7 +653,12 @@ func pushCommand(ctx context.Context, args []string) error { for n := range saidHeld { handled[n] = true } - refused, err := flushBehind(ctx, open, nodes, handled, composeForPush(open, gens), bus, holder, os.Stdout) + // The cascade is a push too, and a machine in it whose data would move is held the same way + // (novox/hq ADR 0217). + keep := keeping(func(held context.Context, sending []readyNode) ([]readyNode, error) { + return holdingBack(held, inv, sending, movable, &heldBack) + }) + refused, err := flushBehind(ctx, open, nodes, handled, composeForPush(open, gens), bus, holder, keep, os.Stdout) refusals = append(refusals, refused...) if err != nil { return err @@ -639,6 +673,11 @@ func pushCommand(ctx context.Context, args []string) error { return err } } + if len(heldBack) > 0 { + sort.Strings(heldBack) + return fmt.Errorf("held %s: a running module's data would move — see above; nothing was sent "+ + "there (novox/hq ADR 0217)", strings.Join(heldBack, ", ")) + } return couldNotBeResolved(refusals, len(sending)) } @@ -747,7 +786,7 @@ func composeEach(names []string, allot func(node string) (order, error), // still sent. Their memberships go before their declarations, as every send's do (issue 249). func sendRound(ctx context.Context, open *stores, names []string, compose func(held context.Context, node string) (sendable, error), - d delivery, holder string) ([]string, error) { + d delivery, holder string, keep keeping) ([]string, error) { held, release, err := holdNodes(ctx, open, names) if err != nil { return nil, err @@ -756,12 +795,46 @@ func sendRound(ctx context.Context, open *stores, names []string, sending, refused := composeEach(names, allotting(held, open.inventory), func(node string) (sendable, error) { return compose(held, node) }) + if keep != nil { + if sending, err = keep(held, sending); err != nil { + return refused, err + } + } if _, err := deliver(held, d, holder, sending); err != nil { return refused, err } return refused, nil } +// keeping is what a round of sends keeps of the machines it composed, before any is delivered: nil +// keeps them all. A push holds back the machines whose running modules' data would move (holdingBack). +type keeping func(held context.Context, sending []readyNode) ([]readyNode, error) + +// holdingBack leaves out each machine whose declaration would move a running module's data, says so, +// and names it among those held back (novox/hq ADR 0217); the rest go on to be delivered, grants first +// (issue 249). A declaration's body is the same bytes however often it is read. +func holdingBack(ctx context.Context, inv *inventory.Inventory, sending []readyNode, movable map[string]bool, + heldBack *[]string) ([]readyNode, error) { + var out []readyNode + for _, s := range sending { + body, err := s.declared.Body() + if err != nil { + return nil, err + } + moves, err := heldMoves(ctx, inv, s.node, body, movable) + if err != nil { + return nil, err + } + if len(moves) > 0 { + sayDataHeld(os.Stdout, s.node, moves) + *heldBack = append(*heldBack, s.node) + continue + } + out = append(out, s) + } + return out, nil +} + // delivery is the two acts of sending machines what they should be, apart, so the order between // them is one function's and can be read and tested there (novox/hq issue 249). type delivery interface { @@ -1473,6 +1546,16 @@ func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body if err := inv.RecordSentUnder(kept, record.ID, digest, builds, epoch); err != nil { return "", err } + // And what it was, summarised, so the next push can be compared with it (novox/hq ADR 0217). + // Never a reason to fail a send that is already away: a summary that cannot be kept means the + // next comparison has nothing to hold against, which is how every machine starts. + if summary, err := summarize(body); err == nil { + if raw, err := json.Marshal(summary); err == nil { + if err := inv.RecordSentSummary(kept, record.ID, raw); err != nil { + fmt.Fprintf(os.Stderr, "%s: what it was sent is not kept for comparison: %v\n", node, err) + } + } + } return digest, nil } diff --git a/cmd/mesh-controller/seatverbs.go b/cmd/mesh-controller/seatverbs.go index 90cf5af..351df3e 100644 --- a/cmd/mesh-controller/seatverbs.go +++ b/cmd/mesh-controller/seatverbs.go @@ -407,6 +407,10 @@ func (a *verbArguments) commandLine() ([]string, error) { if on("files") { return []string{"plan", str("node"), "--files"}, nil } + // What a push would change there (novox/hq ADR 0217); diff with files is passed over, and so refused. + if on("diff") { + return []string{"plan", str("node"), "--diff"}, nil + } return []string{"plan", str("node"), "--json"}, nil case "assign", "unassign": if err := need("node", "module"); err != nil { @@ -441,10 +445,16 @@ func (a *verbArguments) commandLine() ([]string, error) { // behind is not read here: given with a machine, it is refused as passed over — naming // a machine and asking for every machine behind are two requests, and guessing one // would push a machine nobody named, or not push one somebody did. - return append([]string{"push", n, "--wait", "0"}, why...), nil + argv := []string{"push", n, "--wait", "0"} + // A running module's data moving is sent only when said, per module (novox/hq ADR 0217). + if m := str("move"); m != "" { + argv = append(argv, "--move", m) + } + return append(argv, why...), nil } // No machine: the whole mesh, whether or not behind said so. The command's answer says it - // first, so a caller who meant one machine reads that it was not one. + // first, so a caller who meant one machine reads that it was not one. move is not read: a + // machine whose data would move is held and named, and sent by a push that names it. on("behind") return append([]string{"push", "--behind", "--wait", "0"}, why...), nil case "hand-act": @@ -672,13 +682,21 @@ func (a *verbArguments) commandLine() ([]string, error) { return nil, err } var argv []string - if on("clear") { + switch { + case on("clear"): argv = []string{"settings", "clear", str("module")} - } else { - argv = []string{"settings", "set", str("module")} - // Neither values nor clear: the command says its usage, which names both. - if v := str("values"); v != "" { - argv = append(argv, v) + case str("values") != "": + argv = []string{"settings", "set", str("module"), str("values")} + // What a set removes is refused unless meant (novox/hq ADR 0217). + if on("replace") { + argv = append(argv, "--replace") + } + default: + // Neither values nor clear: the layer as it stands, which is what a caller reads before + // replacing it (novox/hq ADR 0217) — and with history, the layers it replaced. + argv = []string{"settings", "show", str("module")} + if on("history") { + argv = append(argv, "--history") } } if n := str("node"); n != "" { diff --git a/cmd/mesh-controller/seatverbs_schema_test.go b/cmd/mesh-controller/seatverbs_schema_test.go index 35656dc..9ab0339 100644 --- a/cmd/mesh-controller/seatverbs_schema_test.go +++ b/cmd/mesh-controller/seatverbs_schema_test.go @@ -258,7 +258,11 @@ var accountedFlags = map[string]map[string]string{ "seats": {"json": "set by the verb: the answer is data"}, "queue": {"json": "not set: the verb answers the table a person reads"}, "plan": {"json": "set by the verb unless files is asked"}, - "push": {"wait": "set by the verb to 0: a tool call cannot hold a connection for a whole apply"}, + "push": { + "wait": "set by the verb to 0: a tool call cannot hold a connection for a whole apply", + "all": "withheld: the verb naming no machine is --behind, and a push of every machine is said at a " + + "shell (novox/hq ADR 0217); `command` reaches it", + }, "build": { "wait": "set by the verb to 0: the id follows the build (issue 176)", "self": "set by the verb from the repository's form: a path on the forge, or a URL", diff --git a/cmd/mesh-controller/seatverbs_test.go b/cmd/mesh-controller/seatverbs_test.go index 0124c7d..64d3799 100644 --- a/cmd/mesh-controller/seatverbs_test.go +++ b/cmd/mesh-controller/seatverbs_test.go @@ -72,9 +72,11 @@ func TestSettingsSetsOrClearsALayer(t *testing.T) { if strings.Join(argv, " ") != "settings clear dnsmasq" { t.Fatalf("clear for the mesh: %v", argv) } + // Neither values nor clear reads the layer as it stands (novox/hq ADR 0217): what a caller reads + // before replacing it, where it used to fall to the command's usage. argv, _ = argvFor("settings", map[string]any{"module": "dnsmasq"}) - if strings.Join(argv, " ") != "settings set dnsmasq" { - t.Fatalf("a set with no values falls to the command's usage: %v", argv) + if strings.Join(argv, " ") != "settings show dnsmasq" { + t.Fatalf("a call with no values reads the layer: %v", argv) } } diff --git a/cmd/mesh-controller/unseen.go b/cmd/mesh-controller/unseen.go new file mode 100644 index 0000000..543a96e --- /dev/null +++ b/cmd/mesh-controller/unseen.go @@ -0,0 +1,338 @@ +package main + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "os" + "reflect" + "sort" + "strings" + + "github.com/novox/mesh-controller/internal/inventory" +) + +// No change to a machine takes effect unseen (novox/hq ADR 0217, to-be 44). +// +// Two incidents in two days had one shape: a change took effect that nobody saw before it did. A +// provisioner read an unreadable file as "nobody asks" and dropped seven databases (issue 241); one +// placement set for one module on one machine replaced its whole settings layer, and the plan then +// ran the module on an empty directory (novox/hq issue 304). Here are the three guards: what a settings layer +// loses is said and refused unless meant; what a push will change can be read before it is sent; and +// a running container's data moving holds that machine's push until the operator says so. Each is +// silent when nothing is at stake, so an ordinary change goes exactly as before. + +// ---- 1. what a settings layer loses ------------------------------------------------------------ + +// settingsChange is what replacing one layer with another adds, changes and removes, by dotted path +// to each leaf — `places.config`, not only `places` — because a layer that keeps a key and loses one +// of its entries has lost something just the same: on 2026-10-05 a node's `places` kept its name and +// lost three of its four directories. +func settingsChange(before, after map[string]any) (added, changed, removed []string) { + was, now := leaves(before, ""), leaves(after, "") + for path, v := range now { + old, had := was[path] + switch { + case !had: + added = append(added, path) + case !reflect.DeepEqual(old, v): + changed = append(changed, path) + } + } + for path := range was { + if _, kept := now[path]; !kept { + removed = append(removed, path) + } + } + sort.Strings(added) + sort.Strings(changed) + sort.Strings(removed) + return added, changed, removed +} + +// leaves flattens a settings layer to its leaves by dotted path; a list is a leaf, compared whole. +func leaves(m map[string]any, prefix string) map[string]any { + out := map[string]any{} + for k, v := range m { + path := k + if prefix != "" { + path = prefix + "." + k + } + if inner, ok := v.(map[string]any); ok && len(inner) > 0 { + for p, leaf := range leaves(inner, path) { + out[p] = leaf + } + continue + } + out[path] = v + } + return out +} + +// ---- 2. what a push will change ----------------------------------------------------------------- + +// sentResource is what is kept of one resource a machine was sent: enough to say what changed and +// whether a container's data moved, and nothing that could carry a secret — a file's content is +// kept as a digest, never as text (the record lands in the store's own backups). +type sentResource struct { + ID string `json:"id"` + Type string `json:"type"` + // Digest is of the whole resource as it was sent. + Digest string `json:"digest"` + // Fields is each field's digest, so a change can be named by field. + Fields map[string]string `json:"fields"` + // Volumes is a container's mounts as sent — paths, which are not secrets, and what a moved + // data directory is read from. + Volumes []string `json:"volumes,omitempty"` +} + +// summarize is what is kept of a declaration body: its resources, in the order sent. +func summarize(body []byte) ([]sentResource, error) { + var envelope struct { + Resources []map[string]any `json:"resources"` + } + if err := json.Unmarshal(body, &envelope); err != nil { + return nil, fmt.Errorf("a declaration that is not one: %w", err) + } + out := make([]sentResource, 0, len(envelope.Resources)) + for _, r := range envelope.Resources { + s := sentResource{ID: fmt.Sprint(r["id"]), Type: fmt.Sprint(r["type"]), Digest: digestValue(r), + Fields: map[string]string{}} + for k, v := range r { + s.Fields[k] = digestValue(v) + } + if s.Type == "container" { + if vols, ok := r["volumes"].([]any); ok { + for _, v := range vols { + s.Volumes = append(s.Volumes, fmt.Sprint(v)) + } + } + } + out = append(out, s) + } + return out, nil +} + +func digestValue(v any) string { + raw, _ := json.Marshal(v) + sum := sha256.Sum256(raw) + return hex.EncodeToString(sum[:8]) +} + +// changedResource is one resource sent differently, with the fields that differ. +type changedResource struct { + ID, Type string + Fields []string +} + +// sentDiff is what would be sent now against what was sent last, by resource id. +type sentDiff struct { + Added, Removed []string + Changed []changedResource +} + +func (d sentDiff) empty() bool { + return len(d.Added) == 0 && len(d.Removed) == 0 && len(d.Changed) == 0 +} + +func diffSent(before, after []sentResource) sentDiff { + was := map[string]sentResource{} + for _, r := range before { + was[r.ID] = r + } + var d sentDiff + seen := map[string]bool{} + for _, r := range after { + seen[r.ID] = true + old, had := was[r.ID] + if !had { + d.Added = append(d.Added, r.ID) + continue + } + if old.Digest == r.Digest { + continue + } + c := changedResource{ID: r.ID, Type: r.Type} + for k, v := range r.Fields { + if old.Fields[k] != v { + c.Fields = append(c.Fields, k) + } + } + for k := range old.Fields { + if _, kept := r.Fields[k]; !kept { + c.Fields = append(c.Fields, k) + } + } + sort.Strings(c.Fields) + d.Changed = append(d.Changed, c) + } + for _, r := range before { + if !seen[r.ID] { + d.Removed = append(d.Removed, r.ID) + } + } + sort.Strings(d.Added) + sort.Strings(d.Removed) + sort.Slice(d.Changed, func(a, b int) bool { return d.Changed[a].ID < d.Changed[b].ID }) + return d +} + +// writeDiff says a diff the way a person reads one: what is added, removed and changed, and a +// changed container's fields — a container sent differently is a container recreated. +func writeDiff(w io.Writer, node string, d sentDiff, moves []dataMove) { + if d.empty() { + fmt.Fprintf(w, "%s: nothing would change — it was last sent exactly this\n", node) + return + } + fmt.Fprintf(w, "%s would change:\n", node) + for _, id := range d.Added { + fmt.Fprintf(w, " + %s\n", id) + } + for _, id := range d.Removed { + fmt.Fprintf(w, " - %s\n", id) + } + for _, c := range d.Changed { + what := "changed" + if c.Type == "container" { + what = "recreated" + } + fmt.Fprintf(w, " ~ %s %s: %s\n", c.ID, what, strings.Join(c.Fields, ", ")) + } + writeMoves(w, node, moves) +} + +// ---- 3. a running module's data moving ------------------------------------------------------------ + +// dataMove is a container keeping a mount at the same place inside it with a different directory +// on the machine behind it: its data moving — or, as on 2026-10-05, a module about to start on an +// empty directory because the placement that pointed at its data was lost. +type dataMove struct { + Container, Inside, From, To string +} + +// Module is the module the container belongs to: resource ids are `.`. +func (m dataMove) Module() string { + module, _, _ := strings.Cut(m.Container, ".") + return module +} + +// movesIn is every data move between what a machine was sent and what it would be sent. A new +// container, a mount added or dropped and a changed image are not moves. +func movesIn(before, after []sentResource) []dataMove { + was := map[string]sentResource{} + for _, r := range before { + if r.Type == "container" { + was[r.ID] = r + } + } + var out []dataMove + for _, r := range after { + old, had := was[r.ID] + if r.Type != "container" || !had { + continue + } + from := mountSources(old.Volumes) + for inside, to := range mountSources(r.Volumes) { + if prev, mounted := from[inside]; mounted && prev != to { + out = append(out, dataMove{Container: r.ID, Inside: inside, From: prev, To: to}) + } + } + } + sort.Slice(out, func(a, b int) bool { + if out[a].Container != out[b].Container { + return out[a].Container < out[b].Container + } + return out[a].Inside < out[b].Inside + }) + return out +} + +// mountSources is a container's mounts by the place inside it: `source:inside[:options]`. +func mountSources(volumes []string) map[string]string { + out := map[string]string{} + for _, v := range volumes { + parts := strings.SplitN(v, ":", 3) + if len(parts) < 2 { + continue + } + out[parts[1]] = parts[0] + } + return out +} + +func writeMoves(w io.Writer, node string, moves []dataMove) { + for _, m := range moves { + fmt.Fprintf(w, " ! %s: %s moves from %s to %s\n", m.Container, m.Inside, m.From, m.To) + } +} + +// heldMoves is the moves in a push to one machine that the operator has not said to send (`--move +// `); empty when there is nothing to hold, including a machine never sent anything before. +func heldMoves(ctx context.Context, inv *inventory.Inventory, node string, body []byte, allowed map[string]bool) ([]dataMove, error) { + prev, err := inv.SentSummary(ctx, node) + if err != nil || len(prev) == 0 { + return nil, err + } + var before []sentResource + if err := json.Unmarshal(prev, &before); err != nil { + // A record this version cannot read compares with nothing: holding every push for it would + // stop the mesh on a format, which is not what is at stake. Said, so it is not passed over. + fmt.Fprintf(os.Stderr, "%s: what it was last sent is kept in a form this version does not read, "+ + "so whether its data would move is not known: %v\n", node, err) + // empty-on-error: said above; the send replaces the unreadable record with one this version reads + return nil, nil + } + after, err := summarize(body) + if err != nil { + return nil, err + } + var held []dataMove + for _, m := range movesIn(before, after) { + if !allowed[m.Module()] { + held = append(held, m) + } + } + return held, nil +} + +// sayDataHeld tells the operator why a machine was not sent, and how to send it. +func sayDataHeld(w io.Writer, node string, moves []dataMove) { + modules := map[string]bool{} + for _, m := range moves { + modules[m.Module()] = true + } + var names []string + for m := range modules { + names = append(names, m) + } + sort.Strings(names) + fmt.Fprintf(w, "held %s: a running module's data would move (novox/hq ADR 0217)\n", node) + writeMoves(w, node, moves) + fmt.Fprintf(w, " if that is meant: push %s --move %s\n", node, strings.Join(names, ",")) +} + +// writePlanDiff says what sending body to a machine would change against what it was last sent. +func writePlanDiff(ctx context.Context, inv *inventory.Inventory, node string, body []byte) error { + after, err := summarize(body) + if err != nil { + return err + } + prev, err := inv.SentSummary(ctx, node) + if err != nil { + return err + } + if len(prev) == 0 { + fmt.Printf("%s: what it was last sent is not kept yet — the first push from this version keeps "+ + "it; %d resource(s) would be sent\n", node, len(after)) + return nil + } + var before []sentResource + if err := json.Unmarshal(prev, &before); err != nil { + return fmt.Errorf("%s: what it was last sent is kept in a form this version does not read: %w", node, err) + } + writeDiff(os.Stdout, node, diffSent(before, after), movesIn(before, after)) + return nil +} diff --git a/cmd/mesh-controller/unseen_test.go b/cmd/mesh-controller/unseen_test.go new file mode 100644 index 0000000..32f9ddf --- /dev/null +++ b/cmd/mesh-controller/unseen_test.go @@ -0,0 +1,190 @@ +package main + +import ( + "bytes" + "context" + "reflect" + "strings" + "testing" +) + +// novox/hq ADR 0217: no change to a machine takes effect unseen. + +// The layer of 2026-10-05: a media server's node layer, and the one that replaced it, which kept +// `places` and lost three of its four directories and every other key. +var before = map[string]any{ + "puid": 1000.0, "pgid": 1000.0, + "expose": map[string]any{"32400": "anywhere"}, + "places": map[string]any{ + "config": map[string]any{"path": "/srv/media/config", "owner": "1000:1000"}, + "data": map[string]any{"path": "/srv/media/data", "owner": "1000:1000"}, + "transcode": map[string]any{"path": "/srv/media/temp", "owner": "1000:1000"}, + }, +} + +func TestASettingThatKeepsAKeyAndLosesItsEntriesIsSaidToRemoveThem(t *testing.T) { + after := map[string]any{"places": map[string]any{ + "previews": map[string]any{"path": "/srv/pool/previews", "owner": "1000:1000"}, + }} + added, changed, removed := settingsChange(before, after) + if !reflect.DeepEqual(added, []string{"places.previews.owner", "places.previews.path"}) { + t.Errorf("added %v", added) + } + if len(changed) != 0 { + t.Errorf("changed %v", changed) + } + for _, lost := range []string{"expose.32400", "pgid", "places.config.path", "places.data.owner", "puid"} { + found := false + for _, r := range removed { + found = found || r == lost + } + if !found { + t.Errorf("removing %s was not said: %v", lost, removed) + } + } +} + +func TestASettingThatOnlyAddsOrChangesRemovesNothing(t *testing.T) { + after := map[string]any{ + "puid": 1001.0, "pgid": 1000.0, + "expose": map[string]any{"32400": "anywhere"}, + "places": map[string]any{ + "config": map[string]any{"path": "/srv/media/config", "owner": "1000:1000"}, + "data": map[string]any{"path": "/srv/media/data", "owner": "1000:1000"}, + "transcode": map[string]any{"path": "/srv/media/temp", "owner": "1000:1000"}, + "previews": map[string]any{"path": "/srv/pool/previews", "owner": "1000:1000"}, + }, + } + _, changed, removed := settingsChange(before, after) + if len(removed) != 0 { + t.Errorf("an additive set was said to remove %v", removed) + } + if !reflect.DeepEqual(changed, []string{"puid"}) { + t.Errorf("changed %v", changed) + } +} + +// What was sent, as a push would send it: a media server's container and a file with a secret. +func declaration(configFrom string, extra ...string) []byte { + vols := `"` + configFrom + `:/config", "/srv/media/data:/data"` + for _, e := range extra { + vols += `, "` + e + `"` + } + return []byte(`{"declaration":1,"sequence":7,"resources":[ + {"id":"plex.server","type":"container","image":"plex@sha256:aa","volumes":[` + vols + `]}, + {"id":"plex.env","type":"file","path":"/srv/media/env","content":"TOKEN=the-secret-itself"}]}`) +} + +func summarized(t *testing.T, body []byte) []sentResource { + t.Helper() + s, err := summarize(body) + if err != nil { + t.Fatal(err) + } + return s +} + +func TestWhatIsKeptOfASendHoldsNoFileContent(t *testing.T) { + s := summarized(t, declaration("/srv/media/config")) + var kept bytes.Buffer + for _, r := range s { + kept.WriteString(r.ID + r.Type + r.Digest + strings.Join(r.Volumes, ",")) + for k, v := range r.Fields { + kept.WriteString(k + v) + } + } + if strings.Contains(kept.String(), "the-secret-itself") { + t.Fatal("a file's content was kept in the record of what was sent") + } + if len(s) != 2 || s[0].Volumes[0] != "/srv/media/config:/config" { + t.Fatalf("summary %+v", s) + } +} + +func TestADiffOfAnUnchangedMachineIsEmptyAndAChangedContainerNamesItsField(t *testing.T) { + was := summarized(t, declaration("/srv/media/config")) + if d := diffSent(was, summarized(t, declaration("/srv/media/config"))); !d.empty() { + t.Fatalf("an unchanged machine differs: %+v", d) + } + d := diffSent(was, summarized(t, declaration("/var/lib/plex/config"))) + if len(d.Changed) != 1 || d.Changed[0].ID != "plex.server" || !reflect.DeepEqual(d.Changed[0].Fields, []string{"volumes"}) { + t.Fatalf("diff %+v", d) + } + var out bytes.Buffer + writeDiff(&out, "home", d, movesIn(was, summarized(t, declaration("/var/lib/plex/config")))) + for _, want := range []string{"~ plex.server recreated: volumes", "! plex.server: /config moves from /srv/media/config to /var/lib/plex/config"} { + if !strings.Contains(out.String(), want) { + t.Errorf("diff does not say %q:\n%s", want, out.String()) + } + } +} + +// The incident: the configuration mount would move to an empty default directory. +func TestARunningContainersDataMovingIsAMoveAndAnAddedMountIsNot(t *testing.T) { + was := summarized(t, declaration("/srv/media/config")) + moves := movesIn(was, summarized(t, declaration("/var/lib/plex/config"))) + want := []dataMove{{Container: "plex.server", Inside: "/config", From: "/srv/media/config", To: "/var/lib/plex/config"}} + if !reflect.DeepEqual(moves, want) { + t.Fatalf("moves %+v", moves) + } + if moves[0].Module() != "plex" { + t.Errorf("module %q", moves[0].Module()) + } + // A mount added — the previews of 2026-10-05 — is not a move. + if m := movesIn(was, summarized(t, declaration("/srv/media/config", "/srv/pool/previews:/config/Media"))); len(m) != 0 { + t.Errorf("an added mount was held as a move: %+v", m) + } + // Nor is a container the machine never ran. + if m := movesIn(nil, was); len(m) != 0 { + t.Errorf("a first send was held as a move: %+v", m) + } +} + +func TestAPushNamingNoMachineIsRefusedUnlessItSaysAll(t *testing.T) { + err := pushCommand(context.Background(), nil) + if err == nil || !strings.Contains(err.Error(), "--all") { + t.Fatalf("a bare push was not refused: %v", err) + } + if err := pushCommand(context.Background(), []string{"--all", "--behind"}); err == nil { + t.Fatal("--all with --behind was accepted") + } +} + +func TestTheVerbsCarryReadReplaceAndMove(t *testing.T) { + for _, c := range []struct { + verb string + args map[string]any + want []string + }{ + {"settings", map[string]any{"module": "plex", "node": "home"}, []string{"settings", "show", "plex", "--node", "home"}}, + {"settings", map[string]any{"module": "plex", "history": "true"}, []string{"settings", "show", "plex", "--history"}}, + {"settings", map[string]any{"module": "plex", "values": "{}", "replace": "true"}, []string{"settings", "set", "plex", "{}", "--replace"}}, + {"push", map[string]any{"node": "home", "move": "plex", "why": "w"}, + []string{"push", "home", "--wait", "0", "--move", "plex", "--why", "w"}}, + {"push", map[string]any{"why": "w"}, []string{"push", "--behind", "--wait", "0", "--why", "w"}}, + {"plan", map[string]any{"node": "home", "diff": "true"}, []string{"plan", "home", "--diff"}}, + } { + got, err := argvFor(c.verb, c.args) + if err != nil || !reflect.DeepEqual(got, c.want) { + t.Errorf("%s %v: %v %v, want %v", c.verb, c.args, got, err, c.want) + } + } +} + +// An argument of the guards given where it cannot take effect is refused, not passed over (issue 244): +// a move with no machine named, a replace beside a clear, a history beside values, a diff beside files. +func TestTheGuardsArgumentsAreNotPassedOver(t *testing.T) { + for _, c := range []struct { + verb string + args map[string]any + }{ + {"push", map[string]any{"move": "plex", "why": "w"}}, + {"settings", map[string]any{"module": "plex", "clear": "true", "replace": "true"}}, + {"settings", map[string]any{"module": "plex", "values": "{}", "history": "true"}}, + {"plan", map[string]any{"node": "home", "files": "true", "diff": "true"}}, + } { + if argv, err := argvFor(c.verb, c.args); err == nil { + t.Errorf("%s %v was taken as %v, and an argument passed over", c.verb, c.args, argv) + } + } +} diff --git a/internal/catalogue/verbs.go b/internal/catalogue/verbs.go index 5e96e81..94f3c61 100644 --- a/internal/catalogue/verbs.go +++ b/internal/catalogue/verbs.go @@ -168,11 +168,14 @@ var ControllerVerbs = []Verb{ Input: schema(map[string]string{"plan": "one walk's id", "limit": "how many ended walks beside the open ones (default 50)"}, nil)}, {Name: "plan", Description: "What one machine would run, and why: the declaration the mesh would send it — " + - "or, with files, the files it would be given.", + "or, with files, the files it would be given; or, with diff, what a push would change there against what " + + "it was last sent: resources added, removed and changed, a container's changed fields (it is recreated), " + + "and any mount whose directory on the machine would move (novox/hq ADR 0217).", Input: schema(map[string]string{ "node": "the machine's name", "files": "\"true\": the files this machine would be given, instead of the declaration as JSON", - }, []string{"node"}, "files")}, + "diff": "\"true\": what a push would change on this machine, against what it was last sent; not with files", + }, []string{"node"}, "files", "diff")}, {Name: "assign", Description: "Put a module on a machine. Refused with the mesh's own words when it cannot resolve there, " + "or when a seat its resources are applied through is held by nothing on the machine (novox/hq ADR 0207).", Input: schema(map[string]string{"node": "the machine's name", @@ -194,9 +197,12 @@ var ControllerVerbs = []Verb{ "WHOLE mesh — every machine that is behind — and the answer says so first; behind says that outright. " + "Answers at once that it is running, with a call id: `calls` with that id says what it sent " + "(a push can reload the bus, which then refuses any answer still to come). A push by hand is a repair, " + - "and says why: recorded in the hand-act log (novox/hq to-be 45 §7).", + "and says why: recorded in the hand-act log (novox/hq to-be 45 §7). A machine whose push would give a " + + "running module's data another directory is held and named, the others sent (novox/hq ADR 0217): read " + + "`plan` with diff, and name the machine with move to send it.", Input: schema(map[string]string{ "node": "the machine's name; without it, every machine that is behind", + "move": "with node: the modules whose data may move with this push, comma-separated (optional)", "behind": "\"true\": every machine that is behind, the whole mesh — the same as naming none, said outright; not with node", "why": "why this is pushed by hand: recorded in the hand-act log", "cause": "the cause in a word, or a condition's kind — the word a second push for the same reason uses (optional)", @@ -222,15 +228,20 @@ var ControllerVerbs = []Verb{ "node": "the machine that runs the module", "module": "the module's name", }, []string{"node", "module"})}, - {Name: "settings", Description: "Set what an assignment is configured with: a module's settings for the whole mesh, " + - "or for one machine. Replaces that layer whole — what it does not name, it no longer sets — and takes effect " + - "at the next push. With clear, removes the layer and the module is back to what its definition says.", + {Name: "settings", Description: "Read or set what an assignment is configured with: a module's settings for the whole " + + "mesh, or for one machine. Without values or clear, answers the layer as it stands — read it before setting it; " + + "with history, the layers it replaced. Setting replaces that layer whole and answers each key it adds (+), " + + "changes (~) and removes (-); a set that would remove a key is refused unless replace says it is meant " + + "(novox/hq ADR 0217). Takes effect at the next push. With clear, removes the layer and the module is back to " + + "what its definition says; a cleared or replaced layer is kept in the history.", Input: schema(map[string]string{ - "module": "the module's name", - "values": "the settings as a JSON object, for set", - "node": "one machine; the whole mesh when absent", - "clear": "\"true\" to remove the layer instead of setting it; not with values", - }, []string{"module"}, "clear")}, + "module": "the module's name", + "values": "the settings as a JSON object, for set", + "node": "one machine; the whole mesh when absent", + "clear": "\"true\" to remove the layer instead of setting it; not with values", + "replace": "\"true\": with values, the set is meant to remove the keys the layer had and it does not name", + "history": "\"true\": without values or clear, the layers this one replaced, the latest first", + }, []string{"module"}, "clear", "replace", "history")}, {Name: "command", Description: "Run one command line of the controller's own, as you would type it at its " + "shell — `node account g14 jochen`, `node show ace`, `module list` — and answer what it printed. The " + "generic verb beside the named ones (novox/hq ADR 0154): everything the binary can do, without a verb " + diff --git a/internal/inventory/catalogue.go b/internal/inventory/catalogue.go index fd56782..1fd0cbd 100644 --- a/internal/inventory/catalogue.go +++ b/internal/inventory/catalogue.go @@ -793,11 +793,23 @@ func (i *Inventory) setSettings(ctx context.Context, nodeName, module string, va return fmt.Errorf("%s: %s is given per node — a port is a fact about one machine; "+ "set it with --node", module, catalogue.PortsSetting) } - _, err = i.store.Pool().Exec(ctx, + // The layer it replaces is kept (novox/hq ADR 0217), in the same transaction as the write: a + // previous value is one command away, not in a backup, and a write that fails leaves no history. + tx, err := i.store.Pool().Begin(ctx) + if err != nil { + return err + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + if _, err := tx.Exec(ctx, keepReplacedSQL, module, nil, "set"); err != nil { + return err + } + if _, err := tx.Exec(ctx, `insert into settings (node, module, values) values (null, $1, $2) on conflict (module) where node is null - do update set values = excluded.values, set_at = now()`, module, raw) - return wrapModule(err, module) + do update set values = excluded.values, set_at = now()`, module, raw); err != nil { + return wrapModule(err, module) + } + return tx.Commit(ctx) } node, err := i.NodeByName(ctx, nodeName) if err != nil { @@ -817,6 +829,10 @@ func (i *Inventory) setSettings(ctx context.Context, nodeName, module string, va return err } } + // The layer it replaces is kept (novox/hq ADR 0217), in the same transaction as the write. + if _, err := tx.Exec(ctx, keepReplacedSQL, module, node.ID, "set"); err != nil { + return err + } _, err = tx.Exec(ctx, `insert into settings (node, module, values) values ($1, $2, $3) on conflict (node, module) where node is not null @@ -1009,18 +1025,24 @@ func wrapModule(err error, module string) error { // ClearSettings removes a layer. func (i *Inventory) ClearSettings(ctx context.Context, nodeName, module string) error { - if nodeName == "" { - _, err := i.store.Pool().Exec(ctx, - `delete from settings where module = $1 and node is null`, module) - return err - } - node, err := i.NodeByName(ctx, nodeName) + nodeID, err := i.layerNode(ctx, nodeName) if err != nil { return err } - _, err = i.store.Pool().Exec(ctx, - `delete from settings where module = $1 and node = $2`, module, node.ID) - return err + // The layer it removes is kept (novox/hq ADR 0217), in the same transaction as the removal. + tx, err := i.store.Pool().Begin(ctx) + if err != nil { + return err + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + if _, err := tx.Exec(ctx, keepReplacedSQL, module, nodeID, "clear"); err != nil { + return err + } + if _, err := tx.Exec(ctx, + `delete from settings where module = $1 and node is not distinct from $2::uuid`, module, nodeID); err != nil { + return err + } + return tx.Commit(ctx) } // SettingsFor is the layers that apply to one module on one node, in the order they are applied. diff --git a/internal/inventory/migrations/0078-what-a-change-replaces.sql b/internal/inventory/migrations/0078-what-a-change-replaces.sql new file mode 100644 index 0000000..c6a6ec8 --- /dev/null +++ b/internal/inventory/migrations/0078-what-a-change-replaces.sql @@ -0,0 +1,26 @@ +-- What a change replaces (novox/hq ADR 0217, to-be 44). +-- +-- Two records the mesh did not keep, and each time a change took effect unseen it was the one +-- missing. A settings layer is replaced whole, and the layer it replaced was nowhere: on 2026-10-05 +-- one placement set for one module on one machine dropped that machine's whole layer for it, and +-- the old one was read back from a database backup (novox/hq issue 304). And of what a machine was sent the +-- mesh kept only a digest — enough to say *whether* it changed, never *what*. + +-- Every layer that was replaced or cleared, with when it had been set and when it went. Not a +-- foreign key to settings: the row it was is the row being replaced. +create table settings_history ( + node uuid references node(id) on delete cascade, + module text not null, + values jsonb not null, + set_at timestamptz, + replaced_at timestamptz not null default now(), + -- set · clear + replaced_by text not null +); +create index settings_history_by_layer on settings_history (module, node, replaced_at desc); + +-- What a machine was last sent, summarised: per resource its id, type, a digest of it and of each +-- field, and a container's mounts. Not the declaration: a file's content may carry a secret, and +-- this lands in the store's backups. Read only to compare a plan with what was sent; what a machine +-- *should* be is composed from the mesh's records every time, as before (migration 0014). +alter table node add column sent_summary jsonb; diff --git a/internal/inventory/unseen.go b/internal/inventory/unseen.go new file mode 100644 index 0000000..be7e482 --- /dev/null +++ b/internal/inventory/unseen.go @@ -0,0 +1,114 @@ +package inventory + +import ( + "context" + "encoding/json" + "errors" + "time" + + "github.com/jackc/pgx/v5" +) + +// What a change replaces (novox/hq ADR 0217, to-be 44): the settings layer a set or a clear replaced, +// and a summary of what a machine was last sent. Both were missing when a change took effect unseen. + +// Layer is one settings layer as it stands — the whole mesh's when nodeName is empty — and whether +// there is one. A layer is replaced whole when set; reading it first is how one key is changed +// without losing the others. +func (i *Inventory) Layer(ctx context.Context, nodeName, module string) (map[string]any, bool, error) { + nodeID, err := i.layerNode(ctx, nodeName) + if err != nil { + return nil, false, err + } + var raw []byte + err = i.store.Pool().QueryRow(ctx, + `select values from settings where module = $1 and node is not distinct from $2::uuid`, + module, nodeID).Scan(&raw) + if errors.Is(err, pgx.ErrNoRows) { + return nil, false, nil + } + if err != nil { + return nil, false, err + } + values := map[string]any{} + if err := json.Unmarshal(raw, &values); err != nil { + return nil, false, err + } + return values, true, nil +} + +// PastLayer is a layer that was replaced or cleared. +type PastLayer struct { + Values map[string]any + SetAt *time.Time + ReplacedAt time.Time + // ReplacedBy is "set" or "clear". + ReplacedBy string +} + +// SettingsHistory is every layer of one module on one machine — the whole mesh's when nodeName is +// empty — that was replaced or cleared, the latest first. +func (i *Inventory) SettingsHistory(ctx context.Context, nodeName, module string) ([]PastLayer, error) { + nodeID, err := i.layerNode(ctx, nodeName) + if err != nil { + return nil, err + } + rows, err := i.store.Pool().Query(ctx, + `select values, set_at, replaced_at, replaced_by from settings_history + where module = $1 and node is not distinct from $2::uuid order by replaced_at desc`, + module, nodeID) + if err != nil { + return nil, err + } + defer rows.Close() + var out []PastLayer + for rows.Next() { + var raw []byte + var p PastLayer + if err := rows.Scan(&raw, &p.SetAt, &p.ReplacedAt, &p.ReplacedBy); err != nil { + return nil, err + } + if err := json.Unmarshal(raw, &p.Values); err != nil { + return nil, err + } + out = append(out, p) + } + return out, rows.Err() +} + +// keepReplacedSQL copies a layer into the history before it is replaced or cleared; run in the same +// transaction as the write where there is one, so a write that fails leaves no history of it. +const keepReplacedSQL = `insert into settings_history (node, module, values, set_at, replaced_by) + select node, module, values, set_at, $3 from settings + where module = $1 and node is not distinct from $2::uuid` + +// layerNode is the node id of a layer, nil for the whole mesh's. +func (i *Inventory) layerNode(ctx context.Context, nodeName string) (*string, error) { + if nodeName == "" { + return nil, nil + } + n, err := i.NodeByName(ctx, nodeName) + if err != nil { + return nil, err + } + return &n.ID, nil +} + +// RecordSentSummary keeps what a machine was last sent, summarised (migration 0078): read only to +// compare what would be sent with it, never as what the machine should be. +func (i *Inventory) RecordSentSummary(ctx context.Context, node string, summary []byte) error { + _, err := i.store.Pool().Exec(ctx, `update node set sent_summary = $2 where id = $1`, node, summary) + return err +} + +// SentSummary is what a machine was last sent, summarised, by its name; empty for a machine sent +// nothing since the summary was first kept. +func (i *Inventory) SentSummary(ctx context.Context, name string) ([]byte, error) { + var raw []byte + err := i.store.Pool().QueryRow(ctx, + `select sent_summary from node where name = $1`, name).Scan(&raw) + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil + } + return raw, err +} diff --git a/internal/inventory/unseen_test.go b/internal/inventory/unseen_test.go new file mode 100644 index 0000000..73a9803 --- /dev/null +++ b/internal/inventory/unseen_test.go @@ -0,0 +1,104 @@ +package inventory + +import ( + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +// novox/hq ADR 0217: a settings layer can be read, and the one a set or a clear replaced is kept, so +// a previous value is one command away rather than in a database backup (novox/hq issue 304). +func TestTheLayerASetReplacesIsKeptAndReadable(t *testing.T) { + inv := fresh(t) + ctx := t.Context() + if _, err := inv.AddNode(ctx, "anchor"); err != nil { + t.Fatal(err) + } + web := catalogue.Manifest{Module: "web", Version: "1", + Resources: []map[string]any{ + {"id": "server", "type": "container", "name": "web", "image": "x"}, + {"id": "conf", "type": "file", "path": "/etc/web.json", "content": "{}", "merge": "json"}, + }} + if err := inv.RegisterModule(ctx, web, Source{}); err != nil { + t.Fatal(err) + } + + if _, has, err := inv.Layer(ctx, "anchor", "web"); err != nil || has { + t.Fatalf("a layer nobody set: %v %v", has, err) + } + first := map[string]any{"colour": "blue", "size": "large"} + if err := inv.SetSettings(ctx, "anchor", "web", first); err != nil { + t.Fatal(err) + } + got, has, err := inv.Layer(ctx, "anchor", "web") + if err != nil || !has || got["colour"] != "blue" || got["size"] != "large" { + t.Fatalf("the layer read back: %v %v %v", got, has, err) + } + if past, err := inv.SettingsHistory(ctx, "anchor", "web"); err != nil || len(past) != 0 { + t.Fatalf("a first set replaced something: %v %v", past, err) + } + + if err := inv.SetSettings(ctx, "anchor", "web", map[string]any{"colour": "red"}); err != nil { + t.Fatal(err) + } + if err := inv.ClearSettings(ctx, "anchor", "web"); err != nil { + t.Fatal(err) + } + past, err := inv.SettingsHistory(ctx, "anchor", "web") + if err != nil || len(past) != 2 { + t.Fatalf("history %v %v", past, err) + } + // The latest first: the clear took the red layer, the set took the first. + if past[0].ReplacedBy != "clear" || past[0].Values["colour"] != "red" { + t.Errorf("the cleared layer: %+v", past[0]) + } + if past[1].ReplacedBy != "set" || past[1].Values["size"] != "large" { + t.Errorf("the replaced layer still has what the set dropped: %+v", past[1]) + } + // The mesh-wide layer keeps its own history, apart from the node's. + if err := inv.SetSettings(ctx, "", "web", map[string]any{"colour": "green"}); err != nil { + t.Fatal(err) + } + if err := inv.SetSettings(ctx, "", "web", map[string]any{"colour": "grey"}); err != nil { + t.Fatal(err) + } + mesh, err := inv.SettingsHistory(ctx, "", "web") + if err != nil || len(mesh) != 1 || mesh[0].Values["colour"] != "green" { + t.Fatalf("the mesh-wide history %v %v", mesh, err) + } + // And a mesh-wide clear keeps the layer it removes, beside the node's untouched. + if err := inv.ClearSettings(ctx, "", "web"); err != nil { + t.Fatal(err) + } + if _, has, err := inv.Layer(ctx, "", "web"); err != nil || has { + t.Fatalf("a cleared mesh-wide layer is still there: %v %v", has, err) + } + mesh, err = inv.SettingsHistory(ctx, "", "web") + if err != nil || len(mesh) != 2 || mesh[0].ReplacedBy != "clear" || mesh[0].Values["colour"] != "grey" { + t.Fatalf("the mesh-wide clear was not kept: %+v %v", mesh, err) + } + if past, err := inv.SettingsHistory(ctx, "anchor", "web"); err != nil || len(past) != 2 { + t.Fatalf("the node's history changed with the mesh's: %v %v", past, err) + } +} + +// What a machine was last sent is kept, summarised, and read back by its name; a machine sent +// nothing since has nothing to compare with. +func TestWhatAMachineWasSentIsKeptForComparison(t *testing.T) { + inv := fresh(t) + ctx := t.Context() + n, err := inv.AddNode(ctx, "anchor") + if err != nil { + t.Fatal(err) + } + if got, err := inv.SentSummary(ctx, "anchor"); err != nil || len(got) != 0 { + t.Fatalf("a machine never sent anything: %s %v", got, err) + } + if err := inv.RecordSentSummary(ctx, n.ID, []byte(`[{"id":"web.server","type":"container"}]`)); err != nil { + t.Fatal(err) + } + got, err := inv.SentSummary(ctx, "anchor") + if err != nil || len(got) == 0 { + t.Fatalf("read back: %s %v", got, err) + } +}