Merge pull request 'No change to a machine takes effect unseen (hq ADR 0217)' (#50) from feat/no-change-takes-effect-unseen into main

This commit was merged in pull request #50.
This commit is contained in:
2026-10-08 00:05:59 +00:00
18 changed files with 1073 additions and 75 deletions
+2 -2
View File
@@ -45,7 +45,7 @@ func TestAMachineIsSentTheEpochOnlyOnceItSaysItReadsOne(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
d := &bodiesDelivery{recordedDelivery: recordedDelivery{inv: inv}, bodies: map[string][]byte{}} d := &bodiesDelivery{recordedDelivery: recordedDelivery{inv: inv}, bodies: map[string][]byte{}}
if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, composeForPush(open, gens), d, ""); err != nil { if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, composeForPush(open, gens), d, "", nil); err != nil {
t.Fatal(err) t.Fatal(err)
} }
carried := func(node string) (epoch float64, has bool) { carried := func(node string) (epoch float64, has bool) {
@@ -92,7 +92,7 @@ func TestNothingIsComposedOrSentWithoutTheLease(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
d := &bodiesDelivery{recordedDelivery: recordedDelivery{inv: open.inventory}, bodies: map[string][]byte{}} d := &bodiesDelivery{recordedDelivery: recordedDelivery{inv: open.inventory}, bodies: map[string][]byte{}}
refused, err := sendRound(ctx, open, []string{"anchor"}, composeForPush(open, gens), d, "") refused, err := sendRound(ctx, open, []string{"anchor"}, composeForPush(open, gens), d, "", nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
+2 -2
View File
@@ -179,7 +179,7 @@ func sayHeld(w io.Writer, node string, why []string) {
// machines that could not be composed, as refusals. // machines that could not be composed, as refusals.
func flushBehind(ctx context.Context, open *stores, nodes []inventory.Node, handled map[string]bool, func flushBehind(ctx context.Context, open *stores, nodes []inventory.Node, handled map[string]bool,
compose func(held context.Context, node string) (sendable, error), d delivery, holder string, compose func(held context.Context, node string) (sendable, error), d delivery, holder string,
w io.Writer) ([]string, error) { keep keeping, w io.Writer) ([]string, error) {
inv := open.inventory inv := open.inventory
var refusals []string var refusals []string
// Bounded by the node count: a node is marked handled the round it is considered and is never // Bounded by the node count: a node is marked handled the round it is considered and is never
@@ -237,7 +237,7 @@ func flushBehind(ctx context.Context, open *stores, nodes []inventory.Node, hand
// a refusal and reported at the end, and the others are still sent (novox/hq ADR 0066). // a refusal and reported at the end, and the others are still sent (novox/hq ADR 0066).
// Held for this round only, and after the last round's were given back, so two pushes // Held for this round only, and after the last round's were given back, so two pushes
// cascading into each other's machines never each wait on the other. // cascading into each other's machines never each wait on the other.
refused, err := sendRound(ctx, open, sending, compose, d, holder) refused, err := sendRound(ctx, open, sending, compose, d, holder, keep)
refusals = append(refusals, refused...) refusals = append(refusals, refused...)
if err != nil { if err != nil {
return refusals, err return refusals, err
+11 -11
View File
@@ -203,7 +203,7 @@ func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) {
} }
compose := composeForPush(open, gens) compose := composeForPush(open, gens)
d := &recordedDelivery{inv: inv} d := &recordedDelivery{inv: inv}
if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, ""); err != nil { if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, "", nil); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if builds, known, err := inv.SentBuilds(ctx, "laptop"); err != nil || !known || builds["resolver"] != "c1c1c1c1c1" { if builds, known, err := inv.SentBuilds(ctx, "laptop"); err != nil || !known || builds["resolver"] != "c1c1c1c1c1" {
@@ -221,13 +221,13 @@ func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) {
// The change merges; the policy is record. `push anchor` sends the anchor... // The change merges; the policy is record. `push anchor` sends the anchor...
aResolver(t, open, "c2c2c2c2c2", asked.Add(time.Minute)) aResolver(t, open, "c2c2c2c2c2", asked.Add(time.Minute))
d.declared = nil d.declared = nil
if _, err := sendRound(ctx, open, []string{"anchor"}, compose, d, ""); err != nil { if _, err := sendRound(ctx, open, []string{"anchor"}, compose, d, "", nil); err != nil {
t.Fatal(err) t.Fatal(err)
} }
// ...and its cascade leaves the laptop, saying so. // ...and its cascade leaves the laptop, saying so.
var said bytes.Buffer var said bytes.Buffer
d.declared = nil d.declared = nil
refused, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true}, compose, d, "", &said) refused, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true}, compose, d, "", nil, &said)
if err != nil || len(refused) != 0 { if err != nil || len(refused) != 0 {
t.Fatalf("the cascade failed: %v %v", refused, err) t.Fatalf("the cascade failed: %v %v", refused, err)
} }
@@ -248,13 +248,13 @@ func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) {
// is held, and that what else it is owed waits with it. // is held, and that what else it is owed waits with it.
aThirdMachine(t, open) aThirdMachine(t, open)
d.declared = nil d.declared = nil
if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, ""); err != nil { if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, "", nil); err != nil {
t.Fatal(err) t.Fatal(err)
} }
d.declared = nil d.declared = nil
said.Reset() said.Reset()
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true}, if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true},
compose, d, "", &said); err != nil { compose, d, "", nil, &said); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if len(d.declared) != 0 || digestOfLaptop() != before { if len(d.declared) != 0 || digestOfLaptop() != before {
@@ -271,7 +271,7 @@ func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) {
} }
said.Reset() said.Reset()
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true}, if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true},
compose, d, "", &said); err != nil { compose, d, "", nil, &said); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if len(d.declared) != 0 || !strings.Contains(said.String(), "has passed no gate yet") { if len(d.declared) != 0 || !strings.Contains(said.String(), "has passed no gate yet") {
@@ -284,7 +284,7 @@ func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) {
} }
said.Reset() said.Reset()
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true}, if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true},
compose, d, "", &said); err != nil { compose, d, "", nil, &said); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if !reflect.DeepEqual(d.declared, []string{"laptop"}) || digestOfLaptop() == before { if !reflect.DeepEqual(d.declared, []string{"laptop"}) || digestOfLaptop() == before {
@@ -311,18 +311,18 @@ func TestANamedPushStillSendsAConsequenceNothingHolds(t *testing.T) {
} }
compose := composeForPush(open, gens) compose := composeForPush(open, gens)
d := &recordedDelivery{inv: inv} d := &recordedDelivery{inv: inv}
if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, ""); err != nil { if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, "", nil); err != nil {
t.Fatal(err) t.Fatal(err)
} }
// `push spare`, the machine just placed: the others' peers change with it. // `push spare`, the machine just placed: the others' peers change with it.
aThirdMachine(t, open) aThirdMachine(t, open)
if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, ""); err != nil { if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, "", nil); err != nil {
t.Fatal(err) t.Fatal(err)
} }
d.declared = nil d.declared = nil
var said bytes.Buffer var said bytes.Buffer
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", &said); err != nil { if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", nil, &said); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if !reflect.DeepEqual(d.declared, []string{"anchor", "laptop"}) { if !reflect.DeepEqual(d.declared, []string{"anchor", "laptop"}) {
@@ -342,7 +342,7 @@ func TestANamedPushStillSendsAConsequenceNothingHolds(t *testing.T) {
} }
d.declared = nil d.declared = nil
said.Reset() said.Reset()
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", &said); err != nil { if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", nil, &said); err != nil {
t.Fatal(err) t.Fatal(err)
} }
// The anchor, the hub, may still be settling from the machine placed above; the laptop is the // The anchor, the hub, may still be settling from the machine placed above; the laptop is the
+2 -2
View File
@@ -23,11 +23,11 @@ func TestASendRoundGivesItsHoldBackOnEveryWayOut(t *testing.T) {
for name, round := range map[string]func() error{ for name, round := range map[string]func() error{
"a body that cannot be marshalled": func() error { "a body that cannot be marshalled": func() error {
_, err := sendRound(ctx, open, []string{"anchor"}, unmarshallable, fine, "") _, err := sendRound(ctx, open, []string{"anchor"}, unmarshallable, fine, "", nil)
return err return err
}, },
"a send that fails": func() error { "a send that fails": func() error {
_, err := sendRound(ctx, open, []string{"anchor"}, plain, failing, "") _, err := sendRound(ctx, open, []string{"anchor"}, plain, failing, "", nil)
return err return err
}, },
} { } {
+4 -4
View File
@@ -232,7 +232,7 @@ func licenceKey(ctx context.Context, args []string) error {
// and printing the value here would put the one copy that matters on a terminal. // and printing the value here would put the one copy that matters on a terminal.
fmt.Printf("sealed to %d holder(s). The mesh has discarded the key and cannot read it back\n", fmt.Printf("sealed to %d holder(s). The mesh has discarded the key and cannot read it back\n",
sealed) sealed)
fmt.Printf(" run `push` to deliver it\n") fmt.Printf(" run `push --behind` to deliver it\n")
return nil return nil
} }
@@ -340,7 +340,7 @@ func licenceSetGrant(ctx context.Context, args []string) error {
return err return err
} }
fmt.Printf("%s now holds a refresh token for %s, sealed to that node's key and readable by it "+ fmt.Printf("%s now holds a refresh token for %s, sealed to that node's key and readable by it "+
"alone.\n the control plane stored the box without opening it; run `push` to deliver it\n", "alone.\n the control plane stored the box without opening it; run `push --behind` to deliver it\n",
"the manager", name) "the manager", name)
return nil return nil
} }
@@ -423,7 +423,7 @@ func licenceSubmitRefresh(ctx context.Context, args []string) error {
"manager node alone" "manager node alone"
} }
fmt.Printf("submitted a refresh for %s: a new access token sealed to %d holder(s), and %s.\n"+ fmt.Printf("submitted a refresh for %s: a new access token sealed to %d holder(s), and %s.\n"+
" run `push` to deliver it\n", name, sealed, rotatedNote) " run `push --behind` to deliver it\n", name, sealed, rotatedNote)
return nil return nil
} }
@@ -456,7 +456,7 @@ func licenceRefresh(ctx context.Context, args []string) error {
return err return err
} }
fmt.Printf("refreshed %s: a new access token sealed to %d holder(s), and the refresh token left "+ fmt.Printf("refreshed %s: a new access token sealed to %d holder(s), and the refresh token left "+
"with its manager.\n run `push` to deliver it\n", name, sealed) "with its manager.\n run `push --behind` to deliver it\n", name, sealed)
return nil return nil
} }
+83 -11
View File
@@ -8,7 +8,6 @@ import (
"fmt" "fmt"
"net" "net"
"os" "os"
"sort"
"strings" "strings"
"github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/broker"
@@ -383,7 +382,8 @@ func assignCommand(ctx context.Context, verb string, args []string) error {
func settingsCommand(ctx context.Context, args []string) error { func settingsCommand(ctx context.Context, args []string) error {
if len(args) == 0 { if len(args) == 0 {
return errors.New("settings set <module> <file> [--node <node>], or settings clear <module> [--node <node>]") return errors.New("settings show <module> [--node <node>] [--history], settings set <module> <file> " +
"[--node <node>] [--replace], or settings clear <module> [--node <node>]")
} }
open, err := openStores(ctx) open, err := openStores(ctx)
if err != nil { if err != nil {
@@ -394,6 +394,11 @@ func settingsCommand(ctx context.Context, args []string) error {
set := flag.NewFlagSet("settings", flag.ContinueOnError) set := flag.NewFlagSet("settings", flag.ContinueOnError)
node := set.String("node", "", "one machine, rather than the whole mesh") node := set.String("node", "", "one machine, rather than the whole mesh")
// **What a set removes is refused unless meant** (novox/hq ADR 0217). A layer is replaced whole,
// and on 2026-10-05 setting one placement dropped a machine's whole layer for a module without a
// word (novox/hq issue 304). Adding and changing keys needs nothing; removing one needs this.
replace := set.Bool("replace", false, "for set: remove the keys the new layer does not name")
history := set.Bool("history", false, "for show: the layers this one replaced, the latest first")
positionals, err := parseAround(set, args[1:]) positionals, err := parseAround(set, args[1:])
if err != nil { if err != nil {
return err return err
@@ -421,17 +426,76 @@ func settingsCommand(ctx context.Context, args []string) error {
if err := json.Unmarshal(raw, &values); err != nil { if err := json.Unmarshal(raw, &values); err != nil {
return fmt.Errorf("%s is not a settings file: %w", positionals[1], err) return fmt.Errorf("%s is not a settings file: %w", positionals[1], err)
} }
before, _, err := inv.Layer(ctx, *node, positionals[0])
if err != nil {
return err
}
added, changed, removed := settingsChange(before, values)
if len(removed) > 0 && !*replace {
return fmt.Errorf("%s on %s: this layer would no longer set %s. A layer is replaced whole; "+
"read it with `settings show %s%s` and include what should stay, or add --replace if the "+
"removal is meant (novox/hq ADR 0217). Nothing was changed",
positionals[0], where, strings.Join(removed, ", "), positionals[0], nodeFlag(*node))
}
if err := inv.SetSettings(ctx, *node, positionals[0], values); err != nil { if err := inv.SetSettings(ctx, *node, positionals[0], values); err != nil {
return err return err
} }
fmt.Printf("%s on %s:\n", positionals[0], where)
var keys []string if len(added)+len(changed)+len(removed) == 0 {
for k := range values { fmt.Println(" nothing changed")
keys = append(keys, k)
} }
sort.Strings(keys) for _, p := range added {
fmt.Printf("%s on %s: %s\n", positionals[0], where, strings.Join(keys, ", ")) fmt.Printf(" + %s\n", p)
fmt.Println(" run `push` to send it") }
for _, p := range changed {
fmt.Printf(" ~ %s\n", p)
}
for _, p := range removed {
fmt.Printf(" - %s\n", p)
}
if before != nil {
fmt.Printf(" the layer it replaced is kept: settings show %s%s --history\n", positionals[0], nodeFlag(*node))
}
if *node != "" {
fmt.Printf(" run `plan %s --diff` to see what it changes, `push %s` to send it\n", *node, *node)
} else {
fmt.Println(" run `push --behind` to send it to the machines running it")
}
return nil
case "show":
if len(positionals) != 1 {
return errors.New("settings show <module> [--node <node>] [--history]")
}
if *history {
past, err := inv.SettingsHistory(ctx, *node, positionals[0])
if err != nil {
return err
}
if len(past) == 0 {
fmt.Printf("%s on %s: no layer has been replaced\n", positionals[0], where)
return nil
}
for _, p := range past {
shown, _ := json.MarshalIndent(p.Values, " ", " ")
fmt.Printf("%s on %s, until %s (%s):\n %s\n", positionals[0], where,
p.ReplacedAt.Local().Format("2006-01-02 15:04:05"), p.ReplacedBy, shown)
}
return nil
}
values, has, err := inv.Layer(ctx, *node, positionals[0])
if err != nil {
return err
}
if !has {
fmt.Printf("%s on %s: no layer — the module's definition says\n", positionals[0], where)
return nil
}
shown, err := json.MarshalIndent(values, "", " ")
if err != nil {
return err
}
fmt.Println(string(shown))
return nil return nil
case "clear": case "clear":
@@ -445,10 +509,18 @@ func settingsCommand(ctx context.Context, args []string) error {
return nil return nil
default: default:
return fmt.Errorf("settings has no %q; it has set and clear", args[0]) return fmt.Errorf("settings has no %q; it has show, set and clear", args[0])
} }
} }
// nodeFlag is ` --node <node>` for a machine's layer, nothing for the whole mesh's.
func nodeFlag(node string) string {
if node == "" {
return ""
}
return " --node " + node
}
// describeOffers says what a module provides, and marks the ones answered from anywhere in the // describeOffers says what a module provides, and marks the ones answered from anywhere in the
// mesh — because "provides a database" and "provides a shell" are read the same way and mean // mesh — because "provides a database" and "provides a shell" are read the same way and mean
// entirely different things about where the answer has to be. // entirely different things about where the answer has to be.
@@ -683,7 +755,7 @@ func issueWith(ctx context.Context, inv *inventory.Inventory, m catalogue.Manife
Emits: m.EmitsAll(), Consumes: m.Consumes, Serves: m.Tools, Emits: m.EmitsAll(), Consumes: m.Consumes, Serves: m.Tools,
}); needed { }); needed {
if busAddress == "" { if busAddress == "" {
fmt.Printf(" %s consumes; its consumer is created when the bus is reachable (`push`, then "+ fmt.Printf(" %s consumes; its consumer is created when the bus is reachable (`push --behind`, then "+
"`rollout mint` again is harmless)\n", m.Module) "`rollout mint` again is harmless)\n", m.Module)
} else { } else {
js, err := broker.Dial(busAddress) js, err := broker.Dial(busAddress)
+15 -1
View File
@@ -1185,12 +1185,15 @@ func planCommand(ctx context.Context, args []string) error {
// checking it against the host's own parser, most usefully, which is the only way to know // checking it against the host's own parser, most usefully, which is the only way to know
// that what the control plane emits is what the host accepts. // that what the control plane emits is what the host accepts.
asJSON := set.Bool("json", false, "print the declaration this node would be sent") asJSON := set.Bool("json", false, "print the declaration this node would be sent")
// What a push would change, against what the machine was last sent (novox/hq ADR 0217): read
// before sending, so a change that removes, moves or recreates something is seen first.
asDiff := set.Bool("diff", false, "what a push would change on this node, against what it was last sent")
positionals, err := parseAround(set, args) positionals, err := parseAround(set, args)
if err != nil { if err != nil {
return err return err
} }
if len(positionals) != 1 { if len(positionals) != 1 {
return errors.New("plan <node> [--files] [--json]") return errors.New("plan <node> [--files] [--json] [--diff]")
} }
args = positionals args = positionals
open, err := openStores(ctx) open, err := openStores(ctx)
@@ -1207,6 +1210,17 @@ func planCommand(ctx context.Context, args []string) error {
fmt.Printf("%s is assigned nothing\n", args[0]) fmt.Printf("%s is assigned nothing\n", args[0])
return nil return nil
} }
if *asDiff {
declared, err := declarationFor(ctx, open, args[0], plan, settings)
if err != nil {
return err
}
body, err := declared.Body()
if err != nil {
return err
}
return writePlanDiff(ctx, open.inventory, args[0], body)
}
if *asJSON { if *asJSON {
declared, err := declarationFor(ctx, open, args[0], plan, settings) declared, err := declarationFor(ctx, open, args[0], plan, settings)
if err != nil { if err != nil {
+91 -8
View File
@@ -359,14 +359,31 @@ func pushCommand(ctx context.Context, args []string) error {
// A push by hand is a repair, and says why (novox/hq to-be 45 §7): required through the seat, // A push by hand is a repair, and says why (novox/hq to-be 45 §7): required through the seat,
// recorded when given at a shell — see handacts.go for why a shell is not refused. // recorded when given at a shell — see handacts.go for why a shell is not refused.
why := addHandActFlags(set) why := addHandActFlags(set)
// Every machine at once is said, not defaulted to (novox/hq ADR 0217): a bare `push` sent the
// whole mesh on 2026-10-05 when its usage was wanted.
all := set.Bool("all", false, "every machine")
// A running module's data moving is sent only when said, per module (novox/hq ADR 0217).
move := set.String("move", "", "modules whose data may move with this push, comma-separated")
positionals, err := parseAround(set, args) positionals, err := parseAround(set, args)
if err != nil { if err != nil {
return err return err
} }
args = positionals args = positionals
if len(args) > 1 { if len(args) > 1 {
return errors.New("push [<node>] [--behind] — one node, or all of them") return errors.New("push <node> | push --behind | push --all — one machine, the ones behind, or all of them")
} }
if len(args) == 0 && !*behind && !*all {
return errors.New("push names a machine, or says --behind (the ones not running what they " +
"should) or --all (every machine) — a push to the whole mesh is not the default (novox/hq ADR 0217)")
}
if *all && (*behind || len(args) == 1) {
return errors.New("push --all is every machine; it takes no machine and no --behind")
}
movable := map[string]bool{}
for _, m := range splitModules(*move) {
movable[m] = true
}
var heldBack []string
if len(args) == 1 && *behind { if len(args) == 1 && *behind {
// Naming a machine and asking for the ones that need it are two different requests, and // Naming a machine and asking for the ones that need it are two different requests, and
// guessing which was meant would sometimes push to a machine somebody did not name. // guessing which was meant would sometimes push to a machine somebody did not name.
@@ -377,6 +394,12 @@ func pushCommand(ctx context.Context, args []string) error {
if *behind { if *behind {
recorded = append(recorded, "--behind") recorded = append(recorded, "--behind")
} }
if *all {
recorded = append(recorded, "--all")
}
if *move != "" {
recorded = append(recorded, "--move", *move)
}
why.record(ctx, "push", recorded) why.record(ctx, "push", recorded)
open, err := openStores(ctx) open, err := openStores(ctx)
if err != nil { if err != nil {
@@ -587,22 +610,28 @@ func pushCommand(ctx context.Context, args []string) error {
}) })
defer release() defer release()
// A machine whose declaration would move a running module's data is held, and only it (novox/hq
// ADR 0217): said, with the command that sends it, and left out of everything below.
toSend, err := holdingBack(ctx, inv, sending, movable, &heldBack)
if err != nil {
return err
}
// **What it recreates, said before it is sent** (novox/hq ADR 0245): a person's push moves every // **What it recreates, said before it is sent** (novox/hq ADR 0245): a person's push moves every
// module whose build changed, and recreates what changed in it — a gated send said so, a push did not. // module whose build changed, and recreates what changed in it — a gated send said so, a push did not.
sayWhatAPushRecreates(ctx, open, sending, os.Stdout) sayWhatAPushRecreates(ctx, open, toSend, os.Stdout)
// Each machine's memberships first, then the declarations (novox/hq issue 249, ADR 0160): a push // Each machine's memberships first, then the declarations (novox/hq issue 249, ADR 0160): a push
// is the one most operators run, and on 2026-10-01 it was the one path that issued none. // is the one most operators run, and on 2026-10-01 it was the one path that issued none.
bus := overTheBus{open: open, server: server, signer: ident} bus := overTheBus{open: open, server: server, signer: ident}
sentDigest, err := deliver(ctx, bus, holder, sending) sentDigest, err := deliver(ctx, bus, holder, toSend)
if err != nil { if err != nil {
return err return err
} }
release() release()
told := make([]string, 0, len(sending)) told := make([]string, 0, len(toSend))
for _, r := range sending { for _, r := range toSend {
told = append(told, r.node) told = append(told, r.node)
} }
fmt.Printf("\n%d node(s) told: %s\n", len(sending), strings.Join(told, ", ")) fmt.Printf("\n%d node(s) told: %s\n", len(toSend), strings.Join(told, ", "))
reportUnheldPushed(os.Stdout, len(args) == 1, asked, unheld) reportUnheldPushed(os.Stdout, len(args) == 1, asked, unheld)
// **A named push leaves the mesh consistent, not just the machine it named** (novox/hq // **A named push leaves the mesh consistent, not just the machine it named** (novox/hq
@@ -624,7 +653,12 @@ func pushCommand(ctx context.Context, args []string) error {
for n := range saidHeld { for n := range saidHeld {
handled[n] = true handled[n] = true
} }
refused, err := flushBehind(ctx, open, nodes, handled, composeForPush(open, gens), bus, holder, os.Stdout) // The cascade is a push too, and a machine in it whose data would move is held the same way
// (novox/hq ADR 0217).
keep := keeping(func(held context.Context, sending []readyNode) ([]readyNode, error) {
return holdingBack(held, inv, sending, movable, &heldBack)
})
refused, err := flushBehind(ctx, open, nodes, handled, composeForPush(open, gens), bus, holder, keep, os.Stdout)
refusals = append(refusals, refused...) refusals = append(refusals, refused...)
if err != nil { if err != nil {
return err return err
@@ -639,6 +673,11 @@ func pushCommand(ctx context.Context, args []string) error {
return err return err
} }
} }
if len(heldBack) > 0 {
sort.Strings(heldBack)
return fmt.Errorf("held %s: a running module's data would move — see above; nothing was sent "+
"there (novox/hq ADR 0217)", strings.Join(heldBack, ", "))
}
return couldNotBeResolved(refusals, len(sending)) return couldNotBeResolved(refusals, len(sending))
} }
@@ -747,7 +786,7 @@ func composeEach(names []string, allot func(node string) (order, error),
// still sent. Their memberships go before their declarations, as every send's do (issue 249). // still sent. Their memberships go before their declarations, as every send's do (issue 249).
func sendRound(ctx context.Context, open *stores, names []string, func sendRound(ctx context.Context, open *stores, names []string,
compose func(held context.Context, node string) (sendable, error), compose func(held context.Context, node string) (sendable, error),
d delivery, holder string) ([]string, error) { d delivery, holder string, keep keeping) ([]string, error) {
held, release, err := holdNodes(ctx, open, names) held, release, err := holdNodes(ctx, open, names)
if err != nil { if err != nil {
return nil, err return nil, err
@@ -756,12 +795,46 @@ func sendRound(ctx context.Context, open *stores, names []string,
sending, refused := composeEach(names, allotting(held, open.inventory), func(node string) (sendable, error) { sending, refused := composeEach(names, allotting(held, open.inventory), func(node string) (sendable, error) {
return compose(held, node) return compose(held, node)
}) })
if keep != nil {
if sending, err = keep(held, sending); err != nil {
return refused, err
}
}
if _, err := deliver(held, d, holder, sending); err != nil { if _, err := deliver(held, d, holder, sending); err != nil {
return refused, err return refused, err
} }
return refused, nil return refused, nil
} }
// keeping is what a round of sends keeps of the machines it composed, before any is delivered: nil
// keeps them all. A push holds back the machines whose running modules' data would move (holdingBack).
type keeping func(held context.Context, sending []readyNode) ([]readyNode, error)
// holdingBack leaves out each machine whose declaration would move a running module's data, says so,
// and names it among those held back (novox/hq ADR 0217); the rest go on to be delivered, grants first
// (issue 249). A declaration's body is the same bytes however often it is read.
func holdingBack(ctx context.Context, inv *inventory.Inventory, sending []readyNode, movable map[string]bool,
heldBack *[]string) ([]readyNode, error) {
var out []readyNode
for _, s := range sending {
body, err := s.declared.Body()
if err != nil {
return nil, err
}
moves, err := heldMoves(ctx, inv, s.node, body, movable)
if err != nil {
return nil, err
}
if len(moves) > 0 {
sayDataHeld(os.Stdout, s.node, moves)
*heldBack = append(*heldBack, s.node)
continue
}
out = append(out, s)
}
return out, nil
}
// delivery is the two acts of sending machines what they should be, apart, so the order between // delivery is the two acts of sending machines what they should be, apart, so the order between
// them is one function's and can be read and tested there (novox/hq issue 249). // them is one function's and can be read and tested there (novox/hq issue 249).
type delivery interface { type delivery interface {
@@ -1473,6 +1546,16 @@ func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body
if err := inv.RecordSentUnder(kept, record.ID, digest, builds, epoch); err != nil { if err := inv.RecordSentUnder(kept, record.ID, digest, builds, epoch); err != nil {
return "", err return "", err
} }
// And what it was, summarised, so the next push can be compared with it (novox/hq ADR 0217).
// Never a reason to fail a send that is already away: a summary that cannot be kept means the
// next comparison has nothing to hold against, which is how every machine starts.
if summary, err := summarize(body); err == nil {
if raw, err := json.Marshal(summary); err == nil {
if err := inv.RecordSentSummary(kept, record.ID, raw); err != nil {
fmt.Fprintf(os.Stderr, "%s: what it was sent is not kept for comparison: %v\n", node, err)
}
}
}
return digest, nil return digest, nil
} }
+26 -8
View File
@@ -407,6 +407,10 @@ func (a *verbArguments) commandLine() ([]string, error) {
if on("files") { if on("files") {
return []string{"plan", str("node"), "--files"}, nil return []string{"plan", str("node"), "--files"}, nil
} }
// What a push would change there (novox/hq ADR 0217); diff with files is passed over, and so refused.
if on("diff") {
return []string{"plan", str("node"), "--diff"}, nil
}
return []string{"plan", str("node"), "--json"}, nil return []string{"plan", str("node"), "--json"}, nil
case "assign", "unassign": case "assign", "unassign":
if err := need("node", "module"); err != nil { if err := need("node", "module"); err != nil {
@@ -441,10 +445,16 @@ func (a *verbArguments) commandLine() ([]string, error) {
// behind is not read here: given with a machine, it is refused as passed over — naming // behind is not read here: given with a machine, it is refused as passed over — naming
// a machine and asking for every machine behind are two requests, and guessing one // a machine and asking for every machine behind are two requests, and guessing one
// would push a machine nobody named, or not push one somebody did. // would push a machine nobody named, or not push one somebody did.
return append([]string{"push", n, "--wait", "0"}, why...), nil argv := []string{"push", n, "--wait", "0"}
// A running module's data moving is sent only when said, per module (novox/hq ADR 0217).
if m := str("move"); m != "" {
argv = append(argv, "--move", m)
}
return append(argv, why...), nil
} }
// No machine: the whole mesh, whether or not behind said so. The command's answer says it // No machine: the whole mesh, whether or not behind said so. The command's answer says it
// first, so a caller who meant one machine reads that it was not one. // first, so a caller who meant one machine reads that it was not one. move is not read: a
// machine whose data would move is held and named, and sent by a push that names it.
on("behind") on("behind")
return append([]string{"push", "--behind", "--wait", "0"}, why...), nil return append([]string{"push", "--behind", "--wait", "0"}, why...), nil
case "hand-act": case "hand-act":
@@ -672,13 +682,21 @@ func (a *verbArguments) commandLine() ([]string, error) {
return nil, err return nil, err
} }
var argv []string var argv []string
if on("clear") { switch {
case on("clear"):
argv = []string{"settings", "clear", str("module")} argv = []string{"settings", "clear", str("module")}
} else { case str("values") != "":
argv = []string{"settings", "set", str("module")} argv = []string{"settings", "set", str("module"), str("values")}
// Neither values nor clear: the command says its usage, which names both. // What a set removes is refused unless meant (novox/hq ADR 0217).
if v := str("values"); v != "" { if on("replace") {
argv = append(argv, v) argv = append(argv, "--replace")
}
default:
// Neither values nor clear: the layer as it stands, which is what a caller reads before
// replacing it (novox/hq ADR 0217) — and with history, the layers it replaced.
argv = []string{"settings", "show", str("module")}
if on("history") {
argv = append(argv, "--history")
} }
} }
if n := str("node"); n != "" { if n := str("node"); n != "" {
+5 -1
View File
@@ -258,7 +258,11 @@ var accountedFlags = map[string]map[string]string{
"seats": {"json": "set by the verb: the answer is data"}, "seats": {"json": "set by the verb: the answer is data"},
"queue": {"json": "not set: the verb answers the table a person reads"}, "queue": {"json": "not set: the verb answers the table a person reads"},
"plan": {"json": "set by the verb unless files is asked"}, "plan": {"json": "set by the verb unless files is asked"},
"push": {"wait": "set by the verb to 0: a tool call cannot hold a connection for a whole apply"}, "push": {
"wait": "set by the verb to 0: a tool call cannot hold a connection for a whole apply",
"all": "withheld: the verb naming no machine is --behind, and a push of every machine is said at a " +
"shell (novox/hq ADR 0217); `command` reaches it",
},
"build": { "build": {
"wait": "set by the verb to 0: the id follows the build (issue 176)", "wait": "set by the verb to 0: the id follows the build (issue 176)",
"self": "set by the verb from the repository's form: a path on the forge, or a URL", "self": "set by the verb from the repository's form: a path on the forge, or a URL",
+4 -2
View File
@@ -72,9 +72,11 @@ func TestSettingsSetsOrClearsALayer(t *testing.T) {
if strings.Join(argv, " ") != "settings clear dnsmasq" { if strings.Join(argv, " ") != "settings clear dnsmasq" {
t.Fatalf("clear for the mesh: %v", argv) t.Fatalf("clear for the mesh: %v", argv)
} }
// Neither values nor clear reads the layer as it stands (novox/hq ADR 0217): what a caller reads
// before replacing it, where it used to fall to the command's usage.
argv, _ = argvFor("settings", map[string]any{"module": "dnsmasq"}) argv, _ = argvFor("settings", map[string]any{"module": "dnsmasq"})
if strings.Join(argv, " ") != "settings set dnsmasq" { if strings.Join(argv, " ") != "settings show dnsmasq" {
t.Fatalf("a set with no values falls to the command's usage: %v", argv) t.Fatalf("a call with no values reads the layer: %v", argv)
} }
} }
+338
View File
@@ -0,0 +1,338 @@
package main
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"os"
"reflect"
"sort"
"strings"
"github.com/novox/mesh-controller/internal/inventory"
)
// No change to a machine takes effect unseen (novox/hq ADR 0217, to-be 44).
//
// Two incidents in two days had one shape: a change took effect that nobody saw before it did. A
// provisioner read an unreadable file as "nobody asks" and dropped seven databases (issue 241); one
// placement set for one module on one machine replaced its whole settings layer, and the plan then
// ran the module on an empty directory (novox/hq issue 304). Here are the three guards: what a settings layer
// loses is said and refused unless meant; what a push will change can be read before it is sent; and
// a running container's data moving holds that machine's push until the operator says so. Each is
// silent when nothing is at stake, so an ordinary change goes exactly as before.
// ---- 1. what a settings layer loses ------------------------------------------------------------
// settingsChange is what replacing one layer with another adds, changes and removes, by dotted path
// to each leaf — `places.config`, not only `places` — because a layer that keeps a key and loses one
// of its entries has lost something just the same: on 2026-10-05 a node's `places` kept its name and
// lost three of its four directories.
func settingsChange(before, after map[string]any) (added, changed, removed []string) {
was, now := leaves(before, ""), leaves(after, "")
for path, v := range now {
old, had := was[path]
switch {
case !had:
added = append(added, path)
case !reflect.DeepEqual(old, v):
changed = append(changed, path)
}
}
for path := range was {
if _, kept := now[path]; !kept {
removed = append(removed, path)
}
}
sort.Strings(added)
sort.Strings(changed)
sort.Strings(removed)
return added, changed, removed
}
// leaves flattens a settings layer to its leaves by dotted path; a list is a leaf, compared whole.
func leaves(m map[string]any, prefix string) map[string]any {
out := map[string]any{}
for k, v := range m {
path := k
if prefix != "" {
path = prefix + "." + k
}
if inner, ok := v.(map[string]any); ok && len(inner) > 0 {
for p, leaf := range leaves(inner, path) {
out[p] = leaf
}
continue
}
out[path] = v
}
return out
}
// ---- 2. what a push will change -----------------------------------------------------------------
// sentResource is what is kept of one resource a machine was sent: enough to say what changed and
// whether a container's data moved, and nothing that could carry a secret — a file's content is
// kept as a digest, never as text (the record lands in the store's own backups).
type sentResource struct {
ID string `json:"id"`
Type string `json:"type"`
// Digest is of the whole resource as it was sent.
Digest string `json:"digest"`
// Fields is each field's digest, so a change can be named by field.
Fields map[string]string `json:"fields"`
// Volumes is a container's mounts as sent — paths, which are not secrets, and what a moved
// data directory is read from.
Volumes []string `json:"volumes,omitempty"`
}
// summarize is what is kept of a declaration body: its resources, in the order sent.
func summarize(body []byte) ([]sentResource, error) {
var envelope struct {
Resources []map[string]any `json:"resources"`
}
if err := json.Unmarshal(body, &envelope); err != nil {
return nil, fmt.Errorf("a declaration that is not one: %w", err)
}
out := make([]sentResource, 0, len(envelope.Resources))
for _, r := range envelope.Resources {
s := sentResource{ID: fmt.Sprint(r["id"]), Type: fmt.Sprint(r["type"]), Digest: digestValue(r),
Fields: map[string]string{}}
for k, v := range r {
s.Fields[k] = digestValue(v)
}
if s.Type == "container" {
if vols, ok := r["volumes"].([]any); ok {
for _, v := range vols {
s.Volumes = append(s.Volumes, fmt.Sprint(v))
}
}
}
out = append(out, s)
}
return out, nil
}
func digestValue(v any) string {
raw, _ := json.Marshal(v)
sum := sha256.Sum256(raw)
return hex.EncodeToString(sum[:8])
}
// changedResource is one resource sent differently, with the fields that differ.
type changedResource struct {
ID, Type string
Fields []string
}
// sentDiff is what would be sent now against what was sent last, by resource id.
type sentDiff struct {
Added, Removed []string
Changed []changedResource
}
func (d sentDiff) empty() bool {
return len(d.Added) == 0 && len(d.Removed) == 0 && len(d.Changed) == 0
}
func diffSent(before, after []sentResource) sentDiff {
was := map[string]sentResource{}
for _, r := range before {
was[r.ID] = r
}
var d sentDiff
seen := map[string]bool{}
for _, r := range after {
seen[r.ID] = true
old, had := was[r.ID]
if !had {
d.Added = append(d.Added, r.ID)
continue
}
if old.Digest == r.Digest {
continue
}
c := changedResource{ID: r.ID, Type: r.Type}
for k, v := range r.Fields {
if old.Fields[k] != v {
c.Fields = append(c.Fields, k)
}
}
for k := range old.Fields {
if _, kept := r.Fields[k]; !kept {
c.Fields = append(c.Fields, k)
}
}
sort.Strings(c.Fields)
d.Changed = append(d.Changed, c)
}
for _, r := range before {
if !seen[r.ID] {
d.Removed = append(d.Removed, r.ID)
}
}
sort.Strings(d.Added)
sort.Strings(d.Removed)
sort.Slice(d.Changed, func(a, b int) bool { return d.Changed[a].ID < d.Changed[b].ID })
return d
}
// writeDiff says a diff the way a person reads one: what is added, removed and changed, and a
// changed container's fields — a container sent differently is a container recreated.
func writeDiff(w io.Writer, node string, d sentDiff, moves []dataMove) {
if d.empty() {
fmt.Fprintf(w, "%s: nothing would change — it was last sent exactly this\n", node)
return
}
fmt.Fprintf(w, "%s would change:\n", node)
for _, id := range d.Added {
fmt.Fprintf(w, " + %s\n", id)
}
for _, id := range d.Removed {
fmt.Fprintf(w, " - %s\n", id)
}
for _, c := range d.Changed {
what := "changed"
if c.Type == "container" {
what = "recreated"
}
fmt.Fprintf(w, " ~ %s %s: %s\n", c.ID, what, strings.Join(c.Fields, ", "))
}
writeMoves(w, node, moves)
}
// ---- 3. a running module's data moving ------------------------------------------------------------
// dataMove is a container keeping a mount at the same place inside it with a different directory
// on the machine behind it: its data moving — or, as on 2026-10-05, a module about to start on an
// empty directory because the placement that pointed at its data was lost.
type dataMove struct {
Container, Inside, From, To string
}
// Module is the module the container belongs to: resource ids are `<module>.<id>`.
func (m dataMove) Module() string {
module, _, _ := strings.Cut(m.Container, ".")
return module
}
// movesIn is every data move between what a machine was sent and what it would be sent. A new
// container, a mount added or dropped and a changed image are not moves.
func movesIn(before, after []sentResource) []dataMove {
was := map[string]sentResource{}
for _, r := range before {
if r.Type == "container" {
was[r.ID] = r
}
}
var out []dataMove
for _, r := range after {
old, had := was[r.ID]
if r.Type != "container" || !had {
continue
}
from := mountSources(old.Volumes)
for inside, to := range mountSources(r.Volumes) {
if prev, mounted := from[inside]; mounted && prev != to {
out = append(out, dataMove{Container: r.ID, Inside: inside, From: prev, To: to})
}
}
}
sort.Slice(out, func(a, b int) bool {
if out[a].Container != out[b].Container {
return out[a].Container < out[b].Container
}
return out[a].Inside < out[b].Inside
})
return out
}
// mountSources is a container's mounts by the place inside it: `source:inside[:options]`.
func mountSources(volumes []string) map[string]string {
out := map[string]string{}
for _, v := range volumes {
parts := strings.SplitN(v, ":", 3)
if len(parts) < 2 {
continue
}
out[parts[1]] = parts[0]
}
return out
}
func writeMoves(w io.Writer, node string, moves []dataMove) {
for _, m := range moves {
fmt.Fprintf(w, " ! %s: %s moves from %s to %s\n", m.Container, m.Inside, m.From, m.To)
}
}
// heldMoves is the moves in a push to one machine that the operator has not said to send (`--move
// <module>`); empty when there is nothing to hold, including a machine never sent anything before.
func heldMoves(ctx context.Context, inv *inventory.Inventory, node string, body []byte, allowed map[string]bool) ([]dataMove, error) {
prev, err := inv.SentSummary(ctx, node)
if err != nil || len(prev) == 0 {
return nil, err
}
var before []sentResource
if err := json.Unmarshal(prev, &before); err != nil {
// A record this version cannot read compares with nothing: holding every push for it would
// stop the mesh on a format, which is not what is at stake. Said, so it is not passed over.
fmt.Fprintf(os.Stderr, "%s: what it was last sent is kept in a form this version does not read, "+
"so whether its data would move is not known: %v\n", node, err)
// empty-on-error: said above; the send replaces the unreadable record with one this version reads
return nil, nil
}
after, err := summarize(body)
if err != nil {
return nil, err
}
var held []dataMove
for _, m := range movesIn(before, after) {
if !allowed[m.Module()] {
held = append(held, m)
}
}
return held, nil
}
// sayDataHeld tells the operator why a machine was not sent, and how to send it.
func sayDataHeld(w io.Writer, node string, moves []dataMove) {
modules := map[string]bool{}
for _, m := range moves {
modules[m.Module()] = true
}
var names []string
for m := range modules {
names = append(names, m)
}
sort.Strings(names)
fmt.Fprintf(w, "held %s: a running module's data would move (novox/hq ADR 0217)\n", node)
writeMoves(w, node, moves)
fmt.Fprintf(w, " if that is meant: push %s --move %s\n", node, strings.Join(names, ","))
}
// writePlanDiff says what sending body to a machine would change against what it was last sent.
func writePlanDiff(ctx context.Context, inv *inventory.Inventory, node string, body []byte) error {
after, err := summarize(body)
if err != nil {
return err
}
prev, err := inv.SentSummary(ctx, node)
if err != nil {
return err
}
if len(prev) == 0 {
fmt.Printf("%s: what it was last sent is not kept yet — the first push from this version keeps "+
"it; %d resource(s) would be sent\n", node, len(after))
return nil
}
var before []sentResource
if err := json.Unmarshal(prev, &before); err != nil {
return fmt.Errorf("%s: what it was last sent is kept in a form this version does not read: %w", node, err)
}
writeDiff(os.Stdout, node, diffSent(before, after), movesIn(before, after))
return nil
}
+190
View File
@@ -0,0 +1,190 @@
package main
import (
"bytes"
"context"
"reflect"
"strings"
"testing"
)
// novox/hq ADR 0217: no change to a machine takes effect unseen.
// The layer of 2026-10-05: a media server's node layer, and the one that replaced it, which kept
// `places` and lost three of its four directories and every other key.
var before = map[string]any{
"puid": 1000.0, "pgid": 1000.0,
"expose": map[string]any{"32400": "anywhere"},
"places": map[string]any{
"config": map[string]any{"path": "/srv/media/config", "owner": "1000:1000"},
"data": map[string]any{"path": "/srv/media/data", "owner": "1000:1000"},
"transcode": map[string]any{"path": "/srv/media/temp", "owner": "1000:1000"},
},
}
func TestASettingThatKeepsAKeyAndLosesItsEntriesIsSaidToRemoveThem(t *testing.T) {
after := map[string]any{"places": map[string]any{
"previews": map[string]any{"path": "/srv/pool/previews", "owner": "1000:1000"},
}}
added, changed, removed := settingsChange(before, after)
if !reflect.DeepEqual(added, []string{"places.previews.owner", "places.previews.path"}) {
t.Errorf("added %v", added)
}
if len(changed) != 0 {
t.Errorf("changed %v", changed)
}
for _, lost := range []string{"expose.32400", "pgid", "places.config.path", "places.data.owner", "puid"} {
found := false
for _, r := range removed {
found = found || r == lost
}
if !found {
t.Errorf("removing %s was not said: %v", lost, removed)
}
}
}
func TestASettingThatOnlyAddsOrChangesRemovesNothing(t *testing.T) {
after := map[string]any{
"puid": 1001.0, "pgid": 1000.0,
"expose": map[string]any{"32400": "anywhere"},
"places": map[string]any{
"config": map[string]any{"path": "/srv/media/config", "owner": "1000:1000"},
"data": map[string]any{"path": "/srv/media/data", "owner": "1000:1000"},
"transcode": map[string]any{"path": "/srv/media/temp", "owner": "1000:1000"},
"previews": map[string]any{"path": "/srv/pool/previews", "owner": "1000:1000"},
},
}
_, changed, removed := settingsChange(before, after)
if len(removed) != 0 {
t.Errorf("an additive set was said to remove %v", removed)
}
if !reflect.DeepEqual(changed, []string{"puid"}) {
t.Errorf("changed %v", changed)
}
}
// What was sent, as a push would send it: a media server's container and a file with a secret.
func declaration(configFrom string, extra ...string) []byte {
vols := `"` + configFrom + `:/config", "/srv/media/data:/data"`
for _, e := range extra {
vols += `, "` + e + `"`
}
return []byte(`{"declaration":1,"sequence":7,"resources":[
{"id":"plex.server","type":"container","image":"plex@sha256:aa","volumes":[` + vols + `]},
{"id":"plex.env","type":"file","path":"/srv/media/env","content":"TOKEN=the-secret-itself"}]}`)
}
func summarized(t *testing.T, body []byte) []sentResource {
t.Helper()
s, err := summarize(body)
if err != nil {
t.Fatal(err)
}
return s
}
func TestWhatIsKeptOfASendHoldsNoFileContent(t *testing.T) {
s := summarized(t, declaration("/srv/media/config"))
var kept bytes.Buffer
for _, r := range s {
kept.WriteString(r.ID + r.Type + r.Digest + strings.Join(r.Volumes, ","))
for k, v := range r.Fields {
kept.WriteString(k + v)
}
}
if strings.Contains(kept.String(), "the-secret-itself") {
t.Fatal("a file's content was kept in the record of what was sent")
}
if len(s) != 2 || s[0].Volumes[0] != "/srv/media/config:/config" {
t.Fatalf("summary %+v", s)
}
}
func TestADiffOfAnUnchangedMachineIsEmptyAndAChangedContainerNamesItsField(t *testing.T) {
was := summarized(t, declaration("/srv/media/config"))
if d := diffSent(was, summarized(t, declaration("/srv/media/config"))); !d.empty() {
t.Fatalf("an unchanged machine differs: %+v", d)
}
d := diffSent(was, summarized(t, declaration("/var/lib/plex/config")))
if len(d.Changed) != 1 || d.Changed[0].ID != "plex.server" || !reflect.DeepEqual(d.Changed[0].Fields, []string{"volumes"}) {
t.Fatalf("diff %+v", d)
}
var out bytes.Buffer
writeDiff(&out, "home", d, movesIn(was, summarized(t, declaration("/var/lib/plex/config"))))
for _, want := range []string{"~ plex.server recreated: volumes", "! plex.server: /config moves from /srv/media/config to /var/lib/plex/config"} {
if !strings.Contains(out.String(), want) {
t.Errorf("diff does not say %q:\n%s", want, out.String())
}
}
}
// The incident: the configuration mount would move to an empty default directory.
func TestARunningContainersDataMovingIsAMoveAndAnAddedMountIsNot(t *testing.T) {
was := summarized(t, declaration("/srv/media/config"))
moves := movesIn(was, summarized(t, declaration("/var/lib/plex/config")))
want := []dataMove{{Container: "plex.server", Inside: "/config", From: "/srv/media/config", To: "/var/lib/plex/config"}}
if !reflect.DeepEqual(moves, want) {
t.Fatalf("moves %+v", moves)
}
if moves[0].Module() != "plex" {
t.Errorf("module %q", moves[0].Module())
}
// A mount added — the previews of 2026-10-05 — is not a move.
if m := movesIn(was, summarized(t, declaration("/srv/media/config", "/srv/pool/previews:/config/Media"))); len(m) != 0 {
t.Errorf("an added mount was held as a move: %+v", m)
}
// Nor is a container the machine never ran.
if m := movesIn(nil, was); len(m) != 0 {
t.Errorf("a first send was held as a move: %+v", m)
}
}
func TestAPushNamingNoMachineIsRefusedUnlessItSaysAll(t *testing.T) {
err := pushCommand(context.Background(), nil)
if err == nil || !strings.Contains(err.Error(), "--all") {
t.Fatalf("a bare push was not refused: %v", err)
}
if err := pushCommand(context.Background(), []string{"--all", "--behind"}); err == nil {
t.Fatal("--all with --behind was accepted")
}
}
func TestTheVerbsCarryReadReplaceAndMove(t *testing.T) {
for _, c := range []struct {
verb string
args map[string]any
want []string
}{
{"settings", map[string]any{"module": "plex", "node": "home"}, []string{"settings", "show", "plex", "--node", "home"}},
{"settings", map[string]any{"module": "plex", "history": "true"}, []string{"settings", "show", "plex", "--history"}},
{"settings", map[string]any{"module": "plex", "values": "{}", "replace": "true"}, []string{"settings", "set", "plex", "{}", "--replace"}},
{"push", map[string]any{"node": "home", "move": "plex", "why": "w"},
[]string{"push", "home", "--wait", "0", "--move", "plex", "--why", "w"}},
{"push", map[string]any{"why": "w"}, []string{"push", "--behind", "--wait", "0", "--why", "w"}},
{"plan", map[string]any{"node": "home", "diff": "true"}, []string{"plan", "home", "--diff"}},
} {
got, err := argvFor(c.verb, c.args)
if err != nil || !reflect.DeepEqual(got, c.want) {
t.Errorf("%s %v: %v %v, want %v", c.verb, c.args, got, err, c.want)
}
}
}
// An argument of the guards given where it cannot take effect is refused, not passed over (issue 244):
// a move with no machine named, a replace beside a clear, a history beside values, a diff beside files.
func TestTheGuardsArgumentsAreNotPassedOver(t *testing.T) {
for _, c := range []struct {
verb string
args map[string]any
}{
{"push", map[string]any{"move": "plex", "why": "w"}},
{"settings", map[string]any{"module": "plex", "clear": "true", "replace": "true"}},
{"settings", map[string]any{"module": "plex", "values": "{}", "history": "true"}},
{"plan", map[string]any{"node": "home", "files": "true", "diff": "true"}},
} {
if argv, err := argvFor(c.verb, c.args); err == nil {
t.Errorf("%s %v was taken as %v, and an argument passed over", c.verb, c.args, argv)
}
}
}
+22 -11
View File
@@ -168,11 +168,14 @@ var ControllerVerbs = []Verb{
Input: schema(map[string]string{"plan": "one walk's id", "limit": "how many ended walks beside the open ones (default 50)"}, Input: schema(map[string]string{"plan": "one walk's id", "limit": "how many ended walks beside the open ones (default 50)"},
nil)}, nil)},
{Name: "plan", Description: "What one machine would run, and why: the declaration the mesh would send it — " + {Name: "plan", Description: "What one machine would run, and why: the declaration the mesh would send it — " +
"or, with files, the files it would be given.", "or, with files, the files it would be given; or, with diff, what a push would change there against what " +
"it was last sent: resources added, removed and changed, a container's changed fields (it is recreated), " +
"and any mount whose directory on the machine would move (novox/hq ADR 0217).",
Input: schema(map[string]string{ Input: schema(map[string]string{
"node": "the machine's name", "node": "the machine's name",
"files": "\"true\": the files this machine would be given, instead of the declaration as JSON", "files": "\"true\": the files this machine would be given, instead of the declaration as JSON",
}, []string{"node"}, "files")}, "diff": "\"true\": what a push would change on this machine, against what it was last sent; not with files",
}, []string{"node"}, "files", "diff")},
{Name: "assign", Description: "Put a module on a machine. Refused with the mesh's own words when it cannot resolve there, " + {Name: "assign", Description: "Put a module on a machine. Refused with the mesh's own words when it cannot resolve there, " +
"or when a seat its resources are applied through is held by nothing on the machine (novox/hq ADR 0207).", "or when a seat its resources are applied through is held by nothing on the machine (novox/hq ADR 0207).",
Input: schema(map[string]string{"node": "the machine's name", Input: schema(map[string]string{"node": "the machine's name",
@@ -194,9 +197,12 @@ var ControllerVerbs = []Verb{
"WHOLE mesh — every machine that is behind — and the answer says so first; behind says that outright. " + "WHOLE mesh — every machine that is behind — and the answer says so first; behind says that outright. " +
"Answers at once that it is running, with a call id: `calls` with that id says what it sent " + "Answers at once that it is running, with a call id: `calls` with that id says what it sent " +
"(a push can reload the bus, which then refuses any answer still to come). A push by hand is a repair, " + "(a push can reload the bus, which then refuses any answer still to come). A push by hand is a repair, " +
"and says why: recorded in the hand-act log (novox/hq to-be 45 §7).", "and says why: recorded in the hand-act log (novox/hq to-be 45 §7). A machine whose push would give a " +
"running module's data another directory is held and named, the others sent (novox/hq ADR 0217): read " +
"`plan` with diff, and name the machine with move to send it.",
Input: schema(map[string]string{ Input: schema(map[string]string{
"node": "the machine's name; without it, every machine that is behind", "node": "the machine's name; without it, every machine that is behind",
"move": "with node: the modules whose data may move with this push, comma-separated (optional)",
"behind": "\"true\": every machine that is behind, the whole mesh — the same as naming none, said outright; not with node", "behind": "\"true\": every machine that is behind, the whole mesh — the same as naming none, said outright; not with node",
"why": "why this is pushed by hand: recorded in the hand-act log", "why": "why this is pushed by hand: recorded in the hand-act log",
"cause": "the cause in a word, or a condition's kind — the word a second push for the same reason uses (optional)", "cause": "the cause in a word, or a condition's kind — the word a second push for the same reason uses (optional)",
@@ -222,15 +228,20 @@ var ControllerVerbs = []Verb{
"node": "the machine that runs the module", "node": "the machine that runs the module",
"module": "the module's name", "module": "the module's name",
}, []string{"node", "module"})}, }, []string{"node", "module"})},
{Name: "settings", Description: "Set what an assignment is configured with: a module's settings for the whole mesh, " + {Name: "settings", Description: "Read or set what an assignment is configured with: a module's settings for the whole " +
"or for one machine. Replaces that layer whole — what it does not name, it no longer sets — and takes effect " + "mesh, or for one machine. Without values or clear, answers the layer as it stands — read it before setting it; " +
"at the next push. With clear, removes the layer and the module is back to what its definition says.", "with history, the layers it replaced. Setting replaces that layer whole and answers each key it adds (+), " +
"changes (~) and removes (-); a set that would remove a key is refused unless replace says it is meant " +
"(novox/hq ADR 0217). Takes effect at the next push. With clear, removes the layer and the module is back to " +
"what its definition says; a cleared or replaced layer is kept in the history.",
Input: schema(map[string]string{ Input: schema(map[string]string{
"module": "the module's name", "module": "the module's name",
"values": "the settings as a JSON object, for set", "values": "the settings as a JSON object, for set",
"node": "one machine; the whole mesh when absent", "node": "one machine; the whole mesh when absent",
"clear": "\"true\" to remove the layer instead of setting it; not with values", "clear": "\"true\" to remove the layer instead of setting it; not with values",
}, []string{"module"}, "clear")}, "replace": "\"true\": with values, the set is meant to remove the keys the layer had and it does not name",
"history": "\"true\": without values or clear, the layers this one replaced, the latest first",
}, []string{"module"}, "clear", "replace", "history")},
{Name: "command", Description: "Run one command line of the controller's own, as you would type it at its " + {Name: "command", Description: "Run one command line of the controller's own, as you would type it at its " +
"shell — `node account g14 jochen`, `node show ace`, `module list` — and answer what it printed. The " + "shell — `node account g14 jochen`, `node show ace`, `module list` — and answer what it printed. The " +
"generic verb beside the named ones (novox/hq ADR 0154): everything the binary can do, without a verb " + "generic verb beside the named ones (novox/hq ADR 0154): everything the binary can do, without a verb " +
+34 -12
View File
@@ -793,11 +793,23 @@ func (i *Inventory) setSettings(ctx context.Context, nodeName, module string, va
return fmt.Errorf("%s: %s is given per node — a port is a fact about one machine; "+ return fmt.Errorf("%s: %s is given per node — a port is a fact about one machine; "+
"set it with --node", module, catalogue.PortsSetting) "set it with --node", module, catalogue.PortsSetting)
} }
_, err = i.store.Pool().Exec(ctx, // The layer it replaces is kept (novox/hq ADR 0217), in the same transaction as the write: a
// previous value is one command away, not in a backup, and a write that fails leaves no history.
tx, err := i.store.Pool().Begin(ctx)
if err != nil {
return err
}
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
if _, err := tx.Exec(ctx, keepReplacedSQL, module, nil, "set"); err != nil {
return err
}
if _, err := tx.Exec(ctx,
`insert into settings (node, module, values) values (null, $1, $2) `insert into settings (node, module, values) values (null, $1, $2)
on conflict (module) where node is null on conflict (module) where node is null
do update set values = excluded.values, set_at = now()`, module, raw) do update set values = excluded.values, set_at = now()`, module, raw); err != nil {
return wrapModule(err, module) return wrapModule(err, module)
}
return tx.Commit(ctx)
} }
node, err := i.NodeByName(ctx, nodeName) node, err := i.NodeByName(ctx, nodeName)
if err != nil { if err != nil {
@@ -817,6 +829,10 @@ func (i *Inventory) setSettings(ctx context.Context, nodeName, module string, va
return err return err
} }
} }
// The layer it replaces is kept (novox/hq ADR 0217), in the same transaction as the write.
if _, err := tx.Exec(ctx, keepReplacedSQL, module, node.ID, "set"); err != nil {
return err
}
_, err = tx.Exec(ctx, _, err = tx.Exec(ctx,
`insert into settings (node, module, values) values ($1, $2, $3) `insert into settings (node, module, values) values ($1, $2, $3)
on conflict (node, module) where node is not null on conflict (node, module) where node is not null
@@ -1009,18 +1025,24 @@ func wrapModule(err error, module string) error {
// ClearSettings removes a layer. // ClearSettings removes a layer.
func (i *Inventory) ClearSettings(ctx context.Context, nodeName, module string) error { func (i *Inventory) ClearSettings(ctx context.Context, nodeName, module string) error {
if nodeName == "" { nodeID, err := i.layerNode(ctx, nodeName)
_, err := i.store.Pool().Exec(ctx,
`delete from settings where module = $1 and node is null`, module)
return err
}
node, err := i.NodeByName(ctx, nodeName)
if err != nil { if err != nil {
return err return err
} }
_, err = i.store.Pool().Exec(ctx, // The layer it removes is kept (novox/hq ADR 0217), in the same transaction as the removal.
`delete from settings where module = $1 and node = $2`, module, node.ID) tx, err := i.store.Pool().Begin(ctx)
return err if err != nil {
return err
}
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
if _, err := tx.Exec(ctx, keepReplacedSQL, module, nodeID, "clear"); err != nil {
return err
}
if _, err := tx.Exec(ctx,
`delete from settings where module = $1 and node is not distinct from $2::uuid`, module, nodeID); err != nil {
return err
}
return tx.Commit(ctx)
} }
// SettingsFor is the layers that apply to one module on one node, in the order they are applied. // SettingsFor is the layers that apply to one module on one node, in the order they are applied.
@@ -0,0 +1,26 @@
-- What a change replaces (novox/hq ADR 0217, to-be 44).
--
-- Two records the mesh did not keep, and each time a change took effect unseen it was the one
-- missing. A settings layer is replaced whole, and the layer it replaced was nowhere: on 2026-10-05
-- one placement set for one module on one machine dropped that machine's whole layer for it, and
-- the old one was read back from a database backup (novox/hq issue 304). And of what a machine was sent the
-- mesh kept only a digest — enough to say *whether* it changed, never *what*.
-- Every layer that was replaced or cleared, with when it had been set and when it went. Not a
-- foreign key to settings: the row it was is the row being replaced.
create table settings_history (
node uuid references node(id) on delete cascade,
module text not null,
values jsonb not null,
set_at timestamptz,
replaced_at timestamptz not null default now(),
-- set · clear
replaced_by text not null
);
create index settings_history_by_layer on settings_history (module, node, replaced_at desc);
-- What a machine was last sent, summarised: per resource its id, type, a digest of it and of each
-- field, and a container's mounts. Not the declaration: a file's content may carry a secret, and
-- this lands in the store's backups. Read only to compare a plan with what was sent; what a machine
-- *should* be is composed from the mesh's records every time, as before (migration 0014).
alter table node add column sent_summary jsonb;
+114
View File
@@ -0,0 +1,114 @@
package inventory
import (
"context"
"encoding/json"
"errors"
"time"
"github.com/jackc/pgx/v5"
)
// What a change replaces (novox/hq ADR 0217, to-be 44): the settings layer a set or a clear replaced,
// and a summary of what a machine was last sent. Both were missing when a change took effect unseen.
// Layer is one settings layer as it stands — the whole mesh's when nodeName is empty — and whether
// there is one. A layer is replaced whole when set; reading it first is how one key is changed
// without losing the others.
func (i *Inventory) Layer(ctx context.Context, nodeName, module string) (map[string]any, bool, error) {
nodeID, err := i.layerNode(ctx, nodeName)
if err != nil {
return nil, false, err
}
var raw []byte
err = i.store.Pool().QueryRow(ctx,
`select values from settings where module = $1 and node is not distinct from $2::uuid`,
module, nodeID).Scan(&raw)
if errors.Is(err, pgx.ErrNoRows) {
return nil, false, nil
}
if err != nil {
return nil, false, err
}
values := map[string]any{}
if err := json.Unmarshal(raw, &values); err != nil {
return nil, false, err
}
return values, true, nil
}
// PastLayer is a layer that was replaced or cleared.
type PastLayer struct {
Values map[string]any
SetAt *time.Time
ReplacedAt time.Time
// ReplacedBy is "set" or "clear".
ReplacedBy string
}
// SettingsHistory is every layer of one module on one machine — the whole mesh's when nodeName is
// empty — that was replaced or cleared, the latest first.
func (i *Inventory) SettingsHistory(ctx context.Context, nodeName, module string) ([]PastLayer, error) {
nodeID, err := i.layerNode(ctx, nodeName)
if err != nil {
return nil, err
}
rows, err := i.store.Pool().Query(ctx,
`select values, set_at, replaced_at, replaced_by from settings_history
where module = $1 and node is not distinct from $2::uuid order by replaced_at desc`,
module, nodeID)
if err != nil {
return nil, err
}
defer rows.Close()
var out []PastLayer
for rows.Next() {
var raw []byte
var p PastLayer
if err := rows.Scan(&raw, &p.SetAt, &p.ReplacedAt, &p.ReplacedBy); err != nil {
return nil, err
}
if err := json.Unmarshal(raw, &p.Values); err != nil {
return nil, err
}
out = append(out, p)
}
return out, rows.Err()
}
// keepReplacedSQL copies a layer into the history before it is replaced or cleared; run in the same
// transaction as the write where there is one, so a write that fails leaves no history of it.
const keepReplacedSQL = `insert into settings_history (node, module, values, set_at, replaced_by)
select node, module, values, set_at, $3 from settings
where module = $1 and node is not distinct from $2::uuid`
// layerNode is the node id of a layer, nil for the whole mesh's.
func (i *Inventory) layerNode(ctx context.Context, nodeName string) (*string, error) {
if nodeName == "" {
return nil, nil
}
n, err := i.NodeByName(ctx, nodeName)
if err != nil {
return nil, err
}
return &n.ID, nil
}
// RecordSentSummary keeps what a machine was last sent, summarised (migration 0078): read only to
// compare what would be sent with it, never as what the machine should be.
func (i *Inventory) RecordSentSummary(ctx context.Context, node string, summary []byte) error {
_, err := i.store.Pool().Exec(ctx, `update node set sent_summary = $2 where id = $1`, node, summary)
return err
}
// SentSummary is what a machine was last sent, summarised, by its name; empty for a machine sent
// nothing since the summary was first kept.
func (i *Inventory) SentSummary(ctx context.Context, name string) ([]byte, error) {
var raw []byte
err := i.store.Pool().QueryRow(ctx,
`select sent_summary from node where name = $1`, name).Scan(&raw)
if errors.Is(err, pgx.ErrNoRows) {
return nil, nil
}
return raw, err
}
+104
View File
@@ -0,0 +1,104 @@
package inventory
import (
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
)
// novox/hq ADR 0217: a settings layer can be read, and the one a set or a clear replaced is kept, so
// a previous value is one command away rather than in a database backup (novox/hq issue 304).
func TestTheLayerASetReplacesIsKeptAndReadable(t *testing.T) {
inv := fresh(t)
ctx := t.Context()
if _, err := inv.AddNode(ctx, "anchor"); err != nil {
t.Fatal(err)
}
web := catalogue.Manifest{Module: "web", Version: "1",
Resources: []map[string]any{
{"id": "server", "type": "container", "name": "web", "image": "x"},
{"id": "conf", "type": "file", "path": "/etc/web.json", "content": "{}", "merge": "json"},
}}
if err := inv.RegisterModule(ctx, web, Source{}); err != nil {
t.Fatal(err)
}
if _, has, err := inv.Layer(ctx, "anchor", "web"); err != nil || has {
t.Fatalf("a layer nobody set: %v %v", has, err)
}
first := map[string]any{"colour": "blue", "size": "large"}
if err := inv.SetSettings(ctx, "anchor", "web", first); err != nil {
t.Fatal(err)
}
got, has, err := inv.Layer(ctx, "anchor", "web")
if err != nil || !has || got["colour"] != "blue" || got["size"] != "large" {
t.Fatalf("the layer read back: %v %v %v", got, has, err)
}
if past, err := inv.SettingsHistory(ctx, "anchor", "web"); err != nil || len(past) != 0 {
t.Fatalf("a first set replaced something: %v %v", past, err)
}
if err := inv.SetSettings(ctx, "anchor", "web", map[string]any{"colour": "red"}); err != nil {
t.Fatal(err)
}
if err := inv.ClearSettings(ctx, "anchor", "web"); err != nil {
t.Fatal(err)
}
past, err := inv.SettingsHistory(ctx, "anchor", "web")
if err != nil || len(past) != 2 {
t.Fatalf("history %v %v", past, err)
}
// The latest first: the clear took the red layer, the set took the first.
if past[0].ReplacedBy != "clear" || past[0].Values["colour"] != "red" {
t.Errorf("the cleared layer: %+v", past[0])
}
if past[1].ReplacedBy != "set" || past[1].Values["size"] != "large" {
t.Errorf("the replaced layer still has what the set dropped: %+v", past[1])
}
// The mesh-wide layer keeps its own history, apart from the node's.
if err := inv.SetSettings(ctx, "", "web", map[string]any{"colour": "green"}); err != nil {
t.Fatal(err)
}
if err := inv.SetSettings(ctx, "", "web", map[string]any{"colour": "grey"}); err != nil {
t.Fatal(err)
}
mesh, err := inv.SettingsHistory(ctx, "", "web")
if err != nil || len(mesh) != 1 || mesh[0].Values["colour"] != "green" {
t.Fatalf("the mesh-wide history %v %v", mesh, err)
}
// And a mesh-wide clear keeps the layer it removes, beside the node's untouched.
if err := inv.ClearSettings(ctx, "", "web"); err != nil {
t.Fatal(err)
}
if _, has, err := inv.Layer(ctx, "", "web"); err != nil || has {
t.Fatalf("a cleared mesh-wide layer is still there: %v %v", has, err)
}
mesh, err = inv.SettingsHistory(ctx, "", "web")
if err != nil || len(mesh) != 2 || mesh[0].ReplacedBy != "clear" || mesh[0].Values["colour"] != "grey" {
t.Fatalf("the mesh-wide clear was not kept: %+v %v", mesh, err)
}
if past, err := inv.SettingsHistory(ctx, "anchor", "web"); err != nil || len(past) != 2 {
t.Fatalf("the node's history changed with the mesh's: %v %v", past, err)
}
}
// What a machine was last sent is kept, summarised, and read back by its name; a machine sent
// nothing since has nothing to compare with.
func TestWhatAMachineWasSentIsKeptForComparison(t *testing.T) {
inv := fresh(t)
ctx := t.Context()
n, err := inv.AddNode(ctx, "anchor")
if err != nil {
t.Fatal(err)
}
if got, err := inv.SentSummary(ctx, "anchor"); err != nil || len(got) != 0 {
t.Fatalf("a machine never sent anything: %s %v", got, err)
}
if err := inv.RecordSentSummary(ctx, n.ID, []byte(`[{"id":"web.server","type":"container"}]`)); err != nil {
t.Fatal(err)
}
got, err := inv.SentSummary(ctx, "anchor")
if err != nil || len(got) == 0 {
t.Fatalf("read back: %s %v", got, err)
}
}