diff --git a/cmd/mesh-controller/facts.go b/cmd/mesh-controller/facts.go index d24cc913..c1b6c891 100644 --- a/cmd/mesh-controller/facts.go +++ b/cmd/mesh-controller/facts.go @@ -514,6 +514,10 @@ func composedAndValidated(ctx context.Context, open *stores, node string, gens m if declared.Epoch, err = open.inventory.SentEpoch(ctx, record.ID); err != nil { return sendable{}, nil, err } + // And the generation it was last sent, as the would-send is (novox/hq issue 234). + if declared.Generation, declared.PutBack, err = open.inventory.SentGeneration(ctx, record.ID); err != nil { + return sendable{}, nil, err + } body, err := declared.Body() if err != nil { return sendable{}, nil, err diff --git a/cmd/mesh-controller/gate.go b/cmd/mesh-controller/gate.go index 36eefc4d..d7a71972 100644 --- a/cmd/mesh-controller/gate.go +++ b/cmd/mesh-controller/gate.go @@ -994,7 +994,8 @@ func gateFailed(ctx context.Context, open *stores, p *inventory.Plan, module str } return } - sent, err := sendRollout(withScope(ctx, sendScope{modules: map[string]bool{module: true}}), open, g.Machines) + sent, err := sendRollout(withScope(ctx, sendScope{modules: map[string]bool{module: true}, putBack: true}), open, + g.Machines) if err != nil { notBack(fmt.Sprintf("its registered build is back at %s, and sending it to %s was refused: %v — `push %s` "+ "sends it", short(previous.Commit), strings.Join(g.Machines, ", "), err, g.Machines[0])) @@ -1087,7 +1088,7 @@ func sendRollbacks(ctx context.Context, open *stores, p *inventory.Plan, b *roll } inv := open.inventory sort.Strings(b.machines) - sent, err := sendRollout(withScope(ctx, sendScope{modules: b.modules}), open, b.machines) + sent, err := sendRollout(withScope(ctx, sendScope{modules: b.modules, putBack: true}), open, b.machines) var back []string for _, r := range b.pending { if err != nil { diff --git a/cmd/mesh-controller/generation.go b/cmd/mesh-controller/generation.go new file mode 100644 index 00000000..f42d3fff --- /dev/null +++ b/cmd/mesh-controller/generation.go @@ -0,0 +1,195 @@ +package main + +import ( + "context" + "fmt" + "io" + "os" + "sort" + "strings" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// The assignment generation a declaration was composed from, and the record of every send (novox/hq +// issue 234). +// +// On 2026-10-04 a declaration newer in sequence than every other named four fewer modules than the +// assignments held, and a machine applied it and undeclared all four. The sequence orders arrival and +// cannot tell a later send that carries an older view of the assignments. So a declaration now carries +// the generation of the assignments it was composed from — a counter the store raises in the same +// transaction as every assignment change — and a machine refuses one older than it applied, unless it is +// a gate's put-back. And every send is written down with who sent it, from which generation and naming +// which modules: the controller logged no send then, and which process sent the stale declaration could +// not be read back from anything the mesh kept. + +// kindOlderGeneration is S20's kind: a machine refused a send for the generation it was composed from. +const kindOlderGeneration = "older-generation" + +// generationRefusalsSaid is how long a refused send is said after its refusal: an hour, as an advisory is. +const generationRefusalsSaid = advisoryQuiet + +// stamp gives a composed declaration the order allotted to it before it was composed: its sequence, the +// epoch and generation when the machine reads them, and the put-back mark beside a generation — and keeps +// the generation and acting epoch for the record of the send whether or not the machine is sent them. +func (o order) stamp(d *sendable) { + d.Sequence, d.Epoch = o.sequence, o.epoch + d.composedFrom, d.actingEpoch, d.putBackSend = o.generation, o.acting, o.putBack + d.Generation, d.PutBack = 0, false + if o.readsGeneration && o.generation > 0 { + d.Generation, d.PutBack = o.generation, o.putBack + } +} + +// sentRecord is what the record of a send keeps beside its digest. +type sentRecord struct { + sequence int64 + epoch uint64 + generation int64 + putBack bool + // told is the generation and put-back mark on the wire, which the would-send is stamped with. + toldGeneration int64 + toldPutBack bool + sender string + modules []string +} + +// sentRecordOf is a composed send's record: its sender is the caller this process acts for. +func sentRecordOf(ctx context.Context, d sendable) sentRecord { + return sentRecord{sequence: d.Sequence, epoch: d.actingEpoch, generation: d.composedFrom, putBack: d.putBackSend, + toldGeneration: d.Generation, toldPutBack: d.PutBack, sender: senderOf(callerOf(ctx)), modules: d.modules} +} + +// callerOf is who asked for what this process does: the seat call's caller when ctx belongs to one, the +// caller the controller ran this command for, or the account at the shell. +func callerOf(ctx context.Context) string { + if c := link.CallerIn(ctx); c != "" { + return c + } + return link.Caller() +} + +// senderOf is a send's sender in words: who asked, and the process and build that composed it — the +// answer issue 234 could not find, because two controllers and a one-shot push were all sending then. +func senderOf(caller string) string { + host, _ := os.Hostname() + return fmt.Sprintf("%s (pid %d on %s, build %s)", caller, os.Getpid(), host, version) +} + +// namedModules are the modules a declaration names: its machine's set, less what was left out of it. +func namedModules(plan catalogue.Resolution, leftOut map[string]string) []string { + out := make([]string, 0, len(plan.Modules)) + for _, m := range plan.Modules { + if _, left := leftOut[m.Module]; !left { + out = append(out, m.Module) + } + } + sort.Strings(out) + return out +} + +// heardGenerationRefusal keeps a machine's refusal of a send for its generation on the send it refused, +// so S20 names its sender, and says it (novox/hq issue 234). +// +// **And raises the mesh's counter past what the machine applied, when the refused send was composed from +// the counter as it stands**: then the sender's view was not stale — the counter is behind the machine, +// which is a store put back from a backup — and every send after would be refused for ever. A stale +// sender's generation is below the counter, and the counter is left alone for it. +func heardGenerationRefusal(ctx context.Context, inv *inventory.Inventory, report link.Report) { + r := report.OlderGeneration + if r == nil || inv == nil || report.Node == "" { + return + } + node, err := inv.NodeByName(ctx, report.Node) + if err != nil { + fmt.Fprintf(os.Stderr, "mesh-controller: %s refused a send for its generation, and the machine cannot be read: %v\n", + report.Node, err) + return + } + send, found, err := inv.RefusedSend(ctx, node.ID, report.Sequence, r.Applied) + switch { + case err != nil: + fmt.Fprintf(os.Stderr, "mesh-controller: %s refused send %d for its generation, and the refusal could not be "+ + "kept: %v\n", report.Node, report.Sequence, err) + case !found: + fmt.Fprintf(os.Stderr, "mesh-controller: %s refused send %d (generation %d; it applied %d), which the mesh "+ + "has no record of sending\n", report.Node, report.Sequence, r.Generation, r.Applied) + default: + fmt.Fprintf(os.Stderr, "mesh-controller: %s refused send %d from %s: composed from assignment generation %d, "+ + "and it applied %d\n", report.Node, report.Sequence, send.Sender, r.Generation, r.Applied) + } + now, err := inv.AssignmentGeneration(ctx) + if err != nil || r.Generation < now || r.Applied < now { + return + } + raised, err := inv.RaiseAssignmentGeneration(ctx, r.Applied) + if err != nil { + fmt.Fprintf(os.Stderr, "mesh-controller: the assignment generation (%d) is behind what %s applied (%d), and "+ + "could not be raised: %v\n", now, report.Node, r.Applied, err) + return + } + fmt.Fprintf(os.Stderr, "mesh-controller: the assignment generation was %d, behind what %s applied (%d) — a store "+ + "put back from a backup — and is raised to %d, so the next send is not refused\n", now, report.Node, r.Applied, + raised) +} + +// watchGenerationRefusals is S20: every send a machine refused within the hour for the generation it was +// composed from, naming who sent it (novox/hq issue 234). One per machine, the newest refusal. +func watchGenerationRefusals(f *signalFacts) []conditions.Observation { + seen := map[string]bool{} + var out []conditions.Observation + for _, s := range f.refusedSends { + if s.RefusedAt == nil || f.now.Sub(*s.RefusedAt) > generationRefusalsSaid || seen[s.NodeName] { + continue + } + seen[s.NodeName] = true + out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: s.NodeName, Kind: kindOlderGeneration, + Machine: s.NodeName, Severity: conditions.Warning, + Summary: fmt.Sprintf("%s refused sequence %d from %s: it was composed from assignment generation %d, and "+ + "%s applied generation %d — a sender composing from a view of the assignments the mesh has moved "+ + "past; it named %s", s.NodeName, s.Sequence, s.Sender, s.Generation, s.NodeName, s.RefusedApplied, + modulesWords(s.Modules)), + Said: fmt.Sprintf("refused at %s", s.RefusedAt.UTC().Format(time.RFC3339)), + Headline: fmt.Sprintf("%s refused an out-of-date update", s.NodeName), + Explanation: fmt.Sprintf("Something sent %s an update made from an older list of what runs there. %s "+ + "refused it, so nothing was removed. Nothing for you to do unless it repeats.", s.NodeName, s.NodeName), + Resolved: fmt.Sprintf("%s has had no out-of-date update for an hour", s.NodeName)}) + } + return out +} + +// modulesWords is a send's modules as a sentence says them. +func modulesWords(modules []string) string { + if len(modules) == 0 { + return "no module the mesh recorded" + } + return strings.Join(modules, ", ") +} + +// writeLastSend says what a machine was last told, by whom and from which generation (novox/hq issue 234): +// nothing when the mesh has not recorded a send to it. +func writeLastSend(ctx context.Context, w io.Writer, inv *inventory.Inventory, node string) error { + s, found, err := inv.LastSend(ctx, node) + if err != nil || !found { + return err + } + generation := "no generation recorded" + if s.Generation > 0 { + generation = fmt.Sprintf("assignment generation %d", s.Generation) + } + kind := "sent" + if s.PutBack { + kind = "put back" + } + fmt.Fprintf(w, "%s was last %s sequence %d at %s by %s, composed from %s, naming %s\n", node, kind, + s.Sequence, s.SentAt.Local().Format("2006-01-02 15:04:05"), s.Sender, generation, modulesWords(s.Modules)) + if s.RefusedAt != nil { + fmt.Fprintf(w, " and refused it at %s: it had applied generation %d\n", + s.RefusedAt.Local().Format("2006-01-02 15:04:05"), s.RefusedApplied) + } + return nil +} diff --git a/cmd/mesh-controller/generation_test.go b/cmd/mesh-controller/generation_test.go new file mode 100644 index 00000000..f794760c --- /dev/null +++ b/cmd/mesh-controller/generation_test.go @@ -0,0 +1,107 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/inventory" +) + +// The assignment generation a declaration was composed from (novox/hq issue 234). + +func bodyKeys(t *testing.T, s sendable) map[string]any { + t.Helper() + raw, err := s.Body() + if err != nil { + t.Fatal(err) + } + var keys map[string]any + if err := json.Unmarshal(raw, &keys); err != nil { + t.Fatal(err) + } + return keys +} + +// **The generation goes on the wire only to a machine whose node-engine said it reads one**: an older +// node-engine decodes strictly and refuses an unknown key, whole. And the put-back mark only with it. +func TestTheGenerationIsSentOnlyToAMachineThatReadsOne(t *testing.T) { + resources := []map[string]any{{"id": "a", "type": "file", "path": "/etc/a", "content": "x\n"}} + reads := order{sequence: 12, epoch: 57, generation: 40, readsGeneration: true, acting: 57, putBack: true} + var told sendable + told.Resources = resources + reads.stamp(&told) + keys := bodyKeys(t, told) + if keys["generation"] != float64(40) || keys["put_back"] != true || keys["sequence"] != float64(12) { + t.Fatalf("a machine that reads a generation was sent %v", keys) + } + if told.composedFrom != 40 || told.actingEpoch != 57 { + t.Errorf("the send does not keep what it was composed from for its record: %+v", told) + } + + older := order{sequence: 12, epoch: 57, generation: 40, readsGeneration: false, acting: 57, putBack: true} + var untold sendable + untold.Resources = resources + older.stamp(&untold) + keys = bodyKeys(t, untold) + if _, there := keys["generation"]; there { + t.Fatalf("a machine whose node-engine never said it reads a generation was sent one: %v", keys) + } + if _, there := keys["put_back"]; there { + t.Fatalf("a machine whose node-engine never said it reads a generation was sent a put-back mark: %v", keys) + } + if untold.composedFrom != 40 { + t.Errorf("the generation it was composed from is recorded whether or not it was sent: %+v", untold) + } + + // An ordinary send carries no put-back mark at all: the body is what it was apart from the generation. + var ordinary sendable + ordinary.Resources = resources + ordinaryOrder := reads + ordinaryOrder.putBack = false + ordinaryOrder.stamp(&ordinary) + if _, there := bodyKeys(t, ordinary)["put_back"]; there { + t.Fatal("an ordinary send says it is a put-back") + } +} + +// **The sender is named**: the caller of the seat call when there is one, else the shell's account, and the +// process and build that composed it. +func TestASendNamesItsSender(t *testing.T) { + said := senderOf("g14.node-tools, through the mesh-controller seat") + if !strings.Contains(said, "g14.node-tools") || !strings.Contains(said, "pid ") || !strings.Contains(said, "build ") { + t.Fatalf("the sender reads %q", said) + } +} + +// refusedSend is a send a machine refused for its generation at a moment. +func refusedSend(at time.Time) inventory.Send { + return inventory.Send{NodeName: "anchor", Sequence: 12, Epoch: 57, Generation: 38, RefusedApplied: 40, + Sender: "a one-shot push by jochen at a shell on anchor (pid 4242 on anchor, build 2026.10.11)", + Modules: []string{"docker"}, SentAt: at.Add(-time.Second), RefusedAt: &at} +} + +// **A refused send is raised naming its sender** (novox/hq issue 234): who sent it, from which generation, +// against which the machine applied, and its sequence — the facts that took a morning to look for. +func TestARefusedSendIsRaisedNamingItsSender(t *testing.T) { + now := time.Date(2026, 10, 11, 12, 0, 0, 0, time.UTC) + f := calm(now) + f.refusedSends = []inventory.Send{refusedSend(now.Add(-time.Minute))} + got := watchGenerationRefusals(f) + if len(got) != 1 { + t.Fatalf("%+v", got) + } + o := got[0] + if o.Key() != "machine.anchor.older-generation" || o.Machine != "anchor" { + t.Errorf("raised as %s about %q", o.Key(), o.Machine) + } + for _, want := range []string{"a one-shot push by jochen", "generation 38", "generation 40", "sequence 12"} { + if !strings.Contains(o.Summary, want) { + t.Errorf("the summary does not say %q: %s", want, o.Summary) + } + } + if o.Headline == "" || o.Explanation == "" || o.Resolved == "" { + t.Errorf("the condition is not worded for the operator: %+v", o) + } +} diff --git a/cmd/mesh-controller/held_back_test.go b/cmd/mesh-controller/held_back_test.go index bcb0fe94..f9d2ba06 100644 --- a/cmd/mesh-controller/held_back_test.go +++ b/cmd/mesh-controller/held_back_test.go @@ -128,7 +128,7 @@ func (r *recordedDelivery) grant(context.Context, []readyNode) error { return ni func (r *recordedDelivery) declare(ctx context.Context, s readyNode, body []byte) (string, error) { r.declared = append(r.declared, s.node) - return recordSent(ctx, r.inv, s.node, body, s.declared.Builds, s.declared.Epoch) + return recordSent(ctx, r.inv, s.node, body, s.declared.Builds, s.declared.Epoch, sentRecordOf(ctx, s.declared)) } // aResolver is a module built from a repository, at a commit, with something on the machine that diff --git a/cmd/mesh-controller/issue_204_test.go b/cmd/mesh-controller/issue_204_test.go index feda54d8..1779292a 100644 --- a/cmd/mesh-controller/issue_204_test.go +++ b/cmd/mesh-controller/issue_204_test.go @@ -77,7 +77,7 @@ func TestASendIsRecordedEvenWhenTheSenderIsBeingCancelled(t *testing.T) { } cancel() // the sender is going away: its context is cancelled between the send and the record body := []byte(`{"declaration":1,"resources":[]}`) - digest, err := recordSent(ctx, inv, "anchor", body, nil, 0) + digest, err := recordSent(ctx, inv, "anchor", body, nil, 0, sentRecord{sender: "a test"}) if err != nil { // NodeByName on the cancelled context may itself refuse; the record must still be possible // through the detached context, so look the node up again on a live one. diff --git a/cmd/mesh-controller/plan.go b/cmd/mesh-controller/plan.go index 82f0fbb4..2ec829b1 100644 --- a/cmd/mesh-controller/plan.go +++ b/cmd/mesh-controller/plan.go @@ -420,7 +420,8 @@ func declarationWith(ctx context.Context, open *stores, node string, out := sendable{Resources: composed.Resources, Adoption: adoption, Received: composed.Received, Mesh: with.Mesh, BusUsers: with.BusUsers, LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut, withheld: with.Withheld, - unbound: with.Unbound, foreseen: composed.Foreseen, unplaced: composed.Unplaced} + unbound: with.Unbound, foreseen: composed.Foreseen, unplaced: composed.Unplaced, + modules: namedModules(plan, composed.LeftOut)} // And which build of each module it carries, for the send to record (novox/hq issue 259, ADR // 0221). Read only on the send path: a question about what would be sent records nothing. if choosing == Allocating { @@ -1290,6 +1291,11 @@ func planCommand(ctx context.Context, args []string) error { return nil } + // What the machine was last told, by whom and from which assignment generation (novox/hq issue 234), + // beside what it would be told now. A record that cannot be read is said, not taken for none. + if err := writeLastSend(ctx, os.Stdout, open.inventory, args[0]); err != nil { + fmt.Printf("what %s was last sent could not be read: %v\n", args[0], err) + } // Which modules a push would leave out, and why — said before the plan, since the plan is of // what the machine would be told (novox/hq ADR 0163, rule 6). Judged, never composed: `plan` // without --json allocates nothing. diff --git a/cmd/mesh-controller/push.go b/cmd/mesh-controller/push.go index 9779b442..be33434b 100644 --- a/cmd/mesh-controller/push.go +++ b/cmd/mesh-controller/push.go @@ -356,10 +356,18 @@ func declare(ctx context.Context, args []string) error { // the mesh did not compose it, so a push that does not name this machine treats it as held. // The epoch it carried, if a person wrote one in, is what the machine heard. var carried struct { - Epoch uint64 `json:"epoch"` + Epoch uint64 `json:"epoch"` + Sequence int64 `json:"sequence"` + Generation int64 `json:"generation"` + PutBack bool `json:"put_back"` } _ = json.Unmarshal(raw, &carried) - if _, err := recordSent(ctx, inv, node, raw, nil, carried.Epoch); err != nil { + // And recorded as every send is (novox/hq issue 234): by hand, from what it carried; the modules it + // named are not known, since the mesh did not compose it. + if _, err := recordSent(ctx, inv, node, raw, nil, carried.Epoch, sentRecord{sequence: carried.Sequence, + epoch: carried.Epoch, generation: carried.Generation, putBack: carried.PutBack, + toldGeneration: carried.Generation, toldPutBack: carried.PutBack, + sender: senderOf(callerOf(ctx)) + ", a declaration sent by hand"}); err != nil { return err } fmt.Printf("sent %s a signed declaration (%d bytes)\n", node, len(raw)) @@ -798,7 +806,7 @@ func composeEach(names []string, allot func(node string) (order, error), refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err)) continue } - declared.Sequence, declared.Epoch = numbered.sequence, numbered.epoch + numbered.stamp(&declared) if len(declared.Resources) == 0 { // Sent, not skipped (novox/hq issue 127). A node whose declaration composes to // nothing may have HELD something before — the broker opening a placement gave it, @@ -1016,7 +1024,8 @@ func (b overTheBus) declare(ctx context.Context, s readyNode, body []byte) (stri } // After it is away, not before. A digest recorded for something that failed to send would make // the machine look current for a declaration it never received. - digest, err := recordSent(ctx, b.open.inventory, s.node, body, s.declared.Builds, s.declared.Epoch) + digest, err := recordSent(ctx, b.open.inventory, s.node, body, s.declared.Builds, s.declared.Epoch, + sentRecordOf(ctx, s.declared)) if err != nil { return "", err } @@ -1228,7 +1237,7 @@ func sendToEach(ctx context.Context, open *stores, names []string) ([]string, er refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err)) continue } - declared.Sequence, declared.Epoch = numbered.sequence, numbered.epoch + numbered.stamp(&declared) reportLeftOut(name, declared) sending = append(sending, readyNode{name, declared}) } @@ -1393,6 +1402,11 @@ func wouldSendFrom(ctx context.Context, open *stores, if declared.Epoch, err = open.inventory.SentEpoch(ctx, n.ID); err != nil { return nil, err } + // And the generation and put-back mark it was last sent, for the same reason (novox/hq issue 234): + // an assignment elsewhere in the mesh is not a change of this machine. + if declared.Generation, declared.PutBack, err = open.inventory.SentGeneration(ctx, n.ID); err != nil { + return nil, err + } body, err := declared.Body() if err != nil { return nil, err @@ -1531,6 +1545,14 @@ var epochForActs = func(ctx context.Context) (uint64, error) { return theLease.e type order struct { sequence int64 epoch uint64 + // generation is the assignment generation read before the composition, and readsGeneration whether the + // machine's node-engine said it reads one (novox/hq issue 234); acting is the lease epoch the sender acts + // under, whether or not the machine is sent it — both kept for the record of the send. + generation int64 + readsGeneration bool + acting uint64 + // putBack is a gate's put-back (sendScope.putBack), read from the send's context. + putBack bool } // allot takes the next sequence for a machine — the number its next declaration carries — under the @@ -1544,6 +1566,7 @@ func allot(ctx context.Context, inv *inventory.Inventory, node string) (order, e if err != nil { return order{}, err } + acting := epoch if epoch > 0 { reads, err := inv.ReadsEpoch(ctx, record.ID) if err != nil { @@ -1553,11 +1576,24 @@ func allot(ctx context.Context, inv *inventory.Inventory, node string) (order, e epoch = 0 } } + // **The generation is read here, before the composition reads a single assignment** (novox/hq issue + // 234). A generation only grows, so one read before is never newer than the view composed after it: + // the declaration may claim a generation older than its content, never newer — and a claim newer than + // the content is exactly the stale send the machine must be able to refuse. + generation, err := inv.AssignmentGeneration(ctx) + if err != nil { + return order{}, err + } + readsGeneration, err := inv.ReadsGeneration(ctx, record.ID) + if err != nil { + return order{}, err + } seq, err := inv.NextSequence(ctx, record.ID) if err != nil { return order{}, err } - return order{sequence: seq, epoch: epoch}, nil + return order{sequence: seq, epoch: epoch, generation: generation, readsGeneration: readsGeneration, + acting: acting, putBack: scopeOf(ctx).putBack}, nil } // recordSent writes down what a machine was just sent, and returns the digest. @@ -1572,7 +1608,7 @@ func allot(ctx context.Context, inv *inventory.Inventory, node string) (order, e // And the build of each module it carried (novox/hq issue 259, ADR 0221), nil when that is not known: // what tells a machine held back by a policy or a plan from one a push left behind. func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body []byte, - builds map[string]string, epoch uint64) (string, error) { + builds map[string]string, epoch uint64, sent sentRecord) (string, error) { kept, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) defer cancel() record, err := inv.NodeByName(kept, node) @@ -1583,6 +1619,14 @@ func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body if err := inv.RecordSentUnder(kept, record.ID, digest, builds, epoch); err != nil { return "", err } + // And the send itself, who sent it and from which generation (novox/hq issue 234): what the machine + // was last told, by whom, is read back from here — and what the would-send is stamped with. + if err := inv.RecordSend(kept, inventory.Send{Node: record.ID, Sequence: sent.sequence, + Epoch: int64(sent.epoch), Sender: sent.sender, Generation: sent.generation, PutBack: sent.putBack, + ToldGeneration: sent.toldGeneration, ToldPutBack: sent.toldPutBack, Digest: digest, + Modules: sent.modules}); err != nil { + return "", fmt.Errorf("%s was sent its declaration, and the send could not be recorded: %w", node, err) + } // And what it was, summarised, so the next push can be compared with it (novox/hq ADR 0217). // Never a reason to fail a send that is already away: a summary that cannot be kept means the // next comparison has nothing to hold against, which is how every machine starts. diff --git a/cmd/mesh-controller/release.go b/cmd/mesh-controller/release.go index 56b702c8..31c1ef87 100644 --- a/cmd/mesh-controller/release.go +++ b/cmd/mesh-controller/release.go @@ -50,6 +50,9 @@ type sendScope struct { modules map[string]bool // person is a person's act: it carries what it carries. person bool + // putBack is a gate putting machines back after a failed send (novox/hq issue 234): its declarations + // say so, and a machine does not refuse them for the generation they carry. + putBack bool } type sendScopeKey struct{} diff --git a/cmd/mesh-controller/sendable.go b/cmd/mesh-controller/sendable.go index 2311522a..cbc89943 100644 --- a/cmd/mesh-controller/sendable.go +++ b/cmd/mesh-controller/sendable.go @@ -27,6 +27,25 @@ type sendable struct { // said it reads one is sent none, because an older node-engine refuses a key it does not know, whole // (link/order.go, the contract). Epoch uint64 + // Generation is the assignment generation it was composed from (novox/hq issue 234): a machine that + // applied a later one refuses it, so a send composed from a view of the assignments the mesh has moved + // past cannot undeclare what is still assigned. Zero is not sent at all — every machine whose + // node-engine has not said it reads one is sent none, for the reason the epoch is not (an older + // node-engine refuses a key it does not know, whole). + Generation int64 + // PutBack marks a gate's put-back (novox/hq issue 234): it carries the generation of what it puts + // back and is not refused for it. Sent only beside a generation. + PutBack bool + // composedFrom is the generation read before this declaration was composed, and actingEpoch the + // lease epoch its sender acted under — whether or not the machine is sent either — for the record of + // the send; never on the wire. + composedFrom int64 + actingEpoch uint64 + // putBackSend is whether a gate sent it to put the machine back, for the record; PutBack is the mark + // on the wire, only beside a generation. + putBackSend bool + // modules are the modules this declaration names, for the record of the send; never on the wire. + modules []string // Adoption is nil for a converged node, and then the body is byte for byte what it was before // adoption existed: an older host parses the envelope strictly and would refuse the key. Adoption *adoptionEnvelope @@ -94,6 +113,12 @@ func (s sendable) Body() ([]byte, error) { if len(s.LeftOut) > 0 { envelope["left_out"] = s.LeftOut } + if s.Generation > 0 { + envelope["generation"] = s.Generation + if s.PutBack { + envelope["put_back"] = true + } + } // An empty declaration is deliberate here — the node owns nothing the mesh put there // (novox/hq issue 127) — and the host refuses an empty body unless it is told the emptiness // is meant, so a truncated or mis-composed body is never mistaken for "own nothing". diff --git a/cmd/mesh-controller/signals.go b/cmd/mesh-controller/signals.go index ab3e8a93..34a6781c 100644 --- a/cmd/mesh-controller/signals.go +++ b/cmd/mesh-controller/signals.go @@ -226,6 +226,20 @@ var signalsTable = []signalRow{ newest: func(f *signalFacts) time.Time { return newestOf(f.batches, func(b batchFacts) time.Time { return b.closed }) }}, + {Row: "S20", Signal: "a send refused for the assignment generation it was composed from", + Emitter: "node-engine", Trigger: "each refusal (novox/hq issue 234)", + Bound: "none: raised at the first refusal, naming its sender, the generation it came from and the one the " + + "machine applied; cleared an hour after the last", + Kind: kindOlderGeneration, Severity: conditions.Warning, Phase: 2, + needs: func(f *signalFacts) error { return f.refusedSendsErr }, watch: watchGenerationRefusals, + newest: func(f *signalFacts) time.Time { + return newestOf(f.refusedSends, func(s inventory.Send) time.Time { + if s.RefusedAt == nil { + return time.Time{} + } + return *s.RefusedAt + }) + }}, {Row: "S17", Signal: "a send held for the bus's planned step is told to a person", Emitter: "controller's plan", Trigger: "each send refused because it would replace the bus outside its step (novox/hq issue 336)", Bound: "none: raised at the first refusal, for the operator, naming what waits, the bus build from and to, " + diff --git a/cmd/mesh-controller/signals_test.go b/cmd/mesh-controller/signals_test.go index c69a9d02..9abf6ce8 100644 --- a/cmd/mesh-controller/signals_test.go +++ b/cmd/mesh-controller/signals_test.go @@ -178,6 +178,14 @@ var suppressions = map[string]suppression{ commit: "c0ffee001122", modules: []string{"app"}, since: f.now.Add(-time.Second)}}} }, }, + // A send refused by its machine for the generation it was composed from (novox/hq issue 234): said at + // once, naming its sender, for an hour after. Inside: the last such refusal was more than an hour ago. + "S20": { + inside: func(f *signalFacts) { + f.refusedSends = []inventory.Send{refusedSend(f.now.Add(-61 * time.Minute))} + }, + past: func(f *signalFacts) { f.refusedSends = []inventory.Send{refusedSend(f.now.Add(-time.Second))} }, + }, // Twice by hand within a fortnight is a healer wanted; once, or the first of two a day too old, is not. "S15": { inside: func(f *signalFacts) { diff --git a/cmd/mesh-controller/status_summary.go b/cmd/mesh-controller/status_summary.go index 7974abd4..a81a0b8e 100644 --- a/cmd/mesh-controller/status_summary.go +++ b/cmd/mesh-controller/status_summary.go @@ -196,6 +196,11 @@ func (l nudgingListener) Heard(ctx context.Context, report link.Report) (bool, e if report.Ordered() { link.StaleRefusals.Lifetime(report.Node, report.RefusedOlder, now) } + // A send refused for the generation it was composed from is kept on the send it refused, so S20 names + // its sender (novox/hq issue 234). + if report.OlderGeneration != nil { + heardGenerationRefusal(ctx, l.Enrolment.Inventory, report) + } // What the machine's witnesses put back and stand by (novox/hq ADR 0236): read by the gate and its // probe. Only from an account of the machine — not a word that a declaration was set aside, nor a rekey. if report.Superseded == "" && report.Rekey == nil && report.Node != "" { diff --git a/cmd/mesh-controller/watchdogs.go b/cmd/mesh-controller/watchdogs.go index 35c2c951..7ccb352d 100644 --- a/cmd/mesh-controller/watchdogs.go +++ b/cmd/mesh-controller/watchdogs.go @@ -55,6 +55,10 @@ type signalFacts struct { waits []waitFacts // batches are the batches not yet cut (S18, S19, novox/hq ADR 0276). batches []batchFacts + // refusedSends are the sends a machine refused within the hour for the generation they were composed + // from, each naming its sender (S20, novox/hq issue 234). + refusedSends []inventory.Send + refusedSendsErr error loop loopFacts loopErr error @@ -364,6 +368,7 @@ func (w *watchdogs) gather(ctx context.Context) *signalFacts { } } f.handActs, f.handActsErr = w.gatherHandActs(ctx, now) + f.refusedSends, f.refusedSendsErr = inv.RefusedSendsSince(ctx, now.Add(-generationRefusalsSaid)) f.facts.taken, _, f.facts.began, f.facts.err = exportedFacts.last() return f } diff --git a/internal/inventory/migrations/0094-a-declaration-says-the-generation-it-came-from.sql b/internal/inventory/migrations/0094-a-declaration-says-the-generation-it-came-from.sql new file mode 100644 index 00000000..07a182d3 --- /dev/null +++ b/internal/inventory/migrations/0094-a-declaration-says-the-generation-it-came-from.sql @@ -0,0 +1,64 @@ +-- A declaration says the assignment generation it was composed from, and every send is recorded +-- (novox/hq issue 234). +-- +-- On 2026-10-04 a declaration newer in sequence than every other named four fewer modules than the +-- assignments held, and the machine undeclared all four. The sequence orders arrival; it cannot tell a +-- later send that carries an older view of the assignments. And nothing the mesh kept said which process +-- sent it, from what view: the node row holds only the digest of the last send. + +-- The assignment generation: one counter for the mesh, raised in the same transaction as every change to +-- what is assigned where. A trigger, not a line in each writer: an assignment is also taken by a node's +-- removal (on delete cascade) and renamed with its module (on update cascade), which no writer in Go +-- sees, and a rule kept only by the writers that remember it is the rule this issue found broken. Seeded +-- at 1, so every declaration composed after this claims a generation and none claims zero ("none"). +create table assignment_generation ( + one boolean primary key default true check (one), + generation bigint not null +); +insert into assignment_generation (one, generation) values (true, 1); + +create function raise_assignment_generation() returns trigger language plpgsql as $$ +begin + update assignment_generation set generation = generation + 1; + return null; +end +$$; + +-- Per row, so a statement that changes nothing (an assignment repeated, `on conflict do nothing`) raises +-- nothing; a statement that changes several raises once per row, which only ever moves it forward. +create trigger assignment_generation_raised + after insert or update or delete on assignment + for each row execute function raise_assignment_generation(); + +-- What a machine was last sent of it: the generation, and whether that send was a gate's put-back, so +-- what the mesh WOULD send is stamped the same way and reads as byte for byte what it DID send when +-- nothing else changed (as sent_epoch, migration 0068). Null and false for a send without. +alter table node add column sent_generation bigint; +alter table node add column sent_put_back boolean not null default false; +-- Whether the machine's node-engine said it reads a generation (its reports' `reads_generation`): until +-- it has, it is sent none, because an older node-engine refuses a key it does not know, whole. +alter table node add column reads_generation boolean not null default false; + +-- Every send: the machine, its sequence, who sent it — the lease epoch the sender acted under and the +-- caller, process and build — the generation it was composed from, whether it was a put-back, its +-- digest and the modules it named. Written after the declaration is away, as the node row's record is. +-- A refusal by the machine of a send for its generation is kept on the send it refused, so the +-- condition it raises names the sender from here. The newest 200 per machine are kept. +create table declaration_send ( + id bigserial primary key, + node uuid not null references node (id) on delete cascade, + sequence bigint, + epoch bigint, + sender text not null, + generation bigint, + put_back boolean not null default false, + digest text not null, + modules text[] not null default '{}', + sent_at timestamptz not null default now(), + refused_at timestamptz, + -- refused_applied is the generation the machine said it had applied when it refused this send. + refused_applied bigint +); +create index declaration_send_node on declaration_send (node, id desc); +create index declaration_send_sequence on declaration_send (node, sequence); +create index declaration_send_refused on declaration_send (refused_at) where refused_at is not null; diff --git a/internal/inventory/sends.go b/internal/inventory/sends.go new file mode 100644 index 00000000..085a2ac9 --- /dev/null +++ b/internal/inventory/sends.go @@ -0,0 +1,202 @@ +package inventory + +import ( + "context" + "fmt" + "time" + + "github.com/jackc/pgx/v5" +) + +// The assignment generation and the record of every send (novox/hq issue 234, migration 0094). +// +// A declaration carries the generation of the assignments it was composed from, and a machine refuses one +// composed from an older generation than it applied — the sequence orders arrival and cannot tell a later +// send carrying an older view. And every send is written down with who sent it, from which generation and +// naming which modules, so what a machine was last told, by whom, is read back rather than guessed at. + +// sendsKept is how many sends are kept per machine: the newest. Enough to read back a morning of pushes +// on a busy machine; the record is for finding who sent what, not an archive. +const sendsKept = 200 + +// AssignmentGeneration is the mesh's assignment generation now: raised by the store itself in the same +// transaction as every change to what is assigned where (a trigger on the assignment table). +func (i *Inventory) AssignmentGeneration(ctx context.Context) (int64, error) { + var g int64 + if err := i.store.Pool().QueryRow(ctx, `select generation from assignment_generation`).Scan(&g); err != nil { + return 0, fmt.Errorf("reading the assignment generation: %w", err) + } + return g, nil +} + +// RaiseAssignmentGeneration raises the generation past one a machine applied, and answers it: one more +// than that, or what it already was when it is past it. It never lowers it. +// +// For one case only: a machine refused a send composed from the generation the mesh holds now, so the +// machine applied a higher one than the mesh's counter — a store put back from a backup. Every send after +// would be refused for ever; raised past it, the next send carries a generation the machine takes, composed +// from the assignments the store holds now. A stale sender never meets this: its generation is below the +// counter, and the counter is not touched for it. +func (i *Inventory) RaiseAssignmentGeneration(ctx context.Context, past int64) (int64, error) { + var g int64 + err := i.store.Pool().QueryRow(ctx, + `update assignment_generation set generation = greatest(generation, $1 + 1) returning generation`, past).Scan(&g) + if err != nil { + return 0, fmt.Errorf("raising the assignment generation past %d: %w", past, err) + } + return g, nil +} + +// SentGeneration is the generation a machine was last sent and whether that send was a put-back, by its +// id; zero for one sent without. What the mesh WOULD send is stamped with these, so it reads as byte for +// byte what it DID send when nothing else changed. +func (i *Inventory) SentGeneration(ctx context.Context, id string) (int64, bool, error) { + var g *int64 + var putBack bool + if err := i.store.Pool().QueryRow(ctx, `select sent_generation, sent_put_back from node where id = $1`, id). + Scan(&g, &putBack); err != nil { + return 0, false, fmt.Errorf("reading the generation %s was last sent: %w", id, err) + } + if g == nil { + return 0, putBack, nil + } + return *g, putBack, nil +} + +// ReadsGeneration says a machine's node-engine said it reads a generation in a declaration, by its id. +func (i *Inventory) ReadsGeneration(ctx context.Context, id string) (bool, error) { + var reads bool + if err := i.store.Pool().QueryRow(ctx, `select reads_generation from node where id = $1`, id).Scan(&reads); err != nil { + return false, fmt.Errorf("reading whether %s reads a generation: %w", id, err) + } + return reads, nil +} + +// RecordReadsGeneration keeps what a machine's latest report said of reading a generation. +func (i *Inventory) RecordReadsGeneration(ctx context.Context, id string, reads bool) error { + _, err := i.store.Pool().Exec(ctx, `update node set reads_generation = $2 where id = $1`, id, reads) + return err +} + +// Send is one declaration sent to a machine, as the mesh records it. +type Send struct { + // Node is the machine's id; NodeName its name, filled where it is read back. + Node string + NodeName string + Sequence int64 + // Epoch is the lease epoch its sender acted under, zero for one that acted under none. + Epoch int64 + // Sender is who sent it, in words: the caller, and the process and build that composed it. + Sender string + // Generation is the assignment generation it was composed from, zero when not known; PutBack whether a + // gate sent it to put the machine back. + Generation int64 + PutBack bool + // ToldGeneration and ToldPutBack are what the machine was told of them on the wire: zero and false for a + // machine whose node-engine has not said it reads a generation. What the would-send is stamped with. + ToldGeneration int64 + ToldPutBack bool + Digest string + // Modules are the modules it named. + Modules []string + SentAt time.Time + // RefusedAt is when the machine refused it for its generation, nil when it did not; RefusedApplied the + // generation the machine said it had applied then. + RefusedAt *time.Time + RefusedApplied int64 +} + +// RecordSend writes one send down, and what the machine was last sent of its generation, in one +// transaction; the oldest beyond the newest 200 for the machine are let go. +func (i *Inventory) RecordSend(ctx context.Context, s Send) error { + tx, err := i.store.Pool().Begin(ctx) + if err != nil { + return err + } + defer func() { _ = tx.Rollback(ctx) }() + if _, err := tx.Exec(ctx, `update node set sent_generation = $2, sent_put_back = $3 where id = $1`, + s.Node, nullIfZero(s.ToldGeneration), s.ToldPutBack); err != nil { + return err + } + modules := s.Modules + if modules == nil { + modules = []string{} + } + if _, err := tx.Exec(ctx, + `insert into declaration_send (node, sequence, epoch, sender, generation, put_back, digest, modules) + values ($1, $2, $3, $4, $5, $6, $7, $8)`, + s.Node, nullIfZero(s.Sequence), nullIfZero(s.Epoch), s.Sender, nullIfZero(s.Generation), s.PutBack, + s.Digest, modules); err != nil { + return err + } + if _, err := tx.Exec(ctx, + `delete from declaration_send where node = $1 and id not in + (select id from declaration_send where node = $1 order by id desc limit $2)`, s.Node, sendsKept); err != nil { + return err + } + return tx.Commit(ctx) +} + +// sendColumns are a send's columns as scanSends reads them. +const sendColumns = `s.node, n.name, coalesce(s.sequence, 0), coalesce(s.epoch, 0), s.sender, coalesce(s.generation, 0), + s.put_back, s.digest, s.modules, s.sent_at, s.refused_at, coalesce(s.refused_applied, 0)` + +func scanSends(rows pgx.Rows) ([]Send, error) { + defer rows.Close() + var out []Send + for rows.Next() { + var s Send + if err := rows.Scan(&s.Node, &s.NodeName, &s.Sequence, &s.Epoch, &s.Sender, &s.Generation, &s.PutBack, + &s.Digest, &s.Modules, &s.SentAt, &s.RefusedAt, &s.RefusedApplied); err != nil { + return nil, err + } + out = append(out, s) + } + return out, rows.Err() +} + +// LastSend is the last declaration a machine was sent, by its name; false when none was recorded. +func (i *Inventory) LastSend(ctx context.Context, name string) (Send, bool, error) { + rows, err := i.store.Pool().Query(ctx, `select `+sendColumns+` + from declaration_send s join node n on n.id = s.node + where n.name = $1 order by s.id desc limit 1`, name) + if err != nil { + return Send{}, false, err + } + sends, err := scanSends(rows) + if err != nil || len(sends) == 0 { + return Send{}, false, err + } + return sends[0], true, nil +} + +// RefusedSend keeps a machine's refusal of the send of a sequence for the generation it came from, and +// answers that send, sender and all; false when no send of that sequence was recorded — sent before the +// record, or by a hand the mesh did not see. The newest of that sequence when there are several: a +// declaration a person sent by hand may repeat one. +func (i *Inventory) RefusedSend(ctx context.Context, node string, sequence, applied int64) (Send, bool, error) { + rows, err := i.store.Pool().Query(ctx, `with refused as ( + update declaration_send set refused_at = now(), refused_applied = $3 + where id = (select id from declaration_send where node = $1 and sequence = $2 order by id desc limit 1) + returning *) + select `+sendColumns+` from refused s join node n on n.id = s.node`, node, sequence, applied) + if err != nil { + return Send{}, false, err + } + sends, err := scanSends(rows) + if err != nil || len(sends) == 0 { + return Send{}, false, err + } + return sends[0], true, nil +} + +// RefusedSendsSince is every send refused for its generation since a moment, newest first. +func (i *Inventory) RefusedSendsSince(ctx context.Context, since time.Time) ([]Send, error) { + rows, err := i.store.Pool().Query(ctx, `select `+sendColumns+` + from declaration_send s join node n on n.id = s.node + where s.refused_at >= $1 order by s.refused_at desc`, since) + if err != nil { + return nil, err + } + return scanSends(rows) +} diff --git a/internal/inventory/sends_test.go b/internal/inventory/sends_test.go new file mode 100644 index 00000000..9f5dfe6b --- /dev/null +++ b/internal/inventory/sends_test.go @@ -0,0 +1,138 @@ +package inventory + +import ( + "slices" + "testing" + "time" +) + +// The assignment generation and the record of every send (novox/hq issue 234). On 2026-10-04 a declaration +// newer in sequence than every other named four fewer modules than the assignments held, a machine applied +// it, and nothing the mesh kept could say who sent it or from what view. + +func generationNow(t *testing.T, inv *Inventory) int64 { + t.Helper() + g, err := inv.AssignmentGeneration(t.Context()) + if err != nil { + t.Fatal(err) + } + return g +} + +// **Every change to what is assigned where raises the generation, in its own transaction** — an +// assignment, an unassignment, and one taken by the machine's removal, which no writer in Go makes — and +// a repeated assignment, which changes nothing, does not. +func TestEveryAssignmentChangeRaisesTheGeneration(t *testing.T) { + inv, node := aNodeWithModules(t, "postgres", "web") + ctx := t.Context() + start := generationNow(t, inv) + if start < 1 { + t.Fatalf("the generation starts at %d; zero is \"none claimed\" on the wire", start) + } + if _, err := inv.Assign(ctx, node, "postgres"); err != nil { + t.Fatal(err) + } + assigned := generationNow(t, inv) + if assigned <= start { + t.Fatalf("an assignment left the generation at %d", assigned) + } + if _, err := inv.Assign(ctx, node, "postgres"); err != nil { + t.Fatal(err) + } + if again := generationNow(t, inv); again != assigned { + t.Errorf("an assignment repeated, which changed nothing, moved the generation from %d to %d", assigned, again) + } + if err := inv.Unassign(ctx, node, "postgres"); err != nil { + t.Fatal(err) + } + unassigned := generationNow(t, inv) + if unassigned <= assigned { + t.Fatalf("an unassignment left the generation at %d", unassigned) + } + if _, err := inv.Assign(ctx, node, "web"); err != nil { + t.Fatal(err) + } + before := generationNow(t, inv) + if _, err := inv.RemoveNodeForTest(ctx, node); err != nil { + t.Fatal(err) + } + if removed := generationNow(t, inv); removed <= before { + t.Errorf("a machine's removal took its assignments and left the generation at %d", removed) + } +} + +// **The generation never goes down, and is raised past what a machine applied when the mesh's own counter +// is behind it** — a store put back from a backup — so the next send is not refused for ever. +func TestTheGenerationIsRaisedPastWhatAMachineApplied(t *testing.T) { + inv, _ := aNodeWithModules(t) + now := generationNow(t, inv) + if raised, err := inv.RaiseAssignmentGeneration(t.Context(), now+40); err != nil || raised != now+41 { + t.Fatalf("raised past %d to %d (%v)", now+40, raised, err) + } + if raised, err := inv.RaiseAssignmentGeneration(t.Context(), 2); err != nil || raised != now+41 { + t.Fatalf("a raise below the counter moved it to %d (%v)", raised, err) + } +} + +// **Every send is recorded: its sequence, its sender, the generation it came from and the modules it +// named** — and the machine's last send is what `plan` reads, the would-send is stamped with its +// generation and put-back mark, and a refusal is kept on the send it refused, naming its sender. +func TestASendIsRecordedWithItsSenderGenerationAndModules(t *testing.T) { + inv, node := aNodeWithModules(t) + ctx := t.Context() + record, err := inv.NodeByName(ctx, node) + if err != nil { + t.Fatal(err) + } + first := Send{Node: record.ID, Sequence: 11, Epoch: 57, Sender: "the controller's daemon (pid 7 on anchor)", + Generation: 40, Digest: "d11", Modules: []string{"docker", "pacman", "sudo"}} + if err := inv.RecordSend(ctx, first); err != nil { + t.Fatal(err) + } + stale := Send{Node: record.ID, Sequence: 12, Epoch: 57, Sender: "a one-shot push by jochen at a shell on anchor", + Generation: 38, ToldGeneration: 38, Digest: "d12", Modules: []string{"docker"}} + if err := inv.RecordSend(ctx, stale); err != nil { + t.Fatal(err) + } + last, found, err := inv.LastSend(ctx, node) + if err != nil || !found { + t.Fatalf("the last send is not kept: %v", err) + } + if last.Sequence != 12 || last.Sender != stale.Sender || last.Generation != 38 || + !slices.Equal(last.Modules, []string{"docker"}) || last.SentAt.IsZero() { + t.Fatalf("the last send reads %+v", last) + } + generation, putBack, err := inv.SentGeneration(ctx, record.ID) + if err != nil || generation != 38 || putBack { + t.Fatalf("what the machine was last sent of it reads %d, %v (%v)", generation, putBack, err) + } + + refused, found, err := inv.RefusedSend(ctx, record.ID, 12, 40) + if err != nil || !found || refused.Sender != stale.Sender || refused.RefusedApplied != 40 { + t.Fatalf("the refusal is not kept on the send it refused: %+v, %v, %v", refused, found, err) + } + since, err := inv.RefusedSendsSince(ctx, time.Now().Add(-time.Hour)) + if err != nil || len(since) != 1 || since[0].NodeName != node || since[0].Sequence != 12 { + t.Fatalf("the refused sends within the hour read %+v (%v)", since, err) + } + if _, found, err := inv.RefusedSend(ctx, record.ID, 99, 40); err != nil || found { + t.Errorf("a refusal of a send never recorded was found: %v, %v", found, err) + } +} + +// **A machine that reads a generation is recorded from its reports**, and one rolled back says so no longer. +func TestWhetherAMachineReadsAGenerationIsItsLatestWord(t *testing.T) { + inv, node := aNodeWithModules(t) + record, err := inv.NodeByName(t.Context(), node) + if err != nil { + t.Fatal(err) + } + for _, reads := range []bool{false, true, false} { + if err := inv.RecordReadsGeneration(t.Context(), record.ID, reads); err != nil { + t.Fatal(err) + } + if got, err := inv.ReadsGeneration(t.Context(), record.ID); err != nil || got != reads { + t.Fatalf("recorded %v, read %v (%v)", reads, got, err) + } + } +} diff --git a/internal/link/enrolment.go b/internal/link/enrolment.go index 6c3cab6b..de36001f 100644 --- a/internal/link/enrolment.go +++ b/internal/link/enrolment.go @@ -447,6 +447,13 @@ func (e Enrolment) Heard(ctx context.Context, report Report) (news bool, err err if err := e.Inventory.RecordReadsEpoch(ctx, node.ID, report.ReadsEpoch()); err != nil { return false, err } + // And whether it reads a generation (novox/hq issue 234), the same way: only from a node-engine that + // orders its reports, so a one-shot report (a rekey) does not say an engine stopped reading one. + if report.Ordered() { + if err := e.Inventory.RecordReadsGeneration(ctx, node.ID, report.ReadsGeneration); err != nil { + return false, err + } + } if report.Ordered() { account := AccountOf(report) news, err = e.Inventory.RecordOrderedDoing(ctx, node.ID, doing, diff --git a/internal/link/generation_test.go b/internal/link/generation_test.go new file mode 100644 index 00000000..7a25283b --- /dev/null +++ b/internal/link/generation_test.go @@ -0,0 +1,33 @@ +package link + +import ( + "encoding/json" + "testing" +) + +// A report's word on the assignment generation (novox/hq issue 234), as mesh-host's report writes it: +// `reads_generation` on every report of a node-engine that reads one, and `older_generation` on a refusal +// of a declaration composed from an older generation than the machine applied. +func TestTheGenerationOnTheWire(t *testing.T) { + raw := []byte(`{"node":"anchor","refused":"older generation","declared":"d12","epoch":57,"sequence":12,` + + `"report_sequence":8,"reads_generation":true,"older_generation":{"generation":38,"applied":40}}`) + var r Report + if err := json.Unmarshal(raw, &r); err != nil { + t.Fatal(err) + } + if !r.ReadsGeneration || r.OlderGeneration == nil || *r.OlderGeneration != (GenerationRefusal{38, 40}) || + r.Sequence != 12 { + t.Fatalf("a node-engine's generation refusal reads as %+v", r) + } + // Not a refusal by the lease's epoch: S13 counts those, and this one is raised naming its sender. + if r.StaleRefusalOf() { + t.Errorf("a generation refusal reads as a stale writer's") + } + var older Report + if err := json.Unmarshal([]byte(`{"node":"anchor","applied":["a"],"report_sequence":3}`), &older); err != nil { + t.Fatal(err) + } + if older.ReadsGeneration || older.OlderGeneration != nil { + t.Fatalf("a node-engine that never said it reads a generation reads as one that does: %+v", older) + } +} diff --git a/internal/link/protocol.go b/internal/link/protocol.go index b64a9a72..63349ece 100644 --- a/internal/link/protocol.go +++ b/internal/link/protocol.go @@ -197,6 +197,15 @@ type Report struct { // RefusedOlder is how many declarations the node-engine has refused as older, ever, on every report: // a refusal whose own report was lost is still counted from the next. RefusedOlder int64 `json:"refused_older,omitempty"` + // ReadsGeneration says the node-engine reads a declaration's assignment generation and its put-back + // mark (novox/hq issue 234), on every report it makes: the mesh sends them only to a machine that said + // so, because an older node-engine refuses a key it does not know, whole. Mirrors mesh-host + // internal/link/messages.go. + ReadsGeneration bool `json:"reads_generation,omitempty"` + // OlderGeneration is set on a report refusing a declaration composed from an older assignment + // generation than the machine applied (novox/hq issue 234). The refused declaration is the report's + // own Declared and Order, and its sender is read from the record of sends by that sequence. + OlderGeneration *GenerationRefusal `json:"older_generation,omitempty"` // Held is what an adopted node found and is keeping as it was until its module is taken // (novox/hq ADR 0100). Without it an adopted node reads as converged. @@ -605,3 +614,10 @@ type HealthSaid struct { Node string `json:"node"` Health Health `json:"health"` } + +// GenerationRefusal is a declaration refused for the assignment generation it was composed from: that +// generation, and the highest the machine applied (novox/hq issue 234; mesh-host's GenerationRefusal). +type GenerationRefusal struct { + Generation int64 `json:"generation"` + Applied int64 `json:"applied"` +}