Archived
A declaration says the assignment generation it came from, and every send is recorded (issue 234) #227
@@ -514,6 +514,10 @@ func composedAndValidated(ctx context.Context, open *stores, node string, gens m
|
||||
if declared.Epoch, err = open.inventory.SentEpoch(ctx, record.ID); err != nil {
|
||||
return sendable{}, nil, err
|
||||
}
|
||||
// And the generation it was last sent, as the would-send is (novox/hq issue 234).
|
||||
if declared.Generation, declared.PutBack, err = open.inventory.SentGeneration(ctx, record.ID); err != nil {
|
||||
return sendable{}, nil, err
|
||||
}
|
||||
body, err := declared.Body()
|
||||
if err != nil {
|
||||
return sendable{}, nil, err
|
||||
|
||||
@@ -994,7 +994,8 @@ func gateFailed(ctx context.Context, open *stores, p *inventory.Plan, module str
|
||||
}
|
||||
return
|
||||
}
|
||||
sent, err := sendRollout(withScope(ctx, sendScope{modules: map[string]bool{module: true}}), open, g.Machines)
|
||||
sent, err := sendRollout(withScope(ctx, sendScope{modules: map[string]bool{module: true}, putBack: true}), open,
|
||||
g.Machines)
|
||||
if err != nil {
|
||||
notBack(fmt.Sprintf("its registered build is back at %s, and sending it to %s was refused: %v — `push %s` "+
|
||||
"sends it", short(previous.Commit), strings.Join(g.Machines, ", "), err, g.Machines[0]))
|
||||
@@ -1087,7 +1088,7 @@ func sendRollbacks(ctx context.Context, open *stores, p *inventory.Plan, b *roll
|
||||
}
|
||||
inv := open.inventory
|
||||
sort.Strings(b.machines)
|
||||
sent, err := sendRollout(withScope(ctx, sendScope{modules: b.modules}), open, b.machines)
|
||||
sent, err := sendRollout(withScope(ctx, sendScope{modules: b.modules, putBack: true}), open, b.machines)
|
||||
var back []string
|
||||
for _, r := range b.pending {
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,195 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The assignment generation a declaration was composed from, and the record of every send (novox/hq
|
||||
// issue 234).
|
||||
//
|
||||
// On 2026-10-04 a declaration newer in sequence than every other named four fewer modules than the
|
||||
// assignments held, and a machine applied it and undeclared all four. The sequence orders arrival and
|
||||
// cannot tell a later send that carries an older view of the assignments. So a declaration now carries
|
||||
// the generation of the assignments it was composed from — a counter the store raises in the same
|
||||
// transaction as every assignment change — and a machine refuses one older than it applied, unless it is
|
||||
// a gate's put-back. And every send is written down with who sent it, from which generation and naming
|
||||
// which modules: the controller logged no send then, and which process sent the stale declaration could
|
||||
// not be read back from anything the mesh kept.
|
||||
|
||||
// kindOlderGeneration is S20's kind: a machine refused a send for the generation it was composed from.
|
||||
const kindOlderGeneration = "older-generation"
|
||||
|
||||
// generationRefusalsSaid is how long a refused send is said after its refusal: an hour, as an advisory is.
|
||||
const generationRefusalsSaid = advisoryQuiet
|
||||
|
||||
// stamp gives a composed declaration the order allotted to it before it was composed: its sequence, the
|
||||
// epoch and generation when the machine reads them, and the put-back mark beside a generation — and keeps
|
||||
// the generation and acting epoch for the record of the send whether or not the machine is sent them.
|
||||
func (o order) stamp(d *sendable) {
|
||||
d.Sequence, d.Epoch = o.sequence, o.epoch
|
||||
d.composedFrom, d.actingEpoch, d.putBackSend = o.generation, o.acting, o.putBack
|
||||
d.Generation, d.PutBack = 0, false
|
||||
if o.readsGeneration && o.generation > 0 {
|
||||
d.Generation, d.PutBack = o.generation, o.putBack
|
||||
}
|
||||
}
|
||||
|
||||
// sentRecord is what the record of a send keeps beside its digest.
|
||||
type sentRecord struct {
|
||||
sequence int64
|
||||
epoch uint64
|
||||
generation int64
|
||||
putBack bool
|
||||
// told is the generation and put-back mark on the wire, which the would-send is stamped with.
|
||||
toldGeneration int64
|
||||
toldPutBack bool
|
||||
sender string
|
||||
modules []string
|
||||
}
|
||||
|
||||
// sentRecordOf is a composed send's record: its sender is the caller this process acts for.
|
||||
func sentRecordOf(ctx context.Context, d sendable) sentRecord {
|
||||
return sentRecord{sequence: d.Sequence, epoch: d.actingEpoch, generation: d.composedFrom, putBack: d.putBackSend,
|
||||
toldGeneration: d.Generation, toldPutBack: d.PutBack, sender: senderOf(callerOf(ctx)), modules: d.modules}
|
||||
}
|
||||
|
||||
// callerOf is who asked for what this process does: the seat call's caller when ctx belongs to one, the
|
||||
// caller the controller ran this command for, or the account at the shell.
|
||||
func callerOf(ctx context.Context) string {
|
||||
if c := link.CallerIn(ctx); c != "" {
|
||||
return c
|
||||
}
|
||||
return link.Caller()
|
||||
}
|
||||
|
||||
// senderOf is a send's sender in words: who asked, and the process and build that composed it — the
|
||||
// answer issue 234 could not find, because two controllers and a one-shot push were all sending then.
|
||||
func senderOf(caller string) string {
|
||||
host, _ := os.Hostname()
|
||||
return fmt.Sprintf("%s (pid %d on %s, build %s)", caller, os.Getpid(), host, version)
|
||||
}
|
||||
|
||||
// namedModules are the modules a declaration names: its machine's set, less what was left out of it.
|
||||
func namedModules(plan catalogue.Resolution, leftOut map[string]string) []string {
|
||||
out := make([]string, 0, len(plan.Modules))
|
||||
for _, m := range plan.Modules {
|
||||
if _, left := leftOut[m.Module]; !left {
|
||||
out = append(out, m.Module)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// heardGenerationRefusal keeps a machine's refusal of a send for its generation on the send it refused,
|
||||
// so S20 names its sender, and says it (novox/hq issue 234).
|
||||
//
|
||||
// **And raises the mesh's counter past what the machine applied, when the refused send was composed from
|
||||
// the counter as it stands**: then the sender's view was not stale — the counter is behind the machine,
|
||||
// which is a store put back from a backup — and every send after would be refused for ever. A stale
|
||||
// sender's generation is below the counter, and the counter is left alone for it.
|
||||
func heardGenerationRefusal(ctx context.Context, inv *inventory.Inventory, report link.Report) {
|
||||
r := report.OlderGeneration
|
||||
if r == nil || inv == nil || report.Node == "" {
|
||||
return
|
||||
}
|
||||
node, err := inv.NodeByName(ctx, report.Node)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "mesh-controller: %s refused a send for its generation, and the machine cannot be read: %v\n",
|
||||
report.Node, err)
|
||||
return
|
||||
}
|
||||
send, found, err := inv.RefusedSend(ctx, node.ID, report.Sequence, r.Applied)
|
||||
switch {
|
||||
case err != nil:
|
||||
fmt.Fprintf(os.Stderr, "mesh-controller: %s refused send %d for its generation, and the refusal could not be "+
|
||||
"kept: %v\n", report.Node, report.Sequence, err)
|
||||
case !found:
|
||||
fmt.Fprintf(os.Stderr, "mesh-controller: %s refused send %d (generation %d; it applied %d), which the mesh "+
|
||||
"has no record of sending\n", report.Node, report.Sequence, r.Generation, r.Applied)
|
||||
default:
|
||||
fmt.Fprintf(os.Stderr, "mesh-controller: %s refused send %d from %s: composed from assignment generation %d, "+
|
||||
"and it applied %d\n", report.Node, report.Sequence, send.Sender, r.Generation, r.Applied)
|
||||
}
|
||||
now, err := inv.AssignmentGeneration(ctx)
|
||||
if err != nil || r.Generation < now || r.Applied < now {
|
||||
return
|
||||
}
|
||||
raised, err := inv.RaiseAssignmentGeneration(ctx, r.Applied)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "mesh-controller: the assignment generation (%d) is behind what %s applied (%d), and "+
|
||||
"could not be raised: %v\n", now, report.Node, r.Applied, err)
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "mesh-controller: the assignment generation was %d, behind what %s applied (%d) — a store "+
|
||||
"put back from a backup — and is raised to %d, so the next send is not refused\n", now, report.Node, r.Applied,
|
||||
raised)
|
||||
}
|
||||
|
||||
// watchGenerationRefusals is S20: every send a machine refused within the hour for the generation it was
|
||||
// composed from, naming who sent it (novox/hq issue 234). One per machine, the newest refusal.
|
||||
func watchGenerationRefusals(f *signalFacts) []conditions.Observation {
|
||||
seen := map[string]bool{}
|
||||
var out []conditions.Observation
|
||||
for _, s := range f.refusedSends {
|
||||
if s.RefusedAt == nil || f.now.Sub(*s.RefusedAt) > generationRefusalsSaid || seen[s.NodeName] {
|
||||
continue
|
||||
}
|
||||
seen[s.NodeName] = true
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: s.NodeName, Kind: kindOlderGeneration,
|
||||
Machine: s.NodeName, Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("%s refused sequence %d from %s: it was composed from assignment generation %d, and "+
|
||||
"%s applied generation %d — a sender composing from a view of the assignments the mesh has moved "+
|
||||
"past; it named %s", s.NodeName, s.Sequence, s.Sender, s.Generation, s.NodeName, s.RefusedApplied,
|
||||
modulesWords(s.Modules)),
|
||||
Said: fmt.Sprintf("refused at %s", s.RefusedAt.UTC().Format(time.RFC3339)),
|
||||
Headline: fmt.Sprintf("%s refused an out-of-date update", s.NodeName),
|
||||
Explanation: fmt.Sprintf("Something sent %s an update made from an older list of what runs there. %s "+
|
||||
"refused it, so nothing was removed. Nothing for you to do unless it repeats.", s.NodeName, s.NodeName),
|
||||
Resolved: fmt.Sprintf("%s has had no out-of-date update for an hour", s.NodeName)})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// modulesWords is a send's modules as a sentence says them.
|
||||
func modulesWords(modules []string) string {
|
||||
if len(modules) == 0 {
|
||||
return "no module the mesh recorded"
|
||||
}
|
||||
return strings.Join(modules, ", ")
|
||||
}
|
||||
|
||||
// writeLastSend says what a machine was last told, by whom and from which generation (novox/hq issue 234):
|
||||
// nothing when the mesh has not recorded a send to it.
|
||||
func writeLastSend(ctx context.Context, w io.Writer, inv *inventory.Inventory, node string) error {
|
||||
s, found, err := inv.LastSend(ctx, node)
|
||||
if err != nil || !found {
|
||||
return err
|
||||
}
|
||||
generation := "no generation recorded"
|
||||
if s.Generation > 0 {
|
||||
generation = fmt.Sprintf("assignment generation %d", s.Generation)
|
||||
}
|
||||
kind := "sent"
|
||||
if s.PutBack {
|
||||
kind = "put back"
|
||||
}
|
||||
fmt.Fprintf(w, "%s was last %s sequence %d at %s by %s, composed from %s, naming %s\n", node, kind,
|
||||
s.Sequence, s.SentAt.Local().Format("2006-01-02 15:04:05"), s.Sender, generation, modulesWords(s.Modules))
|
||||
if s.RefusedAt != nil {
|
||||
fmt.Fprintf(w, " and refused it at %s: it had applied generation %d\n",
|
||||
s.RefusedAt.Local().Format("2006-01-02 15:04:05"), s.RefusedApplied)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// The assignment generation a declaration was composed from (novox/hq issue 234).
|
||||
|
||||
func bodyKeys(t *testing.T, s sendable) map[string]any {
|
||||
t.Helper()
|
||||
raw, err := s.Body()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var keys map[string]any
|
||||
if err := json.Unmarshal(raw, &keys); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
// **The generation goes on the wire only to a machine whose node-engine said it reads one**: an older
|
||||
// node-engine decodes strictly and refuses an unknown key, whole. And the put-back mark only with it.
|
||||
func TestTheGenerationIsSentOnlyToAMachineThatReadsOne(t *testing.T) {
|
||||
resources := []map[string]any{{"id": "a", "type": "file", "path": "/etc/a", "content": "x\n"}}
|
||||
reads := order{sequence: 12, epoch: 57, generation: 40, readsGeneration: true, acting: 57, putBack: true}
|
||||
var told sendable
|
||||
told.Resources = resources
|
||||
reads.stamp(&told)
|
||||
keys := bodyKeys(t, told)
|
||||
if keys["generation"] != float64(40) || keys["put_back"] != true || keys["sequence"] != float64(12) {
|
||||
t.Fatalf("a machine that reads a generation was sent %v", keys)
|
||||
}
|
||||
if told.composedFrom != 40 || told.actingEpoch != 57 {
|
||||
t.Errorf("the send does not keep what it was composed from for its record: %+v", told)
|
||||
}
|
||||
|
||||
older := order{sequence: 12, epoch: 57, generation: 40, readsGeneration: false, acting: 57, putBack: true}
|
||||
var untold sendable
|
||||
untold.Resources = resources
|
||||
older.stamp(&untold)
|
||||
keys = bodyKeys(t, untold)
|
||||
if _, there := keys["generation"]; there {
|
||||
t.Fatalf("a machine whose node-engine never said it reads a generation was sent one: %v", keys)
|
||||
}
|
||||
if _, there := keys["put_back"]; there {
|
||||
t.Fatalf("a machine whose node-engine never said it reads a generation was sent a put-back mark: %v", keys)
|
||||
}
|
||||
if untold.composedFrom != 40 {
|
||||
t.Errorf("the generation it was composed from is recorded whether or not it was sent: %+v", untold)
|
||||
}
|
||||
|
||||
// An ordinary send carries no put-back mark at all: the body is what it was apart from the generation.
|
||||
var ordinary sendable
|
||||
ordinary.Resources = resources
|
||||
ordinaryOrder := reads
|
||||
ordinaryOrder.putBack = false
|
||||
ordinaryOrder.stamp(&ordinary)
|
||||
if _, there := bodyKeys(t, ordinary)["put_back"]; there {
|
||||
t.Fatal("an ordinary send says it is a put-back")
|
||||
}
|
||||
}
|
||||
|
||||
// **The sender is named**: the caller of the seat call when there is one, else the shell's account, and the
|
||||
// process and build that composed it.
|
||||
func TestASendNamesItsSender(t *testing.T) {
|
||||
said := senderOf("g14.node-tools, through the mesh-controller seat")
|
||||
if !strings.Contains(said, "g14.node-tools") || !strings.Contains(said, "pid ") || !strings.Contains(said, "build ") {
|
||||
t.Fatalf("the sender reads %q", said)
|
||||
}
|
||||
}
|
||||
|
||||
// refusedSend is a send a machine refused for its generation at a moment.
|
||||
func refusedSend(at time.Time) inventory.Send {
|
||||
return inventory.Send{NodeName: "anchor", Sequence: 12, Epoch: 57, Generation: 38, RefusedApplied: 40,
|
||||
Sender: "a one-shot push by jochen at a shell on anchor (pid 4242 on anchor, build 2026.10.11)",
|
||||
Modules: []string{"docker"}, SentAt: at.Add(-time.Second), RefusedAt: &at}
|
||||
}
|
||||
|
||||
// **A refused send is raised naming its sender** (novox/hq issue 234): who sent it, from which generation,
|
||||
// against which the machine applied, and its sequence — the facts that took a morning to look for.
|
||||
func TestARefusedSendIsRaisedNamingItsSender(t *testing.T) {
|
||||
now := time.Date(2026, 10, 11, 12, 0, 0, 0, time.UTC)
|
||||
f := calm(now)
|
||||
f.refusedSends = []inventory.Send{refusedSend(now.Add(-time.Minute))}
|
||||
got := watchGenerationRefusals(f)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
o := got[0]
|
||||
if o.Key() != "machine.anchor.older-generation" || o.Machine != "anchor" {
|
||||
t.Errorf("raised as %s about %q", o.Key(), o.Machine)
|
||||
}
|
||||
for _, want := range []string{"a one-shot push by jochen", "generation 38", "generation 40", "sequence 12"} {
|
||||
if !strings.Contains(o.Summary, want) {
|
||||
t.Errorf("the summary does not say %q: %s", want, o.Summary)
|
||||
}
|
||||
}
|
||||
if o.Headline == "" || o.Explanation == "" || o.Resolved == "" {
|
||||
t.Errorf("the condition is not worded for the operator: %+v", o)
|
||||
}
|
||||
}
|
||||
@@ -128,7 +128,7 @@ func (r *recordedDelivery) grant(context.Context, []readyNode) error { return ni
|
||||
|
||||
func (r *recordedDelivery) declare(ctx context.Context, s readyNode, body []byte) (string, error) {
|
||||
r.declared = append(r.declared, s.node)
|
||||
return recordSent(ctx, r.inv, s.node, body, s.declared.Builds, s.declared.Epoch)
|
||||
return recordSent(ctx, r.inv, s.node, body, s.declared.Builds, s.declared.Epoch, sentRecordOf(ctx, s.declared))
|
||||
}
|
||||
|
||||
// aResolver is a module built from a repository, at a commit, with something on the machine that
|
||||
|
||||
@@ -77,7 +77,7 @@ func TestASendIsRecordedEvenWhenTheSenderIsBeingCancelled(t *testing.T) {
|
||||
}
|
||||
cancel() // the sender is going away: its context is cancelled between the send and the record
|
||||
body := []byte(`{"declaration":1,"resources":[]}`)
|
||||
digest, err := recordSent(ctx, inv, "anchor", body, nil, 0)
|
||||
digest, err := recordSent(ctx, inv, "anchor", body, nil, 0, sentRecord{sender: "a test"})
|
||||
if err != nil {
|
||||
// NodeByName on the cancelled context may itself refuse; the record must still be possible
|
||||
// through the detached context, so look the node up again on a live one.
|
||||
|
||||
@@ -420,7 +420,8 @@ func declarationWith(ctx context.Context, open *stores, node string,
|
||||
out := sendable{Resources: composed.Resources, Adoption: adoption,
|
||||
Received: composed.Received, Mesh: with.Mesh, BusUsers: with.BusUsers,
|
||||
LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut, withheld: with.Withheld,
|
||||
unbound: with.Unbound, foreseen: composed.Foreseen, unplaced: composed.Unplaced}
|
||||
unbound: with.Unbound, foreseen: composed.Foreseen, unplaced: composed.Unplaced,
|
||||
modules: namedModules(plan, composed.LeftOut)}
|
||||
// And which build of each module it carries, for the send to record (novox/hq issue 259, ADR
|
||||
// 0221). Read only on the send path: a question about what would be sent records nothing.
|
||||
if choosing == Allocating {
|
||||
@@ -1290,6 +1291,11 @@ func planCommand(ctx context.Context, args []string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// What the machine was last told, by whom and from which assignment generation (novox/hq issue 234),
|
||||
// beside what it would be told now. A record that cannot be read is said, not taken for none.
|
||||
if err := writeLastSend(ctx, os.Stdout, open.inventory, args[0]); err != nil {
|
||||
fmt.Printf("what %s was last sent could not be read: %v\n", args[0], err)
|
||||
}
|
||||
// Which modules a push would leave out, and why — said before the plan, since the plan is of
|
||||
// what the machine would be told (novox/hq ADR 0163, rule 6). Judged, never composed: `plan`
|
||||
// without --json allocates nothing.
|
||||
|
||||
@@ -356,10 +356,18 @@ func declare(ctx context.Context, args []string) error {
|
||||
// the mesh did not compose it, so a push that does not name this machine treats it as held.
|
||||
// The epoch it carried, if a person wrote one in, is what the machine heard.
|
||||
var carried struct {
|
||||
Epoch uint64 `json:"epoch"`
|
||||
Epoch uint64 `json:"epoch"`
|
||||
Sequence int64 `json:"sequence"`
|
||||
Generation int64 `json:"generation"`
|
||||
PutBack bool `json:"put_back"`
|
||||
}
|
||||
_ = json.Unmarshal(raw, &carried)
|
||||
if _, err := recordSent(ctx, inv, node, raw, nil, carried.Epoch); err != nil {
|
||||
// And recorded as every send is (novox/hq issue 234): by hand, from what it carried; the modules it
|
||||
// named are not known, since the mesh did not compose it.
|
||||
if _, err := recordSent(ctx, inv, node, raw, nil, carried.Epoch, sentRecord{sequence: carried.Sequence,
|
||||
epoch: carried.Epoch, generation: carried.Generation, putBack: carried.PutBack,
|
||||
toldGeneration: carried.Generation, toldPutBack: carried.PutBack,
|
||||
sender: senderOf(callerOf(ctx)) + ", a declaration sent by hand"}); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("sent %s a signed declaration (%d bytes)\n", node, len(raw))
|
||||
@@ -798,7 +806,7 @@ func composeEach(names []string, allot func(node string) (order, error),
|
||||
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
|
||||
continue
|
||||
}
|
||||
declared.Sequence, declared.Epoch = numbered.sequence, numbered.epoch
|
||||
numbered.stamp(&declared)
|
||||
if len(declared.Resources) == 0 {
|
||||
// Sent, not skipped (novox/hq issue 127). A node whose declaration composes to
|
||||
// nothing may have HELD something before — the broker opening a placement gave it,
|
||||
@@ -1016,7 +1024,8 @@ func (b overTheBus) declare(ctx context.Context, s readyNode, body []byte) (stri
|
||||
}
|
||||
// After it is away, not before. A digest recorded for something that failed to send would make
|
||||
// the machine look current for a declaration it never received.
|
||||
digest, err := recordSent(ctx, b.open.inventory, s.node, body, s.declared.Builds, s.declared.Epoch)
|
||||
digest, err := recordSent(ctx, b.open.inventory, s.node, body, s.declared.Builds, s.declared.Epoch,
|
||||
sentRecordOf(ctx, s.declared))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -1228,7 +1237,7 @@ func sendToEach(ctx context.Context, open *stores, names []string) ([]string, er
|
||||
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
|
||||
continue
|
||||
}
|
||||
declared.Sequence, declared.Epoch = numbered.sequence, numbered.epoch
|
||||
numbered.stamp(&declared)
|
||||
reportLeftOut(name, declared)
|
||||
sending = append(sending, readyNode{name, declared})
|
||||
}
|
||||
@@ -1393,6 +1402,11 @@ func wouldSendFrom(ctx context.Context, open *stores,
|
||||
if declared.Epoch, err = open.inventory.SentEpoch(ctx, n.ID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// And the generation and put-back mark it was last sent, for the same reason (novox/hq issue 234):
|
||||
// an assignment elsewhere in the mesh is not a change of this machine.
|
||||
if declared.Generation, declared.PutBack, err = open.inventory.SentGeneration(ctx, n.ID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
body, err := declared.Body()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -1531,6 +1545,14 @@ var epochForActs = func(ctx context.Context) (uint64, error) { return theLease.e
|
||||
type order struct {
|
||||
sequence int64
|
||||
epoch uint64
|
||||
// generation is the assignment generation read before the composition, and readsGeneration whether the
|
||||
// machine's node-engine said it reads one (novox/hq issue 234); acting is the lease epoch the sender acts
|
||||
// under, whether or not the machine is sent it — both kept for the record of the send.
|
||||
generation int64
|
||||
readsGeneration bool
|
||||
acting uint64
|
||||
// putBack is a gate's put-back (sendScope.putBack), read from the send's context.
|
||||
putBack bool
|
||||
}
|
||||
|
||||
// allot takes the next sequence for a machine — the number its next declaration carries — under the
|
||||
@@ -1544,6 +1566,7 @@ func allot(ctx context.Context, inv *inventory.Inventory, node string) (order, e
|
||||
if err != nil {
|
||||
return order{}, err
|
||||
}
|
||||
acting := epoch
|
||||
if epoch > 0 {
|
||||
reads, err := inv.ReadsEpoch(ctx, record.ID)
|
||||
if err != nil {
|
||||
@@ -1553,11 +1576,24 @@ func allot(ctx context.Context, inv *inventory.Inventory, node string) (order, e
|
||||
epoch = 0
|
||||
}
|
||||
}
|
||||
// **The generation is read here, before the composition reads a single assignment** (novox/hq issue
|
||||
// 234). A generation only grows, so one read before is never newer than the view composed after it:
|
||||
// the declaration may claim a generation older than its content, never newer — and a claim newer than
|
||||
// the content is exactly the stale send the machine must be able to refuse.
|
||||
generation, err := inv.AssignmentGeneration(ctx)
|
||||
if err != nil {
|
||||
return order{}, err
|
||||
}
|
||||
readsGeneration, err := inv.ReadsGeneration(ctx, record.ID)
|
||||
if err != nil {
|
||||
return order{}, err
|
||||
}
|
||||
seq, err := inv.NextSequence(ctx, record.ID)
|
||||
if err != nil {
|
||||
return order{}, err
|
||||
}
|
||||
return order{sequence: seq, epoch: epoch}, nil
|
||||
return order{sequence: seq, epoch: epoch, generation: generation, readsGeneration: readsGeneration,
|
||||
acting: acting, putBack: scopeOf(ctx).putBack}, nil
|
||||
}
|
||||
|
||||
// recordSent writes down what a machine was just sent, and returns the digest.
|
||||
@@ -1572,7 +1608,7 @@ func allot(ctx context.Context, inv *inventory.Inventory, node string) (order, e
|
||||
// And the build of each module it carried (novox/hq issue 259, ADR 0221), nil when that is not known:
|
||||
// what tells a machine held back by a policy or a plan from one a push left behind.
|
||||
func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body []byte,
|
||||
builds map[string]string, epoch uint64) (string, error) {
|
||||
builds map[string]string, epoch uint64, sent sentRecord) (string, error) {
|
||||
kept, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
defer cancel()
|
||||
record, err := inv.NodeByName(kept, node)
|
||||
@@ -1583,6 +1619,14 @@ func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body
|
||||
if err := inv.RecordSentUnder(kept, record.ID, digest, builds, epoch); err != nil {
|
||||
return "", err
|
||||
}
|
||||
// And the send itself, who sent it and from which generation (novox/hq issue 234): what the machine
|
||||
// was last told, by whom, is read back from here — and what the would-send is stamped with.
|
||||
if err := inv.RecordSend(kept, inventory.Send{Node: record.ID, Sequence: sent.sequence,
|
||||
Epoch: int64(sent.epoch), Sender: sent.sender, Generation: sent.generation, PutBack: sent.putBack,
|
||||
ToldGeneration: sent.toldGeneration, ToldPutBack: sent.toldPutBack, Digest: digest,
|
||||
Modules: sent.modules}); err != nil {
|
||||
return "", fmt.Errorf("%s was sent its declaration, and the send could not be recorded: %w", node, err)
|
||||
}
|
||||
// And what it was, summarised, so the next push can be compared with it (novox/hq ADR 0217).
|
||||
// Never a reason to fail a send that is already away: a summary that cannot be kept means the
|
||||
// next comparison has nothing to hold against, which is how every machine starts.
|
||||
|
||||
@@ -50,6 +50,9 @@ type sendScope struct {
|
||||
modules map[string]bool
|
||||
// person is a person's act: it carries what it carries.
|
||||
person bool
|
||||
// putBack is a gate putting machines back after a failed send (novox/hq issue 234): its declarations
|
||||
// say so, and a machine does not refuse them for the generation they carry.
|
||||
putBack bool
|
||||
}
|
||||
|
||||
type sendScopeKey struct{}
|
||||
|
||||
@@ -27,6 +27,25 @@ type sendable struct {
|
||||
// said it reads one is sent none, because an older node-engine refuses a key it does not know, whole
|
||||
// (link/order.go, the contract).
|
||||
Epoch uint64
|
||||
// Generation is the assignment generation it was composed from (novox/hq issue 234): a machine that
|
||||
// applied a later one refuses it, so a send composed from a view of the assignments the mesh has moved
|
||||
// past cannot undeclare what is still assigned. Zero is not sent at all — every machine whose
|
||||
// node-engine has not said it reads one is sent none, for the reason the epoch is not (an older
|
||||
// node-engine refuses a key it does not know, whole).
|
||||
Generation int64
|
||||
// PutBack marks a gate's put-back (novox/hq issue 234): it carries the generation of what it puts
|
||||
// back and is not refused for it. Sent only beside a generation.
|
||||
PutBack bool
|
||||
// composedFrom is the generation read before this declaration was composed, and actingEpoch the
|
||||
// lease epoch its sender acted under — whether or not the machine is sent either — for the record of
|
||||
// the send; never on the wire.
|
||||
composedFrom int64
|
||||
actingEpoch uint64
|
||||
// putBackSend is whether a gate sent it to put the machine back, for the record; PutBack is the mark
|
||||
// on the wire, only beside a generation.
|
||||
putBackSend bool
|
||||
// modules are the modules this declaration names, for the record of the send; never on the wire.
|
||||
modules []string
|
||||
// Adoption is nil for a converged node, and then the body is byte for byte what it was before
|
||||
// adoption existed: an older host parses the envelope strictly and would refuse the key.
|
||||
Adoption *adoptionEnvelope
|
||||
@@ -94,6 +113,12 @@ func (s sendable) Body() ([]byte, error) {
|
||||
if len(s.LeftOut) > 0 {
|
||||
envelope["left_out"] = s.LeftOut
|
||||
}
|
||||
if s.Generation > 0 {
|
||||
envelope["generation"] = s.Generation
|
||||
if s.PutBack {
|
||||
envelope["put_back"] = true
|
||||
}
|
||||
}
|
||||
// An empty declaration is deliberate here — the node owns nothing the mesh put there
|
||||
// (novox/hq issue 127) — and the host refuses an empty body unless it is told the emptiness
|
||||
// is meant, so a truncated or mis-composed body is never mistaken for "own nothing".
|
||||
|
||||
@@ -226,6 +226,20 @@ var signalsTable = []signalRow{
|
||||
newest: func(f *signalFacts) time.Time {
|
||||
return newestOf(f.batches, func(b batchFacts) time.Time { return b.closed })
|
||||
}},
|
||||
{Row: "S20", Signal: "a send refused for the assignment generation it was composed from",
|
||||
Emitter: "node-engine", Trigger: "each refusal (novox/hq issue 234)",
|
||||
Bound: "none: raised at the first refusal, naming its sender, the generation it came from and the one the " +
|
||||
"machine applied; cleared an hour after the last",
|
||||
Kind: kindOlderGeneration, Severity: conditions.Warning, Phase: 2,
|
||||
needs: func(f *signalFacts) error { return f.refusedSendsErr }, watch: watchGenerationRefusals,
|
||||
newest: func(f *signalFacts) time.Time {
|
||||
return newestOf(f.refusedSends, func(s inventory.Send) time.Time {
|
||||
if s.RefusedAt == nil {
|
||||
return time.Time{}
|
||||
}
|
||||
return *s.RefusedAt
|
||||
})
|
||||
}},
|
||||
{Row: "S17", Signal: "a send held for the bus's planned step is told to a person", Emitter: "controller's plan",
|
||||
Trigger: "each send refused because it would replace the bus outside its step (novox/hq issue 336)",
|
||||
Bound: "none: raised at the first refusal, for the operator, naming what waits, the bus build from and to, " +
|
||||
|
||||
@@ -178,6 +178,14 @@ var suppressions = map[string]suppression{
|
||||
commit: "c0ffee001122", modules: []string{"app"}, since: f.now.Add(-time.Second)}}}
|
||||
},
|
||||
},
|
||||
// A send refused by its machine for the generation it was composed from (novox/hq issue 234): said at
|
||||
// once, naming its sender, for an hour after. Inside: the last such refusal was more than an hour ago.
|
||||
"S20": {
|
||||
inside: func(f *signalFacts) {
|
||||
f.refusedSends = []inventory.Send{refusedSend(f.now.Add(-61 * time.Minute))}
|
||||
},
|
||||
past: func(f *signalFacts) { f.refusedSends = []inventory.Send{refusedSend(f.now.Add(-time.Second))} },
|
||||
},
|
||||
// Twice by hand within a fortnight is a healer wanted; once, or the first of two a day too old, is not.
|
||||
"S15": {
|
||||
inside: func(f *signalFacts) {
|
||||
|
||||
@@ -196,6 +196,11 @@ func (l nudgingListener) Heard(ctx context.Context, report link.Report) (bool, e
|
||||
if report.Ordered() {
|
||||
link.StaleRefusals.Lifetime(report.Node, report.RefusedOlder, now)
|
||||
}
|
||||
// A send refused for the generation it was composed from is kept on the send it refused, so S20 names
|
||||
// its sender (novox/hq issue 234).
|
||||
if report.OlderGeneration != nil {
|
||||
heardGenerationRefusal(ctx, l.Enrolment.Inventory, report)
|
||||
}
|
||||
// What the machine's witnesses put back and stand by (novox/hq ADR 0236): read by the gate and its
|
||||
// probe. Only from an account of the machine — not a word that a declaration was set aside, nor a rekey.
|
||||
if report.Superseded == "" && report.Rekey == nil && report.Node != "" {
|
||||
|
||||
@@ -55,6 +55,10 @@ type signalFacts struct {
|
||||
waits []waitFacts
|
||||
// batches are the batches not yet cut (S18, S19, novox/hq ADR 0276).
|
||||
batches []batchFacts
|
||||
// refusedSends are the sends a machine refused within the hour for the generation they were composed
|
||||
// from, each naming its sender (S20, novox/hq issue 234).
|
||||
refusedSends []inventory.Send
|
||||
refusedSendsErr error
|
||||
|
||||
loop loopFacts
|
||||
loopErr error
|
||||
@@ -364,6 +368,7 @@ func (w *watchdogs) gather(ctx context.Context) *signalFacts {
|
||||
}
|
||||
}
|
||||
f.handActs, f.handActsErr = w.gatherHandActs(ctx, now)
|
||||
f.refusedSends, f.refusedSendsErr = inv.RefusedSendsSince(ctx, now.Add(-generationRefusalsSaid))
|
||||
f.facts.taken, _, f.facts.began, f.facts.err = exportedFacts.last()
|
||||
return f
|
||||
}
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
-- A declaration says the assignment generation it was composed from, and every send is recorded
|
||||
-- (novox/hq issue 234).
|
||||
--
|
||||
-- On 2026-10-04 a declaration newer in sequence than every other named four fewer modules than the
|
||||
-- assignments held, and the machine undeclared all four. The sequence orders arrival; it cannot tell a
|
||||
-- later send that carries an older view of the assignments. And nothing the mesh kept said which process
|
||||
-- sent it, from what view: the node row holds only the digest of the last send.
|
||||
|
||||
-- The assignment generation: one counter for the mesh, raised in the same transaction as every change to
|
||||
-- what is assigned where. A trigger, not a line in each writer: an assignment is also taken by a node's
|
||||
-- removal (on delete cascade) and renamed with its module (on update cascade), which no writer in Go
|
||||
-- sees, and a rule kept only by the writers that remember it is the rule this issue found broken. Seeded
|
||||
-- at 1, so every declaration composed after this claims a generation and none claims zero ("none").
|
||||
create table assignment_generation (
|
||||
one boolean primary key default true check (one),
|
||||
generation bigint not null
|
||||
);
|
||||
insert into assignment_generation (one, generation) values (true, 1);
|
||||
|
||||
create function raise_assignment_generation() returns trigger language plpgsql as $$
|
||||
begin
|
||||
update assignment_generation set generation = generation + 1;
|
||||
return null;
|
||||
end
|
||||
$$;
|
||||
|
||||
-- Per row, so a statement that changes nothing (an assignment repeated, `on conflict do nothing`) raises
|
||||
-- nothing; a statement that changes several raises once per row, which only ever moves it forward.
|
||||
create trigger assignment_generation_raised
|
||||
after insert or update or delete on assignment
|
||||
for each row execute function raise_assignment_generation();
|
||||
|
||||
-- What a machine was last sent of it: the generation, and whether that send was a gate's put-back, so
|
||||
-- what the mesh WOULD send is stamped the same way and reads as byte for byte what it DID send when
|
||||
-- nothing else changed (as sent_epoch, migration 0068). Null and false for a send without.
|
||||
alter table node add column sent_generation bigint;
|
||||
alter table node add column sent_put_back boolean not null default false;
|
||||
-- Whether the machine's node-engine said it reads a generation (its reports' `reads_generation`): until
|
||||
-- it has, it is sent none, because an older node-engine refuses a key it does not know, whole.
|
||||
alter table node add column reads_generation boolean not null default false;
|
||||
|
||||
-- Every send: the machine, its sequence, who sent it — the lease epoch the sender acted under and the
|
||||
-- caller, process and build — the generation it was composed from, whether it was a put-back, its
|
||||
-- digest and the modules it named. Written after the declaration is away, as the node row's record is.
|
||||
-- A refusal by the machine of a send for its generation is kept on the send it refused, so the
|
||||
-- condition it raises names the sender from here. The newest 200 per machine are kept.
|
||||
create table declaration_send (
|
||||
id bigserial primary key,
|
||||
node uuid not null references node (id) on delete cascade,
|
||||
sequence bigint,
|
||||
epoch bigint,
|
||||
sender text not null,
|
||||
generation bigint,
|
||||
put_back boolean not null default false,
|
||||
digest text not null,
|
||||
modules text[] not null default '{}',
|
||||
sent_at timestamptz not null default now(),
|
||||
refused_at timestamptz,
|
||||
-- refused_applied is the generation the machine said it had applied when it refused this send.
|
||||
refused_applied bigint
|
||||
);
|
||||
create index declaration_send_node on declaration_send (node, id desc);
|
||||
create index declaration_send_sequence on declaration_send (node, sequence);
|
||||
create index declaration_send_refused on declaration_send (refused_at) where refused_at is not null;
|
||||
@@ -0,0 +1,202 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// The assignment generation and the record of every send (novox/hq issue 234, migration 0094).
|
||||
//
|
||||
// A declaration carries the generation of the assignments it was composed from, and a machine refuses one
|
||||
// composed from an older generation than it applied — the sequence orders arrival and cannot tell a later
|
||||
// send carrying an older view. And every send is written down with who sent it, from which generation and
|
||||
// naming which modules, so what a machine was last told, by whom, is read back rather than guessed at.
|
||||
|
||||
// sendsKept is how many sends are kept per machine: the newest. Enough to read back a morning of pushes
|
||||
// on a busy machine; the record is for finding who sent what, not an archive.
|
||||
const sendsKept = 200
|
||||
|
||||
// AssignmentGeneration is the mesh's assignment generation now: raised by the store itself in the same
|
||||
// transaction as every change to what is assigned where (a trigger on the assignment table).
|
||||
func (i *Inventory) AssignmentGeneration(ctx context.Context) (int64, error) {
|
||||
var g int64
|
||||
if err := i.store.Pool().QueryRow(ctx, `select generation from assignment_generation`).Scan(&g); err != nil {
|
||||
return 0, fmt.Errorf("reading the assignment generation: %w", err)
|
||||
}
|
||||
return g, nil
|
||||
}
|
||||
|
||||
// RaiseAssignmentGeneration raises the generation past one a machine applied, and answers it: one more
|
||||
// than that, or what it already was when it is past it. It never lowers it.
|
||||
//
|
||||
// For one case only: a machine refused a send composed from the generation the mesh holds now, so the
|
||||
// machine applied a higher one than the mesh's counter — a store put back from a backup. Every send after
|
||||
// would be refused for ever; raised past it, the next send carries a generation the machine takes, composed
|
||||
// from the assignments the store holds now. A stale sender never meets this: its generation is below the
|
||||
// counter, and the counter is not touched for it.
|
||||
func (i *Inventory) RaiseAssignmentGeneration(ctx context.Context, past int64) (int64, error) {
|
||||
var g int64
|
||||
err := i.store.Pool().QueryRow(ctx,
|
||||
`update assignment_generation set generation = greatest(generation, $1 + 1) returning generation`, past).Scan(&g)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("raising the assignment generation past %d: %w", past, err)
|
||||
}
|
||||
return g, nil
|
||||
}
|
||||
|
||||
// SentGeneration is the generation a machine was last sent and whether that send was a put-back, by its
|
||||
// id; zero for one sent without. What the mesh WOULD send is stamped with these, so it reads as byte for
|
||||
// byte what it DID send when nothing else changed.
|
||||
func (i *Inventory) SentGeneration(ctx context.Context, id string) (int64, bool, error) {
|
||||
var g *int64
|
||||
var putBack bool
|
||||
if err := i.store.Pool().QueryRow(ctx, `select sent_generation, sent_put_back from node where id = $1`, id).
|
||||
Scan(&g, &putBack); err != nil {
|
||||
return 0, false, fmt.Errorf("reading the generation %s was last sent: %w", id, err)
|
||||
}
|
||||
if g == nil {
|
||||
return 0, putBack, nil
|
||||
}
|
||||
return *g, putBack, nil
|
||||
}
|
||||
|
||||
// ReadsGeneration says a machine's node-engine said it reads a generation in a declaration, by its id.
|
||||
func (i *Inventory) ReadsGeneration(ctx context.Context, id string) (bool, error) {
|
||||
var reads bool
|
||||
if err := i.store.Pool().QueryRow(ctx, `select reads_generation from node where id = $1`, id).Scan(&reads); err != nil {
|
||||
return false, fmt.Errorf("reading whether %s reads a generation: %w", id, err)
|
||||
}
|
||||
return reads, nil
|
||||
}
|
||||
|
||||
// RecordReadsGeneration keeps what a machine's latest report said of reading a generation.
|
||||
func (i *Inventory) RecordReadsGeneration(ctx context.Context, id string, reads bool) error {
|
||||
_, err := i.store.Pool().Exec(ctx, `update node set reads_generation = $2 where id = $1`, id, reads)
|
||||
return err
|
||||
}
|
||||
|
||||
// Send is one declaration sent to a machine, as the mesh records it.
|
||||
type Send struct {
|
||||
// Node is the machine's id; NodeName its name, filled where it is read back.
|
||||
Node string
|
||||
NodeName string
|
||||
Sequence int64
|
||||
// Epoch is the lease epoch its sender acted under, zero for one that acted under none.
|
||||
Epoch int64
|
||||
// Sender is who sent it, in words: the caller, and the process and build that composed it.
|
||||
Sender string
|
||||
// Generation is the assignment generation it was composed from, zero when not known; PutBack whether a
|
||||
// gate sent it to put the machine back.
|
||||
Generation int64
|
||||
PutBack bool
|
||||
// ToldGeneration and ToldPutBack are what the machine was told of them on the wire: zero and false for a
|
||||
// machine whose node-engine has not said it reads a generation. What the would-send is stamped with.
|
||||
ToldGeneration int64
|
||||
ToldPutBack bool
|
||||
Digest string
|
||||
// Modules are the modules it named.
|
||||
Modules []string
|
||||
SentAt time.Time
|
||||
// RefusedAt is when the machine refused it for its generation, nil when it did not; RefusedApplied the
|
||||
// generation the machine said it had applied then.
|
||||
RefusedAt *time.Time
|
||||
RefusedApplied int64
|
||||
}
|
||||
|
||||
// RecordSend writes one send down, and what the machine was last sent of its generation, in one
|
||||
// transaction; the oldest beyond the newest 200 for the machine are let go.
|
||||
func (i *Inventory) RecordSend(ctx context.Context, s Send) error {
|
||||
tx, err := i.store.Pool().Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
if _, err := tx.Exec(ctx, `update node set sent_generation = $2, sent_put_back = $3 where id = $1`,
|
||||
s.Node, nullIfZero(s.ToldGeneration), s.ToldPutBack); err != nil {
|
||||
return err
|
||||
}
|
||||
modules := s.Modules
|
||||
if modules == nil {
|
||||
modules = []string{}
|
||||
}
|
||||
if _, err := tx.Exec(ctx,
|
||||
`insert into declaration_send (node, sequence, epoch, sender, generation, put_back, digest, modules)
|
||||
values ($1, $2, $3, $4, $5, $6, $7, $8)`,
|
||||
s.Node, nullIfZero(s.Sequence), nullIfZero(s.Epoch), s.Sender, nullIfZero(s.Generation), s.PutBack,
|
||||
s.Digest, modules); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(ctx,
|
||||
`delete from declaration_send where node = $1 and id not in
|
||||
(select id from declaration_send where node = $1 order by id desc limit $2)`, s.Node, sendsKept); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
|
||||
// sendColumns are a send's columns as scanSends reads them.
|
||||
const sendColumns = `s.node, n.name, coalesce(s.sequence, 0), coalesce(s.epoch, 0), s.sender, coalesce(s.generation, 0),
|
||||
s.put_back, s.digest, s.modules, s.sent_at, s.refused_at, coalesce(s.refused_applied, 0)`
|
||||
|
||||
func scanSends(rows pgx.Rows) ([]Send, error) {
|
||||
defer rows.Close()
|
||||
var out []Send
|
||||
for rows.Next() {
|
||||
var s Send
|
||||
if err := rows.Scan(&s.Node, &s.NodeName, &s.Sequence, &s.Epoch, &s.Sender, &s.Generation, &s.PutBack,
|
||||
&s.Digest, &s.Modules, &s.SentAt, &s.RefusedAt, &s.RefusedApplied); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// LastSend is the last declaration a machine was sent, by its name; false when none was recorded.
|
||||
func (i *Inventory) LastSend(ctx context.Context, name string) (Send, bool, error) {
|
||||
rows, err := i.store.Pool().Query(ctx, `select `+sendColumns+`
|
||||
from declaration_send s join node n on n.id = s.node
|
||||
where n.name = $1 order by s.id desc limit 1`, name)
|
||||
if err != nil {
|
||||
return Send{}, false, err
|
||||
}
|
||||
sends, err := scanSends(rows)
|
||||
if err != nil || len(sends) == 0 {
|
||||
return Send{}, false, err
|
||||
}
|
||||
return sends[0], true, nil
|
||||
}
|
||||
|
||||
// RefusedSend keeps a machine's refusal of the send of a sequence for the generation it came from, and
|
||||
// answers that send, sender and all; false when no send of that sequence was recorded — sent before the
|
||||
// record, or by a hand the mesh did not see. The newest of that sequence when there are several: a
|
||||
// declaration a person sent by hand may repeat one.
|
||||
func (i *Inventory) RefusedSend(ctx context.Context, node string, sequence, applied int64) (Send, bool, error) {
|
||||
rows, err := i.store.Pool().Query(ctx, `with refused as (
|
||||
update declaration_send set refused_at = now(), refused_applied = $3
|
||||
where id = (select id from declaration_send where node = $1 and sequence = $2 order by id desc limit 1)
|
||||
returning *)
|
||||
select `+sendColumns+` from refused s join node n on n.id = s.node`, node, sequence, applied)
|
||||
if err != nil {
|
||||
return Send{}, false, err
|
||||
}
|
||||
sends, err := scanSends(rows)
|
||||
if err != nil || len(sends) == 0 {
|
||||
return Send{}, false, err
|
||||
}
|
||||
return sends[0], true, nil
|
||||
}
|
||||
|
||||
// RefusedSendsSince is every send refused for its generation since a moment, newest first.
|
||||
func (i *Inventory) RefusedSendsSince(ctx context.Context, since time.Time) ([]Send, error) {
|
||||
rows, err := i.store.Pool().Query(ctx, `select `+sendColumns+`
|
||||
from declaration_send s join node n on n.id = s.node
|
||||
where s.refused_at >= $1 order by s.refused_at desc`, since)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return scanSends(rows)
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The assignment generation and the record of every send (novox/hq issue 234). On 2026-10-04 a declaration
|
||||
// newer in sequence than every other named four fewer modules than the assignments held, a machine applied
|
||||
// it, and nothing the mesh kept could say who sent it or from what view.
|
||||
|
||||
func generationNow(t *testing.T, inv *Inventory) int64 {
|
||||
t.Helper()
|
||||
g, err := inv.AssignmentGeneration(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return g
|
||||
}
|
||||
|
||||
// **Every change to what is assigned where raises the generation, in its own transaction** — an
|
||||
// assignment, an unassignment, and one taken by the machine's removal, which no writer in Go makes — and
|
||||
// a repeated assignment, which changes nothing, does not.
|
||||
func TestEveryAssignmentChangeRaisesTheGeneration(t *testing.T) {
|
||||
inv, node := aNodeWithModules(t, "postgres", "web")
|
||||
ctx := t.Context()
|
||||
start := generationNow(t, inv)
|
||||
if start < 1 {
|
||||
t.Fatalf("the generation starts at %d; zero is \"none claimed\" on the wire", start)
|
||||
}
|
||||
if _, err := inv.Assign(ctx, node, "postgres"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assigned := generationNow(t, inv)
|
||||
if assigned <= start {
|
||||
t.Fatalf("an assignment left the generation at %d", assigned)
|
||||
}
|
||||
if _, err := inv.Assign(ctx, node, "postgres"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if again := generationNow(t, inv); again != assigned {
|
||||
t.Errorf("an assignment repeated, which changed nothing, moved the generation from %d to %d", assigned, again)
|
||||
}
|
||||
if err := inv.Unassign(ctx, node, "postgres"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
unassigned := generationNow(t, inv)
|
||||
if unassigned <= assigned {
|
||||
t.Fatalf("an unassignment left the generation at %d", unassigned)
|
||||
}
|
||||
if _, err := inv.Assign(ctx, node, "web"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := generationNow(t, inv)
|
||||
if _, err := inv.RemoveNodeForTest(ctx, node); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if removed := generationNow(t, inv); removed <= before {
|
||||
t.Errorf("a machine's removal took its assignments and left the generation at %d", removed)
|
||||
}
|
||||
}
|
||||
|
||||
// **The generation never goes down, and is raised past what a machine applied when the mesh's own counter
|
||||
// is behind it** — a store put back from a backup — so the next send is not refused for ever.
|
||||
func TestTheGenerationIsRaisedPastWhatAMachineApplied(t *testing.T) {
|
||||
inv, _ := aNodeWithModules(t)
|
||||
now := generationNow(t, inv)
|
||||
if raised, err := inv.RaiseAssignmentGeneration(t.Context(), now+40); err != nil || raised != now+41 {
|
||||
t.Fatalf("raised past %d to %d (%v)", now+40, raised, err)
|
||||
}
|
||||
if raised, err := inv.RaiseAssignmentGeneration(t.Context(), 2); err != nil || raised != now+41 {
|
||||
t.Fatalf("a raise below the counter moved it to %d (%v)", raised, err)
|
||||
}
|
||||
}
|
||||
|
||||
// **Every send is recorded: its sequence, its sender, the generation it came from and the modules it
|
||||
// named** — and the machine's last send is what `plan` reads, the would-send is stamped with its
|
||||
// generation and put-back mark, and a refusal is kept on the send it refused, naming its sender.
|
||||
func TestASendIsRecordedWithItsSenderGenerationAndModules(t *testing.T) {
|
||||
inv, node := aNodeWithModules(t)
|
||||
ctx := t.Context()
|
||||
record, err := inv.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first := Send{Node: record.ID, Sequence: 11, Epoch: 57, Sender: "the controller's daemon (pid 7 on anchor)",
|
||||
Generation: 40, Digest: "d11", Modules: []string{"docker", "pacman", "sudo"}}
|
||||
if err := inv.RecordSend(ctx, first); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stale := Send{Node: record.ID, Sequence: 12, Epoch: 57, Sender: "a one-shot push by jochen at a shell on anchor",
|
||||
Generation: 38, ToldGeneration: 38, Digest: "d12", Modules: []string{"docker"}}
|
||||
if err := inv.RecordSend(ctx, stale); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
last, found, err := inv.LastSend(ctx, node)
|
||||
if err != nil || !found {
|
||||
t.Fatalf("the last send is not kept: %v", err)
|
||||
}
|
||||
if last.Sequence != 12 || last.Sender != stale.Sender || last.Generation != 38 ||
|
||||
!slices.Equal(last.Modules, []string{"docker"}) || last.SentAt.IsZero() {
|
||||
t.Fatalf("the last send reads %+v", last)
|
||||
}
|
||||
generation, putBack, err := inv.SentGeneration(ctx, record.ID)
|
||||
if err != nil || generation != 38 || putBack {
|
||||
t.Fatalf("what the machine was last sent of it reads %d, %v (%v)", generation, putBack, err)
|
||||
}
|
||||
|
||||
refused, found, err := inv.RefusedSend(ctx, record.ID, 12, 40)
|
||||
if err != nil || !found || refused.Sender != stale.Sender || refused.RefusedApplied != 40 {
|
||||
t.Fatalf("the refusal is not kept on the send it refused: %+v, %v, %v", refused, found, err)
|
||||
}
|
||||
since, err := inv.RefusedSendsSince(ctx, time.Now().Add(-time.Hour))
|
||||
if err != nil || len(since) != 1 || since[0].NodeName != node || since[0].Sequence != 12 {
|
||||
t.Fatalf("the refused sends within the hour read %+v (%v)", since, err)
|
||||
}
|
||||
if _, found, err := inv.RefusedSend(ctx, record.ID, 99, 40); err != nil || found {
|
||||
t.Errorf("a refusal of a send never recorded was found: %v, %v", found, err)
|
||||
}
|
||||
}
|
||||
|
||||
// **A machine that reads a generation is recorded from its reports**, and one rolled back says so no longer.
|
||||
func TestWhetherAMachineReadsAGenerationIsItsLatestWord(t *testing.T) {
|
||||
inv, node := aNodeWithModules(t)
|
||||
record, err := inv.NodeByName(t.Context(), node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, reads := range []bool{false, true, false} {
|
||||
if err := inv.RecordReadsGeneration(t.Context(), record.ID, reads); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, err := inv.ReadsGeneration(t.Context(), record.ID); err != nil || got != reads {
|
||||
t.Fatalf("recorded %v, read %v (%v)", reads, got, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -447,6 +447,13 @@ func (e Enrolment) Heard(ctx context.Context, report Report) (news bool, err err
|
||||
if err := e.Inventory.RecordReadsEpoch(ctx, node.ID, report.ReadsEpoch()); err != nil {
|
||||
return false, err
|
||||
}
|
||||
// And whether it reads a generation (novox/hq issue 234), the same way: only from a node-engine that
|
||||
// orders its reports, so a one-shot report (a rekey) does not say an engine stopped reading one.
|
||||
if report.Ordered() {
|
||||
if err := e.Inventory.RecordReadsGeneration(ctx, node.ID, report.ReadsGeneration); err != nil {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
if report.Ordered() {
|
||||
account := AccountOf(report)
|
||||
news, err = e.Inventory.RecordOrderedDoing(ctx, node.ID, doing,
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A report's word on the assignment generation (novox/hq issue 234), as mesh-host's report writes it:
|
||||
// `reads_generation` on every report of a node-engine that reads one, and `older_generation` on a refusal
|
||||
// of a declaration composed from an older generation than the machine applied.
|
||||
func TestTheGenerationOnTheWire(t *testing.T) {
|
||||
raw := []byte(`{"node":"anchor","refused":"older generation","declared":"d12","epoch":57,"sequence":12,` +
|
||||
`"report_sequence":8,"reads_generation":true,"older_generation":{"generation":38,"applied":40}}`)
|
||||
var r Report
|
||||
if err := json.Unmarshal(raw, &r); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !r.ReadsGeneration || r.OlderGeneration == nil || *r.OlderGeneration != (GenerationRefusal{38, 40}) ||
|
||||
r.Sequence != 12 {
|
||||
t.Fatalf("a node-engine's generation refusal reads as %+v", r)
|
||||
}
|
||||
// Not a refusal by the lease's epoch: S13 counts those, and this one is raised naming its sender.
|
||||
if r.StaleRefusalOf() {
|
||||
t.Errorf("a generation refusal reads as a stale writer's")
|
||||
}
|
||||
var older Report
|
||||
if err := json.Unmarshal([]byte(`{"node":"anchor","applied":["a"],"report_sequence":3}`), &older); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if older.ReadsGeneration || older.OlderGeneration != nil {
|
||||
t.Fatalf("a node-engine that never said it reads a generation reads as one that does: %+v", older)
|
||||
}
|
||||
}
|
||||
@@ -197,6 +197,15 @@ type Report struct {
|
||||
// RefusedOlder is how many declarations the node-engine has refused as older, ever, on every report:
|
||||
// a refusal whose own report was lost is still counted from the next.
|
||||
RefusedOlder int64 `json:"refused_older,omitempty"`
|
||||
// ReadsGeneration says the node-engine reads a declaration's assignment generation and its put-back
|
||||
// mark (novox/hq issue 234), on every report it makes: the mesh sends them only to a machine that said
|
||||
// so, because an older node-engine refuses a key it does not know, whole. Mirrors mesh-host
|
||||
// internal/link/messages.go.
|
||||
ReadsGeneration bool `json:"reads_generation,omitempty"`
|
||||
// OlderGeneration is set on a report refusing a declaration composed from an older assignment
|
||||
// generation than the machine applied (novox/hq issue 234). The refused declaration is the report's
|
||||
// own Declared and Order, and its sender is read from the record of sends by that sequence.
|
||||
OlderGeneration *GenerationRefusal `json:"older_generation,omitempty"`
|
||||
|
||||
// Held is what an adopted node found and is keeping as it was until its module is taken
|
||||
// (novox/hq ADR 0100). Without it an adopted node reads as converged.
|
||||
@@ -605,3 +614,10 @@ type HealthSaid struct {
|
||||
Node string `json:"node"`
|
||||
Health Health `json:"health"`
|
||||
}
|
||||
|
||||
// GenerationRefusal is a declaration refused for the assignment generation it was composed from: that
|
||||
// generation, and the highest the machine applied (novox/hq issue 234; mesh-host's GenerationRefusal).
|
||||
type GenerationRefusal struct {
|
||||
Generation int64 `json:"generation"`
|
||||
Applied int64 `json:"applied"`
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user