A declaration says the assignment generation it came from, and every send is recorded (issue 234) #227

Merged
mesh-admin merged 5 commits from fix/234-a-declaration-says-the-generation-it-came-from into main 2026-10-11 17:08:48 +00:00
10 changed files with 394 additions and 175 deletions
Showing only changes of commit 5dd0e3c056 - Show all commits
+1 -1
View File
@@ -515,7 +515,7 @@ func composedAndValidated(ctx context.Context, open *stores, node string, gens m
return sendable{}, nil, err
}
// And the generation it was last sent, as the would-send is (novox/hq issue 234).
if declared.Generation, declared.PutBack, err = open.inventory.SentGeneration(ctx, record.ID); err != nil {
if declared.Generation, err = open.inventory.SentGeneration(ctx, record.ID); err != nil {
return sendable{}, nil, err
}
body, err := declared.Body()
+2 -3
View File
@@ -994,8 +994,7 @@ func gateFailed(ctx context.Context, open *stores, p *inventory.Plan, module str
}
return
}
sent, err := sendRollout(withScope(ctx, sendScope{modules: map[string]bool{module: true}, putBack: true}), open,
g.Machines)
sent, err := sendRollout(withScope(ctx, sendScope{modules: map[string]bool{module: true}}), open, g.Machines)
if err != nil {
notBack(fmt.Sprintf("its registered build is back at %s, and sending it to %s was refused: %v — `push %s` "+
"sends it", short(previous.Commit), strings.Join(g.Machines, ", "), err, g.Machines[0]))
@@ -1088,7 +1087,7 @@ func sendRollbacks(ctx context.Context, open *stores, p *inventory.Plan, b *roll
}
inv := open.inventory
sort.Strings(b.machines)
sent, err := sendRollout(withScope(ctx, sendScope{modules: b.modules, putBack: true}), open, b.machines)
sent, err := sendRollout(withScope(ctx, sendScope{modules: b.modules}), open, b.machines)
var back []string
for _, r := range b.pending {
if err != nil {
+73 -46
View File
@@ -33,15 +33,20 @@ const kindOlderGeneration = "older-generation"
// generationRefusalsSaid is how long a refused send is said after its refusal: an hour, as an advisory is.
const generationRefusalsSaid = advisoryQuiet
// stamp gives a composed declaration the order allotted to it before it was composed: its sequence, the
// epoch and generation when the machine reads them, and the put-back mark beside a generation — and keeps
// the generation and acting epoch for the record of the send whether or not the machine is sent them.
// stamp gives a composed declaration the order allotted to it before it was composed: its sequence, and
// the epoch and generation when the machine reads them — and keeps the generation and acting epoch for the
// record of the send whether or not the machine is sent them.
//
// **No put-back mark is sent.** A gate puts a machine back by composing it again (sendRollout), so its
// declaration is allotted here like any other and carries the generation as it stands — never older than
// what the machine applied. The node-engine reads a `put_back` key (mesh-host#82); this controller never
// needs to send one.
func (o order) stamp(d *sendable) {
d.Sequence, d.Epoch = o.sequence, o.epoch
d.composedFrom, d.actingEpoch, d.putBackSend = o.generation, o.acting, o.putBack
d.Generation, d.PutBack = 0, false
d.composedFrom, d.actingEpoch = o.generation, o.acting
d.Generation = 0
if o.readsGeneration && o.generation > 0 {
d.Generation, d.PutBack = o.generation, o.putBack
d.Generation = o.generation
}
}
@@ -50,18 +55,17 @@ type sentRecord struct {
sequence int64
epoch uint64
generation int64
putBack bool
// told is the generation and put-back mark on the wire, which the would-send is stamped with.
// toldGeneration is the generation on the wire, which the would-send is stamped with: zero for a machine
// whose node-engine has not said it reads one.
toldGeneration int64
toldPutBack bool
sender string
modules []string
}
// sentRecordOf is a composed send's record: its sender is the caller this process acts for.
func sentRecordOf(ctx context.Context, d sendable) sentRecord {
return sentRecord{sequence: d.Sequence, epoch: d.actingEpoch, generation: d.composedFrom, putBack: d.putBackSend,
toldGeneration: d.Generation, toldPutBack: d.PutBack, sender: senderOf(callerOf(ctx)), modules: d.modules}
return sentRecord{sequence: d.Sequence, epoch: d.actingEpoch, generation: d.composedFrom,
toldGeneration: d.Generation, sender: senderOf(callerOf(ctx)), modules: d.modules}
}
// callerOf is who asked for what this process does: the seat call's caller when ctx belongs to one, the
@@ -92,13 +96,19 @@ func namedModules(plan catalogue.Resolution, leftOut map[string]string) []string
return out
}
// heardGenerationRefusal keeps a machine's refusal of a send for its generation on the send it refused,
// so S20 names its sender, and says it (novox/hq issue 234).
// heardGenerationRefusal keeps a machine's refusal of a send for its generation, so S20 names its sender
// and says it (novox/hq issue 234): on the send it refused or — when the mesh has no record of sending that
// sequence — as a refusal of its own, naming the sender as unknown. Either way S20 is raised: a refusal the
// mesh cannot attribute is the louder fact, not a quieter one.
//
// **And raises the mesh's counter past what the machine applied, when the refused send was composed from
// the counter as it stands**: then the sender's view was not stale — the counter is behind the machine,
// which is a store put back from a backup — and every send after would be refused for ever. A stale
// sender's generation is below the counter, and the counter is left alone for it.
// the counter as it stands** (counterBehind): then the sender's view was not stale — the counter is behind
// the machine, which is a store put back from a backup — and every send after would be refused for ever. A
// stale sender's generation is below the counter, and the counter is left alone for it.
//
// Nothing is sent from here. This runs in the controller's receive loop, and a push from it would hold that
// loop, and the machine's hold, for as long as the push takes — the deaf controller of issues 184 and 185.
// S20 names `push <node>` for that case instead.
func heardGenerationRefusal(ctx context.Context, inv *inventory.Inventory, report link.Report) {
r := report.OlderGeneration
if r == nil || inv == nil || report.Node == "" {
@@ -106,35 +116,44 @@ func heardGenerationRefusal(ctx context.Context, inv *inventory.Inventory, repor
}
node, err := inv.NodeByName(ctx, report.Node)
if err != nil {
fmt.Fprintf(os.Stderr, "mesh-controller: %s refused a send for its generation, and the machine cannot be read: %v\n",
report.Node, err)
fmt.Fprintf(os.Stderr, "mesh-controller: %s refused a send for its generation, and the machine cannot be "+
"read, so it is not raised: %v\n", report.Node, err)
return
}
send, found, err := inv.RefusedSend(ctx, node.ID, report.Sequence, r.Applied)
switch {
case err != nil:
fmt.Fprintf(os.Stderr, "mesh-controller: %s refused send %d for its generation, and the refusal could not be "+
"kept: %v\n", report.Node, report.Sequence, err)
case !found:
fmt.Fprintf(os.Stderr, "mesh-controller: %s refused send %d (generation %d; it applied %d), which the mesh "+
"has no record of sending\n", report.Node, report.Sequence, r.Generation, r.Applied)
default:
fmt.Fprintf(os.Stderr, "mesh-controller: %s refused send %d from %s: composed from assignment generation %d, "+
"and it applied %d\n", report.Node, report.Sequence, send.Sender, r.Generation, r.Applied)
var raised int64
if now, err := inv.AssignmentGeneration(ctx); err != nil {
fmt.Fprintf(os.Stderr, "mesh-controller: %s refused a send for its generation, and the mesh's own cannot be "+
"read: %v\n", report.Node, err)
} else if counterBehind(*r, now) {
if raised, err = inv.RaiseAssignmentGeneration(ctx, r.Applied); err != nil {
fmt.Fprintf(os.Stderr, "mesh-controller: the assignment generation (%d) is behind what %s applied (%d), "+
"and could not be raised: %v\n", now, report.Node, r.Applied, err)
raised = 0
} else {
fmt.Fprintf(os.Stderr, "mesh-controller: the assignment generation was %d, behind what %s applied (%d) — "+
"a store put back from a backup — and is raised to %d; `push %s` sends it what the mesh holds now\n",
now, report.Node, r.Applied, raised, report.Node)
}
}
now, err := inv.AssignmentGeneration(ctx)
if err != nil || r.Generation < now || r.Applied < now {
return
send, found, err := inv.RefusedSend(ctx, node.ID, report.Sequence, r.Applied, raised)
if err == nil && !found {
send, err = inv.RecordUnrecordedRefusal(ctx, inventory.Send{Node: node.ID, Sequence: report.Sequence,
Epoch: report.Epoch, Generation: r.Generation, Digest: report.Declared, RefusedApplied: r.Applied,
CounterRaisedTo: raised})
}
raised, err := inv.RaiseAssignmentGeneration(ctx, r.Applied)
if err != nil {
fmt.Fprintf(os.Stderr, "mesh-controller: the assignment generation (%d) is behind what %s applied (%d), and "+
"could not be raised: %v\n", now, report.Node, r.Applied, err)
fmt.Fprintf(os.Stderr, "mesh-controller: %s refused send %d for its generation, and the refusal could not be "+
"kept, so it is not raised: %v\n", report.Node, report.Sequence, err)
return
}
fmt.Fprintf(os.Stderr, "mesh-controller: the assignment generation was %d, behind what %s applied (%d) — a store "+
"put back from a backup — and is raised to %d, so the next send is not refused\n", now, report.Node, r.Applied,
raised)
fmt.Fprintf(os.Stderr, "mesh-controller: %s refused send %d from %s: composed from assignment generation %d, "+
"and it applied %d\n", report.Node, report.Sequence, send.Sender, r.Generation, r.Applied)
}
// counterBehind says a refusal shows the mesh's counter behind the machine rather than a stale sender: the
// refused send carried the counter as it stands now, and the machine applied more than that.
func counterBehind(r link.GenerationRefusal, now int64) bool {
return r.Generation >= now && r.Applied > r.Generation
}
// watchGenerationRefusals is S20: every send a machine refused within the hour for the generation it was
@@ -147,7 +166,7 @@ func watchGenerationRefusals(f *signalFacts) []conditions.Observation {
continue
}
seen[s.NodeName] = true
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: s.NodeName, Kind: kindOlderGeneration,
o := conditions.Observation{Scope: conditions.ScopeMachine, ID: s.NodeName, Kind: kindOlderGeneration,
Machine: s.NodeName, Severity: conditions.Warning,
Summary: fmt.Sprintf("%s refused sequence %d from %s: it was composed from assignment generation %d, and "+
"%s applied generation %d — a sender composing from a view of the assignments the mesh has moved "+
@@ -157,7 +176,20 @@ func watchGenerationRefusals(f *signalFacts) []conditions.Observation {
Headline: fmt.Sprintf("%s refused an out-of-date update", s.NodeName),
Explanation: fmt.Sprintf("Something sent %s an update made from an older list of what runs there. %s "+
"refused it, so nothing was removed. Nothing for you to do unless it repeats.", s.NodeName, s.NodeName),
Resolved: fmt.Sprintf("%s has had no out-of-date update for an hour", s.NodeName)})
Resolved: fmt.Sprintf("%s has had no out-of-date update for an hour", s.NodeName)}
if s.CounterRaisedTo > 0 {
// Not a stale sender: the mesh's counter was behind the machine (a store put back from a backup) and
// was raised; nothing has sent the machine its declaration since, so a person is asked to.
o.Summary = fmt.Sprintf("%s refused sequence %d from %s: it was composed from assignment generation %d, "+
"the mesh's own, and %s applied generation %d — the mesh's counter was behind the machine (a store "+
"put back from a backup?) and is raised to %d; `push %s` sends it what the mesh holds now",
s.NodeName, s.Sequence, s.Sender, s.Generation, s.NodeName, s.RefusedApplied, s.CounterRaisedTo,
s.NodeName)
o.Explanation = fmt.Sprintf("Needs you: push %s. The mesh's record was older than %s, so %s refused its "+
"update and kept what it had. The record is repaired; a push sends the update again.",
s.NodeName, s.NodeName, s.NodeName)
}
out = append(out, o)
}
return out
}
@@ -181,12 +213,7 @@ func writeLastSend(ctx context.Context, w io.Writer, inv *inventory.Inventory, n
if s.Generation > 0 {
generation = fmt.Sprintf("assignment generation %d", s.Generation)
}
kind := "sent"
if s.PutBack {
kind = "put back"
}
fmt.Fprintf(w, "%s was last %s sequence %d at %s by %s, composed from %s, naming %s\n", node, kind,
s.Sequence, s.SentAt.Local().Format("2006-01-02 15:04:05"), s.Sender, generation, modulesWords(s.Modules))
fmt.Fprintf(w, "%s was last sent sequence %d at %s by %s, composed from %s, naming %s\n", node, s.Sequence, s.SentAt.Local().Format("2006-01-02 15:04:05"), s.Sender, generation, modulesWords(s.Modules))
if s.RefusedAt != nil {
fmt.Fprintf(w, " and refused it at %s: it had applied generation %d\n",
s.RefusedAt.Local().Format("2006-01-02 15:04:05"), s.RefusedApplied)
+62 -21
View File
@@ -7,6 +7,7 @@ import (
"time"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// The assignment generation a declaration was composed from (novox/hq issue 234).
@@ -25,45 +26,35 @@ func bodyKeys(t *testing.T, s sendable) map[string]any {
}
// **The generation goes on the wire only to a machine whose node-engine said it reads one**: an older
// node-engine decodes strictly and refuses an unknown key, whole. And the put-back mark only with it.
// node-engine decodes strictly and refuses an unknown key, whole. No put-back mark is ever sent: a gate
// composes its put-back afresh, with the generation as it stands.
func TestTheGenerationIsSentOnlyToAMachineThatReadsOne(t *testing.T) {
resources := []map[string]any{{"id": "a", "type": "file", "path": "/etc/a", "content": "x\n"}}
reads := order{sequence: 12, epoch: 57, generation: 40, readsGeneration: true, acting: 57, putBack: true}
reads := order{sequence: 12, epoch: 57, generation: 40, readsGeneration: true, acting: 57}
var told sendable
told.Resources = resources
reads.stamp(&told)
keys := bodyKeys(t, told)
if keys["generation"] != float64(40) || keys["put_back"] != true || keys["sequence"] != float64(12) {
if keys["generation"] != float64(40) || keys["sequence"] != float64(12) {
t.Fatalf("a machine that reads a generation was sent %v", keys)
}
if _, there := keys["put_back"]; there {
t.Fatalf("a put-back mark was sent: %v", keys)
}
if told.composedFrom != 40 || told.actingEpoch != 57 {
t.Errorf("the send does not keep what it was composed from for its record: %+v", told)
}
older := order{sequence: 12, epoch: 57, generation: 40, readsGeneration: false, acting: 57, putBack: true}
older := order{sequence: 12, epoch: 57, generation: 40, readsGeneration: false, acting: 57}
var untold sendable
untold.Resources = resources
older.stamp(&untold)
keys = bodyKeys(t, untold)
if _, there := keys["generation"]; there {
t.Fatalf("a machine whose node-engine never said it reads a generation was sent one: %v", keys)
}
if _, there := keys["put_back"]; there {
t.Fatalf("a machine whose node-engine never said it reads a generation was sent a put-back mark: %v", keys)
if _, there := bodyKeys(t, untold)["generation"]; there {
t.Fatal("a machine whose node-engine never said it reads a generation was sent one")
}
if untold.composedFrom != 40 {
t.Errorf("the generation it was composed from is recorded whether or not it was sent: %+v", untold)
}
// An ordinary send carries no put-back mark at all: the body is what it was apart from the generation.
var ordinary sendable
ordinary.Resources = resources
ordinaryOrder := reads
ordinaryOrder.putBack = false
ordinaryOrder.stamp(&ordinary)
if _, there := bodyKeys(t, ordinary)["put_back"]; there {
t.Fatal("an ordinary send says it is a put-back")
}
}
// **The sender is named**: the caller of the seat call when there is one, else the shell's account, and the
@@ -79,7 +70,7 @@ func TestASendNamesItsSender(t *testing.T) {
func refusedSend(at time.Time) inventory.Send {
return inventory.Send{NodeName: "anchor", Sequence: 12, Epoch: 57, Generation: 38, RefusedApplied: 40,
Sender: "a one-shot push by jochen at a shell on anchor (pid 4242 on anchor, build 2026.10.11)",
Modules: []string{"docker"}, SentAt: at.Add(-time.Second), RefusedAt: &at}
Modules: []string{"docker"}, SentAt: at.Add(-time.Second), RefusedAt: &at, Recorded: true}
}
// **A refused send is raised naming its sender** (novox/hq issue 234): who sent it, from which generation,
@@ -105,3 +96,53 @@ func TestARefusedSendIsRaisedNamingItsSender(t *testing.T) {
t.Errorf("the condition is not worded for the operator: %+v", o)
}
}
// **A refusal of a send the mesh has no record of is raised too, its sender said to be unknown** — never
// only a line on stderr.
func TestARefusalOfAnUnrecordedSendIsRaisedSayingTheSenderIsUnknown(t *testing.T) {
now := time.Date(2026, 10, 11, 12, 0, 0, 0, time.UTC)
f := calm(now)
s := refusedSend(now.Add(-time.Minute))
s.Recorded, s.Sender, s.Modules = false, "a sender the mesh has no record of (no send of this sequence was recorded)", nil
f.refusedSends = []inventory.Send{s}
got := watchGenerationRefusals(f)
if len(got) != 1 || !strings.Contains(got[0].Summary, "no record of") {
t.Fatalf("an unattributed refusal was not raised as one: %+v", got)
}
}
// **When the refusal showed the mesh's counter behind the machine, the condition asks for a push** — it was
// raised, and nothing has sent the machine its declaration since — and does not say there is nothing to do.
func TestACounterBehindTheMachineAsksForAPush(t *testing.T) {
now := time.Date(2026, 10, 11, 12, 0, 0, 0, time.UTC)
f := calm(now)
s := refusedSend(now.Add(-time.Minute))
s.Generation, s.RefusedApplied, s.CounterRaisedTo = 12, 40, 41
f.refusedSends = []inventory.Send{s}
got := watchGenerationRefusals(f)
if len(got) != 1 {
t.Fatalf("%+v", got)
}
if !strings.Contains(got[0].Summary, "`push anchor`") || !strings.Contains(got[0].Explanation, "push anchor") ||
strings.Contains(got[0].Explanation, "Nothing for you to do") {
t.Errorf("a counter behind the machine does not ask for a push: %s / %s", got[0].Summary, got[0].Explanation)
}
}
// **Only a refusal of the counter as it stands is the counter behind**: a stale sender's generation is below
// it, and the counter is left alone for that.
func TestOnlyARefusalOfTheCounterAsItStandsRaisesIt(t *testing.T) {
for _, c := range []struct {
refused, applied, now int64
behind bool
}{
{refused: 12, applied: 40, now: 12, behind: true}, // the store was put back: the mesh says 12, the machine had 40
{refused: 38, applied: 40, now: 41, behind: false}, // a stale sender: the counter is already past
{refused: 38, applied: 40, now: 40, behind: false}, // a stale sender: the counter is where the machine is
} {
r := link.GenerationRefusal{Generation: c.refused, Applied: c.applied}
if got := counterBehind(r, c.now); got != c.behind {
t.Errorf("refused %d, applied %d, counter %d: behind %v, want %v", c.refused, c.applied, c.now, got, c.behind)
}
}
}
+20 -18
View File
@@ -359,14 +359,12 @@ func declare(ctx context.Context, args []string) error {
Epoch uint64 `json:"epoch"`
Sequence int64 `json:"sequence"`
Generation int64 `json:"generation"`
PutBack bool `json:"put_back"`
}
_ = json.Unmarshal(raw, &carried)
// And recorded as every send is (novox/hq issue 234): by hand, from what it carried; the modules it
// named are not known, since the mesh did not compose it.
if _, err := recordSent(ctx, inv, node, raw, nil, carried.Epoch, sentRecord{sequence: carried.Sequence,
epoch: carried.Epoch, generation: carried.Generation, putBack: carried.PutBack,
toldGeneration: carried.Generation, toldPutBack: carried.PutBack,
epoch: carried.Epoch, generation: carried.Generation, toldGeneration: carried.Generation,
sender: senderOf(callerOf(ctx)) + ", a declaration sent by hand"}); err != nil {
return err
}
@@ -1402,9 +1400,9 @@ func wouldSendFrom(ctx context.Context, open *stores,
if declared.Epoch, err = open.inventory.SentEpoch(ctx, n.ID); err != nil {
return nil, err
}
// And the generation and put-back mark it was last sent, for the same reason (novox/hq issue 234):
// an assignment elsewhere in the mesh is not a change of this machine.
if declared.Generation, declared.PutBack, err = open.inventory.SentGeneration(ctx, n.ID); err != nil {
// And the generation it was last sent, for the same reason (novox/hq issue 234): an assignment
// elsewhere in the mesh is not a change of this machine.
if declared.Generation, err = open.inventory.SentGeneration(ctx, n.ID); err != nil {
return nil, err
}
body, err := declared.Body()
@@ -1551,8 +1549,6 @@ type order struct {
generation int64
readsGeneration bool
acting uint64
// putBack is a gate's put-back (sendScope.putBack), read from the send's context.
putBack bool
}
// allot takes the next sequence for a machine — the number its next declaration carries — under the
@@ -1593,7 +1589,7 @@ func allot(ctx context.Context, inv *inventory.Inventory, node string) (order, e
return order{}, err
}
return order{sequence: seq, epoch: epoch, generation: generation, readsGeneration: readsGeneration,
acting: acting, putBack: scopeOf(ctx).putBack}, nil
acting: acting}, nil
}
// recordSent writes down what a machine was just sent, and returns the digest.
@@ -1616,17 +1612,23 @@ func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body
return "", err
}
digest := digestOf(body)
if err := inv.RecordSentUnder(kept, record.ID, digest, builds, epoch); err != nil {
// The digest and the generation it carried are written in one transaction (novox/hq issue 234), so the
// would-send is never stamped with a generation the machine was not sent; and the send itself, who sent
// it and from which generation, beside them. A record of the send that cannot be written does not make a
// send that is away look failed: it is said, loudly, and the machine's own record stands.
err = inv.RecordSentWith(kept, record.ID, digest, builds, epoch, sent.toldGeneration, inventory.Send{
Sequence: sent.sequence, Epoch: int64(sent.epoch), Sender: sent.sender, Generation: sent.generation,
Digest: digest, Modules: sent.modules})
var unrecorded *inventory.SendNotRecordedError
switch {
case errors.As(err, &unrecorded):
fmt.Fprintf(os.Stderr, "mesh-controller: SEND NOT RECORDED: %s was sent declaration %s (sequence %d), and who "+
"sent it and from which generation could not be written down, so a refusal of it will name no sender "+
"(novox/hq issue 234): %v\n", node, short(digest), sent.sequence, unrecorded.Err)
fmt.Printf("%s: sent, and the record of who sent it could NOT be written: %v\n", node, unrecorded.Err)
case err != nil:
return "", err
}
// And the send itself, who sent it and from which generation (novox/hq issue 234): what the machine
// was last told, by whom, is read back from here — and what the would-send is stamped with.
if err := inv.RecordSend(kept, inventory.Send{Node: record.ID, Sequence: sent.sequence,
Epoch: int64(sent.epoch), Sender: sent.sender, Generation: sent.generation, PutBack: sent.putBack,
ToldGeneration: sent.toldGeneration, ToldPutBack: sent.toldPutBack, Digest: digest,
Modules: sent.modules}); err != nil {
return "", fmt.Errorf("%s was sent its declaration, and the send could not be recorded: %w", node, err)
}
// And what it was, summarised, so the next push can be compared with it (novox/hq ADR 0217).
// Never a reason to fail a send that is already away: a summary that cannot be kept means the
// next comparison has nothing to hold against, which is how every machine starts.
-3
View File
@@ -50,9 +50,6 @@ type sendScope struct {
modules map[string]bool
// person is a person's act: it carries what it carries.
person bool
// putBack is a gate putting machines back after a failed send (novox/hq issue 234): its declarations
// say so, and a machine does not refuse them for the generation they carry.
putBack bool
}
type sendScopeKey struct{}
-9
View File
@@ -33,17 +33,11 @@ type sendable struct {
// node-engine has not said it reads one is sent none, for the reason the epoch is not (an older
// node-engine refuses a key it does not know, whole).
Generation int64
// PutBack marks a gate's put-back (novox/hq issue 234): it carries the generation of what it puts
// back and is not refused for it. Sent only beside a generation.
PutBack bool
// composedFrom is the generation read before this declaration was composed, and actingEpoch the
// lease epoch its sender acted under — whether or not the machine is sent either — for the record of
// the send; never on the wire.
composedFrom int64
actingEpoch uint64
// putBackSend is whether a gate sent it to put the machine back, for the record; PutBack is the mark
// on the wire, only beside a generation.
putBackSend bool
// modules are the modules this declaration names, for the record of the send; never on the wire.
modules []string
// Adoption is nil for a converged node, and then the body is byte for byte what it was before
@@ -115,9 +109,6 @@ func (s sendable) Body() ([]byte, error) {
}
if s.Generation > 0 {
envelope["generation"] = s.Generation
if s.PutBack {
envelope["put_back"] = true
}
}
// An empty declaration is deliberate here — the node owns nothing the mesh put there
// (novox/hq issue 127) — and the host refuses an empty body unless it is told the emptiness
@@ -24,26 +24,33 @@ begin
end
$$;
-- Per row, so a statement that changes nothing (an assignment repeated, `on conflict do nothing`) raises
-- nothing; a statement that changes several raises once per row, which only ever moves it forward.
-- Per row, so a statement that changes nothing raises nothing: an assignment repeated (`on conflict do
-- nothing` inserts no row) and an update that leaves a row as it was (the WHEN below; a plain UPDATE fires a
-- row trigger whether or not it changed anything). A statement that changes several rows raises once per
-- row, which only ever moves it forward. Two triggers, because only an update has both OLD and NEW.
create trigger assignment_generation_raised
after insert or update or delete on assignment
after insert or delete on assignment
for each row execute function raise_assignment_generation();
create trigger assignment_generation_raised_by_update
after update on assignment
for each row when (old is distinct from new) execute function raise_assignment_generation();
-- What a machine was last sent of it: the generation, and whether that send was a gate's put-back, so
-- The generation a machine was last sent, written in the same transaction as the digest of that send, so
-- what the mesh WOULD send is stamped the same way and reads as byte for byte what it DID send when
-- nothing else changed (as sent_epoch, migration 0068). Null and false for a send without.
-- nothing else changed (as sent_epoch, migration 0068). Null for a send without one.
alter table node add column sent_generation bigint;
alter table node add column sent_put_back boolean not null default false;
-- Whether the machine's node-engine said it reads a generation (its reports' `reads_generation`): until
-- it has, it is sent none, because an older node-engine refuses a key it does not know, whole.
alter table node add column reads_generation boolean not null default false;
-- Every send: the machine, its sequence, who sent it — the lease epoch the sender acted under and the
-- caller, process and build — the generation it was composed from, whether it was a put-back, its
-- digest and the modules it named. Written after the declaration is away, as the node row's record is.
-- A refusal by the machine of a send for its generation is kept on the send it refused, so the
-- condition it raises names the sender from here. The newest 200 per machine are kept.
-- caller, process and build — the generation it was composed from, its digest and the modules it named.
-- Written after the declaration is away, with the node row's record of it. A refusal by the machine of a
-- send for its generation is kept on the send it refused, so the condition it raises names the sender
-- from here; a refusal of a sequence the mesh has no send for is kept as a row of its own, `recorded`
-- false, whose sender is said to be unknown. `counter_raised_to` is the generation the mesh's counter was
-- raised to when the refusal showed the counter behind the machine (a store put back from a backup). The
-- newest 200 per machine are kept.
create table declaration_send (
id bigserial primary key,
node uuid not null references node (id) on delete cascade,
@@ -51,13 +58,14 @@ create table declaration_send (
epoch bigint,
sender text not null,
generation bigint,
put_back boolean not null default false,
digest text not null,
modules text[] not null default '{}',
sent_at timestamptz not null default now(),
refused_at timestamptz,
-- refused_applied is the generation the machine said it had applied when it refused this send.
refused_applied bigint
refused_applied bigint,
counter_raised_to bigint,
recorded boolean not null default true
);
create index declaration_send_node on declaration_send (node, id desc);
create index declaration_send_sequence on declaration_send (node, sequence);
+133 -48
View File
@@ -2,6 +2,7 @@ package inventory
import (
"context"
"encoding/json"
"fmt"
"time"
@@ -47,20 +48,17 @@ func (i *Inventory) RaiseAssignmentGeneration(ctx context.Context, past int64) (
return g, nil
}
// SentGeneration is the generation a machine was last sent and whether that send was a put-back, by its
// id; zero for one sent without. What the mesh WOULD send is stamped with these, so it reads as byte for
// byte what it DID send when nothing else changed.
func (i *Inventory) SentGeneration(ctx context.Context, id string) (int64, bool, error) {
// SentGeneration is the generation a machine was last sent, by its id; zero for one sent without. What the
// mesh WOULD send is stamped with it, so it reads as byte for byte what it DID send when nothing else changed.
func (i *Inventory) SentGeneration(ctx context.Context, id string) (int64, error) {
var g *int64
var putBack bool
if err := i.store.Pool().QueryRow(ctx, `select sent_generation, sent_put_back from node where id = $1`, id).
Scan(&g, &putBack); err != nil {
return 0, false, fmt.Errorf("reading the generation %s was last sent: %w", id, err)
if err := i.store.Pool().QueryRow(ctx, `select sent_generation from node where id = $1`, id).Scan(&g); err != nil {
return 0, fmt.Errorf("reading the generation %s was last sent: %w", id, err)
}
if g == nil {
return 0, putBack, nil
return 0, nil
}
return *g, putBack, nil
return *g, nil
}
// ReadsGeneration says a machine's node-engine said it reads a generation in a declaration, by its id.
@@ -88,66 +86,122 @@ type Send struct {
Epoch int64
// Sender is who sent it, in words: the caller, and the process and build that composed it.
Sender string
// Generation is the assignment generation it was composed from, zero when not known; PutBack whether a
// gate sent it to put the machine back.
// Generation is the assignment generation it was composed from, zero when not known.
Generation int64
PutBack bool
// ToldGeneration and ToldPutBack are what the machine was told of them on the wire: zero and false for a
// machine whose node-engine has not said it reads a generation. What the would-send is stamped with.
ToldGeneration int64
ToldPutBack bool
Digest string
Digest string
// Modules are the modules it named.
Modules []string
SentAt time.Time
// RefusedAt is when the machine refused it for its generation, nil when it did not; RefusedApplied the
// generation the machine said it had applied then.
RefusedAt *time.Time
RefusedApplied int64
// generation the machine said it had applied then; CounterRaisedTo what the mesh's counter was raised to
// because the refusal showed it behind the machine, zero when it was not.
RefusedAt *time.Time
RefusedApplied int64
CounterRaisedTo int64
// Recorded is false for a refusal of a sequence the mesh has no send for: its sender is unknown.
Recorded bool
}
// RecordSend writes one send down, and what the machine was last sent of its generation, in one
// transaction; the oldest beyond the newest 200 for the machine are let go.
func (i *Inventory) RecordSend(ctx context.Context, s Send) error {
// unknownSender is the sender of a refused sequence the mesh has no record of sending.
const unknownSender = "a sender the mesh has no record of (no send of this sequence was recorded: sent by " +
"hand, before sends were recorded, or by a controller whose record was not written)"
// SendNotRecordedError is a send whose machine's record was written — its digest and the generation it
// carried — and whose record of who sent it was not. The send is away and the machine's record stands; the
// caller says this loudly and does not take the send for failed.
type SendNotRecordedError struct{ Err error }
func (e *SendNotRecordedError) Error() string {
return "the record of the send was not written: " + e.Err.Error()
}
func (e *SendNotRecordedError) Unwrap() error { return e.Err }
// RecordSentWith writes down what a machine was just sent — its digest, the builds it carried, the epoch and
// the generation on the wire — and the send itself, who sent it and from which generation (novox/hq issue
// 234), in one transaction. The digest and the generation are written together, so the would-send is never
// stamped with a generation the machine was not sent. The send's own record is written under a savepoint: if
// it cannot be, the machine's record is still committed and a *SendNotRecordedError says so. The oldest
// sends beyond the newest 200 for the machine are let go.
func (i *Inventory) RecordSentWith(ctx context.Context, node, digest string, builds map[string]string, epoch uint64,
generation int64, s Send) error {
var sentEpoch *int64
if epoch > 0 {
e := int64(epoch)
sentEpoch = &e
}
var carried *string
if builds != nil {
raw, err := json.Marshal(builds)
if err != nil {
return err
}
text := string(raw)
carried = &text
}
tx, err := i.store.Pool().Begin(ctx)
if err != nil {
return err
}
defer func() { _ = tx.Rollback(ctx) }()
if _, err := tx.Exec(ctx, `update node set sent_generation = $2, sent_put_back = $3 where id = $1`,
s.Node, nullIfZero(s.ToldGeneration), s.ToldPutBack); err != nil {
if _, err := tx.Exec(ctx,
`update node set sent = $2, sent_at = now(), sent_builds = $3::jsonb, sent_epoch = $4, sent_generation = $5
where id = $1`, node, digest, carried, sentEpoch, nullIfZero(generation)); err != nil {
return err
}
recordErr := recordSend(ctx, tx, node, s)
if err := tx.Commit(ctx); err != nil {
return err
}
if recordErr != nil {
return &SendNotRecordedError{Err: recordErr}
}
return nil
}
// recordSend writes one send under a savepoint of tx, which it rolls back when the send cannot be written.
func recordSend(ctx context.Context, tx pgx.Tx, node string, s Send) error {
sp, err := tx.Begin(ctx)
if err != nil {
return err
}
defer func() { _ = sp.Rollback(ctx) }()
modules := s.Modules
if modules == nil {
modules = []string{}
}
if _, err := tx.Exec(ctx,
`insert into declaration_send (node, sequence, epoch, sender, generation, put_back, digest, modules)
values ($1, $2, $3, $4, $5, $6, $7, $8)`,
s.Node, nullIfZero(s.Sequence), nullIfZero(s.Epoch), s.Sender, nullIfZero(s.Generation), s.PutBack,
s.Digest, modules); err != nil {
if _, err := sp.Exec(ctx,
`insert into declaration_send (node, sequence, epoch, sender, generation, digest, modules)
values ($1, $2, $3, $4, $5, $6, $7)`,
node, nullIfZero(s.Sequence), nullIfZero(s.Epoch), s.Sender, nullIfZero(s.Generation), s.Digest,
modules); err != nil {
return err
}
if _, err := tx.Exec(ctx,
if err := pruneSends(ctx, sp, node); err != nil {
return err
}
return sp.Commit(ctx)
}
// pruneSends lets go of a machine's sends beyond the newest 200.
func pruneSends(ctx context.Context, q queries, node string) error {
_, err := q.Exec(ctx,
`delete from declaration_send where node = $1 and id not in
(select id from declaration_send where node = $1 order by id desc limit $2)`, s.Node, sendsKept); err != nil {
return err
}
return tx.Commit(ctx)
(select id from declaration_send where node = $1 order by id desc limit $2)`, node, sendsKept)
return err
}
// sendColumns are a send's columns as scanSends reads them.
const sendColumns = `s.node, n.name, coalesce(s.sequence, 0), coalesce(s.epoch, 0), s.sender, coalesce(s.generation, 0),
s.put_back, s.digest, s.modules, s.sent_at, s.refused_at, coalesce(s.refused_applied, 0)`
s.digest, s.modules, s.sent_at, s.refused_at, coalesce(s.refused_applied, 0), coalesce(s.counter_raised_to, 0),
s.recorded`
func scanSends(rows pgx.Rows) ([]Send, error) {
defer rows.Close()
var out []Send
for rows.Next() {
var s Send
if err := rows.Scan(&s.Node, &s.NodeName, &s.Sequence, &s.Epoch, &s.Sender, &s.Generation, &s.PutBack,
&s.Digest, &s.Modules, &s.SentAt, &s.RefusedAt, &s.RefusedApplied); err != nil {
if err := rows.Scan(&s.Node, &s.NodeName, &s.Sequence, &s.Epoch, &s.Sender, &s.Generation, &s.Digest,
&s.Modules, &s.SentAt, &s.RefusedAt, &s.RefusedApplied, &s.CounterRaisedTo, &s.Recorded); err != nil {
return nil, err
}
out = append(out, s)
@@ -155,11 +209,12 @@ func scanSends(rows pgx.Rows) ([]Send, error) {
return out, rows.Err()
}
// LastSend is the last declaration a machine was sent, by its name; false when none was recorded.
// LastSend is the last declaration a machine was sent, by its name; false when none was recorded. A refusal
// of a sequence the mesh has no send for is not a send, and is not it.
func (i *Inventory) LastSend(ctx context.Context, name string) (Send, bool, error) {
rows, err := i.store.Pool().Query(ctx, `select `+sendColumns+`
from declaration_send s join node n on n.id = s.node
where n.name = $1 order by s.id desc limit 1`, name)
where n.name = $1 and s.recorded order by s.id desc limit 1`, name)
if err != nil {
return Send{}, false, err
}
@@ -171,15 +226,17 @@ func (i *Inventory) LastSend(ctx context.Context, name string) (Send, bool, erro
}
// RefusedSend keeps a machine's refusal of the send of a sequence for the generation it came from, and
// answers that send, sender and all; false when no send of that sequence was recorded — sent before the
// record, or by a hand the mesh did not see. The newest of that sequence when there are several: a
// declaration a person sent by hand may repeat one.
func (i *Inventory) RefusedSend(ctx context.Context, node string, sequence, applied int64) (Send, bool, error) {
// answers that send, sender and all; false when no send of that sequence was recorded (RecordUnrecordedRefusal
// keeps that one). The newest of that sequence when there are several: a declaration a person sent by hand may
// repeat one. raisedTo is what the mesh's counter was raised to for it, zero for none.
func (i *Inventory) RefusedSend(ctx context.Context, node string, sequence, applied, raisedTo int64) (Send, bool, error) {
rows, err := i.store.Pool().Query(ctx, `with refused as (
update declaration_send set refused_at = now(), refused_applied = $3
where id = (select id from declaration_send where node = $1 and sequence = $2 order by id desc limit 1)
update declaration_send set refused_at = now(), refused_applied = $3, counter_raised_to = $4
where id = (select id from declaration_send where node = $1 and sequence = $2 and recorded
order by id desc limit 1)
returning *)
select `+sendColumns+` from refused s join node n on n.id = s.node`, node, sequence, applied)
select `+sendColumns+` from refused s join node n on n.id = s.node`, node, sequence, applied,
nullIfZero(raisedTo))
if err != nil {
return Send{}, false, err
}
@@ -190,6 +247,34 @@ func (i *Inventory) RefusedSend(ctx context.Context, node string, sequence, appl
return sends[0], true, nil
}
// RecordUnrecordedRefusal keeps a machine's refusal of a sequence the mesh has no send for, as a row of its
// own whose sender is unknown, and answers it: S20 raises it like any other, because a refusal nobody can
// attribute is no quieter for that. s carries the machine, the sequence, epoch, generation and digest the
// refused declaration carried, and the refusal's applied generation and counter raise.
func (i *Inventory) RecordUnrecordedRefusal(ctx context.Context, s Send) (Send, error) {
rows, err := i.store.Pool().Query(ctx, `with refused as (
insert into declaration_send (node, sequence, epoch, sender, generation, digest, refused_at, refused_applied,
counter_raised_to, recorded)
values ($1, $2, $3, $4, $5, $6, now(), $7, $8, false) returning *)
select `+sendColumns+` from refused s join node n on n.id = s.node`,
s.Node, nullIfZero(s.Sequence), nullIfZero(s.Epoch), unknownSender, nullIfZero(s.Generation), s.Digest,
s.RefusedApplied, nullIfZero(s.CounterRaisedTo))
if err != nil {
return Send{}, err
}
sends, err := scanSends(rows)
if err != nil {
return Send{}, err
}
if len(sends) == 0 {
return Send{}, fmt.Errorf("the refusal of sequence %d was not kept", s.Sequence)
}
if err := pruneSends(ctx, i.store.Pool(), s.Node); err != nil {
return Send{}, err
}
return sends[0], nil
}
// RefusedSendsSince is every send refused for its generation since a moment, newest first.
func (i *Inventory) RefusedSendsSince(ctx context.Context, since time.Time) ([]Send, error) {
rows, err := i.store.Pool().Query(ctx, `select `+sendColumns+`
+83 -14
View File
@@ -1,6 +1,7 @@
package inventory
import (
"errors"
"slices"
"testing"
"time"
@@ -75,8 +76,8 @@ func TestTheGenerationIsRaisedPastWhatAMachineApplied(t *testing.T) {
}
// **Every send is recorded: its sequence, its sender, the generation it came from and the modules it
// named** — and the machine's last send is what `plan` reads, the would-send is stamped with its
// generation and put-back mark, and a refusal is kept on the send it refused, naming its sender.
// named** — and the machine's last send is what `plan` reads, the would-send is stamped with the generation
// written with the digest, and a refusal is kept on the send it refused, naming its sender.
func TestASendIsRecordedWithItsSenderGenerationAndModules(t *testing.T) {
inv, node := aNodeWithModules(t)
ctx := t.Context()
@@ -84,42 +85,110 @@ func TestASendIsRecordedWithItsSenderGenerationAndModules(t *testing.T) {
if err != nil {
t.Fatal(err)
}
first := Send{Node: record.ID, Sequence: 11, Epoch: 57, Sender: "the controller's daemon (pid 7 on anchor)",
first := Send{Sequence: 11, Epoch: 57, Sender: "the controller's daemon (pid 7 on anchor)",
Generation: 40, Digest: "d11", Modules: []string{"docker", "pacman", "sudo"}}
if err := inv.RecordSend(ctx, first); err != nil {
if err := inv.RecordSentWith(ctx, record.ID, "d11", nil, 57, 40, first); err != nil {
t.Fatal(err)
}
stale := Send{Node: record.ID, Sequence: 12, Epoch: 57, Sender: "a one-shot push by jochen at a shell on anchor",
Generation: 38, ToldGeneration: 38, Digest: "d12", Modules: []string{"docker"}}
if err := inv.RecordSend(ctx, stale); err != nil {
stale := Send{Sequence: 12, Epoch: 57, Sender: "a one-shot push by jochen at a shell on anchor",
Generation: 38, Digest: "d12", Modules: []string{"docker"}}
if err := inv.RecordSentWith(ctx, record.ID, "d12", nil, 57, 38, stale); err != nil {
t.Fatal(err)
}
last, found, err := inv.LastSend(ctx, node)
if err != nil || !found {
t.Fatalf("the last send is not kept: %v", err)
}
if last.Sequence != 12 || last.Sender != stale.Sender || last.Generation != 38 ||
if last.Sequence != 12 || last.Sender != stale.Sender || last.Generation != 38 || !last.Recorded ||
!slices.Equal(last.Modules, []string{"docker"}) || last.SentAt.IsZero() {
t.Fatalf("the last send reads %+v", last)
}
generation, putBack, err := inv.SentGeneration(ctx, record.ID)
if err != nil || generation != 38 || putBack {
t.Fatalf("what the machine was last sent of it reads %d, %v (%v)", generation, putBack, err)
if generation, err := inv.SentGeneration(ctx, record.ID); err != nil || generation != 38 {
t.Fatalf("what the machine was last sent of it reads %d (%v)", generation, err)
}
refused, found, err := inv.RefusedSend(ctx, record.ID, 12, 40)
if err != nil || !found || refused.Sender != stale.Sender || refused.RefusedApplied != 40 {
refused, found, err := inv.RefusedSend(ctx, record.ID, 12, 40, 0)
if err != nil || !found || refused.Sender != stale.Sender || refused.RefusedApplied != 40 ||
refused.CounterRaisedTo != 0 {
t.Fatalf("the refusal is not kept on the send it refused: %+v, %v, %v", refused, found, err)
}
since, err := inv.RefusedSendsSince(ctx, time.Now().Add(-time.Hour))
if err != nil || len(since) != 1 || since[0].NodeName != node || since[0].Sequence != 12 {
t.Fatalf("the refused sends within the hour read %+v (%v)", since, err)
}
if _, found, err := inv.RefusedSend(ctx, record.ID, 99, 40); err != nil || found {
if _, found, err := inv.RefusedSend(ctx, record.ID, 99, 40, 0); err != nil || found {
t.Errorf("a refusal of a send never recorded was found: %v, %v", found, err)
}
}
// **A refusal of a sequence the mesh has no send for is kept too, its sender unknown**, so S20 raises it;
// it is not the machine's last send, and it says what the counter was raised to.
func TestARefusalOfASendNeverRecordedIsKeptWithAnUnknownSender(t *testing.T) {
inv, node := aNodeWithModules(t)
ctx := t.Context()
record, err := inv.NodeByName(ctx, node)
if err != nil {
t.Fatal(err)
}
kept, err := inv.RecordUnrecordedRefusal(ctx, Send{Node: record.ID, Sequence: 99, Epoch: 57, Generation: 41,
Digest: "d99", RefusedApplied: 44, CounterRaisedTo: 45})
if err != nil {
t.Fatal(err)
}
if kept.Recorded || kept.Sender != unknownSender || kept.RefusedAt == nil || kept.CounterRaisedTo != 45 ||
kept.NodeName != node {
t.Fatalf("the refusal reads %+v", kept)
}
since, err := inv.RefusedSendsSince(ctx, time.Now().Add(-time.Hour))
if err != nil || len(since) != 1 || since[0].Sequence != 99 {
t.Fatalf("the refused sends within the hour read %+v (%v)", since, err)
}
if _, found, err := inv.LastSend(ctx, node); err != nil || found {
t.Errorf("a refusal of a send never recorded reads as the machine's last send: %v, %v", found, err)
}
}
// **The machine's record and the send's are written together, and a send record that cannot be written
// leaves the machine's record standing and says so**: the digest and generation are the machine's, and a
// send that is away must not read as failed, nor the machine as behind.
func TestASendRecordThatCannotBeWrittenLeavesTheMachinesRecord(t *testing.T) {
inv, node := aNodeWithModules(t)
ctx := t.Context()
record, err := inv.NodeByName(ctx, node)
if err != nil {
t.Fatal(err)
}
// A NUL byte is text PostgreSQL refuses: the send's own row cannot be written.
broken := Send{Sequence: 13, Sender: "a test", Generation: 42, Digest: "d\x0013"}
err = inv.RecordSentWith(ctx, record.ID, "d13", nil, 0, 42, broken)
var unrecorded *SendNotRecordedError
if !errors.As(err, &unrecorded) {
t.Fatalf("a send record that could not be written read as %v", err)
}
if generation, err := inv.SentGeneration(ctx, record.ID); err != nil || generation != 42 {
t.Fatalf("the machine's generation was not written with its digest: %d (%v)", generation, err)
}
if sent, err := inv.Outstanding(ctx, node); err != nil || sent != "d13" {
t.Fatalf("the machine's digest was not written: %q (%v)", sent, err)
}
}
// **An update that leaves an assignment as it was raises nothing** (the trigger's WHEN).
func TestAnUpdateThatChangesNoAssignmentRaisesNothing(t *testing.T) {
inv, node := aNodeWithModules(t, "web")
ctx := t.Context()
if _, err := inv.Assign(ctx, node, "web"); err != nil {
t.Fatal(err)
}
before := generationNow(t, inv)
if _, err := inv.store.Pool().Exec(ctx, `update assignment set module = module`); err != nil {
t.Fatal(err)
}
if after := generationNow(t, inv); after != before {
t.Errorf("an update that changed nothing moved the generation from %d to %d", before, after)
}
}
// **A machine that reads a generation is recorded from its reports**, and one rolled back says so no longer.
func TestWhetherAMachineReadsAGenerationIsItsLatestWord(t *testing.T) {
inv, node := aNodeWithModules(t)