diff --git a/cmd/mesh-controller/facts.go b/cmd/mesh-controller/facts.go index c1b6c891..50b891cf 100644 --- a/cmd/mesh-controller/facts.go +++ b/cmd/mesh-controller/facts.go @@ -515,7 +515,7 @@ func composedAndValidated(ctx context.Context, open *stores, node string, gens m return sendable{}, nil, err } // And the generation it was last sent, as the would-send is (novox/hq issue 234). - if declared.Generation, declared.PutBack, err = open.inventory.SentGeneration(ctx, record.ID); err != nil { + if declared.Generation, err = open.inventory.SentGeneration(ctx, record.ID); err != nil { return sendable{}, nil, err } body, err := declared.Body() diff --git a/cmd/mesh-controller/gate.go b/cmd/mesh-controller/gate.go index d7a71972..36eefc4d 100644 --- a/cmd/mesh-controller/gate.go +++ b/cmd/mesh-controller/gate.go @@ -994,8 +994,7 @@ func gateFailed(ctx context.Context, open *stores, p *inventory.Plan, module str } return } - sent, err := sendRollout(withScope(ctx, sendScope{modules: map[string]bool{module: true}, putBack: true}), open, - g.Machines) + sent, err := sendRollout(withScope(ctx, sendScope{modules: map[string]bool{module: true}}), open, g.Machines) if err != nil { notBack(fmt.Sprintf("its registered build is back at %s, and sending it to %s was refused: %v — `push %s` "+ "sends it", short(previous.Commit), strings.Join(g.Machines, ", "), err, g.Machines[0])) @@ -1088,7 +1087,7 @@ func sendRollbacks(ctx context.Context, open *stores, p *inventory.Plan, b *roll } inv := open.inventory sort.Strings(b.machines) - sent, err := sendRollout(withScope(ctx, sendScope{modules: b.modules, putBack: true}), open, b.machines) + sent, err := sendRollout(withScope(ctx, sendScope{modules: b.modules}), open, b.machines) var back []string for _, r := range b.pending { if err != nil { diff --git a/cmd/mesh-controller/generation.go b/cmd/mesh-controller/generation.go index f42d3fff..15f79cbb 100644 --- a/cmd/mesh-controller/generation.go +++ b/cmd/mesh-controller/generation.go @@ -33,15 +33,20 @@ const kindOlderGeneration = "older-generation" // generationRefusalsSaid is how long a refused send is said after its refusal: an hour, as an advisory is. const generationRefusalsSaid = advisoryQuiet -// stamp gives a composed declaration the order allotted to it before it was composed: its sequence, the -// epoch and generation when the machine reads them, and the put-back mark beside a generation — and keeps -// the generation and acting epoch for the record of the send whether or not the machine is sent them. +// stamp gives a composed declaration the order allotted to it before it was composed: its sequence, and +// the epoch and generation when the machine reads them — and keeps the generation and acting epoch for the +// record of the send whether or not the machine is sent them. +// +// **No put-back mark is sent.** A gate puts a machine back by composing it again (sendRollout), so its +// declaration is allotted here like any other and carries the generation as it stands — never older than +// what the machine applied. The node-engine reads a `put_back` key (mesh-host#82); this controller never +// needs to send one. func (o order) stamp(d *sendable) { d.Sequence, d.Epoch = o.sequence, o.epoch - d.composedFrom, d.actingEpoch, d.putBackSend = o.generation, o.acting, o.putBack - d.Generation, d.PutBack = 0, false + d.composedFrom, d.actingEpoch = o.generation, o.acting + d.Generation = 0 if o.readsGeneration && o.generation > 0 { - d.Generation, d.PutBack = o.generation, o.putBack + d.Generation = o.generation } } @@ -50,18 +55,17 @@ type sentRecord struct { sequence int64 epoch uint64 generation int64 - putBack bool - // told is the generation and put-back mark on the wire, which the would-send is stamped with. + // toldGeneration is the generation on the wire, which the would-send is stamped with: zero for a machine + // whose node-engine has not said it reads one. toldGeneration int64 - toldPutBack bool sender string modules []string } // sentRecordOf is a composed send's record: its sender is the caller this process acts for. func sentRecordOf(ctx context.Context, d sendable) sentRecord { - return sentRecord{sequence: d.Sequence, epoch: d.actingEpoch, generation: d.composedFrom, putBack: d.putBackSend, - toldGeneration: d.Generation, toldPutBack: d.PutBack, sender: senderOf(callerOf(ctx)), modules: d.modules} + return sentRecord{sequence: d.Sequence, epoch: d.actingEpoch, generation: d.composedFrom, + toldGeneration: d.Generation, sender: senderOf(callerOf(ctx)), modules: d.modules} } // callerOf is who asked for what this process does: the seat call's caller when ctx belongs to one, the @@ -92,13 +96,19 @@ func namedModules(plan catalogue.Resolution, leftOut map[string]string) []string return out } -// heardGenerationRefusal keeps a machine's refusal of a send for its generation on the send it refused, -// so S20 names its sender, and says it (novox/hq issue 234). +// heardGenerationRefusal keeps a machine's refusal of a send for its generation, so S20 names its sender +// and says it (novox/hq issue 234): on the send it refused or — when the mesh has no record of sending that +// sequence — as a refusal of its own, naming the sender as unknown. Either way S20 is raised: a refusal the +// mesh cannot attribute is the louder fact, not a quieter one. // // **And raises the mesh's counter past what the machine applied, when the refused send was composed from -// the counter as it stands**: then the sender's view was not stale — the counter is behind the machine, -// which is a store put back from a backup — and every send after would be refused for ever. A stale -// sender's generation is below the counter, and the counter is left alone for it. +// the counter as it stands** (counterBehind): then the sender's view was not stale — the counter is behind +// the machine, which is a store put back from a backup — and every send after would be refused for ever. A +// stale sender's generation is below the counter, and the counter is left alone for it. +// +// Nothing is sent from here. This runs in the controller's receive loop, and a push from it would hold that +// loop, and the machine's hold, for as long as the push takes — the deaf controller of issues 184 and 185. +// S20 names `push ` for that case instead. func heardGenerationRefusal(ctx context.Context, inv *inventory.Inventory, report link.Report) { r := report.OlderGeneration if r == nil || inv == nil || report.Node == "" { @@ -106,35 +116,44 @@ func heardGenerationRefusal(ctx context.Context, inv *inventory.Inventory, repor } node, err := inv.NodeByName(ctx, report.Node) if err != nil { - fmt.Fprintf(os.Stderr, "mesh-controller: %s refused a send for its generation, and the machine cannot be read: %v\n", - report.Node, err) + fmt.Fprintf(os.Stderr, "mesh-controller: %s refused a send for its generation, and the machine cannot be "+ + "read, so it is not raised: %v\n", report.Node, err) return } - send, found, err := inv.RefusedSend(ctx, node.ID, report.Sequence, r.Applied) - switch { - case err != nil: - fmt.Fprintf(os.Stderr, "mesh-controller: %s refused send %d for its generation, and the refusal could not be "+ - "kept: %v\n", report.Node, report.Sequence, err) - case !found: - fmt.Fprintf(os.Stderr, "mesh-controller: %s refused send %d (generation %d; it applied %d), which the mesh "+ - "has no record of sending\n", report.Node, report.Sequence, r.Generation, r.Applied) - default: - fmt.Fprintf(os.Stderr, "mesh-controller: %s refused send %d from %s: composed from assignment generation %d, "+ - "and it applied %d\n", report.Node, report.Sequence, send.Sender, r.Generation, r.Applied) + var raised int64 + if now, err := inv.AssignmentGeneration(ctx); err != nil { + fmt.Fprintf(os.Stderr, "mesh-controller: %s refused a send for its generation, and the mesh's own cannot be "+ + "read: %v\n", report.Node, err) + } else if counterBehind(*r, now) { + if raised, err = inv.RaiseAssignmentGeneration(ctx, r.Applied); err != nil { + fmt.Fprintf(os.Stderr, "mesh-controller: the assignment generation (%d) is behind what %s applied (%d), "+ + "and could not be raised: %v\n", now, report.Node, r.Applied, err) + raised = 0 + } else { + fmt.Fprintf(os.Stderr, "mesh-controller: the assignment generation was %d, behind what %s applied (%d) — "+ + "a store put back from a backup — and is raised to %d; `push %s` sends it what the mesh holds now\n", + now, report.Node, r.Applied, raised, report.Node) + } } - now, err := inv.AssignmentGeneration(ctx) - if err != nil || r.Generation < now || r.Applied < now { - return + send, found, err := inv.RefusedSend(ctx, node.ID, report.Sequence, r.Applied, raised) + if err == nil && !found { + send, err = inv.RecordUnrecordedRefusal(ctx, inventory.Send{Node: node.ID, Sequence: report.Sequence, + Epoch: report.Epoch, Generation: r.Generation, Digest: report.Declared, RefusedApplied: r.Applied, + CounterRaisedTo: raised}) } - raised, err := inv.RaiseAssignmentGeneration(ctx, r.Applied) if err != nil { - fmt.Fprintf(os.Stderr, "mesh-controller: the assignment generation (%d) is behind what %s applied (%d), and "+ - "could not be raised: %v\n", now, report.Node, r.Applied, err) + fmt.Fprintf(os.Stderr, "mesh-controller: %s refused send %d for its generation, and the refusal could not be "+ + "kept, so it is not raised: %v\n", report.Node, report.Sequence, err) return } - fmt.Fprintf(os.Stderr, "mesh-controller: the assignment generation was %d, behind what %s applied (%d) — a store "+ - "put back from a backup — and is raised to %d, so the next send is not refused\n", now, report.Node, r.Applied, - raised) + fmt.Fprintf(os.Stderr, "mesh-controller: %s refused send %d from %s: composed from assignment generation %d, "+ + "and it applied %d\n", report.Node, report.Sequence, send.Sender, r.Generation, r.Applied) +} + +// counterBehind says a refusal shows the mesh's counter behind the machine rather than a stale sender: the +// refused send carried the counter as it stands now, and the machine applied more than that. +func counterBehind(r link.GenerationRefusal, now int64) bool { + return r.Generation >= now && r.Applied > r.Generation } // watchGenerationRefusals is S20: every send a machine refused within the hour for the generation it was @@ -147,7 +166,7 @@ func watchGenerationRefusals(f *signalFacts) []conditions.Observation { continue } seen[s.NodeName] = true - out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: s.NodeName, Kind: kindOlderGeneration, + o := conditions.Observation{Scope: conditions.ScopeMachine, ID: s.NodeName, Kind: kindOlderGeneration, Machine: s.NodeName, Severity: conditions.Warning, Summary: fmt.Sprintf("%s refused sequence %d from %s: it was composed from assignment generation %d, and "+ "%s applied generation %d — a sender composing from a view of the assignments the mesh has moved "+ @@ -157,7 +176,20 @@ func watchGenerationRefusals(f *signalFacts) []conditions.Observation { Headline: fmt.Sprintf("%s refused an out-of-date update", s.NodeName), Explanation: fmt.Sprintf("Something sent %s an update made from an older list of what runs there. %s "+ "refused it, so nothing was removed. Nothing for you to do unless it repeats.", s.NodeName, s.NodeName), - Resolved: fmt.Sprintf("%s has had no out-of-date update for an hour", s.NodeName)}) + Resolved: fmt.Sprintf("%s has had no out-of-date update for an hour", s.NodeName)} + if s.CounterRaisedTo > 0 { + // Not a stale sender: the mesh's counter was behind the machine (a store put back from a backup) and + // was raised; nothing has sent the machine its declaration since, so a person is asked to. + o.Summary = fmt.Sprintf("%s refused sequence %d from %s: it was composed from assignment generation %d, "+ + "the mesh's own, and %s applied generation %d — the mesh's counter was behind the machine (a store "+ + "put back from a backup?) and is raised to %d; `push %s` sends it what the mesh holds now", + s.NodeName, s.Sequence, s.Sender, s.Generation, s.NodeName, s.RefusedApplied, s.CounterRaisedTo, + s.NodeName) + o.Explanation = fmt.Sprintf("Needs you: push %s. The mesh's record was older than %s, so %s refused its "+ + "update and kept what it had. The record is repaired; a push sends the update again.", + s.NodeName, s.NodeName, s.NodeName) + } + out = append(out, o) } return out } @@ -181,12 +213,7 @@ func writeLastSend(ctx context.Context, w io.Writer, inv *inventory.Inventory, n if s.Generation > 0 { generation = fmt.Sprintf("assignment generation %d", s.Generation) } - kind := "sent" - if s.PutBack { - kind = "put back" - } - fmt.Fprintf(w, "%s was last %s sequence %d at %s by %s, composed from %s, naming %s\n", node, kind, - s.Sequence, s.SentAt.Local().Format("2006-01-02 15:04:05"), s.Sender, generation, modulesWords(s.Modules)) + fmt.Fprintf(w, "%s was last sent sequence %d at %s by %s, composed from %s, naming %s\n", node, s.Sequence, s.SentAt.Local().Format("2006-01-02 15:04:05"), s.Sender, generation, modulesWords(s.Modules)) if s.RefusedAt != nil { fmt.Fprintf(w, " and refused it at %s: it had applied generation %d\n", s.RefusedAt.Local().Format("2006-01-02 15:04:05"), s.RefusedApplied) diff --git a/cmd/mesh-controller/generation_test.go b/cmd/mesh-controller/generation_test.go index f794760c..a2372bcc 100644 --- a/cmd/mesh-controller/generation_test.go +++ b/cmd/mesh-controller/generation_test.go @@ -7,6 +7,7 @@ import ( "time" "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" ) // The assignment generation a declaration was composed from (novox/hq issue 234). @@ -25,45 +26,35 @@ func bodyKeys(t *testing.T, s sendable) map[string]any { } // **The generation goes on the wire only to a machine whose node-engine said it reads one**: an older -// node-engine decodes strictly and refuses an unknown key, whole. And the put-back mark only with it. +// node-engine decodes strictly and refuses an unknown key, whole. No put-back mark is ever sent: a gate +// composes its put-back afresh, with the generation as it stands. func TestTheGenerationIsSentOnlyToAMachineThatReadsOne(t *testing.T) { resources := []map[string]any{{"id": "a", "type": "file", "path": "/etc/a", "content": "x\n"}} - reads := order{sequence: 12, epoch: 57, generation: 40, readsGeneration: true, acting: 57, putBack: true} + reads := order{sequence: 12, epoch: 57, generation: 40, readsGeneration: true, acting: 57} var told sendable told.Resources = resources reads.stamp(&told) keys := bodyKeys(t, told) - if keys["generation"] != float64(40) || keys["put_back"] != true || keys["sequence"] != float64(12) { + if keys["generation"] != float64(40) || keys["sequence"] != float64(12) { t.Fatalf("a machine that reads a generation was sent %v", keys) } + if _, there := keys["put_back"]; there { + t.Fatalf("a put-back mark was sent: %v", keys) + } if told.composedFrom != 40 || told.actingEpoch != 57 { t.Errorf("the send does not keep what it was composed from for its record: %+v", told) } - older := order{sequence: 12, epoch: 57, generation: 40, readsGeneration: false, acting: 57, putBack: true} + older := order{sequence: 12, epoch: 57, generation: 40, readsGeneration: false, acting: 57} var untold sendable untold.Resources = resources older.stamp(&untold) - keys = bodyKeys(t, untold) - if _, there := keys["generation"]; there { - t.Fatalf("a machine whose node-engine never said it reads a generation was sent one: %v", keys) - } - if _, there := keys["put_back"]; there { - t.Fatalf("a machine whose node-engine never said it reads a generation was sent a put-back mark: %v", keys) + if _, there := bodyKeys(t, untold)["generation"]; there { + t.Fatal("a machine whose node-engine never said it reads a generation was sent one") } if untold.composedFrom != 40 { t.Errorf("the generation it was composed from is recorded whether or not it was sent: %+v", untold) } - - // An ordinary send carries no put-back mark at all: the body is what it was apart from the generation. - var ordinary sendable - ordinary.Resources = resources - ordinaryOrder := reads - ordinaryOrder.putBack = false - ordinaryOrder.stamp(&ordinary) - if _, there := bodyKeys(t, ordinary)["put_back"]; there { - t.Fatal("an ordinary send says it is a put-back") - } } // **The sender is named**: the caller of the seat call when there is one, else the shell's account, and the @@ -79,7 +70,7 @@ func TestASendNamesItsSender(t *testing.T) { func refusedSend(at time.Time) inventory.Send { return inventory.Send{NodeName: "anchor", Sequence: 12, Epoch: 57, Generation: 38, RefusedApplied: 40, Sender: "a one-shot push by jochen at a shell on anchor (pid 4242 on anchor, build 2026.10.11)", - Modules: []string{"docker"}, SentAt: at.Add(-time.Second), RefusedAt: &at} + Modules: []string{"docker"}, SentAt: at.Add(-time.Second), RefusedAt: &at, Recorded: true} } // **A refused send is raised naming its sender** (novox/hq issue 234): who sent it, from which generation, @@ -105,3 +96,53 @@ func TestARefusedSendIsRaisedNamingItsSender(t *testing.T) { t.Errorf("the condition is not worded for the operator: %+v", o) } } + +// **A refusal of a send the mesh has no record of is raised too, its sender said to be unknown** — never +// only a line on stderr. +func TestARefusalOfAnUnrecordedSendIsRaisedSayingTheSenderIsUnknown(t *testing.T) { + now := time.Date(2026, 10, 11, 12, 0, 0, 0, time.UTC) + f := calm(now) + s := refusedSend(now.Add(-time.Minute)) + s.Recorded, s.Sender, s.Modules = false, "a sender the mesh has no record of (no send of this sequence was recorded)", nil + f.refusedSends = []inventory.Send{s} + got := watchGenerationRefusals(f) + if len(got) != 1 || !strings.Contains(got[0].Summary, "no record of") { + t.Fatalf("an unattributed refusal was not raised as one: %+v", got) + } +} + +// **When the refusal showed the mesh's counter behind the machine, the condition asks for a push** — it was +// raised, and nothing has sent the machine its declaration since — and does not say there is nothing to do. +func TestACounterBehindTheMachineAsksForAPush(t *testing.T) { + now := time.Date(2026, 10, 11, 12, 0, 0, 0, time.UTC) + f := calm(now) + s := refusedSend(now.Add(-time.Minute)) + s.Generation, s.RefusedApplied, s.CounterRaisedTo = 12, 40, 41 + f.refusedSends = []inventory.Send{s} + got := watchGenerationRefusals(f) + if len(got) != 1 { + t.Fatalf("%+v", got) + } + if !strings.Contains(got[0].Summary, "`push anchor`") || !strings.Contains(got[0].Explanation, "push anchor") || + strings.Contains(got[0].Explanation, "Nothing for you to do") { + t.Errorf("a counter behind the machine does not ask for a push: %s / %s", got[0].Summary, got[0].Explanation) + } +} + +// **Only a refusal of the counter as it stands is the counter behind**: a stale sender's generation is below +// it, and the counter is left alone for that. +func TestOnlyARefusalOfTheCounterAsItStandsRaisesIt(t *testing.T) { + for _, c := range []struct { + refused, applied, now int64 + behind bool + }{ + {refused: 12, applied: 40, now: 12, behind: true}, // the store was put back: the mesh says 12, the machine had 40 + {refused: 38, applied: 40, now: 41, behind: false}, // a stale sender: the counter is already past + {refused: 38, applied: 40, now: 40, behind: false}, // a stale sender: the counter is where the machine is + } { + r := link.GenerationRefusal{Generation: c.refused, Applied: c.applied} + if got := counterBehind(r, c.now); got != c.behind { + t.Errorf("refused %d, applied %d, counter %d: behind %v, want %v", c.refused, c.applied, c.now, got, c.behind) + } + } +} diff --git a/cmd/mesh-controller/push.go b/cmd/mesh-controller/push.go index be33434b..9f3c1434 100644 --- a/cmd/mesh-controller/push.go +++ b/cmd/mesh-controller/push.go @@ -359,14 +359,12 @@ func declare(ctx context.Context, args []string) error { Epoch uint64 `json:"epoch"` Sequence int64 `json:"sequence"` Generation int64 `json:"generation"` - PutBack bool `json:"put_back"` } _ = json.Unmarshal(raw, &carried) // And recorded as every send is (novox/hq issue 234): by hand, from what it carried; the modules it // named are not known, since the mesh did not compose it. if _, err := recordSent(ctx, inv, node, raw, nil, carried.Epoch, sentRecord{sequence: carried.Sequence, - epoch: carried.Epoch, generation: carried.Generation, putBack: carried.PutBack, - toldGeneration: carried.Generation, toldPutBack: carried.PutBack, + epoch: carried.Epoch, generation: carried.Generation, toldGeneration: carried.Generation, sender: senderOf(callerOf(ctx)) + ", a declaration sent by hand"}); err != nil { return err } @@ -1402,9 +1400,9 @@ func wouldSendFrom(ctx context.Context, open *stores, if declared.Epoch, err = open.inventory.SentEpoch(ctx, n.ID); err != nil { return nil, err } - // And the generation and put-back mark it was last sent, for the same reason (novox/hq issue 234): - // an assignment elsewhere in the mesh is not a change of this machine. - if declared.Generation, declared.PutBack, err = open.inventory.SentGeneration(ctx, n.ID); err != nil { + // And the generation it was last sent, for the same reason (novox/hq issue 234): an assignment + // elsewhere in the mesh is not a change of this machine. + if declared.Generation, err = open.inventory.SentGeneration(ctx, n.ID); err != nil { return nil, err } body, err := declared.Body() @@ -1551,8 +1549,6 @@ type order struct { generation int64 readsGeneration bool acting uint64 - // putBack is a gate's put-back (sendScope.putBack), read from the send's context. - putBack bool } // allot takes the next sequence for a machine — the number its next declaration carries — under the @@ -1593,7 +1589,7 @@ func allot(ctx context.Context, inv *inventory.Inventory, node string) (order, e return order{}, err } return order{sequence: seq, epoch: epoch, generation: generation, readsGeneration: readsGeneration, - acting: acting, putBack: scopeOf(ctx).putBack}, nil + acting: acting}, nil } // recordSent writes down what a machine was just sent, and returns the digest. @@ -1616,17 +1612,23 @@ func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body return "", err } digest := digestOf(body) - if err := inv.RecordSentUnder(kept, record.ID, digest, builds, epoch); err != nil { + // The digest and the generation it carried are written in one transaction (novox/hq issue 234), so the + // would-send is never stamped with a generation the machine was not sent; and the send itself, who sent + // it and from which generation, beside them. A record of the send that cannot be written does not make a + // send that is away look failed: it is said, loudly, and the machine's own record stands. + err = inv.RecordSentWith(kept, record.ID, digest, builds, epoch, sent.toldGeneration, inventory.Send{ + Sequence: sent.sequence, Epoch: int64(sent.epoch), Sender: sent.sender, Generation: sent.generation, + Digest: digest, Modules: sent.modules}) + var unrecorded *inventory.SendNotRecordedError + switch { + case errors.As(err, &unrecorded): + fmt.Fprintf(os.Stderr, "mesh-controller: SEND NOT RECORDED: %s was sent declaration %s (sequence %d), and who "+ + "sent it and from which generation could not be written down, so a refusal of it will name no sender "+ + "(novox/hq issue 234): %v\n", node, short(digest), sent.sequence, unrecorded.Err) + fmt.Printf("%s: sent, and the record of who sent it could NOT be written: %v\n", node, unrecorded.Err) + case err != nil: return "", err } - // And the send itself, who sent it and from which generation (novox/hq issue 234): what the machine - // was last told, by whom, is read back from here — and what the would-send is stamped with. - if err := inv.RecordSend(kept, inventory.Send{Node: record.ID, Sequence: sent.sequence, - Epoch: int64(sent.epoch), Sender: sent.sender, Generation: sent.generation, PutBack: sent.putBack, - ToldGeneration: sent.toldGeneration, ToldPutBack: sent.toldPutBack, Digest: digest, - Modules: sent.modules}); err != nil { - return "", fmt.Errorf("%s was sent its declaration, and the send could not be recorded: %w", node, err) - } // And what it was, summarised, so the next push can be compared with it (novox/hq ADR 0217). // Never a reason to fail a send that is already away: a summary that cannot be kept means the // next comparison has nothing to hold against, which is how every machine starts. diff --git a/cmd/mesh-controller/release.go b/cmd/mesh-controller/release.go index 31c1ef87..56b702c8 100644 --- a/cmd/mesh-controller/release.go +++ b/cmd/mesh-controller/release.go @@ -50,9 +50,6 @@ type sendScope struct { modules map[string]bool // person is a person's act: it carries what it carries. person bool - // putBack is a gate putting machines back after a failed send (novox/hq issue 234): its declarations - // say so, and a machine does not refuse them for the generation they carry. - putBack bool } type sendScopeKey struct{} diff --git a/cmd/mesh-controller/sendable.go b/cmd/mesh-controller/sendable.go index cbc89943..0c1e2116 100644 --- a/cmd/mesh-controller/sendable.go +++ b/cmd/mesh-controller/sendable.go @@ -33,17 +33,11 @@ type sendable struct { // node-engine has not said it reads one is sent none, for the reason the epoch is not (an older // node-engine refuses a key it does not know, whole). Generation int64 - // PutBack marks a gate's put-back (novox/hq issue 234): it carries the generation of what it puts - // back and is not refused for it. Sent only beside a generation. - PutBack bool // composedFrom is the generation read before this declaration was composed, and actingEpoch the // lease epoch its sender acted under — whether or not the machine is sent either — for the record of // the send; never on the wire. composedFrom int64 actingEpoch uint64 - // putBackSend is whether a gate sent it to put the machine back, for the record; PutBack is the mark - // on the wire, only beside a generation. - putBackSend bool // modules are the modules this declaration names, for the record of the send; never on the wire. modules []string // Adoption is nil for a converged node, and then the body is byte for byte what it was before @@ -115,9 +109,6 @@ func (s sendable) Body() ([]byte, error) { } if s.Generation > 0 { envelope["generation"] = s.Generation - if s.PutBack { - envelope["put_back"] = true - } } // An empty declaration is deliberate here — the node owns nothing the mesh put there // (novox/hq issue 127) — and the host refuses an empty body unless it is told the emptiness diff --git a/internal/inventory/migrations/0094-a-declaration-says-the-generation-it-came-from.sql b/internal/inventory/migrations/0094-a-declaration-says-the-generation-it-came-from.sql index 07a182d3..52dde45b 100644 --- a/internal/inventory/migrations/0094-a-declaration-says-the-generation-it-came-from.sql +++ b/internal/inventory/migrations/0094-a-declaration-says-the-generation-it-came-from.sql @@ -24,26 +24,33 @@ begin end $$; --- Per row, so a statement that changes nothing (an assignment repeated, `on conflict do nothing`) raises --- nothing; a statement that changes several raises once per row, which only ever moves it forward. +-- Per row, so a statement that changes nothing raises nothing: an assignment repeated (`on conflict do +-- nothing` inserts no row) and an update that leaves a row as it was (the WHEN below; a plain UPDATE fires a +-- row trigger whether or not it changed anything). A statement that changes several rows raises once per +-- row, which only ever moves it forward. Two triggers, because only an update has both OLD and NEW. create trigger assignment_generation_raised - after insert or update or delete on assignment + after insert or delete on assignment for each row execute function raise_assignment_generation(); +create trigger assignment_generation_raised_by_update + after update on assignment + for each row when (old is distinct from new) execute function raise_assignment_generation(); --- What a machine was last sent of it: the generation, and whether that send was a gate's put-back, so +-- The generation a machine was last sent, written in the same transaction as the digest of that send, so -- what the mesh WOULD send is stamped the same way and reads as byte for byte what it DID send when --- nothing else changed (as sent_epoch, migration 0068). Null and false for a send without. +-- nothing else changed (as sent_epoch, migration 0068). Null for a send without one. alter table node add column sent_generation bigint; -alter table node add column sent_put_back boolean not null default false; -- Whether the machine's node-engine said it reads a generation (its reports' `reads_generation`): until -- it has, it is sent none, because an older node-engine refuses a key it does not know, whole. alter table node add column reads_generation boolean not null default false; -- Every send: the machine, its sequence, who sent it — the lease epoch the sender acted under and the --- caller, process and build — the generation it was composed from, whether it was a put-back, its --- digest and the modules it named. Written after the declaration is away, as the node row's record is. --- A refusal by the machine of a send for its generation is kept on the send it refused, so the --- condition it raises names the sender from here. The newest 200 per machine are kept. +-- caller, process and build — the generation it was composed from, its digest and the modules it named. +-- Written after the declaration is away, with the node row's record of it. A refusal by the machine of a +-- send for its generation is kept on the send it refused, so the condition it raises names the sender +-- from here; a refusal of a sequence the mesh has no send for is kept as a row of its own, `recorded` +-- false, whose sender is said to be unknown. `counter_raised_to` is the generation the mesh's counter was +-- raised to when the refusal showed the counter behind the machine (a store put back from a backup). The +-- newest 200 per machine are kept. create table declaration_send ( id bigserial primary key, node uuid not null references node (id) on delete cascade, @@ -51,13 +58,14 @@ create table declaration_send ( epoch bigint, sender text not null, generation bigint, - put_back boolean not null default false, digest text not null, modules text[] not null default '{}', sent_at timestamptz not null default now(), refused_at timestamptz, -- refused_applied is the generation the machine said it had applied when it refused this send. - refused_applied bigint + refused_applied bigint, + counter_raised_to bigint, + recorded boolean not null default true ); create index declaration_send_node on declaration_send (node, id desc); create index declaration_send_sequence on declaration_send (node, sequence); diff --git a/internal/inventory/sends.go b/internal/inventory/sends.go index 085a2ac9..8a6efa36 100644 --- a/internal/inventory/sends.go +++ b/internal/inventory/sends.go @@ -2,6 +2,7 @@ package inventory import ( "context" + "encoding/json" "fmt" "time" @@ -47,20 +48,17 @@ func (i *Inventory) RaiseAssignmentGeneration(ctx context.Context, past int64) ( return g, nil } -// SentGeneration is the generation a machine was last sent and whether that send was a put-back, by its -// id; zero for one sent without. What the mesh WOULD send is stamped with these, so it reads as byte for -// byte what it DID send when nothing else changed. -func (i *Inventory) SentGeneration(ctx context.Context, id string) (int64, bool, error) { +// SentGeneration is the generation a machine was last sent, by its id; zero for one sent without. What the +// mesh WOULD send is stamped with it, so it reads as byte for byte what it DID send when nothing else changed. +func (i *Inventory) SentGeneration(ctx context.Context, id string) (int64, error) { var g *int64 - var putBack bool - if err := i.store.Pool().QueryRow(ctx, `select sent_generation, sent_put_back from node where id = $1`, id). - Scan(&g, &putBack); err != nil { - return 0, false, fmt.Errorf("reading the generation %s was last sent: %w", id, err) + if err := i.store.Pool().QueryRow(ctx, `select sent_generation from node where id = $1`, id).Scan(&g); err != nil { + return 0, fmt.Errorf("reading the generation %s was last sent: %w", id, err) } if g == nil { - return 0, putBack, nil + return 0, nil } - return *g, putBack, nil + return *g, nil } // ReadsGeneration says a machine's node-engine said it reads a generation in a declaration, by its id. @@ -88,66 +86,122 @@ type Send struct { Epoch int64 // Sender is who sent it, in words: the caller, and the process and build that composed it. Sender string - // Generation is the assignment generation it was composed from, zero when not known; PutBack whether a - // gate sent it to put the machine back. + // Generation is the assignment generation it was composed from, zero when not known. Generation int64 - PutBack bool - // ToldGeneration and ToldPutBack are what the machine was told of them on the wire: zero and false for a - // machine whose node-engine has not said it reads a generation. What the would-send is stamped with. - ToldGeneration int64 - ToldPutBack bool - Digest string + Digest string // Modules are the modules it named. Modules []string SentAt time.Time // RefusedAt is when the machine refused it for its generation, nil when it did not; RefusedApplied the - // generation the machine said it had applied then. - RefusedAt *time.Time - RefusedApplied int64 + // generation the machine said it had applied then; CounterRaisedTo what the mesh's counter was raised to + // because the refusal showed it behind the machine, zero when it was not. + RefusedAt *time.Time + RefusedApplied int64 + CounterRaisedTo int64 + // Recorded is false for a refusal of a sequence the mesh has no send for: its sender is unknown. + Recorded bool } -// RecordSend writes one send down, and what the machine was last sent of its generation, in one -// transaction; the oldest beyond the newest 200 for the machine are let go. -func (i *Inventory) RecordSend(ctx context.Context, s Send) error { +// unknownSender is the sender of a refused sequence the mesh has no record of sending. +const unknownSender = "a sender the mesh has no record of (no send of this sequence was recorded: sent by " + + "hand, before sends were recorded, or by a controller whose record was not written)" + +// SendNotRecordedError is a send whose machine's record was written — its digest and the generation it +// carried — and whose record of who sent it was not. The send is away and the machine's record stands; the +// caller says this loudly and does not take the send for failed. +type SendNotRecordedError struct{ Err error } + +func (e *SendNotRecordedError) Error() string { + return "the record of the send was not written: " + e.Err.Error() +} +func (e *SendNotRecordedError) Unwrap() error { return e.Err } + +// RecordSentWith writes down what a machine was just sent — its digest, the builds it carried, the epoch and +// the generation on the wire — and the send itself, who sent it and from which generation (novox/hq issue +// 234), in one transaction. The digest and the generation are written together, so the would-send is never +// stamped with a generation the machine was not sent. The send's own record is written under a savepoint: if +// it cannot be, the machine's record is still committed and a *SendNotRecordedError says so. The oldest +// sends beyond the newest 200 for the machine are let go. +func (i *Inventory) RecordSentWith(ctx context.Context, node, digest string, builds map[string]string, epoch uint64, + generation int64, s Send) error { + var sentEpoch *int64 + if epoch > 0 { + e := int64(epoch) + sentEpoch = &e + } + var carried *string + if builds != nil { + raw, err := json.Marshal(builds) + if err != nil { + return err + } + text := string(raw) + carried = &text + } tx, err := i.store.Pool().Begin(ctx) if err != nil { return err } defer func() { _ = tx.Rollback(ctx) }() - if _, err := tx.Exec(ctx, `update node set sent_generation = $2, sent_put_back = $3 where id = $1`, - s.Node, nullIfZero(s.ToldGeneration), s.ToldPutBack); err != nil { + if _, err := tx.Exec(ctx, + `update node set sent = $2, sent_at = now(), sent_builds = $3::jsonb, sent_epoch = $4, sent_generation = $5 + where id = $1`, node, digest, carried, sentEpoch, nullIfZero(generation)); err != nil { return err } + recordErr := recordSend(ctx, tx, node, s) + if err := tx.Commit(ctx); err != nil { + return err + } + if recordErr != nil { + return &SendNotRecordedError{Err: recordErr} + } + return nil +} + +// recordSend writes one send under a savepoint of tx, which it rolls back when the send cannot be written. +func recordSend(ctx context.Context, tx pgx.Tx, node string, s Send) error { + sp, err := tx.Begin(ctx) + if err != nil { + return err + } + defer func() { _ = sp.Rollback(ctx) }() modules := s.Modules if modules == nil { modules = []string{} } - if _, err := tx.Exec(ctx, - `insert into declaration_send (node, sequence, epoch, sender, generation, put_back, digest, modules) - values ($1, $2, $3, $4, $5, $6, $7, $8)`, - s.Node, nullIfZero(s.Sequence), nullIfZero(s.Epoch), s.Sender, nullIfZero(s.Generation), s.PutBack, - s.Digest, modules); err != nil { + if _, err := sp.Exec(ctx, + `insert into declaration_send (node, sequence, epoch, sender, generation, digest, modules) + values ($1, $2, $3, $4, $5, $6, $7)`, + node, nullIfZero(s.Sequence), nullIfZero(s.Epoch), s.Sender, nullIfZero(s.Generation), s.Digest, + modules); err != nil { return err } - if _, err := tx.Exec(ctx, + if err := pruneSends(ctx, sp, node); err != nil { + return err + } + return sp.Commit(ctx) +} + +// pruneSends lets go of a machine's sends beyond the newest 200. +func pruneSends(ctx context.Context, q queries, node string) error { + _, err := q.Exec(ctx, `delete from declaration_send where node = $1 and id not in - (select id from declaration_send where node = $1 order by id desc limit $2)`, s.Node, sendsKept); err != nil { - return err - } - return tx.Commit(ctx) + (select id from declaration_send where node = $1 order by id desc limit $2)`, node, sendsKept) + return err } // sendColumns are a send's columns as scanSends reads them. const sendColumns = `s.node, n.name, coalesce(s.sequence, 0), coalesce(s.epoch, 0), s.sender, coalesce(s.generation, 0), - s.put_back, s.digest, s.modules, s.sent_at, s.refused_at, coalesce(s.refused_applied, 0)` + s.digest, s.modules, s.sent_at, s.refused_at, coalesce(s.refused_applied, 0), coalesce(s.counter_raised_to, 0), + s.recorded` func scanSends(rows pgx.Rows) ([]Send, error) { defer rows.Close() var out []Send for rows.Next() { var s Send - if err := rows.Scan(&s.Node, &s.NodeName, &s.Sequence, &s.Epoch, &s.Sender, &s.Generation, &s.PutBack, - &s.Digest, &s.Modules, &s.SentAt, &s.RefusedAt, &s.RefusedApplied); err != nil { + if err := rows.Scan(&s.Node, &s.NodeName, &s.Sequence, &s.Epoch, &s.Sender, &s.Generation, &s.Digest, + &s.Modules, &s.SentAt, &s.RefusedAt, &s.RefusedApplied, &s.CounterRaisedTo, &s.Recorded); err != nil { return nil, err } out = append(out, s) @@ -155,11 +209,12 @@ func scanSends(rows pgx.Rows) ([]Send, error) { return out, rows.Err() } -// LastSend is the last declaration a machine was sent, by its name; false when none was recorded. +// LastSend is the last declaration a machine was sent, by its name; false when none was recorded. A refusal +// of a sequence the mesh has no send for is not a send, and is not it. func (i *Inventory) LastSend(ctx context.Context, name string) (Send, bool, error) { rows, err := i.store.Pool().Query(ctx, `select `+sendColumns+` from declaration_send s join node n on n.id = s.node - where n.name = $1 order by s.id desc limit 1`, name) + where n.name = $1 and s.recorded order by s.id desc limit 1`, name) if err != nil { return Send{}, false, err } @@ -171,15 +226,17 @@ func (i *Inventory) LastSend(ctx context.Context, name string) (Send, bool, erro } // RefusedSend keeps a machine's refusal of the send of a sequence for the generation it came from, and -// answers that send, sender and all; false when no send of that sequence was recorded — sent before the -// record, or by a hand the mesh did not see. The newest of that sequence when there are several: a -// declaration a person sent by hand may repeat one. -func (i *Inventory) RefusedSend(ctx context.Context, node string, sequence, applied int64) (Send, bool, error) { +// answers that send, sender and all; false when no send of that sequence was recorded (RecordUnrecordedRefusal +// keeps that one). The newest of that sequence when there are several: a declaration a person sent by hand may +// repeat one. raisedTo is what the mesh's counter was raised to for it, zero for none. +func (i *Inventory) RefusedSend(ctx context.Context, node string, sequence, applied, raisedTo int64) (Send, bool, error) { rows, err := i.store.Pool().Query(ctx, `with refused as ( - update declaration_send set refused_at = now(), refused_applied = $3 - where id = (select id from declaration_send where node = $1 and sequence = $2 order by id desc limit 1) + update declaration_send set refused_at = now(), refused_applied = $3, counter_raised_to = $4 + where id = (select id from declaration_send where node = $1 and sequence = $2 and recorded + order by id desc limit 1) returning *) - select `+sendColumns+` from refused s join node n on n.id = s.node`, node, sequence, applied) + select `+sendColumns+` from refused s join node n on n.id = s.node`, node, sequence, applied, + nullIfZero(raisedTo)) if err != nil { return Send{}, false, err } @@ -190,6 +247,34 @@ func (i *Inventory) RefusedSend(ctx context.Context, node string, sequence, appl return sends[0], true, nil } +// RecordUnrecordedRefusal keeps a machine's refusal of a sequence the mesh has no send for, as a row of its +// own whose sender is unknown, and answers it: S20 raises it like any other, because a refusal nobody can +// attribute is no quieter for that. s carries the machine, the sequence, epoch, generation and digest the +// refused declaration carried, and the refusal's applied generation and counter raise. +func (i *Inventory) RecordUnrecordedRefusal(ctx context.Context, s Send) (Send, error) { + rows, err := i.store.Pool().Query(ctx, `with refused as ( + insert into declaration_send (node, sequence, epoch, sender, generation, digest, refused_at, refused_applied, + counter_raised_to, recorded) + values ($1, $2, $3, $4, $5, $6, now(), $7, $8, false) returning *) + select `+sendColumns+` from refused s join node n on n.id = s.node`, + s.Node, nullIfZero(s.Sequence), nullIfZero(s.Epoch), unknownSender, nullIfZero(s.Generation), s.Digest, + s.RefusedApplied, nullIfZero(s.CounterRaisedTo)) + if err != nil { + return Send{}, err + } + sends, err := scanSends(rows) + if err != nil { + return Send{}, err + } + if len(sends) == 0 { + return Send{}, fmt.Errorf("the refusal of sequence %d was not kept", s.Sequence) + } + if err := pruneSends(ctx, i.store.Pool(), s.Node); err != nil { + return Send{}, err + } + return sends[0], nil +} + // RefusedSendsSince is every send refused for its generation since a moment, newest first. func (i *Inventory) RefusedSendsSince(ctx context.Context, since time.Time) ([]Send, error) { rows, err := i.store.Pool().Query(ctx, `select `+sendColumns+` diff --git a/internal/inventory/sends_test.go b/internal/inventory/sends_test.go index 9f5dfe6b..96c98d15 100644 --- a/internal/inventory/sends_test.go +++ b/internal/inventory/sends_test.go @@ -1,6 +1,7 @@ package inventory import ( + "errors" "slices" "testing" "time" @@ -75,8 +76,8 @@ func TestTheGenerationIsRaisedPastWhatAMachineApplied(t *testing.T) { } // **Every send is recorded: its sequence, its sender, the generation it came from and the modules it -// named** — and the machine's last send is what `plan` reads, the would-send is stamped with its -// generation and put-back mark, and a refusal is kept on the send it refused, naming its sender. +// named** — and the machine's last send is what `plan` reads, the would-send is stamped with the generation +// written with the digest, and a refusal is kept on the send it refused, naming its sender. func TestASendIsRecordedWithItsSenderGenerationAndModules(t *testing.T) { inv, node := aNodeWithModules(t) ctx := t.Context() @@ -84,42 +85,110 @@ func TestASendIsRecordedWithItsSenderGenerationAndModules(t *testing.T) { if err != nil { t.Fatal(err) } - first := Send{Node: record.ID, Sequence: 11, Epoch: 57, Sender: "the controller's daemon (pid 7 on anchor)", + first := Send{Sequence: 11, Epoch: 57, Sender: "the controller's daemon (pid 7 on anchor)", Generation: 40, Digest: "d11", Modules: []string{"docker", "pacman", "sudo"}} - if err := inv.RecordSend(ctx, first); err != nil { + if err := inv.RecordSentWith(ctx, record.ID, "d11", nil, 57, 40, first); err != nil { t.Fatal(err) } - stale := Send{Node: record.ID, Sequence: 12, Epoch: 57, Sender: "a one-shot push by jochen at a shell on anchor", - Generation: 38, ToldGeneration: 38, Digest: "d12", Modules: []string{"docker"}} - if err := inv.RecordSend(ctx, stale); err != nil { + stale := Send{Sequence: 12, Epoch: 57, Sender: "a one-shot push by jochen at a shell on anchor", + Generation: 38, Digest: "d12", Modules: []string{"docker"}} + if err := inv.RecordSentWith(ctx, record.ID, "d12", nil, 57, 38, stale); err != nil { t.Fatal(err) } last, found, err := inv.LastSend(ctx, node) if err != nil || !found { t.Fatalf("the last send is not kept: %v", err) } - if last.Sequence != 12 || last.Sender != stale.Sender || last.Generation != 38 || + if last.Sequence != 12 || last.Sender != stale.Sender || last.Generation != 38 || !last.Recorded || !slices.Equal(last.Modules, []string{"docker"}) || last.SentAt.IsZero() { t.Fatalf("the last send reads %+v", last) } - generation, putBack, err := inv.SentGeneration(ctx, record.ID) - if err != nil || generation != 38 || putBack { - t.Fatalf("what the machine was last sent of it reads %d, %v (%v)", generation, putBack, err) + if generation, err := inv.SentGeneration(ctx, record.ID); err != nil || generation != 38 { + t.Fatalf("what the machine was last sent of it reads %d (%v)", generation, err) } - refused, found, err := inv.RefusedSend(ctx, record.ID, 12, 40) - if err != nil || !found || refused.Sender != stale.Sender || refused.RefusedApplied != 40 { + refused, found, err := inv.RefusedSend(ctx, record.ID, 12, 40, 0) + if err != nil || !found || refused.Sender != stale.Sender || refused.RefusedApplied != 40 || + refused.CounterRaisedTo != 0 { t.Fatalf("the refusal is not kept on the send it refused: %+v, %v, %v", refused, found, err) } since, err := inv.RefusedSendsSince(ctx, time.Now().Add(-time.Hour)) if err != nil || len(since) != 1 || since[0].NodeName != node || since[0].Sequence != 12 { t.Fatalf("the refused sends within the hour read %+v (%v)", since, err) } - if _, found, err := inv.RefusedSend(ctx, record.ID, 99, 40); err != nil || found { + if _, found, err := inv.RefusedSend(ctx, record.ID, 99, 40, 0); err != nil || found { t.Errorf("a refusal of a send never recorded was found: %v, %v", found, err) } } +// **A refusal of a sequence the mesh has no send for is kept too, its sender unknown**, so S20 raises it; +// it is not the machine's last send, and it says what the counter was raised to. +func TestARefusalOfASendNeverRecordedIsKeptWithAnUnknownSender(t *testing.T) { + inv, node := aNodeWithModules(t) + ctx := t.Context() + record, err := inv.NodeByName(ctx, node) + if err != nil { + t.Fatal(err) + } + kept, err := inv.RecordUnrecordedRefusal(ctx, Send{Node: record.ID, Sequence: 99, Epoch: 57, Generation: 41, + Digest: "d99", RefusedApplied: 44, CounterRaisedTo: 45}) + if err != nil { + t.Fatal(err) + } + if kept.Recorded || kept.Sender != unknownSender || kept.RefusedAt == nil || kept.CounterRaisedTo != 45 || + kept.NodeName != node { + t.Fatalf("the refusal reads %+v", kept) + } + since, err := inv.RefusedSendsSince(ctx, time.Now().Add(-time.Hour)) + if err != nil || len(since) != 1 || since[0].Sequence != 99 { + t.Fatalf("the refused sends within the hour read %+v (%v)", since, err) + } + if _, found, err := inv.LastSend(ctx, node); err != nil || found { + t.Errorf("a refusal of a send never recorded reads as the machine's last send: %v, %v", found, err) + } +} + +// **The machine's record and the send's are written together, and a send record that cannot be written +// leaves the machine's record standing and says so**: the digest and generation are the machine's, and a +// send that is away must not read as failed, nor the machine as behind. +func TestASendRecordThatCannotBeWrittenLeavesTheMachinesRecord(t *testing.T) { + inv, node := aNodeWithModules(t) + ctx := t.Context() + record, err := inv.NodeByName(ctx, node) + if err != nil { + t.Fatal(err) + } + // A NUL byte is text PostgreSQL refuses: the send's own row cannot be written. + broken := Send{Sequence: 13, Sender: "a test", Generation: 42, Digest: "d\x0013"} + err = inv.RecordSentWith(ctx, record.ID, "d13", nil, 0, 42, broken) + var unrecorded *SendNotRecordedError + if !errors.As(err, &unrecorded) { + t.Fatalf("a send record that could not be written read as %v", err) + } + if generation, err := inv.SentGeneration(ctx, record.ID); err != nil || generation != 42 { + t.Fatalf("the machine's generation was not written with its digest: %d (%v)", generation, err) + } + if sent, err := inv.Outstanding(ctx, node); err != nil || sent != "d13" { + t.Fatalf("the machine's digest was not written: %q (%v)", sent, err) + } +} + +// **An update that leaves an assignment as it was raises nothing** (the trigger's WHEN). +func TestAnUpdateThatChangesNoAssignmentRaisesNothing(t *testing.T) { + inv, node := aNodeWithModules(t, "web") + ctx := t.Context() + if _, err := inv.Assign(ctx, node, "web"); err != nil { + t.Fatal(err) + } + before := generationNow(t, inv) + if _, err := inv.store.Pool().Exec(ctx, `update assignment set module = module`); err != nil { + t.Fatal(err) + } + if after := generationNow(t, inv); after != before { + t.Errorf("an update that changed nothing moved the generation from %d to %d", before, after) + } +} + // **A machine that reads a generation is recorded from its reports**, and one rolled back says so no longer. func TestWhetherAMachineReadsAGenerationIsItsLatestWord(t *testing.T) { inv, node := aNodeWithModules(t)