novox/mesh-controller
6
0
Fork 0
Code Issues Pull Requests 6 Releases Wiki Activity

Act under a lease, keep accounts by order, one writer at composition (hq to-be 45 Phase 2)

Browse Source 
Two controllers could both act (issue 204), a reconcile's report could
overtake the apply after it and the digest decided (issue 267), and a grant
could make a second writer of a machine's report.

- The lease (internal/lease, ADR 0229): mesh-controller_lease key `holder`,
  15 s age, renewed every 5 s by compare-and-set; the epoch is the revision
  it was taken at. The gate is the clock (stops 3 s before expiry); a refused
  renewal is a loss and the process exits; a holder that stops gives it back.
  serve takes it before asserting the bus. Epochs kept in the store
  (migration 0068 controller_epoch) as a floor: a bucket raised from nothing
  is compacted past it. Unleased (no epoch, S12 urgent) only when nobody
  holds it and the bus will not let it be written. A shell command acts
  under the holder's epoch, or its own lease when none.
- Declarations carry `epoch` inside the signed envelope, only to a machine
  whose latest account carried a report_sequence (mesh-host #35); would-send
  is composed with the epoch last sent. Allot and the send both pass the gate.
- Reports: contract in internal/link/order.go (epoch, sequence,
  report_sequence, older_than, refused_older). Accounts kept by epoch, then
  sequence, then report sequence; older refused, counted; unordered reports
  keep the digest rule. Plans by compare-and-set on a revision, with epoch.
  Conditions and calls carry the epoch and are not written off the lease.
- S12 and S13 (naming the writer by epoch) watched, D5 run; reset of the
  bucket said. Writers table compiled in and enforced in PermissionsFor; the
  controller no longer publishes mesh.control.>. A contract per consumed
  kind, and the empty-on-error lint over the repository.
- mesh-host pinned to its main with the epoch in the validator (D1 validates
  the envelope as sent).

Needs mesh-host's genesis lock with the lease grant (mesh-host PR) for
TestTheInstallersFirstUserListIsWhatTheControllerWouldCompose.
This commit is contained in:
jochen
2026-10-06 12:29:18 +02:00
parent 070ecafc07
commit 2eb9a22c24
59 changed files with 3975 additions and 158 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/mesh-controller/push.go
+94 -25
View File
@@ -63,7 +63,7 @@ func connectLink(ctx context.Context, inv *inventory.Inventory, enroller link.En
return link.ConnectNats(js, enroller, listener), nil
}
func serve(ctx context.Context) error {
func serve(ctx context.Context) (err error) {
// The one process whose log is read over time, so the one that says each change to a node's
// unmet seat dependencies once (novox/hq ADR 0207).
logUnheldChanges = true
@@ -104,6 +104,41 @@ func serve(ctx context.Context) error {
// being live is refused, because a mesh half on each is one where a declaration goes out on one
// and the report comes back on the other, and every component logs success while it happens.
// **The lease, before anything that acts** (novox/hq to-be 45 §6): asserting the bus's objects is the
// controller's to do, and so is everything after. A controller starting while another holds it waits
// here, said; one that loses it stops: every act's gate closes at once, and ctx ends so the process
// exits and is started again as a candidate.
busAddress, err := broker.BusAddress()
if err != nil {
return err
}
lost, err := theLease.serveUnderTheLease(ctx, inv, busAddress)
if err != nil {
return err
}
// Given back before the store closes, so the epoch is recorded as given back rather than found
// expired by the next holder.
defer theLease.release()
ctx, stopActing := context.WithCancel(ctx)
defer stopActing()
go func() {
select {
case <-lost:
stopActing()
case <-ctx.Done():
}
}()
defer func() {
select {
case <-lost:
if err == nil {
err = errors.New("the controller lease was lost; this controller stopped acting and exits, to " +
"be started again as a candidate")
}
default:
}
}()
work := link.Enrolment{Inventory: inv, Identity: ident, Broker: known,
OnNATS: true}
// `status` from a summary kept current here (novox/hq to-be 45 Phase 0): a machine saying
@@ -170,6 +205,9 @@ func serve(ctx context.Context) error {
givenEvents = bus
// Composed now and kept current, before the verb that answers from it is served.
go statusFrom.keep(ctx)
// Every call carries the lease's epoch, and its record is written only under the lease (novox/hq
// to-be 45 §6).
link.Calls.UnderLease(func() (uint64, error) { return theLease.epoch(ctx) })
// A call that outlasts its caller's patience is followed by `calls` (novox/hq issue 265).
link.Calls.Follow = catalogue.ControllerSeatName + ".calls"
// And every call is kept on the bus, so a restart of this process keeps what came of each
@@ -178,7 +216,7 @@ func serve(ctx context.Context) error {
said := log.New(os.Stdout, "", log.LstdFlags)
if keeper, err := link.CallsOnTheBus(ctx, bus.Conn); err != nil {
fmt.Printf("calls are kept in memory only, and lost when this controller stops: %v\n", err)
} else if err := link.Calls.Durably(ctx, keeper, controllerProcess(), said); err != nil {
} else if err := link.Calls.Durably(ctx, keeper, instance, said); err != nil {
fmt.Printf("calls are kept on the bus from now on; the ones kept before could not be read: %v\n", err)
}
// What is wrong, kept and said (novox/hq to-be 45 §2): the condition store, the watchdogs of the
@@ -257,7 +295,12 @@ func declare(ctx context.Context, args []string) error {
// is still what the machine was last told, and status must not read it as current for the one
// the mesh would compose. Which builds it carried is recorded as not known (novox/hq issue 259):
// the mesh did not compose it, so a push that does not name this machine treats it as held.
if _, err := recordSent(ctx, inv, node, raw, nil); err != nil {
// The epoch it carried, if a person wrote one in, is what the machine heard.
var carried struct {
Epoch uint64 `json:"epoch"`
}
_ = json.Unmarshal(raw, &carried)
if _, err := recordSent(ctx, inv, node, raw, nil, carried.Epoch); err != nil {
return err
}
fmt.Printf("sent %s a signed declaration (%d bytes)\n", node, len(raw))
@@ -586,7 +629,7 @@ type readyNode struct {
//
// The all-or-nothing rule is kept where it means something — sendTo, which rotates a credential
// across two machines that must agree — and dropped here, where it never did.
func composeEach(names []string, allot func(node string) (int64, error),
func composeEach(names []string, allot func(node string) (order, error),
compose func(node string) (sendable, error)) ([]readyNode, []string) {
var sending []readyNode
@@ -600,7 +643,7 @@ func composeEach(names []string, allot func(node string) (int64, error),
// took the older content as the newer word: on 2026-10-02 a runtime assigned and applied on
// two machines was undone two seconds later by exactly that. Taken here, before the first
// read, what was composed earlier is numbered lower whatever order the sends happen in.
seq, err := allot(name)
numbered, err := allot(name)
if err != nil {
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
continue
@@ -610,7 +653,7 @@ func composeEach(names []string, allot func(node string) (int64, error),
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
continue
}
declared.Sequence = seq
declared.Sequence, declared.Epoch = numbered.sequence, numbered.epoch
if len(declared.Resources) == 0 {
// Sent, not skipped (novox/hq issue 127). A node whose declaration composes to
// nothing may have HELD something before — the broker opening a placement gave it,
@@ -794,7 +837,7 @@ func (b overTheBus) declare(ctx context.Context, s readyNode, body []byte) (stri
}
// After it is away, not before. A digest recorded for something that failed to send would make
// the machine look current for a declaration it never received.
digest, err := recordSent(ctx, b.open.inventory, s.node, body, s.declared.Builds)
digest, err := recordSent(ctx, b.open.inventory, s.node, body, s.declared.Builds, s.declared.Epoch)
if err != nil {
return "", err
}
@@ -947,7 +990,7 @@ func sendToEach(ctx context.Context, open *stores, names []string) ([]string, er
var refusals []string
for _, name := range names {
// Numbered before composing, for the reason composeEach gives (novox/hq issue 204).
seq, err := allot(ctx, inv, name)
numbered, err := allot(ctx, inv, name)
if err != nil {
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
continue
@@ -963,7 +1006,7 @@ func sendToEach(ctx context.Context, open *stores, names []string) ([]string, er
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
continue
}
declared.Sequence = seq
declared.Sequence, declared.Epoch = numbered.sequence, numbered.epoch
reportLeftOut(name, declared)
sending = append(sending, readyNode{name, declared})
}
@@ -1120,6 +1163,11 @@ func wouldSendFrom(ctx context.Context, open *stores,
if declared.Sequence, err = open.inventory.Sequence(ctx, n.ID); err != nil {
return nil, err
}
// And the epoch it was last sent under, for the same reason: a new holder of the lease is not a
// change of the machine (novox/hq to-be 45 §6).
if declared.Epoch, err = open.inventory.SentEpoch(ctx, n.ID); err != nil {
return nil, err
}
body, err := declared.Body()
if err != nil {
return nil, err
@@ -1245,17 +1293,46 @@ func seatHolders(ctx context.Context, inv *inventory.Inventory) (map[string]brok
// number gives one send the next sequence for its node (novox/hq 04-ISSUES/107).
// allotting is allot over one inventory, in the shape composeEach takes.
func allotting(ctx context.Context, inv *inventory.Inventory) func(node string) (int64, error) {
return func(node string) (int64, error) { return allot(ctx, inv, node) }
func allotting(ctx context.Context, inv *inventory.Inventory) func(node string) (order, error) {
return func(node string) (order, error) { return allot(ctx, inv, node) }
}
// allot takes the next sequence for a machine — the number its next declaration carries.
func allot(ctx context.Context, inv *inventory.Inventory, node string) (int64, error) {
// epochForActs is the lease's gate as a composition asks it; a variable so a test can act under an epoch
// without a bus.
var epochForActs = func(ctx context.Context) (uint64, error) { return theLease.epoch(ctx) }
// order is what a declaration carries of its writer's order (link/order.go): its sequence, and the
// epoch of the lease it is composed under — zero for a machine that has not said it reads one.
type order struct {
sequence int64
epoch uint64
}
// allot takes the next sequence for a machine — the number its next declaration carries — under the
// lease: a process that may not act takes none, and composes nothing (novox/hq to-be 45 §6).
func allot(ctx context.Context, inv *inventory.Inventory, node string) (order, error) {
epoch, err := epochForActs(ctx)
if err != nil {
return order{}, fmt.Errorf("nothing was composed for %s: %w", node, err)
}
record, err := inv.NodeByName(ctx, node)
if err != nil {
return 0, err
return order{}, err
}
return inv.NextSequence(ctx, record.ID)
if epoch > 0 {
reads, err := inv.ReadsEpoch(ctx, record.ID)
if err != nil {
return order{}, err
}
if !reads {
epoch = 0
}
}
seq, err := inv.NextSequence(ctx, record.ID)
if err != nil {
return order{}, err
}
return order{sequence: seq, epoch: epoch}, nil
}
// recordSent writes down what a machine was just sent, and returns the digest.
@@ -1270,7 +1347,7 @@ func allot(ctx context.Context, inv *inventory.Inventory, node string) (int64, e
// And the build of each module it carried (novox/hq issue 259, ADR 0221), nil when that is not known:
// what tells a machine held back by a policy or a plan from one a push left behind.
func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body []byte,
builds map[string]string) (string, error) {
builds map[string]string, epoch uint64) (string, error) {
kept, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
defer cancel()
record, err := inv.NodeByName(kept, node)
@@ -1278,7 +1355,7 @@ func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body
return "", err
}
digest := digestOf(body)
if err := inv.RecordSent(kept, record.ID, digest, builds); err != nil {
if err := inv.RecordSentUnder(kept, record.ID, digest, builds, epoch); err != nil {
return "", err
}
return digest, nil
@@ -1304,11 +1381,3 @@ func reportUnheldPushed(w io.Writer, named bool, asked []string, unheld map[stri
fmt.Fprintf(w, "%s: %d unmet seat dependenc(ies) — see `status`\n", node, len(lines))
}
}
// controllerProcess names this serving process among controllers: the machine, the process and when
// it started — what a call kept on the bus carries, so the next controller can tell a call this one
// left running from one it is running itself (novox/hq to-be 45 §6).
func controllerProcess() string {
host, _ := os.Hostname()
return fmt.Sprintf("controller@%s pid %d since %s", host, os.Getpid(), time.Now().UTC().Format(time.RFC3339))
}