package main import ( "context" "errors" "fmt" "os" "sync" "time" "github.com/nats-io/nats.go/jetstream" "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/lease" "github.com/novox/mesh-controller/internal/link" ) // Acting under the lease (novox/hq to-be 45 §6, ADR 0227 rule 1). // // **Only the instance holding the lease acts**: sends a declaration, writes a plan, a condition or a // call. Every one of those passes theLease.epoch, which answers the epoch the act carries or why it may // not happen. Three ways a process stands to the lease: // // - **The serving controller** takes it before it does anything else — before it asserts the bus's // objects, which are the controller's to write — waiting while another holds it, and renews it. // A renewal refused or failed is the lease lost: the gate closes at once and the process exits, so // its service manager restarts it as a candidate (serve, in push.go). // - **A command run at a shell** — `push` in the installer, the lab, a person repairing a mesh whose // controller is down (issue 201) — acts **under the holder's epoch** when a controller holds the // lease: it is the same mesh's word, composed and sent under the store's hold of each machine like // the serving controller's, and the epoch it carries is read at the moment it acts, so a handover // between makes it stale and refused like any other. **When nobody holds the lease, the command // takes it** for as long as it runs and gives it back; a controller starting meanwhile waits for // it, as it would for another controller. // - **A process with no bus** — a test, a command that only reads — acts with no epoch and is // refused nothing: there is nothing to order against, and nothing it does reaches a machine. // // **Unleased, said and temporary.** A serving controller whose bus refuses it the lease's key — the bus's // user list is older than this build and does not grant the bucket yet — and that sees no other holder // serves without one, as every controller did before the lease: declarations carry no epoch, which no // node-engine refuses. Said once, kept as a condition (S12), and tried again every renewal interval; the // first push that sends the bus its new user list grants it, and the next try takes it. Refusing to act // instead would be a controller that can never send the user list that lets it act. // actor is this process's standing to the lease. type actor struct { mu sync.Mutex // held is the lease this process holds: the serving controller's, or a command's own. held *lease.Lease // unleased is why a serving controller acts without the lease; empty while it holds it or is not // serving. unleased string // serving is a serving controller, which never borrows another's epoch. serving bool // kv is the lease bucket, for a command to read the holder's epoch from. kv jetstream.KeyValue close func() // noBus is a process with no bus configured. noBus bool // reset is when the lease bucket was found raised again from nothing and its revisions moved past // the highest epoch issued, and what was said of it; zero when it was not (S12). reset time.Time resetSaid string } // theLease is this process's standing to the lease. var theLease = &actor{} // instance names this process among controller instances: its machine, its process and when it // started. The lease's holder and every call this process keeps carry it. var instance = func() string { host, _ := os.Hostname() return fmt.Sprintf("controller@%s pid %d since %s", host, os.Getpid(), time.Now().UTC().Format(time.RFC3339)) }() // epoch is the gate: the epoch an act carries — zero for none — or why it may not happen. func (a *actor) epoch(ctx context.Context) (uint64, error) { a.mu.Lock() held, serving, unleased, noBus := a.held, a.serving, a.unleased, a.noBus a.mu.Unlock() switch { case held != nil: return held.Epoch() case serving && unleased != "": return 0, nil case serving: return 0, lease.ErrNotHeld case noBus: return 0, nil } return a.forACommand(ctx) } // forACommand is a command's epoch: the holder's, or a lease of its own when nobody holds one. func (a *actor) forACommand(ctx context.Context) (uint64, error) { a.mu.Lock() defer a.mu.Unlock() if a.held != nil { return a.held.Epoch() } if a.kv == nil { address, err := broker.BusAddress() if err != nil { // No bus: this process reaches no machine, and has nothing to order against. a.noBus = true return 0, nil } js, err := broker.Dial(address) if err != nil { return 0, fmt.Errorf("the bus cannot be reached, so whether a controller holds the lease cannot be "+ "read and nothing is done: %w", err) } api, err := jetstream.New(js.Conn()) if err != nil { js.Close() return 0, err } reading, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() if err := broker.EnsureLeaseBucket(reading, api); err != nil { js.Close() return 0, err } kv, err := api.KeyValue(reading, broker.LeaseBucket) if err != nil { js.Close() return 0, err } a.kv, a.close = kv, js.Close if _, found, err := lease.Current(reading, kv); err == nil && !found { // Nobody: this command takes it for as long as it runs. l, err := lease.Open(reading, api, broker.LeaseBucket, lease.Options{Holder: holderOf(instance), Say: func(format string, args ...any) { fmt.Printf(format+"\n", args...) }}) if err != nil { return 0, err } epoch, err := l.TryTake(reading) if err != nil { return 0, fmt.Errorf("no controller holds the lease and this command could not take it: %w", err) } keeping, stop := context.WithCancel(context.Background()) go l.Keep(keeping) a.held = l closeBus := a.close a.close = func() { stop() l.Release(context.Background()) closeBus() } return epoch, nil } } reading, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() holder, found, err := lease.Current(reading, a.kv) if err != nil { return 0, fmt.Errorf("who holds the controller lease cannot be read, so nothing is done: %w", err) } if !found { return 0, errors.New("the controller that held the lease while this command ran let go of it; nothing " + "more is done under an epoch nobody holds — run the command again") } return holder.Epoch, nil } // release gives back what this process holds, at its end. func (a *actor) release() { a.mu.Lock() closing := a.close a.close = nil a.mu.Unlock() if closing != nil { closing() } } // holderOf is this process as the lease's holder. func holderOf(instance string) lease.Holder { host, _ := os.Hostname() return lease.Holder{Instance: instance, Host: host, Build: version} } // serveUnderTheLease takes the lease for the serving controller, waiting while another holds it, and // keeps it until ctx ends. Lost is closed when it is lost; the caller exits on it. func (a *actor) serveUnderTheLease(ctx context.Context, inv *inventory.Inventory, address string) (lost <-chan struct{}, err error) { a.mu.Lock() a.serving = true a.mu.Unlock() js, err := broker.Dial(address) if err != nil { return nil, fmt.Errorf("the mesh is on the bus at %s and this control plane cannot reach it to take the "+ "lease: %w", broker.BareAddress(address), err) } api, err := jetstream.New(js.Conn()) if err != nil { js.Close() return nil, err } asserting, cancel := context.WithTimeout(ctx, 10*time.Second) err = broker.EnsureLeaseBucket(asserting, api) cancel() if err != nil { js.Close() return nil, err } say := func(format string, args ...any) { fmt.Printf(format+"\n", args...) } l, err := lease.Open(ctx, api, broker.LeaseBucket, lease.Options{Holder: holderOf(instance), Floor: inv.HighestEpoch, Say: say, Health: controllerHealth, Moved: func(was, floor uint64) { a.mu.Lock() defer a.mu.Unlock() a.reset = time.Now() a.resetSaid = fmt.Sprintf("the lease bucket was at revision %d with epoch %d already issued: it was "+ "raised again from nothing (a bus whose data was replaced), and its revisions were moved past %d so "+ "no machine refuses the next epoch", was, floor, floor) }}) if err != nil { js.Close() return nil, err } gone := make(chan struct{}) epoch, err := l.Take(ctx) switch { case ctx.Err() != nil: js.Close() return nil, ctx.Err() case err != nil && !errors.Is(err, lease.ErrUnwritable): // Whether another controller acts cannot be told: this one does not act, and exits to try again. js.Close() return nil, err case err != nil: // Nobody holds it and the bus will not let it be written: unleased, said, tried again (see above). a.mu.Lock() a.unleased = err.Error() a.mu.Unlock() say("this controller serves WITHOUT the lease: %v. Its declarations carry no epoch; it tries again "+ "every %s, and the first push that sends the bus its user list grants it", err, lease.RenewEvery) go a.takeWhenGranted(ctx, l, inv, gone) default: a.took(ctx, l, inv, epoch, gone) } a.mu.Lock() a.close = func() { if held, err := l.Epoch(); err == nil { ending, cancel := context.WithTimeout(context.Background(), 5*time.Second) if err := inv.EndEpoch(ending, held, inventory.EpochReleased); err != nil { say("how epoch %d ended could not be recorded: %v", held, err) } cancel() } l.Release(context.Background()) js.Close() } a.mu.Unlock() return gone, nil } // took is the lease taken: recorded, earlier epochs nobody gave back ended as expired, kept. func (a *actor) took(ctx context.Context, l *lease.Lease, inv *inventory.Inventory, epoch uint64, gone chan struct{}) { a.mu.Lock() a.held, a.unleased = l, "" a.mu.Unlock() h := holderOf(instance) recording, cancel := context.WithTimeout(ctx, 10*time.Second) expired, err := inv.TookEpoch(recording, inventory.Epoch{Epoch: epoch, Instance: h.Instance, Host: h.Host, Build: h.Build, Taken: time.Now()}) cancel() if err != nil { fmt.Printf("epoch %d could not be recorded as taken, so a stale refusal from it will not name it: %v\n", epoch, err) } for _, e := range expired { fmt.Printf("the controller of epoch %d (%s) stopped renewing the lease without giving it back: it is "+ "taken over at epoch %d\n", e.Epoch, e.Instance, epoch) } go l.Keep(ctx) go func() { <-l.Lost() if ctx.Err() == nil { why := l.LostWhy() ending, cancel := context.WithTimeout(context.Background(), 5*time.Second) _ = inv.EndEpoch(ending, epoch, inventory.EpochLost) cancel() fmt.Printf("the controller lease was lost (epoch %d): %v — this controller stops and exits, to "+ "be started again as a candidate\n", epoch, why) } close(gone) }() } // takeWhenGranted tries the lease again every renewal interval while serving unleased, and stops this // controller if another took it meanwhile: two serving at once is what the lease is for. func (a *actor) takeWhenGranted(ctx context.Context, l *lease.Lease, inv *inventory.Inventory, gone chan struct{}) { tick := time.NewTicker(lease.RenewEvery) defer tick.Stop() for { select { case <-ctx.Done(): return case <-tick.C: } epoch, err := l.TryTake(ctx) if errors.Is(err, lease.ErrTaken) { fmt.Printf("another controller took the lease while this one served without it: %v — this one "+ "stops and exits\n", err) close(gone) return } if err != nil { // Still not written, or not readable this time: unleased, said by S12, tried again. a.mu.Lock() a.unleased = err.Error() a.mu.Unlock() continue } a.took(ctx, l, inv, epoch, gone) return } } // standing is what `status` and the self-check say of this process and the lease. type standing struct { Epoch uint64 Held bool Renewed time.Time Unleased string // Reset is when the lease bucket was found raised again from nothing, and ResetSaid what of it. Reset time.Time ResetSaid string } func (a *actor) standing() standing { a.mu.Lock() held, unleased, reset, resetSaid := a.held, a.unleased, a.reset, a.resetSaid a.mu.Unlock() st := standing{Unleased: unleased, Reset: reset, ResetSaid: resetSaid} if held == nil { return st } epoch, err := held.Epoch() st.Epoch, st.Held, st.Renewed = epoch, err == nil, held.Renewed() return st } // The gates, given to what acts: a declaration's send (link) and a plan's write (the inventory). func init() { link.ActingGate = func(ctx context.Context) error { _, err := theLease.epoch(ctx) if err != nil { return fmt.Errorf("this controller may not send: %w", err) } return nil } }