jochen
d9289ef6d4
A build that reported applied was sent everywhere; one that then did nothing, served no tools or broke its machine's word reached every machine. Now the first machine is judged by the component's health (the core's definitions, as doctor probes H-*, or a module's own) three times over two minutes within ten; a failing gate puts the previous build back there once, marks the build, and says it as a condition and an event. Upgrades roll out by default; the bus is a planned step; a module deleted at its source is not built (the public-acme plan failure).
354 lines
12 KiB
Go
354 lines
12 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/nats-io/nats.go/jetstream"
|
|
|
|
"github.com/novox/mesh-controller/internal/broker"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/lease"
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
)
|
|
|
|
// Acting under the lease (novox/hq to-be 45 §6, ADR 0227 rule 1).
|
|
//
|
|
// **Only the instance holding the lease acts**: sends a declaration, writes a plan, a condition or a
|
|
// call. Every one of those passes theLease.epoch, which answers the epoch the act carries or why it may
|
|
// not happen. Three ways a process stands to the lease:
|
|
//
|
|
// - **The serving controller** takes it before it does anything else — before it asserts the bus's
|
|
// objects, which are the controller's to write — waiting while another holds it, and renews it.
|
|
// A renewal refused or failed is the lease lost: the gate closes at once and the process exits, so
|
|
// its service manager restarts it as a candidate (serve, in push.go).
|
|
// - **A command run at a shell** — `push` in the installer, the lab, a person repairing a mesh whose
|
|
// controller is down (issue 201) — acts **under the holder's epoch** when a controller holds the
|
|
// lease: it is the same mesh's word, composed and sent under the store's hold of each machine like
|
|
// the serving controller's, and the epoch it carries is read at the moment it acts, so a handover
|
|
// between makes it stale and refused like any other. **When nobody holds the lease, the command
|
|
// takes it** for as long as it runs and gives it back; a controller starting meanwhile waits for
|
|
// it, as it would for another controller.
|
|
// - **A process with no bus** — a test, a command that only reads — acts with no epoch and is
|
|
// refused nothing: there is nothing to order against, and nothing it does reaches a machine.
|
|
//
|
|
// **Unleased, said and temporary.** A serving controller whose bus refuses it the lease's key — the bus's
|
|
// user list is older than this build and does not grant the bucket yet — and that sees no other holder
|
|
// serves without one, as every controller did before the lease: declarations carry no epoch, which no
|
|
// node-engine refuses. Said once, kept as a condition (S12), and tried again every renewal interval; the
|
|
// first push that sends the bus its new user list grants it, and the next try takes it. Refusing to act
|
|
// instead would be a controller that can never send the user list that lets it act.
|
|
|
|
// actor is this process's standing to the lease.
|
|
type actor struct {
|
|
mu sync.Mutex
|
|
// held is the lease this process holds: the serving controller's, or a command's own.
|
|
held *lease.Lease
|
|
// unleased is why a serving controller acts without the lease; empty while it holds it or is not
|
|
// serving.
|
|
unleased string
|
|
// serving is a serving controller, which never borrows another's epoch.
|
|
serving bool
|
|
// kv is the lease bucket, for a command to read the holder's epoch from.
|
|
kv jetstream.KeyValue
|
|
close func()
|
|
// noBus is a process with no bus configured.
|
|
noBus bool
|
|
// reset is when the lease bucket was found raised again from nothing and its revisions moved past
|
|
// the highest epoch issued, and what was said of it; zero when it was not (S12).
|
|
reset time.Time
|
|
resetSaid string
|
|
}
|
|
|
|
// theLease is this process's standing to the lease.
|
|
var theLease = &actor{}
|
|
|
|
// instance names this process among controller instances: its machine, its process and when it
|
|
// started. The lease's holder and every call this process keeps carry it.
|
|
var instance = func() string {
|
|
host, _ := os.Hostname()
|
|
return fmt.Sprintf("controller@%s pid %d since %s", host, os.Getpid(), time.Now().UTC().Format(time.RFC3339))
|
|
}()
|
|
|
|
// epoch is the gate: the epoch an act carries — zero for none — or why it may not happen.
|
|
func (a *actor) epoch(ctx context.Context) (uint64, error) {
|
|
a.mu.Lock()
|
|
held, serving, unleased, noBus := a.held, a.serving, a.unleased, a.noBus
|
|
a.mu.Unlock()
|
|
switch {
|
|
case held != nil:
|
|
return held.Epoch()
|
|
case serving && unleased != "":
|
|
return 0, nil
|
|
case serving:
|
|
return 0, lease.ErrNotHeld
|
|
case noBus:
|
|
return 0, nil
|
|
}
|
|
return a.forACommand(ctx)
|
|
}
|
|
|
|
// forACommand is a command's epoch: the holder's, or a lease of its own when nobody holds one.
|
|
func (a *actor) forACommand(ctx context.Context) (uint64, error) {
|
|
a.mu.Lock()
|
|
defer a.mu.Unlock()
|
|
if a.held != nil {
|
|
return a.held.Epoch()
|
|
}
|
|
if a.kv == nil {
|
|
address, err := broker.BusAddress()
|
|
if err != nil {
|
|
// No bus: this process reaches no machine, and has nothing to order against.
|
|
a.noBus = true
|
|
return 0, nil
|
|
}
|
|
js, err := broker.Dial(address)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("the bus cannot be reached, so whether a controller holds the lease cannot be "+
|
|
"read and nothing is done: %w", err)
|
|
}
|
|
api, err := jetstream.New(js.Conn())
|
|
if err != nil {
|
|
js.Close()
|
|
return 0, err
|
|
}
|
|
reading, cancel := context.WithTimeout(ctx, 10*time.Second)
|
|
defer cancel()
|
|
if err := broker.EnsureLeaseBucket(reading, api); err != nil {
|
|
js.Close()
|
|
return 0, err
|
|
}
|
|
kv, err := api.KeyValue(reading, broker.LeaseBucket)
|
|
if err != nil {
|
|
js.Close()
|
|
return 0, err
|
|
}
|
|
a.kv, a.close = kv, js.Close
|
|
if _, found, err := lease.Current(reading, kv); err == nil && !found {
|
|
// Nobody: this command takes it for as long as it runs.
|
|
l, err := lease.Open(reading, api, broker.LeaseBucket, lease.Options{Holder: holderOf(instance),
|
|
Say: func(format string, args ...any) { fmt.Printf(format+"\n", args...) }})
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
epoch, err := l.TryTake(reading)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("no controller holds the lease and this command could not take it: %w", err)
|
|
}
|
|
keeping, stop := context.WithCancel(context.Background())
|
|
go l.Keep(keeping)
|
|
a.held = l
|
|
closeBus := a.close
|
|
a.close = func() {
|
|
stop()
|
|
l.Release(context.Background())
|
|
closeBus()
|
|
}
|
|
return epoch, nil
|
|
}
|
|
}
|
|
reading, cancel := context.WithTimeout(ctx, 10*time.Second)
|
|
defer cancel()
|
|
holder, found, err := lease.Current(reading, a.kv)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("who holds the controller lease cannot be read, so nothing is done: %w", err)
|
|
}
|
|
if !found {
|
|
return 0, errors.New("the controller that held the lease while this command ran let go of it; nothing " +
|
|
"more is done under an epoch nobody holds — run the command again")
|
|
}
|
|
return holder.Epoch, nil
|
|
}
|
|
|
|
// release gives back what this process holds, at its end.
|
|
func (a *actor) release() {
|
|
a.mu.Lock()
|
|
closing := a.close
|
|
a.close = nil
|
|
a.mu.Unlock()
|
|
if closing != nil {
|
|
closing()
|
|
}
|
|
}
|
|
|
|
// holderOf is this process as the lease's holder.
|
|
func holderOf(instance string) lease.Holder {
|
|
host, _ := os.Hostname()
|
|
return lease.Holder{Instance: instance, Host: host, Build: version}
|
|
}
|
|
|
|
// serveUnderTheLease takes the lease for the serving controller, waiting while another holds it, and
|
|
// keeps it until ctx ends. Lost is closed when it is lost; the caller exits on it.
|
|
func (a *actor) serveUnderTheLease(ctx context.Context, inv *inventory.Inventory, address string) (lost <-chan struct{}, err error) {
|
|
a.mu.Lock()
|
|
a.serving = true
|
|
a.mu.Unlock()
|
|
js, err := broker.Dial(address)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("the mesh is on the bus at %s and this control plane cannot reach it to take the "+
|
|
"lease: %w", broker.BareAddress(address), err)
|
|
}
|
|
api, err := jetstream.New(js.Conn())
|
|
if err != nil {
|
|
js.Close()
|
|
return nil, err
|
|
}
|
|
asserting, cancel := context.WithTimeout(ctx, 10*time.Second)
|
|
err = broker.EnsureLeaseBucket(asserting, api)
|
|
cancel()
|
|
if err != nil {
|
|
js.Close()
|
|
return nil, err
|
|
}
|
|
say := func(format string, args ...any) { fmt.Printf(format+"\n", args...) }
|
|
l, err := lease.Open(ctx, api, broker.LeaseBucket, lease.Options{Holder: holderOf(instance),
|
|
Floor: inv.HighestEpoch, Say: say, Health: controllerHealth, Moved: func(was, floor uint64) {
|
|
a.mu.Lock()
|
|
defer a.mu.Unlock()
|
|
a.reset = time.Now()
|
|
a.resetSaid = fmt.Sprintf("the lease bucket was at revision %d with epoch %d already issued: it was "+
|
|
"raised again from nothing (a bus whose data was replaced), and its revisions were moved past %d so "+
|
|
"no machine refuses the next epoch", was, floor, floor)
|
|
}})
|
|
if err != nil {
|
|
js.Close()
|
|
return nil, err
|
|
}
|
|
gone := make(chan struct{})
|
|
epoch, err := l.Take(ctx)
|
|
switch {
|
|
case ctx.Err() != nil:
|
|
js.Close()
|
|
return nil, ctx.Err()
|
|
case err != nil && !errors.Is(err, lease.ErrUnwritable):
|
|
// Whether another controller acts cannot be told: this one does not act, and exits to try again.
|
|
js.Close()
|
|
return nil, err
|
|
case err != nil:
|
|
// Nobody holds it and the bus will not let it be written: unleased, said, tried again (see above).
|
|
a.mu.Lock()
|
|
a.unleased = err.Error()
|
|
a.mu.Unlock()
|
|
say("this controller serves WITHOUT the lease: %v. Its declarations carry no epoch; it tries again "+
|
|
"every %s, and the first push that sends the bus its user list grants it", err, lease.RenewEvery)
|
|
go a.takeWhenGranted(ctx, l, inv, gone)
|
|
default:
|
|
a.took(ctx, l, inv, epoch, gone)
|
|
}
|
|
a.mu.Lock()
|
|
a.close = func() {
|
|
if held, err := l.Epoch(); err == nil {
|
|
ending, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
if err := inv.EndEpoch(ending, held, inventory.EpochReleased); err != nil {
|
|
say("how epoch %d ended could not be recorded: %v", held, err)
|
|
}
|
|
cancel()
|
|
}
|
|
l.Release(context.Background())
|
|
js.Close()
|
|
}
|
|
a.mu.Unlock()
|
|
return gone, nil
|
|
}
|
|
|
|
// took is the lease taken: recorded, earlier epochs nobody gave back ended as expired, kept.
|
|
func (a *actor) took(ctx context.Context, l *lease.Lease, inv *inventory.Inventory, epoch uint64, gone chan struct{}) {
|
|
a.mu.Lock()
|
|
a.held, a.unleased = l, ""
|
|
a.mu.Unlock()
|
|
h := holderOf(instance)
|
|
recording, cancel := context.WithTimeout(ctx, 10*time.Second)
|
|
expired, err := inv.TookEpoch(recording, inventory.Epoch{Epoch: epoch, Instance: h.Instance, Host: h.Host,
|
|
Build: h.Build, Taken: time.Now()})
|
|
cancel()
|
|
if err != nil {
|
|
fmt.Printf("epoch %d could not be recorded as taken, so a stale refusal from it will not name it: %v\n", epoch, err)
|
|
}
|
|
for _, e := range expired {
|
|
fmt.Printf("the controller of epoch %d (%s) stopped renewing the lease without giving it back: it is "+
|
|
"taken over at epoch %d\n", e.Epoch, e.Instance, epoch)
|
|
}
|
|
go l.Keep(ctx)
|
|
go func() {
|
|
<-l.Lost()
|
|
if ctx.Err() == nil {
|
|
why := l.LostWhy()
|
|
ending, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
_ = inv.EndEpoch(ending, epoch, inventory.EpochLost)
|
|
cancel()
|
|
fmt.Printf("the controller lease was lost (epoch %d): %v — this controller stops and exits, to "+
|
|
"be started again as a candidate\n", epoch, why)
|
|
}
|
|
close(gone)
|
|
}()
|
|
}
|
|
|
|
// takeWhenGranted tries the lease again every renewal interval while serving unleased, and stops this
|
|
// controller if another took it meanwhile: two serving at once is what the lease is for.
|
|
func (a *actor) takeWhenGranted(ctx context.Context, l *lease.Lease, inv *inventory.Inventory, gone chan struct{}) {
|
|
tick := time.NewTicker(lease.RenewEvery)
|
|
defer tick.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-tick.C:
|
|
}
|
|
epoch, err := l.TryTake(ctx)
|
|
if errors.Is(err, lease.ErrTaken) {
|
|
fmt.Printf("another controller took the lease while this one served without it: %v — this one "+
|
|
"stops and exits\n", err)
|
|
close(gone)
|
|
return
|
|
}
|
|
if err != nil {
|
|
// Still not written, or not readable this time: unleased, said by S12, tried again.
|
|
a.mu.Lock()
|
|
a.unleased = err.Error()
|
|
a.mu.Unlock()
|
|
continue
|
|
}
|
|
a.took(ctx, l, inv, epoch, gone)
|
|
return
|
|
}
|
|
}
|
|
|
|
// standing is what `status` and the self-check say of this process and the lease.
|
|
type standing struct {
|
|
Epoch uint64
|
|
Held bool
|
|
Renewed time.Time
|
|
Unleased string
|
|
// Reset is when the lease bucket was found raised again from nothing, and ResetSaid what of it.
|
|
Reset time.Time
|
|
ResetSaid string
|
|
}
|
|
|
|
func (a *actor) standing() standing {
|
|
a.mu.Lock()
|
|
held, unleased, reset, resetSaid := a.held, a.unleased, a.reset, a.resetSaid
|
|
a.mu.Unlock()
|
|
st := standing{Unleased: unleased, Reset: reset, ResetSaid: resetSaid}
|
|
if held == nil {
|
|
return st
|
|
}
|
|
epoch, err := held.Epoch()
|
|
st.Epoch, st.Held, st.Renewed = epoch, err == nil, held.Renewed()
|
|
return st
|
|
}
|
|
|
|
// The gates, given to what acts: a declaration's send (link) and a plan's write (the inventory).
|
|
func init() {
|
|
link.ActingGate = func(ctx context.Context) error {
|
|
_, err := theLease.epoch(ctx)
|
|
if err != nil {
|
|
return fmt.Errorf("this controller may not send: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
}
|