mesh-controller/cmd/mesh-controller/acting.go

package main

import (
	"context"
	"errors"
	"fmt"
	"os"
	"sync"
	"time"

	"github.com/nats-io/nats.go/jetstream"

	"github.com/novox/mesh-controller/internal/broker"
	"github.com/novox/mesh-controller/internal/inventory"
	"github.com/novox/mesh-controller/internal/lease"
	"github.com/novox/mesh-controller/internal/link"
)

// Acting under the lease (novox/hq to-be 45 §6, ADR 0227 rule 1).
//
// **Only the instance holding the lease acts**: sends a declaration, writes a plan, a condition or a
// call. Every one of those passes theLease.epoch, which answers the epoch the act carries or why it may
// not happen. Three ways a process stands to the lease:
//
//   - **The serving controller** takes it before it does anything else — before it asserts the bus's
//     objects, which are the controller's to write — waiting while another holds it, and renews it.
//     A renewal refused or failed is the lease lost: the gate closes at once and the process exits, so
//     its service manager restarts it as a candidate (serve, in push.go).
//   - **A command run at a shell** — `push` in the installer, the lab, a person repairing a mesh whose
//     controller is down (issue 201) — acts **under the holder's epoch** when a controller holds the
//     lease: it is the same mesh's word, composed and sent under the store's hold of each machine like
//     the serving controller's, and the epoch it carries is read at the moment it acts, so a handover
//     between makes it stale and refused like any other. **When nobody holds the lease, the command
//     takes it** for as long as it runs and gives it back; a controller starting meanwhile waits for
//     it, as it would for another controller.
//   - **A process with no bus** — a test, a command that only reads — acts with no epoch and is
//     refused nothing: there is nothing to order against, and nothing it does reaches a machine.
//
// **Unleased, said and temporary.** A serving controller whose bus refuses it the lease's key — the bus's
// user list is older than this build and does not grant the bucket yet — and that sees no other holder
// serves without one, as every controller did before the lease: declarations carry no epoch, which no
// node-engine refuses. Said once, kept as a condition (S12), and tried again every renewal interval; the
// first push that sends the bus its new user list grants it, and the next try takes it. Refusing to act
// instead would be a controller that can never send the user list that lets it act.

// actor is this process's standing to the lease.
type actor struct {
	mu sync.Mutex
	// held is the lease this process holds: the serving controller's, or a command's own.
	held *lease.Lease
	// unleased is why a serving controller acts without the lease; empty while it holds it or is not
	// serving.
	unleased string
	// serving is a serving controller, which never borrows another's epoch.
	serving bool
	// kv is the lease bucket, for a command to read the holder's epoch from.
	kv    jetstream.KeyValue
	close func()
	// noBus is a process with no bus configured.
	noBus bool
	// reset is when the lease bucket was found raised again from nothing and its revisions moved past
	// the highest epoch issued, and what was said of it; zero when it was not (S12).
	reset     time.Time
	resetSaid string
}

// theLease is this process's standing to the lease.
var theLease = &actor{}

// instance names this process among controller instances: its machine, its process and when it
// started. The lease's holder and every call this process keeps carry it.
var instance = func() string {
	host, _ := os.Hostname()
	return fmt.Sprintf("controller@%s pid %d since %s", host, os.Getpid(), time.Now().UTC().Format(time.RFC3339))
}()

// epoch is the gate: the epoch an act carries — zero for none — or why it may not happen.
func (a *actor) epoch(ctx context.Context) (uint64, error) {
	a.mu.Lock()
	held, serving, unleased, noBus := a.held, a.serving, a.unleased, a.noBus
	a.mu.Unlock()
	switch {
	case held != nil:
		return held.Epoch()
	case serving && unleased != "":
		return 0, nil
	case serving:
		return 0, lease.ErrNotHeld
	case noBus:
		return 0, nil
	}
	return a.forACommand(ctx)
}

// forACommand is a command's epoch: the holder's, or a lease of its own when nobody holds one.
func (a *actor) forACommand(ctx context.Context) (uint64, error) {
	a.mu.Lock()
	defer a.mu.Unlock()
	if a.held != nil {
		return a.held.Epoch()
	}
	if a.kv == nil {
		address, err := broker.BusAddress()
		if err != nil {
			// No bus: this process reaches no machine, and has nothing to order against.
			a.noBus = true
			return 0, nil
		}
		js, err := broker.Dial(address)
		if err != nil {
			return 0, fmt.Errorf("the bus cannot be reached, so whether a controller holds the lease cannot be "+
				"read and nothing is done: %w", err)
		}
		api, err := jetstream.New(js.Conn())
		if err != nil {
			js.Close()
			return 0, err
		}
		reading, cancel := context.WithTimeout(ctx, 10*time.Second)
		defer cancel()
		if err := broker.EnsureLeaseBucket(reading, api); err != nil {
			js.Close()
			return 0, err
		}
		kv, err := api.KeyValue(reading, broker.LeaseBucket)
		if err != nil {
			js.Close()
			return 0, err
		}
		a.kv, a.close = kv, js.Close
		if _, found, err := lease.Current(reading, kv); err == nil && !found {
			// Nobody: this command takes it for as long as it runs.
			l, err := lease.Open(reading, api, broker.LeaseBucket, lease.Options{Holder: holderOf(instance),
				Say: func(format string, args ...any) { fmt.Printf(format+"\n", args...) }})
			if err != nil {
				return 0, err
			}
			epoch, err := l.TryTake(reading)
			if err != nil {
				return 0, fmt.Errorf("no controller holds the lease and this command could not take it: %w", err)
			}
			keeping, stop := context.WithCancel(context.Background())
			go l.Keep(keeping)
			a.held = l
			closeBus := a.close
			a.close = func() {
				stop()
				l.Release(context.Background())
				closeBus()
			}
			return epoch, nil
		}
	}
	reading, cancel := context.WithTimeout(ctx, 10*time.Second)
	defer cancel()
	holder, found, err := lease.Current(reading, a.kv)
	if err != nil {
		return 0, fmt.Errorf("who holds the controller lease cannot be read, so nothing is done: %w", err)
	}
	if !found {
		return 0, errors.New("the controller that held the lease while this command ran let go of it; nothing " +
			"more is done under an epoch nobody holds — run the command again")
	}
	return holder.Epoch, nil
}

// release gives back what this process holds, at its end.
func (a *actor) release() {
	a.mu.Lock()
	closing := a.close
	a.close = nil
	a.mu.Unlock()
	if closing != nil {
		closing()
	}
}

// holderOf is this process as the lease's holder.
func holderOf(instance string) lease.Holder {
	host, _ := os.Hostname()
	return lease.Holder{Instance: instance, Host: host, Build: version}
}

// serveUnderTheLease takes the lease for the serving controller, waiting while another holds it, and
// keeps it until ctx ends. Lost is closed when it is lost; the caller exits on it.
func (a *actor) serveUnderTheLease(ctx context.Context, inv *inventory.Inventory, address string) (lost <-chan struct{}, err error) {
	a.mu.Lock()
	a.serving = true
	a.mu.Unlock()
	js, err := broker.Dial(address)
	if err != nil {
		return nil, fmt.Errorf("the mesh is on the bus at %s and this control plane cannot reach it to take the "+
			"lease: %w", broker.BareAddress(address), err)
	}
	api, err := jetstream.New(js.Conn())
	if err != nil {
		js.Close()
		return nil, err
	}
	asserting, cancel := context.WithTimeout(ctx, 10*time.Second)
	err = broker.EnsureLeaseBucket(asserting, api)
	cancel()
	if err != nil {
		js.Close()
		return nil, err
	}
	say := func(format string, args ...any) { fmt.Printf(format+"\n", args...) }
	l, err := lease.Open(ctx, api, broker.LeaseBucket, lease.Options{Holder: holderOf(instance),
		Floor: inv.HighestEpoch, Say: say, Moved: func(was, floor uint64) {
			a.mu.Lock()
			defer a.mu.Unlock()
			a.reset = time.Now()
			a.resetSaid = fmt.Sprintf("the lease bucket was at revision %d with epoch %d already issued: it was "+
				"raised again from nothing (a bus whose data was replaced), and its revisions were moved past %d so "+
				"no machine refuses the next epoch", was, floor, floor)
		}})
	if err != nil {
		js.Close()
		return nil, err
	}
	gone := make(chan struct{})
	epoch, err := l.Take(ctx)
	switch {
	case ctx.Err() != nil:
		js.Close()
		return nil, ctx.Err()
	case err != nil && !errors.Is(err, lease.ErrUnwritable):
		// Whether another controller acts cannot be told: this one does not act, and exits to try again.
		js.Close()
		return nil, err
	case err != nil:
		// Nobody holds it and the bus will not let it be written: unleased, said, tried again (see above).
		a.mu.Lock()
		a.unleased = err.Error()
		a.mu.Unlock()
		say("this controller serves WITHOUT the lease: %v. Its declarations carry no epoch; it tries again "+
			"every %s, and the first push that sends the bus its user list grants it", err, lease.RenewEvery)
		go a.takeWhenGranted(ctx, l, inv, gone)
	default:
		a.took(ctx, l, inv, epoch, gone)
	}
	a.mu.Lock()
	a.close = func() {
		if held, err := l.Epoch(); err == nil {
			ending, cancel := context.WithTimeout(context.Background(), 5*time.Second)
			if err := inv.EndEpoch(ending, held, inventory.EpochReleased); err != nil {
				say("how epoch %d ended could not be recorded: %v", held, err)
			}
			cancel()
		}
		l.Release(context.Background())
		js.Close()
	}
	a.mu.Unlock()
	return gone, nil
}

// took is the lease taken: recorded, earlier epochs nobody gave back ended as expired, kept.
func (a *actor) took(ctx context.Context, l *lease.Lease, inv *inventory.Inventory, epoch uint64, gone chan struct{}) {
	a.mu.Lock()
	a.held, a.unleased = l, ""
	a.mu.Unlock()
	h := holderOf(instance)
	recording, cancel := context.WithTimeout(ctx, 10*time.Second)
	expired, err := inv.TookEpoch(recording, inventory.Epoch{Epoch: epoch, Instance: h.Instance, Host: h.Host,
		Build: h.Build, Taken: time.Now()})
	cancel()
	if err != nil {
		fmt.Printf("epoch %d could not be recorded as taken, so a stale refusal from it will not name it: %v\n", epoch, err)
	}
	for _, e := range expired {
		fmt.Printf("the controller of epoch %d (%s) stopped renewing the lease without giving it back: it is "+
			"taken over at epoch %d\n", e.Epoch, e.Instance, epoch)
	}
	go l.Keep(ctx)
	go func() {
		<-l.Lost()
		if ctx.Err() == nil {
			why := l.LostWhy()
			ending, cancel := context.WithTimeout(context.Background(), 5*time.Second)
			_ = inv.EndEpoch(ending, epoch, inventory.EpochLost)
			cancel()
			fmt.Printf("the controller lease was lost (epoch %d): %v — this controller stops and exits, to "+
				"be started again as a candidate\n", epoch, why)
		}
		close(gone)
	}()
}

// takeWhenGranted tries the lease again every renewal interval while serving unleased, and stops this
// controller if another took it meanwhile: two serving at once is what the lease is for.
func (a *actor) takeWhenGranted(ctx context.Context, l *lease.Lease, inv *inventory.Inventory, gone chan struct{}) {
	tick := time.NewTicker(lease.RenewEvery)
	defer tick.Stop()
	for {
		select {
		case <-ctx.Done():
			return
		case <-tick.C:
		}
		epoch, err := l.TryTake(ctx)
		if errors.Is(err, lease.ErrTaken) {
			fmt.Printf("another controller took the lease while this one served without it: %v — this one "+
				"stops and exits\n", err)
			close(gone)
			return
		}
		if err != nil {
			// Still not written, or not readable this time: unleased, said by S12, tried again.
			a.mu.Lock()
			a.unleased = err.Error()
			a.mu.Unlock()
			continue
		}
		a.took(ctx, l, inv, epoch, gone)
		return
	}
}

// standing is what `status` and the self-check say of this process and the lease.
type standing struct {
	Epoch    uint64
	Held     bool
	Renewed  time.Time
	Unleased string
	// Reset is when the lease bucket was found raised again from nothing, and ResetSaid what of it.
	Reset     time.Time
	ResetSaid string
}

func (a *actor) standing() standing {
	a.mu.Lock()
	held, unleased, reset, resetSaid := a.held, a.unleased, a.reset, a.resetSaid
	a.mu.Unlock()
	st := standing{Unleased: unleased, Reset: reset, ResetSaid: resetSaid}
	if held == nil {
		return st
	}
	epoch, err := held.Epoch()
	st.Epoch, st.Held, st.Renewed = epoch, err == nil, held.Renewed()
	return st
}

// The gates, given to what acts: a declaration's send (link) and a plan's write (the inventory).
func init() {
	link.ActingGate = func(ctx context.Context) error {
		_, err := theLease.epoch(ctx)
		if err != nil {
			return fmt.Errorf("this controller may not send: %w", err)
		}
		return nil
	}
}