120 lines
3.6 KiB
Go
120 lines
3.6 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/lease"
|
|
)
|
|
|
|
// The controller's side of its own rollback witness (novox/hq to-be 45 §8, ADR 0236; the contract is
|
|
// internal/lease/witness.go): what the serving controller says of itself in every write of the lease's
|
|
// key, for the node-engine on the control node to judge a new controller build by.
|
|
|
|
// processStarted is when this process started.
|
|
var processStarted = time.Now().UTC()
|
|
|
|
// readiness remembers when this process first became ready.
|
|
var readiness struct {
|
|
sync.Mutex
|
|
at time.Time
|
|
}
|
|
|
|
// controllerHealth is the controller's health definition (to-be 45 §8) as this process meets it now:
|
|
// it holds the lease — it is asked only while writing the key — its self-check has finished a run, and
|
|
// in that run `status` answered in full within ten seconds (D9). Anything short of that is said, never
|
|
// read as ready.
|
|
func controllerHealth() *lease.Health {
|
|
h := &lease.Health{Started: processStarted}
|
|
d := doctorFrom
|
|
if d == nil {
|
|
h.Why = "the self-check is not running in this process yet"
|
|
return h
|
|
}
|
|
run := d.lastRun()
|
|
if run == nil {
|
|
h.Why = "the self-check has not finished its first run"
|
|
return h
|
|
}
|
|
h.DoctorRan = run.At
|
|
ran := false
|
|
for _, p := range run.Probes {
|
|
if p.ID != "D9" {
|
|
continue
|
|
}
|
|
ran = true
|
|
if p.Verdict != verdictPass {
|
|
h.Why = "in the last self-check, status did not answer in full within ten seconds (D9 " + p.Verdict + ")"
|
|
return h
|
|
}
|
|
}
|
|
if !ran {
|
|
h.Why = "the last self-check did not judge status (D9)"
|
|
return h
|
|
}
|
|
readiness.Lock()
|
|
if readiness.at.IsZero() {
|
|
readiness.at = time.Now().UTC()
|
|
}
|
|
h.Ready, h.ReadyAt = true, readiness.at
|
|
readiness.Unlock()
|
|
return h
|
|
}
|
|
|
|
// witnessed is every rollback the machines' witnesses say they made and still stand by, as their last
|
|
// report carried it (Report.RolledBack). Kept in the serving controller's memory only: a witness says it
|
|
// on every report until it is resolved, so a controller started afresh hears it again with the next
|
|
// report — the host's reconcile is minutes apart, and a restored controller hears the report that
|
|
// follows its own restoring.
|
|
var witnessed = &rollbacksHeard{byNode: map[string]heardRollbacks{}}
|
|
|
|
type rollbacksHeard struct {
|
|
mu sync.Mutex
|
|
byNode map[string]heardRollbacks
|
|
}
|
|
|
|
type heardRollbacks struct {
|
|
at time.Time
|
|
list []lease.Rollback
|
|
}
|
|
|
|
// heard keeps what one machine's account said, replacing what it said before: an account without any
|
|
// says the machine's witnesses stand by none.
|
|
func (w *rollbacksHeard) heard(node string, list []lease.Rollback, at time.Time) {
|
|
w.mu.Lock()
|
|
defer w.mu.Unlock()
|
|
w.byNode[node] = heardRollbacks{at: at, list: append([]lease.Rollback(nil), list...)}
|
|
}
|
|
|
|
// all is what every machine heard from said, by machine.
|
|
func (w *rollbacksHeard) all() map[string][]lease.Rollback {
|
|
w.mu.Lock()
|
|
defer w.mu.Unlock()
|
|
out := map[string][]lease.Rollback{}
|
|
for node, h := range w.byNode {
|
|
if len(h.list) > 0 {
|
|
out[node] = append([]lease.Rollback(nil), h.list...)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// holder is who holds the controller lease now, read from the bucket: through the serving controller's
|
|
// own lease, or the bucket a command opened.
|
|
func (a *actor) holder(ctx context.Context) (lease.Holder, bool, error) {
|
|
a.mu.Lock()
|
|
held, kv := a.held, a.kv
|
|
a.mu.Unlock()
|
|
reading, cancel := context.WithTimeout(ctx, 5*time.Second)
|
|
defer cancel()
|
|
switch {
|
|
case held != nil:
|
|
return held.Current(reading)
|
|
case kv != nil:
|
|
return lease.Current(reading, kv)
|
|
}
|
|
return lease.Holder{}, false, errors.New("this process has no view of the controller lease")
|
|
}
|