The have's failed phase (apply-failing), unhealthy health (module-unhealthy) and kept words (kept-as-it-runs), and each part of its health report (machine-network, network-rewritten, network-unreachable, machine-units, relogin-needed, directory-used-as-found, node-account-fallback), raised and cleared by the held writer where the engine knows the fact. Builds: design 47 G A-197 (ADR 0596, A-192, A-149) Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
282 lines
10 KiB
Go
282 lines
10 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
"git.novox.be/novox/mesh-sdk/go/seatdef"
|
|
"git.novox.be/novox/mesh-sdk/go/seatlist"
|
|
"git.novox.be/novox/mesh-sdk/go/views"
|
|
"github.com/novox/mesh-host/internal/link"
|
|
"github.com/novox/mesh-host/internal/seatwait"
|
|
)
|
|
|
|
// The node-engine reads the seat list as every part does (novox/hq ADR 0635 §1, ADR 0636): through the seat mesh-catalog,
|
|
// from the bootstrapped definitions until it has been read, acting on nothing new meanwhile (the composer's gate), and
|
|
// following the seat's events. What it holds is given to catalogue at every change, which compiles no seat list.
|
|
|
|
// seatListNow is this run's reader, what the node-engine's verbs seat-list and seat-list-again answer from.
|
|
var seatListNow atomic.Pointer[seatListState]
|
|
|
|
// seatListRead answers the verb seat-list: what the engine holds of the list and what it waits for (ADR 0636 §7).
|
|
func seatListRead() (any, error) {
|
|
st := seatListNow.Load()
|
|
if st == nil {
|
|
return nil, errors.New("this engine's node agent has not started, so it holds no seat list yet")
|
|
}
|
|
status := st.reader.Status()
|
|
return map[string]any{"status": status, "health": status.Health()}, nil
|
|
}
|
|
|
|
// seatListAgain answers the verb seat-list-again: the list read now from the seat, and what is held afterwards.
|
|
func seatListAgain() (any, error) {
|
|
st := seatListNow.Load()
|
|
if st == nil {
|
|
return nil, errors.New("this engine's node agent has not started, so it holds no seat list yet")
|
|
}
|
|
err := st.reader.Reload()
|
|
st.waiter.Changed() // a read that changed nothing held still ends the wait
|
|
out, _ := seatListRead()
|
|
if err != nil {
|
|
out.(map[string]any)["error"] = err.Error()
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// seatListState is one run's reader, the finding that it waits, and the channel on which the composer hears a change.
|
|
type seatListState struct {
|
|
reader *seatlist.Reader
|
|
waiter *seatwait.Waiter
|
|
changed chan struct{}
|
|
}
|
|
|
|
// sessionBus is the reader's bus over the link: a verb of the seat is asked in the engine's own name, an event heard on
|
|
// the seat's event subject for as long as ctx lives.
|
|
type sessionBus struct {
|
|
ctx context.Context
|
|
session *link.Session
|
|
}
|
|
|
|
func (b sessionBus) Ask(key string, body any) (json.RawMessage, error) {
|
|
seat, verb, _ := strings.Cut(strings.TrimPrefix(key, "seat:"), ".")
|
|
return b.session.AskMeshSeat(b.ctx, seat, verb, body)
|
|
}
|
|
|
|
func (b sessionBus) Subscribe(pattern string, handle func(string, json.RawMessage) error) error {
|
|
seat, event, _ := strings.Cut(pattern, ".")
|
|
heard, stop, err := b.session.Hear(seatdef.EventSubject(seat, event))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
go func() {
|
|
defer stop()
|
|
for {
|
|
select {
|
|
case <-b.ctx.Done():
|
|
return
|
|
case body, ok := <-heard:
|
|
if !ok {
|
|
return
|
|
}
|
|
_ = handle(pattern, body)
|
|
}
|
|
}
|
|
}()
|
|
return nil
|
|
}
|
|
|
|
// seatWaitClock is the clock of the node's bound; the tests give their own.
|
|
var seatWaitClock seatwait.Clock = realClock{}
|
|
|
|
type realClock struct{}
|
|
|
|
func (realClock) Now() time.Time { return time.Now() }
|
|
func (realClock) AfterFunc(d time.Duration, f func()) seatwait.Timer {
|
|
return time.AfterFunc(d, f)
|
|
}
|
|
|
|
// nodeFindings is the node-engine's finding writer: mesh-sdk's held writer, for the engine's findings are raised as they
|
|
// happen and not judged afresh (design 47 G A-29, A-40), with the contract of the seat mesh-conditions.
|
|
type nodeFindings = views.HeldWriter[condseat.Finding, condseat.FindingCleared, condseat.FindingsStated, condseat.FindingsAskedEvent]
|
|
|
|
// findingsOfTheNode keeps the node's findings through mesh-sdk's held writer (design 47 G §383, §384, §461, A-29): each
|
|
// finding is written in the node's state `findings` and said as an event on the engine's own stream (a foundation part's
|
|
// `mesh-host-findings`, §529, A-30), the engine says what it holds at its start and when the keeper asks, and the keeper
|
|
// reads no state of the engine's.
|
|
type findingsOfTheNode struct {
|
|
*nodeFindings
|
|
}
|
|
|
|
// newFindingsOfTheNode makes the writer at once; it reads the state when it is started (A-40). Its events are published on
|
|
// the engine's own stream awaiting the bus's acknowledgement and its consumer of `findings-asked` is its own.
|
|
func newFindingsOfTheNode(session *link.Session, node, module string, say func(string)) (*findingsOfTheNode, error) {
|
|
writer, err := views.NewHeldWriter(views.WriterConfig[condseat.Finding, condseat.FindingCleared, condseat.FindingsStated, condseat.FindingsAskedEvent]{
|
|
Module: module,
|
|
Node: node,
|
|
State: link.FindingsState{Session: session},
|
|
Kinds: append([]string{seatwait.Kind, seatwait.HeldKind, seatwait.EntryNotFoundKind}, seatwait.NodeKinds...),
|
|
Asked: condseat.SubjectFindingsAsked,
|
|
Logf: func(format string, args ...any) { say(fmt.Sprintf(format, args...)) },
|
|
Publish: session.PublishEvent,
|
|
Listen: func(_ context.Context, subject string, handle func([]byte)) (func(), error) {
|
|
return session.ListenFrom(condseat.StreamFindingsAsked, subject, handle)
|
|
},
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &findingsOfTheNode{writer}, nil
|
|
}
|
|
|
|
// Start seeds what the state holds, states the findings now and hears the keeper until ctx ends.
|
|
func (w *findingsOfTheNode) Start(ctx context.Context) error {
|
|
go func() { <-ctx.Done(); w.Close() }()
|
|
return w.nodeFindings.Start(ctx)
|
|
}
|
|
|
|
// startSeatList starts this run's reader: its bootstrapped seats are given to catalogue at once, and every change of what
|
|
// it holds again, saying so to the composer and to the waiter that writes the five-minute finding, and stating again what
|
|
// the engine reads (link.Session.RestateReads, ADR 0618 §4): the seats it states are those of the list it holds.
|
|
func startSeatList(ctx context.Context, session *link.Session, node string, findings *lateFindings, say func(string)) (*seatListState, error) {
|
|
st := &seatListState{changed: make(chan struct{}, 1)}
|
|
// Start returns at once and reads the seat on a goroutine, which may announce a change before the waiter exists.
|
|
var announce atomic.Pointer[func()]
|
|
reader, err := seatlist.Start(seatlist.Config{Bus: sessionBus{ctx, session}, Bootstrap: seatlist.Bootstrapped(),
|
|
OnChange: func(seats []catalogue.Seat) {
|
|
catalogue.UseSeats(seats)
|
|
// What the engine reads changed with the list: its record and its event say it (ADR 0618 §4), off the
|
|
// reader's goroutine, which returns quickly.
|
|
go session.RestateReads(ctx)
|
|
if f := announce.Load(); f != nil { // the first read may end before the waiter exists
|
|
(*f)()
|
|
}
|
|
}})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("the node-engine cannot follow the seat list: %w", err)
|
|
}
|
|
st.reader = reader
|
|
st.waiter = &seatwait.Waiter{Node: node, ObserverOf: findings.observer, Reader: reader, Writer: findings, Ctx: ctx, Bound: viewWaitsBound(),
|
|
Clock: seatWaitClock, Say: say}
|
|
told := func() {
|
|
st.waiter.Changed()
|
|
select {
|
|
case st.changed <- struct{}{}:
|
|
default:
|
|
}
|
|
}
|
|
announce.Store(&told)
|
|
catalogue.UseSeats(reader.Seats())
|
|
session.RestateReads(ctx) // its first statement holds the bootstrapped seats at least (issue 1966)
|
|
told()
|
|
// The first read ends whether the seat answered or not, and a read that held what the reader held already says no
|
|
// change: its end is told to the waiter and the composer, so a status that moved from reading to heard starts or ends
|
|
// the bound (ADR 0636 §1). Start's return held nobody's start.
|
|
go func() {
|
|
select {
|
|
case <-reader.Ready():
|
|
catalogue.UseSeats(reader.Seats())
|
|
session.RestateReads(ctx)
|
|
told()
|
|
case <-ctx.Done():
|
|
}
|
|
}()
|
|
seatListNow.Store(st)
|
|
return st, nil
|
|
}
|
|
|
|
// lateFindings is the node-engine's finding writer, made when the engine's own module is known (design 47 G §527, §634,
|
|
// A-149): at once from its applied record, or, on a first start with no such record, when the first composing names it. Till
|
|
// then a write or a clear is refused in words, never lost in silence.
|
|
type lateFindings struct {
|
|
own *link.OwnName
|
|
make func(module string) (*findingsOfTheNode, error)
|
|
mu sync.Mutex
|
|
w *findingsOfTheNode
|
|
}
|
|
|
|
// observer is the module the findings are observed as: the engine's own, "" while it is not known.
|
|
func (f *lateFindings) observer() string { return f.own.Get() }
|
|
|
|
// Start makes the writer and starts it as soon as the own name is known: in this call when it already is, else on a goroutine.
|
|
func (f *lateFindings) Start(ctx context.Context, say func(string)) {
|
|
begin := func(module string) {
|
|
w, err := f.make(module)
|
|
if err != nil { // a writer that cannot be made is a fault of its configuration: loud
|
|
say("the node-engine cannot make its finding writer: " + err.Error())
|
|
return
|
|
}
|
|
if err := w.Start(ctx); err != nil {
|
|
say("the node-engine's findings are kept in its state and not said as events until it is started again: " + err.Error())
|
|
}
|
|
f.mu.Lock()
|
|
f.w = w
|
|
f.mu.Unlock()
|
|
}
|
|
if name := f.own.Get(); name != "" {
|
|
begin(name)
|
|
return
|
|
}
|
|
go func() {
|
|
if name, err := f.own.Wait(ctx); err == nil {
|
|
begin(name)
|
|
}
|
|
}()
|
|
}
|
|
|
|
var errOwnNameUnknown = errors.New("the node-engine's own module is not yet named by its applied record or its first composing, so its finding writer is not made")
|
|
|
|
func (f *lateFindings) writer() *findingsOfTheNode {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
return f.w
|
|
}
|
|
|
|
func (f *lateFindings) Write(ctx context.Context, fi condseat.Finding) error {
|
|
if w := f.writer(); w != nil {
|
|
return w.Write(ctx, fi)
|
|
}
|
|
return errOwnNameUnknown
|
|
}
|
|
|
|
func (f *lateFindings) Clear(ctx context.Context, scope, id, kind, why string) error {
|
|
if w := f.writer(); w != nil {
|
|
return w.Clear(ctx, scope, id, kind, why)
|
|
}
|
|
return errOwnNameUnknown
|
|
}
|
|
|
|
func (f *lateFindings) Held() []condseat.Finding {
|
|
if w := f.writer(); w != nil {
|
|
return w.Held()
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// viewWaitsFile is the file the manifest declares with the content ${config:view-waits-minutes} (design 47 G §633), read at
|
|
// the engine's start as recoveryWaitFile is.
|
|
var viewWaitsFile = "/var/lib/mesh-host/view-waits-minutes"
|
|
|
|
// viewWaitsBound is how long the reader of the seat list may wait before the engine says it: the file's minutes, the design's
|
|
// five (seatwait.Bound) when the file is missing, empty or not a positive number.
|
|
func viewWaitsBound() time.Duration {
|
|
raw, err := os.ReadFile(viewWaitsFile)
|
|
if err != nil {
|
|
return seatwait.Bound
|
|
}
|
|
n, err := strconv.Atoi(strings.TrimSpace(string(raw)))
|
|
if err != nil || n <= 0 {
|
|
return seatwait.Bound
|
|
}
|
|
return time.Duration(n) * time.Minute
|
|
}
|