novox/mesh-controller
6
0
Fork 0
Code Issues Pull Requests 5 Releases Wiki Activity

Heal what is known, under a brake, and say every repair (hq to-be 45 Phase 3)

Browse Source 
Research 031 counted the repairs people made by hand: a push to unstick a
plan waiting on a report, a controller restarted to make an object again, a
plan closed, a consumer re-made from now. Each was the ordinary path taken
again by someone who noticed. The healer registry makes each a registered
response to one condition kind, with a budget, a settle and its event:

- H1 sent-not-reported: ask the machine's node-engine to report again
  (mesh.node.<n>.ask.report); if it does not report what it was sent, send
  it again, never moving a build a policy or a plan holds back
- H2 stalled: close a plan whose wait is superseded or finished
- H3 holder-silent / consumer-lost: the send's own assertion of the bus's
  objects (issue 208's note)
- H4 consumer-behind: consumer-reset, only for a consumer the stream table
  marks resettable (the controller's own events consumer)
- H5 is the identity provider's own repair (ADR 0224 §5), registered only

Success is the observation clearing the condition, never the healer; a spent
budget hands the condition to the operator, urgent, with what was tried, and
no healer touches it again. Every act is begun in the store before it is made
(migration 0070), kept in the condition's tried as "healer Hn" and said as
the seat event healer-acted; a heal is never a hand act. More than twelve acts
in an hour stop every healer until an hour after the last, said urgently.
Only the lease holder heals.

S15 is live: a cause repaired by hand twice in a fortnight raises
healer-wanted, naming the healer that was not enough where one exists. D6's
far-behind finding has its own kind, consumer-behind. Nodes are granted the
question; the controller's grant gains healer-acted (genesis lock in
mesh-host). `healers` lists the registry, the acts and the brake; status
counts the week's heals.
This commit is contained in:
jochen
2026-10-06 14:26:26 +02:00
parent 20c147ffdb
commit 751e39186c
31 changed files with 2154 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/conditions/store.go
+97 -4
View File
@@ -76,6 +76,9 @@ type clearing struct {
at time.Time
count int
silenced *Silence
// tried is what healers tried before it cleared: a reopening is the same fault, and what was
// tried on it is still what was tried.
tried []Attempt
}
// Options are what a Keeper is made with.
@@ -113,7 +116,8 @@ func NewKeeper(ctx context.Context, o Options) *Keeper {
if recent, err := k.history.Since(ctx, k.now().Add(-ReopenWithin)); err == nil {
for _, e := range recent {
if e.Change == ChangeCleared {
k.cleared[e.Key] = clearing{at: e.At, count: e.Condition.Count, silenced: e.Condition.Silenced}
k.cleared[e.Key] = clearing{at: e.At, count: e.Condition.Count, silenced: e.Condition.Silenced,
tried: e.Condition.Tried}
}
}
} else {
@@ -191,6 +195,7 @@ func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error)
if before.silenced != nil && now.Before(before.silenced.Until) {
c.Silenced = before.silenced
}
c.Tried = before.tried
}
k.mu.Unlock()
if err := k.stamp(&c); err != nil {
@@ -216,7 +221,8 @@ func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error)
return Condition{}, fmt.Errorf("the condition %s on the bus cannot be read: %w", key, err)
}
var changes []Event
if o.Severity != c.Severity {
// A healer's budget spent made it urgent; the watchdog seeing it again does not undo that.
if o.Severity != c.Severity && !(c.Escalated() && o.Severity != Urgent) {
changes = append(changes, Event{Change: ChangeSeverity, Was: string(c.Severity)})
c.Severity = o.Severity
}
@@ -224,7 +230,9 @@ func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error)
changes = append(changes, Event{Change: ChangeResolver, Was: c.Resolver})
c.Resolver = r
}
c.Summary, c.Source, c.LastObserved = o.Summary, o.Source, now
// The kind as the source says it now: a source that gave the same key a kind of its own since
// (a probe's finding split out for a healer) is read by that kind from its next observation.
c.Kind, c.Summary, c.Source, c.LastObserved = o.Kind, o.Summary, o.Source, now
if o.Machine != "" {
c.Subject.Machine = o.Machine
}
@@ -282,7 +290,7 @@ func (k *Keeper) Clear(ctx context.Context, key, why string) (bool, error) {
}
now := k.now().UTC()
k.mu.Lock()
k.cleared[key] = clearing{at: now, count: c.Count, silenced: c.Silenced}
k.cleared[key] = clearing{at: now, count: c.Count, silenced: c.Silenced, tried: c.Tried}
k.mu.Unlock()
k.tell(Event{Condition: c, At: now, Change: ChangeCleared, Why: why, Cleared: &now})
return true, nil
@@ -290,6 +298,91 @@ func (k *Keeper) Clear(ctx context.Context, key, why string) (bool, error) {
return false, fmt.Errorf("the condition %s kept moving under this clearing; %d tries", key, tries)
}
// Tried records a healer's attempt on an open condition (to-be 45 §7) and makes the healer its
// resolver: said as `condition-changed` when the resolver changes, kept in `tried` either way. **It
// never clears the condition**: a repair that worked is seen by the observation that raised it, which
// clears it — a healer marking its own work done would be a second opinion of the fact. False when no
// condition is open under the key: it cleared meanwhile, and there is nothing to record against.
func (k *Keeper) Tried(ctx context.Context, key string, a Attempt, resolver string) (Condition, bool, error) {
return k.amend(ctx, key, func(c *Condition, now time.Time) []Event {
c.Tried = appendAttempt(c.Tried, a, now)
var changes []Event
if resolver != "" && resolver != c.Resolver && !c.Escalated() {
changes = append(changes, Event{Change: ChangeResolver, Was: c.Resolver, Why: a.Outcome})
c.Resolver = resolver
}
return changes
})
}
// Escalate is a healer's budget spent, or a repair it may not make (to-be 45 §2, §7): the attempt that
// says so is kept in `tried`, the resolver becomes the operator and the severity urgent, each said as
// `condition-changed`. Observation still clears it when the fault goes; nothing else does.
func (k *Keeper) Escalate(ctx context.Context, key string, a Attempt) (Condition, bool, error) {
return k.amend(ctx, key, func(c *Condition, now time.Time) []Event {
c.Tried = appendAttempt(c.Tried, a, now)
var changes []Event
if c.Severity != Urgent {
changes = append(changes, Event{Change: ChangeSeverity, Was: string(c.Severity), Why: a.Outcome})
c.Severity = Urgent
}
if c.Resolver != ResolverOperator {
changes = append(changes, Event{Change: ChangeResolver, Was: c.Resolver, Why: a.Outcome})
c.Resolver = ResolverOperator
}
return changes
})
}
// appendAttempt adds an attempt, stamped when it has no time, keeping the newest KeptAttempts.
func appendAttempt(tried []Attempt, a Attempt, now time.Time) []Attempt {
if a.At.IsZero() {
a.At = now
}
tried = append(tried, a)
if len(tried) > KeptAttempts {
tried = tried[len(tried)-KeptAttempts:]
}
return tried
}
// amend changes one open condition by compare-and-set and says what changed. False when none is open.
func (k *Keeper) amend(ctx context.Context, key string, change func(*Condition, time.Time) []Event) (Condition, bool, error) {
for i := 0; i < tries; i++ {
entry, found, err := k.store.Get(ctx, key)
if err != nil {
return Condition{}, false, fmt.Errorf("reading the condition %s: %w", key, err)
}
if !found {
return Condition{}, false, nil
}
var c Condition
if err := json.Unmarshal(entry.Value, &c); err != nil {
return Condition{}, false, fmt.Errorf("the condition %s on the bus cannot be read: %w", key, err)
}
now := k.now().UTC()
changes := change(&c, now)
if err := k.stamp(&c); err != nil {
return Condition{}, false, err
}
body, err := json.Marshal(c)
if err != nil {
return Condition{}, false, err
}
if err := k.store.Update(ctx, key, body, entry.Revision); errors.Is(err, ErrMoved) {
continue
} else if err != nil {
return Condition{}, false, fmt.Errorf("writing the condition %s: %w", key, err)
}
for _, e := range changes {
e.At, e.Condition = now, c
k.tell(e)
}
return c, true, nil
}
return Condition{}, false, fmt.Errorf("the condition %s kept moving under this write; %d tries", key, tries)
}
// Reconcile is one source's whole observation: every condition it observes is observed, and every
// condition it raised before and no longer observes is cleared — the observation says it is
// resolved. A source that could not observe must not call this: an empty observation clears all it