Heal what is known, under a brake, and say every repair (hq to-be 45 Phase 3)
Research 031 counted the repairs people made by hand: a push to unstick a plan waiting on a report, a controller restarted to make an object again, a plan closed, a consumer re-made from now. Each was the ordinary path taken again by someone who noticed. The healer registry makes each a registered response to one condition kind, with a budget, a settle and its event: - H1 sent-not-reported: ask the machine's node-engine to report again (mesh.node.<n>.ask.report); if it does not report what it was sent, send it again, never moving a build a policy or a plan holds back - H2 stalled: close a plan whose wait is superseded or finished - H3 holder-silent / consumer-lost: the send's own assertion of the bus's objects (issue 208's note) - H4 consumer-behind: consumer-reset, only for a consumer the stream table marks resettable (the controller's own events consumer) - H5 is the identity provider's own repair (ADR 0224 §5), registered only Success is the observation clearing the condition, never the healer; a spent budget hands the condition to the operator, urgent, with what was tried, and no healer touches it again. Every act is begun in the store before it is made (migration 0070), kept in the condition's tried as "healer Hn" and said as the seat event healer-acted; a heal is never a hand act. More than twelve acts in an hour stop every healer until an hour after the last, said urgently. Only the lease holder heals. S15 is live: a cause repaired by hand twice in a fortnight raises healer-wanted, naming the healer that was not enough where one exists. D6's far-behind finding has its own kind, consumer-behind. Nodes are granted the question; the controller's grant gains healer-acted (genesis lock in mesh-host). `healers` lists the registry, the acts and the brake; status counts the week's heals.
This commit is contained in:
jochen
@@ -0,0 +1,126 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The healers' acts (novox/hq to-be 45 §7, Phase 3): see migration 0070. The controller is their
|
||||
// only writer, under its lease; `healers` and `conditions` read them.
|
||||
|
||||
// The outcomes of a heal.
|
||||
const (
|
||||
// HealActing is an act begun and not yet finished: counted against the budget and the brake from
|
||||
// the moment it starts, so a controller that dies mid-act has still spent it.
|
||||
HealActing = "acting"
|
||||
// HealActed is an act that did what it does. Whether it repaired anything is the observation's to
|
||||
// say, never the healer's: the condition clears, or it does not.
|
||||
HealActed = "acted"
|
||||
// HealFailed is an act that could not be done.
|
||||
HealFailed = "failed"
|
||||
// HealEscalated is a budget spent, said: the condition was handed to the operator.
|
||||
HealEscalated = "escalated"
|
||||
)
|
||||
|
||||
// HealsKeptFor is how long a heal is kept: as long as a condition's history.
|
||||
const HealsKeptFor = 90 * 24 * time.Hour
|
||||
|
||||
// Heal is one act of a healer.
|
||||
type Heal struct {
|
||||
ID int64 `json:"id"`
|
||||
Healer string `json:"healer"`
|
||||
ConditionKey string `json:"condition"`
|
||||
Kind string `json:"kind"`
|
||||
BudgetKey string `json:"budget-key"`
|
||||
Act string `json:"act"`
|
||||
Outcome string `json:"outcome"`
|
||||
Said string `json:"said,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
Finished *time.Time `json:"finished,omitempty"`
|
||||
Epoch uint64 `json:"epoch,omitempty"`
|
||||
}
|
||||
|
||||
// BeginHeal writes an act about to be taken, under the lease: the act is counted from here. Refused,
|
||||
// and nothing written, by a process that may not act.
|
||||
func (i *Inventory) BeginHeal(ctx context.Context, h Heal) (Heal, error) {
|
||||
epoch, err := i.actingEpoch(ctx)
|
||||
if err != nil {
|
||||
return h, fmt.Errorf("the heal %s of %s is not begun: %w", h.Healer, h.ConditionKey, err)
|
||||
}
|
||||
if strings.TrimSpace(h.Healer) == "" || strings.TrimSpace(h.ConditionKey) == "" || strings.TrimSpace(h.Act) == "" {
|
||||
return h, errors.New("a heal names its healer, its condition and its act")
|
||||
}
|
||||
if h.BudgetKey == "" {
|
||||
h.BudgetKey = h.ConditionKey
|
||||
}
|
||||
if h.Outcome == "" {
|
||||
h.Outcome = HealActing
|
||||
}
|
||||
// At the runner's moment when it gives one, so its budgets and the brake are counted on one clock.
|
||||
var at *time.Time
|
||||
if !h.At.IsZero() {
|
||||
at = &h.At
|
||||
}
|
||||
err = i.store.Pool().QueryRow(ctx,
|
||||
`insert into heal (healer, condition_key, kind, budget_key, act, outcome, said, epoch, at)
|
||||
values ($1, $2, $3, $4, $5, $6, $7, $8, coalesce($9, now())) returning id, at`,
|
||||
h.Healer, h.ConditionKey, h.Kind, h.BudgetKey, h.Act, h.Outcome, h.Said, epoch, at).Scan(&h.ID, &h.At)
|
||||
if err != nil {
|
||||
return h, err
|
||||
}
|
||||
if epoch != nil {
|
||||
h.Epoch = uint64(*epoch)
|
||||
}
|
||||
return h, nil
|
||||
}
|
||||
|
||||
// FinishHeal says what came of an act begun. Written whatever the lease says by now: what was done was
|
||||
// done, and its record must not depend on the doer still holding the lease a moment later.
|
||||
func (i *Inventory) FinishHeal(ctx context.Context, id int64, outcome, said string) error {
|
||||
tag, err := i.store.Pool().Exec(ctx,
|
||||
`update heal set outcome = $2, said = $3, finished = now() where id = $1`, id, outcome, said)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if tag.RowsAffected() != 1 {
|
||||
return fmt.Errorf("no heal %d to finish", id)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// HealsSince is every heal from a moment, oldest first.
|
||||
func (i *Inventory) HealsSince(ctx context.Context, since time.Time) ([]Heal, error) {
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select id, healer, condition_key, kind, budget_key, act, outcome, said, at, finished, epoch
|
||||
from heal where at >= $1 order by at, id`, since)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []Heal
|
||||
for rows.Next() {
|
||||
var h Heal
|
||||
var epoch *int64
|
||||
if err := rows.Scan(&h.ID, &h.Healer, &h.ConditionKey, &h.Kind, &h.BudgetKey, &h.Act, &h.Outcome, &h.Said,
|
||||
&h.At, &h.Finished, &epoch); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if epoch != nil {
|
||||
h.Epoch = uint64(*epoch)
|
||||
}
|
||||
out = append(out, h)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// ForgetOldHeals removes what is older than HealsKeptFor, and says how many.
|
||||
func (i *Inventory) ForgetOldHeals(ctx context.Context) (int64, error) {
|
||||
tag, err := i.store.Pool().Exec(ctx, `delete from heal where at < $1`, time.Now().Add(-HealsKeptFor))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return tag.RowsAffected(), nil
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// **A heal is begun under the lease and kept with its epoch** (novox/hq to-be 45 §7): refused, and
|
||||
// nothing written, by a process that may not act; finished whatever the lease says by then; read back
|
||||
// in the order they were taken.
|
||||
func TestAHealIsBegunUnderTheLeaseAndFinishedAfter(t *testing.T) {
|
||||
inv := ForTest(t)
|
||||
ctx := t.Context()
|
||||
inv.ActsUnder(func(context.Context) (uint64, error) { return 0, errors.New("the lease is not held") })
|
||||
if _, err := inv.BeginHeal(ctx, Heal{Healer: "H1", ConditionKey: "machine.anchor.sent-not-reported", Act: "asked"}); err == nil {
|
||||
t.Fatal("a heal was begun by a process that may not act")
|
||||
}
|
||||
inv.ActsUnder(func(context.Context) (uint64, error) { return 57, nil })
|
||||
at := time.Now().Add(-time.Minute).UTC().Truncate(time.Microsecond)
|
||||
first, err := inv.BeginHeal(ctx, Heal{Healer: "H1", ConditionKey: "machine.anchor.sent-not-reported",
|
||||
Kind: "sent-not-reported", Act: "asked", At: at})
|
||||
if err != nil || first.ID == 0 || first.Epoch != 57 || first.BudgetKey != first.ConditionKey ||
|
||||
first.Outcome != HealActing || !first.At.Equal(at) {
|
||||
t.Fatalf("begun: %+v %v", first, err)
|
||||
}
|
||||
if _, err := inv.BeginHeal(ctx, Heal{Healer: "H3"}); err == nil {
|
||||
t.Fatal("a heal naming no condition and no act was begun")
|
||||
}
|
||||
inv.ActsUnder(func(context.Context) (uint64, error) { return 0, errors.New("lost meanwhile") })
|
||||
if err := inv.FinishHeal(ctx, first.ID, HealActed, "it reported"); err != nil {
|
||||
t.Fatalf("what was done could not be recorded once the lease was gone: %v", err)
|
||||
}
|
||||
if err := inv.FinishHeal(ctx, first.ID+1000, HealActed, ""); err == nil {
|
||||
t.Fatal("a heal that was never begun was finished")
|
||||
}
|
||||
heals, err := inv.HealsSince(ctx, time.Now().Add(-time.Hour))
|
||||
if err != nil || len(heals) != 1 || heals[0].Outcome != HealActed || heals[0].Said != "it reported" ||
|
||||
heals[0].Finished == nil || heals[0].Epoch != 57 {
|
||||
t.Fatalf("read back: %+v %v", heals, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
-- A known failure heals itself, under a brake, and every repair is said (novox/hq to-be 45 §7,
|
||||
-- Phase 3, ADR 0227 rule 7).
|
||||
--
|
||||
-- Every act a healer takes is a row here, written before the act and finished after it: what the
|
||||
-- budgets and the mesh-wide brake count, and what `healers` reads back. In the store and not in a
|
||||
-- bucket on the bus, because the budget must outlive the controller — a controller restarting in a
|
||||
-- loop must not reset a healer's count each time — and because the bus's user list would have to
|
||||
-- grant a new bucket before the first heal could be counted.
|
||||
--
|
||||
-- Numbered 0070, past 0069, while the consumer-retirement feature (ADR 0230) is built beside this one:
|
||||
-- a migration it adds takes 0069 if it merges first; one merged after this must be numbered above
|
||||
-- 0070, because the store refuses to run a lower number than one already run.
|
||||
create table heal (
|
||||
id bigserial primary key,
|
||||
-- The healer's id in the registry: H1, H2, ...
|
||||
healer text not null,
|
||||
-- The condition it acted on, and the condition's kind.
|
||||
condition_key text not null,
|
||||
kind text not null,
|
||||
-- What its budget is counted against: the condition, a machine, a holder, a consumer.
|
||||
budget_key text not null,
|
||||
-- What it did, in the mesh's words.
|
||||
act text not null,
|
||||
-- 'acting' while the act runs; then 'acted', 'failed' or 'escalated' (a budget spent, said).
|
||||
outcome text not null default 'acting',
|
||||
said text not null default '',
|
||||
at timestamptz not null default now(),
|
||||
finished timestamptz,
|
||||
-- The lease epoch it acted under (to-be 45 §6); null where none was claimed.
|
||||
epoch bigint
|
||||
);
|
||||
|
||||
create index heal_at on heal (at);
|
||||
create index heal_budget on heal (healer, budget_key, at);
|
||||
Reference in New Issue
Block a user