Hear what providers retire, and let a person approve, reject and delete (hq ADR 0230)

A provider now waits for a person before retiring more than three consumers
or half of what it holds, and deletes only when asked. The controller is that
person's way in: it keeps waiting and rejected sets as conditions, answers them
with retire approve|reject, lists and deletes retired consumers through the
provider's own tools on its machine, records each act in the hand-act log, and
probes for anything retired longer than thirty days (D11).
This commit is contained in:
jochen
2026-10-06 14:41:15 +02:00
parent 298daa6fbe
commit 68009b16fe
23 changed files with 1678 additions and 27 deletions
+7 -2
View File
@@ -255,13 +255,18 @@ var ControllerFollows = []string{
// the index is a name.
moduleEventSubject("*", ProvisionerFailing),
moduleEventSubject("*", ProvisionerRecovered),
// **What becomes of a consumer the mesh stopped asking for** (novox/hq ADR 0230): retired, waiting
// for a person, re-enabled, deleted — a provider's third word, from whichever module provides.
// Appended, because the index is a name.
moduleEventSubject("*", ProvisionerRetirement),
}
// The provider standing events, by their local names. Written here as well as in the catalogue
// (catalogue.ProvisionerEvents), which this package cannot import; a test keeps them agreeing.
const (
ProvisionerFailing = "provisioner.failing"
ProvisionerRecovered = "provisioner.recovered"
ProvisionerFailing = "provisioner.failing"
ProvisionerRecovered = "provisioner.recovered"
ProvisionerRetirement = "provisioner.retirement"
)
// moduleEventSubject is where one module's event lands. The same derivation PermissionsFor uses, so
+1 -1
View File
@@ -25,7 +25,7 @@ accounts {
users = [
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*"] }
subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] }
subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] }
allow_responses: { max: 1, ttl: "1m" }
} }
{ user: "enrol.one", password: "$2a$11$eeeeeeeeeeeeeeeeeeeeee", permissions: {
+4 -3
View File
@@ -117,9 +117,10 @@ var WritersTable = []WriterRow{
{State: "a merge announced", Writer: "one announcer per forge (the hook, or the poll when the hook is absent — never both)",
KeptIn: "the bus", Others: "—", Subjects: []string{"mesh.mod.*.event.pull.merged"}, Writes: ownModule},
{State: "a provider's standing", Writer: "the provider", KeptIn: "the provider's events",
Others: "the controller keeps the newest word as a condition",
Subjects: []string{"mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered"},
Writes: ownModule},
Others: "the controller keeps the newest word as a condition",
Subjects: []string{"mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered",
"mesh.mod.*.event.provisioner.retirement"},
Writes: ownModule},
{State: "the operator-channel's open messages", Writer: "the seat's holder", KeptIn: "its own key-value state",
Others: "—"},
{State: "the facts snapshot", Writer: "controller", KeptIn: "the artifact store, facts/latest",
+1 -1
View File
@@ -55,7 +55,7 @@ func TestAWholeMeshComposesWithOneWriterPerState(t *testing.T) {
{Kind: KindEnrolment, Node: "two", PasswordHash: "x"},
{Kind: KindPerson, Module: "jochen", Invokes: []string{"*"}, PasswordHash: "x"},
{Kind: KindModule, Node: "one", Module: "gitea", Emits: []string{"pull.merged"}, PasswordHash: "x"},
{Kind: KindModule, Node: "one", Module: "postgres", Emits: []string{"provisioner.failing", "provisioner.recovered"},
{Kind: KindModule, Node: "one", Module: "postgres", Emits: []string{"provisioner.failing", "provisioner.recovered", "provisioner.retirement"},
PasswordHash: "x"},
{Kind: KindModule, Node: "one", Module: "build-agent", Holds: []Seat{builder}, PasswordHash: "x"},
{Kind: KindNodeTools, Node: "one", Module: RuntimeModule, PasswordHash: "x", Carries: []Declared{
+10 -4
View File
@@ -164,13 +164,19 @@ func consumePattern(pattern string) error {
// The events a provider says about its consumers (novox/hq ADR 0224): a consumer it has failed
// without one success for minutes, and that consumer succeeding again or being withdrawn. The
// controller follows them from every module and `status` names a consumer failing until it recovers.
//
// **And what becomes of a consumer the mesh stopped asking for** (novox/hq ADR 0230): retired — its
// access disabled and its data kept — after the same answer in five passes, waiting for a person when
// more would go than the bound allows, re-enabled when asked for again, and deleted only by a person's
// `cleanup delete`. One event, its `change` saying which.
const (
ProvisionerFailing = "provisioner.failing"
ProvisionerRecovered = "provisioner.recovered"
ProvisionerFailing = "provisioner.failing"
ProvisionerRecovered = "provisioner.recovered"
ProvisionerRetirement = "provisioner.retirement"
)
// ProvisionerEvents are both, in the order they are said.
var ProvisionerEvents = []string{ProvisionerFailing, ProvisionerRecovered}
// ProvisionerEvents are all three, in the order they are said.
var ProvisionerEvents = []string{ProvisionerFailing, ProvisionerRecovered, ProvisionerRetirement}
// EmitsAll is every event a module may publish: what it declares and, for a module that receives
// contributions — a provider, running a provisioner over them — the provider's standing events.
@@ -0,0 +1,22 @@
package catalogue
import (
"slices"
"testing"
)
// Every provider may say what becomes of a consumer the mesh stopped asking for (novox/hq ADR 0230),
// whatever its manifest lists — as it may say a consumer it keeps failing (ADR 0224) — and a module
// that receives nothing may not.
func TestEveryProviderMaySayWhatItRetires(t *testing.T) {
provider := Manifest{Module: "pg", Receives: map[string]string{"postgres-database": "/x/mesh.json"},
Emits: []string{"database.provisioned"}}
for _, e := range []string{ProvisionerFailing, ProvisionerRecovered, ProvisionerRetirement, "database.provisioned"} {
if !slices.Contains(provider.EmitsAll(), e) {
t.Errorf("a provider may not emit %s: %v", e, provider.EmitsAll())
}
}
if slices.Contains(Manifest{Module: "app", Emits: []string{"x"}}.EmitsAll(), ProvisionerRetirement) {
t.Error("a module that provides nothing may say what it retires")
}
}
+27
View File
@@ -267,6 +267,33 @@ var ControllerVerbs = []Verb{
"probes": "\"true\": the registry — what each probe asserts, and the condition it raises",
"signals": "\"true\": the signals table, each row with the age of its newest signal",
}, nil, "run", "probes", "signals")},
// A consumer the mesh stopped asking for: retired, waiting for a person, deleted only by one
// (novox/hq ADR 0230).
{Name: "retire", Description: "A consumer the mesh stops asking for is retired by its provider — access " +
"disabled, data kept — after the same answer in five passes; more than three at once, or more than half " +
"of those held, waits for a person. With no answer, every provider that waits and what it would retire. " +
"With answer approve or reject, a node and a module: retire exactly what that provider waits with, or " +
"keep it active — a hand act, which says why (novox/hq ADR 0230).",
Input: schema(map[string]string{
"answer": "approve or reject: answer what the provider waits with (needs node, module and why)",
"node": "with answer: the machine the provider runs on",
"module": "with answer: the provider module",
"why": "with answer: why — required, and recorded in the hand-act log",
"cause": "with answer: the cause in a word (retire-waiting when absent)",
}, nil)},
{Name: "cleanup", Description: "Every consumer a provider holds retired — its age, its size where the " +
"backend knows, and why it was retired. With consumer (and node, module): the provider deletes that one " +
"retired consumer — never an active one. With older-than: every retired consumer older than that many " +
"days, listed; deleted only with confirm. Deleting is a hand act, which says why (novox/hq ADR 0230).",
Input: schema(map[string]string{
"node": "with consumer: the machine the provider runs on",
"module": "with consumer: the provider module",
"consumer": "delete this retired consumer (needs node, module and why)",
"older-than": "delete every consumer retired more than this many days (needs why; lists only without confirm)",
"confirm": "\"true\": with older-than, delete what is listed",
"why": "with consumer or older-than: why — required, and recorded in the hand-act log",
"cause": "with consumer or older-than: the cause in a word (cleanup-waiting when absent)",
}, nil, "confirm")},
{Name: "build", Description: "Have the build machine build a repository. Answers at once with the build's id: " +
"`builds` with that id follows it line by line, and the module is registered when the outcome comes.",
Input: schema(map[string]string{
+6 -2
View File
@@ -42,6 +42,10 @@ var Contracts = map[string]Contract{
"catalogue's record, which is the order, so a late one reads the same record"},
KindCatchUp: {Unordered: "a catalogue asking what it missed: answered from the record, whenever asked"},
KindProvisioner: {Unordered: "a provider's newest word about a consumer, said again every fifteen minutes " +
"while it holds (ADR 0224): the condition keeps the last observed, and S8 says when the words stop",
Tests: []string{"TestAProviderFailingAConsumerBreaksAllWellUntilItRecovers"}},
"while it holds (ADR 0224): the condition keeps the last observed, and S8 says when the words stop. " +
"Its retirement word (ADR 0230) is unordered too: a waiting set is said again every fifteen minutes, " +
"and every other change is checked against the provider itself — `retire` and `cleanup` ask it, " +
"and the self-check's D11 asks it every run — so an older word read late is corrected by the next",
Tests: []string{"TestAProviderFailingAConsumerBreaksAllWellUntilItRecovers",
"TestARetirementWordIsKeptAsItsConditionNamingTheEmitterFromTheSubject"}},
}
+3 -2
View File
@@ -262,7 +262,7 @@ func kindOfSubject(subject string) (string, bool) {
}
// ProvisionerEmitter is the module a provider's standing event came from, read from its subject
// (`mesh.mod.<module>.event.provisioner.<failing|recovered>`); false for any other subject. The
// (`mesh.mod.<module>.event.provisioner.<failing|recovered|retirement>`); false for any other subject. The
// controller's own follow pattern, with `*` for the module, decodes too.
func ProvisionerEmitter(subject string) (string, bool) {
rest, ok := strings.CutPrefix(subject, "mesh.mod.")
@@ -273,7 +273,8 @@ func ProvisionerEmitter(subject string) (string, bool) {
if !ok || module == "" || strings.Contains(module, ".") {
return "", false
}
if event != broker.ProvisionerFailing && event != broker.ProvisionerRecovered {
if event != broker.ProvisionerFailing && event != broker.ProvisionerRecovered &&
event != broker.ProvisionerRetirement {
return "", false
}
return module, true
+241
View File
@@ -0,0 +1,241 @@
package link
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
)
// What becomes of a consumer the mesh stopped asking for (novox/hq ADR 0230).
//
// **A consumer is retired, not withdrawn, and deleted only by a person.** A provider that has seen the
// same consumers go unasked for in five passes disables their access and keeps their data, marked in
// its backend with when and why; asked for again, it enables them as they were. When more would go
// than its bound allows — more than three, or more than half of those it holds — it retires nothing and
// waits for `retire approve` or `retire reject`. A retired consumer is deleted only by `cleanup delete`,
// which the provider executes. Each of these is the provider's `provisioner.retirement` event, its
// `change` saying which; the controller keeps the ones that need a person as conditions.
// The changes a provider says.
const (
RetireWaiting = "waiting"
RetireSettled = "settled"
RetireApproved = "approved"
RetireRejected = "rejected"
RetireRetired = "retired"
RetireReenabled = "reenabled"
RetireDeleted = "deleted"
RetireAdopted = "adopted"
)
// ViaController is what the controller's verbs say they came through, so an act a provider was asked
// for some other way is told apart and recorded by hand.
const ViaController = "mesh-controller"
// RetiredConsumer is one consumer in a retirement word, or in a provider's answer.
type RetiredConsumer struct {
Consumer string `json:"consumer"`
Node string `json:"node,omitempty"`
RetiredAt string `json:"retired_at,omitempty"`
Why string `json:"why,omitempty"`
// SizeBytes is what it keeps on the backend; -1 or absent where the backend cannot say.
SizeBytes *int64 `json:"size_bytes,omitempty"`
// Kind is "consumer", or a backend's own word for something set aside ("set-aside-database").
Kind string `json:"kind,omitempty"`
}
// Retirement is one provider's word about consumers the mesh stopped asking for.
type Retirement struct {
// Module is the emitter, read from the subject the bus let it publish on — never from the body.
Module string `json:"-"`
Provider string `json:"provider"`
ProviderNode string `json:"provider-node"`
Change string `json:"change"`
Consumers []RetiredConsumer `json:"consumers"`
Held int `json:"held"`
Bound string `json:"bound,omitempty"`
Why string `json:"why,omitempty"`
By string `json:"by,omitempty"`
Via string `json:"via,omitempty"`
At time.Time `json:"at"`
Since time.Time `json:"since,omitempty"`
}
// Names are the consumers it names, in its order.
func (r Retirement) Names() []string {
out := make([]string, 0, len(r.Consumers))
for _, c := range r.Consumers {
out = append(out, c.Consumer)
}
return out
}
// Retirements keeps what providers say about consumers they retire.
type Retirements interface {
// Retired records one word. An error the store is away for is held and asked again, like a report.
Retired(ctx context.Context, r Retirement) error
}
// KeepsRetirements says where retirement words are kept, and asks for them to be delivered.
func (s *Server) KeepsRetirements(r Retirements) error {
if err := s.inbound.Also(KindProvisioner); err != nil {
return err
}
s.retirements = r
return nil
}
// IsRetirement says a subject is a provider's retirement word.
func IsRetirement(subject string) bool {
_, ok := ProvisionerEmitter(subject)
return ok && strings.HasSuffix(subject, ".event."+broker.ProvisionerRetirement)
}
// ReadRetirement is one retirement word as the controller understands it.
func ReadRetirement(subject string, body []byte) (Retirement, error) {
module, ok := ProvisionerEmitter(subject)
if !ok || !IsRetirement(subject) {
return Retirement{}, fmt.Errorf("%s is not a provider's retirement word", subject)
}
var r Retirement
if err := json.Unmarshal(body, &r); err != nil {
return Retirement{}, fmt.Errorf("%s's retirement word could not be read: %w", module, err)
}
switch r.Change {
case RetireWaiting, RetireSettled, RetireApproved, RetireRejected, RetireRetired, RetireReenabled,
RetireDeleted, RetireAdopted:
default:
return Retirement{}, fmt.Errorf("%s said a retirement change the mesh has no name for: %q", module, r.Change)
}
if r.ProviderNode == "" {
return Retirement{}, fmt.Errorf("%s's retirement word named no machine", module)
}
r.Module = module
return r, nil
}
// retirement acts on one retirement word. Like a recovery, a word that clears a condition is said
// once, so a store that is away holds the message rather than dropping it.
func (s *Server) retirement(ctx context.Context, m Control) {
if s.retirements == nil {
_ = m.Took()
return
}
r, err := ReadRetirement(m.Subject(), m.Body())
if err != nil {
s.log.Printf("%v; ignored", err)
_ = m.Took()
return
}
err = s.retirements.Retired(ctx, r)
what := fmt.Sprintf("%s's retirement word (%s)", r.Module, r.Change)
switch s.decide(ctx, m, what, "", "", err) {
case Hold:
return
case Stale, GiveUp:
_ = m.Took()
return
}
if err != nil {
s.log.Printf("%s could not be kept: %v", what, err)
} else {
s.log.Printf("%s on %s %s %s", r.Module, r.ProviderNode, r.Change, strings.Join(r.Names(), ", "))
}
_ = m.Took()
}
// The tools every provider serves about its retired consumers (ADR 0230), asked of one machine.
const (
ToolRetirement = "provisioner_retirement"
ToolRetireApprove = "provisioner_retire_approve"
ToolRetireReject = "provisioner_retire_reject"
ToolRetiredDelete = "provisioner_delete"
)
// ErrNothingServes is a tool nothing on that machine serves: the module is not running there, or
// is older than the tool.
var ErrNothingServes = errors.New("nothing serves it")
// ModuleToolOn is where one machine's instance of a module answers a tool: the module's tool subject
// with the machine as its last token — the subject the node tools bind beside the shared one.
func ModuleToolOn(module, tool, node string) string { return ToolSubject(module, tool) + "." + node }
// AskModuleToolOn asks one machine's instance of a module one of its tools and reads its answer.
func AskModuleToolOn(ctx context.Context, conn *nats.Conn, module, tool, node string, args any,
timeout time.Duration) (Answer, error) {
body, err := json.Marshal(args)
if err != nil {
return Answer{}, err
}
asking, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
subject := ModuleToolOn(module, tool, node)
refused, stop := refusalsOf(conn, subject)
defer stop()
type replied struct {
msg *nats.Msg
err error
}
done := make(chan replied, 1)
go func() {
msg, err := conn.RequestWithContext(asking, subject, body)
done <- replied{msg, err}
}()
var reply *nats.Msg
select {
case r := <-done:
reply, err = r.msg, r.err
case why := <-refused:
cancel()
return Answer{}, fmt.Errorf("the bus refused the controller asking %s.%s on %s: %v", module, tool, node, why)
}
switch {
case errors.Is(err, nats.ErrNoResponders):
return Answer{}, fmt.Errorf("%w: %s on %s does not answer %s — it is not running there, or is "+
"older than the tool", ErrNothingServes, module, node, tool)
case errors.Is(err, context.DeadlineExceeded), errors.Is(err, nats.ErrTimeout):
return Answer{}, fmt.Errorf("%s on %s did not answer %s within %s", module, node, tool, timeout)
case err != nil:
return Answer{}, err
}
var answer Answer
if err := json.Unmarshal(reply.Data, &answer); err != nil {
return Answer{}, fmt.Errorf("%s on %s answered %s with something unreadable: %w", module, node, tool, err)
}
return answer, nil
}
// RetirementWaiting is the set a provider waits with for a person.
type RetirementWaiting struct {
Consumers []RetiredConsumer `json:"consumers"`
Since string `json:"since"`
Held int `json:"held"`
}
// RetirementRejected is the set a person's rejection keeps active.
type RetirementRejected struct {
Consumers []RetiredConsumer `json:"consumers"`
By string `json:"by"`
Why string `json:"why"`
At string `json:"at"`
}
// RetirementState is a provider's answer to provisioner_retirement.
type RetirementState struct {
Resource string `json:"resource"`
Node string `json:"node"`
Held []string `json:"held"`
StablePasses int `json:"stable_passes"`
Bound string `json:"bound"`
Waiting *RetirementWaiting `json:"waiting"`
Rejected *RetirementRejected `json:"rejected"`
Retired []RetiredConsumer `json:"retired"`
}
+2
View File
@@ -81,6 +81,8 @@ type Server struct {
replayer Replayer
// standings keeps what providers say about their consumers (novox/hq ADR 0224).
standings Standings
// retirements keeps what providers say about consumers the mesh stopped asking for (ADR 0230).
retirements Retirements
log *log.Logger
// giveUp is how long one message is held for the store; zero means GiveUpAfter.
+4
View File
@@ -81,6 +81,10 @@ func ReadStanding(subject string, body []byte) (Standing, error) {
// it lasts, so one dropped is replaced; a recovery is said once, and dropping it would leave status
// naming a consumer that is fine. So a store that is away holds the message, as a report is held.
func (s *Server) provisioner(ctx context.Context, m Control) {
if IsRetirement(m.Subject()) {
s.retirement(ctx, m)
return
}
if s.standings == nil {
// Delivered because the consumer's filter names it, with nothing here keeping it: taken,
// because handing it back would not give it anywhere to go.
+6 -2
View File
@@ -37,6 +37,7 @@ func TestTheControllerFollowsEveryProvidersStandingAndNothingElse(t *testing.T)
for subject, want := range map[string]string{
"mesh.mod.keycloak.event.provisioner.failing": "keycloak",
"mesh.mod.postgres.event.provisioner.recovered": "postgres",
"mesh.mod.minio.event.provisioner.retirement": "minio",
"mesh.mod.*.event.provisioner.failing": "*",
} {
got, ok := ProvisionerEmitter(subject)
@@ -63,8 +64,11 @@ func TestTheControllerFollowsEveryProvidersStandingAndNothingElse(t *testing.T)
follows++
}
}
if follows != 2 {
t.Fatalf("the controller follows %d standing subjects, want failing and recovered", follows)
if follows != 3 {
t.Fatalf("the controller follows %d provider subjects, want failing, recovered and retirement (ADR 0230)", follows)
}
if !IsRetirement("mesh.mod.postgres.event.provisioner.retirement") || IsRetirement("mesh.mod.postgres.event.provisioner.failing") {
t.Fatal("a retirement word is not told from a standing")
}
}