Hear what providers retire, and let a person approve, reject and delete (hq ADR 0230)
A provider now waits for a person before retiring more than three consumers or half of what it holds, and deletes only when asked. The controller is that person's way in: it keeps waiting and rejected sets as conditions, answers them with retire approve|reject, lists and deletes retired consumers through the provider's own tools on its machine, records each act in the hand-act log, and probes for anything retired longer than thirty days (D11).
This commit is contained in:
@@ -255,13 +255,18 @@ var ControllerFollows = []string{
|
||||
// the index is a name.
|
||||
moduleEventSubject("*", ProvisionerFailing),
|
||||
moduleEventSubject("*", ProvisionerRecovered),
|
||||
// **What becomes of a consumer the mesh stopped asking for** (novox/hq ADR 0230): retired, waiting
|
||||
// for a person, re-enabled, deleted — a provider's third word, from whichever module provides.
|
||||
// Appended, because the index is a name.
|
||||
moduleEventSubject("*", ProvisionerRetirement),
|
||||
}
|
||||
|
||||
// The provider standing events, by their local names. Written here as well as in the catalogue
|
||||
// (catalogue.ProvisionerEvents), which this package cannot import; a test keeps them agreeing.
|
||||
const (
|
||||
ProvisionerFailing = "provisioner.failing"
|
||||
ProvisionerRecovered = "provisioner.recovered"
|
||||
ProvisionerFailing = "provisioner.failing"
|
||||
ProvisionerRecovered = "provisioner.recovered"
|
||||
ProvisionerRetirement = "provisioner.retirement"
|
||||
)
|
||||
|
||||
// moduleEventSubject is where one module's event lands. The same derivation PermissionsFor uses, so
|
||||
|
||||
+1
-1
@@ -25,7 +25,7 @@ accounts {
|
||||
users = [
|
||||
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
|
||||
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*"] }
|
||||
subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] }
|
||||
subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] }
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
{ user: "enrol.one", password: "$2a$11$eeeeeeeeeeeeeeeeeeeeee", permissions: {
|
||||
|
||||
@@ -117,9 +117,10 @@ var WritersTable = []WriterRow{
|
||||
{State: "a merge announced", Writer: "one announcer per forge (the hook, or the poll when the hook is absent — never both)",
|
||||
KeptIn: "the bus", Others: "—", Subjects: []string{"mesh.mod.*.event.pull.merged"}, Writes: ownModule},
|
||||
{State: "a provider's standing", Writer: "the provider", KeptIn: "the provider's events",
|
||||
Others: "the controller keeps the newest word as a condition",
|
||||
Subjects: []string{"mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered"},
|
||||
Writes: ownModule},
|
||||
Others: "the controller keeps the newest word as a condition",
|
||||
Subjects: []string{"mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered",
|
||||
"mesh.mod.*.event.provisioner.retirement"},
|
||||
Writes: ownModule},
|
||||
{State: "the operator-channel's open messages", Writer: "the seat's holder", KeptIn: "its own key-value state",
|
||||
Others: "—"},
|
||||
{State: "the facts snapshot", Writer: "controller", KeptIn: "the artifact store, facts/latest",
|
||||
|
||||
@@ -55,7 +55,7 @@ func TestAWholeMeshComposesWithOneWriterPerState(t *testing.T) {
|
||||
{Kind: KindEnrolment, Node: "two", PasswordHash: "x"},
|
||||
{Kind: KindPerson, Module: "jochen", Invokes: []string{"*"}, PasswordHash: "x"},
|
||||
{Kind: KindModule, Node: "one", Module: "gitea", Emits: []string{"pull.merged"}, PasswordHash: "x"},
|
||||
{Kind: KindModule, Node: "one", Module: "postgres", Emits: []string{"provisioner.failing", "provisioner.recovered"},
|
||||
{Kind: KindModule, Node: "one", Module: "postgres", Emits: []string{"provisioner.failing", "provisioner.recovered", "provisioner.retirement"},
|
||||
PasswordHash: "x"},
|
||||
{Kind: KindModule, Node: "one", Module: "build-agent", Holds: []Seat{builder}, PasswordHash: "x"},
|
||||
{Kind: KindNodeTools, Node: "one", Module: RuntimeModule, PasswordHash: "x", Carries: []Declared{
|
||||
|
||||
@@ -164,13 +164,19 @@ func consumePattern(pattern string) error {
|
||||
// The events a provider says about its consumers (novox/hq ADR 0224): a consumer it has failed
|
||||
// without one success for minutes, and that consumer succeeding again or being withdrawn. The
|
||||
// controller follows them from every module and `status` names a consumer failing until it recovers.
|
||||
//
|
||||
// **And what becomes of a consumer the mesh stopped asking for** (novox/hq ADR 0230): retired — its
|
||||
// access disabled and its data kept — after the same answer in five passes, waiting for a person when
|
||||
// more would go than the bound allows, re-enabled when asked for again, and deleted only by a person's
|
||||
// `cleanup delete`. One event, its `change` saying which.
|
||||
const (
|
||||
ProvisionerFailing = "provisioner.failing"
|
||||
ProvisionerRecovered = "provisioner.recovered"
|
||||
ProvisionerFailing = "provisioner.failing"
|
||||
ProvisionerRecovered = "provisioner.recovered"
|
||||
ProvisionerRetirement = "provisioner.retirement"
|
||||
)
|
||||
|
||||
// ProvisionerEvents are both, in the order they are said.
|
||||
var ProvisionerEvents = []string{ProvisionerFailing, ProvisionerRecovered}
|
||||
// ProvisionerEvents are all three, in the order they are said.
|
||||
var ProvisionerEvents = []string{ProvisionerFailing, ProvisionerRecovered, ProvisionerRetirement}
|
||||
|
||||
// EmitsAll is every event a module may publish: what it declares and, for a module that receives
|
||||
// contributions — a provider, running a provisioner over them — the provider's standing events.
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Every provider may say what becomes of a consumer the mesh stopped asking for (novox/hq ADR 0230),
|
||||
// whatever its manifest lists — as it may say a consumer it keeps failing (ADR 0224) — and a module
|
||||
// that receives nothing may not.
|
||||
func TestEveryProviderMaySayWhatItRetires(t *testing.T) {
|
||||
provider := Manifest{Module: "pg", Receives: map[string]string{"postgres-database": "/x/mesh.json"},
|
||||
Emits: []string{"database.provisioned"}}
|
||||
for _, e := range []string{ProvisionerFailing, ProvisionerRecovered, ProvisionerRetirement, "database.provisioned"} {
|
||||
if !slices.Contains(provider.EmitsAll(), e) {
|
||||
t.Errorf("a provider may not emit %s: %v", e, provider.EmitsAll())
|
||||
}
|
||||
}
|
||||
if slices.Contains(Manifest{Module: "app", Emits: []string{"x"}}.EmitsAll(), ProvisionerRetirement) {
|
||||
t.Error("a module that provides nothing may say what it retires")
|
||||
}
|
||||
}
|
||||
@@ -267,6 +267,33 @@ var ControllerVerbs = []Verb{
|
||||
"probes": "\"true\": the registry — what each probe asserts, and the condition it raises",
|
||||
"signals": "\"true\": the signals table, each row with the age of its newest signal",
|
||||
}, nil, "run", "probes", "signals")},
|
||||
// A consumer the mesh stopped asking for: retired, waiting for a person, deleted only by one
|
||||
// (novox/hq ADR 0230).
|
||||
{Name: "retire", Description: "A consumer the mesh stops asking for is retired by its provider — access " +
|
||||
"disabled, data kept — after the same answer in five passes; more than three at once, or more than half " +
|
||||
"of those held, waits for a person. With no answer, every provider that waits and what it would retire. " +
|
||||
"With answer approve or reject, a node and a module: retire exactly what that provider waits with, or " +
|
||||
"keep it active — a hand act, which says why (novox/hq ADR 0230).",
|
||||
Input: schema(map[string]string{
|
||||
"answer": "approve or reject: answer what the provider waits with (needs node, module and why)",
|
||||
"node": "with answer: the machine the provider runs on",
|
||||
"module": "with answer: the provider module",
|
||||
"why": "with answer: why — required, and recorded in the hand-act log",
|
||||
"cause": "with answer: the cause in a word (retire-waiting when absent)",
|
||||
}, nil)},
|
||||
{Name: "cleanup", Description: "Every consumer a provider holds retired — its age, its size where the " +
|
||||
"backend knows, and why it was retired. With consumer (and node, module): the provider deletes that one " +
|
||||
"retired consumer — never an active one. With older-than: every retired consumer older than that many " +
|
||||
"days, listed; deleted only with confirm. Deleting is a hand act, which says why (novox/hq ADR 0230).",
|
||||
Input: schema(map[string]string{
|
||||
"node": "with consumer: the machine the provider runs on",
|
||||
"module": "with consumer: the provider module",
|
||||
"consumer": "delete this retired consumer (needs node, module and why)",
|
||||
"older-than": "delete every consumer retired more than this many days (needs why; lists only without confirm)",
|
||||
"confirm": "\"true\": with older-than, delete what is listed",
|
||||
"why": "with consumer or older-than: why — required, and recorded in the hand-act log",
|
||||
"cause": "with consumer or older-than: the cause in a word (cleanup-waiting when absent)",
|
||||
}, nil, "confirm")},
|
||||
{Name: "build", Description: "Have the build machine build a repository. Answers at once with the build's id: " +
|
||||
"`builds` with that id follows it line by line, and the module is registered when the outcome comes.",
|
||||
Input: schema(map[string]string{
|
||||
|
||||
@@ -42,6 +42,10 @@ var Contracts = map[string]Contract{
|
||||
"catalogue's record, which is the order, so a late one reads the same record"},
|
||||
KindCatchUp: {Unordered: "a catalogue asking what it missed: answered from the record, whenever asked"},
|
||||
KindProvisioner: {Unordered: "a provider's newest word about a consumer, said again every fifteen minutes " +
|
||||
"while it holds (ADR 0224): the condition keeps the last observed, and S8 says when the words stop",
|
||||
Tests: []string{"TestAProviderFailingAConsumerBreaksAllWellUntilItRecovers"}},
|
||||
"while it holds (ADR 0224): the condition keeps the last observed, and S8 says when the words stop. " +
|
||||
"Its retirement word (ADR 0230) is unordered too: a waiting set is said again every fifteen minutes, " +
|
||||
"and every other change is checked against the provider itself — `retire` and `cleanup` ask it, " +
|
||||
"and the self-check's D11 asks it every run — so an older word read late is corrected by the next",
|
||||
Tests: []string{"TestAProviderFailingAConsumerBreaksAllWellUntilItRecovers",
|
||||
"TestARetirementWordIsKeptAsItsConditionNamingTheEmitterFromTheSubject"}},
|
||||
}
|
||||
|
||||
@@ -262,7 +262,7 @@ func kindOfSubject(subject string) (string, bool) {
|
||||
}
|
||||
|
||||
// ProvisionerEmitter is the module a provider's standing event came from, read from its subject
|
||||
// (`mesh.mod.<module>.event.provisioner.<failing|recovered>`); false for any other subject. The
|
||||
// (`mesh.mod.<module>.event.provisioner.<failing|recovered|retirement>`); false for any other subject. The
|
||||
// controller's own follow pattern, with `*` for the module, decodes too.
|
||||
func ProvisionerEmitter(subject string) (string, bool) {
|
||||
rest, ok := strings.CutPrefix(subject, "mesh.mod.")
|
||||
@@ -273,7 +273,8 @@ func ProvisionerEmitter(subject string) (string, bool) {
|
||||
if !ok || module == "" || strings.Contains(module, ".") {
|
||||
return "", false
|
||||
}
|
||||
if event != broker.ProvisionerFailing && event != broker.ProvisionerRecovered {
|
||||
if event != broker.ProvisionerFailing && event != broker.ProvisionerRecovered &&
|
||||
event != broker.ProvisionerRetirement {
|
||||
return "", false
|
||||
}
|
||||
return module, true
|
||||
|
||||
@@ -0,0 +1,241 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// What becomes of a consumer the mesh stopped asking for (novox/hq ADR 0230).
|
||||
//
|
||||
// **A consumer is retired, not withdrawn, and deleted only by a person.** A provider that has seen the
|
||||
// same consumers go unasked for in five passes disables their access and keeps their data, marked in
|
||||
// its backend with when and why; asked for again, it enables them as they were. When more would go
|
||||
// than its bound allows — more than three, or more than half of those it holds — it retires nothing and
|
||||
// waits for `retire approve` or `retire reject`. A retired consumer is deleted only by `cleanup delete`,
|
||||
// which the provider executes. Each of these is the provider's `provisioner.retirement` event, its
|
||||
// `change` saying which; the controller keeps the ones that need a person as conditions.
|
||||
|
||||
// The changes a provider says.
|
||||
const (
|
||||
RetireWaiting = "waiting"
|
||||
RetireSettled = "settled"
|
||||
RetireApproved = "approved"
|
||||
RetireRejected = "rejected"
|
||||
RetireRetired = "retired"
|
||||
RetireReenabled = "reenabled"
|
||||
RetireDeleted = "deleted"
|
||||
RetireAdopted = "adopted"
|
||||
)
|
||||
|
||||
// ViaController is what the controller's verbs say they came through, so an act a provider was asked
|
||||
// for some other way is told apart and recorded by hand.
|
||||
const ViaController = "mesh-controller"
|
||||
|
||||
// RetiredConsumer is one consumer in a retirement word, or in a provider's answer.
|
||||
type RetiredConsumer struct {
|
||||
Consumer string `json:"consumer"`
|
||||
Node string `json:"node,omitempty"`
|
||||
RetiredAt string `json:"retired_at,omitempty"`
|
||||
Why string `json:"why,omitempty"`
|
||||
// SizeBytes is what it keeps on the backend; -1 or absent where the backend cannot say.
|
||||
SizeBytes *int64 `json:"size_bytes,omitempty"`
|
||||
// Kind is "consumer", or a backend's own word for something set aside ("set-aside-database").
|
||||
Kind string `json:"kind,omitempty"`
|
||||
}
|
||||
|
||||
// Retirement is one provider's word about consumers the mesh stopped asking for.
|
||||
type Retirement struct {
|
||||
// Module is the emitter, read from the subject the bus let it publish on — never from the body.
|
||||
Module string `json:"-"`
|
||||
|
||||
Provider string `json:"provider"`
|
||||
ProviderNode string `json:"provider-node"`
|
||||
Change string `json:"change"`
|
||||
Consumers []RetiredConsumer `json:"consumers"`
|
||||
Held int `json:"held"`
|
||||
Bound string `json:"bound,omitempty"`
|
||||
Why string `json:"why,omitempty"`
|
||||
By string `json:"by,omitempty"`
|
||||
Via string `json:"via,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
Since time.Time `json:"since,omitempty"`
|
||||
}
|
||||
|
||||
// Names are the consumers it names, in its order.
|
||||
func (r Retirement) Names() []string {
|
||||
out := make([]string, 0, len(r.Consumers))
|
||||
for _, c := range r.Consumers {
|
||||
out = append(out, c.Consumer)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Retirements keeps what providers say about consumers they retire.
|
||||
type Retirements interface {
|
||||
// Retired records one word. An error the store is away for is held and asked again, like a report.
|
||||
Retired(ctx context.Context, r Retirement) error
|
||||
}
|
||||
|
||||
// KeepsRetirements says where retirement words are kept, and asks for them to be delivered.
|
||||
func (s *Server) KeepsRetirements(r Retirements) error {
|
||||
if err := s.inbound.Also(KindProvisioner); err != nil {
|
||||
return err
|
||||
}
|
||||
s.retirements = r
|
||||
return nil
|
||||
}
|
||||
|
||||
// IsRetirement says a subject is a provider's retirement word.
|
||||
func IsRetirement(subject string) bool {
|
||||
_, ok := ProvisionerEmitter(subject)
|
||||
return ok && strings.HasSuffix(subject, ".event."+broker.ProvisionerRetirement)
|
||||
}
|
||||
|
||||
// ReadRetirement is one retirement word as the controller understands it.
|
||||
func ReadRetirement(subject string, body []byte) (Retirement, error) {
|
||||
module, ok := ProvisionerEmitter(subject)
|
||||
if !ok || !IsRetirement(subject) {
|
||||
return Retirement{}, fmt.Errorf("%s is not a provider's retirement word", subject)
|
||||
}
|
||||
var r Retirement
|
||||
if err := json.Unmarshal(body, &r); err != nil {
|
||||
return Retirement{}, fmt.Errorf("%s's retirement word could not be read: %w", module, err)
|
||||
}
|
||||
switch r.Change {
|
||||
case RetireWaiting, RetireSettled, RetireApproved, RetireRejected, RetireRetired, RetireReenabled,
|
||||
RetireDeleted, RetireAdopted:
|
||||
default:
|
||||
return Retirement{}, fmt.Errorf("%s said a retirement change the mesh has no name for: %q", module, r.Change)
|
||||
}
|
||||
if r.ProviderNode == "" {
|
||||
return Retirement{}, fmt.Errorf("%s's retirement word named no machine", module)
|
||||
}
|
||||
r.Module = module
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// retirement acts on one retirement word. Like a recovery, a word that clears a condition is said
|
||||
// once, so a store that is away holds the message rather than dropping it.
|
||||
func (s *Server) retirement(ctx context.Context, m Control) {
|
||||
if s.retirements == nil {
|
||||
_ = m.Took()
|
||||
return
|
||||
}
|
||||
r, err := ReadRetirement(m.Subject(), m.Body())
|
||||
if err != nil {
|
||||
s.log.Printf("%v; ignored", err)
|
||||
_ = m.Took()
|
||||
return
|
||||
}
|
||||
err = s.retirements.Retired(ctx, r)
|
||||
what := fmt.Sprintf("%s's retirement word (%s)", r.Module, r.Change)
|
||||
switch s.decide(ctx, m, what, "", "", err) {
|
||||
case Hold:
|
||||
return
|
||||
case Stale, GiveUp:
|
||||
_ = m.Took()
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
s.log.Printf("%s could not be kept: %v", what, err)
|
||||
} else {
|
||||
s.log.Printf("%s on %s %s %s", r.Module, r.ProviderNode, r.Change, strings.Join(r.Names(), ", "))
|
||||
}
|
||||
_ = m.Took()
|
||||
}
|
||||
|
||||
// The tools every provider serves about its retired consumers (ADR 0230), asked of one machine.
|
||||
const (
|
||||
ToolRetirement = "provisioner_retirement"
|
||||
ToolRetireApprove = "provisioner_retire_approve"
|
||||
ToolRetireReject = "provisioner_retire_reject"
|
||||
ToolRetiredDelete = "provisioner_delete"
|
||||
)
|
||||
|
||||
// ErrNothingServes is a tool nothing on that machine serves: the module is not running there, or
|
||||
// is older than the tool.
|
||||
var ErrNothingServes = errors.New("nothing serves it")
|
||||
|
||||
// ModuleToolOn is where one machine's instance of a module answers a tool: the module's tool subject
|
||||
// with the machine as its last token — the subject the node tools bind beside the shared one.
|
||||
func ModuleToolOn(module, tool, node string) string { return ToolSubject(module, tool) + "." + node }
|
||||
|
||||
// AskModuleToolOn asks one machine's instance of a module one of its tools and reads its answer.
|
||||
func AskModuleToolOn(ctx context.Context, conn *nats.Conn, module, tool, node string, args any,
|
||||
timeout time.Duration) (Answer, error) {
|
||||
body, err := json.Marshal(args)
|
||||
if err != nil {
|
||||
return Answer{}, err
|
||||
}
|
||||
asking, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
subject := ModuleToolOn(module, tool, node)
|
||||
refused, stop := refusalsOf(conn, subject)
|
||||
defer stop()
|
||||
type replied struct {
|
||||
msg *nats.Msg
|
||||
err error
|
||||
}
|
||||
done := make(chan replied, 1)
|
||||
go func() {
|
||||
msg, err := conn.RequestWithContext(asking, subject, body)
|
||||
done <- replied{msg, err}
|
||||
}()
|
||||
var reply *nats.Msg
|
||||
select {
|
||||
case r := <-done:
|
||||
reply, err = r.msg, r.err
|
||||
case why := <-refused:
|
||||
cancel()
|
||||
return Answer{}, fmt.Errorf("the bus refused the controller asking %s.%s on %s: %v", module, tool, node, why)
|
||||
}
|
||||
switch {
|
||||
case errors.Is(err, nats.ErrNoResponders):
|
||||
return Answer{}, fmt.Errorf("%w: %s on %s does not answer %s — it is not running there, or is "+
|
||||
"older than the tool", ErrNothingServes, module, node, tool)
|
||||
case errors.Is(err, context.DeadlineExceeded), errors.Is(err, nats.ErrTimeout):
|
||||
return Answer{}, fmt.Errorf("%s on %s did not answer %s within %s", module, node, tool, timeout)
|
||||
case err != nil:
|
||||
return Answer{}, err
|
||||
}
|
||||
var answer Answer
|
||||
if err := json.Unmarshal(reply.Data, &answer); err != nil {
|
||||
return Answer{}, fmt.Errorf("%s on %s answered %s with something unreadable: %w", module, node, tool, err)
|
||||
}
|
||||
return answer, nil
|
||||
}
|
||||
|
||||
// RetirementWaiting is the set a provider waits with for a person.
|
||||
type RetirementWaiting struct {
|
||||
Consumers []RetiredConsumer `json:"consumers"`
|
||||
Since string `json:"since"`
|
||||
Held int `json:"held"`
|
||||
}
|
||||
|
||||
// RetirementRejected is the set a person's rejection keeps active.
|
||||
type RetirementRejected struct {
|
||||
Consumers []RetiredConsumer `json:"consumers"`
|
||||
By string `json:"by"`
|
||||
Why string `json:"why"`
|
||||
At string `json:"at"`
|
||||
}
|
||||
|
||||
// RetirementState is a provider's answer to provisioner_retirement.
|
||||
type RetirementState struct {
|
||||
Resource string `json:"resource"`
|
||||
Node string `json:"node"`
|
||||
Held []string `json:"held"`
|
||||
StablePasses int `json:"stable_passes"`
|
||||
Bound string `json:"bound"`
|
||||
Waiting *RetirementWaiting `json:"waiting"`
|
||||
Rejected *RetirementRejected `json:"rejected"`
|
||||
Retired []RetiredConsumer `json:"retired"`
|
||||
}
|
||||
@@ -81,6 +81,8 @@ type Server struct {
|
||||
replayer Replayer
|
||||
// standings keeps what providers say about their consumers (novox/hq ADR 0224).
|
||||
standings Standings
|
||||
// retirements keeps what providers say about consumers the mesh stopped asking for (ADR 0230).
|
||||
retirements Retirements
|
||||
|
||||
log *log.Logger
|
||||
// giveUp is how long one message is held for the store; zero means GiveUpAfter.
|
||||
|
||||
@@ -81,6 +81,10 @@ func ReadStanding(subject string, body []byte) (Standing, error) {
|
||||
// it lasts, so one dropped is replaced; a recovery is said once, and dropping it would leave status
|
||||
// naming a consumer that is fine. So a store that is away holds the message, as a report is held.
|
||||
func (s *Server) provisioner(ctx context.Context, m Control) {
|
||||
if IsRetirement(m.Subject()) {
|
||||
s.retirement(ctx, m)
|
||||
return
|
||||
}
|
||||
if s.standings == nil {
|
||||
// Delivered because the consumer's filter names it, with nothing here keeping it: taken,
|
||||
// because handing it back would not give it anywhere to go.
|
||||
|
||||
@@ -37,6 +37,7 @@ func TestTheControllerFollowsEveryProvidersStandingAndNothingElse(t *testing.T)
|
||||
for subject, want := range map[string]string{
|
||||
"mesh.mod.keycloak.event.provisioner.failing": "keycloak",
|
||||
"mesh.mod.postgres.event.provisioner.recovered": "postgres",
|
||||
"mesh.mod.minio.event.provisioner.retirement": "minio",
|
||||
"mesh.mod.*.event.provisioner.failing": "*",
|
||||
} {
|
||||
got, ok := ProvisionerEmitter(subject)
|
||||
@@ -63,8 +64,11 @@ func TestTheControllerFollowsEveryProvidersStandingAndNothingElse(t *testing.T)
|
||||
follows++
|
||||
}
|
||||
}
|
||||
if follows != 2 {
|
||||
t.Fatalf("the controller follows %d standing subjects, want failing and recovered", follows)
|
||||
if follows != 3 {
|
||||
t.Fatalf("the controller follows %d provider subjects, want failing, recovered and retirement (ADR 0230)", follows)
|
||||
}
|
||||
if !IsRetirement("mesh.mod.postgres.event.provisioner.retirement") || IsRetirement("mesh.mod.postgres.event.provisioner.failing") {
|
||||
t.Fatal("a retirement word is not told from a standing")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user