Hear what providers retire, and let a person approve, reject and delete (hq ADR 0230)
A provider now waits for a person before retiring more than three consumers or half of what it holds, and deletes only when asked. The controller is that person's way in: it keeps waiting and rejected sets as conditions, answers them with retire approve|reject, lists and deletes retired consumers through the provider's own tools on its machine, records each act in the hand-act log, and probes for anything retired longer than thirty days (D11).
This commit is contained in:
jochen
@@ -0,0 +1,434 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The verbs a person answers a provider's retirement with, and cleans up with (novox/hq ADR 0230).
|
||||
//
|
||||
// **The controller asks; the provider acts.** Approving, rejecting and deleting are each a question to
|
||||
// the provider on the machine it runs on — its `provisioner_*` tools — because the provider owns its
|
||||
// backend and the controller touches none. Every one says why, is written in the hand-act log before
|
||||
// it is asked, and the provider says what it did as its `provisioner.retirement` event.
|
||||
|
||||
const retireUsage = "retire [--json] | retire approve <node> <module> --why <text> | retire reject <node> <module> --why <text>"
|
||||
|
||||
const cleanupUsage = "cleanup [list] [--json] | cleanup delete <node> <module> <consumer> --why <text> | " +
|
||||
"cleanup delete --older-than <days> --why <text> [--confirm]"
|
||||
|
||||
func isNothingServes(err error) bool { return errors.Is(err, link.ErrNothingServes) }
|
||||
|
||||
func unmarshalAnswer(a link.Answer, v any) error {
|
||||
if len(a.Result) == 0 {
|
||||
return errors.New("an empty answer")
|
||||
}
|
||||
return json.Unmarshal(a.Result, v)
|
||||
}
|
||||
|
||||
// retireCommand is `retire`, `retire approve` and `retire reject`.
|
||||
func retireCommand(ctx context.Context, args []string) error {
|
||||
sub := "list"
|
||||
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
||||
sub, args = args[0], args[1:]
|
||||
}
|
||||
switch sub {
|
||||
case "list":
|
||||
set := flag.NewFlagSet("retire", flag.ContinueOnError)
|
||||
asJSON := set.Bool("json", false, "as data")
|
||||
if rest, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
} else if len(rest) > 0 {
|
||||
return errors.New(retireUsage)
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
return onTheBus(func(conn *nats.Conn) error { return listRetiring(ctx, open.inventory, conn, *asJSON) })
|
||||
case "approve", "reject":
|
||||
set := flag.NewFlagSet("retire "+sub, flag.ContinueOnError)
|
||||
f := addHandActFlags(set)
|
||||
rest, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(rest) != 2 {
|
||||
return errors.New(retireUsage)
|
||||
}
|
||||
if err := f.require("retire " + sub); err != nil {
|
||||
return err
|
||||
}
|
||||
return onTheBus(func(conn *nats.Conn) error {
|
||||
return answerRetirement(ctx, conn, providerInstance{Node: rest[0], Module: rest[1]}, sub == "approve", f)
|
||||
})
|
||||
}
|
||||
return errors.New(retireUsage)
|
||||
}
|
||||
|
||||
// retiringRow is one provider's waiting or rejected set, as `retire` lists it.
|
||||
type retiringRow struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Waiting []link.RetiredConsumer `json:"waiting,omitempty"`
|
||||
Since string `json:"since,omitempty"`
|
||||
Held int `json:"held,omitempty"`
|
||||
Bound string `json:"bound,omitempty"`
|
||||
Rejected []link.RetiredConsumer `json:"rejected,omitempty"`
|
||||
RejectWhy string `json:"rejected-why,omitempty"`
|
||||
Unasked string `json:"unasked,omitempty"`
|
||||
}
|
||||
|
||||
func listRetiring(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn, asJSON bool) error {
|
||||
instances, err := providerInstances(ctx, inv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var rows []retiringRow
|
||||
for _, p := range instances {
|
||||
row := retiringRow{Node: p.Node, Module: p.Module}
|
||||
state, err := askRetirement(ctx, conn, p)
|
||||
switch {
|
||||
case isNothingServes(err):
|
||||
row.Unasked = "answers no retirement question: it predates ADR 0230"
|
||||
case err != nil:
|
||||
row.Unasked = err.Error()
|
||||
default:
|
||||
if state.Waiting != nil {
|
||||
row.Waiting, row.Since, row.Held = state.Waiting.Consumers, state.Waiting.Since, state.Waiting.Held
|
||||
}
|
||||
if state.Rejected != nil {
|
||||
row.Rejected, row.RejectWhy = state.Rejected.Consumers, orWhy(state.Rejected.By, state.Rejected.Why)
|
||||
}
|
||||
row.Bound = state.Bound
|
||||
if row.Waiting == nil && row.Rejected == nil {
|
||||
continue
|
||||
}
|
||||
}
|
||||
rows = append(rows, row)
|
||||
}
|
||||
if asJSON {
|
||||
if rows == nil {
|
||||
rows = []retiringRow{}
|
||||
}
|
||||
return printJSON(map[string]any{"providers": rows})
|
||||
}
|
||||
said := false
|
||||
for _, r := range rows {
|
||||
switch {
|
||||
case r.Unasked != "":
|
||||
fmt.Printf("%s on %s: not asked — %s\n", r.Module, r.Node, r.Unasked)
|
||||
default:
|
||||
if r.Waiting != nil {
|
||||
said = true
|
||||
fmt.Printf("%s on %s WAITS since %s to retire %d of the %d it holds (%s): %s\n"+
|
||||
" retire approve %s %s --why … | retire reject %s %s --why …\n",
|
||||
r.Module, r.Node, r.Since, len(r.Waiting), r.Held, orBound(r.Bound), consumerList(r.Waiting),
|
||||
r.Node, r.Module, r.Node, r.Module)
|
||||
}
|
||||
if r.Rejected != nil {
|
||||
said = true
|
||||
fmt.Printf("%s on %s keeps active, by a rejection (%s): %s\n", r.Module, r.Node, r.RejectWhy,
|
||||
consumerList(r.Rejected))
|
||||
}
|
||||
}
|
||||
}
|
||||
if !said {
|
||||
fmt.Println("no provider waits for a person to approve a retirement")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// answerRetirement approves or rejects what one provider waits with. **The set sent is the set the
|
||||
// provider says it waits with, read now**, and the provider refuses any other: a person approves what
|
||||
// they were shown, never a set that moved since.
|
||||
func answerRetirement(ctx context.Context, conn *nats.Conn, p providerInstance, approve bool, f handActFlags) error {
|
||||
state, err := askRetirement(ctx, conn, p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
verb, tool := "retire reject", link.ToolRetireReject
|
||||
var set []link.RetiredConsumer
|
||||
if state.Waiting != nil {
|
||||
set = state.Waiting.Consumers
|
||||
}
|
||||
if approve {
|
||||
verb, tool = "retire approve", link.ToolRetireApprove
|
||||
if set == nil && state.Rejected != nil {
|
||||
// A rejection can be taken back: the consumers it kept are retired after all.
|
||||
set = state.Rejected.Consumers
|
||||
}
|
||||
}
|
||||
if len(set) == 0 {
|
||||
return fmt.Errorf("%s on %s waits for nobody to approve or reject a retirement. Nothing was done", p.Module, p.Node)
|
||||
}
|
||||
names := make([]string, 0, len(set))
|
||||
for _, c := range set {
|
||||
names = append(names, c.Consumer)
|
||||
}
|
||||
if strings.TrimSpace(*f.cause) == "" {
|
||||
*f.cause = kindRetireWaiting
|
||||
}
|
||||
if strings.TrimSpace(*f.condition) == "" {
|
||||
*f.condition = retireWaitingKey(p.Module, p.Node)
|
||||
}
|
||||
f.record(ctx, verb, append([]string{p.Node, p.Module}, names...))
|
||||
answer, err := link.AskModuleToolOn(ctx, conn, p.Module, tool, p.Node, map[string]any{
|
||||
"consumers": names, "why": strings.TrimSpace(*f.why), "by": link.Caller(), "via": link.ViaController,
|
||||
}, retirementAsk)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return fmt.Errorf("%s on %s refused: %s", p.Module, p.Node, answer.Error)
|
||||
}
|
||||
if approve {
|
||||
fmt.Printf("%s on %s retired %s: access disabled, data kept; `cleanup list` shows them\n", p.Module, p.Node,
|
||||
strings.Join(names, ", "))
|
||||
} else {
|
||||
fmt.Printf("%s on %s keeps %s active; the warning stays open until the mesh asks for them again or the "+
|
||||
"retirement is approved\n", p.Module, p.Node, strings.Join(names, ", "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// cleanupCommand is `cleanup list` and `cleanup delete`.
|
||||
func cleanupCommand(ctx context.Context, args []string) error {
|
||||
sub := "list"
|
||||
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
||||
sub, args = args[0], args[1:]
|
||||
}
|
||||
switch sub {
|
||||
case "list":
|
||||
set := flag.NewFlagSet("cleanup", flag.ContinueOnError)
|
||||
asJSON := set.Bool("json", false, "as data")
|
||||
if rest, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
} else if len(rest) > 0 {
|
||||
return errors.New(cleanupUsage)
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
return onTheBus(func(conn *nats.Conn) error {
|
||||
listing, err := gatherRetired(ctx, open.inventory, conn, time.Now())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return printRetired(listing, *asJSON)
|
||||
})
|
||||
case "delete":
|
||||
set := flag.NewFlagSet("cleanup delete", flag.ContinueOnError)
|
||||
f := addHandActFlags(set)
|
||||
olderThan := set.Int("older-than", 0, "every retired consumer older than this many days")
|
||||
confirm := set.Bool("confirm", false, "with --older-than: delete what is listed, rather than only list it")
|
||||
rest, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := f.require("cleanup delete"); err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.TrimSpace(*f.cause) == "" {
|
||||
*f.cause = kindCleanupWaiting
|
||||
}
|
||||
switch {
|
||||
case *olderThan > 0 && len(rest) == 0:
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
return onTheBus(func(conn *nats.Conn) error {
|
||||
return deleteOlderThan(ctx, open.inventory, conn, *olderThan, *confirm, f, time.Now())
|
||||
})
|
||||
case *olderThan == 0 && len(rest) == 3 && !*confirm:
|
||||
return onTheBus(func(conn *nats.Conn) error {
|
||||
return deleteRetired(ctx, conn, providerInstance{Node: rest[0], Module: rest[1]}, rest[2], f)
|
||||
})
|
||||
}
|
||||
return errors.New(cleanupUsage)
|
||||
}
|
||||
return errors.New(cleanupUsage)
|
||||
}
|
||||
|
||||
// retiredRow is one retired consumer, as `cleanup list` shows it.
|
||||
type retiredRow struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Consumer string `json:"consumer"`
|
||||
// ConsumerNode is where the consumer was, when the provider knows.
|
||||
ConsumerNode string `json:"consumer-node,omitempty"`
|
||||
Kind string `json:"kind,omitempty"`
|
||||
RetiredAt string `json:"retired-at,omitempty"`
|
||||
// AgeDays is whole days since it was retired; -1 when the provider could not say when.
|
||||
AgeDays int `json:"age-days"`
|
||||
SizeBytes *int64 `json:"size-bytes,omitempty"`
|
||||
Why string `json:"why,omitempty"`
|
||||
}
|
||||
|
||||
// retiredListing is every provider's retired consumers, and the providers that could not say.
|
||||
type retiredListing struct {
|
||||
Retired []retiredRow `json:"retired"`
|
||||
// Unasked names each provider not asked, and why: one older than the question, or one that failed.
|
||||
Unasked []string `json:"unasked,omitempty"`
|
||||
}
|
||||
|
||||
func gatherRetired(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn, now time.Time) (retiredListing, error) {
|
||||
instances, err := providerInstances(ctx, inv)
|
||||
if err != nil {
|
||||
return retiredListing{}, err
|
||||
}
|
||||
return retiredOf(ctx, conn, instances, now), nil
|
||||
}
|
||||
|
||||
func retiredOf(ctx context.Context, conn *nats.Conn, instances []providerInstance, now time.Time) retiredListing {
|
||||
out := retiredListing{Retired: []retiredRow{}}
|
||||
for _, p := range instances {
|
||||
state, err := askRetirement(ctx, conn, p)
|
||||
if err != nil {
|
||||
if isNothingServes(err) {
|
||||
out.Unasked = append(out.Unasked, fmt.Sprintf("%s on %s answers no retirement question: it predates ADR 0230", p.Module, p.Node))
|
||||
} else {
|
||||
out.Unasked = append(out.Unasked, err.Error())
|
||||
}
|
||||
continue
|
||||
}
|
||||
for _, c := range state.Retired {
|
||||
row := retiredRow{Node: p.Node, Module: p.Module, Consumer: c.Consumer, ConsumerNode: c.Node, Kind: c.Kind,
|
||||
RetiredAt: c.RetiredAt, AgeDays: -1, SizeBytes: c.SizeBytes, Why: c.Why}
|
||||
if at := retiredAt(c); !at.IsZero() {
|
||||
row.AgeDays = int(now.Sub(at).Hours() / 24)
|
||||
}
|
||||
out.Retired = append(out.Retired, row)
|
||||
}
|
||||
}
|
||||
sort.SliceStable(out.Retired, func(i, j int) bool { return out.Retired[i].AgeDays > out.Retired[j].AgeDays })
|
||||
return out
|
||||
}
|
||||
|
||||
func printRetired(l retiredListing, asJSON bool) error {
|
||||
if asJSON {
|
||||
return printJSON(l)
|
||||
}
|
||||
if len(l.Retired) == 0 {
|
||||
fmt.Println("no provider holds a retired consumer")
|
||||
}
|
||||
for _, r := range l.Retired {
|
||||
age := "age unknown"
|
||||
if r.AgeDays >= 0 {
|
||||
age = strconv.Itoa(r.AgeDays) + " day(s)"
|
||||
}
|
||||
kind := ""
|
||||
if r.Kind != "" && r.Kind != "consumer" {
|
||||
kind = " [" + r.Kind + "]"
|
||||
}
|
||||
fmt.Printf("%s on %s: %s%s — retired %s, %s, %s\n %s\n", r.Module, r.Node, r.Consumer, kind, age,
|
||||
sizeWords(r.SizeBytes), r.RetiredAt, orWhy("", r.Why))
|
||||
}
|
||||
for _, u := range l.Unasked {
|
||||
fmt.Printf("not asked: %s\n", u)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// deleteRetired asks one provider to delete one consumer it holds retired — never an active one: the
|
||||
// provider refuses that, and this refuses it first, from what the provider says it holds.
|
||||
func deleteRetired(ctx context.Context, conn *nats.Conn, p providerInstance, consumer string, f handActFlags) error {
|
||||
state, err := askRetirement(ctx, conn, p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
found := false
|
||||
for _, c := range state.Retired {
|
||||
found = found || c.Consumer == consumer
|
||||
}
|
||||
if !found {
|
||||
return fmt.Errorf("%s on %s holds no retired consumer %s — only a retired consumer is deleted. Nothing was done",
|
||||
p.Module, p.Node, consumer)
|
||||
}
|
||||
f.record(ctx, "cleanup delete", []string{p.Node, p.Module, consumer})
|
||||
answer, err := link.AskModuleToolOn(ctx, conn, p.Module, link.ToolRetiredDelete, p.Node, map[string]any{
|
||||
"consumer": consumer, "confirm": consumer, "why": strings.TrimSpace(*f.why), "by": link.Caller(),
|
||||
"via": link.ViaController,
|
||||
}, retirementAsk)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return fmt.Errorf("%s on %s refused to delete %s: %s", p.Module, p.Node, consumer, answer.Error)
|
||||
}
|
||||
var done struct {
|
||||
FreedBytes *int64 `json:"freed_bytes"`
|
||||
}
|
||||
_ = unmarshalAnswer(answer, &done)
|
||||
fmt.Printf("%s on %s deleted %s (%s freed)\n", p.Module, p.Node, consumer, sizeWords(done.FreedBytes))
|
||||
return nil
|
||||
}
|
||||
|
||||
// deleteOlderThan lists every consumer retired more than days ago, and deletes them only with confirm.
|
||||
// One whose age the provider cannot say is never in it.
|
||||
func deleteOlderThan(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn, days int, confirm bool,
|
||||
f handActFlags, now time.Time) error {
|
||||
listing, err := gatherRetired(ctx, inv, conn, now)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return deleteFrom(ctx, conn, listing, days, confirm, f)
|
||||
}
|
||||
|
||||
func deleteFrom(ctx context.Context, conn *nats.Conn, listing retiredListing, days int, confirm bool, f handActFlags) error {
|
||||
var due []retiredRow
|
||||
unknown := 0
|
||||
for _, r := range listing.Retired {
|
||||
switch {
|
||||
case r.AgeDays < 0:
|
||||
unknown++
|
||||
case r.AgeDays > days:
|
||||
due = append(due, r)
|
||||
}
|
||||
}
|
||||
for _, u := range listing.Unasked {
|
||||
fmt.Printf("not asked: %s\n", u)
|
||||
}
|
||||
if unknown > 0 {
|
||||
fmt.Printf("%d retired consumer(s) whose age their provider cannot say are left out\n", unknown)
|
||||
}
|
||||
if len(due) == 0 {
|
||||
fmt.Printf("nothing has been retired more than %d day(s)\n", days)
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("retired more than %d day(s):\n", days)
|
||||
for _, r := range due {
|
||||
fmt.Printf(" %s on %s: %s — %d day(s), %s\n", r.Module, r.Node, r.Consumer, r.AgeDays, sizeWords(r.SizeBytes))
|
||||
}
|
||||
if !confirm {
|
||||
fmt.Printf("nothing was deleted: add --confirm to delete these %d\n", len(due))
|
||||
return nil
|
||||
}
|
||||
var failed []string
|
||||
for _, r := range due {
|
||||
if err := deleteRetired(ctx, conn, providerInstance{Node: r.Node, Module: r.Module}, r.Consumer, f); err != nil {
|
||||
failed = append(failed, err.Error())
|
||||
}
|
||||
}
|
||||
if len(failed) > 0 {
|
||||
return fmt.Errorf("%d of %d not deleted: %s", len(failed), len(due), strings.Join(failed, "; "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user