jochen
68009b16fe
A provider now waits for a person before retiring more than three consumers or half of what it holds, and deletes only when asked. The controller is that person's way in: it keeps waiting and rejected sets as conditions, answers them with retire approve|reject, lists and deletes retired consumers through the provider's own tools on its machine, records each act in the hand-act log, and probes for anything retired longer than thirty days (D11).
435 lines
14 KiB
Go
435 lines
14 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"flag"
|
|
"fmt"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/nats-io/nats.go"
|
|
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
)
|
|
|
|
// The verbs a person answers a provider's retirement with, and cleans up with (novox/hq ADR 0230).
|
|
//
|
|
// **The controller asks; the provider acts.** Approving, rejecting and deleting are each a question to
|
|
// the provider on the machine it runs on — its `provisioner_*` tools — because the provider owns its
|
|
// backend and the controller touches none. Every one says why, is written in the hand-act log before
|
|
// it is asked, and the provider says what it did as its `provisioner.retirement` event.
|
|
|
|
const retireUsage = "retire [--json] | retire approve <node> <module> --why <text> | retire reject <node> <module> --why <text>"
|
|
|
|
const cleanupUsage = "cleanup [list] [--json] | cleanup delete <node> <module> <consumer> --why <text> | " +
|
|
"cleanup delete --older-than <days> --why <text> [--confirm]"
|
|
|
|
func isNothingServes(err error) bool { return errors.Is(err, link.ErrNothingServes) }
|
|
|
|
func unmarshalAnswer(a link.Answer, v any) error {
|
|
if len(a.Result) == 0 {
|
|
return errors.New("an empty answer")
|
|
}
|
|
return json.Unmarshal(a.Result, v)
|
|
}
|
|
|
|
// retireCommand is `retire`, `retire approve` and `retire reject`.
|
|
func retireCommand(ctx context.Context, args []string) error {
|
|
sub := "list"
|
|
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
|
sub, args = args[0], args[1:]
|
|
}
|
|
switch sub {
|
|
case "list":
|
|
set := flag.NewFlagSet("retire", flag.ContinueOnError)
|
|
asJSON := set.Bool("json", false, "as data")
|
|
if rest, err := parseAround(set, args); err != nil {
|
|
return err
|
|
} else if len(rest) > 0 {
|
|
return errors.New(retireUsage)
|
|
}
|
|
open, err := openStores(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer open.Close()
|
|
return onTheBus(func(conn *nats.Conn) error { return listRetiring(ctx, open.inventory, conn, *asJSON) })
|
|
case "approve", "reject":
|
|
set := flag.NewFlagSet("retire "+sub, flag.ContinueOnError)
|
|
f := addHandActFlags(set)
|
|
rest, err := parseAround(set, args)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(rest) != 2 {
|
|
return errors.New(retireUsage)
|
|
}
|
|
if err := f.require("retire " + sub); err != nil {
|
|
return err
|
|
}
|
|
return onTheBus(func(conn *nats.Conn) error {
|
|
return answerRetirement(ctx, conn, providerInstance{Node: rest[0], Module: rest[1]}, sub == "approve", f)
|
|
})
|
|
}
|
|
return errors.New(retireUsage)
|
|
}
|
|
|
|
// retiringRow is one provider's waiting or rejected set, as `retire` lists it.
|
|
type retiringRow struct {
|
|
Node string `json:"node"`
|
|
Module string `json:"module"`
|
|
Waiting []link.RetiredConsumer `json:"waiting,omitempty"`
|
|
Since string `json:"since,omitempty"`
|
|
Held int `json:"held,omitempty"`
|
|
Bound string `json:"bound,omitempty"`
|
|
Rejected []link.RetiredConsumer `json:"rejected,omitempty"`
|
|
RejectWhy string `json:"rejected-why,omitempty"`
|
|
Unasked string `json:"unasked,omitempty"`
|
|
}
|
|
|
|
func listRetiring(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn, asJSON bool) error {
|
|
instances, err := providerInstances(ctx, inv)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var rows []retiringRow
|
|
for _, p := range instances {
|
|
row := retiringRow{Node: p.Node, Module: p.Module}
|
|
state, err := askRetirement(ctx, conn, p)
|
|
switch {
|
|
case isNothingServes(err):
|
|
row.Unasked = "answers no retirement question: it predates ADR 0230"
|
|
case err != nil:
|
|
row.Unasked = err.Error()
|
|
default:
|
|
if state.Waiting != nil {
|
|
row.Waiting, row.Since, row.Held = state.Waiting.Consumers, state.Waiting.Since, state.Waiting.Held
|
|
}
|
|
if state.Rejected != nil {
|
|
row.Rejected, row.RejectWhy = state.Rejected.Consumers, orWhy(state.Rejected.By, state.Rejected.Why)
|
|
}
|
|
row.Bound = state.Bound
|
|
if row.Waiting == nil && row.Rejected == nil {
|
|
continue
|
|
}
|
|
}
|
|
rows = append(rows, row)
|
|
}
|
|
if asJSON {
|
|
if rows == nil {
|
|
rows = []retiringRow{}
|
|
}
|
|
return printJSON(map[string]any{"providers": rows})
|
|
}
|
|
said := false
|
|
for _, r := range rows {
|
|
switch {
|
|
case r.Unasked != "":
|
|
fmt.Printf("%s on %s: not asked — %s\n", r.Module, r.Node, r.Unasked)
|
|
default:
|
|
if r.Waiting != nil {
|
|
said = true
|
|
fmt.Printf("%s on %s WAITS since %s to retire %d of the %d it holds (%s): %s\n"+
|
|
" retire approve %s %s --why … | retire reject %s %s --why …\n",
|
|
r.Module, r.Node, r.Since, len(r.Waiting), r.Held, orBound(r.Bound), consumerList(r.Waiting),
|
|
r.Node, r.Module, r.Node, r.Module)
|
|
}
|
|
if r.Rejected != nil {
|
|
said = true
|
|
fmt.Printf("%s on %s keeps active, by a rejection (%s): %s\n", r.Module, r.Node, r.RejectWhy,
|
|
consumerList(r.Rejected))
|
|
}
|
|
}
|
|
}
|
|
if !said {
|
|
fmt.Println("no provider waits for a person to approve a retirement")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// answerRetirement approves or rejects what one provider waits with. **The set sent is the set the
|
|
// provider says it waits with, read now**, and the provider refuses any other: a person approves what
|
|
// they were shown, never a set that moved since.
|
|
func answerRetirement(ctx context.Context, conn *nats.Conn, p providerInstance, approve bool, f handActFlags) error {
|
|
state, err := askRetirement(ctx, conn, p)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
verb, tool := "retire reject", link.ToolRetireReject
|
|
var set []link.RetiredConsumer
|
|
if state.Waiting != nil {
|
|
set = state.Waiting.Consumers
|
|
}
|
|
if approve {
|
|
verb, tool = "retire approve", link.ToolRetireApprove
|
|
if set == nil && state.Rejected != nil {
|
|
// A rejection can be taken back: the consumers it kept are retired after all.
|
|
set = state.Rejected.Consumers
|
|
}
|
|
}
|
|
if len(set) == 0 {
|
|
return fmt.Errorf("%s on %s waits for nobody to approve or reject a retirement. Nothing was done", p.Module, p.Node)
|
|
}
|
|
names := make([]string, 0, len(set))
|
|
for _, c := range set {
|
|
names = append(names, c.Consumer)
|
|
}
|
|
if strings.TrimSpace(*f.cause) == "" {
|
|
*f.cause = kindRetireWaiting
|
|
}
|
|
if strings.TrimSpace(*f.condition) == "" {
|
|
*f.condition = retireWaitingKey(p.Module, p.Node)
|
|
}
|
|
f.record(ctx, verb, append([]string{p.Node, p.Module}, names...))
|
|
answer, err := link.AskModuleToolOn(ctx, conn, p.Module, tool, p.Node, map[string]any{
|
|
"consumers": names, "why": strings.TrimSpace(*f.why), "by": link.Caller(), "via": link.ViaController,
|
|
}, retirementAsk)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if answer.Error != "" {
|
|
return fmt.Errorf("%s on %s refused: %s", p.Module, p.Node, answer.Error)
|
|
}
|
|
if approve {
|
|
fmt.Printf("%s on %s retired %s: access disabled, data kept; `cleanup list` shows them\n", p.Module, p.Node,
|
|
strings.Join(names, ", "))
|
|
} else {
|
|
fmt.Printf("%s on %s keeps %s active; the warning stays open until the mesh asks for them again or the "+
|
|
"retirement is approved\n", p.Module, p.Node, strings.Join(names, ", "))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// cleanupCommand is `cleanup list` and `cleanup delete`.
|
|
func cleanupCommand(ctx context.Context, args []string) error {
|
|
sub := "list"
|
|
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
|
sub, args = args[0], args[1:]
|
|
}
|
|
switch sub {
|
|
case "list":
|
|
set := flag.NewFlagSet("cleanup", flag.ContinueOnError)
|
|
asJSON := set.Bool("json", false, "as data")
|
|
if rest, err := parseAround(set, args); err != nil {
|
|
return err
|
|
} else if len(rest) > 0 {
|
|
return errors.New(cleanupUsage)
|
|
}
|
|
open, err := openStores(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer open.Close()
|
|
return onTheBus(func(conn *nats.Conn) error {
|
|
listing, err := gatherRetired(ctx, open.inventory, conn, time.Now())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return printRetired(listing, *asJSON)
|
|
})
|
|
case "delete":
|
|
set := flag.NewFlagSet("cleanup delete", flag.ContinueOnError)
|
|
f := addHandActFlags(set)
|
|
olderThan := set.Int("older-than", 0, "every retired consumer older than this many days")
|
|
confirm := set.Bool("confirm", false, "with --older-than: delete what is listed, rather than only list it")
|
|
rest, err := parseAround(set, args)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := f.require("cleanup delete"); err != nil {
|
|
return err
|
|
}
|
|
if strings.TrimSpace(*f.cause) == "" {
|
|
*f.cause = kindCleanupWaiting
|
|
}
|
|
switch {
|
|
case *olderThan > 0 && len(rest) == 0:
|
|
open, err := openStores(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer open.Close()
|
|
return onTheBus(func(conn *nats.Conn) error {
|
|
return deleteOlderThan(ctx, open.inventory, conn, *olderThan, *confirm, f, time.Now())
|
|
})
|
|
case *olderThan == 0 && len(rest) == 3 && !*confirm:
|
|
return onTheBus(func(conn *nats.Conn) error {
|
|
return deleteRetired(ctx, conn, providerInstance{Node: rest[0], Module: rest[1]}, rest[2], f)
|
|
})
|
|
}
|
|
return errors.New(cleanupUsage)
|
|
}
|
|
return errors.New(cleanupUsage)
|
|
}
|
|
|
|
// retiredRow is one retired consumer, as `cleanup list` shows it.
|
|
type retiredRow struct {
|
|
Node string `json:"node"`
|
|
Module string `json:"module"`
|
|
Consumer string `json:"consumer"`
|
|
// ConsumerNode is where the consumer was, when the provider knows.
|
|
ConsumerNode string `json:"consumer-node,omitempty"`
|
|
Kind string `json:"kind,omitempty"`
|
|
RetiredAt string `json:"retired-at,omitempty"`
|
|
// AgeDays is whole days since it was retired; -1 when the provider could not say when.
|
|
AgeDays int `json:"age-days"`
|
|
SizeBytes *int64 `json:"size-bytes,omitempty"`
|
|
Why string `json:"why,omitempty"`
|
|
}
|
|
|
|
// retiredListing is every provider's retired consumers, and the providers that could not say.
|
|
type retiredListing struct {
|
|
Retired []retiredRow `json:"retired"`
|
|
// Unasked names each provider not asked, and why: one older than the question, or one that failed.
|
|
Unasked []string `json:"unasked,omitempty"`
|
|
}
|
|
|
|
func gatherRetired(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn, now time.Time) (retiredListing, error) {
|
|
instances, err := providerInstances(ctx, inv)
|
|
if err != nil {
|
|
return retiredListing{}, err
|
|
}
|
|
return retiredOf(ctx, conn, instances, now), nil
|
|
}
|
|
|
|
func retiredOf(ctx context.Context, conn *nats.Conn, instances []providerInstance, now time.Time) retiredListing {
|
|
out := retiredListing{Retired: []retiredRow{}}
|
|
for _, p := range instances {
|
|
state, err := askRetirement(ctx, conn, p)
|
|
if err != nil {
|
|
if isNothingServes(err) {
|
|
out.Unasked = append(out.Unasked, fmt.Sprintf("%s on %s answers no retirement question: it predates ADR 0230", p.Module, p.Node))
|
|
} else {
|
|
out.Unasked = append(out.Unasked, err.Error())
|
|
}
|
|
continue
|
|
}
|
|
for _, c := range state.Retired {
|
|
row := retiredRow{Node: p.Node, Module: p.Module, Consumer: c.Consumer, ConsumerNode: c.Node, Kind: c.Kind,
|
|
RetiredAt: c.RetiredAt, AgeDays: -1, SizeBytes: c.SizeBytes, Why: c.Why}
|
|
if at := retiredAt(c); !at.IsZero() {
|
|
row.AgeDays = int(now.Sub(at).Hours() / 24)
|
|
}
|
|
out.Retired = append(out.Retired, row)
|
|
}
|
|
}
|
|
sort.SliceStable(out.Retired, func(i, j int) bool { return out.Retired[i].AgeDays > out.Retired[j].AgeDays })
|
|
return out
|
|
}
|
|
|
|
func printRetired(l retiredListing, asJSON bool) error {
|
|
if asJSON {
|
|
return printJSON(l)
|
|
}
|
|
if len(l.Retired) == 0 {
|
|
fmt.Println("no provider holds a retired consumer")
|
|
}
|
|
for _, r := range l.Retired {
|
|
age := "age unknown"
|
|
if r.AgeDays >= 0 {
|
|
age = strconv.Itoa(r.AgeDays) + " day(s)"
|
|
}
|
|
kind := ""
|
|
if r.Kind != "" && r.Kind != "consumer" {
|
|
kind = " [" + r.Kind + "]"
|
|
}
|
|
fmt.Printf("%s on %s: %s%s — retired %s, %s, %s\n %s\n", r.Module, r.Node, r.Consumer, kind, age,
|
|
sizeWords(r.SizeBytes), r.RetiredAt, orWhy("", r.Why))
|
|
}
|
|
for _, u := range l.Unasked {
|
|
fmt.Printf("not asked: %s\n", u)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// deleteRetired asks one provider to delete one consumer it holds retired — never an active one: the
|
|
// provider refuses that, and this refuses it first, from what the provider says it holds.
|
|
func deleteRetired(ctx context.Context, conn *nats.Conn, p providerInstance, consumer string, f handActFlags) error {
|
|
state, err := askRetirement(ctx, conn, p)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
found := false
|
|
for _, c := range state.Retired {
|
|
found = found || c.Consumer == consumer
|
|
}
|
|
if !found {
|
|
return fmt.Errorf("%s on %s holds no retired consumer %s — only a retired consumer is deleted. Nothing was done",
|
|
p.Module, p.Node, consumer)
|
|
}
|
|
f.record(ctx, "cleanup delete", []string{p.Node, p.Module, consumer})
|
|
answer, err := link.AskModuleToolOn(ctx, conn, p.Module, link.ToolRetiredDelete, p.Node, map[string]any{
|
|
"consumer": consumer, "confirm": consumer, "why": strings.TrimSpace(*f.why), "by": link.Caller(),
|
|
"via": link.ViaController,
|
|
}, retirementAsk)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if answer.Error != "" {
|
|
return fmt.Errorf("%s on %s refused to delete %s: %s", p.Module, p.Node, consumer, answer.Error)
|
|
}
|
|
var done struct {
|
|
FreedBytes *int64 `json:"freed_bytes"`
|
|
}
|
|
_ = unmarshalAnswer(answer, &done)
|
|
fmt.Printf("%s on %s deleted %s (%s freed)\n", p.Module, p.Node, consumer, sizeWords(done.FreedBytes))
|
|
return nil
|
|
}
|
|
|
|
// deleteOlderThan lists every consumer retired more than days ago, and deletes them only with confirm.
|
|
// One whose age the provider cannot say is never in it.
|
|
func deleteOlderThan(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn, days int, confirm bool,
|
|
f handActFlags, now time.Time) error {
|
|
listing, err := gatherRetired(ctx, inv, conn, now)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return deleteFrom(ctx, conn, listing, days, confirm, f)
|
|
}
|
|
|
|
func deleteFrom(ctx context.Context, conn *nats.Conn, listing retiredListing, days int, confirm bool, f handActFlags) error {
|
|
var due []retiredRow
|
|
unknown := 0
|
|
for _, r := range listing.Retired {
|
|
switch {
|
|
case r.AgeDays < 0:
|
|
unknown++
|
|
case r.AgeDays > days:
|
|
due = append(due, r)
|
|
}
|
|
}
|
|
for _, u := range listing.Unasked {
|
|
fmt.Printf("not asked: %s\n", u)
|
|
}
|
|
if unknown > 0 {
|
|
fmt.Printf("%d retired consumer(s) whose age their provider cannot say are left out\n", unknown)
|
|
}
|
|
if len(due) == 0 {
|
|
fmt.Printf("nothing has been retired more than %d day(s)\n", days)
|
|
return nil
|
|
}
|
|
fmt.Printf("retired more than %d day(s):\n", days)
|
|
for _, r := range due {
|
|
fmt.Printf(" %s on %s: %s — %d day(s), %s\n", r.Module, r.Node, r.Consumer, r.AgeDays, sizeWords(r.SizeBytes))
|
|
}
|
|
if !confirm {
|
|
fmt.Printf("nothing was deleted: add --confirm to delete these %d\n", len(due))
|
|
return nil
|
|
}
|
|
var failed []string
|
|
for _, r := range due {
|
|
if err := deleteRetired(ctx, conn, providerInstance{Node: r.Node, Module: r.Module}, r.Consumer, f); err != nil {
|
|
failed = append(failed, err.Error())
|
|
}
|
|
}
|
|
if len(failed) > 0 {
|
|
return fmt.Errorf("%d of %d not deleted: %s", len(failed), len(due), strings.Join(failed, "; "))
|
|
}
|
|
return nil
|
|
}
|