jochen
52af210e47
A module's data section says what it keeps and how precious it is; the backup holder's lines, binding stickiness, retirement on unassign and D13's conditions follow from it, so issue 273's empty replacement is said and an unassigned module's data is remembered, not forgotten.
480 lines
16 KiB
Go
480 lines
16 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"flag"
|
|
"fmt"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/nats-io/nats.go"
|
|
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
)
|
|
|
|
// The verbs a person answers a provider's retirement with, and cleans up with (novox/hq ADR 0230).
|
|
//
|
|
// **The controller asks; the provider acts.** Approving, rejecting and deleting are each a question to
|
|
// the provider on the machine it runs on — its `provisioner_*` tools — because the provider owns its
|
|
// backend and the controller touches none. Every one says why, is written in the hand-act log before
|
|
// it is asked, and the provider says what it did as its `provisioner.retirement` event.
|
|
|
|
const retireUsage = "retire [--json] | retire approve <node> <module> --why <text> | retire reject <node> <module> --why <text>"
|
|
|
|
const cleanupUsage = "cleanup [list] [--json] | cleanup delete <node> <module> <consumer> --why <text> | " +
|
|
"cleanup delete --older-than <days> --why <text> [--confirm]"
|
|
|
|
func isNothingServes(err error) bool { return errors.Is(err, link.ErrNothingServes) }
|
|
|
|
func unmarshalAnswer(a link.Answer, v any) error {
|
|
if len(a.Result) == 0 {
|
|
return errors.New("an empty answer")
|
|
}
|
|
return json.Unmarshal(a.Result, v)
|
|
}
|
|
|
|
// retireCommand is `retire`, `retire approve` and `retire reject`.
|
|
func retireCommand(ctx context.Context, args []string) error {
|
|
sub := "list"
|
|
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
|
sub, args = args[0], args[1:]
|
|
}
|
|
switch sub {
|
|
case "list":
|
|
set := flag.NewFlagSet("retire", flag.ContinueOnError)
|
|
asJSON := set.Bool("json", false, "as data")
|
|
if rest, err := parseAround(set, args); err != nil {
|
|
return err
|
|
} else if len(rest) > 0 {
|
|
return errors.New(retireUsage)
|
|
}
|
|
open, err := openStores(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer open.Close()
|
|
return onTheBus(func(conn *nats.Conn) error { return listRetiring(ctx, open.inventory, conn, *asJSON) })
|
|
case "approve", "reject":
|
|
set := flag.NewFlagSet("retire "+sub, flag.ContinueOnError)
|
|
f := addHandActFlags(set)
|
|
rest, err := parseAround(set, args)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(rest) != 2 {
|
|
return errors.New(retireUsage)
|
|
}
|
|
if err := f.require("retire " + sub); err != nil {
|
|
return err
|
|
}
|
|
return onTheBus(func(conn *nats.Conn) error {
|
|
return answerRetirement(ctx, conn, providerInstance{Node: rest[0], Module: rest[1]}, sub == "approve", f)
|
|
})
|
|
}
|
|
return errors.New(retireUsage)
|
|
}
|
|
|
|
// retiringRow is one provider's waiting or rejected set, as `retire` lists it.
|
|
type retiringRow struct {
|
|
Node string `json:"node"`
|
|
Module string `json:"module"`
|
|
Waiting []link.RetiredConsumer `json:"waiting,omitempty"`
|
|
Since string `json:"since,omitempty"`
|
|
Held int `json:"held,omitempty"`
|
|
Bound string `json:"bound,omitempty"`
|
|
Rejected []link.RetiredConsumer `json:"rejected,omitempty"`
|
|
RejectWhy string `json:"rejected-why,omitempty"`
|
|
Unasked string `json:"unasked,omitempty"`
|
|
}
|
|
|
|
func listRetiring(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn, asJSON bool) error {
|
|
instances, err := providerInstances(ctx, inv)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var rows []retiringRow
|
|
for _, p := range instances {
|
|
row := retiringRow{Node: p.Node, Module: p.Module}
|
|
state, err := askRetirement(ctx, conn, p)
|
|
switch {
|
|
case isNothingServes(err):
|
|
row.Unasked = "answers no retirement question: it predates ADR 0230"
|
|
case err != nil:
|
|
row.Unasked = err.Error()
|
|
default:
|
|
if state.Waiting != nil {
|
|
row.Waiting, row.Since, row.Held = state.Waiting.Consumers, state.Waiting.Since, state.Waiting.Held
|
|
}
|
|
if state.Rejected != nil {
|
|
row.Rejected, row.RejectWhy = state.Rejected.Consumers, orWhy(state.Rejected.By, state.Rejected.Why)
|
|
}
|
|
row.Bound = state.Bound
|
|
if row.Waiting == nil && row.Rejected == nil {
|
|
continue
|
|
}
|
|
}
|
|
rows = append(rows, row)
|
|
}
|
|
if asJSON {
|
|
if rows == nil {
|
|
rows = []retiringRow{}
|
|
}
|
|
return printJSON(map[string]any{"providers": rows})
|
|
}
|
|
said := false
|
|
for _, r := range rows {
|
|
switch {
|
|
case r.Unasked != "":
|
|
fmt.Printf("%s on %s: not asked — %s\n", r.Module, r.Node, r.Unasked)
|
|
default:
|
|
if r.Waiting != nil {
|
|
said = true
|
|
fmt.Printf("%s on %s WAITS since %s to retire %d of the %d it holds (%s): %s\n"+
|
|
" retire approve %s %s --why … | retire reject %s %s --why …\n",
|
|
r.Module, r.Node, r.Since, len(r.Waiting), r.Held, orBound(r.Bound), consumerList(r.Waiting),
|
|
r.Node, r.Module, r.Node, r.Module)
|
|
}
|
|
if r.Rejected != nil {
|
|
said = true
|
|
fmt.Printf("%s on %s keeps active, by a rejection (%s): %s\n", r.Module, r.Node, r.RejectWhy,
|
|
consumerList(r.Rejected))
|
|
}
|
|
}
|
|
}
|
|
if !said {
|
|
fmt.Println("no provider waits for a person to approve a retirement")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// answerRetirement approves or rejects what one provider waits with. **The set sent is the set the
|
|
// provider says it waits with, read now**, and the provider refuses any other: a person approves what
|
|
// they were shown, never a set that moved since.
|
|
func answerRetirement(ctx context.Context, conn *nats.Conn, p providerInstance, approve bool, f handActFlags) error {
|
|
state, err := askRetirement(ctx, conn, p)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
verb, tool := "retire reject", link.ToolRetireReject
|
|
var set []link.RetiredConsumer
|
|
if state.Waiting != nil {
|
|
set = state.Waiting.Consumers
|
|
}
|
|
if approve {
|
|
verb, tool = "retire approve", link.ToolRetireApprove
|
|
if set == nil && state.Rejected != nil {
|
|
// A rejection can be taken back: the consumers it kept are retired after all.
|
|
set = state.Rejected.Consumers
|
|
}
|
|
}
|
|
if len(set) == 0 {
|
|
return fmt.Errorf("%s on %s waits for nobody to approve or reject a retirement. Nothing was done", p.Module, p.Node)
|
|
}
|
|
names := make([]string, 0, len(set))
|
|
for _, c := range set {
|
|
names = append(names, c.Consumer)
|
|
}
|
|
if strings.TrimSpace(*f.cause) == "" {
|
|
*f.cause = kindRetireWaiting
|
|
}
|
|
if strings.TrimSpace(*f.condition) == "" {
|
|
*f.condition = retireWaitingKey(p.Module, p.Node)
|
|
}
|
|
f.record(ctx, verb, append([]string{p.Node, p.Module}, names...))
|
|
answer, err := link.AskModuleToolOn(ctx, conn, p.Module, tool, p.Node, map[string]any{
|
|
"consumers": names, "why": strings.TrimSpace(*f.why), "by": link.Caller(), "via": link.ViaController,
|
|
}, retirementAsk)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if answer.Error != "" {
|
|
return fmt.Errorf("%s on %s refused: %s", p.Module, p.Node, answer.Error)
|
|
}
|
|
if approve && state.RetiresBy == "mark-only" {
|
|
fmt.Printf("%s on %s marked %s retired, MARK ONLY: this provider cannot disable a consumer, so they keep "+
|
|
"their access until `cleanup delete`; `cleanup list` shows them\n", p.Module, p.Node, strings.Join(names, ", "))
|
|
} else if approve {
|
|
fmt.Printf("%s on %s retired %s: access disabled, data kept; `cleanup list` shows them\n", p.Module, p.Node,
|
|
strings.Join(names, ", "))
|
|
} else {
|
|
fmt.Printf("%s on %s keeps %s active; the warning stays open until the mesh asks for them again or the "+
|
|
"retirement is approved\n", p.Module, p.Node, strings.Join(names, ", "))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// cleanupCommand is `cleanup list` and `cleanup delete`.
|
|
func cleanupCommand(ctx context.Context, args []string) error {
|
|
sub := "list"
|
|
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
|
sub, args = args[0], args[1:]
|
|
}
|
|
switch sub {
|
|
case "list":
|
|
set := flag.NewFlagSet("cleanup", flag.ContinueOnError)
|
|
asJSON := set.Bool("json", false, "as data")
|
|
if rest, err := parseAround(set, args); err != nil {
|
|
return err
|
|
} else if len(rest) > 0 {
|
|
return errors.New(cleanupUsage)
|
|
}
|
|
open, err := openStores(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer open.Close()
|
|
return onTheBus(func(conn *nats.Conn) error {
|
|
listing, err := gatherRetired(ctx, open.inventory, conn, time.Now())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return printRetired(listing, *asJSON)
|
|
})
|
|
case "delete":
|
|
set := flag.NewFlagSet("cleanup delete", flag.ContinueOnError)
|
|
f := addHandActFlags(set)
|
|
olderThan := set.Int("older-than", 0, "every retired consumer older than this many days")
|
|
confirm := set.Bool("confirm", false, "with --older-than: delete what is listed, rather than only list it")
|
|
rest, err := parseAround(set, args)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := f.require("cleanup delete"); err != nil {
|
|
return err
|
|
}
|
|
if strings.TrimSpace(*f.cause) == "" {
|
|
*f.cause = kindCleanupWaiting
|
|
}
|
|
switch {
|
|
case *olderThan > 0 && len(rest) == 0:
|
|
open, err := openStores(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer open.Close()
|
|
return onTheBus(func(conn *nats.Conn) error {
|
|
return deleteOlderThan(ctx, open, conn, *olderThan, *confirm, f, time.Now())
|
|
})
|
|
case *olderThan == 0 && len(rest) == 3 && !*confirm:
|
|
open, err := openStores(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer open.Close()
|
|
return onTheBus(func(conn *nats.Conn) error {
|
|
// A module's own retired data, when the mesh holds such an item (novox/hq ADR 0233); a
|
|
// provider's retired consumer otherwise.
|
|
if r, err := open.inventory.DataOf(ctx, rest[0], rest[1], rest[2]); err == nil && r.DeletedAt == nil &&
|
|
r.RetiredAt != nil {
|
|
return deleteRetiredData(ctx, conn, open, r, f)
|
|
}
|
|
return deleteRetired(ctx, conn, providerInstance{Node: rest[0], Module: rest[1]}, rest[2], f)
|
|
})
|
|
}
|
|
return errors.New(cleanupUsage)
|
|
}
|
|
return errors.New(cleanupUsage)
|
|
}
|
|
|
|
// retiredRow is one retired consumer, as `cleanup list` shows it.
|
|
type retiredRow struct {
|
|
Node string `json:"node"`
|
|
Module string `json:"module"`
|
|
Consumer string `json:"consumer"`
|
|
// ConsumerNode is where the consumer was, when the provider knows.
|
|
ConsumerNode string `json:"consumer-node,omitempty"`
|
|
Kind string `json:"kind,omitempty"`
|
|
RetiredAt string `json:"retired-at,omitempty"`
|
|
// AgeDays is whole days since it was retired; -1 when the provider could not say when.
|
|
AgeDays int `json:"age-days"`
|
|
SizeBytes *int64 `json:"size-bytes,omitempty"`
|
|
Why string `json:"why,omitempty"`
|
|
// Access is "kept" for a consumer of a mark-only provider: retired on record, still reachable.
|
|
Access string `json:"access,omitempty"`
|
|
// Path and Class are a module's own retired data's (Kind own-data, novox/hq ADR 0233): where it is
|
|
// kept on its machine, and its class. Consumer is then the item.
|
|
Path string `json:"path,omitempty"`
|
|
Class string `json:"class,omitempty"`
|
|
}
|
|
|
|
// retiredListing is every provider's retired consumers, and the providers that could not say.
|
|
type retiredListing struct {
|
|
Retired []retiredRow `json:"retired"`
|
|
// Unasked names each provider not asked, and why: one older than the question, or one that failed.
|
|
Unasked []string `json:"unasked,omitempty"`
|
|
}
|
|
|
|
func gatherRetired(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn, now time.Time) (retiredListing, error) {
|
|
instances, err := providerInstances(ctx, inv)
|
|
if err != nil {
|
|
return retiredListing{}, err
|
|
}
|
|
listing := retiredOf(ctx, conn, instances, now)
|
|
// And every module's own data retired on its machine (novox/hq ADR 0233).
|
|
records, err := inv.Data(ctx)
|
|
if err != nil {
|
|
return retiredListing{}, err
|
|
}
|
|
listing.Retired = append(listing.Retired, retiredData(records, now)...)
|
|
sort.SliceStable(listing.Retired, func(i, j int) bool { return listing.Retired[i].AgeDays > listing.Retired[j].AgeDays })
|
|
return listing, nil
|
|
}
|
|
|
|
func retiredOf(ctx context.Context, conn *nats.Conn, instances []providerInstance, now time.Time) retiredListing {
|
|
out := retiredListing{Retired: []retiredRow{}}
|
|
for _, p := range instances {
|
|
state, err := askRetirement(ctx, conn, p)
|
|
if err != nil {
|
|
if isNothingServes(err) {
|
|
out.Unasked = append(out.Unasked, fmt.Sprintf("%s on %s answers no retirement question: it predates ADR 0230", p.Module, p.Node))
|
|
} else {
|
|
out.Unasked = append(out.Unasked, err.Error())
|
|
}
|
|
continue
|
|
}
|
|
for _, c := range state.Retired {
|
|
row := retiredRow{Node: p.Node, Module: p.Module, Consumer: c.Consumer, ConsumerNode: c.Node, Kind: c.Kind,
|
|
RetiredAt: c.RetiredAt, AgeDays: -1, SizeBytes: c.SizeBytes, Why: c.Why, Access: c.Access}
|
|
if at := retiredAt(c); !at.IsZero() {
|
|
row.AgeDays = int(now.Sub(at).Hours() / 24)
|
|
}
|
|
out.Retired = append(out.Retired, row)
|
|
}
|
|
}
|
|
sort.SliceStable(out.Retired, func(i, j int) bool { return out.Retired[i].AgeDays > out.Retired[j].AgeDays })
|
|
return out
|
|
}
|
|
|
|
func printRetired(l retiredListing, asJSON bool) error {
|
|
if asJSON {
|
|
return printJSON(l)
|
|
}
|
|
if len(l.Retired) == 0 {
|
|
fmt.Println("no provider holds a retired consumer, and no machine holds retired data")
|
|
}
|
|
for _, r := range l.Retired {
|
|
age := "age unknown"
|
|
if r.AgeDays >= 0 {
|
|
age = strconv.Itoa(r.AgeDays) + " day(s)"
|
|
}
|
|
kind := ""
|
|
if r.Kind != "" && r.Kind != "consumer" {
|
|
kind = " [" + r.Kind + "]"
|
|
}
|
|
if r.Access == "kept" {
|
|
kind += " [MARK ONLY: access kept until deleted]"
|
|
}
|
|
if r.Kind == retiredDataKind {
|
|
fmt.Printf("%s on %s: its own %s, %s, at %s — retired %s, %s, %s\n %s\n", r.Module, r.Node, r.Consumer,
|
|
r.Class, r.Path, age, sizeWords(r.SizeBytes), r.RetiredAt, orWhy("", r.Why))
|
|
continue
|
|
}
|
|
fmt.Printf("%s on %s: %s%s — retired %s, %s, %s\n %s\n", r.Module, r.Node, r.Consumer, kind, age,
|
|
sizeWords(r.SizeBytes), r.RetiredAt, orWhy("", r.Why))
|
|
}
|
|
for _, u := range l.Unasked {
|
|
fmt.Printf("not asked: %s\n", u)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// deleteRetired asks one provider to delete one consumer it holds retired — never an active one: the
|
|
// provider refuses that, and this refuses it first, from what the provider says it holds.
|
|
func deleteRetired(ctx context.Context, conn *nats.Conn, p providerInstance, consumer string, f handActFlags) error {
|
|
state, err := askRetirement(ctx, conn, p)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
found := false
|
|
for _, c := range state.Retired {
|
|
found = found || c.Consumer == consumer
|
|
}
|
|
if !found {
|
|
return fmt.Errorf("%s on %s holds no retired consumer %s — only a retired consumer is deleted. Nothing was done",
|
|
p.Module, p.Node, consumer)
|
|
}
|
|
f.record(ctx, "cleanup delete", []string{p.Node, p.Module, consumer})
|
|
answer, err := link.AskModuleToolOn(ctx, conn, p.Module, link.ToolRetiredDelete, p.Node, map[string]any{
|
|
"consumer": consumer, "confirm": consumer, "why": strings.TrimSpace(*f.why), "by": link.Caller(),
|
|
"via": link.ViaController,
|
|
}, retirementAsk)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if answer.Error != "" {
|
|
return fmt.Errorf("%s on %s refused to delete %s: %s", p.Module, p.Node, consumer, answer.Error)
|
|
}
|
|
var done struct {
|
|
FreedBytes *int64 `json:"freed_bytes"`
|
|
}
|
|
_ = unmarshalAnswer(answer, &done)
|
|
fmt.Printf("%s on %s deleted %s (%s freed)\n", p.Module, p.Node, consumer, sizeWords(done.FreedBytes))
|
|
return nil
|
|
}
|
|
|
|
// deleteOlderThan lists every consumer retired more than days ago, and deletes them only with confirm.
|
|
// One whose age the provider cannot say is never in it.
|
|
func deleteOlderThan(ctx context.Context, open *stores, conn *nats.Conn, days int, confirm bool,
|
|
f handActFlags, now time.Time) error {
|
|
listing, err := gatherRetired(ctx, open.inventory, conn, now)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return deleteFrom(ctx, conn, open, listing, days, confirm, f)
|
|
}
|
|
|
|
func deleteFrom(ctx context.Context, conn *nats.Conn, open *stores, listing retiredListing, days int, confirm bool, f handActFlags) error {
|
|
var due []retiredRow
|
|
unknown := 0
|
|
for _, r := range listing.Retired {
|
|
switch {
|
|
case r.AgeDays < 0:
|
|
unknown++
|
|
case r.AgeDays > days:
|
|
due = append(due, r)
|
|
}
|
|
}
|
|
for _, u := range listing.Unasked {
|
|
fmt.Printf("not asked: %s\n", u)
|
|
}
|
|
if unknown > 0 {
|
|
fmt.Printf("%d retired consumer(s) whose age their provider cannot say are left out\n", unknown)
|
|
}
|
|
if len(due) == 0 {
|
|
fmt.Printf("nothing has been retired more than %d day(s)\n", days)
|
|
return nil
|
|
}
|
|
fmt.Printf("retired more than %d day(s):\n", days)
|
|
for _, r := range due {
|
|
fmt.Printf(" %s on %s: %s — %d day(s), %s\n", r.Module, r.Node, r.Consumer, r.AgeDays, sizeWords(r.SizeBytes))
|
|
}
|
|
if !confirm {
|
|
fmt.Printf("nothing was deleted: add --confirm to delete these %d\n", len(due))
|
|
return nil
|
|
}
|
|
var failed []string
|
|
for _, r := range due {
|
|
var err error
|
|
if r.Kind == retiredDataKind {
|
|
var rec inventory.DataRecord
|
|
if rec, err = open.inventory.DataOf(ctx, r.Node, r.Module, r.Consumer); err == nil {
|
|
err = deleteRetiredData(ctx, conn, open, rec, f)
|
|
}
|
|
} else {
|
|
err = deleteRetired(ctx, conn, providerInstance{Node: r.Node, Module: r.Module}, r.Consumer, f)
|
|
}
|
|
if err != nil {
|
|
failed = append(failed, err.Error())
|
|
}
|
|
}
|
|
if len(failed) > 0 {
|
|
return fmt.Errorf("%d of %d not deleted: %s", len(failed), len(due), strings.Join(failed, "; "))
|
|
}
|
|
return nil
|
|
}
|