Delete go/provider and its harness: the provisioner's client holds it, no importer left on any join head (S-7, B9f; issue 1966)
Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i
This commit is contained in:
@@ -25,7 +25,7 @@ import (
|
||||
var meshSDKPackages = []string{
|
||||
".", "announce", "announcetest", "buildstep", "buscalls", "buscompose", "busstate", "busretry", "calladdress", "catalogue", "channel",
|
||||
"declaration", "delivery", "forgekey", "handacts", "health", "internal/beside", "keys", "kvlimit", "membership", "motion",
|
||||
"objectlink", "outward", "overlay", "pernode", "process", "provider", "provider/harness", "provisions",
|
||||
"objectlink", "outward", "overlay", "pernode", "process", "provisions",
|
||||
"sdkversion", "seal", "seatdef", "seatlist", "seatlist/seatlisttest", "testbus", "views", "views/viewstest",
|
||||
}
|
||||
|
||||
|
||||
@@ -1,72 +0,0 @@
|
||||
package provider
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
)
|
||||
|
||||
// The provider's consumers' credentials, from the file its machine's provisioner client fills (novox/hq ADR 0373,
|
||||
// issue 679).
|
||||
//
|
||||
// **The provider never asks the vault.** A provider in the tool runner has no account of its own, and the vault
|
||||
// hands a secret only to a node's provisioner client or to a module under an account of its own (ADR 0349 clause
|
||||
// 11, ADR 0370 clause 2). So the provider's template names `${consumers:<provision>}`, and its machine's client
|
||||
// fills that file with one entry per consumer bound to it for that provision, each credential asked of the vault in
|
||||
// the client's own name; the file is the provider's alone (mode 0600, owned). ConsumersFile is that file as the
|
||||
// helper's Vault: it reads the file on each ask, so a consumer that binds or a credential the vault remakes reaches
|
||||
// the provider at its next pass.
|
||||
|
||||
// ConsumerCredential is one entry of the file: the consumer, the vault's name of its credential, and its value.
|
||||
type ConsumerCredential struct {
|
||||
// Consumer and Module are the consumer's machine and module, Local its local name for the provision, if any.
|
||||
Consumer string `json:"consumer"`
|
||||
Module string `json:"module"`
|
||||
Local string `json:"local,omitempty"`
|
||||
// Credential is the vault's name of the consumer's credential, PairName's.
|
||||
Credential string `json:"credential"`
|
||||
// Value is the credential itself.
|
||||
Value string `json:"value"`
|
||||
}
|
||||
|
||||
// ConsumersFile answers a credential from the files the provider machine's client fills: one per provision whose
|
||||
// template names `${consumers:<provision>}`, each a JSON list of ConsumerCredential.
|
||||
type ConsumersFile struct {
|
||||
Paths []string
|
||||
}
|
||||
|
||||
// Value is the credential named name, from the first file carrying it. A file not there yet is passed over; one not
|
||||
// in the client's form is an error, never read as no consumers.
|
||||
func (f ConsumersFile) Value(_ context.Context, name string) ([]byte, error) {
|
||||
var missing []string
|
||||
for _, at := range f.Paths {
|
||||
raw, err := os.ReadFile(at)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
missing = append(missing, at)
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the consumers' file %s: %w", at, err)
|
||||
}
|
||||
var entries []ConsumerCredential
|
||||
if err := json.Unmarshal(raw, &entries); err != nil {
|
||||
return nil, fmt.Errorf("the consumers' file %s is not the provisioner client's list of consumers", at)
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.Credential == name {
|
||||
if e.Value == "" {
|
||||
return nil, fmt.Errorf("the consumers' file %s carries %s with no value", at, name)
|
||||
}
|
||||
return []byte(e.Value), nil
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(missing) == len(f.Paths) {
|
||||
return nil, fmt.Errorf("%s: no consumers' file is filled yet (%v)", name, f.Paths)
|
||||
}
|
||||
return nil, fmt.Errorf("%s is in no consumers' file: the vault holds none for it yet, or its machine's client "+
|
||||
"has not filled it", name)
|
||||
}
|
||||
@@ -1,68 +0,0 @@
|
||||
package provider
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The provider reads its consumers' credentials only from the file its machine's provisioner client fills (novox/hq
|
||||
// ADR 0373 clause 3, issue 679): one entry per consumer, naming the consumer and the vault's name for its credential,
|
||||
// and carrying the value. The helper is wired to it as its Vault and asks nothing of the vault itself.
|
||||
func TestTheConsumersFileAnswersEachConsumersCredential(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
at := filepath.Join(dir, "consumers.json")
|
||||
file := `[{"consumer":"ace","module":"letta","credential":"pair/novox/postgres-database/ace/letta","value":"s3cret"},
|
||||
{"consumer":"g14","module":"gitea","local":"cache","credential":"pair/novox/postgres-database/g14/gitea/cache","value":"other"}]`
|
||||
if err := os.WriteFile(at, []byte(file), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v := ConsumersFile{Paths: []string{filepath.Join(dir, "absent.json"), at}}
|
||||
got, err := v.Value(context.Background(), PairName("novox", "postgres-database", "ace", "letta", ""))
|
||||
if err != nil || string(got) != "s3cret" {
|
||||
t.Fatalf("ace's letta: %q, %v; want s3cret", got, err)
|
||||
}
|
||||
got, err = v.Value(context.Background(), PairName("novox", "postgres-database", "g14", "gitea", "cache"))
|
||||
if err != nil || string(got) != "other" {
|
||||
t.Fatalf("g14's gitea under cache: %q, %v; want other", got, err)
|
||||
}
|
||||
// A consumer the file does not carry is waited for, and never told of a login its provider has not made.
|
||||
_, err = v.Value(context.Background(), PairName("novox", "postgres-database", "shanks", "photos", ""))
|
||||
if err == nil || !strings.Contains(err.Error(), "pair/novox/postgres-database/shanks/photos") {
|
||||
t.Fatalf("a consumer the file does not carry: %v", err)
|
||||
}
|
||||
if strings.Contains(err.Error(), "s3cret") || strings.Contains(err.Error(), "other") {
|
||||
t.Fatalf("an error names a credential: %v", err)
|
||||
}
|
||||
|
||||
// Wired as the helper's Vault, the file is what makes each consumer's login.
|
||||
bus := newStandIn()
|
||||
assign(t, bus, "ace", "letta", bind("postgres-database", "", "novox", "postgres"))
|
||||
var made []string
|
||||
p := New(Options{Node: "novox", Module: "postgres", Bus: bus, Vault: v,
|
||||
Offers: map[string]Offer{"postgres-database": {Serves: map[string]any{"port": 5432}}},
|
||||
Make: func(_ context.Context, c Consumer) error {
|
||||
made = append(made, c.Entry.User+"="+string(c.Credential))
|
||||
return nil
|
||||
}, Logf: func(string, ...any) {}})
|
||||
if _, err := p.Pass(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(made) != 1 || made[0] != "mesh_ace_letta=s3cret" {
|
||||
t.Fatalf("made %v, want ace's letta under the file's credential", made)
|
||||
}
|
||||
}
|
||||
|
||||
// A file that is not the client's form is said, never read as no consumers.
|
||||
func TestAConsumersFileNotInItsFormIsRefused(t *testing.T) {
|
||||
at := filepath.Join(t.TempDir(), "consumers.json")
|
||||
if err := os.WriteFile(at, []byte(`{"not": "a list"}`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err := ConsumersFile{Paths: []string{at}}.Value(context.Background(), "pair/novox/p/ace/m")
|
||||
if err == nil || !strings.Contains(err.Error(), at) {
|
||||
t.Fatalf("a file not in the client's form: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -1,20 +0,0 @@
|
||||
package provider
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestStandingIsFalseOnlyForAnEndedOrPendingValue(t *testing.T) {
|
||||
for raw, want := range map[string]bool{
|
||||
`{"node":"a","module":"b"}`: true,
|
||||
`{"node":"a","module":"b","ended":"","state":"","x":{"y":1}}`: true,
|
||||
`{"node":"a","module":"b","ended":null}`: true,
|
||||
`{"node":"a","module":"b","build":"b1","commit":"c","rollout":"r","since":"t","artifact":{"name":"n"}}`: true,
|
||||
`{"node":"a","module":"b","ended":"2026-10-18T01:00:00Z"}`: false,
|
||||
`{"node":"a","module":"b","state":"pending"}`: false,
|
||||
} {
|
||||
if got := Standing([]byte(raw)); got != want {
|
||||
t.Errorf("%s: %v, want %v", raw, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,5 +0,0 @@
|
||||
package harness
|
||||
|
||||
import "context"
|
||||
|
||||
var ctx = context.Background()
|
||||
@@ -1,62 +0,0 @@
|
||||
package harness
|
||||
|
||||
// The moved path started before its runtime answers asks again until it does (issue 943).
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
)
|
||||
|
||||
// earlyBus refuses its first watches as the runtime does before Serve runs, then answers as the bucket bus does.
|
||||
type earlyBus struct {
|
||||
*bucketBus
|
||||
refuse int
|
||||
}
|
||||
|
||||
func (b *earlyBus) Watch(ctx context.Context, bucket string, handle func(string, []byte)) error {
|
||||
if b.refuse > 0 {
|
||||
b.refuse--
|
||||
return errors.New("before Serve: a bundle reaches the bus through the runtime that launched it")
|
||||
}
|
||||
return b.bucketBus.Watch(ctx, bucket, handle)
|
||||
}
|
||||
|
||||
func TestAMovedPathStartedTooEarlyAsksAgainUntilTheRuntimeAnswers(t *testing.T) {
|
||||
w := newMovedWorld(t, map[[2]string]string{})
|
||||
w.run.FirstRetry = time.Millisecond
|
||||
w.run.loaded = false
|
||||
bus := &earlyBus{bucketBus: w.bus, refuse: 3}
|
||||
p := provider.New(provider.Options{Node: "novox", Module: "postgres", Bus: w.bus,
|
||||
Vault: provider.ConsumersFile{Paths: []string{w.file}}, Make: w.run.Make, Gone: w.run.Gone,
|
||||
Every: time.Hour, Logf: func(string, ...any) {}})
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- w.run.Run(ctx, p, bus) }()
|
||||
deadline := time.After(time.Second)
|
||||
for {
|
||||
w.run.mu.Lock()
|
||||
loaded := w.run.loaded
|
||||
w.run.mu.Unlock()
|
||||
if loaded {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case err := <-done:
|
||||
cancel()
|
||||
t.Fatalf("Run stopped before the runtime answered: %v", err)
|
||||
case <-deadline:
|
||||
cancel()
|
||||
t.Fatal("the plans were never watched")
|
||||
case <-time.After(time.Millisecond):
|
||||
}
|
||||
}
|
||||
cancel()
|
||||
<-done
|
||||
if bus.refuse != 0 {
|
||||
t.Fatalf("%d refusals left", bus.refuse)
|
||||
}
|
||||
}
|
||||
@@ -1,132 +0,0 @@
|
||||
package harness
|
||||
|
||||
// postgres handed to the provisioner still serves the consumers the controller's grant file gives (issue 937): a
|
||||
// consumer not moved has no plan naming postgres, only its grant, until R1 (issue 682).
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// granting gives the moved world a grant file, as the controller writes it for the consumers not moved.
|
||||
func (w *movedWorld) granting(given ...map[string]any) {
|
||||
if w.h.Receives == "" {
|
||||
w.h.Receives = filepath.Join(w.t.TempDir(), "mesh.json")
|
||||
}
|
||||
dir := filepath.Dir(w.h.Receives)
|
||||
for _, g := range given {
|
||||
secret := filepath.Join(dir, g["as"].(string)+".secret")
|
||||
if err := os.WriteFile(secret, []byte("pw-"+g["as"].(string)+"\n"), 0o600); err != nil {
|
||||
w.t.Fatal(err)
|
||||
}
|
||||
g["secret"] = secret
|
||||
}
|
||||
if given == nil {
|
||||
given = []map[string]any{}
|
||||
}
|
||||
raw, _ := json.Marshal(map[string]any{"requirement": ProvisionName, "given": given})
|
||||
if err := os.WriteFile(w.h.Receives, raw, 0o600); err != nil {
|
||||
w.t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// settled is enough passes and time for a retirement to be stable.
|
||||
func (w *movedWorld) settled() {
|
||||
for i := 0; i < 8; i++ {
|
||||
w.pass()
|
||||
}
|
||||
w.now = w.now.Add(15 * time.Minute)
|
||||
w.pass()
|
||||
w.pass()
|
||||
}
|
||||
|
||||
// The logins the grant file gives are asked for: made as the grant path makes them, never offered for retirement,
|
||||
// beside a moved consumer the helper makes from its plan.
|
||||
func testAConsumerOnlyTheGrantFileGivesIsAskedForAndMade(t *testing.T) {
|
||||
bound, logins := bindings(1)
|
||||
w := newMovedWorld(t, bound)
|
||||
w.a.inv.Active = []string{"mesh_ace_baserow", "mesh_ace_letta", logins[0]}
|
||||
w.bus.bind("ace", "app1", nil)
|
||||
w.granting(
|
||||
map[string]any{"as": "mesh_ace_baserow", "node": "ace", "values": map[string]any{"name": "baserow"}},
|
||||
map[string]any{"as": "mesh_ace_letta", "node": "ace",
|
||||
"values": map[string]any{"name": "letta", "extensions": []any{"vector"}}},
|
||||
)
|
||||
w.settled()
|
||||
if len(w.a.removed) != 0 || w.h.r.waiting != nil {
|
||||
t.Fatalf("retired %v, waiting %+v: the grant file's logins were counted unasked", w.a.removed, w.h.r.waiting)
|
||||
}
|
||||
made := map[string]Provision{}
|
||||
for _, c := range w.a.created {
|
||||
made[c.As] = c
|
||||
}
|
||||
letta := made["mesh_ace_letta"]
|
||||
if letta.Password != "pw-mesh_ace_letta" || letta.Consumer != "ace" ||
|
||||
!reflect.DeepEqual(letta.Values, map[string]any{"name": "letta", "extensions": []any{"vector"}}) {
|
||||
t.Fatalf("made %+v for letta, as its grant gives it", letta)
|
||||
}
|
||||
if _, ok := made["mesh_ace_baserow"]; !ok {
|
||||
t.Fatalf("baserow not made: %+v", w.a.created)
|
||||
}
|
||||
if _, ok := made[logins[0]]; !ok {
|
||||
t.Fatalf("the moved consumer not made: %+v", w.a.created)
|
||||
}
|
||||
}
|
||||
|
||||
// A login both a plan and the grant file name is the plan's: made once, with the plan's credential.
|
||||
func testALoginAPlanNamesIsNotMadeFromTheGrantFile(t *testing.T) {
|
||||
bound, logins := bindings(1)
|
||||
w := newMovedWorld(t, bound)
|
||||
w.bus.bind("ace", "app1", nil)
|
||||
w.granting(map[string]any{"as": logins[0], "node": "ace", "values": map[string]any{"name": "app1"}})
|
||||
w.pass()
|
||||
w.pass()
|
||||
for _, c := range w.a.created {
|
||||
if c.As == logins[0] && c.Password != "pw-app1" {
|
||||
t.Fatalf("made %s with the grant's password %q, not the plan's", c.As, c.Password)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// An unreadable grant file is not nobody asking (issue 241): nothing is retired while it cannot be read.
|
||||
func testAnUnreadableGrantFileRetiresNothing(t *testing.T) {
|
||||
bound, logins := bindings(1)
|
||||
w := newMovedWorld(t, bound)
|
||||
w.a.inv.Active = []string{"mesh_ace_baserow", logins[0]}
|
||||
w.bus.bind("ace", "app1", nil)
|
||||
w.h.Receives = filepath.Join(t.TempDir(), "absent.json")
|
||||
w.settled()
|
||||
if len(w.a.removed) != 0 || w.h.r.waiting != nil {
|
||||
t.Fatalf("retired %v, waiting %+v with the grant file unreadable", w.a.removed, w.h.r.waiting)
|
||||
}
|
||||
}
|
||||
|
||||
// A login the grant file stops giving is not retired by the provider itself (novox/hq ADR 0570 clause 5): every
|
||||
// retirement waits for the operator's yes on a provision's key.
|
||||
func testALoginTheGrantFileStopsGivingIsRetired(t *testing.T) {
|
||||
bound, _ := bindings(1)
|
||||
w := newMovedWorld(t, bound)
|
||||
w.bus.bind("ace", "app1", nil)
|
||||
w.granting(
|
||||
map[string]any{"as": "mesh_ace_baserow", "node": "ace", "values": map[string]any{"name": "baserow"}},
|
||||
map[string]any{"as": "mesh_ace_letta", "node": "ace", "values": map[string]any{"name": "letta"}},
|
||||
)
|
||||
w.pass()
|
||||
w.granting(map[string]any{"as": "mesh_ace_baserow", "node": "ace", "values": map[string]any{"name": "baserow"}})
|
||||
w.settled()
|
||||
if len(w.a.removed) != 0 || len(w.a.deleted) != 0 {
|
||||
t.Fatalf("retired %v, deleted %v, by itself", w.a.removed, w.a.deleted)
|
||||
}
|
||||
}
|
||||
|
||||
// TestGrantsMoved holds this file's cases, each a subtest of its own name (mesh-sdk's test budget: at most 500 unit tests).
|
||||
func TestGrantsMoved(t *testing.T) {
|
||||
t.Run("AConsumerOnlyTheGrantFileGivesIsAskedForAndMade", testAConsumerOnlyTheGrantFileGivesIsAskedForAndMade)
|
||||
t.Run("ALoginAPlanNamesIsNotMadeFromTheGrantFile", testALoginAPlanNamesIsNotMadeFromTheGrantFile)
|
||||
t.Run("AnUnreadableGrantFileRetiresNothing", testAnUnreadableGrantFileRetiresNothing)
|
||||
t.Run("ALoginTheGrantFileStopsGivingIsRetired", testALoginTheGrantFileStopsGivingIsRetired)
|
||||
}
|
||||
@@ -1,572 +0,0 @@
|
||||
// Package harness is the reconcile loop and the retirement of a provider that keeps a resource per consumer; the
|
||||
// module supplies an Adapter.
|
||||
package harness
|
||||
|
||||
// The reconcile loop every provider shares, as the TypeScript SDK's runProvisioner runs it
|
||||
// (@novox/mesh-sdk/provisioner, 0.1.12), line for line in behaviour. It moved here from the postgres and keycloak
|
||||
// providers, which each carried an identical copy (novox/hq ADR 0039: the loop is the SDK's, the adapter is the
|
||||
// module's; issue 849).
|
||||
//
|
||||
// Read the contributions the mesh delivered; bring each consumer's resource into being through the
|
||||
// adapter, under the login and password the mesh minted; retire what the mesh no longer asks for.
|
||||
// **A provider creates the credential the mesh minted, and seals nothing (novox/hq ADR 0048).**
|
||||
//
|
||||
// **A provider that keeps failing a consumer says so on the bus (novox/hq ADR 0224).** A consumer
|
||||
// whose create, check or secret has failed without one success in between for FailingAfter is
|
||||
// announced as `provisioner.failing` — naming the consumer, its machine and the class of error — and
|
||||
// again every SayAgainEvery while it lasts; the first success after that is `provisioner.recovered`.
|
||||
// The controller keeps the newest per provider and consumer and `status` names it. On 2026-10-05 the
|
||||
// identity provider failed every consumer 31,000 times in a day and said so only in its journal
|
||||
// (novox/hq issue 179).
|
||||
//
|
||||
// **A consumer the mesh stops asking for is retired, not withdrawn, and deleted only by a person
|
||||
// (novox/hq ADR 0230, the operator's model of 2026-10-06).** Retiring disables its access — reversibly —
|
||||
// and marks its login and data "to delete" with when and why; nothing is deleted. Asked for again, the
|
||||
// ordinary create re-enables it as it was. It is retired only once the same set has gone unasked in
|
||||
// StablePasses consecutive passes and for StableFor, and a set larger than the bound waits for a
|
||||
// person. retirement.go.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Provision is one consumer's resource to bring into being — everything the mesh derived and delivered.
|
||||
type Provision struct {
|
||||
// As is the login the mesh derived and gave the consumer to present.
|
||||
As string
|
||||
// Password is the one the mesh minted, read from the file the host unsealed.
|
||||
Password string
|
||||
// Values are what the consumer contributed (e.g. {"name": "letta", "extensions": ["vector"]}).
|
||||
Values map[string]any
|
||||
// Derived is what this provider's own definition derives for the consumer (novox/hq ADR 0201).
|
||||
Derived map[string]any
|
||||
// At is where the consumer is; Consumer is its node.
|
||||
At string
|
||||
Consumer string
|
||||
}
|
||||
|
||||
// Adapter is the per-service half.
|
||||
type Adapter interface {
|
||||
// Create brings the consumer into being under the mesh's login and password — and, for one that
|
||||
// was retired, re-enables it as it was and clears its mark to delete.
|
||||
Create(ctx context.Context, p Provision) error
|
||||
// Retire disables the consumer's access, reversibly, and marks its login and data to delete with
|
||||
// when and why. It deletes nothing (novox/hq ADR 0230). derived is what the mesh last derived,
|
||||
// remembered here; nil for a consumer this process did not make.
|
||||
Retire(ctx context.Context, as string, derived map[string]any, why string, at time.Time) error
|
||||
// Holds says whether the backend still holds the consumer exactly as p says. Read-only.
|
||||
Holds(ctx context.Context, p Provision) (bool, error)
|
||||
// Inventory is what the backend holds that the mesh made: consumers active and retired, read from
|
||||
// the backend itself so a restart forgets nothing. Read-only.
|
||||
Inventory(ctx context.Context) (Inventory, error)
|
||||
// Delete removes one retired consumer — its login and its data — for good. Only ever asked by a
|
||||
// person, through `cleanup delete`; the harness has checked it is retired and not asked for.
|
||||
Delete(ctx context.Context, r Retired) (freedBytes int64, err error)
|
||||
}
|
||||
|
||||
// Harness is the loop's settings and memory.
|
||||
type Harness struct {
|
||||
Resource string
|
||||
Receives string
|
||||
Adapter Adapter
|
||||
Every time.Duration // 5s
|
||||
VerifyEvery time.Duration // 60s
|
||||
HoldsTimeout time.Duration // 30s
|
||||
Log func(format string, args ...any)
|
||||
Now func() time.Time
|
||||
// Announce publishes one of the provider's standing events; nil announces nothing. Node is the
|
||||
// machine this provider runs on, said in each.
|
||||
Announce func(event string, body map[string]any)
|
||||
Node string
|
||||
// FailingAfter is how long a consumer fails without a success before it is announced (5m);
|
||||
// SayAgainEvery is how often it is announced again while it lasts (15m), so a controller that
|
||||
// missed the first hears the next, and a standing nobody repeats can be told from one that holds.
|
||||
FailingAfter time.Duration
|
||||
SayAgainEvery time.Duration
|
||||
// StablePasses (5) is how many consecutive passes must see the same set no longer asked for, and
|
||||
// StableFor (10m) how long it must have held since the first of them, before it is retired. RetireAtOnce (3) and RetireFraction (0.5) bound what is retired without a person:
|
||||
// more consumers than RetireAtOnce, or — where more than one is held — a larger share of those held
|
||||
// than RetireFraction, waits for `retire approve` (novox/hq ADR 0230).
|
||||
StablePasses int
|
||||
StableFor time.Duration
|
||||
RetireAtOnce int
|
||||
RetireFraction float64
|
||||
|
||||
// mu is held by a pass and by every tool a person asks, so the two never interleave.
|
||||
mu sync.Mutex
|
||||
verifiedAt time.Time
|
||||
r retirement
|
||||
applied map[string]appliedEntry
|
||||
lost map[string]brake
|
||||
waiting map[string]int
|
||||
failing map[string]failure
|
||||
trouble map[string]*standing
|
||||
cleared map[string]bool
|
||||
lastWarning string
|
||||
}
|
||||
|
||||
// standing is one consumer's unbroken run of failures: since when, how often, and the last error.
|
||||
type standing struct {
|
||||
node string
|
||||
since time.Time
|
||||
attempts int
|
||||
class string
|
||||
text string
|
||||
saidAt time.Time
|
||||
}
|
||||
|
||||
// The events a provider's standing is announced as (novox/hq ADR 0224). The controller derives the
|
||||
// permission to emit them for every module that receives contributions; no manifest lists them.
|
||||
const (
|
||||
EventFailing = "provisioner.failing"
|
||||
EventRecovered = "provisioner.recovered"
|
||||
)
|
||||
|
||||
// The classes of error a standing is announced with: what a person reading `status` needs to know
|
||||
// before reading the journal. An adapter may say better (Classifier).
|
||||
const (
|
||||
ClassCredentials = "credentials-rejected"
|
||||
ClassUnreachable = "unreachable"
|
||||
ClassSecret = "secret-unreadable"
|
||||
ClassRefused = "refused"
|
||||
)
|
||||
|
||||
// Classifier is an adapter that can say what class an error of its own is.
|
||||
type Classifier interface {
|
||||
Class(err error) string
|
||||
}
|
||||
|
||||
type appliedEntry struct {
|
||||
hash string
|
||||
derived map[string]any
|
||||
node string
|
||||
}
|
||||
|
||||
type brake struct {
|
||||
times int
|
||||
nextAt time.Time
|
||||
}
|
||||
|
||||
type failure struct {
|
||||
text string
|
||||
times int
|
||||
}
|
||||
|
||||
// The longest a consumer whose create keeps failing to satisfy holds waits between checks.
|
||||
const maxBackoff = time.Hour
|
||||
|
||||
// How many passes a secret may be unreadable before it stops being called a race (issue 225), and
|
||||
// once said loudly, how often it is repeated. The same cadence quiets a create that keeps failing
|
||||
// the same way.
|
||||
const (
|
||||
patiently = 12
|
||||
loudlyEvery = 240
|
||||
)
|
||||
|
||||
type contribution struct {
|
||||
As string `json:"as"`
|
||||
Secret string `json:"secret"`
|
||||
Node string `json:"node"`
|
||||
At string `json:"at"`
|
||||
Values map[string]any `json:"values"`
|
||||
Derived map[string]any `json:"derived"`
|
||||
}
|
||||
|
||||
func (h *Harness) init() {
|
||||
if h.Every == 0 {
|
||||
h.Every = 5 * time.Second
|
||||
}
|
||||
if h.VerifyEvery == 0 {
|
||||
h.VerifyEvery = time.Minute
|
||||
}
|
||||
if h.HoldsTimeout == 0 {
|
||||
h.HoldsTimeout = 30 * time.Second
|
||||
}
|
||||
if h.Now == nil {
|
||||
h.Now = time.Now
|
||||
}
|
||||
if h.FailingAfter == 0 {
|
||||
h.FailingAfter = 5 * time.Minute
|
||||
}
|
||||
if h.SayAgainEvery == 0 {
|
||||
h.SayAgainEvery = 15 * time.Minute
|
||||
}
|
||||
if h.StablePasses == 0 {
|
||||
h.StablePasses = 5
|
||||
}
|
||||
if h.StableFor == 0 {
|
||||
h.StableFor = StableFor
|
||||
}
|
||||
if h.RetireAtOnce == 0 {
|
||||
h.RetireAtOnce = 3
|
||||
}
|
||||
if h.RetireFraction == 0 {
|
||||
h.RetireFraction = 0.5
|
||||
}
|
||||
if h.Log == nil {
|
||||
h.Log = func(format string, args ...any) { fmt.Fprintf(os.Stderr, format+"\n", args...) }
|
||||
}
|
||||
if h.applied == nil {
|
||||
h.applied = map[string]appliedEntry{}
|
||||
h.lost = map[string]brake{}
|
||||
h.waiting = map[string]int{}
|
||||
h.failing = map[string]failure{}
|
||||
h.trouble = map[string]*standing{}
|
||||
h.cleared = map[string]bool{}
|
||||
h.r.retired = map[string]retiredEntry{}
|
||||
}
|
||||
}
|
||||
|
||||
// Run reconciles until ctx ends. One consumer's failure never stops the others'.
|
||||
func (h *Harness) Run(ctx context.Context) {
|
||||
h.init()
|
||||
for {
|
||||
h.Reconcile(ctx)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(h.Every):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (h *Harness) say(format string, args ...any) {
|
||||
h.Log("[provisioner:"+h.Resource+"] "+format, args...)
|
||||
}
|
||||
|
||||
func (h *Harness) warn(why string) {
|
||||
if why == h.lastWarning {
|
||||
return
|
||||
}
|
||||
if why != "" {
|
||||
h.say("%s; nothing applied or removed until it can be read", why)
|
||||
} else {
|
||||
h.say("contributions file readable again")
|
||||
}
|
||||
h.lastWarning = why
|
||||
}
|
||||
|
||||
// readContributions answers the consumers asked for, or nil when the file says nothing usable.
|
||||
// **Nothing read is not nobody asking** (novox/hq issue 241): only a file that was read can retire anything.
|
||||
func (h *Harness) readContributions() []contribution {
|
||||
raw, err := os.ReadFile(h.Receives)
|
||||
if err != nil {
|
||||
h.warn(fmt.Sprintf("contributions file unreadable (%s): %v", h.Receives, err))
|
||||
return nil
|
||||
}
|
||||
var doc struct {
|
||||
Requirement string `json:"requirement"`
|
||||
Given json.RawMessage `json:"given"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &doc); err != nil {
|
||||
h.warn(fmt.Sprintf("contributions file is not JSON (%s): %v", h.Receives, err))
|
||||
return nil
|
||||
}
|
||||
if doc.Requirement != "" && doc.Requirement != h.Resource {
|
||||
h.warn(fmt.Sprintf("%s is for %s, not %s", h.Receives, doc.Requirement, h.Resource))
|
||||
return nil
|
||||
}
|
||||
var given []contribution
|
||||
if len(doc.Given) == 0 || string(doc.Given) == "null" || json.Unmarshal(doc.Given, &given) != nil {
|
||||
h.warn(fmt.Sprintf("%s has no given list", h.Receives))
|
||||
return nil
|
||||
}
|
||||
h.warn("")
|
||||
out := []contribution{}
|
||||
for _, g := range given {
|
||||
// No `as` is not a credential grant: nothing to create for it.
|
||||
if g.As != "" && g.Secret != "" {
|
||||
out = append(out, g)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func hashOf(as, password string, values, derived map[string]any) string {
|
||||
// Derived is in the hash: a provider that renames what it derives gave a different resource.
|
||||
b, _ := json.Marshal([]any{as, password, orEmpty(values), orEmpty(derived)})
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func orEmpty(m map[string]any) map[string]any {
|
||||
if m == nil {
|
||||
return map[string]any{}
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// Reconcile is one pass.
|
||||
func (h *Harness) Reconcile(ctx context.Context) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
h.init()
|
||||
given := h.readContributions()
|
||||
if given == nil {
|
||||
// Not a result: the passes that must agree before anything is retired start again.
|
||||
h.r.key, h.r.count = "", 0
|
||||
return
|
||||
}
|
||||
want := map[string]bool{}
|
||||
for _, g := range given {
|
||||
want[g.As] = true
|
||||
}
|
||||
h.seed(ctx, want)
|
||||
h.apply(ctx, given)
|
||||
|
||||
// Retire what the mesh has stably stopped asking for — within the bound, or with a person.
|
||||
h.retireUnasked(ctx, want)
|
||||
h.r.lastWant = want
|
||||
for as := range h.failing {
|
||||
if !want[as] {
|
||||
delete(h.failing, as)
|
||||
}
|
||||
}
|
||||
// A consumer the mesh stopped asking for that this provider holds nothing for is no longer failed by
|
||||
// anyone: said, so a standing the controller keeps for it is cleared rather than left naming a
|
||||
// consumer that is gone. One still held is said recovered when it is retired.
|
||||
for as := range h.trouble {
|
||||
if _, held := h.applied[as]; !want[as] && !held {
|
||||
h.recovered(as, "no longer asked for")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// apply makes each consumer the grant file gives, and checks every VerifyEvery that the backend still holds it. The
|
||||
// caller holds h.mu. One consumer's failure never stops the others'.
|
||||
func (h *Harness) apply(ctx context.Context, given []contribution) {
|
||||
verifying := h.Now().Sub(h.verifiedAt) >= h.VerifyEvery
|
||||
if verifying {
|
||||
h.verifiedAt = h.Now()
|
||||
}
|
||||
|
||||
for _, g := range given {
|
||||
raw, err := os.ReadFile(g.Secret)
|
||||
if err != nil {
|
||||
// A secret the host has not written yet is a race on the first pass; past a minute it is
|
||||
// a person's to look at, and said so (novox/hq issue 225).
|
||||
n := h.waiting[g.As] + 1
|
||||
h.waiting[g.As] = n
|
||||
if n <= patiently {
|
||||
h.say("%s: secret not readable yet (%s): %v", g.As, g.Secret, err)
|
||||
} else if n == patiently+1 || n%loudlyEvery == 0 {
|
||||
h.say("%s: CANNOT READ the secret after %d attempts (%s): %v. This is not a race any more — "+
|
||||
"nothing has been provisioned for this consumer and nothing will be until somebody looks. "+
|
||||
"Check who owns the file and who this process runs as (novox/hq issue 225)", g.As, n, g.Secret, err)
|
||||
}
|
||||
h.failed(g.As, g.Node, ClassSecret, fmt.Sprintf("secret not readable (%s): %v", g.Secret, err))
|
||||
continue
|
||||
}
|
||||
delete(h.waiting, g.As)
|
||||
password := strings.TrimSuffix(string(raw), "\n")
|
||||
p := Provision{As: g.As, Password: password, Values: orEmpty(g.Values), Derived: orEmpty(g.Derived), At: g.At, Consumer: g.Node}
|
||||
hash := hashOf(g.As, password, g.Values, g.Derived)
|
||||
|
||||
reapplying := 0
|
||||
if was, ok := h.applied[g.As]; ok && was.hash == hash {
|
||||
if !verifying {
|
||||
continue
|
||||
}
|
||||
b, braked := h.lost[g.As]
|
||||
if braked && h.Now().Before(b.nextAt) {
|
||||
continue
|
||||
}
|
||||
hctx, cancel := context.WithTimeout(ctx, h.HoldsTimeout)
|
||||
held, err := h.Adapter.Holds(hctx, p)
|
||||
timedOut := errors.Is(hctx.Err(), context.DeadlineExceeded)
|
||||
cancel()
|
||||
if err != nil {
|
||||
// Unable to ask is not evidence of loss. A backend that timed out will time out for
|
||||
// the next consumer too, so the rest of this pass is not asked.
|
||||
text := scrub(err, password)
|
||||
h.say("%s: could not check the backend, will ask again: %s", g.As, text)
|
||||
h.failed(g.As, g.Node, h.classOf(err, text), text)
|
||||
if timedOut {
|
||||
verifying = false
|
||||
}
|
||||
continue
|
||||
}
|
||||
if held {
|
||||
delete(h.lost, g.As)
|
||||
h.succeeded(g.As)
|
||||
continue
|
||||
}
|
||||
reapplying = b.times + 1
|
||||
if reapplying == 1 {
|
||||
h.say("%s: the backend no longer holds it; applying again", g.As)
|
||||
} else {
|
||||
h.say("%s: still not held after being applied again (%d times in a row) — create does not "+
|
||||
"produce what holds checks; applying again", g.As, reapplying)
|
||||
}
|
||||
}
|
||||
if err := h.Adapter.Create(ctx, p); err != nil {
|
||||
text := scrub(err, password)
|
||||
f := h.failing[g.As]
|
||||
if f.text != text {
|
||||
f = failure{text: text}
|
||||
}
|
||||
f.times++
|
||||
h.failing[g.As] = f
|
||||
// Said each time it changes, and while it stays the same, as rarely as a lost secret.
|
||||
if f.times == 1 || f.times%loudlyEvery == 0 {
|
||||
h.say("%s: create failed, will retry: %s", g.As, text)
|
||||
}
|
||||
h.failed(g.As, g.Node, h.classOf(err, text), text)
|
||||
if reapplying > 0 {
|
||||
h.lost[g.As] = brake{times: reapplying - 1}
|
||||
}
|
||||
continue
|
||||
}
|
||||
if f, was := h.failing[g.As]; was {
|
||||
h.say("%s: created, after %d failed attempt(s)", g.As, f.times)
|
||||
delete(h.failing, g.As)
|
||||
}
|
||||
h.succeeded(g.As)
|
||||
h.applied[g.As] = appliedEntry{hash: hash, derived: p.Derived, node: g.Node}
|
||||
h.reenabled(g.As, g.Node)
|
||||
if reapplying == 0 {
|
||||
delete(h.lost, g.As)
|
||||
} else {
|
||||
wait := h.VerifyEvery << (reapplying - 1)
|
||||
if wait > maxBackoff || wait <= 0 {
|
||||
wait = maxBackoff
|
||||
}
|
||||
h.lost[g.As] = brake{times: reapplying, nextAt: h.Now().Add(wait)}
|
||||
if reapplying > 1 {
|
||||
h.say("%s: next check in %s", g.As, wait.Round(time.Second))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// failed counts one more failure in a consumer's unbroken run, and announces the run once it has
|
||||
// lasted FailingAfter — then again every SayAgainEvery while it lasts.
|
||||
func (h *Harness) failed(as, node, class, text string) {
|
||||
now := h.Now()
|
||||
s := h.trouble[as]
|
||||
if s == nil {
|
||||
s = &standing{since: now}
|
||||
h.trouble[as] = s
|
||||
}
|
||||
s.node, s.class, s.text = node, class, text
|
||||
s.attempts++
|
||||
if now.Sub(s.since) < h.FailingAfter {
|
||||
return
|
||||
}
|
||||
if !s.saidAt.IsZero() && now.Sub(s.saidAt) < h.SayAgainEvery {
|
||||
return
|
||||
}
|
||||
first := s.saidAt.IsZero()
|
||||
s.saidAt = now
|
||||
if first {
|
||||
h.say("%s: FAILING for %s (%d attempts, %s): %s. Announced as %s; `status` names it until it "+
|
||||
"succeeds (novox/hq ADR 0224)", as, now.Sub(s.since).Round(time.Second), s.attempts, class, text, EventFailing)
|
||||
}
|
||||
h.announce(EventFailing, map[string]any{
|
||||
"provider": h.Resource, "provider-node": h.Node,
|
||||
"consumer": as, "node": node,
|
||||
"class": class, "error": clip(text),
|
||||
"since": s.since.UTC().Format(time.RFC3339), "attempts": s.attempts,
|
||||
})
|
||||
}
|
||||
|
||||
// succeeded ends a consumer's run of failures; one that was announced is announced recovered.
|
||||
//
|
||||
// **And the first success for a consumer since this process started is announced too**, failing or
|
||||
// not: a provider that announced a failure and was restarted has forgotten it, and without this the
|
||||
// controller would name the consumer failing for ever after it recovered unheard.
|
||||
func (h *Harness) succeeded(as string) {
|
||||
if h.trouble[as] == nil && !h.cleared[as] {
|
||||
h.cleared[as] = true
|
||||
h.announce(EventRecovered, map[string]any{
|
||||
"provider": h.Resource, "provider-node": h.Node, "consumer": as, "why": "first-success",
|
||||
})
|
||||
return
|
||||
}
|
||||
h.cleared[as] = true
|
||||
h.recovered(as, "")
|
||||
}
|
||||
|
||||
func (h *Harness) recovered(as, why string) {
|
||||
s := h.trouble[as]
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
delete(h.trouble, as)
|
||||
if s.saidAt.IsZero() {
|
||||
return // never announced, so there is nothing to take back
|
||||
}
|
||||
if why == "" {
|
||||
h.say("%s: recovered after %s and %d failed attempt(s)", as, h.Now().Sub(s.since).Round(time.Second), s.attempts)
|
||||
}
|
||||
body := map[string]any{
|
||||
"provider": h.Resource, "provider-node": h.Node, "consumer": as, "node": s.node,
|
||||
"since": s.since.UTC().Format(time.RFC3339), "attempts": s.attempts,
|
||||
}
|
||||
if why != "" {
|
||||
body["why"] = why
|
||||
}
|
||||
h.announce(EventRecovered, body)
|
||||
}
|
||||
|
||||
func (h *Harness) announce(event string, body map[string]any) {
|
||||
if h.Announce != nil {
|
||||
h.Announce(event, body)
|
||||
}
|
||||
}
|
||||
|
||||
// classOf is an error's class: the adapter's word when it has one, else read from the text.
|
||||
func (h *Harness) classOf(err error, text string) string {
|
||||
if c, ok := h.Adapter.(Classifier); ok {
|
||||
if class := c.Class(err); class != "" {
|
||||
return class
|
||||
}
|
||||
}
|
||||
return ClassOf(text)
|
||||
}
|
||||
|
||||
// ClassOf reads an error's class from its text — the words the backends the mesh runs use.
|
||||
func ClassOf(text string) string {
|
||||
t := strings.ToLower(text)
|
||||
for _, w := range []string{"invalid_grant", "invalid user credentials", "password authentication failed",
|
||||
"authentication failed", "unauthorized", " 401"} {
|
||||
if strings.Contains(t, w) {
|
||||
return ClassCredentials
|
||||
}
|
||||
}
|
||||
for _, w := range []string{"connection refused", "no such host", "i/o timeout", "deadline exceeded",
|
||||
"connection reset", "network is unreachable", "no route to host", "eof"} {
|
||||
if strings.Contains(t, w) {
|
||||
return ClassUnreachable
|
||||
}
|
||||
}
|
||||
return ClassRefused
|
||||
}
|
||||
|
||||
// clip keeps an announced error to what belongs in a status line.
|
||||
func clip(text string) string {
|
||||
const most = 300
|
||||
if len(text) <= most {
|
||||
return text
|
||||
}
|
||||
return text[:most] + "…"
|
||||
}
|
||||
|
||||
// scrub is an error's text with the consumer's password removed, raw and URL-encoded.
|
||||
func scrub(err error, password string) string {
|
||||
text := err.Error()
|
||||
if password == "" {
|
||||
return text
|
||||
}
|
||||
for _, form := range []string{password, url.QueryEscape(password), url.PathEscape(password)} {
|
||||
text = strings.ReplaceAll(text, form, "***")
|
||||
}
|
||||
return text
|
||||
}
|
||||
@@ -1,164 +0,0 @@
|
||||
package harness
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type world struct {
|
||||
t *testing.T
|
||||
dir string
|
||||
receives string
|
||||
now time.Time
|
||||
h *Harness
|
||||
a *recorder
|
||||
said []string
|
||||
}
|
||||
|
||||
func newWorld(t *testing.T) *world {
|
||||
w := &world{t: t, dir: t.TempDir(), now: time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC), a: &recorder{held: true}}
|
||||
w.receives = filepath.Join(w.dir, "mesh.json")
|
||||
w.h = &Harness{Resource: "postgres-database", Receives: w.receives, Adapter: w.a,
|
||||
Now: func() time.Time { return w.now },
|
||||
Log: func(f string, a ...any) { w.said = append(w.said, f) }}
|
||||
return w
|
||||
}
|
||||
|
||||
func (w *world) give(given ...map[string]any) {
|
||||
for _, g := range given {
|
||||
secret := filepath.Join(w.dir, g["as"].(string)+".secret")
|
||||
if err := os.WriteFile(secret, []byte("pw-"+g["as"].(string)+"\n"), 0o600); err != nil {
|
||||
w.t.Fatal(err)
|
||||
}
|
||||
g["secret"] = secret
|
||||
}
|
||||
if given == nil {
|
||||
given = []map[string]any{}
|
||||
}
|
||||
raw, _ := json.Marshal(map[string]any{"requirement": "postgres-database", "given": given})
|
||||
if err := os.WriteFile(w.receives, raw, 0o600); err != nil {
|
||||
w.t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func testAConsumerIsCreatedOnceUnderTheMeshsLoginAndPassword(t *testing.T) {
|
||||
w := newWorld(t)
|
||||
w.give(map[string]any{"as": "mesh_ace_letta", "node": "ace", "values": map[string]any{"name": "letta"}})
|
||||
w.h.Reconcile(ctx)
|
||||
w.h.Reconcile(ctx)
|
||||
if len(w.a.created) != 1 {
|
||||
t.Fatalf("created %d times", len(w.a.created))
|
||||
}
|
||||
p := w.a.created[0]
|
||||
if p.As != "mesh_ace_letta" || p.Password != "pw-mesh_ace_letta" || p.Consumer != "ace" {
|
||||
t.Fatalf("%+v", p)
|
||||
}
|
||||
}
|
||||
|
||||
func testAConsumerNoLongerAskedForIsRetiredOnceStable(t *testing.T) {
|
||||
w := newWorld(t)
|
||||
w.give(map[string]any{"as": "a"}, map[string]any{"as": "b"})
|
||||
w.h.Reconcile(ctx)
|
||||
w.give(map[string]any{"as": "a"})
|
||||
w.settle() // five passes and ten minutes
|
||||
if strings.Join(w.a.removed, ",") != "b" {
|
||||
t.Fatal(w.a.removed)
|
||||
}
|
||||
// Only a file that says nobody asks retires the last one.
|
||||
w.give()
|
||||
w.settle()
|
||||
if strings.Join(w.a.removed, ",") != "b,a" {
|
||||
t.Fatal(w.a.removed)
|
||||
}
|
||||
}
|
||||
|
||||
func testNothingReadIsNotNobodyAsking(t *testing.T) {
|
||||
for name, content := range map[string]string{
|
||||
"unreadable": "",
|
||||
"not JSON": "{",
|
||||
"no given": `{"requirement": "postgres-database"}`,
|
||||
"another": `{"requirement": "mssql-database", "given": []}`,
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
w := newWorld(t)
|
||||
w.give(map[string]any{"as": "a"})
|
||||
w.h.Reconcile(ctx)
|
||||
if content == "" {
|
||||
os.Remove(w.receives)
|
||||
} else {
|
||||
os.WriteFile(w.receives, []byte(content), 0o600)
|
||||
}
|
||||
w.h.Reconcile(ctx)
|
||||
if len(w.a.removed) != 0 {
|
||||
t.Fatalf("retired %v on a file it could not use", w.a.removed)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func testALostConsumerIsAppliedAgainAndBraked(t *testing.T) {
|
||||
w := newWorld(t)
|
||||
w.give(map[string]any{"as": "a"})
|
||||
w.h.Reconcile(ctx)
|
||||
w.a.held = false
|
||||
w.now = w.now.Add(2 * time.Minute)
|
||||
w.h.Reconcile(ctx)
|
||||
if len(w.a.created) != 2 {
|
||||
t.Fatalf("created %d times", len(w.a.created))
|
||||
}
|
||||
// Still not held a minute later: applied again, then braked for two minutes.
|
||||
w.now = w.now.Add(61 * time.Second)
|
||||
w.h.Reconcile(ctx)
|
||||
w.now = w.now.Add(61 * time.Second)
|
||||
w.h.Reconcile(ctx)
|
||||
if len(w.a.created) != 3 {
|
||||
t.Fatalf("not braked: created %d times", len(w.a.created))
|
||||
}
|
||||
}
|
||||
|
||||
func testAFailingCreateIsRetriedAndSaidOnce(t *testing.T) {
|
||||
w := newWorld(t)
|
||||
w.a.failing = &pgErr{"extension \"nope\" refused, password pw-a"}
|
||||
w.give(map[string]any{"as": "a"})
|
||||
for i := 0; i < 5; i++ {
|
||||
w.h.Reconcile(ctx)
|
||||
}
|
||||
n := 0
|
||||
for _, s := range w.said {
|
||||
if strings.Contains(s, "create failed") {
|
||||
n++
|
||||
}
|
||||
}
|
||||
if n != 1 {
|
||||
t.Fatalf("said %d times", n)
|
||||
}
|
||||
w.a.failing = nil
|
||||
w.h.Reconcile(ctx)
|
||||
if len(w.a.created) != 1 {
|
||||
t.Fatal("not retried")
|
||||
}
|
||||
}
|
||||
|
||||
type pgErr struct{ s string }
|
||||
|
||||
func (e *pgErr) Error() string { return e.s }
|
||||
|
||||
func testScrubRemovesThePassword(t *testing.T) {
|
||||
if got := scrub(&pgErr{"bad pw a/b c and a%2Fb+c"}, "a/b c"); strings.Contains(got, "a/b c") || strings.Contains(got, "a%2Fb+c") {
|
||||
t.Fatal(got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHarness holds this file's cases, each a subtest of its own name (mesh-sdk's test budget: at most 500 unit tests).
|
||||
func TestHarness(t *testing.T) {
|
||||
t.Run("AConsumerIsCreatedOnceUnderTheMeshsLoginAndPassword", testAConsumerIsCreatedOnceUnderTheMeshsLoginAndPassword)
|
||||
t.Run("AConsumerNoLongerAskedForIsRetiredOnceStable", testAConsumerNoLongerAskedForIsRetiredOnceStable)
|
||||
t.Run("NothingReadIsNotNobodyAsking", testNothingReadIsNotNobodyAsking)
|
||||
t.Run("ALostConsumerIsAppliedAgainAndBraked", testALostConsumerIsAppliedAgainAndBraked)
|
||||
t.Run("AFailingCreateIsRetriedAndSaidOnce", testAFailingCreateIsRetriedAndSaidOnce)
|
||||
t.Run("ScrubRemovesThePassword", testScrubRemovesThePassword)
|
||||
}
|
||||
@@ -1,312 +0,0 @@
|
||||
package harness
|
||||
|
||||
// A provider handed to the provisioner (novox/hq ADR 0373, ADR 0349 clause 12, issues 780 and 805): where its
|
||||
// consumers file is set, it runs on the provider helper (package provider) instead of the harness reading the
|
||||
// controller's grant files. Moved is that path, shared by every provider that keeps a resource per consumer; a module
|
||||
// says only what differs, in a MovedSpec.
|
||||
//
|
||||
// The helper learns who binds the provider from the plans, composes each consumer's entry and reads its credential
|
||||
// through ConsumersFile, the file this machine's provisioner client fills. It leaves the rest to the two hooks
|
||||
// here: Make brings the consumer's resource into being exactly as the harness's adapter does, and Gone leaves
|
||||
// retirement to the harness's guards (retirement.go): the resource is disabled and marked to delete, never removed.
|
||||
//
|
||||
// **A consumer not moved is still the grant file's** (issue 937, ADR 0349 clause 17), for a provider whose spec says
|
||||
// WithGrants: only a moved consumer's plan names its provider, so the controller still gives every other consumer
|
||||
// through the grant file (MESH_RECEIVES) until R1 (issue 682). Each login it gives is asked for, and one no plan
|
||||
// names is made and checked by the harness's grant pass exactly as on the grant path; a login a plan names is the
|
||||
// helper's alone. A grant file that cannot be read is not nobody asking (issue 241): nothing is retired until it is
|
||||
// read again.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
"git.novox.be/novox/mesh-sdk/go/provisions"
|
||||
)
|
||||
|
||||
// MovedSpec is what differs per provider on the moved path.
|
||||
type MovedSpec struct {
|
||||
// Module and Provision are the provider module's name and what it provides, as its manifest says.
|
||||
Module, Provision string
|
||||
// Offer is what it serves: its credential form and served values.
|
||||
Offer provider.Offer
|
||||
// Login is the identity a consumer is made under at the provider's end (postgres: the entry's User; keycloak: its
|
||||
// As); an entry with none is not made.
|
||||
Login func(e provider.Entry) string
|
||||
// Values are the values the adapter is handed for a consumer; the entry's contribution when not set.
|
||||
Values func(e provider.Entry) map[string]any
|
||||
// Noun names what Login is in words ("login", "client id"), Made what is made ("login", "client"), and Backend
|
||||
// the service in "no longer holds it" ("the backend", "Keycloak").
|
||||
Noun, Made, Backend string
|
||||
// WithGrants makes the logins the grant file gives, that no plan names, asked for and made by the harness's grant
|
||||
// pass (issue 937).
|
||||
WithGrants bool
|
||||
}
|
||||
|
||||
// Moved is a provider on the provider helper, with the harness's own memory and retirement behind it: what is held,
|
||||
// what is counted unasked, what waits for a person (retirement.go). The harness is never Run here; the helper's Make
|
||||
// and Gone and the loop below drive the same state, so the guards are the harness's, not a copy.
|
||||
type Moved struct {
|
||||
h *Harness
|
||||
spec MovedSpec
|
||||
|
||||
// FirstRetry is the first wait before a watch asked too early is asked again (issue 943); a second when zero.
|
||||
FirstRetry time.Duration
|
||||
|
||||
mu sync.Mutex // guards plans and loaded
|
||||
plans map[string][]byte
|
||||
loaded bool
|
||||
verified map[string]time.Time
|
||||
}
|
||||
|
||||
func newMoved(h *Harness, spec MovedSpec) *Moved {
|
||||
h.mu.Lock()
|
||||
h.init()
|
||||
h.mu.Unlock()
|
||||
return &Moved{h: h, spec: spec, plans: map[string][]byte{}, verified: map[string]time.Time{}}
|
||||
}
|
||||
|
||||
// Make brings one consumer's resource into being under its login and the vault's credential, as the harness does:
|
||||
// at once when it is new or changed, and otherwise only when the backend no longer holds it, which is asked every
|
||||
// VerifyEvery.
|
||||
func (m *Moved) Make(ctx context.Context, c provider.Consumer) error {
|
||||
h, e := m.h, c.Entry
|
||||
as := m.spec.Login(e)
|
||||
if as == "" {
|
||||
return fmt.Errorf("%s has no %s to make", e.Key(), m.spec.Noun)
|
||||
}
|
||||
if len(c.Credential) == 0 {
|
||||
return fmt.Errorf("%s has no credential to make its %s with", e.Key(), m.spec.Made)
|
||||
}
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
password := string(c.Credential)
|
||||
values := orEmpty(e.Contributes)
|
||||
if m.spec.Values != nil {
|
||||
values = m.spec.Values(e)
|
||||
}
|
||||
p := Provision{As: as, Password: password, Values: values, Consumer: e.Consumer}
|
||||
hash := hashOf(as, password, p.Values, nil)
|
||||
if was, ok := h.applied[as]; ok && was.hash == hash {
|
||||
if h.Now().Sub(m.verified[as]) < h.VerifyEvery {
|
||||
return nil
|
||||
}
|
||||
m.verified[as] = h.Now()
|
||||
hctx, cancel := context.WithTimeout(ctx, h.HoldsTimeout)
|
||||
held, err := h.Adapter.Holds(hctx, p)
|
||||
cancel()
|
||||
if err != nil {
|
||||
text := scrub(err, password)
|
||||
h.failed(as, e.Consumer, h.classOf(err, text), text)
|
||||
return errors.New(text)
|
||||
}
|
||||
if held {
|
||||
h.succeeded(as)
|
||||
return nil
|
||||
}
|
||||
h.say("%s: %s no longer holds it; applying again", as, m.spec.Backend)
|
||||
}
|
||||
if err := h.Adapter.Create(ctx, p); err != nil {
|
||||
text := scrub(err, password)
|
||||
h.failed(as, e.Consumer, h.classOf(err, text), text)
|
||||
return errors.New(text)
|
||||
}
|
||||
h.succeeded(as)
|
||||
h.applied[as] = appliedEntry{hash: hash, node: e.Consumer}
|
||||
m.verified[as] = h.Now()
|
||||
h.reenabled(as, e.Consumer)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Gone is the helper letting a consumer go, which it does only on a retiring provision the operator said yes to (novox/hq
|
||||
// ADR 0570 clause 5): its login is retired at once, through the adapter, its resource and data kept and marked to
|
||||
// delete, deleted only by a person's `cleanup delete`. A consumer another plan still binds under the same login loses
|
||||
// nothing.
|
||||
func (m *Moved) Gone(ctx context.Context, e provider.Entry) error {
|
||||
h := m.h
|
||||
as := m.spec.Login(e)
|
||||
if as == "" {
|
||||
return nil // refused: nothing was made for it
|
||||
}
|
||||
want, _ := m.asked(h.Node)
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
if want[as] {
|
||||
return nil // another holder of the same login still binds: nothing to retire
|
||||
}
|
||||
if _, held := h.applied[as]; !held {
|
||||
return nil
|
||||
}
|
||||
if done := h.retire(ctx, []string{as}, "the operator approved its retirement ("+e.Key()+")", nil); len(done) != 1 {
|
||||
return fmt.Errorf("%s was not retired; it is tried again at the next pass", as)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// asked is the logins the seat's plans bind the provider for (novox/hq ADR 0570 clause 2), and whether they could all be
|
||||
// read: a plan that cannot be read, or no read yet, is not an answer (novox/hq issue 241).
|
||||
func (m *Moved) asked(node string) (map[string]bool, bool) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if !m.loaded || len(m.plans) == 0 {
|
||||
return nil, false
|
||||
}
|
||||
want := map[string]bool{}
|
||||
for key, raw := range m.plans {
|
||||
pl, err := provisions.DecodePlan(raw)
|
||||
if err != nil || pl.Node+"."+pl.Module != key {
|
||||
return nil, false
|
||||
}
|
||||
for _, b := range pl.Plan.Provisions {
|
||||
if b.Provider == node && b.ProviderModule == m.spec.Module && b.Provision == m.spec.Provision {
|
||||
as := b.As
|
||||
if as == "" {
|
||||
as = provider.ConsumerIdentity(pl.Node, pl.Module) // a plan that names none
|
||||
}
|
||||
want[as] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
return want, true
|
||||
}
|
||||
|
||||
// granted is what the grant file gives that no plan names (issue 937), and every login it gives; and whether it
|
||||
// could be read. The caller holds h.mu. With no grant file named there is nothing given, and that is a read.
|
||||
func (m *Moved) granted(planned map[string]bool) ([]contribution, map[string]bool, bool) {
|
||||
h := m.h
|
||||
if h.Receives == "" {
|
||||
return nil, map[string]bool{}, true
|
||||
}
|
||||
given := h.readContributions()
|
||||
if given == nil {
|
||||
return nil, nil, false
|
||||
}
|
||||
var own []contribution
|
||||
logins := map[string]bool{}
|
||||
for _, g := range given {
|
||||
logins[g.As] = true
|
||||
if !planned[g.As] {
|
||||
own = append(own, g)
|
||||
}
|
||||
}
|
||||
return own, logins, true
|
||||
}
|
||||
|
||||
// retire is one pass over what the plans bind and the grant file gives now: the logins only the grant file gives are
|
||||
// made and checked, as the grant path does. It retires nothing (ADR 0570 clause 5).
|
||||
func (m *Moved) retire(ctx context.Context) {
|
||||
h := m.h
|
||||
want, ok := m.asked(h.Node)
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
var given []contribution
|
||||
if ok && m.spec.WithGrants {
|
||||
var granted map[string]bool
|
||||
given, granted, ok = m.granted(want)
|
||||
for as := range granted {
|
||||
want[as] = true
|
||||
}
|
||||
}
|
||||
if !ok {
|
||||
// Not a result: the passes that must agree before anything is retired start again.
|
||||
h.r.key, h.r.count = "", 0
|
||||
return
|
||||
}
|
||||
h.seed(ctx, want)
|
||||
h.apply(ctx, given)
|
||||
// No login is retired for going unasked (novox/hq ADR 0570 clause 5): only Gone, on the operator's yes.
|
||||
h.r.lastWant = want
|
||||
for as := range h.failing {
|
||||
if !want[as] {
|
||||
delete(h.failing, as)
|
||||
}
|
||||
}
|
||||
for as := range h.trouble {
|
||||
if _, held := h.applied[as]; !want[as] && !held {
|
||||
h.recovered(as, "no longer asked for")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// watch keeps the plans as the bus holds them, from the first read on.
|
||||
func (m *Moved) watch(ctx context.Context, bus provider.Bus) error {
|
||||
err := bus.Watch(ctx, provisions.PlansBucket, func(key string, value []byte) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if value == nil {
|
||||
delete(m.plans, key)
|
||||
} else {
|
||||
m.plans[key] = append([]byte(nil), value...)
|
||||
}
|
||||
})
|
||||
if err == nil {
|
||||
m.mu.Lock()
|
||||
m.loaded = true
|
||||
m.mu.Unlock()
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Run answers the consumers through the helper and retires through the harness's rules, until ctx ends.
|
||||
//
|
||||
// The bus is reached through the runtime that launched this bundle, which answers only once Serve runs, and Run starts
|
||||
// beside Serve: a watch asked too early is asked again, after a wait that doubles from FirstRetry up to a minute,
|
||||
// until it is answered or ctx ends, so a start that loses the race never leaves the consumers unanswered (issue 943).
|
||||
func (m *Moved) Run(ctx context.Context, p *provider.Provider, bus provider.Bus) error {
|
||||
wait := m.FirstRetry
|
||||
if wait <= 0 {
|
||||
wait = time.Second
|
||||
}
|
||||
for tries := 1; ; tries++ {
|
||||
err := m.watch(ctx, bus)
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
if tries == 4 || tries%20 == 0 {
|
||||
m.h.say("the plans cannot be watched yet (%d tries), asking again: %v", tries, err)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(wait):
|
||||
}
|
||||
wait = min(wait*2, time.Minute)
|
||||
}
|
||||
go func() {
|
||||
for {
|
||||
m.retire(ctx)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(m.h.Every):
|
||||
}
|
||||
}
|
||||
}()
|
||||
return p.Run(ctx)
|
||||
}
|
||||
|
||||
// NewMoved is a provider handed to the provisioner: the helper answering its consumers, and the harness (never run)
|
||||
// whose retirement and tools it shares. consumersFile is where its machine's client fills its consumers. Run the
|
||||
// result with Moved.Run, passing the provider and the bus returned with it.
|
||||
func NewMoved(consumersFile string, h *Harness, spec MovedSpec) (*Moved, *provider.Provider, provider.Bus, error) {
|
||||
if consumersFile == "" {
|
||||
return nil, nil, nil, errors.New("no consumers' file")
|
||||
}
|
||||
run := newMoved(h, spec)
|
||||
bus := provider.StateBus(spec.Module)
|
||||
return run, provider.New(provider.Options{
|
||||
Node: h.Node,
|
||||
Module: spec.Module,
|
||||
Offers: map[string]provider.Offer{spec.Provision: spec.Offer},
|
||||
Bus: bus,
|
||||
Vault: provider.ConsumersFile{Paths: []string{consumersFile}},
|
||||
Make: run.Make,
|
||||
Gone: run.Gone,
|
||||
Every: h.Every,
|
||||
Logf: func(f string, a ...any) { h.say(f, a...) },
|
||||
}), bus, nil
|
||||
}
|
||||
@@ -1,476 +0,0 @@
|
||||
package harness
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
"git.novox.be/novox/mesh-sdk/go/provisions"
|
||||
)
|
||||
|
||||
const (
|
||||
ProvisionName = "postgres-database"
|
||||
servedPort = 5432
|
||||
)
|
||||
|
||||
// postgresSpec is what a postgres-like provider says: a login per consumer, and the grant file's logins.
|
||||
var postgresSpec = MovedSpec{Module: "postgres", Provision: ProvisionName,
|
||||
Offer: provider.Offer{Credential: provider.PerConsumer, Serves: map[string]any{"port": servedPort}},
|
||||
Login: func(e provider.Entry) string { return e.User },
|
||||
Noun: "login", Made: "login", Backend: "the backend", WithGrants: true}
|
||||
|
||||
// bucketBus is the helper's bus over in-memory buckets.
|
||||
type bucketBus struct {
|
||||
mu sync.Mutex
|
||||
buckets map[string]map[string][]byte
|
||||
}
|
||||
|
||||
func (b *bucketBus) Watch(_ context.Context, bucket string, handle func(string, []byte)) error {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
for k, v := range b.buckets[bucket] {
|
||||
handle(k, v)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *bucketBus) Put(_ context.Context, bucket, key string, value []byte) error {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if b.buckets[bucket] == nil {
|
||||
b.buckets[bucket] = map[string][]byte{}
|
||||
}
|
||||
b.buckets[bucket][key] = append([]byte(nil), value...)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *bucketBus) Delete(_ context.Context, bucket, key string) error {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
delete(b.buckets[bucket], key)
|
||||
return nil
|
||||
}
|
||||
|
||||
// putPlan is a consumer's plan on the seat mesh-provisioner, of these plan entries, each provision's key provided (novox/hq
|
||||
// ADR 0570 clauses 1, 2): what the helper binds by.
|
||||
func (b *bucketBus) putPlan(node, module string, entries ...map[string]any) {
|
||||
raw, _ := json.Marshal(entries)
|
||||
var ps []catalogue.PlanProvision
|
||||
if err := json.Unmarshal(raw, &ps); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
for _, pp := range ps {
|
||||
pr := provisions.Provision{Provision: pp.Provision, Local: pp.Local, Consumer: provisions.End{Module: module, Node: node},
|
||||
Provider: provisions.End{Module: pp.ProviderModule, Node: pp.Provider}, State: provisions.Provided}
|
||||
key, _ := pr.Key()
|
||||
value, err := provisions.Encode(pr)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
_ = b.Put(context.Background(), provisions.ProvisionsBucket, key, value)
|
||||
}
|
||||
plan, err := provisions.EncodePlan(provisions.Plan{Node: node, Module: module, Plan: catalogue.Plan{Provisions: ps}})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
_ = b.Put(context.Background(), provisions.PlansBucket, node+"."+module, plan)
|
||||
}
|
||||
|
||||
// retireOn is a consumer's provision retiring with the operator's yes (ADR 0570 clause 5), its plan gone: the one way
|
||||
// a provider lets it go.
|
||||
func (b *bucketBus) retireOn(node, module, provides string) {
|
||||
pr := provisions.Provision{Provision: ProvisionName, Consumer: provisions.End{Module: module, Node: node},
|
||||
Provider: provisions.End{Module: provides, Node: "novox"}, State: provisions.Retiring, Why: module + " left",
|
||||
Warrant: &provisions.Warrant{Ask: "a-1", Option: "yes", By: "the operator"}}
|
||||
key, _ := pr.Key()
|
||||
value, _ := provisions.Encode(pr)
|
||||
_ = b.Delete(context.Background(), provisions.PlansBucket, node+"."+module)
|
||||
_ = b.Put(context.Background(), provisions.ProvisionsBucket, key, value)
|
||||
}
|
||||
|
||||
// bind is one consumer's assignment, whose plan names this machine's postgres as the provider and carries what the
|
||||
// consumer contributes (issue 776).
|
||||
func (b *bucketBus) bind(node, module string, contributes map[string]any) {
|
||||
b.bindNamed("postgres", node, module, contributes, nil)
|
||||
}
|
||||
|
||||
// bindNamed is bind to a provider module, with the names the controller composed for the consumer (issue 792).
|
||||
func (b *bucketBus) bindNamed(provides, node, module string, contributes map[string]any, names map[string]string) {
|
||||
b.putPlan(node, module, map[string]any{"provision": ProvisionName, "provider": "novox", "provider-module": provides,
|
||||
"contributes": contributes, "names": names})
|
||||
}
|
||||
|
||||
// consumersFile writes the file the machine's client fills: one credential per consumer.
|
||||
func consumersFile(t *testing.T, bound map[[2]string]string) string {
|
||||
var list []provider.ConsumerCredential
|
||||
for k, password := range bound {
|
||||
list = append(list, provider.ConsumerCredential{Consumer: k[0], Module: k[1],
|
||||
Credential: provider.PairName("novox", ProvisionName, k[0], k[1], ""), Value: password})
|
||||
}
|
||||
raw, _ := json.Marshal(list)
|
||||
at := filepath.Join(t.TempDir(), "consumers.json")
|
||||
if err := os.WriteFile(at, raw, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return at
|
||||
}
|
||||
|
||||
// movedWorld is postgres on the helper over an in-memory bus, with a recorder for a server and a clock to move.
|
||||
type movedWorld struct {
|
||||
t *testing.T
|
||||
a *recorder
|
||||
bus *bucketBus
|
||||
h *Harness
|
||||
spec MovedSpec
|
||||
run *Moved
|
||||
p *provider.Provider
|
||||
now time.Time
|
||||
file string
|
||||
}
|
||||
|
||||
func newMovedWorld(t *testing.T, bound map[[2]string]string) *movedWorld {
|
||||
return newMovedWorldOf(t, bound, postgresSpec)
|
||||
}
|
||||
|
||||
func newMovedWorldOf(t *testing.T, bound map[[2]string]string, spec MovedSpec) *movedWorld {
|
||||
w := &movedWorld{t: t, spec: spec, a: &recorder{}, bus: &bucketBus{buckets: map[string]map[string][]byte{}},
|
||||
now: time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)}
|
||||
w.file = consumersFile(t, bound)
|
||||
w.h = &Harness{Resource: ProvisionName, Adapter: w.a, Node: "novox", Now: func() time.Time { return w.now },
|
||||
Log: func(string, ...any) {}}
|
||||
var bus provider.Bus
|
||||
w.run, w.p, bus, _ = NewMoved(w.file, w.h, spec)
|
||||
_ = bus
|
||||
if err := w.run.watch(ctx, w.bus); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
// pass is one helper pass and one retirement pass, five seconds after the last.
|
||||
func (w *movedWorld) pass() []provider.Outcome {
|
||||
w.now = w.now.Add(5 * time.Second)
|
||||
// the plans as the bus holds them now, as the watch would hand them
|
||||
w.run.plans = map[string][]byte{}
|
||||
for k, v := range w.bus.buckets[provisions.PlansBucket] {
|
||||
w.run.plans[k] = v
|
||||
}
|
||||
// the helper reads its buckets from the in-memory bus
|
||||
w.p = provider.New(provider.Options{Node: "novox", Module: w.spec.Module, Bus: w.bus,
|
||||
Vault: provider.ConsumersFile{Paths: []string{w.file}},
|
||||
Offers: map[string]provider.Offer{ProvisionName: w.spec.Offer},
|
||||
Make: w.run.Make, Gone: w.run.Gone, Logf: func(string, ...any) {}})
|
||||
out, err := w.p.Pass(ctx)
|
||||
if err != nil {
|
||||
w.t.Fatal(err)
|
||||
}
|
||||
w.run.retire(ctx)
|
||||
return out
|
||||
}
|
||||
|
||||
func bindings(n int) (map[[2]string]string, []string) {
|
||||
bound := map[[2]string]string{}
|
||||
var logins []string
|
||||
for i := 1; i <= n; i++ {
|
||||
module := fmt.Sprintf("app%d", i)
|
||||
bound[[2]string{"ace", module}] = "pw-" + module
|
||||
logins = append(logins, provider.ConsumerIdentity("ace", module))
|
||||
}
|
||||
return bound, logins
|
||||
}
|
||||
|
||||
// The consumers' file makes the same calls on the adapter as the equivalent grant files do (the harness).
|
||||
func testTheConsumersFileMakesWhatTheGrantFilesMake(t *testing.T) {
|
||||
letta := provider.ConsumerIdentity("ace", "letta")
|
||||
plain := provider.ConsumerIdentity("ace", "plain")
|
||||
|
||||
// Today: the grant files, through the harness.
|
||||
g := newWorld(t)
|
||||
g.give(
|
||||
map[string]any{"as": letta, "node": "ace", "values": map[string]any{"name": "letta", "extensions": []any{"vector"}}},
|
||||
map[string]any{"as": plain, "node": "ace", "values": map[string]any{"name": "plain"}},
|
||||
)
|
||||
g.h.Reconcile(ctx)
|
||||
|
||||
// The consumers' file, through the provider helper.
|
||||
w := newMovedWorld(t, map[[2]string]string{{"ace", "letta"}: "pw-" + letta, {"ace", "plain"}: "pw-" + plain})
|
||||
w.bus.bind("ace", "letta", map[string]any{"name": "letta", "extensions": []any{"vector"}})
|
||||
w.bus.bind("ace", "plain", map[string]any{"name": "plain"})
|
||||
for _, o := range w.pass() {
|
||||
if o.Status != provider.Answered {
|
||||
t.Fatalf("%+v", o)
|
||||
}
|
||||
}
|
||||
byAs := map[string]Provision{}
|
||||
for _, c := range w.a.created {
|
||||
byAs[c.As] = c
|
||||
}
|
||||
if len(w.a.created) != 2 {
|
||||
t.Fatalf("created %d, want 2", len(w.a.created))
|
||||
}
|
||||
for _, want := range g.a.created {
|
||||
c := byAs[want.As]
|
||||
if c.As != want.As || c.Password != want.Password || c.Consumer != want.Consumer ||
|
||||
!reflect.DeepEqual(orEmpty(c.Values), orEmpty(want.Values)) {
|
||||
t.Fatalf("made %+v, the grant files make %+v", c, want)
|
||||
}
|
||||
}
|
||||
// Unchanged, and held: nothing is made again, within a minute or after it.
|
||||
w.a.held = true
|
||||
w.pass()
|
||||
w.now = w.now.Add(2 * time.Minute)
|
||||
w.pass()
|
||||
if len(w.a.created) != 2 {
|
||||
t.Fatalf("an unchanged pass created again: %d", len(w.a.created))
|
||||
}
|
||||
// No longer held: made again.
|
||||
w.a.held = false
|
||||
w.now = w.now.Add(2 * time.Minute)
|
||||
w.pass()
|
||||
if len(w.a.created) != 4 {
|
||||
t.Fatalf("not made again: %d", len(w.a.created))
|
||||
}
|
||||
}
|
||||
|
||||
// A consumer whose plan goes retires nothing by itself, however long (novox/hq ADR 0570 clause 5): its login is retired,
|
||||
// never dropped, and its entry removed only once its provision is retiring with the operator's yes.
|
||||
func testAConsumerMissingFromOneReadRetiresNothing(t *testing.T) {
|
||||
bound, logins := bindings(2)
|
||||
w := newMovedWorld(t, bound)
|
||||
w.bus.bind("ace", "app1", nil)
|
||||
w.bus.bind("ace", "app2", nil)
|
||||
w.pass()
|
||||
_ = w.bus.Delete(ctx, provisions.PlansBucket, "ace.app2")
|
||||
for i := 0; i < 8; i++ {
|
||||
w.pass()
|
||||
}
|
||||
w.now = w.now.Add(30 * time.Minute)
|
||||
w.pass()
|
||||
if len(w.a.removed) != 0 || w.h.r.waiting != nil {
|
||||
t.Fatalf("retired %v, waiting %v, with no warrant", w.a.removed, w.h.r.waiting)
|
||||
}
|
||||
if _, kept := w.bus.buckets[provider.BucketOf("postgres")]["ace.app2.postgres-database"]; !kept {
|
||||
t.Fatal("the entry went with no warrant")
|
||||
}
|
||||
w.bus.retireOn("ace", "app2", "postgres")
|
||||
w.pass()
|
||||
if !reflect.DeepEqual(w.a.removed, []string{logins[1]}) || len(w.a.deleted) != 0 {
|
||||
t.Fatalf("retired %v, deleted %v", w.a.removed, w.a.deleted)
|
||||
}
|
||||
if _, kept := w.bus.buckets[provider.BucketOf("postgres")]["ace.app2.postgres-database"]; kept {
|
||||
t.Fatal("the entry stayed after the login was retired")
|
||||
}
|
||||
}
|
||||
|
||||
// A plan list that was never read is not nobody asking.
|
||||
func testNoPlansReadRetiresNothing(t *testing.T) {
|
||||
bound, _ := bindings(1)
|
||||
w := newMovedWorld(t, bound)
|
||||
w.bus.bind("ace", "app1", nil)
|
||||
w.pass()
|
||||
w.bus.Delete(ctx, provisions.PlansBucket, "ace.app1")
|
||||
for i := 0; i < 8; i++ {
|
||||
w.pass()
|
||||
}
|
||||
w.now = w.now.Add(20 * time.Minute)
|
||||
w.pass()
|
||||
if len(w.a.removed) != 0 {
|
||||
t.Fatalf("retired %v with no plans read", w.a.removed)
|
||||
}
|
||||
}
|
||||
|
||||
// A consumer whose credential is not in the file yet is waited for: nothing is made for it.
|
||||
func testAConsumerWithoutACredentialIsWaitedFor(t *testing.T) {
|
||||
w := newMovedWorld(t, map[[2]string]string{})
|
||||
w.bus.bind("ace", "letta", map[string]any{"name": "letta"})
|
||||
outcomes := w.pass()
|
||||
if len(outcomes) != 1 || outcomes[0].Status != provider.Waiting || len(w.a.created) != 0 {
|
||||
t.Fatalf("%+v, created %v", outcomes, w.a.created)
|
||||
}
|
||||
}
|
||||
|
||||
// Where the file is not there, postgres takes the grant directory, as before; where it is, the helper.
|
||||
// The grant path restarts itself when the consumers file is first filled.
|
||||
// A consumer whose module has a slug keeps the login its plan names, the slug first (issue 788): the adapter makes it
|
||||
// under that login, and the retirement counts it as asked for, so a pass retires nothing.
|
||||
func testASluggedConsumerKeepsTheLoginItsPlanNames(t *testing.T) {
|
||||
agent := provider.ConsumerIdentity("ace", "agent")
|
||||
w := newMovedWorld(t, map[[2]string]string{{"ace", "claude-code"}: "pw-agent"})
|
||||
w.bus.putPlan("ace", "claude-code", map[string]any{"provision": ProvisionName, "provider": "novox",
|
||||
"provider-module": "postgres", "as": agent})
|
||||
for _, o := range w.pass() {
|
||||
if o.Status != provider.Answered {
|
||||
t.Fatalf("%+v", o)
|
||||
}
|
||||
}
|
||||
if len(w.a.created) != 1 || w.a.created[0].As != agent {
|
||||
t.Fatalf("made %+v, want one under %s", w.a.created, agent)
|
||||
}
|
||||
for i := 0; i < 8; i++ {
|
||||
w.pass()
|
||||
}
|
||||
w.now = w.now.Add(15 * time.Minute)
|
||||
w.pass()
|
||||
w.pass()
|
||||
if len(w.a.removed) != 0 {
|
||||
t.Fatalf("retired %v: the plan's login was counted unasked", w.a.removed)
|
||||
}
|
||||
}
|
||||
|
||||
// A provider whose spec names the plan's `as` as the login and no grant file (the keycloak shape) makes the consumer
|
||||
// under that name with the values its spec composes, and a grant file that cannot be read does not hold up its
|
||||
// retirement pass.
|
||||
func testASpecWithoutGrantsMakesUnderThePlansAsAndIgnoresTheGrantFile(t *testing.T) {
|
||||
a := &recorder{}
|
||||
bus := &bucketBus{buckets: map[string]map[string][]byte{}}
|
||||
now := time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)
|
||||
h := &Harness{Resource: ProvisionName, Adapter: a, Node: "novox", Now: func() time.Time { return now },
|
||||
Receives: filepath.Join(t.TempDir(), "unreadable-or-missing.json"), Log: func(string, ...any) {}}
|
||||
spec := MovedSpec{Module: "keycloak", Provision: ProvisionName,
|
||||
Offer: provider.Offer{Credential: provider.PerConsumer},
|
||||
Login: func(e provider.Entry) string { return e.As }, Noun: "client id", Made: "client", Backend: "Keycloak",
|
||||
Values: func(e provider.Entry) map[string]any { return map[string]any{"label": e.Contributes["label"]} }}
|
||||
file := consumersFile(t, map[[2]string]string{{"ace", "wiki"}: "pw-wiki"})
|
||||
run, _, _, err := NewMoved(file, h, spec)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bus.putPlan("ace", "wiki", map[string]any{"provision": ProvisionName, "provider": "novox",
|
||||
"provider-module": "keycloak", "as": "wiki-client", "contributes": map[string]any{"label": "Wiki"}})
|
||||
if err := run.watch(ctx, bus); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p := provider.New(provider.Options{Node: "novox", Module: "keycloak", Bus: bus,
|
||||
Vault: provider.ConsumersFile{Paths: []string{file}},
|
||||
Offers: map[string]provider.Offer{ProvisionName: spec.Offer}, Make: run.Make, Gone: run.Gone,
|
||||
Logf: func(string, ...any) {}})
|
||||
out, err := p.Pass(ctx)
|
||||
if err != nil || len(out) != 1 || out[0].Status != provider.Answered {
|
||||
t.Fatalf("%v %+v", err, out)
|
||||
}
|
||||
run.retire(ctx)
|
||||
if len(a.created) != 1 || a.created[0].As != "wiki-client" || a.created[0].Password != "pw-wiki" ||
|
||||
a.created[0].Values["label"] != "Wiki" {
|
||||
t.Fatalf("made %+v", a.created)
|
||||
}
|
||||
}
|
||||
|
||||
const testIssuer = "https://id.example.org/realms/Novox"
|
||||
|
||||
var (
|
||||
grafanaContributes = map[string]any{"label": "grafana", "endpoint": "web", "callback": "/login/generic_oauth"}
|
||||
grafanaNames = map[string]string{"name": "grafana.example.org", "internal-name": "grafana.home.internal"}
|
||||
)
|
||||
|
||||
// keycloakSpec is what a keycloak-like provider says: the client id is the plan's `as`, the values are what the
|
||||
// consumer contributes with the controller's names over them, and no grant file is consulted.
|
||||
var keycloakSpec = MovedSpec{Module: "keycloak", Provision: ProvisionName,
|
||||
Offer: provider.Offer{Credential: provider.PerConsumer, Serves: map[string]any{"issuer": testIssuer,
|
||||
"token-path": "/protocol/openid-connect/token"}},
|
||||
Login: func(e provider.Entry) string { return e.As },
|
||||
Values: func(e provider.Entry) map[string]any {
|
||||
values := map[string]any{}
|
||||
for k, v := range e.Contributes {
|
||||
values[k] = v
|
||||
}
|
||||
for k, name := range e.Names {
|
||||
values[k] = name
|
||||
}
|
||||
return values
|
||||
},
|
||||
Noun: "client id", Made: "client", Backend: "Keycloak"}
|
||||
|
||||
// The consumers' file makes the same call on the adapter as the equivalent grant file does (the harness), for a
|
||||
// provider of the keycloak shape: the same client id, secret and values; and the entry tells the consumer what the
|
||||
// provider serves.
|
||||
func testTheConsumersFileMakesWhatTheGrantFileMakesForTheKeycloakShape(t *testing.T) {
|
||||
grafana := provider.ConsumerIdentity("home", "grafana")
|
||||
values := map[string]any{"name": grafanaNames["name"], "internal-name": grafanaNames["internal-name"]}
|
||||
for k, v := range grafanaContributes {
|
||||
values[k] = v
|
||||
}
|
||||
g := newWorld(t)
|
||||
g.give(map[string]any{"as": grafana, "node": "home", "values": values})
|
||||
g.h.Reconcile(ctx)
|
||||
|
||||
w := newMovedWorldOf(t, map[[2]string]string{{"home", "grafana"}: "pw-" + grafana}, keycloakSpec)
|
||||
w.bus.bindNamed("keycloak", "home", "grafana", grafanaContributes, grafanaNames)
|
||||
for _, o := range w.pass() {
|
||||
if o.Status != provider.Answered {
|
||||
t.Fatalf("%+v", o)
|
||||
}
|
||||
}
|
||||
if len(w.a.created) != 1 || len(g.a.created) != 1 {
|
||||
t.Fatalf("created %d through the helper, %d through the harness, want 1 each", len(w.a.created), len(g.a.created))
|
||||
}
|
||||
c, want := w.a.created[0], g.a.created[0]
|
||||
if c.As != want.As || c.Password != want.Password || c.Consumer != want.Consumer ||
|
||||
!reflect.DeepEqual(orEmpty(c.Values), orEmpty(want.Values)) {
|
||||
t.Fatalf("made %+v, the grant file makes %+v", c, want)
|
||||
}
|
||||
var entry provider.Entry
|
||||
if err := json.Unmarshal(w.bus.buckets[provider.BucketOf("keycloak")]["home.grafana."+ProvisionName], &entry); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if entry.Values["issuer"] != testIssuer || entry.Values["token-path"] != "/protocol/openid-connect/token" {
|
||||
t.Fatalf("served %v", entry.Values)
|
||||
}
|
||||
w.a.held = true
|
||||
w.pass()
|
||||
w.now = w.now.Add(2 * time.Minute)
|
||||
w.pass()
|
||||
if len(w.a.created) != 1 {
|
||||
t.Fatalf("an unchanged pass created again: %d", len(w.a.created))
|
||||
}
|
||||
w.a.held = false
|
||||
w.now = w.now.Add(2 * time.Minute)
|
||||
w.pass()
|
||||
if len(w.a.created) != 2 {
|
||||
t.Fatalf("not made again: %d", len(w.a.created))
|
||||
}
|
||||
}
|
||||
|
||||
// A consumer whose plan names its client id (a slugged module) keeps it, and the retirement counts it as asked for.
|
||||
func testAConsumerNamedByItsPlanKeepsItsClientId(t *testing.T) {
|
||||
agent := provider.ConsumerIdentity("home", "dashboard")
|
||||
w := newMovedWorldOf(t, map[[2]string]string{{"home", "grafana-2"}: "pw"}, keycloakSpec)
|
||||
w.bus.putPlan("home", "grafana-2", map[string]any{"provision": ProvisionName, "provider": "novox", "provider-module": "keycloak",
|
||||
"as": agent, "contributes": grafanaContributes, "names": grafanaNames})
|
||||
for _, o := range w.pass() {
|
||||
if o.Status != provider.Answered {
|
||||
t.Fatalf("%+v", o)
|
||||
}
|
||||
}
|
||||
if len(w.a.created) != 1 || w.a.created[0].As != agent {
|
||||
t.Fatalf("made %+v, want one under %s", w.a.created, agent)
|
||||
}
|
||||
for i := 0; i < 8; i++ {
|
||||
w.pass()
|
||||
}
|
||||
w.now = w.now.Add(15 * time.Minute)
|
||||
w.pass()
|
||||
w.pass()
|
||||
if len(w.a.removed) != 0 {
|
||||
t.Fatalf("retired %v: the plan's client id was counted unasked", w.a.removed)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMoved holds this file's cases, each a subtest of its own name (mesh-sdk's test budget: at most 500 unit tests).
|
||||
func TestMoved(t *testing.T) {
|
||||
t.Run("TheConsumersFileMakesWhatTheGrantFilesMake", testTheConsumersFileMakesWhatTheGrantFilesMake)
|
||||
t.Run("AConsumerMissingFromOneReadRetiresNothing", testAConsumerMissingFromOneReadRetiresNothing)
|
||||
t.Run("NoPlansReadRetiresNothing", testNoPlansReadRetiresNothing)
|
||||
t.Run("AConsumerWithoutACredentialIsWaitedFor", testAConsumerWithoutACredentialIsWaitedFor)
|
||||
t.Run("ASluggedConsumerKeepsTheLoginItsPlanNames", testASluggedConsumerKeepsTheLoginItsPlanNames)
|
||||
t.Run("ASpecWithoutGrantsMakesUnderThePlansAsAndIgnoresTheGrantFile", testASpecWithoutGrantsMakesUnderThePlansAsAndIgnoresTheGrantFile)
|
||||
t.Run("TheConsumersFileMakesWhatTheGrantFileMakesForTheKeycloakShape", testTheConsumersFileMakesWhatTheGrantFileMakesForTheKeycloakShape)
|
||||
t.Run("AConsumerNamedByItsPlanKeepsItsClientId", testAConsumerNamedByItsPlanKeepsItsClientId)
|
||||
}
|
||||
@@ -1,626 +0,0 @@
|
||||
package harness
|
||||
|
||||
// What becomes of a consumer the mesh no longer asks for (novox/hq ADR 0230 — the operator's model,
|
||||
// decided 2026-10-06; it replaces ADR 0229's withdrawal brake, which let one consumer go every hour).
|
||||
//
|
||||
// A consumer — a login, a client, a key, and the data behind it — is in one of three states:
|
||||
//
|
||||
// 1. ACTIVE.
|
||||
// 2. RETIRED. The mesh stopped asking for it: its access is disabled, reversibly, and its login and
|
||||
// data are marked "to delete" with when and why. Nothing is deleted. Asked for again, the ordinary
|
||||
// create re-enables it at once, as it was.
|
||||
// 3. DELETED, only through `cleanup delete`, a person's act, which this provider executes because it
|
||||
// owns its backend.
|
||||
//
|
||||
// **Stable removals.** A consumer is retired only once the same set of consumers has gone unasked BOTH
|
||||
// in StablePasses (5) consecutive passes that read the file AND for at least StableFor (10 minutes)
|
||||
// since the first pass that saw it. Five passes alone are twenty-five seconds — shorter than a
|
||||
// controller restart, a store reconnecting or a file half written. A pass that could not read the file
|
||||
// is not a result and starts both again; so does a different set. Additions and changes are never
|
||||
// delayed.
|
||||
//
|
||||
// **Too many is a person.** A stable set of more than RetireAtOnce (3), or — where more than one is
|
||||
// held — of more than RetireFraction (half) of those held, retires nothing: it WAITS, said in the
|
||||
// journal and announced `provisioner.retirement` `waiting` (again every SayAgainEvery), which the
|
||||
// controller raises as an urgent condition, until `retire approve <node> <module>` or `retire reject`.
|
||||
// An unassignment the controller made itself is no exception: many changes at once means a person is
|
||||
// at work, and a person confirms once. On 2026-10-04 one misread file withdrew seven consumers at once
|
||||
// (issue 241); under this rule that is a question, not an act.
|
||||
//
|
||||
// **The backend remembers, not this process.** What is retired, since when and why is read from the
|
||||
// backend's own mark (Adapter.Inventory), so a restart forgets nothing; a consumer the backend holds
|
||||
// active that the mesh no longer asks for is retired by the same rules after a restart as before one.
|
||||
// A consumer found disabled without the mark — withdrawn before this record — is ADOPTED as retired:
|
||||
// marked, said, announced `adopted`, and its clock starts then.
|
||||
//
|
||||
// **A rejection lives as long as this process.** Rejected, the set is kept active and not asked about
|
||||
// again while it stays the same set; a restart asks again — loudly, never silently.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
// EventRetirement is the one event a provider says about retirement, its `change` saying what
|
||||
// happened. The controller derives the permission to emit it for every module that receives
|
||||
// contributions, as it does the standing events (novox/hq ADR 0224, 0230).
|
||||
const EventRetirement = "provisioner.retirement"
|
||||
|
||||
// The changes EventRetirement carries.
|
||||
const (
|
||||
ChangeWaiting = "waiting" // a stable set over the bound waits for a person
|
||||
ChangeSettled = "settled" // a waiting or rejected set ended without being retired
|
||||
ChangeApproved = "approved" // a person approved the waiting (or rejected) set
|
||||
ChangeRejected = "rejected" // a person kept the waiting set active
|
||||
ChangeRetired = "retired" // consumers disabled and marked to delete
|
||||
ChangeReenabled = "reenabled" // a retired consumer asked for again, enabled as it was
|
||||
ChangeDeleted = "deleted" // a retired consumer deleted, by a person
|
||||
ChangeAdopted = "adopted" // found disabled without the mark, now retired on record
|
||||
)
|
||||
|
||||
// StableFor is the least time the same unasked set must hold before it is retired (novox/hq ADR 0230):
|
||||
// longer than a controller restart, a store reconnecting or a file half written.
|
||||
const StableFor = 10 * time.Minute
|
||||
|
||||
// KindConsumer is a retired consumer. A backend may list other things set aside for deletion under a
|
||||
// word of its own (postgres: a database renamed aside).
|
||||
const KindConsumer = "consumer"
|
||||
|
||||
// Retired is one thing a provider holds retired, as its backend's mark says.
|
||||
type Retired struct {
|
||||
Consumer string
|
||||
// Node is the consumer's machine, where the mark records it.
|
||||
Node string
|
||||
// RetiredAt is when it was retired; zero for one found disabled without the mesh's mark.
|
||||
RetiredAt time.Time
|
||||
Why string
|
||||
// SizeBytes is what deleting it would free; -1 when the backend cannot say.
|
||||
SizeBytes int64
|
||||
Kind string
|
||||
}
|
||||
|
||||
// Inventory is what a backend holds that the mesh made.
|
||||
type Inventory struct {
|
||||
Active []string
|
||||
Retired []Retired
|
||||
// Sizes is what each active consumer keeps on the backend, in bytes, where the backend can say: what
|
||||
// the controller tells an empty replacement of a consumer's data by (novox/hq ADR 0233).
|
||||
Sizes map[string]int64
|
||||
}
|
||||
|
||||
// retirement is the loop's memory of the sets it is counting, waiting on and was told to keep.
|
||||
type retirement struct {
|
||||
key string // the unasked set, joined
|
||||
count int // consecutive passes that saw it
|
||||
firstSeen time.Time
|
||||
waiting *waitingSet
|
||||
rejected *rejectedSet
|
||||
retired map[string]retiredEntry
|
||||
lastWant map[string]bool
|
||||
seeded bool
|
||||
seedSaid string
|
||||
// noneSaid is whether this process has said that nothing waits: an earlier process may have left a set
|
||||
// waiting that this one never knew (issue 937).
|
||||
noneSaid bool
|
||||
}
|
||||
|
||||
type waitingSet struct {
|
||||
set []string
|
||||
since time.Time
|
||||
saidAt time.Time
|
||||
held int
|
||||
}
|
||||
|
||||
type rejectedSet struct {
|
||||
set []string
|
||||
by, why string
|
||||
at time.Time
|
||||
}
|
||||
|
||||
type retiredEntry struct {
|
||||
node string
|
||||
at time.Time
|
||||
why string
|
||||
}
|
||||
|
||||
// seed reads what the backend holds, once per process: an active consumer the mesh no longer asks
|
||||
// for is held, so it is retired by the same rules as one this process made; one found disabled
|
||||
// without the mark is adopted as retired.
|
||||
func (h *Harness) seed(ctx context.Context, want map[string]bool) {
|
||||
if h.r.seeded {
|
||||
return
|
||||
}
|
||||
inv, err := h.Adapter.Inventory(ctx)
|
||||
if err != nil {
|
||||
if said := err.Error(); said != h.r.seedSaid {
|
||||
h.say("cannot list what the backend holds (%v); a consumer the mesh stopped asking for while this "+
|
||||
"provider was down is not retired until it can", err)
|
||||
h.r.seedSaid = said
|
||||
}
|
||||
return
|
||||
}
|
||||
h.r.seeded = true
|
||||
for _, as := range inv.Active {
|
||||
if _, held := h.applied[as]; !held && !want[as] {
|
||||
// No hash: asked for again, it is applied again, which is harmless and marks it.
|
||||
h.applied[as] = appliedEntry{}
|
||||
h.say("%s: held by the backend and no longer asked for; retired once that holds for %d passes "+
|
||||
"and %s (novox/hq ADR 0230)", as, h.StablePasses, h.StableFor)
|
||||
}
|
||||
}
|
||||
now := h.Now()
|
||||
for _, r := range inv.Retired {
|
||||
if r.Kind != "" && r.Kind != KindConsumer {
|
||||
continue
|
||||
}
|
||||
if !r.RetiredAt.IsZero() {
|
||||
h.r.retired[r.Consumer] = retiredEntry{node: r.Node, at: r.RetiredAt, why: r.Why}
|
||||
continue
|
||||
}
|
||||
if want[r.Consumer] {
|
||||
continue // asked for: this pass's create enables it
|
||||
}
|
||||
why := "found disabled without the mesh's mark — withdrawn before retirement was recorded " +
|
||||
"(novox/hq ADR 0230); retired as found"
|
||||
if err := h.Adapter.Retire(ctx, r.Consumer, nil, why, now); err != nil {
|
||||
h.say("%s: found disabled and could not be marked retired, will try at the next start: %v", r.Consumer, err)
|
||||
continue
|
||||
}
|
||||
h.r.retired[r.Consumer] = retiredEntry{node: r.Node, at: now, why: why}
|
||||
h.say("%s: ADOPTED as retired — %s", r.Consumer, why)
|
||||
h.announce(EventRetirement, h.retirementBody(ChangeAdopted, []map[string]any{
|
||||
{"consumer": r.Consumer, "node": r.Node, "retired_at": stamp(now), "why": why, "size_bytes": r.SizeBytes},
|
||||
}, map[string]any{"why": why}))
|
||||
}
|
||||
}
|
||||
|
||||
// retireUnasked counts the passes that saw the same set no longer asked for and, once it is stable,
|
||||
// retires it — or, past the bound, waits for a person.
|
||||
func (h *Harness) retireUnasked(ctx context.Context, want map[string]bool) {
|
||||
var unasked []string
|
||||
for as := range h.applied {
|
||||
if !want[as] {
|
||||
unasked = append(unasked, as)
|
||||
}
|
||||
}
|
||||
sort.Strings(unasked)
|
||||
key := strings.Join(unasked, "\x00")
|
||||
now := h.Now()
|
||||
switch {
|
||||
case len(unasked) == 0:
|
||||
h.settle("every consumer held is asked for again")
|
||||
h.r.key, h.r.count = "", 0
|
||||
return
|
||||
case key != h.r.key:
|
||||
if h.r.waiting != nil || h.r.rejected != nil {
|
||||
h.settle("the set the mesh no longer asks for changed")
|
||||
}
|
||||
h.r.key, h.r.count, h.r.firstSeen = key, 1, now
|
||||
h.say("no longer asked for: %s; retired once the same set holds for %d passes and %s",
|
||||
strings.Join(unasked, ", "), h.StablePasses, h.StableFor)
|
||||
default:
|
||||
h.r.count++
|
||||
}
|
||||
if h.r.rejected != nil || h.r.count < h.StablePasses || now.Sub(h.r.firstSeen) < h.StableFor {
|
||||
return
|
||||
}
|
||||
if h.overTheBound(len(unasked), len(h.applied)) {
|
||||
h.wait(unasked)
|
||||
return
|
||||
}
|
||||
why := fmt.Sprintf("the mesh stopped asking for it: the same result in %d consecutive passes over %s, from %s",
|
||||
h.r.count, now.Sub(h.r.firstSeen).Round(time.Second), stamp(h.r.firstSeen))
|
||||
h.retire(ctx, unasked, why, nil)
|
||||
}
|
||||
|
||||
// overTheBound says retiring n of the held consumers at once needs a person.
|
||||
func (h *Harness) overTheBound(n, held int) bool {
|
||||
if n == 0 {
|
||||
return false
|
||||
}
|
||||
return n > h.RetireAtOnce || (held > 1 && float64(n) > h.RetireFraction*float64(held))
|
||||
}
|
||||
|
||||
func (h *Harness) bound(held int) string {
|
||||
return fmt.Sprintf("more than %d at once, or more than %.0f%% of the %d held", h.RetireAtOnce,
|
||||
h.RetireFraction*100, held)
|
||||
}
|
||||
|
||||
// wait holds a stable set over the bound for a person, said once and announced again every
|
||||
// SayAgainEvery so a controller that missed the first hears the next.
|
||||
func (h *Harness) wait(set []string) {
|
||||
now := h.Now()
|
||||
w := h.r.waiting
|
||||
if w == nil {
|
||||
w = &waitingSet{set: set, since: now, held: len(h.applied)}
|
||||
h.r.waiting = w
|
||||
h.say("RETIREMENT WAITS FOR A PERSON: the mesh no longer asks for %d of the %d consumer(s) this provider "+
|
||||
"holds (%s), %s. Nothing is retired; each stays active until `retire approve %s <module>` or "+
|
||||
"`retire reject` (novox/hq ADR 0230)", len(set), w.held, strings.Join(set, ", "), h.bound(w.held), h.Node)
|
||||
} else if now.Sub(w.saidAt) < h.SayAgainEvery {
|
||||
return
|
||||
}
|
||||
w.saidAt = now
|
||||
h.announce(EventRetirement, h.retirementBody(ChangeWaiting, h.named(set), map[string]any{
|
||||
"held": w.held, "bound": h.bound(w.held), "since": stamp(w.since),
|
||||
}))
|
||||
}
|
||||
|
||||
// settle ends a waiting or rejected set that the mesh's own answer made moot.
|
||||
func (h *Harness) settle(why string) {
|
||||
var set []string
|
||||
switch {
|
||||
case h.r.waiting != nil:
|
||||
set = h.r.waiting.set
|
||||
case h.r.rejected != nil:
|
||||
set = h.r.rejected.set
|
||||
case h.r.noneSaid:
|
||||
return
|
||||
default:
|
||||
// A restart forgets a set an earlier process waited on, and so would never say it ended: the
|
||||
// controller's condition would ask a person to approve what nobody waits on any more (issue 937).
|
||||
// The first time this process finds every consumer held asked for, it says so once, with no set.
|
||||
h.r.noneSaid = true
|
||||
h.say("nothing waits for a person: %s", why)
|
||||
h.announce(EventRetirement, h.retirementBody(ChangeSettled, []map[string]any{}, map[string]any{"why": why}))
|
||||
return
|
||||
}
|
||||
h.r.waiting, h.r.rejected = nil, nil
|
||||
h.say("retirement of %s settled without retiring: %s", strings.Join(set, ", "), why)
|
||||
h.announce(EventRetirement, h.retirementBody(ChangeSettled, h.named(set), map[string]any{"why": why}))
|
||||
}
|
||||
|
||||
// retire disables and marks each consumer of set; one that fails stays held and is tried again once
|
||||
// its set is stable again.
|
||||
func (h *Harness) retire(ctx context.Context, set []string, why string, extra map[string]any) []string {
|
||||
now := h.Now()
|
||||
held := len(h.applied)
|
||||
var done []string
|
||||
var said []map[string]any
|
||||
for _, as := range set {
|
||||
was, ok := h.applied[as]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if err := h.Adapter.Retire(ctx, as, was.derived, why, now); err != nil {
|
||||
h.say("%s: retiring failed, will try again: %v", as, err)
|
||||
continue
|
||||
}
|
||||
delete(h.applied, as)
|
||||
delete(h.lost, as)
|
||||
delete(h.failing, as)
|
||||
h.r.retired[as] = retiredEntry{node: was.node, at: now, why: why}
|
||||
h.recovered(as, "retired")
|
||||
h.say("%s: RETIRED — its access disabled and its data kept, marked to delete (%s). Asked for again "+
|
||||
"it is re-enabled as it was; it is deleted only by `cleanup delete` (novox/hq ADR 0230)", as, why)
|
||||
done = append(done, as)
|
||||
said = append(said, map[string]any{"consumer": as, "node": was.node, "retired_at": stamp(now), "why": why})
|
||||
}
|
||||
h.r.key, h.r.count, h.r.waiting, h.r.rejected = "", 0, nil, nil
|
||||
if len(done) > 0 {
|
||||
body := map[string]any{"held": held, "why": why}
|
||||
for k, v := range extra {
|
||||
body[k] = v
|
||||
}
|
||||
h.announce(EventRetirement, h.retirementBody(ChangeRetired, said, body))
|
||||
}
|
||||
return done
|
||||
}
|
||||
|
||||
// reenabled says a retired consumer was asked for again and its create enabled it.
|
||||
func (h *Harness) reenabled(as, node string) {
|
||||
e, was := h.r.retired[as]
|
||||
if !was {
|
||||
return
|
||||
}
|
||||
delete(h.r.retired, as)
|
||||
h.say("%s: asked for again — re-enabled as it was, its mark to delete cleared (retired %s: %s)", as,
|
||||
stamp(e.at), e.why)
|
||||
h.announce(EventRetirement, h.retirementBody(ChangeReenabled, []map[string]any{
|
||||
{"consumer": as, "node": node, "retired_at": stamp(e.at), "why": e.why},
|
||||
}, nil))
|
||||
}
|
||||
|
||||
// Approve retires exactly the set waiting (or the set rejected) — a person's confirmation.
|
||||
func (h *Harness) Approve(ctx context.Context, consumers []string, why, by, via string) ([]string, error) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
h.init()
|
||||
if strings.TrimSpace(why) == "" {
|
||||
return nil, errors.New("approve: say why")
|
||||
}
|
||||
set := sorted(consumers)
|
||||
var held []string
|
||||
switch {
|
||||
case h.r.waiting != nil && same(set, h.r.waiting.set):
|
||||
held = h.r.waiting.set
|
||||
case h.r.rejected != nil && same(set, h.r.rejected.set):
|
||||
held = h.r.rejected.set
|
||||
default:
|
||||
return nil, fmt.Errorf("approve: %s is not the set waiting here — %s", orNone(set), h.waitingWords())
|
||||
}
|
||||
h.say("retirement of %s APPROVED by %s: %s", strings.Join(held, ", "), orSomebody(by), why)
|
||||
h.announce(EventRetirement, h.retirementBody(ChangeApproved, h.named(held),
|
||||
map[string]any{"why": why, "by": by, "via": via}))
|
||||
reason := fmt.Sprintf("the mesh stopped asking for it; approved by %s: %s", orSomebody(by), why)
|
||||
return h.retire(ctx, held, reason, map[string]any{"by": by, "via": via}), nil
|
||||
}
|
||||
|
||||
// Reject keeps the waiting set active; it is not asked about again while it stays the same set.
|
||||
func (h *Harness) Reject(consumers []string, why, by, via string) ([]string, error) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
h.init()
|
||||
if strings.TrimSpace(why) == "" {
|
||||
return nil, errors.New("reject: say why")
|
||||
}
|
||||
set := sorted(consumers)
|
||||
if h.r.waiting == nil || !same(set, h.r.waiting.set) {
|
||||
return nil, fmt.Errorf("reject: %s is not the set waiting here — %s", orNone(set), h.waitingWords())
|
||||
}
|
||||
h.r.rejected = &rejectedSet{set: h.r.waiting.set, by: by, why: why, at: h.Now()}
|
||||
h.r.waiting = nil
|
||||
h.say("retirement of %s REJECTED by %s: %s. Kept active, and not asked about again while the mesh goes on "+
|
||||
"not asking for exactly these (until this provider restarts)", strings.Join(set, ", "), orSomebody(by), why)
|
||||
h.announce(EventRetirement, h.retirementBody(ChangeRejected, h.named(set),
|
||||
map[string]any{"why": why, "by": by, "via": via}))
|
||||
return set, nil
|
||||
}
|
||||
|
||||
// DeleteRetired deletes one retired consumer, by a person's word: never an active one, never one the
|
||||
// mesh asks for.
|
||||
func (h *Harness) DeleteRetired(ctx context.Context, consumer, why, by, via string) (int64, error) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
h.init()
|
||||
if strings.TrimSpace(why) == "" {
|
||||
return 0, errors.New("delete: say why")
|
||||
}
|
||||
if h.r.lastWant[consumer] {
|
||||
return 0, fmt.Errorf("delete: the mesh asks for %s — it is not retired", consumer)
|
||||
}
|
||||
if _, held := h.applied[consumer]; held {
|
||||
return 0, fmt.Errorf("delete: %s is active here — only a retired consumer is deleted", consumer)
|
||||
}
|
||||
inv, err := h.Adapter.Inventory(ctx)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("delete: what the backend holds cannot be read, so nothing is deleted: %w", err)
|
||||
}
|
||||
var found *Retired
|
||||
for i := range inv.Retired {
|
||||
if inv.Retired[i].Consumer == consumer {
|
||||
found = &inv.Retired[i]
|
||||
}
|
||||
}
|
||||
if found == nil {
|
||||
return 0, fmt.Errorf("delete: %s is not retired here — only a retired consumer is deleted", consumer)
|
||||
}
|
||||
freed, err := h.Adapter.Delete(ctx, *found)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
delete(h.r.retired, consumer)
|
||||
h.say("%s: DELETED by %s: %s (retired %s: %s; %d bytes freed)", consumer, orSomebody(by), why,
|
||||
stampOr(found.RetiredAt), found.Why, freed)
|
||||
h.announce(EventRetirement, h.retirementBody(ChangeDeleted, []map[string]any{{
|
||||
"consumer": consumer, "node": found.Node, "retired_at": stampOr(found.RetiredAt), "why": found.Why,
|
||||
"size_bytes": found.SizeBytes, "kind": kindOf(*found),
|
||||
}}, map[string]any{"why": why, "by": by, "via": via, "freed_bytes": freed}))
|
||||
return freed, nil
|
||||
}
|
||||
|
||||
// Retirement is what a person (and the controller's `cleanup list` and its probe) asks: what is
|
||||
// held, waiting, rejected and retired here.
|
||||
func (h *Harness) Retirement(ctx context.Context) (map[string]any, error) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
h.init()
|
||||
inv, err := h.Adapter.Inventory(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var held []string
|
||||
for as := range h.applied {
|
||||
held = append(held, as)
|
||||
}
|
||||
sort.Strings(held)
|
||||
retired := []map[string]any{}
|
||||
for _, r := range inv.Retired {
|
||||
retired = append(retired, map[string]any{"consumer": r.Consumer, "node": r.Node,
|
||||
"retired_at": stampOr(r.RetiredAt), "why": r.Why, "size_bytes": r.SizeBytes, "kind": kindOf(r)})
|
||||
}
|
||||
sizes := map[string]int64{}
|
||||
for _, as := range held {
|
||||
if size, ok := inv.Sizes[as]; ok && size >= 0 {
|
||||
sizes[as] = size
|
||||
}
|
||||
}
|
||||
out := map[string]any{"resource": h.Resource, "node": h.Node, "held": orEmptyList(held), "held_sizes": sizes,
|
||||
"stable_passes": h.StablePasses, "stable_for_seconds": int(h.StableFor.Seconds()), "bound": h.bound(len(h.applied)), "waiting": nil, "rejected": nil,
|
||||
"retired": retired}
|
||||
if w := h.r.waiting; w != nil {
|
||||
out["waiting"] = map[string]any{"consumers": h.named(w.set), "since": stamp(w.since), "held": w.held}
|
||||
}
|
||||
if r := h.r.rejected; r != nil {
|
||||
out["rejected"] = map[string]any{"consumers": h.named(r.set), "by": r.by, "why": r.why, "at": stamp(r.at)}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (h *Harness) waitingWords() string {
|
||||
switch {
|
||||
case h.r.waiting != nil:
|
||||
return "waiting: " + strings.Join(h.r.waiting.set, ", ")
|
||||
case h.r.rejected != nil:
|
||||
return "nothing waits; rejected and kept: " + strings.Join(h.r.rejected.set, ", ")
|
||||
}
|
||||
return "nothing waits for a person here"
|
||||
}
|
||||
|
||||
// named is a set with each consumer's machine, as the events and the tools say it.
|
||||
func (h *Harness) named(set []string) []map[string]any {
|
||||
out := make([]map[string]any, 0, len(set))
|
||||
for _, as := range set {
|
||||
out = append(out, map[string]any{"consumer": as, "node": h.applied[as].node})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (h *Harness) retirementBody(change string, consumers []map[string]any, extra map[string]any) map[string]any {
|
||||
body := map[string]any{"provider": h.Resource, "provider-node": h.Node, "change": change,
|
||||
"consumers": consumers, "at": stamp(h.Now())}
|
||||
for k, v := range extra {
|
||||
body[k] = v
|
||||
}
|
||||
return body
|
||||
}
|
||||
|
||||
// RetirementTools are the four tools every provider serves for retirement, the same names in every
|
||||
// module: a caller asks them on the provider's machine, with the operator's answer named in why and by.
|
||||
func RetirementTools(h *Harness) []stdio.Tool {
|
||||
if h == nil {
|
||||
return nil
|
||||
}
|
||||
ask := func() (context.Context, context.CancelFunc) {
|
||||
return context.WithTimeout(context.Background(), 2*time.Minute)
|
||||
}
|
||||
who := map[string]any{
|
||||
"why": map[string]any{"type": "string", "description": "why — required, kept in the hand-act log"},
|
||||
"by": map[string]any{"type": "string", "description": "who decided"},
|
||||
"via": map[string]any{"type": "string", "description": "the part or session that asks, when not the caller itself"},
|
||||
}
|
||||
withSet := map[string]any{"consumers": map[string]any{"type": "array", "items": map[string]any{"type": "string"},
|
||||
"description": "the set, exactly as provisioner_retirement names it"}}
|
||||
for k, v := range who {
|
||||
withSet[k] = v
|
||||
}
|
||||
withOne := map[string]any{
|
||||
"consumer": map[string]any{"type": "string", "description": "the retired consumer to delete"},
|
||||
"confirm": map[string]any{"type": "string", "description": "the consumer's name again, to say this is meant"},
|
||||
}
|
||||
for k, v := range who {
|
||||
withOne[k] = v
|
||||
}
|
||||
return []stdio.Tool{
|
||||
{Name: "provisioner_retirement",
|
||||
Description: "What this provider holds, what waits for a person to approve its retirement, what was rejected, " +
|
||||
"and every retired consumer with when, why and its size (novox/hq ADR 0230).",
|
||||
Run: func(map[string]any) (any, error) {
|
||||
ctx, cancel := ask()
|
||||
defer cancel()
|
||||
return h.Retirement(ctx)
|
||||
}},
|
||||
{Name: "provisioner_retire_approve",
|
||||
Description: "Retire exactly the set waiting for a person (or the set rejected earlier): access disabled, data " +
|
||||
"kept and marked to delete. It asks nobody: the operator's answer on the operator channel to the provisioner's " +
|
||||
"ask (novox/hq ADR 0570 §5) is named in why and by, which the hand-act log keeps.",
|
||||
Input: withSet,
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
ctx, cancel := ask()
|
||||
defer cancel()
|
||||
done, err := h.Approve(ctx, strs(args["consumers"]), argString(args, "why"), argString(args, "by"), argString(args, "via"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"retired": orEmptyList(done)}, nil
|
||||
}},
|
||||
{Name: "provisioner_retire_reject",
|
||||
Description: "Keep the set waiting for a person active. It asks nobody: the operator's answer on the operator channel " +
|
||||
"(novox/hq ADR 0570 §5) is named in why and by, which the hand-act log keeps.",
|
||||
Input: withSet,
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
kept, err := h.Reject(strs(args["consumers"]), argString(args, "why"), argString(args, "by"), argString(args, "via"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"kept": kept}, nil
|
||||
}},
|
||||
{Name: "provisioner_delete",
|
||||
Description: "Delete one RETIRED consumer — its login and its data — for good. Refused for anything active or " +
|
||||
"asked for. It asks nobody: ask the operator on the operator channel first, and name that answer in why and " +
|
||||
"by, which the hand-act log keeps (novox/hq ADR 0604 §8).",
|
||||
Input: withOne,
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
consumer := argString(args, "consumer")
|
||||
if consumer == "" || argString(args, "confirm") != consumer {
|
||||
return nil, errors.New("delete: name the consumer, and repeat it in confirm")
|
||||
}
|
||||
ctx, cancel := ask()
|
||||
defer cancel()
|
||||
freed, err := h.DeleteRetired(ctx, consumer, argString(args, "why"), argString(args, "by"), argString(args, "via"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"deleted": consumer, "freed_bytes": freed}, nil
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
func strs(v any) []string {
|
||||
list, _ := v.([]any)
|
||||
out := []string{}
|
||||
for _, x := range list {
|
||||
if s, ok := x.(string); ok && s != "" {
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func sorted(list []string) []string {
|
||||
out := append([]string(nil), list...)
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func same(a, b []string) bool {
|
||||
return strings.Join(sorted(a), "\x00") == strings.Join(sorted(b), "\x00")
|
||||
}
|
||||
|
||||
func orNone(set []string) string {
|
||||
if len(set) == 0 {
|
||||
return "an empty set"
|
||||
}
|
||||
return strings.Join(set, ", ")
|
||||
}
|
||||
|
||||
func orSomebody(by string) string {
|
||||
if by == "" {
|
||||
return "a person"
|
||||
}
|
||||
return by
|
||||
}
|
||||
|
||||
func orEmptyList(list []string) []string {
|
||||
if list == nil {
|
||||
return []string{}
|
||||
}
|
||||
return list
|
||||
}
|
||||
|
||||
func kindOf(r Retired) string {
|
||||
if r.Kind == "" {
|
||||
return KindConsumer
|
||||
}
|
||||
return r.Kind
|
||||
}
|
||||
|
||||
func stamp(t time.Time) string { return t.UTC().Format(time.RFC3339) }
|
||||
|
||||
func stampOr(t time.Time) string {
|
||||
if t.IsZero() {
|
||||
return ""
|
||||
}
|
||||
return stamp(t)
|
||||
}
|
||||
|
||||
func argString(args map[string]any, key string) string {
|
||||
s, _ := args[key].(string)
|
||||
return s
|
||||
}
|
||||
@@ -1,596 +0,0 @@
|
||||
package harness
|
||||
|
||||
// Retirement (novox/hq ADR 0230), as the shared loop does it: a consumer the mesh stops asking for is
|
||||
// retired only after the same result in five consecutive passes, too many at once wait for a person,
|
||||
// asked for again it is re-enabled, and only a person deletes. The same file in every Go provider.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// recorder is a backend that remembers what it holds, active and retired, as a real one's mark does.
|
||||
type recorder struct {
|
||||
created []Provision
|
||||
removed []string // retired, in order
|
||||
deleted []string
|
||||
held bool
|
||||
failing error
|
||||
holdsErr error
|
||||
retErr error
|
||||
inv Inventory
|
||||
invErr error
|
||||
}
|
||||
|
||||
func (r *recorder) Create(_ context.Context, p Provision) error {
|
||||
if r.failing != nil {
|
||||
return r.failing
|
||||
}
|
||||
r.created = append(r.created, p)
|
||||
r.inv.Retired = withoutRetired(r.inv.Retired, p.As)
|
||||
if !listHas(r.inv.Active, p.As) {
|
||||
r.inv.Active = append(r.inv.Active, p.As)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *recorder) Retire(_ context.Context, as string, _ map[string]any, why string, at time.Time) error {
|
||||
if r.retErr != nil {
|
||||
return r.retErr
|
||||
}
|
||||
r.removed = append(r.removed, as)
|
||||
r.inv.Active = without(r.inv.Active, as)
|
||||
r.inv.Retired = append(withoutRetired(r.inv.Retired, as),
|
||||
Retired{Consumer: as, RetiredAt: at, Why: why, SizeBytes: 42, Kind: KindConsumer})
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *recorder) Holds(context.Context, Provision) (bool, error) {
|
||||
if r.holdsErr != nil {
|
||||
return false, r.holdsErr
|
||||
}
|
||||
return r.held, nil
|
||||
}
|
||||
|
||||
func (r *recorder) Inventory(context.Context) (Inventory, error) { return r.inv, r.invErr }
|
||||
|
||||
func (r *recorder) Delete(_ context.Context, x Retired) (int64, error) {
|
||||
r.deleted = append(r.deleted, x.Consumer)
|
||||
r.inv.Retired = withoutRetired(r.inv.Retired, x.Consumer)
|
||||
return x.SizeBytes, nil
|
||||
}
|
||||
|
||||
func listHas(list []string, s string) bool {
|
||||
for _, x := range list {
|
||||
if x == s {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func without(list []string, s string) []string {
|
||||
var out []string
|
||||
for _, x := range list {
|
||||
if x != s {
|
||||
out = append(out, x)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func withoutRetired(list []Retired, s string) []Retired {
|
||||
var out []Retired
|
||||
for _, x := range list {
|
||||
if x.Consumer != s {
|
||||
out = append(out, x)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// holding gives the provider n consumers c1…cn and applies them.
|
||||
func holding(w *world, n int) []map[string]any {
|
||||
var given []map[string]any
|
||||
for i := 1; i <= n; i++ {
|
||||
given = append(given, map[string]any{"as": fmt.Sprintf("c%d", i), "node": "anchor"})
|
||||
}
|
||||
w.give(given...)
|
||||
w.h.Reconcile(ctx)
|
||||
return given
|
||||
}
|
||||
|
||||
// pass is one reconcile five seconds after the last.
|
||||
func (w *world) pass() {
|
||||
w.now = w.now.Add(5 * time.Second)
|
||||
w.h.Reconcile(ctx)
|
||||
}
|
||||
|
||||
func retirements(said []announced) []string {
|
||||
var out []string
|
||||
for _, a := range said {
|
||||
if a.event == EventRetirement {
|
||||
out = append(out, a.body["change"].(string))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func noneWaits(a announced) bool {
|
||||
c, _ := a.body["consumers"].([]map[string]any)
|
||||
return a.body["change"] == ChangeSettled && len(c) == 0
|
||||
}
|
||||
|
||||
func lastRetirement(t *testing.T, said []announced) map[string]any {
|
||||
t.Helper()
|
||||
for i := len(said) - 1; i >= 0; i-- {
|
||||
if said[i].event == EventRetirement {
|
||||
return said[i].body
|
||||
}
|
||||
}
|
||||
t.Fatal("nothing about retirement was announced")
|
||||
return nil
|
||||
}
|
||||
|
||||
func namesOf(body map[string]any) string {
|
||||
var out []string
|
||||
for _, c := range body["consumers"].([]map[string]any) {
|
||||
out = append(out, c["consumer"].(string))
|
||||
}
|
||||
return strings.Join(out, ",")
|
||||
}
|
||||
|
||||
// settle passes every five seconds until the same answer has held for StableFor, and one pass more.
|
||||
func (w *world) settle() { w.passes(StableFor + 5*time.Second) }
|
||||
|
||||
func testATransientEmptyListForFourPassesRetiresNothing(t *testing.T) {
|
||||
w, said := standingWorld(t)
|
||||
given := holding(w, 1)
|
||||
w.give()
|
||||
for i := 0; i < 4; i++ {
|
||||
w.pass()
|
||||
}
|
||||
w.give(given...)
|
||||
w.pass()
|
||||
if len(w.a.removed) != 0 || len(retirements(*said)) != 0 {
|
||||
t.Fatalf("four passes retired %v and said %v", w.a.removed, retirements(*said))
|
||||
}
|
||||
}
|
||||
|
||||
// Five identical passes are twenty-five seconds — shorter than a controller restart. Both must hold:
|
||||
// five passes AND ten minutes of the same answer (novox/hq ADR 0230).
|
||||
func testFivePassesInTwentyFiveSecondsRetireNothingTenMinutesDo(t *testing.T) {
|
||||
w, said := standingWorld(t)
|
||||
holding(w, 1)
|
||||
w.give()
|
||||
for i := 0; i < 5; i++ {
|
||||
w.pass()
|
||||
}
|
||||
if len(w.a.removed) != 0 || len(retirements(*said)) != 0 {
|
||||
t.Fatalf("five passes in 25 s retired %v", w.a.removed)
|
||||
}
|
||||
w.passes(StableFor - 30*time.Second)
|
||||
if len(w.a.removed) != 0 {
|
||||
t.Fatalf("retired before the set held %s: %v", StableFor, w.a.removed)
|
||||
}
|
||||
w.passes(time.Minute)
|
||||
if strings.Join(w.a.removed, ",") != "c1" {
|
||||
t.Fatalf("the same set held %s and was not retired: %v", StableFor, w.a.removed)
|
||||
}
|
||||
// Ten minutes in one slow pass are not five passes.
|
||||
w2 := newWorld(t)
|
||||
holding(w2, 1)
|
||||
w2.give()
|
||||
w2.pass()
|
||||
w2.now = w2.now.Add(StableFor)
|
||||
w2.pass()
|
||||
if len(w2.a.removed) != 0 {
|
||||
t.Fatalf("two passes ten minutes apart retired %v", w2.a.removed)
|
||||
}
|
||||
w2.passes(15 * time.Second)
|
||||
if len(w2.a.removed) != 1 {
|
||||
t.Fatalf("five passes over ten minutes did not retire: %v", w2.a.removed)
|
||||
}
|
||||
}
|
||||
|
||||
func testAStableSetIsRetiredAndAskedAgainReEnables(t *testing.T) {
|
||||
w, said := standingWorld(t)
|
||||
given := holding(w, 3)
|
||||
w.give(given[:2]...)
|
||||
w.settle()
|
||||
if strings.Join(w.a.removed, ",") != "c3" {
|
||||
t.Fatalf("retired %v", w.a.removed)
|
||||
}
|
||||
body := lastRetirement(t, *said)
|
||||
if body["change"] != ChangeRetired || namesOf(body) != "c3" || body["held"] != 3 || body["provider-node"] != "anchor" ||
|
||||
!strings.Contains(body["why"].(string), "consecutive passes over 10m") {
|
||||
t.Fatalf("%v", body)
|
||||
}
|
||||
if len(w.a.inv.Retired) != 1 || w.a.inv.Retired[0].Consumer != "c3" {
|
||||
t.Fatalf("the backend does not hold it retired: %+v", w.a.inv)
|
||||
}
|
||||
// Asked for again: the ordinary create, on the first pass, and said.
|
||||
created := len(w.a.created)
|
||||
w.give(given...)
|
||||
w.pass()
|
||||
if len(w.a.created) != created+1 || w.a.created[created].As != "c3" {
|
||||
t.Fatalf("not created again: %v", w.a.created)
|
||||
}
|
||||
if body := lastRetirement(t, *said); body["change"] != ChangeReenabled || namesOf(body) != "c3" {
|
||||
t.Fatalf("%v", body)
|
||||
}
|
||||
if len(w.a.inv.Retired) != 0 {
|
||||
t.Fatalf("still marked retired: %+v", w.a.inv.Retired)
|
||||
}
|
||||
}
|
||||
|
||||
func testAnUnreadablePassStartsTheCountAgain(t *testing.T) {
|
||||
w := newWorld(t)
|
||||
holding(w, 1)
|
||||
w.give()
|
||||
w.passes(StableFor - time.Minute)
|
||||
os.WriteFile(w.receives, []byte("{"), 0o600)
|
||||
w.pass()
|
||||
w.give()
|
||||
w.passes(StableFor - time.Minute)
|
||||
if len(w.a.removed) != 0 {
|
||||
t.Fatalf("an unreadable pass counted: %v", w.a.removed)
|
||||
}
|
||||
w.passes(2 * time.Minute)
|
||||
if len(w.a.removed) != 1 {
|
||||
t.Fatalf("not retired after ten minutes of readable passes: %v", w.a.removed)
|
||||
}
|
||||
}
|
||||
|
||||
func testADifferentSetStartsTheCountAgain(t *testing.T) {
|
||||
w := newWorld(t)
|
||||
given := holding(w, 5)
|
||||
w.give(given[:4]...)
|
||||
w.passes(StableFor - time.Minute)
|
||||
w.give(given[:3]...)
|
||||
w.passes(StableFor - time.Minute)
|
||||
if len(w.a.removed) != 0 {
|
||||
t.Fatalf("a changed set kept its count: %v", w.a.removed)
|
||||
}
|
||||
w.passes(2 * time.Minute)
|
||||
if strings.Join(w.a.removed, ",") != "c4,c5" {
|
||||
t.Fatalf("%v", w.a.removed)
|
||||
}
|
||||
}
|
||||
|
||||
func testAdditionsAndChangesAreNeverDelayed(t *testing.T) {
|
||||
w := newWorld(t)
|
||||
holding(w, 1)
|
||||
w.give(map[string]any{"as": "c1", "node": "anchor"}, map[string]any{"as": "c2"})
|
||||
w.pass()
|
||||
if len(w.a.created) != 2 || w.a.created[1].As != "c2" {
|
||||
t.Fatalf("an addition waited: %v", w.a.created)
|
||||
}
|
||||
w.give(map[string]any{"as": "c1", "node": "anchor", "values": map[string]any{"name": "rotated"}}, map[string]any{"as": "c2"})
|
||||
w.pass()
|
||||
if len(w.a.created) != 3 || w.a.created[2].As != "c1" {
|
||||
t.Fatalf("a change waited: %v", w.a.created)
|
||||
}
|
||||
}
|
||||
|
||||
func testTooManyWaitsForAPersonAndApproveRetiresExactlyThatSet(t *testing.T) {
|
||||
w, said := standingWorld(t)
|
||||
holding(w, 4)
|
||||
w.give()
|
||||
w.passes(15 * time.Minute)
|
||||
if len(w.a.removed) != 0 {
|
||||
t.Fatalf("four of four were retired without a person: %v", w.a.removed)
|
||||
}
|
||||
if got := strings.Join(retirements(*said), ","); got != ChangeWaiting {
|
||||
t.Fatalf("said %q, want one waiting", got)
|
||||
}
|
||||
body := lastRetirement(t, *said)
|
||||
if namesOf(body) != "c1,c2,c3,c4" || body["held"] != 4 || body["bound"] == "" {
|
||||
t.Fatalf("%v", body)
|
||||
}
|
||||
if !strings.Contains(strings.Join(w.said, "\n"), "RETIREMENT WAITS FOR A PERSON") {
|
||||
t.Fatal("the wait was not said")
|
||||
}
|
||||
// Said again every quarter of an hour while it waits.
|
||||
w.passes(11 * time.Minute)
|
||||
if got := strings.Join(retirements(*said), ","); got != "waiting,waiting" {
|
||||
t.Fatalf("%q", got)
|
||||
}
|
||||
|
||||
if _, err := w.h.Approve(ctx, []string{"c1", "c2"}, "tidy", "operator", "mesh-controller"); err == nil {
|
||||
t.Fatal("approved a set that is not the one waiting")
|
||||
}
|
||||
if _, err := w.h.Approve(ctx, []string{"c4", "c3", "c2", "c1"}, "", "operator", ""); err == nil {
|
||||
t.Fatal("approved without a why")
|
||||
}
|
||||
done, err := w.h.Approve(ctx, []string{"c4", "c3", "c2", "c1"}, "the old machine is gone", "operator", "mesh-controller")
|
||||
if err != nil || strings.Join(done, ",") != "c1,c2,c3,c4" || strings.Join(w.a.removed, ",") != "c1,c2,c3,c4" {
|
||||
t.Fatal(done, err, w.a.removed)
|
||||
}
|
||||
changes := retirements(*said)
|
||||
if strings.Join(changes[len(changes)-2:], ",") != "approved,retired" {
|
||||
t.Fatalf("%v", changes)
|
||||
}
|
||||
if b := lastRetirement(t, *said); b["by"] != "operator" || b["via"] != "mesh-controller" ||
|
||||
!strings.Contains(b["why"].(string), "the old machine is gone") {
|
||||
t.Fatalf("%v", b)
|
||||
}
|
||||
// Nothing more waits.
|
||||
w.passes(time.Minute)
|
||||
if r, _ := w.h.Retirement(ctx); r["waiting"] != nil || len(r["retired"].([]map[string]any)) != 4 {
|
||||
t.Fatalf("%v", r)
|
||||
}
|
||||
}
|
||||
|
||||
func testRejectedIsKeptAndNotAskedAgainUntilTheSetChanges(t *testing.T) {
|
||||
w, said := standingWorld(t)
|
||||
given := holding(w, 4)
|
||||
w.give()
|
||||
w.settle()
|
||||
if _, err := w.h.Reject([]string{"c1"}, "no", "operator", ""); err == nil {
|
||||
t.Fatal("rejected a set that is not the one waiting")
|
||||
}
|
||||
kept, err := w.h.Reject([]string{"c1", "c2", "c3", "c4"}, "a person is moving them", "operator", "mesh-controller")
|
||||
if err != nil || len(kept) != 4 {
|
||||
t.Fatal(kept, err)
|
||||
}
|
||||
w.passes(time.Hour)
|
||||
if len(w.a.removed) != 0 {
|
||||
t.Fatalf("a rejected set was retired: %v", w.a.removed)
|
||||
}
|
||||
if got := strings.Join(retirements(*said), ","); got != "waiting,rejected" {
|
||||
t.Fatalf("asked again after a rejection: %q", got)
|
||||
}
|
||||
r, _ := w.h.Retirement(ctx)
|
||||
if r["rejected"] == nil || r["waiting"] != nil {
|
||||
t.Fatalf("%v", r)
|
||||
}
|
||||
// One of them asked for again: a different answer, so the rejection is settled and counting
|
||||
// starts over — three of four is over the bound again, and waits again.
|
||||
w.give(given[0])
|
||||
w.pass()
|
||||
if got := strings.Join(retirements(*said), ","); got != "waiting,rejected,settled" {
|
||||
t.Fatalf("%q", got)
|
||||
}
|
||||
w.settle()
|
||||
if got := strings.Join(retirements(*said), ","); got != "waiting,rejected,settled,waiting" {
|
||||
t.Fatalf("%q", got)
|
||||
}
|
||||
// A rejected set can still be approved later.
|
||||
w2, _ := standingWorld(t)
|
||||
holding(w2, 4)
|
||||
w2.give()
|
||||
w2.settle()
|
||||
w2.h.Reject([]string{"c1", "c2", "c3", "c4"}, "wait", "operator", "")
|
||||
if done, err := w2.h.Approve(ctx, []string{"c1", "c2", "c3", "c4"}, "now", "operator", ""); err != nil || len(done) != 4 {
|
||||
t.Fatal(done, err)
|
||||
}
|
||||
}
|
||||
|
||||
func testAWaitingSetAskedForAgainSettles(t *testing.T) {
|
||||
w, said := standingWorld(t)
|
||||
given := holding(w, 4)
|
||||
w.give()
|
||||
w.settle()
|
||||
w.give(given...)
|
||||
w.pass()
|
||||
if got := strings.Join(retirements(*said), ","); got != "waiting,settled" || len(w.a.removed) != 0 {
|
||||
t.Fatalf("%q %v", got, w.a.removed)
|
||||
}
|
||||
if _, err := w.h.Approve(ctx, []string{"c1", "c2", "c3", "c4"}, "late", "operator", ""); err == nil {
|
||||
t.Fatal("approved a set that no longer waits")
|
||||
}
|
||||
}
|
||||
|
||||
func testDeleteRemovesOnlyThatRetiredConsumer(t *testing.T) {
|
||||
w, said := standingWorld(t)
|
||||
given := holding(w, 5)
|
||||
w.give(given[:3]...)
|
||||
w.settle()
|
||||
if strings.Join(w.a.removed, ",") != "c4,c5" {
|
||||
t.Fatalf("%v", w.a.removed)
|
||||
}
|
||||
if _, err := w.h.DeleteRetired(ctx, "c1", "tidy", "operator", ""); err == nil {
|
||||
t.Fatal("deleted an active consumer")
|
||||
}
|
||||
if _, err := w.h.DeleteRetired(ctx, "c5", "", "operator", ""); err == nil {
|
||||
t.Fatal("deleted without a why")
|
||||
}
|
||||
if _, err := w.h.DeleteRetired(ctx, "nobody", "tidy", "operator", ""); err == nil {
|
||||
t.Fatal("deleted something not retired")
|
||||
}
|
||||
freed, err := w.h.DeleteRetired(ctx, "c5", "the experiment is over", "operator", "mesh-controller")
|
||||
if err != nil || freed != 42 || strings.Join(w.a.deleted, ",") != "c5" {
|
||||
t.Fatal(freed, err, w.a.deleted)
|
||||
}
|
||||
if b := lastRetirement(t, *said); b["change"] != ChangeDeleted || namesOf(b) != "c5" || b["freed_bytes"] != int64(42) {
|
||||
t.Fatalf("%v", b)
|
||||
}
|
||||
r, _ := w.h.Retirement(ctx)
|
||||
if left := r["retired"].([]map[string]any); len(left) != 1 || left[0]["consumer"] != "c4" {
|
||||
t.Fatalf("%v", r)
|
||||
}
|
||||
// Retired and asked for again before anybody deleted it: refused, it is the mesh's again.
|
||||
w.give(given[:4]...)
|
||||
w.pass()
|
||||
if _, err := w.h.DeleteRetired(ctx, "c4", "tidy", "operator", ""); err == nil {
|
||||
t.Fatal("deleted a consumer the mesh asks for")
|
||||
}
|
||||
}
|
||||
|
||||
func testTheBound(t *testing.T) {
|
||||
h := &Harness{}
|
||||
h.init()
|
||||
for _, c := range []struct{ n, held int }{{1, 1}, {1, 2}, {1, 3}, {3, 7}, {3, 6}, {2, 5}} {
|
||||
if h.overTheBound(c.n, c.held) {
|
||||
t.Errorf("%d of %d waits for a person", c.n, c.held)
|
||||
}
|
||||
}
|
||||
for _, c := range []struct{ n, held int }{{2, 2}, {2, 3}, {4, 7}, {4, 10}, {7, 7}} {
|
||||
if !h.overTheBound(c.n, c.held) {
|
||||
t.Errorf("%d of %d is retired without a person", c.n, c.held)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func testARestartRetiresWhatTheBackendHoldsUnaskedAndAdoptsWhatItFindsDisabled(t *testing.T) {
|
||||
w, said := standingWorld(t)
|
||||
w.a.inv = Inventory{Active: []string{"kept", "orphan"}, Retired: []Retired{
|
||||
{Consumer: "locked-before", SizeBytes: 7, Kind: KindConsumer},
|
||||
{Consumer: "old_deleted_20261005", RetiredAt: time.Date(2026, 10, 5, 0, 0, 0, 0, time.UTC), Kind: "set-aside-database"},
|
||||
}}
|
||||
w.give(map[string]any{"as": "kept"})
|
||||
w.h.Reconcile(ctx)
|
||||
if b := lastRetirement(t, *said); b["change"] != ChangeAdopted || namesOf(b) != "locked-before" {
|
||||
t.Fatalf("%v", b)
|
||||
}
|
||||
if strings.Join(w.a.removed, ",") != "locked-before" {
|
||||
t.Fatalf("adopting did not mark it: %v", w.a.removed)
|
||||
}
|
||||
w.settle()
|
||||
if strings.Join(w.a.removed, ",") != "locked-before,orphan" {
|
||||
t.Fatalf("an orphan the backend holds was not retired: %v", w.a.removed)
|
||||
}
|
||||
}
|
||||
|
||||
func testABackendThatCannotBeListedIsSaidAndAskedAgain(t *testing.T) {
|
||||
w := newWorld(t)
|
||||
w.a.invErr = errors.New("connection refused")
|
||||
holding(w, 1)
|
||||
if !strings.Contains(strings.Join(w.said, "\n"), "cannot list what the backend holds") {
|
||||
t.Fatal(w.said)
|
||||
}
|
||||
w.a.invErr = nil
|
||||
w.a.inv = Inventory{Active: []string{"c1", "orphan"}}
|
||||
w.pass()
|
||||
w.settle()
|
||||
if strings.Join(w.a.removed, ",") != "orphan" {
|
||||
t.Fatalf("%v", w.a.removed)
|
||||
}
|
||||
}
|
||||
|
||||
func testTheToolsAnswer(t *testing.T) {
|
||||
w, _ := standingWorld(t)
|
||||
holding(w, 4)
|
||||
w.give()
|
||||
w.settle()
|
||||
tools := map[string]func(map[string]any) (any, error){}
|
||||
for _, tool := range RetirementTools(w.h) {
|
||||
tools[tool.Name] = tool.Run
|
||||
// L3: the delete, approve and reject name no removed controller verb, and say where the operator answers (ADR 0604 §8).
|
||||
if tool.Name != "provisioner_retirement" && (strings.Contains(tool.Description, "controller") || !strings.Contains(tool.Description, "operator channel")) {
|
||||
t.Errorf("%s says %q", tool.Name, tool.Description)
|
||||
}
|
||||
}
|
||||
if len(tools) != 4 {
|
||||
t.Fatalf("%d tools", len(tools))
|
||||
}
|
||||
got, err := tools["provisioner_retirement"](nil)
|
||||
if err != nil || got.(map[string]any)["waiting"] == nil {
|
||||
t.Fatal(got, err)
|
||||
}
|
||||
set := []any{"c1", "c2", "c3", "c4"}
|
||||
if _, err := tools["provisioner_retire_approve"](map[string]any{"consumers": set}); err == nil {
|
||||
t.Fatal("approved without a why")
|
||||
}
|
||||
if _, err := tools["provisioner_retire_approve"](map[string]any{"consumers": set, "why": "gone", "by": "operator"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := tools["provisioner_delete"](map[string]any{"consumer": "c1", "why": "gone"}); err == nil {
|
||||
t.Fatal("deleted without confirm")
|
||||
}
|
||||
if _, err := tools["provisioner_delete"](map[string]any{"consumer": "c1", "confirm": "c1", "why": "gone"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Join(w.a.deleted, ",") != "c1" {
|
||||
t.Fatal(w.a.deleted)
|
||||
}
|
||||
}
|
||||
|
||||
func testAFailingConsumerRetiredIsSaidRecovered(t *testing.T) {
|
||||
w, said := standingWorld(t)
|
||||
given := holding(w, 2)
|
||||
w.a.held = false
|
||||
w.a.failing = errors.New("boom")
|
||||
w.passes(7 * time.Minute)
|
||||
w.give(given[0])
|
||||
w.settle()
|
||||
recovered := false
|
||||
for _, a := range *said {
|
||||
if a.event == EventRecovered && a.body["consumer"] == "c2" && a.body["why"] == "retired" {
|
||||
recovered = true
|
||||
}
|
||||
}
|
||||
if !recovered {
|
||||
t.Fatalf("%v", *said)
|
||||
}
|
||||
}
|
||||
|
||||
// everyWord is a world that keeps every word the provider announces.
|
||||
func everyWord(t *testing.T) (*world, *[]announced) {
|
||||
w := newWorld(t)
|
||||
var said []announced
|
||||
w.h.Announce = func(e string, b map[string]any) { said = append(said, announced{e, b}) }
|
||||
w.h.Node = "anchor"
|
||||
return w, &said
|
||||
}
|
||||
|
||||
// A restarted provider that finds every consumer it holds asked for says once that nothing waits, so the controller
|
||||
// clears a waiting condition an earlier process raised and this one never knew (issue 937); it says it no more.
|
||||
func testARestartedProviderSaysOnceThatNothingWaits(t *testing.T) {
|
||||
w, said := everyWord(t)
|
||||
holding(w, 2)
|
||||
w.pass()
|
||||
w.pass()
|
||||
var words []announced
|
||||
for _, a := range *said {
|
||||
if a.event == EventRetirement {
|
||||
words = append(words, a)
|
||||
}
|
||||
}
|
||||
if len(words) != 1 || !noneWaits(words[0]) || words[0].body["provider-node"] != w.h.Node {
|
||||
t.Fatalf("said %+v, want one settled with no set", words)
|
||||
}
|
||||
}
|
||||
|
||||
// A provider whose own set waits does not say that nothing waits.
|
||||
func testAProviderThatWaitsDoesNotSayNothingWaits(t *testing.T) {
|
||||
w, said := everyWord(t)
|
||||
w.a.inv.Active = []string{"c1", "c2", "c3", "c4", "c5"}
|
||||
w.give(map[string]any{"as": "c1", "node": "anchor"})
|
||||
w.settle()
|
||||
for _, a := range *said {
|
||||
if a.event == EventRetirement && noneWaits(a) {
|
||||
t.Fatalf("said nothing waits while %v waited", w.h.r.waiting)
|
||||
}
|
||||
}
|
||||
if w.h.r.waiting == nil {
|
||||
t.Fatal("nothing waited: the test does not hold the case")
|
||||
}
|
||||
}
|
||||
|
||||
// TestRetirement holds this file's cases, each a subtest of its own name (mesh-sdk's test budget: at most 500 unit tests).
|
||||
func TestRetirement(t *testing.T) {
|
||||
t.Run("ATransientEmptyListForFourPassesRetiresNothing", testATransientEmptyListForFourPassesRetiresNothing)
|
||||
t.Run("FivePassesInTwentyFiveSecondsRetireNothingTenMinutesDo", testFivePassesInTwentyFiveSecondsRetireNothingTenMinutesDo)
|
||||
t.Run("AStableSetIsRetiredAndAskedAgainReEnables", testAStableSetIsRetiredAndAskedAgainReEnables)
|
||||
t.Run("AnUnreadablePassStartsTheCountAgain", testAnUnreadablePassStartsTheCountAgain)
|
||||
t.Run("ADifferentSetStartsTheCountAgain", testADifferentSetStartsTheCountAgain)
|
||||
t.Run("AdditionsAndChangesAreNeverDelayed", testAdditionsAndChangesAreNeverDelayed)
|
||||
t.Run("TooManyWaitsForAPersonAndApproveRetiresExactlyThatSet", testTooManyWaitsForAPersonAndApproveRetiresExactlyThatSet)
|
||||
t.Run("RejectedIsKeptAndNotAskedAgainUntilTheSetChanges", testRejectedIsKeptAndNotAskedAgainUntilTheSetChanges)
|
||||
t.Run("AWaitingSetAskedForAgainSettles", testAWaitingSetAskedForAgainSettles)
|
||||
t.Run("DeleteRemovesOnlyThatRetiredConsumer", testDeleteRemovesOnlyThatRetiredConsumer)
|
||||
t.Run("TheBound", testTheBound)
|
||||
t.Run("ARestartRetiresWhatTheBackendHoldsUnaskedAndAdoptsWhatItFindsDisabled", testARestartRetiresWhatTheBackendHoldsUnaskedAndAdoptsWhatItFindsDisabled)
|
||||
t.Run("ABackendThatCannotBeListedIsSaidAndAskedAgain", testABackendThatCannotBeListedIsSaidAndAskedAgain)
|
||||
t.Run("TheToolsAnswer", testTheToolsAnswer)
|
||||
t.Run("AFailingConsumerRetiredIsSaidRecovered", testAFailingConsumerRetiredIsSaidRecovered)
|
||||
t.Run("ARestartedProviderSaysOnceThatNothingWaits", testARestartedProviderSaysOnceThatNothingWaits)
|
||||
t.Run("AProviderThatWaitsDoesNotSayNothingWaits", testAProviderThatWaitsDoesNotSayNothingWaits)
|
||||
}
|
||||
@@ -1,205 +0,0 @@
|
||||
package harness
|
||||
|
||||
// A provider that keeps failing a consumer says so on the bus (novox/hq ADR 0224): not on the first
|
||||
// failure, which may be a restart; after FailingAfter of failures with no success between; again
|
||||
// every SayAgainEvery while it lasts; and recovered on the first success, or when the consumer goes.
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type announced struct {
|
||||
event string
|
||||
body map[string]any
|
||||
}
|
||||
|
||||
func standingWorld(t *testing.T) (*world, *[]announced) {
|
||||
w := newWorld(t)
|
||||
var said []announced
|
||||
w.h.Announce = func(e string, b map[string]any) {
|
||||
// The first success since start, and the first word that nothing waits (issue 937), are their own
|
||||
// tests'; every other test reads past them.
|
||||
if c, _ := b["consumers"].([]map[string]any); b["why"] != "first-success" &&
|
||||
!(e == EventRetirement && b["change"] == ChangeSettled && len(c) == 0) {
|
||||
said = append(said, announced{e, b})
|
||||
}
|
||||
}
|
||||
w.h.Node = "anchor"
|
||||
return w, &said
|
||||
}
|
||||
|
||||
// passes reconciles every five seconds for d, as Run would.
|
||||
func (w *world) passes(d time.Duration) {
|
||||
for end := w.now.Add(d); w.now.Before(end); w.now = w.now.Add(5 * time.Second) {
|
||||
w.h.Reconcile(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
func events(said []announced) string {
|
||||
var out []string
|
||||
for _, a := range said {
|
||||
out = append(out, a.event)
|
||||
}
|
||||
return strings.Join(out, ",")
|
||||
}
|
||||
|
||||
func testAConsumerFailedForMinutesIsAnnouncedNamingItAndTheClass(t *testing.T) {
|
||||
w, said := standingWorld(t)
|
||||
w.a.failing = errors.New(`token request failed: 401 {"error":"invalid_grant","error_description":"Invalid user credentials"}`)
|
||||
w.give(map[string]any{"as": "mesh_home_grafana", "node": "home-server"})
|
||||
|
||||
w.passes(4 * time.Minute)
|
||||
if len(*said) != 0 {
|
||||
t.Fatalf("announced before FailingAfter: %v", events(*said))
|
||||
}
|
||||
w.passes(2 * time.Minute)
|
||||
if events(*said) != EventFailing {
|
||||
t.Fatalf("want one %s, got %q", EventFailing, events(*said))
|
||||
}
|
||||
b := (*said)[0].body
|
||||
if b["consumer"] != "mesh_home_grafana" || b["node"] != "home-server" || b["class"] != ClassCredentials ||
|
||||
b["provider"] != "postgres-database" || b["provider-node"] != "anchor" || b["attempts"].(int) < 60 {
|
||||
t.Fatalf("%v", b)
|
||||
}
|
||||
|
||||
// Said again while it lasts, not every pass.
|
||||
w.passes(14 * time.Minute)
|
||||
if events(*said) != EventFailing {
|
||||
t.Fatalf("repeated too soon: %q", events(*said))
|
||||
}
|
||||
w.passes(2 * time.Minute)
|
||||
if events(*said) != EventFailing+","+EventFailing {
|
||||
t.Fatalf("not repeated: %q", events(*said))
|
||||
}
|
||||
|
||||
// The first success takes it back.
|
||||
w.a.failing = nil
|
||||
w.passes(5 * time.Second)
|
||||
if events(*said) != EventFailing+","+EventFailing+","+EventRecovered {
|
||||
t.Fatalf("no recovery: %q", events(*said))
|
||||
}
|
||||
if (*said)[2].body["consumer"] != "mesh_home_grafana" {
|
||||
t.Fatal((*said)[2].body)
|
||||
}
|
||||
}
|
||||
|
||||
func testOneSuccessBetweenFailuresStartsTheRunAgain(t *testing.T) {
|
||||
w, said := standingWorld(t)
|
||||
w.a.failing = errors.New("connection refused")
|
||||
w.give(map[string]any{"as": "a"})
|
||||
w.passes(4 * time.Minute)
|
||||
w.a.failing = nil
|
||||
w.passes(5 * time.Second)
|
||||
w.give(map[string]any{"as": "a", "values": map[string]any{"name": "changed"}})
|
||||
w.a.failing = errors.New("connection refused")
|
||||
w.passes(4 * time.Minute)
|
||||
if len(*said) != 0 {
|
||||
t.Fatalf("two runs of four minutes are not one of eight: %q", events(*said))
|
||||
}
|
||||
}
|
||||
|
||||
// The check that failed for a day on 2026-10-05: clients already made, every minute's check refused
|
||||
// at the token. A check that cannot be asked is a failure too.
|
||||
func testACheckThatKeepsFailingIsAFailureToo(t *testing.T) {
|
||||
w, said := standingWorld(t)
|
||||
w.give(map[string]any{"as": "a"})
|
||||
w.h.Reconcile(ctx)
|
||||
w.a.holdsErr = errors.New("401 invalid_grant")
|
||||
w.passes(7 * time.Minute)
|
||||
if events(*said) != EventFailing || (*said)[0].body["class"] != ClassCredentials {
|
||||
t.Fatalf("%q %v", events(*said), *said)
|
||||
}
|
||||
w.a.holdsErr = nil
|
||||
w.passes(time.Minute + 5*time.Second)
|
||||
if events(*said) != EventFailing+","+EventRecovered {
|
||||
t.Fatalf("%q", events(*said))
|
||||
}
|
||||
}
|
||||
|
||||
func testAnUnreadableSecretIsAnnouncedAsSuch(t *testing.T) {
|
||||
w, said := standingWorld(t)
|
||||
w.give(map[string]any{"as": "a"})
|
||||
os.Remove(w.dir + "/a.secret")
|
||||
w.passes(6 * time.Minute)
|
||||
if events(*said) != EventFailing || (*said)[0].body["class"] != ClassSecret {
|
||||
t.Fatalf("%q %v", events(*said), *said)
|
||||
}
|
||||
}
|
||||
|
||||
func testAConsumerNoLongerAskedForIsNoLongerFailing(t *testing.T) {
|
||||
w, said := standingWorld(t)
|
||||
w.a.failing = errors.New("boom")
|
||||
w.give(map[string]any{"as": "a"}, map[string]any{"as": "b"})
|
||||
w.passes(6 * time.Minute)
|
||||
if events(*said) != EventFailing+","+EventFailing {
|
||||
t.Fatalf("%q", events(*said))
|
||||
}
|
||||
w.give(map[string]any{"as": "a"})
|
||||
w.passes(5 * time.Second)
|
||||
last := (*said)[len(*said)-1]
|
||||
if last.event != EventRecovered || last.body["consumer"] != "b" || last.body["why"] != "no longer asked for" {
|
||||
t.Fatalf("%v", *said)
|
||||
}
|
||||
}
|
||||
|
||||
func testAnErrorIsClassedByItsWords(t *testing.T) {
|
||||
for text, want := range map[string]string{
|
||||
`Keycloak token request failed: 401 {"error":"invalid_grant"}`: ClassCredentials,
|
||||
`FATAL: password authentication failed for user "postgres"`: ClassCredentials,
|
||||
`dial tcp 127.0.0.1:5432: connect: connection refused`: ClassUnreachable,
|
||||
`context deadline exceeded`: ClassUnreachable,
|
||||
`extension "nope" is not available`: ClassRefused,
|
||||
} {
|
||||
if got := ClassOf(text); got != want {
|
||||
t.Errorf("%s: %s, want %s", text, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func testAnAdapterThatClassesItsOwnErrorsIsBelieved(t *testing.T) {
|
||||
w, said := standingWorld(t)
|
||||
w.h.Adapter = classing{w.a}
|
||||
w.a.failing = errors.New("anything")
|
||||
w.give(map[string]any{"as": "a"})
|
||||
w.passes(6 * time.Minute)
|
||||
if (*said)[0].body["class"] != "its-own" {
|
||||
t.Fatal((*said)[0].body)
|
||||
}
|
||||
}
|
||||
|
||||
type classing struct{ *recorder }
|
||||
|
||||
func (classing) Class(error) string { return "its-own" }
|
||||
|
||||
// A provider restarted after announcing a failure has forgotten it; its first success for each
|
||||
// consumer is announced, so the controller clears what it kept rather than naming it for ever.
|
||||
func testTheFirstSuccessSinceStartIsAnnouncedOnce(t *testing.T) {
|
||||
w := newWorld(t)
|
||||
var said []announced
|
||||
w.h.Announce = func(e string, b map[string]any) {
|
||||
if e != EventRetirement {
|
||||
said = append(said, announced{e, b})
|
||||
}
|
||||
}
|
||||
w.give(map[string]any{"as": "a"})
|
||||
w.passes(3 * time.Minute)
|
||||
if events(said) != EventRecovered || said[0].body["why"] != "first-success" || said[0].body["consumer"] != "a" {
|
||||
t.Fatalf("%v", said)
|
||||
}
|
||||
}
|
||||
|
||||
// TestStanding holds this file's cases, each a subtest of its own name (mesh-sdk's test budget: at most 500 unit tests).
|
||||
func TestStanding(t *testing.T) {
|
||||
t.Run("AConsumerFailedForMinutesIsAnnouncedNamingItAndTheClass", testAConsumerFailedForMinutesIsAnnouncedNamingItAndTheClass)
|
||||
t.Run("OneSuccessBetweenFailuresStartsTheRunAgain", testOneSuccessBetweenFailuresStartsTheRunAgain)
|
||||
t.Run("ACheckThatKeepsFailingIsAFailureToo", testACheckThatKeepsFailingIsAFailureToo)
|
||||
t.Run("AnUnreadableSecretIsAnnouncedAsSuch", testAnUnreadableSecretIsAnnouncedAsSuch)
|
||||
t.Run("AConsumerNoLongerAskedForIsNoLongerFailing", testAConsumerNoLongerAskedForIsNoLongerFailing)
|
||||
t.Run("AnErrorIsClassedByItsWords", testAnErrorIsClassedByItsWords)
|
||||
t.Run("AnAdapterThatClassesItsOwnErrorsIsBelieved", testAnAdapterThatClassesItsOwnErrorsIsBelieved)
|
||||
t.Run("TheFirstSuccessSinceStartIsAnnouncedOnce", testTheFirstSuccessSinceStartIsAnnouncedOnce)
|
||||
}
|
||||
@@ -1,858 +0,0 @@
|
||||
// Package provider is a providing module's half of a binding: it answers the provider's consumers itself, over the
|
||||
// bus (novox/hq ADR 0349 clause 12, issue 678): the provider module, not the controller, makes each bound
|
||||
// consumer's entry.
|
||||
//
|
||||
// **Who binds the provider.** Each node's provisioner client records the needs of the modules handed to it from
|
||||
// their plans, and the controller names in each plan, per required provision and local name, the provider that
|
||||
// resolution chose: `provisioning.provisions` in `<node>.<module>` of the assignments mirror (issue 760). A recorded
|
||||
// need names no provider (issue 677's note 7), so the plan's entry is where a bind is seen, as the vault sees it
|
||||
// (mesh-vault pair.go). A consumer binds this provider when an entry names this machine and this module, and
|
||||
// unbinds when the entry, or the assignment, goes. A consumer still filled the old way has no plan and is not seen
|
||||
// here; the controller answers it as before (ADR 0349 clause 17).
|
||||
//
|
||||
// **One entry per consumer, in the provider's own bucket.** For each bind the helper composes the consumer's
|
||||
// served values from the provider's offer (its config already composed into it, ADR 0202 §7) and the consumer's
|
||||
// identity (`${consumer:as}`, `${consumer:as:dns}`, ADR 0202 §1), and writes one entry, `<node>.<module>.
|
||||
// <provision>[.<local>]` of `<provider module>_bound`: the one statement both ends read. It never carries a
|
||||
// secret. It names the credential the vault keeps for the pair, which the consumer's client asks the vault for.
|
||||
//
|
||||
// **The credential is the vault's** (ADR 0349 clause 8). The helper mints nothing and reads nothing of the
|
||||
// provider's own secrets: it reads the value from the vault and hands it to the provider's adapter, which makes the
|
||||
// login its software keeps. An offer whose software holds one credential (ADR 0158) is the vault's one value per
|
||||
// provider and provision, `provision/<provider node>/<provision>`, with the provider's one user name; an offer with
|
||||
// a login per consumer (ADR 0048, ADR 0158 §11, the default) is each consumer's own value,
|
||||
// `pair/<provider node>/<provision>/<node>/<module>[/<local>]`, with the consumer's identity as its login (issue
|
||||
// 677's notes 12 and 13). A credential the vault does not hold yet is waited for: nothing is made and no entry is
|
||||
// written, so a consumer is never told of a login its provider has not made. A provider in the tool runner never asks
|
||||
// the vault itself: its Vault is ConsumersFile, the file its machine's provisioner client fills (ADR 0373).
|
||||
//
|
||||
// **A multi-holder consumer gets no derived value** (ADR 0202 §10 as ADR 0349 clause 12 changes it): a consumer
|
||||
// bound under a local name keeps several holders of the provision, each with its own login, while what the
|
||||
// provider derives names one; its entry refuses it, naming both ends, and its client reports it.
|
||||
//
|
||||
// The helper speaks to the bus and the vault through the two interfaces below, so a provider wires it to its
|
||||
// runtime and a test to a stand-in. Standard library only.
|
||||
package provider
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/provisions"
|
||||
)
|
||||
|
||||
// BucketOf is a providing module's own bucket of entries, one per consumer: `<module>_bound`, the module's state
|
||||
// `bound`.
|
||||
func BucketOf(module string) string { return module + "_bound" }
|
||||
|
||||
// Bus is what the helper needs of the bus: a key-value watch and the provider's writes to its own bucket.
|
||||
type Bus interface {
|
||||
// Watch hands handle every current key and value of bucket, then each change as it comes, a removed key with a
|
||||
// nil value; it returns once the current values are handed, and the watch lasts until ctx ends.
|
||||
Watch(ctx context.Context, bucket string, handle func(key string, value []byte)) error
|
||||
Put(ctx context.Context, bucket, key string, value []byte) error
|
||||
Delete(ctx context.Context, bucket, key string) error
|
||||
}
|
||||
|
||||
// ConditionalBus is a Bus that can also delete a key only while it still holds what the caller read. The helper uses
|
||||
// it when the Bus offers it, so a pass that lets a consumer go never removes an entry another instance of the
|
||||
// provider wrote after the pass read it; a Bus without it deletes unconditionally. DeleteIf returns an error that
|
||||
// is stdio.ErrChanged when the key holds something else now.
|
||||
type ConditionalBus interface {
|
||||
Bus
|
||||
DeleteIf(ctx context.Context, bucket, key string, held []byte) error
|
||||
}
|
||||
|
||||
// Vault is how the provider reads a credential the vault keeps, by its name, opened.
|
||||
type Vault interface {
|
||||
Value(ctx context.Context, name string) ([]byte, error)
|
||||
}
|
||||
|
||||
// CredentialForm is which credential an offer's consumers present.
|
||||
type CredentialForm int
|
||||
|
||||
const (
|
||||
// PerConsumer is a login of each consumer's own (ADR 0048, ADR 0158 §11): the default.
|
||||
PerConsumer CredentialForm = iota
|
||||
// Shared is the one credential the provider's software holds, shared with every consumer (ADR 0158).
|
||||
Shared
|
||||
// NoCredential is an offer whose consumers present none.
|
||||
NoCredential
|
||||
)
|
||||
|
||||
// Offer is what the provider serves one provision with.
|
||||
type Offer struct {
|
||||
// Serves are the values served to each consumer, the provider's config already composed into them; a string
|
||||
// may name the consumer's identity, `${consumer:as}` or `${consumer:as:dns}`.
|
||||
Serves map[string]any
|
||||
// Credential is the form of the credential its consumers present. A consumer's plan entry that names a
|
||||
// credential form of its own wins over it (the plan's `credential-form`).
|
||||
Credential CredentialForm
|
||||
// User is the provider's one user name, for a Shared credential.
|
||||
User string
|
||||
}
|
||||
|
||||
// Entry is one consumer's entry in the provider's bucket: the one statement both ends read. Never a secret.
|
||||
type Entry struct {
|
||||
Provider string `json:"provider"`
|
||||
ProviderModule string `json:"provider-module"`
|
||||
Provision string `json:"provision"`
|
||||
Consumer string `json:"consumer"`
|
||||
Module string `json:"module"`
|
||||
Local string `json:"local,omitempty"`
|
||||
// As is the consumer's identity, the mesh's (ADR 0202 §1).
|
||||
As string `json:"as,omitempty"`
|
||||
// User is the login the consumer presents: its identity, or the provider's one user name for a shared credential.
|
||||
User string `json:"user,omitempty"`
|
||||
// Values are the served values, composed for this consumer: what `${bound:<provision>:<key>}` reads.
|
||||
Values map[string]any `json:"values,omitempty"`
|
||||
// VaultName is the name under which the vault keeps this consumer's credential, which the consumer's client asks
|
||||
// the vault for: a name, never the credential. Its key is not "credential" because the runtime's state guard
|
||||
// refuses any key ending in that word, whatever the value (issue 945).
|
||||
VaultName string `json:"vault-name,omitempty"`
|
||||
// Refused, when set, is why the consumer is not served, naming both ends.
|
||||
Refused string `json:"refused,omitempty"`
|
||||
// State is the provider's outcome for the consumer, which the holder of mesh-provisioner reads to set the
|
||||
// provision's key (novox/hq ADR 0570 clause 3): provided, failed with Why, or waits with what is awaited. Empty from
|
||||
// a provider before it.
|
||||
State string `json:"state,omitempty"`
|
||||
Why string `json:"why,omitempty"`
|
||||
// Contributes is what the consumer's module tells its provider about this provision (a database's name, its
|
||||
// extensions), from its plan's entry (issue 776). Handed to Make and Gone; never written to the bucket.
|
||||
Contributes map[string]any `json:"-"`
|
||||
// At and From are where the consumer's plan says the provider is and which machine it is (issue 788): the values
|
||||
// today's binding file gives that consumer. Written into Values as `at` and `from`; never alone to the bucket.
|
||||
At, From string `json:"-"`
|
||||
// Names are the names the controller composed for this consumer on this provision (issue 792): its own name, its
|
||||
// internal name, and one of each per local name it contributes under, keyed as its binding file names them.
|
||||
// Written into Values over anything the provider serves under the key; never alone to the bucket.
|
||||
Names map[string]string `json:"-"`
|
||||
// CredentialForm and Name are from the consumer's plan entry (issue 825): the form of the credential the
|
||||
// controller chose for this bind (`shared`, `pair`, `issued` or `given`) and the holder's name. When set, they
|
||||
// win over the offer's form. Never written to the bucket.
|
||||
CredentialForm, Name string `json:"-"`
|
||||
}
|
||||
|
||||
// Key is the entry's key in the provider's bucket: `<node>.<module>.<provision>`, and `.<local>` for a bind under a
|
||||
// local name.
|
||||
func (e Entry) Key() string {
|
||||
k := e.Consumer + "." + e.Module + "." + e.Provision
|
||||
if e.Local != "" {
|
||||
k += "." + e.Local
|
||||
}
|
||||
return k
|
||||
}
|
||||
|
||||
// Consumer is what the provider's adapter makes: the consumer's entry and the credential the vault keeps for it.
|
||||
type Consumer struct {
|
||||
Entry Entry
|
||||
Credential []byte
|
||||
}
|
||||
|
||||
// Options are the provider's.
|
||||
type Options struct {
|
||||
// Node and Module are this provider: the machine it runs on and the module serving.
|
||||
Node, Module string
|
||||
// Offers are the provisions it serves, by name.
|
||||
Offers map[string]Offer
|
||||
Bus Bus
|
||||
Vault Vault
|
||||
// Make brings a consumer into being in the provider's software under the vault's credential. Asked on every
|
||||
// pass for every consumer answered, so it must leave one that is already as asked alone; a rotated credential
|
||||
// reaches the software this way.
|
||||
Make func(ctx context.Context, c Consumer) error
|
||||
// Gone, when set, hears that a consumer no longer binds the provider, before its entry is removed; an error
|
||||
// keeps the entry, and it is heard again on the next pass. Retiring what was kept for it is the provider's
|
||||
// (ADR 0230).
|
||||
Gone func(ctx context.Context, e Entry) error
|
||||
// Identity is the consumer's identity for an entry whose plan names none; ConsumerIdentity of the node and the
|
||||
// module when not set.
|
||||
Identity func(node, module string) string
|
||||
// Every is how often a pass runs without a change watched; a minute when not set.
|
||||
Every time.Duration
|
||||
// Logf says what a pass did that differs from the last; the standard log when not set.
|
||||
Logf func(format string, args ...any)
|
||||
}
|
||||
|
||||
// Provider answers its consumers.
|
||||
type Provider struct {
|
||||
opts Options
|
||||
|
||||
mu sync.Mutex
|
||||
entries map[string][]byte
|
||||
// plans and provisions are the seat mesh-provisioner's states (ADR 0570 clauses 1, 2), by key.
|
||||
plans map[string][]byte
|
||||
provisions map[string][]byte
|
||||
said map[string]string
|
||||
poke chan struct{}
|
||||
}
|
||||
|
||||
// New is a provider answering its consumers as opts say.
|
||||
func New(opts Options) *Provider {
|
||||
if opts.Identity == nil {
|
||||
opts.Identity = ConsumerIdentity
|
||||
}
|
||||
if opts.Every <= 0 {
|
||||
opts.Every = time.Minute
|
||||
}
|
||||
if opts.Logf == nil {
|
||||
opts.Logf = func(f string, a ...any) { fmt.Printf("[provider] "+f+"\n", a...) }
|
||||
}
|
||||
return &Provider{opts: opts, entries: map[string][]byte{},
|
||||
plans: map[string][]byte{}, provisions: map[string][]byte{}, said: map[string]string{}, poke: make(chan struct{}, 1)}
|
||||
}
|
||||
|
||||
// ErrNotReady is what a Make returns, wrapped or alone, when the provider cannot make a consumer's provision yet but
|
||||
// will without anyone's hand (its keys not read since a restart): the entry waits and keeps serving what it served.
|
||||
// Any other error is a failure, and its entry serves nothing.
|
||||
var ErrNotReady = errors.New("not ready yet")
|
||||
|
||||
// What a pass did for one bind.
|
||||
const (
|
||||
Answered = "answered"
|
||||
Refused = "refused"
|
||||
Waiting = "waiting"
|
||||
Failed = "failed"
|
||||
Removed = "removed"
|
||||
// Superseded is an entry no bind asks for any more that another instance of the provider wrote after this pass
|
||||
// read it: it is left standing.
|
||||
Superseded = "superseded"
|
||||
)
|
||||
|
||||
// Outcome is what a pass did for one consumer's entry.
|
||||
type Outcome struct {
|
||||
Key string `json:"key"`
|
||||
Status string `json:"status"`
|
||||
Why string `json:"why,omitempty"`
|
||||
}
|
||||
|
||||
// Run watches the binds and the provider's own entries, and passes once at start, on every change and every
|
||||
// Every, until ctx ends.
|
||||
func (p *Provider) Run(ctx context.Context) error {
|
||||
if err := p.watch(ctx, BucketOf(p.opts.Module), p.entries); err != nil {
|
||||
return fmt.Errorf("watching %s: %w", BucketOf(p.opts.Module), err)
|
||||
}
|
||||
for _, bucket := range []string{provisions.PlansBucket, provisions.ProvisionsBucket} {
|
||||
into := p.plans
|
||||
if bucket == provisions.ProvisionsBucket {
|
||||
into = p.provisions
|
||||
}
|
||||
if err := p.watch(ctx, bucket, into); err != nil {
|
||||
return fmt.Errorf("watching %s: %w", bucket, err)
|
||||
}
|
||||
}
|
||||
tick := time.NewTicker(p.opts.Every)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
p.mu.Lock()
|
||||
in := Inputs{Entries: clone(p.entries), Plans: clone(p.plans), Provisions: clone(p.provisions)}
|
||||
p.mu.Unlock()
|
||||
if _, err := p.passIn(ctx, in); err != nil {
|
||||
p.say("pass", err.Error())
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-p.poke:
|
||||
case <-tick.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Provider) watch(ctx context.Context, bucket string, into map[string][]byte) error {
|
||||
return p.opts.Bus.Watch(ctx, bucket, func(key string, value []byte) {
|
||||
p.mu.Lock()
|
||||
if value == nil {
|
||||
delete(into, key)
|
||||
} else {
|
||||
into[key] = append([]byte(nil), value...)
|
||||
}
|
||||
p.mu.Unlock()
|
||||
if bucket != BucketOf(p.opts.Module) {
|
||||
select {
|
||||
case p.poke <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Pass reads the binds and the provider's entries from the bus and answers once: each bind of this provider gets
|
||||
// its entry, and each entry of this provider no bind asks for is removed. An assignment that cannot be read, or that
|
||||
// is not the one its key names, is skipped and said (an outcome of status Failed under the assignment's key, and a
|
||||
// line of the log), and the entries of its consumer are left as they are, since a bind missed would read as an
|
||||
// unbind; the pass goes on for the rest. Run passes by itself; Pass is for a provider that passes when it chooses.
|
||||
//
|
||||
// Over StateBus each call to Pass leaves two watches open for the life of the process, silenced once the call has
|
||||
// ended (see StateBus): a provider that passes often should use Run, which watches once.
|
||||
func (p *Provider) Pass(ctx context.Context) ([]Outcome, error) {
|
||||
watchCtx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
var mu sync.Mutex
|
||||
read := func(bucket string) (map[string][]byte, error) {
|
||||
got := map[string][]byte{}
|
||||
err := p.opts.Bus.Watch(watchCtx, bucket, func(key string, value []byte) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if value == nil {
|
||||
delete(got, key)
|
||||
} else {
|
||||
got[key] = append([]byte(nil), value...)
|
||||
}
|
||||
})
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return clone(got), err
|
||||
}
|
||||
entries, err := read(BucketOf(p.opts.Module))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading %s: %w", BucketOf(p.opts.Module), err)
|
||||
}
|
||||
plans, err := read(provisions.PlansBucket)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading %s: %w", provisions.PlansBucket, err)
|
||||
}
|
||||
provs, err := read(provisions.ProvisionsBucket)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading %s: %w", provisions.ProvisionsBucket, err)
|
||||
}
|
||||
return p.passIn(ctx, Inputs{Entries: entries, Plans: plans, Provisions: provs})
|
||||
}
|
||||
|
||||
// Inputs are what one pass reads: the provider's own entries, and the seat mesh-provisioner's plans and provisions
|
||||
// (novox/hq ADR 0570 clauses 1, 2), each by key. The controller's assignments are read no more (clause 2).
|
||||
type Inputs struct {
|
||||
Entries, Plans, Provisions map[string][]byte
|
||||
}
|
||||
|
||||
func (p *Provider) passIn(ctx context.Context, in Inputs) ([]Outcome, error) {
|
||||
entries := in.Entries
|
||||
var binds []Entry
|
||||
var out []Outcome
|
||||
unknown := map[string]bool{} // consumers whose assignment was skipped: their entries are not judged
|
||||
// Every consumer is bound by its plan and its provision keys (ADR 0570 clauses 2, 3), and an entry no plan names is
|
||||
// never let go by the provider itself (clause 5): only a retiring key with the operator's yes retires it.
|
||||
seat := map[string]bool{}
|
||||
gated := map[string]provisions.Provision{}
|
||||
for _, key := range sortedKeys(in.Plans) {
|
||||
plan, err := provisions.DecodePlan(in.Plans[key])
|
||||
if err != nil || plan.Node+"."+plan.Module != key {
|
||||
o := Outcome{Key: key, Status: Failed, Why: fmt.Sprintf("the plan %s is not readable, so who it binds is not "+
|
||||
"known; it is skipped, and its entries are left as they are", key)}
|
||||
out = append(out, o)
|
||||
p.say(key, o.Status+" "+o.Why)
|
||||
unknown[key], seat[key] = true, true
|
||||
continue
|
||||
}
|
||||
seat[key] = true
|
||||
for _, b := range plan.Plan.Provisions {
|
||||
if b.Provider != p.opts.Node || b.ProviderModule != p.opts.Module {
|
||||
continue
|
||||
}
|
||||
e := Entry{Provider: p.opts.Node, ProviderModule: p.opts.Module, Provision: b.Provision, Consumer: plan.Node,
|
||||
Module: plan.Module, Local: b.Local, Contributes: b.Contributes, At: b.At, From: b.From, As: b.As,
|
||||
Names: b.Names, CredentialForm: b.CredentialForm, Name: b.Name}
|
||||
binds = append(binds, e)
|
||||
}
|
||||
}
|
||||
for _, key := range sortedKeys(in.Provisions) {
|
||||
if pr, err := provisions.Decode(in.Provisions[key]); err == nil && pr.Provider.Node == p.opts.Node &&
|
||||
pr.Provider.Module == p.opts.Module {
|
||||
gated[key] = pr
|
||||
}
|
||||
}
|
||||
// **A key no plan names is bound from the entry it carries** (issue 1126): the seat keys a consumer's provisions
|
||||
// from its wanted and its newest builds, and keeps the plan at its wanted build alone, so a requirement the newest
|
||||
// build adds has a key before any plan names it, and the want that waits for it waits on this provider. The plan
|
||||
// wins where both say it. A key with no entry is said, never left without a word.
|
||||
planned := map[string]bool{}
|
||||
for _, b := range binds {
|
||||
planned[b.Key()] = true
|
||||
}
|
||||
for _, key := range sortedKeys(gated) {
|
||||
pr := gated[key]
|
||||
if planned[key] || pr.State == provisions.Retiring || unknown[consumerOf(key)] {
|
||||
continue
|
||||
}
|
||||
if pr.Plan == nil {
|
||||
o := Outcome{Key: key, Status: Waiting, Why: "asked, but no plan names it and its key carries no plan entry, " +
|
||||
"so whom it binds is not known; mesh-provisioner composes the entry"}
|
||||
out = append(out, o)
|
||||
p.say(key, o.Status+" "+o.Why)
|
||||
continue
|
||||
}
|
||||
b := *pr.Plan
|
||||
binds = append(binds, Entry{Provider: p.opts.Node, ProviderModule: p.opts.Module, Provision: pr.Provision,
|
||||
Consumer: pr.Consumer.Node, Module: pr.Consumer.Module, Local: pr.Local, Contributes: b.Contributes, At: b.At,
|
||||
From: b.From, As: b.As, Names: b.Names, CredentialForm: b.CredentialForm, Name: b.Name})
|
||||
}
|
||||
want := map[string]bool{}
|
||||
for _, b := range binds {
|
||||
key := b.Key()
|
||||
want[key] = true
|
||||
if why, ok := p.ready(key, gated, in.Provisions); !ok {
|
||||
o := Outcome{Key: key, Status: Waiting, Why: why}
|
||||
out = append(out, o)
|
||||
p.say(key, o.Status+" "+o.Why)
|
||||
continue
|
||||
}
|
||||
o := p.answer(ctx, b, entries[key])
|
||||
out = append(out, o)
|
||||
p.say(key, o.Status+" "+o.Why)
|
||||
}
|
||||
// ADR 0570 clause 5: a retiring key the operator said yes to is let go; one asked and not answered, or declined,
|
||||
// keeps its entry.
|
||||
for _, key := range sortedKeys(gated) {
|
||||
if pr := gated[key]; pr.State == provisions.Retiring && pr.Warrant != nil && pr.Warrant.Option == "yes" {
|
||||
want[key] = false
|
||||
if _, held := entries[key]; held {
|
||||
out = append(out, p.letGo(ctx, key, entries[key])...)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// consumerOf is the `<node>.<module>` an entry's key starts with: its first two parts.
|
||||
func consumerOf(key string) string {
|
||||
parts := strings.SplitN(key, ".", 3)
|
||||
if len(parts) < 3 {
|
||||
return key
|
||||
}
|
||||
return parts[0] + "." + parts[1]
|
||||
}
|
||||
|
||||
// remove deletes an entry this instance read as held, and only while it is still as read when the Bus can say so.
|
||||
func (p *Provider) remove(ctx context.Context, key string, held []byte) error {
|
||||
if c, ok := p.opts.Bus.(ConditionalBus); ok {
|
||||
return c.DeleteIf(ctx, BucketOf(p.opts.Module), key, held)
|
||||
}
|
||||
return p.opts.Bus.Delete(ctx, BucketOf(p.opts.Module), key)
|
||||
}
|
||||
|
||||
// answer makes one bind's entry and writes it when it differs from what the bucket holds.
|
||||
func (p *Provider) answer(ctx context.Context, e Entry, have []byte) Outcome {
|
||||
key := e.Key()
|
||||
offer, offered := p.opts.Offers[e.Provision]
|
||||
switch {
|
||||
case !offered:
|
||||
e.Refused = fmt.Sprintf("%s on %s is bound to %s on %s for %s, which %s does not serve", e.Module,
|
||||
e.Consumer, e.ProviderModule, e.Provider, e.Provision, e.ProviderModule)
|
||||
case e.Local != "" && derives(offer.Serves):
|
||||
e.Refused = fmt.Sprintf("%s on %s keeps several holders of %s (this one is %q), and %s on %s derives %s for "+
|
||||
"each consumer from its login: each holder has a login of its own and a consumer is told one value per "+
|
||||
"requirement, so the two ends would name different things (novox/hq ADR 0202 §10, ADR 0349 clause 12)",
|
||||
e.Module, e.Consumer, e.Provision, e.Local, e.ProviderModule, e.Provider,
|
||||
strings.Join(derivedKeys(offer.Serves), ", "))
|
||||
}
|
||||
if e.Refused != "" {
|
||||
e.State, e.Why = provisions.Failed, e.Refused
|
||||
return p.write(ctx, e, have, Outcome{Key: key, Status: Refused, Why: e.Refused})
|
||||
}
|
||||
|
||||
// The plan's identity is the controller's, slug first; Identity only answers for an entry without one.
|
||||
if e.As == "" {
|
||||
e.As = p.opts.Identity(e.Consumer, e.Module)
|
||||
}
|
||||
values, err := ServedTo(offer.Serves, e.As)
|
||||
if err != nil {
|
||||
e.Refused = fmt.Sprintf("%s on %s serving %s to %s on %s: %v", e.ProviderModule, e.Provider, e.Provision,
|
||||
e.Module, e.Consumer, err)
|
||||
e.As = ""
|
||||
e.State, e.Why = provisions.Failed, e.Refused
|
||||
return p.write(ctx, e, have, Outcome{Key: key, Status: Refused, Why: e.Refused})
|
||||
}
|
||||
if e.At != "" || e.From != "" {
|
||||
if values == nil {
|
||||
values = map[string]any{}
|
||||
}
|
||||
if e.At != "" {
|
||||
values["at"] = e.At
|
||||
}
|
||||
if e.From != "" {
|
||||
values["from"] = e.From
|
||||
}
|
||||
}
|
||||
if len(e.Names) > 0 {
|
||||
if values == nil {
|
||||
values = map[string]any{}
|
||||
}
|
||||
for key, name := range e.Names {
|
||||
values[key] = name
|
||||
}
|
||||
}
|
||||
e.Values = values
|
||||
var credential []byte
|
||||
// The plan's form wins; the offer's is the default.
|
||||
// An offer that declares no credential has none whatever the plan says: the controller writes "pair" for it.
|
||||
form := e.CredentialForm
|
||||
if offer.Credential == NoCredential {
|
||||
form = ""
|
||||
} else if form == "" {
|
||||
form = map[CredentialForm]string{Shared: "shared", PerConsumer: "pair"}[offer.Credential]
|
||||
}
|
||||
switch form {
|
||||
case "shared":
|
||||
e.User = offer.User
|
||||
e.VaultName = SharedName(e.Provider, e.Provision)
|
||||
case "pair":
|
||||
e.User = e.As
|
||||
e.VaultName = PairName(e.Provider, e.Provision, e.Consumer, e.Module, e.Local)
|
||||
case "issued", "given":
|
||||
// A per-holder value, as mesh-vault names it (perholder.go).
|
||||
e.User = e.As
|
||||
e.VaultName = HolderName(e.Provider, e.Provision, e.Consumer, e.Module)
|
||||
}
|
||||
if e.VaultName != "" {
|
||||
credential, err = p.opts.Vault.Value(ctx, e.VaultName)
|
||||
if err != nil {
|
||||
return Outcome{Key: key, Status: Waiting, Why: fmt.Sprintf("the vault's %s: %v", e.VaultName, err)}
|
||||
}
|
||||
}
|
||||
if p.opts.Make != nil {
|
||||
if err := p.opts.Make(ctx, Consumer{Entry: e, Credential: credential}); errors.Is(err, ErrNotReady) {
|
||||
// **Not ready is no failure** (the regression of 2026-10-30: gitea after a restart, its keys not read yet,
|
||||
// stopped serving every consumer): the entry says waits with why, and keeps what the last entry served, so a
|
||||
// consumer goes on with it until the provider can make it again.
|
||||
why := "the provider is not ready to make it yet: " + redact(err.Error(), credential)
|
||||
waiting := e
|
||||
var prev Entry
|
||||
if len(have) > 0 && json.Unmarshal(have, &prev) == nil && prev.State == provisions.Provided {
|
||||
waiting.Values, waiting.VaultName, waiting.User = prev.Values, prev.VaultName, prev.User
|
||||
} else {
|
||||
waiting.Values, waiting.VaultName, waiting.User = nil, "", ""
|
||||
}
|
||||
waiting.State, waiting.Why = provisions.Waits, why
|
||||
p.write(ctx, waiting, have, Outcome{Key: key, Status: Waiting, Why: why})
|
||||
return Outcome{Key: key, Status: Waiting, Why: why}
|
||||
} else if err != nil {
|
||||
why := "the provider could not make it: " + redact(err.Error(), credential)
|
||||
// Said in its entry, which the holder of mesh-provisioner reads (ADR 0570 clause 3), with nothing served.
|
||||
failed := e
|
||||
failed.Values, failed.VaultName, failed.User, failed.State, failed.Why = nil, "", "", provisions.Failed, why
|
||||
p.write(ctx, failed, have, Outcome{Key: key, Status: Failed, Why: why})
|
||||
return Outcome{Key: key, Status: Failed, Why: why}
|
||||
}
|
||||
}
|
||||
e.State = provisions.Provided
|
||||
return p.write(ctx, e, have, Outcome{Key: key, Status: Answered})
|
||||
}
|
||||
|
||||
// redact takes a credential out of text in the forms it can come back in: as it is, in base64 (standard and URL,
|
||||
// padded and not) and as a JSON string says it (escapes included).
|
||||
func redact(text string, credential []byte) string {
|
||||
if len(credential) == 0 {
|
||||
return text
|
||||
}
|
||||
quoted, _ := json.Marshal(string(credential))
|
||||
forms := []string{string(credential), base64.StdEncoding.EncodeToString(credential),
|
||||
base64.RawStdEncoding.EncodeToString(credential), base64.URLEncoding.EncodeToString(credential),
|
||||
base64.RawURLEncoding.EncodeToString(credential), string(quoted[1 : len(quoted)-1])}
|
||||
sort.SliceStable(forms, func(i, j int) bool { return len(forms[i]) > len(forms[j]) })
|
||||
for _, f := range forms {
|
||||
text = strings.ReplaceAll(text, f, "<credential>")
|
||||
}
|
||||
return text
|
||||
}
|
||||
|
||||
func (p *Provider) write(ctx context.Context, e Entry, have []byte, o Outcome) Outcome {
|
||||
body, err := json.Marshal(e)
|
||||
if err != nil {
|
||||
return Outcome{Key: o.Key, Status: Failed, Why: err.Error()}
|
||||
}
|
||||
if sameJSON(have, body) {
|
||||
return o
|
||||
}
|
||||
if err := p.opts.Bus.Put(ctx, BucketOf(p.opts.Module), o.Key, body); err != nil {
|
||||
return Outcome{Key: o.Key, Status: Failed, Why: "its entry could not be written: " + err.Error()}
|
||||
}
|
||||
return o
|
||||
}
|
||||
|
||||
// say logs what a key's pass said when it differs from the last.
|
||||
func (p *Provider) say(key, what string) {
|
||||
p.mu.Lock()
|
||||
same := p.said[key] == what
|
||||
p.said[key] = what
|
||||
p.mu.Unlock()
|
||||
if !same {
|
||||
p.opts.Logf("%s: %s", key, strings.TrimSpace(what))
|
||||
}
|
||||
}
|
||||
|
||||
// SharedName is the vault's one value per provider and provision (ADR 0158, issue 677).
|
||||
func SharedName(provider, provision string) string {
|
||||
return provisions.VaultName(provisions.FormShared, provisions.Provision{Provision: provision, Provider: provisions.End{Node: provider}})
|
||||
}
|
||||
|
||||
// HolderName is the vault's value of one holder's own credential, issued or given for it (mesh-vault perholder.go).
|
||||
func HolderName(provider, provision, node, module string) string {
|
||||
return provisions.VaultName(provisions.FormIssued, provisions.Provision{Provision: provision,
|
||||
Provider: provisions.End{Node: provider}, Consumer: provisions.End{Node: node, Module: module}})
|
||||
}
|
||||
|
||||
// PairName is the vault's value of one consumer's own login (ADR 0048, ADR 0158 §11, issue 677's note 12).
|
||||
func PairName(provider, provision, node, module, local string) string {
|
||||
return provisions.VaultName(provisions.FormPair, provisions.Provision{Provision: provision, Local: local,
|
||||
Provider: provisions.End{Node: provider}, Consumer: provisions.End{Node: node, Module: module}})
|
||||
}
|
||||
|
||||
// identityUnusable is every character a consumer's identity does not hold (mesh-controller internal/catalogue
|
||||
// identity.go, the one derivation).
|
||||
var identityUnusable = regexp.MustCompile(`[^a-z0-9_]+`)
|
||||
|
||||
// ConsumerIdentity is what one module on one machine is called wherever it authenticates: `mesh_<node>_<module>`,
|
||||
// lower case, anything but letters, digits and `_` written `_` (mesh-controller ConsumerIdentity). module is the
|
||||
// identity source: the module's slug when it declares one, otherwise its name (ADR 0049).
|
||||
func ConsumerIdentity(node, module string) string {
|
||||
clean := func(s string) string {
|
||||
return strings.Trim(identityUnusable.ReplaceAllString(strings.ToLower(s), "_"), "_")
|
||||
}
|
||||
return "mesh_" + clean(node) + "_" + clean(module)
|
||||
}
|
||||
|
||||
// consumerFact is `${consumer:<fact>}` or `${consumer:<fact>:<alphabet>}`.
|
||||
var consumerFact = regexp.MustCompile(`\$\{consumer:([a-z][a-z0-9-]*)(?::([a-z][a-z0-9-]*))?\}`)
|
||||
|
||||
// ServedTo fills an offer's served values for one consumer whose identity is as (ADR 0202 §1): `${consumer:as}` as
|
||||
// it is, `${consumer:as:dns}` with `-` for `_`. Only strings carry placeholders; any other fact or alphabet is
|
||||
// refused, never left standing.
|
||||
func ServedTo(serves map[string]any, as string) (map[string]any, error) {
|
||||
if serves == nil {
|
||||
return nil, nil
|
||||
}
|
||||
out := make(map[string]any, len(serves))
|
||||
for _, key := range sortedKeys(serves) {
|
||||
text, ok := serves[key].(string)
|
||||
if !ok || !strings.Contains(text, "${consumer:") {
|
||||
out[key] = serves[key]
|
||||
continue
|
||||
}
|
||||
var failed error
|
||||
out[key] = consumerFact.ReplaceAllStringFunc(text, func(match string) string {
|
||||
parts := consumerFact.FindStringSubmatch(match)
|
||||
switch {
|
||||
case parts[1] == "as" && parts[2] == "":
|
||||
return as
|
||||
case parts[1] == "as" && parts[2] == "dns":
|
||||
return strings.ReplaceAll(as, "_", "-")
|
||||
}
|
||||
if failed == nil {
|
||||
failed = fmt.Errorf("the value served as %q says %s, and the mesh states only ${consumer:as} and "+
|
||||
"${consumer:as:dns}", key, match)
|
||||
}
|
||||
return match
|
||||
})
|
||||
if failed != nil {
|
||||
return nil, failed
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func derives(serves map[string]any) bool { return len(derivedKeys(serves)) > 0 }
|
||||
|
||||
func derivedKeys(serves map[string]any) []string {
|
||||
var out []string
|
||||
for _, k := range sortedKeys(serves) {
|
||||
if text, ok := serves[k].(string); ok && strings.Contains(text, "${consumer:") {
|
||||
out = append(out, k)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func clone(m map[string][]byte) map[string][]byte {
|
||||
out := make(map[string][]byte, len(m))
|
||||
for k, v := range m {
|
||||
out[k] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func sameJSON(a, b []byte) bool {
|
||||
if a == nil || b == nil {
|
||||
return false
|
||||
}
|
||||
var x, y any
|
||||
if json.Unmarshal(a, &x) != nil || json.Unmarshal(b, &y) != nil {
|
||||
return false
|
||||
}
|
||||
ja, _ := json.Marshal(x)
|
||||
jb, _ := json.Marshal(y)
|
||||
return string(ja) == string(jb)
|
||||
}
|
||||
|
||||
func sortedKeys[V any](m map[string]V) []string {
|
||||
if len(m) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// StateBus is the Bus of a provider that runs as a module bundle over the runtime's stdio: the provider's own bucket,
|
||||
// BucketOf(module), is the module's state `bound`, and provisions.PlansBucket and provisions.ProvisionsBucket are the
|
||||
// states it reads, `mesh-provisioner.plans` and `mesh-provisioner.provisions` (each declared in the module's manifest).
|
||||
// Any other bucket is refused.
|
||||
//
|
||||
// A stdio state watch lasts as long as the process and cannot be ended (stdio.KeptState.Watch), so Watch honours the
|
||||
// Bus contract by silencing instead: once ctx has ended the handler is never called again, though the runtime keeps
|
||||
// sending changes that are dropped. A Pass, which watches twice per call, is therefore correct, only the dropped
|
||||
// changes of its two spent watches remain.
|
||||
func StateBus(module string) Bus { return stateBus{module: module} }
|
||||
|
||||
type stateBus struct{ module string }
|
||||
|
||||
func (b stateBus) state(bucket string) (stdio.KeptState, error) {
|
||||
switch bucket {
|
||||
case BucketOf(b.module):
|
||||
return stdio.State("bound"), nil
|
||||
case provisions.PlansBucket:
|
||||
return stdio.State(provisions.PlansRead), nil
|
||||
case provisions.ProvisionsBucket:
|
||||
return stdio.State(provisions.ProvisionsRead), nil
|
||||
}
|
||||
return stdio.KeptState{}, fmt.Errorf("%s is not a bucket this provider keeps or reads", bucket)
|
||||
}
|
||||
|
||||
func (b stateBus) Watch(ctx context.Context, bucket string, handle func(key string, value []byte)) error {
|
||||
s, err := b.state(bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.Watch("", func(ch stdio.StateChange) error {
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
if ch.Op == "delete" {
|
||||
handle(ch.Key, nil)
|
||||
} else {
|
||||
handle(ch.Key, ch.Value)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func (b stateBus) Put(_ context.Context, bucket, key string, value []byte) error {
|
||||
s, err := b.state(bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = s.Put(key, json.RawMessage(value))
|
||||
return err
|
||||
}
|
||||
|
||||
// DeleteIf removes the key only while it still holds what the caller read; it reads the key first, so another
|
||||
// writer's change in that short while can still be lost.
|
||||
func (b stateBus) DeleteIf(_ context.Context, bucket, key string, held []byte) error {
|
||||
s, err := b.state(bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cur, err := s.Get(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if cur == nil {
|
||||
return nil
|
||||
}
|
||||
if !sameJSON(cur.Value, held) {
|
||||
return fmt.Errorf("%s is not as it was read: %w", key, stdio.ErrChanged)
|
||||
}
|
||||
return s.Delete(key)
|
||||
}
|
||||
|
||||
func (b stateBus) Delete(_ context.Context, bucket, key string) error {
|
||||
s, err := b.state(bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.Delete(key)
|
||||
}
|
||||
|
||||
// Standing is whether an assignment's value is a standing assignment: false when `ended` is set (the module was
|
||||
// taken off its node, novox/hq ADR 0408 clause 4) or `state` is not empty (a pending one). Such a value is read
|
||||
// exactly as a removed key. Every other field, known or not, changes nothing (issue 1098).
|
||||
func Standing(value []byte) bool {
|
||||
var v struct {
|
||||
Ended json.RawMessage `json:"ended"`
|
||||
State json.RawMessage `json:"state"`
|
||||
}
|
||||
if json.Unmarshal(value, &v) != nil {
|
||||
return true // not decided here: the parse that follows refuses it
|
||||
}
|
||||
return !fieldSet(v.Ended) && !fieldSet(v.State)
|
||||
}
|
||||
|
||||
// fieldSet is whether a field carries something: not absent, null or empty.
|
||||
func fieldSet(f json.RawMessage) bool {
|
||||
s := strings.TrimSpace(string(f))
|
||||
return s != "" && s != "null" && s != `""`
|
||||
}
|
||||
|
||||
// letGo lets a consumer's entry go: the provider hears it gone (Gone, which retires what it kept for the consumer), and
|
||||
// the entry is removed while it is still as read.
|
||||
func (p *Provider) letGo(ctx context.Context, key string, held []byte) []Outcome {
|
||||
var e Entry
|
||||
if err := json.Unmarshal(held, &e); err != nil || e.Provider != p.opts.Node || e.ProviderModule != p.opts.Module {
|
||||
return nil // another machine's instance of this provider wrote it, or nothing of this helper's
|
||||
}
|
||||
said := func(o Outcome) []Outcome {
|
||||
p.say(key, o.Status+" "+o.Why)
|
||||
return []Outcome{o}
|
||||
}
|
||||
if p.opts.Gone != nil {
|
||||
if err := p.opts.Gone(ctx, e); err != nil {
|
||||
return said(Outcome{Key: key, Status: Failed, Why: "no longer bound, and the provider could not let it go: " + err.Error()})
|
||||
}
|
||||
}
|
||||
if err := p.remove(ctx, key, held); err != nil {
|
||||
if errors.Is(err, stdio.ErrChanged) {
|
||||
return said(Outcome{Key: key, Status: Superseded, Why: "another instance of the provider wrote its entry " +
|
||||
"since this pass read it, so it stands"})
|
||||
}
|
||||
return said(Outcome{Key: key, Status: Failed, Why: "no longer bound, and its entry could not be removed: " + err.Error()})
|
||||
}
|
||||
p.say(key, Removed)
|
||||
p.mu.Lock()
|
||||
delete(p.said, key)
|
||||
p.mu.Unlock()
|
||||
return []Outcome{{Key: key, Status: Removed}}
|
||||
}
|
||||
|
||||
// ready says whether the provision under key may be made now (novox/hq ADR 0570 clause 3): its key is on the seat, it is
|
||||
// asked or provided, and every provision it depends on is provided. Otherwise why it waits, naming the key or the
|
||||
// dependency.
|
||||
func (p *Provider) ready(key string, mine map[string]provisions.Provision, all map[string][]byte) (string, bool) {
|
||||
pr, ok := mine[key]
|
||||
if !ok {
|
||||
return "mesh-provisioner holds no key " + key + " naming this provider yet", false
|
||||
}
|
||||
if pr.State != provisions.Asked && pr.State != provisions.Provided {
|
||||
return "its key is " + pr.State + ": " + pr.Why, false
|
||||
}
|
||||
for _, d := range pr.DependsOn {
|
||||
dep, err := provisions.Decode(all[d])
|
||||
if err != nil {
|
||||
return "it depends on " + d + ", which mesh-provisioner holds no readable key of", false
|
||||
}
|
||||
if dep.State != provisions.Provided {
|
||||
return "it depends on " + d + ", which is " + dep.State, false
|
||||
}
|
||||
}
|
||||
return "", true
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,154 +0,0 @@
|
||||
package provider
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/provisions"
|
||||
)
|
||||
|
||||
// runtime drives a bundle over pipes as the runtime does: it reads what the bundle says and sends what the runtime would.
|
||||
type runtime struct {
|
||||
t *testing.T
|
||||
lines *bufio.Scanner
|
||||
in io.Writer
|
||||
}
|
||||
|
||||
func (r *runtime) next() map[string]any {
|
||||
done := make(chan map[string]any, 1)
|
||||
go func() {
|
||||
if r.lines.Scan() {
|
||||
var m map[string]any
|
||||
_ = json.Unmarshal(r.lines.Bytes(), &m)
|
||||
done <- m
|
||||
}
|
||||
}()
|
||||
select {
|
||||
case m := <-done:
|
||||
return m
|
||||
case <-time.After(3 * time.Second):
|
||||
r.t.Fatal("the bundle said nothing")
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (r *runtime) send(m map[string]any) {
|
||||
b, _ := json.Marshal(m)
|
||||
_, _ = r.in.Write(append(b, '\n'))
|
||||
}
|
||||
|
||||
func (r *runtime) call(id int, tool string, args map[string]any) {
|
||||
r.send(map[string]any{"jsonrpc": "2.0", "id": id, "method": "tools/call", "params": map[string]any{"name": tool, "arguments": args}})
|
||||
}
|
||||
|
||||
func serve(t *testing.T, tools []stdio.Tool) *runtime {
|
||||
t.Setenv(stdio.ServedAs, "keeper")
|
||||
inR, inW := io.Pipe()
|
||||
outR, outW := io.Pipe()
|
||||
go func() { _ = stdio.ServeOn("", tools, inR, outW) }()
|
||||
t.Cleanup(func() { _ = inW.Close() })
|
||||
return &runtime{t: t, lines: bufio.NewScanner(outR), in: inW}
|
||||
}
|
||||
|
||||
// The bus of a moved provider maps its own bucket to the state `bound` and the seat's plans to the state
|
||||
// `mesh-provisioner.plans` (novox/hq ADR 0570 clause 2), writes only the own bucket's state, and refuses any other
|
||||
// bucket, the controller's assignments among them.
|
||||
func TestStateBusMapsTheProvidersBucketsToTheRuntimesStates(t *testing.T) {
|
||||
bus := StateBus("postgres")
|
||||
rt := serve(t, []stdio.Tool{
|
||||
{Name: "watch", Run: func(a map[string]any) (any, error) {
|
||||
return nil, bus.Watch(context.Background(), a["bucket"].(string), func(string, []byte) {})
|
||||
}},
|
||||
{Name: "put", Run: func(map[string]any) (any, error) {
|
||||
return "put", bus.Put(context.Background(), BucketOf("postgres"), "k", []byte(`{"a":1}`))
|
||||
}},
|
||||
{Name: "delete", Run: func(map[string]any) (any, error) {
|
||||
return "deleted", bus.Delete(context.Background(), BucketOf("postgres"), "k")
|
||||
}},
|
||||
})
|
||||
for i, c := range []struct{ bucket, state string }{
|
||||
{BucketOf("postgres"), "bound"},
|
||||
{provisions.PlansBucket, "mesh-provisioner.plans"},
|
||||
} {
|
||||
rt.call(i+1, "watch", map[string]any{"bucket": c.bucket})
|
||||
w := rt.next()
|
||||
if w["method"] != "mesh/state.watch" || w["params"].(map[string]any)["state"] != c.state {
|
||||
t.Fatalf("bucket %s asked %v, want the state %s", c.bucket, w, c.state)
|
||||
}
|
||||
rt.send(map[string]any{"jsonrpc": "2.0", "id": w["id"], "result": map[string]any{}})
|
||||
rt.next()
|
||||
}
|
||||
rt.call(3, "put", nil)
|
||||
p := rt.next()
|
||||
if p["method"] != "mesh/state.put" || p["params"].(map[string]any)["state"] != "bound" || p["params"].(map[string]any)["key"] != "k" {
|
||||
t.Fatalf("put asked %v", p)
|
||||
}
|
||||
rt.send(map[string]any{"jsonrpc": "2.0", "id": p["id"], "result": map[string]any{"revision": 1}})
|
||||
rt.next()
|
||||
rt.call(4, "delete", nil)
|
||||
d := rt.next()
|
||||
if d["method"] != "mesh/state.delete" || d["params"].(map[string]any)["state"] != "bound" {
|
||||
t.Fatalf("delete asked %v", d)
|
||||
}
|
||||
rt.send(map[string]any{"jsonrpc": "2.0", "id": d["id"], "result": map[string]any{}})
|
||||
rt.next()
|
||||
|
||||
for _, bucket := range []string{"other_bound", "elsewhere", "mesh-controller_assignments"} {
|
||||
if err := bus.Watch(context.Background(), bucket, func(string, []byte) {}); err == nil || !strings.Contains(err.Error(), "not a bucket") {
|
||||
t.Fatalf("a watch of %s gave %v", bucket, err)
|
||||
}
|
||||
if err := bus.Put(context.Background(), bucket, "k", nil); err == nil {
|
||||
t.Fatalf("a put to %s was allowed", bucket)
|
||||
}
|
||||
if err := bus.Delete(context.Background(), bucket, "k"); err == nil {
|
||||
t.Fatalf("a delete in %s was allowed", bucket)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A watch whose ctx has ended calls its handler no more, though the runtime keeps sending changes: the Bus contract
|
||||
// says a watch lasts until ctx ends, and a stdio state watch lasts as long as the process.
|
||||
func TestStateBusWatchCallsItsHandlerNoMoreOnceItsContextEnded(t *testing.T) {
|
||||
bus := StateBus("postgres")
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
handled := make(chan string, 8)
|
||||
rt := serve(t, []stdio.Tool{
|
||||
{Name: "start", Run: func(map[string]any) (any, error) {
|
||||
return "started", bus.Watch(ctx, provisions.PlansBucket, func(key string, value []byte) {
|
||||
handled <- key + "=" + string(value)
|
||||
})
|
||||
}},
|
||||
{Name: "stop", Run: func(map[string]any) (any, error) { cancel(); return "stopped", nil }},
|
||||
})
|
||||
change := func(id int, watch any, key, op string) {
|
||||
rt.send(map[string]any{"jsonrpc": "2.0", "id": id, "method": "mesh/state", "params": map[string]any{"watch": watch,
|
||||
"change": map[string]any{"state": "mesh-controller.assignments", "key": key, "op": op, "value": "v"}}})
|
||||
if r := rt.next(); r["id"] != float64(id) || r["error"] != nil {
|
||||
t.Fatalf("a change was answered %v", r)
|
||||
}
|
||||
}
|
||||
rt.call(1, "start", nil)
|
||||
w := rt.next()
|
||||
rt.send(map[string]any{"jsonrpc": "2.0", "id": w["id"], "result": map[string]any{}})
|
||||
rt.next()
|
||||
|
||||
change(50, w["id"], "before", "put")
|
||||
if got := <-handled; got != `before="v"` {
|
||||
t.Fatalf("handled %q", got)
|
||||
}
|
||||
rt.call(2, "stop", nil)
|
||||
rt.next()
|
||||
change(51, w["id"], "after", "put")
|
||||
change(52, w["id"], "after", "delete")
|
||||
select {
|
||||
case got := <-handled:
|
||||
t.Fatalf("the handler was called after its context ended: %q", got)
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user