313 lines
11 KiB
Go
313 lines
11 KiB
Go
package harness
|
|
|
|
// A provider handed to the provisioner (novox/hq ADR 0373, ADR 0349 clause 12, issues 780 and 805): where its
|
|
// consumers file is set, it runs on the provider helper (package provider) instead of the harness reading the
|
|
// controller's grant files. Moved is that path, shared by every provider that keeps a resource per consumer; a module
|
|
// says only what differs, in a MovedSpec.
|
|
//
|
|
// The helper learns who binds the provider from the plans, composes each consumer's entry and reads its credential
|
|
// through ConsumersFile, the file this machine's provisioner client fills. It leaves the rest to the two hooks
|
|
// here: Make brings the consumer's resource into being exactly as the harness's adapter does, and Gone leaves
|
|
// retirement to the harness's guards (retirement.go): the resource is disabled and marked to delete, never removed.
|
|
//
|
|
// **A consumer not moved is still the grant file's** (issue 937, ADR 0349 clause 17), for a provider whose spec says
|
|
// WithGrants: only a moved consumer's plan names its provider, so the controller still gives every other consumer
|
|
// through the grant file (MESH_RECEIVES) until R1 (issue 682). Each login it gives is asked for, and one no plan
|
|
// names is made and checked by the harness's grant pass exactly as on the grant path; a login a plan names is the
|
|
// helper's alone. A grant file that cannot be read is not nobody asking (issue 241): nothing is retired until it is
|
|
// read again.
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"sync"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/provider"
|
|
"git.novox.be/novox/mesh-sdk/go/provisions"
|
|
)
|
|
|
|
// MovedSpec is what differs per provider on the moved path.
|
|
type MovedSpec struct {
|
|
// Module and Provision are the provider module's name and what it provides, as its manifest says.
|
|
Module, Provision string
|
|
// Offer is what it serves: its credential form and served values.
|
|
Offer provider.Offer
|
|
// Login is the identity a consumer is made under at the provider's end (postgres: the entry's User; keycloak: its
|
|
// As); an entry with none is not made.
|
|
Login func(e provider.Entry) string
|
|
// Values are the values the adapter is handed for a consumer; the entry's contribution when not set.
|
|
Values func(e provider.Entry) map[string]any
|
|
// Noun names what Login is in words ("login", "client id"), Made what is made ("login", "client"), and Backend
|
|
// the service in "no longer holds it" ("the backend", "Keycloak").
|
|
Noun, Made, Backend string
|
|
// WithGrants makes the logins the grant file gives, that no plan names, asked for and made by the harness's grant
|
|
// pass (issue 937).
|
|
WithGrants bool
|
|
}
|
|
|
|
// Moved is a provider on the provider helper, with the harness's own memory and retirement behind it: what is held,
|
|
// what is counted unasked, what waits for a person (retirement.go). The harness is never Run here; the helper's Make
|
|
// and Gone and the loop below drive the same state, so the guards are the harness's, not a copy.
|
|
type Moved struct {
|
|
h *Harness
|
|
spec MovedSpec
|
|
|
|
// FirstRetry is the first wait before a watch asked too early is asked again (issue 943); a second when zero.
|
|
FirstRetry time.Duration
|
|
|
|
mu sync.Mutex // guards plans and loaded
|
|
plans map[string][]byte
|
|
loaded bool
|
|
verified map[string]time.Time
|
|
}
|
|
|
|
func newMoved(h *Harness, spec MovedSpec) *Moved {
|
|
h.mu.Lock()
|
|
h.init()
|
|
h.mu.Unlock()
|
|
return &Moved{h: h, spec: spec, plans: map[string][]byte{}, verified: map[string]time.Time{}}
|
|
}
|
|
|
|
// Make brings one consumer's resource into being under its login and the vault's credential, as the harness does:
|
|
// at once when it is new or changed, and otherwise only when the backend no longer holds it, which is asked every
|
|
// VerifyEvery.
|
|
func (m *Moved) Make(ctx context.Context, c provider.Consumer) error {
|
|
h, e := m.h, c.Entry
|
|
as := m.spec.Login(e)
|
|
if as == "" {
|
|
return fmt.Errorf("%s has no %s to make", e.Key(), m.spec.Noun)
|
|
}
|
|
if len(c.Credential) == 0 {
|
|
return fmt.Errorf("%s has no credential to make its %s with", e.Key(), m.spec.Made)
|
|
}
|
|
h.mu.Lock()
|
|
defer h.mu.Unlock()
|
|
password := string(c.Credential)
|
|
values := orEmpty(e.Contributes)
|
|
if m.spec.Values != nil {
|
|
values = m.spec.Values(e)
|
|
}
|
|
p := Provision{As: as, Password: password, Values: values, Consumer: e.Consumer}
|
|
hash := hashOf(as, password, p.Values, nil)
|
|
if was, ok := h.applied[as]; ok && was.hash == hash {
|
|
if h.Now().Sub(m.verified[as]) < h.VerifyEvery {
|
|
return nil
|
|
}
|
|
m.verified[as] = h.Now()
|
|
hctx, cancel := context.WithTimeout(ctx, h.HoldsTimeout)
|
|
held, err := h.Adapter.Holds(hctx, p)
|
|
cancel()
|
|
if err != nil {
|
|
text := scrub(err, password)
|
|
h.failed(as, e.Consumer, h.classOf(err, text), text)
|
|
return errors.New(text)
|
|
}
|
|
if held {
|
|
h.succeeded(as)
|
|
return nil
|
|
}
|
|
h.say("%s: %s no longer holds it; applying again", as, m.spec.Backend)
|
|
}
|
|
if err := h.Adapter.Create(ctx, p); err != nil {
|
|
text := scrub(err, password)
|
|
h.failed(as, e.Consumer, h.classOf(err, text), text)
|
|
return errors.New(text)
|
|
}
|
|
h.succeeded(as)
|
|
h.applied[as] = appliedEntry{hash: hash, node: e.Consumer}
|
|
m.verified[as] = h.Now()
|
|
h.reenabled(as, e.Consumer)
|
|
return nil
|
|
}
|
|
|
|
// Gone is the helper letting a consumer go, which it does only on a retiring provision the operator said yes to (novox/hq
|
|
// ADR 0570 clause 5): its login is retired at once, through the adapter, its resource and data kept and marked to
|
|
// delete, deleted only by a person's `cleanup delete`. A consumer another plan still binds under the same login loses
|
|
// nothing.
|
|
func (m *Moved) Gone(ctx context.Context, e provider.Entry) error {
|
|
h := m.h
|
|
as := m.spec.Login(e)
|
|
if as == "" {
|
|
return nil // refused: nothing was made for it
|
|
}
|
|
want, _ := m.asked(h.Node)
|
|
h.mu.Lock()
|
|
defer h.mu.Unlock()
|
|
if want[as] {
|
|
return nil // another holder of the same login still binds: nothing to retire
|
|
}
|
|
if _, held := h.applied[as]; !held {
|
|
return nil
|
|
}
|
|
if done := h.retire(ctx, []string{as}, "the operator approved its retirement ("+e.Key()+")", nil); len(done) != 1 {
|
|
return fmt.Errorf("%s was not retired; it is tried again at the next pass", as)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// asked is the logins the seat's plans bind the provider for (novox/hq ADR 0570 clause 2), and whether they could all be
|
|
// read: a plan that cannot be read, or no read yet, is not an answer (novox/hq issue 241).
|
|
func (m *Moved) asked(node string) (map[string]bool, bool) {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
if !m.loaded || len(m.plans) == 0 {
|
|
return nil, false
|
|
}
|
|
want := map[string]bool{}
|
|
for key, raw := range m.plans {
|
|
pl, err := provisions.DecodePlan(raw)
|
|
if err != nil || pl.Node+"."+pl.Module != key {
|
|
return nil, false
|
|
}
|
|
for _, b := range pl.Plan.Provisions {
|
|
if b.Provider == node && b.ProviderModule == m.spec.Module && b.Provision == m.spec.Provision {
|
|
as := b.As
|
|
if as == "" {
|
|
as = provider.ConsumerIdentity(pl.Node, pl.Module) // a plan that names none
|
|
}
|
|
want[as] = true
|
|
}
|
|
}
|
|
}
|
|
return want, true
|
|
}
|
|
|
|
// granted is what the grant file gives that no plan names (issue 937), and every login it gives; and whether it
|
|
// could be read. The caller holds h.mu. With no grant file named there is nothing given, and that is a read.
|
|
func (m *Moved) granted(planned map[string]bool) ([]contribution, map[string]bool, bool) {
|
|
h := m.h
|
|
if h.Receives == "" {
|
|
return nil, map[string]bool{}, true
|
|
}
|
|
given := h.readContributions()
|
|
if given == nil {
|
|
return nil, nil, false
|
|
}
|
|
var own []contribution
|
|
logins := map[string]bool{}
|
|
for _, g := range given {
|
|
logins[g.As] = true
|
|
if !planned[g.As] {
|
|
own = append(own, g)
|
|
}
|
|
}
|
|
return own, logins, true
|
|
}
|
|
|
|
// retire is one pass over what the plans bind and the grant file gives now: the logins only the grant file gives are
|
|
// made and checked, as the grant path does. It retires nothing (ADR 0570 clause 5).
|
|
func (m *Moved) retire(ctx context.Context) {
|
|
h := m.h
|
|
want, ok := m.asked(h.Node)
|
|
h.mu.Lock()
|
|
defer h.mu.Unlock()
|
|
var given []contribution
|
|
if ok && m.spec.WithGrants {
|
|
var granted map[string]bool
|
|
given, granted, ok = m.granted(want)
|
|
for as := range granted {
|
|
want[as] = true
|
|
}
|
|
}
|
|
if !ok {
|
|
// Not a result: the passes that must agree before anything is retired start again.
|
|
h.r.key, h.r.count = "", 0
|
|
return
|
|
}
|
|
h.seed(ctx, want)
|
|
h.apply(ctx, given)
|
|
// No login is retired for going unasked (novox/hq ADR 0570 clause 5): only Gone, on the operator's yes.
|
|
h.r.lastWant = want
|
|
for as := range h.failing {
|
|
if !want[as] {
|
|
delete(h.failing, as)
|
|
}
|
|
}
|
|
for as := range h.trouble {
|
|
if _, held := h.applied[as]; !want[as] && !held {
|
|
h.recovered(as, "no longer asked for")
|
|
}
|
|
}
|
|
}
|
|
|
|
// watch keeps the plans as the bus holds them, from the first read on.
|
|
func (m *Moved) watch(ctx context.Context, bus provider.Bus) error {
|
|
err := bus.Watch(ctx, provisions.PlansBucket, func(key string, value []byte) {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
if value == nil {
|
|
delete(m.plans, key)
|
|
} else {
|
|
m.plans[key] = append([]byte(nil), value...)
|
|
}
|
|
})
|
|
if err == nil {
|
|
m.mu.Lock()
|
|
m.loaded = true
|
|
m.mu.Unlock()
|
|
}
|
|
return err
|
|
}
|
|
|
|
// Run answers the consumers through the helper and retires through the harness's rules, until ctx ends.
|
|
//
|
|
// The bus is reached through the runtime that launched this bundle, which answers only once Serve runs, and Run starts
|
|
// beside Serve: a watch asked too early is asked again, after a wait that doubles from FirstRetry up to a minute,
|
|
// until it is answered or ctx ends, so a start that loses the race never leaves the consumers unanswered (issue 943).
|
|
func (m *Moved) Run(ctx context.Context, p *provider.Provider, bus provider.Bus) error {
|
|
wait := m.FirstRetry
|
|
if wait <= 0 {
|
|
wait = time.Second
|
|
}
|
|
for tries := 1; ; tries++ {
|
|
err := m.watch(ctx, bus)
|
|
if err == nil {
|
|
break
|
|
}
|
|
if tries == 4 || tries%20 == 0 {
|
|
m.h.say("the plans cannot be watched yet (%d tries), asking again: %v", tries, err)
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
case <-time.After(wait):
|
|
}
|
|
wait = min(wait*2, time.Minute)
|
|
}
|
|
go func() {
|
|
for {
|
|
m.retire(ctx)
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-time.After(m.h.Every):
|
|
}
|
|
}
|
|
}()
|
|
return p.Run(ctx)
|
|
}
|
|
|
|
// NewMoved is a provider handed to the provisioner: the helper answering its consumers, and the harness (never run)
|
|
// whose retirement and tools it shares. consumersFile is where its machine's client fills its consumers. Run the
|
|
// result with Moved.Run, passing the provider and the bus returned with it.
|
|
func NewMoved(consumersFile string, h *Harness, spec MovedSpec) (*Moved, *provider.Provider, provider.Bus, error) {
|
|
if consumersFile == "" {
|
|
return nil, nil, nil, errors.New("no consumers' file")
|
|
}
|
|
run := newMoved(h, spec)
|
|
bus := provider.StateBus(spec.Module)
|
|
return run, provider.New(provider.Options{
|
|
Node: h.Node,
|
|
Module: spec.Module,
|
|
Offers: map[string]provider.Offer{spec.Provision: spec.Offer},
|
|
Bus: bus,
|
|
Vault: provider.ConsumersFile{Paths: []string{consumersFile}},
|
|
Make: run.Make,
|
|
Gone: run.Gone,
|
|
Every: h.Every,
|
|
Logf: func(f string, a ...any) { h.say(f, a...) },
|
|
}), bus, nil
|
|
}
|