mesh-sdk/go/provider/harness/moved.go

package harness

// A provider handed to the provisioner (novox/hq ADR 0373, ADR 0349 clause 12, issues 780 and 805): where its
// consumers file is set, it runs on the provider helper (package provider) instead of the harness reading the
// controller's grant files. Moved is that path, shared by every provider that keeps a resource per consumer; a module
// says only what differs, in a MovedSpec.
//
// The helper learns who binds the provider from the plans, composes each consumer's entry and reads its credential
// through ConsumersFile, the file this machine's provisioner client fills. It leaves the rest to the two hooks
// here: Make brings the consumer's resource into being exactly as the harness's adapter does, and Gone leaves
// retirement to the harness's guards (retirement.go): the resource is disabled and marked to delete, never removed.
//
// **A consumer not moved is still the grant file's** (issue 937, ADR 0349 clause 17), for a provider whose spec says
// WithGrants: only a moved consumer's plan names its provider, so the controller still gives every other consumer
// through the grant file (MESH_RECEIVES) until R1 (issue 682). Each login it gives is asked for, and one no plan
// names is made and checked by the harness's grant pass exactly as on the grant path; a login a plan names is the
// helper's alone. A grant file that cannot be read is not nobody asking (issue 241): nothing is retired until it is
// read again.

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"sync"
	"time"

	"git.novox.be/novox/mesh-sdk/go/provider"
)

// MovedSpec is what differs per provider on the moved path.
type MovedSpec struct {
	// Module and Provision are the provider module's name and what it provides, as its manifest says.
	Module, Provision string
	// Offer is what it serves: its credential form and served values.
	Offer provider.Offer
	// Login is the identity a consumer is made under at the provider's end (postgres: the entry's User; keycloak: its
	// As); an entry with none is not made.
	Login func(e provider.Entry) string
	// Values are the values the adapter is handed for a consumer; the entry's contribution when not set.
	Values func(e provider.Entry) map[string]any
	// Noun names what Login is in words ("login", "client id"), Made what is made ("login", "client"), and Backend
	// the service in "no longer holds it" ("the backend", "Keycloak").
	Noun, Made, Backend string
	// WithGrants makes the logins the grant file gives, that no plan names, asked for and made by the harness's grant
	// pass (issue 937).
	WithGrants bool
}

// Moved is a provider on the provider helper, with the harness's own memory and retirement behind it: what is held,
// what is counted unasked, what waits for a person (retirement.go). The harness is never Run here; the helper's Make
// and Gone and the loop below drive the same state, so the guards are the harness's, not a copy.
type Moved struct {
	h    *Harness
	spec MovedSpec

	// FirstRetry is the first wait before a watch asked too early is asked again (issue 943); a second when zero.
	FirstRetry time.Duration

	mu          sync.Mutex // guards assignments and loaded
	assignments map[string][]byte
	loaded      bool
	verified    map[string]time.Time
}

func newMoved(h *Harness, spec MovedSpec) *Moved {
	h.mu.Lock()
	h.init()
	h.mu.Unlock()
	return &Moved{h: h, spec: spec, assignments: map[string][]byte{}, verified: map[string]time.Time{}}
}

// Make brings one consumer's resource into being under its login and the vault's credential, as the harness does:
// at once when it is new or changed, and otherwise only when the backend no longer holds it, which is asked every
// VerifyEvery.
func (m *Moved) Make(ctx context.Context, c provider.Consumer) error {
	h, e := m.h, c.Entry
	as := m.spec.Login(e)
	if as == "" {
		return fmt.Errorf("%s has no %s to make", e.Key(), m.spec.Noun)
	}
	if len(c.Credential) == 0 {
		return fmt.Errorf("%s has no credential to make its %s with", e.Key(), m.spec.Made)
	}
	h.mu.Lock()
	defer h.mu.Unlock()
	password := string(c.Credential)
	values := orEmpty(e.Contributes)
	if m.spec.Values != nil {
		values = m.spec.Values(e)
	}
	p := Provision{As: as, Password: password, Values: values, Consumer: e.Consumer}
	hash := hashOf(as, password, p.Values, nil)
	if was, ok := h.applied[as]; ok && was.hash == hash {
		if h.Now().Sub(m.verified[as]) < h.VerifyEvery {
			return nil
		}
		m.verified[as] = h.Now()
		hctx, cancel := context.WithTimeout(ctx, h.HoldsTimeout)
		held, err := h.Adapter.Holds(hctx, p)
		cancel()
		if err != nil {
			text := scrub(err, password)
			h.failed(as, e.Consumer, h.classOf(err, text), text)
			return errors.New(text)
		}
		if held {
			h.succeeded(as)
			return nil
		}
		h.say("%s: %s no longer holds it; applying again", as, m.spec.Backend)
	}
	if err := h.Adapter.Create(ctx, p); err != nil {
		text := scrub(err, password)
		h.failed(as, e.Consumer, h.classOf(err, text), text)
		return errors.New(text)
	}
	h.succeeded(as)
	h.applied[as] = appliedEntry{hash: hash, node: e.Consumer}
	m.verified[as] = h.Now()
	h.reenabled(as, e.Consumer)
	return nil
}

// Gone is the helper asking whether it may forget a consumer that no longer binds. **It never retires**: retiring is
// the harness's rule, the same unasked set over StablePasses passes and StableFor, at most RetireAtOnce at a time
// (retirement.go, ADR 0230), run by retire below. So a consumer missing from one read of the plans loses nothing:
// the helper is told no, and asks again at its next pass, until the login is retired, or asked for again.
func (m *Moved) Gone(_ context.Context, e provider.Entry) error {
	h := m.h
	as := m.spec.Login(e)
	if as == "" {
		return nil // refused: nothing was made for it
	}
	// The plans and the grant file as they are now, not as the last retirement pass read them: with either not
	// read, nothing is let go.
	want, read := m.asked(h.Node)
	h.mu.Lock()
	defer h.mu.Unlock()
	asked := want[as]
	if m.spec.WithGrants {
		_, granted, ok := m.granted(want)
		read = read && ok
		asked = asked || granted[as]
	}
	_, held := h.applied[as]
	_, retired := h.r.retired[as]
	switch {
	case !read:
	case asked:
		return nil // another holder of the same login still binds: nothing to retire
	case retired, !held && h.r.seeded:
		return nil
	}
	return fmt.Errorf("%s is kept until its retirement is stable (%d passes and %s) and within the bound, or a "+
		"person approves it", as, h.StablePasses, h.StableFor)
}

// plan is as much of an assignment as is read to learn who binds the provider.
type plan struct {
	Node         string `json:"node"`
	Module       string `json:"module"`
	Provisioning *struct {
		Provisions []struct {
			Provision      string `json:"provision"`
			Provider       string `json:"provider"`
			ProviderModule string `json:"provider-module"`
			// As is who the consumer is at its provider's end, its slug first (issue 788).
			As string `json:"as"`
		} `json:"provisions"`
	} `json:"provisioning"`
}

// asked is the logins the plans bind the provider for, and whether they could all be read: a plan that is not the
// controller's, or no read yet, is not an answer, so nothing is counted unasked from it (novox/hq issue 241).
func (m *Moved) asked(node string) (map[string]bool, bool) {
	m.mu.Lock()
	defer m.mu.Unlock()
	if !m.loaded || len(m.assignments) == 0 {
		return nil, false
	}
	want := map[string]bool{}
	for key, raw := range m.assignments {
		var a plan
		if err := json.Unmarshal(raw, &a); err != nil || a.Node+"."+a.Module != key {
			return nil, false
		}
		if a.Provisioning == nil {
			continue
		}
		for _, b := range a.Provisioning.Provisions {
			if b.Provider == node && b.ProviderModule == m.spec.Module && b.Provision == m.spec.Provision {
				as := b.As
				if as == "" {
					as = provider.ConsumerIdentity(a.Node, a.Module) // a plan that names none
				}
				want[as] = true
			}
		}
	}
	return want, true
}

// granted is what the grant file gives that no plan names (issue 937), and every login it gives; and whether it
// could be read. The caller holds h.mu. With no grant file named there is nothing given, and that is a read.
func (m *Moved) granted(planned map[string]bool) ([]contribution, map[string]bool, bool) {
	h := m.h
	if h.Receives == "" {
		return nil, map[string]bool{}, true
	}
	given := h.readContributions()
	if given == nil {
		return nil, nil, false
	}
	var own []contribution
	logins := map[string]bool{}
	for _, g := range given {
		logins[g.As] = true
		if !planned[g.As] {
			own = append(own, g)
		}
	}
	return own, logins, true
}

// retire is one pass of the harness's retirement over what the plans bind and the grant file gives now; the logins
// only the grant file gives are made and checked first, as the grant path does.
func (m *Moved) retire(ctx context.Context) {
	h := m.h
	want, ok := m.asked(h.Node)
	h.mu.Lock()
	defer h.mu.Unlock()
	var given []contribution
	if ok && m.spec.WithGrants {
		var granted map[string]bool
		given, granted, ok = m.granted(want)
		for as := range granted {
			want[as] = true
		}
	}
	if !ok {
		// Not a result: the passes that must agree before anything is retired start again.
		h.r.key, h.r.count = "", 0
		return
	}
	h.seed(ctx, want)
	h.apply(ctx, given)
	h.retireUnasked(ctx, want)
	h.r.lastWant = want
	for as := range h.failing {
		if !want[as] {
			delete(h.failing, as)
		}
	}
	for as := range h.trouble {
		if _, held := h.applied[as]; !want[as] && !held {
			h.recovered(as, "no longer asked for")
		}
	}
}

// watch keeps the plans as the bus holds them, from the first read on.
func (m *Moved) watch(ctx context.Context, bus provider.Bus) error {
	err := bus.Watch(ctx, provider.AssignmentsBucket, func(key string, value []byte) {
		m.mu.Lock()
		defer m.mu.Unlock()
		if value == nil {
			delete(m.assignments, key)
		} else {
			m.assignments[key] = append([]byte(nil), value...)
		}
	})
	if err == nil {
		m.mu.Lock()
		m.loaded = true
		m.mu.Unlock()
	}
	return err
}

// Run answers the consumers through the helper and retires through the harness's rules, until ctx ends.
//
// The bus is reached through the runtime that launched this bundle, which answers only once Serve runs, and Run starts
// beside Serve: a watch asked too early is asked again, after a wait that doubles from FirstRetry up to a minute,
// until it is answered or ctx ends, so a start that loses the race never leaves the consumers unanswered (issue 943).
func (m *Moved) Run(ctx context.Context, p *provider.Provider, bus provider.Bus) error {
	wait := m.FirstRetry
	if wait <= 0 {
		wait = time.Second
	}
	for tries := 1; ; tries++ {
		err := m.watch(ctx, bus)
		if err == nil {
			break
		}
		if tries == 4 || tries%20 == 0 {
			m.h.say("the plans cannot be watched yet (%d tries), asking again: %v", tries, err)
		}
		select {
		case <-ctx.Done():
			return ctx.Err()
		case <-time.After(wait):
		}
		wait = min(wait*2, time.Minute)
	}
	go func() {
		for {
			m.retire(ctx)
			select {
			case <-ctx.Done():
				return
			case <-time.After(m.h.Every):
			}
		}
	}()
	return p.Run(ctx)
}

// NewMoved is a provider handed to the provisioner: the helper answering its consumers, and the harness (never run)
// whose retirement and tools it shares. consumersFile is where its machine's client fills its consumers. Run the
// result with Moved.Run, passing the provider and the bus returned with it.
func NewMoved(consumersFile string, h *Harness, spec MovedSpec) (*Moved, *provider.Provider, provider.Bus, error) {
	if consumersFile == "" {
		return nil, nil, nil, errors.New("no consumers' file")
	}
	run := newMoved(h, spec)
	bus := provider.StateBus(spec.Module)
	return run, provider.New(provider.Options{
		Node:   h.Node,
		Module: spec.Module,
		Offers: map[string]provider.Offer{spec.Provision: spec.Offer},
		Bus:    bus,
		Vault:  provider.ConsumersFile{Paths: []string{consumersFile}},
		Make:   run.Make,
		Gone:   run.Gone,
		Every:  h.Every,
		Logf:   func(f string, a ...any) { h.say(f, a...) },
	}), bus, nil
}