jschoubben
bfb271a91b
The take-in watched the builder's and the indexer's buckets, started a follower per module that retried on a 30 s timer, and at start took every catalogued module in alphabetically: on 2026-10-25 a restart took in about 2 modules a minute, about 190 followers queued behind 16 slots, and every take-in composed two or three times even with nothing new, so a new build waited 30 minutes and more for its credential. Now two durable consumers named mesh-controller-take-in read the indexer's modules bucket (every module's last trunk value when the consumer is made, then each write) and the builder's builds bucket (each write), at most 16 keys unacknowledged each. The bus keeps what is not taken in: the start's look is the consumers' own catch-up, and a restart costs only what was left unacknowledged. A batch reads the catalogue once, takes each module in once, and asks the composer twice: once for its modules' nodes, once for the bus's node with the grants. A module registered at its trunk whose compositions and grant exist, or that this process took in at the same trunk, composes nothing; a key delivered again is taken in whole. A wait for the builder's or indexer's next write is acknowledged, a wait on a read or a failure refused with a 30 s delay. The backlog is in status, and a backlog standing still for 5 minutes raises take-in-stalled. The controller's account is granted the two consumers' acks. The follower tests go with the followers: twelve tests become three. Research 072: the take-ins on a bounded pull consumer (milestone 4, done keys alone) and the start's look composing only what changed (milestone 2).
385 lines
13 KiB
Go
385 lines
13 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"maps"
|
|
"slices"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
|
"github.com/nats-io/nats.go/jetstream"
|
|
"github.com/novox/mesh-controller/internal/broker"
|
|
"github.com/novox/mesh-controller/internal/builderkeys"
|
|
"github.com/novox/mesh-controller/internal/conditions"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
)
|
|
|
|
// A new fingerprint of a registered module, from its done key to its compositions (novox/hq ADR 0539 clauses 5 and 7,
|
|
// ADR 0501 §2, ADR 0544). With no delivery plan left to take a module's done key in, a write of the builder's or the
|
|
// indexer's keys does it: the controller takes in the done key of the fingerprint the indexer's trunk value names
|
|
// (registers the module at it, as a plan's take-in did), composes every node the module is assigned to (which makes what
|
|
// a module's new manifest needs, a credential among it), composes the node holding the bus (its user list now carries
|
|
// the module's permissions at the new fingerprint), and writes `grants.<module>.<fp>`. The assigner's sixth and seventh
|
|
// rules then let a want name the fingerprint.
|
|
//
|
|
// **A durable pull consumer, not a watch** (ADR 0544; research 072, milestones 2 and 4). The writes are read from two
|
|
// durable consumers (take_in_consumer.go): the bus keeps what is not taken in yet, so a restart takes in only what the
|
|
// controller before it left unacknowledged, and the start's look is the consumers' own catch-up. A batch of keys asks the
|
|
// composer twice whatever its size, and a key with nothing new composes nothing.
|
|
|
|
// takeInSteps are what taking a batch of done keys in does, each a step a test may stand in for.
|
|
type takeInSteps struct {
|
|
// catalogue is every module the catalogue holds, with the nodes it is assigned to: read once a batch. A module it
|
|
// does not hold is the new-module take-in's (new_modules.go).
|
|
catalogue func(ctx context.Context) (map[string][]string, error)
|
|
// trunk is the indexer's trunk value of the module.
|
|
trunk func(ctx context.Context, module string) (sdkdelivery.Module, error)
|
|
// registeredAt says the module is registered at the commit.
|
|
registeredAt func(ctx context.Context, module, commit string) (bool, error)
|
|
// takeIn registers the module from the builder's done key of the trunk's fingerprint (takeInKey).
|
|
takeIn func(ctx context.Context, module string, trunk sdkdelivery.Module) error
|
|
// taken says the module's compositions at fp exist for every node and its grant is written: nothing to compose.
|
|
taken func(ctx context.Context, module, fp string, nodes []string) (bool, error)
|
|
// busHolder is the node holding the bus, "" for none.
|
|
busHolder func(ctx context.Context) (string, error)
|
|
// compose composes the nodes, and writes the grants after when grants: each node's error, and the grants'.
|
|
compose func(ctx context.Context, nodes []string, grants bool) (map[string]error, error, error)
|
|
}
|
|
|
|
// delivered is one key a consumer delivered: its module, and whether it was delivered before (a take-in that may have
|
|
// stopped half-way, so it is taken in whole).
|
|
type delivered struct {
|
|
module string
|
|
again bool
|
|
}
|
|
|
|
// verdict is what becomes of a module's delivered keys.
|
|
type verdict int
|
|
|
|
const (
|
|
// acknowledged: taken in, nothing new, a wait a later write ends, or a build key that failed.
|
|
acknowledged verdict = iota
|
|
// refused: delivered again by the bus after doneKeyRetry (a wait on a read, or a failure).
|
|
refused
|
|
)
|
|
|
|
// settledKey is what a batch did with one module.
|
|
type settledKey struct {
|
|
module string
|
|
verdict verdict
|
|
// taken: registered, composed and granted now (not nothing new); its condition is cleared.
|
|
taken bool
|
|
// nothingNew: registered and composed before, so nothing was done.
|
|
nothingNew bool
|
|
// waits is why it waits, "" when it does not.
|
|
waits string
|
|
// err is a failure: raised as the module's condition.
|
|
err error
|
|
// said is the line said of it, "" for none.
|
|
said string
|
|
}
|
|
|
|
// batch takes the delivered keys in, each module once: it answers one settledKey per module in the order first
|
|
// delivered, and the number of composer passes it asked. memo is, per module, the trunk (fingerprint and commit) this
|
|
// process took in last; a module at the same trunk delivered for the first time has nothing new.
|
|
func (s takeInSteps) batch(ctx context.Context, keys []delivered, memo map[string]string) ([]settledKey, int) {
|
|
var order []string
|
|
again := map[string]bool{}
|
|
for _, k := range keys {
|
|
if _, seen := again[k.module]; !seen {
|
|
order = append(order, k.module)
|
|
}
|
|
again[k.module] = again[k.module] || k.again
|
|
}
|
|
out := make([]settledKey, 0, len(order))
|
|
all := func(v verdict, err error) ([]settledKey, int) {
|
|
for _, m := range order {
|
|
out = append(out, settledKey{module: m, verdict: v, err: err})
|
|
}
|
|
return out, 0
|
|
}
|
|
held, err := s.catalogue(ctx)
|
|
if err != nil {
|
|
return all(refused, fmt.Errorf("the catalogue cannot be read: %w", err))
|
|
}
|
|
type toCompose struct {
|
|
at int
|
|
nodes []string
|
|
mark string
|
|
said string
|
|
}
|
|
var need []toCompose
|
|
for _, m := range order {
|
|
r := settledKey{module: m}
|
|
nodes, isHeld := held[m]
|
|
if !isHeld {
|
|
out = append(out, r)
|
|
continue
|
|
}
|
|
trunk, err := s.trunk(ctx, m)
|
|
if errors.Is(err, builderkeys.ErrNoValue) || (err == nil && !strings.HasPrefix(trunk.FP2, sdkdelivery.SourcePrefix2)) {
|
|
out = append(out, r)
|
|
continue
|
|
}
|
|
if err != nil {
|
|
r.verdict, r.err = refused, err
|
|
out = append(out, r)
|
|
continue
|
|
}
|
|
mark := trunk.FP2 + "@" + trunk.Commit
|
|
if !again[m] && memo[m] == mark {
|
|
r.nothingNew = true
|
|
out = append(out, r)
|
|
continue
|
|
}
|
|
at, err := s.registeredAt(ctx, m, trunk.Commit)
|
|
if err != nil {
|
|
r.verdict, r.err = refused, err
|
|
out = append(out, r)
|
|
continue
|
|
}
|
|
said := ""
|
|
if !at {
|
|
if err := s.takeIn(ctx, m, trunk); err != nil {
|
|
switch {
|
|
case errors.Is(err, errWaitsForAWrite):
|
|
r.waits = err.Error()
|
|
case errors.Is(err, errKeyWaits):
|
|
r.verdict, r.waits = refused, err.Error()
|
|
case errors.Is(err, errKeyFailed):
|
|
r.err = err // acknowledged: only a new build ends it, and its key is delivered then
|
|
default:
|
|
r.verdict, r.err = refused, err
|
|
}
|
|
out = append(out, r)
|
|
continue
|
|
}
|
|
said = fmt.Sprintf("%s registered from the builder's done key %s (%s)", m, short(trunk.FP2), short(trunk.Commit))
|
|
} else if !again[m] {
|
|
// Registered at its trunk already: composed and granted by a take-in before this one, or at this
|
|
// controller's start, unless that take-in stopped half-way, which leaves a composition or the grant missing.
|
|
if done, err := s.taken(ctx, m, trunk.FP2, nodes); err == nil && done {
|
|
memo[m] = mark
|
|
r.nothingNew = true
|
|
out = append(out, r)
|
|
continue
|
|
}
|
|
}
|
|
if len(nodes) == 0 {
|
|
memo[m] = mark
|
|
r.taken, r.said = true, said
|
|
out = append(out, r)
|
|
continue
|
|
}
|
|
if said == "" {
|
|
said = fmt.Sprintf("%s at %s", m, short(trunk.FP2))
|
|
}
|
|
need = append(need, toCompose{at: len(out), nodes: nodes, mark: mark, said: said})
|
|
out = append(out, r)
|
|
}
|
|
if len(need) == 0 {
|
|
return out, 0
|
|
}
|
|
refuseNeed := func(err error) ([]settledKey, int) {
|
|
for _, n := range need {
|
|
out[n.at].verdict, out[n.at].err = refused, err
|
|
}
|
|
return out, 0
|
|
}
|
|
holder, err := s.busHolder(ctx)
|
|
if err != nil {
|
|
return refuseNeed(fmt.Errorf("the node holding the bus cannot be read: %w", err))
|
|
}
|
|
// The modules' nodes first, in one pass: composing them makes what their new manifests need, credentials among it.
|
|
var union []string
|
|
for _, n := range need {
|
|
for _, node := range n.nodes {
|
|
if node != holder && !slices.Contains(union, node) {
|
|
union = append(union, node)
|
|
}
|
|
}
|
|
}
|
|
sort.Strings(union)
|
|
passes := 0
|
|
failed := map[string]error{}
|
|
if len(union) > 0 {
|
|
f, _, err := s.compose(ctx, union, false)
|
|
passes++
|
|
if err != nil {
|
|
return refuseNeed(err)
|
|
}
|
|
maps.Copy(failed, f)
|
|
}
|
|
// Then the bus's user list, which carries the modules' permissions at their new fingerprints, and the grants, which
|
|
// with the compositions let a want name each fingerprint (rules 6 and 7).
|
|
var bus []string
|
|
if holder != "" {
|
|
bus = []string{holder}
|
|
}
|
|
f, grantsErr, err := s.compose(ctx, bus, true)
|
|
passes++
|
|
if err != nil {
|
|
return refuseNeed(err)
|
|
}
|
|
maps.Copy(failed, f)
|
|
for _, n := range need {
|
|
var errs []error
|
|
for _, node := range append(slices.Clone(n.nodes), bus...) {
|
|
if e := failed[node]; e != nil && !slices.ContainsFunc(errs, func(x error) bool { return x == e }) {
|
|
errs = append(errs, e)
|
|
}
|
|
}
|
|
if grantsErr != nil {
|
|
errs = append(errs, grantsErr)
|
|
}
|
|
r := &out[n.at]
|
|
if len(errs) > 0 {
|
|
r.verdict, r.err = refused, errors.Join(errs...)
|
|
continue
|
|
}
|
|
memo[r.module] = n.mark
|
|
r.taken = true
|
|
r.said = n.said + "; composed for " + strings.Join(n.nodes, ", ") + ", its grant written"
|
|
}
|
|
return out, passes
|
|
}
|
|
|
|
// doneKeysSay says a line of the take-ins; a variable so a test may hear it.
|
|
var doneKeysSay = func(s string) { fmt.Println(s) }
|
|
|
|
// doneKeyRetry is how long the bus waits before it delivers again a key whose take-in was refused.
|
|
var doneKeyRetry = 30 * time.Second
|
|
|
|
// kindNotComposed is the condition of a module's new fingerprint whose compositions or grant are not written: until
|
|
// they are, no want can name it.
|
|
const kindNotComposed = "fingerprint-not-composed"
|
|
|
|
func doneKeyObservation(module string) conditions.Observation {
|
|
return conditions.Observation{Scope: conditions.ScopeBuild, ID: module, Token: kindNotComposed, Kind: kindNotComposed,
|
|
Severity: conditions.Urgent, Resolver: conditions.ResolverOperator, Source: "done keys",
|
|
Summary: module + "'s new fingerprint is not composed or granted, so no want can name it: "}
|
|
}
|
|
|
|
// liveTakeInSteps are the steps on the controller's own stores and the bus.
|
|
func liveTakeInSteps(open *stores, keys registerKeys, js jetstream.JetStream) takeInSteps {
|
|
inv := open.inventory
|
|
return takeInSteps{
|
|
catalogue: func(ctx context.Context) (map[string][]string, error) {
|
|
entries, err := inv.Catalogued(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out := make(map[string][]string, len(entries))
|
|
for _, e := range entries {
|
|
out[e.Manifest.Module] = e.On
|
|
}
|
|
return out, nil
|
|
},
|
|
trunk: func(ctx context.Context, module string) (sdkdelivery.Module, error) {
|
|
return keys.ModuleAt(ctx, module, "")
|
|
},
|
|
registeredAt: func(ctx context.Context, module, commit string) (bool, error) {
|
|
src, err := inv.SourceOf(ctx, module)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
return sameCommitID(src.BuiltFrom, commit), nil
|
|
},
|
|
takeIn: func(ctx context.Context, module string, trunk sdkdelivery.Module) error {
|
|
_, _, err := takeInKey(ctx, inv, keys, keyTake{Module: module, Trunk: trunk, Ask: registrationAsk{Merge: trunk.Commit}})
|
|
if errors.Is(err, inventory.ErrSuperseded) {
|
|
return nil
|
|
}
|
|
return err
|
|
},
|
|
taken: func(ctx context.Context, module, fp string, nodes []string) (bool, error) {
|
|
return composedAndGranted(ctx, js, module, fp, nodes)
|
|
},
|
|
busHolder: func(ctx context.Context) (string, error) {
|
|
holders, err := seatHolders(ctx, inv)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return holders[catalogue.BrokerSeat].Node, nil
|
|
},
|
|
// The composer's, never the take-in's own: one pass at a time composes each node asked for once, and writes the
|
|
// grants once (issue 1945, composer.go).
|
|
compose: func(ctx context.Context, nodes []string, grants bool) (map[string]error, error, error) {
|
|
return composerFor(open).composeEach(ctx, nodes, grants)
|
|
},
|
|
}
|
|
}
|
|
|
|
// composedAndGranted says the bus holds module's grant at fp and its composition at fp for every node: a few reads of
|
|
// keys, never a composition.
|
|
func composedAndGranted(ctx context.Context, js jetstream.JetStream, module, fp string, nodes []string) (bool, error) {
|
|
grants, err := js.KeyValue(ctx, broker.GrantsBucket)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
key, err := sdkdelivery.ControllerGrantKey(module, fp)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
if _, err := grants.Get(ctx, key); err != nil {
|
|
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
|
return false, nil
|
|
}
|
|
return false, err
|
|
}
|
|
digests, err := js.KeyValue(ctx, sdkdelivery.ControllerComposedBucket)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
for _, node := range nodes {
|
|
key, err := sdkdelivery.ComposedKey(node, module, fp)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
if _, err := digests.Get(ctx, key); err != nil {
|
|
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
|
return false, nil
|
|
}
|
|
return false, err
|
|
}
|
|
}
|
|
return true, nil
|
|
}
|
|
|
|
// doneKeysComposers are the composers of the done keys' take-ins in this process, one for each set of stores (a test
|
|
// runs several meshes), each started at its first use and run for as long as the process serves.
|
|
var doneKeysComposers sync.Map
|
|
|
|
// composerFor is the composer of the stores, its passes composing on them and writing the grants on the bus.
|
|
func composerFor(open *stores) *composer {
|
|
if c, ok := doneKeysComposers.Load(open); ok {
|
|
return c.(*composer)
|
|
}
|
|
c := newComposer(func(ctx context.Context, nodes []string) map[string]error {
|
|
_, failed := composeModulesByNode(ctx, open, nodes)
|
|
return failed
|
|
}, func(ctx context.Context) error {
|
|
bus, err := aBus()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer bus.Close()
|
|
js, err := jetstream.New(bus.Conn())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return grantsIssued(ctx, open.inventory, js, "done keys: ")
|
|
})
|
|
if was, loaded := doneKeysComposers.LoadOrStore(open, c); loaded {
|
|
return was.(*composer)
|
|
}
|
|
go c.run(context.Background())
|
|
return c
|
|
}
|