jschoubben
666d1b682e
mesh/delivery held for a person: merged without a passing check: only a person decides what follows
mesh/check checking 666d1b682e7f on 045f25bd1353
registeredAt compared commits, so a catalogue merge that moved only the commit (2df05030 at 18:02, a85d9037 before it) took every catalogue module in again and composed it: about 200 take-ins, some 25 minutes, on every mesh-catalog merge, which held nextcloud for an hour. A module whose trunk fingerprint is the one it is registered at now has the new commit recorded (built_from and source_head, as a registration moves them) and composes nothing unless its composition or grant is missing. Readers of the registered commit: the fingerprints read the commit's build or, with none, the indexer's value at it, which names the same fingerprint; status, modules, the board and the facts read it as the source's commit, which it is; the upgrade path counts it as built from that commit, which it is. The stall watch counted only settles the bus accepted, so each refused acknowledgement at a first start left work held for good: it now counts the loop's every attempt. The acknowledgement and refusals at the stop are sent past the stop, and the connection closes only after the readers and the loop have ended. Research 072: the take-ins on a bounded pull consumer.
447 lines
16 KiB
Go
447 lines
16 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"maps"
|
|
"slices"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
|
"github.com/nats-io/nats.go/jetstream"
|
|
"github.com/novox/mesh-controller/internal/broker"
|
|
"github.com/novox/mesh-controller/internal/builderkeys"
|
|
"github.com/novox/mesh-controller/internal/conditions"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
)
|
|
|
|
// A new fingerprint of a registered module, from its done key to its compositions (novox/hq ADR 0539 clauses 5 and 7,
|
|
// ADR 0501 §2, ADR 0544). With no delivery plan left to take a module's done key in, a write of the builder's or the
|
|
// indexer's keys does it: the controller takes in the done key of the fingerprint the indexer's trunk value names
|
|
// (registers the module at it, as a plan's take-in did), composes every node the module is assigned to (which makes what
|
|
// a module's new manifest needs, a credential among it), composes the node holding the bus (its user list now carries
|
|
// the module's permissions at the new fingerprint), and writes `grants.<module>.<fp>`. The assigner's sixth and seventh
|
|
// rules then let a want name the fingerprint.
|
|
//
|
|
// **A durable pull consumer, not a watch** (ADR 0544; research 072, milestones 2 and 4). The writes are read from two
|
|
// durable consumers (take_in_consumer.go): the bus keeps what is not taken in yet, so a restart takes in only what the
|
|
// controller before it left unacknowledged, and the start's look is the consumers' own catch-up. A batch of keys asks the
|
|
// composer twice whatever its size, and a key with nothing new composes nothing.
|
|
|
|
// takeInSteps are what taking a batch of done keys in does, each a step a test may stand in for.
|
|
type takeInSteps struct {
|
|
// catalogue is every module the catalogue holds, with the nodes it is assigned to: read once a batch. A module it
|
|
// does not hold is the new-module take-in's (new_modules.go).
|
|
catalogue func(ctx context.Context) (map[string][]string, error)
|
|
// trunk is the indexer's trunk value of the module.
|
|
trunk func(ctx context.Context, module string) (sdkdelivery.Module, error)
|
|
// registered says whether the module is registered at the trunk's commit, and whether at its fingerprint: a commit
|
|
// that moves while the fingerprint stays (a catalogue merge that leaves the module's sources alone) is no new build.
|
|
registered func(ctx context.Context, module string, trunk sdkdelivery.Module) (atCommit, atFP bool, err error)
|
|
// recordCommit records the module's registration at the trunk's commit, its fingerprint unchanged: nothing composed.
|
|
recordCommit func(ctx context.Context, module, commit string) error
|
|
// takeIn registers the module from the builder's done key of the trunk's fingerprint (takeInKey).
|
|
takeIn func(ctx context.Context, module string, trunk sdkdelivery.Module) error
|
|
// taken says the module's compositions at fp exist for every node and its grant is written: nothing to compose.
|
|
taken func(ctx context.Context, module, fp string, nodes []string) (bool, error)
|
|
// busHolder is the node holding the bus, "" for none.
|
|
busHolder func(ctx context.Context) (string, error)
|
|
// compose composes the nodes, and writes the grants after when grants: each node's error, and the grants'.
|
|
compose func(ctx context.Context, nodes []string, grants bool) (map[string]error, error, error)
|
|
}
|
|
|
|
// delivered is one key a consumer delivered: its module, and whether it was delivered before (a take-in that may have
|
|
// stopped half-way, so it is taken in whole).
|
|
type delivered struct {
|
|
module string
|
|
again bool
|
|
// deliveries is how often the bus delivered it, this time included: what a refusal's delay grows with.
|
|
deliveries uint64
|
|
}
|
|
|
|
// verdict is what becomes of a module's delivered keys.
|
|
type verdict int
|
|
|
|
const (
|
|
// acknowledged: taken in, nothing new, a wait a later write ends, or a build key that failed.
|
|
acknowledged verdict = iota
|
|
// refused: delivered again by the bus after doneKeyRetry (a wait on a read, or a failure).
|
|
refused
|
|
)
|
|
|
|
// settledKey is what a batch did with one module.
|
|
type settledKey struct {
|
|
module string
|
|
verdict verdict
|
|
// taken: registered, composed and granted now (not nothing new); its condition is cleared.
|
|
taken bool
|
|
// nothingNew: registered and composed before, so nothing was done.
|
|
nothingNew bool
|
|
// waits is why it waits, "" when it does not.
|
|
waits string
|
|
// err is a failure: raised as the module's condition.
|
|
err error
|
|
// said is the line said of it, "" for none.
|
|
said string
|
|
}
|
|
|
|
// batch takes the delivered keys in, each module once: it answers one settledKey per module in the order first
|
|
// delivered, and the number of composer passes it asked. memo is, per module, the trunk (fingerprint and commit) this
|
|
// process took in last; a module at the same trunk delivered for the first time has nothing new.
|
|
func (s takeInSteps) batch(ctx context.Context, keys []delivered, memo map[string]string) ([]settledKey, int) {
|
|
var order []string
|
|
again := map[string]bool{}
|
|
for _, k := range keys {
|
|
if _, seen := again[k.module]; !seen {
|
|
order = append(order, k.module)
|
|
}
|
|
again[k.module] = again[k.module] || k.again
|
|
}
|
|
out := make([]settledKey, 0, len(order))
|
|
all := func(v verdict, err error) ([]settledKey, int) {
|
|
for _, m := range order {
|
|
out = append(out, settledKey{module: m, verdict: v, err: err})
|
|
}
|
|
return out, 0
|
|
}
|
|
held, err := s.catalogue(ctx)
|
|
if err != nil {
|
|
return all(refused, fmt.Errorf("the catalogue cannot be read: %w", err))
|
|
}
|
|
type toCompose struct {
|
|
at int
|
|
nodes []string
|
|
mark string
|
|
said string
|
|
}
|
|
var need []toCompose
|
|
for _, m := range order {
|
|
r := settledKey{module: m}
|
|
nodes, isHeld := held[m]
|
|
if !isHeld {
|
|
out = append(out, r)
|
|
continue
|
|
}
|
|
trunk, err := s.trunk(ctx, m)
|
|
if errors.Is(err, builderkeys.ErrNoValue) {
|
|
out = append(out, r)
|
|
continue
|
|
}
|
|
if err == nil && !strings.HasPrefix(trunk.FP2, sdkdelivery.SourcePrefix2) {
|
|
// The indexer's next write names one: said once, as every wait for a write (ADR 0544 §4).
|
|
why := trunk.None2
|
|
if why == "" {
|
|
why = "the indexer wrote no fingerprint of the new form"
|
|
}
|
|
r.waits = fmt.Sprintf("%v: %s at %.8s has no fingerprint to take a build by: %s", errWaitsForAWrite, m,
|
|
trunk.Commit, why)
|
|
out = append(out, r)
|
|
continue
|
|
}
|
|
if err != nil {
|
|
r.verdict, r.err = refused, err
|
|
out = append(out, r)
|
|
continue
|
|
}
|
|
mark := trunk.FP2 + "@" + trunk.Commit
|
|
if !again[m] && memo[m] == mark {
|
|
r.nothingNew = true
|
|
out = append(out, r)
|
|
continue
|
|
}
|
|
at, atFP, err := s.registered(ctx, m, trunk)
|
|
if err == nil && !at && atFP {
|
|
// Only the commit moved (issue 1945: every catalogue merge took about 200 modules in again, 25 minutes): the
|
|
// new commit is recorded, and the module is judged as one registered, composed only if something is missing.
|
|
if err = s.recordCommit(ctx, m, trunk.Commit); err == nil {
|
|
at = true
|
|
}
|
|
}
|
|
if err != nil {
|
|
r.verdict, r.err = refused, err
|
|
out = append(out, r)
|
|
continue
|
|
}
|
|
said := ""
|
|
if !at {
|
|
if err := s.takeIn(ctx, m, trunk); err != nil {
|
|
switch {
|
|
case errors.Is(err, errWaitsForAWrite):
|
|
r.waits = err.Error()
|
|
case errors.Is(err, errKeyWaits):
|
|
r.verdict, r.waits = refused, err.Error()
|
|
case errors.Is(err, errKeyFailed):
|
|
r.err = err // acknowledged: only a new build ends it, and its key is delivered then
|
|
default:
|
|
r.verdict, r.err = refused, err
|
|
}
|
|
out = append(out, r)
|
|
continue
|
|
}
|
|
said = fmt.Sprintf("%s registered from the builder's done key %s (%s)", m, short(trunk.FP2), short(trunk.Commit))
|
|
} else if !again[m] {
|
|
// Registered at its trunk already: composed and granted by a take-in before this one, or at this
|
|
// controller's start, unless that take-in stopped half-way, which leaves a composition or the grant missing.
|
|
if done, err := s.taken(ctx, m, trunk.FP2, nodes); err == nil && done {
|
|
memo[m] = mark
|
|
r.nothingNew = true
|
|
out = append(out, r)
|
|
continue
|
|
}
|
|
}
|
|
if len(nodes) == 0 {
|
|
memo[m] = mark
|
|
r.taken, r.said = true, said
|
|
out = append(out, r)
|
|
continue
|
|
}
|
|
if said == "" {
|
|
said = fmt.Sprintf("%s at %s", m, short(trunk.FP2))
|
|
}
|
|
need = append(need, toCompose{at: len(out), nodes: nodes, mark: mark, said: said})
|
|
out = append(out, r)
|
|
}
|
|
if len(need) == 0 {
|
|
return out, 0
|
|
}
|
|
refuseNeed := func(err error) ([]settledKey, int) {
|
|
for _, n := range need {
|
|
out[n.at].verdict, out[n.at].err = refused, err
|
|
}
|
|
return out, 0
|
|
}
|
|
holder, err := s.busHolder(ctx)
|
|
if err != nil {
|
|
return refuseNeed(fmt.Errorf("the node holding the bus cannot be read: %w", err))
|
|
}
|
|
// The modules' nodes first, in one pass: composing them makes what their new manifests need, credentials among it.
|
|
var union []string
|
|
for _, n := range need {
|
|
for _, node := range n.nodes {
|
|
if node != holder && !slices.Contains(union, node) {
|
|
union = append(union, node)
|
|
}
|
|
}
|
|
}
|
|
sort.Strings(union)
|
|
passes := 0
|
|
failed := map[string]error{}
|
|
if len(union) > 0 {
|
|
f, _, err := s.compose(ctx, union, false)
|
|
passes++
|
|
if err != nil {
|
|
return refuseNeed(err)
|
|
}
|
|
maps.Copy(failed, f)
|
|
}
|
|
// Then the bus's user list, which carries the modules' permissions at their new fingerprints, and the grants, which
|
|
// with the compositions let a want name each fingerprint (rules 6 and 7).
|
|
var bus []string
|
|
if holder != "" {
|
|
bus = []string{holder}
|
|
}
|
|
f, grantsErr, err := s.compose(ctx, bus, true)
|
|
passes++
|
|
if err != nil {
|
|
return refuseNeed(err)
|
|
}
|
|
maps.Copy(failed, f)
|
|
for _, n := range need {
|
|
var errs []error
|
|
for _, node := range append(slices.Clone(n.nodes), bus...) {
|
|
if e := failed[node]; e != nil && !slices.ContainsFunc(errs, func(x error) bool { return x == e }) {
|
|
errs = append(errs, e)
|
|
}
|
|
}
|
|
if grantsErr != nil {
|
|
errs = append(errs, grantsErr)
|
|
}
|
|
r := &out[n.at]
|
|
if len(errs) > 0 {
|
|
r.verdict, r.err = refused, errors.Join(errs...)
|
|
continue
|
|
}
|
|
memo[r.module] = n.mark
|
|
r.taken = true
|
|
r.said = n.said + "; composed for " + strings.Join(n.nodes, ", ") + ", its grant written"
|
|
}
|
|
return out, passes
|
|
}
|
|
|
|
// doneKeysSay says a line of the take-ins; a variable so a test may hear it.
|
|
var doneKeysSay = func(s string) { fmt.Println(s) }
|
|
|
|
// doneKeyRetry is how long the bus waits before it delivers again a key whose take-in was refused the first time; each
|
|
// refusal after doubles it, up to doneKeyRetryMost, so a key refused for long (a manifest word this controller does not
|
|
// read yet, a node whose composition keeps failing) costs a take-in every few minutes, not every half minute.
|
|
var doneKeyRetry = 30 * time.Second
|
|
|
|
// doneKeyRetryMost is the longest a refused key waits before it is delivered again.
|
|
const doneKeyRetryMost = 10 * time.Minute
|
|
|
|
// refusalDelay is how long the bus waits before it delivers again a key refused on its deliveries-th delivery.
|
|
func refusalDelay(deliveries uint64) time.Duration {
|
|
d := doneKeyRetry
|
|
for i := uint64(1); i < deliveries && d < doneKeyRetryMost; i++ {
|
|
d *= 2
|
|
}
|
|
return min(d, doneKeyRetryMost)
|
|
}
|
|
|
|
// kindNotComposed is the condition of a module's new fingerprint whose compositions or grant are not written: until
|
|
// they are, no want can name it.
|
|
const kindNotComposed = "fingerprint-not-composed"
|
|
|
|
func doneKeyObservation(module string) conditions.Observation {
|
|
return conditions.Observation{Scope: conditions.ScopeBuild, ID: module, Token: kindNotComposed, Kind: kindNotComposed,
|
|
Severity: conditions.Urgent, Resolver: conditions.ResolverOperator, Source: "done keys",
|
|
Summary: module + "'s new fingerprint is not composed or granted, so no want can name it: "}
|
|
}
|
|
|
|
// liveTakeInSteps are the steps on the controller's own stores and the bus.
|
|
func liveTakeInSteps(open *stores, keys registerKeys, js jetstream.JetStream) takeInSteps {
|
|
inv := open.inventory
|
|
return takeInSteps{
|
|
catalogue: func(ctx context.Context) (map[string][]string, error) {
|
|
entries, err := inv.Catalogued(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out := make(map[string][]string, len(entries))
|
|
for _, e := range entries {
|
|
out[e.Manifest.Module] = e.On
|
|
}
|
|
return out, nil
|
|
},
|
|
trunk: func(ctx context.Context, module string) (sdkdelivery.Module, error) {
|
|
return keys.ModuleAt(ctx, module, "")
|
|
},
|
|
registered: func(ctx context.Context, module string, trunk sdkdelivery.Module) (bool, bool, error) {
|
|
src, err := inv.SourceOf(ctx, module)
|
|
if err != nil {
|
|
return false, false, err
|
|
}
|
|
if sameCommitID(src.BuiltFrom, trunk.Commit) {
|
|
return true, true, nil
|
|
}
|
|
fp, err := registeredFingerprint(ctx, inv, keys, module, src.BuiltFrom)
|
|
return false, err == nil && fp != "" && fp == trunk.FP2, nil
|
|
},
|
|
recordCommit: func(ctx context.Context, module, commit string) error {
|
|
return inv.RecordBuiltFrom(ctx, module, commit)
|
|
},
|
|
takeIn: func(ctx context.Context, module string, trunk sdkdelivery.Module) error {
|
|
_, _, err := takeInKey(ctx, inv, keys, keyTake{Module: module, Trunk: trunk, Ask: registrationAsk{Merge: trunk.Commit}})
|
|
if errors.Is(err, inventory.ErrSuperseded) {
|
|
return nil
|
|
}
|
|
return err
|
|
},
|
|
taken: func(ctx context.Context, module, fp string, nodes []string) (bool, error) {
|
|
return composedAndGranted(ctx, js, module, fp, nodes)
|
|
},
|
|
busHolder: func(ctx context.Context) (string, error) {
|
|
holders, err := seatHolders(ctx, inv)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return holders[catalogue.BrokerSeat].Node, nil
|
|
},
|
|
// The composer's, never the take-in's own: one pass at a time composes each node asked for once, and writes the
|
|
// grants once (issue 1945, composer.go).
|
|
compose: func(ctx context.Context, nodes []string, grants bool) (map[string]error, error, error) {
|
|
return composerFor(open).composeEach(ctx, nodes, grants)
|
|
},
|
|
}
|
|
}
|
|
|
|
// registeredFingerprint is the fingerprint a module is registered at, as currentFingerprints reads it: its commit's
|
|
// newest build's, or the indexer's value at the commit when the commit has no build of its own; "" when neither says.
|
|
func registeredFingerprint(ctx context.Context, inv *inventory.Inventory, keys registerKeys, module, commit string) (string, error) {
|
|
if commit == "" {
|
|
return "", nil
|
|
}
|
|
if id, err := inv.BuildOf(ctx, module, commit); err == nil && id != "" {
|
|
if b, found, err := inv.BuildByID(ctx, id); err == nil && found && b.SourceFingerprint != "" {
|
|
return b.SourceFingerprint, nil
|
|
}
|
|
}
|
|
at, err := keys.ModuleAt(ctx, module, commit)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return at.FP2, nil
|
|
}
|
|
|
|
// composedAndGranted says the bus holds module's grant at fp and its composition at fp for every node: a few reads of
|
|
// keys, never a composition.
|
|
func composedAndGranted(ctx context.Context, js jetstream.JetStream, module, fp string, nodes []string) (bool, error) {
|
|
grants, err := js.KeyValue(ctx, broker.GrantsBucket)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
key, err := sdkdelivery.ControllerGrantKey(module, fp)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
if _, err := grants.Get(ctx, key); err != nil {
|
|
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
|
return false, nil
|
|
}
|
|
return false, err
|
|
}
|
|
digests, err := js.KeyValue(ctx, sdkdelivery.ControllerComposedBucket)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
for _, node := range nodes {
|
|
key, err := sdkdelivery.ComposedKey(node, module, fp)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
if _, err := digests.Get(ctx, key); err != nil {
|
|
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
|
return false, nil
|
|
}
|
|
return false, err
|
|
}
|
|
}
|
|
return true, nil
|
|
}
|
|
|
|
// doneKeysComposers are the composers of the done keys' take-ins in this process, one for each set of stores (a test
|
|
// runs several meshes), each started at its first use and run for as long as the process serves.
|
|
var doneKeysComposers sync.Map
|
|
|
|
// composerFor is the composer of the stores, its passes composing on them and writing the grants on the bus.
|
|
func composerFor(open *stores) *composer {
|
|
if c, ok := doneKeysComposers.Load(open); ok {
|
|
return c.(*composer)
|
|
}
|
|
c := newComposer(func(ctx context.Context, nodes []string) map[string]error {
|
|
_, failed := composeModulesByNode(ctx, open, nodes)
|
|
return failed
|
|
}, func(ctx context.Context) error {
|
|
bus, err := aBus()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer bus.Close()
|
|
js, err := jetstream.New(bus.Conn())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return grantsIssued(ctx, open.inventory, js, "done keys: ")
|
|
})
|
|
if was, loaded := doneKeysComposers.LoadOrStore(open, c); loaded {
|
|
return was.(*composer)
|
|
}
|
|
go c.run(context.Background())
|
|
return c
|
|
}
|