jschoubben
3e39ccf958
The flag that made the first watch the start's look was cleared after every watch, even one that failed to open or ended before its Caught mark: the next watch then handed every standing assignment as fresh, about 300 settles at once with a catalogue read each, or dropped the keys the cut watch had handed. The flag is now cleared by the start's look itself, and a watch cut before it opens the next with nothing heard. Two rows of the existing table test cover it. Research 072: the grants piece (memberships and credentials heard from placement).
388 lines
16 KiB
Go
388 lines
16 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
|
|
"github.com/novox/mesh-controller/internal/broker"
|
|
"github.com/novox/mesh-controller/internal/chain"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
)
|
|
|
|
// An assignment that stands issues its node's bus accounts and the module's grant (novox/hq ADR 0543, issue 1945; a
|
|
// step of research 072's grants: the memberships and credentials heard from placement, not from a send or a build).
|
|
//
|
|
// ADR 0539 clause 7 issues a module's credential and grant when its build is done. A module already built and assigned
|
|
// to one more node never has a build done again, so nothing issued that node's account: on 2026-10-25 at 17:19
|
|
// mesh-build-node and mesh-build-go were assigned to ace, composed for it, and the bus's user list never named
|
|
// ace.mesh-build-node nor ace.mesh-build-go; one runtime found no credential, the other was refused by the bus, both
|
|
// restarting every 5 s. The assign act composes the assigned node alone (composedAfterAct): never the node holding the
|
|
// bus, never the grant.
|
|
//
|
|
// So the write of an assignments key is the event. When a key `<node>.<module>` of the assignments bucket comes to stand
|
|
// (a new key, or one that was ended or pending), the lease holder issues the module's accounts on that node that are not
|
|
// minted (idempotent: a minted account is never issued again, a rotation is on purpose), composes the node holding the
|
|
// bus so its user list carries them, and writes the grants. A rewrite of an assignment that already stood (the
|
|
// provisioner's plan kept in its value) does nothing. A starting controller looks at every standing assignment once,
|
|
// from one reading of the catalogue: it issues what is missing, then composes the bus's node and writes the grants
|
|
// once. The assign act issues too; both mint only an account no row names, in one statement, so exactly one of them
|
|
// mints and seals it (mintBusUser, the review of issue 1945).
|
|
|
|
// assignmentAccountSteps are what an assignment that comes to stand does, each a step a test may stand in for.
|
|
type assignmentAccountSteps struct {
|
|
// holds says this controller holds the lease: only its holder issues.
|
|
holds func(ctx context.Context) bool
|
|
// catalogue reads the catalogue once, for every assignment a look issues for.
|
|
catalogue func(ctx context.Context) (map[string]catalogue.Manifest, error)
|
|
// issue issues the module's accounts on the node that are not minted, from the catalogue read; issued says it
|
|
// minted one, line what it did.
|
|
issue func(ctx context.Context, shelf map[string]catalogue.Manifest, node, module string) (issued bool, line string, err error)
|
|
// busHolder is the node holding the bus, "" for none.
|
|
busHolder func(ctx context.Context) (string, error)
|
|
// compose writes the nodes' compositions; grants writes the grants issued.
|
|
compose func(ctx context.Context, nodes []string) error
|
|
grants func(ctx context.Context) error
|
|
}
|
|
|
|
// settle issues what the standing assignment of module on node lacks. fresh says the assignment just came to stand: the
|
|
// bus's node is composed and the grants written even when every account was minted before (a module assigned again
|
|
// keeps its account, and the bus's user list left it out while it was unassigned). It answers what it did, "" for
|
|
// nothing.
|
|
func (s assignmentAccountSteps) settle(ctx context.Context, node, module string) (string, error) {
|
|
if module == controllerModule {
|
|
// The controller's own module is issued at the terminal alone: its `broker` secret is the credential it is
|
|
// using (rollout.go), and it is assigned and unassigned there only (refusedMovingTheController).
|
|
return "", nil
|
|
}
|
|
if !s.holds(ctx) {
|
|
return "", errNotHolding
|
|
}
|
|
shelf, err := s.catalogue(ctx)
|
|
if err != nil {
|
|
return "", fmt.Errorf("the catalogue cannot be read: %w", err)
|
|
}
|
|
_, line, err := s.issue(ctx, shelf, node, module)
|
|
if err != nil {
|
|
return "", fmt.Errorf("its bus accounts on %s are not issued: %w", node, err)
|
|
}
|
|
said, err := s.cover(ctx)
|
|
prefix := module + " on " + node + ": "
|
|
if line != "" {
|
|
prefix += line + "; "
|
|
}
|
|
return prefix + said, err
|
|
}
|
|
|
|
// look is the start's look: the accounts every standing assignment lacks, issued one after another from one reading of
|
|
// the catalogue, then the bus's node composed and the grants written once, when it looked at any assignment, whatever
|
|
// it issued (an assignment that came to stand while no controller heard it may have its accounts and still lack its
|
|
// place in the bus's user list or its grant). It answers what it did.
|
|
func (s assignmentAccountSteps) look(ctx context.Context, standing []chain.Assignment) (string, error) {
|
|
var looked []chain.Assignment
|
|
for _, a := range standing {
|
|
if a.Module != controllerModule {
|
|
looked = append(looked, a)
|
|
}
|
|
}
|
|
if len(looked) == 0 {
|
|
return "", nil
|
|
}
|
|
if !s.holds(ctx) {
|
|
return "", errNotHolding
|
|
}
|
|
shelf, err := s.catalogue(ctx)
|
|
if err != nil {
|
|
return "", fmt.Errorf("the catalogue cannot be read: %w", err)
|
|
}
|
|
var lines []string
|
|
for _, a := range looked {
|
|
if ctx.Err() != nil {
|
|
return "", ctx.Err()
|
|
}
|
|
if _, line, err := s.issue(ctx, shelf, a.Node, a.Module); err != nil {
|
|
return "", fmt.Errorf("%s's bus accounts on %s are not issued: %w", a.Module, a.Node, err)
|
|
} else if line != "" {
|
|
lines = append(lines, a.Module+" on "+a.Node+": "+line)
|
|
}
|
|
}
|
|
said, err := s.cover(ctx)
|
|
lines = append(lines, fmt.Sprintf("%d standing assignment(s) looked at; %s", len(looked), said))
|
|
return strings.Join(lines, "\n"), err
|
|
}
|
|
|
|
// errNotHolding is a controller that does not hold the lease: it issues nothing, and tries again until it holds it.
|
|
var errNotHolding = errors.New("this controller does not hold the lease")
|
|
|
|
// cover composes the node holding the bus and writes the grants, and says so.
|
|
func (s assignmentAccountSteps) cover(ctx context.Context) (string, error) {
|
|
holder, err := s.busHolder(ctx)
|
|
if err != nil {
|
|
return "", fmt.Errorf("the node holding the bus cannot be read: %w", err)
|
|
}
|
|
if holder != "" {
|
|
if err := s.compose(ctx, []string{holder}); err != nil {
|
|
return "", fmt.Errorf("the bus's user list on %s is not composed: %w", holder, err)
|
|
}
|
|
}
|
|
if err := s.grants(ctx); err != nil {
|
|
return "", err
|
|
}
|
|
if holder != "" {
|
|
return "the bus's user list composed on " + holder + ", the grants written", nil
|
|
}
|
|
return "the grants written", nil
|
|
}
|
|
|
|
// assignmentStands says a value of the assignments bucket holds an assignment that stands: made (a module only pulled in
|
|
// by another names no time assigned), not ended and in no state (chain.Assignment.Stands).
|
|
func assignmentStands(a chain.Assignment) bool {
|
|
return !a.Assigned.IsZero() && a.Stands() && a.Node != "" && a.Module != ""
|
|
}
|
|
|
|
// assignmentTurns keeps which keys of the assignments bucket stood when last heard, to tell an assignment that comes to
|
|
// stand from a rewrite of one that stood.
|
|
type assignmentTurns struct {
|
|
mu sync.Mutex
|
|
stood map[string]bool
|
|
}
|
|
|
|
// heard records a value of key (nil for a delete or a purge) and answers whether it holds an assignment that just came
|
|
// to stand.
|
|
func (t *assignmentTurns) heard(key string, value []byte) (chain.Assignment, bool) {
|
|
var a chain.Assignment
|
|
stands := false
|
|
if value != nil && json.Unmarshal(value, &a) == nil {
|
|
stands = assignmentStands(a)
|
|
}
|
|
t.mu.Lock()
|
|
defer t.mu.Unlock()
|
|
if t.stood == nil {
|
|
t.stood = map[string]bool{}
|
|
}
|
|
was := t.stood[key]
|
|
t.stood[key] = stands
|
|
return a, stands && !was
|
|
}
|
|
|
|
// ownAccountOf is the username of the account a module speaks on the bus with on node, its own and no process's: the
|
|
// module's when it reads a `broker` own secret, else its provisioner client's when it runs one; false when it has none
|
|
// (a module whose accounts are its processes' alone, or one that does not speak on the bus).
|
|
func ownAccountOf(m catalogue.Manifest, node string) (string, bool) {
|
|
if _, reads := m.OwnSecrets["broker"]; reads {
|
|
return broker.Principal{Kind: broker.KindModule, Node: node, Module: m.Module}.Username(), true
|
|
}
|
|
if runsTheProvisionerClient(m) {
|
|
return broker.Principal{Kind: broker.KindProvisioner, Node: node, Module: m.Module}.Username(), true
|
|
}
|
|
return "", false
|
|
}
|
|
|
|
// issueAssignmentAccounts issues module's accounts on node that are not minted: its own (issueOnTheNewBus) and its
|
|
// processes' (issueMissingProcessAccounts). A module the catalogue does not hold, or one with nowhere to deliver an
|
|
// account (mayIssue), has nothing to issue; an account minted before is kept.
|
|
func issueAssignmentAccounts(ctx context.Context, inv *inventory.Inventory, shelf map[string]catalogue.Manifest,
|
|
node, module string) (bool, string, error) {
|
|
// Mint only what no row names: the assign act hears its own write and may issue at once (the review of issue 1945).
|
|
ctx = onlyIfAbsent(ctx)
|
|
m, known := shelf[module]
|
|
if !known || mayIssue(m) != nil {
|
|
return false, "", nil
|
|
}
|
|
if user, own := ownAccountOf(m, node); own {
|
|
_, minted, err := inv.BusUserHash(ctx, user)
|
|
if err != nil {
|
|
return false, "", fmt.Errorf("whether %s is minted cannot be read: %w", user, err)
|
|
}
|
|
if !minted {
|
|
busAddress, err := broker.BusAddress()
|
|
if err != nil {
|
|
return false, "", err
|
|
}
|
|
// Its processes' accounts too: issueOnTheNewBus issues every one of them with the module's own.
|
|
if err := issueOnTheNewBus(ctx, inv, m, node, busAddress); err != nil {
|
|
return false, "", err
|
|
}
|
|
return true, "its bus credential issued as " + user, nil
|
|
}
|
|
}
|
|
line, err := issueMissingProcessAccounts(ctx, inv, m, node)
|
|
if err != nil {
|
|
return false, "", err
|
|
}
|
|
return line != "", line, nil
|
|
}
|
|
|
|
// liveAssignmentAccountSteps are the steps on the controller's own stores and the bus: the composer's passes, as the
|
|
// done keys' take-ins use them, so a node asked for by both is composed once a pass.
|
|
func liveAssignmentAccountSteps(open *stores) assignmentAccountSteps {
|
|
inv := open.inventory
|
|
return assignmentAccountSteps{
|
|
holds: func(ctx context.Context) bool {
|
|
// The lease's gate, as every act's: a serving controller the bus does not grant the lease yet acts unleased.
|
|
_, err := theLease.epoch(ctx)
|
|
return err == nil
|
|
},
|
|
catalogue: func(ctx context.Context) (map[string]catalogue.Manifest, error) { return inv.Catalogue(ctx) },
|
|
issue: func(ctx context.Context, shelf map[string]catalogue.Manifest, node, module string) (bool, string, error) {
|
|
return issueAssignmentAccounts(ctx, inv, shelf, node, module)
|
|
},
|
|
busHolder: func(ctx context.Context) (string, error) {
|
|
holders, err := seatHolders(ctx, inv)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return holders[catalogue.BrokerSeat].Node, nil
|
|
},
|
|
compose: func(ctx context.Context, nodes []string) error {
|
|
return composerFor(open).compose(ctx, nodes, false)
|
|
},
|
|
grants: func(ctx context.Context) error {
|
|
return composerFor(open).compose(ctx, nil, true)
|
|
},
|
|
}
|
|
}
|
|
|
|
// assignmentAccountsRetry is how long a settle that failed waits before it tries again.
|
|
var assignmentAccountsRetry = 30 * time.Second
|
|
|
|
// untilSettled runs settle until it works or ctx ends: a failure, or a controller not holding the lease, is said and
|
|
// tried again every assignmentAccountsRetry, never dropped, since no later write of the key may come.
|
|
func untilSettled(ctx context.Context, what string, settle func(context.Context) (string, error)) {
|
|
for {
|
|
said, err := settle(ctx)
|
|
if err == nil {
|
|
if said != "" {
|
|
for _, line := range strings.Split(said, "\n") {
|
|
fmt.Printf("assignments: %s\n", line)
|
|
}
|
|
}
|
|
return
|
|
}
|
|
if ctx.Err() != nil {
|
|
return
|
|
}
|
|
fmt.Printf("assignments: %s: %v; tried again in %s\n", what, err, assignmentAccountsRetry)
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-time.After(assignmentAccountsRetry):
|
|
}
|
|
}
|
|
}
|
|
|
|
// watchAssignmentAccounts settles every assignment of the assignments bucket that comes to stand, for as long as ctx: the
|
|
// bucket's current values first (the start's look, settled as not fresh), then each write. A watch that ends, or a
|
|
// bucket not there yet, is watched again a minute later.
|
|
func watchAssignmentAccounts(ctx context.Context, open *stores) {
|
|
steps := liveAssignmentAccountSteps(open)
|
|
goServing(ctx, func() {
|
|
runAssignmentWatches(ctx, open.inventory.WatchAssignments, func(a chain.Assignment) {
|
|
inBackground(ctx, func() {
|
|
untilSettled(ctx, a.Module+" on "+a.Node, func(ctx context.Context) (string, error) {
|
|
return steps.settle(ctx, a.Node, a.Module)
|
|
})
|
|
})
|
|
}, func(look []chain.Assignment) {
|
|
// The start's look, once in the process: one reading of the catalogue, one assignment after another,
|
|
// then one composition of the bus's node and the grants (issues 1945 and 1277).
|
|
inBackground(ctx, func() {
|
|
untilSettled(ctx, "the start's look", func(ctx context.Context) (string, error) {
|
|
return steps.look(ctx, look)
|
|
})
|
|
})
|
|
}, func() bool {
|
|
select {
|
|
case <-ctx.Done():
|
|
return false
|
|
case <-time.After(time.Minute):
|
|
return true
|
|
}
|
|
})
|
|
})
|
|
}
|
|
|
|
// runAssignmentWatches watches the assignments again each time a watch ends, until ctx ends or pause says stop: each
|
|
// assignment that comes to stand is handed to fresh, and the standing ones of the first watch that reads them all (its
|
|
// Caught mark) to look, once. A watch that ends, or fails to open, before the start's look ran is no start's look: the
|
|
// next one is, with what it hears from nothing (a fresh assignmentTurns), so no key it handed is dropped and no burst of
|
|
// fresh settles replaces the look (the review of issue 1945).
|
|
func runAssignmentWatches(ctx context.Context, watch func(context.Context) (<-chan chain.Entry, error),
|
|
fresh func(chain.Assignment), look func([]chain.Assignment), pause func() bool) {
|
|
turns := &assignmentTurns{}
|
|
first := true
|
|
for {
|
|
var atStart []chain.Assignment
|
|
if err := watchAssignmentsOnce(ctx, watch, turns, first, func(a chain.Assignment, isFresh bool) {
|
|
if !isFresh {
|
|
atStart = append(atStart, a)
|
|
return
|
|
}
|
|
fresh(a)
|
|
}, func() {
|
|
first = false
|
|
look(atStart)
|
|
atStart = nil
|
|
}); err != nil && ctx.Err() == nil {
|
|
fmt.Printf("assignments: the assignments are not heard, so a new assignment's bus accounts wait: %v\n", err)
|
|
}
|
|
if first {
|
|
turns = &assignmentTurns{} // the start's look has not run: the next watch hears every standing key anew
|
|
}
|
|
if ctx.Err() != nil || !pause() {
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
// watchAssignmentsOnce watches the assignments until the watch or ctx ends, handing each assignment that comes to stand
|
|
// to settle. On the process's first watch, the current values are the start's look (not fresh) and read is called once
|
|
// they are all handed (the chain's Caught mark); every other value, and every value of a watch opened again after one
|
|
// ended, is fresh: an assignment that came to stand while no watch heard it is settled whole, never as the start's look
|
|
// (the review of issue 1945).
|
|
func watchAssignmentsOnce(ctx context.Context, watch func(context.Context) (<-chan chain.Entry, error),
|
|
turns *assignmentTurns, first bool, settle func(a chain.Assignment, fresh bool), read func()) error {
|
|
ctx, stop := context.WithCancel(ctx)
|
|
defer stop()
|
|
entries, err := watch(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
current := true
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return nil
|
|
case e, ok := <-entries:
|
|
if !ok {
|
|
if ctx.Err() != nil {
|
|
return nil
|
|
}
|
|
return fmt.Errorf("the watch ended")
|
|
}
|
|
if e.Caught {
|
|
if current {
|
|
current = false // the current values are read: what follows is written from now on
|
|
if first {
|
|
read()
|
|
}
|
|
}
|
|
continue
|
|
}
|
|
var value []byte
|
|
if !e.Deleted {
|
|
value = e.Value
|
|
}
|
|
if a, turned := turns.heard(e.Key, value); turned {
|
|
settle(a, !current || !first)
|
|
}
|
|
}
|
|
}
|
|
}
|