novox/mesh-controller
18
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Mint an assignment's account once, whoever finds it missing first (issue 1945, ADR 0543, review)

Browse Source 
The assign act and the assignments' watch both checked whether an account was
minted and then minted it, and minting overwrites the hash: interleaved, the
bus kept one caller's hash and the node was sealed the other's password, the
crash loop this branch fixes. Both now mint through one statement that inserts
the row only when none stands; the caller whose insert wins seals, the other
seals nothing. issueMissingProcessAccounts mints the same way.

A watch opened again after one ended handed the standing assignments as the
start's look, so one that came to stand in the gap got no bus composition or
grant when its account already stood: only the process's first watch is the
start's look now. The start's look reads the catalogue once and ends with one
composition of the bus's node and the grants; a controller not holding the
lease is told so and tries again rather than dropping the key.

Tests folded to stay within the budget: the assignment tests are three table
tests, the process account and mayIssue tests folded into one of them, and
the concurrent mint is a case of the inventory's rotation test.
Research 072: the grants piece (memberships and credentials heard from placement).
This commit is contained in:
jschoubben
2026-10-25 17:51:06 +01:00
parent ca5f74eadb
commit 9ff47f6aaa
8 changed files with 389 additions and 223 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/mesh-controller/assignment_accounts.go
+114 -47
View File
@@ -3,7 +3,9 @@ package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"sync"
"time"
@@ -28,15 +30,20 @@ import (
// (a new key, or one that was ended or pending), the lease holder issues the module's accounts on that node that are not
// minted (idempotent: a minted account is never issued again, a rotation is on purpose), composes the node holding the
// bus so its user list carries them, and writes the grants. A rewrite of an assignment that already stood (the
// provisioner's plan kept in its value) does nothing. A starting controller looks at every standing assignment once:
// it issues what is missing and, when it issued anything, composes the bus's node and writes the grants.
// provisioner's plan kept in its value) does nothing. A starting controller looks at every standing assignment once,
// from one reading of the catalogue: it issues what is missing, then composes the bus's node and writes the grants
// once. The assign act issues too; both mint only an account no row names, in one statement, so exactly one of them
// mints and seals it (mintBusUser, the review of issue 1945).
// assignmentAccountSteps are what an assignment that comes to stand does, each a step a test may stand in for.
type assignmentAccountSteps struct {
// holds says this controller holds the lease: only its holder issues.
holds func(ctx context.Context) bool
// issue issues the module's accounts on the node that are not minted; issued says it minted one, line what it did.
issue func(ctx context.Context, node, module string) (issued bool, line string, err error)
// catalogue reads the catalogue once, for every assignment a look issues for.
catalogue func(ctx context.Context) (map[string]catalogue.Manifest, error)
// issue issues the module's accounts on the node that are not minted, from the catalogue read; issued says it
// minted one, line what it did.
issue func(ctx context.Context, shelf map[string]catalogue.Manifest, node, module string) (issued bool, line string, err error)
// busHolder is the node holding the bus, "" for none.
busHolder func(ctx context.Context) (string, error)
// compose writes the nodes' compositions; grants writes the grants issued.
@@ -48,39 +55,89 @@ type assignmentAccountSteps struct {
// bus's node is composed and the grants written even when every account was minted before (a module assigned again
// keeps its account, and the bus's user list left it out while it was unassigned). It answers what it did, "" for
// nothing.
func (s assignmentAccountSteps) settle(ctx context.Context, node, module string, fresh bool) (string, error) {
if !s.holds(ctx) || module == controllerModule {
func (s assignmentAccountSteps) settle(ctx context.Context, node, module string) (string, error) {
if module == controllerModule {
// The controller's own module is issued at the terminal alone: its `broker` secret is the credential it is
// using (rollout.go), and it is assigned and unassigned there only (refusedMovingTheController).
return "", nil
}
issued, line, err := s.issue(ctx, node, module)
if !s.holds(ctx) {
return "", errNotHolding
}
shelf, err := s.catalogue(ctx)
if err != nil {
return "", fmt.Errorf("the catalogue cannot be read: %w", err)
}
_, line, err := s.issue(ctx, shelf, node, module)
if err != nil {
return "", fmt.Errorf("its bus accounts on %s are not issued: %w", node, err)
}
if !issued && !fresh {
said, err := s.cover(ctx)
prefix := module + " on " + node + ": "
if line != "" {
prefix += line + "; "
}
return prefix + said, err
}
// look is the start's look: the accounts every standing assignment lacks, issued one after another from one reading of
// the catalogue, then the bus's node composed and the grants written once, when it looked at any assignment, whatever
// it issued (an assignment that came to stand while no controller heard it may have its accounts and still lack its
// place in the bus's user list or its grant). It answers what it did.
func (s assignmentAccountSteps) look(ctx context.Context, standing []chain.Assignment) (string, error) {
var looked []chain.Assignment
for _, a := range standing {
if a.Module != controllerModule {
looked = append(looked, a)
}
}
if len(looked) == 0 {
return "", nil
}
if !s.holds(ctx) {
return "", errNotHolding
}
shelf, err := s.catalogue(ctx)
if err != nil {
return "", fmt.Errorf("the catalogue cannot be read: %w", err)
}
var lines []string
for _, a := range looked {
if ctx.Err() != nil {
return "", ctx.Err()
}
if _, line, err := s.issue(ctx, shelf, a.Node, a.Module); err != nil {
return "", fmt.Errorf("%s's bus accounts on %s are not issued: %w", a.Module, a.Node, err)
} else if line != "" {
lines = append(lines, a.Module+" on "+a.Node+": "+line)
}
}
said, err := s.cover(ctx)
lines = append(lines, fmt.Sprintf("%d standing assignment(s) looked at; %s", len(looked), said))
return strings.Join(lines, "\n"), err
}
// errNotHolding is a controller that does not hold the lease: it issues nothing, and tries again until it holds it.
var errNotHolding = errors.New("this controller does not hold the lease")
// cover composes the node holding the bus and writes the grants, and says so.
func (s assignmentAccountSteps) cover(ctx context.Context) (string, error) {
holder, err := s.busHolder(ctx)
if err != nil {
return line, fmt.Errorf("the node holding the bus cannot be read: %w", err)
return "", fmt.Errorf("the node holding the bus cannot be read: %w", err)
}
if holder != "" {
if err := s.compose(ctx, []string{holder}); err != nil {
return line, fmt.Errorf("the bus's user list on %s is not composed: %w", holder, err)
return "", fmt.Errorf("the bus's user list on %s is not composed: %w", holder, err)
}
}
if err := s.grants(ctx); err != nil {
return line, err
}
said := module + " on " + node + ": "
if line != "" {
said += line + "; "
return "", err
}
if holder != "" {
said += "the bus's user list composed on " + holder + ", "
return "the bus's user list composed on " + holder + ", the grants written", nil
}
return said + "its grant written", nil
return "the grants written", nil
}
// assignmentStands says a value of the assignments bucket holds an assignment that stands: made (a module only pulled in
@@ -130,11 +187,10 @@ func ownAccountOf(m catalogue.Manifest, node string) (string, bool) {
// issueAssignmentAccounts issues module's accounts on node that are not minted: its own (issueOnTheNewBus) and its
// processes' (issueMissingProcessAccounts). A module the catalogue does not hold, or one with nowhere to deliver an
// account (mayIssue), has nothing to issue; an account minted before is kept.
func issueAssignmentAccounts(ctx context.Context, inv *inventory.Inventory, node, module string) (bool, string, error) {
shelf, err := inv.Catalogue(ctx)
if err != nil {
return false, "", fmt.Errorf("the catalogue cannot be read: %w", err)
}
func issueAssignmentAccounts(ctx context.Context, inv *inventory.Inventory, shelf map[string]catalogue.Manifest,
node, module string) (bool, string, error) {
// Mint only what no row names: the assign act hears its own write and may issue at once (the review of issue 1945).
ctx = onlyIfAbsent(ctx)
m, known := shelf[module]
if !known || mayIssue(m) != nil {
return false, "", nil
@@ -169,11 +225,13 @@ func liveAssignmentAccountSteps(open *stores) assignmentAccountSteps {
inv := open.inventory
return assignmentAccountSteps{
holds: func(ctx context.Context) bool {
epoch, err := theLease.epoch(ctx)
return err == nil && epoch > 0
// The lease's gate, as every act's: a serving controller the bus does not grant the lease yet acts unleased.
_, err := theLease.epoch(ctx)
return err == nil
},
issue: func(ctx context.Context, node, module string) (bool, string, error) {
return issueAssignmentAccounts(ctx, inv, node, module)
catalogue: func(ctx context.Context) (map[string]catalogue.Manifest, error) { return inv.Catalogue(ctx) },
issue: func(ctx context.Context, shelf map[string]catalogue.Manifest, node, module string) (bool, string, error) {
return issueAssignmentAccounts(ctx, inv, shelf, node, module)
},
busHolder: func(ctx context.Context) (string, error) {
holders, err := seatHolders(ctx, inv)
@@ -194,21 +252,23 @@ func liveAssignmentAccountSteps(open *stores) assignmentAccountSteps {
// assignmentAccountsRetry is how long a settle that failed waits before it tries again.
var assignmentAccountsRetry = 30 * time.Second
// settleAssignment settles one assignment until it works or ctx ends: a failure is said and tried again every
// assignmentAccountsRetry, never dropped, since no later write of the key may come.
func settleAssignment(ctx context.Context, steps assignmentAccountSteps, node, module string, fresh bool) {
// untilSettled runs settle until it works or ctx ends: a failure, or a controller not holding the lease, is said and
// tried again every assignmentAccountsRetry, never dropped, since no later write of the key may come.
func untilSettled(ctx context.Context, what string, settle func(context.Context) (string, error)) {
for {
said, err := steps.settle(ctx, node, module, fresh)
said, err := settle(ctx)
if err == nil {
if said != "" {
fmt.Printf("assignments: %s\n", said)
for _, line := range strings.Split(said, "\n") {
fmt.Printf("assignments: %s\n", line)
}
}
return
}
if ctx.Err() != nil {
return
}
fmt.Printf("assignments: %s on %s: %v; tried again in %s\n", module, node, err, assignmentAccountsRetry)
fmt.Printf("assignments: %s: %v; tried again in %s\n", what, err, assignmentAccountsRetry)
select {
case <-ctx.Done():
return
@@ -224,30 +284,33 @@ func watchAssignmentAccounts(ctx context.Context, open *stores) {
steps := liveAssignmentAccountSteps(open)
turns := &assignmentTurns{}
goServing(ctx, func() {
first := true
for {
var atStart []chain.Assignment
if err := watchAssignmentsOnce(ctx, open.inventory.WatchAssignments, turns, func(a chain.Assignment, fresh bool) {
if err := watchAssignmentsOnce(ctx, open.inventory.WatchAssignments, turns, first, func(a chain.Assignment, fresh bool) {
if !fresh {
atStart = append(atStart, a)
return
}
inBackground(ctx, func() { settleAssignment(ctx, steps, a.Node, a.Module, true) })
inBackground(ctx, func() {
untilSettled(ctx, a.Module+" on "+a.Node, func(ctx context.Context) (string, error) {
return steps.settle(ctx, a.Node, a.Module)
})
})
}, func() {
// The start's look, one assignment after another: a reading of the catalogue each, never all at once
// (issues 1945 and 1277: a burst of take-ins once filled the store).
// The start's look, once in the process: one reading of the catalogue, one assignment after another,
// then one composition of the bus's node and the grants (issues 1945 and 1277).
look := atStart
atStart = nil
inBackground(ctx, func() {
for _, a := range look {
if ctx.Err() != nil {
return
}
settleAssignment(ctx, steps, a.Node, a.Module, false)
}
untilSettled(ctx, "the start's look", func(ctx context.Context) (string, error) {
return steps.look(ctx, look)
})
})
}); err != nil && ctx.Err() == nil {
fmt.Printf("assignments: the assignments are not heard, so a new assignment's bus accounts wait: %v\n", err)
}
first = false
select {
case <-ctx.Done():
return
@@ -258,10 +321,12 @@ func watchAssignmentAccounts(ctx context.Context, open *stores) {
}
// watchAssignmentsOnce watches the assignments until the watch or ctx ends, handing each assignment that comes to stand
// to settle: fresh for one written after the current values were read, not fresh for one of them, and calling read once
// the current values are all handed (the chain's Caught mark).
// to settle. On the process's first watch, the current values are the start's look (not fresh) and read is called once
// they are all handed (the chain's Caught mark); every other value, and every value of a watch opened again after one
// ended, is fresh: an assignment that came to stand while no watch heard it is settled whole, never as the start's look
// (the review of issue 1945).
func watchAssignmentsOnce(ctx context.Context, watch func(context.Context) (<-chan chain.Entry, error),
turns *assignmentTurns, settle func(a chain.Assignment, fresh bool), read func()) error {
turns *assignmentTurns, first bool, settle func(a chain.Assignment, fresh bool), read func()) error {
ctx, stop := context.WithCancel(ctx)
defer stop()
entries, err := watch(ctx)
@@ -283,7 +348,9 @@ func watchAssignmentsOnce(ctx context.Context, watch func(context.Context) (<-ch
if e.Caught {
if current {
current = false // the current values are read: what follows is written from now on
read()
if first {
read()
}
}
continue
}
@@ -292,7 +359,7 @@ func watchAssignmentsOnce(ctx context.Context, watch func(context.Context) (<-ch
value = e.Value
}
if a, turned := turns.heard(e.Key, value); turned {
settle(a, !current)
settle(a, !current || !first)
}
}
}