novox/mesh-host
18
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

ENG-8, ENG-9, ENG-11: a failed pull from a store is tried again once per settled of the store's holder, heard on every node, and the failed have says whom it waits on and what triggered the last retry (design 51, issue 1966; needs mesh-sdk request R1, R2)

Browse Source 
Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig
This commit is contained in:
jschoubben
2026-11-04 11:49:44 +01:00
parent 3eef84d979
commit 2a71a8ff76
8 changed files with 467 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/agent/agent.go
+144 -6
View File
@@ -82,6 +82,9 @@ type Haves interface {
// Result is how one apply or removal came out.
type Result struct {
// Pulls are the images the apply could not find or pull, each with the seat it came through and that seat's holder
// where the declaration names them (a store's image, ENG-10): what a failed apply waits on (ENG-8).
Pulls []PullFailure
Failed bool
// Transient says the failure was the bus not answering for a moment (no responders, a timeout, a reconnect, a
// JetStream API error while the bus reloads), not the module's: the apply is tried again without a new revision.
@@ -89,6 +92,14 @@ type Result struct {
Detail string
}
// PullFailure is one image an apply could not find or pull. Holder is the module holding the seat the image came through,
// "" for an image no store holds (a public registry's): its pull is tried again only by the changes that always start an
// apply. Started is when the pull began.
type PullFailure struct {
Image, Seat, Holder string
Started time.Time
}
// Backup is the answer of the backup asked before a module moves.
// BackupInterrupted is the words of a backup the node agent itself stopped waiting for (its context ended): no failure, so
// the next start backs the module up again rather than holding it for the operator.
@@ -150,6 +161,9 @@ type Env struct {
// client to fill again what it refused (issue 924, the second meeting: ace, g14 and shanks on 2026-10-29 at 00:09
// UTC refused power's three new drop-ins from an engine start on, as no module settled after it). Nil tells nothing.
RunsChanged func(why string)
// Now is the clock the agent compares a pull's start with the time it heard a store's holder settled by; nil is
// time.Now.
Now func() time.Time
// Backoff is how long to wait before the try-th retry (1, 2, …) of an apply that failed transiently. Nil retries a
// transient failure only on RetryTransient or a change of either key.
Backoff func(try int) time.Duration
@@ -232,11 +246,17 @@ type Agent struct {
failedAgain chan string
// tries counts the transient failures in a row of each module; again carries a retry whose wait ran out, and
// transient asks for every transiently failed module at once. stopped is closed when Run returns.
tries map[string]int
gen uint64
again chan againAsk
transient chan struct{}
stopped chan struct{}
tries map[string]int
// pulls is, for each module whose last apply failed on a pull from a store, what it waits on (ENG-8); settledAt and
// settledSeq are, by the module holding a store, when its settled was last heard since this agent started and how many
// have been. Under mu.
pulls map[string]*pullWait
settledAt map[string]time.Time
settledSeq map[string]int
gen uint64
again chan againAsk
transient chan struct{}
stopped chan struct{}
}
type againAsk struct {
@@ -276,6 +296,7 @@ func New(env Env) *Agent {
}
return &Agent{env: env, wants: map[string]wantNow{}, comps: map[string]compNow{}, running: map[string]delivery.Have{}, applied: map[string]compNow{}, had: map[string]compNow{}, keptData: map[string]bool{},
attempt: map[string]attempt{}, checked: map[string]bool{}, prior: map[string]bool{}, said: map[string]string{}, notComposed: map[string]notComposedMark{},
pulls: map[string]*pullWait{}, settledAt: map[string]time.Time{}, settledSeq: map[string]int{},
retry: make(chan []string, 8), reconcile: make(chan []string, 64), failedAgain: make(chan string, 8), tries: map[string]int{}, again: make(chan againAsk, 64),
transient: make(chan struct{}, 1), stopped: make(chan struct{})}
}
@@ -1510,6 +1531,9 @@ func (a *Agent) apply(ctx context.Context, module string, w wantNow, comp compNo
a.writeHave(ctx, Have{Module: module, FP: fp, Composed: dig, Phase: string(delivery.PhaseApplying)})
res := a.env.Apply(ctx, comp.C, others)
a.mu.Lock()
if !res.Failed || res.Transient {
delete(a.pulls, module)
}
what := fmt.Sprintf("%s at %s", module, fp)
if module == NodeModule {
what = "the node's own composition"
@@ -1544,11 +1568,20 @@ func (a *Agent) apply(ctx context.Context, module string, w wantNow, comp compNo
delete(a.tries, module)
if res.Failed {
a.attempt[module] = attempt{FP: fp, Composed: dig, WantRev: w.Rev, CompRev: comp.Rev}
detail := res.Detail
again := a.notePullFailed(module, res.Pulls)
if words := a.pullWords(module); words != "" {
detail += words
}
a.mu.Unlock()
a.env.Say(fmt.Sprintf("applying %s failed: %s", what, res.Detail))
a.writeHave(ctx, Have{Module: module, FP: fp, Composed: dig, Phase: string(delivery.PhaseFailed), Detail: res.Detail})
a.writeHave(ctx, Have{Module: module, FP: fp, Composed: dig, Phase: string(delivery.PhaseFailed), Detail: detail})
if again != "" {
a.askPullAgain(module, again)
}
return
}
delete(a.pulls, module)
delete(a.attempt, module)
delete(a.notComposed, module) // the apply says what it runs now
a.running[module] = delivery.Have{FP: fp, Composed: dig, Phase: delivery.PhaseRunning}
@@ -1584,6 +1617,7 @@ func (a *Agent) remove(ctx context.Context, module string, w wantNow) {
return
}
delete(a.attempt, module)
delete(a.pulls, module)
delete(a.had, module)
a.running[module] = delivery.Have{Phase: delivery.PhaseRemoved}
a.mu.Unlock()
@@ -1794,3 +1828,107 @@ func (a *Agent) readKeptData(ctx context.Context, modules ...string) {
}
}
}
// pullWait is what a module whose apply failed on a pull from a store waits on: the store's holder settling (ENG-8,
// design 51 *distribution's first delivery*).
type pullWait struct {
PullFailure
// retriedFor is the count of the holder's settled the last retry answered: a settled answers once.
retriedFor int
// last is what triggered the last retry, in words, and lastAt when; the have and the journal say it.
last string
lastAt time.Time
}
func (a *Agent) now() time.Time {
if a.env.Now != nil {
return a.env.Now()
}
return time.Now()
}
// notePullFailed records the pull a failed apply met from a store (the first image with a holder), keeping the settled it was
// last tried again for. It answers, when the holder's settled was heard after the pull started and not yet answered, the
// words of the retry to make at once (the race just before `settled`: ENG-11), else "". a.mu is held.
func (a *Agent) notePullFailed(module string, pulls []PullFailure) string {
var found *PullFailure
for i := range pulls {
if pulls[i].Holder != "" {
found = &pulls[i]
break
}
}
if found == nil {
delete(a.pulls, module)
return ""
}
pw := &pullWait{PullFailure: *found}
if before := a.pulls[module]; before != nil {
pw.retriedFor, pw.last, pw.lastAt = before.retriedFor, before.last, before.lastAt
}
a.pulls[module] = pw
holder := found.Holder
if seq := a.settledSeq[holder]; seq > pw.retriedFor && a.settledAt[holder].After(found.Started) {
return a.markPullRetry(pw, seq, fmt.Sprintf("%s settled at %s, after the pull of %s began at %s", holder,
a.settledAt[holder].Format("15:04:05"), found.Image, found.Started.Format("15:04:05")))
}
return ""
}
// markPullRetry notes that pw is tried again for the holder's settled number seq, and answers the words that say so. a.mu
// is held.
func (a *Agent) markPullRetry(pw *pullWait, seq int, why string) string {
pw.retriedFor, pw.last, pw.lastAt = seq, why, a.now()
return why
}
// pullWords is what a failed have says of the wait: whose settling it is tried again on and what triggered the last retry.
// "" for a failure that waits on no holder. a.mu is held.
func (a *Agent) pullWords(module string) string {
pw := a.pulls[module]
if pw == nil {
return ""
}
words := fmt.Sprintf("; it is tried again once %s has settled", pw.Holder)
if pw.last != "" {
words += "; last tried again at " + pw.lastAt.Format("15:04:05") + ": " + pw.last
}
return words
}
// askPullAgain asks the apply of a module again, because why, as a person's verb compose does (RetryFailed): said in the
// journal, never blocking.
func (a *Agent) askPullAgain(module, why string) {
select {
case a.failedAgain <- module:
a.env.Say(fmt.Sprintf("%s: its image could not be pulled from the store; applied again because %s", module, why))
default:
a.env.Say(fmt.Sprintf("%s: its image could not be pulled from the store, and %s, but the engine has too many retries "+
"asked; ask it with compose", module, why))
}
}
// Settled is the status view's word that a module has settled on some node (ENG-11): the engine's hearing tells it for every
// module it hears. When the module holds a store a failed apply of this node waits on, that apply is tried again once for
// this settled; a settled of any other module starts nothing. A pull that began after this settled, or has failed before it
// and is answered, is not tried again for it (notePullFailed).
func (a *Agent) Settled(holder string) {
a.mu.Lock()
a.settledSeq[holder]++
a.settledAt[holder] = a.now()
seq := a.settledSeq[holder]
var asks []string
var modules []string
for module, pw := range a.pulls {
att, failed := a.attempt[module]
if pw.Holder != holder || !failed || att.Held || att.Transient || pw.retriedFor >= seq {
continue
}
modules = append(modules, module)
asks = append(asks, a.markPullRetry(pw, seq, fmt.Sprintf("%s settled", holder)))
}
a.mu.Unlock()
for i, module := range modules {
a.askPullAgain(module, asks[i])
}
}
internal/agent/agent_test.go
+117 -5
View File
@@ -8,6 +8,7 @@ import (
"slices"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
@@ -115,10 +116,14 @@ type bed struct {
// transient marks failWith as the bus not answering; failTimes > 0 fails only the first that many applies.
transient bool
failTimes int
backup func(module string) Backup
kept map[string][]json.RawMessage
others [][]delivery.Composed
said []string
// pullsFn, when set, is the images a failed apply could not pull (called as the apply fails); during runs at the start of
// each apply, before it answers.
pullsFn func() []PullFailure
during func()
backup func(module string) Backup
kept map[string][]json.RawMessage
others [][]delivery.Composed
said []string
// settled is each module Settled was told, and settledRan whether what runs held it then.
settled []string
settledRan []bool
@@ -130,6 +135,9 @@ func newBed(t *testing.T) *bed {
b := &bed{t: t, wants: newFeed(), comps: newFeed(), haves: &haves{}}
b.env = Env{Node: node, Wants: b.wants, Composed: b.comps, Haves: b.haves,
Apply: func(_ context.Context, c delivery.Composed, others []delivery.Composed) Result {
if b.during != nil {
b.during()
}
b.mu.Lock()
defer b.mu.Unlock()
b.others = append(b.others, others)
@@ -139,7 +147,11 @@ func newBed(t *testing.T) *bed {
}
b.applied = append(b.applied, name)
if b.failWith != "" && (b.failTimes == 0 || len(b.applied) <= b.failTimes) {
return Result{Failed: true, Transient: b.transient, Detail: b.failWith}
res := Result{Failed: true, Transient: b.transient, Detail: b.failWith}
if b.pullsFn != nil {
res.Pulls = b.pullsFn()
}
return res
}
return Result{}
},
@@ -461,6 +473,106 @@ func TestAFailedApplyIsRetriedOnlyByAChangeOfEitherKey(t *testing.T) {
t.Errorf("a module running its want was applied again (%d applies)", n)
}
})
// design 51 (ENG-8, ENG-11): a pull from a store that failed is tried again when the store's holder settles, once per
// settled, never on a clock; the have says whom it waits on and what triggered the last retry.
t.Run("a pull from a store waits for its holder to settle", func(t *testing.T) {
b := newBed(t)
var clock atomic.Int64
now := func() time.Time { return time.Unix(1_000_000, clock.Load()) }
tick := func() time.Time { clock.Add(int64(time.Second)); return now() }
b.env.Now = now
b.env.Backoff = func(int) time.Duration { return time.Millisecond }
b.failWith = "pulling image reg/x@sha256:1: manifest unknown"
b.pullsFn = func() []PullFailure {
return []PullFailure{{Image: "reg/x@sha256:1", Seat: "mesh-artifact-store", Holder: "distribution", Started: tick()},
{Image: "public/y@sha256:2"}}
}
stop := b.run()
defer stop()
fp := fpOf("a")
ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1"))
wk, wraw := wantFor(t, "web", fp, dig, "")
b.comps.put(ck, craw)
b.wants.put(wk, wraw)
eventually(t, "not applied", func() bool { return len(b.appliedList()) == 1 })
eventually(t, "the have does not say whom it waits on", func() bool {
h := b.haves.last("web")
return h.Phase == string(delivery.PhaseFailed) && strings.Contains(h.Detail, "tried again once distribution has settled")
})
time.Sleep(100 * time.Millisecond)
if n := len(b.appliedList()); n != 1 {
t.Fatalf("a failed pull was tried again by the clock (%d applies)", n)
}
// A settled of another module starts nothing.
b.ag.Settled("gitea")
time.Sleep(50 * time.Millisecond)
if n := len(b.appliedList()); n != 1 {
t.Fatalf("a settled of another module applied the module (%d applies)", n)
}
// The holder's settled: tried again once, whose pull fails again after it (started later): failed again, no more.
tick()
b.ag.Settled("distribution")
eventually(t, "not tried again on the holder's settled", func() bool { return len(b.appliedList()) == 2 })
eventually(t, "the have does not say what triggered the retry", func() bool {
h := b.haves.last("web")
return h.Phase == string(delivery.PhaseFailed) && strings.Contains(h.Detail, "last tried again at") && strings.Contains(h.Detail, "distribution settled")
})
time.Sleep(100 * time.Millisecond)
if n := len(b.appliedList()); n != 2 {
t.Fatalf("a second failure after the same settled started a further apply (%d applies)", n)
}
// The next settled answers once more, and the pull works: the module runs, and waits on nothing.
b.mu.Lock()
b.failWith = ""
b.mu.Unlock()
tick()
b.ag.Settled("distribution")
eventually(t, "not tried again on the next settled", func() bool { return len(b.appliedList()) == 3 })
eventually(t, "not running", func() bool { return b.haves.last("web").Phase == string(delivery.PhaseRunning) })
b.ag.Settled("distribution")
time.Sleep(50 * time.Millisecond)
if n := len(b.appliedList()); n != 3 {
t.Fatalf("a running module was applied again by a settled (%d applies)", n)
}
})
// The race just before `settled`: the holder's settled is heard while the pull is in flight, and the pull fails after it;
// applied again at once, once, and a settled heard before the pull began applies nothing at once.
t.Run("a settled heard while the pull ran is not missed", func(t *testing.T) {
b := newBed(t)
var clock atomic.Int64
now := func() time.Time { return time.Unix(1_000_000, clock.Load()) }
tick := func() time.Time { clock.Add(int64(time.Second)); return now() }
b.env.Now = now
b.failWith = "pulling image reg/x@sha256:1: connection refused"
began := tick()
first := true
b.during = func() {
if first {
first = false
tick()
b.ag.Settled("distribution") // the holder settles while this apply's pull is in flight
}
}
b.pullsFn = func() []PullFailure {
started := began
if len(b.applied) > 1 {
started = tick() // the retry's pull begins after the settled
}
return []PullFailure{{Image: "reg/x@sha256:1", Seat: "mesh-artifact-store", Holder: "distribution", Started: started}}
}
stop := b.run()
defer stop()
fp := fpOf("a")
ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1"))
wk, wraw := wantFor(t, "web", fp, dig, "")
b.comps.put(ck, craw)
b.wants.put(wk, wraw)
eventually(t, "the settled heard during the pull was missed", func() bool { return len(b.appliedList()) == 2 })
time.Sleep(100 * time.Millisecond)
if n := len(b.appliedList()); n != 2 {
t.Fatalf("the race retry failing again started a further apply (%d applies)", n)
}
})
// The bus did not answer for a moment (mesh-vault on the bus node, 2026-10-25 23:04:24): tried again with no new
// revision of either key, by the backoff or by RetryTransient once the engine knows the bus is back; the have says
// retrying meanwhile, never failed.