novox/mesh-host
9
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

ENG-8, ENG-9, ENG-11: a failed pull from a store is tried again once per settled of the store's holder, heard on every node, and the failed have says whom it waits on and what triggered the last retry (design 51, issue 1966; needs mesh-sdk request R1, R2)

Browse Source 
Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig
This commit is contained in:
jschoubben
2026-11-04 11:49:44 +01:00
parent 3eef84d979
commit 2a71a8ff76
8 changed files with 467 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/agent/agent.go
+144 -6
View File
@@ -82,6 +82,9 @@ type Haves interface {
// Result is how one apply or removal came out.
type Result struct {
// Pulls are the images the apply could not find or pull, each with the seat it came through and that seat's holder
// where the declaration names them (a store's image, ENG-10): what a failed apply waits on (ENG-8).
Pulls []PullFailure
Failed bool
// Transient says the failure was the bus not answering for a moment (no responders, a timeout, a reconnect, a
// JetStream API error while the bus reloads), not the module's: the apply is tried again without a new revision.
@@ -89,6 +92,14 @@ type Result struct {
Detail string
}
// PullFailure is one image an apply could not find or pull. Holder is the module holding the seat the image came through,
// "" for an image no store holds (a public registry's): its pull is tried again only by the changes that always start an
// apply. Started is when the pull began.
type PullFailure struct {
Image, Seat, Holder string
Started time.Time
}
// Backup is the answer of the backup asked before a module moves.
// BackupInterrupted is the words of a backup the node agent itself stopped waiting for (its context ended): no failure, so
// the next start backs the module up again rather than holding it for the operator.
@@ -150,6 +161,9 @@ type Env struct {
// client to fill again what it refused (issue 924, the second meeting: ace, g14 and shanks on 2026-10-29 at 00:09
// UTC refused power's three new drop-ins from an engine start on, as no module settled after it). Nil tells nothing.
RunsChanged func(why string)
// Now is the clock the agent compares a pull's start with the time it heard a store's holder settled by; nil is
// time.Now.
Now func() time.Time
// Backoff is how long to wait before the try-th retry (1, 2, …) of an apply that failed transiently. Nil retries a
// transient failure only on RetryTransient or a change of either key.
Backoff func(try int) time.Duration
@@ -232,11 +246,17 @@ type Agent struct {
failedAgain chan string
// tries counts the transient failures in a row of each module; again carries a retry whose wait ran out, and
// transient asks for every transiently failed module at once. stopped is closed when Run returns.
tries map[string]int
gen uint64
again chan againAsk
transient chan struct{}
stopped chan struct{}
tries map[string]int
// pulls is, for each module whose last apply failed on a pull from a store, what it waits on (ENG-8); settledAt and
// settledSeq are, by the module holding a store, when its settled was last heard since this agent started and how many
// have been. Under mu.
pulls map[string]*pullWait
settledAt map[string]time.Time
settledSeq map[string]int
gen uint64
again chan againAsk
transient chan struct{}
stopped chan struct{}
}
type againAsk struct {
@@ -276,6 +296,7 @@ func New(env Env) *Agent {
}
return &Agent{env: env, wants: map[string]wantNow{}, comps: map[string]compNow{}, running: map[string]delivery.Have{}, applied: map[string]compNow{}, had: map[string]compNow{}, keptData: map[string]bool{},
attempt: map[string]attempt{}, checked: map[string]bool{}, prior: map[string]bool{}, said: map[string]string{}, notComposed: map[string]notComposedMark{},
pulls: map[string]*pullWait{}, settledAt: map[string]time.Time{}, settledSeq: map[string]int{},
retry: make(chan []string, 8), reconcile: make(chan []string, 64), failedAgain: make(chan string, 8), tries: map[string]int{}, again: make(chan againAsk, 64),
transient: make(chan struct{}, 1), stopped: make(chan struct{})}
}
@@ -1510,6 +1531,9 @@ func (a *Agent) apply(ctx context.Context, module string, w wantNow, comp compNo
a.writeHave(ctx, Have{Module: module, FP: fp, Composed: dig, Phase: string(delivery.PhaseApplying)})
res := a.env.Apply(ctx, comp.C, others)
a.mu.Lock()
if !res.Failed || res.Transient {
delete(a.pulls, module)
}
what := fmt.Sprintf("%s at %s", module, fp)
if module == NodeModule {
what = "the node's own composition"
@@ -1544,11 +1568,20 @@ func (a *Agent) apply(ctx context.Context, module string, w wantNow, comp compNo
delete(a.tries, module)
if res.Failed {
a.attempt[module] = attempt{FP: fp, Composed: dig, WantRev: w.Rev, CompRev: comp.Rev}
detail := res.Detail
again := a.notePullFailed(module, res.Pulls)
if words := a.pullWords(module); words != "" {
detail += words
}
a.mu.Unlock()
a.env.Say(fmt.Sprintf("applying %s failed: %s", what, res.Detail))
a.writeHave(ctx, Have{Module: module, FP: fp, Composed: dig, Phase: string(delivery.PhaseFailed), Detail: res.Detail})
a.writeHave(ctx, Have{Module: module, FP: fp, Composed: dig, Phase: string(delivery.PhaseFailed), Detail: detail})
if again != "" {
a.askPullAgain(module, again)
}
return
}
delete(a.pulls, module)
delete(a.attempt, module)
delete(a.notComposed, module) // the apply says what it runs now
a.running[module] = delivery.Have{FP: fp, Composed: dig, Phase: delivery.PhaseRunning}
@@ -1584,6 +1617,7 @@ func (a *Agent) remove(ctx context.Context, module string, w wantNow) {
return
}
delete(a.attempt, module)
delete(a.pulls, module)
delete(a.had, module)
a.running[module] = delivery.Have{Phase: delivery.PhaseRemoved}
a.mu.Unlock()
@@ -1794,3 +1828,107 @@ func (a *Agent) readKeptData(ctx context.Context, modules ...string) {
}
}
}
// pullWait is what a module whose apply failed on a pull from a store waits on: the store's holder settling (ENG-8,
// design 51 *distribution's first delivery*).
type pullWait struct {
PullFailure
// retriedFor is the count of the holder's settled the last retry answered: a settled answers once.
retriedFor int
// last is what triggered the last retry, in words, and lastAt when; the have and the journal say it.
last string
lastAt time.Time
}
func (a *Agent) now() time.Time {
if a.env.Now != nil {
return a.env.Now()
}
return time.Now()
}
// notePullFailed records the pull a failed apply met from a store (the first image with a holder), keeping the settled it was
// last tried again for. It answers, when the holder's settled was heard after the pull started and not yet answered, the
// words of the retry to make at once (the race just before `settled`: ENG-11), else "". a.mu is held.
func (a *Agent) notePullFailed(module string, pulls []PullFailure) string {
var found *PullFailure
for i := range pulls {
if pulls[i].Holder != "" {
found = &pulls[i]
break
}
}
if found == nil {
delete(a.pulls, module)
return ""
}
pw := &pullWait{PullFailure: *found}
if before := a.pulls[module]; before != nil {
pw.retriedFor, pw.last, pw.lastAt = before.retriedFor, before.last, before.lastAt
}
a.pulls[module] = pw
holder := found.Holder
if seq := a.settledSeq[holder]; seq > pw.retriedFor && a.settledAt[holder].After(found.Started) {
return a.markPullRetry(pw, seq, fmt.Sprintf("%s settled at %s, after the pull of %s began at %s", holder,
a.settledAt[holder].Format("15:04:05"), found.Image, found.Started.Format("15:04:05")))
}
return ""
}
// markPullRetry notes that pw is tried again for the holder's settled number seq, and answers the words that say so. a.mu
// is held.
func (a *Agent) markPullRetry(pw *pullWait, seq int, why string) string {
pw.retriedFor, pw.last, pw.lastAt = seq, why, a.now()
return why
}
// pullWords is what a failed have says of the wait: whose settling it is tried again on and what triggered the last retry.
// "" for a failure that waits on no holder. a.mu is held.
func (a *Agent) pullWords(module string) string {
pw := a.pulls[module]
if pw == nil {
return ""
}
words := fmt.Sprintf("; it is tried again once %s has settled", pw.Holder)
if pw.last != "" {
words += "; last tried again at " + pw.lastAt.Format("15:04:05") + ": " + pw.last
}
return words
}
// askPullAgain asks the apply of a module again, because why, as a person's verb compose does (RetryFailed): said in the
// journal, never blocking.
func (a *Agent) askPullAgain(module, why string) {
select {
case a.failedAgain <- module:
a.env.Say(fmt.Sprintf("%s: its image could not be pulled from the store; applied again because %s", module, why))
default:
a.env.Say(fmt.Sprintf("%s: its image could not be pulled from the store, and %s, but the engine has too many retries "+
"asked; ask it with compose", module, why))
}
}
// Settled is the status view's word that a module has settled on some node (ENG-11): the engine's hearing tells it for every
// module it hears. When the module holds a store a failed apply of this node waits on, that apply is tried again once for
// this settled; a settled of any other module starts nothing. A pull that began after this settled, or has failed before it
// and is answered, is not tried again for it (notePullFailed).
func (a *Agent) Settled(holder string) {
a.mu.Lock()
a.settledSeq[holder]++
a.settledAt[holder] = a.now()
seq := a.settledSeq[holder]
var asks []string
var modules []string
for module, pw := range a.pulls {
att, failed := a.attempt[module]
if pw.Holder != holder || !failed || att.Held || att.Transient || pw.retriedFor >= seq {
continue
}
modules = append(modules, module)
asks = append(asks, a.markPullRetry(pw, seq, fmt.Sprintf("%s settled", holder)))
}
a.mu.Unlock()
for i, module := range modules {
a.askPullAgain(module, asks[i])
}
}