novox/mesh-host
18
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

ENG-9: a failed have carries the pull it waits on, read through node_haves

Browse Source 
The image, the seat and its holder, when the pull began, and the last retry with what triggered it ride in the failed
have's pull-wait (mesh-sdk delivery.Have.PullWait, asked of the lead in j1-sdk-requests.patch), from the agent through
the link's module fact to the have bucket; the agent's own read view of them goes with the verb. Rows in the agent's
pull test and the have bucket's test; each failed without the code.

Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig
This commit is contained in:
jschoubben
2026-11-04 14:02:44 +01:00
parent 2b14f166de
commit a2313057c0
6 changed files with 35 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/mesh-host/nodeagent.go
+1 -1
View File
@@ -120,7 +120,7 @@ func (h sessionHaves) Write(ctx context.Context, x agent.Have) {
}
h.session.NoteModule(ctx, link.ModuleFact{Module: x.Module, FP: x.FP, Composed: x.Composed, Phase: phase,
Failed: phase == string(delivery.PhaseFailed), Detail: shorten(x.Detail), Judgement: x.Judgement, Waits: x.Waits,
BuildAlone: x.BuildAlone, Kept: x.Kept, Sealed: x.Sealed})
BuildAlone: x.BuildAlone, Kept: x.Kept, Sealed: x.Sealed, PullWait: x.PullWait})
}
func (h sessionHaves) Removed(ctx context.Context, module string) {
internal/agent/agent.go
+14 -26
View File
@@ -68,6 +68,9 @@ type Have struct {
// slot (delivery.Have.SealedAt): kept in the have so the composer reuses them after a restart (ADR 0547 §3), each at
// its own slot alone (issue 1126, C1).
Sealed delivery.Seals
// PullWait is, on a failed have, the pull from a store its apply failed on and waits on the holder of (issue 1966
// ENG-9, delivery.Have.PullWait): read through the have, as node_haves shows it.
PullWait *delivery.PullWait
}
// Haves is the node's have bucket as the agent uses it.
@@ -1592,9 +1595,11 @@ func (a *Agent) apply(ctx context.Context, module string, w wantNow, comp compNo
if words := a.pullWords(module); words != "" {
detail += words
}
wait := a.pullWaitOf(module)
a.mu.Unlock()
a.env.Say(fmt.Sprintf("applying %s failed: %s", what, res.Detail))
a.writeHave(ctx, Have{Module: module, FP: fp, Composed: dig, Phase: string(delivery.PhaseFailed), Detail: detail})
a.writeHave(ctx, Have{Module: module, FP: fp, Composed: dig, Phase: string(delivery.PhaseFailed), Detail: detail,
PullWait: wait})
if again != "" {
a.askPullAgain(module, again)
}
@@ -1952,30 +1957,13 @@ func (a *Agent) Settled(holder string) {
}
}
// PullWaitView is one failed apply that waits on the holder of a store, as the engine's read verb pull-waits shows it
// (ENG-9, design 51): the module, the image and the seat it came through, whose settling it waits on, when the pull began,
// how many times that holder has settled since this engine started and when last, and what triggered the last retry.
type PullWaitView struct {
Module string `json:"module"`
Image string `json:"image"`
Seat string `json:"seat,omitempty"`
Holder string `json:"holder"`
Started time.Time `json:"pull-started"`
Settled int `json:"holder-settled"`
SettledAt time.Time `json:"holder-last-settled,omitzero"`
LastRetry string `json:"last-retry,omitempty"`
LastRetryAt time.Time `json:"last-retry-at,omitzero"`
}
// PullWaits are the failed applies that wait on a store's holder, by module.
func (a *Agent) PullWaits() []PullWaitView {
a.mu.Lock()
defer a.mu.Unlock()
out := []PullWaitView{}
for module, pw := range a.pulls {
out = append(out, PullWaitView{Module: module, Image: pw.Image, Seat: pw.Seat, Holder: pw.Holder, Started: pw.Started,
Settled: a.settledSeq[pw.Holder], SettledAt: a.settledAt[pw.Holder], LastRetry: pw.last, LastRetryAt: pw.lastAt})
// pullWaitOf is the pull a failed apply of module waits on, as its have carries it (ENG-9): the image, the seat and its
// holder, when the pull began, and the last retry with what triggered it; nil when it waits on no holder. a.mu is held.
func (a *Agent) pullWaitOf(module string) *delivery.PullWait {
pw := a.pulls[module]
if pw == nil {
return nil
}
sort.Slice(out, func(i, j int) bool { return out[i].Module < out[j].Module })
return out
return &delivery.PullWait{Image: pw.Image, Seat: pw.Seat, Holder: pw.Holder, Began: pw.Started, Retried: pw.last,
RetriedAt: pw.lastAt}
}
internal/agent/agent_test.go
+7 -6
View File
@@ -521,10 +521,11 @@ func TestAFailedApplyIsRetriedOnlyByAChangeOfEitherKey(t *testing.T) {
if n := len(b.appliedList()); n != 2 {
t.Fatalf("a second failure after the same settled started a further apply (%d applies)", n)
}
// The read verb's view: whom it waits on, how often that holder settled, and what triggered the last retry.
if w := b.ag.PullWaits(); len(w) != 1 || w[0].Module != "web" || w[0].Holder != "distribution" || w[0].Image != "reg/x@sha256:1" ||
w[0].Settled != 1 || !strings.Contains(w[0].LastRetry, "distribution settled") {
t.Fatalf("the engine says it waits on %+v", w)
// The failed have carries the pull it waits on (ENG-9, read through node_haves): the image, whose settled it waits
// on, and what triggered the last retry.
if w := b.haves.last("web").PullWait; w == nil || w.Holder != "distribution" || w.Image != "reg/x@sha256:1" ||
w.Began.IsZero() || !strings.Contains(w.Retried, "distribution settled") || w.RetriedAt.IsZero() {
t.Fatalf("the failed have says it waits on %+v", w)
}
// The next settled answers once more, and the pull works: the module runs, and waits on nothing.
b.mu.Lock()
@@ -534,8 +535,8 @@ func TestAFailedApplyIsRetriedOnlyByAChangeOfEitherKey(t *testing.T) {
b.ag.Settled("distribution")
eventually(t, "not tried again on the next settled", func() bool { return len(b.appliedList()) == 3 })
eventually(t, "not running", func() bool { return b.haves.last("web").Phase == string(delivery.PhaseRunning) })
if w := b.ag.PullWaits(); len(w) != 0 {
t.Fatalf("a running module still waits on %+v", w)
if w := b.haves.last("web").PullWait; w != nil {
t.Fatalf("a running module's have still waits on %+v", w)
}
b.ag.Settled("distribution")
time.Sleep(50 * time.Millisecond)
internal/link/have.go
+3
View File
@@ -387,6 +387,9 @@ func (q *Session) writeHaves(ctx context.Context, bus Bus) {
h.Phase = delivery.PhaseFailed
}
h.SealedAt = f.Sealed
if h.Phase == delivery.PhaseFailed {
h.PullWait = f.PullWait
}
h.Health = s.healthOf(module, f)
if h.Detail == "" {
h.Detail = s.wordsOf(module, h.Health)
internal/link/have_test.go
+8
View File
@@ -581,6 +581,14 @@ func TestAnAppliedModulesHaveNamesTheFingerprintAndTheDigestAndFollowsItsHealth(
t.Fatalf("an apply lost the user list: %+v", got)
}
// A failed apply waiting on a store's holder carries the pull it waits on in its have (ENG-9, read by node_haves).
wait := &delivery.PullWait{Image: "reg/x@sha256:1", Seat: "container-image-registry", Holder: "distribution",
Began: time.Date(2026, 11, 4, 14, 0, 0, 0, time.UTC), Retried: "distribution settled"}
q.NoteModule(context.Background(), ModuleFact{Module: "web", FP: fp, Composed: dig, Failed: true, Detail: "the pull failed", PullWait: wait})
if got := l.have(t, "n1.web"); got.Phase != delivery.PhaseFailed || got.PullWait == nil || got.PullWait.Holder != "distribution" {
t.Fatalf("the failed have does not say the pull it waits on: %+v", got)
}
// A module held for a failed backup says so and what it keeps running; removed is written once, over its key.
q.NoteModule(context.Background(), ModuleFact{Module: "web", FP: fp, Composed: dig, Phase: string(delivery.PhaseHeld), Detail: "the repository is locked"})
if got := l.have(t, "n1.web"); got.Phase != delivery.PhaseHeld || got.Detail == "" {
internal/link/messages.go
+2
View File
@@ -645,4 +645,6 @@ type ModuleFact struct {
Kept []string
// Sealed is the values the composition it names carries sealed to this node's key, by slot (delivery.Have.SealedAt).
Sealed delivery.Seals
// PullWait is, on a failed fact, the pull from a store its apply waits on (delivery.Have.PullWait, issue 1966 ENG-9).
PullWait *delivery.PullWait
}