diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index f0e67b7..57dfbde 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -120,7 +120,7 @@ func (h sessionHaves) Write(ctx context.Context, x agent.Have) { } h.session.NoteModule(ctx, link.ModuleFact{Module: x.Module, FP: x.FP, Composed: x.Composed, Phase: phase, Failed: phase == string(delivery.PhaseFailed), Detail: shorten(x.Detail), Judgement: x.Judgement, Waits: x.Waits, - BuildAlone: x.BuildAlone, Kept: x.Kept, Sealed: x.Sealed}) + BuildAlone: x.BuildAlone, Kept: x.Kept, Sealed: x.Sealed, PullWait: x.PullWait}) } func (h sessionHaves) Removed(ctx context.Context, module string) { diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 01a69b4..f42b747 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -68,6 +68,9 @@ type Have struct { // slot (delivery.Have.SealedAt): kept in the have so the composer reuses them after a restart (ADR 0547 ยง3), each at // its own slot alone (issue 1126, C1). Sealed delivery.Seals + // PullWait is, on a failed have, the pull from a store its apply failed on and waits on the holder of (issue 1966 + // ENG-9, delivery.Have.PullWait): read through the have, as node_haves shows it. + PullWait *delivery.PullWait } // Haves is the node's have bucket as the agent uses it. @@ -1592,9 +1595,11 @@ func (a *Agent) apply(ctx context.Context, module string, w wantNow, comp compNo if words := a.pullWords(module); words != "" { detail += words } + wait := a.pullWaitOf(module) a.mu.Unlock() a.env.Say(fmt.Sprintf("applying %s failed: %s", what, res.Detail)) - a.writeHave(ctx, Have{Module: module, FP: fp, Composed: dig, Phase: string(delivery.PhaseFailed), Detail: detail}) + a.writeHave(ctx, Have{Module: module, FP: fp, Composed: dig, Phase: string(delivery.PhaseFailed), Detail: detail, + PullWait: wait}) if again != "" { a.askPullAgain(module, again) } @@ -1952,30 +1957,13 @@ func (a *Agent) Settled(holder string) { } } -// PullWaitView is one failed apply that waits on the holder of a store, as the engine's read verb pull-waits shows it -// (ENG-9, design 51): the module, the image and the seat it came through, whose settling it waits on, when the pull began, -// how many times that holder has settled since this engine started and when last, and what triggered the last retry. -type PullWaitView struct { - Module string `json:"module"` - Image string `json:"image"` - Seat string `json:"seat,omitempty"` - Holder string `json:"holder"` - Started time.Time `json:"pull-started"` - Settled int `json:"holder-settled"` - SettledAt time.Time `json:"holder-last-settled,omitzero"` - LastRetry string `json:"last-retry,omitempty"` - LastRetryAt time.Time `json:"last-retry-at,omitzero"` -} - -// PullWaits are the failed applies that wait on a store's holder, by module. -func (a *Agent) PullWaits() []PullWaitView { - a.mu.Lock() - defer a.mu.Unlock() - out := []PullWaitView{} - for module, pw := range a.pulls { - out = append(out, PullWaitView{Module: module, Image: pw.Image, Seat: pw.Seat, Holder: pw.Holder, Started: pw.Started, - Settled: a.settledSeq[pw.Holder], SettledAt: a.settledAt[pw.Holder], LastRetry: pw.last, LastRetryAt: pw.lastAt}) +// pullWaitOf is the pull a failed apply of module waits on, as its have carries it (ENG-9): the image, the seat and its +// holder, when the pull began, and the last retry with what triggered it; nil when it waits on no holder. a.mu is held. +func (a *Agent) pullWaitOf(module string) *delivery.PullWait { + pw := a.pulls[module] + if pw == nil { + return nil } - sort.Slice(out, func(i, j int) bool { return out[i].Module < out[j].Module }) - return out + return &delivery.PullWait{Image: pw.Image, Seat: pw.Seat, Holder: pw.Holder, Began: pw.Started, Retried: pw.last, + RetriedAt: pw.lastAt} } diff --git a/internal/agent/agent_test.go b/internal/agent/agent_test.go index e6ee848..646856b 100644 --- a/internal/agent/agent_test.go +++ b/internal/agent/agent_test.go @@ -521,10 +521,11 @@ func TestAFailedApplyIsRetriedOnlyByAChangeOfEitherKey(t *testing.T) { if n := len(b.appliedList()); n != 2 { t.Fatalf("a second failure after the same settled started a further apply (%d applies)", n) } - // The read verb's view: whom it waits on, how often that holder settled, and what triggered the last retry. - if w := b.ag.PullWaits(); len(w) != 1 || w[0].Module != "web" || w[0].Holder != "distribution" || w[0].Image != "reg/x@sha256:1" || - w[0].Settled != 1 || !strings.Contains(w[0].LastRetry, "distribution settled") { - t.Fatalf("the engine says it waits on %+v", w) + // The failed have carries the pull it waits on (ENG-9, read through node_haves): the image, whose settled it waits + // on, and what triggered the last retry. + if w := b.haves.last("web").PullWait; w == nil || w.Holder != "distribution" || w.Image != "reg/x@sha256:1" || + w.Began.IsZero() || !strings.Contains(w.Retried, "distribution settled") || w.RetriedAt.IsZero() { + t.Fatalf("the failed have says it waits on %+v", w) } // The next settled answers once more, and the pull works: the module runs, and waits on nothing. b.mu.Lock() @@ -534,8 +535,8 @@ func TestAFailedApplyIsRetriedOnlyByAChangeOfEitherKey(t *testing.T) { b.ag.Settled("distribution") eventually(t, "not tried again on the next settled", func() bool { return len(b.appliedList()) == 3 }) eventually(t, "not running", func() bool { return b.haves.last("web").Phase == string(delivery.PhaseRunning) }) - if w := b.ag.PullWaits(); len(w) != 0 { - t.Fatalf("a running module still waits on %+v", w) + if w := b.haves.last("web").PullWait; w != nil { + t.Fatalf("a running module's have still waits on %+v", w) } b.ag.Settled("distribution") time.Sleep(50 * time.Millisecond) diff --git a/internal/link/have.go b/internal/link/have.go index 7f1a2ed..23757db 100644 --- a/internal/link/have.go +++ b/internal/link/have.go @@ -387,6 +387,9 @@ func (q *Session) writeHaves(ctx context.Context, bus Bus) { h.Phase = delivery.PhaseFailed } h.SealedAt = f.Sealed + if h.Phase == delivery.PhaseFailed { + h.PullWait = f.PullWait + } h.Health = s.healthOf(module, f) if h.Detail == "" { h.Detail = s.wordsOf(module, h.Health) diff --git a/internal/link/have_test.go b/internal/link/have_test.go index f0825b2..93916f9 100644 --- a/internal/link/have_test.go +++ b/internal/link/have_test.go @@ -581,6 +581,14 @@ func TestAnAppliedModulesHaveNamesTheFingerprintAndTheDigestAndFollowsItsHealth( t.Fatalf("an apply lost the user list: %+v", got) } + // A failed apply waiting on a store's holder carries the pull it waits on in its have (ENG-9, read by node_haves). + wait := &delivery.PullWait{Image: "reg/x@sha256:1", Seat: "container-image-registry", Holder: "distribution", + Began: time.Date(2026, 11, 4, 14, 0, 0, 0, time.UTC), Retried: "distribution settled"} + q.NoteModule(context.Background(), ModuleFact{Module: "web", FP: fp, Composed: dig, Failed: true, Detail: "the pull failed", PullWait: wait}) + if got := l.have(t, "n1.web"); got.Phase != delivery.PhaseFailed || got.PullWait == nil || got.PullWait.Holder != "distribution" { + t.Fatalf("the failed have does not say the pull it waits on: %+v", got) + } + // A module held for a failed backup says so and what it keeps running; removed is written once, over its key. q.NoteModule(context.Background(), ModuleFact{Module: "web", FP: fp, Composed: dig, Phase: string(delivery.PhaseHeld), Detail: "the repository is locked"}) if got := l.have(t, "n1.web"); got.Phase != delivery.PhaseHeld || got.Detail == "" { diff --git a/internal/link/messages.go b/internal/link/messages.go index dd65cb3..59cebe1 100644 --- a/internal/link/messages.go +++ b/internal/link/messages.go @@ -645,4 +645,6 @@ type ModuleFact struct { Kept []string // Sealed is the values the composition it names carries sealed to this node's key, by slot (delivery.Have.SealedAt). Sealed delivery.Seals + // PullWait is, on a failed fact, the pull from a store its apply waits on (delivery.Have.PullWait, issue 1966 ENG-9). + PullWait *delivery.PullWait }