diff --git a/internal/selfcompose/entries.go b/internal/selfcompose/entries.go index c3b5da11..bc0311e9 100644 --- a/internal/selfcompose/entries.go +++ b/internal/selfcompose/entries.go @@ -589,6 +589,9 @@ func (c *Composer) askTrunkBuilds() (wait func()) { sayLines = append(sayLines, a.Said) } if a.NeedTrunk { + if r, backing := c.trunkRetry[module]; backing && c.now().Before(r.next) { + continue // the catalog gave none and its backoff runs: asked again at its end or at a catalog-revision (A-487a) + } if had, asked := c.trunkBuilds[module]; !asked || had.running != a.RunFP { asks = append(asks, ask{node, module, a.RunFP}) if c.trunkAsking == nil { @@ -614,6 +617,7 @@ func (c *Composer) askTrunkBuilds() (wait func()) { for a := range work { t, why := c.trunkOf(a.module) fp := t.fp + var wait time.Duration c.mu.Lock() delete(c.trunkAsking, a.module) if fp != "" { @@ -623,16 +627,20 @@ func (c *Composer) askTrunkBuilds() (wait func()) { t.running = a.running c.trunkBuilds[a.module] = t delete(c.trunkWhy, a.module) + delete(c.trunkRetry, a.module) } else { if c.trunkWhy == nil { c.trunkWhy = map[string]string{} } c.trunkWhy[a.module] = why + wait = c.trunkBackoffLocked(a.module) } c.mu.Unlock() if fp == "" { - // Said with its reason, and asked again on the next pass: nothing is kept of an answer that was none (A-461e). - c.env.Say(fmt.Sprintf("the trunk build of %s, placed on %s, could not be had, and is asked again: %s", a.module, a.node, why)) + // Said with its reason, and asked again at a catalog-revision or after its backoff, whichever comes first, never + // at every pass (A-461e, A-487a); the backoff wakes a pass at its end. + c.env.Say(fmt.Sprintf("the trunk build of %s, placed on %s, could not be had, and is asked again in %s or when the catalog says a revision: %s", a.module, a.node, wait, why)) + time.AfterFunc(wait, c.askRehand) continue } c.askRehand() @@ -648,6 +656,35 @@ func (c *Composer) askTrunkBuilds() (wait func()) { return all.Wait } +// trunkBackoff is the asks of a module's trunk build that gave none, in a row, and the time before which it is not asked again. +type trunkBackoff struct { + tries int + next time.Time +} + +// trunkBackoffLocked notes one more ask of the module's trunk build that gave none and returns the wait before it is asked again: +// laterAgain's, 5 s growing to 5 minutes (A-487a). Under mu. +func (c *Composer) trunkBackoffLocked(module string) time.Duration { + if c.trunkRetry == nil { + c.trunkRetry = map[string]trunkBackoff{} + } + r := c.trunkRetry[module] + wait := LaterWait(r.tries) + c.trunkRetry[module] = trunkBackoff{tries: r.tries + 1, next: c.now().Add(wait)} + return wait +} + +// trunkRetryCleared ends the wait of every module whose trunk build the catalog gave none of, on the catalog's word that it holds a +// new revision: the next pass asks them again (A-487a). The asks in a row are kept, so one that fails again waits longer. +func (c *Composer) trunkRetryCleared() { + c.mu.Lock() + defer c.mu.Unlock() + for m, r := range c.trunkRetry { + r.next = time.Time{} + c.trunkRetry[m] = r + } +} + // entryManifest is an entry's manifest as a reader takes it, with its digest: the one the indexer set at take-in, or, where it set // none, the digest of the manifest's bytes. func entryManifest(e meshcatalog.CatalogEntry) (digest string, raw []byte, err error) { diff --git a/internal/selfcompose/entries_test.go b/internal/selfcompose/entries_test.go index 937214b7..5c189589 100644 --- a/internal/selfcompose/entries_test.go +++ b/internal/selfcompose/entries_test.go @@ -635,6 +635,7 @@ func TestRunningBuildEntryKeptAndHandedAgain(t *testing.T) { // A-461, A-461e: a node asks the trunk build only of what it needs of another node's placement (the store's holder, the provider // it is bound to), never of every module placed there, and beside the pass. func TestOnlyWhatTheNodeNeedsOfAnotherNodesPlacementIsAsked(t *testing.T) { + t.Run("a trunk build the catalog gave none of waits its backoff or a catalog-revision (A-487a)", trunkAskBackoffAndCatalogRevision) env := newFakeBus().env() env.Node = "n1" var mu sync.Mutex @@ -666,3 +667,107 @@ func TestOnlyWhatTheNodeNeedsOfAnotherNodesPlacementIsAsked(t *testing.T) { t.Errorf("the trunk builds asked are %v, want the store's holder and the bound provider alone", asked) } } + +// A-487a (A-461e): a trunk build the catalog gave none of is not asked again at every pass: it waits laterAgain's step for the module +// (5 s, then 15 s), and the seat's event catalog-revision ends the wait at once; an answer starts the steps again from 5 s. The +// event is heard through Env.Hear on the contract's subject; a refused subscription is said once and tried again. +func trunkAskBackoffAndCatalogRevision(t *testing.T) { + old := meshcatalogclient.SourcePrefix2 + strings.Repeat("1", 64) + now := time.Date(2026, 11, 19, 10, 0, 0, 0, time.UTC) + var asks int + answers := false + env := newFakeBus().env() + env.Node = "n1" + env.Now = func() time.Time { return now } + env.Entry = EntryCall(func(_ context.Context, _, verb string, _ any) (json.RawMessage, error) { + if verb != meshcatalog.VerbModule { + return nil, errors.New("not held") + } + asks++ + if !answers { + return nil, errors.New("the catalog is busy") + } + return json.Marshal(meshcatalog.ModuleAnswer{Module: meshcatalog.ModuleAnswerModule{Fp: old, Manifest: map[string]any{"module": "postgres"}}}) + }) + c := New(env) + c.mu.Lock() + c.wants["postgres"] = meshwants.WantsByWantNodeModuleValue{Fp: old} + c.assignments["n1"] = meshmoduleassigner.Assignments{Node: "n1", Modules: []meshmoduleassigner.PlacedModule{{Module: "postgres"}}} + c.heard = map[string]map[string]heardRun{"n1": {"postgres": {fingerprint: old}}} + c.mu.Unlock() + pass := func(want int, what string) { + t.Helper() + c.askTrunkBuilds()() + if asks != want { + t.Fatalf("%s: the catalog was asked %d times, want %d", what, asks, want) + } + } + pass(1, "first pass") + pass(1, "a second pass at once") + now = now.Add(4 * time.Second) + pass(1, "within the 5 s step") + now = now.Add(2 * time.Second) + pass(2, "after the 5 s step") + now = now.Add(14 * time.Second) + pass(2, "within the 15 s step") + c.trunkRetryCleared() // what hearing catalog-revision does + pass(3, "after catalog-revision") + pass(3, "the revision's ask gave none again: a step waits again") + answers = true + now = now.Add(time.Minute) + pass(4, "after the longer step") + if _, backing := c.trunkRetry["postgres"]; backing { + t.Errorf("an answer came and the backoff of the module stands: %+v", c.trunkRetry) + } + + // The wiring: the composer hears the seat's event on the contract's subject. A subscription Hear refuses is said once with the subject + // and the error, never refuses Watch, and is tried again (principle 3); once it holds, an event clears the waits. + setFor(t, &laterAgain, []time.Duration{time.Nanosecond}) + b := newFakeBus() + env.Say = b.env().Say + attempts := make(chan struct{}, 8) + heard := make(chan []byte) // unbuffered: a second send returns only when the first event was handled + var tries int + env.Hear = func(subject string) (<-chan []byte, func(), error) { + if subject != meshcatalog.SubjectCatalogRevisionAll { + return make(chan []byte), func() {}, nil + } + tries++ + attempts <- struct{}{} + if tries < 2 { + return nil, nil, errors.New("permissions violation") + } + return heard, func() {}, nil + } + answers = false + c = New(env) + c.mu.Lock() + c.trunkRetry = map[string]trunkBackoff{"postgres": {tries: 1, next: now.Add(time.Hour)}} + c.mu.Unlock() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + if _, err := c.Watch(ctx, "n1.>"); err != nil { + t.Fatalf("a refused catalog-revision subscription refused Watch: %v", err) + } + <-attempts + <-attempts // the second attempt comes after the refusal was said + said := 0 + b.mu.Lock() + for _, line := range b.said { + if strings.Contains(line, meshcatalog.SubjectCatalogRevisionAll) && strings.Contains(line, "permissions violation") { + said++ + } + } + b.mu.Unlock() + if said != 1 { + t.Errorf("the refusal was said %d times, want once", said) + } + heard <- []byte(`{}`) + heard <- []byte(`{}`) + c.mu.Lock() + next := c.trunkRetry["postgres"].next + c.mu.Unlock() + if !next.IsZero() { + t.Error("catalog-revision was heard and the wait of the module stands") + } +} diff --git a/internal/selfcompose/selfcompose.go b/internal/selfcompose/selfcompose.go index 7aadc3dc..089d1d56 100644 --- a/internal/selfcompose/selfcompose.go +++ b/internal/selfcompose/selfcompose.go @@ -220,8 +220,11 @@ type Composer struct { // trunkAsking is, by module, a trunk build being asked beside the pass (askTrunkBuilds), under mu. trunkAsking map[string]bool // trunkWhy is, by module, why the last ask of its trunk build gave none, for the words of the one place (A-484). - trunkWhy map[string]string - heardRead bool + trunkWhy map[string]string + // trunkRetry is, by module, the backoff of asking its trunk build again after an answer that was none: the asks made in a row + // without an answer and the time before which it is not asked (A-487a), under mu. + trunkRetry map[string]trunkBackoff + heardRead bool // ends is, by module, the last build end naming no branch the builder said of it (NoteBuildEnded), its newest build that // ended, under mu; endsRead is whether the first read of the builder's stream ended (BuildEndsRead) (design 47 G A-293). ends map[string]builds.BuildEnded @@ -540,6 +543,12 @@ func (c *Composer) Watch(ctx context.Context, _ string) (<-chan agent.Update, er }(i, ch) } if c.env.Hear != nil { + go c.hearCatalogRevisions(ctx, func() { + select { + case merged <- tagged{in: -1, ok: true}: + case <-ctx.Done(): + } + }) heard, stop, err := c.env.Hear(VaultSecretChanged) if err != nil { return nil, fmt.Errorf("the node composes again when a secret changes, and cannot hear the vault say so: %w", err) @@ -697,6 +706,41 @@ func (c *Composer) Watch(ctx context.Context, _ string) (<-chan agent.Update, er return out, nil } +// hearCatalogRevisions hears the seat mesh-catalog's event catalog-revision: each one clears the backoffs of the trunk builds the +// catalog gave none of and wakes a pass (A-487a). A wake-up only, the backoff bounds the asks without it: a subscription refused +// is said once with its subject and error (principle 3), the composer goes on composing, and the subscription is tried again on +// laterAgain's steps until it holds. +func (c *Composer) hearCatalogRevisions(ctx context.Context, wake func()) { + said := false + for try := 0; ; try++ { + revisions, stop, err := c.env.Hear(meshcatalog.SubjectCatalogRevisionAll) + if err == nil { + defer stop() + for { + select { + case <-ctx.Done(): + return + case _, ok := <-revisions: + if !ok { + return + } + c.trunkRetryCleared() + wake() + } + } + } + if !said { + said = true + c.env.Say(fmt.Sprintf("the node cannot hear %s: %v; it asks the trunk builds again on a backoff alone and tries to hear it again", meshcatalog.SubjectCatalogRevisionAll, err)) + } + select { + case <-ctx.Done(): + return + case <-time.After(LaterWait(try)): + } + } +} + func all(b []bool) bool { for _, v := range b { if !v { diff --git a/internal/selfcompose/selfcompose_test.go b/internal/selfcompose/selfcompose_test.go index e97dab75..fe33e2f4 100644 --- a/internal/selfcompose/selfcompose_test.go +++ b/internal/selfcompose/selfcompose_test.go @@ -249,6 +249,9 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { } secretChanged := make(chan []byte, 1) env.Hear = func(subject string) (<-chan []byte, func(), error) { + if subject == meshcatalog.SubjectCatalogRevisionAll { + return make(chan []byte), func() {}, nil + } if subject != VaultSecretChanged { t.Errorf("heard %s", subject) } diff --git a/module.json b/module.json index 011513fe..6697a3a5 100644 --- a/module.json +++ b/module.json @@ -53,6 +53,7 @@ "seat:oci-artifact-registry.versions" ], "consumes": [ + "mesh-catalog.catalog-revision", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", "mesh-conditions.findings-asked", diff --git a/packaging/operator_keys_test.go b/packaging/operator_keys_test.go index 12524586..1fa92620 100644 --- a/packaging/operator_keys_test.go +++ b/packaging/operator_keys_test.go @@ -93,7 +93,7 @@ func TestTheManifestConsumesTheBackupEvent(t *testing.T) { t.Errorf("config recovery-wait-minutes is %+v, want default 90", m.Config["recovery-wait-minutes"]) } // The backup event, the status view's settled, and the two events of the seat that serves the seat list (ADR 0636 §4), the keeper's ask (design 47 G §387), the seat's running (§199) and the builder's build ends (A-293). - if want := []string{"mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", "mesh-conditions.findings-asked", "mesh-delivery-builder.build.ended.**", "mesh-delivery-view.settled", "mesh-recovery.ended", "mesh-recovery.restored", "node-engine.running", "restic.backed-up"}; !slices.Equal(m.Consumes, want) { + if want := []string{"mesh-catalog.catalog-revision", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", "mesh-conditions.findings-asked", "mesh-delivery-builder.build.ended.**", "mesh-delivery-view.settled", "mesh-recovery.ended", "mesh-recovery.restored", "node-engine.running", "restic.backed-up"}; !slices.Equal(m.Consumes, want) { t.Fatalf("consumes %v, want %v", m.Consumes, want) } // 774: the acts bucket mesh-host_acts is not declared as state for now: a declared state of a module with no account