Author SHA1 Message Date
jschoubben 85d1707e82 selfcompose: the trunk-ask backoff test is a subtest of an existing test, no function added
Builds: design 47 G-22 A-487a, the test budget (module total back to main's count)
2026-11-19 01:24:02 +01:00
jschoubben cb6cd9d996 selfcompose: hear catalog-revision on the contract's subject, consume it in the manifest, one test
Subscribe to meshcatalog.SubjectCatalogRevisionAll (no string of its own); module.json consumes
mesh-catalog.catalog-revision so the node account is granted it; the two new tests are folded into
TestTrunkAskBackoffAndCatalogRevision and wait on channels, not time.

Builds: design 47 G-22 A-487a, A-487c, principle 3
2026-11-19 01:18:38 +01:00
jschoubben 2057c9d304 selfcompose: a refused catalog-revision subscription is said once and tried again, never refuses Watch
The subscription is a wake-up; the backoff bounds the asks without it. The refusal is said with its subject and error
(principle 3), the composer goes on, and the subscription is tried again on laterAgain's steps until it holds.

Builds: design 47 G-22 A-487a, principle 3
2026-11-19 00:59:17 +01:00
jschoubben 6ac8952541 selfcompose: ask a trunk build the catalog gave none of again on a backoff or a catalog-revision, not at every pass
A trunk build answered none waits laterAgain's step for its module (5 s growing to 5 min); the seat mesh-catalog's
event catalog-revision, heard through Env.Hear, ends the wait of every module at once; an answer starts the steps
again. A pass is woken at the step's end.

Builds: design 47 G-22 A-487a, A-461e, G-12 line 317
2026-11-19 00:57:21 +01:00
6 changed files with 195 additions and 5 deletions
+39 -2
View File
@@ -589,6 +589,9 @@ func (c *Composer) askTrunkBuilds() (wait func()) {
sayLines = append(sayLines, a.Said) sayLines = append(sayLines, a.Said)
} }
if a.NeedTrunk { if a.NeedTrunk {
if r, backing := c.trunkRetry[module]; backing && c.now().Before(r.next) {
continue // the catalog gave none and its backoff runs: asked again at its end or at a catalog-revision (A-487a)
}
if had, asked := c.trunkBuilds[module]; !asked || had.running != a.RunFP { if had, asked := c.trunkBuilds[module]; !asked || had.running != a.RunFP {
asks = append(asks, ask{node, module, a.RunFP}) asks = append(asks, ask{node, module, a.RunFP})
if c.trunkAsking == nil { if c.trunkAsking == nil {
@@ -614,6 +617,7 @@ func (c *Composer) askTrunkBuilds() (wait func()) {
for a := range work { for a := range work {
t, why := c.trunkOf(a.module) t, why := c.trunkOf(a.module)
fp := t.fp fp := t.fp
var wait time.Duration
c.mu.Lock() c.mu.Lock()
delete(c.trunkAsking, a.module) delete(c.trunkAsking, a.module)
if fp != "" { if fp != "" {
@@ -623,16 +627,20 @@ func (c *Composer) askTrunkBuilds() (wait func()) {
t.running = a.running t.running = a.running
c.trunkBuilds[a.module] = t c.trunkBuilds[a.module] = t
delete(c.trunkWhy, a.module) delete(c.trunkWhy, a.module)
delete(c.trunkRetry, a.module)
} else { } else {
if c.trunkWhy == nil { if c.trunkWhy == nil {
c.trunkWhy = map[string]string{} c.trunkWhy = map[string]string{}
} }
c.trunkWhy[a.module] = why c.trunkWhy[a.module] = why
wait = c.trunkBackoffLocked(a.module)
} }
c.mu.Unlock() c.mu.Unlock()
if fp == "" { if fp == "" {
// Said with its reason, and asked again on the next pass: nothing is kept of an answer that was none (A-461e). // Said with its reason, and asked again at a catalog-revision or after its backoff, whichever comes first, never
c.env.Say(fmt.Sprintf("the trunk build of %s, placed on %s, could not be had, and is asked again: %s", a.module, a.node, why)) // at every pass (A-461e, A-487a); the backoff wakes a pass at its end.
c.env.Say(fmt.Sprintf("the trunk build of %s, placed on %s, could not be had, and is asked again in %s or when the catalog says a revision: %s", a.module, a.node, wait, why))
time.AfterFunc(wait, c.askRehand)
continue continue
} }
c.askRehand() c.askRehand()
@@ -648,6 +656,35 @@ func (c *Composer) askTrunkBuilds() (wait func()) {
return all.Wait return all.Wait
} }
// trunkBackoff is the asks of a module's trunk build that gave none, in a row, and the time before which it is not asked again.
type trunkBackoff struct {
tries int
next time.Time
}
// trunkBackoffLocked notes one more ask of the module's trunk build that gave none and returns the wait before it is asked again:
// laterAgain's, 5 s growing to 5 minutes (A-487a). Under mu.
func (c *Composer) trunkBackoffLocked(module string) time.Duration {
if c.trunkRetry == nil {
c.trunkRetry = map[string]trunkBackoff{}
}
r := c.trunkRetry[module]
wait := LaterWait(r.tries)
c.trunkRetry[module] = trunkBackoff{tries: r.tries + 1, next: c.now().Add(wait)}
return wait
}
// trunkRetryCleared ends the wait of every module whose trunk build the catalog gave none of, on the catalog's word that it holds a
// new revision: the next pass asks them again (A-487a). The asks in a row are kept, so one that fails again waits longer.
func (c *Composer) trunkRetryCleared() {
c.mu.Lock()
defer c.mu.Unlock()
for m, r := range c.trunkRetry {
r.next = time.Time{}
c.trunkRetry[m] = r
}
}
// entryManifest is an entry's manifest as a reader takes it, with its digest: the one the indexer set at take-in, or, where it set // entryManifest is an entry's manifest as a reader takes it, with its digest: the one the indexer set at take-in, or, where it set
// none, the digest of the manifest's bytes. // none, the digest of the manifest's bytes.
func entryManifest(e meshcatalog.CatalogEntry) (digest string, raw []byte, err error) { func entryManifest(e meshcatalog.CatalogEntry) (digest string, raw []byte, err error) {
+105
View File
@@ -635,6 +635,7 @@ func TestRunningBuildEntryKeptAndHandedAgain(t *testing.T) {
// A-461, A-461e: a node asks the trunk build only of what it needs of another node's placement (the store's holder, the provider // A-461, A-461e: a node asks the trunk build only of what it needs of another node's placement (the store's holder, the provider
// it is bound to), never of every module placed there, and beside the pass. // it is bound to), never of every module placed there, and beside the pass.
func TestOnlyWhatTheNodeNeedsOfAnotherNodesPlacementIsAsked(t *testing.T) { func TestOnlyWhatTheNodeNeedsOfAnotherNodesPlacementIsAsked(t *testing.T) {
t.Run("a trunk build the catalog gave none of waits its backoff or a catalog-revision (A-487a)", trunkAskBackoffAndCatalogRevision)
env := newFakeBus().env() env := newFakeBus().env()
env.Node = "n1" env.Node = "n1"
var mu sync.Mutex var mu sync.Mutex
@@ -666,3 +667,107 @@ func TestOnlyWhatTheNodeNeedsOfAnotherNodesPlacementIsAsked(t *testing.T) {
t.Errorf("the trunk builds asked are %v, want the store's holder and the bound provider alone", asked) t.Errorf("the trunk builds asked are %v, want the store's holder and the bound provider alone", asked)
} }
} }
// A-487a (A-461e): a trunk build the catalog gave none of is not asked again at every pass: it waits laterAgain's step for the module
// (5 s, then 15 s), and the seat's event catalog-revision ends the wait at once; an answer starts the steps again from 5 s. The
// event is heard through Env.Hear on the contract's subject; a refused subscription is said once and tried again.
func trunkAskBackoffAndCatalogRevision(t *testing.T) {
old := meshcatalogclient.SourcePrefix2 + strings.Repeat("1", 64)
now := time.Date(2026, 11, 19, 10, 0, 0, 0, time.UTC)
var asks int
answers := false
env := newFakeBus().env()
env.Node = "n1"
env.Now = func() time.Time { return now }
env.Entry = EntryCall(func(_ context.Context, _, verb string, _ any) (json.RawMessage, error) {
if verb != meshcatalog.VerbModule {
return nil, errors.New("not held")
}
asks++
if !answers {
return nil, errors.New("the catalog is busy")
}
return json.Marshal(meshcatalog.ModuleAnswer{Module: meshcatalog.ModuleAnswerModule{Fp: old, Manifest: map[string]any{"module": "postgres"}}})
})
c := New(env)
c.mu.Lock()
c.wants["postgres"] = meshwants.WantsByWantNodeModuleValue{Fp: old}
c.assignments["n1"] = meshmoduleassigner.Assignments{Node: "n1", Modules: []meshmoduleassigner.PlacedModule{{Module: "postgres"}}}
c.heard = map[string]map[string]heardRun{"n1": {"postgres": {fingerprint: old}}}
c.mu.Unlock()
pass := func(want int, what string) {
t.Helper()
c.askTrunkBuilds()()
if asks != want {
t.Fatalf("%s: the catalog was asked %d times, want %d", what, asks, want)
}
}
pass(1, "first pass")
pass(1, "a second pass at once")
now = now.Add(4 * time.Second)
pass(1, "within the 5 s step")
now = now.Add(2 * time.Second)
pass(2, "after the 5 s step")
now = now.Add(14 * time.Second)
pass(2, "within the 15 s step")
c.trunkRetryCleared() // what hearing catalog-revision does
pass(3, "after catalog-revision")
pass(3, "the revision's ask gave none again: a step waits again")
answers = true
now = now.Add(time.Minute)
pass(4, "after the longer step")
if _, backing := c.trunkRetry["postgres"]; backing {
t.Errorf("an answer came and the backoff of the module stands: %+v", c.trunkRetry)
}
// The wiring: the composer hears the seat's event on the contract's subject. A subscription Hear refuses is said once with the subject
// and the error, never refuses Watch, and is tried again (principle 3); once it holds, an event clears the waits.
setFor(t, &laterAgain, []time.Duration{time.Nanosecond})
b := newFakeBus()
env.Say = b.env().Say
attempts := make(chan struct{}, 8)
heard := make(chan []byte) // unbuffered: a second send returns only when the first event was handled
var tries int
env.Hear = func(subject string) (<-chan []byte, func(), error) {
if subject != meshcatalog.SubjectCatalogRevisionAll {
return make(chan []byte), func() {}, nil
}
tries++
attempts <- struct{}{}
if tries < 2 {
return nil, nil, errors.New("permissions violation")
}
return heard, func() {}, nil
}
answers = false
c = New(env)
c.mu.Lock()
c.trunkRetry = map[string]trunkBackoff{"postgres": {tries: 1, next: now.Add(time.Hour)}}
c.mu.Unlock()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
if _, err := c.Watch(ctx, "n1.>"); err != nil {
t.Fatalf("a refused catalog-revision subscription refused Watch: %v", err)
}
<-attempts
<-attempts // the second attempt comes after the refusal was said
said := 0
b.mu.Lock()
for _, line := range b.said {
if strings.Contains(line, meshcatalog.SubjectCatalogRevisionAll) && strings.Contains(line, "permissions violation") {
said++
}
}
b.mu.Unlock()
if said != 1 {
t.Errorf("the refusal was said %d times, want once", said)
}
heard <- []byte(`{}`)
heard <- []byte(`{}`)
c.mu.Lock()
next := c.trunkRetry["postgres"].next
c.mu.Unlock()
if !next.IsZero() {
t.Error("catalog-revision was heard and the wait of the module stands")
}
}
+46 -2
View File
@@ -220,8 +220,11 @@ type Composer struct {
// trunkAsking is, by module, a trunk build being asked beside the pass (askTrunkBuilds), under mu. // trunkAsking is, by module, a trunk build being asked beside the pass (askTrunkBuilds), under mu.
trunkAsking map[string]bool trunkAsking map[string]bool
// trunkWhy is, by module, why the last ask of its trunk build gave none, for the words of the one place (A-484). // trunkWhy is, by module, why the last ask of its trunk build gave none, for the words of the one place (A-484).
trunkWhy map[string]string trunkWhy map[string]string
heardRead bool // trunkRetry is, by module, the backoff of asking its trunk build again after an answer that was none: the asks made in a row
// without an answer and the time before which it is not asked (A-487a), under mu.
trunkRetry map[string]trunkBackoff
heardRead bool
// ends is, by module, the last build end naming no branch the builder said of it (NoteBuildEnded), its newest build that // ends is, by module, the last build end naming no branch the builder said of it (NoteBuildEnded), its newest build that
// ended, under mu; endsRead is whether the first read of the builder's stream ended (BuildEndsRead) (design 47 G A-293). // ended, under mu; endsRead is whether the first read of the builder's stream ended (BuildEndsRead) (design 47 G A-293).
ends map[string]builds.BuildEnded ends map[string]builds.BuildEnded
@@ -540,6 +543,12 @@ func (c *Composer) Watch(ctx context.Context, _ string) (<-chan agent.Update, er
}(i, ch) }(i, ch)
} }
if c.env.Hear != nil { if c.env.Hear != nil {
go c.hearCatalogRevisions(ctx, func() {
select {
case merged <- tagged{in: -1, ok: true}:
case <-ctx.Done():
}
})
heard, stop, err := c.env.Hear(VaultSecretChanged) heard, stop, err := c.env.Hear(VaultSecretChanged)
if err != nil { if err != nil {
return nil, fmt.Errorf("the node composes again when a secret changes, and cannot hear the vault say so: %w", err) return nil, fmt.Errorf("the node composes again when a secret changes, and cannot hear the vault say so: %w", err)
@@ -697,6 +706,41 @@ func (c *Composer) Watch(ctx context.Context, _ string) (<-chan agent.Update, er
return out, nil return out, nil
} }
// hearCatalogRevisions hears the seat mesh-catalog's event catalog-revision: each one clears the backoffs of the trunk builds the
// catalog gave none of and wakes a pass (A-487a). A wake-up only, the backoff bounds the asks without it: a subscription refused
// is said once with its subject and error (principle 3), the composer goes on composing, and the subscription is tried again on
// laterAgain's steps until it holds.
func (c *Composer) hearCatalogRevisions(ctx context.Context, wake func()) {
said := false
for try := 0; ; try++ {
revisions, stop, err := c.env.Hear(meshcatalog.SubjectCatalogRevisionAll)
if err == nil {
defer stop()
for {
select {
case <-ctx.Done():
return
case _, ok := <-revisions:
if !ok {
return
}
c.trunkRetryCleared()
wake()
}
}
}
if !said {
said = true
c.env.Say(fmt.Sprintf("the node cannot hear %s: %v; it asks the trunk builds again on a backoff alone and tries to hear it again", meshcatalog.SubjectCatalogRevisionAll, err))
}
select {
case <-ctx.Done():
return
case <-time.After(LaterWait(try)):
}
}
}
func all(b []bool) bool { func all(b []bool) bool {
for _, v := range b { for _, v := range b {
if !v { if !v {
+3
View File
@@ -249,6 +249,9 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
} }
secretChanged := make(chan []byte, 1) secretChanged := make(chan []byte, 1)
env.Hear = func(subject string) (<-chan []byte, func(), error) { env.Hear = func(subject string) (<-chan []byte, func(), error) {
if subject == meshcatalog.SubjectCatalogRevisionAll {
return make(chan []byte), func() {}, nil
}
if subject != VaultSecretChanged { if subject != VaultSecretChanged {
t.Errorf("heard %s", subject) t.Errorf("heard %s", subject)
} }
+1
View File
@@ -53,6 +53,7 @@
"seat:oci-artifact-registry.versions" "seat:oci-artifact-registry.versions"
], ],
"consumes": [ "consumes": [
"mesh-catalog.catalog-revision",
"mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-changed",
"mesh-catalog.seat-definition-removed", "mesh-catalog.seat-definition-removed",
"mesh-conditions.findings-asked", "mesh-conditions.findings-asked",
+1 -1
View File
@@ -93,7 +93,7 @@ func TestTheManifestConsumesTheBackupEvent(t *testing.T) {
t.Errorf("config recovery-wait-minutes is %+v, want default 90", m.Config["recovery-wait-minutes"]) t.Errorf("config recovery-wait-minutes is %+v, want default 90", m.Config["recovery-wait-minutes"])
} }
// The backup event, the status view's settled, and the two events of the seat that serves the seat list (ADR 0636 §4), the keeper's ask (design 47 G §387), the seat's running (§199) and the builder's build ends (A-293). // The backup event, the status view's settled, and the two events of the seat that serves the seat list (ADR 0636 §4), the keeper's ask (design 47 G §387), the seat's running (§199) and the builder's build ends (A-293).
if want := []string{"mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", "mesh-conditions.findings-asked", "mesh-delivery-builder.build.ended.**", "mesh-delivery-view.settled", "mesh-recovery.ended", "mesh-recovery.restored", "node-engine.running", "restic.backed-up"}; !slices.Equal(m.Consumes, want) { if want := []string{"mesh-catalog.catalog-revision", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", "mesh-conditions.findings-asked", "mesh-delivery-builder.build.ended.**", "mesh-delivery-view.settled", "mesh-recovery.ended", "mesh-recovery.restored", "node-engine.running", "restic.backed-up"}; !slices.Equal(m.Consumes, want) {
t.Fatalf("consumes %v, want %v", m.Consumes, want) t.Fatalf("consumes %v, want %v", m.Consumes, want)
} }
// 774: the acts bucket mesh-host_acts is not declared as state for now: a declared state of a module with no account // 774: the acts bucket mesh-host_acts is not declared as state for now: a declared state of a module with no account