Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
85d1707e82 | ||
|
|
cb6cd9d996 | ||
|
|
2057c9d304 | ||
|
|
6ac8952541 |
@@ -589,6 +589,9 @@ func (c *Composer) askTrunkBuilds() (wait func()) {
|
|||||||
sayLines = append(sayLines, a.Said)
|
sayLines = append(sayLines, a.Said)
|
||||||
}
|
}
|
||||||
if a.NeedTrunk {
|
if a.NeedTrunk {
|
||||||
|
if r, backing := c.trunkRetry[module]; backing && c.now().Before(r.next) {
|
||||||
|
continue // the catalog gave none and its backoff runs: asked again at its end or at a catalog-revision (A-487a)
|
||||||
|
}
|
||||||
if had, asked := c.trunkBuilds[module]; !asked || had.running != a.RunFP {
|
if had, asked := c.trunkBuilds[module]; !asked || had.running != a.RunFP {
|
||||||
asks = append(asks, ask{node, module, a.RunFP})
|
asks = append(asks, ask{node, module, a.RunFP})
|
||||||
if c.trunkAsking == nil {
|
if c.trunkAsking == nil {
|
||||||
@@ -614,6 +617,7 @@ func (c *Composer) askTrunkBuilds() (wait func()) {
|
|||||||
for a := range work {
|
for a := range work {
|
||||||
t, why := c.trunkOf(a.module)
|
t, why := c.trunkOf(a.module)
|
||||||
fp := t.fp
|
fp := t.fp
|
||||||
|
var wait time.Duration
|
||||||
c.mu.Lock()
|
c.mu.Lock()
|
||||||
delete(c.trunkAsking, a.module)
|
delete(c.trunkAsking, a.module)
|
||||||
if fp != "" {
|
if fp != "" {
|
||||||
@@ -623,16 +627,20 @@ func (c *Composer) askTrunkBuilds() (wait func()) {
|
|||||||
t.running = a.running
|
t.running = a.running
|
||||||
c.trunkBuilds[a.module] = t
|
c.trunkBuilds[a.module] = t
|
||||||
delete(c.trunkWhy, a.module)
|
delete(c.trunkWhy, a.module)
|
||||||
|
delete(c.trunkRetry, a.module)
|
||||||
} else {
|
} else {
|
||||||
if c.trunkWhy == nil {
|
if c.trunkWhy == nil {
|
||||||
c.trunkWhy = map[string]string{}
|
c.trunkWhy = map[string]string{}
|
||||||
}
|
}
|
||||||
c.trunkWhy[a.module] = why
|
c.trunkWhy[a.module] = why
|
||||||
|
wait = c.trunkBackoffLocked(a.module)
|
||||||
}
|
}
|
||||||
c.mu.Unlock()
|
c.mu.Unlock()
|
||||||
if fp == "" {
|
if fp == "" {
|
||||||
// Said with its reason, and asked again on the next pass: nothing is kept of an answer that was none (A-461e).
|
// Said with its reason, and asked again at a catalog-revision or after its backoff, whichever comes first, never
|
||||||
c.env.Say(fmt.Sprintf("the trunk build of %s, placed on %s, could not be had, and is asked again: %s", a.module, a.node, why))
|
// at every pass (A-461e, A-487a); the backoff wakes a pass at its end.
|
||||||
|
c.env.Say(fmt.Sprintf("the trunk build of %s, placed on %s, could not be had, and is asked again in %s or when the catalog says a revision: %s", a.module, a.node, wait, why))
|
||||||
|
time.AfterFunc(wait, c.askRehand)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
c.askRehand()
|
c.askRehand()
|
||||||
@@ -648,6 +656,35 @@ func (c *Composer) askTrunkBuilds() (wait func()) {
|
|||||||
return all.Wait
|
return all.Wait
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// trunkBackoff is the asks of a module's trunk build that gave none, in a row, and the time before which it is not asked again.
|
||||||
|
type trunkBackoff struct {
|
||||||
|
tries int
|
||||||
|
next time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// trunkBackoffLocked notes one more ask of the module's trunk build that gave none and returns the wait before it is asked again:
|
||||||
|
// laterAgain's, 5 s growing to 5 minutes (A-487a). Under mu.
|
||||||
|
func (c *Composer) trunkBackoffLocked(module string) time.Duration {
|
||||||
|
if c.trunkRetry == nil {
|
||||||
|
c.trunkRetry = map[string]trunkBackoff{}
|
||||||
|
}
|
||||||
|
r := c.trunkRetry[module]
|
||||||
|
wait := LaterWait(r.tries)
|
||||||
|
c.trunkRetry[module] = trunkBackoff{tries: r.tries + 1, next: c.now().Add(wait)}
|
||||||
|
return wait
|
||||||
|
}
|
||||||
|
|
||||||
|
// trunkRetryCleared ends the wait of every module whose trunk build the catalog gave none of, on the catalog's word that it holds a
|
||||||
|
// new revision: the next pass asks them again (A-487a). The asks in a row are kept, so one that fails again waits longer.
|
||||||
|
func (c *Composer) trunkRetryCleared() {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
for m, r := range c.trunkRetry {
|
||||||
|
r.next = time.Time{}
|
||||||
|
c.trunkRetry[m] = r
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// entryManifest is an entry's manifest as a reader takes it, with its digest: the one the indexer set at take-in, or, where it set
|
// entryManifest is an entry's manifest as a reader takes it, with its digest: the one the indexer set at take-in, or, where it set
|
||||||
// none, the digest of the manifest's bytes.
|
// none, the digest of the manifest's bytes.
|
||||||
func entryManifest(e meshcatalog.CatalogEntry) (digest string, raw []byte, err error) {
|
func entryManifest(e meshcatalog.CatalogEntry) (digest string, raw []byte, err error) {
|
||||||
|
|||||||
@@ -635,6 +635,7 @@ func TestRunningBuildEntryKeptAndHandedAgain(t *testing.T) {
|
|||||||
// A-461, A-461e: a node asks the trunk build only of what it needs of another node's placement (the store's holder, the provider
|
// A-461, A-461e: a node asks the trunk build only of what it needs of another node's placement (the store's holder, the provider
|
||||||
// it is bound to), never of every module placed there, and beside the pass.
|
// it is bound to), never of every module placed there, and beside the pass.
|
||||||
func TestOnlyWhatTheNodeNeedsOfAnotherNodesPlacementIsAsked(t *testing.T) {
|
func TestOnlyWhatTheNodeNeedsOfAnotherNodesPlacementIsAsked(t *testing.T) {
|
||||||
|
t.Run("a trunk build the catalog gave none of waits its backoff or a catalog-revision (A-487a)", trunkAskBackoffAndCatalogRevision)
|
||||||
env := newFakeBus().env()
|
env := newFakeBus().env()
|
||||||
env.Node = "n1"
|
env.Node = "n1"
|
||||||
var mu sync.Mutex
|
var mu sync.Mutex
|
||||||
@@ -666,3 +667,107 @@ func TestOnlyWhatTheNodeNeedsOfAnotherNodesPlacementIsAsked(t *testing.T) {
|
|||||||
t.Errorf("the trunk builds asked are %v, want the store's holder and the bound provider alone", asked)
|
t.Errorf("the trunk builds asked are %v, want the store's holder and the bound provider alone", asked)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A-487a (A-461e): a trunk build the catalog gave none of is not asked again at every pass: it waits laterAgain's step for the module
|
||||||
|
// (5 s, then 15 s), and the seat's event catalog-revision ends the wait at once; an answer starts the steps again from 5 s. The
|
||||||
|
// event is heard through Env.Hear on the contract's subject; a refused subscription is said once and tried again.
|
||||||
|
func trunkAskBackoffAndCatalogRevision(t *testing.T) {
|
||||||
|
old := meshcatalogclient.SourcePrefix2 + strings.Repeat("1", 64)
|
||||||
|
now := time.Date(2026, 11, 19, 10, 0, 0, 0, time.UTC)
|
||||||
|
var asks int
|
||||||
|
answers := false
|
||||||
|
env := newFakeBus().env()
|
||||||
|
env.Node = "n1"
|
||||||
|
env.Now = func() time.Time { return now }
|
||||||
|
env.Entry = EntryCall(func(_ context.Context, _, verb string, _ any) (json.RawMessage, error) {
|
||||||
|
if verb != meshcatalog.VerbModule {
|
||||||
|
return nil, errors.New("not held")
|
||||||
|
}
|
||||||
|
asks++
|
||||||
|
if !answers {
|
||||||
|
return nil, errors.New("the catalog is busy")
|
||||||
|
}
|
||||||
|
return json.Marshal(meshcatalog.ModuleAnswer{Module: meshcatalog.ModuleAnswerModule{Fp: old, Manifest: map[string]any{"module": "postgres"}}})
|
||||||
|
})
|
||||||
|
c := New(env)
|
||||||
|
c.mu.Lock()
|
||||||
|
c.wants["postgres"] = meshwants.WantsByWantNodeModuleValue{Fp: old}
|
||||||
|
c.assignments["n1"] = meshmoduleassigner.Assignments{Node: "n1", Modules: []meshmoduleassigner.PlacedModule{{Module: "postgres"}}}
|
||||||
|
c.heard = map[string]map[string]heardRun{"n1": {"postgres": {fingerprint: old}}}
|
||||||
|
c.mu.Unlock()
|
||||||
|
pass := func(want int, what string) {
|
||||||
|
t.Helper()
|
||||||
|
c.askTrunkBuilds()()
|
||||||
|
if asks != want {
|
||||||
|
t.Fatalf("%s: the catalog was asked %d times, want %d", what, asks, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pass(1, "first pass")
|
||||||
|
pass(1, "a second pass at once")
|
||||||
|
now = now.Add(4 * time.Second)
|
||||||
|
pass(1, "within the 5 s step")
|
||||||
|
now = now.Add(2 * time.Second)
|
||||||
|
pass(2, "after the 5 s step")
|
||||||
|
now = now.Add(14 * time.Second)
|
||||||
|
pass(2, "within the 15 s step")
|
||||||
|
c.trunkRetryCleared() // what hearing catalog-revision does
|
||||||
|
pass(3, "after catalog-revision")
|
||||||
|
pass(3, "the revision's ask gave none again: a step waits again")
|
||||||
|
answers = true
|
||||||
|
now = now.Add(time.Minute)
|
||||||
|
pass(4, "after the longer step")
|
||||||
|
if _, backing := c.trunkRetry["postgres"]; backing {
|
||||||
|
t.Errorf("an answer came and the backoff of the module stands: %+v", c.trunkRetry)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The wiring: the composer hears the seat's event on the contract's subject. A subscription Hear refuses is said once with the subject
|
||||||
|
// and the error, never refuses Watch, and is tried again (principle 3); once it holds, an event clears the waits.
|
||||||
|
setFor(t, &laterAgain, []time.Duration{time.Nanosecond})
|
||||||
|
b := newFakeBus()
|
||||||
|
env.Say = b.env().Say
|
||||||
|
attempts := make(chan struct{}, 8)
|
||||||
|
heard := make(chan []byte) // unbuffered: a second send returns only when the first event was handled
|
||||||
|
var tries int
|
||||||
|
env.Hear = func(subject string) (<-chan []byte, func(), error) {
|
||||||
|
if subject != meshcatalog.SubjectCatalogRevisionAll {
|
||||||
|
return make(chan []byte), func() {}, nil
|
||||||
|
}
|
||||||
|
tries++
|
||||||
|
attempts <- struct{}{}
|
||||||
|
if tries < 2 {
|
||||||
|
return nil, nil, errors.New("permissions violation")
|
||||||
|
}
|
||||||
|
return heard, func() {}, nil
|
||||||
|
}
|
||||||
|
answers = false
|
||||||
|
c = New(env)
|
||||||
|
c.mu.Lock()
|
||||||
|
c.trunkRetry = map[string]trunkBackoff{"postgres": {tries: 1, next: now.Add(time.Hour)}}
|
||||||
|
c.mu.Unlock()
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
if _, err := c.Watch(ctx, "n1.>"); err != nil {
|
||||||
|
t.Fatalf("a refused catalog-revision subscription refused Watch: %v", err)
|
||||||
|
}
|
||||||
|
<-attempts
|
||||||
|
<-attempts // the second attempt comes after the refusal was said
|
||||||
|
said := 0
|
||||||
|
b.mu.Lock()
|
||||||
|
for _, line := range b.said {
|
||||||
|
if strings.Contains(line, meshcatalog.SubjectCatalogRevisionAll) && strings.Contains(line, "permissions violation") {
|
||||||
|
said++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
b.mu.Unlock()
|
||||||
|
if said != 1 {
|
||||||
|
t.Errorf("the refusal was said %d times, want once", said)
|
||||||
|
}
|
||||||
|
heard <- []byte(`{}`)
|
||||||
|
heard <- []byte(`{}`)
|
||||||
|
c.mu.Lock()
|
||||||
|
next := c.trunkRetry["postgres"].next
|
||||||
|
c.mu.Unlock()
|
||||||
|
if !next.IsZero() {
|
||||||
|
t.Error("catalog-revision was heard and the wait of the module stands")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -221,6 +221,9 @@ type Composer struct {
|
|||||||
trunkAsking map[string]bool
|
trunkAsking map[string]bool
|
||||||
// trunkWhy is, by module, why the last ask of its trunk build gave none, for the words of the one place (A-484).
|
// trunkWhy is, by module, why the last ask of its trunk build gave none, for the words of the one place (A-484).
|
||||||
trunkWhy map[string]string
|
trunkWhy map[string]string
|
||||||
|
// trunkRetry is, by module, the backoff of asking its trunk build again after an answer that was none: the asks made in a row
|
||||||
|
// without an answer and the time before which it is not asked (A-487a), under mu.
|
||||||
|
trunkRetry map[string]trunkBackoff
|
||||||
heardRead bool
|
heardRead bool
|
||||||
// ends is, by module, the last build end naming no branch the builder said of it (NoteBuildEnded), its newest build that
|
// ends is, by module, the last build end naming no branch the builder said of it (NoteBuildEnded), its newest build that
|
||||||
// ended, under mu; endsRead is whether the first read of the builder's stream ended (BuildEndsRead) (design 47 G A-293).
|
// ended, under mu; endsRead is whether the first read of the builder's stream ended (BuildEndsRead) (design 47 G A-293).
|
||||||
@@ -540,6 +543,12 @@ func (c *Composer) Watch(ctx context.Context, _ string) (<-chan agent.Update, er
|
|||||||
}(i, ch)
|
}(i, ch)
|
||||||
}
|
}
|
||||||
if c.env.Hear != nil {
|
if c.env.Hear != nil {
|
||||||
|
go c.hearCatalogRevisions(ctx, func() {
|
||||||
|
select {
|
||||||
|
case merged <- tagged{in: -1, ok: true}:
|
||||||
|
case <-ctx.Done():
|
||||||
|
}
|
||||||
|
})
|
||||||
heard, stop, err := c.env.Hear(VaultSecretChanged)
|
heard, stop, err := c.env.Hear(VaultSecretChanged)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("the node composes again when a secret changes, and cannot hear the vault say so: %w", err)
|
return nil, fmt.Errorf("the node composes again when a secret changes, and cannot hear the vault say so: %w", err)
|
||||||
@@ -697,6 +706,41 @@ func (c *Composer) Watch(ctx context.Context, _ string) (<-chan agent.Update, er
|
|||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// hearCatalogRevisions hears the seat mesh-catalog's event catalog-revision: each one clears the backoffs of the trunk builds the
|
||||||
|
// catalog gave none of and wakes a pass (A-487a). A wake-up only, the backoff bounds the asks without it: a subscription refused
|
||||||
|
// is said once with its subject and error (principle 3), the composer goes on composing, and the subscription is tried again on
|
||||||
|
// laterAgain's steps until it holds.
|
||||||
|
func (c *Composer) hearCatalogRevisions(ctx context.Context, wake func()) {
|
||||||
|
said := false
|
||||||
|
for try := 0; ; try++ {
|
||||||
|
revisions, stop, err := c.env.Hear(meshcatalog.SubjectCatalogRevisionAll)
|
||||||
|
if err == nil {
|
||||||
|
defer stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case _, ok := <-revisions:
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
c.trunkRetryCleared()
|
||||||
|
wake()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !said {
|
||||||
|
said = true
|
||||||
|
c.env.Say(fmt.Sprintf("the node cannot hear %s: %v; it asks the trunk builds again on a backoff alone and tries to hear it again", meshcatalog.SubjectCatalogRevisionAll, err))
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-time.After(LaterWait(try)):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func all(b []bool) bool {
|
func all(b []bool) bool {
|
||||||
for _, v := range b {
|
for _, v := range b {
|
||||||
if !v {
|
if !v {
|
||||||
|
|||||||
@@ -249,6 +249,9 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
|||||||
}
|
}
|
||||||
secretChanged := make(chan []byte, 1)
|
secretChanged := make(chan []byte, 1)
|
||||||
env.Hear = func(subject string) (<-chan []byte, func(), error) {
|
env.Hear = func(subject string) (<-chan []byte, func(), error) {
|
||||||
|
if subject == meshcatalog.SubjectCatalogRevisionAll {
|
||||||
|
return make(chan []byte), func() {}, nil
|
||||||
|
}
|
||||||
if subject != VaultSecretChanged {
|
if subject != VaultSecretChanged {
|
||||||
t.Errorf("heard %s", subject)
|
t.Errorf("heard %s", subject)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -53,6 +53,7 @@
|
|||||||
"seat:oci-artifact-registry.versions"
|
"seat:oci-artifact-registry.versions"
|
||||||
],
|
],
|
||||||
"consumes": [
|
"consumes": [
|
||||||
|
"mesh-catalog.catalog-revision",
|
||||||
"mesh-catalog.seat-definition-changed",
|
"mesh-catalog.seat-definition-changed",
|
||||||
"mesh-catalog.seat-definition-removed",
|
"mesh-catalog.seat-definition-removed",
|
||||||
"mesh-conditions.findings-asked",
|
"mesh-conditions.findings-asked",
|
||||||
|
|||||||
@@ -93,7 +93,7 @@ func TestTheManifestConsumesTheBackupEvent(t *testing.T) {
|
|||||||
t.Errorf("config recovery-wait-minutes is %+v, want default 90", m.Config["recovery-wait-minutes"])
|
t.Errorf("config recovery-wait-minutes is %+v, want default 90", m.Config["recovery-wait-minutes"])
|
||||||
}
|
}
|
||||||
// The backup event, the status view's settled, and the two events of the seat that serves the seat list (ADR 0636 §4), the keeper's ask (design 47 G §387), the seat's running (§199) and the builder's build ends (A-293).
|
// The backup event, the status view's settled, and the two events of the seat that serves the seat list (ADR 0636 §4), the keeper's ask (design 47 G §387), the seat's running (§199) and the builder's build ends (A-293).
|
||||||
if want := []string{"mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", "mesh-conditions.findings-asked", "mesh-delivery-builder.build.ended.**", "mesh-delivery-view.settled", "mesh-recovery.ended", "mesh-recovery.restored", "node-engine.running", "restic.backed-up"}; !slices.Equal(m.Consumes, want) {
|
if want := []string{"mesh-catalog.catalog-revision", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", "mesh-conditions.findings-asked", "mesh-delivery-builder.build.ended.**", "mesh-delivery-view.settled", "mesh-recovery.ended", "mesh-recovery.restored", "node-engine.running", "restic.backed-up"}; !slices.Equal(m.Consumes, want) {
|
||||||
t.Fatalf("consumes %v, want %v", m.Consumes, want)
|
t.Fatalf("consumes %v, want %v", m.Consumes, want)
|
||||||
}
|
}
|
||||||
// 774: the acts bucket mesh-host_acts is not declared as state for now: a declared state of a module with no account
|
// 774: the acts bucket mesh-host_acts is not declared as state for now: a declared state of a module with no account
|
||||||
|
|||||||
Reference in New Issue
Block a user