MH-0: an entry whose ask failed is asked again, at most entry-asks times, then unanswered
The node-engine consumes the entries client's asking again (H-A-60, H-A-110): the setting entry-asks (default 5; module.json config and file resource, read at the engine's start) goes to Entries.Asks; Entries.AskAgain is called at each link's start and on each catalog-revision the composer hears (the shadow's cache too); the finding entry-unanswered joins the engine's finding kinds, written with a journal line when an entry ends unanswered and cleared when it is answered or no longer wanted; the shadow's wait and the have's hold end on it instead of waiting without bound. Builds: design 47 H, H-A-60, H-A-110 Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy
This commit is contained in:
@@ -76,6 +76,7 @@ func (c *Composer) newEntries() {
|
||||
}
|
||||
c.entries = meshcatalogclient.NewEntries(c.env.Entry)
|
||||
c.entries.Now = c.now
|
||||
c.entries.Asks = c.env.EntryAsks
|
||||
c.entries.Changed = c.entryChanged
|
||||
c.keptEnt = map[string]keptEntry{}
|
||||
c.keptWants = map[string]bool{}
|
||||
@@ -153,6 +154,7 @@ func (c *Composer) entryChanged(row meshcatalogclient.EntryRow) {
|
||||
c.keep(entryKey(row.Fingerprint), meshcatalog.EntryAnswer{Entry: *row.Entry})
|
||||
}
|
||||
c.tellNotFound(row.Fingerprint, row.State == meshcatalogclient.EntryNotFound)
|
||||
c.tellUnanswered(row.Fingerprint, row.Asks, row.State == meshcatalogclient.EntryUnanswered)
|
||||
if row.State == meshcatalogclient.EntryAnswered {
|
||||
c.handAgainFor(row.Fingerprint)
|
||||
}
|
||||
@@ -211,6 +213,40 @@ func (c *Composer) tellNotFound(fp string, notFound bool) {
|
||||
}
|
||||
}
|
||||
|
||||
// tellUnanswered tells the finding entry-unanswered once when fp's entry is unanswered, with a journal line saying it, and once when
|
||||
// it no longer is (design 47 H, H-A-60).
|
||||
func (c *Composer) tellUnanswered(fp string, asks int, unanswered bool) {
|
||||
c.mu.Lock()
|
||||
was := c.unanswered[fp]
|
||||
if was == unanswered {
|
||||
c.mu.Unlock()
|
||||
return
|
||||
}
|
||||
if c.unanswered == nil {
|
||||
c.unanswered = map[string]bool{}
|
||||
}
|
||||
if unanswered {
|
||||
c.unanswered[fp] = true
|
||||
} else {
|
||||
delete(c.unanswered, fp)
|
||||
}
|
||||
c.mu.Unlock()
|
||||
if unanswered {
|
||||
c.env.Say(fmt.Sprintf("the entry of the build %s is unanswered: its ask failed to reach the catalog half's holder %d times more, so the node composes nothing new of it until it is asked again (the seat node-engine's verb entry-again)", fp, asks))
|
||||
}
|
||||
if c.env.EntryUnanswered != nil {
|
||||
c.env.EntryUnanswered(fp, asks, unanswered)
|
||||
}
|
||||
}
|
||||
|
||||
// EntriesAskAgain asks again every entry whose last ask failed in transport (meshcatalogclient.Entries.AskAgain): the node agent
|
||||
// calls it when its link to the bus opens (design 47 H, H-A-60).
|
||||
func (c *Composer) EntriesAskAgain(ctx context.Context) {
|
||||
if c.entries != nil {
|
||||
c.entries.AskAgain(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
// keep writes one value into the composition state; a state that cannot be written is said, and the value is kept again by
|
||||
// the next pass.
|
||||
func (c *Composer) keep(key string, v any) {
|
||||
@@ -318,6 +354,7 @@ func (c *Composer) syncEntries(ctx context.Context) {
|
||||
if _, named := need[row.Fingerprint]; !named && !added {
|
||||
c.entries.Release(row.Fingerprint)
|
||||
c.tellNotFound(row.Fingerprint, false)
|
||||
c.tellUnanswered(row.Fingerprint, 0, false)
|
||||
}
|
||||
}
|
||||
c.mu.Lock()
|
||||
@@ -701,6 +738,9 @@ func (c *Composer) entryHolds() map[string]string {
|
||||
case meshcatalogclient.EntryAsked:
|
||||
out[module] = fmt.Sprintf("%s the entry of the build %s is asked of the catalog half and not yet answered, so the node composes nothing new of %s",
|
||||
agent.EntryAsked, w.Fp, module)
|
||||
case meshcatalogclient.EntryUnanswered:
|
||||
out[module] = fmt.Sprintf("%s the entry of the build %s is unanswered: its ask failed to reach the catalog half as often as the setting entry-asks allows, so the node composes nothing new of %s",
|
||||
agent.EntryAsked, w.Fp, module)
|
||||
case meshcatalogclient.EntryNotFound:
|
||||
out[module] = fmt.Sprintf("%s the catalog half holds no entry of the build %s, so the node composes nothing new of %s",
|
||||
agent.EntryMissing, w.Fp, module)
|
||||
@@ -951,6 +991,8 @@ func (c *Composer) addedModule(ctx context.Context, module string) (fp string, e
|
||||
return fp, e, true, false, ""
|
||||
case meshcatalogclient.EntryNotFound:
|
||||
return fp, e, false, false, fmt.Sprintf("the catalog half holds no entry of %s's build %s", module, fp)
|
||||
case meshcatalogclient.EntryUnanswered:
|
||||
return fp, e, false, false, fmt.Sprintf("the entry of %s's build %s is unanswered: its ask failed to reach the catalog half as often as the setting entry-asks allows", module, fp)
|
||||
}
|
||||
return fp, e, false, true, fmt.Sprintf("the entry of %s's build %s is asked of the catalog half and not yet answered", module, fp)
|
||||
}
|
||||
|
||||
@@ -635,6 +635,7 @@ func TestRunningBuildEntryKeptAndHandedAgain(t *testing.T) {
|
||||
// it is bound to), never of every module placed there, and beside the pass.
|
||||
func TestOnlyWhatTheNodeNeedsOfAnotherNodesPlacementIsAsked(t *testing.T) {
|
||||
t.Run("a trunk build the catalog gave none of waits its backoff or a catalog-revision (A-487a)", trunkAskBackoffAndCatalogRevision)
|
||||
t.Run("an entry whose ask failed is asked again on each catalog-revision, at most entry-asks times, then unanswered (H-A-60, H-A-110)", entryAskedAgainThenUnanswered)
|
||||
env := newFakeBus().env()
|
||||
env.Node = "n1"
|
||||
var mu sync.Mutex
|
||||
@@ -770,3 +771,81 @@ func trunkAskBackoffAndCatalogRevision(t *testing.T) {
|
||||
t.Error("catalog-revision was heard and the wait of the module stands")
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 H, H-A-60 and H-A-110: an entry whose ask failed in transport is asked again on each catalog-revision the composer hears
|
||||
// (never on a timer), at most the setting entry-asks times; one more failure and it is unanswered, said in the journal and as the
|
||||
// finding entry-unanswered, which the shadow does not wait on; asked again by hand and answered, the finding is cleared.
|
||||
func entryAskedAgainThenUnanswered(t *testing.T) {
|
||||
fp := meshcatalogclient.SourcePrefix2 + strings.Repeat("c", 64)
|
||||
var calls atomic.Int32
|
||||
var answers atomic.Bool
|
||||
env := newFakeBus().env()
|
||||
env.EntryAsks = 2
|
||||
env.Entry = EntryCall(func(context.Context, string, string, any) (json.RawMessage, error) {
|
||||
calls.Add(1)
|
||||
if answers.Load() {
|
||||
return json.Marshal(meshcatalog.EntryAnswer{Entry: meshcatalog.CatalogEntry{Module: "web", Fp: fp}})
|
||||
}
|
||||
return nil, errors.New("the bus did not answer")
|
||||
})
|
||||
var mu sync.Mutex
|
||||
var said []string
|
||||
told := []string{}
|
||||
env.Say = func(line string) { mu.Lock(); said = append(said, line); mu.Unlock() }
|
||||
env.EntryUnanswered = func(got string, asks int, unanswered bool) {
|
||||
mu.Lock()
|
||||
told = append(told, fmt.Sprintf("%s %d %v", got, asks, unanswered))
|
||||
mu.Unlock()
|
||||
}
|
||||
heard := make(chan []byte)
|
||||
env.Hear = func(string) (<-chan []byte, func(), error) { return heard, func() {}, nil }
|
||||
c := New(env)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go c.hearCatalogRevisions(ctx, func() {})
|
||||
state := func() meshcatalogclient.EntryState { _, st := c.entries.Need(ctx, fp); return st }
|
||||
c.entries.Need(ctx, fp)
|
||||
// Each catalog-revision asks the failed entry again once: three asks in all, the third being the one past the bound of two.
|
||||
for deadline := time.Now().Add(10 * time.Second); state() != meshcatalogclient.EntryUnanswered; {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("the entry is %s after %d asks, want unanswered", state(), calls.Load())
|
||||
}
|
||||
select {
|
||||
case heard <- nil:
|
||||
case <-time.After(time.Millisecond):
|
||||
}
|
||||
}
|
||||
for i := 0; i < 3; i++ {
|
||||
heard <- nil // an unanswered entry is not asked again by an event
|
||||
}
|
||||
var gotTold, gotSaid string
|
||||
for deadline := time.Now().Add(10 * time.Second); gotTold == "[]" || gotTold == ""; runtime.Gosched() {
|
||||
mu.Lock()
|
||||
gotTold, gotSaid = fmt.Sprint(told), strings.Join(said, "\n")
|
||||
mu.Unlock()
|
||||
if time.Now().After(deadline) {
|
||||
break
|
||||
}
|
||||
}
|
||||
if calls.Load() != 3 || gotTold != "["+fp+" 2 true]" || !strings.Contains(gotSaid, "is unanswered") || !strings.Contains(gotSaid, fp) {
|
||||
t.Errorf("asked %d times, told %s, said %q; want 3 asks, the finding told once with 2 asks and the journal line", calls.Load(), gotTold, gotSaid)
|
||||
}
|
||||
// The shadow's wait on the entry ends: it is no longer pending.
|
||||
c.mu.Lock()
|
||||
c.wants = map[string]meshwants.WantsByWantNodeModuleValue{"web": {Fp: fp}}
|
||||
c.mu.Unlock()
|
||||
if c.syncShadowEntries(ctx) {
|
||||
t.Error("the shadow still waits on an unanswered entry")
|
||||
}
|
||||
// Asked again by hand and answered, the finding is cleared.
|
||||
answers.Store(true)
|
||||
if err := c.entries.Again(ctx, fp); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mu.Lock()
|
||||
gotTold = fmt.Sprint(told)
|
||||
mu.Unlock()
|
||||
if gotTold != "["+fp+" 2 true "+fp+" 0 false]" {
|
||||
t.Errorf("told %s after the answer, want the finding cleared", gotTold)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -67,6 +67,12 @@ type Env struct {
|
||||
// EntryNotFound is told when the catalog half answered it holds no entry of a fingerprint (notFound), and when that is no longer so:
|
||||
// the entry answered, or nothing the node holds names it (design 47 G §614, A-117). Nil tells nobody.
|
||||
EntryNotFound func(fp string, notFound bool)
|
||||
// EntryUnanswered is told when an ask of the catalog half's entry failed to reach the holder as often as EntryAsks allows
|
||||
// (unanswered), and when that is no longer so (design 47 H, H-A-60). Nil tells nobody.
|
||||
EntryUnanswered func(fp string, asks int, unanswered bool)
|
||||
// EntryAsks is the setting `entry-asks`: how often an entry whose ask failed in transport is asked again before it is
|
||||
// unanswered; meshcatalogclient.DefaultEntryAsks when 0 (design 47 H, H-A-60).
|
||||
EntryAsks int
|
||||
// Watch follows one bucket's keys matching pattern, as the agent's watches do.
|
||||
Watch func(ctx context.Context, bucket, pattern string) (<-chan agent.Update, error)
|
||||
// Get reads one key's latest value; nil and no error when there is none.
|
||||
@@ -168,7 +174,10 @@ type Composer struct {
|
||||
OnEngine func(module string)
|
||||
// AlsoEntryAgain is told each entry asked again by hand, so the shadow's own cache of entries asks it again too.
|
||||
AlsoEntryAgain func(ctx context.Context, fp string)
|
||||
env Env
|
||||
// AlsoEntriesAskAgain is told each catalog-revision the composer hears, so the shadow, which hears nothing itself, asks its
|
||||
// own entries again too (design 47 H, H-A-110).
|
||||
AlsoEntriesAskAgain func(ctx context.Context)
|
||||
env Env
|
||||
|
||||
mu sync.Mutex
|
||||
wants map[string]meshwants.WantsByWantNodeModuleValue // by module, the node's own under agent.NodeModule
|
||||
@@ -202,6 +211,8 @@ type Composer struct {
|
||||
keptEnt map[string]keptEntry
|
||||
// notFound is each fingerprint whose entry-not-found the composer told, so it is told once and cleared once.
|
||||
notFound map[string]bool
|
||||
// unanswered is each fingerprint whose entry-unanswered the composer told, so it is told once and cleared once.
|
||||
unanswered map[string]bool
|
||||
// resolved is each wanted build's digests as its version resolved them through the registry seats (versions.go, A-430a).
|
||||
resolved map[resolvedKey]versionParts
|
||||
keptWantVal map[string]keptWant
|
||||
@@ -706,7 +717,7 @@ func (c *Composer) Watch(ctx context.Context, _ string) (<-chan agent.Update, er
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// hearCatalogRevisions hears the seat mesh-catalog's event catalog-revision: each one clears the backoffs of the trunk builds the
|
||||
// hearCatalogRevisions hears the seat mesh-catalog's event catalog-revision: each one asks again the entries whose ask failed (H-A-110), clears the backoffs of the trunk builds the
|
||||
// catalog gave none of and wakes a pass (A-487a). A wake-up only, the backoff bounds the asks without it: a subscription refused
|
||||
// is said once with its subject and error (principle 3), the composer goes on composing, and the subscription is tried again on
|
||||
// laterAgain's steps until it holds.
|
||||
@@ -725,6 +736,11 @@ func (c *Composer) hearCatalogRevisions(ctx context.Context, wake func()) {
|
||||
return
|
||||
}
|
||||
c.trunkRetryCleared()
|
||||
// And each entry whose ask failed in transport is asked again, once per take-in (design 47 H, H-A-110).
|
||||
c.EntriesAskAgain(ctx)
|
||||
if c.AlsoEntriesAskAgain != nil {
|
||||
c.AlsoEntriesAskAgain(ctx)
|
||||
}
|
||||
wake()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -826,6 +826,9 @@ func (c *Composer) sameValue(a, b string) (same, known bool) {
|
||||
return bytes.Equal(x, y), true
|
||||
}
|
||||
|
||||
// EntriesAskAgain asks again every entry of the shadow's own cache whose last ask failed in transport (design 47 H, H-A-60).
|
||||
func (s *Shadow) EntriesAskAgain(ctx context.Context) { s.c.EntriesAskAgain(ctx) }
|
||||
|
||||
// EntryAgain asks fp's entry of the catalog half again in the shadow's own cache, where the node's own composer was asked
|
||||
// again by hand: an entry the shadow holds as not found stays so until it is asked (a build the half had not yet taken in).
|
||||
func (s *Shadow) EntryAgain(ctx context.Context, fp string) {
|
||||
|
||||
Reference in New Issue
Block a user