MH-0: an entry whose ask failed is asked again, at most entry-asks times, then unanswered
The node-engine consumes the entries client's asking again (H-A-60, H-A-110): the setting entry-asks (default 5; module.json config and file resource, read at the engine's start) goes to Entries.Asks; Entries.AskAgain is called at each link's start and on each catalog-revision the composer hears (the shadow's cache too); the finding entry-unanswered joins the engine's finding kinds, written with a journal line when an entry ends unanswered and cleared when it is answered or no longer wanted; the shadow's wait and the have's hold end on it instead of waiting without bound. Builds: design 47 H, H-A-60, H-A-110 Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy
This commit is contained in:
@@ -13,6 +13,10 @@ import (
|
||||
// running for (design 47 G §614, A-15, A-117): scope module, its id the fingerprint escaped as a name.
|
||||
const EntryNotFoundKind = "entry-not-found"
|
||||
|
||||
// EntryUnansweredKind is the finding a part writes when its ask of the catalog half's entry of a fingerprint failed to reach the
|
||||
// holder as often as the setting `entry-asks` allows (design 47 H, H-A-60): scope module, its id the fingerprint escaped as a name.
|
||||
const EntryUnansweredKind = "entry-unanswered"
|
||||
|
||||
// EntryNotFoundID is the finding's id for a fingerprint.
|
||||
func EntryNotFoundID(fp string) string { return keys.EscapeName(fp) }
|
||||
|
||||
@@ -31,3 +35,20 @@ func SayEntryNotFound(ctx context.Context, w Writer, node, observer, fp string,
|
||||
Explanation: "A want or a running names this build and the catalog half answered that it holds no entry of it. The finding clears when the entry is answered or nothing the node holds names the build.",
|
||||
At: now()})
|
||||
}
|
||||
|
||||
// SayEntryUnanswered writes the finding for fp when unanswered, and clears it when no longer: the entry answered later, asked
|
||||
// again by hand, or nothing the node holds names the fingerprint any more (design 47 H, H-A-60). A write or a clear that fails is
|
||||
// returned for the caller to say.
|
||||
func SayEntryUnanswered(ctx context.Context, w Writer, node, observer, fp string, asks int, unanswered bool, now func() time.Time) error {
|
||||
id := EntryNotFoundID(fp)
|
||||
if !unanswered {
|
||||
return w.Clear(ctx, scopeModule, id, EntryUnansweredKind, "the entry is answered, or nothing the node holds names its fingerprint")
|
||||
}
|
||||
return w.Write(ctx, condseat.Finding{Observer: observer, Kind: EntryUnansweredKind, Scope: scopeModule, ID: id,
|
||||
Severity: condseat.FindingSeverityWarning, Source: EntryUnansweredKind,
|
||||
Subject: condseat.Subject{Scope: scopeModule, ID: observer, Machine: node},
|
||||
Summary: fmt.Sprintf("the ask of the catalog half's entry of the build %s failed to reach the holder %d times, so %s composes nothing new of it", fp, asks, node),
|
||||
Headline: fmt.Sprintf("the entry of the build %s is unanswered on %s", fp, node),
|
||||
Explanation: "The ask failed in transport and was asked again as often as the setting entry-asks allows. The finding clears when the entry is answered, asked again by hand (the seat node-engine's verb entry-again) or nothing the node holds names the build.",
|
||||
At: now()})
|
||||
}
|
||||
|
||||
@@ -230,6 +230,17 @@ func caseAnEntryNotFoundIsAFindingOfTheFingerprint(t *testing.T) {
|
||||
if err := SayEntryNotFound(context.Background(), w, "n1", "mesh-host", fp, false, now); err != nil || len(w.cleared) != 1 || w.cleared[0].ID != w.put[0].ID {
|
||||
t.Fatalf("cleared %+v, %v", w.cleared, err)
|
||||
}
|
||||
// And design 47 H, H-A-60: an entry unanswered is the finding entry-unanswered, of the same scope and id, cleared by the same.
|
||||
u := &fakeFindings{}
|
||||
if err := SayEntryUnanswered(context.Background(), u, "n1", "mesh-host", fp, 5, true, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(u.put) != 1 || u.put[0].Kind != EntryUnansweredKind || u.put[0].Scope != "module" || u.put[0].ID != keys.EscapeName(fp) {
|
||||
t.Fatalf("written %+v", u.put)
|
||||
}
|
||||
if err := SayEntryUnanswered(context.Background(), u, "n1", "mesh-host", fp, 0, false, now); err != nil || len(u.cleared) != 1 || u.cleared[0].Kind != EntryUnansweredKind || u.cleared[0].ID != u.put[0].ID {
|
||||
t.Fatalf("cleared %+v, %v", u.cleared, err)
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0636 §1, the node's finding of a seat list unread.
|
||||
|
||||
@@ -76,6 +76,7 @@ func (c *Composer) newEntries() {
|
||||
}
|
||||
c.entries = meshcatalogclient.NewEntries(c.env.Entry)
|
||||
c.entries.Now = c.now
|
||||
c.entries.Asks = c.env.EntryAsks
|
||||
c.entries.Changed = c.entryChanged
|
||||
c.keptEnt = map[string]keptEntry{}
|
||||
c.keptWants = map[string]bool{}
|
||||
@@ -153,6 +154,7 @@ func (c *Composer) entryChanged(row meshcatalogclient.EntryRow) {
|
||||
c.keep(entryKey(row.Fingerprint), meshcatalog.EntryAnswer{Entry: *row.Entry})
|
||||
}
|
||||
c.tellNotFound(row.Fingerprint, row.State == meshcatalogclient.EntryNotFound)
|
||||
c.tellUnanswered(row.Fingerprint, row.Asks, row.State == meshcatalogclient.EntryUnanswered)
|
||||
if row.State == meshcatalogclient.EntryAnswered {
|
||||
c.handAgainFor(row.Fingerprint)
|
||||
}
|
||||
@@ -211,6 +213,40 @@ func (c *Composer) tellNotFound(fp string, notFound bool) {
|
||||
}
|
||||
}
|
||||
|
||||
// tellUnanswered tells the finding entry-unanswered once when fp's entry is unanswered, with a journal line saying it, and once when
|
||||
// it no longer is (design 47 H, H-A-60).
|
||||
func (c *Composer) tellUnanswered(fp string, asks int, unanswered bool) {
|
||||
c.mu.Lock()
|
||||
was := c.unanswered[fp]
|
||||
if was == unanswered {
|
||||
c.mu.Unlock()
|
||||
return
|
||||
}
|
||||
if c.unanswered == nil {
|
||||
c.unanswered = map[string]bool{}
|
||||
}
|
||||
if unanswered {
|
||||
c.unanswered[fp] = true
|
||||
} else {
|
||||
delete(c.unanswered, fp)
|
||||
}
|
||||
c.mu.Unlock()
|
||||
if unanswered {
|
||||
c.env.Say(fmt.Sprintf("the entry of the build %s is unanswered: its ask failed to reach the catalog half's holder %d times more, so the node composes nothing new of it until it is asked again (the seat node-engine's verb entry-again)", fp, asks))
|
||||
}
|
||||
if c.env.EntryUnanswered != nil {
|
||||
c.env.EntryUnanswered(fp, asks, unanswered)
|
||||
}
|
||||
}
|
||||
|
||||
// EntriesAskAgain asks again every entry whose last ask failed in transport (meshcatalogclient.Entries.AskAgain): the node agent
|
||||
// calls it when its link to the bus opens (design 47 H, H-A-60).
|
||||
func (c *Composer) EntriesAskAgain(ctx context.Context) {
|
||||
if c.entries != nil {
|
||||
c.entries.AskAgain(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
// keep writes one value into the composition state; a state that cannot be written is said, and the value is kept again by
|
||||
// the next pass.
|
||||
func (c *Composer) keep(key string, v any) {
|
||||
@@ -318,6 +354,7 @@ func (c *Composer) syncEntries(ctx context.Context) {
|
||||
if _, named := need[row.Fingerprint]; !named && !added {
|
||||
c.entries.Release(row.Fingerprint)
|
||||
c.tellNotFound(row.Fingerprint, false)
|
||||
c.tellUnanswered(row.Fingerprint, 0, false)
|
||||
}
|
||||
}
|
||||
c.mu.Lock()
|
||||
@@ -701,6 +738,9 @@ func (c *Composer) entryHolds() map[string]string {
|
||||
case meshcatalogclient.EntryAsked:
|
||||
out[module] = fmt.Sprintf("%s the entry of the build %s is asked of the catalog half and not yet answered, so the node composes nothing new of %s",
|
||||
agent.EntryAsked, w.Fp, module)
|
||||
case meshcatalogclient.EntryUnanswered:
|
||||
out[module] = fmt.Sprintf("%s the entry of the build %s is unanswered: its ask failed to reach the catalog half as often as the setting entry-asks allows, so the node composes nothing new of %s",
|
||||
agent.EntryAsked, w.Fp, module)
|
||||
case meshcatalogclient.EntryNotFound:
|
||||
out[module] = fmt.Sprintf("%s the catalog half holds no entry of the build %s, so the node composes nothing new of %s",
|
||||
agent.EntryMissing, w.Fp, module)
|
||||
@@ -951,6 +991,8 @@ func (c *Composer) addedModule(ctx context.Context, module string) (fp string, e
|
||||
return fp, e, true, false, ""
|
||||
case meshcatalogclient.EntryNotFound:
|
||||
return fp, e, false, false, fmt.Sprintf("the catalog half holds no entry of %s's build %s", module, fp)
|
||||
case meshcatalogclient.EntryUnanswered:
|
||||
return fp, e, false, false, fmt.Sprintf("the entry of %s's build %s is unanswered: its ask failed to reach the catalog half as often as the setting entry-asks allows", module, fp)
|
||||
}
|
||||
return fp, e, false, true, fmt.Sprintf("the entry of %s's build %s is asked of the catalog half and not yet answered", module, fp)
|
||||
}
|
||||
|
||||
@@ -635,6 +635,7 @@ func TestRunningBuildEntryKeptAndHandedAgain(t *testing.T) {
|
||||
// it is bound to), never of every module placed there, and beside the pass.
|
||||
func TestOnlyWhatTheNodeNeedsOfAnotherNodesPlacementIsAsked(t *testing.T) {
|
||||
t.Run("a trunk build the catalog gave none of waits its backoff or a catalog-revision (A-487a)", trunkAskBackoffAndCatalogRevision)
|
||||
t.Run("an entry whose ask failed is asked again on each catalog-revision, at most entry-asks times, then unanswered (H-A-60, H-A-110)", entryAskedAgainThenUnanswered)
|
||||
env := newFakeBus().env()
|
||||
env.Node = "n1"
|
||||
var mu sync.Mutex
|
||||
@@ -770,3 +771,81 @@ func trunkAskBackoffAndCatalogRevision(t *testing.T) {
|
||||
t.Error("catalog-revision was heard and the wait of the module stands")
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 H, H-A-60 and H-A-110: an entry whose ask failed in transport is asked again on each catalog-revision the composer hears
|
||||
// (never on a timer), at most the setting entry-asks times; one more failure and it is unanswered, said in the journal and as the
|
||||
// finding entry-unanswered, which the shadow does not wait on; asked again by hand and answered, the finding is cleared.
|
||||
func entryAskedAgainThenUnanswered(t *testing.T) {
|
||||
fp := meshcatalogclient.SourcePrefix2 + strings.Repeat("c", 64)
|
||||
var calls atomic.Int32
|
||||
var answers atomic.Bool
|
||||
env := newFakeBus().env()
|
||||
env.EntryAsks = 2
|
||||
env.Entry = EntryCall(func(context.Context, string, string, any) (json.RawMessage, error) {
|
||||
calls.Add(1)
|
||||
if answers.Load() {
|
||||
return json.Marshal(meshcatalog.EntryAnswer{Entry: meshcatalog.CatalogEntry{Module: "web", Fp: fp}})
|
||||
}
|
||||
return nil, errors.New("the bus did not answer")
|
||||
})
|
||||
var mu sync.Mutex
|
||||
var said []string
|
||||
told := []string{}
|
||||
env.Say = func(line string) { mu.Lock(); said = append(said, line); mu.Unlock() }
|
||||
env.EntryUnanswered = func(got string, asks int, unanswered bool) {
|
||||
mu.Lock()
|
||||
told = append(told, fmt.Sprintf("%s %d %v", got, asks, unanswered))
|
||||
mu.Unlock()
|
||||
}
|
||||
heard := make(chan []byte)
|
||||
env.Hear = func(string) (<-chan []byte, func(), error) { return heard, func() {}, nil }
|
||||
c := New(env)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go c.hearCatalogRevisions(ctx, func() {})
|
||||
state := func() meshcatalogclient.EntryState { _, st := c.entries.Need(ctx, fp); return st }
|
||||
c.entries.Need(ctx, fp)
|
||||
// Each catalog-revision asks the failed entry again once: three asks in all, the third being the one past the bound of two.
|
||||
for deadline := time.Now().Add(10 * time.Second); state() != meshcatalogclient.EntryUnanswered; {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("the entry is %s after %d asks, want unanswered", state(), calls.Load())
|
||||
}
|
||||
select {
|
||||
case heard <- nil:
|
||||
case <-time.After(time.Millisecond):
|
||||
}
|
||||
}
|
||||
for i := 0; i < 3; i++ {
|
||||
heard <- nil // an unanswered entry is not asked again by an event
|
||||
}
|
||||
var gotTold, gotSaid string
|
||||
for deadline := time.Now().Add(10 * time.Second); gotTold == "[]" || gotTold == ""; runtime.Gosched() {
|
||||
mu.Lock()
|
||||
gotTold, gotSaid = fmt.Sprint(told), strings.Join(said, "\n")
|
||||
mu.Unlock()
|
||||
if time.Now().After(deadline) {
|
||||
break
|
||||
}
|
||||
}
|
||||
if calls.Load() != 3 || gotTold != "["+fp+" 2 true]" || !strings.Contains(gotSaid, "is unanswered") || !strings.Contains(gotSaid, fp) {
|
||||
t.Errorf("asked %d times, told %s, said %q; want 3 asks, the finding told once with 2 asks and the journal line", calls.Load(), gotTold, gotSaid)
|
||||
}
|
||||
// The shadow's wait on the entry ends: it is no longer pending.
|
||||
c.mu.Lock()
|
||||
c.wants = map[string]meshwants.WantsByWantNodeModuleValue{"web": {Fp: fp}}
|
||||
c.mu.Unlock()
|
||||
if c.syncShadowEntries(ctx) {
|
||||
t.Error("the shadow still waits on an unanswered entry")
|
||||
}
|
||||
// Asked again by hand and answered, the finding is cleared.
|
||||
answers.Store(true)
|
||||
if err := c.entries.Again(ctx, fp); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mu.Lock()
|
||||
gotTold = fmt.Sprint(told)
|
||||
mu.Unlock()
|
||||
if gotTold != "["+fp+" 2 true "+fp+" 0 false]" {
|
||||
t.Errorf("told %s after the answer, want the finding cleared", gotTold)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -67,6 +67,12 @@ type Env struct {
|
||||
// EntryNotFound is told when the catalog half answered it holds no entry of a fingerprint (notFound), and when that is no longer so:
|
||||
// the entry answered, or nothing the node holds names it (design 47 G §614, A-117). Nil tells nobody.
|
||||
EntryNotFound func(fp string, notFound bool)
|
||||
// EntryUnanswered is told when an ask of the catalog half's entry failed to reach the holder as often as EntryAsks allows
|
||||
// (unanswered), and when that is no longer so (design 47 H, H-A-60). Nil tells nobody.
|
||||
EntryUnanswered func(fp string, asks int, unanswered bool)
|
||||
// EntryAsks is the setting `entry-asks`: how often an entry whose ask failed in transport is asked again before it is
|
||||
// unanswered; meshcatalogclient.DefaultEntryAsks when 0 (design 47 H, H-A-60).
|
||||
EntryAsks int
|
||||
// Watch follows one bucket's keys matching pattern, as the agent's watches do.
|
||||
Watch func(ctx context.Context, bucket, pattern string) (<-chan agent.Update, error)
|
||||
// Get reads one key's latest value; nil and no error when there is none.
|
||||
@@ -168,7 +174,10 @@ type Composer struct {
|
||||
OnEngine func(module string)
|
||||
// AlsoEntryAgain is told each entry asked again by hand, so the shadow's own cache of entries asks it again too.
|
||||
AlsoEntryAgain func(ctx context.Context, fp string)
|
||||
env Env
|
||||
// AlsoEntriesAskAgain is told each catalog-revision the composer hears, so the shadow, which hears nothing itself, asks its
|
||||
// own entries again too (design 47 H, H-A-110).
|
||||
AlsoEntriesAskAgain func(ctx context.Context)
|
||||
env Env
|
||||
|
||||
mu sync.Mutex
|
||||
wants map[string]meshwants.WantsByWantNodeModuleValue // by module, the node's own under agent.NodeModule
|
||||
@@ -202,6 +211,8 @@ type Composer struct {
|
||||
keptEnt map[string]keptEntry
|
||||
// notFound is each fingerprint whose entry-not-found the composer told, so it is told once and cleared once.
|
||||
notFound map[string]bool
|
||||
// unanswered is each fingerprint whose entry-unanswered the composer told, so it is told once and cleared once.
|
||||
unanswered map[string]bool
|
||||
// resolved is each wanted build's digests as its version resolved them through the registry seats (versions.go, A-430a).
|
||||
resolved map[resolvedKey]versionParts
|
||||
keptWantVal map[string]keptWant
|
||||
@@ -706,7 +717,7 @@ func (c *Composer) Watch(ctx context.Context, _ string) (<-chan agent.Update, er
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// hearCatalogRevisions hears the seat mesh-catalog's event catalog-revision: each one clears the backoffs of the trunk builds the
|
||||
// hearCatalogRevisions hears the seat mesh-catalog's event catalog-revision: each one asks again the entries whose ask failed (H-A-110), clears the backoffs of the trunk builds the
|
||||
// catalog gave none of and wakes a pass (A-487a). A wake-up only, the backoff bounds the asks without it: a subscription refused
|
||||
// is said once with its subject and error (principle 3), the composer goes on composing, and the subscription is tried again on
|
||||
// laterAgain's steps until it holds.
|
||||
@@ -725,6 +736,11 @@ func (c *Composer) hearCatalogRevisions(ctx context.Context, wake func()) {
|
||||
return
|
||||
}
|
||||
c.trunkRetryCleared()
|
||||
// And each entry whose ask failed in transport is asked again, once per take-in (design 47 H, H-A-110).
|
||||
c.EntriesAskAgain(ctx)
|
||||
if c.AlsoEntriesAskAgain != nil {
|
||||
c.AlsoEntriesAskAgain(ctx)
|
||||
}
|
||||
wake()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -826,6 +826,9 @@ func (c *Composer) sameValue(a, b string) (same, known bool) {
|
||||
return bytes.Equal(x, y), true
|
||||
}
|
||||
|
||||
// EntriesAskAgain asks again every entry of the shadow's own cache whose last ask failed in transport (design 47 H, H-A-60).
|
||||
func (s *Shadow) EntriesAskAgain(ctx context.Context) { s.c.EntriesAskAgain(ctx) }
|
||||
|
||||
// EntryAgain asks fp's entry of the catalog half again in the shadow's own cache, where the node's own composer was asked
|
||||
// again by hand: an entry the shadow holds as not found stays so until it is asked (a build the half had not yet taken in).
|
||||
func (s *Shadow) EntryAgain(ctx context.Context, fp string) {
|
||||
|
||||
Reference in New Issue
Block a user