Design 47 H1: the node-engine composes its wants from the bus (want fields, version, apply-order) #239
@@ -627,6 +627,11 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus
|
||||
say("the finding that the catalog half holds no entry of " + fp + " cannot be kept: " + err.Error())
|
||||
}
|
||||
},
|
||||
entryUnanswered: func(fp string, asks int, unanswered bool) {
|
||||
if err := seatwait.SayEntryUnanswered(run, findings, node, findings.observer(), fp, asks, unanswered, time.Now); err != nil {
|
||||
say("the finding that the entry of " + fp + " is unanswered cannot be kept: " + err.Error())
|
||||
}
|
||||
},
|
||||
seatList: seats,
|
||||
seatListFailed: err != nil,
|
||||
running: func() []compositions.Composed {
|
||||
@@ -690,10 +695,14 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus
|
||||
// nothing and keeps running what it runs; a assignment it cannot watch stops the agent, said, and it begins again.
|
||||
composer := selfComposer(session, node, hooks, say)
|
||||
composer.AlsoEntryAgain = shadow.EntryAgain
|
||||
composer.AlsoEntriesAskAgain = shadow.EntriesAskAgain
|
||||
composer.OnEngine = session.Own.Set // the first composing names the engine's own module on a first start (§527, §634)
|
||||
machinesNow.Store(composer) // the machines a failing network part points at, by the assignments (issue 774)
|
||||
composerNow.Store(composer) // what the verb compose asks a pass of, this run's alone
|
||||
session.SetManifestOf(composer.ManifestOf) // the event running names the manifest of the build that runs
|
||||
// The link is open: each entry whose ask failed in transport is asked again (design 47 H, H-A-60).
|
||||
composer.EntriesAskAgain(run)
|
||||
shadow.EntriesAskAgain(run)
|
||||
go followRunning(run, session, composer, say)
|
||||
go followBuildEnds(run, session, composer, say)
|
||||
apply.UseEngineModule(composer.EngineModule) // the guard of the engine's own trees fixes no name (§527)
|
||||
@@ -1132,7 +1141,9 @@ type composerHooks struct {
|
||||
kept func(kept map[string][]string)
|
||||
// entryNotFound is selfcompose.Env's EntryNotFound.
|
||||
entryNotFound func(fp string, notFound bool)
|
||||
held func(module string) (compositions.Seals, []string, error)
|
||||
// entryUnanswered is selfcompose.Env's EntryUnanswered.
|
||||
entryUnanswered func(fp string, asks int, unanswered bool)
|
||||
held func(module string) (compositions.Seals, []string, error)
|
||||
// vaultAuthority and certify are selfcompose.Env's VaultAuthority and Certify.
|
||||
vaultAuthority func() string
|
||||
certify func(ctx context.Context) error
|
||||
@@ -1191,7 +1202,7 @@ func composerEnv(session *link.Session, node string, say func(string), hooks com
|
||||
Hear: session.Hear,
|
||||
// And a moved module's templates archive, in the artifact store, as every archive is fetched (issue 1126).
|
||||
Fetch: apply.Fetch,
|
||||
Running: hooks.running, Unseal: hooks.unseal, Refused: hooks.refused, Held: hooks.held, Kept: hooks.kept, EntryNotFound: hooks.entryNotFound,
|
||||
Running: hooks.running, Unseal: hooks.unseal, Refused: hooks.refused, Held: hooks.held, Kept: hooks.kept, EntryNotFound: hooks.entryNotFound, EntryUnanswered: hooks.entryUnanswered, EntryAsks: entryAsks(),
|
||||
VaultAuthority: hooks.vaultAuthority, Certify: hooks.certify}
|
||||
if hooks.seatList != nil {
|
||||
env.SeatList, env.SeatListChanged = hooks.seatList.reader, hooks.seatList.changed
|
||||
|
||||
@@ -79,6 +79,9 @@ type itBed struct {
|
||||
// holder re-serves (issue 1966).
|
||||
seatAway atomic.Bool
|
||||
findings nats.KeyValue
|
||||
// entryFails makes the stand-in answer the verb entry with a refusal in words the holder never gives for an unknown fingerprint:
|
||||
// the ask fails to reach the holder's answer, as one on a bus that drops it does (design 47 H, H-A-60).
|
||||
entryFails atomic.Bool
|
||||
// entries are the entries the stand-in of the seat mesh-catalog answers its verb entry from, by fingerprint (design 47 G §421).
|
||||
entries map[string]meshcatalog.CatalogEntry
|
||||
}
|
||||
@@ -117,6 +120,10 @@ func (b *itBed) standInEntry() {
|
||||
b.mu.Lock()
|
||||
e, ok := b.entries[in.Fingerprint]
|
||||
b.mu.Unlock()
|
||||
if b.entryFails.Load() {
|
||||
_ = m.Respond([]byte(`{"error":"the holder cannot answer now"}`))
|
||||
return
|
||||
}
|
||||
if !ok {
|
||||
_ = m.Respond([]byte(`{"error":"no entry of fingerprint ` + in.Fingerprint + ` is kept"}`))
|
||||
return
|
||||
@@ -334,6 +341,42 @@ func TestNatsANodeTheAssignmentsNameComposesAndAppliesItsOwn(t *testing.T) {
|
||||
t.Errorf("composed and applied under a grant without the served key: %q", b.file("web"))
|
||||
}
|
||||
})
|
||||
// Design 47 H, H-A-60 and H-A-110: an entry whose ask failed is asked again on each catalog-revision, at most entry-asks times; the
|
||||
// shadow's wait on it and the have's end `unanswered`, loudly (the journal and the finding entry-unanswered), instead of waiting
|
||||
// without bound; asked again by hand once the holder answers, the node composes.
|
||||
t.Run("an entry whose ask failed is asked again on each catalog-revision and ends unanswered", func(t *testing.T) {
|
||||
// The golden grant does not list the subscription to catalog-revision, which the manifest consumes: laid over it here.
|
||||
granted := nodegrant.For(itNode, "it2.mailer.domain")
|
||||
granted.Subscribe = append(granted.Subscribe, meshcatalog.SubjectCatalogRevisionAll)
|
||||
b := newItBedAs(t, granted)
|
||||
b.entryFails.Store(true)
|
||||
stop, st := h8(t, b, false)
|
||||
defer stop()
|
||||
revision := func() bool {
|
||||
if err := b.conn.Publish(meshcatalog.SubjectCatalogRevision("main"), []byte(`{}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h := b.have("web")
|
||||
return h.Phase == nec.PhaseHeld && strings.HasPrefix(h.Detail, agent.EntryAsked) && strings.Contains(h.Detail, "is unanswered")
|
||||
}
|
||||
within(t, 20*time.Second, "web's entry did not end unanswered after the asks the setting allows", revision)
|
||||
within(t, 5*time.Second, "the journal does not say the entry is unanswered", func() bool {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
return strings.Contains(strings.Join(b.said, "\n"), "is unanswered")
|
||||
})
|
||||
if b.file("web") != "" {
|
||||
t.Fatal("composed with an entry nobody answered")
|
||||
}
|
||||
b.entryFails.Store(false)
|
||||
if err := composerNow.Load().EntryAgain(context.Background(), st.fp); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := st.resolved.Put(itNode+".web.greeting", []byte(`{"value":"bonjour","source":"module"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
within(t, 5*time.Second, "the node did not compose once the entry was asked again and answered", func() bool { return b.file("web") == "greeting=bonjour\n" })
|
||||
})
|
||||
b := newItBedAs(t, nodegrant.For(itNode, "it2.mailer.domain"))
|
||||
stop, st := h8(t, b, true)
|
||||
defer stop()
|
||||
|
||||
@@ -56,6 +56,26 @@ func TestTheEnginesOwnNameIsReadFromItsAppliedRecordOrItsFirstComposing(t *testi
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 H, H-A-60: how often an entry whose ask failed is asked again is the file the manifest's `entry-asks` feeds, five when it
|
||||
// holds nothing usable.
|
||||
func TestTheEntryAsksAreTheManifestsSetting(t *testing.T) {
|
||||
was := entryAsksFile
|
||||
defer func() { entryAsksFile = was }()
|
||||
for content, want := range map[string]int{"2\n": 2, "": 5, "0": 5, "x": 5} {
|
||||
entryAsksFile = filepath.Join(t.TempDir(), "w")
|
||||
if err := os.WriteFile(entryAsksFile, []byte(content), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := entryAsks(); got != want {
|
||||
t.Errorf("the file %q gives %v, want %v", content, got, want)
|
||||
}
|
||||
}
|
||||
entryAsksFile = filepath.Join(t.TempDir(), "missing")
|
||||
if got := entryAsks(); got != 5 {
|
||||
t.Errorf("no file gives %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// §633: the engine's bound for the first seat list is the file the manifest's `view-waits-minutes` feeds, five minutes
|
||||
// when it holds nothing usable.
|
||||
func TestTheViewWaitsBoundIsTheManifestsSetting(t *testing.T) {
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -124,7 +125,7 @@ func newFindingsOfTheNode(session *link.Session, node, module string, say func(s
|
||||
Module: module,
|
||||
Node: node,
|
||||
State: link.FindingsState{Session: session},
|
||||
Kinds: append([]string{seatwait.Kind, seatwait.HeldKind, seatwait.EntryNotFoundKind}, seatwait.NodeKinds...),
|
||||
Kinds: append([]string{seatwait.Kind, seatwait.HeldKind, seatwait.EntryNotFoundKind, seatwait.EntryUnansweredKind}, seatwait.NodeKinds...),
|
||||
Asked: condseat.SubjectFindingsAsked,
|
||||
Logf: func(format string, args ...any) { say(fmt.Sprintf(format, args...)) },
|
||||
Publish: session.PublishEvent,
|
||||
@@ -279,3 +280,21 @@ func viewWaitsBound() time.Duration {
|
||||
}
|
||||
return time.Duration(n) * time.Minute
|
||||
}
|
||||
|
||||
// entryAsksFile is the file the manifest declares with the content ${config:entry-asks} (design 47 H, H-A-60), read at the
|
||||
// engine's start as recoveryWaitFile is.
|
||||
var entryAsksFile = "/var/lib/mesh-host/entry-asks"
|
||||
|
||||
// entryAsks is how often an entry whose ask failed in transport is asked again before it is unanswered: the file's number, the
|
||||
// design's five (meshcatalogclient.DefaultEntryAsks) when the file is missing, empty or not a positive number.
|
||||
func entryAsks() int {
|
||||
raw, err := os.ReadFile(entryAsksFile)
|
||||
if err != nil {
|
||||
return meshcatalogclient.DefaultEntryAsks
|
||||
}
|
||||
n, err := strconv.Atoi(strings.TrimSpace(string(raw)))
|
||||
if err != nil || n <= 0 {
|
||||
return meshcatalogclient.DefaultEntryAsks
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
@@ -13,6 +13,10 @@ import (
|
||||
// running for (design 47 G §614, A-15, A-117): scope module, its id the fingerprint escaped as a name.
|
||||
const EntryNotFoundKind = "entry-not-found"
|
||||
|
||||
// EntryUnansweredKind is the finding a part writes when its ask of the catalog half's entry of a fingerprint failed to reach the
|
||||
// holder as often as the setting `entry-asks` allows (design 47 H, H-A-60): scope module, its id the fingerprint escaped as a name.
|
||||
const EntryUnansweredKind = "entry-unanswered"
|
||||
|
||||
// EntryNotFoundID is the finding's id for a fingerprint.
|
||||
func EntryNotFoundID(fp string) string { return keys.EscapeName(fp) }
|
||||
|
||||
@@ -31,3 +35,20 @@ func SayEntryNotFound(ctx context.Context, w Writer, node, observer, fp string,
|
||||
Explanation: "A want or a running names this build and the catalog half answered that it holds no entry of it. The finding clears when the entry is answered or nothing the node holds names the build.",
|
||||
At: now()})
|
||||
}
|
||||
|
||||
// SayEntryUnanswered writes the finding for fp when unanswered, and clears it when no longer: the entry answered later, asked
|
||||
// again by hand, or nothing the node holds names the fingerprint any more (design 47 H, H-A-60). A write or a clear that fails is
|
||||
// returned for the caller to say.
|
||||
func SayEntryUnanswered(ctx context.Context, w Writer, node, observer, fp string, asks int, unanswered bool, now func() time.Time) error {
|
||||
id := EntryNotFoundID(fp)
|
||||
if !unanswered {
|
||||
return w.Clear(ctx, scopeModule, id, EntryUnansweredKind, "the entry is answered, or nothing the node holds names its fingerprint")
|
||||
}
|
||||
return w.Write(ctx, condseat.Finding{Observer: observer, Kind: EntryUnansweredKind, Scope: scopeModule, ID: id,
|
||||
Severity: condseat.FindingSeverityWarning, Source: EntryUnansweredKind,
|
||||
Subject: condseat.Subject{Scope: scopeModule, ID: observer, Machine: node},
|
||||
Summary: fmt.Sprintf("the ask of the catalog half's entry of the build %s failed to reach the holder %d times, so %s composes nothing new of it", fp, asks, node),
|
||||
Headline: fmt.Sprintf("the entry of the build %s is unanswered on %s", fp, node),
|
||||
Explanation: "The ask failed in transport and was asked again as often as the setting entry-asks allows. The finding clears when the entry is answered, asked again by hand (the seat node-engine's verb entry-again) or nothing the node holds names the build.",
|
||||
At: now()})
|
||||
}
|
||||
|
||||
@@ -230,6 +230,17 @@ func caseAnEntryNotFoundIsAFindingOfTheFingerprint(t *testing.T) {
|
||||
if err := SayEntryNotFound(context.Background(), w, "n1", "mesh-host", fp, false, now); err != nil || len(w.cleared) != 1 || w.cleared[0].ID != w.put[0].ID {
|
||||
t.Fatalf("cleared %+v, %v", w.cleared, err)
|
||||
}
|
||||
// And design 47 H, H-A-60: an entry unanswered is the finding entry-unanswered, of the same scope and id, cleared by the same.
|
||||
u := &fakeFindings{}
|
||||
if err := SayEntryUnanswered(context.Background(), u, "n1", "mesh-host", fp, 5, true, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(u.put) != 1 || u.put[0].Kind != EntryUnansweredKind || u.put[0].Scope != "module" || u.put[0].ID != keys.EscapeName(fp) {
|
||||
t.Fatalf("written %+v", u.put)
|
||||
}
|
||||
if err := SayEntryUnanswered(context.Background(), u, "n1", "mesh-host", fp, 0, false, now); err != nil || len(u.cleared) != 1 || u.cleared[0].Kind != EntryUnansweredKind || u.cleared[0].ID != u.put[0].ID {
|
||||
t.Fatalf("cleared %+v, %v", u.cleared, err)
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0636 §1, the node's finding of a seat list unread.
|
||||
|
||||
@@ -76,6 +76,7 @@ func (c *Composer) newEntries() {
|
||||
}
|
||||
c.entries = meshcatalogclient.NewEntries(c.env.Entry)
|
||||
c.entries.Now = c.now
|
||||
c.entries.Asks = c.env.EntryAsks
|
||||
c.entries.Changed = c.entryChanged
|
||||
c.keptEnt = map[string]keptEntry{}
|
||||
c.keptWants = map[string]bool{}
|
||||
@@ -153,6 +154,7 @@ func (c *Composer) entryChanged(row meshcatalogclient.EntryRow) {
|
||||
c.keep(entryKey(row.Fingerprint), meshcatalog.EntryAnswer{Entry: *row.Entry})
|
||||
}
|
||||
c.tellNotFound(row.Fingerprint, row.State == meshcatalogclient.EntryNotFound)
|
||||
c.tellUnanswered(row.Fingerprint, row.Asks, row.State == meshcatalogclient.EntryUnanswered)
|
||||
if row.State == meshcatalogclient.EntryAnswered {
|
||||
c.handAgainFor(row.Fingerprint)
|
||||
}
|
||||
@@ -211,6 +213,40 @@ func (c *Composer) tellNotFound(fp string, notFound bool) {
|
||||
}
|
||||
}
|
||||
|
||||
// tellUnanswered tells the finding entry-unanswered once when fp's entry is unanswered, with a journal line saying it, and once when
|
||||
// it no longer is (design 47 H, H-A-60).
|
||||
func (c *Composer) tellUnanswered(fp string, asks int, unanswered bool) {
|
||||
c.mu.Lock()
|
||||
was := c.unanswered[fp]
|
||||
if was == unanswered {
|
||||
c.mu.Unlock()
|
||||
return
|
||||
}
|
||||
if c.unanswered == nil {
|
||||
c.unanswered = map[string]bool{}
|
||||
}
|
||||
if unanswered {
|
||||
c.unanswered[fp] = true
|
||||
} else {
|
||||
delete(c.unanswered, fp)
|
||||
}
|
||||
c.mu.Unlock()
|
||||
if unanswered {
|
||||
c.env.Say(fmt.Sprintf("the entry of the build %s is unanswered: its ask failed to reach the catalog half's holder %d times more, so the node composes nothing new of it until it is asked again (the seat node-engine's verb entry-again)", fp, asks))
|
||||
}
|
||||
if c.env.EntryUnanswered != nil {
|
||||
c.env.EntryUnanswered(fp, asks, unanswered)
|
||||
}
|
||||
}
|
||||
|
||||
// EntriesAskAgain asks again every entry whose last ask failed in transport (meshcatalogclient.Entries.AskAgain): the node agent
|
||||
// calls it when its link to the bus opens (design 47 H, H-A-60).
|
||||
func (c *Composer) EntriesAskAgain(ctx context.Context) {
|
||||
if c.entries != nil {
|
||||
c.entries.AskAgain(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
// keep writes one value into the composition state; a state that cannot be written is said, and the value is kept again by
|
||||
// the next pass.
|
||||
func (c *Composer) keep(key string, v any) {
|
||||
@@ -318,6 +354,7 @@ func (c *Composer) syncEntries(ctx context.Context) {
|
||||
if _, named := need[row.Fingerprint]; !named && !added {
|
||||
c.entries.Release(row.Fingerprint)
|
||||
c.tellNotFound(row.Fingerprint, false)
|
||||
c.tellUnanswered(row.Fingerprint, 0, false)
|
||||
}
|
||||
}
|
||||
c.mu.Lock()
|
||||
@@ -701,6 +738,9 @@ func (c *Composer) entryHolds() map[string]string {
|
||||
case meshcatalogclient.EntryAsked:
|
||||
out[module] = fmt.Sprintf("%s the entry of the build %s is asked of the catalog half and not yet answered, so the node composes nothing new of %s",
|
||||
agent.EntryAsked, w.Fp, module)
|
||||
case meshcatalogclient.EntryUnanswered:
|
||||
out[module] = fmt.Sprintf("%s the entry of the build %s is unanswered: its ask failed to reach the catalog half as often as the setting entry-asks allows, so the node composes nothing new of %s",
|
||||
agent.EntryAsked, w.Fp, module)
|
||||
case meshcatalogclient.EntryNotFound:
|
||||
out[module] = fmt.Sprintf("%s the catalog half holds no entry of the build %s, so the node composes nothing new of %s",
|
||||
agent.EntryMissing, w.Fp, module)
|
||||
@@ -951,6 +991,8 @@ func (c *Composer) addedModule(ctx context.Context, module string) (fp string, e
|
||||
return fp, e, true, false, ""
|
||||
case meshcatalogclient.EntryNotFound:
|
||||
return fp, e, false, false, fmt.Sprintf("the catalog half holds no entry of %s's build %s", module, fp)
|
||||
case meshcatalogclient.EntryUnanswered:
|
||||
return fp, e, false, false, fmt.Sprintf("the entry of %s's build %s is unanswered: its ask failed to reach the catalog half as often as the setting entry-asks allows", module, fp)
|
||||
}
|
||||
return fp, e, false, true, fmt.Sprintf("the entry of %s's build %s is asked of the catalog half and not yet answered", module, fp)
|
||||
}
|
||||
|
||||
@@ -635,6 +635,7 @@ func TestRunningBuildEntryKeptAndHandedAgain(t *testing.T) {
|
||||
// it is bound to), never of every module placed there, and beside the pass.
|
||||
func TestOnlyWhatTheNodeNeedsOfAnotherNodesPlacementIsAsked(t *testing.T) {
|
||||
t.Run("a trunk build the catalog gave none of waits its backoff or a catalog-revision (A-487a)", trunkAskBackoffAndCatalogRevision)
|
||||
t.Run("an entry whose ask failed is asked again on each catalog-revision, at most entry-asks times, then unanswered (H-A-60, H-A-110)", entryAskedAgainThenUnanswered)
|
||||
env := newFakeBus().env()
|
||||
env.Node = "n1"
|
||||
var mu sync.Mutex
|
||||
@@ -770,3 +771,81 @@ func trunkAskBackoffAndCatalogRevision(t *testing.T) {
|
||||
t.Error("catalog-revision was heard and the wait of the module stands")
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 H, H-A-60 and H-A-110: an entry whose ask failed in transport is asked again on each catalog-revision the composer hears
|
||||
// (never on a timer), at most the setting entry-asks times; one more failure and it is unanswered, said in the journal and as the
|
||||
// finding entry-unanswered, which the shadow does not wait on; asked again by hand and answered, the finding is cleared.
|
||||
func entryAskedAgainThenUnanswered(t *testing.T) {
|
||||
fp := meshcatalogclient.SourcePrefix2 + strings.Repeat("c", 64)
|
||||
var calls atomic.Int32
|
||||
var answers atomic.Bool
|
||||
env := newFakeBus().env()
|
||||
env.EntryAsks = 2
|
||||
env.Entry = EntryCall(func(context.Context, string, string, any) (json.RawMessage, error) {
|
||||
calls.Add(1)
|
||||
if answers.Load() {
|
||||
return json.Marshal(meshcatalog.EntryAnswer{Entry: meshcatalog.CatalogEntry{Module: "web", Fp: fp}})
|
||||
}
|
||||
return nil, errors.New("the bus did not answer")
|
||||
})
|
||||
var mu sync.Mutex
|
||||
var said []string
|
||||
told := []string{}
|
||||
env.Say = func(line string) { mu.Lock(); said = append(said, line); mu.Unlock() }
|
||||
env.EntryUnanswered = func(got string, asks int, unanswered bool) {
|
||||
mu.Lock()
|
||||
told = append(told, fmt.Sprintf("%s %d %v", got, asks, unanswered))
|
||||
mu.Unlock()
|
||||
}
|
||||
heard := make(chan []byte)
|
||||
env.Hear = func(string) (<-chan []byte, func(), error) { return heard, func() {}, nil }
|
||||
c := New(env)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go c.hearCatalogRevisions(ctx, func() {})
|
||||
state := func() meshcatalogclient.EntryState { _, st := c.entries.Need(ctx, fp); return st }
|
||||
c.entries.Need(ctx, fp)
|
||||
// Each catalog-revision asks the failed entry again once: three asks in all, the third being the one past the bound of two.
|
||||
for deadline := time.Now().Add(10 * time.Second); state() != meshcatalogclient.EntryUnanswered; {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("the entry is %s after %d asks, want unanswered", state(), calls.Load())
|
||||
}
|
||||
select {
|
||||
case heard <- nil:
|
||||
case <-time.After(time.Millisecond):
|
||||
}
|
||||
}
|
||||
for i := 0; i < 3; i++ {
|
||||
heard <- nil // an unanswered entry is not asked again by an event
|
||||
}
|
||||
var gotTold, gotSaid string
|
||||
for deadline := time.Now().Add(10 * time.Second); gotTold == "[]" || gotTold == ""; runtime.Gosched() {
|
||||
mu.Lock()
|
||||
gotTold, gotSaid = fmt.Sprint(told), strings.Join(said, "\n")
|
||||
mu.Unlock()
|
||||
if time.Now().After(deadline) {
|
||||
break
|
||||
}
|
||||
}
|
||||
if calls.Load() != 3 || gotTold != "["+fp+" 2 true]" || !strings.Contains(gotSaid, "is unanswered") || !strings.Contains(gotSaid, fp) {
|
||||
t.Errorf("asked %d times, told %s, said %q; want 3 asks, the finding told once with 2 asks and the journal line", calls.Load(), gotTold, gotSaid)
|
||||
}
|
||||
// The shadow's wait on the entry ends: it is no longer pending.
|
||||
c.mu.Lock()
|
||||
c.wants = map[string]meshwants.WantsByWantNodeModuleValue{"web": {Fp: fp}}
|
||||
c.mu.Unlock()
|
||||
if c.syncShadowEntries(ctx) {
|
||||
t.Error("the shadow still waits on an unanswered entry")
|
||||
}
|
||||
// Asked again by hand and answered, the finding is cleared.
|
||||
answers.Store(true)
|
||||
if err := c.entries.Again(ctx, fp); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mu.Lock()
|
||||
gotTold = fmt.Sprint(told)
|
||||
mu.Unlock()
|
||||
if gotTold != "["+fp+" 2 true "+fp+" 0 false]" {
|
||||
t.Errorf("told %s after the answer, want the finding cleared", gotTold)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -67,6 +67,12 @@ type Env struct {
|
||||
// EntryNotFound is told when the catalog half answered it holds no entry of a fingerprint (notFound), and when that is no longer so:
|
||||
// the entry answered, or nothing the node holds names it (design 47 G §614, A-117). Nil tells nobody.
|
||||
EntryNotFound func(fp string, notFound bool)
|
||||
// EntryUnanswered is told when an ask of the catalog half's entry failed to reach the holder as often as EntryAsks allows
|
||||
// (unanswered), and when that is no longer so (design 47 H, H-A-60). Nil tells nobody.
|
||||
EntryUnanswered func(fp string, asks int, unanswered bool)
|
||||
// EntryAsks is the setting `entry-asks`: how often an entry whose ask failed in transport is asked again before it is
|
||||
// unanswered; meshcatalogclient.DefaultEntryAsks when 0 (design 47 H, H-A-60).
|
||||
EntryAsks int
|
||||
// Watch follows one bucket's keys matching pattern, as the agent's watches do.
|
||||
Watch func(ctx context.Context, bucket, pattern string) (<-chan agent.Update, error)
|
||||
// Get reads one key's latest value; nil and no error when there is none.
|
||||
@@ -168,7 +174,10 @@ type Composer struct {
|
||||
OnEngine func(module string)
|
||||
// AlsoEntryAgain is told each entry asked again by hand, so the shadow's own cache of entries asks it again too.
|
||||
AlsoEntryAgain func(ctx context.Context, fp string)
|
||||
env Env
|
||||
// AlsoEntriesAskAgain is told each catalog-revision the composer hears, so the shadow, which hears nothing itself, asks its
|
||||
// own entries again too (design 47 H, H-A-110).
|
||||
AlsoEntriesAskAgain func(ctx context.Context)
|
||||
env Env
|
||||
|
||||
mu sync.Mutex
|
||||
wants map[string]meshwants.WantsByWantNodeModuleValue // by module, the node's own under agent.NodeModule
|
||||
@@ -202,6 +211,8 @@ type Composer struct {
|
||||
keptEnt map[string]keptEntry
|
||||
// notFound is each fingerprint whose entry-not-found the composer told, so it is told once and cleared once.
|
||||
notFound map[string]bool
|
||||
// unanswered is each fingerprint whose entry-unanswered the composer told, so it is told once and cleared once.
|
||||
unanswered map[string]bool
|
||||
// resolved is each wanted build's digests as its version resolved them through the registry seats (versions.go, A-430a).
|
||||
resolved map[resolvedKey]versionParts
|
||||
keptWantVal map[string]keptWant
|
||||
@@ -706,7 +717,7 @@ func (c *Composer) Watch(ctx context.Context, _ string) (<-chan agent.Update, er
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// hearCatalogRevisions hears the seat mesh-catalog's event catalog-revision: each one clears the backoffs of the trunk builds the
|
||||
// hearCatalogRevisions hears the seat mesh-catalog's event catalog-revision: each one asks again the entries whose ask failed (H-A-110), clears the backoffs of the trunk builds the
|
||||
// catalog gave none of and wakes a pass (A-487a). A wake-up only, the backoff bounds the asks without it: a subscription refused
|
||||
// is said once with its subject and error (principle 3), the composer goes on composing, and the subscription is tried again on
|
||||
// laterAgain's steps until it holds.
|
||||
@@ -725,6 +736,11 @@ func (c *Composer) hearCatalogRevisions(ctx context.Context, wake func()) {
|
||||
return
|
||||
}
|
||||
c.trunkRetryCleared()
|
||||
// And each entry whose ask failed in transport is asked again, once per take-in (design 47 H, H-A-110).
|
||||
c.EntriesAskAgain(ctx)
|
||||
if c.AlsoEntriesAskAgain != nil {
|
||||
c.AlsoEntriesAskAgain(ctx)
|
||||
}
|
||||
wake()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -826,6 +826,9 @@ func (c *Composer) sameValue(a, b string) (same, known bool) {
|
||||
return bytes.Equal(x, y), true
|
||||
}
|
||||
|
||||
// EntriesAskAgain asks again every entry of the shadow's own cache whose last ask failed in transport (design 47 H, H-A-60).
|
||||
func (s *Shadow) EntriesAskAgain(ctx context.Context) { s.c.EntriesAskAgain(ctx) }
|
||||
|
||||
// EntryAgain asks fp's entry of the catalog half again in the shadow's own cache, where the node's own composer was asked
|
||||
// again by hand: an entry the shadow holds as not found stays so until it is asked (a build the half had not yet taken in).
|
||||
func (s *Shadow) EntryAgain(ctx context.Context, fp string) {
|
||||
|
||||
+13
@@ -105,6 +105,12 @@
|
||||
"minimum": 1,
|
||||
"default": 5
|
||||
},
|
||||
"entry-asks": {
|
||||
"type": "number",
|
||||
"description": "How often the node-engine asks the catalog half's entry of a build again after an ask that failed to reach the holder, before the entry is unanswered and the finding entry-unanswered is written (design 47 H, H-A-60).",
|
||||
"minimum": 1,
|
||||
"default": 5
|
||||
},
|
||||
"operator-signers": {
|
||||
"type": "string",
|
||||
"description": "The operator's public keys, ed25519 in base64, one a line: the node applies a composition made by hand only when one of these signed it (ADR 0539 clause 3). Public keys, no secret. Empty trusts no operator key.",
|
||||
@@ -147,6 +153,13 @@
|
||||
"mode": "0644",
|
||||
"content": "${config:recovery-wait-minutes}\n"
|
||||
},
|
||||
{
|
||||
"id": "entry-asks",
|
||||
"type": "file",
|
||||
"path": "/var/lib/mesh-host/entry-asks",
|
||||
"mode": "0644",
|
||||
"content": "${config:entry-asks}\n"
|
||||
},
|
||||
{
|
||||
"id": "view-waits",
|
||||
"type": "file",
|
||||
|
||||
@@ -89,6 +89,9 @@ func TestTheManifestConsumesTheBackupEvent(t *testing.T) {
|
||||
if !strings.Contains(string(raw), `"content": "${config:recovery-wait-minutes}\n"`) {
|
||||
t.Errorf("module.json declares no file with the content ${config:recovery-wait-minutes} (design 47 G A-112)")
|
||||
}
|
||||
if !strings.Contains(string(raw), `"content": "${config:entry-asks}\n"`) || m.Config["entry-asks"].Default != float64(5) {
|
||||
t.Errorf("module.json declares no file with the content ${config:entry-asks} or its default is not 5 (design 47 H, H-A-60)")
|
||||
}
|
||||
if m.Config["recovery-wait-minutes"].Default != float64(90) {
|
||||
t.Errorf("config recovery-wait-minutes is %+v, want default 90", m.Config["recovery-wait-minutes"])
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user