compose-candidate reads each manifest once and answers within the bus's reply window
Off the control-node an ask of compose-candidate ran into its 60 s bound: each of its two compositions read every manifest in the mesh one key at a time over the bus, on scratch copies of the shadow that kept nothing, so each ask read them all twice again (about 35 s per composition on ace, g14 and shanks; 2 s on novox). The answer at 60 s then came after the bus's one-minute reply window (allow_responses ttl), so the bus refused it and the checker heard nothing. Now the manifests are read eight at a time into the shadow before either composition, a scratch copy keeps what never changes (manifest, archive, done build) in the shadow too, the candidate is laid over a copy taken after the composition without it, and the engine answers within 50 s: unready, naming what it waits on, when not done. Every answer is said in one journal line with its reads and its longest wait. Issue 1126. Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL
This commit is contained in:
@@ -133,3 +133,121 @@ func TestTheNodeComposesACandidateOnAsk(t *testing.T) {
|
||||
t.Errorf("composing candidates wrote %v, marked %d have(s), handed %d answer(s); want nothing", state.written(), applied.Load(), handed.Load())
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1126, 2026-11-03: off the control-node an ask of compose-candidate was not answered within a minute, and said
|
||||
// nothing of why. Each of its two compositions read every manifest in the mesh one by one, on copies of the shadow
|
||||
// that kept nothing, so every ask read them all twice again. Each manifest is now read once, before either
|
||||
// composition, into the shadow, and a later ask reads none; an ask whose read does not answer is answered unready at
|
||||
// its bound, naming what the node waits on; and every answer is said in one line with what it waited on.
|
||||
func TestComposeCandidateReadsOnceAndAnswersWithinItsBound(t *testing.T) {
|
||||
fp := delivery.SourcePrefix2 + strings.Repeat("a", 64)
|
||||
fpNew := delivery.SourcePrefix2 + strings.Repeat("b", 64)
|
||||
lemurs2 := strings.Replace(lemurs, `"version":"1"`, `"version":"2"`, 1)
|
||||
other := strings.NewReplacer(`"module":"lemurs"`, `"module":"other"`, "lemurs.service", "other.service").Replace(lemurs)
|
||||
digest, digest2, digestOther := delivery.ManifestDigest([]byte(lemurs)), delivery.ManifestDigest([]byte(lemurs2)),
|
||||
delivery.ManifestDigest([]byte(other))
|
||||
otherKey, _ := delivery.ManifestsKey(digestOther)
|
||||
|
||||
// shadow is a ready shadow of n1 over b: lemurs placed on it, other placed nowhere (its manifest read for the mesh's
|
||||
// catalogue alone), and a candidate laying lemurs at a new build; its clock moves a second at each reading.
|
||||
shadow := func(b *fakeBus) (*Shadow, string) {
|
||||
for d, body := range map[string]string{digest: lemurs, digest2: lemurs2, digestOther: other} {
|
||||
man, _ := delivery.EncodeManifest(delivery.Manifest{Module: "x", JSON: body})
|
||||
key, _ := delivery.ManifestsKey(d)
|
||||
b.kv[delivery.ManifestsBucket+"/"+key] = man
|
||||
}
|
||||
head := delivery.Head{Repository: "novox/x", Number: 7, Head: "c0ffee"}
|
||||
heads, _ := delivery.HeadsHash([]delivery.Head{head})
|
||||
pulls := []delivery.CandidatePull{{Head: head, Modules: []string{"lemurs"},
|
||||
Fps: map[string]delivery.CandidateModule{"lemurs": {FP2: fpNew, Manifest: digest2}}}}
|
||||
raw, _ := delivery.EncodeCandidate(delivery.Candidate{Group: "fix/1126", HeadsHash: heads, Pulls: pulls,
|
||||
FPSet: delivery.FingerprintSet(pulls)})
|
||||
ckey, _ := delivery.CandidatesKey("fix/1126")
|
||||
b.kv[delivery.CandidatesBucket+"/"+ckey] = raw
|
||||
env := b.env()
|
||||
var tick atomic.Int64
|
||||
env.Now = func() time.Time { return time.Unix(0, 0).Add(time.Duration(tick.Add(1)) * time.Second) }
|
||||
s := NewShadow(env, nil)
|
||||
s.c.placements["n1"] = delivery.Placement{Node: "n1", Decided: delivery.NodeDecided{Hub: true, Address: "10.77.0.1"},
|
||||
Modules: []delivery.PlacedModule{{Module: "lemurs"}}}
|
||||
s.c.records["n1"] = delivery.NodeRecord{Node: "n1", TunnelKey: "k", SealingKey: "sk", Capabilities: []string{"systemd"}}
|
||||
s.c.modules["lemurs"] = delivery.Module{FP2: fp, Manifest: digest, Repository: "novox/x", Commit: "c"}
|
||||
s.c.modules["other"] = delivery.Module{FP2: fp, Manifest: digestOther, Repository: "novox/x", Commit: "c"}
|
||||
s.ready.Store(true)
|
||||
key, _ := delivery.CompositionKey("fix/1126", heads)
|
||||
return s, key
|
||||
}
|
||||
readsOf := func(b *fakeBus, key string) (n int) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
for _, r := range b.reads {
|
||||
if r == [2]string{delivery.ManifestsBucket, key} {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
saidLast := func(b *fakeBus) string {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if len(b.said) == 0 {
|
||||
return ""
|
||||
}
|
||||
return b.said[len(b.said)-1]
|
||||
}
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
// stuck: the read of other's manifest does not answer until the ask's bound ends.
|
||||
stuck bool
|
||||
asks int
|
||||
verdict string
|
||||
reads int // of other's manifest, over every ask
|
||||
words string // in the answer's words
|
||||
said string // in the line said of the last answer
|
||||
}{
|
||||
{name: "one ask reads each manifest once", asks: 1, verdict: delivery.ComposeCandidateComposed, reads: 1,
|
||||
said: "compose-candidate fix/1126."},
|
||||
{name: "a later ask reads none again", asks: 2, verdict: delivery.ComposeCandidateComposed, reads: 1,
|
||||
said: "read(s) waited on"},
|
||||
{name: "a read that does not answer is named at the bound", stuck: true, asks: 1, verdict: delivery.ComposeCandidateUnready,
|
||||
words: "waits on", said: "answered unready"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
b := newFakeBus()
|
||||
s, key := shadow(b)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
if tc.stuck {
|
||||
get := s.c.env.Get
|
||||
s.c.env.Get = func(rctx context.Context, bucket, k string) ([]byte, error) {
|
||||
if bucket == delivery.ManifestsBucket && k == otherKey {
|
||||
b.mu.Lock()
|
||||
b.reads = append(b.reads, [2]string{bucket, k})
|
||||
b.mu.Unlock()
|
||||
cancel() // the ask's bound ends while the read waits
|
||||
<-rctx.Done()
|
||||
return nil, rctx.Err()
|
||||
}
|
||||
return get(rctx, bucket, k)
|
||||
}
|
||||
}
|
||||
var a delivery.ComposeCandidateAnswer
|
||||
for range tc.asks {
|
||||
a = s.ComposeCandidate(ctx, delivery.ComposeCandidateAsk{Candidate: key})
|
||||
}
|
||||
if a.Verdict != tc.verdict || !strings.Contains(a.Refused, tc.words) {
|
||||
t.Errorf("answered %s %q; want %s with %q", a.Verdict, a.Refused, tc.verdict, tc.words)
|
||||
}
|
||||
if tc.stuck && !strings.Contains(a.Refused, "manifest") {
|
||||
t.Errorf("answered %q; want the manifest it waits on named", a.Refused)
|
||||
}
|
||||
if got := readsOf(b, otherKey); tc.reads > 0 && got != tc.reads {
|
||||
t.Errorf("other's manifest was read %d time(s); want %d", got, tc.reads)
|
||||
}
|
||||
if line := saidLast(b); !strings.Contains(line, tc.said) {
|
||||
t.Errorf("said %q; want a line with %q", line, tc.said)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user