novox/mesh-host
18
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

compose-candidate reads each manifest once and answers within the bus's reply window

Browse Source 
Off the control-node an ask of compose-candidate ran into its 60 s bound: each of
its two compositions read every manifest in the mesh one key at a time over the
bus, on scratch copies of the shadow that kept nothing, so each ask read them all
twice again (about 35 s per composition on ace, g14 and shanks; 2 s on novox).
The answer at 60 s then came after the bus's one-minute reply window
(allow_responses ttl), so the bus refused it and the checker heard nothing.

Now the manifests are read eight at a time into the shadow before either
composition, a scratch copy keeps what never changes (manifest, archive, done
build) in the shadow too, the candidate is laid over a copy taken after the
composition without it, and the engine answers within 50 s: unready, naming what
it waits on, when not done. Every answer is said in one journal line with its
reads and its longest wait.

Issue 1126.

Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL
This commit is contained in:
jschoubben
2026-11-03 01:23:17 +01:00
parent 27a956448c
commit ebb4f4af18
4 changed files with 330 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/selfcompose/candidate.go
+154 -14
View File
@@ -6,7 +6,9 @@ import (
"errors"
"fmt"
"maps"
"slices"
"sort"
"sync"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
@@ -28,17 +30,98 @@ import (
// errNotReady is a shadow that has not read every input yet.
var errNotReady = errors.New("this node's shadow has not read every input yet")
// ComposeCandidate is the node's answer to ask, within delivery.ComposeAnswerBound (or ctx's deadline, if sooner).
// candidateWithin is the node's own bound on an answer to compose-candidate: shorter than the checker's
// (delivery.ComposeAnswerBound), because the bus lets the engine answer an ask only within a minute of it (the grants
// part's allow_responses ttl, mesh-grants ResponseTTL): an answer at the checker's 60 s bound was refused by the bus
// (issue 1126, 2026-11-03 01:17:04 on shanks, "Permissions Violation for Publish to _INBOX..."), so the checker heard
// nothing at all. A node not done by then answers unready, naming what it waits on.
var candidateWithin = delivery.ComposeAnswerBound - 10*time.Second
// warmAtOnce is how many of the indexer's manifests the shadow reads at once before it composes a candidate.
const warmAtOnce = 8
// candidateRun is what one ask's compositions wait on: each scratch composer it made, the last the one composing now.
type candidateRun struct {
mu sync.Mutex
parts []*Composer
}
func (r *candidateRun) add(c *Composer) *Composer {
r.mu.Lock()
defer r.mu.Unlock()
r.parts = append(r.parts, c)
return c
}
// said is the run's waits in words: how many reads it waited on the bus or the vault for, and the longest; and, while
// one is still running (now), what it waits on and since when.
func (r *candidateRun) said(now time.Time) (waits string, on string) {
r.mu.Lock()
parts := slices.Clone(r.parts)
r.mu.Unlock()
n, longest, longestOn := 0, time.Duration(0), ""
for _, c := range parts {
c.doingMu.Lock()
n += c.waits.n
if c.waits.longest > longest {
longest, longestOn = c.waits.longest, c.waits.longestOn
}
if c.doing != "" {
on = fmt.Sprintf("%s (for %s)", c.doing, now.Sub(c.doingSince).Round(time.Millisecond))
} else if on == "" && c.waits.lastOn != "" {
on = c.waits.lastOn + " (the last read)"
}
c.doingMu.Unlock()
}
waits = fmt.Sprintf("%d read(s) waited on", n)
if longestOn != "" {
waits += fmt.Sprintf(", the longest %s on %s", longest.Round(time.Millisecond), longestOn)
}
if on == "" {
on = "nothing it reads (composing)"
}
return waits, on
}
// ComposeCandidate is the node's answer to ask, within candidateWithin (or ctx's deadline, if sooner). Each answer is
// said in one line, with what it waited on and for how long (issue 1126: ace, g14 and shanks answered nothing for a
// minute while nothing said why). An answer not composed within the bound is unready, naming what the node waits on.
func (s *Shadow) ComposeCandidate(ctx context.Context, ask delivery.ComposeCandidateAsk) delivery.ComposeCandidateAnswer {
began := time.Now()
s.mu.Lock()
c := s.c
s.mu.Unlock()
began := c.now()
ctx, cancel := context.WithTimeout(ctx, candidateWithin)
defer cancel()
run := &candidateRun{}
done := make(chan delivery.ComposeCandidateAnswer, 1)
go func() { done <- s.composeCandidate(ctx, c, ask, began, run) }()
var a delivery.ComposeCandidateAnswer
select {
case a = <-done:
case <-ctx.Done():
_, on := run.said(c.now())
a = delivery.ComposeCandidateAnswer{Node: c.env.Node, Candidate: ask.Candidate, Verdict: delivery.ComposeCandidateUnready,
Refused: fmt.Sprintf("the node did not compose the candidate within %s: it waits on %s", candidateWithin, on),
Took: c.now().Sub(began).Milliseconds()}
}
waits, _ := run.said(c.now())
words := ""
if a.Refused != "" {
words = ": " + a.Refused
}
c.env.Say(fmt.Sprintf("compose-candidate %s answered %s in %s (%s)%s", ask.Candidate, a.Verdict,
(time.Duration(a.Took) * time.Millisecond).String(), waits, words))
return a
}
// composeCandidate composes ask on scratch copies of c, each added to run as it starts.
func (s *Shadow) composeCandidate(ctx context.Context, c *Composer, ask delivery.ComposeCandidateAsk, began time.Time, run *candidateRun) delivery.ComposeCandidateAnswer {
node := c.env.Node
a := delivery.ComposeCandidateAnswer{Node: node, Candidate: ask.Candidate}
refuse := func(why string) delivery.ComposeCandidateAnswer {
a.Verdict, a.Refused, a.Modules, a.Own = delivery.ComposeCandidateRefused, why, nil, delivery.ModuleComposed{}
a.Took = time.Since(began).Milliseconds()
a.Took = c.now().Sub(began).Milliseconds()
return a
}
// A node that cannot compose now (its shadow not ready, a read the bus did not answer, the bound passed) answers
@@ -58,12 +141,10 @@ func (s *Shadow) ComposeCandidate(ctx context.Context, ask delivery.ComposeCandi
if err != nil {
return refuse(err.Error())
}
ctx, cancel := context.WithTimeout(ctx, delivery.ComposeAnswerBound)
defer cancel()
if !s.ready.Load() {
return unready(errNotReady.Error())
}
base := c.scratch()
base := run.add(c.scratch())
raw, err := base.get(ctx, delivery.CandidatesBucket, ckey)
if err != nil {
return unready(fmt.Sprintf("the indexer's candidate %s cannot be read: %v", ask.Candidate, err))
@@ -80,17 +161,21 @@ func (s *Shadow) ComposeCandidate(ctx context.Context, ask delivery.ComposeCandi
// not asked about. No answer (ADR 0593 §9), never a verdict on the wrong heads.
return unready(fmt.Sprintf("the indexer's candidate of the group %s is at heads %s, not %s: the check asked is replaced", group, cand.HeadsHash, heads))
}
laid := c.scratch()
touched, err := laid.layCandidate(ctx, cand)
if err != nil {
return unready(err.Error())
}
// Every module's manifest is read before either composition, several at once, into the shadow's own: a composition
// reads every manifest in the mesh one by one, which off the control-node took about 35 s each (issue 1126).
base.warm(ctx, c, cand)
var running []delivery.Composed
if c.env.Running != nil {
running = c.env.Running()
}
baseComps, baseHeld, baseErr := base.composeShadow(ctx, running, nil, nil)
// The candidate laid over a copy taken after the composition without it, so what that read is not read again.
laid := run.add(c.scratch())
touched, err := laid.layCandidate(ctx, cand)
if err != nil {
return unready(err.Error())
}
laidComps, laidHeld, laidErr := laid.composeShadow(ctx, running, touched, func(scope frombus.Scope) {
if !ask.Input {
return
@@ -109,7 +194,8 @@ func (s *Shadow) ComposeCandidate(ctx context.Context, ask delivery.ComposeCandi
}
})
if ctx.Err() != nil {
return unready(fmt.Sprintf("the bound of %s passed before the node composed the candidate", delivery.ComposeAnswerBound))
_, on := run.said(c.now())
return unready(fmt.Sprintf("the node did not compose the candidate within %s: it waits on %s", candidateWithin, on))
}
switch {
case laidErr != nil && BusFailed(laidErr):
@@ -118,7 +204,7 @@ func (s *Shadow) ComposeCandidate(ctx context.Context, ask delivery.ComposeCandi
// The node composes none of its own without the candidate either: the candidate changes nothing of that.
a.Verdict, a.Modules = delivery.ComposeCandidateComposed, []delivery.ModuleComposed{}
a.Own = delivery.ModuleComposed{Module: delivery.HaveNodeWord, State: delivery.ModuleHeldState, Words: laidErr.Error()}
a.Took = time.Since(began).Milliseconds()
a.Took = c.now().Sub(began).Milliseconds()
return a
case laidErr != nil:
return refuse(laidErr.Error())
@@ -193,7 +279,7 @@ func (s *Shadow) ComposeCandidate(ctx context.Context, ask delivery.ComposeCandi
if refused {
a.Verdict = delivery.ComposeCandidateRefused
}
a.Took = time.Since(began).Milliseconds()
a.Took = c.now().Sub(began).Milliseconds()
return a
}
@@ -214,9 +300,63 @@ func (c *Composer) scratch() *Composer {
out.ownHanded = maps.Clone(c.ownHanded)
out.certified = c.certified
out.placementKeys = c.placementKeys
out.origin = c
return out
}
// warm reads, several at once, each manifest the indexer's modules and the candidate name that the shadow (origin) does
// not hold yet, keeping each in origin and in c; one not read is left to the composition, which reads it and says why.
// Its reads are not noted as the bus failing: the composition that reads again notes its own.
func (c *Composer) warm(ctx context.Context, origin *Composer, cand delivery.Candidate) {
origin.mu.Lock()
want := map[string]bool{}
for _, m := range origin.modules {
if _, ok := origin.manifests[m.Manifest]; m.Manifest != "" && !ok {
want[m.Manifest] = true
}
}
for _, pull := range cand.Pulls {
for _, cm := range pull.Fps {
if _, ok := origin.manifests[cm.Manifest]; cm.Manifest != "" && !ok {
want[cm.Manifest] = true
}
}
}
origin.mu.Unlock()
if len(want) == 0 {
return
}
defer c.waiting(fmt.Sprintf("%d of the indexer's manifests (%s)", len(want), delivery.ManifestsBucket))()
slots := make(chan struct{}, warmAtOnce)
var wg sync.WaitGroup
for digest := range want {
key, err := delivery.ManifestsKey(digest)
if err != nil {
continue
}
wg.Add(1)
go func() {
defer wg.Done()
select {
case slots <- struct{}{}:
case <-ctx.Done():
return
}
defer func() { <-slots }()
value, err := c.env.Get(ctx, delivery.ManifestsBucket, key)
if err != nil || value == nil {
return
}
m, err := delivery.DecodeManifest(value)
if err != nil {
return
}
c.keepRead(func(k *Composer) { k.manifests[digest] = []byte(m.JSON) })
}()
}
wg.Wait()
}
// layCandidate lays the candidate's modules over the indexer's: each at its fingerprint, with its manifest's digest and
// the indexer's verdict on it (read from its verdicts, none read taken as accepted); a module the candidate removes goes.
// It answers the modules laid.