Wake a parked build puller on its capacity's own change, so a node started at capacity 0 takes asks once raised (issue 1887)
g14's Go builder sat idle at capacity 2 while delivery builds waited: a puller with no room waited on its wake alone, which only a pause, a resume or a step's end gives, and it read its capacity from the settings file only when woken. The puller now follows the module's config value capacity through the runtime's watch of what mesh-config resolved (stdio.WatchConfig), uses the value it hands, and wakes on each change: no timer, no file watch, and no wait for the node-engine to fill the file again. It says when it parks and when it wakes, with its capacity.
This commit is contained in:
+82
-4
@@ -9,6 +9,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
@@ -68,6 +69,9 @@ type Puller struct {
|
||||
// node's pause key (delivery.PauseKey) before it pulls, and Pause and Resume write it with Put, so a pause outlives
|
||||
// a restart.
|
||||
Load func(key string) ([]byte, error)
|
||||
// WatchConfig, when set, follows the module's config value `capacity` (CapacitySetting) as mesh-config resolves it
|
||||
// for this node: stdio.WatchConfig in a module, a test's own double.
|
||||
WatchConfig func(key string, handle func(stdio.ProvisionChange) error) error
|
||||
|
||||
mu sync.Mutex
|
||||
inHand int
|
||||
@@ -75,6 +79,9 @@ type Puller struct {
|
||||
paused bool
|
||||
wake chan struct{}
|
||||
once sync.Once
|
||||
// watched is the capacity the config watch handed last, used in place of Capacity once hasWatched.
|
||||
watched int
|
||||
hasWatched bool
|
||||
}
|
||||
|
||||
func (p *Puller) init() {
|
||||
@@ -186,13 +193,29 @@ func (p *Puller) Running() []RunningAsk {
|
||||
|
||||
// room says whether the puller may take one more ask now.
|
||||
func (p *Puller) room() bool {
|
||||
capacity := 0
|
||||
if p.Capacity != nil {
|
||||
ok, _, _ := p.roomNow()
|
||||
return ok
|
||||
}
|
||||
|
||||
// roomNow is room, with the capacity and the asks in hand it was judged on.
|
||||
func (p *Puller) roomNow() (ok bool, capacity, inHand int) {
|
||||
p.mu.Lock()
|
||||
watched, has := p.watched, p.hasWatched
|
||||
p.mu.Unlock()
|
||||
if has {
|
||||
capacity = watched
|
||||
} else if p.Capacity != nil {
|
||||
capacity = p.Capacity()
|
||||
}
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
return !p.paused && p.inHand < capacity
|
||||
return !p.paused && p.inHand < capacity, capacity, p.inHand
|
||||
}
|
||||
|
||||
// CapacityNow is the capacity the puller judges by: the config watch's last value, else Capacity.
|
||||
func (p *Puller) CapacityNow() int {
|
||||
_, capacity, _ := p.roomNow()
|
||||
return capacity
|
||||
}
|
||||
|
||||
// Pull takes and runs asks until ctx ends. An ask still running then is neither ended nor acknowledged: the stream
|
||||
@@ -205,6 +228,7 @@ func (p *Puller) Pull(ctx context.Context) error {
|
||||
if err := p.restore(); err != nil {
|
||||
return err // pulling on a pause that cannot be read could build on a node paused on purpose
|
||||
}
|
||||
p.watchCapacity()
|
||||
wait := p.PullWait
|
||||
if wait <= 0 {
|
||||
wait = DefaultPullWait
|
||||
@@ -213,12 +237,17 @@ func (p *Puller) Pull(ctx context.Context) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if !p.room() {
|
||||
if ok, capacity, inHand := p.roomNow(); !ok {
|
||||
// Parked until something wakes it: a capacity change (the config watch), a resume, or a step's end. Said
|
||||
// once each way, so a puller that sits idle says why (issue 1887).
|
||||
p.logf("the %s puller on %s parks: %d of capacity %d in hand, paused %v", p.Language, p.Node, inHand, capacity, p.Paused())
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-p.wake:
|
||||
}
|
||||
_, capacity, inHand = p.roomNow()
|
||||
p.logf("the %s puller on %s wakes: %d of capacity %d in hand, paused %v", p.Language, p.Node, inHand, capacity, p.Paused())
|
||||
continue
|
||||
}
|
||||
msg, err := p.next(ctx, wait)
|
||||
@@ -348,3 +377,52 @@ func (p *Puller) run(ctx context.Context, msg *Pulled, ask delivery.StepAsk, key
|
||||
p.logf("the ask %s ended on %s and its acknowledgement failed: %v", ask.ID, p.Node, aerr)
|
||||
}
|
||||
}
|
||||
|
||||
// watchCapacity follows the module's config value `capacity` when WatchConfig is set (issue 1887, 2026-10-25: g14's
|
||||
// puller started at capacity 0 sat parked with capacity 2 set, because nothing woke it to read its settings file
|
||||
// again). The value comes as mesh-config resolved it for this node, the event of the change itself: the runtime hands
|
||||
// the value now and then each change its machine's provisioner client says moved, so nothing polls, and the puller does
|
||||
// not wait for the node-engine to fill its settings file again. Each value handed is kept, used in place of Capacity,
|
||||
// and wakes the puller. A watch that cannot start leaves the puller on Capacity, said once.
|
||||
func (p *Puller) watchCapacity() {
|
||||
if p.WatchConfig == nil {
|
||||
return
|
||||
}
|
||||
err := p.WatchConfig(CapacitySetting, func(c stdio.ProvisionChange) error {
|
||||
n, why := p.capacityFrom(c)
|
||||
p.mu.Lock()
|
||||
before, had := p.watched, p.hasWatched
|
||||
p.watched, p.hasWatched = n, true
|
||||
p.mu.Unlock()
|
||||
switch {
|
||||
case why != "":
|
||||
p.logf("the %s puller on %s pulls nothing: %s", p.Language, p.Node, why)
|
||||
case !had || before != n:
|
||||
p.logf("the %s build capacity of %s is %d, as mesh-config resolved it", p.Language, p.Node, n)
|
||||
}
|
||||
p.Wake()
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
p.logf("the %s build capacity of %s cannot be followed on the bus, so it is read from the settings file when the "+
|
||||
"puller wakes: %v", p.Language, p.Node, err)
|
||||
}
|
||||
}
|
||||
|
||||
// capacityFrom is the node's capacity in one value the watch handed, and why it is 0 when it cannot be read.
|
||||
func (p *Puller) capacityFrom(c stdio.ProvisionChange) (int, string) {
|
||||
if c.Refused != "" {
|
||||
return 0, "its capacity is no longer answered: " + c.Refused
|
||||
}
|
||||
setting := ""
|
||||
if len(c.Value) > 0 && string(c.Value) != "null" {
|
||||
if err := json.Unmarshal(c.Value, &setting); err != nil {
|
||||
return 0, fmt.Sprintf("its capacity %s is not a string", c.Value)
|
||||
}
|
||||
}
|
||||
n, err := capacityOf(setting, p.Node)
|
||||
if err != nil {
|
||||
return 0, err.Error()
|
||||
}
|
||||
return n, ""
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user