Compare commits
14
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
05904f5f44 | ||
|
|
219fdcd7f0 | ||
|
|
55ba7f2cb8 | ||
|
|
102f8dc267 | ||
|
|
296e11d499 | ||
|
|
eed4976483 | ||
|
|
bbadbb2af0 | ||
|
|
1414ed7f7b | ||
|
|
f67342f0d8 | ||
|
|
fae7710bed | ||
|
|
289bbf68e4 | ||
|
|
f2a4363a1a | ||
|
|
fd4d26b42c | ||
|
|
acaf8830ec |
@@ -30,7 +30,9 @@ import (
|
||||
// closes at once and the process exits, so its service manager restarts it as a candidate (serve,
|
||||
// in push.go). A renewal the bus does not answer in time is not a loss (novox/hq issue 477): the
|
||||
// lease is kept, no new act starts once half the key's age passed unrenewed, and the next renewal
|
||||
// made opens the gate again.
|
||||
// made opens the gate again. A candidate backs off before it stands (novox/hq issue 906, lease_backoff.go):
|
||||
// after an epoch lost lately, and while its bus does not answer it, in its own process rather than in a
|
||||
// restart every five seconds.
|
||||
// - **A command run at a shell** — `push` in the installer, the lab, a person repairing a mesh whose
|
||||
// controller is down (issue 201) — acts **under the holder's epoch** when a controller holds the
|
||||
// lease: it is the same mesh's word, composed and sent under the store's hold of each machine like
|
||||
@@ -72,6 +74,9 @@ type actor struct {
|
||||
// lease over in a second rather than in a renewal of five (novox/hq issue 591). A renewal is given
|
||||
// the lease's own RenewEvery to answer however often it is made, so a slow bus does not lose it.
|
||||
renewEvery, poll time.Duration
|
||||
// lossBase, when set, is the backoff after a lost epoch in place of the key's age: a test's (novox/hq
|
||||
// issue 906).
|
||||
lossBase time.Duration
|
||||
}
|
||||
|
||||
// theLease is this process's standing to the lease.
|
||||
@@ -211,48 +216,73 @@ func (a *actor) serveUnderTheLease(ctx context.Context, inv *inventory.Inventory
|
||||
a.mu.Lock()
|
||||
a.serving = true
|
||||
a.mu.Unlock()
|
||||
// Its own connection, held as long as the lease, and named so (novox/hq issue 327).
|
||||
js, err := broker.Dial(address, nats.Name(broker.ConnectionName+" lease"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the mesh is on the bus at %s and this control plane cannot reach it to take the "+
|
||||
"lease: %w", broker.BareAddress(address), err)
|
||||
}
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
js.Close()
|
||||
return nil, err
|
||||
}
|
||||
asserting, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
err = broker.EnsureLeaseBucket(asserting, api)
|
||||
cancel()
|
||||
if err != nil {
|
||||
js.Close()
|
||||
return nil, err
|
||||
}
|
||||
say := func(format string, args ...any) { fmt.Printf(format+"\n", args...) }
|
||||
l, err := lease.Open(ctx, api, broker.LeaseBucket, lease.Options{Holder: holderOf(instance),
|
||||
RenewEvery: a.renewEvery, RenewWithin: lease.RenewEvery, Poll: a.poll, Floor: inv.HighestEpoch, Say: say, Health: controllerHealth, Moved: func(was, floor uint64) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
a.reset = time.Now()
|
||||
a.resetSaid = fmt.Sprintf("the lease bucket was at revision %d with epoch %d already issued: it was "+
|
||||
"raised again from nothing (a bus whose data was replaced), and its revisions were moved past %d so "+
|
||||
"no machine refuses the next epoch", was, floor, floor)
|
||||
}})
|
||||
// **Backing off before standing** (novox/hq issue 906, lease_backoff.go): after an epoch lost lately, before
|
||||
// this process touches the bus; and while the bus does not answer it, here rather than in a restart.
|
||||
if wait := a.backoffAfterLosses(ctx, inv, say); wait > 0 {
|
||||
if err := sleepOrDone(ctx, wait); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
var (
|
||||
js *broker.JetStream
|
||||
l *lease.Lease
|
||||
epoch uint64
|
||||
unwritable error
|
||||
)
|
||||
err = standUp(ctx, sleepOrDone, say, func(ctx context.Context) error {
|
||||
unwritable = nil
|
||||
if js == nil {
|
||||
// Its own connection, held as long as the lease, and named so (novox/hq issue 327).
|
||||
dialled, err := broker.Dial(address, nats.Name(broker.ConnectionName+" lease"))
|
||||
if err != nil {
|
||||
return fmt.Errorf("the mesh is on the bus at %s and this control plane cannot reach it to take the "+
|
||||
"lease: %w", broker.BareAddress(address), err)
|
||||
}
|
||||
js = dialled
|
||||
}
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
asserting, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
err = broker.EnsureLeaseBucket(asserting, api)
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
l, err = lease.Open(ctx, api, broker.LeaseBucket, lease.Options{Holder: holderOf(instance),
|
||||
RenewEvery: a.renewEvery, RenewWithin: lease.RenewEvery, Poll: a.poll, Floor: inv.HighestEpoch, Say: say, Health: controllerHealth, Moved: func(was, floor uint64) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
a.reset = time.Now()
|
||||
a.resetSaid = fmt.Sprintf("the lease bucket was at revision %d with epoch %d already issued: it was "+
|
||||
"raised again from nothing (a bus whose data was replaced), and its revisions were moved past %d so "+
|
||||
"no machine refuses the next epoch", was, floor, floor)
|
||||
}})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
epoch, err = l.Take(ctx)
|
||||
if errors.Is(err, lease.ErrUnwritable) {
|
||||
unwritable = err
|
||||
return nil
|
||||
}
|
||||
// Any other error: whether another controller acts cannot be told, so this one does not act, and tries
|
||||
// again.
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
js.Close()
|
||||
if js != nil {
|
||||
js.Close()
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
gone := make(chan struct{})
|
||||
epoch, err := l.Take(ctx)
|
||||
switch {
|
||||
switch err := unwritable; {
|
||||
case ctx.Err() != nil:
|
||||
js.Close()
|
||||
return nil, ctx.Err()
|
||||
case err != nil && !errors.Is(err, lease.ErrUnwritable):
|
||||
// Whether another controller acts cannot be told: this one does not act, and exits to try again.
|
||||
js.Close()
|
||||
return nil, err
|
||||
case err != nil:
|
||||
// Nobody holds it and the bus will not let it be written: unleased, said, tried again (see above).
|
||||
a.mu.Lock()
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
package main
|
||||
|
||||
import "context"
|
||||
|
||||
// A test's builder says it writes references, as R4e's does, unless the test says otherwise (issue 1886): no test reads
|
||||
// the bus for it.
|
||||
func init() {
|
||||
builderSaysReferences = func(context.Context) (bool, error) { return true, nil }
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/builderkeys"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// theBuilderSays sets what the running builder says for one test.
|
||||
func theBuilderSays(t *testing.T, says bool, err error) {
|
||||
t.Helper()
|
||||
was := builderSaysReferences
|
||||
builderSaysReferences = func(context.Context) (bool, error) { return says, err }
|
||||
t.Cleanup(func() { builderSaysReferences = was })
|
||||
}
|
||||
|
||||
// askingBuildAgent records the modules build-agent is asked for, by path.
|
||||
func askingBuildAgent(t *testing.T) *[]string {
|
||||
t.Helper()
|
||||
var asked []string
|
||||
was := askABuild
|
||||
askABuild = func(_ context.Context, _ buildSource, path, _ string) (string, error) {
|
||||
asked = append(asked, path)
|
||||
return "build-9", nil
|
||||
}
|
||||
t.Cleanup(func() { askABuild = was })
|
||||
return &asked
|
||||
}
|
||||
|
||||
// 1886/AC2: while no running builder says it writes references, a walk asks build-agent for a module of its tier, as
|
||||
// before R4e, and takes nothing from the builder's key.
|
||||
func TestAWalkAsksBuildAgentWhileTheBuilderDoesNotSayItWritesReferences(t *testing.T) {
|
||||
theBuilderSays(t, false, nil)
|
||||
asked := askingBuildAgent(t)
|
||||
p, byName := aWalkOfOne("sudo")
|
||||
askModule(t.Context(), p, "sudo", byName)
|
||||
s := p.Modules["sudo"]
|
||||
if s.Key || s.Build != "build-9" || s.State != "asked" || len(*asked) != 1 {
|
||||
t.Fatalf("the module is not asked of build-agent: %+v, asked %v", s, *asked)
|
||||
}
|
||||
}
|
||||
|
||||
// 1886/AC3: once a running builder says it writes references, a walk takes the module from the builder's key (ADR
|
||||
// 0498 §8) and asks build-agent for nothing.
|
||||
func TestAWalkTakesTheKeyOnceTheBuilderSaysItWritesReferences(t *testing.T) {
|
||||
theBuilderSays(t, true, nil)
|
||||
asked := askingBuildAgent(t)
|
||||
p, byName := aWalkOfOne("sudo")
|
||||
askModule(t.Context(), p, "sudo", byName)
|
||||
if s := p.Modules["sudo"]; !s.Key || s.Build != "" || len(*asked) != 0 {
|
||||
t.Fatalf("the module is not taken from its key: %+v, asked %v", s, *asked)
|
||||
}
|
||||
}
|
||||
|
||||
// 1886/AC2: a say that cannot be read leaves the module unasked and says why, so the next look asks it: neither path is
|
||||
// guessed.
|
||||
func TestASayNotReadLeavesTheModuleUnasked(t *testing.T) {
|
||||
theBuilderSays(t, false, errors.New("the bus did not answer"))
|
||||
asked := askingBuildAgent(t)
|
||||
p, byName := aWalkOfOne("sudo")
|
||||
askModule(t.Context(), p, "sudo", byName)
|
||||
s := p.Modules["sudo"]
|
||||
if s.State != "" || s.Key || len(*asked) != 0 || !strings.Contains(s.Why, "the bus did not answer") {
|
||||
t.Fatalf("a say not read: %+v, asked %v", s, *asked)
|
||||
}
|
||||
if got := unaskedOf(*p); len(got) != 1 {
|
||||
t.Fatalf("the module is not left for the next look to ask: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 1886/AC2: the one function every ask goes through lets build-agent build any module while the builder does not say
|
||||
// it writes references, and refuses it again once it does.
|
||||
func TestBuildAgentIsAskedForAnyModuleWhileTheBuilderDoesNotSay(t *testing.T) {
|
||||
was := askedModuleOf
|
||||
t.Cleanup(func() { askedModuleOf = was })
|
||||
askedModuleOf = func(_ context.Context, _ buildSource, path string) (string, error) {
|
||||
return strings.TrimPrefix(path, "modules/"), nil
|
||||
}
|
||||
theBuilderSays(t, false, nil)
|
||||
if err := onlyTheBuildersOwn(t.Context(), buildSource{Repository: "novox/mesh-catalog", Seat: "git"}, "modules/sudo"); err != nil {
|
||||
t.Fatalf("an ask while the builder does not say is refused: %v", err)
|
||||
}
|
||||
theBuilderSays(t, true, nil)
|
||||
if err := onlyTheBuildersOwn(t.Context(), buildSource{Repository: "novox/mesh-catalog", Seat: "git"}, "modules/sudo"); !errors.Is(err, errNotTheBuildersOwn) {
|
||||
t.Fatalf("an ask once the builder says is not refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1886/AC4: an open walk that waits on a key the running builder cannot reference (done without a reference, as R4e's
|
||||
// own walk did in plan-1792707174627079192) is judged again at the next look: the module is handed back unasked, and
|
||||
// that look's ask goes to build-agent. With the builder saying it writes references, the same walk keeps waiting.
|
||||
func TestAnOpenWalkWaitingOnAnUnreferencedKeyIsAskedOfBuildAgent(t *testing.T) {
|
||||
t.Cleanup(func() {
|
||||
keyTakes.Lock()
|
||||
keyTakes.running, keyTakes.waits, keyTakes.taken = map[string]bool{}, map[string]string{}, map[string]bool{}
|
||||
keyTakes.Unlock()
|
||||
})
|
||||
fp := "src2:" + strings.Repeat("e", 64)
|
||||
unreferenced := builderkeys.Key{Module: "sudo", Fingerprint: fp, Found: true, Revision: 1,
|
||||
Build: sdkdelivery.BuilderBuild{State: sdkdelivery.BuildDone, Manifest: strings.Repeat("b", 64),
|
||||
Artifacts: []sdkdelivery.BuiltArtifact{{Name: "tools", Kind: "bundle", Digest: "sha256:1"}}}}
|
||||
walk := func() (*inventory.Plan, map[string]inventory.Entry, planKeysOf) {
|
||||
theBuilderSays(t, true, nil)
|
||||
p, byName := aWalkOfOne("sudo")
|
||||
askModule(t.Context(), p, "sudo", byName)
|
||||
keys := planKeysOf{values: map[string]sdkdelivery.Module{"sudo@c0ffee0123456789": {FP2: fp}},
|
||||
keys: map[string]builderkeys.Key{"sudo": unreferenced}}
|
||||
return p, byName, keys
|
||||
}
|
||||
take := func(string, sdkdelivery.Module, string) { t.Fatal("an unreferenced key was taken in") }
|
||||
records := recordsOf{builds: map[string]inventory.Build{}}
|
||||
|
||||
// Said: it waits for the reference, as ADR 0498 §8 has it.
|
||||
p, _, keys := walk()
|
||||
if _, err := settleKeys(t.Context(), records, keys, p, p.Tiers[0], take); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if s := p.Modules["sudo"]; s.State != "asked" || !s.Key || !strings.Contains(s.Why, "waits for the reference") {
|
||||
t.Fatalf("with the builder saying, the walk does not wait for the reference: %+v", s)
|
||||
}
|
||||
|
||||
// Not said: handed back, and the next look asks build-agent.
|
||||
p, byName, keys := walk()
|
||||
theBuilderSays(t, false, nil)
|
||||
asked := askingBuildAgent(t)
|
||||
changed, err := settleKeys(t.Context(), records, keys, p, p.Tiers[0], take)
|
||||
if err != nil || !changed {
|
||||
t.Fatalf("the walk was not judged again: %v %v", changed, err)
|
||||
}
|
||||
s := p.Modules["sudo"]
|
||||
if s.Key || s.State != "" || p.State != inventory.PlanBuilding || !strings.Contains(s.Why, "does not say") {
|
||||
t.Fatalf("the module is not handed back to be asked of build-agent: %+v %s", s, p.State)
|
||||
}
|
||||
for _, m := range unaskedOf(*p) {
|
||||
askModule(t.Context(), p, m, byName)
|
||||
}
|
||||
if s := p.Modules["sudo"]; s.Key || s.Build != "build-9" || len(*asked) != 1 {
|
||||
t.Fatalf("the next look did not ask build-agent: %+v, asked %v", s, *asked)
|
||||
}
|
||||
}
|
||||
|
||||
// 1886/AC4: a key the running builder failed while it does not say it writes references fails no walk: the module is
|
||||
// asked of build-agent, whose build is the one delivered.
|
||||
func TestAFailedKeyOfABuilderThatDoesNotSayFailsNoWalk(t *testing.T) {
|
||||
theBuilderSays(t, true, nil)
|
||||
p, byName := aWalkOfOne("sudo")
|
||||
askModule(t.Context(), p, "sudo", byName)
|
||||
_ = byName
|
||||
fp := "src2:" + strings.Repeat("f", 64)
|
||||
theBuilderSays(t, false, nil)
|
||||
keys := planKeysOf{values: map[string]sdkdelivery.Module{"sudo@c0ffee0123456789": {FP2: fp}},
|
||||
keys: map[string]builderkeys.Key{"sudo": {Module: "sudo", Fingerprint: fp, Found: true,
|
||||
Build: sdkdelivery.BuilderBuild{State: sdkdelivery.BuildFailed, Error: "not stored: tools: no reference"}}}}
|
||||
if _, err := settleKeys(t.Context(), recordsOf{}, keys, p, p.Tiers[0], func(string, sdkdelivery.Module, string) {}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if s := p.Modules["sudo"]; p.State != inventory.PlanBuilding || s.State != "" || s.Key {
|
||||
t.Fatalf("a failed key of a builder that does not say failed the walk: %s %q %+v", p.State, p.Note, s)
|
||||
}
|
||||
}
|
||||
|
||||
// 1886/AC6 (ADR 0522 §6): the parts that write references, the builder and the build modules, are build-agent's own until
|
||||
// R4g: asked of build-agent and never taken from keys they write themselves, whatever the builder says.
|
||||
func TestTheReferenceWritersAreAskedOfBuildAgentWhateverTheBuilderSays(t *testing.T) {
|
||||
theBuilderSays(t, true, nil)
|
||||
asked := askingBuildAgent(t)
|
||||
for _, m := range []string{"mesh-build-go", "mesh-build-node", "mesh-build-image", "mesh-delivery-builder"} {
|
||||
p, byName := aWalkOfOne(m)
|
||||
askModule(t.Context(), p, m, byName)
|
||||
if s := p.Modules[m]; s.Key || s.Build != "build-9" {
|
||||
t.Errorf("%s is not asked of build-agent: %+v", m, s)
|
||||
}
|
||||
}
|
||||
if len(*asked) != 4 {
|
||||
t.Fatalf("build-agent was asked %v", *asked)
|
||||
}
|
||||
}
|
||||
|
||||
// 1886 review round 1 (ADR 0522 §2): the guard of the one ask function judges by the say askModule read, never a second
|
||||
// read: a second read that fails, or that says otherwise, fails no plan.
|
||||
func TestTheAskGuardUsesTheSayTheWalkRead(t *testing.T) {
|
||||
was := askedModuleOf
|
||||
t.Cleanup(func() { askedModuleOf = was })
|
||||
askedModuleOf = func(_ context.Context, _ buildSource, path string) (string, error) {
|
||||
return strings.TrimPrefix(path, "modules/"), nil
|
||||
}
|
||||
reads := 0
|
||||
wasSays := builderSaysReferences
|
||||
t.Cleanup(func() { builderSaysReferences = wasSays })
|
||||
for _, second := range []struct {
|
||||
name string
|
||||
says bool
|
||||
err error
|
||||
}{{"a second read that fails", false, errors.New("the bus did not answer")}, {"a say that turned true", true, nil}} {
|
||||
reads = 0
|
||||
builderSaysReferences = func(context.Context) (bool, error) {
|
||||
reads++
|
||||
if reads == 1 {
|
||||
return false, nil
|
||||
}
|
||||
return second.says, second.err
|
||||
}
|
||||
wasAsk := askABuild
|
||||
askABuild = func(ctx context.Context, source buildSource, path, _ string) (string, error) {
|
||||
if err := onlyTheBuildersOwn(ctx, source, path); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "build-9", nil
|
||||
}
|
||||
p, byName := aWalkOfOne("sudo")
|
||||
askModule(t.Context(), p, "sudo", byName)
|
||||
askABuild = wasAsk
|
||||
if s := p.Modules["sudo"]; p.State == inventory.PlanFailed || s.Build != "build-9" || s.Key {
|
||||
t.Errorf("%s: the walk %s, the module %+v", second.name, p.State, s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1886 merge check hang: the look must judge an ask and a settle by one say. A settle that read the say from another
|
||||
// source than the ask (the bus's bucket, against the walk's read) handed a module back, the next ask took it from its key
|
||||
// again, and the look's loop never ended: walk_from_keys_integration_test hung.
|
||||
func TestTheSettleJudgesByTheSameSayAsTheAsk(t *testing.T) {
|
||||
theBuilderSays(t, true, nil)
|
||||
p, _ := aWalkOfOne("sudo")
|
||||
p.Modules["sudo"] = &inventory.PlanModule{}
|
||||
askModule(t.Context(), p, "sudo", map[string]inventory.Entry{"sudo": {Source: inventory.Source{Repository: "novox/mesh-catalog", Path: "modules/sudo", Ref: "main"}}})
|
||||
fp := "src2:" + strings.Repeat("1", 64)
|
||||
keys := planKeysOf{values: map[string]sdkdelivery.Module{"sudo@c0ffee0123456789": {FP2: fp}},
|
||||
keys: map[string]builderkeys.Key{"sudo": {Module: "sudo", Fingerprint: fp, Found: true,
|
||||
Build: sdkdelivery.BuilderBuild{State: sdkdelivery.BuildBuilding, By: "shanks"}}}}
|
||||
if _, err := settleKeys(t.Context(), recordsOf{}, keys, p, p.Tiers[0], func(string, sdkdelivery.Module, string) {}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if s := p.Modules["sudo"]; !s.Key || s.State != "asked" {
|
||||
t.Fatalf("the settle judged by another say than the ask's: %+v", s)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// Backing off before standing (novox/hq issue 906).
|
||||
//
|
||||
// A controller loses its lease when its bus stops answering for longer than the key's age: on the control-node,
|
||||
// a bus whose disks are saturated has taken 30 to 42 seconds to answer (2026-10-23 05:52). The process then
|
||||
// exits, as ADR 0229 rule 4 says, and its service manager starts it again five seconds later. **A start is the
|
||||
// heaviest thing a controller asks of the bus** — every bucket, stream and consumer asserted again — and a start
|
||||
// that cannot assert the lease's bucket used to exit at once and be started again five seconds later, on the
|
||||
// same overloaded bus, nine times in twelve minutes (2026-10-16). So a candidate backs off twice:
|
||||
//
|
||||
// - **After a loss**, before it touches the bus: from the newest epoch lost in the last ten minutes (the
|
||||
// controller's store says it, rule 7), it waits one key's age, doubled for every further loss in that
|
||||
// window, at most two minutes. A single loss costs a key's age; a controller losing it again and again
|
||||
// stands less and less often.
|
||||
// - **While the bus does not answer it**, in its own process: asserting the lease's bucket, opening it or
|
||||
// reading who holds it is tried again after 5 s, doubling to at most 60 s, instead of exiting into a restart
|
||||
// every five seconds. Nothing acts meanwhile: the process holds no lease, and the gate refuses every act.
|
||||
|
||||
const (
|
||||
// lossWindow is how far back lost epochs count towards the backoff.
|
||||
lossWindow = 10 * time.Minute
|
||||
// lossMost is the longest a candidate waits after losses.
|
||||
lossMost = 2 * time.Minute
|
||||
// standFirst and standMost are the waits between tries of a candidate the bus does not answer.
|
||||
standFirst = 5 * time.Second
|
||||
standMost = time.Minute
|
||||
)
|
||||
|
||||
// lossBackoff is how long a candidate waits now before it stands, after the epochs it read: one key's age from
|
||||
// the newest loss in the window, doubled for every further loss in it, at most lossMost.
|
||||
func lossBackoff(epochs []inventory.Epoch, now time.Time) time.Duration {
|
||||
return lossBackoffFrom(epochs, now, broker.LeaseTTL)
|
||||
}
|
||||
|
||||
func lossBackoffFrom(epochs []inventory.Epoch, now time.Time, base time.Duration) time.Duration {
|
||||
lost, newest := lostSince(epochs, now)
|
||||
if lost == 0 {
|
||||
return 0
|
||||
}
|
||||
delay := base
|
||||
for i := 1; i < lost && delay < lossMost; i++ {
|
||||
delay *= 2
|
||||
}
|
||||
delay = min(delay, lossMost)
|
||||
return max(newest.Add(delay).Sub(now), 0)
|
||||
}
|
||||
|
||||
// standUp runs try until it answers nil, waiting standFirst between tries and doubling to standMost; it answers
|
||||
// ctx's error once ctx ends.
|
||||
func standUp(ctx context.Context, sleep func(context.Context, time.Duration) error, say func(string, ...any),
|
||||
try func(context.Context) error) error {
|
||||
wait := standFirst
|
||||
for {
|
||||
err := try(ctx)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
say("this controller cannot stand for the lease yet: %v — it tries again in %s, in this process, rather "+
|
||||
"than start again on a bus that does not answer (novox/hq issue 906)", err, wait)
|
||||
if err := sleep(ctx, wait); err != nil {
|
||||
return err
|
||||
}
|
||||
wait = min(wait*2, standMost)
|
||||
}
|
||||
}
|
||||
|
||||
// backoffAfterLosses is how long this candidate waits before it stands, read from the epochs its store kept,
|
||||
// and said. A store that cannot be read makes it wait nothing: the backoff spares the bus, and never keeps a
|
||||
// controller from standing.
|
||||
func (a *actor) backoffAfterLosses(ctx context.Context, inv *inventory.Inventory, say func(string, ...any)) time.Duration {
|
||||
now := time.Now()
|
||||
reading, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
epochs, err := inv.EpochsSince(reading, now.Add(-lossWindow))
|
||||
cancel()
|
||||
if err != nil {
|
||||
say("whether the lease was lost lately cannot be read, so this controller stands without backing off: %v", err)
|
||||
return 0
|
||||
}
|
||||
base := a.lossBase
|
||||
if base <= 0 {
|
||||
base = broker.LeaseTTL
|
||||
}
|
||||
wait := lossBackoffFrom(epochs, now, base)
|
||||
if wait > 0 {
|
||||
lost, _ := lostSince(epochs, now)
|
||||
say("the controller lease was lost lately (%d time(s) in the last %s): this controller backs off %s "+
|
||||
"before it stands, so its start does not add to the load that lost it (novox/hq issue 906)",
|
||||
lost, lossWindow, wait.Round(time.Second))
|
||||
}
|
||||
return wait
|
||||
}
|
||||
|
||||
// lostSince is how many epochs were lost in the window before now, and when the newest of them was.
|
||||
func lostSince(epochs []inventory.Epoch, now time.Time) (lost int, newest time.Time) {
|
||||
for _, e := range epochs {
|
||||
if e.Ended == nil || e.How != inventory.EpochLost || now.Sub(*e.Ended) > lossWindow {
|
||||
continue
|
||||
}
|
||||
lost++
|
||||
if e.Ended.After(newest) {
|
||||
newest = *e.Ended
|
||||
}
|
||||
}
|
||||
return lost, newest
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A controller that lost its lease to a slow bus backs off before it stands again (novox/hq issue 906): it
|
||||
// does not add its start — every bucket and consumer asserted again — to the load that lost it the lease.
|
||||
|
||||
func lostAt(epoch uint64, ended time.Time, how string) inventory.Epoch {
|
||||
return inventory.Epoch{Epoch: epoch, Instance: "controller@anchor", Taken: ended.Add(-time.Hour), Ended: &ended,
|
||||
How: how}
|
||||
}
|
||||
|
||||
// 906/AC1: given epochs lost in the last ten minutes, when a candidate starts, then it waits from the newest
|
||||
// loss one key's age, doubled for each further loss, at most two minutes; nothing else makes it wait.
|
||||
func TestACandidateAfterALostLeaseBacksOff(t *testing.T) {
|
||||
now := time.Date(2026, 10, 23, 5, 52, 40, 0, time.UTC)
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
epochs []inventory.Epoch
|
||||
want time.Duration
|
||||
}{
|
||||
{"nothing lost", nil, 0},
|
||||
{"one held, one given back", []inventory.Epoch{
|
||||
{Epoch: 3, Taken: now.Add(-time.Minute)},
|
||||
lostAt(2, now.Add(-2*time.Second), inventory.EpochReleased),
|
||||
}, 0},
|
||||
{"one lost five seconds ago", []inventory.Epoch{lostAt(7, now.Add(-5*time.Second), inventory.EpochLost)},
|
||||
10 * time.Second},
|
||||
{"one lost long ago enough", []inventory.Epoch{lostAt(7, now.Add(-20*time.Second), inventory.EpochLost)}, 0},
|
||||
{"three lost, the newest a second ago", []inventory.Epoch{
|
||||
lostAt(9, now.Add(-time.Second), inventory.EpochLost),
|
||||
lostAt(8, now.Add(-3*time.Minute), inventory.EpochLost),
|
||||
lostAt(7, now.Add(-6*time.Minute), inventory.EpochLost),
|
||||
}, 59 * time.Second},
|
||||
{"six lost: at most two minutes", []inventory.Epoch{
|
||||
lostAt(12, now, inventory.EpochLost), lostAt(11, now.Add(-time.Minute), inventory.EpochLost),
|
||||
lostAt(10, now.Add(-2*time.Minute), inventory.EpochLost), lostAt(9, now.Add(-3*time.Minute), inventory.EpochLost),
|
||||
lostAt(8, now.Add(-4*time.Minute), inventory.EpochLost), lostAt(7, now.Add(-5*time.Minute), inventory.EpochLost),
|
||||
}, 2 * time.Minute},
|
||||
{"a loss older than ten minutes does not count", []inventory.Epoch{
|
||||
lostAt(9, now.Add(-time.Second), inventory.EpochLost),
|
||||
lostAt(8, now.Add(-11*time.Minute), inventory.EpochLost),
|
||||
}, 14 * time.Second},
|
||||
{"expired is not lost", []inventory.Epoch{lostAt(9, now.Add(-time.Second), inventory.EpochExpired)}, 0},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
if got := lossBackoff(c.epochs, now); got != c.want {
|
||||
t.Fatalf("a candidate after %d epoch(s) waits %s, want %s", len(c.epochs), got, c.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// 906/AC2: given a bus that does not answer a candidate, when it asserts the lease's bucket or reads who holds
|
||||
// it, then the candidate tries again in its own process after 5 s, doubling to at most 60 s, instead of exiting
|
||||
// to be restarted by its service manager every five seconds.
|
||||
func TestACandidateTheBusDoesNotAnswerTriesAgainWithBackoff(t *testing.T) {
|
||||
var waited []time.Duration
|
||||
sleep := func(ctx context.Context, d time.Duration) error { waited = append(waited, d); return nil }
|
||||
tries := 0
|
||||
slow := errors.New("asserting bucket mesh-controller_lease: context deadline exceeded")
|
||||
err := standUp(t.Context(), sleep, func(string, ...any) {}, func(context.Context) error {
|
||||
tries++
|
||||
if tries < 7 {
|
||||
return slow
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("a candidate whose bus answered at the seventh try gave up: %v", err)
|
||||
}
|
||||
want := []time.Duration{5 * time.Second, 10 * time.Second, 20 * time.Second, 40 * time.Second, time.Minute,
|
||||
time.Minute}
|
||||
if tries != 7 || !slices.Equal(waited, want) {
|
||||
t.Fatalf("a candidate tried %d time(s), waiting %v between; want 7, waiting %v", tries, waited, want)
|
||||
}
|
||||
}
|
||||
|
||||
// 906/AC2: and a candidate stopping while it waits stops, with why.
|
||||
func TestACandidateStoppingWhileItBacksOffStops(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
cancel()
|
||||
err := standUp(ctx, sleepOrDone, func(string, ...any) {}, func(context.Context) error {
|
||||
return errors.New("who holds the lease cannot be read: context deadline exceeded")
|
||||
})
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("a candidate stopped while backing off answered %v", err)
|
||||
}
|
||||
}
|
||||
@@ -152,3 +152,33 @@ func TestTwoControllersOneActs(t *testing.T) {
|
||||
t.Fatalf("B's epoch does not say it was lost: %+v", ended)
|
||||
}
|
||||
}
|
||||
|
||||
// 906/AC1, on a real bus and store: a controller starting right after an epoch was lost backs off before it
|
||||
// stands — it takes the lease no sooner than the backoff after the loss, and then holds it as usual.
|
||||
func TestAControllerAfterALostLeaseBacksOffBeforeItStands(t *testing.T) {
|
||||
url, _ := aBusForTheLease(t)
|
||||
inv := inventory.ForTest(t)
|
||||
ctx := t.Context()
|
||||
// The controller before it lost epoch 5, and said so in the store, as `took` does on a loss.
|
||||
if _, err := inv.TookEpoch(ctx, inventory.Epoch{Epoch: 5, Instance: "controller@anchor pid 1", Host: "anchor",
|
||||
Taken: time.Now()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.EndEpoch(ctx, 5, inventory.EpochLost); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The key's age stands in as a backoff of two seconds, so the case runs in seconds (issue 591).
|
||||
const backoff = 2 * time.Second
|
||||
a := &actor{renewEvery: lease.RenewEvery / 5, poll: lease.Poll / 10, lossBase: backoff}
|
||||
start := time.Now()
|
||||
if _, err := a.serveUnderTheLease(ctx, inv, url); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer a.release()
|
||||
if waited := time.Since(start); waited < backoff-300*time.Millisecond {
|
||||
t.Fatalf("a controller took the lease %s after an epoch was lost, without backing off %s", waited, backoff)
|
||||
}
|
||||
if epoch, err := a.epoch(ctx); err != nil || epoch <= 5 {
|
||||
t.Fatalf("after backing off the controller holds epoch %d (%v), want one above the lost 5", epoch, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"time"
|
||||
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"github.com/novox/mesh-controller/internal/builderkeys"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
@@ -209,7 +210,12 @@ func watchNewModules(ctx context.Context, open *stores) {
|
||||
fmt.Printf("new modules: the builder's keys are not watched: %v\n", err)
|
||||
return
|
||||
}
|
||||
if err := builderkeys.Watch(ctx, keys.JS, func(module string) { wakeNewModule(ctx, open, module) }); err != nil {
|
||||
says := watchingSays{js: keys.JS}
|
||||
says.watch(ctx)
|
||||
if err := builderkeys.Watch(ctx, keys.JS, func(module string) {
|
||||
says.watch(ctx)
|
||||
wakeNewModule(ctx, open, module)
|
||||
}); err != nil {
|
||||
closeKeys()
|
||||
fmt.Printf("new modules: the builder's keys are not watched: %v\n", err)
|
||||
return
|
||||
@@ -272,3 +278,32 @@ func newModulesOf(ctx context.Context, repository string) ([]string, error) {
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// watchingSays watches the builder's bucket of instances once it is there (issue 1886): a change of what the builder says
|
||||
// asks the look, which judges every open walk again. A builder from before R4e makes no such bucket; a write of the
|
||||
// builder's keys asks again whether it is there now, so nothing waits on a timer.
|
||||
type watchingSays struct {
|
||||
js jetstream.JetStream
|
||||
mu *sync.Mutex
|
||||
on *bool
|
||||
}
|
||||
|
||||
func (w *watchingSays) watch(ctx context.Context) {
|
||||
if w.mu == nil {
|
||||
w.mu, w.on = &sync.Mutex{}, new(bool)
|
||||
}
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if *w.on {
|
||||
return
|
||||
}
|
||||
on, err := builderkeys.WatchSays(ctx, w.js, askToLook)
|
||||
if err != nil {
|
||||
fmt.Printf("plans: what the builder says it writes is not watched now: %v\n", err)
|
||||
return
|
||||
}
|
||||
if on {
|
||||
*w.on = true
|
||||
askToLook() // what it says now may differ from what the last look read
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,7 +32,12 @@ func isBuildersOwn(module string) bool { return buildersOwn(module) }
|
||||
// repository's; and mesh-tools, the toolchain image build-agent's builds stand on. The builder writes no key of any of them,
|
||||
// and a walk waiting for one would hold its tier for ever. mesh-tools-go is not among them: it builds from a Dockerfile in
|
||||
// its own directory on a pinned base, and has its key from mesh-build-image.
|
||||
var buildAgentsOwn = map[string]bool{sdkdelivery.BuilderModule: true, builder.AgentModule: true, "mesh-tools": true}
|
||||
//
|
||||
// **And the parts that write references** (ADR 0522 §6, issue 1886): the build modules push and answer each reference,
|
||||
// and the builder writes it into the key, so none of them is ever taken from a key they write themselves. Until R4g
|
||||
// build-agent is the way they are delivered whatever the builder says.
|
||||
var buildAgentsOwn = map[string]bool{sdkdelivery.BuilderModule: true, builder.AgentModule: true, "mesh-tools": true,
|
||||
"mesh-build-go": true, "mesh-build-node": true, "mesh-build-image": true}
|
||||
|
||||
// buildersOwn is isBuildersOwn's rule: a variable only so a test of the asks left (their bus waits, their plan key) may
|
||||
// name modules of its own as build-agent's.
|
||||
@@ -44,6 +49,32 @@ type planKeyReads interface {
|
||||
Build(ctx context.Context, module, fp string) (builderkeys.Key, error)
|
||||
}
|
||||
|
||||
// builderSayKey carries, on an ask's context, the builder's say the walk read before asking (withBuilderSay).
|
||||
type builderSayKey struct{}
|
||||
|
||||
// withBuilderSay hands the say a walk read to the ask's guard, so the guard never reads it a second time.
|
||||
func withBuilderSay(ctx context.Context, says bool) context.Context {
|
||||
return context.WithValue(ctx, builderSayKey{}, says)
|
||||
}
|
||||
|
||||
// builderSayOf is the say a walk handed the ask, and whether it handed one.
|
||||
func builderSayOf(ctx context.Context) (says, handed bool) {
|
||||
says, handed = ctx.Value(builderSayKey{}).(bool)
|
||||
return says, handed
|
||||
}
|
||||
|
||||
// builderSaysReferences is whether a running builder instance says, in its own bucket of instances, that the keys it
|
||||
// writes name references (issue 1886). Until one does, a walk asks build-agent for every module, as before R4e: the
|
||||
// keys the running builder writes could never be delivered from. A variable so a test needs no bus.
|
||||
var builderSaysReferences = func(ctx context.Context) (bool, error) {
|
||||
keys, closeKeys, err := openBuilderKeys()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer closeKeys()
|
||||
return keys.SaysReferences(ctx)
|
||||
}
|
||||
|
||||
// openPlanKeys opens what a walk reads of the indexer and the builder. A variable so a test needs no bus.
|
||||
var openPlanKeys = func() (planKeyReads, func(), error) {
|
||||
r, closeKeys, err := openBuilderKeys()
|
||||
@@ -102,6 +133,20 @@ func settleKeys(ctx context.Context, inv keyRecords, keys planKeyReads, p *inven
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
// One say for the ask and the settle (builderSaysReferences): two sources that differ hand a module back and take it
|
||||
// from its key again at every turn of the look, which then never ends.
|
||||
says, err := builderSaysReferences(ctx)
|
||||
if err != nil {
|
||||
return changed, fmt.Errorf("whether the builder writes references cannot be read: %w", err)
|
||||
}
|
||||
// **A walk never waits on a key the running builder cannot reference** (issue 1886): while no builder says it writes
|
||||
// references, a module that waits on its key, or whose key failed, is handed back unasked, and the look asks it of
|
||||
// build-agent. A done key with every reference is still taken.
|
||||
toAgent := func(s *inventory.PlanModule, why string) {
|
||||
s.State, s.Key, s.AskedAt, s.KeyAt, s.Build, s.Fingerprint = "", false, nil, "", "", ""
|
||||
s.Why = "the builder does not say it writes references, so build-agent is asked (" + why + ")"
|
||||
changed = true
|
||||
}
|
||||
for _, m := range tier {
|
||||
s := p.Modules[m]
|
||||
if s == nil || !s.Key || s.State != "asked" {
|
||||
@@ -109,6 +154,10 @@ func settleKeys(ctx context.Context, inv keyRecords, keys planKeyReads, p *inven
|
||||
}
|
||||
trunk, err := keys.ModuleAt(ctx, m, s.KeyAt)
|
||||
if errors.Is(err, builderkeys.ErrNoValue) {
|
||||
if !says {
|
||||
toAgent(s, "the indexer holds no value of it")
|
||||
continue
|
||||
}
|
||||
set(s, fmt.Sprintf("waits for the indexer's value of %s at %s", m, orNow(s.KeyAt)))
|
||||
continue
|
||||
}
|
||||
@@ -120,6 +169,10 @@ func settleKeys(ctx context.Context, inv keyRecords, keys planKeyReads, p *inven
|
||||
if why == "" {
|
||||
why = "the indexer wrote no fingerprint of the new form"
|
||||
}
|
||||
if !says {
|
||||
toAgent(s, why)
|
||||
continue
|
||||
}
|
||||
set(s, fmt.Sprintf("waits: %s at %s has no fingerprint to take a build by: %s", m, orNow(s.KeyAt), why))
|
||||
continue
|
||||
}
|
||||
@@ -131,6 +184,9 @@ func settleKeys(ctx context.Context, inv keyRecords, keys planKeyReads, p *inven
|
||||
return changed, err
|
||||
}
|
||||
switch {
|
||||
case !says && (k.Failed() || !k.Done() || len(k.Unreferenced()) > 0):
|
||||
toAgent(s, k.Says())
|
||||
continue
|
||||
case k.Failed():
|
||||
// **A failed key fails the walk in the builder's own words** (`digests differ:`, `not stored:`, `not pinned:`,
|
||||
// `version taken:`), and a rebuild of it is a new revision the next walk takes.
|
||||
|
||||
@@ -203,6 +203,21 @@ func retryRefusal(p inventory.Plan, plans []inventory.Plan, onPath map[string]st
|
||||
return oneWalkAtATime(p, plans, onPath)
|
||||
}
|
||||
stopped := stoppedRollouts(p)
|
||||
// **A walk stopped by hand while its tier waited is resumed** (issue 1886): nothing in it failed, so nothing is
|
||||
// asked again; it goes on from where it stopped, unless a newer walk holds what it has yet to send.
|
||||
if len(stopped) == 0 && stoppedByHandWaiting(p) {
|
||||
if q, found := newerOpenPlan(p, plans); found {
|
||||
return fmt.Errorf("%s supersedes it: a newer merge of %s (%s at %s) is open, and resuming %s would build "+
|
||||
"what that one replaced", q.ID, q.Repository, q.ID, short(q.Commit), p.ID)
|
||||
}
|
||||
for _, m := range holdingABuild(p) {
|
||||
if q, found := newerPlanFor(m, p, plans); found {
|
||||
return fmt.Errorf("%s has a newer plan, %s (%s, %s at %s): resuming %s would send the older build of it "+
|
||||
"over the newer one", m, q.ID, q.State, q.Repository, short(q.Commit), p.ID)
|
||||
}
|
||||
}
|
||||
return oneWalkAtATime(p, plans, onPath)
|
||||
}
|
||||
if len(stopped) == 0 {
|
||||
return fmt.Errorf("nothing in tier %d of %s failed to build or stopped rolling out — it stopped at: %s",
|
||||
p.Tier, p.ID, p.Note)
|
||||
@@ -333,6 +348,8 @@ func plansForRetry(ctx context.Context, inv planReader, p inventory.Plan) ([]inv
|
||||
|
||||
// retryPlan asks a failed plan's failed modules again, under new ids, and sets it building again at
|
||||
// that tier: what follows is the plan going on as if they had built the first time.
|
||||
// A walk stopped by hand while its tier waited, with nothing failed, is resumed instead: nothing asked again
|
||||
// (resumeStopped, issue 1886).
|
||||
func retryPlan(ctx context.Context, open *stores, id string) (string, error) {
|
||||
inv := open.inventory
|
||||
release, err := inv.HoldPlans(ctx, true)
|
||||
@@ -388,6 +405,13 @@ func retryPlan(ctx context.Context, open *stores, id string) (string, error) {
|
||||
if again := unjudgedAtGate(p); len(failed) == 0 && len(again) > 0 {
|
||||
return retryTierWhole(ctx, open, &p, again)
|
||||
}
|
||||
if len(failed) == 0 && len(stoppedRollouts(p)) == 0 && stoppedByHandWaiting(p) {
|
||||
said := resumeStopped(&p)
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return said, nil
|
||||
}
|
||||
if len(failed) == 0 {
|
||||
return retryRollouts(ctx, open, &p)
|
||||
}
|
||||
@@ -583,3 +607,89 @@ func toRetry(p *inventory.Plan, recorded map[string][]inventory.Build, byID map[
|
||||
}
|
||||
return failedIn(*p)
|
||||
}
|
||||
|
||||
// stoppedByHandWaiting says a walk was stopped by hand while its current tier still waited (issue 1886): stopped by a
|
||||
// person, nothing of the tier failed, and a module of it asked, built and not sent, or not yet asked. A stop that left
|
||||
// merges out is not one: retryPlan refuses it before this is read (novox/hq ADR 0299).
|
||||
func stoppedByHandWaiting(p inventory.Plan) bool {
|
||||
if p.State != inventory.PlanFailed || p.Delivery == nil || p.Delivery.Stopped == "" ||
|
||||
p.Tier >= len(p.Tiers) || len(failedIn(p)) > 0 {
|
||||
return false
|
||||
}
|
||||
asked, unsent, unasked := waitingIn(p)
|
||||
return asked+unsent+unasked > 0
|
||||
}
|
||||
|
||||
// waitingIn counts what a plan's current tier waits on: modules asked, built and not sent, not yet asked.
|
||||
func waitingIn(p inventory.Plan) (asked, unsent, unasked int) {
|
||||
if p.Tier >= len(p.Tiers) {
|
||||
return 0, 0, 0
|
||||
}
|
||||
for _, m := range p.Tiers[p.Tier] {
|
||||
s := p.Modules[m]
|
||||
switch {
|
||||
case s == nil || s.State == "":
|
||||
unasked++
|
||||
case s.State == "asked":
|
||||
asked++
|
||||
case s.State == "built" && s.SentAt == nil:
|
||||
unsent++
|
||||
}
|
||||
}
|
||||
return asked, unsent, unasked
|
||||
}
|
||||
|
||||
// holdingABuild is the modules of a stopped walk's current tier that hold a build of its commit, not yet sent: built and
|
||||
// not sent, or asked under a build id. Only these could put an older build over a newer plan's (review of issue 1886): a
|
||||
// module not yet asked is asked at its followed branch's head when the walk reaches it, and one asked on the builder's key
|
||||
// is settled by the look from the key of the commit it reads then. Sorted.
|
||||
func holdingABuild(p inventory.Plan) []string {
|
||||
if p.Tier >= len(p.Tiers) {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for _, m := range p.Tiers[p.Tier] {
|
||||
s := p.Modules[m]
|
||||
if s == nil || s.SentAt != nil || s.Key {
|
||||
continue
|
||||
}
|
||||
if s.State == "built" || (s.State == "asked" && s.Build != "") {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// resumeStopped sets a walk stopped by hand building again at the tier it stopped at, its stop cleared and its asks kept but
|
||||
// build-agent's own (issue 1886): the next look carries it on as any walk — a build done waits for its reference, and one done on the
|
||||
// builder's key is handed on (novox/hq ADR 0522 §3). Its note keeps who stopped it and that a person resumed it.
|
||||
func resumeStopped(p *inventory.Plan) string {
|
||||
// **Build-agent's own are asked of build-agent** (novox/hq ADR 0522 §6): one marked taken from a key it writes itself is
|
||||
// set unasked, and the look asks the rest of a partly asked tier (unaskedOf), of build-agent, whatever the builder says.
|
||||
var again []string
|
||||
for _, m := range p.Tiers[p.Tier] {
|
||||
if s := p.Modules[m]; s != nil && s.State == "asked" && s.Key && isBuildersOwn(m) {
|
||||
s.State, s.Key, s.AskedAt, s.Build, s.Why = "", false, nil, "", ""
|
||||
again = append(again, m)
|
||||
}
|
||||
}
|
||||
sort.Strings(again)
|
||||
asked, unsent, unasked := waitingIn(*p)
|
||||
later := 0
|
||||
for i := p.Tier + 1; i < len(p.Tiers); i++ {
|
||||
later += len(p.Tiers[i])
|
||||
}
|
||||
by, why := p.Delivery.Stopped, p.Delivery.StoppedWhy
|
||||
waits := fmt.Sprintf("%d asked, %d built and not sent, %d not yet asked in it; %d not yet asked in the tiers after",
|
||||
asked, unsent, unasked, later)
|
||||
if len(again) > 0 {
|
||||
waits += "; " + strings.Join(again, ", ") + " set unasked, for the look to ask of build-agent (its own, ADR 0522 §6)"
|
||||
}
|
||||
p.State = inventory.PlanBuilding
|
||||
goesOnAgain(p)
|
||||
p.Delivery.Kept = nil
|
||||
p.Note = fmt.Sprintf("tier %d resumed by hand after the stop by %s (%s): %s", p.Tier, by, why, waits)
|
||||
return fmt.Sprintf("%s resumed at tier %d of %d, stopped by %s: %s; this verb asks nothing — the next look carries it "+
|
||||
"on as any walk", p.ID, p.Tier+1, len(p.Tiers), by, waits)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A walk stopped by hand while its tier waited — modules asked on the builder's keys, two built and never sent, the next
|
||||
// tier not yet asked — is resumed by `plans retry` (issue 1886): set building again, its stop cleared, the asks it holds
|
||||
// kept, so the next look carries it on as any walk. Nothing in it failed, so the retry of failed builds had nothing to do
|
||||
// and refused it, leaving a stopped R4e with no verb to carry its 48 modules on.
|
||||
func TestRetryResumesAWalkStoppedByHand(t *testing.T) {
|
||||
at := time.Date(2026, 10, 23, 0, 1, 53, 0, time.UTC)
|
||||
sent := at.Add(-time.Hour)
|
||||
p := inventory.Plan{ID: "plan-r4e", Repository: "novox/mesh-sdk", Branch: "main", Commit: "7aacbc16",
|
||||
Created: at.Add(-2 * time.Hour), State: inventory.PlanFailed, Tier: 1,
|
||||
Note: "stopped by a person at tier 1: issue 1886",
|
||||
Delivery: &inventory.PlanDelivery{Stopped: "a person (fix-1886@shanks)", StoppedWhy: "issue 1886",
|
||||
Kept: []inventory.PlanKept{{Module: "mesh-controller"}}},
|
||||
Modules: map[string]*inventory.PlanModule{}}
|
||||
tier0 := []string{"mesh-controller", "mesh-sdk", "pacman", "route-proxy"}
|
||||
for _, m := range tier0 {
|
||||
p.Modules[m] = &inventory.PlanModule{State: "built", BuiltAt: &sent, SentAt: &sent}
|
||||
}
|
||||
var tier1 []string
|
||||
for _, m := range []string{"build-agent", "mesh-tools"} {
|
||||
tier1 = append(tier1, m)
|
||||
p.Modules[m] = &inventory.PlanModule{State: "built", AskedAt: &sent, BuiltAt: &sent, Build: "build-" + m}
|
||||
}
|
||||
onKeys := []string{"mesh-build-go", "mesh-build-image", "mesh-build-node", "mesh-delivery-checker",
|
||||
"mesh-delivery-forge-bridge", "mesh-delivery-gate", "mesh-delivery-indexer", "mesh-delivery-view"}
|
||||
for _, m := range onKeys {
|
||||
tier1 = append(tier1, m)
|
||||
p.Modules[m] = &inventory.PlanModule{State: "asked", AskedAt: &sent, Key: true, Build: "build-" + m}
|
||||
}
|
||||
tier2 := []string{"gitea", "mesh-catalog", "mongodb"}
|
||||
for _, m := range tier2 {
|
||||
p.Modules[m] = &inventory.PlanModule{}
|
||||
}
|
||||
p.Tiers = [][]string{tier0, tier1, tier2}
|
||||
|
||||
if err := retryRefusal(p, []inventory.Plan{p}, ownPathOf(nil)); err != nil {
|
||||
t.Fatalf("a walk stopped by hand with its tier waiting was refused a retry: %v", err)
|
||||
}
|
||||
said := resumeStopped(&p)
|
||||
if p.State != inventory.PlanBuilding {
|
||||
t.Errorf("resumed walk is %s, wanted building", p.State)
|
||||
}
|
||||
if p.Delivery.Stopped != "" || p.Delivery.StoppedWhy != "" || len(p.Delivery.Kept) > 0 {
|
||||
t.Errorf("resumed walk still says it stopped: %+v", p.Delivery)
|
||||
}
|
||||
if !strings.Contains(p.Note, "resumed by hand") || !strings.Contains(p.Note, "a person (fix-1886@shanks)") {
|
||||
t.Errorf("the note does not record the hand resume and the stop it undid: %q", p.Note)
|
||||
}
|
||||
for _, m := range onKeys {
|
||||
s := p.Modules[m]
|
||||
if isBuildersOwn(m) {
|
||||
// Build-agent's own are asked of build-agent whatever the builder says (ADR 0522 §6): set unasked, for the look.
|
||||
if s.State != "" || s.Key || s.Build != "" || s.AskedAt != nil {
|
||||
t.Errorf("%s, build-agent's own, was left on the builder's key: %+v", m, s)
|
||||
}
|
||||
if !slices.Contains(unaskedOf(p), m) {
|
||||
t.Errorf("%s is not among what the look asks next: %v", m, unaskedOf(p))
|
||||
}
|
||||
continue
|
||||
}
|
||||
if s.State != "asked" || !s.Key || s.Build != "build-"+m {
|
||||
t.Errorf("%s's ask was not kept for the look to hand on: %+v", m, s)
|
||||
}
|
||||
}
|
||||
for _, w := range []string{"plan-r4e", "tier 2 of 3", "5 asked", "2 built and not sent", "3 not yet asked in it",
|
||||
"3 not yet asked in the tiers after", "mesh-build-go, mesh-build-image, mesh-build-node set unasked"} {
|
||||
if !strings.Contains(said, w) {
|
||||
t.Errorf("the answer %q does not say %q", said, w)
|
||||
}
|
||||
}
|
||||
|
||||
// What it still refuses: a walk stopped with nothing left in its tier, and one a related open walk runs beside.
|
||||
done := p
|
||||
done.State, done.Delivery = inventory.PlanFailed, &inventory.PlanDelivery{Stopped: "a person"}
|
||||
done.Modules = map[string]*inventory.PlanModule{}
|
||||
for _, tier := range done.Tiers {
|
||||
for _, m := range tier {
|
||||
done.Modules[m] = &inventory.PlanModule{State: "built", BuiltAt: &sent, SentAt: &sent}
|
||||
}
|
||||
}
|
||||
if err := retryRefusal(done, nil, ownPathOf(nil)); err == nil || !strings.Contains(err.Error(), "nothing in tier 1") {
|
||||
t.Errorf("a stopped walk with nothing left in its tier was retried: %v", err)
|
||||
}
|
||||
stopped := p
|
||||
stopped.State, stopped.Delivery = inventory.PlanFailed, &inventory.PlanDelivery{Stopped: "a person"}
|
||||
beside := inventory.Plan{ID: "plan-442", Repository: "novox/mesh-sdk", Branch: "release", Commit: "bbadbb2a",
|
||||
Created: at, State: inventory.PlanBuilding, Tiers: [][]string{{"mesh-sdk"}},
|
||||
Modules: map[string]*inventory.PlanModule{"mesh-sdk": {State: "asked"}}}
|
||||
if err := retryRefusal(stopped, []inventory.Plan{beside}, ownPathOf(nil)); err == nil || !strings.Contains(err.Error(), "one walk at a time") {
|
||||
t.Errorf("a stopped walk was resumed beside a related open walk: %v", err)
|
||||
}
|
||||
// A newer open plan of its repository and branch supersedes it: resuming would build what that one replaced.
|
||||
newerOpen := inventory.Plan{ID: "plan-newer", Repository: "novox/mesh-sdk", Branch: "main", Commit: "d00dfeed",
|
||||
Created: at, State: inventory.PlanBuilding, Tiers: [][]string{{"zsh"}},
|
||||
Modules: map[string]*inventory.PlanModule{"zsh": {State: "asked"}}}
|
||||
if err := retryRefusal(stopped, []inventory.Plan{stopped, newerOpen}, ownPathOf(nil)); err == nil ||
|
||||
!strings.Contains(err.Error(), "plan-newer supersedes it") {
|
||||
t.Errorf("a stopped walk was resumed under a newer open plan of its branch: %v", err)
|
||||
}
|
||||
// A newer plan holding a build of its tier not yet sent refuses it: resuming would send the older build over the newer.
|
||||
newerBuilt := inventory.Plan{ID: "plan-agent", Repository: "novox/mesh-controller", Branch: "main", Commit: "d00dfeed",
|
||||
Created: at, State: inventory.PlanDone, Tiers: [][]string{{"build-agent"}},
|
||||
Modules: map[string]*inventory.PlanModule{"build-agent": {State: "built"}}}
|
||||
if err := retryRefusal(stopped, []inventory.Plan{stopped, newerBuilt}, ownPathOf(nil)); err == nil ||
|
||||
!strings.Contains(err.Error(), "older build") {
|
||||
t.Errorf("a stopped walk was resumed over a newer plan of a build it has not sent: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A newer plan holding a module the stopped walk has not yet asked does not refuse its resume (review of issue 1886): that
|
||||
// module is asked at its followed branch's head when the walk reaches it, so no older build goes over the newer. Open and
|
||||
// related, the newer walk refuses it under one walk at a time; done, it does not. One holding a build the walk has not sent
|
||||
// still refuses it.
|
||||
func TestResumeIsNotRefusedByANewerPlanOfAModuleNotYetAsked(t *testing.T) {
|
||||
at := time.Date(2026, 10, 23, 0, 1, 53, 0, time.UTC)
|
||||
sent := at.Add(-time.Hour)
|
||||
p := inventory.Plan{ID: "plan-r4e", Repository: "novox/mesh-sdk", Branch: "main", Commit: "7aacbc16",
|
||||
Created: at.Add(-2 * time.Hour), State: inventory.PlanFailed, Tier: 1,
|
||||
Delivery: &inventory.PlanDelivery{Stopped: "a person", StoppedWhy: "issue 1886"},
|
||||
Tiers: [][]string{{"mesh-sdk"}, {"build-agent", "mesh-delivery-gate"}, {"mesh-delivery-builder", "gitea"}},
|
||||
Modules: map[string]*inventory.PlanModule{
|
||||
"mesh-sdk": {State: "built", BuiltAt: &sent, SentAt: &sent},
|
||||
"build-agent": {State: "built", BuiltAt: &sent},
|
||||
"mesh-delivery-gate": {State: "asked", AskedAt: &sent, Key: true},
|
||||
"mesh-delivery-builder": {}, "gitea": {}}}
|
||||
newer := func(state string, holds string) inventory.Plan {
|
||||
return inventory.Plan{ID: "plan-1792715180015327197", Repository: "novox/mesh-delivery", Branch: "main",
|
||||
Commit: "d00dfeed", Created: at, State: state, Tiers: [][]string{{holds}},
|
||||
Modules: map[string]*inventory.PlanModule{holds: {State: "asked"}}}
|
||||
}
|
||||
if err := retryRefusal(p, []inventory.Plan{p, newer(inventory.PlanDone, "mesh-delivery-builder")}, ownPathOf(nil)); err != nil {
|
||||
t.Errorf("a done newer plan of a module not yet asked refused the resume: %v", err)
|
||||
}
|
||||
if err := retryRefusal(p, []inventory.Plan{p, newer(inventory.PlanDone, "mesh-delivery-gate")}, ownPathOf(nil)); err != nil {
|
||||
t.Errorf("a done newer plan of a module on the builder's key refused the resume: %v", err)
|
||||
}
|
||||
if err := retryRefusal(p, []inventory.Plan{p, newer(inventory.PlanBuilding, "mesh-delivery-builder")}, ownPathOf(nil)); err == nil ||
|
||||
!strings.Contains(err.Error(), "one walk at a time") {
|
||||
t.Errorf("an open newer walk related to it did not refuse the resume under one walk at a time: %v", err)
|
||||
}
|
||||
if err := retryRefusal(p, []inventory.Plan{p, newer(inventory.PlanDone, "build-agent")}, ownPathOf(nil)); err == nil ||
|
||||
!strings.Contains(err.Error(), "older build") {
|
||||
t.Errorf("a newer plan of a built module not yet sent did not refuse the resume: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -457,6 +457,18 @@ func onlyTheBuildersOwn(ctx context.Context, source buildSource, path string) er
|
||||
if isBuildersOwn(module) {
|
||||
return nil
|
||||
}
|
||||
// **While no running builder says it writes references, build-agent builds every module** (issue 1886): judged by the
|
||||
// say the walk read when it has one, read here only for an ask that carries none.
|
||||
says, read := builderSayOf(ctx)
|
||||
if !read {
|
||||
var err error
|
||||
if says, err = builderSaysReferences(ctx); err != nil {
|
||||
says = true // not known: refused below, as before the say existed
|
||||
}
|
||||
}
|
||||
if !says {
|
||||
return nil
|
||||
}
|
||||
if module == "" {
|
||||
module = fmt.Sprintf("%s at %q", source, orRoot(path))
|
||||
}
|
||||
@@ -529,10 +541,22 @@ func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[s
|
||||
}
|
||||
// **Every module but the builder's own is taken from the builder's done key, asking no build** (novox/hq ADR 0503
|
||||
// §6, ADR 0501, ADR 0453 §5): the look settles it (settleFromKeys).
|
||||
// **Only while a running builder says it writes references** (issue 1886): until then the keys it writes could never
|
||||
// be delivered from, and the module is asked of build-agent as before R4e. A say not read leaves it unasked, said,
|
||||
// for the next look.
|
||||
if !isBuildersOwn(name) {
|
||||
sayBuilding(ctx, p.ID, now)
|
||||
takeFromKey(p, state, e, now)
|
||||
return
|
||||
says, err := builderSaysReferences(ctx)
|
||||
if err != nil {
|
||||
state.Why = "waits: whether the builder writes references cannot be read: " + firstLine(err.Error())
|
||||
return
|
||||
}
|
||||
if says {
|
||||
sayBuilding(ctx, p.ID, now)
|
||||
takeFromKey(p, state, e, now)
|
||||
return
|
||||
}
|
||||
// The ask's guard judges by this say, never by a second read that could fail or differ (review of issue 1886).
|
||||
ctx = withBuilderSay(ctx, says)
|
||||
}
|
||||
source := buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat}
|
||||
fmt.Printf(" tier %d: ", p.Tier)
|
||||
|
||||
@@ -44,6 +44,11 @@ func replay867(t *testing.T, recheck bool) {
|
||||
if _, err := exec.LookPath("go"); err != nil {
|
||||
t.Skip("no Go toolchain on this machine to build a judge with")
|
||||
}
|
||||
// The check's floor of free space (novox/hq issue 1834) reads the disk the test runs on, which is none of this
|
||||
// replay's business: a build node under the floor failed every mesh-controller check here (issue 1834, note 12).
|
||||
wasFree := builder.FreeBytes
|
||||
builder.FreeBytes = func(string) (uint64, error) { return 500 << 30, nil }
|
||||
t.Cleanup(func() { builder.FreeBytes = wasFree })
|
||||
r := theGroupOfIssue867(t)
|
||||
for _, c := range []struct {
|
||||
what, repo string
|
||||
|
||||
@@ -242,7 +242,8 @@ func build(ctx context.Context, run Runner, publish Publisher,
|
||||
// A fresh clone every time rather than a fetch into a tree that is already there. A build
|
||||
// that reuses a working tree can succeed because of something a previous build left behind,
|
||||
// and that is a build nobody can reproduce.
|
||||
if _, err := run(ctx, workspace, "git", cloneWith(credentials, "clone", "--quiet", repository, tree)...); err != nil {
|
||||
if err := cloneFromForge(ctx, run, workspace, repository, tree,
|
||||
cloneWith(credentials, "clone", "--quiet", repository, tree), say); err != nil {
|
||||
say("clone", "FAILED: %v", err)
|
||||
return Result{}, fmt.Errorf("cannot clone %s: %w", repository, err)
|
||||
}
|
||||
@@ -794,7 +795,7 @@ func contextFrom(ctx context.Context, run Runner, workspace, artifact, credentia
|
||||
os.RemoveAll(dir)
|
||||
}
|
||||
}()
|
||||
if _, err := run(ctx, workspace, "git", cloneWith(credentials, "clone", "--quiet", url, dir)...); err != nil {
|
||||
if err := cloneFromForge(ctx, run, workspace, url, dir, cloneWith(credentials, "clone", "--quiet", url, dir), say); err != nil {
|
||||
return "", fmt.Errorf("cannot clone %s: %w", url, err)
|
||||
}
|
||||
if err := recordedWithoutUserinfo(ctx, run, dir, url); err != nil {
|
||||
|
||||
@@ -428,7 +428,8 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
|
||||
// clone clones a repository and checks out ref; readAt keeps which, by directory.
|
||||
readAt := map[string]string{}
|
||||
clone := func(repository, ref, dir string) error {
|
||||
if _, err := run(ctx, root, "git", cloneWith(credentials, "clone", "--quiet", repository, dir)...); err != nil {
|
||||
if err := cloneFromForge(ctx, run, root, redact.redact(repository), filepath.Join(root, dir),
|
||||
cloneWith(credentials, "clone", "--quiet", repository, dir), say); err != nil {
|
||||
return fmt.Errorf("cannot clone %s: %s", redact.redact(repository), redact.redact(err.Error()))
|
||||
}
|
||||
if err := recordedWithoutUserinfo(ctx, run, filepath.Join(root, dir), repository); err != nil {
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
package builder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A clone from the forge waits out a resolver that did not answer, for a bounded time (novox/hq issue 1773).
|
||||
//
|
||||
// **Why.** A build or a check failed now and then on "Could not resolve host: novox.internal" while cloning, between
|
||||
// clones that resolved the same name a second before and after, and the whole delivery failed with it. The lookup
|
||||
// failing is the resolver's passing fault, not the repository's: nothing reached the forge, so cloning again cannot
|
||||
// do anything twice. This is issue 457's shape (a build waits out a registry held still) applied to the name lookup.
|
||||
//
|
||||
// **Only the lookup.** Any other failure of git — authentication, a repository or a ref that is not there, a 404 —
|
||||
// fails at once, as it always did: trying it again would only make a real failure slower.
|
||||
//
|
||||
// **Bounded, and loud.** Each retry is said in the build's log as the resolver's failure, and past lookupWait the
|
||||
// build fails saying the resolver did not answer for that long.
|
||||
|
||||
var (
|
||||
// lookupWait is how long a clone waits for a resolver that does not answer.
|
||||
lookupWait = 60 * time.Second
|
||||
// lookupFirstPause is the first pause between tries; each pause doubles, up to lookupMostPause.
|
||||
lookupFirstPause = time.Second
|
||||
)
|
||||
|
||||
// lookupMostPause is the longest pause between two tries.
|
||||
const lookupMostPause = 10 * time.Second
|
||||
|
||||
// lookupFailures are what git (through curl) and Go's resolver say when a name did not resolve, in lower case.
|
||||
var lookupFailures = []string{
|
||||
"could not resolve host",
|
||||
"temporary failure in name resolution",
|
||||
}
|
||||
|
||||
// failedLookup is whether an error is a name lookup that failed: git's "Could not resolve host", the system's
|
||||
// "Temporary failure in name resolution", or a lookup that timed out.
|
||||
func failedLookup(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
said := strings.ToLower(err.Error())
|
||||
for _, f := range lookupFailures {
|
||||
if strings.Contains(said, f) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return strings.Contains(said, "lookup ") && strings.Contains(said, "i/o timeout")
|
||||
}
|
||||
|
||||
// cloneFromForge runs `git clone` of url into dest (git's arguments in args, run from dir), and while it fails on the
|
||||
// name lookup, clones again with a growing pause until lookupWait has passed. What a failed try left at dest is
|
||||
// removed before the next. Any other failure is returned at once.
|
||||
func cloneFromForge(ctx context.Context, run Runner, dir, url, dest string, args []string,
|
||||
say func(step, format string, args ...any)) error {
|
||||
_, err := run(ctx, dir, "git", args...)
|
||||
if !failedLookup(err) {
|
||||
return err
|
||||
}
|
||||
started := time.Now()
|
||||
pause := lookupFirstPause
|
||||
for {
|
||||
elapsed := time.Since(started)
|
||||
left := lookupWait - elapsed
|
||||
if left <= 0 {
|
||||
say("clone", "the resolver did not answer for %s, cloning %s, for %s; the clone fails",
|
||||
urlHost(url), url, elapsed.Round(time.Second))
|
||||
return fmt.Errorf("the resolver did not answer a lookup of %s for %s, cloning %s: %w",
|
||||
urlHost(url), elapsed.Round(time.Millisecond), url, err)
|
||||
}
|
||||
wait := min(pause, left)
|
||||
say("clone", "the resolver did not answer for %s, cloning %s; trying again in %s, for up to %s "+
|
||||
"(novox/hq issue 1773)", urlHost(url), url, wait, lookupWait)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return fmt.Errorf("stopped while waiting for the resolver to answer for %s: %w (last: %v)",
|
||||
urlHost(url), ctx.Err(), err)
|
||||
case <-time.After(wait):
|
||||
}
|
||||
pause = min(pause*2, lookupMostPause)
|
||||
if err := os.RemoveAll(dest); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err = run(ctx, dir, "git", args...); !failedLookup(err) {
|
||||
if err == nil {
|
||||
say("clone", "the resolver answered for %s after %s; the clone worked", urlHost(url),
|
||||
time.Since(started).Round(time.Millisecond))
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// urlHost is the host (and port) a URL names, or the URL itself when it names none.
|
||||
func urlHost(url string) string {
|
||||
rest := url
|
||||
if _, after, ok := strings.Cut(url, "://"); ok {
|
||||
rest = after
|
||||
}
|
||||
if i := strings.IndexAny(rest, "/?#"); i >= 0 {
|
||||
rest = rest[:i]
|
||||
}
|
||||
if i := strings.LastIndex(rest, "@"); i >= 0 {
|
||||
rest = rest[i+1:]
|
||||
}
|
||||
if rest == "" {
|
||||
return url
|
||||
}
|
||||
return rest
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
package builder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Issue 1773: a build or a check failed now and then on "Could not resolve host: novox.internal" while cloning from
|
||||
// the forge, between clones that resolved the same name, and its whole delivery failed with it. A clone that fails on
|
||||
// the name lookup is tried again within a bound, says each retry in the build's log as the resolver's failure, and
|
||||
// only past the bound fails, saying the resolver did not answer. Any other failure of git fails at once, as before.
|
||||
|
||||
const lookupFailure = "exit status 128\nfatal: unable to access 'http://novox.internal:20000/novox/x.git/': " +
|
||||
"Could not resolve host: novox.internal"
|
||||
|
||||
// failingClones is a runner whose first n forge clones fail with failure, then clone as r does; n < 0 fails every one.
|
||||
type failingClones struct {
|
||||
r *recorded
|
||||
n int
|
||||
failure string
|
||||
clones int
|
||||
}
|
||||
|
||||
func (f *failingClones) run(ctx context.Context, dir, name string, args ...string) (string, error) {
|
||||
if name == "git" && isACloneOf(args) {
|
||||
f.clones++
|
||||
if f.n < 0 || f.clones <= f.n {
|
||||
return "", fmt.Errorf("git %s: %s", strings.Join(args, " "), f.failure)
|
||||
}
|
||||
}
|
||||
return f.r.run(ctx, dir, name, args...)
|
||||
}
|
||||
|
||||
func isACloneOf(args []string) bool {
|
||||
for _, a := range args {
|
||||
if a == "clone" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// buildLog collects what a build says in its own log.
|
||||
func buildLog() (Log, func() []string) {
|
||||
var mu sync.Mutex
|
||||
var lines []string
|
||||
return func(step, message string) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
lines = append(lines, step+": "+message)
|
||||
}, func() []string {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return append([]string(nil), lines...)
|
||||
}
|
||||
}
|
||||
|
||||
const nothingToBuild = `{"module":"x","version":"1"}`
|
||||
|
||||
// 1773/AC1: a clone failing twice on the lookup, then working, is tried again, each retry said, and the build goes on.
|
||||
func TestACloneThatFailsOnTheLookupIsTriedAgainAndTheBuildGoesOn(t *testing.T) {
|
||||
quickLookups(t)
|
||||
r, workspace := aRepository(t, nothingToBuild, nil)
|
||||
f := &failingClones{r: r, n: 2, failure: lookupFailure}
|
||||
log, said := buildLog()
|
||||
_, err := Build(context.Background(), f.run, r, "http://novox.internal:20000/novox/x.git", "", "", workspace, nil,
|
||||
Npmrc{}, GitCredential{}, log)
|
||||
if err != nil {
|
||||
t.Fatalf("a clone the resolver failed twice, then answered, ended the build: %v", err)
|
||||
}
|
||||
if f.clones != 3 {
|
||||
t.Errorf("the clone ran %d times, want 3 (two lookups failed, the third worked)", f.clones)
|
||||
}
|
||||
retries := 0
|
||||
for _, line := range said() {
|
||||
if strings.Contains(line, "the resolver did not answer") && strings.Contains(line, "novox.internal") {
|
||||
retries++
|
||||
}
|
||||
}
|
||||
if retries != 2 {
|
||||
t.Errorf("the build's log said %d retries as the resolver's failure, want 2:\n%s", retries,
|
||||
strings.Join(said(), "\n"))
|
||||
}
|
||||
}
|
||||
|
||||
// 1773/AC2: a lookup that keeps failing past the bound fails the build, saying the resolver did not answer.
|
||||
func TestACloneWhoseLookupNeverAnswersFailsSayingTheResolverDidNotAnswer(t *testing.T) {
|
||||
quickLookups(t)
|
||||
r, workspace := aRepository(t, nothingToBuild, nil)
|
||||
f := &failingClones{r: r, n: -1, failure: lookupFailure}
|
||||
_, err := Build(context.Background(), f.run, r, "http://novox.internal:20000/novox/x.git", "", "", workspace, nil,
|
||||
Npmrc{}, GitCredential{}, nil)
|
||||
if err == nil {
|
||||
t.Fatal("a clone whose lookup never answered built")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "the resolver did not answer") ||
|
||||
!strings.Contains(err.Error(), "http://novox.internal:20000/novox/x.git") {
|
||||
t.Errorf("the failure does not say the resolver did not answer for the forge's URL: %v", err)
|
||||
}
|
||||
if f.clones < 2 {
|
||||
t.Errorf("the clone ran %d times; a failed lookup is tried again within the bound", f.clones)
|
||||
}
|
||||
}
|
||||
|
||||
// 1773/AC3: any other failure of git (authentication, a missing repository) is not tried again and fails at once.
|
||||
func TestACloneFailingForAnotherReasonIsNotTriedAgain(t *testing.T) {
|
||||
quickLookups(t)
|
||||
for _, failure := range []string{
|
||||
"exit status 128\nfatal: Authentication failed for 'http://novox.internal:20000/novox/x.git/'",
|
||||
"exit status 128\nremote: Not found.\nfatal: repository 'http://novox.internal:20000/novox/x.git/' not found",
|
||||
"exit status 128\nfatal: unable to access 'http://novox.internal:20000/novox/x.git/': The requested URL returned error: 404",
|
||||
} {
|
||||
r, workspace := aRepository(t, nothingToBuild, nil)
|
||||
f := &failingClones{r: r, n: -1, failure: failure}
|
||||
_, err := Build(context.Background(), f.run, r, "http://novox.internal:20000/novox/x.git", "", "", workspace, nil,
|
||||
Npmrc{}, GitCredential{}, nil)
|
||||
if err == nil || !strings.Contains(err.Error(), "cannot clone") {
|
||||
t.Errorf("a clone failing with %q said %v", failure, err)
|
||||
}
|
||||
if f.clones != 1 {
|
||||
t.Errorf("a clone failing with %q ran %d times; it is not a lookup and is not tried again", failure, f.clones)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// quickLookups shortens the bound and the pauses, so a test waits for milliseconds.
|
||||
func quickLookups(t *testing.T) {
|
||||
t.Helper()
|
||||
wasBound, wasFirst := lookupWait, lookupFirstPause
|
||||
lookupWait, lookupFirstPause = 200*time.Millisecond, 10*time.Millisecond
|
||||
t.Cleanup(func() { lookupWait, lookupFirstPause = wasBound, wasFirst })
|
||||
}
|
||||
|
||||
// 1773/AC1, 1773/AC3: a check's clones are the same: a failed lookup is tried again, and the failure after it (here
|
||||
// an authentication refused) ends the check at once, not tried again.
|
||||
func TestACheckCloneTriesAFailedLookupAgain(t *testing.T) {
|
||||
quickLookups(t)
|
||||
clones := 0
|
||||
run := func(_ context.Context, _ string, name string, args ...string) (string, error) {
|
||||
if name == "git" && isACloneOf(args) {
|
||||
clones++
|
||||
if clones <= 2 {
|
||||
return "", fmt.Errorf("git clone: %s", lookupFailure)
|
||||
}
|
||||
return "", fmt.Errorf("git clone: exit status 128\nfatal: Authentication failed")
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
_, err := Check(context.Background(), run, CheckSpec{ID: "build-1", Owner: "novox", Repo: "x", Number: 1,
|
||||
Repository: "http://novox.internal:20000/novox/x.git", Ref: "abc"}, t.TempDir(), "registry.invalid",
|
||||
GitCredential{}, nil)
|
||||
if err == nil || !strings.Contains(err.Error(), "cannot clone") || !strings.Contains(err.Error(), "Authentication failed") {
|
||||
t.Fatalf("the check said %v", err)
|
||||
}
|
||||
if clones != 3 {
|
||||
t.Errorf("the check's clone ran %d times, want 3: two failed lookups, then the refusal, not tried again", clones)
|
||||
}
|
||||
}
|
||||
@@ -9,6 +9,7 @@ package builderkeys
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
@@ -309,3 +310,99 @@ func firstLine(s string) string {
|
||||
line, _, _ := strings.Cut(strings.TrimSpace(s), "\n")
|
||||
return line
|
||||
}
|
||||
|
||||
// The builder's say of what its keys hold (novox/hq issue 1886): each running builder instance keeps a key, named by its
|
||||
// node, in the builder's own announced bucket of instances, saying what the keys it writes hold. An instance that writes
|
||||
// references (ADR 0498 §1) says WritesReferences there; a builder from before R4e keeps no such key, and has no bucket.
|
||||
// The bucket is the builder's and the shape is read by the controller alone, so neither is in mesh-sdk.
|
||||
const (
|
||||
// BuilderInstancesBucket is the builder's bucket of its running instances, `<owner module>_<name>`.
|
||||
BuilderInstancesBucket = delivery.BuilderModule + "_instances"
|
||||
// WritesReferences is the word an instance says when every done key it writes names where its artifacts are.
|
||||
WritesReferences = "references"
|
||||
)
|
||||
|
||||
// instanceSays is an instance's value in BuilderInstancesBucket: what the keys it writes hold.
|
||||
type instanceSays struct {
|
||||
Writes []string `json:"writes"`
|
||||
}
|
||||
|
||||
// saysReferences says whether any instance's value says it writes references. A value that does not decode says
|
||||
// nothing.
|
||||
func saysReferences(values [][]byte) bool {
|
||||
for _, v := range values {
|
||||
var says instanceSays
|
||||
if json.Unmarshal(v, &says) != nil {
|
||||
continue
|
||||
}
|
||||
for _, w := range says.Writes {
|
||||
if w == WritesReferences {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// SaysReferences says whether a running builder instance says, in its bucket of instances, that it writes references. No
|
||||
// bucket (the builder from before R4e, which never made it) and no key say no; a bucket that cannot be read is an error.
|
||||
func (r Reader) SaysReferences(ctx context.Context) (bool, error) {
|
||||
kv, err := r.bucket(ctx, BuilderInstancesBucket)
|
||||
if err != nil || kv == nil {
|
||||
return false, err
|
||||
}
|
||||
keys, err := kv.Keys(ctx)
|
||||
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("listing the builder's instances: %w", err)
|
||||
}
|
||||
var values [][]byte
|
||||
for _, key := range keys {
|
||||
e, err := kv.Get(ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("the builder's instance %s: %w", key, err)
|
||||
}
|
||||
values = append(values, e.Value())
|
||||
}
|
||||
return saysReferences(values), nil
|
||||
}
|
||||
|
||||
// WatchSays calls wake for every change of the builder's bucket of instances from now on, until ctx ends: what makes the
|
||||
// look judge an open walk again when the builder starts or stops saying it writes references (issue 1886), with no
|
||||
// timer. It says false, and watches nothing, while the bucket is not there; the caller asks again when the builder
|
||||
// writes a key, which a builder that makes the bucket does.
|
||||
func WatchSays(ctx context.Context, js jetstream.JetStream, wake func()) (bool, error) {
|
||||
kv, err := js.KeyValue(ctx, BuilderInstancesBucket)
|
||||
if errors.Is(err, jetstream.ErrBucketNotFound) {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("the bucket %s: %w", BuilderInstancesBucket, err)
|
||||
}
|
||||
w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly())
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("watching %s: %w", BuilderInstancesBucket, err)
|
||||
}
|
||||
go func() {
|
||||
defer w.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case e, ok := <-w.Updates():
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if e != nil {
|
||||
wake() // a put and a delete both change what the builder says
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
return true, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
//go:build integration
|
||||
|
||||
package builderkeys
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// 1886/AC1 and AC4 on a real bus: no bucket of instances says no and watches nothing; an instance's key saying
|
||||
// references says yes, and its put and its delete each wake the watcher, with no timer.
|
||||
func TestTheBuildersSayIsReadAndWatchedOnTheBus(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cancel()
|
||||
js := aBus(t)
|
||||
r := Reader{JS: js}
|
||||
if says, err := r.SaysReferences(ctx); err != nil || says {
|
||||
t.Fatalf("no bucket: %v %v", says, err)
|
||||
}
|
||||
if on, err := WatchSays(ctx, js, func() {}); err != nil || on {
|
||||
t.Fatalf("no bucket is watched: %v %v", on, err)
|
||||
}
|
||||
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: BuilderInstancesBucket, History: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
woken := make(chan struct{}, 4)
|
||||
if on, err := WatchSays(ctx, js, func() { woken <- struct{}{} }); err != nil || !on {
|
||||
t.Fatalf("the bucket is not watched: %v %v", on, err)
|
||||
}
|
||||
wake := func(what string) {
|
||||
t.Helper()
|
||||
select {
|
||||
case <-woken:
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("%s woke nothing", what)
|
||||
}
|
||||
}
|
||||
put(t, kv, "shanks", []byte(`{"writes":["references"]}`))
|
||||
wake("a put")
|
||||
if says, err := r.SaysReferences(ctx); err != nil || !says {
|
||||
t.Fatalf("an instance saying references: %v %v", says, err)
|
||||
}
|
||||
if err := kv.Delete(ctx, "shanks"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wake("a delete")
|
||||
if says, err := r.SaysReferences(ctx); err != nil || says {
|
||||
t.Fatalf("no instance left: %v %v", says, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package builderkeys
|
||||
|
||||
import "testing"
|
||||
|
||||
// 1886/AC1: a builder that keeps no say, or a say without the word, is a builder that does not write references; one
|
||||
// instance that says it is enough, since it builds every key done without a reference again (ADR 0498 §8).
|
||||
func TestTheBuilderSaysReferencesOnlyWhenAnInstanceSaysTheWord(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
values [][]byte
|
||||
want bool
|
||||
}{
|
||||
{"no instance", nil, false},
|
||||
{"an instance saying nothing", [][]byte{[]byte(`{"writes":[]}`)}, false},
|
||||
{"a value not read", [][]byte{[]byte(`{not json`)}, false},
|
||||
{"another word", [][]byte{[]byte(`{"writes":["digests"]}`)}, false},
|
||||
{"one instance says it", [][]byte{[]byte(`{"writes":["digests"]}`), []byte(`{"v":1,"writes":["references"]}`)}, true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := saysReferences(c.values); got != c.want {
|
||||
t.Errorf("%s: says references %v, want %v", c.name, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user