Merge branch 'fix/1126-answer-once' into fix/1126-seed-same-3

2026-10-28 12:11:50 +01:00
parent faf6a23d91 cd1d2ff447
commit 16df7d0b7a
9 changed files with 534 additions and 80 deletions
@@ -10,6 +10,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"testing"
 	"time"

@@ -306,6 +307,12 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
 		Transient(fmt.Errorf("the revisions of it2.mailer.domain: %w", context.DeadlineExceeded)) {
 		t.Error("Transient misjudges a bus that does not answer, or a refusal")
 	}
+	// A read the bus did not answer, timed out or not, is tried again later; the run's own end and a refusal are not.
+	if !BusFailed(fmt.Errorf("the revisions of it2.mailer.domain: %w", context.DeadlineExceeded)) || !BusFailed(fmt.Errorf("x: %w", nats.ErrNoResponders)) ||
+		!BusFailed(fmt.Errorf("the have of web: %w", nats.ErrTimeout)) || BusFailed(fmt.Errorf("x: %w", context.Canceled)) ||
+		BusFailed(fmt.Errorf("nats says ${dir:mesh-state}, and nats declares no directory")) || BusFailed(nil) {
+		t.Error("BusFailed misjudges a read the bus did not answer, the run's end, or a refusal")
+	}

 	// Issue 1126, C2: a config value is laid as the controller lays it (catalogue.LaidValue), from the mesh's, the node's
 	// or the module's own and never from a default or an unset one, both of this node's and of what another node serves:
@@ -428,23 +435,28 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
 		name, in, held string
 		slots          delivery.Seals
 		kept           bool
+		fails          bool // the have's read the bus did not answer: no composition, made again later (2026-10-28)
 	}{
-		{"the have holds the rotated value at its slot", "keeper", rotatedBox, delivery.Seals{slot: rotatedBox}, true},
-		{"the have holds the value before the rotation", "keeper", beforeBox, delivery.Seals{slot: beforeBox}, false},
-		{"a have written before slots holds the rotated value", "keeper", rotatedBox, nil, true},
-		{"another module holds the same value", "lemurs", rotatedBox, delivery.Seals{"lemurs.grant-x#sealed": rotatedBox}, false},
-		{"another resource of keeper holds the same value", "keeper", rotatedBox, delivery.Seals{"keeper.other#sealed": rotatedBox}, false},
+		{"the have holds the rotated value at its slot", "keeper", rotatedBox, delivery.Seals{slot: rotatedBox}, true, false},
+		{"the have holds the value before the rotation", "keeper", beforeBox, delivery.Seals{slot: beforeBox}, false, false},
+		{"a have written before slots holds the rotated value", "keeper", rotatedBox, nil, true, false},
+		{"another module holds the same value", "lemurs", rotatedBox, delivery.Seals{"lemurs.grant-x#sealed": rotatedBox}, false, false},
+		{"another resource of keeper holds the same value", "keeper", rotatedBox, delivery.Seals{"keeper.other#sealed": rotatedBox}, false, false},
+		{"the bus does not answer for the have", "keeper", rotatedBox, nil, false, true},
 	} {
 		restarted := env
 		restarted.Running = func() []delivery.Composed { return nil }
-		restarted.Held = func(module string) (delivery.Seals, []string) {
+		restarted.Held = func(module string) (delivery.Seals, []string, error) {
 			if module != row.in {
-				return nil, nil
+				return nil, nil, nil
+			}
+			if row.fails {
+				return nil, nil, fmt.Errorf("the have of %s: %w", module, nats.ErrTimeout)
 			}
 			if row.slots == nil {
-				return nil, []string{row.held}
+				return nil, []string{row.held}, nil
 			}
-			return row.slots, nil
+			return row.slots, nil, nil
 		}
 		restarted.Unseal = func(box string) ([]byte, error) {
 			b.mu.Lock()
@@ -475,6 +487,12 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
 				composed = string(u.Value)
 			}
 		}
+		if row.fails {
+			if composed != "" || !again.owed.Load() {
+				t.Errorf("%s: composed %v and owes a pass again %v; want nothing handed and a pass owed", row.name, composed != "", again.owed.Load())
+			}
+			continue
+		}
 		if composed == "" {
 			t.Fatalf("%s: keeper was not composed", row.name)
 		}
@@ -497,11 +515,11 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
 		}
 		return nil, fmt.Errorf("not sealed to n1: %s", box)
 	}
-	coreEnv.Held = func(module string) (delivery.Seals, []string) {
+	coreEnv.Held = func(module string) (delivery.Seals, []string, error) {
 		if module == agent.NodeModule {
-			return delivery.Seals{"core-box#sealed": rotatedBox}, nil
+			return delivery.Seals{"core-box#sealed": rotatedBox}, nil, nil
 		}
-		return nil, nil
+		return nil, nil, nil
 	}
 	if got := delivery.SealsIn(New(coreEnv).keepRunning(ownComp, nil))["core-box#sealed"]; got != rotatedBox {
 		t.Errorf("the node's own core box after a restart is %s, want the one its key holds %s", got, rotatedBox)
@@ -606,7 +624,11 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
 	readOnly := &fakeBus{kv: map[string][]byte{delivery.ManifestsBucket + "/" + key: man, delivery.ManifestsBucket + "/" + kkey: kman}}
 	senv := readOnly.env()
 	senv.Watch = sb.watch
+	var ownFails atomic.Bool // the vault's own on a bus that does not answer (2026-10-28, ace's answer to seed-7)
 	senv.Own = func(context.Context, map[string]any) (json.RawMessage, error) {
+		if ownFails.Load() {
+			return nil, fmt.Errorf("the vault's own could not be asked: %w", nats.ErrNoResponders)
+		}
 		return json.RawMessage(`{"node":"n1","key":"sk","own":[]}`), nil // nothing issued for keeper: it is not placed here
 	}
 	// What n1 runs: lemurs and its own composition, as its composer composes them from the trunk.
@@ -755,6 +777,77 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
 		t.Errorf("answers handed again for nothing changed: %+v", a)
 	case <-time.After(100 * time.Millisecond):
 	}
+	// Issue 1126, 2026-10-28 (shanks answered probe/1126-seed-7 at revision 84 every 13-17 s): a candidate revision is
+	// answered once. A module's have whose health or words alone move (each look rewrites them), and a placement, a
+	// module or a config value written again with the value it held, answer nothing again; a new revision answers its
+	// group alone, and what runs moving answers every group once.
+	answering := func() int {
+		readOnly.mu.Lock()
+		defer readOnly.mu.Unlock()
+		n := 0
+		for _, l := range readOnly.said {
+			if strings.HasPrefix(l, "answering placement candidate ") {
+				n++
+			}
+		}
+		return n
+	}
+	drain := func() {
+		for {
+			select {
+			case answers = <-answered:
+			case <-time.After(150 * time.Millisecond):
+				return
+			}
+		}
+	}
+	lemursHave := func(detail string, rev uint64) {
+		raw, err := delivery.EncodeHave(delivery.Have{FP: fp, Composed: lemursDigest, Phase: delivery.PhaseRunning,
+			Health: delivery.HealthFailed, Detail: detail})
+		if err != nil {
+			t.Fatal(err)
+		}
+		hk, _ := delivery.HaveKey("n1", "lemurs")
+		sb.put(delivery.HaveBucket, hk, raw, rev)
+	}
+	lemursHave("its tool lemurs_health: no link to the bus is open", 10)
+	drain()
+	settled := answering()
+	lemursHave("its tool lemurs_health: no answer within 10s", 11)
+	lemursHave("its tool lemurs_health: no answer within 20s", 12)
+	sb.put(delivery.PlacementBucket, "placement.n1", tp, 2)
+	lraw, _ := delivery.EncodeModule(mods["lemurs"])
+	sb.put(delivery.ModulesBucket, "lemurs", lraw, 2)
+	drain()
+	if got := answering(); got != settled {
+		t.Errorf("answered %d more time(s) with no candidate revision and no input of the node changed", got-settled)
+	}
+	candidate("same", 8, "lemurs")
+	waitFor("the new revision answered", func(a map[string]delivery.CandidateAnswer) bool { return a["same"].Revision == 8 })
+	drain()
+	if got := answering(); got != settled+1 {
+		t.Errorf("a new revision of one group answered %d group(s), want 1", got-settled)
+	}
+	settled = answering()
+	hk, _ := delivery.HaveKey("n1", "lemurs")
+	moved, _ := delivery.EncodeHave(delivery.Have{FP: fp, Composed: "sha256:" + strings.Repeat("e", 64), Phase: delivery.PhaseRunning})
+	sb.put(delivery.HaveBucket, hk, moved, 13)
+	drain()
+	if got := answering(); got != settled+len(answers) {
+		t.Errorf("what runs moved, and %d answer(s) were composed again for %d group(s)", got-settled, len(answers))
+	}
+	// A read the bus did not answer is no answer to keep: the group is answered again after a bound, though nothing moved.
+	laterAgain = []time.Duration{20 * time.Millisecond}
+	ownFails.Store(true)
+	candidate("adds", 9, "lemurs", "keeper")
+	waitFor("the answer met the vault not answering", func(a map[string]delivery.CandidateAnswer) bool {
+		return a["adds"].Revision == 9 && strings.Contains(a["adds"].Refused, "no responders")
+	})
+	ownFails.Store(false)
+	waitFor("answered again once the bus answers, with nothing else moved", func(a map[string]delivery.CandidateAnswer) bool {
+		return a["adds"].Revision == 9 && a["adds"].Verdict == delivery.CandidateDiffers
+	})
+	drain()
 	k, _ := delivery.PlacementCandidateKey("same", "n1")
 	sb.del(delivery.PlacementCandidatesBucket, k)
 	waitFor("the deleted group's answer stays", func(a map[string]delivery.CandidateAnswer) bool { _, ok := a["same"]; return !ok && len(a) == 4 })