modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/askended.go

@@ -26,17 +26,33 @@ import (
 // askEndedSubject is the teller's event ask-ended, as the manifest's consumes names it (`mesh-delivery-teller.ask-ended`).
 var askEndedSubject = process.Account{Module: buckets.TellerModule}.EventSubject(tellerevents.AskEnded)
 
+// staleOf says why the ask at a key is stale, as the pass reads it (passer.staleAsk), or "" when it is not or when the
+// trunk is not read; nil reads nothing.
+type staleOf func(key string) string
+
+// staleOfWorld is staleOf over the world the pass reads: the same rule as the pass, over the same cache.
+func staleOfWorld(w *world) staleOf {
+	p := &passer{w: w}
+	return func(key string) string {
+		trunk := w.trunk()
+		if len(trunk) == 0 {
+			return ""
+		}
+		return p.staleAsk(key, trunk)
+	}
+}
+
 // hearAskEnded hears the teller's ask-ended on the assigner's one hub of its durable consumer on EVENTS (registered before
 // the hub starts): an ask of the assigner's that ended at the operator channel without an answer while its key stands is
 // written again under a new asking, so the teller asks it again (issue 1966, note 151). asksKV is the assigner's asks; the
 // puts are opened at each event, since a put's record ask is not written again (ADR 0546).
-func hearAskEnded(ctx context.Context, js jetstream.JetStream, hub *process.EventHub, asksKV jetstream.KeyValue, logf func(string, ...any)) error {
+func hearAskEnded(ctx context.Context, js jetstream.JetStream, hub *process.EventHub, asksKV jetstream.KeyValue, stale staleOf, logf func(string, ...any)) error {
 	return hub.Handle(askEndedSubject, func(m jetstream.Msg) {
 		var putsKV jetstream.KeyValue
 		if kv, err := js.KeyValue(ctx, buckets.Puts); err == nil {
 			putsKV = kv
 		}
-		if err := askAgain(ctx, asksKV, putsKV, m.Data(), time.Now(), logf); err != nil {
+		if err := askAgain(ctx, asksKV, putsKV, stale, m.Data(), time.Now(), logf); err != nil {
 			logf("%s: %v; it is heard again", m.Subject(), err)
 			_ = m.Nak()
 			return
@@ -55,13 +71,13 @@ const channelAsksVerb = "seat:" + oc.Seat + "." + oc.VerbAsks
 // assigner's consumer: an ask-ended the teller said before the consumer existed is not delivered to it (issue 1966 note 155,
 // ADR 0668 §4), so each held ask whose asking ended there expired or cancelled is written again as that event writes it
 // (askAgain, with every rule and its compare-and-set). The read is a kept call (CallKept), made off the hub's start.
-func readEndedAtBind(ctx context.Context, js jetstream.JetStream, hub *process.EventHub, asksKV jetstream.KeyValue, bus caller, wait callWait, logf func(string, ...any)) {
+func readEndedAtBind(ctx context.Context, js jetstream.JetStream, hub *process.EventHub, asksKV jetstream.KeyValue, stale staleOf, bus caller, wait callWait, logf func(string, ...any)) {
 	if asksKV == nil || bus == nil {
 		return
 	}
 	hub.OnBound(func(created time.Time) {
 		go func() {
-			if err := endedAtBind(ctx, js, asksKV, bus, wait, created, time.Now(), logf); err != nil {
+			if err := endedAtBind(ctx, js, asksKV, stale, bus, wait, created, time.Now(), logf); err != nil {
 				logf("the operator channel's ended asks are not read at the bind of %s (%v): an ask that ended there before is asked again when the teller says it ended", created.Format(time.RFC3339), err)
 			}
 		}()
@@ -119,7 +135,7 @@ func (cw callWait) keep(ctx context.Context, id, callee string, call func() (jso
 
 // endedAtBind is readEndedAtBind's one read: the held asks, the seat's verb (a kept call whose wait is kept, wait), and
 // each ended asking written again.
-func endedAtBind(ctx context.Context, js jetstream.JetStream, asksKV jetstream.KeyValue, bus caller, wait callWait, created, now time.Time, logf func(string, ...any)) error {
+func endedAtBind(ctx context.Context, js jetstream.JetStream, asksKV jetstream.KeyValue, stale staleOf, bus caller, wait callWait, created, now time.Time, logf func(string, ...any)) error {
 	keys, err := asksKV.Keys(ctx)
 	if errors.Is(err, jetstream.ErrNoKeysFound) {
 		return nil
@@ -157,7 +173,7 @@ func endedAtBind(ctx context.Context, js jetstream.JetStream, asksKV jetstream.K
 	for _, e := range ended {
 		body, err := tellerevents.EncodeEnded(e)
 		if err == nil {
-			err = askAgain(ctx, asksKV, putsKV, body, now, logf)
+			err = askAgain(ctx, asksKV, putsKV, stale, body, now, logf)
 		}
 		if err != nil {
 			return err
@@ -170,7 +186,7 @@ func endedAtBind(ctx context.Context, js jetstream.JetStream, asksKV jetstream.K
 // askAgain writes the ask the teller said ended again under a new asking, by compare-and-set at the revision that holds the
 // asking that ended (assign.AskedAgain), and says in the journal what it wrote or why it wrote nothing. Only a bus that
 // could not be read or written is an error, so the event is heard again; a body that does not read is said and dropped.
-func askAgain(ctx context.Context, asksKV, putsKV jetstream.KeyValue, data []byte, now time.Time, logf func(string, ...any)) error {
+func askAgain(ctx context.Context, asksKV, putsKV jetstream.KeyValue, stale staleOf, data []byte, now time.Time, logf func(string, ...any)) error {
 	e, err := tellerevents.DecodeEnded(data)
 	if err != nil {
 		logf("the teller's %s holds what is not an ended ask, and is dropped: %v", tellerevents.AskEnded, err)
@@ -205,6 +221,13 @@ func askAgain(ctx context.Context, asksKV, putsKV jetstream.KeyValue, data []byt
 		}
 	}
 	value, why := assign.AskedAgain(e, current, put, now)
+	if why == "" && stale != nil {
+		// An ask the pass takes away is not written again: it is about a move or a composition that has been left behind, and the
+		// pass removes it (issue 1966 note 165).
+		if s := stale(e.Key); s != "" {
+			why = "it is stale and left to the pass to take away: " + s
+		}
+	}
 	if why != "" {
 		logf("%s: the ask %s ended (%s) at the operator channel without an answer, and is not written again: %s", e.Key, e.Ask, e.Outcome, why)
 		return nil

modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go

@@ -1037,6 +1037,22 @@ func TestABackupAnswerIsWrittenIntoTheNodesWant(t *testing.T) {
 	if w2.Words == w1.Words {
 		t.Errorf("the second answer left the want as it was: %+v", w2)
 	}
+	// Issue 1966 note 165 (AC165-1): the move is superseded, the trunk and then the want naming another build; the pass takes
+	// the backup ask away and clears the condition of the failed backup.
+	fOther := meshcatalogclient.SourcePrefix2 + strings.Repeat("c", 64)
+	b.fixture(t, "web", fOther, `{"module":"web","version":"2"}`) // the trunk moves on: the want follows
+	for deadline := time.Now().Add(15 * time.Second); ; time.Sleep(50 * time.Millisecond) {
+		cleared := false
+		for k, c := range b.conditionsSaid(t) {
+			cleared = cleared || (strings.Contains(k, "backup") && c.State == occ.StateCleared)
+		}
+		if !b.hasKey(buckets.Asks, akey) && cleared {
+			break
+		}
+		if time.Now().After(deadline) {
+			t.Fatalf("the backup of a move the want left behind: ask there %v, condition cleared %v", b.hasKey(buckets.Asks, akey), cleared)
+		}
+	}
 }
 
 const recorded = `{"module":"web","version":"1","upgrade":{"policy":"record","why":"a person takes each build"}}`

modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go

@@ -481,10 +481,10 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
 	if js != nil {
 		// The teller's ask-ended (issue 1966): an ask ended at the channel without an answer is written again; registered
 		// before hearReadsChanged starts the hub.
-		if err := hearAskEnded(ctx, js, hub, kvs[buckets.Asks], logger.Printf); err != nil {
+		if err := hearAskEnded(ctx, js, hub, kvs[buckets.Asks], staleOfWorld(w), logger.Printf); err != nil {
 			logger.Printf("%s is not heard, so an ask that ends at the operator channel without an answer is not asked again until the assigner starts again: %v", askEndedSubject, err)
 		}
-		readEndedAtBind(ctx, js, hub, kvs[buckets.Asks], bus, callWait{heard: func() bool { return w.seatHeard(oc.Seat) }, findings: findings}, logger.Printf)
+		readEndedAtBind(ctx, js, hub, kvs[buckets.Asks], staleOfWorld(w), bus, callWait{heard: func() bool { return w.seatHeard(oc.Seat) }, findings: findings}, logger.Printf)
 		hearReadsChanged(ctx, js, hub, reads, findings, gate.eventsLive, logger.Printf)
 	}
 	if bus != nil && kvs[buckets.Puts] != nil {

modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go

@@ -666,8 +666,10 @@ func (p *passer) lift(ctx context.Context, module, fp string) {
 
 // staleAsk says why the hold, restore or record ask at key (read as the SDK reads it) is about something the trunk has left
 // behind, or "" when it is not: a module no longer on the trunk, a fingerprint that is no longer the module's trunk fingerprint,
-// or, for a record ask, a composition the module's node no longer waits on. A backup ask and an ask of another kind are never
-// stale here. A module of the trunk whose fingerprint is not read yet, and a have not read yet, are not stale: nothing known is
+// or, for a record ask, a composition the module's node no longer waits on. A backup ask is stale when the module's want on its
+// node names another fingerprint or is removed (the move it is about was superseded; issue 1966 note 165); a want not written
+// yet, or at the same fingerprint in another composition, leaves it (backups asks that again under the next number). An ask of
+// another kind is never stale here. A module of the trunk whose fingerprint is not read yet, and a have not read yet, are not stale: nothing known is
 // nothing taken away.
 func (p *passer) staleAsk(key string, trunk map[string]meshcatalog.ModulesByModuleValue) string {
 	k, err := asks.ParseAnyAskKey(key)
@@ -675,6 +677,25 @@ func (p *passer) staleAsk(key string, trunk map[string]meshcatalog.ModulesByModu
 		return ""
 	}
 	switch k.Kind {
+	case asks.AskKindBackup:
+		wkey, err := mwc.WantKey(k.Node, k.Module)
+		if err != nil {
+			return ""
+		}
+		raw := p.w.value(buckets.Wants, wkey)
+		if raw == nil {
+			return ""
+		}
+		w, err := mwc.DecodeWant(raw)
+		if err != nil {
+			return ""
+		}
+		if w.Removed {
+			return "its module's want on " + k.Node + " is removed"
+		}
+		if w.Fp != k.FP {
+			return "its module's want on " + k.Node + " names " + w.Fp + " now"
+		}
 	case asks.AskKindHold, asks.AskKindRestore:
 		m, on := trunk[k.Module]
 		if !on {
@@ -753,7 +774,7 @@ func (p *passer) fingerprintsOf(node, module, trunkFP string) []string {
 	return out
 }
 
-// staleAsks takes away, at every pass, each hold, restore and record ask that staleAsk names (issue 1906, note 28): an ask
+// staleAsks takes away, at every pass, each hold, restore, record and backup ask that staleAsk names (issue 1906, note 28): an ask
 // outlived its fingerprint, and the teller takes its router ask back from the operator when the key goes. With the trunk not
 // read at all (no module), nothing is taken away.
 func (p *passer) staleAsks(ctx context.Context, trunk map[string]meshcatalog.ModulesByModuleValue) {
@@ -769,6 +790,14 @@ func (p *passer) staleAsks(ctx context.Context, trunk map[string]meshcatalog.Mod
 			if held && !p.clearHeld(k.Module, k.FP) {
 				continue
 			}
+			// A backup ask's condition is cleared with it, raised again if a move to that fingerprint is backed up again.
+			if kerr == nil && k.Kind == asks.AskKindBackup && p.say != nil {
+				if err := p.say(assign.BackupCondition(k.Node, k.Module, k.FP, "", false)); err != nil {
+					p.logf("the condition of the stale backup of %s on %s could not be cleared, and is cleared at the next pass: %v", k.Module, k.Node, err)
+					continue
+				}
+				delete(p.said, assign.BackupCondition(k.Node, k.Module, k.FP, "", false).Key)
+			}
 			p.logf("the ask %s is about a fingerprint or composition the trunk has left behind and is taken away: %s", key, why)
 			p.remove(ctx, buckets.Asks, key)
 			// The held record stays: it is the memory that this fingerprint failed (ADR 0408 §4), and a revert returns the trunk to

modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go

@@ -24,6 +24,7 @@ import (
 	mdseat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery"
 	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
 	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
+	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-teller/tellerevents"
 	"git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate"
 
 	"github.com/nats-io/nats.go/jetstream"
@@ -150,6 +151,53 @@ func TestAHoldAskedAgainAfterALiftAndARestartHasANewID(t *testing.T) {
 	if _, err := askAgainByHand(st, record, key, "again", "lead@shanks", now); err == nil || !strings.Contains(err.Error(), "keep") {
 		t.Errorf("an answered asking is asked again: %v", err)
 	}
+	// Issue 1966 note 165 (AC165-2, AC165-3): the teller's ask-ended for a backup ask whose want names another fingerprint writes
+	// nothing and the journal says it is stale; the same ask with its want at the same fingerprint in another composition, or no
+	// want yet, is written again.
+	backKey, _ := asks.BackupAskKey("ace", "web", fpTest)
+	fpOther := meshcatalogclient.SourcePrefix2 + strings.Repeat("c", 64)
+	for _, c := range []struct {
+		name  string
+		want  *mwc.Want
+		stale bool
+	}{
+		{"its want names another fingerprint", &mwc.Want{Fp: fpOther, Self: true}, true},
+		{"its want is removed", &mwc.Want{Removed: true}, true},
+		{"its want names the fingerprint in another composition", &mwc.Want{Fp: fpTest, Composed: "sha256:" + strings.Repeat("e", 64)}, false},
+		{"no want yet", nil, false},
+	} {
+		t.Run("ended backup ask: "+c.name, func(t *testing.T) {
+			bkv := &memKV{values: map[string][]byte{}}
+			pb := newPasser(kvSet{buckets.Asks: bkv, buckets.Wants: &memKV{values: map[string][]byte{}}}, now)
+			ba := assign.BackupAsk(backKey, "ace", "web", fpTest, "sha256:"+strings.Repeat("d", 64), "w", 2, now)
+			bv, err := assign.EncodeAsk(backKey, ba)
+			if err != nil {
+				t.Fatal(err)
+			}
+			bkv.values[backKey] = bv
+			if c.want != nil {
+				wkey, _ := mwc.WantKey("ace", "web")
+				raw, _ := mwc.EncodeWant(*c.want)
+				pb.w.put(buckets.Wants, wkey, raw)
+			}
+			_, bask, _ := assign.DecodeAsk(backKey, bv)
+			body, err := tellerevents.EncodeEnded(tellerevents.Ended{Key: backKey, Ask: bask.ID, Outcome: occ.OutcomeExpired})
+			if err != nil {
+				t.Fatal(err)
+			}
+			var said []string
+			stale := func(key string) string {
+				return pb.staleAsk(key, map[string]meshcatalog.ModulesByModuleValue{"web": {Fp2: fpOther}})
+			}
+			if err := askAgain(context.Background(), bkv, nil, stale, body, now.Add(time.Hour), func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }); err != nil {
+				t.Fatal(err)
+			}
+			written := !bytes.Equal(bkv.values[backKey], bv)
+			if written == c.stale || (c.stale && !strings.Contains(strings.Join(said, "\n"), "stale")) {
+				t.Errorf("written again = %v (stale %v), journal %q", written, c.stale, said)
+			}
+		})
+	}
 	for _, c := range []struct{ key, by, why string }{{key, stdio.Unproven, "a caller"}, {"hold.none", "lead@shanks", "no ask"}} {
 		if _, err := askAgainByHand(st, record, c.key, "w", c.by, now); err == nil {
 			t.Errorf("%s: asked again", c.why)
@@ -864,7 +912,8 @@ func testAHeldModuleRaisesNoNeitherAndClearsAStandingOne(t *testing.T) {
 
 // 1906 note 28: an ask outlives its fingerprint. Each pass takes away the hold, restore and record asks of a module whose
 // fingerprint (or composition) is not the trunk's now, or whose module left the trunk; a trunk not read takes nothing away; a
-// backup ask is left; and a lift takes the restore asks of its fingerprint on every node.
+// backup ask is taken away when its module's want on the node names another fingerprint or is removed, and left when the want
+// is not written yet or names the same fingerprint (1966 note 165: AC165-1, AC165-3); and a lift takes the restore asks of its fingerprint on every node.
 func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) {
 	fpOld, fpNew := meshcatalogclient.SourcePrefix2+strings.Repeat("a", 64), meshcatalogclient.SourcePrefix2+strings.Repeat("c", 64)
 	digOld, digNew := "sha256:"+strings.Repeat("d", 64), "sha256:"+strings.Repeat("e", 64)
@@ -884,14 +933,19 @@ func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) {
 	for _, c := range []struct {
 		name    string
 		trunk   map[string]meshcatalog.ModulesByModuleValue
-		gone    []string // the keys taken away
-		noDigit bool     // the node's own composition is not read
+		gone    []string  // the keys taken away
+		noDigit bool      // the node's own composition is not read
+		want    *mwc.Want // the want of web on novox, nil when none is written
 	}{
-		{"the trunk moved on", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld}, false},
-		{"the trunk is at the old fingerprint", trunkOf(fpOld), []string{holdNew, holdGone, restNew, recOld}, false},
-		{"the trunk's fingerprint is not read yet", trunkOf(""), []string{holdGone}, false},
-		{"the composition is not read yet", trunkOf(fpNew), []string{holdOld, holdGone, restOld}, true},
-		{"the trunk is not read at all", nil, nil, false},
+		{"the trunk moved on", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld}, false, nil},
+		{"the backup's want names another fingerprint", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld, backOld}, false, &mwc.Want{Fp: fpNew, Self: true}},
+		{"the backup's want is removed", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld, backOld}, false, &mwc.Want{Removed: true}},
+		{"the backup's want names the fingerprint in another composition", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld}, false, &mwc.Want{Fp: fpOld, Composed: digNew}},
+		{"the trunk is at the old fingerprint", trunkOf(fpOld), []string{holdNew, holdGone, restNew, recOld}, false, nil},
+		{"the trunk's fingerprint is not read yet", trunkOf(""), []string{holdGone}, false, nil},
+		{"the composition is not read yet", trunkOf(fpNew), []string{holdOld, holdGone, restOld}, true, nil},
+		{"the trunk is not read at all", nil, nil, false, nil},
+		{"the trunk is not read at all, the backup's want moved", nil, nil, false, &mwc.Want{Fp: fpNew, Self: true}},
 	} {
 		t.Run(c.name, func(t *testing.T) {
 			asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
@@ -928,6 +982,14 @@ func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) {
 				}
 				p.w.put(buckets.Have, hkey, raw)
 			}
+			if c.want != nil {
+				wkey, _ := mwc.WantKey("novox", "web")
+				raw, err := mwc.EncodeWant(*c.want)
+				if err != nil {
+					t.Fatal(err)
+				}
+				p.w.put(buckets.Wants, wkey, raw)
+			}
 			heldOld, _ := mwc.HeldKey("web", fpOld)
 			wantsKV.values[heldOld] = []byte("x")
 			p.w.put(buckets.Wants, heldOld, []byte("x"))
@@ -942,6 +1004,13 @@ func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) {
 			if gone := slices.Contains(c.gone, holdOld); !heldThere || cleared != gone {
 				t.Errorf("hold ask of the old fingerprint gone = %v: its held record stays = %v (must), its condition cleared = %v", gone, heldThere, cleared)
 			}
+			// the failed backup's condition is cleared with its ask, and with no other (AC165-1)
+			backCleared := slices.ContainsFunc(said, func(c oc.Condition) bool {
+				return c.Key == assign.BackupCondition("novox", "web", fpOld, "", false).Key && c.State == occ.StateCleared
+			})
+			if gone := slices.Contains(c.gone, backOld); backCleared != gone {
+				t.Errorf("backup ask gone = %v, its condition cleared = %v", gone, backCleared)
+			}
 			for _, key := range all {
 				_, there := asksKV.values[key]
 				want := !slices.Contains(c.gone, key)