Files
mesh-delivery/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/assignerasks_test.go
T
jschoubben cd8afa47d9 The teller reads the router's record outside its lock, so a catch-up never holds a warrant
catchUp held the teller's lock across the router's record read (Where, through operatorask.Bucket and seatlist.HolderOf,
which retries); with no answer it blocked the module's event pull, whose Said waits on that lock (met when an ask arrived
inside a first view). OnAssignerAsk now catches up after it let go of the lock, catchUp takes the lock only once the record
is read (an asking no longer open by then is left), and OnCurrent reads the records of the asks asked again outside the
lock in the same way.

Builds: design 47 G §717 (the teller raises the assigner's asks whether or not anything else answers, issue 1906); principle 3

Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
2026-11-12 20:19:06 +01:00

330 lines
14 KiB
Go

package main
import (
"errors"
"slices"
"encoding/json"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-teller/tellerevents"
"strings"
"testing"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
)
// 1906/H1: given the assigner's hold ask in its bucket and no answer, when the teller hears the key, then it raises the ask on
// the operator channel as itself, unchanged and once (the key heard again, as a watch replays it, asks nothing more).
func TestAnAssignerAskIsRaisedOnTheOperatorChannelOnce(t *testing.T) {
b := newBed(t)
assigner, _ := withView(b)
key, value, ask := holdAsk(t)
assigner[key] = value
b.t.OnAssignerAsk(key)
b.t.OnAssignerAsk(key)
asked := b.askedOf(oc.AcceptSubjectAsk(tellerModule))
if len(asked) != 1 {
t.Fatalf("published %d asks, want one; log:\n%s", len(asked), b.logs.String())
}
var got oc.Ask
if err := json.Unmarshal(asked[0].body, &got); err != nil {
t.Fatal(err)
}
if got.ID != ask.ID || occ.Digest(got) != occ.Digest(ask) {
t.Errorf("the router's ask %+v differs from the assigner's", got)
}
// Issue 1966, ADR 0668 §1: a publish that fails (the router restarting) is not forgotten: the ask waits, kept, and is
// asked at the teller's next start (or the next end), though nothing writes the assigner's key again.
b = newBed(t)
assigner, _ = withView(b)
assigner[key] = value
publish, failed := b.client.Publish, false
b.client.Publish = func(subject string, body any, id string) error {
if subject == oc.AcceptSubjectAsk(tellerModule) && !failed {
failed = true
return errors.New("nats: no responders available for request")
}
return publish(subject, body, id)
}
b.t.Publish = b.client.Publish
b.t.OnAssignerAsk(key)
if asked := b.askedOf(oc.AcceptSubjectAsk(tellerModule)); !failed || len(asked) != 0 {
t.Fatalf("published %d asks through a failing publish", len(asked))
}
b.t.OnCurrent()
if asked := b.askedOf(oc.AcceptSubjectAsk(tellerModule)); len(asked) != 1 {
t.Errorf("after a failed publish and a start, published %d asks, want one; log:\n%s", len(asked), b.logs.String())
}
}
// 1906/H1: without the view's bucket the assigner's asks are still raised.
func TestAnAssignerAskIsRaisedWithoutTheView(t *testing.T) {
b := newBed(t)
assigner := memGate{}
b.t.Assigner = assigner
key, value, _ := holdAsk(t)
assigner[key] = value
b.t.OnAssignerAsk(key)
if n := len(b.askedOf(oc.AcceptSubjectAsk(tellerModule))); n != 1 {
t.Fatalf("published %d asks, want one; log:\n%s", n, b.logs.String())
}
}
// 1906/H1: the operator's answer on the phone comes back by the same path as the view's: written under the ask's own key in
// the bucket answers, naming the asking, once; a second warrant writes nothing more and nothing is said as a fault.
func TestTheOperatorsAnswerToAnAssignerAskIsWrittenUnderItsKey(t *testing.T) {
b := newBed(t)
assigner, _ := withView(b)
key, value, ask := holdAsk(t)
assigner[key] = value
b.t.OnAssignerAsk(key)
w := ackFor(ask, "release")
b.deliver(t, w)
raw, ok := b.answers.written[answerKey(key)]
if !ok {
t.Fatalf("no answer under %s; log:\n%s", key, b.logs.String())
}
if a, err := meshdeliveryclient.DecodeAnswer(raw); err != nil || a.Option != "release" || answeredAsking(raw) != ask.ID {
t.Errorf("answer = %+v, %v", a, err)
}
b.deliver(t, w)
if b.answers.writes != 1 || len(b.conds) != 0 {
t.Errorf("%d writes, conditions %+v", b.answers.writes, b.conds)
}
}
// 1906/H1: an answer to an asking the assigner has since taken away or asked again changes nothing.
func TestAnAnswerToAnAssignerAskTakenAwayWritesNothing(t *testing.T) {
b := newBed(t)
assigner, _ := withView(b)
key, value, ask := holdAsk(t)
assigner[key] = value
b.t.OnAssignerAsk(key)
delete(assigner, key)
b.deliver(t, ackFor(ask, "release"))
if b.answers.writes != 0 {
t.Errorf("an answer to a withdrawn ask was written")
}
}
// 1906/H1, H2: an ask answered in the view is taken back from the phone, and an ask the assigner took away (a hold it lifted by
// itself) is taken back too; an ask answered before it is heard is never raised.
func TestAnAssignerAskAnsweredInTheViewOrTakenAwayIsTakenBack(t *testing.T) {
cancelled := func(b *bed, id string) bool {
for _, p := range b.askedOf(oc.AcceptSubjectCancel(tellerModule)) {
if string(p.body) == `{"id":"`+id+`"}` {
return true
}
}
return false
}
b := newBed(t)
assigner, view := withView(b)
key, value, ask := holdAsk(t)
assigner[key] = value
b.t.OnAssignerAsk(key)
view[key+".view"] = viewAnswer(t, "release", "")
b.t.OnViewAnswer(key + ".view")
if !cancelled(b, ask.ID) {
t.Errorf("the ask answered in the view was not taken back; log:\n%s", b.logs.String())
}
b = newBed(t)
assigner, _ = withView(b)
assigner[key] = value
b.t.OnAssignerAsk(key)
delete(assigner, key)
b.t.OnAssignerAskGone(key)
if !cancelled(b, ask.ID) {
t.Errorf("the ask taken away was not taken back; log:\n%s", b.logs.String())
}
b = newBed(t)
assigner, view = withView(b)
assigner[key] = value
view[key+".view"] = viewAnswer(t, "keep", "")
b.t.OnAssignerAsk(key)
if n := len(b.askedOf(oc.AcceptSubjectAsk(tellerModule))); n != 0 {
t.Errorf("an ask answered in the view was raised %d times", n)
}
}
// 1906/H1: an assigner ask refused for want of room waits and is asked again when another of the teller's asks ends; a refusal for
// its words is said as a condition about a delivery ask.
func TestAnAssignerAskWaitsForRoomAndARefusalIsSaidAsADeliveryAsk(t *testing.T) {
anAssignerAskWrittenAgainAfterBeingTakenAwayIsAskedAgainUnderANewAsking(t)
b := newBed(t)
assigner, _ := withView(b)
_, first := b.holdOf(t, 1, 1)
key, value, ask := holdAsk(t)
assigner[key] = value
b.t.OnAssignerAsk(key)
b.refuse(t, ask.ID, roomRefusal)
b.deliver(t, warrantFor(first, "keep"))
got := b.askedIDs()
if len(got) != 3 || got[2] != ask.ID {
t.Fatalf("asks = %v, want the assigner's %s asked again", got, ask.ID)
}
b = newBed(t)
assigner, _ = withView(b)
assigner[key] = value
b.t.OnAssignerAsk(key)
b.refuse(t, ask.ID, "the ask has no headline")
if len(b.conds) != 1 || b.conds[0].Subject.Scope != "delivery" || strings.Contains(b.conds[0].Explanation, "merge") {
t.Errorf("conditions = %+v", b.conds)
}
}
// ackFor is the router's warrant for an option of level acknowledge, as the assigner's asks offer.
func ackFor(a oc.Ask, option string) oc.Warrant { return warrantFor(a, option) }
// 1906/H5: given seven holds at once (as on 2026-10-25), when they are raised, then at most mostAssignerOpen are open at the
// router, and each hold answered lets the next waiting one be asked.
func TestABurstOfHoldsAsksNoMoreThanTheMostOpenAndTheNextWhenOneEnds(t *testing.T) {
b := newBed(t)
var held []oc.Ask
for n := 0; n < 7; n++ {
_, ask := b.holdOf(t, n, 1)
held = append(held, ask)
}
if got := b.askedIDs(); len(got) != mostAssignerOpen || got[0] != held[0].ID || got[1] != held[1].ID {
t.Fatalf("asked %v, want the first %d holds only; log:\n%s", got, mostAssignerOpen, b.logs.String())
}
if len(b.dropped) != 0 || len(b.conds) != 0 {
t.Errorf("router dropped %v, conditions %+v", b.dropped, b.conds)
}
b.deliver(t, ackFor(held[0], "release"))
got := b.askedIDs()
if len(got) != mostAssignerOpen+1 || got[len(got)-1] != held[2].ID {
t.Fatalf("asked %v after a hold was answered, want the third hold asked next; log:\n%s", got, b.logs.String())
}
}
// Issue 1966 (nats on novox): an assigner ask taken back when its key went is asked again when the assigner writes the key
// again under a new asking, whatever its kind; written again under the id the router kept it is not (the router would drop it).
// An asking that ended at the channel without an answer while its key stands (taken back and written again under its id, or
// lapsed at the router) is said to the assigner as the teller's event ask-ended, so the assigner writes it again under a new
// asking (note 151); one answered, refused, replaced or ended by hand is not.
func anAssignerAskWrittenAgainAfterBeingTakenAwayIsAskedAgainUnderANewAsking(t *testing.T) {
digest := "sha256:" + strings.Repeat("e", 64)
fp := "src2:" + strings.Repeat("a", 64)
recordKey, _ := asks.RecordAskKey("novox", "nats", digest)
restoreKey, _ := asks.RestoreAskKey("novox", "nats", fp)
holdKey, _ := asks.HoldAskKey("nats", fp)
backupKey, _ := asks.BackupAskKey("novox", "nats", fp)
for kind, key := range map[string]string{"record": recordKey, "restore": restoreKey, "hold": holdKey, "backup": backupKey} {
write := func(t *testing.T, assigner memGate, asking int) oc.Ask {
ask := oc.Ask{ID: meshdeliveryclient.AskingID(key, asking), Headline: "nats waits", Explanation: "e", Who: occ.Operator,
Expires: occ.FormatTime(now.Add(occ.ApproveLasts)), OnExpiry: "stays",
Options: []oc.Option{{ID: "yes", Label: "Yes", Does: "d", Level: occ.Acknowledge}, {ID: "no", Label: "No", Does: "d", Level: occ.Acknowledge}}}
value, err := asks.EncodeAsk(key, asks.Ask{Ask: rawAsk(t, ask), Asking: asking})
if err != nil {
t.Fatal(err)
}
assigner[key] = value
return ask
}
t.Run(kind, func(t *testing.T) {
b := newBed(t)
assigner, _ := withView(b)
first := write(t, assigner, 1000)
b.t.OnAssignerAsk(key)
delete(assigner, key)
b.t.OnAssignerAskGone(key) // the assigner took it away as stale: the router ask is taken back
second := write(t, assigner, 1060)
b.t.OnAssignerAsk(key)
if got := b.askedIDs(); len(got) != 2 || got[0] != first.ID || got[1] != second.ID || first.ID == second.ID {
t.Fatalf("asked %v, want %s then %s; log:\n%s", got, first.ID, second.ID, b.logs.String())
}
if len(b.emitted) != 0 {
t.Fatalf("said %v while every asking was asked", b.emitted)
}
// Written again under the id the router kept, after it was taken back: not asked, and the assigner is told.
delete(assigner, key)
b.t.OnAssignerAsk(key) // a watch's replay, as at the teller's start, before the key went
b.t.OnAssignerAskGone(key)
write(t, assigner, 1060)
b.t.OnAssignerAsk(key)
want := []string{tellerevents.AskEnded + " " + key + " " + second.ID + " " + occ.OutcomeCancelled}
if got := b.askedIDs(); len(got) != 2 || !slices.Equal(b.emitted, want) {
t.Errorf("asked %v and said %v after the same id was written again, want %v; log:\n%s", got, b.emitted, want, b.logs.String())
}
})
for _, end := range []struct {
name string
end func(b *bed, a oc.Ask)
said string // the outcome said to the assigner, or "" for none
}{
{"lapsed at the router", func(b *bed, a oc.Ask) {
b.deliver(t, oc.Warrant{Ask: a.ID, Asker: tellerModule, Outcome: occ.OutcomeExpired, At: occ.FormatTime(now), AskDigest: occ.Digest(a)})
}, occ.OutcomeExpired},
{"cancelled at the router", func(b *bed, a oc.Ask) {
b.deliver(t, oc.Warrant{Ask: a.ID, Asker: tellerModule, Outcome: occ.OutcomeCancelled, At: occ.FormatTime(now), AskDigest: occ.Digest(a)})
}, occ.OutcomeCancelled},
{"replaced at the router", func(b *bed, a oc.Ask) {
b.deliver(t, oc.Warrant{Ask: a.ID, Asker: tellerModule, Outcome: occ.OutcomeReplaced, At: occ.FormatTime(now), AskDigest: occ.Digest(a)})
}, ""},
{"refused for its words", func(b *bed, a oc.Ask) { b.refuse(t, a.ID, "the ask has no headline") }, ""},
{"answered", func(b *bed, a oc.Ask) { b.deliver(t, ackFor(a, "yes")) }, ""},
{"ended by hand", func(b *bed, a oc.Ask) { b.t.EndAsk(a.ID, "the operator asked", "lead@shanks") }, ""},
} {
t.Run(kind+"/"+end.name, func(t *testing.T) {
b := newBed(t)
assigner, _ := withView(b)
a := write(t, assigner, 1000)
b.t.OnAssignerAsk(key)
end.end(b, a)
b.t.OnAssignerAsk(key) // the key still stands under that asking, as at the teller's start
var want []string
if end.said != "" {
want = []string{tellerevents.AskEnded + " " + key + " " + a.ID + " " + end.said}
}
if !slices.Equal(b.emitted, want) {
t.Errorf("said %v, want %v; log:\n%s", b.emitted, want, b.logs.String())
}
if got := b.askedIDs(); len(got) != 1 {
t.Errorf("asked %v, want the one asking only: the assigner writes a new one", got)
}
})
}
}
}
// An ask the teller caught up with inside a first view reads the router's record outside the teller's lock (principle 3, nothing
// stalls silently): while that read waits on the bus, a warrant the module's event pull hands on (Said) is handled, not held
// behind it. The order met: the ask is asked, the first view hands the key on again, and its catch-up's read does not answer.
func TestACatchUpWaitingOnTheRouterHoldsNoWarrant(t *testing.T) {
b := newBed(t)
assigner, _ := withView(b)
key, value, _ := holdAsk(t)
assigner[key] = value
b.t.OnAssignerAsk(key)
record := b.client.Record
entered, release := make(chan struct{}), make(chan struct{})
b.client.Record = func(id string) (*stdio.StateEntry, error) {
close(entered)
<-release
return record(id)
}
caught := make(chan struct{})
go func() { defer close(caught); b.t.OnAssignerAsk(key) }() // the first view hands the key on: the ask is caught up with
<-entered
said := make(chan struct{})
go func() { defer close(said); b.t.Said(occ.Done, oc.Warrant{Ask: "other"}, nil) }()
select {
case <-said:
case <-time.After(5 * time.Second):
close(release)
<-caught
t.Fatal("a warrant waited on the teller's lock while a catch-up read the router's record")
}
close(release)
<-caught
}