Give each delivery's hold bound its own key's time to live and map a seat's finding to the module the placement says holds it, so the keeper holds the right findings for the right time (issue 1952 C7)

This commit is contained in:
2026-10-29 22:16:23 +01:00
parent b5fdcb88c8
commit 4cbdd0fc74
10 changed files with 96 additions and 26 deletions
@@ -52,8 +52,14 @@ func (k *Keeper) aboutOf(kind, source string, s health.Subject) []about {
out = append(out, about{Node: parts[1], Module: parts[0]}) out = append(out, about{Node: parts[1], Module: parts[0]})
} }
case health.ScopeSeat: case health.ScopeSeat:
// The module the placement says holds the seat on that node, or the seat's own name where none is read.
if len(parts) >= 2 { if len(parts) >= 2 {
out = append(out, about{Node: parts[len(parts)-1], Module: strings.Join(parts[:len(parts)-1], ".")}) node, seat := parts[len(parts)-1], strings.Join(parts[:len(parts)-1], ".")
module := k.holders[node+"."+seat]
if module == "" {
module = seat
}
out = append(out, about{Node: node, Module: module})
} }
case health.ScopeMachine: case health.ScopeMachine:
// The machine itself (presence, network, resolver, disk) is never held; its tools are, while its tool runner is // The machine itself (presence, network, resolver, disk) is never held; its tools are, while its tool runner is
@@ -143,9 +149,8 @@ func (k *Keeper) keepHeld(h held) error {
return nil return nil
} }
// bound is a module's delivery bound: its manifest names none yet, so the default (clause 13 (d)). The state `bounds` // bound is a module's delivery bound: its manifest names none yet, so the default (clause 13 (d)). Each key of the state
// keeps each key that long (its ttl-seconds), since the runtime writes no key with a time to live of its own; a bound a // `bounds` is written with it as its own time to live, and its expiry is the bound passing.
// manifest names needs that first.
func (k *Keeper) bound(string) time.Duration { func (k *Keeper) bound(string) time.Duration {
if k.Bound > 0 { if k.Bound > 0 {
return k.Bound return k.Bound
@@ -211,6 +216,33 @@ func (k *Keeper) Have(key string, value []byte, deleted bool) error {
return k.judge(node, module, "") return k.judge(node, module, "")
} }
// Placement takes one change of the placement: which module holds which seat on a node.
func (k *Keeper) Placement(key string, value []byte, deleted bool) {
node, err := delivery.ParsePlacementKey(key)
if err != nil {
return
}
k.mu.Lock()
defer k.mu.Unlock()
k.init()
for hk := range k.holders {
if strings.HasPrefix(hk, node+".") {
delete(k.holders, hk)
}
}
if deleted {
return
}
p, err := delivery.DecodePlacement(value)
if err != nil {
k.Logf("the placement of %s is not readable, so its seats' holders are not known: %v", node, err)
return
}
for _, h := range p.Holds {
k.holders[node+"."+h.Claim] = h.Module
}
}
// BoundPassed is a delivery's bound key expiring: what it holds is raised, saying the delivery did not end. // BoundPassed is a delivery's bound key expiring: what it holds is raised, saying the delivery did not end.
func (k *Keeper) BoundPassed(key string) error { func (k *Keeper) BoundPassed(key string) error {
node, module := splitNodeModule(key) node, module := splitNodeModule(key)
@@ -71,6 +71,10 @@ func TestADeliveryHoldsItsOwnFindingsUntilItEnds(t *testing.T) {
if err := r.k.Controller("core.controller.deaf", deaf, false); err != nil { if err := r.k.Controller("core.controller.deaf", deaf, false); err != nil {
t.Fatal(err) t.Fatal(err)
} }
// The delivery's bound is its own key's time to live, the module's bound (the default where it names none).
if r.st.ttls["bounds/novox.mesh-controller"] != delivery.DefaultDeliveryBound {
t.Errorf("%s: the bound key lives %s", c.name, r.st.ttls["bounds/novox.mesh-controller"])
}
if got := r.changes(); got != "raised "+other+", raised "+machine { if got := r.changes(); got != "raised "+other+", raised "+machine {
t.Errorf("%s: during the delivery said %q: another module's and the machine's at once, nothing of its own", c.name, got) t.Errorf("%s: during the delivery said %q: another module's and the machine's at once, nothing of its own", c.name, got)
} }
@@ -106,4 +110,20 @@ func TestADeliveryHoldsItsOwnFindingsUntilItEnds(t *testing.T) {
t.Errorf("%s: the carried deaf condition was raised, or something is still held: %+v", c.name, r.k.Held()) t.Errorf("%s: the carried deaf condition was raised, or something is still held: %+v", c.name, r.k.Held())
} }
} }
// A seat's finding is about the module the placement says holds the seat on that node (the lead, 2026-10-29): held
// while the assigner is delivered on ace, though the seat is named mesh-placement.
r := newRig()
placed, err := delivery.EncodePlacement(delivery.Placement{Node: "ace", Modules: []delivery.PlacedModule{{Module: "mesh-delivery-assigner"}},
Holds: []delivery.Holding{{Claim: "mesh-placement", Scope: "mesh", Module: "mesh-delivery-assigner"}}})
if err != nil {
t.Fatal(err)
}
r.k.Placement("placement.ace", placed, false)
_ = r.k.Want("want.ace.mesh-delivery-assigner", want(ctrlNext, ctrlNextComp), false)
_ = r.k.Have("ace.mesh-delivery-assigner", have(ctrlRunning, ctrlComposed, delivery.PhaseRunning, delivery.HealthOK), false)
seat, seatRaw := finding(t, selfCheck, health.ScopeSeat, "mesh-placement.ace", "holder-silent", "D3", health.Warning, r.now)
_ = r.k.Finding(selfCheck, seat, seatRaw, false)
if got := r.changes(); got != "" || len(r.k.Held()) != 1 || r.k.Held()[0].Module != "mesh-delivery-assigner" {
t.Errorf("a seat's finding while its holder is delivered: said %q, held %+v", got, r.k.Held())
}
} }
@@ -48,8 +48,11 @@ const (
ControllerLease = "mesh-controller.lease" ControllerLease = "mesh-controller.lease"
ControllerLeaseKey = "holder" ControllerLeaseKey = "holder"
WantsRead = delivery.AssignerModule + ".wants" WantsRead = delivery.AssignerModule + ".wants"
HavesRead = "node-engine.have" // PlacementRead is the mesh-placement seat's placement, read for which module holds a seat on a node (hold.go; the
ConditionWrites = 8 // lead, 2026-10-29, for ADR 0564 clause 2's inputs).
PlacementRead = delivery.PlacementRead
HavesRead = "node-engine.have"
ConditionWrites = 8
) )
// Store is the bus as the keeper uses it: its own states, written by compare-and-set, and a key with its own time to // Store is the bus as the keeper uses it: its own states, written by compare-and-set, and a key with its own time to
@@ -104,8 +107,10 @@ type Keeper struct {
takeover health.Takeover takeover health.Takeover
wants map[string]delivery.Want wants map[string]delivery.Want
haves map[string]delivery.Have haves map[string]delivery.Have
stuck map[string]string // holders are, by `<node>.<seat>`, the module the placement says holds the seat there.
held map[string]held holders map[string]string
stuck map[string]string
held map[string]held
// passed are the deliveries whose bound passed, by node and module, at the fingerprint wanted then; outOfHold the // passed are the deliveries whose bound passed, by node and module, at the fingerprint wanted then; outOfHold the
// line a key raised out of a hold says (hold.go). // line a key raised out of a hold says (hold.go).
passed map[string]string passed map[string]string
@@ -121,7 +126,7 @@ func (k *Keeper) init() {
k.wants, k.haves, k.stuck, k.held, k.cleared = map[string]delivery.Want{}, map[string]delivery.Have{}, k.wants, k.haves, k.stuck, k.held, k.cleared = map[string]delivery.Want{}, map[string]delivery.Have{},
map[string]string{}, map[string]held{}, map[string]clearedMemo{} map[string]string{}, map[string]held{}, map[string]clearedMemo{}
k.takeover = health.Takeover{V: health.TakeoverVersion, Follow: health.FollowController} k.takeover = health.Takeover{V: health.TakeoverVersion, Follow: health.FollowController}
k.passed, k.outOfHold = map[string]string{}, map[string]string{} k.passed, k.outOfHold, k.holders = map[string]string{}, map[string]string{}, map[string]string{}
} }
if k.Now == nil { if k.Now == nil {
k.Now = time.Now k.Now = time.Now
@@ -20,6 +20,8 @@ type fakeStore struct {
vals map[string][]byte vals map[string][]byte
revs map[string]uint64 revs map[string]uint64
next uint64 next uint64
// ttls are the times to live the keys were created with.
ttls map[string]time.Duration
} }
func newFakeStore() *fakeStore { func newFakeStore() *fakeStore {
@@ -79,7 +81,13 @@ func (f *fakeStore) Keys(state string) ([]string, error) {
return out, nil return out, nil
} }
func (f *fakeStore) CreateTTL(state, key string, value any, _ time.Duration) error { func (f *fakeStore) CreateTTL(state, key string, value any, ttl time.Duration) error {
f.mu.Lock()
if f.ttls == nil {
f.ttls = map[string]time.Duration{}
}
f.ttls[state+"/"+key] = ttl
f.mu.Unlock()
_, err := f.Create(state, key, value) _, err := f.Create(state, key, value)
return err return err
} }
@@ -44,10 +44,9 @@ func (runtimeStore) Delete(state, key string) error { return stdio.State(state).
func (runtimeStore) Keys(state string) ([]string, error) { return stdio.State(state).Keys() } func (runtimeStore) Keys(state string) ([]string, error) { return stdio.State(state).Keys() }
// CreateTTL creates the key in a state whose own age is the time to live (`bounds`, ttl-seconds): the runtime writes no // CreateTTL creates the key with its own time to live (`bounds` is key-ttl); its expiry is handed over as OpExpire.
// key with a time to live of its own. func (runtimeStore) CreateTTL(state, key string, value any, ttl time.Duration) error {
func (runtimeStore) CreateTTL(state, key string, value any, _ time.Duration) error { _, err := stdio.State(state).CreateTTL(key, value, ttl)
_, err := stdio.State(state).Create(key, value)
return err return err
} }
@@ -132,11 +131,15 @@ func Keep(k *Keeper) {
}) })
} }
watch(k, BoundsState, "", func(c stdio.StateChange) error { watch(k, BoundsState, "", func(c stdio.StateChange) error {
if c.Op != "put" && !c.Current { if c.Op == stdio.OpExpire {
return said(k.BoundPassed(c.Key)) return said(k.BoundPassed(c.Key))
} }
return nil return nil
}) })
watch(k, PlacementRead, "", func(c stdio.StateChange) error {
k.Placement(c.Key, c.Value, c.Op != stdio.OpPut)
return nil
})
} }
func main() { func main() {
@@ -29,7 +29,7 @@ func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) {
!slices.Equal(Verbs, catalogue.VerbNames(seat.Serves)) { !slices.Equal(Verbs, catalogue.VerbNames(seat.Serves)) {
t.Errorf("module %s runs as %s and claims %+v; the seat serves %v", m.Module, m.RunsAs, m.Claims, catalogue.VerbNames(seat.Serves)) t.Errorf("module %s runs as %s and claims %+v; the seat serves %v", m.Module, m.RunsAs, m.Claims, catalogue.VerbNames(seat.Serves))
} }
reads := []string{ControllerConditions, ControllerLease, WantsRead, HavesRead, AttemptsRead} reads := []string{ControllerConditions, ControllerLease, WantsRead, HavesRead, AttemptsRead, PlacementRead}
for _, o := range Observers { for _, o := range Observers {
reads = append(reads, o+"."+health.FindingsState) reads = append(reads, o+"."+health.FindingsState)
} }
@@ -39,12 +39,14 @@ func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) {
if !slices.Equal(got, reads) || slices.Contains(m.Reads, health.HealersModule+"."+health.WouldState) { if !slices.Equal(got, reads) || slices.Contains(m.Reads, health.HealersModule+"."+health.WouldState) {
t.Errorf("reads %v, want %v and never the healers' would", got, reads) t.Errorf("reads %v, want %v and never the healers' would", got, reads)
} }
ttl := map[string]int{} ttl, keyTTL := map[string]int{}, map[string]bool{}
for _, s := range m.State { for _, s := range m.State {
ttl[s.Name] = s.TTLSeconds ttl[s.Name], keyTTL[s.Name] = s.TTLSeconds, s.KeyTTL
} }
if ttl[HistoryState] != int(HistoryKeptFor.Seconds()) || ttl[BoundsState] != 900 { // Each bound is its own key's time to live (the runtime's per-key TTL), never the bucket's age.
t.Errorf("history is kept %d s and a bound %d s", ttl[HistoryState], ttl[BoundsState]) if ttl[HistoryState] != int(HistoryKeptFor.Seconds()) || ttl[BoundsState] != 0 || !keyTTL[BoundsState] {
t.Errorf("history is kept %d s, a bound's bucket ages %d s, its keys have their own time to live: %v",
ttl[HistoryState], ttl[BoundsState], keyTTL[BoundsState])
} }
// Clause 12: each state with the tool that reads it, each act with the tool that starts it. // Clause 12: each state with the tool that reads it, each act with the tool that starts it.
readBy := map[string]string{ConditionsState: Seat + ".conditions", HistoryState: HistoryTool, TakeoverState: TakeoverTool, readBy := map[string]string{ConditionsState: Seat + ".conditions", HistoryState: HistoryTool, TakeoverState: TakeoverTool,
@@ -261,7 +261,7 @@ func TestTheKeeperWholeRaisesAndClearsOnTheBus(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
buckets := []string{"mesh-controller_conditions", "mesh-controller_lease", "mesh-delivery-assigner_wants", "node-engine_have", buckets := []string{"mesh-controller_conditions", "mesh-controller_lease", "mesh-delivery-assigner_wants", "node-engine_have", "mesh-delivery-assigner_placement",
"mesh-healers_attempts"} "mesh-healers_attempts"}
for _, o := range Observers { for _, o := range Observers {
buckets = append(buckets, o+"_findings") buckets = append(buckets, o+"_findings")
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-conditions
go 1.26.0 go 1.26.0
require ( require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029210043-32f5ce371268 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211442-bc6c8a6d244a
github.com/nats-io/nats.go v1.54.0 github.com/nats-io/nats.go v1.54.0
) )
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029210043-32f5ce371268 h1:+jH77IJmxUiFDdbJljhynQRY0Cc5/5jtwmMV7UUqRI4= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211442-bc6c8a6d244a h1:haFs0zoHDUOrdtKn0Cw9lhCg/K3Me6Ff3Mg+IdVlK34=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029210043-32f5ce371268/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211442-bc6c8a6d244a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
+2 -2
View File
@@ -28,7 +28,8 @@
"mesh-controller.conditions", "mesh-controller.conditions",
"mesh-controller.lease", "mesh-controller.lease",
"mesh-delivery-assigner.wants", "mesh-delivery-assigner.wants",
"node-engine.have" "node-engine.have",
"mesh-delivery-assigner.placement"
], ],
"state": [ "state": [
"conditions", "conditions",
@@ -40,7 +41,6 @@
"held", "held",
{ {
"name": "bounds", "name": "bounds",
"ttl-seconds": 900,
"key-ttl": true "key-ttl": true
}, },
"asked" "asked"