diff --git a/modules/mesh-config/README.md b/modules/mesh-config/README.md index 99f368a43..df2bc093a 100644 --- a/modules/mesh-config/README.md +++ b/modules/mesh-config/README.md @@ -30,7 +30,7 @@ module none of whose manifests can be read is listed under `unresolved` with why values last resolved for it. Then: the manifest's default; mesh config for the predefined key the value falls back to; node config for that key on the node; the module's own value on that node, the most specific winning. A value naming no predefined key resolves from its default and the module's own value only (ADR 0348 §2). It resolves at -start, whenever an assignment, a want or a trunk module moves, and every five minutes, writing only the values that moved. +start, whenever an assignment, a want or a trunk module moves, writing only the values that moved. ## The verb diff --git a/modules/mesh-config/cmd/mesh-config/main.go b/modules/mesh-config/cmd/mesh-config/main.go index 09f0b32dd..b0d927dd8 100644 --- a/modules/mesh-config/cmd/mesh-config/main.go +++ b/modules/mesh-config/cmd/mesh-config/main.go @@ -38,15 +38,19 @@ func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) } var theViews *ownerViews // runtimeStore is the bus, through the runtime that launched this bundle. A state of the delivery owners that the -// owners' tools answer is read from the picture those keep (ownerviews.go), the rest from the bus. -type runtimeStore struct{} +// owners' tools answer is read from the picture those keep (ownerviews.go), this module's own states from the copy their +// watches keep (ownstates.go), the rest from the bus. +type runtimeStore struct{ own *ownCopy } -func (runtimeStore) Get(state, key string) (*Entry, error) { +func (r runtimeStore) Get(state, key string) (*Entry, error) { if theViews != nil { if e, followed, err := theViews.get(state, key); followed { return e, err } } + if r.own != nil && ownStates[state] { + return r.own.get(state, key) + } e, err := stdio.State(state).Get(key) if err != nil || e == nil { return nil, err @@ -54,19 +58,30 @@ func (runtimeStore) Get(state, key string) (*Entry, error) { return &Entry{Key: e.Key, Value: e.Value, Revision: e.Revision}, nil } -func (runtimeStore) Put(state, key string, value any) error { +func (r runtimeStore) Put(state, key string, value any) error { + if r.own != nil && ownStates[state] { + return r.own.write(state, key, value) + } _, err := stdio.State(state).Put(key, value) return err } -func (runtimeStore) Delete(state, key string) error { return stdio.State(state).Delete(key) } +func (r runtimeStore) Delete(state, key string) error { + if r.own != nil && ownStates[state] { + return r.own.remove(state, key) + } + return stdio.State(state).Delete(key) +} -func (runtimeStore) Keys(state string) ([]string, error) { +func (r runtimeStore) Keys(state string) ([]string, error) { if theViews != nil { if keys, followed, err := theViews.keys(state); followed { return keys, err } } + if r.own != nil && ownStates[state] { + return r.own.keys(state) + } return stdio.State(state).Keys() } @@ -120,7 +135,16 @@ func (runtimeAsks) Warranted(ask string) error { } func main() { - c := &Config{Store: runtimeStore{}, Logf: logf, Asks: runtimeAsks{}, Forge: toolForge{ask: stdio.Ask}, + own, rearm := newOwnCopy(), make(chan struct{}, 1) + own.changed = func(state string) { + if state == ProposalsState || state == RetiringState { + select { + case rearm <- struct{}{}: + default: + } + } + } + c := &Config{Store: runtimeStore{own: own}, Logf: logf, Asks: runtimeAsks{}, Forge: toolForge{ask: stdio.Ask}, Delivery: toolDelivery{ask: stdio.Ask}, Vault: toolVault{ask: stdio.Ask, recorder: recorder}} // The module's one held writer (design 47 G §689, A-16, A-29, A-32): the renames' and retirements' findings and the views reader's go through it. w, err := views.NewStdioHeldWriter(views.WriterConfig[meshconditions.Finding, meshconditions.FindingCleared, meshconditions.FindingsStated, meshconditions.FindingsAskedEvent]{ @@ -181,6 +205,7 @@ func main() { theViews.Start(context.Background()) go resolving(c, moved) go hearing(c) + go deciding(c, rearm) go hearingDelivery(c) if err := stdio.Serve("", append(Tools(c), branches.EntryAgainTool("config", c.Store.HandAct))); err != nil { logf("%v", err) @@ -188,23 +213,17 @@ func main() { } } -// resolving resolves at start and again whenever a source moves, and every five minutes besides, so a change -// heard nowhere is still resolved. The runtime answers only once Serve runs, so the watches are asked again -// until they are taken. +// resolving resolves at start and again whenever a source moves: an assignment, a want, a catalog revision, an entry +// answered, the predefined list or a config candidate (each reaches `moved` through its view, ownerviews.go and main). It +// reads the forge once for each pull request it keeps something for, so a merge or a close said while this module was down is +// found (ADR 0502); after that it is heard on git.pull-merged and git.pull-closed (hearing), never asked again on a timer +// (novox/hq principle 1, issue 1982). The runtime answers only once Serve runs. func resolving(c *Config, moved chan struct{}) { - // The owners' states reach `moved` through their views (ownerviews.go); the builder's builds, which has no tool yet, - // is read when a want moves, and nothing else is watched. - tick := time.NewTicker(5 * time.Minute) - defer tick.Stop() - time.Sleep(time.Second) - for sweep := true; ; { - if sweep { - // A merge or a close of a pull request a value is held for, heard nowhere, is found here (ADR 0502). - for _, s := range c.SweepPulls() { - logf("[mesh-config] %s", s) - } - } - sweep = false + <-stdio.Served() + for _, s := range c.SweepPulls() { + logf("[mesh-config] %s", s) + } + for { r, err := c.Reconcile() switch { case err != nil: @@ -215,21 +234,23 @@ func resolving(c *Config, moved chan struct{}) { if r.Published != "" { logf("[mesh-config] published %s", r.Published) } + // A window or a retirement judged while the entry of the build it waits on was still asked stays open until the + // entry answers: the answer moves this pass, and the open ones are judged again here (design 32 *A config value + // renamed*). It reads nothing from the bus when none is open. + for _, s := range c.Windows("") { + logf("[mesh-config] %s", s) + } + <-moved + time.Sleep(500 * time.Millisecond) // a burst of writes is one resolution select { case <-moved: - time.Sleep(500 * time.Millisecond) // a burst of writes is one resolution - select { - case <-moved: - default: - } - case <-tick.C: - sweep = true + default: } } } -// hearing takes the router's word on this module's asks, asked again until the runtime takes the subscription, and -// reads the router's record of every waiting ask once a minute besides, so an answer heard nowhere is still taken. +// hearing takes the router's word on this module's asks, asked again until the runtime takes the subscription, and the +// forge's word on a pull request's merge or close. func hearing(c *Config) { go func() { <-stdio.Served() // subscribed as soon as the runtime serves, never after a guessed wait: an event sent meanwhile is heard @@ -274,9 +295,15 @@ func hearing(c *Config) { } }(pattern) } - tick := time.NewTicker(time.Minute) - defer tick.Stop() - for range tick.C { +} + +// deciding reads the router's record of every waiting ask once at start, which finds an answer said while this module was +// down, and then when the router says an ask ended (hearing's `decided` subscription calls Decide itself) or the next +// deadline of a waiting proposal or retirement passes: one timer for the earliest, set again whenever a proposal or a +// retirement is written (issue 1982; novox/hq ADR 0426 §3: a timer is a deadline, never a poll). +func deciding(c *Config, rearm chan struct{}) { + <-stdio.Served() + decide := func() { said, err := c.Decide() if err != nil { logf("[mesh-config] reading the operator's answers: %v", err) @@ -285,6 +312,22 @@ func hearing(c *Config) { logf("[mesh-config] %s", s) } } + decide() + for { + at, ok := c.NextDeadline() + if !ok { + <-rearm + continue + } + // Not sooner than a minute: a deadline that passed and was not settled (an ask the router refused) is tried again, not spun on. + timer := time.NewTimer(max(time.Until(at), time.Minute)) + select { + case <-timer.C: + decide() + case <-rearm: + timer.Stop() + } + } } // hearingDelivery takes the delivery parts' words that close a rename window or retire a value, each asked again until diff --git a/modules/mesh-config/cmd/mesh-config/ownstates.go b/modules/mesh-config/cmd/mesh-config/ownstates.go new file mode 100644 index 000000000..642976427 --- /dev/null +++ b/modules/mesh-config/cmd/mesh-config/ownstates.go @@ -0,0 +1,78 @@ +package main + +import ( + "encoding/json" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// The module's own states are read from a copy each one's watch keeps, never key by key (novox/hq ADR 0580 §1, issue 1982): +// a resolution reads every key of values, resolved, pulls, declared and the rest, and a get per key for each of them, once per +// pass and once per delivery event, was about 4,000 requests to the bus every time. A state is watched on its first read, the +// watch hands every current value over before the read is answered, and this process's own writes are taken into the copy +// as they are made, so a read right after a write sees it. + +// ownStates are the states this module keeps and reads through its store (its manifest's `state`, but the findings, which its +// held writer alone reads and writes): the ones read from the copy. +var ownStates = map[string]bool{ + ValuesState: true, ResolvedState: true, PredefinedState: true, ProposalsState: true, PullsState: true, KeptState: true, + DeclaredState: true, RenamesState: true, RetiringState: true, +} + +// ownCopy is the copy of the module's own states over a bus: watch hands every current value of a state and then each change, +// get reads one key, put and del write one. +type ownCopy struct { + copy *stdio.WatchedStates + put func(state, key string, value any) (uint64, error) + del func(state, key string) error + // changed, when not nil, is told of each write or removal after it is in the copy: what waits on a deadline is judged again. + changed func(state string) +} + +// newOwnCopy is the copy over this bundle's runtime. +func newOwnCopy() *ownCopy { + return &ownCopy{copy: stdio.NewWatchedStates(), + put: func(state, key string, value any) (uint64, error) { return stdio.State(state).Put(key, value) }, + del: func(state, key string) error { return stdio.State(state).Delete(key) }} +} + +func (o *ownCopy) get(state, key string) (*Entry, error) { + raw, ok, err := o.copy.Get(state, key) + if err != nil || !ok { + return nil, err + } + return &Entry{Key: key, Value: raw}, nil +} + +func (o *ownCopy) keys(state string) ([]string, error) { return o.copy.Keys(state) } + +// write puts a key and takes it into the copy at the revision the bus answered. +func (o *ownCopy) write(state, key string, value any) error { + rev, err := o.put(state, key, value) + if err != nil { + return err + } + raw, err := json.Marshal(value) + if err != nil { + return err + } + err = o.copy.Wrote(state, key, raw, rev) + o.told(state) + return err +} + +func (o *ownCopy) told(state string) { + if o.changed != nil { + o.changed(state) + } +} + +// remove deletes a key and takes that into the copy. +func (o *ownCopy) remove(state, key string) error { + if err := o.del(state, key); err != nil { + return err + } + err := o.copy.Removed(state, key) + o.told(state) + return err +} diff --git a/modules/mesh-config/cmd/mesh-config/proposals.go b/modules/mesh-config/cmd/mesh-config/proposals.go index e38bee873..358f636e8 100644 --- a/modules/mesh-config/cmd/mesh-config/proposals.go +++ b/modules/mesh-config/cmd/mesh-config/proposals.go @@ -910,19 +910,51 @@ func (c *Config) HeardEnd(w oc.Warrant) []string { c.mu.Lock() defer c.mu.Unlock() defer c.tellEnded(w.Ask, endedWord{Outcome: w.Outcome, Words: w.Words}) + // A retirement's ask ended with no choice is taken from the router's record the same way (retiring.go): the router's word + // is the only thing that moves it, and no timer looks. + said := c.decideRetirements() e, err := c.Store.Get(ProposalsState, w.Ask) if err != nil || e == nil { - return nil + return said } var p Proposal if json.Unmarshal(e.Value, &p) != nil || p.ID != w.Ask || p.State != ProposalWaiting { - return nil + return said } why := "the ask " + string(w.Outcome) if w.Words != "" { why += " (" + w.Words + ")" } - return []string{c.end(p, why+"; nothing was set")} + return append(said, c.end(p, why+"; nothing was set")) +} + +// NextDeadline is when a waiting proposal or retirement is next judged for want of an answer: a proposal with no record +// at the router ends an hour after its ask lapses (Decide), and a retirement left unanswered is asked once more after +// HeldFinding. The earliest of them; false when nothing waits on a time. +func (c *Config) NextDeadline() (time.Time, bool) { + c.mu.Lock() + defer c.mu.Unlock() + var at time.Time + earlier := func(t time.Time) { + if at.IsZero() || t.Before(at) { + at = t + } + } + if all, err := c.proposals(); err == nil { + for _, p := range all { + if p.State == ProposalWaiting { + earlier(p.At.Add(ProposalLasts + time.Hour)) + } + } + } + if held, err := c.readRetiring(); err == nil { + for _, e := range held { + if e.State == RetireUnanswered && !e.AskedAgain { + earlier(e.Asked.Add(HeldFinding)) + } + } + } + return at, !at.IsZero() } // ---- the list ---------------------------------------------------------------------------------------------- diff --git a/modules/mesh-config/cmd/mesh-config/pulls.go b/modules/mesh-config/cmd/mesh-config/pulls.go index 50f01da94..3508a95bf 100644 --- a/modules/mesh-config/cmd/mesh-config/pulls.go +++ b/modules/mesh-config/cmd/mesh-config/pulls.go @@ -534,7 +534,7 @@ func (c *Config) HeardPull(p PullEvent, merged bool) []string { } // SweepPulls looks at the forge for every pull request it keeps something for, so a merge or a close it did not hear -// is found: run with each five-minute resolution. +// is found: run once at start (resolving). func (c *Config) SweepPulls() []string { if c.Forge == nil { return nil diff --git a/modules/mesh-config/cmd/mesh-config/quiet_integration_test.go b/modules/mesh-config/cmd/mesh-config/quiet_integration_test.go new file mode 100644 index 000000000..deb93d759 --- /dev/null +++ b/modules/mesh-config/cmd/mesh-config/quiet_integration_test.go @@ -0,0 +1,135 @@ +//go:build integration + +package main + +// Issue 1982, against a real bus: mesh-config whole asks the bus for nothing while no input of it changes, and a changed input +// resolves only what it touches. The binary runs as the node's tool runtime plays it (whole_integration_test.go), and every +// request it makes of the runtime is counted: what a state read, a watch and a write ask of the bus. + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/testbus" +) + +func (p *player) snapshot() (calls map[string]int, puts []string) { + p.mu.Lock() + defer p.mu.Unlock() + calls = map[string]int{} + for k, v := range p.calls { + calls[k] = v + } + return calls, slices.Clone(p.puts) +} + +// asked is how many requests of these methods the module made between two snapshots. +func asked(before, after map[string]int, methods ...string) int { + n := 0 + for _, m := range methods { + n += after[m] - before[m] + } + return n +} + +func TestConfigWholeAsksTheBusForNothingWhileNoInputChanges(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + bin := filepath.Join(t.TempDir(), Module) + build := exec.Command("go", "build", "-o", bin, ".") + build.Stderr = os.Stderr + if err := build.Run(); err != nil { + t.Fatalf("mesh-config does not build: %v", err) + } + nc, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(nc.Close) + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + buckets := []string{"mesh-delivery-assigner_assignments", "mesh-delivery-assigner_wants", "mesh-delivery-indexer_predefined", + "mesh-delivery-indexer_config-candidates"} + for _, s := range []string{ValuesState, ResolvedState, PredefinedState, ProposalsState, PullsState, KeptState, DeclaredState, + RenamesState, RetiringState, FindingsState} { + buckets = append(buckets, Module+"_"+s) + } + for _, b := range buckets { + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b, History: 2}); err != nil { + t.Fatal(err) + } + } + haves := map[string]string{"g14.player": fpOf("player", fp1), "novox.player": fpOf("player", fp1), "g14.radio": fpOf("radio", fp1)} + p := start(t, bin, js, newStandIn(), haves, nil) + p.viewsReady() + + const radio = `{"module":"radio","config":{"band":{"type":"number","default":1}}}` + p.manifestIn("player", oldPlayer) + p.manifestIn("radio", radio) + p.buildIn("g14", "player", fpOf("player", fp1), oldPlayer) + p.buildIn("novox", "player", fpOf("player", fp1), oldPlayer) + p.buildIn("g14", "radio", fpOf("radio", fp1), radio) + p.placed("g14", "player", "radio") + p.placed("novox", "player") + p.hearsTheViews() + eventually(t, "every value is resolved", func() bool { + a, okA := resolvedValue(p, "g14", "player", "volume") + b, okB := resolvedValue(p, "novox", "player", "volume") + c, okC := resolvedValue(p, "g14", "radio", "band") + return okA && okB && okC && a == "5" && b == "5" && c == "1" + }) + // The passes the setup moved have ended when the module answers a tool: it takes its lock for none. + if _, err := p.call(ResyncTool, nil); err != nil { + t.Fatal(err) + } + + // No input changes: the delivery says it delivered, a node-engine says what it reads, the forge says a pull request nothing is + // kept for closed, and the catalog says a manifest was accepted that no retirement waits on. Each is heard and none is a pass: + // nothing is asked of the bus. (The module has no timer: no minute, five or otherwise, asks it anything.) + before, putsBefore := p.snapshot() + for i := 0; i < 20; i++ { + p.event("mesh-delivery-view.delivered", map[string]any{"revision": "r"}) + p.event("node-engine.manifest-reads-changed", map[string]any{"node": "g14"}) + p.event("git.pull-closed", map[string]any{"owner": "novox", "repo": "mesh-catalog", "number": 700 + i}) + } + after, putsAfter := p.snapshot() + if n := asked(before, after, "mesh/state.get", "mesh/state.keys", "mesh/state.watch", "mesh/state.put", "mesh/state.create", + "mesh/state.update", "mesh/state.delete", "mesh/ask"); n != 0 || len(putsAfter) != len(putsBefore) { + t.Errorf("60 events that move nothing asked %d requests of the bus (%v); wrote %v", n, after, putsAfter[len(putsBefore):]) + } + + // One input changes: player's manifest declares a second value. Only player's values on its two nodes are written, and `declared` + // for player; radio's resolved values are not written again, and nothing is read key by key. + before, putsBefore = p.snapshot() + const gain = `{"module":"player","config":{"volume":{"type":"number","default":5},"gain":{"type":"number","default":2}}}` + p.manifestIn("player", gain) + eventually(t, "the new value is resolved on both nodes", func() bool { + a, okA := resolvedValue(p, "g14", "player", "gain") + b, okB := resolvedValue(p, "novox", "player", "gain") + return okA && okB && a == "2" && b == "2" + }) + after, putsAfter = p.snapshot() + var wrote []string + for _, w := range putsAfter[len(putsBefore):] { + wrote = append(wrote, strings.TrimPrefix(w, "mesh/state.")) + } + for _, w := range wrote { + if !strings.Contains(w, ResolvedState+"/g14.player.") && !strings.Contains(w, ResolvedState+"/novox.player.") && + !strings.Contains(w, DeclaredState+"/player") { + t.Errorf("a changed manifest of player wrote %q, which is not player's", w) + } + } + if n := asked(before, after, "mesh/state.get", "mesh/state.keys", "mesh/state.watch"); n != 0 { + t.Errorf("a changed manifest read %d keys of the bus's states one by one (%v)", n, after) + } +} diff --git a/modules/mesh-config/cmd/mesh-config/quiet_test.go b/modules/mesh-config/cmd/mesh-config/quiet_test.go new file mode 100644 index 000000000..d6f89e77c --- /dev/null +++ b/modules/mesh-config/cmd/mesh-config/quiet_test.go @@ -0,0 +1,256 @@ +package main + +// What mesh-config asks of the bus per input (issue 1982): its own states are read from the copy their watches keep, an event +// that moves nothing reads nothing, and the one timer left is a deadline. + +import ( + "encoding/json" + "sync" + "testing" + "time" + + occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient" + oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// countingStore counts the reads of the states by state: what the bus would be asked. +type countingStore struct { + *memStore + mu sync.Mutex + reads map[string]int +} + +func (s *countingStore) Get(state, key string) (*Entry, error) { + s.mu.Lock() + s.reads[state]++ + s.mu.Unlock() + return s.memStore.Get(state, key) +} + +func (s *countingStore) Keys(state string) ([]string, error) { + s.mu.Lock() + s.reads[state]++ + s.mu.Unlock() + return s.memStore.Keys(state) +} + +// readsOf is the reads of every state but those named. +func (s *countingStore) readsExcept(states ...string) map[string]int { + s.mu.Lock() + defer s.mu.Unlock() + out := map[string]int{} + for k, n := range s.reads { + out[k] = n + } + for _, k := range states { + delete(out, k) + } + return out +} + +// **An event that moves nothing reads nothing but what it judges** (design 32 *A config value renamed*, *A value no longer +// read*): with no rename window open, `delivered` and `manifest-reads-changed` read no source, and a `manifest-accepted` +// whose rows are all marked or read again reads none either. Each was a read of every key of every state of the module +// (about 4,000 requests) under a delivery's burst of events. +func TestAnEventThatMovesNothingReadsNoSource(t *testing.T) { + c, m, d, _, _, _ := retiringMesh(t) + cs := &countingStore{memStore: m, reads: map[string]int{}} + c.Store = cs + d.rows = []ReaderRow{{Module: "player", Key: "volume", Kind: KindValue, Readers: []string{"g14"}}} // read: nothing to retire + for name, event := range map[string]func(){ + "delivered": func() { c.Windows("") }, + "manifest-reads-changed": func() { c.Windows("g14") }, + "manifest-accepted": func() { c.Retirements("player") }, + } { + cs.reads = map[string]int{} + event() + if got := cs.readsExcept(RenamesState, RetiringState); len(got) != 0 { + t.Errorf("%s read %v", name, got) + } + } +} + +// An open window is judged, and reads what judging it needs: the same event with a window open reads the sources. +func TestAnEventWithAWindowOpenJudgesIt(t *testing.T) { + c, m, _, _ := aRenamingMesh(t) + m.declare(t, "player", newPlayer) + if _, err := c.Reconcile(); err != nil { + t.Fatal(err) + } + cs := &countingStore{memStore: m, reads: map[string]int{}} + c.Store = cs + c.Windows("") + if cs.reads[ValuesState] == 0 { + t.Errorf("an open window was judged on no source: %v", cs.reads) + } +} + +// **An ask of a retirement that ends with no choice is taken when the router says so** (design 32 *A value no longer read*): +// with no minute's look at the router's record, the `decided` event of an ask that expired moves the retirement. +func TestARetirementWhoseAskExpiredIsTakenOnTheRoutersWord(t *testing.T) { + c, m, d, r, _, _ := retiringMesh(t) + for _, node := range []string{"g14", "novox"} { + m.want(t, node, "player", fp3, donePlayer) + d.haves[node+".player"] = fp3 + } + c.Windows("") + c.Retirements("player") + rows := retiringOf(t, c) + if len(rows) != 1 || rows[0].State != RetireAsked { + t.Fatalf("retiring %+v, want asked", rows) + } + r.choose(t, rows[0].Ask, OptionApprove, func(w *oc.Warrant) { w.Outcome = occ.OutcomeExpired }) + c.HeardEnd(oc.Warrant{Ask: rows[0].Ask, Asker: Module, Outcome: occ.OutcomeExpired}) + if rows = retiringOf(t, c); rows[0].State != RetireUnanswered { + t.Errorf("after the router said the ask expired: %+v", rows) + } +} + +// The next deadline is the earliest of a waiting proposal's (an hour past its ask lapsing, with no record at the router) +// and an unanswered retirement's (asked once more after HeldFinding); none when nothing waits on a time. +func TestTheNextDeadlineIsTheEarliestOfWhatWaitsOnATime(t *testing.T) { + m := newStore() + c := &Config{Store: m, Catalog: m, Logf: t.Logf} + if _, ok := c.NextDeadline(); ok { + t.Fatal("a deadline with nothing waiting") + } + t0 := time.Date(2026, 11, 19, 10, 0, 0, 0, time.UTC) + put := func(state, key string, v any) { + if err := m.Put(state, key, v); err != nil { + t.Fatal(err) + } + } + put(ProposalsState, "p1", Proposal{ID: "p1", State: ProposalWaiting, At: t0.Add(48 * time.Hour)}) + put(ProposalsState, "p2", Proposal{ID: "p2", State: ProposalEnded, At: t0}) + want := t0.Add(48*time.Hour + ProposalLasts + time.Hour) + if at, ok := c.NextDeadline(); !ok || !at.Equal(want) { + t.Errorf("deadline %v, %v; want %v", at, ok, want) + } + put(RetiringState, "player.volume", Retiring{Module: "player", Key: "volume", State: RetireUnanswered, Asked: t0.Add(-6 * 24 * time.Hour)}) + want = t0.Add(HeldFinding - 6*24*time.Hour) + if at, ok := c.NextDeadline(); !ok || !at.Equal(want) { + t.Errorf("deadline %v, %v; want the retirement's %v", at, ok, want) + } +} + +// busOf is a bus of states for the copy's tests: what each state holds, the watches handed out and the requests asked. +type busOf struct { + mu sync.Mutex + held map[string]map[string][]byte + handles map[string]func(stdio.StateChange) error + watches int + gets int + writes int + revision uint64 +} + +func (b *busOf) ownCopy() *ownCopy { + return &ownCopy{ + copy: stdio.WatchedStatesOver(func(state string, handle func(stdio.StateChange) error) error { + b.mu.Lock() + b.watches++ + b.handles[state] = handle + current := map[string][]byte{} + for k, v := range b.held[state] { + current[k] = v + } + b.mu.Unlock() + for k, v := range current { + if err := handle(stdio.StateChange{State: state, Key: k, Op: stdio.OpPut, Value: v, Revision: 1, Current: true}); err != nil { + return err + } + } + return nil + }, func(state, key string) ([]byte, bool, error) { + b.mu.Lock() + defer b.mu.Unlock() + b.gets++ + v, ok := b.held[state][key] + return v, ok, nil + }), + put: func(state, key string, value any) (uint64, error) { + raw, _ := json.Marshal(value) + b.mu.Lock() + defer b.mu.Unlock() + b.writes++ + b.revision++ + if b.held[state] == nil { + b.held[state] = map[string][]byte{} + } + b.held[state][key] = raw + return b.revision + 1, nil + }, + del: func(state, key string) error { + b.mu.Lock() + defer b.mu.Unlock() + b.writes++ + delete(b.held[state], key) + return nil + }} +} + +// **The module's own states are read from a copy, never key by key** (ADR 0580 §1, issue 1982): a whole state read of 1,176 +// keys is one watch and no get, again and again; this process's write is in the copy at once, another's arrives by the watch. +func TestAnOwnStateIsReadFromItsWatchedCopyNotKeyByKey(t *testing.T) { + b := &busOf{held: map[string]map[string][]byte{PullsState: {}}, handles: map[string]func(stdio.StateChange) error{}} + for i := 0; i < 1176; i++ { + b.held[PullsState][pullKey("novox/mesh-catalog#"+string(rune('a'+i%26))+time.Duration(i).String())] = []byte(`{"pull":"x"}`) + } + n := len(b.held[PullsState]) + own := b.ownCopy() + for pass := 0; pass < 3; pass++ { + keys, err := own.keys(PullsState) + if err != nil || len(keys) != n { + t.Fatalf("keys %d, %v; want %d", len(keys), err, n) + } + for _, k := range keys { + if e, err := own.get(PullsState, k); err != nil || e == nil { + t.Fatalf("get %s: %v, %v", k, e, err) + } + } + } + if b.watches != 1 || b.gets != 0 { + t.Errorf("3 passes over %d keys asked the bus for %d watches and %d gets; want 1 and 0", n, b.watches, b.gets) + } + // Its own write and removal are in the copy at once. + if err := own.write(PullsState, "mine", map[string]string{"pull": "mine"}); err != nil { + t.Fatal(err) + } + if e, _ := own.get(PullsState, "mine"); e == nil { + t.Error("a key just written is not read") + } + if err := own.remove(PullsState, "mine"); err != nil { + t.Fatal(err) + } + if e, _ := own.get(PullsState, "mine"); e != nil { + t.Error("a key just removed is still read") + } + // Another writer's change arrives by the watch. + if err := b.handles[PullsState](stdio.StateChange{State: PullsState, Key: "theirs", Op: stdio.OpPut, Value: []byte(`{"pull":"t"}`), Revision: 99}); err != nil { + t.Fatal(err) + } + if e, _ := own.get(PullsState, "theirs"); e == nil { + t.Error("a change another wrote is not read") + } + if b.gets != 0 { + t.Errorf("%d gets", b.gets) + } +} + +// A write and a removal tell what waits on a deadline, once each. +func TestAWriteTellsWhatWaitsOnADeadline(t *testing.T) { + b := &busOf{held: map[string]map[string][]byte{}, handles: map[string]func(stdio.StateChange) error{}} + own := b.ownCopy() + var told []string + own.changed = func(state string) { told = append(told, state) } + if err := own.write(ProposalsState, "p", Proposal{ID: "p"}); err != nil { + t.Fatal(err) + } + if err := own.remove(ProposalsState, "p"); err != nil { + t.Fatal(err) + } + if len(told) != 2 || told[0] != ProposalsState { + t.Errorf("told %v", told) + } +} diff --git a/modules/mesh-config/cmd/mesh-config/renames.go b/modules/mesh-config/cmd/mesh-config/renames.go index f7338a38a..64417796f 100644 --- a/modules/mesh-config/cmd/mesh-config/renames.go +++ b/modules/mesh-config/cmd/mesh-config/renames.go @@ -293,14 +293,26 @@ func (c *Config) Windows(node string) []string { return said } +// windowOpen says whether any rename holds a window open on the node named, or on any node when none is named. +func windowOpen(renames map[string]Rename, only string) bool { + for _, r := range renames { + for _, node := range r.Open() { + if only == "" || only == node { + return true + } + } + } + return false +} + // closeWindows is Windows' rename half; the caller holds the lock. func (c *Config) closeWindows(only string) []string { renames, err := c.readRenames() if err != nil { return []string{"the renames cannot be read: " + err.Error()} } - if c.Delivery == nil { - return nil + if c.Delivery == nil || !windowOpen(renames, only) { + return nil // no window to judge: the delivery's word moves nothing, and no source is read for it (issue 1982) } s, err := c.sources() if err != nil { diff --git a/modules/mesh-config/cmd/mesh-config/retiring.go b/modules/mesh-config/cmd/mesh-config/retiring.go index 6f6bd2c16..3be623f7c 100644 --- a/modules/mesh-config/cmd/mesh-config/retiring.go +++ b/modules/mesh-config/cmd/mesh-config/retiring.go @@ -148,9 +148,15 @@ func (c *Config) Retirements(module string) []string { if err != nil { return []string{"the retirements cannot be read: " + err.Error()} } - s, err := c.sources() - if err != nil { - return []string{"the retirements cannot be judged: " + err.Error()} + // The sources are read only when a row is one to judge: a row already marked, or read again, needs none (issue 1982). + var s Sources + for _, row := range rows { + if _, has := held[retiringKey(module, row.Key)]; (row.Module == "" || row.Module == module) && (len(row.Readers) == 0 && !has) { + if s, err = c.sources(); err != nil { + return []string{"the retirements cannot be judged: " + err.Error()} + } + break + } } var said []string for _, row := range rows { diff --git a/modules/mesh-config/cmd/mesh-config/whole_integration_test.go b/modules/mesh-config/cmd/mesh-config/whole_integration_test.go index 9cc30a578..83d9af240 100644 --- a/modules/mesh-config/cmd/mesh-config/whole_integration_test.go +++ b/modules/mesh-config/cmd/mesh-config/whole_integration_test.go @@ -81,6 +81,11 @@ type player struct { catalog map[string]catalog.CatalogEntry byFp map[string]catalog.CatalogEntry catRev int64 + // calls counts what the module asked of the runtime, by method (and by key for an ask); puts are the keys it wrote, "/". + calls map[string]int + puts []string + // stop ends the module this player runs: the restart of a module is the old process gone and a new one started. + stop func() } // ownedState is one state of an owner (the indexer's modules, the assigner's wants...), kept from its bucket. @@ -232,6 +237,12 @@ func (p *player) handle(m wire) { Bucket string `json:"bucket"` } _ = json.Unmarshal(m.Params, &a) + p.mu.Lock() + p.calls[m.Method]++ + if strings.HasPrefix(m.Method, "mesh/state.") && m.Method != "mesh/state.get" && m.Method != "mesh/state.keys" && m.Method != "mesh/state.watch" { + p.puts = append(p.puts, m.Method+" "+a.State+"/"+a.Key) + } + p.mu.Unlock() kv, kvErr := p.kv(a.State) switch m.Method { case "mesh/state.get": @@ -675,14 +686,16 @@ func start(t *testing.T, bin string, js jetstream.JetStream, router *standIn, ha stdout, _ := cmd.StdoutPipe() cmd.Stderr = os.Stderr p := &player{t: t, ctx: ctx, js: js, router: router, in: stdin, pending: map[string]chan wire{}, haves: haves, readers: readers, - owned: map[string]*ownedState{}, catalog: map[string]catalog.CatalogEntry{}, byFp: map[string]catalog.CatalogEntry{}} + owned: map[string]*ownedState{}, catalog: map[string]catalog.CatalogEntry{}, byFp: map[string]catalog.CatalogEntry{}, + calls: map[string]int{}} for _, f := range followed { p.own(f.Read) } if err := cmd.Start(); err != nil { t.Fatal(err) } - t.Cleanup(func() { _ = stdin.Close(); cancel(); _ = cmd.Wait() }) + p.stop = func() { _ = stdin.Close(); cancel(); _ = cmd.Wait() } + t.Cleanup(p.stop) go p.serve(stdout) t.Cleanup(cancel) return p @@ -785,6 +798,7 @@ func TestConfigWholeRenamesAndRetiresAgainstTheBus(t *testing.T) { // The module restarted inside the window keeps it, and says renamed no more. said := p.publishedCount(EventRenamed) + p.stop() p2 := start(t, bin, js, router, haves, nil) p2.viewsReady() if _, err := p2.call(ResyncTool, nil); err != nil {