diff --git a/modules/mesh-grants/cmd/mesh-grants/main.go b/modules/mesh-grants/cmd/mesh-grants/main.go index ea6860c0..2e86973f 100644 --- a/modules/mesh-grants/cmd/mesh-grants/main.go +++ b/modules/mesh-grants/cmd/mesh-grants/main.go @@ -28,6 +28,7 @@ import ( "os/signal" "strings" "syscall" + "time" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" @@ -117,7 +118,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // followed on the owners' views families, which the hearing below hands them. sayCond := sayVia(ctx, js, account.Module) fv := grants.NewViews(func(tool string) func(context.Context, string) (views.Answer, error) { - return views.AskVia(grants.Unwrapped(conn.Call), tool) + return grants.TimedAsk(tool, views.AskVia(grants.Unwrapped(conn.Call), tool), time.Now, logger.Printf) }, sayCond, f.Wake, logger.Printf) fv.Rest, fv.Watching = bus, bus byState := map[string]*views.Follower{} diff --git a/modules/mesh-grants/grants_test.go b/modules/mesh-grants/grants_test.go index 117f1ee0..09d41dd1 100644 --- a/modules/mesh-grants/grants_test.go +++ b/modules/mesh-grants/grants_test.go @@ -801,6 +801,7 @@ func TestGrants(t *testing.T) { } return nil } + p.Say = channelTakes(t, p.Say) for _, s := range b.streams { if s["name"] == "OBJ_mesh-knowledge_sections" { s["bytes"] = int64(3500 << 20) @@ -1664,6 +1665,7 @@ func TestGrants(t *testing.T) { var said []oc.Condition p.Say = func(c oc.Condition) error { said = append(said, c); return nil } + p.Say = channelTakes(t, p.Say) // settled is the assignments of the lists agreeing, every hash held, and the loaded list as edit leaves it. settled := func(edit func(modX *grants.User)) func() { return func() { @@ -1768,6 +1770,7 @@ func TestGrants(t *testing.T) { } return nil } + fp.Say = channelTakes(t, fp.Say) fp.Compose() if len(fresh) == 0 || fresh[0].Key != grants.HeldCondition || fresh[0].State != occ.StateCleared { t.Errorf("a fresh part's first pass does not say the held warning is cleared: %+v", fresh) @@ -2442,6 +2445,7 @@ func TestGrants(t *testing.T) { } return nil } + p.Say = channelTakes(t, p.Say) // With no lease key (1126/A1). // The private network the mesh provides is placed with no fingerprint by construction: never a missing grant. b.assign("b", "mesh-wireguard") @@ -2636,6 +2640,14 @@ func TestGrants(t *testing.T) { b.put(grants.ReadHave, "b.mod-y", runs) b.put(grants.ReadHave, "a.mesh-delivery-indexer", runs) }, ""}, + // The wants view unanswered, the assignments answered (issue 1966, the list held from 2026-11-08 16:39): what runs + // comes from the haves, the wants are no failed read, and the pass only adds. + {"the wants view unanswered", func(b *fakeBus, p *grants.Part) { + b.refused[grants.ReadWants] = true + if in := grants.ReadInputs(b, &grants.Manifests{}); in.WantsUnread == "" || len(in.ReadFailed()) != 0 { + t.Errorf("an unanswered wants view is a failed read, or composes no fallback: %q, %v", in.WantsUnread, in.ReadFailed()) + } + }, ""}, // The ledger of a held module stays with its module's users, whatever the seat list (held.go). {"the ledger of a module whose manifest is refused", func(b *fakeBus, p *grants.Part) { b.refuse("mod-x", ledgerX, "it is held") @@ -3074,6 +3086,7 @@ func TestGrants(t *testing.T) { sayMu.Unlock() return nil } + say = channelTakes(t, say) fv := grants.NewViews(owner, say, func() {}, nil) fv.Rest = b var minutes atomic.Int64 @@ -3467,6 +3480,7 @@ func TestGrants(t *testing.T) { } return nil } + p.Say = channelTakes(t, p.Say) made["max_deliver"] = 9 res = p.Compose() if len(res.Consumers.Differ) != 1 || !strings.Contains(res.Consumers.Differ[0], "EVENTS.b_mod-c: max deliver 9") || @@ -3538,6 +3552,7 @@ func TestGrants(t *testing.T) { p.Objects = &fakeObjects{} var said []oc.Condition p.Say = func(c oc.Condition) error { said = append(said, c); return nil } + p.Say = channelTakes(t, p.Say) p.Compose() inForce, _, _ := p.Sealed() if inForce.Digest == "" { @@ -3708,6 +3723,7 @@ func TestGrants(t *testing.T) { } return nil } + p.Say = channelTakes(t, p.Say) p.Compose() inForce, _, _ := p.Sealed() if inForce.Digest == "" { @@ -3793,6 +3809,7 @@ func TestGrants(t *testing.T) { } return nil } + p.Say = channelTakes(t, p.Say) p.Compose() inForce, _, _ := p.Sealed() before, _, ok, _ := b.ownGet(grants.StateUserList, grants.SealedUserKey(inForce.Digest, "b.mod-r.mod-r-tools")) @@ -3888,6 +3905,28 @@ func TestGrants(t *testing.T) { t.Errorf("the kept tool runner was not said: %+v", last) } }) + + // Each first view's page says in the log how long it took (issue 1966: the wants page outgrew the call's bound unseen). + t.Run("a first view's page says how long it took", func(t *testing.T) { + at := time.Date(2026, 11, 8, 22, 0, 0, 0, time.UTC) + now := func() time.Time { at = at.Add(1500 * time.Millisecond); return at } + for _, row := range []struct { + name string + err error + want string + }{ + {"answered", nil, `the first view mesh-delivery-assigner.wants, the page after "", took 1.5s: 2 keys at revision 9, next ""`}, + {"failed", errors.New("context deadline exceeded"), `the first view mesh-delivery-assigner.wants, the page after "", failed after 1.5s: context deadline exceeded`}, + } { + var lines []string + ask := grants.TimedAsk(grants.ReadWants, func(context.Context, string) (views.Answer, error) { + return views.Answer{Revision: 9, Values: map[string]json.RawMessage{"a": nil, "b": nil}}, row.err + }, now, func(f string, a ...any) { lines = append(lines, fmt.Sprintf(f, a...)) }) + if _, err := ask(context.Background(), ""); !errors.Is(err, row.err) || len(lines) != 1 || !strings.Contains(lines[0], row.want) { + t.Errorf("%s: said %q (%v), want %q", row.name, lines, err, row.want) + } + } + }) } // fakeObjects is the part's object bucket of sealed lists. @@ -4084,3 +4123,15 @@ func TestPassComposesMembershipFromTheHaveBuild(t *testing.T) { } }) } + +// channelTakes is say, after the operator-channel's own check of each condition: one the channel would refuse (its +// explanation not opening with "Needs you: " and what it needs) fails the test, since the operator would never see it. +func channelTakes(t *testing.T, say func(oc.Condition) error) func(oc.Condition) error { + t.Helper() + return func(c oc.Condition) error { + if _, err := occ.EncodeCondition(c); err != nil { + t.Errorf("the channel refuses the condition %s: %v", c.Key, err) + } + return say(c) + } +} diff --git a/modules/mesh-grants/internal/grants/consumers.go b/modules/mesh-grants/internal/grants/consumers.go index 38d08ba6..c3ee1029 100644 --- a/modules/mesh-grants/internal/grants/consumers.go +++ b/modules/mesh-grants/internal/grants/consumers.go @@ -224,9 +224,9 @@ func (p *Part) sayDiffer(st *ConsumersState) { } else { c.Severity, c.State = occ.SeverityWarning, occ.StateRaised c.Summary = fmt.Sprintf("%d durable consumers differ from their composed configuration", len(st.Differ)) - c.Headline, c.Needs = "A module's consumer is not as composed", "a look, and grants_consumer_match for each" - c.Explanation = occ.NeedsYou + "a look. The bus holds these consumers otherwise than the bus composer composes them, " + - "and the grants part never changes a held one by itself (its delivery position would be lost): " + why + c.Headline = "A module's consumer is not as composed" + needsYou(&c, "a look, and grants_consumer_match for each", "The bus holds these consumers otherwise than the bus composer "+ + "composes them, and the grants part never changes a held one by itself (its delivery position would be lost): "+why) c.Resolved = "every account consumer matches, by grants_consumer_match or otherwise" } if p.Say(c) == nil { @@ -321,9 +321,9 @@ func (p *Part) sayBehind(st *ConsumersState) { } else { c.Severity, c.State = occ.SeverityWarning, occ.StateRaised c.Summary = fmt.Sprintf("%d durable consumers are more than %d messages behind their stream's head", len(st.Behind), BehindBound) - c.Headline, c.Needs = "Messages pile up for a listener", "a look at the module that reads them" - c.Explanation = occ.NeedsYou + "a look. These consumers stayed far behind their stream's head at two passes " + - "running, so what is sent to their modules waits unread (how far each is, in the grants part's last pass): " + why + c.Headline = "Messages pile up for a listener" + needsYou(&c, "a look at the module that reads them", "These consumers stayed far behind their stream's head at two passes "+ + "running, so what is sent to their modules waits unread (how far each is, in the grants part's last pass): "+why) c.Resolved = "every account consumer is back within the bound" } if p.Say(c) == nil { diff --git a/modules/mesh-grants/internal/grants/firstviews.go b/modules/mesh-grants/internal/grants/firstviews.go index 2871c7f5..11aace93 100644 --- a/modules/mesh-grants/internal/grants/firstviews.go +++ b/modules/mesh-grants/internal/grants/firstviews.go @@ -354,9 +354,9 @@ func unansweredCondition(fv FirstView, s views.Status, raised bool) oc.Condition c.Summary = fmt.Sprintf("the grants part has no first view of %s's %s: %s has not answered since %s (%s)", fv.Owner, fv.State, fv.Tool, s.Asked.Format(time.RFC3339), strings.TrimSpace(s.Error)) c.Headline = "The grants part waits for " + fv.Owner - c.Explanation = "The grants part reads " + fv.Owner + "'s " + fv.State + " only through " + fv.Tool + ", and keeps asking it; " + - "it never reads the state itself, and keeps what it composed meanwhile (novox/hq ADR 0615 §6). The answer clears this." - c.Needs = "a look at " + fv.Owner + "'s runtime of its own: is it running and connected?" + needsYou(&c, "a look at "+fv.Owner+"'s runtime of its own: is it running and connected?", "The grants part reads "+fv.Owner+ + "'s "+fv.State+" only through "+fv.Tool+", and keeps asking it; it never reads the state itself, and keeps what it "+ + "composed meanwhile (novox/hq ADR 0615 §6). The answer clears this.") return c } @@ -376,10 +376,10 @@ func emptyCondition(fv FirstView, raised bool) oc.Condition { c.State, c.Severity = occ.StateRaised, occ.SeverityWarning c.Summary = fmt.Sprintf("%s answered %s's %s empty at revision 0, so the grants part keeps the user list in force", fv.Tool, fv.Owner, fv.State) c.Headline = fv.Owner + "'s " + fv.State + " is empty" - c.Explanation = occ.NeedsYou + "a step. The owner holds nothing of " + fv.State + " (its state rebuilt by a recreate that could " + - "not keep the data, or a fresh mesh), so the grants part drops no login and keeps what it holds; a login revoked meanwhile " + - "stays until the owner's first change (novox/hq design 25 § 12.2)." - c.Needs = "let the owner write its state again from its sources (" + step + "), then grants_first_view_again" + needsYou(&c, "let the owner write its state again from its sources ("+step+"), then grants_first_view_again", "The owner holds "+ + "nothing of "+fv.State+" (its state rebuilt by a recreate that could not keep the data, or a fresh mesh), so the grants "+ + "part drops no login and keeps what it holds; a login revoked meanwhile stays until the owner's first change "+ + "(novox/hq design 25 § 12.2).") return c } @@ -479,6 +479,26 @@ func (p *Part) AskFirstViewAgain(caller, state, why string, record func(what, wh "says": "the part's process asks it once more at this ask; grants_first_views shows its answer"}, nil } +// TimedAsk is ask, saying in the log how long each page took and how many keys it brought, or after how long it failed: +// a first view that never answers (issue 1966: the wants page outgrew the call's bound) shows how far from its bound it is. +func TimedAsk(tool string, ask func(context.Context, string) (views.Answer, error), now func() time.Time, + logf func(string, ...any)) func(context.Context, string) (views.Answer, error) { + return func(ctx context.Context, after string) (views.Answer, error) { + start := now() + a, err := ask(ctx, after) + took := now().Sub(start).Round(time.Millisecond) + switch { + case logf == nil: + case err != nil: + logf("[mesh-grants] the first view %s, the page after %q, failed after %s: %v", tool, after, took, err) + default: + logf("[mesh-grants] the first view %s, the page after %q, took %s: %d keys at revision %d, next %q", tool, after, took, + len(a.Values), a.Revision, a.Next) + } + return a, err + } +} + // Unwrapped is a call whose answer is read as the tool's own text where the runtime wraps it as content (unwrap), as // a first view's answer is decoded (views.AskVia). func Unwrapped(call func(key string, body any) (json.RawMessage, error)) func(key string, body any) (json.RawMessage, error) { diff --git a/modules/mesh-grants/internal/grants/grants.go b/modules/mesh-grants/internal/grants/grants.go index 3390df0a..25fc80d8 100644 --- a/modules/mesh-grants/internal/grants/grants.go +++ b/modules/mesh-grants/internal/grants/grants.go @@ -339,9 +339,9 @@ func (p *Part) sayMissing(missing []string) { } else { c.Severity, c.State = occ.SeverityWarning, occ.StateRaised c.Summary = fmt.Sprintf("%d placed module(s) have no grant, so no want can name them", len(missing)) - c.Headline, c.Needs = "A placed module has no bus grant", "a look at why each has none" - c.Explanation = occ.NeedsYou + "a look. The grants part wrote no grant for these modules, so the assigner's " + - "credential rule gives them no want: " + why + c.Headline = "A placed module has no bus grant" + needsYou(&c, "a look at why each has none", "The grants part wrote no grant for these modules, so the assigner's "+ + "credential rule gives them no want: "+why) c.Resolved = "every placed module has a grant at the fingerprint the indexer names" } if p.Say(c) == nil { diff --git a/modules/mesh-grants/internal/grants/held.go b/modules/mesh-grants/internal/grants/held.go index 23dc5c33..6bf1cad2 100644 --- a/modules/mesh-grants/internal/grants/held.go +++ b/modules/mesh-grants/internal/grants/held.go @@ -143,9 +143,10 @@ func (p *Part) sayHeldModules(held map[string]string, kept map[string]User) { } else { c.Severity, c.State = occ.SeverityWarning, occ.StateRaised c.Summary = fmt.Sprintf("%d bus user(s) of a module whose manifest is held are kept in the list: %s", len(users), why) - c.Headline, c.Needs = "A module's manifest is held, its bus users are kept", "a manifest the indexer accepts, or a judge that knows its fields" - c.Explanation = occ.NeedsYou + "a manifest the indexer accepts (a refused one mended, or a waiting one judged by an indexer that knows its fields). " + - "The rest of the user list is written as ever; a held module's users stay as the list in force holds them: " + why + c.Headline = "A module's manifest is held, its bus users are kept" + needsYou(&c, "a manifest the indexer accepts, or a judge that knows its fields", "A refused one mended, or a waiting one "+ + "judged by an indexer that knows its fields. The rest of the user list is written as ever; a held module's users stay "+ + "as the list in force holds them: "+why) c.Resolved = "every module's manifest is accepted" } if p.Say(c) == nil { diff --git a/modules/mesh-grants/internal/grants/inputs.go b/modules/mesh-grants/internal/grants/inputs.go index 4b63aa60..dc8ff14f 100644 --- a/modules/mesh-grants/internal/grants/inputs.go +++ b/modules/mesh-grants/internal/grants/inputs.go @@ -180,6 +180,11 @@ type Inputs struct { // answering (bootstrap only, issue 1966); empty when the assignments placed them. Such a pass retires nothing // (ReadFailures) and seals no list dropping a user of the list in force (holdsBack). ByHaves string + // WantsUnread says why this pass composed what runs from each have's fingerprint, the wants view not answering while + // the assignments view did (bootstrap only, issue 1966: a wants page the running assigner cannot answer in a call's + // bound); empty when the wants were read. As ByHaves, such a pass retires nothing (ReadFailures) and only adds + // (keepsLoaded): the list is held only where it would take a permission from a loaded user (holdsBack). + WantsUnread string // Held is, by module, the manifest a held module's running build was judged on (its have's fingerprint, the builder's // build of it): what the seats it defines, the streams others hear from it and its reads are composed from while its // trunk manifest is refused or waits, so no other principal loses a grant to it (ADR 0601 clause 3). @@ -519,12 +524,20 @@ func ReadInputs(r Reader, cache *Manifests) Inputs { // What each machine is wanted to run: the want's fingerprint, the builder's build of it, the manifest it built. var fromTrunk []string wants := map[string][]byte{} - if in.ByHaves == "" { + _, wantsErr := r.Keys(ReadWants) + switch { + case in.ByHaves == "" && wantsErr == nil: wants = each(r, ReadWants, &in.Gaps) // What each machine runs now is its have's build, which the want may already have left (issue 1966): the // memberships keep what either build reads, calls and consumes. runningByHaves(r, &in, cache, in.Had) - } else { + case in.ByHaves == "": + // **Bootstrap only** (issue 1966), as the assignments' placement by the haves: while the wants view does not answer, + // what runs is each placed module's have's fingerprint, said as a gap that is no failed read, and the pass only adds. + in.WantsUnread = "the wants view did not answer (" + wantsErr.Error() + "), so what runs is composed from each have's fingerprint" + in.Gaps = append(in.Gaps, Gap{Input: ReadWants, Why: "refused or absent: what runs is composed from each have's fingerprint instead, until it answers (" + wantsErr.Error() + ")"}) + fromTrunk = runningByHaves(r, &in, cache, in.Running) + default: // Placed by the haves, what runs is what each have's fingerprint names, not a want: the assigner's wants are its // view too, and not read in such a pass. in.Gaps = append(in.Gaps, Gap{Input: ReadWants, Why: "not read: the modules are placed by the node-engines' haves this pass, so what runs is composed from each have's fingerprint"}) @@ -716,6 +729,9 @@ func (in Inputs) ReadFailures() []string { if in.ByHaves != "" { out = append(out, ReadAssignments+": "+in.ByHaves) } + if in.WantsUnread != "" { + out = append(out, ReadWants+": "+in.WantsUnread) + } for _, g := range in.Refused { out = append(out, g.Input+": "+g.Why) } diff --git a/modules/mesh-grants/internal/grants/keeploaded.go b/modules/mesh-grants/internal/grants/keeploaded.go index dee095b4..23693a87 100644 --- a/modules/mesh-grants/internal/grants/keeploaded.go +++ b/modules/mesh-grants/internal/grants/keeploaded.go @@ -21,6 +21,8 @@ func (in Inputs) keepsLoaded() string { switch { case in.ByHaves != "": return in.ByHaves + case in.WantsUnread != "": + return in.WantsUnread case in.SeatListUnread: return "the seat list is unread, so only the bootstrapped seats compose" } diff --git a/modules/mesh-grants/internal/grants/needsyou.go b/modules/mesh-grants/internal/grants/needsyou.go new file mode 100644 index 00000000..1d9845a5 --- /dev/null +++ b/modules/mesh-grants/internal/grants/needsyou.go @@ -0,0 +1,20 @@ +package grants + +import ( + "strings" + + occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient" + oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" +) + +// needsYou sets what a raised condition needs of the operator and its explanation: the channel takes a condition whose +// explanation opens with "Needs you: " and those very words (operatorchannelclient.EncodeCondition, novox/hq ADR 0253 §1), +// and refuses any other, so the operator would never see it. rest is the explanation after them. +func needsYou(c *oc.Condition, needs, rest string) { + c.Needs = needs + sep := ". " + if strings.HasSuffix(needs, "?") || strings.HasSuffix(needs, ".") { + sep = " " + } + c.Explanation = occ.NeedsYou + needs + sep + rest +} diff --git a/modules/mesh-grants/internal/grants/refused.go b/modules/mesh-grants/internal/grants/refused.go index bff94ec3..54b32caa 100644 --- a/modules/mesh-grants/internal/grants/refused.go +++ b/modules/mesh-grants/internal/grants/refused.go @@ -127,9 +127,10 @@ func (p *Part) sayPrincipalsRefused(refused map[string]string, kept map[string]U } else { c.Severity, c.State = occ.SeverityWarning, occ.StateRaised c.Summary = fmt.Sprintf("%d principal(s) refused their composed grants: %s", len(refused), why) - c.Headline, c.Needs = "A principal's bus grants are refused", "a fix of the manifest or the writers table" - c.Explanation = occ.NeedsYou + "a fix of what makes it a second writer. The rest of the user list is written as ever; " + - "each refused principal keeps the grants the list in force gives it, or stays out where it holds none: " + why + c.Headline = "A principal's bus grants are refused" + needsYou(&c, "a fix of the manifest or the writers table, of what makes it a second writer", + "The rest of the user list is written as ever; each refused principal keeps the grants the list in force gives it, "+ + "or stays out where it holds none: "+why) c.Resolved = "every principal's grants compose again" } if p.Say(c) == nil { diff --git a/modules/mesh-grants/internal/grants/sealed.go b/modules/mesh-grants/internal/grants/sealed.go index 591c5c94..cd5b27ff 100644 --- a/modules/mesh-grants/internal/grants/sealed.go +++ b/modules/mesh-grants/internal/grants/sealed.go @@ -180,8 +180,8 @@ func (p *Part) sayHeld(why string) { } else { reason := strings.TrimPrefix(why, "held: ") c.Severity, c.State = occ.SeverityWarning, occ.StateRaised - c.Summary, c.Headline, c.Needs = "the user list is held: "+reason, "The user list in force is held back", "a look at why" - c.Explanation = occ.NeedsYou + "a look at why. The bus node keeps following the list written before, not the one composed now: " + reason + c.Summary, c.Headline = "the user list is held: "+reason, "The user list in force is held back" + needsYou(&c, "a look at why", "The bus node keeps following the list written before, not the one composed now: "+reason) c.Resolved = "a pass finds nothing to hold back and the list advances" } if p.Say(c) == nil {