novox/mesh-delivery
10
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request 'Grants composes from the haves while the wants view is unanswered, and its conditions reach the operator (issue 1966, recovery)' (#180) from fix/1966-first-view-one-read into main

Browse Source
This commit was merged in pull request #180.
This commit is contained in:
mesh-tools
2026-11-08 22:31:58 +00:00
parent 3f96301177 160fca6a47
commit 7d8e2060a7
11 changed files with 139 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/mesh-grants/cmd/mesh-grants/main.go
+2 -1
View File
@@ -28,6 +28,7 @@ import (
"os/signal"
"strings"
"syscall"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
@@ -117,7 +118,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// followed on the owners' views families, which the hearing below hands them.
sayCond := sayVia(ctx, js, account.Module)
fv := grants.NewViews(func(tool string) func(context.Context, string) (views.Answer, error) {
return views.AskVia(grants.Unwrapped(conn.Call), tool)
return grants.TimedAsk(tool, views.AskVia(grants.Unwrapped(conn.Call), tool), time.Now, logger.Printf)
}, sayCond, f.Wake, logger.Printf)
fv.Rest, fv.Watching = bus, bus
byState := map[string]*views.Follower{}
modules/mesh-grants/grants_test.go
+51
View File
@@ -801,6 +801,7 @@ func TestGrants(t *testing.T) {
}
return nil
}
p.Say = channelTakes(t, p.Say)
for _, s := range b.streams {
if s["name"] == "OBJ_mesh-knowledge_sections" {
s["bytes"] = int64(3500 << 20)
@@ -1664,6 +1665,7 @@ func TestGrants(t *testing.T) {
var said []oc.Condition
p.Say = func(c oc.Condition) error { said = append(said, c); return nil }
p.Say = channelTakes(t, p.Say)
// settled is the assignments of the lists agreeing, every hash held, and the loaded list as edit leaves it.
settled := func(edit func(modX *grants.User)) func() {
return func() {
@@ -1768,6 +1770,7 @@ func TestGrants(t *testing.T) {
}
return nil
}
fp.Say = channelTakes(t, fp.Say)
fp.Compose()
if len(fresh) == 0 || fresh[0].Key != grants.HeldCondition || fresh[0].State != occ.StateCleared {
t.Errorf("a fresh part's first pass does not say the held warning is cleared: %+v", fresh)
@@ -2442,6 +2445,7 @@ func TestGrants(t *testing.T) {
}
return nil
}
p.Say = channelTakes(t, p.Say)
// With no lease key (1126/A1).
// The private network the mesh provides is placed with no fingerprint by construction: never a missing grant.
b.assign("b", "mesh-wireguard")
@@ -2636,6 +2640,14 @@ func TestGrants(t *testing.T) {
b.put(grants.ReadHave, "b.mod-y", runs)
b.put(grants.ReadHave, "a.mesh-delivery-indexer", runs)
}, ""},
// The wants view unanswered, the assignments answered (issue 1966, the list held from 2026-11-08 16:39): what runs
// comes from the haves, the wants are no failed read, and the pass only adds.
{"the wants view unanswered", func(b *fakeBus, p *grants.Part) {
b.refused[grants.ReadWants] = true
if in := grants.ReadInputs(b, &grants.Manifests{}); in.WantsUnread == "" || len(in.ReadFailed()) != 0 {
t.Errorf("an unanswered wants view is a failed read, or composes no fallback: %q, %v", in.WantsUnread, in.ReadFailed())
}
}, ""},
// The ledger of a held module stays with its module's users, whatever the seat list (held.go).
{"the ledger of a module whose manifest is refused", func(b *fakeBus, p *grants.Part) {
b.refuse("mod-x", ledgerX, "it is held")
@@ -3074,6 +3086,7 @@ func TestGrants(t *testing.T) {
sayMu.Unlock()
return nil
}
say = channelTakes(t, say)
fv := grants.NewViews(owner, say, func() {}, nil)
fv.Rest = b
var minutes atomic.Int64
@@ -3467,6 +3480,7 @@ func TestGrants(t *testing.T) {
}
return nil
}
p.Say = channelTakes(t, p.Say)
made["max_deliver"] = 9
res = p.Compose()
if len(res.Consumers.Differ) != 1 || !strings.Contains(res.Consumers.Differ[0], "EVENTS.b_mod-c: max deliver 9") ||
@@ -3538,6 +3552,7 @@ func TestGrants(t *testing.T) {
p.Objects = &fakeObjects{}
var said []oc.Condition
p.Say = func(c oc.Condition) error { said = append(said, c); return nil }
p.Say = channelTakes(t, p.Say)
p.Compose()
inForce, _, _ := p.Sealed()
if inForce.Digest == "" {
@@ -3708,6 +3723,7 @@ func TestGrants(t *testing.T) {
}
return nil
}
p.Say = channelTakes(t, p.Say)
p.Compose()
inForce, _, _ := p.Sealed()
if inForce.Digest == "" {
@@ -3793,6 +3809,7 @@ func TestGrants(t *testing.T) {
}
return nil
}
p.Say = channelTakes(t, p.Say)
p.Compose()
inForce, _, _ := p.Sealed()
before, _, ok, _ := b.ownGet(grants.StateUserList, grants.SealedUserKey(inForce.Digest, "b.mod-r.mod-r-tools"))
@@ -3888,6 +3905,28 @@ func TestGrants(t *testing.T) {
t.Errorf("the kept tool runner was not said: %+v", last)
}
})
// Each first view's page says in the log how long it took (issue 1966: the wants page outgrew the call's bound unseen).
t.Run("a first view's page says how long it took", func(t *testing.T) {
at := time.Date(2026, 11, 8, 22, 0, 0, 0, time.UTC)
now := func() time.Time { at = at.Add(1500 * time.Millisecond); return at }
for _, row := range []struct {
name string
err error
want string
}{
{"answered", nil, `the first view mesh-delivery-assigner.wants, the page after "", took 1.5s: 2 keys at revision 9, next ""`},
{"failed", errors.New("context deadline exceeded"), `the first view mesh-delivery-assigner.wants, the page after "", failed after 1.5s: context deadline exceeded`},
} {
var lines []string
ask := grants.TimedAsk(grants.ReadWants, func(context.Context, string) (views.Answer, error) {
return views.Answer{Revision: 9, Values: map[string]json.RawMessage{"a": nil, "b": nil}}, row.err
}, now, func(f string, a ...any) { lines = append(lines, fmt.Sprintf(f, a...)) })
if _, err := ask(context.Background(), ""); !errors.Is(err, row.err) || len(lines) != 1 || !strings.Contains(lines[0], row.want) {
t.Errorf("%s: said %q (%v), want %q", row.name, lines, err, row.want)
}
}
})
}
// fakeObjects is the part's object bucket of sealed lists.
@@ -4084,3 +4123,15 @@ func TestPassComposesMembershipFromTheHaveBuild(t *testing.T) {
}
})
}
// channelTakes is say, after the operator-channel's own check of each condition: one the channel would refuse (its
// explanation not opening with "Needs you: " and what it needs) fails the test, since the operator would never see it.
func channelTakes(t *testing.T, say func(oc.Condition) error) func(oc.Condition) error {
t.Helper()
return func(c oc.Condition) error {
if _, err := occ.EncodeCondition(c); err != nil {
t.Errorf("the channel refuses the condition %s: %v", c.Key, err)
}
return say(c)
}
}
modules/mesh-grants/internal/grants/consumers.go
+6 -6
View File
@@ -224,9 +224,9 @@ func (p *Part) sayDiffer(st *ConsumersState) {
} else {
c.Severity, c.State = occ.SeverityWarning, occ.StateRaised
c.Summary = fmt.Sprintf("%d durable consumers differ from their composed configuration", len(st.Differ))
c.Headline, c.Needs = "A module's consumer is not as composed", "a look, and grants_consumer_match for each"
c.Explanation = occ.NeedsYou + "a look. The bus holds these consumers otherwise than the bus composer composes them, " +
"and the grants part never changes a held one by itself (its delivery position would be lost): " + why
c.Headline = "A module's consumer is not as composed"
needsYou(&c, "a look, and grants_consumer_match for each", "The bus holds these consumers otherwise than the bus composer "+
"composes them, and the grants part never changes a held one by itself (its delivery position would be lost): "+why)
c.Resolved = "every account consumer matches, by grants_consumer_match or otherwise"
}
if p.Say(c) == nil {
@@ -321,9 +321,9 @@ func (p *Part) sayBehind(st *ConsumersState) {
} else {
c.Severity, c.State = occ.SeverityWarning, occ.StateRaised
c.Summary = fmt.Sprintf("%d durable consumers are more than %d messages behind their stream's head", len(st.Behind), BehindBound)
c.Headline, c.Needs = "Messages pile up for a listener", "a look at the module that reads them"
c.Explanation = occ.NeedsYou + "a look. These consumers stayed far behind their stream's head at two passes " +
"running, so what is sent to their modules waits unread (how far each is, in the grants part's last pass): " + why
c.Headline = "Messages pile up for a listener"
needsYou(&c, "a look at the module that reads them", "These consumers stayed far behind their stream's head at two passes "+
"running, so what is sent to their modules waits unread (how far each is, in the grants part's last pass): "+why)
c.Resolved = "every account consumer is back within the bound"
}
if p.Say(c) == nil {
modules/mesh-grants/internal/grants/firstviews.go
+27 -7
View File
@@ -354,9 +354,9 @@ func unansweredCondition(fv FirstView, s views.Status, raised bool) oc.Condition
c.Summary = fmt.Sprintf("the grants part has no first view of %s's %s: %s has not answered since %s (%s)", fv.Owner, fv.State,
fv.Tool, s.Asked.Format(time.RFC3339), strings.TrimSpace(s.Error))
c.Headline = "The grants part waits for " + fv.Owner
c.Explanation = "The grants part reads " + fv.Owner + "'s " + fv.State + " only through " + fv.Tool + ", and keeps asking it; " +
"it never reads the state itself, and keeps what it composed meanwhile (novox/hq ADR 0615 §6). The answer clears this."
c.Needs = "a look at " + fv.Owner + "'s runtime of its own: is it running and connected?"
needsYou(&c, "a look at "+fv.Owner+"'s runtime of its own: is it running and connected?", "The grants part reads "+fv.Owner+
"'s "+fv.State+" only through "+fv.Tool+", and keeps asking it; it never reads the state itself, and keeps what it "+
"composed meanwhile (novox/hq ADR 0615 §6). The answer clears this.")
return c
}
@@ -376,10 +376,10 @@ func emptyCondition(fv FirstView, raised bool) oc.Condition {
c.State, c.Severity = occ.StateRaised, occ.SeverityWarning
c.Summary = fmt.Sprintf("%s answered %s's %s empty at revision 0, so the grants part keeps the user list in force", fv.Tool, fv.Owner, fv.State)
c.Headline = fv.Owner + "'s " + fv.State + " is empty"
c.Explanation = occ.NeedsYou + "a step. The owner holds nothing of " + fv.State + " (its state rebuilt by a recreate that could " +
"not keep the data, or a fresh mesh), so the grants part drops no login and keeps what it holds; a login revoked meanwhile " +
"stays until the owner's first change (novox/hq design 25 § 12.2)."
c.Needs = "let the owner write its state again from its sources (" + step + "), then grants_first_view_again"
needsYou(&c, "let the owner write its state again from its sources ("+step+"), then grants_first_view_again", "The owner holds "+
"nothing of "+fv.State+" (its state rebuilt by a recreate that could not keep the data, or a fresh mesh), so the grants "+
"part drops no login and keeps what it holds; a login revoked meanwhile stays until the owner's first change "+
"(novox/hq design 25 § 12.2).")
return c
}
@@ -479,6 +479,26 @@ func (p *Part) AskFirstViewAgain(caller, state, why string, record func(what, wh
"says": "the part's process asks it once more at this ask; grants_first_views shows its answer"}, nil
}
// TimedAsk is ask, saying in the log how long each page took and how many keys it brought, or after how long it failed:
// a first view that never answers (issue 1966: the wants page outgrew the call's bound) shows how far from its bound it is.
func TimedAsk(tool string, ask func(context.Context, string) (views.Answer, error), now func() time.Time,
logf func(string, ...any)) func(context.Context, string) (views.Answer, error) {
return func(ctx context.Context, after string) (views.Answer, error) {
start := now()
a, err := ask(ctx, after)
took := now().Sub(start).Round(time.Millisecond)
switch {
case logf == nil:
case err != nil:
logf("[mesh-grants] the first view %s, the page after %q, failed after %s: %v", tool, after, took, err)
default:
logf("[mesh-grants] the first view %s, the page after %q, took %s: %d keys at revision %d, next %q", tool, after, took,
len(a.Values), a.Revision, a.Next)
}
return a, err
}
}
// Unwrapped is a call whose answer is read as the tool's own text where the runtime wraps it as content (unwrap), as
// a first view's answer is decoded (views.AskVia).
func Unwrapped(call func(key string, body any) (json.RawMessage, error)) func(key string, body any) (json.RawMessage, error) {
modules/mesh-grants/internal/grants/grants.go
+3 -3
View File
@@ -339,9 +339,9 @@ func (p *Part) sayMissing(missing []string) {
} else {
c.Severity, c.State = occ.SeverityWarning, occ.StateRaised
c.Summary = fmt.Sprintf("%d placed module(s) have no grant, so no want can name them", len(missing))
c.Headline, c.Needs = "A placed module has no bus grant", "a look at why each has none"
c.Explanation = occ.NeedsYou + "a look. The grants part wrote no grant for these modules, so the assigner's " +
"credential rule gives them no want: " + why
c.Headline = "A placed module has no bus grant"
needsYou(&c, "a look at why each has none", "The grants part wrote no grant for these modules, so the assigner's "+
"credential rule gives them no want: "+why)
c.Resolved = "every placed module has a grant at the fingerprint the indexer names"
}
if p.Say(c) == nil {
modules/mesh-grants/internal/grants/held.go
+4 -3
View File
@@ -143,9 +143,10 @@ func (p *Part) sayHeldModules(held map[string]string, kept map[string]User) {
} else {
c.Severity, c.State = occ.SeverityWarning, occ.StateRaised
c.Summary = fmt.Sprintf("%d bus user(s) of a module whose manifest is held are kept in the list: %s", len(users), why)
c.Headline, c.Needs = "A module's manifest is held, its bus users are kept", "a manifest the indexer accepts, or a judge that knows its fields"
c.Explanation = occ.NeedsYou + "a manifest the indexer accepts (a refused one mended, or a waiting one judged by an indexer that knows its fields). " +
"The rest of the user list is written as ever; a held module's users stay as the list in force holds them: " + why
c.Headline = "A module's manifest is held, its bus users are kept"
needsYou(&c, "a manifest the indexer accepts, or a judge that knows its fields", "A refused one mended, or a waiting one "+
"judged by an indexer that knows its fields. The rest of the user list is written as ever; a held module's users stay "+
"as the list in force holds them: "+why)
c.Resolved = "every module's manifest is accepted"
}
if p.Say(c) == nil {
modules/mesh-grants/internal/grants/inputs.go
+18 -2
View File
@@ -180,6 +180,11 @@ type Inputs struct {
// answering (bootstrap only, issue 1966); empty when the assignments placed them. Such a pass retires nothing
// (ReadFailures) and seals no list dropping a user of the list in force (holdsBack).
ByHaves string
// WantsUnread says why this pass composed what runs from each have's fingerprint, the wants view not answering while
// the assignments view did (bootstrap only, issue 1966: a wants page the running assigner cannot answer in a call's
// bound); empty when the wants were read. As ByHaves, such a pass retires nothing (ReadFailures) and only adds
// (keepsLoaded): the list is held only where it would take a permission from a loaded user (holdsBack).
WantsUnread string
// Held is, by module, the manifest a held module's running build was judged on (its have's fingerprint, the builder's
// build of it): what the seats it defines, the streams others hear from it and its reads are composed from while its
// trunk manifest is refused or waits, so no other principal loses a grant to it (ADR 0601 clause 3).
@@ -519,12 +524,20 @@ func ReadInputs(r Reader, cache *Manifests) Inputs {
// What each machine is wanted to run: the want's fingerprint, the builder's build of it, the manifest it built.
var fromTrunk []string
wants := map[string][]byte{}
if in.ByHaves == "" {
_, wantsErr := r.Keys(ReadWants)
switch {
case in.ByHaves == "" && wantsErr == nil:
wants = each(r, ReadWants, &in.Gaps)
// What each machine runs now is its have's build, which the want may already have left (issue 1966): the
// memberships keep what either build reads, calls and consumes.
runningByHaves(r, &in, cache, in.Had)
} else {
case in.ByHaves == "":
// **Bootstrap only** (issue 1966), as the assignments' placement by the haves: while the wants view does not answer,
// what runs is each placed module's have's fingerprint, said as a gap that is no failed read, and the pass only adds.
in.WantsUnread = "the wants view did not answer (" + wantsErr.Error() + "), so what runs is composed from each have's fingerprint"
in.Gaps = append(in.Gaps, Gap{Input: ReadWants, Why: "refused or absent: what runs is composed from each have's fingerprint instead, until it answers (" + wantsErr.Error() + ")"})
fromTrunk = runningByHaves(r, &in, cache, in.Running)
default:
// Placed by the haves, what runs is what each have's fingerprint names, not a want: the assigner's wants are its
// view too, and not read in such a pass.
in.Gaps = append(in.Gaps, Gap{Input: ReadWants, Why: "not read: the modules are placed by the node-engines' haves this pass, so what runs is composed from each have's fingerprint"})
@@ -716,6 +729,9 @@ func (in Inputs) ReadFailures() []string {
if in.ByHaves != "" {
out = append(out, ReadAssignments+": "+in.ByHaves)
}
if in.WantsUnread != "" {
out = append(out, ReadWants+": "+in.WantsUnread)
}
for _, g := range in.Refused {
out = append(out, g.Input+": "+g.Why)
}
modules/mesh-grants/internal/grants/keeploaded.go
+2
View File
@@ -21,6 +21,8 @@ func (in Inputs) keepsLoaded() string {
switch {
case in.ByHaves != "":
return in.ByHaves
case in.WantsUnread != "":
return in.WantsUnread
case in.SeatListUnread:
return "the seat list is unread, so only the bootstrapped seats compose"
}
modules/mesh-grants/internal/grants/needsyou.go
+20
View File
@@ -0,0 +1,20 @@
package grants
import (
"strings"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
)
// needsYou sets what a raised condition needs of the operator and its explanation: the channel takes a condition whose
// explanation opens with "Needs you: " and those very words (operatorchannelclient.EncodeCondition, novox/hq ADR 0253 §1),
// and refuses any other, so the operator would never see it. rest is the explanation after them.
func needsYou(c *oc.Condition, needs, rest string) {
c.Needs = needs
sep := ". "
if strings.HasSuffix(needs, "?") || strings.HasSuffix(needs, ".") {
sep = " "
}
c.Explanation = occ.NeedsYou + needs + sep + rest
}
modules/mesh-grants/internal/grants/refused.go
+4 -3
View File
@@ -127,9 +127,10 @@ func (p *Part) sayPrincipalsRefused(refused map[string]string, kept map[string]U
} else {
c.Severity, c.State = occ.SeverityWarning, occ.StateRaised
c.Summary = fmt.Sprintf("%d principal(s) refused their composed grants: %s", len(refused), why)
c.Headline, c.Needs = "A principal's bus grants are refused", "a fix of the manifest or the writers table"
c.Explanation = occ.NeedsYou + "a fix of what makes it a second writer. The rest of the user list is written as ever; " +
"each refused principal keeps the grants the list in force gives it, or stays out where it holds none: " + why
c.Headline = "A principal's bus grants are refused"
needsYou(&c, "a fix of the manifest or the writers table, of what makes it a second writer",
"The rest of the user list is written as ever; each refused principal keeps the grants the list in force gives it, "+
"or stays out where it holds none: "+why)
c.Resolved = "every principal's grants compose again"
}
if p.Say(c) == nil {
modules/mesh-grants/internal/grants/sealed.go
+2 -2
View File
@@ -180,8 +180,8 @@ func (p *Part) sayHeld(why string) {
} else {
reason := strings.TrimPrefix(why, "held: ")
c.Severity, c.State = occ.SeverityWarning, occ.StateRaised
c.Summary, c.Headline, c.Needs = "the user list is held: "+reason, "The user list in force is held back", "a look at why"
c.Explanation = occ.NeedsYou + "a look at why. The bus node keeps following the list written before, not the one composed now: " + reason
c.Summary, c.Headline = "the user list is held: "+reason, "The user list in force is held back"
needsYou(&c, "a look at why", "The bus node keeps following the list written before, not the one composed now: "+reason)
c.Resolved = "a pass finds nothing to hold back and the list advances"
}
if p.Say(c) == nil {