Grants composes from the haves while the wants view is unanswered, and its conditions reach the operator (issue 1966, recovery) #180
@@ -28,6 +28,7 @@ import (
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
@@ -117,7 +118,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// followed on the owners' views families, which the hearing below hands them.
|
||||
sayCond := sayVia(ctx, js, account.Module)
|
||||
fv := grants.NewViews(func(tool string) func(context.Context, string) (views.Answer, error) {
|
||||
return views.AskVia(grants.Unwrapped(conn.Call), tool)
|
||||
return grants.TimedAsk(tool, views.AskVia(grants.Unwrapped(conn.Call), tool), time.Now, logger.Printf)
|
||||
}, sayCond, f.Wake, logger.Printf)
|
||||
fv.Rest, fv.Watching = bus, bus
|
||||
byState := map[string]*views.Follower{}
|
||||
|
||||
@@ -801,6 +801,7 @@ func TestGrants(t *testing.T) {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
p.Say = channelTakes(t, p.Say)
|
||||
for _, s := range b.streams {
|
||||
if s["name"] == "OBJ_mesh-knowledge_sections" {
|
||||
s["bytes"] = int64(3500 << 20)
|
||||
@@ -1664,6 +1665,7 @@ func TestGrants(t *testing.T) {
|
||||
|
||||
var said []oc.Condition
|
||||
p.Say = func(c oc.Condition) error { said = append(said, c); return nil }
|
||||
p.Say = channelTakes(t, p.Say)
|
||||
// settled is the assignments of the lists agreeing, every hash held, and the loaded list as edit leaves it.
|
||||
settled := func(edit func(modX *grants.User)) func() {
|
||||
return func() {
|
||||
@@ -1768,6 +1770,7 @@ func TestGrants(t *testing.T) {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
fp.Say = channelTakes(t, fp.Say)
|
||||
fp.Compose()
|
||||
if len(fresh) == 0 || fresh[0].Key != grants.HeldCondition || fresh[0].State != occ.StateCleared {
|
||||
t.Errorf("a fresh part's first pass does not say the held warning is cleared: %+v", fresh)
|
||||
@@ -2442,6 +2445,7 @@ func TestGrants(t *testing.T) {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
p.Say = channelTakes(t, p.Say)
|
||||
// With no lease key (1126/A1).
|
||||
// The private network the mesh provides is placed with no fingerprint by construction: never a missing grant.
|
||||
b.assign("b", "mesh-wireguard")
|
||||
@@ -2636,6 +2640,14 @@ func TestGrants(t *testing.T) {
|
||||
b.put(grants.ReadHave, "b.mod-y", runs)
|
||||
b.put(grants.ReadHave, "a.mesh-delivery-indexer", runs)
|
||||
}, ""},
|
||||
// The wants view unanswered, the assignments answered (issue 1966, the list held from 2026-11-08 16:39): what runs
|
||||
// comes from the haves, the wants are no failed read, and the pass only adds.
|
||||
{"the wants view unanswered", func(b *fakeBus, p *grants.Part) {
|
||||
b.refused[grants.ReadWants] = true
|
||||
if in := grants.ReadInputs(b, &grants.Manifests{}); in.WantsUnread == "" || len(in.ReadFailed()) != 0 {
|
||||
t.Errorf("an unanswered wants view is a failed read, or composes no fallback: %q, %v", in.WantsUnread, in.ReadFailed())
|
||||
}
|
||||
}, ""},
|
||||
// The ledger of a held module stays with its module's users, whatever the seat list (held.go).
|
||||
{"the ledger of a module whose manifest is refused", func(b *fakeBus, p *grants.Part) {
|
||||
b.refuse("mod-x", ledgerX, "it is held")
|
||||
@@ -3074,6 +3086,7 @@ func TestGrants(t *testing.T) {
|
||||
sayMu.Unlock()
|
||||
return nil
|
||||
}
|
||||
say = channelTakes(t, say)
|
||||
fv := grants.NewViews(owner, say, func() {}, nil)
|
||||
fv.Rest = b
|
||||
var minutes atomic.Int64
|
||||
@@ -3467,6 +3480,7 @@ func TestGrants(t *testing.T) {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
p.Say = channelTakes(t, p.Say)
|
||||
made["max_deliver"] = 9
|
||||
res = p.Compose()
|
||||
if len(res.Consumers.Differ) != 1 || !strings.Contains(res.Consumers.Differ[0], "EVENTS.b_mod-c: max deliver 9") ||
|
||||
@@ -3538,6 +3552,7 @@ func TestGrants(t *testing.T) {
|
||||
p.Objects = &fakeObjects{}
|
||||
var said []oc.Condition
|
||||
p.Say = func(c oc.Condition) error { said = append(said, c); return nil }
|
||||
p.Say = channelTakes(t, p.Say)
|
||||
p.Compose()
|
||||
inForce, _, _ := p.Sealed()
|
||||
if inForce.Digest == "" {
|
||||
@@ -3708,6 +3723,7 @@ func TestGrants(t *testing.T) {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
p.Say = channelTakes(t, p.Say)
|
||||
p.Compose()
|
||||
inForce, _, _ := p.Sealed()
|
||||
if inForce.Digest == "" {
|
||||
@@ -3793,6 +3809,7 @@ func TestGrants(t *testing.T) {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
p.Say = channelTakes(t, p.Say)
|
||||
p.Compose()
|
||||
inForce, _, _ := p.Sealed()
|
||||
before, _, ok, _ := b.ownGet(grants.StateUserList, grants.SealedUserKey(inForce.Digest, "b.mod-r.mod-r-tools"))
|
||||
@@ -3888,6 +3905,28 @@ func TestGrants(t *testing.T) {
|
||||
t.Errorf("the kept tool runner was not said: %+v", last)
|
||||
}
|
||||
})
|
||||
|
||||
// Each first view's page says in the log how long it took (issue 1966: the wants page outgrew the call's bound unseen).
|
||||
t.Run("a first view's page says how long it took", func(t *testing.T) {
|
||||
at := time.Date(2026, 11, 8, 22, 0, 0, 0, time.UTC)
|
||||
now := func() time.Time { at = at.Add(1500 * time.Millisecond); return at }
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
err error
|
||||
want string
|
||||
}{
|
||||
{"answered", nil, `the first view mesh-delivery-assigner.wants, the page after "", took 1.5s: 2 keys at revision 9, next ""`},
|
||||
{"failed", errors.New("context deadline exceeded"), `the first view mesh-delivery-assigner.wants, the page after "", failed after 1.5s: context deadline exceeded`},
|
||||
} {
|
||||
var lines []string
|
||||
ask := grants.TimedAsk(grants.ReadWants, func(context.Context, string) (views.Answer, error) {
|
||||
return views.Answer{Revision: 9, Values: map[string]json.RawMessage{"a": nil, "b": nil}}, row.err
|
||||
}, now, func(f string, a ...any) { lines = append(lines, fmt.Sprintf(f, a...)) })
|
||||
if _, err := ask(context.Background(), ""); !errors.Is(err, row.err) || len(lines) != 1 || !strings.Contains(lines[0], row.want) {
|
||||
t.Errorf("%s: said %q (%v), want %q", row.name, lines, err, row.want)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// fakeObjects is the part's object bucket of sealed lists.
|
||||
@@ -4084,3 +4123,15 @@ func TestPassComposesMembershipFromTheHaveBuild(t *testing.T) {
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// channelTakes is say, after the operator-channel's own check of each condition: one the channel would refuse (its
|
||||
// explanation not opening with "Needs you: " and what it needs) fails the test, since the operator would never see it.
|
||||
func channelTakes(t *testing.T, say func(oc.Condition) error) func(oc.Condition) error {
|
||||
t.Helper()
|
||||
return func(c oc.Condition) error {
|
||||
if _, err := occ.EncodeCondition(c); err != nil {
|
||||
t.Errorf("the channel refuses the condition %s: %v", c.Key, err)
|
||||
}
|
||||
return say(c)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -224,9 +224,9 @@ func (p *Part) sayDiffer(st *ConsumersState) {
|
||||
} else {
|
||||
c.Severity, c.State = occ.SeverityWarning, occ.StateRaised
|
||||
c.Summary = fmt.Sprintf("%d durable consumers differ from their composed configuration", len(st.Differ))
|
||||
c.Headline, c.Needs = "A module's consumer is not as composed", "a look, and grants_consumer_match for each"
|
||||
c.Explanation = occ.NeedsYou + "a look. The bus holds these consumers otherwise than the bus composer composes them, " +
|
||||
"and the grants part never changes a held one by itself (its delivery position would be lost): " + why
|
||||
c.Headline = "A module's consumer is not as composed"
|
||||
needsYou(&c, "a look, and grants_consumer_match for each", "The bus holds these consumers otherwise than the bus composer "+
|
||||
"composes them, and the grants part never changes a held one by itself (its delivery position would be lost): "+why)
|
||||
c.Resolved = "every account consumer matches, by grants_consumer_match or otherwise"
|
||||
}
|
||||
if p.Say(c) == nil {
|
||||
@@ -321,9 +321,9 @@ func (p *Part) sayBehind(st *ConsumersState) {
|
||||
} else {
|
||||
c.Severity, c.State = occ.SeverityWarning, occ.StateRaised
|
||||
c.Summary = fmt.Sprintf("%d durable consumers are more than %d messages behind their stream's head", len(st.Behind), BehindBound)
|
||||
c.Headline, c.Needs = "Messages pile up for a listener", "a look at the module that reads them"
|
||||
c.Explanation = occ.NeedsYou + "a look. These consumers stayed far behind their stream's head at two passes " +
|
||||
"running, so what is sent to their modules waits unread (how far each is, in the grants part's last pass): " + why
|
||||
c.Headline = "Messages pile up for a listener"
|
||||
needsYou(&c, "a look at the module that reads them", "These consumers stayed far behind their stream's head at two passes "+
|
||||
"running, so what is sent to their modules waits unread (how far each is, in the grants part's last pass): "+why)
|
||||
c.Resolved = "every account consumer is back within the bound"
|
||||
}
|
||||
if p.Say(c) == nil {
|
||||
|
||||
@@ -354,9 +354,9 @@ func unansweredCondition(fv FirstView, s views.Status, raised bool) oc.Condition
|
||||
c.Summary = fmt.Sprintf("the grants part has no first view of %s's %s: %s has not answered since %s (%s)", fv.Owner, fv.State,
|
||||
fv.Tool, s.Asked.Format(time.RFC3339), strings.TrimSpace(s.Error))
|
||||
c.Headline = "The grants part waits for " + fv.Owner
|
||||
c.Explanation = "The grants part reads " + fv.Owner + "'s " + fv.State + " only through " + fv.Tool + ", and keeps asking it; " +
|
||||
"it never reads the state itself, and keeps what it composed meanwhile (novox/hq ADR 0615 §6). The answer clears this."
|
||||
c.Needs = "a look at " + fv.Owner + "'s runtime of its own: is it running and connected?"
|
||||
needsYou(&c, "a look at "+fv.Owner+"'s runtime of its own: is it running and connected?", "The grants part reads "+fv.Owner+
|
||||
"'s "+fv.State+" only through "+fv.Tool+", and keeps asking it; it never reads the state itself, and keeps what it "+
|
||||
"composed meanwhile (novox/hq ADR 0615 §6). The answer clears this.")
|
||||
return c
|
||||
}
|
||||
|
||||
@@ -376,10 +376,10 @@ func emptyCondition(fv FirstView, raised bool) oc.Condition {
|
||||
c.State, c.Severity = occ.StateRaised, occ.SeverityWarning
|
||||
c.Summary = fmt.Sprintf("%s answered %s's %s empty at revision 0, so the grants part keeps the user list in force", fv.Tool, fv.Owner, fv.State)
|
||||
c.Headline = fv.Owner + "'s " + fv.State + " is empty"
|
||||
c.Explanation = occ.NeedsYou + "a step. The owner holds nothing of " + fv.State + " (its state rebuilt by a recreate that could " +
|
||||
"not keep the data, or a fresh mesh), so the grants part drops no login and keeps what it holds; a login revoked meanwhile " +
|
||||
"stays until the owner's first change (novox/hq design 25 § 12.2)."
|
||||
c.Needs = "let the owner write its state again from its sources (" + step + "), then grants_first_view_again"
|
||||
needsYou(&c, "let the owner write its state again from its sources ("+step+"), then grants_first_view_again", "The owner holds "+
|
||||
"nothing of "+fv.State+" (its state rebuilt by a recreate that could not keep the data, or a fresh mesh), so the grants "+
|
||||
"part drops no login and keeps what it holds; a login revoked meanwhile stays until the owner's first change "+
|
||||
"(novox/hq design 25 § 12.2).")
|
||||
return c
|
||||
}
|
||||
|
||||
@@ -479,6 +479,26 @@ func (p *Part) AskFirstViewAgain(caller, state, why string, record func(what, wh
|
||||
"says": "the part's process asks it once more at this ask; grants_first_views shows its answer"}, nil
|
||||
}
|
||||
|
||||
// TimedAsk is ask, saying in the log how long each page took and how many keys it brought, or after how long it failed:
|
||||
// a first view that never answers (issue 1966: the wants page outgrew the call's bound) shows how far from its bound it is.
|
||||
func TimedAsk(tool string, ask func(context.Context, string) (views.Answer, error), now func() time.Time,
|
||||
logf func(string, ...any)) func(context.Context, string) (views.Answer, error) {
|
||||
return func(ctx context.Context, after string) (views.Answer, error) {
|
||||
start := now()
|
||||
a, err := ask(ctx, after)
|
||||
took := now().Sub(start).Round(time.Millisecond)
|
||||
switch {
|
||||
case logf == nil:
|
||||
case err != nil:
|
||||
logf("[mesh-grants] the first view %s, the page after %q, failed after %s: %v", tool, after, took, err)
|
||||
default:
|
||||
logf("[mesh-grants] the first view %s, the page after %q, took %s: %d keys at revision %d, next %q", tool, after, took,
|
||||
len(a.Values), a.Revision, a.Next)
|
||||
}
|
||||
return a, err
|
||||
}
|
||||
}
|
||||
|
||||
// Unwrapped is a call whose answer is read as the tool's own text where the runtime wraps it as content (unwrap), as
|
||||
// a first view's answer is decoded (views.AskVia).
|
||||
func Unwrapped(call func(key string, body any) (json.RawMessage, error)) func(key string, body any) (json.RawMessage, error) {
|
||||
|
||||
@@ -339,9 +339,9 @@ func (p *Part) sayMissing(missing []string) {
|
||||
} else {
|
||||
c.Severity, c.State = occ.SeverityWarning, occ.StateRaised
|
||||
c.Summary = fmt.Sprintf("%d placed module(s) have no grant, so no want can name them", len(missing))
|
||||
c.Headline, c.Needs = "A placed module has no bus grant", "a look at why each has none"
|
||||
c.Explanation = occ.NeedsYou + "a look. The grants part wrote no grant for these modules, so the assigner's " +
|
||||
"credential rule gives them no want: " + why
|
||||
c.Headline = "A placed module has no bus grant"
|
||||
needsYou(&c, "a look at why each has none", "The grants part wrote no grant for these modules, so the assigner's "+
|
||||
"credential rule gives them no want: "+why)
|
||||
c.Resolved = "every placed module has a grant at the fingerprint the indexer names"
|
||||
}
|
||||
if p.Say(c) == nil {
|
||||
|
||||
@@ -143,9 +143,10 @@ func (p *Part) sayHeldModules(held map[string]string, kept map[string]User) {
|
||||
} else {
|
||||
c.Severity, c.State = occ.SeverityWarning, occ.StateRaised
|
||||
c.Summary = fmt.Sprintf("%d bus user(s) of a module whose manifest is held are kept in the list: %s", len(users), why)
|
||||
c.Headline, c.Needs = "A module's manifest is held, its bus users are kept", "a manifest the indexer accepts, or a judge that knows its fields"
|
||||
c.Explanation = occ.NeedsYou + "a manifest the indexer accepts (a refused one mended, or a waiting one judged by an indexer that knows its fields). " +
|
||||
"The rest of the user list is written as ever; a held module's users stay as the list in force holds them: " + why
|
||||
c.Headline = "A module's manifest is held, its bus users are kept"
|
||||
needsYou(&c, "a manifest the indexer accepts, or a judge that knows its fields", "A refused one mended, or a waiting one "+
|
||||
"judged by an indexer that knows its fields. The rest of the user list is written as ever; a held module's users stay "+
|
||||
"as the list in force holds them: "+why)
|
||||
c.Resolved = "every module's manifest is accepted"
|
||||
}
|
||||
if p.Say(c) == nil {
|
||||
|
||||
@@ -180,6 +180,11 @@ type Inputs struct {
|
||||
// answering (bootstrap only, issue 1966); empty when the assignments placed them. Such a pass retires nothing
|
||||
// (ReadFailures) and seals no list dropping a user of the list in force (holdsBack).
|
||||
ByHaves string
|
||||
// WantsUnread says why this pass composed what runs from each have's fingerprint, the wants view not answering while
|
||||
// the assignments view did (bootstrap only, issue 1966: a wants page the running assigner cannot answer in a call's
|
||||
// bound); empty when the wants were read. As ByHaves, such a pass retires nothing (ReadFailures) and only adds
|
||||
// (keepsLoaded): the list is held only where it would take a permission from a loaded user (holdsBack).
|
||||
WantsUnread string
|
||||
// Held is, by module, the manifest a held module's running build was judged on (its have's fingerprint, the builder's
|
||||
// build of it): what the seats it defines, the streams others hear from it and its reads are composed from while its
|
||||
// trunk manifest is refused or waits, so no other principal loses a grant to it (ADR 0601 clause 3).
|
||||
@@ -519,12 +524,20 @@ func ReadInputs(r Reader, cache *Manifests) Inputs {
|
||||
// What each machine is wanted to run: the want's fingerprint, the builder's build of it, the manifest it built.
|
||||
var fromTrunk []string
|
||||
wants := map[string][]byte{}
|
||||
if in.ByHaves == "" {
|
||||
_, wantsErr := r.Keys(ReadWants)
|
||||
switch {
|
||||
case in.ByHaves == "" && wantsErr == nil:
|
||||
wants = each(r, ReadWants, &in.Gaps)
|
||||
// What each machine runs now is its have's build, which the want may already have left (issue 1966): the
|
||||
// memberships keep what either build reads, calls and consumes.
|
||||
runningByHaves(r, &in, cache, in.Had)
|
||||
} else {
|
||||
case in.ByHaves == "":
|
||||
// **Bootstrap only** (issue 1966), as the assignments' placement by the haves: while the wants view does not answer,
|
||||
// what runs is each placed module's have's fingerprint, said as a gap that is no failed read, and the pass only adds.
|
||||
in.WantsUnread = "the wants view did not answer (" + wantsErr.Error() + "), so what runs is composed from each have's fingerprint"
|
||||
in.Gaps = append(in.Gaps, Gap{Input: ReadWants, Why: "refused or absent: what runs is composed from each have's fingerprint instead, until it answers (" + wantsErr.Error() + ")"})
|
||||
fromTrunk = runningByHaves(r, &in, cache, in.Running)
|
||||
default:
|
||||
// Placed by the haves, what runs is what each have's fingerprint names, not a want: the assigner's wants are its
|
||||
// view too, and not read in such a pass.
|
||||
in.Gaps = append(in.Gaps, Gap{Input: ReadWants, Why: "not read: the modules are placed by the node-engines' haves this pass, so what runs is composed from each have's fingerprint"})
|
||||
@@ -716,6 +729,9 @@ func (in Inputs) ReadFailures() []string {
|
||||
if in.ByHaves != "" {
|
||||
out = append(out, ReadAssignments+": "+in.ByHaves)
|
||||
}
|
||||
if in.WantsUnread != "" {
|
||||
out = append(out, ReadWants+": "+in.WantsUnread)
|
||||
}
|
||||
for _, g := range in.Refused {
|
||||
out = append(out, g.Input+": "+g.Why)
|
||||
}
|
||||
|
||||
@@ -21,6 +21,8 @@ func (in Inputs) keepsLoaded() string {
|
||||
switch {
|
||||
case in.ByHaves != "":
|
||||
return in.ByHaves
|
||||
case in.WantsUnread != "":
|
||||
return in.WantsUnread
|
||||
case in.SeatListUnread:
|
||||
return "the seat list is unread, so only the bootstrapped seats compose"
|
||||
}
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
package grants
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
|
||||
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
|
||||
)
|
||||
|
||||
// needsYou sets what a raised condition needs of the operator and its explanation: the channel takes a condition whose
|
||||
// explanation opens with "Needs you: " and those very words (operatorchannelclient.EncodeCondition, novox/hq ADR 0253 §1),
|
||||
// and refuses any other, so the operator would never see it. rest is the explanation after them.
|
||||
func needsYou(c *oc.Condition, needs, rest string) {
|
||||
c.Needs = needs
|
||||
sep := ". "
|
||||
if strings.HasSuffix(needs, "?") || strings.HasSuffix(needs, ".") {
|
||||
sep = " "
|
||||
}
|
||||
c.Explanation = occ.NeedsYou + needs + sep + rest
|
||||
}
|
||||
@@ -127,9 +127,10 @@ func (p *Part) sayPrincipalsRefused(refused map[string]string, kept map[string]U
|
||||
} else {
|
||||
c.Severity, c.State = occ.SeverityWarning, occ.StateRaised
|
||||
c.Summary = fmt.Sprintf("%d principal(s) refused their composed grants: %s", len(refused), why)
|
||||
c.Headline, c.Needs = "A principal's bus grants are refused", "a fix of the manifest or the writers table"
|
||||
c.Explanation = occ.NeedsYou + "a fix of what makes it a second writer. The rest of the user list is written as ever; " +
|
||||
"each refused principal keeps the grants the list in force gives it, or stays out where it holds none: " + why
|
||||
c.Headline = "A principal's bus grants are refused"
|
||||
needsYou(&c, "a fix of the manifest or the writers table, of what makes it a second writer",
|
||||
"The rest of the user list is written as ever; each refused principal keeps the grants the list in force gives it, "+
|
||||
"or stays out where it holds none: "+why)
|
||||
c.Resolved = "every principal's grants compose again"
|
||||
}
|
||||
if p.Say(c) == nil {
|
||||
|
||||
@@ -180,8 +180,8 @@ func (p *Part) sayHeld(why string) {
|
||||
} else {
|
||||
reason := strings.TrimPrefix(why, "held: ")
|
||||
c.Severity, c.State = occ.SeverityWarning, occ.StateRaised
|
||||
c.Summary, c.Headline, c.Needs = "the user list is held: "+reason, "The user list in force is held back", "a look at why"
|
||||
c.Explanation = occ.NeedsYou + "a look at why. The bus node keeps following the list written before, not the one composed now: " + reason
|
||||
c.Summary, c.Headline = "the user list is held: "+reason, "The user list in force is held back"
|
||||
needsYou(&c, "a look at why", "The bus node keeps following the list written before, not the one composed now: "+reason)
|
||||
c.Resolved = "a pass finds nothing to hold back and the list advances"
|
||||
}
|
||||
if p.Say(c) == nil {
|
||||
|
||||
Reference in New Issue
Block a user