From 9f65ca98d334096ec93a28bac96407cfbd78aafe Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 23:00:41 +0100 Subject: [PATCH] =?UTF-8?q?GR9/GR10=20(A-5,=20A-52,=20=C2=A7684,=20=C2=A74?= =?UTF-8?q?44,=20=C2=A7614):=20the=20grants=20part=20follows=20the=20catal?= =?UTF-8?q?og=20half=20through=20the=20branch=20follower=20on=20a=20views?= =?UTF-8?q?=20reader,=20and=20composes=20a=20want's=20account=20from=20its?= =?UTF-8?q?=20entry?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The part adopts meshcatalogclient's branch follower on a views.ProcessReader holding its held finding writer (so its view-waits is the reader's); the manifest of the build a want or a have names is the catalog graph's entry of that fingerprint, not a builder's build (the `builds` read is gone); until every first view of the catalog half is in, the pass keeps the list in force and only adds, says so in its log and in grants_first_views. Builds: design 47 G §684, §444, §614, §191, A-5, A-52 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- modules/mesh-grants/cmd/mesh-grants/main.go | 19 +- modules/mesh-grants/grants_test.go | 118 ++++++++-- .../mesh-grants/internal/grants/catalog.go | 202 ++++++++++++++++++ .../mesh-grants/internal/grants/firstviews.go | 25 +++ modules/mesh-grants/internal/grants/follow.go | 16 +- modules/mesh-grants/internal/grants/inputs.go | 74 ++----- .../mesh-grants/internal/grants/keeploaded.go | 2 + modules/mesh-grants/internal/grants/part.go | 5 +- .../mesh-grants/internal/grants/rootfree.go | 2 +- modules/mesh-grants/module.json | 8 +- 10 files changed, 393 insertions(+), 78 deletions(-) create mode 100644 modules/mesh-grants/internal/grants/catalog.go diff --git a/modules/mesh-grants/cmd/mesh-grants/main.go b/modules/mesh-grants/cmd/mesh-grants/main.go index 27f4f768..2a57f33a 100644 --- a/modules/mesh-grants/cmd/mesh-grants/main.go +++ b/modules/mesh-grants/cmd/mesh-grants/main.go @@ -37,6 +37,7 @@ import ( "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/announce" @@ -152,6 +153,20 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro byState[v.State] = fv.Followers[v.Read] } hearViews := views.Hear(ctx, byState, logger.Printf) + // The catalog half (design 47 G §614, §684, A-5, A-52): the branch follower of meshcatalogclient, adopted on the part's views + // reader, which holds the part's finding writer, so the part's view-waits is the reader's. The reader hears the events of the + // views it follows; every other owner's views event is the five first views' above (Other). The reader's own first-views state + // is none: grants_first_views says the catalog half's views beside the others. + reader := &views.ProcessReader{Module: account.Module, Call: conn.Call, Logf: logger.Printf, Hub: hub, Other: hearViews, + Findings: func() views.WireWriter { return fw }} + branches, err := meshcatalogclient.FollowBranchesProcess(reader, conn.Call) + if err != nil { + return err + } + branches.Logf = logger.Printf + branches.Changed = func(string, string, json.RawMessage, bool, time.Time) { f.Wake() } + branches.Entries.Changed = func(meshcatalogclient.EntryRow) { f.Wake() } + fv.Catalog = &grants.CatalogOf{Branches: branches, Context: ctx, Wake: f.Wake} f.Part = &grants.Part{Read: fv, Call: conn, Store: ownStore{conn}, Emit: emitting{conn}, Views: fv, FirstViewContext: ctx, Create: creating{ctx: ctx, js: js}, Consumers: creating{ctx: ctx, js: js}, Objects: sealedObjects{ctx: ctx, js: js}, Purging: ownPurging{ctx: ctx, js: js, module: account.Module}, Logf: logger.Printf, @@ -173,6 +188,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro f.Part.SeatList = gate f.Part.Say, f.Part.SayFinding = sayCond, sayFinding fv.Start(ctx, f.Wake) + reader.Start(ctx, js, account, nil) f.Asked = func(wake func()) error { return conn.State(grants.StateUserList).Watch(grants.KeyAsked, func(c stdio.StateChange) error { if !c.Current { @@ -200,9 +216,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro f.ConsumerMade(ctx, created) } }) - if err := hub.Handle(views.EventsPattern, hearViews); err != nil { - return err - } // The seat mesh-catalog's verdict events go to the verdicts' first view (ADR 0622 §2), which takes each at its revision. for _, event := range []string{meshcatalog.EventManifestAccepted, meshcatalog.EventManifestRefused} { if err := hub.Handle("mesh.seat."+meshcatalog.Seat+".event."+event, func(m jetstream.Msg) { diff --git a/modules/mesh-grants/grants_test.go b/modules/mesh-grants/grants_test.go index bfe7edae..2d251a68 100644 --- a/modules/mesh-grants/grants_test.go +++ b/modules/mesh-grants/grants_test.go @@ -36,7 +36,6 @@ import ( "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshwantsclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/keys" @@ -90,6 +89,54 @@ type fakeBus struct { consumerCreates []string // maxValue, when set, refuses an own value larger than it as the bus does (JetStream 10054). maxValue int + // entries are the catalog half's entries by fingerprint, the manifest each holds as the indexer's manifests state has it + // (entry); entriesAsked says every one is asked, not answered: the half cannot be read yet. + entries map[string]string + entriesAsked bool + catalogWaited []string +} + +// entry is the catalog half's entry of fp, holding the manifest of the digest the indexer's manifests state names. +func (b *fakeBus) entry(fp, digest string) { + b.mu.Lock() + defer b.mu.Unlock() + if b.entries == nil { + b.entries = map[string]string{} + } + raw, ok := b.states[grants.ReadManifests][digest] + if !ok { + panic("no manifest " + digest) + } + v, err := meshcatalogclient.DecodeManifest(raw) + if err != nil { + panic(err) + } + b.entries[fp] = v.Manifest +} + +// Resolve is the catalog half's entry of a fingerprint (grants.CatalogHalf). +func (b *fakeBus) Resolve(fp string) (meshcatalog.CatalogEntry, meshcatalogclient.EntryState) { + b.mu.Lock() + defer b.mu.Unlock() + raw, ok := b.entries[fp] + switch { + case b.entriesAsked: + return meshcatalog.CatalogEntry{}, meshcatalogclient.EntryAsked + case !ok: + return meshcatalog.CatalogEntry{}, meshcatalogclient.EntryNotFound + } + var m map[string]any + if err := json.Unmarshal([]byte(raw), &m); err != nil { + panic(err) + } + return meshcatalog.CatalogEntry{Fp: fp, Manifest: m}, meshcatalogclient.EntryAnswered +} + +// Waiting is the first views of the catalog half not in. +func (b *fakeBus) Waiting() []string { + b.mu.Lock() + defer b.mu.Unlock() + return b.catalogWaited } // MatchConsumer is the bus's update of a held durable consumer to the configuration given. @@ -672,8 +719,7 @@ func TestGrants(t *testing.T) { b.put(grants.ReadManifests, digest, meshcatalog.ManifestsByDigestValue{V: keys.Version, Module: "mod-x", Manifest: built}) fp := "src2:" + strings.Repeat("cd", 32) b.put(grants.ReadWants, "want.a.mod-x", meshwantsclient.Want{V: keys.Version, Fp: fp, Composed: "sha256:" + strings.Repeat("ef", 32)}) - key, _ := builds.BuilderBuildKey("mod-x", fp) - b.put(grants.ReadBuilds, key, map[string]any{"v": keys.Version, "state": "done", "manifest": digest}) + b.entry(fp, digest) }, []string{"a.mod-x", "a.nats", "b.mod-y", "node.a", "node.b"}, "people"}, // Issue 1966: refused, each module is placed where its have says it runs (the bootstrap fallback); a has nats. {"assignments refused is a gap; the machines and the modules come from the haves", func(b *fakeBus) { b.refused[grants.ReadAssignments] = true }, @@ -824,8 +870,7 @@ func TestGrants(t *testing.T) { Manifest: meshcatalogclient.ManifestDigest([]byte(`{"module":"mesh-knowledge","version":"2"}`)), Repository: "novox/x", Commit: "c", Verdict: meshcatalogclient.VerdictRefused, Problems: []string{"it waits"}, Unknown: []string{"core-credentials"}}) b.put(grants.ReadHave, "a.mesh-knowledge", nodeengine.HaveByNodeModuleValue{V: keys.Version, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK, Fp: fp}) - heldKey, _ := builds.BuilderBuildKey("mesh-knowledge", fp) - b.put(grants.ReadBuilds, heldKey, map[string]any{"v": keys.Version, "state": "done", "manifest": meshcatalogclient.ManifestDigest([]byte(knowledge))}) + b.entry(fp, meshcatalogclient.ManifestDigest([]byte(knowledge))) if heldWeb := (func() grants.User { us := shadowOf(t, b) return us[slices.IndexFunc(us, func(u grants.User) bool { return u.User == "a.web" })] @@ -933,8 +978,7 @@ func TestGrants(t *testing.T) { digest := meshcatalogclient.ManifestDigest([]byte(router)) b.put(grants.ReadManifests, digest, meshcatalog.ManifestsByDigestValue{V: keys.Version, Module: "messenger", Manifest: router}) b.put(grants.ReadHave, "a.messenger", nodeengine.HaveByNodeModuleValue{V: keys.Version, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK, Fp: fp}) - key, _ := builds.BuilderBuildKey("messenger", fp) - b.put(grants.ReadBuilds, key, map[string]any{"v": keys.Version, "state": "done", "manifest": digest}) + b.entry(fp, digest) } } return b @@ -1118,8 +1162,7 @@ func TestGrants(t *testing.T) { bt.assign("b", "mod-s") runsFP := "src2:" + strings.Repeat("cd", 32) bt.put(grants.ReadWants, "want.b.mod-s", meshwantsclient.Want{V: keys.Version, Fp: runsFP, Composed: "sha256:" + strings.Repeat("ef", 32)}) - runsKey, _ := builds.BuilderBuildKey("mod-s", runsFP) - bt.put(grants.ReadBuilds, runsKey, map[string]any{"v": keys.Version, "state": "done", "manifest": oldDigest}) + bt.entry(runsFP, oldDigest) for _, u := range shadowOf(t, bt) { if u.User == "node.a" && !slices.Contains(u.Publish, served) { t.Errorf("node a is not granted the domain the trunk's mod-s serves, while b runs one that serves none") @@ -4426,8 +4469,7 @@ func TestGrants(t *testing.T) { // mod-r runs a build whose manifest is the accepted one: the held module's seats and streams are composed from it. runFP := "src2:" + strings.Repeat("cd", 32) b.put(grants.ReadHave, "b.mod-r", nodeengine.HaveByNodeModuleValue{V: keys.Version, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK, Fp: runFP}) - runKey, _ := builds.BuilderBuildKey("mod-r", runFP) - b.put(grants.ReadBuilds, runKey, map[string]any{"v": keys.Version, "state": "done", "manifest": meshcatalogclient.ManifestDigest([]byte(ctl))}) + b.entry(runFP, meshcatalogclient.ManifestDigest([]byte(ctl))) p := partOn(b) p.Objects = &fakeObjects{} var said []oc.Condition @@ -4771,6 +4813,51 @@ func TestMembershipKeepsWhatTheRunningBuildReads(t *testing.T) { } } +// TestPassBeforeTheCatalogHalfsFirstViews is design 47 G §684 (A-52): while a first view of the catalog half is not in, the pass keeps +// the list in force and only adds, and says why; the want's account is composed from the entry of its fingerprint, and once the views +// are in the pass composes the whole mesh again. +func TestPassBeforeTheCatalogHalfsFirstViews(t *testing.T) { + waiting := func(b *fakeBus) { + b.mu.Lock() + b.catalogWaited = []string{"the catalog half of the branch main"} + b.mu.Unlock() + } + t.Run("a wait for a first view is the pass's reason to keep what the bus loaded, and says which", func(t *testing.T) { + b := fixture() + waiting(b) + in := grants.ReadInputs(b, &grants.Manifests{}) + if !strings.Contains(in.CatalogUnread, "the catalog half of the branch main") { + t.Fatalf("the pass says no wait for the catalog half's first view: %q", in.CatalogUnread) + } + if f := strings.Join(in.ReadFailures(), "\n"); !strings.Contains(f, "catalog half") { + t.Errorf("a pass before the first views retires something: %s", f) + } + if res := partOn(b).Pass(); res.CatalogWaiting == "" { + t.Errorf("the pass's result does not say it composed before the first views") + } + }) + t.Run("the want's account is composed from its entry before any view, and once they are in the pass is whole", func(t *testing.T) { + b := fixture() + built := strings.Replace(xManifest, `{"name":"kept","history":3}`, `{"name":"newer"}`, 1) + digest := meshcatalogclient.ManifestDigest([]byte(built)) + b.put(grants.ReadManifests, digest, meshcatalog.ManifestsByDigestValue{V: keys.Version, Module: "mod-x", Manifest: built}) + fp := "src2:" + strings.Repeat("cd", 32) + b.put(grants.ReadWants, "want.a.mod-x", meshwantsclient.Want{V: keys.Version, Fp: fp, Composed: "sha256:" + strings.Repeat("ef", 32)}) + b.entry(fp, digest) + waiting(b) + in := grants.ReadInputs(b, &grants.Manifests{}) + if _, ok := in.Running["a.mod-x"]; !ok || in.CatalogUnread == "" { + t.Errorf("before the first views the want's entry composes nothing: running %v, wait %q", in.Running, in.CatalogUnread) + } + b.mu.Lock() + b.catalogWaited = nil + b.mu.Unlock() + if in := grants.ReadInputs(b, &grants.Manifests{}); in.CatalogUnread != "" || len(in.Running) == 0 { + t.Errorf("with every first view in the pass still waits: %q", in.CatalogUnread) + } + }) +} + // A pass reads each machine's have for the build it runs (issue 1966): its membership keeps what that older build reads, a // have or build that cannot be read holds the pass back, and the seats held are the ones that build claims (ADR 0660 §2). func TestPassComposesMembershipFromTheHaveBuild(t *testing.T) { @@ -4781,8 +4868,7 @@ func TestPassComposesMembershipFromTheHaveBuild(t *testing.T) { b.put(grants.ReadManifests, digest, meshcatalog.ManifestsByDigestValue{V: keys.Version, Module: "mod-y", Manifest: old}) fp := "src2:" + strings.Repeat("cd", 32) b.put(grants.ReadHave, "b.mod-y", nodeengine.HaveByNodeModuleValue{V: keys.Version, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK, Fp: fp}) - key, _ := builds.BuilderBuildKey("mod-y", fp) - b.put(grants.ReadBuilds, key, map[string]any{"v": keys.Version, "state": "done", "manifest": digest}) + b.entry(fp, digest) } t.Run("the published membership keeps a read of the build the have names", func(t *testing.T) { b := fixture() @@ -4797,7 +4883,7 @@ func TestPassComposesMembershipFromTheHaveBuild(t *testing.T) { runs(b) part := partOn(b) part.Pass() - b.refused[grants.ReadBuilds] = true + b.entriesAsked = true part.Pass() if got := string(b.own[grants.StateMemberships]["b.mod-y"]); !strings.Contains(got, "mod-x.old") { t.Errorf("b.mod-y's membership lost the running build's read when its build could not be read: %s", got) @@ -4806,8 +4892,8 @@ func TestPassComposesMembershipFromTheHaveBuild(t *testing.T) { t.Run("a build that cannot be read is a failed read", func(t *testing.T) { b := fixture() runs(b) - b.refused[grants.ReadBuilds] = true - if f := grants.ReadInputs(b, &grants.Manifests{}).ReadFailed(); !strings.Contains(strings.Join(f, "\n"), grants.ReadBuilds) { + b.entriesAsked = true + if f := grants.ReadInputs(b, &grants.Manifests{}).ReadFailed(); !strings.Contains(strings.Join(f, "\n"), "catalog entry") { t.Errorf("a build read refused is not a failed read: %v", f) } }) diff --git a/modules/mesh-grants/internal/grants/catalog.go b/modules/mesh-grants/internal/grants/catalog.go new file mode 100644 index 00000000..0ea3ca40 --- /dev/null +++ b/modules/mesh-grants/internal/grants/catalog.go @@ -0,0 +1,202 @@ +package grants + +import ( + "context" + "encoding/json" + "fmt" + "slices" + "sync" + + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/views" +) + +// **The grants part reads the catalog half** (design 47 G §614, §444, §684, A-5, A-52): through meshcatalogclient's branch follower +// (Branches), adopted on the part's views.ProcessReader, which holds the part's finding writer, so the part's `view-waits` is the +// reader's. The manifest of the build a want or a have names is the catalog graph's entry of that fingerprint (Resolve), never a +// builder's build. + +// catalogHalfVerbs are the verbs every part that reads the catalog half names in its `invokes` (design 47 G §614): the revision of a +// branch, the registrations, the gate's source and the entry by fingerprint. (The gate's source is named in the short form: the +// long form with its field `branches`, §558, is not read by mesh-sdk's manifest reader yet.) +func catalogHalfVerbs() []string { + return []string{"seat:" + meshcatalog.Seat + "." + meshcatalog.VerbRevision, "seat:" + meshcatalog.Seat + "." + meshcatalog.VerbRegistrations, + meshcatalogclient.SourceVerb, meshcatalogclient.EntryVerb} +} + +// catalogHalfEvents are the events every such part names in its `consumes`, as `.`: the catalog's revision and +// registration-changed, and the gate's source-moved. +func catalogHalfEvents() []string { + return []string{meshcatalog.Seat + "." + meshcatalog.EventCatalogRevision, meshcatalog.Seat + "." + meshcatalog.EventRegistrationChanged, + meshdelivery.Seat + "." + meshdelivery.EventSourceMoved} +} + +// CatalogHalf is what the part reads of the catalog half: the entry by its fingerprint, the branches whose first view is not in, +// and what a tool says of both. +type CatalogHalf interface { + // Resolve reads the entry of fingerprint: from a followed branch's half, else asked of `entry` once and kept. + Resolve(fingerprint string) (meshcatalog.CatalogEntry, meshcatalogclient.EntryState) + // Waiting says why the part has not every first view of the catalog half, or nil when it has them all: the branches + // followed whose first view is not in, or that none is followed yet. + Waiting() []string + // Statuses are the first view of each followed branch, in the order followed. + Statuses() []views.Status + // Entries are the entries kept by fingerprint, with their state. + Entries() []meshcatalogclient.EntryRow +} + +// CatalogOf is the part's catalog half over the branch follower. Context is the process's life the entry asks run in; Wake +// starts a pass, and is called when a followed branch's first view comes in, which may change nothing a view's own change says. +type CatalogOf struct { + Branches *meshcatalogclient.Branches + Context context.Context + Wake func() + + mu sync.Mutex + watched map[string]bool +} + +// Resolve reads fingerprint's entry (meshcatalogclient.Branches.Resolve). +func (c *CatalogOf) Resolve(fingerprint string) (meshcatalog.CatalogEntry, meshcatalogclient.EntryState) { + return c.Branches.Resolve(c.Context, fingerprint) +} + +// Waiting says which first views of the catalog half are not in. +func (c *CatalogOf) Waiting() []string { + followed := c.Branches.Followed() + if len(followed) == 0 { + return []string{"no branch is followed yet (the registrations and the gate's source are not answered)"} + } + var out []string + for _, b := range followed { + if cat, ok := c.Branches.Catalog(b); ok { + select { + case <-cat.Ready(): + default: + out = append(out, "the catalog half of the branch "+b) + c.wakeAt(b, cat) + } + } + } + return out +} + +// wakeAt starts a pass when branch's first view comes in, once per branch. +func (c *CatalogOf) wakeAt(branch string, cat *meshcatalogclient.Catalog) { + c.mu.Lock() + if c.watched == nil { + c.watched = map[string]bool{} + } + done := c.watched[branch] + c.watched[branch] = true + c.mu.Unlock() + if done || c.Wake == nil { + return + } + go func() { + select { + case <-cat.Ready(): + c.Wake() + case <-c.Context.Done(): + } + }() +} + +// Statuses are the followed branches' first views. +func (c *CatalogOf) Statuses() []views.Status { + var out []views.Status + for _, b := range c.Branches.Followed() { + if cat, ok := c.Branches.Catalog(b); ok { + out = append(out, cat.Follower.Status()) + } + } + return out +} + +// Entries are the entries kept by fingerprint. +func (c *CatalogOf) Entries() []meshcatalogclient.EntryRow { return c.Branches.Entries.Rows() } + +// catalogReader is what a Reader that holds the catalog half answers (Views does): ReadInputs reads the want's entry through it. +type catalogReader interface { + Resolve(fingerprint string) (meshcatalog.CatalogEntry, meshcatalogclient.EntryState) + Waiting() []string +} + +// ErrNoCatalog is an entry asked of a reader that holds no catalog half (a tool's). +var ErrNoCatalog = fmt.Errorf("this reader holds no catalog half") + +// Resolve reads an entry by its fingerprint through the part's catalog half; none held answers asked. +func (v *Views) Resolve(fingerprint string) (meshcatalog.CatalogEntry, meshcatalogclient.EntryState) { + if v == nil || v.Catalog == nil { + return meshcatalog.CatalogEntry{}, "" + } + return v.Catalog.Resolve(fingerprint) +} + +// Waiting says which first views of the catalog half are not in; nil where none is held (a tool's reader waits for none). +func (v *Views) Waiting() []string { + if v == nil || v.Catalog == nil { + return nil + } + return v.Catalog.Waiting() +} + +// entryManifest is the manifest of the entry of fingerprint in the catalog half, kept by fingerprint. state is the entry's +// state as the half answers it ("" where the reader holds no catalog half); a manifest is answered only with an `answered` entry. +func (c *Manifests) entryManifest(r Reader, fingerprint string) (catalogue.Manifest, meshcatalogclient.EntryState, error) { + cr, ok := r.(catalogReader) + if !ok { + return catalogue.Manifest{}, "", ErrNoCatalog + } + c.mu.Lock() + m, ok := c.byFp[fingerprint] + c.mu.Unlock() + if ok { + return m, meshcatalogclient.EntryAnswered, nil + } + entry, state := cr.Resolve(fingerprint) + if state != meshcatalogclient.EntryAnswered { + return catalogue.Manifest{}, state, fmt.Errorf("the catalog half's entry of %s is %s", fingerprint, orNone(string(state))) + } + raw, err := json.Marshal(entry.Manifest) + if err != nil || len(entry.Manifest) == 0 { + return catalogue.Manifest{}, state, fmt.Errorf("the entry of %s holds no manifest", fingerprint) + } + // Decoded, never judged, as a manifest by digest is (Manifests.get). + m, err = catalogue.Decode(raw) + if err != nil { + return catalogue.Manifest{}, state, fmt.Errorf("the manifest of the entry of %s does not decode: %w", fingerprint, err) + } + c.mu.Lock() + defer c.mu.Unlock() + if c.byFp == nil { + c.byFp = map[string]catalogue.Manifest{} + } + c.byFp[fingerprint] = m + return m, state, nil +} + +func orNone(s string) string { + if s == "" { + return "not held" + } + return s +} + +// catalogWaiting is why this pass composes before the catalog half's first views (design 47 G §684): the part keeps the user list it +// last wrote and adds what the entries of the builds wanted give. +func catalogWaiting(r Reader) string { + cr, ok := r.(catalogReader) + if !ok { + return "" + } + w := cr.Waiting() + if len(w) == 0 { + return "" + } + slices.Sort(w) + return fmt.Sprintf("the first views of the catalog half are not in (%v), so each account is composed from its want's entry by fingerprint and the list in force is kept", w) +} diff --git a/modules/mesh-grants/internal/grants/firstviews.go b/modules/mesh-grants/internal/grants/firstviews.go index 8ada2af3..16fcb958 100644 --- a/modules/mesh-grants/internal/grants/firstviews.go +++ b/modules/mesh-grants/internal/grants/firstviews.go @@ -79,6 +79,8 @@ type Views struct { // Rest reads every other state, and Watching watches it (nil: watched by nobody, as in a tool). Rest Reader Watching Watcher + // Catalog is the part's catalog half (catalog.go): nil in a tool, which reads none. + Catalog CatalogHalf said *untilSaid } @@ -399,6 +401,20 @@ type FirstViewsState struct { Bindings *BindingsStatus `json:"bindings,omitempty"` // Streams are the whole reads of streams kept last per subject at the part's start, `waiting` until each ends (design 47 G §556). Streams []StreamRead `json:"streams,omitempty"` + // Catalog is the first view of each branch of the catalog half the part follows (design 47 G §614), and CatalogEntries the entries + // kept by fingerprint, each asked, answered or not found; CatalogWaiting says, while a first view is not in, that the part composes + // each account from its want's entry and keeps the list in force (§684). + Catalog []views.Status `json:"catalog,omitempty"` + CatalogEntries []CatalogEntry `json:"catalog-entries,omitempty"` + CatalogWaiting []string `json:"catalog-waiting,omitempty"` +} + +// CatalogEntry is one entry of the catalog half kept by fingerprint, as grants_first_views says it. +type CatalogEntry struct { + Fingerprint string `json:"fingerprint"` + State string `json:"state"` + Since time.Time `json:"since"` + Detail string `json:"detail,omitempty"` } // sayFirstViews keeps the first views' statuses in the part's own state (for grants_first_views) when they change, and @@ -421,6 +437,15 @@ func (p *Part) sayFirstViews(res *Result) { state.Bindings = &b } state.Streams = p.StreamReads() + if p.Views != nil && p.Views.Catalog != nil { + state.Catalog, state.CatalogWaiting = p.Views.Catalog.Statuses(), p.Views.Catalog.Waiting() + for i := range state.Catalog { + state.Catalog[i].Asked, state.Catalog[i].Answered = state.Catalog[i].Asked.Truncate(time.Second), state.Catalog[i].Answered.Truncate(time.Second) + } + for _, e := range p.Views.Catalog.Entries() { + state.CatalogEntries = append(state.CatalogEntries, CatalogEntry{Fingerprint: e.Fingerprint, State: string(e.State), Since: e.Since.Truncate(time.Second), Detail: e.Detail}) + } + } data, _ := json.Marshal(state) p.changed(res, StateUserList, KeyFirstViews, data) if p.Views == nil { diff --git a/modules/mesh-grants/internal/grants/follow.go b/modules/mesh-grants/internal/grants/follow.go index 7ba13912..07a78196 100644 --- a/modules/mesh-grants/internal/grants/follow.go +++ b/modules/mesh-grants/internal/grants/follow.go @@ -48,9 +48,11 @@ type Follower struct { heard_ bool keptHeld bool rootSaid string - watched map[string]bool - refused map[string]string - have map[string]string + // catalogSaid is the wait for the catalog half's first views last said in the log (§684). + catalogSaid string + watched map[string]bool + refused map[string]string + have map[string]string // held and notAsserted are what the last pass held back and could not create, said again only when they change. held string notAsserted string @@ -200,6 +202,14 @@ func (f *Follower) open() { // the pass could not create, each with why (issue 1790), when the set changes from the last pass's; and every stream it // created, one line per module. func (f *Follower) said(res Result) { + if res.CatalogWaiting != f.catalogSaid { + f.catalogSaid = res.CatalogWaiting + if res.CatalogWaiting == "" { + f.logf("every first view of the catalog half is in: every account is composed again from the whole mesh") + } else { + f.logf("composed before the catalog half's first views: %s", res.CatalogWaiting) + } + } if refused := strings.Join(res.Refused, "\n"); refused != f.refusedMans { f.refusedMans = refused if len(res.Refused) == 0 { diff --git a/modules/mesh-grants/internal/grants/inputs.go b/modules/mesh-grants/internal/grants/inputs.go index 7b78b215..62a56579 100644 --- a/modules/mesh-grants/internal/grants/inputs.go +++ b/modules/mesh-grants/internal/grants/inputs.go @@ -30,7 +30,6 @@ import ( "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshwantsclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go/overlay" "git.novox.be/novox/mesh-sdk/go/catalogue" ) @@ -45,7 +44,6 @@ const ( ReadHave = "node-engine.have" ReadResolved = "mesh-config.resolved" ReadWants = "mesh-delivery-assigner.wants" - ReadBuilds = "mesh-delivery-builder.builds" // ReadAccounts is the vault's bcrypt hashes of the bus accounts it holds, by bus user (ADR 0547 clause 9). ReadAccounts = "mesh-vault.bus-accounts" // ReadAssignments is the seat mesh-module-assigner's assignments (ADR 0567 §1, §9): each node's modules and the seats @@ -61,7 +59,7 @@ const ( // Reads is every state the part reads, in the manifest's order. Not the controller's assignments: the seat's assignments // alone places modules (novox/hq ADR 0549 §5, ADR 0571 §10; issue 774). -var Reads = []string{ReadResolved, ReadHave, ReadBuilds, ReadAccounts, ReadEnrolments, ReadNodeLogins} +var Reads = []string{ReadResolved, ReadHave, ReadAccounts, ReadEnrolments, ReadNodeLogins} // ViewReads are the owners' states the part reads by their first views and views families, never their buckets // (firstviews.go, ADR 0615): each read in Reads' place. @@ -75,7 +73,7 @@ const ( ) // Invokes is every tool the part calls. -var Invokes = append([]string{CallUsers, CallUserCan, CallStreams, CallRootFree, CallHandAct, CallSeatDefinitions, CallBrokerList}, firstViewTools()...) +var Invokes = append([]string{CallUsers, CallUserCan, CallStreams, CallRootFree, CallHandAct, CallSeatDefinitions, CallBrokerList}, append(catalogHalfVerbs(), firstViewTools()...)...) // CallSeatDefinitions reads the seat list (mesh-sdk seatlist): the verb of the seat mesh-catalog. const CallSeatDefinitions = "seat:mesh-catalog.seat-definitions" @@ -205,6 +203,9 @@ type Inputs struct { // bound); empty when the wants were read. As ByHaves, such a pass retires nothing (ReadFailures) and only adds // (keepsLoaded): the list is held only where it would take a permission from a loaded user (holdsBack). WantsUnread string + // CatalogUnread says why this pass composed before the first views of the catalog half were in (design 47 G §684); empty once they are. + // As WantsUnread, such a pass retires nothing (ReadFailures) and only adds: the list in force is kept. + CatalogUnread string // Held is, by module, the manifest a held module's running build was judged on (its have's fingerprint, the builder's // build of it): what the seats it defines, the streams others hear from it and its reads are composed from while its // trunk manifest is refused or waits, so no other principal loses a grant to it (ADR 0601 clause 3). @@ -258,6 +259,8 @@ const BusModule = "nats" type Manifests struct { mu sync.Mutex parsed map[string]catalogue.Manifest + // byFp are the manifests of the catalog half's entries, by fingerprint: a fingerprint names one entry for ever (catalog.go). + byFp map[string]catalogue.Manifest // gate says whether a manifest naming some seats may be composed yet (the seat list reader, ADR 0636 §1); nil gates none. gate SeatGate } @@ -356,6 +359,7 @@ func ReadInputs(r Reader, cache *Manifests) Inputs { NoFingerprint: map[string]string{}, Running: map[string]catalogue.Manifest{}, Had: map[string]catalogue.Manifest{}, HadFailed: map[string]string{}, Values: map[string]any{}, Gaps: append([]Gap(nil), standingGaps...)} in.SeatListUnread = gate != nil && !gate.MayAct() + in.CatalogUnread = catalogWaiting(r) in.Defined = definedEventsOf(nil) // The definitions the reader holds, the bootstrapped ones before the seat list is read (ADR 0636 §2), name the events kept // on a stream of their own, so a durable made at the first pass already hears them there and never on EVENTS (a held @@ -550,7 +554,7 @@ func ReadInputs(r Reader, cache *Manifests) Inputs { Why: "assigned, and the indexer holds no manifest of it at the trunk (a module registered from outside the indexed repositories): composed with no authority"}) } } - // What each machine is wanted to run: the want's fingerprint, the builder's build of it, the manifest it built. + // What each machine is wanted to run: the want's fingerprint, the catalog half's entry of it. var fromTrunk []string wants := map[string][]byte{} _, wantsErr := r.Keys(ReadWants) @@ -585,21 +589,8 @@ func ReadInputs(r Reader, cache *Manifests) Inputs { if w.Removed || w.Fp == "" { continue } - bk, err := builds.BuilderBuildKey(module, w.Fp) - if err != nil { - continue - } - braw, found, err := r.Get(ReadBuilds, bk) - if err != nil || !found { - fromTrunk = append(fromTrunk, node+"."+module) - continue - } - b, err := builds.DecodeBuilderBuild(braw) - if err != nil || b.Manifest == "" { - fromTrunk = append(fromTrunk, node+"."+module) - continue - } - if m, err := cache.get(r, b.Manifest); err == nil { + // The manifest of the build the want names is the catalog graph's entry of its fingerprint (design 47 G §444, §614). + if m, _, err := cache.entryManifest(r, w.Fp); err == nil { in.Running[node+"."+module] = m } else { fromTrunk = append(fromTrunk, node+"."+module) @@ -761,6 +752,9 @@ func (in Inputs) ReadFailures() []string { if in.WantsUnread != "" { out = append(out, ReadWants+": "+in.WantsUnread) } + if in.CatalogUnread != "" { + out = append(out, "catalog half: "+in.CatalogUnread) + } for _, g := range in.Refused { out = append(out, g.Input+": "+g.Why) } @@ -816,30 +810,16 @@ func runningByHaves(r Reader, in *Inputs, cache *Manifests, into map[string]cata fromTrunk = append(fromTrunk, node+"."+module) continue } - bk, err := builds.BuilderBuildKey(module, h.Fp) + m, state, err := cache.entryManifest(r, h.Fp) if err != nil { + if state == meshcatalogclient.EntryAsked { + in.HadFailed[node+"."+module] = "the catalog half's entry of " + h.Fp + " is asked" + in.Gaps = append(in.Gaps, Gap{Input: "catalog entry " + h.Fp, Why: "asked and not answered yet, so the manifest of the build " + node + "." + module + " runs is unknown: " + err.Error(), Failed: true}) + } fromTrunk = append(fromTrunk, node+"."+module) continue } - braw, found, err := r.Get(ReadBuilds, bk) - if err != nil { - in.HadFailed[node+"."+module] = ReadBuilds + " cannot be read" - in.Gaps = append(in.Gaps, Gap{Input: ReadBuilds + " " + bk, Why: "cannot be read, so the manifest of the build " + node + "." + module + " runs is unknown: " + err.Error(), Failed: true}) - } - if err != nil || !found { - fromTrunk = append(fromTrunk, node+"."+module) - continue - } - b, err := builds.DecodeBuilderBuild(braw) - if err != nil || b.Manifest == "" { - fromTrunk = append(fromTrunk, node+"."+module) - continue - } - if m, err := cache.get(r, b.Manifest); err == nil { - into[node+"."+module] = m - } else { - fromTrunk = append(fromTrunk, node+"."+module) - } + into[node+"."+module] = m } } return fromTrunk @@ -864,19 +844,7 @@ func heldManifests(r Reader, in Inputs, cache *Manifests) map[string]catalogue.M if err != nil || h.Fp == "" { continue } - bk, err := builds.BuilderBuildKey(module, h.Fp) - if err != nil { - continue - } - braw, found, err := r.Get(ReadBuilds, bk) - if err != nil || !found { - continue - } - b, err := builds.DecodeBuilderBuild(braw) - if err != nil || b.Manifest == "" { - continue - } - if m, err := cache.get(r, b.Manifest); err == nil { + if m, _, err := cache.entryManifest(r, h.Fp); err == nil { out[module] = m break } diff --git a/modules/mesh-grants/internal/grants/keeploaded.go b/modules/mesh-grants/internal/grants/keeploaded.go index 23693a87..e65dea25 100644 --- a/modules/mesh-grants/internal/grants/keeploaded.go +++ b/modules/mesh-grants/internal/grants/keeploaded.go @@ -23,6 +23,8 @@ func (in Inputs) keepsLoaded() string { return in.ByHaves case in.WantsUnread != "": return in.WantsUnread + case in.CatalogUnread != "": + return in.CatalogUnread case in.SeatListUnread: return "the seat list is unread, so only the bootstrapped seats compose" } diff --git a/modules/mesh-grants/internal/grants/part.go b/modules/mesh-grants/internal/grants/part.go index 5a4669d5..51ea62c9 100644 --- a/modules/mesh-grants/internal/grants/part.go +++ b/modules/mesh-grants/internal/grants/part.go @@ -140,7 +140,9 @@ type Result struct { Withheld *WithheldWritten `json:"withheld,omitempty"` // PrincipalsRefused is each principal whose composed grants the composer refused, `: `: it keeps its grants // of the list in force, or stays out where it holds none (refused.go); the condition mesh-grants.principal-refused says it. - PrincipalsRefused []string `json:"principals_refused,omitempty"` + PrincipalsRefused []string `json:"principals_refused,omitempty"` + // CatalogWaiting is why this pass composed before the first views of the catalog half were in (design 47 G §684); empty once they are. + CatalogWaiting string `json:"catalog_waiting,omitempty"` Errors []string `json:"errors,omitempty"` PublishedOrAsserted string `json:"published_or_asserted"` } @@ -306,6 +308,7 @@ func (p *Part) Compose() Result { p.Views.SayUnsaid() in := p.inputs() p.keepBrokers(in) + res.CatalogWaiting = in.CatalogUnread if err := p.withBound(&in); err != nil { res.Errors = append(res.Errors, err.Error()) } diff --git a/modules/mesh-grants/internal/grants/rootfree.go b/modules/mesh-grants/internal/grants/rootfree.go index b901a577..819c0763 100644 --- a/modules/mesh-grants/internal/grants/rootfree.go +++ b/modules/mesh-grants/internal/grants/rootfree.go @@ -43,7 +43,7 @@ const EventAccountIssued = "mesh-vault." + buscompose.AccountIssuedEvent // Consumes is every event the part hears, as its manifest's consumes name them. // The seat mesh-broker's created, recreated and dropped are heard as every seat's event, `.` on EVENTS (the // lead's ruling, registry milestone 13:0x), never on the bus module's own stream. -var Consumes = append(append(append(slices.Clone(seatEvents), seatListEvents...), viewsEvents()...), ConsumesRunning, ConsumesFindingsAsked) +var Consumes = append(append(append(append(slices.Clone(seatEvents), seatListEvents...), catalogHalfEvents()...), viewsEvents()...), ConsumesRunning, ConsumesFindingsAsked) // ConsumesFindingsAsked is the keeper's ask the part's finding writer hears on a consumer of its own (design 47 G §390, §529): in // the manifest's consumes for the grant, never handed to the event hub. diff --git a/modules/mesh-grants/module.json b/modules/mesh-grants/module.json index 08434e00..e6e28462 100644 --- a/modules/mesh-grants/module.json +++ b/modules/mesh-grants/module.json @@ -4,7 +4,6 @@ "slug": "grants", "reads": [ "mesh-config.resolved", - "mesh-delivery-builder.builds", "node-engine.have", "mesh-vault.bus-accounts", "mesh-enrolment.enrolments", @@ -22,10 +21,13 @@ "mesh-delivery-assigner.views.wants", "mesh-delivery-indexer.views.manifests", "mesh-delivery-indexer.views.modules", + "mesh-catalog.catalog-revision", "mesh-catalog.manifest-accepted", "mesh-catalog.manifest-refused", + "mesh-catalog.registration-changed", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", + "mesh-delivery.source-moved", "node-engine.running.*.*" ], "uses": [ @@ -41,7 +43,11 @@ "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "mesh-delivery-indexer.verdicts", + "seat:mesh-catalog.entry", + "seat:mesh-catalog.registrations", + "seat:mesh-catalog.revision", "seat:mesh-catalog.seat-definitions", + "seat:mesh-delivery.source", "seat:mesh-hand-acts.hand-act", "seat:mesh-broker.list" ],