diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/catalogread.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/catalogread.go new file mode 100644 index 00000000..97803058 --- /dev/null +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/catalogread.go @@ -0,0 +1,83 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "sort" + "sync" + "time" + + "mesh-delivery-view/internal/view" +) + +// catalogWithin is how long a call waits for the seat mesh-catalog's first view before it says there is none. +var catalogWithin = 30 * time.Second + +// catalogRead is the modules the view's tools answer, read from the catalog half through the seat mesh-catalog's verb +// `revision` (design 47, Two graphs): the trunk's newest revision, asked once per call and in pages, each entry in the +// form of the value the tools always answered. With no view the read says so; it never answers no modules. +func catalogRead(ask func(key string, body any) (json.RawMessage, error)) view.StateRead { + var ( + once sync.Once + values map[string]json.RawMessage + rev uint64 + failed error + ) + load := func() { + ctx, cancel := context.WithTimeout(context.Background(), catalogWithin) + defer cancel() + c, err := meshcatalogclient.NewCatalog(ask, "main") + if err != nil { + failed = err + return + } + if err := c.Start(ctx); err != nil { + failed = fmt.Errorf("the catalog half has no first view yet (%s): %w", meshcatalogclient.RevisionVerb, err) + return + } + half, ok := c.View() + if !ok { + failed = fmt.Errorf("the catalog half has no first view yet (%s)", meshcatalogclient.RevisionVerb) + return + } + rev, values = uint64(half.Revision.Revision), map[string]json.RawMessage{} + for module, e := range half.Entries { + raw, _ := json.Marshal(e) + v, err := view.ModuleValue(raw) + if err != nil { + failed = fmt.Errorf("the catalog entry of %s: %w", module, err) + return + } + values[module] = v + } + } + read := func() (map[string]json.RawMessage, uint64, error) { + once.Do(load) + return values, rev, failed + } + return view.StateRead{ + Keys: func() ([]string, error) { + v, _, err := read() + keys := make([]string, 0, len(v)) + for k := range v { + keys = append(keys, k) + } + sort.Strings(keys) + return keys, err + }, + Get: func(key string) (json.RawMessage, error) { + v, _, err := read() + return v[key], err + }, + GetEntry: func(key string) (json.RawMessage, uint64, error) { + v, r, err := read() + return v[key], r, err + }, + All: func() (map[string]json.RawMessage, error) { + v, _, err := read() + return v, err + }, + } +} diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go index 1a8f828a..9baa4b52 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go @@ -396,7 +396,7 @@ func ownersRead(ask func(key string, body any) (json.RawMessage, error)) view.Re // Each owner's read tool as the view's invokes names it. indexer := func(tool string) view.StateRead { return view.OwnerRead(ask, view.IndexerModule+"."+tool) } assigner := func(tool string) view.StateRead { return view.OwnerRead(ask, view.AssignerModule+"."+tool) } - return view.Reader{State: indexer("indexer_read_modules"), CandidatesState: indexer("indexer_read_candidates"), + return view.Reader{State: catalogRead(ask), CandidatesState: indexer("indexer_read_candidates"), WantsState: assigner("assigner_read_wants"), DifferencesState: assigner("assigner_read_differences"), HeardState: indexer("indexer_read_heard"), PredefinedState: indexer("indexer_read_predefined"), diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/owners_test.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/owners_test.go index 80ad8589..8864369f 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/owners_test.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/owners_test.go @@ -3,12 +3,14 @@ package main import ( "encoding/json" "fmt" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" "mesh-delivery-view/internal/view" "sort" "strings" "testing" "time" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/sdk/graph" "git.novox.be/novox/mesh-sdk/go/views" ) @@ -51,7 +53,6 @@ func ownersAnswerTheToolsAsBefore(t *testing.T) { t.Fatal(err) } states := map[string]source{ - view.IndexerModule + ".modules": {"a": withFP2, "b": withNone}, view.IndexerModule + ".candidates": {"feat/x": cand, "fix": `{"v":1,"group":"fix","base":3,"heads":"h","pulls":[{"modules":["netcheck"]}]}`}, view.AssignerModule + ".wants": {"want.ace.zsh": `{"fp":"` + fp2A + `"}`, "want.novox.zsh": `{"fp":"` + fp2B + `"}`, "held." + keyB: `{"node":"ace","words":"failed"}`}, @@ -67,14 +68,12 @@ func ownersAnswerTheToolsAsBefore(t *testing.T) { const heard1 = `{"v":1,"subject":"s","sequence":41,"repository":"novox/mesh-catalog","commit":"9cdac09a"}` states[view.IndexerModule+".heard"] = source{"forge": heard1} histories := map[string]map[string][]views.Revision{ - view.IndexerModule + ".modules": {"a": {{Revision: 7, At: at, Op: "put", Value: json.RawMessage(withNone)}, - {Revision: 9, At: at.Add(time.Minute), Op: "put", Value: json.RawMessage(withFP2)}}}, view.IndexerModule + ".heard": {"forge": {{Revision: 3, At: at, Op: "put", Value: json.RawMessage(heard1)}}}, view.AssignerModule + ".wants": {"want.ace.zsh": {{Revision: 12, At: at, Op: "put", Value: json.RawMessage(`{"fp":"` + fp2A + `"}`)}}}, } // The direct reading, as the tools read the states before (the reads the manifest no longer has). st := func(name string) view.StateRead { return fake(states[name]).reader().State } - direct := view.Reader{State: st(view.IndexerModule + ".modules"), CandidatesState: st(view.IndexerModule + ".candidates"), + direct := view.Reader{CandidatesState: st(view.IndexerModule + ".candidates"), WantsState: st(view.AssignerModule + ".wants"), DifferencesState: st(view.AssignerModule + ".differences"), PredefinedState: st(view.IndexerModule + "." + view.IndexerPredefinedState), ConfigCandidatesState: st(view.IndexerModule + "." + view.IndexerConfigCandidatesState), @@ -89,6 +88,10 @@ func ownersAnswerTheToolsAsBefore(t *testing.T) { } asked := map[string]int{} ask := func(key string, body any) (json.RawMessage, error) { + if key == meshcatalogclient.RevisionVerb { + asked[key]++ + return json.Marshal(catalogAnswer) + } name, ok := tools[key] if !ok { return nil, fmt.Errorf("no tool %s", key) @@ -118,10 +121,6 @@ func ownersAnswerTheToolsAsBefore(t *testing.T) { tool string args map[string]any }{ - {view.ModuleReadTool, map[string]any{"module": "a"}}, - {view.ModuleReadTool, map[string]any{"module": "gone"}}, - {view.ModulesReadTool, map[string]any{}}, - {view.ModulesReadTool, map[string]any{"none2": true}}, {view.CandidateReadTool, map[string]any{"group": "feat/x"}}, {view.CandidatesReadTool, map[string]any{}}, {view.CandidatesReadTool, map[string]any{"module": "zsh"}}, @@ -143,12 +142,24 @@ func ownersAnswerTheToolsAsBefore(t *testing.T) { t.Errorf("%s %v through the owner's read tool = %s, %v; reading the state it was %s, %v", row.tool, row.args, gj, gerr, wj, werr) } } - // A key's history is asked of the owner's read tool and answered from its kept values (A2j finding 1, J4). - out, err := run(t, owned, view.ModuleHistoryTool, map[string]any{"module": "a"}) - if h, _ := out.(view.ModuleHistory); err != nil || len(h.Values) != 2 || h.Values[0].Revision != 9 || h.Values[1].FP2 != "" || !h.Values[1].At.Equal(at) { - t.Errorf("indexer_module_history a through the indexer's read tool = %+v, %v", out, err) + // The modules are the catalog half's, asked of the seat mesh-catalog's verb revision (W16.1): a module is its entry in + // the form of the value the tool always answered, the modules without a fingerprint say why, and the module's history + // is not a thing the catalog half keeps. + a, err := run(t, owned, view.ModuleReadTool, map[string]any{"module": "a"}) + if raw, _ := json.Marshal(a); err != nil || !strings.Contains(string(raw), `"fp2":"`+fp2A+`"`) || !strings.Contains(string(raw), `"commit":"c1"`) || !strings.Contains(string(raw), `"src":7`) { + t.Errorf("indexer_module a = %s, %v", raw, err) } - out, err = run(t, owned, view.HeardTool, map[string]any{}) + if _, err := run(t, owned, view.ModuleReadTool, map[string]any{"module": "gone"}); err == nil { + t.Errorf("indexer_module gone answered a module the catalog half does not hold") + } + all, err := run(t, owned, view.ModulesReadTool, map[string]any{"none2": true}) + if raw, _ := json.Marshal(all); err != nil || !strings.Contains(string(raw), `"b"`) || strings.Contains(string(raw), `"a"`) || !strings.Contains(string(raw), "builder's fingerprint not yet known") { + t.Errorf("indexer_modules none2 = %s, %v", raw, err) + } + if _, err := run(t, owned, view.ModuleHistoryTool, map[string]any{"module": "a"}); err == nil { + t.Errorf("indexer_module_history answered a history the catalog half does not keep") + } + out, err := run(t, owned, view.HeardTool, map[string]any{}) if rows, _ := out.(view.HeardAnswer); err != nil || len(rows.Heard) != 1 || rows.Heard[0].Revision != 3 || string(rows.Heard[0].Heard) != heard1 { t.Errorf("indexer_heard through the indexer's read tool = %+v, %v", out, err) } @@ -156,10 +167,29 @@ func ownersAnswerTheToolsAsBefore(t *testing.T) { if revs, err := owned.WantsState.History("want.ace.zsh"); err != nil || len(revs) != 1 || !revs[0].At.Equal(at) || revs[0].Revision != 12 { t.Errorf("the want's history through the assigner's read tool = %+v, %v", revs, err) } + // With no view of the catalog half the modules are not read: the tools say so, never that no module is held. + catalogWithin = 20 * time.Millisecond + none := ownersRead(func(key string, body any) (json.RawMessage, error) { return nil, fmt.Errorf("the holder is away") }) + if out, err := run(t, none, view.ModulesReadTool, map[string]any{}); err == nil || !strings.Contains(err.Error(), "no first view") { + t.Errorf("indexer_modules with no view = %v, %v, want an error naming it", out, err) + } // Each owner state the tools read was asked of its owner's read tool. for tool, name := range tools { if name != view.IndexerModule+"."+graph.State && asked[tool] == 0 { t.Errorf("%s was never asked", tool) } } + if asked[meshcatalogclient.RevisionVerb] == 0 { + t.Errorf("%s was never asked", meshcatalogclient.RevisionVerb) + } +} + +// catalogAnswer is the seat mesh-catalog's newest trunk revision: a with a fingerprint and the source revision it moved at, +// b with none and the reason. +var catalogAnswer = meshcatalog.RevisionAnswer{ + Revision: meshcatalog.CatalogRevision{Revision: 4, Kind: meshcatalogclient.KindTrunk, Source: 7}, + Entries: []meshcatalog.CatalogEntry{ + {Module: "a", Repository: "novox/r", Commit: "c1", Path: ".", Manifest: map[string]any{"module": "a"}, Fp: fp2A, Inputs: &meshcatalog.EntryInputs{Src: 7}}, + {Module: "b", Repository: "novox/r", Commit: "c2", Path: "modules/b", Manifest: map[string]any{"module": "b"}, None: "builder's fingerprint not yet known"}, + }, } diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view/integration_test.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view/integration_test.go index 77ad3d76..14248181 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view/integration_test.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view/integration_test.go @@ -12,6 +12,7 @@ import ( "path/filepath" "slices" "strings" + "sync" "testing" "time" @@ -47,6 +48,67 @@ func serveOwners(t *testing.T, ctx context.Context, js jetstream.JetStream, nc * } } +// catalogHolder is the seat mesh-catalog's holder as the view's process needs it (W16.1): its verb revision answering the +// trunk's newest revision whole, and its event catalog-revision said on the shared events stream at each change, which the +// view's durable consumer hears. +type catalogHolder struct { + t testing.TB + js jetstream.JetStream + mu sync.Mutex + // rev is the revision's number; entries are its entries by module. + rev int64 + entries map[string]meshcatalog.CatalogEntry + ctx context.Context +} + +func serveCatalog(t testing.TB, ctx context.Context, js jetstream.JetStream, nc *nats.Conn) *catalogHolder { + t.Helper() + h := &catalogHolder{t: t, js: js, ctx: ctx, rev: 1, entries: map[string]meshcatalog.CatalogEntry{}} + events, err := js.Stream(ctx, process.EventsStream) + if err != nil { + t.Fatal(err) + } + info := events.CachedInfo().Config + if !slices.Contains(info.Subjects, "mesh.seat.*.event.>") { + info.Subjects = append(info.Subjects, "mesh.seat.*.event.>") + if _, err := js.UpdateStream(ctx, info); err != nil { + t.Fatal(err) + } + } + answer := func(m *nats.Msg) { + h.mu.Lock() + a := meshcatalog.RevisionAnswer{Revision: meshcatalog.CatalogRevision{Revision: h.rev, Kind: meshcatalogclient.KindTrunk}} + for _, e := range h.entries { + a.Entries = append(a.Entries, e) + } + h.mu.Unlock() + raw, _ := json.Marshal(map[string]any{"result": a, "node": "n0"}) + _ = m.Respond(raw) + } + for _, subject := range []string{"mesh.seat.mesh-catalog.call.revision.>", "mesh.seat.mesh-catalog.call.revision"} { + sub, err := nc.Subscribe(subject, answer) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = sub.Unsubscribe() }) + } + return h +} + +// put is the holder's next trunk revision with the entry of one module changed, said as catalog-revision. +func (h *catalogHolder) put(e meshcatalog.CatalogEntry) { + h.t.Helper() + h.mu.Lock() + h.rev++ + h.entries[e.Module] = e + rev := h.rev + h.mu.Unlock() + body, _ := json.Marshal(meshcatalog.CatalogRevisionEvent{Kind: meshcatalogclient.KindTrunk, Branch: "main", Revision: rev, Changed: []string{e.Module}}) + if _, err := h.js.Publish(h.ctx, "mesh.seat.mesh-catalog.event."+meshcatalog.EventCatalogRevision, body); err != nil { + h.t.Fatal(err) + } +} + // TestProcessWholeSeesTheBuckets starts the program the way the node does (its entry, run, with its environment and a // broker file) against an embedded bus holding the three buckets, and asks its socket. func TestProcessWholeSeesTheBuckets(t *testing.T) { // 1134/AC8 @@ -126,6 +188,7 @@ func TestProcessWholeSeesTheBuckets(t *testing.T) { // 1134/AC8 runCtx, stop := context.WithCancel(ctx) done := make(chan error, 1) serveOwners(t, ctx, js, nc) + serveCatalog(t, ctx, js, nc) go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() t.Cleanup(func() { stop(); <-done }) @@ -226,6 +289,7 @@ func TestProcessWholeShowsWhatTheWatchmanSays(t *testing.T) { // 1168/AC5 runCtx, stop := context.WithCancel(ctx) done := make(chan error, 1) serveOwners(t, ctx, js, nc) + serveCatalog(t, ctx, js, nc) go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() t.Cleanup(func() { stop(); <-done }) @@ -293,6 +357,7 @@ func TestProcessWholeShowsTheIndexersBucketsOnceTheyExist(t *testing.T) { // 124 runCtx, stop := context.WithCancel(ctx) done := make(chan error, 1) serveOwners(t, ctx, js, nc) + catalog := serveCatalog(t, ctx, js, nc) go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() t.Cleanup(func() { stop(); <-done }) @@ -306,26 +371,16 @@ func TestProcessWholeShowsTheIndexersBucketsOnceTheyExist(t *testing.T) { // 124 if err != nil { t.Fatal(err) } - mods, err := js.KeyValue(ctx, view.ModulesBucket) - if err != nil { - t.Fatal(err) - } const head = "0123456789abcdef0123456789abcdef01234567" hash, _ := meshdeliveryclient.HeadsHash([]meshdeliveryclient.Head{{Repository: "novox/mesh-delivery", Number: 12, Head: head}}) cand, err := json.Marshal(meshcatalog.CandidatesByGroupValue{Group: "feat/x", Heads: hash, Pulls: []meshcatalog.CandidatesByGroupValuePullsItem{{Repository: "novox/mesh-delivery", Number: 12, Head: head, Modules: []string{"a"}}}}) if err != nil { t.Fatal(err) } - mod, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{None: "no tree", Repository: "novox/mesh-delivery", Commit: head}) - if err != nil { - t.Fatal(err) - } if _, err := cands.Put(ctx, "feat/x", cand); err != nil { t.Fatal(err) } - if _, err := mods.Put(ctx, "a", mod); err != nil { - t.Fatal(err) - } + catalog.put(meshcatalog.CatalogEntry{Module: "a", None: "no tree", Repository: "novox/mesh-delivery", Commit: head, Manifest: map[string]any{"module": "a"}}) var last view.Snapshot for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) { if s, err := view.Ask(socket); err == nil { @@ -380,6 +435,7 @@ func TestProcessWholeShowsTheOpenAsksAndDropsTheAnsweredOnes(t *testing.T) { // runCtx, stop := context.WithCancel(ctx) done := make(chan error, 1) serveOwners(t, ctx, js, nc) + serveCatalog(t, ctx, js, nc) go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() t.Cleanup(func() { stop(); <-done }) @@ -471,6 +527,7 @@ func TestProcessWholeListsEveryDifference(t *testing.T) { // 1297/B8 AC runCtx, stop := context.WithCancel(ctx) done := make(chan error, 1) serveOwners(t, ctx, js, nc) + serveCatalog(t, ctx, js, nc) go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() t.Cleanup(func() { stop(); <-done }) @@ -569,6 +626,7 @@ func TestProcessWholeWritesTheAnswerToAnAskNeedingNoProof(t *testing.T) { // 129 runCtx, stop := context.WithCancel(ctx) done := make(chan error, 1) serveOwners(t, ctx, js, nc) + serveCatalog(t, ctx, js, nc) go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() t.Cleanup(func() { stop(); <-done }) @@ -721,6 +779,7 @@ func TestProcessWholeShowsTheBacklogOfBuildAsks(t *testing.T) { // 1887/A8a AC runCtx, stop := context.WithCancel(ctx) done := make(chan error, 1) serveOwners(t, ctx, js, nc) + serveCatalog(t, ctx, js, nc) go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() t.Cleanup(func() { stop(); <-done }) @@ -768,7 +827,7 @@ func TestProcessWholeSaysTheSourceRevisionIsDelivered(t *testing.T) { t.Fatal(err) } kvs := map[string]jetstream.KeyValue{} - for _, name := range []string{view.SourceBucket, view.ModulesBucket, view.AssignerWantsBucket, view.HaveBucket} { + for _, name := range []string{view.SourceBucket, view.AssignerWantsBucket, view.HaveBucket} { kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: 5}) if err != nil { t.Fatal(err) @@ -789,20 +848,16 @@ func TestProcessWholeSaysTheSourceRevisionIsDelivered(t *testing.T) { runCtx, stop := context.WithCancel(ctx) done := make(chan error, 1) serveOwners(t, ctx, js, nc) + catalog := serveCatalog(t, ctx, js, nc) go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() t.Cleanup(func() { stop(); <-done }) fp := "src2:" + strings.Repeat("2", 64) - mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: strings.Repeat("a", 64), Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath}) want, _ := meshwantsclient.EncodeWant(meshwantsclient.Want{Fp: fp, Composed: "sha256:" + strings.Repeat("c", 64)}) failed, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed}) good, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}) src, _ := meshdeliveryclient.EncodeSource(meshdelivery.SourceByBranchValue{Group: "feat/x", Heads: map[string]string{"novox/web": strings.Repeat("1", 40)}}) - for bucket, kv := range map[string][]byte{view.ModulesBucket: mod} { - if _, err := kvs[bucket].Put(ctx, "web", kv); err != nil { - t.Fatal(err) - } - } + catalog.put(meshcatalog.CatalogEntry{Module: "web", Fp: fp, Manifest: map[string]any{"module": "a"}, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath}) if _, err := kvs[view.AssignerWantsBucket].Put(ctx, "want.ace.web", want); err != nil { t.Fatal(err) } @@ -886,6 +941,7 @@ func TestProcessWholeSaysModulesInMotion(t *testing.T) { runCtx, stop := context.WithCancel(ctx) done := make(chan error, 1) serveOwners(t, ctx, js, nc) + serveCatalog(t, ctx, js, nc) go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() t.Cleanup(func() { stop(); <-done }) time.Sleep(2 * time.Second) // the view has read what stood: nothing diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go index 471875c1..ecd9dcd5 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go @@ -11,6 +11,7 @@ import ( "encoding/json" "errors" "fmt" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" "io" "log" "mesh-delivery-view/internal/view" @@ -126,15 +127,21 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } } open := buckets{js: js, reader: reader} - for _, bucket := range []string{view.HaveBucket, view.AssignmentsBucket, view.PresenceBucket, view.StuckBucket, view.CandidatesBucket, view.ModulesBucket, view.AsksBucket, view.AnswersBucket, builds.BuilderBuildsBucket, view.AssignerWantsBucket, view.SourceBucket, view.AssignerAsksBucket, viewstate.ViewAnswersBucket, view.TakenBucket} { + for _, bucket := range []string{view.HaveBucket, view.AssignmentsBucket, view.PresenceBucket, view.StuckBucket, view.CandidatesBucket, view.AsksBucket, view.AnswersBucket, builds.BuilderBuildsBucket, view.AssignerWantsBucket, view.SourceBucket, view.AssignerAsksBucket, viewstate.ViewAnswersBucket, view.TakenBucket} { go follow(ctx, open, bucket, false, model, logger) } - // The modules' and the source's history give each fingerprint's first commit and each head's source revision (ADR 0556). - // The modules' picture holds no history: from its first view on, each change it takes is the history (finding of DLV-M4). - for _, bucket := range []string{view.ModulesBucket, view.SourceBucket} { - go follow(ctx, open, bucket, true, model, logger) + // The source's history gives each head's source revision (ADR 0556); the modules' history is each revision of the + // catalog half the view takes, in its order (below). + go follow(ctx, open, view.SourceBucket, true, model, logger) + statusBucket := process.AwaitBucket(ctx, js, conn, firstViewsBucket, "view_first_views reads nothing and view_first_view_again is not heard", logger.Printf) + // The modules are the catalog half's, adopted whole by the seat's client: its first view, its status, its hand asks and + // its event catalog-revision (on the module's durable consumer) are the reader's. + catalog, _, err := meshcatalogclient.WireCatalogProcess(reader, calls.Call, "main", catalogChanged(model, logger)) + if err != nil { + return err } - reader.Start(ctx, js, cred, process.AwaitBucket(ctx, js, conn, firstViewsBucket, "view_first_views reads nothing and view_first_view_again is not heard", logger.Printf)) + reader.Start(ctx, js, cred, statusBucket) + catalogHeld(ctx, catalog, model) select { case <-ctx.Done(): case err := <-webUp: @@ -189,7 +196,6 @@ var ownersFollowed = []struct{ owner, state, tool string }{ {view.AssignerModule, "wants", view.AssignerModule + ".assigner_read_wants"}, {view.AssignerModule, "asks", view.AssignerModule + ".assigner_read_asks"}, {view.IndexerModule, "candidates", view.IndexerModule + ".indexer_read_candidates"}, - {view.IndexerModule, "modules", view.IndexerModule + ".indexer_read_modules"}, {view.IndexerModule, "taken", view.IndexerModule + ".indexer_read_taken"}, } @@ -277,3 +283,35 @@ func emitEvent(ctx context.Context, js jetstream.JetStream, cred process.Account _, err = js.PublishMsg(pub, msg, jetstream.WithMsgID(id)) return err } + +// catalogChanged hands one module of a revision the view took to the model, in the form of its value; the modules' +// history is these changes in the order of the revisions. +func catalogChanged(model *view.Model, logger *log.Logger) func(string, json.RawMessage, bool, time.Time) { + return func(module string, entry json.RawMessage, gone bool, at time.Time) { + e := view.Entry{Bucket: view.ModulesBucket, Key: module, At: at, Deleted: gone} + if !gone { + v, err := view.ModuleValue(entry) + if err != nil { + logger.Printf("the catalog entry of %s is not taken: %v", module, err) + return + } + e.Value = v + } + model.Apply(e) + e.History = true + model.Apply(e) + } +} + +// catalogHeld makes the model's pictures of the modules initial once the catalog half's first view is held: the process +// starts at once and waits for the seat's holder, saying so. +func catalogHeld(ctx context.Context, cat *meshcatalogclient.Catalog, model *view.Model) { + go func() { + select { + case <-cat.Follower.Ready(): + model.Initial(view.ModulesBucket) + model.Initial(view.ReachHistory(view.ModulesBucket)) + case <-ctx.Done(): + } + }() +} diff --git a/modules/mesh-delivery-view/internal/view/catalog.go b/modules/mesh-delivery-view/internal/view/catalog.go new file mode 100644 index 00000000..385de245 --- /dev/null +++ b/modules/mesh-delivery-view/internal/view/catalog.go @@ -0,0 +1,58 @@ +package view + +// The view reads the catalog half of the delivery through the seat mesh-catalog (design 47, Two graphs; ADR 0656 §6): +// a first view of its verb `revision`, then each trunk revision its event `catalog-revision` says, never the indexer's +// state `modules`. The model keeps the picture it always kept (one value per module, its history for the reach), so each +// entry of a revision is handed to it in the form of that value. + +import ( + "encoding/json" + "fmt" + + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" +) + +// CatalogSeat is the seat whose verb `revision` and event `catalog-revision` the view reads the modules from, as its +// manifest's `invokes` and `consumes` name them. +const CatalogSeat = meshcatalog.Seat + +// ModulesBucket names the model's picture of the catalog half's modules (and its history): the model's label, no +// bucket of anybody's. +const ModulesBucket = CatalogSeat + "_modules" + +// ModuleValue is the value of one module of the model, as the model decodes it, made from the catalog entry (its JSON): +// the fingerprint is the entry's `fp` (the only form there is), the source revisions are the entry's inputs'. A manifest +// held for unknown fields has no verdict here: the model reads the fingerprint and the commit. +func ModuleValue(entry json.RawMessage) (json.RawMessage, error) { + var e meshcatalog.CatalogEntry + if err := json.Unmarshal(entry, &e); err != nil || e.Module == "" { + return nil, fmt.Errorf("a catalog entry does not read: %v", err) + } + v := meshcatalog.ModulesByModuleValue{Commit: e.Commit, Fp2: e.Fp, None2: e.None, Path: e.Path, Repository: e.Repository, Packages: e.Packages} + if len(e.Manifest) > 0 { + // The entry holds its manifest whole; the model keeps the digest of its bytes, as the value it always kept does. + raw, _ := json.Marshal(e.Manifest) + v.Manifest = meshcatalogclient.ManifestDigest(raw) + } + if e.Verdict == meshcatalogclient.VerdictAccepted || e.Verdict == meshcatalogclient.VerdictRefused { + v.Verdict = e.Verdict + } + if in := e.Inputs; in != nil { + v.Src, v.FpSrc, v.Toolchains, v.Paths, v.Warnings = in.Src, in.FpSrc, in.Toolchains, in.Paths, in.Warnings + if f := in.From; f != nil && v.Manifest != "" { + v.Inputs2 = &meshcatalog.ModulesByModuleValueInputs2{Bases: f.Bases, Builder: f.Builder, Contexts: f.Contexts, + ImportForm: f.ImportForm, Imports: f.Imports, Locks: f.Locks, Packages: f.Packages, Paths: f.Paths, Tree: f.Tree} + } + if v.Verdict == meshcatalogclient.VerdictRefused { + v.Problems = in.Problems + } + } + if v.Verdict == meshcatalogclient.VerdictRefused && len(v.Problems) == 0 { + v.Verdict = "" + } + if v.Verdict != meshcatalogclient.VerdictRefused { + v.Warnings = nil + } + return meshcatalogclient.EncodeModule(v) +} diff --git a/modules/mesh-delivery-view/internal/view/names.go b/modules/mesh-delivery-view/internal/view/names.go index 4a2d62f1..24e69c85 100644 --- a/modules/mesh-delivery-view/internal/view/names.go +++ b/modules/mesh-delivery-view/internal/view/names.go @@ -24,7 +24,6 @@ const ( EngineModule = "node-engine" // The indexer's states the view reads. - ModulesBucket = IndexerModule + "_modules" CandidatesBucket = IndexerModule + "_candidates" TakenBucket = IndexerModule + "_taken" PackagesState = "packages" diff --git a/modules/mesh-delivery-view/manifest_test.go b/modules/mesh-delivery-view/manifest_test.go index b35cf6f8..693fe5d5 100644 --- a/modules/mesh-delivery-view/manifest_test.go +++ b/modules/mesh-delivery-view/manifest_test.go @@ -3,6 +3,7 @@ package manifest import ( "bytes" "encoding/json" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" "io/fs" "mesh-delivery-view/internal/view" "os" @@ -119,8 +120,9 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7 // follows then heard on their views families (ADR 0615 §1, §5; DLV-M4). if want := []string{"mesh-delivery-assigner.assigner_read_asks", "mesh-delivery-assigner.assigner_read_differences", "mesh-delivery-assigner.assigner_read_assignments", "mesh-delivery-assigner.assigner_read_wants", "mesh-delivery-indexer.indexer_read_candidates", "mesh-delivery-indexer.indexer_read_config_candidates", - "mesh-delivery-indexer.indexer_read_graph", "mesh-delivery-indexer.indexer_read_heard", "mesh-delivery-indexer.indexer_pending", "mesh-delivery-indexer.indexer_read_modules", + "mesh-delivery-indexer.indexer_read_graph", "mesh-delivery-indexer.indexer_read_heard", "mesh-delivery-indexer.indexer_pending", "mesh-delivery-indexer.indexer_read_packages", "mesh-delivery-indexer.indexer_read_predefined", "mesh-delivery-indexer.indexer_read_taken", + "seat:mesh-catalog.revision", // ADR 0663: the modules are the catalog half's, read through the seat's verb revision, never the indexer's "seat:mesh-hand-acts.hand-act", // the join: its first view asked again by hand is a hand act (views.StatusTools) "seat:node-engine.manifest-reads"}; !reflect.DeepEqual(m.Invokes, want) { // ADR 0618 §3: node_manifest_reads asks the node-engine seat's verb of every node; 1585/T24: view_tiers asks indexer_pending, a verb edge (ADR 0614 §2) t.Errorf("invokes = %v, want %v", m.Invokes, want) @@ -133,7 +135,7 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7 t.Errorf("version = %q, want 0.1", m.Version) } if want := []string{"mesh-conditions.findings-asked", "mesh-delivery-assigner.views.asks", "mesh-delivery-assigner.views.assignments", "mesh-delivery-assigner.views.wants", - "mesh-delivery-indexer.views.candidates", "mesh-delivery-indexer.views.modules", "mesh-delivery-indexer.views.taken"}; !reflect.DeepEqual(m.Consumes, want) { + "mesh-catalog.catalog-revision", "mesh-delivery-indexer.views.candidates", "mesh-delivery-indexer.views.taken"}; !reflect.DeepEqual(m.Consumes, want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } // The one event the view says: a source revision is delivered (1906/D9, ADR 0539 §17), derived by it from the wants and the haves. @@ -180,7 +182,7 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7 if !slices.Contains(m.Reads, view.SignersRead) { t.Errorf("the view reads %s, its manifest's reads are %v", view.SignersRead, m.Reads) } - followed := map[string]string{view.ModulesBucket: view.IndexerModule + ".views.modules", view.CandidatesBucket: view.IndexerModule + ".views.candidates", + followed := map[string]string{view.CandidatesBucket: view.IndexerModule + ".views.candidates", view.TakenBucket: view.IndexerModule + ".views.taken", view.AssignmentsBucket: view.AssignerModule + ".views.assignments", view.AssignerWantsBucket: view.AssignerModule + ".views.wants", view.AssignerAsksBucket: view.AssignerModule + ".views.asks"} for bucket, family := range followed { @@ -191,11 +193,20 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7 t.Errorf("the view follows %s, its manifest consumes %v", family, m.Consumes) } } - for _, tool := range []string{view.IndexerModule + ".indexer_read_modules", view.IndexerModule + ".indexer_read_candidates", view.IndexerModule + ".indexer_pending", view.AssignerModule + ".assigner_read_wants"} { + for _, tool := range []string{meshcatalogclient.RevisionVerb, view.IndexerModule + ".indexer_read_candidates", view.IndexerModule + ".indexer_pending", view.AssignerModule + ".assigner_read_wants"} { if !slices.Contains(m.Invokes, tool) { t.Errorf("the view asks %s, its manifest invokes %v", tool, m.Invokes) } } + // The modules are the catalog half's: the seat's verb is invoked, its event consumed, and the indexer's state modules is neither. + if !slices.Contains(m.Consumes, view.CatalogSeat+"."+meshcatalog.EventCatalogRevision) { + t.Errorf("the view hears the seat's %s, its manifest consumes %v", meshcatalog.EventCatalogRevision, m.Consumes) + } + for _, name := range append(slices.Clone(m.Invokes), m.Consumes...) { + if strings.Contains(name, "indexer_read_modules") || strings.Contains(name, "views.modules") { + t.Errorf("the manifest names %s: the modules are the catalog half's (ADR 0663)", name) + } + } if viewstate.ViewModule != m.Module || viewstate.ViewAnswersBucket != m.Module+"_answers" || viewstate.ReachBucket != m.Module+"_reach" { t.Errorf("the view names itself %s (%s, %s), its manifest %s", viewstate.ViewModule, viewstate.ViewAnswersBucket, viewstate.ReachBucket, m.Module) } diff --git a/modules/mesh-delivery-view/module.json b/modules/mesh-delivery-view/module.json index a8544e42..9e727992 100644 --- a/modules/mesh-delivery-view/module.json +++ b/modules/mesh-delivery-view/module.json @@ -46,8 +46,8 @@ "mesh-delivery-assigner.views.asks", "mesh-delivery-assigner.views.assignments", "mesh-delivery-assigner.views.wants", + "mesh-catalog.catalog-revision", "mesh-delivery-indexer.views.candidates", - "mesh-delivery-indexer.views.modules", "mesh-delivery-indexer.views.taken" ], "streams": [ @@ -68,10 +68,10 @@ "mesh-delivery-indexer.indexer_read_graph", "mesh-delivery-indexer.indexer_read_heard", "mesh-delivery-indexer.indexer_pending", - "mesh-delivery-indexer.indexer_read_modules", "mesh-delivery-indexer.indexer_read_packages", "mesh-delivery-indexer.indexer_read_predefined", "mesh-delivery-indexer.indexer_read_taken", + "seat:mesh-catalog.revision", "seat:mesh-hand-acts.hand-act", "seat:node-engine.manifest-reads" ],