From 839bc4a6e5cbfe3866badcc090f559d3a7cec645 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 11:44:42 +0100 Subject: [PATCH 01/33] TL0: the teller's test follows the contracts, the settle ask gone with the verb settle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §431 (the settle ask goes with the verb settle) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-teller/teller.go | 4 ++-- .../cmd/mesh-delivery-teller/teller_test.go | 9 ++++----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/teller.go b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/teller.go index 83506c23..4426cb51 100644 --- a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/teller.go +++ b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/teller.go @@ -324,8 +324,8 @@ func (t *Teller) catchUp(id string) { t.said(h, *w, err) } -// mergeAct is what the approve option of the gate's ask binds: the ask's verb (meshdeliveryclient.AskVerb: mesh-delivery.merge, or -// mesh-delivery.settle for a person's settle ask, novox/hq ADR 0556 clause 5), this ask, the answer, and the heads it covers. +// mergeAct is what the approve option of the gate's ask binds: the ask's verb (meshdeliveryclient.AskVerb: mesh-delivery.merge; the settle ask +// went with the verb settle, design 47 G §431), this ask, the answer, and the heads it covers. // It is spelled as the gate composes it (mesh-catalog modules/mesh-delivery-gate approval.go mergeAct, from // modules/mesh-delivery approval.go:63-65) and as mesh-controller verifies it (cmd/mesh-controller/moved.go:98-106). func mergeAct(verb, ask, option string, heads map[string]string) occ.Act { diff --git a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/teller_test.go b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/teller_test.go index 34758645..dac5305b 100644 --- a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/teller_test.go +++ b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/teller_test.go @@ -412,16 +412,15 @@ func TestAWarrantThatHoldsWritesOneAnswerAndASecondWritesNone(t *testing.T) { // if len(b.conds) != 0 { t.Errorf("conditions said: %+v", b.conds) } - // 1280 (ADR 0556 clause 5): a settle ask holds when its options bind mesh-delivery.settle, and not when they bind a merge. - for verb, holds := range map[string]bool{"mesh-delivery.settle": true, "mesh-delivery.merge": false} { + // design 47 G §431: the settle ask went with the verb settle, so an ask whose options bind it holds no more. + for verb, holds := range map[string]bool{"mesh-delivery.settle": false, "mesh-delivery.merge": true} { b := newBed(t) - group := meshdeliveryclient.SettleGroup([]string{"novox/mesh-delivery@" + headA[:12]}) - key, value, ga := gateAskOf(t, group, verb, twoHeads(), twoHeads(), 1) + key, value, ga := gateAskOf(t, "g", verb, twoHeads(), twoHeads(), 1) b.gate[key] = value b.t.OnAsk(key, 1, value) b.deliver(t, warrantFor(ga.Ask, "approve")) if _, written := b.answers.written[answerKey(key)]; written != holds { - t.Errorf("a settle ask binding %s: answer written %v, want %v", verb, written, holds) + t.Errorf("an ask binding %s: answer written %v, want %v", verb, written, holds) } } } From 9b9e9f68bb538c4938adb036c50981d5a696d927 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 11:48:34 +0100 Subject: [PATCH 02/33] TL4: the teller's manifest says version 0.1, declares its finding stream and hears the keeper's ask MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §383, §387, §368, §411 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- modules/mesh-delivery-teller/manifest_test.go | 22 ++++++++++++++++--- modules/mesh-delivery-teller/module.json | 11 +++++++++- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/modules/mesh-delivery-teller/manifest_test.go b/modules/mesh-delivery-teller/manifest_test.go index 829136df..953960be 100644 --- a/modules/mesh-delivery-teller/manifest_test.go +++ b/modules/mesh-delivery-teller/manifest_test.go @@ -23,6 +23,13 @@ type state struct { History int `json:"history,omitempty"` } +type stream struct { + Name string `json:"name"` + Events []string `json:"events"` + MaxAgeDays int `json:"max-age-days"` + MaxMiB int `json:"max-mib"` +} + type artifact struct { Name string `json:"name"` Kind string `json:"kind"` @@ -44,6 +51,7 @@ type manifest struct { Invokes []string `json:"invokes"` Consumes []string `json:"consumes"` Emits []string `json:"emits"` + Streams []stream `json:"streams"` OwnSecrets map[string]string `json:"own-secrets"` SecretsOwner string `json:"secrets-owner"` Resources []map[string]any `json:"resources"` @@ -102,7 +110,8 @@ func TestManifestDeclarations(t *testing.T) { // 1240/AC1 if want := []string{"mesh-delivery-assigner.assigner_read_asks", "seat:mesh-hand-acts.hand-act"}; !reflect.DeepEqual(m.Invokes, want) { t.Errorf("invokes = %v, want %v", m.Invokes, want) } - if want := []string{"mesh-delivery-assigner.views.asks"}; !reflect.DeepEqual(m.Consumes, want) { + // design 47 G §387: the keeper's ask for the findings is heard, and the finding stream is declared (§383). + if want := []string{"mesh-conditions.findings-asked", "mesh-delivery-assigner.views.asks"}; !reflect.DeepEqual(m.Consumes, want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } if want := []string{"operator-channel"}; !reflect.DeepEqual(sorted(m.Uses), want) { @@ -122,8 +131,15 @@ func TestManifestDeclarations(t *testing.T) { // 1240/AC1 if want := []string{tellerevents.AskEnded}; !reflect.DeepEqual(m.Emits, want) { t.Errorf("emits = %v, want %v", m.Emits, want) } - // No streams, bus or seat (ADR 0437 §4). - for _, k := range []string{"streams", "logs", "claims", "provides", "requires", "defines-seats", "seat", "seats", "bus"} { + // design 47 G §383, §387: the stream -findings, 7 days and 16 MiB; §368: version is MAJOR.MINOR. + if want := []stream{{Name: "mesh-delivery-teller-findings", Events: []string{"finding-written.*", "finding-cleared.*", "findings-stated.*"}, MaxAgeDays: 7, MaxMiB: 16}}; !reflect.DeepEqual(m.Streams, want) { + t.Errorf("streams = %+v, want %+v", m.Streams, want) + } + if m.Version != "0.1" { + t.Errorf("version = %q, want 0.1", m.Version) + } + // No bus or seat (ADR 0437 §4). + for _, k := range []string{"logs", "claims", "provides", "requires", "defines-seats", "seat", "seats", "bus"} { if _, ok := top[k]; ok { t.Errorf("the manifest declares %q: the teller has none", k) } diff --git a/modules/mesh-delivery-teller/module.json b/modules/mesh-delivery-teller/module.json index c6e1a80c..cc176fd7 100644 --- a/modules/mesh-delivery-teller/module.json +++ b/modules/mesh-delivery-teller/module.json @@ -1,6 +1,6 @@ { "module": "mesh-delivery-teller", - "version": "1", + "version": "0.1", "slug": "teller", "tools": [ "teller_end_ask", @@ -12,8 +12,17 @@ "seat:mesh-hand-acts.hand-act" ], "consumes": [ + "mesh-conditions.findings-asked", "mesh-delivery-assigner.views.asks" ], + "streams": [ + { + "name": "mesh-delivery-teller-findings", + "events": ["finding-written.*", "finding-cleared.*", "findings-stated.*"], + "max-age-days": 7, + "max-mib": 16 + } + ], "emits": [ "ask-ended" ], From 3a4768ab43768b6460395a134ccaad93a9f7ae91 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 11:48:36 +0100 Subject: [PATCH 03/33] FB6a: the forge bridge reads trees, files, pulls and tags through the seat git and hears its pull and delete events MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Taken over from the join's 473f9323, 5f40d230, 159a679b; the events are the seat's pull-updated, pull-merged, pull-closed and forge-delete. Builds: design 47 G §252, §310, §312, §317 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../mesh-delivery-forge-bridge-tools/main.go | 29 +-- .../main_test.go | 62 +++-- .../cmd/mesh-delivery-forge-bridge/call.go | 30 ++- .../cmd/mesh-delivery-forge-bridge/main.go | 4 +- .../mesh-delivery-forge-bridge/main_test.go | 120 +++++----- .../cmd/mesh-delivery-forge-bridge/open.go | 9 +- .../mesh-delivery-forge-bridge/open_test.go | 17 +- modules/mesh-delivery-forge-bridge/go.mod | 1 + .../internal/bridge/bridge.go | 162 +++++++------ .../internal/bridge/bridge_test.go | 214 ++++++++++-------- .../internal/bridge/call.go | 105 +++++---- .../internal/bridge/forgemade.go | 67 ++---- .../internal/bridge/tags.go | 66 ++---- .../internal/gitfixture/gitfixture.go | 42 ++++ .../manifest_test.go | 9 +- .../mesh-delivery-forge-bridge/module.json | 17 +- 16 files changed, 537 insertions(+), 417 deletions(-) create mode 100644 modules/mesh-delivery-forge-bridge/internal/gitfixture/gitfixture.go diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main.go index 0309cf12..d123a475 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main.go @@ -1,6 +1,6 @@ // Command mesh-delivery-forge-bridge-tools serves the forge bridge's tools (novox/hq ADR 0633 §4, issue 1585; design 51): -// the runtime launches it for a call. bridge_tags reads a package's git tags from the forge through the forge module's -// gitea_api, for the builder's floor. bridge_groups reads the delivery groups the gate holds and what the bridge kept of their ends; +// the runtime launches it for a call. bridge_tags reads a package's git tags through the seat git's verb tags, for the +// builder's floor. bridge_groups reads the delivery groups the gate holds and what the bridge kept of their ends; // bridge_group_end_again writes an ask into the bridge's state end-asks and waits for the bridge's answer there, since the // bridge is the other program and serves no request subject (ADR 0437 §4). package main @@ -43,18 +43,18 @@ const ( type asker func(key string, value []byte) (groups.EndAsk, error) // Tools are the bridge's tools: groupsRead reads the groups the gate holds with what the bridge kept of them. -func Tools(ask asker, now func() time.Time, groupsRead func() ([]groups.Row, []string, error), merged func(repository, merge string, number int) (json.RawMessage, bool, error), api bridge.API) []stdio.Tool { +func Tools(ask asker, now func() time.Time, groupsRead func() ([]groups.Row, []string, error), merged func(repository, merge string, number int) (json.RawMessage, bool, error), forge bridge.Caller) []stdio.Tool { str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) } return []stdio.Tool{ {Name: bridge.TagsTool, Description: "A package's git tags on the forge, for the builder's floor (design 51): every tag of repository whose name starts " + - "with prefix (go/v for a Go package in the go directory, v for one at the root), sorted. Read-only; read from the forge " + - "through the forge module's gitea_api, at most " + fmt.Sprint(bridge.TagsMaxPages*bridge.TagsPageSize) + " tags (more says so).", + "with prefix (go/v for a Go package in the go directory, v for one at the root), sorted. Read-only; read through the seat " + + "git's verb tags, at most " + fmt.Sprint(bridge.TagsMaxPages) + " pages (more says so).", Input: map[string]any{"type": "object", "required": []string{"repository"}, "properties": map[string]any{ "repository": map[string]any{"type": "string", "description": "/"}, "prefix": map[string]any{"type": "string", "description": "only tags starting with it"}}}, Run: func(args map[string]any) (any, error) { - return bridge.Tags(api, str(args, "repository"), str(args, "prefix")) + return bridge.Tags(context.Background(), forge, str(args, "repository"), str(args, "prefix")) }}, {Name: MergedTool, Description: "The merged message of one merge, asked again (issue 1585): the message the bridge wrote on its stream FORGE when the " + @@ -135,10 +135,10 @@ func Tools(ask asker, now func() time.Time, groupsRead func() ([]groups.Row, []s // (bridge.MergedMessage), and, for a member of a delivery group, the group and its members as the process sets them. // // When the gate holds no merge with that commit, the merge was made on the forge (a recovery merge, the operator's own): -// the pull request is read from the forge through api (by number when given, else by the merge commit), the message is +// the pull request is read through the seat git's verb pull (by number when given, else by the merge commit), the message is // built the same way, and madeOnForge is true, for the answer and its log line to say. It is a lone pull request: no // group is set. -func MergedOf(raws map[string][]byte, repository, merge string, number int, trees bridge.Trees, files bridge.Files, api bridge.API) (out json.RawMessage, madeOnForge bool, err error) { +func MergedOf(raws map[string][]byte, repository, merge string, number int, trees bridge.Trees, files bridge.Files, forge bridge.Caller) (out json.RawMessage, madeOnForge bool, err error) { owner, repo, ok := strings.Cut(repository, "/") if !ok || owner == "" || repo == "" { return nil, false, fmt.Errorf("%q is not owner/name", repository) @@ -169,10 +169,10 @@ func MergedOf(raws map[string][]byte, repository, merge string, number int, tree out, err = forgemsg.EncodeForge(f) return out, false, err } - if api == nil { + if forge == nil { return nil, false, fmt.Errorf("the gate's merges hold no merge of %s with the commit %s, so its pull request is not known", repository, merge) } - fp, err := bridge.ForgeMade(api, owner, repo, number, merge) + fp, err := bridge.ForgeMade(context.Background(), forge, owner, repo, number, merge) if err != nil { return nil, false, fmt.Errorf("the gate's merges hold no merge of %s with the commit %s, and the forge does not give it as a merged pull request either: %w", repository, merge, err) } @@ -282,11 +282,6 @@ func askCaller(_ context.Context, key string, body any) (json.RawMessage, error) return stdio.Ask(key, body) } -// forgeAPI is one GET of the forge's API through the forge module's gitea_api, the path the bridge reads the forge by. -func forgeAPI(path string) (json.RawMessage, error) { - return stdio.Ask("gitea.gitea_api", map[string]any{"path": path, "method": "GET"}) -} - func main() { merges, performed, asks := stdio.State(gateMergesState), stdio.State(groups.PerformedState), stdio.State(groups.EndAsksState) tools := Tools(askEnd(asks, 25*time.Second), time.Now, @@ -303,8 +298,8 @@ func main() { } } ctx := context.Background() - return MergedOf(raws, repository, merge, number, bridge.TreesVia(ctx, askCaller), bridge.FilesVia(ctx, askCaller), forgeAPI) - }, forgeAPI) + return MergedOf(raws, repository, merge, number, bridge.TreesVia(ctx, askCaller), bridge.FilesVia(ctx, askCaller), askCaller) + }, askCaller) if err := stdio.Serve("", tools); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-forge-bridge-tools] %v\n", err) os.Exit(1) diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main_test.go index 946eece1..616d23ea 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main_test.go @@ -2,6 +2,7 @@ package main import ( "bytes" + "context" "encoding/json" "errors" "strings" @@ -9,11 +10,13 @@ import ( "time" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" + git "git.novox.be/novox/mesh-delivery/contracts/go/seats/git" seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "mesh-delivery-forge-bridge/internal/bridge" + "mesh-delivery-forge-bridge/internal/gitfixture" "mesh-delivery-forge-bridge/internal/groups" ) @@ -136,7 +139,7 @@ func TestTheBridgeToolsReadTheGroupsAndAskTheEndAgain(t *testing.T) { return nil, bridge.ErrNotFound } published := func(repo string, number int, head, branch, merge string) forgemsg.Forge { - body, _ := json.Marshal(map[string]any{"owner": "novox", "repo": repo, "number": number, "head": branch, "head_sha": head, "base": "main", "merge_commit_sha": merge}) + body, _ := json.Marshal(git.PullMergedEvent{Repository: "novox/" + repo, Number: int64(number), Branch: branch, Head: head, Base: "main", MergeCommit: merge}) msg, err := bridge.Convert(bridge.MergedSubject, body, trees, files) if err != nil { t.Fatal(err) @@ -158,7 +161,7 @@ func TestTheBridgeToolsReadTheGroupsAndAskTheEndAgain(t *testing.T) { repository string merge string number int - api bridge.API + api bridge.Caller onForge bool want forgemsg.Forge fail string @@ -173,23 +176,25 @@ func TestTheBridgeToolsReadTheGroupsAndAskTheEndAgain(t *testing.T) { // Issue 1966: a merge made on the forge (a recovery merge) has no merge in the gate; the bridge reads the pull request // from the forge, by number or by the merge commit, and checks its merge commit. {name: "a merge made on the forge, by number", raws: map[string][]byte{"solo": lone}, repository: "novox/a", merge: c2, number: 9, onForge: true, - api: forgeAnswers(t, "/repos/novox/a/pulls/9", true, "fix/x", h2, c2, "main"), want: published("a", 9, h2, "fix/x", c2)}, + api: forgeAnswers(t, 9, "", "merged", "fix/x", h2, c2, "main"), want: published("a", 9, h2, "fix/x", c2)}, {name: "a merge made on the forge, by the merge commit", raws: map[string][]byte{}, repository: "novox/a", merge: c2, onForge: true, - api: forgeAnswers(t, "/repos/novox/a/commits/"+c2+"/pull", true, "fix/x", h2, c2, "main"), want: published("a", 9, h2, "fix/x", c2)}, + api: forgeAnswers(t, 0, c2, "merged", "fix/x", h2, c2, "main"), want: published("a", 9, h2, "fix/x", c2)}, {name: "a forge pull request merged at another commit", raws: map[string][]byte{}, repository: "novox/a", merge: c2, number: 9, - api: forgeAnswers(t, "/repos/novox/a/pulls/9", true, "fix/x", h2, c1, "main"), fail: "not at " + c2}, + api: forgeAnswers(t, 9, "", "merged", "fix/x", h2, c1, "main"), fail: "not at " + c2}, {name: "a forge pull request not merged", raws: map[string][]byte{}, repository: "novox/a", merge: c2, number: 9, - api: forgeAnswers(t, "/repos/novox/a/pulls/9", false, "fix/x", h2, "", "main"), fail: "not merged"}, + api: forgeAnswers(t, 9, "", "open", "fix/x", h2, "", "main"), fail: "not merged"}, {name: "a forge pull request merged into another branch", raws: map[string][]byte{}, repository: "novox/a", merge: c2, number: 9, - api: forgeAnswers(t, "/repos/novox/a/pulls/9", true, "fix/x", h2, c2, "dev"), fail: "not the trunk"}, + api: forgeAnswers(t, 9, "", "merged", "fix/x", h2, c2, "dev"), fail: "not the trunk"}, {name: "a forge pull request with no head branch", raws: map[string][]byte{}, repository: "novox/a", merge: c2, number: 9, - api: forgeAnswers(t, "/repos/novox/a/pulls/9", true, "refs/pull/9/head", h2, c2, "main"), fail: "no head branch"}, - {name: "a forge answer that is the bare body, no wrapper", raws: map[string][]byte{}, repository: "novox/a", merge: c2, number: 9, - api: func(string) (json.RawMessage, error) { return json.RawMessage(`{"number":9,"merged":true}`), nil }, fail: `{"result"`}, - {name: "a forge answer with no pull request number", raws: map[string][]byte{}, repository: "novox/a", merge: c2, number: 9, - api: func(string) (json.RawMessage, error) { return json.RawMessage(`{"result":{"merged":true}}`), nil }, fail: "no number"}, + api: forgeAnswers(t, 9, "", "merged", "", h2, c2, "main"), fail: "no head branch"}, + {name: "a seat answer that says an error", raws: map[string][]byte{}, repository: "novox/a", merge: c2, number: 9, + api: answering(gitfixture.Given(t, "pull answers an error, never none", "verb:pull.answer")), fail: "holds no pull request"}, + {name: "a seat answer with no pull request", raws: map[string][]byte{}, repository: "novox/a", merge: c2, number: 9, + api: answering(json.RawMessage(`{}`)), fail: "carries nothing"}, {name: "a forge that does not answer", raws: map[string][]byte{}, repository: "novox/a", merge: c2, - api: func(string) (json.RawMessage, error) { return nil, errors.New("the forge is down") }, fail: "the forge is down"}, + api: func(context.Context, string, any) (json.RawMessage, error) { + return nil, errors.New("the forge is down") + }, fail: "the forge is down"}, } { t.Run(c.name, func(t *testing.T) { raw, onForge, err := MergedOf(c.raws, c.repository, c.merge, c.number, trees, files, c.api) @@ -216,21 +221,28 @@ func TestTheBridgeToolsReadTheGroupsAndAskTheEndAgain(t *testing.T) { }) } -// forgeAnswers is the forge's gitea_api for one GET path: that pull request as the forge says it; any other path is not found. -func forgeAnswers(t *testing.T, path string, merged bool, branch, head, merge, base string) bridge.API { +// forgeAnswers is the seat git's verb pull for one pull request, asked by number (when number is given) or by the merge +// commit asked: the seat's own fixture with that pull request's state, branch, head, merge commit and base; any other +// ask is refused as the seat refuses it. +func forgeAnswers(t *testing.T, number int, byMerge, state, branch, head, merge, base string) bridge.Caller { t.Helper() - body, _ := json.Marshal(map[string]any{"number": 9, "merged": merged, "merge_commit_sha": merge, - "head": map[string]any{"ref": "refs/pull/9/head", "label": branch, "sha": head}, "base": map[string]any{"ref": base}}) - return func(p string) (json.RawMessage, error) { - if p != path { - return nil, errors.New("the forge has no " + p) + var a git.PullAnswer + if err := json.Unmarshal(gitfixture.Given(t, "pull answers", "verb:pull.answer"), &a); err != nil || a.Pull == nil { + t.Fatalf("the seat's pull fixture: %v", err) + } + p := *a.Pull + p.Repository, p.Number, p.State, p.Branch, p.Head, p.MergeCommit, p.Base = "novox/a", 9, state, branch, head, merge, base + answer, _ := json.Marshal(git.PullAnswer{Pull: &p}) + return func(_ context.Context, key string, body any) (json.RawMessage, error) { + in, ok := body.(git.PullInput) + if key != "seat:git.pull" || !ok || in.Repository != "novox/a" || int(in.Number) != number || in.MergeCommit != byMerge { + return json.RawMessage(`{"error":"the forge holds no such pull request"}`), nil } - return wrapped(body), nil + return answer, nil } } -// wrapped is gitea_api's answer for a body: {"result": } (mesh-catalog modules/gitea/tools/index.ts). -func wrapped(body []byte) json.RawMessage { - b, _ := json.Marshal(map[string]json.RawMessage{"result": body}) - return b +// answering is a seat that answers raw to every ask. +func answering(raw json.RawMessage) bridge.Caller { + return func(context.Context, string, any) (json.RawMessage, error) { return raw, nil } } diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/call.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/call.go index abf7f86b..fae52344 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/call.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/call.go @@ -19,7 +19,7 @@ const callTimeout = 30 * time.Second // caller calls a module's tool by its manifest key (`.`) with a JSON body and returns the tool's result. type caller func(ctx context.Context, key string, body any) (json.RawMessage, error) -// callerOn calls tools as the bridge's own principal. It copies mesh-sdk go/process/call.go (Call, callSubjects, +// callerOn calls tools and seat verbs as the bridge's own principal. It copies mesh-sdk go/process/call.go (Call, callSubjects, // request; hand-written here until the SDK's helper is used, issue 1242): the request is the tool's arguments as // JSON, published on the caller-named subject `mesh.mod..call..`, or on the subject that // names no caller, `mesh.mod..tool.`, when nothing answers the first; the reply is @@ -49,22 +49,40 @@ func callerOn(nc *nats.Conn, user string) caller { return r.Result, nil } return func(ctx context.Context, key string, body any) (json.RawMessage, error) { - module, tool, ok := strings.Cut(key, ".") - if !ok || module == "" || tool == "" { - return nil, fmt.Errorf("%q is not .", key) + called, fallback, err := callSubjects(key, token) + if err != nil { + return nil, err } data, err := json.Marshal(body) if err != nil { return nil, err } - result, err := request(ctx, "mesh.mod."+module+".call."+tool+"."+token, data) + result, err := request(ctx, called, data) if errors.Is(err, nats.ErrNoResponders) { - result, err = request(ctx, "mesh.mod."+module+".tool."+tool, data) + result, err = request(ctx, fallback, data) } return result, err } } +// callSubjects is the caller-named subject a key is called on and its plain fallback: a seat's verb +// (`seat:.`) on mesh.seat..call.., else a module's tool (`.`) on +// mesh.mod..call.. (mesh-sdk go/process/call.go callSubjectsOn). +func callSubjects(key, token string) (called, fallback string, err error) { + if rest, isSeat := strings.CutPrefix(key, "seat:"); isSeat { + seat, verb, ok := strings.Cut(rest, ".") + if !ok || seat == "" || verb == "" { + return "", "", fmt.Errorf("%q is not seat:.", key) + } + return "mesh.seat." + seat + ".call." + verb + "." + token, "mesh.seat." + seat + ".tool." + verb, nil + } + module, tool, ok := strings.Cut(key, ".") + if !ok || module == "" || tool == "" { + return "", "", fmt.Errorf("%q is not .", key) + } + return "mesh.mod." + module + ".call." + tool + "." + token, "mesh.mod." + module + ".tool." + tool, nil +} + // treesVia asks the forge for a commit's trees through call. func treesVia(ctx context.Context, call caller) bridge.Trees { return bridge.TreesVia(ctx, bridge.Caller(call)) diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go index e412765d..19ad49af 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go @@ -21,6 +21,7 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + git "git.novox.be/novox/mesh-delivery/contracts/go/seats/git" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "mesh-delivery-forge-bridge/internal/bridge" @@ -197,7 +198,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // The module's one pull of its durable consumer on the shared events stream is its event hub; the bridge has the one // handler, for every event its consumes filter lets through. Essential: the forge's events are what the bridge carries. hub := process.NewEventHub(logger.Printf) - if err := hub.Handle("mesh.mod.*.event.>", func(m jetstream.Msg) { + // The forge's events are the seat git's (novox/hq design 47 *The seat `git`*), whichever module holds it. + if err := hub.Handle("mesh.seat."+git.Seat+".event.>", func(m jetstream.Msg) { hear(&turn, func() { // Bounded: the turn is held, and no sync or other event goes on without it. hctx, cancel := context.WithTimeout(ctx, syncDeadline) diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go index ede623fd..865a7c88 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go @@ -13,6 +13,7 @@ import ( "testing" "time" + git "git.novox.be/novox/mesh-delivery/contracts/go/seats/git" "git.novox.be/novox/mesh-sdk/go/process" "github.com/nats-io/nats.go" @@ -21,6 +22,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "mesh-delivery-forge-bridge/internal/bridge" + "mesh-delivery-forge-bridge/internal/gitfixture" ) const head = "0123456789abcdef0123456789abcdef01234567" @@ -53,18 +55,14 @@ func (p *fakePub) Publish(_ context.Context, subject string, payload []byte, _ . return &jetstream.PubAck{}, nil } -// body is gitea.pull.updated as mesh-catalog modules/gitea/trunk.ts:233-250 writes it (the fields the bridge reads). +// body is the seat git's pull.updated (contracts/seats/git), the head head_sha laid over it when given. func body(m map[string]any) []byte { - base := map[string]any{"owner": "novox", "repo": "mesh-delivery", "number": 12, "head": "feat/x", "head_sha": head, - "module_dirs": []string{"modules/a"}, "module_dirs_said": true} - for k, v := range m { - if v == nil { - delete(base, k) - } else { - base[k] = v - } + e := map[string]any{"repository": "novox/mesh-delivery", "number": 12, "branch": "feat/x", "head": head, "base": "main", + "change": map[string]any{"paths": []string{"modules/a/x.go"}, "module-dirs": []string{"modules/a"}}} + if h, ok := m["head_sha"]; ok { + e["head"] = h } - b, _ := json.Marshal(base) + b, _ := json.Marshal(e) return b } @@ -80,7 +78,7 @@ func TestReadableEventIsWrittenOnForgeAndAcked(t *testing.T) { // 1235/AC2 (unit } }) var logs bytes.Buffer - m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(nil)}, &fakePub{} + m, pub := &fakeMsg{subject: git.SubjectPullUpdated, data: body(nil)}, &fakePub{} handle(context.Background(), m, pub, nil, someFiles, nil, log.New(&logs, "", 0)) if len(pub.subjects) != 1 || pub.subjects[0] != "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.pull" { t.Fatalf("published on %v", pub.subjects) @@ -96,7 +94,7 @@ func TestReadableEventIsWrittenOnForgeAndAcked(t *testing.T) { // 1235/AC2 (unit func TestUnreadableEventIsTerminatedAndLogged(t *testing.T) { // 1235/AC5 var logs bytes.Buffer - m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(map[string]any{"head_sha": "short"})}, &fakePub{} + m, pub := &fakeMsg{subject: git.SubjectPullUpdated, data: body(map[string]any{"head_sha": "short"})}, &fakePub{} handle(context.Background(), m, pub, nil, someFiles, nil, log.New(&logs, "", 0)) if len(pub.subjects) != 0 { t.Errorf("a message was written: %v", pub.subjects) @@ -104,13 +102,13 @@ func TestUnreadableEventIsTerminatedAndLogged(t *testing.T) { // 1235/AC5 if m.termd != 1 || m.acked != 0 { t.Errorf("terminated %d acked %d, want terminated once", m.termd, m.acked) } - if !strings.Contains(logs.String(), "mesh.mod.gitea.event.pull.updated") || !strings.Contains(logs.String(), "head") { + if !strings.Contains(logs.String(), git.SubjectPullUpdated) || !strings.Contains(logs.String(), "head") { t.Errorf("log %q names neither the subject nor why", logs.String()) } // 1235: an unwritten message is not acked (a failed write is redelivered by the server, not retried here); merged here // from a test of its own, for the test budget. logs.Reset() - m, pub = &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(nil)}, &fakePub{err: errors.New("no stream")} + m, pub = &fakeMsg{subject: git.SubjectPullUpdated, data: body(nil)}, &fakePub{err: errors.New("no stream")} handle(context.Background(), m, pub, nil, someFiles, nil, log.New(&logs, "", 0)) if m.acked != 0 || m.termd != 0 { t.Errorf("acked %d terminated %d, want neither", m.acked, m.termd) @@ -120,16 +118,16 @@ func TestUnreadableEventIsTerminatedAndLogged(t *testing.T) { // 1235/AC5 } } -// mergedEvent is gitea.pull.merged as mesh-catalog modules/gitea/index.ts:186-207 writes it (the fields the bridge reads). +// mergedEvent is the seat git's pull.merged. func mergedEvent() []byte { - b, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-delivery", "number": 12, "head": "feat/x", "head_sha": head, - "base": "main", "merge_commit_sha": "89abcdef0123456789abcdef0123456789abcdef"}) + b, _ := json.Marshal(map[string]any{"repository": "novox/mesh-delivery", "number": 12, "branch": "feat/x", "head": head, + "base": "main", "merge-commit": "89abcdef0123456789abcdef0123456789abcdef"}) return b } func TestTreesRefusedLeavesTheEventForTenSecondsAndLogs(t *testing.T) { // 1236/AC3 var logs bytes.Buffer - m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.merged", data: mergedEvent()}, &fakePub{} + m, pub := &fakeMsg{subject: git.SubjectPullMerged, data: mergedEvent()}, &fakePub{} call := func(context.Context, string, any) (json.RawMessage, error) { return nil, errors.New("the tree listing of novox/mesh-delivery was cut") } @@ -147,33 +145,36 @@ func TestTreesRefusedLeavesTheEventForTenSecondsAndLogs(t *testing.T) { // 1236/ } } -func TestMergeCallsTheToolOnceAndWritesTheMergedMessage(t *testing.T) { // 1236/AC1 (through the tool's request and answer) +func TestMergeCallsTheToolOnceAndWritesTheMergedMessage(t *testing.T) { // 1236/AC1; design 47 *The seat git* (verbs trees, file) var logs bytes.Buffer - m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.merged", data: mergedEvent()}, &fakePub{} + m, pub := &fakeMsg{subject: git.SubjectPullMerged, data: mergedEvent()}, &fakePub{} var keys []string var args []any - var mu sync.Mutex // the files are asked for concurrently (issue 1458) + var mu sync.Mutex // the files are asked for concurrently (issue 1458) + trees := gitfixture.Given(t, "trees answers", "verb:trees.answer") // root ddd…, "" ddd…, contracts eee… + file := gitfixture.Given(t, "file answers", "verb:file.answer") call := func(_ context.Context, key string, body any) (json.RawMessage, error) { mu.Lock() keys, args = append(keys, key), append(args, body) mu.Unlock() - if key == "gitea.gitea_get_file" { // the manifests, 1448; the trees are what this test is about - return json.RawMessage(`{"file":{"path":"p","sha":"0123","size":2,"content":"{}"}}`), nil + if key == "seat:git.file" { // the manifests, 1448; the trees are what this test is about + return file, nil } - return json.RawMessage(`{"root":"r0","trees":{"":"t0","modules/a":"t1"}}`), nil + return trees, nil } handle(context.Background(), m, pub, treesVia(context.Background(), call), filesVia(context.Background(), call), nil, log.New(&logs, "", 0)) - if len(keys) == 0 || keys[0] != "gitea.gitea_commit_trees" || slices.Index(keys[1:], "gitea.gitea_commit_trees") >= 0 { + if len(keys) == 0 || keys[0] != "seat:git.trees" || slices.Index(keys[1:], "seat:git.trees") >= 0 { t.Fatalf("calls %v, want the trees once and first", keys) } - if got, _ := json.Marshal(args[0]); string(got) != `{"owner":"novox","repo":"mesh-delivery","sha":"89abcdef0123456789abcdef0123456789abcdef"}` { + if got, _ := json.Marshal(args[0]); !sameJSON(got, []byte(`{"repository":"novox/mesh-delivery","commit":"89abcdef0123456789abcdef0123456789abcdef"}`)) { t.Errorf("arguments %s", got) } if len(pub.subjects) != 1 || pub.subjects[0] != "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.merged" { t.Fatalf("published on %v", pub.subjects) } f, err := forgemsg.DecodeForge(pub.data[0]) - if err != nil || f.Root != "r0" || f.Trees[""] != "t0" || f.Trees["modules/a"] != "t1" || f.Merge == "" { + d, e := strings.Repeat("d", 40), strings.Repeat("e", 40) + if err != nil || f.Root != d || f.Trees[""] != d || f.Trees["contracts"] != e || f.Merge == "" { t.Errorf("message %+v err %v", f, err) } if m.acked != 1 { @@ -181,60 +182,65 @@ func TestMergeCallsTheToolOnceAndWritesTheMergedMessage(t *testing.T) { // 1236/ } } -func TestAnAnswerWithoutARootIsNotUsed(t *testing.T) { // 1236 (a shape the tool does not answer) - call := func(context.Context, string, any) (json.RawMessage, error) { - return json.RawMessage(`{"trees":{"a":"t"}}`), nil - } - if _, _, err := treesVia(context.Background(), call)("o", "r", "s"); err == nil { - t.Error("an answer with no root was taken") - } - call = func(context.Context, string, any) (json.RawMessage, error) { return json.RawMessage(`[1]`), nil } - if _, _, err := treesVia(context.Background(), call)("o", "r", "s"); err == nil { - t.Error("an answer that is not an object was taken") +func TestAnAnswerWithoutARootIsNotUsed(t *testing.T) { // 1236 (a shape the seat does not answer); design 47 *A failed read answers an error* + for _, raw := range []json.RawMessage{json.RawMessage(`{"trees":{"a":"t"}}`), json.RawMessage(`[1]`), + gitfixture.Given(t, "trees answers an error, never none", "verb:trees.answer")} { + call := func(context.Context, string, any) (json.RawMessage, error) { return raw, nil } + if _, _, err := treesVia(context.Background(), call)("o", "r", "s"); err == nil { + t.Errorf("the answer %s was taken", raw) + } } } -// The answer of gitea_get_file as mesh-catalog modules/gitea tools/index.ts and client.ts getFile write it, and the -// error of a path the forge does not have as client.ts request writes it (`Gitea API ${path}: ${status} ${text}`). -const getFileAnswer = `{"file":{"path":"modules/a/go.sum","ref":"89abcdef0123456789abcdef0123456789abcdef","sha":"0123","size":4,"content":"x y\n"}}` -const getFile404 = "Gitea API /repos/novox/mesh-delivery/contents/modules/a/go.sum?ref=89abcdef: 404 {\"message\":\"GetContentsOrList\",\"url\":\"https://forge.example/api/swagger\"}" - -func TestFilesViaAsksByOwnerRepoPathRefAndTakesTheContent(t *testing.T) { // 1448/AC4 +func TestFilesViaAsksByOwnerRepoPathRefAndTakesTheContent(t *testing.T) { // 1448/AC4; design 47 *The seat git* (verb file) var keys []string var args []any call := func(_ context.Context, key string, body any) (json.RawMessage, error) { keys, args = append(keys, key), append(args, body) - return json.RawMessage(getFileAnswer), nil + return gitfixture.Given(t, "file answers", "verb:file.answer"), nil } - got, err := filesVia(context.Background(), call)("novox", "mesh-delivery", "modules/a/go.sum", "89abcdef0123456789abcdef0123456789abcdef") - if err != nil || string(got) != "x y\n" { + got, err := filesVia(context.Background(), call)("novox", "mesh-contracts", "seats/index.json", strings.Repeat("a", 40)) + if err != nil || string(got) != `{"format":1,"seats":["mesh-broker"]}` { t.Fatalf("got %q err %v", got, err) } - if len(keys) != 1 || keys[0] != "gitea.gitea_get_file" { + if len(keys) != 1 || keys[0] != "seat:git.file" { t.Fatalf("calls %v", keys) } - if b, _ := json.Marshal(args[0]); string(b) != `{"owner":"novox","path":"modules/a/go.sum","ref":"89abcdef0123456789abcdef0123456789abcdef","repo":"mesh-delivery"}` { - t.Errorf("arguments %s", b) + want := gitfixture.Given(t, "file asks", "verb:file.input") + if b, _ := json.Marshal(args[0]); !sameJSON(b, want) { + t.Errorf("arguments %s, want the seat's %s", b, want) } } -func TestFilesViaTellsNotFoundFromOtherErrorsAndOtherShapes(t *testing.T) { // 1448/AC3 - call := func(err error, raw string) caller { - return func(context.Context, string, any) (json.RawMessage, error) { return json.RawMessage(raw), err } +func TestFilesViaTellsNotFoundFromOtherErrorsAndOtherShapes(t *testing.T) { // 1448/AC3; design 47 *A failed read answers an error* + call := func(err error, raw json.RawMessage) caller { + return func(context.Context, string, any) (json.RawMessage, error) { return raw, err } } - if _, err := filesVia(context.Background(), call(errors.New(getFile404), ""))("o", "r", "p", "s"); !errors.Is(err, bridge.ErrNotFound) { - t.Errorf("a 404 is %v, want ErrNotFound", err) + absent := gitfixture.All(t, "file answers", "verb:file.answer")[1] + if _, err := filesVia(context.Background(), call(nil, absent))("o", "r", "p", "s"); !errors.Is(err, bridge.ErrNotFound) { + t.Errorf("an absent file is %v, want ErrNotFound", err) } - if _, err := filesVia(context.Background(), call(errors.New("Gitea API /repos/o/r/contents/p: 502 bad gateway"), ""))("o", "r", "p", "s"); err == nil || errors.Is(err, bridge.ErrNotFound) { - t.Errorf("a 502 is %v, want an error that is not ErrNotFound", err) + if _, err := filesVia(context.Background(), call(errors.New("nats: timeout"), nil))("o", "r", "p", "s"); err == nil || errors.Is(err, bridge.ErrNotFound) { + t.Errorf("a seat that does not answer is %v, want an error that is not ErrNotFound", err) } - for _, raw := range []string{`{"file":null}`, `{}`, `[1]`, `{"file":{"path":"p"}}`} { + for _, raw := range []json.RawMessage{gitfixture.Given(t, "file answers an error, never none", "verb:file.answer"), json.RawMessage(`{}`), json.RawMessage(`[1]`), json.RawMessage(`{"path":"p"}`)} { if _, err := filesVia(context.Background(), call(nil, raw))("o", "r", "p", "s"); err == nil || errors.Is(err, bridge.ErrNotFound) { t.Errorf("the answer %s gave %v, want an error", raw, err) } } } +// sameJSON is whether two JSON texts say the same value. +func sameJSON(a, b []byte) bool { + var x, y any + if json.Unmarshal(a, &x) != nil || json.Unmarshal(b, &y) != nil { + return false + } + ax, _ := json.Marshal(x) + by, _ := json.Marshal(y) + return bytes.Equal(ax, by) +} + // someFiles is a forge that has a module.json at every path and no other file, for the tests that are not about the files. func someFiles(_, _, path, _ string) ([]byte, error) { if strings.HasSuffix(path, "module.json") { diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/open.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/open.go index a93543dd..93f4d07b 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/open.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/open.go @@ -10,14 +10,15 @@ import ( "github.com/nats-io/nats.go/jetstream" + git "git.novox.be/novox/mesh-delivery/contracts/go/seats/git" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "mesh-delivery-forge-bridge/internal/bridge" ) -// openPullsTool is the forge's tool for every open pull request (mesh-catalog modules/gitea/tools/index.ts -// gitea_open_pulls: no arguments, answers {pulls: [...]}). -const openPullsTool = "gitea.gitea_open_pulls" +// openPullsTool is the seat git's verb for every open pull request (novox/hq design 47 *The seat `git`*): no arguments, +// answers {pulls: [...]} or an error. +var openPullsTool = bridge.GitKey(git.VerbOpenPulls) // headStore is the bucket heads: the head last written on FORGE for each pull request, by bridge.Track's key // (novox/hq ADR 0443 §5). @@ -84,7 +85,7 @@ func syncOpen(ctx context.Context, call caller, pub publisher, heads headStore, defer mu.Unlock() ctx, cancel := context.WithTimeout(ctx, syncDeadline) defer cancel() - raw, err := call(ctx, openPullsTool, struct{}{}) + raw, err := call(ctx, openPullsTool, git.OpenPullsInput{}) if err != nil { return fmt.Errorf("%s: %w", openPullsTool, err) } diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/open_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/open_test.go index a9caade0..5861b9b7 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/open_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/open_test.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "log" + "strconv" "strings" "sync" "testing" @@ -16,12 +17,14 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" ) -// openAnswer is gitea_open_pulls's answer as mesh-catalog modules/gitea/client.ts:369-381 builds its entries inside {pulls}. +// openAnswer is the seat git's answer of open-pulls: two into the trunk, one into another branch. func openAnswer() json.RawMessage { - return json.RawMessage(`{"pulls":[ - {"owner":"novox","repo":"mesh-delivery","number":12,"base":"main","head":"feat/x","head_sha":"` + head + `","html_url":"h","required":[],"statuses":[],"checks":{},"labels":[]}, - {"owner":"novox","repo":"mesh-catalog","number":300,"base":"main","head":"feat/x","head_sha":"` + strings.Repeat("a", 40) + `","html_url":"h","required":[],"statuses":[],"checks":{},"labels":[]}, - {"owner":"novox","repo":"mesh-catalog","number":301,"base":"feat/y","head":"feat/z","head_sha":"` + strings.Repeat("b", 40) + `","html_url":"h","required":[],"statuses":[],"checks":{},"labels":[]}]}`) + entry := func(repo string, number int, base, branch, sha string) string { + return `{"pull":{"repository":"novox/` + repo + `","number":` + strconv.Itoa(number) + `,"base":"` + base + `","branch":"` + branch + + `","head":"` + sha + `","state":"open","url":"h"},"required":[],"checks":{}}` + } + return json.RawMessage(`{"pulls":[` + entry("mesh-delivery", 12, "main", "feat/x", head) + `,` + + entry("mesh-catalog", 300, "main", "feat/x", strings.Repeat("a", 40)) + `,` + entry("mesh-catalog", 301, "feat/y", "feat/z", strings.Repeat("b", 40)) + `]}`) } func TestSyncOpenWritesOneUpdatedMessagePerOpenPull(t *testing.T) { // 1237/AC1, AC4 @@ -35,7 +38,7 @@ func TestSyncOpenWritesOneUpdatedMessagePerOpenPull(t *testing.T) { // 1237/AC1, if err := runSync(call, pub, newHeads(), &sync.Mutex{}, log.New(&logs, "", 0)); err != nil { t.Fatal(err) } - if len(keys) != 1 || keys[0] != "gitea.gitea_open_pulls" { + if len(keys) != 1 || keys[0] != "seat:git.open-pulls" { t.Errorf("calls %v", keys) } if len(pub.subjects) != 3 { @@ -232,7 +235,7 @@ func TestSyncOpenFailureNamesTheCallAndTheForgesWords(t *testing.T) { // 1237/AC return nil, errors.New("the forge said 502") } err := runSync(call, pub, newHeads(), &sync.Mutex{}, log.New(&bytes.Buffer{}, "", 0)) - if err == nil || !strings.Contains(err.Error(), "gitea.gitea_open_pulls") || !strings.Contains(err.Error(), "the forge said 502") { + if err == nil || !strings.Contains(err.Error(), "seat:git.open-pulls") || !strings.Contains(err.Error(), "the forge said 502") { t.Errorf("err = %v", err) } if len(pub.subjects) != 0 { diff --git a/modules/mesh-delivery-forge-bridge/go.mod b/modules/mesh-delivery-forge-bridge/go.mod index 042b8ff1..9a5a262e 100644 --- a/modules/mesh-delivery-forge-bridge/go.mod +++ b/modules/mesh-delivery-forge-bridge/go.mod @@ -4,6 +4,7 @@ go 1.26.0 require ( git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/contracts/go/seats/git v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/sdk v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000 diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go b/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go index a920e158..1a7a7439 100644 --- a/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go @@ -1,4 +1,4 @@ -// Package bridge turns the forge module's pull request events into the messages the forge bridge writes on FORGE. +// Package bridge turns the seat git's pull request events into the messages the forge bridge writes on FORGE. package bridge import ( @@ -15,6 +15,7 @@ import ( "strings" "sync" + git "git.novox.be/novox/mesh-delivery/contracts/go/seats/git" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/keys" @@ -35,13 +36,13 @@ var ErrRetry = errors.New("the forge could not say") const Trunk = "main" // Trees asks the forge for the trees of a commit: the root tree and the tree of every directory holding a module.json -// (the root directory spelled ""), as the forge's gitea_commit_trees answers (issue 1234/AC4). +// (the root directory spelled ""), as the seat git's verb trees answers (issue 1234/AC4). type Trees func(owner, repo, sha string) (root string, trees map[string]string, err error) // ErrNotFound says the forge has no file at that path and ref (the forge's "404", issue 1448). var ErrNotFound = errors.New("the forge has no such file") -// Files asks the forge for one file's bytes at a ref (a commit), as the forge's gitea_get_file answers; a file the forge +// Files asks the forge for one file's bytes at a ref (a commit), as the seat git's verb file answers; a file the forge // says is not there is ErrNotFound, any other failure any other error (issue 1448). type Files func(owner, repo, path, ref string) ([]byte, error) @@ -64,16 +65,16 @@ type Message struct { Said string } -// The subjects of the forge module's events the bridge writes pull messages for (mesh-catalog modules/gitea, -// published by the module as mesh.mod..event.). +// The subjects of the seat git's events the bridge writes pull messages for (novox/hq design 47 *The seat `git`*): the +// seat's holder says them on mesh.seat.git.event., whichever module holds it; the bridge names none. const ( - UpdatedSubject = "mesh.mod.gitea.event.pull.updated" - ClosedSubject = "mesh.mod.gitea.event.pull.closed" - MergedSubject = "mesh.mod.gitea.event.pull.merged" - // DeletedSubject is the forge module's forge.delete: a branch or a tag deleted (mesh-catalog modules/gitea - // webhook.ts: owner, repo, ref, ref_type, sha). Only the forge module's account may publish it, the subject being - // in its own namespace (design 32 § 2), so no other module can end a branch's tagged versions. - DeletedSubject = "mesh.mod.gitea.event.forge.delete" + UpdatedSubject = git.SubjectPullUpdated + ClosedSubject = git.SubjectPullClosed + MergedSubject = git.SubjectPullMerged + // DeletedSubject is the seat's forge.delete: a branch or a tag deleted (repository, ref, ref-type). Only the seat's + // holder may publish on the seat's subjects (novox/hq ADR 0363 §2), so no other module can end a branch's tagged + // versions. + DeletedSubject = git.SubjectForgeDelete ) // ForgeBranchDeleted is the kind of a branch-deleted message on FORGE (novox/hq design 51 *The forge bridge says a @@ -91,21 +92,14 @@ type BranchDeleted struct { Head string `json:"head,omitempty"` } -// deleted is the part of forge.delete the bridge reads. -type deleted struct { - Owner string `json:"owner"` - Repo string `json:"repo"` - Ref string `json:"ref"` - RefType string `json:"ref_type"` - SHA string `json:"sha"` -} - // branchDeleted turns forge.delete into a branch-deleted message; a deleted tag is not handled. func branchDeleted(subject string, data []byte) (Message, error) { - var d deleted - if err := json.Unmarshal(data, &d); err != nil { + var e git.ForgeDeleteEvent + if err := json.Unmarshal(data, &e); err != nil { return Message{}, unreadable(subject, "the body is not json: %v", err) } + d := struct{ Owner, Repo, Ref, RefType string }{Ref: e.Ref, RefType: e.RefType} + d.Owner, d.Repo, _ = strings.Cut(e.Repository, "/") if d.RefType != "branch" { return Message{}, fmt.Errorf("%w: %s of a %s (%s)", ErrNotHandled, subject, d.RefType, d.Ref) } @@ -118,7 +112,7 @@ func branchDeleted(subject string, data []byte) (Message, error) { case branch == Trunk: return Message{}, fmt.Errorf("%w: %s of the trunk %s", ErrNotHandled, subject, Trunk) } - body, err := json.Marshal(BranchDeleted{V: keys.Version, Kind: ForgeBranchDeleted, Repository: d.Owner + "/" + d.Repo, Branch: branch, Head: d.SHA}) + body, err := json.Marshal(BranchDeleted{V: keys.Version, Kind: ForgeBranchDeleted, Repository: d.Owner + "/" + d.Repo, Branch: branch}) if err != nil { return Message{}, unreadable(subject, "%v", err) } @@ -128,22 +122,48 @@ func branchDeleted(subject string, data []byte) (Message, error) { var sha40 = regexp.MustCompile(`^[0-9a-f]{40}$`) -// pull is the part of gitea.pull.merged (index.ts:186-207), gitea.pull.updated (mesh-catalog modules/gitea/trunk.ts:233-250) and gitea.pull.closed -// (index.ts:127-128) the bridge reads; head is the branch and head_sha the commit. +// pull is the part of the seat git's pull.updated, pull.merged and pull.closed the bridge reads: Head is the branch and +// HeadSHA the commit, Owner and Repo the repository's two halves. type pull struct { - Owner string `json:"owner"` - Repo string `json:"repo"` - Number int `json:"number"` - Head string `json:"head"` - HeadSHA string `json:"head_sha"` - ModuleDirs []string `json:"module_dirs"` - ModuleDirsSaid bool `json:"module_dirs_said"` - Base string `json:"base"` - MergeSHA string `json:"merge_commit_sha"` - // Paths are the files the pull request changes (a removed or renamed one included), PathsTruncated says the forge - // cut the list; nil when the forge did not say them. - Paths *[]string `json:"paths"` - PathsTruncated bool `json:"paths_truncated"` + Owner, Repo string + Number int + Head, HeadSHA string + ModuleDirs []string + ModuleDirsSaid bool + Base string + MergeSHA string +} + +// pullOf reads one of the seat's pull events. The directories are the seat's word only when its change says them: an +// absent `module-dirs` is "could not be read", never "none". +func pullOf(data []byte) (pull, error) { + var e struct { + Repository string `json:"repository"` + Number int64 `json:"number"` + Branch string `json:"branch"` + Head string `json:"head"` + Base string `json:"base"` + MergeCommit string `json:"merge-commit"` + Change json.RawMessage `json:"change"` + } + if err := json.Unmarshal(data, &e); err != nil { + return pull{}, err + } + p := pull{Number: int(e.Number), Head: e.Branch, HeadSHA: e.Head, Base: e.Base, MergeSHA: e.MergeCommit} + p.Owner, p.Repo, _ = strings.Cut(e.Repository, "/") + if len(e.Change) > 0 && string(e.Change) != "null" { + var c map[string]json.RawMessage + if err := json.Unmarshal(e.Change, &c); err != nil { + return pull{}, fmt.Errorf("its change: %w", err) + } + if raw, said := c["module-dirs"]; said { + if err := json.Unmarshal(raw, &p.ModuleDirs); err != nil { + return pull{}, fmt.Errorf("its change's module-dirs: %w", err) + } + p.ModuleDirsSaid = true + } + } + return p, nil } func unreadable(subject, why string, args ...any) error { @@ -176,12 +196,12 @@ func ConvertKept(subject string, data []byte, trees Trees, files Files, kept Kep default: return Message{}, fmt.Errorf("%w: %s", ErrNotHandled, subject) } - var p pull - if err := json.Unmarshal(data, &p); err != nil { - return Message{}, unreadable(subject, "the body is not json: %v", err) + p, err := pullOf(data) + if err != nil { + return Message{}, unreadable(subject, "the body is not the seat's json: %v", err) } switch { - case p.Owner == "" || p.Repo == "": + case p.Owner == "" || p.Repo == "" || strings.Contains(p.Repo, "/"): return Message{}, unreadable(subject, "no owner or repo") case p.Number <= 0: return Message{}, unreadable(subject, "no pull request number") @@ -462,29 +482,39 @@ func merged(subject string, p pull, trees Trees, files Files, kept Kept) (Messag return msg, nil } -// openPull is an entry of gitea_open_pulls's answer (mesh-catalog modules/gitea/client.ts:369-381 openPulls, with the -// branch as `head`, issue 1234/AC5), the part the bridge reads. -type openPull struct { - Owner string `json:"owner"` - Repo string `json:"repo"` - Number int `json:"number"` - Base string `json:"base"` - Head string `json:"head"` - HeadSHA string `json:"head_sha"` +// openPulls reads the seat git's answer of open-pulls: an answer saying an error is the error, never none open. +func openPulls(answer []byte) ([]pull, error) { + var a git.OpenPullsAnswer + if err := json.Unmarshal(answer, &a); err != nil { + return nil, fmt.Errorf("the seat git's answer of open-pulls is not its contract's: %v", err) + } + if a.Error != "" { + return nil, fmt.Errorf("the seat git refused open-pulls: %s", a.Error) + } + var said map[string]json.RawMessage + if _ = json.Unmarshal(answer, &said); said["pulls"] == nil || string(said["pulls"]) == "null" { + // None open is `pulls: []`; an answer naming no list would drop every candidate on a holder's fault. + return nil, fmt.Errorf("open-pulls: %w", errNoAnswer) + } + out := make([]pull, 0, len(a.Pulls)) + for _, o := range a.Pulls { + p := pull{Number: int(o.Pull.Number), Head: o.Pull.Branch, HeadSHA: o.Pull.Head, Base: o.Pull.Base} + p.Owner, p.Repo, _ = strings.Cut(o.Pull.Repository, "/") + out = append(out, p) + } + return out, nil } -// OpenPulls turns the answer of gitea_open_pulls, {pulls: [...]}, into one pull message per open pull request into +// OpenPulls turns the seat git's answer of open-pulls into one pull message per open pull request into // the trunk, action updated and its directories not known (the forge's answer does not say them). A pull request into // another branch gets none. An entry that cannot be read is named in skipped and gets none; an answer that is not -// {pulls} is an error. +// the contract's, or that says an error, is an error. func OpenPulls(answer []byte) (msgs []Message, skipped []string, err error) { - var a struct { - Pulls *[]openPull `json:"pulls"` + pulls, err := openPulls(answer) + if err != nil { + return nil, nil, err } - if err := json.Unmarshal(answer, &a); err != nil || a.Pulls == nil { - return nil, nil, fmt.Errorf("the answer of gitea_open_pulls is not {pulls: [...]}: %v", err) - } - for _, p := range *a.Pulls { + for _, p := range pulls { if p.Base != Trunk { continue } @@ -523,18 +553,16 @@ func Track(f forgemsg.Forge) (key, head string, ok bool) { return "", "", false } -// OpenMessage turns the answer of gitea_open_pulls into the open message: every open pull request into the trunk, by +// OpenMessage turns the seat git's answer of open-pulls into the open message: every open pull request into the trunk, by // repository and number, whether its head reads or not and whether FORGE holds it or not, in repository and number // order. A pull request into another branch is not named. An answer that names none is a message too: nothing is open. func OpenMessage(answer []byte) (Message, error) { - var a struct { - Pulls *[]openPull `json:"pulls"` - } - if err := json.Unmarshal(answer, &a); err != nil || a.Pulls == nil { - return Message{}, fmt.Errorf("the answer of gitea_open_pulls is not {pulls: [...]}: %v", err) + pulls, err := openPulls(answer) + if err != nil { + return Message{}, err } open := []forgemsg.OpenPull{} - for _, p := range *a.Pulls { + for _, p := range pulls { if p.Base != Trunk || p.Owner == "" || p.Repo == "" || p.Number <= 0 { continue } diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/bridge_test.go b/modules/mesh-delivery-forge-bridge/internal/bridge/bridge_test.go index 6744636b..2ab29a65 100644 --- a/modules/mesh-delivery-forge-bridge/internal/bridge/bridge_test.go +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/bridge_test.go @@ -1,6 +1,7 @@ package bridge import ( + "context" "encoding/json" "errors" "strings" @@ -8,44 +9,63 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-sdk/go/keys" + + "mesh-delivery-forge-bridge/internal/gitfixture" ) const head = "0123456789abcdef0123456789abcdef01234567" -// updated is the body of gitea.pull.updated as mesh-catalog modules/gitea/trunk.ts:233-250 (announcementOf) writes -// it: owner, repo, number, base_sha, title, body, base, head (the branch), head_sha, clone_url, html_url, paths, -// paths_truncated, removed, module_dirs, module_dirs_said, merge_check, merge_check_said. +// updated is the body of the seat git's pull.updated (contracts/seats/git): the pull request, its base's head and its +// change. extra names the parts a test changes by their short names (owner, repo, number, head for the branch, +// head_sha for the commit, base, module_dirs, module_dirs_said), a nil value leaving the part out (seatPull). func updated(extra map[string]any) []byte { - m := map[string]any{ - "owner": "novox", "repo": "mesh-delivery", "number": 12, - "base_sha": "ffffffffffffffffffffffffffffffffffffffff", "title": "A change", "body": "text", "base": "main", - "head": "feat/x", "head_sha": head, "clone_url": "https://forge.example/novox/mesh-delivery.git", - "html_url": "https://forge.example/novox/mesh-delivery/pulls/12", - "paths": []string{"modules/a/x.go"}, "paths_truncated": false, "removed": []string{}, - "module_dirs": []string{"modules/a"}, "module_dirs_said": true, - "merge_check": true, "merge_check_said": true, - } - for k, v := range extra { - if v == nil { - delete(m, k) - } else { - m[k] = v + return seatPull(map[string]any{"base_sha": "ffffffffffffffffffffffffffffffffffffffff", + "paths": []string{"modules/a/x.go"}, "module_dirs": []string{"modules/a"}, "module_dirs_said": true}, extra) +} + +// closed is the body of the seat git's pull.closed. +func closed() []byte { return seatPull(nil, nil) } + +// seatPull is a pull event of the seat git: the pull request novox/mesh-delivery#12 of feat/x at head into main, with +// the parts of base and extra laid over it by their short names, a nil value leaving the part out. module_dirs is said +// in the change only when module_dirs_said is true: its absence is "could not be read". +func seatPull(base, extra map[string]any) []byte { + m := map[string]any{"owner": "novox", "repo": "mesh-delivery", "number": 12, "title": "A change", "head": "feat/x", + "head_sha": head, "base": "main", "html_url": "https://forge.example/novox/mesh-delivery/pulls/12"} + for _, over := range []map[string]any{base, extra} { + for k, v := range over { + if v == nil { + delete(m, k) + } else { + m[k] = v + } } } - b, _ := json.Marshal(m) + e := map[string]any{} + owner, _ := m["owner"].(string) + repo, _ := m["repo"].(string) + if owner != "" || repo != "" { + e["repository"] = owner + "/" + repo + } + for short, name := range map[string]string{"number": "number", "title": "title", "head": "branch", "head_sha": "head", + "base": "base", "html_url": "url", "merge_commit_sha": "merge-commit", "base_sha": "base-head", "merged_at": "merged-at"} { + if v, ok := m[short]; ok { + e[name] = v + } + } + if paths, ok := m["paths"]; ok { + c := map[string]any{"paths": paths} + if m["module_dirs_said"] == true && m["module_dirs"] != nil { + c["module-dirs"] = m["module_dirs"] + } + e["change"] = c + } + b, _ := json.Marshal(e) return b } -// closed is the body of gitea.pull.closed as mesh-catalog modules/gitea/index.ts:127-128 writes it: owner, repo, -// number, title, head, head_sha, base, html_url. -func closed() []byte { - b, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-delivery", "number": 12, "title": "A change", - "head": "feat/x", "head_sha": head, "base": "main", "html_url": "https://forge.example/novox/mesh-delivery/pulls/12"}) - return b -} - -const updatedSubject = "mesh.mod.gitea.event.pull.updated" -const closedSubject = "mesh.mod.gitea.event.pull.closed" +const updatedSubject = "mesh.seat.git.event.pull-updated" +const closedSubject = "mesh.seat.git.event.pull-closed" func TestUpdatedBecomesAPullMessage(t *testing.T) { // 1235/AC2 (the conversion; the bus is the integration test) msg, err := Convert(updatedSubject, updated(nil), nil, anyFiles) @@ -142,8 +162,8 @@ func TestUnreadableEventsWriteNothing(t *testing.T) { // 1235/AC5 "upper-case head": {updatedSubject, updated(map[string]any{"head_sha": strings.ToUpper(head)}), "head"}, "no branch": {updatedSubject, updated(map[string]any{"head": nil}), "branch"}, "not json": {updatedSubject, []byte("{"), "json"}, - "closed no head": {closedSubject, mustDrop(closed(), "head_sha"), "head"}, - "closed no repo": {closedSubject, mustDrop(closed(), "repo"), "repo"}, + "closed no head": {closedSubject, mustDrop(closed(), "head"), "head"}, + "closed no repo": {closedSubject, mustDrop(closed(), "repository"), "repo"}, } { msg, err := Convert(c.subject, c.data, nil, anyFiles) if err == nil { @@ -171,21 +191,21 @@ func mustDrop(body []byte, key string) []byte { } func TestOtherSubjectsAreNotHandled(t *testing.T) { // 1235, 1236 - for _, s := range []string{"mesh.mod.gitea.event.something.else", "mesh.mod.other.event.pull.updated", "mesh.mod.other.event.forge.delete"} { + for _, s := range []string{"mesh.seat.git.event.something.else", "mesh.mod.gitea.event.pull.updated", "mesh.mod.other.event.forge.delete"} { if _, err := Convert(s, updated(nil), nil, anyFiles); !errors.Is(err, ErrNotHandled) { t.Errorf("%s: err = %v, want ErrNotHandled", s, err) } } - // The forge module's forge.delete of a branch is a branch-deleted message (DLV-FB1, design 51); of a tag or the + // The seat's forge.delete of a branch is a branch-deleted message (DLV-FB1, design 51); of a tag or the // trunk it is not handled, and a body naming no repository is unreadable. del := func(refType, ref string) []byte { - b, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-sdk", "full_name": "novox/mesh-sdk", "ref": ref, "ref_type": refType, "sha": head}) + b, _ := json.Marshal(map[string]any{"repository": "novox/mesh-sdk", "ref": ref, "ref-type": refType, "sender": "jochen"}) return b } msg, err := Convert(DeletedSubject, del("branch", "feat/x"), nil, anyFiles) var bd BranchDeleted if err != nil || msg.Event != "forge.novox.mesh-sdk.branch-deleted" || json.Unmarshal(msg.Data, &bd) != nil || - bd.Kind != ForgeBranchDeleted || bd.Repository != "novox/mesh-sdk" || bd.Branch != "feat/x" || bd.Head != head || msg.Said == "" { + bd.Kind != ForgeBranchDeleted || bd.Repository != "novox/mesh-sdk" || bd.Branch != "feat/x" || bd.Head != "" || msg.Said == "" { t.Errorf("forge.delete of a branch = %+v (%s), %v", msg, msg.Data, err) } for _, c := range []struct{ refType, ref string }{{"tag", "go/v0.1.13"}, {"branch", "main"}} { @@ -193,7 +213,7 @@ func TestOtherSubjectsAreNotHandled(t *testing.T) { // 1235, 1236 t.Errorf("forge.delete of %s %s: err = %v, want ErrNotHandled", c.refType, c.ref, err) } } - if _, err := Convert(DeletedSubject, []byte(`{"ref":"x","ref_type":"branch"}`), nil, anyFiles); !errors.Is(err, ErrUnreadable) { + if _, err := Convert(DeletedSubject, []byte(`{"ref":"x","ref-type":"branch"}`), nil, anyFiles); !errors.Is(err, ErrUnreadable) { t.Errorf("forge.delete naming no repository: err = %v, want ErrUnreadable", err) } // 1236: a merged event without a full merge commit is unreadable, and the forge is not asked. @@ -206,27 +226,13 @@ func TestOtherSubjectsAreNotHandled(t *testing.T) { // 1235, 1236 const merge = "89abcdef0123456789abcdef0123456789abcdef" -// mergedBody is gitea.pull.merged as mesh-catalog modules/gitea/index.ts:186-207 writes it. +// mergedBody is the seat git's pull.merged, extra by the short names seatPull reads. func mergedBody(extra map[string]any) []byte { - m := map[string]any{ - "owner": "novox", "repo": "mesh-delivery", "number": 12, "title": "A change", "body": "text", - "head_sha": head, "head_checks": map[string]string{"mesh/merge-gate": "success"}, "head": "feat/x", "base": "main", - "merge_commit_sha": merge, "merged_at": "2026-10-19T12:00:00Z", "clone_url": "https://forge.example/novox/mesh-delivery.git", - "html_url": "https://forge.example/novox/mesh-delivery/pulls/12", "paths": []string{"modules/a/x.go"}, "paths_truncated": false, - "removed": []string{}, "module_dirs": []string{"modules/a"}, "module_dirs_said": true, - } - for k, v := range extra { - if v == nil { - delete(m, k) - } else { - m[k] = v - } - } - b, _ := json.Marshal(m) - return b + return seatPull(map[string]any{"merge_commit_sha": merge, "merged_at": "2026-10-19T12:00:00Z", + "paths": []string{"modules/a/x.go"}, "module_dirs": []string{"modules/a"}, "module_dirs_said": true}, extra) } -const mergedSubject = "mesh.mod.gitea.event.pull.merged" +const mergedSubject = "mesh.seat.git.event.pull-merged" type call struct{ owner, repo, sha string } @@ -311,16 +317,15 @@ func TestNoModuleJSONAnywhereWritesAMergedMessageWithNoTrees(t *testing.T) { } } -// openAnswer is gitea_open_pulls's answer as mesh-catalog modules/gitea/client.ts:369-381 builds each entry -// (owner, repo, number, base, head, head_sha, html_url, required, statuses, checks, labels) inside {pulls}. +// openAnswer is the seat git's answer of open-pulls. func openAnswer(pulls ...map[string]any) []byte { b, _ := json.Marshal(map[string]any{"pulls": pulls}) return b } func openEntry(repo string, number int, base, branch, sha string) map[string]any { - return map[string]any{"owner": "novox", "repo": repo, "number": number, "base": base, "head": branch, "head_sha": sha, - "html_url": "https://forge.example/x", "required": []string{"mesh/merge-gate"}, "statuses": []string{}, "checks": map[string]string{}, "labels": []string{}} + return map[string]any{"pull": map[string]any{"repository": "novox/" + repo, "number": number, "base": base, "branch": branch, "head": sha, + "state": "open", "url": "https://forge.example/x"}, "required": []string{"mesh/check"}, "checks": map[string]any{}} } func TestOpenPullsAreUpdatedPullMessagesWithUnknownDirs(t *testing.T) { // 1237/AC1 @@ -355,7 +360,7 @@ func TestOpenPullsUnreadableEntryIsNamedAndTheRestGo(t *testing.T) { // 1237, 12 if err != nil || len(msgs) != 1 || len(skipped) != 1 || !strings.Contains(skipped[0], "novox/a#1") { t.Errorf("msgs %d skipped %v err %v", len(msgs), skipped, err) } - for _, bad := range []string{`{"pulls":null}`, `{}`, `[]`, `x`} { + for _, bad := range []string{`{"pulls":null}`, `{}`, `[]`, `x`, string(gitFixture(t, "open-pulls answers an error, never none", "verb:open-pulls.answer"))} { if _, _, err := OpenPulls([]byte(bad)); err == nil { t.Errorf("%s was taken as an answer", bad) } @@ -381,7 +386,7 @@ func TestOpenMessageNamesEveryOpenPullIntoTheTrunk(t *testing.T) { // 1279/AC6 if f, derr := forgemsg.DecodeForge(m.Data); err != nil || derr != nil || f.Kind != forgemsg.ForgeOpen || len(f.Open) != 0 { t.Errorf("empty answer: %+v %v %v", f, err, derr) } - for _, bad := range []string{`{"pulls":null}`, `{}`, `x`} { + for _, bad := range []string{`{"pulls":null}`, `{}`, `x`, `{"error":"the forge answered 502","pulls":[]}`} { if _, err := OpenMessage([]byte(bad)); err == nil { t.Errorf("%s was taken as an answer", bad) } @@ -411,41 +416,66 @@ func TestTrackSaysWhichHeadOfWhichPullRequestWasWritten(t *testing.T) { // 1279/ } } -func TestBridgeTagsAnswersAPackagesTagsByPrefixPageByPage(t *testing.T) { // DLV-FB1, design 51 *The floor* - var asked []string - page := func(n int) json.RawMessage { - var tags []map[string]string - for i := 0; i < n; i++ { - tags = append(tags, map[string]string{"name": "x" + strings.Repeat("y", i)}) +// gitFixture is one answer of the seat git's fixtures, by its name and what it is of. +func gitFixture(t *testing.T, name, of string) json.RawMessage { + t.Helper() + return gitfixture.Given(t, name, of) +} + +func TestBridgeTagsAnswersAPackagesTagsByPrefixPageByPage(t *testing.T) { // DLV-FB1, design 51 *The floor*; design 47 *The seat git* + more := gitFixture(t, "tags answers", "verb:tags.answer") // go/v0.1.15, go/v0.1.14, more + failed := gitFixture(t, "tags answers an error, never none", "verb:tags.answer") + last := json.RawMessage(`{"repository":"novox/mesh-sdk","tags":["go/v0.1.2","v0.0.1"],"page":2,"bound":50}`) + type asked struct { + key string + in string + } + for _, row := range []struct { + name string + answer func(page int) (json.RawMessage, error) + repo string + tags []string + more bool + pages int + failing string + }{ + {name: "two pages, the second says no more", repo: "novox/mesh-sdk", tags: []string{"go/v0.1.14", "go/v0.1.15", "go/v0.1.2"}, pages: 2, + answer: func(p int) (json.RawMessage, error) { + if p == 1 { + return more, nil + } + return last, nil + }}, + {name: "every page says more: stopped at the bound", repo: "novox/mesh-sdk", more: true, pages: TagsMaxPages, + answer: func(int) (json.RawMessage, error) { return json.RawMessage(`{"tags":[],"more":true}`), nil }}, + {name: "the seat does not answer", repo: "novox/mesh-sdk", failing: "page 1", pages: 1, + answer: func(int) (json.RawMessage, error) { return nil, errors.New("down") }}, + {name: "an error answer is an error, never none", repo: "novox/mesh-sdk", failing: "404", pages: 1, + answer: func(int) (json.RawMessage, error) { return failed, nil }}, + {name: "a repository with no owner", repo: "mesh-sdk", failing: "/"}, + } { + var calls []asked + call := func(_ context.Context, key string, body any) (json.RawMessage, error) { + in, _ := json.Marshal(body) + calls = append(calls, asked{key, string(in)}) + var p struct{ Page int } + _ = json.Unmarshal(in, &p) + return row.answer(p.Page) } - b, _ := json.Marshal(tags) - return b - } - api := func(path string) (json.RawMessage, error) { - asked = append(asked, path) - if strings.Contains(path, "page=1&") { - full := page(TagsPageSize - 2) - var tags []map[string]string - _ = json.Unmarshal(full, &tags) - tags = append(tags, map[string]string{"name": "go/v0.1.13"}, map[string]string{"name": "v0.0.1"}) - b, _ := json.Marshal(map[string]any{"result": tags}) - return b, nil + a, err := Tags(context.Background(), call, row.repo, "go/v") + if row.failing != "" { + if err == nil || !strings.Contains(err.Error(), row.failing) { + t.Errorf("%s: %v, want an error naming %q", row.name, err, row.failing) + } + } else if err != nil || !slicesEqual(a.Tags, row.tags) || a.More != row.more { + t.Errorf("%s: Tags = %+v, %v", row.name, a, err) + } + if len(calls) != row.pages { + t.Errorf("%s: %d calls, want %d", row.name, len(calls), row.pages) + } + if len(calls) > 0 && (calls[0].key != "seat:git.tags" || calls[0].in != `{"page":1,"prefix":"go/v","repository":"novox/mesh-sdk"}`) { + t.Errorf("%s: first call %+v", row.name, calls[0]) } - return json.RawMessage(`{"result":[{"name":"go/v0.1.2"}]}`), nil - } - a, err := Tags(api, "novox/mesh-sdk", "go/v") - if err != nil || !slicesEqual(a.Tags, []string{"go/v0.1.13", "go/v0.1.2"}) || a.More || len(asked) != 2 || - asked[0] != "/repos/novox/mesh-sdk/tags?page=1&limit=50" { - t.Errorf("Tags = %+v, %v; asked %v", a, err, asked) - } - if _, err := Tags(func(string) (json.RawMessage, error) { return nil, errors.New("down") }, "novox/mesh-sdk", ""); err == nil || !strings.Contains(err.Error(), "page 1") { - t.Errorf("a forge not answering = %v, want an error naming the page", err) - } - if _, err := Tags(func(string) (json.RawMessage, error) { return json.RawMessage(`[{"name":"go/v0.1.2"}]`), nil }, "novox/mesh-sdk", ""); err == nil || !strings.Contains(err.Error(), `{"result"`) { - t.Errorf("a bare list with no wrapper = %v, want the shape named", err) - } - if _, err := Tags(api, "mesh-sdk", ""); err == nil { - t.Error("a repository with no owner answered") } } diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/call.go b/modules/mesh-delivery-forge-bridge/internal/bridge/call.go index d1160c79..344cc9bd 100644 --- a/modules/mesh-delivery-forge-bridge/internal/bridge/call.go +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/call.go @@ -3,77 +3,106 @@ package bridge import ( "context" "encoding/json" + "errors" "fmt" - "regexp" + "strings" + + git "git.novox.be/novox/mesh-delivery/contracts/go/seats/git" ) -// Caller calls a module's tool by its manifest key (`.`) with a JSON body and returns the tool's result: -// the bridge's process calls through the bus as its own principal, its tools bundle through the runtime (stdio.Ask). +// Caller calls a seat's verb by its manifest key (`seat:.`) with a JSON body and returns its answer: the +// bridge's process calls through the bus as its own principal, its tools bundle through the runtime (stdio.Ask). type Caller func(ctx context.Context, key string, body any) (json.RawMessage, error) -// CommitTreesTool is the forge's tool for a commit's trees (mesh-catalog modules/gitea, issue 1234): arguments owner, -// repo and sha, answer {root, trees}, the root directory spelled "" (1234/AC4). -const CommitTreesTool = "gitea.gitea_commit_trees" +// GitKey is the manifest key of a verb of the seat `git` (novox/hq design 47 *The seat `git`*): the bridge reaches the +// forge through the seat alone and names no module that holds it. +func GitKey(verb string) string { return "seat:" + git.Seat + "." + verb } -// TreesVia asks the forge for a commit's trees through call. +// askGit calls one verb of the seat `git` and decodes its answer into A. An answer the contract cannot type, or one +// that says an error, is the call's error: a failed read is never read as none (design 47 *A failed read answers an +// error*). +func askGit[A any](ctx context.Context, call Caller, verb string, in any) (A, error) { + var a A + raw, err := call(ctx, GitKey(verb), in) + if err != nil { + return a, fmt.Errorf("the seat git did not answer %s: %w", verb, err) + } + if err := json.Unmarshal(raw, &a); err != nil { + return a, fmt.Errorf("the seat git's answer of %s is not its contract's: %w", verb, err) + } + var said struct { + Error string `json:"error"` + } + _ = json.Unmarshal(raw, &said) + if said.Error != "" { + return a, fmt.Errorf("the seat git refused %s: %s", verb, said.Error) + } + return a, nil +} + +// repository is `/`, as the seat names a repository. +func repository(owner, repo string) string { return owner + "/" + repo } + +// TreesVia asks the seat `git` for a commit's trees (verb trees). func TreesVia(ctx context.Context, call Caller) Trees { return func(owner, repo, sha string) (string, map[string]string, error) { - raw, err := call(ctx, CommitTreesTool, map[string]string{"owner": owner, "repo": repo, "sha": sha}) + a, err := askGit[git.TreesAnswer](ctx, call, git.VerbTrees, git.TreesInput{Repository: repository(owner, repo), Commit: sha}) if err != nil { return "", nil, err } - var a struct { - Root string `json:"root"` - Trees map[string]string `json:"trees"` - } - if err := json.Unmarshal(raw, &a); err != nil { - return "", nil, fmt.Errorf("the answer of %s is not {root, trees}: %w", CommitTreesTool, err) - } if a.Root == "" { - return "", nil, fmt.Errorf("the answer of %s names no root tree", CommitTreesTool) + return "", nil, fmt.Errorf("the seat git's trees of %s at %s name no root tree", repository(owner, repo), sha) } return a.Root, a.Trees, nil } } -// GetFileTool is the forge's tool for one file at a ref (mesh-catalog modules/gitea tools/index.ts gitea_get_file): -// arguments owner, repo, path and ref, answer {file: {path, ref, sha, size, content}}, the content decoded to text -// (client.ts getFile). A path the forge does not have fails with `Gitea API : 404 ` (client.ts request). -const GetFileTool = "gitea.gitea_get_file" - -// notFound is how the forge's error says 404 (client.ts request: `Gitea API ${path}: ${status} ${text}`; client.ts:670 -// recognises it as /: 404 /). -var notFound = regexp.MustCompile(`: 404 `) - -// FilesVia asks the forge for a file through call. A file the forge says is not there is ErrNotFound. +// FilesVia asks the seat `git` for one file at a ref (verb file). A file the seat says is absent is ErrNotFound; an +// answer that says neither absent nor content is an error, never an empty file. func FilesVia(ctx context.Context, call Caller) Files { return func(owner, repo, path, ref string) ([]byte, error) { - raw, err := call(ctx, GetFileTool, map[string]string{"owner": owner, "repo": repo, "path": path, "ref": ref}) + raw, err := call(ctx, GitKey(git.VerbFile), git.FileInput{Repository: repository(owner, repo), Path: path, Ref: ref}) if err != nil { - if notFound.MatchString(err.Error()) { - return nil, fmt.Errorf("%w: %v", ErrNotFound, err) - } - return nil, err + return nil, fmt.Errorf("the seat git did not answer file: %w", err) } var a struct { - File *struct { - Content *string `json:"content"` - } `json:"file"` + git.FileAnswer + Content *string `json:"content"` } - if err := json.Unmarshal(raw, &a); err != nil || a.File == nil || a.File.Content == nil { - return nil, fmt.Errorf("the answer of %s is not {file: {content}}: %v", GetFileTool, err) + if err := json.Unmarshal(raw, &a); err != nil { + return nil, fmt.Errorf("the seat git's answer of file is not its contract's: %w", err) } - return []byte(*a.File.Content), nil + switch { + case a.Error != "": + return nil, fmt.Errorf("the seat git refused file %s at %s: %s", path, ref, a.Error) + case a.Absent: + return nil, fmt.Errorf("%w: %s at %s in %s", ErrNotFound, path, ref, repository(owner, repo)) + case a.Content == nil: + return nil, fmt.Errorf("the seat git's file %s at %s says neither its content nor that it is absent", path, ref) + } + return []byte(*a.Content), nil } } +// errNoAnswer is a read whose answer carries nothing the contract says it must. +var errNoAnswer = errors.New("the seat git's answer carries nothing") + // MergedMessage is the merged message of a merge into the trunk, asked again: built as the bridge builds the one it writes // on FORGE (it is that event's body through ConvertKept), from the pull request's owner, repository, number, branch, head // and merge commit, which the gate's merges state holds. It reads the forge for every module directory (no kept inputs). func MergedMessage(owner, repo string, number int, branch, head, merge string, trees Trees, files Files) (Message, error) { - body, err := json.Marshal(pull{Owner: owner, Repo: repo, Number: number, Head: branch, HeadSHA: head, Base: Trunk, MergeSHA: merge}) + body, err := json.Marshal(git.PullMergedEvent{Repository: repository(owner, repo), Number: int64(number), Branch: branch, Head: head, Base: Trunk, MergeCommit: merge}) if err != nil { return Message{}, err } return ConvertKept(MergedSubject, body, trees, files, nil) } + +// splitRepository is a `/` cut in two, refusing anything else. +func splitRepository(repository string) (string, string, error) { + owner, name, ok := strings.Cut(strings.TrimSpace(repository), "/") + if !ok || owner == "" || name == "" || strings.Contains(name, "/") { + return "", "", fmt.Errorf("repository %q is not /", repository) + } + return owner, name, nil +} diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/forgemade.go b/modules/mesh-delivery-forge-bridge/internal/bridge/forgemade.go index 811504cc..022b74fb 100644 --- a/modules/mesh-delivery-forge-bridge/internal/bridge/forgemade.go +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/forgemade.go @@ -1,10 +1,10 @@ package bridge import ( - "encoding/json" + "context" "fmt" - "net/url" - "strings" + + git "git.novox.be/novox/mesh-delivery/contracts/go/seats/git" ) // ForgePull is a pull request as the forge itself says it: what the bridge needs of a merge made on the forge, not @@ -16,51 +16,30 @@ type ForgePull struct { Merge string // the merge commit into the trunk } -// ForgeMade reads, through the forge module's gitea_api (the path the bridge reads the forge by), the merged pull -// request whose merge commit is merge: by number when number is given (the pull request's merge commit must then equal -// merge), else by the merge commit (the forge's commit-to-pull-request lookup). A pull request not merged, merged into -// another branch than the trunk, or merged at another commit is refused. -func ForgeMade(api API, owner, repo string, number int, merge string) (ForgePull, error) { - path := fmt.Sprintf("/repos/%s/%s/commits/%s/pull", url.PathEscape(owner), url.PathEscape(repo), url.PathEscape(merge)) +// ForgeMade reads, through the seat `git`'s verb pull, the merged pull request whose merge commit is merge: by number +// when number is given (its merge commit must then equal merge), else by the merge commit. A pull request not merged, +// merged into another branch than the trunk, or merged at another commit is refused. +func ForgeMade(ctx context.Context, call Caller, owner, repo string, number int, merge string) (ForgePull, error) { + in := git.PullInput{Repository: repository(owner, repo), MergeCommit: merge} if number > 0 { - path = fmt.Sprintf("/repos/%s/%s/pulls/%d", url.PathEscape(owner), url.PathEscape(repo), number) + in = git.PullInput{Repository: repository(owner, repo), Number: int64(number)} } - raw, err := api(path) + a, err := askGit[git.PullAnswer](ctx, call, git.VerbPull, in) if err != nil { - return ForgePull{}, fmt.Errorf("the forge did not answer %s: %w", path, err) - } - body, err := forgeBody(raw) - if err != nil { - return ForgePull{}, fmt.Errorf("the forge's answer for %s: %w", path, err) - } - var p struct { - Number int `json:"number"` - Merged bool `json:"merged"` - MergeCommitSHA string `json:"merge_commit_sha"` - Head struct { - Label string `json:"label"` - SHA string `json:"sha"` - } `json:"head"` - Base struct { - Ref string `json:"ref"` - } `json:"base"` - } - if err := json.Unmarshal(body, &p); err != nil { - return ForgePull{}, fmt.Errorf("the forge's answer for %s is no pull request: %w", path, err) + return ForgePull{}, err } + p := a.Pull switch { - case p.Number == 0: - return ForgePull{}, fmt.Errorf("the forge's answer for %s is no pull request (no number)", path) - case !p.Merged: - return ForgePull{}, fmt.Errorf("the forge says the pull request %s/%s#%d is not merged", owner, repo, p.Number) - case p.MergeCommitSHA != merge: - return ForgePull{}, fmt.Errorf("the forge says %s/%s#%d was merged at %s, not at %s", owner, repo, p.Number, p.MergeCommitSHA, merge) - case p.Base.Ref != Trunk: - return ForgePull{}, fmt.Errorf("the forge says %s/%s#%d was merged into %s, not the trunk %s", owner, repo, p.Number, p.Base.Ref, Trunk) + case p == nil || p.Number == 0: + return ForgePull{}, fmt.Errorf("the pull request of %s at %s: %w", in.Repository, merge, errNoAnswer) + case p.State != git.PullStateMerged: + return ForgePull{}, fmt.Errorf("the forge says the pull request %s#%d is %s, not merged", in.Repository, p.Number, p.State) + case p.MergeCommit != merge: + return ForgePull{}, fmt.Errorf("the forge says %s#%d was merged at %s, not at %s", in.Repository, p.Number, p.MergeCommit, merge) + case p.Base != Trunk: + return ForgePull{}, fmt.Errorf("the forge says %s#%d was merged into %s, not the trunk %s", in.Repository, p.Number, p.Base, Trunk) + case p.Branch == "": + return ForgePull{}, fmt.Errorf("the forge gives no head branch for %s#%d", in.Repository, p.Number) } - // For a merged pull request the forge's head.ref is "refs/pull/N/head"; the branch is in head.label (same-repository pull requests). - if p.Head.Label == "" || strings.HasPrefix(p.Head.Label, "refs/") { - return ForgePull{}, fmt.Errorf("the forge gives no head branch for %s/%s#%d (label %q)", owner, repo, p.Number, p.Head.Label) - } - return ForgePull{Number: p.Number, Branch: p.Head.Label, Head: p.Head.SHA, Merge: p.MergeCommitSHA}, nil + return ForgePull{Number: int(p.Number), Branch: p.Branch, Head: p.Head, Merge: p.MergeCommit}, nil } diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/tags.go b/modules/mesh-delivery-forge-bridge/internal/bridge/tags.go index b4a942ec..df9d4380 100644 --- a/modules/mesh-delivery-forge-bridge/internal/bridge/tags.go +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/tags.go @@ -1,39 +1,20 @@ package bridge import ( - "encoding/json" + "context" "fmt" - "net/url" "sort" "strings" + + git "git.novox.be/novox/mesh-delivery/contracts/go/seats/git" ) // TagsTool is the forge bridge's tool bridge_tags (novox/hq design 51 *The floor*): a package's git tags, which the // builder's floor reads as one of its three sources. const TagsTool = "bridge_tags" -// TagsPageSize is how many tags one page of the forge's tags answers; TagsMaxPages bounds the read. -const ( - TagsPageSize = 50 - TagsMaxPages = 40 -) - -// API asks the forge one GET of its API (the forge module's gitea_api), answering what the tool answers: the body -// wrapped as {"result": }. Every reader of an API answer takes the body through forgeBody, the one place that -// knows that shape (issue 1966: two readers took the wrapper for the body). -type API func(path string) (json.RawMessage, error) - -// forgeBody is the body of the forge's answer: the tool's "result". An answer without one is refused, naming the shape, -// never read as an empty body. -func forgeBody(raw json.RawMessage) (json.RawMessage, error) { - var w struct { - Result json.RawMessage `json:"result"` - } - if err := json.Unmarshal(raw, &w); err != nil || len(w.Result) == 0 || string(w.Result) == "null" { - return nil, fmt.Errorf(`the forge module's answer is not {"result": }`) - } - return w.Result, nil -} +// TagsMaxPages bounds the read: at most this many pages of the seat `git`'s verb tags. +const TagsMaxPages = 40 // TagsAnswer is what bridge_tags answers. type TagsAnswer struct { @@ -45,39 +26,30 @@ type TagsAnswer struct { } // Tags is bridge_tags: every tag of repository (`/`) whose name starts with prefix (`go/v` for the Go -// package of mesh-sdk's `go` directory, `v` or empty for a package at the root), sorted. A forge that does not answer -// is an error naming the page: the builder then counts this source silent. -func Tags(api API, repository, prefix string) (TagsAnswer, error) { - owner, name, ok := strings.Cut(strings.TrimSpace(repository), "/") - if !ok || owner == "" || name == "" || strings.Contains(name, "/") { - return TagsAnswer{}, fmt.Errorf("repository %q is not /", repository) +// package of mesh-sdk's `go` directory, `v` or empty for a package at the root), sorted, read page by page through the +// seat `git`'s verb tags until a page says no more. A page the seat does not answer is an error naming it: the builder +// then counts this source silent. +func Tags(ctx context.Context, call Caller, repository, prefix string) (TagsAnswer, error) { + owner, name, err := splitRepository(repository) + if err != nil { + return TagsAnswer{}, err } out := TagsAnswer{Repository: owner + "/" + name, Prefix: prefix, Tags: []string{}} - for page := 1; ; page++ { + for page := int64(1); ; page++ { if page > TagsMaxPages { out.More = true break } - raw, err := api(fmt.Sprintf("/repos/%s/%s/tags?page=%d&limit=%d", url.PathEscape(owner), url.PathEscape(name), page, TagsPageSize)) + a, err := askGit[git.TagsAnswer](ctx, call, git.VerbTags, git.TagsInput{Repository: out.Repository, Prefix: prefix, Page: page}) if err != nil { - return TagsAnswer{}, fmt.Errorf("the forge did not answer the tags of %s (page %d): %w", out.Repository, page, err) + return TagsAnswer{}, fmt.Errorf("the tags of %s (page %d): %w", out.Repository, page, err) } - body, err := forgeBody(raw) - if err != nil { - return TagsAnswer{}, fmt.Errorf("the forge's tags of %s (page %d): %w", out.Repository, page, err) - } - var tags []struct { - Name string `json:"name"` - } - if err := json.Unmarshal(body, &tags); err != nil { - return TagsAnswer{}, fmt.Errorf("the forge's tags of %s (page %d) are no list of tags: %w", out.Repository, page, err) - } - for _, t := range tags { - if strings.HasPrefix(t.Name, prefix) { - out.Tags = append(out.Tags, t.Name) + for _, t := range a.Tags { + if strings.HasPrefix(t, prefix) { + out.Tags = append(out.Tags, t) } } - if len(tags) < TagsPageSize { + if !a.More { break } } diff --git a/modules/mesh-delivery-forge-bridge/internal/gitfixture/gitfixture.go b/modules/mesh-delivery-forge-bridge/internal/gitfixture/gitfixture.go new file mode 100644 index 00000000..f3e5f09d --- /dev/null +++ b/modules/mesh-delivery-forge-bridge/internal/gitfixture/gitfixture.go @@ -0,0 +1,42 @@ +// Package gitfixture reads the seat git's own fixtures (contracts/seats/git/fixtures/verbs.json), so the bridge's unit +// fakes answer what the seat's holder answers and nothing the contract does not say (design 47 *The seat `git`*). +package gitfixture + +import ( + "encoding/json" + "os" + "path/filepath" + "runtime" + "testing" +) + +// All is every fixture's body named name of of (such as "file answers", "verb:file.answer"), in the file's order; a +// name with none fails the test. +func All(t testing.TB, name, of string) []json.RawMessage { + t.Helper() + _, here, _, _ := runtime.Caller(0) + raw, err := os.ReadFile(filepath.Join(filepath.Dir(here), "../../../../contracts/seats/git/fixtures/verbs.json")) + if err != nil { + t.Fatal(err) + } + var all []struct { + Name, Of string + Given json.RawMessage + } + if err := json.Unmarshal(raw, &all); err != nil { + t.Fatal(err) + } + var out []json.RawMessage + for _, f := range all { + if f.Name == name && f.Of == of { + out = append(out, f.Given) + } + } + if len(out) == 0 { + t.Fatalf("no fixture %q of %s", name, of) + } + return out +} + +// Given is the first fixture's body named name of of. +func Given(t testing.TB, name, of string) json.RawMessage { t.Helper(); return All(t, name, of)[0] } diff --git a/modules/mesh-delivery-forge-bridge/manifest_test.go b/modules/mesh-delivery-forge-bridge/manifest_test.go index 0f4b8860..7a786eaa 100644 --- a/modules/mesh-delivery-forge-bridge/manifest_test.go +++ b/modules/mesh-delivery-forge-bridge/manifest_test.go @@ -91,15 +91,16 @@ func TestManifestDeclarations(t *testing.T) { // 1235/AC1 t.Errorf("streams = %+v", m.Streams) } // The checker's owed is the forge module's to write (ADR 0521, R4e finding R2-1): the bridge neither hears nor reads it. - // gitea.forge.delete: a branch's end, said as a branch-deleted kind (design 51, DLV-FB1). - if want := []string{"gitea.forge.delete", "gitea.pull.closed", "gitea.pull.merged", "gitea.pull.updated"}; !reflect.DeepEqual(sorted(m.Consumes), want) { + // The forge's events are the seat git's (design 47 *The seat git*); forge.delete: a branch's end, said as a + // branch-deleted kind (design 51, DLV-FB1). + if want := []string{"git.forge-delete", "git.pull-closed", "git.pull-merged", "git.pull-updated"}; !reflect.DeepEqual(sorted(m.Consumes), want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } if want := []string{"mesh-delivery-gate.merges", "mesh-delivery-gate.owed"}; !reflect.DeepEqual(sorted(m.Reads), want) { t.Errorf("reads = %v, want %v", m.Reads, want) } - if want := []string{"gitea.gitea_add_comment", "gitea.gitea_api", "gitea.gitea_commit_status", "gitea.gitea_commit_trees", "gitea.gitea_delivery_view", - "gitea.gitea_get_file", "gitea.gitea_merge_pull_request", "gitea.gitea_note_append", "gitea.gitea_open_pulls"}; !reflect.DeepEqual(sorted(m.Invokes), want) { + if want := []string{"gitea.gitea_add_comment", "gitea.gitea_commit_status", "gitea.gitea_delivery_view", "gitea.gitea_merge_pull_request", + "gitea.gitea_note_append", "seat:git.file", "seat:git.open-pulls", "seat:git.pull", "seat:git.tags", "seat:git.trees"}; !reflect.DeepEqual(sorted(m.Invokes), want) { t.Errorf("invokes = %v, want %v", m.Invokes, want) } if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" { diff --git a/modules/mesh-delivery-forge-bridge/module.json b/modules/mesh-delivery-forge-bridge/module.json index 2b214bd6..0105502f 100644 --- a/modules/mesh-delivery-forge-bridge/module.json +++ b/modules/mesh-delivery-forge-bridge/module.json @@ -3,10 +3,10 @@ "version": "1", "slug": "forge-bridge", "consumes": [ - "gitea.pull.updated", - "gitea.pull.merged", - "gitea.pull.closed", - "gitea.forge.delete" + "git.pull-updated", + "git.pull-merged", + "git.pull-closed", + "git.forge-delete" ], "streams": [ { "name": "forge", "events": ["forge.**"] } @@ -33,10 +33,11 @@ "gitea.gitea_delivery_view", "gitea.gitea_note_append", "gitea.gitea_add_comment", - "gitea.gitea_open_pulls", - "gitea.gitea_commit_trees", - "gitea.gitea_get_file", - "gitea.gitea_api" + "seat:git.open-pulls", + "seat:git.trees", + "seat:git.file", + "seat:git.pull", + "seat:git.tags" ], "own-secrets": { "broker": "${dir:state}/broker" From 0f618f1998770ddbddec8ce5f4cccc1aeffe8bdc Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 11:48:38 +0100 Subject: [PATCH 04/33] WM1 (head half): the watchman raises pipeline-stands for a watched branch's head with no pipeline within five minutes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §12, §75, §112 (the head half; the step half waits on a design question) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-watchman/consumes.go | 17 ++++ modules/mesh-delivery-watchman/go.mod | 2 + .../internal/watch/pipelines.go | 91 +++++++++++++++++++ .../internal/watch/pipelines_test.go | 76 ++++++++++++++++ .../internal/watch/watch.go | 4 +- .../mesh-delivery-watchman/manifest_test.go | 2 +- modules/mesh-delivery-watchman/module.json | 2 + 7 files changed, 192 insertions(+), 2 deletions(-) create mode 100644 modules/mesh-delivery-watchman/internal/watch/pipelines.go create mode 100644 modules/mesh-delivery-watchman/internal/watch/pipelines_test.go diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/consumes.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/consumes.go index a154a11e..fb66bc3d 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/consumes.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/consumes.go @@ -5,6 +5,8 @@ import ( "encoding/json" "git.novox.be/novox/mesh-contracts/go/seats/meshbroker" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner" "strings" "sync" @@ -41,6 +43,7 @@ func consumed(w *watch.Watchman, subject string, body []byte, logf func(string, return } w.Accepted(a) + case pipelineSubject(w, subject, body): case brokerSubject(subject) && strings.HasSuffix(subject, ".event."+eventCreated): if s, err := watch.ReadBrokerSaid(body); err != nil { logf("%v; left", err) @@ -140,3 +143,17 @@ func hearConsumed(ctx context.Context, js jetstream.JetStream, cred process.Acco func brokerSubject(subject string) bool { return strings.HasPrefix(subject, "mesh.seat."+meshbroker.Seat+".event.") } + +// pipelineSubject hands the gate's `source-moved` and the runner's `pipeline-started` to the watchman (watch/pipelines.go, +// design 47 G §12): the head of a watched branch and its pipeline. It says whether the subject was one of them. +func pipelineSubject(w *watch.Watchman, subject string, body []byte) bool { + switch { + case strings.HasPrefix(subject, "mesh.seat."+meshdelivery.Seat+".event."+meshdelivery.EventSourceMoved+"."): + w.SourceMoved(body) + case strings.HasPrefix(subject, "mesh.seat."+meshpipelinerunner.Seat+".event."+meshpipelinerunner.EventPipelineStarted+"."): + w.PipelineStarted(body) + default: + return false + } + return true +} diff --git a/modules/mesh-delivery-watchman/go.mod b/modules/mesh-delivery-watchman/go.mod index 6a4e9198..55006f65 100644 --- a/modules/mesh-delivery-watchman/go.mod +++ b/modules/mesh-delivery-watchman/go.mod @@ -14,6 +14,8 @@ require ( git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/clients/meshwantsclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/modules/mesh-grants/busgrants v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000 diff --git a/modules/mesh-delivery-watchman/internal/watch/pipelines.go b/modules/mesh-delivery-watchman/internal/watch/pipelines.go new file mode 100644 index 00000000..1a04cf0a --- /dev/null +++ b/modules/mesh-delivery-watchman/internal/watch/pipelines.go @@ -0,0 +1,91 @@ +package watch + +import ( + "encoding/json" + "fmt" + "time" + + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner" +) + +// What the runner and the gate say of a watched branch's head, heard as events (design 47 G §12, §75, §112): a head the +// gate says moved (`source-moved`) that starts no pipeline within five minutes is the finding `pipeline-stands`, naming the +// branch and its head, raised by the condition keeper and cleared by the runner's `pipeline-started` for that head. + +// KindPipelineStands is a watched branch's head, or a step, standing still (design 47 G §12, §75). +const KindPipelineStands = "pipeline-stands" + +// HeadBound is how long a watched branch's head may stand with no pipeline started (design 47 G §12, §112: the record's five +// minutes). +const HeadBound = 5 * time.Minute + +// headRow is the signals table's row of a head with no pipeline. +const headRow = "head-pipeline" + +// head is a watched branch's head still waiting for its pipeline. +type head struct{ revision int64 } + +// SourceMoved hears the seat mesh-delivery's `source-moved`: the branch's head is a new source revision, and its pipeline +// is due within HeadBound. A move made by a gathered group is a member governed by the take-in's own bound (ADR 0633 §5), +// so it is not timed here (design 47 G §12). A later move of the same branch replaces the head that waits. +func (w *Watchman) SourceMoved(body []byte) { + var m meshdelivery.SourceMovedEvent + if err := json.Unmarshal(body, &m); err != nil || m.Branch == "" || m.Revision == 0 { + w.say("a source-moved does not read, left: %v", err) + return + } + w.mu.Lock() + defer w.mu.Unlock() + id := headID(m.Branch) + w.unsignal(headRow, id) + w.unfind(FindingKey(ScopeDelivery, id, KindPipelineStands)) + if m.Group != "" { + delete(w.heads, m.Branch) + return + } + at := w.now() + if m.At != nil && !m.At.IsZero() { + at = *m.At + } + w.heads[m.Branch] = head{revision: m.Revision} + f := Finding{Scope: ScopeDelivery, ID: id, Kind: KindPipelineStands, Severity: Warning, Source: headRow, Since: at, + Summary: fmt.Sprintf("the head of %s (source revision %d) has started no pipeline since %s", m.Branch, m.Revision, stamp(at)), + Said: fmt.Sprintf("no pipeline in %s", HeadBound), + Headline: "A change has started no pipeline", + Explanation: "The head of the watched branch " + m.Branch + " (source revision " + fmt.Sprint(m.Revision) + ") moved " + + HeadBound.String() + " ago or more and the pipeline runner has said no pipeline started for it.", + Needs: "the runner to start the pipeline (its pipeline_stands and list tools say where it is)"} + left := HeadBound - w.now().Sub(at) + if left > 0 { + w.signal(headRow, id, left, signalDue{Do: dueFind, Finding: f}) + return + } + w.find(f) +} + +// PipelineStarted hears the runner's `pipeline-started`: it ends the wait of the head it is for (the branch's head at that +// source revision or a later one). A run on request is no head's pipeline. +func (w *Watchman) PipelineStarted(body []byte) { + var p meshpipelinerunner.PipelineStartedEvent + if err := json.Unmarshal(body, &p); err != nil || p.Branch == "" { + w.say("a pipeline-started does not read, left: %v", err) + return + } + if p.Run != "" { + return + } + w.mu.Lock() + defer w.mu.Unlock() + h, ok := w.heads[p.Branch] + if !ok || p.Source < h.revision { + return + } + delete(w.heads, p.Branch) + id := headID(p.Branch) + w.unsignal(headRow, id) + w.unfind(FindingKey(ScopeDelivery, id, KindPipelineStands)) +} + +// headID is the finding's id of a branch's head. +func headID(branch string) string { return "head-" + unsafeKey.ReplaceAllString(branch, "_") } diff --git a/modules/mesh-delivery-watchman/internal/watch/pipelines_test.go b/modules/mesh-delivery-watchman/internal/watch/pipelines_test.go new file mode 100644 index 00000000..82fa5545 --- /dev/null +++ b/modules/mesh-delivery-watchman/internal/watch/pipelines_test.go @@ -0,0 +1,76 @@ +package watch + +import ( + "encoding/json" + "testing" + "time" + + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner" +) + +func moved(branch string, rev int64, group string, at time.Time) []byte { + b, _ := json.Marshal(meshdelivery.SourceMovedEvent{Branch: branch, Revision: rev, Group: group, At: &at}) + return b +} + +func started(branch string, rev int64, run string) []byte { + b, _ := json.Marshal(meshpipelinerunner.PipelineStartedEvent{Branch: branch, Source: rev, Run: run, Module: "m", Pipeline: "p"}) + return b +} + +// design 47 G §12, §79 (test 76): a head with no pipeline within five minutes is `pipeline-stands`, cleared by the +// runner's `pipeline-started` for that head; a gathered group's member and a run on request do not. +func TestAHeadWithNoPipelineStands(t *testing.T) { + key := FindingKey(ScopeDelivery, headID("main"), KindPipelineStands) + cases := []struct { + name string + run func(w *Watchman, o *fakeOut, c *clock) + stand bool + }{ + {"five minutes and no pipeline", func(w *Watchman, o *fakeOut, c *clock) { + w.SourceMoved(moved("main", 7, "", c.t)) + if o.ttls[signalKey(headRow, headID("main"))] != HeadBound || len(o.findings) != 0 { + t.Fatalf("within the bound: %v %v", o.ttls, o.findings) + } + expire(w, c, signalKey(headRow, headID("main")), HeadBound) + }, true}, + {"its pipeline-started clears it", func(w *Watchman, o *fakeOut, c *clock) { + w.SourceMoved(moved("main", 7, "", c.t)) + expire(w, c, signalKey(headRow, headID("main")), HeadBound) + w.PipelineStarted(started("main", 7, "")) + }, false}, + {"a pipeline before the bound raises none", func(w *Watchman, o *fakeOut, c *clock) { + w.SourceMoved(moved("main", 7, "", c.t)) + w.PipelineStarted(started("main", 8, "")) + if _, ok := o.pulses[signalKey(headRow, headID("main"))]; ok { + t.Fatal("the pulse stands") + } + }, false}, + {"a pipeline of an older head does not clear a newer head", func(w *Watchman, o *fakeOut, c *clock) { + w.SourceMoved(moved("main", 7, "", c.t)) + w.PipelineStarted(started("main", 6, "")) + expire(w, c, signalKey(headRow, headID("main")), HeadBound) + }, true}, + {"a gathered group's member is not timed", func(w *Watchman, o *fakeOut, c *clock) { + w.SourceMoved(moved("main", 7, "grp", c.t)) + if len(o.pulses) != 0 { + t.Fatalf("a pulse for a member: %v", o.pulses) + } + }, false}, + {"a run on request is no head's pipeline", func(w *Watchman, o *fakeOut, c *clock) { + w.SourceMoved(moved("main", 7, "", c.t)) + w.PipelineStarted(started("main", 7, "run-1")) + expire(w, c, signalKey(headRow, headID("main")), HeadBound) + }, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + w, o, c := setup() + tc.run(w, o, c) + if _, ok := o.findings[key]; ok != tc.stand { + t.Fatalf("the finding stands %v, want %v: %v", ok, tc.stand, o.findings) + } + }) + } +} diff --git a/modules/mesh-delivery-watchman/internal/watch/watch.go b/modules/mesh-delivery-watchman/internal/watch/watch.go index e73bc987..45fdf1f2 100644 --- a/modules/mesh-delivery-watchman/internal/watch/watch.go +++ b/modules/mesh-delivery-watchman/internal/watch/watch.go @@ -112,6 +112,8 @@ type Watchman struct { // pulses, stuck keys and findings wait for its own end (ReadLate), so a restart while its owner is down neither drops // nor raises again what it holds. awaited map[string]bool + // heads is, by watched branch, the head still waiting for its pipeline (pipelines.go). + heads map[string]head } // New is a watchman writing to out, reading the time from now. @@ -122,7 +124,7 @@ func New(out Out, now func() time.Time) *Watchman { haveState: map[string]nodeengine.HaveByNodeModuleValue{}, unhealthy: map[string]int{}, advised: map[string]int{}, advisedSince: map[string]time.Time{}, control: map[string]bool{}, failedSince: map[string]time.Time{}, provFailed: map[string]map[string]ProvReport{}, closures: map[string]map[string]time.Time{}, waiting: map[string]map[string]string{}, health: map[string]nodeengine.HealthByNodeValue{}, tunnels: map[string]string{}, - standing: map[string]map[string]bool{}, readWhole: map[string]bool{}, awaited: map[string]bool{}} + standing: map[string]map[string]bool{}, readWhole: map[string]bool{}, awaited: map[string]bool{}, heads: map[string]head{}} } // Logf sets where a write that failed is said; it is silent until set. diff --git a/modules/mesh-delivery-watchman/manifest_test.go b/modules/mesh-delivery-watchman/manifest_test.go index 79799276..e312196c 100644 --- a/modules/mesh-delivery-watchman/manifest_test.go +++ b/modules/mesh-delivery-watchman/manifest_test.go @@ -111,7 +111,7 @@ func TestManifestStateAndReads(t *testing.T) { // 1135/AC6 t.Errorf("invokes = %v, want %v", m.Invokes, want) } // DLV-M5 its first views, 1585/T20 the indexer's take-in events, and DLV-W1 (ADR 0595 §4): the accepted verdicts and the bus module's created and refused, durably. - if want := []string{"mesh-delivery-assigner.views.wants", "mesh-delivery-indexer.take-in.pending", "mesh-delivery-indexer.take-in.taken", "mesh-delivery-indexer.views.modules", "mesh-catalog.manifest-accepted", "mesh-broker.created", "mesh-broker.refused"}; !reflect.DeepEqual(m.Consumes, want) { + if want := []string{"mesh-delivery-assigner.views.wants", "mesh-delivery-indexer.take-in.pending", "mesh-delivery-indexer.take-in.taken", "mesh-delivery-indexer.views.modules", "mesh-catalog.manifest-accepted", "mesh-delivery.source-moved", "mesh-pipeline-runner.pipeline-started", "mesh-broker.created", "mesh-broker.refused"}; !reflect.DeepEqual(m.Consumes, want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } // 1952/WM1: the watchman holds the seat mesh-watchdogs at mesh scope, its first seat (novox/hq ADR 0564 §1, §4). diff --git a/modules/mesh-delivery-watchman/module.json b/modules/mesh-delivery-watchman/module.json index 25acacd9..9629fa8f 100644 --- a/modules/mesh-delivery-watchman/module.json +++ b/modules/mesh-delivery-watchman/module.json @@ -36,6 +36,8 @@ "mesh-delivery-indexer.take-in.taken", "mesh-delivery-indexer.views.modules", "mesh-catalog.manifest-accepted", + "mesh-delivery.source-moved", + "mesh-pipeline-runner.pipeline-started", "mesh-broker.created", "mesh-broker.refused" ], From 69aeb817e229c61e78a481c3ba99db9b2901ae3c Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 11:49:32 +0100 Subject: [PATCH 05/33] WM2: the watchman raises build-capacity-waits for a build seat whose oldest queued step has waited 60 minutes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §86 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-watchman/consumes.go | 3 + .../internal/watch/pipelines.go | 78 +++++++++++++++++++ .../internal/watch/pipelines_test.go | 37 +++++++++ .../internal/watch/watch.go | 4 +- .../mesh-delivery-watchman/manifest_test.go | 2 +- modules/mesh-delivery-watchman/module.json | 2 + 6 files changed, 124 insertions(+), 2 deletions(-) diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/consumes.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/consumes.go index fb66bc3d..f2ccce58 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/consumes.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/consumes.go @@ -152,6 +152,9 @@ func pipelineSubject(w *watch.Watchman, subject string, body []byte) bool { w.SourceMoved(body) case strings.HasPrefix(subject, "mesh.seat."+meshpipelinerunner.Seat+".event."+meshpipelinerunner.EventPipelineStarted+"."): w.PipelineStarted(body) + case strings.HasPrefix(subject, "mesh.seat."+meshpipelinerunner.Seat+".event."+meshpipelinerunner.EventStepStarted+"."), + strings.HasPrefix(subject, "mesh.seat."+meshpipelinerunner.Seat+".event."+meshpipelinerunner.EventStepEnded+"."): + w.StepEvent(body) default: return false } diff --git a/modules/mesh-delivery-watchman/internal/watch/pipelines.go b/modules/mesh-delivery-watchman/internal/watch/pipelines.go index 1a04cf0a..ed9e62f2 100644 --- a/modules/mesh-delivery-watchman/internal/watch/pipelines.go +++ b/modules/mesh-delivery-watchman/internal/watch/pipelines.go @@ -89,3 +89,81 @@ func (w *Watchman) PipelineStarted(body []byte) { // headID is the finding's id of a branch's head. func headID(branch string) string { return "head-" + unsafeKey.ReplaceAllString(branch, "_") } + +// KindBuildCapacityWaits is a build seat whose oldest queued step has waited the bound (design 47 G §86). +const KindBuildCapacityWaits = "build-capacity-waits" + +// CapacityBound is how long a step may wait queued on a build seat's stream before the seat is a finding (design 47 G §86). +const CapacityBound = 60 * time.Minute + +// capacityRow is the signals table's row of a build seat's queue. +const capacityRow = "build-capacity" + +// StepEvent hears the runner's `step-started` and `step-ended` (design 47 G §86): a step said waiting for a build node is +// queued on its seat, owed no progress; it leaves the queue when it starts running or ends. A seat's finding is due when +// its oldest queued step has waited CapacityBound, and ends when that seat has none that old. +func (w *Watchman) StepEvent(body []byte) { + var e meshpipelinerunner.StepEvent + if err := json.Unmarshal(body, &e); err != nil || e.Pipeline == "" || e.Result.Step == "" { + w.say("a step event does not read, left: %v", err) + return + } + id := e.Pipeline + "/" + e.Result.Step + w.mu.Lock() + defer w.mu.Unlock() + for seat, q := range w.queued { + if _, ok := q[id]; ok && (e.Result.Outcome != meshpipelinerunner.StepResultOutcomeWaitingForABuildNode || e.Result.Seat != seat) { + delete(q, id) + w.capacity(seat) + } + } + if e.Result.Outcome != meshpipelinerunner.StepResultOutcomeWaitingForABuildNode || e.Result.Seat == "" { + return + } + since := w.now() + switch { + case e.Result.Started != nil: + since = *e.Result.Started + case e.At != nil: + since = *e.At + } + seat := e.Result.Seat + if w.queued[seat] == nil { + w.queued[seat] = map[string]time.Time{} + } + if _, ok := w.queued[seat][id]; !ok { + w.queued[seat][id] = since + } + w.capacity(seat) +} + +// capacity judges a build seat's queue again, w.mu held: one signal for its oldest queued step. +func (w *Watchman) capacity(seat string) { + w.unsignal(capacityRow, seat) + key := FindingKey(ScopeDelivery, seat, KindBuildCapacityWaits) + var oldest time.Time + for _, t := range w.queued[seat] { + if oldest.IsZero() || t.Before(oldest) { + oldest = t + } + } + if oldest.IsZero() { + delete(w.queued, seat) + w.unfind(key) + return + } + f := Finding{Scope: ScopeDelivery, ID: seat, Kind: KindBuildCapacityWaits, Severity: Warning, Source: capacityRow, Since: oldest, + Summary: fmt.Sprintf("the oldest step queued on %s has waited since %s", seat, stamp(oldest)), + Said: "queued for " + CapacityBound.String(), + Headline: "Builds wait for a build node", + Explanation: "The oldest step queued on the build seat " + seat + " has waited " + CapacityBound.String() + + " for a build node to take it.", + Needs: "build capacity: a build node present and not paused, or fewer steps asked"} + left := CapacityBound - w.now().Sub(oldest) + if left > 0 { + w.unfind(key) + w.signal(capacityRow, seat, left, signalDue{Do: dueFind, Finding: f}) + return + } + w.find(f) +} diff --git a/modules/mesh-delivery-watchman/internal/watch/pipelines_test.go b/modules/mesh-delivery-watchman/internal/watch/pipelines_test.go index 82fa5545..a5aa1afa 100644 --- a/modules/mesh-delivery-watchman/internal/watch/pipelines_test.go +++ b/modules/mesh-delivery-watchman/internal/watch/pipelines_test.go @@ -74,3 +74,40 @@ func TestAHeadWithNoPipelineStands(t *testing.T) { }) } } + +func queuedStep(pipeline, step, seat, outcome string, started time.Time) []byte { + b, _ := json.Marshal(meshpipelinerunner.StepEvent{Pipeline: pipeline, Branch: "main", Module: "m", Commit: "c", + Result: meshpipelinerunner.StepResult{Step: step, Seat: seat, Outcome: outcome, Started: &started}}) + return b +} + +// design 47 G §86: a build seat whose oldest queued step has waited 60 minutes raises one `build-capacity-waits`, cleared +// when that step starts. +func TestABuildSeatsOldestQueuedStepWaitsAnHour(t *testing.T) { + const seat = "node-build-go" + key := FindingKey(ScopeDelivery, seat, KindBuildCapacityWaits) + wait := meshpipelinerunner.StepResultOutcomeWaitingForABuildNode + w, o, c := setup() + w.StepEvent(queuedStep("p1", "unit", seat, wait, c.t)) + c.t = c.t.Add(10 * time.Minute) + w.StepEvent(queuedStep("p2", "unit", seat, wait, c.t)) + pk := signalKey(capacityRow, seat) + if o.ttls[pk] != 50*time.Minute || len(o.findings) != 0 { + t.Fatalf("within the bound: %v %v", o.ttls, o.findings) + } + expire(w, c, pk, 50*time.Minute) + if _, ok := o.findings[key]; !ok || len(o.findings) != 1 { + t.Fatalf("an hour for the oldest: %v", o.findings) + } + w.StepEvent(queuedStep("p1", "unit", seat, meshpipelinerunner.StepResultOutcomeRunning, c.t)) + if len(o.findings) != 0 { + t.Fatalf("the oldest started, the next has waited 50 minutes: %v", o.findings) + } + if o.ttls[pk] != 10*time.Minute { + t.Fatalf("the next one's hour: %v", o.ttls[pk]) + } + w.StepEvent(queuedStep("p2", "unit", seat, meshpipelinerunner.StepResultOutcomePassed, c.t)) + if _, ok := o.pulses[pk]; ok || len(o.findings) != 0 { + t.Fatalf("the queue is empty: %v %v", o.pulses, o.findings) + } +} diff --git a/modules/mesh-delivery-watchman/internal/watch/watch.go b/modules/mesh-delivery-watchman/internal/watch/watch.go index 45fdf1f2..41083c1d 100644 --- a/modules/mesh-delivery-watchman/internal/watch/watch.go +++ b/modules/mesh-delivery-watchman/internal/watch/watch.go @@ -114,6 +114,8 @@ type Watchman struct { awaited map[string]bool // heads is, by watched branch, the head still waiting for its pipeline (pipelines.go). heads map[string]head + // queued is, by build seat, each step queued on it for a build node and since when (pipelines.go). + queued map[string]map[string]time.Time } // New is a watchman writing to out, reading the time from now. @@ -124,7 +126,7 @@ func New(out Out, now func() time.Time) *Watchman { haveState: map[string]nodeengine.HaveByNodeModuleValue{}, unhealthy: map[string]int{}, advised: map[string]int{}, advisedSince: map[string]time.Time{}, control: map[string]bool{}, failedSince: map[string]time.Time{}, provFailed: map[string]map[string]ProvReport{}, closures: map[string]map[string]time.Time{}, waiting: map[string]map[string]string{}, health: map[string]nodeengine.HealthByNodeValue{}, tunnels: map[string]string{}, - standing: map[string]map[string]bool{}, readWhole: map[string]bool{}, awaited: map[string]bool{}, heads: map[string]head{}} + standing: map[string]map[string]bool{}, readWhole: map[string]bool{}, awaited: map[string]bool{}, heads: map[string]head{}, queued: map[string]map[string]time.Time{}} } // Logf sets where a write that failed is said; it is silent until set. diff --git a/modules/mesh-delivery-watchman/manifest_test.go b/modules/mesh-delivery-watchman/manifest_test.go index e312196c..826e749b 100644 --- a/modules/mesh-delivery-watchman/manifest_test.go +++ b/modules/mesh-delivery-watchman/manifest_test.go @@ -111,7 +111,7 @@ func TestManifestStateAndReads(t *testing.T) { // 1135/AC6 t.Errorf("invokes = %v, want %v", m.Invokes, want) } // DLV-M5 its first views, 1585/T20 the indexer's take-in events, and DLV-W1 (ADR 0595 §4): the accepted verdicts and the bus module's created and refused, durably. - if want := []string{"mesh-delivery-assigner.views.wants", "mesh-delivery-indexer.take-in.pending", "mesh-delivery-indexer.take-in.taken", "mesh-delivery-indexer.views.modules", "mesh-catalog.manifest-accepted", "mesh-delivery.source-moved", "mesh-pipeline-runner.pipeline-started", "mesh-broker.created", "mesh-broker.refused"}; !reflect.DeepEqual(m.Consumes, want) { + if want := []string{"mesh-delivery-assigner.views.wants", "mesh-delivery-indexer.take-in.pending", "mesh-delivery-indexer.take-in.taken", "mesh-delivery-indexer.views.modules", "mesh-catalog.manifest-accepted", "mesh-delivery.source-moved", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.step-started", "mesh-pipeline-runner.step-ended", "mesh-broker.created", "mesh-broker.refused"}; !reflect.DeepEqual(m.Consumes, want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } // 1952/WM1: the watchman holds the seat mesh-watchdogs at mesh scope, its first seat (novox/hq ADR 0564 §1, §4). diff --git a/modules/mesh-delivery-watchman/module.json b/modules/mesh-delivery-watchman/module.json index 9629fa8f..35bf7289 100644 --- a/modules/mesh-delivery-watchman/module.json +++ b/modules/mesh-delivery-watchman/module.json @@ -38,6 +38,8 @@ "mesh-catalog.manifest-accepted", "mesh-delivery.source-moved", "mesh-pipeline-runner.pipeline-started", + "mesh-pipeline-runner.step-started", + "mesh-pipeline-runner.step-ended", "mesh-broker.created", "mesh-broker.refused" ], From 799241364fea7d581c67cf9a7988c6f6192c5653 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 11:50:04 +0100 Subject: [PATCH 06/33] VW0: the view reads the trunk's source by the branch key and the per-branch value MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §305 (the state source is kept per watched branch), §431 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-view/integration_test.go | 4 ++-- modules/mesh-delivery-view/internal/view/names.go | 5 +++++ modules/mesh-delivery-view/internal/view/reach.go | 2 +- modules/mesh-delivery-view/internal/view/reach_test.go | 6 +++--- modules/mesh-delivery-view/internal/view/tiers.go | 3 +-- modules/mesh-delivery-view/internal/view/view.go | 2 +- modules/mesh-delivery-view/internal/view/wants_test.go | 4 ++-- 7 files changed, 15 insertions(+), 11 deletions(-) diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view/integration_test.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view/integration_test.go index cd326086..77ad3d76 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view/integration_test.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view/integration_test.go @@ -797,7 +797,7 @@ func TestProcessWholeSaysTheSourceRevisionIsDelivered(t *testing.T) { want, _ := meshwantsclient.EncodeWant(meshwantsclient.Want{Fp: fp, Composed: "sha256:" + strings.Repeat("c", 64)}) failed, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed}) good, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}) - src, _ := meshdeliveryclient.EncodeSource(meshdelivery.SourceByMainValue{Group: "feat/x", Heads: map[string]string{"novox/web": strings.Repeat("1", 40)}}) + src, _ := meshdeliveryclient.EncodeSource(meshdelivery.SourceByBranchValue{Group: "feat/x", Heads: map[string]string{"novox/web": strings.Repeat("1", 40)}}) for bucket, kv := range map[string][]byte{view.ModulesBucket: mod} { if _, err := kvs[bucket].Put(ctx, "web", kv); err != nil { t.Fatal(err) @@ -809,7 +809,7 @@ func TestProcessWholeSaysTheSourceRevisionIsDelivered(t *testing.T) { if _, err := kvs[view.HaveBucket].Put(ctx, "ace.web", failed); err != nil { t.Fatal(err) } - rev, err := kvs[view.SourceBucket].Put(ctx, meshdeliveryclient.SourceMainKey, src) + rev, err := kvs[view.SourceBucket].Put(ctx, meshdeliveryclient.SourceKey("main"), src) if err != nil { t.Fatal(err) } diff --git a/modules/mesh-delivery-view/internal/view/names.go b/modules/mesh-delivery-view/internal/view/names.go index 0fc71ca2..4a2d62f1 100644 --- a/modules/mesh-delivery-view/internal/view/names.go +++ b/modules/mesh-delivery-view/internal/view/names.go @@ -3,6 +3,7 @@ package view import ( "fmt" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/sdk/trunk" "git.novox.be/novox/mesh-sdk/go/keys" @@ -88,3 +89,7 @@ func PackagesKey(module string) (string, error) { } return meshcatalog.PackagesByModuleKey(keys.EscapeName(module)), nil } + +// trunkSourceKey is the key of the gate's state source for the trunk, the watched branch `main` (design 47 G §305: the source +// is kept per watched branch, meshdeliveryclient.SourceKey; the view's reads are of the trunk's). +var trunkSourceKey = meshdeliveryclient.SourceKey("main") diff --git a/modules/mesh-delivery-view/internal/view/reach.go b/modules/mesh-delivery-view/internal/view/reach.go index b11e0648..d74e058c 100644 --- a/modules/mesh-delivery-view/internal/view/reach.go +++ b/modules/mesh-delivery-view/internal/view/reach.go @@ -71,7 +71,7 @@ func (m *Model) applyHistory(e Entry) { } m.modHist[e.Key] = h case SourceBucket: - if e.Key != meshdeliveryclient.SourceMainKey || e.Deleted || e.Revision == 0 { + if e.Key != trunkSourceKey || e.Deleted || e.Revision == 0 { return } src, err := meshdeliveryclient.DecodeSource(e.Value) diff --git a/modules/mesh-delivery-view/internal/view/reach_test.go b/modules/mesh-delivery-view/internal/view/reach_test.go index 12e32744..5d53d583 100644 --- a/modules/mesh-delivery-view/internal/view/reach_test.go +++ b/modules/mesh-delivery-view/internal/view/reach_test.go @@ -49,11 +49,11 @@ func TestTheViewDerivesHowFarEachRepositoryReached(t *testing.T) { return append([]Entry{modIn(fpLater, commit, 12, to, true), modIn(fpLater, commit, 12, to, false)}, extra...) } src := func(rev uint64, webHead string) Entry { - v, err := meshdeliveryclient.EncodeSource(meshdelivery.SourceByMainValue{Group: "g", Heads: map[string]string{web: webHead, sdk: commitS}}) + v, err := meshdeliveryclient.EncodeSource(meshdelivery.SourceByBranchValue{Group: "g", Heads: map[string]string{web: webHead, sdk: commitS}}) if err != nil { t.Fatal(err) } - return Entry{Bucket: SourceBucket, Key: meshdeliveryclient.SourceMainKey, Value: v, At: t0, Revision: rev, History: true} + return Entry{Bucket: SourceBucket, Key: trunkSourceKey, Value: v, At: t0, Revision: rev, History: true} } taken := func(repository, merge string) Entry { key, _ := trunk.RepositoryKey(repository) @@ -72,7 +72,7 @@ func TestTheViewDerivesHowFarEachRepositoryReached(t *testing.T) { // web changed at A (fpOld) and at C (fpNew); mail first appeared at A. The source named A, B, then C as web's head. world := func(t *testing.T) []Entry { cur, _ := meshdeliveryclient.DecodeSource(src(12, commitC).Value) - current := Entry{Bucket: SourceBucket, Key: meshdeliveryclient.SourceMainKey, Revision: 12, At: t0} + current := Entry{Bucket: SourceBucket, Key: trunkSourceKey, Revision: 12, At: t0} current.Value, _ = meshdeliveryclient.EncodeSource(cur) return []Entry{src(10, commitA), src(11, commitB), src(12, commitC), current, mod("web", fpOld, commitA, 10, true), modIn(fpNew, commitC, 12, in("t1", "mesh-sdk/go"), true), mod("mail", fpMail, commitA, 0, true), diff --git a/modules/mesh-delivery-view/internal/view/tiers.go b/modules/mesh-delivery-view/internal/view/tiers.go index a579c044..1d720ef5 100644 --- a/modules/mesh-delivery-view/internal/view/tiers.go +++ b/modules/mesh-delivery-view/internal/view/tiers.go @@ -13,7 +13,6 @@ import ( "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshwatchdogsclient" "git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshwantsclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" @@ -83,7 +82,7 @@ func (r Reader) Tiers(revision uint64) (Tiers, error) { if r.SourceState.GetEntry == nil { return Tiers{}, fmt.Errorf("the gate's source cannot be read here: name the source revision") } - _, rev, err := r.SourceState.GetEntry(meshdeliveryclient.SourceMainKey) + _, rev, err := r.SourceState.GetEntry(trunkSourceKey) if err != nil { return Tiers{}, fmt.Errorf("the gate's source: %w", err) } diff --git a/modules/mesh-delivery-view/internal/view/view.go b/modules/mesh-delivery-view/internal/view/view.go index 35f1c077..49db0b8e 100644 --- a/modules/mesh-delivery-view/internal/view/view.go +++ b/modules/mesh-delivery-view/internal/view/view.go @@ -621,7 +621,7 @@ func (m *Model) apply(e Entry) { delete(m.taken, e.Key) } case SourceBucket: - if e.Key != meshdeliveryclient.SourceMainKey { + if e.Key != trunkSourceKey { return } if e.Deleted { diff --git a/modules/mesh-delivery-view/internal/view/wants_test.go b/modules/mesh-delivery-view/internal/view/wants_test.go index 6599683e..23a84f33 100644 --- a/modules/mesh-delivery-view/internal/view/wants_test.go +++ b/modules/mesh-delivery-view/internal/view/wants_test.go @@ -61,11 +61,11 @@ func moduleE(t *testing.T, module, fp string) Entry { func sourceE(t *testing.T, revision uint64) Entry { t.Helper() - v, err := meshdeliveryclient.EncodeSource(meshdelivery.SourceByMainValue{Group: "g", Heads: map[string]string{"novox/web": strings.Repeat("1", 40)}}) + v, err := meshdeliveryclient.EncodeSource(meshdelivery.SourceByBranchValue{Group: "g", Heads: map[string]string{"novox/web": strings.Repeat("1", 40)}}) if err != nil { t.Fatal(err) } - return Entry{Bucket: SourceBucket, Key: meshdeliveryclient.SourceMainKey, Value: v, At: t0, Revision: revision} + return Entry{Bucket: SourceBucket, Key: trunkSourceKey, Value: v, At: t0, Revision: revision} } // 1906/D7 AC7: every want is a row beside the have of its node, with where it stands. From 43bebef626f35fde2d46afecec206e34db64f1cc Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 11:52:13 +0100 Subject: [PATCH 07/33] WM3: the watchman says machine-away and machine-back of the seat mesh-watchdogs when a machine's presence expires and when it is heard again MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §389, §436 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-watchman/main.go | 19 +++++++++- .../cmd/mesh-delivery-watchman/main_test.go | 1 + .../internal/watch/pipelines_test.go | 26 +++++++++++++ .../internal/watch/presence.go | 37 +++++++++++++++++++ .../internal/watch/signals.go | 5 +++ .../internal/watch/watch.go | 6 ++- .../internal/watch/watch_test.go | 8 ++++ 7 files changed, 100 insertions(+), 2 deletions(-) diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go index 3fb22fe7..dc99c4ef 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go @@ -55,6 +55,23 @@ func main() { type kvOut struct { ctx context.Context presence, pulse, stuk, findings jetstream.KeyValue + js jetstream.JetStream + cred process.Account +} + +// Say publishes an event of the seat mesh-watchdogs, said under the watchman's own account (design 47 G §389, §436). +func (o *kvOut) Say(subject string, body []byte, id string) error { + msg := nats.NewMsg(subject) + msg.Header.Set("x-event-id", id) + msg.Header.Set("x-source", o.cred.Module) + msg.Header.Set("x-node", o.cred.Node) + msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano)) + msg.Header.Set("content-type", "application/json") + msg.Data = body + pub, cancel := context.WithTimeout(context.WithoutCancel(o.ctx), 10*time.Second) + defer cancel() + _, err := o.js.PublishMsg(pub, msg, jetstream.WithMsgID(id)) + return err } func (o *kvOut) PutFinding(key string, value []byte) error { @@ -251,7 +268,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return err } - w := watch.New(&kvOut{ctx: ctx, presence: kvs.presence, pulse: kvs.pulse, stuk: kvs.stuck, findings: kvs.findings}, time.Now) + w := watch.New(&kvOut{ctx: ctx, presence: kvs.presence, pulse: kvs.pulse, stuk: kvs.stuck, findings: kvs.findings, js: js, cred: cred}, time.Now) w.Logf(logger.Printf) // What the wants and the modules hold is ended by their own late read (watch.ReadLate), never by the start's end. w.Awaits(watch.AssignerWantsBucket) diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go index a3f306da..a0f47f2d 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go @@ -152,5 +152,6 @@ func (o *findingsOut) PutPresence(string, bool) error { return nil func (o *findingsOut) PutPulse(string, []byte, time.Duration) error { return nil } func (o *findingsOut) DeletePulse(string) error { o.pulseDeletes++; return nil } func (o *findingsOut) PutStuck(string, []byte) error { return nil } +func (o *findingsOut) Say(string, []byte, string) error { return nil } func (o *findingsOut) PutFinding(key string, _ []byte) error { o.findings[key] = true; return nil } func (o *findingsOut) DeleteFinding(key string) error { delete(o.findings, key); return nil } diff --git a/modules/mesh-delivery-watchman/internal/watch/pipelines_test.go b/modules/mesh-delivery-watchman/internal/watch/pipelines_test.go index a5aa1afa..c9cb3b38 100644 --- a/modules/mesh-delivery-watchman/internal/watch/pipelines_test.go +++ b/modules/mesh-delivery-watchman/internal/watch/pipelines_test.go @@ -111,3 +111,29 @@ func TestABuildSeatsOldestQueuedStepWaitsAnHour(t *testing.T) { t.Fatalf("the queue is empty: %v %v", o.pulses, o.findings) } } + +// design 47 G §389, §436 (test 106's cause): a machine whose presence key expires says `machine-away`, once, and one +// heard again says `machine-back`; a machine heard again that was never away says nothing. +func TestAMachinesPresenceIsSaidAsEvents(t *testing.T) { + w, o, c := setup() + w.Connect("node.ace") + w.Connect("node.ace") // heard again while never away + w.Disconnect("node.ace") + w.Disconnect("node.ace") + if len(o.said) != 0 { + t.Fatalf("said before the expiry: %v", o.said) + } + c.t = t0.Add(120 * time.Second) + w.Pulse(pulseKeyOf(t, presenceNS, "ace"), Expired, c.t) + w.Pulse(pulseKeyOf(t, presenceNS, "ace"), Expired, c.t) // the same expiry heard twice + w.Connect("node.ace") + w.Connect("node.ace") + got := []string{} + for _, s := range o.said { + got = append(got, s.subject) + } + want := []string{"mesh.seat.mesh-watchdogs.event.machine-away.ace", "mesh.seat.mesh-watchdogs.event.machine-back.ace"} + if len(got) != 2 || got[0] != want[0] || got[1] != want[1] { + t.Fatalf("said %v, want %v", got, want) + } +} diff --git a/modules/mesh-delivery-watchman/internal/watch/presence.go b/modules/mesh-delivery-watchman/internal/watch/presence.go index 5fe16e07..b9881a31 100644 --- a/modules/mesh-delivery-watchman/internal/watch/presence.go +++ b/modules/mesh-delivery-watchman/internal/watch/presence.go @@ -8,6 +8,7 @@ import ( "time" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshwatchdogsclient" + "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs" ) // partPrefix is what the watchman keeps presence of among the module users: the parts of delivery. @@ -186,3 +187,39 @@ func (w *Watchman) Rebuild(ask func(body []byte) ([]byte, error)) error { w.Connections(users) return nil } + +// machineAway says `machine-away` of the seat mesh-watchdogs when a machine's presence key has expired (design 47 G §389, §436): +// the machine, when it was last heard and when this was said, once until the machine is heard again. +func (w *Watchman) machineAway(node string, lastHeard time.Time) { + if _, said := w.awaySaid[node]; said { + return + } + at := w.now().UTC() + body, err := json.Marshal(meshwatchdogs.MachineAwayEvent{V: 1, Machine: node, At: at, LastHeard: lastHeard.UTC()}) + if err != nil { + return + } + if err := w.out.Say(meshwatchdogs.SubjectMachineAway(node), body, "machine-away."+node+"."+at.Format(time.RFC3339Nano)); err != nil { + w.say("cannot say machine-away of %s: %v", node, err) + return + } + w.awaySaid[node] = lastHeard.UTC() +} + +// machineBack says `machine-back` when a machine whose machine-away was said is heard again (design 47 G §389). +func (w *Watchman) machineBack(node string) { + lastHeard, said := w.awaySaid[node] + if !said { + return + } + at := w.now().UTC() + body, err := json.Marshal(meshwatchdogs.MachineBackEvent{V: 1, Machine: node, At: at, LastHeard: lastHeard}) + if err != nil { + return + } + if err := w.out.Say(meshwatchdogs.SubjectMachineBack(node), body, "machine-back."+node+"."+at.Format(time.RFC3339Nano)); err != nil { + w.say("cannot say machine-back of %s: %v", node, err) + return + } + delete(w.awaySaid, node) +} diff --git a/modules/mesh-delivery-watchman/internal/watch/signals.go b/modules/mesh-delivery-watchman/internal/watch/signals.go index 92b43f17..2d9e7913 100644 --- a/modules/mesh-delivery-watchman/internal/watch/signals.go +++ b/modules/mesh-delivery-watchman/internal/watch/signals.go @@ -254,6 +254,11 @@ func (w *Watchman) stuckFinding(sk string, s meshwatchdogsclient.Stuck) { if s.Bucket == PresenceBucket { if node, part, _, err := meshwatchdogsclient.ParsePresenceKey(s.Key); err == nil && part == "" && node != "" { key := FindingKey(ScopeMachine, node, KindSilent) + if s.Moved { + w.machineBack(node) + } else { + w.machineAway(node, s.Since) + } defer w.networks() // a silent machine is down on the record: what points at it is held there, said by its silence if s.Moved || w.away[node] { w.unfind(key) diff --git a/modules/mesh-delivery-watchman/internal/watch/watch.go b/modules/mesh-delivery-watchman/internal/watch/watch.go index 41083c1d..9a05ef6e 100644 --- a/modules/mesh-delivery-watchman/internal/watch/watch.go +++ b/modules/mesh-delivery-watchman/internal/watch/watch.go @@ -36,6 +36,8 @@ type Out interface { // PutFinding writes one of the watchman's findings; DeleteFinding deletes one it no longer observes. PutFinding(key string, value []byte) error DeleteFinding(key string) error + // Say publishes an event of the seat mesh-watchdogs on its subject (design 47 G §389), once under its id. + Say(subject string, body []byte, id string) error } type presence struct { @@ -116,6 +118,8 @@ type Watchman struct { heads map[string]head // queued is, by build seat, each step queued on it for a build node and since when (pipelines.go). queued map[string]map[string]time.Time + // awaySaid is each machine whose machine-away was said and not yet its machine-back, with when it was last heard (presence.go). + awaySaid map[string]time.Time } // New is a watchman writing to out, reading the time from now. @@ -126,7 +130,7 @@ func New(out Out, now func() time.Time) *Watchman { haveState: map[string]nodeengine.HaveByNodeModuleValue{}, unhealthy: map[string]int{}, advised: map[string]int{}, advisedSince: map[string]time.Time{}, control: map[string]bool{}, failedSince: map[string]time.Time{}, provFailed: map[string]map[string]ProvReport{}, closures: map[string]map[string]time.Time{}, waiting: map[string]map[string]string{}, health: map[string]nodeengine.HealthByNodeValue{}, tunnels: map[string]string{}, - standing: map[string]map[string]bool{}, readWhole: map[string]bool{}, awaited: map[string]bool{}, heads: map[string]head{}, queued: map[string]map[string]time.Time{}} + standing: map[string]map[string]bool{}, readWhole: map[string]bool{}, awaited: map[string]bool{}, heads: map[string]head{}, queued: map[string]map[string]time.Time{}, awaySaid: map[string]time.Time{}} } // Logf sets where a write that failed is said; it is silent until set. diff --git a/modules/mesh-delivery-watchman/internal/watch/watch_test.go b/modules/mesh-delivery-watchman/internal/watch/watch_test.go index 004b7ad9..740d3834 100644 --- a/modules/mesh-delivery-watchman/internal/watch/watch_test.go +++ b/modules/mesh-delivery-watchman/internal/watch/watch_test.go @@ -26,6 +26,14 @@ type fakeOut struct { stuckWrites int findings map[string]Finding findingWrites int + said []said +} + +type said struct{ subject, id string } + +func (o *fakeOut) Say(subject string, _ []byte, id string) error { + o.said = append(o.said, said{subject, id}) + return nil } func newOut() *fakeOut { From 61277b6b9b47e651f8e7b19fcfbf2f9574d2ca4b Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 11:53:41 +0100 Subject: [PATCH 08/33] VW4: the view's manifest says version 0.1, declares its finding stream and hears the keeper's ask MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §383, §387, §368, §411 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- modules/mesh-delivery-view/manifest_test.go | 17 ++++++++++++++++- modules/mesh-delivery-view/module.json | 11 ++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/modules/mesh-delivery-view/manifest_test.go b/modules/mesh-delivery-view/manifest_test.go index a2e58671..b35cf6f8 100644 --- a/modules/mesh-delivery-view/manifest_test.go +++ b/modules/mesh-delivery-view/manifest_test.go @@ -51,6 +51,13 @@ type state struct { History int `json:"history,omitempty"` } +type stream struct { + Name string `json:"name"` + Events []string `json:"events"` + MaxAgeDays int `json:"max-age-days"` + MaxMiB int `json:"max-mib"` +} + type manifest struct { State []state `json:"state"` Module string `json:"module"` @@ -63,6 +70,7 @@ type manifest struct { Uses []string `json:"uses"` Tools []string `json:"tools"` Emits []string `json:"emits"` + Streams []stream `json:"streams"` OwnSecrets map[string]string `json:"own-secrets"` SecretsOwner string `json:"secrets-owner"` Resources []map[string]any `json:"resources"` @@ -117,7 +125,14 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7 "seat:node-engine.manifest-reads"}; !reflect.DeepEqual(m.Invokes, want) { // ADR 0618 §3: node_manifest_reads asks the node-engine seat's verb of every node; 1585/T24: view_tiers asks indexer_pending, a verb edge (ADR 0614 §2) t.Errorf("invokes = %v, want %v", m.Invokes, want) } - if want := []string{"mesh-delivery-assigner.views.asks", "mesh-delivery-assigner.views.assignments", "mesh-delivery-assigner.views.wants", + // design 47 G §387: the keeper's ask for the findings is heard, the finding stream declared (§383), and §368: version is MAJOR.MINOR. + if want := []stream{{Name: "mesh-delivery-view-findings", Events: []string{"finding-written.*", "finding-cleared.*", "findings-stated.*"}, MaxAgeDays: 7, MaxMiB: 16}}; !reflect.DeepEqual(m.Streams, want) { + t.Errorf("streams = %+v, want %+v", m.Streams, want) + } + if m.Version != "0.1" { + t.Errorf("version = %q, want 0.1", m.Version) + } + if want := []string{"mesh-conditions.findings-asked", "mesh-delivery-assigner.views.asks", "mesh-delivery-assigner.views.assignments", "mesh-delivery-assigner.views.wants", "mesh-delivery-indexer.views.candidates", "mesh-delivery-indexer.views.modules", "mesh-delivery-indexer.views.taken"}; !reflect.DeepEqual(m.Consumes, want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } diff --git a/modules/mesh-delivery-view/module.json b/modules/mesh-delivery-view/module.json index 81d90b74..a8544e42 100644 --- a/modules/mesh-delivery-view/module.json +++ b/modules/mesh-delivery-view/module.json @@ -1,6 +1,6 @@ { "module": "mesh-delivery-view", - "version": "1", + "version": "0.1", "moved": true, "slug": "view", "tools": [ @@ -42,6 +42,7 @@ { "name": "findings", "history": 1 } ], "consumes": [ + "mesh-conditions.findings-asked", "mesh-delivery-assigner.views.asks", "mesh-delivery-assigner.views.assignments", "mesh-delivery-assigner.views.wants", @@ -49,6 +50,14 @@ "mesh-delivery-indexer.views.modules", "mesh-delivery-indexer.views.taken" ], + "streams": [ + { + "name": "mesh-delivery-view-findings", + "events": ["finding-written.*", "finding-cleared.*", "findings-stated.*"], + "max-age-days": 7, + "max-mib": 16 + } + ], "invokes": [ "mesh-delivery-assigner.assigner_read_asks", "mesh-delivery-assigner.assigner_read_differences", From 1778888acca14459a3ccc23ecceda029921cd578 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 11:54:50 +0100 Subject: [PATCH 09/33] AS2: the wants keep the trunk-only form until the first registration, and follow the pipelines' ends from it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §95, §130, §353, §352 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-assigner/main.go | 10 +- .../cmd/mesh-delivery-assigner/pass.go | 4 +- .../cmd/mesh-delivery-assigner/pipelines.go | 96 ++++++++++++++++++- .../mesh-delivery-assigner/pipelines_test.go | 39 ++++++++ .../mesh-delivery-assigner/manifest_test.go | 4 +- modules/mesh-delivery-assigner/module.json | 2 + 6 files changed, 143 insertions(+), 12 deletions(-) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index 4f040bed..3ed9dab2 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -493,9 +493,13 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, if kv := kvs[writtenBucket]; kv != nil { // The runner's pipeline-ended and group-ended (design 47 G §29): kept in the state written, which the wants follow // once the first registrations are written (§95, §353). - p.heard = newPipelineEnds(kvEnds{ctx, kv}, wake, logger.Printf) - if err := hearEnds(hub, p.heard); err != nil { - logger.Printf("a pipeline's end is not heard, so no want follows a pipeline until the assigner starts again: %v", err) + p.ends = newPipelineEnds(kvEnds{ctx, kv}, wake, logger.Printf) + err := hearEnds(hub, p.ends) + if err == nil && bus != nil { + err = followRegistrations(ctx, hub, bus.Call, p.ends) + } + if err != nil { + logger.Printf("a pipeline's end or a registration is not heard, so the wants keep the trunk-only form until the assigner starts again: %v", err) } } hearReadsChanged(ctx, js, hub, reads, findings, gate.eventsLive, logger.Printf) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go index 69d369af..8c5d6aa2 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go @@ -58,10 +58,8 @@ type passer struct { // neither is, by node and module, each module on neither list whose condition this process raised (issue 1945). Nil until the // first pass after a start, which clears what a previous process may have raised (startClears). neither map[string]neitherOne - // ends are the pipeline ends the wants follow (pipelines.go); nil is the trunk-only want, kept until the first registrations. + // ends are the pipeline ends the wants follow (pipelines.go), once the first registrations are heard; until then, and nil, the trunk-only want. ends *pipelineEnds - // heard is the same follower while the wants do not follow it yet (ends is nil). - heard *pipelineEnds // acts are the puts' acts (ADR 0546); nil: the puts are not taken further (no bus to call, or no puts bucket). acts *putActs // noWant is, by node and module, the reason last said why the node gets no want for the module (issue 1945): a reason is diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go index 743125a5..5e9ab67e 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go @@ -5,13 +5,19 @@ import ( "encoding/json" "errors" "fmt" + "slices" "sort" + "strings" "sync" + "sync/atomic" + "time" "github.com/nats-io/nats.go/jetstream" + mcc "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" mprc "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshpipelinerunnerclient" mwc "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshwantsclient" + mpcat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" mpr "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/keys" @@ -81,6 +87,9 @@ type endStore interface { get(key string) (value []byte, revision uint64, err error) // swap writes value at key if the key stands at revision (0: if it does not stand); errSwapped when it does not. swap(key string, value []byte, revision uint64) error + // keys are the keys that stand, and drop deletes one. + keys() ([]string, error) + drop(key string) error } var errSwapped = errors.New("the key changed") @@ -115,6 +124,16 @@ func (k kvEnds) swap(key string, value []byte, revision uint64) error { return err } +func (k kvEnds) keys() ([]string, error) { + ks, err := k.kv.Keys(k.ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + return nil, nil + } + return ks, err +} + +func (k kvEnds) drop(key string) error { return k.kv.Delete(k.ctx, key) } + func isWrongSequence(err error) bool { var ae *jetstream.APIError return errors.As(err, &ae) && ae.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequence @@ -126,7 +145,9 @@ type pipelineEnds struct { st endStore wake func() logf func(string, ...any) - mu sync.Mutex + // on is whether the wants follow the ends: set by the first registration heard, never before (design 47 G §95, §353). + on atomic.Bool + mu sync.Mutex // firsts is, by module, the first watched branch of its registration, the branch of a placement that names none (design // 47 G §14). firsts map[string]string @@ -278,7 +299,7 @@ func writtenTools(st store) []stdio.Tool { // branchesOf is the watched branches the placements of module chose, each once; the default branch when none names one. func (e *pipelineEnds) branchesOf(module string, placed map[string]map[string]string) []string { - if e == nil { + if !e.following() { return nil } seen := map[string]bool{} @@ -300,7 +321,7 @@ func (e *pipelineEnds) branchesOf(module string, placed map[string]map[string]st // mayWant says whether module's wants may name fp: a pipeline passed at fp on every branch, or the wants name fp already // (a node placed later, a module taken away). func (e *pipelineEnds) mayWant(module, fp string, branches []string, wanted map[string]mwc.Want) bool { - if e == nil { + if !e.following() { return true } for _, w := range wanted { @@ -318,10 +339,77 @@ func (e *pipelineEnds) mayWant(module, fp string, branches []string, wanted map[ // wrote marks the end the wants of module were written from. func (e *pipelineEnds) wrote(module, fp string, branches []string) { - if e == nil { + if !e.following() { return } for _, b := range branches { e.markWritten(module, b, fp) } } + +// following says whether the wants follow the pipeline ends yet; a nil follower never does. +func (e *pipelineEnds) following() bool { return e != nil && e.on.Load() } + +// registered takes a module's registration (design 47 G §353): the first one heard switches the wants to the ends, its first +// watched branch is the default a placement runs, and the ends kept for a branch it no longer watches are deleted (§352). gone +// is a registration removed: all its ends go. +func (e *pipelineEnds) registered(reg mpcat.Registration, gone bool) { + e.mu.Lock() + if gone || len(reg.Branches) == 0 { + delete(e.firsts, reg.Module) + } else { + e.firsts[reg.Module] = reg.Branches[0] + } + e.mu.Unlock() + kept, err := e.st.keys() + if err != nil { + e.logf("the ends kept cannot be listed: %v", err) + } + prefix := keys.EscapeName(reg.Module) + "." + for _, k := range kept { + rest, ok := strings.CutPrefix(k, prefix) + if !ok { + continue + } + branch, err := keys.UnescapeName(rest) + if err == nil && !gone && slices.Contains(reg.Branches, branch) { + continue + } + if err := e.st.drop(k); err != nil { + e.logf("the end %s is not deleted: %v", k, err) + } + } + if !e.on.Swap(true) { + e.logf("the first registration is heard (%s): the wants follow the pipelines' ends from now", reg.Module) + } + e.wake() +} + +// followRegistrations follows the registrations through the seat mesh-catalog (design 47 G §353): a first view of its verb +// and each registration-changed on the hub. It returns when the hub's handler is registered; the first view is asked on its own +// goroutine, the wants keep the trunk-only form until a registration is heard. +func followRegistrations(ctx context.Context, hub *process.EventHub, call mcc.Call, e *pipelineEnds) error { + regs := mcc.NewRegistrations(call) + regs.Follower.Logf = e.logf + regs.Follower.Changed = func(module string, value json.RawMessage, gone bool, _ time.Time) { + reg := mpcat.Registration{Module: module} + if !gone { + _ = json.Unmarshal(value, ®) + } + e.registered(reg, gone) + } + if err := hub.Handle(mpcat.SubjectRegistrationChanged, func(m jetstream.Msg) { + if err := regs.Hear(ctx, m.Data()); err != nil { + e.logf("%s: %v", m.Subject(), err) + } + _ = m.Ack() + }); err != nil { + return err + } + go func() { + if err := regs.Start(ctx); err != nil && ctx.Err() == nil { + e.logf("the first view of the registrations is not answered: %v", err) + } + }() + return nil +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines_test.go index 17597d49..acb0c207 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines_test.go @@ -10,6 +10,7 @@ import ( "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" mwc "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshwantsclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + mpcat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" mpr "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner" oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" @@ -34,6 +35,13 @@ type memEnds struct { func newMemEnds() *memEnds { return &memEnds{vals: map[string][]byte{}, revs: map[string]uint64{}} } func (m *memEnds) get(key string) ([]byte, uint64, error) { return m.vals[key], m.revs[key], nil } +func (m *memEnds) keys() (out []string, _ error) { + for k := range m.vals { + out = append(out, k) + } + return out, nil +} +func (m *memEnds) drop(key string) error { delete(m.vals, key); delete(m.revs, key); return nil } func (m *memEnds) swap(key string, v []byte, rev uint64) error { if m.revs[key] != rev { return errSwapped @@ -79,6 +87,7 @@ func TestAWantFollowsAPassedPipeline(t *testing.T) { } { woken := 0 e := newPipelineEnds(newMemEnds(), func() { woken++ }, t.Logf) + e.on.Store(true) for _, p := range c.ends { if p.Group != "" { continue // heard and ignored, as hearPipeline does @@ -100,12 +109,14 @@ func TestAWantFollowsAPassedPipeline(t *testing.T) { } // A want that names the fingerprint already is not held by a later failure (a node placed afterwards gets it). e := newPipelineEnds(newMemEnds(), func() {}, t.Logf) + e.on.Store(true) if !e.mayWant("web", fp1, []string{"main"}, map[string]mwc.Want{"g14": {Fp: fp1}}) { t.Error("a want naming the fingerprint was held") } // markWritten keeps the end and says it was written from; a newer end replaces it unwritten. st := newMemEnds() e = newPipelineEnds(st, func() {}, t.Logf) + e.on.Store(true) _ = e.take(pipe("web", "main", 5, 1, fp1, pass), true) e.wrote("web", fp1, []string{"main"}) var end End @@ -153,6 +164,7 @@ func TestAPassWritesNoWantWithoutAPassedPipeline(t *testing.T) { p.w.put(buckets.Presence, "g14", on) if c.follow { p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf) + p.ends.on.Store(true) if c.passed { _ = p.ends.take(pipe("web", trunkBranch, 5, 1, fp, mpr.PipelineEndedOutcomePassed), true) } @@ -172,3 +184,30 @@ func TestAPassWritesNoWantWithoutAPassedPipeline(t *testing.T) { } } } + +// Design 47 G §95, §130, §353, §352 (test 100): the wants keep the trunk-only form until the first registration is heard and +// switch on it; a branch a registration no longer watches loses its end, and its first branch is a placement's default. +func TestTheWantsSwitchOnTheFirstRegistration(t *testing.T) { + st := newMemEnds() + woken := 0 + e := newPipelineEnds(st, func() { woken++ }, t.Logf) + _ = e.take(pipe("web", "main", 5, 1, "fp1", mpr.PipelineEndedOutcomeFailed), false) + _ = e.take(pipe("web", "beta", 5, 1, "fp1", mpr.PipelineEndedOutcomePassed), true) + if e.following() || !e.mayWant("web", "fp1", []string{"main"}, nil) { + t.Fatal("the follower held a want before any registration") + } + e.registered(mpcat.Registration{Module: "web", Branches: []string{"beta"}}, false) + if !e.following() || woken == 0 { + t.Error("the first registration did not switch the wants on") + } + if e.mayWant("web", "fp1", []string{"main"}, nil) || !e.mayWant("web", "fp1", []string{"beta"}, nil) { + t.Error("after the switch the wants did not follow the ends") + } + if len(st.vals) != 1 || e.branchOf("web", "") != "beta" || e.branchOf("other", "") != trunkBranch { + t.Errorf("ends kept %d, default branch %q: an unwatched branch's end stayed, or the default is wrong", len(st.vals), e.branchOf("web", "")) + } + e.registered(mpcat.Registration{Module: "web"}, true) + if len(st.vals) != 0 { + t.Errorf("a registration gone left %d ends", len(st.vals)) + } +} diff --git a/modules/mesh-delivery-assigner/manifest_test.go b/modules/mesh-delivery-assigner/manifest_test.go index 629dcee0..036dbc34 100644 --- a/modules/mesh-delivery-assigner/manifest_test.go +++ b/modules/mesh-delivery-assigner/manifest_test.go @@ -123,11 +123,11 @@ func TestManifest(t *testing.T) { } // ADR 0618 §4: and it hears each node's engine say its reads changed; ADR 0636 §4 (issue 1966): and the seat list's two events; // issue 1966 note 151: and the teller's ask-ended, its subject the one the assigner hears. - if !slices.Equal(m.Consumes, []string{"mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", "mesh-delivery-indexer.views.graph", "mesh-delivery-indexer.views.manifests", "mesh-delivery-indexer.views.modules", buckets.TellerModule + "." + tellerevents.AskEnded, "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.group-ended", "node-engine.manifest-reads-changed"}) { + if !slices.Equal(m.Consumes, []string{"mesh-catalog.registration-changed", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", "mesh-delivery-indexer.views.graph", "mesh-delivery-indexer.views.manifests", "mesh-delivery-indexer.views.modules", buckets.TellerModule + "." + tellerevents.AskEnded, "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.group-ended", "node-engine.manifest-reads-changed"}) { t.Errorf("consumes = %v, want the indexer's views of graph, manifests and modules, the teller's ask-ended and the engines' reads changed", m.Consumes) } if want := []string{"mesh-delivery-indexer.graph", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", - "seat:mesh-catalog.seat-definitions", // ADR 0636 §4: the seat list + "seat:mesh-catalog.registrations", "seat:mesh-catalog.seat-definitions", // ADR 0636 §4: the seat list "seat:mesh-hand-acts.hand-act", "seat:mesh-hand-acts.hand-acts", "seat:mesh-hand-acts.warranted", // ADR 0565 §9, §11: the seat alone "seat:node-backup.backed-up", "seat:node-backup.now", "seat:node-engine.manifest-reads", // ADR 0618 §3 diff --git a/modules/mesh-delivery-assigner/module.json b/modules/mesh-delivery-assigner/module.json index d81cca86..d6252329 100644 --- a/modules/mesh-delivery-assigner/module.json +++ b/modules/mesh-delivery-assigner/module.json @@ -39,6 +39,7 @@ "operator-channel" ], "consumes": [ + "mesh-catalog.registration-changed", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", "mesh-delivery-indexer.views.graph", @@ -53,6 +54,7 @@ "mesh-delivery-indexer.graph", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", + "seat:mesh-catalog.registrations", "seat:mesh-catalog.seat-definitions", "seat:mesh-hand-acts.hand-act", "seat:mesh-hand-acts.hand-acts", From 0b597eb41749cbdbe576536e28bb431aaa4049ee Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 11:54:56 +0100 Subject: [PATCH 10/33] FB3, FB4, FB5: the bridge performs what the gate owes through the seat git and says each outcome, reads groups from the seat, and its owed tools MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Taken over from the join (6ada2089, f1417ab0, 6e162a60, 3a4e62b0, 73fb42d7, ee15a280, a5476f62, 94a4e62a, 33907a81, e519cc87, e8f268c1) and fitted to the final contracts: the seat's events are pull-updated, pull-merged, pull-closed, forge-delete; OwedEnded keeps only the fields §164 names; merge-pull takes no ask. A merge act is performed only as a switched act (§169); from G the gate owes none (§167). Builds: design 47 G §43, §45, §157, §161, §162, §163, §164, §167, §168, §169, §171, §250 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../clients/meshdeliveryclient/gate_test.go | 31 + .../mesh-delivery-forge-bridge-tools/main.go | 209 +++--- .../main_test.go | 148 ++-- .../mesh-delivery-forge-bridge-tools/owed.go | 132 ++++ .../cmd/mesh-delivery-forge-bridge/gate.go | 132 ---- .../mesh-delivery-forge-bridge/gate_test.go | 381 +++++----- .../mesh-delivery-forge-bridge/groupask.go | 44 +- .../cmd/mesh-delivery-forge-bridge/groups.go | 162 ++--- .../mesh-delivery-forge-bridge/groups_test.go | 109 ++- .../integration_test.go | 688 +++++++++--------- .../cmd/mesh-delivery-forge-bridge/main.go | 82 ++- .../cmd/mesh-delivery-forge-bridge/names.go | 8 +- .../mesh-delivery-forge-bridge/names_test.go | 20 +- .../cmd/mesh-delivery-forge-bridge/owed.go | 342 +++++++++ .../forgemsg/forge.go | 2 +- modules/mesh-delivery-forge-bridge/go.mod | 2 + .../internal/bridge/acts.go | 160 ---- .../internal/bridge/acts_test.go | 188 ----- .../internal/bridge/call.go | 2 +- .../internal/bridge/owed.go | 147 ++++ .../internal/bridge/owed_test.go | 95 +++ .../internal/gitfixture/gitfixture.go | 9 +- .../internal/groups/groups.go | 157 ++-- .../internal/groups/owedask.go | 95 +++ .../manifest_test.go | 48 +- .../mesh-delivery-forge-bridge/module.json | 39 +- 26 files changed, 1940 insertions(+), 1492 deletions(-) create mode 100644 modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/owed.go create mode 100644 modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/owed.go delete mode 100644 modules/mesh-delivery-forge-bridge/internal/bridge/acts.go delete mode 100644 modules/mesh-delivery-forge-bridge/internal/bridge/acts_test.go create mode 100644 modules/mesh-delivery-forge-bridge/internal/bridge/owed.go create mode 100644 modules/mesh-delivery-forge-bridge/internal/bridge/owed_test.go create mode 100644 modules/mesh-delivery-forge-bridge/internal/groups/owedask.go diff --git a/contracts/go/clients/meshdeliveryclient/gate_test.go b/contracts/go/clients/meshdeliveryclient/gate_test.go index 92be0bf2..afcee870 100644 --- a/contracts/go/clients/meshdeliveryclient/gate_test.go +++ b/contracts/go/clients/meshdeliveryclient/gate_test.go @@ -142,6 +142,37 @@ func TestGateKeysAsGoDeliveryHadThem(t *testing.T) { t.Errorf("%q reads as a switched act's id", other) } } + // A switched act's id carries its state, key and revision, and reads back; any other id is not one. + for _, c := range []struct { + state, key string + rev uint64 + id string + }{ + {SwitchedOwed, "novox.app.7.0123abcd.status=mesh/delivery", 4, "switched.owed.4.novox.app.7.0123abcd.status=mesh/delivery"}, + {SwitchedMerges, "release-g", 12, "switched.merges.12.release-g"}, + } { + id, err := SwitchedActID(c.state, c.key, c.rev) + if err != nil || id != c.id { + t.Errorf("SwitchedActID(%q, %q, %d) = %q, %v; want %q", c.state, c.key, c.rev, id, err, c.id) + } + if s, k, r, ok := ParseSwitchedActID(id); !ok || s != c.state || k != c.key || r != c.rev { + t.Errorf("ParseSwitchedActID(%q) = %q %q %d %v", id, s, k, r, ok) + } + } + for _, bad := range []struct { + state, key string + rev uint64 + }{{"turns", "k", 1}, {SwitchedOwed, "", 1}, {SwitchedOwed, "a@b", 1}, {SwitchedMerges, "k", 0}} { + if id, err := SwitchedActID(bad.state, bad.key, bad.rev); err == nil { + t.Errorf("SwitchedActID(%+v) = %q, no error", bad, id) + } + } + for _, other := range []string{"novox.app.7.0123abcd.view@1700000000", "switched.turns.1.k", "switched.owed.x.k", + "switched.owed.0.k", "switched.owed.3", "switched.owed.3.k@1"} { + if _, _, _, ok := ParseSwitchedActID(other); ok { + t.Errorf("%q reads as a switched act's id", other) + } + } } func TestGateValuesRefuseWhatTheirRulesRefuse(t *testing.T) { diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main.go index d123a475..4bc1619d 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main.go @@ -9,17 +9,13 @@ import ( "context" "encoding/json" "fmt" - "maps" "os" - "slices" - "strconv" "strings" "sync" "time" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" - "git.novox.be/novox/mesh-delivery/sdk/trunk" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" @@ -28,9 +24,6 @@ import ( "mesh-delivery-forge-bridge/internal/groups" ) -// gateMergesState is the gate's state merges, which the tools read (reads: mesh-delivery-gate.merges in the manifest). -const gateMergesState = "mesh-delivery-gate.merges" - // Tool names. const ( GroupsTool = "bridge_groups" @@ -42,8 +35,8 @@ const ( // asker writes an end-asks value and waits for the bridge's answer to it. type asker func(key string, value []byte) (groups.EndAsk, error) -// Tools are the bridge's tools: groupsRead reads the groups the gate holds with what the bridge kept of them. -func Tools(ask asker, now func() time.Time, groupsRead func() ([]groups.Row, []string, error), merged func(repository, merge string, number int) (json.RawMessage, bool, error), forge bridge.Caller) []stdio.Tool { +// Tools are the bridge's tools: groupsRead reads the seat mesh-delivery's groups with what the bridge kept of them. +func Tools(ask asker, now func() time.Time, groupsRead func() ([]groups.Row, []string, error), merged func(repository, merge string, number int) (json.RawMessage, error), forge bridge.Caller) []stdio.Tool { str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) } return []stdio.Tool{ {Name: bridge.TagsTool, @@ -59,37 +52,32 @@ func Tools(ask asker, now func() time.Time, groupsRead func() ([]groups.Row, []s {Name: MergedTool, Description: "The merged message of one merge, asked again (issue 1585): the message the bridge wrote on its stream FORGE when the " + "pull request was merged into the trunk, built the same way from the forge (the merge commit's trees and the manifests " + - "and lock files of its module directories) and from the gate's merges state (the pull request's number, head and " + - "group). For the indexer, which asks it when it takes in a held merge it does not hold in memory (a restart, a new " + + "and lock files of its module directories), the pull request's number, branch and head from the seat git's verb pull, and " + + "for a delivery group's member its group and members from the seat mesh-delivery's verb groups. For the indexer, which asks it when it takes in a held merge it does not hold in memory (a restart, a new " + "build), and the ask-again a lost message lacked. About 0.5 MB for a repository of many modules. Answers {message} " + - "with the forge message whole, or why it cannot: neither the gate nor the forge gives a merged pull request with that commit, or the forge did not answer. " + - "A merge made on the forge (a recovery merge, the operator's own) has no merge in the gate: the bridge reads that pull request from the forge, by number when given, and says made_on_forge.", + "with the forge message whole, or why it cannot: the forge gives no merged pull request with that commit, or a seat did not answer. " + + "A merge made on the forge (a recovery merge, the operator's own) is read the same way.", Input: map[string]any{"type": "object", "required": []string{"repository", "merge"}, "properties": map[string]any{ "repository": map[string]any{"type": "string", "description": "the repository, owner/name, as the forge names it"}, "merge": map[string]any{"type": "string", "description": "the merge commit, 40 hex digits"}, - "number": map[string]any{"type": "integer", "description": "the pull request's number, when known: for a merge made on the forge the bridge reads that pull request and checks its merge commit"}}}, + "number": map[string]any{"type": "integer", "description": "the pull request's number, when known: the bridge reads that pull request and checks its merge commit"}}}, Run: func(args map[string]any) (any, error) { number, _ := args["number"].(float64) - raw, onForge, err := merged(str(args, "repository"), str(args, "merge"), int(number)) + raw, err := merged(str(args, "repository"), str(args, "merge"), int(number)) if err != nil { return nil, err } - out := map[string]any{"message": raw} - if onForge { - out["made_on_forge"] = true - out["note"] = "the merge was made on the forge, not through the gate: its pull request was read from the forge" - fmt.Fprintf(os.Stderr, "[mesh-delivery-forge-bridge-tools] merged message of %s at %s answered from the forge: the merge was made on the forge, not through the gate\n", str(args, "repository"), str(args, "merge")) - } - return out, nil + return map[string]any{"message": raw}, nil }}, {Name: GroupsTool, - Description: "The delivery groups the gate holds (more than one member), each with its members' merge state and whether each " + - "merged message is written on the forge stream, and whether the group's end (forge.group.merged) is written: ended, " + - "or open with why (gathered, or merged and its end not written yet). An open group with every member merged is " + + Description: "The delivery groups the seat mesh-delivery's verb groups answers (more than one member), each with its members' delivery " + + "state and whether each merged message is written on the forge stream (with the merge commit it named), and whether the " + + "group's end (forge.group.merged) is written: ended, or open with why (gathered, or every message written and its end " + + "not written yet). An open group with every member's message written is " + "what bridge_group_end_again says the end of. Each group names its last end ask (who, why, when) and the bridge's " + "answer to it, from the state end-asks. A group's name narrows it to that group; open lists only open ones.", Input: map[string]any{"type": "object", "properties": map[string]any{ - "group": map[string]any{"type": "string", "description": "only this group, by the name the gate keys it"}, + "group": map[string]any{"type": "string", "description": "only this group, by its id (its branch)"}, "open": map[string]any{"type": "boolean", "description": "only the groups whose end is not written"}}}, Run: func(args map[string]any) (any, error) { rows, unread, err := groupsRead() @@ -110,11 +98,11 @@ func Tools(ask asker, now func() time.Time, groupsRead func() ([]groups.Row, []s {Name: EndAgainTool, Description: "Say a delivery group's end (forge.group.merged) again, by hand, for a group whose end the indexer did not take " + "in (a group bridge_groups names as merged with its end not written, or one the indexer waits for). The bridge " + - "writes it again from the gate's merges, with the time appended to the message id so the stream does not drop it as " + + "writes it again from the seat mesh-delivery's groups, with the time appended to the message id so the stream does not drop it as " + "a duplicate; the indexer takes an end for a group it already took in without changing anything. Refused while " + - "a member is not merged or its merged message is not written. Answers whether it was written.", + "a member's merged message is not written. Answers whether it was written.", Input: map[string]any{"type": "object", "required": []string{"group"}, "properties": map[string]any{ - "group": map[string]any{"type": "string", "description": "the group, by the name the gate keys it"}, + "group": map[string]any{"type": "string", "description": "the group, by its id (its branch)"}, "why": map[string]any{"type": "string", "description": "why its end is said again, in plain words"}}}, Run: func(args map[string]any) (any, error) { key, value, err := groups.NewEndAsk(str(args, "group"), str(args, "why"), stdio.CallerOf(args), now()) @@ -131,87 +119,50 @@ func Tools(ask asker, now func() time.Time, groupsRead func() ([]groups.Row, []s } // MergedOf is the merged message of the merge of repository at commit merge, as the bridge writes it: the pull request's -// number, head and group from the gate's merges (raw, by key), the rest from the forge through trees and files -// (bridge.MergedMessage), and, for a member of a delivery group, the group and its members as the process sets them. -// -// When the gate holds no merge with that commit, the merge was made on the forge (a recovery merge, the operator's own): -// the pull request is read through the seat git's verb pull (by number when given, else by the merge commit), the message is -// built the same way, and madeOnForge is true, for the answer and its log line to say. It is a lone pull request: no -// group is set. -func MergedOf(raws map[string][]byte, repository, merge string, number int, trees bridge.Trees, files bridge.Files, forge bridge.Caller) (out json.RawMessage, madeOnForge bool, err error) { +// number, branch and head through the seat git's verb pull (by number when given, else by the merge commit; a merge made +// on the forge is read the same way), the rest from the forge through trees and files (bridge.MergedMessage), and, for a +// member of a delivery group the seat mesh-delivery names (of), the group and its members as the process sets them. +func MergedOf(of []seat.Group, repository, merge string, number int, trees bridge.Trees, files bridge.Files, forge bridge.Caller) (json.RawMessage, error) { owner, repo, ok := strings.Cut(repository, "/") if !ok || owner == "" || repo == "" { - return nil, false, fmt.Errorf("%q is not owner/name", repository) - } - for _, key := range slices.Sorted(maps.Keys(raws)) { - m, err := meshdeliveryclient.DecodeMerge(raws[key]) - if err != nil { - continue - } - at := slices.IndexFunc(m.Pulls, func(p seat.MergesByGroupValuePullsItem) bool { - return trunk.SameRepository(p.Repository, repository) && p.Commit == merge - }) - if at < 0 { - continue - } - p := m.Pulls[at] - msg, err := bridge.MergedMessage(owner, repo, int(p.Number), m.Group, p.Head, merge, trees, files) - if err != nil { - return nil, false, fmt.Errorf("the merged message of %s at %s cannot be built: %w", repository, merge, err) - } - f, err := forgemsg.DecodeForge(msg.Data) - if err != nil { - return nil, false, err - } - if _, approved := m.Approved[meshdeliveryclient.ApprovedKey(f.Repository, f.Number)]; approved && len(m.Approved) >= 2 { - f.Group, f.Members = m.Group, groups.MembersOf(m) - } - out, err = forgemsg.EncodeForge(f) - return out, false, err - } - if forge == nil { - return nil, false, fmt.Errorf("the gate's merges hold no merge of %s with the commit %s, so its pull request is not known", repository, merge) + return nil, fmt.Errorf("%q is not owner/name", repository) } fp, err := bridge.ForgeMade(context.Background(), forge, owner, repo, number, merge) if err != nil { - return nil, false, fmt.Errorf("the gate's merges hold no merge of %s with the commit %s, and the forge does not give it as a merged pull request either: %w", repository, merge, err) + return nil, fmt.Errorf("the forge gives no merged pull request of %s with the commit %s: %w", repository, merge, err) } msg, err := bridge.MergedMessage(owner, repo, fp.Number, fp.Branch, fp.Head, merge, trees, files) if err != nil { - return nil, false, fmt.Errorf("the merged message of %s at %s (a merge made on the forge) cannot be built: %w", repository, merge, err) + return nil, fmt.Errorf("the merged message of %s at %s cannot be built: %w", repository, merge, err) } - return msg.Data, true, nil + f, err := forgemsg.DecodeForge(msg.Data) + if err != nil { + return nil, err + } + if g, ok := groups.Named(of, f.Branch, f.Repository, f.Number); ok { + f.Group, f.Members = g.ID, groups.MembersOf(g) + } + return forgemsg.EncodeForge(f) } -// read is the groups the gate holds with what the bridge kept of them: their ends in performed, and the last ask of -// each in end-asks with the bridge's answer. -func read(merges, performed, asks stdio.KeptState) ([]groups.Row, []string, error) { - keys, err := merges.Keys() - if err != nil { - return nil, nil, fmt.Errorf("the gate's merges cannot be listed: %w", err) - } - raws := map[string][]byte{} - for _, k := range keys { - e, err := merges.Get(k) - if err != nil || e == nil { - continue - } - raws[k] = e.Value - } +// read is the groups the seat mesh-delivery answers with what the bridge kept of them: the merged messages written and +// their ends in performed, and the last ask of each in end-asks with the bridge's answer. +func read(of []seat.Group, performed, asks stdio.KeptState) ([]groups.Row, []string, error) { pkeys, err := performed.Keys() if err != nil { return nil, nil, fmt.Errorf("the bridge's performed state cannot be listed: %w", err) } - published, ended := map[string]bool{}, map[string]uint64{} + published, ended := map[string]string{}, map[string]string{} for _, k := range pkeys { + into := ended switch { case strings.HasPrefix(k, "published."): - published[k] = true - case strings.HasPrefix(k, "groups."): - if e, err := performed.Get(k); err == nil && e != nil { - rev, _ := strconv.ParseUint(strings.TrimSpace(string(e.Value)), 10, 64) - ended[strings.TrimPrefix(k, "groups.")] = rev - } + into = published + case !strings.HasPrefix(k, "groups."): + continue + } + if e, err := performed.Get(k); err == nil && e != nil { + into[k] = strings.TrimSpace(string(e.Value)) } } akeys, err := asks.Keys() @@ -224,28 +175,39 @@ func read(merges, performed, asks stdio.KeptState) ([]groups.Row, []string, erro askRaws[k] = e.Value } } - rows, unread := groups.Rows(raws, published, ended, askRaws) + rows, unread := groups.Rows(of, published, ended, askRaws) return rows, unread, nil } -// askEnd writes the end-asks value at key and waits, on a watch of the key, at most bound for the bridge's answer. +// askEnd writes the end-asks value at key and waits at most bound for the bridge's answer. func askEnd(s stdio.KeptState, bound time.Duration) asker { + wait := askState(s, groups.EndAsksBucket, bound, func(v []byte) bool { a, err := groups.DecodeEndAsk(v); return err == nil && a.Done() }) return func(key string, value []byte) (groups.EndAsk, error) { - answered := make(chan groups.EndAsk, 1) + v, err := wait(key, value) + if err != nil { + return groups.EndAsk{}, err + } + return groups.DecodeEndAsk(v) + } +} + +// askState writes an ask's value at key of the state s (its bucket named bucket) and waits, on a watch of the key, at most +// bound for the bridge's answer, a later value that done says is answered. +func askState(s stdio.KeptState, bucket string, bound time.Duration, done func([]byte) bool) func(key string, value []byte) ([]byte, error) { + return func(key string, value []byte) ([]byte, error) { + answered := make(chan []byte, 1) var rev uint64 var mu sync.Mutex take := func(revision uint64, v []byte) { mu.Lock() mine := rev mu.Unlock() - if mine == 0 || revision <= mine { + if mine == 0 || revision <= mine || !done(v) { return } - if a, err := groups.DecodeEndAsk(v); err == nil && a.Done() { - select { - case answered <- a: - default: - } + select { + case answered <- v: + default: } } if err := s.Watch(key, func(c stdio.StateChange) error { @@ -254,11 +216,11 @@ func askEnd(s stdio.KeptState, bound time.Duration) asker { } return nil }); err != nil { - return groups.EndAsk{}, fmt.Errorf("watching %s.%s: %w", groups.EndAsksState, key, err) + return nil, fmt.Errorf("watching %s.%s: %w", bucket, key, err) } r, err := s.Put(key, json.RawMessage(value)) if err != nil { - return groups.EndAsk{}, fmt.Errorf("the ask could not be written to %s: %w", groups.EndAsksBucket, err) + return nil, fmt.Errorf("the ask could not be written to %s: %w", bucket, err) } mu.Lock() rev = r @@ -269,10 +231,10 @@ func askEnd(s stdio.KeptState, bound time.Duration) asker { wait, cancel := context.WithTimeout(context.Background(), bound) // a context's bound, no timer (ADR 0437 §8) defer cancel() select { - case a := <-answered: - return a, nil + case v := <-answered: + return v, nil case <-wait.Done(): - return groups.EndAsk{}, fmt.Errorf("the bridge did not answer within %s: its answer will be in %s.%s", bound, groups.EndAsksBucket, key) + return nil, fmt.Errorf("the bridge did not answer within %s: its answer will be in %s.%s", bound, bucket, key) } } } @@ -283,23 +245,32 @@ func askCaller(_ context.Context, key string, body any) (json.RawMessage, error) } func main() { - merges, performed, asks := stdio.State(gateMergesState), stdio.State(groups.PerformedState), stdio.State(groups.EndAsksState) + performed, asks := stdio.State(groups.PerformedState), stdio.State(groups.EndAsksState) tools := Tools(askEnd(asks, 25*time.Second), time.Now, - func() ([]groups.Row, []string, error) { return read(merges, performed, asks) }, - func(repository, merge string, number int) (json.RawMessage, bool, error) { - keys, err := merges.Keys() + func() ([]groups.Row, []string, error) { + of, err := groups.Read(context.Background(), askCaller) if err != nil { - return nil, false, fmt.Errorf("the gate's merges cannot be listed: %w", err) - } - raws := map[string][]byte{} - for _, k := range keys { - if e, err := merges.Get(k); err == nil && e != nil { - raws[k] = e.Value - } + return nil, nil, err } + return read(of, performed, asks) + }, + func(repository, merge string, number int) (json.RawMessage, error) { ctx := context.Background() - return MergedOf(raws, repository, merge, number, bridge.TreesVia(ctx, askCaller), bridge.FilesVia(ctx, askCaller), askCaller) + of, err := groups.Read(ctx, askCaller) + if err != nil { + return nil, err + } + return MergedOf(of, repository, merge, number, bridge.TreesVia(ctx, askCaller), bridge.FilesVia(ctx, askCaller), askCaller) }, askCaller) + owedAsks := stdio.State(groups.OwedAsksState) + recorder := meshhandactsclient.Recorder{Ask: stdio.Ask} + tools = append(tools, OwedTools(askState(owedAsks, groups.OwedAsksBucket, 25*time.Second, func(v []byte) bool { + a, err := groups.DecodeOwedAsk(v) + return err == nil && a.Done() + }), func() ([]OwedRow, error) { return performedRows(performed) }, func(what, why string) error { + _, err := recorder.Record(meshhandactsclient.Request{What: what, Why: why, Cause: "owed-dropped"}) + return err + }, time.Now)...) if err := stdio.Serve("", tools); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-forge-bridge-tools] %v\n", err) os.Exit(1) diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main_test.go index 616d23ea..d065eac3 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main_test.go @@ -20,28 +20,24 @@ import ( "mesh-delivery-forge-bridge/internal/groups" ) -// 1585/T18, T19 (ADR 0633 §4): bridge_groups names each group the gate holds with its members' state and whether its end is -// written (open or ended, with why), narrowed by group and by open; bridge_group_end_again writes an ask naming the group, -// who asked and why, and answers what the bridge answered. +// 1585/T18, T19 (ADR 0633 §4), acceptance 49: bridge_groups names each group the seat mesh-delivery's verb groups answers +// with its members' state and whether its end is written (open or ended, with why), narrowed by group and by open; +// bridge_group_end_again writes an ask naming the group, who asked and why, and answers what the bridge answered. func TestTheBridgeToolsReadTheGroupsAndAskTheEndAgain(t *testing.T) { h1, h2 := strings.Repeat("1", 40), strings.Repeat("2", 40) c1, c2 := strings.Repeat("c", 40), strings.Repeat("d", 40) - enc := func(group string, state2 meshdeliveryclient.MergeState, c string) []byte { - v, err := meshdeliveryclient.EncodeMerge(seat.MergesByGroupValue{Group: group, Ask: "ask", Approved: map[string]string{"novox/a@1": h1, "novox/b@2": h2}, - Pulls: []seat.MergesByGroupValuePullsItem{{Repository: "novox/a", Number: 1, Head: h1, State: string(meshdeliveryclient.MergeMerged), Commit: c1}, - {Repository: "novox/b", Number: 2, Head: h2, State: string(state2), Commit: c}}}) - if err != nil { - t.Fatal(err) - } - return v + grp := func(id string, a, b int, stateB string) seat.Group { + return seat.Group{ID: id, State: "merging", Members: []seat.Member{{Repository: "novox/a", Number: int64(a), Head: h1, State: "merged"}, + {Repository: "novox/b", Number: int64(b), Head: h2, State: stateB}}} } - lone, _ := meshdeliveryclient.EncodeMerge(seat.MergesByGroupValue{Group: "solo", Ask: "ask", Approved: map[string]string{"novox/a@5": h1}, - Pulls: []seat.MergesByGroupValuePullsItem{{Repository: "novox/a", Number: 5, Head: h1, State: string(meshdeliveryclient.MergeMerged), Commit: c1}}}) - merges := map[string][]byte{"ended": enc("ended", meshdeliveryclient.MergeMerged, c2), "ready": enc("ready", meshdeliveryclient.MergeMerged, c2), - "gathered": enc("gathered", meshdeliveryclient.MergeMerging, ""), "solo": lone, "bad": []byte("{")} - performed := map[string]bool{} - for _, c := range []string{c1, c2} { - performed[groups.PublishedKey("novox/a", 1, c)], performed[groups.PublishedKey("novox/b", 2, c)] = true, true + of := []seat.Group{grp("ended", 1, 2, "merged"), grp("ready", 3, 4, "merged"), grp("gathered", 5, 6, "merging"), + {ID: "solo", Members: []seat.Member{{Repository: "novox/a", Number: 7, Head: h1}}}, grp("", 8, 9, "merged")} + published := map[string]string{} + for _, n := range []int{1, 3, 5} { + published[groups.PublishedKey("novox/a", n)] = c1 + } + for _, n := range []int{2, 4} { + published[groups.PublishedKey("novox/b", n)] = c2 } // The state end-asks keeps the last ask of the group ready, answered, and one that does not read. answeredAsk, _ := json.Marshal(groups.EndAsk{V: 1, Group: "ready", Why: "lost", By: "lead@shanks", At: time.Date(2026, 11, 4, 22, 0, 0, 0, time.UTC), @@ -58,7 +54,7 @@ func TestTheBridgeToolsReadTheGroupsAndAskTheEndAgain(t *testing.T) { } tools := Tools(ask, func() time.Time { return time.Date(2026, 11, 4, 23, 0, 0, 0, time.UTC) }, func() ([]groups.Row, []string, error) { - rows, unread := groups.Rows(merges, performed, map[string]uint64{"ended": 7}, asks) + rows, unread := groups.Rows(of, published, map[string]string{"groups.ended": c2}, asks) return rows, unread, nil }, nil, nil) call := func(name string, args map[string]any) any { @@ -87,22 +83,22 @@ func TestTheBridgeToolsReadTheGroupsAndAskTheEndAgain(t *testing.T) { for _, r := range out.Groups { m[r.Group] = r } - if len(args) == 0 && strings.Join(out.Unread, " ") != "bad end-asks.gathered" { - t.Errorf("unread = %v, want bad and end-asks.gathered", out.Unread) + if len(args) == 0 && strings.Join(out.Unread, ", ") != "a group with no id, end-asks.gathered" { + t.Errorf("unread = %v, want the group with no id and end-asks.gathered", out.Unread) } return m } all := rowsOf(map[string]any{}) - if len(all) != 3 || !all["ended"].Ended || all["ended"].EndedAt != 7 { + if len(all) != 3 || !all["ended"].Ended || all["ended"].EndedWith != c2 { t.Fatalf("groups = %+v", all) } - if r := all["ready"]; !r.Open || !strings.Contains(r.Words, "bridge_group_end_again") || !r.Members[1].Published { - t.Errorf("ready = %+v, want open, merged and published, its end the thing to say", r) + if r := all["ready"]; !r.Open || !strings.Contains(r.Words, "bridge_group_end_again") || !r.Members[1].Published || r.Members[1].Commit != c2 { + t.Errorf("ready = %+v, want open and published with its merge commit, its end the thing to say", r) } if a := all["ready"].LastEndAsk; a == nil || a.By != "lead@shanks" || a.Why != "lost" || !a.Written || a.Answer == "" || all["ended"].LastEndAsk != nil { t.Errorf("ready's last end ask = %+v, ended's %+v: want ready's ask and its answer, none for ended", a, all["ended"].LastEndAsk) } - if r := all["gathered"]; !r.Open || !strings.Contains(r.Words, "gathered") || r.Members[1].State != string(meshdeliveryclient.MergeMerging) { + if r := all["gathered"]; !r.Open || !strings.Contains(r.Words, "gathered") || r.Members[1].State != "merging" || r.Members[1].Published { t.Errorf("gathered = %+v", r) } if open := rowsOf(map[string]any{"open": true}); len(open) != 2 || open["ended"].Group != "" { @@ -150,54 +146,47 @@ func TestTheBridgeToolsReadTheGroupsAndAskTheEndAgain(t *testing.T) { } return f } - lone, _ := meshdeliveryclient.EncodeMerge(seat.MergesByGroupValue{Group: "solo", Ask: "a", Approved: map[string]string{"novox/a@5": h1}, - Pulls: []seat.MergesByGroupValuePullsItem{{Repository: "novox/a", Number: 5, Head: h1, State: string(meshdeliveryclient.MergeMerged), Commit: c1}}}) - group, _ := meshdeliveryclient.EncodeMerge(seat.MergesByGroupValue{Group: "feat/g", Ask: "a", Approved: map[string]string{"novox/a@6": h1, "novox/b@7": h2}, - Pulls: []seat.MergesByGroupValuePullsItem{{Repository: "novox/a", Number: 6, Head: h1, State: string(meshdeliveryclient.MergeMerged), Commit: c1}, - {Repository: "novox/b", Number: 7, Head: h2, State: string(meshdeliveryclient.MergeMerging)}}}) + // The seat mesh-delivery's groups: feat/g of novox/a#9 and novox/b#7; a lone pull request is in none. + of := []seat.Group{{ID: "fix/x", Members: []seat.Member{{Repository: "novox/a", Number: 9, Head: h2}, {Repository: "novox/b", Number: 7, Head: h1}}}} for _, c := range []struct { name string - raws map[string][]byte + of []seat.Group repository string merge string number int api bridge.Caller - onForge bool want forgemsg.Forge fail string }{ - {name: "a lone pull request", raws: map[string][]byte{"solo": lone}, repository: "novox/a", merge: c1, want: published("a", 5, h1, "solo", c1)}, - {name: "a member of a group", raws: map[string][]byte{"feat~g": group}, repository: "novox/a", merge: c1, want: func() forgemsg.Forge { - f := published("a", 6, h1, "feat/g", c1) - f.Group, f.Members = "feat/g", []meshdeliveryclient.Head{{Repository: "novox/a", Number: 6, Head: h1}, {Repository: "novox/b", Number: 7, Head: h2}} - return f - }()}, - {name: "a merge the gate does not hold", raws: map[string][]byte{"solo": lone, "bad": []byte("{")}, repository: "novox/a", merge: c2, fail: "hold no merge"}, - // Issue 1966: a merge made on the forge (a recovery merge) has no merge in the gate; the bridge reads the pull request - // from the forge, by number or by the merge commit, and checks its merge commit. - {name: "a merge made on the forge, by number", raws: map[string][]byte{"solo": lone}, repository: "novox/a", merge: c2, number: 9, onForge: true, + {name: "a lone pull request, by number", repository: "novox/a", merge: c2, number: 9, api: forgeAnswers(t, 9, "", "merged", "fix/x", h2, c2, "main"), want: published("a", 9, h2, "fix/x", c2)}, - {name: "a merge made on the forge, by the merge commit", raws: map[string][]byte{}, repository: "novox/a", merge: c2, onForge: true, + {name: "a lone pull request, by the merge commit", repository: "novox/a", merge: c2, api: forgeAnswers(t, 0, c2, "merged", "fix/x", h2, c2, "main"), want: published("a", 9, h2, "fix/x", c2)}, - {name: "a forge pull request merged at another commit", raws: map[string][]byte{}, repository: "novox/a", merge: c2, number: 9, + {name: "a member of a group", of: of, repository: "novox/a", merge: c2, + api: forgeAnswers(t, 0, c2, "merged", "fix/x", h2, c2, "main"), want: func() forgemsg.Forge { + f := published("a", 9, h2, "fix/x", c2) + f.Group, f.Members = "fix/x", []meshdeliveryclient.Head{{Repository: "novox/a", Number: 9, Head: h2}, {Repository: "novox/b", Number: 7, Head: h1}} + return f + }()}, + {name: "a forge pull request merged at another commit", repository: "novox/a", merge: c2, number: 9, api: forgeAnswers(t, 9, "", "merged", "fix/x", h2, c1, "main"), fail: "not at " + c2}, - {name: "a forge pull request not merged", raws: map[string][]byte{}, repository: "novox/a", merge: c2, number: 9, + {name: "a forge pull request not merged", repository: "novox/a", merge: c2, number: 9, api: forgeAnswers(t, 9, "", "open", "fix/x", h2, "", "main"), fail: "not merged"}, - {name: "a forge pull request merged into another branch", raws: map[string][]byte{}, repository: "novox/a", merge: c2, number: 9, + {name: "a forge pull request merged into another branch", repository: "novox/a", merge: c2, number: 9, api: forgeAnswers(t, 9, "", "merged", "fix/x", h2, c2, "dev"), fail: "not the trunk"}, - {name: "a forge pull request with no head branch", raws: map[string][]byte{}, repository: "novox/a", merge: c2, number: 9, + {name: "a forge pull request with no head branch", repository: "novox/a", merge: c2, number: 9, api: forgeAnswers(t, 9, "", "merged", "", h2, c2, "main"), fail: "no head branch"}, - {name: "a seat answer that says an error", raws: map[string][]byte{}, repository: "novox/a", merge: c2, number: 9, + {name: "a seat answer that says an error", repository: "novox/a", merge: c2, number: 9, api: answering(gitfixture.Given(t, "pull answers an error, never none", "verb:pull.answer")), fail: "holds no pull request"}, - {name: "a seat answer with no pull request", raws: map[string][]byte{}, repository: "novox/a", merge: c2, number: 9, + {name: "a seat answer with no pull request", repository: "novox/a", merge: c2, number: 9, api: answering(json.RawMessage(`{}`)), fail: "carries nothing"}, - {name: "a forge that does not answer", raws: map[string][]byte{}, repository: "novox/a", merge: c2, + {name: "a forge that does not answer", repository: "novox/a", merge: c2, api: func(context.Context, string, any) (json.RawMessage, error) { return nil, errors.New("the forge is down") }, fail: "the forge is down"}, } { t.Run(c.name, func(t *testing.T) { - raw, onForge, err := MergedOf(c.raws, c.repository, c.merge, c.number, trees, files, c.api) + raw, err := MergedOf(c.of, c.repository, c.merge, c.number, trees, files, c.api) if c.fail != "" { if err == nil || !strings.Contains(err.Error(), c.fail) { t.Fatalf("err %v, want %q", err, c.fail) @@ -205,20 +194,67 @@ func TestTheBridgeToolsReadTheGroupsAndAskTheEndAgain(t *testing.T) { return } want, _ := forgemsg.EncodeForge(c.want) - if err == nil && onForge != c.onForge { - t.Fatalf("made on the forge %v, want %v", onForge, c.onForge) - } if err != nil || !bytes.Equal(raw, want) { t.Fatalf("err %v, answer\n%s\nwant the published message\n%s", err, raw, want) } }) } - if _, _, err := MergedOf(map[string][]byte{"solo": lone}, "novox/a", c1, 0, func(string, string, string) (string, map[string]string, error) { + if _, err := MergedOf(nil, "novox/a", c2, 9, func(string, string, string) (string, map[string]string, error) { return "", nil, errors.New("the forge does not answer") - }, files, nil); err == nil || !strings.Contains(err.Error(), "does not answer") { + }, files, forgeAnswers(t, 9, "", "merged", "fix/x", h2, c2, "main")); err == nil || !strings.Contains(err.Error(), "does not answer") { t.Errorf("a forge that does not answer: %v", err) } }) + + // Design 47 *What the gate owes the forge*: bridge_owed reads performed and asks the backlog; bridge_owed_again asks the + // act again; bridge_owed_drop writes the hand-act log first and drops nothing it could not record. + t.Run("the owed tools", func(t *testing.T) { + at := time.Date(2026, 11, 9, 21, 0, 0, 0, time.UTC) + var asked []groups.OwedAsk + ask := func(key string, value []byte) ([]byte, error) { + a, err := groups.DecodeOwedAsk(value) + if err != nil { + return nil, err + } + asked = append(asked, a) + a.Answered, a.Outcome, a.Answer = at, forgemsg.OwedPerformed, "said as owed.ended: performed" + if a.Kind == groups.OwedBacklog { + a.Backlog = []groups.Backlog{{Stream: seat.StreamOwed, Waiting: 1}} + } + return json.Marshal(a) + } + rows := []OwedRow{{Of: seat.Seat, ID: "a", Outcome: forgemsg.OwedPerformed, At: at}, {Of: "mesh-delivery-checker", ID: "b", Outcome: forgemsg.OwedRefused, At: at}} + var recorded []string + recordErr := error(nil) + record := func(what, why string) error { recorded = append(recorded, what+": "+why); return recordErr } + tools := map[string]func(map[string]any) (any, error){} + for _, tl := range OwedTools(ask, func() ([]OwedRow, error) { return rows, nil }, record, func() time.Time { return at }) { + tools[tl.Name] = tl.Run + } + out, err := tools[OwedTool](map[string]any{"of": seat.Seat}) + if m, _ := out.(map[string]any); err != nil || len(m["performed"].([]OwedRow)) != 1 || len(m["backlog"].([]groups.Backlog)) != 1 { + t.Errorf("bridge_owed: %v %v", out, err) + } + if out, err := tools[OwedAgainTool](map[string]any{"of": seat.Seat, "id": "a"}); err != nil || out.(map[string]any)["outcome"] != forgemsg.OwedPerformed || + asked[len(asked)-1].Kind != groups.OwedAgain { + t.Errorf("bridge_owed_again: %v %v", out, err) + } + if _, err := tools[OwedDropTool](map[string]any{"of": seat.Seat, "id": "a"}); err == nil || len(recorded) != 0 { + t.Errorf("a drop with no why: %v, recorded %v", err, recorded) + } + n := len(asked) + recordErr = errors.New("mesh-hand-acts has no holder that answers") + if _, err := tools[OwedDropTool](map[string]any{"of": seat.Seat, "id": "a", "why": "merged by hand"}); err == nil || len(asked) != n { + t.Errorf("a drop whose hand act was not recorded: %v, asked %d", err, len(asked)-n) + } + recordErr = nil + if _, err := tools[OwedDropTool](map[string]any{"of": seat.Seat, "id": "a", "why": "merged by hand"}); err != nil || asked[len(asked)-1].Why != "merged by hand" { + t.Errorf("bridge_owed_drop: %v %+v", err, asked) + } + if _, _, err := groups.NewOwedAsk(groups.OwedAgain, "", "a", "", "", at); err == nil { + t.Error("an ask naming no seat was taken") + } + }) } // forgeAnswers is the seat git's verb pull for one pull request, asked by number (when number is given) or by the merge diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/owed.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/owed.go new file mode 100644 index 00000000..2c7459b0 --- /dev/null +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/owed.go @@ -0,0 +1,132 @@ +package main + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" + + "mesh-delivery-forge-bridge/internal/groups" +) + +// The tools of what the bridge performs of what is owed to the forge (novox/hq design 47 *What the gate owes the forge*). +const ( + OwedTool = "bridge_owed" + OwedAgainTool = "bridge_owed_again" + OwedDropTool = "bridge_owed_drop" +) + +// OwedRow is one act the bridge performed, as the state performed keeps it: the outcome said as owed.ended. +type OwedRow = forgemsg.OwedEnded + +// performedRows reads every act the state performed holds; a key that is not an act (a group end's bookkeeping) is left. +func performedRows(s stdio.KeptState) ([]OwedRow, error) { + keys, err := s.Keys() + if err != nil { + return nil, fmt.Errorf("the bridge's performed state cannot be listed: %w", err) + } + var rows []OwedRow + for _, k := range keys { + e, err := s.Get(k) + if err != nil || e == nil { + continue + } + var p struct { + Ended OwedRow `json:"ended"` + } + if json.Unmarshal(e.Value, &p) == nil && p.Ended.Of != "" { + rows = append(rows, p.Ended) + } + } + sort.Slice(rows, func(i, j int) bool { return rows[i].At.After(rows[j].At) }) + return rows, nil +} + +// OwedTools are bridge_owed, bridge_owed_again and bridge_owed_drop: ask writes an owed-asks value and waits for the +// bridge's answer; performed reads the state performed; record writes a hand act in the seat mesh-hand-acts' log before +// a drop is asked, so a drop is never made without its record. +func OwedTools(ask func(key string, value []byte) ([]byte, error), performed func() ([]OwedRow, error), record func(what, why string) error, now func() time.Time) []stdio.Tool { + str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) } + act := map[string]any{ + "of": map[string]any{"type": "string", "description": "the seat that said the act: mesh-delivery (the gate) or mesh-delivery-checker"}, + "id": map[string]any{"type": "string", "description": "the act's id, as owed and owed.ended say it"}} + asked := func(kind string, args map[string]any) (groups.OwedAsk, error) { + key, value, err := groups.NewOwedAsk(kind, str(args, "of"), str(args, "id"), str(args, "why"), stdio.CallerOf(args), now()) + if err != nil { + return groups.OwedAsk{}, err + } + v, err := ask(key, value) + if err != nil { + return groups.OwedAsk{}, err + } + return groups.DecodeOwedAsk(v) + } + withWhy := map[string]any{"why": map[string]any{"type": "string", "description": "why it is dropped, in plain words"}} + for k, v := range act { + withWhy[k] = v + } + return []stdio.Tool{ + {Name: OwedTool, + Description: "What the forge bridge performed of what the gate and the checker owe the forge (design 47): each act it said as " + + "owed.ended, newest first, with its seat, id, pull request, outcome (performed, refused, could not be performed, " + + "undecodable, dropped), the forge's words and when, from its state performed (kept 7 days); and the backlog of its " + + "durable on each seat's stream owed: the last event applied, the events waiting and held, the age of the oldest. " + + "of or id narrow it. Read-only.", + Input: map[string]any{"type": "object", "properties": act}, + Run: func(args map[string]any) (any, error) { + rows, err := performed() + if err != nil { + return nil, err + } + kept := []OwedRow{} + for _, r := range rows { + if (str(args, "of") == "" || r.Of == str(args, "of")) && (str(args, "id") == "" || r.ID == str(args, "id")) { + kept = append(kept, r) + } + } + out := map[string]any{"performed": kept} + if a, err := asked(groups.OwedBacklog, args); err != nil { + out["backlog-error"] = err.Error() + } else { + out["backlog"] = a.Backlog + } + return out, nil + }}, + {Name: OwedAgainTool, + Description: "Perform one act owed to the forge again now, by hand, whatever its last outcome (a refusal the forge no longer " + + "makes, an act that could not be performed): the bridge calls the seat git's verb of the act under its id (safe to " + + "repeat) and says its outcome as owed.ended. The act is the one the bridge kept, or, for the gate's, the seat " + + "mesh-delivery's groups. Answers the outcome.", + Input: map[string]any{"type": "object", "required": []string{"of", "id"}, "properties": act}, + Run: func(args map[string]any) (any, error) { + a, err := asked(groups.OwedAgain, args) + if err != nil { + return nil, err + } + return map[string]any{"of": a.Of, "id": a.ID, "outcome": a.Outcome, "answer": a.Answer}, nil + }}, + {Name: OwedDropTool, + Description: "Drop one act owed to the forge, by hand, with why: written first in the hand-act log of the seat mesh-hand-acts, " + + "then said by the bridge as owed.ended with the outcome dropped, so its owner records it answered and stops saying " + + "it again. Nothing is done at the forge. Answers the outcome said.", + Input: map[string]any{"type": "object", "required": []string{"of", "id", "why"}, "properties": withWhy}, + Run: func(args map[string]any) (any, error) { + if str(args, "why") == "" { + return nil, fmt.Errorf("a drop says why") + } + if err := record(fmt.Sprintf("dropped the act %s that %s owes the forge", str(args, "id"), str(args, "of")), str(args, "why")); err != nil { + return nil, err + } + a, err := asked(groups.OwedDrop, args) + if err != nil { + return nil, err + } + return map[string]any{"of": a.Of, "id": a.ID, "outcome": a.Outcome, "answer": a.Answer}, nil + }}, + } +} diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/gate.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/gate.go index a005d465..d0120eb9 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/gate.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/gate.go @@ -3,142 +3,10 @@ package main import ( "context" "errors" - "fmt" - "log" - "strconv" "github.com/nats-io/nats.go/jetstream" - - "mesh-delivery-forge-bridge/internal/bridge" ) -// performedBucket is the module's state bucket performed, `_` as heads is: for each key of the gate's -// buckets the bridge has acted on, the revision it acted at (manifest state entry performed, novox/hq ADR 0437 §8). -// Its keys are `owed.` and `merges.`; its value is the revision in decimal. -const performedBucket = forgeBridgeModule + "_performed" - -// kvRevisions is the bucket performed as the performer's revisions. -type kvRevisions struct{ kv jetstream.KeyValue } - -func (s kvRevisions) Get(ctx context.Context, key string) (uint64, bool, error) { - e, err := s.kv.Get(ctx, key) - if errors.Is(err, jetstream.ErrKeyNotFound) { - return 0, false, nil - } - if err != nil { - return 0, false, err - } - rev, err := strconv.ParseUint(string(e.Value()), 10, 64) - if err != nil { - return 0, false, fmt.Errorf("%s holds %q, which is no revision: %w", key, e.Value(), err) - } - return rev, true, nil -} - -func (s kvRevisions) Put(ctx context.Context, key string, rev uint64) error { - _, err := s.kv.PutString(ctx, key, strconv.FormatUint(rev, 10)) - return err -} - -// revisions is what the performer keeps of the revisions it acted at. -type revisions interface { - Get(ctx context.Context, key string) (rev uint64, found bool, err error) - Put(ctx context.Context, key string, rev uint64) error -} - -// performer does on the forge what the gate's keys ask (novox/hq ADR 0437 §8): one act for each revision of an owed -// key, one merge for each revision of a merges key. A refusal is written on FORGE and the act is not tried again until -// the key has a new revision; the performer holds no timer and serves no request. -type performer struct { - call caller - pub publisher - done revisions - logger *log.Logger -} - -// seen says whether the revision of the key was acted on already. A revision that cannot be read is not seen, so the -// act is done: a status set twice is the same status. -func (p *performer) seen(ctx context.Context, doneKey string, rev uint64) bool { - kept, found, err := p.done.Get(ctx, doneKey) - if err != nil { - p.logger.Printf("%s: the revision acted at could not be read, so the act is done again: %v", doneKey, err) - return false - } - return found && kept >= rev -} - -func (p *performer) keep(ctx context.Context, doneKey string, rev uint64) { - if err := p.done.Put(ctx, doneKey, rev); err != nil { - p.logger.Printf("%s: revision %d was acted on but could not be kept, so a restart acts on it again: %v", doneKey, rev, err) - } -} - -// refuse writes the refused message of an act on FORGE; it reports whether it was written. The message id makes a -// second write of one revision's refusal the same message to the stream. -func (p *performer) refuse(ctx context.Context, doneKey string, rev uint64, a bridge.Act, words string) bool { - msg, err := bridge.RefusedMessage(a, words) - if err != nil { - p.logger.Printf("%s: the forge refused %s (%s) and the refusal cannot be said on FORGE: %v", doneKey, a.Name, words, err) - return false - } - id := fmt.Sprintf("refused.%s.%d", doneKey, rev) - if _, err := p.pub.Publish(ctx, forgeSubject(msg.Event), msg.Data, jetstream.WithMsgID(id)); err != nil { - p.logger.Printf("%s: the forge refused %s (%s) and FORGE could not be written: %v", doneKey, a.Name, words, err) - return false - } - return true -} - -// owed performs the act an owed key of the gate asks at revision rev, once. -func (p *performer) owed(ctx context.Context, key string, rev uint64, value []byte) { - p.perform(ctx, "owed."+key, key, rev, value) -} - -// perform does the act of an owed key once for the revision, under the bookkeeping key doneKey. -func (p *performer) perform(ctx context.Context, doneKey, key string, rev uint64, value []byte) { - if p.seen(ctx, doneKey, rev) { - return - } - a, err := bridge.OwedAct(key, value) - if err != nil { - p.logger.Printf("%s at revision %d is not an act the bridge can perform, and is left: %v", key, rev, err) - p.keep(ctx, doneKey, rev) - return - } - if _, err := p.call(ctx, a.Call.Tool, a.Call.Args); err != nil { - p.logger.Printf("%s at revision %d: %s refused: %v", key, rev, a.Call.Tool, err) - if !p.refuse(ctx, doneKey, rev, a, err.Error()) { - return - } - } - p.keep(ctx, doneKey, rev) -} - -// merges performs the merges a merges key hands the bridge at revision rev, once: each pull request in the state -// merging, in the order the value names them, stopping at the first refusal (novox/hq ADR 0441 §3). -func (p *performer) merges(ctx context.Context, key string, rev uint64, value []byte) { - doneKey := "merges." + key - if p.seen(ctx, doneKey, rev) { - return - } - acts, err := bridge.MergeActs(value) - if err != nil { - p.logger.Printf("%s at revision %d cannot be merged, and is left: %v", key, rev, err) - p.keep(ctx, doneKey, rev) - return - } - for _, a := range acts { - if _, err := p.call(ctx, a.Call.Tool, a.Call.Args); err != nil { - p.logger.Printf("%s at revision %d: the merge of %s#%d refused: %v", key, rev, a.Repository, a.Number, err) - if !p.refuse(ctx, doneKey, rev, a, err.Error()) { - return - } - break - } - } - p.keep(ctx, doneKey, rev) -} - // watch hands each key of a bucket, with the revision it has now and then each new one, to react until ctx ends or the // watch fails. A deleted key is not an act. func watch(ctx context.Context, kv jetstream.KeyValue, react func(key string, rev uint64, value []byte)) error { diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/gate_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/gate_test.go index 7fced163..942df8cd 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/gate_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/gate_test.go @@ -8,10 +8,15 @@ import ( "log" "strings" "testing" + "time" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" + checker "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdeliverychecker" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" + + "mesh-delivery-forge-bridge/internal/gitfixture" + "mesh-delivery-forge-bridge/internal/groups" ) const ( @@ -19,195 +24,231 @@ const ( headB = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" ) -type memRevisions map[string]uint64 +type memStrs map[string]string -func (m memRevisions) Get(_ context.Context, key string) (uint64, bool, error) { +func (m memStrs) Get(_ context.Context, key string) (string, bool, error) { r, ok := m[key] return r, ok, nil } -func (m memRevisions) Put(_ context.Context, key string, rev uint64) error { m[key] = rev; return nil } +func (m memStrs) Put(_ context.Context, key, v string) error { m[key] = v; return nil } -type tcall struct { - key string - body string +type memPerformed map[string][]byte + +func (m memPerformed) Get(_ context.Context, key string) ([]byte, bool, error) { + v, ok := m[key] + return v, ok, nil +} +func (m memPerformed) Put(_ context.Context, key string, v []byte) error { m[key] = v; return nil } + +// seatFake answers each seat verb from answers, by its key; one absent is not answered (an error, as a bound passed). +type seatFake struct { + calls []string + answers map[string]json.RawMessage } -// forgeFake records the calls and answers each from refuse, a map from a tool to its refusal. -type forgeFake struct { - calls []tcall - refuse map[string]string -} - -func (f *forgeFake) call(_ context.Context, key string, body any) (json.RawMessage, error) { - b, _ := json.Marshal(body) - f.calls = append(f.calls, tcall{key, string(b)}) - if w, ok := f.refuse[key]; ok { - return nil, errors.New(w) +func (f *seatFake) call(_ context.Context, key string, _ any) (json.RawMessage, error) { + f.calls = append(f.calls, key) + if a, ok := f.answers[key]; ok { + return a, nil } - return json.RawMessage(`{}`), nil + return nil, errors.New("nats: timeout") } -func newPerformer(f *forgeFake) (*performer, *fakePub, memRevisions, *bytes.Buffer) { - pub, done, logs := &fakePub{}, memRevisions{}, &bytes.Buffer{} - return &performer{call: f.call, pub: pub, done: done, logger: log.New(logs, "", 0)}, pub, done, logs +func newOwing(f *seatFake) (*owing, *fakePub, memPerformed) { + pub, kept := &fakePub{}, memPerformed{} + at := time.Date(2026, 11, 9, 21, 0, 0, 0, time.UTC) + return &owing{call: f.call, pub: pub, performed: kept, now: func() time.Time { return at }, logger: log.New(&bytes.Buffer{}, "", 0)}, pub, kept } -// The values are the SDK's own encoding (mesh-sdk go/delivery EncodeOwed, EncodeMerge), the keys its OwedKey. -func owedKV(t *testing.T, repository string, number int, act string, o seat.OwedByRepositoryNumberHeadActValue) (string, []byte) { - t.Helper() - key, err := meshdeliveryclient.OwedKey(repository, number, headA, act) - if err != nil { - t.Fatal(err) - } - v, err := meshdeliveryclient.EncodeOwed(o) - if err != nil { - t.Fatal(err) - } - return key, v -} - -func TestAnOwedStatusIsSetOnceForARevision(t *testing.T) { // 1341/AC1 +// Design 47 *What the gate owes the forge*: an owed act is the seat git's verb under its id, its outcome said as +// owed.ended on the stream forge before it is kept in performed, an act it cannot read said undecodable; and at its start +// the bridge performs from the seat's groups every act said and not answered. +func TestWhatIsOwedIsPerformedSaidAndKept(t *testing.T) { t.Run("the names are the manifest's", theNamesAreTheManifests) - f := &forgeFake{} - p, _, _, _ := newPerformer(f) - key, v := owedKV(t, "novox/mesh-delivery", 12, "status=mesh/merge-gate", - seat.OwedByRepositoryNumberHeadActValue{Act: meshdeliveryclient.OwedStatus, Context: "mesh/merge-gate", State: "success", Text: "all green", Target: "https://forge.example/p/12"}) - p.owed(context.Background(), key, 7, v) - if len(f.calls) != 1 || f.calls[0].key != "gitea.gitea_commit_status" || - f.calls[0].body != `{"context":"mesh/merge-gate","description":"all green","owner":"novox","repo":"mesh-delivery","sha":"`+headA+`","state":"success","target_url":"https://forge.example/p/12"}` { - t.Fatalf("calls = %+v", f.calls) - } - p.owed(context.Background(), key, 7, v) // the same revision seen again (a restart's replay) - if len(f.calls) != 1 { - t.Errorf("the same revision called the tool again: %+v", f.calls) - } - p.owed(context.Background(), key, 8, v) // a new revision of the key is a new act - if len(f.calls) != 2 { - t.Errorf("a new revision was not performed: %+v", f.calls) - } -} - -func TestViewNoteAndCommentEachCallTheirToolOnce(t *testing.T) { // 1341/AC2 - f := &forgeFake{} - p, _, _, _ := newPerformer(f) + gate := func(name string) []byte { return gitfixture.Seat(t, seat.Seat, "events", name, "event:owed")[0] } + answer := func(verb, name string) json.RawMessage { return gitfixture.Given(t, name, "verb:"+verb+".answer") } for _, c := range []struct { - act string - o seat.OwedByRepositoryNumberHeadActValue + name, of string + data []byte + answers map[string]json.RawMessage + pubErr error + call string + outcome string + kept bool + err string }{ - {"view", seat.OwedByRepositoryNumberHeadActValue{Act: meshdeliveryclient.OwedView, Text: "the view"}}, - {"note", seat.OwedByRepositoryNumberHeadActValue{Act: meshdeliveryclient.OwedNote, Text: "a line", Ref: "mesh-plan"}}, - {"comment.1", seat.OwedByRepositoryNumberHeadActValue{Act: meshdeliveryclient.OwedComment, Text: "hello"}}, + {"a merge performed", seat.Seat, gate("owed is said, a merge"), + map[string]json.RawMessage{"seat:git.merge-pull": answer("merge-pull", "merge-pull answers")}, nil, + "seat:git.merge-pull", forgemsg.OwedPerformed, true, ""}, + {"a status refused", seat.Seat, gate("owed is said, a status"), + map[string]json.RawMessage{"seat:git.status": answer("status", "status answers an error, never none")}, nil, + "seat:git.status", forgemsg.OwedRefused, true, ""}, + {"the checker's status not answered", checker.Seat, gitfixture.Seat(t, checker.Seat, "events", "owed is said", "event:owed")[0], + nil, nil, "seat:git.status", forgemsg.OwedCouldNotBePerformed, true, ""}, + {"an act it cannot read", seat.Seat, []byte(`{`), nil, nil, "", forgemsg.OwedUndecodable, false, ""}, + {"a seat that says no owed act", "mesh-wants", gate("owed is said, a merge"), nil, nil, "", forgemsg.OwedUndecodable, false, ""}, + {"the stream forge refusing the outcome", seat.Seat, gate("owed is said, a merge"), + map[string]json.RawMessage{"seat:git.merge-pull": answer("merge-pull", "merge-pull answers")}, errors.New("no responders"), + "seat:git.merge-pull", "", false, "could not be said"}, } { - key, v := owedKV(t, "novox/mesh-delivery", 12, c.act, c.o) - p.owed(context.Background(), key, 1, v) - p.owed(context.Background(), key, 1, v) + f := &seatFake{answers: c.answers} + o, pub, kept := newOwing(f) + pub.err = c.pubErr + _, err := o.heard(context.Background(), c.of, c.data) + if c.err != "" { + if err == nil || !strings.Contains(err.Error(), c.err) || len(kept) != 0 { + t.Errorf("%s: %v, kept %d; want an error naming %q and nothing kept", c.name, err, len(kept), c.err) + } + continue + } + if err != nil || len(pub.data) != 1 || pub.subjects[0] != forgeSubject(forgemsg.OwedEndedEvent) { + t.Fatalf("%s: %v, said %v; want one owed.ended", c.name, err, pub.subjects) + } + e, err := forgemsg.DecodeOwedEnded(pub.data[0]) + if err != nil || e.Outcome != c.outcome || e.Of != c.of { + t.Errorf("%s: said %+v %v, want %s of %s", c.name, e, err, c.outcome, c.of) + } + if (c.call == "") != (len(f.calls) == 0) || (c.call != "" && f.calls[0] != c.call) { + t.Errorf("%s: called %v, want %q", c.name, f.calls, c.call) + } + if v, ok := kept[performedKey(c.of, e.ID)]; ok != c.kept { + t.Errorf("%s: kept %v (%s), want %v", c.name, ok, v, c.kept) + } else if ok { + var p Performed + if err := json.Unmarshal(v, &p); err != nil || p.Ended.Outcome != c.outcome || !json.Valid(p.Owed) { + t.Errorf("%s: kept %s, want the outcome and the owed event", c.name, v) + } + } } - var tools []string - for _, c := range f.calls { - tools = append(tools, c.key) - } - if got := strings.Join(tools, " "); got != "gitea.gitea_delivery_view gitea.gitea_note_append gitea.gitea_add_comment" { - t.Errorf("calls = %s", got) - } -} -func TestARefusedActIsWrittenOnForgeOnceAndNotRetried(t *testing.T) { // 1341/AC3 - f := &forgeFake{refuse: map[string]string{"gitea.gitea_add_comment": "the pull request is locked"}} - p, pub, _, _ := newPerformer(f) - key, v := owedKV(t, "novox/mesh-delivery", 12, "comment.1", seat.OwedByRepositoryNumberHeadActValue{Act: meshdeliveryclient.OwedComment, Text: "hello"}) - p.owed(context.Background(), key, 3, v) - p.owed(context.Background(), key, 3, v) - if len(f.calls) != 1 { - t.Errorf("the refused act was tried %d times at one revision", len(f.calls)) - } - if len(pub.subjects) != 1 || pub.subjects[0] != "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.refused" { - t.Fatalf("published on %v", pub.subjects) - } - m, err := forgemsg.DecodeForge(pub.data[0]) - if err != nil || m.Kind != forgemsg.ForgeRefused || m.Act != "comment.1" || m.Number != 12 || m.Head != headA || m.Words != "the pull request is locked" { - t.Errorf("message %+v err %v", m, err) - } - f.refuse = nil - p.owed(context.Background(), key, 4, v) // only a new revision tries again - if len(f.calls) != 2 { - t.Errorf("a new revision was not tried: %d calls", len(f.calls)) - } -} + // Design 47 *The switch carries what is owed*: an act the gate switched from its old state is performed only when the old + // bridge's own bookkeeping does not name its key at its revision or later; one it names is said performed, not done. + t.Run("a switched act the old bridge performed", func(t *testing.T) { + var owed map[string]any + if err := json.Unmarshal(gate("owed is said, a merge"), &owed); err != nil { + t.Fatal(err) + } + id, err := meshdeliveryclient.SwitchedActID(meshdeliveryclient.SwitchedMerges, "feat~x", 7) + if err != nil { + t.Fatal(err) + } + owed["id"] = id + data, _ := json.Marshal(owed) + for _, c := range []struct { + name string + kept string // the old bridge's merges.feat~x, "" for none + called bool + }{ + {"kept at its revision", "7", false}, + {"kept at a later revision", "9", false}, + {"kept at an earlier revision", "6", true}, + {"not kept", "", true}, + } { + f := &seatFake{answers: map[string]json.RawMessage{"seat:git.merge-pull": answer("merge-pull", "merge-pull answers")}} + o, pub, kept := newOwing(f) + if c.kept != "" { + kept["merges.feat~x"] = []byte(c.kept) + } + if _, err := o.heard(context.Background(), seat.Seat, data); err != nil || len(pub.data) != 1 { + t.Fatalf("%s: %v, said %d", c.name, err, len(pub.data)) + } + e, _ := forgemsg.DecodeOwedEnded(pub.data[0]) + if (len(f.calls) == 1) != c.called || e.Outcome != forgemsg.OwedPerformed || e.ID != id { + t.Errorf("%s: called %v, said %+v; want called %v and performed", c.name, f.calls, e, c.called) + } + if _, ok := kept[performedKey(seat.Seat, id)]; !ok { + t.Errorf("%s: the outcome is not kept under the act's id", c.name) + } + } + }) -func TestAnOwedValueThatCannotBeReadIsLoggedAndNotRetried(t *testing.T) { - f := &forgeFake{} - p, pub, done, logs := newPerformer(f) - p.owed(context.Background(), "not-a-key", 2, []byte(`{}`)) - if len(f.calls) != 0 || len(pub.subjects) != 0 { - t.Errorf("calls %v published %v", f.calls, pub.subjects) - } - if !strings.Contains(logs.String(), "not-a-key") { - t.Errorf("log %q does not name the key", logs.String()) - } - if rev, ok := done["owed.not-a-key"]; !ok || rev != 2 { - t.Errorf("revision kept = %v %v", rev, ok) - } -} + t.Run("heard from a seat's stream owed", func(t *testing.T) { + f := &seatFake{answers: map[string]json.RawMessage{"seat:git.merge-pull": answer("merge-pull", "merge-pull answers")}} + o, pub, _ := newOwing(f) + m := &fakeMsg{subject: "mesh.seat.mesh-delivery.event.owed.novox/mesh-sdk.178", data: gate("owed is said, a merge")} + o.hear(context.Background(), seat.Seat, m) + again := &fakeMsg{subject: m.subject, data: m.data} + o.hear(context.Background(), seat.Seat, again) + if m.acked != 1 || again.acked != 1 || len(f.calls) != 1 || len(pub.data) != 1 { + t.Errorf("an act said twice: acked %d and %d, called %v, said %d; want performed and said once, both acknowledged", + m.acked, again.acked, f.calls, len(pub.data)) + } + pub.err = errors.New("no responders") + lost := &fakeMsg{subject: m.subject, data: gate("owed is said, a status")} + o.hear(context.Background(), seat.Seat, lost) + if lost.acked != 0 || len(lost.nak) != 1 { + t.Errorf("an outcome not said: acked %d, nak %v; want it heard again", lost.acked, lost.nak) + } + }) -func mergeKV(t *testing.T, m seat.MergesByGroupValue) []byte { - t.Helper() - v, err := meshdeliveryclient.EncodeMerge(m) - if err != nil { - t.Fatal(err) - } - return v -} + t.Run("asked again, dropped, and its backlog read by hand", func(t *testing.T) { + groupsAnswer := gitfixture.Seat(t, seat.Seat, "verbs", "groups answers a group with its members and acts", "verb:groups.answer")[0] + f := &seatFake{answers: map[string]json.RawMessage{"seat:git.merge-pull": answer("merge-pull", "merge-pull answers"), + "seat:mesh-delivery.groups": groupsAnswer, "seat:git.status": answer("status", "status answers")}} + o, pub, kept := newOwing(f) + merge := "merge.feat=47-G.0123456789ab.novox/mesh-sdk#178" + if _, err := o.heard(context.Background(), seat.Seat, gate("owed is said, a merge")); err != nil { + t.Fatal(err) + } + backlog := func(context.Context) []groups.Backlog { return []groups.Backlog{{Stream: seat.StreamOwed, Waiting: 2}} } + for _, c := range []struct { + name string + ask groups.OwedAsk + calls, said int + outcome, answers string + }{ + {"again, from performed", groups.OwedAsk{Kind: groups.OwedAgain, Of: seat.Seat, ID: merge}, 2, 2, forgemsg.OwedPerformed, "performed"}, + // The fixture's status is in the groups only: the bridge never heard it. + {"again, from the groups", groups.OwedAsk{Kind: groups.OwedAgain, Of: seat.Seat, ID: "status.novox/mesh-sdk#178.aaaaaaaaaaaa.mesh/merge-gate"}, 4, 3, forgemsg.OwedPerformed, "performed"}, + {"dropped", groups.OwedAsk{Kind: groups.OwedDrop, Of: seat.Seat, ID: merge, Why: "merged by hand", By: "lead@shanks"}, 4, 4, forgemsg.OwedDropped, "merged by hand"}, + {"again, an act nobody holds", groups.OwedAsk{Kind: groups.OwedAgain, Of: checker.Seat, ID: "x"}, 4, 4, "", "checker says"}, + {"the backlog", groups.OwedAsk{Kind: groups.OwedBacklog}, 4, 4, "", "backlog"}, + } { + a := o.answer(context.Background(), c.ask, backlog) + if !a.Done() || a.Outcome != c.outcome || !strings.Contains(a.Answer, c.answers) || len(f.calls) != c.calls || len(pub.data) != c.said { + t.Errorf("%s: %+v, called %v, said %d; want %q naming %q, %d calls, %d said", c.name, a, f.calls, len(pub.data), c.outcome, c.answers, c.calls, c.said) + } + if c.ask.Kind == groups.OwedBacklog && (len(a.Backlog) != 1 || a.Backlog[0].Waiting != 2) { + t.Errorf("%s: %+v", c.name, a.Backlog) + } + } + var p Performed + if err := json.Unmarshal(kept[performedKey(seat.Seat, merge)], &p); err != nil || p.Ended.Outcome != forgemsg.OwedDropped { + t.Errorf("performed after the drop: %+v %v", p, err) + } + }) -func twoMembers(a, b meshdeliveryclient.MergeState) seat.MergesByGroupValue { - return seat.MergesByGroupValue{Group: "g", Ask: "merge.g.0123456789ab", - Approved: map[string]string{meshdeliveryclient.ApprovedKey("novox/mesh-sdk", 4): headB, meshdeliveryclient.ApprovedKey("novox/mesh-delivery", 12): headA}, - Pulls: []seat.MergesByGroupValuePullsItem{ - {Repository: "novox/mesh-delivery", Number: 12, Head: headA, State: string(a), Commit: map[bool]string{true: "c1"}[a == meshdeliveryclient.MergeMerged]}, - {Repository: "novox/mesh-sdk", Number: 4, Head: headB, State: string(b), Commit: map[bool]string{true: "c2"}[b == meshdeliveryclient.MergeMerged]}, - }} -} - -func TestAMergeRevisionMergesOnlyTheMemberHanded(t *testing.T) { // 1342/AC1 - f := &forgeFake{} - p, _, _, _ := newPerformer(f) - m := twoMembers(meshdeliveryclient.MergeMerging, meshdeliveryclient.MergeMerged) - p.merges(context.Background(), "g", 5, mergeKV(t, m)) - want := `{"approved_heads":["novox/mesh-delivery@` + headA + `","novox/mesh-sdk@` + headB + `"],"head":"` + headA + `","number":12,"owner":"novox","repo":"mesh-delivery"}` - if len(f.calls) != 1 || f.calls[0].key != "gitea.gitea_merge_pull_request" || f.calls[0].body != want { - t.Fatalf("calls = %+v", f.calls) - } - p.merges(context.Background(), "g", 5, mergeKV(t, m)) - if len(f.calls) != 1 { - t.Errorf("a revision merged twice: %+v", f.calls) - } -} - -func TestARefusedMergeIsWrittenOnForgeOnceAndNotRetried(t *testing.T) { // 1342/AC2 - f := &forgeFake{refuse: map[string]string{"gitea.gitea_merge_pull_request": "novox/mesh-delivery#12 is not merged: its head moved"}} - p, pub, _, _ := newPerformer(f) - v := mergeKV(t, twoMembers(meshdeliveryclient.MergeMerging, meshdeliveryclient.MergeMerged)) - p.merges(context.Background(), "g", 5, v) - p.merges(context.Background(), "g", 5, v) - if len(f.calls) != 1 || len(pub.data) != 1 { - t.Fatalf("calls %d messages %d, want one each", len(f.calls), len(pub.data)) - } - m, err := forgemsg.DecodeForge(pub.data[0]) - if err != nil || m.Kind != forgemsg.ForgeRefused || m.Repository != "novox/mesh-delivery" || m.Number != 12 || m.Head != headA || - m.Act != "merge" || m.Words != "novox/mesh-delivery#12 is not merged: its head moved" { - t.Errorf("message %+v err %v", m, err) - } - f.refuse = nil - p.merges(context.Background(), "g", 6, v) - if len(f.calls) != 2 { - t.Errorf("a new revision was not tried: %d calls", len(f.calls)) - } -} - -func TestAMergeWithTwoHandedStopsAtTheFirstRefusal(t *testing.T) { // ADR 0441 §3 - f := &forgeFake{refuse: map[string]string{"gitea.gitea_merge_pull_request": "no"}} - p, pub, _, _ := newPerformer(f) - p.merges(context.Background(), "g", 1, mergeKV(t, twoMembers(meshdeliveryclient.MergeMerging, meshdeliveryclient.MergeMerging))) - if len(f.calls) != 1 || len(pub.data) != 1 { - t.Errorf("calls %d messages %d, want the first refusal to stop the pass", len(f.calls), len(pub.data)) - } + t.Run("caught up from the groups", func(t *testing.T) { + groups := gitfixture.Seat(t, seat.Seat, "verbs", "groups answers a group with its members and acts", "verb:groups.answer")[0] + f := &seatFake{answers: map[string]json.RawMessage{"seat:mesh-delivery.groups": groups, + "seat:git.status": answer("status", "status answers")}} + o, pub, kept := newOwing(f) + // The fixture's merge has its outcome recorded; its status is said and unanswered. + if n, err := o.catchUp(context.Background()); err != nil || n != 1 || len(pub.data) != 1 || f.calls[1] != "seat:git.status" { + t.Fatalf("catch-up: %d %v, called %v; want the one unanswered status performed", n, err, f.calls) + } + if n, err := o.catchUp(context.Background()); err != nil || n != 0 || len(f.calls) != 3 { + t.Errorf("catch-up again: %d %v, called %v; want nothing performed that performed holds", n, err, f.calls) + } + if len(kept) != 1 { + t.Errorf("kept %v, want the status", kept) + } + // The seat answers a group it does not know with no groups, never an error (W11.10): nothing is performed and + // nothing fails; a groups the seat leaves unanswered is the catch-up's error. + none := gitfixture.Seat(t, seat.Seat, "verbs", "groups answers a group it does not know with no groups", "verb:groups.answer")[0] + for _, row := range []struct { + answer json.RawMessage + fails bool + }{{none, false}, {nil, true}} { + f := &seatFake{answers: map[string]json.RawMessage{}} + if row.answer != nil { + f.answers["seat:mesh-delivery.groups"] = row.answer + } + o, pub, _ := newOwing(f) + n, err := o.catchUp(context.Background()) + if (err != nil) != row.fails || n != 0 || len(pub.data) != 0 { + t.Errorf("catch-up from %s: %d performed, %v; want none performed, failing %v", row.answer, n, err, row.fails) + } + } + }) } diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groupask.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groupask.go index 3c0365b8..6ccd2b76 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groupask.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groupask.go @@ -8,11 +8,13 @@ import ( "github.com/nats-io/nats.go/jetstream" + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" + "mesh-delivery-forge-bridge/internal/groups" ) // endAsked answers one value of the state end-asks (the tool bridge_group_end_again, ADR 0633 §4): a value not yet answered -// says the group's end again from the gate's merges, with the ask's time appended to the message id so the stream's duplicate +// says the group's end again from the seat's groups, with the ask's time appended to the message id so the stream's duplicate // window does not drop it, and is written back by compare-and-set at its revision with whether it was written. A value // answered already (this bridge's own write) is left. func (g *grouper) endAsked(ctx context.Context, kv jetstream.KeyValue, key string, rev uint64, value []byte) { @@ -24,7 +26,7 @@ func (g *grouper) endAsked(ctx context.Context, kv jetstream.KeyValue, key strin if a.Done() { return } - a = g.answerEnd(ctx, key, a, time.Now()) + a = g.answerEnd(ctx, a, time.Now()) g.logger.Printf("the end of the group %q said again by %s (%s): %s", a.Group, a.By, a.Why, a.Answer) data, err := json.Marshal(a) if err != nil { @@ -35,27 +37,31 @@ func (g *grouper) endAsked(ctx context.Context, kv jetstream.KeyValue, key strin } } -// readMerges reads one key of the gate's merges state. -func (g *grouper) readMerges(ctx context.Context, key string) ([]byte, uint64, bool, error) { - if g.merges == nil { - return nil, 0, false, errors.New("the bridge does not read the gate's merges (its bucket was not there at the start)") - } - value, rev, found, err := g.merges(ctx, key) - return value, rev, found, err -} - // answerEnd says the group's end again for an ask and returns the ask with its answer. -func (g *grouper) answerEnd(ctx context.Context, key string, a groups.EndAsk, at time.Time) groups.EndAsk { +func (g *grouper) answerEnd(ctx context.Context, a groups.EndAsk, at time.Time) groups.EndAsk { a.Answered = at.UTC() - switch merges, mrev, found, err := g.readMerges(ctx, key); { - case err != nil: - a.Answer = "the gate's merges " + key + " cannot be read: " + err.Error() - case !found: - a.Answer = "the gate holds no merge of the group " + a.Group - case g.end(ctx, key, mrev, merges, groups.AgainStamp(a.At)): + if g.groups == nil { + a.Answer = "the bridge reads no groups" + return a + } + of, err := g.groups(ctx) + if err != nil { + a.Answer = "the groups cannot be read: " + err.Error() + return a + } + m, ok := seat.Group{}, false + for _, c := range of { + if c.ID == a.Group { + m, ok = c, true + } + } + switch { + case !ok || len(m.Members) < 2: + a.Answer = "the seat " + seat.Seat + " holds no group " + a.Group + " of more than one member" + case g.end(ctx, m, groups.AgainStamp(a.At)): a.Written, a.Answer = true, "the end of the group is written again" default: - a.Answer = "the end is not written: a member is not merged or its merged message is not written, or FORGE refused it (the bridge's journal says which)" + a.Answer = "the end is not written: a member's merged message is not written, or FORGE refused it (the bridge's journal says which)" } return a } diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups.go index c840f23a..1a4d65c9 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups.go @@ -2,37 +2,40 @@ package main import ( "context" - "errors" "log" - "slices" "github.com/nats-io/nats.go/jetstream" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" - "git.novox.be/novox/mesh-delivery/sdk/trunk" groupkeys "mesh-delivery-forge-bridge/internal/groups" ) // A delivery group's merges reach the indexer together (novox/hq ADR 0633 §4): the bridge sets the group on each -// member's merged message, from the gate's merges state it reads (ADR 0437 §8), and once every approved member is -// merged and its merged message written on FORGE it says the group's end, forge.group.merged, once. +// member's merged message, from the seat mesh-delivery's verb groups (design 47 *A group's merges are taken in together*; +// it reads no state of the gate's), and once every member's merged message is written on FORGE it says the group's end, +// forge.group.merged, once, after the last of them. // -// What it keeps of it is in its state performed, beside the gate's keys it acted on: `published.` for each -// member's merged message written (the hash is the first 16 hex digits of the sha256 of `#@`), -// and `groups.` once the group's end is written, its value the revision of the gate's merges key it was -// worked out from. A group whose end is not written yet is open (bridge_groups reads them). +// What it keeps of it is in its state performed: `published.` for each member's merged message written, its value +// the merge commit it named (the hash is the first 16 hex digits of the sha256 of `#`), and +// `groups.` once the group's end is written, its value the last member's merge commit. A group whose end is not +// written yet is open (bridge_groups reads them). -// merges reads one key of the gate's merges state. -type mergesRead func(ctx context.Context, key string) (value []byte, rev uint64, found bool, err error) +// groupsRead is every delivery group the seat answers. +type groupsRead func(ctx context.Context) ([]seat.Group, error) + +// strs is performed as string values by key: the grouper's `published.` and `groups.` keys. +type strs interface { + Get(ctx context.Context, key string) (value string, found bool, err error) + Put(ctx context.Context, key, value string) error +} // grouper sets a member's group and says a group's end. type grouper struct { - merges mergesRead + groups groupsRead send func(ctx context.Context, subject string, data []byte, id string) error // writes on FORGE with the message id - done revisions + done strs logger *log.Logger } @@ -44,63 +47,40 @@ func sendVia(pub publisher) func(context.Context, string, []byte, string) error } } -func kvMerges(kv jetstream.KeyValue) mergesRead { - return func(ctx context.Context, key string) ([]byte, uint64, bool, error) { - e, err := kv.Get(ctx, key) - if errors.Is(err, jetstream.ErrKeyNotFound) { - return nil, 0, false, nil - } - if err != nil { - return nil, 0, false, err - } - return e.Value(), e.Revision(), true, nil - } +// groupsVia reads the groups through the seat mesh-delivery's verb groups. +func groupsVia(call caller) groupsRead { + return func(ctx context.Context) ([]seat.Group, error) { return groupkeys.Read(ctx, call) } } -// groupOf is the gate's merge of the group a merged message's branch names, when the message's pull request is one of -// its approved members and the group has more than one: a lone pull request is no group. -func (g *grouper) groupOf(ctx context.Context, f forgemsg.Forge) (seat.MergesByGroupValue, uint64, bool) { - if g == nil || g.merges == nil || f.Branch == "" { - return seat.MergesByGroupValue{}, 0, false +// groupOf is the group a merged message's branch names, when the seat names the message's pull request among its +// members and the group has more than one: a lone pull request is no group. +func (g *grouper) groupOf(ctx context.Context, f forgemsg.Forge) (seat.Group, bool) { + if g == nil || g.groups == nil || f.Branch == "" { + return seat.Group{}, false } - key, err := meshdeliveryclient.MergesKey(f.Branch) + of, err := g.groups(ctx) if err != nil { - return seat.MergesByGroupValue{}, 0, false + g.logger.Printf("%s#%d: the groups cannot be read, so its merged message names no group: %v", f.Repository, f.Number, err) + return seat.Group{}, false } - value, rev, found, err := g.merges(ctx, key) - if err != nil { - g.logger.Printf("%s#%d: the gate's merges.%s cannot be read, so its merged message names no group: %v", f.Repository, f.Number, key, err) - return seat.MergesByGroupValue{}, 0, false - } - if !found { - return seat.MergesByGroupValue{}, 0, false - } - m, err := meshdeliveryclient.DecodeMerge(value) - if err != nil { - g.logger.Printf("%s#%d: the gate's merges.%s is not a merge, so its merged message names no group: %v", f.Repository, f.Number, key, err) - return seat.MergesByGroupValue{}, 0, false - } - if _, ok := m.Approved[meshdeliveryclient.ApprovedKey(f.Repository, f.Number)]; !ok || len(m.Approved) < 2 { - return seat.MergesByGroupValue{}, 0, false - } - return m, rev, true + return groupkeys.Named(of, f.Branch, f.Repository, f.Number) } -// member sets the group and its approved members on a merged message's body (ADR 0633 §4); any other body, and a -// merge of no group, is left as it is. +// member sets the group and its members on a merged message's body (ADR 0633 §4); any other body, and a merge of no +// group, is left as it is. func (g *grouper) member(ctx context.Context, data []byte) ([]byte, forgemsg.Forge, bool) { f, err := forgemsg.DecodeForge(data) if err != nil || f.Kind != forgemsg.ForgeMerged { return data, f, false } - m, _, ok := g.groupOf(ctx, f) + m, ok := g.groupOf(ctx, f) if !ok { return data, f, false } - f.Group, f.Members = m.Group, groupkeys.MembersOf(m) + f.Group, f.Members = m.ID, groupkeys.MembersOf(m) out, err := forgemsg.EncodeForge(f) if err != nil { - g.logger.Printf("%s#%d: its merged message cannot name the group %q, written without: %v", f.Repository, f.Number, m.Group, err) + g.logger.Printf("%s#%d: its merged message cannot name the group %q, written without: %v", f.Repository, f.Number, m.ID, err) return data, f, false } return out, f, true @@ -108,73 +88,55 @@ func (g *grouper) member(ctx context.Context, data []byte) ([]byte, forgemsg.For // published keeps that a member's merged message is written, and says the group's end when it was the last. func (g *grouper) published(ctx context.Context, f forgemsg.Forge) { - if err := g.done.Put(ctx, groupkeys.PublishedKey(f.Repository, f.Number, f.Merge), 1); err != nil { - g.logger.Printf("%s#%d: that its merged message is written is not kept, so the group's end waits for the next act: %v", f.Repository, f.Number, err) + if err := g.done.Put(ctx, groupkeys.PublishedKey(f.Repository, f.Number), f.Merge); err != nil { + g.logger.Printf("%s#%d: that its merged message is written is not kept, so the group's end waits for bridge_group_end_again: %v", f.Repository, f.Number, err) return } - key, err := meshdeliveryclient.MergesKey(f.Branch) - if err != nil { - return + if m, ok := g.groupOf(ctx, f); ok { + g.end(ctx, m, "") } - value, rev, found, err := g.merges(ctx, key) - if err != nil || !found { - return - } - g.ended(ctx, key, rev, value) } -// ended says the group's end (forge.group.merged) once every approved member of the gate's merge is merged and its -// merged message written, after the last of them; never on a refusal (ADR 0441 §3: a held group merges on from it). -// It is written once per group: `groups.` in performed says it was. again says it is written again by hand +// end says the group's end (forge.group.merged) once every member's merged message is written, after the last of them; +// never on a refusal (ADR 0441 §3: a held group merges on from it), since a refused member's message is never written. It +// is written once per group: `groups.` in performed says it was. again says it is written again by hand // (bridge_group_end_again), with the time appended to the message id, so the duplicate window does not drop it. -func (g *grouper) ended(ctx context.Context, key string, rev uint64, value []byte) { - g.end(ctx, key, rev, value, "") -} - -func (g *grouper) end(ctx context.Context, key string, rev uint64, value []byte, again string) bool { - m, err := meshdeliveryclient.DecodeMerge(value) - if err != nil || len(m.Approved) < 2 { +func (g *grouper) end(ctx context.Context, m seat.Group, again string) bool { + if len(m.Members) < 2 { + return false + } + key, err := groupkeys.Key(m.ID) + if err != nil { + g.logger.Printf("the group %q cannot be kept, so its end is not said: %v", m.ID, err) return false } if again == "" { - if _, seen, err := g.done.Get(ctx, groupkeys.Key(key)); err == nil && seen { + if _, seen, err := g.done.Get(ctx, key); err == nil && seen { return false } } - end := forgemsg.ForgeGroupEnd{Group: m.Group} - for _, member := range groupkeys.MembersOf(m) { - at := slices.IndexFunc(m.Pulls, func(p seat.MergesByGroupValuePullsItem) bool { - return trunk.SameRepository(p.Repository, member.Repository) && int(p.Number) == member.Number - }) - if at < 0 || m.Pulls[at].Commit == "" { - return false // not every member is merged (a merge commit is written only for a merged pull): the group stays gathered - } - if _, ok, err := g.done.Get(ctx, groupkeys.PublishedKey(member.Repository, member.Number, m.Pulls[at].Commit)); err != nil || !ok { + // The members in the group's order, which the gate merges them in (ADR 0441 §5), so the last commit names the end. + end := forgemsg.ForgeGroupEnd{Group: m.ID} + for _, member := range m.Members { + merge, ok, err := g.done.Get(ctx, groupkeys.PublishedKey(member.Repository, int(member.Number))) + if err != nil || !ok || merge == "" { return false // its merged message is not written yet: said when it is (published) } - end.Members = append(end.Members, forgemsg.GroupMerge{Repository: member.Repository, Number: member.Number, Merge: m.Pulls[at].Commit}) + end.Members = append(end.Members, forgemsg.GroupMerge{Repository: member.Repository, Number: int(member.Number), Merge: merge}) } - // The members in the order the gate merged them, so the last commit names the end. - slices.SortStableFunc(end.Members, func(a, b forgemsg.GroupMerge) int { - return slices.IndexFunc(m.Pulls, func(p seat.MergesByGroupValuePullsItem) bool { return p.Commit == a.Merge }) - - slices.IndexFunc(m.Pulls, func(p seat.MergesByGroupValuePullsItem) bool { return p.Commit == b.Merge }) - }) data, err := forgemsg.EncodeForgeGroupEnd(end) if err != nil { - g.logger.Printf("the end of the group %q cannot be written: %v", m.Group, err) + g.logger.Printf("the end of the group %q cannot be written: %v", m.ID, err) return false } - id := groupkeys.EndID(m.Group, end.Members[len(end.Members)-1].Merge, "") - if again != "" { - id += "." + again - } - if err := g.send(ctx, forgeSubject(forgemsg.ForgeGroupEndEvent), data, id); err != nil { - g.logger.Printf("the end of the group %q is not written on FORGE, written at the gate's next write or by bridge_group_end_again: %v", m.Group, err) + last := end.Members[len(end.Members)-1].Merge + if err := g.send(ctx, forgeSubject(forgemsg.ForgeGroupEndEvent), data, groupkeys.EndID(m.ID, last, again)); err != nil { + g.logger.Printf("the end of the group %q is not written on FORGE, written by bridge_group_end_again: %v", m.ID, err) return false } - if err := g.done.Put(ctx, groupkeys.Key(key), rev); err != nil { - g.logger.Printf("the end of the group %q is written and not kept: the stream's duplicate window drops a second one: %v", m.Group, err) + if err := g.done.Put(ctx, key, last); err != nil { + g.logger.Printf("the end of the group %q is written and not kept: the stream's duplicate window drops a second one: %v", m.ID, err) } - g.logger.Printf("the end of the group %q is written: %d members merged, the last %s", m.Group, len(end.Members), end.Members[len(end.Members)-1].Merge) + g.logger.Printf("the end of the group %q is written: %d members merged, the last %s", m.ID, len(end.Members), last) return true } diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups_test.go index b4100d67..3c24b39e 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups_test.go @@ -3,12 +3,12 @@ package main import ( "bytes" "context" + "errors" "log" "strings" "testing" "time" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" @@ -26,32 +26,20 @@ func (p *idPub) send(_ context.Context, subject string, data []byte, id string) return nil } -func TestAGroupsMembersNameItAndItsEndIsSaidOnce(t *testing.T) { // 1585/T16, T17, T19 (ADR 0633 §4) +// 1585/T16, T17, T19 (ADR 0633 §4), acceptance 49: a member's merged message names its group from the seat +// mesh-delivery's verb groups, the group's end is said once after the last member's merged message, and said again by hand. +func TestAGroupsMembersNameItAndItsEndIsSaidOnce(t *testing.T) { ctx := context.Background() h1, h2 := strings.Repeat("1", 40), strings.Repeat("2", 40) c1, c2 := strings.Repeat("c", 40), strings.Repeat("d", 40) - merge := func(states ...seat.MergesByGroupValuePullsItem) []byte { - v, err := meshdeliveryclient.EncodeMerge(seat.MergesByGroupValue{Group: "feat/take-in", Ask: "ask", Pulls: states, - Approved: map[string]string{"novox/mesh-sdk@37": h1, "novox/mesh-delivery@9": h2}}) - if err != nil { - t.Fatal(err) - } - return v - } - sdk := func(state meshdeliveryclient.MergeState, commit, words string) seat.MergesByGroupValuePullsItem { - return seat.MergesByGroupValuePullsItem{Repository: "novox/mesh-sdk", Number: 37, Head: h1, State: string(state), Commit: commit, Words: words} - } - dlv := func(state meshdeliveryclient.MergeState, commit string) seat.MergesByGroupValuePullsItem { - return seat.MergesByGroupValuePullsItem{Repository: "novox/mesh-delivery", Number: 9, Head: h2, State: string(state), Commit: commit} - } - gate := map[string][]byte{} + group := seat.Group{ID: "feat/take-in", State: "merging", Members: []seat.Member{ + {Repository: "novox/mesh-sdk", Number: 37, Head: h1, State: "merged"}, {Repository: "novox/mesh-delivery", Number: 9, Head: h2, State: "merging"}}} + var of []seat.Group + var read error var logs bytes.Buffer - pub, done := &idPub{}, memRevisions{} + pub, done := &idPub{}, memStrs{} g := &grouper{send: pub.send, done: done, logger: log.New(&logs, "", 0), - merges: func(_ context.Context, key string) ([]byte, uint64, bool, error) { - v, ok := gate[key] - return v, 7, ok, nil - }} + groups: func(context.Context) ([]seat.Group, error) { return of, read }} mergedOf := func(repo string, number int, head, commit string) []byte { data, err := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: repo, Branch: "feat/take-in", Number: number, Head: head, Merge: commit, Root: "r"}) if err != nil { @@ -59,79 +47,64 @@ func TestAGroupsMembersNameItAndItsEndIsSaidOnce(t *testing.T) { // 1585/T16, T1 } return data } - // A lone pull request's merged message carries no group. - if _, f, ok := g.member(ctx, mergedOf("novox/hq", 4, h1, c1)); ok || f.Group != "" { + // A lone pull request's merged message carries no group, and neither does one whose groups cannot be read. + of = []seat.Group{group, {ID: "solo", Members: []seat.Member{{Repository: "novox/hq", Number: 4, Head: h1}}}} + lone, _ := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/hq", Branch: "solo", Number: 4, Head: h1, Merge: c1, Root: "r"}) + if _, f, ok := g.member(ctx, lone); ok || f.Group != "" { t.Errorf("a lone pull request names the group %q", f.Group) } - // The first member: its message names the group and both members; the group is not ended. - gate["feat/take-in"] = merge(sdk(meshdeliveryclient.MergeMerged, c1, ""), dlv(meshdeliveryclient.MergeMerging, "")) + read = errors.New("the seat did not answer") + if _, f, ok := g.member(ctx, mergedOf("novox/mesh-sdk", 37, h1, c1)); ok || f.Group != "" || !strings.Contains(logs.String(), "the seat did not answer") { + t.Errorf("a group read that failed names the group %q: %s", f.Group, logs.String()) + } + read = nil + // The first member: its message names the group and both members in the group's order; the group is not ended. data, f, ok := g.member(ctx, mergedOf("novox/mesh-sdk", 37, h1, c1)) - if back, err := forgemsg.DecodeForge(data); !ok || err != nil || back.Group != "feat/take-in" || len(back.Members) != 2 { + if back, err := forgemsg.DecodeForge(data); !ok || err != nil || back.Group != "feat/take-in" || len(back.Members) != 2 || back.Members[0].Number != 37 { t.Fatalf("a member's message: %+v %v", back, err) } g.published(ctx, f) - // A refusal of the other member is no end (ADR 0441 §3: the group merges on from it later). - gate["feat/take-in"] = merge(sdk(meshdeliveryclient.MergeMerged, c1, ""), seat.MergesByGroupValuePullsItem{Repository: "novox/mesh-delivery", Number: 9, Head: h2, State: string(meshdeliveryclient.MergeRefused), Words: "405"}) - g.ended(ctx, "feat/take-in", 8, gate["feat/take-in"]) - // Merged on at the gate before its merged message is written: still no end; then its message ends the group. - gate["feat/take-in"] = merge(sdk(meshdeliveryclient.MergeMerged, c1, ""), dlv(meshdeliveryclient.MergeMerged, c2)) - g.ended(ctx, "feat/take-in", 9, gate["feat/take-in"]) - if len(pub.subjects) != 0 { - t.Fatalf("an end before every member's merged message is written: %v", pub.subjects) + if len(pub.subjects) != 0 || done[groupkeys.PublishedKey("novox/mesh-sdk", 37)] != c1 { + t.Fatalf("after the first member: ends %v, performed %v", pub.subjects, done) } + // The last member's merged message ends the group; a second merged message of it says no second end. _, f, _ = g.member(ctx, mergedOf("novox/mesh-delivery", 9, h2, c2)) g.published(ctx, f) - g.ended(ctx, "feat/take-in", 10, gate["feat/take-in"]) // a later write of the gate says no second end + g.published(ctx, f) if len(pub.subjects) != 1 || pub.subjects[0] != "mesh.stream.mesh-delivery-forge-bridge.event.forge.group.merged" || pub.ids[0] != "group-end.feat/take-in."+c2 { t.Fatalf("ends %v ids %v, want one forge.group.merged with id group-end.feat/take-in.%s", pub.subjects, pub.ids, c2) } end, err := forgemsg.DecodeForgeGroupEnd(pub.data[0]) if err != nil || len(end.Members) != 2 || end.Members[0].Merge != c1 || end.Members[1].Merge != c2 { - t.Errorf("the end = %+v %v, want both members in their merge order", end, err) + t.Errorf("the end = %+v %v, want both members in the group's order", end, err) } - if rev, ok := done[groupkeys.Key("feat/take-in")]; !ok || rev != 7 { - t.Errorf("performed %v: want groups.feat/take-in kept", done) - } - // Again by hand: written once more, its id not the first's. - if !g.end(ctx, "feat/take-in", 10, gate["feat/take-in"], "20261104T2300") || pub.ids[1] == pub.ids[0] { - t.Errorf("the end again by hand: ids %v", pub.ids) - } - // 1585/T19: an ask to say a group's end again is answered by the bridge from the gate's merges, with the ask's time on - // the message id; not written while a member is not merged, with the reason; not for a group the gate does not hold. - pull := func(repo string, n int, head string, state meshdeliveryclient.MergeState, commit string) seat.MergesByGroupValuePullsItem { - return seat.MergesByGroupValuePullsItem{Repository: repo, Number: int64(n), Head: head, State: string(state), Commit: commit} - } - askMerge := func(second meshdeliveryclient.MergeState, commit2 string) []byte { - v, err := meshdeliveryclient.EncodeMerge(seat.MergesByGroupValue{Group: "feat/take-in", Ask: "ask", Approved: map[string]string{"novox/mesh-sdk@37": h1, "novox/mesh-delivery@9": h2}, - Pulls: []seat.MergesByGroupValuePullsItem{pull("novox/mesh-sdk", 37, h1, meshdeliveryclient.MergeMerged, c1), pull("novox/mesh-delivery", 9, h2, second, commit2)}}) - if err != nil { - t.Fatal(err) - } - return v + if key, _ := groupkeys.Key("feat/take-in"); done[key] != c2 { + t.Errorf("performed %v: want groups.feat~take-in kept with the last merge", done) } + // 1585/T19: an ask to say a group's end again is answered from the seat's groups, with the ask's time on the message id; + // not written while a member's merged message is not written, with the reason; not for a group the seat does not hold, + // nor when the groups cannot be read. ask := groupkeys.EndAsk{V: 1, Group: "feat/take-in", By: "operator@shanks", Why: "the indexer never took it in", At: time.Date(2026, 11, 4, 23, 0, 0, 123, time.UTC)} for _, tc := range []struct { name string - gate map[string][]byte + of []seat.Group + read error published bool written bool answer string }{ - {"merged and published: written again", map[string][]byte{"feat/take-in": askMerge(meshdeliveryclient.MergeMerged, c2)}, true, true, "written again"}, - {"a member's message not written", map[string][]byte{"feat/take-in": askMerge(meshdeliveryclient.MergeMerged, c2)}, false, false, "not written"}, - {"a member not merged", map[string][]byte{"feat/take-in": askMerge(meshdeliveryclient.MergeMerging, "")}, true, false, "not written"}, - {"no merge at the gate", map[string][]byte{}, true, false, "holds no merge"}, + {"every message written: written again", []seat.Group{group}, nil, true, true, "written again"}, + {"a member's message not written", []seat.Group{group}, nil, false, false, "not written"}, + {"no such group at the seat", []seat.Group{{ID: "other", Members: group.Members}}, nil, true, false, "holds no group"}, + {"the groups cannot be read", nil, errors.New("no answer"), true, false, "cannot be read"}, } { - pub, done := &idPub{}, memRevisions{} + pub, done := &idPub{}, memStrs{} if tc.published { - done[groupkeys.PublishedKey("novox/mesh-sdk", 37, c1)], done[groupkeys.PublishedKey("novox/mesh-delivery", 9, c2)] = 1, 1 + done[groupkeys.PublishedKey("novox/mesh-sdk", 37)], done[groupkeys.PublishedKey("novox/mesh-delivery", 9)] = c1, c2 } g := &grouper{send: pub.send, done: done, logger: log.New(&bytes.Buffer{}, "", 0), - merges: func(_ context.Context, key string) ([]byte, uint64, bool, error) { - v, ok := tc.gate[key] - return v, 4, ok, nil - }} - got := g.answerEnd(ctx, "feat/take-in", ask, ask.At.Add(time.Second)) + groups: func(context.Context) ([]seat.Group, error) { return tc.of, tc.read }} + got := g.answerEnd(ctx, ask, ask.At.Add(time.Second)) if got.Written != tc.written || !strings.Contains(got.Answer, tc.answer) || !got.Done() || len(pub.ids) != map[bool]int{true: 1}[tc.written] { t.Errorf("%s: %+v, ids %v", tc.name, got, pub.ids) } diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/integration_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/integration_test.go index 2c54ba13..a33ce668 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/integration_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/integration_test.go @@ -7,6 +7,7 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "errors" "fmt" "io" "os" @@ -20,28 +21,27 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" + git "git.novox.be/novox/mesh-delivery/contracts/go/seats/git" seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" + checker "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdeliverychecker" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-sdk/go/keys" "git.novox.be/novox/mesh-sdk/go/process" + + "mesh-delivery-forge-bridge/internal/gitfixture" + groupkeys "mesh-delivery-forge-bridge/internal/groups" ) -// busConf is the shape of the composed accounts file (mesh-controller internal/broker): one account with JetStream, -// the controller's user, and the bridge's own principal . with the permissions the controller composes for -// it, and no others, so a permission the bridge lacks fails a test here and not on the mesh (issue 1279). +// busConf is the shape of the composed accounts file (the grants part's buscompose): one account with JetStream, the +// controller's user, and the bridge's own principal . with the permissions composed for its manifest, and +// no others, so a permission the bridge lacks fails a test here and not on the mesh (issue 1279). // -// The gate's buckets mesh-delivery-gate_merges and mesh-delivery-gate_owed are read and watched (the manifest's reads), the -// bucket performed is the bridge's own (its state), and the five forge tools of the gate's acts are invoked (issue 1341, 1342). -// -// The bridge neither reads the checker's owed nor hears its stream (ADR 0521, R4e finding R2-1): the forge module writes -// mesh/check from it. The stream checks is made here so a test can say the checker wrote a status. -// -// The lists are what mesh-controller's broker.PermissionsFor answers (internal/broker/nats.go:577) for a module principal -// with the manifest of module.json: consumes the three gitea.pull events, invokes gitea.gitea_open_pulls, -// gitea.gitea_commit_trees and gitea.gitea_get_file, state heads, performed, inputs and end-asks (ADR 0633 §5's hand end), and the stream forge; `**` is written `>` on the wire (modstreams.go -// WirePattern). The consumer's durable is _ on EVENTS (nats.go consumerDurable). The controller's own -// user keeps all rights, as it has. +// From module.json: the seat git's events heard through its durable on EVENTS; what the gate and the checker owe heard +// through its durable on each seat's own stream owed (SEAT__owed); the seat git's verbs, the seat mesh-delivery's +// groups and the seat mesh-hand-acts' hand-act called on mesh.seat..call.. (or its plain fallback); +// its own states heads, performed, inputs, end-asks and owed-asks; its stream forge. It reads no state of the gate's or +// the checker's (design 47, acceptance 49 and 53). `**` is written `>` on the wire. The controller's own user keeps all +// rights, as it has. const busConf = ` listen: 127.0.0.1:-1 jetstream { store_dir: %q } @@ -53,38 +53,23 @@ accounts { { user: "n1.mesh-delivery-forge-bridge", password: "pw" permissions { publish { allow: [ - "$JS.ACK.EVENTS.n1_mesh-delivery-forge-bridge.>" "$JS.API.CONSUMER.CREATE.KV_mesh-delivery-forge-bridge_heads.>" "$JS.API.CONSUMER.DELETE.KV_mesh-delivery-forge-bridge_heads.>" - "$JS.API.CONSUMER.INFO.EVENTS.n1_mesh-delivery-forge-bridge" - "$JS.API.CONSUMER.MSG.NEXT.EVENTS.n1_mesh-delivery-forge-bridge" "$JS.API.DIRECT.GET.KV_mesh-delivery-forge-bridge_heads.>" "$JS.API.STREAM.INFO.KV_mesh-delivery-forge-bridge_heads" "$JS.FC.KV_mesh-delivery-forge-bridge_heads.>" "$KV.mesh-delivery-forge-bridge_heads.>" - "mesh.mod.gitea.call.gitea_commit_trees.*.n1~mesh-delivery-forge-bridge" - "mesh.mod.gitea.call.gitea_commit_trees.n1~mesh-delivery-forge-bridge" - "mesh.mod.gitea.call.gitea_open_pulls.*.n1~mesh-delivery-forge-bridge" - "mesh.mod.gitea.call.gitea_open_pulls.n1~mesh-delivery-forge-bridge" - "mesh.mod.gitea.tool.gitea_commit_trees" - "mesh.mod.gitea.tool.gitea_commit_trees.*" - "mesh.mod.gitea.call.gitea_get_file.*.n1~mesh-delivery-forge-bridge" - "mesh.mod.gitea.call.gitea_get_file.n1~mesh-delivery-forge-bridge" - "mesh.mod.gitea.tool.gitea_get_file" - "mesh.mod.gitea.tool.gitea_get_file.*" - "mesh.mod.gitea.tool.gitea_open_pulls" - "mesh.mod.gitea.tool.gitea_open_pulls.*" "$JS.API.CONSUMER.CREATE.KV_mesh-delivery-forge-bridge_performed.>" - "$JS.API.CONSUMER.CREATE.KV_mesh-delivery-forge-bridge_inputs.>" "$JS.API.CONSUMER.DELETE.KV_mesh-delivery-forge-bridge_performed.>" - "$JS.API.CONSUMER.DELETE.KV_mesh-delivery-forge-bridge_inputs.>" "$JS.API.DIRECT.GET.KV_mesh-delivery-forge-bridge_performed.>" - "$JS.API.DIRECT.GET.KV_mesh-delivery-forge-bridge_inputs.>" "$JS.API.STREAM.INFO.KV_mesh-delivery-forge-bridge_performed" - "$JS.API.STREAM.INFO.KV_mesh-delivery-forge-bridge_inputs" "$JS.FC.KV_mesh-delivery-forge-bridge_performed.>" - "$JS.FC.KV_mesh-delivery-forge-bridge_inputs.>" "$KV.mesh-delivery-forge-bridge_performed.>" + "$JS.API.CONSUMER.CREATE.KV_mesh-delivery-forge-bridge_inputs.>" + "$JS.API.CONSUMER.DELETE.KV_mesh-delivery-forge-bridge_inputs.>" + "$JS.API.DIRECT.GET.KV_mesh-delivery-forge-bridge_inputs.>" + "$JS.API.STREAM.INFO.KV_mesh-delivery-forge-bridge_inputs" + "$JS.FC.KV_mesh-delivery-forge-bridge_inputs.>" "$KV.mesh-delivery-forge-bridge_inputs.>" "$JS.API.CONSUMER.CREATE.KV_mesh-delivery-forge-bridge_end-asks.>" "$JS.API.CONSUMER.DELETE.KV_mesh-delivery-forge-bridge_end-asks.>" @@ -92,45 +77,51 @@ accounts { "$JS.API.STREAM.INFO.KV_mesh-delivery-forge-bridge_end-asks" "$JS.FC.KV_mesh-delivery-forge-bridge_end-asks.>" "$KV.mesh-delivery-forge-bridge_end-asks.>" - "$JS.API.CONSUMER.CREATE.KV_mesh-delivery-gate_merges.>" - "$JS.API.CONSUMER.DELETE.KV_mesh-delivery-gate_merges.>" - "$JS.API.DIRECT.GET.KV_mesh-delivery-gate_merges.>" - "$JS.API.STREAM.INFO.KV_mesh-delivery-gate_merges" - "$JS.FC.KV_mesh-delivery-gate_merges.>" - "$JS.API.CONSUMER.CREATE.KV_mesh-delivery-gate_owed.>" - "$JS.API.CONSUMER.DELETE.KV_mesh-delivery-gate_owed.>" - "$JS.API.DIRECT.GET.KV_mesh-delivery-gate_owed.>" - "$JS.API.STREAM.INFO.KV_mesh-delivery-gate_owed" - "$JS.FC.KV_mesh-delivery-gate_owed.>" - "mesh.mod.gitea.call.gitea_merge_pull_request.*.n1~mesh-delivery-forge-bridge" - "mesh.mod.gitea.call.gitea_merge_pull_request.n1~mesh-delivery-forge-bridge" - "mesh.mod.gitea.tool.gitea_merge_pull_request" - "mesh.mod.gitea.tool.gitea_merge_pull_request.*" - "mesh.mod.gitea.call.gitea_commit_status.*.n1~mesh-delivery-forge-bridge" - "mesh.mod.gitea.call.gitea_commit_status.n1~mesh-delivery-forge-bridge" - "mesh.mod.gitea.tool.gitea_commit_status" - "mesh.mod.gitea.tool.gitea_commit_status.*" - "mesh.mod.gitea.call.gitea_delivery_view.*.n1~mesh-delivery-forge-bridge" - "mesh.mod.gitea.call.gitea_delivery_view.n1~mesh-delivery-forge-bridge" - "mesh.mod.gitea.tool.gitea_delivery_view" - "mesh.mod.gitea.tool.gitea_delivery_view.*" - "mesh.mod.gitea.call.gitea_note_append.*.n1~mesh-delivery-forge-bridge" - "mesh.mod.gitea.call.gitea_note_append.n1~mesh-delivery-forge-bridge" - "mesh.mod.gitea.tool.gitea_note_append" - "mesh.mod.gitea.tool.gitea_note_append.*" - "mesh.mod.gitea.call.gitea_add_comment.*.n1~mesh-delivery-forge-bridge" - "mesh.mod.gitea.call.gitea_add_comment.n1~mesh-delivery-forge-bridge" - "mesh.mod.gitea.tool.gitea_add_comment" - "mesh.mod.gitea.tool.gitea_add_comment.*" + "$JS.API.CONSUMER.CREATE.KV_mesh-delivery-forge-bridge_owed-asks.>" + "$JS.API.CONSUMER.DELETE.KV_mesh-delivery-forge-bridge_owed-asks.>" + "$JS.API.DIRECT.GET.KV_mesh-delivery-forge-bridge_owed-asks.>" + "$JS.API.STREAM.INFO.KV_mesh-delivery-forge-bridge_owed-asks" + "$JS.FC.KV_mesh-delivery-forge-bridge_owed-asks.>" + "$KV.mesh-delivery-forge-bridge_owed-asks.>" + "$JS.ACK.EVENTS.n1_mesh-delivery-forge-bridge.>" + "$JS.API.CONSUMER.INFO.EVENTS.n1_mesh-delivery-forge-bridge" + "$JS.API.CONSUMER.MSG.NEXT.EVENTS.n1_mesh-delivery-forge-bridge" + "$JS.ACK.SEAT_mesh-delivery_owed.n1_mesh-delivery-forge-bridge.>" + "$JS.API.CONSUMER.INFO.SEAT_mesh-delivery_owed.n1_mesh-delivery-forge-bridge" + "$JS.API.CONSUMER.MSG.NEXT.SEAT_mesh-delivery_owed.n1_mesh-delivery-forge-bridge" + "$JS.ACK.SEAT_mesh-delivery-checker_owed.n1_mesh-delivery-forge-bridge.>" + "$JS.API.CONSUMER.INFO.SEAT_mesh-delivery-checker_owed.n1_mesh-delivery-forge-bridge" + "$JS.API.CONSUMER.MSG.NEXT.SEAT_mesh-delivery-checker_owed.n1_mesh-delivery-forge-bridge" + "mesh.seat.git.call.status.n1~mesh-delivery-forge-bridge" + "mesh.seat.git.tool.status" + "mesh.seat.git.call.comment.n1~mesh-delivery-forge-bridge" + "mesh.seat.git.tool.comment" + "mesh.seat.git.call.view.n1~mesh-delivery-forge-bridge" + "mesh.seat.git.tool.view" + "mesh.seat.git.call.note.n1~mesh-delivery-forge-bridge" + "mesh.seat.git.tool.note" + "mesh.seat.git.call.merge-pull.n1~mesh-delivery-forge-bridge" + "mesh.seat.git.tool.merge-pull" + "mesh.seat.git.call.open-pulls.n1~mesh-delivery-forge-bridge" + "mesh.seat.git.tool.open-pulls" + "mesh.seat.git.call.trees.n1~mesh-delivery-forge-bridge" + "mesh.seat.git.tool.trees" + "mesh.seat.git.call.file.n1~mesh-delivery-forge-bridge" + "mesh.seat.git.tool.file" + "mesh.seat.git.call.pull.n1~mesh-delivery-forge-bridge" + "mesh.seat.git.tool.pull" + "mesh.seat.git.call.tags.n1~mesh-delivery-forge-bridge" + "mesh.seat.git.tool.tags" + "mesh.seat.mesh-delivery.call.groups.n1~mesh-delivery-forge-bridge" + "mesh.seat.mesh-delivery.tool.groups" + "mesh.seat.mesh-hand-acts.call.hand-act.n1~mesh-delivery-forge-bridge" + "mesh.seat.mesh-hand-acts.tool.hand-act" "mesh.stream.mesh-delivery-forge-bridge.event.forge.>" + "mesh.stream.mesh-delivery-forge-bridge.event.owed.ended" ] } subscribe { allow: [ "_INBOX.n1.mesh-delivery-forge-bridge.>" "mesh.assignment.n1.mesh-delivery-forge-bridge" - "mesh.mod.gitea.event.forge.delete" - "mesh.mod.gitea.event.pull.closed" - "mesh.mod.gitea.event.pull.merged" - "mesh.mod.gitea.event.pull.updated" "mesh.mod.mesh-delivery-forge-bridge.call.>" "mesh.mod.mesh-delivery-forge-bridge.tool.>" ] } @@ -186,26 +177,32 @@ type bed struct { ctx context.Context done <-chan error reply func(string) - // files sets what the stand-in forge answers to gitea_get_file from then on; it is asked with the tool's arguments. - files func(func(owner, repo, path, ref string) (string, bool)) + // files sets what the stand-in seat git answers to file from then on; it is asked with the verb's input. + files func(func(repository, path, ref string) (string, bool)) + // answer sets the stand-in's answer to one verb of a seat (`.`) from then on. + answer func(verb string, answer string) + // calls are the verbs the stand-in was called with, in order, as `. `. + calls func() []string + // restart stops the bridge, runs between, and starts it again on the same bus. + restart func(between func()) } -// getFileReply is the reply envelope for gitea_get_file as mesh-catalog modules/gitea tools/index.ts and client.ts getFile -// answer it ({file: {path, ref, sha, size, content}}), and for a path the forge does not have the error client.ts request -// writes: `Gitea API : 404 `. -func getFileReply(path, ref, content string, found bool) []byte { +// answerOf is a verb's answer in the reply envelope of mesh-sdk go/process/call.go ({result, error, node}). +func answerOf(answer string) []byte { return []byte(`{"result":` + answer + `,"node":"n0"}`) } + +// fileAnswer is the seat git's answer to file: the content, or absent. +func fileAnswer(path, ref, content string, found bool) string { if !found { - b, _ := json.Marshal(map[string]any{"error": "Gitea API /repos/novox/mesh-delivery/contents/" + path + "?ref=" + ref + `: 404 {"message":"GetContentsOrList","url":"https://forge.example/api/swagger"}`}) - return b + return `{"absent":true}` } - b, _ := json.Marshal(map[string]any{"result": map[string]any{"file": map[string]any{"path": path, "ref": ref, "sha": "0123456789abcdef0123456789abcdef01234567", - "size": len(content), "content": content}}, "node": "n0"}) - return b + b, _ := json.Marshal(git.FileAnswer{Path: path, Ref: ref, Content: content, Size: int64(len(content))}) + return string(b) } -// startBed is startBridge with a step between the streams and the consumer: prepare writes what the streams hold before -// the bridge's consumer is made, as the controller makes it, from now (ADR 0443 §1), and reply sets the answer of the -// forge's gitea_open_pulls from then on. +// startBed is startBridge with a step between the streams and the consumers: prepare writes what the streams hold before +// the bridge's consumers are made, as the grants part makes them, from now (ADR 0443 §1), and reply sets the answer of the +// seat git's open-pulls from then on. A stand-in holds the seats the bridge calls: git (every verb, trees r0 by default, +// a module.json of {} at every path), mesh-delivery's groups (none by default) and mesh-hand-acts' hand-act. func startBed(t *testing.T, openReply string, prepare func(ctx context.Context, js jetstream.JetStream)) *bed { srv := startBus(t) ctl, err := nats.Connect(srv.ClientURL(), nats.UserInfo("ctl", "pw")) @@ -219,70 +216,74 @@ func startBed(t *testing.T, openReply string, prepare func(ctx context.Context, } ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) t.Cleanup(cancel) - if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>"}}); err != nil { + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"}}); err != nil { t.Fatal(err) } - if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "FORGE", Subjects: []string{"mesh.stream.mesh-delivery-forge-bridge.event.forge.>"}}); err != nil { + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "FORGE", Subjects: []string{"mesh.stream.mesh-delivery-forge-bridge.event.>"}}); err != nil { t.Fatal(err) } - // The checker's stream checks, which the bridge no longer hears (ADR 0521): a test says on it that the checker wrote a status. - if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "CHECKS", Subjects: []string{"mesh.stream.mesh-delivery-checker.event.>"}}); err != nil { - t.Fatal(err) + // Each seat's own stream owed, as its contract keeps it (Stream, SEAT__owed), as the grants part makes it: no + // subjects of its own, sourced from EVENTS filtered by the event's subject. + for name, subjects := range map[string]string{seat.StreamOwed: seat.SubjectOwedAll, checker.StreamOwed: checker.SubjectOwedAll} { + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: name, MaxAge: 7 * 24 * time.Hour, + Sources: []*jetstream.StreamSource{{Name: "EVENTS", FilterSubject: subjects}}}); err != nil { + t.Fatal(err) + } } - // The bridge's state bucket, made by the controller from the manifest's state entry before the process starts. - if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: headsBucket, History: 1}); err != nil { - t.Fatal(err) - } - // Its other state bucket, and the gate's buckets the bridge reads, made by the controller from the gate's manifest - // (the gate's merges and owed with their histories). - for name, history := range map[string]uint8{performedBucket: 1, inputsBucket: 1, mergesBucket: 10, owedBucket: 5} { - if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: history}); err != nil { + // The bridge's states, made from the manifest's state entries and its requires before the process starts. + for _, name := range []string{headsBucket, performedBucket, inputsBucket, groupkeys.EndAsksBucket, groupkeys.OwedAsksBucket} { + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: 1}); err != nil { t.Fatal(err) } } if prepare != nil { prepare(ctx, js) } + acct := process.Account{Node: "n1", Module: "mesh-delivery-forge-bridge"} if _, err := js.CreateOrUpdateConsumer(ctx, "EVENTS", jetstream.ConsumerConfig{ - Durable: "n1_mesh-delivery-forge-bridge", AckPolicy: jetstream.AckExplicitPolicy, DeliverPolicy: jetstream.DeliverNewPolicy, - FilterSubjects: []string{"mesh.mod.gitea.event.pull.updated", "mesh.mod.gitea.event.pull.merged", "mesh.mod.gitea.event.pull.closed", "mesh.mod.gitea.event.forge.delete"}, + Durable: acct.EventConsumerName(), AckPolicy: jetstream.AckExplicitPolicy, DeliverPolicy: jetstream.DeliverNewPolicy, + FilterSubjects: []string{git.SubjectPullUpdated, git.SubjectPullMerged, git.SubjectPullClosed, git.SubjectForgeDelete}, }); err != nil { t.Fatal(err) } + for name, subjects := range map[string]string{seat.StreamOwed: seat.SubjectOwedAll, checker.StreamOwed: checker.SubjectOwedAll} { + if _, err := js.CreateOrUpdateConsumer(ctx, name, acct.SeatEventConsumerConfig([]string{subjects}, false)); err != nil { + t.Fatal(err) + } + } - // A stand-in answers the forge's gitea_open_pulls, which the bridge asks at its start and after a reconnect, on its tool - // subject (mesh-control internal/broker/nats.go invokedSubjects) in the reply envelope of mesh-sdk go/process/call.go. + // The stand-in seats, answering on the caller-named subject and its plain fallback. var mu sync.Mutex - current := openReply - open, err := ctl.Subscribe("mesh.mod.gitea.tool.gitea_open_pulls", func(m *nats.Msg) { - mu.Lock() - defer mu.Unlock() - _ = m.Respond([]byte(current)) - }) - if err != nil { - t.Fatal(err) + answers := map[string]string{ + "git.open-pulls": openReply, "git.trees": `{"root":"r0","trees":{"":"t0","modules/a":"t1"}}`, + "mesh-delivery.groups": `{"groups":[]}`, "mesh-hand-acts.hand-act": `{"recorded":true}`, } - t.Cleanup(func() { _ = open.Unsubscribe() }) - if err := ctl.Flush(); err != nil { - t.Fatal(err) + for _, verb := range []string{"status", "comment", "view", "note", "merge-pull"} { + answers["git."+verb] = string(gitfixture.Given(t, verb+" answers", "verb:"+verb+".answer")) } - - // A stand-in answers the forge's gitea_get_file: by default a module.json of {} at every path and no other file. - var fmu sync.Mutex - filesOf := func(_, _, path, _ string) (string, bool) { return "{}", strings.HasSuffix(path, "module.json") } - getFile, err := ctl.Subscribe("mesh.mod.gitea.tool.gitea_get_file", func(m *nats.Msg) { - var a struct{ Owner, Repo, Path, Ref string } - _ = json.Unmarshal(m.Data, &a) - fmu.Lock() - f := filesOf - fmu.Unlock() - content, found := f(a.Owner, a.Repo, a.Path, a.Ref) - _ = m.Respond(getFileReply(a.Path, a.Ref, content, found)) - }) - if err != nil { - t.Fatal(err) + filesOf := func(_, path, _ string) (string, bool) { return "{}", strings.HasSuffix(path, "module.json") } + var called []string + for _, subject := range []string{"mesh.seat.*.call.*.*", "mesh.seat.*.tool.*"} { + sub, err := ctl.Subscribe(subject, func(m *nats.Msg) { + parts := strings.Split(m.Subject, ".") + verb := parts[2] + "." + parts[4] + mu.Lock() + called = append(called, verb+" "+string(m.Data)) + answer, files := answers[verb], filesOf + mu.Unlock() + if verb == "git.file" { + var in git.FileInput + _ = json.Unmarshal(m.Data, &in) + content, found := files(in.Repository, in.Path, in.Ref) + answer = fileAnswer(in.Path, in.Ref, content, found) + } + _ = m.Respond(answerOf(answer)) + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = sub.Unsubscribe() }) } - t.Cleanup(func() { _ = getFile.Unsubscribe() }) if err := ctl.Flush(); err != nil { t.Fatal(err) } @@ -293,24 +294,37 @@ func startBed(t *testing.T, openReply string, prepare func(ctx context.Context, t.Fatal(err) } env := map[string]string{"MESH_BROKER_FILE": broker} - runCtx, stop := context.WithCancel(ctx) done := make(chan error, 1) - go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() - t.Cleanup(stop) - return &bed{srv: srv, ctl: ctl, js: js, ctx: ctx, done: done, reply: func(s string) { mu.Lock(); current = s; mu.Unlock() }, - files: func(f func(owner, repo, path, ref string) (string, bool)) { fmu.Lock(); filesOf = f; fmu.Unlock() }} + var stop context.CancelFunc + start := func() { + var runCtx context.Context + runCtx, stop = context.WithCancel(ctx) + go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() + } + start() + t.Cleanup(func() { stop() }) + set := func(verb, answer string) { mu.Lock(); answers[verb] = answer; mu.Unlock() } + return &bed{srv: srv, ctl: ctl, js: js, ctx: ctx, done: done, reply: func(s string) { set("git.open-pulls", s) }, + files: func(f func(repository, path, ref string) (string, bool)) { mu.Lock(); filesOf = f; mu.Unlock() }, + answer: set, + calls: func() []string { mu.Lock(); defer mu.Unlock(); return append([]string(nil), called...) }, + restart: func(between func()) { + stop() + <-done + between() + start() + }} } // TestBridgeWritesPullMessages starts the program the way the node does (its entry, run, with MESH_BROKER_FILE). func TestBridgeWritesPullMessages(t *testing.T) { // 1235/AC2 - ctl, js, ctx, _ := startBridge(t, `{"result":{"pulls":[]}}`) + ctl, js, ctx, _ := startBridge(t, `{"pulls":[]}`) defer branchDeletedCase(t, ctl.ConnectedUrl(), js, ctx) - // gitea.pull.updated as the forge module publishes it: the body of trunk.ts:233-250 on mesh.mod.gitea.event.pull.updated. - ev, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-delivery", "number": 12, "title": "t", "body": "b", "base": "main", - "head": "feat/x", "head_sha": head, "clone_url": "u", "html_url": "h", "paths": []string{"modules/a/x.go"}, "paths_truncated": false, - "removed": []string{}, "module_dirs": []string{"modules/a"}, "module_dirs_said": true}) - if _, err := js.Publish(ctx, "mesh.mod.gitea.event.pull.updated", ev); err != nil { + // pull.updated as the seat git says it (its contract's PullUpdatedEvent). + ev, _ := json.Marshal(git.PullUpdatedEvent{Repository: "novox/mesh-delivery", Number: 12, Title: "t", Base: "main", Branch: "feat/x", Head: head, + Change: &git.Change{Paths: []string{"modules/a/x.go"}, ModuleDirs: []string{"modules/a"}}}) + if _, err := js.Publish(ctx, git.SubjectPullUpdated, ev); err != nil { t.Fatal(err) } @@ -357,13 +371,13 @@ func TestBridgeWritesPullMessages(t *testing.T) { // 1235/AC2 t.Errorf("the event is not acknowledged: %+v", ci) } -// branchDeletedCase is DLV-FB1's case of the bridge's whole-module test (design 51): the forge module's forge.delete of -// a branch is a branch-deleted message on FORGE, and a forge.delete from another account than the forge module's is -// refused by the bus. +// branchDeletedCase is DLV-FB1's case of the bridge's whole-module test (design 51): the seat git's forge.delete of a +// branch is a branch-deleted message on FORGE, and a forge.delete from another account than the seat's holder is refused +// by the bus. func branchDeletedCase(t *testing.T, srv string, js jetstream.JetStream, ctx context.Context) { t.Helper() - ev, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-sdk", "ref": "feat/x", "ref_type": "branch", "sha": head}) - if _, err := js.Publish(ctx, "mesh.mod.gitea.event.forge.delete", ev); err != nil { + ev, _ := json.Marshal(git.ForgeDeleteEvent{Repository: "novox/mesh-sdk", Ref: "feat/x", RefType: "branch"}) + if _, err := js.Publish(ctx, git.SubjectForgeDelete, ev); err != nil { t.Fatal(err) } forge, err := js.Stream(ctx, "FORGE") @@ -388,21 +402,26 @@ func branchDeletedCase(t *testing.T, srv string, js jetstream.JetStream, ctx con ojs, _ := jetstream.New(other) octx, cancel := context.WithTimeout(ctx, 2*time.Second) defer cancel() - if _, err := ojs.Publish(octx, "mesh.mod.gitea.event.forge.delete", ev); err == nil { - t.Error("a forge.delete from the bridge's account was taken by the bus: only the forge module's account may say it") + if _, err := ojs.Publish(octx, git.SubjectForgeDelete, ev); err == nil { + t.Error("a forge.delete from the bridge's account was taken by the bus: only the seat git's holder may say it") } } const merge = "89abcdef0123456789abcdef0123456789abcdef" -// TestBridgeWritesMergedMessage has a stand-in answer the forge's gitea_commit_trees on its tool subject -// (mesh-control internal/broker/nats.go invokedSubjects: mesh.mod..tool.), in the reply envelope of -// mesh-sdk go/process/call.go ({result, error, node}). +// openPull is one entry of the seat git's open-pulls answer. +func openPull(repo string, number int, branch, head string) string { + b, _ := json.Marshal(git.OpenPull{Pull: git.Pull{Repository: "novox/" + repo, Number: int64(number), Base: "main", Branch: branch, Head: head, State: "open"}, + Checks: map[string]git.Status{}, Required: []string{}}) + return string(b) +} + +// TestBridgeWritesMergedMessage: the stand-in seat git answers trees and file on the caller-named subjects. func TestBridgeWritesMergedMessage(t *testing.T) { // 1236/AC4, 1448/AC5 - bed := startBed(t, `{"result":{"pulls":[]}}`, nil) - ctl, js, ctx := bed.ctl, bed.js, bed.ctx + bed := startBed(t, `{"pulls":[]}`, nil) + js, ctx := bed.js, bed.ctx // The stand-in forge's files are two real files of this repository (this module's module.json and go.sum, read from the - // working tree), in gitea_get_file's real answer shape: the root has a manifest only, modules/a a manifest and go.sum. + // working tree), in the seat git's file answer: the root has a manifest only, modules/a a manifest and go.sum. manifest, err := os.ReadFile("../../module.json") if err != nil { t.Fatal(err) @@ -411,7 +430,7 @@ func TestBridgeWritesMergedMessage(t *testing.T) { // 1236/AC4, 1448/AC5 if err != nil { t.Fatal(err) } - bed.files(func(_, _, path, ref string) (string, bool) { + bed.files(func(_, path, ref string) (string, bool) { if ref != merge { t.Errorf("the forge was asked for %s at %s, want the merge commit %s", path, ref, merge) } @@ -423,25 +442,10 @@ func TestBridgeWritesMergedMessage(t *testing.T) { // 1236/AC4, 1448/AC5 } return "", false }) - var asked chan []byte = make(chan []byte, 4) - sub, err := ctl.Subscribe("mesh.mod.gitea.tool.gitea_commit_trees", func(m *nats.Msg) { - asked <- m.Data - _ = m.Respond([]byte(`{"result":{"root":"r0","trees":{"":"t0","modules/a":"t1"}},"node":"n0"}`)) - }) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = sub.Unsubscribe() }) - if err := ctl.Flush(); err != nil { - t.Fatal(err) - } - - // gitea.pull.merged as mesh-catalog modules/gitea/index.ts:186-207 publishes it. - ev, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-delivery", "number": 12, "title": "t", "body": "b", - "head_sha": head, "head": "feat/x", "base": "main", "merge_commit_sha": merge, "merged_at": "2026-10-19T12:00:00Z", - "clone_url": "u", "html_url": "h", "paths": []string{"modules/a/x.go"}, "paths_truncated": false, "removed": []string{}, - "module_dirs": []string{"modules/a"}, "module_dirs_said": true}) - if _, err := js.Publish(ctx, "mesh.mod.gitea.event.pull.merged", ev); err != nil { + // pull.merged as the seat git says it (its contract's PullMergedEvent). + ev, _ := json.Marshal(git.PullMergedEvent{Repository: "novox/mesh-delivery", Number: 12, Title: "t", Head: head, Branch: "feat/x", Base: "main", + MergeCommit: merge, Change: &git.Change{Paths: []string{"modules/a/x.go"}, ModuleDirs: []string{"modules/a"}}}) + if _, err := js.Publish(ctx, git.SubjectPullMerged, ev); err != nil { t.Fatal(err) } @@ -475,16 +479,15 @@ func TestBridgeWritesMergedMessage(t *testing.T) { // 1236/AC4, 1448/AC5 if !forgemsg.SameForge(f, want) { t.Errorf("message = %+v, want %+v", f, want) } - select { - case body := <-asked: - if string(body) != `{"owner":"novox","repo":"mesh-delivery","sha":"`+merge+`"}` { - t.Errorf("the tool was asked %s", body) + var trees []string + for _, c := range bed.calls() { + if strings.HasPrefix(c, "git.trees ") { + trees = append(trees, c) } - default: - t.Error("the stand-in was not asked") } - if len(asked) != 0 { - t.Error("the tool was asked more than once") + in, _ := json.Marshal(git.TreesInput{Repository: "novox/mesh-delivery", Commit: merge}) + if len(trees) != 1 || trees[0] != "git.trees "+string(in) { + t.Errorf("the seat git was asked for trees %v, want once at the merge commit", trees) } cons, err := js.Consumer(ctx, "EVENTS", "n1_mesh-delivery-forge-bridge") if err != nil { @@ -503,10 +506,10 @@ func TestBridgeWritesMergedMessage(t *testing.T) { // 1236/AC4, 1448/AC5 // TestBridgeSaysTheOpenPullsAtStart: the forge holds two open pull requests when the bridge starts; one message each is on // FORGE, and the open message naming both after them (ADR 0443 §5). func TestBridgeSaysTheOpenPullsAtStart(t *testing.T) { // 1237/AC1 - // The entries are gitea_open_pulls's (mesh-catalog modules/gitea/client.ts:369-381, head the branch). - _, js, ctx, _ := startBridge(t, `{"result":{"pulls":[ - {"owner":"novox","repo":"mesh-delivery","number":12,"base":"main","head":"feat/x","head_sha":"`+head+`","html_url":"h","required":[],"statuses":[],"checks":{},"labels":[]}, - {"owner":"novox","repo":"mesh-catalog","number":300,"base":"main","head":"feat/x","head_sha":"`+strings.Repeat("a", 40)+`","html_url":"h","required":[],"statuses":[],"checks":{},"labels":[]}]},"node":"n0"}`) + // The entries are the seat git's open-pulls answer (its contract's OpenPull). + _, js, ctx, _ := startBridge(t, `{"pulls":[ + `+openPull("mesh-delivery", 12, "feat/x", head)+`, + `+openPull("mesh-catalog", 300, "feat/x", strings.Repeat("a", 40))+`]}`) forge, err := js.Stream(ctx, "FORGE") if err != nil { t.Fatal(err) @@ -533,10 +536,10 @@ func TestBridgeSaysTheOpenPullsAtStart(t *testing.T) { // 1237/AC1 // TestBridgeExitsWhenTheOpenPullsCannotBeRead: the process manager starts it again. func TestBridgeExitsWhenTheOpenPullsCannotBeRead(t *testing.T) { // 1237/AC3 - _, _, _, done := startBridge(t, `{"error":"the forge said 502"}`) + _, _, _, done := startBridge(t, `{"error":"the forge said 502","error-kind":"failed","pulls":[]}`) select { case err := <-done: - if err == nil || !strings.Contains(err.Error(), "gitea.gitea_open_pulls") || !strings.Contains(err.Error(), "the forge said 502") { + if err == nil || !strings.Contains(err.Error(), "open-pulls") || !strings.Contains(err.Error(), "the forge said 502") { t.Errorf("run returned %v", err) } case <-time.After(10 * time.Second): @@ -555,14 +558,12 @@ func TestBridgeReadsTheForgeAtStartAndAfterAReconnectAndHearsNoOlderEvent(t *tes return "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox." + repo + ".pull" } openSubject := "mesh.stream.mesh-delivery-forge-bridge.event.forge.open" - b := startBed(t, `{"result":{"pulls":[ - {"owner":"novox","repo":"mesh-delivery","number":12,"base":"main","head":"feat/x","head_sha":"`+head+`","html_url":"h","required":[],"statuses":[],"checks":{},"labels":[]}, - {"owner":"novox","repo":"mesh-catalog","number":300,"base":"main","head":"feat/x","head_sha":"`+h2+`","html_url":"h","required":[],"statuses":[],"checks":{},"labels":[]}]},"node":"n0"}`, + b := startBed(t, `{"pulls":[`+openPull("mesh-delivery", 12, "feat/x", head)+`,`+openPull("mesh-catalog", 300, "feat/x", h2)+`]}`, func(ctx context.Context, js jetstream.JetStream) { // Older events: a pull request long closed and one long updated, from before the bridge's consumer. - for subject, ev := range map[string]map[string]any{ - "mesh.mod.gitea.event.pull.updated": {"owner": "novox", "repo": "mesh-old", "number": 5, "head": "feat/old", "head_sha": h2}, - "mesh.mod.gitea.event.pull.closed": {"owner": "novox", "repo": "mesh-older", "number": 6, "head": "feat/older", "head_sha": h2}, + for subject, ev := range map[string]any{ + git.SubjectPullUpdated: git.PullUpdatedEvent{Repository: "novox/mesh-old", Number: 5, Base: "main", Branch: "feat/old", Head: h2}, + git.SubjectPullClosed: git.PullClosedEvent{Repository: "novox/mesh-older", Number: 6, Base: "main", Branch: "feat/older", Head: h2}, } { raw, _ := json.Marshal(ev) if _, err := js.Publish(ctx, subject, raw); err != nil { @@ -651,171 +652,186 @@ func TestBridgeReadsTheForgeAtStartAndAfterAReconnectAndHearsNoOlderEvent(t *tes } } -// TestBridgeMergesAGroupOneMemberAtATime: the gate (played by the test) writes merges.g with member A handed and the -// Approve's heads of A and B; a stand-in forge merges A and announces it; the gate then hands B; both end merged on FORGE -// (1342/AC3, ADR 0437 §8's check). -func TestBridgeMergesAGroupOneMemberAtATime(t *testing.T) { // 1342/AC3 - b := startBed(t, `{"result":{"pulls":[]}}`, nil) - const headB = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" - const mergeA, mergeB = "1111111111111111111111111111111111111111", "2222222222222222222222222222222222222222" - merges := make(chan string, 4) - // The stand-in for gitea_merge_pull_request answers in the reply envelope of mesh-sdk go/process/call.go and publishes - // gitea.pull.merged as mesh-catalog modules/gitea/index.ts:186-207 does, with the merge commit of the member merged. - sub, err := b.ctl.Subscribe("mesh.mod.gitea.tool.gitea_merge_pull_request", func(m *nats.Msg) { - var a struct { - Repo string `json:"repo"` - Head string `json:"head"` - } - _ = json.Unmarshal(m.Data, &a) - merges <- string(m.Data) - commit, number := mergeA, 12 - if a.Repo == "mesh-sdk" { - commit, number = mergeB, 4 - } - ev, _ := json.Marshal(map[string]any{"owner": "novox", "repo": a.Repo, "number": number, "head_sha": a.Head, "head": "feat/x", - "base": "main", "merge_commit_sha": commit, "module_dirs": []string{"modules/a"}, "module_dirs_said": true}) - _, _ = b.js.Publish(b.ctx, "mesh.mod.gitea.event.pull.merged", ev) - _ = m.Respond([]byte(`{"result":{"merged":true},"node":"n0"}`)) - }) +// heard is every message published under the bridge's stream subjects from now, by subject. +func heard(t *testing.T, b *bed) <-chan *nats.Msg { + t.Helper() + ch := make(chan *nats.Msg, 64) + sub, err := b.ctl.ChanSubscribe("mesh.stream.mesh-delivery-forge-bridge.event.>", ch) if err != nil { t.Fatal(err) } t.Cleanup(func() { _ = sub.Unsubscribe() }) - trees, err := b.ctl.Subscribe("mesh.mod.gitea.tool.gitea_commit_trees", func(m *nats.Msg) { - _ = m.Respond([]byte(`{"result":{"root":"r0","trees":{"":"t0","modules/a":"t1"}},"node":"n0"}`)) - }) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = trees.Unsubscribe() }) if err := b.ctl.Flush(); err != nil { t.Fatal(err) } + return ch +} - kv, err := b.js.KeyValue(b.ctx, mergesBucket) - if err != nil { - t.Fatal(err) - } - approved := map[string]string{meshdeliveryclient.ApprovedKey("novox/mesh-delivery", 12): head, meshdeliveryclient.ApprovedKey("novox/mesh-sdk", 4): headB} - pullA := seat.MergesByGroupValuePullsItem{Repository: "novox/mesh-delivery", Number: 12, Head: head, State: string(meshdeliveryclient.MergeMerging)} - pullB := seat.MergesByGroupValuePullsItem{Repository: "novox/mesh-sdk", Number: 4, Head: headB, State: string(meshdeliveryclient.MergeMerging)} - put := func(pulls ...seat.MergesByGroupValuePullsItem) { - v, err := meshdeliveryclient.EncodeMerge(seat.MergesByGroupValue{Group: "g", Ask: "merge.g.0123456789ab", Approved: approved, Pulls: pulls}) - if err != nil { - t.Fatal(err) - } - if _, err := kv.Put(b.ctx, "g", v); err != nil { - t.Fatal(err) - } - } - next := func(repo string) { +// next is the next message heard on subject within a bound; the others are passed by. +func next(t *testing.T, ch <-chan *nats.Msg, subject string) []byte { + t.Helper() + bound := time.After(20 * time.Second) + for { select { - case got := <-merges: - if !strings.Contains(got, `"repo":"`+repo+`"`) { - t.Errorf("the forge was asked %s, want %s", got, repo) + case m := <-ch: + if m.Subject == subject { + return m.Data } - for _, want := range []string{"novox/mesh-delivery@" + head, "novox/mesh-sdk@" + headB} { - if !strings.Contains(got, want) { - t.Errorf("the merge %s does not carry the approved head %s", got, want) - } - } - case <-time.After(10 * time.Second): - t.Fatalf("%s was not merged within 10 s", repo) + case <-bound: + t.Fatalf("nothing on %s within 20 s", subject) + return nil } } - put(pullA) - next("mesh-delivery") - // The gate hands the next member only after the bridge reported the one before merged (ADR 0441 §5). - for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && len(forgeMerged(t, b)) < 1; time.Sleep(50 * time.Millisecond) { - } - pullA.State, pullA.Commit = string(meshdeliveryclient.MergeMerged), mergeA - put(pullA, pullB) - next("mesh-sdk") - var got []string - for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { - if got = forgeMerged(t, b); len(got) == 2 { - break - } - } - if len(got) != 2 { - t.Fatalf("FORGE holds merged messages for %v, want both members", got) - } - select { - case extra := <-merges: - t.Errorf("a third merge was asked: %s", extra) - default: - } } -// forgeMerged lists the repositories FORGE holds a merged message for, sorted. -func forgeMerged(t *testing.T, b *bed) []string { +// refusedRead says the bridge's account cannot read the bucket: a key-value read of another module's state is refused. +func refusedRead(t *testing.T, b *bed, bucket string) { t.Helper() - forge, err := b.js.Stream(b.ctx, "FORGE") + if _, err := b.js.CreateKeyValue(b.ctx, jetstream.KeyValueConfig{Bucket: bucket}); err != nil { + t.Fatal(err) + } + nc, err := nats.Connect(b.srv.ClientURL(), nats.UserInfo("n1.mesh-delivery-forge-bridge", "pw")) if err != nil { t.Fatal(err) } - var out []string - for _, repo := range []string{"mesh-delivery", "mesh-sdk"} { - raw, err := forge.GetLastMsgForSubject(b.ctx, "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox."+repo+".merged") - if err != nil { - continue - } - if f, err := forgemsg.DecodeForge(raw.Data); err == nil && f.Kind == forgemsg.ForgeMerged { - out = append(out, f.Repository) + defer nc.Close() + js, _ := jetstream.New(nc) + ctx, cancel := context.WithTimeout(b.ctx, 2*time.Second) + defer cancel() + if kv, err := js.KeyValue(ctx, bucket); err == nil { + if _, err := kv.Get(ctx, "any"); err == nil || errors.Is(err, jetstream.ErrKeyNotFound) { + t.Errorf("the bridge's account reads %s", bucket) } } - return out +} + +// Acceptance 49: the forge bridge sets the group on each member's merged message and says the group's end once, after the +// last member's, from the seat mesh-delivery's verb groups; its account reads no state of the gate's. +func TestBridgeSaysAGroupFromTheSeatsGroups(t *testing.T) { // 1585, ADR 0633 §4 + b := startBed(t, `{"pulls":[]}`, nil) + headB := strings.Repeat("b", 40) + mergeB := strings.Repeat("e", 40) + groups, _ := json.Marshal(seat.GroupsAnswer{Groups: []seat.Group{{ID: "feat/x", State: "merging", Members: []seat.Member{ + {Repository: "novox/mesh-delivery", Number: 12, Head: head, State: "merged"}, {Repository: "novox/mesh-sdk", Number: 4, Head: headB, State: "merging"}}}}}) + b.answer("mesh-delivery.groups", string(groups)) + ch := heard(t, b) + for _, m := range []git.PullMergedEvent{ + {Repository: "novox/mesh-delivery", Number: 12, Head: head, Branch: "feat/x", Base: "main", MergeCommit: merge, Change: &git.Change{Paths: []string{"modules/a/x.go"}, ModuleDirs: []string{"modules/a"}}}, + {Repository: "novox/mesh-sdk", Number: 4, Head: headB, Branch: "feat/x", Base: "main", MergeCommit: mergeB, Change: &git.Change{Paths: []string{"go/x.go"}}}, + } { + ev, _ := json.Marshal(m) + if _, err := b.js.Publish(b.ctx, git.SubjectPullMerged, ev); err != nil { + t.Fatal(err) + } + repo := strings.TrimPrefix(m.Repository, "novox/") + f, err := forgemsg.DecodeForge(next(t, ch, "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox."+repo+".merged")) + if err != nil || f.Group != "feat/x" || len(f.Members) != 2 || f.Members[1].Number != 4 { + t.Fatalf("%s's merged message: %+v %v, want the group feat/x with both members", repo, f, err) + } + } + end, err := forgemsg.DecodeForgeGroupEnd(next(t, ch, forgeSubject(forgemsg.ForgeGroupEndEvent))) + if err != nil || end.Group != "feat/x" || len(end.Members) != 2 || end.Members[0].Merge != merge || end.Members[1].Merge != mergeB { + t.Errorf("the group's end: %+v %v", end, err) + } + refusedRead(t, b, "mesh-delivery-gate_merges") +} + +// Acceptance 53 (design 47 *What the gate owes the forge*): an owed act said on a seat's own stream owed is performed through +// the seat git and its outcome said as owed.ended; said twice it is performed once; one it cannot read is said undecodable; +// one said while the bridge is stopped is performed once after its restart; bridge_owed_again's ask is answered by the +// process; and its account reads neither the gate's nor the checker's state. +func TestBridgePerformsWhatIsOwedOnce(t *testing.T) { + b := startBed(t, `{"pulls":[]}`, nil) + ch := heard(t, b) + ended := func() forgemsg.OwedEnded { + t.Helper() + e, err := forgemsg.DecodeOwedEnded(next(t, ch, forgeSubject(forgemsg.OwedEndedEvent))) + if err != nil { + t.Fatal(err) + } + return e + } + count := func(verb string) int { + n := 0 + for _, c := range b.calls() { + if strings.HasPrefix(c, verb+" ") { + n++ + } + } + return n + } + gateOwed := gitfixture.Seat(t, seat.Seat, "events", "owed is said, a merge", "event:owed")[0] + subject := seat.SubjectOwed("novox~mesh-sdk", "178") + for range 2 { + if _, err := b.js.Publish(b.ctx, subject, gateOwed); err != nil { + t.Fatal(err) + } + } + if e := ended(); e.Outcome != forgemsg.OwedPerformed || e.Of != seat.Seat { + t.Errorf("the gate's merge: %+v, want performed", e) + } + if _, err := b.js.Publish(b.ctx, subject, []byte("{")); err != nil { + t.Fatal(err) + } + if e := ended(); e.Outcome != forgemsg.OwedUndecodable { + t.Errorf("an act it cannot read: %+v, want undecodable", e) + } + if n := count("git.merge-pull"); n != 1 { + t.Errorf("an act said twice was performed %d times", n) + } + // Said while the bridge is stopped: performed once after its restart. + checkerOwed := gitfixture.Seat(t, checker.Seat, "events", "owed is said", "event:owed")[0] + b.restart(func() { + if _, err := b.js.Publish(b.ctx, checker.SubjectOwed("novox~mesh-sdk", "178"), checkerOwed); err != nil { + t.Fatal(err) + } + }) + if e := ended(); e.Of != checker.Seat || e.Outcome != forgemsg.OwedPerformed { + t.Errorf("the checker's status after the restart: %+v, want performed", e) + } + if n := count("git.status"); n != 1 { + t.Errorf("the checker's status was performed %d times, want once", n) + } + // bridge_owed_again asks through the state owed-asks; the process performs the act again and answers there. + var g struct { + ID string `json:"id"` + } + _ = json.Unmarshal(gateOwed, &g) + key, value, err := groupkeys.NewOwedAsk(groupkeys.OwedAgain, seat.Seat, g.ID, "a test", "lead@n1", time.Now()) + if err != nil { + t.Fatal(err) + } + asks, err := b.js.KeyValue(b.ctx, groupkeys.OwedAsksBucket) + if err != nil { + t.Fatal(err) + } + w, err := asks.Watch(b.ctx, key) + if err != nil { + t.Fatal(err) + } + defer w.Stop() + if _, err := asks.Put(b.ctx, key, value); err != nil { + t.Fatal(err) + } + bound := time.After(20 * time.Second) + for answered := false; !answered; { + select { + case e := <-w.Updates(): + if e == nil { + continue + } + if a, err := groupkeys.DecodeOwedAsk(e.Value()); err == nil && a.Done() { + answered = true + if a.Outcome != forgemsg.OwedPerformed || count("git.merge-pull") != 2 { + t.Errorf("bridge_owed_again: %+v, merge-pull asked %d times; want performed again", a, count("git.merge-pull")) + } + } + case <-bound: + t.Fatal("bridge_owed_again was not answered within 20 s") + } + } + refusedRead(t, b, "mesh-delivery-gate_owed") + refusedRead(t, b, "mesh-delivery-checker_owed") } func sha256Sum(b []byte) []byte { s := sha256.Sum256(b); return s[:] } - -// TestBridgePerformsNoMeshCheckAct: the checker owes mesh/check on a head and says so on its stream; the bridge calls no -// tool with mesh/check and publishes nothing of its own on EVENTS for it. The forge module writes mesh/check from the -// checker's owed state itself (ADR 0521; R4e round 2 finding R2-1). -func TestBridgePerformsNoMeshCheckAct(t *testing.T) { // R2-1, ADR 0521 - b := startBed(t, `{"result":{"pulls":[]}}`, nil) - calls := make(chan string, 8) - // Every forge call is watched, and none answered here: the bed's stand-ins answer the ones the bridge asks. - sub, err := b.ctl.Subscribe("mesh.mod.gitea.>", func(m *nats.Msg) { - if strings.Contains(string(m.Data), "mesh/check") { - calls <- m.Subject + " " + string(m.Data) - } - }) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = sub.Unsubscribe() }) - if err := b.ctl.Flush(); err != nil { - t.Fatal(err) - } - kv, err := b.js.CreateKeyValue(b.ctx, jetstream.KeyValueConfig{Bucket: "mesh-delivery-checker_owed", History: 5}) - if err != nil { - t.Fatal(err) - } - key, err := meshdeliveryclient.OwedKey("novox/mesh-delivery", 12, head, "status=mesh/check") - if err != nil { - t.Fatal(err) - } - v, err := meshdeliveryclient.EncodeOwed(seat.OwedByRepositoryNumberHeadActValue{Act: meshdeliveryclient.OwedStatus, Context: meshdeliveryclient.StatusContextCheck, State: "success", Text: "pass", Target: "https://view.example/checks/g.0123456789ab"}) - if err != nil { - t.Fatal(err) - } - if _, err := kv.Put(b.ctx, key, v); err != nil { - t.Fatal(err) - } - if _, err := b.js.Publish(b.ctx, "mesh.stream.mesh-delivery-checker.event.owed.written", []byte(`{"key":"`+key+`"}`)); err != nil { - t.Fatal(err) - } - select { - case c := <-calls: - t.Errorf("a forge tool was called with mesh/check: %s", c) - case <-time.After(2 * time.Second): - } - events, err := b.js.Stream(b.ctx, "EVENTS") - if err != nil { - t.Fatal(err) - } - if _, err := events.GetLastMsgForSubject(b.ctx, "mesh.mod.mesh-delivery-forge-bridge.event.>"); err == nil { - t.Error("the bridge published an event of its own for the checker's mesh/check") - } -} diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go index 19ad49af..f91af302 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go @@ -22,6 +22,8 @@ import ( "github.com/nats-io/nats.go/jetstream" git "git.novox.be/novox/mesh-delivery/contracts/go/seats/git" + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" + checker "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdeliverychecker" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "mesh-delivery-forge-bridge/internal/bridge" @@ -146,39 +148,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // Everything written on FORGE, an event's and a sync's, keeps the head in the bucket. rec := &recorder{pub: js, heads: heads, logger: logger} said := func(ctx context.Context) error { return syncOpen(ctx, call, rec, heads, &turn, logger) } - // What the gate asks of the forge (ADR 0437 §8): its owed acts and its merges, each performed once per revision. - performedKV, err := process.EssentialBucket(ctx, js, "bridge", performedBucket, "perform an act of the gate only once") + performedKV, err := process.EssentialBucket(ctx, js, "bridge", performedBucket, "keep what it performed of what is owed to the forge") if err != nil { return err } - perf := &performer{call: call, pub: rec, done: kvRevisions{performedKV}, logger: logger} - // A delivery group's member names its group, and its end is said once (ADR 0633 §4), from the gate's merges. - groups := &grouper{send: sendVia(js), done: kvRevisions{performedKV}, logger: logger} - watching := make(chan error, 3) - for _, w := range []struct { - bucket string - react func(ctx context.Context, key string, rev uint64, value []byte) - }{{owedBucket, perf.owed}, {mergesBucket, func(ctx context.Context, key string, rev uint64, value []byte) { - perf.merges(ctx, key, rev, value) - groups.ended(ctx, key, rev, value) - }}} { - // The gate's buckets come with the gate and its grant; the bridge does not wait for them (no timer), and is started - // again by its delivery when the gate arrives. A bucket refused or not answered is the same (issue 1945). - kv := process.OptionalBucket(ctx, js, w.bucket, "nothing of it is performed until the bridge starts again", logger.Printf) - if kv == nil { - continue - } - if w.bucket == mergesBucket { - groups.merges = kvMerges(kv) - } - go func() { - watching <- watch(ctx, kv, func(key string, rev uint64, value []byte) { - hctx, cancel := context.WithTimeout(ctx, syncDeadline) - defer cancel() - w.react(hctx, key, rev, value) - }) - }() - } + // What the gate and the checker owe the forge, performed through the seat git (design 47 *What the gate owes the forge*). + owe := &owing{call: call, pub: js, performed: kvPerformed{performedKV}, now: time.Now, logger: logger} + // A delivery group's member names its group, and its end is said once (ADR 0633 §4), from the seat mesh-delivery's groups. + groups := &grouper{groups: groupsVia(call), send: sendVia(js), done: kvHeads{performedKV}, logger: logger} // A person says a group's end again with bridge_group_end_again, which writes the state end-asks (ADR 0633 §4): watched, and // answered there; a state not there at this start is answered from the next start. if kv := process.OptionalBucket(ctx, js, groupkeys.EndAsksBucket, "bridge_group_end_again is not answered until the bridge starts again", logger.Printf); kv != nil { @@ -192,6 +169,43 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err := said(ctx); err != nil { return err } + // What the gate and the checker say they owe, each on its seat's own stream owed through this module's durable there + // (made from its consumes). A stream not there yet is said, and the bridge goes on: its other work does not wait on the + // gate, and what was said meanwhile is caught up from the groups at the next start. + for _, owner := range []struct{ of, kept string }{{seat.Seat, seat.StreamOwed}, {checker.Seat, checker.StreamOwed}} { + cc, err := cred.HearSeatEvents(ctx, js, owner.kept, func(m jetstream.Msg) { + hctx, cancel := context.WithTimeout(ctx, syncDeadline) + defer cancel() + owe.hear(hctx, owner.of, m) + }, logger.Printf) + if err != nil { + logger.Printf("nothing %s owes the forge is performed until the bridge starts again: %v", owner.of, err) + continue + } + defer cc.Stop() + } + // A person performs an act again, drops one or reads the consumers' backlog with the tools bridge_owed_again, + // bridge_owed_drop and bridge_owed, which write the state owed-asks: watched, and answered there. + if kv := process.OptionalBucket(ctx, js, groupkeys.OwedAsksBucket, "bridge_owed_again, bridge_owed_drop and bridge_owed's backlog are not answered until the bridge starts again", logger.Printf); kv != nil { + backlog := owedBacklog(js, cred, time.Now) + go func() { + err := watch(ctx, kv, func(key string, rev uint64, value []byte) { + hctx, cancel := context.WithTimeout(ctx, syncDeadline) + defer cancel() + owe.asked(hctx, kv, key, rev, value, backlog) + }) + if err != nil && ctx.Err() == nil { + logger.Printf("the owed asks are no longer watched, so they are not answered until the bridge starts again: %v", err) + } + }() + } + // What was said while the bridge was away and is still unanswered, read from the seat (design 47 *The bridge catches up + // from the seat*). The gate comes after the bridge, so a seat that does not answer yet is said, and the bridge goes on. + if n, err := owe.catchUp(ctx); err != nil { + logger.Printf("nothing owed is caught up from the groups at this start, so an act said while the bridge was away waits for its saying again: %v", err) + } else if n > 0 { + logger.Printf("%d acts owed to the forge, said while the bridge was away, performed from the groups", n) + } go onReconnect(ctx, again, said, logger) // What a merge read of each module directory, by its tree (issue 1126): a merge reads only the directories it changed. inputs := newKept(process.OptionalBucket(ctx, js, inputsBucket, "a merge's module directories are kept in memory only, so the first merge after a start reads every one", logger.Printf), logger) @@ -213,10 +227,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return fmt.Errorf("%w: the bridge hears no forge event without it, so it stops; where the bus refused it or did not answer, this module's consumes (made by the controller with its grant) have not reached the bus yet", err) } defer hub.Stop() - select { - case <-ctx.Done(): - return nil - case err := <-watching: - return fmt.Errorf("watching the gate's keys: %w", err) - } + <-ctx.Done() + return nil } diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/names.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/names.go index 4e59a705..0cc0db7d 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/names.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/names.go @@ -2,15 +2,9 @@ package main import groupkeys "mesh-delivery-forge-bridge/internal/groups" -// The holders and buckets the forge bridge names, taken from its own manifest (module.json: its name, its `reads`, its +// The holders and buckets the forge bridge names, taken from its own manifest (module.json: its name and its // states); a unit row checks them against it (TestTheNamesAreTheManifests). No contract names a holder // (novox/hq ADR 0625 §4, ADR 0634 §1). const ( forgeBridgeModule = groupkeys.Module - gateModule = "mesh-delivery-gate" - - // mergesBucket and owedBucket are the gate's states merges and owed, read by the bridge (reads: mesh-delivery-gate.merges, - // mesh-delivery-gate.owed). - mergesBucket = gateModule + "_merges" - owedBucket = gateModule + "_owed" ) diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/names_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/names_test.go index 95746226..f2e9e83f 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/names_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/names_test.go @@ -7,8 +7,8 @@ import ( "testing" ) -// theNamesAreTheManifests (a subtest of TestAnOwedStatusIsSetOnceForARevision): the holders and buckets the bridge names -// are its manifest's (its module, its `reads`, its states), since no contract names a holder (novox/hq ADR 0634 §1). +// theNamesAreTheManifests (a subtest of TestWhatIsOwedIsPerformedSaidAndKept): the holders and buckets the bridge names +// are its manifest's (its module, its `reads`, its states and its kept data in `requires`), since no contract names a holder (novox/hq ADR 0634 §1). func theNamesAreTheManifests(t *testing.T) { raw, err := os.ReadFile("../../module.json") if err != nil { @@ -20,6 +20,7 @@ func theNamesAreTheManifests(t *testing.T) { State []struct { Name string `json:"name"` } `json:"state"` + Requires []json.RawMessage `json:"requires"` } if err := json.Unmarshal(raw, &m); err != nil { t.Fatal(err) @@ -27,15 +28,22 @@ func theNamesAreTheManifests(t *testing.T) { if m.Module != forgeBridgeModule { t.Errorf("the manifest's module is %q, the bridge names itself %q", m.Module, forgeBridgeModule) } - for _, read := range []string{gateModule + ".merges", gateModule + ".owed"} { - if !slices.Contains(m.Reads, read) { - t.Errorf("the bridge reads %s, its manifest's reads are %v", read, m.Reads) - } + if len(m.Reads) != 0 { + t.Errorf("the bridge reads no other module's state (acceptance 49), its manifest's reads are %v", m.Reads) } var own []string for _, s := range m.State { own = append(own, forgeBridgeModule+"_"+s.Name) } + for _, r := range m.Requires { + var kept struct { + Name string `json:"name"` + Template string `json:"template"` + } + if json.Unmarshal(r, &kept) == nil && kept.Template == "state" { + own = append(own, forgeBridgeModule+"_"+kept.Name) + } + } for _, b := range []string{headsBucket, performedBucket, inputsBucket} { if !slices.Contains(own, b) { t.Errorf("the bridge keeps %s, its manifest's states are %v", b, own) diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/owed.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/owed.go new file mode 100644 index 00000000..7066e2bc --- /dev/null +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/owed.go @@ -0,0 +1,342 @@ +package main + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "log" + "strconv" + "strings" + "time" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" + git "git.novox.be/novox/mesh-delivery/contracts/go/seats/git" + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" + checker "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdeliverychecker" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" + + "git.novox.be/novox/mesh-sdk/go/process" + + "mesh-delivery-forge-bridge/internal/bridge" + "mesh-delivery-forge-bridge/internal/groups" +) + +// performedBucket is the module's state performed (manifest `requires`, the template state): one key per owed act it +// performed, the act's outcome and when, with the owed event as heard (novox/hq design 47 *What the gate owes the +// forge*). It is a saving and a read, never what keeps an act from happening twice: every act is safe to repeat at the +// forge. The group ends' bookkeeping (`published.`, `groups.`, groups.go) is in it too, under the same 7 days. +const performedBucket = forgeBridgeModule + "_performed" + +// performedKey is the key of one act in performed: the seat that said it and its id, the id in unpadded base64url, since +// an id may hold what a key may not (`#`, `/`). +func performedKey(of, id string) string { + return of + "." + base64.RawURLEncoding.EncodeToString([]byte(id)) +} + +// Performed is one value of the state performed: the outcome said as owed.ended, and the owed event it answers, so +// bridge_owed_again performs it again from here. +type Performed struct { + Ended forgemsg.OwedEnded `json:"ended"` + Owed json.RawMessage `json:"owed,omitempty"` +} + +// performedStore is the state performed as the owing reads and writes it. +type performedStore interface { + Get(ctx context.Context, key string) (value []byte, found bool, err error) + Put(ctx context.Context, key string, value []byte) error +} + +type kvPerformed struct{ kv jetstream.KeyValue } + +func (s kvPerformed) Get(ctx context.Context, key string) ([]byte, bool, error) { + e, err := s.kv.Get(ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return nil, false, nil + } + if err != nil { + return nil, false, err + } + return e.Value(), true, nil +} + +func (s kvPerformed) Put(ctx context.Context, key string, value []byte) error { + _, err := s.kv.Put(ctx, key, value) + return err +} + +// owing performs what the gate and the checker owe the forge (design 47 *What the gate owes the forge*): each owed act +// through the seat git's verb under its id, its outcome said as owed.ended on the stream forge and then kept in +// performed. It holds no timer. +type owing struct { + call caller + pub publisher + performed performedStore + now func() time.Time + logger *log.Logger +} + +// readerOf is the reading of the seat of's owed event into the seat git's verb. +func readerOf(of string) (func([]byte) (bridge.OwedCall, error), error) { + switch of { + case seat.Seat: + return bridge.GateOwed, nil + case checker.Seat: + return bridge.CheckerOwed, nil + } + return nil, fmt.Errorf("the seat %q says no owed act the bridge performs", of) +} + +// heard performs one owed event the seat of said. Its error is only an outcome that could not be said on the stream +// forge: the event is then heard again, and the act, safe to repeat, performed again. +func (o *owing) heard(ctx context.Context, of string, data []byte) (forgemsg.OwedEnded, error) { + read, err := readerOf(of) + if err != nil { + return o.end(ctx, forgemsg.OwedEnded{Of: of, Outcome: forgemsg.OwedUndecodable, Words: err.Error()}, data) + } + c, err := read(data) + if err != nil { + return o.end(ctx, forgemsg.OwedEnded{Of: of, ID: c.ID, + Outcome: forgemsg.OwedUndecodable, Words: err.Error()}, data) + } + if o.held(ctx, c.Of, c.ID) { + // said twice, or said again while its outcome was on its way: performed once, the outcome said once + return forgemsg.OwedEnded{Of: c.Of, ID: c.ID}, nil + } + if words, done := o.switchedDone(ctx, c.Of, c.ID); done { + // performed by the old bridge before the switch: said performed, never done twice + return o.end(ctx, forgemsg.OwedEnded{Of: c.Of, ID: c.ID, + Outcome: forgemsg.OwedPerformed, Words: words}, data) + } + return o.perform(ctx, c, data) +} + +// switchedDone says whether a switched act (design 47 *The switch carries what is owed*: the gate's act of a key of its +// old state owed or merges, its id meshdeliveryclient.SwitchedActID) was performed by the old bridge: its own bookkeeping +// in performed, `.`, holds that key at the act's revision or a later one. Any other id, and bookkeeping that +// cannot be read, is not: the act is performed, which changes nothing at the forge. +func (o *owing) switchedDone(ctx context.Context, of, id string) (string, bool) { + state, key, rev, ok := meshdeliveryclient.ParseSwitchedActID(id) + if !ok || of != seat.Seat { + return "", false + } + kept, found, err := o.performed.Get(ctx, state+"."+key) + if err != nil || !found { + return "", false + } + at, err := strconv.ParseUint(strings.TrimSpace(string(kept)), 10, 64) + if err != nil || at < rev { + return "", false + } + return fmt.Sprintf("performed by the bridge before the switch: its %s.%s at revision %d", state, key, at), true +} + +// hear is one message of a seat's stream owed: performed and acknowledged, or, when its outcome could not be said, +// left to come again (design 47: the bridge stopped between an act and its owed.ended performs it again). +func (o *owing) hear(ctx context.Context, of string, m delivered) { + if _, err := o.heard(ctx, of, m.Data()); err != nil { + o.logger.Printf("%s, so the act is heard again: %v", m.Subject(), err) + _ = m.NakWithDelay(retryDelay) + return + } + _ = m.Ack() +} + +// perform calls the seat git's verb of one act and says what it came to. +func (o *owing) perform(ctx context.Context, c bridge.OwedCall, data []byte) (forgemsg.OwedEnded, error) { + answer, callErr := o.call(ctx, "seat:"+git.Seat+"."+c.Verb, c.Input) + outcome, words := bridge.Outcome(answer, callErr) + if outcome != forgemsg.OwedPerformed { + o.logger.Printf("the owed %s %s of %s#%d (said by %s) is %s: %s", c.Act, c.ID, c.Repository, c.Number, c.Of, outcome, words) + } + return o.end(ctx, forgemsg.OwedEnded{Of: c.Of, ID: c.ID, + Outcome: outcome, Words: words}, data) +} + +// end says an act's outcome as owed.ended on the stream forge, then keeps it in performed. An outcome not kept is +// logged: the act may be performed again, which changes nothing at the forge. +func (o *owing) end(ctx context.Context, e forgemsg.OwedEnded, owed []byte) (forgemsg.OwedEnded, error) { + e.At = o.now().UTC() + data, err := forgemsg.EncodeOwedEnded(e) + if err != nil { + return e, fmt.Errorf("the outcome of the owed %s cannot be said: %w", e.ID, err) + } + if _, err := o.pub.Publish(ctx, forgeSubject(forgemsg.OwedEndedEvent), data); err != nil { + return e, fmt.Errorf("the outcome %s of the owed %s could not be said on the stream forge: %w", e.Outcome, e.ID, err) + } + if e.ID == "" { + return e, nil // an undecodable act with no id is said, and has no key to keep + } + if !json.Valid(owed) { + owed = nil + } + v, err := json.Marshal(Performed{Ended: e, Owed: owed}) + if err == nil { + err = o.performed.Put(ctx, performedKey(e.Of, e.ID), v) + } + if err != nil { + o.logger.Printf("the outcome %s of the owed %s is said but could not be kept in performed: %v", e.Outcome, e.ID, err) + } + return e, nil +} + +// held says whether performed holds an outcome of the act already. One that cannot be read is not held: the act is +// performed again, which changes nothing at the forge. +func (o *owing) held(ctx context.Context, of, id string) bool { + _, found, err := o.performed.Get(ctx, performedKey(of, id)) + if err != nil { + o.logger.Printf("performed of the owed %s cannot be read, so it is performed again: %v", id, err) + return false + } + return found +} + +// catchUp reads the seat mesh-delivery's groups and performs every act said there with no recorded outcome that +// performed does not hold either (design 47 *The bridge catches up from the seat*). It answers how many it performed. +func (o *owing) catchUp(ctx context.Context) (int, error) { + of, err := groups.Read(ctx, o.call) + if err != nil { + return 0, err + } + n := 0 + for _, g := range of { + for _, act := range g.Acts { + if act.Outcome != "" || o.held(ctx, seat.Seat, act.Owed.ID) { + continue // answered, or performed and its outcome on its way to the gate + } + data, err := json.Marshal(act.Owed) + if err != nil { + return n, err + } + if _, err := o.heard(ctx, seat.Seat, data); err != nil { + return n, err + } + n++ + } + } + return n, nil +} + +// owedOf is the owed event of an act to perform again: as performed keeps it, else, for the gate's, as its groups say it. +func (o *owing) owedOf(ctx context.Context, of, id string) ([]byte, error) { + v, found, err := o.performed.Get(ctx, performedKey(of, id)) + if err != nil { + return nil, fmt.Errorf("performed cannot be read: %w", err) + } + if found { + var p Performed + if err := json.Unmarshal(v, &p); err == nil && len(p.Owed) > 0 { + return p.Owed, nil + } + } + if of != seat.Seat { + return nil, fmt.Errorf("the bridge holds no owed act %s of %s: the checker says its unanswered acts again at its start", id, of) + } + raw, err := o.call(ctx, "seat:"+seat.Seat+"."+seat.VerbGroups, seat.GroupsInput{All: seat.GroupsInputAllTrue}) + if err != nil { + return nil, fmt.Errorf("the seat %s's groups could not be read: %w", seat.Seat, err) + } + var a seat.GroupsAnswer + if err := json.Unmarshal(raw, &a); err != nil || a.Error != "" { + return nil, fmt.Errorf("the seat %s's groups do not answer: %v %s", seat.Seat, err, a.Error) + } + for _, g := range a.Groups { + for _, act := range g.Acts { + if act.Owed.ID == id { + return json.Marshal(act.Owed) + } + } + } + return nil, fmt.Errorf("neither the bridge nor the seat %s's groups hold an owed act %s", seat.Seat, id) +} + +// answer answers one owed ask: an act performed again now, whatever performed holds of it; an act dropped, said as +// owed.ended with the outcome dropped and the ask's why (the tool wrote the hand-act log first); or the consumers' backlog. +func (o *owing) answer(ctx context.Context, a groups.OwedAsk, backlog func(context.Context) []groups.Backlog) groups.OwedAsk { + a.Answered = o.now().UTC() + switch a.Kind { + case groups.OwedBacklog: + a.Backlog, a.Answer = backlog(ctx), "the backlog of each seat's stream owed" + return a + } + data, err := o.owedOf(ctx, a.Of, a.ID) + if err != nil { + a.Answer = err.Error() + return a + } + var e forgemsg.OwedEnded + if a.Kind == groups.OwedDrop { + e, err = o.end(ctx, forgemsg.OwedEnded{Of: a.Of, ID: a.ID, Outcome: forgemsg.OwedDropped, Words: a.Why + " (dropped by " + a.By + ")"}, data) + } else if read, rerr := readerOf(a.Of); rerr != nil { + err = rerr + } else if c, rerr := read(data); rerr != nil { + e, err = o.end(ctx, forgemsg.OwedEnded{Of: a.Of, ID: a.ID, Outcome: forgemsg.OwedUndecodable, Words: rerr.Error()}, data) + } else { + e, err = o.perform(ctx, c, data) + } + if err != nil { + a.Answer = err.Error() + return a + } + a.Outcome, a.Answer = e.Outcome, "said as owed.ended: "+e.Outcome + if e.Words != "" { + a.Answer += ": " + e.Words + } + return a +} + +// asked answers one value of the state owed-asks not yet answered, written back by compare-and-set at its revision; an +// answered one (this bridge's own write) is left. +func (o *owing) asked(ctx context.Context, kv jetstream.KeyValue, key string, rev uint64, value []byte, backlog func(context.Context) []groups.Backlog) { + a, err := groups.DecodeOwedAsk(value) + if err != nil { + o.logger.Printf("%s.%s holds what is not an owed ask, left: %v", groups.OwedAsksState, key, err) + return + } + if a.Done() { + return + } + a = o.answer(ctx, a, backlog) + o.logger.Printf("the owed ask %s of %s %s by %s (%s): %s", a.Kind, a.Of, a.ID, a.By, a.Why, a.Answer) + data, err := json.Marshal(a) + if err != nil { + return + } + if _, err := kv.Update(ctx, key, data, rev); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) { + o.logger.Printf("%s.%s: the answer could not be written: %v", groups.OwedAsksState, key, err) + } +} + +// owedBacklog reads the bridge's durable on each seat's stream owed: its acknowledged floor, what waits and what is held, +// and the time of the first event after the floor. +func owedBacklog(js jetstream.JetStream, cred process.Account, now func() time.Time) func(context.Context) []groups.Backlog { + return func(ctx context.Context) []groups.Backlog { + var out []groups.Backlog + for _, kept := range []string{seat.StreamOwed, checker.StreamOwed} { + row := groups.Backlog{Stream: kept, Consumer: cred.EventConsumerName()} + c, err := js.Consumer(ctx, row.Stream, row.Consumer) + var info *jetstream.ConsumerInfo + if err == nil { + info, err = c.Info(ctx) + } + if err != nil { + row.Error = err.Error() + out = append(out, row) + continue + } + row.LastApplied, row.Waiting, row.Held = info.AckFloor.Stream, info.NumPending, uint64(info.NumAckPending) + if row.Waiting+row.Held > 0 { + if s, err := js.Stream(ctx, row.Stream); err == nil { + if m, err := s.GetMsg(ctx, info.AckFloor.Stream+1); err == nil { + row.Oldest, row.OldestSecs = m.Time.UTC(), int64(now().Sub(m.Time).Seconds()) + } + } + } + out = append(out, row) + } + return out + } +} diff --git a/modules/mesh-delivery-forge-bridge/forgemsg/forge.go b/modules/mesh-delivery-forge-bridge/forgemsg/forge.go index 822de445..03568714 100644 --- a/modules/mesh-delivery-forge-bridge/forgemsg/forge.go +++ b/modules/mesh-delivery-forge-bridge/forgemsg/forge.go @@ -156,7 +156,7 @@ type Forge struct { PredefinedRead bool `json:"predefined-read,omitempty"` // Group is the delivery group a merged message's pull request was merged in, and Members every approved member of // that group with its approved head, this one included (novox/hq ADR 0633 §4): the forge bridge sets both from the - // gate's merges state, and the indexer gathers the group's merges until the group's end (ForgeGroupEnd). A merge + // seat mesh-delivery's verb groups, and the indexer gathers the group's merges until the group's end (ForgeGroupEnd). A merge // with no group carries neither, and an older bridge sends neither; an older reader ignores both. Group string `json:"group,omitempty"` Members []meshdeliveryclient.Head `json:"members,omitempty"` diff --git a/modules/mesh-delivery-forge-bridge/go.mod b/modules/mesh-delivery-forge-bridge/go.mod index 9a5a262e..e7af2fcd 100644 --- a/modules/mesh-delivery-forge-bridge/go.mod +++ b/modules/mesh-delivery-forge-bridge/go.mod @@ -3,8 +3,10 @@ module mesh-delivery-forge-bridge go 1.26.0 require ( + git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/seats/git v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdeliverychecker v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/sdk v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000 diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/acts.go b/modules/mesh-delivery-forge-bridge/internal/bridge/acts.go deleted file mode 100644 index 65e58eff..00000000 --- a/modules/mesh-delivery-forge-bridge/internal/bridge/acts.go +++ /dev/null @@ -1,160 +0,0 @@ -package bridge - -import ( - "fmt" - "slices" - "strings" - - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" - seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" -) - -// Call is a call of a forge tool: the tool by its manifest key and its arguments. -type Call struct { - Tool string - Args map[string]any -} - -// Act is one thing the bridge does on the forge for a key of the gate: the act's name (an owed key's act, `status=`, -// `view`, `note`, `comment.`, or `merge`), the pull request and head it is on (Number 0 for a commit no pull -// request names) and the forge call that performs it. -type Act struct { - Name string - Repository string - Number int - Head string - Call Call -} - -// The forge's tools, by manifest key; their arguments are the ones of mesh-catalog modules/gitea: meshdeliveryclient.ts -// commitStatusTool (owner, repo, sha, context, state, description, target_url), tools/index.ts gitea_delivery_view -// (owner, repo, number, body), gitea_note_append (owner, repo, sha, ref, line), gitea_add_comment (owner, repo, -// number, body), merge.ts gitea_merge_pull_request (owner, repo, number, head, approved_heads). -const ( - commitStatusTool = "gitea.gitea_commit_status" - deliveryViewTool = "gitea.gitea_delivery_view" - noteAppendTool = "gitea.gitea_note_append" - addCommentTool = "gitea.gitea_add_comment" - mergePullTool = "gitea.gitea_merge_pull_request" -) - -// MergeAct is the name of the act of merging a pull request. -const MergeAct = "merge" - -func ownerRepo(repository string) (owner, repo string) { - owner, repo, _ = strings.Cut(repository, "/") - return owner, repo -} - -// OwedAct turns an owed key and its value (mesh-sdk go/delivery OwedKey, EncodeOwed) into the forge call that performs -// it. A key that is no owed key, a value that is no owed act, a value whose act is not the key's, and a view or a comment -// on no pull request are errors, each saying why. -func OwedAct(key string, value []byte) (Act, error) { - repository, number, head, name, err := meshdeliveryclient.ParseOwedKey(key) - if err != nil { - return Act{}, err - } - o, err := meshdeliveryclient.DecodeOwed(value) - if err != nil { - return Act{}, fmt.Errorf("the owed act %s: %w", key, err) - } - kind, context, err := meshdeliveryclient.ParseOwedAct(name) - if err != nil { - return Act{}, err - } - if o.Act != kind { - return Act{}, fmt.Errorf("the owed act %s says %q, not the %q its key names", key, o.Act, kind) - } - owner, repo := ownerRepo(repository) - a := Act{Name: name, Repository: repository, Number: number, Head: head} - needsPull := func() error { - if number <= 0 { - return fmt.Errorf("the owed %s %s is on a commit no pull request names, and it needs the pull request", kind, key) - } - return nil - } - switch kind { - case meshdeliveryclient.OwedStatus: - if o.Context != context { - return Act{}, fmt.Errorf("the owed status %s says the context %q, not the %q its key names", key, o.Context, context) - } - if o.Context == meshdeliveryclient.StatusContextCheck { - // No tool writes mesh/check (R4e finding K1), and the bridge performs no act of it: the forge module writes it from - // the checker's state mesh-delivery-checker.owed, read by a watch (ADR 0521, finding R2-1). - return Act{}, fmt.Errorf("the owed status %s is %s, which the bridge does not perform: the forge module writes it from mesh-delivery-checker.owed", - key, meshdeliveryclient.StatusContextCheck) - } - a.Call = Call{Tool: commitStatusTool, Args: map[string]any{"owner": owner, "repo": repo, "sha": head, "context": o.Context, - "state": o.State, "description": o.Text, "target_url": o.Target}} - case meshdeliveryclient.OwedView: - if err := needsPull(); err != nil { - return Act{}, err - } - a.Call = Call{Tool: deliveryViewTool, Args: map[string]any{"owner": owner, "repo": repo, "number": number, "body": o.Text}} - case meshdeliveryclient.OwedNote: - a.Call = Call{Tool: noteAppendTool, Args: map[string]any{"owner": owner, "repo": repo, "sha": head, "ref": o.Ref, "line": o.Text}} - case meshdeliveryclient.OwedComment: - if err := needsPull(); err != nil { - return Act{}, err - } - a.Call = Call{Tool: addCommentTool, Args: map[string]any{"owner": owner, "repo": repo, "number": number, "body": o.Text}} - } - return a, nil -} - -// MergeActs turns the value of a merges key (the delivery client's EncodeMerge) into the merges to perform: one for each -// pull request in the state merging, in the order the value names them, at the head it expects and with the approved -// heads of every member of the group, `/@` sorted (the merge tool's approved_heads). A pull -// request merged or refused already gets none. A value with no approved heads is an error: the forge's merge tool -// needs them for a group. -func MergeActs(value []byte) ([]Act, error) { - m, err := meshdeliveryclient.DecodeMerge(value) - if err != nil { - return nil, err - } - var handed []seat.MergesByGroupValuePullsItem - for _, p := range m.Pulls { - if meshdeliveryclient.MergeState(p.State) == meshdeliveryclient.MergeMerging { - handed = append(handed, p) - } - } - if len(handed) == 0 { - return nil, nil - } - if len(m.Approved) == 0 { - return nil, fmt.Errorf("the merge of the group %s names no approved heads, so the forge's merge tool has nothing to merge at", m.Group) - } - approved := make([]string, 0, len(m.Approved)) - for key, sha := range m.Approved { - repository, _, ok := strings.Cut(key, "@") - if !ok { - return nil, fmt.Errorf("the approved head %q of the group %s is not @", key, m.Group) - } - approved = append(approved, repository+"@"+sha) - } - slices.Sort(approved) - var acts []Act - for _, p := range handed { - owner, repo := ownerRepo(p.Repository) - acts = append(acts, Act{Name: MergeAct, Repository: p.Repository, Number: int(p.Number), Head: p.Head, - Call: Call{Tool: mergePullTool, Args: map[string]any{"owner": owner, "repo": repo, "number": int(p.Number), "head": p.Head, - "approved_heads": approved}}}) - } - return acts, nil -} - -// RefusedMessage is the refused message the bridge writes on FORGE when the forge refuses an act: the act, the pull -// request and head it was on, and the forge's own words. -func RefusedMessage(a Act, words string) (Message, error) { - event, err := forgemsg.ForgeEvent(a.Repository, forgemsg.ForgeRefused) - if err != nil { - return Message{}, err - } - body, err := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgeRefused, Repository: a.Repository, Number: a.Number, - Head: a.Head, Act: a.Name, Words: words}) - if err != nil { - return Message{}, err - } - return Message{Event: event, Data: body}, nil -} diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/acts_test.go b/modules/mesh-delivery-forge-bridge/internal/bridge/acts_test.go deleted file mode 100644 index a1cb4669..00000000 --- a/modules/mesh-delivery-forge-bridge/internal/bridge/acts_test.go +++ /dev/null @@ -1,188 +0,0 @@ -package bridge - -import ( - "reflect" - "strings" - "testing" - - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" - seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" - "git.novox.be/novox/mesh-sdk/go/keys" -) - -const ( - headA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - headB = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" -) - -// owedValue is an owed value as the SDK writes it (mesh-sdk go/delivery EncodeOwed), and its key as OwedKey spells it. -func owedValue(t *testing.T, repository string, number int, act string, o seat.OwedByRepositoryNumberHeadActValue) (string, []byte) { - t.Helper() - key, err := meshdeliveryclient.OwedKey(repository, number, headA, act) - if err != nil { - t.Fatal(err) - } - b, err := meshdeliveryclient.EncodeOwed(o) - if err != nil { - t.Fatal(err) - } - return key, b -} - -func TestOwedStatusIsACommitStatusCall(t *testing.T) { // 1341/AC1, K1 - key, v := owedValue(t, "novox/mesh-delivery", 12, "status=mesh/merge-gate", - seat.OwedByRepositoryNumberHeadActValue{Act: meshdeliveryclient.OwedStatus, Context: "mesh/merge-gate", State: "pending", Text: "waiting", Target: "https://forge.example/novox/mesh-delivery/pulls/12"}) - a, err := OwedAct(key, v) - if err != nil { - t.Fatal(err) - } - want := Call{Tool: "gitea.gitea_commit_status", Args: map[string]any{"owner": "novox", "repo": "mesh-delivery", "sha": headA, - "context": "mesh/merge-gate", "state": "pending", "description": "waiting", "target_url": "https://forge.example/novox/mesh-delivery/pulls/12"}} - if !reflect.DeepEqual(a.Call, want) { - t.Errorf("call = %+v, want %+v", a.Call, want) - } - if a.Repository != "novox/mesh-delivery" || a.Number != 12 || a.Head != headA || a.Name != "status=mesh/merge-gate" { - t.Errorf("act = %+v", a) - } - // R4e findings K1 and R2-1 (issue 1301 S4; ADR 0503 §3, ADR 0521): no tool writes mesh/check, since a tool is told no - // caller and node-tools invokes every tool; the forge module writes it from the checker's owed state, and the bridge - // performs no mesh/check act. - ck, cv := checkerKV(t, seat.OwedByRepositoryNumberHeadActValue{Act: meshdeliveryclient.OwedStatus, Context: meshdeliveryclient.StatusContextCheck, State: "success", Text: "pass", Target: "https://view.example/checks/g.1"}) - if a, err := OwedAct(ck, cv); err == nil || !strings.Contains(err.Error(), "mesh-delivery-checker.owed") { - t.Fatalf("OwedAct(mesh/check) = %+v, %v; want refused, naming where the forge module reads it", a, err) - } - // Nor through the gate's own owed bucket: the context is refused whichever key carries it. - gk, gv := owedValue(t, "novox/mesh-delivery", 12, "status=mesh/check", - seat.OwedByRepositoryNumberHeadActValue{Act: meshdeliveryclient.OwedStatus, Context: meshdeliveryclient.StatusContextCheck, State: "success", Text: "pass", Target: "https://x.example"}) - if a, err := OwedAct(gk, gv); err == nil || !strings.Contains(err.Error(), "mesh-delivery-checker.owed") { - t.Fatalf("OwedAct(gate's mesh/check) = %+v, %v; want refused", a, err) - } -} - -// checkerKV is the status the checker owes, as it writes it (the gate's owed key with the act status=mesh/check). -func checkerKV(t *testing.T, o seat.OwedByRepositoryNumberHeadActValue) (string, []byte) { - t.Helper() - key, err := meshdeliveryclient.OwedKey("novox/mesh-delivery", 12, headA, "status=mesh/check") - if err != nil { - t.Fatal(err) - } - v, err := meshdeliveryclient.EncodeOwed(o) - if err != nil { - t.Fatal(err) - } - return key, v -} - -func TestOwedViewNoteAndCommentCalls(t *testing.T) { // 1341/AC2 - cases := []struct { - act string - o seat.OwedByRepositoryNumberHeadActValue - want Call - }{ - {"view", seat.OwedByRepositoryNumberHeadActValue{Act: meshdeliveryclient.OwedView, Text: "the view"}, - Call{Tool: "gitea.gitea_delivery_view", Args: map[string]any{"owner": "novox", "repo": "mesh-delivery", "number": 12, "body": "the view"}}}, - {"note", seat.OwedByRepositoryNumberHeadActValue{Act: meshdeliveryclient.OwedNote, Text: "a line", Ref: "mesh-plan"}, - Call{Tool: "gitea.gitea_note_append", Args: map[string]any{"owner": "novox", "repo": "mesh-delivery", "sha": headA, "ref": "mesh-plan", "line": "a line"}}}, - {"comment.2", seat.OwedByRepositoryNumberHeadActValue{Act: meshdeliveryclient.OwedComment, Text: "hello"}, - Call{Tool: "gitea.gitea_add_comment", Args: map[string]any{"owner": "novox", "repo": "mesh-delivery", "number": 12, "body": "hello"}}}, - } - for _, c := range cases { - key, v := owedValue(t, "novox/mesh-delivery", 12, c.act, c.o) - a, err := OwedAct(key, v) - if err != nil { - t.Fatalf("%s: %v", c.act, err) - } - if !reflect.DeepEqual(a.Call, c.want) { - t.Errorf("%s: call = %+v, want %+v", c.act, a.Call, c.want) - } - } -} - -func TestOwedActWithoutAPullRequestNeedsOneOnlyWhereTheToolDoes(t *testing.T) { - key, v := owedValue(t, "novox/mesh-delivery", 0, "note", seat.OwedByRepositoryNumberHeadActValue{Act: meshdeliveryclient.OwedNote, Text: "l", Ref: "mesh-plan"}) - if _, err := OwedAct(key, v); err != nil { - t.Errorf("a note on a commit no pull request names: %v", err) - } - key, v = owedValue(t, "novox/mesh-delivery", 0, "view", seat.OwedByRepositoryNumberHeadActValue{Act: meshdeliveryclient.OwedView, Text: "v"}) - if _, err := OwedAct(key, v); err == nil { - t.Error("a view of no pull request was taken") - } -} - -func TestOwedActThatDisagreesWithItsKeyIsRefused(t *testing.T) { - // A dotted repository's key is unescaped, and its act names the repository whole. - dk, dv := owedValue(t, "novox/my.repo", 3, "comment.1", seat.OwedByRepositoryNumberHeadActValue{Act: meshdeliveryclient.OwedComment, Text: "x"}) - if a, err := OwedAct(dk, dv); err != nil || a.Call.Args["repo"] != "my.repo" || a.Repository != "novox/my.repo" { - t.Errorf("a dotted repository's act = %+v, %v", a, err) - } - key, _ := owedValue(t, "novox/mesh-delivery", 12, "view", seat.OwedByRepositoryNumberHeadActValue{Act: meshdeliveryclient.OwedView, Text: "v"}) - v, _ := meshdeliveryclient.EncodeOwed(seat.OwedByRepositoryNumberHeadActValue{Act: meshdeliveryclient.OwedComment, Text: "v"}) - if _, err := OwedAct(key, v); err == nil { - t.Error("a comment under a view's key was taken") - } - if _, err := OwedAct("nonsense", v); err == nil { - t.Error("a key that is no owed key was taken") - } - if _, err := OwedAct(key, []byte(`{"v":1`)); err == nil { - t.Error("a value that is not json was taken") - } -} - -func mergeValue(t *testing.T, m seat.MergesByGroupValue) []byte { - t.Helper() - b, err := meshdeliveryclient.EncodeMerge(m) - if err != nil { - t.Fatal(err) - } - return b -} - -func TestMergeActsHandOnlyTheMemberHandedWithEveryApprovedHead(t *testing.T) { // 1342/AC1 - approved := map[string]string{meshdeliveryclient.ApprovedKey("novox/mesh-sdk", 4): headB, meshdeliveryclient.ApprovedKey("novox/mesh-delivery", 12): headA} - v := mergeValue(t, seat.MergesByGroupValue{Group: "g", Ask: "merge.g.0123456789ab", Approved: approved, Pulls: []seat.MergesByGroupValuePullsItem{ - {Repository: "novox/mesh-sdk", Number: 4, Head: headB, State: string(meshdeliveryclient.MergeMerged), Commit: "c0"}, - {Repository: "novox/mesh-delivery", Number: 12, Head: headA, State: string(meshdeliveryclient.MergeMerging)}, - }}) - acts, err := MergeActs(v) - if err != nil { - t.Fatal(err) - } - if len(acts) != 1 { - t.Fatalf("%d acts, want 1 (the merged member is not merged again)", len(acts)) - } - want := Call{Tool: "gitea.gitea_merge_pull_request", Args: map[string]any{"owner": "novox", "repo": "mesh-delivery", "number": 12, "head": headA, - "approved_heads": []string{"novox/mesh-delivery@" + headA, "novox/mesh-sdk@" + headB}}} - if !reflect.DeepEqual(acts[0].Call, want) { - t.Errorf("call = %+v, want %+v", acts[0].Call, want) - } - if acts[0].Name != "merge" || acts[0].Number != 12 || acts[0].Head != headA { - t.Errorf("act = %+v", acts[0]) - } - // A merge with no approved heads is refused: the forge's tool would refuse it with nothing to say which. - v = mergeValue(t, seat.MergesByGroupValue{Group: "g", Ask: "a", Pulls: []seat.MergesByGroupValuePullsItem{ - {Repository: "novox/mesh-delivery", Number: 12, Head: headA, State: string(meshdeliveryclient.MergeMerging)}}}) - if _, err := MergeActs(v); err == nil { - t.Error("a merge with no approved heads was taken") - } -} - -func TestRefusedMessageNamesTheActAndTheForgesWords(t *testing.T) { // 1341/AC3, 1342/AC2 - a := Act{Name: "merge", Repository: "novox/mesh-delivery", Number: 12, Head: headA} - m, err := RefusedMessage(a, "novox/mesh-delivery#12 is not merged: its head moved") - if err != nil { - t.Fatal(err) - } - if m.Event != "forge.novox.mesh-delivery.refused" { - t.Errorf("event = %q", m.Event) - } - f, err := forgemsg.DecodeForge(m.Data) - if err != nil { - t.Fatal(err) - } - want := forgemsg.Forge{V: keys.Version, Kind: forgemsg.ForgeRefused, Repository: "novox/mesh-delivery", Number: 12, Head: headA, - Act: "merge", Words: "novox/mesh-delivery#12 is not merged: its head moved"} - if !forgemsg.SameForge(f, want) { - t.Errorf("message = %+v", f) - } -} diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/call.go b/modules/mesh-delivery-forge-bridge/internal/bridge/call.go index 344cc9bd..fee308e4 100644 --- a/modules/mesh-delivery-forge-bridge/internal/bridge/call.go +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/call.go @@ -89,7 +89,7 @@ var errNoAnswer = errors.New("the seat git's answer carries nothing") // MergedMessage is the merged message of a merge into the trunk, asked again: built as the bridge builds the one it writes // on FORGE (it is that event's body through ConvertKept), from the pull request's owner, repository, number, branch, head -// and merge commit, which the gate's merges state holds. It reads the forge for every module directory (no kept inputs). +// and merge commit, which the seat git's verb pull answers. It reads the forge for every module directory (no kept inputs). func MergedMessage(owner, repo string, number int, branch, head, merge string, trees Trees, files Files) (Message, error) { body, err := json.Marshal(git.PullMergedEvent{Repository: repository(owner, repo), Number: int64(number), Branch: branch, Head: head, Base: Trunk, MergeCommit: merge}) if err != nil { diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/owed.go b/modules/mesh-delivery-forge-bridge/internal/bridge/owed.go new file mode 100644 index 00000000..4fd274da --- /dev/null +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/owed.go @@ -0,0 +1,147 @@ +package bridge + +import ( + "encoding/json" + "errors" + "fmt" + + git "git.novox.be/novox/mesh-delivery/contracts/go/seats/git" + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" + checker "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdeliverychecker" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" +) + +// OwedCall is one act owed to the forge as the seat git's verb that performs it: the seat that said it (Of), the act's +// id, and the verb with its input (novox/hq design 47 *What the gate owes the forge*). Every verb is safe to repeat at +// the forge: a status and the view replace what stands, a merge of a pull request merged at that head is answered +// performed, and a comment and a note line carry the id, which the seat's holder skips when it already holds it. +type OwedCall struct { + Of string + ID string + Act string + Repository string + Number int + Verb string + Input any +} + +// ErrUndecodable is an owed event the bridge cannot read: it is said as owed.ended `undecodable`, never only logged. +var ErrUndecodable = errors.New("an owed act the bridge cannot read") + +func undecodable(why string, args ...any) error { + return fmt.Errorf("%w: %s", ErrUndecodable, fmt.Sprintf(why, args...)) +} + +// GateOwed reads the seat mesh-delivery's event owed into the seat git's verb that performs it. +func GateOwed(data []byte) (OwedCall, error) { + var o seat.OwedEvent + if err := json.Unmarshal(data, &o); err != nil { + return OwedCall{}, undecodable("the gate's owed is not json: %v", err) + } + c := OwedCall{Of: seat.Seat, ID: o.ID, Act: o.Act, Repository: o.Repository, Number: int(o.Number)} + if err := c.common(o.Head); err != nil { + return c, err + } + commit := o.Commit + if commit == "" { + commit = o.Head + } + needsPull := func() error { + if o.Number <= 0 { + return undecodable("the owed %s %s names no pull request", o.Act, o.ID) + } + return nil + } + switch o.Act { + case seat.OwedActMerge: + // Only a switched act is a merge (§169): from G's delivery the gate owes none (§167), a session calls merge-pull + // itself; a merge of the old state, not yet performed by the old bridge, is not dropped. + if err := needsPull(); err != nil { + return c, err + } + c.Verb, c.Input = git.VerbMergePull, git.MergePullInput{ID: o.ID, Repository: o.Repository, Number: o.Number, Head: o.Head} + case seat.OwedActStatus: + if o.Context == "" || o.State == "" { + return c, undecodable("the owed status %s names no context or state", o.ID) + } + c.Verb, c.Input = git.VerbStatus, git.StatusInput{ID: o.ID, Repository: o.Repository, Commit: commit, Context: o.Context, + State: o.State, Description: o.Description, Target: o.Target} + case seat.OwedActView: + if err := needsPull(); err != nil { + return c, err + } + c.Verb, c.Input = git.VerbView, git.ViewInput{ID: o.ID, Repository: o.Repository, Number: o.Number, Text: o.Text} + case seat.OwedActComment: + if err := needsPull(); err != nil { + return c, err + } + c.Verb, c.Input = git.VerbComment, git.CommentInput{ID: o.ID, Repository: o.Repository, Number: o.Number, Text: o.Text} + case seat.OwedActNote: + if o.Line == "" { + return c, undecodable("the owed note line %s says no line", o.ID) + } + c.Verb, c.Input = git.VerbNote, git.NoteInput{ID: o.ID, Repository: o.Repository, Commit: commit, Line: o.Line} + default: + return c, undecodable("the owed %s is the act %q, none the seat git performs", o.ID, o.Act) + } + return c, nil +} + +// CheckerOwed reads the seat mesh-delivery-checker's event owed (its mesh/check status) into the seat git's verb status. +func CheckerOwed(data []byte) (OwedCall, error) { + var o checker.OwedEvent + if err := json.Unmarshal(data, &o); err != nil { + return OwedCall{}, undecodable("the checker's owed is not json: %v", err) + } + c := OwedCall{Of: checker.Seat, ID: o.ID, Act: o.Act, Repository: o.Repository, Number: int(o.Number)} + if err := c.common(o.Head); err != nil { + return c, err + } + if o.Act != checker.OwedActStatus || o.Context == "" || o.State == "" { + return c, undecodable("the checker's owed %s is %q of the context %q in the state %q, not a status", o.ID, o.Act, o.Context, o.State) + } + c.Verb, c.Input = git.VerbStatus, git.StatusInput{ID: o.ID, Repository: o.Repository, Commit: o.Head, Context: o.Context, + State: o.State, Description: o.Description, Target: o.Target} + return c, nil +} + +func (c OwedCall) common(head string) error { + switch { + case c.ID == "": + return undecodable("an owed %s of %s names no id", c.Act, c.Repository) + case c.Repository == "": + return undecodable("the owed %s names no repository", c.ID) + case !sha40.MatchString(head): + return undecodable("the owed %s names the head %q, not 40 hex digits", c.ID, head) + } + if _, _, err := splitRepository(c.Repository); err != nil { + return undecodable("the owed %s: %v", c.ID, err) + } + return nil +} + +// Outcome is what an act's call of the seat git came to, as owed.ended says it: performed when the forge holds the act +// (done now, or its id already held); refused with the forge's own words when the seat answers a refusal or an error +// of the kind refused; could not be performed with the error when the seat did not answer, answered an error of the +// kind failed, or answered what the contract cannot read. +func Outcome(raw json.RawMessage, callErr error) (outcome, words string) { + if callErr != nil { + return forgemsg.OwedCouldNotBePerformed, callErr.Error() + } + var a git.MergePullAnswer // every act verb's answer is this one's fields or fewer + if err := json.Unmarshal(raw, &a); err != nil { + return forgemsg.OwedCouldNotBePerformed, "the seat git's answer is not its contract's: " + err.Error() + } + switch { + case a.Refused != "": + return forgemsg.OwedRefused, a.Refused + case a.Error != "" && a.ErrorKind == git.ErrorKindRefused: + return forgemsg.OwedRefused, a.Error + case a.Error != "": + // failed, or a kind the contract does not name: the forge did not do it, and did not say no + return forgemsg.OwedCouldNotBePerformed, a.Error + case a.Performed || a.Held: + return forgemsg.OwedPerformed, "" + } + return forgemsg.OwedCouldNotBePerformed, "the seat git's answer says neither performed, held, refused nor an error: " + string(raw) +} diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/owed_test.go b/modules/mesh-delivery-forge-bridge/internal/bridge/owed_test.go new file mode 100644 index 00000000..bdc235e6 --- /dev/null +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/owed_test.go @@ -0,0 +1,95 @@ +package bridge + +import ( + "encoding/json" + "errors" + "reflect" + "strings" + "testing" + + git "git.novox.be/novox/mesh-delivery/contracts/go/seats/git" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" + + "mesh-delivery-forge-bridge/internal/gitfixture" +) + +// Design 47 *What the gate owes the forge*: each act the gate or the checker says as owed is the seat git's verb of that +// act, under the act's id; what cannot be read is undecodable, and every answer is one outcome. +func TestAnOwedEventIsTheSeatGitsVerbAndItsAnswerOneOutcome(t *testing.T) { + gate := func(name string) []byte { return gitfixture.Seat(t, "mesh-delivery", "events", name, "event:owed")[0] } + chk := gitfixture.Seat(t, "mesh-delivery-checker", "events", "owed is said", "event:owed")[0] + a40 := strings.Repeat("a", 40) + edit := func(raw []byte, k string, v any) []byte { + var m map[string]any + _ = json.Unmarshal(raw, &m) + if v == nil { + delete(m, k) + } else { + m[k] = v + } + b, _ := json.Marshal(m) + return b + } + for _, c := range []struct { + name string + read func([]byte) (OwedCall, error) + data []byte + verb string + input any + bad string + }{ + {"a merge on no pull request", GateOwed, edit(gate("owed is said, a merge"), "number", 0), "", nil, "no pull request"}, + {"a status on the head", GateOwed, gate("owed is said, a status"), "status", git.StatusInput{ID: "status.novox/mesh-sdk#178.aaaaaaaaaaaa.mesh/merge-gate", + Repository: "novox/mesh-sdk", Commit: a40, Context: "mesh/merge-gate", State: "pending", Description: "waits for the operator's Approve", Target: "https://forge.test/x"}, ""}, + {"a view", GateOwed, edit(edit(gate("owed is said, a status"), "act", "view"), "text", "the view"), "view", + git.ViewInput{ID: "status.novox/mesh-sdk#178.aaaaaaaaaaaa.mesh/merge-gate", Repository: "novox/mesh-sdk", Number: 178, Text: "the view"}, ""}, + {"a comment", GateOwed, edit(edit(gate("owed is said, a status"), "act", "comment"), "text", "hello"), "comment", + git.CommentInput{ID: "status.novox/mesh-sdk#178.aaaaaaaaaaaa.mesh/merge-gate", Repository: "novox/mesh-sdk", Number: 178, Text: "hello"}, ""}, + {"a note line on its commit", GateOwed, edit(edit(edit(gate("owed is said, a status"), "act", "note"), "line", "a line"), "commit", strings.Repeat("c", 40)), "note", + git.NoteInput{ID: "status.novox/mesh-sdk#178.aaaaaaaaaaaa.mesh/merge-gate", Repository: "novox/mesh-sdk", Commit: strings.Repeat("c", 40), Line: "a line"}, ""}, + {"the checker's status", CheckerOwed, chk, "status", git.StatusInput{ID: "status.novox/mesh-sdk#178.aaaaaaaaaaaa.mesh/check", + Repository: "novox/mesh-sdk", Commit: a40, Context: "mesh/check", State: "success", Description: "passed", Target: "https://forge.test/x"}, ""}, + {"not json", GateOwed, []byte("{"), "", nil, "not json"}, + {"no id", GateOwed, edit(gate("owed is said, a merge"), "id", nil), "", nil, "no id"}, + {"a short head", CheckerOwed, edit(chk, "head", "abc"), "", nil, "40 hex"}, + {"a merge of the switch", GateOwed, gate("owed is said, a merge"), "merge-pull", git.MergePullInput{ID: "merge.feat=47-G.0123456789ab.novox/mesh-sdk#178", + Repository: "novox/mesh-sdk", Number: 178, Head: a40}, ""}, + {"a comment on no pull request", GateOwed, edit(edit(gate("owed is said, a status"), "act", "comment"), "number", 0), "", nil, "no pull request"}, + {"an act of no kind", GateOwed, edit(gate("owed is said, a status"), "act", "label"), "", nil, "none the seat git"}, + {"a repository not owner/name", GateOwed, edit(gate("owed is said, a status"), "repository", "mesh-sdk"), "", nil, "owner"}, + } { + got, err := c.read(c.data) + switch { + case c.bad != "": + if !errors.Is(err, ErrUndecodable) || !strings.Contains(err.Error(), c.bad) { + t.Errorf("%s: %v, want undecodable naming %q", c.name, err, c.bad) + } + case err != nil || got.Verb != c.verb || !reflect.DeepEqual(got.Input, c.input): + t.Errorf("%s: %s %+v %v, want %s %+v", c.name, got.Verb, got.Input, err, c.verb, c.input) + } + } + answers := func(verb string) []json.RawMessage { return gitfixture.All(t, verb+" answers", "verb:"+verb+".answer") } + failed := func(verb string) json.RawMessage { + return gitfixture.Given(t, verb+" answers an error, never none", "verb:"+verb+".answer") + } + for _, c := range []struct { + name string + raw json.RawMessage + err error + outcome, words string + }{ + {"a merge performed", answers("merge-pull")[0], nil, forgemsg.OwedPerformed, ""}, + {"a merge refused", answers("merge-pull")[1], nil, forgemsg.OwedRefused, "the head moved"}, + {"a comment already held", answers("comment")[0], nil, forgemsg.OwedPerformed, ""}, + {"the forge's error, of the kind refused", failed("comment"), nil, forgemsg.OwedRefused, "403"}, + {"the forge's error, of the kind failed", failed("view"), nil, forgemsg.OwedCouldNotBePerformed, "502"}, + {"no answer in the call's bound", nil, errors.New("nats: timeout"), forgemsg.OwedCouldNotBePerformed, "timeout"}, + {"an answer saying nothing", json.RawMessage(`{"id":"x"}`), nil, forgemsg.OwedCouldNotBePerformed, "neither"}, + {"an answer not the contract's", json.RawMessage(`[1]`), nil, forgemsg.OwedCouldNotBePerformed, "contract"}, + } { + outcome, words := Outcome(c.raw, c.err) + if outcome != c.outcome || !strings.Contains(words, c.words) { + t.Errorf("%s: %q %q, want %q naming %q", c.name, outcome, words, c.outcome, c.words) + } + } +} diff --git a/modules/mesh-delivery-forge-bridge/internal/gitfixture/gitfixture.go b/modules/mesh-delivery-forge-bridge/internal/gitfixture/gitfixture.go index f3e5f09d..29641622 100644 --- a/modules/mesh-delivery-forge-bridge/internal/gitfixture/gitfixture.go +++ b/modules/mesh-delivery-forge-bridge/internal/gitfixture/gitfixture.go @@ -13,9 +13,16 @@ import ( // All is every fixture's body named name of of (such as "file answers", "verb:file.answer"), in the file's order; a // name with none fails the test. func All(t testing.TB, name, of string) []json.RawMessage { + t.Helper() + return Seat(t, "git", "verbs", name, of) +} + +// Seat is every fixture's body named name of of in a seat's fixtures file (such as "mesh-delivery", "events"), so the +// bridge's fakes of the seats it hears say what those seats say. +func Seat(t testing.TB, seat, file, name, of string) []json.RawMessage { t.Helper() _, here, _, _ := runtime.Caller(0) - raw, err := os.ReadFile(filepath.Join(filepath.Dir(here), "../../../../contracts/seats/git/fixtures/verbs.json")) + raw, err := os.ReadFile(filepath.Join(filepath.Dir(here), "../../../../contracts/seats", seat, "fixtures", file+".json")) if err != nil { t.Fatal(err) } diff --git a/modules/mesh-delivery-forge-bridge/internal/groups/groups.go b/modules/mesh-delivery-forge-bridge/internal/groups/groups.go index e3ddbc03..5cfde96d 100644 --- a/modules/mesh-delivery-forge-bridge/internal/groups/groups.go +++ b/modules/mesh-delivery-forge-bridge/internal/groups/groups.go @@ -5,6 +5,7 @@ package groups import ( + "context" "crypto/sha256" "encoding/hex" "encoding/json" @@ -12,12 +13,13 @@ import ( "fmt" "slices" "sort" - "strconv" "strings" "time" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" + "git.novox.be/novox/mesh-delivery/sdk/trunk" + "git.novox.be/novox/mesh-sdk/go/keys" ) // The bridge's states the tools read and write. @@ -26,20 +28,34 @@ const ( Module = "mesh-delivery-forge-bridge" // PerformedState is the bridge's state of what it performed and published. PerformedState = "performed" - // EndAsksState is the state of asks to say a group's end again, keyed by the gate's merges key of the group. + // EndAsksState is the state of asks to say a group's end again, keyed by the group escaped as a key. EndAsksState = "end-asks" // EndAsksBucket is its bucket on the bus. EndAsksBucket = Module + "_" + EndAsksState ) -// Key is the key of performed that says a group's end is written: `groups.`, its value the revision of the -// gate's merges key it was worked out from. -func Key(mergesKey string) string { return "groups." + mergesKey } +// Key is the key of performed that says a group's end is written: `groups.`, its value the last +// member's merge commit the end named. +func Key(group string) (string, error) { + part, err := KeyOf(group) + if err != nil { + return "", err + } + return "groups." + part, nil +} + +// KeyOf is a group's id (its branch) escaped as a key, as the states performed and end-asks key it. +func KeyOf(group string) (string, error) { + if strings.TrimSpace(group) == "" { + return "", errors.New("a group's id is empty") + } + return keys.EscapeName(group), nil +} // PublishedKey is the key of performed that says a member's merged message is written: `published.<16 hex digits of the -// sha256 of #@>`. -func PublishedKey(repository string, number int, merge string) string { - sum := sha256.Sum256([]byte(fmt.Sprintf("%s#%d@%s", strings.ToLower(repository), number, merge))) +// sha256 of #>`, its value the merge commit the message named (a pull request merges once). +func PublishedKey(repository string, number int) string { + sum := sha256.Sum256([]byte(fmt.Sprintf("%s#%d", strings.ToLower(repository), number))) return "published." + hex.EncodeToString(sum[:8]) } @@ -72,13 +88,13 @@ type EndAsk struct { // Done says the ask is answered. func (a EndAsk) Done() bool { return !a.Answered.IsZero() } -// NewEndAsk is the value bridge_group_end_again writes for a group: its key (the gate's merges key) and the value. +// NewEndAsk is the value bridge_group_end_again writes for a group: its key (the group escaped as a key) and the value. func NewEndAsk(group, why, by string, at time.Time) (string, []byte, error) { group = strings.TrimSpace(group) if group == "" { return "", nil, errors.New("name the group whose end is said again (group)") } - key, err := meshdeliveryclient.MergesKey(group) + key, err := KeyOf(group) if err != nil { return "", nil, err } @@ -98,21 +114,24 @@ func DecodeEndAsk(data []byte) (EndAsk, error) { return a, nil } -// Row is one group as bridge_groups answers it: the gate's merge of it and what the bridge kept of its end. +// Row is one group as bridge_groups answers it: the seat mesh-delivery's group and what the bridge kept of its end. type Row struct { Key string `json:"key"` Group string `json:"group"` + State string `json:"state,omitempty"` Members []RowMember `json:"members"` Ended bool `json:"ended"` - EndedAt uint64 `json:"ended-at-revision,omitempty"` - Open bool `json:"open"` - Words string `json:"words"` + // EndedWith is the last member's merge commit the end named, when it is written. + EndedWith string `json:"ended-with,omitempty"` + Open bool `json:"open"` + Words string `json:"words"` // LastEndAsk is the last ask to say the group's end again (bridge_group_end_again), its who, why and when, and the // bridge's answer once it read it; absent when none was asked. The state end-asks keeps the last per group. LastEndAsk *EndAsk `json:"last-end-ask,omitempty"` } -// RowMember is one approved member of a group: its merge's state at the gate and whether its merged message is written. +// RowMember is one member of a group: its delivery state at the seat and, once its merged message is written, the merge +// commit it named. type RowMember struct { Repository string `json:"repository"` Number int `json:"number"` @@ -121,12 +140,12 @@ type RowMember struct { Published bool `json:"published"` } -// Rows are the groups the gate's merges hold with more than one member, each with what the bridge kept: ended (its end -// written, the revision it was worked out from) or open (some member is not merged or its merged message not written, or its -// end not written). merges is the gate's merges by key, performed the bridge's performed keys by key; unread are the gate's -// keys that do not read. asks are the end-asks values by the same key; one that does not read is named unread as -// `end-asks.`. -func Rows(merges map[string][]byte, performed map[string]bool, endedAt map[string]uint64, asks map[string][]byte) (rows []Row, unread []string) { +// Rows are the groups the seat mesh-delivery's verb groups answers, each with what the bridge kept: ended (its end written, +// with the last merge it named) or open (some member's merged message is not written, or its end not written). published +// is performed's `published.` values by key (the merge commit), ended its `groups.` values by key; asks are the end-asks +// values by the same key, and one that does not read is named unread as `end-asks.`. A group whose id cannot be a +// key is named unread by its id. +func Rows(of []seat.Group, published, ended map[string]string, asks map[string][]byte) (rows []Row, unread []string) { lastAsk := map[string]*EndAsk{} for key, raw := range asks { a, err := DecodeEndAsk(raw) @@ -136,37 +155,32 @@ func Rows(merges map[string][]byte, performed map[string]bool, endedAt map[strin } lastAsk[key] = &a } - for key, raw := range merges { - m, err := meshdeliveryclient.DecodeMerge(raw) + for _, g := range of { + if len(g.Members) < 2 { + continue + } + key, err := KeyOf(g.ID) if err != nil { - unread = append(unread, key) + unread = append(unread, "a group with no id") continue } - if len(m.Approved) < 2 { - continue + r := Row{Key: key, Group: g.ID, State: g.State} + allPublished := true + for _, m := range g.Members { + commit, pub := published[PublishedKey(m.Repository, int(m.Number))] + r.Members = append(r.Members, RowMember{Repository: m.Repository, Number: int(m.Number), State: m.State, Commit: commit, Published: pub}) + allPublished = allPublished && pub } - r := Row{Key: key, Group: m.Group} - allMerged := true - for _, p := range m.Pulls { - pub := p.Commit != "" && performed[PublishedKey(p.Repository, int(p.Number), p.Commit)] - r.Members = append(r.Members, RowMember{Repository: p.Repository, Number: int(p.Number), State: p.State, Commit: p.Commit, Published: pub}) - if meshdeliveryclient.MergeState(p.State) != meshdeliveryclient.MergeMerged || !pub { - allMerged = false - } - } - if len(m.Pulls) < len(m.Approved) { - allMerged = false - } - r.EndedAt, r.Ended = endedAt[key] + r.EndedWith, r.Ended = ended["groups."+key] r.LastEndAsk = lastAsk[key] r.Open = !r.Ended switch { case r.Ended: r.Words = "its end is written" - case allMerged: - r.Words = "every member is merged and its message written, and its end is not written: bridge_group_end_again says it" + case allPublished: + r.Words = "every member's merged message is written, and its end is not written: bridge_group_end_again says it" default: - r.Words = "gathered: not every member is merged and its merged message written" + r.Words = "gathered: not every member's merged message is written" } rows = append(rows, r) } @@ -177,25 +191,46 @@ func Rows(merges map[string][]byte, performed map[string]bool, endedAt map[strin func sortRows(rows []Row) { sort.Slice(rows, func(i, j int) bool { return rows[i].Key < rows[j].Key }) } -// MembersOf are a merge's approved members with their heads, in the order of their keys. -func MembersOf(m seat.MergesByGroupValue) []meshdeliveryclient.Head { - var out []meshdeliveryclient.Head - for _, k := range slices.Sorted(func(yield func(string) bool) { - for k := range m.Approved { - if !yield(k) { - return - } - } - }) { - at := strings.LastIndex(k, "@") - if at < 0 { - continue - } - n, err := strconv.Atoi(k[at+1:]) - if err != nil { - continue - } - out = append(out, meshdeliveryclient.Head{Repository: k[:at], Number: n, Head: m.Approved[k]}) +// MembersOf are a group's members with their heads, in the group's order. +func MembersOf(g seat.Group) []meshdeliveryclient.Head { + out := make([]meshdeliveryclient.Head, 0, len(g.Members)) + for _, m := range g.Members { + out = append(out, meshdeliveryclient.Head{Repository: m.Repository, Number: int(m.Number), Head: m.Head}) } return out } + +// Named is the group of id among the groups the seat answered, when it has more than one member and names the pull +// request number of repository among them (a lone pull request is no group). +func Named(of []seat.Group, id, repository string, number int) (seat.Group, bool) { + for _, g := range of { + if g.ID != id || len(g.Members) < 2 { + continue + } + if slices.ContainsFunc(g.Members, func(m seat.Member) bool { + return trunk.SameRepository(m.Repository, repository) && int(m.Number) == number + }) { + return g, true + } + } + return seat.Group{}, false +} + +// Read is every delivery group the seat mesh-delivery's verb groups answers, the ended ones too (all), called through call +// by the verb's manifest key: the bridge reads a group from the seat, never from the gate's state (design 47 *A group's +// merges are taken in together*). An answer saying an error, or one the contract cannot type, is an error: a failed read +// is never read as no group. +func Read(ctx context.Context, call func(ctx context.Context, key string, body any) (json.RawMessage, error)) ([]seat.Group, error) { + raw, err := call(ctx, "seat:"+seat.Seat+"."+seat.VerbGroups, seat.GroupsInput{All: seat.GroupsInputAllTrue}) + if err != nil { + return nil, fmt.Errorf("the seat %s did not answer %s: %w", seat.Seat, seat.VerbGroups, err) + } + var a seat.GroupsAnswer + if err := json.Unmarshal(raw, &a); err != nil { + return nil, fmt.Errorf("the seat %s's answer of %s is not its contract's: %w", seat.Seat, seat.VerbGroups, err) + } + if a.Error != "" { + return nil, fmt.Errorf("the seat %s refused %s: %s", seat.Seat, seat.VerbGroups, a.Error) + } + return a.Groups, nil +} diff --git a/modules/mesh-delivery-forge-bridge/internal/groups/owedask.go b/modules/mesh-delivery-forge-bridge/internal/groups/owedask.go new file mode 100644 index 00000000..e8ec3319 --- /dev/null +++ b/modules/mesh-delivery-forge-bridge/internal/groups/owedask.go @@ -0,0 +1,95 @@ +package groups + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "strings" + "time" +) + +// What a person asks of the bridge's performing of what is owed (novox/hq design 47 *What the gate owes the forge*): the +// tools bridge_owed (its consumers' backlog), bridge_owed_again and bridge_owed_drop write an ask into the state +// owed-asks and wait for the bridge's answer there, as bridge_group_end_again does, since the bridge is the other program +// and serves no request subject (ADR 0437 §4). +const ( + // OwedAsksState is the state of asks about what is owed, one key per kind of ask and act. + OwedAsksState = "owed-asks" + // OwedAsksBucket is its bucket on the bus. + OwedAsksBucket = Module + "_" + OwedAsksState +) + +// The kinds of an owed ask. +const ( + OwedAgain = "again" + OwedDrop = "drop" + OwedBacklog = "backlog" +) + +// Backlog is one durable consumer of a seat's stream owed as the bridge reads it: the last event applied (its stream +// sequence), the events waiting and those delivered and not yet acknowledged, and the oldest of them. +type Backlog struct { + Stream string `json:"stream"` + Consumer string `json:"consumer"` + LastApplied uint64 `json:"last-applied"` + Waiting uint64 `json:"waiting"` + Held uint64 `json:"held"` + Oldest time.Time `json:"oldest,omitzero"` + OldestSecs int64 `json:"oldest-secs,omitempty"` + Error string `json:"error,omitempty"` +} + +// OwedAsk is a value of owed-asks: what was asked (of the act Of said under ID, for again and drop), why, who and when, +// and once the bridge answered, when, what it came to (the outcome said as owed.ended) and its words; a backlog ask is +// answered with the consumers' backlog. +type OwedAsk struct { + Kind string `json:"kind"` + Of string `json:"of,omitempty"` + ID string `json:"id,omitempty"` + Why string `json:"why,omitempty"` + By string `json:"by,omitempty"` + At time.Time `json:"at"` + Answered time.Time `json:"answered,omitzero"` + Outcome string `json:"outcome,omitempty"` + Answer string `json:"answer,omitempty"` + Backlog []Backlog `json:"backlog,omitempty"` +} + +// Done says the bridge answered it. +func (a OwedAsk) Done() bool { return !a.Answered.IsZero() } + +// NewOwedAsk is an ask's key and value: a backlog ask under the key backlog, an act's under its kind, its seat and its id +// (unpadded base64url, as an id may hold what a key may not). A drop says why. +func NewOwedAsk(kind, of, id, why, by string, at time.Time) (string, []byte, error) { + a := OwedAsk{Kind: kind, Of: of, ID: id, Why: strings.TrimSpace(why), By: by, At: at.UTC()} + key := kind + switch kind { + case OwedBacklog: + a.Of, a.ID = "", "" + case OwedAgain, OwedDrop: + if of == "" || id == "" { + return "", nil, fmt.Errorf("an owed act is named by the seat that said it and its id") + } + if kind == OwedDrop && a.Why == "" { + return "", nil, fmt.Errorf("a drop says why") + } + key = kind + "." + of + "." + base64.RawURLEncoding.EncodeToString([]byte(id)) + default: + return "", nil, fmt.Errorf("no owed ask %q: again, drop or backlog", kind) + } + v, err := json.Marshal(a) + return key, v, err +} + +// DecodeOwedAsk reads a value of owed-asks. +func DecodeOwedAsk(data []byte) (OwedAsk, error) { + var a OwedAsk + if err := json.Unmarshal(data, &a); err != nil { + return a, err + } + switch a.Kind { + case OwedAgain, OwedDrop, OwedBacklog: + return a, nil + } + return a, fmt.Errorf("no owed ask %q", a.Kind) +} diff --git a/modules/mesh-delivery-forge-bridge/manifest_test.go b/modules/mesh-delivery-forge-bridge/manifest_test.go index 7a786eaa..034778b3 100644 --- a/modules/mesh-delivery-forge-bridge/manifest_test.go +++ b/modules/mesh-delivery-forge-bridge/manifest_test.go @@ -27,6 +27,14 @@ type state struct { History int `json:"history"` } +// kept is an entry of requires in its long form: kept data on the bus (novox/hq ADR 0595). +type kept struct { + Name string `json:"name"` + Storage string `json:"storage"` + Template string `json:"template"` + Options map[string]int `json:"options"` +} + type artifact struct { Name string `json:"name"` Kind string `json:"kind"` @@ -46,6 +54,7 @@ type manifest struct { Consumes []string `json:"consumes"` Streams []stream `json:"streams"` State []state `json:"state"` + Requires []kept `json:"requires"` Reads []string `json:"reads"` Tools []string `json:"tools"` Invokes []string `json:"invokes"` @@ -87,20 +96,22 @@ func TestManifestDeclarations(t *testing.T) { // 1235/AC1 if m.Module != "mesh-delivery-forge-bridge" { t.Errorf("module = %q", m.Module) } - if !reflect.DeepEqual(m.Streams, []stream{{Name: "forge", Events: []string{"forge.**"}}}) { + if !reflect.DeepEqual(m.Streams, []stream{{Name: "forge", Events: []string{"forge.**", "owed.ended"}}}) { t.Errorf("streams = %+v", m.Streams) } - // The checker's owed is the forge module's to write (ADR 0521, R4e finding R2-1): the bridge neither hears nor reads it. - // The forge's events are the seat git's (design 47 *The seat git*); forge.delete: a branch's end, said as a + // What the gate and the checker owe the forge is each seat's event owed, heard on its own stream (design 47 *What the + // gate owes the forge*), never read from their state. The forge's events are the seat git's (design 47 *The seat git*); forge.delete: a branch's end, said as a // branch-deleted kind (design 51, DLV-FB1). - if want := []string{"git.forge-delete", "git.pull-closed", "git.pull-merged", "git.pull-updated"}; !reflect.DeepEqual(sorted(m.Consumes), want) { + if want := []string{"git.forge-delete", "git.pull-closed", "git.pull-merged", "git.pull-updated", "mesh-delivery-checker.owed", "mesh-delivery.owed"}; !reflect.DeepEqual(sorted(m.Consumes), want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } - if want := []string{"mesh-delivery-gate.merges", "mesh-delivery-gate.owed"}; !reflect.DeepEqual(sorted(m.Reads), want) { - t.Errorf("reads = %v, want %v", m.Reads, want) + // A group is read from the seat mesh-delivery's verb groups (design 47, acceptance 49): the bridge reads no state of + // another module's. + if len(m.Reads) != 0 { + t.Errorf("reads = %v, want none", m.Reads) } - if want := []string{"gitea.gitea_add_comment", "gitea.gitea_commit_status", "gitea.gitea_delivery_view", "gitea.gitea_merge_pull_request", - "gitea.gitea_note_append", "seat:git.file", "seat:git.open-pulls", "seat:git.pull", "seat:git.tags", "seat:git.trees"}; !reflect.DeepEqual(sorted(m.Invokes), want) { + if want := []string{"seat:git.comment", "seat:git.file", "seat:git.merge-pull", "seat:git.note", "seat:git.open-pulls", + "seat:git.pull", "seat:git.status", "seat:git.tags", "seat:git.trees", "seat:git.view", "seat:mesh-delivery.groups", "seat:mesh-hand-acts.hand-act"}; !reflect.DeepEqual(sorted(m.Invokes), want) { t.Errorf("invokes = %v, want %v", m.Invokes, want) } if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" { @@ -109,19 +120,22 @@ func TestManifestDeclarations(t *testing.T) { // 1235/AC1 if m.SecretsOwner != "mesh-delivery-forge-bridge" { t.Errorf("secrets-owner = %q", m.SecretsOwner) } - // Three buckets: the head last written on FORGE for each pull request, so a sync writes only what differs (ADR 0443 - // §5), the revision of each key of the gate's buckets the bridge performed, so a key is performed once per - // revision (ADR 0437 §8; issue 1341), and what a merge read of each module directory by its tree, so a merge reads - // the forge only for the directories it changed (issue 1126). - if !reflect.DeepEqual(m.State, []state{{Name: "heads", History: 1}, {Name: "performed", History: 1}, {Name: "inputs", History: 1}, {Name: "end-asks", History: 1}}) { // 1279/AC6, 1341, 1126, 1585 - t.Errorf("state = %+v, want the buckets heads, performed, inputs and end-asks", m.State) + // The head last written on FORGE for each pull request, so a sync writes only what differs (ADR 0443 §5), and what a + // merge read of each module directory by its tree, so a merge reads the forge only for the directories it changed + // (issue 1126). What it performed of what is owed is kept data in the new form, one key per act, kept 7 days as the + // owners' streams owed are; its size is the template's own (design 47 *What the gate owes the forge*, ADR 0595 §3). + if !reflect.DeepEqual(m.State, []state{{Name: "heads", History: 1}, {Name: "inputs", History: 1}, {Name: "end-asks", History: 1}, {Name: "owed-asks", History: 1}}) { // 1279/AC6, 1126, 1585 + t.Errorf("state = %+v, want the buckets heads, inputs, end-asks and owed-asks", m.State) + } + if want := []kept{{Name: "performed", Storage: "key-value", Template: "state", Options: map[string]int{"history": 1, "ttl-seconds": 604800}}}; !reflect.DeepEqual(m.Requires, want) { + t.Errorf("requires = %+v, want performed as %+v", m.Requires, want) } // 1585/T18: the tools read the groups and ask the bridge to say a group's end again, through its state end-asks. The tool bridge_merged_message answers a merge's merged message again (issue 1585); bridge_tags reads a package's tags for the builder's floor (design 51). - if want := []string{"bridge_group_end_again", "bridge_groups", "bridge_merged_message", "bridge_tags"}; !reflect.DeepEqual(sorted(m.Tools), want) { + if want := []string{"bridge_group_end_again", "bridge_groups", "bridge_merged_message", "bridge_owed", "bridge_owed_again", "bridge_owed_drop", "bridge_tags"}; !reflect.DeepEqual(sorted(m.Tools), want) { t.Errorf("tools = %v, want %v", m.Tools, want) } - // No other state, no bus, no seat, no event (ADR 0437 §4; ADR 0521: no check.status). - for _, k := range []string{"logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "seat", "seats"} { + // No other state, no seat, no event (ADR 0437 §4; ADR 0521: no check.status); its kept data on the bus is the one entry of requires above. + for _, k := range []string{"logs", "emits", "claims", "provides", "uses", "defines-seats", "seat", "seats"} { if _, ok := top[k]; ok { t.Errorf("the manifest declares %q: the bridge has none", k) } diff --git a/modules/mesh-delivery-forge-bridge/module.json b/modules/mesh-delivery-forge-bridge/module.json index 0105502f..c26d1d23 100644 --- a/modules/mesh-delivery-forge-bridge/module.json +++ b/modules/mesh-delivery-forge-bridge/module.json @@ -6,39 +6,50 @@ "git.pull-updated", "git.pull-merged", "git.pull-closed", - "git.forge-delete" + "git.forge-delete", + "mesh-delivery.owed", + "mesh-delivery-checker.owed" ], "streams": [ - { "name": "forge", "events": ["forge.**"] } + { "name": "forge", "events": ["forge.**", "owed.ended"] } ], "state": [ { "name": "heads", "history": 1 }, - { "name": "performed", "history": 1 }, { "name": "inputs", "history": 1 }, - { "name": "end-asks", "history": 1 } + { "name": "end-asks", "history": 1 }, + { "name": "owed-asks", "history": 1 } ], "tools": [ "bridge_tags", "bridge_groups", "bridge_group_end_again", - "bridge_merged_message" - ], - "reads": [ - "mesh-delivery-gate.merges", - "mesh-delivery-gate.owed" + "bridge_merged_message", + "bridge_owed", + "bridge_owed_again", + "bridge_owed_drop" ], "invokes": [ - "gitea.gitea_merge_pull_request", - "gitea.gitea_commit_status", - "gitea.gitea_delivery_view", - "gitea.gitea_note_append", - "gitea.gitea_add_comment", + "seat:git.status", + "seat:git.comment", + "seat:git.view", + "seat:git.note", + "seat:git.merge-pull", + "seat:mesh-delivery.groups", + "seat:mesh-hand-acts.hand-act", "seat:git.open-pulls", "seat:git.trees", "seat:git.file", "seat:git.pull", "seat:git.tags" ], + "requires": [ + { + "name": "performed", + "storage": "key-value", + "template": "state", + "options": { "history": 1, "ttl-seconds": 604800 } + } + ], "own-secrets": { "broker": "${dir:state}/broker" }, From 91a9c4129056bcd4bdd038074f72eccd5b657ecd Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 11:55:02 +0100 Subject: [PATCH 11/33] gofmt --- modules/mesh-delivery-assigner/manifest_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/mesh-delivery-assigner/manifest_test.go b/modules/mesh-delivery-assigner/manifest_test.go index 036dbc34..1db630c4 100644 --- a/modules/mesh-delivery-assigner/manifest_test.go +++ b/modules/mesh-delivery-assigner/manifest_test.go @@ -127,7 +127,7 @@ func TestManifest(t *testing.T) { t.Errorf("consumes = %v, want the indexer's views of graph, manifests and modules, the teller's ask-ended and the engines' reads changed", m.Consumes) } if want := []string{"mesh-delivery-indexer.graph", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", - "seat:mesh-catalog.registrations", "seat:mesh-catalog.seat-definitions", // ADR 0636 §4: the seat list + "seat:mesh-catalog.registrations", "seat:mesh-catalog.seat-definitions", // ADR 0636 §4: the seat list "seat:mesh-hand-acts.hand-act", "seat:mesh-hand-acts.hand-acts", "seat:mesh-hand-acts.warranted", // ADR 0565 §9, §11: the seat alone "seat:node-backup.backed-up", "seat:node-backup.now", "seat:node-engine.manifest-reads", // ADR 0618 §3 From 5fcd76a9f936f867eda88f92bdd7e81aec515671 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 11:56:50 +0100 Subject: [PATCH 12/33] WM7: the watchman raises record-open 24 hours after a merge unless another condition covers it, and store-filling from the seats' filling, full and room MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §148, §160, §181, §218, §263 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-watchman/consumes.go | 22 +++ .../internal/watch/pipelines_test.go | 66 ++++++++ .../internal/watch/records.go | 149 ++++++++++++++++++ .../internal/watch/signals.go | 10 ++ .../internal/watch/watch.go | 4 +- .../mesh-delivery-watchman/manifest_test.go | 2 +- modules/mesh-delivery-watchman/module.json | 14 ++ 7 files changed, 265 insertions(+), 2 deletions(-) create mode 100644 modules/mesh-delivery-watchman/internal/watch/records.go diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/consumes.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/consumes.go index f2ccce58..a0c405d4 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/consumes.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/consumes.go @@ -44,6 +44,7 @@ func consumed(w *watch.Watchman, subject string, body []byte, logf func(string, } w.Accepted(a) case pipelineSubject(w, subject, body): + case recordSubject(w, subject, body): case brokerSubject(subject) && strings.HasSuffix(subject, ".event."+eventCreated): if s, err := watch.ReadBrokerSaid(body); err != nil { logf("%v; left", err) @@ -160,3 +161,24 @@ func pipelineSubject(w *watch.Watchman, subject string, body []byte) bool { } return true } + +// recordSubject hands the seats' `filling`, `full` and `room` of a store and the seat mesh-delivery's `record-opened` and +// `record-ended` to the watchman (watch/records.go, design 47 G §148, §181). It says whether the subject was one of them. +func recordSubject(w *watch.Watchman, subject string, body []byte) bool { + parts := strings.Split(subject, ".") + if len(parts) != 5 || parts[0] != "mesh" || parts[1] != "seat" || parts[3] != "event" { + return false + } + seat, event := parts[2], parts[4] + switch { + case event == meshdelivery.EventFilling || event == meshdelivery.EventFull || event == meshdelivery.EventRoom: + w.StoreFill(seat, event, body) + case seat == meshdelivery.Seat && event == meshdelivery.EventRecordOpened: + w.RecordOpen(body) + case seat == meshdelivery.Seat && event == meshdelivery.EventRecordEnded: + w.RecordEnded(body) + default: + return false + } + return true +} diff --git a/modules/mesh-delivery-watchman/internal/watch/pipelines_test.go b/modules/mesh-delivery-watchman/internal/watch/pipelines_test.go index c9cb3b38..b384cafd 100644 --- a/modules/mesh-delivery-watchman/internal/watch/pipelines_test.go +++ b/modules/mesh-delivery-watchman/internal/watch/pipelines_test.go @@ -137,3 +137,69 @@ func TestAMachinesPresenceIsSaidAsEvents(t *testing.T) { t.Fatalf("said %v, want %v", got, want) } } + +func fillBody(store string, bytes int64) []byte { + b, _ := json.Marshal(meshdelivery.FillingEvent{Store: store, Bytes: bytes, Cap: 100}) + return b +} + +// design 47 G §181, §263: a store past its share is `store-filling` naming the part and the store, full is urgent, `room` and a +// fill under 70 % clear it. +func TestAStoreFillingIsAFinding(t *testing.T) { + w, o, _ := setup() + key := FindingKey(ScopeDelivery, "mesh-delivery-records", KindStoreFilling) + w.StoreFill("mesh-delivery", "filling", fillBody("records", 60)) + if len(o.findings) != 0 { + t.Fatalf("under 70 %%: %v", o.findings) + } + w.StoreFill("mesh-delivery", "filling", fillBody("records", 85)) + if o.findings[key].Severity != Warning { + t.Fatalf("filling: %v", o.findings) + } + w.StoreFill("mesh-delivery", "full", fillBody("records", 100)) + if o.findings[key].Severity != Urgent { + t.Fatalf("full: %v", o.findings) + } + w.StoreFill("mesh-delivery", "room", fillBody("records", 60)) + if len(o.findings) != 0 { + t.Fatalf("room: %v", o.findings) + } +} + +func recordBody(id string, at time.Time) []byte { + b, _ := json.Marshal(meshdelivery.RecordOpenedEvent{Record: id, Milestone: "47/G", At: &at}) + return b +} + +// design 47 G §148, §160: a record open 24 hours is `record-open`, cleared by `record-ended`; a record that ends in time +// raises none; while a store is full, which already tells why, it raises none. +func TestARecordOpenADayIsAFinding(t *testing.T) { + w, o, c := setup() + pk := signalKey(recordRow, "r1") + w.RecordOpen(recordBody("r1", c.t)) + if o.ttls[pk] != RecordOpenBound || len(o.findings) != 0 { + t.Fatalf("within the bound: %v %v", o.ttls, o.findings) + } + expire(w, c, pk, RecordOpenBound) + key := FindingKey(ScopeDelivery, "r1", KindRecordOpen) + if _, ok := o.findings[key]; !ok { + t.Fatalf("past 24 hours: %v", o.findings) + } + w.StoreFill("mesh-delivery", "full", fillBody("records", 100)) + if _, ok := o.findings[key]; ok { + t.Fatalf("a full store covers the record: %v", o.findings) + } + w.StoreFill("mesh-delivery", "room", fillBody("records", 10)) + if _, ok := o.findings[key]; !ok { + t.Fatalf("room, and the record stands open again: %v", o.findings) + } + w.RecordEnded(recordBody("r1", c.t)) + if len(o.findings) != 0 { + t.Fatalf("record-ended: %v", o.findings) + } + w.RecordOpen(recordBody("r2", c.t)) + w.RecordEnded(recordBody("r2", c.t)) + if _, ok := o.pulses[signalKey(recordRow, "r2")]; ok { + t.Fatal("the pulse of an ended record stands") + } +} diff --git a/modules/mesh-delivery-watchman/internal/watch/records.go b/modules/mesh-delivery-watchman/internal/watch/records.go new file mode 100644 index 00000000..f51e2bfa --- /dev/null +++ b/modules/mesh-delivery-watchman/internal/watch/records.go @@ -0,0 +1,149 @@ +package watch + +import ( + "encoding/json" + "fmt" + "time" + + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" +) + +// What the seats say of the delivery's records and stores, heard as the seats' events (design 47 G §148, §181, §218; ADR 0596 +// §2), never read from their state: +// - `filling`, `full` and `room` of a store against its cap: `store-filling` while it stands past its cap's share, urgent +// when it is full, cleared by `room` or by a `filling` under FillingFrom; +// - `record-opened` and `record-ended` of a delivery record: `record-open` when it has stood open for 24 hours, unless +// another condition already tells why (`acting-part-late`, or a store `full`). +// Each is a row of the signals table: its pulse is its bound, written at the event, so a restart loses none. + +// The kinds of these findings. +const ( + KindActingPartLate = "acting-part-late" + KindStoreFilling = "store-filling" + KindRecordOpen = "record-open" +) + +// The rows of the signals table of these findings, which a restart keeps. +const ( + storeRow = "store-fill" + recordRow = "record-open" +) + +// uncovered is the While of a record-open signal: the finding is written at the bound only while no other condition covers +// the record. +const uncovered = "uncovered:record-open" + +// RecordOpenBound is how long a delivery record may stay open before it is a finding (design 47 G §148). +const RecordOpenBound = 24 * time.Hour + +// FillingFrom is the share of a store's cap under which a `filling` clears its finding (design 47 G §181). +const FillingFrom = 0.70 + +// StoreFill hears a seat's `filling`, `full` or `room` of one of its stores. A `filling` is a warning, naming the part and the +// store, unless its share is under FillingFrom, which clears it; `full` is urgent, naming what it refuses; `room` clears either. +func (w *Watchman) StoreFill(seat, event string, body []byte) { + var f meshdelivery.FillingEvent + if err := json.Unmarshal(body, &f); err != nil || f.Store == "" { + w.say("the %s of the seat %s does not read, left: %v", event, seat, err) + return + } + id := seat + "-" + f.Store + key := FindingKey(ScopeDelivery, id, KindStoreFilling) + w.mu.Lock() + defer w.mu.Unlock() + share := 0.0 + if f.Cap > 0 { + share = float64(f.Bytes) / float64(f.Cap) + } + since := w.now() + if f.At != nil && !f.At.IsZero() { + since = *f.At + } + switch { + case event == "room" || (event == "filling" && share < FillingFrom): + w.unfind(key) + case event == "full": + w.find(Finding{Scope: ScopeDelivery, ID: id, Kind: KindStoreFilling, Severity: Urgent, Source: storeRow, Since: since, + Summary: fmt.Sprintf("%s's store %s is full (%d of %d bytes)", seat, f.Store, f.Bytes, f.Cap), + Said: "full", Headline: "A store of the delivery is full", + Explanation: "The store " + f.Store + " of the seat " + seat + " is at its cap, so it refuses " + orWords(f.Refused, "what it is asked to keep") + + " until room is made.", + Needs: "room made in the store (a removal, or its cap raised)"}) + default: + w.find(Finding{Scope: ScopeDelivery, ID: id, Kind: KindStoreFilling, Severity: Warning, Source: storeRow, Since: since, + Summary: fmt.Sprintf("%s's store %s is at %.0f %% of its cap (%d of %d bytes)", seat, f.Store, share*100, f.Bytes, f.Cap), + Said: "filling", Headline: "A store of the delivery is filling", + Explanation: "The store " + f.Store + " of the seat " + seat + " is nearly full; once it is full it refuses new entries.", + Needs: "room made before it is full"}) + } + w.rejudgeRecords() +} + +// RecordOpen hears the seat mesh-delivery's `record-opened`: its finding is due when the record has stood open for 24 hours. +func (w *Watchman) RecordOpen(body []byte) { + var r meshdelivery.RecordOpenedEvent + if err := json.Unmarshal(body, &r); err != nil || r.Record == "" { + w.say("a record-opened does not read, left: %v", err) + return + } + w.mu.Lock() + defer w.mu.Unlock() + since := w.now() + if r.At != nil && !r.At.IsZero() { + since = *r.At + } + f := Finding{Scope: ScopeDelivery, ID: r.Record, Kind: KindRecordOpen, Severity: Warning, Source: recordRow, Since: since, + Summary: fmt.Sprintf("the delivery record of %s has stood open since %s", r.Milestone, stamp(since)), + Said: "open for " + RecordOpenBound.String(), Headline: "A delivery is not on every machine yet", + Explanation: "The delivery record of " + r.Milestone + " has stood open for " + RecordOpenBound.String() + + ": a node does not run what it changed, or a step never answered.", + Needs: "what it waits for (gate_record names it), or the record ended by hand if a node is gone for good (gate_record_end)"} + w.unsignal(recordRow, r.Record) + w.unfind(f.Key()) + if left := RecordOpenBound - w.now().Sub(since); left > 0 { + w.signal(recordRow, r.Record, left, signalDue{Do: dueFind, While: uncovered, Finding: f}) + return + } + w.opened[r.Record] = f + w.rejudgeRecords() +} + +// RecordEnded hears the seat mesh-delivery's `record-ended`: its pulse and its finding end. +func (w *Watchman) RecordEnded(body []byte) { + var r meshdelivery.RecordEndedEvent + if err := json.Unmarshal(body, &r); err != nil || r.Record == "" { + w.say("a record-ended does not read, left: %v", err) + return + } + w.mu.Lock() + defer w.mu.Unlock() + w.unsignal(recordRow, r.Record) + delete(w.opened, r.Record) + w.unfind(FindingKey(ScopeDelivery, r.Record, KindRecordOpen)) +} + +// recordCovered says another condition tells the operator why a record is open: a wait past its bound or a full store. +func (w *Watchman) recordCovered() bool { + for _, v := range w.findings { + f, err := DecodeFinding(v) + if err != nil { + continue + } + if f.Kind == KindActingPartLate || (f.Kind == KindStoreFilling && f.Severity == Urgent) { + return true + } + } + return false +} + +// rejudgeRecords writes the finding of each record past its bound that nothing covers, and ends it when something does. +func (w *Watchman) rejudgeRecords() { + for id, f := range w.opened { + if w.recordCovered() { + w.unfind(f.Key()) + continue + } + w.find(f) + _ = id + } +} diff --git a/modules/mesh-delivery-watchman/internal/watch/signals.go b/modules/mesh-delivery-watchman/internal/watch/signals.go index 2d9e7913..4829dc6f 100644 --- a/modules/mesh-delivery-watchman/internal/watch/signals.go +++ b/modules/mesh-delivery-watchman/internal/watch/signals.go @@ -171,6 +171,11 @@ func (w *Watchman) signalExpired(pk string) { case dueEnd: w.unfind(due.Finding.Key()) case dueFind: + if due.While == uncovered { + w.opened[due.Finding.ID] = due.Finding + w.rejudgeRecords() + return + } if due.While != "" && !w.stillWrong(due.While, due.Finding.Kind) { return } @@ -181,6 +186,9 @@ func (w *Watchman) signalExpired(pk string) { // stillWrong says a have's key still says what a row's finding was waited on: failed for apply-failing, unhealthy for // unhealthy. func (w *Watchman) stillWrong(haveKey, kind string) bool { + if haveKey == uncovered { + return !w.recordCovered() + } if node, ok := strings.CutPrefix(haveKey, "silent:"); ok { _, open := w.findings[FindingKey(ScopeMachine, node, KindSilent)] return open @@ -936,6 +944,8 @@ var SignalRows = []struct{ Row, Signal, Bound, Kind string }{ {"S21", "a machine's have failed or its provisioner client's files not placed", ApplyEvery.String(), KindApplyFailing}, {"health", "a have unhealthy in two writes in a row; urgent after " + UnhealthyUrgentAfter.String(), "two writes", KindUnhealthy}, {"take-in", "a source revision or a delivery group the indexer holds back (ADR 0633 §5)", TakeInRevisionBound.String() + " a revision, " + TakeInGroupBound.String() + " a group", KindTakeInWaits}, + {storeRow, "a store of the delivery's parts filling or full (the seats' filling, full and room)", "filling from " + fmt.Sprintf("%.0f", FillingFrom*100) + " %, full urgent", KindStoreFilling}, + {recordRow, "a delivery record open (the seat mesh-delivery's record-opened, ended by record-ended)", RecordOpenBound.String(), KindRecordOpen}, {"provisions", "a provision of mesh-provisioner in waits (ADR 0570 §7)", "at once", KindNeedsOperator}, } diff --git a/modules/mesh-delivery-watchman/internal/watch/watch.go b/modules/mesh-delivery-watchman/internal/watch/watch.go index 9a05ef6e..88fb281e 100644 --- a/modules/mesh-delivery-watchman/internal/watch/watch.go +++ b/modules/mesh-delivery-watchman/internal/watch/watch.go @@ -120,6 +120,8 @@ type Watchman struct { queued map[string]map[string]time.Time // awaySaid is each machine whose machine-away was said and not yet its machine-back, with when it was last heard (presence.go). awaySaid map[string]time.Time + // opened is each record past its bound and not ended, with its finding (records.go). + opened map[string]Finding } // New is a watchman writing to out, reading the time from now. @@ -130,7 +132,7 @@ func New(out Out, now func() time.Time) *Watchman { haveState: map[string]nodeengine.HaveByNodeModuleValue{}, unhealthy: map[string]int{}, advised: map[string]int{}, advisedSince: map[string]time.Time{}, control: map[string]bool{}, failedSince: map[string]time.Time{}, provFailed: map[string]map[string]ProvReport{}, closures: map[string]map[string]time.Time{}, waiting: map[string]map[string]string{}, health: map[string]nodeengine.HealthByNodeValue{}, tunnels: map[string]string{}, - standing: map[string]map[string]bool{}, readWhole: map[string]bool{}, awaited: map[string]bool{}, heads: map[string]head{}, queued: map[string]map[string]time.Time{}, awaySaid: map[string]time.Time{}} + standing: map[string]map[string]bool{}, readWhole: map[string]bool{}, awaited: map[string]bool{}, heads: map[string]head{}, queued: map[string]map[string]time.Time{}, awaySaid: map[string]time.Time{}, opened: map[string]Finding{}} } // Logf sets where a write that failed is said; it is silent until set. diff --git a/modules/mesh-delivery-watchman/manifest_test.go b/modules/mesh-delivery-watchman/manifest_test.go index 826e749b..4926b296 100644 --- a/modules/mesh-delivery-watchman/manifest_test.go +++ b/modules/mesh-delivery-watchman/manifest_test.go @@ -111,7 +111,7 @@ func TestManifestStateAndReads(t *testing.T) { // 1135/AC6 t.Errorf("invokes = %v, want %v", m.Invokes, want) } // DLV-M5 its first views, 1585/T20 the indexer's take-in events, and DLV-W1 (ADR 0595 §4): the accepted verdicts and the bus module's created and refused, durably. - if want := []string{"mesh-delivery-assigner.views.wants", "mesh-delivery-indexer.take-in.pending", "mesh-delivery-indexer.take-in.taken", "mesh-delivery-indexer.views.modules", "mesh-catalog.manifest-accepted", "mesh-delivery.source-moved", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.step-started", "mesh-pipeline-runner.step-ended", "mesh-broker.created", "mesh-broker.refused"}; !reflect.DeepEqual(m.Consumes, want) { + if want := []string{"mesh-delivery-assigner.views.wants", "mesh-delivery-indexer.take-in.pending", "mesh-delivery-indexer.take-in.taken", "mesh-delivery-indexer.views.modules", "mesh-catalog.manifest-accepted", "mesh-delivery.source-moved", "mesh-delivery.filling", "mesh-delivery.full", "mesh-delivery.room", "mesh-delivery.record-opened", "mesh-delivery.record-ended", "mesh-catalog.filling", "mesh-catalog.full", "mesh-catalog.room", "mesh-wants.filling", "mesh-wants.full", "mesh-wants.room", "mesh-grants.filling", "mesh-grants.full", "mesh-grants.room", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.step-started", "mesh-pipeline-runner.step-ended", "mesh-broker.created", "mesh-broker.refused"}; !reflect.DeepEqual(m.Consumes, want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } // 1952/WM1: the watchman holds the seat mesh-watchdogs at mesh scope, its first seat (novox/hq ADR 0564 §1, §4). diff --git a/modules/mesh-delivery-watchman/module.json b/modules/mesh-delivery-watchman/module.json index 35bf7289..08b23559 100644 --- a/modules/mesh-delivery-watchman/module.json +++ b/modules/mesh-delivery-watchman/module.json @@ -37,6 +37,20 @@ "mesh-delivery-indexer.views.modules", "mesh-catalog.manifest-accepted", "mesh-delivery.source-moved", + "mesh-delivery.filling", + "mesh-delivery.full", + "mesh-delivery.room", + "mesh-delivery.record-opened", + "mesh-delivery.record-ended", + "mesh-catalog.filling", + "mesh-catalog.full", + "mesh-catalog.room", + "mesh-wants.filling", + "mesh-wants.full", + "mesh-wants.room", + "mesh-grants.filling", + "mesh-grants.full", + "mesh-grants.room", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.step-started", "mesh-pipeline-runner.step-ended", From b1fcabcebe549e550546bd803fadef16889fe22c Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 11:57:53 +0100 Subject: [PATCH 13/33] AS3: a placement's branch and build, set by a proposal, bound by the ask, and its build a held put MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A proposal sets a module's watched branch and build (a re-place keeps them), the changes digest an ask binds to carries both, and a placement's build is a held put of that build, released when the placement drops it while a person's put stays. Taken over from 6bc6fe53 and b8e50fff. Builds: design 47 G §14, §78, §114, §360, §361 Tests: TestAssignments (place change branch, build, digest rows), aPlacementsBuildIsPut. Test 77 (§79) covered by the put row. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-assigner/pass_test.go | 58 +++++++++++ .../mesh-delivery-assigner/placementbuild.go | 95 +++++++++++++++++++ .../cmd/mesh-delivery-assigner/puts.go | 1 + .../internal/assignment/assignments_test.go | 18 ++++ .../internal/assignment/digest.go | 8 +- .../internal/assignment/proposal.go | 10 +- 6 files changed, 187 insertions(+), 3 deletions(-) create mode 100644 modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/placementbuild.go diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go index 0bc60e9a..5e8e1ce4 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go @@ -272,6 +272,7 @@ func TestAnAnswerCountsOnlyForTheAskingItAnswered(t *testing.T) { t.Error("an answer to an ask taken away was taken") } t.Run("a put of the bus acts on the operator's warrant alone", putOfTheBus) + t.Run("a placement's build is put held, and released when the placement names it no more", aPlacementsBuildIsPut) // ADR 0614 §4: the rules read what each node's engine states it reads from its record in `.node`; a node with no // record, or one that states nothing, is read as stating nothing (the rules then read the baseline). t.Run("the rules read what each node's engine reads", func(t *testing.T) { @@ -467,6 +468,63 @@ func (b *fakeBus) Emit(eventType string, body any) error { return nil } +// Design 47 line 912 and line 914 ("a placement's `build` delivers the earlier build"; ADR 0666 §24, B-K1): a placed +// module's build becomes a held put of that build, which then goes a put's steps; a placement naming no build any more +// releases the put it made, never a person's. +func aPlacementsBuildIsPut(t *testing.T) { + ctx := context.Background() + putsKV := &memKV{values: map[string][]byte{}} + p := newPasser(kvSet{buckets.Puts: putsKV}, time.Date(2026, 11, 10, 18, 0, 0, 0, time.UTC)) + earlier := meshcatalogclient.SourcePrefix2 + strings.Repeat("e", 64) + place := func(build string) { + values, err := mmac.AssignmentsValues(mma.Assignments{Node: "g14", Modules: []mma.PlacedModule{{Module: "web", Build: build}, {Module: "db"}}}) + if err != nil { + t.Fatal(err) + } + for key, raw := range values { + p.w.putEntry(plc.AssignmentsBucket, key, raw, uint64(len(raw)), time.Now()) + } + } + read := func() (puts.Put, bool) { + raw, ok := putsKV.values["g14.web"] + if !ok { + return puts.Put{}, false + } + got, err := puts.Decode(raw) + if err != nil { + t.Fatal(err) + } + return got, true + } + place(earlier) + p.placementPuts(ctx) + got, ok := read() + if !ok || got.FP != earlier || !got.Hold || got.By != placementBy || got.Step != puts.Asked { + t.Fatalf("the placement's build made the put %+v (there %v)", got, ok) + } + if _, there := putsKV.values["g14.db"]; there { + t.Error("a placed module naming no build was put") + } + p.placementPuts(ctx) // again: the same build makes nothing new + if again, _ := read(); !again.At.Equal(got.At) || again.Step != puts.Asked { + t.Errorf("a second pass made the put again: %+v", again) + } + place("") + p.placementPuts(ctx) + if got, _ := read(); got.Step != puts.Released || !strings.Contains(got.Words, placementBy) { + t.Errorf("the placement naming no build left the put %+v", got) + } + // A person's put stands while the placement names no build. + person := puts.Put{FP: earlier, Hold: true, Why: "a try", By: "lead@shanks", At: time.Now()}.Said(time.Now(), puts.Holding, "held") + raw, _ := puts.Encode(person) + putsKV.values["g14.web"] = raw + p.w.put(buckets.Puts, "g14.web", raw) + p.placementPuts(ctx) + if got, _ := read(); got.Step != puts.Holding || got.By != "lead@shanks" { + t.Errorf("a person's put was released by the placement: %+v", got) + } +} + // ADR 0546, the first put (2026-10-25, want.novox.nats): the tool writes the put; a pass waits for the composition and grant // the controller's take-in writes; once they are there it asks the operator with a yes that approves; only on that warrant does it have // the streams snapshotted, then writes the want held and records the warrant; the have running it healthy holds it; a release diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/placementbuild.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/placementbuild.go new file mode 100644 index 00000000..3fb5d39c --- /dev/null +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/placementbuild.go @@ -0,0 +1,95 @@ +package main + +import ( + "context" + "fmt" + "sort" + + "mesh-delivery-assigner/internal/buckets" + "mesh-delivery-assigner/internal/puts" +) + +// placementBy is who a put the placement's build makes names as its asker: the placement, whose proposal the operator +// approved, is the warrant (design 47 *Going back is one act*; ADR 0666 §24). +const placementBy = "the placement" + +// placementPuts makes each placed module's `build` a held put of that build, and releases a put the placement made once its +// placement names that build no more (design 47 line 912: "a return to an earlier build is one value in the placement, the +// module's `build` in its placement key, set by a placement proposal"; ADR 0666 §24; B-K1). The put then goes its own steps +// (the builder's build, the composition, an ask where the module's policy needs one, the want, holding) and is read with +// `assigner_puts` and released with `assigner_release` like any other; a placement's build the builder never built is refused +// by its build step, said in the put's words. A put asked by a person stands until the placement names another build. +func (p *passer) placementPuts(ctx context.Context) { + kv := p.kvs[buckets.Puts] + if kv == nil { + return + } + now := p.clock().UTC() + named := map[string]string{} // put key -> the build its placement names + for node, a := range p.w.assignments() { + for _, m := range a.Modules { + if m.Build == "" { + continue + } + if key, err := puts.Key(node, m.Module); err == nil { + named[key] = m.Build + } + } + } + held := p.w.read(buckets.Puts) + keys := make([]string, 0, len(named)+len(held)) + for k := range named { + keys = append(keys, k) + } + for k := range held { + if _, there := named[k]; !there { + keys = append(keys, k) + } + } + sort.Strings(keys) + for _, key := range keys { + build, names := named[key] + var before *puts.Put + if raw, ok := held[key]; ok { + if got, err := puts.Decode(raw); err == nil { + before = &got + } + } + var next puts.Put + switch { + case names && before != nil && before.FP == build: + continue // made already, or a person's put of the same build: it goes its own steps, ended or not + case names: + words := fmt.Sprintf("asked by %s: it names build %s of %s", placementBy, shortFP(build), key) + if before != nil && !puts.Ended(before.Step) { + words += fmt.Sprintf("; it replaces the put of %s, which stood at %s", shortFP(before.FP), before.Step) + } + next = puts.Put{FP: build, Hold: true, Why: "the placement names this build (ADR 0666 §24)", By: placementBy, At: now}.Said(now, puts.Asked, words) + case before != nil && before.By == placementBy && !puts.Ended(before.Step): + next = before.Said(now, puts.Released, fmt.Sprintf("released by %s: it names no build of its own any more; the assigner's rules write the want again", placementBy)) + default: + continue + } + data, err := puts.Encode(next) + if err != nil { + p.logf("the placement's put %s is not written: %v", key, err) + continue + } + if before == nil { + _, err = kv.Create(ctx, key, data) + } else if e, gerr := kv.Get(ctx, key); gerr != nil { + err = gerr + } else { + _, err = kv.Update(ctx, key, data, e.Revision()) + } + if err != nil { + p.logf("the placement's put %s could not be written, and is written at the next pass: %v", key, err) + continue + } + p.w.put(buckets.Puts, key, data) + p.logf("put %s: %s: %s", key, next.Step, next.Words) + if p.acts != nil { + p.acts.say(key, next) + } + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go index 929ecbdb..77ccfaf2 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go @@ -105,6 +105,7 @@ func (p *passer) puts(ctx context.Context) { if p.acts == nil { return } + p.placementPuts(ctx) all := p.putsByModule() if len(all) == 0 { return diff --git a/modules/mesh-delivery-assigner/internal/assignment/assignments_test.go b/modules/mesh-delivery-assigner/internal/assignment/assignments_test.go index 13789020..33a30427 100644 --- a/modules/mesh-delivery-assigner/internal/assignment/assignments_test.go +++ b/modules/mesh-delivery-assigner/internal/assignment/assignments_test.go @@ -129,6 +129,24 @@ func TestAssignments(t *testing.T) { if !reflect.DeepEqual(client.Assigned(current["g14"]), []string{"zsh", "app"}) { t.Error("laying changed the current assignments") } + // Design 47 line 1051 (the Choices table: a module's watched branch and its build in the placement key, set by a + // proposal): a place change sets them, and a re-place naming none keeps the module's. + onBeta, f := LayChanges(current, []seat.ProposalChange{{Op: client.OpPlace, Node: "g14", Module: "zsh", Branch: "beta", Build: "b1"}}) + again, g := LayChanges(onBeta, []seat.ProposalChange{{Op: client.OpPlace, Node: "g14", Module: "zsh", Ports: map[string]int64{"80": 8081}}}) + if m := onBeta["g14"].Modules[0]; len(f) != 0 || m.Branch != "beta" || m.Build != "b1" { + t.Errorf("a place change's branch and build: %+v, %v", m, f) + } + if m := again["g14"].Modules[0]; len(g) != 0 || m.Branch != "beta" || m.Build != "b1" || m.Ports["80"] != 8081 { + t.Errorf("a re-place dropped the branch and build: %+v, %v", m, g) + } + if ChangesDigest(seat.Proposal{Changes: []seat.ProposalChange{{Op: client.OpPlace, Node: "g14", Module: "zsh", Branch: "beta"}}}) == + ChangesDigest(seat.Proposal{Changes: []seat.ProposalChange{{Op: client.OpPlace, Node: "g14", Module: "zsh"}}}) { + t.Error("an ask bound to a place change's digest does not bind its branch") + } + if ChangesDigest(seat.Proposal{Changes: []seat.ProposalChange{{Op: client.OpPlace, Node: "g14", Module: "zsh", Build: "b1"}}}) == + ChangesDigest(seat.Proposal{Changes: []seat.ProposalChange{{Op: client.OpPlace, Node: "g14", Module: "zsh"}}}) { + t.Error("an ask bound to a place change's digest does not bind its build (design 47 G 360)") + } if fresh, f := LayChanges(current, []seat.ProposalChange{{Op: client.OpPlace, Node: "pi", Module: "zsh"}}); len(f) != 0 || fresh["pi"].Node != "pi" { t.Errorf("a node with no key: %+v, %v", fresh, f) } diff --git a/modules/mesh-delivery-assigner/internal/assignment/digest.go b/modules/mesh-delivery-assigner/internal/assignment/digest.go index b22e05c6..a7166a1a 100644 --- a/modules/mesh-delivery-assigner/internal/assignment/digest.go +++ b/modules/mesh-delivery-assigner/internal/assignment/digest.go @@ -29,6 +29,8 @@ type placedForm struct { HasLayer bool `json:"has-layer,omitempty"` MeshValues map[string]any `json:"mesh-wide-values,omitempty"` Values map[string]any `json:"values,omitempty"` + Branch string `json:"branch,omitempty"` + Build string `json:"build,omitempty"` } type holdingForm struct { @@ -110,6 +112,8 @@ type changeForm struct { Whole *assignmentsForm `json:"whole,omitempty"` Decided *decidedForm `json:"decided,omitempty"` Keys *joiningForm `json:"keys,omitempty"` + Branch string `json:"branch,omitempty"` + Build string `json:"build,omitempty"` } func providerOf(p seat.Provider) providerForm { return providerForm{Node: p.Node, Module: p.Module} } @@ -150,7 +154,7 @@ func assignmentsOf(p seat.Assignments) assignmentsForm { } for _, m := range p.Modules { out.Modules = append(out.Modules, placedForm{Module: m.Module, Ports: m.Ports, Bindings: providersOf(m.Bindings), - MeshWide: m.MeshWide, HasMesh: m.HasMeshWide, Layer: m.Layer, HasLayer: m.HasLayer, MeshValues: m.MeshWideValues, Values: m.Values}) + MeshWide: m.MeshWide, HasMesh: m.HasMeshWide, Layer: m.Layer, HasLayer: m.HasLayer, MeshValues: m.MeshWideValues, Values: m.Values, Branch: m.Branch, Build: m.Build}) } for _, h := range p.Holds { out.Holds = append(out.Holds, holdingForm{Claim: h.Claim, Scope: h.Scope, Module: h.Module, Site: h.Site, Kind: h.Kind}) @@ -161,7 +165,7 @@ func assignmentsOf(p seat.Assignments) assignmentsForm { func changeOf(c seat.ProposalChange) changeForm { out := changeForm{Op: c.Op, Node: c.Node, Module: c.Module, Ports: c.Ports, Bindings: providersOf(c.Bindings), Provision: c.Provision, Claim: c.Claim, Scope: c.Scope, Kind: c.Kind, Site: c.Site, Field: c.Field, MeshWide: c.MeshWide, Layer: c.Layer, - MeshValues: c.MeshWideValues, Values: c.Values} + MeshValues: c.MeshWideValues, Values: c.Values, Branch: c.Branch, Build: c.Build} if c.Provider != nil { p := providerOf(*c.Provider) out.Provider = &p diff --git a/modules/mesh-delivery-assigner/internal/assignment/proposal.go b/modules/mesh-delivery-assigner/internal/assignment/proposal.go index b66cf164..1680e598 100644 --- a/modules/mesh-delivery-assigner/internal/assignment/proposal.go +++ b/modules/mesh-delivery-assigner/internal/assignment/proposal.go @@ -148,11 +148,19 @@ func LayChanges(current map[string]seat.Assignments, changes []seat.ProposalChan } continue case client.OpPlace: - m := seat.PlacedModule{Module: c.Module, Ports: c.Ports, Bindings: c.Bindings} + // A module's watched branch and its build are in the placement key, set by a proposal (design 47 line 1051, + // the Choices table's row): the change's when it names them, else kept from the module placed there. + m := seat.PlacedModule{Module: c.Module, Ports: c.Ports, Bindings: c.Bindings, Branch: c.Branch, Build: c.Build} if at >= 0 { o := p.Modules[at] m.MeshWide, m.HasMeshWide, m.Layer, m.HasLayer = o.MeshWide, o.HasMeshWide, o.Layer, o.HasLayer m.MeshWideValues, m.Values = o.MeshWideValues, o.Values + if m.Branch == "" { + m.Branch = o.Branch + } + if m.Build == "" { + m.Build = o.Build + } p.Modules[at] = m } else { p.Modules = append(p.Modules, m) From cbbb971d35938c5208dc19fe583de66a1ee1c786 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 11:58:33 +0100 Subject: [PATCH 14/33] WM12: the watchman's manifest says its version as MAJOR.MINOR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §368, §372 (0.1 for a module with none published) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- modules/mesh-delivery-watchman/manifest_test.go | 5 +++++ modules/mesh-delivery-watchman/module.json | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/modules/mesh-delivery-watchman/manifest_test.go b/modules/mesh-delivery-watchman/manifest_test.go index 4926b296..e8d37103 100644 --- a/modules/mesh-delivery-watchman/manifest_test.go +++ b/modules/mesh-delivery-watchman/manifest_test.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "reflect" + "regexp" "slices" "strings" "testing" @@ -98,6 +99,10 @@ func TestManifestStateAndReads(t *testing.T) { // 1135/AC6 "mesh-delivery-builder.builds", "mesh-host.health", "mesh-provisioner.provisions", "node-engine.have"}) { t.Errorf("reads = %v", reads) } + // Design 47 G §368: the version is MAJOR.MINOR and nothing else. + if !regexp.MustCompile(`^[0-9]+\.[0-9]+$`).MatchString(m.Version) { + t.Errorf("version = %q, want MAJOR.MINOR", m.Version) + } // DLV-M5 (ADR 0615 §1): the assigner's wants and the indexer's modules are their owners' states, asked by their read // tools and followed by their views families, never read. for _, r := range m.Reads { diff --git a/modules/mesh-delivery-watchman/module.json b/modules/mesh-delivery-watchman/module.json index 08b23559..bd081f2f 100644 --- a/modules/mesh-delivery-watchman/module.json +++ b/modules/mesh-delivery-watchman/module.json @@ -1,6 +1,6 @@ { "module": "mesh-delivery-watchman", - "version": "1", + "version": "0.1", "slug": "watchman", "claims": [ { From ec14823429efee2b5a1e28393a314b04ebd8b2d7 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 12:05:11 +0100 Subject: [PATCH 15/33] AS15, AS16: a provider counts healthy by the node-engine seat's running, and a put ends on it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The assigner hears running.. through its durable consumer on the seat's stream (last message per subject), counts a provider healthy while its running build is not stopped whatever a newer want of it waits for, and ends a put when running names the put's fingerprint (a running without one, or stopped, ends none); the have no longer ends a put. With the stream not heard, rule 5 reads the have as before. Builds: design 47 G §199, §309; main file "A provider before its consumers" and "Until milestone G, from the have; from G, from running" Tests: TestAProviderRunsHealthyByItsRunning, the put rows of TestTheBusMovesByTheRulesAndItsUserListRevisionIsWrittenAtOnce, TestAnAnswerCountsOnlyForTheAskingItAnswered (running ends the put) Manifest: consumes node-engine.running; validator: no finding new against trunk. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-assigner/main.go | 1 + .../cmd/mesh-delivery-assigner/pass_test.go | 17 +++- .../cmd/mesh-delivery-assigner/puts.go | 3 + .../cmd/mesh-delivery-assigner/running.go | 88 +++++++++++++++++++ .../cmd/mesh-delivery-assigner/world.go | 8 +- .../internal/assign/put.go | 32 +++---- .../internal/assign/wants.go | 17 +++- .../internal/assign/wants_test.go | 47 ++++++++-- .../mesh-delivery-assigner/manifest_test.go | 4 +- modules/mesh-delivery-assigner/module.json | 3 +- 10 files changed, 188 insertions(+), 32 deletions(-) create mode 100644 modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/running.go diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index 3ed9dab2..1af26f74 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -503,6 +503,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, } } hearReadsChanged(ctx, js, hub, reads, findings, gate.eventsLive, logger.Printf) + hearRunning(ctx, js, w, wake, findings, logger.Printf) } if bus != nil && kvs[buckets.Puts] != nil { p.acts = newPutActs(bus, wake, logger.Printf) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go index 5e8e1ce4..716ac50c 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go @@ -651,8 +651,21 @@ func putOfTheBus(t *testing.T) { t.Errorf("the warrant was recorded as %q", call) } world(buckets.Wants, "want.novox.nats", wantsKV.values["want.novox.nats"]) - running, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Composed: digest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}) - world(buckets.Have, hkey, running) + // Design 47 G §309: the put ends on the node-engine seat's running naming its fingerprint, never on the have. + haveRunning, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Composed: digest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}) + world(buckets.Have, hkey, haveRunning) + if got := step(); got.Step != puts.Wanted { + t.Fatalf("the have runs it and no running says so, the put stands at %s: %s", got.Step, got.Words) + } + if err := p.w.hearRunning("mesh.seat.node-engine.event.running.novox.nats", []byte(`{"v":1,"at":"2026-11-10T20:00:00Z","node":"novox","module":"nats"}`)); err != nil { + t.Fatal(err) + } + if got := step(); got.Step != puts.Wanted { + t.Fatalf("a running without a fingerprint ends none, the put stands at %s: %s", got.Step, got.Words) + } + if err := p.w.hearRunning("mesh.seat.node-engine.event.running.novox.nats", []byte(`{"v":1,"at":"2026-11-10T20:01:00Z","node":"novox","module":"nats","fingerprint":"`+fp+`"}`)); err != nil { + t.Fatal(err) + } if got := step(); got.Step != puts.Holding { t.Fatalf("running healthy, the put stands at %s: %s", got.Step, got.Words) } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go index 77ccfaf2..0322e53c 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go @@ -221,6 +221,9 @@ func (p *passer) putWorld(node, module string, put puts.Put, manifest catalogue. } } } + if r, ok := p.w.runningOf(node, module); ok { + w.Running = &r + } if hkey, err := nodeengineclient.HaveKey(node, module); err == nil { if raw := p.w.value(buckets.Have, hkey); raw != nil { if h, err := nodeengineclient.DecodeHave(raw); err == nil { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/running.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/running.go new file mode 100644 index 00000000..1aa6d079 --- /dev/null +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/running.go @@ -0,0 +1,88 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + "git.novox.be/novox/mesh-sdk/go/process" + + "mesh-delivery-assigner/internal/assign" + "mesh-delivery-assigner/internal/assignment" +) + +// What each node runs is heard, never asked (design 47 G §199, §309): the node-engine seat's event `running..` is +// kept last per subject on its own stream, and the assigner's durable consumer on it starts from the last message of every +// subject, so a node away still has its last word here. The assigner counts a provider healthy by it (rule 5) and ends a put on it. + +// hearRunning keeps each `running` the stream holds and each it says from now, and wakes the pass on every one. +func hearRunning(ctx context.Context, js jetstream.JetStream, w *world, wake func(), findings assign.Findings, logf func(string, ...any)) { + acct := process.Account{Node: thisNode, Module: assignment.Module} + nh := notHeardWriter(findings, logf) + cc, err := acct.HearSeatEvents(ctx, js, nodeengine.StreamRunning, func(m jetstream.Msg) { + if err := m.Ack(); err != nil { + logf("acknowledging %s: %v", m.Subject(), err) + } + if err := w.hearRunning(m.Subject(), m.Data()); err != nil { + logf("%v", err) + return + } + wake() + }, logf) + if err != nil { + nh.NotHeard(nodeengine.StreamRunning, "what each node runs is not heard until the assigner starts again, so a provider is counted healthy by its have and a put does not end", err) + return + } + w.mu.Lock() + w.runningHeard = true + w.mu.Unlock() + nh.Heard(nodeengine.StreamRunning) + go func() { + <-ctx.Done() + cc.Stop() + }() +} + +// hearRunning keeps one `running` event said on subject. +func (w *world) hearRunning(subject string, data []byte) error { + var r nodeengine.Running + if err := json.Unmarshal(data, &r); err != nil { + return fmt.Errorf("%s holds what is not a running: %w", subject, err) + } + if r.Node == "" || r.Module == "" { + return fmt.Errorf("%s names no node or no module, so it is kept for none", subject) + } + w.mu.Lock() + w.running[r.Node+"."+r.Module] = r + w.mu.Unlock() + return nil +} + +// runningOf is the last `running` heard of module on node. +func (w *world) runningOf(node, module string) (nodeengine.Running, bool) { + w.mu.Lock() + defer w.mu.Unlock() + r, ok := w.running[node+"."+module] + return r, ok +} + +// runningNow is every `running` heard, by node and module; nil while the seat's events are not heard (the rules then read the +// have). +func (w *world) runningNow() map[string]map[string]nodeengine.Running { + w.mu.Lock() + defer w.mu.Unlock() + if !w.runningHeard { + return nil + } + out := map[string]map[string]nodeengine.Running{} + for _, r := range w.running { + if out[r.Node] == nil { + out[r.Node] = map[string]nodeengine.Running{} + } + out[r.Node][r.Module] = r + } + return out +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go index a5a1ca9a..be881eb6 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go @@ -52,6 +52,10 @@ type world struct { // sourceGroups is, by revision of the gate's source.main, the release (group) that moved it, from every revision its // history keeps (watched with history): a build's source revision names the release that carries it (ADR 0601 §1). sourceGroups map[uint64]string + // running is, by `.`, the last `running` the node-engine seat said of it, kept from its stream by the assigner's + // durable consumer (design 47 G §199, §309); runningHeard says that consumer is bound. + running map[string]nodeengine.Running + runningHeard bool } // entryMeta is a value's revision in its bucket and when it was written. @@ -70,7 +74,7 @@ type parsed struct { func newWorld() *world { return &world{values: map[string]map[string][]byte{}, manifests: map[string]parsed{}, healthy: map[string][]string{}, changes: map[string]uint64{}, meta: map[string]entryMeta{}, trunkBuilt: map[string]builds.BuildLogEntry{}, assignmentKeys: mmac.NewKeys(), - sourceGroups: map[uint64]string{}} + sourceGroups: map[uint64]string{}, running: map[string]nodeengine.Running{}} } // putEntry is put, keeping the value's revision and the time it was written. @@ -391,7 +395,7 @@ func (w *world) wantsOf() map[string]map[string]mwc.Want { // nodes is what the six rules read of the nodes: presence, haves, and every provision's providers with their nodes. func (w *world) nodes(manifests map[string]catalogue.Manifest, assigned map[string][]string) assign.Nodes { - n := assign.Nodes{Online: map[string]bool{}, Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{}, Providers: map[string]map[string][]string{}, + n := assign.Nodes{Online: map[string]bool{}, Running: w.runningNow(), Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{}, Providers: map[string]map[string][]string{}, Reads: map[string][]string{}} for key, value := range w.read(buckets.Presence) { if node, online, ok := presenceOf(key, value); ok { diff --git a/modules/mesh-delivery-assigner/internal/assign/put.go b/modules/mesh-delivery-assigner/internal/assign/put.go index 34d7ae9b..eb139832 100644 --- a/modules/mesh-delivery-assigner/internal/assign/put.go +++ b/modules/mesh-delivery-assigner/internal/assign/put.go @@ -2,7 +2,6 @@ package assign import ( "fmt" - "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" mwc "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshwantsclient" @@ -50,7 +49,10 @@ type PutWorld struct { SnapshotFailed string // Have is what the node says it runs of the module; nil when it says nothing. Have *nodeengine.HaveByNodeModuleValue - Now time.Time + // Running is the node-engine seat's last `running` of the module on the node; nil when it said none (design 47 G §309). A put + // ends on it, never on the have. + Running *nodeengine.Running + Now time.Time } // PutNext is what a step decides: the put as it stands after it, and the acts the pass does for it. @@ -154,26 +156,18 @@ func StepPut(w PutWorld) PutNext { return next } -// applied is a wanted put once the node's have says how it went: holding or done when it runs the composition healthy. +// applied is a wanted put once the node says how it went: holding or done when the node-engine seat's `running` names the put's +// fingerprint (design 47, *Compositions, wants and the apply*: from G a put ends on `running..` naming its +// fingerprint, never on the have). A `running` without a fingerprint, or one that says the module stopped, ends none. The have +// still says a failed health check in the put's words. func applied(w PutWorld, p puts.Put) puts.Put { - h := w.Have - if h == nil || h.Fp != p.FP { - return p - } - // A put without a hold ends on the fingerprint alone: a node that composes its own digests no composition of the put's, and - // a put that waits for that one pins the module's want for good (issue 1966). A held put keeps its composition. - if !p.Hold && h.Phase == nodeengineclient.PhaseRunning && h.Health == nodeengineclient.HealthOK { + if r := w.Running; r != nil && !r.Stopped && r.Fingerprint != "" && r.Fingerprint == p.FP { + if p.Hold { + return p.Said(w.Now, puts.Holding, fmt.Sprintf("%s runs %s at %s healthy; held there until released", w.Node, w.Module, short(p.FP))) + } return p.Said(w.Now, puts.Done, fmt.Sprintf("%s runs %s at %s healthy; the rules write its want again", w.Node, w.Module, short(p.FP))) } - if h.Composed != p.Composed { - return p - } - switch { - case h.Phase == nodeengineclient.PhaseRunning && h.Health == nodeengineclient.HealthOK && p.Hold: - return p.Said(w.Now, puts.Holding, fmt.Sprintf("%s runs %s at %s healthy; held there until released", w.Node, w.Module, short(p.FP))) - case h.Phase == nodeengineclient.PhaseRunning && h.Health == nodeengineclient.HealthOK: - return p.Said(w.Now, puts.Done, fmt.Sprintf("%s runs %s at %s healthy; the rules write its want again", w.Node, w.Module, short(p.FP))) - case failed(*h): + if h := w.Have; h != nil && h.Fp == p.FP && h.Composed == p.Composed && failed(*h) { return p.Said(w.Now, p.Step, fmt.Sprintf("%s runs %s at %s and its health check failed: %s", w.Node, w.Module, short(p.FP), h.Detail)) } return p diff --git a/modules/mesh-delivery-assigner/internal/assign/wants.go b/modules/mesh-delivery-assigner/internal/assign/wants.go index 6e7845cd..63a15400 100644 --- a/modules/mesh-delivery-assigner/internal/assign/wants.go +++ b/modules/mesh-delivery-assigner/internal/assign/wants.go @@ -109,6 +109,10 @@ func (m Module) composition(node, fp string) (ComposedDigest, bool) { type Nodes struct { Online map[string]bool Haves map[string]map[string]nodeengine.HaveByNodeModuleValue + // Running is, by node and module, the last `running` the node-engine seat said (design 47 G §309): a provider runs healthy + // while the build it runs does, as this says it, whatever a newer want of it waits for (rule 5, main file *A provider + // before its consumers*). Nil: the seat's events are not heard, and the have says it. + Running map[string]map[string]nodeengine.Running // Providers is, by provision, the modules that provide it, with the nodes each is assigned to. Providers map[string]map[string][]string // Reads is, by node, the reader capabilities its node-engine states in its node record (nodeengine.NodeRecord.ManifestReads, @@ -276,6 +280,17 @@ func (m Module) MissingProvisions(node string) []string { return out } +// runsHealthy says whether module runs healthy on node: from G the node-engine seat's last `running` of it names a build and +// does not say it stopped; with the seat's events not heard, the have's phase and health. +func (n Nodes) runsHealthy(node, module string) bool { + if n.Running != nil { + r, ok := n.Running[node][module] + return ok && !r.Stopped && r.Fingerprint != "" + } + h, ok := n.Haves[node][module] + return ok && h.Phase == nodeengineclient.PhaseRunning && h.Health == nodeengineclient.HealthOK +} + // providedHealthy is rule 5 for one provision: some module providing it runs healthy on the node, or, assigned // elsewhere only, at its canary. A provision no module provides is not a module's to judge (the mesh or the node // offers it) and holds. @@ -295,7 +310,7 @@ func providedHealthy(provision, node string, n Nodes) (bool, string) { if !slices.Contains(nodes, node) { at = Canary(nodes, n.Online) } - if h, ok := n.Haves[at][p]; ok && at != "" && h.Phase == nodeengineclient.PhaseRunning && h.Health == nodeengineclient.HealthOK { + if at != "" && n.runsHealthy(at, p) { return true, "" } } diff --git a/modules/mesh-delivery-assigner/internal/assign/wants_test.go b/modules/mesh-delivery-assigner/internal/assign/wants_test.go index 1c38e3a7..e3d46708 100644 --- a/modules/mesh-delivery-assigner/internal/assign/wants_test.go +++ b/modules/mesh-delivery-assigner/internal/assign/wants_test.go @@ -923,7 +923,7 @@ func stepsOfAPut(t *testing.T) { w := bus() w.Put = w.Put.Said(now, puts.Wanted, "wanted") w.Put.Composed = d.Digest - w.Have = &nodeengine.HaveByNodeModuleValue{Fp: fNew, Composed: d.Digest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} + w.Running = &nodeengine.Running{Node: "novox", Module: "nats", Fingerprint: fNew} return w }, puts.Holding, "held there until released", false, false, false}, {"wanted without hold: done", func() PutWorld { @@ -931,7 +931,7 @@ func stepsOfAPut(t *testing.T) { w.Put.Hold = false w.Put = w.Put.Said(now, puts.Wanted, "wanted") w.Put.Composed = d.Digest - w.Have = &nodeengine.HaveByNodeModuleValue{Fp: fNew, Composed: d.Digest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} + w.Running = &nodeengine.Running{Node: "novox", Module: "nats", Fingerprint: fNew} return w }, puts.Done, "the rules write its want again", false, false, false}, {"wanted without hold, the node composes its own: done at the fingerprint", func() PutWorld { @@ -939,14 +939,27 @@ func stepsOfAPut(t *testing.T) { w.Put.Hold = false w.Put = w.Put.Said(now, puts.Wanted, "wanted") w.Put.Composed = d.Digest - w.Have = &nodeengine.HaveByNodeModuleValue{Fp: fNew, Composed: dig(fOld), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} + w.Running = &nodeengine.Running{Node: "novox", Module: "nats", Fingerprint: fNew} return w }, puts.Done, "the rules write its want again", false, false, false}, - {"wanted with hold, the node composes its own: stays wanted", func() PutWorld { + {"wanted, the node's running names no fingerprint: stays wanted (the have runs it, which ends nothing)", func() PutWorld { w := bus() w.Put = w.Put.Said(now, puts.Wanted, "wanted") w.Put.Composed = d.Digest - w.Have = &nodeengine.HaveByNodeModuleValue{Fp: fNew, Composed: dig(fOld), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} + w.Running = &nodeengine.Running{Node: "novox", Module: "nats"} + w.Have = &nodeengine.HaveByNodeModuleValue{Fp: fNew, Composed: d.Digest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} + return w + }, puts.Wanted, "wanted", false, false, false}, + {"wanted, the node's running says it stopped: stays wanted", func() PutWorld { + w := bus() + w.Put = w.Put.Said(now, puts.Wanted, "wanted") + w.Running = &nodeengine.Running{Node: "novox", Module: "nats", Fingerprint: fNew, Stopped: true} + return w + }, puts.Wanted, "wanted", false, false, false}, + {"wanted, the node's running names another build: stays wanted", func() PutWorld { + w := bus() + w.Put = w.Put.Said(now, puts.Wanted, "wanted") + w.Running = &nodeengine.Running{Node: "novox", Module: "nats", Fingerprint: fOld} return w }, puts.Wanted, "wanted", false, false, false}, {"released: nothing more", func() PutWorld { w := bus(); w.Put = w.Put.Said(now, puts.Released, "released"); return w }, puts.Released, "released", false, false, false}, @@ -1546,3 +1559,27 @@ func TestAHeldModuleWithNoWantIsNotOnNeitherList(t *testing.T) { } } } + +// Main file *A provider before its consumers* (design 47 G §309): from G a provider runs healthy while the build it runs is +// healthy, as the node-engine seat's `running` says it, whatever a newer want of it waits for (its have's phase waiting-yes). +func TestAProviderRunsHealthyByItsRunning(t *testing.T) { + waiting := nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseWaitingYes, Health: nodeengineclient.HealthOK} + for _, c := range []struct { + name string + running map[string]map[string]nodeengine.Running + ok bool + }{ + {"its new want waits for a yes, its old build runs", map[string]map[string]nodeengine.Running{"novox": {"nats": {Node: "novox", Module: "nats", Fingerprint: fOld}}}, true}, + {"its running build is stopped", map[string]map[string]nodeengine.Running{"novox": {"nats": {Node: "novox", Module: "nats", Stopped: true}}}, false}, + {"it said no running", map[string]map[string]nodeengine.Running{"novox": {}}, false}, + {"its running names no build", map[string]map[string]nodeengine.Running{"novox": {"nats": {Node: "novox", Module: "nats"}}}, false}, + {"the seat's events are not heard: the have says it, and waiting-yes is not running", nil, false}, + } { + n := Nodes{Online: map[string]bool{"novox": true}, Running: c.running, + Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{"novox": {"nats": waiting}}, + Providers: map[string]map[string][]string{"bus-state": {"nats": {"novox"}}}} + if ok, words := providedHealthy("bus-state", "novox", n); ok != c.ok { + t.Errorf("%s: healthy = %v (%s), want %v", c.name, ok, words, c.ok) + } + } +} diff --git a/modules/mesh-delivery-assigner/manifest_test.go b/modules/mesh-delivery-assigner/manifest_test.go index 1db630c4..ba1b4796 100644 --- a/modules/mesh-delivery-assigner/manifest_test.go +++ b/modules/mesh-delivery-assigner/manifest_test.go @@ -123,8 +123,8 @@ func TestManifest(t *testing.T) { } // ADR 0618 §4: and it hears each node's engine say its reads changed; ADR 0636 §4 (issue 1966): and the seat list's two events; // issue 1966 note 151: and the teller's ask-ended, its subject the one the assigner hears. - if !slices.Equal(m.Consumes, []string{"mesh-catalog.registration-changed", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", "mesh-delivery-indexer.views.graph", "mesh-delivery-indexer.views.manifests", "mesh-delivery-indexer.views.modules", buckets.TellerModule + "." + tellerevents.AskEnded, "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.group-ended", "node-engine.manifest-reads-changed"}) { - t.Errorf("consumes = %v, want the indexer's views of graph, manifests and modules, the teller's ask-ended and the engines' reads changed", m.Consumes) + if !slices.Equal(m.Consumes, []string{"mesh-catalog.registration-changed", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", "mesh-delivery-indexer.views.graph", "mesh-delivery-indexer.views.manifests", "mesh-delivery-indexer.views.modules", buckets.TellerModule + "." + tellerevents.AskEnded, "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.group-ended", "node-engine.manifest-reads-changed", "node-engine.running"}) { + t.Errorf("consumes = %v, want the indexer's views of graph, manifests and modules, the teller's ask-ended, the engines' reads changed and what they run", m.Consumes) } if want := []string{"mesh-delivery-indexer.graph", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "seat:mesh-catalog.registrations", "seat:mesh-catalog.seat-definitions", // ADR 0636 §4: the seat list diff --git a/modules/mesh-delivery-assigner/module.json b/modules/mesh-delivery-assigner/module.json index d6252329..e69ae235 100644 --- a/modules/mesh-delivery-assigner/module.json +++ b/modules/mesh-delivery-assigner/module.json @@ -48,7 +48,8 @@ "mesh-delivery-teller.ask-ended", "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.group-ended", - "node-engine.manifest-reads-changed" + "node-engine.manifest-reads-changed", + "node-engine.running" ], "invokes": [ "mesh-delivery-indexer.graph", From 190fdef8b3ed4691e68a5061bdc9af6d29e3745a Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 12:07:39 +0100 Subject: [PATCH 16/33] AS19 (the seat list): the assigner wires its seat list reader through mesh-sdk's seatlist.Part MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Its hand-wired seatGate goes; the reader, its tools assigner_seat_list and assigner_seat_list_again and the re-read when the events are live are the SDK's Part. A test searches the code for a wiring of its own. Left of AS19: the indexer's views and the gate's reads, which wait for AS4's catalog half. Builds: design 47 G §204 Test: TestTheSeatListIsWiredByTheSDKAlone Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-assigner/main.go | 8 +- .../cmd/mesh-delivery-assigner/pass.go | 9 +- .../cmd/mesh-delivery-assigner/seat.go | 4 +- .../cmd/mesh-delivery-assigner/seat_tools.go | 5 +- .../mesh-delivery-assigner/seat_tools_test.go | 2 +- .../cmd/mesh-delivery-assigner/seatlist.go | 100 +----------------- .../mesh-delivery-assigner/seatlist_test.go | 36 ++++++- .../cmd/mesh-delivery-assigner/tools.go | 6 +- 8 files changed, 55 insertions(+), 115 deletions(-) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index 1af26f74..72177123 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -425,9 +425,9 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, hub := process.NewEventHub(logger.Printf) // The seat list (seatlist.go), read over the connection and heard on the hub, given to catalogue at every change, and a // pass made then: registered before the hub starts. - gate := &seatGate{} + gate := seatlist.NewPart("assigner", seatListAgainCallers) if bus != nil { - gate.follow(seatlist.ProcessBus(bus.Call, hub), wake, logger.Printf) + gate.Follow(seatlist.ProcessBus(bus.Call, hub), wake, logger.Printf) } // The indexer's manifests, modules and module graph, through its first views and views family, never its buckets // (DLV-F2r, ADR 0631): the first pass waits for the three answers, as it waits for every watch to hand what stands. @@ -502,7 +502,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, logger.Printf("a pipeline's end or a registration is not heard, so the wants keep the trunk-only form until the assigner starts again: %v", err) } } - hearReadsChanged(ctx, js, hub, reads, findings, gate.eventsLive, logger.Printf) + hearReadsChanged(ctx, js, hub, reads, findings, gate.EventsLive, logger.Printf) hearRunning(ctx, js, w, wake, findings, logger.Printf) } if bus != nil && kvs[buckets.Puts] != nil { @@ -603,7 +603,7 @@ func hearReadsChanged(ctx context.Context, js jetstream.JetStream, hub *process. } w.Heard(process.EventsStream) if live != nil { - live() // the seat list, unread while the seat was silent, is read now (seatGate.eventsLive) + live() // the seat list, unread while the seat was silent, is read now (seatlist.Part.EventsLive) } go func() { <-ctx.Done() diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go index 8c5d6aa2..ef0ccc4f 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go @@ -18,6 +18,8 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate" "git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go/overlay" "git.novox.be/novox/mesh-sdk/go/motion" + "git.novox.be/novox/mesh-sdk/go/seatlist" + "sync" oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" "git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient" @@ -79,7 +81,8 @@ type passer struct { seat *seatActs // seats is the seat list the process reads (seatlist.go): until it is read, seat-holders is not written and a proposal // laid again is not ended on a seat it does not name (ADR 0636 §1). Nil reads nothing. - seats *seatGate + seats *seatlist.Part + seatWaitSaid sync.Once publish publisher saidProposed map[string]bool // copied says the retired state was copied into `assignments` by this process, or found empty (copyRetired). @@ -1248,11 +1251,11 @@ func (p *passer) seatHolders(ctx context.Context) { if len(assignments) == 0 { return // no assignments read yet: nothing is known of who holds what, and nothing is deleted } - if !p.seats.read() { + if !p.seats.Heard() { // ADR 0636 §1: which claim names which seat is the list's, so nothing is written from the bootstrap alone; the // reader's change makes a pass. if p.seats != nil { - p.seats.waitSaid.Do(func() { p.logf("seat-holders is not written until the seat list is read: %s", p.seats.health()) }) + p.seatWaitSaid.Do(func() { p.logf("seat-holders is not written until the seat list is read: %s", p.seats.Health()) }) } return } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat.go index dfb5d04d..8efd714f 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat.go @@ -440,10 +440,10 @@ func (p *passer) mustAnswer(ctx context.Context, prop mma.Proposal, rev uint64, func (p *passer) layAgain(ctx context.Context, prop mma.Proposal, rev uint64, current map[string]mma.Assignments, revs map[string]uint64, why string) { held, recorded := p.lays(ctx) again, laid, findings := assign.Propose(assign.ProposeIn{ID: prop.ID, By: prop.By, Why: prop.Why, At: prop.At, Changes: prop.Changes, - Current: current, Revisions: revs, Held: held, Recorded: recorded, SeatListRead: p.seats.read()}) + Current: current, Revisions: revs, Held: held, Recorded: recorded, SeatListRead: p.seats.Heard()}) if slices.ContainsFunc(findings, plc.WaitsForSeatList) { // ADR 0636 §1: nothing is ended on a list not read; the reader's change makes a pass, which lays it again. - p.logf("the proposal %s is not laid again yet: %s", prop.ID, p.seats.health()) + p.logf("the proposal %s is not laid again yet: %s", prop.ID, p.seats.Health()) return } if len(findings) > 0 { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go index 6d63a9f9..e459a792 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go @@ -19,6 +19,7 @@ import ( mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient" mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/seatlist" "git.novox.be/novox/mesh-sdk/go/views" "mesh-delivery-assigner/internal/assign" @@ -86,7 +87,7 @@ type seatTools struct { now func() time.Time // seats is the seat list the bundle reads (seatlist.go): until it is read, propose judges no mesh seat the bootstrapped // definitions do not name as held once (ADR 0636 §1). Nil reads nothing. - seatList *seatGate + seatList *seatlist.Part } func newSeatTools(st seatStore, ask func(string, any) (json.RawMessage, error)) *seatTools { @@ -265,7 +266,7 @@ func (s *seatTools) doPropose(args map[string]any, by string) (any, error) { enrollers := plc.EnrollersOf(current) now := s.now() p, laid, findings := assign.Propose(assign.ProposeIn{ID: plc.NewProposalID(now), By: by, Why: arg(args, "why"), At: now, - Changes: changes, Current: current, Revisions: revs, Held: held, Recorded: recorded, Warrant: warrant, SeatListRead: s.seatList.read()}) + Changes: changes, Current: current, Revisions: revs, Held: held, Recorded: recorded, Warrant: warrant, SeatListRead: s.seatList.Heard()}) if refusal := plc.AddNodeRefusal(p, moduleOfCaller(by, enrollers), enrollers); refusal != "" { findings = append(findings, refusal) } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go index 704f820c..c884c495 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go @@ -257,7 +257,7 @@ func TestTheSeatsVerbs(t *testing.T) { // Such a session is proved as its machine's tool runner (issue 627); the healers too, no other module. for caller, refused := range map[string]bool{"person.jochen": false, "shanks.node-tools": false, "ace.mesh-healers": false, "ace.messenger": true} { - if err := seatlist.Refusal(seatListAgainTool, seatListAgainCallers, caller); (err != nil) != refused { + if err := seatlist.Refusal("assigner_seat_list_again", seatListAgainCallers, caller); (err != nil) != refused { t.Errorf("%s: refused=%v, err %v", caller, refused, err) } } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seatlist.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seatlist.go index ec80d6e0..61e8d076 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seatlist.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seatlist.go @@ -1,103 +1,13 @@ package main -import ( - "errors" - "sync" - "sync/atomic" +import stdio "git.novox.be/novox/mesh-sdk/go" - stdio "git.novox.be/novox/mesh-sdk/go" - "git.novox.be/novox/mesh-sdk/go/catalogue" - "git.novox.be/novox/mesh-sdk/go/seatlist" -) - -// The assigner reads the seat list as every part does (novox/hq ADR 0635 §1, ADR 0636 §1, §7; issue 1966): through the seat -// mesh-catalog's verb seat-definitions and its two events, from the bootstrapped definitions until the seat has answered. What -// the reader holds is given to catalogue at every change, which the check of a proposal and the seat-holders state read: which -// mesh seat is replicated, and which name is a seat's. Both of its runtimes hold one: the process (the proposals it lays again, -// the state seat-holders) over its connection and the event hub, and the tools' bundle (the seat's verb propose) over the -// runtime. Until the list is read, a mesh seat the bootstrapped definitions do not name is never judged held once +// The assigner reads the seat list as every part does, through mesh-sdk's one wiring (go/seatlist.Part; design 47 G §204): its +// reader over the seat mesh-catalog's verb seat-definitions and its two events, its tools `assigner_seat_list` and +// `assigner_seat_list_again`. What the reader holds is given to catalogue at every change, which the check of a proposal and the +// seat-holders state read; until the list is read, a mesh seat the bootstrapped definitions do not name is never judged held once // (plc.CheckAssignments), and seat-holders is not written. -// Seat list tools (ADR 0636 §7): what the reader holds, and reading it again now, which every operator's session and the -// healers may ask. -const ( - seatListTool = "assigner_seat_list" - seatListAgainTool = "assigner_seat_list_again" -) - // A call through the mesh MCP server is proved only as its machine's tool runner (.node-tools, issue 627), so // node-tools stands for an operator's session until a session is proved (issue 1966). var seatListAgainCallers = []string{stdio.AnyPerson, "node-tools", "mesh-healers"} - -// seatGate is the reader a runtime of the assigner judges by, set once it is started; before, the list counts as unread. -type seatGate struct { - reader atomic.Pointer[seatlist.Reader] - // waitSaid says the journal was told that seat-holders waits for the list, so it is told once per wait. - waitSaid sync.Once -} - -// read says the seat mesh-catalog has answered the reader: the list is read (seatlist.Status's Heard). A nil gate, or one -// whose reader has not started, has read nothing. -func (g *seatGate) read() bool { - if g == nil { - return false - } - r := g.reader.Load() - return r != nil && r.Status().Heard -} - -// health is why the list is not read, or empty. -func (g *seatGate) health() string { - if g == nil { - return "the assigner follows no seat list" - } - r := g.reader.Load() - if r == nil { - return "the seat list reader has not started yet" - } - return r.Status().Health() -} - -// follow starts the reader over bus: what it holds is given to catalogue at once and at every change, and changed is called -// then (the process makes a pass, which lays again what waited). -func (g *seatGate) follow(bus seatlist.Bus, changed func(), logf func(string, ...any)) *seatlist.Reader { - reader, err := seatlist.Start(seatlist.Config{Bus: bus, Bootstrap: seatlist.Bootstrapped(), - OnChange: func(seats []catalogue.Seat) { - catalogue.UseSeats(seats) - if changed != nil { - changed() - } - }}) - if err != nil { - logf("the assigner cannot follow its seat list, so a mesh seat the bootstrapped definitions do not name waits for it: %v", err) - } - if reader != nil { - catalogue.UseSeats(reader.Seats()) - g.reader.Store(reader) - } - return reader -} - -// eventsLive is called once the process's consumer on EVENTS is heard: the seat may have said it speaks before that consumer -// existed, whose events start at the stream's end, so while the list is unread it is read now (ADR 0636 §1). -func (g *seatGate) eventsLive() { - if r := g.reader.Load(); r != nil { - r.ReloadIfUnread() - } -} - -// tools are the reader's two tools over whatever reader the gate holds at the call. -func (g *seatGate) tools() []stdio.Tool { - shape := seatlist.Tools(&seatlist.Reader{}, seatListTool, seatListAgainTool, seatListAgainCallers) - for i := range shape { - run := i - shape[i].Run = func(args map[string]any) (any, error) { - r := g.reader.Load() - if r == nil { - return nil, errors.New("the seat list reader has not started yet: the runtime has not served this module") - } - return seatlist.Tools(r, seatListTool, seatListAgainTool, seatListAgainCallers)[run].Run(args) - } - } - return shape -} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seatlist_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seatlist_test.go index 67f44e2f..8cf119c9 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seatlist_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seatlist_test.go @@ -3,6 +3,9 @@ package main import ( "encoding/json" "errors" + "os" + "path/filepath" + "strings" "sync" "testing" "time" @@ -50,21 +53,44 @@ func (standInCatalog) Subscribe(string, func(string, json.RawMessage) error) err // gateOver is a seat gate following the stand-in, once its first read has ended. A read one gives catalogue the test list // (catalogue.UseSeats), as a host's does, for the tests after it too: the list is the same everywhere. -func gateOver(t *testing.T, silent bool) *seatGate { +func gateOver(t *testing.T, silent bool) *seatlist.Part { t.Helper() if silent { // A silent seat reached no one: its reader asks again within the bus's retry bound (issue 1966), which a test // that waits for it to stay unread keeps short. t.Cleanup(busretry.Waits(20*time.Millisecond, time.Millisecond, 5*time.Millisecond)) } - g := &seatGate{} - r := g.follow(standInCatalog{silent: silent}, nil, t.Logf) + g := seatlist.NewPart("assigner", seatListAgainCallers) + r := g.Follow(standInCatalog{silent: silent}, nil, t.Logf) if r == nil { t.Fatal("the reader did not start") } <-r.Ready() - if g.read() == silent { - t.Fatalf("the gate read %v over a stand-in silent %v: %s", g.read(), silent, g.health()) + if g.Heard() == silent { + t.Fatalf("the gate read %v over a stand-in silent %v: %s", g.Heard(), silent, g.Health()) } return g } + +// Design 47 G §204: the seat list's reader is wired once, by mesh-sdk's go/seatlist.Part; no part wires its own. A search of the +// assigner's code finds no start of a reader and no tools of its own. +func TestTheSeatListIsWiredByTheSDKAlone(t *testing.T) { + files, err := filepath.Glob("*.go") + if err != nil { + t.Fatal(err) + } + for _, f := range files { + if strings.HasSuffix(f, "_test.go") { + continue + } + raw, err := os.ReadFile(f) + if err != nil { + t.Fatal(err) + } + for _, own := range []string{"seatlist.Start(", "seatlist.Tools(", "OnChange:"} { + if strings.Contains(string(raw), own) { + t.Errorf("%s wires the seat list itself (%s), where mesh-sdk's seatlist.Part does it", f, own) + } + } + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go index 9b946428..1c7ec141 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go @@ -386,15 +386,15 @@ func serveTools() error { } }() // The seat list (seatlist.go), read through the runtime once Serve has its channel: the seat's verbs judge by it. - gate := &seatGate{} + gate := seatlist.NewPart("assigner", seatListAgainCallers) go func() { <-stdio.Served() - gate.follow(seatlist.StdioBus(), nil, func(f string, a ...any) { fmt.Fprintf(os.Stderr, "mesh-delivery-assigner: "+f+"\n", a...) }) + gate.Follow(seatlist.StdioBus(), nil, func(f string, a ...any) { fmt.Fprintf(os.Stderr, "mesh-delivery-assigner: "+f+"\n", a...) }) }() seatVerbs := newSeatTools(runtimeStore{}, stdio.Ask) seatVerbs.seatList = gate tools := append(s.tools(), seatVerbs.tools()...) - tools = append(tools, gate.tools()...) + tools = append(tools, gate.Tools()...) wants := newWantsTools(runtimeStore{}, stdio.Ask, func(e, id string, b any) error { // The seat's event, said on the seat: a claimed seat's event is never the module's own. return stdio.SeatPublish(wantsEventSubject(e), b, id) From 21195a06be7c5faf2d18a5d9d70fc476fe21fe5f Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 12:08:09 +0100 Subject: [PATCH 17/33] CK2: the checker takes the trunk heads from the catalog half's revision and catalog-revision, not the gate's source (repairs the checker against the new contracts) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §197, §195, §200 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-checker-tools/main.go | 10 ++-- .../mesh-delivery-checker/integration_test.go | 35 ++++++++---- .../cmd/mesh-delivery-checker/main.go | 26 ++------- .../cmd/mesh-delivery-checker/trunk.go | 57 +++++++++++++++++++ .../cmd/mesh-delivery-checker/trunk_test.go | 38 +++++++++++++ .../internal/check/case.go | 15 ++--- .../internal/check/checker_test.go | 7 +-- .../internal/check/examiner_test.go | 10 +--- .../internal/check/names.go | 1 - .../internal/check/trunkview.go | 28 +++++++++ .../mesh-delivery-checker/manifest_test.go | 8 +-- modules/mesh-delivery-checker/module.json | 3 +- 12 files changed, 171 insertions(+), 67 deletions(-) create mode 100644 modules/mesh-delivery-checker/cmd/mesh-delivery-checker/trunk.go create mode 100644 modules/mesh-delivery-checker/cmd/mesh-delivery-checker/trunk_test.go create mode 100644 modules/mesh-delivery-checker/internal/check/trunkview.go diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go index d32b387c..9d6caafe 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go @@ -14,7 +14,7 @@ import ( "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdeliverychecker" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" stdio "git.novox.be/novox/mesh-sdk/go" @@ -426,14 +426,12 @@ func main() { AskState: read(stdio.State(check.AsksState)), BuildState: read(stdio.State("mesh-delivery-builder.builds"))} r.Runs = runsOf{ownerRead(stdio.Ask, indexerModulesTool)} // the node-engine at the commit the mesh runs, as the checker reads it tries := stdio.State(check.TriesState) - source := stdio.State("mesh-delivery-gate.source") trunks := func() (map[string]string, error) { - e, err := source.Get(meshdeliveryclient.SourceMainKey) - if err != nil || e == nil { + raw, err := stdio.Ask(check.TrunkVerb, meshcatalog.RevisionInput{}) + if err != nil { return nil, err } - s, err := meshdeliveryclient.DecodeSource(e.Value) - return s.Heads, err + return check.TrunkHeads(raw) } r.Trunks = trunks store := stateStore{StateRead: read(tries), create: func(key string, value []byte) error { diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go index 8421e99b..0ddb4046 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go @@ -417,9 +417,11 @@ type bed struct { inputsAnswered atomic.Int32 fake *fakeBuildModules srvURL string - announce []string - mu sync.Mutex - logs saidLog + // trunk is the catalog half's trunk revision the bed's seat answers `revision` with (design 47 G section 197). + trunkNow *atomic.Value + announce []string + mu sync.Mutex + logs saidLog } // saidLog is what the program said, safe for its goroutines and the test's. @@ -536,9 +538,24 @@ func startBed(t *testing.T) *bed { t.Fatal(err) } } + trunkView := &atomic.Value{} + trunkView.Store(map[string]string{}) + for _, subject := range []string{"mesh.seat.mesh-catalog.call.revision", "mesh.seat.mesh-catalog.call.revision.>"} { + if _, err := ctl.Subscribe(subject, func(m *nats.Msg) { + var heads []meshcatalog.Head + for repo, commit := range trunkView.Load().(map[string]string) { + heads = append(heads, meshcatalog.Head{Repository: repo, Commit: commit}) + } + body, _ := json.Marshal(meshcatalog.RevisionAnswer{Revision: meshcatalog.CatalogRevision{Kind: "trunk", Revision: 1, Trunk: heads}}) + reply, _ := json.Marshal(map[string]any{"result": json.RawMessage(body), "node": "ctl"}) + _ = m.Respond(reply) + }); err != nil { + t.Fatal(err) + } + } ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) t.Cleanup(cancel) - b := &bed{ctx: ctx, js: js, buckets: map[string]jetstream.KeyValue{}, tools: map[string]jetstream.KeyValue{}, srvURL: srv.ClientURL()} + b := &bed{trunkNow: trunkView, ctx: ctx, js: js, buckets: map[string]jetstream.KeyValue{}, tools: map[string]jetstream.KeyValue{}, srvURL: srv.ClientURL()} for module, stream := range toolchainStreams { if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: stream, Subjects: []string{"mesh.stream." + module + ".event.toolchain.reported"}}); err != nil { t.Fatal(err) @@ -589,7 +606,7 @@ func startBed(t *testing.T) *bed { } for name, history := range map[string]int{check.CheckerVerdictsBucket: 5, check.CheckerKeptBucket: 1, check.CheckerOwedBucket: 5, check.PreviewsBucket: 5, check.CandidatesBucket: 10, - check.ModulesBucket: 20, check.ManifestsBucket: 1, check.SourceBucket: 64, + check.ModulesBucket: 20, check.ManifestsBucket: 1, check.PresenceBucket: 20, check.HaveBucket: 20, check.TriesBucket: 1, check.TurnsBucket: 5, builds.BuilderBuildsBucket: 1, check.InputsBucket: 1} { kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: uint8(history)}) @@ -631,11 +648,9 @@ func startBed(t *testing.T) *bed { func (b *bed) setTrunk(t *testing.T, heads map[string]string) { t.Helper() - data, err := meshdeliveryclient.EncodeSource(meshdelivery.SourceByMainValue{Group: "g", Heads: heads}) - if err != nil { - t.Fatal(err) - } - if _, err := b.buckets[check.SourceBucket].Put(b.ctx, meshdeliveryclient.SourceMainKey, data); err != nil { + b.trunkNow.Store(heads) + body, _ := json.Marshal(meshcatalog.CatalogRevisionEvent{Kind: "trunk", Branch: "main", Revision: 1, V: 1}) + if _, err := b.js.Publish(b.ctx, meshcatalog.SubjectCatalogRevision("main"), body); err != nil { t.Fatal(err) } } diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go index 9a3a26e6..81433a09 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go @@ -26,7 +26,6 @@ import ( "mesh-delivery-checker/internal/check" "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" @@ -375,26 +374,13 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro }() } } - // The gate's source says the trunk head of each repository, which a pull request is merged onto. Where it is not there - // yet no trunk is known, so nothing is checked. - if kv := process.OptionalBucket(ctx, js, check.SourceBucket, "no trunk is known, so nothing is checked until the checker starts again", logger.Printf); kv != nil { - sw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { - if e.Key() == meshdeliveryclient.SourceMainKey { - trunks.Put(e.Value(), !opOf(e)) - } - }) - if err != nil { - process.NotWatched(logger.Printf, check.SourceBucket, err, "no trunk is known, so nothing is checked until the checker starts again") - } else { - go func() { - errs <- follow(ctx, sw, func(e jetstream.KeyValueEntry) { - if e.Key() == meshdeliveryclient.SourceMainKey && trunks.Put(e.Value(), !opOf(e)) { - ch.TrunkMoved(ctx) // the trunk moved: its statuses go pending, and a verdict for the old trunk is judged again - } - }) - }() - } + // The catalog half says the trunk head of each repository, which a pull request is merged onto (design 47 G §197). Where it + // does not answer yet no trunk is known, so nothing is checked. + firstTrunks, err := followTrunks(hub, caller.Call, trunks, ch.TrunkMoved, logger.Printf) + if err != nil { + return fmt.Errorf("the checker cannot hear the catalog half's revisions: %w", err) } + firstTrunks(ctx) // The gate's merge turns (novox/hq ADR 0548 §3): no check is claimed of a repository an approved merge waits on, and the // delete of its turn claims it. Read before the candidates, so no check starts past a turn written before this start. // Where the gate's bucket is not there yet it is read as empty, and opened again before each claim until it is there diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/trunk.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/trunk.go new file mode 100644 index 00000000..b5b639e1 --- /dev/null +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/trunk.go @@ -0,0 +1,57 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + + "mesh-delivery-checker/internal/check" +) + +// The checker takes the trunk heads a check merges onto from the catalog half (design 47 G §197), never from the gate's state +// `source`: a first view by the seat mesh-catalog's verb `revision`, which carries the trunk head of every repository, then +// the seat's event `catalog-revision` of a trunk revision, each read again by the verb. + +// readTrunks asks the catalog half's newest trunk revision and sets the heads it carries. It says whether they changed. A +// candidate's revision, or an answer that does not read, changes nothing. +func readTrunks(ask func(string, any) (json.RawMessage, error), trunks *check.Trunks) (bool, error) { + raw, err := ask(check.TrunkVerb, meshcatalog.RevisionInput{}) + if err != nil { + return false, fmt.Errorf("the catalog half's revision cannot be read: %w", err) + } + heads, err := check.TrunkHeads(raw) + if err != nil { + return false, err + } + return trunks.Set(heads), nil +} + +// followTrunks reads the first view and then follows `catalog-revision`: a trunk revision's event reads the heads again, and a +// change of them is the trunk moving (statuses go pending, a verdict for the old trunk is judged again). A read that fails is +// said, and made again at the next event. +func followTrunks(hub interface { + Handle(string, func(jetstream.Msg)) error +}, ask func(string, any) (json.RawMessage, error), trunks *check.Trunks, moved func(context.Context), logf func(string, ...any)) (first func(context.Context), err error) { + run := func(ctx context.Context) { + changed, err := readTrunks(ask, trunks) + if err != nil { + logf("no trunk is known until the catalog half answers, so nothing is checked: %v", err) + return + } + if changed { + moved(ctx) + } + } + err = hub.Handle(meshcatalog.SubjectCatalogRevisionAll, func(m jetstream.Msg) { + var e meshcatalog.CatalogRevisionEvent + if json.Unmarshal(m.Data(), &e) == nil && e.Kind == "trunk" { + run(context.Background()) + } + _ = m.Ack() + }) + return run, err +} diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/trunk_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/trunk_test.go new file mode 100644 index 00000000..47929f13 --- /dev/null +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/trunk_test.go @@ -0,0 +1,38 @@ +package main + +import ( + "encoding/json" + "errors" + "testing" + + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + + "mesh-delivery-checker/internal/check" +) + +// design 47 G §197: the trunk heads come from the catalog half's trunk revision, by the seat verb revision, and never from the +// gate's source; a candidate's revision and an unanswered verb change nothing. +func TestTrunkHeadsComeFromTheCatalogHalf(t *testing.T) { + answer := func(kind string, heads ...meshcatalog.Head) func(string, any) (json.RawMessage, error) { + return func(key string, _ any) (json.RawMessage, error) { + if key != "seat:mesh-catalog.revision" { + t.Fatalf("asked %s", key) + } + return json.Marshal(meshcatalog.RevisionAnswer{Revision: meshcatalog.CatalogRevision{Kind: kind, Trunk: heads}}) + } + } + trunks := &check.Trunks{} + h := meshcatalog.Head{Repository: "r", Commit: "c1"} + if changed, err := readTrunks(answer("trunk", h), trunks); err != nil || !changed || trunks.Heads()["r"] != "c1" { + t.Fatalf("first view: %v %v %v", changed, err, trunks.Heads()) + } + if changed, _ := readTrunks(answer("trunk", h), trunks); changed { + t.Fatal("the same heads moved the trunk") + } + if _, err := readTrunks(answer("candidate", meshcatalog.Head{Repository: "r", Commit: "c2"}), trunks); err == nil || trunks.Heads()["r"] != "c1" { + t.Fatalf("a candidate's revision: %v %v", err, trunks.Heads()) + } + if _, err := readTrunks(func(string, any) (json.RawMessage, error) { return nil, errors.New("down") }, trunks); err == nil || trunks.Heads()["r"] != "c1" { + t.Fatalf("an unanswered verb: %v %v", err, trunks.Heads()) + } +} diff --git a/modules/mesh-delivery-checker/internal/check/case.go b/modules/mesh-delivery-checker/internal/check/case.go index 6557b8f6..e6358bc7 100644 --- a/modules/mesh-delivery-checker/internal/check/case.go +++ b/modules/mesh-delivery-checker/internal/check/case.go @@ -234,22 +234,15 @@ func GateHeads(c meshcatalog.CandidatesByGroupValue) string { // sameTrunk says whether two verdicts' trunk heads are the same. func sameTrunk(a, b map[string]string) bool { return maps.Equal(a, b) } -// Trunks holds the trunk head of each repository, as the gate's source.main says (a watch puts it in). +// Trunks holds the trunk head of each repository, as the catalog half's trunk revision says (a follower of the seat mesh-catalog's +// verb revision and its event catalog-revision sets it; design 47 G §197). type Trunks struct { mu sync.Mutex heads map[string]string } -// Put takes a source.main value in and answers whether the heads changed. -func (t *Trunks) Put(value []byte, deleted bool) bool { - var heads map[string]string - if !deleted { - s, err := meshdeliveryclient.DecodeSource(value) - if err != nil { - return false - } - heads = s.Heads - } +// Set takes the trunk heads in and answers whether they changed. +func (t *Trunks) Set(heads map[string]string) bool { t.mu.Lock() defer t.mu.Unlock() if maps.Equal(t.heads, heads) { diff --git a/modules/mesh-delivery-checker/internal/check/checker_test.go b/modules/mesh-delivery-checker/internal/check/checker_test.go index f68c1a7b..694d277b 100644 --- a/modules/mesh-delivery-checker/internal/check/checker_test.go +++ b/modules/mesh-delivery-checker/internal/check/checker_test.go @@ -14,7 +14,6 @@ import ( "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliverycheckerclient" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdeliverychecker" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "git.novox.be/novox/mesh-delivery/sdk/buildstep" @@ -107,11 +106,7 @@ func newRig(t testing.TB, j Judge) *rig { func (r *rig) setTrunks(t testing.TB, heads map[string]string) bool { t.Helper() - data, err := meshdeliveryclient.EncodeSource(meshdelivery.SourceByMainValue{Group: "g", Heads: heads}) - if err != nil { - t.Fatal(err) - } - return r.trunks.Put(data, false) + return r.trunks.Set(heads) } func (r *rig) verdict(t testing.TB, key string) meshdeliverychecker.VerdictsByGroupHeadsValue { diff --git a/modules/mesh-delivery-checker/internal/check/examiner_test.go b/modules/mesh-delivery-checker/internal/check/examiner_test.go index 1c5f9146..80d166a4 100644 --- a/modules/mesh-delivery-checker/internal/check/examiner_test.go +++ b/modules/mesh-delivery-checker/internal/check/examiner_test.go @@ -25,9 +25,7 @@ import ( gobuild "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep" "git.novox.be/novox/mesh-contracts/go/seats/meshconfig" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliverycheckerclient" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdeliverychecker" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" @@ -880,15 +878,11 @@ func TestTheOutcomeSaysIfEachCandidatesModuleIsBuiltFailedOrWaiting(t *testing.T } } -// trunksOf are trunk heads of every repository, as the gate's source names them. +// trunksOf are trunk heads of every repository, as the catalog half's trunk revision names them. func trunksOf(t *testing.T, heads map[string]string) *Trunks { t.Helper() - data, err := meshdeliveryclient.EncodeSource(meshdelivery.SourceByMainValue{Group: "g", Heads: heads}) - if err != nil { - t.Fatal(err) - } tr := &Trunks{} - tr.Put(data, false) + tr.Set(heads) return tr } diff --git a/modules/mesh-delivery-checker/internal/check/names.go b/modules/mesh-delivery-checker/internal/check/names.go index d34e2bee..76f76fea 100644 --- a/modules/mesh-delivery-checker/internal/check/names.go +++ b/modules/mesh-delivery-checker/internal/check/names.go @@ -22,7 +22,6 @@ const ( // GateModule holds the seat mesh-delivery's states source and turns, which the checker reads (reads: mesh-delivery-gate.source, // mesh-delivery-gate.turns). GateModule = "mesh-delivery-gate" - SourceBucket = GateModule + "_" + meshdelivery.StateSource TurnsBucket = GateModule + "_" + meshdelivery.StateTurns // IndexerModule holds the seat mesh-catalog's states, which the checker follows by their views (consumes: diff --git a/modules/mesh-delivery-checker/internal/check/trunkview.go b/modules/mesh-delivery-checker/internal/check/trunkview.go new file mode 100644 index 00000000..d316a77c --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/trunkview.go @@ -0,0 +1,28 @@ +package check + +import ( + "encoding/json" + "fmt" + + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" +) + +// TrunkVerb is the key of the seat verb the trunk heads are asked by (design 47 G §197). +const TrunkVerb = "seat:" + meshcatalog.Seat + "." + meshcatalog.VerbRevision + +// TrunkHeads reads the trunk head of every repository out of the seat mesh-catalog's answer of `revision`. A candidate's +// revision is no trunk's and is refused. +func TrunkHeads(raw json.RawMessage) (map[string]string, error) { + var a meshcatalog.RevisionAnswer + if err := json.Unmarshal(raw, &a); err != nil { + return nil, fmt.Errorf("the catalog half's revision does not read: %w", err) + } + if a.Revision.Kind != "trunk" { + return nil, fmt.Errorf("the catalog half's revision %d is a %q revision, not a trunk's", a.Revision.Revision, a.Revision.Kind) + } + heads := map[string]string{} + for _, h := range a.Revision.Trunk { + heads[h.Repository] = h.Commit + } + return heads, nil +} diff --git a/modules/mesh-delivery-checker/manifest_test.go b/modules/mesh-delivery-checker/manifest_test.go index ac47d009..a3978a5e 100644 --- a/modules/mesh-delivery-checker/manifest_test.go +++ b/modules/mesh-delivery-checker/manifest_test.go @@ -112,7 +112,7 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1 t.Errorf("state = %+v, want %+v", m.State, want) } if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-node.steps", "mesh-build-node.toolchains", - "mesh-delivery-builder.builds", "mesh-delivery-gate.source", "mesh-delivery-gate.turns", "mesh-delivery-watchman.presence", "node-engine.have"}; !reflect.DeepEqual(sorted(m.Reads), want) { + "mesh-delivery-builder.builds", "mesh-delivery-gate.turns", "mesh-delivery-watchman.presence", "node-engine.have"}; !reflect.DeepEqual(sorted(m.Reads), want) { t.Errorf("reads = %v, want %v", m.Reads, want) } // DLV-M2 (ADR 0615 §1, ADR 0596): the indexer's candidates, modules, manifests, packages and predefined keys and the @@ -131,7 +131,7 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1 "mesh-delivery-assigner.assigner_read_assignments", "mesh-delivery-indexer.indexer_read_candidates", "mesh-delivery-indexer.indexer_read_manifests", "mesh-delivery-indexer.indexer_read_modules", "mesh-delivery-indexer.indexer_read_packages", "mesh-delivery-indexer.indexer_read_predefined", "mesh-delivery-indexer.indexer_read_seat_definitions", - "seat:mesh-catalog.seat-definitions", "seat:mesh-config.renames", "seat:mesh-hand-acts.hand-act", "seat:node-build-go.drop", "seat:node-build-node.drop", "seat:node-engine.compose-candidate"}; !reflect.DeepEqual(sorted(m.Invokes), want) { + "seat:mesh-catalog.revision", "seat:mesh-catalog.seat-definitions", "seat:mesh-config.renames", "seat:mesh-hand-acts.hand-act", "seat:node-build-go.drop", "seat:node-build-node.drop", "seat:node-engine.compose-candidate"}; !reflect.DeepEqual(sorted(m.Invokes), want) { t.Errorf("invokes = %v, want the forge's reads, the seats' drop and compose-candidate", m.Invokes) } if want := []string{"node-build-go", "node-build-node"}; !reflect.DeepEqual(sorted(m.Uses), want) { @@ -140,7 +140,7 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1 // The checker is woken by a build module's report of a toolchain, as the indexer is: the bucket may not exist when it starts. // And a group's check by the package registry seats' published of its tagged versions (design 51 *Tagged versions*). if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", - "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", "mesh-delivery-assigner.views.assignments", "mesh-delivery-indexer.views.candidates", "mesh-delivery-indexer.views.manifests", "mesh-delivery-indexer.views.modules", + "mesh-catalog.catalog-revision", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", "mesh-delivery-assigner.views.assignments", "mesh-delivery-indexer.views.candidates", "mesh-delivery-indexer.views.manifests", "mesh-delivery-indexer.views.modules", "mesh-delivery-indexer.views.packages", "mesh-delivery-indexer.views.predefined", "mesh-delivery-indexer.views.seat-definitions", "npm-package-registry.published"}; !reflect.DeepEqual(sorted(m.Consumes), want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } @@ -202,7 +202,7 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1 } // D-11 (ADR 0634 §1): the holders and buckets internal/check names, no contract naming a holder, are the manifest's: a state // the checker reads is a `reads` (.) and a state of the indexer it follows a `consumes` of its views family. - for bucket, read := range map[string]string{check.SourceBucket: "mesh-delivery-gate.source", check.TurnsBucket: "mesh-delivery-gate.turns", + for bucket, read := range map[string]string{check.TurnsBucket: "mesh-delivery-gate.turns", check.PresenceBucket: "mesh-delivery-watchman.presence", check.HaveBucket: "node-engine.have"} { if strings.Replace(bucket, "_", ".", 1) != read || !slices.Contains(m.Reads, read) { t.Errorf("the bucket %s is not the declared read %s", bucket, read) diff --git a/modules/mesh-delivery-checker/module.json b/modules/mesh-delivery-checker/module.json index 49efcfe4..261ab3e9 100644 --- a/modules/mesh-delivery-checker/module.json +++ b/modules/mesh-delivery-checker/module.json @@ -16,6 +16,7 @@ ], "consumes": [ "go-package-registry.published", + "mesh-catalog.catalog-revision", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", "mesh-build-go.toolchain.reported", @@ -52,7 +53,6 @@ "mesh-build-node.steps", "mesh-build-node.toolchains", "mesh-delivery-builder.builds", - "mesh-delivery-gate.source", "mesh-delivery-gate.turns", "mesh-delivery-watchman.presence", "node-engine.have" @@ -90,6 +90,7 @@ "mesh-delivery-indexer.indexer_read_packages", "mesh-delivery-indexer.indexer_read_predefined", "mesh-delivery-indexer.indexer_read_seat_definitions", + "seat:mesh-catalog.revision", "seat:mesh-catalog.seat-definitions", "seat:mesh-config.renames", "seat:mesh-hand-acts.hand-act", From c3068b15e19e4427fb2c52130dfc8042c4153ffe Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 12:10:49 +0100 Subject: [PATCH 18/33] CK1: the checker reads no merge turn of the gate and claims a new check at once MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §278, §197 (test 52) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-checker-tools/main.go | 4 +- .../mesh-delivery-checker/integration_test.go | 23 +---- .../cmd/mesh-delivery-checker/lazyturns.go | 50 ----------- .../cmd/mesh-delivery-checker/main.go | 28 ------ .../internal/check/adr548_test.go | 89 +------------------ .../internal/check/checker.go | 58 +----------- .../internal/check/names.go | 4 +- .../internal/check/read.go | 50 +---------- .../internal/check/read_test.go | 12 --- .../internal/check/turns.go | 79 ---------------- .../mesh-delivery-checker/manifest_test.go | 4 +- modules/mesh-delivery-checker/module.json | 1 - 12 files changed, 14 insertions(+), 388 deletions(-) delete mode 100644 modules/mesh-delivery-checker/cmd/mesh-delivery-checker/lazyturns.go delete mode 100644 modules/mesh-delivery-checker/internal/check/turns.go diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go index 9d6caafe..f74619f1 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go @@ -422,8 +422,8 @@ func main() { PreviewState: read(stdio.State("previews")), InputState: read(stdio.State(check.InputsStateName)), StepStates: map[string]check.StateRead{"mesh-build-go": read(stdio.State("mesh-build-go.steps")), "mesh-build-node": read(stdio.State("mesh-build-node.steps"))}, HaveState: read(stdio.State("node-engine.have")), StepBound: stepBound(), - TurnState: read(stdio.State("mesh-delivery-gate.turns")), HearingState: read(stdio.State(buildrun.HearingState)), - AskState: read(stdio.State(check.AsksState)), BuildState: read(stdio.State("mesh-delivery-builder.builds"))} + HearingState: read(stdio.State(buildrun.HearingState)), + AskState: read(stdio.State(check.AsksState)), BuildState: read(stdio.State("mesh-delivery-builder.builds"))} r.Runs = runsOf{ownerRead(stdio.Ask, indexerModulesTool)} // the node-engine at the commit the mesh runs, as the checker reads it tries := stdio.State(check.TriesState) trunks := func() (map[string]string, error) { diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go index 0ddb4046..19930604 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go @@ -38,7 +38,6 @@ import ( "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/gopackageregistry" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdeliverychecker" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" @@ -608,7 +607,7 @@ func startBed(t *testing.T) *bed { check.CheckerOwedBucket: 5, check.PreviewsBucket: 5, check.CandidatesBucket: 10, check.ModulesBucket: 20, check.ManifestsBucket: 1, check.PresenceBucket: 20, check.HaveBucket: 20, check.TriesBucket: 1, - check.TurnsBucket: 5, builds.BuilderBuildsBucket: 1, check.InputsBucket: 1} { + builds.BuilderBuildsBucket: 1, check.InputsBucket: 1} { kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: uint8(history)}) if err != nil { t.Fatal(err) @@ -936,26 +935,8 @@ func TestACheckOrACarryOfEqualInputsRunsNothing(t *testing.T) { // 1287/AC5, 128 t.Errorf("%d kept keys, was %d", got, kept) } - // §3: the gate's turn on the repository holds a new check off; its delete claims it. - at := time.Now().UTC() - turn, err := meshdeliveryclient.EncodeTurn(meshdelivery.TurnsByRepositoryValue{Repository: repo, Waiting: []meshdelivery.TurnsByRepositoryValueWaitingItem{{Group: "fix/other", Heads: "0123456789ab", Approved: at, Since: at}}}) - if err != nil { - t.Fatal(err) - } - tk, _ := meshdeliveryclient.TurnsKey(repo) - turnRev, err := b.buckets[check.TurnsBucket].Put(b.ctx, tk, turn) - if err != nil { - t.Fatal(err) - } - b.said(t, fmt.Sprintf("merge turn %s changed (revision %d)", tk, turnRev)) // the checker heard the turn + // Design 47 G §278 (test 52): a new check is claimed at once, with no state of the gate granted to the checker. third := b.candidate(t, "feat/z", pull(repo, 9, "3333333333333333333333333333333333333333")) - b.said(t, third+": not claimed: the approved merge of fix/other") // it held the check off - if _, err := b.buckets[check.CheckerVerdictsBucket].Get(b.ctx, third); !errors.Is(err, jetstream.ErrKeyNotFound) { - t.Fatalf("verdicts.%s exists (%v) while a merge turn holds its repository", third, err) - } - if err := b.buckets[check.TurnsBucket].Delete(b.ctx, tk); err != nil { - t.Fatal(err) - } b.verdict(t, third, ended) // §5: the trunk moves; the first verdict is carried by its digests, nothing runs. diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/lazyturns.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/lazyturns.go deleted file mode 100644 index b39611bc..00000000 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/lazyturns.go +++ /dev/null @@ -1,50 +0,0 @@ -package main - -import ( - "sync" - - "mesh-delivery-checker/internal/check" - - "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" -) - -// lazyTurns is the gate's merge turns as the checker reads them (novox/hq ADR 0548 §3), opened when the gate's bucket is -// there: before each claim it is opened again until it is, so a checker that started before the gate made the bucket holds -// checks from the gate's first turn on, with no timer and no restart. Until then it holds nothing, and says so once, loudly, -// and once again when it opens. -type lazyTurns struct { - turns *check.Turns - open func() error - logf func(format string, args ...any) - - mu sync.Mutex - opened bool - said bool -} - -// ensure opens the bucket once it is there; a failure is said once. -func (l *lazyTurns) ensure() { - l.mu.Lock() - defer l.mu.Unlock() - if l.opened || l.open == nil { - return - } - if err := l.open(); err != nil { - if !l.said { - l.said = true - l.logf("NO MERGE TURN HOLDS A CHECK: the gate's bucket %s cannot be read (%v); every check is claimed as if no approved merge waited, "+ - "and the bucket is opened again before each claim until it is there", check.TurnsBucket, err) - } - return - } - l.opened = true - if l.said { - l.logf("the gate's merge turns are read from %s from now on: an approved merge's turn holds checks off its repositories again", check.TurnsBucket) - } -} - -// Holding opens the bucket when it is not yet, then answers what the turns hold. -func (l *lazyTurns) Holding(repositories []string, heads string) (string, meshdelivery.TurnsByRepositoryValueWaitingItem, bool) { - l.ensure() - return l.turns.Holding(repositories, heads) -} diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go index 81433a09..d567a1df 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go @@ -204,8 +204,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro tools := &buildrun.ToolchainSet{} presence := &buildrun.PresenceSet{} trunks := &check.Trunks{} - turns := &check.Turns{} - lazy := &lazyTurns{turns: turns, logf: logger.Printf} // A check is one ask per pull request and build module on the build seats' check stream, pulled by a build node with a // free slot (novox/hq ADR 0525 §1, §3): the checker chooses no node and starts no step. asks := buildrun.JetStreamAsks{JS: js} @@ -232,7 +230,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro Verdicts: kvStore{buckets[check.CheckerVerdictsBucket]}, Owed: kvStore{buckets[check.CheckerOwedBucket]}, Trunks: trunks, - Turns: lazy, Bound: checkBound, Link: link, Logf: logger.Printf, @@ -381,31 +378,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return fmt.Errorf("the checker cannot hear the catalog half's revisions: %w", err) } firstTrunks(ctx) - // The gate's merge turns (novox/hq ADR 0548 §3): no check is claimed of a repository an approved merge waits on, and the - // delete of its turn claims it. Read before the candidates, so no check starts past a turn written before this start. - // Where the gate's bucket is not there yet it is read as empty, and opened again before each claim until it is there - // (lazyTurns), so a gate delivered after the checker started holds checks from its first turn on. - lazy.open = func() error { - kv, err := js.KeyValue(ctx, check.TurnsBucket) - if err != nil { - return err - } - tw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { turns.Put(e.Key(), e.Value(), !opOf(e)) }) - if err != nil { - return err - } - go func() { - errs <- follow(ctx, tw, func(e jetstream.KeyValueEntry) { - if turns.Put(e.Key(), e.Value(), !opOf(e)) { - // Said, so a wait on a held check can be read (and a test waits on it rather than on a pause). - logger.Printf("the gate's merge turn %s changed (revision %d): the checks it holds are held or claimed from now on", e.Key(), e.Revision()) - ch.Reconsider(ctx) // a turn went or changed: a check it held is claimed now - } - }) - }() - return nil - } - lazy.ensure() afterFirstViews := func() error { // The modules say which repository holds a TypeScript module, so the Node build module judges it too. pw, err := settle(ctx, packages, func(e jetstream.KeyValueEntry) { published.Put(e.Key(), e.Value(), !opOf(e)) }) diff --git a/modules/mesh-delivery-checker/internal/check/adr548_test.go b/modules/mesh-delivery-checker/internal/check/adr548_test.go index d88c4f6b..2d236d4b 100644 --- a/modules/mesh-delivery-checker/internal/check/adr548_test.go +++ b/modules/mesh-delivery-checker/internal/check/adr548_test.go @@ -7,9 +7,7 @@ import ( "time" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliverycheckerclient" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdeliverychecker" ) @@ -17,26 +15,6 @@ import ( // a trunk or sibling move never ends a running check and carries its verdict once it ended (§1, §5, §6); only a head git // refused gets a whole check after a move (§7); the line of a carried verdict says so (§8). -// turnOf is a turn's value on a repository for a delivery of the heads hash. -func turnOf(t testing.TB, repo, group, heads string) []byte { - t.Helper() - at := time.Date(2026, 10, 26, 9, 0, 0, 0, time.UTC) - data, err := meshdeliveryclient.EncodeTurn(meshdelivery.TurnsByRepositoryValue{Repository: repo, Waiting: []meshdelivery.TurnsByRepositoryValueWaitingItem{{Group: group, Heads: heads, Approved: at, Since: at}}}) - if err != nil { - t.Fatal(err) - } - return data -} - -func turnKey(t testing.TB, repo string) string { - t.Helper() - k, err := meshdeliveryclient.TurnsKey(repo) - if err != nil { - t.Fatal(err) - } - return k -} - // waitFor polls a condition the checker's goroutines make true, failing after ten seconds: a bound a loaded machine does // not reach, waited out only by a failing test. func waitFor(t testing.TB, what string, ok func() bool) { @@ -59,66 +37,15 @@ func pullWithFP(repo string, number int, head string, d string) meshcatalog.Cand func TestOneCheckPerDelivery(t *testing.T) { ctx := context.Background() for name, run := range map[string]func(t *testing.T){ - // §3: no claim while a turn names one of its repositories; a claim on the turn's delete event. - "a turn on its repository holds a new check off until the turn goes": func(t *testing.T) { + // Design 47 G §278 (test 52): a new check is claimed at once; the checker reads no state of the gate, so no merge turn holds it. + "a new check is claimed at once": func(t *testing.T) { j := passJudge() r := newRig(t, j) - turns := &Turns{} - r.ch.Turns = turns - turns.Put(turnKey(t, repoA), turnOf(t, repoA, "fix/other", "0123456789ab"), false) cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1)) r.ch.Candidate(ctx, "feat%2Fx", cand) r.ch.Wait() - if j.count() != 0 || len(r.verdicts.vals) != 0 { - t.Fatalf("judged %d times with a turn on %s, verdicts %v", j.count(), repoA, r.verdicts.vals) - } - if held := r.ch.Held()[keyOf(t, "feat/x", cand)]; !strings.Contains(held, repoA) || !strings.Contains(held, "fix/other") { - t.Errorf("the hold is said as %q, want the turn's repository and group", held) - } - if !turns.Put(turnKey(t, repoA), nil, true) { - t.Fatal("the turn's delete changed nothing") - } - r.ch.Reconsider(ctx) // what the watch of turns does on the delete - r.ch.Wait() if j.count() != 1 || r.verdict(t, keyOf(t, "feat/x", cand)).State != meshdeliverycheckerclient.VerdictPass { - t.Errorf("judged %d times after the turn went, want once", j.count()) - } - }, - "a turn on another repository, or of its own heads, holds nothing": func(t *testing.T) { - j := passJudge() - r := newRig(t, j) - turns := &Turns{} - r.ch.Turns = turns - cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1)) - c, _ := meshcatalog.DecodeCandidatesByGroupValue(cand) - turns.Put(turnKey(t, repoSDK), turnOf(t, repoSDK, "fix/other", "0123456789ab"), false) - turns.Put(turnKey(t, repoA), turnOf(t, repoA, "feat/x", GateHeads(c)), false) - r.ch.Candidate(ctx, "feat%2Fx", cand) - r.ch.Wait() - if j.count() != 1 { - t.Errorf("judged %d times, want once: the turns hold another repository and its own merge", j.count()) - } - }, - // §3, the race: a check claimed before the gate's turn is judged on the commit it read, never restarted. - "a check claimed before a turn was written runs to its verdict": func(t *testing.T) { - release, started := make(chan struct{}), make(chan struct{}) - j := &judgeFunc{f: func(context.Context, *Case, int) (Outcome, error) { - close(started) - <-release - return Outcome{Verdict: passing()}, nil - }} - r := newRig(t, j) - turns := &Turns{} - r.ch.Turns = turns - cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1)) - r.ch.Candidate(ctx, "feat%2Fx", cand) - <-started - turns.Put(turnKey(t, repoA), turnOf(t, repoA, "fix/other", "0123456789ab"), false) - r.ch.Reconsider(ctx) - close(release) - r.ch.Wait() - if j.count() != 1 || r.verdict(t, keyOf(t, "feat/x", cand)).State != meshdeliverycheckerclient.VerdictPass { - t.Errorf("judged %d times; the running check is not ended by the turn", j.count()) + t.Errorf("judged %d times, want once at once", j.count()) } }, // §1, §5: a trunk move during a running check calls the judge once, writes its verdict, keeps the status pending (a pass @@ -280,15 +207,7 @@ func TestOneCheckPerDelivery(t *testing.T) { r.ch.Wait() a, _ := meshcatalog.DecodeCandidatesByGroupValue(candA) c, _ := meshcatalog.DecodeCandidatesByGroupValue(candC) - at := time.Date(2026, 10, 26, 9, 0, 0, 0, time.UTC) - turn, err := meshdeliveryclient.EncodeTurn(meshdelivery.TurnsByRepositoryValue{Repository: repoA, Waiting: []meshdelivery.TurnsByRepositoryValueWaitingItem{ - {Group: "feat/a", Heads: GateHeads(a), Approved: at, Since: at}, {Group: "feat/c", Heads: GateHeads(c), Approved: at.Add(time.Minute), Since: at}}}) - if err != nil { - t.Fatal(err) - } - turns := &Turns{} - turns.Put(turnKey(t, repoA), turn, false) - r.ch.Turns = turns + _, _ = a, c r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS}) // a third merge moves the trunk under both r.ch.TrunkMoved(ctx) r.ch.Wait() diff --git a/modules/mesh-delivery-checker/internal/check/checker.go b/modules/mesh-delivery-checker/internal/check/checker.go index 1e583732..b5c72ada 100644 --- a/modules/mesh-delivery-checker/internal/check/checker.go +++ b/modules/mesh-delivery-checker/internal/check/checker.go @@ -16,7 +16,6 @@ import ( "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliverycheckerclient" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdeliverychecker" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "git.novox.be/novox/mesh-delivery/sdk/buildstep" @@ -86,11 +85,6 @@ type Checker struct { // Runs says which commit the mesh runs of a module (the indexer's modules): the node-engine a verdict read beside at its // running commit is out of date when that commit moves (ADR 0548 §6); nil reads every sibling at its trunk. Runs interface{ CommitOf(module string) string } - // Turns are the gate's merge turns (ADR 0548 §3): no check is claimed of a repository a turn names, but the check of the - // waiting delivery's own heads; nil holds none. - Turns interface { - Holding(repositories []string, heads string) (string, meshdelivery.TurnsByRepositoryValueWaitingItem, bool) - } // Bound is how long one check may run; none when 0 (DefaultBound). Bound time.Duration // Rebuilds hears the builder's build keys a failed verdict names (failedBuilds): once each is written done, the check of @@ -120,7 +114,6 @@ type Checker struct { tried map[string]bool // by verdict key: looked at since this process started epoch uint64 // changes of the nodes seen since this process started nodesFail map[string]uint64 // by verdict key: the epoch a failure of the nodes was written at - held map[string]string // by verdict key: the turn that holds its check off, as last said live map[string]liveCheck // by verdict key: the check running in this process, ended when its heads go settling map[string]*window // by candidates key: the window its heads settle in rebuilt map[string]bool // by verdict key: every failed build its verdict names is done again @@ -184,7 +177,7 @@ func (s superseded) Unwrap() error { return buildrun.Withdrawn{Why: s.Error()} } func (ch *Checker) init() { ch.once.Do(func() { ch.known, ch.inflight, ch.again, ch.tried = map[string]meshcatalog.CandidatesByGroupValue{}, map[string]bool{}, map[string]bool{}, map[string]bool{} - ch.nodesFail, ch.held = map[string]uint64{}, map[string]string{} + ch.nodesFail = map[string]uint64{} ch.live, ch.settling = map[string]liveCheck{}, map[string]*window{} ch.rebuilt, ch.awaiting = map[string]bool{}, map[string]bool{} ch.outsideSaid = map[string]bool{} @@ -695,9 +688,6 @@ func (ch *Checker) claim(ctx context.Context, cs *Case, first bool, epoch uint64 return nil, fmt.Errorf("%s: %w", cs.Key, err) } if !found { - if ch.heldByTurn(cs) { - return nil, nil - } rev, err := ch.Verdicts.Create(ctx, cs.Key, data) if errors.Is(err, ErrConflict) { return nil, nil // another checker made it first @@ -728,7 +718,7 @@ func (ch *Checker) claim(ctx context.Context, cs *Case, first bool, epoch uint64 ch.mu.Unlock() retake = !ok || epoch > at } - if !retake || ch.heldByTurn(cs) { + if !retake { return nil, nil } rev, err = ch.Verdicts.Update(ctx, cs.Key, data, rev) @@ -741,50 +731,6 @@ func (ch *Checker) claim(ctx context.Context, cs *Case, first bool, epoch uint64 return &claimed{rev}, nil } -// heldByTurn says whether a merge turn holds the case's check off (ADR 0548 §3): an approved merge waits on one of the -// repositories it merges heads into, and it is not the merge of the case's own heads. The check is claimed when the turn's -// key goes (Reconsider on the turns watch). Said once per turn in the log. A check already claimed is never held: one -// claimed in the moment before the gate wrote its turn is judged on the commit it read and carried once after. -func (ch *Checker) heldByTurn(cs *Case) bool { - if ch.Turns == nil { - return false - } - repos := make([]string, 0, len(cs.Trunk)) - for r := range cs.Trunk { - repos = append(repos, r) - } - repo, w, held := ch.Turns.Holding(repos, cs.Heads) - why := "" - if held { - why = repo + " " + w.Group + " " + w.Heads - } - ch.mu.Lock() - said := ch.held[cs.Key] - if why == "" { - delete(ch.held, cs.Key) - } else { - ch.held[cs.Key] = why - } - ch.mu.Unlock() - if held && said != why { - ch.logf("%s: not claimed: the approved merge of %s (heads %s) holds a turn on %s, so its check waits for that merge (ADR 0548 §3)", - cs.Key, w.Group, w.Heads, repo) - } - return held -} - -// Held is every check a merge turn holds off now, by verdict key, with the turn's repository, group and heads. -func (ch *Checker) Held() map[string]string { - ch.init() - ch.mu.Lock() - defer ch.mu.Unlock() - out := make(map[string]string, len(ch.held)) - for k, v := range ch.held { - out[k] = v - } - return out -} - // ConflictWords are what the words of a head that git would not merge onto its trunk say before the files in conflict. const ConflictWords = "does not merge into the trunk: files in conflict: " diff --git a/modules/mesh-delivery-checker/internal/check/names.go b/modules/mesh-delivery-checker/internal/check/names.go index 76f76fea..44809d1b 100644 --- a/modules/mesh-delivery-checker/internal/check/names.go +++ b/modules/mesh-delivery-checker/internal/check/names.go @@ -4,7 +4,6 @@ import ( "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" checker "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdeliverychecker" ) @@ -21,8 +20,7 @@ const ( // GateModule holds the seat mesh-delivery's states source and turns, which the checker reads (reads: mesh-delivery-gate.source, // mesh-delivery-gate.turns). - GateModule = "mesh-delivery-gate" - TurnsBucket = GateModule + "_" + meshdelivery.StateTurns + GateModule = "mesh-delivery-gate" // IndexerModule holds the seat mesh-catalog's states, which the checker follows by their views (consumes: // mesh-delivery-indexer.views.*). diff --git a/modules/mesh-delivery-checker/internal/check/read.go b/modules/mesh-delivery-checker/internal/check/read.go index 54ee1798..0c677b90 100644 --- a/modules/mesh-delivery-checker/internal/check/read.go +++ b/modules/mesh-delivery-checker/internal/check/read.go @@ -51,8 +51,6 @@ type Reader struct { StepBound time.Duration // how long the checker waits for one step; DefaultStepBound when 0 // Trunks are the trunk heads of every repository (the gate's source); nil names no re-judge after a trunk move. Trunks func() (map[string]string, error) - // TurnState is the gate's merge turns (ADR 0548 §3); without it no check is named as held by a turn. - TurnState StateRead // Runs says which commit the mesh runs of a module (the indexer's modules), as the checker's decision reads it: the // node-engine a verdict names is compared at its running commit, not at the trunk's (ADR 0548 §6); nil knows none. Runs interface{ CommitOf(module string) string } @@ -411,9 +409,6 @@ type Status struct { // Waiting are the candidates whose check waits, by kind (ADR 0536 §4): new (no verdict for its heads) and nodes (failed // for the nodes, asked again when they change) start before trunk (judged on trunk heads that moved since). Waiting map[string][]string `json:"waiting_by_kind"` - // HeldByTurn are the candidates whose check an approved merge's turn holds off (ADR 0548 §3), by group: the repository and - // the group of the merge that waits. Their check is claimed when that turn goes. - HeldByTurn map[string]string `json:"held_by_turn"` // CannotBeJudged are the candidates no pull request of which can be judged, by group, with why: the gate's source.main // names no trunk head of their repositories (issue 1422). They are judged once it does; until then their status is error. CannotBeJudged map[string]string `json:"cannot_be_judged"` @@ -451,42 +446,6 @@ func (r Reader) deadLetters() DeadLettersHeard { return DeadLettersHeard{Heard: a.Healthy, Why: a.Why} } -// turns reads the gate's merge turns, by repository; nil when they cannot be read, said in a note. -func (r Reader) turns(st *Status) *Turns { - if r.TurnState.Keys == nil || r.TurnState.Get == nil { - st.Notes = append(st.Notes, "the gate's merge turns are not readable here, so no check is named as held by one") - return nil - } - keys, err := r.TurnState.Keys() - if err != nil { - st.Notes = append(st.Notes, "the gate's merge turns cannot be listed, so no check is named as held by one: "+err.Error()) - return nil - } - t := &Turns{} - for _, k := range keys { - if raw, err := r.TurnState.Get(k); err == nil && len(raw) > 0 { - t.Put(k, raw, false) - } - } - return t -} - -// heldBy says the turn that holds a candidate's check off, or "". -func heldBy(t *Turns, c meshcatalog.CandidatesByGroupValue, heads map[string]string) string { - cs := CaseOf(c, heads) - if t == nil || cs == nil { - return "" - } - repos := make([]string, 0, len(cs.Trunk)) - for r := range cs.Trunk { - repos = append(repos, r) - } - if repo, w, ok := t.Holding(repos, cs.Heads); ok { - return repo + ": the approved merge of " + w.Group + " waits on it" - } - return "" -} - // Status reads the build modules' steps and the candidates against their verdicts. func (r Reader) Status() (Status, error) { st := Status{RunningByNode: map[string]int{}, Lost: map[string]string{}} @@ -503,8 +462,7 @@ func (r Reader) Status() (Status, error) { } st.Unjudged = []string{} st.Waiting = map[string][]string{KindNew: {}, KindNodes: {}, KindTrunk: {}} - st.HeldByTurn, st.CannotBeJudged, st.Outside = map[string]string{}, map[string]string{}, map[string]string{} - turns := r.turns(&st) + st.CannotBeJudged, st.Outside = map[string]string{}, map[string]string{} var heads map[string]string if r.Trunks == nil { st.Notes = append(st.Notes, "the trunk heads are not readable here, so no check is named as waiting for a trunk move") @@ -571,9 +529,6 @@ func (r Reader) Status() (Status, error) { if len(raw) == 0 { st.Unjudged = append(st.Unjudged, c.Group) st.Waiting[KindNew] = append(st.Waiting[KindNew], c.Group) - if why := heldBy(turns, c, heads); why != "" { - st.HeldByTurn[c.Group] = why - } continue } v, err := meshdeliverycheckerclient.DecodeVerdict(raw) @@ -597,9 +552,6 @@ func (r Reader) Status() (Status, error) { case heads != nil: if cs := CaseOf(c, heads); cs != nil && (!sameTrunk(v.Trunk, cs.Trunk) || besideMoved(v.Beside, besideAt(heads, r.Runs))) { st.Waiting[KindTrunk] = append(st.Waiting[KindTrunk], c.Group) - if why := heldBy(turns, c, heads); why != "" { - st.HeldByTurn[c.Group] = why - } } } } diff --git a/modules/mesh-delivery-checker/internal/check/read_test.go b/modules/mesh-delivery-checker/internal/check/read_test.go index cf5d63a5..cad97946 100644 --- a/modules/mesh-delivery-checker/internal/check/read_test.go +++ b/modules/mesh-delivery-checker/internal/check/read_test.go @@ -316,18 +316,6 @@ func TestTheStatusCountsAnHQOnlyGroupAsUnjudged(t *testing.T) { if len(st.Unjudged) != 1 || st.Unjudged[0] != "pb-x" { t.Errorf("unjudged = %v, want pb-x", st.Unjudged) } - // ADR 0548 §3: a candidate whose check an approved merge's turn holds off is named with the turn. - held := candidateOf(t, "feat/x", pullOf(repoA, 7, head1)) - hkey, _ := meshcatalogclient.CandidatesKey("feat/x") - st, err = Reader{CandidateState: fakeState{hkey: held}.read(), VerdictState: fakeState{}.read(), - Trunks: func() (map[string]string, error) { return map[string]string{repoA: trunkA}, nil }, - TurnState: fakeState{turnKey(t, repoA): turnOf(t, repoA, "fix/other", "0123456789ab")}.read()}.Status() - if err != nil { - t.Fatal(err) - } - if why := st.HeldByTurn["feat/x"]; !strings.Contains(why, repoA) || !strings.Contains(why, "fix/other") { - t.Errorf("held_by_turn = %v, want feat/x held by the merge of fix/other on %s", st.HeldByTurn, repoA) - } } // slowBucket wraps a state with a Get that takes delay and counts the values read, the lists and the most Gets in flight. diff --git a/modules/mesh-delivery-checker/internal/check/turns.go b/modules/mesh-delivery-checker/internal/check/turns.go deleted file mode 100644 index cb8da5e0..00000000 --- a/modules/mesh-delivery-checker/internal/check/turns.go +++ /dev/null @@ -1,79 +0,0 @@ -package check - -import ( - "sort" - "strings" - "sync" - - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" -) - -// Turns holds the gate's merge turns (novox/hq ADR 0548 §3), by repository, as a watch of the gate's turns bucket puts them -// in: the checker claims no check of a repository a turn names, except the check of a waiting delivery's own heads, so an -// approved merge goes before a check not yet started. A bucket not there is read as empty. -type Turns struct { - mu sync.Mutex - byRepo map[string]meshdelivery.TurnsByRepositoryValue -} - -// Put takes a value of the turns bucket in, or its delete, and answers whether what the checker holds changed. A value that -// is not a turn is dropped as a delete would be: a turn the checker cannot read holds nothing. -func (t *Turns) Put(key string, value []byte, deleted bool) bool { - repository, err := meshdeliveryclient.ParseTurnsKey(key) - if err != nil { - return false - } - t.mu.Lock() - defer t.mu.Unlock() - if t.byRepo == nil { - t.byRepo = map[string]meshdelivery.TurnsByRepositoryValue{} - } - cur, had := t.byRepo[repository] - if !deleted { - if turn, err := meshdeliveryclient.DecodeTurn(value); err == nil { - t.byRepo[repository] = turn - return !had || !meshdeliveryclient.SameTurn(cur, turn) - } - } - delete(t.byRepo, repository) - return had -} - -// Holding names the first of the repositories (by name) a turn holds off the check of the heads, and the delivery that -// waits first in it; false when none does. -func (t *Turns) Holding(repositories []string, heads string) (string, meshdelivery.TurnsByRepositoryValueWaitingItem, bool) { - if t == nil { - return "", meshdelivery.TurnsByRepositoryValueWaitingItem{}, false - } - sorted := make([]string, 0, len(repositories)) - for _, r := range repositories { - sorted = append(sorted, strings.ToLower(r)) - } - sort.Strings(sorted) - t.mu.Lock() - defer t.mu.Unlock() - for _, r := range sorted { - turn, ok := t.byRepo[r] - if !ok || !meshdeliveryclient.TurnHolds(turn, heads) { - continue - } - for _, w := range turn.Waiting { - if w.Heads != heads { - return r, w, true - } - } - } - return "", meshdelivery.TurnsByRepositoryValueWaitingItem{}, false -} - -// All is every turn held now, by repository (a copy). -func (t *Turns) All() map[string]meshdelivery.TurnsByRepositoryValue { - t.mu.Lock() - defer t.mu.Unlock() - out := make(map[string]meshdelivery.TurnsByRepositoryValue, len(t.byRepo)) - for r, turn := range t.byRepo { - out[r] = turn - } - return out -} diff --git a/modules/mesh-delivery-checker/manifest_test.go b/modules/mesh-delivery-checker/manifest_test.go index a3978a5e..29506cab 100644 --- a/modules/mesh-delivery-checker/manifest_test.go +++ b/modules/mesh-delivery-checker/manifest_test.go @@ -112,7 +112,7 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1 t.Errorf("state = %+v, want %+v", m.State, want) } if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-node.steps", "mesh-build-node.toolchains", - "mesh-delivery-builder.builds", "mesh-delivery-gate.turns", "mesh-delivery-watchman.presence", "node-engine.have"}; !reflect.DeepEqual(sorted(m.Reads), want) { + "mesh-delivery-builder.builds", "mesh-delivery-watchman.presence", "node-engine.have"}; !reflect.DeepEqual(sorted(m.Reads), want) { t.Errorf("reads = %v, want %v", m.Reads, want) } // DLV-M2 (ADR 0615 §1, ADR 0596): the indexer's candidates, modules, manifests, packages and predefined keys and the @@ -202,7 +202,7 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1 } // D-11 (ADR 0634 §1): the holders and buckets internal/check names, no contract naming a holder, are the manifest's: a state // the checker reads is a `reads` (.) and a state of the indexer it follows a `consumes` of its views family. - for bucket, read := range map[string]string{check.TurnsBucket: "mesh-delivery-gate.turns", + for bucket, read := range map[string]string{ check.PresenceBucket: "mesh-delivery-watchman.presence", check.HaveBucket: "node-engine.have"} { if strings.Replace(bucket, "_", ".", 1) != read || !slices.Contains(m.Reads, read) { t.Errorf("the bucket %s is not the declared read %s", bucket, read) diff --git a/modules/mesh-delivery-checker/module.json b/modules/mesh-delivery-checker/module.json index 261ab3e9..02b78a1e 100644 --- a/modules/mesh-delivery-checker/module.json +++ b/modules/mesh-delivery-checker/module.json @@ -53,7 +53,6 @@ "mesh-build-node.steps", "mesh-build-node.toolchains", "mesh-delivery-builder.builds", - "mesh-delivery-gate.turns", "mesh-delivery-watchman.presence", "node-engine.have" ], From 5fc1dbd264379148c46b8122d6154036f7ee7c84 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 12:14:07 +0100 Subject: [PATCH 19/33] WM11, CK12, EN1 (manifest half): finding stream, the keeper's ask and MAJOR.MINOR version in the manifests of the watchman, the checker and enrolment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §383, §387, §368 (the findings writer half waits for the generated writer of §439) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- modules/mesh-delivery-checker/manifest_test.go | 5 +++-- modules/mesh-delivery-checker/module.json | 8 +++++--- modules/mesh-delivery-watchman/manifest_test.go | 13 ++++++++++++- modules/mesh-delivery-watchman/module.json | 6 +++++- .../cmd/mesh-enrolment/tokens_test.go | 2 +- modules/mesh-enrolment/module.json | 8 ++++++-- 6 files changed, 32 insertions(+), 10 deletions(-) diff --git a/modules/mesh-delivery-checker/manifest_test.go b/modules/mesh-delivery-checker/manifest_test.go index 29506cab..78a167bb 100644 --- a/modules/mesh-delivery-checker/manifest_test.go +++ b/modules/mesh-delivery-checker/manifest_test.go @@ -41,6 +41,7 @@ type stream struct { Name string `json:"name"` Events []string `json:"events"` MaxAgeDays int `json:"max-age-days,omitempty"` + MaxMiB int `json:"max-mib,omitempty"` Retention string `json:"retention,omitempty"` } @@ -140,7 +141,7 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1 // The checker is woken by a build module's report of a toolchain, as the indexer is: the bucket may not exist when it starts. // And a group's check by the package registry seats' published of its tagged versions (design 51 *Tagged versions*). if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", - "mesh-catalog.catalog-revision", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", "mesh-delivery-assigner.views.assignments", "mesh-delivery-indexer.views.candidates", "mesh-delivery-indexer.views.manifests", "mesh-delivery-indexer.views.modules", + "mesh-catalog.catalog-revision", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", "mesh-conditions.findings-asked", "mesh-delivery-assigner.views.assignments", "mesh-delivery-indexer.views.candidates", "mesh-delivery-indexer.views.manifests", "mesh-delivery-indexer.views.modules", "mesh-delivery-indexer.views.packages", "mesh-delivery-indexer.views.predefined", "mesh-delivery-indexer.views.seat-definitions", "npm-package-registry.published"}; !reflect.DeepEqual(sorted(m.Consumes), want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } @@ -154,7 +155,7 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1 // Its family asks (design 51 *Tagged versions*, design 26): tag-wanted and verdict, seven days, apart from checks so a // waiter on a verdict does not wake on a tag ask; a stream that keeps its history, the predefined form of an event family // (ADR 0592 §6, the manifest validator). - if want := []stream{{Name: "checks", Events: []string{"preview.**", "owed.**", "try.**"}}, {Name: "asks", Events: []string{"verdict", "tag-wanted"}, Retention: "history"}}; !reflect.DeepEqual(m.Streams, want) { + if want := []stream{{Name: "checks", Events: []string{"preview.**", "owed.**", "try.**"}}, {Name: "asks", Events: []string{"verdict", "tag-wanted"}, Retention: "history"}, {Name: "mesh-delivery-checker-findings", Events: []string{"finding-written.*", "finding-cleared.*", "findings-stated.*"}, MaxAgeDays: 7, MaxMiB: 16}}; !reflect.DeepEqual(m.Streams, want) { t.Errorf("streams = %+v, want %+v", m.Streams, want) } // Only the checker's account publishes try.* on its stream checks: the stream is declared by its owner alone, and no other diff --git a/modules/mesh-delivery-checker/module.json b/modules/mesh-delivery-checker/module.json index 02b78a1e..3ed9b598 100644 --- a/modules/mesh-delivery-checker/module.json +++ b/modules/mesh-delivery-checker/module.json @@ -1,6 +1,6 @@ { "module": "mesh-delivery-checker", - "version": "1", + "version": "0.1", "slug": "checker", "claims": [ { @@ -28,11 +28,13 @@ "mesh-delivery-indexer.views.packages", "mesh-delivery-indexer.views.predefined", "mesh-delivery-indexer.views.seat-definitions", - "npm-package-registry.published" + "npm-package-registry.published", + "mesh-conditions.findings-asked" ], "streams": [ { "name": "checks", "events": ["preview.**", "owed.**", "try.**"] }, - { "name": "asks", "events": ["verdict", "tag-wanted"], "retention": "history" } + { "name": "asks", "events": ["verdict", "tag-wanted"], "retention": "history" }, + { "name": "mesh-delivery-checker-findings", "events": ["finding-written.*", "finding-cleared.*", "findings-stated.*"], "max-age-days": 7, "max-mib": 16 } ], "state": [ { "name": "asks", "history": 1 }, diff --git a/modules/mesh-delivery-watchman/manifest_test.go b/modules/mesh-delivery-watchman/manifest_test.go index e8d37103..ddf43111 100644 --- a/modules/mesh-delivery-watchman/manifest_test.go +++ b/modules/mesh-delivery-watchman/manifest_test.go @@ -57,6 +57,12 @@ type manifest struct { Consumes []string `json:"consumes"` Invokes []string `json:"invokes"` State []state `json:"state"` + Streams []struct { + Name string `json:"name"` + Events []string `json:"events"` + MaxAgeDays int `json:"max-age-days"` + MaxMiB int `json:"max-mib"` + } `json:"streams"` OwnSecrets map[string]string `json:"own-secrets"` SecretsOwner string `json:"secrets-owner"` Resources []map[string]any `json:"resources"` @@ -103,6 +109,11 @@ func TestManifestStateAndReads(t *testing.T) { // 1135/AC6 if !regexp.MustCompile(`^[0-9]+\.[0-9]+$`).MatchString(m.Version) { t.Errorf("version = %q, want MAJOR.MINOR", m.Version) } + // Design 47 G §387: the finding stream, 7 days and 16 MiB, and the keeper's ask. + if len(m.Streams) != 1 || m.Streams[0].Name != "mesh-delivery-watchman-findings" || m.Streams[0].MaxAgeDays != 7 || m.Streams[0].MaxMiB != 16 || + !reflect.DeepEqual(m.Streams[0].Events, []string{"finding-written.*", "finding-cleared.*", "findings-stated.*"}) { + t.Errorf("streams = %+v, want the finding stream", m.Streams) + } // DLV-M5 (ADR 0615 §1): the assigner's wants and the indexer's modules are their owners' states, asked by their read // tools and followed by their views families, never read. for _, r := range m.Reads { @@ -116,7 +127,7 @@ func TestManifestStateAndReads(t *testing.T) { // 1135/AC6 t.Errorf("invokes = %v, want %v", m.Invokes, want) } // DLV-M5 its first views, 1585/T20 the indexer's take-in events, and DLV-W1 (ADR 0595 §4): the accepted verdicts and the bus module's created and refused, durably. - if want := []string{"mesh-delivery-assigner.views.wants", "mesh-delivery-indexer.take-in.pending", "mesh-delivery-indexer.take-in.taken", "mesh-delivery-indexer.views.modules", "mesh-catalog.manifest-accepted", "mesh-delivery.source-moved", "mesh-delivery.filling", "mesh-delivery.full", "mesh-delivery.room", "mesh-delivery.record-opened", "mesh-delivery.record-ended", "mesh-catalog.filling", "mesh-catalog.full", "mesh-catalog.room", "mesh-wants.filling", "mesh-wants.full", "mesh-wants.room", "mesh-grants.filling", "mesh-grants.full", "mesh-grants.room", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.step-started", "mesh-pipeline-runner.step-ended", "mesh-broker.created", "mesh-broker.refused"}; !reflect.DeepEqual(m.Consumes, want) { + if want := []string{"mesh-delivery-assigner.views.wants", "mesh-delivery-indexer.take-in.pending", "mesh-delivery-indexer.take-in.taken", "mesh-delivery-indexer.views.modules", "mesh-catalog.manifest-accepted", "mesh-delivery.source-moved", "mesh-delivery.filling", "mesh-delivery.full", "mesh-delivery.room", "mesh-delivery.record-opened", "mesh-delivery.record-ended", "mesh-catalog.filling", "mesh-catalog.full", "mesh-catalog.room", "mesh-wants.filling", "mesh-wants.full", "mesh-wants.room", "mesh-grants.filling", "mesh-grants.full", "mesh-grants.room", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.step-started", "mesh-pipeline-runner.step-ended", "mesh-broker.created", "mesh-broker.refused", "mesh-conditions.findings-asked"}; !reflect.DeepEqual(m.Consumes, want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } // 1952/WM1: the watchman holds the seat mesh-watchdogs at mesh scope, its first seat (novox/hq ADR 0564 §1, §4). diff --git a/modules/mesh-delivery-watchman/module.json b/modules/mesh-delivery-watchman/module.json index bd081f2f..908c9988 100644 --- a/modules/mesh-delivery-watchman/module.json +++ b/modules/mesh-delivery-watchman/module.json @@ -30,6 +30,9 @@ "mesh-delivery-indexer.indexer_read_modules", "seat:mesh-hand-acts.hand-act" ], + "streams": [ + { "name": "mesh-delivery-watchman-findings", "events": ["finding-written.*", "finding-cleared.*", "findings-stated.*"], "max-age-days": 7, "max-mib": 16 } + ], "consumes": [ "mesh-delivery-assigner.views.wants", "mesh-delivery-indexer.take-in.pending", @@ -55,7 +58,8 @@ "mesh-pipeline-runner.step-started", "mesh-pipeline-runner.step-ended", "mesh-broker.created", - "mesh-broker.refused" + "mesh-broker.refused", + "mesh-conditions.findings-asked" ], "tools": [ "watchman_findings", diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/tokens_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/tokens_test.go index 5cb2e3f2..fa8dfbe3 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/tokens_test.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/tokens_test.go @@ -119,7 +119,7 @@ func theManifestSaysWhatTheCodeDoes(t *testing.T) { for _, e := range EndEvents { heard = append(heard, Seat+"."+e) } - heard = append(heard, assignerModule+".views."+assignmentsState) + heard = append(heard, assignerModule+".views."+assignmentsState, "mesh-conditions.findings-asked") // the keeper's ask (design 47 G §387) if !slices.Equal(m.Consumes, heard) { t.Errorf("consumes %v, the end events are %v", m.Consumes, heard) } diff --git a/modules/mesh-enrolment/module.json b/modules/mesh-enrolment/module.json index 302cdf2b..506c568a 100644 --- a/modules/mesh-enrolment/module.json +++ b/modules/mesh-enrolment/module.json @@ -1,6 +1,6 @@ { "module": "mesh-enrolment", - "version": "1", + "version": "0.1", "runs-as": "mesh-enrolment", "claims": [ { @@ -16,12 +16,16 @@ ] } ], + "streams": [ + { "name": "mesh-enrolment-findings", "events": ["finding-written.*", "finding-cleared.*", "findings-stated.*"], "max-age-days": 7, "max-mib": 16 } + ], "consumes": [ "mesh-node-enroller.node-enrolled", "mesh-node-enroller.enrolment-refused", "mesh-node-enroller.enrolment-revoked", "mesh-node-enroller.enrolment-expired", - "mesh-delivery-assigner.views.assignments" + "mesh-delivery-assigner.views.assignments", + "mesh-conditions.findings-asked" ], "uses": [ "operator-channel" From 8cfc24fc2eead75849e0f0354109b951764239d9 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 12:14:50 +0100 Subject: [PATCH 20/33] Remove a binary built by mistake in the checker's folder Builds: none (housekeeping of CK2) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ From 5e0c04c3cfbc0a718690330a0f5ac4e2e6cec114 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 12:15:39 +0100 Subject: [PATCH 21/33] CT10: meshcatalogclient follows the catalog half per watched branch, a first view then each catalog-revision of that branch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The view of a branch is named mesh-catalog.catalog-revisions.; two branches give two views. A revision heard and not held asks revision once more, said in the log. Builds: design 47 G §358, §196, §479, §495 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../go/clients/meshcatalogclient/catalog.go | 227 ++++++++++++++++++ .../meshcatalogclient/catalog_view_test.go | 147 ++++++++++++ 2 files changed, 374 insertions(+) create mode 100644 contracts/go/clients/meshcatalogclient/catalog.go create mode 100644 contracts/go/clients/meshcatalogclient/catalog_view_test.go diff --git a/contracts/go/clients/meshcatalogclient/catalog.go b/contracts/go/clients/meshcatalogclient/catalog.go new file mode 100644 index 00000000..95d46cf5 --- /dev/null +++ b/contracts/go/clients/meshcatalogclient/catalog.go @@ -0,0 +1,227 @@ +package meshcatalogclient + +import ( + "context" + "encoding/json" + "fmt" + "maps" + "strconv" + "sync" + "time" + + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-sdk/go/buscalls" + "git.novox.be/novox/mesh-sdk/go/keys" + "git.novox.be/novox/mesh-sdk/go/views" +) + +// RevisionVerb is the seat verb a part takes its first view of the catalog half by, as its manifest's invokes names it. +const RevisionVerb = "seat:" + seat.Seat + "." + seat.VerbRevision + +// The kinds a catalog graph revision is of. +const ( + KindTrunk = "trunk" + KindCandidate = "candidate" +) + +// Half is one catalog graph revision whole: the revision with its entries by module. +type Half struct { + Revision seat.CatalogRevision + Entries map[string]seat.CatalogEntry +} + +// ViewName is how the catalog half's revision is named in a part's first-view status (views.Status.View, WaitsFor). +func ViewName(revision int64) string { return strconv.FormatInt(revision, 10) } + +// catalogCreated stands for the view's Created in its follower: a revision view is never made again, so one fixed time, +// never zero, lets the follower tell a revision heard from one it holds. +var catalogCreated = time.Unix(0, 0).UTC() + +// Catalog is a part's view of the catalog half of one watched branch (design 47 G §358, §479): a first view of the +// branch's newest trunk revision asked of the seat's verb `revision`, then kept from its event `catalog-revision` of that +// branch, built on go/views' Follower, so the runtime's tools `_first_views` and `_first_view_again` read +// and steer it as every first view. The view is named `mesh-catalog.catalog-revisions.`, so a part that watches two +// branches makes two Catalogs, two views. A `catalog-revision` the view missed (a revision heard that it does not hold) +// asks `revision` once more, said in the log (§196, ADR 0615 §5), and is waited for, said in the status and past +// views.WaitsBound raised as view-waits. +type Catalog struct { + Follower *views.Follower + // Branch is the watched branch the view is of, as the caller names it. + Branch string + + call Call + mu sync.Mutex + // held is the half the view holds; paging the one being read, page by page, of one revision. + held *Half + paging *Half + // waits is the revision waited for, 0 when none. + waits int64 +} + +// NewCatalog is a Catalog of branch asking the seat through call. Nothing is asked before Start. +func NewCatalog(call Call, branch string) (*Catalog, error) { + if err := keys.CheckName("branch", branch); err != nil { + return nil, err + } + c := &Catalog{call: call, Branch: branch} + c.Follower = &views.Follower{Owner: seat.Seat, Tool: RevisionVerb, State: views.BranchState(seat.StateCatalogRevisions, branch), Ask: c.ask} + return c, nil +} + +// WireCatalog is a Catalog of branch adopted by a stdio part's views.Reader: the reader writes its conditions and its +// view-waits finding, reads it as `mesh-catalog.catalog-revisions.` (a key a module, its value the module's +// entry), lists and asks it again in its first-view tools, and at its Start subscribes the seat's event catalog-revision +// (the part's manifest consumes it) into Hear and asks the first view. changed, when not nil, is the Follower's Changed. +// The part declares the seat's verb revision in invokes. Call it once per watched branch. +func WireCatalog(r *views.Reader, call Call, branch string, changed func(module string, entry json.RawMessage, deleted bool, at time.Time)) (*Catalog, error) { + c, err := NewCatalog(call, branch) + if err != nil { + return nil, err + } + c.Follower.Changed = changed + if err := r.Adopt(c.Follower, seat.Seat, seat.EventCatalogRevision, c.Hear); err != nil { + return nil, err + } + return c, nil +} + +// WireCatalogProcess is WireCatalog for a process's views.ProcessReader: its picture +// (`mesh-catalog_catalog-revisions.`) reads the half as a read-only bucket, and the seat's event catalog-revision +// on the module's durable consumer goes to Hear. +func WireCatalogProcess(r *views.ProcessReader, call Call, branch string, changed func(module string, entry json.RawMessage, deleted bool, at time.Time)) (*Catalog, *views.Picture, error) { + c, err := NewCatalog(call, branch) + if err != nil { + return nil, nil, err + } + c.Follower.Changed = changed + p, err := r.Adopt(c.Follower, seat.Seat, seat.EventCatalogRevision, c.Hear) + if err != nil { + return nil, nil, err + } + return c, p, nil +} + +// Start asks the first view, the branch's newest trunk revision, until it is answered or ctx ends. +func (c *Catalog) Start(ctx context.Context) error { return c.Follower.Start(ctx) } + +// Ready is closed once the first view is answered. +func (c *Catalog) Ready() <-chan struct{} { return c.Follower.Ready() } + +// View is the half the view holds; false while it has none, when the part acts on nothing new. +func (c *Catalog) View() (Half, bool) { + c.mu.Lock() + defer c.mu.Unlock() + if c.held == nil { + return Half{}, false + } + return Half{Revision: c.held.Revision, Entries: maps.Clone(c.held.Entries)}, true +} + +// Entry is one module's entry in the half the view holds. +func (c *Catalog) Entry(module string) (seat.CatalogEntry, bool) { + c.mu.Lock() + defer c.mu.Unlock() + if c.held == nil { + return seat.CatalogEntry{}, false + } + e, ok := c.held.Entries[module] + return e, ok +} + +// Hear takes one `catalog-revision` event, as the part's durable consumer of the seat's events hands it. A candidate's +// revision and another branch's move nothing here; a trunk revision of this branch the view holds, or one older, is passed +// over; a newer one is a revision the view missed: waited for, said in the log, and asked again on its own goroutine, so +// the consumer is never held while the holder is away. Data that is no such event is an error. +func (c *Catalog) Hear(ctx context.Context, data []byte) error { + var ev seat.CatalogRevisionEvent + if err := json.Unmarshal(data, &ev); err != nil || ev.Revision <= 0 { + return fmt.Errorf("meshcatalogclient: not a %s event: %s", seat.EventCatalogRevision, clipBytes(data)) + } + if ev.Kind != KindTrunk || ev.Branch != keys.EscapeName(c.Branch) { + return nil + } + c.mu.Lock() + held := int64(0) + if c.held != nil { + held = c.held.Revision.Revision + } + if ev.Revision > c.waits { + c.waits = ev.Revision + } + c.mu.Unlock() + if ev.Revision <= held { + return nil + } + // The follower keeps the newest revision heard, so an ask that was running when it was heard is asked once more. + c.Follower.Apply(views.Event{Owner: seat.Seat, State: c.Follower.State, Revision: uint64(ev.Revision), Created: catalogCreated, Gap: true}) + c.Follower.WaitFor(ctx, ViewName(ev.Revision)) + if c.Follower.Logf != nil { + c.Follower.Logf("meshcatalogclient: catalog revision %d of %s said, the view holds %d: %s is asked once more", ev.Revision, c.Branch, held, RevisionVerb) + } + go func() { + _ = c.Follower.Gap(ctx, fmt.Sprintf("catalog revision %d of %s said, the view holds %d", ev.Revision, c.Branch, held)) + }() + return nil +} + +// ask is the Follower's Ask: one page of the branch's newest trunk revision, the next pages of the same revision; once +// whole, it is the half the view holds. +func (c *Catalog) ask(_ context.Context, after string) (views.Answer, error) { + in := seat.RevisionInput{After: after, Branch: keys.EscapeName(c.Branch)} + c.mu.Lock() + if after != "" && c.paging != nil { + in.Revision = c.paging.Revision.Revision + } + c.mu.Unlock() + a, err := askRevision(c.call, in) + if err != nil { + return views.Answer{}, err + } + if a.Revision.Kind != KindTrunk { + return views.Answer{}, fmt.Errorf("%s answered a %s revision for the trunk", RevisionVerb, a.Revision.Kind) + } + c.mu.Lock() + if after == "" || c.paging == nil { + c.paging = &Half{Revision: a.Revision, Entries: map[string]seat.CatalogEntry{}} + } else if c.paging.Revision.Revision != a.Revision.Revision { + was := c.paging.Revision.Revision + c.paging = nil + c.mu.Unlock() + return views.Answer{}, fmt.Errorf("%s answered revision %d for a page of revision %d", RevisionVerb, a.Revision.Revision, was) + } + values := map[string]json.RawMessage{} + for _, e := range a.Entries { + c.paging.Entries[e.Module] = e + raw, _ := json.Marshal(e) + values[e.Module] = raw + } + rev := a.Revision.Revision + holds := false + if a.Next == "" { + c.held, c.paging = c.paging, nil + if c.waits <= rev { + c.waits, holds = 0, true + } + } + c.mu.Unlock() + if holds { + c.Follower.Holds(ViewName(rev)) + } + return views.Answer{V: 1, Owner: seat.Seat, State: c.Follower.State, Revision: uint64(rev), Created: catalogCreated, Values: values, Next: a.Next, At: time.Now()}, nil +} + +func askRevision(call Call, in seat.RevisionInput) (seat.RevisionAnswer, error) { + raw, err := call(RevisionVerb, in) + if err != nil { + return seat.RevisionAnswer{}, err + } + answer, said, ok := buscalls.Unwrap(raw) + if !ok { + return seat.RevisionAnswer{}, fmt.Errorf("%s answered: %s", RevisionVerb, said) + } + var a seat.RevisionAnswer + if err := json.Unmarshal(answer, &a); err != nil || a.Revision.Revision <= 0 { + return seat.RevisionAnswer{}, fmt.Errorf("%s answered no revision: %s", RevisionVerb, clipBytes(answer)) + } + return a, nil +} diff --git a/contracts/go/clients/meshcatalogclient/catalog_view_test.go b/contracts/go/clients/meshcatalogclient/catalog_view_test.go new file mode 100644 index 00000000..6077fede --- /dev/null +++ b/contracts/go/clients/meshcatalogclient/catalog_view_test.go @@ -0,0 +1,147 @@ +package meshcatalogclient + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "sync" + "testing" + "time" + + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-sdk/go/keys" + "git.novox.be/novox/mesh-sdk/go/views" +) + +// fakeRevisions answers the verb revision per branch from the newest revision each branch holds, and counts each ask. +type fakeRevisions struct { + mu sync.Mutex + newest map[string]int64 // by escaped branch + asks map[string]int + branches []string +} + +func (f *fakeRevisions) call(key string, body any) (json.RawMessage, error) { + in := body.(seat.RevisionInput) + f.mu.Lock() + defer f.mu.Unlock() + rev, ok := f.newest[in.Branch] + if !ok { + return nil, fmt.Errorf("asked for the branch %q", in.Branch) + } + f.asks[in.Branch]++ + return json.Marshal(seat.RevisionAnswer{ + Revision: seat.CatalogRevision{Revision: rev, Kind: KindTrunk, Branch: in.Branch}, + Entries: []seat.CatalogEntry{{Module: fmt.Sprintf("m-of-%s-%d", in.Branch, rev)}}, + }) +} + +func (f *fakeRevisions) set(branch string, rev int64) { + f.mu.Lock() + defer f.mu.Unlock() + f.newest[keys.EscapeName(branch)] = rev +} + +func (f *fakeRevisions) asked(branch string) int { + f.mu.Lock() + defer f.mu.Unlock() + return f.asks[keys.EscapeName(branch)] +} + +func revisionEvent(branch, kind string, rev int64) []byte { + b, _ := json.Marshal(seat.CatalogRevisionEvent{V: 1, Branch: keys.EscapeName(branch), Kind: kind, Revision: rev}) + return b +} + +// CT10 (design 47 G §358, §479, §196): the catalog half is a view per watched branch, named by its caller: two branches +// give two views, each named `mesh-catalog.catalog-revisions.`, each following only its own branch's +// `catalog-revision`; a revision the view missed asks `revision` once more, said in the log. +func TestCatalogIsAViewPerWatchedBranch(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + f := &fakeRevisions{newest: map[string]int64{"main": 3, keys.EscapeName("release/1"): 7}, asks: map[string]int{}} + var logMu sync.Mutex + var logged []string + logf := func(format string, args ...any) { + logMu.Lock() + defer logMu.Unlock() + logged = append(logged, fmt.Sprintf(format, args...)) + } + main, err := NewCatalog(f.call, "main") + if err != nil { + t.Fatal(err) + } + rel, err := NewCatalog(f.call, "release/1") + if err != nil { + t.Fatal(err) + } + for _, c := range []*Catalog{main, rel} { + c.Follower.Wait = time.Nanosecond + c.Follower.Logf = logf + } + if main.Follower.State == rel.Follower.State || main.Follower.State != "catalog-revisions.main" || rel.Follower.State != views.BranchState(seat.StateCatalogRevisions, "release/1") { + t.Fatalf("views are named %q and %q", main.Follower.State, rel.Follower.State) + } + for _, c := range []*Catalog{main, rel} { + if err := c.Start(ctx); err != nil { + t.Fatal(err) + } + <-c.Ready() + } + if h, ok := main.View(); !ok || h.Revision.Revision != 3 || len(h.Entries) != 1 { + t.Fatalf("main holds %+v, %v", h, ok) + } + if h, ok := rel.View(); !ok || h.Revision.Revision != 7 { + t.Fatalf("release/1 holds %+v, %v", h, ok) + } + + // An event of the other branch, a candidate's and one the view holds ask nothing. + for _, data := range [][]byte{revisionEvent("release/1", KindTrunk, 8), revisionEvent("main", KindCandidate, 4), revisionEvent("main", KindTrunk, 3)} { + if err := main.Hear(ctx, data); err != nil { + t.Fatal(err) + } + } + if n := f.asked("main"); n != 1 { + t.Errorf("main asked %d times, want only its first view", n) + } + + // A missed revision asks `revision` once more, and is said. + f.set("main", 4) + if err := main.Hear(ctx, revisionEvent("main", KindTrunk, 4)); err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(5 * time.Second) + for { + if h, _ := main.View(); h.Revision.Revision == 4 { + break + } + if time.Now().After(deadline) { + t.Fatal("main never held revision 4") + } + time.Sleep(time.Millisecond) + } + if n := f.asked("release/1"); n != 1 { + t.Errorf("release/1 asked %d times after main's revision", n) + } + logMu.Lock() + said := false + for _, l := range logged { + if strings.Contains(l, "catalog revision 4 of main said") { + said = true + } + } + logMu.Unlock() + if !said { + t.Errorf("the gap was not said in the log: %q", logged) + } + + for _, data := range []string{`{}`, `not json`} { + if err := main.Hear(ctx, []byte(data)); err == nil { + t.Errorf("Hear(%s) took it", data) + } + } + if _, err := NewCatalog(f.call, ""); err == nil { + t.Error("a Catalog of no branch was made") + } +} From ce286795e0183d67f1bb8e883fcaed67e53a0ecd Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 12:17:16 +0100 Subject: [PATCH 22/33] CT10: a branch with no revision yet is a ready, empty catalog view MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §479, §495 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../go/clients/meshcatalogclient/catalog.go | 15 +++++++++++++++ .../meshcatalogclient/catalog_view_test.go | 17 +++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/contracts/go/clients/meshcatalogclient/catalog.go b/contracts/go/clients/meshcatalogclient/catalog.go index 95d46cf5..95403eb4 100644 --- a/contracts/go/clients/meshcatalogclient/catalog.go +++ b/contracts/go/clients/meshcatalogclient/catalog.go @@ -3,9 +3,11 @@ package meshcatalogclient import ( "context" "encoding/json" + "errors" "fmt" "maps" "strconv" + "strings" "sync" "time" @@ -174,6 +176,10 @@ func (c *Catalog) ask(_ context.Context, after string) (views.Answer, error) { } c.mu.Unlock() a, err := askRevision(c.call, in) + if errors.Is(err, errNoneYet) { + // A branch with no revision yet is ready and empty, and nothing waits for it (§479, §495). + return views.Answer{V: 1, Owner: seat.Seat, State: c.Follower.State, Created: catalogCreated, Values: map[string]json.RawMessage{}, At: time.Now()}, nil + } if err != nil { return views.Answer{}, err } @@ -210,13 +216,22 @@ func (c *Catalog) ask(_ context.Context, after string) (views.Answer, error) { return views.Answer{V: 1, Owner: seat.Seat, State: c.Follower.State, Revision: uint64(rev), Created: catalogCreated, Values: values, Next: a.Next, At: time.Now()}, nil } +// errNoneYet is the seat's answer `none yet`: the branch has no revision yet. +var errNoneYet = errors.New("none yet") + func askRevision(call Call, in seat.RevisionInput) (seat.RevisionAnswer, error) { raw, err := call(RevisionVerb, in) if err != nil { + if strings.Contains(err.Error(), views.NoneYet) { + return seat.RevisionAnswer{}, errNoneYet + } return seat.RevisionAnswer{}, err } answer, said, ok := buscalls.Unwrap(raw) if !ok { + if strings.Contains(said, views.NoneYet) { + return seat.RevisionAnswer{}, errNoneYet + } return seat.RevisionAnswer{}, fmt.Errorf("%s answered: %s", RevisionVerb, said) } var a seat.RevisionAnswer diff --git a/contracts/go/clients/meshcatalogclient/catalog_view_test.go b/contracts/go/clients/meshcatalogclient/catalog_view_test.go index 6077fede..7af15ae0 100644 --- a/contracts/go/clients/meshcatalogclient/catalog_view_test.go +++ b/contracts/go/clients/meshcatalogclient/catalog_view_test.go @@ -27,6 +27,9 @@ func (f *fakeRevisions) call(key string, body any) (json.RawMessage, error) { f.mu.Lock() defer f.mu.Unlock() rev, ok := f.newest[in.Branch] + if ok && rev == 0 { + return nil, fmt.Errorf("revision of %s: none yet", in.Branch) + } if !ok { return nil, fmt.Errorf("asked for the branch %q", in.Branch) } @@ -136,6 +139,20 @@ func TestCatalogIsAViewPerWatchedBranch(t *testing.T) { t.Errorf("the gap was not said in the log: %q", logged) } + // A branch with no revision yet is ready and empty, and nothing waits for it. + f.set("new", 0) + none, err := NewCatalog(f.call, "new") + if err != nil { + t.Fatal(err) + } + if err := none.Start(ctx); err != nil { + t.Fatal(err) + } + <-none.Ready() + if _, ok := none.View(); ok || none.Follower.Status().WaitsFor != "" { + t.Errorf("a branch with no revision yet holds a view or waits: %+v", none.Follower.Status()) + } + for _, data := range []string{`{}`, `not json`} { if err := main.Hear(ctx, []byte(data)); err == nil { t.Errorf("Hear(%s) took it", data) From 940d42c659837062ebd973c99e264784bd5c1c0c Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 12:20:54 +0100 Subject: [PATCH 23/33] CT5: the build seats' steps takes drop and why, fetch gains builds, a build names what it resolved, steps are keyed by asker, toolchain-reported and step-progress; regenerated MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit steps callers: the runner, the indexer, the operator and sessions (D-18); BLD12 builds in the fetch; BLD13 resolved and the toolchain digest; BLD14 the step key .. beside main's .; the events toolchain-reported and step-progress on the three build seats, and step-progress on mesh-wants. Builds: design 47 G §463, §480, §465, §466, §467, §481, §483, §492, §496 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- contracts/go/seats/meshwants/contract.go | 30 ++- contracts/go/seats/meshwants/contract_test.go | 2 + contracts/go/seats/nodebuildgo/contract.go | 146 ++++++++++++++- .../go/seats/nodebuildgo/contract_test.go | 16 ++ contracts/go/seats/nodebuildimage/contract.go | 146 ++++++++++++++- .../go/seats/nodebuildimage/contract_test.go | 16 ++ contracts/go/seats/nodebuildnode/contract.go | 146 ++++++++++++++- .../go/seats/nodebuildnode/contract_test.go | 16 ++ contracts/parts/build-step.json | 174 +++++++++++++++++- .../seats/mesh-wants/fixtures/events.json | 11 ++ contracts/seats/mesh-wants/seat.json | 39 ++++ .../seats/node-build-go/fixtures/events.json | 23 +++ .../seats/node-build-go/fixtures/states.json | 12 ++ .../seats/node-build-go/fixtures/verbs.json | 24 +++ contracts/seats/node-build-go/seat.json | 9 +- .../node-build-image/fixtures/events.json | 23 +++ .../node-build-image/fixtures/states.json | 12 ++ .../node-build-image/fixtures/verbs.json | 24 +++ contracts/seats/node-build-image/seat.json | 9 +- .../node-build-node/fixtures/events.json | 23 +++ .../node-build-node/fixtures/states.json | 12 ++ .../seats/node-build-node/fixtures/verbs.json | 24 +++ contracts/seats/node-build-node/seat.json | 9 +- contracts/ts/seats/meshwants/index.d.ts | 15 ++ contracts/ts/seats/meshwants/index.js | 9 + contracts/ts/seats/nodebuildgo/index.d.ts | 29 +++ contracts/ts/seats/nodebuildgo/index.js | 41 +++++ .../seats/nodebuildgo/parts/build-step.d.ts | 64 ++++++- .../ts/seats/nodebuildgo/parts/build-step.js | 2 + contracts/ts/seats/nodebuildimage/index.d.ts | 29 +++ contracts/ts/seats/nodebuildimage/index.js | 41 +++++ .../nodebuildimage/parts/build-step.d.ts | 64 ++++++- .../seats/nodebuildimage/parts/build-step.js | 2 + contracts/ts/seats/nodebuildnode/index.d.ts | 29 +++ contracts/ts/seats/nodebuildnode/index.js | 41 +++++ .../seats/nodebuildnode/parts/build-step.d.ts | 64 ++++++- .../seats/nodebuildnode/parts/build-step.js | 2 + 37 files changed, 1331 insertions(+), 47 deletions(-) diff --git a/contracts/go/seats/meshwants/contract.go b/contracts/go/seats/meshwants/contract.go index 97003e33..9b9f4754 100644 --- a/contracts/go/seats/meshwants/contract.go +++ b/contracts/go/seats/meshwants/contract.go @@ -39,6 +39,7 @@ const ( EventRoom = "room" EventStepEnded = "step-ended" EventVersionDelivered = "version-delivered" + EventStepProgress = "step-progress" ) // Verbs is every verb the seat serves, in its definition's order: a new copy each call. @@ -48,7 +49,7 @@ func Verbs() []string { // Events is every event published on the seat's subject: a new copy each call. func Events() []string { - return []string{EventPolicyChanged, EventAssignmentRevision, EventDifference, EventWanted, EventWaitHeld, EventWaitEnded, EventFilling, EventFull, EventRoom, EventStepEnded, EventVersionDelivered} + return []string{EventPolicyChanged, EventAssignmentRevision, EventDifference, EventWanted, EventWaitHeld, EventWaitEnded, EventFilling, EventFull, EventRoom, EventStepEnded, EventVersionDelivered, EventStepProgress} } // Family is where the seat's events are kept: the shared events stream, filtered to the seat's subjects. @@ -99,6 +100,17 @@ const StreamVersionDelivered = "SEAT_mesh-wants_version-delivered" // SubjectVersionDelivered is where version-delivered is published. const SubjectVersionDelivered = "mesh.seat.mesh-wants.event.version-delivered" +// StreamStepProgress is the stream step-progress is kept in. +const StreamStepProgress = "SEAT_mesh-wants_step-progress" + +// SubjectStepProgress is where step-progress is published, on mesh.seat.mesh-wants.event.step-progress.. +func SubjectStepProgress(node string) string { + return "mesh.seat.mesh-wants.event.step-progress." + node +} + +// SubjectStepProgressAll hears every step-progress. +const SubjectStepProgressAll = "mesh.seat.mesh-wants.event.step-progress.*" + // AcceptSubjectStep is where step is sent. const AcceptSubjectStep = "mesh.seat.mesh-wants.accept.step" @@ -823,6 +835,22 @@ type VersionDeliveredEvent struct { Version string `json:"version"` } +// StepProgressEvent is the body of the event step-progress. +type StepProgressEvent struct { + Ask string `json:"ask"` + Asker string `json:"asker"` + At time.Time `json:"at"` + Phase string `json:"phase"` +} + +// The values of StepProgressEvent's phase. +const ( + StepProgressEventPhaseFetch = "fetch" + StepProgressEventPhaseBuild = "build" + StepProgressEventPhaseRun = "run" + StepProgressEventPhaseReport = "report" +) + // WantsByWantNodeModuleValue is one value of the state wants under the key want... type WantsByWantNodeModuleValue struct { // again or skip diff --git a/contracts/go/seats/meshwants/contract_test.go b/contracts/go/seats/meshwants/contract_test.go index dcfe1e6c..6a6825e6 100644 --- a/contracts/go/seats/meshwants/contract_test.go +++ b/contracts/go/seats/meshwants/contract_test.go @@ -43,6 +43,8 @@ func TestFixtures(t *testing.T) { decode: func(b []byte) (any, error) { var v StepEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-wants\\.event\\.step-ended\\.[^.*> ]+$")}, {name: "a version delivered with the machines away", subject: "mesh.seat.mesh-wants.event.version-delivered", given: "{\"away\":[\"laptop\"],\"module\":\"mesh-sdk\",\"parts\":{\"mesh-sdk:go\":\"sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\"},\"v\":1,\"version\":\"0.1.15\"}", wire: "{\"away\":[\"laptop\"],\"module\":\"mesh-sdk\",\"parts\":{\"mesh-sdk:go\":\"sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\"},\"v\":1,\"version\":\"0.1.15\"}", decode: func(b []byte) (any, error) { var v VersionDeliveredEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-wants\\.event\\.version-delivered$")}, + {name: "a composition step's progress", subject: "mesh.seat.mesh-wants.event.step-progress.ace", given: "{\"ask\":\"a1b2\",\"asker\":\"assigner\",\"at\":\"2026-11-11T10:01:00Z\",\"phase\":\"build\"}", wire: "{\"ask\":\"a1b2\",\"asker\":\"assigner\",\"at\":\"2026-11-11T10:01:00Z\",\"phase\":\"build\"}", + decode: func(b []byte) (any, error) { var v StepProgressEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-wants\\.event\\.step-progress\\.[^.*> ]+$")}, {name: "a want", subject: "", given: "{\"fp\":\"ab12\",\"rule\":3,\"v\":1}", wire: "{\"fp\":\"ab12\",\"rule\":3,\"v\":1}", decode: func(b []byte) (any, error) { var v WantsByWantNodeModuleValue diff --git a/contracts/go/seats/nodebuildgo/contract.go b/contracts/go/seats/nodebuildgo/contract.go index 34785b37..7b1eaece 100644 --- a/contracts/go/seats/nodebuildgo/contract.go +++ b/contracts/go/seats/nodebuildgo/contract.go @@ -33,7 +33,9 @@ const ( ) const ( - EventStepEnded = "step-ended" + EventStepEnded = "step-ended" + EventToolchainReported = "toolchain-reported" + EventStepProgress = "step-progress" ) // Verbs is every verb the seat serves, in its definition's order: a new copy each call. @@ -43,7 +45,7 @@ func Verbs() []string { // Events is every event published on the seat's subject: a new copy each call. func Events() []string { - return []string{EventStepEnded} + return []string{EventStepEnded, EventToolchainReported, EventStepProgress} } // Family is where the seat's events are kept: the shared events stream, filtered to the seat's subjects. @@ -61,6 +63,28 @@ func SubjectStepEnded(node string) string { return "mesh.seat.node-build-go.even // SubjectStepEndedAll hears every step-ended. const SubjectStepEndedAll = "mesh.seat.node-build-go.event.step-ended.*" +// StreamToolchainReported is the stream toolchain-reported is kept in. +const StreamToolchainReported = "SEAT_node-build-go_toolchain-reported" + +// SubjectToolchainReported is where toolchain-reported is published, on mesh.seat.node-build-go.event.toolchain-reported.. +func SubjectToolchainReported(node string) string { + return "mesh.seat.node-build-go.event.toolchain-reported." + node +} + +// SubjectToolchainReportedAll hears every toolchain-reported. +const SubjectToolchainReportedAll = "mesh.seat.node-build-go.event.toolchain-reported.*" + +// StreamStepProgress is the stream step-progress is kept in. +const StreamStepProgress = "SEAT_node-build-go_step-progress" + +// SubjectStepProgress is where step-progress is published, on mesh.seat.node-build-go.event.step-progress.. +func SubjectStepProgress(node string) string { + return "mesh.seat.node-build-go.event.step-progress." + node +} + +// SubjectStepProgressAll hears every step-progress. +const SubjectStepProgressAll = "mesh.seat.node-build-go.event.step-progress.*" + // AcceptSubjectDelivery is where delivery is sent. const AcceptSubjectDelivery = "mesh.seat.node-build-go.accept.delivery" @@ -86,6 +110,47 @@ const ( StateDetectedSteps = "detected-steps" ) +// StepsByAskerNodeIDKey is the key of one value of the state steps under the form "..". +func StepsByAskerNodeIDKey(asker, node, id string) string { return asker + "." + node + "." + id } + +// ParseStepsByAskerNodeIDKey reads the parts of a key of the form "..", in the order of the form; the last result is false for a key of another form. +func ParseStepsByAskerNodeIDKey(key string) (string, string, string, bool) { + rest := key + v0, after0, found0 := strings.Cut(rest, ".") + if !found0 || v0 == "" { + return "", "", "", false + } + rest = after0 + v2, after2, found2 := strings.Cut(rest, ".") + if !found2 || v2 == "" { + return "", "", "", false + } + rest = after2 + if rest == "" { + return "", "", "", false + } + v4 := rest + rest = "" + if rest != "" { + return "", "", "", false + } + return v0, v2, v4, true +} + +// DecodeStepsByAskerNodeIDValue reads one value of the state steps under the form "..", refusing a field StepsByAskerNodeIDValue does not know and any text after the value. +func DecodeStepsByAskerNodeIDValue(raw []byte) (StepsByAskerNodeIDValue, error) { + var v StepsByAskerNodeIDValue + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.DisallowUnknownFields() + if err := dec.Decode(&v); err != nil { + return v, err + } + if dec.More() { + return v, errors.New("text after the value") + } + return v, nil +} + // StepsByNodeIDKey is the key of one value of the state steps under the form ".". func StepsByNodeIDKey(node, id string) string { return node + "." + id } @@ -317,11 +382,21 @@ type BuildAnswer struct { Toolchain *ToolchainUsed `json:"toolchain,omitempty"` } +// BuiltArtifactResolvedItem is a type of the contract. +type BuiltArtifactResolvedItem struct { + // sha256: of the package's bytes as the build fetched them + Digest string `json:"digest"` + Package string `json:"package"` + Version string `json:"version"` +} + // BuiltArtifact is one artifact a build made: its digest and the reference it was published under. type BuiltArtifact struct { Artifact string `json:"artifact"` Digest string `json:"digest"` Reference string `json:"reference,omitempty"` + // one per mesh package the build resolved (the catalog graph entry's packages, the edges of kind 1) + Resolved []BuiltArtifactResolvedItem `json:"resolved,omitempty"` } // Capacity is how many steps the node runs at once. @@ -396,6 +471,16 @@ type FetchBesideItem struct { Repository string `json:"repository"` } +// FetchBuildsItem is a type of the contract. +type FetchBuildsItem struct { + Artifact string `json:"artifact"` + // the member's build digest, sha256: + Digest string `json:"digest"` + Module string `json:"module"` + // the name in its ecosystem (the Go module path, the npm name) + Package string `json:"package"` +} + // FetchPackage is the TypeScript module's group package: the group's tagged version of the SDK ({name, version}), placed from the registry in each member whatever its lock names, or the group's package member ({repository, commit, ref}), packed by the fetch and served to the members in place of the registry's. type FetchPackage struct { Commit string `json:"commit,omitempty"` @@ -411,6 +496,8 @@ type Fetch struct { Base string `json:"base,omitempty"` // the sibling repositories placed beside the fetched tree Beside []FetchBesideItem `json:"beside,omitempty"` + // a group run's package members, each by its build's digest: the build module reads each from the artifact store at artifact-store:////blobs/, checks the bytes against the digest and lays them as the group package; a package named here is named in neither packages nor package + Builds []FetchBuildsItem `json:"builds,omitempty"` // the full commit Commit string `json:"commit"` // the module's directory in the repository ("." for its root), with paths @@ -735,6 +822,22 @@ type StepNames struct { Repositories []string `json:"repositories"` } +// StepProgress is a running ask's progress, said once a minute while the holder runs it (design 47 G section 496). +type StepProgress struct { + Ask string `json:"ask"` + Asker string `json:"asker"` + At time.Time `json:"at"` + Phase string `json:"phase"` +} + +// The values of StepProgress's phase. +const ( + StepProgressPhaseFetch = "fetch" + StepProgressPhaseBuild = "build" + StepProgressPhaseRun = "run" + StepProgressPhaseReport = "report" +) + // StepResult is what a step answered, by the kind of ask: a build's artifacts, toolchain, imports, out-of-step modules and the artifact it kept; a check's fetch and levels; a compose's built, key and verdict; a run's built, key, exit, output and said; a declined ask's declined, needs and lacks. The fields of the kinds do not meet, so one layout holds the five. type StepResult struct { Artifacts []BuiltArtifact `json:"artifacts,omitempty"` @@ -828,8 +931,20 @@ type Toolchain struct { Version string `json:"version"` } +// ToolchainReported is a node's toolchain of a language, said at the build module's start and each time its toolchains key of the language changes (design 47 G section 492). +type ToolchainReported struct { + // the digest of the program of the provision the build module requires for that language's toolchain, never the build module's own + Digest string `json:"digest"` + Language string `json:"language"` + Node string `json:"node"` + Recipe string `json:"recipe"` + Version string `json:"version"` +} + // ToolchainUsed is the toolchain a build was made with: the language, the installed version and the build module's recipe. type ToolchainUsed struct { + // the building node's digest of its toolchain program (kind 3's entry), never the build module's own program + Digest string `json:"digest,omitempty"` Language string `json:"language"` Recipe string `json:"recipe"` Version string `json:"version"` @@ -912,20 +1027,28 @@ const ( type StepsInput struct { // find them anew, not from the kept answer Again bool `json:"again,omitempty"` - // the artifact reference, : - Artifact string `json:"artifact"` - Fetch Fetch `json:"fetch"` + // the key of detected-steps, . + Artifact string `json:"artifact"` + // delete the artifact's kept key instead of detecting; needs why + Drop bool `json:"drop,omitempty"` + Fetch *Fetch `json:"fetch,omitempty"` Fingerprint string `json:"fingerprint,omitempty"` Module string `json:"module"` + // with drop, why: a non-empty text, written in the hand-act log with the caller the bus proved + Why string `json:"why,omitempty"` } // StepsAnswer is what steps answers. type StepsAnswer struct { - Artifact string `json:"artifact"` + Artifact string `json:"artifact,omitempty"` + // the artifact whose key a drop deleted + Dropped string `json:"dropped,omitempty"` + // why a drop deleted nothing, naming the hand-act log when it refused + Error string `json:"error,omitempty"` Fingerprint string `json:"fingerprint,omitempty"` // answered from the kept answer Kept bool `json:"kept,omitempty"` - Steps []DetectedStep `json:"steps"` + Steps []DetectedStep `json:"steps,omitempty"` } // DeliveryAccept is the body of delivery. @@ -949,6 +1072,15 @@ type CheckRunAccept = StepAsk // StepEndedEvent is the body of the event step-ended. type StepEndedEvent = StepEnded +// ToolchainReportedEvent is the body of the event toolchain-reported. +type ToolchainReportedEvent = ToolchainReported + +// StepProgressEvent is the body of the event step-progress. +type StepProgressEvent = StepProgress + +// StepsByAskerNodeIDValue is one value of the state steps under the key ... +type StepsByAskerNodeIDValue = Step + // StepsByNodeIDValue is one value of the state steps under the key .. type StepsByNodeIDValue = Step diff --git a/contracts/go/seats/nodebuildgo/contract_test.go b/contracts/go/seats/nodebuildgo/contract_test.go index b35c655d..14aca33e 100644 --- a/contracts/go/seats/nodebuildgo/contract_test.go +++ b/contracts/go/seats/nodebuildgo/contract_test.go @@ -21,6 +21,10 @@ func TestFixtures(t *testing.T) { decode: func(b []byte) (any, error) { var v StepEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.node-build-go\\.event\\.step-ended\\.[^.*> ]+$")}, {name: "an integration step that could not run, naming the part not heard", subject: "mesh.seat.node-build-go.event.step-ended.g14", given: "{\"artifact\":\"nats:program\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d5f\",\"module\":\"nats\",\"node\":\"g14\",\"outcome\":\"could not run\",\"output-object\":\"mesh-build-go_sent-objects/m2kx9a1b3c4d5f\",\"pipeline\":\"main.12.nats\",\"said\":\"the sandbox could not fetch novox/mesh-sdk\",\"step\":\"integration\",\"v\":1}", wire: "{\"artifact\":\"nats:program\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d5f\",\"module\":\"nats\",\"node\":\"g14\",\"outcome\":\"could not run\",\"output-object\":\"mesh-build-go_sent-objects/m2kx9a1b3c4d5f\",\"pipeline\":\"main.12.nats\",\"said\":\"the sandbox could not fetch novox/mesh-sdk\",\"step\":\"integration\",\"v\":1}", decode: func(b []byte) (any, error) { var v StepEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.node-build-go\\.event\\.step-ended\\.[^.*> ]+$")}, + {name: "a node's toolchain reported", subject: "mesh.seat.node-build-go.event.toolchain-reported.ace", given: "{\"digest\":\"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"language\":\"go\",\"node\":\"ace\",\"recipe\":\"1\",\"version\":\"1.26.0\"}", wire: "{\"digest\":\"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"language\":\"go\",\"node\":\"ace\",\"recipe\":\"1\",\"version\":\"1.26.0\"}", + decode: func(b []byte) (any, error) { var v ToolchainReportedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.node-build-go\\.event\\.toolchain-reported\\.[^.*> ]+$")}, + {name: "a running ask says its progress", subject: "mesh.seat.node-build-go.event.step-progress.ace", given: "{\"ask\":\"a1b2\",\"asker\":\"runner\",\"at\":\"2026-11-11T10:01:00Z\",\"phase\":\"fetch\"}", wire: "{\"ask\":\"a1b2\",\"asker\":\"runner\",\"at\":\"2026-11-11T10:01:00Z\",\"phase\":\"fetch\"}", + decode: func(b []byte) (any, error) { var v StepProgressEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.node-build-go\\.event\\.step-progress\\.[^.*> ]+$")}, {name: "a running step", subject: "", given: "{\"id\":\"a1b2\",\"node\":\"ace\",\"started\":\"2026-11-05T10:00:00Z\",\"state\":\"running\",\"tool\":\"build\",\"v\":1}", wire: "{\"id\":\"a1b2\",\"node\":\"ace\",\"started\":\"2026-11-05T10:00:00Z\",\"state\":\"running\",\"tool\":\"build\",\"v\":1}", decode: func(b []byte) (any, error) { var v StepsByNodeIDValue; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, {name: "a paused node", subject: "", given: "{\"at\":\"2026-11-05T10:00:00Z\",\"by\":\"operator@shanks\",\"paused\":true,\"v\":1}", wire: "{\"at\":\"2026-11-05T10:00:00Z\",\"by\":\"operator@shanks\",\"paused\":true,\"v\":1}", @@ -75,6 +79,12 @@ func TestFixtures(t *testing.T) { err := json.Unmarshal(b, &v) return v, err }, subjectIs: nil}, + {name: "a step keyed under its asker", subject: "", given: "{\"id\":\"a1b2\",\"node\":\"ace\",\"started\":\"2026-11-05T10:00:00Z\",\"state\":\"running\",\"tool\":\"build\",\"v\":1}", wire: "{\"id\":\"a1b2\",\"node\":\"ace\",\"started\":\"2026-11-05T10:00:00Z\",\"state\":\"running\",\"tool\":\"build\",\"v\":1}", + decode: func(b []byte) (any, error) { + var v StepsByAskerNodeIDValue + err := json.Unmarshal(b, &v) + return v, err + }, subjectIs: nil}, {name: "drop names the ask and why", subject: "", given: "{\"by\":\"mesh-delivery-builder\",\"id\":\"a1b2\",\"why\":\"a newer fingerprint supersedes it\"}", wire: "{\"by\":\"mesh-delivery-builder\",\"id\":\"a1b2\",\"why\":\"a newer fingerprint supersedes it\"}", decode: func(b []byte) (any, error) { var v DropInput; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, {name: "kept-artifact asks", subject: "", given: "{\"digest\":\"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd\",\"fp\":\"src2:1480985e\",\"module\":\"artifact-store\"}", wire: "{\"digest\":\"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd\",\"fp\":\"src2:1480985e\",\"module\":\"artifact-store\"}", @@ -93,6 +103,12 @@ func TestFixtures(t *testing.T) { decode: func(b []byte) (any, error) { var v StepsInput; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, {name: "steps answers lint and unit tests", subject: "", given: "{\"artifact\":\"nats:program\",\"fingerprint\":\"fp-0123\",\"steps\":[{\"step\":\"lint\"},{\"packages\":[\"./modules/nats\"],\"step\":\"unit\"}]}", wire: "{\"artifact\":\"nats:program\",\"fingerprint\":\"fp-0123\",\"steps\":[{\"step\":\"lint\"},{\"packages\":[\"./modules/nats\"],\"step\":\"unit\"}]}", decode: func(b []byte) (any, error) { var v StepsAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, + {name: "steps drops an artifact's kept steps with a reason", subject: "", given: "{\"artifact\":\"nats.program\",\"drop\":true,\"module\":\"nats\",\"why\":\"the detection was wrong\"}", wire: "{\"artifact\":\"nats.program\",\"drop\":true,\"module\":\"nats\",\"why\":\"the detection was wrong\"}", + decode: func(b []byte) (any, error) { var v StepsInput; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, + {name: "steps answers a drop with the artifact", subject: "", given: "{\"dropped\":\"nats.program\"}", wire: "{\"dropped\":\"nats.program\"}", + decode: func(b []byte) (any, error) { var v StepsAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, + {name: "steps answers a drop that the hand-act log refused", subject: "", given: "{\"error\":\"the hand-act log refused: nothing was deleted\"}", wire: "{\"error\":\"the hand-act log refused: nothing was deleted\"}", + decode: func(b []byte) (any, error) { var v StepsAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, } for _, c := range cases { v, err := c.decode([]byte(c.given)) diff --git a/contracts/go/seats/nodebuildimage/contract.go b/contracts/go/seats/nodebuildimage/contract.go index 00f25faa..9769b994 100644 --- a/contracts/go/seats/nodebuildimage/contract.go +++ b/contracts/go/seats/nodebuildimage/contract.go @@ -32,7 +32,9 @@ const ( ) const ( - EventStepEnded = "step-ended" + EventStepEnded = "step-ended" + EventToolchainReported = "toolchain-reported" + EventStepProgress = "step-progress" ) // Verbs is every verb the seat serves, in its definition's order: a new copy each call. @@ -42,7 +44,7 @@ func Verbs() []string { // Events is every event published on the seat's subject: a new copy each call. func Events() []string { - return []string{EventStepEnded} + return []string{EventStepEnded, EventToolchainReported, EventStepProgress} } // Family is where the seat's events are kept: the shared events stream, filtered to the seat's subjects. @@ -62,6 +64,28 @@ func SubjectStepEnded(node string) string { // SubjectStepEndedAll hears every step-ended. const SubjectStepEndedAll = "mesh.seat.node-build-image.event.step-ended.*" +// StreamToolchainReported is the stream toolchain-reported is kept in. +const StreamToolchainReported = "SEAT_node-build-image_toolchain-reported" + +// SubjectToolchainReported is where toolchain-reported is published, on mesh.seat.node-build-image.event.toolchain-reported.. +func SubjectToolchainReported(node string) string { + return "mesh.seat.node-build-image.event.toolchain-reported." + node +} + +// SubjectToolchainReportedAll hears every toolchain-reported. +const SubjectToolchainReportedAll = "mesh.seat.node-build-image.event.toolchain-reported.*" + +// StreamStepProgress is the stream step-progress is kept in. +const StreamStepProgress = "SEAT_node-build-image_step-progress" + +// SubjectStepProgress is where step-progress is published, on mesh.seat.node-build-image.event.step-progress.. +func SubjectStepProgress(node string) string { + return "mesh.seat.node-build-image.event.step-progress." + node +} + +// SubjectStepProgressAll hears every step-progress. +const SubjectStepProgressAll = "mesh.seat.node-build-image.event.step-progress.*" + // AcceptSubjectDelivery is where delivery is sent. const AcceptSubjectDelivery = "mesh.seat.node-build-image.accept.delivery" @@ -84,6 +108,47 @@ const ( StateDetectedSteps = "detected-steps" ) +// StepsByAskerNodeIDKey is the key of one value of the state steps under the form "..". +func StepsByAskerNodeIDKey(asker, node, id string) string { return asker + "." + node + "." + id } + +// ParseStepsByAskerNodeIDKey reads the parts of a key of the form "..", in the order of the form; the last result is false for a key of another form. +func ParseStepsByAskerNodeIDKey(key string) (string, string, string, bool) { + rest := key + v0, after0, found0 := strings.Cut(rest, ".") + if !found0 || v0 == "" { + return "", "", "", false + } + rest = after0 + v2, after2, found2 := strings.Cut(rest, ".") + if !found2 || v2 == "" { + return "", "", "", false + } + rest = after2 + if rest == "" { + return "", "", "", false + } + v4 := rest + rest = "" + if rest != "" { + return "", "", "", false + } + return v0, v2, v4, true +} + +// DecodeStepsByAskerNodeIDValue reads one value of the state steps under the form "..", refusing a field StepsByAskerNodeIDValue does not know and any text after the value. +func DecodeStepsByAskerNodeIDValue(raw []byte) (StepsByAskerNodeIDValue, error) { + var v StepsByAskerNodeIDValue + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.DisallowUnknownFields() + if err := dec.Decode(&v); err != nil { + return v, err + } + if dec.More() { + return v, errors.New("text after the value") + } + return v, nil +} + // StepsByNodeIDKey is the key of one value of the state steps under the form ".". func StepsByNodeIDKey(node, id string) string { return node + "." + id } @@ -315,11 +380,21 @@ type BuildAnswer struct { Toolchain *ToolchainUsed `json:"toolchain,omitempty"` } +// BuiltArtifactResolvedItem is a type of the contract. +type BuiltArtifactResolvedItem struct { + // sha256: of the package's bytes as the build fetched them + Digest string `json:"digest"` + Package string `json:"package"` + Version string `json:"version"` +} + // BuiltArtifact is one artifact a build made: its digest and the reference it was published under. type BuiltArtifact struct { Artifact string `json:"artifact"` Digest string `json:"digest"` Reference string `json:"reference,omitempty"` + // one per mesh package the build resolved (the catalog graph entry's packages, the edges of kind 1) + Resolved []BuiltArtifactResolvedItem `json:"resolved,omitempty"` } // Capacity is how many steps the node runs at once. @@ -394,6 +469,16 @@ type FetchBesideItem struct { Repository string `json:"repository"` } +// FetchBuildsItem is a type of the contract. +type FetchBuildsItem struct { + Artifact string `json:"artifact"` + // the member's build digest, sha256: + Digest string `json:"digest"` + Module string `json:"module"` + // the name in its ecosystem (the Go module path, the npm name) + Package string `json:"package"` +} + // FetchPackage is the TypeScript module's group package: the group's tagged version of the SDK ({name, version}), placed from the registry in each member whatever its lock names, or the group's package member ({repository, commit, ref}), packed by the fetch and served to the members in place of the registry's. type FetchPackage struct { Commit string `json:"commit,omitempty"` @@ -409,6 +494,8 @@ type Fetch struct { Base string `json:"base,omitempty"` // the sibling repositories placed beside the fetched tree Beside []FetchBesideItem `json:"beside,omitempty"` + // a group run's package members, each by its build's digest: the build module reads each from the artifact store at artifact-store:////blobs/, checks the bytes against the digest and lays them as the group package; a package named here is named in neither packages nor package + Builds []FetchBuildsItem `json:"builds,omitempty"` // the full commit Commit string `json:"commit"` // the module's directory in the repository ("." for its root), with paths @@ -733,6 +820,22 @@ type StepNames struct { Repositories []string `json:"repositories"` } +// StepProgress is a running ask's progress, said once a minute while the holder runs it (design 47 G section 496). +type StepProgress struct { + Ask string `json:"ask"` + Asker string `json:"asker"` + At time.Time `json:"at"` + Phase string `json:"phase"` +} + +// The values of StepProgress's phase. +const ( + StepProgressPhaseFetch = "fetch" + StepProgressPhaseBuild = "build" + StepProgressPhaseRun = "run" + StepProgressPhaseReport = "report" +) + // StepResult is what a step answered, by the kind of ask: a build's artifacts, toolchain, imports, out-of-step modules and the artifact it kept; a check's fetch and levels; a compose's built, key and verdict; a run's built, key, exit, output and said; a declined ask's declined, needs and lacks. The fields of the kinds do not meet, so one layout holds the five. type StepResult struct { Artifacts []BuiltArtifact `json:"artifacts,omitempty"` @@ -826,8 +929,20 @@ type Toolchain struct { Version string `json:"version"` } +// ToolchainReported is a node's toolchain of a language, said at the build module's start and each time its toolchains key of the language changes (design 47 G section 492). +type ToolchainReported struct { + // the digest of the program of the provision the build module requires for that language's toolchain, never the build module's own + Digest string `json:"digest"` + Language string `json:"language"` + Node string `json:"node"` + Recipe string `json:"recipe"` + Version string `json:"version"` +} + // ToolchainUsed is the toolchain a build was made with: the language, the installed version and the build module's recipe. type ToolchainUsed struct { + // the building node's digest of its toolchain program (kind 3's entry), never the build module's own program + Digest string `json:"digest,omitempty"` Language string `json:"language"` Recipe string `json:"recipe"` Version string `json:"version"` @@ -910,20 +1025,28 @@ const ( type StepsInput struct { // find them anew, not from the kept answer Again bool `json:"again,omitempty"` - // the artifact reference, : - Artifact string `json:"artifact"` - Fetch Fetch `json:"fetch"` + // the key of detected-steps, . + Artifact string `json:"artifact"` + // delete the artifact's kept key instead of detecting; needs why + Drop bool `json:"drop,omitempty"` + Fetch *Fetch `json:"fetch,omitempty"` Fingerprint string `json:"fingerprint,omitempty"` Module string `json:"module"` + // with drop, why: a non-empty text, written in the hand-act log with the caller the bus proved + Why string `json:"why,omitempty"` } // StepsAnswer is what steps answers. type StepsAnswer struct { - Artifact string `json:"artifact"` + Artifact string `json:"artifact,omitempty"` + // the artifact whose key a drop deleted + Dropped string `json:"dropped,omitempty"` + // why a drop deleted nothing, naming the hand-act log when it refused + Error string `json:"error,omitempty"` Fingerprint string `json:"fingerprint,omitempty"` // answered from the kept answer Kept bool `json:"kept,omitempty"` - Steps []DetectedStep `json:"steps"` + Steps []DetectedStep `json:"steps,omitempty"` } // DeliveryAccept is the body of delivery. @@ -944,6 +1067,15 @@ type RecheckAccept = StepAsk // StepEndedEvent is the body of the event step-ended. type StepEndedEvent = StepEnded +// ToolchainReportedEvent is the body of the event toolchain-reported. +type ToolchainReportedEvent = ToolchainReported + +// StepProgressEvent is the body of the event step-progress. +type StepProgressEvent = StepProgress + +// StepsByAskerNodeIDValue is one value of the state steps under the key ... +type StepsByAskerNodeIDValue = Step + // StepsByNodeIDValue is one value of the state steps under the key .. type StepsByNodeIDValue = Step diff --git a/contracts/go/seats/nodebuildimage/contract_test.go b/contracts/go/seats/nodebuildimage/contract_test.go index fbbbb9c8..3ec60a4f 100644 --- a/contracts/go/seats/nodebuildimage/contract_test.go +++ b/contracts/go/seats/nodebuildimage/contract_test.go @@ -21,6 +21,10 @@ func TestFixtures(t *testing.T) { decode: func(b []byte) (any, error) { var v StepEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.node-build-image\\.event\\.step-ended\\.[^.*> ]+$")}, {name: "a lint step that could not run, naming the part not heard", subject: "mesh.seat.node-build-image.event.step-ended.g14", given: "{\"artifact\":\"nats:program\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d5f\",\"module\":\"nats\",\"node\":\"g14\",\"outcome\":\"could not run\",\"output-object\":\"mesh-build-go_sent-objects/m2kx9a1b3c4d5f\",\"pipeline\":\"main.12.nats\",\"said\":\"the sandbox could not fetch novox/mesh-sdk\",\"step\":\"lint\",\"v\":1}", wire: "{\"artifact\":\"nats:program\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d5f\",\"module\":\"nats\",\"node\":\"g14\",\"outcome\":\"could not run\",\"output-object\":\"mesh-build-go_sent-objects/m2kx9a1b3c4d5f\",\"pipeline\":\"main.12.nats\",\"said\":\"the sandbox could not fetch novox/mesh-sdk\",\"step\":\"lint\",\"v\":1}", decode: func(b []byte) (any, error) { var v StepEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.node-build-image\\.event\\.step-ended\\.[^.*> ]+$")}, + {name: "a node's toolchain reported", subject: "mesh.seat.node-build-image.event.toolchain-reported.ace", given: "{\"digest\":\"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"language\":\"go\",\"node\":\"ace\",\"recipe\":\"1\",\"version\":\"1.26.0\"}", wire: "{\"digest\":\"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"language\":\"go\",\"node\":\"ace\",\"recipe\":\"1\",\"version\":\"1.26.0\"}", + decode: func(b []byte) (any, error) { var v ToolchainReportedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.node-build-image\\.event\\.toolchain-reported\\.[^.*> ]+$")}, + {name: "a running ask says its progress", subject: "mesh.seat.node-build-image.event.step-progress.ace", given: "{\"ask\":\"a1b2\",\"asker\":\"runner\",\"at\":\"2026-11-11T10:01:00Z\",\"phase\":\"fetch\"}", wire: "{\"ask\":\"a1b2\",\"asker\":\"runner\",\"at\":\"2026-11-11T10:01:00Z\",\"phase\":\"fetch\"}", + decode: func(b []byte) (any, error) { var v StepProgressEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.node-build-image\\.event\\.step-progress\\.[^.*> ]+$")}, {name: "a running step", subject: "", given: "{\"id\":\"a1b2\",\"node\":\"ace\",\"started\":\"2026-11-05T10:00:00Z\",\"state\":\"running\",\"tool\":\"build\",\"v\":1}", wire: "{\"id\":\"a1b2\",\"node\":\"ace\",\"started\":\"2026-11-05T10:00:00Z\",\"state\":\"running\",\"tool\":\"build\",\"v\":1}", decode: func(b []byte) (any, error) { var v StepsByNodeIDValue; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, {name: "a paused node", subject: "", given: "{\"at\":\"2026-11-05T10:00:00Z\",\"by\":\"operator@shanks\",\"paused\":true,\"v\":1}", wire: "{\"at\":\"2026-11-05T10:00:00Z\",\"by\":\"operator@shanks\",\"paused\":true,\"v\":1}", @@ -57,6 +61,12 @@ func TestFixtures(t *testing.T) { err := json.Unmarshal(b, &v) return v, err }, subjectIs: nil}, + {name: "a step keyed under its asker", subject: "", given: "{\"id\":\"a1b2\",\"node\":\"ace\",\"started\":\"2026-11-05T10:00:00Z\",\"state\":\"running\",\"tool\":\"build\",\"v\":1}", wire: "{\"id\":\"a1b2\",\"node\":\"ace\",\"started\":\"2026-11-05T10:00:00Z\",\"state\":\"running\",\"tool\":\"build\",\"v\":1}", + decode: func(b []byte) (any, error) { + var v StepsByAskerNodeIDValue + err := json.Unmarshal(b, &v) + return v, err + }, subjectIs: nil}, {name: "drop names the ask and why", subject: "", given: "{\"by\":\"mesh-delivery-builder\",\"id\":\"a1b2\",\"why\":\"a newer fingerprint supersedes it\"}", wire: "{\"by\":\"mesh-delivery-builder\",\"id\":\"a1b2\",\"why\":\"a newer fingerprint supersedes it\"}", decode: func(b []byte) (any, error) { var v DropInput; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, {name: "kept-artifact asks", subject: "", given: "{\"digest\":\"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd\",\"fp\":\"src2:1480985e\",\"module\":\"artifact-store\"}", wire: "{\"digest\":\"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd\",\"fp\":\"src2:1480985e\",\"module\":\"artifact-store\"}", @@ -75,6 +85,12 @@ func TestFixtures(t *testing.T) { decode: func(b []byte) (any, error) { var v StepsInput; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, {name: "steps answers lint and unit tests", subject: "", given: "{\"artifact\":\"nats:program\",\"fingerprint\":\"fp-0123\",\"steps\":[{\"step\":\"lint\"},{\"packages\":[\"./modules/nats\"],\"step\":\"unit\"}]}", wire: "{\"artifact\":\"nats:program\",\"fingerprint\":\"fp-0123\",\"steps\":[{\"step\":\"lint\"},{\"packages\":[\"./modules/nats\"],\"step\":\"unit\"}]}", decode: func(b []byte) (any, error) { var v StepsAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, + {name: "steps drops an artifact's kept steps with a reason", subject: "", given: "{\"artifact\":\"nats.program\",\"drop\":true,\"module\":\"nats\",\"why\":\"the detection was wrong\"}", wire: "{\"artifact\":\"nats.program\",\"drop\":true,\"module\":\"nats\",\"why\":\"the detection was wrong\"}", + decode: func(b []byte) (any, error) { var v StepsInput; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, + {name: "steps answers a drop with the artifact", subject: "", given: "{\"dropped\":\"nats.program\"}", wire: "{\"dropped\":\"nats.program\"}", + decode: func(b []byte) (any, error) { var v StepsAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, + {name: "steps answers a drop that the hand-act log refused", subject: "", given: "{\"error\":\"the hand-act log refused: nothing was deleted\"}", wire: "{\"error\":\"the hand-act log refused: nothing was deleted\"}", + decode: func(b []byte) (any, error) { var v StepsAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, } for _, c := range cases { v, err := c.decode([]byte(c.given)) diff --git a/contracts/go/seats/nodebuildnode/contract.go b/contracts/go/seats/nodebuildnode/contract.go index 2df88e8f..5004106f 100644 --- a/contracts/go/seats/nodebuildnode/contract.go +++ b/contracts/go/seats/nodebuildnode/contract.go @@ -32,7 +32,9 @@ const ( ) const ( - EventStepEnded = "step-ended" + EventStepEnded = "step-ended" + EventToolchainReported = "toolchain-reported" + EventStepProgress = "step-progress" ) // Verbs is every verb the seat serves, in its definition's order: a new copy each call. @@ -42,7 +44,7 @@ func Verbs() []string { // Events is every event published on the seat's subject: a new copy each call. func Events() []string { - return []string{EventStepEnded} + return []string{EventStepEnded, EventToolchainReported, EventStepProgress} } // Family is where the seat's events are kept: the shared events stream, filtered to the seat's subjects. @@ -62,6 +64,28 @@ func SubjectStepEnded(node string) string { // SubjectStepEndedAll hears every step-ended. const SubjectStepEndedAll = "mesh.seat.node-build-node.event.step-ended.*" +// StreamToolchainReported is the stream toolchain-reported is kept in. +const StreamToolchainReported = "SEAT_node-build-node_toolchain-reported" + +// SubjectToolchainReported is where toolchain-reported is published, on mesh.seat.node-build-node.event.toolchain-reported.. +func SubjectToolchainReported(node string) string { + return "mesh.seat.node-build-node.event.toolchain-reported." + node +} + +// SubjectToolchainReportedAll hears every toolchain-reported. +const SubjectToolchainReportedAll = "mesh.seat.node-build-node.event.toolchain-reported.*" + +// StreamStepProgress is the stream step-progress is kept in. +const StreamStepProgress = "SEAT_node-build-node_step-progress" + +// SubjectStepProgress is where step-progress is published, on mesh.seat.node-build-node.event.step-progress.. +func SubjectStepProgress(node string) string { + return "mesh.seat.node-build-node.event.step-progress." + node +} + +// SubjectStepProgressAll hears every step-progress. +const SubjectStepProgressAll = "mesh.seat.node-build-node.event.step-progress.*" + // AcceptSubjectDelivery is where delivery is sent. const AcceptSubjectDelivery = "mesh.seat.node-build-node.accept.delivery" @@ -84,6 +108,47 @@ const ( StateDetectedSteps = "detected-steps" ) +// StepsByAskerNodeIDKey is the key of one value of the state steps under the form "..". +func StepsByAskerNodeIDKey(asker, node, id string) string { return asker + "." + node + "." + id } + +// ParseStepsByAskerNodeIDKey reads the parts of a key of the form "..", in the order of the form; the last result is false for a key of another form. +func ParseStepsByAskerNodeIDKey(key string) (string, string, string, bool) { + rest := key + v0, after0, found0 := strings.Cut(rest, ".") + if !found0 || v0 == "" { + return "", "", "", false + } + rest = after0 + v2, after2, found2 := strings.Cut(rest, ".") + if !found2 || v2 == "" { + return "", "", "", false + } + rest = after2 + if rest == "" { + return "", "", "", false + } + v4 := rest + rest = "" + if rest != "" { + return "", "", "", false + } + return v0, v2, v4, true +} + +// DecodeStepsByAskerNodeIDValue reads one value of the state steps under the form "..", refusing a field StepsByAskerNodeIDValue does not know and any text after the value. +func DecodeStepsByAskerNodeIDValue(raw []byte) (StepsByAskerNodeIDValue, error) { + var v StepsByAskerNodeIDValue + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.DisallowUnknownFields() + if err := dec.Decode(&v); err != nil { + return v, err + } + if dec.More() { + return v, errors.New("text after the value") + } + return v, nil +} + // StepsByNodeIDKey is the key of one value of the state steps under the form ".". func StepsByNodeIDKey(node, id string) string { return node + "." + id } @@ -315,11 +380,21 @@ type BuildAnswer struct { Toolchain *ToolchainUsed `json:"toolchain,omitempty"` } +// BuiltArtifactResolvedItem is a type of the contract. +type BuiltArtifactResolvedItem struct { + // sha256: of the package's bytes as the build fetched them + Digest string `json:"digest"` + Package string `json:"package"` + Version string `json:"version"` +} + // BuiltArtifact is one artifact a build made: its digest and the reference it was published under. type BuiltArtifact struct { Artifact string `json:"artifact"` Digest string `json:"digest"` Reference string `json:"reference,omitempty"` + // one per mesh package the build resolved (the catalog graph entry's packages, the edges of kind 1) + Resolved []BuiltArtifactResolvedItem `json:"resolved,omitempty"` } // Capacity is how many steps the node runs at once. @@ -394,6 +469,16 @@ type FetchBesideItem struct { Repository string `json:"repository"` } +// FetchBuildsItem is a type of the contract. +type FetchBuildsItem struct { + Artifact string `json:"artifact"` + // the member's build digest, sha256: + Digest string `json:"digest"` + Module string `json:"module"` + // the name in its ecosystem (the Go module path, the npm name) + Package string `json:"package"` +} + // FetchPackage is the TypeScript module's group package: the group's tagged version of the SDK ({name, version}), placed from the registry in each member whatever its lock names, or the group's package member ({repository, commit, ref}), packed by the fetch and served to the members in place of the registry's. type FetchPackage struct { Commit string `json:"commit,omitempty"` @@ -409,6 +494,8 @@ type Fetch struct { Base string `json:"base,omitempty"` // the sibling repositories placed beside the fetched tree Beside []FetchBesideItem `json:"beside,omitempty"` + // a group run's package members, each by its build's digest: the build module reads each from the artifact store at artifact-store:////blobs/, checks the bytes against the digest and lays them as the group package; a package named here is named in neither packages nor package + Builds []FetchBuildsItem `json:"builds,omitempty"` // the full commit Commit string `json:"commit"` // the module's directory in the repository ("." for its root), with paths @@ -733,6 +820,22 @@ type StepNames struct { Repositories []string `json:"repositories"` } +// StepProgress is a running ask's progress, said once a minute while the holder runs it (design 47 G section 496). +type StepProgress struct { + Ask string `json:"ask"` + Asker string `json:"asker"` + At time.Time `json:"at"` + Phase string `json:"phase"` +} + +// The values of StepProgress's phase. +const ( + StepProgressPhaseFetch = "fetch" + StepProgressPhaseBuild = "build" + StepProgressPhaseRun = "run" + StepProgressPhaseReport = "report" +) + // StepResult is what a step answered, by the kind of ask: a build's artifacts, toolchain, imports, out-of-step modules and the artifact it kept; a check's fetch and levels; a compose's built, key and verdict; a run's built, key, exit, output and said; a declined ask's declined, needs and lacks. The fields of the kinds do not meet, so one layout holds the five. type StepResult struct { Artifacts []BuiltArtifact `json:"artifacts,omitempty"` @@ -826,8 +929,20 @@ type Toolchain struct { Version string `json:"version"` } +// ToolchainReported is a node's toolchain of a language, said at the build module's start and each time its toolchains key of the language changes (design 47 G section 492). +type ToolchainReported struct { + // the digest of the program of the provision the build module requires for that language's toolchain, never the build module's own + Digest string `json:"digest"` + Language string `json:"language"` + Node string `json:"node"` + Recipe string `json:"recipe"` + Version string `json:"version"` +} + // ToolchainUsed is the toolchain a build was made with: the language, the installed version and the build module's recipe. type ToolchainUsed struct { + // the building node's digest of its toolchain program (kind 3's entry), never the build module's own program + Digest string `json:"digest,omitempty"` Language string `json:"language"` Recipe string `json:"recipe"` Version string `json:"version"` @@ -910,20 +1025,28 @@ const ( type StepsInput struct { // find them anew, not from the kept answer Again bool `json:"again,omitempty"` - // the artifact reference, : - Artifact string `json:"artifact"` - Fetch Fetch `json:"fetch"` + // the key of detected-steps, . + Artifact string `json:"artifact"` + // delete the artifact's kept key instead of detecting; needs why + Drop bool `json:"drop,omitempty"` + Fetch *Fetch `json:"fetch,omitempty"` Fingerprint string `json:"fingerprint,omitempty"` Module string `json:"module"` + // with drop, why: a non-empty text, written in the hand-act log with the caller the bus proved + Why string `json:"why,omitempty"` } // StepsAnswer is what steps answers. type StepsAnswer struct { - Artifact string `json:"artifact"` + Artifact string `json:"artifact,omitempty"` + // the artifact whose key a drop deleted + Dropped string `json:"dropped,omitempty"` + // why a drop deleted nothing, naming the hand-act log when it refused + Error string `json:"error,omitempty"` Fingerprint string `json:"fingerprint,omitempty"` // answered from the kept answer Kept bool `json:"kept,omitempty"` - Steps []DetectedStep `json:"steps"` + Steps []DetectedStep `json:"steps,omitempty"` } // DeliveryAccept is the body of delivery. @@ -944,6 +1067,15 @@ type RecheckAccept = StepAsk // StepEndedEvent is the body of the event step-ended. type StepEndedEvent = StepEnded +// ToolchainReportedEvent is the body of the event toolchain-reported. +type ToolchainReportedEvent = ToolchainReported + +// StepProgressEvent is the body of the event step-progress. +type StepProgressEvent = StepProgress + +// StepsByAskerNodeIDValue is one value of the state steps under the key ... +type StepsByAskerNodeIDValue = Step + // StepsByNodeIDValue is one value of the state steps under the key .. type StepsByNodeIDValue = Step diff --git a/contracts/go/seats/nodebuildnode/contract_test.go b/contracts/go/seats/nodebuildnode/contract_test.go index 982cd713..931da365 100644 --- a/contracts/go/seats/nodebuildnode/contract_test.go +++ b/contracts/go/seats/nodebuildnode/contract_test.go @@ -21,6 +21,10 @@ func TestFixtures(t *testing.T) { decode: func(b []byte) (any, error) { var v StepEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.node-build-node\\.event\\.step-ended\\.[^.*> ]+$")}, {name: "an integration step that could not run, naming the part not heard", subject: "mesh.seat.node-build-node.event.step-ended.g14", given: "{\"artifact\":\"nats:program\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d5f\",\"module\":\"nats\",\"node\":\"g14\",\"outcome\":\"could not run\",\"output-object\":\"mesh-build-go_sent-objects/m2kx9a1b3c4d5f\",\"pipeline\":\"main.12.nats\",\"said\":\"the sandbox could not fetch novox/mesh-sdk\",\"step\":\"integration\",\"v\":1}", wire: "{\"artifact\":\"nats:program\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d5f\",\"module\":\"nats\",\"node\":\"g14\",\"outcome\":\"could not run\",\"output-object\":\"mesh-build-go_sent-objects/m2kx9a1b3c4d5f\",\"pipeline\":\"main.12.nats\",\"said\":\"the sandbox could not fetch novox/mesh-sdk\",\"step\":\"integration\",\"v\":1}", decode: func(b []byte) (any, error) { var v StepEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.node-build-node\\.event\\.step-ended\\.[^.*> ]+$")}, + {name: "a node's toolchain reported", subject: "mesh.seat.node-build-node.event.toolchain-reported.ace", given: "{\"digest\":\"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"language\":\"go\",\"node\":\"ace\",\"recipe\":\"1\",\"version\":\"1.26.0\"}", wire: "{\"digest\":\"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"language\":\"go\",\"node\":\"ace\",\"recipe\":\"1\",\"version\":\"1.26.0\"}", + decode: func(b []byte) (any, error) { var v ToolchainReportedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.node-build-node\\.event\\.toolchain-reported\\.[^.*> ]+$")}, + {name: "a running ask says its progress", subject: "mesh.seat.node-build-node.event.step-progress.ace", given: "{\"ask\":\"a1b2\",\"asker\":\"runner\",\"at\":\"2026-11-11T10:01:00Z\",\"phase\":\"fetch\"}", wire: "{\"ask\":\"a1b2\",\"asker\":\"runner\",\"at\":\"2026-11-11T10:01:00Z\",\"phase\":\"fetch\"}", + decode: func(b []byte) (any, error) { var v StepProgressEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.node-build-node\\.event\\.step-progress\\.[^.*> ]+$")}, {name: "a running step", subject: "", given: "{\"id\":\"a1b2\",\"node\":\"ace\",\"started\":\"2026-11-05T10:00:00Z\",\"state\":\"running\",\"tool\":\"build\",\"v\":1}", wire: "{\"id\":\"a1b2\",\"node\":\"ace\",\"started\":\"2026-11-05T10:00:00Z\",\"state\":\"running\",\"tool\":\"build\",\"v\":1}", decode: func(b []byte) (any, error) { var v StepsByNodeIDValue; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, {name: "a paused node", subject: "", given: "{\"at\":\"2026-11-05T10:00:00Z\",\"by\":\"operator@shanks\",\"paused\":true,\"v\":1}", wire: "{\"at\":\"2026-11-05T10:00:00Z\",\"by\":\"operator@shanks\",\"paused\":true,\"v\":1}", @@ -61,6 +65,12 @@ func TestFixtures(t *testing.T) { err := json.Unmarshal(b, &v) return v, err }, subjectIs: nil}, + {name: "a step keyed under its asker", subject: "", given: "{\"id\":\"a1b2\",\"node\":\"ace\",\"started\":\"2026-11-05T10:00:00Z\",\"state\":\"running\",\"tool\":\"build\",\"v\":1}", wire: "{\"id\":\"a1b2\",\"node\":\"ace\",\"started\":\"2026-11-05T10:00:00Z\",\"state\":\"running\",\"tool\":\"build\",\"v\":1}", + decode: func(b []byte) (any, error) { + var v StepsByAskerNodeIDValue + err := json.Unmarshal(b, &v) + return v, err + }, subjectIs: nil}, {name: "drop names the ask and why", subject: "", given: "{\"by\":\"mesh-delivery-builder\",\"id\":\"a1b2\",\"why\":\"a newer fingerprint supersedes it\"}", wire: "{\"by\":\"mesh-delivery-builder\",\"id\":\"a1b2\",\"why\":\"a newer fingerprint supersedes it\"}", decode: func(b []byte) (any, error) { var v DropInput; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, {name: "kept-artifact asks", subject: "", given: "{\"digest\":\"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd\",\"fp\":\"src2:1480985e\",\"module\":\"artifact-store\"}", wire: "{\"digest\":\"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd\",\"fp\":\"src2:1480985e\",\"module\":\"artifact-store\"}", @@ -79,6 +89,12 @@ func TestFixtures(t *testing.T) { decode: func(b []byte) (any, error) { var v StepsInput; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, {name: "steps answers lint and unit tests", subject: "", given: "{\"artifact\":\"nats:program\",\"fingerprint\":\"fp-0123\",\"steps\":[{\"step\":\"lint\"},{\"packages\":[\"./modules/nats\"],\"step\":\"unit\"}]}", wire: "{\"artifact\":\"nats:program\",\"fingerprint\":\"fp-0123\",\"steps\":[{\"step\":\"lint\"},{\"packages\":[\"./modules/nats\"],\"step\":\"unit\"}]}", decode: func(b []byte) (any, error) { var v StepsAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, + {name: "steps drops an artifact's kept steps with a reason", subject: "", given: "{\"artifact\":\"nats.program\",\"drop\":true,\"module\":\"nats\",\"why\":\"the detection was wrong\"}", wire: "{\"artifact\":\"nats.program\",\"drop\":true,\"module\":\"nats\",\"why\":\"the detection was wrong\"}", + decode: func(b []byte) (any, error) { var v StepsInput; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, + {name: "steps answers a drop with the artifact", subject: "", given: "{\"dropped\":\"nats.program\"}", wire: "{\"dropped\":\"nats.program\"}", + decode: func(b []byte) (any, error) { var v StepsAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, + {name: "steps answers a drop that the hand-act log refused", subject: "", given: "{\"error\":\"the hand-act log refused: nothing was deleted\"}", wire: "{\"error\":\"the hand-act log refused: nothing was deleted\"}", + decode: func(b []byte) (any, error) { var v StepsAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, } for _, c := range cases { v, err := c.decode([]byte(c.given)) diff --git a/contracts/parts/build-step.json b/contracts/parts/build-step.json index 3aed594f..2179c177 100644 --- a/contracts/parts/build-step.json +++ b/contracts/parts/build-step.json @@ -159,10 +159,12 @@ }, { "name": "steps", - "description": "Which pipeline steps an artifact can run, found by its language's convention (design 47 *Pipeline steps and the policy*): lint always, unit and integration tests where its packages hold them. Kept by the artifact's fingerprint; again finds them anew.", + "description": "Which pipeline steps an artifact can run, found by its language's convention (design 47 *Pipeline steps and the policy*): lint always, unit and integration tests where its packages hold them. Kept by the artifact's fingerprint; again finds them anew. With drop and why it deletes the artifact's kept key instead, writes the act in the hand-act log of the seat mesh-hand-acts first (a log that refuses deletes nothing and answers error), and answers dropped (design 47 G section 463, section 480).", "callers": [ "seat:mesh-pipeline-runner", - "seat:mesh-catalog" + "seat:mesh-catalog", + "operator", + "sessions" ], "input": { "type": "object", @@ -172,7 +174,7 @@ }, "artifact": { "type": "string", - "description": "the artifact reference, :" + "description": "the key of detected-steps, ." }, "fingerprint": { "type": "string" @@ -183,13 +185,21 @@ "again": { "type": "boolean", "description": "find them anew, not from the kept answer" + }, + "drop": { + "type": "boolean", + "description": "delete the artifact's kept key instead of detecting; needs why" + }, + "why": { + "type": "string", + "description": "with drop, why: a non-empty text, written in the hand-act log with the caller the bus proved" } }, "required": [ "module", - "artifact", - "fetch" - ] + "artifact" + ], + "description": "fetch is required unless drop is given, and why with drop" }, "answer": { "type": "object", @@ -209,12 +219,17 @@ "kept": { "type": "boolean", "description": "answered from the kept answer" + }, + "dropped": { + "type": "string", + "description": "the artifact whose key a drop deleted" + }, + "error": { + "type": "string", + "description": "why a drop deleted nothing, naming the hand-act log when it refused" } }, - "required": [ - "artifact", - "steps" - ] + "required": [] }, "class": "act" } @@ -857,6 +872,35 @@ ] }, "description": "the sibling repositories placed beside the fetched tree" + }, + "builds": { + "type": "array", + "description": "a group run's package members, each by its build's digest: the build module reads each from the artifact store at artifact-store:////blobs/, checks the bytes against the digest and lays them as the group package; a package named here is named in neither packages nor package", + "items": { + "type": "object", + "properties": { + "package": { + "type": "string", + "description": "the name in its ecosystem (the Go module path, the npm name)" + }, + "module": { + "type": "string" + }, + "artifact": { + "type": "string" + }, + "digest": { + "type": "string", + "description": "the member's build digest, sha256:" + } + }, + "required": [ + "package", + "module", + "artifact", + "digest" + ] + } } }, "required": [ @@ -875,6 +919,10 @@ }, "recipe": { "type": "string" + }, + "digest": { + "type": "string", + "description": "the building node's digest of its toolchain program (kind 3's entry), never the build module's own program" } }, "required": [ @@ -916,6 +964,30 @@ }, "reference": { "type": "string" + }, + "resolved": { + "type": "array", + "description": "one per mesh package the build resolved (the catalog graph entry's packages, the edges of kind 1)", + "items": { + "type": "object", + "properties": { + "package": { + "type": "string" + }, + "version": { + "type": "string" + }, + "digest": { + "type": "string", + "description": "sha256: of the package's bytes as the build fetched them" + } + }, + "required": [ + "package", + "version", + "digest" + ] + } } }, "required": [ @@ -1628,6 +1700,66 @@ "fingerprint", "steps" ] + }, + "ToolchainReported": { + "type": "object", + "description": "a node's toolchain of a language, said at the build module's start and each time its toolchains key of the language changes (design 47 G section 492)", + "properties": { + "node": { + "type": "string" + }, + "language": { + "type": "string" + }, + "version": { + "type": "string" + }, + "recipe": { + "type": "string" + }, + "digest": { + "type": "string", + "description": "the digest of the program of the provision the build module requires for that language's toolchain, never the build module's own" + } + }, + "required": [ + "node", + "language", + "version", + "recipe", + "digest" + ] + }, + "StepProgress": { + "type": "object", + "description": "a running ask's progress, said once a minute while the holder runs it (design 47 G section 496)", + "properties": { + "asker": { + "type": "string" + }, + "ask": { + "type": "string" + }, + "phase": { + "type": "string", + "enum": [ + "fetch", + "build", + "run", + "report" + ] + }, + "at": { + "type": "string", + "format": "date-time" + } + }, + "required": [ + "asker", + "ask", + "phase", + "at" + ] } }, "events": [ @@ -1641,6 +1773,28 @@ "max-age-days": 90, "max-mib": 128 } + }, + { + "name": "toolchain-reported", + "subject": "toolchain-reported.", + "body": { + "$ref": "#/defs/ToolchainReported" + }, + "keep": { + "max-age-days": 90, + "max-mib": 16 + } + }, + { + "name": "step-progress", + "subject": "step-progress.", + "body": { + "$ref": "#/defs/StepProgress" + }, + "keep": { + "max-age-days": 1, + "max-mib": 16 + } } ] } diff --git a/contracts/seats/mesh-wants/fixtures/events.json b/contracts/seats/mesh-wants/fixtures/events.json index 70fbc77c..7c00e058 100644 --- a/contracts/seats/mesh-wants/fixtures/events.json +++ b/contracts/seats/mesh-wants/fixtures/events.json @@ -166,5 +166,16 @@ "laptop" ] } + }, + { + "name": "a composition step's progress", + "of": "event:step-progress", + "subject": "mesh.seat.mesh-wants.event.step-progress.ace", + "given": { + "asker": "assigner", + "ask": "a1b2", + "phase": "build", + "at": "2026-11-11T10:01:00Z" + } } ] diff --git a/contracts/seats/mesh-wants/seat.json b/contracts/seats/mesh-wants/seat.json index e002258b..aca8f3b9 100644 --- a/contracts/seats/mesh-wants/seat.json +++ b/contracts/seats/mesh-wants/seat.json @@ -427,6 +427,45 @@ "parts" ] } + }, + { + "name": "step-progress", + "subject": "step-progress.", + "keep": { + "max-age-days": 1, + "max-mib": 16 + }, + "body": { + "type": "object", + "description": "a composition step's progress, said once a minute while the assigner runs it (design 47 G section 496)", + "properties": { + "asker": { + "type": "string" + }, + "ask": { + "type": "string" + }, + "phase": { + "type": "string", + "enum": [ + "fetch", + "build", + "run", + "report" + ] + }, + "at": { + "type": "string", + "format": "date-time" + } + }, + "required": [ + "asker", + "ask", + "phase", + "at" + ] + } } ], "family": { diff --git a/contracts/seats/node-build-go/fixtures/events.json b/contracts/seats/node-build-go/fixtures/events.json index 06ad58d6..1d3351e3 100644 --- a/contracts/seats/node-build-go/fixtures/events.json +++ b/contracts/seats/node-build-go/fixtures/events.json @@ -53,5 +53,28 @@ "said": "the sandbox could not fetch novox/mesh-sdk", "output-object": "mesh-build-go_sent-objects/m2kx9a1b3c4d5f" } + }, + { + "name": "a node's toolchain reported", + "of": "event:toolchain-reported", + "subject": "mesh.seat.node-build-go.event.toolchain-reported.ace", + "given": { + "node": "ace", + "language": "go", + "version": "1.26.0", + "recipe": "1", + "digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + }, + { + "name": "a running ask says its progress", + "of": "event:step-progress", + "subject": "mesh.seat.node-build-go.event.step-progress.ace", + "given": { + "asker": "runner", + "ask": "a1b2", + "phase": "fetch", + "at": "2026-11-11T10:01:00Z" + } } ] diff --git a/contracts/seats/node-build-go/fixtures/states.json b/contracts/seats/node-build-go/fixtures/states.json index 9ee56a47..bb3685b7 100644 --- a/contracts/seats/node-build-go/fixtures/states.json +++ b/contracts/seats/node-build-go/fixtures/states.json @@ -526,5 +526,17 @@ ], "at": "2026-11-10T10:00:00Z" } + }, + { + "name": "a step keyed under its asker", + "of": "state:steps/..", + "given": { + "v": 1, + "id": "a1b2", + "tool": "build", + "node": "ace", + "state": "running", + "started": "2026-11-05T10:00:00Z" + } } ] diff --git a/contracts/seats/node-build-go/fixtures/verbs.json b/contracts/seats/node-build-go/fixtures/verbs.json index 1e4af4dd..7a02ff62 100644 --- a/contracts/seats/node-build-go/fixtures/verbs.json +++ b/contracts/seats/node-build-go/fixtures/verbs.json @@ -100,5 +100,29 @@ } ] } + }, + { + "name": "steps drops an artifact's kept steps with a reason", + "of": "verb:steps.input", + "given": { + "module": "nats", + "artifact": "nats.program", + "drop": true, + "why": "the detection was wrong" + } + }, + { + "name": "steps answers a drop with the artifact", + "of": "verb:steps.answer", + "given": { + "dropped": "nats.program" + } + }, + { + "name": "steps answers a drop that the hand-act log refused", + "of": "verb:steps.answer", + "given": { + "error": "the hand-act log refused: nothing was deleted" + } } ] diff --git a/contracts/seats/node-build-go/seat.json b/contracts/seats/node-build-go/seat.json index 4d30fcd7..f9be14f8 100644 --- a/contracts/seats/node-build-go/seat.json +++ b/contracts/seats/node-build-go/seat.json @@ -87,12 +87,19 @@ "name": "steps", "template": "state", "key-forms": [ + { + "key": "..", + "value": { + "$ref": "#/defs/Step" + }, + "description": "a step on a node, keyed by the asker the bus proves (design 47 G section 467, section 481)" + }, { "key": ".", "value": { "$ref": "#/defs/Step" }, - "description": "a step on a node" + "description": "a step whose ask came on main's subject, with no asker's name (design 47 G section 483)" } ], "description": "the steps this module runs", diff --git a/contracts/seats/node-build-image/fixtures/events.json b/contracts/seats/node-build-image/fixtures/events.json index 1e20d2ed..e8dca58f 100644 --- a/contracts/seats/node-build-image/fixtures/events.json +++ b/contracts/seats/node-build-image/fixtures/events.json @@ -36,5 +36,28 @@ "said": "the sandbox could not fetch novox/mesh-sdk", "output-object": "mesh-build-go_sent-objects/m2kx9a1b3c4d5f" } + }, + { + "name": "a node's toolchain reported", + "of": "event:toolchain-reported", + "subject": "mesh.seat.node-build-image.event.toolchain-reported.ace", + "given": { + "node": "ace", + "language": "go", + "version": "1.26.0", + "recipe": "1", + "digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + }, + { + "name": "a running ask says its progress", + "of": "event:step-progress", + "subject": "mesh.seat.node-build-image.event.step-progress.ace", + "given": { + "asker": "runner", + "ask": "a1b2", + "phase": "fetch", + "at": "2026-11-11T10:01:00Z" + } } ] diff --git a/contracts/seats/node-build-image/fixtures/states.json b/contracts/seats/node-build-image/fixtures/states.json index 3cbef09e..149ea1ca 100644 --- a/contracts/seats/node-build-image/fixtures/states.json +++ b/contracts/seats/node-build-image/fixtures/states.json @@ -186,5 +186,17 @@ ], "at": "2026-11-10T10:00:00Z" } + }, + { + "name": "a step keyed under its asker", + "of": "state:steps/..", + "given": { + "v": 1, + "id": "a1b2", + "tool": "build", + "node": "ace", + "state": "running", + "started": "2026-11-05T10:00:00Z" + } } ] diff --git a/contracts/seats/node-build-image/fixtures/verbs.json b/contracts/seats/node-build-image/fixtures/verbs.json index 1e4af4dd..7a02ff62 100644 --- a/contracts/seats/node-build-image/fixtures/verbs.json +++ b/contracts/seats/node-build-image/fixtures/verbs.json @@ -100,5 +100,29 @@ } ] } + }, + { + "name": "steps drops an artifact's kept steps with a reason", + "of": "verb:steps.input", + "given": { + "module": "nats", + "artifact": "nats.program", + "drop": true, + "why": "the detection was wrong" + } + }, + { + "name": "steps answers a drop with the artifact", + "of": "verb:steps.answer", + "given": { + "dropped": "nats.program" + } + }, + { + "name": "steps answers a drop that the hand-act log refused", + "of": "verb:steps.answer", + "given": { + "error": "the hand-act log refused: nothing was deleted" + } } ] diff --git a/contracts/seats/node-build-image/seat.json b/contracts/seats/node-build-image/seat.json index 2d5f83ab..5528da1f 100644 --- a/contracts/seats/node-build-image/seat.json +++ b/contracts/seats/node-build-image/seat.json @@ -65,12 +65,19 @@ "name": "steps", "template": "state", "key-forms": [ + { + "key": "..", + "value": { + "$ref": "#/defs/Step" + }, + "description": "a step on a node, keyed by the asker the bus proves (design 47 G section 467, section 481)" + }, { "key": ".", "value": { "$ref": "#/defs/Step" }, - "description": "a step on a node" + "description": "a step whose ask came on main's subject, with no asker's name (design 47 G section 483)" } ], "description": "the steps this module runs", diff --git a/contracts/seats/node-build-node/fixtures/events.json b/contracts/seats/node-build-node/fixtures/events.json index 0fb1ac2d..1f1924a0 100644 --- a/contracts/seats/node-build-node/fixtures/events.json +++ b/contracts/seats/node-build-node/fixtures/events.json @@ -53,5 +53,28 @@ "said": "the sandbox could not fetch novox/mesh-sdk", "output-object": "mesh-build-go_sent-objects/m2kx9a1b3c4d5f" } + }, + { + "name": "a node's toolchain reported", + "of": "event:toolchain-reported", + "subject": "mesh.seat.node-build-node.event.toolchain-reported.ace", + "given": { + "node": "ace", + "language": "go", + "version": "1.26.0", + "recipe": "1", + "digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + }, + { + "name": "a running ask says its progress", + "of": "event:step-progress", + "subject": "mesh.seat.node-build-node.event.step-progress.ace", + "given": { + "asker": "runner", + "ask": "a1b2", + "phase": "fetch", + "at": "2026-11-11T10:01:00Z" + } } ] diff --git a/contracts/seats/node-build-node/fixtures/states.json b/contracts/seats/node-build-node/fixtures/states.json index ccf06869..e077d0d6 100644 --- a/contracts/seats/node-build-node/fixtures/states.json +++ b/contracts/seats/node-build-node/fixtures/states.json @@ -274,5 +274,17 @@ ], "at": "2026-11-10T10:00:00Z" } + }, + { + "name": "a step keyed under its asker", + "of": "state:steps/..", + "given": { + "v": 1, + "id": "a1b2", + "tool": "build", + "node": "ace", + "state": "running", + "started": "2026-11-05T10:00:00Z" + } } ] diff --git a/contracts/seats/node-build-node/fixtures/verbs.json b/contracts/seats/node-build-node/fixtures/verbs.json index 1e4af4dd..7a02ff62 100644 --- a/contracts/seats/node-build-node/fixtures/verbs.json +++ b/contracts/seats/node-build-node/fixtures/verbs.json @@ -100,5 +100,29 @@ } ] } + }, + { + "name": "steps drops an artifact's kept steps with a reason", + "of": "verb:steps.input", + "given": { + "module": "nats", + "artifact": "nats.program", + "drop": true, + "why": "the detection was wrong" + } + }, + { + "name": "steps answers a drop with the artifact", + "of": "verb:steps.answer", + "given": { + "dropped": "nats.program" + } + }, + { + "name": "steps answers a drop that the hand-act log refused", + "of": "verb:steps.answer", + "given": { + "error": "the hand-act log refused: nothing was deleted" + } } ] diff --git a/contracts/seats/node-build-node/seat.json b/contracts/seats/node-build-node/seat.json index 978f9dde..910ff77c 100644 --- a/contracts/seats/node-build-node/seat.json +++ b/contracts/seats/node-build-node/seat.json @@ -72,12 +72,19 @@ "name": "steps", "template": "state", "key-forms": [ + { + "key": "..", + "value": { + "$ref": "#/defs/Step" + }, + "description": "a step on a node, keyed by the asker the bus proves (design 47 G section 467, section 481)" + }, { "key": ".", "value": { "$ref": "#/defs/Step" }, - "description": "a step on a node" + "description": "a step whose ask came on main's subject, with no asker's name (design 47 G section 483)" } ], "description": "the steps this module runs", diff --git a/contracts/ts/seats/meshwants/index.d.ts b/contracts/ts/seats/meshwants/index.d.ts index 804dc8c1..a2760e6c 100644 --- a/contracts/ts/seats/meshwants/index.d.ts +++ b/contracts/ts/seats/meshwants/index.d.ts @@ -29,6 +29,7 @@ export declare const events: { readonly room: "room"; readonly stepEnded: "step-ended"; readonly versionDelivered: "version-delivered"; + readonly stepProgress: "step-progress"; }; export declare const family: { readonly name: "mesh-wants.events"; readonly maxAgeDays: 7 }; @@ -61,6 +62,12 @@ export declare const versionDeliveredStream: "SEAT_mesh-wants_version-delivered" export declare const versionDeliveredSubject: "mesh.seat.mesh-wants.event.version-delivered"; +export declare const stepProgressStream: "SEAT_mesh-wants_step-progress"; + +export declare function stepProgressSubject(node: string): string; + +export declare const stepProgressSubjectAll: "mesh.seat.mesh-wants.event.step-progress.*"; + export declare const stepSubject: "mesh.seat.mesh-wants.accept.step"; export declare const states: { @@ -474,6 +481,14 @@ export interface VersionDeliveredEvent { "version": string; } +/** is the body of the event step-progress. */ +export interface StepProgressEvent { + "ask": string; + "asker": string; + "at": string; + "phase": "fetch" | "build" | "run" | "report"; +} + /** is one value of the state wants under the key want... */ export interface WantsByWantNodeModuleValue { /** again or skip */ diff --git a/contracts/ts/seats/meshwants/index.js b/contracts/ts/seats/meshwants/index.js index 96565037..488f1228 100644 --- a/contracts/ts/seats/meshwants/index.js +++ b/contracts/ts/seats/meshwants/index.js @@ -29,6 +29,7 @@ export const events = { room: "room", stepEnded: "step-ended", versionDelivered: "version-delivered", + stepProgress: "step-progress", }; export const family = { name: "mesh-wants.events", maxAgeDays: 7 }; @@ -63,6 +64,14 @@ export const versionDeliveredStream = "SEAT_mesh-wants_version-delivered"; export const versionDeliveredSubject = "mesh.seat.mesh-wants.event.version-delivered"; +export const stepProgressStream = "SEAT_mesh-wants_step-progress"; + +export function stepProgressSubject(node) { + return "mesh.seat.mesh-wants.event.step-progress." + node; +} + +export const stepProgressSubjectAll = "mesh.seat.mesh-wants.event.step-progress.*"; + export const stepSubject = "mesh.seat.mesh-wants.accept.step"; export const states = { diff --git a/contracts/ts/seats/nodebuildgo/index.d.ts b/contracts/ts/seats/nodebuildgo/index.d.ts index ff6e8d0e..ac126e27 100644 --- a/contracts/ts/seats/nodebuildgo/index.d.ts +++ b/contracts/ts/seats/nodebuildgo/index.d.ts @@ -26,6 +26,8 @@ export declare const accepts: { export declare const events: { readonly stepEnded: "step-ended"; + readonly toolchainReported: "toolchain-reported"; + readonly stepProgress: "step-progress"; }; export declare const family: { readonly name: "node-build-go.events"; readonly maxAgeDays: 7 }; @@ -36,6 +38,18 @@ export declare function stepEndedSubject(node: string): string; export declare const stepEndedSubjectAll: "mesh.seat.node-build-go.event.step-ended.*"; +export declare const toolchainReportedStream: "SEAT_node-build-go_toolchain-reported"; + +export declare function toolchainReportedSubject(node: string): string; + +export declare const toolchainReportedSubjectAll: "mesh.seat.node-build-go.event.toolchain-reported.*"; + +export declare const stepProgressStream: "SEAT_node-build-go_step-progress"; + +export declare function stepProgressSubject(node: string): string; + +export declare const stepProgressSubjectAll: "mesh.seat.node-build-go.event.step-progress.*"; + export declare const deliverySubject: "mesh.seat.node-build-go.accept.delivery"; export declare const checkSubject: "mesh.seat.node-build-go.accept.check"; @@ -55,6 +69,12 @@ export declare const states: { readonly detectedSteps: "detected-steps"; }; +/** The key of one value of the state steps under the form "..". */ +export declare function stepsByAskerNodeIDKey(asker: string, node: string, id: string): string; + +/** The parts of a key of the form "..", or undefined for a key of another form. */ +export declare function parseStepsByAskerNodeIDKey(_key: string): { asker: string; node: string; id: string } | undefined; + /** The key of one value of the state steps under the form ".". */ export declare function stepsByNodeIDKey(node: string, id: string): string; @@ -94,6 +114,7 @@ export declare function parseDetectedStepsByModuleArtifactKey(_key: string): { m export type AskArtifact = buildStep.AskArtifact; export type BuildAnswer = buildStep.BuildAnswer; export type BuiltArtifact = buildStep.BuiltArtifact; +export type BuiltArtifactResolvedItem = buildStep.BuiltArtifactResolvedItem; export type Capacity = buildStep.Capacity; export type CheckAnswer = buildStep.CheckAnswer; export type ComposeAnswer = buildStep.ComposeAnswer; @@ -107,6 +128,7 @@ export type DropInput = buildStep.DropInput; export type Fetch = buildStep.Fetch; export type FetchAnswer = buildStep.FetchAnswer; export type FetchBesideItem = buildStep.FetchBesideItem; +export type FetchBuildsItem = buildStep.FetchBuildsItem; export type FetchPackage = buildStep.FetchPackage; export type FetchRefusal = buildStep.FetchRefusal; export type KeptArtifactAnswer = buildStep.KeptArtifactAnswer; @@ -139,6 +161,8 @@ export type StepAskCompose = buildStep.StepAskCompose; export type StepEnded = buildStep.StepEnded; export type StepEndedEvent = buildStep.StepEndedEvent; export type StepNames = buildStep.StepNames; +export type StepProgress = buildStep.StepProgress; +export type StepProgressEvent = buildStep.StepProgressEvent; export type StepResult = buildStep.StepResult; export type StepRun = buildStep.StepRun; export type StepsAnswer = buildStep.StepsAnswer; @@ -146,6 +170,8 @@ export type StepsInput = buildStep.StepsInput; export type TakenVersion = buildStep.TakenVersion; export type TestCounts = buildStep.TestCounts; export type Toolchain = buildStep.Toolchain; +export type ToolchainReported = buildStep.ToolchainReported; +export type ToolchainReportedEvent = buildStep.ToolchainReportedEvent; export type ToolchainUsed = buildStep.ToolchainUsed; /** is the body of delivery. */ @@ -166,6 +192,9 @@ export type RecheckAccept = StepAsk; /** is the body of check.run. */ export type CheckRunAccept = StepAsk; +/** is one value of the state steps under the key ... */ +export type StepsByAskerNodeIDValue = Step; + /** is one value of the state steps under the key .. */ export type StepsByNodeIDValue = Step; diff --git a/contracts/ts/seats/nodebuildgo/index.js b/contracts/ts/seats/nodebuildgo/index.js index e80d7956..cf4a3f99 100644 --- a/contracts/ts/seats/nodebuildgo/index.js +++ b/contracts/ts/seats/nodebuildgo/index.js @@ -24,6 +24,8 @@ export const accepts = { export const events = { stepEnded: "step-ended", + toolchainReported: "toolchain-reported", + stepProgress: "step-progress", }; export const family = { name: "node-build-go.events", maxAgeDays: 7 }; @@ -36,6 +38,22 @@ export function stepEndedSubject(node) { export const stepEndedSubjectAll = "mesh.seat.node-build-go.event.step-ended.*"; +export const toolchainReportedStream = "SEAT_node-build-go_toolchain-reported"; + +export function toolchainReportedSubject(node) { + return "mesh.seat.node-build-go.event.toolchain-reported." + node; +} + +export const toolchainReportedSubjectAll = "mesh.seat.node-build-go.event.toolchain-reported.*"; + +export const stepProgressStream = "SEAT_node-build-go_step-progress"; + +export function stepProgressSubject(node) { + return "mesh.seat.node-build-go.event.step-progress." + node; +} + +export const stepProgressSubjectAll = "mesh.seat.node-build-go.event.step-progress.*"; + export const deliverySubject = "mesh.seat.node-build-go.accept.delivery"; export const checkSubject = "mesh.seat.node-build-go.accept.check"; @@ -55,6 +73,29 @@ export const states = { detectedSteps: "detected-steps", }; +/** The key of one value of the state steps under the form "..". */ +export function stepsByAskerNodeIDKey(asker, node, id) { + return asker + "." + node + "." + id; +} + +/** The parts of a key of the form "..", or undefined for a key of another form. */ +export function parseStepsByAskerNodeIDKey(_key) { + let _rest = _key; + const _at0 = _rest.indexOf("."); + if (_at0 <= 0) return undefined; + const asker = _rest.slice(0, _at0); + _rest = _rest.slice(_at0 + 1); + const _at2 = _rest.indexOf("."); + if (_at2 <= 0) return undefined; + const node = _rest.slice(0, _at2); + _rest = _rest.slice(_at2 + 1); + if (_rest === "") return undefined; + const id = _rest; + _rest = ""; + if (_rest !== "") return undefined; + return { asker, node, id }; +} + /** The key of one value of the state steps under the form ".". */ export function stepsByNodeIDKey(node, id) { return node + "." + id; diff --git a/contracts/ts/seats/nodebuildgo/parts/build-step.d.ts b/contracts/ts/seats/nodebuildgo/parts/build-step.d.ts index 3a3c18ba..a6cc4551 100644 --- a/contracts/ts/seats/nodebuildgo/parts/build-step.d.ts +++ b/contracts/ts/seats/nodebuildgo/parts/build-step.d.ts @@ -13,6 +13,8 @@ export declare const verbs: { export declare const events: { readonly stepEnded: "step-ended"; + readonly toolchainReported: "toolchain-reported"; + readonly stepProgress: "step-progress"; }; /** one artifact of a build ask: built from the fetched tree, or copied from an upstream pin */ @@ -32,11 +34,20 @@ export interface BuildAnswer { "toolchain"?: ToolchainUsed; } +export interface BuiltArtifactResolvedItem { + /** sha256: of the package's bytes as the build fetched them */ + "digest": string; + "package": string; + "version": string; +} + /** one artifact a build made: its digest and the reference it was published under */ export interface BuiltArtifact { "artifact": string; "digest": string; "reference"?: string; + /** one per mesh package the build resolved (the catalog graph entry's packages, the edges of kind 1) */ + "resolved"?: BuiltArtifactResolvedItem[]; } /** how many steps the node runs at once */ @@ -100,6 +111,15 @@ export interface FetchBesideItem { "repository": string; } +export interface FetchBuildsItem { + "artifact": string; + /** the member's build digest, sha256: */ + "digest": string; + "module": string; + /** the name in its ecosystem (the Go module path, the npm name) */ + "package": string; +} + /** the TypeScript module's group package: the group's tagged version of the SDK ({name, version}), placed from the registry in each member whatever its lock names, or the group's package member ({repository, commit, ref}), packed by the fetch and served to the members in place of the registry's */ export interface FetchPackage { "commit"?: string; @@ -115,6 +135,8 @@ export interface Fetch { "base"?: string; /** the sibling repositories placed beside the fetched tree */ "beside"?: FetchBesideItem[]; + /** a group run's package members, each by its build's digest: the build module reads each from the artifact store at artifact-store:////blobs/, checks the bytes against the digest and lays them as the group package; a package named here is named in neither packages nor package */ + "builds"?: FetchBuildsItem[]; /** the full commit */ "commit": string; /** the module's directory in the repository ("." for its root), with paths */ @@ -416,6 +438,14 @@ export interface StepNames { "repositories": string[]; } +/** a running ask's progress, said once a minute while the holder runs it (design 47 G section 496) */ +export interface StepProgress { + "ask": string; + "asker": string; + "at": string; + "phase": "fetch" | "build" | "run" | "report"; +} + /** what a step answered, by the kind of ask: a build's artifacts, toolchain, imports, out-of-step modules and the artifact it kept; a check's fetch and levels; a compose's built, key and verdict; a run's built, key, exit, output and said; a declined ask's declined, needs and lacks. The fields of the kinds do not meet, so one layout holds the five */ export interface StepResult { "artifacts"?: BuiltArtifact[]; @@ -502,8 +532,20 @@ export interface Toolchain { "version": string; } +/** a node's toolchain of a language, said at the build module's start and each time its toolchains key of the language changes (design 47 G section 492) */ +export interface ToolchainReported { + /** the digest of the program of the provision the build module requires for that language's toolchain, never the build module's own */ + "digest": string; + "language": string; + "node": string; + "recipe": string; + "version": string; +} + /** the toolchain a build was made with: the language, the installed version and the build module's recipe */ export interface ToolchainUsed { + /** the building node's digest of its toolchain program (kind 3's entry), never the build module's own program */ + "digest"?: string; "language": string; "recipe": string; "version": string; @@ -579,22 +621,36 @@ export interface KeptArtifactAnswer { export interface StepsInput { /** find them anew, not from the kept answer */ "again"?: boolean; - /** the artifact reference, : */ + /** the key of detected-steps, . */ "artifact": string; - "fetch": Fetch; + /** delete the artifact's kept key instead of detecting; needs why */ + "drop"?: boolean; + "fetch"?: Fetch; "fingerprint"?: string; "module": string; + /** with drop, why: a non-empty text, written in the hand-act log with the caller the bus proved */ + "why"?: string; } /** is what steps answers. */ export interface StepsAnswer { - "artifact": string; + "artifact"?: string; + /** the artifact whose key a drop deleted */ + "dropped"?: string; + /** why a drop deleted nothing, naming the hand-act log when it refused */ + "error"?: string; "fingerprint"?: string; /** answered from the kept answer */ "kept"?: boolean; - "steps": DetectedStep[]; + "steps"?: DetectedStep[]; } /** is the body of the event step-ended. */ export type StepEndedEvent = StepEnded; +/** is the body of the event toolchain-reported. */ +export type ToolchainReportedEvent = ToolchainReported; + +/** is the body of the event step-progress. */ +export type StepProgressEvent = StepProgress; + diff --git a/contracts/ts/seats/nodebuildgo/parts/build-step.js b/contracts/ts/seats/nodebuildgo/parts/build-step.js index 027da325..155a68be 100644 --- a/contracts/ts/seats/nodebuildgo/parts/build-step.js +++ b/contracts/ts/seats/nodebuildgo/parts/build-step.js @@ -13,6 +13,8 @@ export const verbs = { export const events = { stepEnded: "step-ended", + toolchainReported: "toolchain-reported", + stepProgress: "step-progress", }; export {}; diff --git a/contracts/ts/seats/nodebuildimage/index.d.ts b/contracts/ts/seats/nodebuildimage/index.d.ts index 2cfdc13b..06659b29 100644 --- a/contracts/ts/seats/nodebuildimage/index.d.ts +++ b/contracts/ts/seats/nodebuildimage/index.d.ts @@ -25,6 +25,8 @@ export declare const accepts: { export declare const events: { readonly stepEnded: "step-ended"; + readonly toolchainReported: "toolchain-reported"; + readonly stepProgress: "step-progress"; }; export declare const family: { readonly name: "node-build-image.events"; readonly maxAgeDays: 7 }; @@ -35,6 +37,18 @@ export declare function stepEndedSubject(node: string): string; export declare const stepEndedSubjectAll: "mesh.seat.node-build-image.event.step-ended.*"; +export declare const toolchainReportedStream: "SEAT_node-build-image_toolchain-reported"; + +export declare function toolchainReportedSubject(node: string): string; + +export declare const toolchainReportedSubjectAll: "mesh.seat.node-build-image.event.toolchain-reported.*"; + +export declare const stepProgressStream: "SEAT_node-build-image_step-progress"; + +export declare function stepProgressSubject(node: string): string; + +export declare const stepProgressSubjectAll: "mesh.seat.node-build-image.event.step-progress.*"; + export declare const deliverySubject: "mesh.seat.node-build-image.accept.delivery"; export declare const checkSubject: "mesh.seat.node-build-image.accept.check"; @@ -52,6 +66,12 @@ export declare const states: { readonly detectedSteps: "detected-steps"; }; +/** The key of one value of the state steps under the form "..". */ +export declare function stepsByAskerNodeIDKey(asker: string, node: string, id: string): string; + +/** The parts of a key of the form "..", or undefined for a key of another form. */ +export declare function parseStepsByAskerNodeIDKey(_key: string): { asker: string; node: string; id: string } | undefined; + /** The key of one value of the state steps under the form ".". */ export declare function stepsByNodeIDKey(node: string, id: string): string; @@ -91,6 +111,7 @@ export declare function parseDetectedStepsByModuleArtifactKey(_key: string): { m export type AskArtifact = buildStep.AskArtifact; export type BuildAnswer = buildStep.BuildAnswer; export type BuiltArtifact = buildStep.BuiltArtifact; +export type BuiltArtifactResolvedItem = buildStep.BuiltArtifactResolvedItem; export type Capacity = buildStep.Capacity; export type CheckAnswer = buildStep.CheckAnswer; export type ComposeAnswer = buildStep.ComposeAnswer; @@ -104,6 +125,7 @@ export type DropInput = buildStep.DropInput; export type Fetch = buildStep.Fetch; export type FetchAnswer = buildStep.FetchAnswer; export type FetchBesideItem = buildStep.FetchBesideItem; +export type FetchBuildsItem = buildStep.FetchBuildsItem; export type FetchPackage = buildStep.FetchPackage; export type FetchRefusal = buildStep.FetchRefusal; export type KeptArtifactAnswer = buildStep.KeptArtifactAnswer; @@ -136,6 +158,8 @@ export type StepAskCompose = buildStep.StepAskCompose; export type StepEnded = buildStep.StepEnded; export type StepEndedEvent = buildStep.StepEndedEvent; export type StepNames = buildStep.StepNames; +export type StepProgress = buildStep.StepProgress; +export type StepProgressEvent = buildStep.StepProgressEvent; export type StepResult = buildStep.StepResult; export type StepRun = buildStep.StepRun; export type StepsAnswer = buildStep.StepsAnswer; @@ -143,6 +167,8 @@ export type StepsInput = buildStep.StepsInput; export type TakenVersion = buildStep.TakenVersion; export type TestCounts = buildStep.TestCounts; export type Toolchain = buildStep.Toolchain; +export type ToolchainReported = buildStep.ToolchainReported; +export type ToolchainReportedEvent = buildStep.ToolchainReportedEvent; export type ToolchainUsed = buildStep.ToolchainUsed; /** is the body of delivery. */ @@ -160,6 +186,9 @@ export type DeliveryStepAccept = StepAsk; /** is the body of recheck. */ export type RecheckAccept = StepAsk; +/** is one value of the state steps under the key ... */ +export type StepsByAskerNodeIDValue = Step; + /** is one value of the state steps under the key .. */ export type StepsByNodeIDValue = Step; diff --git a/contracts/ts/seats/nodebuildimage/index.js b/contracts/ts/seats/nodebuildimage/index.js index 6df298da..8a23cc70 100644 --- a/contracts/ts/seats/nodebuildimage/index.js +++ b/contracts/ts/seats/nodebuildimage/index.js @@ -23,6 +23,8 @@ export const accepts = { export const events = { stepEnded: "step-ended", + toolchainReported: "toolchain-reported", + stepProgress: "step-progress", }; export const family = { name: "node-build-image.events", maxAgeDays: 7 }; @@ -35,6 +37,22 @@ export function stepEndedSubject(node) { export const stepEndedSubjectAll = "mesh.seat.node-build-image.event.step-ended.*"; +export const toolchainReportedStream = "SEAT_node-build-image_toolchain-reported"; + +export function toolchainReportedSubject(node) { + return "mesh.seat.node-build-image.event.toolchain-reported." + node; +} + +export const toolchainReportedSubjectAll = "mesh.seat.node-build-image.event.toolchain-reported.*"; + +export const stepProgressStream = "SEAT_node-build-image_step-progress"; + +export function stepProgressSubject(node) { + return "mesh.seat.node-build-image.event.step-progress." + node; +} + +export const stepProgressSubjectAll = "mesh.seat.node-build-image.event.step-progress.*"; + export const deliverySubject = "mesh.seat.node-build-image.accept.delivery"; export const checkSubject = "mesh.seat.node-build-image.accept.check"; @@ -52,6 +70,29 @@ export const states = { detectedSteps: "detected-steps", }; +/** The key of one value of the state steps under the form "..". */ +export function stepsByAskerNodeIDKey(asker, node, id) { + return asker + "." + node + "." + id; +} + +/** The parts of a key of the form "..", or undefined for a key of another form. */ +export function parseStepsByAskerNodeIDKey(_key) { + let _rest = _key; + const _at0 = _rest.indexOf("."); + if (_at0 <= 0) return undefined; + const asker = _rest.slice(0, _at0); + _rest = _rest.slice(_at0 + 1); + const _at2 = _rest.indexOf("."); + if (_at2 <= 0) return undefined; + const node = _rest.slice(0, _at2); + _rest = _rest.slice(_at2 + 1); + if (_rest === "") return undefined; + const id = _rest; + _rest = ""; + if (_rest !== "") return undefined; + return { asker, node, id }; +} + /** The key of one value of the state steps under the form ".". */ export function stepsByNodeIDKey(node, id) { return node + "." + id; diff --git a/contracts/ts/seats/nodebuildimage/parts/build-step.d.ts b/contracts/ts/seats/nodebuildimage/parts/build-step.d.ts index 3a3c18ba..a6cc4551 100644 --- a/contracts/ts/seats/nodebuildimage/parts/build-step.d.ts +++ b/contracts/ts/seats/nodebuildimage/parts/build-step.d.ts @@ -13,6 +13,8 @@ export declare const verbs: { export declare const events: { readonly stepEnded: "step-ended"; + readonly toolchainReported: "toolchain-reported"; + readonly stepProgress: "step-progress"; }; /** one artifact of a build ask: built from the fetched tree, or copied from an upstream pin */ @@ -32,11 +34,20 @@ export interface BuildAnswer { "toolchain"?: ToolchainUsed; } +export interface BuiltArtifactResolvedItem { + /** sha256: of the package's bytes as the build fetched them */ + "digest": string; + "package": string; + "version": string; +} + /** one artifact a build made: its digest and the reference it was published under */ export interface BuiltArtifact { "artifact": string; "digest": string; "reference"?: string; + /** one per mesh package the build resolved (the catalog graph entry's packages, the edges of kind 1) */ + "resolved"?: BuiltArtifactResolvedItem[]; } /** how many steps the node runs at once */ @@ -100,6 +111,15 @@ export interface FetchBesideItem { "repository": string; } +export interface FetchBuildsItem { + "artifact": string; + /** the member's build digest, sha256: */ + "digest": string; + "module": string; + /** the name in its ecosystem (the Go module path, the npm name) */ + "package": string; +} + /** the TypeScript module's group package: the group's tagged version of the SDK ({name, version}), placed from the registry in each member whatever its lock names, or the group's package member ({repository, commit, ref}), packed by the fetch and served to the members in place of the registry's */ export interface FetchPackage { "commit"?: string; @@ -115,6 +135,8 @@ export interface Fetch { "base"?: string; /** the sibling repositories placed beside the fetched tree */ "beside"?: FetchBesideItem[]; + /** a group run's package members, each by its build's digest: the build module reads each from the artifact store at artifact-store:////blobs/, checks the bytes against the digest and lays them as the group package; a package named here is named in neither packages nor package */ + "builds"?: FetchBuildsItem[]; /** the full commit */ "commit": string; /** the module's directory in the repository ("." for its root), with paths */ @@ -416,6 +438,14 @@ export interface StepNames { "repositories": string[]; } +/** a running ask's progress, said once a minute while the holder runs it (design 47 G section 496) */ +export interface StepProgress { + "ask": string; + "asker": string; + "at": string; + "phase": "fetch" | "build" | "run" | "report"; +} + /** what a step answered, by the kind of ask: a build's artifacts, toolchain, imports, out-of-step modules and the artifact it kept; a check's fetch and levels; a compose's built, key and verdict; a run's built, key, exit, output and said; a declined ask's declined, needs and lacks. The fields of the kinds do not meet, so one layout holds the five */ export interface StepResult { "artifacts"?: BuiltArtifact[]; @@ -502,8 +532,20 @@ export interface Toolchain { "version": string; } +/** a node's toolchain of a language, said at the build module's start and each time its toolchains key of the language changes (design 47 G section 492) */ +export interface ToolchainReported { + /** the digest of the program of the provision the build module requires for that language's toolchain, never the build module's own */ + "digest": string; + "language": string; + "node": string; + "recipe": string; + "version": string; +} + /** the toolchain a build was made with: the language, the installed version and the build module's recipe */ export interface ToolchainUsed { + /** the building node's digest of its toolchain program (kind 3's entry), never the build module's own program */ + "digest"?: string; "language": string; "recipe": string; "version": string; @@ -579,22 +621,36 @@ export interface KeptArtifactAnswer { export interface StepsInput { /** find them anew, not from the kept answer */ "again"?: boolean; - /** the artifact reference, : */ + /** the key of detected-steps, . */ "artifact": string; - "fetch": Fetch; + /** delete the artifact's kept key instead of detecting; needs why */ + "drop"?: boolean; + "fetch"?: Fetch; "fingerprint"?: string; "module": string; + /** with drop, why: a non-empty text, written in the hand-act log with the caller the bus proved */ + "why"?: string; } /** is what steps answers. */ export interface StepsAnswer { - "artifact": string; + "artifact"?: string; + /** the artifact whose key a drop deleted */ + "dropped"?: string; + /** why a drop deleted nothing, naming the hand-act log when it refused */ + "error"?: string; "fingerprint"?: string; /** answered from the kept answer */ "kept"?: boolean; - "steps": DetectedStep[]; + "steps"?: DetectedStep[]; } /** is the body of the event step-ended. */ export type StepEndedEvent = StepEnded; +/** is the body of the event toolchain-reported. */ +export type ToolchainReportedEvent = ToolchainReported; + +/** is the body of the event step-progress. */ +export type StepProgressEvent = StepProgress; + diff --git a/contracts/ts/seats/nodebuildimage/parts/build-step.js b/contracts/ts/seats/nodebuildimage/parts/build-step.js index 027da325..155a68be 100644 --- a/contracts/ts/seats/nodebuildimage/parts/build-step.js +++ b/contracts/ts/seats/nodebuildimage/parts/build-step.js @@ -13,6 +13,8 @@ export const verbs = { export const events = { stepEnded: "step-ended", + toolchainReported: "toolchain-reported", + stepProgress: "step-progress", }; export {}; diff --git a/contracts/ts/seats/nodebuildnode/index.d.ts b/contracts/ts/seats/nodebuildnode/index.d.ts index 3a87dc86..0d4743ec 100644 --- a/contracts/ts/seats/nodebuildnode/index.d.ts +++ b/contracts/ts/seats/nodebuildnode/index.d.ts @@ -25,6 +25,8 @@ export declare const accepts: { export declare const events: { readonly stepEnded: "step-ended"; + readonly toolchainReported: "toolchain-reported"; + readonly stepProgress: "step-progress"; }; export declare const family: { readonly name: "node-build-node.events"; readonly maxAgeDays: 7 }; @@ -35,6 +37,18 @@ export declare function stepEndedSubject(node: string): string; export declare const stepEndedSubjectAll: "mesh.seat.node-build-node.event.step-ended.*"; +export declare const toolchainReportedStream: "SEAT_node-build-node_toolchain-reported"; + +export declare function toolchainReportedSubject(node: string): string; + +export declare const toolchainReportedSubjectAll: "mesh.seat.node-build-node.event.toolchain-reported.*"; + +export declare const stepProgressStream: "SEAT_node-build-node_step-progress"; + +export declare function stepProgressSubject(node: string): string; + +export declare const stepProgressSubjectAll: "mesh.seat.node-build-node.event.step-progress.*"; + export declare const deliverySubject: "mesh.seat.node-build-node.accept.delivery"; export declare const checkSubject: "mesh.seat.node-build-node.accept.check"; @@ -52,6 +66,12 @@ export declare const states: { readonly detectedSteps: "detected-steps"; }; +/** The key of one value of the state steps under the form "..". */ +export declare function stepsByAskerNodeIDKey(asker: string, node: string, id: string): string; + +/** The parts of a key of the form "..", or undefined for a key of another form. */ +export declare function parseStepsByAskerNodeIDKey(_key: string): { asker: string; node: string; id: string } | undefined; + /** The key of one value of the state steps under the form ".". */ export declare function stepsByNodeIDKey(node: string, id: string): string; @@ -91,6 +111,7 @@ export declare function parseDetectedStepsByModuleArtifactKey(_key: string): { m export type AskArtifact = buildStep.AskArtifact; export type BuildAnswer = buildStep.BuildAnswer; export type BuiltArtifact = buildStep.BuiltArtifact; +export type BuiltArtifactResolvedItem = buildStep.BuiltArtifactResolvedItem; export type Capacity = buildStep.Capacity; export type CheckAnswer = buildStep.CheckAnswer; export type ComposeAnswer = buildStep.ComposeAnswer; @@ -104,6 +125,7 @@ export type DropInput = buildStep.DropInput; export type Fetch = buildStep.Fetch; export type FetchAnswer = buildStep.FetchAnswer; export type FetchBesideItem = buildStep.FetchBesideItem; +export type FetchBuildsItem = buildStep.FetchBuildsItem; export type FetchPackage = buildStep.FetchPackage; export type FetchRefusal = buildStep.FetchRefusal; export type KeptArtifactAnswer = buildStep.KeptArtifactAnswer; @@ -136,6 +158,8 @@ export type StepAskCompose = buildStep.StepAskCompose; export type StepEnded = buildStep.StepEnded; export type StepEndedEvent = buildStep.StepEndedEvent; export type StepNames = buildStep.StepNames; +export type StepProgress = buildStep.StepProgress; +export type StepProgressEvent = buildStep.StepProgressEvent; export type StepResult = buildStep.StepResult; export type StepRun = buildStep.StepRun; export type StepsAnswer = buildStep.StepsAnswer; @@ -143,6 +167,8 @@ export type StepsInput = buildStep.StepsInput; export type TakenVersion = buildStep.TakenVersion; export type TestCounts = buildStep.TestCounts; export type Toolchain = buildStep.Toolchain; +export type ToolchainReported = buildStep.ToolchainReported; +export type ToolchainReportedEvent = buildStep.ToolchainReportedEvent; export type ToolchainUsed = buildStep.ToolchainUsed; /** is the body of delivery. */ @@ -160,6 +186,9 @@ export type DeliveryStepAccept = StepAsk; /** is the body of recheck. */ export type RecheckAccept = StepAsk; +/** is one value of the state steps under the key ... */ +export type StepsByAskerNodeIDValue = Step; + /** is one value of the state steps under the key .. */ export type StepsByNodeIDValue = Step; diff --git a/contracts/ts/seats/nodebuildnode/index.js b/contracts/ts/seats/nodebuildnode/index.js index 3c9047d8..6cda8ea2 100644 --- a/contracts/ts/seats/nodebuildnode/index.js +++ b/contracts/ts/seats/nodebuildnode/index.js @@ -23,6 +23,8 @@ export const accepts = { export const events = { stepEnded: "step-ended", + toolchainReported: "toolchain-reported", + stepProgress: "step-progress", }; export const family = { name: "node-build-node.events", maxAgeDays: 7 }; @@ -35,6 +37,22 @@ export function stepEndedSubject(node) { export const stepEndedSubjectAll = "mesh.seat.node-build-node.event.step-ended.*"; +export const toolchainReportedStream = "SEAT_node-build-node_toolchain-reported"; + +export function toolchainReportedSubject(node) { + return "mesh.seat.node-build-node.event.toolchain-reported." + node; +} + +export const toolchainReportedSubjectAll = "mesh.seat.node-build-node.event.toolchain-reported.*"; + +export const stepProgressStream = "SEAT_node-build-node_step-progress"; + +export function stepProgressSubject(node) { + return "mesh.seat.node-build-node.event.step-progress." + node; +} + +export const stepProgressSubjectAll = "mesh.seat.node-build-node.event.step-progress.*"; + export const deliverySubject = "mesh.seat.node-build-node.accept.delivery"; export const checkSubject = "mesh.seat.node-build-node.accept.check"; @@ -52,6 +70,29 @@ export const states = { detectedSteps: "detected-steps", }; +/** The key of one value of the state steps under the form "..". */ +export function stepsByAskerNodeIDKey(asker, node, id) { + return asker + "." + node + "." + id; +} + +/** The parts of a key of the form "..", or undefined for a key of another form. */ +export function parseStepsByAskerNodeIDKey(_key) { + let _rest = _key; + const _at0 = _rest.indexOf("."); + if (_at0 <= 0) return undefined; + const asker = _rest.slice(0, _at0); + _rest = _rest.slice(_at0 + 1); + const _at2 = _rest.indexOf("."); + if (_at2 <= 0) return undefined; + const node = _rest.slice(0, _at2); + _rest = _rest.slice(_at2 + 1); + if (_rest === "") return undefined; + const id = _rest; + _rest = ""; + if (_rest !== "") return undefined; + return { asker, node, id }; +} + /** The key of one value of the state steps under the form ".". */ export function stepsByNodeIDKey(node, id) { return node + "." + id; diff --git a/contracts/ts/seats/nodebuildnode/parts/build-step.d.ts b/contracts/ts/seats/nodebuildnode/parts/build-step.d.ts index 3a3c18ba..a6cc4551 100644 --- a/contracts/ts/seats/nodebuildnode/parts/build-step.d.ts +++ b/contracts/ts/seats/nodebuildnode/parts/build-step.d.ts @@ -13,6 +13,8 @@ export declare const verbs: { export declare const events: { readonly stepEnded: "step-ended"; + readonly toolchainReported: "toolchain-reported"; + readonly stepProgress: "step-progress"; }; /** one artifact of a build ask: built from the fetched tree, or copied from an upstream pin */ @@ -32,11 +34,20 @@ export interface BuildAnswer { "toolchain"?: ToolchainUsed; } +export interface BuiltArtifactResolvedItem { + /** sha256: of the package's bytes as the build fetched them */ + "digest": string; + "package": string; + "version": string; +} + /** one artifact a build made: its digest and the reference it was published under */ export interface BuiltArtifact { "artifact": string; "digest": string; "reference"?: string; + /** one per mesh package the build resolved (the catalog graph entry's packages, the edges of kind 1) */ + "resolved"?: BuiltArtifactResolvedItem[]; } /** how many steps the node runs at once */ @@ -100,6 +111,15 @@ export interface FetchBesideItem { "repository": string; } +export interface FetchBuildsItem { + "artifact": string; + /** the member's build digest, sha256: */ + "digest": string; + "module": string; + /** the name in its ecosystem (the Go module path, the npm name) */ + "package": string; +} + /** the TypeScript module's group package: the group's tagged version of the SDK ({name, version}), placed from the registry in each member whatever its lock names, or the group's package member ({repository, commit, ref}), packed by the fetch and served to the members in place of the registry's */ export interface FetchPackage { "commit"?: string; @@ -115,6 +135,8 @@ export interface Fetch { "base"?: string; /** the sibling repositories placed beside the fetched tree */ "beside"?: FetchBesideItem[]; + /** a group run's package members, each by its build's digest: the build module reads each from the artifact store at artifact-store:////blobs/, checks the bytes against the digest and lays them as the group package; a package named here is named in neither packages nor package */ + "builds"?: FetchBuildsItem[]; /** the full commit */ "commit": string; /** the module's directory in the repository ("." for its root), with paths */ @@ -416,6 +438,14 @@ export interface StepNames { "repositories": string[]; } +/** a running ask's progress, said once a minute while the holder runs it (design 47 G section 496) */ +export interface StepProgress { + "ask": string; + "asker": string; + "at": string; + "phase": "fetch" | "build" | "run" | "report"; +} + /** what a step answered, by the kind of ask: a build's artifacts, toolchain, imports, out-of-step modules and the artifact it kept; a check's fetch and levels; a compose's built, key and verdict; a run's built, key, exit, output and said; a declined ask's declined, needs and lacks. The fields of the kinds do not meet, so one layout holds the five */ export interface StepResult { "artifacts"?: BuiltArtifact[]; @@ -502,8 +532,20 @@ export interface Toolchain { "version": string; } +/** a node's toolchain of a language, said at the build module's start and each time its toolchains key of the language changes (design 47 G section 492) */ +export interface ToolchainReported { + /** the digest of the program of the provision the build module requires for that language's toolchain, never the build module's own */ + "digest": string; + "language": string; + "node": string; + "recipe": string; + "version": string; +} + /** the toolchain a build was made with: the language, the installed version and the build module's recipe */ export interface ToolchainUsed { + /** the building node's digest of its toolchain program (kind 3's entry), never the build module's own program */ + "digest"?: string; "language": string; "recipe": string; "version": string; @@ -579,22 +621,36 @@ export interface KeptArtifactAnswer { export interface StepsInput { /** find them anew, not from the kept answer */ "again"?: boolean; - /** the artifact reference, : */ + /** the key of detected-steps, . */ "artifact": string; - "fetch": Fetch; + /** delete the artifact's kept key instead of detecting; needs why */ + "drop"?: boolean; + "fetch"?: Fetch; "fingerprint"?: string; "module": string; + /** with drop, why: a non-empty text, written in the hand-act log with the caller the bus proved */ + "why"?: string; } /** is what steps answers. */ export interface StepsAnswer { - "artifact": string; + "artifact"?: string; + /** the artifact whose key a drop deleted */ + "dropped"?: string; + /** why a drop deleted nothing, naming the hand-act log when it refused */ + "error"?: string; "fingerprint"?: string; /** answered from the kept answer */ "kept"?: boolean; - "steps": DetectedStep[]; + "steps"?: DetectedStep[]; } /** is the body of the event step-ended. */ export type StepEndedEvent = StepEnded; +/** is the body of the event toolchain-reported. */ +export type ToolchainReportedEvent = ToolchainReported; + +/** is the body of the event step-progress. */ +export type StepProgressEvent = StepProgress; + diff --git a/contracts/ts/seats/nodebuildnode/parts/build-step.js b/contracts/ts/seats/nodebuildnode/parts/build-step.js index 027da325..155a68be 100644 --- a/contracts/ts/seats/nodebuildnode/parts/build-step.js +++ b/contracts/ts/seats/nodebuildnode/parts/build-step.js @@ -13,6 +13,8 @@ export const verbs = { export const events = { stepEnded: "step-ended", + toolchainReported: "toolchain-reported", + stepProgress: "step-progress", }; export {}; From 2c1f5a40332720fc564612add61160c5c10e243b Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 12:21:20 +0100 Subject: [PATCH 24/33] VW1: the view reads the modules from the catalog half of the trunk, by meshcatalogclient.Catalog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The view's process follows the trunk's catalog half (first view of the seat mesh-catalog's verb revision, then each catalog-revision of branch main) and hands each changed entry to its model in the form of the value it always kept; its tools read the newest trunk revision through the verb. The manifest invokes seat:mesh-catalog.revision and consumes mesh-catalog.catalog-revision instead of the indexer's modules read tool and views family. Builds: design 47 G §196, test 61 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../mesh-delivery-view-tools/catalogread.go | 83 +++++++++++++++++ .../cmd/mesh-delivery-view-tools/main.go | 2 +- .../mesh-delivery-view-tools/owners_test.go | 56 ++++++++--- .../mesh-delivery-view/integration_test.go | 92 +++++++++++++++---- .../cmd/mesh-delivery-view/main.go | 52 +++++++++-- .../internal/view/catalog.go | 58 ++++++++++++ .../mesh-delivery-view/internal/view/names.go | 1 - modules/mesh-delivery-view/manifest_test.go | 19 +++- modules/mesh-delivery-view/module.json | 4 +- 9 files changed, 321 insertions(+), 46 deletions(-) create mode 100644 modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/catalogread.go create mode 100644 modules/mesh-delivery-view/internal/view/catalog.go diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/catalogread.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/catalogread.go new file mode 100644 index 00000000..97803058 --- /dev/null +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/catalogread.go @@ -0,0 +1,83 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "sort" + "sync" + "time" + + "mesh-delivery-view/internal/view" +) + +// catalogWithin is how long a call waits for the seat mesh-catalog's first view before it says there is none. +var catalogWithin = 30 * time.Second + +// catalogRead is the modules the view's tools answer, read from the catalog half through the seat mesh-catalog's verb +// `revision` (design 47, Two graphs): the trunk's newest revision, asked once per call and in pages, each entry in the +// form of the value the tools always answered. With no view the read says so; it never answers no modules. +func catalogRead(ask func(key string, body any) (json.RawMessage, error)) view.StateRead { + var ( + once sync.Once + values map[string]json.RawMessage + rev uint64 + failed error + ) + load := func() { + ctx, cancel := context.WithTimeout(context.Background(), catalogWithin) + defer cancel() + c, err := meshcatalogclient.NewCatalog(ask, "main") + if err != nil { + failed = err + return + } + if err := c.Start(ctx); err != nil { + failed = fmt.Errorf("the catalog half has no first view yet (%s): %w", meshcatalogclient.RevisionVerb, err) + return + } + half, ok := c.View() + if !ok { + failed = fmt.Errorf("the catalog half has no first view yet (%s)", meshcatalogclient.RevisionVerb) + return + } + rev, values = uint64(half.Revision.Revision), map[string]json.RawMessage{} + for module, e := range half.Entries { + raw, _ := json.Marshal(e) + v, err := view.ModuleValue(raw) + if err != nil { + failed = fmt.Errorf("the catalog entry of %s: %w", module, err) + return + } + values[module] = v + } + } + read := func() (map[string]json.RawMessage, uint64, error) { + once.Do(load) + return values, rev, failed + } + return view.StateRead{ + Keys: func() ([]string, error) { + v, _, err := read() + keys := make([]string, 0, len(v)) + for k := range v { + keys = append(keys, k) + } + sort.Strings(keys) + return keys, err + }, + Get: func(key string) (json.RawMessage, error) { + v, _, err := read() + return v[key], err + }, + GetEntry: func(key string) (json.RawMessage, uint64, error) { + v, r, err := read() + return v[key], r, err + }, + All: func() (map[string]json.RawMessage, error) { + v, _, err := read() + return v, err + }, + } +} diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go index 1a8f828a..9baa4b52 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go @@ -396,7 +396,7 @@ func ownersRead(ask func(key string, body any) (json.RawMessage, error)) view.Re // Each owner's read tool as the view's invokes names it. indexer := func(tool string) view.StateRead { return view.OwnerRead(ask, view.IndexerModule+"."+tool) } assigner := func(tool string) view.StateRead { return view.OwnerRead(ask, view.AssignerModule+"."+tool) } - return view.Reader{State: indexer("indexer_read_modules"), CandidatesState: indexer("indexer_read_candidates"), + return view.Reader{State: catalogRead(ask), CandidatesState: indexer("indexer_read_candidates"), WantsState: assigner("assigner_read_wants"), DifferencesState: assigner("assigner_read_differences"), HeardState: indexer("indexer_read_heard"), PredefinedState: indexer("indexer_read_predefined"), diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/owners_test.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/owners_test.go index 80ad8589..8864369f 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/owners_test.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/owners_test.go @@ -3,12 +3,14 @@ package main import ( "encoding/json" "fmt" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" "mesh-delivery-view/internal/view" "sort" "strings" "testing" "time" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/sdk/graph" "git.novox.be/novox/mesh-sdk/go/views" ) @@ -51,7 +53,6 @@ func ownersAnswerTheToolsAsBefore(t *testing.T) { t.Fatal(err) } states := map[string]source{ - view.IndexerModule + ".modules": {"a": withFP2, "b": withNone}, view.IndexerModule + ".candidates": {"feat/x": cand, "fix": `{"v":1,"group":"fix","base":3,"heads":"h","pulls":[{"modules":["netcheck"]}]}`}, view.AssignerModule + ".wants": {"want.ace.zsh": `{"fp":"` + fp2A + `"}`, "want.novox.zsh": `{"fp":"` + fp2B + `"}`, "held." + keyB: `{"node":"ace","words":"failed"}`}, @@ -67,14 +68,12 @@ func ownersAnswerTheToolsAsBefore(t *testing.T) { const heard1 = `{"v":1,"subject":"s","sequence":41,"repository":"novox/mesh-catalog","commit":"9cdac09a"}` states[view.IndexerModule+".heard"] = source{"forge": heard1} histories := map[string]map[string][]views.Revision{ - view.IndexerModule + ".modules": {"a": {{Revision: 7, At: at, Op: "put", Value: json.RawMessage(withNone)}, - {Revision: 9, At: at.Add(time.Minute), Op: "put", Value: json.RawMessage(withFP2)}}}, view.IndexerModule + ".heard": {"forge": {{Revision: 3, At: at, Op: "put", Value: json.RawMessage(heard1)}}}, view.AssignerModule + ".wants": {"want.ace.zsh": {{Revision: 12, At: at, Op: "put", Value: json.RawMessage(`{"fp":"` + fp2A + `"}`)}}}, } // The direct reading, as the tools read the states before (the reads the manifest no longer has). st := func(name string) view.StateRead { return fake(states[name]).reader().State } - direct := view.Reader{State: st(view.IndexerModule + ".modules"), CandidatesState: st(view.IndexerModule + ".candidates"), + direct := view.Reader{CandidatesState: st(view.IndexerModule + ".candidates"), WantsState: st(view.AssignerModule + ".wants"), DifferencesState: st(view.AssignerModule + ".differences"), PredefinedState: st(view.IndexerModule + "." + view.IndexerPredefinedState), ConfigCandidatesState: st(view.IndexerModule + "." + view.IndexerConfigCandidatesState), @@ -89,6 +88,10 @@ func ownersAnswerTheToolsAsBefore(t *testing.T) { } asked := map[string]int{} ask := func(key string, body any) (json.RawMessage, error) { + if key == meshcatalogclient.RevisionVerb { + asked[key]++ + return json.Marshal(catalogAnswer) + } name, ok := tools[key] if !ok { return nil, fmt.Errorf("no tool %s", key) @@ -118,10 +121,6 @@ func ownersAnswerTheToolsAsBefore(t *testing.T) { tool string args map[string]any }{ - {view.ModuleReadTool, map[string]any{"module": "a"}}, - {view.ModuleReadTool, map[string]any{"module": "gone"}}, - {view.ModulesReadTool, map[string]any{}}, - {view.ModulesReadTool, map[string]any{"none2": true}}, {view.CandidateReadTool, map[string]any{"group": "feat/x"}}, {view.CandidatesReadTool, map[string]any{}}, {view.CandidatesReadTool, map[string]any{"module": "zsh"}}, @@ -143,12 +142,24 @@ func ownersAnswerTheToolsAsBefore(t *testing.T) { t.Errorf("%s %v through the owner's read tool = %s, %v; reading the state it was %s, %v", row.tool, row.args, gj, gerr, wj, werr) } } - // A key's history is asked of the owner's read tool and answered from its kept values (A2j finding 1, J4). - out, err := run(t, owned, view.ModuleHistoryTool, map[string]any{"module": "a"}) - if h, _ := out.(view.ModuleHistory); err != nil || len(h.Values) != 2 || h.Values[0].Revision != 9 || h.Values[1].FP2 != "" || !h.Values[1].At.Equal(at) { - t.Errorf("indexer_module_history a through the indexer's read tool = %+v, %v", out, err) + // The modules are the catalog half's, asked of the seat mesh-catalog's verb revision (W16.1): a module is its entry in + // the form of the value the tool always answered, the modules without a fingerprint say why, and the module's history + // is not a thing the catalog half keeps. + a, err := run(t, owned, view.ModuleReadTool, map[string]any{"module": "a"}) + if raw, _ := json.Marshal(a); err != nil || !strings.Contains(string(raw), `"fp2":"`+fp2A+`"`) || !strings.Contains(string(raw), `"commit":"c1"`) || !strings.Contains(string(raw), `"src":7`) { + t.Errorf("indexer_module a = %s, %v", raw, err) } - out, err = run(t, owned, view.HeardTool, map[string]any{}) + if _, err := run(t, owned, view.ModuleReadTool, map[string]any{"module": "gone"}); err == nil { + t.Errorf("indexer_module gone answered a module the catalog half does not hold") + } + all, err := run(t, owned, view.ModulesReadTool, map[string]any{"none2": true}) + if raw, _ := json.Marshal(all); err != nil || !strings.Contains(string(raw), `"b"`) || strings.Contains(string(raw), `"a"`) || !strings.Contains(string(raw), "builder's fingerprint not yet known") { + t.Errorf("indexer_modules none2 = %s, %v", raw, err) + } + if _, err := run(t, owned, view.ModuleHistoryTool, map[string]any{"module": "a"}); err == nil { + t.Errorf("indexer_module_history answered a history the catalog half does not keep") + } + out, err := run(t, owned, view.HeardTool, map[string]any{}) if rows, _ := out.(view.HeardAnswer); err != nil || len(rows.Heard) != 1 || rows.Heard[0].Revision != 3 || string(rows.Heard[0].Heard) != heard1 { t.Errorf("indexer_heard through the indexer's read tool = %+v, %v", out, err) } @@ -156,10 +167,29 @@ func ownersAnswerTheToolsAsBefore(t *testing.T) { if revs, err := owned.WantsState.History("want.ace.zsh"); err != nil || len(revs) != 1 || !revs[0].At.Equal(at) || revs[0].Revision != 12 { t.Errorf("the want's history through the assigner's read tool = %+v, %v", revs, err) } + // With no view of the catalog half the modules are not read: the tools say so, never that no module is held. + catalogWithin = 20 * time.Millisecond + none := ownersRead(func(key string, body any) (json.RawMessage, error) { return nil, fmt.Errorf("the holder is away") }) + if out, err := run(t, none, view.ModulesReadTool, map[string]any{}); err == nil || !strings.Contains(err.Error(), "no first view") { + t.Errorf("indexer_modules with no view = %v, %v, want an error naming it", out, err) + } // Each owner state the tools read was asked of its owner's read tool. for tool, name := range tools { if name != view.IndexerModule+"."+graph.State && asked[tool] == 0 { t.Errorf("%s was never asked", tool) } } + if asked[meshcatalogclient.RevisionVerb] == 0 { + t.Errorf("%s was never asked", meshcatalogclient.RevisionVerb) + } +} + +// catalogAnswer is the seat mesh-catalog's newest trunk revision: a with a fingerprint and the source revision it moved at, +// b with none and the reason. +var catalogAnswer = meshcatalog.RevisionAnswer{ + Revision: meshcatalog.CatalogRevision{Revision: 4, Kind: meshcatalogclient.KindTrunk, Source: 7}, + Entries: []meshcatalog.CatalogEntry{ + {Module: "a", Repository: "novox/r", Commit: "c1", Path: ".", Manifest: map[string]any{"module": "a"}, Fp: fp2A, Inputs: &meshcatalog.EntryInputs{Src: 7}}, + {Module: "b", Repository: "novox/r", Commit: "c2", Path: "modules/b", Manifest: map[string]any{"module": "b"}, None: "builder's fingerprint not yet known"}, + }, } diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view/integration_test.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view/integration_test.go index 77ad3d76..14248181 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view/integration_test.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view/integration_test.go @@ -12,6 +12,7 @@ import ( "path/filepath" "slices" "strings" + "sync" "testing" "time" @@ -47,6 +48,67 @@ func serveOwners(t *testing.T, ctx context.Context, js jetstream.JetStream, nc * } } +// catalogHolder is the seat mesh-catalog's holder as the view's process needs it (W16.1): its verb revision answering the +// trunk's newest revision whole, and its event catalog-revision said on the shared events stream at each change, which the +// view's durable consumer hears. +type catalogHolder struct { + t testing.TB + js jetstream.JetStream + mu sync.Mutex + // rev is the revision's number; entries are its entries by module. + rev int64 + entries map[string]meshcatalog.CatalogEntry + ctx context.Context +} + +func serveCatalog(t testing.TB, ctx context.Context, js jetstream.JetStream, nc *nats.Conn) *catalogHolder { + t.Helper() + h := &catalogHolder{t: t, js: js, ctx: ctx, rev: 1, entries: map[string]meshcatalog.CatalogEntry{}} + events, err := js.Stream(ctx, process.EventsStream) + if err != nil { + t.Fatal(err) + } + info := events.CachedInfo().Config + if !slices.Contains(info.Subjects, "mesh.seat.*.event.>") { + info.Subjects = append(info.Subjects, "mesh.seat.*.event.>") + if _, err := js.UpdateStream(ctx, info); err != nil { + t.Fatal(err) + } + } + answer := func(m *nats.Msg) { + h.mu.Lock() + a := meshcatalog.RevisionAnswer{Revision: meshcatalog.CatalogRevision{Revision: h.rev, Kind: meshcatalogclient.KindTrunk}} + for _, e := range h.entries { + a.Entries = append(a.Entries, e) + } + h.mu.Unlock() + raw, _ := json.Marshal(map[string]any{"result": a, "node": "n0"}) + _ = m.Respond(raw) + } + for _, subject := range []string{"mesh.seat.mesh-catalog.call.revision.>", "mesh.seat.mesh-catalog.call.revision"} { + sub, err := nc.Subscribe(subject, answer) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = sub.Unsubscribe() }) + } + return h +} + +// put is the holder's next trunk revision with the entry of one module changed, said as catalog-revision. +func (h *catalogHolder) put(e meshcatalog.CatalogEntry) { + h.t.Helper() + h.mu.Lock() + h.rev++ + h.entries[e.Module] = e + rev := h.rev + h.mu.Unlock() + body, _ := json.Marshal(meshcatalog.CatalogRevisionEvent{Kind: meshcatalogclient.KindTrunk, Branch: "main", Revision: rev, Changed: []string{e.Module}}) + if _, err := h.js.Publish(h.ctx, "mesh.seat.mesh-catalog.event."+meshcatalog.EventCatalogRevision, body); err != nil { + h.t.Fatal(err) + } +} + // TestProcessWholeSeesTheBuckets starts the program the way the node does (its entry, run, with its environment and a // broker file) against an embedded bus holding the three buckets, and asks its socket. func TestProcessWholeSeesTheBuckets(t *testing.T) { // 1134/AC8 @@ -126,6 +188,7 @@ func TestProcessWholeSeesTheBuckets(t *testing.T) { // 1134/AC8 runCtx, stop := context.WithCancel(ctx) done := make(chan error, 1) serveOwners(t, ctx, js, nc) + serveCatalog(t, ctx, js, nc) go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() t.Cleanup(func() { stop(); <-done }) @@ -226,6 +289,7 @@ func TestProcessWholeShowsWhatTheWatchmanSays(t *testing.T) { // 1168/AC5 runCtx, stop := context.WithCancel(ctx) done := make(chan error, 1) serveOwners(t, ctx, js, nc) + serveCatalog(t, ctx, js, nc) go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() t.Cleanup(func() { stop(); <-done }) @@ -293,6 +357,7 @@ func TestProcessWholeShowsTheIndexersBucketsOnceTheyExist(t *testing.T) { // 124 runCtx, stop := context.WithCancel(ctx) done := make(chan error, 1) serveOwners(t, ctx, js, nc) + catalog := serveCatalog(t, ctx, js, nc) go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() t.Cleanup(func() { stop(); <-done }) @@ -306,26 +371,16 @@ func TestProcessWholeShowsTheIndexersBucketsOnceTheyExist(t *testing.T) { // 124 if err != nil { t.Fatal(err) } - mods, err := js.KeyValue(ctx, view.ModulesBucket) - if err != nil { - t.Fatal(err) - } const head = "0123456789abcdef0123456789abcdef01234567" hash, _ := meshdeliveryclient.HeadsHash([]meshdeliveryclient.Head{{Repository: "novox/mesh-delivery", Number: 12, Head: head}}) cand, err := json.Marshal(meshcatalog.CandidatesByGroupValue{Group: "feat/x", Heads: hash, Pulls: []meshcatalog.CandidatesByGroupValuePullsItem{{Repository: "novox/mesh-delivery", Number: 12, Head: head, Modules: []string{"a"}}}}) if err != nil { t.Fatal(err) } - mod, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{None: "no tree", Repository: "novox/mesh-delivery", Commit: head}) - if err != nil { - t.Fatal(err) - } if _, err := cands.Put(ctx, "feat/x", cand); err != nil { t.Fatal(err) } - if _, err := mods.Put(ctx, "a", mod); err != nil { - t.Fatal(err) - } + catalog.put(meshcatalog.CatalogEntry{Module: "a", None: "no tree", Repository: "novox/mesh-delivery", Commit: head, Manifest: map[string]any{"module": "a"}}) var last view.Snapshot for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) { if s, err := view.Ask(socket); err == nil { @@ -380,6 +435,7 @@ func TestProcessWholeShowsTheOpenAsksAndDropsTheAnsweredOnes(t *testing.T) { // runCtx, stop := context.WithCancel(ctx) done := make(chan error, 1) serveOwners(t, ctx, js, nc) + serveCatalog(t, ctx, js, nc) go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() t.Cleanup(func() { stop(); <-done }) @@ -471,6 +527,7 @@ func TestProcessWholeListsEveryDifference(t *testing.T) { // 1297/B8 AC runCtx, stop := context.WithCancel(ctx) done := make(chan error, 1) serveOwners(t, ctx, js, nc) + serveCatalog(t, ctx, js, nc) go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() t.Cleanup(func() { stop(); <-done }) @@ -569,6 +626,7 @@ func TestProcessWholeWritesTheAnswerToAnAskNeedingNoProof(t *testing.T) { // 129 runCtx, stop := context.WithCancel(ctx) done := make(chan error, 1) serveOwners(t, ctx, js, nc) + serveCatalog(t, ctx, js, nc) go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() t.Cleanup(func() { stop(); <-done }) @@ -721,6 +779,7 @@ func TestProcessWholeShowsTheBacklogOfBuildAsks(t *testing.T) { // 1887/A8a AC runCtx, stop := context.WithCancel(ctx) done := make(chan error, 1) serveOwners(t, ctx, js, nc) + serveCatalog(t, ctx, js, nc) go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() t.Cleanup(func() { stop(); <-done }) @@ -768,7 +827,7 @@ func TestProcessWholeSaysTheSourceRevisionIsDelivered(t *testing.T) { t.Fatal(err) } kvs := map[string]jetstream.KeyValue{} - for _, name := range []string{view.SourceBucket, view.ModulesBucket, view.AssignerWantsBucket, view.HaveBucket} { + for _, name := range []string{view.SourceBucket, view.AssignerWantsBucket, view.HaveBucket} { kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: 5}) if err != nil { t.Fatal(err) @@ -789,20 +848,16 @@ func TestProcessWholeSaysTheSourceRevisionIsDelivered(t *testing.T) { runCtx, stop := context.WithCancel(ctx) done := make(chan error, 1) serveOwners(t, ctx, js, nc) + catalog := serveCatalog(t, ctx, js, nc) go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() t.Cleanup(func() { stop(); <-done }) fp := "src2:" + strings.Repeat("2", 64) - mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: strings.Repeat("a", 64), Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath}) want, _ := meshwantsclient.EncodeWant(meshwantsclient.Want{Fp: fp, Composed: "sha256:" + strings.Repeat("c", 64)}) failed, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed}) good, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}) src, _ := meshdeliveryclient.EncodeSource(meshdelivery.SourceByBranchValue{Group: "feat/x", Heads: map[string]string{"novox/web": strings.Repeat("1", 40)}}) - for bucket, kv := range map[string][]byte{view.ModulesBucket: mod} { - if _, err := kvs[bucket].Put(ctx, "web", kv); err != nil { - t.Fatal(err) - } - } + catalog.put(meshcatalog.CatalogEntry{Module: "web", Fp: fp, Manifest: map[string]any{"module": "a"}, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath}) if _, err := kvs[view.AssignerWantsBucket].Put(ctx, "want.ace.web", want); err != nil { t.Fatal(err) } @@ -886,6 +941,7 @@ func TestProcessWholeSaysModulesInMotion(t *testing.T) { runCtx, stop := context.WithCancel(ctx) done := make(chan error, 1) serveOwners(t, ctx, js, nc) + serveCatalog(t, ctx, js, nc) go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() t.Cleanup(func() { stop(); <-done }) time.Sleep(2 * time.Second) // the view has read what stood: nothing diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go index 471875c1..ecd9dcd5 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go @@ -11,6 +11,7 @@ import ( "encoding/json" "errors" "fmt" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" "io" "log" "mesh-delivery-view/internal/view" @@ -126,15 +127,21 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } } open := buckets{js: js, reader: reader} - for _, bucket := range []string{view.HaveBucket, view.AssignmentsBucket, view.PresenceBucket, view.StuckBucket, view.CandidatesBucket, view.ModulesBucket, view.AsksBucket, view.AnswersBucket, builds.BuilderBuildsBucket, view.AssignerWantsBucket, view.SourceBucket, view.AssignerAsksBucket, viewstate.ViewAnswersBucket, view.TakenBucket} { + for _, bucket := range []string{view.HaveBucket, view.AssignmentsBucket, view.PresenceBucket, view.StuckBucket, view.CandidatesBucket, view.AsksBucket, view.AnswersBucket, builds.BuilderBuildsBucket, view.AssignerWantsBucket, view.SourceBucket, view.AssignerAsksBucket, viewstate.ViewAnswersBucket, view.TakenBucket} { go follow(ctx, open, bucket, false, model, logger) } - // The modules' and the source's history give each fingerprint's first commit and each head's source revision (ADR 0556). - // The modules' picture holds no history: from its first view on, each change it takes is the history (finding of DLV-M4). - for _, bucket := range []string{view.ModulesBucket, view.SourceBucket} { - go follow(ctx, open, bucket, true, model, logger) + // The source's history gives each head's source revision (ADR 0556); the modules' history is each revision of the + // catalog half the view takes, in its order (below). + go follow(ctx, open, view.SourceBucket, true, model, logger) + statusBucket := process.AwaitBucket(ctx, js, conn, firstViewsBucket, "view_first_views reads nothing and view_first_view_again is not heard", logger.Printf) + // The modules are the catalog half's, adopted whole by the seat's client: its first view, its status, its hand asks and + // its event catalog-revision (on the module's durable consumer) are the reader's. + catalog, _, err := meshcatalogclient.WireCatalogProcess(reader, calls.Call, "main", catalogChanged(model, logger)) + if err != nil { + return err } - reader.Start(ctx, js, cred, process.AwaitBucket(ctx, js, conn, firstViewsBucket, "view_first_views reads nothing and view_first_view_again is not heard", logger.Printf)) + reader.Start(ctx, js, cred, statusBucket) + catalogHeld(ctx, catalog, model) select { case <-ctx.Done(): case err := <-webUp: @@ -189,7 +196,6 @@ var ownersFollowed = []struct{ owner, state, tool string }{ {view.AssignerModule, "wants", view.AssignerModule + ".assigner_read_wants"}, {view.AssignerModule, "asks", view.AssignerModule + ".assigner_read_asks"}, {view.IndexerModule, "candidates", view.IndexerModule + ".indexer_read_candidates"}, - {view.IndexerModule, "modules", view.IndexerModule + ".indexer_read_modules"}, {view.IndexerModule, "taken", view.IndexerModule + ".indexer_read_taken"}, } @@ -277,3 +283,35 @@ func emitEvent(ctx context.Context, js jetstream.JetStream, cred process.Account _, err = js.PublishMsg(pub, msg, jetstream.WithMsgID(id)) return err } + +// catalogChanged hands one module of a revision the view took to the model, in the form of its value; the modules' +// history is these changes in the order of the revisions. +func catalogChanged(model *view.Model, logger *log.Logger) func(string, json.RawMessage, bool, time.Time) { + return func(module string, entry json.RawMessage, gone bool, at time.Time) { + e := view.Entry{Bucket: view.ModulesBucket, Key: module, At: at, Deleted: gone} + if !gone { + v, err := view.ModuleValue(entry) + if err != nil { + logger.Printf("the catalog entry of %s is not taken: %v", module, err) + return + } + e.Value = v + } + model.Apply(e) + e.History = true + model.Apply(e) + } +} + +// catalogHeld makes the model's pictures of the modules initial once the catalog half's first view is held: the process +// starts at once and waits for the seat's holder, saying so. +func catalogHeld(ctx context.Context, cat *meshcatalogclient.Catalog, model *view.Model) { + go func() { + select { + case <-cat.Follower.Ready(): + model.Initial(view.ModulesBucket) + model.Initial(view.ReachHistory(view.ModulesBucket)) + case <-ctx.Done(): + } + }() +} diff --git a/modules/mesh-delivery-view/internal/view/catalog.go b/modules/mesh-delivery-view/internal/view/catalog.go new file mode 100644 index 00000000..385de245 --- /dev/null +++ b/modules/mesh-delivery-view/internal/view/catalog.go @@ -0,0 +1,58 @@ +package view + +// The view reads the catalog half of the delivery through the seat mesh-catalog (design 47, Two graphs; ADR 0656 §6): +// a first view of its verb `revision`, then each trunk revision its event `catalog-revision` says, never the indexer's +// state `modules`. The model keeps the picture it always kept (one value per module, its history for the reach), so each +// entry of a revision is handed to it in the form of that value. + +import ( + "encoding/json" + "fmt" + + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" +) + +// CatalogSeat is the seat whose verb `revision` and event `catalog-revision` the view reads the modules from, as its +// manifest's `invokes` and `consumes` name them. +const CatalogSeat = meshcatalog.Seat + +// ModulesBucket names the model's picture of the catalog half's modules (and its history): the model's label, no +// bucket of anybody's. +const ModulesBucket = CatalogSeat + "_modules" + +// ModuleValue is the value of one module of the model, as the model decodes it, made from the catalog entry (its JSON): +// the fingerprint is the entry's `fp` (the only form there is), the source revisions are the entry's inputs'. A manifest +// held for unknown fields has no verdict here: the model reads the fingerprint and the commit. +func ModuleValue(entry json.RawMessage) (json.RawMessage, error) { + var e meshcatalog.CatalogEntry + if err := json.Unmarshal(entry, &e); err != nil || e.Module == "" { + return nil, fmt.Errorf("a catalog entry does not read: %v", err) + } + v := meshcatalog.ModulesByModuleValue{Commit: e.Commit, Fp2: e.Fp, None2: e.None, Path: e.Path, Repository: e.Repository, Packages: e.Packages} + if len(e.Manifest) > 0 { + // The entry holds its manifest whole; the model keeps the digest of its bytes, as the value it always kept does. + raw, _ := json.Marshal(e.Manifest) + v.Manifest = meshcatalogclient.ManifestDigest(raw) + } + if e.Verdict == meshcatalogclient.VerdictAccepted || e.Verdict == meshcatalogclient.VerdictRefused { + v.Verdict = e.Verdict + } + if in := e.Inputs; in != nil { + v.Src, v.FpSrc, v.Toolchains, v.Paths, v.Warnings = in.Src, in.FpSrc, in.Toolchains, in.Paths, in.Warnings + if f := in.From; f != nil && v.Manifest != "" { + v.Inputs2 = &meshcatalog.ModulesByModuleValueInputs2{Bases: f.Bases, Builder: f.Builder, Contexts: f.Contexts, + ImportForm: f.ImportForm, Imports: f.Imports, Locks: f.Locks, Packages: f.Packages, Paths: f.Paths, Tree: f.Tree} + } + if v.Verdict == meshcatalogclient.VerdictRefused { + v.Problems = in.Problems + } + } + if v.Verdict == meshcatalogclient.VerdictRefused && len(v.Problems) == 0 { + v.Verdict = "" + } + if v.Verdict != meshcatalogclient.VerdictRefused { + v.Warnings = nil + } + return meshcatalogclient.EncodeModule(v) +} diff --git a/modules/mesh-delivery-view/internal/view/names.go b/modules/mesh-delivery-view/internal/view/names.go index 4a2d62f1..24e69c85 100644 --- a/modules/mesh-delivery-view/internal/view/names.go +++ b/modules/mesh-delivery-view/internal/view/names.go @@ -24,7 +24,6 @@ const ( EngineModule = "node-engine" // The indexer's states the view reads. - ModulesBucket = IndexerModule + "_modules" CandidatesBucket = IndexerModule + "_candidates" TakenBucket = IndexerModule + "_taken" PackagesState = "packages" diff --git a/modules/mesh-delivery-view/manifest_test.go b/modules/mesh-delivery-view/manifest_test.go index b35cf6f8..693fe5d5 100644 --- a/modules/mesh-delivery-view/manifest_test.go +++ b/modules/mesh-delivery-view/manifest_test.go @@ -3,6 +3,7 @@ package manifest import ( "bytes" "encoding/json" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" "io/fs" "mesh-delivery-view/internal/view" "os" @@ -119,8 +120,9 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7 // follows then heard on their views families (ADR 0615 §1, §5; DLV-M4). if want := []string{"mesh-delivery-assigner.assigner_read_asks", "mesh-delivery-assigner.assigner_read_differences", "mesh-delivery-assigner.assigner_read_assignments", "mesh-delivery-assigner.assigner_read_wants", "mesh-delivery-indexer.indexer_read_candidates", "mesh-delivery-indexer.indexer_read_config_candidates", - "mesh-delivery-indexer.indexer_read_graph", "mesh-delivery-indexer.indexer_read_heard", "mesh-delivery-indexer.indexer_pending", "mesh-delivery-indexer.indexer_read_modules", + "mesh-delivery-indexer.indexer_read_graph", "mesh-delivery-indexer.indexer_read_heard", "mesh-delivery-indexer.indexer_pending", "mesh-delivery-indexer.indexer_read_packages", "mesh-delivery-indexer.indexer_read_predefined", "mesh-delivery-indexer.indexer_read_taken", + "seat:mesh-catalog.revision", // ADR 0663: the modules are the catalog half's, read through the seat's verb revision, never the indexer's "seat:mesh-hand-acts.hand-act", // the join: its first view asked again by hand is a hand act (views.StatusTools) "seat:node-engine.manifest-reads"}; !reflect.DeepEqual(m.Invokes, want) { // ADR 0618 §3: node_manifest_reads asks the node-engine seat's verb of every node; 1585/T24: view_tiers asks indexer_pending, a verb edge (ADR 0614 §2) t.Errorf("invokes = %v, want %v", m.Invokes, want) @@ -133,7 +135,7 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7 t.Errorf("version = %q, want 0.1", m.Version) } if want := []string{"mesh-conditions.findings-asked", "mesh-delivery-assigner.views.asks", "mesh-delivery-assigner.views.assignments", "mesh-delivery-assigner.views.wants", - "mesh-delivery-indexer.views.candidates", "mesh-delivery-indexer.views.modules", "mesh-delivery-indexer.views.taken"}; !reflect.DeepEqual(m.Consumes, want) { + "mesh-catalog.catalog-revision", "mesh-delivery-indexer.views.candidates", "mesh-delivery-indexer.views.taken"}; !reflect.DeepEqual(m.Consumes, want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } // The one event the view says: a source revision is delivered (1906/D9, ADR 0539 §17), derived by it from the wants and the haves. @@ -180,7 +182,7 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7 if !slices.Contains(m.Reads, view.SignersRead) { t.Errorf("the view reads %s, its manifest's reads are %v", view.SignersRead, m.Reads) } - followed := map[string]string{view.ModulesBucket: view.IndexerModule + ".views.modules", view.CandidatesBucket: view.IndexerModule + ".views.candidates", + followed := map[string]string{view.CandidatesBucket: view.IndexerModule + ".views.candidates", view.TakenBucket: view.IndexerModule + ".views.taken", view.AssignmentsBucket: view.AssignerModule + ".views.assignments", view.AssignerWantsBucket: view.AssignerModule + ".views.wants", view.AssignerAsksBucket: view.AssignerModule + ".views.asks"} for bucket, family := range followed { @@ -191,11 +193,20 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7 t.Errorf("the view follows %s, its manifest consumes %v", family, m.Consumes) } } - for _, tool := range []string{view.IndexerModule + ".indexer_read_modules", view.IndexerModule + ".indexer_read_candidates", view.IndexerModule + ".indexer_pending", view.AssignerModule + ".assigner_read_wants"} { + for _, tool := range []string{meshcatalogclient.RevisionVerb, view.IndexerModule + ".indexer_read_candidates", view.IndexerModule + ".indexer_pending", view.AssignerModule + ".assigner_read_wants"} { if !slices.Contains(m.Invokes, tool) { t.Errorf("the view asks %s, its manifest invokes %v", tool, m.Invokes) } } + // The modules are the catalog half's: the seat's verb is invoked, its event consumed, and the indexer's state modules is neither. + if !slices.Contains(m.Consumes, view.CatalogSeat+"."+meshcatalog.EventCatalogRevision) { + t.Errorf("the view hears the seat's %s, its manifest consumes %v", meshcatalog.EventCatalogRevision, m.Consumes) + } + for _, name := range append(slices.Clone(m.Invokes), m.Consumes...) { + if strings.Contains(name, "indexer_read_modules") || strings.Contains(name, "views.modules") { + t.Errorf("the manifest names %s: the modules are the catalog half's (ADR 0663)", name) + } + } if viewstate.ViewModule != m.Module || viewstate.ViewAnswersBucket != m.Module+"_answers" || viewstate.ReachBucket != m.Module+"_reach" { t.Errorf("the view names itself %s (%s, %s), its manifest %s", viewstate.ViewModule, viewstate.ViewAnswersBucket, viewstate.ReachBucket, m.Module) } diff --git a/modules/mesh-delivery-view/module.json b/modules/mesh-delivery-view/module.json index a8544e42..9e727992 100644 --- a/modules/mesh-delivery-view/module.json +++ b/modules/mesh-delivery-view/module.json @@ -46,8 +46,8 @@ "mesh-delivery-assigner.views.asks", "mesh-delivery-assigner.views.assignments", "mesh-delivery-assigner.views.wants", + "mesh-catalog.catalog-revision", "mesh-delivery-indexer.views.candidates", - "mesh-delivery-indexer.views.modules", "mesh-delivery-indexer.views.taken" ], "streams": [ @@ -68,10 +68,10 @@ "mesh-delivery-indexer.indexer_read_graph", "mesh-delivery-indexer.indexer_read_heard", "mesh-delivery-indexer.indexer_pending", - "mesh-delivery-indexer.indexer_read_modules", "mesh-delivery-indexer.indexer_read_packages", "mesh-delivery-indexer.indexer_read_predefined", "mesh-delivery-indexer.indexer_read_taken", + "seat:mesh-catalog.revision", "seat:mesh-hand-acts.hand-act", "seat:node-engine.manifest-reads" ], From e1499f0617a321c18ef93907a93af88309daff49 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 12:21:56 +0100 Subject: [PATCH 25/33] CT4: a wait names its clock and its kind: clock-from, step of the acting steps, a put's fingerprint, a delivery wait's machines; regenerated MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit mesh-wants' Wait gains clock-from, the step enum of section 203 with put, delivery, kept-build and backup, fingerprint and machines, and requires step, module and since only (a put's wait has no part); the runner's wait body gains clock-from. The runner's `step` already names pipeline steps: see the report. Builds: design 47 G §498, §266, §269 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../go/seats/meshpipelinerunner/contract.go | 2 ++ contracts/go/seats/meshwants/contract.go | 31 +++++++++++++--- contracts/go/seats/meshwants/contract_test.go | 4 +++ .../seats/mesh-pipeline-runner/seat.json | 5 +++ .../seats/mesh-wants/fixtures/events.json | 32 +++++++++++++++++ contracts/seats/mesh-wants/seat.json | 35 +++++++++++++++++-- .../ts/seats/meshpipelinerunner/index.d.ts | 2 ++ contracts/ts/seats/meshwants/index.d.ts | 13 +++++-- 8 files changed, 114 insertions(+), 10 deletions(-) diff --git a/contracts/go/seats/meshpipelinerunner/contract.go b/contracts/go/seats/meshpipelinerunner/contract.go index 39817255..7c17dd91 100644 --- a/contracts/go/seats/meshpipelinerunner/contract.go +++ b/contracts/go/seats/meshpipelinerunner/contract.go @@ -675,6 +675,8 @@ type Wait struct { Artifact string `json:"artifact,omitempty"` // the watched branch, or the branch of a run on request Branch string `json:"branch"` + // the time the bound counts from (design 47 G section 266, section 269, section 498) + ClockFrom *time.Time `json:"clock-from,omitempty"` // the commit the module's entry was last changed at Commit string `json:"commit,omitempty"` // the module's build-input fingerprint at the commit diff --git a/contracts/go/seats/meshwants/contract.go b/contracts/go/seats/meshwants/contract.go index 9b9f4754..ad44c028 100644 --- a/contracts/go/seats/meshwants/contract.go +++ b/contracts/go/seats/meshwants/contract.go @@ -659,20 +659,43 @@ type Wait struct { Bound string `json:"bound,omitempty"` // on a release, who released it and why By string `json:"by,omitempty"` - Fp string `json:"fp,omitempty"` + // the time the bound counts from (design 47 G section 266, section 269, section 498) + ClockFrom *time.Time `json:"clock-from,omitempty"` + // a put's wait: the fingerprint of the build put + Fingerprint string `json:"fingerprint,omitempty"` + Fp string `json:"fp,omitempty"` // on wait-ended: ran or released How string `json:"how,omitempty"` + // a dependent's delivery wait: the machines the dependency's version has yet to reach + Machines []string `json:"machines,omitempty"` // the entry that waits Module string `json:"module"` Node string `json:"node,omitempty"` // the acting part waited for - Part string `json:"part"` + Part string `json:"part,omitempty"` Since time.Time `json:"since"` - Step string `json:"step"` + // which wait: an acting step of section 203 or put, delivery, kept-build, backup (design 47 G section 498) + Step string `json:"step"` // its new version's fingerprint - Version string `json:"version"` + Version string `json:"version,omitempty"` } +// The values of Wait's step. +const ( + WaitStepTakeIn = "take-in" + WaitStepValidate = "validate" + WaitStepCheck = "check" + WaitStepBuild = "build" + WaitStepCompose = "compose" + WaitStepGrant = "grant" + WaitStepPlace = "place" + WaitStepPipeline = "pipeline" + WaitStepPut = "put" + WaitStepDelivery = "delivery" + WaitStepKeptBuild = "kept-build" + WaitStepBackup = "backup" +) + // UpgradeInput is what upgrade takes. type UpgradeInput struct { // "true": each node's own composition, by its digest, and the assigner's findings diff --git a/contracts/go/seats/meshwants/contract_test.go b/contracts/go/seats/meshwants/contract_test.go index 6a6825e6..b0624c3e 100644 --- a/contracts/go/seats/meshwants/contract_test.go +++ b/contracts/go/seats/meshwants/contract_test.go @@ -45,6 +45,10 @@ func TestFixtures(t *testing.T) { decode: func(b []byte) (any, error) { var v VersionDeliveredEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-wants\\.event\\.version-delivered$")}, {name: "a composition step's progress", subject: "mesh.seat.mesh-wants.event.step-progress.ace", given: "{\"ask\":\"a1b2\",\"asker\":\"assigner\",\"at\":\"2026-11-11T10:01:00Z\",\"phase\":\"build\"}", wire: "{\"ask\":\"a1b2\",\"asker\":\"assigner\",\"at\":\"2026-11-11T10:01:00Z\",\"phase\":\"build\"}", decode: func(b []byte) (any, error) { var v StepProgressEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-wants\\.event\\.step-progress\\.[^.*> ]+$")}, + {name: "a put without hold waits, counted from its start", subject: "mesh.seat.mesh-wants.event.wait-held", given: "{\"bound\":\"30m\",\"clock-from\":\"2026-11-11T10:00:00Z\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"ace\",\"since\":\"2026-11-11T10:00:00Z\",\"step\":\"put\"}", wire: "{\"bound\":\"30m\",\"clock-from\":\"2026-11-11T10:00:00Z\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"ace\",\"since\":\"2026-11-11T10:00:00Z\",\"step\":\"put\"}", + decode: func(b []byte) (any, error) { var v WaitHeldEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-wants\\.event\\.wait-held$")}, + {name: "a dependent's delivery waits for a dependency's version", subject: "mesh.seat.mesh-wants.event.wait-held", given: "{\"bound\":\"60m\",\"clock-from\":\"2026-11-11T09:58:00Z\",\"machines\":[\"ace\",\"g14\"],\"module\":\"nats\",\"part\":\"mesh-sdk\",\"since\":\"2026-11-11T10:00:00Z\",\"step\":\"delivery\",\"version\":\"0.1.4\"}", wire: "{\"bound\":\"60m\",\"clock-from\":\"2026-11-11T09:58:00Z\",\"machines\":[\"ace\",\"g14\"],\"module\":\"nats\",\"part\":\"mesh-sdk\",\"since\":\"2026-11-11T10:00:00Z\",\"step\":\"delivery\",\"version\":\"0.1.4\"}", + decode: func(b []byte) (any, error) { var v WaitHeldEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-wants\\.event\\.wait-held$")}, {name: "a want", subject: "", given: "{\"fp\":\"ab12\",\"rule\":3,\"v\":1}", wire: "{\"fp\":\"ab12\",\"rule\":3,\"v\":1}", decode: func(b []byte) (any, error) { var v WantsByWantNodeModuleValue diff --git a/contracts/seats/mesh-pipeline-runner/seat.json b/contracts/seats/mesh-pipeline-runner/seat.json index 5d972333..29d7b274 100644 --- a/contracts/seats/mesh-pipeline-runner/seat.json +++ b/contracts/seats/mesh-pipeline-runner/seat.json @@ -1483,6 +1483,11 @@ "how": { "type": "string", "description": "for its end, how it ended: answered, ran, released, superseded" + }, + "clock-from": { + "type": "string", + "format": "date-time", + "description": "the time the bound counts from (design 47 G section 266, section 269, section 498)" } }, "description": "a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283)", diff --git a/contracts/seats/mesh-wants/fixtures/events.json b/contracts/seats/mesh-wants/fixtures/events.json index 7c00e058..550c6c38 100644 --- a/contracts/seats/mesh-wants/fixtures/events.json +++ b/contracts/seats/mesh-wants/fixtures/events.json @@ -177,5 +177,37 @@ "phase": "build", "at": "2026-11-11T10:01:00Z" } + }, + { + "name": "a put without hold waits, counted from its start", + "of": "event:wait-held", + "subject": "mesh.seat.mesh-wants.event.wait-held", + "given": { + "step": "put", + "node": "ace", + "module": "nats", + "fingerprint": "fp-0123", + "since": "2026-11-11T10:00:00Z", + "clock-from": "2026-11-11T10:00:00Z", + "bound": "30m" + } + }, + { + "name": "a dependent's delivery waits for a dependency's version", + "of": "event:wait-held", + "subject": "mesh.seat.mesh-wants.event.wait-held", + "given": { + "step": "delivery", + "part": "mesh-sdk", + "version": "0.1.4", + "machines": [ + "ace", + "g14" + ], + "module": "nats", + "since": "2026-11-11T10:00:00Z", + "clock-from": "2026-11-11T09:58:00Z", + "bound": "60m" + } } ] diff --git a/contracts/seats/mesh-wants/seat.json b/contracts/seats/mesh-wants/seat.json index aca8f3b9..6e41ad9f 100644 --- a/contracts/seats/mesh-wants/seat.json +++ b/contracts/seats/mesh-wants/seat.json @@ -1025,7 +1025,22 @@ "description": "a step's wait the assigner holds: placing, wanting or composing until an acting part's new version runs", "properties": { "step": { - "type": "string" + "type": "string", + "enum": [ + "take-in", + "validate", + "check", + "build", + "compose", + "grant", + "place", + "pipeline", + "put", + "delivery", + "kept-build", + "backup" + ], + "description": "which wait: an acting step of section 203 or put, delivery, kept-build, backup (design 47 G section 498)" }, "part": { "type": "string", @@ -1060,12 +1075,26 @@ "by": { "type": "string", "description": "on a release, who released it and why" + }, + "clock-from": { + "type": "string", + "format": "date-time", + "description": "the time the bound counts from (design 47 G section 266, section 269, section 498)" + }, + "fingerprint": { + "type": "string", + "description": "a put's wait: the fingerprint of the build put" + }, + "machines": { + "type": "array", + "items": { + "type": "string" + }, + "description": "a dependent's delivery wait: the machines the dependency's version has yet to reach" } }, "required": [ "step", - "part", - "version", "module", "since" ] diff --git a/contracts/ts/seats/meshpipelinerunner/index.d.ts b/contracts/ts/seats/meshpipelinerunner/index.d.ts index cf1d8263..abcd517c 100644 --- a/contracts/ts/seats/meshpipelinerunner/index.d.ts +++ b/contracts/ts/seats/meshpipelinerunner/index.d.ts @@ -364,6 +364,8 @@ export interface Wait { "artifact"?: string; /** the watched branch, or the branch of a run on request */ "branch": string; + /** the time the bound counts from (design 47 G section 266, section 269, section 498) */ + "clock-from"?: string; /** the commit the module's entry was last changed at */ "commit"?: string; /** the module's build-input fingerprint at the commit */ diff --git a/contracts/ts/seats/meshwants/index.d.ts b/contracts/ts/seats/meshwants/index.d.ts index a2760e6c..91a28584 100644 --- a/contracts/ts/seats/meshwants/index.d.ts +++ b/contracts/ts/seats/meshwants/index.d.ts @@ -311,18 +311,25 @@ export interface Wait { "bound"?: string; /** on a release, who released it and why */ "by"?: string; + /** the time the bound counts from (design 47 G section 266, section 269, section 498) */ + "clock-from"?: string; + /** a put's wait: the fingerprint of the build put */ + "fingerprint"?: string; "fp"?: string; /** on wait-ended: ran or released */ "how"?: string; + /** a dependent's delivery wait: the machines the dependency's version has yet to reach */ + "machines"?: string[]; /** the entry that waits */ "module": string; "node"?: string; /** the acting part waited for */ - "part": string; + "part"?: string; "since": string; - "step": string; + /** which wait: an acting step of section 203 or put, delivery, kept-build, backup (design 47 G section 498) */ + "step": "take-in" | "validate" | "check" | "build" | "compose" | "grant" | "place" | "pipeline" | "put" | "delivery" | "kept-build" | "backup"; /** its new version's fingerprint */ - "version": string; + "version"?: string; } /** is what upgrade takes. */ From 2b052791dac88f2250663033ce51d0319154e146 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 12:28:00 +0100 Subject: [PATCH 26/33] RN2: the runner reads its policy from the registrations and each watched branch's catalog half, and says when it waits for a view MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The runner follows the registrations (a first view, then registration-changed) and, for each watched branch they name, that branch's catalog half (a first view, then catalog-revision of that branch), each in its state first-views, read by pipeline_first_views and asked again by hand with pipeline_first_view_again; past five minutes without a view it writes view-waits. A step's status is read per module and artifact from the registration, and the build seat of an artifact from the acting rule build naming its language. Builds: design 47 G §21, §23, §323, §77, §195, §200 Open: design questions Q-RN7 (where a build seat's holder is read, §323) and Q-RN8 (a status left empty, §23). Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-pipeline-tools/main.go | 19 +- .../mesh-delivery-pipeline-tools/main_test.go | 11 +- .../cmd/mesh-delivery-pipeline/main.go | 17 ++ .../cmd/mesh-delivery-pipeline/views.go | 140 +++++++++++++++ modules/mesh-delivery-pipeline/go.mod | 4 + .../internal/pipeline/registrations.go | 170 ++++++++++++++++++ .../internal/pipeline/registrations_test.go | 85 +++++++++ .../mesh-delivery-pipeline/manifest_test.go | 4 + modules/mesh-delivery-pipeline/module.json | 9 + 9 files changed, 456 insertions(+), 3 deletions(-) create mode 100644 modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go create mode 100644 modules/mesh-delivery-pipeline/internal/pipeline/registrations.go create mode 100644 modules/mesh-delivery-pipeline/internal/pipeline/registrations_test.go diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go index bebc4d19..dcc14e41 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go @@ -7,6 +7,10 @@ import ( "encoding/json" "fmt" "os" + "time" + + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" + "git.novox.be/novox/mesh-sdk/go/views" seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner" stdio "git.novox.be/novox/mesh-sdk/go" @@ -82,9 +86,22 @@ func Tools(r pipeline.Reads) []stdio.Tool { return out } +// Module and Slug are the runner's module and its tools' prefix, as its manifest names them. +const ( + Module = "mesh-delivery-pipeline" + Slug = "pipeline" +) + +// FirstViewTools are the runner's `_first_views` and `_first_view_again` over its state first-views +// (design 47 G §77, §200): each view it holds or waits for, since when, and asked again by hand, a hand act. +func FirstViewTools(st stdio.KeptState, acts views.HandActs, now func() time.Time) []stdio.Tool { + return views.StatusTools(Module, Slug+"_first_views", Slug+"_first_view_again", st, acts, now) +} + func main() { r := pipeline.Reads{Pipelines: kept{stdio.State(seat.StatePipelines)}, Kept: kept{stdio.State(seat.StateKept)}} - if err := stdio.Serve("", Tools(r)); err != nil { + tools := append(Tools(r), FirstViewTools(stdio.State(views.StatusState), meshhandactsclient.Recorder{Ask: stdio.Ask}, time.Now)...) + if err := stdio.Serve("", tools); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-pipeline-tools] %v\n", err) os.Exit(1) } diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main_test.go index 0565790a..73d96db0 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main_test.go @@ -5,6 +5,9 @@ import ( "strings" "testing" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/views" + "mesh-delivery-pipeline/internal/pipeline" ) @@ -13,13 +16,17 @@ type none struct{} func (none) Keys() ([]string, error) { return nil, nil } func (none) Value(string) (json.RawMessage, error) { return nil, nil } -// TestTools: the bundle serves the five read verbs as . (§83, §306; RN15/AC1). +// TestTools: the bundle serves the five read verbs as . (§83, §306; RN15/AC1), and the first-view tools +// (§77, §200; RN2/AC4). func TestTools(t *testing.T) { var names []string for _, tl := range Tools(pipeline.Reads{Pipelines: none{}, Kept: none{}}) { names = append(names, tl.Name) } - want := "mesh-pipeline-runner.show mesh-pipeline-runner.list mesh-pipeline-runner.stands mesh-pipeline-runner.why-carried mesh-pipeline-runner.kept" + for _, tl := range FirstViewTools(stdio.State(views.StatusState), nil, nil) { + names = append(names, tl.Name) + } + want := "mesh-pipeline-runner.show mesh-pipeline-runner.list mesh-pipeline-runner.stands mesh-pipeline-runner.why-carried mesh-pipeline-runner.kept pipeline_first_views pipeline_first_view_again" if got := strings.Join(names, " "); got != want { t.Errorf("tools %q, want %q", got, want) } diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index a931e9eb..9c35c107 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -12,6 +12,8 @@ import ( "os/signal" "syscall" + "github.com/nats-io/nats.go" + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner" "git.novox.be/novox/mesh-sdk/go/process" ) @@ -42,6 +44,21 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if _, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, seat.StatePipelines), "keep a pipeline"); err != nil { return err } + // The registrations and each watched branch's catalog half, first views then events (§21, §195, §200): until a + // branch's view is held the runner acts on nothing new of it, and says so. + calls, err := process.Connect(cred, nats.Name(cred.Node+"."+cred.Module+".calls"), nats.RetryOnFailedConnect(true)) + if err != nil { + return err + } + defer calls.Close() + hub := process.NewEventHub(logger.Printf) + if _, err := startViews(ctx, js, conn, cred, hub, calls.Call, logger.Printf); err != nil { + return err + } + if err := hub.StartUntilBound(ctx, js, cred, nil, nil); err != nil { + logger.Printf("the consumer %s is not bound yet (%v): bound once the bus makes it", cred.EventConsumerName(), err) + } + go func() { <-ctx.Done(); hub.Stop() }() logger.Printf("holding %s", seat.Seat) <-ctx.Done() return nil diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go new file mode 100644 index 00000000..0ff4228f --- /dev/null +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go @@ -0,0 +1,140 @@ +package main + +import ( + "context" + "sync" + "time" + + "github.com/nats-io/nats.go/jetstream" + + mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + sdkfindings "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient/findings" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-sdk/go/process" + "git.novox.be/novox/mesh-sdk/go/views" + + "mesh-delivery-pipeline/internal/pipeline" +) + +// catalogRevisions is every watched branch's catalog-revision, `mesh.seat.mesh-catalog.event.catalog-revision.`. +const catalogRevisions = "mesh.seat." + catalog.Seat + ".event." + catalog.EventCatalogRevision + ".>" + +// runnerViews are the runner's first views (design 47 G §21, §195, §200): the registrations, then the catalog half of +// each watched branch they name, each said in `_first_views` (its state first-views) and, past five minutes +// without a view, as view-waits in its findings. +type runnerViews struct { + Registrations *meshcatalogclient.Registrations + Catalog *pipeline.Views +} + +// startViews wires the runner's views on hub and asks each first view; register every other handler on hub before or +// after, then start hub once. +func startViews(ctx context.Context, js jetstream.JetStream, conn process.Subscriber, cred process.Account, hub *process.EventHub, + call meshcatalogclient.Call, logf func(string, ...any)) (*runnerViews, error) { + findings := process.AwaitBucket(ctx, js, conn, mcc.FindingsBucket(cred.Module), "a view waited for past five minutes is said in the log alone", logf) + status := process.AwaitBucket(ctx, js, conn, cred.Module+"_"+views.StatusState, "the runner's first views are not said in its first-view tool, nor asked again by hand", logf) + writer := func() views.FindingWriter { return sdkfindings.Writer(ctx, findings.KV) } + reader := &views.ProcessReader{Module: cred.Module, Call: call, Logf: logf, Hub: hub, Findings: writer} + + // A watched branch first named by a registration gets its view of the catalog half (CT10), its conditions and its + // status kept as every first view's, asked at once. + wire := func(f *views.Follower) { + f.Logf = logf + f.Raise = func(s views.Status) { + putFinding(writer(), views.UnansweredFinding(cred.Module, s, time.Now()), true, logf) + } + f.Clear = func(s views.Status) { + putFinding(writer(), views.UnansweredFinding(cred.Module, s, time.Now()), false, logf) + } + f.RaiseWaits = func(s views.Status) { + logf("waits for %s's view %s since %s", s.Owner, s.WaitsFor, s.WaitingSince.Format(time.RFC3339)) + putFinding(writer(), views.ViewWaitsFinding(cred.Module, s, time.Now()), true, logf) + } + f.ClearWaits = func(s views.Status) { + putFinding(writer(), views.ViewWaitsFinding(cred.Module, s, time.Now()), false, logf) + } + f.Said = func(s views.Status) { + if kv := status.KV(); kv != nil { + views.KeepStatus(ctx, kv, logf)(s) + } + } + status.Ready(func(kv jetstream.KeyValue) { + if err := views.HearAgain(ctx, kv, []*views.Follower{f}, logf); err != nil { + logf("the hand asks of the first view %s are not heard (%v) until the runner starts again", f.Tool, err) + } + }) + } + var ( + mu sync.Mutex + made []*views.Follower + ) + cv := &pipeline.Views{Logf: logf, Open: func(branch string) (pipeline.BranchView, error) { + c, err := meshcatalogclient.NewCatalog(call, branch) + if err != nil { + return nil, err + } + wire(c.Follower) + mu.Lock() + made = append(made, c.Follower) + mu.Unlock() + go func() { + if err := c.Start(ctx); err != nil { + logf("the first view of the catalog half of %s was not answered: %v", branch, err) + } + }() + return catalogView{c}, nil + }} + regs, _, err := meshcatalogclient.WireRegistrationsProcess(reader, call, func(reg catalog.Registration, gone bool) { + if !gone { + cv.Registered(reg) + } + }) + if err != nil { + return nil, err + } + if err := hub.Handle(catalogRevisions, func(m jetstream.Msg) { + if err := cv.HearRevision(ctx, m.Data()); err != nil { + logf("%s is not taken: %v", m.Subject(), err) + } + _ = m.Ack() + }); err != nil { + return nil, err + } + // A consumer made after a view was asked does not deliver what was said before it: each branch's view asks again. + hub.OnBound(func(created time.Time) { + mu.Lock() + fs := append([]*views.Follower(nil), made...) + mu.Unlock() + for _, f := range fs { + f.ConsumerMade(ctx, created) + } + }) + reader.Start(ctx, js, cred, status) + return &runnerViews{Registrations: regs, Catalog: cv}, nil +} + +func putFinding(w views.FindingWriter, f views.Finding, raised bool, logf func(string, ...any)) { + var err error + if raised { + err = w.Put(f) + } else { + err = w.Delete(f) + } + if err != nil { + logf("the finding %s is not written: %v", f.Kind, err) + } +} + +// catalogView is a branch's meshcatalogclient.Catalog as the runner reads it. +type catalogView struct{ c *meshcatalogclient.Catalog } + +func (v catalogView) Hear(ctx context.Context, data []byte) error { return v.c.Hear(ctx, data) } + +func (v catalogView) View() (pipeline.Half, bool) { + h, ok := v.c.View() + if !ok { + return pipeline.Half{}, false + } + return pipeline.Half{Revision: h.Revision.Revision, Source: h.Revision.Source, Entries: h.Entries}, true +} diff --git a/modules/mesh-delivery-pipeline/go.mod b/modules/mesh-delivery-pipeline/go.mod index 511f96a5..41926627 100644 --- a/modules/mesh-delivery-pipeline/go.mod +++ b/modules/mesh-delivery-pipeline/go.mod @@ -9,6 +9,10 @@ require ( ) require ( + git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000 github.com/klauspost/compress v1.20.1 // indirect github.com/nats-io/nkeys v0.4.16 // indirect github.com/nats-io/nuid v1.0.1 // indirect diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/registrations.go b/modules/mesh-delivery-pipeline/internal/pipeline/registrations.go new file mode 100644 index 00000000..4eaa2ae2 --- /dev/null +++ b/modules/mesh-delivery-pipeline/internal/pipeline/registrations.go @@ -0,0 +1,170 @@ +package pipeline + +import ( + "context" + "fmt" + "sort" + "sync" + + catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner" +) + +// Status is a step's status for one artifact of a module, read from the module's registration and never from a +// manifest (design 47 G §21, §23): the artifact's own status where its policy names one, else the module's. A step +// neither names is "" (design question Q-RN8, open). +func Status(reg catalog.Registration, artifact, step string) string { + if a, ok := reg.Policy.Artifacts[artifact]; ok { + if s := statusOf(a, step); s != "" { + return s + } + } + return statusOf(reg.Policy.Steps, step) +} + +func statusOf(s catalog.StepStatuses, step string) string { + switch step { + case seat.StepResultStepLint: + return s.Lint + case seat.StepResultStepUnit: + return s.Unit + case seat.StepResultStepIntegration: + return s.Integration + case seat.StepResultStepComposition: + return s.Composition + case seat.StepResultStepReview: + return s.Review + case seat.StepResultStepApproval: + return s.Approval + } + return "" +} + +// buildStep is the acting rule's step a build seat names for the languages it builds (design 47 G §323). +const buildStep = "build" + +// BuildSeat is the build seat that runs lint and the test steps of an artifact of language, read from the catalog half +// of the pipeline's revision view (design 47 G §323): the seat whose acting rule `build` has `on` naming that language, +// as an entry of the half that claims it carries it in `acts`. Two seats naming one language are an error naming both. +func BuildSeat(entries map[string]catalog.CatalogEntry, language string) (string, error) { + var seats []string + for _, e := range entries { + for _, r := range e.Acts { + if r.Step == buildStep && r.On == language && r.Seat != "" && !contains(seats, r.Seat) { + seats = append(seats, r.Seat) + } + } + } + sort.Strings(seats) + switch len(seats) { + case 1: + return seats[0], nil + case 0: + return "", fmt.Errorf("no build seat's acting rule %s names the language %q in the catalog half", buildStep, language) + } + return "", fmt.Errorf("the build seats %v each name the language %q in their acting rule %s", seats, language, buildStep) +} + +func contains(list []string, s string) bool { + for _, x := range list { + if x == s { + return true + } + } + return false +} + +// BranchView is the runner's view of one watched branch's catalog half (design 47 G §195, §200, §479): a first view, then +// that branch's catalog-revision events. meshcatalogclient.Catalog is one. +type BranchView interface { + Hear(ctx context.Context, data []byte) error + View() (Half, bool) +} + +// Half is the catalog half a branch view holds, as the runner reads it: its revision and its entries by module. +type Half struct { + Revision int64 + Source int64 + Entries map[string]catalog.CatalogEntry +} + +// Views is the runner's views of the catalog half, one per watched branch the registrations name (design 47 G §21, +// §195): a branch first named by a registration gets its view made and started by Open, and every catalog-revision +// heard goes to every branch's view, each taking its own branch's alone. +type Views struct { + // Open makes and starts the view of a branch; its error is said in the journal and the branch asked again at the + // next registration heard. + Open func(branch string) (BranchView, error) + Logf func(string, ...any) + + mu sync.Mutex + branches map[string]BranchView +} + +// Registered takes the watched branches of a module's registration (design 47 G §21): each branch the views do not hold +// yet is opened. +func (v *Views) Registered(reg catalog.Registration) { + for _, b := range reg.Branches { + v.ensure(b) + } +} + +func (v *Views) ensure(branch string) { + v.mu.Lock() + defer v.mu.Unlock() + if v.branches == nil { + v.branches = map[string]BranchView{} + } + if _, ok := v.branches[branch]; ok || branch == "" { + return + } + bv, err := v.Open(branch) + if err != nil { + if v.Logf != nil { + v.Logf("the catalog half of the watched branch %s is not followed (asked again at the next registration heard): %v", branch, err) + } + return + } + v.branches[branch] = bv +} + +// Branches are the watched branches the views hold, sorted. +func (v *Views) Branches() []string { + v.mu.Lock() + defer v.mu.Unlock() + out := make([]string, 0, len(v.branches)) + for b := range v.branches { + out = append(out, b) + } + sort.Strings(out) + return out +} + +// View is the catalog half the view of branch holds; false while it has none, when the runner acts on nothing new of +// that branch (design 47 G §200). +func (v *Views) View(branch string) (Half, bool) { + v.mu.Lock() + bv := v.branches[branch] + v.mu.Unlock() + if bv == nil { + return Half{}, false + } + return bv.View() +} + +// HearRevision hands one catalog-revision event to every branch's view. +func (v *Views) HearRevision(ctx context.Context, data []byte) error { + v.mu.Lock() + all := make([]BranchView, 0, len(v.branches)) + for _, bv := range v.branches { + all = append(all, bv) + } + v.mu.Unlock() + var first error + for _, bv := range all { + if err := bv.Hear(ctx, data); err != nil && first == nil { + first = err + } + } + return first +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/registrations_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/registrations_test.go new file mode 100644 index 00000000..0f150d36 --- /dev/null +++ b/modules/mesh-delivery-pipeline/internal/pipeline/registrations_test.go @@ -0,0 +1,85 @@ +package pipeline + +import ( + "context" + "errors" + "strings" + "testing" + + catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" +) + +// The policy, the build seat by language and the views per watched branch (design 47 G §21, §23, §323, §195). +func TestPolicyAndBuildSeat(t *testing.T) { + reg := catalog.Registration{Module: "m", Branches: []string{"main"}, Policy: catalog.Policy{ + Steps: catalog.StepStatuses{Lint: "required", Unit: "required", Integration: "optional", Approval: "required"}, + Artifacts: map[string]catalog.StepStatuses{"tools": {Unit: "bypass"}}, + }} + for _, c := range []struct{ artifact, step, want string }{ + {"program", "unit", "required"}, + {"tools", "unit", "bypass"}, + {"tools", "integration", "optional"}, + {"tools", "approval", "required"}, + {"program", "composition", ""}, + } { + if got := Status(reg, c.artifact, c.step); got != c.want { + t.Errorf("Status(%s, %s) = %q, want %q", c.artifact, c.step, got, c.want) + } + } + + entries := map[string]catalog.CatalogEntry{ + "mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}, + "mesh-build-node": {Module: "mesh-build-node", Acts: []catalog.ActingRule{{Seat: "node-build-node", Step: "build", On: "typescript"}, {Seat: "node-build-node", Step: "check", On: "go"}}}, + "app": {Module: "app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}}}, + } + for _, c := range []struct{ language, want, err string }{ + {"go", "node-build-go", ""}, + {"typescript", "node-build-node", ""}, + {"rust", "", `"rust"`}, + } { + got, err := BuildSeat(entries, c.language) + if got != c.want || (c.err == "") != (err == nil) || (err != nil && !strings.Contains(err.Error(), c.err)) { + t.Errorf("BuildSeat(%s) = %q, %v; want %q, an error naming %s", c.language, got, err, c.want, c.err) + } + } + entries["other"] = catalog.CatalogEntry{Acts: []catalog.ActingRule{{Seat: "node-build-other", Step: "build", On: "go"}}} + if _, err := BuildSeat(entries, "go"); err == nil || !strings.Contains(err.Error(), "node-build-other") { + t.Errorf("two seats naming go: %v, want an error naming both", err) + } + + heard := map[string]int{} + opened := 0 + v := &Views{Open: func(b string) (BranchView, error) { + opened++ + if b == "broken" { + return nil, errors.New("refused") + } + return fakeView{branch: b, heard: heard}, nil + }} + v.Registered(reg) + v.Registered(catalog.Registration{Module: "n", Branches: []string{"main", "feature/x", "broken"}}) + if got := v.Branches(); strings.Join(got, ",") != "feature/x,main" || opened != 3 { + t.Fatalf("branches %v opened %d, want feature/x and main, three opens", got, opened) + } + v.Registered(catalog.Registration{Module: "n", Branches: []string{"broken"}}) + if opened != 4 { + t.Errorf("a branch whose view was refused is opened again at the next registration: %d opens", opened) + } + if err := v.HearRevision(context.Background(), []byte(`{}`)); err != nil || heard["main"] != 1 || heard["feature/x"] != 1 { + t.Errorf("a revision heard reaches every branch's view: %v %v", err, heard) + } + if h, ok := v.View("main"); !ok || h.Revision != 7 { + t.Errorf("View(main) = %v %v", h, ok) + } + if _, ok := v.View("gone"); ok { + t.Error("a branch not watched has no view") + } +} + +type fakeView struct { + branch string + heard map[string]int +} + +func (f fakeView) Hear(context.Context, []byte) error { f.heard[f.branch]++; return nil } +func (f fakeView) View() (Half, bool) { return Half{Revision: 7}, true } diff --git a/modules/mesh-delivery-pipeline/manifest_test.go b/modules/mesh-delivery-pipeline/manifest_test.go index 59aa7378..3404113b 100644 --- a/modules/mesh-delivery-pipeline/manifest_test.go +++ b/modules/mesh-delivery-pipeline/manifest_test.go @@ -59,6 +59,10 @@ func TestManifest(t *testing.T) { "operator-channel.decided", "node-build-go.step-ended", "node-build-node.step-ended", "node-build-image.step-ended", "mesh-wants.step-ended", "mesh-catalog.registration-changed", "mesh-catalog.catalog-revision"})}, + // RN2 (§21, §195, §200, §77): the registrations and the catalog half asked by their verbs, the first views read and + // asked again by hand (a hand act). + {"invokes", equalSets(m.Invokes, []string{"seat:mesh-catalog.registrations", "seat:mesh-catalog.revision", "seat:mesh-hand-acts.hand-act"})}, + {"first-view tools", equalSets(m.Tools, []string{"pipeline_first_views", "pipeline_first_view_again"})}, // RN1/AC5 (§95, §130): the new assigner is delivered before the runner. {"program needs the assigner's program", slices.Equal(program.Needs, []string{"mesh-delivery-assigner:program"})}, } diff --git a/modules/mesh-delivery-pipeline/module.json b/modules/mesh-delivery-pipeline/module.json index 3a618fe0..9d85a776 100644 --- a/modules/mesh-delivery-pipeline/module.json +++ b/modules/mesh-delivery-pipeline/module.json @@ -22,6 +22,10 @@ ] } ], + "tools": [ + "pipeline_first_views", + "pipeline_first_view_again" + ], "consumes": [ "mesh-delivery-indexer.take-in.taken", "mesh-builds.package-published", @@ -34,6 +38,11 @@ "mesh-catalog.registration-changed", "mesh-catalog.catalog-revision" ], + "invokes": [ + "seat:mesh-catalog.registrations", + "seat:mesh-catalog.revision", + "seat:mesh-hand-acts.hand-act" + ], "state": [ { "name": "pipelines", From 68c10c5664e9c81d1907a3f0cbd0323af2e906a6 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 12:31:58 +0100 Subject: [PATCH 27/33] RN24: the runner's manifest says version 0.1, declares its finding stream and hears the keeper's ask MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §368, §383, §387 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- modules/mesh-delivery-pipeline/manifest_test.go | 11 ++++++++--- modules/mesh-delivery-pipeline/module.json | 15 +++++++++++++-- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/modules/mesh-delivery-pipeline/manifest_test.go b/modules/mesh-delivery-pipeline/manifest_test.go index 3404113b..dbad350d 100644 --- a/modules/mesh-delivery-pipeline/manifest_test.go +++ b/modules/mesh-delivery-pipeline/manifest_test.go @@ -49,8 +49,13 @@ func TestManifest(t *testing.T) { // §27, §85, §330, §355, §77: the old state sections. {"old state sections", equalSets(states, []string{seat.StatePipelines, seat.StateKept, "aged", "findings", "first-views"})}, // §27, §85: the step log on an old stream, its outputs in an old objects section. - {"old streams section", equalSets(streams, []string{seat.StateStepLog})}, - {"step-log 128 MiB", len(m.Streams) == 1 && m.Streams[0].MaxMiB == 128}, + // RN24 (§383, §387): beside it, the finding stream its findings are said on. + {"old streams section", equalSets(streams, []string{seat.StateStepLog, "mesh-delivery-pipeline-findings"})}, + {"step-log 128 MiB", len(m.Streams) == 2 && m.Streams[0].MaxMiB == 128}, + {"finding stream 7 days 16 MiB", len(m.Streams) == 2 && m.Streams[1].MaxAgeDays == 7 && m.Streams[1].MaxMiB == 16 && + equalSets(m.Streams[1].Events, []string{"finding-written.*", "finding-cleared.*", "findings-stated.*"})}, + // RN24 (§368): the version as MAJOR.MINOR, 0.1 with none published. + {"version 0.1", m.Version == "0.1"}, {"old objects section", equalSets(objects, []string{seat.StateStepOutputs})}, {"step-outputs 1 GiB", len(m.Objects) == 1 && m.Objects[0].MaxMiB == 1024}, // §333, §54, §44, §53, §376, §24, §37, §4, §77: what the runner hears. @@ -58,7 +63,7 @@ func TestManifest(t *testing.T) { "mesh-delivery-indexer.take-in.taken", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "operator-channel.decided", "node-build-go.step-ended", "node-build-node.step-ended", "node-build-image.step-ended", "mesh-wants.step-ended", "mesh-catalog.registration-changed", - "mesh-catalog.catalog-revision"})}, + "mesh-catalog.catalog-revision", "mesh-conditions.findings-asked"})}, // RN2 (§21, §195, §200, §77): the registrations and the catalog half asked by their verbs, the first views read and // asked again by hand (a hand act). {"invokes", equalSets(m.Invokes, []string{"seat:mesh-catalog.registrations", "seat:mesh-catalog.revision", "seat:mesh-hand-acts.hand-act"})}, diff --git a/modules/mesh-delivery-pipeline/module.json b/modules/mesh-delivery-pipeline/module.json index 9d85a776..5c71ce13 100644 --- a/modules/mesh-delivery-pipeline/module.json +++ b/modules/mesh-delivery-pipeline/module.json @@ -1,6 +1,6 @@ { "module": "mesh-delivery-pipeline", - "version": "1", + "version": "0.1", "slug": "pipeline", "claims": [ { @@ -36,7 +36,8 @@ "node-build-image.step-ended", "mesh-wants.step-ended", "mesh-catalog.registration-changed", - "mesh-catalog.catalog-revision" + "mesh-catalog.catalog-revision", + "mesh-conditions.findings-asked" ], "invokes": [ "seat:mesh-catalog.registrations", @@ -72,6 +73,16 @@ "step-log.**" ], "max-mib": 128 + }, + { + "name": "mesh-delivery-pipeline-findings", + "events": [ + "finding-written.*", + "finding-cleared.*", + "findings-stated.*" + ], + "max-age-days": 7, + "max-mib": 16 } ], "objects": [ From 38a249239417ffc2d7c29e0a498329af3efa1ee4 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 12:34:26 +0100 Subject: [PATCH 28/33] RN3 (publish half): a pipeline starts by compare-and-set on its key, and a package-published starts one per dependent behind MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A start writes the pipeline's key (watched branch, source revision, module) only when it is absent and says pipeline-started; a second start of the same revision and module writes nothing. A package-published asks the catalog's outdated for that package on that branch, a kept call, and starts a pipeline for each dependent behind by kind 1, keyed by the publication's branch and source revision. The start on take-in.taken, and a gathered group's one pipeline per branch, wait for take-in.taken's body (design 47 G §476, in review). Builds: design 47 G §54, §103, §334, §53, §75 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-pipeline/main.go | 22 ++- .../cmd/mesh-delivery-pipeline/start.go | 71 +++++++++ modules/mesh-delivery-pipeline/go.mod | 2 + .../internal/pipeline/start.go | 136 ++++++++++++++++++ .../internal/pipeline/start_test.go | 87 +++++++++++ .../mesh-delivery-pipeline/manifest_test.go | 4 +- modules/mesh-delivery-pipeline/module.json | 3 +- 7 files changed, 321 insertions(+), 4 deletions(-) create mode 100644 modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go create mode 100644 modules/mesh-delivery-pipeline/internal/pipeline/start.go create mode 100644 modules/mesh-delivery-pipeline/internal/pipeline/start_test.go diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index 9c35c107..5805b1e0 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -5,6 +5,7 @@ package main import ( "context" + "encoding/json" "fmt" "io" "log" @@ -16,6 +17,8 @@ import ( seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner" "git.novox.be/novox/mesh-sdk/go/process" + + "mesh-delivery-pipeline/internal/pipeline" ) func main() { @@ -41,7 +44,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro defer conn.Close() // The state pipelines, `_pipelines`, holds every pipeline by compare-and-set (§27): without it the // runner keeps nothing, so it stops. - if _, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, seat.StatePipelines), "keep a pipeline"); err != nil { + pipelines, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, seat.StatePipelines), "keep a pipeline") + if err != nil { return err } // The registrations and each watched branch's catalog half, first views then events (§21, §195, §200): until a @@ -52,7 +56,21 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } defer calls.Close() hub := process.NewEventHub(logger.Printf) - if _, err := startViews(ctx, js, conn, cred, hub, calls.Call, logger.Printf); err != nil { + rv, err := startViews(ctx, js, conn, cred, hub, calls.Call, logger.Printf) + if err != nil { + return err + } + // A pipeline starts by compare-and-set on its key's absence and says pipeline-started (§54, §103); a publish starts + // one per dependent behind (§53, §334). + starter := &pipeline.Starter{Pipelines: createdKV{ctx: ctx, kv: pipelines}, Logf: logger.Printf, + Say: func(subject string, body any) error { + data, err := json.Marshal(body) + if err == nil { + _, err = js.Publish(ctx, subject, data) + } + return err + }} + if err := hearPublished(ctx, hub, starter, calls, rv.Catalog, logger.Printf); err != nil { return err } if err := hub.StartUntilBound(ctx, js, cred, nil, nil); err != nil { diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go new file mode 100644 index 00000000..a2341544 --- /dev/null +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go @@ -0,0 +1,71 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + "github.com/nats-io/nats.go/jetstream" + + builds "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" + catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-sdk/go/buscalls" + "git.novox.be/novox/mesh-sdk/go/process" + + "mesh-delivery-pipeline/internal/pipeline" +) + +// outdatedVerb is the catalog's verb a publication's dependents behind are asked by, as the manifest's invokes names it. +const outdatedVerb = "seat:" + catalog.Seat + "." + catalog.VerbOutdated + +// createdKV is the state pipelines written by compare-and-set on a key's absence (§54). +type createdKV struct { + ctx context.Context + kv jetstream.KeyValue +} + +func (c createdKV) Create(key string, value []byte) error { + _, err := c.kv.Create(c.ctx, key, value) + if errors.Is(err, jetstream.ErrKeyExists) { + return pipeline.ErrExists + } + return err +} + +// hearPublished starts the pipelines of each package-published heard (§53, §334): the dependents the catalog's outdated +// names behind, each asked as a kept call. The message is acknowledged once its starts are written, on a goroutine of its +// own, so a slow catalog holds no other event. +func hearPublished(ctx context.Context, hub *process.EventHub, starter *pipeline.Starter, calls *process.Conn, v *pipeline.Views, logf func(string, ...any)) error { + outdated := func(in catalog.OutdatedInput) (catalog.OutdatedAnswer, error) { + raw, err := calls.CallKept(outdatedVerb, in) + if err != nil { + return catalog.OutdatedAnswer{}, err + } + answer, said, ok := buscalls.Unwrap(raw) + if !ok { + return catalog.OutdatedAnswer{}, fmt.Errorf("%s answered: %s", outdatedVerb, said) + } + var a catalog.OutdatedAnswer + err = json.Unmarshal(answer, &a) + return a, err + } + return hub.Handle(builds.SubjectPackagePublished, func(m jetstream.Msg) { + var ev builds.PackagePublished + if err := json.Unmarshal(m.Data(), &ev); err != nil { + logf("%s is no package-published: %v", m.Subject(), err) + _ = m.Ack() + return + } + go func() { + n, err := starter.Published(ev, outdated, v.View) + if err != nil { + logf("the publish of %s %s on %s: %d pipelines started, then %v; heard again", ev.Artifact, ev.Version, ev.Branch, n, err) + _ = m.Nak() + return + } + logf("the publish of %s %s on %s started %d pipelines", ev.Artifact, ev.Version, ev.Branch, n) + _ = m.Ack() + }() + }) +} diff --git a/modules/mesh-delivery-pipeline/go.mod b/modules/mesh-delivery-pipeline/go.mod index 41926627..daa61da4 100644 --- a/modules/mesh-delivery-pipeline/go.mod +++ b/modules/mesh-delivery-pipeline/go.mod @@ -12,6 +12,8 @@ require ( git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/contracts/go/clients/meshpipelinerunnerclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000 github.com/klauspost/compress v1.20.1 // indirect github.com/nats-io/nkeys v0.4.16 // indirect diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go new file mode 100644 index 00000000..3d693669 --- /dev/null +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -0,0 +1,136 @@ +package pipeline + +import ( + "encoding/json" + "errors" + "fmt" + "strconv" + "time" + + runnerclient "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshpipelinerunnerclient" + builds "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" + catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner" + "git.novox.be/novox/mesh-sdk/go/keys" +) + +// ErrExists is a Create of a pipeline key that is written already: the second start, which writes nothing (§54). +var ErrExists = errors.New("the pipeline's key is written already") + +// Created is the runner's state pipelines as a start writes it: a key written only when it is absent (compare-and-set on +// its absence), ErrExists when it is not. +type Created interface { + Create(key string, value []byte) error +} + +// Begin is what a start names: the pipeline's key (watched branch, source revision, module; design 47 G §27, §54) and +// what the catalog half of that branch's view says of the module. +type Begin struct { + Branch string + Source int64 + Module string + Commit string + Fp string + Group string + Repository string + Revision int64 +} + +// Key is a pipeline's key in the state pipelines, `..`, the branch escaped as a key's +// name is (§27). +func Key(branch string, source int64, module string) string { + return seat.PipelinesByBranchRevisionModuleKey(keys.EscapeName(branch), strconv.FormatInt(source, 10), module) +} + +// Starter starts pipelines (design 47 G §54, §103): it writes a pipeline's key by compare-and-set on its absence, so one +// source revision and module has one pipeline whichever event starts it first, and says pipeline-started for the one it +// wrote. +type Starter struct { + Pipelines Created + // Say says one of the seat's events on its subject. + Say func(subject string, body any) error + Now func() time.Time + Logf func(string, ...any) +} + +// Start writes b's pipeline, running, and says pipeline-started; false, nil when its key was written already (the second +// start writes nothing, §54). +func (s *Starter) Start(b Begin) (bool, error) { + if b.Branch == "" || b.Module == "" || b.Source <= 0 { + return false, fmt.Errorf("a pipeline names its watched branch, source revision and module: %+v", b) + } + now := time.Now().UTC() + if s.Now != nil { + now = s.Now().UTC() + } + key := Key(b.Branch, b.Source, b.Module) + p := seat.Pipeline{V: 1, Pipeline: key, Branch: b.Branch, Source: b.Source, Module: b.Module, Commit: b.Commit, Fp: b.Fp, + Group: b.Group, Repository: b.Repository, Revision: b.Revision, Outcome: seat.PipelineOutcomeRunning, Started: &now, + Steps: []seat.StepResult{}} + raw, err := json.Marshal(p) + if err != nil { + return false, err + } + if err := s.Pipelines.Create(key, raw); errors.Is(err, ErrExists) { + s.logf("%s is started already: this start writes nothing", key) + return false, nil + } else if err != nil { + return false, fmt.Errorf("the pipeline %s is not written: %w", key, err) + } + ev := seat.PipelineStarted{V: 1, At: &now, Pipeline: key, Branch: b.Branch, Source: b.Source, Module: b.Module, + Commit: b.Commit, Fp: b.Fp, Group: b.Group, Revision: b.Revision} + commitOrGroup := b.Commit + if b.Group != "" { + commitOrGroup = b.Group + } + if err := s.Say(runnerclient.Subject(seat.EventPipelineStarted, b.Module, b.Branch, commitOrGroup), ev); err != nil { + s.logf("pipeline-started of %s is not said: %v", key, err) + } + return true, nil +} + +func (s *Starter) logf(format string, args ...any) { + if s.Logf != nil { + s.Logf(format, args...) + } +} + +// outdatedKind is the edge `outdated` names a dependent behind by for a rebuild on a publish: B published before A builds, +// a package A imports (§53). +const outdatedKind = 1 + +// Published starts, for a package-published, a pipeline for each dependent the catalog's `outdated` names behind for that +// package on that branch by kind 1, keyed by the publication's watched branch and source revision and the dependent +// (design 47 G §53, §334). outdated is the kept call of the verb; view the catalog half of the publication's branch. A +// dependent the half does not hold is said and started with what the event names. +func (s *Starter) Published(ev builds.PackagePublished, outdated func(catalog.OutdatedInput) (catalog.OutdatedAnswer, error), view func(branch string) (Half, bool)) (int, error) { + if ev.Artifact == "" || ev.Branch == "" || ev.Source <= 0 { + return 0, fmt.Errorf("a package-published names its artifact, watched branch and source revision: %+v", ev) + } + a, err := outdated(catalog.OutdatedInput{Branch: ev.Branch, Dependency: ev.Artifact}) + if err != nil { + return 0, fmt.Errorf("outdated of %s on %s: %w", ev.Artifact, ev.Branch, err) + } + half, _ := view(ev.Branch) + started := 0 + var first error + for _, o := range a.Behind { + if o.Kind != outdatedKind || o.Module == "" { + continue + } + b := Begin{Branch: ev.Branch, Source: ev.Source, Module: o.Module, Revision: half.Revision} + if e, ok := half.Entries[o.Module]; ok { + b.Commit, b.Fp, b.Repository = e.Commit, e.Fp, e.Repository + } else { + s.logf("the catalog half of %s holds no entry of %s, behind %s: its pipeline starts without its commit", ev.Branch, o.Module, ev.Artifact) + } + ok, err := s.Start(b) + if err != nil && first == nil { + first = err + } + if ok { + started++ + } + } + return started, first +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go new file mode 100644 index 00000000..da18b6ba --- /dev/null +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go @@ -0,0 +1,87 @@ +package pipeline + +import ( + "encoding/json" + "errors" + "strings" + "testing" + + builds "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" + catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner" +) + +type created map[string][]byte + +func (c created) Create(key string, value []byte) error { + if _, ok := c[key]; ok { + return ErrExists + } + c[key] = value + return nil +} + +// The starts (design 47 G §54, §103, §334): a pipeline's key written by compare-and-set on its absence, so a second start +// of the same source revision and module writes nothing; a publish starts one pipeline per dependent outdated names +// behind by kind 1, keyed by the publishing branch and revision. +func TestStarts(t *testing.T) { + store := created{} + var said []string + s := &Starter{Pipelines: store, Say: func(subject string, body any) error { + ev := body.(seat.PipelineStarted) + said = append(said, subject+" "+ev.Pipeline) + return nil + }} + view := func(branch string) (Half, bool) { + return Half{Revision: 9, Entries: map[string]catalog.CatalogEntry{ + "app": {Module: "app", Commit: "c1", Fp: "src2:aa", Repository: "o/app"}}}, true + } + var asked catalog.OutdatedInput + outdated := func(in catalog.OutdatedInput) (catalog.OutdatedAnswer, error) { + asked = in + return catalog.OutdatedAnswer{Behind: []catalog.Outdated{ + {Module: "app", Kind: 1}, {Module: "tool", Kind: 1}, {Module: "ordered", Kind: 4}}}, nil + } + n, err := s.Published(builds.PackagePublished{Artifact: "mesh-sdk:go", Branch: "main", Source: 41, Module: "mesh-sdk", Version: "0.1.16"}, outdated, view) + if err != nil || n != 2 { + t.Fatalf("a publish started %d (%v), want the two dependents behind by kind 1", n, err) + } + if asked.Dependency != "mesh-sdk:go" || asked.Branch != "main" { + t.Errorf("outdated asked %+v, want the package on its branch", asked) + } + var p seat.Pipeline + if err := json.Unmarshal(store["main.41.app"], &p); err != nil || p.Commit != "c1" || p.Fp != "src2:aa" || p.Revision != 9 || + p.Outcome != seat.PipelineOutcomeRunning || p.Source != 41 { + t.Errorf("the pipeline main.41.app = %+v (%v)", p, err) + } + if _, ok := store["main.41.ordered"]; ok { + t.Error("an ordering edge starts no pipeline") + } + // The take-in of the same source revision and module is the same pipeline: the second start writes nothing. + before := string(store["main.41.app"]) + ok, err := s.Start(Begin{Branch: "main", Source: 41, Module: "app", Commit: "c2"}) + if ok || err != nil || string(store["main.41.app"]) != before || len(said) != 2 { + t.Errorf("a second start wrote %v (%v); said %v", ok, err, said) + } + if !strings.HasPrefix(said[0], "mesh.seat.mesh-pipeline-runner.event.pipeline-started.app.main.c1 main.41.app") { + t.Errorf("pipeline-started said on %q", said[0]) + } + // A branch holding a dot is escaped in the key, so the key reads back; a start naming no source revision is refused. + if _, err := s.Start(Begin{Branch: "release.1", Source: 3, Module: "app"}); err != nil { + t.Fatal(err) + } + if b, r, m, ok := seat.ParsePipelinesByBranchRevisionModuleKey(Key("release.1", 3, "app")); !ok || r != "3" || m != "app" || b == "release" { + t.Errorf("the key of release.1 is %q", Key("release.1", 3, "app")) + } + if _, err := s.Start(Begin{Branch: "main", Module: "app"}); err == nil { + t.Error("a start with no source revision is refused") + } + s.Pipelines = failing{} + if _, err := s.Start(Begin{Branch: "main", Source: 5, Module: "app"}); err == nil { + t.Error("a write the bus refuses is an error") + } +} + +type failing struct{} + +func (failing) Create(string, []byte) error { return errors.New("bus away") } diff --git a/modules/mesh-delivery-pipeline/manifest_test.go b/modules/mesh-delivery-pipeline/manifest_test.go index dbad350d..c60c345f 100644 --- a/modules/mesh-delivery-pipeline/manifest_test.go +++ b/modules/mesh-delivery-pipeline/manifest_test.go @@ -66,7 +66,9 @@ func TestManifest(t *testing.T) { "mesh-catalog.catalog-revision", "mesh-conditions.findings-asked"})}, // RN2 (§21, §195, §200, §77): the registrations and the catalog half asked by their verbs, the first views read and // asked again by hand (a hand act). - {"invokes", equalSets(m.Invokes, []string{"seat:mesh-catalog.registrations", "seat:mesh-catalog.revision", "seat:mesh-hand-acts.hand-act"})}, + {"invokes", equalSets(m.Invokes, []string{"seat:mesh-catalog.registrations", "seat:mesh-catalog.revision", "seat:mesh-hand-acts.hand-act", + // RN3 (§53, §334): a publication's dependents behind. + "seat:mesh-catalog.outdated"})}, {"first-view tools", equalSets(m.Tools, []string{"pipeline_first_views", "pipeline_first_view_again"})}, // RN1/AC5 (§95, §130): the new assigner is delivered before the runner. {"program needs the assigner's program", slices.Equal(program.Needs, []string{"mesh-delivery-assigner:program"})}, diff --git a/modules/mesh-delivery-pipeline/module.json b/modules/mesh-delivery-pipeline/module.json index 5c71ce13..d0055720 100644 --- a/modules/mesh-delivery-pipeline/module.json +++ b/modules/mesh-delivery-pipeline/module.json @@ -42,7 +42,8 @@ "invokes": [ "seat:mesh-catalog.registrations", "seat:mesh-catalog.revision", - "seat:mesh-hand-acts.hand-act" + "seat:mesh-hand-acts.hand-act", + "seat:mesh-catalog.outdated" ], "state": [ { From 846340c5539dff4e9d9fea600b290f7f928a0b53 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 12:37:06 +0100 Subject: [PATCH 29/33] AS4: the assigner computes a node's entries from the catalog half of the branch its placement names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The indexer's first views of manifests, modules and graph are replaced by one meshcatalogclient.Catalog per watched branch (the trunk's from the start, each branch a registration names from then on); the world keeps each branch's modules, graph nodes and manifests, and a pass splits a module's nodes by the branch their placements name and computes each group's wants from that branch's half. Not built: AC2 (a composition step on beta only), which is AS5's step. Builds: design 47 G §14, §184, §191, §194, §195 Tests: TestACommitOnBetaMovesOnlyTheNodeThatChoseBeta (failed with the groups on the trunk alone), TestTheCatalogHalfIsTakenInPerBranch Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-assigner/halves.go | 147 ++++++++ .../integration_test.go | 138 ++++++-- .../cmd/mesh-delivery-assigner/main.go | 11 +- .../cmd/mesh-delivery-assigner/pass.go | 314 +++++++++++------- .../cmd/mesh-delivery-assigner/pass_test.go | 31 +- .../cmd/mesh-delivery-assigner/pipelines.go | 5 + .../mesh-delivery-assigner/pipelines_test.go | 105 +++++- .../cmd/mesh-delivery-assigner/puts.go | 3 +- .../cmd/mesh-delivery-assigner/seat.go | 7 +- .../cmd/mesh-delivery-assigner/verbs.go | 34 +- .../cmd/mesh-delivery-assigner/views.go | 83 +++-- .../cmd/mesh-delivery-assigner/world.go | 146 ++++---- .../cmd/mesh-delivery-assigner/world_test.go | 10 + .../mesh-delivery-assigner/manifest_test.go | 8 +- modules/mesh-delivery-assigner/module.json | 6 +- 15 files changed, 739 insertions(+), 309 deletions(-) create mode 100644 modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/halves.go diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/halves.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/halves.go new file mode 100644 index 00000000..07829862 --- /dev/null +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/halves.go @@ -0,0 +1,147 @@ +package main + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "slices" + "sync" + "time" + + "github.com/nats-io/nats.go/jetstream" + + mcc "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/sdk/graph" + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// halves follows the catalog half of each watched branch through the seat mesh-catalog (design 47 G §14, §184, §195): one +// meshcatalogclient.Catalog per branch, a first view of the verb `revision` and then each `catalog-revision` of that branch, +// each entry taken into the world as the value and the graph node the rules read. Of the view the assigner reads only the catalog +// half, never its own output back (§195). +type halves struct { + w *world + call mcc.Call + wake func() + logf func(string, ...any) + ctx context.Context + + mu sync.Mutex + cats map[string]*mcc.Catalog + // started is called with each catalog made, for the part that reads their status. + started func(*mcc.Catalog) + // settled is told once when a branch's first view is answered, for the first pass to wait on (the trunk's alone). + settled chan<- struct{} +} + +// follow makes the catalog of branch and asks its first view, once; true when it was made now. +func (h *halves) follow(branch string) bool { + h.mu.Lock() + if h.cats[branch] != nil { + h.mu.Unlock() + return false + } + c, err := mcc.NewCatalog(h.call, branch) + if err != nil { + h.mu.Unlock() + h.logf("the catalog half of branch %q is not followed: %v", branch, err) + return false + } + c.Follower.Logf = h.logf + c.Follower.Changed = func(module string, entry json.RawMessage, deleted bool, _ time.Time) { + h.changed(branch, module, entry, deleted) + } + if h.cats == nil { + h.cats = map[string]*mcc.Catalog{} + } + h.cats[branch] = c + h.mu.Unlock() + if h.started != nil { + h.started(c) + } + go func() { + if err := c.Start(h.ctx); err != nil { + if h.ctx.Err() == nil { + h.logf("the first view of the catalog half of branch %q is not answered: %v", branch, err) + } + return + } + if branch == trunkBranch && h.settled != nil { + h.settled <- struct{}{} + } + }() + return true +} + +// hear gives a catalog-revision to every catalog: each takes its own branch's and passes over another's. +func (h *halves) hear(m jetstream.Msg) { + h.mu.Lock() + cats := make([]*mcc.Catalog, 0, len(h.cats)) + for _, c := range h.cats { + cats = append(cats, c) + } + h.mu.Unlock() + for _, c := range cats { + if err := c.Hear(h.ctx, m.Data()); err != nil { + h.logf("%s on branch %q: %v", m.Subject(), c.Branch, err) + } + } + _ = m.Ack() +} + +// followers are the first views the catalogs hold. +func (h *halves) followers() []*mcc.Catalog { + h.mu.Lock() + defer h.mu.Unlock() + out := make([]*mcc.Catalog, 0, len(h.cats)) + for _, c := range h.cats { + out = append(out, c) + } + return out +} + +// changed takes one entry of branch's half into the world, or a module taken out of it. +func (h *halves) changed(branch, module string, raw json.RawMessage, deleted bool) { + defer h.wake() + if deleted { + h.w.dropModule(branch, module) + return + } + var e meshcatalog.CatalogEntry + if err := json.Unmarshal(raw, &e); err != nil { + h.logf("the entry of %s on branch %q does not read: %v", module, branch, err) + return + } + v, n, m := fromEntry(e) + h.w.putModule(branch, module, v, n, m) +} + +// fromEntry is a catalog entry as the value and the graph node the rules read, and its manifest parsed once under the digest of +// its words. +func fromEntry(e meshcatalog.CatalogEntry) (meshcatalog.ModulesByModuleValue, graph.Node, *parsed) { + v := meshcatalog.ModulesByModuleValue{Commit: e.Commit, Fp2: e.Fp, Needs: e.Needs, None2: e.None, Packages: e.Packages, + Path: e.Path, Repository: e.Repository, Verdict: e.Verdict} + n := graph.Node{FP: e.Fp, None: e.None, Repository: e.Repository, Path: e.Path, Bases: e.Bases, Provides: e.Provides, Requires: e.Requires, + Awaits: e.Awaits, Invokes: e.Invokes, Serves: e.Serves, LeftOut: e.LeftOut} + for p := range e.Packages { + n.Packages = append(n.Packages, p) + } + slices.Sort(n.Packages) + if len(e.Manifest) == 0 { + return v, n, nil + } + raw, err := json.Marshal(e.Manifest) + if err != nil { + return v, n, nil + } + sum := sha256.Sum256(raw) + v.Manifest = "sha256:" + hex.EncodeToString(sum[:]) + man, err := catalogue.Decode(raw) + if err != nil { + return v, n, &parsed{module: e.Module, err: "it does not decode: " + err.Error()} + } + n.Serves = graph.ServesOf(man) + return v, n, &parsed{module: e.Module, manifest: man} +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go index 0ce2b023..e10e9312 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go @@ -7,6 +7,7 @@ import ( "encoding/json" "errors" "fmt" + "git.novox.be/novox/mesh-sdk/go/keys" "io" "maps" "mesh-delivery-assigner/internal/assignment" @@ -52,7 +53,6 @@ import ( "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient/enginereads" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" - "git.novox.be/novox/mesh-delivery/sdk/graph" "git.novox.be/novox/mesh-sdk/go/views" "mesh-delivery-assigner/internal/assign" @@ -71,6 +71,17 @@ type bed struct { placed map[string][]string // journal is what the programs the bed starts write in their journal. journal *journal + // cat is the catalog half the seat mesh-catalog serves the assigner (serveCatalog). + cat *catalogBed +} + +// catalogBed is the catalog half of each watched branch as the seat mesh-catalog keeps it: entries by branch and module, a +// revision per branch, and the manifest bodies by digest. +type catalogBed struct { + mu sync.Mutex + entries map[string]map[string]meshcatalog.CatalogEntry + rev map[string]int64 + bodies map[string]string } // journal is a program's journal as the bed hears it: a test awaits a line in it (await) where the line is the only sign @@ -144,7 +155,8 @@ func startBed(t *testing.T, leaveOut string) *bed { } ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) t.Cleanup(cancel) - b := &bed{srv: srv, js: js, ctx: ctx, dir: dir, buckets: map[string]jetstream.KeyValue{}, journal: &journal{grew: make(chan struct{})}} + b := &bed{srv: srv, js: js, ctx: ctx, dir: dir, buckets: map[string]jetstream.KeyValue{}, journal: &journal{grew: make(chan struct{})}, + cat: &catalogBed{entries: map[string]map[string]meshcatalog.CatalogEntry{}, rev: map[string]int64{}, bodies: map[string]string{}}} for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) { if name == leaveOut { continue @@ -171,7 +183,7 @@ func startBed(t *testing.T, leaveOut string) *bed { // event names the engine's machine last, so the filter is the event with every machine (mesh-grants buscompose // machineEvent; the grants part's own test holds the derivation). if _, err := js.CreateOrUpdateConsumer(ctx, "EVENTS", jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-assigner", - FilterSubjects: []string{enginereads.SubjectPrefix + "*", "mesh.mod." + buckets.IndexerModule + ".event.views.>"}, AckPolicy: jetstream.AckExplicitPolicy}); err != nil { + FilterSubjects: []string{enginereads.SubjectPrefix + "*", "mesh.mod." + buckets.IndexerModule + ".event.views.>", meshcatalog.SubjectCatalogRevisionAll}, AckPolicy: jetstream.AckExplicitPolicy}); err != nil { t.Fatal(err) } // The operator-channel seat's stream of conditions, which the assigner says its conditions on. @@ -180,6 +192,8 @@ func startBed(t *testing.T, leaveOut string) *bed { } // The indexer as the owner of its manifests and modules (DLV-F2r), with the real owner's side. serveIndexer(t, b, nc) + // The seat mesh-catalog's catalog half (design 47 G §194), which the assigner follows per branch. + serveCatalog(t, b, nc) // The assigner's views family is its ordinary events on EVENTS (ADR 0635 §5); its marks are its state. if kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: marksBucket, History: 1}); err != nil { t.Fatal(err) @@ -422,6 +436,7 @@ func (b *bed) manifest(t *testing.T, module, body string) string { t.Fatal(err) } b.putJSON(t, buckets.Manifests, key, value) + b.cat.bodies[digest] = body return digest } @@ -447,11 +462,11 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) { // DLV-F5r (ADR 0621 §3): the indexer's runtime starts together with the assigner and answers its first views only // after the assigner asked; within the bound the assigner writes the wants from what they answer. up := make(chan struct{}) - indexerLate = up + catalogLate = up // ADR 0631 §2: while the indexer does not answer, past the first-view bound the assigner raises the condition naming // the indexer and graph, and reads nothing of the indexer's state; the answer clears it. firstViewBound, firstViewWait = 300*time.Millisecond, 50*time.Millisecond - t.Cleanup(func() { indexerLate, firstViewBound, firstViewWait = nil, 0, 0 }) + t.Cleanup(func() { catalogLate, firstViewBound, firstViewWait = nil, 0, 0 }) for _, name := range append(append([]string{}, readBuckets...), watched...) { if strings.HasPrefix(name, buckets.IndexerModule+"_") { t.Errorf("the assigner reads the indexer's state %s itself", name) @@ -461,7 +476,7 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) { fNew, fOld := meshcatalogclient.SourcePrefix2+strings.Repeat("b", 64), meshcatalogclient.SourcePrefix2+strings.Repeat("a", 64) digest := b.manifest(t, "web", `{"module":"web","version":"1"}`) mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fNew, Manifest: digest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath}) - b.putJSON(t, buckets.Modules, "web", mod) + b.putModule(t, "web", mod) built, err := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: digest}) if err != nil { t.Fatal(err) @@ -493,20 +508,20 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) { "seat-holders": {"n1.node-tools"}, "wants": {"n1.mesh-grants"}, }) - graphKey := meshconditionsclient.Key(meshconditionsclient.ScopeModule, assignment.Module+"."+buckets.IndexerModule+"."+graph.State, views.UnansweredKind) + graphKey := meshconditionsclient.Key(meshconditionsclient.ScopeModule, assignment.Module+"."+meshcatalog.Seat+"."+views.BranchState(meshcatalog.StateCatalogRevisions, trunkBranch), views.UnansweredKind) graphRaised := func() bool { e, err := b.buckets[assign.FindingsBucket].Get(b.ctx, graphKey) - return err == nil && strings.Contains(string(e.Value()), buckets.IndexerModule+".graph") + return err == nil && strings.Contains(string(e.Value()), "mesh-catalog.revision") } for deadline := time.Now().Add(10 * time.Second); !graphRaised(); time.Sleep(50 * time.Millisecond) { if time.Now().After(deadline) { - t.Fatalf("with the indexer down, no condition names its graph (finding %s)", graphKey) + t.Fatalf("with the seat mesh-catalog down, no condition names its revision verb (finding %s)", graphKey) } } close(up) for deadline := time.Now().Add(10 * time.Second); graphRaised(); time.Sleep(50 * time.Millisecond) { if time.Now().After(deadline) { - t.Fatalf("the indexer answered, and the condition naming its graph stands (finding %s)", graphKey) + t.Fatalf("the seat answered, and the condition naming its revision verb stands (finding %s)", graphKey) } } @@ -608,7 +623,7 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) { fp, digest string }{"db": {fpOf('2'), dbDigest}, "app": {fpOf('3'), appDigest}, "tool": {fpOf('4'), toolDigest}} { mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: row.fp, Manifest: row.digest, Repository: "novox/" + module, Commit: strings.Repeat("2", 40), Path: meshcatalogclient.RootPath}) - b.putJSON(t, buckets.Modules, module, mod) + b.putModule(t, module, mod) fps := []string{row.fp} if module == "tool" { fps = append(fps, toolOld) @@ -680,7 +695,7 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) { formsDigest := b.manifest(t, "forms", `{"module":"forms","version":"1"}`) formsAt := func(fp string) { mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: formsDigest, Repository: "novox/forms", Commit: strings.Repeat("3", 40), Path: meshcatalogclient.RootPath}) - b.putJSON(t, buckets.Modules, "forms", mod) + b.putModule(t, "forms", mod) } for _, fp := range []string{formsOld, formsNew} { build := builds.BuilderBuild{State: builds.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: formsDigest} @@ -802,7 +817,7 @@ func TestAHeldFingerprintIsAskedAbout(t *testing.T) { fNew := meshcatalogclient.SourcePrefix2 + strings.Repeat("c", 64) digest := b.manifest(t, "web", `{"module":"web","version":"1"}`) mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fNew, Manifest: digest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath}) - b.putJSON(t, buckets.Modules, "web", mod) + b.putModule(t, "web", mod) on, _ := presenceValue(true) b.putJSON(t, buckets.Presence, "ace", on) failed, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed, Detail: "connection refused"}) @@ -860,7 +875,7 @@ func TestAFailedDataModuleIsAskedToRestore(t *testing.T) { fNew := meshcatalogclient.SourcePrefix2 + strings.Repeat("d", 64) digest := b.manifest(t, "mail", `{"module":"mail","version":"1","resources":[{"id":"data","type":"directory","path":"/var/lib/mail"}],"data":{"own":[{"id":"mail","path":"${dir:data}","class":"irreplaceable","backup":"copy"}]}}`) mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fNew, Manifest: digest, Repository: "novox/mail", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath}) - b.putJSON(t, buckets.Modules, "mail", mod) + b.putModule(t, "mail", mod) on, _ := presenceValue(true) b.putJSON(t, buckets.Presence, "ace", on) failed, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed, Detail: "connection refused"}) @@ -910,7 +925,7 @@ func TestTheAssignerReplacedResumesWithNothingLostOrRepeated(t *testing.T) { fp := meshcatalogclient.SourcePrefix2 + strings.Repeat(string(rune('a'+i%6)), 63) + fmt.Sprintf("%x", i%16) digest := b.manifest(t, name, fmt.Sprintf(`{"module":%q,"version":"1"}`, name)) mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: digest, Repository: "novox/" + name, Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath}) - b.putJSON(t, buckets.Modules, name, mod) + b.putModule(t, name, mod) built, err := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: digest}) if err != nil { t.Fatal(err) @@ -956,7 +971,7 @@ func (b *bed) fixture(t *testing.T, name, fp, manifest string) { t.Helper() digest := b.manifest(t, name, manifest) mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: digest, Repository: "novox/" + name, Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath}) - b.putJSON(t, buckets.Modules, name, mod) + b.putModule(t, name, mod) built, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: digest}) bkey, _ := builds.BuilderBuildKey(name, fp) b.putJSON(t, buckets.Builds, bkey, built) @@ -1305,9 +1320,9 @@ func TestAProposalIsAnsweredApprovedAndWritten(t *testing.T) { rec, _ := nodeengineclient.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Record: &nodeengine.NodeRecord{Node: node}}) b.putJSON(t, buckets.Have, node+".node", rec) } - b.putJSON(t, buckets.Modules, "web", []byte(`{"v":1}`)) - b.putJSON(t, buckets.Modules, "shell", []byte(`{"v":1}`)) - b.putJSON(t, buckets.Modules, "mesh-enrolment", []byte(`{"v":1}`)) + b.putModule(t, "web", []byte(`{"v":1}`)) + b.putModule(t, "shell", []byte(`{"v":1}`)) + b.putModule(t, "mesh-enrolment", []byte(`{"v":1}`)) // place writes a node's assignments as another writer would, in its node and module keys (research 072 row 37), and // answers its node key's revision. place := func(node string, modules ...string) uint64 { @@ -1611,9 +1626,9 @@ func assignmentsSubjects() []string { return out } -// indexerLate, when set before the bed is made, fails every ask of the indexer's first views until it is closed: the +// catalogLate, when set before the bed is made, fails every ask of the indexer's first views until it is closed: the // indexer down, its runtime coming up after the assigner asked (DLV-F5r, ADR 0631 §2). -var indexerLate chan struct{} +var catalogLate chan struct{} // serveIndexer is the indexer as the owner of its manifests, modules and module graph (ADR 0615, ADR 0621, ADR 0631): its // buckets, which a test fills as the indexer writes them; its views family said by the views emitter as ordinary events on EVENTS, @@ -1635,7 +1650,7 @@ func serveIndexer(t *testing.T, b *bed, nc *nats.Conn) { if err := em.Start(b.ctx, state, kv, nil); err != nil { t.Fatal(err) } - late := indexerLate + late := catalogLate sub, err := nc.Subscribe("mesh.mod.mesh-delivery-indexer.call."+state+".>", func(m *nats.Msg) { if late != nil { // its runtime comes up after the assigner started (DLV-F5r): until then every ask fails, as on a down indexer select { @@ -1797,3 +1812,82 @@ func changeIn(body map[string]any) plc.Change { _ = json.Unmarshal(raw, &c) return c } + +// indexerFirstViews are the indexer's states the bed still serves (the verbs' check of a proposal asks the indexer's modules and +// manifests): the bucket each is kept in. +var indexerFirstViews = map[string]string{"manifests": buckets.Manifests, "modules": buckets.Modules} + +// putModule keeps a module's value as the indexer does for the verbs, and as an entry of the trunk's catalog half; a value that +// names no manifest the bed holds is kept for the verbs alone. +func (b *bed) putModule(t *testing.T, module string, value []byte) { + t.Helper() + b.putJSON(t, buckets.Modules, module, value) + v, err := meshcatalogclient.DecodeModule(value) + if err != nil { // a value the bed writes only for the module to exist + v = meshcatalog.ModulesByModuleValue{Repository: "novox/" + module} + } + b.publish(t, trunkBranch, module, v) +} + +// publish makes the trunk's catalog half of branch name module at v, a new revision of it, and says catalog-revision. +func (b *bed) publish(t *testing.T, branch, module string, v meshcatalog.ModulesByModuleValue) { + t.Helper() + e := meshcatalog.CatalogEntry{Module: module, Fp: v.Fp2, Commit: v.Commit, Repository: v.Repository, Path: v.Path, Needs: v.Needs, Verdict: "accepted"} + if body, ok := b.cat.bodies[v.Manifest]; ok { + _ = json.Unmarshal([]byte(body), &e.Manifest) + } + b.cat.mu.Lock() + if b.cat.entries[branch] == nil { + b.cat.entries[branch] = map[string]meshcatalog.CatalogEntry{} + } + b.cat.entries[branch][module] = e + b.cat.rev[branch]++ + rev, n := b.cat.rev[branch], int64(len(b.cat.entries[branch])) + b.cat.mu.Unlock() + raw, _ := json.Marshal(meshcatalog.CatalogRevisionEvent{Kind: "trunk", Branch: keys.EscapeName(branch), Revision: rev, Entries: n, Changed: []string{module}}) + if _, err := b.js.Publish(b.ctx, meshcatalog.SubjectCatalogRevision(branch), raw); err != nil { + t.Fatal(err) + } +} + +// serveCatalog answers the seat mesh-catalog's verb revision from the bed's catalog half, all in one page; until catalogLate is +// closed every ask fails, as on a seat whose holder is down. +func serveCatalog(t *testing.T, b *bed, nc *nats.Conn) { + t.Helper() + late := catalogLate + answer := func(m *nats.Msg) { + if late != nil { + select { + case <-late: + default: + _ = m.Respond([]byte(`{"error":"the seat mesh-catalog's holder is not up"}`)) + return + } + } + var in meshcatalog.RevisionInput + _ = json.Unmarshal(m.Data, &in) + branch, _ := keys.UnescapeName(in.Branch) + b.cat.mu.Lock() + defer b.cat.mu.Unlock() + if b.cat.rev[branch] == 0 { + _ = m.Respond([]byte(`{"error":"` + views.NoneYet + `"}`)) + return + } + rev := meshcatalog.CatalogRevision{Kind: "trunk", Branch: in.Branch, Revision: b.cat.rev[branch]} + var entries []meshcatalog.CatalogEntry + for _, name := range slices.Sorted(maps.Keys(b.cat.entries[branch])) { + entries = append(entries, b.cat.entries[branch][name]) + rev.Entries = append(rev.Entries, meshcatalog.EntryRef{Module: name, Digest: "sha256:" + strings.Repeat("0", 64)}) + } + raw, _ := json.Marshal(map[string]any{"result": meshcatalog.RevisionAnswer{Revision: rev, Entries: entries}, "node": "n0"}) + _ = m.Respond(raw) + } + for _, subject := range []string{"mesh.seat.mesh-catalog.tool.revision", "mesh.seat.mesh-catalog.call.revision.*", "mesh.seat.mesh-catalog.tool.revision.*"} { + sub, err := nc.Subscribe(subject, answer) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = sub.Unsubscribe() }) + } + subscribed(t, nc) +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index 72177123..e99492ea 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -248,7 +248,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro conn.Close() } }() - iv := &indexerViews{js: js, account: account, call: said.Call, findings: findings, logf: logger.Printf, + iv := &catalogViews{js: js, account: account, call: said.Call, findings: findings, logf: logger.Printf, status: process.AwaitBucket(ctx, js, conn, firstViewsBucket, "assigner_first_views reads nothing and assigner_first_view_again is not heard", logger.Printf)} return follow(ctx, js, conn, kvs, findings.Say(sayVia(ctx, js)), findings, emitting{said}, iv, logger) } @@ -306,7 +306,7 @@ var watched = []string{ // follow watches every bucket a pass reads and makes a pass at start, once every watch handed what stands, and after // each change; changes that come during a pass make one more. It returns when ctx ends or a watch ends. -func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, kvs kvSet, say func(oc.Condition) error, findings assign.Findings, bus caller, iv *indexerViews, logger *log.Logger) error { +func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, kvs kvSet, say func(oc.Condition) error, findings assign.Findings, bus caller, iv *catalogViews, logger *log.Logger) error { w := newWorld() dirty := make(chan struct{}, 1) wake := func() { @@ -494,6 +494,13 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, // The runner's pipeline-ended and group-ended (design 47 G §29): kept in the state written, which the wants follow // once the first registrations are written (§95, §353). p.ends = newPipelineEnds(kvEnds{ctx, kv}, wake, logger.Printf) + if iv != nil && iv.h != nil { + p.ends.onBranches = func(branches []string) { + for _, b := range branches { + iv.h.follow(b) + } + } + } err := hearEnds(hub, p.ends) if err == nil && bus != nil { err = followRegistrations(ctx, hub, bus.Call, p.ends) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go index ef0ccc4f..33382b2d 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go @@ -8,6 +8,7 @@ import ( "maps" "mesh-delivery-assigner/internal/buckets" "slices" + "sort" "strings" "time" @@ -23,6 +24,7 @@ import ( oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" "git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient" + "git.novox.be/novox/mesh-delivery/sdk/graph" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient/enginereads" @@ -219,19 +221,21 @@ func (p *passer) nodeWants(ctx context.Context) { // the hold of a fingerprint whose health check failed. The assigner compares its wants with nothing the controller // sends: the differences bucket is kept read-only with its history (ADR 0539 §8). func (p *passer) wants(ctx context.Context) { - get := p.w.getter(buckets.Manifests) // the indexer's manifests, as its first view and views family say them assigned := p.w.assigned() + assignedBranches := p.w.placedBranches() + // Every module of every branch's catalog half, for the parts that ask which exist; a node's entries come from the half of the + // branch its placement names (design 47 G §14, §184). trunk := p.w.trunk() - manifests := map[string]catalogue.Manifest{} - for name, mod := range trunk { - if mod.Manifest == "" { - continue - } - if got := p.w.manifest(ctx, get, mod.Manifest); got.err == "" { - manifests[name] = got.manifest + views := map[string]*branchView{} + viewOf := func(branch string) *branchView { + if v := views[branch]; v != nil { + return v } + v := p.branchView(branch, assigned) + views[branch] = v + return v } - nodes := p.w.nodes(manifests, assigned) + nodes := viewOf(trunkBranch).nodes p.engineReads(nodes) nodesOf := map[string][]string{} for node, modules := range assigned { @@ -268,111 +272,118 @@ func (p *passer) wants(ctx context.Context) { defer func() { p.sayNeither(neither, stale, left) }() // Where grants are read from, once for the pass: the grants part's (issue 774, ADR 0571 clause 9). grants := p.w.grantsBucket() - // The verb edges, once for the pass (ADR 0614 §2, §3). - moduleGraph := p.moduleGraph() - served := p.served(ctx, get, moduleGraph, nodes) - offering := p.offering(ctx, get, manifests, nodes) // bus state (ADR 0614 §5) - assignedBranches := p.w.placedBranches() for name := range names { - fp := trunk[name].Fp2 - name := name - // A want follows a passed pipeline: a module whose fingerprint no pipeline passed at on every branch its placements - // chose, and whose wants do not name it already, gets none (design 47 G §29, §35). - branches := p.ends.branchesOf(name, assignedBranches) - if fp != "" && !p.ends.mayWant(name, fp, branches, wanted[name]) { - continue - } - m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Needs: trunk[name].Needs, Chosen: chosenOf(chosen, name), Provisions: provisionsOf(name), Answering: answering(manifests, nodes), Offered: offered(manifests), Nodes: nodesOf[name], Now: time.Now(), Puts: puts[name], FormNeeds: p.w.buildNeeds(name, fp), - Self: func(node string) bool { return placed[node] }, - Composed: func(node, fp string) (assign.ComposedDigest, bool) { - return p.w.composed(node, name, fp) // a placed node's own: the digest its own have names (ADR 0571 §6) - }, - GrantedAt: func(fp string) bool { - key, err := grantsstate.PartGrantKey(name, fp) - return err == nil && p.w.has(grants, key) - }, - Restored: func(node string) bool { - key, err := asks.RestoreAskKey(node, name, fp) - return err == nil && p.answeredWith(key, assign.OptionRestore) - }, - Allowed: func(node, digest string) bool { - key, err := asks.RecordAskKey(node, name, digest) - return err == nil && p.answeredWith(key, assign.OptionYes) - }, - Release: func(fp string) (string, bool) { return p.w.release(name, fp) }} - if fp != "" { - held, _ := mwc.HeldKey(name, fp) - grant, _ := grantsstate.PartGrantKey(name, fp) - hold, _ := asks.HoldAskKey(name, fp) - m.Released = p.answeredWith(hold, "release") - m.Kept = p.answeredWith(hold, "keep") - m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(buckets.Wants, held) && !m.Released, p.w.has(grants, grant) - if raw := p.w.value(grants, grant); raw != nil { - if g, err := grantsstate.DecodeGrant(raw); err == nil { - m.GrantedOn = func(node string) bool { return assign.GrantNamesNode(g, node, name) } + groups := p.groups(name, nodesOf[name], assignedBranches[name]) + for gi, g := range groups { + func() { + bv := viewOf(g.branch) + trunk, manifests, nodes, moduleGraph, served, offering := bv.trunk, bv.manifests, bv.nodes, bv.graph, bv.served, bv.offering + fp := trunk[name].Fp2 + name := name + nodesOf := map[string][]string{name: g.nodes} + wantedOf := map[string]map[string]mwc.Want{name: wantedHere(wanted[name], g.nodes, groups, gi == 0)} + // A want follows a passed pipeline: a module whose fingerprint no pipeline passed at on the branch its placements + // chose, and whose wants do not name it already, gets none (design 47 G §29, §35). + var branches []string + if p.ends.following() { + branches = []string{g.branch} } - } - if m.Released { - p.clearHeld(name, fp) - } - } - if served != nil { - m.Invokes, m.Served = keptInvokes(moduleGraph, name), served - } - m.Offering = offering - last := map[string]string{} - for _, node := range m.Nodes { - last[node] = p.w.lastHealthy(node, name, fp) - } - out := assign.Wants(m, nodes, wanted[name], last) - for _, node := range m.Nodes { - for _, prov := range m.MissingProvisions(node) { - missing[node+"."+name+"."+prov] = true - } - } - p.yeses(ctx, name, m, wanted[name], &out) - gatherNeither(neither, name, out) - leftNeither(left, m, wanted[name], out) - p.sayNoWant(name, m.Nodes, wanted[name], out) - p.sayUnreached(name, m.Nodes, out) - p.sayNodeFailed(name, fp, m.Nodes, nodes, out, first) - if first { - startClears(stale, m, out) - if fp != "" && !m.Held && out.Held == nil && !out.Lifted { - p.clearHeld(name, fp) - } - } - if out.Held != nil { - key, err := mwc.HeldKey(name, fp) - if err == nil { - held := *out.Held - p.put(ctx, buckets.Wants, key, func() ([]byte, error) { return mwc.EncodeHeld(held) }) - } - } - if out.Lifted { - p.lift(ctx, name, fp) - } - if out.Held != nil || (m.Held && !out.Lifted) { - p.holdAsk(ctx, name, fp, out.Held) - p.heldCondition(ctx, name, fp, out.Held, len(out.Restore) > 0) - } - for _, node := range out.Restore { - p.restoreAsk(ctx, name, node, fp) - } - for _, need := range out.Record { - p.recordAsk(ctx, name, fp, need) - } - for node, want := range out.Wants { - key, err := mwc.WantKey(node, name) - if err != nil { - p.logf("no want of %s on %s is written: %v", name, node, err) - continue - } - want := want - p.put(ctx, buckets.Wants, key, func() ([]byte, error) { return mwc.EncodeWant(want) }) - } - if len(out.Wants) > 0 { - p.ends.wrote(name, fp, branches) + if fp != "" && !p.ends.mayWant(name, fp, branches, wantedOf[name]) { + return + } + m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Needs: trunk[name].Needs, Chosen: chosenOf(chosen, name), Provisions: provisionsOf(name), Answering: answering(manifests, nodes), Offered: offered(manifests), Nodes: nodesOf[name], Now: time.Now(), Puts: puts[name], FormNeeds: p.w.buildNeeds(name, fp), + Self: func(node string) bool { return placed[node] }, + Composed: func(node, fp string) (assign.ComposedDigest, bool) { + return p.w.composed(node, name, fp) // a placed node's own: the digest its own have names (ADR 0571 §6) + }, + GrantedAt: func(fp string) bool { + key, err := grantsstate.PartGrantKey(name, fp) + return err == nil && p.w.has(grants, key) + }, + Restored: func(node string) bool { + key, err := asks.RestoreAskKey(node, name, fp) + return err == nil && p.answeredWith(key, assign.OptionRestore) + }, + Allowed: func(node, digest string) bool { + key, err := asks.RecordAskKey(node, name, digest) + return err == nil && p.answeredWith(key, assign.OptionYes) + }, + Release: func(fp string) (string, bool) { return p.w.release(name, fp) }} + if fp != "" { + held, _ := mwc.HeldKey(name, fp) + grant, _ := grantsstate.PartGrantKey(name, fp) + hold, _ := asks.HoldAskKey(name, fp) + m.Released = p.answeredWith(hold, "release") + m.Kept = p.answeredWith(hold, "keep") + m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(buckets.Wants, held) && !m.Released, p.w.has(grants, grant) + if raw := p.w.value(grants, grant); raw != nil { + if g, err := grantsstate.DecodeGrant(raw); err == nil { + m.GrantedOn = func(node string) bool { return assign.GrantNamesNode(g, node, name) } + } + } + if m.Released { + p.clearHeld(name, fp) + } + } + if served != nil { + m.Invokes, m.Served = keptInvokes(moduleGraph, name), served + } + m.Offering = offering + last := map[string]string{} + for _, node := range m.Nodes { + last[node] = p.w.lastHealthy(node, name, fp) + } + out := assign.Wants(m, nodes, wantedOf[name], last) + for _, node := range m.Nodes { + for _, prov := range m.MissingProvisions(node) { + missing[node+"."+name+"."+prov] = true + } + } + p.yeses(ctx, name, m, wantedOf[name], &out) + gatherNeither(neither, name, out) + leftNeither(left, m, wantedOf[name], out) + p.sayNoWant(name, m.Nodes, wantedOf[name], out) + p.sayUnreached(name, m.Nodes, out) + p.sayNodeFailed(name, fp, m.Nodes, nodes, out, first) + if first { + startClears(stale, m, out) + if fp != "" && !m.Held && out.Held == nil && !out.Lifted { + p.clearHeld(name, fp) + } + } + if out.Held != nil { + key, err := mwc.HeldKey(name, fp) + if err == nil { + held := *out.Held + p.put(ctx, buckets.Wants, key, func() ([]byte, error) { return mwc.EncodeHeld(held) }) + } + } + if out.Lifted { + p.lift(ctx, name, fp) + } + if out.Held != nil || (m.Held && !out.Lifted) { + p.holdAsk(ctx, name, fp, out.Held) + p.heldCondition(ctx, name, fp, out.Held, len(out.Restore) > 0) + } + for _, node := range out.Restore { + p.restoreAsk(ctx, name, node, fp) + } + for _, need := range out.Record { + p.recordAsk(ctx, name, fp, need) + } + for node, want := range out.Wants { + key, err := mwc.WantKey(node, name) + if err != nil { + p.logf("no want of %s on %s is written: %v", name, node, err) + continue + } + want := want + p.put(ctx, buckets.Wants, key, func() ([]byte, error) { return mwc.EncodeWant(want) }) + } + if len(out.Wants) > 0 { + p.ends.wrote(name, fp, branches) + } + }() } } } @@ -1313,3 +1324,82 @@ func answering(manifests map[string]catalogue.Manifest, n assign.Nodes) func(str return out } } + +// branchView is what one pass reads of the catalog half of one watched branch (design 47 G §184, §195): its modules, their +// manifests, the nodes the rules read through them, the module graph and the two readings of what runs. +type branchView struct { + trunk map[string]meshcatalog.ModulesByModuleValue + manifests map[string]catalogue.Manifest + nodes assign.Nodes + graph graph.Graph + served func(string) bool + offering func(string) bool +} + +// branchView reads the catalog half of branch for this pass. +func (p *passer) branchView(branch string, assigned map[string][]string) *branchView { + v := &branchView{trunk: p.w.trunkOf(branch), manifests: map[string]catalogue.Manifest{}} + for name, mod := range v.trunk { + if mod.Manifest == "" { + continue + } + if got := p.w.manifest(mod.Manifest); got.err == "" { + v.manifests[name] = got.manifest + } + } + v.nodes = p.w.nodes(v.manifests, assigned) + v.graph = p.moduleGraph(branch) + v.served = p.served(v.graph, v.nodes) + v.offering = p.offering(v.manifests, v.nodes) // bus state (ADR 0614 §5) + return v +} + +// group is the nodes of one module whose placements name one watched branch. +type group struct { + branch string + nodes []string +} + +// groups splits the nodes of module by the branch their placements name, the branches in name order (the trunk's first); a +// module placed nowhere is one group on its default branch. +func (p *passer) groups(module string, nodes []string, placed map[string]string) []group { + by := map[string][]string{} + for _, node := range nodes { + b := trunkBranch + if p.ends.following() { + b = p.ends.branchOf(module, placed[node]) + } + by[b] = append(by[b], node) + } + if len(by) == 0 { + b := trunkBranch + if p.ends.following() { + b = p.ends.branchOf(module, "") + } + by[b] = nil + } + var out []group + for b, ns := range by { + out = append(out, group{b, ns}) + } + sort.Slice(out, func(i, j int) bool { return out[i].branch < out[j].branch }) + return out +} + +// wantedHere is the wants of a module a group answers for: those of its own nodes, and (the first group only) those of nodes no +// group holds, which the rules take away. +func wantedHere(wanted map[string]mwc.Want, own []string, groups []group, first bool) map[string]mwc.Want { + held := map[string]bool{} + for _, g := range groups { + for _, n := range g.nodes { + held[n] = true + } + } + out := map[string]mwc.Want{} + for node, w := range wanted { + if slices.Contains(own, node) || (first && !held[node]) { + out[node] = w + } + } + return out +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go index 716ac50c..93654d0d 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go @@ -558,7 +558,7 @@ func putOfTheBus(t *testing.T) { placeOn(t, p.w, "novox", "nats") mdigest := strings.Repeat("e", 64) mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: mdigest, Repository: "novox/mesh-catalog", Commit: strings.Repeat("c", 40)}) - world(buckets.Modules, "nats", mod) + putModuleRaw(p.w, trunkBranch, "nats", mod) p.w.manifests[mdigest] = parsed{module: "nats", manifest: catalogue.Manifest{Module: "nats", Provides: []catalogue.Offer{{Name: "mesh-bus"}}}} bkey, _ := builds.BuilderBuildKey("nats", fp) built, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: mdigest}) @@ -901,7 +901,7 @@ func TestTheFirstPassAfterAStartClearsWhatAPreviousProcessRaised(t *testing.T) { var said []oc.Condition p.say = func(c oc.Condition) error { said = append(said, c); return nil } mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fpTest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath}) - p.w.put(buckets.Modules, "web", mod) + putModuleRaw(p.w, trunkBranch, "web", mod) placeOn(t, p.w, "g14", "web") on, _ := presenceValue(true) p.w.put(buckets.Presence, "g14", on) @@ -939,7 +939,7 @@ func testAHeldModuleRaisesNoNeitherAndClearsAStandingOne(t *testing.T) { said := &[]oc.Condition{} p.say = func(c oc.Condition) error { *said = append(*said, c); return nil } mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fpTest, Repository: "novox/resolved", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath}) - p.w.put(buckets.Modules, "systemd-resolved", mod) + putModuleRaw(p.w, trunkBranch, "systemd-resolved", mod) placeOn(t, p.w, "shanks", "systemd-resolved") on, _ := presenceValue(true) p.w.put(buckets.Presence, "shanks", on) @@ -1198,7 +1198,7 @@ func aReleasesApproveAnswersTheMoveOfItsBuildAlone(t *testing.T) { p := newPasser(kvSet{buckets.Asks: asksKV}, time.Now()) p.logf = t.Logf mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fpTest, Repository: "novox/web", Commit: commit, Src: 7, FpSrc: c.fpSrc, Path: meshcatalogclient.RootPath}) - p.w.put(buckets.Modules, "web", mod) + putModuleRaw(p.w, trunkBranch, "web", mod) src, _ := mdc.EncodeSource(mdseat.SourceByBranchValue{Group: "feat/web", Heads: map[string]string{"novox/web": commit}}) p.w.putEntry(buckets.Source, mdc.SourceKey(trunkBranch), src, 7, time.Now()) later, _ := mdc.EncodeSource(mdseat.SourceByBranchValue{Group: "feat/other", Heads: map[string]string{"novox/web": commit, "novox/x": commit}}) @@ -1261,7 +1261,7 @@ func aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t *testing.T) { p.logf = t.Logf put := func(bucket, key string, v []byte) { p.w.put(bucket, key, v) } mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fpTest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath}) - put(buckets.Modules, "web", mod) // the trunk's build is not done: rule 1 holds g14 at fpOld + putModuleRaw(p.w, trunkBranch, "web", mod) // the trunk's build is not done: rule 1 holds g14 at fpOld placeOn(t, p.w, "g14", "web") pl, _ := mmac.EncodeAssignments(mma.Assignments{Node: "g14", Modules: []mma.PlacedModule{{Module: "web"}}}) put(plc.AssignmentsBucket, "assignments.g14", pl) @@ -1380,7 +1380,7 @@ func TestAnApprovedAssignmentsCallsNoControllerVerb(t *testing.T) { func aWantWaitsOnTheVerbEdgesTheGraphKeeps(t *testing.T) { p := newPasser(kvSet{}, time.Now()) fp := func(c string) string { return meshcatalogclient.SourcePrefix2 + strings.Repeat(c, 64) } - if p.served(context.Background(), nil, graph.Graph{}, assign.Nodes{}) != nil { + if p.served(graph.Graph{}, assign.Nodes{}) != nil { t.Error("with no graph read, a want waits on a verb") } for module, n := range map[string]graph.Node{ @@ -1394,26 +1394,25 @@ func aWantWaitsOnTheVerbEdgesTheGraphKeeps(t *testing.T) { if err != nil { t.Fatal(err) } - p.w.put(buckets.Graph, module, raw) + _ = raw + p.w.putModule(trunkBranch, module, meshcatalog.ModulesByModuleValue{}, n, nil) } - g := p.moduleGraph() + g := p.moduleGraph(trunkBranch) if got := keptInvokes(g, "caller"); !slices.Equal(got, []string{"seat:x.v"}) || keptInvokes(g, "unknown") != nil { t.Errorf("the kept verb edges of caller = %v", got) } digest := strings.Repeat("d", 64) - man, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: "server", Manifest: `{"module":"server","claims":[{"name":"x","scope":"mesh","serves":["w"]}]}`}) - get := func(_ context.Context, key string) ([]byte, error) { - if key == digest { - return man, nil - } - return nil, errNotThere + parsedServer, err := catalogue.Decode([]byte(`{"module":"server","claims":[{"name":"x","scope":"mesh","serves":["w"]}]}`)) + if err != nil { + t.Fatal(err) } + p.w.manifests[digest] = parsed{module: "server", manifest: parsedServer} build, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: digest}) key, _ := builds.BuilderBuildKey("server", fp("f")) p.w.put(buckets.Builds, key, build) running := nodeengine.HaveByNodeModuleValue{Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} at := func(f string) nodeengine.HaveByNodeModuleValue { h := running; h.Fp = f; return h } - served := p.served(context.Background(), get, g, assign.Nodes{Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{ + served := p.served(g, assign.Nodes{Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{ "g14": {"server": at(fp("f")), "late": at(fp("2"))}}}) for v, want := range map[string]bool{"seat:x.w": true, "seat:x.v": false, "late.t": true} { if served(v) != want { @@ -1424,7 +1423,7 @@ func aWantWaitsOnTheVerbEdgesTheGraphKeeps(t *testing.T) { // offline node offers nothing, and one whose manifest is not read counts its trunk manifest's offer. bus := map[string]catalogue.Manifest{"bus": {Module: "bus", Provides: []catalogue.Offer{{Name: graph.BusState, Scope: catalogue.ScopeMesh}}}} for _, online := range []bool{false, true} { - offering := p.offering(context.Background(), get, bus, assign.Nodes{Online: map[string]bool{"n2": online}, + offering := p.offering(bus, assign.Nodes{Online: map[string]bool{"n2": online}, Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{"n2": {"bus": at(fp("2"))}}}) if offering(graph.BusState) != online { t.Errorf("with the bus module's node online %v, bus-state offered %v", online, offering(graph.BusState)) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go index 5e9ab67e..efddeb7a 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go @@ -151,6 +151,8 @@ type pipelineEnds struct { // firsts is, by module, the first watched branch of its registration, the branch of a placement that names none (design // 47 G §14). firsts map[string]string + // onBranches is told the branches a registration names, for the catalog half of each to be followed (design 47 G §14, §195). + onBranches func([]string) } func newPipelineEnds(st endStore, wake func(), logf func(string, ...any)) *pipelineEnds { @@ -379,6 +381,9 @@ func (e *pipelineEnds) registered(reg mpcat.Registration, gone bool) { e.logf("the end %s is not deleted: %v", k, err) } } + if e.onBranches != nil && !gone { + e.onBranches(reg.Branches) + } if !e.on.Swap(true) { e.logf("the first registration is heard (%s): the wants follow the pipelines' ends from now", reg.Module) } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines_test.go index acb0c207..db115598 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines_test.go @@ -3,6 +3,11 @@ package main import ( "context" "encoding/json" + "fmt" + mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient" + mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" + plc "mesh-delivery-assigner/internal/assignment" + "slices" "strings" "testing" "time" @@ -147,7 +152,7 @@ func TestAPassWritesNoWantWithoutAPassedPipeline(t *testing.T) { p.say = func(oc.Condition) error { return nil } mdigest := strings.Repeat("e", 64) mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: mdigest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath}) - p.w.put(buckets.Modules, "web", mod) + putModuleRaw(p.w, trunkBranch, "web", mod) p.w.manifests[mdigest] = parsed{module: "web", manifest: catalogue.Manifest{Module: "web"}} bkey, _ := builds.BuilderBuildKey("web", fp) built, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: mdigest}) @@ -211,3 +216,101 @@ func TestTheWantsSwitchOnTheFirstRegistration(t *testing.T) { t.Errorf("a registration gone left %d ends", len(st.vals)) } } + +// Design 47 G §14, §184 (test 68): a node's entry is computed from the catalog half of the branch its placement names, so a +// change on `beta` moves only the want of the node whose placement chose `beta`. +func TestACommitOnBetaMovesOnlyTheNodeThatChoseBeta(t *testing.T) { + fp := func(c string) string { return meshcatalogclient.SourcePrefix2 + strings.Repeat(c, 64) } + wantsKV := &memKV{values: map[string][]byte{}} + p := newPasser(kvSet{buckets.Wants: wantsKV, buckets.Asks: &memKV{values: map[string][]byte{}}}, time.Now()) + p.say = func(oc.Condition) error { return nil } + p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf) + p.ends.registered(mpcat.Registration{Module: "web", Branches: []string{"main", "beta"}}, false) + digest := "sha256:" + strings.Repeat("c", 64) + on, _ := presenceValue(true) + for _, node := range []string{"g14", "ace"} { + composes(t, p.w, node, digest) + p.w.put(buckets.Presence, node, on) + } + for node, branch := range map[string]string{"g14": "", "ace": "beta"} { + pl := mma.Assignments{Node: node, Modules: []mma.PlacedModule{{Module: "web", Branch: branch}}} + values, err := mmac.AssignmentsValues(pl) + if err != nil { + t.Fatal(err) + } + for key, raw := range values { + p.w.put(plc.AssignmentsBucket, key, raw) + } + } + // publish makes the half of branch name web at fingerprint f: built, granted and passed by its pipeline. + publish := func(branch, f string, source int64) { + mdigest := strings.Repeat("e", 62) + fmt.Sprintf("%02d", source) + if branch == "beta" { + mdigest = strings.Repeat("f", 62) + fmt.Sprintf("%02d", source) + } + mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: f, Manifest: mdigest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath}) + putModuleRaw(p.w, branch, "web", mod) + p.w.manifests[mdigest] = parsed{module: "web", manifest: catalogue.Manifest{Module: "web"}} + bkey, _ := builds.BuilderBuildKey("web", f) + built, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: mdigest}) + p.w.put(buckets.Builds, bkey, built) + gkey, _ := grantsstate.PartGrantKey("web", f) + grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{}) + p.w.put(buckets.Grants, gkey, grant) + _ = p.ends.take(pipe("web", branch, source, 1, f, mpr.PipelineEndedOutcomePassed), true) + } + run := func(node, f string) { + have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: f, Composed: digest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}) + p.w.put(buckets.Have, node+".web", have) + } + fpWant := func(node string) string { + key, _ := mwc.WantKey(node, "web") + w, err := mwc.DecodeWant(wantsKV.values[key]) + if err != nil { + t.Fatalf("the want of web on %s: %v", node, err) + } + return w.Fp + } + publish("main", fp("a"), 5) + publish("beta", fp("b"), 5) + run("g14", fp("a")) + run("ace", fp("b")) + p.wants(context.Background()) + if fpWant("g14") != fp("a") || fpWant("ace") != fp("b") { + t.Fatalf("each node is wanted at its branch's build: g14 %s, ace %s", fpWant("g14"), fpWant("ace")) + } + publish("beta", fp("d"), 6) + p.wants(context.Background()) + if fpWant("g14") != fp("a") || fpWant("ace") != fp("d") { + t.Errorf("a commit on beta moves only ace: g14 %s (want main's), ace %s (want beta's new)", fpWant("g14"), fpWant("ace")) + } +} + +// Design 47 G §14, §195: the catalog half of each branch is taken into the world per branch, an entry as the value and graph node +// the rules read, and a registration's branches are each followed. +func TestTheCatalogHalfIsTakenInPerBranch(t *testing.T) { + w := newWorld() + woken := 0 + h := &halves{w: w, wake: func() { woken++ }, logf: t.Logf} + entry := func(fp string) json.RawMessage { + raw, _ := json.Marshal(meshcatalog.CatalogEntry{Module: "web", Fp: fp, Repository: "novox/web", Commit: "c1", Manifest: map[string]any{"module": "web", "version": "1"}, + Invokes: []string{"seat:x.v"}}) + return raw + } + h.changed("main", "web", entry("src2:1"), false) + h.changed("beta", "web", entry("src2:2"), false) + if w.trunkOf("main")["web"].Fp2 != "src2:1" || w.trunkOf("beta")["web"].Fp2 != "src2:2" || woken != 2 { + t.Errorf("main %+v, beta %+v, woken %d: each branch keeps its own entry", w.trunkOf("main")["web"], w.trunkOf("beta")["web"], woken) + } + v := w.trunkOf("beta")["web"] + if got := w.manifest(v.Manifest); got.err != "" || got.manifest.Module != "web" { + t.Errorf("the entry's manifest is not read by its digest: %+v", got) + } + if n := w.graphOf("main")["web"]; n.FP != "src2:1" || !slices.Equal(n.Invokes, []string{"seat:x.v"}) { + t.Errorf("graph node %+v", n) + } + h.changed("beta", "web", nil, true) + if _, ok := w.trunkOf("beta")["web"]; ok || w.trunkOf("main")["web"].Fp2 == "" { + t.Error("a module taken out of beta's half leaves main's") + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go index 0322e53c..75ffe180 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go @@ -112,7 +112,6 @@ func (p *passer) puts(ctx context.Context) { } assigned := p.w.assigned() trunk := p.w.trunk() - get := p.w.getter(buckets.Manifests) // the indexer's manifests, as its first view and views family say them wanted := p.w.wantsOf() modules := make([]string, 0, len(all)) for m := range all { @@ -122,7 +121,7 @@ func (p *passer) puts(ctx context.Context) { for _, module := range modules { var manifest catalogue.Manifest if mod, ok := trunk[module]; ok && mod.Manifest != "" { - if got := p.w.manifest(ctx, get, mod.Manifest); got.err == "" { + if got := p.w.manifest(mod.Manifest); got.err == "" { manifest = got.manifest } } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat.go index 8efd714f..fb1a8330 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat.go @@ -16,7 +16,6 @@ import ( "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" mdcc "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliverycheckerclient" mdc "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks" @@ -465,10 +464,8 @@ func (p *passer) layAgain(ctx context.Context, prop mma.Proposal, rev uint64, cu // nodes with a record. func (p *passer) lays(ctx context.Context) (func(string) bool, func(string) bool) { held := map[string]bool{} - for key := range p.w.read(buckets.Modules) { - if name, err := meshcatalogclient.ParseModulesKey(key); err == nil { - held[name] = true - } + for name := range p.w.trunk() { + held[name] = true } records := p.w.records() return func(m string) bool { return held[m] }, func(n string) bool { _, ok := records[n]; return ok } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/verbs.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/verbs.go index bc3f55e6..9fa5ea2f 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/verbs.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/verbs.go @@ -1,7 +1,6 @@ package main import ( - "context" "mesh-delivery-assigner/internal/buckets" "slices" @@ -20,21 +19,8 @@ import ( // each kept edge until a build serving its verb runs healthy, read from the haves, the builder's builds and the indexer's // manifests, as a provider is counted (ADR 0603 §5). -// moduleGraph is the module graph as the indexer's first view and views family last said it (ADR 0631); a key that -// does not decode is left out, so its module waits on no verb. -func (p *passer) moduleGraph() graph.Graph { - g := graph.Graph{} - for key, raw := range p.w.read(buckets.Graph) { - module, err := graph.ParseKey(key) - if err != nil { - continue - } - if n, err := graph.Decode(raw); err == nil { - g[module] = n - } - } - return g -} +// moduleGraph is the module graph of the catalog half of branch, from the entries' invokes and serves (design 47 G §184, §195). +func (p *passer) moduleGraph(branch string) graph.Graph { return p.w.graphOf(branch) } // keptInvokes are the verbs of module the graph keeps as edges: each it invokes and does not leave out, that a module // other than itself serves. A verb no module of the graph serves is no edge, so it holds nothing; nil when the graph holds @@ -61,11 +47,11 @@ func keptInvokes(g graph.Graph, module string) []string { // serves are served. A healthy build whose manifest cannot be read counts what the graph says its module serves, said in // the log, so a build read late never holds its callers for good; nil when the graph holds nothing, and no want waits on // a verb. -func (p *passer) served(ctx context.Context, get getter, g graph.Graph, n assign.Nodes) func(string) bool { +func (p *passer) served(g graph.Graph, n assign.Nodes) func(string) bool { if len(g) == 0 { return nil } - served, unread := graph.Served(n.Haves, func(module, fp string) ([]string, bool) { return p.servesOf(ctx, get, module, fp) }, + served, unread := graph.Served(n.Haves, func(module, fp string) ([]string, bool) { return p.servesOf(module, fp) }, func(module string) []string { return g[module].Serves }) if len(unread) > 0 { p.logf("the manifest of the running build of %v cannot be read, so what the module graph says they serve is counted as served", unread) @@ -77,7 +63,7 @@ func (p *passer) served(ctx context.Context, get getter, g graph.Graph, n assign // §5, ADR 0614 §5): every running healthy have on an online node, its build's manifest read as servesOf reads it. A // healthy build whose manifest cannot be read counts what its module's trunk manifest offers, so a build read late never // holds a consumer for good. -func (p *passer) offering(ctx context.Context, get getter, trunk map[string]catalogue.Manifest, n assign.Nodes) func(string) bool { +func (p *passer) offering(trunk map[string]catalogue.Manifest, n assign.Nodes) func(string) bool { offered := map[string]bool{} for node, haves := range n.Haves { if !n.Online[node] { @@ -87,7 +73,7 @@ func (p *passer) offering(ctx context.Context, get getter, trunk map[string]cata if h.Phase != nodeengineclient.PhaseRunning || h.Health != nodeengineclient.HealthOK { continue } - m, ok := p.manifestOf(ctx, get, module, h.Fp) + m, ok := p.manifestOf(module, h.Fp) if !ok { m = trunk[module] } @@ -100,8 +86,8 @@ func (p *passer) offering(ctx context.Context, get getter, trunk map[string]cata } // servesOf are the verbs the manifest of module's build at fp serves (graph.ServesOf), and whether it could be read. -func (p *passer) servesOf(ctx context.Context, get getter, module, fp string) ([]string, bool) { - m, ok := p.manifestOf(ctx, get, module, fp) +func (p *passer) servesOf(module, fp string) ([]string, bool) { + m, ok := p.manifestOf(module, fp) if !ok { return nil, false } @@ -110,7 +96,7 @@ func (p *passer) servesOf(ctx context.Context, get getter, module, fp string) ([ // manifestOf is the manifest of module's build at fp (the builder's build names its digest in the indexer's manifests), // and whether it could be read. -func (p *passer) manifestOf(ctx context.Context, get getter, module, fp string) (catalogue.Manifest, bool) { +func (p *passer) manifestOf(module, fp string) (catalogue.Manifest, bool) { key, err := builds.BuilderBuildKey(module, fp) if err != nil { return catalogue.Manifest{}, false @@ -123,7 +109,7 @@ func (p *passer) manifestOf(ctx context.Context, get getter, module, fp string) if err != nil || b.Manifest == "" { return catalogue.Manifest{}, false } - got := p.w.manifest(ctx, get, b.Manifest) + got := p.w.manifest(b.Manifest) if got.err != "" { return catalogue.Manifest{}, false } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/views.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/views.go index e1b6659f..4186a832 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/views.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/views.go @@ -6,6 +6,7 @@ import ( "mesh-delivery-assigner/internal/assignment" "mesh-delivery-assigner/internal/buckets" "slices" + "sync" "time" "github.com/nats-io/nats.go/jetstream" @@ -20,7 +21,8 @@ import ( sdkfindings "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient/findings" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient/heard" - "git.novox.be/novox/mesh-delivery/sdk/graph" + mcat "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "mesh-delivery-assigner/internal/assign" plc "mesh-delivery-assigner/internal/assignment" @@ -126,10 +128,11 @@ func viewFindings(f assign.Findings) views.FindingWriter { // firstViewsBucket is the assigner's own state of its first views (views.StatusState). var firstViewsBucket = assignment.Module + "_" + views.StatusState -// indexerViews are the assigner's first views of the indexer's manifests, modules and module graph (ADR 0615 §1, -// ADR 0621 §2, ADR 0631; DLV-F2r): asked at the start through the indexer's tools, then followed on indexer-views, asking once more on a gap; -// past the bound its finding names the indexer and the tool, cleared by the answer. -type indexerViews struct { +// catalogViews is the assigner's first view of the catalog half of each watched branch (design 47 G §14, §184, §195): asked +// at the start through the seat mesh-catalog's verb `revision` (the trunk's, then each branch a registration names), then +// followed on its event `catalog-revision`, asking once more on a gap; past the bound its finding names the seat and the verb, +// cleared by the answer. Its status is read by assigner_first_views and asked again by assigner_first_view_again. +type catalogViews struct { js jetstream.JetStream account process.Account call func(string, any) (json.RawMessage, error) @@ -137,21 +140,17 @@ type indexerViews struct { status *process.AwaitedBucket logf func(string, ...any) hub *process.EventHub -} -// indexerFirstViews are the indexer's states the assigner takes by first view and the views family, each by the name of -// the indexer's bucket the world keeps it under: manifests and modules (ADR 0621 §2), and the module graph, whose verb -// edges a want waits on (ADR 0614 §2, §3; ADR 0631 §1). -var indexerFirstViews = map[string]string{"manifests": buckets.Manifests, "modules": buckets.Modules, graph.State: buckets.Graph} + h *halves +} // firstViewBound and firstViewWait are each first view's bound before its condition is raised, and its first wait // between two failed asks; zero is the views package's (views.Bound, two minutes, ADR 0615 §6; and a first wait of five // seconds). A test shortens them. var firstViewBound, firstViewWait time.Duration -// start asks the three first views, each answer taken into w under the indexer's bucket names and settled once (a pass waits -// for it), and hears the views family; it answers how many settles to wait for. -func (iv *indexerViews) start(ctx context.Context, w *world, wake func(), settled chan<- struct{}) int { +// start follows the trunk's catalog half and hears the seat's catalog-revision; it answers how many settles to wait for. +func (iv *catalogViews) start(ctx context.Context, w *world, wake func(), settled chan<- struct{}) int { finding := func(s views.Status, raised bool) { if iv.findings.Put == nil { return @@ -167,33 +166,28 @@ func (iv *indexerViews) start(ctx context.Context, w *world, wake func(), settle iv.logf("views: the finding of the first view %s is not written: %v", s.Tool, err) } } - followers := map[string]*views.Follower{} - var all []*views.Follower - for state, bucket := range indexerFirstViews { - bucket := bucket - tool := buckets.IndexerModule + "." + state - f := &views.Follower{Owner: buckets.IndexerModule, Tool: tool, State: state, Ask: views.AskVia(iv.call, tool), Logf: iv.logf, - Bound: firstViewBound, Wait: firstViewWait, - Raise: func(s views.Status) { finding(s, true) }, Clear: func(s views.Status) { finding(s, false) }, - Changed: func(key string, value json.RawMessage, deleted bool, at time.Time) { - if deleted { - value = nil - } - w.putEntry(bucket, key, value, 0, at) - wake() - }} - followers[state] = f - all = append(all, f) - go func() { - if f.Start(ctx) == nil { - settled <- struct{}{} - } - }() + var statusKV jetstream.KeyValue + var mu sync.Mutex + iv.h = &halves{w: w, call: iv.call, wake: wake, logf: iv.logf, ctx: ctx, settled: settled} + iv.h.started = func(c *mcat.Catalog) { + f := c.Follower + f.Bound, f.Wait = firstViewBound, firstViewWait + f.Raise, f.Clear = func(s views.Status) { finding(s, true) }, func(s views.Status) { finding(s, false) } + mu.Lock() + defer mu.Unlock() + if statusKV != nil { + f.Said = views.KeepStatus(ctx, statusKV, iv.logf) + } } if iv.status != nil { iv.status.Ready(func(kv jetstream.KeyValue) { - for _, f := range all { - f.Said = views.KeepStatus(ctx, kv, iv.logf) + mu.Lock() + statusKV = kv + mu.Unlock() + var all []*views.Follower + for _, c := range iv.h.followers() { + c.Follower.Said = views.KeepStatus(ctx, kv, iv.logf) + all = append(all, c.Follower) } if err := views.HearAgain(ctx, kv, all, iv.logf); err != nil { iv.logf("views: assigner_first_view_again is not heard (%v) until the assigner starts again", err) @@ -202,16 +196,17 @@ func (iv *indexerViews) start(ctx context.Context, w *world, wake func(), settle } // The hub says when the bus made its consumer: a picture asked before then is asked again (views.Follower.ConsumerMade). iv.hub.OnBound(func(created time.Time) { - for _, f := range all { - f.ConsumerMade(ctx, created) + for _, c := range iv.h.followers() { + c.Follower.ConsumerMade(ctx, created) } }) - // Not heard, the indexer's views family is the finding family-not-heard until it binds (ADR 0615 §5): the handler is - // registered on the module's hub, which the assigner starts once every handler is. - _ = notHeardWriter(iv.findings, iv.logf).Bind(process.EventsStream, "the indexer's manifests, modules and graph are the first views' alone until the assigner starts again", func() error { - return iv.hub.Handle("mesh.mod."+buckets.IndexerModule+".event."+views.EventPrefix+"*", views.Hear(ctx, followers, iv.logf)) + // Not heard, the catalog-revision is the finding family-not-heard until it binds (ADR 0615 §5): the handler is registered + // on the module's hub, which the assigner starts once every handler is. + _ = notHeardWriter(iv.findings, iv.logf).Bind(process.EventsStream, "the catalog half is the first views' alone until the assigner starts again", func() error { + return iv.hub.Handle(meshcatalog.SubjectCatalogRevisionAll, iv.h.hear) }) - return len(all) + iv.h.follow(trunkBranch) + return 1 } // notHeardWriter writes the assigner's finding of a family it does not hear into its findings (heard.Writer); with no diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go index be881eb6..876e04e0 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go @@ -1,7 +1,6 @@ package main import ( - "context" "encoding/json" "errors" "maps" @@ -14,16 +13,14 @@ import ( "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" wdc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshwatchdogsclient" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" mdc "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" mwc "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshwantsclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "git.novox.be/novox/mesh-delivery/sdk/graph" "git.novox.be/novox/mesh-delivery/sdk/trunk" "git.novox.be/novox/mesh-sdk/go/keys" - "github.com/nats-io/nats.go/jetstream" - mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient" mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" "git.novox.be/novox/mesh-sdk/go/catalogue" @@ -56,6 +53,10 @@ type world struct { // durable consumer (design 47 G §199, §309); runningHeard says that consumer is bound. running map[string]nodeengine.Running runningHeard bool + // modules and graphs are, by watched branch and module, the catalog half the assigner follows per branch (design 47 G §14, + // §184, §195), as the value and the graph node the rules read: written by putModule from the entries of meshcatalogclient.Catalog. + modules map[string]map[string]meshcatalog.ModulesByModuleValue + graphs map[string]map[string]graph.Node } // entryMeta is a value's revision in its bucket and when it was written. @@ -74,7 +75,8 @@ type parsed struct { func newWorld() *world { return &world{values: map[string]map[string][]byte{}, manifests: map[string]parsed{}, healthy: map[string][]string{}, changes: map[string]uint64{}, meta: map[string]entryMeta{}, trunkBuilt: map[string]builds.BuildLogEntry{}, assignmentKeys: mmac.NewKeys(), - sourceGroups: map[uint64]string{}, running: map[string]nodeengine.Running{}} + sourceGroups: map[uint64]string{}, running: map[string]nodeengine.Running{}, + modules: map[string]map[string]meshcatalog.ModulesByModuleValue{}, graphs: map[string]map[string]graph.Node{}} } // putEntry is put, keeping the value's revision and the time it was written. @@ -194,71 +196,16 @@ func (w *world) read(bucket string) map[string][]byte { return out } -// getter reads one key's value; errNotThere when the key does not exist. -type getter func(ctx context.Context, key string) ([]byte, error) - var errNotThere = errors.New("no such key") -// kvGetter reads keys of a bucket. -func kvGetter(kv jetstream.KeyValue) getter { - return func(ctx context.Context, key string) ([]byte, error) { - e, err := kv.Get(ctx, key) - if errors.Is(err, jetstream.ErrKeyNotFound) { - return nil, errNotThere - } - if err != nil { - return nil, err - } - return e.Value(), nil - } -} - -// getter reads a key of a bucket the world holds: errNotThere where it holds none. -func (w *world) getter(bucket string) getter { - return func(_ context.Context, key string) ([]byte, error) { - w.mu.Lock() - defer w.mu.Unlock() - v, ok := w.values[bucket][key] - if !ok { - return nil, errNotThere - } - return v, nil - } -} - -// manifest reads the manifest of a digest from the indexer's bucket once and keeps it: a digest's manifest never changes. -func (w *world) manifest(ctx context.Context, get getter, digest string) parsed { +// manifest is the manifest of a digest the catalog half said (putModule keeps each once); one not said yet is not readable. +func (w *world) manifest(digest string) parsed { w.mu.Lock() - p, ok := w.manifests[digest] - w.mu.Unlock() - if ok { + defer w.mu.Unlock() + if p, ok := w.manifests[digest]; ok { return p } - key, err := meshcatalogclient.ManifestsKey(digest) - if err != nil { - return parsed{err: err.Error()} - } - value, err := get(ctx, key) - if errors.Is(err, errNotThere) { - return parsed{err: "the indexer holds no manifest of digest " + digest + " yet"} - } - if err != nil { - return parsed{err: err.Error()} - } - m, err := meshcatalogclient.DecodeManifest(value) - if err != nil { - p = parsed{err: err.Error()} - } else if man, err := catalogue.Decode([]byte(m.Manifest)); err != nil { - // Decoded, never judged (novox/hq ADR 0578 §3): the indexer's judge judged it, and its verdict is taken from the - // module's value where it is read (world.mesh). - p = parsed{module: m.Module, err: "it does not decode: " + err.Error()} - } else { - p = parsed{module: m.Module, manifest: man} - } - w.mu.Lock() - w.manifests[digest] = p - w.mu.Unlock() - return p + return parsed{err: "the catalog half holds no manifest of digest " + digest + " yet"} } // assigned is each node's modules the seat mesh-module-assigner's assignments names, in name order (ADR 0549 §5, ADR 0567 §8): the @@ -299,21 +246,74 @@ func (w *world) placedNodes() map[string]bool { return out } -// trunk is every module the indexer holds at the trunk, with its value. +// trunkOf is every module the catalog half of branch holds, with its value (design 47 G §184). +func (w *world) trunkOf(branch string) map[string]meshcatalog.ModulesByModuleValue { + w.mu.Lock() + defer w.mu.Unlock() + return maps.Clone(w.modules[branch]) +} + +// graphOf is the module graph of the catalog half of branch. +func (w *world) graphOf(branch string) graph.Graph { + w.mu.Lock() + defer w.mu.Unlock() + return graph.Graph(maps.Clone(w.graphs[branch])) +} + +// branches are the watched branches the catalog half is held of, the trunk's first. +func (w *world) branches() []string { + w.mu.Lock() + defer w.mu.Unlock() + out := slices.Sorted(maps.Keys(w.modules)) + if i := slices.Index(out, trunkBranch); i > 0 { + out = slices.Insert(slices.Delete(out, i, i+1), 0, trunkBranch) + } + return out +} + +// trunk is every module any branch's catalog half holds, each with its value on the first branch that holds it (the trunk's +// first): the modules that exist, for the parts that ask which, not at which fingerprint a node runs it. func (w *world) trunk() map[string]meshcatalog.ModulesByModuleValue { out := map[string]meshcatalog.ModulesByModuleValue{} - for key, value := range w.read(buckets.Modules) { - name, err := meshcatalogclient.ParseModulesKey(key) - if err != nil { - continue - } - if m, err := meshcatalogclient.DecodeModule(value); err == nil { - out[name] = m + for _, b := range w.branches() { + for name, v := range w.trunkOf(b) { + if _, ok := out[name]; !ok { + out[name] = v + } } } return out } +// putModule takes one module of branch's catalog half in: its value, its graph node and its parsed manifest by the digest the +// value names. +func (w *world) putModule(branch, module string, v meshcatalog.ModulesByModuleValue, n graph.Node, m *parsed) { + w.mu.Lock() + defer w.mu.Unlock() + if w.modules[branch] == nil { + w.modules[branch], w.graphs[branch] = map[string]meshcatalog.ModulesByModuleValue{}, map[string]graph.Node{} + } + w.modules[branch][module], w.graphs[branch][module] = v, n + if m != nil && v.Manifest != "" { + w.manifests[v.Manifest] = *m + } +} + +// dropModule takes a module out of branch's catalog half. +func (w *world) dropModule(branch, module string) { + w.mu.Lock() + defer w.mu.Unlock() + delete(w.modules[branch], module) + delete(w.graphs[branch], module) +} + +// holdsBranch says whether the catalog half of branch has been taken in (even empty). +func (w *world) holdsBranch(branch string) bool { + w.mu.Lock() + defer w.mu.Unlock() + return w.modules[branch] != nil +} + // records is each node's record in its own have key (`.node`): the nodes an assignments proposal is answered by, // with those assignments names (ADR 0593 §8). func (w *world) records() map[string]nodeengine.NodeRecord { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world_test.go index 73a1d5cf..7dcc316d 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world_test.go @@ -10,6 +10,7 @@ import ( "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate" + "git.novox.be/novox/mesh-delivery/sdk/graph" ) // fakeManifests is the indexer's manifests bucket in memory. @@ -71,3 +72,12 @@ func theWorldCountsTheValuesAKeyTakes(t *testing.T) { } } } + +// putModuleRaw takes a module value encoded as the catalog's modules say it into the world as branch's catalog half holds it. +func putModuleRaw(w *world, branch, module string, raw []byte) { + v, err := meshcatalogclient.DecodeModule(raw) + if err != nil { + panic(err) + } + w.putModule(branch, module, v, graph.Node{FP: v.Fp2, Repository: v.Repository}, nil) +} diff --git a/modules/mesh-delivery-assigner/manifest_test.go b/modules/mesh-delivery-assigner/manifest_test.go index ba1b4796..0843516b 100644 --- a/modules/mesh-delivery-assigner/manifest_test.go +++ b/modules/mesh-delivery-assigner/manifest_test.go @@ -123,11 +123,11 @@ func TestManifest(t *testing.T) { } // ADR 0618 §4: and it hears each node's engine say its reads changed; ADR 0636 §4 (issue 1966): and the seat list's two events; // issue 1966 note 151: and the teller's ask-ended, its subject the one the assigner hears. - if !slices.Equal(m.Consumes, []string{"mesh-catalog.registration-changed", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", "mesh-delivery-indexer.views.graph", "mesh-delivery-indexer.views.manifests", "mesh-delivery-indexer.views.modules", buckets.TellerModule + "." + tellerevents.AskEnded, "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.group-ended", "node-engine.manifest-reads-changed", "node-engine.running"}) { - t.Errorf("consumes = %v, want the indexer's views of graph, manifests and modules, the teller's ask-ended, the engines' reads changed and what they run", m.Consumes) + if !slices.Equal(m.Consumes, []string{"mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", buckets.TellerModule + "." + tellerevents.AskEnded, "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.group-ended", "node-engine.manifest-reads-changed", "node-engine.running"}) { + t.Errorf("consumes = %v, want the catalog half's revisions (design 47 G §195), the teller's ask-ended, the engines' reads changed and what they run", m.Consumes) } - if want := []string{"mesh-delivery-indexer.graph", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", - "seat:mesh-catalog.registrations", "seat:mesh-catalog.seat-definitions", // ADR 0636 §4: the seat list + if want := []string{"mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", + "seat:mesh-catalog.registrations", "seat:mesh-catalog.revision", "seat:mesh-catalog.seat-definitions", // ADR 0636 §4: the seat list "seat:mesh-hand-acts.hand-act", "seat:mesh-hand-acts.hand-acts", "seat:mesh-hand-acts.warranted", // ADR 0565 §9, §11: the seat alone "seat:node-backup.backed-up", "seat:node-backup.now", "seat:node-engine.manifest-reads", // ADR 0618 §3 diff --git a/modules/mesh-delivery-assigner/module.json b/modules/mesh-delivery-assigner/module.json index e69ae235..78ce919a 100644 --- a/modules/mesh-delivery-assigner/module.json +++ b/modules/mesh-delivery-assigner/module.json @@ -39,12 +39,10 @@ "operator-channel" ], "consumes": [ + "mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", - "mesh-delivery-indexer.views.graph", - "mesh-delivery-indexer.views.manifests", - "mesh-delivery-indexer.views.modules", "mesh-delivery-teller.ask-ended", "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.group-ended", @@ -52,10 +50,10 @@ "node-engine.running" ], "invokes": [ - "mesh-delivery-indexer.graph", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "seat:mesh-catalog.registrations", + "seat:mesh-catalog.revision", "seat:mesh-catalog.seat-definitions", "seat:mesh-hand-acts.hand-act", "seat:mesh-hand-acts.hand-acts", From 9d79964487940acaaf2a94ad2edf4d6bb9f7923f Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 13:53:17 +0100 Subject: [PATCH 30/33] CT2 (answers): mesh-builds' reserved takes a package's name, toolchain-agreed carries digests with their nodes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §478 (reserved takes module or name, with version; an unknown or doubled name answered error, §491), §472, §495 (toolchain-agreed carries digests, each with the nodes reporting it, in place of one digest). Regenerated by seatgen (-check clean). Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- contracts/go/seats/meshbuilds/contract.go | 18 +++++++-- .../go/seats/meshbuilds/contract_test.go | 4 +- .../seats/mesh-builds/fixtures/events.json | 16 +++++++- .../seats/mesh-builds/fixtures/verbs.json | 8 ++++ contracts/seats/mesh-builds/seat.json | 40 ++++++++++++++----- contracts/ts/seats/meshbuilds/index.d.ts | 17 ++++++-- 6 files changed, 84 insertions(+), 19 deletions(-) diff --git a/contracts/go/seats/meshbuilds/contract.go b/contracts/go/seats/meshbuilds/contract.go index 76437fcc..f95ae661 100644 --- a/contracts/go/seats/meshbuilds/contract.go +++ b/contracts/go/seats/meshbuilds/contract.go @@ -77,11 +77,19 @@ type PackagePublished struct { Version string `json:"version"` } +// ToolchainAgreedDigestsItem is a type of the contract. +type ToolchainAgreedDigestsItem struct { + // the toolchain part's digest + Digest string `json:"digest"` + // the nodes reporting that digest + Nodes []string `json:"nodes"` +} + // ToolchainAgreed is the agreed toolchain of a language changed (design 47 G section 376, ADR 0584 section 1). type ToolchainAgreed struct { At *time.Time `json:"at,omitempty"` - // the toolchain part's digest - Digest string `json:"digest"` + // each digest the nodes reporting the agreed version report, as it stood when the version was agreed, with those nodes (design 47 G section 472, section 495) + Digests []ToolchainAgreedDigestsItem `json:"digests"` // the language whose agreed toolchain changed Language string `json:"language"` V int64 `json:"v"` @@ -91,8 +99,10 @@ type ToolchainAgreed struct { // ReservedInput is what reserved takes. type ReservedInput struct { - // the module - Module string `json:"module"` + // the module; give this or name + Module string `json:"module,omitempty"` + // a package's or an image's name in its ecosystem (the Go module path, the npm name, the image name), in place of module; a name no entry carries, or more than one entry carries, is answered error naming it (design 47 G section 478, section 491) + Name string `json:"name,omitempty"` // the version, when asked for a particular one Version string `json:"version,omitempty"` } diff --git a/contracts/go/seats/meshbuilds/contract_test.go b/contracts/go/seats/meshbuilds/contract_test.go index abbc1cf1..65fb7dfc 100644 --- a/contracts/go/seats/meshbuilds/contract_test.go +++ b/contracts/go/seats/meshbuilds/contract_test.go @@ -19,10 +19,12 @@ func TestFixtures(t *testing.T) { }{ {name: "a package published with the dependencies the build resolved", subject: "mesh.seat.mesh-builds.event.package-published", given: "{\"artifact\":\"mesh-sdk:go\",\"at\":\"2026-11-10T10:00:00Z\",\"branch\":\"main\",\"dependencies\":[{\"artifact\":\"mesh-contracts:go-nodeengine\",\"digest\":\"sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"kind\":1,\"version\":\"0.1.4\"},{\"artifact\":\"mesh-build-go:toolchain\",\"digest\":\"sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"kind\":3,\"version\":\"1.26.0\"}],\"digest\":\"sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"module\":\"mesh-sdk\",\"source\":3589,\"v\":1,\"version\":\"0.1.15\"}", wire: "{\"artifact\":\"mesh-sdk:go\",\"at\":\"2026-11-10T10:00:00Z\",\"branch\":\"main\",\"dependencies\":[{\"artifact\":\"mesh-contracts:go-nodeengine\",\"digest\":\"sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"kind\":1,\"version\":\"0.1.4\"},{\"artifact\":\"mesh-build-go:toolchain\",\"digest\":\"sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"kind\":3,\"version\":\"1.26.0\"}],\"digest\":\"sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"module\":\"mesh-sdk\",\"source\":3589,\"v\":1,\"version\":\"0.1.15\"}", decode: func(b []byte) (any, error) { var v PackagePublishedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-builds\\.event\\.package-published$")}, - {name: "a toolchain agreed", subject: "mesh.seat.mesh-builds.event.toolchain-agreed", given: "{\"at\":\"2026-11-10T10:00:00Z\",\"digest\":\"sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"language\":\"go\",\"v\":1,\"version\":\"1.26.0\"}", wire: "{\"at\":\"2026-11-10T10:00:00Z\",\"digest\":\"sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"language\":\"go\",\"v\":1,\"version\":\"1.26.0\"}", + {name: "a toolchain agreed", subject: "mesh.seat.mesh-builds.event.toolchain-agreed", given: "{\"at\":\"2026-11-10T10:00:00Z\",\"digests\":[{\"digest\":\"sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"nodes\":[\"ace\",\"g14\"]},{\"digest\":\"sha256:fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210\",\"nodes\":[\"novox\"]}],\"language\":\"go\",\"v\":1,\"version\":\"1.26.0\"}", wire: "{\"at\":\"2026-11-10T10:00:00Z\",\"digests\":[{\"digest\":\"sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"nodes\":[\"ace\",\"g14\"]},{\"digest\":\"sha256:fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210\",\"nodes\":[\"novox\"]}],\"language\":\"go\",\"v\":1,\"version\":\"1.26.0\"}", decode: func(b []byte) (any, error) { var v ToolchainAgreedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-builds\\.event\\.toolchain-agreed$")}, {name: "reserved asks a module", subject: "", given: "{\"module\":\"mesh-sdk\"}", wire: "{\"module\":\"mesh-sdk\"}", decode: func(b []byte) (any, error) { var v ReservedInput; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, + {name: "reserved asks a package by its name", subject: "", given: "{\"name\":\"git.novox.be/novox/mesh-sdk/go\",\"version\":\"0.1.15\"}", wire: "{\"name\":\"git.novox.be/novox/mesh-sdk/go\",\"version\":\"0.1.15\"}", + decode: func(b []byte) (any, error) { var v ReservedInput; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, {name: "reserved answers the version and each artifact's digest", subject: "", given: "{\"artifacts\":[{\"artifact\":\"mesh-sdk:go\",\"digest\":\"sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\"},{\"artifact\":\"mesh-sdk:ts\"}],\"module\":\"mesh-sdk\",\"version\":\"0.1.15\"}", wire: "{\"artifacts\":[{\"artifact\":\"mesh-sdk:go\",\"digest\":\"sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\"},{\"artifact\":\"mesh-sdk:ts\"}],\"module\":\"mesh-sdk\",\"version\":\"0.1.15\"}", decode: func(b []byte) (any, error) { var v ReservedAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, } diff --git a/contracts/seats/mesh-builds/fixtures/events.json b/contracts/seats/mesh-builds/fixtures/events.json index 25c23b77..44192577 100644 --- a/contracts/seats/mesh-builds/fixtures/events.json +++ b/contracts/seats/mesh-builds/fixtures/events.json @@ -36,7 +36,21 @@ "v": 1, "language": "go", "version": "1.26.0", - "digest": "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "digests": [ + { + "digest": "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "nodes": [ + "ace", + "g14" + ] + }, + { + "digest": "sha256:fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210", + "nodes": [ + "novox" + ] + } + ], "at": "2026-11-10T10:00:00Z" } } diff --git a/contracts/seats/mesh-builds/fixtures/verbs.json b/contracts/seats/mesh-builds/fixtures/verbs.json index 668a960c..b8c50afb 100644 --- a/contracts/seats/mesh-builds/fixtures/verbs.json +++ b/contracts/seats/mesh-builds/fixtures/verbs.json @@ -6,6 +6,14 @@ "module": "mesh-sdk" } }, + { + "name": "reserved asks a package by its name", + "of": "verb:reserved.input", + "given": { + "name": "git.novox.be/novox/mesh-sdk/go", + "version": "0.1.15" + } + }, { "name": "reserved answers the version and each artifact's digest", "of": "verb:reserved.answer", diff --git a/contracts/seats/mesh-builds/seat.json b/contracts/seats/mesh-builds/seat.json index cfae1a92..29e6a5c3 100644 --- a/contracts/seats/mesh-builds/seat.json +++ b/contracts/seats/mesh-builds/seat.json @@ -16,22 +16,24 @@ "seat:container-image-registry", "seat:oci-artifact-registry" ], - "description": "A module's reserved version with each artifact's digest, for a registry holder deciding whether to publish (design 47 G section 369; design 51, Publishing).", + "description": "A module's reserved version with each artifact's digest, for a registry holder deciding whether to publish; asked by the module or by a package's name in its ecosystem (design 47 G section 369, section 478; design 51, Publishing).", "input": { "type": "object", "properties": { "module": { "type": "string", - "description": "the module" + "description": "the module; give this or name" + }, + "name": { + "type": "string", + "description": "a package's or an image's name in its ecosystem (the Go module path, the npm name, the image name), in place of module; a name no entry carries, or more than one entry carries, is answered error naming it (design 47 G section 478, section 491)" }, "version": { "type": "string", "description": "the version, when asked for a particular one" } }, - "required": [ - "module" - ] + "required": [] }, "answer": { "type": "object", @@ -198,9 +200,29 @@ "type": "string", "description": "the agreed toolchain's version" }, - "digest": { - "type": "string", - "description": "the toolchain part's digest" + "digests": { + "type": "array", + "description": "each digest the nodes reporting the agreed version report, as it stood when the version was agreed, with those nodes (design 47 G section 472, section 495)", + "items": { + "type": "object", + "properties": { + "digest": { + "type": "string", + "description": "the toolchain part's digest" + }, + "nodes": { + "type": "array", + "items": { + "type": "string" + }, + "description": "the nodes reporting that digest" + } + }, + "required": [ + "digest", + "nodes" + ] + } }, "at": { "type": "string", @@ -211,7 +233,7 @@ "v", "language", "version", - "digest" + "digests" ], "description": "the agreed toolchain of a language changed (design 47 G section 376, ADR 0584 section 1)" } diff --git a/contracts/ts/seats/meshbuilds/index.d.ts b/contracts/ts/seats/meshbuilds/index.d.ts index 5c6dfedc..9732e7cd 100644 --- a/contracts/ts/seats/meshbuilds/index.d.ts +++ b/contracts/ts/seats/meshbuilds/index.d.ts @@ -55,11 +55,18 @@ export interface PackagePublished { "version": string; } +export interface ToolchainAgreedDigestsItem { + /** the toolchain part's digest */ + "digest": string; + /** the nodes reporting that digest */ + "nodes": string[]; +} + /** the agreed toolchain of a language changed (design 47 G section 376, ADR 0584 section 1) */ export interface ToolchainAgreed { "at"?: string; - /** the toolchain part's digest */ - "digest": string; + /** each digest the nodes reporting the agreed version report, as it stood when the version was agreed, with those nodes (design 47 G section 472, section 495) */ + "digests": ToolchainAgreedDigestsItem[]; /** the language whose agreed toolchain changed */ "language": string; "v": number; @@ -69,8 +76,10 @@ export interface ToolchainAgreed { /** is what reserved takes. */ export interface ReservedInput { - /** the module */ - "module": string; + /** the module; give this or name */ + "module"?: string; + /** a package's or an image's name in its ecosystem (the Go module path, the npm name, the image name), in place of module; a name no entry carries, or more than one entry carries, is answered error naming it (design 47 G section 478, section 491) */ + "name"?: string; /** the version, when asked for a particular one */ "version"?: string; } From 9f43e927e4881b8eca3a3a699c37c70464d26c89 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 13:53:50 +0100 Subject: [PATCH 31/33] WM7 (store-filling): a store clears below 80 % of its cap, not 70 % MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §181, §263 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../internal/watch/pipelines_test.go | 8 ++++++-- modules/mesh-delivery-watchman/internal/watch/records.go | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/modules/mesh-delivery-watchman/internal/watch/pipelines_test.go b/modules/mesh-delivery-watchman/internal/watch/pipelines_test.go index b384cafd..4459ee23 100644 --- a/modules/mesh-delivery-watchman/internal/watch/pipelines_test.go +++ b/modules/mesh-delivery-watchman/internal/watch/pipelines_test.go @@ -144,13 +144,17 @@ func fillBody(store string, bytes int64) []byte { } // design 47 G §181, §263: a store past its share is `store-filling` naming the part and the store, full is urgent, `room` and a -// fill under 70 % clear it. +// fill under 80 % clear it. func TestAStoreFillingIsAFinding(t *testing.T) { w, o, _ := setup() key := FindingKey(ScopeDelivery, "mesh-delivery-records", KindStoreFilling) w.StoreFill("mesh-delivery", "filling", fillBody("records", 60)) if len(o.findings) != 0 { - t.Fatalf("under 70 %%: %v", o.findings) + t.Fatalf("under 80 %%: %v", o.findings) + } + w.StoreFill("mesh-delivery", "filling", fillBody("records", 75)) + if len(o.findings) != 0 { + t.Fatalf("75 %% is under §181's 80 %%: %v", o.findings) } w.StoreFill("mesh-delivery", "filling", fillBody("records", 85)) if o.findings[key].Severity != Warning { diff --git a/modules/mesh-delivery-watchman/internal/watch/records.go b/modules/mesh-delivery-watchman/internal/watch/records.go index f51e2bfa..95e5032b 100644 --- a/modules/mesh-delivery-watchman/internal/watch/records.go +++ b/modules/mesh-delivery-watchman/internal/watch/records.go @@ -37,7 +37,7 @@ const uncovered = "uncovered:record-open" const RecordOpenBound = 24 * time.Hour // FillingFrom is the share of a store's cap under which a `filling` clears its finding (design 47 G §181). -const FillingFrom = 0.70 +const FillingFrom = 0.80 // StoreFill hears a seat's `filling`, `full` or `room` of one of its stores. A `filling` is a warning, naming the part and the // store, unless its share is under FillingFrom, which clears it; `full` is urgent, naming what it refuses; `room` clears either. From 73d15c08f267c8540f602f922dc0a92dab0e1391 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 13:57:00 +0100 Subject: [PATCH 32/33] CT3 (answers): the catalog graph entry carries manifest-digest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §506 (the entry gains manifest-digest, the sha256: digest of the manifest's bytes as the forge answered them, set by the indexer at take-in). Regenerated by seatgen (-check clean). Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- contracts/go/seats/meshcatalog/contract.go | 4 +++- contracts/go/seats/meshcatalog/contract_test.go | 2 +- contracts/seats/mesh-catalog/fixtures/states.json | 1 + contracts/seats/mesh-catalog/seat.json | 4 ++++ contracts/ts/seats/meshcatalog/index.d.ts | 2 ++ 5 files changed, 11 insertions(+), 2 deletions(-) diff --git a/contracts/go/seats/meshcatalog/contract.go b/contracts/go/seats/meshcatalog/contract.go index 1a6b43f6..db822342 100644 --- a/contracts/go/seats/meshcatalog/contract.go +++ b/contracts/go/seats/meshcatalog/contract.go @@ -736,7 +736,9 @@ type CatalogEntry struct { Main string `json:"main,omitempty"` // the validated manifest whole, kept as an object by the entry's digest, so no size limit applies; read by anyone but the indexer through the entry, never as a manifest (design 47 G section 443, section 442) Manifest map[string]any `json:"manifest"` - Module string `json:"module"` + // the sha256: digest of the manifest's bytes as the forge answered them, set by the indexer at take-in and never computed again from the manifest whole; the module's value and the manifests key are this digest (design 47 G section 506) + ManifestDigest string `json:"manifest-digest,omitempty"` + Module string `json:"module"` // the manifest reads a node-engine must state to compose it, its templates' forms included, as the forge bridge read them at the merge (ADR 0614 §4, ADR 0632 §2) Needs []string `json:"needs,omitempty"` // why it has none diff --git a/contracts/go/seats/meshcatalog/contract_test.go b/contracts/go/seats/meshcatalog/contract_test.go index 5ef39f04..e6bca015 100644 --- a/contracts/go/seats/meshcatalog/contract_test.go +++ b/contracts/go/seats/meshcatalog/contract_test.go @@ -85,7 +85,7 @@ func TestFixtures(t *testing.T) { err := json.Unmarshal(b, &v) return v, err }, subjectIs: nil}, - {name: "a catalog graph entry (xclip)", subject: "", given: "{\"bases\":[\"arch\"],\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fp\":\"src2:b427bdde1f100e2f52490cfa9df0c5112dd3700226b4892349f0c14a15a4af56\",\"inputs\":{\"fp-src\":41,\"from\":{\"bases\":[\"arch\"],\"locks\":{\"go.sum\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\"},\"packages\":[\"git.novox.be/novox/mesh-sdk/go@v0.0.0-20261101000000-f469eb387a30\"],\"tree\":\"4b825dc642cb6eb9a060e54bf8d69288fbee4904\"},\"src\":41,\"toolchains\":{\"go\":{\"language\":\"go\",\"recipe\":\"1\",\"version\":\"1.26.0\"}}},\"invokes\":[\"node-files.read\"],\"manifest\":{\"module\":\"xclip\",\"version\":\"1\"},\"module\":\"xclip\",\"packages\":{\"git.novox.be/novox/mesh-sdk/go\":\"v0.0.0-20261101000000-f469eb387a30\"},\"path\":\"modules/xclip\",\"repository\":\"novox/mesh-catalog\",\"toolchains\":{\"go\":{\"module\":\"mesh-build-go\",\"recipe\":\"1\",\"version\":\"1.26.0\"}},\"v\":1,\"verdict\":\"accepted\"}", wire: "{\"bases\":[\"arch\"],\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fp\":\"src2:b427bdde1f100e2f52490cfa9df0c5112dd3700226b4892349f0c14a15a4af56\",\"inputs\":{\"fp-src\":41,\"from\":{\"bases\":[\"arch\"],\"locks\":{\"go.sum\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\"},\"packages\":[\"git.novox.be/novox/mesh-sdk/go@v0.0.0-20261101000000-f469eb387a30\"],\"tree\":\"4b825dc642cb6eb9a060e54bf8d69288fbee4904\"},\"src\":41,\"toolchains\":{\"go\":{\"language\":\"go\",\"recipe\":\"1\",\"version\":\"1.26.0\"}}},\"invokes\":[\"node-files.read\"],\"manifest\":{\"module\":\"xclip\",\"version\":\"1\"},\"module\":\"xclip\",\"packages\":{\"git.novox.be/novox/mesh-sdk/go\":\"v0.0.0-20261101000000-f469eb387a30\"},\"path\":\"modules/xclip\",\"repository\":\"novox/mesh-catalog\",\"toolchains\":{\"go\":{\"module\":\"mesh-build-go\",\"recipe\":\"1\",\"version\":\"1.26.0\"}},\"v\":1,\"verdict\":\"accepted\"}", + {name: "a catalog graph entry (xclip)", subject: "", given: "{\"bases\":[\"arch\"],\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fp\":\"src2:b427bdde1f100e2f52490cfa9df0c5112dd3700226b4892349f0c14a15a4af56\",\"inputs\":{\"fp-src\":41,\"from\":{\"bases\":[\"arch\"],\"locks\":{\"go.sum\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\"},\"packages\":[\"git.novox.be/novox/mesh-sdk/go@v0.0.0-20261101000000-f469eb387a30\"],\"tree\":\"4b825dc642cb6eb9a060e54bf8d69288fbee4904\"},\"src\":41,\"toolchains\":{\"go\":{\"language\":\"go\",\"recipe\":\"1\",\"version\":\"1.26.0\"}}},\"invokes\":[\"node-files.read\"],\"manifest\":{\"module\":\"xclip\",\"version\":\"1\"},\"manifest-digest\":\"sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"module\":\"xclip\",\"packages\":{\"git.novox.be/novox/mesh-sdk/go\":\"v0.0.0-20261101000000-f469eb387a30\"},\"path\":\"modules/xclip\",\"repository\":\"novox/mesh-catalog\",\"toolchains\":{\"go\":{\"module\":\"mesh-build-go\",\"recipe\":\"1\",\"version\":\"1.26.0\"}},\"v\":1,\"verdict\":\"accepted\"}", wire: "{\"bases\":[\"arch\"],\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fp\":\"src2:b427bdde1f100e2f52490cfa9df0c5112dd3700226b4892349f0c14a15a4af56\",\"inputs\":{\"fp-src\":41,\"from\":{\"bases\":[\"arch\"],\"locks\":{\"go.sum\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\"},\"packages\":[\"git.novox.be/novox/mesh-sdk/go@v0.0.0-20261101000000-f469eb387a30\"],\"tree\":\"4b825dc642cb6eb9a060e54bf8d69288fbee4904\"},\"src\":41,\"toolchains\":{\"go\":{\"language\":\"go\",\"recipe\":\"1\",\"version\":\"1.26.0\"}}},\"invokes\":[\"node-files.read\"],\"manifest\":{\"module\":\"xclip\",\"version\":\"1\"},\"manifest-digest\":\"sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"module\":\"xclip\",\"packages\":{\"git.novox.be/novox/mesh-sdk/go\":\"v0.0.0-20261101000000-f469eb387a30\"},\"path\":\"modules/xclip\",\"repository\":\"novox/mesh-catalog\",\"toolchains\":{\"go\":{\"module\":\"mesh-build-go\",\"recipe\":\"1\",\"version\":\"1.26.0\"}},\"v\":1,\"verdict\":\"accepted\"}", decode: func(b []byte) (any, error) { var v CatalogRevisionsByEntryDigestValue err := json.Unmarshal(b, &v) diff --git a/contracts/seats/mesh-catalog/fixtures/states.json b/contracts/seats/mesh-catalog/fixtures/states.json index c3c1aacc..b1174d11 100644 --- a/contracts/seats/mesh-catalog/fixtures/states.json +++ b/contracts/seats/mesh-catalog/fixtures/states.json @@ -302,6 +302,7 @@ "module": "xclip", "version": "1" }, + "manifest-digest": "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "verdict": "accepted", "fp": "src2:b427bdde1f100e2f52490cfa9df0c5112dd3700226b4892349f0c14a15a4af56", "inputs": { diff --git a/contracts/seats/mesh-catalog/seat.json b/contracts/seats/mesh-catalog/seat.json index fd80e3bf..978e31dd 100644 --- a/contracts/seats/mesh-catalog/seat.json +++ b/contracts/seats/mesh-catalog/seat.json @@ -1872,6 +1872,10 @@ "type": "object", "description": "the validated manifest whole, kept as an object by the entry's digest, so no size limit applies; read by anyone but the indexer through the entry, never as a manifest (design 47 G section 443, section 442)" }, + "manifest-digest": { + "type": "string", + "description": "the sha256: digest of the manifest's bytes as the forge answered them, set by the indexer at take-in and never computed again from the manifest whole; the module's value and the manifests key are this digest (design 47 G section 506)" + }, "verdict": { "type": "string", "description": "its manifest's verdict: accepted, refused or held" diff --git a/contracts/ts/seats/meshcatalog/index.d.ts b/contracts/ts/seats/meshcatalog/index.d.ts index f935a038..53832de7 100644 --- a/contracts/ts/seats/meshcatalog/index.d.ts +++ b/contracts/ts/seats/meshcatalog/index.d.ts @@ -247,6 +247,8 @@ export interface CatalogEntry { "main"?: string; /** the validated manifest whole, kept as an object by the entry's digest, so no size limit applies; read by anyone but the indexer through the entry, never as a manifest (design 47 G section 443, section 442) */ "manifest": Record; + /** the sha256: digest of the manifest's bytes as the forge answered them, set by the indexer at take-in and never computed again from the manifest whole; the module's value and the manifests key are this digest (design 47 G section 506) */ + "manifest-digest"?: string; "module": string; /** the manifest reads a node-engine must state to compose it, its templates' forms included, as the forge bridge read them at the merge (ADR 0614 §4, ADR 0632 §2) */ "needs"?: string[]; From cb6dc7c3208c659e55c6ce130c48fa3b8d8ea7f6 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 13:57:54 +0100 Subject: [PATCH 33/33] BL4: the git tag go/vMAJOR.MINOR.PATCH is written on the catalog's version-released, at the commit the version was reserved from MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The reservation keeps the repository and commit of the build and whether the module has a Go package; version-released writes the tag through the seat git's verb tag before the key goes, and a tag the forge did not write leaves the release to be heard again. Builds: design 47 G §370 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-builder/reserve.go | 43 +++++++++++- .../cmd/mesh-delivery-builder/reserve_test.go | 68 ++++++++++++++++++- modules/mesh-delivery-builder/go.mod | 2 + .../internal/build/numbers.go | 11 ++- .../internal/build/reserve_test.go | 2 +- .../internal/build/run.go | 2 +- .../internal/versions/reserve.go | 20 +++++- .../internal/versions/reserve_test.go | 16 ++--- .../internal/versions/tag.go | 4 ++ .../mesh-delivery-builder/manifest_test.go | 1 + modules/mesh-delivery-builder/module.json | 1 + 11 files changed, 149 insertions(+), 21 deletions(-) create mode 100644 modules/mesh-delivery-builder/internal/versions/tag.go diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go index bb41bbd8..7d286318 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go @@ -12,6 +12,7 @@ import ( "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/git" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "mesh-delivery-builder/internal/build" @@ -45,12 +46,14 @@ type reservations struct { bound time.Duration wake func(context.Context) logf func(string, ...any) + // call asks a seat's verb: the git seat's tag (design 47 G §370). + call toolCaller } // newReservations makes the reserving over store; the released versions of a module are asked of the seat mesh-catalog's // verb versions through call (design 47 G §348), the only source numbering reads them from. func newReservations(store versions.ReserveStore, call toolCaller, bounds func() jetstream.KeyValue, found *findings, wake func(context.Context), logf func(string, ...any)) *reservations { - rv := &reservations{bounds: bounds, found: found, bound: versions.UnreleasedBound, wake: wake, logf: logf} + rv := &reservations{call: call, bounds: bounds, found: found, bound: versions.UnreleasedBound, wake: wake, logf: logf} rv.r = &versions.Reserver{Store: store, Filled: rv.filled.Load, ReleasedOf: func(_ context.Context, module string) ([]string, error) { raw, err := call("seat:"+meshcatalog.Seat+".versions", meshcatalog.VersionsInput{Module: module}) if err != nil { @@ -73,8 +76,8 @@ var _ build.Reservations = (*reservations)(nil) func (rv *reservations) Filled() bool { return rv.filled.Load() } -func (rv *reservations) Reserve(ctx context.Context, module, series string, artifacts []string) (versions.ReservedVersion, error) { - return rv.r.Reserve(ctx, module, series, artifacts) +func (rv *reservations) Reserve(ctx context.Context, module, series string, artifacts []string, src versions.Source) (versions.ReservedVersion, error) { + return rv.r.Reserve(ctx, module, series, artifacts, src) } // Built records the digests; once every artifact has one, the publish's bound starts (a key of the state bounds with its @@ -165,6 +168,13 @@ func (rv *reservations) heard(ctx context.Context, m jetstream.Msg) bool { rv.logf("%s holds what is no version-released, left: %v", m.Subject(), err) break } + if err := rv.tagged(ctx, ev); err != nil { + rv.logf("%v; heard again", err) + if nakErr := m.Nak(); nakErr != nil { + rv.logf("leaving %s for the bus to give again: %v", m.Subject(), nakErr) + } + return true + } gone, err := rv.r.Released(ctx, ev.Module, ev.Version) if err != nil { rv.logf("%v; heard again", err) @@ -206,3 +216,30 @@ func (rv *reservations) dropped(ctx context.Context, module, series, by, why str rv.found.clear(ctx, versions.VersionUnreleased, meshconditionsclient.ScopeModule, module) return fmt.Sprintf("the reservation of %s %s is dropped (%s, %s)", module, series, by, why), nil } + +// tagged writes the git tag of a released version (design 47 G §370): on the catalog's version-released, through the seat +// git's verb tag, at the commit the reservation of that version was made from. A version with no reservation (a superseded +// build's version is never released, and the key of a version released by another builder is gone) or one that names no +// tag (a module with no Go package) is tagged nowhere. A tag the forge could not write is an error, the message heard again. +func (rv *reservations) tagged(ctx context.Context, ev meshcatalog.VersionReleasedEvent) error { + res, found, err := rv.r.Of(ctx, ev.Module, ev.Version) + if err != nil { + return fmt.Errorf("the reservation of %s %s cannot be read to tag it: %w", ev.Module, ev.Version, err) + } + if !found || res.Tag == "" || res.Commit == "" { + return nil + } + raw, err := rv.call("seat:"+git.Seat+".tag", git.TagInput{Repository: res.Repository, Commit: res.Commit, Tag: res.Tag}) + if err != nil { + return fmt.Errorf("the tag %s of %s is not written: %w", res.Tag, ev.Module, err) + } + var a git.TagAnswer + if err := json.Unmarshal(raw, &a); err != nil { + return fmt.Errorf("the git seat answered what is no tag answer for %s: %w", res.Tag, err) + } + if a.Error != "" || !a.Performed { + return fmt.Errorf("the tag %s of %s is not written: %s (%s)", res.Tag, ev.Module, a.Error, a.ErrorKind) + } + rv.logf("%s is tagged %s at %s", ev.Module, res.Tag, res.Commit) + return nil +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go index d1a04e2f..679e1b95 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go @@ -10,6 +10,7 @@ import ( "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/git" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "mesh-delivery-builder/internal/versions" @@ -85,14 +86,14 @@ func TestTheCatalogsWordsMoveTheReservations(t *testing.T) { } rv := newReservations(&memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}, call, func() jetstream.KeyValue { return bounds }, found, func(context.Context) { woke++ }, t.Logf) - if _, err := rv.Reserve(ctx, "m", "1.4", []string{"m:a"}); err == nil || rv.Filled() { + if _, err := rv.Reserve(ctx, "m", "1.4", []string{"m:a"}, versions.Source{}); err == nil || rv.Filled() { t.Fatalf("a version was reserved before versions-filled (%v)", err) } filled := &wakeMsg{subject: meshcatalog.SubjectVersionsFilled, data: []byte(`{"v":1}`)} if !rv.heard(ctx, filled) || !filled.acked || !rv.Filled() || woke != 1 { t.Fatalf("versions-filled: acked %v, filled %v, woke %d", filled.acked, rv.Filled(), woke) } - res, err := rv.Reserve(ctx, "m", "1.4", []string{"m:a", "m:b"}) + res, err := rv.Reserve(ctx, "m", "1.4", []string{"m:a", "m:b"}, versions.Source{}) if err != nil || res.Version != "1.4.3" { t.Fatalf("reserved %q (%v), want 1.4.3 after the catalog's 1.4.2", res.Version, err) } @@ -119,7 +120,7 @@ func TestTheCatalogsWordsMoveTheReservations(t *testing.T) { if all, _ := rv.r.All(ctx); len(all) != 0 || bounds.keys[boundKey] || len(st.vals) != 0 { t.Errorf("after version-released: reservations %v, bounds %v, findings %v; want none", all, bounds.keys, st.vals) } - if _, err := rv.Reserve(ctx, "m", "1.4", []string{"m:a"}); err != nil { + if _, err := rv.Reserve(ctx, "m", "1.4", []string{"m:a"}, versions.Source{}); err != nil { t.Fatal(err) } if answer, err := rv.dropped(ctx, "m", "1.4", "op", "never released", time.Unix(0, 0)); err != nil || !strings.Contains(answer, "dropped") { @@ -129,3 +130,64 @@ func TestTheCatalogsWordsMoveTheReservations(t *testing.T) { t.Error("a reservation that is gone was dropped again") } } + +// TestAReleasedVersionIsTaggedAtTheCommitItWasBuiltFrom: design 47 G §370 — the git tag go/vMAJOR.MINOR.PATCH is written through +// the seat git's tag on the catalog's version-released, at the commit the version's reservation was made from; a module with no +// Go package, and a version never released (a superseded build's), are tagged nowhere; a tag the forge did not write leaves +// version-released to be heard again with the reservation standing. +func TestAReleasedVersionIsTaggedAtTheCommitItWasBuiltFrom(t *testing.T) { + ctx := context.Background() + var tags []git.TagInput + refuse := "" + call := func(addr string, body any) (json.RawMessage, error) { + if addr != "seat:git.tag" { + return json.Marshal(meshcatalog.VersionsAnswer{}) + } + tags = append(tags, body.(git.TagInput)) + return json.Marshal(git.TagAnswer{Performed: refuse == "", Error: refuse}) + } + found := &findings{store: &memFindings{vals: map[string][]byte{}}, logf: t.Logf, now: func() time.Time { return time.Unix(0, 0) }} + rv := newReservations(&memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}, call, func() jetstream.KeyValue { return &boundsKV{keys: map[string]bool{}} }, found, nil, t.Logf) + rv.filled.Store(true) + goPkg := versions.Source{Repository: "o/r", Commit: "c0ffee", GoPackage: true} + if _, err := rv.Reserve(ctx, "pkg", "0.1", []string{"pkg:lib"}, goPkg); err != nil { + t.Fatal(err) + } + if _, err := rv.Reserve(ctx, "prog", "2.0", []string{"prog:bin"}, versions.Source{Repository: "o/p", Commit: "beef"}); err != nil { + t.Fatal(err) + } + release := func(module, version string) *wakeMsg { + m := &wakeMsg{subject: meshcatalog.SubjectVersionReleased, data: []byte(`{"v":1,"module":"` + module + `","version":"` + version + `","source":1,"artifacts":[]}`)} + rv.heard(ctx, m) + return m + } + if m := release("prog", "2.0.0"); !m.acked || len(tags) != 0 { + t.Fatalf("a module with no Go package: acked %v, tags %v; want none written", m.acked, tags) + } + refuse = "forge down" + if m := release("pkg", "0.1.0"); !m.naked || m.acked { + t.Fatalf("a tag the forge did not write: naked %v acked %v; want the release heard again", m.naked, m.acked) + } + if all, _ := rv.r.All(ctx); len(all) != 1 { + t.Fatalf("the reservation went with an unwritten tag: %v", all) + } + refuse = "" + tags = nil + if m := release("pkg", "0.1.0"); !m.acked || len(tags) != 1 || tags[0] != (git.TagInput{Repository: "o/r", Commit: "c0ffee", Tag: "go/v0.1.0"}) { + t.Fatalf("the release: acked %v, tags %+v; want go/v0.1.0 at c0ffee of o/r", m.acked, tags) + } + tags = nil + if _, err := rv.Reserve(ctx, "pkg", "0.1", []string{"pkg:lib"}, goPkg); err != nil { // build A: 0.1.0 (the stub catalog holds none released) + t.Fatal(err) + } + if v, err := rv.Reserve(ctx, "pkg", "0.1", []string{"pkg:lib"}, versions.Source{Repository: "o/r", Commit: "b0b", GoPackage: true}); err != nil || v.Version != "0.1.1" { // build B supersedes A + t.Fatalf("second build reserved %q (%v)", v.Version, err) + } + release("pkg", "0.1.0") // A's version is never released in fact; were it said, it is no reservation of the key + if len(tags) != 0 { + t.Fatalf("a superseded build's version was tagged: %+v", tags) + } + if release("pkg", "0.1.1"); len(tags) != 1 || tags[0].Commit != "b0b" { + t.Fatalf("the newer build's version: tags %+v; want go/v0.1.1 at b0b", tags) + } +} diff --git a/modules/mesh-delivery-builder/go.mod b/modules/mesh-delivery-builder/go.mod index 916455c4..3d333a4d 100644 --- a/modules/mesh-delivery-builder/go.mod +++ b/modules/mesh-delivery-builder/go.mod @@ -11,6 +11,8 @@ require ( git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/contracts/go/seats/git v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/seats/containerimageregistry v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/seats/gopackageregistry v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000 diff --git a/modules/mesh-delivery-builder/internal/build/numbers.go b/modules/mesh-delivery-builder/internal/build/numbers.go index f1b9cef5..2fc70614 100644 --- a/modules/mesh-delivery-builder/internal/build/numbers.go +++ b/modules/mesh-delivery-builder/internal/build/numbers.go @@ -3,6 +3,7 @@ package build import ( "context" "fmt" + "git.novox.be/novox/mesh-sdk/go/catalogue" "strconv" "strings" "time" @@ -148,7 +149,7 @@ type Reservations interface { // Filled says the catalog said versions-filled: nothing is reserved before it. Filled() bool // Reserve reserves module's next version in series for the artifacts (their references, :). - Reserve(ctx context.Context, module, series string, artifacts []string) (versions.ReservedVersion, error) + Reserve(ctx context.Context, module, series string, artifacts []string, src versions.Source) (versions.ReservedVersion, error) // Built records the digests of the artifacts built under the reserved version, by reference. Built(ctx context.Context, module, version string, digests map[string]string) error } @@ -184,12 +185,16 @@ func (b *Builder) waitFill(module string, m meshcatalog.ModulesByModuleValue) bo // reserve reserves the version of the build that starts; a reservation that cannot be made is said and the build goes on // unreserved, as an unnumbered build does. -func (b *Builder) reserve(ctx context.Context, module string, plan Plan) (versions.ReservedVersion, bool) { +func (b *Builder) reserve(ctx context.Context, module string, m meshcatalog.ModulesByModuleValue, plan Plan) (versions.ReservedVersion, bool) { refs := make([]string, 0, len(plan.Tasks)) for _, t := range plan.Tasks { refs = append(refs, module+":"+t.Artifact) } - res, err := b.Reserved.Reserve(ctx, module, plan.Series, refs) + src := versions.Source{Repository: m.Repository, Commit: m.Commit} + for _, t := range plan.Tasks { + src.GoPackage = src.GoPackage || (t.Kind == catalogue.ArtifactPackage && t.Language == "go") + } + res, err := b.Reserved.Reserve(ctx, module, plan.Series, refs, src) if err != nil { b.logf("%s: no version is reserved for this build: %v", module, err) return versions.ReservedVersion{}, false diff --git a/modules/mesh-delivery-builder/internal/build/reserve_test.go b/modules/mesh-delivery-builder/internal/build/reserve_test.go index cc1fc2dd..b9a33735 100644 --- a/modules/mesh-delivery-builder/internal/build/reserve_test.go +++ b/modules/mesh-delivery-builder/internal/build/reserve_test.go @@ -23,7 +23,7 @@ type fakeReservations struct { func (f *fakeReservations) Filled() bool { return f.filled } -func (f *fakeReservations) Reserve(_ context.Context, module, series string, artifacts []string) (versions.ReservedVersion, error) { +func (f *fakeReservations) Reserve(_ context.Context, module, series string, artifacts []string, _ versions.Source) (versions.ReservedVersion, error) { f.reserved, f.series = append(f.reserved, artifacts), append(f.series, series) return versions.ReservedVersion{Module: module, Series: series, Version: series + ".5"}, nil } diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index f5f8bf3a..4a242237 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -536,7 +536,7 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB var reserved versions.ReservedVersion hasReserved := false if reserving && cl.Build.State != builds.BuildDone { - reserved, hasReserved = b.reserve(ctx, module, plan) + reserved, hasReserved = b.reserve(ctx, module, m, plan) } var result builds.BuilderBuild if nothing { diff --git a/modules/mesh-delivery-builder/internal/versions/reserve.go b/modules/mesh-delivery-builder/internal/versions/reserve.go index 36b02425..89b0dc32 100644 --- a/modules/mesh-delivery-builder/internal/versions/reserve.go +++ b/modules/mesh-delivery-builder/internal/versions/reserve.go @@ -49,6 +49,18 @@ type ReservedVersion struct { // Published are the artifacts a registry seat said published at this version with the built digest. Published map[string]bool `json:"published,omitempty"` At time.Time `json:"at"` + // Repository and Commit are what the build was made from, and Tag the git tag its release is written as (a Go + // package's, GoTagOf): empty for a module with no Go package (design 47 G §370). + Repository string `json:"repository,omitempty"` + Commit string `json:"commit,omitempty"` + Tag string `json:"tag,omitempty"` +} + +// Source is what a build reserving a version is made from (design 47 G §370): the repository and commit the git tag is +// written at, and whether the module has a Go package, whose version is tagged. +type Source struct { + Repository, Commit string + GoPackage bool } // ReservedKey is the key of module's reservation of series (MAJOR.MINOR). @@ -96,7 +108,7 @@ func SplitVersion(version string) (series string, patch int, err error) { // Reserve reserves the next version of module's series for a build that starts, for artifacts (their names), and answers // it. The next patch is the one after the higher of the highest released and the one reserved, 0 when there is neither. A // conflicting write (another build reserved first) is read again and the next taken. -func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts []string) (ReservedVersion, error) { +func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts []string, src Source) (ReservedVersion, error) { if r.Filled != nil && !r.Filled() { return ReservedVersion{}, ErrNotFilled } @@ -122,7 +134,11 @@ func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts next = p } } - out := ReservedVersion{V: 1, Module: module, Series: series, Version: fmt.Sprintf("%s.%d", series, next+1), Artifacts: map[string]string{}, At: r.now()} + out := ReservedVersion{V: 1, Module: module, Series: series, Version: fmt.Sprintf("%s.%d", series, next+1), Artifacts: map[string]string{}, At: r.now(), + Repository: src.Repository, Commit: src.Commit} + if src.GoPackage { + out.Tag = GoTagOf(out.Version) + } for _, a := range artifacts { out.Artifacts[a] = "" } diff --git a/modules/mesh-delivery-builder/internal/versions/reserve_test.go b/modules/mesh-delivery-builder/internal/versions/reserve_test.go index c02c0c96..cfecc500 100644 --- a/modules/mesh-delivery-builder/internal/versions/reserve_test.go +++ b/modules/mesh-delivery-builder/internal/versions/reserve_test.go @@ -46,7 +46,7 @@ func TestReservation(t *testing.T) { arts := []string{"m:a", "m:b"} filled = false - if _, err := r.Reserve(ctx, "m", "0.1", arts); !errors.Is(err, ErrNotFilled) { + if _, err := r.Reserve(ctx, "m", "0.1", arts, Source{}); !errors.Is(err, ErrNotFilled) { t.Fatalf("before versions-filled: %v, want ErrNotFilled", err) } if got, _ := r.All(ctx); len(got) != 0 { @@ -54,25 +54,25 @@ func TestReservation(t *testing.T) { } filled = true - first, err := r.Reserve(ctx, "m", "0.1", arts) + first, err := r.Reserve(ctx, "m", "0.1", arts, Source{}) if err != nil || first.Version != "0.1.20" { t.Fatalf("first build reserves %q (%v), want 0.1.20 after the highest released 0.1.19", first.Version, err) } - second, err := r.Reserve(ctx, "m", "0.1", arts) + second, err := r.Reserve(ctx, "m", "0.1", arts, Source{}) if err != nil || second.Version != "0.1.21" { t.Fatalf("second build reserves %q (%v), want 0.1.21: never the first's", second.Version, err) } // Two builds at once: the rival reserves between this one's read and write, and this one takes the next. store.onUpdate = func() { - if _, err := r.Reserve(ctx, "m", "0.1", arts); err != nil { + if _, err := r.Reserve(ctx, "m", "0.1", arts, Source{}); err != nil { t.Error(err) } } - third, err := r.Reserve(ctx, "m", "0.1", arts) + third, err := r.Reserve(ctx, "m", "0.1", arts, Source{}) if err != nil || third.Version != "0.1.23" { t.Fatalf("a build that lost the compare-and-set reserves %q (%v), want 0.1.23", third.Version, err) } - if v, _ := r.Reserve(ctx, "m", "0.3", arts); v.Version != "0.3.0" { + if v, _ := r.Reserve(ctx, "m", "0.3", arts, Source{}); v.Version != "0.3.0" { t.Errorf("a series with nothing released reserves %q, want 0.3.0", v.Version) } @@ -103,7 +103,7 @@ func TestReservation(t *testing.T) { if all, _ := r.All(ctx); len(all) != 0 { t.Errorf("reservations left: %v", all) } - if again, _ := r.Reserve(ctx, "m", "0.1", arts); again.Version != "0.1.20" { + if again, _ := r.Reserve(ctx, "m", "0.1", arts, Source{}); again.Version != "0.1.20" { t.Logf("with the catalog list unchanged the next is %s", again.Version) } if ok, _ := r.Drop(ctx, "m", "0.1"); !ok { @@ -118,7 +118,7 @@ func TestReservationIsReadAndMarkedPublished(t *testing.T) { ctx := context.Background() store := &reserveMem{mem: newMem()} r := &Reserver{Store: store, ReleasedOf: func(context.Context, string) ([]string, error) { return []string{"1.4.2"}, nil }} - v, err := r.Reserve(ctx, "m", "1.4", []string{"m:a", "m:b"}) + v, err := r.Reserve(ctx, "m", "1.4", []string{"m:a", "m:b"}, Source{}) if err != nil || v.Version != "1.4.3" { t.Fatalf("reserved %q (%v), want 1.4.3", v.Version, err) } diff --git a/modules/mesh-delivery-builder/internal/versions/tag.go b/modules/mesh-delivery-builder/internal/versions/tag.go new file mode 100644 index 00000000..e83bff9e --- /dev/null +++ b/modules/mesh-delivery-builder/internal/versions/tag.go @@ -0,0 +1,4 @@ +package versions + +// GoTagOf is the git tag a Go package's released version is written as (design 47 G §370, design 51 *The versions*). +func GoTagOf(version string) string { return "go/v" + version } diff --git a/modules/mesh-delivery-builder/manifest_test.go b/modules/mesh-delivery-builder/manifest_test.go index d03eb00c..6fe5c6b9 100644 --- a/modules/mesh-delivery-builder/manifest_test.go +++ b/modules/mesh-delivery-builder/manifest_test.go @@ -132,6 +132,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 // The watchdogs seat's presence says a node's machine is heard (issue 1966, ADR 0596 §2, ADR 0668 §3). if want := []string{"mesh-delivery-assigner.assigner_read_assignments", "mesh-delivery-forge-bridge.bridge_tags", "mesh-delivery-indexer.indexer_read_candidates", "mesh-delivery-indexer.indexer_read_manifests", "mesh-delivery-indexer.indexer_read_modules", "mesh-delivery-indexer.indexer_read_packages", "mesh-delivery-indexer.indexer_requirements", "seat:container-image-registry.remove", "seat:container-image-registry.versions", + "seat:git.tag", "seat:go-package-registry.remove", "seat:go-package-registry.versions", "seat:mesh-artifact-sweep.held", "seat:mesh-catalog.versions", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.drop", "seat:node-build-image.drop", "seat:node-build-node.drop", "seat:node-engine.manifest-reads", "seat:npm-package-registry.remove", "seat:npm-package-registry.versions", "seat:oci-artifact-registry.remove", "seat:oci-artifact-registry.versions"}; !reflect.DeepEqual(sorted(m.Invokes), want) || len(m.Config) != 0 { t.Errorf("invokes = %v, config = %v, want %v and no config", m.Invokes, m.Config, want) diff --git a/modules/mesh-delivery-builder/module.json b/modules/mesh-delivery-builder/module.json index 39a0cc6b..8ed6a82d 100644 --- a/modules/mesh-delivery-builder/module.json +++ b/modules/mesh-delivery-builder/module.json @@ -77,6 +77,7 @@ "mesh-delivery-indexer.indexer_requirements", "seat:container-image-registry.remove", "seat:container-image-registry.versions", + "seat:git.tag", "seat:go-package-registry.remove", "seat:go-package-registry.versions", "seat:mesh-artifact-sweep.held",