diff --git a/contracts/go/clients/meshcatalogclient/catalog.go b/contracts/go/clients/meshcatalogclient/catalog.go new file mode 100644 index 00000000..95403eb4 --- /dev/null +++ b/contracts/go/clients/meshcatalogclient/catalog.go @@ -0,0 +1,242 @@ +package meshcatalogclient + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "maps" + "strconv" + "strings" + "sync" + "time" + + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-sdk/go/buscalls" + "git.novox.be/novox/mesh-sdk/go/keys" + "git.novox.be/novox/mesh-sdk/go/views" +) + +// RevisionVerb is the seat verb a part takes its first view of the catalog half by, as its manifest's invokes names it. +const RevisionVerb = "seat:" + seat.Seat + "." + seat.VerbRevision + +// The kinds a catalog graph revision is of. +const ( + KindTrunk = "trunk" + KindCandidate = "candidate" +) + +// Half is one catalog graph revision whole: the revision with its entries by module. +type Half struct { + Revision seat.CatalogRevision + Entries map[string]seat.CatalogEntry +} + +// ViewName is how the catalog half's revision is named in a part's first-view status (views.Status.View, WaitsFor). +func ViewName(revision int64) string { return strconv.FormatInt(revision, 10) } + +// catalogCreated stands for the view's Created in its follower: a revision view is never made again, so one fixed time, +// never zero, lets the follower tell a revision heard from one it holds. +var catalogCreated = time.Unix(0, 0).UTC() + +// Catalog is a part's view of the catalog half of one watched branch (design 47 G §358, §479): a first view of the +// branch's newest trunk revision asked of the seat's verb `revision`, then kept from its event `catalog-revision` of that +// branch, built on go/views' Follower, so the runtime's tools `_first_views` and `_first_view_again` read +// and steer it as every first view. The view is named `mesh-catalog.catalog-revisions.`, so a part that watches two +// branches makes two Catalogs, two views. A `catalog-revision` the view missed (a revision heard that it does not hold) +// asks `revision` once more, said in the log (§196, ADR 0615 §5), and is waited for, said in the status and past +// views.WaitsBound raised as view-waits. +type Catalog struct { + Follower *views.Follower + // Branch is the watched branch the view is of, as the caller names it. + Branch string + + call Call + mu sync.Mutex + // held is the half the view holds; paging the one being read, page by page, of one revision. + held *Half + paging *Half + // waits is the revision waited for, 0 when none. + waits int64 +} + +// NewCatalog is a Catalog of branch asking the seat through call. Nothing is asked before Start. +func NewCatalog(call Call, branch string) (*Catalog, error) { + if err := keys.CheckName("branch", branch); err != nil { + return nil, err + } + c := &Catalog{call: call, Branch: branch} + c.Follower = &views.Follower{Owner: seat.Seat, Tool: RevisionVerb, State: views.BranchState(seat.StateCatalogRevisions, branch), Ask: c.ask} + return c, nil +} + +// WireCatalog is a Catalog of branch adopted by a stdio part's views.Reader: the reader writes its conditions and its +// view-waits finding, reads it as `mesh-catalog.catalog-revisions.` (a key a module, its value the module's +// entry), lists and asks it again in its first-view tools, and at its Start subscribes the seat's event catalog-revision +// (the part's manifest consumes it) into Hear and asks the first view. changed, when not nil, is the Follower's Changed. +// The part declares the seat's verb revision in invokes. Call it once per watched branch. +func WireCatalog(r *views.Reader, call Call, branch string, changed func(module string, entry json.RawMessage, deleted bool, at time.Time)) (*Catalog, error) { + c, err := NewCatalog(call, branch) + if err != nil { + return nil, err + } + c.Follower.Changed = changed + if err := r.Adopt(c.Follower, seat.Seat, seat.EventCatalogRevision, c.Hear); err != nil { + return nil, err + } + return c, nil +} + +// WireCatalogProcess is WireCatalog for a process's views.ProcessReader: its picture +// (`mesh-catalog_catalog-revisions.`) reads the half as a read-only bucket, and the seat's event catalog-revision +// on the module's durable consumer goes to Hear. +func WireCatalogProcess(r *views.ProcessReader, call Call, branch string, changed func(module string, entry json.RawMessage, deleted bool, at time.Time)) (*Catalog, *views.Picture, error) { + c, err := NewCatalog(call, branch) + if err != nil { + return nil, nil, err + } + c.Follower.Changed = changed + p, err := r.Adopt(c.Follower, seat.Seat, seat.EventCatalogRevision, c.Hear) + if err != nil { + return nil, nil, err + } + return c, p, nil +} + +// Start asks the first view, the branch's newest trunk revision, until it is answered or ctx ends. +func (c *Catalog) Start(ctx context.Context) error { return c.Follower.Start(ctx) } + +// Ready is closed once the first view is answered. +func (c *Catalog) Ready() <-chan struct{} { return c.Follower.Ready() } + +// View is the half the view holds; false while it has none, when the part acts on nothing new. +func (c *Catalog) View() (Half, bool) { + c.mu.Lock() + defer c.mu.Unlock() + if c.held == nil { + return Half{}, false + } + return Half{Revision: c.held.Revision, Entries: maps.Clone(c.held.Entries)}, true +} + +// Entry is one module's entry in the half the view holds. +func (c *Catalog) Entry(module string) (seat.CatalogEntry, bool) { + c.mu.Lock() + defer c.mu.Unlock() + if c.held == nil { + return seat.CatalogEntry{}, false + } + e, ok := c.held.Entries[module] + return e, ok +} + +// Hear takes one `catalog-revision` event, as the part's durable consumer of the seat's events hands it. A candidate's +// revision and another branch's move nothing here; a trunk revision of this branch the view holds, or one older, is passed +// over; a newer one is a revision the view missed: waited for, said in the log, and asked again on its own goroutine, so +// the consumer is never held while the holder is away. Data that is no such event is an error. +func (c *Catalog) Hear(ctx context.Context, data []byte) error { + var ev seat.CatalogRevisionEvent + if err := json.Unmarshal(data, &ev); err != nil || ev.Revision <= 0 { + return fmt.Errorf("meshcatalogclient: not a %s event: %s", seat.EventCatalogRevision, clipBytes(data)) + } + if ev.Kind != KindTrunk || ev.Branch != keys.EscapeName(c.Branch) { + return nil + } + c.mu.Lock() + held := int64(0) + if c.held != nil { + held = c.held.Revision.Revision + } + if ev.Revision > c.waits { + c.waits = ev.Revision + } + c.mu.Unlock() + if ev.Revision <= held { + return nil + } + // The follower keeps the newest revision heard, so an ask that was running when it was heard is asked once more. + c.Follower.Apply(views.Event{Owner: seat.Seat, State: c.Follower.State, Revision: uint64(ev.Revision), Created: catalogCreated, Gap: true}) + c.Follower.WaitFor(ctx, ViewName(ev.Revision)) + if c.Follower.Logf != nil { + c.Follower.Logf("meshcatalogclient: catalog revision %d of %s said, the view holds %d: %s is asked once more", ev.Revision, c.Branch, held, RevisionVerb) + } + go func() { + _ = c.Follower.Gap(ctx, fmt.Sprintf("catalog revision %d of %s said, the view holds %d", ev.Revision, c.Branch, held)) + }() + return nil +} + +// ask is the Follower's Ask: one page of the branch's newest trunk revision, the next pages of the same revision; once +// whole, it is the half the view holds. +func (c *Catalog) ask(_ context.Context, after string) (views.Answer, error) { + in := seat.RevisionInput{After: after, Branch: keys.EscapeName(c.Branch)} + c.mu.Lock() + if after != "" && c.paging != nil { + in.Revision = c.paging.Revision.Revision + } + c.mu.Unlock() + a, err := askRevision(c.call, in) + if errors.Is(err, errNoneYet) { + // A branch with no revision yet is ready and empty, and nothing waits for it (§479, §495). + return views.Answer{V: 1, Owner: seat.Seat, State: c.Follower.State, Created: catalogCreated, Values: map[string]json.RawMessage{}, At: time.Now()}, nil + } + if err != nil { + return views.Answer{}, err + } + if a.Revision.Kind != KindTrunk { + return views.Answer{}, fmt.Errorf("%s answered a %s revision for the trunk", RevisionVerb, a.Revision.Kind) + } + c.mu.Lock() + if after == "" || c.paging == nil { + c.paging = &Half{Revision: a.Revision, Entries: map[string]seat.CatalogEntry{}} + } else if c.paging.Revision.Revision != a.Revision.Revision { + was := c.paging.Revision.Revision + c.paging = nil + c.mu.Unlock() + return views.Answer{}, fmt.Errorf("%s answered revision %d for a page of revision %d", RevisionVerb, a.Revision.Revision, was) + } + values := map[string]json.RawMessage{} + for _, e := range a.Entries { + c.paging.Entries[e.Module] = e + raw, _ := json.Marshal(e) + values[e.Module] = raw + } + rev := a.Revision.Revision + holds := false + if a.Next == "" { + c.held, c.paging = c.paging, nil + if c.waits <= rev { + c.waits, holds = 0, true + } + } + c.mu.Unlock() + if holds { + c.Follower.Holds(ViewName(rev)) + } + return views.Answer{V: 1, Owner: seat.Seat, State: c.Follower.State, Revision: uint64(rev), Created: catalogCreated, Values: values, Next: a.Next, At: time.Now()}, nil +} + +// errNoneYet is the seat's answer `none yet`: the branch has no revision yet. +var errNoneYet = errors.New("none yet") + +func askRevision(call Call, in seat.RevisionInput) (seat.RevisionAnswer, error) { + raw, err := call(RevisionVerb, in) + if err != nil { + if strings.Contains(err.Error(), views.NoneYet) { + return seat.RevisionAnswer{}, errNoneYet + } + return seat.RevisionAnswer{}, err + } + answer, said, ok := buscalls.Unwrap(raw) + if !ok { + if strings.Contains(said, views.NoneYet) { + return seat.RevisionAnswer{}, errNoneYet + } + return seat.RevisionAnswer{}, fmt.Errorf("%s answered: %s", RevisionVerb, said) + } + var a seat.RevisionAnswer + if err := json.Unmarshal(answer, &a); err != nil || a.Revision.Revision <= 0 { + return seat.RevisionAnswer{}, fmt.Errorf("%s answered no revision: %s", RevisionVerb, clipBytes(answer)) + } + return a, nil +} diff --git a/contracts/go/clients/meshcatalogclient/catalog_view_test.go b/contracts/go/clients/meshcatalogclient/catalog_view_test.go new file mode 100644 index 00000000..7af15ae0 --- /dev/null +++ b/contracts/go/clients/meshcatalogclient/catalog_view_test.go @@ -0,0 +1,164 @@ +package meshcatalogclient + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "sync" + "testing" + "time" + + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-sdk/go/keys" + "git.novox.be/novox/mesh-sdk/go/views" +) + +// fakeRevisions answers the verb revision per branch from the newest revision each branch holds, and counts each ask. +type fakeRevisions struct { + mu sync.Mutex + newest map[string]int64 // by escaped branch + asks map[string]int + branches []string +} + +func (f *fakeRevisions) call(key string, body any) (json.RawMessage, error) { + in := body.(seat.RevisionInput) + f.mu.Lock() + defer f.mu.Unlock() + rev, ok := f.newest[in.Branch] + if ok && rev == 0 { + return nil, fmt.Errorf("revision of %s: none yet", in.Branch) + } + if !ok { + return nil, fmt.Errorf("asked for the branch %q", in.Branch) + } + f.asks[in.Branch]++ + return json.Marshal(seat.RevisionAnswer{ + Revision: seat.CatalogRevision{Revision: rev, Kind: KindTrunk, Branch: in.Branch}, + Entries: []seat.CatalogEntry{{Module: fmt.Sprintf("m-of-%s-%d", in.Branch, rev)}}, + }) +} + +func (f *fakeRevisions) set(branch string, rev int64) { + f.mu.Lock() + defer f.mu.Unlock() + f.newest[keys.EscapeName(branch)] = rev +} + +func (f *fakeRevisions) asked(branch string) int { + f.mu.Lock() + defer f.mu.Unlock() + return f.asks[keys.EscapeName(branch)] +} + +func revisionEvent(branch, kind string, rev int64) []byte { + b, _ := json.Marshal(seat.CatalogRevisionEvent{V: 1, Branch: keys.EscapeName(branch), Kind: kind, Revision: rev}) + return b +} + +// CT10 (design 47 G §358, §479, §196): the catalog half is a view per watched branch, named by its caller: two branches +// give two views, each named `mesh-catalog.catalog-revisions.`, each following only its own branch's +// `catalog-revision`; a revision the view missed asks `revision` once more, said in the log. +func TestCatalogIsAViewPerWatchedBranch(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + f := &fakeRevisions{newest: map[string]int64{"main": 3, keys.EscapeName("release/1"): 7}, asks: map[string]int{}} + var logMu sync.Mutex + var logged []string + logf := func(format string, args ...any) { + logMu.Lock() + defer logMu.Unlock() + logged = append(logged, fmt.Sprintf(format, args...)) + } + main, err := NewCatalog(f.call, "main") + if err != nil { + t.Fatal(err) + } + rel, err := NewCatalog(f.call, "release/1") + if err != nil { + t.Fatal(err) + } + for _, c := range []*Catalog{main, rel} { + c.Follower.Wait = time.Nanosecond + c.Follower.Logf = logf + } + if main.Follower.State == rel.Follower.State || main.Follower.State != "catalog-revisions.main" || rel.Follower.State != views.BranchState(seat.StateCatalogRevisions, "release/1") { + t.Fatalf("views are named %q and %q", main.Follower.State, rel.Follower.State) + } + for _, c := range []*Catalog{main, rel} { + if err := c.Start(ctx); err != nil { + t.Fatal(err) + } + <-c.Ready() + } + if h, ok := main.View(); !ok || h.Revision.Revision != 3 || len(h.Entries) != 1 { + t.Fatalf("main holds %+v, %v", h, ok) + } + if h, ok := rel.View(); !ok || h.Revision.Revision != 7 { + t.Fatalf("release/1 holds %+v, %v", h, ok) + } + + // An event of the other branch, a candidate's and one the view holds ask nothing. + for _, data := range [][]byte{revisionEvent("release/1", KindTrunk, 8), revisionEvent("main", KindCandidate, 4), revisionEvent("main", KindTrunk, 3)} { + if err := main.Hear(ctx, data); err != nil { + t.Fatal(err) + } + } + if n := f.asked("main"); n != 1 { + t.Errorf("main asked %d times, want only its first view", n) + } + + // A missed revision asks `revision` once more, and is said. + f.set("main", 4) + if err := main.Hear(ctx, revisionEvent("main", KindTrunk, 4)); err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(5 * time.Second) + for { + if h, _ := main.View(); h.Revision.Revision == 4 { + break + } + if time.Now().After(deadline) { + t.Fatal("main never held revision 4") + } + time.Sleep(time.Millisecond) + } + if n := f.asked("release/1"); n != 1 { + t.Errorf("release/1 asked %d times after main's revision", n) + } + logMu.Lock() + said := false + for _, l := range logged { + if strings.Contains(l, "catalog revision 4 of main said") { + said = true + } + } + logMu.Unlock() + if !said { + t.Errorf("the gap was not said in the log: %q", logged) + } + + // A branch with no revision yet is ready and empty, and nothing waits for it. + f.set("new", 0) + none, err := NewCatalog(f.call, "new") + if err != nil { + t.Fatal(err) + } + if err := none.Start(ctx); err != nil { + t.Fatal(err) + } + <-none.Ready() + if _, ok := none.View(); ok || none.Follower.Status().WaitsFor != "" { + t.Errorf("a branch with no revision yet holds a view or waits: %+v", none.Follower.Status()) + } + + for _, data := range []string{`{}`, `not json`} { + if err := main.Hear(ctx, []byte(data)); err == nil { + t.Errorf("Hear(%s) took it", data) + } + } + if _, err := NewCatalog(f.call, ""); err == nil { + t.Error("a Catalog of no branch was made") + } +} diff --git a/contracts/go/seats/meshbuilds/contract.go b/contracts/go/seats/meshbuilds/contract.go index 76437fcc..f95ae661 100644 --- a/contracts/go/seats/meshbuilds/contract.go +++ b/contracts/go/seats/meshbuilds/contract.go @@ -77,11 +77,19 @@ type PackagePublished struct { Version string `json:"version"` } +// ToolchainAgreedDigestsItem is a type of the contract. +type ToolchainAgreedDigestsItem struct { + // the toolchain part's digest + Digest string `json:"digest"` + // the nodes reporting that digest + Nodes []string `json:"nodes"` +} + // ToolchainAgreed is the agreed toolchain of a language changed (design 47 G section 376, ADR 0584 section 1). type ToolchainAgreed struct { At *time.Time `json:"at,omitempty"` - // the toolchain part's digest - Digest string `json:"digest"` + // each digest the nodes reporting the agreed version report, as it stood when the version was agreed, with those nodes (design 47 G section 472, section 495) + Digests []ToolchainAgreedDigestsItem `json:"digests"` // the language whose agreed toolchain changed Language string `json:"language"` V int64 `json:"v"` @@ -91,8 +99,10 @@ type ToolchainAgreed struct { // ReservedInput is what reserved takes. type ReservedInput struct { - // the module - Module string `json:"module"` + // the module; give this or name + Module string `json:"module,omitempty"` + // a package's or an image's name in its ecosystem (the Go module path, the npm name, the image name), in place of module; a name no entry carries, or more than one entry carries, is answered error naming it (design 47 G section 478, section 491) + Name string `json:"name,omitempty"` // the version, when asked for a particular one Version string `json:"version,omitempty"` } diff --git a/contracts/go/seats/meshbuilds/contract_test.go b/contracts/go/seats/meshbuilds/contract_test.go index abbc1cf1..65fb7dfc 100644 --- a/contracts/go/seats/meshbuilds/contract_test.go +++ b/contracts/go/seats/meshbuilds/contract_test.go @@ -19,10 +19,12 @@ func TestFixtures(t *testing.T) { }{ {name: "a package published with the dependencies the build resolved", subject: "mesh.seat.mesh-builds.event.package-published", given: "{\"artifact\":\"mesh-sdk:go\",\"at\":\"2026-11-10T10:00:00Z\",\"branch\":\"main\",\"dependencies\":[{\"artifact\":\"mesh-contracts:go-nodeengine\",\"digest\":\"sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"kind\":1,\"version\":\"0.1.4\"},{\"artifact\":\"mesh-build-go:toolchain\",\"digest\":\"sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"kind\":3,\"version\":\"1.26.0\"}],\"digest\":\"sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"module\":\"mesh-sdk\",\"source\":3589,\"v\":1,\"version\":\"0.1.15\"}", wire: "{\"artifact\":\"mesh-sdk:go\",\"at\":\"2026-11-10T10:00:00Z\",\"branch\":\"main\",\"dependencies\":[{\"artifact\":\"mesh-contracts:go-nodeengine\",\"digest\":\"sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"kind\":1,\"version\":\"0.1.4\"},{\"artifact\":\"mesh-build-go:toolchain\",\"digest\":\"sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"kind\":3,\"version\":\"1.26.0\"}],\"digest\":\"sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"module\":\"mesh-sdk\",\"source\":3589,\"v\":1,\"version\":\"0.1.15\"}", decode: func(b []byte) (any, error) { var v PackagePublishedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-builds\\.event\\.package-published$")}, - {name: "a toolchain agreed", subject: "mesh.seat.mesh-builds.event.toolchain-agreed", given: "{\"at\":\"2026-11-10T10:00:00Z\",\"digest\":\"sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"language\":\"go\",\"v\":1,\"version\":\"1.26.0\"}", wire: "{\"at\":\"2026-11-10T10:00:00Z\",\"digest\":\"sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"language\":\"go\",\"v\":1,\"version\":\"1.26.0\"}", + {name: "a toolchain agreed", subject: "mesh.seat.mesh-builds.event.toolchain-agreed", given: "{\"at\":\"2026-11-10T10:00:00Z\",\"digests\":[{\"digest\":\"sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"nodes\":[\"ace\",\"g14\"]},{\"digest\":\"sha256:fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210\",\"nodes\":[\"novox\"]}],\"language\":\"go\",\"v\":1,\"version\":\"1.26.0\"}", wire: "{\"at\":\"2026-11-10T10:00:00Z\",\"digests\":[{\"digest\":\"sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"nodes\":[\"ace\",\"g14\"]},{\"digest\":\"sha256:fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210\",\"nodes\":[\"novox\"]}],\"language\":\"go\",\"v\":1,\"version\":\"1.26.0\"}", decode: func(b []byte) (any, error) { var v ToolchainAgreedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-builds\\.event\\.toolchain-agreed$")}, {name: "reserved asks a module", subject: "", given: "{\"module\":\"mesh-sdk\"}", wire: "{\"module\":\"mesh-sdk\"}", decode: func(b []byte) (any, error) { var v ReservedInput; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, + {name: "reserved asks a package by its name", subject: "", given: "{\"name\":\"git.novox.be/novox/mesh-sdk/go\",\"version\":\"0.1.15\"}", wire: "{\"name\":\"git.novox.be/novox/mesh-sdk/go\",\"version\":\"0.1.15\"}", + decode: func(b []byte) (any, error) { var v ReservedInput; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, {name: "reserved answers the version and each artifact's digest", subject: "", given: "{\"artifacts\":[{\"artifact\":\"mesh-sdk:go\",\"digest\":\"sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\"},{\"artifact\":\"mesh-sdk:ts\"}],\"module\":\"mesh-sdk\",\"version\":\"0.1.15\"}", wire: "{\"artifacts\":[{\"artifact\":\"mesh-sdk:go\",\"digest\":\"sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\"},{\"artifact\":\"mesh-sdk:ts\"}],\"module\":\"mesh-sdk\",\"version\":\"0.1.15\"}", decode: func(b []byte) (any, error) { var v ReservedAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, } diff --git a/contracts/go/seats/meshpipelinerunner/contract.go b/contracts/go/seats/meshpipelinerunner/contract.go index 39817255..7c17dd91 100644 --- a/contracts/go/seats/meshpipelinerunner/contract.go +++ b/contracts/go/seats/meshpipelinerunner/contract.go @@ -675,6 +675,8 @@ type Wait struct { Artifact string `json:"artifact,omitempty"` // the watched branch, or the branch of a run on request Branch string `json:"branch"` + // the time the bound counts from (design 47 G section 266, section 269, section 498) + ClockFrom *time.Time `json:"clock-from,omitempty"` // the commit the module's entry was last changed at Commit string `json:"commit,omitempty"` // the module's build-input fingerprint at the commit diff --git a/contracts/go/seats/meshwants/contract.go b/contracts/go/seats/meshwants/contract.go index 97003e33..ad44c028 100644 --- a/contracts/go/seats/meshwants/contract.go +++ b/contracts/go/seats/meshwants/contract.go @@ -39,6 +39,7 @@ const ( EventRoom = "room" EventStepEnded = "step-ended" EventVersionDelivered = "version-delivered" + EventStepProgress = "step-progress" ) // Verbs is every verb the seat serves, in its definition's order: a new copy each call. @@ -48,7 +49,7 @@ func Verbs() []string { // Events is every event published on the seat's subject: a new copy each call. func Events() []string { - return []string{EventPolicyChanged, EventAssignmentRevision, EventDifference, EventWanted, EventWaitHeld, EventWaitEnded, EventFilling, EventFull, EventRoom, EventStepEnded, EventVersionDelivered} + return []string{EventPolicyChanged, EventAssignmentRevision, EventDifference, EventWanted, EventWaitHeld, EventWaitEnded, EventFilling, EventFull, EventRoom, EventStepEnded, EventVersionDelivered, EventStepProgress} } // Family is where the seat's events are kept: the shared events stream, filtered to the seat's subjects. @@ -99,6 +100,17 @@ const StreamVersionDelivered = "SEAT_mesh-wants_version-delivered" // SubjectVersionDelivered is where version-delivered is published. const SubjectVersionDelivered = "mesh.seat.mesh-wants.event.version-delivered" +// StreamStepProgress is the stream step-progress is kept in. +const StreamStepProgress = "SEAT_mesh-wants_step-progress" + +// SubjectStepProgress is where step-progress is published, on mesh.seat.mesh-wants.event.step-progress.. +func SubjectStepProgress(node string) string { + return "mesh.seat.mesh-wants.event.step-progress." + node +} + +// SubjectStepProgressAll hears every step-progress. +const SubjectStepProgressAll = "mesh.seat.mesh-wants.event.step-progress.*" + // AcceptSubjectStep is where step is sent. const AcceptSubjectStep = "mesh.seat.mesh-wants.accept.step" @@ -647,20 +659,43 @@ type Wait struct { Bound string `json:"bound,omitempty"` // on a release, who released it and why By string `json:"by,omitempty"` - Fp string `json:"fp,omitempty"` + // the time the bound counts from (design 47 G section 266, section 269, section 498) + ClockFrom *time.Time `json:"clock-from,omitempty"` + // a put's wait: the fingerprint of the build put + Fingerprint string `json:"fingerprint,omitempty"` + Fp string `json:"fp,omitempty"` // on wait-ended: ran or released How string `json:"how,omitempty"` + // a dependent's delivery wait: the machines the dependency's version has yet to reach + Machines []string `json:"machines,omitempty"` // the entry that waits Module string `json:"module"` Node string `json:"node,omitempty"` // the acting part waited for - Part string `json:"part"` + Part string `json:"part,omitempty"` Since time.Time `json:"since"` - Step string `json:"step"` + // which wait: an acting step of section 203 or put, delivery, kept-build, backup (design 47 G section 498) + Step string `json:"step"` // its new version's fingerprint - Version string `json:"version"` + Version string `json:"version,omitempty"` } +// The values of Wait's step. +const ( + WaitStepTakeIn = "take-in" + WaitStepValidate = "validate" + WaitStepCheck = "check" + WaitStepBuild = "build" + WaitStepCompose = "compose" + WaitStepGrant = "grant" + WaitStepPlace = "place" + WaitStepPipeline = "pipeline" + WaitStepPut = "put" + WaitStepDelivery = "delivery" + WaitStepKeptBuild = "kept-build" + WaitStepBackup = "backup" +) + // UpgradeInput is what upgrade takes. type UpgradeInput struct { // "true": each node's own composition, by its digest, and the assigner's findings @@ -823,6 +858,22 @@ type VersionDeliveredEvent struct { Version string `json:"version"` } +// StepProgressEvent is the body of the event step-progress. +type StepProgressEvent struct { + Ask string `json:"ask"` + Asker string `json:"asker"` + At time.Time `json:"at"` + Phase string `json:"phase"` +} + +// The values of StepProgressEvent's phase. +const ( + StepProgressEventPhaseFetch = "fetch" + StepProgressEventPhaseBuild = "build" + StepProgressEventPhaseRun = "run" + StepProgressEventPhaseReport = "report" +) + // WantsByWantNodeModuleValue is one value of the state wants under the key want... type WantsByWantNodeModuleValue struct { // again or skip diff --git a/contracts/go/seats/meshwants/contract_test.go b/contracts/go/seats/meshwants/contract_test.go index dcfe1e6c..b0624c3e 100644 --- a/contracts/go/seats/meshwants/contract_test.go +++ b/contracts/go/seats/meshwants/contract_test.go @@ -43,6 +43,12 @@ func TestFixtures(t *testing.T) { decode: func(b []byte) (any, error) { var v StepEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-wants\\.event\\.step-ended\\.[^.*> ]+$")}, {name: "a version delivered with the machines away", subject: "mesh.seat.mesh-wants.event.version-delivered", given: "{\"away\":[\"laptop\"],\"module\":\"mesh-sdk\",\"parts\":{\"mesh-sdk:go\":\"sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\"},\"v\":1,\"version\":\"0.1.15\"}", wire: "{\"away\":[\"laptop\"],\"module\":\"mesh-sdk\",\"parts\":{\"mesh-sdk:go\":\"sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\"},\"v\":1,\"version\":\"0.1.15\"}", decode: func(b []byte) (any, error) { var v VersionDeliveredEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-wants\\.event\\.version-delivered$")}, + {name: "a composition step's progress", subject: "mesh.seat.mesh-wants.event.step-progress.ace", given: "{\"ask\":\"a1b2\",\"asker\":\"assigner\",\"at\":\"2026-11-11T10:01:00Z\",\"phase\":\"build\"}", wire: "{\"ask\":\"a1b2\",\"asker\":\"assigner\",\"at\":\"2026-11-11T10:01:00Z\",\"phase\":\"build\"}", + decode: func(b []byte) (any, error) { var v StepProgressEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-wants\\.event\\.step-progress\\.[^.*> ]+$")}, + {name: "a put without hold waits, counted from its start", subject: "mesh.seat.mesh-wants.event.wait-held", given: "{\"bound\":\"30m\",\"clock-from\":\"2026-11-11T10:00:00Z\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"ace\",\"since\":\"2026-11-11T10:00:00Z\",\"step\":\"put\"}", wire: "{\"bound\":\"30m\",\"clock-from\":\"2026-11-11T10:00:00Z\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"ace\",\"since\":\"2026-11-11T10:00:00Z\",\"step\":\"put\"}", + decode: func(b []byte) (any, error) { var v WaitHeldEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-wants\\.event\\.wait-held$")}, + {name: "a dependent's delivery waits for a dependency's version", subject: "mesh.seat.mesh-wants.event.wait-held", given: "{\"bound\":\"60m\",\"clock-from\":\"2026-11-11T09:58:00Z\",\"machines\":[\"ace\",\"g14\"],\"module\":\"nats\",\"part\":\"mesh-sdk\",\"since\":\"2026-11-11T10:00:00Z\",\"step\":\"delivery\",\"version\":\"0.1.4\"}", wire: "{\"bound\":\"60m\",\"clock-from\":\"2026-11-11T09:58:00Z\",\"machines\":[\"ace\",\"g14\"],\"module\":\"nats\",\"part\":\"mesh-sdk\",\"since\":\"2026-11-11T10:00:00Z\",\"step\":\"delivery\",\"version\":\"0.1.4\"}", + decode: func(b []byte) (any, error) { var v WaitHeldEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-wants\\.event\\.wait-held$")}, {name: "a want", subject: "", given: "{\"fp\":\"ab12\",\"rule\":3,\"v\":1}", wire: "{\"fp\":\"ab12\",\"rule\":3,\"v\":1}", decode: func(b []byte) (any, error) { var v WantsByWantNodeModuleValue diff --git a/contracts/go/seats/nodebuildgo/contract.go b/contracts/go/seats/nodebuildgo/contract.go index 34785b37..7b1eaece 100644 --- a/contracts/go/seats/nodebuildgo/contract.go +++ b/contracts/go/seats/nodebuildgo/contract.go @@ -33,7 +33,9 @@ const ( ) const ( - EventStepEnded = "step-ended" + EventStepEnded = "step-ended" + EventToolchainReported = "toolchain-reported" + EventStepProgress = "step-progress" ) // Verbs is every verb the seat serves, in its definition's order: a new copy each call. @@ -43,7 +45,7 @@ func Verbs() []string { // Events is every event published on the seat's subject: a new copy each call. func Events() []string { - return []string{EventStepEnded} + return []string{EventStepEnded, EventToolchainReported, EventStepProgress} } // Family is where the seat's events are kept: the shared events stream, filtered to the seat's subjects. @@ -61,6 +63,28 @@ func SubjectStepEnded(node string) string { return "mesh.seat.node-build-go.even // SubjectStepEndedAll hears every step-ended. const SubjectStepEndedAll = "mesh.seat.node-build-go.event.step-ended.*" +// StreamToolchainReported is the stream toolchain-reported is kept in. +const StreamToolchainReported = "SEAT_node-build-go_toolchain-reported" + +// SubjectToolchainReported is where toolchain-reported is published, on mesh.seat.node-build-go.event.toolchain-reported.. +func SubjectToolchainReported(node string) string { + return "mesh.seat.node-build-go.event.toolchain-reported." + node +} + +// SubjectToolchainReportedAll hears every toolchain-reported. +const SubjectToolchainReportedAll = "mesh.seat.node-build-go.event.toolchain-reported.*" + +// StreamStepProgress is the stream step-progress is kept in. +const StreamStepProgress = "SEAT_node-build-go_step-progress" + +// SubjectStepProgress is where step-progress is published, on mesh.seat.node-build-go.event.step-progress.. +func SubjectStepProgress(node string) string { + return "mesh.seat.node-build-go.event.step-progress." + node +} + +// SubjectStepProgressAll hears every step-progress. +const SubjectStepProgressAll = "mesh.seat.node-build-go.event.step-progress.*" + // AcceptSubjectDelivery is where delivery is sent. const AcceptSubjectDelivery = "mesh.seat.node-build-go.accept.delivery" @@ -86,6 +110,47 @@ const ( StateDetectedSteps = "detected-steps" ) +// StepsByAskerNodeIDKey is the key of one value of the state steps under the form "..". +func StepsByAskerNodeIDKey(asker, node, id string) string { return asker + "." + node + "." + id } + +// ParseStepsByAskerNodeIDKey reads the parts of a key of the form "..", in the order of the form; the last result is false for a key of another form. +func ParseStepsByAskerNodeIDKey(key string) (string, string, string, bool) { + rest := key + v0, after0, found0 := strings.Cut(rest, ".") + if !found0 || v0 == "" { + return "", "", "", false + } + rest = after0 + v2, after2, found2 := strings.Cut(rest, ".") + if !found2 || v2 == "" { + return "", "", "", false + } + rest = after2 + if rest == "" { + return "", "", "", false + } + v4 := rest + rest = "" + if rest != "" { + return "", "", "", false + } + return v0, v2, v4, true +} + +// DecodeStepsByAskerNodeIDValue reads one value of the state steps under the form "..", refusing a field StepsByAskerNodeIDValue does not know and any text after the value. +func DecodeStepsByAskerNodeIDValue(raw []byte) (StepsByAskerNodeIDValue, error) { + var v StepsByAskerNodeIDValue + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.DisallowUnknownFields() + if err := dec.Decode(&v); err != nil { + return v, err + } + if dec.More() { + return v, errors.New("text after the value") + } + return v, nil +} + // StepsByNodeIDKey is the key of one value of the state steps under the form ".". func StepsByNodeIDKey(node, id string) string { return node + "." + id } @@ -317,11 +382,21 @@ type BuildAnswer struct { Toolchain *ToolchainUsed `json:"toolchain,omitempty"` } +// BuiltArtifactResolvedItem is a type of the contract. +type BuiltArtifactResolvedItem struct { + // sha256: of the package's bytes as the build fetched them + Digest string `json:"digest"` + Package string `json:"package"` + Version string `json:"version"` +} + // BuiltArtifact is one artifact a build made: its digest and the reference it was published under. type BuiltArtifact struct { Artifact string `json:"artifact"` Digest string `json:"digest"` Reference string `json:"reference,omitempty"` + // one per mesh package the build resolved (the catalog graph entry's packages, the edges of kind 1) + Resolved []BuiltArtifactResolvedItem `json:"resolved,omitempty"` } // Capacity is how many steps the node runs at once. @@ -396,6 +471,16 @@ type FetchBesideItem struct { Repository string `json:"repository"` } +// FetchBuildsItem is a type of the contract. +type FetchBuildsItem struct { + Artifact string `json:"artifact"` + // the member's build digest, sha256: + Digest string `json:"digest"` + Module string `json:"module"` + // the name in its ecosystem (the Go module path, the npm name) + Package string `json:"package"` +} + // FetchPackage is the TypeScript module's group package: the group's tagged version of the SDK ({name, version}), placed from the registry in each member whatever its lock names, or the group's package member ({repository, commit, ref}), packed by the fetch and served to the members in place of the registry's. type FetchPackage struct { Commit string `json:"commit,omitempty"` @@ -411,6 +496,8 @@ type Fetch struct { Base string `json:"base,omitempty"` // the sibling repositories placed beside the fetched tree Beside []FetchBesideItem `json:"beside,omitempty"` + // a group run's package members, each by its build's digest: the build module reads each from the artifact store at artifact-store:////blobs/, checks the bytes against the digest and lays them as the group package; a package named here is named in neither packages nor package + Builds []FetchBuildsItem `json:"builds,omitempty"` // the full commit Commit string `json:"commit"` // the module's directory in the repository ("." for its root), with paths @@ -735,6 +822,22 @@ type StepNames struct { Repositories []string `json:"repositories"` } +// StepProgress is a running ask's progress, said once a minute while the holder runs it (design 47 G section 496). +type StepProgress struct { + Ask string `json:"ask"` + Asker string `json:"asker"` + At time.Time `json:"at"` + Phase string `json:"phase"` +} + +// The values of StepProgress's phase. +const ( + StepProgressPhaseFetch = "fetch" + StepProgressPhaseBuild = "build" + StepProgressPhaseRun = "run" + StepProgressPhaseReport = "report" +) + // StepResult is what a step answered, by the kind of ask: a build's artifacts, toolchain, imports, out-of-step modules and the artifact it kept; a check's fetch and levels; a compose's built, key and verdict; a run's built, key, exit, output and said; a declined ask's declined, needs and lacks. The fields of the kinds do not meet, so one layout holds the five. type StepResult struct { Artifacts []BuiltArtifact `json:"artifacts,omitempty"` @@ -828,8 +931,20 @@ type Toolchain struct { Version string `json:"version"` } +// ToolchainReported is a node's toolchain of a language, said at the build module's start and each time its toolchains key of the language changes (design 47 G section 492). +type ToolchainReported struct { + // the digest of the program of the provision the build module requires for that language's toolchain, never the build module's own + Digest string `json:"digest"` + Language string `json:"language"` + Node string `json:"node"` + Recipe string `json:"recipe"` + Version string `json:"version"` +} + // ToolchainUsed is the toolchain a build was made with: the language, the installed version and the build module's recipe. type ToolchainUsed struct { + // the building node's digest of its toolchain program (kind 3's entry), never the build module's own program + Digest string `json:"digest,omitempty"` Language string `json:"language"` Recipe string `json:"recipe"` Version string `json:"version"` @@ -912,20 +1027,28 @@ const ( type StepsInput struct { // find them anew, not from the kept answer Again bool `json:"again,omitempty"` - // the artifact reference, : - Artifact string `json:"artifact"` - Fetch Fetch `json:"fetch"` + // the key of detected-steps, . + Artifact string `json:"artifact"` + // delete the artifact's kept key instead of detecting; needs why + Drop bool `json:"drop,omitempty"` + Fetch *Fetch `json:"fetch,omitempty"` Fingerprint string `json:"fingerprint,omitempty"` Module string `json:"module"` + // with drop, why: a non-empty text, written in the hand-act log with the caller the bus proved + Why string `json:"why,omitempty"` } // StepsAnswer is what steps answers. type StepsAnswer struct { - Artifact string `json:"artifact"` + Artifact string `json:"artifact,omitempty"` + // the artifact whose key a drop deleted + Dropped string `json:"dropped,omitempty"` + // why a drop deleted nothing, naming the hand-act log when it refused + Error string `json:"error,omitempty"` Fingerprint string `json:"fingerprint,omitempty"` // answered from the kept answer Kept bool `json:"kept,omitempty"` - Steps []DetectedStep `json:"steps"` + Steps []DetectedStep `json:"steps,omitempty"` } // DeliveryAccept is the body of delivery. @@ -949,6 +1072,15 @@ type CheckRunAccept = StepAsk // StepEndedEvent is the body of the event step-ended. type StepEndedEvent = StepEnded +// ToolchainReportedEvent is the body of the event toolchain-reported. +type ToolchainReportedEvent = ToolchainReported + +// StepProgressEvent is the body of the event step-progress. +type StepProgressEvent = StepProgress + +// StepsByAskerNodeIDValue is one value of the state steps under the key ... +type StepsByAskerNodeIDValue = Step + // StepsByNodeIDValue is one value of the state steps under the key .. type StepsByNodeIDValue = Step diff --git a/contracts/go/seats/nodebuildgo/contract_test.go b/contracts/go/seats/nodebuildgo/contract_test.go index b35c655d..14aca33e 100644 --- a/contracts/go/seats/nodebuildgo/contract_test.go +++ b/contracts/go/seats/nodebuildgo/contract_test.go @@ -21,6 +21,10 @@ func TestFixtures(t *testing.T) { decode: func(b []byte) (any, error) { var v StepEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.node-build-go\\.event\\.step-ended\\.[^.*> ]+$")}, {name: "an integration step that could not run, naming the part not heard", subject: "mesh.seat.node-build-go.event.step-ended.g14", given: "{\"artifact\":\"nats:program\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d5f\",\"module\":\"nats\",\"node\":\"g14\",\"outcome\":\"could not run\",\"output-object\":\"mesh-build-go_sent-objects/m2kx9a1b3c4d5f\",\"pipeline\":\"main.12.nats\",\"said\":\"the sandbox could not fetch novox/mesh-sdk\",\"step\":\"integration\",\"v\":1}", wire: "{\"artifact\":\"nats:program\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d5f\",\"module\":\"nats\",\"node\":\"g14\",\"outcome\":\"could not run\",\"output-object\":\"mesh-build-go_sent-objects/m2kx9a1b3c4d5f\",\"pipeline\":\"main.12.nats\",\"said\":\"the sandbox could not fetch novox/mesh-sdk\",\"step\":\"integration\",\"v\":1}", decode: func(b []byte) (any, error) { var v StepEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.node-build-go\\.event\\.step-ended\\.[^.*> ]+$")}, + {name: "a node's toolchain reported", subject: "mesh.seat.node-build-go.event.toolchain-reported.ace", given: "{\"digest\":\"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"language\":\"go\",\"node\":\"ace\",\"recipe\":\"1\",\"version\":\"1.26.0\"}", wire: "{\"digest\":\"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"language\":\"go\",\"node\":\"ace\",\"recipe\":\"1\",\"version\":\"1.26.0\"}", + decode: func(b []byte) (any, error) { var v ToolchainReportedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.node-build-go\\.event\\.toolchain-reported\\.[^.*> ]+$")}, + {name: "a running ask says its progress", subject: "mesh.seat.node-build-go.event.step-progress.ace", given: "{\"ask\":\"a1b2\",\"asker\":\"runner\",\"at\":\"2026-11-11T10:01:00Z\",\"phase\":\"fetch\"}", wire: "{\"ask\":\"a1b2\",\"asker\":\"runner\",\"at\":\"2026-11-11T10:01:00Z\",\"phase\":\"fetch\"}", + decode: func(b []byte) (any, error) { var v StepProgressEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.node-build-go\\.event\\.step-progress\\.[^.*> ]+$")}, {name: "a running step", subject: "", given: "{\"id\":\"a1b2\",\"node\":\"ace\",\"started\":\"2026-11-05T10:00:00Z\",\"state\":\"running\",\"tool\":\"build\",\"v\":1}", wire: "{\"id\":\"a1b2\",\"node\":\"ace\",\"started\":\"2026-11-05T10:00:00Z\",\"state\":\"running\",\"tool\":\"build\",\"v\":1}", decode: func(b []byte) (any, error) { var v StepsByNodeIDValue; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, {name: "a paused node", subject: "", given: "{\"at\":\"2026-11-05T10:00:00Z\",\"by\":\"operator@shanks\",\"paused\":true,\"v\":1}", wire: "{\"at\":\"2026-11-05T10:00:00Z\",\"by\":\"operator@shanks\",\"paused\":true,\"v\":1}", @@ -75,6 +79,12 @@ func TestFixtures(t *testing.T) { err := json.Unmarshal(b, &v) return v, err }, subjectIs: nil}, + {name: "a step keyed under its asker", subject: "", given: "{\"id\":\"a1b2\",\"node\":\"ace\",\"started\":\"2026-11-05T10:00:00Z\",\"state\":\"running\",\"tool\":\"build\",\"v\":1}", wire: "{\"id\":\"a1b2\",\"node\":\"ace\",\"started\":\"2026-11-05T10:00:00Z\",\"state\":\"running\",\"tool\":\"build\",\"v\":1}", + decode: func(b []byte) (any, error) { + var v StepsByAskerNodeIDValue + err := json.Unmarshal(b, &v) + return v, err + }, subjectIs: nil}, {name: "drop names the ask and why", subject: "", given: "{\"by\":\"mesh-delivery-builder\",\"id\":\"a1b2\",\"why\":\"a newer fingerprint supersedes it\"}", wire: "{\"by\":\"mesh-delivery-builder\",\"id\":\"a1b2\",\"why\":\"a newer fingerprint supersedes it\"}", decode: func(b []byte) (any, error) { var v DropInput; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, {name: "kept-artifact asks", subject: "", given: "{\"digest\":\"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd\",\"fp\":\"src2:1480985e\",\"module\":\"artifact-store\"}", wire: "{\"digest\":\"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd\",\"fp\":\"src2:1480985e\",\"module\":\"artifact-store\"}", @@ -93,6 +103,12 @@ func TestFixtures(t *testing.T) { decode: func(b []byte) (any, error) { var v StepsInput; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, {name: "steps answers lint and unit tests", subject: "", given: "{\"artifact\":\"nats:program\",\"fingerprint\":\"fp-0123\",\"steps\":[{\"step\":\"lint\"},{\"packages\":[\"./modules/nats\"],\"step\":\"unit\"}]}", wire: "{\"artifact\":\"nats:program\",\"fingerprint\":\"fp-0123\",\"steps\":[{\"step\":\"lint\"},{\"packages\":[\"./modules/nats\"],\"step\":\"unit\"}]}", decode: func(b []byte) (any, error) { var v StepsAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, + {name: "steps drops an artifact's kept steps with a reason", subject: "", given: "{\"artifact\":\"nats.program\",\"drop\":true,\"module\":\"nats\",\"why\":\"the detection was wrong\"}", wire: "{\"artifact\":\"nats.program\",\"drop\":true,\"module\":\"nats\",\"why\":\"the detection was wrong\"}", + decode: func(b []byte) (any, error) { var v StepsInput; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, + {name: "steps answers a drop with the artifact", subject: "", given: "{\"dropped\":\"nats.program\"}", wire: "{\"dropped\":\"nats.program\"}", + decode: func(b []byte) (any, error) { var v StepsAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, + {name: "steps answers a drop that the hand-act log refused", subject: "", given: "{\"error\":\"the hand-act log refused: nothing was deleted\"}", wire: "{\"error\":\"the hand-act log refused: nothing was deleted\"}", + decode: func(b []byte) (any, error) { var v StepsAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, } for _, c := range cases { v, err := c.decode([]byte(c.given)) diff --git a/contracts/go/seats/nodebuildimage/contract.go b/contracts/go/seats/nodebuildimage/contract.go index 00f25faa..9769b994 100644 --- a/contracts/go/seats/nodebuildimage/contract.go +++ b/contracts/go/seats/nodebuildimage/contract.go @@ -32,7 +32,9 @@ const ( ) const ( - EventStepEnded = "step-ended" + EventStepEnded = "step-ended" + EventToolchainReported = "toolchain-reported" + EventStepProgress = "step-progress" ) // Verbs is every verb the seat serves, in its definition's order: a new copy each call. @@ -42,7 +44,7 @@ func Verbs() []string { // Events is every event published on the seat's subject: a new copy each call. func Events() []string { - return []string{EventStepEnded} + return []string{EventStepEnded, EventToolchainReported, EventStepProgress} } // Family is where the seat's events are kept: the shared events stream, filtered to the seat's subjects. @@ -62,6 +64,28 @@ func SubjectStepEnded(node string) string { // SubjectStepEndedAll hears every step-ended. const SubjectStepEndedAll = "mesh.seat.node-build-image.event.step-ended.*" +// StreamToolchainReported is the stream toolchain-reported is kept in. +const StreamToolchainReported = "SEAT_node-build-image_toolchain-reported" + +// SubjectToolchainReported is where toolchain-reported is published, on mesh.seat.node-build-image.event.toolchain-reported.. +func SubjectToolchainReported(node string) string { + return "mesh.seat.node-build-image.event.toolchain-reported." + node +} + +// SubjectToolchainReportedAll hears every toolchain-reported. +const SubjectToolchainReportedAll = "mesh.seat.node-build-image.event.toolchain-reported.*" + +// StreamStepProgress is the stream step-progress is kept in. +const StreamStepProgress = "SEAT_node-build-image_step-progress" + +// SubjectStepProgress is where step-progress is published, on mesh.seat.node-build-image.event.step-progress.. +func SubjectStepProgress(node string) string { + return "mesh.seat.node-build-image.event.step-progress." + node +} + +// SubjectStepProgressAll hears every step-progress. +const SubjectStepProgressAll = "mesh.seat.node-build-image.event.step-progress.*" + // AcceptSubjectDelivery is where delivery is sent. const AcceptSubjectDelivery = "mesh.seat.node-build-image.accept.delivery" @@ -84,6 +108,47 @@ const ( StateDetectedSteps = "detected-steps" ) +// StepsByAskerNodeIDKey is the key of one value of the state steps under the form "..". +func StepsByAskerNodeIDKey(asker, node, id string) string { return asker + "." + node + "." + id } + +// ParseStepsByAskerNodeIDKey reads the parts of a key of the form "..", in the order of the form; the last result is false for a key of another form. +func ParseStepsByAskerNodeIDKey(key string) (string, string, string, bool) { + rest := key + v0, after0, found0 := strings.Cut(rest, ".") + if !found0 || v0 == "" { + return "", "", "", false + } + rest = after0 + v2, after2, found2 := strings.Cut(rest, ".") + if !found2 || v2 == "" { + return "", "", "", false + } + rest = after2 + if rest == "" { + return "", "", "", false + } + v4 := rest + rest = "" + if rest != "" { + return "", "", "", false + } + return v0, v2, v4, true +} + +// DecodeStepsByAskerNodeIDValue reads one value of the state steps under the form "..", refusing a field StepsByAskerNodeIDValue does not know and any text after the value. +func DecodeStepsByAskerNodeIDValue(raw []byte) (StepsByAskerNodeIDValue, error) { + var v StepsByAskerNodeIDValue + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.DisallowUnknownFields() + if err := dec.Decode(&v); err != nil { + return v, err + } + if dec.More() { + return v, errors.New("text after the value") + } + return v, nil +} + // StepsByNodeIDKey is the key of one value of the state steps under the form ".". func StepsByNodeIDKey(node, id string) string { return node + "." + id } @@ -315,11 +380,21 @@ type BuildAnswer struct { Toolchain *ToolchainUsed `json:"toolchain,omitempty"` } +// BuiltArtifactResolvedItem is a type of the contract. +type BuiltArtifactResolvedItem struct { + // sha256: of the package's bytes as the build fetched them + Digest string `json:"digest"` + Package string `json:"package"` + Version string `json:"version"` +} + // BuiltArtifact is one artifact a build made: its digest and the reference it was published under. type BuiltArtifact struct { Artifact string `json:"artifact"` Digest string `json:"digest"` Reference string `json:"reference,omitempty"` + // one per mesh package the build resolved (the catalog graph entry's packages, the edges of kind 1) + Resolved []BuiltArtifactResolvedItem `json:"resolved,omitempty"` } // Capacity is how many steps the node runs at once. @@ -394,6 +469,16 @@ type FetchBesideItem struct { Repository string `json:"repository"` } +// FetchBuildsItem is a type of the contract. +type FetchBuildsItem struct { + Artifact string `json:"artifact"` + // the member's build digest, sha256: + Digest string `json:"digest"` + Module string `json:"module"` + // the name in its ecosystem (the Go module path, the npm name) + Package string `json:"package"` +} + // FetchPackage is the TypeScript module's group package: the group's tagged version of the SDK ({name, version}), placed from the registry in each member whatever its lock names, or the group's package member ({repository, commit, ref}), packed by the fetch and served to the members in place of the registry's. type FetchPackage struct { Commit string `json:"commit,omitempty"` @@ -409,6 +494,8 @@ type Fetch struct { Base string `json:"base,omitempty"` // the sibling repositories placed beside the fetched tree Beside []FetchBesideItem `json:"beside,omitempty"` + // a group run's package members, each by its build's digest: the build module reads each from the artifact store at artifact-store:////blobs/, checks the bytes against the digest and lays them as the group package; a package named here is named in neither packages nor package + Builds []FetchBuildsItem `json:"builds,omitempty"` // the full commit Commit string `json:"commit"` // the module's directory in the repository ("." for its root), with paths @@ -733,6 +820,22 @@ type StepNames struct { Repositories []string `json:"repositories"` } +// StepProgress is a running ask's progress, said once a minute while the holder runs it (design 47 G section 496). +type StepProgress struct { + Ask string `json:"ask"` + Asker string `json:"asker"` + At time.Time `json:"at"` + Phase string `json:"phase"` +} + +// The values of StepProgress's phase. +const ( + StepProgressPhaseFetch = "fetch" + StepProgressPhaseBuild = "build" + StepProgressPhaseRun = "run" + StepProgressPhaseReport = "report" +) + // StepResult is what a step answered, by the kind of ask: a build's artifacts, toolchain, imports, out-of-step modules and the artifact it kept; a check's fetch and levels; a compose's built, key and verdict; a run's built, key, exit, output and said; a declined ask's declined, needs and lacks. The fields of the kinds do not meet, so one layout holds the five. type StepResult struct { Artifacts []BuiltArtifact `json:"artifacts,omitempty"` @@ -826,8 +929,20 @@ type Toolchain struct { Version string `json:"version"` } +// ToolchainReported is a node's toolchain of a language, said at the build module's start and each time its toolchains key of the language changes (design 47 G section 492). +type ToolchainReported struct { + // the digest of the program of the provision the build module requires for that language's toolchain, never the build module's own + Digest string `json:"digest"` + Language string `json:"language"` + Node string `json:"node"` + Recipe string `json:"recipe"` + Version string `json:"version"` +} + // ToolchainUsed is the toolchain a build was made with: the language, the installed version and the build module's recipe. type ToolchainUsed struct { + // the building node's digest of its toolchain program (kind 3's entry), never the build module's own program + Digest string `json:"digest,omitempty"` Language string `json:"language"` Recipe string `json:"recipe"` Version string `json:"version"` @@ -910,20 +1025,28 @@ const ( type StepsInput struct { // find them anew, not from the kept answer Again bool `json:"again,omitempty"` - // the artifact reference, : - Artifact string `json:"artifact"` - Fetch Fetch `json:"fetch"` + // the key of detected-steps, . + Artifact string `json:"artifact"` + // delete the artifact's kept key instead of detecting; needs why + Drop bool `json:"drop,omitempty"` + Fetch *Fetch `json:"fetch,omitempty"` Fingerprint string `json:"fingerprint,omitempty"` Module string `json:"module"` + // with drop, why: a non-empty text, written in the hand-act log with the caller the bus proved + Why string `json:"why,omitempty"` } // StepsAnswer is what steps answers. type StepsAnswer struct { - Artifact string `json:"artifact"` + Artifact string `json:"artifact,omitempty"` + // the artifact whose key a drop deleted + Dropped string `json:"dropped,omitempty"` + // why a drop deleted nothing, naming the hand-act log when it refused + Error string `json:"error,omitempty"` Fingerprint string `json:"fingerprint,omitempty"` // answered from the kept answer Kept bool `json:"kept,omitempty"` - Steps []DetectedStep `json:"steps"` + Steps []DetectedStep `json:"steps,omitempty"` } // DeliveryAccept is the body of delivery. @@ -944,6 +1067,15 @@ type RecheckAccept = StepAsk // StepEndedEvent is the body of the event step-ended. type StepEndedEvent = StepEnded +// ToolchainReportedEvent is the body of the event toolchain-reported. +type ToolchainReportedEvent = ToolchainReported + +// StepProgressEvent is the body of the event step-progress. +type StepProgressEvent = StepProgress + +// StepsByAskerNodeIDValue is one value of the state steps under the key ... +type StepsByAskerNodeIDValue = Step + // StepsByNodeIDValue is one value of the state steps under the key .. type StepsByNodeIDValue = Step diff --git a/contracts/go/seats/nodebuildimage/contract_test.go b/contracts/go/seats/nodebuildimage/contract_test.go index fbbbb9c8..3ec60a4f 100644 --- a/contracts/go/seats/nodebuildimage/contract_test.go +++ b/contracts/go/seats/nodebuildimage/contract_test.go @@ -21,6 +21,10 @@ func TestFixtures(t *testing.T) { decode: func(b []byte) (any, error) { var v StepEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.node-build-image\\.event\\.step-ended\\.[^.*> ]+$")}, {name: "a lint step that could not run, naming the part not heard", subject: "mesh.seat.node-build-image.event.step-ended.g14", given: "{\"artifact\":\"nats:program\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d5f\",\"module\":\"nats\",\"node\":\"g14\",\"outcome\":\"could not run\",\"output-object\":\"mesh-build-go_sent-objects/m2kx9a1b3c4d5f\",\"pipeline\":\"main.12.nats\",\"said\":\"the sandbox could not fetch novox/mesh-sdk\",\"step\":\"lint\",\"v\":1}", wire: "{\"artifact\":\"nats:program\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d5f\",\"module\":\"nats\",\"node\":\"g14\",\"outcome\":\"could not run\",\"output-object\":\"mesh-build-go_sent-objects/m2kx9a1b3c4d5f\",\"pipeline\":\"main.12.nats\",\"said\":\"the sandbox could not fetch novox/mesh-sdk\",\"step\":\"lint\",\"v\":1}", decode: func(b []byte) (any, error) { var v StepEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.node-build-image\\.event\\.step-ended\\.[^.*> ]+$")}, + {name: "a node's toolchain reported", subject: "mesh.seat.node-build-image.event.toolchain-reported.ace", given: "{\"digest\":\"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"language\":\"go\",\"node\":\"ace\",\"recipe\":\"1\",\"version\":\"1.26.0\"}", wire: "{\"digest\":\"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"language\":\"go\",\"node\":\"ace\",\"recipe\":\"1\",\"version\":\"1.26.0\"}", + decode: func(b []byte) (any, error) { var v ToolchainReportedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.node-build-image\\.event\\.toolchain-reported\\.[^.*> ]+$")}, + {name: "a running ask says its progress", subject: "mesh.seat.node-build-image.event.step-progress.ace", given: "{\"ask\":\"a1b2\",\"asker\":\"runner\",\"at\":\"2026-11-11T10:01:00Z\",\"phase\":\"fetch\"}", wire: "{\"ask\":\"a1b2\",\"asker\":\"runner\",\"at\":\"2026-11-11T10:01:00Z\",\"phase\":\"fetch\"}", + decode: func(b []byte) (any, error) { var v StepProgressEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.node-build-image\\.event\\.step-progress\\.[^.*> ]+$")}, {name: "a running step", subject: "", given: "{\"id\":\"a1b2\",\"node\":\"ace\",\"started\":\"2026-11-05T10:00:00Z\",\"state\":\"running\",\"tool\":\"build\",\"v\":1}", wire: "{\"id\":\"a1b2\",\"node\":\"ace\",\"started\":\"2026-11-05T10:00:00Z\",\"state\":\"running\",\"tool\":\"build\",\"v\":1}", decode: func(b []byte) (any, error) { var v StepsByNodeIDValue; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, {name: "a paused node", subject: "", given: "{\"at\":\"2026-11-05T10:00:00Z\",\"by\":\"operator@shanks\",\"paused\":true,\"v\":1}", wire: "{\"at\":\"2026-11-05T10:00:00Z\",\"by\":\"operator@shanks\",\"paused\":true,\"v\":1}", @@ -57,6 +61,12 @@ func TestFixtures(t *testing.T) { err := json.Unmarshal(b, &v) return v, err }, subjectIs: nil}, + {name: "a step keyed under its asker", subject: "", given: "{\"id\":\"a1b2\",\"node\":\"ace\",\"started\":\"2026-11-05T10:00:00Z\",\"state\":\"running\",\"tool\":\"build\",\"v\":1}", wire: "{\"id\":\"a1b2\",\"node\":\"ace\",\"started\":\"2026-11-05T10:00:00Z\",\"state\":\"running\",\"tool\":\"build\",\"v\":1}", + decode: func(b []byte) (any, error) { + var v StepsByAskerNodeIDValue + err := json.Unmarshal(b, &v) + return v, err + }, subjectIs: nil}, {name: "drop names the ask and why", subject: "", given: "{\"by\":\"mesh-delivery-builder\",\"id\":\"a1b2\",\"why\":\"a newer fingerprint supersedes it\"}", wire: "{\"by\":\"mesh-delivery-builder\",\"id\":\"a1b2\",\"why\":\"a newer fingerprint supersedes it\"}", decode: func(b []byte) (any, error) { var v DropInput; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, {name: "kept-artifact asks", subject: "", given: "{\"digest\":\"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd\",\"fp\":\"src2:1480985e\",\"module\":\"artifact-store\"}", wire: "{\"digest\":\"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd\",\"fp\":\"src2:1480985e\",\"module\":\"artifact-store\"}", @@ -75,6 +85,12 @@ func TestFixtures(t *testing.T) { decode: func(b []byte) (any, error) { var v StepsInput; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, {name: "steps answers lint and unit tests", subject: "", given: "{\"artifact\":\"nats:program\",\"fingerprint\":\"fp-0123\",\"steps\":[{\"step\":\"lint\"},{\"packages\":[\"./modules/nats\"],\"step\":\"unit\"}]}", wire: "{\"artifact\":\"nats:program\",\"fingerprint\":\"fp-0123\",\"steps\":[{\"step\":\"lint\"},{\"packages\":[\"./modules/nats\"],\"step\":\"unit\"}]}", decode: func(b []byte) (any, error) { var v StepsAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, + {name: "steps drops an artifact's kept steps with a reason", subject: "", given: "{\"artifact\":\"nats.program\",\"drop\":true,\"module\":\"nats\",\"why\":\"the detection was wrong\"}", wire: "{\"artifact\":\"nats.program\",\"drop\":true,\"module\":\"nats\",\"why\":\"the detection was wrong\"}", + decode: func(b []byte) (any, error) { var v StepsInput; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, + {name: "steps answers a drop with the artifact", subject: "", given: "{\"dropped\":\"nats.program\"}", wire: "{\"dropped\":\"nats.program\"}", + decode: func(b []byte) (any, error) { var v StepsAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, + {name: "steps answers a drop that the hand-act log refused", subject: "", given: "{\"error\":\"the hand-act log refused: nothing was deleted\"}", wire: "{\"error\":\"the hand-act log refused: nothing was deleted\"}", + decode: func(b []byte) (any, error) { var v StepsAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, } for _, c := range cases { v, err := c.decode([]byte(c.given)) diff --git a/contracts/go/seats/nodebuildnode/contract.go b/contracts/go/seats/nodebuildnode/contract.go index 2df88e8f..5004106f 100644 --- a/contracts/go/seats/nodebuildnode/contract.go +++ b/contracts/go/seats/nodebuildnode/contract.go @@ -32,7 +32,9 @@ const ( ) const ( - EventStepEnded = "step-ended" + EventStepEnded = "step-ended" + EventToolchainReported = "toolchain-reported" + EventStepProgress = "step-progress" ) // Verbs is every verb the seat serves, in its definition's order: a new copy each call. @@ -42,7 +44,7 @@ func Verbs() []string { // Events is every event published on the seat's subject: a new copy each call. func Events() []string { - return []string{EventStepEnded} + return []string{EventStepEnded, EventToolchainReported, EventStepProgress} } // Family is where the seat's events are kept: the shared events stream, filtered to the seat's subjects. @@ -62,6 +64,28 @@ func SubjectStepEnded(node string) string { // SubjectStepEndedAll hears every step-ended. const SubjectStepEndedAll = "mesh.seat.node-build-node.event.step-ended.*" +// StreamToolchainReported is the stream toolchain-reported is kept in. +const StreamToolchainReported = "SEAT_node-build-node_toolchain-reported" + +// SubjectToolchainReported is where toolchain-reported is published, on mesh.seat.node-build-node.event.toolchain-reported.. +func SubjectToolchainReported(node string) string { + return "mesh.seat.node-build-node.event.toolchain-reported." + node +} + +// SubjectToolchainReportedAll hears every toolchain-reported. +const SubjectToolchainReportedAll = "mesh.seat.node-build-node.event.toolchain-reported.*" + +// StreamStepProgress is the stream step-progress is kept in. +const StreamStepProgress = "SEAT_node-build-node_step-progress" + +// SubjectStepProgress is where step-progress is published, on mesh.seat.node-build-node.event.step-progress.. +func SubjectStepProgress(node string) string { + return "mesh.seat.node-build-node.event.step-progress." + node +} + +// SubjectStepProgressAll hears every step-progress. +const SubjectStepProgressAll = "mesh.seat.node-build-node.event.step-progress.*" + // AcceptSubjectDelivery is where delivery is sent. const AcceptSubjectDelivery = "mesh.seat.node-build-node.accept.delivery" @@ -84,6 +108,47 @@ const ( StateDetectedSteps = "detected-steps" ) +// StepsByAskerNodeIDKey is the key of one value of the state steps under the form "..". +func StepsByAskerNodeIDKey(asker, node, id string) string { return asker + "." + node + "." + id } + +// ParseStepsByAskerNodeIDKey reads the parts of a key of the form "..", in the order of the form; the last result is false for a key of another form. +func ParseStepsByAskerNodeIDKey(key string) (string, string, string, bool) { + rest := key + v0, after0, found0 := strings.Cut(rest, ".") + if !found0 || v0 == "" { + return "", "", "", false + } + rest = after0 + v2, after2, found2 := strings.Cut(rest, ".") + if !found2 || v2 == "" { + return "", "", "", false + } + rest = after2 + if rest == "" { + return "", "", "", false + } + v4 := rest + rest = "" + if rest != "" { + return "", "", "", false + } + return v0, v2, v4, true +} + +// DecodeStepsByAskerNodeIDValue reads one value of the state steps under the form "..", refusing a field StepsByAskerNodeIDValue does not know and any text after the value. +func DecodeStepsByAskerNodeIDValue(raw []byte) (StepsByAskerNodeIDValue, error) { + var v StepsByAskerNodeIDValue + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.DisallowUnknownFields() + if err := dec.Decode(&v); err != nil { + return v, err + } + if dec.More() { + return v, errors.New("text after the value") + } + return v, nil +} + // StepsByNodeIDKey is the key of one value of the state steps under the form ".". func StepsByNodeIDKey(node, id string) string { return node + "." + id } @@ -315,11 +380,21 @@ type BuildAnswer struct { Toolchain *ToolchainUsed `json:"toolchain,omitempty"` } +// BuiltArtifactResolvedItem is a type of the contract. +type BuiltArtifactResolvedItem struct { + // sha256: of the package's bytes as the build fetched them + Digest string `json:"digest"` + Package string `json:"package"` + Version string `json:"version"` +} + // BuiltArtifact is one artifact a build made: its digest and the reference it was published under. type BuiltArtifact struct { Artifact string `json:"artifact"` Digest string `json:"digest"` Reference string `json:"reference,omitempty"` + // one per mesh package the build resolved (the catalog graph entry's packages, the edges of kind 1) + Resolved []BuiltArtifactResolvedItem `json:"resolved,omitempty"` } // Capacity is how many steps the node runs at once. @@ -394,6 +469,16 @@ type FetchBesideItem struct { Repository string `json:"repository"` } +// FetchBuildsItem is a type of the contract. +type FetchBuildsItem struct { + Artifact string `json:"artifact"` + // the member's build digest, sha256: + Digest string `json:"digest"` + Module string `json:"module"` + // the name in its ecosystem (the Go module path, the npm name) + Package string `json:"package"` +} + // FetchPackage is the TypeScript module's group package: the group's tagged version of the SDK ({name, version}), placed from the registry in each member whatever its lock names, or the group's package member ({repository, commit, ref}), packed by the fetch and served to the members in place of the registry's. type FetchPackage struct { Commit string `json:"commit,omitempty"` @@ -409,6 +494,8 @@ type Fetch struct { Base string `json:"base,omitempty"` // the sibling repositories placed beside the fetched tree Beside []FetchBesideItem `json:"beside,omitempty"` + // a group run's package members, each by its build's digest: the build module reads each from the artifact store at artifact-store:////blobs/, checks the bytes against the digest and lays them as the group package; a package named here is named in neither packages nor package + Builds []FetchBuildsItem `json:"builds,omitempty"` // the full commit Commit string `json:"commit"` // the module's directory in the repository ("." for its root), with paths @@ -733,6 +820,22 @@ type StepNames struct { Repositories []string `json:"repositories"` } +// StepProgress is a running ask's progress, said once a minute while the holder runs it (design 47 G section 496). +type StepProgress struct { + Ask string `json:"ask"` + Asker string `json:"asker"` + At time.Time `json:"at"` + Phase string `json:"phase"` +} + +// The values of StepProgress's phase. +const ( + StepProgressPhaseFetch = "fetch" + StepProgressPhaseBuild = "build" + StepProgressPhaseRun = "run" + StepProgressPhaseReport = "report" +) + // StepResult is what a step answered, by the kind of ask: a build's artifacts, toolchain, imports, out-of-step modules and the artifact it kept; a check's fetch and levels; a compose's built, key and verdict; a run's built, key, exit, output and said; a declined ask's declined, needs and lacks. The fields of the kinds do not meet, so one layout holds the five. type StepResult struct { Artifacts []BuiltArtifact `json:"artifacts,omitempty"` @@ -826,8 +929,20 @@ type Toolchain struct { Version string `json:"version"` } +// ToolchainReported is a node's toolchain of a language, said at the build module's start and each time its toolchains key of the language changes (design 47 G section 492). +type ToolchainReported struct { + // the digest of the program of the provision the build module requires for that language's toolchain, never the build module's own + Digest string `json:"digest"` + Language string `json:"language"` + Node string `json:"node"` + Recipe string `json:"recipe"` + Version string `json:"version"` +} + // ToolchainUsed is the toolchain a build was made with: the language, the installed version and the build module's recipe. type ToolchainUsed struct { + // the building node's digest of its toolchain program (kind 3's entry), never the build module's own program + Digest string `json:"digest,omitempty"` Language string `json:"language"` Recipe string `json:"recipe"` Version string `json:"version"` @@ -910,20 +1025,28 @@ const ( type StepsInput struct { // find them anew, not from the kept answer Again bool `json:"again,omitempty"` - // the artifact reference, : - Artifact string `json:"artifact"` - Fetch Fetch `json:"fetch"` + // the key of detected-steps, . + Artifact string `json:"artifact"` + // delete the artifact's kept key instead of detecting; needs why + Drop bool `json:"drop,omitempty"` + Fetch *Fetch `json:"fetch,omitempty"` Fingerprint string `json:"fingerprint,omitempty"` Module string `json:"module"` + // with drop, why: a non-empty text, written in the hand-act log with the caller the bus proved + Why string `json:"why,omitempty"` } // StepsAnswer is what steps answers. type StepsAnswer struct { - Artifact string `json:"artifact"` + Artifact string `json:"artifact,omitempty"` + // the artifact whose key a drop deleted + Dropped string `json:"dropped,omitempty"` + // why a drop deleted nothing, naming the hand-act log when it refused + Error string `json:"error,omitempty"` Fingerprint string `json:"fingerprint,omitempty"` // answered from the kept answer Kept bool `json:"kept,omitempty"` - Steps []DetectedStep `json:"steps"` + Steps []DetectedStep `json:"steps,omitempty"` } // DeliveryAccept is the body of delivery. @@ -944,6 +1067,15 @@ type RecheckAccept = StepAsk // StepEndedEvent is the body of the event step-ended. type StepEndedEvent = StepEnded +// ToolchainReportedEvent is the body of the event toolchain-reported. +type ToolchainReportedEvent = ToolchainReported + +// StepProgressEvent is the body of the event step-progress. +type StepProgressEvent = StepProgress + +// StepsByAskerNodeIDValue is one value of the state steps under the key ... +type StepsByAskerNodeIDValue = Step + // StepsByNodeIDValue is one value of the state steps under the key .. type StepsByNodeIDValue = Step diff --git a/contracts/go/seats/nodebuildnode/contract_test.go b/contracts/go/seats/nodebuildnode/contract_test.go index 982cd713..931da365 100644 --- a/contracts/go/seats/nodebuildnode/contract_test.go +++ b/contracts/go/seats/nodebuildnode/contract_test.go @@ -21,6 +21,10 @@ func TestFixtures(t *testing.T) { decode: func(b []byte) (any, error) { var v StepEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.node-build-node\\.event\\.step-ended\\.[^.*> ]+$")}, {name: "an integration step that could not run, naming the part not heard", subject: "mesh.seat.node-build-node.event.step-ended.g14", given: "{\"artifact\":\"nats:program\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d5f\",\"module\":\"nats\",\"node\":\"g14\",\"outcome\":\"could not run\",\"output-object\":\"mesh-build-go_sent-objects/m2kx9a1b3c4d5f\",\"pipeline\":\"main.12.nats\",\"said\":\"the sandbox could not fetch novox/mesh-sdk\",\"step\":\"integration\",\"v\":1}", wire: "{\"artifact\":\"nats:program\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d5f\",\"module\":\"nats\",\"node\":\"g14\",\"outcome\":\"could not run\",\"output-object\":\"mesh-build-go_sent-objects/m2kx9a1b3c4d5f\",\"pipeline\":\"main.12.nats\",\"said\":\"the sandbox could not fetch novox/mesh-sdk\",\"step\":\"integration\",\"v\":1}", decode: func(b []byte) (any, error) { var v StepEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.node-build-node\\.event\\.step-ended\\.[^.*> ]+$")}, + {name: "a node's toolchain reported", subject: "mesh.seat.node-build-node.event.toolchain-reported.ace", given: "{\"digest\":\"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"language\":\"go\",\"node\":\"ace\",\"recipe\":\"1\",\"version\":\"1.26.0\"}", wire: "{\"digest\":\"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"language\":\"go\",\"node\":\"ace\",\"recipe\":\"1\",\"version\":\"1.26.0\"}", + decode: func(b []byte) (any, error) { var v ToolchainReportedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.node-build-node\\.event\\.toolchain-reported\\.[^.*> ]+$")}, + {name: "a running ask says its progress", subject: "mesh.seat.node-build-node.event.step-progress.ace", given: "{\"ask\":\"a1b2\",\"asker\":\"runner\",\"at\":\"2026-11-11T10:01:00Z\",\"phase\":\"fetch\"}", wire: "{\"ask\":\"a1b2\",\"asker\":\"runner\",\"at\":\"2026-11-11T10:01:00Z\",\"phase\":\"fetch\"}", + decode: func(b []byte) (any, error) { var v StepProgressEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.node-build-node\\.event\\.step-progress\\.[^.*> ]+$")}, {name: "a running step", subject: "", given: "{\"id\":\"a1b2\",\"node\":\"ace\",\"started\":\"2026-11-05T10:00:00Z\",\"state\":\"running\",\"tool\":\"build\",\"v\":1}", wire: "{\"id\":\"a1b2\",\"node\":\"ace\",\"started\":\"2026-11-05T10:00:00Z\",\"state\":\"running\",\"tool\":\"build\",\"v\":1}", decode: func(b []byte) (any, error) { var v StepsByNodeIDValue; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, {name: "a paused node", subject: "", given: "{\"at\":\"2026-11-05T10:00:00Z\",\"by\":\"operator@shanks\",\"paused\":true,\"v\":1}", wire: "{\"at\":\"2026-11-05T10:00:00Z\",\"by\":\"operator@shanks\",\"paused\":true,\"v\":1}", @@ -61,6 +65,12 @@ func TestFixtures(t *testing.T) { err := json.Unmarshal(b, &v) return v, err }, subjectIs: nil}, + {name: "a step keyed under its asker", subject: "", given: "{\"id\":\"a1b2\",\"node\":\"ace\",\"started\":\"2026-11-05T10:00:00Z\",\"state\":\"running\",\"tool\":\"build\",\"v\":1}", wire: "{\"id\":\"a1b2\",\"node\":\"ace\",\"started\":\"2026-11-05T10:00:00Z\",\"state\":\"running\",\"tool\":\"build\",\"v\":1}", + decode: func(b []byte) (any, error) { + var v StepsByAskerNodeIDValue + err := json.Unmarshal(b, &v) + return v, err + }, subjectIs: nil}, {name: "drop names the ask and why", subject: "", given: "{\"by\":\"mesh-delivery-builder\",\"id\":\"a1b2\",\"why\":\"a newer fingerprint supersedes it\"}", wire: "{\"by\":\"mesh-delivery-builder\",\"id\":\"a1b2\",\"why\":\"a newer fingerprint supersedes it\"}", decode: func(b []byte) (any, error) { var v DropInput; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, {name: "kept-artifact asks", subject: "", given: "{\"digest\":\"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd\",\"fp\":\"src2:1480985e\",\"module\":\"artifact-store\"}", wire: "{\"digest\":\"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd\",\"fp\":\"src2:1480985e\",\"module\":\"artifact-store\"}", @@ -79,6 +89,12 @@ func TestFixtures(t *testing.T) { decode: func(b []byte) (any, error) { var v StepsInput; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, {name: "steps answers lint and unit tests", subject: "", given: "{\"artifact\":\"nats:program\",\"fingerprint\":\"fp-0123\",\"steps\":[{\"step\":\"lint\"},{\"packages\":[\"./modules/nats\"],\"step\":\"unit\"}]}", wire: "{\"artifact\":\"nats:program\",\"fingerprint\":\"fp-0123\",\"steps\":[{\"step\":\"lint\"},{\"packages\":[\"./modules/nats\"],\"step\":\"unit\"}]}", decode: func(b []byte) (any, error) { var v StepsAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, + {name: "steps drops an artifact's kept steps with a reason", subject: "", given: "{\"artifact\":\"nats.program\",\"drop\":true,\"module\":\"nats\",\"why\":\"the detection was wrong\"}", wire: "{\"artifact\":\"nats.program\",\"drop\":true,\"module\":\"nats\",\"why\":\"the detection was wrong\"}", + decode: func(b []byte) (any, error) { var v StepsInput; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, + {name: "steps answers a drop with the artifact", subject: "", given: "{\"dropped\":\"nats.program\"}", wire: "{\"dropped\":\"nats.program\"}", + decode: func(b []byte) (any, error) { var v StepsAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, + {name: "steps answers a drop that the hand-act log refused", subject: "", given: "{\"error\":\"the hand-act log refused: nothing was deleted\"}", wire: "{\"error\":\"the hand-act log refused: nothing was deleted\"}", + decode: func(b []byte) (any, error) { var v StepsAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, } for _, c := range cases { v, err := c.decode([]byte(c.given)) diff --git a/contracts/parts/build-step.json b/contracts/parts/build-step.json index 3aed594f..2179c177 100644 --- a/contracts/parts/build-step.json +++ b/contracts/parts/build-step.json @@ -159,10 +159,12 @@ }, { "name": "steps", - "description": "Which pipeline steps an artifact can run, found by its language's convention (design 47 *Pipeline steps and the policy*): lint always, unit and integration tests where its packages hold them. Kept by the artifact's fingerprint; again finds them anew.", + "description": "Which pipeline steps an artifact can run, found by its language's convention (design 47 *Pipeline steps and the policy*): lint always, unit and integration tests where its packages hold them. Kept by the artifact's fingerprint; again finds them anew. With drop and why it deletes the artifact's kept key instead, writes the act in the hand-act log of the seat mesh-hand-acts first (a log that refuses deletes nothing and answers error), and answers dropped (design 47 G section 463, section 480).", "callers": [ "seat:mesh-pipeline-runner", - "seat:mesh-catalog" + "seat:mesh-catalog", + "operator", + "sessions" ], "input": { "type": "object", @@ -172,7 +174,7 @@ }, "artifact": { "type": "string", - "description": "the artifact reference, :" + "description": "the key of detected-steps, ." }, "fingerprint": { "type": "string" @@ -183,13 +185,21 @@ "again": { "type": "boolean", "description": "find them anew, not from the kept answer" + }, + "drop": { + "type": "boolean", + "description": "delete the artifact's kept key instead of detecting; needs why" + }, + "why": { + "type": "string", + "description": "with drop, why: a non-empty text, written in the hand-act log with the caller the bus proved" } }, "required": [ "module", - "artifact", - "fetch" - ] + "artifact" + ], + "description": "fetch is required unless drop is given, and why with drop" }, "answer": { "type": "object", @@ -209,12 +219,17 @@ "kept": { "type": "boolean", "description": "answered from the kept answer" + }, + "dropped": { + "type": "string", + "description": "the artifact whose key a drop deleted" + }, + "error": { + "type": "string", + "description": "why a drop deleted nothing, naming the hand-act log when it refused" } }, - "required": [ - "artifact", - "steps" - ] + "required": [] }, "class": "act" } @@ -857,6 +872,35 @@ ] }, "description": "the sibling repositories placed beside the fetched tree" + }, + "builds": { + "type": "array", + "description": "a group run's package members, each by its build's digest: the build module reads each from the artifact store at artifact-store:////blobs/, checks the bytes against the digest and lays them as the group package; a package named here is named in neither packages nor package", + "items": { + "type": "object", + "properties": { + "package": { + "type": "string", + "description": "the name in its ecosystem (the Go module path, the npm name)" + }, + "module": { + "type": "string" + }, + "artifact": { + "type": "string" + }, + "digest": { + "type": "string", + "description": "the member's build digest, sha256:" + } + }, + "required": [ + "package", + "module", + "artifact", + "digest" + ] + } } }, "required": [ @@ -875,6 +919,10 @@ }, "recipe": { "type": "string" + }, + "digest": { + "type": "string", + "description": "the building node's digest of its toolchain program (kind 3's entry), never the build module's own program" } }, "required": [ @@ -916,6 +964,30 @@ }, "reference": { "type": "string" + }, + "resolved": { + "type": "array", + "description": "one per mesh package the build resolved (the catalog graph entry's packages, the edges of kind 1)", + "items": { + "type": "object", + "properties": { + "package": { + "type": "string" + }, + "version": { + "type": "string" + }, + "digest": { + "type": "string", + "description": "sha256: of the package's bytes as the build fetched them" + } + }, + "required": [ + "package", + "version", + "digest" + ] + } } }, "required": [ @@ -1628,6 +1700,66 @@ "fingerprint", "steps" ] + }, + "ToolchainReported": { + "type": "object", + "description": "a node's toolchain of a language, said at the build module's start and each time its toolchains key of the language changes (design 47 G section 492)", + "properties": { + "node": { + "type": "string" + }, + "language": { + "type": "string" + }, + "version": { + "type": "string" + }, + "recipe": { + "type": "string" + }, + "digest": { + "type": "string", + "description": "the digest of the program of the provision the build module requires for that language's toolchain, never the build module's own" + } + }, + "required": [ + "node", + "language", + "version", + "recipe", + "digest" + ] + }, + "StepProgress": { + "type": "object", + "description": "a running ask's progress, said once a minute while the holder runs it (design 47 G section 496)", + "properties": { + "asker": { + "type": "string" + }, + "ask": { + "type": "string" + }, + "phase": { + "type": "string", + "enum": [ + "fetch", + "build", + "run", + "report" + ] + }, + "at": { + "type": "string", + "format": "date-time" + } + }, + "required": [ + "asker", + "ask", + "phase", + "at" + ] } }, "events": [ @@ -1641,6 +1773,28 @@ "max-age-days": 90, "max-mib": 128 } + }, + { + "name": "toolchain-reported", + "subject": "toolchain-reported.", + "body": { + "$ref": "#/defs/ToolchainReported" + }, + "keep": { + "max-age-days": 90, + "max-mib": 16 + } + }, + { + "name": "step-progress", + "subject": "step-progress.", + "body": { + "$ref": "#/defs/StepProgress" + }, + "keep": { + "max-age-days": 1, + "max-mib": 16 + } } ] } diff --git a/contracts/seats/mesh-builds/fixtures/events.json b/contracts/seats/mesh-builds/fixtures/events.json index 25c23b77..44192577 100644 --- a/contracts/seats/mesh-builds/fixtures/events.json +++ b/contracts/seats/mesh-builds/fixtures/events.json @@ -36,7 +36,21 @@ "v": 1, "language": "go", "version": "1.26.0", - "digest": "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "digests": [ + { + "digest": "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "nodes": [ + "ace", + "g14" + ] + }, + { + "digest": "sha256:fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210", + "nodes": [ + "novox" + ] + } + ], "at": "2026-11-10T10:00:00Z" } } diff --git a/contracts/seats/mesh-builds/fixtures/verbs.json b/contracts/seats/mesh-builds/fixtures/verbs.json index 668a960c..b8c50afb 100644 --- a/contracts/seats/mesh-builds/fixtures/verbs.json +++ b/contracts/seats/mesh-builds/fixtures/verbs.json @@ -6,6 +6,14 @@ "module": "mesh-sdk" } }, + { + "name": "reserved asks a package by its name", + "of": "verb:reserved.input", + "given": { + "name": "git.novox.be/novox/mesh-sdk/go", + "version": "0.1.15" + } + }, { "name": "reserved answers the version and each artifact's digest", "of": "verb:reserved.answer", diff --git a/contracts/seats/mesh-builds/seat.json b/contracts/seats/mesh-builds/seat.json index cfae1a92..29e6a5c3 100644 --- a/contracts/seats/mesh-builds/seat.json +++ b/contracts/seats/mesh-builds/seat.json @@ -16,22 +16,24 @@ "seat:container-image-registry", "seat:oci-artifact-registry" ], - "description": "A module's reserved version with each artifact's digest, for a registry holder deciding whether to publish (design 47 G section 369; design 51, Publishing).", + "description": "A module's reserved version with each artifact's digest, for a registry holder deciding whether to publish; asked by the module or by a package's name in its ecosystem (design 47 G section 369, section 478; design 51, Publishing).", "input": { "type": "object", "properties": { "module": { "type": "string", - "description": "the module" + "description": "the module; give this or name" + }, + "name": { + "type": "string", + "description": "a package's or an image's name in its ecosystem (the Go module path, the npm name, the image name), in place of module; a name no entry carries, or more than one entry carries, is answered error naming it (design 47 G section 478, section 491)" }, "version": { "type": "string", "description": "the version, when asked for a particular one" } }, - "required": [ - "module" - ] + "required": [] }, "answer": { "type": "object", @@ -198,9 +200,29 @@ "type": "string", "description": "the agreed toolchain's version" }, - "digest": { - "type": "string", - "description": "the toolchain part's digest" + "digests": { + "type": "array", + "description": "each digest the nodes reporting the agreed version report, as it stood when the version was agreed, with those nodes (design 47 G section 472, section 495)", + "items": { + "type": "object", + "properties": { + "digest": { + "type": "string", + "description": "the toolchain part's digest" + }, + "nodes": { + "type": "array", + "items": { + "type": "string" + }, + "description": "the nodes reporting that digest" + } + }, + "required": [ + "digest", + "nodes" + ] + } }, "at": { "type": "string", @@ -211,7 +233,7 @@ "v", "language", "version", - "digest" + "digests" ], "description": "the agreed toolchain of a language changed (design 47 G section 376, ADR 0584 section 1)" } diff --git a/contracts/seats/mesh-pipeline-runner/seat.json b/contracts/seats/mesh-pipeline-runner/seat.json index 5d972333..29d7b274 100644 --- a/contracts/seats/mesh-pipeline-runner/seat.json +++ b/contracts/seats/mesh-pipeline-runner/seat.json @@ -1483,6 +1483,11 @@ "how": { "type": "string", "description": "for its end, how it ended: answered, ran, released, superseded" + }, + "clock-from": { + "type": "string", + "format": "date-time", + "description": "the time the bound counts from (design 47 G section 266, section 269, section 498)" } }, "description": "a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283)", diff --git a/contracts/seats/mesh-wants/fixtures/events.json b/contracts/seats/mesh-wants/fixtures/events.json index 70fbc77c..550c6c38 100644 --- a/contracts/seats/mesh-wants/fixtures/events.json +++ b/contracts/seats/mesh-wants/fixtures/events.json @@ -166,5 +166,48 @@ "laptop" ] } + }, + { + "name": "a composition step's progress", + "of": "event:step-progress", + "subject": "mesh.seat.mesh-wants.event.step-progress.ace", + "given": { + "asker": "assigner", + "ask": "a1b2", + "phase": "build", + "at": "2026-11-11T10:01:00Z" + } + }, + { + "name": "a put without hold waits, counted from its start", + "of": "event:wait-held", + "subject": "mesh.seat.mesh-wants.event.wait-held", + "given": { + "step": "put", + "node": "ace", + "module": "nats", + "fingerprint": "fp-0123", + "since": "2026-11-11T10:00:00Z", + "clock-from": "2026-11-11T10:00:00Z", + "bound": "30m" + } + }, + { + "name": "a dependent's delivery waits for a dependency's version", + "of": "event:wait-held", + "subject": "mesh.seat.mesh-wants.event.wait-held", + "given": { + "step": "delivery", + "part": "mesh-sdk", + "version": "0.1.4", + "machines": [ + "ace", + "g14" + ], + "module": "nats", + "since": "2026-11-11T10:00:00Z", + "clock-from": "2026-11-11T09:58:00Z", + "bound": "60m" + } } ] diff --git a/contracts/seats/mesh-wants/seat.json b/contracts/seats/mesh-wants/seat.json index e002258b..6e41ad9f 100644 --- a/contracts/seats/mesh-wants/seat.json +++ b/contracts/seats/mesh-wants/seat.json @@ -427,6 +427,45 @@ "parts" ] } + }, + { + "name": "step-progress", + "subject": "step-progress.", + "keep": { + "max-age-days": 1, + "max-mib": 16 + }, + "body": { + "type": "object", + "description": "a composition step's progress, said once a minute while the assigner runs it (design 47 G section 496)", + "properties": { + "asker": { + "type": "string" + }, + "ask": { + "type": "string" + }, + "phase": { + "type": "string", + "enum": [ + "fetch", + "build", + "run", + "report" + ] + }, + "at": { + "type": "string", + "format": "date-time" + } + }, + "required": [ + "asker", + "ask", + "phase", + "at" + ] + } } ], "family": { @@ -986,7 +1025,22 @@ "description": "a step's wait the assigner holds: placing, wanting or composing until an acting part's new version runs", "properties": { "step": { - "type": "string" + "type": "string", + "enum": [ + "take-in", + "validate", + "check", + "build", + "compose", + "grant", + "place", + "pipeline", + "put", + "delivery", + "kept-build", + "backup" + ], + "description": "which wait: an acting step of section 203 or put, delivery, kept-build, backup (design 47 G section 498)" }, "part": { "type": "string", @@ -1021,12 +1075,26 @@ "by": { "type": "string", "description": "on a release, who released it and why" + }, + "clock-from": { + "type": "string", + "format": "date-time", + "description": "the time the bound counts from (design 47 G section 266, section 269, section 498)" + }, + "fingerprint": { + "type": "string", + "description": "a put's wait: the fingerprint of the build put" + }, + "machines": { + "type": "array", + "items": { + "type": "string" + }, + "description": "a dependent's delivery wait: the machines the dependency's version has yet to reach" } }, "required": [ "step", - "part", - "version", "module", "since" ] diff --git a/contracts/seats/node-build-go/fixtures/events.json b/contracts/seats/node-build-go/fixtures/events.json index 06ad58d6..1d3351e3 100644 --- a/contracts/seats/node-build-go/fixtures/events.json +++ b/contracts/seats/node-build-go/fixtures/events.json @@ -53,5 +53,28 @@ "said": "the sandbox could not fetch novox/mesh-sdk", "output-object": "mesh-build-go_sent-objects/m2kx9a1b3c4d5f" } + }, + { + "name": "a node's toolchain reported", + "of": "event:toolchain-reported", + "subject": "mesh.seat.node-build-go.event.toolchain-reported.ace", + "given": { + "node": "ace", + "language": "go", + "version": "1.26.0", + "recipe": "1", + "digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + }, + { + "name": "a running ask says its progress", + "of": "event:step-progress", + "subject": "mesh.seat.node-build-go.event.step-progress.ace", + "given": { + "asker": "runner", + "ask": "a1b2", + "phase": "fetch", + "at": "2026-11-11T10:01:00Z" + } } ] diff --git a/contracts/seats/node-build-go/fixtures/states.json b/contracts/seats/node-build-go/fixtures/states.json index 9ee56a47..bb3685b7 100644 --- a/contracts/seats/node-build-go/fixtures/states.json +++ b/contracts/seats/node-build-go/fixtures/states.json @@ -526,5 +526,17 @@ ], "at": "2026-11-10T10:00:00Z" } + }, + { + "name": "a step keyed under its asker", + "of": "state:steps/..", + "given": { + "v": 1, + "id": "a1b2", + "tool": "build", + "node": "ace", + "state": "running", + "started": "2026-11-05T10:00:00Z" + } } ] diff --git a/contracts/seats/node-build-go/fixtures/verbs.json b/contracts/seats/node-build-go/fixtures/verbs.json index 1e4af4dd..7a02ff62 100644 --- a/contracts/seats/node-build-go/fixtures/verbs.json +++ b/contracts/seats/node-build-go/fixtures/verbs.json @@ -100,5 +100,29 @@ } ] } + }, + { + "name": "steps drops an artifact's kept steps with a reason", + "of": "verb:steps.input", + "given": { + "module": "nats", + "artifact": "nats.program", + "drop": true, + "why": "the detection was wrong" + } + }, + { + "name": "steps answers a drop with the artifact", + "of": "verb:steps.answer", + "given": { + "dropped": "nats.program" + } + }, + { + "name": "steps answers a drop that the hand-act log refused", + "of": "verb:steps.answer", + "given": { + "error": "the hand-act log refused: nothing was deleted" + } } ] diff --git a/contracts/seats/node-build-go/seat.json b/contracts/seats/node-build-go/seat.json index 4d30fcd7..f9be14f8 100644 --- a/contracts/seats/node-build-go/seat.json +++ b/contracts/seats/node-build-go/seat.json @@ -87,12 +87,19 @@ "name": "steps", "template": "state", "key-forms": [ + { + "key": "..", + "value": { + "$ref": "#/defs/Step" + }, + "description": "a step on a node, keyed by the asker the bus proves (design 47 G section 467, section 481)" + }, { "key": ".", "value": { "$ref": "#/defs/Step" }, - "description": "a step on a node" + "description": "a step whose ask came on main's subject, with no asker's name (design 47 G section 483)" } ], "description": "the steps this module runs", diff --git a/contracts/seats/node-build-image/fixtures/events.json b/contracts/seats/node-build-image/fixtures/events.json index 1e20d2ed..e8dca58f 100644 --- a/contracts/seats/node-build-image/fixtures/events.json +++ b/contracts/seats/node-build-image/fixtures/events.json @@ -36,5 +36,28 @@ "said": "the sandbox could not fetch novox/mesh-sdk", "output-object": "mesh-build-go_sent-objects/m2kx9a1b3c4d5f" } + }, + { + "name": "a node's toolchain reported", + "of": "event:toolchain-reported", + "subject": "mesh.seat.node-build-image.event.toolchain-reported.ace", + "given": { + "node": "ace", + "language": "go", + "version": "1.26.0", + "recipe": "1", + "digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + }, + { + "name": "a running ask says its progress", + "of": "event:step-progress", + "subject": "mesh.seat.node-build-image.event.step-progress.ace", + "given": { + "asker": "runner", + "ask": "a1b2", + "phase": "fetch", + "at": "2026-11-11T10:01:00Z" + } } ] diff --git a/contracts/seats/node-build-image/fixtures/states.json b/contracts/seats/node-build-image/fixtures/states.json index 3cbef09e..149ea1ca 100644 --- a/contracts/seats/node-build-image/fixtures/states.json +++ b/contracts/seats/node-build-image/fixtures/states.json @@ -186,5 +186,17 @@ ], "at": "2026-11-10T10:00:00Z" } + }, + { + "name": "a step keyed under its asker", + "of": "state:steps/..", + "given": { + "v": 1, + "id": "a1b2", + "tool": "build", + "node": "ace", + "state": "running", + "started": "2026-11-05T10:00:00Z" + } } ] diff --git a/contracts/seats/node-build-image/fixtures/verbs.json b/contracts/seats/node-build-image/fixtures/verbs.json index 1e4af4dd..7a02ff62 100644 --- a/contracts/seats/node-build-image/fixtures/verbs.json +++ b/contracts/seats/node-build-image/fixtures/verbs.json @@ -100,5 +100,29 @@ } ] } + }, + { + "name": "steps drops an artifact's kept steps with a reason", + "of": "verb:steps.input", + "given": { + "module": "nats", + "artifact": "nats.program", + "drop": true, + "why": "the detection was wrong" + } + }, + { + "name": "steps answers a drop with the artifact", + "of": "verb:steps.answer", + "given": { + "dropped": "nats.program" + } + }, + { + "name": "steps answers a drop that the hand-act log refused", + "of": "verb:steps.answer", + "given": { + "error": "the hand-act log refused: nothing was deleted" + } } ] diff --git a/contracts/seats/node-build-image/seat.json b/contracts/seats/node-build-image/seat.json index 2d5f83ab..5528da1f 100644 --- a/contracts/seats/node-build-image/seat.json +++ b/contracts/seats/node-build-image/seat.json @@ -65,12 +65,19 @@ "name": "steps", "template": "state", "key-forms": [ + { + "key": "..", + "value": { + "$ref": "#/defs/Step" + }, + "description": "a step on a node, keyed by the asker the bus proves (design 47 G section 467, section 481)" + }, { "key": ".", "value": { "$ref": "#/defs/Step" }, - "description": "a step on a node" + "description": "a step whose ask came on main's subject, with no asker's name (design 47 G section 483)" } ], "description": "the steps this module runs", diff --git a/contracts/seats/node-build-node/fixtures/events.json b/contracts/seats/node-build-node/fixtures/events.json index 0fb1ac2d..1f1924a0 100644 --- a/contracts/seats/node-build-node/fixtures/events.json +++ b/contracts/seats/node-build-node/fixtures/events.json @@ -53,5 +53,28 @@ "said": "the sandbox could not fetch novox/mesh-sdk", "output-object": "mesh-build-go_sent-objects/m2kx9a1b3c4d5f" } + }, + { + "name": "a node's toolchain reported", + "of": "event:toolchain-reported", + "subject": "mesh.seat.node-build-node.event.toolchain-reported.ace", + "given": { + "node": "ace", + "language": "go", + "version": "1.26.0", + "recipe": "1", + "digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + }, + { + "name": "a running ask says its progress", + "of": "event:step-progress", + "subject": "mesh.seat.node-build-node.event.step-progress.ace", + "given": { + "asker": "runner", + "ask": "a1b2", + "phase": "fetch", + "at": "2026-11-11T10:01:00Z" + } } ] diff --git a/contracts/seats/node-build-node/fixtures/states.json b/contracts/seats/node-build-node/fixtures/states.json index ccf06869..e077d0d6 100644 --- a/contracts/seats/node-build-node/fixtures/states.json +++ b/contracts/seats/node-build-node/fixtures/states.json @@ -274,5 +274,17 @@ ], "at": "2026-11-10T10:00:00Z" } + }, + { + "name": "a step keyed under its asker", + "of": "state:steps/..", + "given": { + "v": 1, + "id": "a1b2", + "tool": "build", + "node": "ace", + "state": "running", + "started": "2026-11-05T10:00:00Z" + } } ] diff --git a/contracts/seats/node-build-node/fixtures/verbs.json b/contracts/seats/node-build-node/fixtures/verbs.json index 1e4af4dd..7a02ff62 100644 --- a/contracts/seats/node-build-node/fixtures/verbs.json +++ b/contracts/seats/node-build-node/fixtures/verbs.json @@ -100,5 +100,29 @@ } ] } + }, + { + "name": "steps drops an artifact's kept steps with a reason", + "of": "verb:steps.input", + "given": { + "module": "nats", + "artifact": "nats.program", + "drop": true, + "why": "the detection was wrong" + } + }, + { + "name": "steps answers a drop with the artifact", + "of": "verb:steps.answer", + "given": { + "dropped": "nats.program" + } + }, + { + "name": "steps answers a drop that the hand-act log refused", + "of": "verb:steps.answer", + "given": { + "error": "the hand-act log refused: nothing was deleted" + } } ] diff --git a/contracts/seats/node-build-node/seat.json b/contracts/seats/node-build-node/seat.json index 978f9dde..910ff77c 100644 --- a/contracts/seats/node-build-node/seat.json +++ b/contracts/seats/node-build-node/seat.json @@ -72,12 +72,19 @@ "name": "steps", "template": "state", "key-forms": [ + { + "key": "..", + "value": { + "$ref": "#/defs/Step" + }, + "description": "a step on a node, keyed by the asker the bus proves (design 47 G section 467, section 481)" + }, { "key": ".", "value": { "$ref": "#/defs/Step" }, - "description": "a step on a node" + "description": "a step whose ask came on main's subject, with no asker's name (design 47 G section 483)" } ], "description": "the steps this module runs", diff --git a/contracts/ts/seats/meshbuilds/index.d.ts b/contracts/ts/seats/meshbuilds/index.d.ts index 5c6dfedc..9732e7cd 100644 --- a/contracts/ts/seats/meshbuilds/index.d.ts +++ b/contracts/ts/seats/meshbuilds/index.d.ts @@ -55,11 +55,18 @@ export interface PackagePublished { "version": string; } +export interface ToolchainAgreedDigestsItem { + /** the toolchain part's digest */ + "digest": string; + /** the nodes reporting that digest */ + "nodes": string[]; +} + /** the agreed toolchain of a language changed (design 47 G section 376, ADR 0584 section 1) */ export interface ToolchainAgreed { "at"?: string; - /** the toolchain part's digest */ - "digest": string; + /** each digest the nodes reporting the agreed version report, as it stood when the version was agreed, with those nodes (design 47 G section 472, section 495) */ + "digests": ToolchainAgreedDigestsItem[]; /** the language whose agreed toolchain changed */ "language": string; "v": number; @@ -69,8 +76,10 @@ export interface ToolchainAgreed { /** is what reserved takes. */ export interface ReservedInput { - /** the module */ - "module": string; + /** the module; give this or name */ + "module"?: string; + /** a package's or an image's name in its ecosystem (the Go module path, the npm name, the image name), in place of module; a name no entry carries, or more than one entry carries, is answered error naming it (design 47 G section 478, section 491) */ + "name"?: string; /** the version, when asked for a particular one */ "version"?: string; } diff --git a/contracts/ts/seats/meshpipelinerunner/index.d.ts b/contracts/ts/seats/meshpipelinerunner/index.d.ts index cf1d8263..abcd517c 100644 --- a/contracts/ts/seats/meshpipelinerunner/index.d.ts +++ b/contracts/ts/seats/meshpipelinerunner/index.d.ts @@ -364,6 +364,8 @@ export interface Wait { "artifact"?: string; /** the watched branch, or the branch of a run on request */ "branch": string; + /** the time the bound counts from (design 47 G section 266, section 269, section 498) */ + "clock-from"?: string; /** the commit the module's entry was last changed at */ "commit"?: string; /** the module's build-input fingerprint at the commit */ diff --git a/contracts/ts/seats/meshwants/index.d.ts b/contracts/ts/seats/meshwants/index.d.ts index 804dc8c1..91a28584 100644 --- a/contracts/ts/seats/meshwants/index.d.ts +++ b/contracts/ts/seats/meshwants/index.d.ts @@ -29,6 +29,7 @@ export declare const events: { readonly room: "room"; readonly stepEnded: "step-ended"; readonly versionDelivered: "version-delivered"; + readonly stepProgress: "step-progress"; }; export declare const family: { readonly name: "mesh-wants.events"; readonly maxAgeDays: 7 }; @@ -61,6 +62,12 @@ export declare const versionDeliveredStream: "SEAT_mesh-wants_version-delivered" export declare const versionDeliveredSubject: "mesh.seat.mesh-wants.event.version-delivered"; +export declare const stepProgressStream: "SEAT_mesh-wants_step-progress"; + +export declare function stepProgressSubject(node: string): string; + +export declare const stepProgressSubjectAll: "mesh.seat.mesh-wants.event.step-progress.*"; + export declare const stepSubject: "mesh.seat.mesh-wants.accept.step"; export declare const states: { @@ -304,18 +311,25 @@ export interface Wait { "bound"?: string; /** on a release, who released it and why */ "by"?: string; + /** the time the bound counts from (design 47 G section 266, section 269, section 498) */ + "clock-from"?: string; + /** a put's wait: the fingerprint of the build put */ + "fingerprint"?: string; "fp"?: string; /** on wait-ended: ran or released */ "how"?: string; + /** a dependent's delivery wait: the machines the dependency's version has yet to reach */ + "machines"?: string[]; /** the entry that waits */ "module": string; "node"?: string; /** the acting part waited for */ - "part": string; + "part"?: string; "since": string; - "step": string; + /** which wait: an acting step of section 203 or put, delivery, kept-build, backup (design 47 G section 498) */ + "step": "take-in" | "validate" | "check" | "build" | "compose" | "grant" | "place" | "pipeline" | "put" | "delivery" | "kept-build" | "backup"; /** its new version's fingerprint */ - "version": string; + "version"?: string; } /** is what upgrade takes. */ @@ -474,6 +488,14 @@ export interface VersionDeliveredEvent { "version": string; } +/** is the body of the event step-progress. */ +export interface StepProgressEvent { + "ask": string; + "asker": string; + "at": string; + "phase": "fetch" | "build" | "run" | "report"; +} + /** is one value of the state wants under the key want... */ export interface WantsByWantNodeModuleValue { /** again or skip */ diff --git a/contracts/ts/seats/meshwants/index.js b/contracts/ts/seats/meshwants/index.js index 96565037..488f1228 100644 --- a/contracts/ts/seats/meshwants/index.js +++ b/contracts/ts/seats/meshwants/index.js @@ -29,6 +29,7 @@ export const events = { room: "room", stepEnded: "step-ended", versionDelivered: "version-delivered", + stepProgress: "step-progress", }; export const family = { name: "mesh-wants.events", maxAgeDays: 7 }; @@ -63,6 +64,14 @@ export const versionDeliveredStream = "SEAT_mesh-wants_version-delivered"; export const versionDeliveredSubject = "mesh.seat.mesh-wants.event.version-delivered"; +export const stepProgressStream = "SEAT_mesh-wants_step-progress"; + +export function stepProgressSubject(node) { + return "mesh.seat.mesh-wants.event.step-progress." + node; +} + +export const stepProgressSubjectAll = "mesh.seat.mesh-wants.event.step-progress.*"; + export const stepSubject = "mesh.seat.mesh-wants.accept.step"; export const states = { diff --git a/contracts/ts/seats/nodebuildgo/index.d.ts b/contracts/ts/seats/nodebuildgo/index.d.ts index ff6e8d0e..ac126e27 100644 --- a/contracts/ts/seats/nodebuildgo/index.d.ts +++ b/contracts/ts/seats/nodebuildgo/index.d.ts @@ -26,6 +26,8 @@ export declare const accepts: { export declare const events: { readonly stepEnded: "step-ended"; + readonly toolchainReported: "toolchain-reported"; + readonly stepProgress: "step-progress"; }; export declare const family: { readonly name: "node-build-go.events"; readonly maxAgeDays: 7 }; @@ -36,6 +38,18 @@ export declare function stepEndedSubject(node: string): string; export declare const stepEndedSubjectAll: "mesh.seat.node-build-go.event.step-ended.*"; +export declare const toolchainReportedStream: "SEAT_node-build-go_toolchain-reported"; + +export declare function toolchainReportedSubject(node: string): string; + +export declare const toolchainReportedSubjectAll: "mesh.seat.node-build-go.event.toolchain-reported.*"; + +export declare const stepProgressStream: "SEAT_node-build-go_step-progress"; + +export declare function stepProgressSubject(node: string): string; + +export declare const stepProgressSubjectAll: "mesh.seat.node-build-go.event.step-progress.*"; + export declare const deliverySubject: "mesh.seat.node-build-go.accept.delivery"; export declare const checkSubject: "mesh.seat.node-build-go.accept.check"; @@ -55,6 +69,12 @@ export declare const states: { readonly detectedSteps: "detected-steps"; }; +/** The key of one value of the state steps under the form "..". */ +export declare function stepsByAskerNodeIDKey(asker: string, node: string, id: string): string; + +/** The parts of a key of the form "..", or undefined for a key of another form. */ +export declare function parseStepsByAskerNodeIDKey(_key: string): { asker: string; node: string; id: string } | undefined; + /** The key of one value of the state steps under the form ".". */ export declare function stepsByNodeIDKey(node: string, id: string): string; @@ -94,6 +114,7 @@ export declare function parseDetectedStepsByModuleArtifactKey(_key: string): { m export type AskArtifact = buildStep.AskArtifact; export type BuildAnswer = buildStep.BuildAnswer; export type BuiltArtifact = buildStep.BuiltArtifact; +export type BuiltArtifactResolvedItem = buildStep.BuiltArtifactResolvedItem; export type Capacity = buildStep.Capacity; export type CheckAnswer = buildStep.CheckAnswer; export type ComposeAnswer = buildStep.ComposeAnswer; @@ -107,6 +128,7 @@ export type DropInput = buildStep.DropInput; export type Fetch = buildStep.Fetch; export type FetchAnswer = buildStep.FetchAnswer; export type FetchBesideItem = buildStep.FetchBesideItem; +export type FetchBuildsItem = buildStep.FetchBuildsItem; export type FetchPackage = buildStep.FetchPackage; export type FetchRefusal = buildStep.FetchRefusal; export type KeptArtifactAnswer = buildStep.KeptArtifactAnswer; @@ -139,6 +161,8 @@ export type StepAskCompose = buildStep.StepAskCompose; export type StepEnded = buildStep.StepEnded; export type StepEndedEvent = buildStep.StepEndedEvent; export type StepNames = buildStep.StepNames; +export type StepProgress = buildStep.StepProgress; +export type StepProgressEvent = buildStep.StepProgressEvent; export type StepResult = buildStep.StepResult; export type StepRun = buildStep.StepRun; export type StepsAnswer = buildStep.StepsAnswer; @@ -146,6 +170,8 @@ export type StepsInput = buildStep.StepsInput; export type TakenVersion = buildStep.TakenVersion; export type TestCounts = buildStep.TestCounts; export type Toolchain = buildStep.Toolchain; +export type ToolchainReported = buildStep.ToolchainReported; +export type ToolchainReportedEvent = buildStep.ToolchainReportedEvent; export type ToolchainUsed = buildStep.ToolchainUsed; /** is the body of delivery. */ @@ -166,6 +192,9 @@ export type RecheckAccept = StepAsk; /** is the body of check.run. */ export type CheckRunAccept = StepAsk; +/** is one value of the state steps under the key ... */ +export type StepsByAskerNodeIDValue = Step; + /** is one value of the state steps under the key .. */ export type StepsByNodeIDValue = Step; diff --git a/contracts/ts/seats/nodebuildgo/index.js b/contracts/ts/seats/nodebuildgo/index.js index e80d7956..cf4a3f99 100644 --- a/contracts/ts/seats/nodebuildgo/index.js +++ b/contracts/ts/seats/nodebuildgo/index.js @@ -24,6 +24,8 @@ export const accepts = { export const events = { stepEnded: "step-ended", + toolchainReported: "toolchain-reported", + stepProgress: "step-progress", }; export const family = { name: "node-build-go.events", maxAgeDays: 7 }; @@ -36,6 +38,22 @@ export function stepEndedSubject(node) { export const stepEndedSubjectAll = "mesh.seat.node-build-go.event.step-ended.*"; +export const toolchainReportedStream = "SEAT_node-build-go_toolchain-reported"; + +export function toolchainReportedSubject(node) { + return "mesh.seat.node-build-go.event.toolchain-reported." + node; +} + +export const toolchainReportedSubjectAll = "mesh.seat.node-build-go.event.toolchain-reported.*"; + +export const stepProgressStream = "SEAT_node-build-go_step-progress"; + +export function stepProgressSubject(node) { + return "mesh.seat.node-build-go.event.step-progress." + node; +} + +export const stepProgressSubjectAll = "mesh.seat.node-build-go.event.step-progress.*"; + export const deliverySubject = "mesh.seat.node-build-go.accept.delivery"; export const checkSubject = "mesh.seat.node-build-go.accept.check"; @@ -55,6 +73,29 @@ export const states = { detectedSteps: "detected-steps", }; +/** The key of one value of the state steps under the form "..". */ +export function stepsByAskerNodeIDKey(asker, node, id) { + return asker + "." + node + "." + id; +} + +/** The parts of a key of the form "..", or undefined for a key of another form. */ +export function parseStepsByAskerNodeIDKey(_key) { + let _rest = _key; + const _at0 = _rest.indexOf("."); + if (_at0 <= 0) return undefined; + const asker = _rest.slice(0, _at0); + _rest = _rest.slice(_at0 + 1); + const _at2 = _rest.indexOf("."); + if (_at2 <= 0) return undefined; + const node = _rest.slice(0, _at2); + _rest = _rest.slice(_at2 + 1); + if (_rest === "") return undefined; + const id = _rest; + _rest = ""; + if (_rest !== "") return undefined; + return { asker, node, id }; +} + /** The key of one value of the state steps under the form ".". */ export function stepsByNodeIDKey(node, id) { return node + "." + id; diff --git a/contracts/ts/seats/nodebuildgo/parts/build-step.d.ts b/contracts/ts/seats/nodebuildgo/parts/build-step.d.ts index 3a3c18ba..a6cc4551 100644 --- a/contracts/ts/seats/nodebuildgo/parts/build-step.d.ts +++ b/contracts/ts/seats/nodebuildgo/parts/build-step.d.ts @@ -13,6 +13,8 @@ export declare const verbs: { export declare const events: { readonly stepEnded: "step-ended"; + readonly toolchainReported: "toolchain-reported"; + readonly stepProgress: "step-progress"; }; /** one artifact of a build ask: built from the fetched tree, or copied from an upstream pin */ @@ -32,11 +34,20 @@ export interface BuildAnswer { "toolchain"?: ToolchainUsed; } +export interface BuiltArtifactResolvedItem { + /** sha256: of the package's bytes as the build fetched them */ + "digest": string; + "package": string; + "version": string; +} + /** one artifact a build made: its digest and the reference it was published under */ export interface BuiltArtifact { "artifact": string; "digest": string; "reference"?: string; + /** one per mesh package the build resolved (the catalog graph entry's packages, the edges of kind 1) */ + "resolved"?: BuiltArtifactResolvedItem[]; } /** how many steps the node runs at once */ @@ -100,6 +111,15 @@ export interface FetchBesideItem { "repository": string; } +export interface FetchBuildsItem { + "artifact": string; + /** the member's build digest, sha256: */ + "digest": string; + "module": string; + /** the name in its ecosystem (the Go module path, the npm name) */ + "package": string; +} + /** the TypeScript module's group package: the group's tagged version of the SDK ({name, version}), placed from the registry in each member whatever its lock names, or the group's package member ({repository, commit, ref}), packed by the fetch and served to the members in place of the registry's */ export interface FetchPackage { "commit"?: string; @@ -115,6 +135,8 @@ export interface Fetch { "base"?: string; /** the sibling repositories placed beside the fetched tree */ "beside"?: FetchBesideItem[]; + /** a group run's package members, each by its build's digest: the build module reads each from the artifact store at artifact-store:////blobs/, checks the bytes against the digest and lays them as the group package; a package named here is named in neither packages nor package */ + "builds"?: FetchBuildsItem[]; /** the full commit */ "commit": string; /** the module's directory in the repository ("." for its root), with paths */ @@ -416,6 +438,14 @@ export interface StepNames { "repositories": string[]; } +/** a running ask's progress, said once a minute while the holder runs it (design 47 G section 496) */ +export interface StepProgress { + "ask": string; + "asker": string; + "at": string; + "phase": "fetch" | "build" | "run" | "report"; +} + /** what a step answered, by the kind of ask: a build's artifacts, toolchain, imports, out-of-step modules and the artifact it kept; a check's fetch and levels; a compose's built, key and verdict; a run's built, key, exit, output and said; a declined ask's declined, needs and lacks. The fields of the kinds do not meet, so one layout holds the five */ export interface StepResult { "artifacts"?: BuiltArtifact[]; @@ -502,8 +532,20 @@ export interface Toolchain { "version": string; } +/** a node's toolchain of a language, said at the build module's start and each time its toolchains key of the language changes (design 47 G section 492) */ +export interface ToolchainReported { + /** the digest of the program of the provision the build module requires for that language's toolchain, never the build module's own */ + "digest": string; + "language": string; + "node": string; + "recipe": string; + "version": string; +} + /** the toolchain a build was made with: the language, the installed version and the build module's recipe */ export interface ToolchainUsed { + /** the building node's digest of its toolchain program (kind 3's entry), never the build module's own program */ + "digest"?: string; "language": string; "recipe": string; "version": string; @@ -579,22 +621,36 @@ export interface KeptArtifactAnswer { export interface StepsInput { /** find them anew, not from the kept answer */ "again"?: boolean; - /** the artifact reference, : */ + /** the key of detected-steps, . */ "artifact": string; - "fetch": Fetch; + /** delete the artifact's kept key instead of detecting; needs why */ + "drop"?: boolean; + "fetch"?: Fetch; "fingerprint"?: string; "module": string; + /** with drop, why: a non-empty text, written in the hand-act log with the caller the bus proved */ + "why"?: string; } /** is what steps answers. */ export interface StepsAnswer { - "artifact": string; + "artifact"?: string; + /** the artifact whose key a drop deleted */ + "dropped"?: string; + /** why a drop deleted nothing, naming the hand-act log when it refused */ + "error"?: string; "fingerprint"?: string; /** answered from the kept answer */ "kept"?: boolean; - "steps": DetectedStep[]; + "steps"?: DetectedStep[]; } /** is the body of the event step-ended. */ export type StepEndedEvent = StepEnded; +/** is the body of the event toolchain-reported. */ +export type ToolchainReportedEvent = ToolchainReported; + +/** is the body of the event step-progress. */ +export type StepProgressEvent = StepProgress; + diff --git a/contracts/ts/seats/nodebuildgo/parts/build-step.js b/contracts/ts/seats/nodebuildgo/parts/build-step.js index 027da325..155a68be 100644 --- a/contracts/ts/seats/nodebuildgo/parts/build-step.js +++ b/contracts/ts/seats/nodebuildgo/parts/build-step.js @@ -13,6 +13,8 @@ export const verbs = { export const events = { stepEnded: "step-ended", + toolchainReported: "toolchain-reported", + stepProgress: "step-progress", }; export {}; diff --git a/contracts/ts/seats/nodebuildimage/index.d.ts b/contracts/ts/seats/nodebuildimage/index.d.ts index 2cfdc13b..06659b29 100644 --- a/contracts/ts/seats/nodebuildimage/index.d.ts +++ b/contracts/ts/seats/nodebuildimage/index.d.ts @@ -25,6 +25,8 @@ export declare const accepts: { export declare const events: { readonly stepEnded: "step-ended"; + readonly toolchainReported: "toolchain-reported"; + readonly stepProgress: "step-progress"; }; export declare const family: { readonly name: "node-build-image.events"; readonly maxAgeDays: 7 }; @@ -35,6 +37,18 @@ export declare function stepEndedSubject(node: string): string; export declare const stepEndedSubjectAll: "mesh.seat.node-build-image.event.step-ended.*"; +export declare const toolchainReportedStream: "SEAT_node-build-image_toolchain-reported"; + +export declare function toolchainReportedSubject(node: string): string; + +export declare const toolchainReportedSubjectAll: "mesh.seat.node-build-image.event.toolchain-reported.*"; + +export declare const stepProgressStream: "SEAT_node-build-image_step-progress"; + +export declare function stepProgressSubject(node: string): string; + +export declare const stepProgressSubjectAll: "mesh.seat.node-build-image.event.step-progress.*"; + export declare const deliverySubject: "mesh.seat.node-build-image.accept.delivery"; export declare const checkSubject: "mesh.seat.node-build-image.accept.check"; @@ -52,6 +66,12 @@ export declare const states: { readonly detectedSteps: "detected-steps"; }; +/** The key of one value of the state steps under the form "..". */ +export declare function stepsByAskerNodeIDKey(asker: string, node: string, id: string): string; + +/** The parts of a key of the form "..", or undefined for a key of another form. */ +export declare function parseStepsByAskerNodeIDKey(_key: string): { asker: string; node: string; id: string } | undefined; + /** The key of one value of the state steps under the form ".". */ export declare function stepsByNodeIDKey(node: string, id: string): string; @@ -91,6 +111,7 @@ export declare function parseDetectedStepsByModuleArtifactKey(_key: string): { m export type AskArtifact = buildStep.AskArtifact; export type BuildAnswer = buildStep.BuildAnswer; export type BuiltArtifact = buildStep.BuiltArtifact; +export type BuiltArtifactResolvedItem = buildStep.BuiltArtifactResolvedItem; export type Capacity = buildStep.Capacity; export type CheckAnswer = buildStep.CheckAnswer; export type ComposeAnswer = buildStep.ComposeAnswer; @@ -104,6 +125,7 @@ export type DropInput = buildStep.DropInput; export type Fetch = buildStep.Fetch; export type FetchAnswer = buildStep.FetchAnswer; export type FetchBesideItem = buildStep.FetchBesideItem; +export type FetchBuildsItem = buildStep.FetchBuildsItem; export type FetchPackage = buildStep.FetchPackage; export type FetchRefusal = buildStep.FetchRefusal; export type KeptArtifactAnswer = buildStep.KeptArtifactAnswer; @@ -136,6 +158,8 @@ export type StepAskCompose = buildStep.StepAskCompose; export type StepEnded = buildStep.StepEnded; export type StepEndedEvent = buildStep.StepEndedEvent; export type StepNames = buildStep.StepNames; +export type StepProgress = buildStep.StepProgress; +export type StepProgressEvent = buildStep.StepProgressEvent; export type StepResult = buildStep.StepResult; export type StepRun = buildStep.StepRun; export type StepsAnswer = buildStep.StepsAnswer; @@ -143,6 +167,8 @@ export type StepsInput = buildStep.StepsInput; export type TakenVersion = buildStep.TakenVersion; export type TestCounts = buildStep.TestCounts; export type Toolchain = buildStep.Toolchain; +export type ToolchainReported = buildStep.ToolchainReported; +export type ToolchainReportedEvent = buildStep.ToolchainReportedEvent; export type ToolchainUsed = buildStep.ToolchainUsed; /** is the body of delivery. */ @@ -160,6 +186,9 @@ export type DeliveryStepAccept = StepAsk; /** is the body of recheck. */ export type RecheckAccept = StepAsk; +/** is one value of the state steps under the key ... */ +export type StepsByAskerNodeIDValue = Step; + /** is one value of the state steps under the key .. */ export type StepsByNodeIDValue = Step; diff --git a/contracts/ts/seats/nodebuildimage/index.js b/contracts/ts/seats/nodebuildimage/index.js index 6df298da..8a23cc70 100644 --- a/contracts/ts/seats/nodebuildimage/index.js +++ b/contracts/ts/seats/nodebuildimage/index.js @@ -23,6 +23,8 @@ export const accepts = { export const events = { stepEnded: "step-ended", + toolchainReported: "toolchain-reported", + stepProgress: "step-progress", }; export const family = { name: "node-build-image.events", maxAgeDays: 7 }; @@ -35,6 +37,22 @@ export function stepEndedSubject(node) { export const stepEndedSubjectAll = "mesh.seat.node-build-image.event.step-ended.*"; +export const toolchainReportedStream = "SEAT_node-build-image_toolchain-reported"; + +export function toolchainReportedSubject(node) { + return "mesh.seat.node-build-image.event.toolchain-reported." + node; +} + +export const toolchainReportedSubjectAll = "mesh.seat.node-build-image.event.toolchain-reported.*"; + +export const stepProgressStream = "SEAT_node-build-image_step-progress"; + +export function stepProgressSubject(node) { + return "mesh.seat.node-build-image.event.step-progress." + node; +} + +export const stepProgressSubjectAll = "mesh.seat.node-build-image.event.step-progress.*"; + export const deliverySubject = "mesh.seat.node-build-image.accept.delivery"; export const checkSubject = "mesh.seat.node-build-image.accept.check"; @@ -52,6 +70,29 @@ export const states = { detectedSteps: "detected-steps", }; +/** The key of one value of the state steps under the form "..". */ +export function stepsByAskerNodeIDKey(asker, node, id) { + return asker + "." + node + "." + id; +} + +/** The parts of a key of the form "..", or undefined for a key of another form. */ +export function parseStepsByAskerNodeIDKey(_key) { + let _rest = _key; + const _at0 = _rest.indexOf("."); + if (_at0 <= 0) return undefined; + const asker = _rest.slice(0, _at0); + _rest = _rest.slice(_at0 + 1); + const _at2 = _rest.indexOf("."); + if (_at2 <= 0) return undefined; + const node = _rest.slice(0, _at2); + _rest = _rest.slice(_at2 + 1); + if (_rest === "") return undefined; + const id = _rest; + _rest = ""; + if (_rest !== "") return undefined; + return { asker, node, id }; +} + /** The key of one value of the state steps under the form ".". */ export function stepsByNodeIDKey(node, id) { return node + "." + id; diff --git a/contracts/ts/seats/nodebuildimage/parts/build-step.d.ts b/contracts/ts/seats/nodebuildimage/parts/build-step.d.ts index 3a3c18ba..a6cc4551 100644 --- a/contracts/ts/seats/nodebuildimage/parts/build-step.d.ts +++ b/contracts/ts/seats/nodebuildimage/parts/build-step.d.ts @@ -13,6 +13,8 @@ export declare const verbs: { export declare const events: { readonly stepEnded: "step-ended"; + readonly toolchainReported: "toolchain-reported"; + readonly stepProgress: "step-progress"; }; /** one artifact of a build ask: built from the fetched tree, or copied from an upstream pin */ @@ -32,11 +34,20 @@ export interface BuildAnswer { "toolchain"?: ToolchainUsed; } +export interface BuiltArtifactResolvedItem { + /** sha256: of the package's bytes as the build fetched them */ + "digest": string; + "package": string; + "version": string; +} + /** one artifact a build made: its digest and the reference it was published under */ export interface BuiltArtifact { "artifact": string; "digest": string; "reference"?: string; + /** one per mesh package the build resolved (the catalog graph entry's packages, the edges of kind 1) */ + "resolved"?: BuiltArtifactResolvedItem[]; } /** how many steps the node runs at once */ @@ -100,6 +111,15 @@ export interface FetchBesideItem { "repository": string; } +export interface FetchBuildsItem { + "artifact": string; + /** the member's build digest, sha256: */ + "digest": string; + "module": string; + /** the name in its ecosystem (the Go module path, the npm name) */ + "package": string; +} + /** the TypeScript module's group package: the group's tagged version of the SDK ({name, version}), placed from the registry in each member whatever its lock names, or the group's package member ({repository, commit, ref}), packed by the fetch and served to the members in place of the registry's */ export interface FetchPackage { "commit"?: string; @@ -115,6 +135,8 @@ export interface Fetch { "base"?: string; /** the sibling repositories placed beside the fetched tree */ "beside"?: FetchBesideItem[]; + /** a group run's package members, each by its build's digest: the build module reads each from the artifact store at artifact-store:////blobs/, checks the bytes against the digest and lays them as the group package; a package named here is named in neither packages nor package */ + "builds"?: FetchBuildsItem[]; /** the full commit */ "commit": string; /** the module's directory in the repository ("." for its root), with paths */ @@ -416,6 +438,14 @@ export interface StepNames { "repositories": string[]; } +/** a running ask's progress, said once a minute while the holder runs it (design 47 G section 496) */ +export interface StepProgress { + "ask": string; + "asker": string; + "at": string; + "phase": "fetch" | "build" | "run" | "report"; +} + /** what a step answered, by the kind of ask: a build's artifacts, toolchain, imports, out-of-step modules and the artifact it kept; a check's fetch and levels; a compose's built, key and verdict; a run's built, key, exit, output and said; a declined ask's declined, needs and lacks. The fields of the kinds do not meet, so one layout holds the five */ export interface StepResult { "artifacts"?: BuiltArtifact[]; @@ -502,8 +532,20 @@ export interface Toolchain { "version": string; } +/** a node's toolchain of a language, said at the build module's start and each time its toolchains key of the language changes (design 47 G section 492) */ +export interface ToolchainReported { + /** the digest of the program of the provision the build module requires for that language's toolchain, never the build module's own */ + "digest": string; + "language": string; + "node": string; + "recipe": string; + "version": string; +} + /** the toolchain a build was made with: the language, the installed version and the build module's recipe */ export interface ToolchainUsed { + /** the building node's digest of its toolchain program (kind 3's entry), never the build module's own program */ + "digest"?: string; "language": string; "recipe": string; "version": string; @@ -579,22 +621,36 @@ export interface KeptArtifactAnswer { export interface StepsInput { /** find them anew, not from the kept answer */ "again"?: boolean; - /** the artifact reference, : */ + /** the key of detected-steps, . */ "artifact": string; - "fetch": Fetch; + /** delete the artifact's kept key instead of detecting; needs why */ + "drop"?: boolean; + "fetch"?: Fetch; "fingerprint"?: string; "module": string; + /** with drop, why: a non-empty text, written in the hand-act log with the caller the bus proved */ + "why"?: string; } /** is what steps answers. */ export interface StepsAnswer { - "artifact": string; + "artifact"?: string; + /** the artifact whose key a drop deleted */ + "dropped"?: string; + /** why a drop deleted nothing, naming the hand-act log when it refused */ + "error"?: string; "fingerprint"?: string; /** answered from the kept answer */ "kept"?: boolean; - "steps": DetectedStep[]; + "steps"?: DetectedStep[]; } /** is the body of the event step-ended. */ export type StepEndedEvent = StepEnded; +/** is the body of the event toolchain-reported. */ +export type ToolchainReportedEvent = ToolchainReported; + +/** is the body of the event step-progress. */ +export type StepProgressEvent = StepProgress; + diff --git a/contracts/ts/seats/nodebuildimage/parts/build-step.js b/contracts/ts/seats/nodebuildimage/parts/build-step.js index 027da325..155a68be 100644 --- a/contracts/ts/seats/nodebuildimage/parts/build-step.js +++ b/contracts/ts/seats/nodebuildimage/parts/build-step.js @@ -13,6 +13,8 @@ export const verbs = { export const events = { stepEnded: "step-ended", + toolchainReported: "toolchain-reported", + stepProgress: "step-progress", }; export {}; diff --git a/contracts/ts/seats/nodebuildnode/index.d.ts b/contracts/ts/seats/nodebuildnode/index.d.ts index 3a87dc86..0d4743ec 100644 --- a/contracts/ts/seats/nodebuildnode/index.d.ts +++ b/contracts/ts/seats/nodebuildnode/index.d.ts @@ -25,6 +25,8 @@ export declare const accepts: { export declare const events: { readonly stepEnded: "step-ended"; + readonly toolchainReported: "toolchain-reported"; + readonly stepProgress: "step-progress"; }; export declare const family: { readonly name: "node-build-node.events"; readonly maxAgeDays: 7 }; @@ -35,6 +37,18 @@ export declare function stepEndedSubject(node: string): string; export declare const stepEndedSubjectAll: "mesh.seat.node-build-node.event.step-ended.*"; +export declare const toolchainReportedStream: "SEAT_node-build-node_toolchain-reported"; + +export declare function toolchainReportedSubject(node: string): string; + +export declare const toolchainReportedSubjectAll: "mesh.seat.node-build-node.event.toolchain-reported.*"; + +export declare const stepProgressStream: "SEAT_node-build-node_step-progress"; + +export declare function stepProgressSubject(node: string): string; + +export declare const stepProgressSubjectAll: "mesh.seat.node-build-node.event.step-progress.*"; + export declare const deliverySubject: "mesh.seat.node-build-node.accept.delivery"; export declare const checkSubject: "mesh.seat.node-build-node.accept.check"; @@ -52,6 +66,12 @@ export declare const states: { readonly detectedSteps: "detected-steps"; }; +/** The key of one value of the state steps under the form "..". */ +export declare function stepsByAskerNodeIDKey(asker: string, node: string, id: string): string; + +/** The parts of a key of the form "..", or undefined for a key of another form. */ +export declare function parseStepsByAskerNodeIDKey(_key: string): { asker: string; node: string; id: string } | undefined; + /** The key of one value of the state steps under the form ".". */ export declare function stepsByNodeIDKey(node: string, id: string): string; @@ -91,6 +111,7 @@ export declare function parseDetectedStepsByModuleArtifactKey(_key: string): { m export type AskArtifact = buildStep.AskArtifact; export type BuildAnswer = buildStep.BuildAnswer; export type BuiltArtifact = buildStep.BuiltArtifact; +export type BuiltArtifactResolvedItem = buildStep.BuiltArtifactResolvedItem; export type Capacity = buildStep.Capacity; export type CheckAnswer = buildStep.CheckAnswer; export type ComposeAnswer = buildStep.ComposeAnswer; @@ -104,6 +125,7 @@ export type DropInput = buildStep.DropInput; export type Fetch = buildStep.Fetch; export type FetchAnswer = buildStep.FetchAnswer; export type FetchBesideItem = buildStep.FetchBesideItem; +export type FetchBuildsItem = buildStep.FetchBuildsItem; export type FetchPackage = buildStep.FetchPackage; export type FetchRefusal = buildStep.FetchRefusal; export type KeptArtifactAnswer = buildStep.KeptArtifactAnswer; @@ -136,6 +158,8 @@ export type StepAskCompose = buildStep.StepAskCompose; export type StepEnded = buildStep.StepEnded; export type StepEndedEvent = buildStep.StepEndedEvent; export type StepNames = buildStep.StepNames; +export type StepProgress = buildStep.StepProgress; +export type StepProgressEvent = buildStep.StepProgressEvent; export type StepResult = buildStep.StepResult; export type StepRun = buildStep.StepRun; export type StepsAnswer = buildStep.StepsAnswer; @@ -143,6 +167,8 @@ export type StepsInput = buildStep.StepsInput; export type TakenVersion = buildStep.TakenVersion; export type TestCounts = buildStep.TestCounts; export type Toolchain = buildStep.Toolchain; +export type ToolchainReported = buildStep.ToolchainReported; +export type ToolchainReportedEvent = buildStep.ToolchainReportedEvent; export type ToolchainUsed = buildStep.ToolchainUsed; /** is the body of delivery. */ @@ -160,6 +186,9 @@ export type DeliveryStepAccept = StepAsk; /** is the body of recheck. */ export type RecheckAccept = StepAsk; +/** is one value of the state steps under the key ... */ +export type StepsByAskerNodeIDValue = Step; + /** is one value of the state steps under the key .. */ export type StepsByNodeIDValue = Step; diff --git a/contracts/ts/seats/nodebuildnode/index.js b/contracts/ts/seats/nodebuildnode/index.js index 3c9047d8..6cda8ea2 100644 --- a/contracts/ts/seats/nodebuildnode/index.js +++ b/contracts/ts/seats/nodebuildnode/index.js @@ -23,6 +23,8 @@ export const accepts = { export const events = { stepEnded: "step-ended", + toolchainReported: "toolchain-reported", + stepProgress: "step-progress", }; export const family = { name: "node-build-node.events", maxAgeDays: 7 }; @@ -35,6 +37,22 @@ export function stepEndedSubject(node) { export const stepEndedSubjectAll = "mesh.seat.node-build-node.event.step-ended.*"; +export const toolchainReportedStream = "SEAT_node-build-node_toolchain-reported"; + +export function toolchainReportedSubject(node) { + return "mesh.seat.node-build-node.event.toolchain-reported." + node; +} + +export const toolchainReportedSubjectAll = "mesh.seat.node-build-node.event.toolchain-reported.*"; + +export const stepProgressStream = "SEAT_node-build-node_step-progress"; + +export function stepProgressSubject(node) { + return "mesh.seat.node-build-node.event.step-progress." + node; +} + +export const stepProgressSubjectAll = "mesh.seat.node-build-node.event.step-progress.*"; + export const deliverySubject = "mesh.seat.node-build-node.accept.delivery"; export const checkSubject = "mesh.seat.node-build-node.accept.check"; @@ -52,6 +70,29 @@ export const states = { detectedSteps: "detected-steps", }; +/** The key of one value of the state steps under the form "..". */ +export function stepsByAskerNodeIDKey(asker, node, id) { + return asker + "." + node + "." + id; +} + +/** The parts of a key of the form "..", or undefined for a key of another form. */ +export function parseStepsByAskerNodeIDKey(_key) { + let _rest = _key; + const _at0 = _rest.indexOf("."); + if (_at0 <= 0) return undefined; + const asker = _rest.slice(0, _at0); + _rest = _rest.slice(_at0 + 1); + const _at2 = _rest.indexOf("."); + if (_at2 <= 0) return undefined; + const node = _rest.slice(0, _at2); + _rest = _rest.slice(_at2 + 1); + if (_rest === "") return undefined; + const id = _rest; + _rest = ""; + if (_rest !== "") return undefined; + return { asker, node, id }; +} + /** The key of one value of the state steps under the form ".". */ export function stepsByNodeIDKey(node, id) { return node + "." + id; diff --git a/contracts/ts/seats/nodebuildnode/parts/build-step.d.ts b/contracts/ts/seats/nodebuildnode/parts/build-step.d.ts index 3a3c18ba..a6cc4551 100644 --- a/contracts/ts/seats/nodebuildnode/parts/build-step.d.ts +++ b/contracts/ts/seats/nodebuildnode/parts/build-step.d.ts @@ -13,6 +13,8 @@ export declare const verbs: { export declare const events: { readonly stepEnded: "step-ended"; + readonly toolchainReported: "toolchain-reported"; + readonly stepProgress: "step-progress"; }; /** one artifact of a build ask: built from the fetched tree, or copied from an upstream pin */ @@ -32,11 +34,20 @@ export interface BuildAnswer { "toolchain"?: ToolchainUsed; } +export interface BuiltArtifactResolvedItem { + /** sha256: of the package's bytes as the build fetched them */ + "digest": string; + "package": string; + "version": string; +} + /** one artifact a build made: its digest and the reference it was published under */ export interface BuiltArtifact { "artifact": string; "digest": string; "reference"?: string; + /** one per mesh package the build resolved (the catalog graph entry's packages, the edges of kind 1) */ + "resolved"?: BuiltArtifactResolvedItem[]; } /** how many steps the node runs at once */ @@ -100,6 +111,15 @@ export interface FetchBesideItem { "repository": string; } +export interface FetchBuildsItem { + "artifact": string; + /** the member's build digest, sha256: */ + "digest": string; + "module": string; + /** the name in its ecosystem (the Go module path, the npm name) */ + "package": string; +} + /** the TypeScript module's group package: the group's tagged version of the SDK ({name, version}), placed from the registry in each member whatever its lock names, or the group's package member ({repository, commit, ref}), packed by the fetch and served to the members in place of the registry's */ export interface FetchPackage { "commit"?: string; @@ -115,6 +135,8 @@ export interface Fetch { "base"?: string; /** the sibling repositories placed beside the fetched tree */ "beside"?: FetchBesideItem[]; + /** a group run's package members, each by its build's digest: the build module reads each from the artifact store at artifact-store:////blobs/, checks the bytes against the digest and lays them as the group package; a package named here is named in neither packages nor package */ + "builds"?: FetchBuildsItem[]; /** the full commit */ "commit": string; /** the module's directory in the repository ("." for its root), with paths */ @@ -416,6 +438,14 @@ export interface StepNames { "repositories": string[]; } +/** a running ask's progress, said once a minute while the holder runs it (design 47 G section 496) */ +export interface StepProgress { + "ask": string; + "asker": string; + "at": string; + "phase": "fetch" | "build" | "run" | "report"; +} + /** what a step answered, by the kind of ask: a build's artifacts, toolchain, imports, out-of-step modules and the artifact it kept; a check's fetch and levels; a compose's built, key and verdict; a run's built, key, exit, output and said; a declined ask's declined, needs and lacks. The fields of the kinds do not meet, so one layout holds the five */ export interface StepResult { "artifacts"?: BuiltArtifact[]; @@ -502,8 +532,20 @@ export interface Toolchain { "version": string; } +/** a node's toolchain of a language, said at the build module's start and each time its toolchains key of the language changes (design 47 G section 492) */ +export interface ToolchainReported { + /** the digest of the program of the provision the build module requires for that language's toolchain, never the build module's own */ + "digest": string; + "language": string; + "node": string; + "recipe": string; + "version": string; +} + /** the toolchain a build was made with: the language, the installed version and the build module's recipe */ export interface ToolchainUsed { + /** the building node's digest of its toolchain program (kind 3's entry), never the build module's own program */ + "digest"?: string; "language": string; "recipe": string; "version": string; @@ -579,22 +621,36 @@ export interface KeptArtifactAnswer { export interface StepsInput { /** find them anew, not from the kept answer */ "again"?: boolean; - /** the artifact reference, : */ + /** the key of detected-steps, . */ "artifact": string; - "fetch": Fetch; + /** delete the artifact's kept key instead of detecting; needs why */ + "drop"?: boolean; + "fetch"?: Fetch; "fingerprint"?: string; "module": string; + /** with drop, why: a non-empty text, written in the hand-act log with the caller the bus proved */ + "why"?: string; } /** is what steps answers. */ export interface StepsAnswer { - "artifact": string; + "artifact"?: string; + /** the artifact whose key a drop deleted */ + "dropped"?: string; + /** why a drop deleted nothing, naming the hand-act log when it refused */ + "error"?: string; "fingerprint"?: string; /** answered from the kept answer */ "kept"?: boolean; - "steps": DetectedStep[]; + "steps"?: DetectedStep[]; } /** is the body of the event step-ended. */ export type StepEndedEvent = StepEnded; +/** is the body of the event toolchain-reported. */ +export type ToolchainReportedEvent = ToolchainReported; + +/** is the body of the event step-progress. */ +export type StepProgressEvent = StepProgress; + diff --git a/contracts/ts/seats/nodebuildnode/parts/build-step.js b/contracts/ts/seats/nodebuildnode/parts/build-step.js index 027da325..155a68be 100644 --- a/contracts/ts/seats/nodebuildnode/parts/build-step.js +++ b/contracts/ts/seats/nodebuildnode/parts/build-step.js @@ -13,6 +13,8 @@ export const verbs = { export const events = { stepEnded: "step-ended", + toolchainReported: "toolchain-reported", + stepProgress: "step-progress", }; export {}; diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/halves.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/halves.go new file mode 100644 index 00000000..07829862 --- /dev/null +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/halves.go @@ -0,0 +1,147 @@ +package main + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "slices" + "sync" + "time" + + "github.com/nats-io/nats.go/jetstream" + + mcc "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/sdk/graph" + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// halves follows the catalog half of each watched branch through the seat mesh-catalog (design 47 G §14, §184, §195): one +// meshcatalogclient.Catalog per branch, a first view of the verb `revision` and then each `catalog-revision` of that branch, +// each entry taken into the world as the value and the graph node the rules read. Of the view the assigner reads only the catalog +// half, never its own output back (§195). +type halves struct { + w *world + call mcc.Call + wake func() + logf func(string, ...any) + ctx context.Context + + mu sync.Mutex + cats map[string]*mcc.Catalog + // started is called with each catalog made, for the part that reads their status. + started func(*mcc.Catalog) + // settled is told once when a branch's first view is answered, for the first pass to wait on (the trunk's alone). + settled chan<- struct{} +} + +// follow makes the catalog of branch and asks its first view, once; true when it was made now. +func (h *halves) follow(branch string) bool { + h.mu.Lock() + if h.cats[branch] != nil { + h.mu.Unlock() + return false + } + c, err := mcc.NewCatalog(h.call, branch) + if err != nil { + h.mu.Unlock() + h.logf("the catalog half of branch %q is not followed: %v", branch, err) + return false + } + c.Follower.Logf = h.logf + c.Follower.Changed = func(module string, entry json.RawMessage, deleted bool, _ time.Time) { + h.changed(branch, module, entry, deleted) + } + if h.cats == nil { + h.cats = map[string]*mcc.Catalog{} + } + h.cats[branch] = c + h.mu.Unlock() + if h.started != nil { + h.started(c) + } + go func() { + if err := c.Start(h.ctx); err != nil { + if h.ctx.Err() == nil { + h.logf("the first view of the catalog half of branch %q is not answered: %v", branch, err) + } + return + } + if branch == trunkBranch && h.settled != nil { + h.settled <- struct{}{} + } + }() + return true +} + +// hear gives a catalog-revision to every catalog: each takes its own branch's and passes over another's. +func (h *halves) hear(m jetstream.Msg) { + h.mu.Lock() + cats := make([]*mcc.Catalog, 0, len(h.cats)) + for _, c := range h.cats { + cats = append(cats, c) + } + h.mu.Unlock() + for _, c := range cats { + if err := c.Hear(h.ctx, m.Data()); err != nil { + h.logf("%s on branch %q: %v", m.Subject(), c.Branch, err) + } + } + _ = m.Ack() +} + +// followers are the first views the catalogs hold. +func (h *halves) followers() []*mcc.Catalog { + h.mu.Lock() + defer h.mu.Unlock() + out := make([]*mcc.Catalog, 0, len(h.cats)) + for _, c := range h.cats { + out = append(out, c) + } + return out +} + +// changed takes one entry of branch's half into the world, or a module taken out of it. +func (h *halves) changed(branch, module string, raw json.RawMessage, deleted bool) { + defer h.wake() + if deleted { + h.w.dropModule(branch, module) + return + } + var e meshcatalog.CatalogEntry + if err := json.Unmarshal(raw, &e); err != nil { + h.logf("the entry of %s on branch %q does not read: %v", module, branch, err) + return + } + v, n, m := fromEntry(e) + h.w.putModule(branch, module, v, n, m) +} + +// fromEntry is a catalog entry as the value and the graph node the rules read, and its manifest parsed once under the digest of +// its words. +func fromEntry(e meshcatalog.CatalogEntry) (meshcatalog.ModulesByModuleValue, graph.Node, *parsed) { + v := meshcatalog.ModulesByModuleValue{Commit: e.Commit, Fp2: e.Fp, Needs: e.Needs, None2: e.None, Packages: e.Packages, + Path: e.Path, Repository: e.Repository, Verdict: e.Verdict} + n := graph.Node{FP: e.Fp, None: e.None, Repository: e.Repository, Path: e.Path, Bases: e.Bases, Provides: e.Provides, Requires: e.Requires, + Awaits: e.Awaits, Invokes: e.Invokes, Serves: e.Serves, LeftOut: e.LeftOut} + for p := range e.Packages { + n.Packages = append(n.Packages, p) + } + slices.Sort(n.Packages) + if len(e.Manifest) == 0 { + return v, n, nil + } + raw, err := json.Marshal(e.Manifest) + if err != nil { + return v, n, nil + } + sum := sha256.Sum256(raw) + v.Manifest = "sha256:" + hex.EncodeToString(sum[:]) + man, err := catalogue.Decode(raw) + if err != nil { + return v, n, &parsed{module: e.Module, err: "it does not decode: " + err.Error()} + } + n.Serves = graph.ServesOf(man) + return v, n, &parsed{module: e.Module, manifest: man} +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go index 0ce2b023..e10e9312 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go @@ -7,6 +7,7 @@ import ( "encoding/json" "errors" "fmt" + "git.novox.be/novox/mesh-sdk/go/keys" "io" "maps" "mesh-delivery-assigner/internal/assignment" @@ -52,7 +53,6 @@ import ( "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient/enginereads" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" - "git.novox.be/novox/mesh-delivery/sdk/graph" "git.novox.be/novox/mesh-sdk/go/views" "mesh-delivery-assigner/internal/assign" @@ -71,6 +71,17 @@ type bed struct { placed map[string][]string // journal is what the programs the bed starts write in their journal. journal *journal + // cat is the catalog half the seat mesh-catalog serves the assigner (serveCatalog). + cat *catalogBed +} + +// catalogBed is the catalog half of each watched branch as the seat mesh-catalog keeps it: entries by branch and module, a +// revision per branch, and the manifest bodies by digest. +type catalogBed struct { + mu sync.Mutex + entries map[string]map[string]meshcatalog.CatalogEntry + rev map[string]int64 + bodies map[string]string } // journal is a program's journal as the bed hears it: a test awaits a line in it (await) where the line is the only sign @@ -144,7 +155,8 @@ func startBed(t *testing.T, leaveOut string) *bed { } ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) t.Cleanup(cancel) - b := &bed{srv: srv, js: js, ctx: ctx, dir: dir, buckets: map[string]jetstream.KeyValue{}, journal: &journal{grew: make(chan struct{})}} + b := &bed{srv: srv, js: js, ctx: ctx, dir: dir, buckets: map[string]jetstream.KeyValue{}, journal: &journal{grew: make(chan struct{})}, + cat: &catalogBed{entries: map[string]map[string]meshcatalog.CatalogEntry{}, rev: map[string]int64{}, bodies: map[string]string{}}} for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) { if name == leaveOut { continue @@ -171,7 +183,7 @@ func startBed(t *testing.T, leaveOut string) *bed { // event names the engine's machine last, so the filter is the event with every machine (mesh-grants buscompose // machineEvent; the grants part's own test holds the derivation). if _, err := js.CreateOrUpdateConsumer(ctx, "EVENTS", jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-assigner", - FilterSubjects: []string{enginereads.SubjectPrefix + "*", "mesh.mod." + buckets.IndexerModule + ".event.views.>"}, AckPolicy: jetstream.AckExplicitPolicy}); err != nil { + FilterSubjects: []string{enginereads.SubjectPrefix + "*", "mesh.mod." + buckets.IndexerModule + ".event.views.>", meshcatalog.SubjectCatalogRevisionAll}, AckPolicy: jetstream.AckExplicitPolicy}); err != nil { t.Fatal(err) } // The operator-channel seat's stream of conditions, which the assigner says its conditions on. @@ -180,6 +192,8 @@ func startBed(t *testing.T, leaveOut string) *bed { } // The indexer as the owner of its manifests and modules (DLV-F2r), with the real owner's side. serveIndexer(t, b, nc) + // The seat mesh-catalog's catalog half (design 47 G §194), which the assigner follows per branch. + serveCatalog(t, b, nc) // The assigner's views family is its ordinary events on EVENTS (ADR 0635 §5); its marks are its state. if kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: marksBucket, History: 1}); err != nil { t.Fatal(err) @@ -422,6 +436,7 @@ func (b *bed) manifest(t *testing.T, module, body string) string { t.Fatal(err) } b.putJSON(t, buckets.Manifests, key, value) + b.cat.bodies[digest] = body return digest } @@ -447,11 +462,11 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) { // DLV-F5r (ADR 0621 §3): the indexer's runtime starts together with the assigner and answers its first views only // after the assigner asked; within the bound the assigner writes the wants from what they answer. up := make(chan struct{}) - indexerLate = up + catalogLate = up // ADR 0631 §2: while the indexer does not answer, past the first-view bound the assigner raises the condition naming // the indexer and graph, and reads nothing of the indexer's state; the answer clears it. firstViewBound, firstViewWait = 300*time.Millisecond, 50*time.Millisecond - t.Cleanup(func() { indexerLate, firstViewBound, firstViewWait = nil, 0, 0 }) + t.Cleanup(func() { catalogLate, firstViewBound, firstViewWait = nil, 0, 0 }) for _, name := range append(append([]string{}, readBuckets...), watched...) { if strings.HasPrefix(name, buckets.IndexerModule+"_") { t.Errorf("the assigner reads the indexer's state %s itself", name) @@ -461,7 +476,7 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) { fNew, fOld := meshcatalogclient.SourcePrefix2+strings.Repeat("b", 64), meshcatalogclient.SourcePrefix2+strings.Repeat("a", 64) digest := b.manifest(t, "web", `{"module":"web","version":"1"}`) mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fNew, Manifest: digest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath}) - b.putJSON(t, buckets.Modules, "web", mod) + b.putModule(t, "web", mod) built, err := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: digest}) if err != nil { t.Fatal(err) @@ -493,20 +508,20 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) { "seat-holders": {"n1.node-tools"}, "wants": {"n1.mesh-grants"}, }) - graphKey := meshconditionsclient.Key(meshconditionsclient.ScopeModule, assignment.Module+"."+buckets.IndexerModule+"."+graph.State, views.UnansweredKind) + graphKey := meshconditionsclient.Key(meshconditionsclient.ScopeModule, assignment.Module+"."+meshcatalog.Seat+"."+views.BranchState(meshcatalog.StateCatalogRevisions, trunkBranch), views.UnansweredKind) graphRaised := func() bool { e, err := b.buckets[assign.FindingsBucket].Get(b.ctx, graphKey) - return err == nil && strings.Contains(string(e.Value()), buckets.IndexerModule+".graph") + return err == nil && strings.Contains(string(e.Value()), "mesh-catalog.revision") } for deadline := time.Now().Add(10 * time.Second); !graphRaised(); time.Sleep(50 * time.Millisecond) { if time.Now().After(deadline) { - t.Fatalf("with the indexer down, no condition names its graph (finding %s)", graphKey) + t.Fatalf("with the seat mesh-catalog down, no condition names its revision verb (finding %s)", graphKey) } } close(up) for deadline := time.Now().Add(10 * time.Second); graphRaised(); time.Sleep(50 * time.Millisecond) { if time.Now().After(deadline) { - t.Fatalf("the indexer answered, and the condition naming its graph stands (finding %s)", graphKey) + t.Fatalf("the seat answered, and the condition naming its revision verb stands (finding %s)", graphKey) } } @@ -608,7 +623,7 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) { fp, digest string }{"db": {fpOf('2'), dbDigest}, "app": {fpOf('3'), appDigest}, "tool": {fpOf('4'), toolDigest}} { mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: row.fp, Manifest: row.digest, Repository: "novox/" + module, Commit: strings.Repeat("2", 40), Path: meshcatalogclient.RootPath}) - b.putJSON(t, buckets.Modules, module, mod) + b.putModule(t, module, mod) fps := []string{row.fp} if module == "tool" { fps = append(fps, toolOld) @@ -680,7 +695,7 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) { formsDigest := b.manifest(t, "forms", `{"module":"forms","version":"1"}`) formsAt := func(fp string) { mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: formsDigest, Repository: "novox/forms", Commit: strings.Repeat("3", 40), Path: meshcatalogclient.RootPath}) - b.putJSON(t, buckets.Modules, "forms", mod) + b.putModule(t, "forms", mod) } for _, fp := range []string{formsOld, formsNew} { build := builds.BuilderBuild{State: builds.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: formsDigest} @@ -802,7 +817,7 @@ func TestAHeldFingerprintIsAskedAbout(t *testing.T) { fNew := meshcatalogclient.SourcePrefix2 + strings.Repeat("c", 64) digest := b.manifest(t, "web", `{"module":"web","version":"1"}`) mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fNew, Manifest: digest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath}) - b.putJSON(t, buckets.Modules, "web", mod) + b.putModule(t, "web", mod) on, _ := presenceValue(true) b.putJSON(t, buckets.Presence, "ace", on) failed, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed, Detail: "connection refused"}) @@ -860,7 +875,7 @@ func TestAFailedDataModuleIsAskedToRestore(t *testing.T) { fNew := meshcatalogclient.SourcePrefix2 + strings.Repeat("d", 64) digest := b.manifest(t, "mail", `{"module":"mail","version":"1","resources":[{"id":"data","type":"directory","path":"/var/lib/mail"}],"data":{"own":[{"id":"mail","path":"${dir:data}","class":"irreplaceable","backup":"copy"}]}}`) mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fNew, Manifest: digest, Repository: "novox/mail", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath}) - b.putJSON(t, buckets.Modules, "mail", mod) + b.putModule(t, "mail", mod) on, _ := presenceValue(true) b.putJSON(t, buckets.Presence, "ace", on) failed, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed, Detail: "connection refused"}) @@ -910,7 +925,7 @@ func TestTheAssignerReplacedResumesWithNothingLostOrRepeated(t *testing.T) { fp := meshcatalogclient.SourcePrefix2 + strings.Repeat(string(rune('a'+i%6)), 63) + fmt.Sprintf("%x", i%16) digest := b.manifest(t, name, fmt.Sprintf(`{"module":%q,"version":"1"}`, name)) mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: digest, Repository: "novox/" + name, Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath}) - b.putJSON(t, buckets.Modules, name, mod) + b.putModule(t, name, mod) built, err := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: digest}) if err != nil { t.Fatal(err) @@ -956,7 +971,7 @@ func (b *bed) fixture(t *testing.T, name, fp, manifest string) { t.Helper() digest := b.manifest(t, name, manifest) mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: digest, Repository: "novox/" + name, Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath}) - b.putJSON(t, buckets.Modules, name, mod) + b.putModule(t, name, mod) built, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: digest}) bkey, _ := builds.BuilderBuildKey(name, fp) b.putJSON(t, buckets.Builds, bkey, built) @@ -1305,9 +1320,9 @@ func TestAProposalIsAnsweredApprovedAndWritten(t *testing.T) { rec, _ := nodeengineclient.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Record: &nodeengine.NodeRecord{Node: node}}) b.putJSON(t, buckets.Have, node+".node", rec) } - b.putJSON(t, buckets.Modules, "web", []byte(`{"v":1}`)) - b.putJSON(t, buckets.Modules, "shell", []byte(`{"v":1}`)) - b.putJSON(t, buckets.Modules, "mesh-enrolment", []byte(`{"v":1}`)) + b.putModule(t, "web", []byte(`{"v":1}`)) + b.putModule(t, "shell", []byte(`{"v":1}`)) + b.putModule(t, "mesh-enrolment", []byte(`{"v":1}`)) // place writes a node's assignments as another writer would, in its node and module keys (research 072 row 37), and // answers its node key's revision. place := func(node string, modules ...string) uint64 { @@ -1611,9 +1626,9 @@ func assignmentsSubjects() []string { return out } -// indexerLate, when set before the bed is made, fails every ask of the indexer's first views until it is closed: the +// catalogLate, when set before the bed is made, fails every ask of the indexer's first views until it is closed: the // indexer down, its runtime coming up after the assigner asked (DLV-F5r, ADR 0631 §2). -var indexerLate chan struct{} +var catalogLate chan struct{} // serveIndexer is the indexer as the owner of its manifests, modules and module graph (ADR 0615, ADR 0621, ADR 0631): its // buckets, which a test fills as the indexer writes them; its views family said by the views emitter as ordinary events on EVENTS, @@ -1635,7 +1650,7 @@ func serveIndexer(t *testing.T, b *bed, nc *nats.Conn) { if err := em.Start(b.ctx, state, kv, nil); err != nil { t.Fatal(err) } - late := indexerLate + late := catalogLate sub, err := nc.Subscribe("mesh.mod.mesh-delivery-indexer.call."+state+".>", func(m *nats.Msg) { if late != nil { // its runtime comes up after the assigner started (DLV-F5r): until then every ask fails, as on a down indexer select { @@ -1797,3 +1812,82 @@ func changeIn(body map[string]any) plc.Change { _ = json.Unmarshal(raw, &c) return c } + +// indexerFirstViews are the indexer's states the bed still serves (the verbs' check of a proposal asks the indexer's modules and +// manifests): the bucket each is kept in. +var indexerFirstViews = map[string]string{"manifests": buckets.Manifests, "modules": buckets.Modules} + +// putModule keeps a module's value as the indexer does for the verbs, and as an entry of the trunk's catalog half; a value that +// names no manifest the bed holds is kept for the verbs alone. +func (b *bed) putModule(t *testing.T, module string, value []byte) { + t.Helper() + b.putJSON(t, buckets.Modules, module, value) + v, err := meshcatalogclient.DecodeModule(value) + if err != nil { // a value the bed writes only for the module to exist + v = meshcatalog.ModulesByModuleValue{Repository: "novox/" + module} + } + b.publish(t, trunkBranch, module, v) +} + +// publish makes the trunk's catalog half of branch name module at v, a new revision of it, and says catalog-revision. +func (b *bed) publish(t *testing.T, branch, module string, v meshcatalog.ModulesByModuleValue) { + t.Helper() + e := meshcatalog.CatalogEntry{Module: module, Fp: v.Fp2, Commit: v.Commit, Repository: v.Repository, Path: v.Path, Needs: v.Needs, Verdict: "accepted"} + if body, ok := b.cat.bodies[v.Manifest]; ok { + _ = json.Unmarshal([]byte(body), &e.Manifest) + } + b.cat.mu.Lock() + if b.cat.entries[branch] == nil { + b.cat.entries[branch] = map[string]meshcatalog.CatalogEntry{} + } + b.cat.entries[branch][module] = e + b.cat.rev[branch]++ + rev, n := b.cat.rev[branch], int64(len(b.cat.entries[branch])) + b.cat.mu.Unlock() + raw, _ := json.Marshal(meshcatalog.CatalogRevisionEvent{Kind: "trunk", Branch: keys.EscapeName(branch), Revision: rev, Entries: n, Changed: []string{module}}) + if _, err := b.js.Publish(b.ctx, meshcatalog.SubjectCatalogRevision(branch), raw); err != nil { + t.Fatal(err) + } +} + +// serveCatalog answers the seat mesh-catalog's verb revision from the bed's catalog half, all in one page; until catalogLate is +// closed every ask fails, as on a seat whose holder is down. +func serveCatalog(t *testing.T, b *bed, nc *nats.Conn) { + t.Helper() + late := catalogLate + answer := func(m *nats.Msg) { + if late != nil { + select { + case <-late: + default: + _ = m.Respond([]byte(`{"error":"the seat mesh-catalog's holder is not up"}`)) + return + } + } + var in meshcatalog.RevisionInput + _ = json.Unmarshal(m.Data, &in) + branch, _ := keys.UnescapeName(in.Branch) + b.cat.mu.Lock() + defer b.cat.mu.Unlock() + if b.cat.rev[branch] == 0 { + _ = m.Respond([]byte(`{"error":"` + views.NoneYet + `"}`)) + return + } + rev := meshcatalog.CatalogRevision{Kind: "trunk", Branch: in.Branch, Revision: b.cat.rev[branch]} + var entries []meshcatalog.CatalogEntry + for _, name := range slices.Sorted(maps.Keys(b.cat.entries[branch])) { + entries = append(entries, b.cat.entries[branch][name]) + rev.Entries = append(rev.Entries, meshcatalog.EntryRef{Module: name, Digest: "sha256:" + strings.Repeat("0", 64)}) + } + raw, _ := json.Marshal(map[string]any{"result": meshcatalog.RevisionAnswer{Revision: rev, Entries: entries}, "node": "n0"}) + _ = m.Respond(raw) + } + for _, subject := range []string{"mesh.seat.mesh-catalog.tool.revision", "mesh.seat.mesh-catalog.call.revision.*", "mesh.seat.mesh-catalog.tool.revision.*"} { + sub, err := nc.Subscribe(subject, answer) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = sub.Unsubscribe() }) + } + subscribed(t, nc) +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index 72177123..e99492ea 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -248,7 +248,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro conn.Close() } }() - iv := &indexerViews{js: js, account: account, call: said.Call, findings: findings, logf: logger.Printf, + iv := &catalogViews{js: js, account: account, call: said.Call, findings: findings, logf: logger.Printf, status: process.AwaitBucket(ctx, js, conn, firstViewsBucket, "assigner_first_views reads nothing and assigner_first_view_again is not heard", logger.Printf)} return follow(ctx, js, conn, kvs, findings.Say(sayVia(ctx, js)), findings, emitting{said}, iv, logger) } @@ -306,7 +306,7 @@ var watched = []string{ // follow watches every bucket a pass reads and makes a pass at start, once every watch handed what stands, and after // each change; changes that come during a pass make one more. It returns when ctx ends or a watch ends. -func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, kvs kvSet, say func(oc.Condition) error, findings assign.Findings, bus caller, iv *indexerViews, logger *log.Logger) error { +func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, kvs kvSet, say func(oc.Condition) error, findings assign.Findings, bus caller, iv *catalogViews, logger *log.Logger) error { w := newWorld() dirty := make(chan struct{}, 1) wake := func() { @@ -494,6 +494,13 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, // The runner's pipeline-ended and group-ended (design 47 G §29): kept in the state written, which the wants follow // once the first registrations are written (§95, §353). p.ends = newPipelineEnds(kvEnds{ctx, kv}, wake, logger.Printf) + if iv != nil && iv.h != nil { + p.ends.onBranches = func(branches []string) { + for _, b := range branches { + iv.h.follow(b) + } + } + } err := hearEnds(hub, p.ends) if err == nil && bus != nil { err = followRegistrations(ctx, hub, bus.Call, p.ends) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go index ef0ccc4f..33382b2d 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go @@ -8,6 +8,7 @@ import ( "maps" "mesh-delivery-assigner/internal/buckets" "slices" + "sort" "strings" "time" @@ -23,6 +24,7 @@ import ( oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" "git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient" + "git.novox.be/novox/mesh-delivery/sdk/graph" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient/enginereads" @@ -219,19 +221,21 @@ func (p *passer) nodeWants(ctx context.Context) { // the hold of a fingerprint whose health check failed. The assigner compares its wants with nothing the controller // sends: the differences bucket is kept read-only with its history (ADR 0539 §8). func (p *passer) wants(ctx context.Context) { - get := p.w.getter(buckets.Manifests) // the indexer's manifests, as its first view and views family say them assigned := p.w.assigned() + assignedBranches := p.w.placedBranches() + // Every module of every branch's catalog half, for the parts that ask which exist; a node's entries come from the half of the + // branch its placement names (design 47 G §14, §184). trunk := p.w.trunk() - manifests := map[string]catalogue.Manifest{} - for name, mod := range trunk { - if mod.Manifest == "" { - continue - } - if got := p.w.manifest(ctx, get, mod.Manifest); got.err == "" { - manifests[name] = got.manifest + views := map[string]*branchView{} + viewOf := func(branch string) *branchView { + if v := views[branch]; v != nil { + return v } + v := p.branchView(branch, assigned) + views[branch] = v + return v } - nodes := p.w.nodes(manifests, assigned) + nodes := viewOf(trunkBranch).nodes p.engineReads(nodes) nodesOf := map[string][]string{} for node, modules := range assigned { @@ -268,111 +272,118 @@ func (p *passer) wants(ctx context.Context) { defer func() { p.sayNeither(neither, stale, left) }() // Where grants are read from, once for the pass: the grants part's (issue 774, ADR 0571 clause 9). grants := p.w.grantsBucket() - // The verb edges, once for the pass (ADR 0614 §2, §3). - moduleGraph := p.moduleGraph() - served := p.served(ctx, get, moduleGraph, nodes) - offering := p.offering(ctx, get, manifests, nodes) // bus state (ADR 0614 §5) - assignedBranches := p.w.placedBranches() for name := range names { - fp := trunk[name].Fp2 - name := name - // A want follows a passed pipeline: a module whose fingerprint no pipeline passed at on every branch its placements - // chose, and whose wants do not name it already, gets none (design 47 G §29, §35). - branches := p.ends.branchesOf(name, assignedBranches) - if fp != "" && !p.ends.mayWant(name, fp, branches, wanted[name]) { - continue - } - m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Needs: trunk[name].Needs, Chosen: chosenOf(chosen, name), Provisions: provisionsOf(name), Answering: answering(manifests, nodes), Offered: offered(manifests), Nodes: nodesOf[name], Now: time.Now(), Puts: puts[name], FormNeeds: p.w.buildNeeds(name, fp), - Self: func(node string) bool { return placed[node] }, - Composed: func(node, fp string) (assign.ComposedDigest, bool) { - return p.w.composed(node, name, fp) // a placed node's own: the digest its own have names (ADR 0571 §6) - }, - GrantedAt: func(fp string) bool { - key, err := grantsstate.PartGrantKey(name, fp) - return err == nil && p.w.has(grants, key) - }, - Restored: func(node string) bool { - key, err := asks.RestoreAskKey(node, name, fp) - return err == nil && p.answeredWith(key, assign.OptionRestore) - }, - Allowed: func(node, digest string) bool { - key, err := asks.RecordAskKey(node, name, digest) - return err == nil && p.answeredWith(key, assign.OptionYes) - }, - Release: func(fp string) (string, bool) { return p.w.release(name, fp) }} - if fp != "" { - held, _ := mwc.HeldKey(name, fp) - grant, _ := grantsstate.PartGrantKey(name, fp) - hold, _ := asks.HoldAskKey(name, fp) - m.Released = p.answeredWith(hold, "release") - m.Kept = p.answeredWith(hold, "keep") - m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(buckets.Wants, held) && !m.Released, p.w.has(grants, grant) - if raw := p.w.value(grants, grant); raw != nil { - if g, err := grantsstate.DecodeGrant(raw); err == nil { - m.GrantedOn = func(node string) bool { return assign.GrantNamesNode(g, node, name) } + groups := p.groups(name, nodesOf[name], assignedBranches[name]) + for gi, g := range groups { + func() { + bv := viewOf(g.branch) + trunk, manifests, nodes, moduleGraph, served, offering := bv.trunk, bv.manifests, bv.nodes, bv.graph, bv.served, bv.offering + fp := trunk[name].Fp2 + name := name + nodesOf := map[string][]string{name: g.nodes} + wantedOf := map[string]map[string]mwc.Want{name: wantedHere(wanted[name], g.nodes, groups, gi == 0)} + // A want follows a passed pipeline: a module whose fingerprint no pipeline passed at on the branch its placements + // chose, and whose wants do not name it already, gets none (design 47 G §29, §35). + var branches []string + if p.ends.following() { + branches = []string{g.branch} } - } - if m.Released { - p.clearHeld(name, fp) - } - } - if served != nil { - m.Invokes, m.Served = keptInvokes(moduleGraph, name), served - } - m.Offering = offering - last := map[string]string{} - for _, node := range m.Nodes { - last[node] = p.w.lastHealthy(node, name, fp) - } - out := assign.Wants(m, nodes, wanted[name], last) - for _, node := range m.Nodes { - for _, prov := range m.MissingProvisions(node) { - missing[node+"."+name+"."+prov] = true - } - } - p.yeses(ctx, name, m, wanted[name], &out) - gatherNeither(neither, name, out) - leftNeither(left, m, wanted[name], out) - p.sayNoWant(name, m.Nodes, wanted[name], out) - p.sayUnreached(name, m.Nodes, out) - p.sayNodeFailed(name, fp, m.Nodes, nodes, out, first) - if first { - startClears(stale, m, out) - if fp != "" && !m.Held && out.Held == nil && !out.Lifted { - p.clearHeld(name, fp) - } - } - if out.Held != nil { - key, err := mwc.HeldKey(name, fp) - if err == nil { - held := *out.Held - p.put(ctx, buckets.Wants, key, func() ([]byte, error) { return mwc.EncodeHeld(held) }) - } - } - if out.Lifted { - p.lift(ctx, name, fp) - } - if out.Held != nil || (m.Held && !out.Lifted) { - p.holdAsk(ctx, name, fp, out.Held) - p.heldCondition(ctx, name, fp, out.Held, len(out.Restore) > 0) - } - for _, node := range out.Restore { - p.restoreAsk(ctx, name, node, fp) - } - for _, need := range out.Record { - p.recordAsk(ctx, name, fp, need) - } - for node, want := range out.Wants { - key, err := mwc.WantKey(node, name) - if err != nil { - p.logf("no want of %s on %s is written: %v", name, node, err) - continue - } - want := want - p.put(ctx, buckets.Wants, key, func() ([]byte, error) { return mwc.EncodeWant(want) }) - } - if len(out.Wants) > 0 { - p.ends.wrote(name, fp, branches) + if fp != "" && !p.ends.mayWant(name, fp, branches, wantedOf[name]) { + return + } + m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Needs: trunk[name].Needs, Chosen: chosenOf(chosen, name), Provisions: provisionsOf(name), Answering: answering(manifests, nodes), Offered: offered(manifests), Nodes: nodesOf[name], Now: time.Now(), Puts: puts[name], FormNeeds: p.w.buildNeeds(name, fp), + Self: func(node string) bool { return placed[node] }, + Composed: func(node, fp string) (assign.ComposedDigest, bool) { + return p.w.composed(node, name, fp) // a placed node's own: the digest its own have names (ADR 0571 §6) + }, + GrantedAt: func(fp string) bool { + key, err := grantsstate.PartGrantKey(name, fp) + return err == nil && p.w.has(grants, key) + }, + Restored: func(node string) bool { + key, err := asks.RestoreAskKey(node, name, fp) + return err == nil && p.answeredWith(key, assign.OptionRestore) + }, + Allowed: func(node, digest string) bool { + key, err := asks.RecordAskKey(node, name, digest) + return err == nil && p.answeredWith(key, assign.OptionYes) + }, + Release: func(fp string) (string, bool) { return p.w.release(name, fp) }} + if fp != "" { + held, _ := mwc.HeldKey(name, fp) + grant, _ := grantsstate.PartGrantKey(name, fp) + hold, _ := asks.HoldAskKey(name, fp) + m.Released = p.answeredWith(hold, "release") + m.Kept = p.answeredWith(hold, "keep") + m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(buckets.Wants, held) && !m.Released, p.w.has(grants, grant) + if raw := p.w.value(grants, grant); raw != nil { + if g, err := grantsstate.DecodeGrant(raw); err == nil { + m.GrantedOn = func(node string) bool { return assign.GrantNamesNode(g, node, name) } + } + } + if m.Released { + p.clearHeld(name, fp) + } + } + if served != nil { + m.Invokes, m.Served = keptInvokes(moduleGraph, name), served + } + m.Offering = offering + last := map[string]string{} + for _, node := range m.Nodes { + last[node] = p.w.lastHealthy(node, name, fp) + } + out := assign.Wants(m, nodes, wantedOf[name], last) + for _, node := range m.Nodes { + for _, prov := range m.MissingProvisions(node) { + missing[node+"."+name+"."+prov] = true + } + } + p.yeses(ctx, name, m, wantedOf[name], &out) + gatherNeither(neither, name, out) + leftNeither(left, m, wantedOf[name], out) + p.sayNoWant(name, m.Nodes, wantedOf[name], out) + p.sayUnreached(name, m.Nodes, out) + p.sayNodeFailed(name, fp, m.Nodes, nodes, out, first) + if first { + startClears(stale, m, out) + if fp != "" && !m.Held && out.Held == nil && !out.Lifted { + p.clearHeld(name, fp) + } + } + if out.Held != nil { + key, err := mwc.HeldKey(name, fp) + if err == nil { + held := *out.Held + p.put(ctx, buckets.Wants, key, func() ([]byte, error) { return mwc.EncodeHeld(held) }) + } + } + if out.Lifted { + p.lift(ctx, name, fp) + } + if out.Held != nil || (m.Held && !out.Lifted) { + p.holdAsk(ctx, name, fp, out.Held) + p.heldCondition(ctx, name, fp, out.Held, len(out.Restore) > 0) + } + for _, node := range out.Restore { + p.restoreAsk(ctx, name, node, fp) + } + for _, need := range out.Record { + p.recordAsk(ctx, name, fp, need) + } + for node, want := range out.Wants { + key, err := mwc.WantKey(node, name) + if err != nil { + p.logf("no want of %s on %s is written: %v", name, node, err) + continue + } + want := want + p.put(ctx, buckets.Wants, key, func() ([]byte, error) { return mwc.EncodeWant(want) }) + } + if len(out.Wants) > 0 { + p.ends.wrote(name, fp, branches) + } + }() } } } @@ -1313,3 +1324,82 @@ func answering(manifests map[string]catalogue.Manifest, n assign.Nodes) func(str return out } } + +// branchView is what one pass reads of the catalog half of one watched branch (design 47 G §184, §195): its modules, their +// manifests, the nodes the rules read through them, the module graph and the two readings of what runs. +type branchView struct { + trunk map[string]meshcatalog.ModulesByModuleValue + manifests map[string]catalogue.Manifest + nodes assign.Nodes + graph graph.Graph + served func(string) bool + offering func(string) bool +} + +// branchView reads the catalog half of branch for this pass. +func (p *passer) branchView(branch string, assigned map[string][]string) *branchView { + v := &branchView{trunk: p.w.trunkOf(branch), manifests: map[string]catalogue.Manifest{}} + for name, mod := range v.trunk { + if mod.Manifest == "" { + continue + } + if got := p.w.manifest(mod.Manifest); got.err == "" { + v.manifests[name] = got.manifest + } + } + v.nodes = p.w.nodes(v.manifests, assigned) + v.graph = p.moduleGraph(branch) + v.served = p.served(v.graph, v.nodes) + v.offering = p.offering(v.manifests, v.nodes) // bus state (ADR 0614 §5) + return v +} + +// group is the nodes of one module whose placements name one watched branch. +type group struct { + branch string + nodes []string +} + +// groups splits the nodes of module by the branch their placements name, the branches in name order (the trunk's first); a +// module placed nowhere is one group on its default branch. +func (p *passer) groups(module string, nodes []string, placed map[string]string) []group { + by := map[string][]string{} + for _, node := range nodes { + b := trunkBranch + if p.ends.following() { + b = p.ends.branchOf(module, placed[node]) + } + by[b] = append(by[b], node) + } + if len(by) == 0 { + b := trunkBranch + if p.ends.following() { + b = p.ends.branchOf(module, "") + } + by[b] = nil + } + var out []group + for b, ns := range by { + out = append(out, group{b, ns}) + } + sort.Slice(out, func(i, j int) bool { return out[i].branch < out[j].branch }) + return out +} + +// wantedHere is the wants of a module a group answers for: those of its own nodes, and (the first group only) those of nodes no +// group holds, which the rules take away. +func wantedHere(wanted map[string]mwc.Want, own []string, groups []group, first bool) map[string]mwc.Want { + held := map[string]bool{} + for _, g := range groups { + for _, n := range g.nodes { + held[n] = true + } + } + out := map[string]mwc.Want{} + for node, w := range wanted { + if slices.Contains(own, node) || (first && !held[node]) { + out[node] = w + } + } + return out +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go index 716ac50c..93654d0d 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go @@ -558,7 +558,7 @@ func putOfTheBus(t *testing.T) { placeOn(t, p.w, "novox", "nats") mdigest := strings.Repeat("e", 64) mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: mdigest, Repository: "novox/mesh-catalog", Commit: strings.Repeat("c", 40)}) - world(buckets.Modules, "nats", mod) + putModuleRaw(p.w, trunkBranch, "nats", mod) p.w.manifests[mdigest] = parsed{module: "nats", manifest: catalogue.Manifest{Module: "nats", Provides: []catalogue.Offer{{Name: "mesh-bus"}}}} bkey, _ := builds.BuilderBuildKey("nats", fp) built, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: mdigest}) @@ -901,7 +901,7 @@ func TestTheFirstPassAfterAStartClearsWhatAPreviousProcessRaised(t *testing.T) { var said []oc.Condition p.say = func(c oc.Condition) error { said = append(said, c); return nil } mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fpTest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath}) - p.w.put(buckets.Modules, "web", mod) + putModuleRaw(p.w, trunkBranch, "web", mod) placeOn(t, p.w, "g14", "web") on, _ := presenceValue(true) p.w.put(buckets.Presence, "g14", on) @@ -939,7 +939,7 @@ func testAHeldModuleRaisesNoNeitherAndClearsAStandingOne(t *testing.T) { said := &[]oc.Condition{} p.say = func(c oc.Condition) error { *said = append(*said, c); return nil } mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fpTest, Repository: "novox/resolved", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath}) - p.w.put(buckets.Modules, "systemd-resolved", mod) + putModuleRaw(p.w, trunkBranch, "systemd-resolved", mod) placeOn(t, p.w, "shanks", "systemd-resolved") on, _ := presenceValue(true) p.w.put(buckets.Presence, "shanks", on) @@ -1198,7 +1198,7 @@ func aReleasesApproveAnswersTheMoveOfItsBuildAlone(t *testing.T) { p := newPasser(kvSet{buckets.Asks: asksKV}, time.Now()) p.logf = t.Logf mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fpTest, Repository: "novox/web", Commit: commit, Src: 7, FpSrc: c.fpSrc, Path: meshcatalogclient.RootPath}) - p.w.put(buckets.Modules, "web", mod) + putModuleRaw(p.w, trunkBranch, "web", mod) src, _ := mdc.EncodeSource(mdseat.SourceByBranchValue{Group: "feat/web", Heads: map[string]string{"novox/web": commit}}) p.w.putEntry(buckets.Source, mdc.SourceKey(trunkBranch), src, 7, time.Now()) later, _ := mdc.EncodeSource(mdseat.SourceByBranchValue{Group: "feat/other", Heads: map[string]string{"novox/web": commit, "novox/x": commit}}) @@ -1261,7 +1261,7 @@ func aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t *testing.T) { p.logf = t.Logf put := func(bucket, key string, v []byte) { p.w.put(bucket, key, v) } mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fpTest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath}) - put(buckets.Modules, "web", mod) // the trunk's build is not done: rule 1 holds g14 at fpOld + putModuleRaw(p.w, trunkBranch, "web", mod) // the trunk's build is not done: rule 1 holds g14 at fpOld placeOn(t, p.w, "g14", "web") pl, _ := mmac.EncodeAssignments(mma.Assignments{Node: "g14", Modules: []mma.PlacedModule{{Module: "web"}}}) put(plc.AssignmentsBucket, "assignments.g14", pl) @@ -1380,7 +1380,7 @@ func TestAnApprovedAssignmentsCallsNoControllerVerb(t *testing.T) { func aWantWaitsOnTheVerbEdgesTheGraphKeeps(t *testing.T) { p := newPasser(kvSet{}, time.Now()) fp := func(c string) string { return meshcatalogclient.SourcePrefix2 + strings.Repeat(c, 64) } - if p.served(context.Background(), nil, graph.Graph{}, assign.Nodes{}) != nil { + if p.served(graph.Graph{}, assign.Nodes{}) != nil { t.Error("with no graph read, a want waits on a verb") } for module, n := range map[string]graph.Node{ @@ -1394,26 +1394,25 @@ func aWantWaitsOnTheVerbEdgesTheGraphKeeps(t *testing.T) { if err != nil { t.Fatal(err) } - p.w.put(buckets.Graph, module, raw) + _ = raw + p.w.putModule(trunkBranch, module, meshcatalog.ModulesByModuleValue{}, n, nil) } - g := p.moduleGraph() + g := p.moduleGraph(trunkBranch) if got := keptInvokes(g, "caller"); !slices.Equal(got, []string{"seat:x.v"}) || keptInvokes(g, "unknown") != nil { t.Errorf("the kept verb edges of caller = %v", got) } digest := strings.Repeat("d", 64) - man, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: "server", Manifest: `{"module":"server","claims":[{"name":"x","scope":"mesh","serves":["w"]}]}`}) - get := func(_ context.Context, key string) ([]byte, error) { - if key == digest { - return man, nil - } - return nil, errNotThere + parsedServer, err := catalogue.Decode([]byte(`{"module":"server","claims":[{"name":"x","scope":"mesh","serves":["w"]}]}`)) + if err != nil { + t.Fatal(err) } + p.w.manifests[digest] = parsed{module: "server", manifest: parsedServer} build, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: digest}) key, _ := builds.BuilderBuildKey("server", fp("f")) p.w.put(buckets.Builds, key, build) running := nodeengine.HaveByNodeModuleValue{Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} at := func(f string) nodeengine.HaveByNodeModuleValue { h := running; h.Fp = f; return h } - served := p.served(context.Background(), get, g, assign.Nodes{Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{ + served := p.served(g, assign.Nodes{Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{ "g14": {"server": at(fp("f")), "late": at(fp("2"))}}}) for v, want := range map[string]bool{"seat:x.w": true, "seat:x.v": false, "late.t": true} { if served(v) != want { @@ -1424,7 +1423,7 @@ func aWantWaitsOnTheVerbEdgesTheGraphKeeps(t *testing.T) { // offline node offers nothing, and one whose manifest is not read counts its trunk manifest's offer. bus := map[string]catalogue.Manifest{"bus": {Module: "bus", Provides: []catalogue.Offer{{Name: graph.BusState, Scope: catalogue.ScopeMesh}}}} for _, online := range []bool{false, true} { - offering := p.offering(context.Background(), get, bus, assign.Nodes{Online: map[string]bool{"n2": online}, + offering := p.offering(bus, assign.Nodes{Online: map[string]bool{"n2": online}, Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{"n2": {"bus": at(fp("2"))}}}) if offering(graph.BusState) != online { t.Errorf("with the bus module's node online %v, bus-state offered %v", online, offering(graph.BusState)) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go index 5e9ab67e..efddeb7a 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go @@ -151,6 +151,8 @@ type pipelineEnds struct { // firsts is, by module, the first watched branch of its registration, the branch of a placement that names none (design // 47 G §14). firsts map[string]string + // onBranches is told the branches a registration names, for the catalog half of each to be followed (design 47 G §14, §195). + onBranches func([]string) } func newPipelineEnds(st endStore, wake func(), logf func(string, ...any)) *pipelineEnds { @@ -379,6 +381,9 @@ func (e *pipelineEnds) registered(reg mpcat.Registration, gone bool) { e.logf("the end %s is not deleted: %v", k, err) } } + if e.onBranches != nil && !gone { + e.onBranches(reg.Branches) + } if !e.on.Swap(true) { e.logf("the first registration is heard (%s): the wants follow the pipelines' ends from now", reg.Module) } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines_test.go index acb0c207..db115598 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines_test.go @@ -3,6 +3,11 @@ package main import ( "context" "encoding/json" + "fmt" + mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient" + mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" + plc "mesh-delivery-assigner/internal/assignment" + "slices" "strings" "testing" "time" @@ -147,7 +152,7 @@ func TestAPassWritesNoWantWithoutAPassedPipeline(t *testing.T) { p.say = func(oc.Condition) error { return nil } mdigest := strings.Repeat("e", 64) mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: mdigest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath}) - p.w.put(buckets.Modules, "web", mod) + putModuleRaw(p.w, trunkBranch, "web", mod) p.w.manifests[mdigest] = parsed{module: "web", manifest: catalogue.Manifest{Module: "web"}} bkey, _ := builds.BuilderBuildKey("web", fp) built, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: mdigest}) @@ -211,3 +216,101 @@ func TestTheWantsSwitchOnTheFirstRegistration(t *testing.T) { t.Errorf("a registration gone left %d ends", len(st.vals)) } } + +// Design 47 G §14, §184 (test 68): a node's entry is computed from the catalog half of the branch its placement names, so a +// change on `beta` moves only the want of the node whose placement chose `beta`. +func TestACommitOnBetaMovesOnlyTheNodeThatChoseBeta(t *testing.T) { + fp := func(c string) string { return meshcatalogclient.SourcePrefix2 + strings.Repeat(c, 64) } + wantsKV := &memKV{values: map[string][]byte{}} + p := newPasser(kvSet{buckets.Wants: wantsKV, buckets.Asks: &memKV{values: map[string][]byte{}}}, time.Now()) + p.say = func(oc.Condition) error { return nil } + p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf) + p.ends.registered(mpcat.Registration{Module: "web", Branches: []string{"main", "beta"}}, false) + digest := "sha256:" + strings.Repeat("c", 64) + on, _ := presenceValue(true) + for _, node := range []string{"g14", "ace"} { + composes(t, p.w, node, digest) + p.w.put(buckets.Presence, node, on) + } + for node, branch := range map[string]string{"g14": "", "ace": "beta"} { + pl := mma.Assignments{Node: node, Modules: []mma.PlacedModule{{Module: "web", Branch: branch}}} + values, err := mmac.AssignmentsValues(pl) + if err != nil { + t.Fatal(err) + } + for key, raw := range values { + p.w.put(plc.AssignmentsBucket, key, raw) + } + } + // publish makes the half of branch name web at fingerprint f: built, granted and passed by its pipeline. + publish := func(branch, f string, source int64) { + mdigest := strings.Repeat("e", 62) + fmt.Sprintf("%02d", source) + if branch == "beta" { + mdigest = strings.Repeat("f", 62) + fmt.Sprintf("%02d", source) + } + mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: f, Manifest: mdigest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath}) + putModuleRaw(p.w, branch, "web", mod) + p.w.manifests[mdigest] = parsed{module: "web", manifest: catalogue.Manifest{Module: "web"}} + bkey, _ := builds.BuilderBuildKey("web", f) + built, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: mdigest}) + p.w.put(buckets.Builds, bkey, built) + gkey, _ := grantsstate.PartGrantKey("web", f) + grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{}) + p.w.put(buckets.Grants, gkey, grant) + _ = p.ends.take(pipe("web", branch, source, 1, f, mpr.PipelineEndedOutcomePassed), true) + } + run := func(node, f string) { + have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: f, Composed: digest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}) + p.w.put(buckets.Have, node+".web", have) + } + fpWant := func(node string) string { + key, _ := mwc.WantKey(node, "web") + w, err := mwc.DecodeWant(wantsKV.values[key]) + if err != nil { + t.Fatalf("the want of web on %s: %v", node, err) + } + return w.Fp + } + publish("main", fp("a"), 5) + publish("beta", fp("b"), 5) + run("g14", fp("a")) + run("ace", fp("b")) + p.wants(context.Background()) + if fpWant("g14") != fp("a") || fpWant("ace") != fp("b") { + t.Fatalf("each node is wanted at its branch's build: g14 %s, ace %s", fpWant("g14"), fpWant("ace")) + } + publish("beta", fp("d"), 6) + p.wants(context.Background()) + if fpWant("g14") != fp("a") || fpWant("ace") != fp("d") { + t.Errorf("a commit on beta moves only ace: g14 %s (want main's), ace %s (want beta's new)", fpWant("g14"), fpWant("ace")) + } +} + +// Design 47 G §14, §195: the catalog half of each branch is taken into the world per branch, an entry as the value and graph node +// the rules read, and a registration's branches are each followed. +func TestTheCatalogHalfIsTakenInPerBranch(t *testing.T) { + w := newWorld() + woken := 0 + h := &halves{w: w, wake: func() { woken++ }, logf: t.Logf} + entry := func(fp string) json.RawMessage { + raw, _ := json.Marshal(meshcatalog.CatalogEntry{Module: "web", Fp: fp, Repository: "novox/web", Commit: "c1", Manifest: map[string]any{"module": "web", "version": "1"}, + Invokes: []string{"seat:x.v"}}) + return raw + } + h.changed("main", "web", entry("src2:1"), false) + h.changed("beta", "web", entry("src2:2"), false) + if w.trunkOf("main")["web"].Fp2 != "src2:1" || w.trunkOf("beta")["web"].Fp2 != "src2:2" || woken != 2 { + t.Errorf("main %+v, beta %+v, woken %d: each branch keeps its own entry", w.trunkOf("main")["web"], w.trunkOf("beta")["web"], woken) + } + v := w.trunkOf("beta")["web"] + if got := w.manifest(v.Manifest); got.err != "" || got.manifest.Module != "web" { + t.Errorf("the entry's manifest is not read by its digest: %+v", got) + } + if n := w.graphOf("main")["web"]; n.FP != "src2:1" || !slices.Equal(n.Invokes, []string{"seat:x.v"}) { + t.Errorf("graph node %+v", n) + } + h.changed("beta", "web", nil, true) + if _, ok := w.trunkOf("beta")["web"]; ok || w.trunkOf("main")["web"].Fp2 == "" { + t.Error("a module taken out of beta's half leaves main's") + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go index 0322e53c..75ffe180 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go @@ -112,7 +112,6 @@ func (p *passer) puts(ctx context.Context) { } assigned := p.w.assigned() trunk := p.w.trunk() - get := p.w.getter(buckets.Manifests) // the indexer's manifests, as its first view and views family say them wanted := p.w.wantsOf() modules := make([]string, 0, len(all)) for m := range all { @@ -122,7 +121,7 @@ func (p *passer) puts(ctx context.Context) { for _, module := range modules { var manifest catalogue.Manifest if mod, ok := trunk[module]; ok && mod.Manifest != "" { - if got := p.w.manifest(ctx, get, mod.Manifest); got.err == "" { + if got := p.w.manifest(mod.Manifest); got.err == "" { manifest = got.manifest } } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat.go index 8efd714f..fb1a8330 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat.go @@ -16,7 +16,6 @@ import ( "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" mdcc "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliverycheckerclient" mdc "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks" @@ -465,10 +464,8 @@ func (p *passer) layAgain(ctx context.Context, prop mma.Proposal, rev uint64, cu // nodes with a record. func (p *passer) lays(ctx context.Context) (func(string) bool, func(string) bool) { held := map[string]bool{} - for key := range p.w.read(buckets.Modules) { - if name, err := meshcatalogclient.ParseModulesKey(key); err == nil { - held[name] = true - } + for name := range p.w.trunk() { + held[name] = true } records := p.w.records() return func(m string) bool { return held[m] }, func(n string) bool { _, ok := records[n]; return ok } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/verbs.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/verbs.go index bc3f55e6..9fa5ea2f 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/verbs.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/verbs.go @@ -1,7 +1,6 @@ package main import ( - "context" "mesh-delivery-assigner/internal/buckets" "slices" @@ -20,21 +19,8 @@ import ( // each kept edge until a build serving its verb runs healthy, read from the haves, the builder's builds and the indexer's // manifests, as a provider is counted (ADR 0603 §5). -// moduleGraph is the module graph as the indexer's first view and views family last said it (ADR 0631); a key that -// does not decode is left out, so its module waits on no verb. -func (p *passer) moduleGraph() graph.Graph { - g := graph.Graph{} - for key, raw := range p.w.read(buckets.Graph) { - module, err := graph.ParseKey(key) - if err != nil { - continue - } - if n, err := graph.Decode(raw); err == nil { - g[module] = n - } - } - return g -} +// moduleGraph is the module graph of the catalog half of branch, from the entries' invokes and serves (design 47 G §184, §195). +func (p *passer) moduleGraph(branch string) graph.Graph { return p.w.graphOf(branch) } // keptInvokes are the verbs of module the graph keeps as edges: each it invokes and does not leave out, that a module // other than itself serves. A verb no module of the graph serves is no edge, so it holds nothing; nil when the graph holds @@ -61,11 +47,11 @@ func keptInvokes(g graph.Graph, module string) []string { // serves are served. A healthy build whose manifest cannot be read counts what the graph says its module serves, said in // the log, so a build read late never holds its callers for good; nil when the graph holds nothing, and no want waits on // a verb. -func (p *passer) served(ctx context.Context, get getter, g graph.Graph, n assign.Nodes) func(string) bool { +func (p *passer) served(g graph.Graph, n assign.Nodes) func(string) bool { if len(g) == 0 { return nil } - served, unread := graph.Served(n.Haves, func(module, fp string) ([]string, bool) { return p.servesOf(ctx, get, module, fp) }, + served, unread := graph.Served(n.Haves, func(module, fp string) ([]string, bool) { return p.servesOf(module, fp) }, func(module string) []string { return g[module].Serves }) if len(unread) > 0 { p.logf("the manifest of the running build of %v cannot be read, so what the module graph says they serve is counted as served", unread) @@ -77,7 +63,7 @@ func (p *passer) served(ctx context.Context, get getter, g graph.Graph, n assign // §5, ADR 0614 §5): every running healthy have on an online node, its build's manifest read as servesOf reads it. A // healthy build whose manifest cannot be read counts what its module's trunk manifest offers, so a build read late never // holds a consumer for good. -func (p *passer) offering(ctx context.Context, get getter, trunk map[string]catalogue.Manifest, n assign.Nodes) func(string) bool { +func (p *passer) offering(trunk map[string]catalogue.Manifest, n assign.Nodes) func(string) bool { offered := map[string]bool{} for node, haves := range n.Haves { if !n.Online[node] { @@ -87,7 +73,7 @@ func (p *passer) offering(ctx context.Context, get getter, trunk map[string]cata if h.Phase != nodeengineclient.PhaseRunning || h.Health != nodeengineclient.HealthOK { continue } - m, ok := p.manifestOf(ctx, get, module, h.Fp) + m, ok := p.manifestOf(module, h.Fp) if !ok { m = trunk[module] } @@ -100,8 +86,8 @@ func (p *passer) offering(ctx context.Context, get getter, trunk map[string]cata } // servesOf are the verbs the manifest of module's build at fp serves (graph.ServesOf), and whether it could be read. -func (p *passer) servesOf(ctx context.Context, get getter, module, fp string) ([]string, bool) { - m, ok := p.manifestOf(ctx, get, module, fp) +func (p *passer) servesOf(module, fp string) ([]string, bool) { + m, ok := p.manifestOf(module, fp) if !ok { return nil, false } @@ -110,7 +96,7 @@ func (p *passer) servesOf(ctx context.Context, get getter, module, fp string) ([ // manifestOf is the manifest of module's build at fp (the builder's build names its digest in the indexer's manifests), // and whether it could be read. -func (p *passer) manifestOf(ctx context.Context, get getter, module, fp string) (catalogue.Manifest, bool) { +func (p *passer) manifestOf(module, fp string) (catalogue.Manifest, bool) { key, err := builds.BuilderBuildKey(module, fp) if err != nil { return catalogue.Manifest{}, false @@ -123,7 +109,7 @@ func (p *passer) manifestOf(ctx context.Context, get getter, module, fp string) if err != nil || b.Manifest == "" { return catalogue.Manifest{}, false } - got := p.w.manifest(ctx, get, b.Manifest) + got := p.w.manifest(b.Manifest) if got.err != "" { return catalogue.Manifest{}, false } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/views.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/views.go index e1b6659f..4186a832 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/views.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/views.go @@ -6,6 +6,7 @@ import ( "mesh-delivery-assigner/internal/assignment" "mesh-delivery-assigner/internal/buckets" "slices" + "sync" "time" "github.com/nats-io/nats.go/jetstream" @@ -20,7 +21,8 @@ import ( sdkfindings "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient/findings" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient/heard" - "git.novox.be/novox/mesh-delivery/sdk/graph" + mcat "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "mesh-delivery-assigner/internal/assign" plc "mesh-delivery-assigner/internal/assignment" @@ -126,10 +128,11 @@ func viewFindings(f assign.Findings) views.FindingWriter { // firstViewsBucket is the assigner's own state of its first views (views.StatusState). var firstViewsBucket = assignment.Module + "_" + views.StatusState -// indexerViews are the assigner's first views of the indexer's manifests, modules and module graph (ADR 0615 §1, -// ADR 0621 §2, ADR 0631; DLV-F2r): asked at the start through the indexer's tools, then followed on indexer-views, asking once more on a gap; -// past the bound its finding names the indexer and the tool, cleared by the answer. -type indexerViews struct { +// catalogViews is the assigner's first view of the catalog half of each watched branch (design 47 G §14, §184, §195): asked +// at the start through the seat mesh-catalog's verb `revision` (the trunk's, then each branch a registration names), then +// followed on its event `catalog-revision`, asking once more on a gap; past the bound its finding names the seat and the verb, +// cleared by the answer. Its status is read by assigner_first_views and asked again by assigner_first_view_again. +type catalogViews struct { js jetstream.JetStream account process.Account call func(string, any) (json.RawMessage, error) @@ -137,21 +140,17 @@ type indexerViews struct { status *process.AwaitedBucket logf func(string, ...any) hub *process.EventHub -} -// indexerFirstViews are the indexer's states the assigner takes by first view and the views family, each by the name of -// the indexer's bucket the world keeps it under: manifests and modules (ADR 0621 §2), and the module graph, whose verb -// edges a want waits on (ADR 0614 §2, §3; ADR 0631 §1). -var indexerFirstViews = map[string]string{"manifests": buckets.Manifests, "modules": buckets.Modules, graph.State: buckets.Graph} + h *halves +} // firstViewBound and firstViewWait are each first view's bound before its condition is raised, and its first wait // between two failed asks; zero is the views package's (views.Bound, two minutes, ADR 0615 §6; and a first wait of five // seconds). A test shortens them. var firstViewBound, firstViewWait time.Duration -// start asks the three first views, each answer taken into w under the indexer's bucket names and settled once (a pass waits -// for it), and hears the views family; it answers how many settles to wait for. -func (iv *indexerViews) start(ctx context.Context, w *world, wake func(), settled chan<- struct{}) int { +// start follows the trunk's catalog half and hears the seat's catalog-revision; it answers how many settles to wait for. +func (iv *catalogViews) start(ctx context.Context, w *world, wake func(), settled chan<- struct{}) int { finding := func(s views.Status, raised bool) { if iv.findings.Put == nil { return @@ -167,33 +166,28 @@ func (iv *indexerViews) start(ctx context.Context, w *world, wake func(), settle iv.logf("views: the finding of the first view %s is not written: %v", s.Tool, err) } } - followers := map[string]*views.Follower{} - var all []*views.Follower - for state, bucket := range indexerFirstViews { - bucket := bucket - tool := buckets.IndexerModule + "." + state - f := &views.Follower{Owner: buckets.IndexerModule, Tool: tool, State: state, Ask: views.AskVia(iv.call, tool), Logf: iv.logf, - Bound: firstViewBound, Wait: firstViewWait, - Raise: func(s views.Status) { finding(s, true) }, Clear: func(s views.Status) { finding(s, false) }, - Changed: func(key string, value json.RawMessage, deleted bool, at time.Time) { - if deleted { - value = nil - } - w.putEntry(bucket, key, value, 0, at) - wake() - }} - followers[state] = f - all = append(all, f) - go func() { - if f.Start(ctx) == nil { - settled <- struct{}{} - } - }() + var statusKV jetstream.KeyValue + var mu sync.Mutex + iv.h = &halves{w: w, call: iv.call, wake: wake, logf: iv.logf, ctx: ctx, settled: settled} + iv.h.started = func(c *mcat.Catalog) { + f := c.Follower + f.Bound, f.Wait = firstViewBound, firstViewWait + f.Raise, f.Clear = func(s views.Status) { finding(s, true) }, func(s views.Status) { finding(s, false) } + mu.Lock() + defer mu.Unlock() + if statusKV != nil { + f.Said = views.KeepStatus(ctx, statusKV, iv.logf) + } } if iv.status != nil { iv.status.Ready(func(kv jetstream.KeyValue) { - for _, f := range all { - f.Said = views.KeepStatus(ctx, kv, iv.logf) + mu.Lock() + statusKV = kv + mu.Unlock() + var all []*views.Follower + for _, c := range iv.h.followers() { + c.Follower.Said = views.KeepStatus(ctx, kv, iv.logf) + all = append(all, c.Follower) } if err := views.HearAgain(ctx, kv, all, iv.logf); err != nil { iv.logf("views: assigner_first_view_again is not heard (%v) until the assigner starts again", err) @@ -202,16 +196,17 @@ func (iv *indexerViews) start(ctx context.Context, w *world, wake func(), settle } // The hub says when the bus made its consumer: a picture asked before then is asked again (views.Follower.ConsumerMade). iv.hub.OnBound(func(created time.Time) { - for _, f := range all { - f.ConsumerMade(ctx, created) + for _, c := range iv.h.followers() { + c.Follower.ConsumerMade(ctx, created) } }) - // Not heard, the indexer's views family is the finding family-not-heard until it binds (ADR 0615 §5): the handler is - // registered on the module's hub, which the assigner starts once every handler is. - _ = notHeardWriter(iv.findings, iv.logf).Bind(process.EventsStream, "the indexer's manifests, modules and graph are the first views' alone until the assigner starts again", func() error { - return iv.hub.Handle("mesh.mod."+buckets.IndexerModule+".event."+views.EventPrefix+"*", views.Hear(ctx, followers, iv.logf)) + // Not heard, the catalog-revision is the finding family-not-heard until it binds (ADR 0615 §5): the handler is registered + // on the module's hub, which the assigner starts once every handler is. + _ = notHeardWriter(iv.findings, iv.logf).Bind(process.EventsStream, "the catalog half is the first views' alone until the assigner starts again", func() error { + return iv.hub.Handle(meshcatalog.SubjectCatalogRevisionAll, iv.h.hear) }) - return len(all) + iv.h.follow(trunkBranch) + return 1 } // notHeardWriter writes the assigner's finding of a family it does not hear into its findings (heard.Writer); with no diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go index be881eb6..876e04e0 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go @@ -1,7 +1,6 @@ package main import ( - "context" "encoding/json" "errors" "maps" @@ -14,16 +13,14 @@ import ( "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" wdc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshwatchdogsclient" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" mdc "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" mwc "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshwantsclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "git.novox.be/novox/mesh-delivery/sdk/graph" "git.novox.be/novox/mesh-delivery/sdk/trunk" "git.novox.be/novox/mesh-sdk/go/keys" - "github.com/nats-io/nats.go/jetstream" - mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient" mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" "git.novox.be/novox/mesh-sdk/go/catalogue" @@ -56,6 +53,10 @@ type world struct { // durable consumer (design 47 G §199, §309); runningHeard says that consumer is bound. running map[string]nodeengine.Running runningHeard bool + // modules and graphs are, by watched branch and module, the catalog half the assigner follows per branch (design 47 G §14, + // §184, §195), as the value and the graph node the rules read: written by putModule from the entries of meshcatalogclient.Catalog. + modules map[string]map[string]meshcatalog.ModulesByModuleValue + graphs map[string]map[string]graph.Node } // entryMeta is a value's revision in its bucket and when it was written. @@ -74,7 +75,8 @@ type parsed struct { func newWorld() *world { return &world{values: map[string]map[string][]byte{}, manifests: map[string]parsed{}, healthy: map[string][]string{}, changes: map[string]uint64{}, meta: map[string]entryMeta{}, trunkBuilt: map[string]builds.BuildLogEntry{}, assignmentKeys: mmac.NewKeys(), - sourceGroups: map[uint64]string{}, running: map[string]nodeengine.Running{}} + sourceGroups: map[uint64]string{}, running: map[string]nodeengine.Running{}, + modules: map[string]map[string]meshcatalog.ModulesByModuleValue{}, graphs: map[string]map[string]graph.Node{}} } // putEntry is put, keeping the value's revision and the time it was written. @@ -194,71 +196,16 @@ func (w *world) read(bucket string) map[string][]byte { return out } -// getter reads one key's value; errNotThere when the key does not exist. -type getter func(ctx context.Context, key string) ([]byte, error) - var errNotThere = errors.New("no such key") -// kvGetter reads keys of a bucket. -func kvGetter(kv jetstream.KeyValue) getter { - return func(ctx context.Context, key string) ([]byte, error) { - e, err := kv.Get(ctx, key) - if errors.Is(err, jetstream.ErrKeyNotFound) { - return nil, errNotThere - } - if err != nil { - return nil, err - } - return e.Value(), nil - } -} - -// getter reads a key of a bucket the world holds: errNotThere where it holds none. -func (w *world) getter(bucket string) getter { - return func(_ context.Context, key string) ([]byte, error) { - w.mu.Lock() - defer w.mu.Unlock() - v, ok := w.values[bucket][key] - if !ok { - return nil, errNotThere - } - return v, nil - } -} - -// manifest reads the manifest of a digest from the indexer's bucket once and keeps it: a digest's manifest never changes. -func (w *world) manifest(ctx context.Context, get getter, digest string) parsed { +// manifest is the manifest of a digest the catalog half said (putModule keeps each once); one not said yet is not readable. +func (w *world) manifest(digest string) parsed { w.mu.Lock() - p, ok := w.manifests[digest] - w.mu.Unlock() - if ok { + defer w.mu.Unlock() + if p, ok := w.manifests[digest]; ok { return p } - key, err := meshcatalogclient.ManifestsKey(digest) - if err != nil { - return parsed{err: err.Error()} - } - value, err := get(ctx, key) - if errors.Is(err, errNotThere) { - return parsed{err: "the indexer holds no manifest of digest " + digest + " yet"} - } - if err != nil { - return parsed{err: err.Error()} - } - m, err := meshcatalogclient.DecodeManifest(value) - if err != nil { - p = parsed{err: err.Error()} - } else if man, err := catalogue.Decode([]byte(m.Manifest)); err != nil { - // Decoded, never judged (novox/hq ADR 0578 §3): the indexer's judge judged it, and its verdict is taken from the - // module's value where it is read (world.mesh). - p = parsed{module: m.Module, err: "it does not decode: " + err.Error()} - } else { - p = parsed{module: m.Module, manifest: man} - } - w.mu.Lock() - w.manifests[digest] = p - w.mu.Unlock() - return p + return parsed{err: "the catalog half holds no manifest of digest " + digest + " yet"} } // assigned is each node's modules the seat mesh-module-assigner's assignments names, in name order (ADR 0549 §5, ADR 0567 §8): the @@ -299,21 +246,74 @@ func (w *world) placedNodes() map[string]bool { return out } -// trunk is every module the indexer holds at the trunk, with its value. +// trunkOf is every module the catalog half of branch holds, with its value (design 47 G §184). +func (w *world) trunkOf(branch string) map[string]meshcatalog.ModulesByModuleValue { + w.mu.Lock() + defer w.mu.Unlock() + return maps.Clone(w.modules[branch]) +} + +// graphOf is the module graph of the catalog half of branch. +func (w *world) graphOf(branch string) graph.Graph { + w.mu.Lock() + defer w.mu.Unlock() + return graph.Graph(maps.Clone(w.graphs[branch])) +} + +// branches are the watched branches the catalog half is held of, the trunk's first. +func (w *world) branches() []string { + w.mu.Lock() + defer w.mu.Unlock() + out := slices.Sorted(maps.Keys(w.modules)) + if i := slices.Index(out, trunkBranch); i > 0 { + out = slices.Insert(slices.Delete(out, i, i+1), 0, trunkBranch) + } + return out +} + +// trunk is every module any branch's catalog half holds, each with its value on the first branch that holds it (the trunk's +// first): the modules that exist, for the parts that ask which, not at which fingerprint a node runs it. func (w *world) trunk() map[string]meshcatalog.ModulesByModuleValue { out := map[string]meshcatalog.ModulesByModuleValue{} - for key, value := range w.read(buckets.Modules) { - name, err := meshcatalogclient.ParseModulesKey(key) - if err != nil { - continue - } - if m, err := meshcatalogclient.DecodeModule(value); err == nil { - out[name] = m + for _, b := range w.branches() { + for name, v := range w.trunkOf(b) { + if _, ok := out[name]; !ok { + out[name] = v + } } } return out } +// putModule takes one module of branch's catalog half in: its value, its graph node and its parsed manifest by the digest the +// value names. +func (w *world) putModule(branch, module string, v meshcatalog.ModulesByModuleValue, n graph.Node, m *parsed) { + w.mu.Lock() + defer w.mu.Unlock() + if w.modules[branch] == nil { + w.modules[branch], w.graphs[branch] = map[string]meshcatalog.ModulesByModuleValue{}, map[string]graph.Node{} + } + w.modules[branch][module], w.graphs[branch][module] = v, n + if m != nil && v.Manifest != "" { + w.manifests[v.Manifest] = *m + } +} + +// dropModule takes a module out of branch's catalog half. +func (w *world) dropModule(branch, module string) { + w.mu.Lock() + defer w.mu.Unlock() + delete(w.modules[branch], module) + delete(w.graphs[branch], module) +} + +// holdsBranch says whether the catalog half of branch has been taken in (even empty). +func (w *world) holdsBranch(branch string) bool { + w.mu.Lock() + defer w.mu.Unlock() + return w.modules[branch] != nil +} + // records is each node's record in its own have key (`.node`): the nodes an assignments proposal is answered by, // with those assignments names (ADR 0593 §8). func (w *world) records() map[string]nodeengine.NodeRecord { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world_test.go index 73a1d5cf..7dcc316d 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world_test.go @@ -10,6 +10,7 @@ import ( "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate" + "git.novox.be/novox/mesh-delivery/sdk/graph" ) // fakeManifests is the indexer's manifests bucket in memory. @@ -71,3 +72,12 @@ func theWorldCountsTheValuesAKeyTakes(t *testing.T) { } } } + +// putModuleRaw takes a module value encoded as the catalog's modules say it into the world as branch's catalog half holds it. +func putModuleRaw(w *world, branch, module string, raw []byte) { + v, err := meshcatalogclient.DecodeModule(raw) + if err != nil { + panic(err) + } + w.putModule(branch, module, v, graph.Node{FP: v.Fp2, Repository: v.Repository}, nil) +} diff --git a/modules/mesh-delivery-assigner/manifest_test.go b/modules/mesh-delivery-assigner/manifest_test.go index ba1b4796..0843516b 100644 --- a/modules/mesh-delivery-assigner/manifest_test.go +++ b/modules/mesh-delivery-assigner/manifest_test.go @@ -123,11 +123,11 @@ func TestManifest(t *testing.T) { } // ADR 0618 §4: and it hears each node's engine say its reads changed; ADR 0636 §4 (issue 1966): and the seat list's two events; // issue 1966 note 151: and the teller's ask-ended, its subject the one the assigner hears. - if !slices.Equal(m.Consumes, []string{"mesh-catalog.registration-changed", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", "mesh-delivery-indexer.views.graph", "mesh-delivery-indexer.views.manifests", "mesh-delivery-indexer.views.modules", buckets.TellerModule + "." + tellerevents.AskEnded, "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.group-ended", "node-engine.manifest-reads-changed", "node-engine.running"}) { - t.Errorf("consumes = %v, want the indexer's views of graph, manifests and modules, the teller's ask-ended, the engines' reads changed and what they run", m.Consumes) + if !slices.Equal(m.Consumes, []string{"mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", buckets.TellerModule + "." + tellerevents.AskEnded, "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.group-ended", "node-engine.manifest-reads-changed", "node-engine.running"}) { + t.Errorf("consumes = %v, want the catalog half's revisions (design 47 G §195), the teller's ask-ended, the engines' reads changed and what they run", m.Consumes) } - if want := []string{"mesh-delivery-indexer.graph", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", - "seat:mesh-catalog.registrations", "seat:mesh-catalog.seat-definitions", // ADR 0636 §4: the seat list + if want := []string{"mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", + "seat:mesh-catalog.registrations", "seat:mesh-catalog.revision", "seat:mesh-catalog.seat-definitions", // ADR 0636 §4: the seat list "seat:mesh-hand-acts.hand-act", "seat:mesh-hand-acts.hand-acts", "seat:mesh-hand-acts.warranted", // ADR 0565 §9, §11: the seat alone "seat:node-backup.backed-up", "seat:node-backup.now", "seat:node-engine.manifest-reads", // ADR 0618 §3 diff --git a/modules/mesh-delivery-assigner/module.json b/modules/mesh-delivery-assigner/module.json index e69ae235..78ce919a 100644 --- a/modules/mesh-delivery-assigner/module.json +++ b/modules/mesh-delivery-assigner/module.json @@ -39,12 +39,10 @@ "operator-channel" ], "consumes": [ + "mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", - "mesh-delivery-indexer.views.graph", - "mesh-delivery-indexer.views.manifests", - "mesh-delivery-indexer.views.modules", "mesh-delivery-teller.ask-ended", "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.group-ended", @@ -52,10 +50,10 @@ "node-engine.running" ], "invokes": [ - "mesh-delivery-indexer.graph", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "seat:mesh-catalog.registrations", + "seat:mesh-catalog.revision", "seat:mesh-catalog.seat-definitions", "seat:mesh-hand-acts.hand-act", "seat:mesh-hand-acts.hand-acts", diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go index bebc4d19..dcc14e41 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go @@ -7,6 +7,10 @@ import ( "encoding/json" "fmt" "os" + "time" + + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" + "git.novox.be/novox/mesh-sdk/go/views" seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner" stdio "git.novox.be/novox/mesh-sdk/go" @@ -82,9 +86,22 @@ func Tools(r pipeline.Reads) []stdio.Tool { return out } +// Module and Slug are the runner's module and its tools' prefix, as its manifest names them. +const ( + Module = "mesh-delivery-pipeline" + Slug = "pipeline" +) + +// FirstViewTools are the runner's `_first_views` and `_first_view_again` over its state first-views +// (design 47 G §77, §200): each view it holds or waits for, since when, and asked again by hand, a hand act. +func FirstViewTools(st stdio.KeptState, acts views.HandActs, now func() time.Time) []stdio.Tool { + return views.StatusTools(Module, Slug+"_first_views", Slug+"_first_view_again", st, acts, now) +} + func main() { r := pipeline.Reads{Pipelines: kept{stdio.State(seat.StatePipelines)}, Kept: kept{stdio.State(seat.StateKept)}} - if err := stdio.Serve("", Tools(r)); err != nil { + tools := append(Tools(r), FirstViewTools(stdio.State(views.StatusState), meshhandactsclient.Recorder{Ask: stdio.Ask}, time.Now)...) + if err := stdio.Serve("", tools); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-pipeline-tools] %v\n", err) os.Exit(1) } diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main_test.go index 0565790a..73d96db0 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main_test.go @@ -5,6 +5,9 @@ import ( "strings" "testing" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/views" + "mesh-delivery-pipeline/internal/pipeline" ) @@ -13,13 +16,17 @@ type none struct{} func (none) Keys() ([]string, error) { return nil, nil } func (none) Value(string) (json.RawMessage, error) { return nil, nil } -// TestTools: the bundle serves the five read verbs as . (§83, §306; RN15/AC1). +// TestTools: the bundle serves the five read verbs as . (§83, §306; RN15/AC1), and the first-view tools +// (§77, §200; RN2/AC4). func TestTools(t *testing.T) { var names []string for _, tl := range Tools(pipeline.Reads{Pipelines: none{}, Kept: none{}}) { names = append(names, tl.Name) } - want := "mesh-pipeline-runner.show mesh-pipeline-runner.list mesh-pipeline-runner.stands mesh-pipeline-runner.why-carried mesh-pipeline-runner.kept" + for _, tl := range FirstViewTools(stdio.State(views.StatusState), nil, nil) { + names = append(names, tl.Name) + } + want := "mesh-pipeline-runner.show mesh-pipeline-runner.list mesh-pipeline-runner.stands mesh-pipeline-runner.why-carried mesh-pipeline-runner.kept pipeline_first_views pipeline_first_view_again" if got := strings.Join(names, " "); got != want { t.Errorf("tools %q, want %q", got, want) } diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index a931e9eb..5805b1e0 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -5,6 +5,7 @@ package main import ( "context" + "encoding/json" "fmt" "io" "log" @@ -12,8 +13,12 @@ import ( "os/signal" "syscall" + "github.com/nats-io/nats.go" + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner" "git.novox.be/novox/mesh-sdk/go/process" + + "mesh-delivery-pipeline/internal/pipeline" ) func main() { @@ -39,9 +44,39 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro defer conn.Close() // The state pipelines, `_pipelines`, holds every pipeline by compare-and-set (§27): without it the // runner keeps nothing, so it stops. - if _, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, seat.StatePipelines), "keep a pipeline"); err != nil { + pipelines, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, seat.StatePipelines), "keep a pipeline") + if err != nil { return err } + // The registrations and each watched branch's catalog half, first views then events (§21, §195, §200): until a + // branch's view is held the runner acts on nothing new of it, and says so. + calls, err := process.Connect(cred, nats.Name(cred.Node+"."+cred.Module+".calls"), nats.RetryOnFailedConnect(true)) + if err != nil { + return err + } + defer calls.Close() + hub := process.NewEventHub(logger.Printf) + rv, err := startViews(ctx, js, conn, cred, hub, calls.Call, logger.Printf) + if err != nil { + return err + } + // A pipeline starts by compare-and-set on its key's absence and says pipeline-started (§54, §103); a publish starts + // one per dependent behind (§53, §334). + starter := &pipeline.Starter{Pipelines: createdKV{ctx: ctx, kv: pipelines}, Logf: logger.Printf, + Say: func(subject string, body any) error { + data, err := json.Marshal(body) + if err == nil { + _, err = js.Publish(ctx, subject, data) + } + return err + }} + if err := hearPublished(ctx, hub, starter, calls, rv.Catalog, logger.Printf); err != nil { + return err + } + if err := hub.StartUntilBound(ctx, js, cred, nil, nil); err != nil { + logger.Printf("the consumer %s is not bound yet (%v): bound once the bus makes it", cred.EventConsumerName(), err) + } + go func() { <-ctx.Done(); hub.Stop() }() logger.Printf("holding %s", seat.Seat) <-ctx.Done() return nil diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go new file mode 100644 index 00000000..a2341544 --- /dev/null +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go @@ -0,0 +1,71 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + "github.com/nats-io/nats.go/jetstream" + + builds "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" + catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-sdk/go/buscalls" + "git.novox.be/novox/mesh-sdk/go/process" + + "mesh-delivery-pipeline/internal/pipeline" +) + +// outdatedVerb is the catalog's verb a publication's dependents behind are asked by, as the manifest's invokes names it. +const outdatedVerb = "seat:" + catalog.Seat + "." + catalog.VerbOutdated + +// createdKV is the state pipelines written by compare-and-set on a key's absence (§54). +type createdKV struct { + ctx context.Context + kv jetstream.KeyValue +} + +func (c createdKV) Create(key string, value []byte) error { + _, err := c.kv.Create(c.ctx, key, value) + if errors.Is(err, jetstream.ErrKeyExists) { + return pipeline.ErrExists + } + return err +} + +// hearPublished starts the pipelines of each package-published heard (§53, §334): the dependents the catalog's outdated +// names behind, each asked as a kept call. The message is acknowledged once its starts are written, on a goroutine of its +// own, so a slow catalog holds no other event. +func hearPublished(ctx context.Context, hub *process.EventHub, starter *pipeline.Starter, calls *process.Conn, v *pipeline.Views, logf func(string, ...any)) error { + outdated := func(in catalog.OutdatedInput) (catalog.OutdatedAnswer, error) { + raw, err := calls.CallKept(outdatedVerb, in) + if err != nil { + return catalog.OutdatedAnswer{}, err + } + answer, said, ok := buscalls.Unwrap(raw) + if !ok { + return catalog.OutdatedAnswer{}, fmt.Errorf("%s answered: %s", outdatedVerb, said) + } + var a catalog.OutdatedAnswer + err = json.Unmarshal(answer, &a) + return a, err + } + return hub.Handle(builds.SubjectPackagePublished, func(m jetstream.Msg) { + var ev builds.PackagePublished + if err := json.Unmarshal(m.Data(), &ev); err != nil { + logf("%s is no package-published: %v", m.Subject(), err) + _ = m.Ack() + return + } + go func() { + n, err := starter.Published(ev, outdated, v.View) + if err != nil { + logf("the publish of %s %s on %s: %d pipelines started, then %v; heard again", ev.Artifact, ev.Version, ev.Branch, n, err) + _ = m.Nak() + return + } + logf("the publish of %s %s on %s started %d pipelines", ev.Artifact, ev.Version, ev.Branch, n) + _ = m.Ack() + }() + }) +} diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go new file mode 100644 index 00000000..0ff4228f --- /dev/null +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go @@ -0,0 +1,140 @@ +package main + +import ( + "context" + "sync" + "time" + + "github.com/nats-io/nats.go/jetstream" + + mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + sdkfindings "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient/findings" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-sdk/go/process" + "git.novox.be/novox/mesh-sdk/go/views" + + "mesh-delivery-pipeline/internal/pipeline" +) + +// catalogRevisions is every watched branch's catalog-revision, `mesh.seat.mesh-catalog.event.catalog-revision.`. +const catalogRevisions = "mesh.seat." + catalog.Seat + ".event." + catalog.EventCatalogRevision + ".>" + +// runnerViews are the runner's first views (design 47 G §21, §195, §200): the registrations, then the catalog half of +// each watched branch they name, each said in `_first_views` (its state first-views) and, past five minutes +// without a view, as view-waits in its findings. +type runnerViews struct { + Registrations *meshcatalogclient.Registrations + Catalog *pipeline.Views +} + +// startViews wires the runner's views on hub and asks each first view; register every other handler on hub before or +// after, then start hub once. +func startViews(ctx context.Context, js jetstream.JetStream, conn process.Subscriber, cred process.Account, hub *process.EventHub, + call meshcatalogclient.Call, logf func(string, ...any)) (*runnerViews, error) { + findings := process.AwaitBucket(ctx, js, conn, mcc.FindingsBucket(cred.Module), "a view waited for past five minutes is said in the log alone", logf) + status := process.AwaitBucket(ctx, js, conn, cred.Module+"_"+views.StatusState, "the runner's first views are not said in its first-view tool, nor asked again by hand", logf) + writer := func() views.FindingWriter { return sdkfindings.Writer(ctx, findings.KV) } + reader := &views.ProcessReader{Module: cred.Module, Call: call, Logf: logf, Hub: hub, Findings: writer} + + // A watched branch first named by a registration gets its view of the catalog half (CT10), its conditions and its + // status kept as every first view's, asked at once. + wire := func(f *views.Follower) { + f.Logf = logf + f.Raise = func(s views.Status) { + putFinding(writer(), views.UnansweredFinding(cred.Module, s, time.Now()), true, logf) + } + f.Clear = func(s views.Status) { + putFinding(writer(), views.UnansweredFinding(cred.Module, s, time.Now()), false, logf) + } + f.RaiseWaits = func(s views.Status) { + logf("waits for %s's view %s since %s", s.Owner, s.WaitsFor, s.WaitingSince.Format(time.RFC3339)) + putFinding(writer(), views.ViewWaitsFinding(cred.Module, s, time.Now()), true, logf) + } + f.ClearWaits = func(s views.Status) { + putFinding(writer(), views.ViewWaitsFinding(cred.Module, s, time.Now()), false, logf) + } + f.Said = func(s views.Status) { + if kv := status.KV(); kv != nil { + views.KeepStatus(ctx, kv, logf)(s) + } + } + status.Ready(func(kv jetstream.KeyValue) { + if err := views.HearAgain(ctx, kv, []*views.Follower{f}, logf); err != nil { + logf("the hand asks of the first view %s are not heard (%v) until the runner starts again", f.Tool, err) + } + }) + } + var ( + mu sync.Mutex + made []*views.Follower + ) + cv := &pipeline.Views{Logf: logf, Open: func(branch string) (pipeline.BranchView, error) { + c, err := meshcatalogclient.NewCatalog(call, branch) + if err != nil { + return nil, err + } + wire(c.Follower) + mu.Lock() + made = append(made, c.Follower) + mu.Unlock() + go func() { + if err := c.Start(ctx); err != nil { + logf("the first view of the catalog half of %s was not answered: %v", branch, err) + } + }() + return catalogView{c}, nil + }} + regs, _, err := meshcatalogclient.WireRegistrationsProcess(reader, call, func(reg catalog.Registration, gone bool) { + if !gone { + cv.Registered(reg) + } + }) + if err != nil { + return nil, err + } + if err := hub.Handle(catalogRevisions, func(m jetstream.Msg) { + if err := cv.HearRevision(ctx, m.Data()); err != nil { + logf("%s is not taken: %v", m.Subject(), err) + } + _ = m.Ack() + }); err != nil { + return nil, err + } + // A consumer made after a view was asked does not deliver what was said before it: each branch's view asks again. + hub.OnBound(func(created time.Time) { + mu.Lock() + fs := append([]*views.Follower(nil), made...) + mu.Unlock() + for _, f := range fs { + f.ConsumerMade(ctx, created) + } + }) + reader.Start(ctx, js, cred, status) + return &runnerViews{Registrations: regs, Catalog: cv}, nil +} + +func putFinding(w views.FindingWriter, f views.Finding, raised bool, logf func(string, ...any)) { + var err error + if raised { + err = w.Put(f) + } else { + err = w.Delete(f) + } + if err != nil { + logf("the finding %s is not written: %v", f.Kind, err) + } +} + +// catalogView is a branch's meshcatalogclient.Catalog as the runner reads it. +type catalogView struct{ c *meshcatalogclient.Catalog } + +func (v catalogView) Hear(ctx context.Context, data []byte) error { return v.c.Hear(ctx, data) } + +func (v catalogView) View() (pipeline.Half, bool) { + h, ok := v.c.View() + if !ok { + return pipeline.Half{}, false + } + return pipeline.Half{Revision: h.Revision.Revision, Source: h.Revision.Source, Entries: h.Entries}, true +} diff --git a/modules/mesh-delivery-pipeline/go.mod b/modules/mesh-delivery-pipeline/go.mod index 511f96a5..daa61da4 100644 --- a/modules/mesh-delivery-pipeline/go.mod +++ b/modules/mesh-delivery-pipeline/go.mod @@ -9,6 +9,12 @@ require ( ) require ( + git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/contracts/go/clients/meshpipelinerunnerclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000 github.com/klauspost/compress v1.20.1 // indirect github.com/nats-io/nkeys v0.4.16 // indirect github.com/nats-io/nuid v1.0.1 // indirect diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/registrations.go b/modules/mesh-delivery-pipeline/internal/pipeline/registrations.go new file mode 100644 index 00000000..4eaa2ae2 --- /dev/null +++ b/modules/mesh-delivery-pipeline/internal/pipeline/registrations.go @@ -0,0 +1,170 @@ +package pipeline + +import ( + "context" + "fmt" + "sort" + "sync" + + catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner" +) + +// Status is a step's status for one artifact of a module, read from the module's registration and never from a +// manifest (design 47 G §21, §23): the artifact's own status where its policy names one, else the module's. A step +// neither names is "" (design question Q-RN8, open). +func Status(reg catalog.Registration, artifact, step string) string { + if a, ok := reg.Policy.Artifacts[artifact]; ok { + if s := statusOf(a, step); s != "" { + return s + } + } + return statusOf(reg.Policy.Steps, step) +} + +func statusOf(s catalog.StepStatuses, step string) string { + switch step { + case seat.StepResultStepLint: + return s.Lint + case seat.StepResultStepUnit: + return s.Unit + case seat.StepResultStepIntegration: + return s.Integration + case seat.StepResultStepComposition: + return s.Composition + case seat.StepResultStepReview: + return s.Review + case seat.StepResultStepApproval: + return s.Approval + } + return "" +} + +// buildStep is the acting rule's step a build seat names for the languages it builds (design 47 G §323). +const buildStep = "build" + +// BuildSeat is the build seat that runs lint and the test steps of an artifact of language, read from the catalog half +// of the pipeline's revision view (design 47 G §323): the seat whose acting rule `build` has `on` naming that language, +// as an entry of the half that claims it carries it in `acts`. Two seats naming one language are an error naming both. +func BuildSeat(entries map[string]catalog.CatalogEntry, language string) (string, error) { + var seats []string + for _, e := range entries { + for _, r := range e.Acts { + if r.Step == buildStep && r.On == language && r.Seat != "" && !contains(seats, r.Seat) { + seats = append(seats, r.Seat) + } + } + } + sort.Strings(seats) + switch len(seats) { + case 1: + return seats[0], nil + case 0: + return "", fmt.Errorf("no build seat's acting rule %s names the language %q in the catalog half", buildStep, language) + } + return "", fmt.Errorf("the build seats %v each name the language %q in their acting rule %s", seats, language, buildStep) +} + +func contains(list []string, s string) bool { + for _, x := range list { + if x == s { + return true + } + } + return false +} + +// BranchView is the runner's view of one watched branch's catalog half (design 47 G §195, §200, §479): a first view, then +// that branch's catalog-revision events. meshcatalogclient.Catalog is one. +type BranchView interface { + Hear(ctx context.Context, data []byte) error + View() (Half, bool) +} + +// Half is the catalog half a branch view holds, as the runner reads it: its revision and its entries by module. +type Half struct { + Revision int64 + Source int64 + Entries map[string]catalog.CatalogEntry +} + +// Views is the runner's views of the catalog half, one per watched branch the registrations name (design 47 G §21, +// §195): a branch first named by a registration gets its view made and started by Open, and every catalog-revision +// heard goes to every branch's view, each taking its own branch's alone. +type Views struct { + // Open makes and starts the view of a branch; its error is said in the journal and the branch asked again at the + // next registration heard. + Open func(branch string) (BranchView, error) + Logf func(string, ...any) + + mu sync.Mutex + branches map[string]BranchView +} + +// Registered takes the watched branches of a module's registration (design 47 G §21): each branch the views do not hold +// yet is opened. +func (v *Views) Registered(reg catalog.Registration) { + for _, b := range reg.Branches { + v.ensure(b) + } +} + +func (v *Views) ensure(branch string) { + v.mu.Lock() + defer v.mu.Unlock() + if v.branches == nil { + v.branches = map[string]BranchView{} + } + if _, ok := v.branches[branch]; ok || branch == "" { + return + } + bv, err := v.Open(branch) + if err != nil { + if v.Logf != nil { + v.Logf("the catalog half of the watched branch %s is not followed (asked again at the next registration heard): %v", branch, err) + } + return + } + v.branches[branch] = bv +} + +// Branches are the watched branches the views hold, sorted. +func (v *Views) Branches() []string { + v.mu.Lock() + defer v.mu.Unlock() + out := make([]string, 0, len(v.branches)) + for b := range v.branches { + out = append(out, b) + } + sort.Strings(out) + return out +} + +// View is the catalog half the view of branch holds; false while it has none, when the runner acts on nothing new of +// that branch (design 47 G §200). +func (v *Views) View(branch string) (Half, bool) { + v.mu.Lock() + bv := v.branches[branch] + v.mu.Unlock() + if bv == nil { + return Half{}, false + } + return bv.View() +} + +// HearRevision hands one catalog-revision event to every branch's view. +func (v *Views) HearRevision(ctx context.Context, data []byte) error { + v.mu.Lock() + all := make([]BranchView, 0, len(v.branches)) + for _, bv := range v.branches { + all = append(all, bv) + } + v.mu.Unlock() + var first error + for _, bv := range all { + if err := bv.Hear(ctx, data); err != nil && first == nil { + first = err + } + } + return first +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/registrations_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/registrations_test.go new file mode 100644 index 00000000..0f150d36 --- /dev/null +++ b/modules/mesh-delivery-pipeline/internal/pipeline/registrations_test.go @@ -0,0 +1,85 @@ +package pipeline + +import ( + "context" + "errors" + "strings" + "testing" + + catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" +) + +// The policy, the build seat by language and the views per watched branch (design 47 G §21, §23, §323, §195). +func TestPolicyAndBuildSeat(t *testing.T) { + reg := catalog.Registration{Module: "m", Branches: []string{"main"}, Policy: catalog.Policy{ + Steps: catalog.StepStatuses{Lint: "required", Unit: "required", Integration: "optional", Approval: "required"}, + Artifacts: map[string]catalog.StepStatuses{"tools": {Unit: "bypass"}}, + }} + for _, c := range []struct{ artifact, step, want string }{ + {"program", "unit", "required"}, + {"tools", "unit", "bypass"}, + {"tools", "integration", "optional"}, + {"tools", "approval", "required"}, + {"program", "composition", ""}, + } { + if got := Status(reg, c.artifact, c.step); got != c.want { + t.Errorf("Status(%s, %s) = %q, want %q", c.artifact, c.step, got, c.want) + } + } + + entries := map[string]catalog.CatalogEntry{ + "mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}, + "mesh-build-node": {Module: "mesh-build-node", Acts: []catalog.ActingRule{{Seat: "node-build-node", Step: "build", On: "typescript"}, {Seat: "node-build-node", Step: "check", On: "go"}}}, + "app": {Module: "app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}}}, + } + for _, c := range []struct{ language, want, err string }{ + {"go", "node-build-go", ""}, + {"typescript", "node-build-node", ""}, + {"rust", "", `"rust"`}, + } { + got, err := BuildSeat(entries, c.language) + if got != c.want || (c.err == "") != (err == nil) || (err != nil && !strings.Contains(err.Error(), c.err)) { + t.Errorf("BuildSeat(%s) = %q, %v; want %q, an error naming %s", c.language, got, err, c.want, c.err) + } + } + entries["other"] = catalog.CatalogEntry{Acts: []catalog.ActingRule{{Seat: "node-build-other", Step: "build", On: "go"}}} + if _, err := BuildSeat(entries, "go"); err == nil || !strings.Contains(err.Error(), "node-build-other") { + t.Errorf("two seats naming go: %v, want an error naming both", err) + } + + heard := map[string]int{} + opened := 0 + v := &Views{Open: func(b string) (BranchView, error) { + opened++ + if b == "broken" { + return nil, errors.New("refused") + } + return fakeView{branch: b, heard: heard}, nil + }} + v.Registered(reg) + v.Registered(catalog.Registration{Module: "n", Branches: []string{"main", "feature/x", "broken"}}) + if got := v.Branches(); strings.Join(got, ",") != "feature/x,main" || opened != 3 { + t.Fatalf("branches %v opened %d, want feature/x and main, three opens", got, opened) + } + v.Registered(catalog.Registration{Module: "n", Branches: []string{"broken"}}) + if opened != 4 { + t.Errorf("a branch whose view was refused is opened again at the next registration: %d opens", opened) + } + if err := v.HearRevision(context.Background(), []byte(`{}`)); err != nil || heard["main"] != 1 || heard["feature/x"] != 1 { + t.Errorf("a revision heard reaches every branch's view: %v %v", err, heard) + } + if h, ok := v.View("main"); !ok || h.Revision != 7 { + t.Errorf("View(main) = %v %v", h, ok) + } + if _, ok := v.View("gone"); ok { + t.Error("a branch not watched has no view") + } +} + +type fakeView struct { + branch string + heard map[string]int +} + +func (f fakeView) Hear(context.Context, []byte) error { f.heard[f.branch]++; return nil } +func (f fakeView) View() (Half, bool) { return Half{Revision: 7}, true } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go new file mode 100644 index 00000000..3d693669 --- /dev/null +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -0,0 +1,136 @@ +package pipeline + +import ( + "encoding/json" + "errors" + "fmt" + "strconv" + "time" + + runnerclient "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshpipelinerunnerclient" + builds "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" + catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner" + "git.novox.be/novox/mesh-sdk/go/keys" +) + +// ErrExists is a Create of a pipeline key that is written already: the second start, which writes nothing (§54). +var ErrExists = errors.New("the pipeline's key is written already") + +// Created is the runner's state pipelines as a start writes it: a key written only when it is absent (compare-and-set on +// its absence), ErrExists when it is not. +type Created interface { + Create(key string, value []byte) error +} + +// Begin is what a start names: the pipeline's key (watched branch, source revision, module; design 47 G §27, §54) and +// what the catalog half of that branch's view says of the module. +type Begin struct { + Branch string + Source int64 + Module string + Commit string + Fp string + Group string + Repository string + Revision int64 +} + +// Key is a pipeline's key in the state pipelines, `..`, the branch escaped as a key's +// name is (§27). +func Key(branch string, source int64, module string) string { + return seat.PipelinesByBranchRevisionModuleKey(keys.EscapeName(branch), strconv.FormatInt(source, 10), module) +} + +// Starter starts pipelines (design 47 G §54, §103): it writes a pipeline's key by compare-and-set on its absence, so one +// source revision and module has one pipeline whichever event starts it first, and says pipeline-started for the one it +// wrote. +type Starter struct { + Pipelines Created + // Say says one of the seat's events on its subject. + Say func(subject string, body any) error + Now func() time.Time + Logf func(string, ...any) +} + +// Start writes b's pipeline, running, and says pipeline-started; false, nil when its key was written already (the second +// start writes nothing, §54). +func (s *Starter) Start(b Begin) (bool, error) { + if b.Branch == "" || b.Module == "" || b.Source <= 0 { + return false, fmt.Errorf("a pipeline names its watched branch, source revision and module: %+v", b) + } + now := time.Now().UTC() + if s.Now != nil { + now = s.Now().UTC() + } + key := Key(b.Branch, b.Source, b.Module) + p := seat.Pipeline{V: 1, Pipeline: key, Branch: b.Branch, Source: b.Source, Module: b.Module, Commit: b.Commit, Fp: b.Fp, + Group: b.Group, Repository: b.Repository, Revision: b.Revision, Outcome: seat.PipelineOutcomeRunning, Started: &now, + Steps: []seat.StepResult{}} + raw, err := json.Marshal(p) + if err != nil { + return false, err + } + if err := s.Pipelines.Create(key, raw); errors.Is(err, ErrExists) { + s.logf("%s is started already: this start writes nothing", key) + return false, nil + } else if err != nil { + return false, fmt.Errorf("the pipeline %s is not written: %w", key, err) + } + ev := seat.PipelineStarted{V: 1, At: &now, Pipeline: key, Branch: b.Branch, Source: b.Source, Module: b.Module, + Commit: b.Commit, Fp: b.Fp, Group: b.Group, Revision: b.Revision} + commitOrGroup := b.Commit + if b.Group != "" { + commitOrGroup = b.Group + } + if err := s.Say(runnerclient.Subject(seat.EventPipelineStarted, b.Module, b.Branch, commitOrGroup), ev); err != nil { + s.logf("pipeline-started of %s is not said: %v", key, err) + } + return true, nil +} + +func (s *Starter) logf(format string, args ...any) { + if s.Logf != nil { + s.Logf(format, args...) + } +} + +// outdatedKind is the edge `outdated` names a dependent behind by for a rebuild on a publish: B published before A builds, +// a package A imports (§53). +const outdatedKind = 1 + +// Published starts, for a package-published, a pipeline for each dependent the catalog's `outdated` names behind for that +// package on that branch by kind 1, keyed by the publication's watched branch and source revision and the dependent +// (design 47 G §53, §334). outdated is the kept call of the verb; view the catalog half of the publication's branch. A +// dependent the half does not hold is said and started with what the event names. +func (s *Starter) Published(ev builds.PackagePublished, outdated func(catalog.OutdatedInput) (catalog.OutdatedAnswer, error), view func(branch string) (Half, bool)) (int, error) { + if ev.Artifact == "" || ev.Branch == "" || ev.Source <= 0 { + return 0, fmt.Errorf("a package-published names its artifact, watched branch and source revision: %+v", ev) + } + a, err := outdated(catalog.OutdatedInput{Branch: ev.Branch, Dependency: ev.Artifact}) + if err != nil { + return 0, fmt.Errorf("outdated of %s on %s: %w", ev.Artifact, ev.Branch, err) + } + half, _ := view(ev.Branch) + started := 0 + var first error + for _, o := range a.Behind { + if o.Kind != outdatedKind || o.Module == "" { + continue + } + b := Begin{Branch: ev.Branch, Source: ev.Source, Module: o.Module, Revision: half.Revision} + if e, ok := half.Entries[o.Module]; ok { + b.Commit, b.Fp, b.Repository = e.Commit, e.Fp, e.Repository + } else { + s.logf("the catalog half of %s holds no entry of %s, behind %s: its pipeline starts without its commit", ev.Branch, o.Module, ev.Artifact) + } + ok, err := s.Start(b) + if err != nil && first == nil { + first = err + } + if ok { + started++ + } + } + return started, first +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go new file mode 100644 index 00000000..da18b6ba --- /dev/null +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go @@ -0,0 +1,87 @@ +package pipeline + +import ( + "encoding/json" + "errors" + "strings" + "testing" + + builds "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" + catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner" +) + +type created map[string][]byte + +func (c created) Create(key string, value []byte) error { + if _, ok := c[key]; ok { + return ErrExists + } + c[key] = value + return nil +} + +// The starts (design 47 G §54, §103, §334): a pipeline's key written by compare-and-set on its absence, so a second start +// of the same source revision and module writes nothing; a publish starts one pipeline per dependent outdated names +// behind by kind 1, keyed by the publishing branch and revision. +func TestStarts(t *testing.T) { + store := created{} + var said []string + s := &Starter{Pipelines: store, Say: func(subject string, body any) error { + ev := body.(seat.PipelineStarted) + said = append(said, subject+" "+ev.Pipeline) + return nil + }} + view := func(branch string) (Half, bool) { + return Half{Revision: 9, Entries: map[string]catalog.CatalogEntry{ + "app": {Module: "app", Commit: "c1", Fp: "src2:aa", Repository: "o/app"}}}, true + } + var asked catalog.OutdatedInput + outdated := func(in catalog.OutdatedInput) (catalog.OutdatedAnswer, error) { + asked = in + return catalog.OutdatedAnswer{Behind: []catalog.Outdated{ + {Module: "app", Kind: 1}, {Module: "tool", Kind: 1}, {Module: "ordered", Kind: 4}}}, nil + } + n, err := s.Published(builds.PackagePublished{Artifact: "mesh-sdk:go", Branch: "main", Source: 41, Module: "mesh-sdk", Version: "0.1.16"}, outdated, view) + if err != nil || n != 2 { + t.Fatalf("a publish started %d (%v), want the two dependents behind by kind 1", n, err) + } + if asked.Dependency != "mesh-sdk:go" || asked.Branch != "main" { + t.Errorf("outdated asked %+v, want the package on its branch", asked) + } + var p seat.Pipeline + if err := json.Unmarshal(store["main.41.app"], &p); err != nil || p.Commit != "c1" || p.Fp != "src2:aa" || p.Revision != 9 || + p.Outcome != seat.PipelineOutcomeRunning || p.Source != 41 { + t.Errorf("the pipeline main.41.app = %+v (%v)", p, err) + } + if _, ok := store["main.41.ordered"]; ok { + t.Error("an ordering edge starts no pipeline") + } + // The take-in of the same source revision and module is the same pipeline: the second start writes nothing. + before := string(store["main.41.app"]) + ok, err := s.Start(Begin{Branch: "main", Source: 41, Module: "app", Commit: "c2"}) + if ok || err != nil || string(store["main.41.app"]) != before || len(said) != 2 { + t.Errorf("a second start wrote %v (%v); said %v", ok, err, said) + } + if !strings.HasPrefix(said[0], "mesh.seat.mesh-pipeline-runner.event.pipeline-started.app.main.c1 main.41.app") { + t.Errorf("pipeline-started said on %q", said[0]) + } + // A branch holding a dot is escaped in the key, so the key reads back; a start naming no source revision is refused. + if _, err := s.Start(Begin{Branch: "release.1", Source: 3, Module: "app"}); err != nil { + t.Fatal(err) + } + if b, r, m, ok := seat.ParsePipelinesByBranchRevisionModuleKey(Key("release.1", 3, "app")); !ok || r != "3" || m != "app" || b == "release" { + t.Errorf("the key of release.1 is %q", Key("release.1", 3, "app")) + } + if _, err := s.Start(Begin{Branch: "main", Module: "app"}); err == nil { + t.Error("a start with no source revision is refused") + } + s.Pipelines = failing{} + if _, err := s.Start(Begin{Branch: "main", Source: 5, Module: "app"}); err == nil { + t.Error("a write the bus refuses is an error") + } +} + +type failing struct{} + +func (failing) Create(string, []byte) error { return errors.New("bus away") } diff --git a/modules/mesh-delivery-pipeline/manifest_test.go b/modules/mesh-delivery-pipeline/manifest_test.go index 59aa7378..c60c345f 100644 --- a/modules/mesh-delivery-pipeline/manifest_test.go +++ b/modules/mesh-delivery-pipeline/manifest_test.go @@ -49,8 +49,13 @@ func TestManifest(t *testing.T) { // §27, §85, §330, §355, §77: the old state sections. {"old state sections", equalSets(states, []string{seat.StatePipelines, seat.StateKept, "aged", "findings", "first-views"})}, // §27, §85: the step log on an old stream, its outputs in an old objects section. - {"old streams section", equalSets(streams, []string{seat.StateStepLog})}, - {"step-log 128 MiB", len(m.Streams) == 1 && m.Streams[0].MaxMiB == 128}, + // RN24 (§383, §387): beside it, the finding stream its findings are said on. + {"old streams section", equalSets(streams, []string{seat.StateStepLog, "mesh-delivery-pipeline-findings"})}, + {"step-log 128 MiB", len(m.Streams) == 2 && m.Streams[0].MaxMiB == 128}, + {"finding stream 7 days 16 MiB", len(m.Streams) == 2 && m.Streams[1].MaxAgeDays == 7 && m.Streams[1].MaxMiB == 16 && + equalSets(m.Streams[1].Events, []string{"finding-written.*", "finding-cleared.*", "findings-stated.*"})}, + // RN24 (§368): the version as MAJOR.MINOR, 0.1 with none published. + {"version 0.1", m.Version == "0.1"}, {"old objects section", equalSets(objects, []string{seat.StateStepOutputs})}, {"step-outputs 1 GiB", len(m.Objects) == 1 && m.Objects[0].MaxMiB == 1024}, // §333, §54, §44, §53, §376, §24, §37, §4, §77: what the runner hears. @@ -58,7 +63,13 @@ func TestManifest(t *testing.T) { "mesh-delivery-indexer.take-in.taken", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "operator-channel.decided", "node-build-go.step-ended", "node-build-node.step-ended", "node-build-image.step-ended", "mesh-wants.step-ended", "mesh-catalog.registration-changed", - "mesh-catalog.catalog-revision"})}, + "mesh-catalog.catalog-revision", "mesh-conditions.findings-asked"})}, + // RN2 (§21, §195, §200, §77): the registrations and the catalog half asked by their verbs, the first views read and + // asked again by hand (a hand act). + {"invokes", equalSets(m.Invokes, []string{"seat:mesh-catalog.registrations", "seat:mesh-catalog.revision", "seat:mesh-hand-acts.hand-act", + // RN3 (§53, §334): a publication's dependents behind. + "seat:mesh-catalog.outdated"})}, + {"first-view tools", equalSets(m.Tools, []string{"pipeline_first_views", "pipeline_first_view_again"})}, // RN1/AC5 (§95, §130): the new assigner is delivered before the runner. {"program needs the assigner's program", slices.Equal(program.Needs, []string{"mesh-delivery-assigner:program"})}, } diff --git a/modules/mesh-delivery-pipeline/module.json b/modules/mesh-delivery-pipeline/module.json index 3a618fe0..d0055720 100644 --- a/modules/mesh-delivery-pipeline/module.json +++ b/modules/mesh-delivery-pipeline/module.json @@ -1,6 +1,6 @@ { "module": "mesh-delivery-pipeline", - "version": "1", + "version": "0.1", "slug": "pipeline", "claims": [ { @@ -22,6 +22,10 @@ ] } ], + "tools": [ + "pipeline_first_views", + "pipeline_first_view_again" + ], "consumes": [ "mesh-delivery-indexer.take-in.taken", "mesh-builds.package-published", @@ -32,7 +36,14 @@ "node-build-image.step-ended", "mesh-wants.step-ended", "mesh-catalog.registration-changed", - "mesh-catalog.catalog-revision" + "mesh-catalog.catalog-revision", + "mesh-conditions.findings-asked" + ], + "invokes": [ + "seat:mesh-catalog.registrations", + "seat:mesh-catalog.revision", + "seat:mesh-hand-acts.hand-act", + "seat:mesh-catalog.outdated" ], "state": [ { @@ -63,6 +74,16 @@ "step-log.**" ], "max-mib": 128 + }, + { + "name": "mesh-delivery-pipeline-findings", + "events": [ + "finding-written.*", + "finding-cleared.*", + "findings-stated.*" + ], + "max-age-days": 7, + "max-mib": 16 } ], "objects": [ diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/catalogread.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/catalogread.go new file mode 100644 index 00000000..97803058 --- /dev/null +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/catalogread.go @@ -0,0 +1,83 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "sort" + "sync" + "time" + + "mesh-delivery-view/internal/view" +) + +// catalogWithin is how long a call waits for the seat mesh-catalog's first view before it says there is none. +var catalogWithin = 30 * time.Second + +// catalogRead is the modules the view's tools answer, read from the catalog half through the seat mesh-catalog's verb +// `revision` (design 47, Two graphs): the trunk's newest revision, asked once per call and in pages, each entry in the +// form of the value the tools always answered. With no view the read says so; it never answers no modules. +func catalogRead(ask func(key string, body any) (json.RawMessage, error)) view.StateRead { + var ( + once sync.Once + values map[string]json.RawMessage + rev uint64 + failed error + ) + load := func() { + ctx, cancel := context.WithTimeout(context.Background(), catalogWithin) + defer cancel() + c, err := meshcatalogclient.NewCatalog(ask, "main") + if err != nil { + failed = err + return + } + if err := c.Start(ctx); err != nil { + failed = fmt.Errorf("the catalog half has no first view yet (%s): %w", meshcatalogclient.RevisionVerb, err) + return + } + half, ok := c.View() + if !ok { + failed = fmt.Errorf("the catalog half has no first view yet (%s)", meshcatalogclient.RevisionVerb) + return + } + rev, values = uint64(half.Revision.Revision), map[string]json.RawMessage{} + for module, e := range half.Entries { + raw, _ := json.Marshal(e) + v, err := view.ModuleValue(raw) + if err != nil { + failed = fmt.Errorf("the catalog entry of %s: %w", module, err) + return + } + values[module] = v + } + } + read := func() (map[string]json.RawMessage, uint64, error) { + once.Do(load) + return values, rev, failed + } + return view.StateRead{ + Keys: func() ([]string, error) { + v, _, err := read() + keys := make([]string, 0, len(v)) + for k := range v { + keys = append(keys, k) + } + sort.Strings(keys) + return keys, err + }, + Get: func(key string) (json.RawMessage, error) { + v, _, err := read() + return v[key], err + }, + GetEntry: func(key string) (json.RawMessage, uint64, error) { + v, r, err := read() + return v[key], r, err + }, + All: func() (map[string]json.RawMessage, error) { + v, _, err := read() + return v, err + }, + } +} diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go index 1a8f828a..9baa4b52 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go @@ -396,7 +396,7 @@ func ownersRead(ask func(key string, body any) (json.RawMessage, error)) view.Re // Each owner's read tool as the view's invokes names it. indexer := func(tool string) view.StateRead { return view.OwnerRead(ask, view.IndexerModule+"."+tool) } assigner := func(tool string) view.StateRead { return view.OwnerRead(ask, view.AssignerModule+"."+tool) } - return view.Reader{State: indexer("indexer_read_modules"), CandidatesState: indexer("indexer_read_candidates"), + return view.Reader{State: catalogRead(ask), CandidatesState: indexer("indexer_read_candidates"), WantsState: assigner("assigner_read_wants"), DifferencesState: assigner("assigner_read_differences"), HeardState: indexer("indexer_read_heard"), PredefinedState: indexer("indexer_read_predefined"), diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/owners_test.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/owners_test.go index 80ad8589..8864369f 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/owners_test.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/owners_test.go @@ -3,12 +3,14 @@ package main import ( "encoding/json" "fmt" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" "mesh-delivery-view/internal/view" "sort" "strings" "testing" "time" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/sdk/graph" "git.novox.be/novox/mesh-sdk/go/views" ) @@ -51,7 +53,6 @@ func ownersAnswerTheToolsAsBefore(t *testing.T) { t.Fatal(err) } states := map[string]source{ - view.IndexerModule + ".modules": {"a": withFP2, "b": withNone}, view.IndexerModule + ".candidates": {"feat/x": cand, "fix": `{"v":1,"group":"fix","base":3,"heads":"h","pulls":[{"modules":["netcheck"]}]}`}, view.AssignerModule + ".wants": {"want.ace.zsh": `{"fp":"` + fp2A + `"}`, "want.novox.zsh": `{"fp":"` + fp2B + `"}`, "held." + keyB: `{"node":"ace","words":"failed"}`}, @@ -67,14 +68,12 @@ func ownersAnswerTheToolsAsBefore(t *testing.T) { const heard1 = `{"v":1,"subject":"s","sequence":41,"repository":"novox/mesh-catalog","commit":"9cdac09a"}` states[view.IndexerModule+".heard"] = source{"forge": heard1} histories := map[string]map[string][]views.Revision{ - view.IndexerModule + ".modules": {"a": {{Revision: 7, At: at, Op: "put", Value: json.RawMessage(withNone)}, - {Revision: 9, At: at.Add(time.Minute), Op: "put", Value: json.RawMessage(withFP2)}}}, view.IndexerModule + ".heard": {"forge": {{Revision: 3, At: at, Op: "put", Value: json.RawMessage(heard1)}}}, view.AssignerModule + ".wants": {"want.ace.zsh": {{Revision: 12, At: at, Op: "put", Value: json.RawMessage(`{"fp":"` + fp2A + `"}`)}}}, } // The direct reading, as the tools read the states before (the reads the manifest no longer has). st := func(name string) view.StateRead { return fake(states[name]).reader().State } - direct := view.Reader{State: st(view.IndexerModule + ".modules"), CandidatesState: st(view.IndexerModule + ".candidates"), + direct := view.Reader{CandidatesState: st(view.IndexerModule + ".candidates"), WantsState: st(view.AssignerModule + ".wants"), DifferencesState: st(view.AssignerModule + ".differences"), PredefinedState: st(view.IndexerModule + "." + view.IndexerPredefinedState), ConfigCandidatesState: st(view.IndexerModule + "." + view.IndexerConfigCandidatesState), @@ -89,6 +88,10 @@ func ownersAnswerTheToolsAsBefore(t *testing.T) { } asked := map[string]int{} ask := func(key string, body any) (json.RawMessage, error) { + if key == meshcatalogclient.RevisionVerb { + asked[key]++ + return json.Marshal(catalogAnswer) + } name, ok := tools[key] if !ok { return nil, fmt.Errorf("no tool %s", key) @@ -118,10 +121,6 @@ func ownersAnswerTheToolsAsBefore(t *testing.T) { tool string args map[string]any }{ - {view.ModuleReadTool, map[string]any{"module": "a"}}, - {view.ModuleReadTool, map[string]any{"module": "gone"}}, - {view.ModulesReadTool, map[string]any{}}, - {view.ModulesReadTool, map[string]any{"none2": true}}, {view.CandidateReadTool, map[string]any{"group": "feat/x"}}, {view.CandidatesReadTool, map[string]any{}}, {view.CandidatesReadTool, map[string]any{"module": "zsh"}}, @@ -143,12 +142,24 @@ func ownersAnswerTheToolsAsBefore(t *testing.T) { t.Errorf("%s %v through the owner's read tool = %s, %v; reading the state it was %s, %v", row.tool, row.args, gj, gerr, wj, werr) } } - // A key's history is asked of the owner's read tool and answered from its kept values (A2j finding 1, J4). - out, err := run(t, owned, view.ModuleHistoryTool, map[string]any{"module": "a"}) - if h, _ := out.(view.ModuleHistory); err != nil || len(h.Values) != 2 || h.Values[0].Revision != 9 || h.Values[1].FP2 != "" || !h.Values[1].At.Equal(at) { - t.Errorf("indexer_module_history a through the indexer's read tool = %+v, %v", out, err) + // The modules are the catalog half's, asked of the seat mesh-catalog's verb revision (W16.1): a module is its entry in + // the form of the value the tool always answered, the modules without a fingerprint say why, and the module's history + // is not a thing the catalog half keeps. + a, err := run(t, owned, view.ModuleReadTool, map[string]any{"module": "a"}) + if raw, _ := json.Marshal(a); err != nil || !strings.Contains(string(raw), `"fp2":"`+fp2A+`"`) || !strings.Contains(string(raw), `"commit":"c1"`) || !strings.Contains(string(raw), `"src":7`) { + t.Errorf("indexer_module a = %s, %v", raw, err) } - out, err = run(t, owned, view.HeardTool, map[string]any{}) + if _, err := run(t, owned, view.ModuleReadTool, map[string]any{"module": "gone"}); err == nil { + t.Errorf("indexer_module gone answered a module the catalog half does not hold") + } + all, err := run(t, owned, view.ModulesReadTool, map[string]any{"none2": true}) + if raw, _ := json.Marshal(all); err != nil || !strings.Contains(string(raw), `"b"`) || strings.Contains(string(raw), `"a"`) || !strings.Contains(string(raw), "builder's fingerprint not yet known") { + t.Errorf("indexer_modules none2 = %s, %v", raw, err) + } + if _, err := run(t, owned, view.ModuleHistoryTool, map[string]any{"module": "a"}); err == nil { + t.Errorf("indexer_module_history answered a history the catalog half does not keep") + } + out, err := run(t, owned, view.HeardTool, map[string]any{}) if rows, _ := out.(view.HeardAnswer); err != nil || len(rows.Heard) != 1 || rows.Heard[0].Revision != 3 || string(rows.Heard[0].Heard) != heard1 { t.Errorf("indexer_heard through the indexer's read tool = %+v, %v", out, err) } @@ -156,10 +167,29 @@ func ownersAnswerTheToolsAsBefore(t *testing.T) { if revs, err := owned.WantsState.History("want.ace.zsh"); err != nil || len(revs) != 1 || !revs[0].At.Equal(at) || revs[0].Revision != 12 { t.Errorf("the want's history through the assigner's read tool = %+v, %v", revs, err) } + // With no view of the catalog half the modules are not read: the tools say so, never that no module is held. + catalogWithin = 20 * time.Millisecond + none := ownersRead(func(key string, body any) (json.RawMessage, error) { return nil, fmt.Errorf("the holder is away") }) + if out, err := run(t, none, view.ModulesReadTool, map[string]any{}); err == nil || !strings.Contains(err.Error(), "no first view") { + t.Errorf("indexer_modules with no view = %v, %v, want an error naming it", out, err) + } // Each owner state the tools read was asked of its owner's read tool. for tool, name := range tools { if name != view.IndexerModule+"."+graph.State && asked[tool] == 0 { t.Errorf("%s was never asked", tool) } } + if asked[meshcatalogclient.RevisionVerb] == 0 { + t.Errorf("%s was never asked", meshcatalogclient.RevisionVerb) + } +} + +// catalogAnswer is the seat mesh-catalog's newest trunk revision: a with a fingerprint and the source revision it moved at, +// b with none and the reason. +var catalogAnswer = meshcatalog.RevisionAnswer{ + Revision: meshcatalog.CatalogRevision{Revision: 4, Kind: meshcatalogclient.KindTrunk, Source: 7}, + Entries: []meshcatalog.CatalogEntry{ + {Module: "a", Repository: "novox/r", Commit: "c1", Path: ".", Manifest: map[string]any{"module": "a"}, Fp: fp2A, Inputs: &meshcatalog.EntryInputs{Src: 7}}, + {Module: "b", Repository: "novox/r", Commit: "c2", Path: "modules/b", Manifest: map[string]any{"module": "b"}, None: "builder's fingerprint not yet known"}, + }, } diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view/integration_test.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view/integration_test.go index 77ad3d76..14248181 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view/integration_test.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view/integration_test.go @@ -12,6 +12,7 @@ import ( "path/filepath" "slices" "strings" + "sync" "testing" "time" @@ -47,6 +48,67 @@ func serveOwners(t *testing.T, ctx context.Context, js jetstream.JetStream, nc * } } +// catalogHolder is the seat mesh-catalog's holder as the view's process needs it (W16.1): its verb revision answering the +// trunk's newest revision whole, and its event catalog-revision said on the shared events stream at each change, which the +// view's durable consumer hears. +type catalogHolder struct { + t testing.TB + js jetstream.JetStream + mu sync.Mutex + // rev is the revision's number; entries are its entries by module. + rev int64 + entries map[string]meshcatalog.CatalogEntry + ctx context.Context +} + +func serveCatalog(t testing.TB, ctx context.Context, js jetstream.JetStream, nc *nats.Conn) *catalogHolder { + t.Helper() + h := &catalogHolder{t: t, js: js, ctx: ctx, rev: 1, entries: map[string]meshcatalog.CatalogEntry{}} + events, err := js.Stream(ctx, process.EventsStream) + if err != nil { + t.Fatal(err) + } + info := events.CachedInfo().Config + if !slices.Contains(info.Subjects, "mesh.seat.*.event.>") { + info.Subjects = append(info.Subjects, "mesh.seat.*.event.>") + if _, err := js.UpdateStream(ctx, info); err != nil { + t.Fatal(err) + } + } + answer := func(m *nats.Msg) { + h.mu.Lock() + a := meshcatalog.RevisionAnswer{Revision: meshcatalog.CatalogRevision{Revision: h.rev, Kind: meshcatalogclient.KindTrunk}} + for _, e := range h.entries { + a.Entries = append(a.Entries, e) + } + h.mu.Unlock() + raw, _ := json.Marshal(map[string]any{"result": a, "node": "n0"}) + _ = m.Respond(raw) + } + for _, subject := range []string{"mesh.seat.mesh-catalog.call.revision.>", "mesh.seat.mesh-catalog.call.revision"} { + sub, err := nc.Subscribe(subject, answer) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = sub.Unsubscribe() }) + } + return h +} + +// put is the holder's next trunk revision with the entry of one module changed, said as catalog-revision. +func (h *catalogHolder) put(e meshcatalog.CatalogEntry) { + h.t.Helper() + h.mu.Lock() + h.rev++ + h.entries[e.Module] = e + rev := h.rev + h.mu.Unlock() + body, _ := json.Marshal(meshcatalog.CatalogRevisionEvent{Kind: meshcatalogclient.KindTrunk, Branch: "main", Revision: rev, Changed: []string{e.Module}}) + if _, err := h.js.Publish(h.ctx, "mesh.seat.mesh-catalog.event."+meshcatalog.EventCatalogRevision, body); err != nil { + h.t.Fatal(err) + } +} + // TestProcessWholeSeesTheBuckets starts the program the way the node does (its entry, run, with its environment and a // broker file) against an embedded bus holding the three buckets, and asks its socket. func TestProcessWholeSeesTheBuckets(t *testing.T) { // 1134/AC8 @@ -126,6 +188,7 @@ func TestProcessWholeSeesTheBuckets(t *testing.T) { // 1134/AC8 runCtx, stop := context.WithCancel(ctx) done := make(chan error, 1) serveOwners(t, ctx, js, nc) + serveCatalog(t, ctx, js, nc) go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() t.Cleanup(func() { stop(); <-done }) @@ -226,6 +289,7 @@ func TestProcessWholeShowsWhatTheWatchmanSays(t *testing.T) { // 1168/AC5 runCtx, stop := context.WithCancel(ctx) done := make(chan error, 1) serveOwners(t, ctx, js, nc) + serveCatalog(t, ctx, js, nc) go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() t.Cleanup(func() { stop(); <-done }) @@ -293,6 +357,7 @@ func TestProcessWholeShowsTheIndexersBucketsOnceTheyExist(t *testing.T) { // 124 runCtx, stop := context.WithCancel(ctx) done := make(chan error, 1) serveOwners(t, ctx, js, nc) + catalog := serveCatalog(t, ctx, js, nc) go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() t.Cleanup(func() { stop(); <-done }) @@ -306,26 +371,16 @@ func TestProcessWholeShowsTheIndexersBucketsOnceTheyExist(t *testing.T) { // 124 if err != nil { t.Fatal(err) } - mods, err := js.KeyValue(ctx, view.ModulesBucket) - if err != nil { - t.Fatal(err) - } const head = "0123456789abcdef0123456789abcdef01234567" hash, _ := meshdeliveryclient.HeadsHash([]meshdeliveryclient.Head{{Repository: "novox/mesh-delivery", Number: 12, Head: head}}) cand, err := json.Marshal(meshcatalog.CandidatesByGroupValue{Group: "feat/x", Heads: hash, Pulls: []meshcatalog.CandidatesByGroupValuePullsItem{{Repository: "novox/mesh-delivery", Number: 12, Head: head, Modules: []string{"a"}}}}) if err != nil { t.Fatal(err) } - mod, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{None: "no tree", Repository: "novox/mesh-delivery", Commit: head}) - if err != nil { - t.Fatal(err) - } if _, err := cands.Put(ctx, "feat/x", cand); err != nil { t.Fatal(err) } - if _, err := mods.Put(ctx, "a", mod); err != nil { - t.Fatal(err) - } + catalog.put(meshcatalog.CatalogEntry{Module: "a", None: "no tree", Repository: "novox/mesh-delivery", Commit: head, Manifest: map[string]any{"module": "a"}}) var last view.Snapshot for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) { if s, err := view.Ask(socket); err == nil { @@ -380,6 +435,7 @@ func TestProcessWholeShowsTheOpenAsksAndDropsTheAnsweredOnes(t *testing.T) { // runCtx, stop := context.WithCancel(ctx) done := make(chan error, 1) serveOwners(t, ctx, js, nc) + serveCatalog(t, ctx, js, nc) go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() t.Cleanup(func() { stop(); <-done }) @@ -471,6 +527,7 @@ func TestProcessWholeListsEveryDifference(t *testing.T) { // 1297/B8 AC runCtx, stop := context.WithCancel(ctx) done := make(chan error, 1) serveOwners(t, ctx, js, nc) + serveCatalog(t, ctx, js, nc) go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() t.Cleanup(func() { stop(); <-done }) @@ -569,6 +626,7 @@ func TestProcessWholeWritesTheAnswerToAnAskNeedingNoProof(t *testing.T) { // 129 runCtx, stop := context.WithCancel(ctx) done := make(chan error, 1) serveOwners(t, ctx, js, nc) + serveCatalog(t, ctx, js, nc) go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() t.Cleanup(func() { stop(); <-done }) @@ -721,6 +779,7 @@ func TestProcessWholeShowsTheBacklogOfBuildAsks(t *testing.T) { // 1887/A8a AC runCtx, stop := context.WithCancel(ctx) done := make(chan error, 1) serveOwners(t, ctx, js, nc) + serveCatalog(t, ctx, js, nc) go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() t.Cleanup(func() { stop(); <-done }) @@ -768,7 +827,7 @@ func TestProcessWholeSaysTheSourceRevisionIsDelivered(t *testing.T) { t.Fatal(err) } kvs := map[string]jetstream.KeyValue{} - for _, name := range []string{view.SourceBucket, view.ModulesBucket, view.AssignerWantsBucket, view.HaveBucket} { + for _, name := range []string{view.SourceBucket, view.AssignerWantsBucket, view.HaveBucket} { kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: 5}) if err != nil { t.Fatal(err) @@ -789,20 +848,16 @@ func TestProcessWholeSaysTheSourceRevisionIsDelivered(t *testing.T) { runCtx, stop := context.WithCancel(ctx) done := make(chan error, 1) serveOwners(t, ctx, js, nc) + catalog := serveCatalog(t, ctx, js, nc) go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() t.Cleanup(func() { stop(); <-done }) fp := "src2:" + strings.Repeat("2", 64) - mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: strings.Repeat("a", 64), Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath}) want, _ := meshwantsclient.EncodeWant(meshwantsclient.Want{Fp: fp, Composed: "sha256:" + strings.Repeat("c", 64)}) failed, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed}) good, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}) src, _ := meshdeliveryclient.EncodeSource(meshdelivery.SourceByBranchValue{Group: "feat/x", Heads: map[string]string{"novox/web": strings.Repeat("1", 40)}}) - for bucket, kv := range map[string][]byte{view.ModulesBucket: mod} { - if _, err := kvs[bucket].Put(ctx, "web", kv); err != nil { - t.Fatal(err) - } - } + catalog.put(meshcatalog.CatalogEntry{Module: "web", Fp: fp, Manifest: map[string]any{"module": "a"}, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath}) if _, err := kvs[view.AssignerWantsBucket].Put(ctx, "want.ace.web", want); err != nil { t.Fatal(err) } @@ -886,6 +941,7 @@ func TestProcessWholeSaysModulesInMotion(t *testing.T) { runCtx, stop := context.WithCancel(ctx) done := make(chan error, 1) serveOwners(t, ctx, js, nc) + serveCatalog(t, ctx, js, nc) go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() t.Cleanup(func() { stop(); <-done }) time.Sleep(2 * time.Second) // the view has read what stood: nothing diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go index 471875c1..ecd9dcd5 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go @@ -11,6 +11,7 @@ import ( "encoding/json" "errors" "fmt" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" "io" "log" "mesh-delivery-view/internal/view" @@ -126,15 +127,21 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } } open := buckets{js: js, reader: reader} - for _, bucket := range []string{view.HaveBucket, view.AssignmentsBucket, view.PresenceBucket, view.StuckBucket, view.CandidatesBucket, view.ModulesBucket, view.AsksBucket, view.AnswersBucket, builds.BuilderBuildsBucket, view.AssignerWantsBucket, view.SourceBucket, view.AssignerAsksBucket, viewstate.ViewAnswersBucket, view.TakenBucket} { + for _, bucket := range []string{view.HaveBucket, view.AssignmentsBucket, view.PresenceBucket, view.StuckBucket, view.CandidatesBucket, view.AsksBucket, view.AnswersBucket, builds.BuilderBuildsBucket, view.AssignerWantsBucket, view.SourceBucket, view.AssignerAsksBucket, viewstate.ViewAnswersBucket, view.TakenBucket} { go follow(ctx, open, bucket, false, model, logger) } - // The modules' and the source's history give each fingerprint's first commit and each head's source revision (ADR 0556). - // The modules' picture holds no history: from its first view on, each change it takes is the history (finding of DLV-M4). - for _, bucket := range []string{view.ModulesBucket, view.SourceBucket} { - go follow(ctx, open, bucket, true, model, logger) + // The source's history gives each head's source revision (ADR 0556); the modules' history is each revision of the + // catalog half the view takes, in its order (below). + go follow(ctx, open, view.SourceBucket, true, model, logger) + statusBucket := process.AwaitBucket(ctx, js, conn, firstViewsBucket, "view_first_views reads nothing and view_first_view_again is not heard", logger.Printf) + // The modules are the catalog half's, adopted whole by the seat's client: its first view, its status, its hand asks and + // its event catalog-revision (on the module's durable consumer) are the reader's. + catalog, _, err := meshcatalogclient.WireCatalogProcess(reader, calls.Call, "main", catalogChanged(model, logger)) + if err != nil { + return err } - reader.Start(ctx, js, cred, process.AwaitBucket(ctx, js, conn, firstViewsBucket, "view_first_views reads nothing and view_first_view_again is not heard", logger.Printf)) + reader.Start(ctx, js, cred, statusBucket) + catalogHeld(ctx, catalog, model) select { case <-ctx.Done(): case err := <-webUp: @@ -189,7 +196,6 @@ var ownersFollowed = []struct{ owner, state, tool string }{ {view.AssignerModule, "wants", view.AssignerModule + ".assigner_read_wants"}, {view.AssignerModule, "asks", view.AssignerModule + ".assigner_read_asks"}, {view.IndexerModule, "candidates", view.IndexerModule + ".indexer_read_candidates"}, - {view.IndexerModule, "modules", view.IndexerModule + ".indexer_read_modules"}, {view.IndexerModule, "taken", view.IndexerModule + ".indexer_read_taken"}, } @@ -277,3 +283,35 @@ func emitEvent(ctx context.Context, js jetstream.JetStream, cred process.Account _, err = js.PublishMsg(pub, msg, jetstream.WithMsgID(id)) return err } + +// catalogChanged hands one module of a revision the view took to the model, in the form of its value; the modules' +// history is these changes in the order of the revisions. +func catalogChanged(model *view.Model, logger *log.Logger) func(string, json.RawMessage, bool, time.Time) { + return func(module string, entry json.RawMessage, gone bool, at time.Time) { + e := view.Entry{Bucket: view.ModulesBucket, Key: module, At: at, Deleted: gone} + if !gone { + v, err := view.ModuleValue(entry) + if err != nil { + logger.Printf("the catalog entry of %s is not taken: %v", module, err) + return + } + e.Value = v + } + model.Apply(e) + e.History = true + model.Apply(e) + } +} + +// catalogHeld makes the model's pictures of the modules initial once the catalog half's first view is held: the process +// starts at once and waits for the seat's holder, saying so. +func catalogHeld(ctx context.Context, cat *meshcatalogclient.Catalog, model *view.Model) { + go func() { + select { + case <-cat.Follower.Ready(): + model.Initial(view.ModulesBucket) + model.Initial(view.ReachHistory(view.ModulesBucket)) + case <-ctx.Done(): + } + }() +} diff --git a/modules/mesh-delivery-view/internal/view/catalog.go b/modules/mesh-delivery-view/internal/view/catalog.go new file mode 100644 index 00000000..385de245 --- /dev/null +++ b/modules/mesh-delivery-view/internal/view/catalog.go @@ -0,0 +1,58 @@ +package view + +// The view reads the catalog half of the delivery through the seat mesh-catalog (design 47, Two graphs; ADR 0656 §6): +// a first view of its verb `revision`, then each trunk revision its event `catalog-revision` says, never the indexer's +// state `modules`. The model keeps the picture it always kept (one value per module, its history for the reach), so each +// entry of a revision is handed to it in the form of that value. + +import ( + "encoding/json" + "fmt" + + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" +) + +// CatalogSeat is the seat whose verb `revision` and event `catalog-revision` the view reads the modules from, as its +// manifest's `invokes` and `consumes` name them. +const CatalogSeat = meshcatalog.Seat + +// ModulesBucket names the model's picture of the catalog half's modules (and its history): the model's label, no +// bucket of anybody's. +const ModulesBucket = CatalogSeat + "_modules" + +// ModuleValue is the value of one module of the model, as the model decodes it, made from the catalog entry (its JSON): +// the fingerprint is the entry's `fp` (the only form there is), the source revisions are the entry's inputs'. A manifest +// held for unknown fields has no verdict here: the model reads the fingerprint and the commit. +func ModuleValue(entry json.RawMessage) (json.RawMessage, error) { + var e meshcatalog.CatalogEntry + if err := json.Unmarshal(entry, &e); err != nil || e.Module == "" { + return nil, fmt.Errorf("a catalog entry does not read: %v", err) + } + v := meshcatalog.ModulesByModuleValue{Commit: e.Commit, Fp2: e.Fp, None2: e.None, Path: e.Path, Repository: e.Repository, Packages: e.Packages} + if len(e.Manifest) > 0 { + // The entry holds its manifest whole; the model keeps the digest of its bytes, as the value it always kept does. + raw, _ := json.Marshal(e.Manifest) + v.Manifest = meshcatalogclient.ManifestDigest(raw) + } + if e.Verdict == meshcatalogclient.VerdictAccepted || e.Verdict == meshcatalogclient.VerdictRefused { + v.Verdict = e.Verdict + } + if in := e.Inputs; in != nil { + v.Src, v.FpSrc, v.Toolchains, v.Paths, v.Warnings = in.Src, in.FpSrc, in.Toolchains, in.Paths, in.Warnings + if f := in.From; f != nil && v.Manifest != "" { + v.Inputs2 = &meshcatalog.ModulesByModuleValueInputs2{Bases: f.Bases, Builder: f.Builder, Contexts: f.Contexts, + ImportForm: f.ImportForm, Imports: f.Imports, Locks: f.Locks, Packages: f.Packages, Paths: f.Paths, Tree: f.Tree} + } + if v.Verdict == meshcatalogclient.VerdictRefused { + v.Problems = in.Problems + } + } + if v.Verdict == meshcatalogclient.VerdictRefused && len(v.Problems) == 0 { + v.Verdict = "" + } + if v.Verdict != meshcatalogclient.VerdictRefused { + v.Warnings = nil + } + return meshcatalogclient.EncodeModule(v) +} diff --git a/modules/mesh-delivery-view/internal/view/names.go b/modules/mesh-delivery-view/internal/view/names.go index 4a2d62f1..24e69c85 100644 --- a/modules/mesh-delivery-view/internal/view/names.go +++ b/modules/mesh-delivery-view/internal/view/names.go @@ -24,7 +24,6 @@ const ( EngineModule = "node-engine" // The indexer's states the view reads. - ModulesBucket = IndexerModule + "_modules" CandidatesBucket = IndexerModule + "_candidates" TakenBucket = IndexerModule + "_taken" PackagesState = "packages" diff --git a/modules/mesh-delivery-view/manifest_test.go b/modules/mesh-delivery-view/manifest_test.go index b35cf6f8..693fe5d5 100644 --- a/modules/mesh-delivery-view/manifest_test.go +++ b/modules/mesh-delivery-view/manifest_test.go @@ -3,6 +3,7 @@ package manifest import ( "bytes" "encoding/json" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" "io/fs" "mesh-delivery-view/internal/view" "os" @@ -119,8 +120,9 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7 // follows then heard on their views families (ADR 0615 §1, §5; DLV-M4). if want := []string{"mesh-delivery-assigner.assigner_read_asks", "mesh-delivery-assigner.assigner_read_differences", "mesh-delivery-assigner.assigner_read_assignments", "mesh-delivery-assigner.assigner_read_wants", "mesh-delivery-indexer.indexer_read_candidates", "mesh-delivery-indexer.indexer_read_config_candidates", - "mesh-delivery-indexer.indexer_read_graph", "mesh-delivery-indexer.indexer_read_heard", "mesh-delivery-indexer.indexer_pending", "mesh-delivery-indexer.indexer_read_modules", + "mesh-delivery-indexer.indexer_read_graph", "mesh-delivery-indexer.indexer_read_heard", "mesh-delivery-indexer.indexer_pending", "mesh-delivery-indexer.indexer_read_packages", "mesh-delivery-indexer.indexer_read_predefined", "mesh-delivery-indexer.indexer_read_taken", + "seat:mesh-catalog.revision", // ADR 0663: the modules are the catalog half's, read through the seat's verb revision, never the indexer's "seat:mesh-hand-acts.hand-act", // the join: its first view asked again by hand is a hand act (views.StatusTools) "seat:node-engine.manifest-reads"}; !reflect.DeepEqual(m.Invokes, want) { // ADR 0618 §3: node_manifest_reads asks the node-engine seat's verb of every node; 1585/T24: view_tiers asks indexer_pending, a verb edge (ADR 0614 §2) t.Errorf("invokes = %v, want %v", m.Invokes, want) @@ -133,7 +135,7 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7 t.Errorf("version = %q, want 0.1", m.Version) } if want := []string{"mesh-conditions.findings-asked", "mesh-delivery-assigner.views.asks", "mesh-delivery-assigner.views.assignments", "mesh-delivery-assigner.views.wants", - "mesh-delivery-indexer.views.candidates", "mesh-delivery-indexer.views.modules", "mesh-delivery-indexer.views.taken"}; !reflect.DeepEqual(m.Consumes, want) { + "mesh-catalog.catalog-revision", "mesh-delivery-indexer.views.candidates", "mesh-delivery-indexer.views.taken"}; !reflect.DeepEqual(m.Consumes, want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } // The one event the view says: a source revision is delivered (1906/D9, ADR 0539 §17), derived by it from the wants and the haves. @@ -180,7 +182,7 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7 if !slices.Contains(m.Reads, view.SignersRead) { t.Errorf("the view reads %s, its manifest's reads are %v", view.SignersRead, m.Reads) } - followed := map[string]string{view.ModulesBucket: view.IndexerModule + ".views.modules", view.CandidatesBucket: view.IndexerModule + ".views.candidates", + followed := map[string]string{view.CandidatesBucket: view.IndexerModule + ".views.candidates", view.TakenBucket: view.IndexerModule + ".views.taken", view.AssignmentsBucket: view.AssignerModule + ".views.assignments", view.AssignerWantsBucket: view.AssignerModule + ".views.wants", view.AssignerAsksBucket: view.AssignerModule + ".views.asks"} for bucket, family := range followed { @@ -191,11 +193,20 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7 t.Errorf("the view follows %s, its manifest consumes %v", family, m.Consumes) } } - for _, tool := range []string{view.IndexerModule + ".indexer_read_modules", view.IndexerModule + ".indexer_read_candidates", view.IndexerModule + ".indexer_pending", view.AssignerModule + ".assigner_read_wants"} { + for _, tool := range []string{meshcatalogclient.RevisionVerb, view.IndexerModule + ".indexer_read_candidates", view.IndexerModule + ".indexer_pending", view.AssignerModule + ".assigner_read_wants"} { if !slices.Contains(m.Invokes, tool) { t.Errorf("the view asks %s, its manifest invokes %v", tool, m.Invokes) } } + // The modules are the catalog half's: the seat's verb is invoked, its event consumed, and the indexer's state modules is neither. + if !slices.Contains(m.Consumes, view.CatalogSeat+"."+meshcatalog.EventCatalogRevision) { + t.Errorf("the view hears the seat's %s, its manifest consumes %v", meshcatalog.EventCatalogRevision, m.Consumes) + } + for _, name := range append(slices.Clone(m.Invokes), m.Consumes...) { + if strings.Contains(name, "indexer_read_modules") || strings.Contains(name, "views.modules") { + t.Errorf("the manifest names %s: the modules are the catalog half's (ADR 0663)", name) + } + } if viewstate.ViewModule != m.Module || viewstate.ViewAnswersBucket != m.Module+"_answers" || viewstate.ReachBucket != m.Module+"_reach" { t.Errorf("the view names itself %s (%s, %s), its manifest %s", viewstate.ViewModule, viewstate.ViewAnswersBucket, viewstate.ReachBucket, m.Module) } diff --git a/modules/mesh-delivery-view/module.json b/modules/mesh-delivery-view/module.json index a8544e42..9e727992 100644 --- a/modules/mesh-delivery-view/module.json +++ b/modules/mesh-delivery-view/module.json @@ -46,8 +46,8 @@ "mesh-delivery-assigner.views.asks", "mesh-delivery-assigner.views.assignments", "mesh-delivery-assigner.views.wants", + "mesh-catalog.catalog-revision", "mesh-delivery-indexer.views.candidates", - "mesh-delivery-indexer.views.modules", "mesh-delivery-indexer.views.taken" ], "streams": [ @@ -68,10 +68,10 @@ "mesh-delivery-indexer.indexer_read_graph", "mesh-delivery-indexer.indexer_read_heard", "mesh-delivery-indexer.indexer_pending", - "mesh-delivery-indexer.indexer_read_modules", "mesh-delivery-indexer.indexer_read_packages", "mesh-delivery-indexer.indexer_read_predefined", "mesh-delivery-indexer.indexer_read_taken", + "seat:mesh-catalog.revision", "seat:mesh-hand-acts.hand-act", "seat:node-engine.manifest-reads" ],