diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index 2bfff267..2f50c05a 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -473,13 +473,15 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // modules again (design 47 G A-82, §204). // The runner's build step ask, the builder's one entry point for a candidate (design 47 G A-97). b.StepEntry = newEntryWaits(branches, half.manifests).entry - // The seat mesh-builds' verb reserved, answered here (A-115, A-127): the reservations, the catalog half's carriers and the seat - // mesh-catalog's versions. + // The seat mesh-builds' verbs reserved, answered here (A-115, A-127): the reservations, the catalog half's carriers and the seat + // mesh-catalog's versions; and published, from the reservations and releases (H-A-328). stopReserved, err := calls.AnswerSeat(meshbuilds.Seat, seatlist.Definition(seatGate.Reader, meshbuilds.Seat), map[string]process.SeatHandler{ meshbuilds.VerbReserved: reservedVerb(ctx, reservedReader(ctx, reserved.r.Store), versions.CarriersFrom(versions.EntriesOf(branches)), catalogReleased(calls.Call), func(name string) { logger.Printf("reserved: no entry carries the name %s; the entries looked at: %s", name, versions.LookedAt(versions.EntriesOf(branches))) - })}) + }), + // published (H-A-328): whether a package built at a fingerprint is published, asked by the runner writing a wait on it. + meshbuilds.VerbPublished: publishedVerb(ctx, reserved.r.PublishedOf)}) if err != nil { return fmt.Errorf("answering the verb reserved: %w", err) } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reservedverb.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reservedverb.go index b45a2905..eca2412a 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reservedverb.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reservedverb.go @@ -49,6 +49,18 @@ func reservedVerb(ctx context.Context, rr versions.Reader, carriers func(context } } +// publishedVerb is the handler of the verb published (design 47 H-A-328): whether an artifact built at a fingerprint is +// published, from the reservations and the releases the builder keeps. +func publishedVerb(ctx context.Context, of func(ctx context.Context, module, artifact, fingerprint string) (meshbuilds.PublishedAnswer, error)) process.SeatHandler { + return func(_ string, body json.RawMessage) (any, error) { + var in meshbuilds.PublishedInput + if err := json.Unmarshal(body, &in); err != nil { + return nil, fmt.Errorf("the arguments do not fit the verb published: %w", err) + } + return of(ctx, in.Module, in.Artifact, in.Fingerprint) + } +} + // catalogReleased are a module's released versions, asked of the seat mesh-catalog's verb versions through call. func catalogReleased(call toolCaller) versions.CatalogVersions { return func(_ context.Context, module string) ([]meshcatalog.VersionEntry, error) { diff --git a/modules/mesh-delivery-builder/internal/versions/reserve.go b/modules/mesh-delivery-builder/internal/versions/reserve.go index 8b819733..0c07361d 100644 --- a/modules/mesh-delivery-builder/internal/versions/reserve.go +++ b/modules/mesh-delivery-builder/internal/versions/reserve.go @@ -54,7 +54,10 @@ type ReservedVersion struct { BuiltAt time.Time `json:"built-at,omitzero"` // Published are the artifacts a registry seat said published at this version with the built digest. Published map[string]bool `json:"published,omitempty"` - At time.Time `json:"at"` + // PublishedAt is when the builder heard or confirmed each artifact's publish (H-A-331); absent for a mark written before it + // was kept. + PublishedAt map[string]time.Time `json:"published-at,omitempty"` + At time.Time `json:"at"` // Repository and Commit are what the build was made from, and Tag the git tag its release is written as (a Go // package's, GoTagOf): empty for a module with no Go package (design 47 G §370). Repository string `json:"repository,omitempty"` @@ -84,6 +87,17 @@ type ReservedVersion struct { Withdrawn map[string]string `json:"withdrawn,omitempty"` } +// markPublished marks artifact a published at at, the moment the builder heard or confirmed its publish (H-A-331). +func (v *ReservedVersion) markPublished(a string, at time.Time) { + if v.Published == nil { + v.Published = map[string]bool{} + } + if v.PublishedAt == nil { + v.PublishedAt = map[string]time.Time{} + } + v.Published[a], v.PublishedAt[a] = true, at.UTC() +} + // KeptPublished is a registry seat's published kept on a reservation, as heard (design 47 G A-415). type KeptPublished struct { Name string `json:"name"` @@ -396,6 +410,8 @@ type ReleasedRecord struct { type ReleasedBuild struct { Fingerprint string `json:"fingerprint"` Version string `json:"version"` + // Digests are the published digest of each artifact, by reference (H-A-331); absent for a release kept before. + Digests map[string]string `json:"digests,omitempty"` } // ReleasedFor says whether the builder saw a version released for the build of fingerprint among the module's last releases. @@ -419,8 +435,54 @@ func (r *Reserver) ReleasedFor(ctx context.Context, module, fingerprint string) return false, nil } +// PublishedOf answers the seat mesh-builds' verb published (design 47 H-A-328): whether artifact of module built at fingerprint +// is published. A reservation of that fingerprint whose artifact a registry seat said published at the built digest (marked +// from the registry's record too, H-A-274), its number not withdrawn, answers its version, digest and, where kept, the +// publish time (H-A-331); a release of that fingerprint among the module's +// last releases answers published at its version, every artifact of a release being published; otherwise not published. +func (r *Reserver) PublishedOf(ctx context.Context, module, artifact, fingerprint string) (meshbuilds.PublishedAnswer, error) { + module, artifact, fingerprint = strings.TrimSpace(module), strings.TrimSpace(artifact), strings.TrimSpace(fingerprint) + if module == "" || artifact == "" || fingerprint == "" { + return meshbuilds.PublishedAnswer{}, errors.New("module, artifact and fingerprint are required") + } + keys, err := r.Store.KeysOf(ctx, NameFilter(module)) + if err != nil { + return meshbuilds.PublishedAnswer{}, fmt.Errorf("the reservations of %s cannot be listed: %w", module, err) + } + sort.Strings(keys) + for _, k := range keys { + cur, _, found, err := r.get(ctx, k) + if err != nil { + return meshbuilds.PublishedAnswer{}, err + } + // A number withdrawn by a take-over is never answered published (H-A-331). + if found && cur.Module == module && cur.Fingerprint == fingerprint && cur.Published[artifact] && !hasKey(cur.Withdrawn, cur.Version) { + a := meshbuilds.PublishedAnswer{Published: true, Version: cur.Version, Digest: cur.Artifacts[artifact]} + if at, ok := cur.PublishedAt[artifact]; ok { + a.At = &at + } + return a, nil + } + } + if r.Releases != nil { + raw, _, found, err := r.Releases.Get(ctx, ReleasedKey(module)) + if err != nil { + return meshbuilds.PublishedAnswer{}, fmt.Errorf("the releases of %s cannot be read: %w", module, err) + } + var rec ReleasedRecord + if found && json.Unmarshal(raw, &rec) == nil { + for i := len(rec.Kept) - 1; i >= 0; i-- { + if rec.Kept[i].Fingerprint == fingerprint { + return meshbuilds.PublishedAnswer{Published: true, Version: rec.Kept[i].Version, Digest: rec.Kept[i].Digests[artifact]}, nil + } + } + } + } + return meshbuilds.PublishedAnswer{Published: false}, nil +} + // keepRelease adds a release to the module's record, dropping the oldest past ReleasedKept. -func (r *Reserver) keepRelease(ctx context.Context, module, fingerprint, version string) error { +func (r *Reserver) keepRelease(ctx context.Context, module, fingerprint, version string, digests map[string]string) error { n := r.ReleasedKept if n < 1 { n = DefaultReleasedKept @@ -440,7 +502,7 @@ func (r *Reserver) keepRelease(ctx context.Context, module, fingerprint, version kept = append(kept, k) } } - kept = append(kept, ReleasedBuild{Fingerprint: fingerprint, Version: version}) + kept = append(kept, ReleasedBuild{Fingerprint: fingerprint, Version: version, Digests: digests}) if len(kept) > n { kept = kept[len(kept)-n:] } @@ -479,7 +541,7 @@ func (r *Reserver) Released(ctx context.Context, module, version string) ([]stri continue } if cur.Version == version && cur.Fingerprint != "" && r.Releases != nil { - if err := r.keepRelease(ctx, module, cur.Fingerprint, version); err != nil { + if err := r.keepRelease(ctx, module, cur.Fingerprint, version, cur.Artifacts); err != nil { return gone, fmt.Errorf("the release of %s %s cannot be recorded: %w", module, version, err) } } @@ -695,11 +757,9 @@ func (r *Reserver) publishedOn(ctx context.Context, keys []string, name, version } matched = true if !cur.Published[a] { - cur.Published = maps.Clone(cur.Published) - if cur.Published == nil { - cur.Published = map[string]bool{} - } - cur.Published[a], changed = true, true + cur.Published, cur.PublishedAt = maps.Clone(cur.Published), maps.Clone(cur.PublishedAt) + cur.markPublished(a, r.now()) + changed = true arts = append(arts, a) } } @@ -768,10 +828,7 @@ func (r *Reserver) Kept(ctx context.Context, module, version string) ([]Marked, } matched = true if !cur.Published[a] { - if cur.Published == nil { - cur.Published = map[string]bool{} - } - cur.Published[a] = true + cur.markPublished(a, r.now()) arts = append(arts, a) } } diff --git a/modules/mesh-delivery-builder/internal/versions/reserve_test.go b/modules/mesh-delivery-builder/internal/versions/reserve_test.go index dbe5b5ba..327ad2aa 100644 --- a/modules/mesh-delivery-builder/internal/versions/reserve_test.go +++ b/modules/mesh-delivery-builder/internal/versions/reserve_test.go @@ -221,6 +221,87 @@ func TestAReleaseIsRecordedByFingerprintBeforeItsReservationGoes(t *testing.T) { } } +// TestPublishedAnswersByTheBuildFingerprint: design 47 H-A-328 — the verb published answers published for a reservation of the +// fingerprint whose artifact a registry's record marked published, with its version and digest, and for a release of it; an +// artifact not yet marked, and another fingerprint, are not published. +func TestPublishedAnswersByTheBuildFingerprint(t *testing.T) { + ctx := context.Background() + heard := time.Date(2026, 11, 21, 19, 0, 0, 0, time.UTC) + store := &reserveMem{mem: newMem()} + r := &Reserver{Store: store, Releases: &reserveMem{mem: newMem()}, ReleasedKept: 2, Filled: func() bool { return true }, + ReleasedOf: func(context.Context, string) ([]string, error) { return nil, nil }, Now: func() time.Time { return heard }} + v, err := r.Reserve(ctx, "mesh-grants", "0.1", []string{"mesh-grants:busgrants", "mesh-grants:grantsstate"}, Source{Fingerprint: "src2:aa"}) + if err != nil { + t.Fatal(err) + } + if _, err := r.Built(ctx, "mesh-grants", v.Version, "mesh-grants:busgrants", "sha256:bb", nil); err != nil { + t.Fatal(err) + } + ask := func(artifact, fp string) meshbuilds.PublishedAnswer { + t.Helper() + a, err := r.PublishedOf(ctx, "mesh-grants", artifact, fp) + if err != nil { + t.Fatal(err) + } + return a + } + if a := ask("mesh-grants:busgrants", "src2:aa"); a.Published { + t.Errorf("a built artifact no registry holds is answered %+v, want not published", a) + } + if marked, _, err := r.Published(ctx, "mesh-grants-busgrants", v.Version, "sha256:bb"); err != nil || len(marked) != 1 { + t.Fatalf("the registry's published marked %v (%v)", marked, err) + } + // H-A-331: the mark keeps when the publish was heard, answered as at. + if a := ask("mesh-grants:busgrants", "src2:aa"); !a.Published || a.Version != v.Version || a.Digest != "sha256:bb" || a.At == nil || !a.At.Equal(heard) { + t.Errorf("published answered %+v, want %s at sha256:bb, heard %s", a, v.Version, heard) + } + // A mark written before the publish time was kept answers no time, never an invented one; a withdrawn number is never + // answered published. + key := ReservedKey("mesh-grants", "0.1") + cur, rev, _, _ := r.get(ctx, key) + older := cur + older.PublishedAt = nil + raw, _ := json.Marshal(older) + if rev, err = store.Update(ctx, key, raw, rev); err != nil { + t.Fatal(err) + } + if a := ask("mesh-grants:busgrants", "src2:aa"); !a.Published || a.At != nil { + t.Errorf("an older mark answered %+v, want published with no at", a) + } + withdrawn := older + withdrawn.Withdrawn = map[string]string{v.Version: "src2:aa"} + raw, _ = json.Marshal(withdrawn) + if rev, err = store.Update(ctx, key, raw, rev); err != nil { + t.Fatal(err) + } + if a := ask("mesh-grants:busgrants", "src2:aa"); a.Published { + t.Errorf("a withdrawn number is answered %+v, want not published", a) + } + raw, _ = json.Marshal(cur) + if _, err = store.Update(ctx, key, raw, rev); err != nil { + t.Fatal(err) + } + if a := ask("mesh-grants:grantsstate", "src2:aa"); a.Published { + t.Errorf("an artifact not marked is answered %+v", a) + } + if a := ask("mesh-grants:busgrants", "src2:other"); a.Published { + t.Errorf("another fingerprint is answered %+v", a) + } + if _, err := r.Released(ctx, "mesh-grants", v.Version); err != nil { + t.Fatal(err) + } + if a := ask("mesh-grants:grantsstate", "src2:aa"); !a.Published || a.Version != v.Version { + t.Errorf("after its release the fingerprint's artifact is answered %+v, want published at %s", a, v.Version) + } + // H-A-331: the kept release keeps each published digest with its version. + if a := ask("mesh-grants:busgrants", "src2:aa"); !a.Published || a.Digest != "sha256:bb" { + t.Errorf("after its release busgrants is answered %+v, want its digest sha256:bb", a) + } + if _, err := r.PublishedOf(ctx, "mesh-grants", "", "src2:aa"); err == nil { + t.Error("an ask naming no artifact is answered") + } +} + // TestTheWithdrawnVersionsAreDroppedOnceReleasedPast: design 47 H H-A-225, H-A-226, H-A-244 — a dropped number stays a gap, kept in // the series key's Withdrawn while it is above the catalog's highest released; at each write those at or below it go, so the // map never grows without end. diff --git a/modules/mesh-delivery-builder/manifest_test.go b/modules/mesh-delivery-builder/manifest_test.go index cdec75a2..2574b54d 100644 --- a/modules/mesh-delivery-builder/manifest_test.go +++ b/modules/mesh-delivery-builder/manifest_test.go @@ -198,9 +198,9 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 if len(m.Logs) != 1 || m.Logs[0].Name != builds.BuilderLog || m.Logs[0].MaxAgeDays != builds.BuilderLogDays || m.Logs[0].MaxMiB != builds.BuilderLogMiB { t.Errorf("logs = %+v, want %s kept %d days in %d MiB", m.Logs, builds.BuilderLog, builds.BuilderLogDays, builds.BuilderLogMiB) } - // Design 47 G §52, §369: the seat mesh-builds, held once for the mesh, serving reserved. - if len(m.Claims) != 1 || m.Claims[0].Name != "mesh-builds" || m.Claims[0].Scope != "mesh" || !reflect.DeepEqual(m.Claims[0].Serves, []string{"reserved"}) { - t.Errorf("claims = %+v, want mesh-builds, scope mesh, serving reserved", m.Claims) + // Design 47 G §52, §369, H-A-328: the seat mesh-builds, held once for the mesh, serving reserved and published. + if len(m.Claims) != 1 || m.Claims[0].Name != "mesh-builds" || m.Claims[0].Scope != "mesh" || !reflect.DeepEqual(m.Claims[0].Serves, []string{"reserved", "published"}) { + t.Errorf("claims = %+v, want mesh-builds, scope mesh, serving reserved and published", m.Claims) } // A-115: the process answers the verb itself and says so in `serves` beside its process, with no `bus`; the bundle serves no tool named so. var resources []map[string]any @@ -208,8 +208,8 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 t.Fatal(err) } for _, r := range resources { - if r["type"] == "process" && !reflect.DeepEqual(r["serves"], []any{"seat:mesh-builds.reserved"}) { - t.Errorf("the process serves %v, want seat:mesh-builds.reserved (A-115)", r["serves"]) + if r["type"] == "process" && !reflect.DeepEqual(r["serves"], []any{"seat:mesh-builds.reserved", "seat:mesh-builds.published"}) { + t.Errorf("the process serves %v, want seat:mesh-builds.reserved and published (A-115, H-A-328)", r["serves"]) } } if slices.Contains(m.Tools, "reserved") { diff --git a/modules/mesh-delivery-builder/module.json b/modules/mesh-delivery-builder/module.json index 7419cf8d..f58f40c8 100644 --- a/modules/mesh-delivery-builder/module.json +++ b/modules/mesh-delivery-builder/module.json @@ -7,7 +7,8 @@ "name": "mesh-builds", "scope": "mesh", "serves": [ - "reserved" + "reserved", + "published" ] } ], @@ -293,7 +294,8 @@ "name": "mesh-delivery-builder", "artifact": "program", "serves": [ - "seat:mesh-builds.reserved" + "seat:mesh-builds.reserved", + "seat:mesh-builds.published" ], "run": [ "./mesh-delivery-builder" diff --git a/modules/mesh-delivery-contracts/go/seats/meshbuilds/contract.go b/modules/mesh-delivery-contracts/go/seats/meshbuilds/contract.go index 179a6890..10c632d2 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshbuilds/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/meshbuilds/contract.go @@ -9,7 +9,8 @@ import "time" const Seat = "mesh-builds" const ( - VerbReserved = "reserved" + VerbReserved = "reserved" + VerbPublished = "published" ) const ( @@ -24,7 +25,7 @@ const ( // Verbs is every verb the seat serves, in its definition's order: a new copy each call. func Verbs() []string { - return []string{VerbReserved} + return []string{VerbReserved, VerbPublished} } // Events is every event published on the seat's subject: a new copy each call. @@ -217,6 +218,28 @@ type ReservedAnswer struct { Version string `json:"version,omitempty"` } +// PublishedInput is what published takes. +type PublishedInput struct { + // the package's artifact reference, : + Artifact string `json:"artifact"` + // the build fingerprint the package is built at, which the answer is matched by + Fingerprint string `json:"fingerprint"` + // the module that builds the package + Module string `json:"module"` +} + +// PublishedAnswer is what published answers. +type PublishedAnswer struct { + // when the builder heard or confirmed the publish, when published; absent for a mark written before the publish time was kept (design 47 H-A-331) + At *time.Time `json:"at,omitempty"` + // the published artifact's digest, when published; absent for a release kept before the digest was kept (design 47 H-A-331) + Digest string `json:"digest,omitempty"` + // true when the artifact built at the fingerprint is published; a number a take-over withdrew is never published (design 47 H-A-331) + Published bool `json:"published"` + // the version it is published at, MAJOR.MINOR.PATCH, when published + Version string `json:"version,omitempty"` +} + // StepAccept is the body of step. type StepAccept = StepRun diff --git a/modules/mesh-delivery-contracts/go/seats/meshbuilds/contract_test.go b/modules/mesh-delivery-contracts/go/seats/meshbuilds/contract_test.go index bb9c0680..47673e2f 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshbuilds/contract_test.go +++ b/modules/mesh-delivery-contracts/go/seats/meshbuilds/contract_test.go @@ -35,6 +35,12 @@ func TestFixtures(t *testing.T) { decode: func(b []byte) (any, error) { var v ReservedAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, {name: "reserved answers an error alone, naming the name or the version", subject: "", given: "{\"error\":\"no entry carries the name example.org/none\"}", wire: "{\"error\":\"no entry carries the name example.org/none\"}", decode: func(b []byte) (any, error) { var v ReservedAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, + {name: "published asks an artifact at its build fingerprint", subject: "", given: "{\"artifact\":\"mesh-grants:busgrants\",\"fingerprint\":\"src2:aa\",\"module\":\"mesh-grants\"}", wire: "{\"artifact\":\"mesh-grants:busgrants\",\"fingerprint\":\"src2:aa\",\"module\":\"mesh-grants\"}", + decode: func(b []byte) (any, error) { var v PublishedInput; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, + {name: "published answers the version, digest and time of a published artifact", subject: "", given: "{\"at\":\"2026-11-21T19:00:00Z\",\"digest\":\"sha256:aa\",\"published\":true,\"version\":\"0.1.4\"}", wire: "{\"at\":\"2026-11-21T19:00:00Z\",\"digest\":\"sha256:aa\",\"published\":true,\"version\":\"0.1.4\"}", + decode: func(b []byte) (any, error) { var v PublishedAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, + {name: "published answers an artifact not published", subject: "", given: "{\"published\":false}", wire: "{\"published\":false}", + decode: func(b []byte) (any, error) { var v PublishedAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, } for _, c := range cases { v, err := c.decode([]byte(c.given)) diff --git a/modules/mesh-delivery-contracts/seats/mesh-builds/fixtures/verbs.json b/modules/mesh-delivery-contracts/seats/mesh-builds/fixtures/verbs.json index 2ebcc767..a9df904d 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-builds/fixtures/verbs.json +++ b/modules/mesh-delivery-contracts/seats/mesh-builds/fixtures/verbs.json @@ -52,5 +52,31 @@ "given": { "error": "no entry carries the name example.org/none" } + }, + { + "name": "published asks an artifact at its build fingerprint", + "of": "verb:published.input", + "given": { + "module": "mesh-grants", + "artifact": "mesh-grants:busgrants", + "fingerprint": "src2:aa" + } + }, + { + "name": "published answers the version, digest and time of a published artifact", + "of": "verb:published.answer", + "given": { + "published": true, + "version": "0.1.4", + "digest": "sha256:aa", + "at": "2026-11-21T19:00:00Z" + } + }, + { + "name": "published answers an artifact not published", + "of": "verb:published.answer", + "given": { + "published": false + } } ] diff --git a/modules/mesh-delivery-contracts/seats/mesh-builds/seat.json b/modules/mesh-delivery-contracts/seats/mesh-builds/seat.json index e100e48c..4f68b5eb 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-builds/seat.json +++ b/modules/mesh-delivery-contracts/seats/mesh-builds/seat.json @@ -70,6 +70,58 @@ }, "description": "the reservation or release, or the error; module, version and artifacts are present whenever error is not (design 47 G 679)" } + }, + { + "name": "published", + "class": "read", + "description": "Whether an artifact built at a build fingerprint is published, from the builder's reservations and the registries' record, for the pipeline runner writing a mover's wait on a package: a package published before the wait was written is read, not waited for (design 47 H-A-326, H-A-328, H-A-274).", + "input": { + "type": "object", + "properties": { + "module": { + "type": "string", + "description": "the module that builds the package" + }, + "artifact": { + "type": "string", + "description": "the package's artifact reference, :" + }, + "fingerprint": { + "type": "string", + "description": "the build fingerprint the package is built at, which the answer is matched by" + } + }, + "required": [ + "module", + "artifact", + "fingerprint" + ] + }, + "answer": { + "type": "object", + "properties": { + "published": { + "type": "boolean", + "description": "true when the artifact built at the fingerprint is published; a number a take-over withdrew is never published (design 47 H-A-331)" + }, + "version": { + "type": "string", + "description": "the version it is published at, MAJOR.MINOR.PATCH, when published" + }, + "digest": { + "type": "string", + "description": "the published artifact's digest, when published; absent for a release kept before the digest was kept (design 47 H-A-331)" + }, + "at": { + "type": "string", + "format": "date-time", + "description": "when the builder heard or confirmed the publish, when published; absent for a mark written before the publish time was kept (design 47 H-A-331)" + } + }, + "required": [ + "published" + ] + } } ], "accepts": [ diff --git a/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.d.ts b/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.d.ts index bf8e68c7..5ac18af1 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.d.ts @@ -6,6 +6,7 @@ export declare const seat: "mesh-builds"; export declare const verbs: { readonly reserved: "reserved"; + readonly published: "published"; }; export declare const accepts: { @@ -171,6 +172,28 @@ export interface ReservedAnswer { "version"?: string; } +/** is what published takes. */ +export interface PublishedInput { + /** the package's artifact reference, : */ + "artifact": string; + /** the build fingerprint the package is built at, which the answer is matched by */ + "fingerprint": string; + /** the module that builds the package */ + "module": string; +} + +/** is what published answers. */ +export interface PublishedAnswer { + /** when the builder heard or confirmed the publish, when published; absent for a mark written before the publish time was kept (design 47 H-A-331) */ + "at"?: string; + /** the published artifact's digest, when published; absent for a release kept before the digest was kept (design 47 H-A-331) */ + "digest"?: string; + /** true when the artifact built at the fingerprint is published; a number a take-over withdrew is never published (design 47 H-A-331) */ + "published": boolean; + /** the version it is published at, MAJOR.MINOR.PATCH, when published */ + "version"?: string; +} + /** is the body of step. */ export type StepAccept = StepRun; diff --git a/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.js b/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.js index 31cc6f7c..6710b83b 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.js +++ b/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.js @@ -6,6 +6,7 @@ export const seat = "mesh-builds"; export const verbs = { reserved: "reserved", + published: "published", }; export const accepts = { diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go index 41964399..facab772 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go @@ -7,6 +7,7 @@ import ( "encoding/json" "fmt" "os" + "sync" "time" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" @@ -98,13 +99,29 @@ func FirstViewTools(module string, st stdio.KeptState, acts views.HandActs, now } // HealthTool is the runner's health check, run outside its process (design 47 G §76, §468, §495; A-164): it reads the -// runner's state pipelines as show does and the ages its process writes in the state consumers, and asks no seat verb. -func HealthTool(pipelines, consumers pipeline.Kept, now func() time.Time) stdio.Tool { +// runner's state pipelines as show does and the ages its process writes in the state consumers, and asks no seat verb. A +// running pipeline it first names stalled is said in this program's journal through logf, once per key for the +// program's life (H-A-311 (3), H-A-313). +func HealthTool(pipelines, consumers pipeline.Kept, now func() time.Time, logf func(string, ...any)) stdio.Tool { + var mu sync.Mutex + said := map[string]bool{} return stdio.Tool{Name: Prefix + "_health", - Description: "The pipeline runner's health: healthy when it answers show and each of its durable consumers' oldest " + - "unacknowledged message waits less than five minutes; otherwise it names each consumer behind.", + Description: "The pipeline runner's health: healthy when it answers show, each of its durable consumers' oldest " + + "unacknowledged message waits less than five minutes and no running pipeline has had nothing open and nothing " + + "newly placed for five minutes; otherwise it names each consumer behind and each such pipeline by its key.", Input: map[string]any{"type": "object", "properties": map[string]any{}}, - Run: func(map[string]any) (any, error) { return pipeline.Health(pipelines, consumers, now()), nil }} + Run: func(map[string]any) (any, error) { + a, stalled := pipeline.HealthStalled(pipelines, consumers, now()) + mu.Lock() + defer mu.Unlock() + for _, key := range stalled { + if !said[key] && logf != nil { + said[key] = true + logf("the running pipeline %s has had nothing open and nothing newly placed for %s or more: runner_health names it unhealthy (H-A-311)", key, pipeline.StalledAfter) + } + } + return a, nil + }} } // StoresFill is each of the runner's stores' fill through the bundle's calls (design 47 G §366, §623, A-214): the states @@ -154,7 +171,8 @@ func main() { tools = append(tools, CancelTool(r.List, stdio.Ask), RerunTool(stdio.Ask)) tools = append(tools, VerbTools(stdio.Ask)...) tools = append(tools, TestRunsTools(stdio.Ask)...) - tools = append(tools, HealthTool(r.Pipelines, kept{stdio.State(pipeline.StateConsumers)}, time.Now)) + tools = append(tools, HealthTool(r.Pipelines, kept{stdio.State(pipeline.StateConsumers)}, time.Now, + func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-pipeline-tools] "+f+"\n", a...) })) tools = append(tools, WaitingTool(kept{stdio.State(seat.StateQueue)})) if err := stdio.Serve("", tools); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-pipeline-tools] %v\n", err) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main_test.go index 95bb4bda..1b2873b3 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main_test.go @@ -3,6 +3,7 @@ package main import ( "encoding/json" "errors" + "fmt" "strings" "testing" @@ -37,7 +38,7 @@ func TestTools(t *testing.T) { names = append(names, tl.Name) } // RN25 (§468, §76; A-164): the health check is the bundle's runner_health. - names = append(names, HealthTool(none{}, none{}, nil).Name, WaitingTool(none{}).Name) + names = append(names, HealthTool(none{}, none{}, nil, nil).Name, WaitingTool(none{}).Name) // §606, §549: the unlogged entries are listed and appended by hand. for _, tl := range UnloggedTools(stdio.State("pipelines"), stdio.Log("step-log"), nil, nil) { names = append(names, tl.Name) @@ -53,6 +54,22 @@ func TestTools(t *testing.T) { if got, _ := json.Marshal(out); err != nil || string(got) != `[{"sequence":1,"pipeline":"main.5"},{"sequence":3,"pipeline":"main.7"}]` { t.Errorf("runner_waiting = %s (%v)", got, err) } + // H-A-311 (3), H-A-313: runner_health names a running pipeline quiet five minutes, and this program's journal says it + // once per key, however often the health is asked. + now := time.Date(2026, 11, 21, 20, 35, 0, 0, time.UTC) + quiet := queueOf{"main.137": `{"pipeline":"main.137","branch":"main","outcome":"running","source":137,"started":"2026-11-21T20:15:00Z",` + + `"movers":[{"module":"app"}],"steps":[{"module":"app","step":"build","outcome":"passed","status":"required","started":"2026-11-21T20:15:52Z","ended":"2026-11-21T20:17:55Z"}]}`} + var journal []string + health := HealthTool(quiet, none{}, func() time.Time { return now }, func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) }) + for range 2 { + out, err := health.Run(nil) + if got, _ := json.Marshal(out); err != nil || !strings.Contains(string(got), "main.137 (nothing open and nothing placed for 17m5s)") { + t.Errorf("runner_health = %s (%v), want main.137 named stalled", got, err) + } + } + if len(journal) != 1 || !strings.Contains(journal[0], "main.137") { + t.Errorf("the journal said %q, want main.137 said once", journal) + } } // queueOf is a state queue in memory, its keys in no order. diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index 19e9c98a..0e6ecb14 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -179,6 +179,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro starter.Supersede = func(newer seat.Pipeline) { hearer.SupersedeRunning(newer) } starter.Left = hearer.LeftInPlace // a same-source re-pin's superseded steps leave their runs (H-A-284) starter.Revision = widener.Revision // a kept mover is placed from its kept revision (H-A-286) + // A mover's wait on a package is read first: a package published before the wait places it at once (H-A-326, H-A-328). + starter.AskPublished = publishedOf(boundedCalls(calls.Call, readBound, time.After, logger.Printf)) hearer.Unplaced = func(p seat.Pipeline, module string) string { return starter.Unplaced(p, module, rv.Catalog.View) } hearer.AskPinHeld = func(key, module string, pins map[string]string) { if _, err := starter.AskPinHeld(hearer.Store, key, module, pins, widener.Revision); err != nil { diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/published.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/published.go new file mode 100644 index 00000000..1b4fbdbb --- /dev/null +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/published.go @@ -0,0 +1,29 @@ +package main + +import ( + "encoding/json" + "fmt" + + builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" + "git.novox.be/novox/mesh-sdk/go/buscalls" +) + +// publishedOf asks the seat mesh-builds' verb published through call whether a package built at a fingerprint is published +// (design 47 H-A-326 (1), H-A-328), asked when a mover's wait on it is written. +func publishedOf(call func(string, any) (json.RawMessage, error)) func(builds.PublishedInput) (builds.PublishedAnswer, error) { + return func(in builds.PublishedInput) (builds.PublishedAnswer, error) { + raw, err := call("seat:"+builds.Seat+"."+builds.VerbPublished, in) + if err != nil { + return builds.PublishedAnswer{}, err + } + answer, said, ok := buscalls.Unwrap(raw) + if !ok { + return builds.PublishedAnswer{}, fmt.Errorf("%s answered: %s", builds.VerbPublished, said) + } + var a builds.PublishedAnswer + if err := json.Unmarshal(answer, &a); err != nil { + return builds.PublishedAnswer{}, fmt.Errorf("%s answered what does not read: %w", builds.VerbPublished, err) + } + return a, nil + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/consumers.go b/modules/mesh-delivery-pipeline/internal/pipeline/consumers.go index d7a85fab..47f81be7 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/consumers.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/consumers.go @@ -7,6 +7,7 @@ import ( "strings" "time" + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" "git.novox.be/novox/mesh-sdk/go/health" "git.novox.be/novox/mesh-sdk/go/process" ) @@ -19,13 +20,83 @@ const StateConsumers = process.ConsumersState // behind (§76). const BehindAfter = 5 * time.Minute -// Health is the runner's health as its tool runner_health answers it (§468, §76, §495; A-164): healthy when its state -// pipelines answers, as show reads it, and each durable consumer's oldest unacknowledged message waits less than -// BehindAfter; otherwise unhealthy naming each consumer behind and how long. -func Health(pipelines, consumers Kept, now time.Time) health.Answer { - if _, err := pipelines.Keys(); err != nil { - return health.Unhealthy(fmt.Sprintf("The runner does not answer show: its state pipelines cannot be listed (%v).", err)) +// StalledAfter is how long a running pipeline may have nothing open and nothing newly placed before the runner's health +// names it stalled (H-A-311 (3)): the mesh's own rule for nothing moving (nothing-moved-in-five-minutes), not a setting +// (H-A-322). +const StalledAfter = 5 * time.Minute + +// Stalled is the key of each running pipeline, as show reads it, that has had nothing open (no step asked and not ended, +// no required step waiting but on a package's publish) and nothing newly placed, started or ended for StalledAfter at now (H-A-311 (3)), each with +// how long, in key order. +func Stalled(pipelines Kept, now time.Time) ([]string, error) { + all, err := Reads{Pipelines: pipelines}.pipelines() + if err != nil { + return nil, err } + var out []string + for _, p := range all { + if p.Outcome != seat.PipelineOutcomeRunning { + continue + } + last, busy := startedOf(p), false + for _, st := range p.Steps { + // A required step waiting (an approval, a composition on its validate) is open, as the end's judgement holds it; + // an optional one waiting holds nothing (A-104), and neither does one waiting on a package's publish, as a mover + // with no steps waiting on one does not (H-A-326 (2)). + if open(st) || (st.Outcome == seat.StepResultOutcomeWaiting && st.Status == seat.StepResultStatusRequired && !pinHeld(st)) { + busy = true + break + } + for _, at := range []*time.Time{st.Started, st.Ended} { + if at != nil && at.After(last) { + last = *at + } + } + } + if busy || last.IsZero() || now.Sub(last) < StalledAfter { + continue + } + out = append(out, fmt.Sprintf("%s (nothing open and nothing placed for %s)", p.Pipeline, now.Sub(last).Round(time.Second))) + } + return out, nil +} + +// Health is the runner's health as its tool runner_health answers it (§468, §76, §495; A-164): healthy when its state +// pipelines answers, as show reads it, each durable consumer's oldest unacknowledged message waits less than +// BehindAfter, and no running pipeline is stalled (H-A-311 (3)); otherwise unhealthy naming each consumer behind and +// how long, and each stalled pipeline by its key. +func Health(pipelines, consumers Kept, now time.Time) health.Answer { + a, _ := HealthStalled(pipelines, consumers, now) + return a +} + +// HealthStalled is Health with the keys of the running pipelines it names stalled, so the tools program says each in its +// journal once (H-A-313). +func HealthStalled(pipelines, consumers Kept, now time.Time) (health.Answer, []string) { + if _, err := pipelines.Keys(); err != nil { + return health.Unhealthy(fmt.Sprintf("The runner does not answer show: its state pipelines cannot be listed (%v).", err)), nil + } + stalled, err := Stalled(pipelines, now) + if err != nil { + return health.Unhealthy(fmt.Sprintf("The runner's running pipelines cannot be read as show reads them (%v).", err)), nil + } + var keys []string + for _, s := range stalled { + keys = append(keys, strings.SplitN(s, " ", 2)[0]) + } + a := consumersHealth(consumers, now) + if len(stalled) == 0 { + return a, nil + } + why := fmt.Sprintf("The runner's running pipelines with nothing open and nothing newly placed for %s or more: %s.", StalledAfter, strings.Join(stalled, ", ")) + if !a.Healthy { + why = a.Why + " " + why + } + return health.Unhealthy(why), keys +} + +// consumersHealth is the part of Health its consumers' ages give. +func consumersHealth(consumers Kept, now time.Time) health.Answer { keys, err := consumers.Keys() if err != nil { return health.Unhealthy(fmt.Sprintf("The ages of the runner's consumers cannot be read from its state %s (%v).", StateConsumers, err)) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/consumers_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/consumers_test.go index 8decf31c..de47a48c 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/consumers_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/consumers_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" "git.novox.be/novox/mesh-sdk/go/process" ) @@ -50,6 +51,19 @@ func TestHealth(t *testing.T) { {"six minutes names the consumer", mapKept{}, age("n_runner", 6*time.Minute), false, "n_runner (its oldest unacknowledged message waits 6m0s)"}, {"show does not answer", failingKept{}, age("n_runner", 0), false, "does not answer show"}, {"no event heard yet", mapKept{}, mapKept{}, true, "has heard no event yet"}, + // H-A-311 (3): a running pipeline with nothing open and nothing newly placed for five minutes is named by its key. + {"a pipeline quiet six minutes is named", piped(now, "main.137", -6*time.Minute, seat.StepResultOutcomePassed), age("n_runner", 0), false, + "main.137 (nothing open and nothing placed for 6m0s)"}, + {"a pipeline quiet four minutes is not", piped(now, "main.137", -4*time.Minute, seat.StepResultOutcomePassed), age("n_runner", 0), true, ""}, + {"a pipeline with a step open is not", piped(now, "main.137", -20*time.Minute, seat.StepResultOutcomeRunning), age("n_runner", 0), true, ""}, + // H-A-285, H-A-309: a record carrying the runner's kept-revisions is read, not refused. + {"a quiet pipeline keeping revisions is named", kept(piped(now, "main.131", -6*time.Minute, seat.StepResultOutcomePassed), "main.131"), age("n_runner", 0), false, + "main.131 (nothing open and nothing placed for 6m0s)"}, + // A required step waiting (an approval on the operator) is open, as the end's judgement holds it. + {"a pipeline whose approval waits is not", piped(now, "main.137", -20*time.Minute, seat.StepResultOutcomeWaiting), age("n_runner", 0), true, ""}, + // H-A-326 (2): a pipeline whose only movers wait on packages (a step held on a publish, a mover with no steps) is named. + {"a pipeline waiting only on packages is named", heldOnPublish(piped(now, "main.137", -20*time.Minute, seat.StepResultOutcomeWaiting), "main.137"), age("n_runner", 0), false, + "main.137 (nothing open and nothing placed for 20m0s)"}, } for _, r := range rows { a := Health(r.pipelines, r.consumers, now) @@ -58,3 +72,41 @@ func TestHealth(t *testing.T) { } } } + +// piped is the state pipelines holding one running pipeline key whose one step was placed and last moved at now+moved, +// with outcome. +func piped(now time.Time, key string, moved time.Duration, outcome string) mapKept { + at, started := now.Add(moved), now.Add(-time.Hour) + st := seat.StepResult{Module: "app", Step: seat.StepResultStepBuild, Outcome: outcome, Status: seat.StepResultStatusRequired, Started: &at} + if outcome == seat.StepResultOutcomeWaiting { + st.Step, st.On = seat.StepResultStepApproval, seat.WaitOnPerson + } else if outcome != seat.StepResultOutcomeRunning { + st.Ended = &at + } + raw, _ := json.Marshal(seat.Pipeline{Pipeline: key, Branch: "main", Outcome: seat.PipelineOutcomeRunning, Started: &started, + Movers: []seat.Mover{{Module: "app"}}, Steps: []seat.StepResult{st}}) + return mapKept{key: raw} +} + +// kept is m with key's record carrying the runner's kept-revisions (H-A-285). +func kept(m mapKept, key string) mapKept { + raw, err := withKept(m[key], map[string]int64{"app": 130}) + if err != nil { + panic(err) + } + m[key] = raw + return m +} + +// heldOnPublish is m with key's one step held on a package's publish (H-A-206) and a second mover with no steps, waiting on one. +func heldOnPublish(m mapKept, key string) mapKept { + var p seat.Pipeline + if err := json.Unmarshal(m[key], &p); err != nil { + panic(err) + } + p.Steps[0].Step, p.Steps[0].On, p.Steps[0].Why = seat.StepResultStepBuild, "", PinWait+"mesh-grants:busgrants" + p.Movers = append(p.Movers, seat.Mover{Module: "nats"}) + raw, _ := json.Marshal(p) + m[key] = raw + return m +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/moverkeys.go b/modules/mesh-delivery-pipeline/internal/pipeline/moverkeys.go index ca162ce9..1342dd0e 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/moverkeys.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/moverkeys.go @@ -332,6 +332,20 @@ func keptOf(raw []byte) (map[string]int64, error) { return kept, nil } +// withoutKept is record without its kept-revisions, the runner's own property and never part of the seat's Go form, so the +// seat's strict decode of the record the runner's reads make refuses no record carrying it (H-A-285, H-A-309). +func withoutKept(record []byte) ([]byte, error) { + var fields map[string]json.RawMessage + if err := json.Unmarshal(record, &fields); err != nil { + return nil, fmt.Errorf("the record is no object: %w", err) + } + if _, ok := fields[keptRevisionsField]; !ok { + return record, nil + } + delete(fields, keptRevisionsField) + return json.Marshal(fields) +} + // withKept is record with kept as its kept-revisions. func withKept(record []byte, kept map[string]int64) ([]byte, error) { var fields map[string]json.RawMessage diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/order_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/order_test.go index 95f5536a..3e830fd0 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/order_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/order_test.go @@ -1,6 +1,7 @@ package pipeline import ( + "fmt" "slices" "strings" "testing" @@ -94,6 +95,23 @@ func TestAMoversBuildWaitsForItsDependenciesInThePipeline(t *testing.T) { return e } before := mem["main.7"] + // H-A-311 (2): the build's end written while its placing was cut (heard by a hearer that places nothing): the same end + // heard again, late, moves nothing of the build and still places tool, which that end unlocked. + cut := *h + cut.Release = nil + if _, err := cut.OnStepEnded(bs, bs.Prefix+"n1", end("passed")); err != nil { + t.Fatal(err) + } + if p, _, _ = mem.Get("main.7"); placed(p, "tool") { + t.Fatalf("the cut placing placed tool: %+v", stepsOf(p, "tool")) + } + if _, err := h.OnStepEnded(bs, bs.Prefix+"n1", end("passed")); err != nil { + t.Fatal(err) + } + if p, _, _ = mem.Get("main.7"); !placed(p, "tool") || stepsOf(p, "tool")[0].Outcome != seat.StepResultOutcomeRunning { + t.Errorf("after builder's build passed, heard late, tool's steps = %+v, want placed and asked", stepsOf(p, "tool")) + } + mem["main.7"] = before if _, err := h.OnStepEnded(bs, bs.Prefix+"n1", end("passed")); err != nil { t.Fatal(err) } @@ -124,6 +142,43 @@ func TestAMoversBuildWaitsForItsDependenciesInThePipeline(t *testing.T) { t.Errorf("tool's step-ended = %+v, want not asked naming builder", ev.Result) } } + + // H-A-326 (1), (3), H-A-328: a package published before its importer's wait was written (by an earlier pipeline) is read of + // the build seat when the wait is written, matched by its mover's fingerprint, and the importer is placed with no event; + // the journal says the wait was resolved from the build seat's published, and a retried placing asks it once. + mem2 := memStore{} + var asked []builds.PublishedInput + var journal []string + s2 := &Starter{Pipelines: memCreated{mem2}, Store: mem2, Say: func(string, any) error { return nil }, Steps: &stepAsks{}, Name: "the-runner", + Registration: func(string) (catalog.Registration, bool) { return reg, true }, + Logf: func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) }, + AskPublished: func(in builds.PublishedInput) (builds.PublishedAnswer, error) { + asked = append(asked, in) + return builds.PublishedAnswer{Published: in.Artifact == "mesh-sdk:go" && in.Fingerprint == "src2:00", Version: "0.2.0"}, nil + }} + if n, err := s2.TakenIn(ev, view); n != 1 || err != nil { + t.Fatalf("the take-in started %d (%v)", n, err) + } + p, _, _ = mem2.Get("main.7") + if sdk, _ := MoverOf(p, "mesh-sdk"); !placed(p, "app") || placed(p, "tool") || !slices.Contains(sdk.Published, "mesh-sdk:go") { + t.Errorf("with mesh-sdk:go published before the wait: app placed %v, tool placed %v, mesh-sdk %+v; want app placed with no event", placed(p, "app"), placed(p, "tool"), sdk) + } + if !slices.ContainsFunc(journal, func(l string) bool { + return strings.Contains(l, "app's wait on mesh-sdk:go") && strings.Contains(l, "placed from the published record") + }) { + t.Errorf("the journal %q does not say app's wait was resolved from the build seat's published", journal) + } + before2 := len(asked) + if _, err := s2.Waiting(mem2, "main", view); err != nil { + t.Fatal(err) + } + if len(asked) != before2 { + t.Errorf("a placing again asked published %d more times, want each package asked once", len(asked)-before2) + } + // At the pipeline's end its kept answers go. + if _, err := s2.Turn("main.7", view); err != nil || len(s2.publishedRead) != 0 { + t.Errorf("after main.7's end %d answers are kept (%v), want none", len(s2.publishedRead), err) + } } // Design 47 G §270, A-326 (H §746, H-A-83 (2)): what a build module's new version is built from is built by its running diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/queue.go b/modules/mesh-delivery-pipeline/internal/pipeline/queue.go index cecc84a8..8a8c1695 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/queue.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/queue.go @@ -236,6 +236,7 @@ func (s *Starter) StartQueue(view func(branch string) (Half, bool)) (int, error) // Turn takes the end of pipeline key (its pipeline-ended, a cancel alike, H-A-53): its running key is deleted and the // queue placed (PlaceQueued). func (s *Starter) Turn(key string, view func(branch string) (Half, bool)) (int, error) { + s.forgetPublished(key) // the ended pipeline's published answers go with it (H-A-328) if s.Queue == nil { return 0, nil } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/read.go b/modules/mesh-delivery-pipeline/internal/pipeline/read.go index 668d2ed8..9f056599 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/read.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/read.go @@ -206,6 +206,10 @@ func (r Reads) pipelineOf(key string, all []string) (seat.Pipeline, bool, error) if raw == nil { return seat.Pipeline{}, false, nil } + raw, err = withoutKept(raw) + if err != nil { + return seat.Pipeline{}, false, fmt.Errorf("%s %s: %w", seat.StatePipelines, key, err) + } p, err := seat.DecodePipelinesByBranchRevisionValue(raw) if err != nil { return seat.Pipeline{}, false, fmt.Errorf("%s %s: %w", seat.StatePipelines, key, err) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/read_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/read_test.go index 375d0289..3b1ee9d7 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/read_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/read_test.go @@ -53,6 +53,12 @@ func TestReads(t *testing.T) { // A pipeline kept as H-A-187 keeps it: its record's steps empty, each mover's under steps... put(t, pipes, "next.8", seat.Pipeline{Pipeline: "next.8", Branch: "next", Source: 8, Outcome: seat.PipelineOutcomeRunning, Started: &t0, Steps: []seat.StepResult{}, Movers: []seat.Mover{{Module: "delta", Commit: "n8"}, {Module: "epsilon", Commit: "n8"}}}) + // H-A-285, H-A-309: the record carries the runner's own kept-revisions, which show, list and stands read past. + withRevs, err := withKept(pipes["next.8"], map[string]int64{"delta": 7}) + if err != nil { + t.Fatal(err) + } + pipes["next.8"] = withRevs put(t, pipes, "steps.next.8.epsilon", seat.PipelinesByStepsPipelineMoverValue{Steps: []seat.StepResult{{Module: "epsilon", Step: seat.StepResultStepBuild, Status: "required", Outcome: seat.StepResultOutcomePassed, Output: "next.8.build.epsilon:program"}}}) put(t, pipes, "steps.next.8.delta", seat.PipelinesByStepsPipelineMoverValue{Steps: []seat.StepResult{{Module: "delta", diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/repin.go b/modules/mesh-delivery-pipeline/internal/pipeline/repin.go index a13a850f..93a2dcc1 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/repin.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/repin.go @@ -177,6 +177,9 @@ func (s *Starter) placeAgain(p *seat.Pipeline, m seat.Mover, half Half, now time return nil, nil, true } due, waiting, failed := ready(*p, m.Module, half.Entries) + if !due && failed == "" && s.readPublished(p, m.Module, waiting) { + due, waiting, failed = ready(*p, m.Module, half.Entries) + } if !due && failed == "" { s.logf("%s: %s waits for %s before its build", p.Pipeline, m.Module, strings.Join(waiting, ", ")) return nil, nil, false diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/runends.go b/modules/mesh-delivery-pipeline/internal/pipeline/runends.go index c54faba5..ac1ad1dd 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/runends.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/runends.go @@ -75,12 +75,18 @@ func IsRunBound(key string) bool { // runOfAsk is the test run asked under ask: the run its asking step names in Shared, read from the state pipelines. func (h *Hearer) runOfAsk(ask string) (string, bool) { + run, _, ok := h.runAndPipelineOfAsk(ask) + return run, ok +} + +// runAndPipelineOfAsk is runOfAsk with the key of the pipeline whose step asked it. +func (h *Hearer) runAndPipelineOfAsk(ask string) (string, string, bool) { if h.TestRuns == nil || ask == "" { - return "", false + return "", "", false } pkeys, err := allKeys(h.Store) if err != nil { - return "", false + return "", "", false } for _, key := range pkeys { p, _, err := h.Store.Get(key) @@ -89,10 +95,26 @@ func (h *Hearer) runOfAsk(ask string) (string, bool) { } if i := stepOfAsk(p, ask); i >= 0 { run := sharedRun(p.Steps[i]) - return run, run != "" + return run, key, run != "" + } + } + return "", "", false +} + +// releaseLate places what the end of a step heard late unlocks (H-A-311 (2)): a step-progress or step-end that arrives +// after its ask ended moves nothing of the step itself, yet the movers waiting on what that end gave (the dependents' +// builds) are placed, in each pipeline named, as the end heard in time would have placed them (Release). +func (h *Hearer) releaseLate(pipelines ...string) { + if h.Release == nil { + return + } + seen := map[string]bool{} + for _, key := range pipelines { + if key != "" && !seen[key] { + seen[key] = true + h.Release(key) } } - return "", false } // RunTaken takes the ask-taken of a test run's ask (§725, A-470): the node that took it is kept on the run, its untaken @@ -158,13 +180,18 @@ func armBound(clock RunClock, runs TestRuns, key, run, ask string, ttl time.Dura // RunProgress takes a build seat's step-progress of a test run's ask (§725, A-466p): its clock begins again, so a run that // keeps saying its progress is never ended by it. An ask of no running, taken test run takes nothing. func (h *Hearer) RunProgress(ask string) error { - run, ok := h.runOfAsk(ask) + run, key, ok := h.runAndPipelineOfAsk(ask) if !ok { return nil } r, _, err := h.TestRuns.Get(run) if errors.Is(err, ErrNoRun) || (err == nil && (r.Ask != ask || r.Ended != nil)) { - h.logf("the step-progress of ask %s is late: the test run %s names ask %q (ended %v), so it moves nothing (A-470n)", ask, run, r.Ask, r.Ended != nil) + h.logf("the step-progress of ask %s is late: the test run %s names ask %q (ended %v), so it moves nothing of its own; what its end unlocks is placed (A-470n, H-A-311)", ask, run, r.Ask, r.Ended != nil) + late := []string{key} + for _, reader := range r.Readers { + late = append(late, reader.Pipeline) + } + h.releaseLate(late...) return nil } if err != nil { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index 1040b7ef..e9f4f552 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -8,6 +8,7 @@ import ( "sort" "strconv" "strings" + "sync" "time" runnerclient "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient" @@ -199,6 +200,67 @@ type Starter struct { // registration-missing (H-A-262); Observer is the runner's own module name. nil raises none. Findings Findings Observer string + // AskPublished asks the seat mesh-builds' verb published whether a package built at a fingerprint is published (H-A-326 + // (1), H-A-328): read when a mover's wait on it is written, so a publish made before the wait places the mover without + // any event; nil reads nothing, the publish event alone placing it. + AskPublished func(in builds.PublishedInput) (builds.PublishedAnswer, error) + + publishedMu sync.Mutex + publishedRead map[string]builds.PublishedAnswer +} + +// readPublished asks, once per pipeline, package and fingerprint, the seat mesh-builds whether each package mover waits on +// in p is published already (H-A-326 (1), H-A-328): a wait is a read first and an event second. Each package answered +// published goes into its From mover's published in p, written with the placing, and the journal says the wait resolved +// from the build seat's published, apart from one a package-published places (H-A-326 (3)). The answer is kept for the +// placing's compare-and-set retries, so no retry asks again. It says whether it added any. +func (s *Starter) readPublished(p *seat.Pipeline, mover string, waiting []string) bool { + if s.AskPublished == nil { + return false + } + added := false + for _, ref := range waiting { + i := slices.IndexFunc(p.Movers, func(m seat.Mover) bool { return m.Module == moduleOf(ref) }) + if i < 0 || slices.Contains(p.Movers[i].Published, ref) { + continue + } + in := builds.PublishedInput{Module: p.Movers[i].Module, Artifact: ref, Fingerprint: p.Movers[i].Fingerprint} + key := p.Pipeline + "\x00" + ref + "\x00" + in.Fingerprint + s.publishedMu.Lock() + a, asked := s.publishedRead[key] + s.publishedMu.Unlock() + if !asked { + var err error + if a, err = s.AskPublished(in); err != nil { + s.logf("%s: whether %s at %s is published is not read, so %s waits for its publish event: %v", p.Pipeline, ref, in.Fingerprint, mover, err) + continue + } + s.publishedMu.Lock() + if s.publishedRead == nil { + s.publishedRead = map[string]builds.PublishedAnswer{} + } + s.publishedRead[key] = a + s.publishedMu.Unlock() + } + if !a.Published { + continue + } + p.Movers[i].Published = append(p.Movers[i].Published, ref) + added = true + s.logf("%s: %s's wait on %s is resolved from the build seat's published (at %s), not from an event: placed from the published record (H-A-326)", p.Pipeline, mover, ref, a.Version) + } + return added +} + +// forgetPublished drops the published answers readPublished kept for pipeline key, at its end (H-A-328). +func (s *Starter) forgetPublished(key string) { + s.publishedMu.Lock() + defer s.publishedMu.Unlock() + for k := range s.publishedRead { + if strings.HasPrefix(k, key+"\x00") { + delete(s.publishedRead, k) + } + } } // holderless ends at once, as it is placed, a pipeline with a build step whose seat the seat mesh-module-assigner's view @@ -365,6 +427,9 @@ func (s *Starter) placeMovers(p *seat.Pipeline, reg func(module string) (catalog // A mover's steps wait for the publish of each mover it builds on (§746, H-A-83 (2)): placed once each is heard, // ended not asked when one's build ended without passing. due, waiting, failed := ready(*p, m.Module, entries) + if !due && failed == "" && s.readPublished(p, m.Module, waiting) { + due, waiting, failed = ready(*p, m.Module, entries) + } if !due && failed == "" { s.logf("%s: %s waits for %s before its build", p.Pipeline, m.Module, strings.Join(waiting, ", ")) waits = append(waits, m) @@ -746,40 +811,15 @@ func (s *Starter) placeWaiting(store Updater, due func(seat.Pipeline) bool, reg } } for _, key := range pkeys { - p, rev, err := store.Get(key) + p, asks, ended, notAsked, n, err := s.placeOne(store, key, due, reg, view, now) if err != nil { - note(fmt.Errorf("the pipeline %s is not read: %w", key, err)) - continue + note(err) } - if p.Outcome != seat.PipelineOutcomeRunning || p.Run != "" || !due(p) { - continue - } - half, ok := view(p.Branch) - if !ok { - s.logf("the catalog half of %s is not held yet: %s waits on it for its steps, placed once it is held", p.Branch, key) - continue - } - if p.Revision > 0 && half.Revision < p.Revision { - s.logf("the catalog half of %s holds revision %d: %s waits for revision %d, which its take-in made, to place its steps (§200)", p.Branch, half.Revision, key, p.Revision) - continue - } - before := len(p.Steps) - asks, unregistered, ended, notAsked := s.placeMovers(&p, reg, half.Entries, now) - // Until its registration arrives, the pipeline says it waits on it (A-327): a mover with no registration has no step, - // and it holds the pipeline's end only while every placed mover's required steps passed (H-A-163, H-A-171). - for _, m := range unregistered { - s.logf("%s: %s has no registration yet: its steps wait on its registration, which places them (A-327)", key, m) - s.registrationMissing(m, key) - } - if len(p.Steps) == before { - continue - } - if _, err := store.Update(key, p, rev); err != nil { - s.logf("%s is written meanwhile, so nothing is placed in it now: %v", key, err) + if n == 0 { continue } count++ - s.logf("%s: %d steps placed from %s (A-327)", key, len(p.Steps)-before, from) + s.logf("%s: %d steps placed from %s (A-327)", key, n, from) // A pipeline whose steps were never asked is started by this placing (A-446a); the pipeline-started of its start, // said already, is said again harmlessly. s.sayStartedPipeline(p, now) @@ -799,6 +839,50 @@ func (s *Starter) placeWaiting(store Updater, due func(seat.Pipeline) bool, reg return count, first } +// placeOne places, in pipeline key, the steps of each mover with none, when the pipeline is running, not a run on request +// and due names it (placeWaiting). Its write is by compare-and-set: a write refused because the pipeline was written +// meanwhile is done again at once from the record as read again, until it lands or the pipeline no longer waits on a +// placing (it ended, or every mover due is placed), so a concurrent write never cuts a placing for good (H-A-311 (1)). It +// answers the pipeline as written, its asks, the steps ended or not asked at the placing, and how many steps it placed. +func (s *Starter) placeOne(store Updater, key string, due func(seat.Pipeline) bool, reg func(module string) (catalog.Registration, bool), view func(branch string) (Half, bool), now time.Time) (seat.Pipeline, []Asked, []int, []int, int, error) { + for { + p, rev, err := store.Get(key) + if err != nil { + return p, nil, nil, nil, 0, fmt.Errorf("the pipeline %s is not read: %w", key, err) + } + if p.Outcome != seat.PipelineOutcomeRunning || p.Run != "" || !due(p) { + return p, nil, nil, nil, 0, nil + } + half, ok := view(p.Branch) + if !ok { + s.logf("the catalog half of %s is not held yet: %s waits on it for its steps, placed once it is held", p.Branch, key) + return p, nil, nil, nil, 0, nil + } + if p.Revision > 0 && half.Revision < p.Revision { + s.logf("the catalog half of %s holds revision %d: %s waits for revision %d, which its take-in made, to place its steps (§200)", p.Branch, half.Revision, key, p.Revision) + return p, nil, nil, nil, 0, nil + } + before := len(p.Steps) + asks, unregistered, ended, notAsked := s.placeMovers(&p, reg, half.Entries, now) + // Until its registration arrives, the pipeline says it waits on it (A-327): a mover with no registration has no step, + // and it holds the pipeline's end only while every placed mover's required steps passed (H-A-163, H-A-171). + for _, m := range unregistered { + s.logf("%s: %s has no registration yet: its steps wait on its registration, which places them (A-327)", key, m) + s.registrationMissing(m, key) + } + if len(p.Steps) == before { + return p, nil, nil, nil, 0, nil + } + if _, err := store.Update(key, p, rev); errors.Is(err, ErrChanged) { + s.logf("%s is written meanwhile, so its placing is done again from the record read again (H-A-311): %v", key, err) + continue + } else if err != nil { + return p, nil, nil, nil, 0, fmt.Errorf("the placing of %s is not written: %w", key, err) + } + return p, asks, ended, notAsked, len(p.Steps) - before, nil + } +} + // recovered says whether the commit of pipeline key is named a recovery; a read that fails asks on the stream check, said, since // a step is asked either way (a recovery's only goes ahead of the first round). func (s *Starter) recovered(key, commit string) bool { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go index ac547827..8994d742 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go @@ -300,6 +300,25 @@ func TestAPipelineStartedBeforeItsRegistrationIsPlacedWhenItArrives(t *testing.T t.Errorf("the pipeline a runner before this one set aside: placed %d (%v), %d steps, %d asks more, want it started as a newly started one", n, err, len(p.Steps), len(asks.asks)-before) } + // H-A-311 (1): a placing whose write is refused because the pipeline was written meanwhile (a step's end, a publish) is + // done again at once from the record read again, never left unplaced until a restart. + held = false + taken = catalog.TakeInTaken{Branch: "next", Source: 7, Entries: []catalog.TakeInEntry{{Module: "app", Commit: "c4", Fingerprint: "src2:dd"}}} + if n, err := s.TakenIn(taken, late); n != 1 || err != nil { + t.Fatalf("the take-in of next.7 started %d (%v)", n, err) + } + held = true + raced := &racingStore{memStore: store.memStore, before: func(m memStore) { + e := m["next.7"] + e.p.Superseded, e.rev = append(e.p.Superseded, "other"), e.rev+1 // a concurrent write the placing did not read + m["next.7"] = e + }} + n, err = s.Waiting(raced, "next", late) + p, _, _ = store.Get("next.7") + if n != 1 || err != nil || len(p.Steps) == 0 || !slices.Contains(p.Superseded, "other") { + t.Errorf("a placing cut by a concurrent write placed %d (%v): %d steps, superseded %v, want its steps placed on the record read again (H-A-311)", n, err, len(p.Steps), p.Superseded) + } + // A-327: a mover with no registration, in a pipeline placed from the half held later (the queue's turn, the half held), // is said to wait on its registration, so the pipeline that cannot end before it is placed says why. var logged []string @@ -411,3 +430,17 @@ func TestAPublishIsTakenIntoThePipelineOfItsRevision(t *testing.T) { t.Errorf("a publish naming no revision (%v): main.7 %v, main.8 %v; want main.7's, the source's lowest", err, published("main.7"), published("main.8")) } } + +// racingStore is memStore whose first Update is preceded by before, a write another part makes between a read and its write. +type racingStore struct { + memStore + before func(memStore) +} + +func (r *racingStore) Update(key string, p seat.Pipeline, rev uint64) (uint64, error) { + if b := r.before; b != nil { + r.before = nil + b(r.memStore) + } + return r.memStore.Update(key, p, rev) +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go index c492ce6f..3a629f1b 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go @@ -273,6 +273,8 @@ func (h *Hearer) OnStepEndedPinned(bs StepSeat, subject string, ev StepEnded, sh ev.Module, ev.Commit, ev.Fingerprint, p.Steps[i].Step, p.Steps[i].Module, m.Commit, m.Fingerprint) case !open(p.Steps[i]): h.Asks.answeredLate(p.Steps[i], h.logf) + // An end heard late moves nothing of its step, and still places what that step's end unlocks (H-A-311 (2)). + defer h.releaseLate(ev.Pipeline) if run := sharedRun(p.Steps[i]); run != "" && h.TestRuns != nil { if r, _, err := h.TestRuns.Get(run); err == nil && r.Ended == nil && r.Ask == ev.ID { // The asking step's end was written and its run's was not (a write failed): the run ends now, heard again. diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go index ba0712e4..07138de5 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go @@ -2,6 +2,7 @@ package pipeline import ( "fmt" + "slices" "strings" "testing" "time" @@ -599,6 +600,8 @@ func TestNoWriteOfATestRunActsOnAReadBeforeItsLatestChange(t *testing.T) { run := unit.Shared[0].Run asked(f, run) delete(f.clock.bound, UntakenPrefix+run) + var released []string + f.h.Release = func(key string) { released = append(released, key) } _ = f.h.RunTaken(unit.Ask, "ace") _ = f.h.RunProgress(unit.Ask) late := 0 @@ -610,6 +613,10 @@ func TestNoWriteOfATestRunActsOnAReadBeforeItsLatestChange(t *testing.T) { if _, ok := f.clock.bound[BoundPrefix+run]; ok || late != 2 { t.Errorf("bounds %v, journal %q; want no clock and both late signals journaled", f.clock.bound, f.logged) } + // H-A-311 (2): the late step-progress still places what its step's end unlocks, in the pipeline that asked it. + if !slices.Contains(released, p.Pipeline) { + t.Errorf("the late step-progress placed in %v, want %s's waiting movers placed", released, p.Pipeline) + } }) // A run's end is written as its step-ended is heard, before the finding the step says (A-470o). t.Run("the run ends before the step's finding is written", func(t *testing.T) { diff --git a/modules/mesh-delivery-pipeline/manifest_test.go b/modules/mesh-delivery-pipeline/manifest_test.go index 7de59100..5c6ef768 100644 --- a/modules/mesh-delivery-pipeline/manifest_test.go +++ b/modules/mesh-delivery-pipeline/manifest_test.go @@ -106,6 +106,8 @@ func TestManifest(t *testing.T) { // A-102 (§22, §24): the accepted kinds the runner puts its steps on. "seat:mesh-builds.step", "seat:mesh-wants.step", "seat:node-build-go.check.step", "seat:node-build-node.check.step", "seat:node-build-image.check.step", + // H-A-328: whether a package a mover waits on is published, read when the wait is written. + "seat:mesh-builds.published", // RN22 (§86, §415, A-207): a recovery's steps on the build seats' stream delivery. "seat:node-build-go.delivery.step", "seat:node-build-node.delivery.step", "seat:node-build-image.delivery.step", // A-231: the queues of the build seats it asks, check for the steps and delivery for a recovery's. diff --git a/modules/mesh-delivery-pipeline/module.json b/modules/mesh-delivery-pipeline/module.json index d25b519d..ebe2436a 100644 --- a/modules/mesh-delivery-pipeline/module.json +++ b/modules/mesh-delivery-pipeline/module.json @@ -78,6 +78,7 @@ "seat:mesh-module-assigner.assignments", "seat:mesh-grants.grants-for", "seat:mesh-builds.step", + "seat:mesh-builds.published", "seat:mesh-wants.step", "seat:node-build-go.check.step", "seat:node-build-node.check.step",