From 7ffec1b56b266128578b6e80bdcb6f35be0002bb Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 00:53:49 +0100 Subject: [PATCH 01/15] I1: a module's fingerprint takes a package by what is published, never by a commit of its repository MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A package of another repository none publishes leaves the module without a fingerprint (NoPackageVersion) instead of taking the gate's source head; one of the module's own repository none publishes enters by its directory's tree, which the tree input carries, so the same source gives the same fingerprint. Builds: design 47 H2 §802, H-A-349 DQ-8, H-A-350 DMC-5 --- .../internal/fingerprint/fingerprint.go | 7 ++-- .../internal/fingerprint/fingerprint_test.go | 13 +++++- .../internal/index/form.go | 37 ++++++----------- .../internal/index/imports_test.go | 41 +++++++++++-------- .../internal/index/source.go | 7 ---- .../internal/index/toolchains.go | 8 ++-- 6 files changed, 54 insertions(+), 59 deletions(-) diff --git a/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint.go b/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint.go index 64bf2d128..6c3cd5637 100644 --- a/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint.go +++ b/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint.go @@ -229,8 +229,10 @@ func ReplacedSiblings(goMod []byte, repository string) []string { // VersionLine is the version line's value of each package (Inputs.Packages) of a module of repository: a mesh Go // module of another repository by its version (versions), and one of repository itself, replaced by its directory // (ReplacedSiblings), by the content its directory publishes, `content:` and the hex sha256 of its requirements' digest -// and each package's digest (empty while it is not published, which gives no fingerprint until it is, as a version -// none publishes does). +// and each package's digest. A package of another repository with no version is empty, which gives no fingerprint until +// it is published (NoPackageVersion); one of repository that is not published yet has no line: it enters by its +// directory's tree, which the module's tree input carries (its own directory, or a derived directory of H-A-202), never by +// a commit (design 47 H2 §802, H-A-349 DQ-8). func VersionLine(repository string, packages []string, versions sdkstep.PackageVersions, published map[string]meshcatalog.PackagesByModuleValue) map[string]string { if len(packages) == 0 { return nil @@ -244,7 +246,6 @@ func VersionLine(repository string, packages []string, versions sdkstep.PackageV } pub, held := published[p] if !held { - out[p] = "" continue } lines := []string{"requires " + pub.Requires} diff --git a/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint_test.go b/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint_test.go index 6a1d68bbc..257e6781f 100644 --- a/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint_test.go +++ b/modules/mesh-delivery-indexer/internal/fingerprint/fingerprint_test.go @@ -62,8 +62,17 @@ func versionLineTakesASiblingDirectoryByItsContent(t *testing.T) { if a, b := line("d1"), line("d2"); a == b { t.Error("a change of the sibling directory alone left the importer's fingerprint where it was") } - if got := VersionLine(repo, []string{buildrun}, nil, nil)[buildrun]; got != "" { - t.Errorf("an unpublished sibling is taken as %q, want none (no fingerprint until it is published)", got) + // An unpublished sibling has no line: it enters by its directory's tree, which the tree input carries (design 47 H2 + // §802, H-A-349 DQ-8); one of another repository with no version still gives no fingerprint. + if got, ok := VersionLine(repo, []string{buildrun}, nil, nil)[buildrun]; ok { + t.Errorf("an unpublished sibling is taken as %q, want no line (it enters by its directory's tree)", got) + } + in := base() + if in.Packages = VersionLine(repo, []string{sdk}, nil, nil); in.Packages[sdk] != "" { + t.Fatalf("a package of another repository with no version is taken as %q", in.Packages[sdk]) + } + if fp, none := Of(in); fp != "" || !strings.HasPrefix(none, NoPackageVersion) { + t.Errorf("a package of another repository with no version gives %q (%q), want none naming it", fp, none) } } diff --git a/modules/mesh-delivery-indexer/internal/index/form.go b/modules/mesh-delivery-indexer/internal/index/form.go index 133f42e1b..8e9c0687f 100644 --- a/modules/mesh-delivery-indexer/internal/index/form.go +++ b/modules/mesh-delivery-indexer/internal/index/form.go @@ -7,7 +7,6 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk" "mesh-delivery-indexer/internal/fingerprint" ) @@ -39,14 +38,15 @@ type form struct { // another repository its go.mod requires, at any version (design 47 H-A-157), read as the indexer always read it // (fingerprint.HeldVersionsOf), and the content of each mesh Go module of its own repository its go.mod replaces by a // directory (fingerprint.VersionLine), so a change of that directory alone moves it (review finding 2). Its versions are the other repositories' alone. -// A package of another repository none of them publishes takes the commit the gate's source names for its repository -// (heads, design 47 G A-365). +// A package of another repository none of them publishes leaves the module without a fingerprint until one does +// (fingerprint.NoPackageVersion), and one of its own repository none publishes enters by its directory's tree alone, +// never by a commit: the same source, the same fingerprint (design 47 H2 §802, H-A-349 DQ-8). // - **An accepted series** (DLV-I6): published and pins come from acceptedSeries, so a mesh Go module of another // repository whose series the module accepts enters, in either form, by its series' newest published version and the // digests of that version's commit. // // It sets kept.ImportForm when it writes the import form. -func formOf(module, repository string, kept *meshcatalog.ModulesByModuleValueInputs2, toolchains map[string]meshcatalogclient.Toolchain2, prev meshcatalog.ModulesByModuleValue, held map[string]meshcatalog.ModulesByModuleValue, published map[string]meshcatalog.PackagesByModuleValue, pins sdkstep.PackageVersions, heads map[string]string, logf func(string, ...any)) form { +func formOf(module, repository string, kept *meshcatalog.ModulesByModuleValueInputs2, toolchains map[string]meshcatalogclient.Toolchain2, prev meshcatalog.ModulesByModuleValue, held map[string]meshcatalog.ModulesByModuleValue, published map[string]meshcatalog.PackagesByModuleValue, pins sdkstep.PackageVersions, logf func(string, ...any)) form { in := inputsFrom(module, *kept, toolchains) if len(kept.Imports) > 0 { r := fingerprint.Resolve(repository, kept.Imports, published) @@ -58,11 +58,11 @@ func formOf(module, repository string, kept *meshcatalog.ModulesByModuleValueInp return form{none: none} case kept.ImportForm == fp && prev.Fp2 != "": // The build that reported the imports stands, and so do its versions. - return form{fp: prev.Fp2, versions: refined(sdkstep.PackageVersions(prev.Packages), r.Versions, module, repository, kept, held, pins, heads), + return form{fp: prev.Fp2, versions: refined(sdkstep.PackageVersions(prev.Packages), r.Versions, module, repository, kept, held, pins), stands: true, raw: fp, built: prev.Packages} } kept.ImportForm = fp - return form{fp: fp, versions: refined(r.Versions, nil, module, repository, kept, held, pins, heads)} + return form{fp: fp, versions: refined(r.Versions, nil, module, repository, kept, held, pins)} } if logf != nil { logf("%s: no published package holds %s, so its fingerprint takes the version line", module, strings.Join(r.Missing, ", ")) @@ -71,7 +71,7 @@ func formOf(module, repository string, kept *meshcatalog.ModulesByModuleValueInp // The fingerprint written here is the version line's, not the import form's: no import form is held for the inputs it was // computed from, so a later pass whose import form computes the same does not take this one for the build's (issue 1966). kept.ImportForm = "" - versions := lineVersions(module, repository, kept, held, pins, heads, logf) + versions := lineVersions(module, repository, kept, held, pins, logf) in.Packages = fingerprint.VersionLine(repository, kept.Packages, versions, published) fp, none := fingerprint.Of(in) if fp == "" { @@ -83,22 +83,22 @@ func formOf(module, repository string, kept *meshcatalog.ModulesByModuleValueInp // refined are an import form's versions: the first of sets that names a package, else, while the module's go.mod // requires a mesh package of another repository (kept.Packages), the version line's, so a build's reported imports // refine the go.mod's set and never empty it (design 47 H-A-157); nil when nothing names one. -func refined(versions, then sdkstep.PackageVersions, module, repository string, kept *meshcatalog.ModulesByModuleValueInputs2, held map[string]meshcatalog.ModulesByModuleValue, pins sdkstep.PackageVersions, heads map[string]string) sdkstep.PackageVersions { +func refined(versions, then sdkstep.PackageVersions, module, repository string, kept *meshcatalog.ModulesByModuleValueInputs2, held map[string]meshcatalog.ModulesByModuleValue, pins sdkstep.PackageVersions) sdkstep.PackageVersions { switch { case len(versions) > 0: return versions case len(then) > 0: return then case len(fingerprint.OtherPackages(kept.Packages, repository)) > 0: - return lineVersions(module, repository, kept, held, pins, heads, nil) + return lineVersions(module, repository, kept, held, pins, nil) } return versions } // lineVersions are the version line's versions of the mesh Go modules of other repositories the module's go.mod requires -// (kept.Packages): each as the indexer has always read it, an accepted series' pin over it, else the commit the gate's -// source names for its repository. -func lineVersions(module, repository string, kept *meshcatalog.ModulesByModuleValueInputs2, held map[string]meshcatalog.ModulesByModuleValue, pins sdkstep.PackageVersions, heads map[string]string, logf func(string, ...any)) sdkstep.PackageVersions { +// (kept.Packages): each as the indexer has always read it, an accepted series' pin over it; one neither gives has no +// version, so the module has no fingerprint until it is published (design 47 H2 §802). +func lineVersions(module, repository string, kept *meshcatalog.ModulesByModuleValueInputs2, held map[string]meshcatalog.ModulesByModuleValue, pins sdkstep.PackageVersions, logf func(string, ...any)) sdkstep.PackageVersions { others := fingerprint.OtherPackages(kept.Packages, repository) versions, unpublished := fingerprint.HeldVersionsOf(others, held) for p, v := range pins { // an accepted series' newest published version (acceptedSeries, DLV-I6) @@ -110,19 +110,6 @@ func lineVersions(module, repository string, kept *meshcatalog.ModulesByModuleVa unpublished = slices.DeleteFunc(unpublished, func(u string) bool { return u == p }) } } - // A mesh Go package no module the indexer holds publishes is built at the commit the gate's source names for its - // repository, as an import of the same repository is; once a version is published, the version applies (design 47 G A-365). - for _, p := range slices.Clone(unpublished) { - if repo, _, ok := trunk.PackageRepository(p); ok { - if head := heads[repo]; len(head) == 40 { - if versions == nil { - versions = sdkstep.PackageVersions{} - } - versions[p] = head - unpublished = slices.DeleteFunc(unpublished, func(u string) bool { return u == p }) - } - } - } if len(unpublished) > 0 && logf != nil { logf("%s: no module the indexer holds publishes %s, so its fingerprint takes no version of it", module, strings.Join(unpublished, ", ")) } diff --git a/modules/mesh-delivery-indexer/internal/index/imports_test.go b/modules/mesh-delivery-indexer/internal/index/imports_test.go index 65876c137..762b8bbd9 100644 --- a/modules/mesh-delivery-indexer/internal/index/imports_test.go +++ b/modules/mesh-delivery-indexer/internal/index/imports_test.go @@ -11,6 +11,8 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-sdk/go/process" + + "mesh-delivery-indexer/internal/fingerprint" ) const meshSDK = "git.novox.be/novox/mesh-sdk/go" @@ -402,41 +404,44 @@ func anUnreadPackagesStateWritesNoFingerprintAndFingerprintsAreComputedAgain(t * t.Fatalf("fingerprints again of a module not held answered stored: %+v", r) } }) - t.Run("formOf builds a package no module publishes at the source's commit of its repository", func(t *testing.T) { // design 47 G A-365 - head := strings.Repeat("c", 40) + t.Run("formOf takes a package by what is published, never by a commit of its repository", func(t *testing.T) { // design 47 H2 §802, H-A-349 DQ-8 contracts := "git.novox.be/novox/mesh-contracts/go/seats/findings" held := map[string]meshcatalog.ModulesByModuleValue{"other": {Repository: "novox/mesh-sdk", Commit: strings.Repeat("d", 40), Path: "go"}} // A build's imports of its own repository alone (design 47 H-A-157): they refine the go.mod's set and never empty it. own := "git.novox.be/novox/mesh-catalog/modules/avahi/lib" - published := map[string]meshcatalog.PackagesByModuleValue{own: {Module: own, Repository: "novox/mesh-catalog", Commit: head, Requires: "r1", + published := map[string]meshcatalog.PackagesByModuleValue{own: {Module: own, Repository: "novox/mesh-catalog", Commit: strings.Repeat("c", 40), Requires: "r1", Packages: []meshcatalog.PackagesByModuleValuePackagesItem{{Package: own, Digest: "d1"}}}} for _, c := range []struct { name string - heads map[string]string held map[string]meshcatalog.ModulesByModuleValue imports []string want string }{ - {"the source names its repository", map[string]string{"novox/mesh-contracts": head}, held, nil, head}, - {"the source names another repository only", map[string]string{"novox/mesh-sdk": head}, held, nil, ""}, - {"a module the indexer holds publishes it", map[string]string{"novox/mesh-contracts": head}, - map[string]meshcatalog.ModulesByModuleValue{"c": {Repository: "novox/mesh-contracts", Commit: strings.Repeat("e", 40), Path: "go/seats/findings"}}, nil, strings.Repeat("e", 40)}, - {"a build's imports naming none of another repository keep the go.mod's", map[string]string{"novox/mesh-contracts": head}, held, []string{own}, head}, + {"a package of another repository none publishes", held, nil, ""}, + {"a module the indexer holds publishes it", map[string]meshcatalog.ModulesByModuleValue{"c": {Repository: "novox/mesh-contracts", Commit: strings.Repeat("e", 40), Path: "go/seats/findings"}}, nil, strings.Repeat("e", 40)}, + {"a build's imports naming none of another repository, which none publishes", held, []string{own}, ""}, } { kept := meshcatalog.ModulesByModuleValueInputs2{Tree: "t", Packages: []string{contracts}, Imports: c.imports} - f := formOf("avahi", "novox/mesh-catalog", &kept, nil, meshcatalog.ModulesByModuleValue{}, c.held, published, nil, c.heads, nil) + f := formOf("avahi", "novox/mesh-catalog", &kept, nil, meshcatalog.ModulesByModuleValue{}, c.held, published, nil, nil) if got := f.versions[contracts]; got != c.want { t.Errorf("%s: version of %s = %q, want %q", c.name, contracts, got, c.want) } - if c.imports == nil { - continue - } - // The build that reported them stands, its held versions empty (a version line read from go.sum). - f = formOf("avahi", "novox/mesh-catalog", &kept, nil, meshcatalog.ModulesByModuleValue{Fp2: "src2:" + strings.Repeat("b", 64)}, c.held, published, nil, c.heads, nil) - if got := f.versions[contracts]; got != c.want || f.fp != "src2:"+strings.Repeat("b", 64) { - t.Errorf("%s, the build standing: version of %s = %q (fp %q), want %q", c.name, contracts, got, f.fp, c.want) + if c.want == "" && c.imports == nil && (f.fp != "" || !strings.HasPrefix(f.none, fingerprint.NoPackageVersion)) { + t.Errorf("%s: fingerprint %q (none %q), want none naming %s", c.name, f.fp, f.none, fingerprint.NoPackageVersion) } } + // A package of the module's own repository no module has published enters by its directory's tree (the tree input), + // so the module has a fingerprint, and the same tree gives the same one whatever commit names it. + sibling := "git.novox.be/novox/mesh-catalog/modules/sibling" + kept := meshcatalog.ModulesByModuleValueInputs2{Tree: "t\nderived modules/sibling o1", Packages: []string{sibling}} + f := formOf("avahi", "novox/mesh-catalog", &kept, nil, meshcatalog.ModulesByModuleValue{}, nil, nil, nil, nil) + if f.fp == "" { + t.Fatalf("an unpublished package of the same repository: none %q, want a fingerprint by its directory's tree", f.none) + } + kept.Tree = "t\nderived modules/sibling o2" + if g := formOf("avahi", "novox/mesh-catalog", &kept, nil, meshcatalog.ModulesByModuleValue{}, nil, nil, nil, nil); g.fp == "" || g.fp == f.fp { + t.Errorf("the sibling's directory changed: fingerprint %q, was %q; want a new one", g.fp, f.fp) + } }) t.Run("formOf clears the import form when it writes the version line", func(t *testing.T) { for _, c := range []struct { @@ -444,7 +449,7 @@ func anUnreadPackagesStateWritesNoFingerprintAndFingerprintsAreComputedAgain(t * imports []string }{{"an import no package holds", []string{meshSDK + "/announce"}}, {"no import reported", nil}} { kept := meshcatalog.ModulesByModuleValueInputs2{Tree: "t", Imports: c.imports, ImportForm: "src2:" + strings.Repeat("a", 64)} - f := formOf("avahi", "novox/mesh-catalog", &kept, nil, meshcatalog.ModulesByModuleValue{}, nil, nil, nil, nil, nil) + f := formOf("avahi", "novox/mesh-catalog", &kept, nil, meshcatalog.ModulesByModuleValue{}, nil, nil, nil, nil) if kept.ImportForm != "" { t.Errorf("%s: kept import form %q after the version line %q", c.name, kept.ImportForm, f.fp) } diff --git a/modules/mesh-delivery-indexer/internal/index/source.go b/modules/mesh-delivery-indexer/internal/index/source.go index a0d2ba74f..55fbf1937 100644 --- a/modules/mesh-delivery-indexer/internal/index/source.go +++ b/modules/mesh-delivery-indexer/internal/index/source.go @@ -24,13 +24,6 @@ type sourceNow struct { heads map[string]string } -// sourceHeads is the head the latest source seen names for each repository (a copy), nil while none has been seen. -func (ix *Indexer) sourceHeads() map[string]string { - ix.mu.Lock() - defer ix.mu.Unlock() - return maps.Clone(ix.source.heads) -} - // wholeAt reads repository's trees at the commit head through the seat git (design 47 H §743): every module directory's manifest, // templates' needs, lock files and packages, and the catalogue's predefined list where the repository is the catalogue's (§769). func (ix *Indexer) wholeAt(ctx context.Context, repository, head string) (whole, error) { diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains.go b/modules/mesh-delivery-indexer/internal/index/toolchains.go index dfba512a6..0a4553916 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains.go @@ -303,11 +303,11 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc r.none = strings.Join(missing, "; ") case noToolchain != "": r.none = noToolchain - if fp, pkgs, ok := stillComputed(module, kept, k, toolchains, held, pub, pins, ix.sourceHeads(), series); ok { + if fp, pkgs, ok := stillComputed(module, kept, k, toolchains, held, pub, pins, series); ok { r = result{fp: fp, tcs: k.held.Toolchains, pkgs: pkgs} } default: - f := formOf(module, k.cur.Repository, &kept, toolchains, k.held, held, pub, pins, ix.sourceHeads(), ix.logf) + f := formOf(module, k.cur.Repository, &kept, toolchains, k.held, held, pub, pins, ix.logf) r.fp, r.none = f.fp, f.none if r.fp != "" { // a pin is a released version, never a commit (H-A-161), and the version line's fingerprint takes it (H-A-167) @@ -510,7 +510,7 @@ func hasContext(m catalogue.Manifest) bool { // with the toolchains it was computed with in place of the languages agreed on none, give it again, in its form. Any // other input that moved (a merge, a base, an agreed toolchain, an imported package) gives another fingerprint, and the // module has none until they agree. -func stillComputed(module string, kept meshcatalog.ModulesByModuleValueInputs2, k *known, agreed map[string]meshcatalogclient.Toolchain2, held map[string]meshcatalog.ModulesByModuleValue, published map[string]meshcatalog.PackagesByModuleValue, pins sdkstep.PackageVersions, heads map[string]string, series map[string]SeriesPackage) (string, sdkstep.PackageVersions, bool) { +func stillComputed(module string, kept meshcatalog.ModulesByModuleValueInputs2, k *known, agreed map[string]meshcatalogclient.Toolchain2, held map[string]meshcatalog.ModulesByModuleValue, published map[string]meshcatalog.PackagesByModuleValue, pins sdkstep.PackageVersions, series map[string]SeriesPackage) (string, sdkstep.PackageVersions, bool) { if k.held.Fp2 == "" { return "", nil, false } @@ -526,7 +526,7 @@ func stillComputed(module string, kept meshcatalog.ModulesByModuleValueInputs2, } toolchains[artifact] = meshcatalogclient.Toolchain2{Language: language, Version: used.Version, Recipe: used.Recipe} } - f := formOf(module, k.held.Repository, &kept, toolchains, k.held, held, published, pins, heads, nil) + f := formOf(module, k.held.Repository, &kept, toolchains, k.held, held, published, pins, nil) if f.none != "" || f.fp == "" { return "", nil, false } From 8bd66112f07534064f5c0ffff2b097627f352ff8 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 01:02:11 +0100 Subject: [PATCH 02/15] I2: a take-in says what it waits on MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A source-moved take-in waiting over a minute on the seat git writes one journal line and its waiting-on (part, verb, since) in the state takeins under running., removed at the take-in's end; past five minutes it raises take-in-waiting (scope merge), cleared when the wait ends. indexer_pending answers the running take-ins beside the held ones. The wait is measured by a context's bound, no timer (ADR 0437 §8). Builds: design 47 H2 §803, H-A-409 --- .../cmd/mesh-delivery-indexer-tools/main.go | 2 +- .../cmd/mesh-delivery-indexer/main.go | 6 +- .../cmd/mesh-delivery-indexer/takein.go | 5 +- .../takein_integration_test.go | 2 +- .../cmd/mesh-delivery-indexer/takein_test.go | 2 +- .../internal/index/candidates.go | 9 +- .../internal/index/dryrun.go | 4 +- .../internal/index/findings.go | 2 +- .../internal/index/held.go | 15 +- .../internal/index/held_test.go | 4 +- .../internal/index/modules.go | 10 +- .../internal/index/moves.go | 4 +- .../internal/index/order.go | 2 +- .../internal/index/package_trees.go | 6 +- .../internal/index/packages.go | 4 +- .../internal/index/source.go | 6 +- .../internal/index/takein_source.go | 3 +- .../internal/index/takein_source_test.go | 111 ++++++++++ .../internal/index/takein_waiting.go | 194 ++++++++++++++++++ .../internal/index/taken.go | 2 +- 20 files changed, 359 insertions(+), 34 deletions(-) create mode 100644 modules/mesh-delivery-indexer/internal/index/takein_waiting.go diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go index a2eba94a8..09fd60658 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go @@ -208,7 +208,7 @@ func main() { _, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.PinDropTool, Verb: "pin-drop"}) return err })...) - tools = append(tools, index.HeldTools(keptRead{stdio.State(index.HeldState)}, asks, time.Now, func(what, why string) error { + tools = append(tools, index.HeldTools(keptRead{stdio.State(index.HeldState)}, keptRead{stdio.State(index.TakeInsState)}, asks, time.Now, func(what, why string) error { _, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.TakeInSkipTool, Verb: "take-in-skip"}) return err })...) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index c87463d79..aca38fc31 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -331,7 +331,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return fmt.Errorf("answering the verb input: %w", err) } defer stopInput() - ix := &index.Indexer{Module: cred.Module, NamedSets: named, Findings: writer, OldSections: awaitedStore{b: oldSections}, Requirements: awaitedStore{b: requirements}, Store: kvStore{candidates}, Modules: modules, Manifests: kvStore{manifests}, Taken: awaitedStore{b: taken}, Builds: known, + // The state takeins: hand acts ask there (watched below), and a running take-in keeps what it waits on (H2 §803, H-A-409). + takeIns := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.TakeInsState), "no hand act of the indexer is answered", logger.Printf) + ix := &index.Indexer{Module: cred.Module, TakeIns: awaitedStore{b: takeIns}, NamedSets: named, Findings: writer, OldSections: awaitedStore{b: oldSections}, Requirements: awaitedStore{b: requirements}, Store: kvStore{candidates}, Modules: modules, Manifests: kvStore{manifests}, Taken: awaitedStore{b: taken}, Builds: known, Predefined: awaitedStore{b: predefined}, Logf: logger.Printf, Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), BranchHead: branchHeadVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call), Packages: awaitedStore{b: packages, js: js, name: index.Bucket(cred.Module, index.PackagesState)}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call), @@ -502,7 +504,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro done() // A person's hand act writes the takeins state (issue 1945): watched, as every other input. // A state not on the bus yet at this start is watched once it appears (issue 1790). - process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.TakeInsState), "no hand act of the indexer is answered", logger.Printf).Ready(func(kv jetstream.KeyValue) { + takeIns.Ready(func(kv jetstream.KeyValue) { eachTakeIn := func(e jetstream.KeyValueEntry) { takenIn(ctx, actor{ix, sources}, kv, e, logger.Printf) } if tw, err := settle(ctx, kv, eachTakeIn); err != nil { process.NotWatched(logger.Printf, index.Bucket(cred.Module, index.TakeInsState), err, "no hand act of the indexer is answered until the indexer starts again") diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go index 31c7f4a84..11a52cd33 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "strings" "time" "github.com/nats-io/nats.go/jetstream" @@ -32,8 +33,8 @@ type takerIn interface { // written back, by compare-and-set at its revision, with what the indexer stored or why not. A value // answered already (this indexer's own write) or deleted is left. func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream.KeyValueEntry, logf func(string, ...any)) { - if e.Operation() != jetstream.KeyValuePut { - return + if e.Operation() != jetstream.KeyValuePut || strings.HasPrefix(e.Key(), index.RunningPrefix) { + return // a running take-in's waiting-on is the indexer's own record, no ask (H-A-409) } a, err := index.DecodeTakeIn(e.Value()) if err != nil { diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go index 3055a5ac7..474cfc4f8 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go @@ -592,7 +592,7 @@ func TestIndexerWhole(t *testing.T) { t.Errorf("main 14 at main 13's head said %+v, want no mover", next) } var log []string - tools := index.HeldTools(kvStore{held}, askVia(t, ctx, takeinsKV), time.Now, func(what, why string) error { + tools := index.HeldTools(kvStore{held}, kvStore{takeinsKV}, askVia(t, ctx, takeinsKV), time.Now, func(what, why string) error { log = append(log, what+": "+why) return nil }) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_test.go index 713eb631d..495844ebf 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_test.go @@ -92,7 +92,7 @@ func TestEveryHandToolsAskIsReadAndActedOn(t *testing.T) { } record := func(string, string) error { return nil } tools := map[string]func(map[string]any) (any, error){} - for _, tl := range slices.Concat(index.HeldTools(nil, ask, time.Now, record), index.PinsTools(nil, nil, ask, time.Now, record), + for _, tl := range slices.Concat(index.HeldTools(nil, nil, ask, time.Now, record), index.PinsTools(nil, nil, ask, time.Now, record), index.RunsTools(nil, nil, ask, time.Now, record), index.NamedTools(ask, time.Now, record), index.FingerprintsTools(ask, time.Now, record, time.Minute), index.CatalogRevisionTools(nil, ask, time.Now, record), index.RegistrationTools(nil, nil, ask, time.Now, func(string, string, string, string) error { return nil })) { diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index b880cdb9c..90dbbde3c 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -126,8 +126,13 @@ type Indexer struct { // content; nil reads none. GoMod func(ctx context.Context, repository, sha, dir string) ([]byte, error) // Now is the clock of a taken value; nil is time.Now. - Now func() time.Time - Builds *Builds + Now func() time.Time + // Bound bounds a running take-in's wait (takein_waiting.go), a context's bound and no timer (ADR 0437 §8); nil is + // context.WithTimeout. + Bound func(context.Context, time.Duration) (context.Context, context.CancelFunc) + // TakeIns is the state takeins, where a running take-in keeps what it waits on (design 47 H2 §803, H-A-409); nil keeps none. + TakeIns Store + Builds *Builds // Named asks the forge for the git object of each named path of the module directories given, at a commit // (the seat git's trees with paths; novox/hq ADR 0499 clauses 2 and 3): the indexer is the one reader of a // manifest's paths (ADR 0495 clause 1, the lead's decision L8 of R4e). Nil asks nothing, and a module naming paths diff --git a/modules/mesh-delivery-indexer/internal/index/dryrun.go b/modules/mesh-delivery-indexer/internal/index/dryrun.go index bdf7a4bdb..2fe12d418 100644 --- a/modules/mesh-delivery-indexer/internal/index/dryrun.go +++ b/modules/mesh-delivery-indexer/internal/index/dryrun.go @@ -29,7 +29,9 @@ func (ix *Indexer) dry() *dryRun { } d := &dryRun{} x := &Indexer{Module: ix.Module, Manifests: ix.Manifests, Packages: ix.Packages, JudgeID: ix.JudgeID, Files: ix.Files, GoSum: ix.GoSum, - GoMod: ix.GoMod, Now: ix.Now, Builds: ix.Builds, Named: ix.Named, Trunk: ix.Trunk, BranchHead: ix.BranchHead, Read: ix.Read, Contains: ix.Contains, + GoMod: ix.GoMod, Now: ix.Now, Bound: ix.Bound, + // A dry run keeps no waiting-on: the take-in it computes for says its waits through the live indexer (H-A-409). + TakeIns: nil, Builds: ix.Builds, Named: ix.Named, Trunk: ix.Trunk, BranchHead: ix.BranchHead, Read: ix.Read, Contains: ix.Contains, SayTakeIn: nil, // The registry milestone's fields: what a take-in would write on the bus stays out of a dry run, so the states of the // requirements, findings, seat definitions, holds and their bounds are not overlaid, nothing is said, and no hold is diff --git a/modules/mesh-delivery-indexer/internal/index/findings.go b/modules/mesh-delivery-indexer/internal/index/findings.go index 542c707b2..f21c16227 100644 --- a/modules/mesh-delivery-indexer/internal/index/findings.go +++ b/modules/mesh-delivery-indexer/internal/index/findings.go @@ -22,7 +22,7 @@ const CallUnansweredKind = "indexer-call-unanswered" // H-A-242, H-A-247): its own, the process reader's event-unhandled and the family-not-heard of the heard writer it is handed // (H-A-247 (1)); a views reader's and the emitter's (views.ViewKinds) are always the writer's besides. The tool runner's // writer over the same state `findings` owns its own. TestFindingKindsHoldEveryKindTheCodeRaises holds the code to this list. -var FindingKinds = []string{PinMissingKind, OutOfStepKind, PackagesNotPublishedKind, HistoryFillingKind, SeedWaitsKind, +var FindingKinds = []string{TakeInWaitingKind, PinMissingKind, OutOfStepKind, PackagesNotPublishedKind, HistoryFillingKind, SeedWaitsKind, SeedLeftOutKind, SeatDefinitionUnparsedKind, SeatDefinedTwiceKind, SeatOutsideItsDomainKind, CallUnansweredKind, views.EventUnhandledKind, heard.Kind} diff --git a/modules/mesh-delivery-indexer/internal/index/held.go b/modules/mesh-delivery-indexer/internal/index/held.go index 5e8774ad8..ca1e685da 100644 --- a/modules/mesh-delivery-indexer/internal/index/held.go +++ b/modules/mesh-delivery-indexer/internal/index/held.go @@ -49,6 +49,8 @@ func (h HeldTakeIn) Event() meshdelivery.SourceMovedEvent { // PendingAnswer is indexer_pending's answer (H-A-102): every branch's held take-ins, in the state held's form (since in RFC 3339). type PendingAnswer struct { Held []HeldTakeIn `json:"held"` + // Running are the take-ins that run, each with what it waits on (design 47 H2 §803, H-A-409). + Running []RunningTakeIn `json:"running,omitempty"` } // ErrHeldUnwritten says a source-moved that could not be taken in is not written into the state held either: its message @@ -224,20 +226,25 @@ func ReadHeld(ctx context.Context, st Store) ([]HeldTakeIn, error) { return out, nil } -// HeldTools are the held take-in's tools (§744): indexer_pending reads the state held; indexer_take_in_pending takes a branch's +// HeldTools are the held take-in's tools (§744): indexer_pending reads the state held and the running take-ins' waiting-on in +// the state takeins (H2 §803, H-A-409); indexer_take_in_pending takes a branch's // held source-moved again now and indexer_take_in_skip acknowledges it unwritten, each an ask in the state takeins the indexer // acts on (H-A-97), the skip a hand act recorded through mesh-hand-acts before it is asked. -func HeldTools(held Store, ask func(key string, value []byte) (TakeInAsk, error), now func() time.Time, record func(what, why string) error) []stdio.Tool { +func HeldTools(held, takeIns Store, ask func(key string, value []byte) (TakeInAsk, error), now func() time.Time, record func(what, why string) error) []stdio.Tool { str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) } return []stdio.Tool{ {Name: PendingTool, Description: "Each watched branch's held take-in (design 47 H §744): the source revision of the gate's source-moved the indexer " + "could not take in (the catalog store full, a read of the forge unanswered), why and since when, in order, each kept here once " + - "acknowledged and taken in again on the indexer's room and at its start. indexer_take_in_pending takes it again now, indexer_take_in_skip ends it unwritten.", + "acknowledged and taken in again on the indexer's room and at its start; and each running take-in that waits over a minute on another part, its waiting-on (part, verb, since). indexer_take_in_pending takes it again now, indexer_take_in_skip ends it unwritten.", Input: map[string]any{"type": "object", "properties": map[string]any{}}, Run: func(map[string]any) (any, error) { h, err := ReadHeld(context.Background(), held) - return PendingAnswer{Held: h}, err + if err != nil { + return PendingAnswer{Held: h}, err + } + r, err := ReadRunning(context.Background(), takeIns) + return PendingAnswer{Held: h, Running: r}, err }}, {Name: TakeInPendingTool, Description: "Take a watched branch's held source-moved in again now (design 47 H §744), as the indexer's room would: " + diff --git a/modules/mesh-delivery-indexer/internal/index/held_test.go b/modules/mesh-delivery-indexer/internal/index/held_test.go index 976e29eb0..fd9c77ca0 100644 --- a/modules/mesh-delivery-indexer/internal/index/held_test.go +++ b/modules/mesh-delivery-indexer/internal/index/held_test.go @@ -26,7 +26,7 @@ func TestTheHeldAndPinToolsAskWhatTheIndexerReads(t *testing.T) { } record := func(what, _ string) error { recorded = append(recorded, what); return nil } tools := map[string]stdio.Tool{} - for _, tl := range append(HeldTools(newMem(), ask, time.Now, record), PinsTools(newMem(), newMemObjects(0), ask, time.Now, record)...) { + for _, tl := range append(HeldTools(newMem(), nil, ask, time.Now, record), PinsTools(newMem(), newMemObjects(0), ask, time.Now, record)...) { tools[tl.Name] = tl } for _, c := range []struct { @@ -56,7 +56,7 @@ func TestTheHeldAndPinToolsAskWhatTheIndexerReads(t *testing.T) { held := newMem() ix := &Indexer{Held: held} ix.hold(t.Context(), meshdelivery.SourceMovedEvent{Branch: "main", Source: 4, Call: "0123456789abcdef"}, "the catalog store is full") - got, err := HeldTools(held, ask, time.Now, record)[0].Run(map[string]any{}) + got, err := HeldTools(held, nil, ask, time.Now, record)[0].Run(map[string]any{}) data, _ := json.Marshal(got) var answer struct { Held []struct { diff --git a/modules/mesh-delivery-indexer/internal/index/modules.go b/modules/mesh-delivery-indexer/internal/index/modules.go index 9a59b83d6..60321160f 100644 --- a/modules/mesh-delivery-indexer/internal/index/modules.go +++ b/modules/mesh-delivery-indexer/internal/index/modules.go @@ -227,7 +227,7 @@ func (ix *Indexer) namedObjects(ctx context.Context, f whole) (map[string]map[st if len(need) == 0 || ix.Named == nil { return objects, nil } - got, err := ix.Named(ctx, f.Repository, f.Merge, need) + got, err := waited(ctx, "git", "trees", func() (map[string]map[string]string, error) { return ix.Named(ctx, f.Repository, f.Merge, need) }) if err != nil { words := map[string]string{} for dir := range need { @@ -320,7 +320,7 @@ func (ix *Indexer) reconcile(ctx context.Context) error { if len(mine) == 0 { continue } - dirs, err := ix.Trunk(ctx, repository, heads[repository]) + dirs, err := waited(ctx, "git", "trees", func() ([]string, error) { return ix.Trunk(ctx, repository, heads[repository]) }) if err != nil { ix.logf("%s at %s: the modules of the trunk cannot be read, so none is dropped: %v", repository, heads[repository], err) continue @@ -346,7 +346,7 @@ func (ix *Indexer) reconcile(ctx context.Context) error { if m.Commit == "" || ix.Contains == nil { return true } - in, err := ix.Contains(ctx, repository, m.Commit, head) + in, err := waited(ctx, "git", "contains", func() (bool, error) { return ix.Contains(ctx, repository, m.Commit, head) }) if err != nil { ix.logf("%s: whether %s is in %s cannot be read, so the module at %s is kept: %v", repository, m.Commit, head, m.Path, err) return true @@ -421,7 +421,7 @@ func (ix *Indexer) goModOf(ctx context.Context, f whole, dir string) ([]byte, bo if ix.GoMod == nil { return nil, false } - raw, err := ix.GoMod(ctx, f.Repository, f.Merge, dir) + raw, err := waited(ctx, "git", "file", func() ([]byte, error) { return ix.GoMod(ctx, f.Repository, f.Merge, dir) }) if err != nil { ix.logf("%s at %s: the go.mod of %q cannot be read, so its module keeps the packages it held: %v", f.Repository, f.Merge, dir, err) return nil, false @@ -444,7 +444,7 @@ func (ix *Indexer) requirementsOf(ctx context.Context, f whole, dir string, cur } } } - raw, err := ix.GoSum(ctx, f.Repository, f.Merge, dir) + raw, err := waited(ctx, "git", "file", func() ([]byte, error) { return ix.GoSum(ctx, f.Repository, f.Merge, dir) }) if err != nil { ix.logf("%s at %s: the go.sum of %q cannot be read, so what it requires is not kept: %v", f.Repository, f.Merge, dir, err) return diff --git a/modules/mesh-delivery-indexer/internal/index/moves.go b/modules/mesh-delivery-indexer/internal/index/moves.go index 03c7de2b6..17088ef94 100644 --- a/modules/mesh-delivery-indexer/internal/index/moves.go +++ b/modules/mesh-delivery-indexer/internal/index/moves.go @@ -26,7 +26,7 @@ func (ix *Indexer) heldByAnotherTrunk(ctx context.Context, module, repository st if head == "" || trunk.SameRepository(other, repository) { continue } - dirs, err := ix.Trunk(ctx, other, head) + dirs, err := waited(ctx, "git", "trees", func() ([]string, error) { return ix.Trunk(ctx, other, head) }) if err != nil { ix.logf("%s at %s: the modules of the trunk cannot be read, so %s is not looked for there: %v", other, head, module, err) continue @@ -36,7 +36,7 @@ func (ix *Indexer) heldByAnotherTrunk(ctx context.Context, module, repository st if nameOfDirectory(dir) != module { continue } - read, err := ix.Read(ctx, other, head, dir) + read, err := waited(ctx, "git", "trees", func() (RepoRead, error) { return ix.Read(ctx, other, head, dir) }) if err != nil { ix.logf("%s at %s: %q cannot be read, so %s is not handed over from it: %v", other, head, dir, module, err) continue diff --git a/modules/mesh-delivery-indexer/internal/index/order.go b/modules/mesh-delivery-indexer/internal/index/order.go index 395cebd96..90a99491f 100644 --- a/modules/mesh-delivery-indexer/internal/index/order.go +++ b/modules/mesh-delivery-indexer/internal/index/order.go @@ -67,7 +67,7 @@ func (ix *Indexer) older(ctx context.Context, repository, commit, later string) if ix.Contains == nil || commit == "" || later == "" || commit == later { return false, nil } - in, err := ix.Contains(ctx, repository, commit, later) + in, err := waited(ctx, "git", "contains", func() (bool, error) { return ix.Contains(ctx, repository, commit, later) }) if notFound(err) { ix.logf("%s: the forge does not have %.12s or %.12s (a rewritten history?), so %.12s is not known to be older and is written: %v", repository, commit, later, commit, err) return false, nil diff --git a/modules/mesh-delivery-indexer/internal/index/package_trees.go b/modules/mesh-delivery-indexer/internal/index/package_trees.go index 128150fff..9cb36c2e6 100644 --- a/modules/mesh-delivery-indexer/internal/index/package_trees.go +++ b/modules/mesh-delivery-indexer/internal/index/package_trees.go @@ -36,7 +36,7 @@ func (t *goModTree) goMod(dir string) []byte { var raw []byte if t.ix.GoMod != nil { var err error - if raw, err = t.ix.GoMod(t.ctx, t.f.Repository, t.f.Merge, dir); err != nil { + if raw, err = waited(t.ctx, "git", "file", func() ([]byte, error) { return t.ix.GoMod(t.ctx, t.f.Repository, t.f.Merge, dir) }); err != nil { t.ix.logf("%s at %s: the go.mod of %q cannot be read, so no module of the tree is found there: %v", t.f.Repository, t.f.Merge, dir, err) raw = nil } @@ -167,7 +167,9 @@ func (ix *Indexer) derivedTree(ctx context.Context, f whole, t *goModTree, m cat if ix.Named == nil { return "", "the forge cannot be asked the git objects of the derived directories " + strings.Join(dirs, ", ") } - got, err := ix.Named(ctx, f.Repository, f.Merge, map[string][]string{"": dirs}) + got, err := waited(ctx, "git", "trees", func() (map[string]map[string]string, error) { + return ix.Named(ctx, f.Repository, f.Merge, map[string][]string{"": dirs}) + }) if err != nil { return "", fmt.Sprintf("the git objects of the derived directories %s: %v", strings.Join(dirs, ", "), err) } diff --git a/modules/mesh-delivery-indexer/internal/index/packages.go b/modules/mesh-delivery-indexer/internal/index/packages.go index d898e2c9a..a730f4d14 100644 --- a/modules/mesh-delivery-indexer/internal/index/packages.go +++ b/modules/mesh-delivery-indexer/internal/index/packages.go @@ -174,7 +174,7 @@ func (ix *Indexer) packagesIn(ctx context.Context, repository, commit string, mo if len(ask) == 0 { return nil } - got, err := ix.Files(ctx, repository, commit, ask) + got, err := waited(ctx, "git", "trees", func() (map[string]map[string]string, error) { return ix.Files(ctx, repository, commit, ask) }) if err != nil { return fmt.Errorf("the forge's listing of %s at %s (%s): %w", repository, commit, strings.Join(ask, ", "), err) } @@ -231,7 +231,7 @@ func (ix *Indexer) packagesIn(ctx context.Context, repository, commit string, mo if ix.Trunk == nil { return nil, nil, whole } - if _, err := ix.Trunk(ctx, repository, commit); err != nil { + if _, err := waited(ctx, "git", "trees", func() ([]string, error) { return ix.Trunk(ctx, repository, commit) }); err != nil { return nil, nil, fmt.Errorf("%w (and the commit's trees: %v)", whole, err) } ix.logf("%s at %s: the commit holds a tree, so every directory asked (%s) is gone", repository, commit, strings.Join(dirs, ", ")) diff --git a/modules/mesh-delivery-indexer/internal/index/source.go b/modules/mesh-delivery-indexer/internal/index/source.go index 55fbf1937..ea7531504 100644 --- a/modules/mesh-delivery-indexer/internal/index/source.go +++ b/modules/mesh-delivery-indexer/internal/index/source.go @@ -32,9 +32,9 @@ func (ix *Indexer) wholeAt(ctx context.Context, repository, head string) (whole, if ix.ReadFile == nil { return nil, errors.New("the indexer reads no file of the forge") } - return ix.ReadFile(ctx, repository, head, p) + return waited(ctx, "git", "file", func() ([]byte, error) { return ix.ReadFile(ctx, repository, head, p) }) } - first, err := ix.Read(ctx, repository, head, "") + first, err := waited(ctx, "git", "trees", func() (RepoRead, error) { return ix.Read(ctx, repository, head, "") }) if err != nil { return f, fmt.Errorf("the seat git could not read %s at %.12s: %w", repository, head, err) } @@ -42,7 +42,7 @@ func (ix *Indexer) wholeAt(ctx context.Context, repository, head string) (whole, for _, dir := range slices.Sorted(maps.Keys(first.Trees)) { read := first if dir != "" { - if read, err = ix.Read(ctx, repository, head, dir); err != nil { + if read, err = waited(ctx, "git", "trees", func() (RepoRead, error) { return ix.Read(ctx, repository, head, dir) }); err != nil { return f, fmt.Errorf("the seat git could not read %q of %s at %.12s: %w", dir, repository, head, err) } } diff --git a/modules/mesh-delivery-indexer/internal/index/takein_source.go b/modules/mesh-delivery-indexer/internal/index/takein_source.go index 667eb11a2..ecb392210 100644 --- a/modules/mesh-delivery-indexer/internal/index/takein_source.go +++ b/modules/mesh-delivery-indexer/internal/index/takein_source.go @@ -38,7 +38,8 @@ func TakenID(branch string, source int64) string { // ErrObjectsFull waits for the indexer's own room, any other is the failed read or write it says; ErrOldForm is never taken in // and said as no take-in. func (ix *Indexer) TakeInSource(ctx context.Context, e meshdelivery.SourceMovedEvent) (meshcatalog.TakeInTaken, error) { - taken, err := ix.takeInSource(ctx, e) + taken, err := ix.takeInSource(ix.withTakeIn(ctx, e.Branch, e.Source), e) + ix.endTakeIn(ctx, e.Source) switch { case err == nil: ix.unhold(ctx, e.Branch, e.Source) diff --git a/modules/mesh-delivery-indexer/internal/index/takein_source_test.go b/modules/mesh-delivery-indexer/internal/index/takein_source_test.go index 2e439e85d..f0da42a54 100644 --- a/modules/mesh-delivery-indexer/internal/index/takein_source_test.go +++ b/modules/mesh-delivery-indexer/internal/index/takein_source_test.go @@ -6,7 +6,14 @@ import ( "errors" "fmt" "reflect" + "slices" + "strings" + "sync" "testing" + "time" + + mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" @@ -98,6 +105,110 @@ func TestASourceRevisionIsTakenInAsOneRevision(t *testing.T) { if _, err := ix.TakeInSource(ctx, old); !errors.Is(err, ErrOldForm) { t.Errorf("a source-moved with no call answered %v, want ErrOldForm", err) } + // Design 47 H2 §803, H-A-409: a take-in says what it waits on, in the journal, in takeins and past five minutes as a finding. + t.Run("a take-in says what it waits on", func(t *testing.T) { + running := newMem() + ix.TakeIns = running + ix.Findings = newFindings() + // The take-in's end removes its waiting-on. + if err := running.Create(ctx, RunningKey(10), []byte(`{"source":10}`)); err != nil { + t.Fatal(err) + } + next := moved + next.Source, next.Call = 10, "00112233445566ff" + if _, err := ix.TakeInSource(ctx, next); err != nil { + t.Fatal(err) + } + if _, _, found, _ := running.Get(ctx, RunningKey(10)); found { + t.Error("the take-in's end left its waiting-on in takeins") + } + start := time.Date(2026, 11, 22, 1, 0, 0, 0, time.UTC) + now := start + timers := make(chan context.CancelFunc) + var mu sync.Mutex + var lines []string + wrote := make(chan struct{}, 1) + findings := newFindings() + w := &Indexer{Module: testSelf, TakeIns: running, Findings: findings, Now: func() time.Time { mu.Lock(); defer mu.Unlock(); return now }, + Bound: func(parent context.Context, _ time.Duration) (context.Context, context.CancelFunc) { + b, cancel := context.WithCancel(parent) + timers <- cancel + return b, cancel + }, + Logf: func(format string, args ...any) { + mu.Lock() + defer mu.Unlock() + lines = append(lines, fmt.Sprintf(format, args...)) + }} + w.Findings = findingsSignal{findings, wrote} + listing := func(answerAfter time.Duration) { + t.Helper() + mu.Lock() + now, lines = start, nil + mu.Unlock() + release, res := make(chan struct{}), make(chan error) + go func() { + _, err := waited(w.withTakeIn(ctx, "main", 9), "git", "trees", func() (int, error) { <-release; return 1, nil }) + res <- err + }() + first := <-timers + if answerAfter < WaitingLineAfter { + close(release) + <-res + return + } + mu.Lock() + now = start.Add(70 * time.Second) + mu.Unlock() + first() + second := <-timers // asked once the line is said and the waiting-on written + if answerAfter >= WaitingFindingAfter { + second() + <-wrote + if h := findings.Held(); len(h) != 1 || h[0].Kind != TakeInWaitingKind || h[0].Scope != mcc.ScopeMerge || h[0].ID != "9" { + t.Errorf("after %s: findings %+v, want take-in-waiting of merge 9", answerAfter, h) + } + } + close(release) + if err := <-res; err != nil { + t.Fatal(err) + } + } + for _, c := range []struct { + after time.Duration + lines int + finding bool + }{{30 * time.Second, 0, false}, {70 * time.Second, 1, false}, {6 * time.Minute, 1, false}} { + listing(c.after) + mu.Lock() + got := slices.Clone(lines) + mu.Unlock() + if len(got) != c.lines || (c.lines == 1 && !strings.Contains(got[0], "waits on git trees since 2026-11-22T01:00:00Z")) { + t.Errorf("answered after %s: lines %q, want %d naming git trees and since when", c.after, got, c.lines) + } + r, err := ReadRunning(ctx, running) + want := c.lines == 1 + if err != nil || want != (len(r) == 1) || (want && (r[0].Source != 9 || r[0].Branch != "main" || r[0].WaitingOn != WaitingOn{Part: "git", Verb: "trees", Since: start})) { + t.Errorf("answered after %s: running %+v (%v), want waiting-on git trees since %s: %v", c.after, r, err, start, want) + } + if len(findings.Held()) != 0 { + t.Errorf("answered after %s: take-in-waiting still raised: %+v", c.after, findings.Held()) + } + w.endTakeIn(ctx, 9) + } + }) +} + +// findingsSignal is a finding writer that says each write it passes on. +type findingsSignal struct { + *memFindings + wrote chan struct{} +} + +func (f findingsSignal) Write(ctx context.Context, x mcseat.Finding) error { + err := f.memFindings.Write(ctx, x) + f.wrote <- struct{}{} + return err } // Design 47 H §768, §743, H-A-19: a branch's first source revision (start true) is taken in with no mover, says take-in.taken diff --git a/modules/mesh-delivery-indexer/internal/index/takein_waiting.go b/modules/mesh-delivery-indexer/internal/index/takein_waiting.go new file mode 100644 index 000000000..e732b658f --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/takein_waiting.go @@ -0,0 +1,194 @@ +package index + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strconv" + "time" + + mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" +) + +// A take-in says what it waits on (design 47 H2 §803, H-A-409): a source-moved take-in that waits longer than +// WaitingLineAfter on another part (the seat git's listing or a file, a package's versions, a registry) writes one journal +// line naming the part, the verb and since when, and keeps its waiting-on in the state takeins under the key +// RunningPrefix and the source revision, rewritten at each new wait and removed at the take-in's end; past +// WaitingFindingAfter it raises the finding take-in-waiting (scope merge, id the source revision), cleared when the wait +// ends. indexer_pending answers it beside the held take-ins. +const ( + TakeInWaitingKind = "take-in-waiting" + RunningPrefix = "running." + WaitingLineAfter = time.Minute + WaitingFindingAfter = 5 * time.Minute +) + +// WaitingOn is what a running take-in waits on: the part, its verb and since when. +type WaitingOn struct { + Part string `json:"part"` + Verb string `json:"verb"` + Since time.Time `json:"since"` +} + +// RunningTakeIn is the value of a running take-in in the state takeins (H-A-409). +type RunningTakeIn struct { + Branch string `json:"branch"` + Source int64 `json:"source"` + WaitingOn WaitingOn `json:"waiting-on"` +} + +// RunningKey is the takeins key of the running take-in of a source revision. +func RunningKey(source int64) string { return RunningPrefix + strconv.FormatInt(source, 10) } + +type takeInRunKey struct{} + +// takeInRun is the running take-in a context carries, and the live indexer that says its waits (a branch's take-in +// computes on a dry copy, which writes nothing). +type takeInRun struct { + ix *Indexer + branch string + source int64 +} + +// withTakeIn is ctx carrying the running take-in of the source revision of branch. +func (ix *Indexer) withTakeIn(ctx context.Context, branch string, source int64) context.Context { + return context.WithValue(ctx, takeInRunKey{}, &takeInRun{ix: ix, branch: branch, source: source}) +} + +// endTakeIn removes the running take-in's value from the state takeins, when one was written. +func (ix *Indexer) endTakeIn(ctx context.Context, source int64) { + if ix.TakeIns == nil { + return + } + key := RunningKey(source) + _, rev, found, err := ix.TakeIns.Get(ctx, key) + if err == nil && found { + err = ix.TakeIns.Delete(ctx, key, rev) + } + if err != nil && !errors.Is(err, ErrConflict) { + ix.logf("take-in: source revision %d: its waiting-on is not removed from %s: %v", source, TakeInsState, err) + } +} + +// bound is ctx bounded by d, a context's bound and no timer (ADR 0437 §8, as fingerprints.go bounds its wait); Bound +// when set. +func (ix *Indexer) bound(ctx context.Context, d time.Duration) (context.Context, context.CancelFunc) { + if ix.Bound != nil { + return ix.Bound(ctx, d) + } + return context.WithTimeout(ctx, d) +} + +// passed is whether the wait outlasted the bound b: false once it ended (done) or the take-in's own context ended. +func passed(ctx, b context.Context, done <-chan struct{}) bool { + select { + case <-done: + return false + case <-b.Done(): + } + select { + case <-done: + return false + default: + } + return ctx.Err() == nil +} + +// waited is f, a call on part's verb: within a running take-in (ctx), a wait past WaitingLineAfter is said and kept, one +// past WaitingFindingAfter raises take-in-waiting until it ends. Outside a take-in it is f alone. +func waited[T any](ctx context.Context, part, verb string, f func() (T, error)) (T, error) { + run, ok := ctx.Value(takeInRunKey{}).(*takeInRun) + if !ok { + return f() + } + done, finished := make(chan struct{}), make(chan struct{}) + go func() { + defer close(finished) + run.ix.watchWait(ctx, run, part, verb, done) + }() + v, err := f() + close(done) + <-finished + return v, err +} + +func (ix *Indexer) watchWait(ctx context.Context, run *takeInRun, part, verb string, done <-chan struct{}) { + since := ix.now().UTC() + line, cancel := ix.bound(ctx, WaitingLineAfter) + defer cancel() + if !passed(ctx, line, done) { + return + } + ix.logf("take-in: source revision %d of %s waits on %s %s since %s", run.source, run.branch, part, verb, since.Format(time.RFC3339)) + ix.writeWaiting(ctx, RunningTakeIn{Branch: run.branch, Source: run.source, WaitingOn: WaitingOn{Part: part, Verb: verb, Since: since}}) + id := strconv.FormatInt(run.source, 10) + finding, cancelFinding := ix.bound(ctx, WaitingFindingAfter-WaitingLineAfter) + defer cancelFinding() + if !passed(ctx, finding, done) { + return + } + if ix.Findings != nil { + f := ix.observed(mcc.ScopeMerge, id, TakeInWaitingKind, + fmt.Sprintf("the take-in of source revision %d of %s waits on %s %s", run.source, run.branch, part, verb), + "A take-in waits on another part", + fmt.Sprintf("The take-in of source revision %d of %s has waited on %s's %s since %s, so the revision is not taken in.", run.source, run.branch, part, verb, since.Format(time.RFC3339)), + "indexer."+TakeInWaitingKind) + if err := ix.Findings.Write(ctx, f); err != nil { + ix.logf("take-in: the finding %s is not written: %v", TakeInWaitingKind, err) + } + } + <-done + if ix.Findings != nil { + if err := ix.clearFinding(context.WithoutCancel(ctx), mcc.ScopeMerge, id, TakeInWaitingKind, part+" "+verb+" answered"); err != nil { + ix.logf("take-in: the finding %s is not cleared: %v", TakeInWaitingKind, err) + } + } +} + +// writeWaiting writes the running take-in's value in the state takeins, over the one it holds. +func (ix *Indexer) writeWaiting(ctx context.Context, v RunningTakeIn) { + if ix.TakeIns == nil { + return + } + raw, _ := json.Marshal(v) + key := RunningKey(v.Source) + _, rev, found, err := ix.TakeIns.Get(ctx, key) + switch { + case err != nil: + case found: + err = ix.TakeIns.Update(ctx, key, raw, rev) + default: + err = ix.TakeIns.Create(ctx, key, raw) + } + if err != nil { + ix.logf("take-in: source revision %d: its waiting-on is not written in %s: %v", v.Source, TakeInsState, err) + } +} + +// ReadRunning is every running take-in's value of the state takeins (indexer_pending's running part); none for no state. +func ReadRunning(ctx context.Context, st Store) ([]RunningTakeIn, error) { + if st == nil { + return nil, nil + } + keys, err := st.Keys(ctx) + if err != nil { + return nil, err + } + var out []RunningTakeIn + for _, k := range keys { + if len(k) <= len(RunningPrefix) || k[:len(RunningPrefix)] != RunningPrefix { + continue + } + raw, _, found, err := st.Get(ctx, k) + if err != nil { + return nil, err + } + var r RunningTakeIn + if !found || json.Unmarshal(raw, &r) != nil { + continue + } + out = append(out, r) + } + return out, nil +} diff --git a/modules/mesh-delivery-indexer/internal/index/taken.go b/modules/mesh-delivery-indexer/internal/index/taken.go index 3d5df687b..4727a25c3 100644 --- a/modules/mesh-delivery-indexer/internal/index/taken.go +++ b/modules/mesh-delivery-indexer/internal/index/taken.go @@ -43,7 +43,7 @@ func (ix *Indexer) writeTakenKnown(ctx context.Context, f whole, notOlder string return nil, false, nil } if ix.Contains != nil && cur.Merge != notOlder { - later, err := ix.Contains(ctx, f.Repository, f.Merge, cur.Merge) + later, err := waited(ctx, "git", "contains", func() (bool, error) { return ix.Contains(ctx, f.Repository, f.Merge, cur.Merge) }) if err != nil { ix.logf("%s: whether %s is in %s cannot be read, so taken.%s stays at %s: %v", f.Repository, f.Merge, cur.Merge, key, cur.Merge, err) return nil, false, nil From 62e8274bf9d6ca1ff75a80188b0e39e0d2a611af Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 01:02:58 +0100 Subject: [PATCH 03/15] R5: a mover left unfinished by another's failure is put in a pipeline again MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A pipeline ending failed or cancelled keeps each mover whose own required steps passed or never ran in the runner's state unfinished and names it in pipeline-ended's unfinished; the branch's next kept catalog revision carries it, or starts a pipeline for it alone. runner_unfinished reads the state, runner_unfinished_drop drops one key, runner_unfinished_place asks the new seat verb place-unfinished (contracts regenerated), both hand acts. Builds: design 47 H2 §786, §820 (unfinished), H-A-398 --- .../go/seats/meshpipelinerunner/contract.go | 103 ++++++-- .../seats/mesh-pipeline-runner/seat.json | 41 +++ .../ts/seats/meshpipelinerunner/index.d.ts | 26 ++ .../ts/seats/meshpipelinerunner/index.js | 24 ++ .../cmd/mesh-delivery-pipeline-tools/main.go | 1 + .../mesh-delivery-pipeline-tools/main_test.go | 51 +++- .../unfinished.go | 105 ++++++++ .../mesh-delivery-pipeline-tools/verbtools.go | 13 +- .../verbtools_test.go | 4 + .../cmd/mesh-delivery-pipeline/main.go | 10 +- .../queue_integration_test.go | 12 +- .../run_integration_test.go | 236 ++++++++++-------- .../testruns_integration_test.go | 2 +- .../cmd/mesh-delivery-pipeline/unfinished.go | 49 ++++ .../unfinished_integration_test.go | 143 +++++++++++ .../cmd/mesh-delivery-pipeline/verbs.go | 22 +- .../cmd/mesh-delivery-pipeline/verbs_test.go | 2 +- .../internal/pipeline/ended_test.go | 52 +++- .../internal/pipeline/moverkeys.go | 5 +- .../internal/pipeline/released.go | 2 +- .../internal/pipeline/start.go | 62 ++++- .../internal/pipeline/start_test.go | 44 ++++ .../internal/pipeline/steps.go | 8 +- .../internal/pipeline/unfinished.go | 222 ++++++++++++++++ .../internal/pipeline/verbs.go | 60 +++++ .../mesh-delivery-pipeline/manifest_test.go | 6 +- modules/mesh-delivery-pipeline/module.json | 11 +- 27 files changed, 1149 insertions(+), 167 deletions(-) create mode 100644 modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/unfinished.go create mode 100644 modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/unfinished.go create mode 100644 modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/unfinished_integration_test.go create mode 100644 modules/mesh-delivery-pipeline/internal/pipeline/unfinished.go diff --git a/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract.go b/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract.go index e4b34e038..f024bb547 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract.go @@ -20,26 +20,27 @@ const Form = 2 // whose fields changed while its forms did not (design 47 H2 §780). const ( generatedForms = "2" - generatedFieldsDigest = "AcceptAnswer=224df0394653 Accepted=d2b011e5090f AskAgainAnswer=224df0394653 CancelAnswer=a0dcc32b3f57 EndAgainAnswer=a0dcc32b3f57 Fill=7a76c9ceb157 FillingEvent=5bd4df78ed84 FullEvent=5bd4df78ed84 KeptAnswer=dfe0d48bc169 KeptByLevelDigestValue=f06c90f7da7f KeptResult=af5f81f189b7 ListAnswer=75717fd670a7 LogEntry=b5c78894cc5b Member=8a103baec01f Mover=5659e54b6dd0 MoverEnded=4b35095f6829 Pipeline=2663792125bf PipelineEnded=523f8fc0fe41 PipelineEndedEvent=cb44007f7173 PipelineEndedPhases=9d1c33ce4cc4 PipelineEndedPhasesBuild=a08d49692a09 PipelineEndedPhasesComposition=a08d49692a09 PipelineEndedPhasesIntegration=a08d49692a09 PipelineEndedPhasesLint=a08d49692a09 PipelineEndedPhasesUnit=a08d49692a09 PipelineEndedPhasesValidate=a08d49692a09 PipelineStarted=a3d000f095db PipelineStartedEvent=eaa401af4117 PipelinesByBranchRevisionValue=693580eed498 PipelinesByBranchRunRunValue=693580eed498 PipelinesByBranchSourcePublishedPackageVersionValue=693580eed498 PipelinesByStepsPipelineMoverValue=94de99db7816 PlaceAnswer=a0dcc32b3f57 Published=d9af3024dacc QueueByQueueNextValue=765a2918a728 QueueByRunningPipelineValue=ed94ced4513a QueueByWaitingSequenceValue=99605298ae13 RePinnedEvent=92fd04bfa6d3 RerunAnswer=9b7efd0df344 ReviewAnswer=f397b8965179 RoomEvent=5bd4df78ed84 RunAnswer=96bd45aa2e62 RunStarted=b280617410c3 RunStartedEvent=0767a9358fa3 Shared=c981cf6aa860 SharedAskedBy=c9b30cc4d0d0 ShowAnswer=5beff763f531 StandsAnswer=0afa3693bf3b StartAnswer=ed94ced4513a StepEndedEvent=799020642c19 StepEvent=64f18c40ed5d StepLogByPipelineValue=ffdde090a101 StepOutputsByPipelineStepArtifactValue=a2f06147f4b6 StepResult=94c2bea6ddcd StepStartedEvent=799020642c19 TestCounts=9c1332427159 TestRun=ca5a87f3b6b8 TestRunPackage=a7d30558470d TestRunReader=c9b30cc4d0d0 TestRunsAnswer=7ab81a45ca33 TestRunsDropAnswer=8ee70b35469d ToolchainUsed=43b6d42376fb Wait=21e4ee7a3ef1 WaitEndedEvent=2bf6d38041c0 WaitHeldEvent=2bf6d38041c0 Waiting=ed94ced4513a WhyCarriedAnswer=8e2fcea6479f Widening=e2790a307466" + generatedFieldsDigest = "AcceptAnswer=224df0394653 Accepted=d2b011e5090f AskAgainAnswer=224df0394653 CancelAnswer=a0dcc32b3f57 EndAgainAnswer=a0dcc32b3f57 Fill=7a76c9ceb157 FillingEvent=5bd4df78ed84 FullEvent=5bd4df78ed84 KeptAnswer=dfe0d48bc169 KeptByLevelDigestValue=f06c90f7da7f KeptResult=af5f81f189b7 ListAnswer=75717fd670a7 LogEntry=b5c78894cc5b Member=8a103baec01f Mover=5659e54b6dd0 MoverEnded=4b35095f6829 Pipeline=2663792125bf PipelineEnded=523f8fc0fe41 PipelineEndedEvent=cb44007f7173 PipelineEndedPhases=9d1c33ce4cc4 PipelineEndedPhasesBuild=a08d49692a09 PipelineEndedPhasesComposition=a08d49692a09 PipelineEndedPhasesIntegration=a08d49692a09 PipelineEndedPhasesLint=a08d49692a09 PipelineEndedPhasesUnit=a08d49692a09 PipelineEndedPhasesValidate=a08d49692a09 PipelineStarted=a3d000f095db PipelineStartedEvent=eaa401af4117 PipelinesByBranchRevisionUnfinishedModuleValue=693580eed498 PipelinesByBranchRevisionValue=693580eed498 PipelinesByBranchRunRunValue=693580eed498 PipelinesByBranchSourcePublishedPackageVersionValue=693580eed498 PipelinesByStepsPipelineMoverValue=94de99db7816 PlaceAnswer=a0dcc32b3f57 PlaceUnfinishedAnswer=ed94ced4513a Published=d9af3024dacc QueueByQueueNextValue=765a2918a728 QueueByRunningPipelineValue=ed94ced4513a QueueByWaitingSequenceValue=99605298ae13 RePinnedEvent=92fd04bfa6d3 RerunAnswer=9b7efd0df344 ReviewAnswer=f397b8965179 RoomEvent=5bd4df78ed84 RunAnswer=96bd45aa2e62 RunStarted=b280617410c3 RunStartedEvent=0767a9358fa3 Shared=c981cf6aa860 SharedAskedBy=c9b30cc4d0d0 ShowAnswer=5beff763f531 StandsAnswer=0afa3693bf3b StartAnswer=ed94ced4513a StepEndedEvent=799020642c19 StepEvent=64f18c40ed5d StepLogByPipelineValue=ffdde090a101 StepOutputsByPipelineStepArtifactValue=a2f06147f4b6 StepResult=94c2bea6ddcd StepStartedEvent=799020642c19 TestCounts=9c1332427159 TestRun=ca5a87f3b6b8 TestRunPackage=a7d30558470d TestRunReader=c9b30cc4d0d0 TestRunsAnswer=7ab81a45ca33 TestRunsDropAnswer=8ee70b35469d ToolchainUsed=43b6d42376fb Wait=21e4ee7a3ef1 WaitEndedEvent=2bf6d38041c0 WaitHeldEvent=2bf6d38041c0 Waiting=ed94ced4513a WhyCarriedAnswer=8e2fcea6479f Widening=e2790a307466" ) const ( - VerbRun = "run" - VerbStart = "start" - VerbShow = "show" - VerbList = "list" - VerbStands = "stands" - VerbWhyCarried = "why-carried" - VerbKept = "kept" - VerbReview = "review" - VerbRerun = "rerun" - VerbAccept = "accept" - VerbCancel = "cancel" - VerbEndAgain = "end-again" - VerbPlace = "place" - VerbAskAgain = "ask-again" - VerbTestRuns = "test-runs" - VerbTestRunsDrop = "test-runs-drop" + VerbRun = "run" + VerbStart = "start" + VerbShow = "show" + VerbList = "list" + VerbStands = "stands" + VerbWhyCarried = "why-carried" + VerbKept = "kept" + VerbReview = "review" + VerbRerun = "rerun" + VerbAccept = "accept" + VerbCancel = "cancel" + VerbEndAgain = "end-again" + VerbPlace = "place" + VerbPlaceUnfinished = "place-unfinished" + VerbAskAgain = "ask-again" + VerbTestRuns = "test-runs" + VerbTestRunsDrop = "test-runs-drop" ) const ( @@ -58,7 +59,7 @@ const ( // Verbs is every verb the seat serves, in its definition's order: a new copy each call. func Verbs() []string { - return []string{VerbRun, VerbStart, VerbShow, VerbList, VerbStands, VerbWhyCarried, VerbKept, VerbReview, VerbRerun, VerbAccept, VerbCancel, VerbEndAgain, VerbPlace, VerbAskAgain, VerbTestRuns, VerbTestRunsDrop} + return []string{VerbRun, VerbStart, VerbShow, VerbList, VerbStands, VerbWhyCarried, VerbKept, VerbReview, VerbRerun, VerbAccept, VerbCancel, VerbEndAgain, VerbPlace, VerbPlaceUnfinished, VerbAskAgain, VerbTestRuns, VerbTestRunsDrop} } // Events is every event published on the seat's subject: a new copy each call. @@ -274,6 +275,44 @@ func DecodePipelinesByBranchRunRunValue(raw []byte) (PipelinesByBranchRunRunValu return v, formRead(raw), nil } +// PipelinesByBranchRevisionUnfinishedModuleKey is the key of one value of the state pipelines under the form "..unfinished.". +func PipelinesByBranchRevisionUnfinishedModuleKey(branch, revision, module string) string { + return branch + "." + revision + ".unfinished." + module +} + +// ParsePipelinesByBranchRevisionUnfinishedModuleKey reads the parts of a key of the form "..unfinished.", in the order of the form; the last result is false for a key of another form. +func ParsePipelinesByBranchRevisionUnfinishedModuleKey(key string) (string, string, string, bool) { + rest := key + v0, after0, found0 := strings.Cut(rest, ".") + if !found0 || v0 == "" { + return "", "", "", false + } + rest = after0 + v2, after2, found2 := strings.Cut(rest, ".unfinished.") + if !found2 || v2 == "" { + return "", "", "", false + } + rest = after2 + if rest == "" { + return "", "", "", false + } + v4 := rest + rest = "" + if rest != "" { + return "", "", "", false + } + return v0, v2, v4, true +} + +// DecodePipelinesByBranchRevisionUnfinishedModuleValue reads one value of the state pipelines under the form "..unfinished." as another part wrote it: a field PipelinesByBranchRevisionUnfinishedModuleValue does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780). +func DecodePipelinesByBranchRevisionUnfinishedModuleValue(raw []byte) (PipelinesByBranchRevisionUnfinishedModuleValue, int, error) { + var v PipelinesByBranchRevisionUnfinishedModuleValue + if err := json.Unmarshal(raw, &v); err != nil { + return v, 0, err + } + return v, formRead(raw), nil +} + // PipelinesByStepsPipelineMoverKey is the key of one value of the state pipelines under the form "steps..". func PipelinesByStepsPipelineMoverKey(pipeline, mover string) string { return "steps." + pipeline + "." + mover @@ -1383,6 +1422,22 @@ type PlaceAnswer struct { Pipeline Pipeline `json:"pipeline"` } +// PlaceUnfinishedInput is what place-unfinished takes. +type PlaceUnfinishedInput struct { + // the unfinished entry's key, . + Key string `json:"key"` + // why, written with the caller the runtime proves in the hand-act log + Why string `json:"why"` +} + +// PlaceUnfinishedAnswer is what place-unfinished answers. +type PlaceUnfinishedAnswer struct { + // is the form of what this value carries, left out while it is 1 (design 47 H2 §780). + Form int64 `json:"form,omitempty"` + // the key of the pipeline the mover is put in + Pipeline string `json:"pipeline"` +} + // AskAgainInput is what ask-again takes. type AskAgainInput struct { // the mover's module, so a pipeline of several movers is never ambiguous; a call naming none on a pipeline of one mover is taken as that mover (design 47 H-A-101) @@ -1493,6 +1548,9 @@ type PipelinesByBranchSourcePublishedPackageVersionValue = Pipeline // PipelinesByBranchRunRunValue is one value of the state pipelines under the key .run.. type PipelinesByBranchRunRunValue = Pipeline +// PipelinesByBranchRevisionUnfinishedModuleValue is one value of the state pipelines under the key ..unfinished.. +type PipelinesByBranchRevisionUnfinishedModuleValue = Pipeline + // PipelinesByStepsPipelineMoverValue is one value of the state pipelines under the key steps... type PipelinesByStepsPipelineMoverValue struct { // is the form of what this value carries, left out while it is 1 (design 47 H2 §780). @@ -1643,6 +1701,15 @@ func DecodePlaceAnswer(raw []byte) (PlaceAnswer, int, error) { return v, formRead(raw), nil } +// DecodePlaceUnfinishedAnswer reads a PlaceUnfinishedAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780). +func DecodePlaceUnfinishedAnswer(raw []byte) (PlaceUnfinishedAnswer, int, error) { + var v PlaceUnfinishedAnswer + if err := json.Unmarshal(raw, &v); err != nil { + return v, 0, err + } + return v, formRead(raw), nil +} + // DecodeAskAgainAnswer reads a AskAgainAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780). func DecodeAskAgainAnswer(raw []byte) (AskAgainAnswer, int, error) { var v AskAgainAnswer diff --git a/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json b/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json index 68e7c9497..bb11edc77 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json +++ b/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json @@ -578,6 +578,40 @@ }, "class": "act" }, + { + "name": "place-unfinished", + "description": "A mover kept in the state unfinished put in a pipeline of its own at once, keyed ..unfinished.; written in the hand-act log (design 47 H2 §786, H-A-398).", + "input": { + "type": "object", + "properties": { + "key": { + "type": "string", + "description": "the unfinished entry's key, ." + }, + "why": { + "type": "string", + "description": "why, written with the caller the runtime proves in the hand-act log" + } + }, + "required": [ + "key", + "why" + ] + }, + "answer": { + "type": "object", + "properties": { + "pipeline": { + "type": "string", + "description": "the key of the pipeline the mover is put in" + } + }, + "required": [ + "pipeline" + ] + }, + "class": "act" + }, { "name": "ask-again", "description": "A declined approval step asked again (ADR 0666 section 10). Written in the hand-act log.", @@ -858,6 +892,13 @@ }, "description": "a run on request's pipeline, one per run, its movers the named commits' modules; it delivers nothing (H-A-83)" }, + { + "key": "..unfinished.", + "value": { + "$ref": "#/defs/Pipeline" + }, + "description": "a pipeline of its own of one mover kept in the runner's state unfinished, put in it by place-unfinished; the revision the unfinished entry names (design 47 H2 §86, H-A-398)" + }, { "key": "steps..", "value": { diff --git a/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.d.ts b/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.d.ts index 1108ddefe..c051eb0d0 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.d.ts @@ -18,6 +18,7 @@ export declare const verbs: { readonly cancel: "cancel"; readonly endAgain: "end-again"; readonly place: "place"; + readonly placeUnfinished: "place-unfinished"; readonly askAgain: "ask-again"; readonly testRuns: "test-runs"; readonly testRunsDrop: "test-runs-drop"; @@ -115,6 +116,12 @@ export declare function pipelinesByBranchRunRunKey(branch: string, run: string): /** The parts of a key of the form ".run.", or undefined for a key of another form. */ export declare function parsePipelinesByBranchRunRunKey(_key: string): { branch: string; run: string } | undefined; +/** The key of one value of the state pipelines under the form "..unfinished.". */ +export declare function pipelinesByBranchRevisionUnfinishedModuleKey(branch: string, revision: string, module: string): string; + +/** The parts of a key of the form "..unfinished.", or undefined for a key of another form. */ +export declare function parsePipelinesByBranchRevisionUnfinishedModuleKey(_key: string): { branch: string; revision: string; module: string } | undefined; + /** The key of one value of the state pipelines under the form "steps..". */ export declare function pipelinesByStepsPipelineMoverKey(pipeline: string, mover: string): string; @@ -863,6 +870,22 @@ export interface PlaceAnswer { "pipeline": Pipeline; } +/** is what place-unfinished takes. */ +export interface PlaceUnfinishedInput { + /** the unfinished entry's key, . */ + "key": string; + /** why, written with the caller the runtime proves in the hand-act log */ + "why": string; +} + +/** is what place-unfinished answers. */ +export interface PlaceUnfinishedAnswer { + /** is the form of what this value carries, left out while it is 1 (design 47 H2 §780). */ + "form"?: number; + /** the key of the pipeline the mover is put in */ + "pipeline": string; +} + /** is what ask-again takes. */ export interface AskAgainInput { /** the mover's module, so a pipeline of several movers is never ambiguous; a call naming none on a pipeline of one mover is taken as that mover (design 47 H-A-101) */ @@ -966,6 +989,9 @@ export type PipelinesByBranchSourcePublishedPackageVersionValue = Pipeline; /** is one value of the state pipelines under the key .run.. */ export type PipelinesByBranchRunRunValue = Pipeline; +/** is one value of the state pipelines under the key ..unfinished.. */ +export type PipelinesByBranchRevisionUnfinishedModuleValue = Pipeline; + /** is one value of the state pipelines under the key steps... */ export interface PipelinesByStepsPipelineMoverValue { /** is the form of what this value carries, left out while it is 1 (design 47 H2 §780). */ diff --git a/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.js b/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.js index f1004d04f..17a2ba3d0 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.js +++ b/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.js @@ -18,6 +18,7 @@ export const verbs = { cancel: "cancel", endAgain: "end-again", place: "place", + placeUnfinished: "place-unfinished", askAgain: "ask-again", testRuns: "test-runs", testRunsDrop: "test-runs-drop", @@ -176,6 +177,29 @@ export function parsePipelinesByBranchRunRunKey(_key) { return { branch, run }; } +/** The key of one value of the state pipelines under the form "..unfinished.". */ +export function pipelinesByBranchRevisionUnfinishedModuleKey(branch, revision, module) { + return branch + "." + revision + ".unfinished." + module; +} + +/** The parts of a key of the form "..unfinished.", or undefined for a key of another form. */ +export function parsePipelinesByBranchRevisionUnfinishedModuleKey(_key) { + let _rest = _key; + const _at0 = _rest.indexOf("."); + if (_at0 <= 0) return undefined; + const branch = _rest.slice(0, _at0); + _rest = _rest.slice(_at0 + 1); + const _at2 = _rest.indexOf(".unfinished."); + if (_at2 <= 0) return undefined; + const revision = _rest.slice(0, _at2); + _rest = _rest.slice(_at2 + 12); + if (_rest === "") return undefined; + const module = _rest; + _rest = ""; + if (_rest !== "") return undefined; + return { branch, revision, module }; +} + /** The key of one value of the state pipelines under the form "steps..". */ export function pipelinesByStepsPipelineMoverKey(pipeline, mover) { return "steps." + pipeline + "." + mover; diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go index facab7727..2a0aba98c 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go @@ -174,6 +174,7 @@ func main() { tools = append(tools, HealthTool(r.Pipelines, kept{stdio.State(pipeline.StateConsumers)}, time.Now, func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-pipeline-tools] "+f+"\n", a...) })) tools = append(tools, WaitingTool(kept{stdio.State(seat.StateQueue)})) + tools = append(tools, UnfinishedTools(keptDelete{kept{stdio.State(pipeline.StateUnfinished)}}, meshhandactsclient.Recorder{Ask: stdio.Ask})...) if err := stdio.Serve("", tools); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-pipeline-tools] %v\n", err) os.Exit(1) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main_test.go index 1b2873b3a..14b6c44c0 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main_test.go @@ -43,7 +43,13 @@ func TestTools(t *testing.T) { for _, tl := range UnloggedTools(stdio.State("pipelines"), stdio.Log("step-log"), nil, nil) { names = append(names, tl.Name) } - want := "mesh-pipeline-runner.show mesh-pipeline-runner.list mesh-pipeline-runner.stands mesh-pipeline-runner.why-carried mesh-pipeline-runner.kept runner_first_views runner_first_view_again runner_cancel runner_rerun runner_start runner_run runner_accept runner_place runner_end_again runner_ask_again runner_health runner_waiting runner_unlogged runner_unlogged_append" + // design 47 H2 §786: the unfinished movers are read and dropped by hand. + unfinished := unfinishedOfMem{"main.app": `{"fingerprint":"src2:aa","revision":4,"pipeline":"main.4","outcome":"failed"}`} + tools := UnfinishedTools(unfinished, nil) + for _, tl := range tools { + names = append(names, tl.Name) + } + want := "mesh-pipeline-runner.show mesh-pipeline-runner.list mesh-pipeline-runner.stands mesh-pipeline-runner.why-carried mesh-pipeline-runner.kept runner_first_views runner_first_view_again runner_cancel runner_rerun runner_start runner_run runner_accept runner_place runner_end_again runner_unfinished_place runner_ask_again runner_health runner_waiting runner_unlogged runner_unlogged_append runner_unfinished runner_unfinished_drop" if got := strings.Join(names, " "); got != want { t.Errorf("tools %q, want %q", got, want) } @@ -70,8 +76,51 @@ func TestTools(t *testing.T) { if len(journal) != 1 || !strings.Contains(journal[0], "main.137") { t.Errorf("the journal said %q, want main.137 said once", journal) } + // design 47 H2 §786: runner_unfinished reads each key and its entry; runner_unfinished_drop names why, drops nothing when + // its hand act is not recorded, and drops the key once it is. + if got, err := tools[0].Run(nil); err != nil || fmt.Sprint(got) != "[{main.app {src2:aa 4 main.4 failed}}]" { + t.Errorf("runner_unfinished = %v (%v)", got, err) + } + for _, c := range []struct { + name string + acts views.HandActs + args map[string]any + refusal string + }{ + {"no why", recordingActs{}, map[string]any{"key": "main.app"}, "why"}, + {"no such key", recordingActs{}, map[string]any{"key": "main.lib", "why": "x"}, "holds no main.lib"}, + {"not recorded", refusingActs{}, map[string]any{"key": "main.app", "why": "x"}, "not recorded"}, + {"dropped", recordingActs{}, map[string]any{"key": "main.app", "why": "the change is reverted"}, ""}, + } { + _, err := UnfinishedTools(unfinished, c.acts)[1].Run(c.args) + _, held := unfinished["main.app"] + if c.refusal != "" && (err == nil || !strings.Contains(err.Error(), c.refusal) || !held) { + t.Errorf("runner_unfinished_drop %s: %v, the key held %t", c.name, err, held) + } + if c.refusal == "" && (err != nil || held) { + t.Errorf("runner_unfinished_drop %s: %v, the key held %t", c.name, err, held) + } + } } +// unfinishedOfMem is a state unfinished in memory. +type unfinishedOfMem map[string]string + +func (u unfinishedOfMem) Keys() ([]string, error) { return queueOf(u).Keys() } + +func (u unfinishedOfMem) Value(k string) (json.RawMessage, error) { + if v, ok := u[k]; ok { + return json.RawMessage(v), nil + } + return nil, nil +} + +func (u unfinishedOfMem) Delete(k string) error { delete(u, k); return nil } + +type recordingActs struct{} + +func (recordingActs) RecordHandAct(views.HandAct) error { return nil } + // queueOf is a state queue in memory, its keys in no order. type queueOf map[string]string diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/unfinished.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/unfinished.go new file mode 100644 index 000000000..a91071c13 --- /dev/null +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/unfinished.go @@ -0,0 +1,105 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "slices" + "strings" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/views" + + "mesh-delivery-pipeline/internal/pipeline" +) + +// UnfinishedState is the runner's state unfinished as the tools read and drop it. +type UnfinishedState interface { + pipeline.Kept + Delete(key string) error +} + +// keptDelete is a state of the runner read and deleted through the runtime. +type keptDelete struct{ kept } + +func (k keptDelete) Delete(key string) error { return k.s.Delete(key) } + +// Unfinished is one mover kept in the state unfinished, as runner_unfinished lists it. +type Unfinished struct { + Key string `json:"key"` + pipeline.UnfinishedEntry +} + +// UnfinishedTools are runner_unfinished and runner_unfinished_drop over the runner's state unfinished (design 47 H2 §786): +// every mover a pipeline's end left unfinished, and one key dropped by hand, with why, written in the hand-act log first. +// runner_unfinished_place, which puts one in a pipeline of its own, asks the seat verb place-unfinished (VerbTools). +func UnfinishedTools(st UnfinishedState, acts views.HandActs) []stdio.Tool { + str := map[string]any{"type": "string"} + return []stdio.Tool{ + {Name: Prefix + "_unfinished", + Description: "The movers a pipeline's end left unfinished (design 47 H2 §786): each key . with its fingerprint, the catalog revision and the pipeline it was left in, and that pipeline's outcome. Each is put in the pipeline of its branch's next catalog revision; runner_unfinished_place puts one in a pipeline of its own now, runner_unfinished_drop drops one.", + Input: map[string]any{"type": "object", "properties": map[string]any{}}, + Run: func(map[string]any) (any, error) { return unfinishedOf(st) }}, + {Name: Prefix + "_unfinished_drop", + Description: "Drop one mover from the state unfinished by hand (design 47 H2 §786): its key . and why, written in the hand-act log first; the mover is then put in no pipeline again until its fingerprint moves.", + Input: map[string]any{"type": "object", "properties": map[string]any{"key": str, "why": str}, "required": []string{"key", "why"}}, + Run: func(args map[string]any) (any, error) { + in, err := as[struct { + Key string `json:"key"` + Why string `json:"why"` + }](args) + if err != nil { + return nil, err + } + return dropUnfinished(st, acts, in.Key, in.Why) + }}, + } +} + +func unfinishedOf(st pipeline.Kept) ([]Unfinished, error) { + keys, err := st.Keys() + if err != nil { + return nil, fmt.Errorf("the state unfinished is not read: %w", err) + } + slices.Sort(keys) + out := []Unfinished{} + for _, k := range keys { + raw, err := st.Value(k) + if err != nil || raw == nil { + continue + } + u := Unfinished{Key: k} + if err := json.Unmarshal(raw, &u.UnfinishedEntry); err != nil { + return nil, fmt.Errorf("the unfinished entry %s is not read: %w", k, err) + } + out = append(out, u) + } + return out, nil +} + +func dropUnfinished(st UnfinishedState, acts views.HandActs, key, why string) (any, error) { + if strings.TrimSpace(key) == "" { + return nil, errors.New("runner_unfinished_drop names the unfinished entry's key") + } + if msg := whyOf(why); msg != "" { + return nil, errors.New(msg) + } + raw, err := st.Value(key) + if err != nil { + return nil, fmt.Errorf("the unfinished entry %s is not read: %w", key, err) + } + if raw == nil { + return nil, fmt.Errorf("the state unfinished holds no %s: nothing was dropped", key) + } + if acts == nil { + return nil, errors.New("no way to record a hand act is given: nothing was dropped") + } + if err := acts.RecordHandAct(views.HandAct{What: "dropped the unfinished mover " + key + " by hand", Why: strings.TrimSpace(why), + Cause: "unfinished-drop", Verb: Prefix + "_unfinished_drop"}); err != nil { + return nil, fmt.Errorf("the hand act is not recorded, so nothing was dropped: %w", err) + } + if err := st.Delete(key); err != nil { + return nil, fmt.Errorf("the unfinished entry %s is not deleted: %w", key, err) + } + return map[string]any{"dropped": key}, nil +} diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/verbtools.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/verbtools.go index 7b887bf6f..c2bc97410 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/verbtools.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/verbtools.go @@ -9,8 +9,8 @@ import ( stdio "git.novox.be/novox/mesh-sdk/go" ) -// VerbTools are runner_start, runner_run, runner_accept, runner_place and runner_ask_again: the runner's seat verbs start, -// run, accept, place (§748) and ask-again, reachable as tools (design 47 G §83, §482), as runner_rerun reaches rerun. Each asks its verb over the bus with +// VerbTools are runner_start, runner_run, runner_accept, runner_place, runner_unfinished_place and runner_ask_again: the +// runner's seat verbs start, run, accept, place (§748), place-unfinished (H2 §786) and ask-again, reachable as tools (design 47 G §83, §482), as runner_rerun reaches rerun. Each asks its verb over the bus with // the arguments it was given and answers the verb's answer; the verb records the hand act and refuses what it refuses. func VerbTools(ask func(key string, body any) (json.RawMessage, error)) []stdio.Tool { str := map[string]any{"type": "string"} @@ -63,6 +63,15 @@ func VerbTools(ask func(key string, body any) (json.RawMessage, error)) []stdio. } return whyOf(in.Why) }), + verbTool[seat.PlaceUnfinishedInput](ask, "unfinished_place", seat.VerbPlaceUnfinished, + "Put a mover kept in the state unfinished in a pipeline of its own at once, by the seat verb place-unfinished (design 47 H2 §786): the entry's key . (runner_unfinished lists them) and why; written in the hand-act log. The answer names the pipeline, keyed ..unfinished..", + obj([]string{"key", "why"}, map[string]any{"key": str, "why": str}), + func(in seat.PlaceUnfinishedInput) string { + if in.Key == "" { + return "place-unfinished names the unfinished entry's key" + } + return whyOf(in.Why) + }), verbTool[seat.AskAgainInput](ask, "ask_again", seat.VerbAskAgain, "Ask a declined approval step of a pipeline once more by the seat verb ask-again: the pipeline's key, the mover's module (needed when the pipeline moves several) and why.", obj([]string{"pipeline", "why"}, map[string]any{"pipeline": str, "module": str, "why": str}), diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/verbtools_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/verbtools_test.go index 40f7558e8..8eafe1981 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/verbtools_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/verbtools_test.go @@ -36,6 +36,10 @@ func TestVerbTools(t *testing.T) { {"end-again", "runner_end_again", "seat:mesh-pipeline-runner.end-again", "", map[string]any{"pipeline": "main.20", "why": "left running"}, seat.EndAgainInput{Pipeline: "main.20", Why: "left running"}}, {"end-again no why", "runner_end_again", "", "names why", map[string]any{"pipeline": "main.20"}, nil}, + // design 47 H2 §786, H-A-398: runner_unfinished_place asks the verb place-unfinished with the entry's key and why. + {"unfinished place", "runner_unfinished_place", "seat:mesh-pipeline-runner.place-unfinished", "", map[string]any{"key": "main.app", "why": "now"}, + seat.PlaceUnfinishedInput{Key: "main.app", Why: "now"}}, + {"unfinished place no key", "runner_unfinished_place", "", "entry's key", map[string]any{"why": "now"}, nil}, {"ask-again no why", "runner_ask_again", "", "names why", map[string]any{"pipeline": "p"}, nil}, } { t.Run(c.name, func(t *testing.T) { diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index 0456d9075..14d0832c1 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -73,6 +73,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err != nil { return err } + // The movers a pipeline's end left unfinished, put in a pipeline again at their branch's next revision (design 47 H2 §786). + unfinishedKV, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, pipeline.StateUnfinished), "put a mover left unfinished in a pipeline again") + if err != nil { + return err + } + unfinished := &pipeline.Unfinished{Store: kvUnfinished{ctx, unfinishedKV}, Logf: logger.Printf} // The toolchain of each language as heard agreed, for the steps' asks (§416, A-104). toolchains, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, "toolchains"), "name a step's toolchain") if err != nil { @@ -166,7 +172,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro Registration: rv.Registrations.Registration, Steps: busStepAsker{ctx: ctx, publish: js.PublishMsg, cred: cred, bound: askBound, logf: logger.Printf}, Name: cred.Module, Kept: keptReader(ctx, kept), Toolchain: toolchainReader(ctx, toolchains), Fills: fills, Recovered: kvRecoveries{ctx, recovered}.Has, Holders: rv.Holders.Holders, Ended: hearer.EndedAtPlacing, - Findings: rv.Findings, Observer: cred.Module} // a mover waiting on its registration is registration-missing (H-A-262) + Findings: rv.Findings, Observer: cred.Module, // a mover waiting on its registration is registration-missing (H-A-262) + Unfinished: unfinished} + hearer.Unfinished = unfinished hearer.AskSteps = starter.AskSteps hearer.RegistrationWaits = func() { starter.JudgeRegistrationMissing(store) } // at every end (H-A-265 (4)) logger.Printf("asks: an ask's bound %s, counted from its put until its holder's ask-taken, then from the take and each step-progress; past it the step ends could not run and its ask is dropped", askBound) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go index 23dc7a86c..53bd45166 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go @@ -54,7 +54,7 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) { defer cancel() const module = "mesh-delivery-pipeline" - for _, state := range []string{"pipelines", "kept", "toolchains", "findings", "first-views", "consumers", "aged", "recoveries", seat.StateQueue} { + for _, state := range []string{"pipelines", "kept", "toolchains", "findings", "first-views", "consumers", "aged", "recoveries", seat.StateQueue, pipeline.StateUnfinished} { if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_" + state, History: 1}); err != nil { t.Fatal(err) } @@ -304,6 +304,16 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) { t.Fatal(err) } outcome("main.4", seat.PipelineOutcomeCancelled) + // design 47 H2 §786: the cancel leaves main.4's lib unfinished; dropped here, as runner_unfinished_drop does, so the + // next take-ins move only what they name. + unfinished, err := js.KeyValue(ctx, module+"_"+pipeline.StateUnfinished) + if err != nil { + t.Fatal(err) + } + waitFor(t, ctx, logs, func() bool { _, err := unfinished.Get(ctx, "main.lib"); return err == nil }) + if err := unfinished.Delete(ctx, "main.lib"); err != nil { + t.Fatal(err) + } outcome("main.6", seat.PipelineOutcomeRunning) waitFor(t, ctx, logs, func() bool { _, ok := finding(pipeline.QueueFullKind); return !ok }) if p := get("main.7"); p.Outcome != seat.PipelineOutcomeWaiting { diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go index a7b2e3cbe..fab8b6a12 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go @@ -47,124 +47,21 @@ func (s *syncBuffer) String() string { s.mu.Lock(); defer s.mu.Unlock(); return // of the build, and ends the placed review step when a session calls `review`, answered by the process with the caller the // bus proved; the pipeline then ends passed. func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *testing.T) { - url := testbus.URL(t) - nc, err := nats.Connect(url) - if err != nil { - t.Fatal(err) - } - defer nc.Close() - js, err := jetstream.New(nc) - if err != nil { - t.Fatal(err) - } ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - - const module = "mesh-delivery-pipeline" - for _, state := range []string{"pipelines", "kept", "toolchains", "findings", "first-views", "consumers", "aged", "recoveries", "queue"} { - if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_" + state, History: 1}); err != nil { - t.Fatal(err) - } - } - // bounds and test-runs keep keys with their own time to live (§725), as the bus makes them from the manifest's key-ttl. - for _, state := range []string{"bounds", TestRunsState} { - if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_" + state, History: 1, LimitMarkerTTL: time.Minute}); err != nil { - t.Fatal(err) - } - } - // Kept pipelines in the old form, their steps inline in the record, as the runner's start finds them on a live bus - // (main.41-46, one of 437 steps): each start path reads them at once from its own goroutine (H-A-187, H §774). - keptPipelines(t, ctx, js, module) - // The step log, the runner's old logs section (§470): the stream the bus makes for it. - if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buscompose.LogStreamName(module, seat.StateStepLog), Subjects: []string{buscompose.LogSubject(module, seat.StateStepLog) + ".>"}}); err != nil { - t.Fatal(err) - } - findingsStream(t, ctx, js, module) - events, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: process.EventsStream, Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"}}) - if err != nil { - t.Fatal(err) - } - if _, err := events.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{Durable: "n1_" + module, AckPolicy: jetstream.AckExplicitPolicy}); err != nil { - t.Fatal(err) - } - asks, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "ASKS", Subjects: []string{"mesh.seat.*.accept.>"}, Retention: jetstream.WorkQueuePolicy}) - if err != nil { - t.Fatal(err) - } - - // The seats the runner calls, answered as their holders do. reg := catalog.Registration{Module: "app", Branches: []string{"main"}, Origin: "o/app", Policy: catalog.Policy{Steps: catalog.StepStatuses{Review: "optional", Validate: "bypass", Composition: "bypass", Approval: "bypass"}}} - answer := func(subject string, body any) { - if _, err := nc.Subscribe(subject, func(m *nats.Msg) { - raw, _ := json.Marshal(map[string]any{"result": body}) - _ = m.Respond(raw) - }); err != nil { - t.Fatal(err) - } - } - answer("mesh.seat.mesh-catalog.call.registrations.*", catalog.RegistrationsAnswer{Registrations: []catalog.Registration{reg}}) // The branch main has no revision yet at the runner's start (none-yet), then revision 1 holding app (A-336). var revised atomic.Bool - if _, err := nc.Subscribe("mesh.seat.mesh-catalog.call.revision.*", func(m *nats.Msg) { - var a catalog.RevisionAnswer + w := startWhole(t, ctx, []catalog.Registration{reg}, func([]byte) catalog.RevisionAnswer { if revised.Load() { - a = catalog.RevisionAnswer{Revision: &catalog.CatalogRevision{Revision: 1, Kind: "trunk", Branch: "main", Source: 3}, + return catalog.RevisionAnswer{Revision: &catalog.CatalogRevision{Revision: 1, Kind: "trunk", Branch: "main", Source: 3}, Entries: []catalog.CatalogEntry{{Module: "app", Commit: "c1", Fp: "src2:aa"}}} - } else { - a.NoneYet = true } - raw, _ := json.Marshal(map[string]any{"result": a}) - _ = m.Respond(raw) - }); err != nil { - t.Fatal(err) - } - answer("mesh.seat.mesh-hand-acts.call.hand-act.*", map[string]any{}) - // The seat list (A-186): the seat mesh-catalog's seat-definitions names the runner's seat, whose definition its write - // verbs' callers are checked against, as the contracts' file holds it. - def, err := os.ReadFile("../../../mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json") - if err != nil { - t.Fatal(err) - } - if _, err := nc.Subscribe("mesh.seat.mesh-catalog.call."+seatlist.Verb+".*", func(m *nats.Msg) { - var in struct { - Seat string `json:"seat"` - } - _ = json.Unmarshal(m.Data, &in) - entry := seatlist.Entry{Seat: seat.Seat, Source: seatlist.SourceTrunk, Digest: "sha256:runner"} - var result any = seatlist.Listing{Revision: 1, Seats: []seatlist.Entry{entry}} - if in.Seat != "" { - entry.Definition = def - result = seatlist.Answer{Seat: in.Seat, Found: in.Seat == seat.Seat, Entry: &entry, Revision: 1} - } - raw, _ := json.Marshal(map[string]any{"result": result}) - _ = m.Respond(raw) - }); err != nil { - t.Fatal(err) - } + return catalog.RevisionAnswer{NoneYet: true} + }) + const module = "mesh-delivery-pipeline" + nc, js, asks, pipelines, logs := w.nc, w.js, w.asks, w.pipelines, w.logs - dir := t.TempDir() - broker := filepath.Join(dir, "broker") - raw, _ := json.Marshal(process.Account{URL: url, Node: "n1", Module: module, User: "runner", Password: "x"}) - if err := os.WriteFile(broker, raw, 0o600); err != nil { - t.Fatal(err) - } - logs := &syncBuffer{} - runCtx, stop := context.WithCancel(ctx) - done := make(chan error, 1) - go func() { - done <- run(runCtx, func(k string) string { - if k == process.BrokerFileEnv { - return broker - } - return "" - }, logs) - }() - defer func() { stop(); <-done }() - - pipelines, err := js.KeyValue(ctx, module+"_pipelines") - if err != nil { - t.Fatal(err) - } // A take-in.taken names one entry of the source revision; the runner places the pipeline of the registered module. taken, _ := json.Marshal(catalog.TakeInTaken{Branch: "main", Source: 3, Call: "0123456789abcdef", Entries: []catalog.TakeInEntry{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}) @@ -275,6 +172,127 @@ func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *tes } } +// whole is the runner started whole as its process is, against this test's bus (design 47 G §22, §54): its states, streams +// and seat list made as the bus makes them from its manifest, the catalog's registrations and revisions answered by regs and +// revision, its journal in logs. +type whole struct { + nc *nats.Conn + js jetstream.JetStream + asks jetstream.Stream + pipelines jetstream.KeyValue + logs *syncBuffer +} + +func startWhole(t *testing.T, ctx context.Context, regs []catalog.Registration, revision func(in []byte) catalog.RevisionAnswer) whole { + t.Helper() + url := testbus.URL(t) + nc, err := nats.Connect(url) + if err != nil { + t.Fatal(err) + } + t.Cleanup(nc.Close) + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + + const module = "mesh-delivery-pipeline" + for _, state := range []string{"pipelines", "kept", "toolchains", "findings", "first-views", "consumers", "aged", "recoveries", "queue", pipeline.StateUnfinished} { + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_" + state, History: 1}); err != nil { + t.Fatal(err) + } + } + // bounds and test-runs keep keys with their own time to live (§725), as the bus makes them from the manifest's key-ttl. + for _, state := range []string{"bounds", TestRunsState} { + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_" + state, History: 1, LimitMarkerTTL: time.Minute}); err != nil { + t.Fatal(err) + } + } + // Kept pipelines in the old form, their steps inline in the record, as the runner's start finds them on a live bus + // (main.41-46, one of 437 steps): each start path reads them at once from its own goroutine (H-A-187, H §774). + keptPipelines(t, ctx, js, module) + // The step log, the runner's old logs section (§470): the stream the bus makes for it. + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buscompose.LogStreamName(module, seat.StateStepLog), Subjects: []string{buscompose.LogSubject(module, seat.StateStepLog) + ".>"}}); err != nil { + t.Fatal(err) + } + findingsStream(t, ctx, js, module) + events, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: process.EventsStream, Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"}}) + if err != nil { + t.Fatal(err) + } + if _, err := events.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{Durable: "n1_" + module, AckPolicy: jetstream.AckExplicitPolicy}); err != nil { + t.Fatal(err) + } + asks, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "ASKS", Subjects: []string{"mesh.seat.*.accept.>"}, Retention: jetstream.WorkQueuePolicy}) + if err != nil { + t.Fatal(err) + } + + // The seats the runner calls, answered as their holders do. + answer := func(subject string, body any) { + if _, err := nc.Subscribe(subject, func(m *nats.Msg) { + raw, _ := json.Marshal(map[string]any{"result": body}) + _ = m.Respond(raw) + }); err != nil { + t.Fatal(err) + } + } + answer("mesh.seat.mesh-catalog.call.registrations.*", catalog.RegistrationsAnswer{Registrations: regs}) + if _, err := nc.Subscribe("mesh.seat.mesh-catalog.call.revision.*", func(m *nats.Msg) { + raw, _ := json.Marshal(map[string]any{"result": revision(m.Data)}) + _ = m.Respond(raw) + }); err != nil { + t.Fatal(err) + } + answer("mesh.seat.mesh-hand-acts.call.hand-act.*", map[string]any{}) + // The seat list (A-186): the seat mesh-catalog's seat-definitions names the runner's seat, whose definition its write + // verbs' callers are checked against, as the contracts' file holds it. + def, err := os.ReadFile("../../../mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json") + if err != nil { + t.Fatal(err) + } + if _, err := nc.Subscribe("mesh.seat.mesh-catalog.call."+seatlist.Verb+".*", func(m *nats.Msg) { + var in struct { + Seat string `json:"seat"` + } + _ = json.Unmarshal(m.Data, &in) + entry := seatlist.Entry{Seat: seat.Seat, Source: seatlist.SourceTrunk, Digest: "sha256:runner"} + var result any = seatlist.Listing{Revision: 1, Seats: []seatlist.Entry{entry}} + if in.Seat != "" { + entry.Definition = def + result = seatlist.Answer{Seat: in.Seat, Found: in.Seat == seat.Seat, Entry: &entry, Revision: 1} + } + raw, _ := json.Marshal(map[string]any{"result": result}) + _ = m.Respond(raw) + }); err != nil { + t.Fatal(err) + } + + dir := t.TempDir() + broker := filepath.Join(dir, "broker") + raw, _ := json.Marshal(process.Account{URL: url, Node: "n1", Module: module, User: "runner", Password: "x"}) + if err := os.WriteFile(broker, raw, 0o600); err != nil { + t.Fatal(err) + } + logs := &syncBuffer{} + runCtx, stop := context.WithCancel(ctx) + done := make(chan error, 1) + go func() { + done <- run(runCtx, func(k string) string { + if k == process.BrokerFileEnv { + return broker + } + return "" + }, logs) + }() + t.Cleanup(func() { stop(); <-done }) + pipelines, err := js.KeyValue(ctx, module+"_pipelines") + if err != nil { + t.Fatal(err) + } + return whole{nc: nc, js: js, asks: asks, pipelines: pipelines, logs: logs} +} + // kvKept reads a bucket as the tools bundle's states do. type kvKept struct { ctx context.Context diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go index d0c242096..69b901247 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go @@ -58,7 +58,7 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) { defer cancel() const module = "mesh-delivery-pipeline" - for _, state := range []string{"pipelines", "kept", "toolchains", "findings", "first-views", "consumers", "aged", "recoveries", "queue"} { + for _, state := range []string{"pipelines", "kept", "toolchains", "findings", "first-views", "consumers", "aged", "recoveries", "queue", pipeline.StateUnfinished} { if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_" + state, History: 1}); err != nil { t.Fatal(err) } diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/unfinished.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/unfinished.go new file mode 100644 index 000000000..8d85d6eba --- /dev/null +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/unfinished.go @@ -0,0 +1,49 @@ +package main + +import ( + "context" + "errors" + + "github.com/nats-io/nats.go/jetstream" +) + +// kvUnfinished is the runner's state unfinished on the bus (design 47 H2 §786), as pipeline.UnfinishedStore reads it: a key +// it does not hold is nil. +type kvUnfinished struct { + ctx context.Context + kv jetstream.KeyValue +} + +func (u kvUnfinished) Get(key string) ([]byte, error) { + e, err := u.kv.Get(u.ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return nil, nil + } + if err != nil { + return nil, err + } + return e.Value(), nil +} + +func (u kvUnfinished) Put(key string, value []byte) error { + _, err := u.kv.Put(u.ctx, key, value) + return err +} + +func (u kvUnfinished) Delete(key string) error { return u.kv.Delete(u.ctx, key) } + +// Keys are the keys matching filter, `*` one key part. +func (u kvUnfinished) Keys(filter string) ([]string, error) { + lister, err := u.kv.ListKeysFiltered(u.ctx, filter) + if errors.Is(err, jetstream.ErrNoKeysFound) { + return nil, nil + } + if err != nil { + return nil, err + } + var out []string + for k := range lister.Keys() { + out = append(out, k) + } + return out, nil +} diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/unfinished_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/unfinished_integration_test.go new file mode 100644 index 000000000..5abc6df67 --- /dev/null +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/unfinished_integration_test.go @@ -0,0 +1,143 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/json" + "slices" + "sync/atomic" + "testing" + "time" + + "github.com/nats-io/nats.go" + + builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" + catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" + + "mesh-delivery-pipeline/internal/pipeline" +) + +// design 47 H2 §786, whole: of a pipeline of two movers one fails its unit step; pipeline-ended names the other in +// unfinished and the runner keeps it in its state unfinished; the next take-in's pipeline carries it, its composition is +// asked and passes (it is wanted), and its key goes once that pipeline ends passed. +func TestAMoverLeftUnfinishedIsPutInTheNextPipelineAndWanted(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + steps := catalog.StepStatuses{Lint: "bypass", Unit: "required", Integration: "bypass", Validate: "required", Composition: "required", + Review: "bypass", Approval: "bypass"} + regs := []catalog.Registration{{Module: "app", Branches: []string{"main"}, Origin: "o/app", Policy: catalog.Policy{Steps: steps}}, + {Module: "lib", Branches: []string{"main"}, Origin: "o/lib", Policy: catalog.Policy{Steps: steps}}} + var newest atomic.Int64 + newest.Store(1) + w := startWhole(t, ctx, regs, func(in []byte) catalog.RevisionAnswer { + var q catalog.RevisionInput + _ = json.Unmarshal(in, &q) + n := q.Revision + if n == 0 { + n = newest.Load() + } + lib := "l1" + if n == 2 { + lib = "l2" + } + return catalog.RevisionAnswer{Revision: &catalog.CatalogRevision{Revision: n, Kind: "trunk", Branch: "main", Source: n + 2}, + Entries: []catalog.CatalogEntry{{Module: "app", Commit: "c1", Fp: "src2:aa"}, + {Module: "lib", Commit: lib, Fp: "src2:" + lib, Artifacts: []catalog.Artifact{{Ref: "lib:lib", Language: "go", Kind: "program"}}}, + {Module: "mesh-build-go", Commit: "c0", Fp: "src2:go", Acts: []catalog.ActingRule{{Step: "build", On: "go", Seat: nodebuildgo.Seat}}}}} + }) + ended := make(chan seat.PipelineEnded, 4) + if _, err := w.nc.Subscribe("mesh.seat.mesh-pipeline-runner.event.pipeline-ended.>", func(m *nats.Msg) { + var pe seat.PipelineEnded + if json.Unmarshal(m.Data, &pe) == nil { + ended <- pe + } + }); err != nil { + t.Fatal(err) + } + unfinished, err := w.js.KeyValue(ctx, "mesh-delivery-pipeline_"+pipeline.StateUnfinished) + if err != nil { + t.Fatal(err) + } + takeIn := func(source, revision int64, entries ...catalog.TakeInEntry) { + raw, _ := json.Marshal(catalog.TakeInTaken{Branch: "main", Source: source, Revision: revision, Call: "0123456789abcdef", Entries: entries}) + if _, err := w.js.Publish(ctx, "mesh.mod.mesh-delivery-indexer.event.take-in.taken", raw); err != nil { + t.Fatal(err) + } + } + // answer ends every running step of key once, as its seat's holder says it: lib's unit fails when fail says so. + answered := map[string]bool{} + answer := func(key string, fail bool) func() bool { + return func() bool { + var p seat.Pipeline + if !readPipeline(ctx, w.pipelines, key, &p) { + return false + } + for _, s := range p.Steps { + open := s.Outcome == seat.StepResultOutcomeRunning || s.Outcome == seat.StepResultOutcomeWaitingForABuildNode + if !open || s.Ask == "" || answered[s.Ask+s.Module] { + continue + } + answered[s.Ask+s.Module] = true + m, _ := pipeline.MoverOf(p, s.Module) + outcome := "passed" + if fail && s.Module == "lib" && s.Step == seat.StepResultStepUnit { + outcome = "failed" + } + body := map[string]any{"v": 1, "id": s.Ask, "pipeline": key, "step": s.Step, "module": s.Module, "commit": m.Commit, + "fingerprint": m.Fingerprint, "node": "n1", "outcome": outcome, "artifact": s.Artifact} + if s.Seat == builds.Seat { + body["entries"] = []builds.BuildEntry{{Module: s.Module, Commit: m.Commit, Fingerprint: m.Fingerprint}} + } + raw, _ := json.Marshal(body) + if _, err := w.js.Publish(ctx, "mesh.seat."+s.Seat+".event.step-ended.n1", raw); err != nil { + t.Fatal(err) + } + } + return p.Outcome != seat.PipelineOutcomeRunning + } + } + endOf := func() seat.PipelineEnded { + select { + case pe := <-ended: + return pe + case <-ctx.Done(): + t.Fatalf("no pipeline-ended was said; the runner said:\n%s", w.logs.String()) + } + return seat.PipelineEnded{} + } + + takeIn(3, 1, catalog.TakeInEntry{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}, catalog.TakeInEntry{Module: "lib", Commit: "l1", Fingerprint: "src2:l1"}) + waitFor(t, ctx, w.logs, answer("main.1", true)) + if pe := endOf(); pe.Pipeline != "main.1" || pe.Outcome != seat.PipelineEndedOutcomeFailed || !slices.Equal(pe.Unfinished, []string{"app"}) { + t.Fatalf("pipeline-ended = %s %s unfinished %v, want main.1 failed leaving app unfinished\n%s", pe.Pipeline, pe.Outcome, pe.Unfinished, w.logs.String()) + } + if e, err := unfinished.Get(ctx, "main.app"); err != nil || !json.Valid(e.Value()) { + t.Fatalf("the state unfinished holds no main.app: %v", err) + } + + // The next take-in moves lib alone, at a new fingerprint: its pipeline carries app, whose composition is asked and passes. + newest.Store(2) + rev, _ := json.Marshal(catalog.CatalogRevisionEvent{V: 1, Revision: 2, Kind: "trunk", Branch: "main", Source: 4, CallKind: catalog.CatalogRevisionEventCallKindTakeIn}) + if _, err := w.js.Publish(ctx, catalog.SubjectCatalogRevision("main"), rev); err != nil { + t.Fatal(err) + } + takeIn(4, 2, catalog.TakeInEntry{Module: "lib", Commit: "l2", Fingerprint: "src2:l2"}) + waitFor(t, ctx, w.logs, answer("main.2", false)) + var p seat.Pipeline + if !readPipeline(ctx, w.pipelines, "main.2", &p) || !pipeline.Moves(p, "app") { + t.Fatalf("main.2 moves %+v, want app carried", p.Movers) + } + composed := false + for _, s := range p.Steps { + composed = composed || (s.Module == "app" && s.Step == seat.StepResultStepComposition && s.Outcome == seat.StepResultOutcomePassed && s.Ask != "") + } + if pe := endOf(); pe.Pipeline != "main.2" || pe.Outcome != seat.PipelineEndedOutcomePassed || !composed || len(pe.Unfinished) != 0 { + t.Fatalf("main.2 ended %s, app composed %t, unfinished %v\n%s", pe.Outcome, composed, pe.Unfinished, w.logs.String()) + } + if _, err := unfinished.Get(ctx, "main.app"); err == nil { + t.Error("main.app is still unfinished after main.2 passed") + } +} diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs.go index 20a754bcf..9daa4715d 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs.go @@ -38,19 +38,21 @@ func answerVerbs(conn *process.Conn, seats *seatlist.Part, v pipeline.Verbs) (fu return conn.AnswerSeat(seat.Seat, seatlist.Definition(seats.Reader, seat.Seat), writeHandlers(v)) } -// writeHandlers are the seat's write verbs the process answers: run, review, rerun, accept, cancel, end-again, start, and ask-again; and +// writeHandlers are the seat's write verbs the process answers: run, review, rerun, accept, cancel, end-again, place, +// place-unfinished (design 47 H2 §786), start, and ask-again; and // the test runs' verbs test-runs and test-runs-drop (A-469g, A-470i). func writeHandlers(v pipeline.Verbs) map[string]process.SeatHandler { return map[string]process.SeatHandler{ - seat.VerbRun: verbOf(func(c string, in seat.RunInput) (any, error) { return v.Run(c, in) }), - seat.VerbReview: verbOf(func(c string, in seat.ReviewInput) (any, error) { return v.Review(c, in) }), - seat.VerbRerun: verbOf(func(c string, in seat.RerunInput) (any, error) { return v.Rerun(c, in) }), - seat.VerbAccept: verbOf(func(c string, in seat.AcceptInput) (any, error) { return v.Accept(c, in) }), - seat.VerbCancel: verbOf(func(c string, in seat.CancelInput) (any, error) { return v.Cancel(c, in) }), - seat.VerbEndAgain: verbOf(func(c string, in seat.EndAgainInput) (any, error) { return v.EndAgain(c, in) }), - seat.VerbPlace: verbOf(func(c string, in seat.PlaceInput) (any, error) { return v.Place(c, in) }), - seat.VerbStart: verbOf(func(c string, in seat.StartInput) (any, error) { return v.Start(c, in) }), - seat.VerbAskAgain: verbOf(func(c string, in seat.AskAgainInput) (any, error) { return v.AskAgain(c, in) }), + seat.VerbRun: verbOf(func(c string, in seat.RunInput) (any, error) { return v.Run(c, in) }), + seat.VerbReview: verbOf(func(c string, in seat.ReviewInput) (any, error) { return v.Review(c, in) }), + seat.VerbRerun: verbOf(func(c string, in seat.RerunInput) (any, error) { return v.Rerun(c, in) }), + seat.VerbAccept: verbOf(func(c string, in seat.AcceptInput) (any, error) { return v.Accept(c, in) }), + seat.VerbCancel: verbOf(func(c string, in seat.CancelInput) (any, error) { return v.Cancel(c, in) }), + seat.VerbEndAgain: verbOf(func(c string, in seat.EndAgainInput) (any, error) { return v.EndAgain(c, in) }), + seat.VerbPlace: verbOf(func(c string, in seat.PlaceInput) (any, error) { return v.Place(c, in) }), + seat.VerbPlaceUnfinished: verbOf(func(c string, in seat.PlaceUnfinishedInput) (any, error) { return v.PlaceUnfinished(c, in) }), + seat.VerbStart: verbOf(func(c string, in seat.StartInput) (any, error) { return v.Start(c, in) }), + seat.VerbAskAgain: verbOf(func(c string, in seat.AskAgainInput) (any, error) { return v.AskAgain(c, in) }), // §728, A-469g: a test run's drop is answered by the process, the only writer of pipelines and test-runs. seat.VerbTestRunsDrop: verbOf(func(c string, in seat.TestRunsDropInput) (any, error) { return v.TestRunsDrop(c, in) }), // A-470i: test-runs too, since the process holds test-runs-cap-mib, the cap its fill is said against. diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs_test.go index d5745ddcb..0946dd8c6 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs_test.go @@ -23,7 +23,7 @@ func TestEveryWriteVerbIsServed(t *testing.T) { got = append(got, v) } slices.Sort(got) - want := []string{seat.VerbAccept, seat.VerbAskAgain, seat.VerbCancel, seat.VerbEndAgain, seat.VerbPlace, seat.VerbRerun, seat.VerbReview, seat.VerbStart, seat.VerbRun, seat.VerbTestRuns, seat.VerbTestRunsDrop} + want := []string{seat.VerbAccept, seat.VerbAskAgain, seat.VerbCancel, seat.VerbEndAgain, seat.VerbPlace, seat.VerbPlaceUnfinished, seat.VerbRerun, seat.VerbReview, seat.VerbStart, seat.VerbRun, seat.VerbTestRuns, seat.VerbTestRunsDrop} slices.Sort(want) if !slices.Equal(got, want) { t.Errorf("the process answers %v, want %v", got, want) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/ended_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/ended_test.go index 69464e34c..edc405b6a 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/ended_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/ended_test.go @@ -2,6 +2,8 @@ package pipeline import ( "errors" + "maps" + "slices" "strings" "testing" "time" @@ -87,14 +89,23 @@ func TestOnePipelineEndedNamesEveryMoverWithItsTier(t *testing.T) { // R6/AC1 and AC2 (design 47 H §746; ADR 0682 §8, H-A-3): a mover's required step that ends `nothing to run` counts as // passed, so the pipeline passes; one mover's required step failed fails the pipeline, naming that mover's step. +// +// design 47 H2 §786: the failed end names the other mover in pipeline-ended's unfinished and keeps it in the state +// unfinished; the failing mover's own key goes (it waits for its fingerprint to move), and a passed end deletes its movers' +// keys. func TestThePipelinePassesOnNothingToRunAndFailsOnOneMover(t *testing.T) { for _, c := range []struct { name, lib, want string + unfinished []string + keys string }{ - {"nothing to run", seat.StepResultOutcomeNothingToRun, seat.PipelineOutcomePassed}, - {"one mover failed", seat.StepResultOutcomeFailed, seat.PipelineOutcomeFailed}, + {"nothing to run", seat.StepResultOutcomeNothingToRun, seat.PipelineOutcomePassed, nil, ""}, + {"one mover failed", seat.StepResultOutcomeFailed, seat.PipelineOutcomeFailed, []string{"app"}, + `main.app={"fingerprint":"src2:aa","revision":0,"pipeline":"main.5","outcome":"failed"}`}, } { - h, store, _, _ := twoMovers(t) + h, store, bodies, _ := twoMovers(t) + u := memUnfinished{"main.lib": `{"fingerprint":"src2:bb"}`, "main.app": `{"fingerprint":"src2:aa"}`} + h.Unfinished = &Unfinished{Store: u} for _, ev := range []StepEnded{ended("a1", "lint", "passed"), libEnded(c.lib)} { if _, err := h.OnStepEnded(goSeat, goSeat.Prefix+"n1", ev); err != nil { t.Fatal(err) @@ -104,9 +115,44 @@ func TestThePipelinePassesOnNothingToRunAndFailsOnOneMover(t *testing.T) { if p.Outcome != c.want || (c.want == seat.PipelineOutcomeFailed && !strings.Contains(p.Why, "lib lib:lib unit failed")) { t.Errorf("%s: the pipeline ended %s (%s), want %s", c.name, p.Outcome, p.Why, c.want) } + pe, _ := bodies["pipeline-ended.main.5"].(seat.PipelineEnded) + if !slices.Equal(pe.Unfinished, c.unfinished) || u.String() != c.keys { + t.Errorf("%s: pipeline-ended's unfinished %v and the state %q, want %v and %q", c.name, pe.Unfinished, u.String(), c.unfinished, c.keys) + } } } +// memUnfinished is a state unfinished in memory. +type memUnfinished map[string]string + +func (m memUnfinished) Get(k string) ([]byte, error) { + if v, ok := m[k]; ok { + return []byte(v), nil + } + return nil, nil +} +func (m memUnfinished) Put(k string, v []byte) error { m[k] = string(v); return nil } +func (m memUnfinished) Delete(k string) error { delete(m, k); return nil } +func (m memUnfinished) Keys(filter string) ([]string, error) { + prefix := strings.TrimSuffix(filter, "*") + var out []string + for k := range m { + if strings.HasPrefix(k, prefix) { + out = append(out, k) + } + } + return out, nil +} + +// String is the state's keys and values in key order. +func (m memUnfinished) String() string { + var out []string + for _, k := range slices.Sorted(maps.Keys(m)) { + out = append(out, k+"="+m[k]) + } + return strings.Join(out, " ") +} + // R8 (design 47 H §746, H-A-52, H-A-62): a version-released writes its version into the mover of that module in the pipeline // of the source revision built, on the branch it was released for, so pipeline-ended names it; another branch's pipeline, // and a version heard again, write nothing. diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/moverkeys.go b/modules/mesh-delivery-pipeline/internal/pipeline/moverkeys.go index 6d6d43522..a10595152 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/moverkeys.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/moverkeys.go @@ -29,7 +29,8 @@ func StepsKey(pipeline, mover string) string { func stepsPrefix(pipeline string) string { return StepsKey(pipeline, "") } // IsStepsKey says whether a key of the state pipelines is a mover's steps rather than a pipeline's record: `steps.` and a -// pipeline key of a take-in's (4 parts), a run's (5 parts, its third `run`) or a publish's (7 parts) and one mover. +// pipeline key of a take-in's (4 parts), a run's (5 parts, its third `run`), an unfinished mover's own (6 parts, its fourth +// `unfinished`, H-A-398) or a publish's (7 parts) and one mover. func IsStepsKey(key string) bool { parts := strings.Split(key, ".") if parts[0] != "steps" { @@ -40,6 +41,8 @@ func IsStepsKey(key string) bool { return true case 5: return parts[2] == runToken + case 6: + return parts[3] == unfinishedToken } return false } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/released.go b/modules/mesh-delivery-pipeline/internal/pipeline/released.go index 8e5056302..f885b6ba4 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/released.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/released.go @@ -25,7 +25,7 @@ func (h *Hearer) Released(ev catalog.VersionReleasedEvent, branch string) (int, // A pipeline is keyed by its revision (H-A-213 (4)), so the pipelines of a source are those naming it, of either key // form a source's pipelines were kept under. var pkeys []string - for _, f := range []string{b + ".*", b + ".*.published.*.*"} { + for _, f := range []string{b + ".*", b + ".*.published.*.*", b + ".*." + unfinishedToken + ".*"} { k, err := h.Store.Keys(f) if err != nil { return 0, fmt.Errorf("the pipelines of source %d are not listed: %w", ev.Source, err) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index e2617d4d9..b326380dc 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -67,9 +67,9 @@ func keyRevision(b Begin) int64 { return b.Source } -// keyForms are the filters of every key form of the state pipelines: a revision's, a run's, and a publish's kept from before -// a release revision started one pipeline (H-A-213 (2)). -var keyForms = []string{"*.*", "*." + runToken + ".*", "*.*.published.*.*"} +// keyForms are the filters of every key form of the state pipelines: a revision's, a run's, a publish's kept from before +// a release revision started one pipeline (H-A-213 (2)), and an unfinished mover's pipeline of its own (H-A-398). +var keyForms = []string{"*.*", "*." + runToken + ".*", "*.*.published.*.*", "*.*." + unfinishedToken + ".*"} // AllKeys are every pipeline's key, of each key form (§746, §747; H-A-83). func AllKeys(store interface { @@ -220,6 +220,9 @@ type Starter struct { // (1), H-A-328): read when a mover's wait on it is written, so a publish made before the wait places the mover without // any event; nil reads nothing, the publish event alone placing it. AskPublished func(in builds.PublishedInput) (builds.PublishedAnswer, error) + // Unfinished puts the movers an earlier pipeline left unfinished in the pipeline of the branch's next catalog revision + // (design 47 H2 §786); nil puts none. + Unfinished *Unfinished publishedMu sync.Mutex publishedRead map[string]builds.PublishedAnswer @@ -992,11 +995,11 @@ func DecodeTakenIn(raw []byte) (catalog.TakeInTaken, error) { // skip, starts nothing; a second hearing writes nothing. view is the catalog half of the event's branch, read for the // entries the steps and the tiers are made from. func (s *Starter) TakenIn(ev catalog.TakeInTaken, view func(branch string) (Half, bool)) (int, error) { - if len(ev.Entries) == 0 { - s.logf("the take-in of %s source %d names no entries: it starts no pipeline", ev.Branch, ev.Source) - return 0, nil - } if ev.Branch == "" || ev.Source <= 0 { + if len(ev.Entries) == 0 { + s.logf("the take-in of %s source %d names no entries: it starts no pipeline", ev.Branch, ev.Source) + return 0, nil + } return 0, fmt.Errorf("a take-in.taken names its watched branch and source revision: branch %q source %d", ev.Branch, ev.Source) } // A take-in heard before its branch's half is held starts its pipeline with no steps, never placed from empty entries @@ -1011,6 +1014,15 @@ func (s *Starter) TakenIn(ev catalog.TakeInTaken, view func(branch string) (Half for _, e := range ev.Entries { movers = append(movers, seat.Mover{Module: e.Module, Commit: e.Commit, Fingerprint: e.Fingerprint, Repository: e.Repository}) } + // The revision the take-in kept carries the movers an earlier pipeline left unfinished (design 47 H2 §786), judged + // against that revision's own entries. + if !ev.Skipped && ev.Revision > 0 { + movers = s.carryUnfinished(ev.Branch, ev.Revision, half, held, movers) + } + if len(movers) == 0 { + s.logf("the take-in of %s source %d names no entries: it starts no pipeline", ev.Branch, ev.Source) + return 0, nil + } movers = Tiers(movers, half.Entries, s.logf) ok, err := s.Start(Begin{Branch: ev.Branch, Source: ev.Source, Call: ev.Call, Revision: ev.Revision, Movers: movers, Entries: half.Entries, unheld: !held}) @@ -1030,6 +1042,27 @@ func (s *Starter) TakenIn(ev catalog.TakeInTaken, view func(branch string) (Half return started, err } +// carryUnfinished adds to the movers of revision of branch the unfinished movers whose fingerprint is still their entry's +// there (§786): half when it is that revision, else the revision read by its number; an unread revision carries none, said. +func (s *Starter) carryUnfinished(branch string, revision int64, half Half, held bool, movers []seat.Mover) []seat.Mover { + if s.Unfinished == nil { + return movers + } + if !held || half.Revision != revision { + if s.Revision == nil { + s.logf("revision %d of %s is not read, so it carries no unfinished mover: the runner holds no revision reader", revision, branch) + return movers + } + read, err := s.Revision(branch, revision) + if err != nil { + s.logf("revision %d of %s is not read, so it carries no unfinished mover: %v", revision, branch, err) + return movers + } + half = read + } + return s.Unfinished.Carry(branch, revision, half.Entries, movers) +} + // Release places the movers of pipeline key whose awaited publishes are all heard now (design 47 H §746, H-A-84). It says // how many pipelines it placed in. func (s *Starter) Release(store Updater, key string, view func(branch string) (Half, bool)) (int, error) { @@ -1147,18 +1180,21 @@ func (s *Starter) Revised(ev catalog.CatalogRevisionEvent, read func(branch stri if half.Revision != ev.Revision { return 0, fmt.Errorf("the revision verb answered revision %d of %s for %d", half.Revision, branch, ev.Revision) } - if len(half.Movers) == 0 { - s.logf("the revision %d of %s (%s) names no mover: it starts no pipeline", ev.Revision, branch, ev.Call) - return 0, nil - } movers := make([]seat.Mover, 0, len(half.Movers)) for _, e := range half.Movers { movers = append(movers, seat.Mover{Module: e.Module, Commit: e.Commit, Fingerprint: e.Fingerprint, Repository: e.Repository}) } // A revision of the same source moves no mover out of the open pipeline of that source holding it: that pipeline keeps // it, and this revision starts no pipeline of its own for it (H-A-283); only a revision of a new source moves movers. - if movers = s.keepSameSource(branch, half, movers); len(movers) == 0 { - s.logf("the revision %d of %s (%s) moves only movers the open pipelines of its source %d keep: it starts no pipeline (H-A-283)", ev.Revision, branch, ev.Call, half.Source) + if len(movers) > 0 { + if movers = s.keepSameSource(branch, half, movers); len(movers) == 0 { + s.logf("the revision %d of %s (%s) moves only movers the open pipelines of its source %d keep (H-A-283)", ev.Revision, branch, ev.Call, half.Source) + } + } + // It carries the movers an earlier pipeline left unfinished, or starts a pipeline for them alone (design 47 H2 §786). + movers = s.carryUnfinished(branch, half.Revision, half, true, movers) + if len(movers) == 0 { + s.logf("the revision %d of %s (%s) moves no mover: it starts no pipeline", ev.Revision, branch, ev.Call) return 0, nil } movers = Tiers(movers, half.Entries, s.logf) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go index 8994d7427..b26b82a7b 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go @@ -392,6 +392,50 @@ func TestTakenInStartsOnlyItsEntries(t *testing.T) { if p, _, err := mem.Get("main.12"); err != nil || len(p.Movers) != 1 || !Moves(p, "moved") { t.Errorf("main.12 = %+v (%v), want the one mover moved: batched is not named by the take-in", p, err) } + + // design 47 H2 §786: the next kept revision carries each unfinished mover still at its entry's fingerprint, one journal + // line each; a key the revision moves or holds at another fingerprint, or not at all, goes; a revision that moves + // nothing starts a pipeline for the unfinished alone; place-unfinished puts one in a pipeline of its own, a hand act. + var journal []string + s.Logf = func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) } + u := memUnfinished{"main.batched": `{"fingerprint":"src2:bb","revision":11}`, "main.moved": `{"fingerprint":"src2:00"}`, + "main.gone": `{"fingerprint":"src2:cc"}`, "dev.batched": `{"fingerprint":"src2:bb"}`} + s.Unfinished = &Unfinished{Store: u, Logf: s.Logf} + half.Revision = 13 + ev.Revision, ev.Source = 13, 10 + if n, err := s.TakenIn(ev, view); err != nil || n != 1 { + t.Fatalf("the take-in of revision 13 started %d (%v)", n, err) + } + if p, _, _ := mem.Get("main.13"); len(p.Movers) != 2 || !Moves(p, "moved") || !Moves(p, "batched") { + t.Errorf("main.13 moves %+v, want moved and the unfinished batched", p.Movers) + } + if u.String() != `dev.batched={"fingerprint":"src2:bb"} main.batched={"fingerprint":"src2:bb","revision":11}` { + t.Errorf("the state unfinished = %s, want main.moved (moved at src2:aa) and main.gone (no entry) gone", u.String()) + } + if !strings.Contains(strings.Join(journal, "\n"), "revision 13 of main puts batched in a pipeline again") { + t.Errorf("the journal says %q, want batched put in a pipeline again", journal) + } + half.Revision = 14 + if n, err := s.TakenIn(catalog.TakeInTaken{Branch: "main", Source: 11, Revision: 14}, view); err != nil || n != 1 { + t.Fatalf("a take-in moving nothing started %d (%v), want one pipeline for the unfinished", n, err) + } + if p, _, _ := mem.Get("main.14"); len(p.Movers) != 1 || !Moves(p, "batched") { + t.Errorf("main.14 moves %+v, want batched alone", p.Movers) + } + half.Revision = 11 + acts := &actLog{} + v := Verbs{Starter: s, View: view, Acts: acts} + if a, err := v.PlaceUnfinished("agent.dev", seat.PlaceUnfinishedInput{Key: "main.batched", Why: "now"}); err != nil || a.Pipeline != "main.11.unfinished.batched" || len(*acts) != 1 { + t.Errorf("place-unfinished = %+v (%v), %d hand acts", a, err, len(*acts)) + } + if p, _, _ := mem.Get("main.11.unfinished.batched"); len(p.Movers) != 1 || !Moves(p, "batched") { + t.Errorf("main.11.unfinished.batched moves %+v, want batched", p.Movers) + } + for _, in := range []seat.PlaceUnfinishedInput{{Key: "main.batched", Why: "again"}, {Key: "main.none", Why: "x"}, {Key: "dev.batched"}} { + if _, err := v.PlaceUnfinished("agent.dev", in); err == nil { + t.Errorf("place-unfinished %+v is refused", in) + } + } } // R16/AC1 (design 47 H §765, H-A-58, H-A-98): a take-in.taken of the old form, with no call (or no branch or source), is diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go index 9645e77ba..eff88bfe0 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go @@ -186,6 +186,9 @@ type Hearer struct { // Rejoin keys a pipeline running again once its approval is answered and it still runs (H-A-106 (b); Starter.Rejoin); // nil keys none. Rejoin func(key string) + // Unfinished keeps, at every end, the movers it leaves unfinished in the state unfinished (design 47 H2 §786); nil keeps + // none. + Unfinished *Unfinished } // EndReaders ends each of readers' steps of run with r's result (§725, §726): a step that read an ended run, or the steps @@ -488,6 +491,7 @@ func (h *Hearer) said(p seat.Pipeline, asked []int, ended bool) error { h.logf("%s ended %s", p.Pipeline, p.Outcome) } h.Failures.Ended(p) + h.Unfinished.Ended(p) if h.RegistrationWaits != nil { h.RegistrationWaits() } @@ -666,7 +670,7 @@ func stepsOf(p seat.Pipeline, module string) []seat.StepResult { // pipelineEnded is p's pipeline-ended (design 47 H §746): its key, branch, source, call, revision and outcome, and per // mover, in the delivery order of its tiers, its module, fingerprint, commit, widens (H-A-119), released version, step results -// and tier. +// and tier; and the movers its end left unfinished (design 47 H2 §786). func pipelineEnded(p seat.Pipeline) seat.PipelineEnded { movers := make([]seat.MoverEnded, 0, len(p.Movers)) for _, m := range p.Movers { @@ -676,7 +680,7 @@ func pipelineEnded(p seat.Pipeline) seat.PipelineEnded { sort.SliceStable(movers, func(i, j int) bool { return movers[i].Tier < movers[j].Tier }) return seat.PipelineEnded{V: 1, Pipeline: p.Pipeline, Branch: p.Branch, Source: p.Source, Call: p.Call, Revision: p.Revision, Run: p.Run, Outcome: p.Outcome, Why: p.Why, Started: p.Started, Ended: p.Ended, EndCount: p.EndCount, Movers: movers, - Phases: phasesOf(p)} + Phases: phasesOf(p), Unfinished: unfinishedOf(p)} } // phasesOf is p's phases (design 47 H2 §785): taken, when the runner wrote the pipeline at its take-in; placed, the diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/unfinished.go b/modules/mesh-delivery-pipeline/internal/pipeline/unfinished.go new file mode 100644 index 000000000..dfc20deb6 --- /dev/null +++ b/modules/mesh-delivery-pipeline/internal/pipeline/unfinished.go @@ -0,0 +1,222 @@ +package pipeline + +import ( + "encoding/json" + "errors" + "fmt" + "slices" + "strconv" + "strings" + + catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + "git.novox.be/novox/mesh-sdk/go/keys" +) + +// StateUnfinished is the runner's state of the movers left unfinished by another mover's failure (design 47 H2 §786, §820): +// one key `.` per such mover, deleted by the runner or by runner_unfinished_drop. +const StateUnfinished = "unfinished" + +// unfinishedToken is the third part of the key of a pipeline of its own of an unfinished mover, +// `..unfinished.` (H-A-398). +const unfinishedToken = "unfinished" + +// UnfinishedEntry is one value of the state unfinished (§786): the mover's fingerprint, the catalog revision and the pipeline +// it was left unfinished in, and that pipeline's outcome. +type UnfinishedEntry struct { + Fingerprint string `json:"fingerprint"` + Revision int64 `json:"revision"` + Pipeline string `json:"pipeline"` + Outcome string `json:"outcome"` +} + +// UnfinishedStore is the state unfinished: a key's value (nil when it has none), a put, a delete and the keys matching a +// filter, `*` one key part. +type UnfinishedStore interface { + Get(key string) ([]byte, error) + Put(key string, value []byte) error + Delete(key string) error + Keys(filter string) ([]string, error) +} + +// UnfinishedKey is a mover's key in the state unfinished, `.`, each escaped as a key's name is (§786). +func UnfinishedKey(branch, module string) string { + return keys.EscapeName(branch) + "." + keys.EscapeName(module) +} + +// UnfinishedPipelineKey is the key of an unfinished mover's pipeline of its own, `..unfinished.` +// (H-A-398). +func UnfinishedPipelineKey(branch string, revision int64, module string) string { + return seat.PipelinesByBranchRevisionUnfinishedModuleKey(keys.EscapeName(branch), strconv.FormatInt(revision, 10), keys.EscapeName(module)) +} + +// neverRan are the outcomes of a step that never ran: still waiting, not asked, cancelled, held by a dependency that did not +// build, or waiting for a build node (§786). +var neverRan = []string{seat.StepResultOutcomeWaiting, seat.StepResultOutcomeNotAsked, seat.StepResultOutcomeCancelled, + seat.StepResultOutcomeDidNotBuild, seat.StepResultOutcomeWaitingForABuildNode} + +// leftUnfinished says whether a pipeline's end leaves movers unfinished: a delivering pipeline (no run on request) that ended +// failed or cancelled (§786; a could-not-run step ends a pipeline failed, H-A-398). +func leftUnfinished(p seat.Pipeline) bool { + return p.Run == "" && (p.Outcome == seat.PipelineOutcomeFailed || p.Outcome == seat.PipelineOutcomeCancelled) +} + +// unfinishedOf are the movers p's end leaves unfinished (§786): in a pipeline ended failed or cancelled, each mover no newer +// revision moved out of it whose own required steps all passed or never ran. +func unfinishedOf(p seat.Pipeline) []string { + if !leftUnfinished(p) { + return nil + } + var out []string + for _, m := range p.Movers { + if slices.Contains(p.Superseded, m.Module) { + continue + } + if ownStepsHeld(p, m.Module) { + out = append(out, m.Module) + } + } + return out +} + +// ownStepsHeld says whether every required step of module in p passed or never ran. +func ownStepsHeld(p seat.Pipeline, module string) bool { + for _, s := range p.Steps { + if s.Module != module || s.Status != seat.StepResultStatusRequired { + continue + } + if !stepPassed(s) && !slices.Contains(neverRan, s.Outcome) { + return false + } + } + return true +} + +// Unfinished keeps the movers left unfinished in the state unfinished and puts them in a pipeline again (§786). +type Unfinished struct { + Store UnfinishedStore + Logf func(string, ...any) +} + +func (u *Unfinished) logf(format string, args ...any) { + if u.Logf != nil { + u.Logf(format, args...) + } +} + +// read is one key's entry, and whether it is held. +func (u *Unfinished) read(key string) (UnfinishedEntry, bool, error) { + raw, err := u.Store.Get(key) + if err != nil || raw == nil { + return UnfinishedEntry{}, false, err + } + var e UnfinishedEntry + if err := json.Unmarshal(raw, &e); err != nil { + return UnfinishedEntry{}, false, fmt.Errorf("the unfinished entry %s is not read: %w", key, err) + } + return e, true, nil +} + +// Ended takes a pipeline's end into the state unfinished (§786): a mover p leaves unfinished is kept under its key; a mover +// whose pipeline ended passed, or whose own required step did not pass (it waits for its fingerprint to move), has its key +// deleted when the key names its fingerprint. A run on request's pipeline, and one every mover left, change nothing. +func (u *Unfinished) Ended(p seat.Pipeline) { + if u == nil || p.Run != "" { + return + } + switch p.Outcome { + case seat.PipelineOutcomePassed, seat.PipelineOutcomeFailed, seat.PipelineOutcomeCancelled: + default: + return + } + left := unfinishedOf(p) + for _, m := range p.Movers { + if slices.Contains(p.Superseded, m.Module) { + continue + } + key := UnfinishedKey(p.Branch, m.Module) + if slices.Contains(left, m.Module) { + raw, _ := json.Marshal(UnfinishedEntry{Fingerprint: m.Fingerprint, Revision: p.Revision, Pipeline: p.Pipeline, Outcome: p.Outcome}) + if err := u.Store.Put(key, raw); err != nil { + u.logf("%s ended %s and leaves %s unfinished, but it is not kept in the state unfinished: %v", p.Pipeline, p.Outcome, m.Module, err) + continue + } + u.logf("%s ended %s and leaves %s unfinished: it is put in a pipeline again at %s's next catalog revision (§786)", p.Pipeline, p.Outcome, m.Module, p.Branch) + continue + } + e, held, err := u.read(key) + if err != nil { + u.logf("%s: the unfinished entry of %s is not read: %v", p.Pipeline, m.Module, err) + continue + } + if !held || e.Fingerprint != m.Fingerprint { + continue + } + if err := u.Store.Delete(key); err != nil { + u.logf("%s: the unfinished entry of %s is not deleted: %v", p.Pipeline, m.Module, err) + continue + } + u.logf("%s ended %s: %s is no longer unfinished", p.Pipeline, p.Outcome, m.Module) + } +} + +// Carry adds to the movers of a catalog revision of branch each unfinished mover whose fingerprint is still its entry's in +// entries, the revision's entries, with one journal line per mover added again (§786). An unfinished mover the revision +// moves at a newer fingerprint, or whose entry holds another fingerprint or none, has its key deleted; one it moves at its +// own fingerprint is carried by the revision already. It answers movers with the added ones after them. +func (u *Unfinished) Carry(branch string, revision int64, entries map[string]catalog.CatalogEntry, movers []seat.Mover) []seat.Mover { + if u == nil { + return movers + } + ukeys, err := u.Store.Keys(keys.EscapeName(branch) + ".*") + if err != nil { + u.logf("the unfinished movers of %s are not listed, so revision %d carries none: %v", branch, revision, err) + return movers + } + slices.Sort(ukeys) + for _, key := range ukeys { + _, name, _ := strings.Cut(key, ".") + module, err := keys.UnescapeName(name) + if err != nil { + continue + } + e, held, err := u.read(key) + if err != nil || !held { + if err != nil { + u.logf("revision %d of %s: %v", revision, branch, err) + } + continue + } + if i := slices.IndexFunc(movers, func(m seat.Mover) bool { return m.Module == module }); i >= 0 { + if movers[i].Fingerprint != e.Fingerprint { + u.dropMoved(key, module, fmt.Sprintf("revision %d of %s moves it at %s", revision, branch, movers[i].Fingerprint)) + } + continue + } + entry, ok := entries[module] + if !ok || entry.Fp != e.Fingerprint { + u.dropMoved(key, module, fmt.Sprintf("revision %d of %s holds it at %q", revision, branch, entry.Fp)) + continue + } + movers = append(movers, seat.Mover{Module: module, Commit: entry.Commit, Fingerprint: entry.Fp, Repository: entry.Repository}) + u.logf("revision %d of %s puts %s in a pipeline again: %s left it unfinished (%s) at fingerprint %s (§786)", revision, branch, module, e.Pipeline, e.Outcome, e.Fingerprint) + } + return movers +} + +// dropMoved deletes the key of an unfinished mover a newer fingerprint moves (§786). +func (u *Unfinished) dropMoved(key, module, why string) { + if err := u.Store.Delete(key); err != nil { + u.logf("the unfinished entry of %s is not deleted: %v", module, err) + return + } + u.logf("%s is no longer unfinished: %s", module, why) +} + +// Entry is one key's entry; false when the state holds none. +func (u *Unfinished) Entry(key string) (UnfinishedEntry, bool, error) { + if u == nil { + return UnfinishedEntry{}, false, errors.New("the runner keeps no state unfinished") + } + return u.read(key) +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go index 5aa00c890..7934aa8f0 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go @@ -726,3 +726,63 @@ func (v Verbs) Place(caller string, in seat.PlaceInput) (seat.PlaceAnswer, error p, _, err = s.Store.Get(in.Pipeline) return seat.PlaceAnswer{Pipeline: p}, err } + +// PlaceUnfinished puts a mover kept in the state unfinished in a pipeline of its own at once, recorded in the hand-act log +// first (design 47 H2 §786, H-A-398): keyed `..unfinished.`, its steps placed from the entries of +// the catalog revision the entry names, the mover at the entry's fingerprint. The key stays until that pipeline ends passed. +// A key the state does not hold, a revision whose entry holds another fingerprint, or a pipeline written already is refused, +// nothing done. +func (v Verbs) PlaceUnfinished(_ string, in seat.PlaceUnfinishedInput) (seat.PlaceUnfinishedAnswer, error) { + s := v.Starter + if strings.TrimSpace(in.Key) == "" { + return seat.PlaceUnfinishedAnswer{}, errors.New("place-unfinished names the unfinished entry's key") + } + if s == nil || s.Unfinished == nil { + return seat.PlaceUnfinishedAnswer{}, errors.New("the runner keeps no state unfinished: nothing was placed") + } + e, held, err := s.Unfinished.Entry(in.Key) + if err != nil { + return seat.PlaceUnfinishedAnswer{}, fmt.Errorf("the unfinished entry %s is not read: %w", in.Key, err) + } + if !held { + return seat.PlaceUnfinishedAnswer{}, fmt.Errorf("the state unfinished holds no %s: nothing was placed", in.Key) + } + b, m, _ := strings.Cut(in.Key, ".") + branch, berr := keys.UnescapeName(b) + module, merr := keys.UnescapeName(m) + if berr != nil || merr != nil || branch == "" || module == "" { + return seat.PlaceUnfinishedAnswer{}, fmt.Errorf("%s is no key .: nothing was placed", in.Key) + } + var half Half + switch { + case s.Revision != nil: + if half, err = s.Revision(branch, e.Revision); err != nil { + return seat.PlaceUnfinishedAnswer{}, fmt.Errorf("the revision %d of %s is not read: nothing was placed: %w", e.Revision, branch, err) + } + case v.View != nil: + var ok bool + if half, ok = v.View(branch); !ok || half.Revision != e.Revision { + return seat.PlaceUnfinishedAnswer{}, fmt.Errorf("the revision %d of %s is not held: nothing was placed", e.Revision, branch) + } + default: + return seat.PlaceUnfinishedAnswer{}, errors.New("the runner reads no catalog revision: nothing was placed") + } + entry, ok := half.Entries[module] + if !ok || entry.Fp != e.Fingerprint || half.Source <= 0 { + return seat.PlaceUnfinishedAnswer{}, fmt.Errorf("the revision %d of %s holds %s at %q, not its unfinished fingerprint %s: nothing was placed", e.Revision, branch, module, entry.Fp, e.Fingerprint) + } + key := UnfinishedPipelineKey(branch, e.Revision, module) + if err := v.record(seat.VerbPlaceUnfinished, "put the unfinished mover "+module+" of "+branch+" in the pipeline "+key+" by hand", in.Why); err != nil { + return seat.PlaceUnfinishedAnswer{}, err + } + started, err := s.Start(Begin{Branch: branch, Source: half.Source, Revision: e.Revision, Entries: half.Entries, Key: key, + Movers: []seat.Mover{{Module: module, Commit: entry.Commit, Fingerprint: entry.Fp, Repository: entry.Repository, Tier: 1}}}) + if err != nil { + return seat.PlaceUnfinishedAnswer{}, err + } + if !started { + return seat.PlaceUnfinishedAnswer{}, fmt.Errorf("the pipeline %s is started already: nothing was placed", key) + } + s.logf("%s puts the unfinished mover %s in a pipeline of its own by hand (§786)", key, module) + return seat.PlaceUnfinishedAnswer{Pipeline: key}, nil +} diff --git a/modules/mesh-delivery-pipeline/manifest_test.go b/modules/mesh-delivery-pipeline/manifest_test.go index 5c6ef7684..49cc90e35 100644 --- a/modules/mesh-delivery-pipeline/manifest_test.go +++ b/modules/mesh-delivery-pipeline/manifest_test.go @@ -55,7 +55,9 @@ func TestManifest(t *testing.T) { {"claims the seat serving every verb of its definition", len(m.Claims) == 1 && m.Claims[0].Name == seat.Seat && equalSets(m.Claims[0].Serves, seat.Verbs())}, // §27, §85, §330, §355, §77: the old state sections. - {"old state sections", equalSets(states, []string{seat.StatePipelines, seat.StateKept, "aged", "recoveries", "bounds", "test-runs", "toolchains", "findings", "first-views", "consumers", seat.StateQueue})}, + {"old state sections", equalSets(states, []string{seat.StatePipelines, seat.StateKept, "aged", "recoveries", "bounds", "test-runs", "toolchains", "findings", "first-views", "consumers", seat.StateQueue, "unfinished"})}, + // design 47 H2 §786, §820: unfinished is a state of 8 MiB. + {"unfinished 8 MiB", stateMiB["unfinished"] == 8}, // §535, §647, A-77, A-104: an ask no holder takes is bound by a key with its own time to live. {"bounds is key-ttl", keyTTL["bounds"]}, // §725, A-469w, A-470: test-runs holds its runs' bound keys, each with its own time to live; 64 MiB its ceiling. @@ -117,7 +119,7 @@ func TestManifest(t *testing.T) { // RN25 (§539): the runner's tools carry the prefix runner_. {"first-view tools", equalSets(m.Tools, []string{"runner_first_views", "runner_first_view_again", "runner_health", "runner_unlogged", "runner_unlogged_append", "runner_start", "runner_run", "runner_accept", "runner_ask_again", "runner_place", "runner_end_again", "runner_waiting", - "runner_test_runs", "runner_test_runs_drop"})}, + "runner_test_runs", "runner_test_runs_drop", "runner_unfinished", "runner_unfinished_drop", "runner_unfinished_place"})}, // A-305 (ADR 0227 rule 8, ADR 0240 rule 2): the process is judged by its unit, as every process of mesh-delivery, // never by a tool of its own; runner_health stays a tool. {"health is the unit", healthTool(raw) == "unit"}, diff --git a/modules/mesh-delivery-pipeline/module.json b/modules/mesh-delivery-pipeline/module.json index ebe2436af..fabd0a8b3 100644 --- a/modules/mesh-delivery-pipeline/module.json +++ b/modules/mesh-delivery-pipeline/module.json @@ -20,6 +20,7 @@ "cancel", "end-again", "place", + "place-unfinished", "ask-again", "test-runs", "test-runs-drop" @@ -40,7 +41,10 @@ "runner_end_again", "runner_waiting", "runner_test_runs", - "runner_test_runs_drop" + "runner_test_runs_drop", + "runner_unfinished", + "runner_unfinished_drop", + "runner_unfinished_place" ], "consumes": [ "mesh-delivery-indexer.take-in.taken", @@ -217,6 +221,11 @@ { "name": "consumers", "history": 1 + }, + { + "name": "unfinished", + "history": 1, + "max-mib": 8 } ], "streams": [ From d719a69b7de6696db4ff86f3c78f31755edd5b91 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 01:08:34 +0100 Subject: [PATCH 04/15] I3: a dry run keeps its packages in its own scratch state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A branch's take-in, a run's candidate and a branch's re-pin compute as dry runs whose packages went to the trunk's packages state. They now write them to the state dry-runs under ..packages., read through to the live state, deleted when the dry run answers and every key at the indexer's start; the manifests stay create-once by digest as every take-in's. indexer_dry_runs reads the state; the manifest asks for it, 16 MiB, one hour's TTL. Builds: design 47 H2 §804, §820; H-A-413 and its addendum --- .../cmd/mesh-delivery-indexer-tools/main.go | 1 + .../mesh-delivery-indexer/integration_test.go | 4 + .../cmd/mesh-delivery-indexer/main.go | 10 +- .../packages_integration_test.go | 65 ++++- .../internal/index/candidates.go | 5 +- .../internal/index/dryrun.go | 267 +++++++++++++++++- .../internal/index/packages_gone_test.go | 41 ++- .../internal/index/runcandidates.go | 7 +- .../internal/index/takein_source.go | 3 +- .../internal/index/toolchains.go | 55 ++-- .../mesh-delivery-indexer/manifest_test.go | 9 +- modules/mesh-delivery-indexer/module.json | 6 +- 12 files changed, 419 insertions(+), 54 deletions(-) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go index 09fd60658..d1bf3dcac 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go @@ -212,6 +212,7 @@ func main() { _, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.TakeInSkipTool, Verb: "take-in-skip"}) return err })...) + tools = append(tools, index.DryRunsTools(keptRead{stdio.State(index.DryRunsState)})...) tools = append(tools, index.RunsTools(keptRead{stdio.State(index.RunsState)}, keptRead{stdio.State(index.CandidatesState)}, asks, time.Now, func(what, why string) error { _, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.RunDropTool, Verb: "run-drop"}) return err diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go index 737b27c38..c9bd95909 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go @@ -363,6 +363,10 @@ func startIndexerRun(t *testing.T, put func(context.Context, buildLog), before . if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(testIndexer, index.ManifestsState), History: index.ManifestsHistory}); err != nil { t.Fatal(err) } + // The dry runs' scratch, made by the grants part from the manifest's state entry dry-runs (design 47 H2 §804, §820). + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(testIndexer, index.DryRunsState), History: 1, TTL: time.Hour, MaxBytes: 16 << 20}); err != nil { + t.Fatal(err) + } // The builder's module log, as the runtime raises it (ADR 0297), filled by put before the indexer starts. if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buscompose.LogStreamName(builds.BuilderModule, builds.BuilderLog), Subjects: []string{buscompose.LogSubject(builds.BuilderModule, builds.BuilderLog) + ".>"}, AllowDirect: true}); err != nil { diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index aca38fc31..043bad17d 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -333,7 +333,15 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro defer stopInput() // The state takeins: hand acts ask there (watched below), and a running take-in keeps what it waits on (H2 §803, H-A-409). takeIns := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.TakeInsState), "no hand act of the indexer is answered", logger.Printf) - ix := &index.Indexer{Module: cred.Module, TakeIns: awaitedStore{b: takeIns}, NamedSets: named, Findings: writer, OldSections: awaitedStore{b: oldSections}, Requirements: awaitedStore{b: requirements}, Store: kvStore{candidates}, Modules: modules, Manifests: kvStore{manifests}, Taken: awaitedStore{b: taken}, Builds: known, + // The state dry-runs, a dry run's scratch of the packages it writes (H2 §804): every key it holds goes at this start, before + // any dry run, so no dry run outlives a start. + dryRuns := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.DryRunsState), "no dry run is computed: a branch's take-in, a run's candidate and a branch's re-pin wait for it", logger.Printf) + dryRuns.Ready(func(kv jetstream.KeyValue) { + if err := index.ClearDryRuns(ctx, kvStore{kv}); err != nil { + logger.Printf("the state %s is not cleared at this start: %v", index.DryRunsState, err) + } + }) + ix := &index.Indexer{Module: cred.Module, TakeIns: awaitedStore{b: takeIns}, DryRuns: awaitedStore{b: dryRuns}, NamedSets: named, Findings: writer, OldSections: awaitedStore{b: oldSections}, Requirements: awaitedStore{b: requirements}, Store: kvStore{candidates}, Modules: modules, Manifests: kvStore{manifests}, Taken: awaitedStore{b: taken}, Builds: known, Predefined: awaitedStore{b: predefined}, Logf: logger.Printf, Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), BranchHead: branchHeadVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call), Packages: awaitedStore{b: packages, js: js, name: index.Bucket(cred.Module, index.PackagesState)}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call), diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/packages_integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/packages_integration_test.go index e2f463232..9976e707b 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/packages_integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/packages_integration_test.go @@ -5,7 +5,9 @@ package main import ( "context" "encoding/json" + "errors" "os" + "reflect" "slices" "strconv" "strings" @@ -62,14 +64,17 @@ func untilModule(t *testing.T, ctx context.Context, js jetstream.JetStream, name // go.sum line names it, ADR 0573 §2; design 47 H-A-157), and plain, which requires // nothing of it, are taken in with mesh-sdk as main's first source revision. A source revision moving mesh-sdk to a commit // changing its buildstep package is taken in as one revision whose movers are mesh-sdk and importer, of another repository, -// and not plain; the packages key of mesh-sdk/go names that commit with a digest per package. +// and not plain; the packages key of mesh-sdk/go names that commit with a digest per package. Design 47 H2 §804: a key a dry +// run left in the state dry-runs is gone at the start, and the take-in of a branch moving mesh-sdk again, a dry run, leaves +// the trunk's packages state byte for byte as before and no key in dry-runs once it answered. func TestASourceRevisionMovingMeshSDKNamesItAndItsImporters(t *testing.T) { const ( sdkGo = "git.novox.be/novox/mesh-sdk/go" - m1, m2 = "1111111111111111111111111111111111111111", "2222222222222222222222222222222222222222" + m1, m2, m3 = "1111111111111111111111111111111111111111", "2222222222222222222222222222222222222222", "4444444444444444444444444444444444444444" catalogMerge = "3333333333333333333333333333333333333333" goModContent = "module importer\n\nrequire git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000\n" sdkRoot1, sdk2 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + sdk3 = "cccccccccccccccccccccccccccccccccccccccc" ) avahi, err := os.ReadFile("../../internal/index/testdata/catalogue-a67f0196-avahi.module.json") if err != nil { @@ -92,7 +97,7 @@ func TestASourceRevisionMovingMeshSDKNamesItAndItsImporters(t *testing.T) { "package.json": "3333333333333333333333333333333333333333", "package-lock.json": "4444444444444444444444444444444444444444", "tsconfig.json": "5555555555555555555555555555555555555555"}}} } - var packages jetstream.KeyValue + var packages, dryRuns jetstream.KeyValue var heard *takeIns js, ctx, _, _ := startIndexer(t, func(context.Context, buildLog) {}, func(ctx context.Context, js jetstream.JetStream) { var err error @@ -118,6 +123,14 @@ func TestASourceRevisionMovingMeshSDKNamesItAndItsImporters(t *testing.T) { forge := serveCommits(t, js.Conn()) forge.put(m1, sdkAt(sdkRoot1, "f5", "6666666666666666666666666666666666666666")) forge.put(m2, sdkAt(sdk2, "f6", "7777777777777777777777777777777777777777")) + forge.put(m3, sdkAt(sdk3, "f7", "8888888888888888888888888888888888888888")) + // A dry run's key left by an indexer that stopped mid-run (§804). + if dryRuns, err = js.KeyValue(ctx, index.Bucket(testIndexer, index.DryRunsState)); err != nil { + t.Fatal(err) + } + if _, err := dryRuns.Put(ctx, index.DryRunPrefix("beta", "0123")+index.PackagesState+".left", []byte(`{"value":"e30="}`)); err != nil { + t.Fatal(err) + } forge.put(catalogMerge, repoAt{trees: map[string]string{"modules/importer": "t1", "modules/plain": "t2"}, files: map[string]string{"modules/importer/module.json": named("importer"), "modules/plain/module.json": named("plain"), "modules/importer/go.mod": goModContent, "modules/importer/go.sum": "golang.org/x/sys v0.1.0 h1:x=\n", @@ -125,7 +138,7 @@ func TestASourceRevisionMovingMeshSDKNamesItAndItsImporters(t *testing.T) { regs := registerAvahi(t, ctx, js, "main") for _, r := range []meshcatalog.Registration{{V: 1, Module: "importer", Origin: "novox/mesh-catalog", Path: "modules/importer", Branches: []string{"main"}}, {V: 1, Module: "plain", Origin: "novox/mesh-catalog", Path: "modules/plain", Branches: []string{"main"}}, - {V: 1, Module: "mesh-sdk", Origin: "novox/mesh-sdk", Path: ".", Branches: []string{"main"}}} { + {V: 1, Module: "mesh-sdk", Origin: "novox/mesh-sdk", Path: ".", Branches: []string{"main", "beta"}}} { raw, _ := json.Marshal(r) if _, err := regs.Put(ctx, index.RegistrationKey(r.Module), raw); err != nil { t.Fatal(err) @@ -134,16 +147,40 @@ func TestASourceRevisionMovingMeshSDKNamesItAndItsImporters(t *testing.T) { seedBuildModules(t, ctx, js, nil) heard = hearTakeIns(t, js.Conn()) }) - say := func(source int64, start bool, heads ...meshdelivery.SourceMovedEventHeadsItem) { + sayOn := func(branch string, source int64, start bool, heads ...meshdelivery.SourceMovedEventHeadsItem) { t.Helper() - raw, _ := json.Marshal(meshdelivery.SourceMovedEvent{V: 1, Branch: "main", Source: source, Call: "0123456789abcdef", Start: start, Heads: heads}) - if _, err := js.Publish(ctx, meshdelivery.SubjectSourceMoved("main", strconv.FormatInt(source, 10)), raw); err != nil { + raw, _ := json.Marshal(meshdelivery.SourceMovedEvent{V: 1, Branch: branch, Source: source, Call: "0123456789abcdef", Start: start, Heads: heads}) + if _, err := js.Publish(ctx, meshdelivery.SubjectSourceMoved(branch, strconv.FormatInt(source, 10)), raw); err != nil { t.Fatal(err) } } + say := func(source int64, start bool, heads ...meshdelivery.SourceMovedEventHeadsItem) { + t.Helper() + sayOn("main", source, start, heads...) + } + // every is each key of kv with its value. + every := func(kv jetstream.KeyValue) map[string]string { + t.Helper() + out := map[string]string{} + ks, err := kv.Keys(ctx) + if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) { + t.Fatal(err) + } + for _, k := range ks { + e, err := kv.Get(ctx, k) + if err != nil { + t.Fatal(err) + } + out[k] = string(e.Value()) + } + return out + } say(10, true, meshdelivery.SourceMovedEventHeadsItem{Repository: "novox/mesh-sdk", Commit: m1}, meshdelivery.SourceMovedEventHeadsItem{Repository: "novox/mesh-catalog", Commit: catalogMerge}) heard.next(t, "main", 10) heard.acked(t, "main") + if left := every(dryRuns); len(left) != 0 { + t.Errorf("dry-runs holds %v after the start, want every key gone (§804)", left) + } // mesh-sdk has released no version: importer keeps the package for its kind-1 edge, with no pin (H-A-161). untilModule(t, ctx, js, "importer", func(m meshcatalog.ModulesByModuleValue) bool { pin, kept := m.Packages[sdkGo] @@ -169,4 +206,18 @@ func TestASourceRevisionMovingMeshSDKNamesItAndItsImporters(t *testing.T) { pub, err := index.DecodePublishedPackages(v) return v != nil && err == nil && pub.Commit == m2 && len(pub.Packages) == 3 }) + + // A branch's take-in moving mesh-sdk to m3 is a dry run: its packages stay in dry-runs while it runs, the trunk's are + // untouched, and its keys go once it answered (§804). + heard.acked(t, "main") + before := every(packages) + sayOn("beta", 1, true, meshdelivery.SourceMovedEventHeadsItem{Repository: "novox/mesh-catalog", Commit: catalogMerge}, meshdelivery.SourceMovedEventHeadsItem{Repository: "novox/mesh-sdk", Commit: m3}) + heard.next(t, "beta", 1) + heard.acked(t, "beta") + if after := every(packages); !reflect.DeepEqual(after, before) { + t.Errorf("the branch's take-in changed the trunk's packages state:\n%v\nwant\n%v", after, before) + } + if left := every(dryRuns); len(left) != 0 { + t.Errorf("dry-runs holds %v once the branch's take-in answered, want none", left) + } } diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index 90dbbde3c..4f4df5137 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -132,6 +132,9 @@ type Indexer struct { Bound func(context.Context, time.Duration) (context.Context, context.CancelFunc) // TakeIns is the state takeins, where a running take-in keeps what it waits on (design 47 H2 §803, H-A-409); nil keeps none. TakeIns Store + // DryRuns is the state dry-runs, a dry run's scratch of the packages it writes (dryrun.go; design 47 H2 §804); nil keeps + // a dry run's packages in memory. + DryRuns Store Builds *Builds // Named asks the forge for the git object of each named path of the module directories given, at a commit // (the seat git's trees with paths; novox/hq ADR 0499 clauses 2 and 3): the indexer is the one reader of a @@ -161,7 +164,7 @@ type Indexer struct { filling atomic.Bool // a FillTaken runs in the background (KickFill) fillAgain atomic.Bool // a kick arrived while it ran underSeats bool // a dry run inside judge.WithSeats: its judge passes do not hold the seat list (verdicts.go) - isDry bool // a dry run (dryrun.go): it shares the live packages state, so it drops no kept package key (packages.go) + isDry bool // a dry run (dryrun.go): its packages are a working copy over the live state, and it drops no kept package key (packages.go) judgeMu sync.Mutex // one judge pass at a time (verdicts.go) keepMu sync.Mutex // one catalog graph revision kept at a time (keepEntries) // Pins is the state pins (pins.go); nil keeps none. PinsPending says how many messages the indexer's durable consumer of the diff --git a/modules/mesh-delivery-indexer/internal/index/dryrun.go b/modules/mesh-delivery-indexer/internal/index/dryrun.go index 2fe12d418..51b3cb61d 100644 --- a/modules/mesh-delivery-indexer/internal/index/dryrun.go +++ b/modules/mesh-delivery-indexer/internal/index/dryrun.go @@ -1,12 +1,20 @@ package index import ( + "context" + "encoding/json" + "errors" "fmt" "maps" "reflect" + "slices" + "strings" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/keys" + "git.novox.be/novox/mesh-sdk/go/views" ) type dryRun struct { @@ -14,12 +22,20 @@ type dryRun struct { modules, graph *overlayStore candidates, taken, predefined *overlayStore oldSections *overlayStore + // scratch is the state dry-runs this dry run's packages are written to, under prefix (§804); nil when the indexer keeps + // none, and the packages then stay in memory. + scratch Store + prefix string + logf func(string, ...any) } -// dry is an indexer over overlays of this one's states (overlay.go), for a run on request's candidate (runcandidates.go): what -// it computes stays in memory and nothing reaches the bus but the manifests and the packages, which clause 1 writes all the -// same. It says no event, kicks nothing in the background, and knows what this one knows of the source and the toolchains. Every field of Indexer is named here (TestDryNamesEveryField holds it so). -func (ix *Indexer) dry() *dryRun { +// dry is an indexer over overlays of this one's states (overlay.go), for a run on request's candidate (runcandidates.go), a +// branch's take-in or re-pin: what it computes stays in memory but the packages it writes, kept in the state dry-runs under +// the branch and call given (scratchStore; design 47 H2 §804, H-A-413), so the trunk's packages state is left untouched; its +// manifests are written create-once by digest as every take-in's (H-A-413's addendum). The caller ends it (end) once it +// answered. It says no event, kicks nothing in the background, and knows what this one knows of the source and the +// toolchains. Every field of Indexer is named here. +func (ix *Indexer) dry(branch, call string) *dryRun { over := func(st Store) (Store, *overlayStore) { if st == nil { return nil, nil @@ -27,8 +43,14 @@ func (ix *Indexer) dry() *dryRun { o := newOverlay(st) return o, o } - d := &dryRun{} - x := &Indexer{Module: ix.Module, Manifests: ix.Manifests, Packages: ix.Packages, JudgeID: ix.JudgeID, Files: ix.Files, GoSum: ix.GoSum, + d := &dryRun{scratch: ix.DryRuns, prefix: DryRunPrefix(branch, call), logf: ix.logf} + packages := ix.Packages + if packages != nil && d.scratch != nil { + packages = &scratchStore{under: ix.Packages, scratch: d.scratch, prefix: d.prefix + PackagesState + "."} + } else if packages != nil { + packages = newOverlay(ix.Packages) + } + x := &Indexer{Module: ix.Module, Manifests: ix.Manifests, Packages: packages, DryRuns: nil, JudgeID: ix.JudgeID, Files: ix.Files, GoSum: ix.GoSum, GoMod: ix.GoMod, Now: ix.Now, Bound: ix.Bound, // A dry run keeps no waiting-on: the take-in it computes for says its waits through the live indexer (H-A-409). TakeIns: nil, Builds: ix.Builds, Named: ix.Named, Trunk: ix.Trunk, BranchHead: ix.BranchHead, Read: ix.Read, Contains: ix.Contains, @@ -85,3 +107,236 @@ func (ix *Indexer) refuseDisagreement(c meshcatalog.CandidatesByGroupValue) erro } return nil } + +// DryRunsState is the indexer's state dry-runs (design 47 H2 §804, §820; H-A-413): a dry run's scratch, the overlay of the +// packages state a dry run writes, keyed ... (DryRunPrefix, the kind the overlaid state's name and +// the name its key), so a dry run leaves the trunk's packages state untouched. The indexer deletes a dry run's keys when it +// answers, and every key at its start; each lives an hour at most (the state's TTL). indexer_dry_runs reads it. +const ( + DryRunsState = "dry-runs" + DryRunsTool = "indexer_dry_runs" +) + +// DryRunPrefix is the prefix of a dry run's keys in the state dry-runs: its branch, escaped as a key's name is, and its call. +func DryRunPrefix(branch, call string) string { + return keys.EscapeName(branch) + "." + keys.EscapeName(call) + "." +} + +// scratchRevision lifts a revision of the state dry-runs above every revision of the state it overlays, as overlayStore does. +const scratchRevision = 1 << 62 + +// scratched is a value of the state dry-runs: the value a dry run wrote over its key, or that it deleted it. +type scratched struct { + Value []byte `json:"value,omitempty"` + Deleted bool `json:"deleted,omitempty"` +} + +// scratchStore is a store whose writes go to the state dry-runs under its prefix, over the store it reads through: a key +// it wrote reads as written, a key it deleted as absent, every other as the store under it holds it. Its compare-and-set +// holds against what it reads, a key it wrote carrying the dry-runs revision lifted by scratchRevision. +type scratchStore struct { + under, scratch Store + prefix string +} + +func (s *scratchStore) held(ctx context.Context, key string) (scratched, uint64, bool, error) { + raw, rev, found, err := s.scratch.Get(ctx, s.prefix+key) + if err != nil || !found { + return scratched{}, 0, false, err + } + var v scratched + if err := json.Unmarshal(raw, &v); err != nil { + return v, 0, false, fmt.Errorf("%s.%s%s: %w", DryRunsState, s.prefix, key, err) + } + return v, rev, true, nil +} + +func (s *scratchStore) Get(ctx context.Context, key string) ([]byte, uint64, bool, error) { + v, rev, found, err := s.held(ctx, key) + if err != nil { + return nil, 0, false, err + } + if found { + if v.Deleted { + return nil, rev + scratchRevision, false, nil + } + return v.Value, rev + scratchRevision, true, nil + } + if s.under == nil { + return nil, 0, false, nil + } + return s.under.Get(ctx, key) +} + +// put checks the revision read against what the key reads as now and writes it into the state dry-runs. +func (s *scratchStore) put(ctx context.Context, key string, value []byte, revision uint64, create, del bool) error { + _, own, mine, err := s.held(ctx, key) + if err != nil { + return err + } + _, rev, found, err := s.Get(ctx, key) + if err != nil { + return err + } + if create && found || !create && (!found || rev != revision) { + return ErrConflict + } + data, err := json.Marshal(scratched{Value: value, Deleted: del}) + if err != nil { + return err + } + if mine { + return s.scratch.Update(ctx, s.prefix+key, data, own) + } + return s.scratch.Create(ctx, s.prefix+key, data) +} + +func (s *scratchStore) Create(ctx context.Context, key string, value []byte) error { + return s.put(ctx, key, value, 0, true, false) +} + +func (s *scratchStore) Update(ctx context.Context, key string, value []byte, revision uint64) error { + return s.put(ctx, key, value, revision, false, false) +} + +func (s *scratchStore) Delete(ctx context.Context, key string, revision uint64) error { + return s.put(ctx, key, nil, revision, false, true) +} + +func (s *scratchStore) Keys(ctx context.Context) ([]string, error) { + var out []string + if s.under != nil { + var err error + if out, err = s.under.Keys(ctx); err != nil { + return nil, err + } + } + all, err := s.scratch.Keys(ctx) + if err != nil { + return nil, err + } + mine := map[string]bool{} + for _, k := range all { + if !strings.HasPrefix(k, s.prefix) { + continue + } + key := strings.TrimPrefix(k, s.prefix) + v, _, found, err := s.held(ctx, key) + if err != nil { + return nil, err + } + if found { + mine[key] = !v.Deleted + } + } + out = slices.DeleteFunc(out, func(k string) bool { _, ok := mine[k]; return ok }) + for k, live := range mine { + if live { + out = append(out, k) + } + } + slices.Sort(out) + return out, nil +} + +// end deletes the dry run's keys from the state dry-runs, once it answered (§804); a key it cannot delete is said, and goes +// with the state's TTL or at the indexer's next start. +func (d *dryRun) end(ctx context.Context) { + if d == nil || d.scratch == nil { + return + } + if err := dropPrefixed(ctx, d.scratch, d.prefix); err != nil { + d.logf("the dry run's keys %s* are not all deleted from %s: %v", d.prefix, DryRunsState, err) + } +} + +// ClearDryRuns deletes every key of the state dry-runs, at the indexer's start (§804): no dry run outlives a start. +func ClearDryRuns(ctx context.Context, st Store) error { return dropPrefixed(ctx, st, "") } + +func dropPrefixed(ctx context.Context, st Store, prefix string) error { + all, err := st.Keys(ctx) + if err != nil { + return err + } + var errs []error + for _, k := range all { + if !strings.HasPrefix(k, prefix) { + continue + } + _, rev, found, err := st.Get(ctx, k) + if err == nil && found { + err = st.Delete(ctx, k, rev) + } + if err != nil && !errors.Is(err, ErrConflict) { + errs = append(errs, fmt.Errorf("%s: %w", k, err)) + } + } + return errors.Join(errs...) +} + +// DryRunKey is one key of the state dry-runs as indexer_dry_runs answers it: the key, the value the dry run wrote (none when +// it deleted the key) and whether it deleted it. +type DryRunKey struct { + Key string `json:"key"` + Value json.RawMessage `json:"value,omitempty"` + Deleted bool `json:"deleted,omitempty"` +} + +// DryRunsAnswer is indexer_dry_runs' answer: the keys in key order up to views.PageBytes, capped when more are held, and +// how many the state holds in all. +type DryRunsAnswer struct { + Keys []DryRunKey `json:"keys"` + Capped bool `json:"capped,omitempty"` + Total int `json:"total"` +} + +// ReadDryRuns is what the state dry-runs holds now, at most views.PageBytes of values (§804, §811). +func ReadDryRuns(ctx context.Context, st Store) (DryRunsAnswer, error) { + out := DryRunsAnswer{Keys: []DryRunKey{}} + if st == nil { + return out, nil + } + all, err := st.Keys(ctx) + if err != nil { + return out, err + } + slices.Sort(all) + out.Total = len(all) + size := 0 + for _, k := range all { + raw, _, found, err := st.Get(ctx, k) + if err != nil { + return out, err + } + if !found { + continue + } + var v scratched + if err := json.Unmarshal(raw, &v); err != nil { + return out, fmt.Errorf("%s.%s: %w", DryRunsState, k, err) + } + if size += len(k) + len(v.Value); size > views.PageBytes && len(out.Keys) > 0 { + out.Capped = true + break + } + e := DryRunKey{Key: k, Deleted: v.Deleted} + if !v.Deleted { + e.Value = v.Value + } + out.Keys = append(out.Keys, e) + } + return out, nil +} + +// DryRunsTools are the dry runs' tools: indexer_dry_runs reads the state dry-runs (§804). +func DryRunsTools(st Store) []stdio.Tool { + return []stdio.Tool{{Name: DryRunsTool, + Description: "The dry runs' scratch state dry-runs (design 47 H2 §804): the packages a branch's take-in, a run's candidate " + + "or a branch's re-pin computes, kept as ..packages. while it runs, so the trunk's packages state " + + "is left untouched; a dry run's keys go when it answers and every key at the indexer's start. Each key with the " + + "value it wrote or deleted, up to 768 KB (capped) and the total held.", + Input: map[string]any{"type": "object", "properties": map[string]any{}}, + Run: func(map[string]any) (any, error) { + return ReadDryRuns(context.Background(), st) + }}} +} diff --git a/modules/mesh-delivery-indexer/internal/index/packages_gone_test.go b/modules/mesh-delivery-indexer/internal/index/packages_gone_test.go index 56ac91e81..4c3b23c0d 100644 --- a/modules/mesh-delivery-indexer/internal/index/packages_gone_test.go +++ b/modules/mesh-delivery-indexer/internal/index/packages_gone_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "reflect" "strings" "testing" "time" @@ -115,18 +116,26 @@ func TestARepositorysPackagesAreThoseInTheMergesTree(t *testing.T) { }) for _, row := range []struct { - name string - dry bool - branch string + name string + dry, scratch bool + branch string }{ - {"a branch's take-in keeps the key: only the trunk's merge says a directory is gone", false, "feat/x"}, - {"a dry run keeps the key: it shares the live packages state", true, TrunkBranch}, + {"a branch's take-in keeps the key: only the trunk's merge says a directory is gone", false, false, "feat/x"}, + {"a dry run keeps the key: its packages are a working copy over the live state", true, false, TrunkBranch}, + // Design 47 H2 §804: over the state dry-runs, a dry run's packages are kept there alone, under its branch and call, + // read by indexer_dry_runs and gone at its end; the live packages state is left as it was. + {"a dry run over the state dry-runs writes its packages there alone and drops them at its end", true, true, TrunkBranch}, } { t.Run(row.name, func(t *testing.T) { s := newSetup(t, live, gone) - ix := s.ix + if row.scratch { + s.ix.DryRuns = newMem() + } + before := s.ix.publishedAll(ctx) + ix, d := s.ix, (*dryRun)(nil) if row.dry { - ix = s.ix.dry().ix + d = s.ix.dry("beta", "t") + ix = d.ix } if err := ix.publish(ctx, catalogueRepo, merge, reaching(live), false, row.branch); err != nil { t.Fatal(err) @@ -134,6 +143,24 @@ func TestARepositorysPackagesAreThoseInTheMergesTree(t *testing.T) { if _, kept := s.ix.publishedAll(ctx)[gone]; !kept { t.Errorf("the kept key of %s was dropped", gone) } + if !row.scratch { + return + } + if got := s.ix.publishedAll(ctx); !reflect.DeepEqual(got, before) { + t.Errorf("the live packages state is %+v after the dry run, want %+v as before", got, before) + } + if mine := ix.publishedAll(ctx); mine[live].Commit != merge { + t.Errorf("the dry run reads %s at %q, want its own write at %s", live, mine[live].Commit, merge) + } + read, err := DryRunsTools(s.ix.DryRuns)[0].Run(map[string]any{}) + a, _ := read.(DryRunsAnswer) + if err != nil || a.Total == 0 || len(a.Keys) != a.Total || !strings.HasPrefix(a.Keys[0].Key, DryRunPrefix("beta", "t")+PackagesState+".") { + t.Errorf("indexer_dry_runs = %+v (%v), want the dry run's packages under beta.t.packages.", read, err) + } + d.end(ctx) + if a, err := ReadDryRuns(ctx, s.ix.DryRuns); err != nil || a.Total != 0 { + t.Errorf("dry-runs holds %+v (%v) after the dry run's end, want none", a, err) + } }) } diff --git a/modules/mesh-delivery-indexer/internal/index/runcandidates.go b/modules/mesh-delivery-indexer/internal/index/runcandidates.go index bfcfead88..9353b42b1 100644 --- a/modules/mesh-delivery-indexer/internal/index/runcandidates.go +++ b/modules/mesh-delivery-indexer/internal/index/runcandidates.go @@ -211,6 +211,11 @@ func (ix *Indexer) writeRunCandidate(ctx context.Context, group string, members pulls := make([]meshcatalog.CandidatesByGroupValuePullsItem, len(placed)) // One take-in per base branch: the members on one base are taken in together, so one member sees another's. bases := map[string]*dryRun{} + defer func() { + for _, d := range bases { + d.end(ctx) // the run's scratch goes once it answered (design 47 H2 §804) + } + }() for i, m := range placed { d, ok := bases[m.base] if !ok { @@ -218,7 +223,7 @@ func (ix *Indexer) writeRunCandidate(ctx context.Context, group string, members if err != nil { return fmt.Errorf("the run %s: the revision view of %s: %w", group, m.base, err) } - d = ix.dry() + d = ix.dry(m.base, group) d.ix.Modules, d.ix.Graph, d.ix.Taken = view, nodes, newOverlay(nil) d.ix.Logf = func(format string, args ...any) { ix.logf("the run "+group+", computed and not written: "+format, args...) diff --git a/modules/mesh-delivery-indexer/internal/index/takein_source.go b/modules/mesh-delivery-indexer/internal/index/takein_source.go index ecb392210..4d108e3a5 100644 --- a/modules/mesh-delivery-indexer/internal/index/takein_source.go +++ b/modules/mesh-delivery-indexer/internal/index/takein_source.go @@ -79,7 +79,8 @@ func (ix *Indexer) takeInSource(ctx context.Context, e meshdelivery.SourceMovedE if err != nil { return taken, err } - d := ix.dry() + d := ix.dry(e.Branch, e.Call) + defer d.end(ctx) d.ix.Modules, d.ix.Graph, d.ix.Taken = view, nodes, newOverlay(nil) d.ix.Logf = func(format string, args ...any) { ix.logf("take-in of "+e.Branch+": "+format, args...) } x = d.ix diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains.go b/modules/mesh-delivery-indexer/internal/index/toolchains.go index 0a4553916..8300429f3 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains.go @@ -636,38 +636,43 @@ func (ix *Indexer) Repin(ctx context.Context, module, version string) (int, erro } named := 0 for _, branch := range branches { - x := ix - if !isTrunkBranch(branch) { - view, nodes, _, err := BranchView(ctx, ix.Revisions, branch) - if err != nil { - return named, err - } - d := ix.dry() - d.ix.Modules, d.ix.Graph, d.ix.Taken = view, nodes, newOverlay(nil) - d.ix.Logf = func(format string, args ...any) { ix.logf("re-pin of "+branch+": "+format, args...) } - x = d.ix - } - only, err := x.repinned(ctx, module) + n, err := ix.repinBranch(ctx, branch, module, version) + named += n if err != nil { return named, err } - if len(only) == 0 { - continue - } - named += len(only) - x.recompute.Lock() - err = x.recomputeLocked(ctx, nil, nil, "", only) - x.recompute.Unlock() - if err != nil { - return named, err - } - if err := ix.keepRepinned(ctx, x, branch, releaseCall(module, version), only); err != nil { - return named, err - } } return named, nil } +// repinBranch re-pins branch's importers of module at version: the trunk in place, another branch as a dry run over its +// revision view, whose scratch goes once it is kept (design 47 H2 §804). It answers how many modules it named. +func (ix *Indexer) repinBranch(ctx context.Context, branch, module, version string) (int, error) { + x, call := ix, releaseCall(module, version) + if !isTrunkBranch(branch) { + view, nodes, _, err := BranchView(ctx, ix.Revisions, branch) + if err != nil { + return 0, err + } + d := ix.dry(branch, call.Call) + defer d.end(ctx) + d.ix.Modules, d.ix.Graph, d.ix.Taken = view, nodes, newOverlay(nil) + d.ix.Logf = func(format string, args ...any) { ix.logf("re-pin of "+branch+": "+format, args...) } + x = d.ix + } + only, err := x.repinned(ctx, module) + if err != nil || len(only) == 0 { + return 0, err + } + x.recompute.Lock() + err = x.recomputeLocked(ctx, nil, nil, "", only) + x.recompute.Unlock() + if err != nil { + return len(only), err + } + return len(only), ix.keepRepinned(ctx, x, branch, call, only) +} + // repinned are the modules the indexer's modules state holds whose packages name a package of module: by their pins, and // by their kept requirements, which name a package they hold no pin of yet (design 47 H-A-204, H-A-165, H-A-167). func (ix *Indexer) repinned(ctx context.Context, module string) (map[string]bool, error) { diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index 81ef37ba3..99f0974c0 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -31,6 +31,7 @@ type state struct { History int `json:"history,omitempty"` KeyTTL bool `json:"key-ttl,omitempty"` MaxMiB int `json:"max-mib,omitempty"` + TTL int `json:"ttl-seconds,omitempty"` } type artifact struct { @@ -138,7 +139,8 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build {Name: index.RequirementsState, History: 5}, {Name: index.SeatDefinitionsState, History: 5}, {Name: index.EntryIndexState, History: 1, MaxMiB: 8}, {Name: index.NamedState, History: 1, MaxMiB: 4}, // design 47 G §695 {Name: index.RegistrationsState, History: 1}, {Name: index.ProposalsState, History: 1, MaxMiB: 4}, - {Name: index.VersionsState, History: 1, MaxMiB: 64}, {Name: index.BuildEdgesState, History: 1}, {Name: index.InputsState, History: 1}, {Name: index.PinsState, History: 1, MaxMiB: index.PinsCapMiB}, {Name: index.HeldState, History: 1}}; !reflect.DeepEqual(m.State, want) { // DLV-I7, DLV-I17 + {Name: index.VersionsState, History: 1, MaxMiB: 64}, {Name: index.BuildEdgesState, History: 1}, {Name: index.InputsState, History: 1}, {Name: index.PinsState, History: 1, MaxMiB: index.PinsCapMiB}, {Name: index.HeldState, History: 1}, + {Name: index.DryRunsState, History: 1, TTL: 3600, MaxMiB: 16}}; !reflect.DeepEqual(m.State, want) { // DLV-I7, DLV-I17; design 47 H2 §804, §820 t.Errorf("state = %+v, want %+v", m.State, want) } // Milestone 47/G W4.2 (design 47, Two graphs; ADR 0619 §2): the catalog graph revisions are objects in the old section, @@ -206,8 +208,9 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // milestone 47/G W4.12 (design 47, the indexer's tools): the catalog graph revisions read and steered. index.CatalogRevisionTool, index.CatalogRevisionsTool, index.CandidateGraphsTool, index.RecomputeTool, index.DropCandidateGraphTool, index.EntryIndexTool, index.EntryPinsTool, index.NamedDropTool, // design 47 G §675, H-A-204 - index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.PinDropTool, index.PendingTool, index.TakeInPendingTool, index.TakeInAgainTool, index.TakeInSkipTool, index.RunsTool, index.RunDropTool) // design 47 G §667, §8, §579, §64 (A-194) - if !reflect.DeepEqual(m.Tools, want) || len(want) != 44 || !slices.Contains(m.Tools, "modules") { + index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.PinDropTool, index.PendingTool, index.TakeInPendingTool, index.TakeInAgainTool, index.TakeInSkipTool, index.RunsTool, index.RunDropTool, // design 47 G §667, §8, §579, §64 (A-194) + index.DryRunsTool) // design 47 H2 §804 + if !reflect.DeepEqual(m.Tools, want) || len(want) != 45 || !slices.Contains(m.Tools, "modules") { t.Errorf("tools = %v, want %v", m.Tools, want) } // Design 47 G §2, §9: every version of every registration is kept with no age limit, 64 MiB, on the old streams section. diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index bc75c22c2..8926fa796 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -70,7 +70,8 @@ "indexer_take_in_again", "indexer_take_in_skip", "indexer_runs", - "indexer_run_drop" + "indexer_run_drop", + "indexer_dry_runs" ], "consumes": [ "go-package-registry.published", @@ -119,7 +120,8 @@ { "name": "build-edges", "history": 1 }, { "name": "inputs", "history": 1 }, { "name": "pins", "history": 1, "max-mib": 4 }, - { "name": "held", "history": 1 } + { "name": "held", "history": 1 }, + { "name": "dry-runs", "history": 1, "ttl-seconds": 3600, "max-mib": 16 } ], "streams": [ { "name": "registration-history", "events": ["registration.*"], "retention": "history", "max-mib": 64 }, From cc5f803f8797e391c109d1961091ae6aa7b6d8d4 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 01:12:04 +0100 Subject: [PATCH 05/15] I5: the registrations are read in one call, filtered and paged MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The verb registrations and the new tool indexer_registrations filter by module and repository while they read, at most 768 KB a page, capped with the cursor next: the tools bundle reads the state from the bus's cursor (mesh/state.page), a store that pages none by its keys in order; total is said when one page holds the whole read. Builds: design 47 H2 §806, §811, §779 --- .../cmd/mesh-delivery-indexer-tools/main.go | 5 ++ .../internal/index/registrations.go | 75 ++++++++++++++--- .../internal/index/registrations_tools.go | 50 ++++++----- .../index/registrations_tools_test.go | 84 +++++++++++++++++++ .../mesh-delivery-indexer/manifest_test.go | 4 +- modules/mesh-delivery-indexer/module.json | 3 +- 6 files changed, 187 insertions(+), 34 deletions(-) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go index d1bf3dcac..6f7f219d9 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go @@ -156,6 +156,11 @@ func (k keptRead) Get(_ context.Context, key string) ([]byte, uint64, bool, erro func (k keptRead) Keys(context.Context) ([]string, error) { return k.s.Keys() } +// Page reads the state a page at a time from the bus's cursor (design 47 H2 §779): the registrations are read so (§806). +func (k keptRead) Page(cursor string, maxBytes int, budget time.Duration) (stdio.StatePage, error) { + return k.s.Page(cursor, maxBytes, budget) +} + func (keptRead) Create(context.Context, string, []byte) error { return errReadOnly } func (keptRead) Update(context.Context, string, []byte, uint64) error { return errReadOnly } diff --git a/modules/mesh-delivery-indexer/internal/index/registrations.go b/modules/mesh-delivery-indexer/internal/index/registrations.go index 519515998..9b0205979 100644 --- a/modules/mesh-delivery-indexer/internal/index/registrations.go +++ b/modules/mesh-delivery-indexer/internal/index/registrations.go @@ -16,7 +16,10 @@ import ( oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" gitseat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk" + stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/keys" + "git.novox.be/novox/mesh-sdk/go/views" ) // A module's registration, and the trusted change of one (design 47 G §2 to §4, §10, §342, §343, §667): the indexer keeps each @@ -169,15 +172,34 @@ func (r *Registry) Registration(ctx context.Context, module string) (meshcatalog return reg, true, nil } -// Registrations is the verb registrations (§4): every registration, or one module's, with its history. +// RegistrationsTool is the indexer's tool of the registrations (design 47 H2 §806): the verb registrations' answer, by hand. +const RegistrationsTool = "indexer_registrations" + +// StatePager reads a state a page at a time, in the order the bus wrote it, after an opaque cursor (mesh-sdk +// KeptState.Page, design 47 H2 §779). +type StatePager interface { + Page(cursor string, maxBytes int, budget time.Duration) (stdio.StatePage, error) +} + +// keyCursor is the cursor of a store read by its keys (a store that pages none): after this key, in key order. +const keyCursor = "key:" + +// RegistrationsAnswer is the verb registrations and the tool indexer_registrations (§4; design 47 H2 §806, §811): every +// registration, or one module's with its history, filtered by module and repository while it reads, at most views.PageBytes +// a page, capped with the cursor next when more is held; a store that pages (StatePager) is read from the bus's cursor, +// any other by its keys in order. total is how many the filters hold, said when one page holds the whole read. func (r *Registry) RegistrationsAnswer(ctx context.Context, in meshcatalog.RegistrationsInput) (meshcatalog.RegistrationsAnswer, error) { out := meshcatalog.RegistrationsAnswer{Registrations: []meshcatalog.Registration{}} + repository := trunk.RepositoryName(strings.TrimSpace(in.Repository)) + keep := func(reg meshcatalog.Registration) bool { + return (in.Module == "" || reg.Module == in.Module) && (repository == "" || trunk.SameRepository(reg.Origin, repository)) + } if in.Module != "" { reg, found, err := r.Registration(ctx, in.Module) if err != nil { return out, err } - if found { + if found && keep(reg) { out.Registrations = append(out.Registrations, reg) } if in.History && r.HistoryOf != nil { @@ -185,22 +207,53 @@ func (r *Registry) RegistrationsAnswer(ctx context.Context, in meshcatalog.Regis return out, fmt.Errorf("the history of %s cannot be read: %w", in.Module, err) } } + out.Total = int64(len(out.Registrations)) return out, nil } - all, err := r.Registrations.Keys(ctx) - if err != nil { - return out, err + match := func(value []byte) (meshcatalog.Registration, bool) { + var reg meshcatalog.Registration + return reg, json.Unmarshal(value, ®) == nil && keep(reg) } - slices.Sort(all) - for _, k := range all { - v, _, found, err := r.Registrations.Get(ctx, k) + if p, ok := r.Registrations.(StatePager); ok { + page, err := p.Page(in.Next, views.PageBytes, views.PageBudget) if err != nil { return out, err } - var reg meshcatalog.Registration - if found && json.Unmarshal(v, ®) == nil { - out.Registrations = append(out.Registrations, reg) + // A page holds at most views.PageBytes, so what the filters keep of it always fits one answer. + for _, e := range page.Entries { + if reg, ok := match(e.Value); ok { + out.Registrations = append(out.Registrations, reg) + } } + out.Next, out.Capped = page.Next, page.Next != "" + } else { + all, err := r.Registrations.Keys(ctx) + if err != nil { + return out, err + } + slices.Sort(all) + after, size := strings.TrimPrefix(in.Next, keyCursor), 0 + for _, k := range all { + if in.Next != "" && k <= after { + continue + } + v, _, found, err := r.Registrations.Get(ctx, k) + if err != nil { + return out, err + } + reg, ok := match(v) + if !found || !ok { + continue + } + if size += len(k) + len(v); size > views.PageBytes && len(out.Registrations) > 0 { + out.Capped, out.Next = true, keyCursor+after + return out, nil + } + out.Registrations, after = append(out.Registrations, reg), k + } + } + if in.Next == "" && !out.Capped { + out.Total = int64(len(out.Registrations)) } return out, nil } diff --git a/modules/mesh-delivery-indexer/internal/index/registrations_tools.go b/modules/mesh-delivery-indexer/internal/index/registrations_tools.go index 9baa5cf5b..0a61d5375 100644 --- a/modules/mesh-delivery-indexer/internal/index/registrations_tools.go +++ b/modules/mesh-delivery-indexer/internal/index/registrations_tools.go @@ -43,27 +43,37 @@ func RegistrationTools(registrations, proposals Store, ask func(key string, valu // in, never queued behind a running take-in (design 47 G A-370). Only a module's history is read from the stream the // indexer keeps, so that is still asked of the indexer. viaIndexer := act(ActRegistrations, true) + read := func(args map[string]any) (any, error) { + var in meshcatalog.RegistrationsInput + clean := map[string]any{} + for k, v := range args { + if k != "caller" && !strings.HasPrefix(k, "_") { + clean[k] = v + } + } + raw, err := json.Marshal(clean) + if err == nil { + err = json.Unmarshal(raw, &in) + } + if err != nil { + return nil, err + } + if in.History { + return viaIndexer(args) + } + return (&Registry{Registrations: registrations}).RegistrationsAnswer(context.Background(), in) + } return []stdio.Tool{ - {Name: meshcatalog.Seat + "." + meshcatalog.VerbRegistrations, Run: func(args map[string]any) (any, error) { - var in meshcatalog.RegistrationsInput - clean := map[string]any{} - for k, v := range args { - if k != "caller" && !strings.HasPrefix(k, "_") { - clean[k] = v - } - } - raw, err := json.Marshal(clean) - if err == nil { - err = json.Unmarshal(raw, &in) - } - if err != nil { - return nil, err - } - if in.History { - return viaIndexer(args) - } - return (&Registry{Registrations: registrations}).RegistrationsAnswer(context.Background(), in) - }}, + {Name: meshcatalog.Seat + "." + meshcatalog.VerbRegistrations, Run: read}, + {Name: RegistrationsTool, Description: "The registrations (design 47 H2 §806), as the seat mesh-catalog's verb registrations answers them: " + + "every module's and every source-only one, filtered by module or repository (owner/name) while it reads, at most 768 KB a page " + + "(capped), the rest read from the cursor next; one module's with every version (history).", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "module": map[string]any{"type": "string", "description": "one module; absent, every registration"}, + "repository": map[string]any{"type": "string", "description": "only the registrations of this repository, owner/name"}, + "history": map[string]any{"type": "boolean", "description": "with every version of the one module"}, + "next": map[string]any{"type": "string", "description": "the cursor the previous page answered"}}}, + Run: read}, {Name: meshcatalog.Seat + "." + meshcatalog.VerbRegister, Run: act(verbRegister, true)}, {Name: meshcatalog.Seat + "." + meshcatalog.VerbPolicy, Run: act(verbPolicy, true)}, {Name: meshcatalog.Seat + "." + meshcatalog.VerbWatch, Run: act(verbWatch, true)}, diff --git a/modules/mesh-delivery-indexer/internal/index/registrations_tools_test.go b/modules/mesh-delivery-indexer/internal/index/registrations_tools_test.go index f9be83ff9..98b77ccbd 100644 --- a/modules/mesh-delivery-indexer/internal/index/registrations_tools_test.go +++ b/modules/mesh-delivery-indexer/internal/index/registrations_tools_test.go @@ -4,12 +4,28 @@ import ( "context" "encoding/json" "errors" + "fmt" + "slices" + "strings" "testing" "time" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + stdio "git.novox.be/novox/mesh-sdk/go" ) +// pagedStore is a store whose Page answers the pages given, by cursor, and records the cursors asked. +type pagedStore struct { + Store + pages map[string]stdio.StatePage + asked []string +} + +func (p *pagedStore) Page(cursor string, _ int, _ time.Duration) (stdio.StatePage, error) { + p.asked = append(p.asked, cursor) + return p.pages[cursor], nil +} + // Design 47 G A-370: the verb registrations answers from the state at once; it never asks the indexer (whose take-in may be // running), but for a module's history, which the indexer reads from its stream. func TestTheRegistrationsVerbIsAnsweredFromItsStateNotAskedOfTheIndexer(t *testing.T) { @@ -41,4 +57,72 @@ func TestTheRegistrationsVerbIsAnsweredFromItsStateNotAskedOfTheIndexer(t *testi if _, err := tool(map[string]any{"module": "a", "history": true}); err == nil || asked != 1 { t.Errorf("a history is asked of the indexer: err %v, asked %d", err, asked) } + + // Design 47 H2 §806: indexer_registrations answers as the verb, filtered by module and repository while it reads, at most + // 768 KB a page, capped with the cursor next; a state that pages is read from the bus's cursor. + t.Run("filtered and paged", func(t *testing.T) { + ctx := context.Background() + big := MemoryStore() + var want []string + for i := range 12 { + origin := "novox/mesh-catalog" + if i%3 == 0 { + origin = "novox/mesh-sdk" + } + module := fmt.Sprintf("m%02d", i) + data, _ := json.Marshal(meshcatalog.Registration{Module: module, Origin: origin, Path: strings.Repeat("p", 100<<10)}) + if err := big.Create(ctx, RegistrationKey(module), data); err != nil { + t.Fatal(err) + } + if origin == "novox/mesh-catalog" { + want = append(want, module) + } + } + named := func(tools []stdio.Tool) func(map[string]any) (any, error) { + for _, tl := range tools { + if tl.Name == RegistrationsTool { + return tl.Run + } + } + t.Fatalf("no tool %s", RegistrationsTool) + return nil + } + run := named(RegistrationTools(big, MemoryStore(), ask, time.Now, nil)) + var got []string + pages := 0 + for next := ""; ; pages++ { + raw, err := run(map[string]any{"repository": "novox/mesh-catalog", "next": next}) + a, _ := raw.(meshcatalog.RegistrationsAnswer) + if err != nil || a.Capped != (a.Next != "") { + t.Fatalf("page %d = %+v (%v), want capped exactly when a cursor follows", pages, a, err) + } + for _, r := range a.Registrations { + got = append(got, r.Module) + } + if next = a.Next; next == "" { + break + } + } + if !slices.Equal(got, want) || pages == 0 { + t.Errorf("the pages hold %v in %d pages after the first, want %v over more than one page", got, pages, want) + } + if a, _ := run(map[string]any{"module": "m00", "repository": "novox/mesh-catalog"}); len(a.(meshcatalog.RegistrationsAnswer).Registrations) != 0 { + t.Errorf("m00 of novox/mesh-sdk answered for the repository novox/mesh-catalog: %+v", a) + } + reg := func(module, origin string) stdio.StateEntry { + data, _ := json.Marshal(meshcatalog.Registration{Module: module, Origin: origin}) + return stdio.StateEntry{Key: RegistrationKey(module), Value: data} + } + pager := &pagedStore{Store: MemoryStore(), pages: map[string]stdio.StatePage{ + "": {Entries: []stdio.StateEntry{reg("a", "novox/x"), reg("b", "novox/y")}, Next: "c1"}, + "c1": {Entries: []stdio.StateEntry{reg("c", "novox/x")}}}} + run = named(RegistrationTools(pager, MemoryStore(), ask, time.Now, nil)) + first, _ := run(map[string]any{"repository": "novox/x"}) + second, _ := run(map[string]any{"repository": "novox/x", "next": "c1"}) + f, g := first.(meshcatalog.RegistrationsAnswer), second.(meshcatalog.RegistrationsAnswer) + if len(f.Registrations) != 1 || f.Registrations[0].Module != "a" || f.Next != "c1" || !f.Capped || + len(g.Registrations) != 1 || g.Registrations[0].Module != "c" || g.Next != "" || !slices.Equal(pager.asked, []string{"", "c1"}) { + t.Errorf("the paged read answered %+v then %+v, asking %v, want a then c of novox/x by the bus's cursor", f, g, pager.asked) + } + }) } diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index 99f0974c0..ac6d421df 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -209,8 +209,8 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build index.CatalogRevisionTool, index.CatalogRevisionsTool, index.CandidateGraphsTool, index.RecomputeTool, index.DropCandidateGraphTool, index.EntryIndexTool, index.EntryPinsTool, index.NamedDropTool, // design 47 G §675, H-A-204 index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.PinDropTool, index.PendingTool, index.TakeInPendingTool, index.TakeInAgainTool, index.TakeInSkipTool, index.RunsTool, index.RunDropTool, // design 47 G §667, §8, §579, §64 (A-194) - index.DryRunsTool) // design 47 H2 §804 - if !reflect.DeepEqual(m.Tools, want) || len(want) != 45 || !slices.Contains(m.Tools, "modules") { + index.DryRunsTool, index.RegistrationsTool) // design 47 H2 §804, §806 + if !reflect.DeepEqual(m.Tools, want) || len(want) != 46 || !slices.Contains(m.Tools, "modules") { t.Errorf("tools = %v, want %v", m.Tools, want) } // Design 47 G §2, §9: every version of every registration is kept with no age limit, 64 MiB, on the old streams section. diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index 8926fa796..b6a7943b6 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -71,7 +71,8 @@ "indexer_take_in_skip", "indexer_runs", "indexer_run_drop", - "indexer_dry_runs" + "indexer_dry_runs", + "indexer_registrations" ], "consumes": [ "go-package-registry.published", From 427885013ec6a16017b738b40c21ee7c99af6720 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 01:12:57 +0100 Subject: [PATCH 06/15] R8: a pipeline is read in parts: show filters while reading and pages at 768 KB; runner_show, runner_movers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 H2 §789, §779 (the cursor), §811 --- .../cmd/mesh-delivery-pipeline-tools/main.go | 1 + .../mesh-delivery-pipeline-tools/main_test.go | 6 +- .../cmd/mesh-delivery-pipeline-tools/show.go | 47 ++++++ .../internal/pipeline/read.go | 136 ++++++++++++++++-- .../internal/pipeline/read_test.go | 67 +++++++++ .../mesh-delivery-pipeline/manifest_test.go | 4 +- modules/mesh-delivery-pipeline/module.json | 4 +- 7 files changed, 251 insertions(+), 14 deletions(-) create mode 100644 modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/show.go diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go index 2a0aba98c..224947099 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go @@ -168,6 +168,7 @@ func main() { tools := append(Tools(r), FirstViewTools(os.Getenv(stdio.ServedAs), stdio.State(views.StatusState), meshhandactsclient.Recorder{Ask: stdio.Ask}, time.Now)...) tools = append(tools, UnloggedTools(stdio.State(seat.StatePipelines), stdio.Log(seat.StateStepLog), meshhandactsclient.Recorder{Ask: stdio.Ask}, func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-pipeline-tools] "+f+"\n", a...) })...) + tools = append(tools, ShowTools(r)...) tools = append(tools, CancelTool(r.List, stdio.Ask), RerunTool(stdio.Ask)) tools = append(tools, VerbTools(stdio.Ask)...) tools = append(tools, TestRunsTools(stdio.Ask)...) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main_test.go index 14b6c44c0..f3136b8e0 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main_test.go @@ -33,6 +33,10 @@ func TestTools(t *testing.T) { for _, tl := range FirstViewTools("the-runner", stdio.State(views.StatusState), nil, nil) { names = append(names, tl.Name) } + // design 47 H2 §789: a pipeline read in parts. + for _, tl := range ShowTools(pipeline.Reads{Pipelines: none{}, Kept: none{}}) { + names = append(names, tl.Name) + } names = append(names, CancelTool(nil, nil).Name, RerunTool(nil).Name) for _, tl := range VerbTools(nil) { names = append(names, tl.Name) @@ -49,7 +53,7 @@ func TestTools(t *testing.T) { for _, tl := range tools { names = append(names, tl.Name) } - want := "mesh-pipeline-runner.show mesh-pipeline-runner.list mesh-pipeline-runner.stands mesh-pipeline-runner.why-carried mesh-pipeline-runner.kept runner_first_views runner_first_view_again runner_cancel runner_rerun runner_start runner_run runner_accept runner_place runner_end_again runner_unfinished_place runner_ask_again runner_health runner_waiting runner_unlogged runner_unlogged_append runner_unfinished runner_unfinished_drop" + want := "mesh-pipeline-runner.show mesh-pipeline-runner.list mesh-pipeline-runner.stands mesh-pipeline-runner.why-carried mesh-pipeline-runner.kept runner_first_views runner_first_view_again runner_show runner_movers runner_cancel runner_rerun runner_start runner_run runner_accept runner_place runner_end_again runner_unfinished_place runner_ask_again runner_health runner_waiting runner_unlogged runner_unlogged_append runner_unfinished runner_unfinished_drop" if got := strings.Join(names, " "); got != want { t.Errorf("tools %q, want %q", got, want) } diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/show.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/show.go new file mode 100644 index 000000000..07a332f1a --- /dev/null +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/show.go @@ -0,0 +1,47 @@ +package main + +import ( + "fmt" + "strings" + + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + stdio "git.novox.be/novox/mesh-sdk/go" + + "mesh-delivery-pipeline/internal/pipeline" +) + +// ShowTools are runner_show and runner_movers, a pipeline read in parts (design 47 H2 §789): show's filters applied while +// reading, at most 768 KB with capped, total and the next page's cursor; and one word per step of each mover. +func ShowTools(r pipeline.Reads) []stdio.Tool { + return []stdio.Tool{ + {Name: Prefix + "_show", + Description: "One pipeline in parts: its record with the steps the filters hold (a mover, a step kind, a step's " + + "outcome), at most 768 KB with capped, the filters' total and next, the cursor of the next page.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "pipeline": map[string]any{"type": "string", "description": "the pipeline's key"}, + "mover": map[string]any{"type": "string", "description": "only this mover's steps"}, + "step": map[string]any{"type": "string", "description": "only the steps of this kind (build, lint, unit, integration, validate, composition, review, approval)"}, + "state": map[string]any{"type": "string", "description": "only the steps whose outcome is this (passed, failed, running, could not run, ...)"}, + "next": map[string]any{"type": "string", "description": "the cursor the previous page answered"}, + }, "required": []string{"pipeline"}}, + Run: func(a map[string]any) (any, error) { + in, err := as[seat.ShowInput](a) + if err != nil { + return nil, err + } + return r.Show(in) + }}, + {Name: Prefix + "_movers", + Description: "Each mover of one pipeline with one word per step: passed, failed, running, waiting or not-run.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "pipeline": map[string]any{"type": "string", "description": "the pipeline's key"}, + }, "required": []string{"pipeline"}}, + Run: func(a map[string]any) (any, error) { + key, _ := a["pipeline"].(string) + if strings.TrimSpace(key) == "" { + return nil, fmt.Errorf("name the pipeline's key") + } + return r.Movers(key) + }}, + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/read.go b/modules/mesh-delivery-pipeline/internal/pipeline/read.go index fe5542471..a69a24595 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/read.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/read.go @@ -3,9 +3,11 @@ package pipeline import ( + "encoding/base64" "encoding/json" "fmt" "sort" + "strconv" "strings" "time" @@ -39,8 +41,12 @@ func (r Reads) fills() ([]seat.Fill, error) { return f, nil } -// Show is one pipeline whole, with its steps and the names of their outputs in step-outputs (§83), or, naming a mover, the -// pipeline's record with that mover's steps alone (H-A-187). +// MaxAnswer is the most a read answers (design 47 H2 §789, §811): 768 KB, the rest read from its cursor. +const MaxAnswer = 768 << 10 + +// Show is one pipeline, with its steps and the names of their outputs in step-outputs (§83), or, naming a mover, the +// pipeline's record with that mover's steps alone (H-A-187). Its filters mover, step and state apply while it reads, and it +// answers at most MaxAnswer with capped, the filters' total and the cursor of the next page (design 47 H2 §789, §779). func (r Reads) Show(in seat.ShowInput) (seat.ShowAnswer, error) { p, ok, err := r.pipeline(in.Pipeline) if err != nil { @@ -53,22 +59,130 @@ func (r Reads) Show(in seat.ShowInput) (seat.ShowAnswer, error) { if err != nil { return seat.ShowAnswer{}, err } - // Named, a mover is answered by its steps alone (H-A-187): one mover's of a pipeline of a hundred movers. - if m := strings.TrimSpace(in.Mover); m != "" { - if !Moves(p, m) { - return seat.ShowAnswer{}, fmt.Errorf("the pipeline %s does not move %s", in.Pipeline, m) - } - p.Steps = stepsOf(p, m) + from, err := pageFrom(in.Next) + if err != nil { + return seat.ShowAnswer{}, err } - a := seat.ShowAnswer{Pipeline: p, Fill: fills} + // Named, a mover is answered by its steps alone (H-A-187): one mover's of a pipeline of a hundred movers. + m := strings.TrimSpace(in.Mover) + if m != "" && !Moves(p, m) { + return seat.ShowAnswer{}, fmt.Errorf("the pipeline %s does not move %s", in.Pipeline, m) + } + held := []seat.StepResult{} for _, s := range p.Steps { - if s.Output != "" { - a.Outputs = append(a.Outputs, s.Output) + if (m == "" || s.Module == m) && (in.Step == "" || s.Step == in.Step) && (in.State == "" || s.Outcome == in.State) { + held = append(held, s) + } + } + p.Steps = []seat.StepResult{} + a := seat.ShowAnswer{Pipeline: p, Fill: fills, Total: int64(len(held))} + size := sizeOf(a) + for i := from; i < len(held); i++ { + n := sizeOf(held[i]) + 1 + if s := held[i]; s.Output != "" { + n += len(s.Output) + 3 + } + if size+n > MaxAnswer && len(a.Pipeline.Steps) > 0 { + a.Capped, a.Next = true, pageCursor(i) + break + } + size += n + a.Pipeline.Steps = append(a.Pipeline.Steps, held[i]) + if held[i].Output != "" { + a.Outputs = append(a.Outputs, held[i].Output) } } return a, nil } +// pageCursor is the opaque cursor of a page starting at the i-th step the filters hold (§779): the caller hands it back. +func pageCursor(i int) string { + return base64.RawURLEncoding.EncodeToString([]byte("step:" + strconv.Itoa(i))) +} + +// pageFrom is the step a cursor starts at, 0 for none; a cursor this reader did not give is refused. +func pageFrom(next string) (int, error) { + if next = strings.TrimSpace(next); next == "" { + return 0, nil + } + raw, err := base64.RawURLEncoding.DecodeString(next) + if err == nil { + if n, ok := strings.CutPrefix(string(raw), "step:"); ok { + if i, err := strconv.Atoi(n); err == nil && i >= 0 { + return i, nil + } + } + } + return 0, fmt.Errorf("the cursor %q is not one show answered", next) +} + +// sizeOf is v's size as JSON. +func sizeOf(v any) int { + b, _ := json.Marshal(v) + return len(b) +} + +// Movers is one word per step of each mover of a pipeline (design 47 H2 §789), in the record's movers' order: passed, +// failed, running, waiting or not-run. +func (r Reads) Movers(pipeline string) (MoversAnswer, error) { + p, ok, err := r.pipeline(pipeline) + if err != nil { + return MoversAnswer{}, err + } + if !ok { + return MoversAnswer{}, fmt.Errorf("no pipeline %q in %s", pipeline, seat.StatePipelines) + } + a := MoversAnswer{Pipeline: p.Pipeline, Outcome: p.Outcome, Movers: make([]MoverWords, 0, len(p.Movers))} + at := map[string]int{} + for _, m := range p.Movers { + at[m.Module] = len(a.Movers) + a.Movers = append(a.Movers, MoverWords{Module: m.Module, Steps: map[string]string{}}) + } + for _, s := range p.Steps { + i, ok := at[s.Module] + if !ok { + continue + } + name := s.Step + if s.Artifact != "" && s.Step != seat.StepResultStepValidate && s.Step != seat.StepResultStepComposition { + name += " " + s.Artifact + } + a.Movers[i].Steps[name] = StepWord(s.Outcome) + } + return a, nil +} + +// MoversAnswer is runner_movers' answer: a pipeline's movers, each with one word per step (design 47 H2 §789). +type MoversAnswer struct { + Pipeline string `json:"pipeline"` + Outcome string `json:"outcome"` + Movers []MoverWords `json:"movers"` +} + +// MoverWords is one mover's steps, each by its kind (and artifact where it has one) with one word. +type MoverWords struct { + Module string `json:"module"` + Steps map[string]string `json:"steps"` +} + +// StepWord is a step outcome's one word (design 47 H2 §789): passed for one that passed, was carried, accepted, bypassed +// or had nothing to run; failed for one that failed, could not run or did not build; running; waiting for one waiting, on +// a build node too; not-run for one not asked, declined, superseded or cancelled. +func StepWord(outcome string) string { + switch outcome { + case seat.StepResultOutcomePassed, seat.StepResultOutcomeCarried, seat.StepResultOutcomeNothingToRun, + seat.StepResultOutcomeAccepted, seat.StepResultOutcomeBypassed: + return "passed" + case seat.StepResultOutcomeFailed, seat.StepResultOutcomeCouldNotRun, seat.StepResultOutcomeDidNotBuild: + return "failed" + case seat.StepResultOutcomeRunning: + return "running" + case seat.StepResultOutcomeWaiting, seat.StepResultOutcomeWaitingForABuildNode: + return "waiting" + } + return "not-run" +} + // List is the pipelines and runs of a module, a branch, a commit or a run, newest started first, at most Limit when it // names one (§83): each one's record with its movers and steps counted, its steps empty (H-A-191). func (r Reads) List(in seat.ListInput) (seat.ListAnswer, error) { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/read_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/read_test.go index 3b1ee9d71..3821a0101 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/read_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/read_test.go @@ -247,3 +247,70 @@ func TestStandsAndStepStartedNameWhatAReadingStepWaitsOn(t *testing.T) { t.Error("resume said no step-started") } } + +// TestAPipelineIsReadInParts holds show and runner_movers to design 47 H2 §789: one mover's steps of a record of 129 movers +// answer under 64 KB; the filters apply while reading; a capped answer continues from its cursor; runner_movers of 1,000 +// movers answers under 768 KB with one word per step. +func TestAPipelineIsReadInParts(t *testing.T) { + t0 := time.Date(2026, 11, 22, 1, 0, 0, 0, time.UTC) + record := func(n int) memKept { + pipes := memKept{} + p := seat.Pipeline{Pipeline: "main.9", Branch: "main", Source: 9, Outcome: seat.PipelineOutcomeRunning, Started: &t0, + Steps: []seat.StepResult{}} + for i := 0; i < n; i++ { + m := fmt.Sprintf("module-%04d", i) + p.Movers = append(p.Movers, seat.Mover{Module: m, Commit: "c9", Fingerprint: "src2:" + strings.Repeat("a", 64)}) + var steps []seat.StepResult + for _, k := range []string{seat.StepResultStepBuild, seat.StepResultStepLint, seat.StepResultStepUnit, + seat.StepResultStepIntegration, seat.StepResultStepValidate, seat.StepResultStepComposition} { + o := seat.StepResultOutcomePassed + if k == seat.StepResultStepUnit && i%10 == 3 { + o = seat.StepResultOutcomeFailed + } + steps = append(steps, seat.StepResult{Module: m, Step: k, Artifact: m + ":program", Status: "required", Outcome: o, + Ask: "ask-" + m + "-" + k, Started: &t0, Ended: &t0, Why: strings.Repeat("w", 200)}) + } + put(t, pipes, "steps.main.9."+m, seat.PipelinesByStepsPipelineMoverValue{Steps: steps}) + } + put(t, pipes, "main.9", p) + return pipes + } + r := Reads{Pipelines: record(129), Kept: memKept{}} + a, err := r.Show(seat.ShowInput{Pipeline: "main.9", Mover: "module-0042"}) + if raw, _ := json.Marshal(a); err != nil || len(raw) >= 64<<10 || len(a.Pipeline.Steps) != 6 || a.Total != 6 || a.Capped { + t.Fatalf("one mover of 129: %d bytes, %d steps, total %d, capped %v, %v", len(raw), len(a.Pipeline.Steps), a.Total, a.Capped, err) + } + a, err = r.Show(seat.ShowInput{Pipeline: "main.9", Step: seat.StepResultStepUnit, State: seat.StepResultOutcomeFailed}) + if err != nil || a.Total != 13 || len(a.Pipeline.Steps) != 13 || a.Pipeline.Steps[0].Module != "module-0003" { + t.Fatalf("the failed unit steps: total %d, %d steps, %v", a.Total, len(a.Pipeline.Steps), err) + } + // The whole record is larger than one answer: it is read in pages, each under 768 KB, every step once. + big := Reads{Pipelines: record(1000), Kept: memKept{}} + seen, pages, next := 0, 0, "" + for { + a, err := big.Show(seat.ShowInput{Pipeline: "main.9", Next: next}) + raw, _ := json.Marshal(a) + if err != nil || len(raw) > MaxAnswer || a.Total != 6000 { + t.Fatalf("page %d: %d bytes, total %d, %v", pages, len(raw), a.Total, err) + } + seen, pages = seen+len(a.Pipeline.Steps), pages+1 + if !a.Capped { + break + } + next = a.Next + } + if seen != 6000 || pages < 2 { + t.Fatalf("pages %d read %d steps, want every one of 6000 over more than one page", pages, seen) + } + if _, err := big.Show(seat.ShowInput{Pipeline: "main.9", Next: "not-a-cursor"}); err == nil { + t.Fatal("a cursor show never answered is refused") + } + m, err := big.Movers("main.9") + raw, _ := json.Marshal(m) + if err != nil || len(raw) >= MaxAnswer || len(m.Movers) != 1000 { + t.Fatalf("runner_movers of 1,000: %d bytes, %d movers, %v", len(raw), len(m.Movers), err) + } + if got := m.Movers[3].Steps["unit module-0003:program"] + " " + m.Movers[4].Steps["validate"]; got != "failed passed" { + t.Fatalf("words %q", got) + } +} diff --git a/modules/mesh-delivery-pipeline/manifest_test.go b/modules/mesh-delivery-pipeline/manifest_test.go index 49cc90e35..de97d07d5 100644 --- a/modules/mesh-delivery-pipeline/manifest_test.go +++ b/modules/mesh-delivery-pipeline/manifest_test.go @@ -119,7 +119,9 @@ func TestManifest(t *testing.T) { // RN25 (§539): the runner's tools carry the prefix runner_. {"first-view tools", equalSets(m.Tools, []string{"runner_first_views", "runner_first_view_again", "runner_health", "runner_unlogged", "runner_unlogged_append", "runner_start", "runner_run", "runner_accept", "runner_ask_again", "runner_place", "runner_end_again", "runner_waiting", - "runner_test_runs", "runner_test_runs_drop", "runner_unfinished", "runner_unfinished_drop", "runner_unfinished_place"})}, + "runner_test_runs", "runner_test_runs_drop", "runner_unfinished", "runner_unfinished_drop", "runner_unfinished_place", + // design 47 H2 §789: a pipeline read in parts. + "runner_show", "runner_movers"})}, // A-305 (ADR 0227 rule 8, ADR 0240 rule 2): the process is judged by its unit, as every process of mesh-delivery, // never by a tool of its own; runner_health stays a tool. {"health is the unit", healthTool(raw) == "unit"}, diff --git a/modules/mesh-delivery-pipeline/module.json b/modules/mesh-delivery-pipeline/module.json index fabd0a8b3..b5b28ce67 100644 --- a/modules/mesh-delivery-pipeline/module.json +++ b/modules/mesh-delivery-pipeline/module.json @@ -44,7 +44,9 @@ "runner_test_runs_drop", "runner_unfinished", "runner_unfinished_drop", - "runner_unfinished_place" + "runner_unfinished_place", + "runner_show", + "runner_movers" ], "consumes": [ "mesh-delivery-indexer.take-in.taken", From 8d43ce0cb91f7aa1f0a1c44cfb4163187fdbb207 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 01:16:51 +0100 Subject: [PATCH 07/15] R6: a kept mover waiting on its kept revision says so; the record's kept-revisions are read past, not dropped MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The runner seat's Mover gains unplaced (kept-revision-unread) and revision, the seat's form 3. Builds: design 47 H2 §787, §780 (fate: the drop of kept-revisions before the decode goes) --- .../go/seats/meshpipelinerunner/contract.go | 21 ++++-- .../seats/mesh-pipeline-runner/seat.json | 13 +++- .../ts/seats/meshpipelinerunner/index.d.ts | 4 + .../internal/pipeline/moverkeys.go | 16 +--- .../internal/pipeline/place.go | 15 ++-- .../internal/pipeline/read.go | 4 - .../internal/pipeline/repin_test.go | 73 +++++++++++++++++++ .../internal/pipeline/start.go | 41 ++++++++++- .../internal/pipeline/verbs.go | 5 +- 9 files changed, 153 insertions(+), 39 deletions(-) diff --git a/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract.go b/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract.go index f024bb547..07592ec7e 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract.go @@ -14,13 +14,13 @@ const Seat = "mesh-pipeline-runner" // Form is the form of the seat's values, events and answers this contract was generated at (design 47 H2 §780): // a reader of a value of another form says "form N, this reader knows M". -const Form = 2 +const Form = 3 // The forms and the digest of the fields this contract was generated from: the generator's compare fails a definition // whose fields changed while its forms did not (design 47 H2 §780). const ( - generatedForms = "2" - generatedFieldsDigest = "AcceptAnswer=224df0394653 Accepted=d2b011e5090f AskAgainAnswer=224df0394653 CancelAnswer=a0dcc32b3f57 EndAgainAnswer=a0dcc32b3f57 Fill=7a76c9ceb157 FillingEvent=5bd4df78ed84 FullEvent=5bd4df78ed84 KeptAnswer=dfe0d48bc169 KeptByLevelDigestValue=f06c90f7da7f KeptResult=af5f81f189b7 ListAnswer=75717fd670a7 LogEntry=b5c78894cc5b Member=8a103baec01f Mover=5659e54b6dd0 MoverEnded=4b35095f6829 Pipeline=2663792125bf PipelineEnded=523f8fc0fe41 PipelineEndedEvent=cb44007f7173 PipelineEndedPhases=9d1c33ce4cc4 PipelineEndedPhasesBuild=a08d49692a09 PipelineEndedPhasesComposition=a08d49692a09 PipelineEndedPhasesIntegration=a08d49692a09 PipelineEndedPhasesLint=a08d49692a09 PipelineEndedPhasesUnit=a08d49692a09 PipelineEndedPhasesValidate=a08d49692a09 PipelineStarted=a3d000f095db PipelineStartedEvent=eaa401af4117 PipelinesByBranchRevisionUnfinishedModuleValue=693580eed498 PipelinesByBranchRevisionValue=693580eed498 PipelinesByBranchRunRunValue=693580eed498 PipelinesByBranchSourcePublishedPackageVersionValue=693580eed498 PipelinesByStepsPipelineMoverValue=94de99db7816 PlaceAnswer=a0dcc32b3f57 PlaceUnfinishedAnswer=ed94ced4513a Published=d9af3024dacc QueueByQueueNextValue=765a2918a728 QueueByRunningPipelineValue=ed94ced4513a QueueByWaitingSequenceValue=99605298ae13 RePinnedEvent=92fd04bfa6d3 RerunAnswer=9b7efd0df344 ReviewAnswer=f397b8965179 RoomEvent=5bd4df78ed84 RunAnswer=96bd45aa2e62 RunStarted=b280617410c3 RunStartedEvent=0767a9358fa3 Shared=c981cf6aa860 SharedAskedBy=c9b30cc4d0d0 ShowAnswer=5beff763f531 StandsAnswer=0afa3693bf3b StartAnswer=ed94ced4513a StepEndedEvent=799020642c19 StepEvent=64f18c40ed5d StepLogByPipelineValue=ffdde090a101 StepOutputsByPipelineStepArtifactValue=a2f06147f4b6 StepResult=94c2bea6ddcd StepStartedEvent=799020642c19 TestCounts=9c1332427159 TestRun=ca5a87f3b6b8 TestRunPackage=a7d30558470d TestRunReader=c9b30cc4d0d0 TestRunsAnswer=7ab81a45ca33 TestRunsDropAnswer=8ee70b35469d ToolchainUsed=43b6d42376fb Wait=21e4ee7a3ef1 WaitEndedEvent=2bf6d38041c0 WaitHeldEvent=2bf6d38041c0 Waiting=ed94ced4513a WhyCarriedAnswer=8e2fcea6479f Widening=e2790a307466" + generatedForms = "3" + generatedFieldsDigest = "AcceptAnswer=224df0394653 Accepted=d2b011e5090f AskAgainAnswer=224df0394653 CancelAnswer=a0dcc32b3f57 EndAgainAnswer=a0dcc32b3f57 Fill=7a76c9ceb157 FillingEvent=5bd4df78ed84 FullEvent=5bd4df78ed84 KeptAnswer=dfe0d48bc169 KeptByLevelDigestValue=f06c90f7da7f KeptResult=af5f81f189b7 ListAnswer=75717fd670a7 LogEntry=b5c78894cc5b Member=8a103baec01f Mover=5db376275bcf MoverEnded=4b35095f6829 Pipeline=2663792125bf PipelineEnded=523f8fc0fe41 PipelineEndedEvent=cb44007f7173 PipelineEndedPhases=9d1c33ce4cc4 PipelineEndedPhasesBuild=a08d49692a09 PipelineEndedPhasesComposition=a08d49692a09 PipelineEndedPhasesIntegration=a08d49692a09 PipelineEndedPhasesLint=a08d49692a09 PipelineEndedPhasesUnit=a08d49692a09 PipelineEndedPhasesValidate=a08d49692a09 PipelineStarted=a3d000f095db PipelineStartedEvent=eaa401af4117 PipelinesByBranchRevisionUnfinishedModuleValue=693580eed498 PipelinesByBranchRevisionValue=693580eed498 PipelinesByBranchRunRunValue=693580eed498 PipelinesByBranchSourcePublishedPackageVersionValue=693580eed498 PipelinesByStepsPipelineMoverValue=94de99db7816 PlaceAnswer=a0dcc32b3f57 PlaceUnfinishedAnswer=ed94ced4513a Published=d9af3024dacc QueueByQueueNextValue=765a2918a728 QueueByRunningPipelineValue=ed94ced4513a QueueByWaitingSequenceValue=99605298ae13 RePinnedEvent=92fd04bfa6d3 RerunAnswer=9b7efd0df344 ReviewAnswer=f397b8965179 RoomEvent=5bd4df78ed84 RunAnswer=96bd45aa2e62 RunStarted=b280617410c3 RunStartedEvent=0767a9358fa3 Shared=c981cf6aa860 SharedAskedBy=c9b30cc4d0d0 ShowAnswer=5beff763f531 StandsAnswer=0afa3693bf3b StartAnswer=ed94ced4513a StepEndedEvent=799020642c19 StepEvent=64f18c40ed5d StepLogByPipelineValue=ffdde090a101 StepOutputsByPipelineStepArtifactValue=a2f06147f4b6 StepResult=94c2bea6ddcd StepStartedEvent=799020642c19 TestCounts=9c1332427159 TestRun=ca5a87f3b6b8 TestRunPackage=a7d30558470d TestRunReader=c9b30cc4d0d0 TestRunsAnswer=7ab81a45ca33 TestRunsDropAnswer=8ee70b35469d ToolchainUsed=43b6d42376fb Wait=21e4ee7a3ef1 WaitEndedEvent=2bf6d38041c0 WaitHeldEvent=2bf6d38041c0 Waiting=ed94ced4513a WhyCarriedAnswer=8e2fcea6479f Widening=e2790a307466" ) const ( @@ -637,16 +637,25 @@ type Mover struct { // yes when the module is over its test budget (design 47 G section 419) OverBudget bool `json:"over-budget,omitempty"` // the artifact references, :, of the mover's dependencies whose package-published (and, for a kind-3 edge, whose build step's passed end) was heard at the pipeline's branch and source, written with the pipeline by compare-and-set; absent when none (H-A-84) - Published []string `json:"published,omitempty"` - Repository string `json:"repository"` - Tests *TestCounts `json:"tests,omitempty"` + Published []string `json:"published,omitempty"` + Repository string `json:"repository"` + // the catalog revision a kept-revision-unread mover waits for (design 47 H2 §787) + Revision int64 `json:"revision,omitempty"` + Tests *TestCounts `json:"tests,omitempty"` // the mover's tier in the pipeline's delivery order, from 1 (ADR 0666 section 14; design 47 H §746) Tier int64 `json:"tier"` + // why the mover has no step placed yet, where the runner knows it: kept-revision-unread for a kept mover whose kept revision the runner's view does not hold yet (design 47 H2 §787); absent once placed + Unplaced string `json:"unplaced,omitempty"` // the version released for the mover in this pipeline (version-released), absent until one is (design 47 H-A-52) Version string `json:"version,omitempty"` Widens []Widening `json:"widens,omitempty"` } +// The values of Mover's unplaced. +const ( + MoverUnplacedKeptRevisionUnread = "kept-revision-unread" +) + // MoverEnded is one mover at its pipeline's end (design 47 H §746). type MoverEnded struct { // the commit the mover's entry was last changed at, from the pipeline's value (design 47 H-A-119) diff --git a/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json b/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json index bb11edc77..3801baadc 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json +++ b/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json @@ -2149,6 +2149,17 @@ "tests": { "$ref": "#/defs/TestCounts" }, + "unplaced": { + "type": "string", + "enum": [ + "kept-revision-unread" + ], + "description": "why the mover has no step placed yet, where the runner knows it: kept-revision-unread for a kept mover whose kept revision the runner's view does not hold yet (design 47 H2 §787); absent once placed" + }, + "revision": { + "type": "integer", + "description": "the catalog revision a kept-revision-unread mover waits for (design 47 H2 §787)" + }, "over-budget": { "type": "boolean", "description": "yes when the module is over its test budget (design 47 G section 419)" @@ -2222,5 +2233,5 @@ ] } }, - "form": 2 + "form": 3 } diff --git a/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.d.ts b/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.d.ts index c051eb0d0..04c0ca54c 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.d.ts @@ -238,9 +238,13 @@ export interface Mover { /** the artifact references, :, of the mover's dependencies whose package-published (and, for a kind-3 edge, whose build step's passed end) was heard at the pipeline's branch and source, written with the pipeline by compare-and-set; absent when none (H-A-84) */ "published"?: string[]; "repository": string; + /** the catalog revision a kept-revision-unread mover waits for (design 47 H2 §787) */ + "revision"?: number; "tests"?: TestCounts; /** the mover's tier in the pipeline's delivery order, from 1 (ADR 0666 section 14; design 47 H §746) */ "tier": number; + /** why the mover has no step placed yet, where the runner knows it: kept-revision-unread for a kept mover whose kept revision the runner's view does not hold yet (design 47 H2 §787); absent once placed */ + "unplaced"?: "kept-revision-unread"; /** the version released for the mover in this pipeline (version-released), absent until one is (design 47 H-A-52) */ "version"?: string; "widens"?: Widening[]; diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/moverkeys.go b/modules/mesh-delivery-pipeline/internal/pipeline/moverkeys.go index a10595152..d63c79bd4 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/moverkeys.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/moverkeys.go @@ -315,7 +315,7 @@ func (s *MoverStore) KeptRevisions(key string) (map[string]int64, error) { } // keptRevisionsField is the pipeline record's property kept-revisions, beside its movers (H-A-285): written and read by the -// runner alone, never part of the seat's Go form of a pipeline, which drops it on every decode. +// runner alone, never part of the seat's Go form of a pipeline, whose tolerant decode reads past it (design 47 H2 §780). const keptRevisionsField = "kept-revisions" // keptOf is the kept-revisions a record carries; none when it carries none. @@ -335,20 +335,6 @@ func keptOf(raw []byte) (map[string]int64, error) { return kept, nil } -// withoutKept is record without its kept-revisions, the runner's own property and never part of the seat's Go form, so the -// seat's strict decode of the record the runner's reads make refuses no record carrying it (H-A-285, H-A-309). -func withoutKept(record []byte) ([]byte, error) { - var fields map[string]json.RawMessage - if err := json.Unmarshal(record, &fields); err != nil { - return nil, fmt.Errorf("the record is no object: %w", err) - } - if _, ok := fields[keptRevisionsField]; !ok { - return record, nil - } - delete(fields, keptRevisionsField) - return json.Marshal(fields) -} - // withKept is record with kept as its kept-revisions. func withKept(record []byte, kept map[string]int64) ([]byte, error) { var fields map[string]json.RawMessage diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place.go b/modules/mesh-delivery-pipeline/internal/pipeline/place.go index f34cb9d43..f9be05712 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place.go @@ -408,17 +408,18 @@ func keptOfStore(store any, key string) (map[string]int64, error) { // keptEntries are the entries of branch's catalog revision rev, the one a same-source re-pin kept a mover at, read by its // number (H-A-286): a kept mover is placed and pin-checked from them, never from the branch's held view; false while they // cannot be read, and the mover waits. -func (s *Starter) keptEntries(branch string, rev int64) (map[string]catalog.CatalogEntry, bool) { +func (s *Starter) keptEntries(branch string, rev int64) (map[string]catalog.CatalogEntry, error) { if s.Revision == nil { - s.logf("the runner reads no catalog revision, so a mover kept at revision %d of %s waits", rev, branch) - return nil, false + return nil, fmt.Errorf("the runner reads no catalog revision") } half, err := s.Revision(branch, rev) - if err != nil || half.Revision != rev { - s.logf("revision %d of %s, which a mover is kept at, is not read (%v): the mover waits for it (H-A-286)", rev, branch, err) - return nil, false + if err != nil { + return nil, err } - return half.Entries, true + if half.Revision != rev { + return nil, fmt.Errorf("the view holds revision %d", half.Revision) + } + return half.Entries, nil } func stepAsk(p seat.Pipeline, m seat.Mover, e catalog.CatalogEntry, a catalog.Artifact, language, step, id, asker string, now time.Time) nodebuildgo.StepAsk { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/read.go b/modules/mesh-delivery-pipeline/internal/pipeline/read.go index a69a24595..98378b212 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/read.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/read.go @@ -320,10 +320,6 @@ func (r Reads) pipelineOf(key string, all []string) (seat.Pipeline, bool, error) if raw == nil { return seat.Pipeline{}, false, nil } - raw, err = withoutKept(raw) - if err != nil { - return seat.Pipeline{}, false, fmt.Errorf("%s %s: %w", seat.StatePipelines, key, err) - } p, _, err := seat.DecodePipelinesByBranchRevisionValue(raw) if err != nil { return seat.Pipeline{}, false, fmt.Errorf("%s %s: %w", seat.StatePipelines, key, err) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/repin_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/repin_test.go index 54ba43b1a..3faf3472b 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/repin_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/repin_test.go @@ -1,6 +1,8 @@ package pipeline import ( + "encoding/json" + "errors" "fmt" "slices" "strings" @@ -189,3 +191,74 @@ func TestAKeptMoverIsPlacedLateAndResumedAtItsKeptRevision(t *testing.T) { t.Fatalf("Resume asked %d (%v), ids %v: want the kept build %s again", n, err, asks.ids, build.Ask) } } + +// design 47 H2 §787: a kept mover whose kept revision the runner's view does not hold yet carries unplaced +// kept-revision-unread with that revision, said once in the journal and read with show; once the revision is read it is +// placed and the mark goes. +func TestAKeptMoverWaitingOnItsRevisionSaysSo(t *testing.T) { + reg, entries, _, read := repinFixture() + unread := true + revision := func(branch string, rev int64) (Half, error) { + if unread && rev == 11 { + return Half{}, errors.New("revision 11 is not held yet") + } + return read(branch, rev) + } + store := &MoverStore{Raw: newMemRaw()} + var lines []string + registered := false + s := &Starter{Pipelines: store, Store: store, Steps: &stepAsks{}, Name: "the-runner", Now: ticking(), Revision: revision, + Say: func(string, any) error { return nil }, Logf: func(f string, a ...any) { lines = append(lines, fmt.Sprintf(f, a...)) }, + Registration: func(m string) (catalog.Registration, bool) { return reg, registered && m == "app" }} + key := Key("main", 10) + if ok, err := s.Start(Begin{Branch: "main", Source: 41, Revision: 10, Entries: entries, + Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:aa", Repository: "o/app"}}}); !ok || err != nil { + t.Fatalf("start = %v, %v", ok, err) + } + if n, err := s.Revised(revisionEvent(11), read); n != 0 || err != nil { + t.Fatalf("the re-pin started %d (%v)", n, err) + } + registered = true + lagging := func(string) (Half, bool) { return Half{Revision: 10, Source: 41, Entries: entries}, true } + for range 2 { + if n, err := s.Registered(store, reg, lagging); n != 0 || err != nil { + t.Fatalf("placed in %d pipelines (%v) while the kept revision is unread", n, err) + } + } + said := 0 + for _, l := range lines { + if strings.Contains(l, "app waits for its kept revision 11 to be read") { + said++ + } + } + shown, err := Reads{Pipelines: rawKept{store.Raw.(*memRaw)}, Kept: memKept{}}.Show(seat.ShowInput{Pipeline: key}) + if err != nil || said != 1 || shown.Pipeline.Movers[0].Unplaced != seat.MoverUnplacedKeptRevisionUnread || shown.Pipeline.Movers[0].Revision != 11 { + t.Fatalf("said %d times, show %+v (%v)", said, shown.Pipeline.Movers, err) + } + if why := s.Unplaced(shown.Pipeline, "app", lagging); why != "app waits for its kept revision 11 to be read" { + t.Fatalf("the unplaced line %q", why) + } + unread = false + if n, err := s.Registered(store, reg, lagging); n != 1 || err != nil { + t.Fatalf("once read: placed in %d pipelines (%v)", n, err) + } + if p, _, _ := store.Get(key); p.Movers[0].Unplaced != "" || p.Movers[0].Revision != 0 { + t.Fatalf("the mark stays once placed: %+v", p.Movers[0]) + } +} + +// rawKept reads a memRaw as the reads do. +type rawKept struct{ r *memRaw } + +func (k rawKept) Keys() ([]string, error) { + var out []string + for key := range k.r.vals { + out = append(out, key) + } + return out, nil +} + +func (k rawKept) Value(key string) (json.RawMessage, error) { + v, _, err := k.r.Get(key) + return v, err +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index b326380dc..eaf7850bf 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -427,7 +427,7 @@ func (s *Starter) placeMovers(p *seat.Pipeline, reg func(module string) (catalog s.logf("the kept-revisions of %s are not read, so its movers are placed as movers not kept: %v", p.Pipeline, err) } all := entries - for _, m := range p.Movers { + for i, m := range p.Movers { // A mover that left for a newer pipeline is placed there, never here (H-A-223, H-A-361). if placed(*p, m.Module) || slices.Contains(p.Superseded, m.Module) { continue @@ -440,10 +440,17 @@ func (s *Starter) placeMovers(p *seat.Pipeline, reg func(module string) (catalog // A mover a same-source re-pin kept is placed from the revision it is kept at, its ids its own (H-A-286). entries := all if rev := kept[m.Module]; rev > 0 { - if entries, ok = s.keptEntries(p.Branch, rev); !ok { + var err error + if entries, err = s.keptEntries(p.Branch, rev); err != nil { + // It waits for its kept revision to be read, said once and carried on the mover (design 47 H2 §787). + if m.Unplaced != seat.MoverUnplacedKeptRevisionUnread || m.Revision != rev { + s.logf("%s: %s waits for its kept revision %d to be read (%v)", p.Pipeline, m.Module, rev, err) + p.Movers[i].Unplaced, p.Movers[i].Revision = seat.MoverUnplacedKeptRevisionUnread, rev + } continue } } + p.Movers[i].Unplaced, p.Movers[i].Revision = "", 0 // A mover's steps wait for the publish of each mover it builds on (§746, H-A-83 (2)): placed once each is heard, // ended not asked when one's build ended without passing. due, waiting, failed := ready(*p, m.Module, entries) @@ -728,7 +735,9 @@ func (s *Starter) Resume(store Updater, branch string, half Half) (int, error) { // A kept mover's steps are found by the ids its kept revision gives, from that revision's entries (H-A-286). entries, rev := half.Entries, kept[m.Module] if rev > 0 { - if entries, ok = s.keptEntries(p.Branch, rev); !ok { + var err error + if entries, err = s.keptEntries(p.Branch, rev); err != nil { + s.logf("%s: revision %d of %s, which %s is kept at, is not read (%v): its steps are asked again once it is", key, rev, p.Branch, m.Module, err) continue } } @@ -761,6 +770,10 @@ func (s *Starter) Resume(store Updater, branch string, half Half) (int, error) { // do not name it (A-327), else " waits on the publish of , …", the packages it awaits not yet published // as the catalog half of p's branch holds its edges (§746). func (s *Starter) Unplaced(p seat.Pipeline, module string, view func(branch string) (Half, bool)) string { + // A kept mover waiting for its kept revision says so (design 47 H2 §787). + if m, ok := MoverOf(p, module); ok && m.Unplaced == seat.MoverUnplacedKeptRevisionUnread { + return fmt.Sprintf("%s waits for its kept revision %d to be read", module, m.Revision) + } if s.Registration == nil { return module + " has no registration" } @@ -883,7 +896,7 @@ func (s *Starter) placeOne(store Updater, key string, due func(seat.Pipeline) bo s.logf("the catalog half of %s holds revision %d: %s waits for revision %d, which its take-in made, to place its steps (§200)", p.Branch, half.Revision, key, p.Revision) return p, nil, nil, nil, 0, nil } - before := len(p.Steps) + before, marks := len(p.Steps), unplacedMarks(p) asks, unregistered, ended, notAsked := s.placeMovers(&p, reg, half.Entries, now) // Until its registration arrives, the pipeline says it waits on it (A-327): a mover with no registration has no step, // and it holds the pipeline's end only while every placed mover's required steps passed (H-A-163, H-A-171). @@ -892,6 +905,15 @@ func (s *Starter) placeOne(store Updater, key string, due func(seat.Pipeline) bo s.registrationMissing(m, key) } if len(p.Steps) == before { + // A mover that now waits for its kept revision, or no longer does, is written so (design 47 H2 §787). + if unplacedMarks(p) == marks { + return p, nil, nil, nil, 0, nil + } + if _, err := store.Update(key, p, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + return p, nil, nil, nil, 0, fmt.Errorf("the movers of %s are not written: %w", key, err) + } return p, nil, nil, nil, 0, nil } if _, err := store.Update(key, p, rev); errors.Is(err, ErrChanged) { @@ -904,6 +926,17 @@ func (s *Starter) placeOne(store Updater, key string, due func(seat.Pipeline) bo } } +// unplacedMarks is what p's movers carry of why they have no step placed (design 47 H2 §787), one line. +func unplacedMarks(p seat.Pipeline) string { + var b strings.Builder + for _, m := range p.Movers { + if m.Unplaced != "" { + fmt.Fprintf(&b, "%s %s %d;", m.Module, m.Unplaced, m.Revision) + } + } + return b.String() +} + // recovered says whether the commit of pipeline key is named a recovery; a read that fails asks on the stream check, said, since // a step is asked either way (a recovery's only goes ahead of the first round). func (s *Starter) recovered(key, commit string) bool { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go index 7934aa8f0..47e6ac144 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go @@ -594,8 +594,9 @@ func (v Verbs) placedAsk(p seat.Pipeline, s seat.StepResult, id string, now time // A kept mover's step is the one placed from the revision it is kept at, under its kept ids (H-A-286). entries, rev := half.Entries, v.keptOf(p.Pipeline)[m.Module] if rev > 0 { - if entries, ok = v.Starter.keptEntries(p.Branch, rev); !ok { - return nil, fmt.Errorf("revision %d of %s, which %s is kept at, is not read: the step is asked when it is", rev, p.Branch, m.Module) + var err error + if entries, err = v.Starter.keptEntries(p.Branch, rev); err != nil { + return nil, fmt.Errorf("revision %d of %s, which %s is kept at, is not read (%v): the step is asked when it is", rev, p.Branch, m.Module, err) } } _, placed := place(p, m, reg, entries, now, v.Starter.Name, false, rev) From 43a88ef96c5e295d42ca5f30069f26fdda94d6eb Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 01:19:27 +0100 Subject: [PATCH 08/15] R7: the registration check waits for its view; runner_health answers registrations-read MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 H2 §788 --- .../cmd/mesh-delivery-pipeline-tools/main.go | 21 ++++++++-- .../mesh-delivery-pipeline-tools/main_test.go | 16 +++++++- .../cmd/mesh-delivery-pipeline/main.go | 12 +++++- .../internal/pipeline/consumers.go | 34 ++++++++++++++++ .../internal/pipeline/registration_missing.go | 7 ++++ .../internal/pipeline/start.go | 5 +++ .../internal/pipeline/start_test.go | 39 +++++++++++++++++++ 7 files changed, 127 insertions(+), 7 deletions(-) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go index 224947099..b1a2627e1 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go @@ -13,6 +13,7 @@ import ( "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" "git.novox.be/novox/mesh-sdk/go/buscalls" "git.novox.be/novox/mesh-sdk/go/busstate" + "git.novox.be/novox/mesh-sdk/go/health" "git.novox.be/novox/mesh-sdk/go/views" seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" @@ -102,16 +103,22 @@ func FirstViewTools(module string, st stdio.KeptState, acts views.HandActs, now // runner's state pipelines as show does and the ages its process writes in the state consumers, and asks no seat verb. A // running pipeline it first names stalled is said in this program's journal through logf, once per key for the // program's life (H-A-311 (3), H-A-313). -func HealthTool(pipelines, consumers pipeline.Kept, now func() time.Time, logf func(string, ...any)) stdio.Tool { +func HealthTool(pipelines, consumers, firstViews pipeline.Kept, now func() time.Time, logf func(string, ...any)) stdio.Tool { var mu sync.Mutex said := map[string]bool{} return stdio.Tool{Name: Prefix + "_health", Description: "The pipeline runner's health: healthy when it answers show, each of its durable consumers' oldest " + "unacknowledged message waits less than five minutes and no running pipeline has had nothing open and nothing " + - "newly placed for five minutes; otherwise it names each consumer behind and each such pipeline by its key.", + "newly placed for five minutes; otherwise it names each consumer behind and each such pipeline by its key. It " + + "answers registrations-read: whether the runner's registrations view's first read is in, and since when.", Input: map[string]any{"type": "object", "properties": map[string]any{}}, Run: func(map[string]any) (any, error) { a, stalled := pipeline.HealthStalled(pipelines, consumers, now()) + // design 47 H2 §788: whether the registrations are read, from the runner's own first-views state. + read, err := pipeline.RegistrationsReadOf(firstViews) + if err != nil { + return nil, err + } mu.Lock() defer mu.Unlock() for _, key := range stalled { @@ -120,10 +127,16 @@ func HealthTool(pipelines, consumers pipeline.Kept, now func() time.Time, logf f logf("the running pipeline %s has had nothing open and nothing newly placed for %s or more: runner_health names it unhealthy (H-A-311)", key, pipeline.StalledAfter) } } - return a, nil + return HealthAnswer{Answer: a, RegistrationsRead: read}, nil }} } +// HealthAnswer is runner_health's answer: the health and registrations-read (design 47 H2 §788). +type HealthAnswer struct { + health.Answer + RegistrationsRead pipeline.RegistrationsRead `json:"registrations-read"` +} + // StoresFill is each of the runner's stores' fill through the bundle's calls (design 47 G §366, §623, A-214): the states // pipelines and kept by `mesh/state.info`, step-outputs by the object store's fill and step-log by the log's; the runner's // process writes nothing for it. A store whose fill cannot be read is an error naming it. @@ -172,7 +185,7 @@ func main() { tools = append(tools, CancelTool(r.List, stdio.Ask), RerunTool(stdio.Ask)) tools = append(tools, VerbTools(stdio.Ask)...) tools = append(tools, TestRunsTools(stdio.Ask)...) - tools = append(tools, HealthTool(r.Pipelines, kept{stdio.State(pipeline.StateConsumers)}, time.Now, + tools = append(tools, HealthTool(r.Pipelines, kept{stdio.State(pipeline.StateConsumers)}, kept{stdio.State(views.StatusState)}, time.Now, func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-pipeline-tools] "+f+"\n", a...) })) tools = append(tools, WaitingTool(kept{stdio.State(seat.StateQueue)})) tools = append(tools, UnfinishedTools(keptDelete{kept{stdio.State(pipeline.StateUnfinished)}}, meshhandactsclient.Recorder{Ask: stdio.Ask})...) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main_test.go index f3136b8e0..768f3daa8 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main_test.go @@ -42,7 +42,7 @@ func TestTools(t *testing.T) { names = append(names, tl.Name) } // RN25 (§468, §76; A-164): the health check is the bundle's runner_health. - names = append(names, HealthTool(none{}, none{}, nil, nil).Name, WaitingTool(none{}).Name) + names = append(names, HealthTool(none{}, none{}, none{}, nil, nil).Name, WaitingTool(none{}).Name) // §606, §549: the unlogged entries are listed and appended by hand. for _, tl := range UnloggedTools(stdio.State("pipelines"), stdio.Log("step-log"), nil, nil) { names = append(names, tl.Name) @@ -70,7 +70,8 @@ func TestTools(t *testing.T) { quiet := queueOf{"main.137": `{"pipeline":"main.137","branch":"main","outcome":"running","source":137,"started":"2026-11-21T20:15:00Z",` + `"movers":[{"module":"app"}],"steps":[{"module":"app","step":"build","outcome":"passed","status":"required","started":"2026-11-21T20:15:52Z","ended":"2026-11-21T20:17:55Z"}]}`} var journal []string - health := HealthTool(quiet, none{}, func() time.Time { return now }, func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) }) + firstViews := queueOf{} + health := HealthTool(quiet, none{}, firstViews, func() time.Time { return now }, func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) }) for range 2 { out, err := health.Run(nil) if got, _ := json.Marshal(out); err != nil || !strings.Contains(string(got), "main.137 (nothing open and nothing placed for 17m5s)") { @@ -80,6 +81,17 @@ func TestTools(t *testing.T) { if len(journal) != 1 || !strings.Contains(journal[0], "main.137") { t.Errorf("the journal said %q, want main.137 said once", journal) } + // design 47 H2 §788: registrations-read is false, since the ask, until the registrations' first view is answered. + for _, c := range []struct{ status, want string }{ + {`{"owner":"mesh-catalog","state":"registrations","asked":"2026-11-21T20:30:00Z"}`, `"registrations-read":{"read":false,"since":"2026-11-21T20:30:00Z"}`}, + {`{"owner":"mesh-catalog","state":"registrations","asked":"2026-11-21T20:30:00Z","answered":"2026-11-21T20:30:02Z"}`, `"registrations-read":{"read":true,"since":"2026-11-21T20:30:02Z"}`}, + } { + firstViews["mesh-catalog.registrations"] = c.status + out, err := health.Run(nil) + if got, _ := json.Marshal(out); err != nil || !strings.Contains(string(got), c.want) { + t.Errorf("runner_health = %s (%v), want %s", got, err, c.want) + } + } // design 47 H2 §786: runner_unfinished reads each key and its entry; runner_unfinished_drop names why, drops nothing when // its hand act is not recorded, and drops the key once it is. if got, err := tools[0].Run(nil); err != nil || fmt.Sprint(got) != "[{main.app {src2:aa 4 main.4 failed}}]" { diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index 14d0832c1..0936a06ea 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -169,7 +169,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // Its asking half (§22, §24): a pipeline's steps are placed from the module's registration and each build seat is asked // for its step. starter := &pipeline.Starter{Pipelines: store, Logf: logger.Printf, Say: say, - Registration: rv.Registrations.Registration, Steps: busStepAsker{ctx: ctx, publish: js.PublishMsg, cred: cred, bound: askBound, logf: logger.Printf}, Name: cred.Module, + Registration: rv.Registrations.Registration, RegistrationsRead: func() bool { return chanClosed(rv.Registrations.Ready()) }, Steps: busStepAsker{ctx: ctx, publish: js.PublishMsg, cred: cred, bound: askBound, logf: logger.Printf}, Name: cred.Module, Kept: keptReader(ctx, kept), Toolchain: toolchainReader(ctx, toolchains), Fills: fills, Recovered: kvRecoveries{ctx, recovered}.Has, Holders: rv.Holders.Holders, Ended: hearer.EndedAtPlacing, Findings: rv.Findings, Observer: cred.Module, // a mover waiting on its registration is registration-missing (H-A-262) @@ -397,3 +397,13 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // ownBucket is a state of the runner's old state section on the bus, `_`. func ownBucket(module, state string) string { return module + "_" + state } + +// chanClosed says whether c is closed now, without waiting. +func chanClosed(c <-chan struct{}) bool { + select { + case <-c: + return true + default: + return false + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/consumers.go b/modules/mesh-delivery-pipeline/internal/pipeline/consumers.go index 47f81be7a..792c9b92c 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/consumers.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/consumers.go @@ -7,9 +7,11 @@ import ( "strings" "time" + catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" "git.novox.be/novox/mesh-sdk/go/health" "git.novox.be/novox/mesh-sdk/go/process" + "git.novox.be/novox/mesh-sdk/go/views" ) // StateConsumers is the runner's old state section of its durable consumers' facts, a key per consumer, written by @@ -128,3 +130,35 @@ func consumersHealth(consumers Kept, now time.Time) health.Answer { } return health.Healthy(fmt.Sprintf("The runner answers show, and each of its %d consumers is less than %s behind.", len(keys), BehindAfter)) } + +// RegistrationsRead is runner_health's registrations-read (design 47 H2 §788): whether the runner's registrations view's +// first read is in, and since when; before it, since when it was asked. +type RegistrationsRead struct { + Read bool `json:"read"` + Since *time.Time `json:"since,omitempty"` +} + +// RegistrationsReadOf reads registrations-read from the runner's own first-views state, where its process keeps each first +// view's status (views.StatusState): read once the seat mesh-catalog's registrations view was answered. +func RegistrationsReadOf(firstViews Kept) (RegistrationsRead, error) { + raw, err := firstViews.Value(views.StatusKey(catalog.Seat, catalog.StateRegistrations)) + if err != nil { + return RegistrationsRead{}, fmt.Errorf("the first view of the registrations is not read from %s: %w", views.StatusState, err) + } + if len(raw) == 0 { + return RegistrationsRead{}, nil + } + var st views.Status + if err := json.Unmarshal(raw, &st); err != nil { + return RegistrationsRead{}, fmt.Errorf("the first view of the registrations in %s: %w", views.StatusState, err) + } + if !st.Answered.IsZero() { + at := st.Answered + return RegistrationsRead{Read: true, Since: &at}, nil + } + if !st.Asked.IsZero() { + at := st.Asked + return RegistrationsRead{Since: &at}, nil + } + return RegistrationsRead{}, nil +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/registration_missing.go b/modules/mesh-delivery-pipeline/internal/pipeline/registration_missing.go index bba0f2832..893168196 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/registration_missing.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/registration_missing.go @@ -20,6 +20,13 @@ func registrationMissingID(module string) string { return keys.EscapeName(module // registrationMissing raises registration-missing of module, waited on by the pipeline key (H-A-262). func (s *Starter) registrationMissing(module, key string) { + // Judged only once the registrations view's first read is in (design 47 H2 §788); before it, said once. + if s.RegistrationsRead != nil && !s.RegistrationsRead() { + if s.unreadSaid.CompareAndSwap(false, true) { + s.logf("registrations not yet read: no step is placed from a registration not read, and no %s is judged before they are (%s waits, in %s)", RegistrationMissingKind, module, key) + } + return + } if s.Findings == nil { return } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index eaf7850bf..9427957b6 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -9,6 +9,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" runnerclient "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient" @@ -162,6 +163,10 @@ type Starter struct { // Registration is a module's registration, whose policy names the steps of its pipeline (§21, §23); with none, a // pipeline starts with no step and says so. Registration func(module string) (catalog.Registration, bool) + // RegistrationsRead says whether the registrations view's first read is in (design 47 H2 §788): before it no + // registration-missing is judged and "registrations not yet read" is said once; nil reads as read. + RegistrationsRead func() bool + unreadSaid atomic.Bool // Steps asks each build seat for its step, as the pipeline's asking half (§22, §24); Name is the runner's own name as an // asker. Steps StepAsker diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go index b26b82a7b..caa04973a 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go @@ -197,6 +197,45 @@ func TestATakeInIsOnePipeline(t *testing.T) { } } +// design 47 H2 §788: before the registrations view's first read is in, a start raises no registration-missing, says +// "registrations not yet read" once and places no step; once read, its movers are placed from the registrations held. +func TestTheRegistrationCheckWaitsForItsView(t *testing.T) { + reg, entries := placeFixture() + store := memCreated{memStore{}} + asks := &stepAsks{} + read := false + var lines []string + s := &Starter{Pipelines: store, Steps: asks, Name: "the-runner", Say: func(string, any) error { return nil }, + Logf: func(f string, a ...any) { lines = append(lines, fmt.Sprintf(f, a...)) }, + RegistrationsRead: func() bool { return read }, + Registration: func(m string) (catalog.Registration, bool) { return reg, read && m == "app" }} + findings := newHeld() + s.Findings, s.Observer = findings, "the-runner" + view := func(string) (Half, bool) { return Half{Entries: entries}, true } + for _, src := range []int64{4, 5} { + taken := catalog.TakeInTaken{Branch: "main", Source: src, Entries: []catalog.TakeInEntry{{Module: "app", Commit: fmt.Sprint("c", src), Fingerprint: fmt.Sprint("src2:a", src)}}} + if _, err := s.TakenIn(taken, view); err != nil { + t.Fatal(err) + } + } + said := 0 + for _, l := range lines { + if strings.HasPrefix(l, "registrations not yet read") { + said++ + } + } + if len(findings.held) != 0 || said != 1 || len(asks.asks) != 0 { + t.Fatalf("unread: findings %v, said %d times, %d asks; want none, once, none", findings.held, said, len(asks.asks)) + } + read = true + if n, err := s.Waiting(store.memStore, "", view); n == 0 || err != nil || len(asks.asks) == 0 { + t.Fatalf("once read: placed in %d (%v), %d asks", n, err, len(asks.asks)) + } + if len(findings.held) != 0 { + t.Fatalf("once read, a registered mover raised %v", findings.held) + } +} + // A-327 (design 47 G §21, §23, §54): a pipeline started before its module has a registration starts with no step; the // registration-changed that brings it places that pipeline's steps from it and asks them, once: heard again it places nothing. func TestAPipelineStartedBeforeItsRegistrationIsPlacedWhenItArrives(t *testing.T) { From d13efa8abe2c248cdd30c05c269f1fc2ec4f0b04 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 01:22:06 +0100 Subject: [PATCH 09/15] R9: the stalled bound is the setting stalled-bound; the health names a step whose ask is untaken past it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 H2 §790, §778 (the health line), §781 (read through mesh-sdk's settings reader) --- .../cmd/mesh-delivery-pipeline-tools/main.go | 17 +++++--- .../mesh-delivery-pipeline-tools/main_test.go | 4 +- .../cmd/mesh-delivery-pipeline/main.go | 7 ++++ .../run_integration_test.go | 28 ++++++++++++- .../internal/pipeline/consumers.go | 41 +++++++++++-------- .../internal/pipeline/consumers_test.go | 31 +++++++++++++- .../mesh-delivery-pipeline/manifest_test.go | 10 ++++- modules/mesh-delivery-pipeline/module.json | 7 ++++ 8 files changed, 118 insertions(+), 27 deletions(-) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go index b1a2627e1..ec50980ba 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go @@ -103,17 +103,18 @@ func FirstViewTools(module string, st stdio.KeptState, acts views.HandActs, now // runner's state pipelines as show does and the ages its process writes in the state consumers, and asks no seat verb. A // running pipeline it first names stalled is said in this program's journal through logf, once per key for the // program's life (H-A-311 (3), H-A-313). -func HealthTool(pipelines, consumers, firstViews pipeline.Kept, now func() time.Time, logf func(string, ...any)) stdio.Tool { +func HealthTool(pipelines, consumers, firstViews pipeline.Kept, bound time.Duration, now func() time.Time, logf func(string, ...any)) stdio.Tool { var mu sync.Mutex said := map[string]bool{} return stdio.Tool{Name: Prefix + "_health", Description: "The pipeline runner's health: healthy when it answers show, each of its durable consumers' oldest " + "unacknowledged message waits less than five minutes and no running pipeline has had nothing open and nothing " + - "newly placed for five minutes; otherwise it names each consumer behind and each such pipeline by its key. It " + + "newly placed, nor a step's ask stayed untaken, for the setting stalled-bound (five minutes by default); otherwise it " + + "names each consumer behind, each such pipeline by its key and each such step. It " + "answers registrations-read: whether the runner's registrations view's first read is in, and since when.", Input: map[string]any{"type": "object", "properties": map[string]any{}}, Run: func(map[string]any) (any, error) { - a, stalled := pipeline.HealthStalled(pipelines, consumers, now()) + a, stalled := pipeline.HealthStalled(pipelines, consumers, now(), bound) // design 47 H2 §788: whether the registrations are read, from the runner's own first-views state. read, err := pipeline.RegistrationsReadOf(firstViews) if err != nil { @@ -124,7 +125,7 @@ func HealthTool(pipelines, consumers, firstViews pipeline.Kept, now func() time. for _, key := range stalled { if !said[key] && logf != nil { said[key] = true - logf("the running pipeline %s has had nothing open and nothing newly placed for %s or more: runner_health names it unhealthy (H-A-311)", key, pipeline.StalledAfter) + logf("%s has stood still for %s or more (nothing open and nothing newly placed, or its ask untaken): runner_health names it unhealthy (H-A-311, design 47 H2 §790)", key, bound) } } return HealthAnswer{Answer: a, RegistrationsRead: read}, nil @@ -174,6 +175,12 @@ func StoresFill(state func(string) (*stdio.StateInfo, error), objects func(strin } func main() { + // design 47 H2 §790, §781: the stalled bound is the runner's setting, read here as its process reads it. + stalledBound, err := stdio.SettingDuration(pipeline.StalledBoundSetting, pipeline.DefaultStalledBound) + if err != nil { + fmt.Fprintf(os.Stderr, "[mesh-delivery-pipeline-tools] %v\n", err) + os.Exit(1) + } fill := StoresFill(func(n string) (*stdio.StateInfo, error) { return stdio.State(n).Info() }, func(n string) (buscalls.Fill, error) { return busstate.ObjectsOf(n).Fill() }, func(n string) (*stdio.LogInfo, error) { return stdio.Log(n).Info() }, time.Now) @@ -185,7 +192,7 @@ func main() { tools = append(tools, CancelTool(r.List, stdio.Ask), RerunTool(stdio.Ask)) tools = append(tools, VerbTools(stdio.Ask)...) tools = append(tools, TestRunsTools(stdio.Ask)...) - tools = append(tools, HealthTool(r.Pipelines, kept{stdio.State(pipeline.StateConsumers)}, kept{stdio.State(views.StatusState)}, time.Now, + tools = append(tools, HealthTool(r.Pipelines, kept{stdio.State(pipeline.StateConsumers)}, kept{stdio.State(views.StatusState)}, stalledBound, time.Now, func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-pipeline-tools] "+f+"\n", a...) })) tools = append(tools, WaitingTool(kept{stdio.State(seat.StateQueue)})) tools = append(tools, UnfinishedTools(keptDelete{kept{stdio.State(pipeline.StateUnfinished)}}, meshhandactsclient.Recorder{Ask: stdio.Ask})...) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main_test.go index 768f3daa8..2b1d63a42 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main_test.go @@ -42,7 +42,7 @@ func TestTools(t *testing.T) { names = append(names, tl.Name) } // RN25 (§468, §76; A-164): the health check is the bundle's runner_health. - names = append(names, HealthTool(none{}, none{}, none{}, nil, nil).Name, WaitingTool(none{}).Name) + names = append(names, HealthTool(none{}, none{}, none{}, pipeline.DefaultStalledBound, nil, nil).Name, WaitingTool(none{}).Name) // §606, §549: the unlogged entries are listed and appended by hand. for _, tl := range UnloggedTools(stdio.State("pipelines"), stdio.Log("step-log"), nil, nil) { names = append(names, tl.Name) @@ -71,7 +71,7 @@ func TestTools(t *testing.T) { `"movers":[{"module":"app"}],"steps":[{"module":"app","step":"build","outcome":"passed","status":"required","started":"2026-11-21T20:15:52Z","ended":"2026-11-21T20:17:55Z"}]}`} var journal []string firstViews := queueOf{} - health := HealthTool(quiet, none{}, firstViews, func() time.Time { return now }, func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) }) + health := HealthTool(quiet, none{}, firstViews, pipeline.DefaultStalledBound, func() time.Time { return now }, func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) }) for range 2 { out, err := health.Run(nil) if got, _ := json.Marshal(out); err != nil || !strings.Contains(string(got), "main.137 (nothing open and nothing placed for 17m5s)") { diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index 0936a06ea..96dcee407 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -20,6 +20,7 @@ import ( catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-sdk/go/seatlist" @@ -166,6 +167,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err != nil { return err } + // design 47 H2 §790, §781: the stalled bound, read as the tools program reads it; a value that is no duration stops the start. + stalledBound, err := stdio.SettingDuration(pipeline.StalledBoundSetting, pipeline.DefaultStalledBound) + if err != nil { + return err + } + logger.Printf("health: a running pipeline with nothing open and nothing newly placed, or a step whose ask is untaken, is named after %s (%s)", stalledBound, pipeline.StalledBoundSetting) // Its asking half (§22, §24): a pipeline's steps are placed from the module's registration and each build seat is asked // for its step. starter := &pipeline.Starter{Pipelines: store, Logf: logger.Printf, Say: say, diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go index fab8b6a12..ea9c12ccb 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go @@ -22,6 +22,7 @@ import ( builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-sdk/go/seatlist" @@ -50,6 +51,8 @@ func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *tes ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() reg := catalog.Registration{Module: "app", Branches: []string{"main"}, Origin: "o/app", Policy: catalog.Policy{Steps: catalog.StepStatuses{Review: "optional", Validate: "bypass", Composition: "bypass", Approval: "bypass"}}} + // design 47 H2 §790: the stalled bound is the setting stalled-bound, here one minute, read as the node-engine composes it. + t.Setenv(stdio.SettingWord(pipeline.StalledBoundSetting), "1m") // The branch main has no revision yet at the runner's start (none-yet), then revision 1 holding app (A-336). var revised atomic.Bool w := startWhole(t, ctx, []catalog.Registration{reg}, func([]byte) catalog.RevisionAnswer { @@ -77,6 +80,15 @@ func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *tes if len(p.Steps) != 0 { t.Fatalf("a take-in before its branch's half placed %d steps, want none (A-336)", len(p.Steps)) } + // design 47 H2 §790: at one minute the health names the pipeline with nothing placed, a minute after its start. + bound, err := stdio.SettingDuration(pipeline.StalledBoundSetting, pipeline.DefaultStalledBound) + if err != nil || bound != time.Minute || !strings.Contains(logs.String(), "is named after 1m0s (stalled-bound)") { + t.Fatalf("the stalled bound %s (%v); the process says it: %t", bound, err, strings.Contains(logs.String(), "is named after 1m0s")) + } + noConsumers := kvKept{ctx, mustKV(t, ctx, js, module+"_consumers")} + if a := pipeline.Health(kvKept{ctx, pipelines}, noConsumers, p.Started.Add(61*time.Second), bound); a.Healthy || !strings.Contains(a.Why, key+" (nothing open") { + t.Fatalf("a minute after its start, with nothing placed, the health says %+v", a) + } // The revision said is main's first half: held, it places the pipeline's steps from the registration (A-327). revised.Store(true) rev, _ := json.Marshal(catalog.CatalogRevisionEvent{V: 1, Revision: 1, Kind: "trunk", Branch: "main", Source: 3}) @@ -88,6 +100,10 @@ func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *tes if build.Outcome != seat.StepResultOutcomeRunning || build.Ask == "" { t.Fatalf("the build step = %+v", build) } + // Once a step is placed in it, the health no longer names it (§790). + if a := pipeline.Health(kvKept{ctx, pipelines}, noConsumers, p.Started.Add(61*time.Second), bound); strings.Contains(a.Why, key+" (nothing open") { + t.Fatalf("with its build placed, the health still names %s: %+v", key, a) + } // The build is asked of the seat mesh-builds: its ask is on its work queue. waitFor(t, ctx, logs, func() bool { info, err := asks.Info(ctx); return err == nil && info.State.Msgs >= 1 }) @@ -167,7 +183,7 @@ func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *tes e, err := consumers.Get(ctx, "n1_"+module) return err == nil && json.Unmarshal(e.Value(), &facts) == nil && facts.OldestUnacknowledged.IsZero() && facts.Acknowledged > 0 }) - if a := pipeline.Health(kvKept{ctx, pipelines}, kvKept{ctx, consumers}, time.Now()); !a.Healthy { + if a := pipeline.Health(kvKept{ctx, pipelines}, kvKept{ctx, consumers}, time.Now(), bound); !a.Healthy { t.Errorf("runner_health over the bus: %+v", a) } } @@ -293,6 +309,16 @@ func startWhole(t *testing.T, ctx context.Context, regs []catalog.Registration, return whole{nc: nc, js: js, asks: asks, pipelines: pipelines, logs: logs} } +// mustKV is the bucket name. +func mustKV(t *testing.T, ctx context.Context, js jetstream.JetStream, name string) jetstream.KeyValue { + t.Helper() + kv, err := js.KeyValue(ctx, name) + if err != nil { + t.Fatal(err) + } + return kv +} + // kvKept reads a bucket as the tools bundle's states do. type kvKept struct { ctx context.Context diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/consumers.go b/modules/mesh-delivery-pipeline/internal/pipeline/consumers.go index 792c9b92c..50076dbd2 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/consumers.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/consumers.go @@ -22,32 +22,39 @@ const StateConsumers = process.ConsumersState // behind (§76). const BehindAfter = 5 * time.Minute -// StalledAfter is how long a running pipeline may have nothing open and nothing newly placed before the runner's health -// names it stalled (H-A-311 (3)): the mesh's own rule for nothing moving (nothing-moved-in-five-minutes), not a setting -// (H-A-322). -const StalledAfter = 5 * time.Minute +// DefaultStalledBound is the setting stalled-bound's default (design 47 H2 §790): how long a running pipeline may have +// nothing open and nothing newly placed, or a step's ask stay untaken (§778), before the runner's health names it; the +// mesh's own rule for nothing moving (nothing-moved-in-five-minutes). +const DefaultStalledBound = 5 * time.Minute + +// StalledBoundSetting is the runner's setting of that bound, read by its process and its tools program (§790, §781). +const StalledBoundSetting = "stalled-bound" // Stalled is the key of each running pipeline, as show reads it, that has had nothing open (no step asked and not ended, -// no required step waiting but on a package's publish) and nothing newly placed, started or ended for StalledAfter at now (H-A-311 (3)), each with -// how long, in key order. -func Stalled(pipelines Kept, now time.Time) ([]string, error) { +// no required step waiting but on a package's publish) and nothing newly placed, started or ended for bound at now +// (H-A-311 (3), design 47 H2 §790), each with how long, in key order; and each step of a running pipeline whose ask is +// still untaken bound after it was put (§778), each with its seat and how long. +func Stalled(pipelines Kept, now time.Time, bound time.Duration) ([]string, error) { all, err := Reads{Pipelines: pipelines}.pipelines() if err != nil { return nil, err } - var out []string + var out, untaken []string for _, p := range all { if p.Outcome != seat.PipelineOutcomeRunning { continue } last, busy := startedOf(p), false for _, st := range p.Steps { + if st.Ask != "" && st.Ended == nil && st.TakenAt == nil && st.AskedAt != nil && open(st) && now.Sub(*st.AskedAt) >= bound { + untaken = append(untaken, fmt.Sprintf("%s %s %s (its ask on %s untaken for %s)", p.Pipeline, st.Module, st.Step, + st.Seat, now.Sub(*st.AskedAt).Round(time.Second))) + } // A required step waiting (an approval, a composition on its validate) is open, as the end's judgement holds it; // an optional one waiting holds nothing (A-104), and neither does one waiting on a package's publish, as a mover // with no steps waiting on one does not (H-A-326 (2)). if open(st) || (st.Outcome == seat.StepResultOutcomeWaiting && st.Status == seat.StepResultStatusRequired && !pinHeld(st)) { busy = true - break } for _, at := range []*time.Time{st.Started, st.Ended} { if at != nil && at.After(last) { @@ -55,42 +62,42 @@ func Stalled(pipelines Kept, now time.Time) ([]string, error) { } } } - if busy || last.IsZero() || now.Sub(last) < StalledAfter { + if busy || last.IsZero() || now.Sub(last) < bound { continue } out = append(out, fmt.Sprintf("%s (nothing open and nothing placed for %s)", p.Pipeline, now.Sub(last).Round(time.Second))) } - return out, nil + return append(out, untaken...), nil } // Health is the runner's health as its tool runner_health answers it (§468, §76, §495; A-164): healthy when its state // pipelines answers, as show reads it, each durable consumer's oldest unacknowledged message waits less than // BehindAfter, and no running pipeline is stalled (H-A-311 (3)); otherwise unhealthy naming each consumer behind and // how long, and each stalled pipeline by its key. -func Health(pipelines, consumers Kept, now time.Time) health.Answer { - a, _ := HealthStalled(pipelines, consumers, now) +func Health(pipelines, consumers Kept, now time.Time, bound time.Duration) health.Answer { + a, _ := HealthStalled(pipelines, consumers, now, bound) return a } // HealthStalled is Health with the keys of the running pipelines it names stalled, so the tools program says each in its // journal once (H-A-313). -func HealthStalled(pipelines, consumers Kept, now time.Time) (health.Answer, []string) { +func HealthStalled(pipelines, consumers Kept, now time.Time, bound time.Duration) (health.Answer, []string) { if _, err := pipelines.Keys(); err != nil { return health.Unhealthy(fmt.Sprintf("The runner does not answer show: its state pipelines cannot be listed (%v).", err)), nil } - stalled, err := Stalled(pipelines, now) + stalled, err := Stalled(pipelines, now, bound) if err != nil { return health.Unhealthy(fmt.Sprintf("The runner's running pipelines cannot be read as show reads them (%v).", err)), nil } var keys []string for _, s := range stalled { - keys = append(keys, strings.SplitN(s, " ", 2)[0]) + keys = append(keys, strings.SplitN(s, " (", 2)[0]) } a := consumersHealth(consumers, now) if len(stalled) == 0 { return a, nil } - why := fmt.Sprintf("The runner's running pipelines with nothing open and nothing newly placed for %s or more: %s.", StalledAfter, strings.Join(stalled, ", ")) + why := fmt.Sprintf("The runner's running pipelines with nothing open and nothing newly placed, or a step whose ask is untaken, for %s or more: %s.", bound, strings.Join(stalled, ", ")) if !a.Healthy { why = a.Why + " " + why } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/consumers_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/consumers_test.go index de47a48cc..3c4e0edc6 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/consumers_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/consumers_test.go @@ -65,8 +65,37 @@ func TestHealth(t *testing.T) { {"a pipeline waiting only on packages is named", heldOnPublish(piped(now, "main.137", -20*time.Minute, seat.StepResultOutcomeWaiting), "main.137"), age("n_runner", 0), false, "main.137 (nothing open and nothing placed for 20m0s)"}, } + // design 47 H2 §778, §790: a step whose ask is untaken past the bound is named with its seat; the bound is the setting's. + untaken := func(asked time.Duration, taken bool) mapKept { + k := piped(now, "main.140", -time.Minute, seat.StepResultOutcomeWaitingForABuildNode) + var p seat.Pipeline + _ = json.Unmarshal(k["main.140"], &p) + at := now.Add(-asked) + p.Steps[0].Ask, p.Steps[0].Seat, p.Steps[0].AskedAt, p.Steps[0].Ended = "ask-1", "mesh-builds", &at, nil + if taken { + p.Steps[0].TakenAt = &at + } + raw, _ := json.Marshal(p) + k["main.140"] = raw + return k + } + rows = append(rows, []struct { + name string + pipelines Kept + consumers Kept + healthy bool + says string + }{ + {"an ask untaken six minutes is named", untaken(6*time.Minute, false), age("n_runner", 0), false, "main.140 app build (its ask on mesh-builds untaken for 6m0s)"}, + {"an ask untaken four minutes is not", untaken(4*time.Minute, false), age("n_runner", 0), true, ""}, + {"a taken ask is not", untaken(20*time.Minute, true), age("n_runner", 0), true, ""}, + }...) + if a := Health(piped(now, "main.137", -2*time.Minute, seat.StepResultOutcomePassed), age("n_runner", 0), now, time.Minute); a.Healthy || + !strings.Contains(a.Why, "main.137 (nothing open and nothing placed for 2m0s)") { + t.Errorf("at a one-minute bound a pipeline quiet two minutes: %q", a.Why) + } for _, r := range rows { - a := Health(r.pipelines, r.consumers, now) + a := Health(r.pipelines, r.consumers, now, DefaultStalledBound) if a.Healthy != r.healthy || !strings.Contains(a.Why, r.says) { t.Errorf("%s: healthy %v %q; want %v saying %q", r.name, a.Healthy, a.Why, r.healthy, r.says) } diff --git a/modules/mesh-delivery-pipeline/manifest_test.go b/modules/mesh-delivery-pipeline/manifest_test.go index de97d07d5..00ed76286 100644 --- a/modules/mesh-delivery-pipeline/manifest_test.go +++ b/modules/mesh-delivery-pipeline/manifest_test.go @@ -207,7 +207,15 @@ func runSettingsOf(raw []byte) bool { } want := map[string]string{"test-runs-cap-mib": "number", "test-runs-kept": "string", "test-step-bound": "string", "run-untaken-bound": "string", "ask-unanswered-bound": "string", "ask-silent-bound": "string", "widens-read-bound": "string", "revisions-kept": "number", "pipelines-at-once": "number", "pipelines-waiting-cap": "number"} - if len(m.Config) != len(want) || m.Config["test-runs-cap-mib"]["maximum"] != float64(64) || m.Config["test-runs-cap-mib"]["default"] != float64(16) || + // design 47 H2 §790, §781: stalled-bound is read through mesh-sdk's settings reader, by the process and the tools + // program alike, from the words the node-engine composes: no file of its own. + words := map[string]string{"stalled-bound": "string"} + for key, kind := range words { + if m.Config[key]["type"] != kind || m.Config[key]["default"] != "5m" || m.Config[key]["runtime"] != true { + return false + } + } + if len(m.Config) != len(want)+len(words) || m.Config["test-runs-cap-mib"]["maximum"] != float64(64) || m.Config["test-runs-cap-mib"]["default"] != float64(16) || m.Config["pipelines-at-once"]["minimum"] != float64(1) || m.Config["pipelines-at-once"]["maximum"] != float64(8) || m.Config["pipelines-at-once"]["default"] != float64(1) || m.Config["pipelines-waiting-cap"]["default"] != float64(64) { return false diff --git a/modules/mesh-delivery-pipeline/module.json b/modules/mesh-delivery-pipeline/module.json index b5b28ce67..c438b8f3d 100644 --- a/modules/mesh-delivery-pipeline/module.json +++ b/modules/mesh-delivery-pipeline/module.json @@ -167,6 +167,13 @@ "trusted": false, "default": "10m" }, + "stalled-bound": { + "type": "string", + "description": "how long a running pipeline may have nothing open and nothing newly placed, or a step's ask stay untaken, before the runner's health names it, a Go duration such as 5m (design 47 H2 sections 778, 790).", + "trusted": false, + "runtime": true, + "default": "5m" + }, "ask-silent-bound": { "type": "string", "description": "how long an ask the runner puts on a seat whose holder says nothing before the step's end (mesh-builds, mesh-wants) may go without that end before its step ends could not run, a Go duration such as 45m (design 47 H H-A-243).", From 84989dafb2e88d10637908a481d26bdf832601ba Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 01:28:07 +0100 Subject: [PATCH 10/15] D7: a catalog entry carries its class MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The seat mesh-catalog's CatalogEntry gains class, core or leaf; an entry without it reads as leaf. The indexer writes it from the manifest. The seat's form rises to 3; regenerated with seatgen, -check clean. Builds: design 47 H2 §810; H-A-394 (1), H-A-431 (1) --- .../go/seats/meshcatalog/contract.go | 8 ++-- .../go/seats/meshcatalog/contract_test.go | 6 +++ .../seats/mesh-catalog/fixtures/states.json | 40 +++++++++++++++++++ .../seats/mesh-catalog/seat.json | 6 ++- .../ts/seats/meshcatalog/index.d.ts | 2 + 5 files changed, 58 insertions(+), 4 deletions(-) diff --git a/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract.go b/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract.go index 6a09c17a2..19cadc72e 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract.go @@ -14,13 +14,13 @@ const Seat = "mesh-catalog" // Form is the form of the seat's values, events and answers this contract was generated at (design 47 H2 §780): // a reader of a value of another form says "form N, this reader knows M". -const Form = 2 +const Form = 3 // The forms and the digest of the fields this contract was generated from: the generator's compare fails a definition // whose fields changed while its forms did not (design 47 H2 §780). const ( - generatedForms = "2" - generatedFieldsDigest = "ActingRule=8a83ce7e0457 Artifact=78b05c1a9ec3 ArtifactPackage=2a3a660a934d ArtifactRef=a2f06147f4b6 BuildEdge=76f0684e254a BuildEdges=d2b95000259e BuildEdgesByModuleBranchValue=4fdf7cc1e723 CandidatesByGroupValue=9ea8c6c580d2 CandidatesByGroupValuePullsItem=90fb03643258 CandidatesByGroupValuePullsItemFpsValue=4c6783201aa3 CatalogEntry=d154f232237b CatalogRevision=d1ecaaa484af CatalogRevisionDefinitionsItem=77fcff620d48 CatalogRevisionEvent=710cbc5bfff1 CatalogRevisionsByBranchRevisionValue=17e54f635d35 CatalogRevisionsByCandidateGroupHeadsValue=17e54f635d35 CatalogRevisionsByEntryDigestValue=8755036a3051 DependentsAnswer=7538f2f5e68e Edge=7242732afaef EntryAnswer=805c2ff28e1c EntryInputs=76a7f8690dd2 EntryRef=ff109e5089e1 Fill=80f442916429 FillingEvent=5bd4df78ed84 FingerprintInputs=99a6df7f991f FullEvent=5bd4df78ed84 Head=774b4bf88ba4 InputAnswer=cd41ee65565d InputsByTimeDigestValue=a2f06147f4b6 ManifestsByDigestValue=51b3822b29c6 ModuleAnswer=499a03ce9b27 ModuleAnswerModule=c0f679e09f08 ModulesAnswer=6d119571c69e ModulesAnswerModulesItem=a70f39c60b67 ModulesByModuleValue=fc7530ba893a ModulesByModuleValueInputs2=99a6df7f991f Outdated=e820214f66bc OutdatedAnswer=275fb25266b9 OutdatedDependency=dc35dca1b683 PackagesByModuleValue=2b62574b709f PackagesByModuleValuePackagesItem=a7d30558470d PackagesByPackageSeriesValue=07421d8d3411 PackagesByPackageSeriesValueNewest=aba144994370 PackagesByPackageSeriesValueNewestContents=eca2ccf56913 PackagesByPackageSeriesValueNewestContentsPackagesItem=a7d30558470d PackagesByPackageSeriesValueNumbered=c8ffc628599c PackagesByPackageSeriesValueNumberedContents=0def6c0794ee PackagesByPackageSeriesValueNumberedContentsPackagesItem=a7d30558470d Policy=719e2f460c6f PolicyAnswer=46cdf8430557 PredefinedByListValue=ec8e3febe8ac ProvidesAnswer=7538f2f5e68e RegisterAnswer=46cdf8430557 Registration=ef03d1b2e73f RegistrationAsked=55adba349c24 RegistrationChangedEvent=99a3268edf41 RegistrationHistoryByModuleVersionValue=0999b5bb7dc7 RegistrationHistoryBySourceOwnerNameVersionValue=0999b5bb7dc7 RegistrationsAnswer=8849ef618cf3 RegistrationsByModuleValue=0999b5bb7dc7 RegistrationsBySourceOwnerNameValue=0999b5bb7dc7 RevisionAnswer=258ea5124931 RoomEvent=5bd4df78ed84 SeatDefinitionChangedEvent=e271d24d58f7 SeatDefinitionRemovedEvent=f7d6171111e6 SeatDefinitionsAnswer=0fd6a59fec59 SeatDefinitionsAnswerEntry=d006f68d2be5 SeatDefinitionsAnswerSeatsItem=e271d24d58f7 SeatListAnswer=912c0cebcf0a SourceCheck=d1ed7cbbd579 StepStatuses=4f50aa633e61 TakeInEntry=545ec1528499 ToolchainUsed=43b6d42376fb UnwatchAnswer=46cdf8430557 VersionArtifact=438c49a392b0 VersionEdge=b0f80e83f4dc VersionEntry=33f5b00dbda9 VersionReleasedEvent=4ccf1ea90e7f VersionsAnswer=b59e6e057e7b VersionsByModuleVersionValue=e546dfadb913 VersionsFilledEvent=4cc437912dc3 WatchAnswer=46cdf8430557" + generatedForms = "3" + generatedFieldsDigest = "ActingRule=8a83ce7e0457 Artifact=78b05c1a9ec3 ArtifactPackage=2a3a660a934d ArtifactRef=a2f06147f4b6 BuildEdge=76f0684e254a BuildEdges=d2b95000259e BuildEdgesByModuleBranchValue=4fdf7cc1e723 CandidatesByGroupValue=9ea8c6c580d2 CandidatesByGroupValuePullsItem=90fb03643258 CandidatesByGroupValuePullsItemFpsValue=4c6783201aa3 CatalogEntry=c0a4ea25d523 CatalogRevision=d1ecaaa484af CatalogRevisionDefinitionsItem=77fcff620d48 CatalogRevisionEvent=710cbc5bfff1 CatalogRevisionsByBranchRevisionValue=17e54f635d35 CatalogRevisionsByCandidateGroupHeadsValue=17e54f635d35 CatalogRevisionsByEntryDigestValue=8755036a3051 DependentsAnswer=7538f2f5e68e Edge=7242732afaef EntryAnswer=805c2ff28e1c EntryInputs=76a7f8690dd2 EntryRef=ff109e5089e1 Fill=80f442916429 FillingEvent=5bd4df78ed84 FingerprintInputs=99a6df7f991f FullEvent=5bd4df78ed84 Head=774b4bf88ba4 InputAnswer=cd41ee65565d InputsByTimeDigestValue=a2f06147f4b6 ManifestsByDigestValue=51b3822b29c6 ModuleAnswer=499a03ce9b27 ModuleAnswerModule=c0f679e09f08 ModulesAnswer=6d119571c69e ModulesAnswerModulesItem=a70f39c60b67 ModulesByModuleValue=fc7530ba893a ModulesByModuleValueInputs2=99a6df7f991f Outdated=e820214f66bc OutdatedAnswer=275fb25266b9 OutdatedDependency=dc35dca1b683 PackagesByModuleValue=2b62574b709f PackagesByModuleValuePackagesItem=a7d30558470d PackagesByPackageSeriesValue=07421d8d3411 PackagesByPackageSeriesValueNewest=aba144994370 PackagesByPackageSeriesValueNewestContents=eca2ccf56913 PackagesByPackageSeriesValueNewestContentsPackagesItem=a7d30558470d PackagesByPackageSeriesValueNumbered=c8ffc628599c PackagesByPackageSeriesValueNumberedContents=0def6c0794ee PackagesByPackageSeriesValueNumberedContentsPackagesItem=a7d30558470d Policy=719e2f460c6f PolicyAnswer=46cdf8430557 PredefinedByListValue=ec8e3febe8ac ProvidesAnswer=7538f2f5e68e RegisterAnswer=46cdf8430557 Registration=ef03d1b2e73f RegistrationAsked=55adba349c24 RegistrationChangedEvent=99a3268edf41 RegistrationHistoryByModuleVersionValue=0999b5bb7dc7 RegistrationHistoryBySourceOwnerNameVersionValue=0999b5bb7dc7 RegistrationsAnswer=8849ef618cf3 RegistrationsByModuleValue=0999b5bb7dc7 RegistrationsBySourceOwnerNameValue=0999b5bb7dc7 RevisionAnswer=258ea5124931 RoomEvent=5bd4df78ed84 SeatDefinitionChangedEvent=e271d24d58f7 SeatDefinitionRemovedEvent=f7d6171111e6 SeatDefinitionsAnswer=0fd6a59fec59 SeatDefinitionsAnswerEntry=d006f68d2be5 SeatDefinitionsAnswerSeatsItem=e271d24d58f7 SeatListAnswer=912c0cebcf0a SourceCheck=d1ed7cbbd579 StepStatuses=4f50aa633e61 TakeInEntry=545ec1528499 ToolchainUsed=43b6d42376fb UnwatchAnswer=46cdf8430557 VersionArtifact=438c49a392b0 VersionEdge=b0f80e83f4dc VersionEntry=33f5b00dbda9 VersionReleasedEvent=4ccf1ea90e7f VersionsAnswer=b59e6e057e7b VersionsByModuleVersionValue=e546dfadb913 VersionsFilledEvent=4cc437912dc3 WatchAnswer=46cdf8430557" ) const ( @@ -679,6 +679,8 @@ type CatalogEntry struct { Awaits []string `json:"awaits,omitempty"` // the images and modules it is built on Bases []string `json:"bases,omitempty"` + // core or leaf: core when its manifest claims a seat, leaf when it claims none; an entry without it reads as leaf (design 47 H2 H-A-394, H-A-431) + Class string `json:"class,omitempty"` // the merge commit it was last changed at Commit string `json:"commit"` // the requirements the manifest's contributes object names as keys, sorted, its local names not carried; each is also in requires (design 47 G section 709) diff --git a/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract_test.go b/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract_test.go index 0f4791843..7116d74ee 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract_test.go +++ b/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract_test.go @@ -83,6 +83,12 @@ func TestFixtures(t *testing.T) { err := json.Unmarshal(b, &v) return v, err }, subjectIs: nil}, + {name: "a seat-claiming module's entry carries its class core (H-A-394, H-A-431)", subject: "", given: "{\"acts\":[{\"bound\":\"60m\",\"on\":\"go\",\"seat\":\"node-build-go\",\"step\":\"build\"}],\"awaits\":[\"database\"],\"class\":\"core\",\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fp\":\"src2:b427bdde1f100e2f52490cfa9df0c5112dd3700226b4892349f0c14a15a4af56\",\"left-out\":[\"mesh-catalog.revision\"],\"main\":\"cmd/mesh-build-go\",\"manifest\":{\"module\":\"mesh-build-go\",\"version\":\"1\"},\"module\":\"mesh-build-go\",\"needs\":[\"form:setting\",\"word:claims\"],\"path\":\"modules/mesh-build-go\",\"repository\":\"novox/mesh-catalog\",\"serves\":[\"node-build-go.run\"],\"v\":1,\"verdict\":\"accepted\"}", wire: "{\"acts\":[{\"bound\":\"60m\",\"on\":\"go\",\"seat\":\"node-build-go\",\"step\":\"build\"}],\"awaits\":[\"database\"],\"class\":\"core\",\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fp\":\"src2:b427bdde1f100e2f52490cfa9df0c5112dd3700226b4892349f0c14a15a4af56\",\"left-out\":[\"mesh-catalog.revision\"],\"main\":\"cmd/mesh-build-go\",\"manifest\":{\"module\":\"mesh-build-go\",\"version\":\"1\"},\"module\":\"mesh-build-go\",\"needs\":[\"form:setting\",\"word:claims\"],\"path\":\"modules/mesh-build-go\",\"repository\":\"novox/mesh-catalog\",\"serves\":[\"node-build-go.run\"],\"v\":1,\"verdict\":\"accepted\"}", + decode: func(b []byte) (any, error) { + var v CatalogRevisionsByEntryDigestValue + err := json.Unmarshal(b, &v) + return v, err + }, subjectIs: nil}, {name: "a module's registration", subject: "", given: "{\"at\":\"2026-11-10T10:00:00Z\",\"branches\":[\"main\"],\"by\":\"operator@workstation\",\"module\":\"nats\",\"origin\":\"novox/mesh-catalog\",\"path\":\"modules/nats/module.json\",\"policy\":{\"artifacts\":{\"program\":{\"unit\":\"bypass\"}},\"steps\":{\"approval\":\"bypass\",\"composition\":\"required\",\"integration\":\"required\",\"lint\":\"required\",\"review\":\"optional\",\"unit\":\"required\"}},\"v\":1,\"version\":2,\"warrant\":\"ask-m2kx9a1b3c4d5e\"}", wire: "{\"at\":\"2026-11-10T10:00:00Z\",\"branches\":[\"main\"],\"by\":\"operator@workstation\",\"module\":\"nats\",\"origin\":\"novox/mesh-catalog\",\"path\":\"modules/nats/module.json\",\"policy\":{\"artifacts\":{\"program\":{\"unit\":\"bypass\"}},\"steps\":{\"approval\":\"bypass\",\"composition\":\"required\",\"integration\":\"required\",\"lint\":\"required\",\"review\":\"optional\",\"unit\":\"required\"}},\"v\":1,\"version\":2,\"warrant\":\"ask-m2kx9a1b3c4d5e\"}", decode: func(b []byte) (any, error) { var v RegistrationsByModuleValue diff --git a/modules/mesh-delivery-contracts/seats/mesh-catalog/fixtures/states.json b/modules/mesh-delivery-contracts/seats/mesh-catalog/fixtures/states.json index 18e92dfa7..9a6ee5360 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-catalog/fixtures/states.json +++ b/modules/mesh-delivery-contracts/seats/mesh-catalog/fixtures/states.json @@ -345,6 +345,46 @@ ] } }, + { + "name": "a seat-claiming module's entry carries its class core (H-A-394, H-A-431)", + "of": "state:catalog-revisions/entry.", + "given": { + "v": 1, + "module": "mesh-build-go", + "repository": "novox/mesh-catalog", + "path": "modules/mesh-build-go", + "commit": "f469eb387a3015569d9b16c66ed418e9dd098f64", + "manifest": { + "module": "mesh-build-go", + "version": "1" + }, + "verdict": "accepted", + "fp": "src2:b427bdde1f100e2f52490cfa9df0c5112dd3700226b4892349f0c14a15a4af56", + "serves": [ + "node-build-go.run" + ], + "left-out": [ + "mesh-catalog.revision" + ], + "needs": [ + "form:setting", + "word:claims" + ], + "awaits": [ + "database" + ], + "main": "cmd/mesh-build-go", + "acts": [ + { + "seat": "node-build-go", + "step": "build", + "on": "go", + "bound": "60m" + } + ], + "class": "core" + } + }, { "name": "a module's registration", "of": "state:registrations/", diff --git a/modules/mesh-delivery-contracts/seats/mesh-catalog/seat.json b/modules/mesh-delivery-contracts/seats/mesh-catalog/seat.json index 2d5ab2d84..da8e95508 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-catalog/seat.json +++ b/modules/mesh-delivery-contracts/seats/mesh-catalog/seat.json @@ -1686,6 +1686,10 @@ "type": "string", "description": "its manifest's verdict: accepted, refused or held" }, + "class": { + "type": "string", + "description": "core or leaf: core when its manifest claims a seat, leaf when it claims none; an entry without it reads as leaf (design 47 H2 H-A-394, H-A-431)" + }, "fp": { "type": "string", "description": "its fingerprint (src2: and 64 hex digits); absent when it has none" @@ -2721,5 +2725,5 @@ ] } }, - "form": 2 + "form": 3 } diff --git a/modules/mesh-delivery-contracts/ts/seats/meshcatalog/index.d.ts b/modules/mesh-delivery-contracts/ts/seats/meshcatalog/index.d.ts index 8e6e6aa9f..19e3606ab 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshcatalog/index.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/meshcatalog/index.d.ts @@ -236,6 +236,8 @@ export interface CatalogEntry { "awaits"?: string[]; /** the images and modules it is built on */ "bases"?: string[]; + /** core or leaf: core when its manifest claims a seat, leaf when it claims none; an entry without it reads as leaf (design 47 H2 H-A-394, H-A-431) */ + "class"?: string; /** the merge commit it was last changed at */ "commit": string; /** the requirements the manifest's contributes object names as keys, sorted, its local names not carried; each is also in requires (design 47 G section 709) */ From 7ed668fa53d525027b21b2ddd0b2a1e6f77becd6 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 01:32:06 +0100 Subject: [PATCH 11/15] I7: a catalog entry carries its manifest's class MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gate reads a mover's class from its catalog entry when it opens the delivery record, so the indexer writes the manifest's class there. Builds: design 47 H2 §810; H-A-394 (1), H-A-431 (1) --- .../mesh-delivery-indexer/internal/index/catalog_graph.go | 5 +++-- .../internal/index/catalog_graph_test.go | 8 ++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_graph.go b/modules/mesh-delivery-indexer/internal/index/catalog_graph.go index 906f2263f..93ff2d057 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_graph.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_graph.go @@ -27,13 +27,14 @@ const CatalogEntryLayout = 1 // CatalogEntryOf is the catalog graph entry of one module: the module graph's node of it as a recompute computed it // (its build, provision and verb edges, the verb edges left out on a cycle, ADR 0614), with where the trunk holds it, // its manifest's digest and verdict, the main package its process runs (MainOf), the package versions it builds against -// and what the indexer computes its next revision from (Inputs: the lead's ruling on milestone 47/G finding 1), so that +// and what the indexer computes its next revision from (Inputs: the lead's ruling on milestone 47/G finding 1), and its +// manifest's class, core or leaf, absent for none (design 47 H2 §810, H-A-394 (1): the gate reads it), so that // ModuleValueOf gives the module's value back from the entry alone. The acting rules are the revision's to add. func CatalogEntryOf(module string, cur meshcatalog.ModulesByModuleValue, m catalogue.Manifest, raw string, n graph.Node) meshcatalog.CatalogEntry { e := meshcatalog.CatalogEntry{V: CatalogEntryLayout, Module: module, Repository: cur.Repository, Path: cur.Path, Commit: cur.Commit, Manifest: manifestWhole(raw), ManifestDigest: meshcatalogclient.EntryManifestDigest(cur.Manifest), Verdict: cur.Verdict, Fp: n.FP, None: n.None, Bases: n.Bases, Packages: maps.Clone(cur.Packages), Provides: n.Provides, Requires: n.Requires, Awaits: n.Awaits, Invokes: n.Invokes, - Serves: n.Serves, LeftOut: n.LeftOut, Main: MainOf(m), Contributes: contributesOf(m)} + Serves: n.Serves, LeftOut: n.LeftOut, Main: MainOf(m), Contributes: contributesOf(m), Class: m.Class} // requires names every requirement as catalogue.Manifest.Wants lists them, optional ones and those only contributes names // included, since contributing to a requirement is requiring it (design 47 G §709, A-229). for _, r := range m.Wants() { diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_graph_test.go b/modules/mesh-delivery-indexer/internal/index/catalog_graph_test.go index 36cb65242..69dca5113 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_graph_test.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_graph_test.go @@ -106,7 +106,11 @@ func TestAnEntryNamesWhatItContributesToAndEveryRequirement(t *testing.T) { if n := graph.NodeOf(e); !reflect.DeepEqual(n.Requires, []string{"database", graph.BusState, "route"}) { t.Errorf("the node's requires = %v, want the optional cache left out", n.Requires) } - if e := CatalogEntryOf("kv", meshcatalog.ModulesByModuleValue{}, catalogue.Manifest{}, "", graph.Node{}); e.Contributes != nil { - t.Errorf("an entry of a manifest contributing nothing carries contributes %v", e.Contributes) + if e := CatalogEntryOf("kv", meshcatalog.ModulesByModuleValue{}, catalogue.Manifest{}, "", graph.Node{}); e.Contributes != nil || e.Class != "" { + t.Errorf("an entry of a manifest contributing nothing and naming no class carries contributes %v, class %q", e.Contributes, e.Class) + } + // Design 47 H2 §810, H-A-394 (1): the entry carries its manifest's class, which the gate reads. + if e := CatalogEntryOf("kv", meshcatalog.ModulesByModuleValue{}, catalogue.Manifest{Class: catalogue.ClassCore}, "", graph.Node{}); e.Class != catalogue.ClassCore { + t.Errorf("the entry of a core manifest carries the class %q, want core", e.Class) } } From 40c2870a716cb751395c14579c21b09da1349e57 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 01:35:56 +0100 Subject: [PATCH 12/15] I4: a release re-pins once, at its reservation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The builder's version-reserved writes the series' numbered version with the contents of the module's entry commit, so acceptedSeries and releasedPins pin it; the indexer keeps each reservation in heard (reserved.., reserved-keep) and gathers a revision's reservations until its pipeline's start, its first step-ended or the expiry of settling. (reserved-settle), then keeps one re-pin revision of that branch. The release's package-published moves no fingerprint; a pipeline-ended not passed withdraws its reservations to the newest released version. Builds: design 47 H2 §805; H-A-418, H-A-420, H-A-424, H-A-436 --- .../cmd/mesh-delivery-indexer/main.go | 53 +++ .../cmd/mesh-delivery-indexer/versions.go | 32 ++ .../internal/index/candidates.go | 4 + .../internal/index/packages_values.go | 9 + .../internal/index/pins.go | 15 +- .../internal/index/repin_test.go | 63 +++ .../internal/index/reserved.go | 432 ++++++++++++++++++ .../internal/index/series.go | 18 +- .../internal/index/toolchains.go | 27 +- .../internal/index/versions.go | 5 +- .../mesh-delivery-indexer/manifest_test.go | 8 +- modules/mesh-delivery-indexer/module.json | 20 +- 12 files changed, 669 insertions(+), 17 deletions(-) create mode 100644 modules/mesh-delivery-indexer/internal/index/reserved.go diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index 043bad17d..352505c32 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -92,6 +92,16 @@ func (s awaitedStore) Create(ctx context.Context, key string, value []byte) erro return st.Create(ctx, key, value) } +// CreateTTL creates key living ttl: the state is declared key-ttl (index.TTLStore, design 47 H2 §805). +func (s awaitedStore) CreateTTL(ctx context.Context, key string, value []byte, ttl time.Duration) error { + st, err := s.store() + if err != nil { + return err + } + _, err = st.kv.Create(ctx, key, value, jetstream.KeyTTL(ttl)) + return conflict(err) +} + func (s awaitedStore) Update(ctx context.Context, key string, value []byte, revision uint64) error { st, err := s.store() if err != nil { @@ -346,6 +356,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), BranchHead: branchHeadVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call), Packages: awaitedStore{b: packages, js: js, name: index.Bucket(cred.Module, index.PackagesState)}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call), Heard: heardStore, Runs: awaitedStore{b: runs}, Graph: graph, + // How long a reservation is kept and how long its gathering waits at most (design 47 H2 §805, H-A-424). + ReservedKeep: reservedSetting("reserved-keep", index.DefaultReservedKeep, logger.Printf), + ReservedSettle: reservedSetting("reserved-settle", index.DefaultReservedSettle, logger.Printf), SeatDefinitions: awaitedStore{b: seatDefs}, ReadFile: readFileVia(calls.Call), SaySeatDefinition: saySeats, Revisions: revisions, EntryIndex: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.EntryIndexState), "no entry is found by its fingerprint", logger.Printf)}, // Message a kept call: a read, called again through a transport fault (ADR 0668 §1) // Each catalog revision may bring another holder of a build seat: the build modules are found again (buildmodules.go). @@ -571,6 +584,36 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err := hub.Handle(meshbuilds.SubjectPackagePublished, func(m jetstream.Msg) { heardPackagePublished(ctx, ix, m, logger.Printf) }); err != nil { return err } + // A release re-pins once, at its reservation (design 47 H2 §805): the builder's version-reserved, and the step-ended of + // a pipeline whose reservations settle then. + if err := hub.Handle(meshbuilds.SubjectVersionReserved, func(m jetstream.Msg) { heardVersionReserved(ctx, ix, m, logger.Printf) }); err != nil { + return err + } + if err := hub.Handle(meshbuilds.SubjectStepEndedAll, func(m jetstream.Msg) { heardBuildStepEnded(ctx, ix, m, logger.Printf) }); err != nil { + return err + } + // A gathering's key settling. settles it when its time to live ends: the watch of heard hands the bus's + // expiry marker over (H-A-436); a gathering a restart cut settles once heard is bound. + heardStore.b.Ready(func(kv jetstream.KeyValue) { + eachHeard := func(e jetstream.KeyValueEntry) { + if e.Operation() != jetstream.KeyValuePurge { + return + } + if err := ix.Expired(ctx, e.Key()); err != nil { + logger.Printf("versions: %s expired and its reservations are not re-pinned: %v", e.Key(), err) + } + } + if hw, err := settle(ctx, kv, eachHeard); err != nil { + process.NotWatched(logger.Printf, index.Bucket(cred.Module, index.HeardState), err, "no gathering of reservations settles at its bound until the indexer starts again") + } else { + go func() { errs <- follow(ctx, hw, eachHeard) }() + } + go func() { + if err := ix.GatherKept(ctx); err != nil { + logger.Printf("versions: the reservations kept before the start are not settled again: %v", err) + } + }() + }) if err := hub.Handle(meshbuilds.SubjectToolchainAgreed, func(m jetstream.Msg) { heardToolchainAgreed(ctx, ix, m, logger.Printf) }); err != nil { return err } @@ -641,3 +684,13 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return err } } + +// reservedSetting is the setting key read as a duration (stdio.SettingDuration); one that does not read is said and takes def. +func reservedSetting(key string, def time.Duration, logf func(string, ...any)) time.Duration { + d, err := stdio.SettingDuration(key, def) + if err != nil { + logf("%v: %s is taken", err, def) + return def + } + return d +} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go index 0e60ef7fd..fa9a746b2 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go @@ -98,6 +98,38 @@ func heardPackagePublished(ctx context.Context, ix *index.Indexer, m jetstream.M _ = m.Ack() } +// heardVersionReserved takes one version-reserved of the seat mesh-builds in (design 47 H2 §805): one that does not decode +// is said and acknowledged; one that cannot be kept is left, so the bus gives it again. +func heardVersionReserved(ctx context.Context, ix *index.Indexer, m jetstream.Msg, logf func(string, ...any)) { + var e meshbuilds.VersionReservedEvent + if err := json.Unmarshal(m.Data(), &e); err != nil { + logf("%s holds what is no version-reserved, left: %v", m.Subject(), err) + _ = m.Ack() + return + } + if err := ix.HearVersionReserved(ctx, e); err != nil { + logf("versions: the version-reserved of %s %s could not be kept, so the bus gives it again: %v", e.Artifact, e.Version, err) + leftForBus(m, fmt.Sprintf("the version-reserved of %s %s", e.Artifact, e.Version), err, logf) + return + } + _ = m.Ack() +} + +// heardBuildStepEnded settles the reservations of the step's pipeline, if they gather (design 47 H2 §805, H-A-424 (1)): +// the builder reserves every mover before it asks the first build. A settle that fails is said; its bound settles again. +func heardBuildStepEnded(ctx context.Context, ix *index.Indexer, m jetstream.Msg, logf func(string, ...any)) { + var e meshbuilds.StepEnded + if err := json.Unmarshal(m.Data(), &e); err != nil { + logf("%s holds what is no step-ended, left: %v", m.Subject(), err) + _ = m.Ack() + return + } + if err := ix.Settle(ctx, e.Pipeline); err != nil { + logf("versions: the reservations of %s are not re-pinned at its step-ended: %v", e.Pipeline, err) + } + _ = m.Ack() +} + // heardToolchainAgreed keeps the newest toolchain-agreed of its language (A-68); one that does not decode is said and // acknowledged, one that cannot be kept is left, so the bus gives it again. func heardToolchainAgreed(ctx context.Context, ix *index.Indexer, m jetstream.Msg, logf func(string, ...any)) { diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index 4f4df5137..bb94bccbe 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -192,6 +192,10 @@ type Indexer struct { runMu sync.Mutex // one run event at a time (runcandidates.go) heldFull atomic.Bool // a take-in is held because the catalog store is full: room is said when it has room again (RoomAgain) pendingMu sync.Mutex // one take-in step at a time: a merge, a group's end, the start and a hand act (pending.go) + // ReservedKeep is how long a reservation is kept in the state heard, and ReservedSettle how long after a revision's + // first reservation its gathering settles at the latest: the settings reserved-keep and reserved-settle (design 47 H2 + // §805, H-A-424); zero takes the default. + ReservedKeep, ReservedSettle time.Duration } func (ix *Indexer) logf(format string, args ...any) { diff --git a/modules/mesh-delivery-indexer/internal/index/packages_values.go b/modules/mesh-delivery-indexer/internal/index/packages_values.go index bc511574e..3ac6cbd38 100644 --- a/modules/mesh-delivery-indexer/internal/index/packages_values.go +++ b/modules/mesh-delivery-indexer/internal/index/packages_values.go @@ -178,3 +178,12 @@ func contentsOfPublished(p meshcatalog.PackagesByModuleValue) *meshcatalog.Packa } return c } + +// publishedOfNumbered is a series' numbered version's contents as the value of a per-module packages key (the same fields). +func publishedOfNumbered(c meshcatalog.PackagesByPackageSeriesValueNumberedContents) meshcatalog.PackagesByModuleValue { + p := meshcatalog.PackagesByModuleValue{At: c.At, Commit: c.Commit, Dir: c.Dir, Module: c.Module, Repository: c.Repository, Requires: c.Requires, V: c.V} + for _, d := range c.Packages { + p.Packages = append(p.Packages, meshcatalog.PackagesByModuleValuePackagesItem{Package: d.Package, Digest: d.Digest}) + } + return p +} diff --git a/modules/mesh-delivery-indexer/internal/index/pins.go b/modules/mesh-delivery-indexer/internal/index/pins.go index 92a543d29..309015e2d 100644 --- a/modules/mesh-delivery-indexer/internal/index/pins.go +++ b/modules/mesh-delivery-indexer/internal/index/pins.go @@ -83,7 +83,14 @@ func (ix *Indexer) HeardPins(ctx context.Context, started *runner.PipelineStarte } switch { case started != nil && started.Revision > 0 && started.Pipeline != "": - return ix.pin(ctx, PinKey(started.Branch, started.Revision), PipelineRecord(started.Pipeline)) + if err := ix.pin(ctx, PinKey(started.Branch, started.Revision), PipelineRecord(started.Pipeline)); err != nil { + return err + } + // The reservations of the pipeline's revision settle at its start, if they gather (design 47 H2 §805, H-A-424 (1)). + if err := ix.Settle(ctx, BranchObject(started.Branch, started.Revision)); err != nil { + ix.logf("versions: the reservations of %s are not re-pinned at its pipeline-started: %v", BranchObject(started.Branch, started.Revision), err) + } + return nil case ended != nil && ended.Revision > 0 && ended.Pipeline != "": // A passed pipeline's release pins the revision its movers' entries live in, before the pipeline's own pin is freed, // one record per mover that released a version in it (H-A-168, H-A-170, H-A-173); a mover that released none, and a run @@ -98,6 +105,12 @@ func (ix *Indexer) HeardPins(ctx context.Context, started *runner.PipelineStarte } } } + // A pipeline that did not pass withdraws the reservations its revision holds (design 47 H2 §805, H-A-418 (1)). + if ended.Outcome != runner.PipelineEndedOutcomePassed { + if err := ix.Withdraw(ctx, ended.Branch, ended.Revision); err != nil { + return fmt.Errorf("the reservations of %s are not withdrawn: %w", BranchObject(ended.Branch, ended.Revision), err) + } + } record := PipelineRecord(ended.Pipeline) if err := ix.unpin(ctx, PinKey(ended.Branch, ended.Revision), record); err != nil { return err diff --git a/modules/mesh-delivery-indexer/internal/index/repin_test.go b/modules/mesh-delivery-indexer/internal/index/repin_test.go index 584c37083..c9b58fb78 100644 --- a/modules/mesh-delivery-indexer/internal/index/repin_test.go +++ b/modules/mesh-delivery-indexer/internal/index/repin_test.go @@ -2,11 +2,13 @@ package index import ( "context" + "fmt" "slices" "strings" "testing" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" "git.novox.be/novox/mesh-sdk/go/catalogue" @@ -97,4 +99,65 @@ func TestAReleaseRepinsAnEntryNamingItsPackageWithNoPin(t *testing.T) { said[0].CallKind != meshcatalog.CatalogRevisionEventCallKindRelease || said[1].Module != "mesh-sdk" || said[1].Version != "0.1.27" { t.Errorf("said %+v, want catalog-revision of each branch with the release's call", said) } + + // Design 47 H2 §805, H-A-418, H-A-420, H-A-424, H-A-436: a pipeline reserving twenty packages of mesh-sdk in main's + // revision 2 keeps one re-pin revision of main when its gathering settles, at its first step-ended; its release then + // keeps none on main, and a pipeline that did not pass withdraws its reservations back to the newest released version. + ix.Files = func(_ context.Context, _, _ string, dirs []string) (map[string]map[string]string, error) { + out := map[string]map[string]string{} + for _, d := range dirs { + out[d] = map[string]string{"go.mod": strings.Repeat("d", 40), "sdk.go": strings.Repeat("e", 40)} + } + return out, nil + } + heard := newMem() + ix.Heard = heard + revision := BranchObject("main", 2) + newest := func() int64 { + rev, _, _, err := readNewestOf(ctx, ix.Revisions, "main") + if err != nil { + t.Fatal(err) + } + return rev.Revision + } + for i := range 20 { + e := meshbuilds.VersionReservedEvent{Module: "mesh-sdk", Artifact: fmt.Sprintf("mesh-sdk:p%d", i), Path: "mesh-sdk/go", Version: "0.1.28", Revision: revision} + if err := ix.HearVersionReserved(ctx, e); err != nil { + t.Fatal(err) + } + } + if _, _, open, _ := heard.Get(ctx, SettlingKey(revision)); !open || newest() != 2 { + t.Fatalf("twenty reservations: gathering open %v, main's newest %d, want open and no revision kept before it settles", open, newest()) + } + if err := ix.Settle(ctx, revision); err != nil || newest() != 3 { + t.Fatalf("the gathering settled (%v) keeping main's newest %d, want one re-pin revision, 3", err, newest()) + } + if now, _ := moduleOf(t, mods, "g"); now.Packages["git.novox.be/novox/mesh-sdk/go"] != "v0.1.28" { + t.Errorf("g after the reservations is pinned %v, want v0.1.28", now.Packages) + } + if err := ix.Settle(ctx, revision); err != nil || newest() != 3 { + t.Errorf("a second step-ended settled (%v) keeping main's newest %d, want still 3", err, newest()) + } + if _, err := ix.HearVersion(ctx, HeardVersion{Package: "mesh-sdk/go", GoModule: "git.novox.be/novox/mesh-sdk/go", Version: "0.1.28", Commit: strings.Repeat("1", 40), Published: true}); err != nil { + t.Fatal(err) + } + if _, err := ix.Repin(ctx, "mesh-sdk", "0.1.28"); err != nil || newest() != 3 { + t.Errorf("the release of the reserved 0.1.28 re-pinned (%v) keeping main's newest %d, want no revision more", err, newest()) + } + next := BranchObject("main", 3) + if err := ix.HearVersionReserved(ctx, meshbuilds.VersionReservedEvent{Module: "mesh-sdk", Artifact: "mesh-sdk:p0", Path: "mesh-sdk/go", Version: "0.1.29", Revision: next}); err != nil { + t.Fatal(err) + } + if err := ix.Settle(ctx, next); err != nil || newest() != 4 { + t.Fatalf("the reservation of 0.1.29 settled (%v) keeping main's newest %d, want 4", err, newest()) + } + if err := ix.Withdraw(ctx, "main", 3); err != nil || newest() != 5 { + t.Fatalf("the withdrawal (%v) kept main's newest %d, want 5", err, newest()) + } + if now, _ := moduleOf(t, mods, "g"); now.Packages["git.novox.be/novox/mesh-sdk/go"] != "v0.1.28" { + t.Errorf("g after the withdrawal is pinned %v, want the newest released v0.1.28", now.Packages) + } + if _, _, kept, _ := heard.Get(ctx, ReservedKey(next, "mesh-sdk")); kept { + t.Errorf("the withdrawn reservation is still kept") + } } diff --git a/modules/mesh-delivery-indexer/internal/index/reserved.go b/modules/mesh-delivery-indexer/internal/index/reserved.go new file mode 100644 index 000000000..20540d277 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/reserved.go @@ -0,0 +1,432 @@ +package index + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "maps" + "slices" + "strings" + "time" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-sdk/go/keys" +) + +// A release re-pins once, at its reservation (design 47 H2 §805; H-A-418, H-A-420, H-A-424). The builder says +// version-reserved when it reserves a package's number, naming the revision `.` its pipeline runs. The indexer +// writes the series' numbered version with the contents read at the commit of the module's entry in that revision, so +// acceptedSeries and releasedPins pin it, and keeps the reservation in its state heard under +// `reserved..`, each key living reserved-keep. It gathers a pipeline's reservations per revision and +// writes one re-pin revision of that revision's branch when the gathering settles: at the first pipeline-started or +// step-ended of that pipeline heard after them, or at the expiry of the key settling., written reserved-settle +// after the first. The release's package-published then +// moves no fingerprint. A pipeline-ended not passed whose revision holds reservations withdraws them: the series' +// numbered goes back to its newest released version and the importers are re-pinned to it. + +// ReservedPrefix begins every key of the state heard holding a revision's reservations of one module. +const ReservedPrefix = "reserved." + +// The defaults of the settings reserved-keep and reserved-settle (H-A-424 (1), (2)). +const ( + DefaultReservedKeep = 24 * time.Hour + DefaultReservedSettle = 30 * time.Second +) + +// ReservedKey is the key of module's reservations in revision, `reserved..`, the module escaped as a +// key's name is. +func ReservedKey(revision, module string) string { + return ReservedPrefix + revision + "." + keys.EscapeName(module) +} + +// Reservation is one version-reserved as the indexer keeps it. +type Reservation struct { + Artifact string `json:"artifact"` + Path string `json:"path"` + Version string `json:"version"` + Fingerprint string `json:"fingerprint,omitempty"` +} + +// Reserved is the value of a reserved key: a module's reservations in one revision. +type Reserved struct { + V int `json:"v"` + Revision string `json:"revision"` + Module string `json:"module"` + Reservations []Reservation `json:"reservations"` + At time.Time `json:"at"` +} + +// TTLStore is a Store that writes a new key with its own time to live (a state declared `key-ttl`). +type TTLStore interface { + CreateTTL(ctx context.Context, key string, value []byte, ttl time.Duration) error +} + +// SettlingPrefix begins the key of the state heard that holds a revision's gathering open, `settling.`, written +// with the key-ttl reserved-settle at its first reservation: its expiry, heard through the indexer's watch of heard +// (Expired), settles the gathering; the indexer holds no timer (ADR 0437 §8; H-A-436). +const SettlingPrefix = "settling." + +// SettlingKey is the key holding revision's gathering open. +func SettlingKey(revision string) string { return SettlingPrefix + revision } + +// HearVersionReserved takes one version-reserved in: the series' numbered version, the reservation kept and the revision's +// gathering begun. An error leaves the event for the bus to give again. +func (ix *Indexer) HearVersionReserved(ctx context.Context, e meshbuilds.VersionReservedEvent) error { + if ix.Heard == nil { + return errors.New("the indexer keeps no state " + HeardState) + } + _, ok, err := ix.branchOfRevision(ctx, e.Revision) + if err != nil { + return err + } + if !ok { + ix.logf("versions: the version-reserved of %s %s names the revision %q, which is of no watched branch: left", e.Artifact, e.Version, e.Revision) + return nil + } + if err := ix.keepNumbered(ctx, e); err != nil { + return err + } + if err := ix.keepReservation(ctx, e); err != nil { + return err + } + ix.logf("versions: %s reserved %s %s in %s; its importers are re-pinned once the pipeline's reservations settle", e.Module, e.Path, e.Version, e.Revision) + return ix.gather(ctx, e.Revision) +} + +// branchOfRevision is the watched branch (the trunk among them) whose revision name is revision, `.`. +func (ix *Indexer) branchOfRevision(ctx context.Context, revision string) (string, bool, error) { + branches, err := ix.watchedBranches(ctx) + if err != nil { + return "", false, err + } + if !slices.Contains(branches, TrunkBranch) { + branches = append(branches, TrunkBranch) + } + for _, b := range branches { + if _, ok := BranchRevisionOf(b, revision); ok { + return b, true, nil + } + } + return "", false, nil +} + +// seriesOfPath is the series key and value of the package path names in series: the key whose name or Go module is path. +func (ix *Indexer) seriesOfPath(ctx context.Context, path, series string) (string, SeriesPackage, bool) { + all := ix.SeriesAll(ctx) + for _, key := range slices.Sorted(maps.Keys(all)) { + if s := all[key]; s.Series == series && (s.Name == path || s.GoModule == path) { + return key, s, true + } + } + return SeriesKey(path, series), SeriesPackage{}, false +} + +// keepNumbered writes the reserved version as its series' numbered one, with the contents read at the commit of the +// module's entry in the event's revision (H-A-418 (2)); an older version than the numbered held changes nothing. +func (ix *Indexer) keepNumbered(ctx context.Context, e meshbuilds.VersionReservedEvent) error { + series, ok := SeriesOf(e.Version) + if ix.Packages == nil || e.Path == "" || !ok { + return nil + } + key, s, _ := ix.seriesOfPath(ctx, e.Path, series) + v := SeriesVersion{Version: strings.TrimPrefix(e.Version, "v"), Fingerprint: e.Fingerprint} + if ix.Revisions != nil { + _, entries, err := ReadRevision(ctx, ix.Revisions, e.Revision) + if err != nil { + return fmt.Errorf("the revision %s of %s's reservation: %w", e.Revision, e.Module, err) + } + if i := slices.IndexFunc(entries, func(x meshcatalog.CatalogEntry) bool { return x.Module == e.Module }); i >= 0 { + v.Commit = entries[i].Commit + if s.GoModule != "" && v.Commit != "" && ix.Files != nil { + got, err := ix.packagesAt(ctx, entries[i].Repository, v.Commit, []string{s.GoModule}, nil) + switch p, found := got[s.GoModule]; { + case err != nil: + ix.logf("versions: %s %s's contents at %.12s are not read, so its importers are not pinned to it before its publish: %v", e.Path, e.Version, v.Commit, err) + case !found: + ix.logf("versions: the forge's listing at %.12s holds no Go module %s, so %s's importers are not pinned to it before its publish", v.Commit, s.GoModule, e.Version) + default: + p.At = ptr(ix.now().UTC()) + v.Contents = contentsOfPublished(p) + } + } + } else { + ix.logf("versions: the revision %s holds no entry of %s, so %s %s is numbered with no contents", e.Revision, e.Module, e.Path, e.Version) + } + } + return write(ctx, ix.Packages, key, func(value []byte, found bool) ([]byte, bool, error) { + held := SeriesPackage{V: keys.Version, Name: e.Path, Series: series} + if found { + if err := json.Unmarshal(value, &held); err != nil { + return nil, false, err + } + } + if n := held.Numbered; n != nil && (CompareVersions("v"+n.Version, "v"+v.Version) > 0 || n.Version == v.Version && n.Contents != nil) { + return nil, false, nil + } + held.Numbered = numberedOf(v) + held.At = ptr(ix.now()) + next, err := json.Marshal(held) + return next, false, err + }) +} + +// keepReservation keeps e under its revision and module in the state heard, the key living reserved-keep: a module's +// further reservation in the revision is added to its key, which is written again with its time to live. +func (ix *Indexer) keepReservation(ctx context.Context, e meshbuilds.VersionReservedEvent) error { + key := ReservedKey(e.Revision, e.Module) + r := Reservation{Artifact: e.Artifact, Path: e.Path, Version: e.Version, Fingerprint: e.Fingerprint} + for range writeTries { + value, rev, found, err := ix.Heard.Get(ctx, key) + if err != nil { + return err + } + held := Reserved{V: keys.Version, Revision: e.Revision, Module: e.Module, At: ix.now().UTC()} + if found { + if err := json.Unmarshal(value, &held); err != nil { + return fmt.Errorf("heard.%s: %w", key, err) + } + if slices.Contains(held.Reservations, r) { + return nil + } + } + held.Reservations = append(held.Reservations, r) + data, err := json.Marshal(held) + if err != nil { + return err + } + if found { + if err := ix.Heard.Delete(ctx, key, rev); errors.Is(err, ErrConflict) { + continue + } else if err != nil { + return err + } + } + err = ix.createTTL(ctx, key, data) + if errors.Is(err, ErrConflict) { + continue + } + return err + } + return fmt.Errorf("heard.%s: %w", key, ErrConflict) +} + +// createTTL creates key in the state heard living reserved-keep (createTTLFor). +func (ix *Indexer) createTTL(ctx context.Context, key string, data []byte) error { + keep := ix.ReservedKeep + if keep <= 0 { + keep = DefaultReservedKeep + } + return ix.createTTLFor(ctx, key, data, keep) +} + +// createTTLFor creates key in the state heard living ttl; a store that keeps no time to live per key keeps it with none, +// said. +func (ix *Indexer) createTTLFor(ctx context.Context, key string, data []byte, ttl time.Duration) error { + if st, ok := ix.Heard.(TTLStore); ok { + return st.CreateTTL(ctx, key, data, ttl) + } + ix.logf("versions: the state %s keeps no time to live per key here, so %s is kept with none", HeardState, key) + return ix.Heard.Create(ctx, key, data) +} + +// reservedOf are the reservations revision holds, by module. +func (ix *Indexer) reservedOf(ctx context.Context, revision string) (map[string]Reserved, map[string]uint64, error) { + if ix.Heard == nil { + return nil, nil, nil + } + all, err := ix.Heard.Keys(ctx) + if err != nil { + return nil, nil, err + } + out, revs := map[string]Reserved{}, map[string]uint64{} + prefix := ReservedPrefix + revision + "." + for _, key := range all { + if !strings.HasPrefix(key, prefix) { + continue + } + value, rev, found, err := ix.Heard.Get(ctx, key) + if err != nil { + return nil, nil, err + } + var r Reserved + if !found || json.Unmarshal(value, &r) != nil || r.Revision != revision { + continue + } + out[key], revs[key] = r, rev + } + return out, revs, nil +} + +// gather begins revision's gathering, if none is open: the key settling., living reserved-settle. +func (ix *Indexer) gather(ctx context.Context, revision string) error { + settle := ix.ReservedSettle + if settle <= 0 { + settle = DefaultReservedSettle + } + data, err := json.Marshal(map[string]any{"revision": revision, "at": ix.now().UTC()}) + if err != nil { + return err + } + err = ix.createTTLFor(ctx, SettlingKey(revision), data, settle) + if errors.Is(err, ErrConflict) { + return nil // a gathering is open already + } + return err +} + +// GatherKept settles, at the start, every revision the state heard holds reservations of whose gathering is not open any +// more: a settle cut by a restart is made then (one that moves nothing keeps no revision). An open one settles at its +// expiry or its pipeline's next word. +func (ix *Indexer) GatherKept(ctx context.Context) error { + if ix.Heard == nil { + return nil + } + all, err := ix.Heard.Keys(ctx) + if err != nil { + return err + } + revisions := map[string]bool{} + for _, key := range all { + if !strings.HasPrefix(key, ReservedPrefix) { + continue + } + value, _, found, err := ix.Heard.Get(ctx, key) + var r Reserved + if err != nil || !found || json.Unmarshal(value, &r) != nil || slices.Contains(all, SettlingKey(r.Revision)) { + continue + } + revisions[r.Revision] = true + } + var errs []error + for _, revision := range slices.Sorted(maps.Keys(revisions)) { + errs = append(errs, ix.settleNow(ctx, revision)) + } + return errors.Join(errs...) +} + +// Settle settles revision's gathering, if one is open (a pipeline-started or step-ended of its pipeline): its key goes, and +// every importer of the modules it reserved is re-pinned in one revision of its branch. None open settles nothing. +func (ix *Indexer) Settle(ctx context.Context, revision string) error { + if ix.Heard == nil { + return nil + } + key := SettlingKey(revision) + _, rev, found, err := ix.Heard.Get(ctx, key) + if err != nil || !found { + return err + } + if err := ix.Heard.Delete(ctx, key, rev); errors.Is(err, ErrConflict) { + return nil // settled by another word, or by its expiry + } else if err != nil { + return err + } + return ix.settleNow(ctx, revision) +} + +// Expired takes in a key of the state heard that its time to live ended, as the watch of heard hands it over: a +// gathering's key settles that gathering (H-A-436); any other key is passed over. +func (ix *Indexer) Expired(ctx context.Context, key string) error { + revision, ok := strings.CutPrefix(key, SettlingPrefix) + if !ok { + return nil + } + return ix.settleNow(ctx, revision) +} + +// settleNow re-pins every importer of the modules revision reserved in one revision of its branch. +func (ix *Indexer) settleNow(ctx context.Context, revision string) error { + held, _, err := ix.reservedOf(ctx, revision) + if err != nil || len(held) == 0 { + return err + } + branch, ok, err := ix.branchOfRevision(ctx, revision) + if err != nil || !ok { + return err + } + modules := modulesOf(held) + _, err = ix.repinModules(ctx, branch, modules, reservedCall(revision, len(modules), false)) + return err +} + +// Withdraw takes a pipeline-ended not passed of branch's revision in: each reservation the revision holds is withdrawn, its +// series' numbered set back to the newest released version, its importers re-pinned to that, and its key deleted. +func (ix *Indexer) Withdraw(ctx context.Context, branch string, revision int64) error { + if ix.Heard == nil { + return nil + } + name := BranchObject(branch, revision) + if _, rev, found, err := ix.Heard.Get(ctx, SettlingKey(name)); err == nil && found { + _ = ix.Heard.Delete(ctx, SettlingKey(name), rev) // the gathering is withdrawn with it + } + held, revs, err := ix.reservedOf(ctx, name) + if err != nil || len(held) == 0 { + return err + } + for _, r := range held { + for _, one := range r.Reservations { + if err := ix.withdrawNumbered(ctx, one); err != nil { + return err + } + } + } + modules := modulesOf(held) + if _, err := ix.repinModules(ctx, branch, modules, reservedCall(name, len(modules), true)); err != nil { + return err + } + for key, rev := range revs { + if err := ix.Heard.Delete(ctx, key, rev); err != nil && !errors.Is(err, ErrConflict) { + return err + } + } + ix.logf("versions: the %d reservations of %s are withdrawn, their importers re-pinned to the newest released versions", len(held), name) + return nil +} + +// withdrawNumbered sets the series' numbered version back to its newest released one while it is the version r reserved. +func (ix *Indexer) withdrawNumbered(ctx context.Context, r Reservation) error { + series, ok := SeriesOf(r.Version) + if ix.Packages == nil || !ok { + return nil + } + key, _, found := ix.seriesOfPath(ctx, r.Path, series) + if !found { + return nil + } + return write(ctx, ix.Packages, key, func(value []byte, found bool) ([]byte, bool, error) { + var s SeriesPackage + if !found || json.Unmarshal(value, &s) != nil || s.Numbered == nil || s.Numbered.Version != strings.TrimPrefix(r.Version, "v") { + return nil, false, nil + } + s.Numbered = nil + if s.Newest != nil { + s.Numbered = numberedOf(*s.Newest) + } + s.At = ptr(ix.now()) + next, err := json.Marshal(s) + return next, false, err + }) +} + +// modulesOf are the modules of a revision's reservations, sorted. +func modulesOf(held map[string]Reserved) []string { + var out []string + for _, r := range held { + if !slices.Contains(out, r.Module) { + out = append(out, r.Module) + } + } + slices.Sort(out) + return out +} + +// reservedCall is the call of the re-pin revision of a revision's reservations: a release's re-pin, made at its reservation +// (§805), or at their withdrawal. +func reservedCall(revision string, modules int, withdrawn bool) revisionCall { + call := fmt.Sprintf("release of %s's %d reserved modules", revision, modules) + if withdrawn { + call = fmt.Sprintf("release of %s's %d reserved modules withdrawn", revision, modules) + } + return revisionCall{Call: call, Kind: meshcatalog.CatalogRevisionCallKindRelease} +} diff --git a/modules/mesh-delivery-indexer/internal/index/series.go b/modules/mesh-delivery-indexer/internal/index/series.go index 8b61e4828..c6df3fa9e 100644 --- a/modules/mesh-delivery-indexer/internal/index/series.go +++ b/modules/mesh-delivery-indexer/internal/index/series.go @@ -177,7 +177,8 @@ func acceptedSeries(series map[string]SeriesPackage, m catalogue.Manifest, repos var pins sdkstep.PackageVersions for _, key := range slices.Sorted(maps.Keys(series)) { s := series[key] - if s.GoModule == "" || s.Newest == nil || s.Newest.Contents == nil || m.AcceptedSeries(s.Name) != s.Series { + pinned := pinnedVersion(s) + if s.GoModule == "" || pinned == nil || pinned.Contents == nil || m.AcceptedSeries(s.Name) != s.Series { continue } if repo, _, ok := trunk.PackageRepository(s.GoModule); !ok || trunk.SameRepository(repo, repository) { @@ -186,8 +187,8 @@ func acceptedSeries(series map[string]SeriesPackage, m catalogue.Manifest, repos if out == nil { out, pins = maps.Clone(published), sdkstep.PackageVersions{} } - p := publishedOfContents(*s.Newest.Contents) - version := "v" + s.Newest.Version + p := publishedOfContents(*pinned.Contents) + version := "v" + pinned.Version p.Commit = version out[s.GoModule], pins[s.GoModule] = p, version } @@ -197,6 +198,17 @@ func acceptedSeries(series map[string]SeriesPackage, m catalogue.Manifest, repos return out, pins } +// pinnedVersion is the version of a series its users are pinned to: its numbered one when a reservation wrote it with its +// contents and it is newer than the newest released (design 47 H2 §805, H-A-418 (2), H-A-424 (4)), else the newest; nil +// for none. +func pinnedVersion(s SeriesPackage) *SeriesVersion { + if n := s.Numbered; n != nil && n.Contents != nil && (s.Newest == nil || CompareVersions("v"+s.Newest.Version, "v"+n.Version) < 0) { + p := publishedOfNumbered(*n.Contents) + return &SeriesVersion{Version: n.Version, Commit: n.Commit, Digest: n.Digest, Fingerprint: n.Fingerprint, Contents: contentsOfPublished(p)} + } + return s.Newest +} + // KeepPackageNames writes, at a trunk merge, each package artifact's name in its own ecosystem into its series key // (design 32): a Go package's module path from the go.mod under its paths at the merge commit, a TypeScript package's // package.json name, each read through the forge. The key keeps what it held and gains the name and the module and diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains.go b/modules/mesh-delivery-indexer/internal/index/toolchains.go index 8300429f3..394a2851a 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains.go @@ -645,10 +645,16 @@ func (ix *Indexer) Repin(ctx context.Context, module, version string) (int, erro return named, nil } -// repinBranch re-pins branch's importers of module at version: the trunk in place, another branch as a dry run over its -// revision view, whose scratch goes once it is kept (design 47 H2 §804). It answers how many modules it named. +// repinBranch re-pins branch's importers of module at version (repinModules). func (ix *Indexer) repinBranch(ctx context.Context, branch, module, version string) (int, error) { - x, call := ix, releaseCall(module, version) + return ix.repinModules(ctx, branch, []string{module}, releaseCall(module, version)) +} + +// repinModules re-pins branch's importers of modules in one revision kept with call: the trunk in place, another branch +// as a dry run over its revision view, whose scratch goes once it is kept (design 47 H2 §804, §805). It answers how many +// modules it named. +func (ix *Indexer) repinModules(ctx context.Context, branch string, modules []string, call revisionCall) (int, error) { + x := ix if !isTrunkBranch(branch) { view, nodes, _, err := BranchView(ctx, ix.Revisions, branch) if err != nil { @@ -660,12 +666,19 @@ func (ix *Indexer) repinBranch(ctx context.Context, branch, module, version stri d.ix.Logf = func(format string, args ...any) { ix.logf("re-pin of "+branch+": "+format, args...) } x = d.ix } - only, err := x.repinned(ctx, module) - if err != nil || len(only) == 0 { - return 0, err + only := map[string]bool{} + for _, module := range modules { + some, err := x.repinned(ctx, module) + if err != nil { + return 0, err + } + maps.Copy(only, some) + } + if len(only) == 0 { + return 0, nil } x.recompute.Lock() - err = x.recomputeLocked(ctx, nil, nil, "", only) + err := x.recomputeLocked(ctx, nil, nil, "", only) x.recompute.Unlock() if err != nil { return len(only), err diff --git a/modules/mesh-delivery-indexer/internal/index/versions.go b/modules/mesh-delivery-indexer/internal/index/versions.go index 52a08b239..71336e1f3 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions.go +++ b/modules/mesh-delivery-indexer/internal/index/versions.go @@ -395,10 +395,11 @@ func releasedPins(versions sdkstep.PackageVersions, series map[string]SeriesPack } out[p] = "" for _, s := range series { - if s.GoModule != p || s.Newest == nil || s.Series != m.AcceptedSeries(s.Name) { + pinned := pinnedVersion(s) + if s.GoModule != p || pinned == nil || s.Series != m.AcceptedSeries(s.Name) { continue } - v := strings.TrimPrefix(s.Newest.Version, "v") + v := strings.TrimPrefix(pinned.Version, "v") if _, ok := versionParts(v); ok && (out[p] == "" || compareVersions(v, strings.TrimPrefix(out[p], "v")) > 0) { out[p] = "v" + v } diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index ac6d421df..acdaa0f5b 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -72,6 +72,7 @@ type manifest struct { Scope string `json:"scope"` Serves []string `json:"serves"` } `json:"claims"` + Config map[string]any `json:"config"` // the settings (design 47 H2 §805) Tools []string `json:"tools"` Emits []string `json:"emits"` Consumes []string `json:"consumes"` @@ -133,8 +134,9 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build if !slices.Equal(m.Uses, []string{"operator-channel"}) || !slices.Contains(m.Invokes, "seat:mesh-module-assigner.assignments") || !slices.Contains(m.Invokes, "seat:mesh-module-assigner.placement") { t.Errorf("uses = %v, invokes = %v, want the seat operator-channel used and seat:mesh-module-assigner.assignments invoked", m.Uses, m.Invokes) } - // 1126 note 76: heard keeps the last 64 FORGE messages the indexer handled, for indexer_heard. - if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "predefined", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "oldsections", History: 1}, {Name: index.RunsState, History: 1}, {Name: index.PackagesState, History: index.PackagesHistory}, + // 1126 note 76: heard keeps the last 64 FORGE messages the indexer handled, for indexer_heard; 8 MiB, each reservation + // and each gathering of them a key of its own time to live (design 47 H2 §805, H-A-420 (3), H-A-436). + if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64, MaxMiB: 8, KeyTTL: true}, {Name: "manifests", History: 1}, {Name: "predefined", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "oldsections", History: 1}, {Name: index.RunsState, History: 1}, {Name: index.PackagesState, History: index.PackagesHistory}, {Name: views.MarksState, History: 1}, {Name: index.FindingsState, History: 5}, {Name: views.StatusState, History: 1}, {Name: index.RequirementsState, History: 5}, {Name: index.SeatDefinitionsState, History: 5}, {Name: index.EntryIndexState, History: 1, MaxMiB: 8}, {Name: index.NamedState, History: 1, MaxMiB: 4}, // design 47 G §695 @@ -173,7 +175,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // "Reads FORGE" is a consumes of the indexer's stream: the broker grants only buckets by reads (mesh-control // internal/broker/state.go:62-93). // The registry seats' published fill each (§413: no consume of version.numbered) package's series (DLV-I5, design 51 *The versions*). - if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", "mesh-catalog.room", "mesh-conditions.findings-asked", + if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-builds.package-published", "mesh-builds.step-ended", "mesh-builds.toolchain-agreed", "mesh-builds.version-reserved", "mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", "mesh-catalog.room", "mesh-conditions.findings-asked", "mesh-delivery.source-moved", "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.run-started", "mesh-wants.pipeline-wanted", "mesh-wants.wait-ended", "mesh-wants.wanted", "node-build-go.step-ended", "node-engine.running", "npm-package-registry.published", "operator-channel.decided", "seat:mesh-module-assigner.views.assignments", "seat:mesh-module-assigner.views.seat-holders"}; !reflect.DeepEqual(sorted(m.Consumes), want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index b6a7943b6..dc1ad3aac 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -82,6 +82,8 @@ "seat:mesh-module-assigner.views.assignments", "seat:mesh-module-assigner.views.seat-holders", "mesh-builds.package-published", + "mesh-builds.version-reserved", + "mesh-builds.step-ended", "mesh-builds.toolchain-agreed", "node-build-go.step-ended", "mesh-wants.wait-ended", @@ -98,9 +100,25 @@ "operator-channel.decided", "npm-package-registry.published" ], + "config": { + "reserved-keep": { + "type": "string", + "description": "how long the indexer keeps a pipeline's reservation of a package's version in its state heard (reserved..), a Go duration (design 47 H2 §805, H-A-424 (2)).", + "trusted": false, + "default": "24h", + "runtime": true + }, + "reserved-settle": { + "type": "string", + "description": "how long after a revision's first reservation its importers are re-pinned at the latest, when no pipeline-started or step-ended of its pipeline came first, a Go duration (design 47 H2 §805, H-A-424 (1), H-A-436).", + "trusted": false, + "default": "30s", + "runtime": true + } + }, "state": [ { "name": "candidates", "history": 10 }, - { "name": "heard", "history": 64 }, + { "name": "heard", "history": 64, "max-mib": 8, "key-ttl": true }, { "name": "manifests", "history": 1 }, { "name": "predefined", "history": 5 }, { "name": "taken", "history": 1 }, From ba54f82a701d0c957db9909ba2b83b4356c32bb6 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 01:49:26 +0100 Subject: [PATCH 13/15] I6 (1): the judge refuses an uncapped delivery store and a seat claim with no class MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A delivery module (one claiming a seat of the delivery system, read from the seat list's definitions) is refused for a state, log or objects entry with no cap, in its old section or in requires; a seat-claiming manifest names class core or leaf; finding-events is 8 MiB, 7 days; the objects template takes max-age-days. internal/beside says a group's check reads the catalog member's head beside it, main only when the catalog is not in the group. Red until the captured catalog is refreshed (I6 (5)): judge TestRulesSecrets/EveryCatalogueIdentityFitsWhatItRequires, TestRulesSecrets/ParseManifestForInputsReadsAValueFallingBackToAPredefinedKey, TestRulesPlacement/TheCataloguesBarPlacesBlocksAndItsShapeCoversEveryShow, TestRulesBus/NoCatalogueManifestNamesAnInstallation; internal/index TestAnArtifactOfNoToolchainHasNoLanguageAndNoLock, TestAMergedManifestIsKeptOnceByItsDigestAndNamedByTheModule, TestAModuleWithNoToolchainArtifactGetsItsFingerprintAtOnce, TestAManifestFallingBackToAPredefinedKeyIsRead (each a captured catalog manifest with no class, or finding-events at 16). Builds: design 47 H2 §814, §820, §783; H-A-394 (1), H-A-411 (1) (4), H-A-415 (1), H-A-428, H-A-430 (3), H-A-434 --- .../internal/beside/beside.go | 5 +- .../internal/index/inputs_test.go | 2 + .../internal/index/toolchains_test.go | 2 +- .../mesh-delivery-indexer/judge/busstate.go | 8 +- .../judge/busstate_test.go | 3 +- .../judge/g_words_test.go | 20 ++++- modules/mesh-delivery-indexer/judge/gwords.go | 7 +- modules/mesh-delivery-indexer/judge/judge.go | 1 + .../mesh-delivery-indexer/judge/judge_test.go | 4 +- modules/mesh-delivery-indexer/judge/rules.go | 2 + .../mesh-delivery-indexer/judge/rules_test.go | 58 +++++++------- modules/mesh-delivery-indexer/judge/stores.go | 79 +++++++++++++++++++ 12 files changed, 148 insertions(+), 43 deletions(-) create mode 100644 modules/mesh-delivery-indexer/judge/stores.go diff --git a/modules/mesh-delivery-indexer/internal/beside/beside.go b/modules/mesh-delivery-indexer/internal/beside/beside.go index d1cd8ffdd..dbc5d3250 100644 --- a/modules/mesh-delivery-indexer/internal/beside/beside.go +++ b/modules/mesh-delivery-indexer/internal/beside/beside.go @@ -21,8 +21,9 @@ // read mesh-sdk and mesh-catalog. None reads mesh-controller, which ended by its unassignment (novox/hq // ADR 0590). In a mesh where either module has no source repository nothing is // cloned, and these tests fail loudly with "not beside this check": a cause in the setup, not in the -// change. And in a delivery group that holds a mesh-catalog pull request, these tests read the -// catalogue's main, not the group's head. +// change. In a delivery group that holds a mesh-catalog member, these tests read that member's head, the +// sibling the group's check places in MESH_CHECK_BESIDE, and the catalogue's main only when the catalogue is +// not in the group: a group is judged against its own heads (ADR 0573 §2; design 47 H2 §814, H-A-428). // - Anywhere else, the test judges against the copy captured in this repository's testdata/beside, at the // commit testdata/beside/CAPTURED names. Never against a developer's own checkout: to judge one, set // MESH_CHECK_BESIDE to the directory holding it, as the check does. diff --git a/modules/mesh-delivery-indexer/internal/index/inputs_test.go b/modules/mesh-delivery-indexer/internal/index/inputs_test.go index cfd34caaa..5b1dd2af0 100644 --- a/modules/mesh-delivery-indexer/internal/index/inputs_test.go +++ b/modules/mesh-delivery-indexer/internal/index/inputs_test.go @@ -160,7 +160,9 @@ func TestAContextIsNotPinned(t *testing.T) { // 1450 (the rule 1444 keeps: conte // build-agent's manifest from mesh-controller at f2d7bc94 has an image artifact built from a context in another repository. // Its claim of the retired seat node-build-agent is read as a claim of a build seat: the parser refuses a retired seat // since R4g (mesh-sdk A9a), and the claim is not what this test is about. + // It names no class, which a seat-claiming manifest names since design 47 H2 §814: given here. raw := strings.ReplaceAll(fixture(t, "controller-f2d7bc94-build-agent"), `"node-build-agent"`, `"node-build-go"`) + raw = strings.Replace(raw, `"claims"`, `"class": "core", "claims"`, 1) in := inputsOf("build-agent", raw, "t", nil, noBases) if in.Kept == nil || in.None2 != None2Context { t.Errorf("kept %+v none2 %q, want none2 %q", in.Kept, in.None2, None2Context) diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go index 059aa977b..5daee1f26 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go @@ -208,7 +208,7 @@ func theBuildModuleOfALanguageIsTheHolderOfItsBuildSeat(t *testing.T) { t.Errorf("two seats building go and two holders of node-build-node: %+v %q, want none, both said", by, problems) } // The indexer finds them among the manifests it holds, and a report of a module that holds no build seat is left out. - builder := `{"module":"go-builds","version":"1","claims":[{"name":"node-build-go"}],"streams":[{"name":"toolchains-go","events":["toolchain.**"]}]}` + builder := `{"module":"go-builds","version":"1","class":"core","claims":[{"name":"node-build-go"}],"streams":[{"name":"toolchains-go","events":["toolchain.**"]}]}` ix, _ := newFormIndexer(t, mod{"avahi", catalogueManifest(t, "avahi")}, mod{"go-builds", builder}) ix.UseBuildModules(nil) got, err := ix.BuildModulesNow(context.Background(), defs) diff --git a/modules/mesh-delivery-indexer/judge/busstate.go b/modules/mesh-delivery-indexer/judge/busstate.go index ef911ecdf..b287c2676 100644 --- a/modules/mesh-delivery-indexer/judge/busstate.go +++ b/modules/mesh-delivery-indexer/judge/busstate.go @@ -87,10 +87,12 @@ var busTemplates = map[string]busTemplate{ }, catalogue.TemplateObjects: { storage: catalogue.StorageObjectStore, - fixed: "what happens when full (a new object is refused) and that objects are kept for good", + fixed: "what happens when full (a new object is refused)", + // max-age-days is how long the bus keeps an object; absent, objects are kept for good (design 47 H2 §820; H-A-434). options: map[string]optionRule{ - catalogue.OptionCapMiB: {kind: optionInt, least: catalogue.ObjectsLeastMiB, most: catalogue.ObjectsMostMiB, required: true}, - catalogue.OptionGrant: {kind: optionText}, + catalogue.OptionCapMiB: {kind: optionInt, least: catalogue.ObjectsLeastMiB, most: catalogue.ObjectsMostMiB, required: true}, + catalogue.OptionAgeDays: {kind: optionInt, least: 1, most: 3650}, + catalogue.OptionGrant: {kind: optionText}, }, }, catalogue.TemplateLedger: { diff --git a/modules/mesh-delivery-indexer/judge/busstate_test.go b/modules/mesh-delivery-indexer/judge/busstate_test.go index 8616e4253..6c526d7b9 100644 --- a/modules/mesh-delivery-indexer/judge/busstate_test.go +++ b/modules/mesh-delivery-indexer/judge/busstate_test.go @@ -27,7 +27,7 @@ func TestBusStateIsJudgedOnce(t *testing.T) { // 1966/AC8: a state, a log, objects and a ledger within bounds {"within bounds", manifest(`{"name":"cache","storage":"key-value","template":"state","options":{"history":8,"key-ttl":true}},` + `{"name":"audit","storage":"stream","template":"log","options":{"max-mib":64,"max-age-days":365}},` + - `{"name":"files","storage":"object-store","template":"objects","options":{"max-mib":2048}},` + ledger), nil}, + `{"name":"files","storage":"object-store","template":"objects","options":{"max-mib":2048,"max-age-days":7}},` + ledger), nil}, // Design 47: every kept data may say its form and its catch-up tool, in an old section and in `requires` by template, // and is accepted as read; one saying neither reads as before. {"a form and a catch-up tool on every kept kind", `{"module":"a","version":"1.0","tools":["a_take_in"],` + @@ -60,6 +60,7 @@ func TestBusStateIsJudgedOnce(t *testing.T) { {"deliveries 11", manifest(`{"name":"x","storage":"stream","template":"work-queue","options":{"events":["job"],"max-deliveries":11}}`), []string{"\"x\"", "\"max-deliveries\" is 11", "1 to 10"}}, {"duplicates past the age", manifest(`{"name":"x","storage":"stream","template":"work-queue","options":{"events":["job"],"max-age-days":1,"duplicates":86401}}`), []string{"\"x\"", "\"duplicates\"", "longer than the queue's age of 1 days"}}, {"log age 3651", manifest(`{"name":"x","storage":"stream","template":"log","options":{"max-age-days":3651}}`), []string{"\"x\"", "\"max-age-days\" is 3651"}}, + {"objects age 3651 (H2 §820)", manifest(`{"name":"x","storage":"object-store","template":"objects","options":{"max-mib":64,"max-age-days":3651}}`), []string{"\"x\"", "\"max-age-days\" is 3651"}}, {"objects without a cap", manifest(`{"name":"x","storage":"object-store","template":"objects"}`), []string{"\"x\"", "\"max-mib\" is missing"}}, {"when full given as an option", manifest(`{"name":"x","storage":"stream","template":"log","options":{"when-full":"drop-oldest"}}`), []string{"\"x\"", "\"when-full\"", "fixed"}}, // A-277: a state's max-mib is an option of its template; an approvals state is §677's (key-ttl, 48 hours, at most 64 MiB). diff --git a/modules/mesh-delivery-indexer/judge/g_words_test.go b/modules/mesh-delivery-indexer/judge/g_words_test.go index 7e74fe5a8..6f08177e6 100644 --- a/modules/mesh-delivery-indexer/judge/g_words_test.go +++ b/modules/mesh-delivery-indexer/judge/g_words_test.go @@ -2,6 +2,7 @@ package judge import ( "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/seatdef" "regexp" "strings" "testing" @@ -9,6 +10,9 @@ import ( // Design 47 G §368, §386, §387, §89: the manifest validator reads the manifests' new words before any manifest uses them. func TestTheJudgeReadsMilestoneGsWords(t *testing.T) { + was := deliverySeats.Load() + useDeliverySeats([]seatdef.Definition{{Seat: "thing", Domain: deliveryDomain}}) + t.Cleanup(func() { deliverySeats.Store(was) }) for _, row := range []struct { name string raw string @@ -33,12 +37,24 @@ func TestTheJudgeReadsMilestoneGsWords(t *testing.T) { `"streams":[{"name":"a-findings","events":["finding-written"],"retention":"history"}],"consumes":["mesh-conditions.findings-asked"]}`, want: "old sections"}, {name: "another part's findings in its requires (§529)", raw: `{"module":"a","version":"1.0","consumes":["mesh-conditions.findings-asked"],"requires":[` + `{"name":"findings","storage":"key-value","template":"state"},` + - `{"name":"finding-events","storage":"stream","template":"events","options":{"events":["finding-written.*","finding-cleared.*","findings-stated.*"],"max-age-days":7,"max-mib":16}}]}`}, + `{"name":"finding-events","storage":"stream","template":"events","options":{"events":["finding-written.*","finding-cleared.*","findings-stated.*"],"max-age-days":7,"max-mib":8}}]}`}, {name: "another part's findings without the finding events (§529)", raw: `{"module":"a","version":"1.0","consumes":["mesh-conditions.findings-asked"],"requires":[` + `{"name":"findings","storage":"key-value","template":"state"}]}`, want: "finding-events"}, {name: "another part's finding events not kept 7 days (§529)", raw: `{"module":"a","version":"1.0","consumes":["mesh-conditions.findings-asked"],"requires":[` + `{"name":"findings","storage":"key-value","template":"state"},` + - `{"name":"finding-events","storage":"stream","template":"events","options":{"events":["finding-written.*","finding-cleared.*","findings-stated.*"],"max-age-days":30,"max-mib":16}}]}`, want: "max-age-days"}, + `{"name":"finding-events","storage":"stream","template":"events","options":{"events":["finding-written.*","finding-cleared.*","findings-stated.*"],"max-age-days":30,"max-mib":8}}]}`, want: "max-age-days"}, + {name: "another part's finding events of 16 MiB (H2 §820, H-A-411 (1))", raw: `{"module":"a","version":"1.0","consumes":["mesh-conditions.findings-asked"],"requires":[` + + `{"name":"findings","storage":"key-value","template":"state"},` + + `{"name":"finding-events","storage":"stream","template":"events","options":{"events":["finding-written.*","finding-cleared.*","findings-stated.*"],"max-age-days":7,"max-mib":16}}]}`, want: "max-mib"}, + {name: "a delivery module's uncapped old-section state (H2 §814)", raw: `{"module":"d","version":"1.0","class":"core","claims":[{"name":"thing","scope":"mesh"}],"state":["s"]}`, want: `state "s" in its old section names none`}, + {name: "a delivery module's uncapped state in requires (H2 §814)", raw: `{"module":"d","version":"1.0","class":"core","claims":[{"name":"thing","scope":"mesh"}],` + + `"requires":[{"name":"s","storage":"key-value","template":"state"}]}`, want: `state "s" in ` + "`requires`"}, + {name: "a delivery module's uncapped objects (H2 §814)", raw: `{"module":"d","version":"1.0","class":"core","claims":[{"name":"thing","scope":"mesh"}],"objects":[{"name":"o"}]}`, want: `objects "o"`}, + {name: "a foundation part's capped old-section state (H2 §814)", raw: `{"module":"mesh-delivery-builder","version":"1.0","class":"core","claims":[{"name":"thing","scope":"mesh"}],"state":[{"name":"s","max-mib":8}]}`}, + {name: "an uncapped state of a module claiming no delivery seat (H2 §814)", raw: `{"module":"a","version":"1.0","state":["s"]}`}, + {name: "a seat-claiming manifest with no class (H2 §810, §814)", raw: `{"module":"d","version":"1.0","claims":[{"name":"thing","scope":"mesh"}]}`, want: "no `class`"}, + {name: "a seat-claiming manifest of another class (H2 §810, §814)", raw: `{"module":"d","version":"1.0","class":"middle","claims":[{"name":"thing","scope":"mesh"}]}`, want: `class "middle"`}, + {name: "a seat-claiming leaf (H2 §810)", raw: `{"module":"d","version":"1.0","class":"leaf","claims":[{"name":"thing","scope":"mesh"}]}`}, } { module := "a" if m := regexp.MustCompile(`"module":"([a-z-]+)"`).FindStringSubmatch(row.raw); m != nil { diff --git a/modules/mesh-delivery-indexer/judge/gwords.go b/modules/mesh-delivery-indexer/judge/gwords.go index a1fe4b7b1..dc5c9c34c 100644 --- a/modules/mesh-delivery-indexer/judge/gwords.go +++ b/modules/mesh-delivery-indexer/judge/gwords.go @@ -88,7 +88,8 @@ func gWordsProblems(m catalogue.Manifest) []string { } // findingEventsProblems judges the entry `finding-events` of a part that is no foundation part: it lists the three finding -// events in its option `events` and keeps them 7 days and 16 MiB (design 47 G §529). +// events in its option `events` and keeps them 7 days and 8 MiB (design 47 G §529, H2 §820; H-A-411 (1)); the template +// `events` drops the oldest when full. func findingEventsProblems(module string, e catalogue.BusStateEntry) []string { var problems []string var listed []string @@ -100,9 +101,9 @@ func findingEventsProblems(module string, e catalogue.BusStateEntry) []string { problems = append(problems, fmt.Sprintf("%s's %s does not list the event %q in its `events` (design 47 G §529)", module, FindingEventsEntry, want)) } } - for option, want := range map[string]string{"max-age-days": "7", "max-mib": "16"} { + for option, want := range map[string]string{"max-age-days": "7", "max-mib": "8"} { if got := strings.TrimSpace(string(e.Options[option])); got != want { - problems = append(problems, fmt.Sprintf("%s's %s sets %s to %q, not %s (design 47 G §529)", module, FindingEventsEntry, option, got, want)) + problems = append(problems, fmt.Sprintf("%s's %s sets %s to %q, not %s (design 47 G §529, H2 §820)", module, FindingEventsEntry, option, got, want)) } } slices.Sort(problems) diff --git a/modules/mesh-delivery-indexer/judge/judge.go b/modules/mesh-delivery-indexer/judge/judge.go index bd352368c..42a2b4984 100644 --- a/modules/mesh-delivery-indexer/judge/judge.go +++ b/modules/mesh-delivery-indexer/judge/judge.go @@ -290,6 +290,7 @@ func UseSeatList(defs []seatdef.Definition) { seatsMu.Lock() // not while a pass judges with seats put over the list (WithSeats) defer seatsMu.Unlock() catalogue.UseSeats(catalogue.SeatsFromDefinitions(defs)) + useDeliverySeats(defs) } // TrunkCase is the case of a request holding the catalogue as the trunk holds it (the checker's JudgeTrunk). diff --git a/modules/mesh-delivery-indexer/judge/judge_test.go b/modules/mesh-delivery-indexer/judge/judge_test.go index 58f7c1587..4113c30e9 100644 --- a/modules/mesh-delivery-indexer/judge/judge_test.go +++ b/modules/mesh-delivery-indexer/judge/judge_test.go @@ -43,7 +43,7 @@ func TestTheJudge(t *testing.T) { {name: "a claimed seat's module with tools of its own", raws: map[string]string{ "def": `{"module":"def","version":"1.0","seats":[{"name":"enrolment","scope":"mesh","serves":["issue","again"]}]}`, - "enrolment": `{"module":"enrolment","version":"1.0","claims":[{"name":"enrolment","scope":"mesh","serves":["issue","again"]}],` + + "enrolment": `{"module":"enrolment","version":"1.0","class":"core","claims":[{"name":"enrolment","scope":"mesh","serves":["issue","again"]}],` + `"tools":["enrolment_first_views","enrolment_first_view_again"]}`}}, } { raws := map[string][]byte{"node-engine": []byte(engineManifest("node-engine"))} // a revision has one claim of the seat node-engine (§527) @@ -185,7 +185,7 @@ func engineManifest(module string) string { for _, v := range seat.Serves { verbs = append(verbs, `"`+v.Name+`"`) } - return `{"module":"` + module + `","version":"1.0","claims":[{"name":"node-engine","scope":"node","serves":[` + strings.Join(verbs, ",") + `]}]}` + return `{"module":"` + module + `","version":"1.0","class":"core","claims":[{"name":"node-engine","scope":"node","serves":[` + strings.Join(verbs, ",") + `]}]}` } // Design 47 G §508, §527: one module of a revision claims the seat node-engine; a second is refused naming both, and none diff --git a/modules/mesh-delivery-indexer/judge/rules.go b/modules/mesh-delivery-indexer/judge/rules.go index d675cb5f6..9aef082ca 100644 --- a/modules/mesh-delivery-indexer/judge/rules.go +++ b/modules/mesh-delivery-indexer/judge/rules.go @@ -2599,6 +2599,8 @@ func parseManifest(raw []byte, predefined catalogue.PredefinedList) (catalogue.M m.SetPredefined(predefined) problems := gWordsProblems(m) + problems = append(problems, deliveryStoreProblems(m)...) + problems = append(problems, classProblems(m)...) if !name.MatchString(m.Module) { problems = append(problems, fmt.Sprintf( "%q is not a usable module name: lower-case letters, digits, dashes and dots", m.Module)) diff --git a/modules/mesh-delivery-indexer/judge/rules_test.go b/modules/mesh-delivery-indexer/judge/rules_test.go index 8d6a0a41e..a82cdca43 100644 --- a/modules/mesh-delivery-indexer/judge/rules_test.go +++ b/modules/mesh-delivery-indexer/judge/rules_test.go @@ -261,7 +261,7 @@ func caseAClaimOnASeatAnotherModuleDeclaresIsAccepted(t *testing.T) { // would say it where no consumer hears it, so the manifest is refused, naming the seat's subject; the seat's // holder saying nothing of its own is accepted. holder := func(emits string) []byte { - return []byte(`{"module":"mesh-self-check","version":"1.0","emits":` + emits + `,"claims":[{"name":"mesh-self-check","scope":"mesh","serves":["doctor","probes"]}]}`) + return []byte(`{"module":"mesh-self-check","version":"1.0","emits":` + emits + `,"class":"core","claims":[{"name":"mesh-self-check","scope":"mesh","serves":["doctor","probes"]}]}`) } if _, err := ParseManifest(holder(`["checked"]`)); err == nil || !strings.Contains(err.Error(), "mesh.seat.mesh-self-check.event.checked") { t.Errorf("a seat's event listed as the module's own: %v", err) @@ -383,7 +383,7 @@ func caseAContributionPlaceholderOutsideTheHolderIsRefused(t *testing.T) { if _, err := ParseManifest([]byte(raw)); err == nil || !strings.Contains(err.Error(), "laptop does not claim node-hotkeys") { t.Errorf("a placeholder outside the holder was accepted: %v", err) } - raw = `{"module":"triggerhappy","claims":[{"name":"node-hotkeys"}],"resources":[{"id":"t","type":"file","path":"/etc/t","content":"${contribution:node-hotkeys:keys}"}]}` + raw = `{"module":"triggerhappy","class":"core","claims":[{"name":"node-hotkeys"}],"resources":[{"id":"t","type":"file","path":"/etc/t","content":"${contribution:node-hotkeys:keys}"}]}` if _, err := ParseManifest([]byte(raw)); err == nil || !strings.Contains(err.Error(), "it receives trigger") { t.Errorf("a placeholder for a kind the seat does not receive was accepted: %v", err) } @@ -508,7 +508,7 @@ func caseADeliveringSeatIsOnlyHeldByAModuleThatProvides(t *testing.T) { // would be the answer anyway, and every consumer would be sent to it. // And refused at registration, where the seat set is the store's: what a seat delivers is // data, so a compiled copy of it may not be what refuses a build (novox/hq ADR 0122). - raw := []byte(`{"module":"thing","version":"1.0","claims":[{"name":"git","scope":"mesh"}]}`) + raw := []byte(`{"module":"thing","version":"1.0","class":"core","claims":[{"name":"git","scope":"mesh"}]}`) m, err := ParseManifest(raw) if err != nil { t.Fatalf("the parser judged what a seat delivers: %v", err) @@ -723,7 +723,7 @@ func caseAHolderOfNodeFilesDeclaringRunsAsIsRefused(t *testing.T) { // carry — as the operator's account — is refused at parse. func caseAHolderOfNodeReadsWithoutAnAccountOfItsOwnIsRefused(t *testing.T) { _, err := ParseManifest([]byte(`{"module": "reads", "version":"1.0", - "claims": [{"name": "node-reads", "scope": "node"}]}`)) + "class":"core","claims":[{"name": "node-reads", "scope": "node"}]}`)) if err == nil || !strings.Contains(err.Error(), "runs-as") || !strings.Contains(err.Error(), "0310") { t.Errorf("a holder of node-reads without runs-as was not refused for it: %v", err) } @@ -745,7 +745,7 @@ func caseAHolderPlacesEveryCombinationOnceWithATemplateThatRendersEveryExample(t if template != "" { renders = `,"renders":{"block":` + string(tb) + `}` } - return `{"module":"a-bar","claims":[{"name":"node-bar"` + renders + `}],"resources":[` + strings.Join(res, ",") + `]}` + return `{"module":"a-bar","class":"core","claims":[{"name":"node-bar"` + renders + `}],"resources":[` + strings.Join(res, ",") + `]}` } all := "${contribution:node-bar:block:bar=bottom,place=resources}${contribution:node-bar:block:bar=bottom,place=status}" + "${contribution:node-bar:block:bar=top,place=resources}${contribution:node-bar:block:bar=top,place=status}" @@ -1304,7 +1304,7 @@ func caseAModuleDeclaresItsOwnSeatAndHoldsIt(t *testing.T) { // what it means (novox/hq ADR 0121). But it may not define one in the mesh's own namespace. func caseAModuleDefinesAndClaimsItsOwnSeat(t *testing.T) { ok := []byte(`{"module":"showcase","version":"1.0","seats":[{"name":"the-showcase","scope":"node"}],` + - `"claims":[{"name":"the-showcase","scope":"node"}]}`) + `"class":"core","claims":[{"name":"the-showcase","scope":"node"}]}`) if _, err := ParseManifest(ok); err != nil { t.Fatalf("a module could not define and claim its own seat: %v", err) } @@ -1321,7 +1321,7 @@ func caseAModuleDefinesAndClaimsItsOwnSeat(t *testing.T) { } // A module may not carve its seat out of the mesh's own namespace. bad := []byte(`{"module":"x","version":"1.0","seats":[{"name":"node-mine","scope":"node"}],` + - `"claims":[{"name":"node-mine","scope":"node"}]}`) + `"class":"core","claims":[{"name":"node-mine","scope":"node"}]}`) if _, err := ParseManifest(bad); err == nil || !strings.Contains(err.Error(), "own namespace") { t.Fatalf("a module defined a seat in the mesh's namespace and was not refused: %v", err) } @@ -2266,14 +2266,14 @@ func caseAnUnknownPlaceholderKeyIsRefused(t *testing.T) { {"${shell:tcsh:first}", "names ${shell:tcsh:first}; shell code is ${shell::}"}, {"${shell:zsh}", "names ${shell:zsh}; shell code is ${shell::}"}, } { - raw := `{"module":"holder","claims":[{"name":"node-environment","scope":"node"},{"name":"node-login-shell","scope":"node"}],` + + raw := `{"module":"holder","class":"core","claims":[{"name":"node-environment","scope":"node"},{"name":"node-login-shell","scope":"node"}],` + `"resources":[{"id":"rc","type":"file","path":"/etc/rc","content":"` + c.content + `"}]}` if _, err := ParseManifest([]byte(raw)); err == nil || !strings.Contains(err.Error(), c.want) { t.Errorf("%s was accepted: %v", c.content, err) } } // And outside a file's content, where nothing could be placed. - raw := `{"module":"holder","claims":[{"name":"node-environment","scope":"node"}],` + + raw := `{"module":"holder","class":"core","claims":[{"name":"node-environment","scope":"node"}],` + `"resources":[{"id":"rc","type":"file","path":"/etc/${environment:posix}","content":"x"}]}` if _, err := ParseManifest([]byte(raw)); err == nil || !strings.Contains(err.Error(), "names ${environment:posix} in its path; the environment and the shell's code are placed only in a file's content") { @@ -2408,7 +2408,7 @@ func caseAPackageParsesLikeABundleAndNeedsALanguage(t *testing.T) { } { raw := `{"module":"a","version":"1.0","build":{"artifacts":[` + c.artifact + `]}}` if before, ok := strings.CutSuffix(c.artifact, claims); ok { - raw = `{"module":"a","version":"1.0","claims":[{"name":"mesh-vault","scope":"mesh"}],"build":{"artifacts":[` + before + `]}}` + raw = `{"module":"a","version":"1.0","class":"core","claims":[{"name":"mesh-vault","scope":"mesh"}],"build":{"artifacts":[` + before + `]}}` } _, err := ParseManifest([]byte(raw)) switch { @@ -2560,7 +2560,7 @@ func caseAProcessBusNamesOnlyTheModulesOwn(t *testing.T) { func caseAProcessBusNeverServesOrHoldsASeat(t *testing.T) { for name, c := range map[string]struct{ bus, says string }{ "holds": {`{"emits":["mail.send.sent"],"holds":["mail-sender"]}`, "holds"}, - "claims": {`{"emits":["mail.send.sent"],"claims":["mail-sender"]}`, "claims"}, + "claims": {`{"emits":["mail.send.sent"],"class":"core","claims":["mail-sender"]}`, "claims"}, "seat": {`{"emits":["mail.send.sent"],"seat":"mail-sender"}`, "seat"}, } { _, err := ParseManifest([]byte(mailSenderProcessManifest(t, c.bus))) @@ -2610,7 +2610,7 @@ func caseAProcessDeclaresItsBusAndIsAccepted(t *testing.T) { // process's, named after its binary and kept through resolution; a removal is from the closed set, alone, and the // bus module's (mesh-broker) alone. toolsIn := func(claims, bus, mode, where string) string { - return `{"module":"nats","version":"1.0","claims":` + claims + `,"own-secrets":{"broker":"${dir:mesh-state}/broker",` + + return `{"module":"nats","version":"1.0","class":"core","claims":` + claims + `,"own-secrets":{"broker":"${dir:mesh-state}/broker",` + `"broker-nats-tools":"${dir:tools-secret}/broker-nats-tools"},"resources":[{"id":"mesh-state","type":"directory",` + `"mode":"0700","place":"mesh","unknown":"program"},{"id":"tools-secret","type":"directory","mode":"` + mode + `","path":"` + where + `","unknown":"program"}],"build":{"artifacts":[{"name":"tools","kind":"bundle",` + @@ -3276,10 +3276,10 @@ func caseAManifestCarryingWaitsIsAFieldNotKnown(t *testing.T) { func caseAMarkNothingEnforcesIsRefused(t *testing.T) { const bundle = `"build":{"artifacts":[{"name":"tools","kind":"bundle","language":"typescript","entrypoints":["tools/index.js"]}]}` const oc = `"uses":["operator-channel"]` - shell := `{"module":"zsh","claims":[{"name":"node-login-shell","scope":"node","serves":["execute"]}],"tools":["zsh_config"],` + bundle + shell := `{"module":"zsh","class":"core","claims":[{"name":"node-login-shell","scope":"node","serves":["execute"]}],"tools":["zsh_config"],` + bundle for _, row := range []struct{ name, manifest, want string }{ {"a tool its tools do not list", `{"module":"shop","tools":["sell"],"needs-approval":["refund"],` + oc + `,` + bundle + `}`, "its tools do not list"}, - {"a verb its claims do not serve", `{"module":"shop","claims":[{"name":"shop-till","serves":["price"]}],"tools":["price"],"needs-approval":["shop-till.refund"],` + oc + `,` + bundle + `}`, "does not serve"}, + {"a verb its claims do not serve", `{"module":"shop","class":"core","claims":[{"name":"shop-till","serves":["price"]}],"tools":["price"],"needs-approval":["shop-till.refund"],` + oc + `,` + bundle + `}`, "does not serve"}, {"a marked tool without operator-channel", `{"module":"shop","tools":["sell"],"needs-approval":["sell"],` + bundle + `}`, "without operator-channel"}, {"a held seat verb its seat marks without operator-channel", shell + `}`, "without operator-channel"}, {"every bundle run by its own process", `{"module":"shop","tools":["sell"],"needs-approval":["sell"],` + oc + `,` + bundle + @@ -4130,7 +4130,7 @@ func caseNodeBuildAgentIsRetiredForTheBuildSeats(t *testing.T) { t.Errorf("the mesh still defines %s", gone) } } - _, err := ParseManifest([]byte(`{"module":"build-agent","version":"1.0","claims":[{"name":"node-build-agent","scope":"node"}]}`)) + _, err := ParseManifest([]byte(`{"module":"build-agent","version":"1.0","class":"core","claims":[{"name":"node-build-agent","scope":"node"}]}`)) if err == nil || !strings.Contains(err.Error(), `"node-build-agent", which is retired`) || !strings.Contains(err.Error(), "node-build-go") { t.Fatalf("the claim is not refused naming its replacement: %v", err) } @@ -4193,7 +4193,7 @@ func caseNoModuleMayDeclareTheLoginShell(t *testing.T) { } // And a shell module claiming the mesh's seat, serving execute, is what the seat is for. m, err := ParseManifest([]byte(`{"module":"zsh","tools":["execute"],"uses":["operator-channel"],` + - `"claims":[{"name":"node-login-shell","scope":"node"}]}`)) + `"class":"core","claims":[{"name":"node-login-shell","scope":"node"}]}`)) if err != nil { t.Fatal(err) } @@ -4506,7 +4506,7 @@ func caseAStoreHolderThatBuildsIsNotRefusedByItsManifest(t *testing.T) { // nothing, so it is refused at the parser rather than silently dropped. func caseTheBussAccountSaysNothingOnTheBus(t *testing.T) { raw := []byte(`{"module":"bus","version":"1.0","provides":[{"name":"mesh-bus","scope":"mesh"}], - "claims":[{"name":"mesh-broker","scope":"mesh"}],"own-secrets":{"broker":"/run/broker"}, + "class":"core","claims":[{"name":"mesh-broker","scope":"mesh"}],"own-secrets":{"broker":"/run/broker"}, "emits":["something.happened"]}`) _, err := ParseManifest(raw) if err == nil || !strings.Contains(err.Error(), "granted the bus's snapshot API and nothing else") { @@ -4514,7 +4514,7 @@ func caseTheBussAccountSaysNothingOnTheBus(t *testing.T) { } // Nor what it invokes, with no bundle for the runtime to ask on its behalf. unbundledInvoke := []byte(`{"module":"bus","version":"1.0","provides":[{"name":"mesh-bus","scope":"mesh"}], - "claims":[{"name":"mesh-broker","scope":"mesh"}],"own-secrets":{"broker":"/run/broker"}, + "class":"core","claims":[{"name":"mesh-broker","scope":"mesh"}],"own-secrets":{"broker":"/run/broker"}, "invokes":["seat:mesh-hand-acts.hand-act"]}`) if _, err := ParseManifest(unbundledInvoke); err == nil || !strings.Contains(err.Error(), "its invokes would be granted nothing") { t.Fatalf("a bus module with no bundle declaring what it invokes was not refused, or not for the reason: %v", err) @@ -4525,7 +4525,7 @@ func caseTheBussAccountSaysNothingOnTheBus(t *testing.T) { // ships one may declare them. The shape of mesh-catalog's modules/nats/module.json at fc5b849d, cut to the // parts the rule reads, was refused and held every bus user list from 2026-11-02 01:35. nats := []byte(`{"module":"nats","version":"1.0","provides":[{"name":"mesh-bus","scope":"mesh","credential":{"form":"none"}}], - "claims":[{"name":"mesh-broker","scope":"mesh"}], + "class":"core","claims":[{"name":"mesh-broker","scope":"mesh"}], "own-secrets":{"broker":"/run/broker","broker-nats-tools":"${dir:tools-secret}/broker-nats-tools"}, "resources":[{"id":"tools-secret","type":"directory","path":"/var/lib/mesh/nats-tools","mode":"0711","unknown":"program"}], "emits":["consumer-removed","bucket-keys-removed","stream-removed","bucket-expiry-set"], @@ -4551,7 +4551,7 @@ func caseTheBussAccountSaysNothingOnTheBus(t *testing.T) { // Its tools bundle says what it emits (novox/hq ADR 0357): the node's runtime carries the bundle and is // granted the module's events, so a bus module that ships one may declare them. bundled := []byte(`{"module":"bus","version":"1.0","provides":[{"name":"mesh-bus","scope":"mesh"}], - "claims":[{"name":"mesh-broker","scope":"mesh"}],"own-secrets":{"broker":"/run/broker"}, + "class":"core","claims":[{"name":"mesh-broker","scope":"mesh"}],"own-secrets":{"broker":"/run/broker"}, "emits":["bus.slow-consumer"],"tools":["bus_streams"], "build":{"artifacts":[{"name":"tools","kind":"bundle","language":"go","system":"arch","from":"cmd/t","binary":"t"}]}}`) m, err := ParseManifest(bundled) @@ -4577,7 +4577,7 @@ func caseTheBussAccountSaysNothingOnTheBus(t *testing.T) { // parts the rule reads. keeper := func(bus string, more string) []byte { return []byte(`{"module":"nats","version":"1.0","provides":[{"name":"mesh-bus","scope":"mesh"}], - "claims":[{"name":"mesh-broker","scope":"mesh","serves":["list"]}],"own-secrets":{"broker":"/run/broker"}, + "class":"core","claims":[{"name":"mesh-broker","scope":"mesh","serves":["list"]}],"own-secrets":{"broker":"/run/broker"}, "tools":["nats_server"],"consumes":["mesh-catalog.manifest-accepted"],"uses":["operator-channel"], "state":[{"name":"requests","history":5}],"streams":[{"name":"broker-events","events":["created"]}], "emits":["consumer-removed"], @@ -4621,7 +4621,7 @@ func caseTheBussAccountSaysNothingOnTheBus(t *testing.T) { } quiet := []byte(`{"module":"bus","version":"1.0","provides":[{"name":"mesh-bus","scope":"mesh"}], - "claims":[{"name":"mesh-broker","scope":"mesh"}],"own-secrets":{"broker":"/run/broker"}, + "class":"core","claims":[{"name":"mesh-broker","scope":"mesh"}],"own-secrets":{"broker":"/run/broker"}, "tools":["bus_streams"]}`) m, err = ParseManifest(quiet) if err != nil { @@ -4974,7 +4974,7 @@ func caseTheParserDoesNotJudgeWhatOnlyTheStoreKnows(t *testing.T) { catalogue.UseSeats([]catalogue.Seat{{Name: "mesh-broker", Scope: catalogue.ScopeMesh, Delivers: "mesh-bus", Decision: "test"}}) raw := []byte(`{"module":"a-bus","version":"1.0",` + `"provides":[{"name":"mesh-bus","scope":"mesh","credential":{"form":"none"}}],` + - `"claims":[{"name":"mesh-broker","scope":"mesh"}]}`) + `"class":"core","claims":[{"name":"mesh-broker","scope":"mesh"}]}`) m, err := ParseManifest(raw) if err != nil { @@ -5185,13 +5185,13 @@ func caseTheSessionsFilesArePlacedByTheDisplayServersHolderAlone(t *testing.T) { {`{"name":"node-display-server"}`, "${shell:zsh:normal}", "does not claim node-login-shell"}, {`{"name":"node-display-server"}`, "${shell:xsession:normal}", "the session's file one of xinitrc, xresources"}, } { - raw := `{"module":"holder","claims":[` + c.claims + `],"resources":[{"id":"rc","type":"file","path":"/etc/rc","content":"` + + raw := `{"module":"holder","class":"core","claims":[` + c.claims + `],"resources":[{"id":"rc","type":"file","path":"/etc/rc","content":"` + c.content + `"}]}` if _, err := ParseManifest([]byte(raw)); err == nil || !strings.Contains(err.Error(), c.want) { t.Errorf("%s with claims [%s]: want %q, got %v", c.content, c.claims, c.want, err) } } - if _, err := ParseManifest([]byte(`{"module":"xorg","claims":[{"name":"node-display-server"}],` + + if _, err := ParseManifest([]byte(`{"module":"xorg","class":"core","claims":[{"name":"node-display-server"}],` + `"resources":[{"id":"rc","type":"file","path":"/home/op/.xinitrc","content":"${shell:xinitrc:last}"}]}`)); err != nil { t.Errorf("the display server's holder could not place the session's slots: %v", err) } @@ -5259,7 +5259,7 @@ func caseTheUplinkIsANodeSeatThatDeliversNothing(t *testing.T) { t.Fatalf("the uplink is %+v, not a node seat delivering nothing by ADR 0117", seat) } // And a manager's module can hold it without providing anything. - raw := []byte(`{"module":"networkmanager","version":"1.0","claims":[{"name":"node-uplink","scope":"node"}]}`) + raw := []byte(`{"module":"networkmanager","version":"1.0","class":"core","claims":[{"name":"node-uplink","scope":"node"}]}`) if _, err := ParseManifest(raw); err != nil { t.Fatalf("a network manager's module could not hold the uplink: %v", err) } @@ -5573,7 +5573,7 @@ func problemsFor(t *testing.T, shelf catalogue.Shelf) string { } func claimed(claims string) []byte { - return []byte(`{"module":"thing","version":"1.0","provides":[{"name":"npm-package-registry","scope":"mesh","credential":{"form":"none"}}],"claims":` + claims + `}`) + return []byte(`{"module":"thing","version":"1.0","provides":[{"name":"npm-package-registry","scope":"mesh","credential":{"form":"none"}}],"class":"core","claims":` + claims + `}`) } func hostnameSeat(t *testing.T) catalogue.Seat { @@ -6098,7 +6098,7 @@ func mod(name string, provides, requires, capabilities []string, claims ...catal } const collectorZsh = `{"module":"zsh","version":"1.0","moved":true, - "claims":[{"name":"node-login-shell","scope":"node"}], + "class":"core","claims":[{"name":"node-login-shell","scope":"node"}], "collections":[{"name":"widgets","comment":"#"}], "build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]}, "resources":[{"id":"rc","type":"file","path":"${machine:account-home}/.zshrc","owner":"${machine:account}", @@ -6660,7 +6660,7 @@ func caseOneModuleClaimsTheNodeEnginesSeat(t *testing.T) { t.Cleanup(func() { catalogue.UseSeats(was) }) catalogue.UseSeats([]catalogue.Seat{{Name: NodeEngineSeat, Scope: catalogue.ScopeNode, Decision: "test", HeldByTheEngine: true}}) engine := func(module string) []byte { - return []byte(`{"module":"` + module + `","version":"1.0","claims":[{"name":"node-engine","scope":"node"}]}`) + return []byte(`{"module":"` + module + `","version":"1.0","class":"core","claims":[{"name":"node-engine","scope":"node"}]}`) } for _, row := range []struct { name string diff --git a/modules/mesh-delivery-indexer/judge/stores.go b/modules/mesh-delivery-indexer/judge/stores.go new file mode 100644 index 000000000..17edbbf89 --- /dev/null +++ b/modules/mesh-delivery-indexer/judge/stores.go @@ -0,0 +1,79 @@ +package judge + +import ( + "fmt" + "slices" + "sync/atomic" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/seatdef" +) + +// The manifest check of design 47 H2 §814 and §810: every kept store of a delivery module names its cap (§812), and a +// module that claims a seat names its class (H-A-394 (1), H-A-411 (4)). + +// deliveryDomain is the glossary domain of the delivery system's seats, as a seat definition names it (seatdef.Domains). +const deliveryDomain = "Delivery" + +// deliverySeats are the seats of the seat list the judge judges against whose definition is of the delivery system, +// by name: set with the list (UseSeatList), read by deliveryStoreProblems. +var deliverySeats atomic.Pointer[map[string]bool] + +// useDeliverySeats keeps the delivery system's seats of defs (deliverySeats). +func useDeliverySeats(defs []seatdef.Definition) { + seats := map[string]bool{} + for _, d := range defs { + if d.Domain == deliveryDomain { + seats[d.Seat] = true + } + } + deliverySeats.Store(&seats) +} + +// deliverySeatOf is the first seat of the delivery system m claims, by the seat list's definitions; "" for none. +func deliverySeatOf(m catalogue.Manifest) string { + seats := deliverySeats.Load() + if seats == nil { + return "" + } + for _, c := range m.Claims { + if (*seats)[c.Name] { + return c.Name + } + } + return "" +} + +// deliveryStoreProblems refuses each state, log and objects entry of a delivery module (one whose manifest claims a seat +// of the delivery system) that names no cap, in its old section or in `requires` alike (design 47 H2 §812, §814): a +// foundation part's old-section entry names its cap in that section. +func deliveryStoreProblems(m catalogue.Manifest) []string { + seat := deliverySeatOf(m) + if seat == "" { + return nil + } + var problems []string + for _, u := range m.UncappedStores() { + where := "in `requires`" + if u.OldSection { + where = "in its old section" + } + problems = append(problems, fmt.Sprintf("%s claims the seat %s of the delivery system, so every store it keeps names its cap; "+ + "its %s %q %s names none (design 47 H2 §812, §814)", m.Module, seat, u.Kind, u.Name, where)) + } + return problems +} + +// classProblems refuses a manifest that claims a seat and names no class, or a class that is neither core nor leaf +// (design 47 H2 §810, §814; H-A-394 (1)). +func classProblems(m catalogue.Manifest) []string { + if len(m.Claims) == 0 || slices.Contains([]string{catalogue.ClassCore, catalogue.ClassLeaf}, m.Class) { + return nil + } + if m.Class == "" { + return []string{fmt.Sprintf("%s claims a seat and names no `class`; a module that claims a seat names `class` %s or %s "+ + "(design 47 H2 §810, §814)", m.Module, catalogue.ClassCore, catalogue.ClassLeaf)} + } + return []string{fmt.Sprintf("%s names the class %q; a class is %s or %s (design 47 H2 §810, §814)", m.Module, m.Class, + catalogue.ClassCore, catalogue.ClassLeaf)} +} From 92ba593639a52d209950022f0bb30de2599cdcc9 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 01:57:10 +0100 Subject: [PATCH 14/15] I6 (2a): the seat mesh-catalog's seat-list hands the seat list's bytes in chunks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit seat-list is a chunked read verb, as input is: input call (128 random bits), answer digest (the list's sha256, its object's name in seat-lists), size and chunks, then the bytes on the call's chunk subject. The object name leaves the answer. The seat's form rises to 4; regenerated with seatgen, -check clean; the fixtures read so. Builds: design 47 H2 §814, §820; H-A-441, H-A-443 (1) --- .../go/seats/meshcatalog/contract.go | 16 +++++----- .../go/seats/meshcatalog/contract_test.go | 4 ++- .../seats/mesh-catalog/fixtures/verbs.json | 15 ++++++--- .../seats/mesh-catalog/seat.json | 31 +++++++++++++------ .../ts/seats/meshcatalog/index.d.ts | 10 +++--- 5 files changed, 50 insertions(+), 26 deletions(-) diff --git a/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract.go b/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract.go index 19cadc72e..43d307e91 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract.go @@ -14,13 +14,13 @@ const Seat = "mesh-catalog" // Form is the form of the seat's values, events and answers this contract was generated at (design 47 H2 §780): // a reader of a value of another form says "form N, this reader knows M". -const Form = 3 +const Form = 4 // The forms and the digest of the fields this contract was generated from: the generator's compare fails a definition // whose fields changed while its forms did not (design 47 H2 §780). const ( - generatedForms = "3" - generatedFieldsDigest = "ActingRule=8a83ce7e0457 Artifact=78b05c1a9ec3 ArtifactPackage=2a3a660a934d ArtifactRef=a2f06147f4b6 BuildEdge=76f0684e254a BuildEdges=d2b95000259e BuildEdgesByModuleBranchValue=4fdf7cc1e723 CandidatesByGroupValue=9ea8c6c580d2 CandidatesByGroupValuePullsItem=90fb03643258 CandidatesByGroupValuePullsItemFpsValue=4c6783201aa3 CatalogEntry=c0a4ea25d523 CatalogRevision=d1ecaaa484af CatalogRevisionDefinitionsItem=77fcff620d48 CatalogRevisionEvent=710cbc5bfff1 CatalogRevisionsByBranchRevisionValue=17e54f635d35 CatalogRevisionsByCandidateGroupHeadsValue=17e54f635d35 CatalogRevisionsByEntryDigestValue=8755036a3051 DependentsAnswer=7538f2f5e68e Edge=7242732afaef EntryAnswer=805c2ff28e1c EntryInputs=76a7f8690dd2 EntryRef=ff109e5089e1 Fill=80f442916429 FillingEvent=5bd4df78ed84 FingerprintInputs=99a6df7f991f FullEvent=5bd4df78ed84 Head=774b4bf88ba4 InputAnswer=cd41ee65565d InputsByTimeDigestValue=a2f06147f4b6 ManifestsByDigestValue=51b3822b29c6 ModuleAnswer=499a03ce9b27 ModuleAnswerModule=c0f679e09f08 ModulesAnswer=6d119571c69e ModulesAnswerModulesItem=a70f39c60b67 ModulesByModuleValue=fc7530ba893a ModulesByModuleValueInputs2=99a6df7f991f Outdated=e820214f66bc OutdatedAnswer=275fb25266b9 OutdatedDependency=dc35dca1b683 PackagesByModuleValue=2b62574b709f PackagesByModuleValuePackagesItem=a7d30558470d PackagesByPackageSeriesValue=07421d8d3411 PackagesByPackageSeriesValueNewest=aba144994370 PackagesByPackageSeriesValueNewestContents=eca2ccf56913 PackagesByPackageSeriesValueNewestContentsPackagesItem=a7d30558470d PackagesByPackageSeriesValueNumbered=c8ffc628599c PackagesByPackageSeriesValueNumberedContents=0def6c0794ee PackagesByPackageSeriesValueNumberedContentsPackagesItem=a7d30558470d Policy=719e2f460c6f PolicyAnswer=46cdf8430557 PredefinedByListValue=ec8e3febe8ac ProvidesAnswer=7538f2f5e68e RegisterAnswer=46cdf8430557 Registration=ef03d1b2e73f RegistrationAsked=55adba349c24 RegistrationChangedEvent=99a3268edf41 RegistrationHistoryByModuleVersionValue=0999b5bb7dc7 RegistrationHistoryBySourceOwnerNameVersionValue=0999b5bb7dc7 RegistrationsAnswer=8849ef618cf3 RegistrationsByModuleValue=0999b5bb7dc7 RegistrationsBySourceOwnerNameValue=0999b5bb7dc7 RevisionAnswer=258ea5124931 RoomEvent=5bd4df78ed84 SeatDefinitionChangedEvent=e271d24d58f7 SeatDefinitionRemovedEvent=f7d6171111e6 SeatDefinitionsAnswer=0fd6a59fec59 SeatDefinitionsAnswerEntry=d006f68d2be5 SeatDefinitionsAnswerSeatsItem=e271d24d58f7 SeatListAnswer=912c0cebcf0a SourceCheck=d1ed7cbbd579 StepStatuses=4f50aa633e61 TakeInEntry=545ec1528499 ToolchainUsed=43b6d42376fb UnwatchAnswer=46cdf8430557 VersionArtifact=438c49a392b0 VersionEdge=b0f80e83f4dc VersionEntry=33f5b00dbda9 VersionReleasedEvent=4ccf1ea90e7f VersionsAnswer=b59e6e057e7b VersionsByModuleVersionValue=e546dfadb913 VersionsFilledEvent=4cc437912dc3 WatchAnswer=46cdf8430557" + generatedForms = "4" + generatedFieldsDigest = "ActingRule=8a83ce7e0457 Artifact=78b05c1a9ec3 ArtifactPackage=2a3a660a934d ArtifactRef=a2f06147f4b6 BuildEdge=76f0684e254a BuildEdges=d2b95000259e BuildEdgesByModuleBranchValue=4fdf7cc1e723 CandidatesByGroupValue=9ea8c6c580d2 CandidatesByGroupValuePullsItem=90fb03643258 CandidatesByGroupValuePullsItemFpsValue=4c6783201aa3 CatalogEntry=c0a4ea25d523 CatalogRevision=d1ecaaa484af CatalogRevisionDefinitionsItem=77fcff620d48 CatalogRevisionEvent=710cbc5bfff1 CatalogRevisionsByBranchRevisionValue=17e54f635d35 CatalogRevisionsByCandidateGroupHeadsValue=17e54f635d35 CatalogRevisionsByEntryDigestValue=8755036a3051 DependentsAnswer=7538f2f5e68e Edge=7242732afaef EntryAnswer=805c2ff28e1c EntryInputs=76a7f8690dd2 EntryRef=ff109e5089e1 Fill=80f442916429 FillingEvent=5bd4df78ed84 FingerprintInputs=99a6df7f991f FullEvent=5bd4df78ed84 Head=774b4bf88ba4 InputAnswer=cd41ee65565d InputsByTimeDigestValue=a2f06147f4b6 ManifestsByDigestValue=51b3822b29c6 ModuleAnswer=499a03ce9b27 ModuleAnswerModule=c0f679e09f08 ModulesAnswer=6d119571c69e ModulesAnswerModulesItem=a70f39c60b67 ModulesByModuleValue=fc7530ba893a ModulesByModuleValueInputs2=99a6df7f991f Outdated=e820214f66bc OutdatedAnswer=275fb25266b9 OutdatedDependency=dc35dca1b683 PackagesByModuleValue=2b62574b709f PackagesByModuleValuePackagesItem=a7d30558470d PackagesByPackageSeriesValue=07421d8d3411 PackagesByPackageSeriesValueNewest=aba144994370 PackagesByPackageSeriesValueNewestContents=eca2ccf56913 PackagesByPackageSeriesValueNewestContentsPackagesItem=a7d30558470d PackagesByPackageSeriesValueNumbered=c8ffc628599c PackagesByPackageSeriesValueNumberedContents=0def6c0794ee PackagesByPackageSeriesValueNumberedContentsPackagesItem=a7d30558470d Policy=719e2f460c6f PolicyAnswer=46cdf8430557 PredefinedByListValue=ec8e3febe8ac ProvidesAnswer=7538f2f5e68e RegisterAnswer=46cdf8430557 Registration=ef03d1b2e73f RegistrationAsked=55adba349c24 RegistrationChangedEvent=99a3268edf41 RegistrationHistoryByModuleVersionValue=0999b5bb7dc7 RegistrationHistoryBySourceOwnerNameVersionValue=0999b5bb7dc7 RegistrationsAnswer=8849ef618cf3 RegistrationsByModuleValue=0999b5bb7dc7 RegistrationsBySourceOwnerNameValue=0999b5bb7dc7 RevisionAnswer=258ea5124931 RoomEvent=5bd4df78ed84 SeatDefinitionChangedEvent=e271d24d58f7 SeatDefinitionRemovedEvent=f7d6171111e6 SeatDefinitionsAnswer=0fd6a59fec59 SeatDefinitionsAnswerEntry=d006f68d2be5 SeatDefinitionsAnswerSeatsItem=e271d24d58f7 SeatListAnswer=04477a8044b1 SourceCheck=d1ed7cbbd579 StepStatuses=4f50aa633e61 TakeInEntry=545ec1528499 ToolchainUsed=43b6d42376fb UnwatchAnswer=46cdf8430557 VersionArtifact=438c49a392b0 VersionEdge=b0f80e83f4dc VersionEntry=33f5b00dbda9 VersionReleasedEvent=4ccf1ea90e7f VersionsAnswer=b59e6e057e7b VersionsByModuleVersionValue=e546dfadb913 VersionsFilledEvent=4cc437912dc3 WatchAnswer=46cdf8430557" ) const ( @@ -1232,17 +1232,19 @@ type SeatDefinitionsAnswer struct { // SeatListInput is what seat-list takes. type SeatListInput struct { + // 128 random bits the caller chooses: the subject token of this send's chunks and acknowledgements + Call string `json:"call"` } // SeatListAnswer is what seat-list answers. type SeatListAnswer struct { - // the seat list's digest + // how many chunks are sent + Chunks int64 `json:"chunks"` + // the seat list's sha256, the object's name in seat-lists Digest string `json:"digest"` // is the form of what this value carries, left out while it is 1 (design 47 H2 §780). Form int64 `json:"form,omitempty"` - // the object's name in seat-lists - Object string `json:"object"` - // the object's size in bytes + // the seat list's size in bytes Size int64 `json:"size"` } diff --git a/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract_test.go b/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract_test.go index 7116d74ee..9981aea73 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract_test.go +++ b/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract_test.go @@ -185,7 +185,9 @@ func TestFixtures(t *testing.T) { decode: func(b []byte) (any, error) { var v RegistrationsInput; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, {name: "registrations answers a capped page with a source-only registration and the next cursor", subject: "", given: "{\"capped\":true,\"next\":\"c-121\",\"registrations\":[{\"at\":\"2026-11-22T00:10:00Z\",\"branches\":[\"main\"],\"by\":\"operator@workstation\",\"check\":{\"seat\":\"node-build-go\",\"step\":\"unit\"},\"origin\":\"novox/hq\",\"policy\":{\"steps\":{\"approval\":\"bypass\",\"review\":\"optional\",\"unit\":\"required\"}},\"v\":1,\"version\":1,\"warrant\":\"ask-m2kx9a1b3c4d6a\"}],\"total\":118}", wire: "{\"capped\":true,\"next\":\"c-121\",\"registrations\":[{\"at\":\"2026-11-22T00:10:00Z\",\"branches\":[\"main\"],\"by\":\"operator@workstation\",\"check\":{\"seat\":\"node-build-go\",\"step\":\"unit\"},\"origin\":\"novox/hq\",\"policy\":{\"steps\":{\"approval\":\"bypass\",\"review\":\"optional\",\"unit\":\"required\"}},\"v\":1,\"version\":1,\"warrant\":\"ask-m2kx9a1b3c4d6a\"}],\"total\":118}", decode: func(b []byte) (any, error) { var v RegistrationsAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, - {name: "seat-list answers the seat list's object by its digest", subject: "", given: "{\"digest\":\"sha256:eeee\",\"object\":\"sha256-eeee\",\"size\":412345}", wire: "{\"digest\":\"sha256:eeee\",\"object\":\"sha256-eeee\",\"size\":412345}", + {name: "seat-list asks the seat list's bytes on a call's chunks", subject: "", given: "{\"call\":\"0123456789abcdef0123456789abcdef\"}", wire: "{\"call\":\"0123456789abcdef0123456789abcdef\"}", + decode: func(b []byte) (any, error) { var v SeatListInput; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, + {name: "seat-list answers the seat list's digest, size and how many chunks carry its bytes", subject: "", given: "{\"chunks\":1,\"digest\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"size\":412345}", wire: "{\"chunks\":1,\"digest\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"size\":412345}", decode: func(b []byte) (any, error) { var v SeatListAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, } for _, c := range cases { diff --git a/modules/mesh-delivery-contracts/seats/mesh-catalog/fixtures/verbs.json b/modules/mesh-delivery-contracts/seats/mesh-catalog/fixtures/verbs.json index 542c676bf..10c49b397 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-catalog/fixtures/verbs.json +++ b/modules/mesh-delivery-contracts/seats/mesh-catalog/fixtures/verbs.json @@ -485,12 +485,19 @@ } }, { - "name": "seat-list answers the seat list's object by its digest", + "name": "seat-list asks the seat list's bytes on a call's chunks", + "of": "verb:seat-list.input", + "given": { + "call": "0123456789abcdef0123456789abcdef" + } + }, + { + "name": "seat-list answers the seat list's digest, size and how many chunks carry its bytes", "of": "verb:seat-list.answer", "given": { - "digest": "sha256:eeee", - "size": 412345, - "object": "sha256-eeee" + "chunks": 1, + "digest": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "size": 412345 } } ] diff --git a/modules/mesh-delivery-contracts/seats/mesh-catalog/seat.json b/modules/mesh-delivery-contracts/seats/mesh-catalog/seat.json index da8e95508..5a656ddc1 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-catalog/seat.json +++ b/modules/mesh-delivery-contracts/seats/mesh-catalog/seat.json @@ -256,34 +256,45 @@ { "name": "seat-list", "class": "read", - "description": "The seat list the mesh holds, as one object in the holder's objects seat-lists, one object per digest (design 47 H2 §814): the manifest validator program takes it by address, so a run by hand reads what the live validate reads. Reads only.", + "description": "The trunk's seat list as the mesh holds it, the object of the holder's objects seat-lists of that digest (design 47 H2 §814, §820; H-A-441, H-A-443): its digest and size, then its bytes (the definitions resolved, a JSON array in the form of a composer request's seats, sorted by seat) in chunks of at most 512 KiB on the chunk subject of the call, each acknowledged by the caller on the ack subject within 30 seconds. A validate run's asker fills the request's seats from these bytes. Reads only.", "input": { "type": "object", - "properties": {} + "properties": { + "call": { + "type": "string", + "description": "128 random bits the caller chooses: the subject token of this send's chunks and acknowledgements", + "minLength": 1 + } + }, + "required": [ + "call" + ] }, "answer": { "type": "object", "properties": { "digest": { "type": "string", - "description": "the seat list's digest" + "description": "the seat list's sha256, the object's name in seat-lists" }, "size": { "type": "integer", "minimum": 0, - "description": "the object's size in bytes" + "description": "the seat list's size in bytes" }, - "object": { - "type": "string", - "description": "the object's name in seat-lists" + "chunks": { + "type": "integer", + "minimum": 0, + "description": "how many chunks are sent" } }, "required": [ "digest", "size", - "object" + "chunks" ] - } + }, + "chunks": true }, { "name": "revision", @@ -2725,5 +2736,5 @@ ] } }, - "form": 3 + "form": 4 } diff --git a/modules/mesh-delivery-contracts/ts/seats/meshcatalog/index.d.ts b/modules/mesh-delivery-contracts/ts/seats/meshcatalog/index.d.ts index 19e3606ab..6862be0b2 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshcatalog/index.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/meshcatalog/index.d.ts @@ -699,17 +699,19 @@ export interface SeatDefinitionsAnswer { /** is what seat-list takes. */ export interface SeatListInput { + /** 128 random bits the caller chooses: the subject token of this send's chunks and acknowledgements */ + "call": string; } /** is what seat-list answers. */ export interface SeatListAnswer { - /** the seat list's digest */ + /** how many chunks are sent */ + "chunks": number; + /** the seat list's sha256, the object's name in seat-lists */ "digest": string; /** is the form of what this value carries, left out while it is 1 (design 47 H2 §780). */ "form"?: number; - /** the object's name in seat-lists */ - "object": string; - /** the object's size in bytes */ + /** the seat list's size in bytes */ "size": number; } From c8ba486318817e4141b8b54ad678ca378e75dbb3 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 02:00:07 +0100 Subject: [PATCH 15/15] I6 (2b): the indexer keeps the seat list as one object and hands it by seat-list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit At each trunk seat-definition move (said or not) the indexer writes the trunk's definitions, resolved, as one JSON array sorted by seat into its objects seat-lists under its sha256, when not held, and deletes every object but the newest two, never the one just kept; one journal line per write and delete. Its process serves the chunked verb seat-list: digest, size and chunks, then the object's bytes; a list not kept yet is refused naming it. The manifest serves seat-list and declares seat-lists in its old objects section at 64 MiB. composer.Request gains seat-list, the digest of the bytes Seats were filled from; the judge reads Seats alone. Builds: design 47 H2 §814, §820; H-A-441, H-A-443 (1) --- .../cmd/mesh-delivery-indexer/main.go | 10 +- .../cmd/mesh-delivery-indexer/objects.go | 23 +++ .../cmd/mesh-delivery-indexer/seatlist.go | 20 ++- .../mesh-delivery-indexer/seatlist_test.go | 26 ++- .../internal/index/seatlists.go | 162 ++++++++++++++++++ .../internal/index/seatlists_test.go | 123 +++++++++++++ .../mesh-delivery-indexer/manifest_test.go | 12 +- modules/mesh-delivery-indexer/module.json | 4 +- .../mesh-delivery-sdk/composer/composer.go | 4 + .../composer/composer_test.go | 5 + 10 files changed, 373 insertions(+), 16 deletions(-) create mode 100644 modules/mesh-delivery-indexer/internal/index/seatlists.go create mode 100644 modules/mesh-delivery-indexer/internal/index/seatlists_test.go diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index 352505c32..b43acbc05 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -318,13 +318,16 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro seatDefs := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.SeatDefinitionsState), "no seat definition is kept", logger.Printf) marks := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, views.MarksState), "no change is said on the views family, so every reader waits on its first view", logger.Printf) seatBus := &seatBus{defs: awaitedSource{ctx, seatDefs}, marks: awaitedSource{ctx, marks}} + // The seat lists, one object per digest (design 47 H2 §814, §820; H-A-441, H-A-443), kept at each trunk definition's move + // and handed by the verb seat-list. + seatLists := newBusObjects(js, cred, index.SeatListsObjects) // The build modules are found again at every change of the seat list (buildmodules.go), once the indexer is made. var buildFollow atomic.Pointer[buildModulesFollow] seatReader, saySeats := followSeatList(seatBus, saySeatDefinition(js, cred), logger.Printf, func() { if b := buildFollow.Load(); b != nil { go b.refresh() // never inside the reader's change: the refresh reads the reader's definitions } - }) + }, seatLists) readWhenBound(seatReader, seatDefs, marks) var fillFollow atomic.Pointer[versionsFill] // Kept while named (design 47 G §675): the want and the running build that name a fingerprint, heard below. @@ -336,9 +339,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro inputsStore := awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.InputsState), "no input of a run ask is kept, so the verb input answers none", logger.Printf)} inObjects := &inputObjects{js: js, module: cred.Module} stopInput, err := calls.AnswerSeat(meshcatalog.Seat, seatlist.Definition(seatReader.Reader, meshcatalog.Seat), map[string]process.SeatHandler{ - meshcatalog.VerbInput: index.InputVerb(inputsStore, inObjects, conn, buscalls.SeatAckWithin, logger.Printf)}) + meshcatalog.VerbInput: index.InputVerb(inputsStore, inObjects, conn, buscalls.SeatAckWithin, logger.Printf), + meshcatalog.VerbSeatList: index.SeatListVerb(seatBus.defs, seatLists, conn, buscalls.SeatAckWithin, logger.Printf)}) if err != nil { - return fmt.Errorf("answering the verb input: %w", err) + return fmt.Errorf("answering the verbs input and seat-list: %w", err) } defer stopInput() // The state takeins: hand acts ask there (watched below), and a running take-in keeps what it waits on (H2 §803, H-A-409). diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/objects.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/objects.go index 6cc947290..8b306ec4a 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/objects.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/objects.go @@ -14,6 +14,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-sdk/go/buscalls" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-sdk/go/views" @@ -109,6 +110,28 @@ func (b *busObjects) Names(ctx context.Context) ([]string, error) { return names, nil } +// List is every object the store holds, a deleted one left out (index.SeatLists). +func (b *busObjects) List(ctx context.Context) ([]buscalls.ObjectInfo, error) { + o, err := b.open(ctx) + if err != nil { + return nil, err + } + infos, err := o.List(ctx) + if errors.Is(err, jetstream.ErrNoObjectsFound) { + return []buscalls.ObjectInfo{}, nil + } + if err != nil { + return nil, err + } + out := make([]buscalls.ObjectInfo, 0, len(infos)) + for _, i := range infos { + if !i.Deleted { + out = append(out, buscalls.ObjectInfo{Name: i.Name, Size: i.Size, Chunks: i.Chunks, Digest: i.Digest, Modified: i.ModTime}) + } + } + return out, nil +} + func (b *busObjects) Fill(ctx context.Context) (int64, int64, error) { o, err := b.open(ctx) if err != nil { diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/seatlist.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/seatlist.go index 9c25c0283..69fa8d95e 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/seatlist.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/seatlist.go @@ -110,18 +110,26 @@ func (b *seatBus) said(c index.SeatDefinitionChanged) { // seatlist.Part, novox/hq ADR 0636 §7), which gives catalogue what it holds at every change, and wraps say so that each move // the indexer says is also heard by the reader. Its two tools are not served: the indexer's process serves no request // subject (ADR 0437 §4), and its list is its own state seat-definitions, which indexer_seat_definition and the seat's verb -// seat-definitions read. It returns the part, whose status the indexer's health can show. -func followSeatList(bus *seatBus, say func(context.Context, index.SeatDefinitionChanged) error, logf func(string, ...any), changed func()) (*seatlist.Part, func(context.Context, index.SeatDefinitionChanged) error) { +// seat-definitions read. It returns the part, whose status the indexer's health can show. At each trunk definition's move, +// said or not, the seat list is kept as one object in lists, the objects seat-lists (design 47 H2 §814, §820, H-A-441), a +// failure said in logf. +func followSeatList(bus *seatBus, say func(context.Context, index.SeatDefinitionChanged) error, logf func(string, ...any), changed func(), lists index.SeatLists) (*seatlist.Part, func(context.Context, index.SeatDefinitionChanged) error) { part := seatlist.NewPart(toolPrefix) part.Follow(bus, changed, logf) return part, func(ctx context.Context, c index.SeatDefinitionChanged) error { - if err := say(ctx, c); err != nil { - return err + if c.Source != index.SourceTrunk { // only a trunk's definition is the seat list's + return say(ctx, c) } - if c.Source == index.SourceTrunk { // only a trunk's definition is the seat list's + err := say(ctx, c) + if err == nil { bus.said(c) } - return nil + if lists != nil { + if _, kerr := index.KeepSeatList(ctx, bus.defs, lists, logf); kerr != nil { + logf("the seat list is not kept in %s after %s moved: %v", index.SeatListsObjects, c.Seat, kerr) + } + } + return err } } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/seatlist_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/seatlist_test.go index 1eb5b65e0..2b5b6c514 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/seatlist_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/seatlist_test.go @@ -1,7 +1,10 @@ package main import ( + "context" "errors" + "fmt" + "strings" "sync" "testing" "time" @@ -10,6 +13,8 @@ import ( "git.novox.be/novox/mesh-sdk/go/seatlist" "git.novox.be/novox/mesh-sdk/go/views" + + "mesh-delivery-indexer/internal/index" ) type unboundSource struct { @@ -44,7 +49,17 @@ func (b *awaitedFake) Ready(f func(jetstream.KeyValue)) { b.ready = append(b.rea func TestTheSeatListIsReadAgainWhenItsStatesAreBound(t *testing.T) { src := &unboundSource{} bus := &seatBus{defs: src, marks: src} - part, _ := followSeatList(bus, nil, t.Logf, nil) + var mu sync.Mutex + var unkept []string + logf := func(f string, a ...any) { + t.Logf(f, a...) + if strings.HasPrefix(f, "the seat list is not kept") { + mu.Lock() + unkept = append(unkept, fmt.Sprintf(f, a...)) + mu.Unlock() + } + } + part, say := followSeatList(bus, func(context.Context, index.SeatDefinitionChanged) error { return nil }, logf, nil, &busObjects{}) reader := part.Reader() <-reader.Ready() if reader.Status().Heard { @@ -76,5 +91,14 @@ func TestTheSeatListIsReadAgainWhenItsStatesAreBound(t *testing.T) { if _, ok := seatlist.Definition(part.Reader, "no-such-seat")(); ok { t.Error("a seat the list does not hold is answered a definition") } + // Design 47 H2 §814 (H-A-441): a trunk definition's move keeps the seat list in seat-lists, a pull request's does not; here + // the list holds no definition, so the keeping is refused and said. + _ = say(context.Background(), index.SeatDefinitionChanged{Seat: "a", Source: "pull-7", Digest: "d"}) + _ = say(context.Background(), index.SeatDefinitionChanged{Seat: "a", Source: index.SourceTrunk, Digest: "d"}) + mu.Lock() + defer mu.Unlock() + if len(unkept) != 1 || !strings.Contains(unkept[0], "holds no trunk definition") { + t.Errorf("the moves kept the seat list as %q, want one try, for the trunk's", unkept) + } var _ seatlist.Bus = bus } diff --git a/modules/mesh-delivery-indexer/internal/index/seatlists.go b/modules/mesh-delivery-indexer/internal/index/seatlists.go new file mode 100644 index 000000000..6c7593630 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/seatlists.go @@ -0,0 +1,162 @@ +package index + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "slices" + "strings" + "time" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + + "git.novox.be/novox/mesh-sdk/go/buscalls" + "git.novox.be/novox/mesh-sdk/go/views" +) + +// The seat list as one object (design 47 H2 §814, §820; H-A-441): the trunk's seat definitions, resolved with their +// includes, as one JSON array in composer.Request.Seats' form sorted by seat, kept in the indexer's objects seat-lists +// under its digest (the sha256 of those bytes), written at each trunk seat-definition-changed or -removed; every object +// but the newest two is deleted at each new list. The seat mesh-catalog's verb seat-list hands its digest, size and bytes, and +// a validate run's asker fills its request's seats from them, so every run of one seat list judges the same bytes. + +// SeatListsObjects is the indexer's objects store of seat lists, in its old objects section (a foundation part, ADR 0619 +// §2): 64 MiB, one object per digest. +const SeatListsObjects = "seat-lists" + +// seatListsKept is how many seat lists the objects keep: the newest two. +const seatListsKept = 2 + +// SeatLists is the objects seat-lists as the indexer's process reaches them, to write, prune and hand out. +type SeatLists interface { + // Put stores data whole under name, replacing one of that name. + Put(ctx context.Context, name string, data []byte) error + // Get is the object's bytes; ErrNoObject when it holds none of that name. + Get(ctx context.Context, name string) ([]byte, error) + // Delete removes the object; one it does not hold is no error. + Delete(ctx context.Context, name string) error + // List is every object the store holds, a deleted one left out. + List(ctx context.Context) ([]buscalls.ObjectInfo, error) +} + +// SeatListOf is the trunk's seat list as the object holds it, and its digest: each trunk definition resolved, in the +// order of its seat's name. A list with no seat is an error: no validator judges by an empty list. +func SeatListOf(defs views.Source) ([]byte, string, error) { + keys, err := defs.Keys() + if err != nil { + return nil, "", fmt.Errorf("the %s state cannot be listed: %w", SeatDefinitionsState, err) + } + type seatDef struct { + seat string + def json.RawMessage + } + var seats []seatDef + for _, key := range keys { + seat, ok := strings.CutSuffix(key, "."+SourceTrunk) + if !ok || strings.HasPrefix(key, outOfStepPrefix) { + continue + } + v, ok, err := defs.Get(key) + if err != nil { + return nil, "", fmt.Errorf("%s.%s cannot be read: %w", SeatDefinitionsState, key, err) + } + if !ok || v == nil { + continue + } + var e SeatDefinitionEntry + if err := json.Unmarshal(v.Value, &e); err != nil || e.Digest == "" || len(e.Definition) == 0 { + continue // an entry that holds no definition is no seat of the list + } + seats = append(seats, seatDef{seat, e.Definition}) + } + slices.SortFunc(seats, func(a, b seatDef) int { return strings.Compare(a.seat, b.seat) }) + list := make([]json.RawMessage, 0, len(seats)) + for _, s := range seats { + list = append(list, s.def) + } + if len(list) == 0 { + return nil, "", fmt.Errorf("the %s state holds no trunk definition, so there is no seat list", SeatDefinitionsState) + } + raw, err := json.Marshal(list) + if err != nil { + return nil, "", err + } + sum := sha256.Sum256(raw) + return raw, hex.EncodeToString(sum[:]), nil +} + +// KeepSeatList writes the trunk's seat list into seat-lists under its digest when the objects do not hold it, then deletes +// every object but the newest two, never the list just kept. It answers the list's digest; each write and each delete is +// one line in logf. +func KeepSeatList(ctx context.Context, defs views.Source, o SeatLists, logf func(string, ...any)) (string, error) { + raw, digest, err := SeatListOf(defs) + if err != nil { + return "", err + } + held, err := o.List(ctx) + if err != nil { + return "", fmt.Errorf("the objects %s cannot be listed: %w", SeatListsObjects, err) + } + if !slices.ContainsFunc(held, func(i buscalls.ObjectInfo) bool { return i.Name == digest }) { + if err := o.Put(ctx, digest, raw); err != nil { + return "", fmt.Errorf("the seat list %s cannot be kept in %s: %w", digest, SeatListsObjects, err) + } + logf("seat list %s kept in %s (%d bytes)", digest, SeatListsObjects, len(raw)) + if held, err = o.List(ctx); err != nil { + return digest, fmt.Errorf("the objects %s cannot be listed: %w", SeatListsObjects, err) + } + } + slices.SortFunc(held, func(a, b buscalls.ObjectInfo) int { return b.Modified.Compare(a.Modified) }) // newest first + kept := 0 + for _, i := range held { + if i.Name == digest || kept < seatListsKept { + kept++ + continue + } + if err := o.Delete(ctx, i.Name); err != nil { + return digest, fmt.Errorf("the seat list %s cannot be deleted from %s: %w", i.Name, SeatListsObjects, err) + } + logf("seat list %s deleted from %s: only the newest %d are kept", i.Name, SeatListsObjects, seatListsKept) + } + return digest, nil +} + +// SeatListVerb is the seat mesh-catalog's verb seat-list, served by the indexer's process (H-A-443: the owner hands its +// objects, ADR 0596): the trunk's seat list's digest and size, then the bytes of its object in seat-lists in chunks of at +// most buscalls.SeatChunkBytes on the call's chunk subjects, each acknowledged before the next. A list the objects do not +// hold yet is refused naming it, never answered with other bytes. A caller that does not acknowledge within within ends +// the send, said in logf. +func SeatListVerb(defs views.Source, o SeatLists, line buscalls.ChunkLine, within time.Duration, logf func(string, ...any)) func(caller string, body json.RawMessage) (any, error) { + return func(_ string, raw json.RawMessage) (any, error) { + var in meshcatalog.SeatListInput + if err := json.Unmarshal(raw, &in); err != nil { + return nil, fmt.Errorf("the arguments are not JSON: %w", err) + } + if !callToken.MatchString(in.Call) { + return nil, errors.New("call is one token of 16 to 64 letters, digits, - or _: 128 random bits the caller chooses") + } + ctx := context.Background() + _, digest, err := SeatListOf(defs) + if err != nil { + return nil, err + } + body, err := o.Get(ctx, digest) + if errors.Is(err, ErrNoObject) { + return nil, fmt.Errorf("the seat list %s is not kept in %s yet: it is written at the trunk's next seat-definition change", digest, SeatListsObjects) + } + if err != nil { + return nil, fmt.Errorf("the seat list %s cannot be read from %s: %w", digest, SeatListsObjects, err) + } + send := buscalls.SeatChunks{Seat: meshcatalog.Seat, Verb: meshcatalog.VerbSeatList, Call: in.Call} + total := int64((len(body) + buscalls.SeatChunkBytes - 1) / buscalls.SeatChunkBytes) + go func() { // the send subscribes to the acknowledgements itself and ends when every chunk is sent or one is not acknowledged + if err := send.Send(line, body, within, logf); err != nil { + logf("the send %s of the seat list %s: %v", in.Call, digest, err) + } + }() + return meshcatalog.SeatListAnswer{Digest: digest, Size: int64(len(body)), Chunks: total}, nil + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/seatlists_test.go b/modules/mesh-delivery-indexer/internal/index/seatlists_test.go new file mode 100644 index 000000000..d00f43b52 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/seatlists_test.go @@ -0,0 +1,123 @@ +package index + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "slices" + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + + "git.novox.be/novox/mesh-sdk/go/buscalls" + "git.novox.be/novox/mesh-sdk/go/views" +) + +// memSeatLists is seat-lists in memory; each put is a second newer than the last. +type memSeatLists struct { + objects map[string][]byte + at map[string]time.Time + clock time.Time +} + +func (m *memSeatLists) Put(_ context.Context, name string, data []byte) error { + m.clock = m.clock.Add(time.Second) + m.objects[name], m.at[name] = data, m.clock + return nil +} + +func (m *memSeatLists) Get(_ context.Context, name string) ([]byte, error) { + d, ok := m.objects[name] + if !ok { + return nil, ErrNoObject + } + return d, nil +} + +func (m *memSeatLists) Delete(_ context.Context, name string) error { + delete(m.objects, name) + return nil +} + +func (m *memSeatLists) List(context.Context) ([]buscalls.ObjectInfo, error) { + out := []buscalls.ObjectInfo{} + for n, d := range m.objects { + out = append(out, buscalls.ObjectInfo{Name: n, Size: uint64(len(d)), Modified: m.at[n]}) + } + return out, nil +} + +func (m *memSeatLists) names() []string { + var out []string + for n := range m.objects { + out = append(out, n) + } + slices.Sort(out) + return out +} + +// noLine is a chunk line that hears nothing, so a send ends at once, said. +type noLine struct{} + +func (noLine) Publish(string, []byte) error { return nil } +func (noLine) Subscribe(string, nats.MsgHandler) (*nats.Subscription, error) { + return nil, errors.New("no bus in this test") +} + +// Design 47 H2 §814, §820 (H-A-441, H-A-443): the trunk's seat list is one object of seat-lists named by its sha256, in the +// order of its seats, kept at each new list with the newest two, and handed by the verb seat-list. +func TestTheSeatListIsOneObjectByItsDigest(t *testing.T) { + entry := func(seat, source, def string) *views.Entry { + raw, _ := json.Marshal(SeatDefinitionEntry{Seat: seat, Source: source, Digest: "d-" + seat, Definition: json.RawMessage(def)}) + return &views.Entry{Value: raw} + } + defs := memSource{ + SeatDefinitionKey("a-b"): entry("a-b", SourceTrunk, `{"seat":"a-b"}`), + SeatDefinitionKey("a"): entry("a", SourceTrunk, `{"seat":"a"}`), + "a.pull-7": entry("a", "pull-7", `{"seat":"a","pull":7}`), + outOfStepPrefix + "c." + SourceTrunk: entry("c", SourceTrunk, `{"seat":"c"}`), + } + want := `[{"seat":"a"},{"seat":"a-b"}]` // by seat, not by key: "a-b.trunk" sorts before "a.trunk" + sum := sha256.Sum256([]byte(want)) + digest := hex.EncodeToString(sum[:]) + raw, got, err := SeatListOf(defs) + if err != nil || string(raw) != want || got != digest { + t.Fatalf("SeatListOf = %s %s (%v), want %s %s", raw, got, err, want, digest) + } + if _, _, err := SeatListOf(memSource{}); err == nil { + t.Error("an empty seat list was given a digest") + } + + lists := &memSeatLists{objects: map[string][]byte{}, at: map[string]time.Time{}} + verb := SeatListVerb(defs, lists, noLine{}, time.Second, t.Logf) + if _, err := verb("", json.RawMessage(`{"call":"0123456789abcdef0123456789abcdef"}`)); err == nil || !strings.Contains(err.Error(), "not kept in seat-lists yet") { + t.Errorf("a list not kept yet was answered: %v", err) + } + for _, old := range []string{"old-1", "old-2", "old-3"} { + _ = lists.Put(context.Background(), old, []byte("[]")) + } + if d, err := KeepSeatList(context.Background(), defs, lists, t.Logf); err != nil || d != digest { + t.Fatalf("KeepSeatList = %s (%v), want %s", d, err, digest) + } + if names := lists.names(); !slices.Equal(names, []string{digest, "old-3"}) || string(lists.objects[digest]) != want { + t.Errorf("seat-lists holds %v, want the list kept and the newest other", names) + } + before := lists.at[digest] + if _, err := KeepSeatList(context.Background(), defs, lists, t.Logf); err != nil || !lists.at[digest].Equal(before) || len(lists.objects) != 2 { + t.Errorf("the same list kept again was written again, or another deleted: %v %v", lists.names(), err) + } + + if _, err := verb("", json.RawMessage(`{"call":"x"}`)); err == nil || !strings.Contains(err.Error(), "128 random bits") { + t.Errorf("a call that is no token was taken: %v", err) + } + a, err := verb("", json.RawMessage(`{"call":"0123456789abcdef0123456789abcdef"}`)) + if err != nil || a != (meshcatalog.SeatListAnswer{Digest: digest, Size: int64(len(want)), Chunks: 1}) { + t.Errorf("seat-list = %+v (%v), want its digest, size and one chunk", a, err) + } +} diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index acdaa0f5b..39223ee20 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -149,9 +149,11 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // capped at 512 MiB, under the local name the indexer opens. W4.14 (design 47, Reverted): their form is 1. Design 47 H // §742 (H-A-105): no catch-up tool of the indexer's follows a restore, indexer_take_in having gone with the builder-log // index; the gate's gate_head_take is the one hand take of a head. - if len(m.Objects) != 1 || m.Objects[0].Name != index.CatalogRevisionsObjects || m.Objects[0].MaxMiB != 512 || - m.Objects[0].Form != 1 || m.Objects[0].CatchUp != "" { - t.Errorf("objects = %+v, want %s capped at 512 MiB, form 1, no catch-up tool", m.Objects, index.CatalogRevisionsObjects) + // Design 47 H2 §814, §820 (H-A-441): the seat lists, one object per digest, 64 MiB, in the old section too. + if len(m.Objects) != 2 || m.Objects[0].Name != index.CatalogRevisionsObjects || m.Objects[0].MaxMiB != 512 || + m.Objects[0].Form != 1 || m.Objects[0].CatchUp != "" || m.Objects[1].Name != index.SeatListsObjects || m.Objects[1].MaxMiB != 64 { + t.Errorf("objects = %+v, want %s capped at 512 MiB, form 1, no catch-up tool, and %s at 64 MiB", m.Objects, + index.CatalogRevisionsObjects, index.SeatListsObjects) } // D-11 (ADR 0634 §1): the holders and buckets the indexer names (internal/index/names.go) are its manifest's: its own // name and states with their revisions, and the states it reads, each as `.`. @@ -306,8 +308,8 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // ADR 0604 §2, ADR 0636 §4: it holds the seat mesh-catalog, serving its four verbs and seat-definitions from its tools // bundle (which stays without a request subject); ADR 0663 §1: and revision, in the same runtime of its own. if len(m.Claims) != 1 || m.Claims[0].Name != index.CatalogSeat || m.Claims[0].Scope != catalogue.ScopeMesh || - !slices.Equal(m.Claims[0].Serves, []string{"modules", "module", "provides", "dependents", index.SeatDefinitionsVerb, "revision", "entry", "registrations", "register", "policy", "watch", "unwatch", "versions", "outdated", "input"}) { - t.Errorf("claims = %+v, want mesh-catalog with its seven verbs (entry: design 47 G §675)", m.Claims) + !slices.Equal(m.Claims[0].Serves, []string{"modules", "module", "provides", "dependents", index.SeatDefinitionsVerb, "seat-list", "revision", "entry", "registrations", "register", "policy", "watch", "unwatch", "versions", "outdated", "input"}) { + t.Errorf("claims = %+v, want mesh-catalog with its verbs (entry: design 47 G §675; seat-list: H2 §814)", m.Claims) } for _, e := range []string{index.SeatDefinitionChangedEvent, index.SeatDefinitionRemovedEvent} { if slices.Contains(m.Emits, e) { diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index dc1ad3aac..e28281ab1 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -13,6 +13,7 @@ "provides", "dependents", "seat-definitions", + "seat-list", "revision", "entry", "registrations", @@ -147,7 +148,8 @@ { "name": "mesh-delivery-indexer-findings", "events": ["finding-written.*", "finding-cleared.*", "findings-stated.*"], "max-age-days": 7, "max-mib": 16 } ], "objects": [ - { "name": "catalog-revisions", "max-mib": 512, "form": 1 } + { "name": "catalog-revisions", "max-mib": 512, "form": 1 }, + { "name": "seat-lists", "max-mib": 64 } ], "emits": [ "take-in.pending", diff --git a/modules/mesh-delivery-sdk/composer/composer.go b/modules/mesh-delivery-sdk/composer/composer.go index 019708074..eac635813 100644 --- a/modules/mesh-delivery-sdk/composer/composer.go +++ b/modules/mesh-delivery-sdk/composer/composer.go @@ -67,6 +67,10 @@ type Request struct { // them (the definitions of the seat mesh-catalog, resolved), raw so that this package names no seat (ADR 0635 §4, ADR 0636 // §1). The judge and the composer read them. None is no list at hand: a composer handed none judges no machine (ADR 0636 §1). Seats []json.RawMessage `json:"seats,omitempty"` + // SeatList is the sha256 of the bytes Seats were filled from: the object of the seat mesh-catalog's seat-list, or a + // revision's own definitions in the same form (design 47 H2 §814; H-A-441, H-A-443), so a verdict says which seat list + // judged it. The asker fetches the bytes and fills Seats before the run; the judge reads Seats alone. Empty is none named. + SeatList string `json:"seat-list,omitempty"` // BlameTries bounds the composer's search for the modules a machine that does not compose blames: it composes at // most this many times more, the assigner's setting blame-tries, since the composer has no settings of its own // (design 47 G §721, §729; A-469i). None is no search: the composer blames only what its composition holds or diff --git a/modules/mesh-delivery-sdk/composer/composer_test.go b/modules/mesh-delivery-sdk/composer/composer_test.go index c8c80c300..b14d35f09 100644 --- a/modules/mesh-delivery-sdk/composer/composer_test.go +++ b/modules/mesh-delivery-sdk/composer/composer_test.go @@ -63,6 +63,11 @@ func TestAManifestIsItsBytesOrItsDigest(t *testing.T) { if back, derr := Decode(wire); err != nil || derr != nil || back.BlameTries != 20 || !strings.Contains(string(wire), `"blame_tries":20`) { t.Errorf("blame_tries 20 went as %s and read back %d (%v, %v)", wire, back.BlameTries, err, derr) } + // Design 47 H2 §814 (H-A-441): the seat list a request's seats were filled from reads back by its digest. + wire, err = json.Marshal(Request{V: Version, SeatList: "0123abcd"}) + if back, derr := Decode(wire); err != nil || derr != nil || back.SeatList != "0123abcd" || !strings.Contains(string(wire), `"seat-list":"0123abcd"`) { + t.Errorf("seat-list went as %s and read back %q (%v, %v)", wire, back.SeatList, err, derr) + } // H3 (design 47 G A-470j, A-470k): a case's none, each module the revision gives no fingerprint with its none words, // reads back whole; a case naming none leaves the field off the wire. wire, err = json.Marshal(Request{V: Version, Cases: []Case{{Name: "change", Manifests: map[string]Manifest{},