Compare commits

..
65 changed files with 278 additions and 2364 deletions
@@ -19,7 +19,6 @@ import (
"runtime"
"slices"
"strings"
"sync"
"testing"
"time"
@@ -204,17 +203,11 @@ func (w *watchKV) Keys(context.Context, ...jetstream.WatchOpt) ([]string, error)
return nil, jetstream.ErrNoKeysFound
}
// putIntoMu holds putInto's writes one at a time: the composition step starts its machines' runs side by side, and each may
// raise a finding into the same map.
var putIntoMu sync.Mutex
// putInto is an assign.Findings Put that keeps each finding in into, as the finding writer would, by its key and its words.
func putInto(into map[string][]byte) func(mcseat.Finding) error {
return func(f mcseat.Finding) error {
raw, err := json.Marshal(f)
putIntoMu.Lock()
into[views.FindingKey(f.Scope, f.ID, f.Kind)] = raw
putIntoMu.Unlock()
return err
}
}
@@ -119,13 +119,6 @@ func TestManifest(t *testing.T) {
}
}
}
// H-A-186 (design 47 G §370): asks, which the gate, the teller and the view import, is a declared Go package of the
// module, so every release publishes it; H-A-188: in the module's version series, 0.3.
if !slices.ContainsFunc(m.Build.Artifacts, func(a catalogue.Artifact) bool {
return a.Name == "asks" && a.Kind == catalogue.ArtifactPackage && a.Language == "go" && slices.Equal(a.Paths, []string{"asks"}) && a.Series == m.Version && m.Version == "0.3"
}) {
t.Errorf("artifacts = %+v, want the Go package asks of series 0.3, the module's", m.Build.Artifacts)
}
// ADR 0635 §5: its views family is its ordinary events, `views.<state>` among its emits, on no stream of its own.
// The seat's four events are said on the seat's own subject (the seat's family), so the module keeps no stream for them
// and lists none among its emits.
@@ -370,15 +370,6 @@
"loads": [
"mesh-delivery-assigner"
]
},
{
"name": "asks",
"kind": "package",
"language": "go",
"paths": [
"asks"
],
"series": "0.3"
}
]
}
@@ -45,7 +45,7 @@ func (m *entryManifests) keep(e meshcatalog.CatalogEntry) error {
if m.byFp == nil {
m.byFp = map[string]catalogue.Manifest{}
}
m.byFp[meshcatalogclient.BareManifestDigest(e.ManifestDigest)] = mf
m.byFp[meshcatalog.BareManifestDigest(e.ManifestDigest)] = mf
return nil
}
@@ -228,8 +228,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return err
}
b.PackageWait = time.Duration(packageWait) * time.Minute
// Before it waits for an unpinned package, the build reads what the Go package registry seat holds of it (H-A-194).
b.PublishedVersions = publishedVersionsVia(calls.Call)
waitsBucket := process.AwaitBucket(ctx, js, conn, builds.StateBucket(cred.Module, build.WaitsState), "the toolchain waits are kept in memory only, and builder_waits shows none", logger.Printf)
b.Waits = newWaitsKept(waitsBucket, func(ctx context.Context, event, id string, body any) error {
return emitEvent(context.WithoutCancel(ctx), js, cred, event, id, body)
@@ -244,6 +242,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return emitSeatEvent(context.WithoutCancel(ctx), js, cred, meshbuilds.SubjectPackagePublished, "package-published-"+p.Module+"-"+p.Version+"-"+p.Artifact, p)
}
b.Reserved = reserved
// A release heard, or the newest the builder's record keeps, pins a package an entry names with none (H-A-165).
reserved.heardReleased = b.VersionReleased
b.NewestReleased = reserved.r.NewestReleased
// The last versions-filled the stream keeps is read at the start (design 47 G §369, §372): a restart does not lose it.
reserved.readFilled(ctx, lastKeptOf(js, cred, logger.Printf), sleeps)
// A state bound later numbers the builds looked at without it.
@@ -44,12 +44,15 @@ type reservations struct {
// built asks a done build's publish (design 47 G A-457a), released ends its item; nil are none.
built func(ctx context.Context, module, version string) error
released func(ctx context.Context, module, version string)
filled atomic.Bool
bounds func() jetstream.KeyValue
found *findings
bound time.Duration
wake func(context.Context)
logf func(string, ...any)
// heardReleased is told each version-released taken in (design 47 H H-A-165): a step ask waiting for a package of the
// module wakes. Nil tells nobody.
heardReleased func(module, version string)
filled atomic.Bool
bounds func() jetstream.KeyValue
found *findings
bound time.Duration
wake func(context.Context)
logf func(string, ...any)
// call asks a seat's verb: the git seat's tag (design 47 G §370).
call toolCaller
// say says package-published on the seat mesh-builds; nil says none.
@@ -252,6 +255,9 @@ func (rv *reservations) heard(ctx context.Context, m jetstream.Msg) bool {
if rv.released != nil {
rv.released(ctx, ev.Module, ev.Version)
}
if rv.heardReleased != nil {
rv.heardReleased(ev.Module, ev.Version)
}
rv.found.clear(ctx, versions.VersionUnreleased, meshconditionsclient.ScopeModule, ev.Module)
default:
return false
@@ -134,10 +134,15 @@ func TestTheCatalogsWordsMoveTheReservations(t *testing.T) {
if err != nil || !strings.Contains(got.Summary, "m:b unpublished") || strings.Contains(got.Summary, "m:a unpublished") || !strings.Contains(got.Summary, "1.4.3") {
t.Fatalf("finding = %+v (%v), want version-unreleased naming m:b of 1.4.3", got, err)
}
var heardReleases []string // H-A-165: a release heard wakes a step ask waiting for a package of the module
rv.heardReleased = func(m, v string) { heardReleases = append(heardReleases, m+" "+v) }
released := &wakeMsg{subject: meshcatalog.SubjectVersionReleased, data: []byte(`{"v":1,"module":"m","version":"1.4.3","source":1,"artifacts":[]}`)}
if !rv.heard(ctx, released) || !released.acked {
t.Fatalf("version-released was not taken in (acked %v)", released.acked)
}
if strings.Join(heardReleases, ", ") != "m 1.4.3" {
t.Errorf("the release heard was told %v, want m 1.4.3 (H-A-165)", heardReleases)
}
if strings.Join(items, ", ") != "asked m 1.4.3, ended m 1.4.3" {
t.Errorf("publish items %v, want the build's end to ask the publish and the release to end it", items)
}
@@ -105,10 +105,6 @@ func (h *seatHearer) heard(ctx context.Context, m jetstream.Msg) bool {
if h.res != nil && se.event == versions.HeardPublished {
h.res.published(ctx, o)
}
if se.event == versions.HeardPublished && o.Kind == versions.KindPackage && h.b != nil {
// A package registry seat's published pins a package an entry names with none, at the version it carries (H-A-192).
h.b.PackagePublished(o.Name, o.Version)
}
h.ack(m)
if len(wake) > 0 {
h.logf("the floor of %s is asked again: %s said %s", strings.Join(wake, ", "), se.seat, se.event)
@@ -13,7 +13,6 @@ import (
"git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
gitseat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/gopackageregistry"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-builder/internal/build"
@@ -97,28 +96,6 @@ func (s awaitedVersions) KeysOf(ctx context.Context, filter string) ([]string, e
return out, nil
}
// publishedVersionsVia asks the Go package registry seat's verb versions for one package path (design 47 H H-A-194): its
// versions, newest first.
func publishedVersionsVia(call toolCaller) func(context.Context, string) ([]string, error) {
return func(_ context.Context, path string) ([]string, error) {
raw, err := call("seat:"+gopackageregistry.Seat+"."+gopackageregistry.VerbVersions, gopackageregistry.VersionsInput{Name: path})
if err != nil {
return nil, err
}
var a gopackageregistry.VersionsAnswer
if err := json.Unmarshal(raw, &a); err != nil {
return nil, fmt.Errorf("%s's versions answered what is no list of versions: %w", gopackageregistry.Seat, err)
}
var out []string
for _, v := range a.Versions {
if v.Version != "" {
out = append(out, v.Version)
}
}
return out, nil
}
}
// floorVia is the floor's three sources as calls of the builder's own account (design 51 *The floor*): the seat's verb
// versions, the seat git's tags and the requirements the catalog half's entries name (A-119). A call that fails is that source
// silent.
@@ -13,7 +13,6 @@ import (
func TestAnEntryOfTheCatalogHalfIsTheFormTheBuilderBuildsBy(t *testing.T) {
e := meshcatalog.CatalogEntry{Module: "netcheck", Repository: "novox/mesh-catalog", Path: "modules/netcheck", Commit: "abc", Fp: "src2:f",
ManifestDigest: "sha256:m", Bases: []string{"base"}, Packages: map[string]string{"p": "1"}, Needs: []string{"n"},
Artifacts: []meshcatalog.Artifact{{Ref: "netcheck:lib", Kind: "package", Language: "go"}},
Inputs: &meshcatalog.EntryInputs{Toolchains: map[string]meshcatalog.ToolchainUsed{"go": {Version: "go1", Recipe: "1"}},
Paths: map[string][]string{"lib": {"a/b", "c"}}}}
got := ModuleOf(e)
@@ -31,11 +30,4 @@ func TestAnEntryOfTheCatalogHalfIsTheFormTheBuilderBuildsBy(t *testing.T) {
if bare := ModuleOf(meshcatalog.CatalogEntry{Module: "m"}); bare.Inputs2 != nil || bare.Fp2 != "" {
t.Errorf("ModuleOf of an entry with none = %+v, want nothing set", bare)
}
// A module with an artifact that names no paths (a Go program beside its packages) is built from its directory whole
// (ADR 0499 clauses 1 and 4): its fetch names no paths, so the program's go.mod and main package stay in the tree.
mixed := e
mixed.Artifacts = append([]meshcatalog.Artifact{{Ref: "netcheck:program", Kind: "bundle", Language: "go"}}, e.Artifacts...)
if got := ModuleOf(mixed); got.Inputs2 == nil || len(got.Inputs2.Paths) != 0 {
t.Errorf("ModuleOf of a module with a program naming no paths = %+v, want bases and no named paths", got.Inputs2)
}
}
@@ -5,12 +5,11 @@ import (
"fmt"
"maps"
"slices"
"strconv"
"strings"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
)
// **A build waits for its revision; a build with no version is never made** (design 47 H H-A-155, H-A-156; H-A-141): a step
@@ -105,16 +104,13 @@ func (b *Builder) awaitRevision(ctx context.Context, run meshbuilds.StepRun) (ou
}
}
// **A package with no pin waits for its publish** (design 47 H H-A-192, H-A-165, H-A-183, H-A-184, H-A-185): an entry
// keeps a mesh package the registry has not published within the series its module accepts with an empty version. Its
// build, a step ask's or a lane's, first reads the versions the registry seat holds of the path (its verb versions,
// H-A-194), then waits for the registry seat's published of that package path in that series (the
// series the module's manifest names in `packages`, else the default; catalogue.Manifest.AcceptedSeries), up to
// PackageWait (the setting package-wait-minutes), and pins the version that event carries; the builder never derives a
// pin (no release number of the module that holds the package). Past the bound the build fails, naming the package, and
// nothing is built.
// **A package with no pin waits for its release** (design 47 H H-A-165, H-A-161): an entry keeps a mesh package none of
// whose versions is released with an empty version. Its step ask waits, as for its revision, for the release of the module
// that holds the package (the catalog's version-released, VersionReleased), up to PackageWait (the setting
// package-wait-minutes), then builds with that release as the package's pin (v<version>); past the bound the step fails,
// naming the package, and nothing is built.
// DefaultPackageWait is the bound of a build's wait for a package's publish, its setting package-wait-minutes' default.
// DefaultPackageWait is the bound of a step ask's wait for a package's release, its setting package-wait-minutes' default.
const DefaultPackageWait = 10 * time.Minute
func (b *Builder) packageWait() time.Duration {
@@ -124,152 +120,91 @@ func (b *Builder) packageWait() time.Duration {
return DefaultPackageWait
}
// PackagePublished takes the Go package registry seat's published of one package path in (its name and the version the
// event carries, H-A-192): the newest published in the version's series is kept, and a build waiting for the package
// wakes. A version that is no MAJOR.MINOR.PATCH (a group's tagged version) pins nothing.
func (b *Builder) PackagePublished(path, version string) {
v := strings.TrimPrefix(version, "v")
series, ok := seriesOfVersion(v)
if path == "" || !ok {
return
}
// VersionReleased takes the catalog's version-released of module in: a step ask waiting for a package of it wakes.
func (b *Builder) VersionReleased(module, version string) {
b.init()
b.mu.Lock()
in := b.publishedPins[path]
if in == nil {
in = map[string]string{}
b.publishedPins[path] = in
}
if old := in[series]; old == "" || compareVersions(v, old) > 0 {
in[series] = v
}
b.releases[module] = version
b.mu.Unlock()
b.pictureMoved()
}
// publishedPin is the pin the registry seat's published gave the package path in the series, as Go names it; "" for none.
func (b *Builder) publishedPin(path, series string) string {
// releaseOf is the pin a package is built against: the release of the module holding it, as Go names it; "" for none yet.
func (b *Builder) releaseOf(ctx context.Context, pkg string) string {
if b.Members == nil {
return ""
}
module, _, found, err := b.Members(ctx, pkg)
if err != nil || !found {
return ""
}
b.mu.Lock()
v := b.publishedPins[path][series]
v := b.releases[module]
b.mu.Unlock()
if v == "" && b.NewestReleased != nil {
if kept, ok, err := b.NewestReleased(ctx, module); err == nil && ok {
v = kept
}
}
if v == "" {
return ""
}
return "v" + v
}
// awaitPublished fills each package of packages with no pin from the published of its path within the series the manifest
// with this digest accepts, waiting for one (H-A-192, H-A-165). It answers the packages with every pin, or an outcome and
// its words: failed past the bound naming the package, could not run when the manifest cannot be read or the builder stops.
func (b *Builder) awaitPublished(ctx context.Context, module, fp, digest string, packages map[string]string) (map[string]string, string, string) {
// awaitPins fills each package entry names with no pin from its release, waiting for one; no outcome once every package has
// a pin, failed past the bound, could not run at the builder's stop.
func (b *Builder) awaitPins(ctx context.Context, entry *meshcatalog.CatalogEntry) (outcome, said string) {
var unpinned []string
for p, v := range packages {
for p, v := range entry.Packages {
if v == "" {
unpinned = append(unpinned, p)
}
}
if len(unpinned) == 0 {
return packages, "", ""
return "", ""
}
slices.Sort(unpinned)
if b.Manifests == nil {
return nil, meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("the series %s accepts of %s cannot be read: no manifests", module, unpinned[0])
}
man, err := b.Manifests.Manifest(ctx, digest)
if err != nil {
return nil, meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("the series %s accepts of %s cannot be read: its manifest %s: %v", module, unpinned[0], digest, err)
}
// What the registry seat holds is read once before the wait (H-A-194): the publishes heard are a cache of it.
for _, p := range unpinned {
if b.PublishedVersions == nil || b.publishedPin(p, man.AcceptedSeries(p)) != "" {
continue
}
vs, err := b.PublishedVersions(ctx, p)
if err != nil {
b.logf("%s at %s: the versions the registry seat holds of %s cannot be read, so the build waits for its publish: %v", module, fp, p, err)
continue
}
for _, v := range vs {
b.PackagePublished(p, v)
}
}
bound := b.packageWait()
wait, stop := context.WithTimeout(ctx, bound) // a context's bound, no timer (ADR 0437 §8)
defer stop()
pins := map[string]string{}
told := false
for {
b.mu.Lock()
moved := b.moved
b.mu.Unlock()
pins := map[string]string{}
missing, missingSeries := "", ""
missing := ""
for _, p := range unpinned {
series := man.AcceptedSeries(p)
pins[p] = b.publishedPin(p, series)
if pins[p] == "" {
pins[p] = b.releaseOf(ctx, p)
}
if pins[p] == "" && missing == "" {
missing, missingSeries = p, series
missing = p
}
}
if missing == "" {
out := maps.Clone(packages)
maps.Copy(out, pins)
packages := maps.Clone(entry.Packages)
maps.Copy(packages, pins)
entry.Packages = packages
if told {
b.logf("%s at %s: every package it names is published: built with %v", module, fp, pins)
b.logf("%s at %s: every package it names has a released version: built with %v", entry.Module, entry.Fp, pins)
}
return out, "", ""
return "", ""
}
notReleased := fmt.Sprintf("package %s has no released version in the series %s", missing, missingSeries)
notReleased := fmt.Sprintf("package %s has no released version", missing)
if !told {
b.logf("%s at %s: %s yet: the build waits for its publish, at most %s", module, fp, notReleased, bound)
b.logf("%s at %s: %s yet: the ask waits for its release, at most %s", entry.Module, entry.Fp, notReleased, bound)
told = true
}
select {
case <-moved:
case <-wait.Done():
if ctx.Err() != nil {
return nil, meshbuilds.StepEndedOutcomeCouldNotRun, notReleased + ": the builder stopped before it was"
return meshbuilds.StepEndedOutcomeCouldNotRun, notReleased + ": the builder stopped before it was"
}
b.logf("%s at %s: %s within %s: the build fails, nothing is built", module, fp, notReleased, bound)
return nil, meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s within %s", notReleased, bound)
b.logf("%s at %s: %s within %s: the step fails, nothing is built", entry.Module, entry.Fp, notReleased, bound)
return meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s within %s", notReleased, bound)
}
}
}
// seriesOfVersion is a MAJOR.MINOR.PATCH version's series, MAJOR.MINOR; false for any other form.
func seriesOfVersion(v string) (string, bool) {
if _, ok := versionParts(v); !ok {
return "", false
}
major, rest, _ := strings.Cut(v, ".")
minor, _, _ := strings.Cut(rest, ".")
return major + "." + minor, true
}
// versionParts are a MAJOR.MINOR.PATCH version's numbers; false for any other form.
func versionParts(v string) ([3]int, bool) {
var out [3]int
parts := strings.Split(v, ".")
if len(parts) != 3 {
return out, false
}
for i, p := range parts {
n, err := strconv.Atoi(p)
if err != nil || n < 0 {
return out, false
}
out[i] = n
}
return out, true
}
// compareVersions orders two MAJOR.MINOR.PATCH versions numerically.
func compareVersions(a, b string) int {
x, _ := versionParts(a)
y, _ := versionParts(b)
for i := range x {
if x[i] != y[i] {
return x[i] - y[i]
}
}
return 0
}
@@ -113,13 +113,11 @@ type Builder struct {
// RevisionWait bounds a step ask's wait for its revision, its setting revision-wait-minutes (H-A-156); DefaultRevisionWait
// when 0.
RevisionWait time.Duration
// PackageWait bounds a build's wait for the publish of a package its entry names with no pin, its setting
// package-wait-minutes (H-A-165); DefaultPackageWait when 0. Only a registry seat's published heard (PackagePublished)
// gives a pin (H-A-192).
PackageWait time.Duration
// PublishedVersions asks the Go package registry seat's verb versions for one package path, its versions newest first
// (H-A-194): read once before a build waits for that package. Nil reads none.
PublishedVersions func(ctx context.Context, path string) ([]string, error)
// PackageWait bounds a step ask's wait for the release of a package its entry names with no pin, its setting
// package-wait-minutes (H-A-165); DefaultPackageWait when 0. NewestReleased reads the newest version the builder's
// record of releases keeps for a module; nil reads none, and only a release heard (VersionReleased) gives a pin.
PackageWait time.Duration
NewestReleased func(ctx context.Context, module string) (version string, found bool, err error)
// Forms answers what each node's engine states it reads (ADR 0618 §2, ADR 0624 §1): present, the nodes whose engine
// answered when last asked, and counted, every node the assignments names, present or away, a node with no statement
// stating nothing (the baseline's first forms). A done key names each artifact in the forms these keep, the oldest in
@@ -164,9 +162,8 @@ type Builder struct {
builds map[string]context.CancelCauseFunc
// moved is closed and made again at each entry the picture takes in or drops: the step asks waiting for a revision wake.
moved chan struct{}
// publishedPins are the newest version a registry seat said published, by package path and series (PackagePublished):
// a package with no pin waits for one (H-A-192).
publishedPins map[string]map[string]string
// releases are the versions heard released, by module (VersionReleased): a package with no pin waits for one.
releases map[string]string
}
type planned struct {
@@ -197,7 +194,7 @@ func (b *Builder) init() {
b.hands, b.events = map[string]*handWatch{}, map[string]bool{}
b.builds = map[string]context.CancelCauseFunc{}
b.moved = make(chan struct{})
b.publishedPins = map[string]map[string]string{}
b.releases = map[string]string{}
})
}
@@ -288,19 +285,16 @@ func (b *Builder) take(ctx context.Context, o Origin, key string, m meshcatalog.
// ModuleOf is a catalog entry in the form the builder builds by: its fingerprint, manifest digest, source (repository,
// directory, commit; no source stamp, design 47 H H-A-48), the toolchains its fingerprint was computed with, the bases it is
// built against, its packages and needs, and the named paths of its artifacts when every artifact names them.
// built against, its packages and needs, and the named paths of its artifacts.
func ModuleOf(e meshcatalog.CatalogEntry) meshcatalog.ModulesByModuleValue {
m := meshcatalog.ModulesByModuleValue{Repository: e.Repository, Path: e.Path, Commit: e.Commit, Fp2: e.Fp, None2: e.None,
Manifest: meshcatalogclient.BareManifestDigest(e.ManifestDigest), Packages: e.Packages, Needs: e.Needs}
Manifest: meshcatalog.BareManifestDigest(e.ManifestDigest), Packages: e.Packages, Needs: e.Needs}
if len(e.Bases) > 0 {
m.Inputs2 = &meshcatalog.ModulesByModuleValueInputs2{Bases: e.Bases}
}
if in := e.Inputs; in != nil {
m.Toolchains, m.Problems = in.Toolchains, in.Problems
// Only a module whose every artifact names its paths is built from those alone (ADR 0499 clauses 1, 2 and 4): an
// artifact without paths, such as a Go program, keeps its module's directory as its build source, so a module
// with one names no paths and its fetch keeps the directory whole.
if len(in.Paths) > 0 && everyArtifactNamesPaths(e) {
if len(in.Paths) > 0 {
if m.Inputs2 == nil {
m.Inputs2 = &meshcatalog.ModulesByModuleValueInputs2{}
}
@@ -315,20 +309,6 @@ func ModuleOf(e meshcatalog.CatalogEntry) meshcatalog.ModulesByModuleValue {
return m
}
// everyArtifactNamesPaths is whether the entry names artifacts and the named paths of each, by its reference or its name.
func everyArtifactNamesPaths(e meshcatalog.CatalogEntry) bool {
if len(e.Artifacts) == 0 || e.Inputs == nil {
return false
}
for _, a := range e.Artifacts {
_, name, _ := strings.Cut(a.Ref, ":")
if len(e.Inputs.Paths[a.Ref]) == 0 && len(e.Inputs.Paths[name]) == 0 {
return false
}
}
return true
}
// originOf is the catalog revision the entry of a build key came from; a key no entry named is of the first branch, as a build
// of the trunk's value was before branches were watched.
func (b *Builder) originOf(key string) Origin {
@@ -1132,15 +1112,9 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m mes
ask := sdkstep.StepAsk{ID: sdkstep.NewID(), Kind: kind, Language: language, Asker: b.OwnModule, At: time.Now(), Version: l.version, Build: job}
if language == sdkstep.AskGo && job.Fetch != nil && len(m.Packages) > 0 {
// The versions of the mesh packages the fingerprint was computed with (novox/hq ADR 0573 §2): the build module writes
// them into the workspace, whatever the repository's go.mod names. A package the entry names with no pin waits for
// its publish and is built with the version that publish carries; past the bound the lane fails naming it, and no
// ask is put (H-A-192, H-A-165).
pinned, outcome, said := b.awaitPublished(ctx, module, m.Fp2, m.Manifest, m.Packages)
if outcome != "" {
l.err = errors.New(said)
return
}
ask.Packages = maps.Clone(pinned)
// them into the workspace, whatever the repository's go.mod names.
ask.Packages = maps.Clone(m.Packages)
maps.DeleteFunc(ask.Packages, func(_, v string) bool { return v == "" }) // a package with no released version is no pin (H-A-161)
for _, fb := range req.Builds {
delete(ask.Packages, fb.Package) // a package `builds` names is named in neither packages nor package (§465)
}
@@ -333,10 +333,8 @@ func TestATrunkFingerprintIsOneAskOnTheDeliveryStream(t *testing.T) {
r := newRig(t, goBundle)
r.asks.Answer = nil // no node has pulled it yet
// Issue 1585 (ADR 0573 §2): the ask carries the module's pins; a package the entry keeps with no version (none
// published, design 47 H-A-161) is pinned at the version the registry seat's published carries (H-A-192), never
// asked without it.
r.b.PackagePublished("git.novox.be/novox/mesh-contracts/go/seats/x", "v0.1.7")
sdk := buildstep.PackageVersions{"git.novox.be/novox/mesh-sdk/go": "v0.1.20", "git.novox.be/novox/mesh-contracts/go/seats/x": "v0.1.7"}
// released, design 47 H-A-161) is no pin.
sdk := buildstep.PackageVersions{"git.novox.be/novox/mesh-sdk/go": "v0.1.20"}
data, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20",
Fp2: fp1, Manifest: mdigest('a'), Path: "modules/m", Toolchains: CatalogToolchains(agreedToolchains),
Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.20", "git.novox.be/novox/mesh-contracts/go/seats/x": ""}})
@@ -6,7 +6,6 @@ import (
"strings"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
)
@@ -64,12 +63,10 @@ func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry) (outc
if entry.Module != e.Module || (e.Commit != "" && entry.Commit != e.Commit) {
return meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("the entry of %s at %s is of %s at %s", e.Module, e.Fingerprint, entry.Module, entry.Commit)
}
// A package the entry names with no pin is built against its publish, waited for (H-A-192, H-A-165).
pinned, outcome, said := b.awaitPublished(ctx, entry.Module, entry.Fp, meshcatalogclient.BareManifestDigest(entry.ManifestDigest), entry.Packages)
if outcome != "" {
// A package the entry names with no pin is built against its release, waited for (H-A-165).
if outcome, said := b.awaitPins(ctx, &entry); outcome != "" {
return outcome, said
}
entry.Packages = pinned
key, err := builds.BuilderBuildKey(entry.Module, entry.Fp)
if err != nil {
return meshbuilds.StepEndedOutcomeCouldNotRun, err.Error()
@@ -82,8 +79,8 @@ func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry) (outc
}
return meshbuilds.StepEndedOutcomeFailed, failedWords
}
m := meshcatalog.ModulesByModuleValue{Repository: entry.Repository, Commit: entry.Commit, Fp2: entry.Fp, Manifest: meshcatalogclient.BareManifestDigest(entry.ManifestDigest), Path: entry.Path,
Toolchains: CatalogToolchains(b.agreedFor(ctx, meshcatalogclient.BareManifestDigest(entry.ManifestDigest))), Packages: entry.Packages}
m := meshcatalog.ModulesByModuleValue{Repository: entry.Repository, Commit: entry.Commit, Fp2: entry.Fp, Manifest: meshcatalog.BareManifestDigest(entry.ManifestDigest), Path: entry.Path,
Toolchains: CatalogToolchains(b.agreedFor(ctx, meshcatalog.BareManifestDigest(entry.ManifestDigest))), Packages: entry.Packages}
b.consider(ctx, entry.Module, m)
select {
case log := <-ended:
@@ -114,7 +111,7 @@ func (b *Builder) passedWithItsVersion(ctx context.Context, entry meshcatalog.Ca
if !reserving {
return meshbuilds.StepEndedOutcomePassed, ""
}
plan, planErr, _ := b.planOf(ctx, meshcatalogclient.BareManifestDigest(entry.ManifestDigest))
plan, planErr, _ := b.planOf(ctx, meshcatalog.BareManifestDigest(entry.ManifestDigest))
switch {
case planErr != nil:
return meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("%s at %s: the plan cannot be read to judge its version: %v", entry.Module, entry.Fp, planErr)
@@ -3,10 +3,9 @@ package build
import (
"context"
"errors"
"fmt"
"reflect"
"slices"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
@@ -122,155 +121,76 @@ func TestABuildStepPassesAReservingBuildOnlyWithItsReservedVersion(t *testing.T)
t.Run("H-A-162: a revision held that changes no entry wakes the ask", aRevisionHeldWakesTheAsk)
t.Run("H-A-165: a package with no pin waits for its release and is built with it", aPackageWithNoPinWaitsForItsRelease)
t.Run("H-A-165: past its bound a package with no release fails the step", aPackageWithNoReleaseFailsPastItsBound)
t.Run("H-A-192: a lane waits for an unpinned package's publish and fails past its bound", aLaneWaitsForAnUnpinnedPackage)
t.Run("H-A-194: what the registry seat holds pins the package before any wait", theSeatsVersionsPinBeforeTheWait)
}
// unpinnedRig is a rig whose step entry m names the package x with no version. Its manifest names no `packages`, so it
// accepts x in the default series 0.1. waiting is told when a build starts its wait for a package's publish.
func unpinnedRig(t *testing.T) (*rig, meshcatalog.CatalogEntry, <-chan struct{}) {
// unpinnedRig is a rig whose step entry m names the package x with no version, built by the module contracts.
func unpinnedRig(t *testing.T) (*rig, *fakeReservations) {
var held atomic.Int64
r, _, entry := revisionRig(t, &held, make(chan struct{}, 1))
r, res, entry := revisionRig(t, &held, make(chan struct{}, 1))
r.b.Picture = nil // no revision to wait for here
entry.Packages = map[string]string{unpinnedX: "", "git.novox.be/novox/mesh-sdk/go": "v0.1.20"}
entry.Packages = map[string]string{"git.novox.be/novox/mesh-contracts/go/seats/x": "", "git.novox.be/novox/mesh-sdk/go": "v0.1.20"}
r.b.StepEntry = func(context.Context, string) (meshcatalog.CatalogEntry, error) { return entry, nil }
waiting := make(chan struct{}, 1)
r.b.Logf = func(format string, args ...any) {
t.Logf(format, args...)
if strings.Contains(fmt.Sprintf(format, args...), "waits for its publish") {
select {
case waiting <- struct{}{}:
default:
}
}
r.b.Members = func(_ context.Context, pkg string) (string, string, bool, error) {
return "contracts", "contracts:x", pkg == "git.novox.be/novox/mesh-contracts/go/seats/x", nil
}
return r, entry, waiting
return r, res
}
const unpinnedX = "git.novox.be/novox/mesh-contracts/go/seats/x"
// awaitWaiting fails the test unless the build started its wait before ended said anything.
func awaitWaiting[T any](t *testing.T, waiting <-chan struct{}, ended <-chan T) {
t.Helper()
select {
case <-waiting:
case end := <-ended:
t.Fatalf("the build ended before the package was published: %+v", end)
case <-time.After(5 * time.Second):
t.Fatal("the build never waited for the package's publish")
}
}
// H-A-192: the pin is the version the registry seat's published carries for the path in the accepted series; a publish in
// another series and the module's release number pin nothing.
func aPackageWithNoPinWaitsForItsRelease(t *testing.T) {
r, _, waiting := unpinnedRig(t)
r.b.PackagePublished(unpinnedX, "v0.2.9") // another series than the 0.1 the module accepts: no pin
r.b.PackagePublished(unpinnedX, "0.1.4-feat-x.2") // a group's tagged version: no pin
r, _ := unpinnedRig(t)
var once sync.Once
looked := make(chan struct{})
r.b.NewestReleased = func(context.Context, string) (string, bool, error) { // the record holds none yet
once.Do(func() { close(looked) })
return "", false, nil
}
ended := make(chan meshbuilds.StepEnded, 1)
go func() {
ended <- r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
}()
awaitWaiting(t, waiting, ended)
if _, ok := r.st.vals[r.key]; ok {
t.Fatal("the build was made before the package was published")
select {
case <-looked:
case end := <-ended:
t.Fatalf("the ask ended before the package was released: %+v, asks %+v", end, r.asks.Asks())
case <-time.After(5 * time.Second):
t.Fatal("the ask never looked for the package's release")
}
r.b.PackagePublished(unpinnedX, "v0.1.3")
if _, ok := r.st.vals[r.key]; ok {
t.Fatal("the build was made before the package had a release")
}
r.b.VersionReleased("contracts", "0.1.3")
var end meshbuilds.StepEnded
select {
case end = <-ended:
case <-time.After(5 * time.Second):
t.Fatal("the publish did not wake the ask")
t.Fatal("the release did not wake the ask")
}
r.b.Wait()
asks := r.asks.Asks()
if end.Outcome != meshbuilds.StepEndedOutcomePassed || len(asks) == 0 ||
!reflect.DeepEqual(asks[0].Packages, buildstep.PackageVersions{unpinnedX: "v0.1.3", "git.novox.be/novox/mesh-sdk/go": "v0.1.20"}) {
!reflect.DeepEqual(asks[0].Packages, buildstep.PackageVersions{"git.novox.be/novox/mesh-contracts/go/seats/x": "v0.1.3", "git.novox.be/novox/mesh-sdk/go": "v0.1.20"}) {
t.Errorf("end = %+v, asks %+v; want passed, built with x at v0.1.3", end, asks)
}
}
func aPackageWithNoReleaseFailsPastItsBound(t *testing.T) {
r, _, _ := unpinnedRig(t)
r, _ := unpinnedRig(t)
r.b.PackageWait = time.Nanosecond
end := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
r.b.Wait()
if _, built := r.st.vals[r.key]; end.Outcome != meshbuilds.StepEndedOutcomeFailed || built ||
!strings.Contains(end.Said, "package "+unpinnedX+" has no released version in the series 0.1") {
!strings.Contains(end.Said, "package git.novox.be/novox/mesh-contracts/go/seats/x has no released version") {
t.Errorf("end = %+v, built %v; want failed naming the package, nothing built", end, built)
}
// A publish heard before gives the pin at once.
r.b.PackagePublished(unpinnedX, "v0.1.2")
// A release the builder's own record keeps gives the pin at once.
r.b.NewestReleased = func(_ context.Context, module string) (string, bool, error) {
return "0.1.2", module == "contracts", nil
}
if end := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}}); strings.Contains(end.Said, "no released version") {
t.Errorf("a publish heard did not pin the package: %+v", end)
}
}
// H-A-194: the build reads the registry seat's versions of the path once before it waits: a version in the accepted series
// pins the package with no event heard; one only in another series leaves the build waiting for a publish.
func theSeatsVersionsPinBeforeTheWait(t *testing.T) {
r, _, _ := unpinnedRig(t)
r.b.PackageWait = time.Nanosecond // no wait: the seat's answer alone gives the pin
var read []string
r.b.PublishedVersions = func(_ context.Context, path string) ([]string, error) {
read = append(read, path)
return []string{"0.2.1", "0.1.9", "0.1.8"}, nil // newest first
}
end := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
r.b.Wait()
asks := r.asks.Asks()
if end.Outcome != meshbuilds.StepEndedOutcomePassed || len(asks) == 0 || asks[0].Packages[unpinnedX] != "v0.1.9" || !slices.Equal(read, []string{unpinnedX}) {
t.Errorf("end = %+v, asks %+v, read %v; want passed, built with x at v0.1.9 read once from the seat", end, asks, read)
}
r2, _, waiting := unpinnedRig(t)
r2.b.PublishedVersions = func(context.Context, string) ([]string, error) { return []string{"0.2.1"}, nil }
ended := make(chan meshbuilds.StepEnded, 1)
go func() {
ended <- r2.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
}()
awaitWaiting(t, waiting, ended)
r2.b.PackagePublished(unpinnedX, "0.1.10")
if end := <-ended; end.Outcome != meshbuilds.StepEndedOutcomePassed {
t.Errorf("end = %+v, want passed once x was published in its series", end)
}
r2.b.Wait()
}
// H-A-192, H-A-165: the lane of an entry taken in waits the same way, never asking a build without the pin, and fails the
// key past the bound naming the package.
func aLaneWaitsForAnUnpinnedPackage(t *testing.T) {
r, entry, waiting := unpinnedRig(t)
done := make(chan struct{})
go func() {
r.b.Entry(t.Context(), Origin{Branch: "main", Source: 41, First: true}, entry)
r.b.Wait()
close(done)
}()
awaitWaiting(t, waiting, done)
if asks := r.asks.Asks(); len(asks) != 0 {
t.Fatalf("a build was asked without the package's pin: %+v", asks)
}
r.b.PackagePublished(unpinnedX, "0.1.5")
select {
case <-done:
case <-time.After(5 * time.Second):
t.Fatal("the publish did not wake the lane")
}
if asks := r.asks.Asks(); len(asks) == 0 || asks[0].Packages[unpinnedX] != "v0.1.5" {
t.Errorf("asks %+v; want one asked with x at v0.1.5", asks)
}
// Past its bound the key fails naming the package, and nothing is asked.
r2, entry2, _ := unpinnedRig(t)
r2.b.PackageWait = time.Nanosecond
r2.b.Entry(t.Context(), Origin{Branch: "main", Source: 41, First: true}, entry2)
r2.b.Wait()
if asks := r2.asks.Asks(); len(asks) != 0 || !strings.Contains(string(r2.st.vals[r2.key]), "package "+unpinnedX+" has no released version") {
t.Errorf("asks %+v, key %s; want none asked and the key failed naming the package", asks, r2.st.vals[r2.key])
t.Errorf("a release the record keeps did not pin the package: %+v", end)
}
}
@@ -295,6 +295,23 @@ func (r *Reserver) ReleasedFor(ctx context.Context, module, fingerprint string)
return false, nil
}
// NewestReleased is the newest version the builder's own record of releases keeps for module (design 47 H H-A-165): a
// package with no pin is built against it; false when the record holds none.
func (r *Reserver) NewestReleased(ctx context.Context, module string) (string, bool, error) {
if r.Releases == nil {
return "", false, nil
}
raw, _, found, err := r.Releases.Get(ctx, ReleasedKey(module))
if err != nil || !found {
return "", false, err
}
var rec ReleasedRecord
if json.Unmarshal(raw, &rec) != nil || len(rec.Kept) == 0 {
return "", false, nil
}
return rec.Kept[len(rec.Kept)-1].Version, true, nil
}
// keepRelease adds a release to the module's record, dropping the oldest past ReleasedKept.
func (r *Reserver) keepRelease(ctx context.Context, module, fingerprint, version string) error {
n := r.ReleasedKept
@@ -201,11 +201,13 @@ func TestAReleaseIsRecordedByFingerprintBeforeItsReservationGoes(t *testing.T) {
t.Errorf("reservations left: %v", all)
}
// One key per module holds the last releases, the oldest dropped (A-456b).
newest := ""
for _, fp := range []string{"src2:b1", "src2:b2"} {
n, err := r.Reserve(ctx, "m", "0.1", []string{"m:a"}, Source{Fingerprint: fp})
if err != nil {
t.Fatal(err)
}
newest = n.Version
if _, err := r.Released(ctx, "m", n.Version); err != nil {
t.Fatal(err)
}
@@ -219,6 +221,13 @@ func TestAReleaseIsRecordedByFingerprintBeforeItsReservationGoes(t *testing.T) {
if ok, _ := r.ReleasedFor(ctx, "m", "src2:b2"); !ok {
t.Error("the newest release is not kept")
}
// H-A-165: the newest version the record keeps is the pin of a package with none.
if v, ok, err := r.NewestReleased(ctx, "m"); !ok || err != nil || v != newest {
t.Errorf("newest released of m = %q, %v (%v), want %s", v, ok, err, newest)
}
if _, ok, _ := r.NewestReleased(ctx, "other"); ok {
t.Error("a module never released has a newest release")
}
}
// TestARebuildReservesAboveTheCatalogsHighestWhateverItsStandingNumberWas: design 47 G A-490b — a build again of a done key replaces
+4 -10
View File
@@ -266,16 +266,10 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1
if _, bus := p["bus"]; bus {
t.Errorf("the process declares a bus: it runs on its own broker credential")
}
// H-A-186, design 47 G §370: a Go package another module imports is a declared package artifact of its module, so every
// release publishes it: builds (ADR 0634 §1, ADR 0606 §4), imported by the build modules and the node-engine, and
// buildrun, imported by the indexer, the view and the watchman.
if len(m.Build.Artifacts) != 4 {
t.Fatalf("artifacts = %+v, want the program, the tools and the packages builds and buildrun", m.Build.Artifacts)
}
for i, name := range []string{"builds", "buildrun"} {
if a := m.Build.Artifacts[2+i]; a.Name != name || a.Kind != "package" || a.Language != "go" || !reflect.DeepEqual(a.Paths, []string{name}) || a.Series != "0.1" {
t.Errorf("package artifact = %+v, want the Go package %s of series 0.1", a, name)
}
// Issue 1966's bootstrap: the package artifact builds is out of the manifest until the new builder runs; the follow-up
// puts it back with its assertion (ADR 0634 §1, ADR 0606 §4: three artifacts, the third the package builds).
if len(m.Build.Artifacts) != 2 {
t.Fatalf("artifacts = %+v, want the program and the tools", m.Build.Artifacts)
}
if a := m.Build.Artifacts[1]; a.Name != "tools" || a.Kind != "bundle" || a.Binary != "mesh-delivery-builder-tools" || !reflect.DeepEqual(a.Loads, []string{"mesh-delivery-builder-tools"}) {
t.Errorf("tools artifact = %+v", a) // 1945
-18
View File
@@ -334,24 +334,6 @@
"loads": [
"mesh-delivery-builder-tools"
]
},
{
"name": "builds",
"kind": "package",
"language": "go",
"paths": [
"builds"
],
"series": "0.1"
},
{
"name": "buildrun",
"kind": "package",
"language": "go",
"paths": [
"buildrun"
],
"series": "0.1"
}
]
}
@@ -1,22 +0,0 @@
package meshcatalogclient
import "testing"
// An entry's manifest-digest carries sha256: and the module's value and the key of `manifests` the 64 digits bare, one
// digest in two forms (design 47 G §578); the two helpers live in the seat's client, outside its contract's generated
// root (H-A-200).
func TestAManifestDigestIsPrefixedForTheEntryAndBareForTheKey(t *testing.T) {
const bare = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
for _, tc := range []struct{ in, entry, key string }{
{bare, "sha256:" + bare, bare},
{"sha256:" + bare, "sha256:" + bare, bare},
{"", "", ""},
} {
if got := EntryManifestDigest(tc.in); got != tc.entry {
t.Errorf("EntryManifestDigest(%q) = %q, want %q", tc.in, got, tc.entry)
}
if got := BareManifestDigest(tc.in); got != tc.key {
t.Errorf("BareManifestDigest(%q) = %q, want %q", tc.in, got, tc.key)
}
}
}
@@ -1,7 +1,4 @@
package meshcatalogclient
// The two helpers below are hand-written, so they live in the seat's client, never under its contract's generated
// root (design 47 H, H-A-200; ADR 0608 §5).
package meshcatalog
import "strings"
@@ -276,48 +276,6 @@ func DecodePipelinesByBranchRunRunValue(raw []byte) (PipelinesByBranchRunRunValu
return v, nil
}
// PipelinesByStepsPipelineMoverKey is the key of one value of the state pipelines under the form "steps.<pipeline>.<mover>".
func PipelinesByStepsPipelineMoverKey(pipeline, mover string) string {
return "steps." + pipeline + "." + mover
}
// ParsePipelinesByStepsPipelineMoverKey reads the parts of a key of the form "steps.<pipeline>.<mover>", in the order of the form; the last result is false for a key of another form.
func ParsePipelinesByStepsPipelineMoverKey(key string) (string, string, bool) {
rest := key
if !strings.HasPrefix(rest, "steps.") {
return "", "", false
}
rest = rest[6:]
v1, after1, found1 := strings.Cut(rest, ".")
if !found1 || v1 == "" {
return "", "", false
}
rest = after1
if rest == "" {
return "", "", false
}
v3 := rest
rest = ""
if rest != "" {
return "", "", false
}
return v1, v3, true
}
// DecodePipelinesByStepsPipelineMoverValue reads one value of the state pipelines under the form "steps.<pipeline>.<mover>", refusing a field PipelinesByStepsPipelineMoverValue does not know and any text after the value.
func DecodePipelinesByStepsPipelineMoverValue(raw []byte) (PipelinesByStepsPipelineMoverValue, error) {
var v PipelinesByStepsPipelineMoverValue
dec := json.NewDecoder(bytes.NewReader(raw))
dec.DisallowUnknownFields()
if err := dec.Decode(&v); err != nil {
return v, err
}
if dec.More() {
return v, errors.New("text after the value")
}
return v, nil
}
// QueueByQueueNextKey is the key of one value of the state queue under the form "queue.next".
func QueueByQueueNextKey() string { return "queue.next" }
@@ -565,10 +523,8 @@ type LogEntry struct {
Artifact string `json:"artifact,omitempty"`
At time.Time `json:"at"`
// the keys of the kept results the step wrote or carried, <level>.<digest>, which the runner ages by their last-read (design 47 G sections 330, 470)
Kept []string `json:"kept,omitempty"`
// the module of the mover whose step the entry is of (H-A-190)
Module string `json:"module,omitempty"`
Outcome string `json:"outcome,omitempty"`
Kept []string `json:"kept,omitempty"`
Outcome string `json:"outcome,omitempty"`
// the output's name in step-outputs
Output string `json:"output,omitempty"`
Pipeline string `json:"pipeline"`
@@ -665,8 +621,6 @@ type Pipeline struct {
// how often the pipeline has ended: 1 at its first end, one higher at each end again after a passing rerun, so the assigner takes a later end of the same source revision (design 47 G sections 29, 325)
EndCount int64 `json:"end-count,omitempty"`
Ended *time.Time `json:"ended,omitempty"`
// how many movers the pipeline has, set by list alone (H-A-191)
MoverCount int64 `json:"mover-count,omitempty"`
// the modules the pipeline moves, its take-in's entries (design 47 H §746)
Movers []Mover `json:"movers"`
Outcome string `json:"outcome"`
@@ -683,13 +637,11 @@ type Pipeline struct {
// the source revision of the watched branch; the pipeline is keyed by it (ADR 0666 section 9)
Source int64 `json:"source"`
Started *time.Time `json:"started,omitempty"`
// how many steps the pipeline's movers have, set by list alone, which answers the record with its steps empty (H-A-191)
StepCount int64 `json:"step-count,omitempty"`
// every mover's steps, each naming its mover, approval last; in the state pipelines a record's own are empty, each mover's kept under its own key (H-A-187)
// every mover's steps, each naming its mover, approval last
Steps []StepResult `json:"steps"`
// the modules a newer waiting take-in of the same branch took out of the pipeline, written with the movers by compare-and-set; at its placing the runner writes a mover's build-step why from it (design 47 H §748, H-A-107)
Superseded []string `json:"superseded,omitempty"`
// the step-log entries of the steps that ended and are not appended to the step log yet, in order; written with the step's end in one write and cleared when appended (design 47 G sections 486, 606); in the state pipelines a record's own are empty, each mover's kept under its own key beside its steps (H-A-189)
// the step-log entries of the steps that ended and are not appended to the step log yet, in order; written with the step's end in one write and cleared when appended (design 47 G sections 486, 606)
Unlogged []LogEntry `json:"unlogged,omitempty"`
// the layout version
V int64 `json:"v,omitempty"`
@@ -1074,8 +1026,6 @@ type StartAnswer struct {
// ShowInput is what show takes.
type ShowInput struct {
// a mover's module: the answer's pipeline then holds that mover's steps alone (H-A-187)
Mover string `json:"mover,omitempty"`
// the pipeline's key
Pipeline string `json:"pipeline"`
}
@@ -1364,14 +1314,6 @@ type PipelinesByBranchSourcePublishedPackageVersionValue = Pipeline
// PipelinesByBranchRunRunValue is one value of the state pipelines under the key <branch>.run.<run>.
type PipelinesByBranchRunRunValue = Pipeline
// PipelinesByStepsPipelineMoverValue is one value of the state pipelines under the key steps.<pipeline>.<mover>.
type PipelinesByStepsPipelineMoverValue struct {
// the mover's steps
Steps []StepResult `json:"steps"`
// the step-log entries of the mover's steps that ended and are not appended to the step log yet, in order (H-A-189)
Unlogged []LogEntry `json:"unlogged,omitempty"`
}
// QueueByQueueNextValue is one value of the state queue under the key queue.next.
type QueueByQueueNextValue = int64
@@ -86,17 +86,13 @@
},
{
"name": "show",
"description": "One pipeline whole: its record with every mover's steps, or one mover's steps alone when it names the mover, its step log and its outputs, with the stores' fill. Reads only.",
"description": "One pipeline whole: its steps, its step log and its outputs, with the stores' fill. Reads only.",
"input": {
"type": "object",
"properties": {
"pipeline": {
"type": "string",
"description": "the pipeline's key"
},
"mover": {
"type": "string",
"description": "a mover's module: the answer's pipeline then holds that mover's steps alone (H-A-187)"
}
},
"required": [
@@ -137,7 +133,7 @@
},
{
"name": "list",
"description": "The pipelines and runs of a module, a branch or a commit, newest first, with the store's fill: each pipeline's record with its movers and steps counted, never its steps (show gives those). Reads only.",
"description": "The pipelines and runs of a module, a branch or a commit, newest first, with the store's fill. Reads only.",
"input": {
"type": "object",
"properties": {
@@ -775,7 +771,7 @@
"name": "pipelines",
"template": "state",
"history": 1,
"description": "one key per pipeline, one per take-in, written by compare-and-set on its absence (design 47 H §746, §747), and one key per mover of a pipeline holding that mover's steps (H-A-187)",
"description": "one key per pipeline, one per take-in, written by compare-and-set on its absence (design 47 H §746, §747)",
"key-forms": [
{
"key": "<branch>.<source>",
@@ -797,32 +793,6 @@
"$ref": "#/defs/Pipeline"
},
"description": "a run on request's pipeline, one per run, its movers the named commits' modules; it delivers nothing (H-A-83)"
},
{
"key": "steps.<pipeline>.<mover>",
"value": {
"type": "object",
"properties": {
"steps": {
"type": "array",
"items": {
"$ref": "#/defs/StepResult"
},
"description": "the mover's steps"
},
"unlogged": {
"type": "array",
"items": {
"$ref": "#/defs/LogEntry"
},
"description": "the step-log entries of the mover's steps that ended and are not appended to the step log yet, in order (H-A-189)"
}
},
"required": [
"steps"
]
},
"description": "one mover's steps and unlogged step-log entries in a pipeline, its key the pipeline's and the mover's module escaped as a key's name, written by compare-and-set as the pipeline's record is; the record's own steps stay empty, so a record never grows with the catalog (H-A-187)"
}
]
},
@@ -1268,7 +1238,7 @@
"items": {
"$ref": "#/defs/StepResult"
},
"description": "every mover's steps, each naming its mover, approval last; in the state pipelines a record's own are empty, each mover's kept under its own key (H-A-187)"
"description": "every mover's steps, each naming its mover, approval last"
},
"started": {
"type": "string",
@@ -1283,15 +1253,7 @@
"items": {
"$ref": "#/defs/LogEntry"
},
"description": "the step-log entries of the steps that ended and are not appended to the step log yet, in order; written with the step's end in one write and cleared when appended (design 47 G sections 486, 606); in the state pipelines a record's own are empty, each mover's kept under its own key beside its steps (H-A-189)"
},
"mover-count": {
"type": "integer",
"description": "how many movers the pipeline has, set by list alone (H-A-191)"
},
"step-count": {
"type": "integer",
"description": "how many steps the pipeline's movers have, set by list alone, which answers the record with its steps empty (H-A-191)"
"description": "the step-log entries of the steps that ended and are not appended to the step log yet, in order; written with the step's end in one write and cleared when appended (design 47 G sections 486, 606)"
},
"published": {
"$ref": "#/defs/Published",
@@ -1651,10 +1613,6 @@
"pipeline": {
"type": "string"
},
"module": {
"type": "string",
"description": "the module of the mover whose step the entry is of (H-A-190)"
},
"step": {
"type": "string",
"enum": [
@@ -112,12 +112,6 @@ export declare function pipelinesByBranchRunRunKey(branch: string, run: string):
/** The parts of a key of the form "<branch>.run.<run>", or undefined for a key of another form. */
export declare function parsePipelinesByBranchRunRunKey(_key: string): { branch: string; run: string } | undefined;
/** The key of one value of the state pipelines under the form "steps.<pipeline>.<mover>". */
export declare function pipelinesByStepsPipelineMoverKey(pipeline: string, mover: string): string;
/** The parts of a key of the form "steps.<pipeline>.<mover>", or undefined for a key of another form. */
export declare function parsePipelinesByStepsPipelineMoverKey(_key: string): { pipeline: string; mover: string } | undefined;
/** The key of one value of the state queue under the form "queue.next". */
export declare function queueByQueueNextKey(): string;
@@ -192,8 +186,6 @@ export interface LogEntry {
"at": string;
/** the keys of the kept results the step wrote or carried, <level>.<digest>, which the runner ages by their last-read (design 47 G sections 330, 470) */
"kept"?: string[];
/** the module of the mover whose step the entry is of (H-A-190) */
"module"?: string;
"outcome"?: "running" | "waiting" | "passed" | "failed" | "nothing to run" | "bypassed" | "carried" | "could not run" | "waiting for a build node" | "did not build" | "accepted" | "declined" | "superseded" | "cancelled";
/** the output's name in step-outputs */
"output"?: string;
@@ -256,8 +248,6 @@ export interface Pipeline {
/** how often the pipeline has ended: 1 at its first end, one higher at each end again after a passing rerun, so the assigner takes a later end of the same source revision (design 47 G sections 29, 325) */
"end-count"?: number;
"ended"?: string;
/** how many movers the pipeline has, set by list alone (H-A-191) */
"mover-count"?: number;
/** the modules the pipeline moves, its take-in's entries (design 47 H §746) */
"movers": Mover[];
"outcome": "waiting" | "running" | "passed" | "failed" | "refused" | "superseded" | "cancelled";
@@ -274,13 +264,11 @@ export interface Pipeline {
/** the source revision of the watched branch; the pipeline is keyed by it (ADR 0666 section 9) */
"source": number;
"started"?: string;
/** how many steps the pipeline's movers have, set by list alone, which answers the record with its steps empty (H-A-191) */
"step-count"?: number;
/** every mover's steps, each naming its mover, approval last; in the state pipelines a record's own are empty, each mover's kept under its own key (H-A-187) */
/** every mover's steps, each naming its mover, approval last */
"steps": StepResult[];
/** the modules a newer waiting take-in of the same branch took out of the pipeline, written with the movers by compare-and-set; at its placing the runner writes a mover's build-step why from it (design 47 H §748, H-A-107) */
"superseded"?: string[];
/** the step-log entries of the steps that ended and are not appended to the step log yet, in order; written with the step's end in one write and cleared when appended (design 47 G sections 486, 606); in the state pipelines a record's own are empty, each mover's kept under its own key beside its steps (H-A-189) */
/** the step-log entries of the steps that ended and are not appended to the step log yet, in order; written with the step's end in one write and cleared when appended (design 47 G sections 486, 606) */
"unlogged"?: LogEntry[];
/** the layout version */
"v"?: number;
@@ -568,8 +556,6 @@ export interface StartAnswer {
/** is what show takes. */
export interface ShowInput {
/** a mover's module: the answer's pipeline then holds that mover's steps alone (H-A-187) */
"mover"?: string;
/** the pipeline's key */
"pipeline": string;
}
@@ -809,14 +795,6 @@ export type PipelinesByBranchSourcePublishedPackageVersionValue = Pipeline;
/** is one value of the state pipelines under the key <branch>.run.<run>. */
export type PipelinesByBranchRunRunValue = Pipeline;
/** is one value of the state pipelines under the key steps.<pipeline>.<mover>. */
export interface PipelinesByStepsPipelineMoverValue {
/** the mover's steps */
"steps": StepResult[];
/** the step-log entries of the mover's steps that ended and are not appended to the step log yet, in order (H-A-189) */
"unlogged"?: LogEntry[];
}
/** is one value of the state queue under the key queue.next. */
export type QueueByQueueNextValue = number;
@@ -173,27 +173,6 @@ export function parsePipelinesByBranchRunRunKey(_key) {
return { branch, run };
}
/** The key of one value of the state pipelines under the form "steps.<pipeline>.<mover>". */
export function pipelinesByStepsPipelineMoverKey(pipeline, mover) {
return "steps." + pipeline + "." + mover;
}
/** The parts of a key of the form "steps.<pipeline>.<mover>", or undefined for a key of another form. */
export function parsePipelinesByStepsPipelineMoverKey(_key) {
let _rest = _key;
if (!_rest.startsWith("steps.")) return undefined;
_rest = _rest.slice(6);
const _at1 = _rest.indexOf(".");
if (_at1 <= 0) return undefined;
const pipeline = _rest.slice(0, _at1);
_rest = _rest.slice(_at1 + 1);
if (_rest === "") return undefined;
const mover = _rest;
_rest = "";
if (_rest !== "") return undefined;
return { pipeline, mover };
}
/** The key of one value of the state queue under the form "queue.next". */
export function queueByQueueNextKey() {
return "queue.next";
@@ -187,13 +187,10 @@ func TestManifestProcess(t *testing.T) { // 1235/AC1
if _, bus := p["bus"]; bus {
t.Errorf("the process declares a bus: it runs on its own broker credential")
}
if len(m.Build.Artifacts) != 3 || m.Build.Artifacts[1].Name != "tools" || m.Build.Artifacts[1].From != "cmd/mesh-delivery-forge-bridge-tools" {
t.Fatalf("artifacts = %+v, want the program, the tools bundle and the package forgemsg", m.Build.Artifacts)
}
// ADR 0573 §1, H-A-186 (design 47 G §370): the gate, the builder and the indexer import forgemsg, so the module declares
// it as a package and every release publishes it.
if a := m.Build.Artifacts[2]; a.Name != "forgemsg" || a.Kind != "package" || a.Language != "go" || len(a.Paths) != 1 || a.Paths[0] != "forgemsg" || a.Series != "0.1" {
t.Errorf("package artifact = %+v", a)
// Issue 1966's bootstrap: the package artifact forgemsg is out of the manifest until the new builder runs; the follow-up
// puts it back with its assertion (ADR 0573 §1: three artifacts, the third the package forgemsg).
if len(m.Build.Artifacts) != 2 || m.Build.Artifacts[1].Name != "tools" || m.Build.Artifacts[1].From != "cmd/mesh-delivery-forge-bridge-tools" {
t.Fatalf("artifacts = %+v, want the program and the tools bundle", m.Build.Artifacts)
}
a := m.Build.Artifacts[0]
if st, err := os.Stat(a.From); err != nil || !st.IsDir() || a.Binary != "mesh-delivery-forge-bridge" || !strings.HasSuffix(a.From, a.Binary) {
@@ -101,15 +101,6 @@
"loads": [
"mesh-delivery-forge-bridge-tools"
]
},
{
"name": "forgemsg",
"kind": "package",
"language": "go",
"paths": [
"forgemsg"
],
"series": "0.1"
}
]
}
@@ -9,7 +9,6 @@ import (
"reflect"
"slices"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph"
"git.novox.be/novox/mesh-sdk/go/catalogue"
@@ -31,7 +30,7 @@ const CatalogEntryLayout = 1
// ModuleValueOf gives the module's value back from the entry alone. The acting rules are the revision's to add.
func CatalogEntryOf(module string, cur meshcatalog.ModulesByModuleValue, m catalogue.Manifest, raw string, n graph.Node) meshcatalog.CatalogEntry {
e := meshcatalog.CatalogEntry{V: CatalogEntryLayout, Module: module, Repository: cur.Repository, Path: cur.Path,
Commit: cur.Commit, Manifest: manifestWhole(raw), ManifestDigest: meshcatalogclient.EntryManifestDigest(cur.Manifest), Verdict: cur.Verdict, Fp: n.FP, None: n.None, Bases: n.Bases,
Commit: cur.Commit, Manifest: manifestWhole(raw), ManifestDigest: meshcatalog.EntryManifestDigest(cur.Manifest), Verdict: cur.Verdict, Fp: n.FP, None: n.None, Bases: n.Bases,
Packages: maps.Clone(cur.Packages), Provides: n.Provides, Requires: n.Requires, Awaits: n.Awaits, Invokes: n.Invokes,
Serves: n.Serves, LeftOut: n.LeftOut, Main: MainOf(m), Contributes: contributesOf(m)}
// requires names every requirement as catalogue.Manifest.Wants lists them, optional ones and those only contributes names
@@ -68,7 +67,7 @@ func CatalogEntryOf(module string, cur meshcatalog.ModulesByModuleValue, m catal
// ruling on milestone 47/G finding 1; ADR 0656 §6: one form of the fact). The old line's fields (the old-form
// fingerprint, the reader needs) are not kept.
func ModuleValueOf(e meshcatalog.CatalogEntry) meshcatalog.ModulesByModuleValue {
v := meshcatalog.ModulesByModuleValue{V: keys.Version, Repository: e.Repository, Path: e.Path, Commit: e.Commit, Manifest: meshcatalogclient.BareManifestDigest(e.ManifestDigest),
v := meshcatalog.ModulesByModuleValue{V: keys.Version, Repository: e.Repository, Path: e.Path, Commit: e.Commit, Manifest: meshcatalog.BareManifestDigest(e.ManifestDigest),
Verdict: e.Verdict, Fp2: e.Fp, None2: e.None, Packages: maps.Clone(e.Packages)}
if in := e.Inputs; in != nil {
v.Toolchains, v.Paths, v.Problems, v.Warnings, v.Unknown = maps.Clone(in.Toolchains), maps.Clone(in.Paths), in.Problems, in.Warnings, in.Unknown
@@ -544,7 +544,7 @@ func (k *KeptTrunk) reread(ctx context.Context, o Objects) error {
}
manifests := map[string]string{}
for _, e := range entries {
if d := meshcatalogclient.BareManifestDigest(e.ManifestDigest); d != "" && e.Manifest != nil {
if d := meshcatalog.BareManifestDigest(e.ManifestDigest); d != "" && e.Manifest != nil {
if raw, err := json.Marshal(e.Manifest); err == nil {
manifests[d] = string(raw)
}
@@ -10,7 +10,6 @@ import (
"testing"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/views"
)
@@ -95,7 +94,7 @@ func TestTheKeptTrunkIsReadAtTheStartAndOncePerHeardRevision(t *testing.T) {
}
// A-487c: the manifests by digest and the verb revision asked of the trunk's newest answer from the view held.
listed = objects.n.Load()
if raw, err := kept.Manifest(meshcatalogclient.BareManifestDigest(keptDigest("ca-trust"))); err != nil || raw != `{"module":"ca-trust"}` {
if raw, err := kept.Manifest(meshcatalog.BareManifestDigest(keptDigest("ca-trust"))); err != nil || raw != `{"module":"ca-trust"}` {
t.Errorf("the held manifest = %q (%v), want the entry's manifest", raw, err)
}
if a, err := Revision(context.Background(), objects, kept, meshcatalog.RevisionInput{}); err != nil || a.Revision.Revision != 2 || len(a.Entries) != 2 {
@@ -63,7 +63,6 @@ func (ix *Indexer) applyMoving(ctx context.Context, f whole, drop bool, moves ma
return w, err
}
objects, namedErr := ix.namedObjects(ctx, f)
trees := ix.goModTree(ctx, f)
overlay := map[string]meshcatalog.ModulesByModuleValue{}
dirOf := map[string]string{}
named, every := map[string]bool{}, true // every module the message's manifests name, and whether each names one
@@ -110,23 +109,7 @@ func (ix *Indexer) applyMoving(ctx context.Context, f whole, drop bool, moves ma
next.None2 = "the manifest cannot be read: it cannot be kept as a value"
} else {
packages := ix.goPackages(ctx, f, dir, cur, has)
if m, err := judge.ForInputs([]byte(raw)); err == nil && usesGo(languagesOf(m)) {
// The entry pins every mesh package of another repository the module's build requires, transitively across
// the same-repository modules of its graph (design 47 H-A-201).
packages = unionSorted(packages, trees.otherRequired(goModuleDirs(m, dir)))
}
in := inputsOfNamed(name, raw, f.Trees[dir], f.Locks[dir], objects[dir], packages, ix.baseFP(ctx))
if in.Kept != nil {
// A Go build resolves the same-repository modules its go.mod files require, transitively, from the tree: their
// directories' objects join the tree input of every form, never the entry's paths (design 47 H-A-202, H-A-172).
if m, err := judge.ForInputs([]byte(raw)); err == nil && usesGo(languagesOf(m)) {
if tree, none := ix.derivedTree(ctx, f, trees, m, dir, in.Kept.Tree); none != "" {
in.Kept, in.None2 = nil, none
} else {
in.Kept.Tree = tree
}
}
}
next.Inputs2, next.None2, next.Paths = in.Kept, in.None2, in.ArtifactPaths
if in.Kept != nil && has && cur.Inputs2 != nil {
// The imports its last trunk build reported stand until its next build reports again (ADR 0593 §2, §4).
@@ -1,196 +0,0 @@
package index
import (
"context"
"fmt"
"path"
"path/filepath"
"slices"
"strings"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"mesh-delivery-indexer/internal/fingerprint"
)
// goModTree reads the go.mod files of one merge's tree, each directory once (Indexer.GoMod), for the walk of the module
// graph a Go build resolves from the tree (design 47 H-A-202, H-A-157).
type goModTree struct {
ix *Indexer
ctx context.Context
f whole
read map[string][]byte // by directory from the repository's root; nil for none
}
func (ix *Indexer) goModTree(ctx context.Context, f whole) *goModTree {
return &goModTree{ix: ix, ctx: ctx, f: f, read: map[string][]byte{}}
}
// goMod is the go.mod of the directory dir (from the root, "" for the root) at the merge commit; nil when there is none
// or it cannot be read, said.
func (t *goModTree) goMod(dir string) []byte {
if raw, ok := t.read[dir]; ok {
return raw
}
var raw []byte
if t.ix.GoMod != nil {
var err error
if raw, err = t.ix.GoMod(t.ctx, t.f.Repository, t.f.Merge, dir); err != nil {
t.ix.logf("%s at %s: the go.mod of %q cannot be read, so no module of the tree is found there: %v", t.f.Repository, t.f.Merge, dir, err)
raw = nil
}
}
t.read[dir] = raw
return raw
}
// sameRepositoryDir is the directory of the merge's tree that holds the Go module required: the required path starts with
// the repository's module prefix, and the directory it names holds a go.mod declaring that path (H-A-198, H-A-202). false
// for a module of another repository, or one the tree holds no go.mod of.
func (t *goModTree) sameRepositoryDir(required string) (string, bool) {
repo, dir, ok := trunk.PackageRepository(required)
if !ok || dir == "" || !trunk.SameRepository(repo, t.f.Repository) {
return "", false
}
if goModModule(t.goMod(dir)) != required {
return "", false
}
return dir, true
}
// sameRepositoryRequired are the directories (from the root, sorted, each once) of every same-repository Go module the
// go.mod of each start directory requires, transitively through those modules' own go.mod files (H-A-202, H-A-157). The
// start directories themselves are not among them.
func (t *goModTree) sameRepositoryRequired(start []string) []string {
seen := map[string]bool{}
for _, s := range start {
seen[s] = true
}
queue := slices.Clone(start)
var out []string
for len(queue) > 0 {
dir := queue[0]
queue = queue[1:]
for _, req := range fingerprint.MeshRequiresOf(t.goMod(dir)) {
d, ok := t.sameRepositoryDir(req)
if !ok || seen[d] {
continue
}
seen[d] = true
out = append(out, d)
queue = append(queue, d)
}
}
slices.Sort(out)
return out
}
// goModuleDirs are the directories (from the root) a module's Go build starts from: its own directory and the directory
// of each path of each Go package artifact of its manifest m.
func goModuleDirs(m catalogue.Manifest, dir string) []string {
out := []string{dir}
if m.Build == nil {
return out
}
for _, a := range m.Build.Artifacts {
if a.Kind != catalogue.ArtifactPackage || a.Language != "go" {
continue
}
for _, p := range a.Paths {
if d := path.Join(dir, strings.TrimSuffix(filepath.ToSlash(p), "/")); !slices.Contains(out, d) {
out = append(out, d)
}
}
}
return out
}
// otherRequired are the mesh packages of other repositories (sorted, each once) that the go.mod of each start directory,
// and of every same-repository module those require transitively (sameRepositoryRequired), requires at any version, the
// zero version included: every one the module's build resolves, which the entry pins (design 47 H-A-201, H-A-183,
// H-A-157). A same-repository module is none: it is built from the tree (H-A-198, H-A-202). nil for none.
func (t *goModTree) otherRequired(start []string) []string {
var out []string
for _, d := range append(slices.Clone(start), t.sameRepositoryRequired(start)...) {
out = append(out, fingerprint.OtherPackages(fingerprint.MeshRequiresOf(t.goMod(d)), t.f.Repository)...)
}
if len(out) == 0 {
return nil
}
return slices.Compact(slices.Sorted(slices.Values(out)))
}
// unionSorted is a and b together, sorted and each once; nil for none.
func unionSorted(a, b []string) []string {
if len(a)+len(b) == 0 {
return nil
}
return slices.Compact(slices.Sorted(slices.Values(append(slices.Clone(a), b...))))
}
// derivedDirs are the same-repository module directories (from the root, sorted) the module's Go build resolves from the
// tree beyond its own tree input (H-A-202): every one its go.mod files require, transitively, that its tree input does
// not hold already, which is the module's directory for the directory's tree, and its named paths for the named tree.
func (t *goModTree) derivedDirs(m catalogue.Manifest, dir string) []string {
var held []string
if paths, every := m.NamedPaths(); every {
for _, p := range paths {
held = append(held, path.Join(dir, filepath.ToSlash(p)))
}
} else {
held = []string{dir}
}
var out []string
for _, d := range t.sameRepositoryRequired(goModuleDirs(m, dir)) {
if !slices.ContainsFunc(held, func(h string) bool { return h == "" || h == "." || d == h || strings.HasPrefix(d, h+"/") }) {
out = append(out, d)
}
}
return out
}
// DerivedLine opens each line of the tree input naming a derived directory and its git object (H-A-202, H-A-172).
const DerivedLine = "derived "
// derivedTree is the module's tree input with a line per derived directory and its object appended, sorted by directory
// (H-A-202, H-A-172: what a build reads moves its fingerprint), so a change of such a directory moves the fingerprint of
// every form. The objects are asked of the forge at the merge commit by their paths from the root (Named); an answer that
// lacks one gives the reason the module has no inputs, naming them. A module with no derived directory keeps its tree.
func (ix *Indexer) derivedTree(ctx context.Context, f whole, t *goModTree, m catalogue.Manifest, dir, tree string) (string, string) {
dirs := t.derivedDirs(m, dir)
if len(dirs) == 0 {
return tree, ""
}
if ix.Named == nil {
return "", "the forge cannot be asked the git objects of the derived directories " + strings.Join(dirs, ", ")
}
got, err := ix.Named(ctx, f.Repository, f.Merge, map[string][]string{"": dirs})
if err != nil {
return "", fmt.Sprintf("the git objects of the derived directories %s: %v", strings.Join(dirs, ", "), err)
}
lines := []string{tree}
var missing []string
for _, d := range dirs {
o := got[""][d]
if o == "" {
missing = append(missing, d)
continue
}
lines = append(lines, DerivedLine+d+" "+o)
}
if len(missing) > 0 {
return "", "the forge names no git object of the derived directories " + strings.Join(missing, ", ")
}
return strings.Join(lines, "\n"), ""
}
// goModModule is the module path a go.mod declares; "" for none.
func goModModule(raw []byte) string {
for _, line := range strings.Split(string(raw), "\n") {
if f := strings.Fields(line); len(f) == 2 && f[0] == "module" {
return strings.Trim(f[1], `"`)
}
}
return ""
}
@@ -1,144 +0,0 @@
package index
import (
"context"
"strings"
"testing"
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"mesh-delivery-indexer/internal/fingerprint"
)
// A fixture repository novox/mesh-delivery: the Go package state of modules/g requires the same-repository module
// modules/b/builds at the zero version, which requires modules/c/client in turn; mesh-sdk is of another repository, and
// modules/b/gone names a directory the tree holds no go.mod of.
const ptpZero = " v0.0.0-00010101000000-000000000000"
var ptpGoMods = map[string]string{
"modules/g/state": "module git.novox.be/novox/mesh-delivery/modules/g/state\n\nrequire (\n\tgit.novox.be/novox/mesh-delivery/modules/b/builds" + ptpZero +
"\n\tgit.novox.be/novox/mesh-sdk/go" + ptpZero + "\n\tgit.novox.be/novox/mesh-delivery/modules/b/gone" + ptpZero + "\n)\n",
"modules/b/builds": "module git.novox.be/novox/mesh-delivery/modules/b/builds\n\nrequire git.novox.be/novox/mesh-delivery/modules/c/client" + ptpZero + " // indirect\n",
"modules/c/client": "module git.novox.be/novox/mesh-delivery/modules/c/client\n",
}
// ptpProgram is modules/g's manifest: a program naming no paths beside the package state.
const ptpProgram = `{"module":"g","version":"1.0","build":{"artifacts":[{"name":"program","kind":"bundle","language":"go","system":"arch","from":"cmd/g","binary":"g","loads":["g"]},` +
`{"name":"state","kind":"package","language":"go","paths":["state"],"series":"0.1"}]}}`
// ptpIndexer answers the fixture's go.mod files and, for a path asked from the root, the object objects names for it.
func ptpIndexer(t *testing.T, objects map[string]string) (*Indexer, *memStore, *[]map[string][]string) {
ix, mods, _ := manifestsIndexer(t)
ix.GoMod = func(_ context.Context, _, _, dir string) ([]byte, error) {
if c, ok := ptpGoMods[dir]; ok {
return []byte(c), nil
}
return nil, nil
}
var asked []map[string][]string
ix.Named = func(_ context.Context, _, _ string, paths map[string][]string) (map[string]map[string]string, error) {
asked = append(asked, paths)
out := map[string]map[string]string{}
for dir, ps := range paths {
out[dir] = map[string]string{}
for _, p := range ps {
out[dir][p] = objects[p]
}
}
return out, nil
}
return ix, mods, &asked
}
// Design 47 H-A-202, H-A-172: the same-repository modules a Go build resolves from the tree, transitively, join the tree
// input of every form with their objects, so a change of one moves the fingerprint of a module whose program names no
// paths; the entry's paths stay the manifest's.
func TestADerivedDirectoryMovesTheFingerprintOfEveryForm(t *testing.T) {
objects := map[string]string{"modules/b/builds": strings.Repeat("b", 40), "modules/c/client": strings.Repeat("c", 40)}
tree := func(objects map[string]string) string {
ix, mods, asked := ptpIndexer(t, objects)
f := withManifests(mergedMsg("novox/mesh-delivery", map[string]string{"modules/g": "gtree"}), map[string]string{"modules/g": ptpProgram}, nil)
if err := ix.Apply(context.Background(), f); err != nil {
t.Fatal(err)
}
m, _ := moduleOf(t, mods, "g")
if got := strings.Join(m.Paths["state"], ","); got != "state" {
t.Errorf("the entry's paths of state are %q, want the manifest's", got)
}
if len(*asked) != 1 || strings.Join((*asked)[0][""], ",") != "modules/b/builds,modules/c/client" {
t.Fatalf("asked the forge %v, want the derived directories from the root once", *asked)
}
if m.Inputs2 == nil || !strings.HasPrefix(m.Inputs2.Tree, "gtree\n") {
t.Fatalf("inputs = %+v (%s), want the directory's tree with the derived lines", m.Inputs2, m.None2)
}
return m.Inputs2.Tree
}
before := tree(objects)
if want := "gtree\nderived modules/b/builds " + objects["modules/b/builds"] + "\nderived modules/c/client " + objects["modules/c/client"]; before != want {
t.Errorf("the tree input is %q, want %q", before, want)
}
objects["modules/c/client"] = strings.Repeat("d", 40)
after := tree(objects)
fp := func(tree string) string {
v, _ := fingerprint.Of(fingerprint.Inputs{Module: "g", Tree: tree})
return v
}
if fp(before) == fp(after) {
t.Errorf("a change of modules/c/client left the fingerprint %s", fp(after))
}
// The named tree takes them too: a module whose every artifact names its paths.
named := `{"module":"g","version":"1.0","build":{"artifacts":[{"name":"state","kind":"package","language":"go","paths":["state"],"series":"0.1"}]}}`
ix, mods, _ := ptpIndexer(t, objects)
f := withManifests(mergedMsg("novox/mesh-delivery", map[string]string{"modules/g": "gtree"}), map[string]string{"modules/g": named}, nil)
f.Paths = map[string]map[string]string{"modules/g": {"state": strings.Repeat("1", 40)}}
if err := ix.Apply(context.Background(), f); err != nil {
t.Fatal(err)
}
if m, _ := moduleOf(t, mods, "g"); m.Inputs2 == nil || !strings.HasPrefix(m.Inputs2.Tree, fingerprint.NamedTreePrefix) ||
!strings.HasSuffix(m.Inputs2.Tree, "\nderived modules/c/client "+objects["modules/c/client"]) || strings.Join(m.Paths["state"], ",") != "state" {
t.Errorf("module = %+v, want the named tree with the derived lines and the manifest's paths", m)
}
// A forge that names no object of a derived directory gives the module none, naming it.
ix, mods, _ = ptpIndexer(t, map[string]string{"modules/b/builds": strings.Repeat("b", 40)})
if err := ix.Apply(context.Background(), withManifests(mergedMsg("novox/mesh-delivery", map[string]string{"modules/g": "gtree"}),
map[string]string{"modules/g": ptpProgram}, nil)); err != nil {
t.Fatal(err)
}
if m, _ := moduleOf(t, mods, "g"); m.Fp2 != "" || !strings.Contains(m.None2, "modules/c/client") {
t.Errorf("module = %+v, want none naming the directory the forge gave no object of", m)
}
}
// Design 47 H-A-201: the entry pins every mesh package of another repository the module's build requires, across the
// same-repository modules of its graph: here the one modules/c/client requires, which modules/g's go.mod never names.
// A module of the module's own repository is never pinned, whatever a build reported (H-A-198, H-A-202).
func TestTheEntryPinsEveryRequirementOfTheModuleGraph(t *testing.T) {
const conditions = "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
objects := map[string]string{"modules/b/builds": strings.Repeat("b", 40), "modules/c/client": strings.Repeat("c", 40)}
ix, mods, _ := ptpIndexer(t, objects)
mod := map[string]string{
"modules/g": "module g\n\nrequire git.novox.be/novox/mesh-delivery/modules/b/builds" + ptpZero + "\n",
"modules/c/client": "module git.novox.be/novox/mesh-delivery/modules/c/client\n\nrequire " + conditions + ptpZero + "\n",
}
ix.GoMod = func(_ context.Context, _, _, dir string) ([]byte, error) {
if c, ok := mod[dir]; ok {
return []byte(c), nil
}
return []byte(ptpGoMods[dir]), nil
}
f := withManifests(mergedMsg("novox/mesh-delivery", map[string]string{"modules/g": "gtree"}), map[string]string{"modules/g": ptpProgram},
map[string]map[string]string{"modules/g": {"go.sum": avahiGoSum}})
if err := ix.Apply(context.Background(), f); err != nil {
t.Fatal(err)
}
m, _ := moduleOf(t, mods, "g")
if m.Inputs2 == nil || strings.Join(m.Inputs2.Packages, ",") != conditions+",git.novox.be/novox/mesh-sdk/go" {
t.Errorf("packages = %+v, want the client's requirement and state's mesh-sdk, and no module of the repository", m.Inputs2)
}
builds := "git.novox.be/novox/mesh-delivery/modules/b/builds"
pins := releasedPins(sdkstep.PackageVersions{builds: "v0.1.106", conditions: "v0.1.4"}, nil, catalogue.Manifest{}, "novox/mesh-delivery")
if _, pinned := pins[builds]; pinned || len(pins) != 1 {
t.Errorf("pins = %v, want %s only, never a module of the repository", pins, conditions)
}
}
@@ -298,7 +298,7 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc
r.fp, r.none = f.fp, f.none
if r.fp != "" {
// a pin is a released version, never a commit (H-A-161), and the version line's fingerprint takes it (H-A-167)
r.fp, r.pkgs = pinnedOf(f, series, k.manifest, k.cur.Repository)
r.fp, r.pkgs = pinnedOf(f, series, k.manifest)
r.tcs = toolchainsUsed(toolchains)
}
}
@@ -517,7 +517,7 @@ func stillComputed(module string, kept meshcatalog.ModulesByModuleValueInputs2,
if f.none != "" || f.fp == "" {
return "", nil, false
}
fp, pkgs := pinnedOf(f, series, k.manifest, k.held.Repository)
fp, pkgs := pinnedOf(f, series, k.manifest)
return fp, pkgs, fp == k.held.Fp2
}
@@ -15,7 +15,6 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/keys"
@@ -374,17 +373,13 @@ func isFloor(raw []byte) bool {
// within the series the module accepts, the one its manifest m names or the default (Manifest.AcceptedSeries), whether
// or not that series holds contents yet; a publish in another series pins nothing and moves nothing, and a module's
// release number is never a pin. A package with none published in that series is kept with no version, so its kind-1
// edge stands (H §746) and its build waits for the publish (H-A-165). A module of the module's own repository is never
// pinned, whatever a build reported: it is built from the tree (design 47 H-A-198, H-A-201, H-A-202). nil for none.
func releasedPins(versions sdkstep.PackageVersions, series map[string]SeriesPackage, m catalogue.Manifest, repository string) sdkstep.PackageVersions {
// edge stands (H §746) and its build waits for the publish (H-A-165). nil for none.
func releasedPins(versions sdkstep.PackageVersions, series map[string]SeriesPackage, m catalogue.Manifest) sdkstep.PackageVersions {
if len(versions) == 0 {
return nil
}
out := sdkstep.PackageVersions{}
for p := range versions {
if repo, _, ok := trunk.PackageRepository(p); ok && repository != "" && trunk.SameRepository(repo, repository) {
continue
}
out[p] = ""
for _, s := range series {
if s.GoModule != p || s.Newest == nil || s.Series != m.AcceptedSeries(s.Name) {
@@ -403,8 +398,8 @@ func releasedPins(versions sdkstep.PackageVersions, series map[string]SeriesPack
// form's fingerprint takes them (design 47 H-A-167, H-A-172; ADR 0573 §4) and a re-pin moves it, the module a mover
// built again against the release; a fingerprint with no pin is the form's as it is. The held fingerprint of a build that
// stands is kept while its pins are the ones it was built with, else its import form takes the new pins.
func pinnedOf(f form, series map[string]SeriesPackage, m catalogue.Manifest, repository string) (string, sdkstep.PackageVersions) {
pins := releasedPins(f.versions, series, m, repository)
func pinnedOf(f form, series map[string]SeriesPackage, m catalogue.Manifest) (string, sdkstep.PackageVersions) {
pins := releasedPins(f.versions, series, m)
if f.stands {
if maps.Equal(f.built, map[string]string(pins)) {
return f.fp, pins
@@ -42,8 +42,6 @@ type artifact struct {
Binary string `json:"binary"`
Loads []string `json:"loads"`
Env map[string]string `json:"env"`
Paths []string `json:"paths"`
Series string `json:"series"`
}
type stream struct {
@@ -346,12 +344,8 @@ func TestManifestProcess(t *testing.T) { // 1238/AC1
if _, bus := p["bus"]; bus {
t.Errorf("the process declares a bus: it runs on its own broker credential")
}
if len(m.Build.Artifacts) != 3 {
t.Fatalf("artifacts = %+v, want the program, the tools and the package takein", m.Build.Artifacts)
}
// H-A-186 (design 47 G §370): takein, which the gate, the view and the watchman import, is a declared package.
if a := m.Build.Artifacts[2]; a.Name != "takein" || a.Kind != "package" || a.Language != "go" || !reflect.DeepEqual(a.Paths, []string{"takein"}) || a.Series != "0.1" {
t.Errorf("package artifact = %+v", a)
if len(m.Build.Artifacts) != 2 {
t.Fatalf("artifacts = %+v, want the program and the tools", m.Build.Artifacts)
}
if a := m.Build.Artifacts[1]; a.Name != "tools" || a.Kind != "bundle" || a.Binary != "mesh-delivery-indexer-tools" || !reflect.DeepEqual(a.Loads, []string{"mesh-delivery-indexer-tools"}) {
t.Errorf("tools artifact = %+v", a) // 1945
@@ -223,15 +223,6 @@
"loads": [
"mesh-delivery-indexer-tools"
]
},
{
"name": "takein",
"kind": "package",
"language": "go",
"paths": [
"takein"
],
"series": "0.1"
}
]
}
@@ -19,7 +19,7 @@ import (
// §549): runner_unlogged lists them, runner_unlogged_append appends them by hand, written in the hand-act log; it is of class
// act and takes §391's default callers.
func UnloggedTools(pipelines stdio.KeptState, log stdio.KeptLog, acts views.HandActs, logf func(string, ...any)) []stdio.Tool {
h := &pipeline.Hearer{Store: &pipeline.MoverStore{Raw: stdioPipelines{pipelines}, Logf: logf}, Logf: logf,
h := &pipeline.Hearer{Store: stdioPipelines{pipelines}, Logf: logf,
Log: func(e seat.LogEntry) error {
_, err := log.Append(keys.EscapeName(e.Pipeline), e)
return err
@@ -67,36 +67,26 @@ func UnloggedTools(pipelines stdio.KeptState, log stdio.KeptLog, acts views.Hand
}
}
// stdioPipelines is the state pipelines as bytes, as the runtime serves it, for the by-hand append: read and written through
// a pipeline.MoverStore, which keeps each mover's steps under its own key (H-A-187).
// stdioPipelines is the state pipelines as the runtime serves it, for the by-hand append.
type stdioPipelines struct{ s stdio.KeptState }
func (s stdioPipelines) Get(key string) ([]byte, uint64, error) {
func (s stdioPipelines) Get(key string) (seat.Pipeline, uint64, error) {
e, err := s.s.Get(key)
if err != nil || e == nil {
return nil, 0, err
return seat.Pipeline{}, 0, errors.Join(err, errors.New("no such pipeline"))
}
return e.Value, e.Revision, nil
var p seat.Pipeline
return p, e.Revision, json.Unmarshal(e.Value, &p)
}
func (s stdioPipelines) Create(key string, value []byte) (uint64, error) {
r, err := s.s.Create(key, json.RawMessage(value))
if err != nil && strings.Contains(err.Error(), "exists") {
return 0, pipeline.ErrExists
}
return r, err
}
func (s stdioPipelines) Update(key string, value []byte, rev uint64) (uint64, error) {
r, err := s.s.Update(key, json.RawMessage(value), rev)
func (s stdioPipelines) Update(key string, p seat.Pipeline, rev uint64) (uint64, error) {
r, err := s.s.Update(key, p, rev)
if err != nil && (strings.Contains(err.Error(), "wrong last sequence") || strings.Contains(err.Error(), "revision")) {
return 0, pipeline.ErrChanged
}
return r, err
}
func (s stdioPipelines) Delete(key string) error { return s.s.Delete(key) }
func (s stdioPipelines) Keys(filter string) ([]string, error) {
all, err := s.s.Keys()
var out []string
@@ -136,9 +136,9 @@ func (k keptStore) Get(key string) (seat.KeptResult, bool, error) {
func (k keptStore) Delete(key string) error { return k.kv.Delete(k.ctx, key) }
// newAger is the ageing over the runner's own stores.
func newAger(ctx context.Context, js jetstream.JetStream, cred process.Account, pipelines *pipeline.MoverStore, kept, aged jetstream.KeyValue, logf func(string, ...any)) *pipeline.Ager {
func newAger(ctx context.Context, js jetstream.JetStream, cred process.Account, pipelines, kept, aged jetstream.KeyValue, logf func(string, ...any)) *pipeline.Ager {
return &pipeline.Ager{Log: stepLog{ctx: ctx, js: js, module: cred.Module}.Read, Aged: agedState{ctx: ctx, kv: aged},
Pipelines: pipelines, Kept: keptStore{ctx: ctx, kv: kept}, Logf: logf,
Pipelines: pipelineDeleter{pipelineStore{ctx: ctx, kv: pipelines}}, Kept: keptStore{ctx: ctx, kv: kept}, Logf: logf,
Outputs: func(name string) error {
o, err := js.ObjectStore(ctx, catalogue.ObjectsBucket(cred.Module, seat.StateStepOutputs))
if err != nil {
@@ -151,6 +151,11 @@ func newAger(ctx context.Context, js jetstream.JetStream, cred process.Account,
}}
}
// pipelineDeleter is the pipelines store with the delete only its owner makes (§330).
type pipelineDeleter struct{ pipelineStore }
func (d pipelineDeleter) Delete(key string) error { return d.kv.Delete(d.ctx, key) }
// storeFills reads the fill of the runner's stores against their caps: pipelines and kept (their key-value streams), step-outputs
// (its object store's) and step-log (its log's), as the bus's own streams say it (design 47 G §366, §486).
func storeFills(ctx context.Context, js jetstream.JetStream, cred process.Account, now func() time.Time) func() ([]seat.Fill, error) {
@@ -128,10 +128,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
return err
}
store := newPipelineStore(ctx, pipelines, logger.Printf)
hearer := newHearer(ctx, js, cred, store, kept, say, logger.Printf)
hearer := newHearer(ctx, js, cred, pipelines, kept, say, logger.Printf)
hearer.Asker = busAsker{conn: conn, cred: cred}
hearer.Ager = newAger(ctx, js, cred, store, kept, aged, logger.Printf)
hearer.Ager = newAger(ctx, js, cred, pipelines, kept, aged, logger.Printf)
// The stores' fill is said before a store is full, and a full one refuses a start (§366).
fills := &pipeline.Fills{Read: storeFills(ctx, js, cred, time.Now), Logf: logger.Printf,
Say: func(event string, f seat.Fill) error { return say("mesh.seat."+seat.Seat+".event."+event, f) }}
@@ -148,7 +147,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
hearer.Failures = &pipeline.Failures{Findings: rv.Findings, Observer: cred.Module, Author: commitAuthor(calls.Call), Logf: logger.Printf}
// Its asking half (§22, §24): a pipeline's steps are placed from the module's registration and each build seat is asked
// for its step.
starter := &pipeline.Starter{Pipelines: store, Logf: logger.Printf, Say: say,
starter := &pipeline.Starter{Pipelines: createdKV{ctx: ctx, kv: pipelines}, Logf: logger.Printf, Say: say,
Registration: rv.Registrations.Registration, Steps: busStepAsker{ctx: ctx, js: js, cred: cred}, Name: cred.Module,
Kept: keptReader(ctx, kept), Toolchain: toolchainReader(ctx, toolchains),
Fills: fills, Recovered: kvRecoveries{ctx, recovered}.Has, Holders: rv.Holders.Holders, Ended: hearer.EndedAtPlacing}
@@ -171,10 +170,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return
}
// A pipeline of the branch whose registration was heard before this half was held gets its steps now (A-327).
if n, err := starter.Waiting(store, branch, rv.Catalog.View); err != nil || n > 0 {
if n, err := starter.Waiting(pipelineStore{ctx: ctx, kv: pipelines}, branch, rv.Catalog.View); err != nil || n > 0 {
logger.Printf("the pipelines of %s that waited on its half for their steps: %d placed (%v)", branch, n, err)
}
n, err := starter.Resume(store, branch, half)
n, err := starter.Resume(pipelineStore{ctx: ctx, kv: pipelines}, branch, half)
if err != nil {
logger.Printf("the steps placed on %s and not yet asked: %d asked, then %v", branch, n, err)
return
@@ -185,19 +184,19 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// The view reaching a newer revision places the steps of the pipelines that waited for it, and asks nothing again (A-448, §200).
advance := func(branch string) {
rv.Holders.refresh()
if n, err := starter.Waiting(store, branch, rv.Catalog.View); err != nil || n > 0 {
if n, err := starter.Waiting(pipelineStore{ctx: ctx, kv: pipelines}, branch, rv.Catalog.View); err != nil || n > 0 {
logger.Printf("the pipelines of %s that waited for a newer revision of its half: %d placed (%v)", branch, n, err)
}
}
advancer.Store(&advance)
// A registration-changed places the steps of its module's pipelines started before it, from it (A-327).
register := func(reg catalog.Registration) {
if n, err := starter.Registered(store, reg, rv.Catalog.View); err != nil || n > 0 {
if n, err := starter.Registered(pipelineStore{ctx: ctx, kv: pipelines}, reg, rv.Catalog.View); err != nil || n > 0 {
logger.Printf("the pipelines of %s started before its registration: %d placed (%v)", reg.Module, n, err)
}
}
registrar.Store(&register)
starter.Store = store
starter.Store = pipelineStore{ctx: ctx, kv: pipelines}
// The queue (design 47 H §748): at most pipelines-at-once pipelines have their steps asked, the others wait in order in
// the state queue, placed at the start from it and on every end.
atOnce, waitingCap, err := queueSettings(getenv)
@@ -401,8 +401,6 @@ func pipelineKeys(t *testing.T, ctx context.Context, kv jetstream.KeyValue) []st
if err != nil {
t.Fatal(err)
}
// A mover's steps are a key of their own (H-A-187), placed with its pipeline: no pipeline of their own.
ks = slices.DeleteFunc(ks, pipeline.IsStepsKey)
slices.Sort(ks)
return ks
}
@@ -7,7 +7,6 @@ import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
@@ -222,34 +221,6 @@ func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *tes
waitFor(t, ctx, logs, func() bool { return readPipeline(ctx, pipelines, key, &p) && p.Outcome == seat.PipelineOutcomePassed })
// H-A-187 over the bus: show composes the pipeline from its record and mover keys, naming a mover its steps alone, and a
// record written before it, its steps inline, is read whole and split at its next write.
shown, err := pipeline.Reads{Pipelines: kvKept{ctx, pipelines}}.Show(seat.ShowInput{Pipeline: key, Mover: "app"})
if err != nil || len(shown.Pipeline.Steps) != len(p.Steps) || shown.Pipeline.Steps[0].Module != "app" {
t.Fatalf("show of %s naming app = %d steps (%v), want its %d", key, len(shown.Pipeline.Steps), err, len(p.Steps))
}
inline := p
inline.Pipeline = "main.2"
inlineRaw, _ := json.Marshal(inline)
if _, err := pipelines.Put(ctx, "main.2", inlineRaw); err != nil {
t.Fatal(err)
}
store := newPipelineStore(ctx, pipelines, func(f string, a ...any) { fmt.Fprintf(logs, f+"\n", a...) })
old, oldRev, err := store.Get("main.2")
if err != nil || len(old.Steps) != len(p.Steps) {
t.Fatalf("the inline record read %d steps (%v), want %d", len(old.Steps), err, len(p.Steps))
}
if _, err := store.Update("main.2", old, oldRev); err != nil {
t.Fatal(err)
}
var split seat.Pipeline
if !readPipeline(ctx, pipelines, "main.2", &split) || len(split.Steps) != len(p.Steps) {
t.Fatalf("the record split reads %d steps, want %d", len(split.Steps), len(p.Steps))
}
if _, err := pipelines.Get(ctx, pipeline.StepsKey("main.2", "app")); err != nil || !strings.Contains(logs.String(), "split into 1 mover keys") {
t.Fatalf("the mover key of main.2's app: %v; the journal says the split: %t", err, strings.Contains(logs.String(), "split into 1 mover keys"))
}
// §470, §330: the step log is the log's stream, read onward by sequence the way the ageing reads it.
logged, err := stepLog{ctx: ctx, js: js, module: module}.Read(1, 100)
if err != nil || len(logged) == 0 || logged[0].Entry.Pipeline != key {
@@ -303,20 +274,13 @@ func stepOf(p seat.Pipeline, step string) seat.StepResult {
return seat.StepResult{}
}
// readPipeline reads pipeline key as the runner does, its steps composed from its mover keys, and holds its record to
// H-A-187: a record the runner wrote carries no steps of its own, so it never grows with the catalog.
func readPipeline(ctx context.Context, kv jetstream.KeyValue, key string, p *seat.Pipeline) bool {
e, err := kv.Get(ctx, key)
if err != nil {
return false
}
var record seat.Pipeline
if json.Unmarshal(e.Value(), &record) != nil || len(record.Steps) > 0 {
panic(fmt.Sprintf("the record of %s carries %d steps of its own; each mover's steps are its own key (H-A-187)", key, len(record.Steps)))
}
got, _, err := newPipelineStore(ctx, kv, nil).Get(key)
*p = got
return err == nil
*p = seat.Pipeline{}
return json.Unmarshal(e.Value(), p) == nil
}
// waitFor asks cond until it holds or the test's context ends, the module's journal in the failure.
@@ -1,7 +1,9 @@
package main
import (
"context"
"encoding/json"
"errors"
"github.com/nats-io/nats.go/jetstream"
@@ -11,6 +13,20 @@ import (
"mesh-delivery-pipeline/internal/pipeline"
)
// createdKV is the state pipelines written by compare-and-set on a key's absence (§54).
type createdKV struct {
ctx context.Context
kv jetstream.KeyValue
}
func (c createdKV) Create(key string, value []byte) error {
_, err := c.kv.Create(c.ctx, key, value)
if errors.Is(err, jetstream.ErrKeyExists) {
return pipeline.ErrExists
}
return err
}
// hearPublished hears each package-published (design 47 H §746, §747): in its source's running pipeline, or starting one
// pipeline of the package's kind-1 dependents that pipeline does not move. The message is acknowledged once that is
// written, on a goroutine of its own, so a slow write holds no other event; the durable consumer delivers a publish heard
@@ -19,74 +19,64 @@ import (
"mesh-delivery-pipeline/internal/pipeline"
)
// kvState is the state pipelines as bytes on the bus, by compare-and-set (§27); the runner reads and writes pipelines
// through one pipeline.MoverStore over it, which keeps each mover's steps under its own key (H-A-187).
type kvState struct {
// pipelineStore is the state pipelines as the hearer reads and writes it: a write by compare-and-set on the revision read (§27).
type pipelineStore struct {
ctx context.Context
kv jetstream.KeyValue
}
// newPipelineStore is the process's one store of the state pipelines.
func newPipelineStore(ctx context.Context, kv jetstream.KeyValue, logf func(string, ...any)) *pipeline.MoverStore {
return &pipeline.MoverStore{Raw: kvState{ctx: ctx, kv: kv}, Logf: logf}
}
func (s kvState) Get(key string) ([]byte, uint64, error) {
func (s pipelineStore) Get(key string) (seat.Pipeline, uint64, error) {
e, err := s.kv.Get(s.ctx, key)
if errors.Is(err, jetstream.ErrKeyNotFound) {
return nil, 0, nil
}
if err != nil {
return nil, 0, err
return seat.Pipeline{}, 0, err
}
return e.Value(), e.Revision(), nil
var p seat.Pipeline
if err := json.Unmarshal(e.Value(), &p); err != nil {
return seat.Pipeline{}, 0, fmt.Errorf("the pipeline %s is no pipeline: %w", key, err)
}
return p, e.Revision(), nil
}
func (s kvState) Create(key string, value []byte) (uint64, error) {
rev, err := s.kv.Create(s.ctx, key, value)
if errors.Is(err, jetstream.ErrKeyExists) {
return 0, pipeline.ErrExists
func (s pipelineStore) Update(key string, p seat.Pipeline, revision uint64) (uint64, error) {
raw, err := json.Marshal(p)
if err != nil {
return 0, err
}
return rev, err
}
func (s kvState) Update(key string, value []byte, revision uint64) (uint64, error) {
rev, err := s.kv.Update(s.ctx, key, value, revision)
rev, err := s.kv.Update(s.ctx, key, raw, revision)
if err != nil && (errors.Is(err, jetstream.ErrKeyExists) || strings.Contains(err.Error(), "wrong last sequence")) {
return 0, pipeline.ErrChanged
}
return rev, err
}
func (s kvState) Delete(key string) error {
if err := s.kv.Delete(s.ctx, key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
return err
func (s pipelineStore) Keys(filter string) ([]string, error) {
all, err := s.kv.Keys(s.ctx)
if errors.Is(err, jetstream.ErrNoKeysFound) {
return nil, nil
}
return nil
}
// Keys are the keys matching filter, `*` one key part, as the bus filters a key-value's subjects.
func (s kvState) Keys(filter string) ([]string, error) {
l, err := s.kv.ListKeysFiltered(s.ctx, filter)
if err != nil {
if errors.Is(err, jetstream.ErrNoKeysFound) {
return nil, nil
}
return nil, err
}
// The lister ends its own watch once the current keys are listed.
var out []string
for k := range l.Keys() {
out = append(out, k)
want := strings.Split(filter, ".")
for _, k := range all {
got := strings.Split(k, ".")
if len(got) != len(want) {
continue
}
ok := true
for i := range want {
ok = ok && (want[i] == "*" || want[i] == got[i])
}
if ok {
out = append(out, k)
}
}
return out, nil
return out, err
}
// newHearer is the hearer over the runner's own states: pipelines, kept, the step log on its stream and step-outputs on its
// objects (§27). Each store is opened where it is used, so one the grants part has not made yet fails that write loudly.
func newHearer(ctx context.Context, js jetstream.JetStream, cred process.Account, pipelines pipeline.Updater, kept jetstream.KeyValue,
func newHearer(ctx context.Context, js jetstream.JetStream, cred process.Account, pipelines, kept jetstream.KeyValue,
say func(string, any) error, logf func(string, ...any)) *pipeline.Hearer {
return &pipeline.Hearer{Store: pipelines, Say: say, Logf: logf, Name: cred.Module,
return &pipeline.Hearer{Store: pipelineStore{ctx: ctx, kv: pipelines}, Say: say, Logf: logf, Name: cred.Module,
Log: stepLog{ctx: ctx, js: js, module: cred.Module}.Append,
Keep: func(key string, k seat.KeptResult) error {
raw, err := json.Marshal(k)
@@ -1,396 +0,0 @@
package pipeline
import (
"encoding/json"
"errors"
"fmt"
"maps"
"slices"
"sort"
"strings"
"sync"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// A pipeline's steps live one key per mover (design 47 H-A-187): the state pipelines keeps the pipeline's record (its
// movers, tiers, call, source and outcome) under the pipeline's key with its own steps empty, and each mover's steps under
// `steps.<pipeline>.<mover>`, written by compare-and-set as the record is, so a record never grows with the catalog and no
// value nears the bus's message limit (ADR 0486). Every read of a pipeline composes its steps from the mover keys it names.
// StepsKey is the key of one mover's steps in the state pipelines, `steps.<pipeline>.<mover>`, the mover's module escaped
// as a key's name is (H-A-187).
func StepsKey(pipeline, mover string) string {
return seat.PipelinesByStepsPipelineMoverKey(pipeline, keys.EscapeName(mover))
}
// stepsPrefix is the start of every mover key of one pipeline.
func stepsPrefix(pipeline string) string { return StepsKey(pipeline, "") }
// IsStepsKey says whether a key of the state pipelines is a mover's steps rather than a pipeline's record: `steps.` and a
// pipeline key of a take-in's (4 parts), a run's (5 parts, its third `run`) or a publish's (7 parts) and one mover.
func IsStepsKey(key string) bool {
parts := strings.Split(key, ".")
if parts[0] != "steps" {
return false
}
switch len(parts) {
case 4, 7:
return true
case 5:
return parts[2] == runToken
}
return false
}
// RawState is the state pipelines as bytes: a key's value and revision (nil, 0 and no error when it has none), a create by
// compare-and-set on absence (ErrExists), an update on the revision read (ErrChanged), a delete, and the keys matching a
// filter, `*` one key part.
type RawState interface {
Get(key string) ([]byte, uint64, error)
Create(key string, value []byte) (uint64, error)
Update(key string, value []byte, revision uint64) (uint64, error)
Delete(key string) error
Keys(filter string) ([]string, error)
}
// ErrNoPipeline is a read of a pipeline key the state pipelines does not hold.
var ErrNoPipeline = errors.New("no such pipeline")
// MoverStore is the state pipelines as the runner reads and writes it (§27, H-A-187): Get composes a pipeline from its
// record and its mover keys, Update writes the record and each mover key that changed, each by compare-and-set on the
// revision read, and nothing when any of them was written since. One MoverStore serves a process: a write and a read of one
// pipeline take its lock, so the process never reads a record and mover keys of two writes.
type MoverStore struct {
Raw RawState
Logf func(string, ...any)
mu sync.Mutex
locks map[string]*sync.Mutex
read map[string]readAt
inline map[string]bool
}
// readAt is what the last read of a pipeline found: its record's revision and each mover key's revision.
type readAt struct {
rev uint64
movers map[string]uint64
}
func (s *MoverStore) lock(key string) func() {
s.mu.Lock()
if s.locks == nil {
s.locks, s.read, s.inline = map[string]*sync.Mutex{}, map[string]readAt{}, map[string]bool{}
}
l, ok := s.locks[key]
if !ok {
l = &sync.Mutex{}
s.locks[key] = l
}
s.mu.Unlock()
l.Lock()
return l.Unlock
}
func (s *MoverStore) logf(f string, a ...any) {
if s.Logf != nil {
s.Logf(f, a...)
}
}
// moverKeys are the mover keys of pipeline key with their values and revisions.
func (s *MoverStore) moverKeys(key string) (map[string][]byte, map[string]uint64, error) {
ks, err := s.Raw.Keys(stepsPrefix(key) + "*")
if err != nil {
return nil, nil, fmt.Errorf("the mover keys of %s are not listed: %w", key, err)
}
vals, revs := map[string][]byte{}, map[string]uint64{}
for _, k := range ks {
v, rev, err := s.Raw.Get(k)
if err != nil {
return nil, nil, fmt.Errorf("%s is not read: %w", k, err)
}
if v != nil {
vals[k], revs[k] = v, rev
}
}
return vals, revs, nil
}
// composeSteps sets p's steps from its mover keys, in the order of p's movers, a key no mover names last in key order, and
// its unlogged step-log entries: the record's own (an entry naming no mover, written before H-A-190) and each mover's, in
// the order they ended (H-A-189).
func composeSteps(p *seat.Pipeline, key string, vals map[string][]byte) error {
var names []string
for k := range vals {
names = append(names, k)
}
sort.Strings(names)
order := make([]string, 0, len(names))
for _, m := range p.Movers {
if k := StepsKey(key, m.Module); vals[k] != nil && !slices.Contains(order, k) {
order = append(order, k)
}
}
for _, k := range names {
if !slices.Contains(order, k) {
order = append(order, k)
}
}
steps := []seat.StepResult{}
unlogged := append([]seat.LogEntry(nil), p.Unlogged...)
for _, k := range order {
v, err := seat.DecodePipelinesByStepsPipelineMoverValue(vals[k])
if err != nil {
return fmt.Errorf("%s holds no mover's steps: %w", k, err)
}
steps = append(steps, v.Steps...)
unlogged = append(unlogged, v.Unlogged...)
}
sort.SliceStable(unlogged, func(i, j int) bool { return unlogged[i].At.Before(unlogged[j].At) })
p.Steps, p.Unlogged = steps, unlogged
return nil
}
// split is p's record, its own steps and unlogged entries empty, and each mover's steps and unlogged step-log entries by
// its mover key (H-A-187, H-A-189). An entry naming no mover (one written before H-A-190) goes under the mover whose step
// it alone matches, else it stays in the record.
func split(key string, p seat.Pipeline) ([]byte, map[string][]byte, error) {
by := map[string]*seat.PipelinesByStepsPipelineMoverValue{}
at := func(module string) *seat.PipelinesByStepsPipelineMoverValue {
k := StepsKey(key, module)
if by[k] == nil {
by[k] = &seat.PipelinesByStepsPipelineMoverValue{Steps: []seat.StepResult{}}
}
return by[k]
}
for _, st := range p.Steps {
if st.Module == "" {
return nil, nil, fmt.Errorf("a step %s of %s names no mover, so it has no mover key", st.Step, key)
}
v := at(st.Module)
v.Steps = append(v.Steps, st)
}
var kept []seat.LogEntry
for _, e := range p.Unlogged {
module := e.Module
if module == "" {
module = moverOfEntry(p, e)
}
if module == "" {
kept = append(kept, e)
continue
}
v := at(module)
v.Unlogged = append(v.Unlogged, e)
}
movers := map[string][]byte{}
for k, v := range by {
raw, err := json.Marshal(v)
if err != nil {
return nil, nil, err
}
movers[k] = raw
}
p.Steps, p.Unlogged = []seat.StepResult{}, kept
record, err := json.Marshal(p)
return record, movers, err
}
// moverOfEntry is the mover of the one step an entry naming no mover matches by step and artifact, empty when none or
// more than one does.
func moverOfEntry(p seat.Pipeline, e seat.LogEntry) string {
found := ""
for _, st := range p.Steps {
if st.Step != e.Step || st.Artifact != e.Artifact {
continue
}
if found != "" && found != st.Module {
return ""
}
found = st.Module
}
return found
}
// Get is pipeline key with every mover's steps and the record's revision. A record that still carries its steps inline (one
// written before H-A-187) is read whole, said once in the journal, and split at its next write.
func (s *MoverStore) Get(key string) (seat.Pipeline, uint64, error) {
defer s.lock(key)()
raw, rev, err := s.Raw.Get(key)
if err != nil {
return seat.Pipeline{}, 0, err
}
if raw == nil {
return seat.Pipeline{}, 0, fmt.Errorf("%w: %s", ErrNoPipeline, key)
}
var p seat.Pipeline
if err := json.Unmarshal(raw, &p); err != nil {
return seat.Pipeline{}, 0, fmt.Errorf("the pipeline %s is no pipeline: %w", key, err)
}
vals, revs, err := s.moverKeys(key)
if err != nil {
return seat.Pipeline{}, 0, err
}
s.read[key] = readAt{rev: rev, movers: revs}
if len(p.Steps) > 0 {
if !s.inline[key] {
s.inline[key] = true
s.logf("the pipeline %s carries its %d steps inline in its record: they are read whole and split into one key per mover at its next write (H-A-187)", key, len(p.Steps))
}
return p, rev, nil
}
if p.Steps == nil {
p.Steps = []seat.StepResult{}
}
return p, rev, composeSteps(&p, key, vals)
}
// Update writes p as pipeline key's record and its mover keys, each by compare-and-set: nothing is written, ErrChanged,
// when the record or a mover key was written since the read revision gave. A record carrying its steps inline has its
// mover keys written before the record, so its steps are never lost between the two.
func (s *MoverStore) Update(key string, p seat.Pipeline, revision uint64) (uint64, error) {
defer s.lock(key)()
record, want, err := split(key, p)
if err != nil {
return 0, err
}
raw, cur, err := s.Raw.Get(key)
if err != nil {
return 0, err
}
if raw == nil {
return 0, fmt.Errorf("%w: %s", ErrNoPipeline, key)
}
if cur != revision {
return 0, ErrChanged
}
vals, revs, err := s.moverKeys(key)
if err != nil {
return 0, err
}
// A mover key written since the read is another write between: the read named its revision.
if seen, ok := s.read[key]; ok && seen.rev == revision {
for k, r := range revs {
if seen.movers[k] != r {
return 0, ErrChanged
}
}
for k := range seen.movers {
if _, ok := revs[k]; !ok {
return 0, ErrChanged
}
}
}
var was seat.Pipeline
inline := json.Unmarshal(raw, &was) == nil && len(was.Steps) > 0
var after map[string]uint64
if inline {
if after, err = s.writeMovers(key, want, vals, revs); err != nil {
return 0, err
}
}
rev, err := s.Raw.Update(key, record, revision)
if err != nil {
return 0, err
}
if !inline {
if after, err = s.writeMovers(key, want, vals, revs); err != nil {
return 0, err
}
} else {
s.logf("the pipeline %s's steps are split into %d mover keys and its record no longer carries them (H-A-187)", key, len(want))
delete(s.inline, key)
}
s.read[key] = readAt{rev: rev, movers: after}
return rev, nil
}
// writeMovers writes each mover key whose steps changed and deletes each that p no longer names, by compare-and-set on the
// revisions read, and answers every mover key's revision after it.
func (s *MoverStore) writeMovers(key string, want, vals map[string][]byte, revs map[string]uint64) (map[string]uint64, error) {
after := map[string]uint64{}
for _, k := range slices.Sorted(maps.Keys(want)) {
v := want[k]
old, ok := vals[k]
var rev uint64
var err error
switch {
case ok && string(old) == string(v):
after[k] = revs[k]
continue
case ok:
rev, err = s.Raw.Update(k, v, revs[k])
default:
rev, err = s.Raw.Create(k, v)
}
if err != nil {
return nil, fmt.Errorf("the steps of %s are not written: %w", k, err)
}
after[k] = rev
}
for k := range vals {
if _, ok := want[k]; !ok {
if err := s.Raw.Delete(k); err != nil {
return nil, fmt.Errorf("the steps of %s are not deleted: %w", k, err)
}
}
}
return after, nil
}
// Create writes a new pipeline, its record by compare-and-set on its absence (ErrExists: the second start writes nothing,
// §54), then its mover keys; a mover key an earlier pipeline of the key left is written over, said in the journal.
func (s *MoverStore) Create(key string, value []byte) error {
var p seat.Pipeline
if err := json.Unmarshal(value, &p); err != nil {
return fmt.Errorf("the pipeline %s is no pipeline: %w", key, err)
}
defer s.lock(key)()
record, want, err := split(key, p)
if err != nil {
return err
}
rev, err := s.Raw.Create(key, record)
if err != nil {
return err
}
vals, revs, err := s.moverKeys(key)
if err != nil {
return err
}
for k := range vals {
s.logf("%s was left by an earlier pipeline of the key: it is written over (H-A-187)", k)
}
after, err := s.writeMovers(key, want, vals, revs)
if err != nil {
return err
}
s.read[key] = readAt{rev: rev, movers: after}
return nil
}
// Delete removes pipeline key's mover keys, then its record (§330): a delete stopped between leaves the record, which the
// next ageing deletes again.
func (s *MoverStore) Delete(key string) error {
unlock := s.lock(key)
vals, _, err := s.moverKeys(key)
if err == nil {
for k := range vals {
if err = s.Raw.Delete(k); err != nil {
break
}
}
}
if err == nil {
err = s.Raw.Delete(key)
}
delete(s.read, key)
delete(s.inline, key)
unlock()
return err
}
// Keys are the keys of the state pipelines matching filter, `*` one key part.
func (s *MoverStore) Keys(filter string) ([]string, error) { return s.Raw.Keys(filter) }
@@ -1,254 +0,0 @@
package pipeline
import (
"encoding/json"
"errors"
"fmt"
"strings"
"testing"
"time"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
)
// memRaw is the state pipelines as bytes, one revision sequence for the bucket as a key-value's stream has.
type memRaw struct {
vals map[string][]byte
revs map[string]uint64
seq uint64
}
func newMemRaw() *memRaw { return &memRaw{vals: map[string][]byte{}, revs: map[string]uint64{}} }
func (m *memRaw) Get(key string) ([]byte, uint64, error) {
v, ok := m.vals[key]
if !ok {
return nil, 0, nil
}
return append([]byte(nil), v...), m.revs[key], nil
}
func (m *memRaw) put(key string, value []byte) uint64 {
m.seq++
m.vals[key], m.revs[key] = append([]byte(nil), value...), m.seq
return m.seq
}
func (m *memRaw) Create(key string, value []byte) (uint64, error) {
if _, ok := m.vals[key]; ok {
return 0, ErrExists
}
return m.put(key, value), nil
}
func (m *memRaw) Update(key string, value []byte, rev uint64) (uint64, error) {
if m.revs[key] != rev {
return 0, ErrChanged
}
return m.put(key, value), nil
}
func (m *memRaw) Delete(key string) error {
delete(m.vals, key)
delete(m.revs, key)
return nil
}
func (m *memRaw) Keys(filter string) ([]string, error) {
var out []string
for k := range m.vals {
if ok, _ := matchKey(filter, k); ok {
out = append(out, k)
}
}
return out, nil
}
// recordSteps is the steps the raw record of key carries itself.
func recordSteps(t *testing.T, m *memRaw, key string) int {
t.Helper()
var p seat.Pipeline
if err := json.Unmarshal(m.vals[key], &p); err != nil {
t.Fatal(err)
}
return len(p.Steps)
}
// TestMoverKeys holds the state pipelines' shape to H-A-187: a pipeline's record keeps its movers and outcome, each mover's
// steps their own key written by compare-and-set as the record is, a read composing them, a record carrying its steps inline
// read whole and split at its next write, one row per case.
func TestMoverKeys(t *testing.T) {
const key = "main.7"
two := func() seat.Pipeline {
return seat.Pipeline{V: 1, Pipeline: key, Branch: "main", Source: 7, Outcome: seat.PipelineOutcomeRunning,
Movers: []seat.Mover{{Module: "beta"}, {Module: "alpha"}},
Steps: []seat.StepResult{{Module: "alpha", Step: seat.StepResultStepBuild, Outcome: seat.StepResultOutcomeRunning},
{Module: "beta", Step: seat.StepResultStepBuild, Outcome: seat.StepResultOutcomeRunning},
{Module: "beta", Step: seat.StepResultStepApproval, Outcome: seat.StepResultOutcomeNotAsked}}}
}
created := func(t *testing.T) (*memRaw, *MoverStore, *[]string) {
m, logs := newMemRaw(), &[]string{}
s := &MoverStore{Raw: m, Logf: func(f string, a ...any) { *logs = append(*logs, fmt.Sprintf(f, a...)) }}
raw, _ := json.Marshal(two())
if err := s.Create(key, raw); err != nil {
t.Fatal(err)
}
return m, s, logs
}
steps := func(p seat.Pipeline) string {
var out []string
for _, st := range p.Steps {
out = append(out, st.Module+" "+st.Step+" "+st.Outcome)
}
return strings.Join(out, "; ")
}
rows := []struct {
name string
run func(t *testing.T) string
want string
}{
{"a write keeps the record's steps empty and each mover's under its own key; a read composes them in the movers' order", func(t *testing.T) string {
m, s, _ := created(t)
p, _, err := s.Get(key)
return fmt.Sprintf("record %d, alpha %t, beta %t: %s %v", recordSteps(t, m, key), m.vals["steps.main.7.alpha"] != nil,
m.vals["steps.main.7.beta"] != nil, steps(p), err)
}, "record 0, alpha true, beta true: beta build running; beta approval not asked; alpha build running <nil>"},
{"an update writes the mover whose steps changed and leaves the other's key as it was", func(t *testing.T) string {
m, s, _ := created(t)
p, rev, _ := s.Get(key)
beta := m.revs["steps.main.7.beta"]
p.Steps[2].Outcome = seat.StepResultOutcomePassed
_, err := s.Update(key, p, rev)
q, _, _ := s.Get(key)
return fmt.Sprintf("beta kept %t: %s %v", m.revs["steps.main.7.beta"] == beta, steps(q), err)
}, "beta kept true: beta build running; beta approval not asked; alpha build passed <nil>"},
{"a mover key written since the read refuses the update, nothing written", func(t *testing.T) string {
m, s, _ := created(t)
p, rev, _ := s.Get(key)
m.put("steps.main.7.alpha", m.vals["steps.main.7.alpha"])
p.Outcome = seat.PipelineOutcomePassed
_, err := s.Update(key, p, rev)
return fmt.Sprintf("%t, record at %t", errors.Is(err, ErrChanged), m.revs[key] == rev)
}, "true, record at true"},
{"a record written since the read refuses the update", func(t *testing.T) string {
_, s, _ := created(t)
p, rev, _ := s.Get(key)
if _, err := s.Update(key, p, rev); err != nil {
return err.Error()
}
_, err := s.Update(key, p, rev)
return fmt.Sprint(errors.Is(err, ErrChanged))
}, "true"},
{"a record carrying its steps inline is read whole, said once, and split at its next write", func(t *testing.T) string {
m, logs := newMemRaw(), &[]string{}
s := &MoverStore{Raw: m, Logf: func(f string, a ...any) { *logs = append(*logs, fmt.Sprintf(f, a...)) }}
raw, _ := json.Marshal(two())
m.put(key, raw)
p, rev, _ := s.Get(key)
_, _, _ = s.Get(key)
read := fmt.Sprintf("read %d steps, %d lines", len(p.Steps), len(*logs))
_, err := s.Update(key, p, rev)
q, _, _ := s.Get(key)
return fmt.Sprintf("%s; written: record %d, %s, split said %t %v", read, recordSteps(t, m, key), steps(q),
strings.Contains(strings.Join(*logs, "\n"), "split into 2 mover keys"), err)
}, "read 3 steps, 1 lines; written: record 0, beta build running; beta approval not asked; alpha build running, split said true <nil>"},
{"a second create writes nothing", func(t *testing.T) string {
m, s, _ := created(t)
p := two()
p.Steps[0].Outcome = seat.StepResultOutcomeFailed
raw, _ := json.Marshal(p)
err := s.Create(key, raw)
q, _, _ := s.Get(key)
return fmt.Sprintf("%t %d: %s", errors.Is(err, ErrExists), len(m.vals), steps(q))
}, "true 3: beta build running; beta approval not asked; alpha build running"},
{"a delete removes the mover keys and the record", func(t *testing.T) string {
m, s, _ := created(t)
err := s.Delete(key)
return fmt.Sprintf("%d %v", len(m.vals), err)
}, "0 <nil>"},
{"a record of 129 movers never grows with their steps (main.26)", func(t *testing.T) string {
m, s, _ := created(t)
p, rev, _ := s.Get(key)
p.Movers, p.Steps = nil, nil
for i := range 129 {
mod := fmt.Sprintf("module-%03d", i)
p.Movers = append(p.Movers, seat.Mover{Module: mod, Commit: strings.Repeat("c", 40), Fingerprint: "fp-" + mod})
for _, step := range []string{seat.StepResultStepBuild, seat.StepResultStepLint, seat.StepResultStepUnit,
seat.StepResultStepIntegration, seat.StepResultStepValidate, seat.StepResultStepComposition,
seat.StepResultStepReview, seat.StepResultStepApproval} {
p.Steps = append(p.Steps, seat.StepResult{Module: mod, Step: step, Artifact: mod + ":program",
Outcome: seat.StepResultOutcomeFailed, Why: strings.Repeat("why ", 250)})
}
}
if _, err := s.Update(key, p, rev); err != nil {
return err.Error()
}
largest := 0
for k, v := range m.vals {
if IsStepsKey(k) && len(v) > largest {
largest = len(v)
}
}
q, _, _ := s.Get(key)
return fmt.Sprintf("record under 64 KiB %t, a mover key under 16 KiB %t, %d steps read", len(m.vals[key]) < 64<<10,
largest < 16<<10, len(q.Steps))
}, "record under 64 KiB true, a mover key under 16 KiB true, 1032 steps read"},
{"a mover's unlogged entries live under its mover key, each filed by the mover it names (H-A-189, H-A-190)", func(t *testing.T) string {
m, s, _ := created(t)
p, rev, _ := s.Get(key)
at := time.Date(2026, 11, 20, 20, 0, 0, 0, time.UTC)
// Two movers' approvals end alike at one moment; each entry names its mover.
p.Unlogged = []seat.LogEntry{{Pipeline: key, Module: "beta", Step: seat.StepResultStepApproval, What: seat.LogEntryWhatEnded, At: at},
{Pipeline: key, Module: "alpha", Step: seat.StepResultStepApproval, What: seat.LogEntryWhatEnded, At: at.Add(time.Second)}}
if _, err := s.Update(key, p, rev); err != nil {
return err.Error()
}
var rec seat.Pipeline
_ = json.Unmarshal(m.vals[key], &rec)
beta, _ := seat.DecodePipelinesByStepsPipelineMoverValue(m.vals["steps.main.7.beta"])
q, _, _ := s.Get(key)
var got []string
for _, e := range q.Unlogged {
got = append(got, e.Module)
}
return fmt.Sprintf("record %d unlogged, beta's key %d (%s), read %s", len(rec.Unlogged), len(beta.Unlogged),
beta.Unlogged[0].Module, strings.Join(got, " "))
}, "record 0 unlogged, beta's key 1 (beta), read beta alpha"},
{"a 129-mover pipeline with 1,000 unlogged entries keeps its record small (H-A-189)", func(t *testing.T) string {
m, s, _ := created(t)
p, rev, _ := s.Get(key)
p.Movers, p.Steps, p.Unlogged = nil, nil, nil
at := time.Date(2026, 11, 20, 20, 0, 0, 0, time.UTC)
for i := range 129 {
mod := fmt.Sprintf("module-%03d", i)
p.Movers = append(p.Movers, seat.Mover{Module: mod, Commit: strings.Repeat("c", 40), Fingerprint: "fp-" + mod})
p.Steps = append(p.Steps, seat.StepResult{Module: mod, Step: seat.StepResultStepBuild, Outcome: seat.StepResultOutcomeNotAsked})
}
for i := range 1000 {
mod := fmt.Sprintf("module-%03d", i%129)
p.Unlogged = append(p.Unlogged, seat.LogEntry{Pipeline: key, Module: mod, Step: seat.StepResultStepBuild, Artifact: mod + ":program",
What: seat.LogEntryWhatEnded, Outcome: seat.StepResultOutcomeNotAsked, Output: strings.Repeat("o", 200), At: at.Add(time.Duration(i) * time.Millisecond)})
}
if _, err := s.Update(key, p, rev); err != nil {
return err.Error()
}
q, _, _ := s.Get(key)
return fmt.Sprintf("record under 64 KiB %t, %d unlogged read", len(m.vals[key]) < 64<<10, len(q.Unlogged))
}, "record under 64 KiB true, 1000 unlogged read"},
{"mover keys are told from records of every key form", func(t *testing.T) string {
var out []string
for _, k := range []string{"steps.main.7.alpha", "steps.main.run.ab12.alpha", "steps.main.7.published.p.v.alpha",
"main.7", "steps.7", "steps.run.ab12", "steps.7.published.p.v"} {
out = append(out, fmt.Sprint(IsStepsKey(k)))
}
return strings.Join(out, " ")
}, "true true true false false false false"},
}
for _, r := range rows {
t.Run(r.name, func(t *testing.T) {
if got := r.run(t); got != r.want {
t.Errorf("got %s\nwant %s", got, r.want)
}
})
}
}
@@ -39,8 +39,7 @@ func (r Reads) fills() ([]seat.Fill, error) {
return f, nil
}
// Show is one pipeline whole, with its steps and the names of their outputs in step-outputs (§83), or, naming a mover, the
// pipeline's record with that mover's steps alone (H-A-187).
// Show is one pipeline whole, with its steps and the names of their outputs in step-outputs (§83).
func (r Reads) Show(in seat.ShowInput) (seat.ShowAnswer, error) {
p, ok, err := r.pipeline(in.Pipeline)
if err != nil {
@@ -53,13 +52,6 @@ func (r Reads) Show(in seat.ShowInput) (seat.ShowAnswer, error) {
if err != nil {
return seat.ShowAnswer{}, err
}
// Named, a mover is answered by its steps alone (H-A-187): one mover's of a pipeline of a hundred movers.
if m := strings.TrimSpace(in.Mover); m != "" {
if !Moves(p, m) {
return seat.ShowAnswer{}, fmt.Errorf("the pipeline %s does not move %s", in.Pipeline, m)
}
p.Steps = stepsOf(p, m)
}
a := seat.ShowAnswer{Pipeline: p, Fill: fills}
for _, s := range p.Steps {
if s.Output != "" {
@@ -70,7 +62,7 @@ func (r Reads) Show(in seat.ShowInput) (seat.ShowAnswer, error) {
}
// List is the pipelines and runs of a module, a branch, a commit or a run, newest started first, at most Limit when it
// names one (§83): each one's record with its movers and steps counted, its steps empty (H-A-191).
// names one (§83).
func (r Reads) List(in seat.ListInput) (seat.ListAnswer, error) {
all, err := r.pipelines()
if err != nil {
@@ -80,9 +72,6 @@ func (r Reads) List(in seat.ListInput) (seat.ListAnswer, error) {
for _, p := range all {
if (in.Module == "" || Moves(p, in.Module)) && (in.Branch == "" || p.Branch == in.Branch) &&
(in.Commit == "" || movesCommit(p, in.Commit)) && (in.Run == "" || p.Run == in.Run) {
// Each pipeline's record with its movers and steps counted, never its steps: show gives those (H-A-191).
p.MoverCount, p.StepCount = int64(len(p.Movers)), int64(len(p.Steps))
p.Steps, p.Unlogged = []seat.StepResult{}, nil
out = append(out, p)
}
}
@@ -189,16 +178,6 @@ func (r Reads) pipeline(key string) (seat.Pipeline, bool, error) {
if key == "" {
return seat.Pipeline{}, false, fmt.Errorf("name the pipeline's key, <branch>.<revision>.<module>")
}
all, err := r.Pipelines.Keys()
if err != nil {
return seat.Pipeline{}, false, fmt.Errorf("%s cannot be read: %w", seat.StatePipelines, err)
}
return r.pipelineOf(key, all)
}
// pipelineOf is pipeline key, its steps composed from the mover keys among all that it names (H-A-187); a record that
// still carries its steps inline is read whole.
func (r Reads) pipelineOf(key string, all []string) (seat.Pipeline, bool, error) {
raw, err := r.Pipelines.Value(key)
if err != nil {
return seat.Pipeline{}, false, fmt.Errorf("%s %s cannot be read: %w", seat.StatePipelines, key, err)
@@ -213,24 +192,7 @@ func (r Reads) pipelineOf(key string, all []string) (seat.Pipeline, bool, error)
if p.Pipeline == "" {
p.Pipeline = key
}
if len(p.Steps) > 0 {
return p, true, nil
}
prefix := stepsPrefix(key)
vals := map[string][]byte{}
for _, k := range all {
if !strings.HasPrefix(k, prefix) || strings.Contains(k[len(prefix):], ".") {
continue
}
v, err := r.Pipelines.Value(k)
if err != nil {
return seat.Pipeline{}, false, fmt.Errorf("%s %s cannot be read: %w", seat.StatePipelines, k, err)
}
if v != nil {
vals[k] = v
}
}
return p, true, composeSteps(&p, key, vals)
return p, true, nil
}
func (r Reads) pipelines() ([]seat.Pipeline, error) {
@@ -241,10 +203,7 @@ func (r Reads) pipelines() ([]seat.Pipeline, error) {
sort.Strings(keys)
var out []seat.Pipeline
for _, k := range keys {
if IsStepsKey(k) {
continue
}
p, ok, err := r.pipelineOf(k, keys)
p, ok, err := r.pipeline(k)
if err != nil {
return nil, err
}
@@ -3,7 +3,6 @@ package pipeline
import (
"encoding/json"
"errors"
"fmt"
"strings"
"testing"
"time"
@@ -50,15 +49,6 @@ func TestReads(t *testing.T) {
Outcome: seat.PipelineOutcomePassed, Started: &t2, Movers: []seat.Mover{{Module: "alpha", Commit: "b3"}}})
put(t, pipes, "main.7.gamma", seat.Pipeline{Pipeline: "main.7.gamma", Branch: "main",
Outcome: seat.PipelineOutcomeRunning, Started: &t1, Movers: []seat.Mover{{Module: "gamma", Commit: "c7"}}})
// A pipeline kept as H-A-187 keeps it: its record's steps empty, each mover's under steps.<pipeline>.<mover>.
put(t, pipes, "next.8", seat.Pipeline{Pipeline: "next.8", Branch: "next", Source: 8, Outcome: seat.PipelineOutcomeRunning,
Started: &t0, Steps: []seat.StepResult{}, Movers: []seat.Mover{{Module: "delta", Commit: "n8"}, {Module: "epsilon", Commit: "n8"}}})
put(t, pipes, "steps.next.8.epsilon", seat.PipelinesByStepsPipelineMoverValue{Steps: []seat.StepResult{{Module: "epsilon",
Step: seat.StepResultStepBuild, Status: "required", Outcome: seat.StepResultOutcomePassed, Output: "next.8.build.epsilon:program"}}})
put(t, pipes, "steps.next.8.delta", seat.PipelinesByStepsPipelineMoverValue{Steps: []seat.StepResult{{Module: "delta",
Step: seat.StepResultStepBuild, Status: "required", Outcome: seat.StepResultOutcomePassed}, {Module: "delta",
Step: seat.StepResultStepApproval, Status: "required", Outcome: seat.StepResultOutcomeWaiting, On: seat.WaitOnPerson, Started: &t2}},
Unlogged: []seat.LogEntry{{Pipeline: "next.8", Module: "delta", Step: seat.StepResultStepBuild, What: seat.LogEntryWhatEnded, At: t1}}})
put(t, kept, "unit.sha256-aa", seat.KeptResult{Level: "unit", Digest: "sha256-aa", Package: "alpha/x"})
put(t, kept, "integration.sha256-bb", seat.KeptResult{Level: "integration", Digest: "sha256-bb", Package: "alpha/y"})
r := Reads{Pipelines: pipes, Kept: kept}
@@ -83,35 +73,6 @@ func TestReads(t *testing.T) {
_, err := r.Show(seat.ShowInput{Pipeline: "main.9.alpha"})
return "", err
}, `error: no pipeline "main.9.alpha" in pipelines`},
{"show composes a pipeline's steps from its mover keys, in its movers' order (H-A-187)", func() (string, error) {
a, err := r.Show(seat.ShowInput{Pipeline: "next.8"})
var s []string
for _, st := range a.Pipeline.Steps {
s = append(s, st.Module+" "+st.Step)
}
return strings.Join(s, ", ") + " | " + strings.Join(a.Outputs, ","), err
}, "delta build, delta approval, epsilon build | next.8.build.epsilon:program"},
{"show naming a mover answers its steps alone", func() (string, error) {
a, err := r.Show(seat.ShowInput{Pipeline: "next.8", Mover: "epsilon"})
var s []string
for _, st := range a.Pipeline.Steps {
s = append(s, st.Module+" "+st.Step)
}
return fmt.Sprintf("%s, %d movers", strings.Join(s, ", "), len(a.Pipeline.Movers)), err
}, "epsilon build, 2 movers"},
{"show refuses a mover the pipeline does not move", func() (string, error) {
_, err := r.Show(seat.ShowInput{Pipeline: "next.8", Mover: "alpha"})
return "", err
}, "error: the pipeline next.8 does not move alpha"},
{"list names a pipeline, never a mover key, its record with its movers and steps counted and no step (H-A-191)", func() (string, error) {
a, err := r.List(seat.ListInput{Branch: "next"})
var s []string
for _, p := range a.Pipelines {
s = append(s, fmt.Sprintf("%s %d movers %d steps, %d steps and %d unlogged answered", p.Pipeline, p.MoverCount, p.StepCount,
len(p.Steps), len(p.Unlogged)))
}
return strings.Join(s, "; "), err
}, "next.8 2 movers 3 steps, 0 steps and 0 unlogged answered"},
{"list by module, newest first", func() (string, error) {
a, err := r.List(seat.ListInput{Module: "alpha"})
return names(a.Pipelines), err
@@ -131,7 +92,7 @@ func TestReads(t *testing.T) {
s = append(s, w.Pipeline+" "+w.Step+" "+w.On+" "+w.Since.Format("15:04"))
}
return strings.Join(s, ";"), err
}, "main.7.alpha approval person 10:01;next.8 approval person 10:02"},
}, "main.7.alpha approval person 10:01"},
{"why-carried names where a carried step came from", func() (string, error) {
a, err := r.WhyCarried(seat.WhyCarriedInput{Pipeline: "main.7.alpha", Step: "unit", Artifact: "alpha:program"})
return a.CarriedFrom, err
@@ -441,7 +441,7 @@ func (h *Hearer) said(p seat.Pipeline, asked []int, ended bool) error {
// endEntry is the step log's entry of a step's end, with the keys of the kept results it wrote or carried.
func endEntry(key string, s seat.StepResult, at time.Time, kept ...string) seat.LogEntry {
return seat.LogEntry{Pipeline: key, Module: s.Module, Step: s.Step, Artifact: s.Artifact, What: seat.LogEntryWhatEnded, Outcome: s.Outcome,
return seat.LogEntry{Pipeline: key, Step: s.Step, Artifact: s.Artifact, What: seat.LogEntryWhatEnded, Outcome: s.Outcome,
Output: s.Output, Kept: kept, At: at}
}
@@ -7,10 +7,9 @@ import (
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
)
// sameEntry says whether two step-log entries are one: the same pipeline, mover (H-A-190), step, artifact, end or start and
// time.
// sameEntry says whether two step-log entries are one: the same pipeline, step, artifact, end or start and time.
func sameEntry(a, b seat.LogEntry) bool {
return a.Pipeline == b.Pipeline && a.Module == b.Module && a.Step == b.Step && a.Artifact == b.Artifact && a.What == b.What && a.At.Equal(b.At)
return a.Pipeline == b.Pipeline && a.Step == b.Step && a.Artifact == b.Artifact && a.What == b.What && a.At.Equal(b.At)
}
// appendUnlogged appends the unlogged entries of pipeline key to the step log in their order and clears each appended one in a
@@ -146,15 +146,9 @@ func (r FetchRequest) Check() error {
if r.Dir != "" && (outside(r.Dir) || strings.HasPrefix(r.Dir, "/")) {
return fmt.Errorf("the module directory %q leaves the repository", r.Dir)
}
// A path is relative to the module's directory; one that leaves it but stays inside the repository (a sibling the
// artifact's go.mod requires, "../<module>/<dir>") is accepted and narrowed from the repository's root, and one that
// leaves the repository is refused (design 47 H-A-198, H-A-199).
for _, p := range r.Paths {
if p == "" || strings.HasPrefix(p, "/") || path.Clean(p) == "." {
return fmt.Errorf("the path %q is not a path relative to the module's directory", p)
}
if outside(path.Join(r.Dir, p)) {
return fmt.Errorf("the path %q leaves the repository", p)
if p == "" || strings.HasPrefix(p, "/") || outside(p) || path.Clean(p) == "." {
return fmt.Errorf("the path %q leaves the module's directory or is not a path in it", p)
}
}
return nil
@@ -37,11 +37,6 @@ func TestAFetchRequestCarriesTheNamedPathsThroughTheToolCall(t *testing.T) {
if dirOnly, err := ParseFetchRequest(map[string]any{"repository": "novox/x", "commit": strings.Repeat("b", 40), "dir": "modules/a"}); err != nil || dirOnly.Dir != "modules/a" || dirOnly.Paths != nil {
t.Fatalf("a directory with no paths: %+v, %v", dirOnly, err)
}
// A sibling of the same repository is a path that leaves the module's directory, not the repository (H-A-199).
if sib, err := ParseFetchRequest(map[string]any{"repository": "novox/x", "commit": strings.Repeat("b", 40), "dir": "modules/a",
"paths": []any{"pkg", "../b/client"}}); err != nil || len(sib.Paths) != 2 {
t.Fatalf("a sibling path inside the repository: %+v, %v", sib, err)
}
for _, c := range []struct {
set map[string]any
says string
@@ -51,8 +46,7 @@ func TestAFetchRequestCarriesTheNamedPathsThroughTheToolCall(t *testing.T) {
{map[string]any{"commit": "abc"}, "is not a full commit"},
{map[string]any{"base": "abc"}, "is not a trunk commit"},
{map[string]any{"paths": []any{"src"}}, "names no module directory"},
{map[string]any{"dir": "modules/a", "paths": []any{"../../../b"}}, "leaves the repository"},
{map[string]any{"dir": "modules/a", "paths": []any{"/b"}}, "not a path relative to the module's directory"},
{map[string]any{"dir": "modules/a", "paths": []any{"../b"}}, "leaves the module's directory"},
{map[string]any{"dir": "../a", "paths": []any{"src"}}, "leaves the repository"},
{map[string]any{"dir": "modules/a", "paths": []any{3}}, "paths is a list of strings"},
{map[string]any{"packages": map[string]any{"git.novox.be/novox/mesh-sdk/go": "main"}}, "neither a full commit nor a Go semantic version"},
@@ -5,6 +5,7 @@
// name for each package whoever reads it (mesh-sdk composer.Request.Packages).
// Design 47 H1, proof 2: this comment is the mesh-delivery-sdk half of a two-module change its pipeline delivers.
// Design 47 H1, proof 2 again: the provider half of the edge mesh-delivery-sdk to mesh-build-node.
// Design 47 H1, proof 2 once more: the provider half of the edge mesh-delivery-sdk to mesh-delivery-builder.
package packagename
import (
@@ -100,14 +100,6 @@ func TestAContractsModulesGeneratedFilesAreComparedWithTheGenerator(t *testing.T
if got, err := OutOfStep(plain, GeneratedRootGo); err != nil || len(got) != 0 {
t.Errorf("a module naming no generated-from compared: %v %v", got, err)
}
// This repository's own contracts module is in step with the generator under both roots: a hand-written file
// under a generated root (go/seats/meshcatalog/manifestdigest.go, pipeline main.43) failed every unit step of its
// mover; a seat's hand-written helper lives in its client (design 47 H, H-A-200; ADR 0608 §5).
for _, root := range []string{GeneratedRootGo, GeneratedRootTS} {
if got, err := OutOfStep(filepath.Join("..", "..", "mesh-delivery-contracts"), root); err != nil || len(got) != 0 {
t.Errorf("mesh-delivery-contracts, root %s: %v %v, want nothing out of step", root, got, err)
}
}
// The build ask answers them beside what it built, and with a failed build too.
built := 0
+1 -3
View File
@@ -25,11 +25,9 @@ func Narrow(tree string, r sdkstep.FetchRequest) error {
}
keep := map[string]bool{".git": true}
for _, p := range r.Paths {
// Kept by its path from the repository's root: a path leaving the module's directory but not the repository is
// narrowed from the root like any other (design 47 H-A-199).
rel := path.Join(r.Dir, p)
if strings.HasPrefix(rel, "../") || rel == ".." || path.IsAbs(p) {
return fmt.Errorf("the named path %q leaves the repository", p)
return fmt.Errorf("the named path %q leaves the module's directory", p)
}
if _, err := os.Lstat(filepath.Join(tree, filepath.FromSlash(rel))); err != nil {
return fmt.Errorf("the named path %q is not in the fetched tree: %w", p, err)
@@ -68,25 +68,6 @@ func TestNarrowKeepsAModuleDirectorysPathsAndFailsOnAnAbsentOne(t *testing.T) {
}
}
// A path leaving the module's directory but not the repository is kept from the root; one leaving the repository is
// refused, naming the repository (design 47 H-A-198, H-A-199).
func TestNarrowKeepsASiblingOfTheRepositoryAndRefusesOneOutsideIt(t *testing.T) {
tree := t.TempDir()
for _, f := range []string{"modules/a/module.json", "modules/a/pkg/go.mod", "modules/a/notes.md", "modules/b/client/go.mod",
"modules/b/other.go", "root.txt"} {
writeTreeFile(t, tree, f, "x")
}
if err := Narrow(tree, sdkstep.FetchRequest{Dir: "modules/a", Paths: []string{"../../../x"}}); err == nil || !strings.Contains(err.Error(), "leaves the repository") {
t.Fatalf("a path leaving the repository gave %v", err)
}
if err := Narrow(tree, sdkstep.FetchRequest{Dir: "modules/a", Paths: []string{"pkg", "../b/client"}}); err != nil {
t.Fatal(err)
}
if got := strings.Join(filesUnder(t, tree), ","); got != "modules/a/module.json,modules/a/pkg/go.mod,modules/b/client/go.mod" {
t.Fatalf("the narrowed tree holds %s", got)
}
}
// A build that needs package.json or go.mod names it among its paths, and so keys on it (ADR 0499 clause 4).
func TestNarrowKeepsAManifestOfTheLanguageOnlyWhenNamed(t *testing.T) {
tree := t.TempDir()
+2 -13
View File
@@ -38,12 +38,9 @@ type artifact struct {
From string `json:"from"`
Binary string `json:"binary"`
Loads []string `json:"loads"`
Paths []string `json:"paths"`
Series string `json:"series"`
}
type manifest struct {
Packages map[string]string `json:"packages"`
Module string `json:"module"`
Version string `json:"version"`
Slug string `json:"slug"`
@@ -179,16 +176,8 @@ func TestManifestProcess(t *testing.T) { // 1240/AC1
if _, bus := p["bus"]; bus {
t.Errorf("the process declares a bus: it runs on its own broker credential")
}
if len(m.Build.Artifacts) != 3 {
t.Fatalf("artifacts = %+v, want the program, the tools and the package tellerevents", m.Build.Artifacts)
}
// H-A-188: the assigner is at 0.3, so its package asks, which this module imports, is accepted in that series.
if got := m.Packages["git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"]; got != "0.3" || len(m.Packages) != 1 {
t.Errorf("packages = %v, want asks accepted in the assigner's series 0.3", m.Packages)
}
// H-A-186 (design 47 G §370): tellerevents, which the assigner imports, is a declared package.
if a := m.Build.Artifacts[2]; a.Name != "tellerevents" || a.Kind != "package" || a.Language != "go" || !reflect.DeepEqual(a.Paths, []string{"tellerevents"}) || a.Series != "0.1" {
t.Errorf("package artifact = %+v", a)
if len(m.Build.Artifacts) != 2 {
t.Fatalf("artifacts = %+v, want the program and the tools", m.Build.Artifacts)
}
a := m.Build.Artifacts[0]
if st, err := os.Stat(a.From); err != nil || !st.IsDir() || a.Binary != "mesh-delivery-teller" || !strings.HasSuffix(a.From, a.Binary) {
-12
View File
@@ -1,9 +1,6 @@
{
"module": "mesh-delivery-teller",
"version": "0.1",
"packages": {
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks": "0.3"
},
"slug": "teller",
"tools": [
"teller_end_ask",
@@ -126,15 +123,6 @@
"loads": [
"mesh-delivery-teller-tools"
]
},
{
"name": "tellerevents",
"kind": "package",
"language": "go",
"paths": [
"tellerevents"
],
"series": "0.1"
}
]
},
+3 -7
View File
@@ -70,7 +70,6 @@ type claim struct {
}
type manifest struct {
Packages map[string]string `json:"packages"`
Claims []claim `json:"claims"`
State []state `json:"state"`
Module string `json:"module"`
@@ -314,12 +313,9 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7
func TestManifestIsWholeInItsRepository(t *testing.T) { // 1134/AC7
m, _ := load(t)
// H-A-188: the assigner is at 0.3, so its package asks, which this module imports, is accepted in that series.
if got := m.Packages["git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"]; got != "0.3" || len(m.Packages) != 1 {
t.Errorf("packages = %v, want asks accepted in the assigner's series 0.3", m.Packages)
}
// H-A-186 (design 47 G §370): viewstate, which the gate, the assigner and the teller import, is a declared package.
if len(m.Build.Artifacts) != 5 || m.Build.Artifacts[4].Name != "viewstate" {
// Issue 1966's bootstrap: the package artifact viewstate is out of the manifest until the new builder runs; the
// follow-up puts the count back at 5.
if len(m.Build.Artifacts) != 4 {
t.Fatalf("artifacts = %+v", m.Build.Artifacts)
}
for _, a := range m.Build.Artifacts {
-12
View File
@@ -1,9 +1,6 @@
{
"module": "mesh-delivery-view",
"version": "0.1",
"packages": {
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks": "0.3"
},
"moved": true,
"slug": "view",
"tools": [
@@ -237,15 +234,6 @@
"name": "files",
"kind": "archive",
"from": "files"
},
{
"name": "viewstate",
"kind": "package",
"language": "go",
"paths": [
"viewstate"
],
"series": "0.1"
}
]
},
@@ -39,7 +39,6 @@ import (
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/announce"
@@ -181,20 +180,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
Create: creating{ctx: ctx, js: js}, Consumers: creating{ctx: ctx, js: js}, NamedPurge: creating{ctx: ctx, js: js},
Objects: sealedObjects{ctx: ctx, js: js}, Purging: ownPurging{ctx: ctx, js: js, module: account.Module}, Logf: logger.Printf,
Bus: grants.BusAddress{URL: account.URL, Fingerprint: account.Fingerprint}}
// The setting grants-for-revisions-per-branch, read at the start, and the watched branches the part follows: grants-for
// keeps those branches' revisions alone, so many to a branch (design 47 H H-A-195).
perBranch, err := grants.RevisionsPerBranchSetting(strings.TrimSpace(getenv(grants.EnvRevisionsPerBranch)))
if err != nil {
return err
}
f.Part.RevisionsPerBranch, f.Part.Followed = perBranch, branches.Followed
f.Watch = fv
f.Part.ReadingRunning(nodeengine.StreamRunning)
// The catalog-revision event names each branch's newest revision, so grants-for asks the seat mesh-catalog's revision only
// for a revision it does not hold (design 47 H H-A-193), heard on the same reader beside the branch follower.
if err := reader.AdoptEvent(meshcatalog.Seat, meshcatalog.EventCatalogRevision, f.Part.HearCatalogRevision); err != nil {
return err
}
f.HearKept = func() error { return hearRunning(ctx, account, js, f, logger.Printf) }
// The first view of the bindings (design 25 bindings-first-view): the seat mesh-broker's list, asked at each start and by
// hand, with the seat's events held until its answer is applied or its bound passes.
-146
View File
@@ -83,8 +83,6 @@ type fakeBus struct {
refuseCreate bool
// refuseUsers refuses the bus module's nats_users: the loaded list cannot be read.
refuseUsers bool
// revisionNumber, when set, numbers each revision mesh-catalog's revision answers (newest, candidate), one entry a page.
revisionNumber map[string]int64
// rootFree answers a machine's engine's root-free verb: nil is no answerer at all (no responders).
rootFree func(machines []string) (any, error)
// consumers are the durable consumers the bus holds, by stream, as nats_streams answers them for one stream; and
@@ -381,19 +379,6 @@ func (b *fakeBus) Call(key string, body any) (json.RawMessage, error) {
_ = json.Unmarshal([]byte(manifest), &m)
out.Entries = append(out.Entries, meshcatalog.CatalogEntry{Module: module, Manifest: m})
}
if n, numbered := b.revisionNumber[which]; numbered {
slices.SortFunc(out.Entries, func(x, y meshcatalog.CatalogEntry) int { return strings.Compare(x.Module, y.Module) })
out.Revision = &meshcatalog.CatalogRevision{Branch: in.Branch, Kind: "trunk", Revision: n}
if in.Group != "" {
out.Revision.Kind, out.Revision.Group, out.Revision.Heads = "candidate", in.Group, "h1"
}
for len(out.Entries) > 0 && out.Entries[0].Module <= in.After {
out.Entries = out.Entries[1:]
}
if len(out.Entries) > 1 {
out.Entries, out.Next = out.Entries[:1], out.Entries[0].Module
}
}
return json.Marshal(out)
case grants.CallStreams:
if stream, _ := args["stream"].(string); stream != "" {
@@ -663,17 +648,6 @@ func TestGrants(t *testing.T) {
if _, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || m.SecretsOwner != "mesh-grants" {
t.Errorf("own-secrets %v owner %q: a credential of its own, broker", m.OwnSecrets, m.SecretsOwner)
}
// H-A-186 (design 47 G §370): grantsstate (imported by the assigner, the vault and the node-engine) and busgrants
// (imported by the bus module and the watchman) are declared Go packages of the module, so every release publishes them.
var packages []string
for _, a := range m.Build.Artifacts {
if a.Kind == catalogue.ArtifactPackage && a.Language == "go" && len(a.Paths) == 1 && a.Paths[0] == a.Name && a.Series == "0.1" {
packages = append(packages, a.Name)
}
}
if !slices.Equal(packages, []string{"grantsstate", "busgrants"}) {
t.Errorf("Go packages %v, want grantsstate and busgrants of series 0.1", packages)
}
})
t.Run("no timer, no update or delete, a publish only of conditions, a create only of declared streams", func(t *testing.T) {
@@ -5172,126 +5146,6 @@ func TestGrantsFor(t *testing.T) {
if _, err := p.GrantsFor(meshgrants.GrantsForInput{Branch: "main"}); err == nil {
t.Error("grants-for with neither a source nor a run answered")
}
// A revision never changes, so one read whole is kept: a second grants-for of the same revisions reads only the first page
// of the candidate, which names a revision kept, and none of the newest, whose number the first ask named (H-A-193: it asks
// once); once the catalog-revision events name both, a grants-for reads no page.
b = fixture()
b.revision = map[string]map[string]string{"newest": {"mod-x": xManifest, "mod-y": xManifest}, "candidate": {"mod-x": candidateX, "mod-y": xManifest}}
b.revisionNumber = map[string]int64{"newest": 9, "candidate": 8}
p = partOn(b)
pages := func() int {
b.mu.Lock()
defer b.mu.Unlock()
n := 0
for _, c := range b.calls {
if c == grants.CallRevision {
n++
}
}
return n
}
first, _ := p.GrantsFor(meshgrants.GrantsForInput{Branch: "main", Source: "7"})
read := pages()
again, _ := p.GrantsFor(meshgrants.GrantsForInput{Branch: "main", Source: "7"})
if read != 4 || pages()-read != 1 {
t.Errorf("the revisions were read in %d pages, then %d again, want 4 then 1 (the candidate's first page)", read, pages()-read)
}
if len(first.Modules) != 1 || !reflect.DeepEqual(first, again) {
t.Errorf("grants-for from kept revisions answers %+v, first %+v", again, first)
}
p = partOn(b)
for _, ev := range []string{`{"v":1,"kind":"trunk","branch":"main","revision":8,"source":7}`, `{"v":1,"kind":"trunk","branch":"main","revision":9,"source":8}`} {
if err := p.HearCatalogRevision(context.Background(), []byte(ev)); err != nil {
t.Fatal(err)
}
}
_, _ = p.GrantsFor(meshgrants.GrantsForInput{Branch: "main", Source: "7"})
read = pages()
heard, _ := p.GrantsFor(meshgrants.GrantsForInput{Branch: "main", Source: "7"})
if pages() != read || !reflect.DeepEqual(first, heard) {
t.Errorf("after the events named both revisions held, grants-for read %d pages and answered %+v, want none and %+v", pages()-read, heard, first)
}
}
// grants-for keeps what H-A-195 says of the catalog: so many revisions to a branch as the setting
// grants-for-revisions-per-branch says, a run's candidate counted within its branch's; only the followed branches', freed when
// one leaves them; and an event below the newest held is the numbering started again, which drops what is kept.
func TestGrantsForKeepsWhatItMay(t *testing.T) {
setting := func(t *testing.T, text string) int {
path := filepath.Join(t.TempDir(), "grants-for-revisions-per-branch")
if err := os.WriteFile(path, []byte(text), 0o600); err != nil {
t.Fatal(err)
}
n, err := grants.RevisionsPerBranchSetting(path)
if err != nil {
t.Fatal(err)
}
return n
}
hear := func(t *testing.T, p *grants.Part, revision, source int) {
ev := fmt.Sprintf(`{"v":1,"kind":"trunk","branch":"main","revision":%d,"source":%d}`, revision, source)
if err := p.HearCatalogRevision(context.Background(), []byte(ev)); err != nil {
t.Fatal(err)
}
}
for _, row := range []struct {
name string
// run asks the candidate by a run, else by the source revision 7; then is done after one grants-for; pages the
// revision pages the grants-for after it reads.
run bool
// perBranch is the part's bound from the start; 0 its default.
perBranch int
then func(t *testing.T, b *fakeBus, p *grants.Part, followed *[]string)
pages int
}{
{name: "the setting bounds a branch's revisions: with 2, the candidate is dropped once two newer are kept", pages: 2,
then: func(t *testing.T, b *fakeBus, p *grants.Part, _ *[]string) {
p.RevisionsPerBranch = setting(t, "2\n")
hear(t, p, 10, 0)
b.revisionNumber["newest"] = 10
_, _ = p.GrantsFor(meshgrants.GrantsForInput{Branch: "main", Source: "7"})
}},
{name: "a run's candidate counts within its branch's bound: with 1, the candidate and the newest each drop the other", run: true,
perBranch: 1, pages: 4, then: func(*testing.T, *fakeBus, *grants.Part, *[]string) {}},
{name: "a branch that leaves the watched branches frees its revisions", pages: 4,
then: func(t *testing.T, _ *fakeBus, p *grants.Part, followed *[]string) {
*followed = nil
_, _ = p.GrantsFor(meshgrants.GrantsForInput{Branch: "main", Source: "7"})
*followed = []string{"main"}
}},
{name: "an event below the newest held drops what is kept of the branch", pages: 4,
then: func(t *testing.T, _ *fakeBus, p *grants.Part, _ *[]string) { hear(t, p, 3, 0) }},
} {
t.Run(row.name, func(t *testing.T) {
b := fixture()
b.revision = map[string]map[string]string{"newest": {"mod-x": xManifest, "mod-y": xManifest}, "candidate": {"mod-x": xManifest, "mod-y": xManifest}}
b.revisionNumber = map[string]int64{"newest": 9, "candidate": 8}
followed := []string{"main"}
p := partOn(b)
p.Followed = func() []string { return followed }
p.RevisionsPerBranch = row.perBranch
in := meshgrants.GrantsForInput{Branch: "main", Source: "7"}
if row.run {
in = meshgrants.GrantsForInput{Branch: "main", Run: "r1"}
} else {
hear(t, p, 8, 7)
hear(t, p, 9, 8)
}
if got, err := p.GrantsFor(in); err != nil || got.Error != "" {
t.Fatalf("grants-for: %+v, %v", got, err)
}
row.then(t, b, p, &followed)
before := len(slices.DeleteFunc(slices.Clone(b.calls), func(c string) bool { return c != grants.CallRevision }))
_, _ = p.GrantsFor(in)
if got := len(slices.DeleteFunc(slices.Clone(b.calls), func(c string) bool { return c != grants.CallRevision })) - before; got != row.pages {
t.Errorf("grants-for read %d revision pages, want %d", got, row.pages)
}
})
}
if n, err := grants.RevisionsPerBranchSetting(filepath.Join(t.TempDir(), "none")); n != grants.DefaultRevisionsPerBranch || err != nil {
t.Errorf("with no file the setting is %d, %v; want its default", n, err)
}
}
// heldHalf is an assignment half holding rev, or none.
@@ -1,23 +1,18 @@
package grants
import (
"context"
"encoding/json"
"errors"
"fmt"
"io/fs"
"maps"
"os"
"slices"
"sort"
"strconv"
"strings"
"sync"
"git.novox.be/novox/mesh-contracts/identity-and-access/go/seats/meshgrants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// **The seat mesh-grants' read `grants-for`** (design 47 G §441, §401, A-205): the grants a candidate would compose, per
@@ -32,240 +27,10 @@ const RunGroup = "run-"
// CallRevision is the seat mesh-catalog's `revision`, as the part calls it.
const CallRevision = "seat:" + meshcatalog.Seat + "." + meshcatalog.VerbRevision
// DefaultRevisionsPerBranch is the setting grants-for-revisions-per-branch's default (design 47 H H-A-195): how many catalog
// revisions of one branch grants-for keeps, a run's candidate counted within its branch's, the oldest read dropped first;
// twice as many source revisions are kept named by the revision taken in from each.
const DefaultRevisionsPerBranch = 4
// EnvRevisionsPerBranch names the file the manifest writes the setting grants-for-revisions-per-branch into.
const EnvRevisionsPerBranch = "MESH_GRANTS_REVISIONS_PER_BRANCH_FILE"
// RevisionsPerBranchSetting reads the setting grants-for-revisions-per-branch from its file, read at the start: a missing or
// empty file is its default, one that holds no positive whole number an error naming the file.
func RevisionsPerBranchSetting(path string) (int, error) {
if path == "" {
return DefaultRevisionsPerBranch, nil
}
raw, err := os.ReadFile(path)
if errors.Is(err, fs.ErrNotExist) {
return DefaultRevisionsPerBranch, nil
}
if err != nil {
return 0, err
}
v := strings.TrimSpace(string(raw))
if v == "" {
return DefaultRevisionsPerBranch, nil
}
n, err := strconv.Atoi(v)
if err != nil || n < 1 {
return 0, fmt.Errorf("%s holds %q, which is not a positive whole number", path, v)
}
return n, nil
}
// revisionsPerBranch is the part's bound of kept revisions per branch: its setting, else the default.
func (p *Part) revisionsPerBranch() int {
if p.RevisionsPerBranch > 0 {
return p.RevisionsPerBranch
}
return DefaultRevisionsPerBranch
}
// following frees what is kept of every branch the part follows no more, and says whether branch is followed: only a followed
// branch's revisions are kept (H-A-195). With no Followed (the tools) every branch is.
func (p *Part) following(branch string) bool {
if p.Followed == nil {
return true
}
followed := map[string]bool{}
for _, b := range p.Followed() {
followed[keys.EscapeName(b)] = true
}
p.revisions.free(followed)
return followed[branch]
}
// revisionsKept keeps the catalog revisions grants-for read, whole, as manifests by module (design 47 H H-A-195): a revision
// never changes, so one read is read for good (as the runner keeps its revisionCache), a cache of the catalog seat's own truth
// and never a state of the mesh. Each is kept by the branch grants-for was asked of (escaped as a key name is), its kind,
// group, heads and number; the newest few of each followed branch are kept in memory for the process's life. It also
// holds what the catalog-revision event names (H-A-193): each branch's newest trunk revision, and the revision each source
// revision was taken in as, so grants-for asks the seat mesh-catalog's revision only for a revision it does not hold, never
// to learn the newest.
type revisionsKept struct {
mu sync.Mutex
order map[string][]string
kept map[string]map[string]catalogue.Manifest
newest map[string]int64
sources map[string]map[int64]int64
sourced map[string][]int64
}
// revisionKey names a revision of branch; every kind but a candidate's is the branch's own series.
func revisionKey(branch string, r meshcatalog.CatalogRevision) string {
if r.Kind != "candidate" {
r.Kind, r.Group, r.Heads = "trunk", "", ""
}
return strings.Join([]string{branch, r.Kind, r.Group, r.Heads, strconv.FormatInt(r.Revision, 10)}, "\x00")
}
func (k *revisionsKept) init() {
if k.kept == nil {
k.kept, k.order, k.newest = map[string]map[string]catalogue.Manifest{}, map[string][]string{}, map[string]int64{}
k.sources, k.sourced = map[string]map[int64]int64{}, map[string][]int64{}
}
}
func (k *revisionsKept) get(branch string, r meshcatalog.CatalogRevision) (map[string]catalogue.Manifest, bool) {
k.mu.Lock()
defer k.mu.Unlock()
m, ok := k.kept[revisionKey(branch, r)]
return m, ok
}
func (k *revisionsKept) put(branch string, r meshcatalog.CatalogRevision, manifests map[string]catalogue.Manifest, max int) {
k.mu.Lock()
defer k.mu.Unlock()
k.init()
key := revisionKey(branch, r)
if _, ok := k.kept[key]; !ok {
k.order[branch] = append(k.order[branch], key)
}
k.kept[key] = manifests
for len(k.order[branch]) > max {
delete(k.kept, k.order[branch][0])
k.order[branch] = k.order[branch][1:]
}
if r.Kind != "candidate" && r.Source != 0 {
k.sourceLocked(branch, r.Source, r.Revision, max)
}
}
// free drops what is kept of each branch not in followed.
func (k *revisionsKept) free(followed map[string]bool) {
k.mu.Lock()
defer k.mu.Unlock()
for _, m := range []map[string]bool{boolsOf(k.order), boolsOf(k.newest), boolsOf(k.sources)} {
for branch := range m {
if !followed[branch] {
k.dropLocked(branch)
}
}
}
}
func boolsOf[V any](m map[string]V) map[string]bool {
out := map[string]bool{}
for k := range m {
out[k] = true
}
return out
}
// dropLocked drops everything kept of branch.
func (k *revisionsKept) dropLocked(branch string) {
for _, key := range k.order[branch] {
delete(k.kept, key)
}
delete(k.order, branch)
delete(k.newest, branch)
delete(k.sources, branch)
delete(k.sourced, branch)
}
func (k *revisionsKept) sourceLocked(branch string, source, revision int64, max int) {
if k.sources[branch] == nil {
k.sources[branch] = map[int64]int64{}
}
if _, ok := k.sources[branch][source]; !ok {
k.sourced[branch] = append(k.sourced[branch], source)
}
k.sources[branch][source] = revision
for len(k.sourced[branch]) > 2*max {
delete(k.sources[branch], k.sourced[branch][0])
k.sourced[branch] = k.sourced[branch][1:]
}
}
// heard takes a trunk revision a catalog-revision event names as its branch's newest. One below the newest held is the
// numbering started again (design 47 H H-A-141): what is kept of the branch was of the earlier numbering, and is dropped.
func (k *revisionsKept) heard(branch string, revision, source int64, max int) {
k.mu.Lock()
defer k.mu.Unlock()
k.init()
if held, ok := k.newest[branch]; ok && revision < held {
k.dropLocked(branch)
}
k.newest[branch] = revision
if source != 0 {
k.sourceLocked(branch, source, revision, max)
}
}
// asked takes the newest revision an ask answered, when no event named one since the start (H-A-193: it asks once).
func (k *revisionsKept) asked(branch string, revision int64) {
k.mu.Lock()
defer k.mu.Unlock()
k.init()
if _, ok := k.newest[branch]; !ok {
k.newest[branch] = revision
}
}
// known is the revision in names that the part knows the number of without asking: the branch's newest an event named, or
// the revision a source revision was taken in as; false when it knows none.
func (k *revisionsKept) known(in meshcatalog.RevisionInput) (int64, bool) {
k.mu.Lock()
defer k.mu.Unlock()
switch {
case in.Group != "" || in.Heads != "" || in.Revision != 0:
return 0, false
case in.Source != 0:
n, ok := k.sources[in.Branch][in.Source]
return n, ok
default:
n, ok := k.newest[in.Branch]
return n, ok
}
}
// HearCatalogRevision takes one catalog-revision event of the seat mesh-catalog, as the part's process hears it (design 47 H
// H-A-193): a trunk revision is its branch's newest, and the revision its source revision was taken in as; a candidate's moves
// nothing here. Data that is no such event is an error.
func (p *Part) HearCatalogRevision(_ context.Context, data []byte) error {
var ev meshcatalog.CatalogRevisionEvent
if err := json.Unmarshal(data, &ev); err != nil || ev.Revision <= 0 {
return fmt.Errorf("not a %s event: %.200s", meshcatalog.EventCatalogRevision, data)
}
if ev.Kind == "candidate" || !p.following(ev.Branch) {
return nil
}
p.revisions.heard(ev.Branch, ev.Revision, ev.Source, p.revisionsPerBranch())
return nil
}
// revisionEntries reads a catalog graph revision's entries, page after page, as manifests by module. none says the branch has
// no revision yet. A revision whose number the part knows (the branch's newest an event named, a source revision's) and holds
// is answered from what it holds with no read; one it knows but does not hold is read by that number. Otherwise a revision
// read whole is kept: once the first page of an answer names a revision that is kept, it is answered from what is kept,
// without the pages after it. The branch's newest is read on by the number its first page names, so its pages are of one
// revision though a newer one is written between them; a revision whose pages name another is not kept.
// no revision yet.
func (p *Part) revisionEntries(in meshcatalog.RevisionInput) (manifests map[string]catalogue.Manifest, none bool, err error) {
branch := in.Branch
followed := p.following(branch)
newestAsked := in.Group == "" && in.Heads == "" && in.Source == 0 && in.Revision == 0
if n, ok := p.revisions.known(in); ok {
if m, held := p.revisions.get(branch, meshcatalog.CatalogRevision{Revision: n}); held {
return m, false, nil
}
if newestAsked {
in.Revision = n
}
}
manifests = map[string]catalogue.Manifest{}
var first *meshcatalog.CatalogRevision
keep := true
for {
raw, err := p.Call.Call(CallRevision, in)
if err != nil {
@@ -278,23 +43,6 @@ func (p *Part) revisionEntries(in meshcatalog.RevisionInput) (manifests map[stri
if a.NoneYet {
return nil, true, nil
}
switch {
case a.Revision == nil:
keep = false
case first == nil:
if newestAsked && followed {
p.revisions.asked(branch, a.Revision.Revision)
}
if m, ok := p.revisions.get(branch, *a.Revision); ok {
return m, false, nil
}
first = a.Revision
if newestAsked {
in.Revision = a.Revision.Revision
}
case revisionKey(branch, *a.Revision) != revisionKey(branch, *first):
keep = false
}
for _, e := range a.Entries {
body, err := json.Marshal(e.Manifest)
if err != nil || len(e.Manifest) == 0 {
@@ -307,9 +55,6 @@ func (p *Part) revisionEntries(in meshcatalog.RevisionInput) (manifests map[stri
manifests[e.Module] = m
}
if a.Next == "" || a.Next == in.After {
if keep && first != nil && followed {
p.revisions.put(branch, *first, manifests, p.revisionsPerBranch())
}
return manifests, false, nil
}
in.After = a.Next
+3 -11
View File
@@ -194,19 +194,11 @@ type Part struct {
// SeatList is the seat list reader, which gates a module whose seats the list has not named (inputs.go, SeatGate); nil
// gates none.
SeatList SeatGate
// RevisionsPerBranch is the setting grants-for-revisions-per-branch, read at the start (grantsfor.go, design 47 H H-A-195);
// 0 is its default.
RevisionsPerBranch int
// Followed is the watched branches the part follows, as named (the branch follower's): grants-for keeps the revisions
// of these alone, and frees a branch's once it leaves them; nil in the tools, where every branch is kept.
Followed func() []string
cache Manifests
// revisions are the catalog revisions grants-for read, kept whole by their number (grantsfor.go).
revisions revisionsKept
mu sync.Mutex
last Result
made creations
mu sync.Mutex
last Result
made creations
// askSeen is the revision of the last ask to compose (KeyAsked) this part acted on.
askSeen uint64
// namedPurged says the last pass found the build seats' named asks unused and purged them, so the next purges only
+1 -38
View File
@@ -64,16 +64,6 @@
"seat:mesh-broker.list",
"seat:mesh-recovery.kept-points"
],
"config": {
"grants-for-revisions-per-branch": {
"type": "number",
"description": "how many catalog revisions of one watched branch the grants part keeps once read for grants-for, a run's candidate counted within its branch's, the oldest dropped first; each some 1.4 MB (design 47 H H-A-195).",
"minimum": 1,
"maximum": 32,
"trusted": false,
"default": 4
}
},
"emits": [
"asserted",
"grant-written",
@@ -166,14 +156,6 @@
"place": ".",
"unknown": "program"
},
{
"id": "grants-for-revisions-per-branch",
"type": "file",
"path": "${dir:state}/grants-for-revisions-per-branch",
"owner": "mesh-grants",
"mode": "0600",
"content": "${config:grants-for-revisions-per-branch}"
},
{
"id": "grants",
"type": "process",
@@ -183,8 +165,7 @@
"./mesh-grants"
],
"env": {
"MESH_BROKER_FILE": "${dir:state}/broker",
"MESH_GRANTS_REVISIONS_PER_BRANCH_FILE": "${dir:state}/grants-for-revisions-per-branch"
"MESH_BROKER_FILE": "${dir:state}/broker"
},
"user": "mesh-grants",
"health": {
@@ -204,24 +185,6 @@
"loads": [
"mesh-grants"
]
},
{
"name": "grantsstate",
"kind": "package",
"language": "go",
"paths": [
"grantsstate"
],
"series": "0.1"
},
{
"name": "busgrants",
"kind": "package",
"language": "go",
"paths": [
"busgrants"
],
"series": "0.1"
}
]
}