Compare commits

..
149 changed files with 844 additions and 4901 deletions
@@ -19,7 +19,6 @@ import (
"runtime"
"slices"
"strings"
"sync"
"testing"
"time"
@@ -204,17 +203,11 @@ func (w *watchKV) Keys(context.Context, ...jetstream.WatchOpt) ([]string, error)
return nil, jetstream.ErrNoKeysFound
}
// putIntoMu holds putInto's writes one at a time: the composition step starts its machines' runs side by side, and each may
// raise a finding into the same map.
var putIntoMu sync.Mutex
// putInto is an assign.Findings Put that keeps each finding in into, as the finding writer would, by its key and its words.
func putInto(into map[string][]byte) func(mcseat.Finding) error {
return func(f mcseat.Finding) error {
raw, err := json.Marshal(f)
putIntoMu.Lock()
into[views.FindingKey(f.Scope, f.ID, f.Kind)] = raw
putIntoMu.Unlock()
return err
}
}
@@ -169,7 +169,7 @@ func (e *pipelineEnds) take(p mpr.PipelineEnded) (bool, error) {
changed := false
var kept []tierMover
defer func() {
if err := e.startTiers(p.Pipeline, p.Source, kept); err != nil {
if err := e.startTiers(p.Pipeline, kept); err != nil {
e.logf("the tier progress of %s is not written: %v", p.Pipeline, err)
}
}()
@@ -42,14 +42,11 @@ type tierMover struct {
Tier int64 `json:"tier"`
}
// tierProgress is a pipeline's tier progress (H-A-56): the tier whose wants are written, since when, the movers, and the source
// revision its pipeline-ended names, said in pipeline-wanted beside the key's revision (design 47 H-A-214, H-A-216); zero in a
// value kept before the field, read as the key's number.
// tierProgress is a pipeline's tier progress (H-A-56): the tier whose wants are written, since when, and the movers.
type tierProgress struct {
Tier int64 `json:"tier"`
Since time.Time `json:"since"`
Movers []tierMover `json:"movers"`
Source int64 `json:"source,omitempty"`
}
// tierKey is the key of pipeline's tier progress; pipeline is the runner's key `<branch>.<source>`, its branch escaped.
@@ -63,7 +60,7 @@ func isTierKey(key string) bool {
// startTiers writes the tier progress of a passed pipeline whose movers' versions are now kept, its first tier the lowest of
// theirs, since now. A pipeline heard again writes nothing.
func (e *pipelineEnds) startTiers(pipeline string, source int64, movers []tierMover) error {
func (e *pipelineEnds) startTiers(pipeline string, movers []tierMover) error {
if len(movers) == 0 {
return nil
}
@@ -71,7 +68,7 @@ func (e *pipelineEnds) startTiers(pipeline string, source int64, movers []tierMo
for _, m := range movers {
first = min(first, m.Tier)
}
value, err := json.Marshal(tierProgress{Tier: first, Since: e.clock().UTC(), Movers: movers, Source: source})
value, err := json.Marshal(tierProgress{Tier: first, Since: e.clock().UTC(), Movers: movers})
if err != nil {
return err
}
@@ -134,7 +131,7 @@ func (p *passer) tiers(n assign.Nodes, placedOn func(module, branch string) []st
continue
}
pipeline := strings.TrimPrefix(key, tierPrefix)
branchEsc, number, _ := strings.Cut(pipeline, ".")
branchEsc, source, _ := strings.Cut(pipeline, ".")
branch, err := keys.UnescapeName(branchEsc)
if err != nil {
continue
@@ -147,7 +144,7 @@ func (p *passer) tiers(n assign.Nodes, placedOn func(module, branch string) []st
notRunning := p.notRunning(n, placedOn, pipeline, branch, t, t.Tier)
late := !now.Before(t.Since.Add(bound))
if late {
p.tierLate(pipeline, branch, number, t, notRunning)
p.tierLate(pipeline, branch, source, t, notRunning)
break
}
if len(notRunning) > 0 {
@@ -168,10 +165,10 @@ func (p *passer) tiers(n assign.Nodes, placedOn func(module, branch string) []st
p.logf("the tier progress %s is not deleted: %v", key, err)
break
}
p.sayPipelineWanted(pipeline, branch, number, t.Source, meshwants.PipelineWantedOutcomeRunning, nil)
p.sayPipelineWanted(pipeline, branch, source, meshwants.PipelineWantedOutcomeRunning, nil)
break
}
value, err := json.Marshal(tierProgress{Tier: later, Since: now, Movers: t.Movers, Source: t.Source})
value, err := json.Marshal(tierProgress{Tier: later, Since: now, Movers: t.Movers})
if err == nil {
err = p.ends.st.swap(key, value, rev)
}
@@ -220,7 +217,7 @@ func (p *passer) notRunning(n assign.Nodes, placedOn func(module, branch string)
// tierLate ends a pipeline's delivery at a tier past tier-bound: the finding tier-late is raised and pipeline-wanted says tier-late,
// once each; the finding clears once the late movers run or a later pipeline moved them. The progress stays, so no later tier is
// written, until no later tier's mover is kept at this pipeline any more.
func (p *passer) tierLate(pipeline, branch, number string, t tierProgress, notRunning []meshwants.NotRunning) {
func (p *passer) tierLate(pipeline, branch, source string, t tierProgress, notRunning []meshwants.NotRunning) {
o := tierLateFinding(pipeline, t.Tier, notRunning)
key := views.FindingKey(o.Scope, assign.KeyID(o.ID), o.Kind)
if len(notRunning) > 0 {
@@ -234,7 +231,7 @@ func (p *passer) tierLate(pipeline, branch, number string, t tierProgress, notRu
p.said[key+".raised"] = true
delete(p.said, key+".cleared")
p.logf("pipeline %s: tier %d is late (tier-bound %s): %s; no later tier is written", pipeline, t.Tier, p.tierBoundOf(), o.Summary)
p.sayPipelineWanted(pipeline, branch, number, t.Source, meshwants.PipelineWantedOutcomeTierLate, notRunning)
p.sayPipelineWanted(pipeline, branch, source, meshwants.PipelineWantedOutcomeTierLate, notRunning)
}
} else if !p.said[key+".cleared"] {
if p.findings.Delete != nil {
@@ -274,16 +271,11 @@ func tierLateFinding(pipeline string, tier int64, notRunning []meshwants.NotRunn
Source: "assigner." + kindTierLate}
}
// sayPipelineWanted says the seat mesh-wants' pipeline-wanted of pipeline, once (its id the pipeline and the outcome): its
// revision the number of the pipeline's key `<branch>.<revision>`, its source the one the pipeline-ended named (design 47
// H-A-214, H-A-216), the key's number for a progress kept before it held one.
func (p *passer) sayPipelineWanted(pipeline, branch, number string, source int64, outcome string, notRunning []meshwants.NotRunning) {
var revision int64
_, _ = fmt.Sscan(number, &revision)
if source <= 0 {
source = revision
}
body := meshwants.PipelineWantedEvent{V: keys.Version, Pipeline: pipeline, Branch: branch, Source: source, Revision: revision, Outcome: outcome, NotRunning: notRunning}
// sayPipelineWanted says the seat mesh-wants' pipeline-wanted of pipeline, once (its id the pipeline and the outcome).
func (p *passer) sayPipelineWanted(pipeline, branch, source, outcome string, notRunning []meshwants.NotRunning) {
var src int64
_, _ = fmt.Sscan(source, &src)
body := meshwants.PipelineWantedEvent{V: keys.Version, Pipeline: pipeline, Branch: branch, Source: src, Outcome: outcome, NotRunning: notRunning}
event := meshwants.EventPipelineWanted + "." + pipeline
if err := p.wantsSay(event, body, "pipeline-wanted."+pipeline+"."+outcome); err != nil {
p.logf("pipeline-wanted of %s is not said: %v", pipeline, err)
@@ -62,11 +62,8 @@ func TestAPipelinesWantsAreWrittenTierByTier(t *testing.T) {
p.w.mu.Unlock()
p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf)
p.ends.trunk, p.ends.now = p.w.trunkName, func() time.Time { return now }
// The pipeline of revision 5 (its key's number) of source 4: a re-pin's revision (H-A-213, H-A-216).
e := ended(trunkBranch, 5, mpr.PipelineEndedOutcomePassed, mover("db", db, "0.2.0", 1, mpr.StepResultOutcomePassed),
mover("web", web, "0.1.2", 2, mpr.StepResultOutcomePassed))
e.Source = 4
if _, err := p.ends.take(e); err != nil {
if _, err := p.ends.take(ended(trunkBranch, 5, mpr.PipelineEndedOutcomePassed, mover("db", db, "0.2.0", 1, mpr.StepResultOutcomePassed),
mover("web", web, "0.1.2", 2, mpr.StepResultOutcomePassed))); err != nil {
t.Fatal(err)
}
wanted := func(node, module string) bool {
@@ -91,7 +88,7 @@ func TestAPipelinesWantsAreWrittenTierByTier(t *testing.T) {
raw, _ = json.Marshal(nodeengine.Running{Node: "g14", Module: "web", Fingerprint: web})
_ = p.w.hearRunning("running.g14.web", raw)
p.wants(context.Background())
if len(said) != 1 || said[0].Outcome != meshwants.PipelineWantedOutcomeRunning || said[0].Revision != 5 || said[0].Source != 4 || len(findings) != 0 {
if len(said) != 1 || said[0].Outcome != meshwants.PipelineWantedOutcomeRunning || len(findings) != 0 {
t.Errorf("every tier runs: said %+v, findings %v", said, findings)
}
if _, _, ok := p.ends.tiersOf(trunkBranch + ".5"); ok {
@@ -111,14 +108,8 @@ func TestAPipelinesWantsAreWrittenTierByTier(t *testing.T) {
if !ok || !strings.Contains(f.Summary, "db on g14") || strings.Contains(f.Summary, "ace") {
t.Errorf("tier-late is not raised naming db on g14 alone: %v", findings)
}
if len(said) != 1 || said[0].Outcome != meshwants.PipelineWantedOutcomeTierLate || len(said[0].NotRunning) != 1 || said[0].NotRunning[0].Module != "db" ||
said[0].Revision != 5 || said[0].Source != 4 {
t.Errorf("pipeline-wanted tier-late is not said once, naming revision 5 of source 4: %+v", said)
}
// A tier progress kept before it held its source says the key's number as both (H-A-216).
p.sayPipelineWanted(trunkBranch+".5", trunkBranch, "5", 0, meshwants.PipelineWantedOutcomeRunning, nil)
if len(said) != 2 || said[1].Revision != 5 || said[1].Source != 5 {
t.Errorf("a progress with no source: said %+v, want revision and source 5", said)
if len(said) != 1 || said[0].Outcome != meshwants.PipelineWantedOutcomeTierLate || len(said[0].NotRunning) != 1 || said[0].NotRunning[0].Module != "db" {
t.Errorf("pipeline-wanted tier-late is not said once: %+v", said)
}
if wanted("g14", "web") || wanted("ace", "web") {
t.Error("a consumer's want is written after its provider's tier was late")
@@ -119,13 +119,6 @@ func TestManifest(t *testing.T) {
}
}
}
// H-A-186 (design 47 G §370): asks, which the gate, the teller and the view import, is a declared Go package of the
// module, so every release publishes it; H-A-188: in the module's version series, 0.3.
if !slices.ContainsFunc(m.Build.Artifacts, func(a catalogue.Artifact) bool {
return a.Name == "asks" && a.Kind == catalogue.ArtifactPackage && a.Language == "go" && slices.Equal(a.Paths, []string{"asks"}) && a.Series == m.Version && m.Version == "0.3"
}) {
t.Errorf("artifacts = %+v, want the Go package asks of series 0.3, the module's", m.Build.Artifacts)
}
// ADR 0635 §5: its views family is its ordinary events, `views.<state>` among its emits, on no stream of its own.
// The seat's four events are said on the seat's own subject (the seat's family), so the module keeps no stream for them
// and lists none among its emits.
@@ -370,15 +370,6 @@
"loads": [
"mesh-delivery-assigner"
]
},
{
"name": "asks",
"kind": "package",
"language": "go",
"paths": [
"asks"
],
"series": "0.3"
}
]
}
@@ -242,11 +242,6 @@ type BuilderBuild struct {
// Imports are, on a done key, the mesh packages the built programs import, as the build module answered them
// (BuildAnswer.Imports; novox/hq ADR 0593 §2): the builder carries them onto the build's end (BuildLogEntry.Imports).
Imports []string `json:"imports,omitempty"`
// Packages are, on a done key only, the mesh packages' versions its Go build was built with, by module path, the ones
// the registry gave a package the entry named with none included (H-A-207): the build step's end reports them, so the
// runner asks the steps that waited for the pins with what the build was built against. A reader that does not know
// the field reads the build as reporting none.
Packages map[string]string `json:"packages,omitempty"`
// Needs are, on a done key only, the reader capabilities (`form:<name>`) the build's reference forms need of a
// node-engine, beside the ones its manifest needs (novox/hq ADR 0618 §1). A reader that does not know the field
// reads the build as needing none.
@@ -45,7 +45,7 @@ func (m *entryManifests) keep(e meshcatalog.CatalogEntry) error {
if m.byFp == nil {
m.byFp = map[string]catalogue.Manifest{}
}
m.byFp[meshcatalogclient.BareManifestDigest(e.ManifestDigest)] = mf
m.byFp[meshcatalog.BareManifestDigest(e.ManifestDigest)] = mf
return nil
}
@@ -101,7 +101,9 @@ func (c *catalogHalf) Changed(ctx context.Context) func(branch, module string, e
}
origin := build.Origin{Branch: branch, First: first}
if cat, ok := c.branches.Catalog(branch); ok {
origin = pictureOrigin(origin, cat.View)
if half, ok := cat.View(); ok {
origin.Source = half.Revision.Source
}
}
var e meshcatalog.CatalogEntry
if err := json.Unmarshal(entry, &e); err != nil || e.Module != module {
@@ -130,16 +132,6 @@ func (c *catalogHalf) Changed(ctx context.Context) func(branch, module string, e
}
}
// pictureOrigin is the origin of an entry the picture takes in: the source revision and the catalog revision number of the
// half the view holds, one revision for both (design 47 G §371, H-A-218, H-A-220), so a build the picture starts before any
// step ask reserves a version that names its revision; none while the view holds no half.
func pictureOrigin(o build.Origin, view func() (meshcatalogclient.Half, bool)) build.Origin {
if half, ok := view(); ok {
o.Source, o.Revision = half.Revision.Source, half.Revision.Revision
}
return o
}
// picture is build.Builder.Picture: the source revision the view of branch holds (design 47 H H-A-155), false while the branch
// is not followed or its view holds none.
func (c *catalogHalf) picture(branch string) (int64, bool) {
@@ -4,9 +4,7 @@ import (
"context"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"mesh-delivery-builder/internal/build"
)
// TestTheManifestOfAnEntryIsReadByItsDigest: the builder reads a manifest from the entry that holds it, by the entry's manifest
@@ -27,18 +25,3 @@ func TestTheManifestOfAnEntryIsReadByItsDigest(t *testing.T) {
t.Errorf("an entry with no manifest kept one: %v, %v", err, m.byFp)
}
}
// H-A-220: an entry the picture takes in has the source revision and the catalog revision number of the half its view holds,
// one revision for both; a view holding no half names neither.
func TestAnEntryThePictureTakesInNamesItsHalfsRevision(t *testing.T) {
o := build.Origin{Branch: "main", First: true}
got := pictureOrigin(o, func() (meshcatalogclient.Half, bool) {
return meshcatalogclient.Half{Revision: meshcatalog.CatalogRevision{Revision: 47, Source: 45}}, true
})
if got != (build.Origin{Branch: "main", First: true, Source: 45, Revision: 47}) {
t.Errorf("origin = %+v, want source 45 at revision 47", got)
}
if got := pictureOrigin(o, func() (meshcatalogclient.Half, bool) { return meshcatalogclient.Half{}, false }); got != o {
t.Errorf("origin with no half = %+v, want %+v", got, o)
}
}
@@ -1,47 +0,0 @@
package main
import (
"context"
"encoding/json"
"fmt"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/buscalls"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// keptRevisionVia reads one kept revision of a branch by its number through the catalog seat's verb revision, page by page
// (design 47 H H-A-205, H-A-147): the source revision it was taken in from and the fingerprint of each module it names.
func keptRevisionVia(call toolCaller) func(context.Context, string, int64) (int64, map[string]string, error) {
return func(_ context.Context, branch string, revision int64) (int64, map[string]string, error) {
fps := map[string]string{}
var source int64
after := ""
for {
raw, err := call(meshcatalogclient.RevisionVerb, meshcatalog.RevisionInput{Branch: keys.EscapeName(branch), Revision: revision, After: after})
if err != nil {
return 0, nil, err
}
answer, said, ok := buscalls.Unwrap(raw)
if !ok {
return 0, nil, fmt.Errorf("%s answered: %s", meshcatalogclient.RevisionVerb, said)
}
var a meshcatalog.RevisionAnswer
if err := json.Unmarshal(answer, &a); err != nil {
return 0, nil, fmt.Errorf("%s answered no revision: %w", meshcatalogclient.RevisionVerb, err)
}
if a.NoneYet || a.Revision == nil || a.Revision.Revision != revision {
return 0, nil, fmt.Errorf("%s answered no revision %d of %s", meshcatalogclient.RevisionVerb, revision, branch)
}
source = a.Revision.Source
for _, e := range a.Entries {
fps[e.Module] = e.Fp
}
if a.Next == "" {
return source, fps, nil
}
after = a.Next
}
}
}
@@ -228,8 +228,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return err
}
b.PackageWait = time.Duration(packageWait) * time.Minute
// Before it waits for an unpinned package, the build reads what the Go package registry seat holds of it (H-A-194).
b.PublishedVersions = publishedVersionsVia(calls.Call)
waitsBucket := process.AwaitBucket(ctx, js, conn, builds.StateBucket(cred.Module, build.WaitsState), "the toolchain waits are kept in memory only, and builder_waits shows none", logger.Printf)
b.Waits = newWaitsKept(waitsBucket, func(ctx context.Context, event, id string, body any) error {
return emitEvent(context.WithoutCancel(ctx), js, cred, event, id, body)
@@ -244,6 +242,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return emitSeatEvent(context.WithoutCancel(ctx), js, cred, meshbuilds.SubjectPackagePublished, "package-published-"+p.Module+"-"+p.Version+"-"+p.Artifact, p)
}
b.Reserved = reserved
// A release heard, or the newest the builder's record keeps, pins a package an entry names with none (H-A-165).
reserved.heardReleased = b.VersionReleased
b.NewestReleased = reserved.r.NewestReleased
// The last versions-filled the stream keeps is read at the start (design 47 G §369, §372): a restart does not lose it.
reserved.readFilled(ctx, lastKeptOf(js, cred, logger.Printf), sleeps)
// A state bound later numbers the builds looked at without it.
@@ -469,9 +470,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// a holder of the seat that is down holds no other part of it).
half.b = b
b.Picture = half.picture
// An ask of a revision below the newest the picture holds is read by number from the catalog seat (H-A-205).
b.KeptRevision = keptRevisionVia(calls.Call)
b.FirstBranch = half.first
branches.Held = func(string, meshcatalog.CatalogRevision) { b.PictureHeld() } // a revision held wakes a step ask's wait (H-A-162)
half.ready = func() {
if err := b.Sweep(ctx, half.pictured()...); err != nil {
@@ -44,12 +44,15 @@ type reservations struct {
// built asks a done build's publish (design 47 G A-457a), released ends its item; nil are none.
built func(ctx context.Context, module, version string) error
released func(ctx context.Context, module, version string)
filled atomic.Bool
bounds func() jetstream.KeyValue
found *findings
bound time.Duration
wake func(context.Context)
logf func(string, ...any)
// heardReleased is told each version-released taken in (design 47 H H-A-165): a step ask waiting for a package of the
// module wakes. Nil tells nobody.
heardReleased func(module, version string)
filled atomic.Bool
bounds func() jetstream.KeyValue
found *findings
bound time.Duration
wake func(context.Context)
logf func(string, ...any)
// call asks a seat's verb: the git seat's tag (design 47 G §370).
call toolCaller
// say says package-published on the seat mesh-builds; nil says none.
@@ -252,6 +255,9 @@ func (rv *reservations) heard(ctx context.Context, m jetstream.Msg) bool {
if rv.released != nil {
rv.released(ctx, ev.Module, ev.Version)
}
if rv.heardReleased != nil {
rv.heardReleased(ev.Module, ev.Version)
}
rv.found.clear(ctx, versions.VersionUnreleased, meshconditionsclient.ScopeModule, ev.Module)
default:
return false
@@ -134,10 +134,15 @@ func TestTheCatalogsWordsMoveTheReservations(t *testing.T) {
if err != nil || !strings.Contains(got.Summary, "m:b unpublished") || strings.Contains(got.Summary, "m:a unpublished") || !strings.Contains(got.Summary, "1.4.3") {
t.Fatalf("finding = %+v (%v), want version-unreleased naming m:b of 1.4.3", got, err)
}
var heardReleases []string // H-A-165: a release heard wakes a step ask waiting for a package of the module
rv.heardReleased = func(m, v string) { heardReleases = append(heardReleases, m+" "+v) }
released := &wakeMsg{subject: meshcatalog.SubjectVersionReleased, data: []byte(`{"v":1,"module":"m","version":"1.4.3","source":1,"artifacts":[]}`)}
if !rv.heard(ctx, released) || !released.acked {
t.Fatalf("version-released was not taken in (acked %v)", released.acked)
}
if strings.Join(heardReleases, ", ") != "m 1.4.3" {
t.Errorf("the release heard was told %v, want m 1.4.3 (H-A-165)", heardReleases)
}
if strings.Join(items, ", ") != "asked m 1.4.3, ended m 1.4.3" {
t.Errorf("publish items %v, want the build's end to ask the publish and the release to end it", items)
}
@@ -105,10 +105,6 @@ func (h *seatHearer) heard(ctx context.Context, m jetstream.Msg) bool {
if h.res != nil && se.event == versions.HeardPublished {
h.res.published(ctx, o)
}
if se.event == versions.HeardPublished && o.Kind == versions.KindPackage && h.b != nil {
// A package registry seat's published pins a package an entry names with none, at the version it carries (H-A-192).
h.b.PackagePublished(o.Name, o.Version)
}
h.ack(m)
if len(wake) > 0 {
h.logf("the floor of %s is asked again: %s said %s", strings.Join(wake, ", "), se.seat, se.event)
@@ -13,7 +13,6 @@ import (
"git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
gitseat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/gopackageregistry"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-builder/internal/build"
@@ -97,28 +96,6 @@ func (s awaitedVersions) KeysOf(ctx context.Context, filter string) ([]string, e
return out, nil
}
// publishedVersionsVia asks the Go package registry seat's verb versions for one package path (design 47 H H-A-194): its
// versions, newest first.
func publishedVersionsVia(call toolCaller) func(context.Context, string) ([]string, error) {
return func(_ context.Context, path string) ([]string, error) {
raw, err := call("seat:"+gopackageregistry.Seat+"."+gopackageregistry.VerbVersions, gopackageregistry.VersionsInput{Name: path})
if err != nil {
return nil, err
}
var a gopackageregistry.VersionsAnswer
if err := json.Unmarshal(raw, &a); err != nil {
return nil, fmt.Errorf("%s's versions answered what is no list of versions: %w", gopackageregistry.Seat, err)
}
var out []string
for _, v := range a.Versions {
if v.Version != "" {
out = append(out, v.Version)
}
}
return out, nil
}
}
// floorVia is the floor's three sources as calls of the builder's own account (design 51 *The floor*): the seat's verb
// versions, the seat git's tags and the requirements the catalog half's entries name (A-119). A call that fails is that source
// silent.
@@ -13,7 +13,6 @@ import (
func TestAnEntryOfTheCatalogHalfIsTheFormTheBuilderBuildsBy(t *testing.T) {
e := meshcatalog.CatalogEntry{Module: "netcheck", Repository: "novox/mesh-catalog", Path: "modules/netcheck", Commit: "abc", Fp: "src2:f",
ManifestDigest: "sha256:m", Bases: []string{"base"}, Packages: map[string]string{"p": "1"}, Needs: []string{"n"},
Artifacts: []meshcatalog.Artifact{{Ref: "netcheck:lib", Kind: "package", Language: "go"}},
Inputs: &meshcatalog.EntryInputs{Toolchains: map[string]meshcatalog.ToolchainUsed{"go": {Version: "go1", Recipe: "1"}},
Paths: map[string][]string{"lib": {"a/b", "c"}}}}
got := ModuleOf(e)
@@ -31,11 +30,4 @@ func TestAnEntryOfTheCatalogHalfIsTheFormTheBuilderBuildsBy(t *testing.T) {
if bare := ModuleOf(meshcatalog.CatalogEntry{Module: "m"}); bare.Inputs2 != nil || bare.Fp2 != "" {
t.Errorf("ModuleOf of an entry with none = %+v, want nothing set", bare)
}
// A module with an artifact that names no paths (a Go program beside its packages) is built from its directory whole
// (ADR 0499 clauses 1 and 4): its fetch names no paths, so the program's go.mod and main package stay in the tree.
mixed := e
mixed.Artifacts = append([]meshcatalog.Artifact{{Ref: "netcheck:program", Kind: "bundle", Language: "go"}}, e.Artifacts...)
if got := ModuleOf(mixed); got.Inputs2 == nil || len(got.Inputs2.Paths) != 0 {
t.Errorf("ModuleOf of a module with a program naming no paths = %+v, want bases and no named paths", got.Inputs2)
}
}
@@ -194,7 +194,7 @@ func (b *Builder) reserve(ctx context.Context, module string, m meshcatalog.Modu
for _, t := range plan.Tasks {
refs = append(refs, module+":"+t.Artifact)
}
src := versions.Source{Repository: m.Repository, Commit: m.Commit, Fingerprint: m.Fp2, Branch: o.Branch, Revision: o.Source, CatalogRevision: o.Revision, Rebuild: rebuild}
src := versions.Source{Repository: m.Repository, Commit: m.Commit, Fingerprint: m.Fp2, Branch: o.Branch, Revision: o.Source, Rebuild: rebuild}
for _, t := range plan.Tasks {
src.GoPackage = src.GoPackage || (t.Kind == catalogue.ArtifactPackage && t.Language == "go")
}
@@ -5,12 +5,11 @@ import (
"fmt"
"maps"
"slices"
"strconv"
"strings"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
)
// **A build waits for its revision; a build with no version is never made** (design 47 H H-A-155, H-A-156; H-A-141): a step
@@ -20,11 +19,6 @@ import (
// revision its view holds (PictureHeld, H-A-162), never a
// timer or a poll, up to RevisionWait (the setting revision-wait-minutes); past it the ask fails, naming the revision and the
// branch, and nothing is built. A step ask with no source (a run on request) does not wait.
//
// **The picture holds every kept revision, not only the newest** (design 47 H H-A-205, H-A-147): an ask of a source revision
// below the newest the picture holds, whose entries the picture no longer names, asks the catalog seat's verb revision for
// its catalog revision by number, once, and is built when that kept revision is of its source and names every entry it
// asks; it never waits, since the picture only moves on. Only a revision above the newest is waited for.
// DefaultRevisionWait is the bound of a step ask's wait for its revision, its setting revision-wait-minutes' default (H-A-156).
const DefaultRevisionWait = 2 * time.Minute
@@ -73,52 +67,12 @@ func (b *Builder) holdsRevision(branch string, source int64, entries []meshbuild
return true
}
// keptRevisionHolds answers no outcome when the catalog keeps the ask's revision (its catalog revision by number, of its
// source revision) and that revision names every entry of the ask; else failed or could not run, with its words.
func (b *Builder) keptRevisionHolds(ctx context.Context, run meshbuilds.StepRun, held int64) (outcome, said string) {
below := fmt.Sprintf("revision %d of %s is below the %d the builder's picture holds", run.Source, run.Branch, held)
if b.KeptRevision == nil || run.Revision <= 0 {
return meshbuilds.StepEndedOutcomeFailed, below + ", and the ask names no catalog revision to read it by"
}
source, fps, err := b.KeptRevision(ctx, run.Branch, run.Revision)
if err != nil {
b.logf("%s: %s, and the catalog's revision %d cannot be read: %v", run.Pipeline, below, run.Revision, err)
return meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("%s, and the catalog's revision %d cannot be read: %v", below, run.Revision, err)
}
if source != run.Source {
return meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s, and the catalog's revision %d is of source revision %d", below, run.Revision, source)
}
for _, e := range run.Entries {
if fps[e.Module] != e.Fingerprint {
return meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s, and the catalog's revision %d names %s at %q, not %s", below, run.Revision, e.Module, fps[e.Module], e.Fingerprint)
}
}
// Each asked fingerprint is recorded as the branch's, so its build reserves its version (H-A-155) as one the picture
// took in does; what the picture heard stays as it is, so the newer fingerprint's build is not superseded.
b.mu.Lock()
for _, e := range run.Entries {
key, err := builds.BuilderBuildKey(e.Module, e.Fingerprint)
if err != nil || b.origin[key].Branch != "" {
continue
}
first := b.FirstBranch == nil || b.FirstBranch(run.Branch, e.Module)
b.trunkFP[key] = true
b.origin[key] = Origin{Branch: run.Branch, Source: run.Source, Revision: run.Revision, First: first}
}
b.mu.Unlock()
b.logf("%s: %s; the catalog keeps revision %d with its entries: the build goes on", run.Pipeline, below, run.Revision)
return "", ""
}
// awaitRevision returns no outcome once the builder's picture of the ask's branch holds its source revision; past the bound the
// step's outcome failed, at the builder's stop could not run, each with its words.
func (b *Builder) awaitRevision(ctx context.Context, run meshbuilds.StepRun) (outcome, said string) {
if b.Picture == nil || run.Source <= 0 || run.Branch == "" {
return "", ""
}
if held, ok := b.Picture(run.Branch); ok && run.Source < held && !b.holdsRevision(run.Branch, run.Source, run.Entries) {
return b.keptRevisionHolds(ctx, run, held)
}
bound := b.revisionWait()
notHeld := fmt.Sprintf("revision %d of %s not in the builder's picture", run.Source, run.Branch)
wait, stop := context.WithTimeout(ctx, bound) // a context's bound, no timer (ADR 0437 §8)
@@ -150,16 +104,13 @@ func (b *Builder) awaitRevision(ctx context.Context, run meshbuilds.StepRun) (ou
}
}
// **A package with no pin waits for its publish** (design 47 H H-A-192, H-A-165, H-A-183, H-A-184, H-A-185): an entry
// keeps a mesh package the registry has not published within the series its module accepts with an empty version. Its
// build, a step ask's or a lane's, first reads the versions the registry seat holds of the path (its verb versions,
// H-A-194), then waits for the registry seat's published of that package path in that series (the
// series the module's manifest names in `packages`, else the default; catalogue.Manifest.AcceptedSeries), up to
// PackageWait (the setting package-wait-minutes), and pins the version that event carries; the builder never derives a
// pin (no release number of the module that holds the package). Past the bound the build fails, naming the package, and
// nothing is built.
// **A package with no pin waits for its release** (design 47 H H-A-165, H-A-161): an entry keeps a mesh package none of
// whose versions is released with an empty version. Its step ask waits, as for its revision, for the release of the module
// that holds the package (the catalog's version-released, VersionReleased), up to PackageWait (the setting
// package-wait-minutes), then builds with that release as the package's pin (v<version>); past the bound the step fails,
// naming the package, and nothing is built.
// DefaultPackageWait is the bound of a build's wait for a package's publish, its setting package-wait-minutes' default.
// DefaultPackageWait is the bound of a step ask's wait for a package's release, its setting package-wait-minutes' default.
const DefaultPackageWait = 10 * time.Minute
func (b *Builder) packageWait() time.Duration {
@@ -169,152 +120,91 @@ func (b *Builder) packageWait() time.Duration {
return DefaultPackageWait
}
// PackagePublished takes the Go package registry seat's published of one package path in (its name and the version the
// event carries, H-A-192): the newest published in the version's series is kept, and a build waiting for the package
// wakes. A version that is no MAJOR.MINOR.PATCH (a group's tagged version) pins nothing.
func (b *Builder) PackagePublished(path, version string) {
v := strings.TrimPrefix(version, "v")
series, ok := seriesOfVersion(v)
if path == "" || !ok {
return
}
// VersionReleased takes the catalog's version-released of module in: a step ask waiting for a package of it wakes.
func (b *Builder) VersionReleased(module, version string) {
b.init()
b.mu.Lock()
in := b.publishedPins[path]
if in == nil {
in = map[string]string{}
b.publishedPins[path] = in
}
if old := in[series]; old == "" || compareVersions(v, old) > 0 {
in[series] = v
}
b.releases[module] = version
b.mu.Unlock()
b.pictureMoved()
}
// publishedPin is the pin the registry seat's published gave the package path in the series, as Go names it; "" for none.
func (b *Builder) publishedPin(path, series string) string {
// releaseOf is the pin a package is built against: the release of the module holding it, as Go names it; "" for none yet.
func (b *Builder) releaseOf(ctx context.Context, pkg string) string {
if b.Members == nil {
return ""
}
module, _, found, err := b.Members(ctx, pkg)
if err != nil || !found {
return ""
}
b.mu.Lock()
v := b.publishedPins[path][series]
v := b.releases[module]
b.mu.Unlock()
if v == "" && b.NewestReleased != nil {
if kept, ok, err := b.NewestReleased(ctx, module); err == nil && ok {
v = kept
}
}
if v == "" {
return ""
}
return "v" + v
}
// awaitPublished fills each package of packages with no pin from the published of its path within the series the manifest
// with this digest accepts, waiting for one (H-A-192, H-A-165). It answers the packages with every pin, or an outcome and
// its words: failed past the bound naming the package, could not run when the manifest cannot be read or the builder stops.
func (b *Builder) awaitPublished(ctx context.Context, module, fp, digest string, packages map[string]string) (map[string]string, string, string) {
// awaitPins fills each package entry names with no pin from its release, waiting for one; no outcome once every package has
// a pin, failed past the bound, could not run at the builder's stop.
func (b *Builder) awaitPins(ctx context.Context, entry *meshcatalog.CatalogEntry) (outcome, said string) {
var unpinned []string
for p, v := range packages {
for p, v := range entry.Packages {
if v == "" {
unpinned = append(unpinned, p)
}
}
if len(unpinned) == 0 {
return packages, "", ""
return "", ""
}
slices.Sort(unpinned)
if b.Manifests == nil {
return nil, meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("the series %s accepts of %s cannot be read: no manifests", module, unpinned[0])
}
man, err := b.Manifests.Manifest(ctx, digest)
if err != nil {
return nil, meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("the series %s accepts of %s cannot be read: its manifest %s: %v", module, unpinned[0], digest, err)
}
// What the registry seat holds is read once before the wait (H-A-194): the publishes heard are a cache of it.
for _, p := range unpinned {
if b.PublishedVersions == nil || b.publishedPin(p, man.AcceptedSeries(p)) != "" {
continue
}
vs, err := b.PublishedVersions(ctx, p)
if err != nil {
b.logf("%s at %s: the versions the registry seat holds of %s cannot be read, so the build waits for its publish: %v", module, fp, p, err)
continue
}
for _, v := range vs {
b.PackagePublished(p, v)
}
}
bound := b.packageWait()
wait, stop := context.WithTimeout(ctx, bound) // a context's bound, no timer (ADR 0437 §8)
defer stop()
pins := map[string]string{}
told := false
for {
b.mu.Lock()
moved := b.moved
b.mu.Unlock()
pins := map[string]string{}
missing, missingSeries := "", ""
missing := ""
for _, p := range unpinned {
series := man.AcceptedSeries(p)
pins[p] = b.publishedPin(p, series)
if pins[p] == "" {
pins[p] = b.releaseOf(ctx, p)
}
if pins[p] == "" && missing == "" {
missing, missingSeries = p, series
missing = p
}
}
if missing == "" {
out := maps.Clone(packages)
maps.Copy(out, pins)
packages := maps.Clone(entry.Packages)
maps.Copy(packages, pins)
entry.Packages = packages
if told {
b.logf("%s at %s: every package it names is published: built with %v", module, fp, pins)
b.logf("%s at %s: every package it names has a released version: built with %v", entry.Module, entry.Fp, pins)
}
return out, "", ""
return "", ""
}
notReleased := fmt.Sprintf("package %s has no released version in the series %s", missing, missingSeries)
notReleased := fmt.Sprintf("package %s has no released version", missing)
if !told {
b.logf("%s at %s: %s yet: the build waits for its publish, at most %s", module, fp, notReleased, bound)
b.logf("%s at %s: %s yet: the ask waits for its release, at most %s", entry.Module, entry.Fp, notReleased, bound)
told = true
}
select {
case <-moved:
case <-wait.Done():
if ctx.Err() != nil {
return nil, meshbuilds.StepEndedOutcomeCouldNotRun, notReleased + ": the builder stopped before it was"
return meshbuilds.StepEndedOutcomeCouldNotRun, notReleased + ": the builder stopped before it was"
}
b.logf("%s at %s: %s within %s: the build fails, nothing is built", module, fp, notReleased, bound)
return nil, meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s within %s", notReleased, bound)
b.logf("%s at %s: %s within %s: the step fails, nothing is built", entry.Module, entry.Fp, notReleased, bound)
return meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s within %s", notReleased, bound)
}
}
}
// seriesOfVersion is a MAJOR.MINOR.PATCH version's series, MAJOR.MINOR; false for any other form.
func seriesOfVersion(v string) (string, bool) {
if _, ok := versionParts(v); !ok {
return "", false
}
major, rest, _ := strings.Cut(v, ".")
minor, _, _ := strings.Cut(rest, ".")
return major + "." + minor, true
}
// versionParts are a MAJOR.MINOR.PATCH version's numbers; false for any other form.
func versionParts(v string) ([3]int, bool) {
var out [3]int
parts := strings.Split(v, ".")
if len(parts) != 3 {
return out, false
}
for i, p := range parts {
n, err := strconv.Atoi(p)
if err != nil || n < 0 {
return out, false
}
out[i] = n
}
return out, true
}
// compareVersions orders two MAJOR.MINOR.PATCH versions numerically.
func compareVersions(a, b string) int {
x, _ := versionParts(a)
y, _ := versionParts(b)
for i := range x {
if x[i] != y[i] {
return x[i] - y[i]
}
}
return 0
}
@@ -110,24 +110,14 @@ type Builder struct {
// Picture answers the source revision the builder's picture of a watched branch holds, false while it holds none (design 47
// H H-A-155, H-A-141): a step ask naming a newer one waits for it. Nil waits for nothing.
Picture func(branch string) (source int64, held bool)
// KeptRevision asks the catalog seat's verb revision for one kept revision of a branch by its number (design 47 H
// H-A-205, H-A-147): the source revision it was taken in from and the fingerprint of each module it names. A step ask of
// a source revision below the newest the picture holds is built when that kept revision names its entries. Nil reads none.
KeptRevision func(ctx context.Context, branch string, revision int64) (source int64, fingerprints map[string]string, err error)
// FirstBranch says whether branch is the first watched branch of the module's registration, the one whose builds reserve
// a version (design 47 G §371): an ask admitted by its kept revision is recorded as that branch's (H-A-205, H-A-155). Nil
// takes every branch as the first, as an origin never heard is.
FirstBranch func(branch, module string) bool
// RevisionWait bounds a step ask's wait for its revision, its setting revision-wait-minutes (H-A-156); DefaultRevisionWait
// when 0.
RevisionWait time.Duration
// PackageWait bounds a build's wait for the publish of a package its entry names with no pin, its setting
// package-wait-minutes (H-A-165); DefaultPackageWait when 0. Only a registry seat's published heard (PackagePublished)
// gives a pin (H-A-192).
PackageWait time.Duration
// PublishedVersions asks the Go package registry seat's verb versions for one package path, its versions newest first
// (H-A-194): read once before a build waits for that package. Nil reads none.
PublishedVersions func(ctx context.Context, path string) ([]string, error)
// PackageWait bounds a step ask's wait for the release of a package its entry names with no pin, its setting
// package-wait-minutes (H-A-165); DefaultPackageWait when 0. NewestReleased reads the newest version the builder's
// record of releases keeps for a module; nil reads none, and only a release heard (VersionReleased) gives a pin.
PackageWait time.Duration
NewestReleased func(ctx context.Context, module string) (version string, found bool, err error)
// Forms answers what each node's engine states it reads (ADR 0618 §2, ADR 0624 §1): present, the nodes whose engine
// answered when last asked, and counted, every node the assignments names, present or away, a node with no statement
// stating nothing (the baseline's first forms). A done key names each artifact in the forms these keep, the oldest in
@@ -172,9 +162,8 @@ type Builder struct {
builds map[string]context.CancelCauseFunc
// moved is closed and made again at each entry the picture takes in or drops: the step asks waiting for a revision wake.
moved chan struct{}
// publishedPins are the newest version a registry seat said published, by package path and series (PackagePublished):
// a package with no pin waits for one (H-A-192).
publishedPins map[string]map[string]string
// releases are the versions heard released, by module (VersionReleased): a package with no pin waits for one.
releases map[string]string
}
type planned struct {
@@ -205,7 +194,7 @@ func (b *Builder) init() {
b.hands, b.events = map[string]*handWatch{}, map[string]bool{}
b.builds = map[string]context.CancelCauseFunc{}
b.moved = make(chan struct{})
b.publishedPins = map[string]map[string]string{}
b.releases = map[string]string{}
})
}
@@ -237,9 +226,7 @@ func (b *Builder) Module(ctx context.Context, key string, value []byte) {
type Origin struct {
Branch string
Source int64
// Revision is the catalog revision number a step ask of the build named (design 47 H H-A-218), 0 until one did.
Revision int64
First bool
First bool
}
// Entry takes the catalog graph entry of one module on one watched branch in (design 47 G §348, §614) and builds its
@@ -298,19 +285,16 @@ func (b *Builder) take(ctx context.Context, o Origin, key string, m meshcatalog.
// ModuleOf is a catalog entry in the form the builder builds by: its fingerprint, manifest digest, source (repository,
// directory, commit; no source stamp, design 47 H H-A-48), the toolchains its fingerprint was computed with, the bases it is
// built against, its packages and needs, and the named paths of its artifacts when every artifact names them.
// built against, its packages and needs, and the named paths of its artifacts.
func ModuleOf(e meshcatalog.CatalogEntry) meshcatalog.ModulesByModuleValue {
m := meshcatalog.ModulesByModuleValue{Repository: e.Repository, Path: e.Path, Commit: e.Commit, Fp2: e.Fp, None2: e.None,
Manifest: meshcatalogclient.BareManifestDigest(e.ManifestDigest), Packages: e.Packages, Needs: e.Needs}
Manifest: meshcatalog.BareManifestDigest(e.ManifestDigest), Packages: e.Packages, Needs: e.Needs}
if len(e.Bases) > 0 {
m.Inputs2 = &meshcatalog.ModulesByModuleValueInputs2{Bases: e.Bases}
}
if in := e.Inputs; in != nil {
m.Toolchains, m.Problems = in.Toolchains, in.Problems
// Only a module whose every artifact names its paths is built from those alone (ADR 0499 clauses 1, 2 and 4): an
// artifact without paths, such as a Go program, keeps its module's directory as its build source, so a module
// with one names no paths and its fetch keeps the directory whole.
if len(in.Paths) > 0 && everyArtifactNamesPaths(e) {
if len(in.Paths) > 0 {
if m.Inputs2 == nil {
m.Inputs2 = &meshcatalog.ModulesByModuleValueInputs2{}
}
@@ -325,20 +309,6 @@ func ModuleOf(e meshcatalog.CatalogEntry) meshcatalog.ModulesByModuleValue {
return m
}
// everyArtifactNamesPaths is whether the entry names artifacts and the named paths of each, by its reference or its name.
func everyArtifactNamesPaths(e meshcatalog.CatalogEntry) bool {
if len(e.Artifacts) == 0 || e.Inputs == nil {
return false
}
for _, a := range e.Artifacts {
_, name, _ := strings.Cut(a.Ref, ":")
if len(e.Inputs.Paths[a.Ref]) == 0 && len(e.Inputs.Paths[name]) == 0 {
return false
}
}
return true
}
// originOf is the catalog revision the entry of a build key came from; a key no entry named is of the first branch, as a build
// of the trunk's value was before branches were watched.
func (b *Builder) originOf(key string) Origin {
@@ -843,7 +813,6 @@ type lane struct {
tcVersion string // and the version it ran
kept string // the digest of the artifact the build node kept instead of storing it (BuildAnswer.Kept), else empty
outOfStep []string // the generated files its build module's compare found out of step, passed or failed
pins map[string]string // the mesh packages' versions its Go ask was built with (H-A-207)
err error
}
@@ -993,7 +962,7 @@ func (b *Builder) build(ctx context.Context, module string, m meshcatalog.Module
used = nil
}
done := builds.BuilderBuild{State: builds.BuildDone, By: b.Claimer.By, Manifest: m.Manifest, Step: step, Artifacts: artifacts, Toolchains: used,
Imports: importsOf(lanes), Needs: forms.Current.Needs(artifacts), Packages: pinsOf(lanes)} // the forms it is written in (ADR 0618 §1)
Imports: importsOf(lanes), Needs: forms.Current.Needs(artifacts)} // the forms it is written in (ADR 0618 §1)
for _, l := range lanes {
if l.kept != "" {
done.KeptOn = l.node // design 47 G §671: the build node that keeps the build
@@ -1143,19 +1112,12 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m mes
ask := sdkstep.StepAsk{ID: sdkstep.NewID(), Kind: kind, Language: language, Asker: b.OwnModule, At: time.Now(), Version: l.version, Build: job}
if language == sdkstep.AskGo && job.Fetch != nil && len(m.Packages) > 0 {
// The versions of the mesh packages the fingerprint was computed with (novox/hq ADR 0573 §2): the build module writes
// them into the workspace, whatever the repository's go.mod names. A package the entry names with no pin waits for
// its publish and is built with the version that publish carries; past the bound the lane fails naming it, and no
// ask is put (H-A-192, H-A-165).
pinned, outcome, said := b.awaitPublished(ctx, module, m.Fp2, m.Manifest, m.Packages)
if outcome != "" {
l.err = errors.New(said)
return
}
ask.Packages = maps.Clone(pinned)
// them into the workspace, whatever the repository's go.mod names.
ask.Packages = maps.Clone(m.Packages)
maps.DeleteFunc(ask.Packages, func(_, v string) bool { return v == "" }) // a package with no released version is no pin (H-A-161)
for _, fb := range req.Builds {
delete(ask.Packages, fb.Package) // a package `builds` names is named in neither packages nor package (§465)
}
l.pins = maps.Clone(ask.Packages) // kept with the done key, so its step's end reports them (H-A-207)
}
if id := asked.resume(language); id != "" {
ask.ID = id // put before this builder started again: the stream holds it, or a node already ran it
@@ -1338,20 +1300,6 @@ func (b *Builder) takeOutOfStep(key string) []string {
// importsOf are the mesh packages the lanes' programs import, as their build modules answered them (novox/hq ADR 0593
// §2): sorted and each once; nil when none answered any. A lane kept from a prior build answered none, so a build that
// kept one reports only what it built.
// pinsOf is the mesh packages' versions the lanes were built with (H-A-207), nil when none was pinned.
func pinsOf(lanes []*lane) map[string]string {
var out map[string]string
for _, l := range lanes {
for p, v := range l.pins {
if out == nil {
out = map[string]string{}
}
out[p] = v
}
}
return out
}
func importsOf(lanes []*lane) []string {
seen := map[string]bool{}
for _, l := range lanes {
@@ -333,10 +333,8 @@ func TestATrunkFingerprintIsOneAskOnTheDeliveryStream(t *testing.T) {
r := newRig(t, goBundle)
r.asks.Answer = nil // no node has pulled it yet
// Issue 1585 (ADR 0573 §2): the ask carries the module's pins; a package the entry keeps with no version (none
// published, design 47 H-A-161) is pinned at the version the registry seat's published carries (H-A-192), never
// asked without it.
r.b.PackagePublished("git.novox.be/novox/mesh-contracts/go/seats/x", "v0.1.7")
sdk := buildstep.PackageVersions{"git.novox.be/novox/mesh-sdk/go": "v0.1.20", "git.novox.be/novox/mesh-contracts/go/seats/x": "v0.1.7"}
// released, design 47 H-A-161) is no pin.
sdk := buildstep.PackageVersions{"git.novox.be/novox/mesh-sdk/go": "v0.1.20"}
data, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20",
Fp2: fp1, Manifest: mdigest('a'), Path: "modules/m", Toolchains: CatalogToolchains(agreedToolchains),
Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.20", "git.novox.be/novox/mesh-contracts/go/seats/x": ""}})
@@ -6,7 +6,6 @@ import (
"strings"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
)
@@ -39,21 +38,9 @@ func (b *Builder) BuildStep(ctx context.Context, run meshbuilds.StepRun) meshbui
if outcome, said := b.awaitRevision(ctx, run); outcome != "" {
return finish(outcome, said)
}
b.askedRevision(run)
var failed []string
for _, e := range run.Entries {
var pins map[string]string
outcome, said := b.buildEntry(ctx, e, &pins)
// The end reports the pins each entry was built with, so the runner asks its held test steps with them (H-A-207).
for path, version := range pins {
if version == "" {
continue
}
if end.Packages == nil {
end.Packages = map[string]string{}
}
end.Packages[path] = version
}
outcome, said := b.buildEntry(ctx, e)
switch outcome {
case meshbuilds.StepEndedOutcomeCouldNotRun:
return finish(outcome, said)
@@ -67,30 +54,8 @@ func (b *Builder) BuildStep(ctx context.Context, run meshbuilds.StepRun) meshbui
return finish(meshbuilds.StepEndedOutcomePassed, "")
}
// askedRevision records on the origin of each entry of run the catalog revision the ask names (design 47 H H-A-218), where
// the origin is of the ask's branch and source revision: the version reserved for its build names it, and so does its
// package-published, so the runner finds the publishing pipeline by branch and revision.
func (b *Builder) askedRevision(run meshbuilds.StepRun) {
if run.Revision <= 0 {
return
}
b.mu.Lock()
defer b.mu.Unlock()
for _, e := range run.Entries {
key, err := builds.BuilderBuildKey(e.Module, e.Fingerprint)
if err != nil {
continue
}
if o, ok := b.origin[key]; ok && o.Branch == run.Branch && o.Source == run.Source {
o.Revision = run.Revision
b.origin[key] = o
}
}
}
// buildEntry builds one entry at its fingerprint and waits for its end; pins is set, when it is done, to the pins its
// done key keeps it was built with, the registry's included (H-A-207).
func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry, pins *map[string]string) (outcome, said string) {
// buildEntry builds one entry at its fingerprint and waits for its end.
func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry) (outcome, said string) {
entry, err := b.StepEntry(ctx, e.Fingerprint)
if err != nil {
return meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("the entry %s at %s cannot be read: %v", e.Module, e.Fingerprint, err)
@@ -98,12 +63,10 @@ func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry, pins
if entry.Module != e.Module || (e.Commit != "" && entry.Commit != e.Commit) {
return meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("the entry of %s at %s is of %s at %s", e.Module, e.Fingerprint, entry.Module, entry.Commit)
}
// A package the entry names with no pin is built against its publish, waited for (H-A-192, H-A-165).
pinned, outcome, said := b.awaitPublished(ctx, entry.Module, entry.Fp, meshcatalogclient.BareManifestDigest(entry.ManifestDigest), entry.Packages)
if outcome != "" {
// A package the entry names with no pin is built against its release, waited for (H-A-165).
if outcome, said := b.awaitPins(ctx, &entry); outcome != "" {
return outcome, said
}
entry.Packages = pinned
key, err := builds.BuilderBuildKey(entry.Module, entry.Fp)
if err != nil {
return meshbuilds.StepEndedOutcomeCouldNotRun, err.Error()
@@ -112,19 +75,16 @@ func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry, pins
defer b.dropEnd(key, ended)
if done, failedWords, found := b.keyEnd(ctx, key); found {
if done {
// A key done before reports the pins it was built with, kept with it, never the ones a publish gives now.
*pins = b.keptPins(ctx, key)
return b.passedWithItsVersion(ctx, entry)
}
return meshbuilds.StepEndedOutcomeFailed, failedWords
}
m := meshcatalog.ModulesByModuleValue{Repository: entry.Repository, Commit: entry.Commit, Fp2: entry.Fp, Manifest: meshcatalogclient.BareManifestDigest(entry.ManifestDigest), Path: entry.Path,
Toolchains: CatalogToolchains(b.agreedFor(ctx, meshcatalogclient.BareManifestDigest(entry.ManifestDigest))), Packages: entry.Packages}
m := meshcatalog.ModulesByModuleValue{Repository: entry.Repository, Commit: entry.Commit, Fp2: entry.Fp, Manifest: meshcatalog.BareManifestDigest(entry.ManifestDigest), Path: entry.Path,
Toolchains: CatalogToolchains(b.agreedFor(ctx, meshcatalog.BareManifestDigest(entry.ManifestDigest))), Packages: entry.Packages}
b.consider(ctx, entry.Module, m)
select {
case log := <-ended:
if log.Outcome == builds.BuildDone {
*pins = b.keptPins(ctx, key)
return b.passedWithItsVersion(ctx, entry)
}
return meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s at %s: %s %s", log.Module, log.FP, log.Outcome, log.Error)
@@ -151,7 +111,7 @@ func (b *Builder) passedWithItsVersion(ctx context.Context, entry meshcatalog.Ca
if !reserving {
return meshbuilds.StepEndedOutcomePassed, ""
}
plan, planErr, _ := b.planOf(ctx, meshcatalogclient.BareManifestDigest(entry.ManifestDigest))
plan, planErr, _ := b.planOf(ctx, meshcatalog.BareManifestDigest(entry.ManifestDigest))
switch {
case planErr != nil:
return meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("%s at %s: the plan cannot be read to judge its version: %v", entry.Module, entry.Fp, planErr)
@@ -173,20 +133,6 @@ func (b *Builder) passedWithItsVersion(ctx context.Context, entry meshcatalog.Ca
return meshbuilds.StepEndedOutcomePassed, ""
}
// keptPins are the pins a done key keeps it was built with (H-A-207); none when it keeps none or cannot be read, so the
// runner ends the steps that waited for them could not run, loud.
func (b *Builder) keptPins(ctx context.Context, key string) map[string]string {
raw, _, ok, err := b.Claimer.Store.Get(ctx, key)
if err != nil || !ok {
return nil
}
v, err := builds.DecodeBuilderBuild(raw)
if err != nil || v.State != builds.BuildDone {
return nil
}
return v.Packages
}
// keyEnd reads the build key: whether it is done, and its error when it failed for good; found only for a key that has ended.
func (b *Builder) keyEnd(ctx context.Context, key string) (done bool, failed string, found bool) {
raw, _, ok, err := b.Claimer.Store.Get(ctx, key)
@@ -3,10 +3,9 @@ package build
import (
"context"
"errors"
"fmt"
"reflect"
"slices"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
@@ -119,185 +118,79 @@ func TestABuildStepPassesAReservingBuildOnlyWithItsReservedVersion(t *testing.T)
}
t.Run("H-A-155: an ask waits for the picture to take its revision in, then builds with its version reserved", aStepAskWaitsForItsRevision)
t.Run("H-A-155: past the bound the ask fails naming its revision, and nothing is built", aStepAskPastItsRevisionBoundFails)
t.Run("H-A-205: an ask of a revision below the picture's newest is read by number and built", anOlderKeptRevisionIsReadByNumber)
t.Run("H-A-162: a revision held that changes no entry wakes the ask", aRevisionHeldWakesTheAsk)
t.Run("H-A-165: a package with no pin waits for its release and is built with it", aPackageWithNoPinWaitsForItsRelease)
t.Run("H-A-165: past its bound a package with no release fails the step", aPackageWithNoReleaseFailsPastItsBound)
t.Run("H-A-192: a lane waits for an unpinned package's publish and fails past its bound", aLaneWaitsForAnUnpinnedPackage)
t.Run("H-A-194: what the registry seat holds pins the package before any wait", theSeatsVersionsPinBeforeTheWait)
}
// unpinnedRig is a rig whose step entry m names the package x with no version. Its manifest names no `packages`, so it
// accepts x in the default series 0.1. waiting is told when a build starts its wait for a package's publish.
func unpinnedRig(t *testing.T) (*rig, meshcatalog.CatalogEntry, <-chan struct{}) {
// unpinnedRig is a rig whose step entry m names the package x with no version, built by the module contracts.
func unpinnedRig(t *testing.T) (*rig, *fakeReservations) {
var held atomic.Int64
r, _, entry := revisionRig(t, &held, make(chan struct{}, 1))
r, res, entry := revisionRig(t, &held, make(chan struct{}, 1))
r.b.Picture = nil // no revision to wait for here
entry.Packages = map[string]string{unpinnedX: "", "git.novox.be/novox/mesh-sdk/go": "v0.1.20"}
entry.Packages = map[string]string{"git.novox.be/novox/mesh-contracts/go/seats/x": "", "git.novox.be/novox/mesh-sdk/go": "v0.1.20"}
r.b.StepEntry = func(context.Context, string) (meshcatalog.CatalogEntry, error) { return entry, nil }
waiting := make(chan struct{}, 1)
r.b.Logf = func(format string, args ...any) {
t.Logf(format, args...)
if strings.Contains(fmt.Sprintf(format, args...), "waits for its publish") {
select {
case waiting <- struct{}{}:
default:
}
}
r.b.Members = func(_ context.Context, pkg string) (string, string, bool, error) {
return "contracts", "contracts:x", pkg == "git.novox.be/novox/mesh-contracts/go/seats/x", nil
}
return r, entry, waiting
return r, res
}
const unpinnedX = "git.novox.be/novox/mesh-contracts/go/seats/x"
// awaitWaiting fails the test unless the build started its wait before ended said anything.
func awaitWaiting[T any](t *testing.T, waiting <-chan struct{}, ended <-chan T) {
t.Helper()
select {
case <-waiting:
case end := <-ended:
t.Fatalf("the build ended before the package was published: %+v", end)
case <-time.After(5 * time.Second):
t.Fatal("the build never waited for the package's publish")
}
}
// H-A-192: the pin is the version the registry seat's published carries for the path in the accepted series; a publish in
// another series and the module's release number pin nothing.
func aPackageWithNoPinWaitsForItsRelease(t *testing.T) {
r, _, waiting := unpinnedRig(t)
r.b.PackagePublished(unpinnedX, "v0.2.9") // another series than the 0.1 the module accepts: no pin
r.b.PackagePublished(unpinnedX, "0.1.4-feat-x.2") // a group's tagged version: no pin
r, _ := unpinnedRig(t)
var once sync.Once
looked := make(chan struct{})
r.b.NewestReleased = func(context.Context, string) (string, bool, error) { // the record holds none yet
once.Do(func() { close(looked) })
return "", false, nil
}
ended := make(chan meshbuilds.StepEnded, 1)
go func() {
ended <- r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
}()
awaitWaiting(t, waiting, ended)
if _, ok := r.st.vals[r.key]; ok {
t.Fatal("the build was made before the package was published")
select {
case <-looked:
case end := <-ended:
t.Fatalf("the ask ended before the package was released: %+v, asks %+v", end, r.asks.Asks())
case <-time.After(5 * time.Second):
t.Fatal("the ask never looked for the package's release")
}
r.b.PackagePublished(unpinnedX, "v0.1.3")
if _, ok := r.st.vals[r.key]; ok {
t.Fatal("the build was made before the package had a release")
}
r.b.VersionReleased("contracts", "0.1.3")
var end meshbuilds.StepEnded
select {
case end = <-ended:
case <-time.After(5 * time.Second):
t.Fatal("the publish did not wake the ask")
t.Fatal("the release did not wake the ask")
}
r.b.Wait()
asks := r.asks.Asks()
if end.Outcome != meshbuilds.StepEndedOutcomePassed || len(asks) == 0 ||
!reflect.DeepEqual(asks[0].Packages, buildstep.PackageVersions{unpinnedX: "v0.1.3", "git.novox.be/novox/mesh-sdk/go": "v0.1.20"}) {
!reflect.DeepEqual(asks[0].Packages, buildstep.PackageVersions{"git.novox.be/novox/mesh-contracts/go/seats/x": "v0.1.3", "git.novox.be/novox/mesh-sdk/go": "v0.1.20"}) {
t.Errorf("end = %+v, asks %+v; want passed, built with x at v0.1.3", end, asks)
}
// The end reports the pins the build was built with, the one the publish gave included (H-A-207).
if !reflect.DeepEqual(end.Packages, map[string]string{unpinnedX: "v0.1.3", "git.novox.be/novox/mesh-sdk/go": "v0.1.20"}) {
t.Errorf("the end reports the pins %v; want the ones it was built with", end.Packages)
}
// A done key reports the pins it keeps it was built with, never the newer one a publish gives now (H-A-207).
r.b.PackagePublished(unpinnedX, "v0.1.5")
again := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
if again.Outcome != meshbuilds.StepEndedOutcomePassed || again.Packages[unpinnedX] != "v0.1.3" {
t.Errorf("the done key's end = %+v; want passed reporting x at v0.1.3, as built", again)
}
// A done key that keeps no pins reports none, so the runner ends the waiting steps loud.
v, err := builds.DecodeBuilderBuild(r.st.vals[r.key])
if err != nil {
t.Fatal(err)
}
v.Packages = nil
raw, err := builds.EncodeBuilderBuild(v)
if err != nil {
t.Fatal(err)
}
r.st.vals[r.key] = raw
if bare := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}}); len(bare.Packages) != 0 {
t.Errorf("a done key keeping no pins reported %v; want none", bare.Packages)
}
}
func aPackageWithNoReleaseFailsPastItsBound(t *testing.T) {
r, _, _ := unpinnedRig(t)
r, _ := unpinnedRig(t)
r.b.PackageWait = time.Nanosecond
end := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
r.b.Wait()
if _, built := r.st.vals[r.key]; end.Outcome != meshbuilds.StepEndedOutcomeFailed || built ||
!strings.Contains(end.Said, "package "+unpinnedX+" has no released version in the series 0.1") {
!strings.Contains(end.Said, "package git.novox.be/novox/mesh-contracts/go/seats/x has no released version") {
t.Errorf("end = %+v, built %v; want failed naming the package, nothing built", end, built)
}
// A publish heard before gives the pin at once.
r.b.PackagePublished(unpinnedX, "v0.1.2")
// A release the builder's own record keeps gives the pin at once.
r.b.NewestReleased = func(_ context.Context, module string) (string, bool, error) {
return "0.1.2", module == "contracts", nil
}
if end := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}}); strings.Contains(end.Said, "no released version") {
t.Errorf("a publish heard did not pin the package: %+v", end)
}
}
// H-A-194: the build reads the registry seat's versions of the path once before it waits: a version in the accepted series
// pins the package with no event heard; one only in another series leaves the build waiting for a publish.
func theSeatsVersionsPinBeforeTheWait(t *testing.T) {
r, _, _ := unpinnedRig(t)
r.b.PackageWait = time.Nanosecond // no wait: the seat's answer alone gives the pin
var read []string
r.b.PublishedVersions = func(_ context.Context, path string) ([]string, error) {
read = append(read, path)
return []string{"0.2.1", "0.1.9", "0.1.8"}, nil // newest first
}
end := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
r.b.Wait()
asks := r.asks.Asks()
if end.Outcome != meshbuilds.StepEndedOutcomePassed || len(asks) == 0 || asks[0].Packages[unpinnedX] != "v0.1.9" || !slices.Equal(read, []string{unpinnedX}) {
t.Errorf("end = %+v, asks %+v, read %v; want passed, built with x at v0.1.9 read once from the seat", end, asks, read)
}
r2, _, waiting := unpinnedRig(t)
r2.b.PublishedVersions = func(context.Context, string) ([]string, error) { return []string{"0.2.1"}, nil }
ended := make(chan meshbuilds.StepEnded, 1)
go func() {
ended <- r2.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
}()
awaitWaiting(t, waiting, ended)
r2.b.PackagePublished(unpinnedX, "0.1.10")
if end := <-ended; end.Outcome != meshbuilds.StepEndedOutcomePassed {
t.Errorf("end = %+v, want passed once x was published in its series", end)
}
r2.b.Wait()
}
// H-A-192, H-A-165: the lane of an entry taken in waits the same way, never asking a build without the pin, and fails the
// key past the bound naming the package.
func aLaneWaitsForAnUnpinnedPackage(t *testing.T) {
r, entry, waiting := unpinnedRig(t)
done := make(chan struct{})
go func() {
r.b.Entry(t.Context(), Origin{Branch: "main", Source: 41, First: true}, entry)
r.b.Wait()
close(done)
}()
awaitWaiting(t, waiting, done)
if asks := r.asks.Asks(); len(asks) != 0 {
t.Fatalf("a build was asked without the package's pin: %+v", asks)
}
r.b.PackagePublished(unpinnedX, "0.1.5")
select {
case <-done:
case <-time.After(5 * time.Second):
t.Fatal("the publish did not wake the lane")
}
if asks := r.asks.Asks(); len(asks) == 0 || asks[0].Packages[unpinnedX] != "v0.1.5" {
t.Errorf("asks %+v; want one asked with x at v0.1.5", asks)
}
// Past its bound the key fails naming the package, and nothing is asked.
r2, entry2, _ := unpinnedRig(t)
r2.b.PackageWait = time.Nanosecond
r2.b.Entry(t.Context(), Origin{Branch: "main", Source: 41, First: true}, entry2)
r2.b.Wait()
if asks := r2.asks.Asks(); len(asks) != 0 || !strings.Contains(string(r2.st.vals[r2.key]), "package "+unpinnedX+" has no released version") {
t.Errorf("asks %+v, key %s; want none asked and the key failed naming the package", asks, r2.st.vals[r2.key])
t.Errorf("a release the record keeps did not pin the package: %+v", end)
}
}
@@ -356,11 +249,6 @@ func aStepAskWaitsForItsRevision(t *testing.T) {
held.Store(40)
asked := make(chan struct{}, 1)
r, res, entry := revisionRig(t, &held, asked)
var keptReads atomic.Int64 // H-A-205: a revision above the newest is waited for, never read by number
r.b.KeptRevision = func(context.Context, string, int64) (int64, map[string]string, error) {
keptReads.Add(1)
return 0, nil, errors.New("not asked")
}
ended := make(chan meshbuilds.StepEnded, 1)
go func() {
ended <- r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.7.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Revision: 7, Source: 41,
@@ -385,45 +273,8 @@ func aStepAskWaitsForItsRevision(t *testing.T) {
t.Fatal("the ask was not woken by the entry the picture took in")
}
r.b.Wait()
if end.Outcome != meshbuilds.StepEndedOutcomePassed || len(res.sources) != 1 || res.sources[0].Revision != 41 || keptReads.Load() != 0 {
t.Errorf("end = %+v, reserved from %+v, kept revisions read %d; want passed with one version reserved from revision 41, none read", end, res.sources, keptReads.Load())
}
}
// H-A-205: two pipelines of consecutive revisions run at once; the picture holds source revision 46, whose entry of m moved
// on, while the older pipeline asks the build of revision 45. Its kept catalog revision is read by number and the build runs;
// a kept revision naming another fingerprint fails the step, naming it.
func anOlderKeptRevisionIsReadByNumber(t *testing.T) {
var held atomic.Int64
held.Store(46)
r, res, _ := revisionRig(t, &held, make(chan struct{}, 1)) // a picture at 46 that never heard 45 (a restart)
r.b.RevisionWait = time.Nanosecond // a wait would fail at once: the older revision is never waited for
r.b.FirstBranch = func(branch, module string) bool { return branch == "main" && module == "m" }
var read []string
r.b.KeptRevision = func(_ context.Context, branch string, revision int64) (int64, map[string]string, error) {
read = append(read, fmt.Sprintf("%s %d", branch, revision))
return 45, map[string]string{"m": fp1}, nil
}
run := meshbuilds.StepRun{Pipeline: "main.47.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Revision: 47, Source: 45,
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}}
end := r.b.BuildStep(t.Context(), run)
r.b.Wait()
if end.Outcome != meshbuilds.StepEndedOutcomePassed || !slices.Equal(read, []string{"main 47"}) {
t.Errorf("end = %+v, kept revisions read %v; want passed, revision 47 of main read once", end, read)
}
// Its fingerprint is recorded as the branch's: its build reserves its version from source revision 45 (H-A-155).
// The version reserved names the catalog revision the ask named, which its package-published says (H-A-218).
if len(res.sources) != 1 || res.sources[0].Revision != 45 || res.sources[0].CatalogRevision != 47 {
t.Errorf("reserved from %+v; want one version reserved from source revision 45 at catalog revision 47 (H-A-218)", res.sources)
}
r2, _, _ := revisionRig(t, &held, make(chan struct{}, 1))
r2.b.RevisionWait = time.Nanosecond
r2.b.KeptRevision = func(context.Context, string, int64) (int64, map[string]string, error) {
return 45, map[string]string{"m": "src2:other"}, nil
}
if end := r2.b.BuildStep(t.Context(), run); end.Outcome != meshbuilds.StepEndedOutcomeFailed || !strings.Contains(end.Said, "names m at \"src2:other\"") {
t.Errorf("end = %+v; want failed naming the fingerprint the kept revision names", end)
if end.Outcome != meshbuilds.StepEndedOutcomePassed || len(res.sources) != 1 || res.sources[0].Revision != 41 {
t.Errorf("end = %+v, reserved from %+v; want passed with one version reserved from revision 41", end, res.sources)
}
}
@@ -65,9 +65,6 @@ type ReservedVersion struct {
// (design 47 G §371, A-57): what `package-published` says of the publication.
Branch string `json:"branch,omitempty"`
Source int64 `json:"source,omitempty"`
// Revision is the catalog revision number the build's step ask named, the indexer's count (design 47 H H-A-218): what
// `package-published` names beside the source, so the runner finds the publishing pipeline; 0 when no step ask named one.
Revision int64 `json:"revision,omitempty"`
// Dependencies are, by artifact reference, what the step's result says each artifact resolved (kind 1) and was built with
// (kind 3), kept for `package-published` (design 47 G §466, §673, A-57).
Dependencies map[string][]meshbuilds.Dependency `json:"dependencies,omitempty"`
@@ -94,8 +91,6 @@ type Source struct {
// that catalog revision (design 47 G §371, A-57), kept on the reservation for the publication.
Branch string
Revision int64
// CatalogRevision is the catalog revision number the build's step ask named (H-A-218), 0 when none did.
CatalogRevision int64
// Rebuild marks a build again of a done key whose version is not released: the standing reservation of Fingerprint is replaced
// and is no floor, so the number is the next above the catalog's highest alone (design 47 G A-490b).
Rebuild bool
@@ -182,7 +177,7 @@ func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts
}
}
out := ReservedVersion{V: 1, Module: module, Series: series, Version: fmt.Sprintf("%s.%d", series, next+1), Artifacts: map[string]string{}, At: r.now(),
Repository: src.Repository, Commit: src.Commit, Fingerprint: src.Fingerprint, Branch: src.Branch, Source: src.Revision, Revision: src.CatalogRevision}
Repository: src.Repository, Commit: src.Commit, Fingerprint: src.Fingerprint, Branch: src.Branch, Source: src.Revision}
if src.GoPackage {
out.Tag = GoTagOf(out.Version)
}
@@ -300,6 +295,23 @@ func (r *Reserver) ReleasedFor(ctx context.Context, module, fingerprint string)
return false, nil
}
// NewestReleased is the newest version the builder's own record of releases keeps for module (design 47 H H-A-165): a
// package with no pin is built against it; false when the record holds none.
func (r *Reserver) NewestReleased(ctx context.Context, module string) (string, bool, error) {
if r.Releases == nil {
return "", false, nil
}
raw, _, found, err := r.Releases.Get(ctx, ReleasedKey(module))
if err != nil || !found {
return "", false, err
}
var rec ReleasedRecord
if json.Unmarshal(raw, &rec) != nil || len(rec.Kept) == 0 {
return "", false, nil
}
return rec.Kept[len(rec.Kept)-1].Version, true, nil
}
// keepRelease adds a release to the module's record, dropping the oldest past ReleasedKept.
func (r *Reserver) keepRelease(ctx context.Context, module, fingerprint, version string) error {
n := r.ReleasedKept
@@ -448,12 +460,11 @@ type Marked struct {
}
// PackagePublished is the seat mesh-builds' package-published of one artifact the holder said published at the reservation's
// version (design 47 G §52, §334, §673, A-57): the artifact reference, the version and the digest published, the watched branch,
// the source revision the build was made from and the catalog revision its step ask named (H-A-218), and the dependencies its
// step result named.
// version (design 47 G §52, §334, §673, A-57): the artifact reference, the version and the digest published, the watched branch and
// the source revision the build was made from, and the dependencies its step result named.
func (m Marked) PackagePublished(digest string, at time.Time) meshbuilds.PackagePublished {
at = at.UTC()
return meshbuilds.PackagePublished{V: 1, Artifact: m.Artifact, At: &at, Branch: m.Reserved.Branch, Source: m.Reserved.Source, Revision: m.Reserved.Revision, Module: m.Reserved.Module,
return meshbuilds.PackagePublished{V: 1, Artifact: m.Artifact, At: &at, Branch: m.Reserved.Branch, Source: m.Reserved.Source, Module: m.Reserved.Module,
Version: m.Reserved.Version, Digest: digest, Dependencies: m.Reserved.Dependencies[m.Artifact]}
}
@@ -125,7 +125,7 @@ func TestReservationIsReadAndMarkedPublished(t *testing.T) {
ctx := context.Background()
store := &reserveMem{mem: newMem()}
r := &Reserver{Store: store, ReleasedOf: func(context.Context, string) ([]string, error) { return []string{"1.4.2"}, nil }}
v, err := r.Reserve(ctx, "m", "1.4", []string{"m:a", "m:b"}, Source{Branch: "main", Revision: 41, CatalogRevision: 44})
v, err := r.Reserve(ctx, "m", "1.4", []string{"m:a", "m:b"}, Source{Branch: "main", Revision: 41})
if err != nil || v.Version != "1.4.3" {
t.Fatalf("reserved %q (%v), want 1.4.3", v.Version, err)
}
@@ -154,12 +154,12 @@ func TestReservationIsReadAndMarkedPublished(t *testing.T) {
if err != nil || len(marked) != 1 {
t.Fatalf("a published of 1.4.3 with a built digest marked %v (%v)", marked, err)
}
// Design 47 G §52, §673, A-57: the package-published of that artifact carries the digest published, the branch, the source
// revision and the catalog revision (H-A-218) the reservation was made for, and the dependencies its step result named.
// Design 47 G §52, §673, A-57: the package-published of that artifact carries the digest published, the branch and the source
// revision the reservation was made for, and the dependencies its step result named.
at := time.Unix(100, 0)
said := marked[0].PackagePublished("sha256:aa", at)
atUTC := at.UTC()
if wantSaid := (meshbuilds.PackagePublished{V: 1, Artifact: "m:a", At: &atUTC, Branch: "main", Source: 41, Revision: 44, Module: "m", Version: "1.4.3", Digest: "sha256:aa", Dependencies: deps}); !reflect.DeepEqual(said, wantSaid) {
if wantSaid := (meshbuilds.PackagePublished{V: 1, Artifact: "m:a", At: &atUTC, Branch: "main", Source: 41, Module: "m", Version: "1.4.3", Digest: "sha256:aa", Dependencies: deps}); !reflect.DeepEqual(said, wantSaid) {
t.Errorf("package-published = %+v, want %+v", said, wantSaid)
}
if marked, _, _ := r.Published(ctx, "m-a", "1.4.3", "sha256:other"); len(marked) != 0 {
@@ -201,11 +201,13 @@ func TestAReleaseIsRecordedByFingerprintBeforeItsReservationGoes(t *testing.T) {
t.Errorf("reservations left: %v", all)
}
// One key per module holds the last releases, the oldest dropped (A-456b).
newest := ""
for _, fp := range []string{"src2:b1", "src2:b2"} {
n, err := r.Reserve(ctx, "m", "0.1", []string{"m:a"}, Source{Fingerprint: fp})
if err != nil {
t.Fatal(err)
}
newest = n.Version
if _, err := r.Released(ctx, "m", n.Version); err != nil {
t.Fatal(err)
}
@@ -219,6 +221,13 @@ func TestAReleaseIsRecordedByFingerprintBeforeItsReservationGoes(t *testing.T) {
if ok, _ := r.ReleasedFor(ctx, "m", "src2:b2"); !ok {
t.Error("the newest release is not kept")
}
// H-A-165: the newest version the record keeps is the pin of a package with none.
if v, ok, err := r.NewestReleased(ctx, "m"); !ok || err != nil || v != newest {
t.Errorf("newest released of m = %q, %v (%v), want %s", v, ok, err, newest)
}
if _, ok, _ := r.NewestReleased(ctx, "other"); ok {
t.Error("a module never released has a newest release")
}
}
// TestARebuildReservesAboveTheCatalogsHighestWhateverItsStandingNumberWas: design 47 G A-490b — a build again of a done key replaces
+4 -10
View File
@@ -266,16 +266,10 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1
if _, bus := p["bus"]; bus {
t.Errorf("the process declares a bus: it runs on its own broker credential")
}
// H-A-186, design 47 G §370: a Go package another module imports is a declared package artifact of its module, so every
// release publishes it: builds (ADR 0634 §1, ADR 0606 §4), imported by the build modules and the node-engine, and
// buildrun, imported by the indexer, the view and the watchman.
if len(m.Build.Artifacts) != 4 {
t.Fatalf("artifacts = %+v, want the program, the tools and the packages builds and buildrun", m.Build.Artifacts)
}
for i, name := range []string{"builds", "buildrun"} {
if a := m.Build.Artifacts[2+i]; a.Name != name || a.Kind != "package" || a.Language != "go" || !reflect.DeepEqual(a.Paths, []string{name}) || a.Series != "0.1" {
t.Errorf("package artifact = %+v, want the Go package %s of series 0.1", a, name)
}
// Issue 1966's bootstrap: the package artifact builds is out of the manifest until the new builder runs; the follow-up
// puts it back with its assertion (ADR 0634 §1, ADR 0606 §4: three artifacts, the third the package builds).
if len(m.Build.Artifacts) != 2 {
t.Fatalf("artifacts = %+v, want the program and the tools", m.Build.Artifacts)
}
if a := m.Build.Artifacts[1]; a.Name != "tools" || a.Kind != "bundle" || a.Binary != "mesh-delivery-builder-tools" || !reflect.DeepEqual(a.Loads, []string{"mesh-delivery-builder-tools"}) {
t.Errorf("tools artifact = %+v", a) // 1945
-18
View File
@@ -334,24 +334,6 @@
"loads": [
"mesh-delivery-builder-tools"
]
},
{
"name": "builds",
"kind": "package",
"language": "go",
"paths": [
"builds"
],
"series": "0.1"
},
{
"name": "buildrun",
"kind": "package",
"language": "go",
"paths": [
"buildrun"
],
"series": "0.1"
}
]
}
@@ -1,22 +0,0 @@
package meshcatalogclient
import "testing"
// An entry's manifest-digest carries sha256: and the module's value and the key of `manifests` the 64 digits bare, one
// digest in two forms (design 47 G §578); the two helpers live in the seat's client, outside its contract's generated
// root (H-A-200).
func TestAManifestDigestIsPrefixedForTheEntryAndBareForTheKey(t *testing.T) {
const bare = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
for _, tc := range []struct{ in, entry, key string }{
{bare, "sha256:" + bare, bare},
{"sha256:" + bare, "sha256:" + bare, bare},
{"", "", ""},
} {
if got := EntryManifestDigest(tc.in); got != tc.entry {
t.Errorf("EntryManifestDigest(%q) = %q, want %q", tc.in, got, tc.entry)
}
if got := BareManifestDigest(tc.in); got != tc.key {
t.Errorf("BareManifestDigest(%q) = %q, want %q", tc.in, got, tc.key)
}
}
}
@@ -17,9 +17,8 @@ import (
// eventPrefix starts every event subject of the seat.
const eventPrefix = "mesh.seat." + seat.Seat + ".event."
// perRevision are the events said per pipeline, on <event>.<branch>.<revision>, its catalog revision's number as its key
// names it (design 47 H §746, H-A-72 (3), H-A-221).
var perRevision = []string{seat.EventPipelineStarted, seat.EventPipelineEnded, seat.EventWaitHeld, seat.EventWaitEnded}
// perSource are the events said per source revision, on <event>.<branch>.<source> (design 47 H §746, H-A-72 (3)).
var perSource = []string{seat.EventPipelineStarted, seat.EventPipelineEnded, seat.EventWaitHeld, seat.EventWaitEnded}
// Subject is where the runner says a step's event or a run's start: mesh.seat.mesh-pipeline-runner.event.<event>.
// <module>.<branch>.<commit>, each name escaped as a key's name is, so a branch holding a dot is one token (design 47 H
@@ -28,42 +27,41 @@ func Subject(event, module, branch, commit string) string {
return eventPrefix + event + "." + keys.EscapeName(module) + "." + keys.EscapeName(branch) + "." + keys.EscapeName(commit)
}
// RevisionSubject is where the runner says a pipeline's start and end and its waits: mesh.seat.mesh-pipeline-runner.
// event.<event>.<branch>.<revision>, the pipeline's catalog revision number, the branch escaped as a key's name is (design
// 47 H §746, H-A-72 (3), H-A-221).
func RevisionSubject(event, branch string, revision int64) string {
return eventPrefix + event + "." + keys.EscapeName(branch) + "." + strconv.FormatInt(revision, 10)
// SourceSubject is where the runner says a pipeline's start and end and its waits: mesh.seat.mesh-pipeline-runner.
// event.<event>.<branch>.<source>, the branch escaped as a key's name is (design 47 H §746, H-A-72 (3)).
func SourceSubject(event, branch string, source int64) string {
return eventPrefix + event + "." + keys.EscapeName(branch) + "." + strconv.FormatInt(source, 10)
}
// Of is what an event's subject names: a step's event or a run's start names the module, the branch and the commit; an
// event per pipeline names the branch and the pipeline's catalog revision (H-A-221).
// event per source revision names the branch and the source.
type Of struct {
Event, Module, Branch, Commit string
Revision int64
Source int64
}
// Parse reads a subject Subject or RevisionSubject built: an event of the seat's, and its names unescaped.
// Parse reads a subject Subject or SourceSubject built: an event of the seat's, and its names unescaped.
func Parse(subject string) (Of, error) {
rest, ok := strings.CutPrefix(subject, eventPrefix)
toks := strings.Split(rest, ".")
byRevision := len(toks) > 0 && slices.Contains(perRevision, toks[0])
bySource := len(toks) > 0 && slices.Contains(perSource, toks[0])
want := 4
if byRevision {
if bySource {
want = 3
}
if !ok || len(toks) != want || !slices.Contains(seat.Events(), toks[0]) {
return Of{}, fmt.Errorf("meshpipelinerunnerclient: %q is no event subject of the seat %s", subject, seat.Seat)
}
if byRevision {
if bySource {
branch, err := keys.UnescapeName(toks[1])
if err != nil || branch == "" {
return Of{}, fmt.Errorf("meshpipelinerunnerclient: %q: branch: %v", subject, err)
}
revision, err := strconv.ParseInt(toks[2], 10, 64)
if err != nil || revision < 1 {
return Of{}, fmt.Errorf("meshpipelinerunnerclient: %q: no revision", subject)
source, err := strconv.ParseInt(toks[2], 10, 64)
if err != nil || source < 1 {
return Of{}, fmt.Errorf("meshpipelinerunnerclient: %q: no source revision", subject)
}
return Of{Event: toks[0], Branch: branch, Revision: revision}, nil
return Of{Event: toks[0], Branch: branch, Source: source}, nil
}
names := make([]string, 0, 3)
for i, t := range toks[1:] {
@@ -17,14 +17,14 @@ func TestSubjectsAndBodies(t *testing.T) {
if of, err := Parse(s); err != nil || of != (Of{Event: seat.EventStepEnded, Module: "nats", Branch: "release.1", Commit: "c1"}) {
t.Errorf("parse %s: %+v, %v", s, of, err)
}
ps := RevisionSubject(seat.EventPipelineEnded, "release.1", 3589)
ps := SourceSubject(seat.EventPipelineEnded, "release.1", 3589)
if ps != "mesh.seat.mesh-pipeline-runner.event.pipeline-ended.release=2e1.3589" {
t.Errorf("pipeline-ended subject %s", ps)
}
if of, err := Parse(ps); err != nil || of != (Of{Event: seat.EventPipelineEnded, Branch: "release.1", Revision: 3589}) {
if of, err := Parse(ps); err != nil || of != (Of{Event: seat.EventPipelineEnded, Branch: "release.1", Source: 3589}) {
t.Errorf("parse %s: %+v, %v", ps, of, err)
}
if of, err := Parse(RevisionSubject(seat.EventWaitHeld, "main", 7)); err != nil || of.Revision != 7 || of.Module != "" {
if of, err := Parse(SourceSubject(seat.EventWaitHeld, "main", 7)); err != nil || of.Source != 7 || of.Module != "" {
t.Errorf("wait-held: %+v, %v", of, err)
}
for _, bad := range []string{"not.an.event.subject",
@@ -95,8 +95,6 @@ type PackagePublished struct {
Digest string `json:"digest"`
// the module
Module string `json:"module"`
// the catalog revision number of that branch the build was made from, the indexer's count (design 47 H-A-218, H-A-220); absent from a reservation made before the field
Revision int64 `json:"revision,omitempty"`
// the source revision of that branch the build was made from
Source int64 `json:"source"`
V int64 `json:"v"`
@@ -116,11 +114,9 @@ type StepEnded struct {
// the node the builder ran it on, as in the subject
Node string `json:"node"`
// never passed for a build that built nothing
Outcome string `json:"outcome"`
// the pins the step's entries were built with, each mesh package's version by its module path, the ones the builder took from the registry for a package the entry named with none included; the runner asks the entry's held test steps with them (H-A-207)
Packages map[string]string `json:"packages,omitempty"`
Pipeline string `json:"pipeline"`
Rerun bool `json:"rerun,omitempty"`
Outcome string `json:"outcome"`
Pipeline string `json:"pipeline"`
Rerun bool `json:"rerun,omitempty"`
// why it failed or could not run: the entry not built, the part not heard
Said string `json:"said,omitempty"`
Started *time.Time `json:"started,omitempty"`
@@ -715,10 +715,6 @@ type CatalogRevision struct {
Base int64 `json:"base,omitempty"`
// the watched branch the revision is of
Branch string `json:"branch,omitempty"`
// what made the revision (design 47 H-A-212, H-A-213): a take-in's the gate's call its source revision names, a re-pin's release of <module> <version>, a take-in again's take-in again of <repository>; absent for a candidate's
Call string `json:"call,omitempty"`
// the kind of what made the revision, which a reader reads instead of call's words (design 47 H-A-213 (3), H-A-214, H-A-215, H-A-217): a take-in's, a re-pin's on a module's release, a take-in again's, a recompute's that moved a fingerprint (its call naming the reason); absent for a candidate's
CallKind string `json:"call-kind,omitempty"`
// the seat definitions it was judged with, each by its digest, read whole through seat-definitions
Definitions []CatalogRevisionDefinitionsItem `json:"definitions,omitempty"`
// one per module, by name
@@ -729,12 +725,8 @@ type CatalogRevision struct {
Heads string `json:"heads,omitempty"`
// trunk or candidate
Kind string `json:"kind"`
// for call-kind release, the module released (H-A-215)
Module string `json:"module,omitempty"`
// the movers of the take-in that wrote it, so a redelivered source-moved whose revision is kept says take-in.taken again from them, recomputing nothing (design 47 H §743, H-A-92); absent for one with no mover
Movers []TakeInEntry `json:"movers,omitempty"`
// for call-kind take-in-again, the repository taken in again (H-A-215)
Repository string `json:"repository,omitempty"`
// the catalog graph revision's number, counted by its holder
Revision int64 `json:"revision"`
// the source revision it was taken in from; for a candidate, the source revision its heads are merged onto
@@ -743,18 +735,8 @@ type CatalogRevision struct {
Trunk []Head `json:"trunk,omitempty"`
// the layout version
V int64 `json:"v,omitempty"`
// for call-kind release, the version released (H-A-215)
Version string `json:"version,omitempty"`
}
// The values of CatalogRevision's call-kind.
const (
CatalogRevisionCallKindTakeIn = "take-in"
CatalogRevisionCallKindRelease = "release"
CatalogRevisionCallKindTakeInAgain = "take-in-again"
CatalogRevisionCallKindRecompute = "recompute"
)
// Edge is one edge of the catalog graph between artifact references, B before A (ADR 0666 section 14): kind 1 B published before A builds (a package A imports), 2 B published before A delivers (fetched by reference at its start), 3 B delivered before A builds (its build module and toolchain), 4 B delivered before A delivers (a provision, bus state, a format's reader), 5 B has acted for A's new version (an acting rule), 6 B runs a version that serves A (a verb or a manifest word A uses).
type Edge struct {
// declared in the artifact entry's needs of A's manifest, where nothing else says it
@@ -1319,10 +1301,6 @@ type CatalogRevisionEvent struct {
At *time.Time `json:"at,omitempty"`
// the watched branch the revision is of, escaped as a key name is
Branch string `json:"branch,omitempty"`
// what made the revision, as the revision's call names it (design 47 H-A-212, H-A-213); absent for a candidate's
Call string `json:"call,omitempty"`
// the kind of what made the revision, which a reader reads instead of call's words (design 47 H-A-213 (3), H-A-214, H-A-215, H-A-217); absent for a candidate's
CallKind string `json:"call-kind,omitempty"`
// the modules whose entries differ from the revision before it of the same kind
Changed []string `json:"changed,omitempty"`
// how many entries it lists
@@ -1330,26 +1308,12 @@ type CatalogRevisionEvent struct {
Group string `json:"group,omitempty"`
Heads string `json:"heads,omitempty"`
// trunk or candidate: a candidate's moves no running part
Kind string `json:"kind"`
// for call-kind release, the module released (H-A-215)
Module string `json:"module,omitempty"`
// for call-kind take-in-again, the repository taken in again (H-A-215)
Repository string `json:"repository,omitempty"`
Revision int64 `json:"revision"`
Source int64 `json:"source"`
V int64 `json:"v"`
// for call-kind release, the version released (H-A-215)
Version string `json:"version,omitempty"`
Kind string `json:"kind"`
Revision int64 `json:"revision"`
Source int64 `json:"source"`
V int64 `json:"v"`
}
// The values of CatalogRevisionEvent's call-kind.
const (
CatalogRevisionEventCallKindTakeIn = "take-in"
CatalogRevisionEventCallKindRelease = "release"
CatalogRevisionEventCallKindTakeInAgain = "take-in-again"
CatalogRevisionEventCallKindRecompute = "recompute"
)
// FillingEvent is the body of the event filling.
type FillingEvent = Fill
@@ -17,7 +17,7 @@ func TestFixtures(t *testing.T) {
decode func([]byte) (any, error)
subjectIs *regexp.Regexp
}{
{name: "a trunk revision kept, on the seat's subject", subject: "mesh.seat.mesh-catalog.event.catalog-revision.main", given: "{\"at\":\"2026-11-09T19:00:00Z\",\"branch\":\"main\",\"call\":\"release of mesh-sdk 0.1.27\",\"call-kind\":\"release\",\"changed\":[\"xclip\"],\"entries\":2,\"kind\":\"trunk\",\"module\":\"mesh-sdk\",\"revision\":12,\"source\":3589,\"v\":1,\"version\":\"0.1.27\"}", wire: "{\"at\":\"2026-11-09T19:00:00Z\",\"branch\":\"main\",\"call\":\"release of mesh-sdk 0.1.27\",\"call-kind\":\"release\",\"changed\":[\"xclip\"],\"entries\":2,\"kind\":\"trunk\",\"module\":\"mesh-sdk\",\"revision\":12,\"source\":3589,\"v\":1,\"version\":\"0.1.27\"}",
{name: "a trunk revision kept, on the seat's subject", subject: "mesh.seat.mesh-catalog.event.catalog-revision.main", given: "{\"at\":\"2026-11-09T19:00:00Z\",\"branch\":\"main\",\"changed\":[\"xclip\"],\"entries\":2,\"kind\":\"trunk\",\"revision\":12,\"source\":3589,\"v\":1}", wire: "{\"at\":\"2026-11-09T19:00:00Z\",\"branch\":\"main\",\"changed\":[\"xclip\"],\"entries\":2,\"kind\":\"trunk\",\"revision\":12,\"source\":3589,\"v\":1}",
decode: func(b []byte) (any, error) { var v CatalogRevisionEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-catalog\\.event\\.catalog-revision\\.[^.*> ]+$")},
{name: "a candidate's revision kept", subject: "mesh.seat.mesh-catalog.event.catalog-revision.main", given: "{\"branch\":\"main\",\"entries\":2,\"group\":\"feat/47-G\",\"heads\":\"0123456789ab\",\"kind\":\"candidate\",\"revision\":13,\"source\":3589,\"v\":1}", wire: "{\"branch\":\"main\",\"entries\":2,\"group\":\"feat/47-G\",\"heads\":\"0123456789ab\",\"kind\":\"candidate\",\"revision\":13,\"source\":3589,\"v\":1}",
decode: func(b []byte) (any, error) { var v CatalogRevisionEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-catalog\\.event\\.catalog-revision\\.[^.*> ]+$")},
@@ -59,7 +59,7 @@ func TestFixtures(t *testing.T) {
decode: func(b []byte) (any, error) { var v ModulesAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
{name: "the module verb (captured: xclip)", subject: "", given: "{\"module\":{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fp\":\"src2:b427bdde1f100e2f52490cfa9df0c5112dd3700226b4892349f0c14a15a4af56\",\"manifest\":{\"build\":{\"artifacts\":[{\"binary\":\"xclip-tools\",\"from\":\"cmd/xclip-tools\",\"kind\":\"bundle\",\"language\":\"go\",\"loads\":[\"xclip-tools\"],\"name\":\"tools\",\"system\":\"arch\"}]},\"capabilities\":[\"package-manager\"],\"module\":\"xclip\",\"resources\":[{\"id\":\"package\",\"package\":\"xclip\",\"type\":\"package\"}],\"tools\":[\"xclip_copy\",\"xclip_paste\",\"xclip_targets\",\"xclip_session\"],\"version\":\"1\"},\"module\":\"xclip\",\"path\":\"modules/xclip\",\"repository\":\"novox/mesh-catalog\"}}", wire: "{\"module\":{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fp\":\"src2:b427bdde1f100e2f52490cfa9df0c5112dd3700226b4892349f0c14a15a4af56\",\"manifest\":{\"build\":{\"artifacts\":[{\"binary\":\"xclip-tools\",\"from\":\"cmd/xclip-tools\",\"kind\":\"bundle\",\"language\":\"go\",\"loads\":[\"xclip-tools\"],\"name\":\"tools\",\"system\":\"arch\"}]},\"capabilities\":[\"package-manager\"],\"module\":\"xclip\",\"resources\":[{\"id\":\"package\",\"package\":\"xclip\",\"type\":\"package\"}],\"tools\":[\"xclip_copy\",\"xclip_paste\",\"xclip_targets\",\"xclip_session\"],\"version\":\"1\"},\"module\":\"xclip\",\"path\":\"modules/xclip\",\"repository\":\"novox/mesh-catalog\"}}",
decode: func(b []byte) (any, error) { var v ModuleAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
{name: "a watched branch's catalog graph revision", subject: "", given: "{\"at\":\"2026-11-09T19:00:00Z\",\"base\":9,\"branch\":\"main\",\"call\":\"release of mesh-sdk 0.1.27\",\"call-kind\":\"release\",\"definitions\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"seat\":\"node-build-go\"}],\"entries\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"mesh-build-go\"},{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"xclip\"}],\"kind\":\"trunk\",\"module\":\"mesh-sdk\",\"movers\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fingerprint\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"module\":\"xclip\",\"repository\":\"novox/mesh-catalog\"}],\"revision\":12,\"source\":3589,\"trunk\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"repository\":\"novox/mesh-catalog\"}],\"v\":1,\"version\":\"0.1.27\"}", wire: "{\"at\":\"2026-11-09T19:00:00Z\",\"base\":9,\"branch\":\"main\",\"call\":\"release of mesh-sdk 0.1.27\",\"call-kind\":\"release\",\"definitions\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"seat\":\"node-build-go\"}],\"entries\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"mesh-build-go\"},{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"xclip\"}],\"kind\":\"trunk\",\"module\":\"mesh-sdk\",\"movers\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fingerprint\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"module\":\"xclip\",\"repository\":\"novox/mesh-catalog\"}],\"revision\":12,\"source\":3589,\"trunk\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"repository\":\"novox/mesh-catalog\"}],\"v\":1,\"version\":\"0.1.27\"}",
{name: "a watched branch's catalog graph revision", subject: "", given: "{\"at\":\"2026-11-09T19:00:00Z\",\"base\":9,\"branch\":\"main\",\"definitions\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"seat\":\"node-build-go\"}],\"entries\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"mesh-build-go\"},{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"xclip\"}],\"kind\":\"trunk\",\"movers\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fingerprint\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"module\":\"xclip\",\"repository\":\"novox/mesh-catalog\"}],\"revision\":12,\"source\":3589,\"trunk\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"repository\":\"novox/mesh-catalog\"}],\"v\":1}", wire: "{\"at\":\"2026-11-09T19:00:00Z\",\"base\":9,\"branch\":\"main\",\"definitions\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"seat\":\"node-build-go\"}],\"entries\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"mesh-build-go\"},{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"xclip\"}],\"kind\":\"trunk\",\"movers\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fingerprint\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"module\":\"xclip\",\"repository\":\"novox/mesh-catalog\"}],\"revision\":12,\"source\":3589,\"trunk\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"repository\":\"novox/mesh-catalog\"}],\"v\":1}",
decode: func(b []byte) (any, error) {
var v CatalogRevisionsByBranchRevisionValue
err := json.Unmarshal(b, &v)
@@ -1,7 +1,4 @@
package meshcatalogclient
// The two helpers below are hand-written, so they live in the seat's client, never under its contract's generated
// root (design 47 H, H-A-200; ADR 0608 §5).
package meshcatalog
import "strings"
@@ -65,9 +65,9 @@ const (
// StreamPipelineStarted is the stream pipeline-started is kept in.
const StreamPipelineStarted = "SEAT_mesh-pipeline-runner_pipeline-started"
// SubjectPipelineStarted is where pipeline-started is published, on mesh.seat.mesh-pipeline-runner.event.pipeline-started.<branch>.<revision>.
func SubjectPipelineStarted(branch, revision string) string {
return "mesh.seat.mesh-pipeline-runner.event.pipeline-started." + branch + "." + revision
// SubjectPipelineStarted is where pipeline-started is published, on mesh.seat.mesh-pipeline-runner.event.pipeline-started.<branch>.<source>.
func SubjectPipelineStarted(branch, source string) string {
return "mesh.seat.mesh-pipeline-runner.event.pipeline-started." + branch + "." + source
}
// SubjectPipelineStartedAll hears every pipeline-started.
@@ -76,9 +76,9 @@ const SubjectPipelineStartedAll = "mesh.seat.mesh-pipeline-runner.event.pipeline
// StreamPipelineEnded is the stream pipeline-ended is kept in.
const StreamPipelineEnded = "SEAT_mesh-pipeline-runner_pipeline-ended"
// SubjectPipelineEnded is where pipeline-ended is published, on mesh.seat.mesh-pipeline-runner.event.pipeline-ended.<branch>.<revision>.
func SubjectPipelineEnded(branch, revision string) string {
return "mesh.seat.mesh-pipeline-runner.event.pipeline-ended." + branch + "." + revision
// SubjectPipelineEnded is where pipeline-ended is published, on mesh.seat.mesh-pipeline-runner.event.pipeline-ended.<branch>.<source>.
func SubjectPipelineEnded(branch, source string) string {
return "mesh.seat.mesh-pipeline-runner.event.pipeline-ended." + branch + "." + source
}
// SubjectPipelineEndedAll hears every pipeline-ended.
@@ -120,9 +120,9 @@ const SubjectStepEndedAll = "mesh.seat.mesh-pipeline-runner.event.step-ended.*.*
// StreamWaitHeld is the stream wait-held is kept in.
const StreamWaitHeld = "SEAT_mesh-pipeline-runner_wait-held"
// SubjectWaitHeld is where wait-held is published, on mesh.seat.mesh-pipeline-runner.event.wait-held.<branch>.<revision>.
func SubjectWaitHeld(branch, revision string) string {
return "mesh.seat.mesh-pipeline-runner.event.wait-held." + branch + "." + revision
// SubjectWaitHeld is where wait-held is published, on mesh.seat.mesh-pipeline-runner.event.wait-held.<branch>.<source>.
func SubjectWaitHeld(branch, source string) string {
return "mesh.seat.mesh-pipeline-runner.event.wait-held." + branch + "." + source
}
// SubjectWaitHeldAll hears every wait-held.
@@ -131,9 +131,9 @@ const SubjectWaitHeldAll = "mesh.seat.mesh-pipeline-runner.event.wait-held.*.*"
// StreamWaitEnded is the stream wait-ended is kept in.
const StreamWaitEnded = "SEAT_mesh-pipeline-runner_wait-ended"
// SubjectWaitEnded is where wait-ended is published, on mesh.seat.mesh-pipeline-runner.event.wait-ended.<branch>.<revision>.
func SubjectWaitEnded(branch, revision string) string {
return "mesh.seat.mesh-pipeline-runner.event.wait-ended." + branch + "." + revision
// SubjectWaitEnded is where wait-ended is published, on mesh.seat.mesh-pipeline-runner.event.wait-ended.<branch>.<source>.
func SubjectWaitEnded(branch, source string) string {
return "mesh.seat.mesh-pipeline-runner.event.wait-ended." + branch + "." + source
}
// SubjectWaitEndedAll hears every wait-ended.
@@ -156,11 +156,11 @@ const (
StateStepOutputs = "step-outputs"
)
// PipelinesByBranchRevisionKey is the key of one value of the state pipelines under the form "<branch>.<revision>".
func PipelinesByBranchRevisionKey(branch, revision string) string { return branch + "." + revision }
// PipelinesByBranchSourceKey is the key of one value of the state pipelines under the form "<branch>.<source>".
func PipelinesByBranchSourceKey(branch, source string) string { return branch + "." + source }
// ParsePipelinesByBranchRevisionKey reads the parts of a key of the form "<branch>.<revision>", in the order of the form; the last result is false for a key of another form.
func ParsePipelinesByBranchRevisionKey(key string) (string, string, bool) {
// ParsePipelinesByBranchSourceKey reads the parts of a key of the form "<branch>.<source>", in the order of the form; the last result is false for a key of another form.
func ParsePipelinesByBranchSourceKey(key string) (string, string, bool) {
rest := key
v0, after0, found0 := strings.Cut(rest, ".")
if !found0 || v0 == "" {
@@ -178,9 +178,9 @@ func ParsePipelinesByBranchRevisionKey(key string) (string, string, bool) {
return v0, v2, true
}
// DecodePipelinesByBranchRevisionValue reads one value of the state pipelines under the form "<branch>.<revision>", refusing a field PipelinesByBranchRevisionValue does not know and any text after the value.
func DecodePipelinesByBranchRevisionValue(raw []byte) (PipelinesByBranchRevisionValue, error) {
var v PipelinesByBranchRevisionValue
// DecodePipelinesByBranchSourceValue reads one value of the state pipelines under the form "<branch>.<source>", refusing a field PipelinesByBranchSourceValue does not know and any text after the value.
func DecodePipelinesByBranchSourceValue(raw []byte) (PipelinesByBranchSourceValue, error) {
var v PipelinesByBranchSourceValue
dec := json.NewDecoder(bytes.NewReader(raw))
dec.DisallowUnknownFields()
if err := dec.Decode(&v); err != nil {
@@ -276,48 +276,6 @@ func DecodePipelinesByBranchRunRunValue(raw []byte) (PipelinesByBranchRunRunValu
return v, nil
}
// PipelinesByStepsPipelineMoverKey is the key of one value of the state pipelines under the form "steps.<pipeline>.<mover>".
func PipelinesByStepsPipelineMoverKey(pipeline, mover string) string {
return "steps." + pipeline + "." + mover
}
// ParsePipelinesByStepsPipelineMoverKey reads the parts of a key of the form "steps.<pipeline>.<mover>", in the order of the form; the last result is false for a key of another form.
func ParsePipelinesByStepsPipelineMoverKey(key string) (string, string, bool) {
rest := key
if !strings.HasPrefix(rest, "steps.") {
return "", "", false
}
rest = rest[6:]
v1, after1, found1 := strings.Cut(rest, ".")
if !found1 || v1 == "" {
return "", "", false
}
rest = after1
if rest == "" {
return "", "", false
}
v3 := rest
rest = ""
if rest != "" {
return "", "", false
}
return v1, v3, true
}
// DecodePipelinesByStepsPipelineMoverValue reads one value of the state pipelines under the form "steps.<pipeline>.<mover>", refusing a field PipelinesByStepsPipelineMoverValue does not know and any text after the value.
func DecodePipelinesByStepsPipelineMoverValue(raw []byte) (PipelinesByStepsPipelineMoverValue, error) {
var v PipelinesByStepsPipelineMoverValue
dec := json.NewDecoder(bytes.NewReader(raw))
dec.DisallowUnknownFields()
if err := dec.Decode(&v); err != nil {
return v, err
}
if dec.More() {
return v, errors.New("text after the value")
}
return v, nil
}
// QueueByQueueNextKey is the key of one value of the state queue under the form "queue.next".
func QueueByQueueNextKey() string { return "queue.next" }
@@ -565,10 +523,8 @@ type LogEntry struct {
Artifact string `json:"artifact,omitempty"`
At time.Time `json:"at"`
// the keys of the kept results the step wrote or carried, <level>.<digest>, which the runner ages by their last-read (design 47 G sections 330, 470)
Kept []string `json:"kept,omitempty"`
// the module of the mover whose step the entry is of (H-A-190)
Module string `json:"module,omitempty"`
Outcome string `json:"outcome,omitempty"`
Kept []string `json:"kept,omitempty"`
Outcome string `json:"outcome,omitempty"`
// the output's name in step-outputs
Output string `json:"output,omitempty"`
Pipeline string `json:"pipeline"`
@@ -665,12 +621,10 @@ type Pipeline struct {
// how often the pipeline has ended: 1 at its first end, one higher at each end again after a passing rerun, so the assigner takes a later end of the same source revision (design 47 G sections 29, 325)
EndCount int64 `json:"end-count,omitempty"`
Ended *time.Time `json:"ended,omitempty"`
// how many movers the pipeline has, set by list alone (H-A-191)
MoverCount int64 `json:"mover-count,omitempty"`
// the modules the pipeline moves, its take-in's entries (design 47 H §746)
Movers []Mover `json:"movers"`
Outcome string `json:"outcome"`
// the pipeline's key, <branch>.<revision>, <branch>.<source>.published.<package>.<version> for a pipeline a publish started, or <branch>.run.<run> for a run on request; a branch escaped as a key's name is (design 47 H §746, §747; H-A-83)
// the pipeline's key, <branch>.<source>, <branch>.<source>.published.<package>.<version> for a pipeline a publish started, or <branch>.run.<run> for a run on request; a branch escaped as a key's name is (design 47 H §746, §747; H-A-83)
Pipeline string `json:"pipeline"`
// for a pipeline a package's publish started, that publish (A-202)
Published *Published `json:"published,omitempty"`
@@ -683,13 +637,11 @@ type Pipeline struct {
// the source revision of the watched branch; the pipeline is keyed by it (ADR 0666 section 9)
Source int64 `json:"source"`
Started *time.Time `json:"started,omitempty"`
// how many steps the pipeline's movers have, set by list alone, which answers the record with its steps empty (H-A-191)
StepCount int64 `json:"step-count,omitempty"`
// every mover's steps, each naming its mover, approval last; in the state pipelines a record's own are empty, each mover's kept under its own key (H-A-187)
// every mover's steps, each naming its mover, approval last
Steps []StepResult `json:"steps"`
// the modules a newer waiting take-in of the same branch took out of the pipeline, written with the movers by compare-and-set; at its placing the runner writes a mover's build-step why from it (design 47 H §748, H-A-107)
Superseded []string `json:"superseded,omitempty"`
// the step-log entries of the steps that ended and are not appended to the step log yet, in order; written with the step's end in one write and cleared when appended (design 47 G sections 486, 606); in the state pipelines a record's own are empty, each mover's kept under its own key beside its steps (H-A-189)
// the step-log entries of the steps that ended and are not appended to the step log yet, in order; written with the step's end in one write and cleared when appended (design 47 G sections 486, 606)
Unlogged []LogEntry `json:"unlogged,omitempty"`
// the layout version
V int64 `json:"v,omitempty"`
@@ -720,7 +672,7 @@ type PipelineEnded struct {
// per mover its fingerprint, released version, step results and tier, in the delivery order (design 47 H §746, H-A-2)
Movers []MoverEnded `json:"movers"`
Outcome string `json:"outcome"`
// the pipeline's key, <branch>.<revision>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)
// the pipeline's key, <branch>.<source>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)
Pipeline string `json:"pipeline"`
// the catalog graph revision of the branch the pipeline runs for
Revision int64 `json:"revision,omitempty"`
@@ -753,7 +705,7 @@ type PipelineStarted struct {
Call string `json:"call,omitempty"`
// the pipeline's movers
Movers []Mover `json:"movers"`
// the pipeline's key, <branch>.<revision>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)
// the pipeline's key, <branch>.<source>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)
Pipeline string `json:"pipeline"`
// for a pipeline a package's publish started, that publish (A-202)
Published *Published `json:"published,omitempty"`
@@ -814,7 +766,7 @@ type StepEvent struct {
// the module's build-input fingerprint at the commit
Fingerprint string `json:"fingerprint,omitempty"`
Module string `json:"module"`
// the pipeline's key, <branch>.<revision>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)
// the pipeline's key, <branch>.<source>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)
Pipeline string `json:"pipeline"`
Result StepResult `json:"result"`
// the catalog graph revision of the branch the pipeline runs for
@@ -966,7 +918,7 @@ type ToolchainUsed struct {
Version string `json:"version"`
}
// Wait is a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283); said on <branch>.<revision> (design 47 H-A-72, H-A-221).
// Wait is a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283); said on <branch>.<source> (design 47 H-A-72).
type Wait struct {
Artifact string `json:"artifact,omitempty"`
// the step's own ask, where it has one (design 47 G §731)
@@ -983,7 +935,7 @@ type Wait struct {
Module string `json:"module,omitempty"`
// what the step waits on (design 47 *Health and going back*)
On string `json:"on"`
// the pipeline's key, <branch>.<revision>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)
// the pipeline's key, <branch>.<source>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)
Pipeline string `json:"pipeline"`
// the catalog graph revision of the branch the pipeline runs for
Revision int64 `json:"revision,omitempty"`
@@ -1024,7 +976,7 @@ const (
// Waiting is a pipeline waiting its turn under pipelines-at-once (design 47 H §748, H-A-72).
type Waiting struct {
// the pipeline's key, <branch>.<revision>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)
// the pipeline's key, <branch>.<source>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)
Pipeline string `json:"pipeline"`
}
@@ -1074,8 +1026,6 @@ type StartAnswer struct {
// ShowInput is what show takes.
type ShowInput struct {
// a mover's module: the answer's pipeline then holds that mover's steps alone (H-A-187)
Mover string `json:"mover,omitempty"`
// the pipeline's key
Pipeline string `json:"pipeline"`
}
@@ -1355,8 +1305,8 @@ type FullEvent = Fill
// RoomEvent is the body of the event room.
type RoomEvent = Fill
// PipelinesByBranchRevisionValue is one value of the state pipelines under the key <branch>.<revision>.
type PipelinesByBranchRevisionValue = Pipeline
// PipelinesByBranchSourceValue is one value of the state pipelines under the key <branch>.<source>.
type PipelinesByBranchSourceValue = Pipeline
// PipelinesByBranchSourcePublishedPackageVersionValue is one value of the state pipelines under the key <branch>.<source>.published.<package>.<version>.
type PipelinesByBranchSourcePublishedPackageVersionValue = Pipeline
@@ -1364,14 +1314,6 @@ type PipelinesByBranchSourcePublishedPackageVersionValue = Pipeline
// PipelinesByBranchRunRunValue is one value of the state pipelines under the key <branch>.run.<run>.
type PipelinesByBranchRunRunValue = Pipeline
// PipelinesByStepsPipelineMoverValue is one value of the state pipelines under the key steps.<pipeline>.<mover>.
type PipelinesByStepsPipelineMoverValue struct {
// the mover's steps
Steps []StepResult `json:"steps"`
// the step-log entries of the mover's steps that ended and are not appended to the step log yet, in order (H-A-189)
Unlogged []LogEntry `json:"unlogged,omitempty"`
}
// QueueByQueueNextValue is one value of the state queue under the key queue.next.
type QueueByQueueNextValue = int64
@@ -45,7 +45,7 @@ func TestFixtures(t *testing.T) {
decode: func(b []byte) (any, error) { var v StepEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-pipeline-runner\\.event\\.step-ended\\.[^.*> ]+\\.[^.*> ]+\\.[^.*> ]+$")},
{name: "a take-in's pipeline kept, its steps naming their mover", subject: "", given: "{\"branch\":\"main\",\"call\":\"0123456789abcdef\",\"end-count\":1,\"ended\":\"2026-11-10T10:20:00Z\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":1}],\"outcome\":\"passed\",\"pipeline\":\"main.3589\",\"revision\":12,\"source\":3589,\"started\":\"2026-11-10T10:00:00Z\",\"steps\":[{\"artifact\":\"nats:program\",\"ask\":\"m2kx9a1b3c4d5e\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"anchor\",\"outcome\":\"passed\",\"seat\":\"node-build-go\",\"status\":\"required\",\"step\":\"lint\"},{\"artifact\":\"nats:program\",\"carried-from\":\"main.3588\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"outcome\":\"carried\",\"status\":\"required\",\"step\":\"unit\",\"tests\":{\"unit\":42}},{\"artifact\":\"nats:program\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"home-server\",\"outcome\":\"failed\",\"seat\":\"node-build-go\",\"status\":\"optional\",\"step\":\"integration\",\"tests\":{\"integration\":3},\"why\":\"TestTheBusWhole failed\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"seat\":\"mesh-wants\",\"status\":\"required\",\"step\":\"composition\"},{\"module\":\"nats\",\"outcome\":\"bypassed\",\"status\":\"bypass\",\"step\":\"review\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"status\":\"required\",\"step\":\"approval\"}],\"v\":1}", wire: "{\"branch\":\"main\",\"call\":\"0123456789abcdef\",\"end-count\":1,\"ended\":\"2026-11-10T10:20:00Z\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":1}],\"outcome\":\"passed\",\"pipeline\":\"main.3589\",\"revision\":12,\"source\":3589,\"started\":\"2026-11-10T10:00:00Z\",\"steps\":[{\"artifact\":\"nats:program\",\"ask\":\"m2kx9a1b3c4d5e\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"anchor\",\"outcome\":\"passed\",\"seat\":\"node-build-go\",\"status\":\"required\",\"step\":\"lint\"},{\"artifact\":\"nats:program\",\"carried-from\":\"main.3588\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"outcome\":\"carried\",\"status\":\"required\",\"step\":\"unit\",\"tests\":{\"unit\":42}},{\"artifact\":\"nats:program\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"home-server\",\"outcome\":\"failed\",\"seat\":\"node-build-go\",\"status\":\"optional\",\"step\":\"integration\",\"tests\":{\"integration\":3},\"why\":\"TestTheBusWhole failed\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"seat\":\"mesh-wants\",\"status\":\"required\",\"step\":\"composition\"},{\"module\":\"nats\",\"outcome\":\"bypassed\",\"status\":\"bypass\",\"step\":\"review\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"status\":\"required\",\"step\":\"approval\"}],\"v\":1}",
decode: func(b []byte) (any, error) {
var v PipelinesByBranchRevisionValue
var v PipelinesByBranchSourceValue
err := json.Unmarshal(b, &v)
return v, err
}, subjectIs: nil},
@@ -73,13 +73,13 @@ func TestFixtures(t *testing.T) {
}, subjectIs: nil},
{name: "a pipeline waiting its turn, its seq given", subject: "", given: "{\"branch\":\"main\",\"call\":\"fedcba9876543210\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0124\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":1}],\"outcome\":\"waiting\",\"pipeline\":\"main.3590\",\"revision\":13,\"seq\":2,\"source\":3590,\"steps\":[],\"v\":1}", wire: "{\"branch\":\"main\",\"call\":\"fedcba9876543210\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0124\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":1}],\"outcome\":\"waiting\",\"pipeline\":\"main.3590\",\"revision\":13,\"seq\":2,\"source\":3590,\"steps\":[],\"v\":1}",
decode: func(b []byte) (any, error) {
var v PipelinesByBranchRevisionValue
var v PipelinesByBranchSourceValue
err := json.Unmarshal(b, &v)
return v, err
}, subjectIs: nil},
{name: "a waiting pipeline names the movers a newer take-in took out of it", subject: "", given: "{\"branch\":\"main\",\"call\":\"fedcba9876543210\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0124\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":1}],\"outcome\":\"waiting\",\"pipeline\":\"main.3590\",\"revision\":13,\"seq\":2,\"source\":3590,\"steps\":[],\"superseded\":[\"mesh-config\"],\"v\":1}", wire: "{\"branch\":\"main\",\"call\":\"fedcba9876543210\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0124\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":1}],\"outcome\":\"waiting\",\"pipeline\":\"main.3590\",\"revision\":13,\"seq\":2,\"source\":3590,\"steps\":[],\"superseded\":[\"mesh-config\"],\"v\":1}",
decode: func(b []byte) (any, error) {
var v PipelinesByBranchRevisionValue
var v PipelinesByBranchSourceValue
err := json.Unmarshal(b, &v)
return v, err
}, subjectIs: nil},
@@ -83,9 +83,9 @@ const SubjectWantedAll = "mesh.seat.mesh-wants.event.wanted.*.*"
// StreamPipelineWanted is the stream pipeline-wanted is kept in.
const StreamPipelineWanted = "SEAT_mesh-wants_pipeline-wanted"
// SubjectPipelineWanted is where pipeline-wanted is published, on mesh.seat.mesh-wants.event.pipeline-wanted.<branch>.<revision>.
func SubjectPipelineWanted(branch, revision string) string {
return "mesh.seat.mesh-wants.event.pipeline-wanted." + branch + "." + revision
// SubjectPipelineWanted is where pipeline-wanted is published, on mesh.seat.mesh-wants.event.pipeline-wanted.<branch>.<source>.
func SubjectPipelineWanted(branch, source string) string {
return "mesh.seat.mesh-wants.event.pipeline-wanted." + branch + "." + source
}
// SubjectPipelineWantedAll hears every pipeline-wanted.
@@ -1001,12 +1001,10 @@ type PipelineWanted struct {
// for tier-late, the late tier's movers not running and the machines they do not run on
NotRunning []NotRunning `json:"not-running,omitempty"`
Outcome string `json:"outcome"`
// the runner's pipeline key, <branch>.<revision>, a branch escaped as a key's name is (design 47 H §746, H-A-213 (4))
// the runner's pipeline key, <branch>.<source>, a branch escaped as a key's name is (design 47 H §746)
Pipeline string `json:"pipeline"`
// the pipeline's catalog revision number, the number of its key; the indexer frees the pins of that revision, not of another revision of the same source (design 47 H-A-214); absent from an older event
Revision int64 `json:"revision,omitempty"`
Source int64 `json:"source"`
V int64 `json:"v"`
Source int64 `json:"source"`
V int64 `json:"v"`
}
// The values of PipelineWanted's outcome.
@@ -29,7 +29,7 @@ func TestFixtures(t *testing.T) {
decode: func(b []byte) (any, error) { var v WantedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-wants\\.event\\.wanted\\.[^.*> ]+\\.[^.*> ]+$")},
{name: "a want removed, its node left or its module unplaced", subject: "mesh.seat.mesh-wants.event.wanted.n1.xclip", given: "{\"at\":\"2026-11-09T19:05:00Z\",\"fp\":\"src2:b427bdde1f100e2f52490cfa9df0c5112dd3700226b4892349f0c14a15a4af56\",\"module\":\"xclip\",\"node\":\"n1\",\"removed\":true,\"revision\":41,\"v\":1}", wire: "{\"at\":\"2026-11-09T19:05:00Z\",\"fp\":\"src2:b427bdde1f100e2f52490cfa9df0c5112dd3700226b4892349f0c14a15a4af56\",\"module\":\"xclip\",\"node\":\"n1\",\"removed\":true,\"revision\":41,\"v\":1}",
decode: func(b []byte) (any, error) { var v WantedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-wants\\.event\\.wanted\\.[^.*> ]+\\.[^.*> ]+$")},
{name: "a pipeline's wants all run", subject: "mesh.seat.mesh-wants.event.pipeline-wanted.main.3590", given: "{\"branch\":\"main\",\"outcome\":\"running\",\"pipeline\":\"main.3590\",\"revision\":3590,\"source\":3589,\"v\":1}", wire: "{\"branch\":\"main\",\"outcome\":\"running\",\"pipeline\":\"main.3590\",\"revision\":3590,\"source\":3589,\"v\":1}",
{name: "a pipeline's wants all run", subject: "mesh.seat.mesh-wants.event.pipeline-wanted.main.3589", given: "{\"branch\":\"main\",\"outcome\":\"running\",\"pipeline\":\"main.3589\",\"source\":3589,\"v\":1}", wire: "{\"branch\":\"main\",\"outcome\":\"running\",\"pipeline\":\"main.3589\",\"source\":3589,\"v\":1}",
decode: func(b []byte) (any, error) { var v PipelineWantedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-wants\\.event\\.pipeline-wanted\\.[^.*> ]+\\.[^.*> ]+$")},
{name: "a pipeline's tier late, naming the movers and machines not running", subject: "mesh.seat.mesh-wants.event.pipeline-wanted.main.3590", given: "{\"branch\":\"main\",\"not-running\":[{\"machines\":[\"anchor\",\"laptop\"],\"module\":\"nats\"}],\"outcome\":\"tier-late\",\"pipeline\":\"main.3590\",\"source\":3590,\"v\":1}", wire: "{\"branch\":\"main\",\"not-running\":[{\"machines\":[\"anchor\",\"laptop\"],\"module\":\"nats\"}],\"outcome\":\"tier-late\",\"pipeline\":\"main.3590\",\"source\":3590,\"v\":1}",
decode: func(b []byte) (any, error) { var v PipelineWantedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-wants\\.event\\.pipeline-wanted\\.[^.*> ]+\\.[^.*> ]+$")},
@@ -905,10 +905,8 @@ type StepRun struct {
Pipeline string `json:"pipeline"`
// the step is asked again by the runner's rerun
Rerun bool `json:"rerun,omitempty"`
// the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request
// the source revision of the watched branch the pipeline runs for; absent for a run on request
Revision int64 `json:"revision,omitempty"`
// the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request
Source int64 `json:"source,omitempty"`
// the pipeline step
Step string `json:"step"`
Toolchain *ToolchainUsed `json:"toolchain,omitempty"`
@@ -903,10 +903,8 @@ type StepRun struct {
Pipeline string `json:"pipeline"`
// the step is asked again by the runner's rerun
Rerun bool `json:"rerun,omitempty"`
// the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request
// the source revision of the watched branch the pipeline runs for; absent for a run on request
Revision int64 `json:"revision,omitempty"`
// the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request
Source int64 `json:"source,omitempty"`
// the pipeline step
Step string `json:"step"`
Toolchain *ToolchainUsed `json:"toolchain,omitempty"`
@@ -903,10 +903,8 @@ type StepRun struct {
Pipeline string `json:"pipeline"`
// the step is asked again by the runner's rerun
Rerun bool `json:"rerun,omitempty"`
// the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request
// the source revision of the watched branch the pipeline runs for; absent for a run on request
Revision int64 `json:"revision,omitempty"`
// the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request
Source int64 `json:"source,omitempty"`
// the pipeline step
Step string `json:"step"`
Toolchain *ToolchainUsed `json:"toolchain,omitempty"`
@@ -1521,11 +1521,7 @@
},
"revision": {
"type": "integer",
"description": "the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request"
},
"source": {
"type": "integer",
"description": "the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request"
"description": "the source revision of the watched branch the pipeline runs for; absent for a run on request"
},
"fingerprint": {
"type": "string",
@@ -180,10 +180,6 @@
"type": "integer",
"description": "the source revision of that branch the build was made from"
},
"revision": {
"type": "integer",
"description": "the catalog revision number of that branch the build was made from, the indexer's count (design 47 H-A-218, H-A-220); absent from a reservation made before the field"
},
"dependencies": {
"type": "array",
"items": {
@@ -367,13 +363,6 @@
"$ref": "#/defs/BuildEntry"
},
"description": "each entry the step built, by module and fingerprint"
},
"packages": {
"type": "object",
"additionalProperties": {
"type": "string"
},
"description": "the pins the step's entries were built with, each mesh package's version by its module path, the ones the builder took from the registry for a package the entry named with none included; the runner asks the entry's held test steps with them (H-A-207)"
}
},
"required": [
@@ -8,10 +8,6 @@
"revision": 12,
"kind": "trunk",
"source": 3589,
"call": "release of mesh-sdk 0.1.27",
"call-kind": "release",
"module": "mesh-sdk",
"version": "0.1.27",
"entries": 2,
"changed": [
"xclip"
@@ -209,11 +209,7 @@
"repository": "novox/mesh-catalog"
}
],
"base": 9,
"call": "release of mesh-sdk 0.1.27",
"call-kind": "release",
"module": "mesh-sdk",
"version": "0.1.27"
"base": 9
}
},
{
@@ -657,32 +657,6 @@
"source": {
"type": "integer"
},
"call": {
"type": "string",
"description": "what made the revision, as the revision's call names it (design 47 H-A-212, H-A-213); absent for a candidate's"
},
"call-kind": {
"type": "string",
"enum": [
"take-in",
"release",
"take-in-again",
"recompute"
],
"description": "the kind of what made the revision, which a reader reads instead of call's words (design 47 H-A-213 (3), H-A-214, H-A-215, H-A-217); absent for a candidate's"
},
"module": {
"type": "string",
"description": "for call-kind release, the module released (H-A-215)"
},
"version": {
"type": "string",
"description": "for call-kind release, the version released (H-A-215)"
},
"repository": {
"type": "string",
"description": "for call-kind take-in-again, the repository taken in again (H-A-215)"
},
"group": {
"type": "string"
},
@@ -1897,32 +1871,6 @@
"type": "integer",
"description": "the source revision it was taken in from; for a candidate, the source revision its heads are merged onto"
},
"call": {
"type": "string",
"description": "what made the revision (design 47 H-A-212, H-A-213): a take-in's the gate's call its source revision names, a re-pin's release of <module> <version>, a take-in again's take-in again of <repository>; absent for a candidate's"
},
"call-kind": {
"type": "string",
"enum": [
"take-in",
"release",
"take-in-again",
"recompute"
],
"description": "the kind of what made the revision, which a reader reads instead of call's words (design 47 H-A-213 (3), H-A-214, H-A-215, H-A-217): a take-in's, a re-pin's on a module's release, a take-in again's, a recompute's that moved a fingerprint (its call naming the reason); absent for a candidate's"
},
"module": {
"type": "string",
"description": "for call-kind release, the module released (H-A-215)"
},
"version": {
"type": "string",
"description": "for call-kind release, the version released (H-A-215)"
},
"repository": {
"type": "string",
"description": "for call-kind take-in-again, the repository taken in again (H-A-215)"
},
"group": {
"type": "string",
"description": "a candidate's group"
@@ -1,7 +1,7 @@
[
{
"name": "a take-in's pipeline kept, its steps naming their mover",
"of": "state:pipelines/<branch>.<revision>",
"of": "state:pipelines/<branch>.<source>",
"given": {
"pipeline": "main.3589",
"branch": "main",
@@ -200,7 +200,7 @@
},
{
"name": "a pipeline waiting its turn, its seq given",
"of": "state:pipelines/<branch>.<revision>",
"of": "state:pipelines/<branch>.<source>",
"given": {
"pipeline": "main.3590",
"branch": "main",
@@ -224,7 +224,7 @@
},
{
"name": "a waiting pipeline names the movers a newer take-in took out of it",
"of": "state:pipelines/<branch>.<revision>",
"of": "state:pipelines/<branch>.<source>",
"given": {
"pipeline": "main.3590",
"branch": "main",
@@ -86,17 +86,13 @@
},
{
"name": "show",
"description": "One pipeline whole: its record with every mover's steps, or one mover's steps alone when it names the mover, its step log and its outputs, with the stores' fill. Reads only.",
"description": "One pipeline whole: its steps, its step log and its outputs, with the stores' fill. Reads only.",
"input": {
"type": "object",
"properties": {
"pipeline": {
"type": "string",
"description": "the pipeline's key"
},
"mover": {
"type": "string",
"description": "a mover's module: the answer's pipeline then holds that mover's steps alone (H-A-187)"
}
},
"required": [
@@ -137,7 +133,7 @@
},
{
"name": "list",
"description": "The pipelines and runs of a module, a branch or a commit, newest first, with the store's fill: each pipeline's record with its movers and steps counted, never its steps (show gives those). Reads only.",
"description": "The pipelines and runs of a module, a branch or a commit, newest first, with the store's fill. Reads only.",
"input": {
"type": "object",
"properties": {
@@ -672,7 +668,7 @@
"events": [
{
"name": "pipeline-started",
"subject": "pipeline-started.<branch>.<revision>",
"subject": "pipeline-started.<branch>.<source>",
"body": {
"$ref": "#/defs/PipelineStarted"
},
@@ -683,7 +679,7 @@
},
{
"name": "pipeline-ended",
"subject": "pipeline-ended.<branch>.<revision>",
"subject": "pipeline-ended.<branch>.<source>",
"body": {
"$ref": "#/defs/PipelineEnded"
},
@@ -727,7 +723,7 @@
},
{
"name": "wait-held",
"subject": "wait-held.<branch>.<revision>",
"subject": "wait-held.<branch>.<source>",
"body": {
"$ref": "#/defs/Wait"
},
@@ -738,7 +734,7 @@
},
{
"name": "wait-ended",
"subject": "wait-ended.<branch>.<revision>",
"subject": "wait-ended.<branch>.<source>",
"body": {
"$ref": "#/defs/Wait"
},
@@ -775,14 +771,14 @@
"name": "pipelines",
"template": "state",
"history": 1,
"description": "one key per pipeline, one per take-in, written by compare-and-set on its absence (design 47 H §746, §747), and one key per mover of a pipeline holding that mover's steps (H-A-187)",
"description": "one key per pipeline, one per take-in, written by compare-and-set on its absence (design 47 H §746, §747)",
"key-forms": [
{
"key": "<branch>.<revision>",
"key": "<branch>.<source>",
"value": {
"$ref": "#/defs/Pipeline"
},
"description": "a pipeline of one catalog revision, keyed by the revision's number (design 47 H-A-213 (4), H-A-219); one kept from before under its source has the same form"
"description": "a take-in's pipeline"
},
{
"key": "<branch>.<source>.published.<package>.<version>",
@@ -797,32 +793,6 @@
"$ref": "#/defs/Pipeline"
},
"description": "a run on request's pipeline, one per run, its movers the named commits' modules; it delivers nothing (H-A-83)"
},
{
"key": "steps.<pipeline>.<mover>",
"value": {
"type": "object",
"properties": {
"steps": {
"type": "array",
"items": {
"$ref": "#/defs/StepResult"
},
"description": "the mover's steps"
},
"unlogged": {
"type": "array",
"items": {
"$ref": "#/defs/LogEntry"
},
"description": "the step-log entries of the mover's steps that ended and are not appended to the step log yet, in order (H-A-189)"
}
},
"required": [
"steps"
]
},
"description": "one mover's steps and unlogged step-log entries in a pipeline, its key the pipeline's and the mover's module escaped as a key's name, written by compare-and-set as the pipeline's record is; the record's own steps stay empty, so a record never grows with the catalog (H-A-187)"
}
]
},
@@ -1199,7 +1169,7 @@
"properties": {
"pipeline": {
"type": "string",
"description": "the pipeline's key, <branch>.<revision>, <branch>.<source>.published.<package>.<version> for a pipeline a publish started, or <branch>.run.<run> for a run on request; a branch escaped as a key's name is (design 47 H §746, §747; H-A-83)"
"description": "the pipeline's key, <branch>.<source>, <branch>.<source>.published.<package>.<version> for a pipeline a publish started, or <branch>.run.<run> for a run on request; a branch escaped as a key's name is (design 47 H §746, §747; H-A-83)"
},
"branch": {
"type": "string",
@@ -1268,7 +1238,7 @@
"items": {
"$ref": "#/defs/StepResult"
},
"description": "every mover's steps, each naming its mover, approval last; in the state pipelines a record's own are empty, each mover's kept under its own key (H-A-187)"
"description": "every mover's steps, each naming its mover, approval last"
},
"started": {
"type": "string",
@@ -1283,15 +1253,7 @@
"items": {
"$ref": "#/defs/LogEntry"
},
"description": "the step-log entries of the steps that ended and are not appended to the step log yet, in order; written with the step's end in one write and cleared when appended (design 47 G sections 486, 606); in the state pipelines a record's own are empty, each mover's kept under its own key beside its steps (H-A-189)"
},
"mover-count": {
"type": "integer",
"description": "how many movers the pipeline has, set by list alone (H-A-191)"
},
"step-count": {
"type": "integer",
"description": "how many steps the pipeline's movers have, set by list alone, which answers the record with its steps empty (H-A-191)"
"description": "the step-log entries of the steps that ended and are not appended to the step log yet, in order; written with the step's end in one write and cleared when appended (design 47 G sections 486, 606)"
},
"published": {
"$ref": "#/defs/Published",
@@ -1313,7 +1275,7 @@
"properties": {
"pipeline": {
"type": "string",
"description": "the pipeline's key, <branch>.<revision>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)"
"description": "the pipeline's key, <branch>.<source>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)"
},
"branch": {
"type": "string",
@@ -1388,7 +1350,7 @@
"properties": {
"pipeline": {
"type": "string",
"description": "the pipeline's key, <branch>.<revision>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)"
"description": "the pipeline's key, <branch>.<source>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)"
},
"branch": {
"type": "string",
@@ -1495,7 +1457,7 @@
"properties": {
"pipeline": {
"type": "string",
"description": "the pipeline's key, <branch>.<revision>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)"
"description": "the pipeline's key, <branch>.<source>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)"
},
"module": {
"type": "string"
@@ -1550,7 +1512,7 @@
"properties": {
"pipeline": {
"type": "string",
"description": "the pipeline's key, <branch>.<revision>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)"
"description": "the pipeline's key, <branch>.<source>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)"
},
"module": {
"type": "string",
@@ -1635,7 +1597,7 @@
"description": "the run the step waits on, where the runner knows it: a test step's run key and ask (design 47 G §731, A-469v)"
}
},
"description": "a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283); said on <branch>.<revision> (design 47 H-A-72, H-A-221)",
"description": "a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283); said on <branch>.<source> (design 47 H-A-72)",
"required": [
"v",
"pipeline",
@@ -1651,10 +1613,6 @@
"pipeline": {
"type": "string"
},
"module": {
"type": "string",
"description": "the module of the mover whose step the entry is of (H-A-190)"
},
"step": {
"type": "string",
"enum": [
@@ -1978,7 +1936,7 @@
"properties": {
"pipeline": {
"type": "string",
"description": "the pipeline's key, <branch>.<revision>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)"
"description": "the pipeline's key, <branch>.<source>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)"
}
},
"description": "a pipeline waiting its turn under pipelines-at-once (design 47 H §748, H-A-72)",
@@ -65,13 +65,12 @@
{
"name": "a pipeline's wants all run",
"of": "event:pipeline-wanted",
"subject": "mesh.seat.mesh-wants.event.pipeline-wanted.main.3590",
"subject": "mesh.seat.mesh-wants.event.pipeline-wanted.main.3589",
"given": {
"v": 1,
"pipeline": "main.3590",
"pipeline": "main.3589",
"branch": "main",
"source": 3589,
"revision": 3590,
"outcome": "running"
}
},
@@ -434,7 +434,7 @@
},
{
"name": "pipeline-wanted",
"subject": "pipeline-wanted.<branch>.<revision>",
"subject": "pipeline-wanted.<branch>.<source>",
"keep": {
"max-age-days": 30,
"max-mib": 64
@@ -1725,7 +1725,7 @@
},
"pipeline": {
"type": "string",
"description": "the runner's pipeline key, <branch>.<revision>, a branch escaped as a key's name is (design 47 H §746, H-A-213 (4))"
"description": "the runner's pipeline key, <branch>.<source>, a branch escaped as a key's name is (design 47 H §746)"
},
"branch": {
"type": "string"
@@ -1733,10 +1733,6 @@
"source": {
"type": "integer"
},
"revision": {
"type": "integer",
"description": "the pipeline's catalog revision number, the number of its key; the indexer frees the pins of that revision, not of another revision of the same source (design 47 H-A-214); absent from an older event"
},
"outcome": {
"type": "string",
"enum": [
@@ -69,8 +69,6 @@ export interface PackagePublished {
"digest": string;
/** the module */
"module": string;
/** the catalog revision number of that branch the build was made from, the indexer's count (design 47 H-A-218, H-A-220); absent from a reservation made before the field */
"revision"?: number;
/** the source revision of that branch the build was made from */
"source": number;
"v": number;
@@ -91,8 +89,6 @@ export interface StepEnded {
"node": string;
/** never passed for a build that built nothing */
"outcome": "passed" | "failed" | "nothing to run" | "could not run";
/** the pins the step's entries were built with, each mesh package's version by its module path, the ones the builder took from the registry for a package the entry named with none included; the runner asks the entry's held test steps with them (H-A-207) */
"packages"?: Record<string, string>;
"pipeline": string;
"rerun"?: boolean;
/** why it failed or could not run: the entry not built, the part not heard */
@@ -277,10 +277,6 @@ export interface CatalogRevision {
"base"?: number;
/** the watched branch the revision is of */
"branch"?: string;
/** what made the revision (design 47 H-A-212, H-A-213): a take-in's the gate's call its source revision names, a re-pin's release of <module> <version>, a take-in again's take-in again of <repository>; absent for a candidate's */
"call"?: string;
/** the kind of what made the revision, which a reader reads instead of call's words (design 47 H-A-213 (3), H-A-214, H-A-215, H-A-217): a take-in's, a re-pin's on a module's release, a take-in again's, a recompute's that moved a fingerprint (its call naming the reason); absent for a candidate's */
"call-kind"?: "take-in" | "release" | "take-in-again" | "recompute";
/** the seat definitions it was judged with, each by its digest, read whole through seat-definitions */
"definitions"?: CatalogRevisionDefinitionsItem[];
/** one per module, by name */
@@ -291,12 +287,8 @@ export interface CatalogRevision {
"heads"?: string;
/** trunk or candidate */
"kind": string;
/** for call-kind release, the module released (H-A-215) */
"module"?: string;
/** the movers of the take-in that wrote it, so a redelivered source-moved whose revision is kept says take-in.taken again from them, recomputing nothing (design 47 H §743, H-A-92); absent for one with no mover */
"movers"?: TakeInEntry[];
/** for call-kind take-in-again, the repository taken in again (H-A-215) */
"repository"?: string;
/** the catalog graph revision's number, counted by its holder */
"revision": number;
/** the source revision it was taken in from; for a candidate, the source revision its heads are merged onto */
@@ -305,8 +297,6 @@ export interface CatalogRevision {
"trunk"?: Head[];
/** the layout version */
"v"?: number;
/** for call-kind release, the version released (H-A-215) */
"version"?: string;
}
/** one edge of the catalog graph between artifact references, B before A (ADR 0666 section 14): kind 1 B published before A builds (a package A imports), 2 B published before A delivers (fetched by reference at its start), 3 B delivered before A builds (its build module and toolchain), 4 B delivered before A delivers (a provision, bus state, a format's reader), 5 B has acted for A's new version (an acting rule), 6 B runs a version that serves A (a verb or a manifest word A uses) */
@@ -806,10 +796,6 @@ export interface CatalogRevisionEvent {
"at"?: string;
/** the watched branch the revision is of, escaped as a key name is */
"branch"?: string;
/** what made the revision, as the revision's call names it (design 47 H-A-212, H-A-213); absent for a candidate's */
"call"?: string;
/** the kind of what made the revision, which a reader reads instead of call's words (design 47 H-A-213 (3), H-A-214, H-A-215, H-A-217); absent for a candidate's */
"call-kind"?: "take-in" | "release" | "take-in-again" | "recompute";
/** the modules whose entries differ from the revision before it of the same kind */
"changed"?: string[];
/** how many entries it lists */
@@ -818,15 +804,9 @@ export interface CatalogRevisionEvent {
"heads"?: string;
/** trunk or candidate: a candidate's moves no running part */
"kind": string;
/** for call-kind release, the module released (H-A-215) */
"module"?: string;
/** for call-kind take-in-again, the repository taken in again (H-A-215) */
"repository"?: string;
"revision": number;
"source": number;
"v": number;
/** for call-kind release, the version released (H-A-215) */
"version"?: string;
}
/** is the body of the event filling. */
@@ -40,13 +40,13 @@ export declare const family: { readonly name: "mesh-pipeline-runner.events"; rea
export declare const pipelineStartedStream: "SEAT_mesh-pipeline-runner_pipeline-started";
export declare function pipelineStartedSubject(branch: string, revision: string): string;
export declare function pipelineStartedSubject(branch: string, source: string): string;
export declare const pipelineStartedSubjectAll: "mesh.seat.mesh-pipeline-runner.event.pipeline-started.*.*";
export declare const pipelineEndedStream: "SEAT_mesh-pipeline-runner_pipeline-ended";
export declare function pipelineEndedSubject(branch: string, revision: string): string;
export declare function pipelineEndedSubject(branch: string, source: string): string;
export declare const pipelineEndedSubjectAll: "mesh.seat.mesh-pipeline-runner.event.pipeline-ended.*.*";
@@ -70,13 +70,13 @@ export declare const stepEndedSubjectAll: "mesh.seat.mesh-pipeline-runner.event.
export declare const waitHeldStream: "SEAT_mesh-pipeline-runner_wait-held";
export declare function waitHeldSubject(branch: string, revision: string): string;
export declare function waitHeldSubject(branch: string, source: string): string;
export declare const waitHeldSubjectAll: "mesh.seat.mesh-pipeline-runner.event.wait-held.*.*";
export declare const waitEndedStream: "SEAT_mesh-pipeline-runner_wait-ended";
export declare function waitEndedSubject(branch: string, revision: string): string;
export declare function waitEndedSubject(branch: string, source: string): string;
export declare const waitEndedSubjectAll: "mesh.seat.mesh-pipeline-runner.event.wait-ended.*.*";
@@ -94,11 +94,11 @@ export declare const states: {
readonly stepOutputs: "step-outputs";
};
/** The key of one value of the state pipelines under the form "<branch>.<revision>". */
export declare function pipelinesByBranchRevisionKey(branch: string, revision: string): string;
/** The key of one value of the state pipelines under the form "<branch>.<source>". */
export declare function pipelinesByBranchSourceKey(branch: string, source: string): string;
/** The parts of a key of the form "<branch>.<revision>", or undefined for a key of another form. */
export declare function parsePipelinesByBranchRevisionKey(_key: string): { branch: string; revision: string } | undefined;
/** The parts of a key of the form "<branch>.<source>", or undefined for a key of another form. */
export declare function parsePipelinesByBranchSourceKey(_key: string): { branch: string; source: string } | undefined;
/** The key of one value of the state pipelines under the form "<branch>.<source>.published.<package>.<version>". */
export declare function pipelinesByBranchSourcePublishedPackageVersionKey(branch: string, source: string, packagePart: string, version: string): string;
@@ -112,12 +112,6 @@ export declare function pipelinesByBranchRunRunKey(branch: string, run: string):
/** The parts of a key of the form "<branch>.run.<run>", or undefined for a key of another form. */
export declare function parsePipelinesByBranchRunRunKey(_key: string): { branch: string; run: string } | undefined;
/** The key of one value of the state pipelines under the form "steps.<pipeline>.<mover>". */
export declare function pipelinesByStepsPipelineMoverKey(pipeline: string, mover: string): string;
/** The parts of a key of the form "steps.<pipeline>.<mover>", or undefined for a key of another form. */
export declare function parsePipelinesByStepsPipelineMoverKey(_key: string): { pipeline: string; mover: string } | undefined;
/** The key of one value of the state queue under the form "queue.next". */
export declare function queueByQueueNextKey(): string;
@@ -192,8 +186,6 @@ export interface LogEntry {
"at": string;
/** the keys of the kept results the step wrote or carried, <level>.<digest>, which the runner ages by their last-read (design 47 G sections 330, 470) */
"kept"?: string[];
/** the module of the mover whose step the entry is of (H-A-190) */
"module"?: string;
"outcome"?: "running" | "waiting" | "passed" | "failed" | "nothing to run" | "bypassed" | "carried" | "could not run" | "waiting for a build node" | "did not build" | "accepted" | "declined" | "superseded" | "cancelled";
/** the output's name in step-outputs */
"output"?: string;
@@ -256,12 +248,10 @@ export interface Pipeline {
/** how often the pipeline has ended: 1 at its first end, one higher at each end again after a passing rerun, so the assigner takes a later end of the same source revision (design 47 G sections 29, 325) */
"end-count"?: number;
"ended"?: string;
/** how many movers the pipeline has, set by list alone (H-A-191) */
"mover-count"?: number;
/** the modules the pipeline moves, its take-in's entries (design 47 H §746) */
"movers": Mover[];
"outcome": "waiting" | "running" | "passed" | "failed" | "refused" | "superseded" | "cancelled";
/** the pipeline's key, <branch>.<revision>, <branch>.<source>.published.<package>.<version> for a pipeline a publish started, or <branch>.run.<run> for a run on request; a branch escaped as a key's name is (design 47 H §746, §747; H-A-83) */
/** the pipeline's key, <branch>.<source>, <branch>.<source>.published.<package>.<version> for a pipeline a publish started, or <branch>.run.<run> for a run on request; a branch escaped as a key's name is (design 47 H §746, §747; H-A-83) */
"pipeline": string;
/** for a pipeline a package's publish started, that publish (A-202) */
"published"?: Published;
@@ -274,13 +264,11 @@ export interface Pipeline {
/** the source revision of the watched branch; the pipeline is keyed by it (ADR 0666 section 9) */
"source": number;
"started"?: string;
/** how many steps the pipeline's movers have, set by list alone, which answers the record with its steps empty (H-A-191) */
"step-count"?: number;
/** every mover's steps, each naming its mover, approval last; in the state pipelines a record's own are empty, each mover's kept under its own key (H-A-187) */
/** every mover's steps, each naming its mover, approval last */
"steps": StepResult[];
/** the modules a newer waiting take-in of the same branch took out of the pipeline, written with the movers by compare-and-set; at its placing the runner writes a mover's build-step why from it (design 47 H §748, H-A-107) */
"superseded"?: string[];
/** the step-log entries of the steps that ended and are not appended to the step log yet, in order; written with the step's end in one write and cleared when appended (design 47 G sections 486, 606); in the state pipelines a record's own are empty, each mover's kept under its own key beside its steps (H-A-189) */
/** the step-log entries of the steps that ended and are not appended to the step log yet, in order; written with the step's end in one write and cleared when appended (design 47 G sections 486, 606) */
"unlogged"?: LogEntry[];
/** the layout version */
"v"?: number;
@@ -300,7 +288,7 @@ export interface PipelineEnded {
/** per mover its fingerprint, released version, step results and tier, in the delivery order (design 47 H §746, H-A-2) */
"movers": MoverEnded[];
"outcome": "waiting" | "running" | "passed" | "failed" | "refused" | "superseded" | "cancelled";
/** the pipeline's key, <branch>.<revision>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */
/** the pipeline's key, <branch>.<source>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */
"pipeline": string;
/** the catalog graph revision of the branch the pipeline runs for */
"revision"?: number;
@@ -322,7 +310,7 @@ export interface PipelineStarted {
"call"?: string;
/** the pipeline's movers */
"movers": Mover[];
/** the pipeline's key, <branch>.<revision>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */
/** the pipeline's key, <branch>.<source>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */
"pipeline": string;
/** for a pipeline a package's publish started, that publish (A-202) */
"published"?: Published;
@@ -383,7 +371,7 @@ export interface StepEvent {
/** the module's build-input fingerprint at the commit */
"fingerprint"?: string;
"module": string;
/** the pipeline's key, <branch>.<revision>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */
/** the pipeline's key, <branch>.<source>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */
"pipeline": string;
"result": StepResult;
/** the catalog graph revision of the branch the pipeline runs for */
@@ -490,7 +478,7 @@ export interface ToolchainUsed {
"version": string;
}
/** a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283); said on <branch>.<revision> (design 47 H-A-72, H-A-221) */
/** a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283); said on <branch>.<source> (design 47 H-A-72) */
export interface Wait {
"artifact"?: string;
/** the step's own ask, where it has one (design 47 G §731) */
@@ -507,7 +495,7 @@ export interface Wait {
"module"?: string;
/** what the step waits on (design 47 *Health and going back*) */
"on": "person" | "build node" | "revision view" | "pipelines-at-once";
/** the pipeline's key, <branch>.<revision>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */
/** the pipeline's key, <branch>.<source>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */
"pipeline": string;
/** the catalog graph revision of the branch the pipeline runs for */
"revision"?: number;
@@ -527,7 +515,7 @@ export interface Wait {
/** a pipeline waiting its turn under pipelines-at-once (design 47 H §748, H-A-72) */
export interface Waiting {
/** the pipeline's key, <branch>.<revision>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */
/** the pipeline's key, <branch>.<source>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */
"pipeline": string;
}
@@ -568,8 +556,6 @@ export interface StartAnswer {
/** is what show takes. */
export interface ShowInput {
/** a mover's module: the answer's pipeline then holds that mover's steps alone (H-A-187) */
"mover"?: string;
/** the pipeline's key */
"pipeline": string;
}
@@ -800,8 +786,8 @@ export type FullEvent = Fill;
/** is the body of the event room. */
export type RoomEvent = Fill;
/** is one value of the state pipelines under the key <branch>.<revision>. */
export type PipelinesByBranchRevisionValue = Pipeline;
/** is one value of the state pipelines under the key <branch>.<source>. */
export type PipelinesByBranchSourceValue = Pipeline;
/** is one value of the state pipelines under the key <branch>.<source>.published.<package>.<version>. */
export type PipelinesByBranchSourcePublishedPackageVersionValue = Pipeline;
@@ -809,14 +795,6 @@ export type PipelinesByBranchSourcePublishedPackageVersionValue = Pipeline;
/** is one value of the state pipelines under the key <branch>.run.<run>. */
export type PipelinesByBranchRunRunValue = Pipeline;
/** is one value of the state pipelines under the key steps.<pipeline>.<mover>. */
export interface PipelinesByStepsPipelineMoverValue {
/** the mover's steps */
"steps": StepResult[];
/** the step-log entries of the mover's steps that ended and are not appended to the step log yet, in order (H-A-189) */
"unlogged"?: LogEntry[];
}
/** is one value of the state queue under the key queue.next. */
export type QueueByQueueNextValue = number;
@@ -40,16 +40,16 @@ export const family = { name: "mesh-pipeline-runner.events", maxAgeDays: 7 };
export const pipelineStartedStream = "SEAT_mesh-pipeline-runner_pipeline-started";
export function pipelineStartedSubject(branch, revision) {
return "mesh.seat.mesh-pipeline-runner.event.pipeline-started." + branch + "." + revision;
export function pipelineStartedSubject(branch, source) {
return "mesh.seat.mesh-pipeline-runner.event.pipeline-started." + branch + "." + source;
}
export const pipelineStartedSubjectAll = "mesh.seat.mesh-pipeline-runner.event.pipeline-started.*.*";
export const pipelineEndedStream = "SEAT_mesh-pipeline-runner_pipeline-ended";
export function pipelineEndedSubject(branch, revision) {
return "mesh.seat.mesh-pipeline-runner.event.pipeline-ended." + branch + "." + revision;
export function pipelineEndedSubject(branch, source) {
return "mesh.seat.mesh-pipeline-runner.event.pipeline-ended." + branch + "." + source;
}
export const pipelineEndedSubjectAll = "mesh.seat.mesh-pipeline-runner.event.pipeline-ended.*.*";
@@ -80,16 +80,16 @@ export const stepEndedSubjectAll = "mesh.seat.mesh-pipeline-runner.event.step-en
export const waitHeldStream = "SEAT_mesh-pipeline-runner_wait-held";
export function waitHeldSubject(branch, revision) {
return "mesh.seat.mesh-pipeline-runner.event.wait-held." + branch + "." + revision;
export function waitHeldSubject(branch, source) {
return "mesh.seat.mesh-pipeline-runner.event.wait-held." + branch + "." + source;
}
export const waitHeldSubjectAll = "mesh.seat.mesh-pipeline-runner.event.wait-held.*.*";
export const waitEndedStream = "SEAT_mesh-pipeline-runner_wait-ended";
export function waitEndedSubject(branch, revision) {
return "mesh.seat.mesh-pipeline-runner.event.wait-ended." + branch + "." + revision;
export function waitEndedSubject(branch, source) {
return "mesh.seat.mesh-pipeline-runner.event.wait-ended." + branch + "." + source;
}
export const waitEndedSubjectAll = "mesh.seat.mesh-pipeline-runner.event.wait-ended.*.*";
@@ -108,23 +108,23 @@ export const states = {
stepOutputs: "step-outputs",
};
/** The key of one value of the state pipelines under the form "<branch>.<revision>". */
export function pipelinesByBranchRevisionKey(branch, revision) {
return branch + "." + revision;
/** The key of one value of the state pipelines under the form "<branch>.<source>". */
export function pipelinesByBranchSourceKey(branch, source) {
return branch + "." + source;
}
/** The parts of a key of the form "<branch>.<revision>", or undefined for a key of another form. */
export function parsePipelinesByBranchRevisionKey(_key) {
/** The parts of a key of the form "<branch>.<source>", or undefined for a key of another form. */
export function parsePipelinesByBranchSourceKey(_key) {
let _rest = _key;
const _at0 = _rest.indexOf(".");
if (_at0 <= 0) return undefined;
const branch = _rest.slice(0, _at0);
_rest = _rest.slice(_at0 + 1);
if (_rest === "") return undefined;
const revision = _rest;
const source = _rest;
_rest = "";
if (_rest !== "") return undefined;
return { branch, revision };
return { branch, source };
}
/** The key of one value of the state pipelines under the form "<branch>.<source>.published.<package>.<version>". */
@@ -173,27 +173,6 @@ export function parsePipelinesByBranchRunRunKey(_key) {
return { branch, run };
}
/** The key of one value of the state pipelines under the form "steps.<pipeline>.<mover>". */
export function pipelinesByStepsPipelineMoverKey(pipeline, mover) {
return "steps." + pipeline + "." + mover;
}
/** The parts of a key of the form "steps.<pipeline>.<mover>", or undefined for a key of another form. */
export function parsePipelinesByStepsPipelineMoverKey(_key) {
let _rest = _key;
if (!_rest.startsWith("steps.")) return undefined;
_rest = _rest.slice(6);
const _at1 = _rest.indexOf(".");
if (_at1 <= 0) return undefined;
const pipeline = _rest.slice(0, _at1);
_rest = _rest.slice(_at1 + 1);
if (_rest === "") return undefined;
const mover = _rest;
_rest = "";
if (_rest !== "") return undefined;
return { pipeline, mover };
}
/** The key of one value of the state queue under the form "queue.next". */
export function queueByQueueNextKey() {
return "queue.next";
@@ -50,7 +50,7 @@ export declare const wantedSubjectAll: "mesh.seat.mesh-wants.event.wanted.*.*";
export declare const pipelineWantedStream: "SEAT_mesh-wants_pipeline-wanted";
export declare function pipelineWantedSubject(branch: string, revision: string): string;
export declare function pipelineWantedSubject(branch: string, source: string): string;
export declare const pipelineWantedSubjectAll: "mesh.seat.mesh-wants.event.pipeline-wanted.*.*";
@@ -412,10 +412,8 @@ export interface PipelineWanted {
/** for tier-late, the late tier's movers not running and the machines they do not run on */
"not-running"?: NotRunning[];
"outcome": "running" | "tier-late";
/** the runner's pipeline key, <branch>.<revision>, a branch escaped as a key's name is (design 47 H §746, H-A-213 (4)) */
/** the runner's pipeline key, <branch>.<source>, a branch escaped as a key's name is (design 47 H §746) */
"pipeline": string;
/** the pipeline's catalog revision number, the number of its key; the indexer frees the pins of that revision, not of another revision of the same source (design 47 H-A-214); absent from an older event */
"revision"?: number;
"source": number;
"v": number;
}
@@ -52,8 +52,8 @@ export const wantedSubjectAll = "mesh.seat.mesh-wants.event.wanted.*.*";
export const pipelineWantedStream = "SEAT_mesh-wants_pipeline-wanted";
export function pipelineWantedSubject(branch, revision) {
return "mesh.seat.mesh-wants.event.pipeline-wanted." + branch + "." + revision;
export function pipelineWantedSubject(branch, source) {
return "mesh.seat.mesh-wants.event.pipeline-wanted." + branch + "." + source;
}
export const pipelineWantedSubjectAll = "mesh.seat.mesh-wants.event.pipeline-wanted.*.*";
@@ -510,10 +510,8 @@ export interface StepRun {
"pipeline": string;
/** the step is asked again by the runner's rerun */
"rerun"?: boolean;
/** the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request */
/** the source revision of the watched branch the pipeline runs for; absent for a run on request */
"revision"?: number;
/** the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request */
"source"?: number;
/** the pipeline step */
"step": "lint" | "unit" | "integration";
"toolchain"?: ToolchainUsed;
@@ -510,10 +510,8 @@ export interface StepRun {
"pipeline": string;
/** the step is asked again by the runner's rerun */
"rerun"?: boolean;
/** the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request */
/** the source revision of the watched branch the pipeline runs for; absent for a run on request */
"revision"?: number;
/** the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request */
"source"?: number;
/** the pipeline step */
"step": "lint" | "unit" | "integration";
"toolchain"?: ToolchainUsed;
@@ -510,10 +510,8 @@ export interface StepRun {
"pipeline": string;
/** the step is asked again by the runner's rerun */
"rerun"?: boolean;
/** the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request */
/** the source revision of the watched branch the pipeline runs for; absent for a run on request */
"revision"?: number;
/** the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request */
"source"?: number;
/** the pipeline step */
"step": "lint" | "unit" | "integration";
"toolchain"?: ToolchainUsed;
@@ -187,13 +187,10 @@ func TestManifestProcess(t *testing.T) { // 1235/AC1
if _, bus := p["bus"]; bus {
t.Errorf("the process declares a bus: it runs on its own broker credential")
}
if len(m.Build.Artifacts) != 3 || m.Build.Artifacts[1].Name != "tools" || m.Build.Artifacts[1].From != "cmd/mesh-delivery-forge-bridge-tools" {
t.Fatalf("artifacts = %+v, want the program, the tools bundle and the package forgemsg", m.Build.Artifacts)
}
// ADR 0573 §1, H-A-186 (design 47 G §370): the gate, the builder and the indexer import forgemsg, so the module declares
// it as a package and every release publishes it.
if a := m.Build.Artifacts[2]; a.Name != "forgemsg" || a.Kind != "package" || a.Language != "go" || len(a.Paths) != 1 || a.Paths[0] != "forgemsg" || a.Series != "0.1" {
t.Errorf("package artifact = %+v", a)
// Issue 1966's bootstrap: the package artifact forgemsg is out of the manifest until the new builder runs; the follow-up
// puts it back with its assertion (ADR 0573 §1: three artifacts, the third the package forgemsg).
if len(m.Build.Artifacts) != 2 || m.Build.Artifacts[1].Name != "tools" || m.Build.Artifacts[1].From != "cmd/mesh-delivery-forge-bridge-tools" {
t.Fatalf("artifacts = %+v, want the program and the tools bundle", m.Build.Artifacts)
}
a := m.Build.Artifacts[0]
if st, err := os.Stat(a.From); err != nil || !st.IsDir() || a.Binary != "mesh-delivery-forge-bridge" || !strings.HasSuffix(a.From, a.Binary) {
@@ -101,15 +101,6 @@
"loads": [
"mesh-delivery-forge-bridge-tools"
]
},
{
"name": "forgemsg",
"kind": "package",
"language": "go",
"paths": [
"forgemsg"
],
"series": "0.1"
}
]
}
@@ -1,85 +0,0 @@
package main
import (
"context"
"fmt"
"strconv"
"strings"
stdio "git.novox.be/novox/mesh-sdk/go"
"mesh-delivery-indexer/internal/index"
)
// entryAnswer is what indexer_entry answers: the trunk revision read, the entry's packages (the pins: each other-repository
// package with the version it builds against, empty while none is published), its named paths by artifact, its fingerprint
// and its manifest's digest.
type entryAnswer struct {
Module string `json:"module"`
Revision int64 `json:"revision"`
Packages map[string]string `json:"packages"`
Paths map[string][]string `json:"paths"`
Fp string `json:"fp,omitempty"`
ManifestDigest string `json:"manifest-digest,omitempty"`
}
// entryReadTool is indexer_entry (H-A-204; tools-are-how-a-module-is-spoken-to) over the indexer's catalog-revisions objects:
// one module's entry pins and paths at a trunk revision, small, so a pin can be read without the revision whole (1.3 MB,
// more than the bus carries).
func entryReadTool(o index.Objects) stdio.Tool {
return stdio.Tool{Name: index.EntryPinsTool,
Description: "Read one module's catalog graph entry pins: the packages it builds against with the version each is pinned to " +
"(an empty version while none is published), the paths each artifact is built from, its fingerprint and its manifest " +
"digest, at the newest trunk revision or at a trunk revision given; a small answer, for when the revision whole is too " +
"large for the bus (H-A-204).",
Input: map[string]any{"type": "object", "required": []string{"module"}, "properties": map[string]any{
"module": map[string]any{"type": "string", "description": "the module whose entry is read"},
"revision": map[string]any{"type": "integer", "description": "the trunk revision; none reads the newest"}}},
Run: func(args map[string]any) (any, error) {
module, _ := args["module"].(string)
module = strings.TrimSpace(module)
if module == "" {
return nil, fmt.Errorf("%s names the module", index.EntryPinsTool)
}
revision, err := revisionArg(args["revision"])
if err != nil {
return nil, err
}
rev, e, err := index.TrunkEntryAt(context.Background(), o, module, revision)
if err != nil {
return nil, err
}
out := entryAnswer{Module: e.Module, Revision: rev.Revision, Packages: e.Packages, Paths: map[string][]string{},
Fp: e.Fp, ManifestDigest: e.ManifestDigest}
if out.Packages == nil {
out.Packages = map[string]string{}
}
if e.Inputs != nil && e.Inputs.Paths != nil {
out.Paths = e.Inputs.Paths
}
return out, nil
}}
}
// revisionArg is the revision argument as a number: absent is 0 (the newest); a number or its digits as a string.
func revisionArg(v any) (int64, error) {
switch r := v.(type) {
case nil:
return 0, nil
case float64:
if r < 0 || r != float64(int64(r)) {
return 0, fmt.Errorf("revision %v is not a trunk revision's number", r)
}
return int64(r), nil
case string:
if strings.TrimSpace(r) == "" {
return 0, nil
}
n, err := strconv.ParseInt(strings.TrimSpace(r), 10, 64)
if err != nil || n < 0 {
return 0, fmt.Errorf("revision %q is not a trunk revision's number", r)
}
return n, nil
}
return 0, fmt.Errorf("revision %v is not a trunk revision's number", v)
}
@@ -1,100 +0,0 @@
package main
import (
"context"
"encoding/json"
"reflect"
"strings"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"mesh-delivery-indexer/internal/index"
)
// memObjects is catalog-revisions in memory.
type memObjects map[string][]byte
func (m memObjects) Put(_ context.Context, name string, data []byte) error {
m[name] = data
return nil
}
func (m memObjects) Get(_ context.Context, name string) ([]byte, error) {
d, ok := m[name]
if !ok {
return nil, index.ErrNoObject
}
return d, nil
}
func (m memObjects) Delete(_ context.Context, name string) error { delete(m, name); return nil }
func (m memObjects) Names(context.Context) ([]string, error) {
var out []string
for n := range m {
out = append(out, n)
}
return out, nil
}
func (memObjects) Fill(context.Context) (int64, int64, error) { return 0, 0, nil }
// TestEntryTool: indexer_entry answers one entry's pins, paths, fingerprint and manifest digest, small, at the newest trunk
// revision and at a given one read from its kept object; an unknown module or revision is an error (H-A-204).
func TestEntryTool(t *testing.T) {
o := memObjects{}
keep := func(revision int64, entries ...meshcatalog.CatalogEntry) {
rev := meshcatalog.CatalogRevision{Branch: index.TrunkBranch, Revision: revision}
for _, e := range entries {
e.Manifest = map[string]any{"module": e.Module, "big": strings.Repeat("x", 64<<10)} // the manifest whole is never answered
data, _ := json.Marshal(e)
digest := e.Module + "-" + e.Fp
o[index.EntryObject(digest)] = data
rev.Entries = append(rev.Entries, meshcatalog.EntryRef{Module: e.Module, Digest: digest})
}
data, _ := json.Marshal(rev)
o[index.BranchObject(index.TrunkBranch, revision)] = data
}
paths := &meshcatalog.EntryInputs{Paths: map[string][]string{"mesh-grants:program": {"modules/mesh-grants", "sdk"}}}
keep(7, meshcatalog.CatalogEntry{Module: "mesh-grants", Fp: "src2:old", ManifestDigest: "sha256:a", Inputs: paths,
Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.26"}}, meshcatalog.CatalogEntry{Module: "other", Fp: "src2:o"})
keep(9, meshcatalog.CatalogEntry{Module: "mesh-grants", Fp: "src2:new", ManifestDigest: "sha256:b", Inputs: paths,
Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.27", "busgrants": ""}})
tool := entryReadTool(o)
if tool.Name != "indexer_entry" {
t.Fatalf("name %s", tool.Name)
}
for _, c := range []struct {
name string
args map[string]any
want entryAnswer
err string
}{
{"newest", map[string]any{"module": "mesh-grants"}, entryAnswer{Module: "mesh-grants", Revision: 9, Fp: "src2:new", ManifestDigest: "sha256:b",
Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.27", "busgrants": ""}, Paths: paths.Paths}, ""},
{"given", map[string]any{"module": "mesh-grants", "revision": float64(7)}, entryAnswer{Module: "mesh-grants", Revision: 7, Fp: "src2:old", ManifestDigest: "sha256:a",
Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.26"}, Paths: paths.Paths}, ""},
{"given as digits, no paths", map[string]any{"module": "other", "revision": "7"}, entryAnswer{Module: "other", Revision: 7, Fp: "src2:o",
Packages: map[string]string{}, Paths: map[string][]string{}}, ""},
{"not in the newest", map[string]any{"module": "other"}, entryAnswer{}, "holds no module other"},
{"revision not kept", map[string]any{"module": "mesh-grants", "revision": float64(8)}, entryAnswer{}, "revision 8 is not kept"},
{"no module", map[string]any{}, entryAnswer{}, "names the module"},
} {
t.Run(c.name, func(t *testing.T) {
got, err := tool.Run(c.args)
if c.err != "" {
if err == nil || !strings.Contains(err.Error(), c.err) {
t.Fatalf("error %v, want %q", err, c.err)
}
return
}
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(got, c.want) {
t.Fatalf("answer %+v, want %+v", got, c.want)
}
data, _ := json.Marshal(got)
if len(data) >= 8<<10 {
t.Fatalf("answer of %d bytes, want under 8 KB", len(data))
}
})
}
}
@@ -193,7 +193,7 @@ func main() {
tools = append(tools, catalogTools(trunk.State, kept, watched)...)
// The seat mesh-catalog's verb revision, a first view of the catalog half answered in this runtime of its own to exactly
// the parts whose entry declares it (novox/hq ADR 0663).
tools = append(tools, index.RevisionTool(revisions, kept), index.EntryTool(revisions, keptRead{stdio.State(index.EntryIndexState)}), index.EntriesTool(revisions, keptRead{stdio.State(index.EntryIndexState)}, keptRead{stdio.State(index.NamedState)}), entryReadTool(revisions), index.VersionsTool(keptRead{stdio.State(index.VersionsState)}),
tools = append(tools, index.RevisionTool(revisions, kept), index.EntryTool(revisions, keptRead{stdio.State(index.EntryIndexState)}), index.EntriesTool(revisions, keptRead{stdio.State(index.EntryIndexState)}, keptRead{stdio.State(index.NamedState)}), index.VersionsTool(keptRead{stdio.State(index.VersionsState)}),
index.OutdatedTool(&index.Indexer{Versions: keptRead{stdio.State(index.VersionsState)}, Heard: keptRead{stdio.State(index.HeardState)}, Revisions: revisions}),
index.InputsReadTool(keptRead{stdio.State(index.InputsState)}, stateobjects.ForState(index.InputsState).Objects))
tools = append(tools, index.CatalogRevisionTools(revisions, asks, time.Now, func(what, why string) error {
@@ -433,7 +433,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
done()
done = step("following the assignments")
followAssignments(ctx, reader, seatHolders.holders(), process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, views.StatusState), "indexer_first_views reads nothing and indexer_first_view_again is not heard", logger.Printf), cred, js, gate, func(ctx context.Context) {
if err := ix.RecomputeKept(ctx, "the assignments of a build module"); err != nil {
if err := ix.Recompute(ctx); err != nil {
logger.Printf("writing the modules again after the assignments of a build module changed: %v", err)
}
}, logger.Printf)
@@ -449,7 +449,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// the toolchain.
return ix.ToolchainReported(ctx)
},
recompute: func(ctx context.Context) error { return ix.RecomputeKept(ctx, "the toolchains read") }}, func() []string { return buildModuleNames(ix.BuildModules()) }, logger)
recompute: ix.Recompute}, func() []string { return buildModuleNames(ix.BuildModules()) }, logger)
if err := tools.start(); err != nil {
return err
}
@@ -90,7 +90,7 @@ func TestTheTrunkIsTheDirectoriesTheForgeListsTrees(t *testing.T) { // 1900/AC3
}{
{name: "a run started", body: `{"v":1,"run":"7","members":[{"module":"a","commit":"c"}],"by":"s"}`, acked: 1, started: "7"},
{name: "a run's pipeline ended", body: `{"v":1,"pipeline":"p","module":"a","branch":"b","commit":"c","outcome":"passed","steps":[],"run":"7"}`, ended: true, acked: 1, endedOf: "7"},
{name: "a pipeline's wants written to the end", body: `{"v":1,"pipeline":"main.13","branch":"main","source":12,"revision":13,"outcome":"tier-late"}`, wanted: true, acked: 1},
{name: "a pipeline's wants written to the end", body: `{"v":1,"pipeline":"main.12","branch":"main","source":12,"outcome":"tier-late"}`, wanted: true, acked: 1},
{name: "not json", body: `nope`, termd: 1},
{name: "a run never taken in", body: `{"v":1,"run":"7","members":[]}`, err: fmt.Errorf("%w: no member", index.ErrUnreadable), termd: 1, started: "7"},
{name: "a bus fault", body: `{"v":1,"run":"7","members":[{"module":"a","commit":"c"}]}`, err: errors.New("the bus did not answer"), started: "7"},
@@ -100,8 +100,8 @@ func TestTheTrunkIsTheDirectoriesTheForgeListsTrees(t *testing.T) { // 1900/AC3
switch {
case row.wanted:
heardPipelineWanted(context.Background(), ix, m, t.Logf)
if ix.wanted != "main.12.13 tier-late" { // its revision handed beside its source (H-A-214)
t.Errorf("%s: was handed %q, want main.12.13 tier-late", row.name, ix.wanted)
if ix.wanted != "main.12 tier-late" {
t.Errorf("%s: was handed %q, want main.12 tier-late", row.name, ix.wanted)
}
case row.ended:
heardPipelineEnded(context.Background(), ix, m, t.Logf)
@@ -169,11 +169,8 @@ func (f *fakeRuns) HeardPins(context.Context, *meshpipelinerunner.PipelineStarte
return nil
}
func (f *fakeRuns) HeardPipelineWanted(_ context.Context, branch string, source, revision int64, outcome string) error {
func (f *fakeRuns) HeardPipelineWanted(_ context.Context, branch string, source int64, outcome string) error {
f.wanted = fmt.Sprintf("%s.%d %s", branch, source, outcome)
if revision > 0 {
f.wanted = fmt.Sprintf("%s.%d.%d %s", branch, source, revision, outcome)
}
return f.err
}
@@ -21,7 +21,7 @@ type runTaker interface {
StartRun(ctx context.Context, e meshpipelinerunner.RunStarted) error
EndRunPipeline(ctx context.Context, e meshpipelinerunner.PipelineEnded) error
HeardPins(ctx context.Context, started *meshpipelinerunner.PipelineStarted, ended *meshpipelinerunner.PipelineEnded) error
HeardPipelineWanted(ctx context.Context, branch string, source, revision int64, outcome string) error
HeardPipelineWanted(ctx context.Context, branch string, source int64, outcome string) error
JudgePins(ctx context.Context)
}
@@ -99,7 +99,7 @@ func heardPipelineWanted(ctx context.Context, ix runTaker, m delivered, logf fun
_ = m.Term()
return
}
err := ix.HeardPipelineWanted(ctx, e.Branch, e.Source, e.Revision, e.Outcome)
err := ix.HeardPipelineWanted(ctx, e.Branch, e.Source, e.Outcome)
settleRun(m, err, "pipeline-wanted of "+e.Pipeline, logf)
if err == nil {
ix.JudgePins(ctx)
@@ -22,9 +22,8 @@ var (
)
// heardVersion reads one message of the consumer as a version, false for any other subject. A package's is kept in its
// series; one that does not decode is said and acknowledged; a write that fails is left unacknowledged, so the bus gives it
// again. It recomputes no module: a fingerprint moves only with a kept catalog revision, a release's by the re-pin its
// package-published starts, one revision per module release (design 47 H-A-211, H-A-213 (2)).
// series; one that does not decode is said and acknowledged; a write that fails is left unacknowledged, so the bus gives
// it again.
func heardVersion(ctx context.Context, ix *index.Indexer, m jetstream.Msg, logf func(string, ...any)) bool {
var h index.HeardVersion
switch seat, ok := publishedSeats[m.Subject()]; {
@@ -52,6 +51,12 @@ func heardVersion(ctx context.Context, ix *index.Indexer, m jetstream.Msg, logf
if kept {
logf("packages: %s %s kept in its series (published %v)", h.Package, h.Version, h.Published)
}
if kept && h.Published {
// A series' newest moved: its users' fingerprints take it (DLV-I6), in the same turn as the publish is heard.
if err := ix.Recompute(ctx); err != nil {
logf("writing the modules again after %s %s was published: %v", h.Package, h.Version, err)
}
}
_ = m.Ack()
return true
}
@@ -1,52 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"testing"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/gopackageregistry"
"mesh-delivery-indexer/internal/index"
)
// ackedMsg is a message that counts its acknowledgements.
type ackedMsg struct {
jetstream.Msg
subject string
data []byte
acked int
}
func (a *ackedMsg) Subject() string { return a.subject }
func (a *ackedMsg) Data() []byte { return a.data }
func (a *ackedMsg) Ack() error { a.acked++; return nil }
// unlistedStore is a store whose listing fails: a recompute over it fails.
type unlistedStore struct{ memStore }
func (unlistedStore) Keys(context.Context) ([]string, error) {
return nil, errors.New("the bus does not answer")
}
// TestHeardVersionMovesNoFingerprint: a registry's published is kept in its series and acknowledged, and recomputes no
// module: a fingerprint moves only with a kept revision, a release's by its re-pin on the package-published (design 47
// H-A-211, H-A-213 (2); the pin specification's leftover L3). The modules state here fails any recompute.
func TestHeardVersionMovesNoFingerprint(t *testing.T) {
data, err := json.Marshal(gopackageregistry.PublishedEvent{Kind: "package", Name: "git.example/novox/lib", Version: "0.1.4"})
if err != nil {
t.Fatal(err)
}
ix := &index.Indexer{Packages: &memStore{vals: map[string][]byte{}}, Modules: &unlistedStore{memStore{vals: map[string][]byte{}}},
Manifests: &memStore{vals: map[string][]byte{}}}
m := &ackedMsg{subject: gopackageregistry.SubjectPublished, data: data}
if !heardVersion(context.Background(), ix, m, t.Logf) {
t.Fatal("the registry's published is not taken as a version")
}
if m.acked != 1 {
t.Errorf("acknowledged %d times, want once: a publish recomputes nothing that could fail", m.acked)
}
}
@@ -286,14 +286,6 @@ func (a actor) HeldAgain(ctx context.Context, branch string) index.TakenIn {
return r
}
// TakeInAgain takes a repository in again at its kept merge commit (indexer_take_in_again, H-A-208, H-A-210), one take-in
// again at a time beside the held ones.
func (a actor) TakeInAgain(ctx context.Context, repository string) index.TakenIn {
a.s.again.Lock()
defer a.s.again.Unlock()
return a.Indexer.TakeInAgain(ctx, repository)
}
// HeldSkip ends a branch's held source revision unwritten (indexer_take_in_skip): take-in.taken says it skipped.
func (a actor) HeldSkip(ctx context.Context, branch string, source int64, by, why string) index.TakenIn {
r := index.TakenIn{Module: branch}
@@ -24,8 +24,6 @@ type takerIn interface {
// HeldAgain and HeldSkip act on a watched branch's held source-moved (design 47 H §744).
HeldAgain(ctx context.Context, branch string) index.TakenIn
HeldSkip(ctx context.Context, branch string, source int64, by, why string) index.TakenIn
// TakeInAgain takes a repository in again at its kept merge commit (H-A-208, H-A-210).
TakeInAgain(ctx context.Context, repository string) index.TakenIn
}
// takenIn answers one value of the takeins state (a hand act, issue 1945): a value not yet answered is acted on now and
@@ -55,9 +53,6 @@ func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream
} else if a.Held != "" {
asked, r = "take-in again of a held take-in", ix.HeldAgain(ctx, a.Held)
a.Module = a.Held
} else if a.Again != "" {
asked, r = "take-in again of a repository", ix.TakeInAgain(ctx, a.Again)
a.Module = a.Again
} else if a.Registry != nil {
asked, r = "registrations' act "+a.Registry.Verb, ix.RegistryAct(ctx, a)
} else if a.Recompute {
@@ -6,7 +6,6 @@ import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"slices"
"strconv"
@@ -448,42 +447,37 @@ func TestIndexerWhole(t *testing.T) {
// The first revision: no mover, no pin; its entries are the base.
sayMoved(t, ctx, js, "main", 10, c10, true)
// A revision's number is the indexer's own count, the branch's newest + 1 (design 47 H-A-213 (1)): the later take-ins of
// sources 11 to 13 are counted on from the number of source 10's.
first := heard.next(t, "main", 10)
r := func(source int64) int64 { return first.Revision + source - 10 }
rn := func(source int64) string { return fmt.Sprintf("main.%d", r(source)) }
if len(first.Entries) != 0 || first.Revision != 2 || first.Source != 10 {
t.Fatalf("the first revision said %+v, want revision 2 (the bed's seed is main.1, of source 0) of source 10 with no entries", first)
if first := heard.next(t, "main", 10); len(first.Entries) != 0 || first.Revision != 10 {
t.Fatalf("the first revision said %+v, want revision 10 with no entries", first)
}
untilKey(t, ctx, pins, index.PinKey("main", r(10)), absent)
untilKey(t, ctx, pins, index.PinKey("main", 10), absent)
// One take-in, one revision carrying its movers, pinned for its pipeline; said again from the kept movers.
sayMoved(t, ctx, js, "main", 11, c11, false)
taken := heard.next(t, "main", 11)
if len(taken.Entries) != 1 || taken.Entries[0].Module != "avahi" || taken.Entries[0].Commit != c11 || taken.Entries[0].Fingerprint == "" ||
taken.Entries[0].Repository != "novox/mesh-catalog" || taken.Revision != r(11) || taken.Call == "" {
taken.Entries[0].Repository != "novox/mesh-catalog" || taken.Revision != 11 || taken.Call == "" {
t.Fatalf("take-in.taken of main 11 = %+v, want avahi at %s as its one mover", taken, c11)
}
rev, _, err := index.ReadRevision(ctx, objects, rn(11))
rev, _, err := index.ReadRevision(ctx, objects, "main.11")
if err != nil || len(rev.Movers) != 1 || rev.Movers[0] != taken.Entries[0] {
t.Fatalf("main.11 = %+v (%v), want it kept carrying the movers said", rev, err)
}
untilKey(t, ctx, pins, index.PinKey("main", r(11)), pinned("pipeline."+rn(11)))
untilKey(t, ctx, pins, index.PinKey("main", r(10)), pinned("pipeline."+rn(11))) // validate's base, H-A-147
untilKey(t, ctx, pins, index.PinKey("main", 11), pinned("pipeline.main.11"))
untilKey(t, ctx, pins, index.PinKey("main", 10), pinned("pipeline.main.11")) // validate's base, H-A-147
info, err := objects.open(ctx)
if err != nil {
t.Fatal(err)
}
before, err := info.GetInfo(ctx, rn(11))
before, err := info.GetInfo(ctx, "main.11")
if err != nil {
t.Fatal(err)
}
sayMoved(t, ctx, js, "main", 11, c11, false)
if again := heard.next(t, "main", 11); len(again.Entries) != 1 || again.Entries[0] != taken.Entries[0] || again.Revision != r(11) {
if again := heard.next(t, "main", 11); len(again.Entries) != 1 || again.Entries[0] != taken.Entries[0] || again.Revision != 11 {
t.Errorf("the source-moved given again said %+v, want the kept movers %+v", again, taken.Entries)
}
if after, err := info.GetInfo(ctx, rn(11)); err != nil || after.NUID != before.NUID {
if after, err := info.GetInfo(ctx, "main.11"); err != nil || after.NUID != before.NUID {
t.Errorf("main.11 was written again (%v): a source-moved given again keeps no second revision", err)
}
@@ -491,11 +485,11 @@ func TestIndexerWhole(t *testing.T) {
// (H-A-147), which the take-in pinned for it.
sayMoved(t, ctx, js, "main", 12, c12, false)
heard.next(t, "main", 12)
if err := pins.Delete(ctx, index.PinKey("main", r(12))); err != nil { // the take-in's own pin, so pipeline-started alone writes it below
if err := pins.Delete(ctx, index.PinKey("main", 12)); err != nil { // the take-in's own pin, so pipeline-started alone writes it below
t.Fatal(err)
}
sayPipeline(t, ctx, js, false, "main", r(12))
untilKey(t, ctx, pins, index.PinKey("main", r(12)), pinned("pipeline."+rn(12)))
sayPipeline(t, ctx, js, false, "main", 12)
untilKey(t, ctx, pins, index.PinKey("main", 12), pinned("pipeline.main.12"))
// A run on request: its candidate and its key written on run-started, from avahi's files at the run's commit (A-160, A-174).
manifest, err := os.ReadFile("../../internal/index/testdata/catalogue-a67f0196-avahi.module.json")
if err != nil {
@@ -521,7 +515,7 @@ func TestIndexerWhole(t *testing.T) {
heard.acked(t, "main")
}
restart()
for _, name := range []string{rn(10), rn(11)} {
for _, name := range []string{"main.10", "main.11"} {
if _, _, err := index.ReadRevision(ctx, objects, name); err != nil {
t.Fatalf("%s is not kept across a restart while main.11's pipeline runs: %v", name, err)
}
@@ -559,27 +553,27 @@ func TestIndexerWhole(t *testing.T) {
t.Errorf("the held source-moved's consumer %+v (%v), want nothing unacknowledged", info, err)
}
setCap(t, ctx, js, 1<<30) // a larger cap alone wakes nothing: no timer takes a held take-in in
sayPipeline(t, ctx, js, true, "main", r(11))
sayPipeline(t, ctx, js, true, "main", 11)
if got := heard.next(t, "main", 13); len(got.Entries) != 1 || got.Entries[0].Commit != c13 {
t.Errorf("main 13 taken in on room said %+v, want avahi at %s", got, c13)
}
untilKey(t, ctx, held, keys.EscapeName("main"), absent)
// main.12's validate base stays (H-A-147), and main.11's passed pipeline's release pins it (H-A-168, H-A-170).
untilKey(t, ctx, pins, index.PinKey("main", r(11)), pinned("pipeline."+rn(12), index.ReleaseRecord("main", "avahi")))
untilRevisionGone(t, ctx, js, rn(10)) // the end freed both of main.11's pins
untilKey(t, ctx, pins, index.PinKey("main", 11), pinned("pipeline.main.12", index.ReleaseRecord("main", "avahi")))
untilRevisionGone(t, ctx, js, "main.10") // the end freed both of main.11's pins
// main.12's passed pipeline releases avahi again; its pipeline-wanted, the seat mesh-wants' (its wants written to the end),
// frees main.11's older release, and main.11 goes (H-A-168, H-A-170, H-A-173).
sayPipeline(t, ctx, js, true, "main", r(12))
untilKey(t, ctx, pins, index.PinKey("main", r(12)), func(v []byte) bool {
sayPipeline(t, ctx, js, true, "main", 12)
untilKey(t, ctx, pins, index.PinKey("main", 12), func(v []byte) bool {
var p index.PinValue
return v != nil && json.Unmarshal(v, &p) == nil && slices.Contains(p.Records, index.ReleaseRecord("main", "avahi"))
})
wantedRaw, _ := json.Marshal(meshwants.PipelineWanted{V: 1, Pipeline: rn(12), Branch: "main", Source: 12, Outcome: meshwants.PipelineWantedOutcomeRunning})
wantedRaw, _ := json.Marshal(meshwants.PipelineWanted{V: 1, Pipeline: "main.12", Branch: "main", Source: 12, Outcome: meshwants.PipelineWantedOutcomeRunning})
if _, err := js.Publish(ctx, meshwants.SubjectPipelineWanted("main", "12"), wantedRaw); err != nil {
t.Fatal(err)
}
untilKey(t, ctx, pins, index.PinKey("main", r(11)), absent)
untilRevisionGone(t, ctx, js, rn(11))
untilKey(t, ctx, pins, index.PinKey("main", 11), absent)
untilRevisionGone(t, ctx, js, "main.11")
if n := len(heard.pending); n != 0 {
t.Errorf("take-in.pending said %d more times for one held source revision, want once", n)
}
@@ -39,9 +39,7 @@ func checkCandidateModule(m meshcatalog.CandidatesByGroupValuePullsItemFpsValue)
return fmt.Errorf("index: the named path %q names no git object", p)
}
}
// An entry names a package with no published version yet with an empty version (design 47 H S8, H-A-204): it is checked
// as an entry's packages, not as an ask's.
if err := sdkstep.PackageVersions(m.Packages).CheckEntry(); err != nil {
if err := sdkstep.PackageVersions(m.Packages).Check(); err != nil {
return err
}
if m.Fp2 != "" {
@@ -1,30 +0,0 @@
package index
import (
"strings"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
)
// TestCheckCandidateModulePins: an entry's package with no published version (an empty pin) passes a run on request's
// candidate; a pin that is no version is refused (design 47 H S8, the pin specification's item 4).
func TestCheckCandidateModulePins(t *testing.T) {
for _, c := range []struct {
name string
packages map[string]string
ok bool
}{
{"an empty pin", map[string]string{"git.example/novox/lib/go": ""}, true},
{"a version", map[string]string{"git.example/novox/lib/go": "v0.1.4"}, true},
{"no version", map[string]string{"git.example/novox/lib/go": "latest"}, false},
} {
t.Run(c.name, func(t *testing.T) {
m := meshcatalog.CandidatesByGroupValuePullsItemFpsValue{Fp2: "src2:" + strings.Repeat("ab", 32), Packages: c.packages}
err := checkCandidateModule(m)
if (err == nil) != c.ok {
t.Errorf("checkCandidateModule(%v) = %v, want ok %v", c.packages, err, c.ok)
}
})
}
}
@@ -36,13 +36,13 @@ func TestACatalogRevisionIsKeptPerWatchedBranch(t *testing.T) {
for _, n := range names {
have[n] = true
}
for _, want := range []string{"main.1", "feature/x=2ey.1", "feature/x=2ey.2"} {
for _, want := range []string{"main.5", "feature/x=2ey.7", "feature/x=2ey.8"} {
if !have[want] {
t.Errorf("the objects hold %v, want %s", names, want)
}
}
if len(said) != 3 || said[0].Branch != "main" || said[1].Branch != "feature/x=2ey" || said[2].Revision != 2 || said[2].Source != 8 {
t.Errorf("said %+v, want main 1 of source 5, then the escaped branch 1 and 2 of sources 7 and 8", said)
if len(said) != 3 || said[0].Branch != "main" || said[1].Branch != "feature/x=2ey" || said[2].Revision != 8 {
t.Errorf("said %+v, want main 5, then the escaped branch 7 and 8", said)
}
answer := func(branch string, revision int64) meshcatalog.RevisionAnswer {
a, err := Revision(ctx, ix.Revisions, nil, meshcatalog.RevisionInput{Branch: branch, Revision: revision})
@@ -51,14 +51,14 @@ func TestACatalogRevisionIsKeptPerWatchedBranch(t *testing.T) {
}
return a
}
if a := answer("main", 0); a.Revision.Revision != 1 || a.Revision.Source != 5 || a.Entries[0].Commit != "c1" {
if a := answer("main", 0); a.Revision.Revision != 5 || a.Entries[0].Commit != "c1" {
t.Errorf("main answered %+v", a)
}
if a := answer("feature/x=2ey", 0); a.Revision.Revision != 2 || a.Revision.Branch != "feature/x.y" || a.Entries[0].Commit != "c3" {
t.Errorf("the branch's newest answered %+v, want its revision 2 at c3", a.Revision)
if a := answer("feature/x=2ey", 0); a.Revision.Revision != 8 || a.Revision.Branch != "feature/x.y" || a.Entries[0].Commit != "c3" {
t.Errorf("the branch's newest answered %+v, want its revision 8 at c3", a.Revision)
}
if a := answer("feature/x=2ey", 1); a.Revision.Revision != 1 || a.Entries[0].Commit != "c2" {
t.Errorf("the branch's revision 1 answered %+v", a.Revision)
if a := answer("feature/x=2ey", 7); a.Revision.Revision != 7 || a.Entries[0].Commit != "c2" {
t.Errorf("the branch's revision 7 answered %+v", a.Revision)
}
if a := answer("beta", 0); !a.NoneYet {
t.Errorf("a branch with no revision answered %+v, want none-yet", a)
@@ -68,22 +68,15 @@ func TestACatalogRevisionIsKeptPerWatchedBranch(t *testing.T) {
bySource := func(branch string, source int64) (meshcatalog.RevisionAnswer, error) {
return Revision(ctx, ix.Revisions, nil, meshcatalog.RevisionInput{Branch: branch, Source: source})
}
if a, err := bySource("main", 5); err != nil || a.Revision.Revision != 1 || a.Revision.Source != 5 || a.Entries[0].Commit != "c1" {
if a, err := bySource("main", 5); err != nil || a.Revision.Revision != 5 || a.Entries[0].Commit != "c1" {
t.Errorf("main's source 5 answered %+v (%v)", a.Revision, err)
}
if a, err := bySource("feature/x=2ey", 7); err != nil || a.Revision.Revision != 1 || a.Entries[0].Commit != "c2" {
t.Errorf("the branch's source 7 answered %+v (%v), want its revision 1 at c2", a.Revision, err)
if a, err := bySource("feature/x=2ey", 7); err != nil || a.Revision.Revision != 7 || a.Entries[0].Commit != "c2" {
t.Errorf("the branch's source 7 answered %+v (%v), want its revision 7 at c2", a.Revision, err)
}
if a, err := bySource("feature/x=2ey", 8); err != nil || a.Revision.Revision != 2 {
if a, err := bySource("feature/x=2ey", 8); err != nil || a.Revision.Revision != 8 {
t.Errorf("the branch's source 8 answered %+v (%v)", a.Revision, err)
}
// H-A-213: a re-pin's revision of source 8 answers below the take-in's: source 8 answers revision 2.
if _, err := ix.keepBranchEntries(ctx, "feature/x.y", 8, releaseCall("mesh-sdk", "0.1.27"), tip("c4"), nil); err != nil {
t.Fatal(err)
}
if a, err := bySource("feature/x=2ey", 8); err != nil || a.Revision.Revision != 2 || a.Entries[0].Commit != "c3" {
t.Errorf("the branch's source 8 after a re-pin answered %+v (%v), want the take-in's revision 2", a.Revision, err)
}
if _, err := bySource("feature/x=2ey", 9); err == nil {
t.Error("a source no revision was taken in from was answered")
}
@@ -9,7 +9,6 @@ import (
"reflect"
"slices"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph"
"git.novox.be/novox/mesh-sdk/go/catalogue"
@@ -31,7 +30,7 @@ const CatalogEntryLayout = 1
// ModuleValueOf gives the module's value back from the entry alone. The acting rules are the revision's to add.
func CatalogEntryOf(module string, cur meshcatalog.ModulesByModuleValue, m catalogue.Manifest, raw string, n graph.Node) meshcatalog.CatalogEntry {
e := meshcatalog.CatalogEntry{V: CatalogEntryLayout, Module: module, Repository: cur.Repository, Path: cur.Path,
Commit: cur.Commit, Manifest: manifestWhole(raw), ManifestDigest: meshcatalogclient.EntryManifestDigest(cur.Manifest), Verdict: cur.Verdict, Fp: n.FP, None: n.None, Bases: n.Bases,
Commit: cur.Commit, Manifest: manifestWhole(raw), ManifestDigest: meshcatalog.EntryManifestDigest(cur.Manifest), Verdict: cur.Verdict, Fp: n.FP, None: n.None, Bases: n.Bases,
Packages: maps.Clone(cur.Packages), Provides: n.Provides, Requires: n.Requires, Awaits: n.Awaits, Invokes: n.Invokes,
Serves: n.Serves, LeftOut: n.LeftOut, Main: MainOf(m), Contributes: contributesOf(m)}
// requires names every requirement as catalogue.Manifest.Wants lists them, optional ones and those only contributes names
@@ -68,7 +67,7 @@ func CatalogEntryOf(module string, cur meshcatalog.ModulesByModuleValue, m catal
// ruling on milestone 47/G finding 1; ADR 0656 §6: one form of the fact). The old line's fields (the old-form
// fingerprint, the reader needs) are not kept.
func ModuleValueOf(e meshcatalog.CatalogEntry) meshcatalog.ModulesByModuleValue {
v := meshcatalog.ModulesByModuleValue{V: keys.Version, Repository: e.Repository, Path: e.Path, Commit: e.Commit, Manifest: meshcatalogclient.BareManifestDigest(e.ManifestDigest),
v := meshcatalog.ModulesByModuleValue{V: keys.Version, Repository: e.Repository, Path: e.Path, Commit: e.Commit, Manifest: meshcatalog.BareManifestDigest(e.ManifestDigest),
Verdict: e.Verdict, Fp2: e.Fp, None2: e.None, Packages: maps.Clone(e.Packages)}
if in := e.Inputs; in != nil {
v.Toolchains, v.Paths, v.Problems, v.Warnings, v.Unknown = maps.Clone(in.Toolchains), maps.Clone(in.Paths), in.Problems, in.Warnings, in.Unknown
@@ -5,7 +5,6 @@ import (
"encoding/json"
"errors"
"fmt"
"maps"
"slices"
"strconv"
"strings"
@@ -88,37 +87,6 @@ func BranchRevisionOf(branch, name string) (int64, bool) {
return r, err == nil && r > 0
}
// watchedBranches are the watched branches the bucket keeps a revision of, the first included when it keeps one, sorted:
// each name <branch>.<n> but an entry's, a candidate's and the earlier forms' (earlierForm).
func (ix *Indexer) watchedBranches(ctx context.Context) ([]string, error) {
if ix.Revisions == nil {
return nil, nil
}
names, err := ix.Revisions.Names(ctx)
if err != nil {
return nil, fmt.Errorf("the objects %s: %w", CatalogRevisionsObjects, err)
}
seen := map[string]bool{}
for _, n := range names {
if strings.HasPrefix(n, "entry.") || strings.HasPrefix(n, "candidate.") || earlierForm(n) {
continue
}
i := strings.LastIndexByte(n, '.')
if i <= 0 {
continue
}
if r, err := strconv.ParseInt(n[i+1:], 10, 64); err != nil || r <= 0 {
continue
}
branch, err := keys.UnescapeName(n[:i])
if err != nil {
continue
}
seen[branch] = true
}
return slices.Sorted(maps.Keys(seen)), nil
}
// TrunkRevisionOf is the revision number a trunk object's name holds, and whether it is one.
func TrunkRevisionOf(name string) (int64, bool) { return BranchRevisionOf(TrunkBranch, name) }
@@ -242,54 +210,6 @@ func readNewestOf(ctx context.Context, o Objects, branch string) (meshcatalog.Ca
}
}
// TrunkEntryAt is a module's entry in a trunk revision, with that revision's list (H-A-204): for revision 0 the newest trunk
// revision read as entryOf reads it (readNewestOf); else the kept list of that trunk revision and only the module's entry
// object, so one entry is read without the revision whole.
func TrunkEntryAt(ctx context.Context, o Objects, module string, revision int64) (meshcatalog.CatalogRevision, meshcatalog.CatalogEntry, error) {
if revision == 0 {
rev, entries, found, err := readNewestOf(ctx, o, TrunkBranch)
if err != nil {
return rev, meshcatalog.CatalogEntry{}, err
}
if !found {
return rev, meshcatalog.CatalogEntry{}, errors.New("the indexer keeps no trunk revision yet")
}
for _, e := range entries {
if e.Module == module {
return rev, e, nil
}
}
return rev, meshcatalog.CatalogEntry{}, fmt.Errorf("the trunk revision %d holds no module %s", rev.Revision, module)
}
name := BranchObject(TrunkBranch, revision)
var rev meshcatalog.CatalogRevision
data, err := o.Get(ctx, name)
if errors.Is(err, ErrNoObject) {
return rev, meshcatalog.CatalogEntry{}, fmt.Errorf("the trunk revision %d is not kept", revision)
}
if err != nil {
return rev, meshcatalog.CatalogEntry{}, fmt.Errorf("revision %s: %w", name, err)
}
if err := json.Unmarshal(data, &rev); err != nil {
return rev, meshcatalog.CatalogEntry{}, fmt.Errorf("revision %s: %w", name, err)
}
for _, ref := range rev.Entries {
if ref.Module != module {
continue
}
data, err := o.Get(ctx, EntryObject(ref.Digest))
if err != nil {
return rev, meshcatalog.CatalogEntry{}, fmt.Errorf("revision %s, the entry of %s (%s): %w", name, module, ref.Digest, err)
}
var e meshcatalog.CatalogEntry
if err := json.Unmarshal(data, &e); err != nil {
return rev, meshcatalog.CatalogEntry{}, fmt.Errorf("revision %s, the entry of %s: %w", name, module, err)
}
return rev, e, nil
}
return rev, meshcatalog.CatalogEntry{}, fmt.Errorf("the trunk revision %d holds no module %s", revision, module)
}
// changedEntries are the modules whose entry differs between two revisions' lists, one gone or added included, by name.
func changedEntries(before, after []meshcatalog.EntryRef) []string {
was := map[string]string{}
@@ -346,22 +266,17 @@ func (ix *Indexer) viewEntries(ctx context.Context) (map[string]meshcatalog.Cata
}
// KeepBranchEntries keeps entries as the catalog revision of a watched branch taken in from its source revision src (design
// 47 H §743, H-A-32), as keepBranchEntries does with no call.
// 47 H §743, H-A-32): <branch>.<src>, the trunk's included, numbered by the source, and says catalog-revision naming the
// branch. movers are the take-in's: kept on the revision (H-A-92), and the revision pinned for the take-in's pipeline,
// pipeline.<branch>.<src>, in the same write when there is one (A-449a), with the branch's previous revision (H-A-147). A revision already kept under that name is not written
// again (read before write, under the branch's one consumer); ErrObjectsFull is a store too full to keep it.
func (ix *Indexer) KeepBranchEntries(ctx context.Context, branch string, src int64, entries map[string]meshcatalog.CatalogEntry, movers []meshcatalog.TakeInEntry) error {
_, err := ix.keepBranchEntries(ctx, branch, src, revisionCall{}, entries, movers)
return err
return ix.keepBranchEntries(ctx, branch, src, entries, movers)
}
// keepBranchEntries keeps entries as the next catalog revision of a watched branch, <branch>.<n>, the trunk's included: n is
// the indexer's own count, the branch's newest revision + 1, and src the gate's source revision it was read at, its field
// source (design 47 H-A-213 (1)); call is what made it (H-A-212, H-A-213 (3)): a take-in's the gate's call, a re-pin's
// "release of <module> <version>", a take-in again's "take-in again of <repository>". It says catalog-revision naming the
// branch, its source and its call. movers are kept on the revision (H-A-92), and the revision pinned for its one pipeline,
// pipeline.<branch>.<n>, in the same write when there is one (A-449a, H-A-213 (4)), with the branch's previous revision
// (H-A-147). It answers the number it kept; ErrObjectsFull is a store too full to keep it.
func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int64, call revisionCall, entries map[string]meshcatalog.CatalogEntry, movers []meshcatalog.TakeInEntry) (int64, error) {
func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int64, entries map[string]meshcatalog.CatalogEntry, movers []meshcatalog.TakeInEntry) error {
if ix.Revisions == nil {
return 0, nil
return nil
}
if branch == "" {
branch = TrunkBranch
@@ -371,24 +286,28 @@ func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int
newest, _, err := newestOf(ctx, ix.Revisions, branch)
if errors.Is(err, process.ErrNotYet) {
ix.logf("catalog graph: no revision is kept: its objects %s are not on the bus yet", CatalogRevisionsObjects)
return 0, err
return err
}
if err != nil {
ix.logf("catalog graph: no revision is kept: the newest is not read: %v", err)
return 0, fmt.Errorf("the newest catalog graph revision: %w", err)
return fmt.Errorf("the newest catalog graph revision: %w", err)
}
if _, err := ix.Revisions.Get(ctx, BranchObject(branch, src)); err == nil {
ix.logf("catalog graph: revision %s is kept already: not written again", BranchObject(branch, src))
return nil
} else if !errors.Is(err, ErrNoObject) {
return fmt.Errorf("revision %s: %w", BranchObject(branch, src), err)
}
number := newest.Revision + 1
entries = WithEdges(ix.withActs(ctx, entries)) // the acting rules and the edges between artifact references are written here, once, for every reader (A-208)
at := ix.now().UTC()
rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: number, Kind: "trunk", Branch: branch, Source: src, Call: call.Call,
CallKind: call.Kind, Module: call.Module, Version: call.Version, Repository: call.Repository, At: &at}, entries)
rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: src, Kind: "trunk", Branch: branch, Source: src, At: &at}, entries)
if err != nil {
return 0, err
return err
}
rev.Movers = movers
// The base of the validate step's case before: the newest revision kept before this write, which the take-in pins for its
// pipeline below, named on the revision in this same write (H-A-147, H-A-151, H-A-154); none on the branch's first take-in.
pinsBase := len(movers) > 0 && ix.Pins != nil && newest.Revision > 0
pinsBase := len(movers) > 0 && ix.Pins != nil && newest.Revision > 0 && newest.Revision != src
if pinsBase {
rev.Base = newest.Revision
}
@@ -400,32 +319,32 @@ func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int
refused := fmt.Sprintf("the next revision; source revision %d held pending with the reason %s", src, HeldObjectsFull)
ix.logf("catalog graph: revision %d of source revision %d is not kept: %s is full (%d of %d bytes)", rev.Revision, src, CatalogRevisionsObjects, used, capacity)
ix.sayFill(ctx, meshcatalog.EventFull, fmt.Sprintf("%d", src), meshcatalog.Fill{Bytes: used, Cap: capacity, Refused: refused})
return 0, fmt.Errorf("revision %d of source revision %d: %w", rev.Revision, src, ErrObjectsFull)
return fmt.Errorf("revision %d of source revision %d: %w", rev.Revision, src, ErrObjectsFull)
}
if err := ix.indexEntries(ctx, rev, entries); err != nil {
ix.logf("catalog graph: revision %d of source revision %d is not kept: %v", rev.Revision, src, err)
return 0, err
return err
}
if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil {
ix.logf("catalog graph: revision %d of source revision %d is not kept: %v", rev.Revision, src, err)
return 0, err
return err
}
// The take-in's revision pinned for its one pipeline before any judge can remove it (§743, §749, A-449a).
if len(movers) > 0 && ix.Pins != nil {
record := PipelineRecord(runner.PipelinesByBranchRevisionKey(keys.EscapeName(branch), strconv.FormatInt(number, 10)))
if err := ix.pin(ctx, PinKey(branch, number), record); err != nil {
ix.logf("pins: the revision %s is not pinned for %s: %v", BranchObject(branch, number), record, err)
return 0, fmt.Errorf("the pin of %s: %w", BranchObject(branch, number), err)
record := PipelineRecord(runner.PipelinesByBranchSourceKey(keys.EscapeName(branch), strconv.FormatInt(src, 10)))
if err := ix.pin(ctx, PinKey(branch, src), record); err != nil {
ix.logf("pins: the revision %s is not pinned for %s: %v", BranchObject(branch, src), record, err)
return fmt.Errorf("the pin of %s: %w", BranchObject(branch, src), err)
}
// The branch's previous revision, the newest kept before this write, pinned for the same pipeline: it is the base of
// the validate step's case before (H-A-108 (3), H-A-147), kept as long as the pipeline that needs it.
if pinsBase {
if err := ix.pin(ctx, PinKey(branch, rev.Base), record); err != nil {
ix.logf("pins: the previous revision %s is not pinned for %s: %v", BranchObject(branch, newest.Revision), record, err)
return 0, fmt.Errorf("the pin of %s: %w", BranchObject(branch, newest.Revision), err)
return fmt.Errorf("the pin of %s: %w", BranchObject(branch, newest.Revision), err)
}
} else {
ix.logf("pins: %s has no revision before it on its branch: none is pinned for %s as its validate base", BranchObject(branch, number), record)
ix.logf("pins: %s has no revision before it on its branch: none is pinned for %s as its validate base", BranchObject(branch, src), record)
}
}
// The revision before this one in its branch ceases to be current (§579): it goes unless a record pins it.
@@ -445,48 +364,16 @@ func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int
if after, capacity, err := ix.Revisions.Fill(ctx); fillErr == nil && err == nil && capacity > 0 && used*5 < capacity*4 && after*5 >= capacity*4 {
ix.sayFill(ctx, meshcatalog.EventFilling, fmt.Sprintf("%d", rev.Revision), meshcatalog.Fill{Bytes: after, Cap: capacity})
}
ix.logf("catalog graph: revision %d of source revision %d kept (%s), %d entries, %d changed", rev.Revision, src, call.Call, len(rev.Entries), len(changed))
ix.logf("catalog graph: revision %d of source revision %d kept, %d entries, %d changed", rev.Revision, src, len(rev.Entries), len(changed))
if ix.SayCatalogRevision != nil {
if err := ix.SayCatalogRevision(ctx, meshcatalog.CatalogRevisionEvent{V: CatalogEntryLayout, Revision: rev.Revision, Kind: rev.Kind,
Branch: keys.EscapeName(branch), Source: src, Call: call.Call, CallKind: call.Kind, Module: call.Module, Version: call.Version,
Repository: call.Repository, At: &at, Changed: changed, Entries: int64(len(rev.Entries))}); err != nil {
Branch: keys.EscapeName(branch), Source: src, At: &at, Changed: changed, Entries: int64(len(rev.Entries))}); err != nil {
ix.logf("catalog graph: revision %d is not said: %v", rev.Revision, err)
}
}
return number, nil
return nil
}
// revisionCall is what made a catalog revision (design 47 H-A-213 (3), H-A-215): call, the words for a person; Kind, one of the
// seat mesh-catalog's three call kinds, which a reader reads instead of the words; Module and Version for a release, Repository
// for a take-in again. The zero value names no call (a revision kept by hand or a test).
type revisionCall struct {
Call, Kind, Module, Version, Repository string
}
// takeInCall is a take-in's: the gate's call its source revision names.
func takeInCall(call string) revisionCall {
return revisionCall{Call: call, Kind: meshcatalog.CatalogRevisionCallKindTakeIn}
}
// releaseCall is a re-pin's on the release of module at version (H-A-213 (2)).
func releaseCall(module, version string) revisionCall {
return revisionCall{Call: ReleaseCall(module, version), Kind: meshcatalog.CatalogRevisionCallKindRelease, Module: module, Version: version}
}
// takeInAgainCall is a take-in again's of repository (H-A-212).
func takeInAgainCall(repository string) revisionCall {
return revisionCall{Call: TakeInAgainCall(repository), Kind: meshcatalog.CatalogRevisionCallKindTakeInAgain, Repository: repository}
}
// recomputeCall is a recompute's that moved a fingerprint, its call naming the reason (H-A-217).
func recomputeCall(reason string) revisionCall {
return revisionCall{Call: RecomputeCall(reason), Kind: meshcatalog.CatalogRevisionCallKindRecompute}
}
// RecomputeCall is the call of the catalog revision a recompute keeps: its reason, such as the assignments of a build module,
// a toolchain or a hand act (design 47 H-A-217).
func RecomputeCall(reason string) string { return "recompute on " + reason }
// HeldObjectsFull is the reason a pending merge is held while the catalog graph revisions are full (Pending.Held).
const HeldObjectsFull = "objects full"
@@ -657,7 +544,7 @@ func (k *KeptTrunk) reread(ctx context.Context, o Objects) error {
}
manifests := map[string]string{}
for _, e := range entries {
if d := meshcatalogclient.BareManifestDigest(e.ManifestDigest); d != "" && e.Manifest != nil {
if d := meshcatalog.BareManifestDigest(e.ManifestDigest); d != "" && e.Manifest != nil {
if raw, err := json.Marshal(e.Manifest); err == nil {
manifests[d] = string(raw)
}
@@ -21,8 +21,6 @@ const (
EntryIndexState = "entry-index"
// EntryIndexTool is the tool that reads a fingerprint's key and its object's digest (class read).
EntryIndexTool = "indexer_entries"
// EntryPinsTool is the tool that reads one module's entry pins and paths at a trunk revision (class read; H-A-204).
EntryPinsTool = "indexer_entry"
)
// EntryIndexValue is one key of entry-index: the digest of the entry object of the fingerprint the key names.
@@ -49,7 +49,7 @@ func TestAKeptRevisionIndexesItsEntries(t *testing.T) {
ctx := context.Background()
fp := "src2:" + strings.Repeat("d", 64)
ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), EntryIndex: newMem(), Now: func() time.Time { return time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC) }}
if err := ix.KeepBranchEntries(ctx, TrunkBranch, 1, map[string]meshcatalog.CatalogEntry{"avahi": {Module: "avahi", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}}}, nil); err != nil {
if err := ix.keepBranchEntries(ctx, TrunkBranch, 1, map[string]meshcatalog.CatalogEntry{"avahi": {Module: "avahi", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}}}, nil); err != nil {
t.Fatal(err)
}
if got, err := Entry(ctx, ix.Revisions, ix.EntryIndex, meshcatalog.EntryInput{Fingerprint: fp}); err != nil || got.Entry.Module != "avahi" {
@@ -260,32 +260,6 @@ func HeldTools(held Store, ask func(key string, value []byte) (TakeInAsk, error)
}
return a.Result, nil
}},
{Name: TakeInAgainTool,
Description: "Take a repository in again now at the merge commit the indexer keeps of it (design 47 H-A-208, H-A-210): its " +
"module graph is read again (each module's requirements, transitively) and the modules state recomputed as a " +
"package's publish re-pins it, so a changed take-in rule reaches a repository with no merge since; pins and " +
"fingerprints move, no catalog revision is kept and the gate is not asked. An unknown repository is refused.",
Input: map[string]any{"type": "object", "required": []string{"repository"}, "properties": map[string]any{
"repository": map[string]any{"type": "string", "description": "the repository as owner/name, such as novox/mesh-host"},
"why": map[string]any{"type": "string", "description": "why it is taken in again now"}}},
Run: func(args map[string]any) (any, error) {
repository := str(args, "repository")
if repository == "" {
return nil, errors.New("name the repository (repository) as owner/name")
}
data, err := json.Marshal(TakeInAsk{V: 1, Again: repository, Why: str(args, "why"), By: stdio.CallerOf(args), At: now().UTC()})
if err != nil {
return nil, err
}
a, err := ask("again."+keys.EscapeName(repository), data)
if err != nil {
return nil, err
}
if a.Result != nil && !a.Result.Stored {
return a.Result, fmt.Errorf("%s is not taken in again: %s", repository, a.Result.Answer)
}
return a.Result, nil
}},
{Name: TakeInSkipTool,
Description: "End a watched branch's held source-moved unwritten (design 47 H §744): it is acknowledged, take-in.taken is said " +
"with no entries and skipped, and the hand act is recorded through mesh-hand-acts before it is asked. indexer_pending " +
@@ -10,7 +10,6 @@ import (
"testing"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/views"
)
@@ -95,7 +94,7 @@ func TestTheKeptTrunkIsReadAtTheStartAndOncePerHeardRevision(t *testing.T) {
}
// A-487c: the manifests by digest and the verb revision asked of the trunk's newest answer from the view held.
listed = objects.n.Load()
if raw, err := kept.Manifest(meshcatalogclient.BareManifestDigest(keptDigest("ca-trust"))); err != nil || raw != `{"module":"ca-trust"}` {
if raw, err := kept.Manifest(meshcatalog.BareManifestDigest(keptDigest("ca-trust"))); err != nil || raw != `{"module":"ca-trust"}` {
t.Errorf("the held manifest = %q (%v), want the entry's manifest", raw, err)
}
if a, err := Revision(context.Background(), objects, kept, meshcatalog.RevisionInput{}); err != nil || a.Revision.Revision != 2 || len(a.Entries) != 2 {
@@ -63,7 +63,6 @@ func (ix *Indexer) applyMoving(ctx context.Context, f whole, drop bool, moves ma
return w, err
}
objects, namedErr := ix.namedObjects(ctx, f)
trees := ix.goModTree(ctx, f)
overlay := map[string]meshcatalog.ModulesByModuleValue{}
dirOf := map[string]string{}
named, every := map[string]bool{}, true // every module the message's manifests name, and whether each names one
@@ -110,23 +109,7 @@ func (ix *Indexer) applyMoving(ctx context.Context, f whole, drop bool, moves ma
next.None2 = "the manifest cannot be read: it cannot be kept as a value"
} else {
packages := ix.goPackages(ctx, f, dir, cur, has)
if m, err := judge.ForInputs([]byte(raw)); err == nil && usesGo(languagesOf(m)) {
// The entry pins every mesh package of another repository the module's build requires, transitively across
// the same-repository modules of its graph (design 47 H-A-201).
packages = unionSorted(packages, trees.otherRequired(goModuleDirs(m, dir)))
}
in := inputsOfNamed(name, raw, f.Trees[dir], f.Locks[dir], objects[dir], packages, ix.baseFP(ctx))
if in.Kept != nil {
// A Go build resolves the same-repository modules its go.mod files require, transitively, from the tree: their
// directories' objects join the tree input of every form, never the entry's paths (design 47 H-A-202, H-A-172).
if m, err := judge.ForInputs([]byte(raw)); err == nil && usesGo(languagesOf(m)) {
if tree, none := ix.derivedTree(ctx, f, trees, m, dir, in.Kept.Tree); none != "" {
in.Kept, in.None2 = nil, none
} else {
in.Kept.Tree = tree
}
}
}
next.Inputs2, next.None2, next.Paths = in.Kept, in.None2, in.ArtifactPaths
if in.Kept != nil && has && cur.Inputs2 != nil {
// The imports its last trunk build reported stand until its next build reports again (ADR 0593 §2, §4).
@@ -1,196 +0,0 @@
package index
import (
"context"
"fmt"
"path"
"path/filepath"
"slices"
"strings"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"mesh-delivery-indexer/internal/fingerprint"
)
// goModTree reads the go.mod files of one merge's tree, each directory once (Indexer.GoMod), for the walk of the module
// graph a Go build resolves from the tree (design 47 H-A-202, H-A-157).
type goModTree struct {
ix *Indexer
ctx context.Context
f whole
read map[string][]byte // by directory from the repository's root; nil for none
}
func (ix *Indexer) goModTree(ctx context.Context, f whole) *goModTree {
return &goModTree{ix: ix, ctx: ctx, f: f, read: map[string][]byte{}}
}
// goMod is the go.mod of the directory dir (from the root, "" for the root) at the merge commit; nil when there is none
// or it cannot be read, said.
func (t *goModTree) goMod(dir string) []byte {
if raw, ok := t.read[dir]; ok {
return raw
}
var raw []byte
if t.ix.GoMod != nil {
var err error
if raw, err = t.ix.GoMod(t.ctx, t.f.Repository, t.f.Merge, dir); err != nil {
t.ix.logf("%s at %s: the go.mod of %q cannot be read, so no module of the tree is found there: %v", t.f.Repository, t.f.Merge, dir, err)
raw = nil
}
}
t.read[dir] = raw
return raw
}
// sameRepositoryDir is the directory of the merge's tree that holds the Go module required: the required path starts with
// the repository's module prefix, and the directory it names holds a go.mod declaring that path (H-A-198, H-A-202). false
// for a module of another repository, or one the tree holds no go.mod of.
func (t *goModTree) sameRepositoryDir(required string) (string, bool) {
repo, dir, ok := trunk.PackageRepository(required)
if !ok || dir == "" || !trunk.SameRepository(repo, t.f.Repository) {
return "", false
}
if goModModule(t.goMod(dir)) != required {
return "", false
}
return dir, true
}
// sameRepositoryRequired are the directories (from the root, sorted, each once) of every same-repository Go module the
// go.mod of each start directory requires, transitively through those modules' own go.mod files (H-A-202, H-A-157). The
// start directories themselves are not among them.
func (t *goModTree) sameRepositoryRequired(start []string) []string {
seen := map[string]bool{}
for _, s := range start {
seen[s] = true
}
queue := slices.Clone(start)
var out []string
for len(queue) > 0 {
dir := queue[0]
queue = queue[1:]
for _, req := range fingerprint.MeshRequiresOf(t.goMod(dir)) {
d, ok := t.sameRepositoryDir(req)
if !ok || seen[d] {
continue
}
seen[d] = true
out = append(out, d)
queue = append(queue, d)
}
}
slices.Sort(out)
return out
}
// goModuleDirs are the directories (from the root) a module's Go build starts from: its own directory and the directory
// of each path of each Go package artifact of its manifest m.
func goModuleDirs(m catalogue.Manifest, dir string) []string {
out := []string{dir}
if m.Build == nil {
return out
}
for _, a := range m.Build.Artifacts {
if a.Kind != catalogue.ArtifactPackage || a.Language != "go" {
continue
}
for _, p := range a.Paths {
if d := path.Join(dir, strings.TrimSuffix(filepath.ToSlash(p), "/")); !slices.Contains(out, d) {
out = append(out, d)
}
}
}
return out
}
// otherRequired are the mesh packages of other repositories (sorted, each once) that the go.mod of each start directory,
// and of every same-repository module those require transitively (sameRepositoryRequired), requires at any version, the
// zero version included: every one the module's build resolves, which the entry pins (design 47 H-A-201, H-A-183,
// H-A-157). A same-repository module is none: it is built from the tree (H-A-198, H-A-202). nil for none.
func (t *goModTree) otherRequired(start []string) []string {
var out []string
for _, d := range append(slices.Clone(start), t.sameRepositoryRequired(start)...) {
out = append(out, fingerprint.OtherPackages(fingerprint.MeshRequiresOf(t.goMod(d)), t.f.Repository)...)
}
if len(out) == 0 {
return nil
}
return slices.Compact(slices.Sorted(slices.Values(out)))
}
// unionSorted is a and b together, sorted and each once; nil for none.
func unionSorted(a, b []string) []string {
if len(a)+len(b) == 0 {
return nil
}
return slices.Compact(slices.Sorted(slices.Values(append(slices.Clone(a), b...))))
}
// derivedDirs are the same-repository module directories (from the root, sorted) the module's Go build resolves from the
// tree beyond its own tree input (H-A-202): every one its go.mod files require, transitively, that its tree input does
// not hold already, which is the module's directory for the directory's tree, and its named paths for the named tree.
func (t *goModTree) derivedDirs(m catalogue.Manifest, dir string) []string {
var held []string
if paths, every := m.NamedPaths(); every {
for _, p := range paths {
held = append(held, path.Join(dir, filepath.ToSlash(p)))
}
} else {
held = []string{dir}
}
var out []string
for _, d := range t.sameRepositoryRequired(goModuleDirs(m, dir)) {
if !slices.ContainsFunc(held, func(h string) bool { return h == "" || h == "." || d == h || strings.HasPrefix(d, h+"/") }) {
out = append(out, d)
}
}
return out
}
// DerivedLine opens each line of the tree input naming a derived directory and its git object (H-A-202, H-A-172).
const DerivedLine = "derived "
// derivedTree is the module's tree input with a line per derived directory and its object appended, sorted by directory
// (H-A-202, H-A-172: what a build reads moves its fingerprint), so a change of such a directory moves the fingerprint of
// every form. The objects are asked of the forge at the merge commit by their paths from the root (Named); an answer that
// lacks one gives the reason the module has no inputs, naming them. A module with no derived directory keeps its tree.
func (ix *Indexer) derivedTree(ctx context.Context, f whole, t *goModTree, m catalogue.Manifest, dir, tree string) (string, string) {
dirs := t.derivedDirs(m, dir)
if len(dirs) == 0 {
return tree, ""
}
if ix.Named == nil {
return "", "the forge cannot be asked the git objects of the derived directories " + strings.Join(dirs, ", ")
}
got, err := ix.Named(ctx, f.Repository, f.Merge, map[string][]string{"": dirs})
if err != nil {
return "", fmt.Sprintf("the git objects of the derived directories %s: %v", strings.Join(dirs, ", "), err)
}
lines := []string{tree}
var missing []string
for _, d := range dirs {
o := got[""][d]
if o == "" {
missing = append(missing, d)
continue
}
lines = append(lines, DerivedLine+d+" "+o)
}
if len(missing) > 0 {
return "", "the forge names no git object of the derived directories " + strings.Join(missing, ", ")
}
return strings.Join(lines, "\n"), ""
}
// goModModule is the module path a go.mod declares; "" for none.
func goModModule(raw []byte) string {
for _, line := range strings.Split(string(raw), "\n") {
if f := strings.Fields(line); len(f) == 2 && f[0] == "module" {
return strings.Trim(f[1], `"`)
}
}
return ""
}
@@ -1,144 +0,0 @@
package index
import (
"context"
"strings"
"testing"
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"mesh-delivery-indexer/internal/fingerprint"
)
// A fixture repository novox/mesh-delivery: the Go package state of modules/g requires the same-repository module
// modules/b/builds at the zero version, which requires modules/c/client in turn; mesh-sdk is of another repository, and
// modules/b/gone names a directory the tree holds no go.mod of.
const ptpZero = " v0.0.0-00010101000000-000000000000"
var ptpGoMods = map[string]string{
"modules/g/state": "module git.novox.be/novox/mesh-delivery/modules/g/state\n\nrequire (\n\tgit.novox.be/novox/mesh-delivery/modules/b/builds" + ptpZero +
"\n\tgit.novox.be/novox/mesh-sdk/go" + ptpZero + "\n\tgit.novox.be/novox/mesh-delivery/modules/b/gone" + ptpZero + "\n)\n",
"modules/b/builds": "module git.novox.be/novox/mesh-delivery/modules/b/builds\n\nrequire git.novox.be/novox/mesh-delivery/modules/c/client" + ptpZero + " // indirect\n",
"modules/c/client": "module git.novox.be/novox/mesh-delivery/modules/c/client\n",
}
// ptpProgram is modules/g's manifest: a program naming no paths beside the package state.
const ptpProgram = `{"module":"g","version":"1.0","build":{"artifacts":[{"name":"program","kind":"bundle","language":"go","system":"arch","from":"cmd/g","binary":"g","loads":["g"]},` +
`{"name":"state","kind":"package","language":"go","paths":["state"],"series":"0.1"}]}}`
// ptpIndexer answers the fixture's go.mod files and, for a path asked from the root, the object objects names for it.
func ptpIndexer(t *testing.T, objects map[string]string) (*Indexer, *memStore, *[]map[string][]string) {
ix, mods, _ := manifestsIndexer(t)
ix.GoMod = func(_ context.Context, _, _, dir string) ([]byte, error) {
if c, ok := ptpGoMods[dir]; ok {
return []byte(c), nil
}
return nil, nil
}
var asked []map[string][]string
ix.Named = func(_ context.Context, _, _ string, paths map[string][]string) (map[string]map[string]string, error) {
asked = append(asked, paths)
out := map[string]map[string]string{}
for dir, ps := range paths {
out[dir] = map[string]string{}
for _, p := range ps {
out[dir][p] = objects[p]
}
}
return out, nil
}
return ix, mods, &asked
}
// Design 47 H-A-202, H-A-172: the same-repository modules a Go build resolves from the tree, transitively, join the tree
// input of every form with their objects, so a change of one moves the fingerprint of a module whose program names no
// paths; the entry's paths stay the manifest's.
func TestADerivedDirectoryMovesTheFingerprintOfEveryForm(t *testing.T) {
objects := map[string]string{"modules/b/builds": strings.Repeat("b", 40), "modules/c/client": strings.Repeat("c", 40)}
tree := func(objects map[string]string) string {
ix, mods, asked := ptpIndexer(t, objects)
f := withManifests(mergedMsg("novox/mesh-delivery", map[string]string{"modules/g": "gtree"}), map[string]string{"modules/g": ptpProgram}, nil)
if err := ix.Apply(context.Background(), f); err != nil {
t.Fatal(err)
}
m, _ := moduleOf(t, mods, "g")
if got := strings.Join(m.Paths["state"], ","); got != "state" {
t.Errorf("the entry's paths of state are %q, want the manifest's", got)
}
if len(*asked) != 1 || strings.Join((*asked)[0][""], ",") != "modules/b/builds,modules/c/client" {
t.Fatalf("asked the forge %v, want the derived directories from the root once", *asked)
}
if m.Inputs2 == nil || !strings.HasPrefix(m.Inputs2.Tree, "gtree\n") {
t.Fatalf("inputs = %+v (%s), want the directory's tree with the derived lines", m.Inputs2, m.None2)
}
return m.Inputs2.Tree
}
before := tree(objects)
if want := "gtree\nderived modules/b/builds " + objects["modules/b/builds"] + "\nderived modules/c/client " + objects["modules/c/client"]; before != want {
t.Errorf("the tree input is %q, want %q", before, want)
}
objects["modules/c/client"] = strings.Repeat("d", 40)
after := tree(objects)
fp := func(tree string) string {
v, _ := fingerprint.Of(fingerprint.Inputs{Module: "g", Tree: tree})
return v
}
if fp(before) == fp(after) {
t.Errorf("a change of modules/c/client left the fingerprint %s", fp(after))
}
// The named tree takes them too: a module whose every artifact names its paths.
named := `{"module":"g","version":"1.0","build":{"artifacts":[{"name":"state","kind":"package","language":"go","paths":["state"],"series":"0.1"}]}}`
ix, mods, _ := ptpIndexer(t, objects)
f := withManifests(mergedMsg("novox/mesh-delivery", map[string]string{"modules/g": "gtree"}), map[string]string{"modules/g": named}, nil)
f.Paths = map[string]map[string]string{"modules/g": {"state": strings.Repeat("1", 40)}}
if err := ix.Apply(context.Background(), f); err != nil {
t.Fatal(err)
}
if m, _ := moduleOf(t, mods, "g"); m.Inputs2 == nil || !strings.HasPrefix(m.Inputs2.Tree, fingerprint.NamedTreePrefix) ||
!strings.HasSuffix(m.Inputs2.Tree, "\nderived modules/c/client "+objects["modules/c/client"]) || strings.Join(m.Paths["state"], ",") != "state" {
t.Errorf("module = %+v, want the named tree with the derived lines and the manifest's paths", m)
}
// A forge that names no object of a derived directory gives the module none, naming it.
ix, mods, _ = ptpIndexer(t, map[string]string{"modules/b/builds": strings.Repeat("b", 40)})
if err := ix.Apply(context.Background(), withManifests(mergedMsg("novox/mesh-delivery", map[string]string{"modules/g": "gtree"}),
map[string]string{"modules/g": ptpProgram}, nil)); err != nil {
t.Fatal(err)
}
if m, _ := moduleOf(t, mods, "g"); m.Fp2 != "" || !strings.Contains(m.None2, "modules/c/client") {
t.Errorf("module = %+v, want none naming the directory the forge gave no object of", m)
}
}
// Design 47 H-A-201: the entry pins every mesh package of another repository the module's build requires, across the
// same-repository modules of its graph: here the one modules/c/client requires, which modules/g's go.mod never names.
// A module of the module's own repository is never pinned, whatever a build reported (H-A-198, H-A-202).
func TestTheEntryPinsEveryRequirementOfTheModuleGraph(t *testing.T) {
const conditions = "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
objects := map[string]string{"modules/b/builds": strings.Repeat("b", 40), "modules/c/client": strings.Repeat("c", 40)}
ix, mods, _ := ptpIndexer(t, objects)
mod := map[string]string{
"modules/g": "module g\n\nrequire git.novox.be/novox/mesh-delivery/modules/b/builds" + ptpZero + "\n",
"modules/c/client": "module git.novox.be/novox/mesh-delivery/modules/c/client\n\nrequire " + conditions + ptpZero + "\n",
}
ix.GoMod = func(_ context.Context, _, _, dir string) ([]byte, error) {
if c, ok := mod[dir]; ok {
return []byte(c), nil
}
return []byte(ptpGoMods[dir]), nil
}
f := withManifests(mergedMsg("novox/mesh-delivery", map[string]string{"modules/g": "gtree"}), map[string]string{"modules/g": ptpProgram},
map[string]map[string]string{"modules/g": {"go.sum": avahiGoSum}})
if err := ix.Apply(context.Background(), f); err != nil {
t.Fatal(err)
}
m, _ := moduleOf(t, mods, "g")
if m.Inputs2 == nil || strings.Join(m.Inputs2.Packages, ",") != conditions+",git.novox.be/novox/mesh-sdk/go" {
t.Errorf("packages = %+v, want the client's requirement and state's mesh-sdk, and no module of the repository", m.Inputs2)
}
builds := "git.novox.be/novox/mesh-delivery/modules/b/builds"
pins := releasedPins(sdkstep.PackageVersions{builds: "v0.1.106", conditions: "v0.1.4"}, nil, catalogue.Manifest{}, "novox/mesh-delivery", nil)
if _, pinned := pins[builds]; pinned || len(pins) != 1 {
t.Errorf("pins = %v, want %s only, never a module of the repository", pins, conditions)
}
}
@@ -119,9 +119,8 @@ func ReleaseRecord(branch, module string) string {
// every revision of the branch below it; a mover the revision holds no release of (superseded, no version released) keeps its
// older release's revision. Outcome tier-late frees nothing: its later tiers' wants are not written, though their release
// records are held, and the same pipeline's later running frees them. A pipeline whose revision is not kept frees nothing and
// says so; one applied twice changes nothing. The pipeline's revision is the one the event names (design 47 H-A-214); an event
// naming none (revision zero, said before the field) is of the source's lowest-numbered revision, the take-in's.
func (ix *Indexer) HeardPipelineWanted(ctx context.Context, branch string, source, revision int64, outcome string) error {
// says so; one applied twice changes nothing.
func (ix *Indexer) HeardPipelineWanted(ctx context.Context, branch string, source int64, outcome string) error {
if ix.Pins == nil || ix.Revisions == nil {
return errors.New("the indexer keeps no state " + PinsState)
}
@@ -129,21 +128,12 @@ func (ix *Indexer) HeardPipelineWanted(ctx context.Context, branch string, sourc
ix.logf("pins: pipeline-wanted of %s %d is %s: it frees no release until its wants are written everywhere (H-A-174)", branch, source, outcome)
return nil
}
var name string
if revision > 0 {
name = RevisionObject(meshcatalog.CatalogRevision{Branch: branch, Revision: revision})
} else {
var err error
if name, err = revisionOfSource(ctx, ix.Revisions, branch, source); err != nil {
ix.logf("pins: pipeline-wanted of %s %d frees no release: its revision is not kept: %v", branch, source, err)
return nil
}
}
rev, _, err := ReadRevision(ctx, ix.Revisions, name)
if revision > 0 && errors.Is(err, ErrNoObject) {
ix.logf("pins: pipeline-wanted of %s revision %d frees no release: %s is not kept", branch, revision, name)
name, err := revisionOfSource(ctx, ix.Revisions, branch, source)
if err != nil {
ix.logf("pins: pipeline-wanted of %s %d frees no release: its revision is not kept: %v", branch, source, err)
return nil
}
rev, _, err := ReadRevision(ctx, ix.Revisions, name)
if err != nil {
return fmt.Errorf("%s: %w", name, err)
}
@@ -106,54 +106,42 @@ func TestTheIndexerRemovesARevisionOnlyWhenNoRecordPinsItAndItIsNotCurrent(t *te
t.Fatal(err)
}
}
endOf := func(source, revision int64, outcome, version string) {
end := func(source int64, outcome, version string) {
t.Helper()
e := runner.PipelineEnded{Branch: "main", Pipeline: fmt.Sprintf("main.%d", revision), Revision: revision, Source: source, Outcome: outcome,
e := runner.PipelineEnded{Branch: "main", Pipeline: fmt.Sprintf("main.%d", source), Revision: source, Source: source, Outcome: outcome,
Movers: []runner.MoverEnded{{Module: "reader", Fingerprint: "f", Version: version}}}
if err := ix.HeardPins(ctx, nil, &e); err != nil {
t.Fatal(err)
}
ix.JudgePins(ctx)
}
end := func(source int64, outcome, version string) { t.Helper(); endOf(source, source, outcome, version) }
// An event naming no revision (said before the field) is of the source's lowest-numbered revision, the take-in's (H-A-214).
wanted := func(source int64, outcome string) {
t.Helper()
if err := ix.HeardPipelineWanted(ctx, "main", source, 0, outcome); err != nil {
if err := ix.HeardPipelineWanted(ctx, "main", source, outcome); err != nil {
t.Fatal(err)
}
ix.JudgePins(ctx)
}
take(4)
end(4, runner.PipelineEndedOutcomePassed, "0.1.10")
take(5)
end(5, runner.PipelineEndedOutcomeFailed, "")
want("a passed release past a later failed pipeline", "main.4", "main.5")
take(6)
end(6, runner.PipelineEndedOutcomePassed, "")
wanted(6, meshwants.PipelineWantedOutcomeRunning)
want("a passed pipeline releasing no version frees no older release", "main.4", "main.6")
take(7)
end(7, runner.PipelineEndedOutcomePassed, "0.1.13")
want("two releases, the unreleased one between them gone", "main.4", "main.7")
wanted(7, meshwants.PipelineWantedOutcomeTierLate) // H-A-174: its later tiers' wants are not written, so it frees nothing
want("a tier-late pipeline-wanted", "main.4", "main.7")
wanted(7, meshwants.PipelineWantedOutcomeRunning)
wanted(7, meshwants.PipelineWantedOutcomeRunning) // heard twice: changes nothing more
want("the newer release written everywhere", "main.7")
if v, _, _, _ := ix.Pins.Get(ctx, "main.7"); !slices.Equal(mustRecords(t, v), []string{ReleaseRecord("main", "reader")}) {
t.Errorf("the pin of main.7 holds %s, want its release alone", v)
take(10)
end(10, runner.PipelineEndedOutcomePassed, "0.1.10")
take(11)
end(11, runner.PipelineEndedOutcomeFailed, "")
want("a passed release past a later failed pipeline", "main.10", "main.11")
take(12)
end(12, runner.PipelineEndedOutcomePassed, "")
wanted(12, meshwants.PipelineWantedOutcomeRunning)
want("a passed pipeline releasing no version frees no older release", "main.10", "main.12")
take(13)
end(13, runner.PipelineEndedOutcomePassed, "0.1.13")
want("two releases, the unreleased one between them gone", "main.10", "main.13")
wanted(13, meshwants.PipelineWantedOutcomeTierLate) // H-A-174: its later tiers' wants are not written, so it frees nothing
want("a tier-late pipeline-wanted", "main.10", "main.13")
wanted(13, meshwants.PipelineWantedOutcomeRunning)
wanted(13, meshwants.PipelineWantedOutcomeRunning) // heard twice: changes nothing more
want("the newer release written everywhere", "main.13")
if v, _, _, _ := ix.Pins.Get(ctx, "main.13"); !slices.Equal(mustRecords(t, v), []string{ReleaseRecord("main", "reader")}) {
t.Errorf("the pin of main.13 holds %s, want its release alone", v)
}
// H-A-214: a re-pin's revision 8 of the same source 7 releases 0.1.14; its pipeline-wanted names revision 8, so the release
// on revision 8 frees the older one on revision 7, which the source's lowest-numbered revision would not.
take(7)
endOf(7, 8, runner.PipelineEndedOutcomePassed, "0.1.14")
want("a re-pin's release beside the take-in's", "main.7", "main.8")
if err := ix.HeardPipelineWanted(ctx, "main", 7, 8, meshwants.PipelineWantedOutcomeRunning); err != nil {
t.Fatal(err)
}
ix.JudgePins(ctx)
want("the re-pin's release written everywhere", "main.8")
}
func mustRecords(t *testing.T, v []byte) []string {
@@ -180,22 +168,21 @@ func TestATakeInPinsTheBranchsPreviousRevisionForItsPipeline(t *testing.T) {
before []int64 // sources taken in with no mover before the take-in of source 5
whileRuns []string
afterTheEnd []string
at int64 // the number the take-in of source 5 keeps: the branch's newest + 1 (H-A-213 (1))
base int64 // the base that revision names (H-A-154)
base int64 // the base main.5 names (H-A-154)
passed bool // main.5's pipeline passed: its release pins main.5
unreleased bool // its mover released no version there, so nothing pins main.5 (H-A-173)
then []int64 // sources taken in with no mover after the end
released []string
}{
{name: "the previous revision kept and pinned", before: []int64{3, 4}, at: 3, whileRuns: []string{"main.2", "main.3"}, afterTheEnd: []string{"main.3"}, base: 2},
{name: "the newest kept before, whatever its source", before: []int64{2}, at: 2, whileRuns: []string{"main.1", "main.2"}, afterTheEnd: []string{"main.2"}, base: 1},
{name: "no revision before it pins only its own", at: 1, whileRuns: []string{"main.1"}, afterTheEnd: []string{"main.1"}},
{name: "a passed pipeline's release keeps its revision once no longer current", before: []int64{4}, at: 2, whileRuns: []string{"main.1", "main.2"},
base: 1, passed: true, then: []int64{6}, afterTheEnd: []string{"main.2", "main.3"}, released: []string{"main.2"}},
{name: "a passed pipeline whose mover released no version pins nothing", before: []int64{4}, at: 2, whileRuns: []string{"main.1", "main.2"},
base: 1, passed: true, unreleased: true, then: []int64{6}, afterTheEnd: []string{"main.3"}},
{name: "a failed pipeline's revision goes once no longer current", before: []int64{4}, at: 2, whileRuns: []string{"main.1", "main.2"},
base: 1, then: []int64{6}, afterTheEnd: []string{"main.3"}},
{name: "the previous revision kept and pinned", before: []int64{3, 4}, whileRuns: []string{"main.4", "main.5"}, afterTheEnd: []string{"main.5"}, base: 4},
{name: "the newest kept before, whatever its number", before: []int64{2}, whileRuns: []string{"main.2", "main.5"}, afterTheEnd: []string{"main.5"}, base: 2},
{name: "no revision before it pins only its own", whileRuns: []string{"main.5"}, afterTheEnd: []string{"main.5"}},
{name: "a passed pipeline's release keeps its revision once no longer current", before: []int64{4}, whileRuns: []string{"main.4", "main.5"},
base: 4, passed: true, then: []int64{6}, afterTheEnd: []string{"main.5", "main.6"}, released: []string{"main.5"}},
{name: "a passed pipeline whose mover released no version pins nothing", before: []int64{4}, whileRuns: []string{"main.4", "main.5"},
base: 4, passed: true, unreleased: true, then: []int64{6}, afterTheEnd: []string{"main.6"}},
{name: "a failed pipeline's revision goes once no longer current", before: []int64{4}, whileRuns: []string{"main.4", "main.5"},
base: 4, then: []int64{6}, afterTheEnd: []string{"main.6"}},
} {
t.Run(tc.name, func(t *testing.T) {
ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), Pins: newMem(), Findings: newFindings(),
@@ -219,19 +206,18 @@ func TestATakeInPinsTheBranchsPreviousRevisionForItsPipeline(t *testing.T) {
keep(s, nil)
}
keep(5, mover)
at := fmt.Sprintf("main.%d", tc.at)
if rev, _, err := ReadRevision(ctx, ix.Revisions, at); err != nil || rev.Base != tc.base || rev.Source != 5 {
t.Errorf("%s names the source %d and the base %d (%v), want 5 and %d, the revision its take-in pinned (H-A-154)", at, rev.Source, rev.Base, err, tc.base)
if rev, _, err := ReadRevision(ctx, ix.Revisions, "main.5"); err != nil || rev.Base != tc.base {
t.Errorf("main.5 names the base %d (%v), want %d, the revision its take-in pinned (H-A-154)", rev.Base, err, tc.base)
}
if got := kept(); !slices.Equal(got, tc.whileRuns) {
t.Errorf("while the pipeline runs: kept %v, want %v", got, tc.whileRuns)
}
for _, k := range tc.whileRuns {
if v, _, _, _ := ix.Pins.Get(ctx, k); !slices.Equal(mustRecords(t, v), []string{"pipeline." + at}) {
if v, _, _, _ := ix.Pins.Get(ctx, k); !slices.Equal(mustRecords(t, v), []string{"pipeline.main.5"}) {
t.Errorf("the pin of %s holds %s, want the pipeline's record", k, v)
}
}
end := runner.PipelineEnded{Branch: "main", Pipeline: at, Revision: tc.at, Outcome: runner.PipelineEndedOutcomeFailed,
end := runner.PipelineEnded{Branch: "main", Pipeline: "main.5", Revision: 5, Outcome: runner.PipelineEndedOutcomeFailed,
Movers: []runner.MoverEnded{{Module: "reader", Fingerprint: "f", Version: "0.1.5"}}}
if tc.unreleased {
end.Movers[0].Version = ""
@@ -1,100 +0,0 @@
package index
import (
"context"
"slices"
"strings"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// Design 47 H-A-204, H-A-201, H-A-165: an entry's packages name every requirement of its module graph of another
// repository: one the form's versions leave out (an import form naming only what the build imported) is pinned by its
// series' newest published version, and one none publishes yet is kept with no version, never left out.
func TestAnEntryNamesEveryRequirementPinnedOrWaiting(t *testing.T) {
const (
wireguard = "git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go"
busgrants = "git.novox.be/novox/mesh-delivery/modules/mesh-grants/busgrants"
sdk = "git.novox.be/novox/mesh-sdk/go"
)
series := map[string]SeriesPackage{"wg": {Name: "mesh-wireguard/go", GoModule: wireguard, Series: catalogue.DefaultSeries,
Newest: &meshcatalog.PackagesByPackageSeriesValueNewest{Version: "0.1.47"}}}
pins := releasedPins(sdkstep.PackageVersions{sdk: "c0ffee"}, series, catalogue.Manifest{}, "novox/mesh-catalog", []string{busgrants, sdk, wireguard})
want := sdkstep.PackageVersions{sdk: "", wireguard: "v0.1.47", busgrants: ""}
if len(pins) != len(want) {
t.Fatalf("pins = %v, want %v", pins, want)
}
for p, v := range want {
if got, named := pins[p]; !named || got != v {
t.Errorf("the pin of %s is %q (named %v), want %q", p, got, named, v)
}
}
}
// Design 47 H-A-204, H-A-165: a release re-pins every kept entry naming one of its packages, of any repository and
// changed or not, by its pins or by its kept requirements alone: here g, of another repository, holds no pin of mesh-sdk.
// H-A-211, H-A-212, H-A-213: on every watched branch holding g, the re-pin keeps the branch's next revision, of the same
// source as the one before it, with the call "release of mesh-sdk 0.1.27" and g as its mover at its new fingerprint, and
// says catalog-revision with that call.
func TestAReleaseRepinsAnEntryNamingItsPackageWithNoPin(t *testing.T) {
ctx := context.Background()
objects := map[string]string{"modules/b/builds": strings.Repeat("b", 40), "modules/c/client": strings.Repeat("c", 40)}
ix, mods, _ := ptpIndexer(t, objects)
ix.Revisions, ix.Packages = newMemObjects(1<<20), newMem()
reportToolchain(t, ix, "mesh-build-go", "n1", "go", "1.26.0")
var said []meshcatalog.CatalogRevisionEvent
ix.SayCatalogRevision = func(_ context.Context, e meshcatalog.CatalogRevisionEvent) error { said = append(said, e); return nil }
f := withManifests(mergedMsg("novox/mesh-delivery", map[string]string{"modules/g": "gtree"}), map[string]string{"modules/g": ptpProgram}, nil)
if err := ix.Apply(ctx, f); err != nil {
t.Fatal(err)
}
g, _ := moduleOf(t, mods, "g")
if g.Inputs2 == nil || !slices.Contains(g.Inputs2.Packages, "git.novox.be/novox/mesh-sdk/go") {
t.Fatalf("g = %+v, want mesh-sdk among its kept requirements", g.Inputs2)
}
g.Packages = nil
mods.put("g", mustEncode(t, g))
mods.put("mesh-sdk", mustEncode(t, meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-sdk", Commit: strings.Repeat("1", 40),
Path: meshcatalogclient.RootPath, None2: None2Pending}))
entries, err := ix.viewEntries(ctx)
if err != nil {
t.Fatal(err)
}
sources := map[string]int64{"main": 5, "feature/x": 9}
for branch, src := range sources {
if err := ix.KeepBranchEntries(ctx, branch, src, entries, nil); err != nil {
t.Fatal(err)
}
}
said = nil
if _, err := ix.HearVersion(ctx, HeardVersion{Package: "mesh-sdk/go", GoModule: "git.novox.be/novox/mesh-sdk/go", Version: "0.1.27", Published: true}); err != nil {
t.Fatal(err)
}
if n, err := ix.Repin(ctx, "mesh-sdk", "0.1.27"); err != nil || n != 2 {
t.Errorf("a release of mesh-sdk re-pinned %d modules (%v), want g on each of the two branches, which requires it with no pin", n, err)
}
now, _ := moduleOf(t, mods, "g")
if now.Fp2 == "" || now.Fp2 == g.Fp2 || now.Packages["git.novox.be/novox/mesh-sdk/go"] != "v0.1.27" {
t.Fatalf("g after the re-pin = %s pinned %v, want a new fingerprint pinned to v0.1.27 (was %s)", now.Fp2, now.Packages, g.Fp2)
}
for branch, src := range sources {
rev, kept, found, err := readNewestOf(ctx, ix.Revisions, branch)
if err != nil || !found || rev.Revision != 2 || rev.Source != src || rev.Call != "release of mesh-sdk 0.1.27" ||
rev.CallKind != meshcatalog.CatalogRevisionCallKindRelease || rev.Module != "mesh-sdk" || rev.Version != "0.1.27" || // H-A-215
len(rev.Movers) != 1 || rev.Movers[0].Module != "g" || rev.Movers[0].Fingerprint != now.Fp2 {
t.Errorf("%s's newest = %+v (%v), want revision 2 of source %d, its call the release, g its mover at %s", branch, rev, err, src, now.Fp2)
continue
}
if i := slices.IndexFunc(kept, func(e meshcatalog.CatalogEntry) bool { return e.Module == "g" }); i < 0 || kept[i].Fp != now.Fp2 {
t.Errorf("%s's newest holds no entry of g at %s", branch, now.Fp2)
}
}
if len(said) != 2 || said[0].Call != "release of mesh-sdk 0.1.27" || said[1].Call != said[0].Call ||
said[0].CallKind != meshcatalog.CatalogRevisionEventCallKindRelease || said[1].Module != "mesh-sdk" || said[1].Version != "0.1.27" {
t.Errorf("said %+v, want catalog-revision of each branch with the release's call", said)
}
}
@@ -272,7 +272,7 @@ func (ix *Indexer) RecomputeByHand(ctx context.Context, by, why string) TakenIn
r.Answer = "the newest catalog graph revision cannot be read: " + err.Error()
return r
}
if err := ix.RecomputeKept(ctx, fmt.Sprintf("a hand act by %s (%s)", by, why)); err != nil {
if err := ix.Recompute(ctx); err != nil {
r.Answer = "the recompute failed: " + err.Error()
return r
}
@@ -167,24 +167,18 @@ func revisionOfSource(ctx context.Context, o Objects, branch string, source int6
}
}
slices.SortFunc(found, func(a, b int64) int { return int(b - a) })
// A re-pin or a take-in again keeps a later revision of the same source (H-A-211, H-A-213): the source's own revision is
// the lowest numbered of it, the take-in's.
lowest := ""
for _, r := range found {
name := RevisionObject(meshcatalog.CatalogRevision{Branch: branch, Revision: r})
rev, _, err := ReadRevision(ctx, o, name)
if err != nil {
return "", err
}
if rev.Source == source {
return name, nil
}
if rev.Source < source {
break
}
if rev.Source == source {
lowest = name
}
}
if lowest != "" {
return lowest, nil
}
return "", fmt.Errorf("no revision of %q was taken in from the source revision %d", branch, source)
}
@@ -1,78 +0,0 @@
package index
import (
"context"
"errors"
"fmt"
"strings"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
"git.novox.be/novox/mesh-sdk/go/process"
)
// TakeInAgainTool takes a repository in again at its kept merge commit (design 47 H-A-208, H-A-210): a rule change of the
// take-in (such as the walk of the module graph, H-A-201) reaches a repository with no merge since.
const TakeInAgainTool = "indexer_take_in_again"
// TakeInAgainCall is the call of the catalog revision a take-in again keeps (design 47 H-A-212, H-A-213 (3)).
func TakeInAgainCall(repository string) string { return "take-in again of " + repository }
// TakeInAgain re-reads the repository's module graph at the merge commit its taken key keeps (wholeAt, the reading of every
// take-in: the kept requirements of each of its modules, H-A-201, H-A-157) and applies it as a take-in does, whose recompute
// of the modules state is the one a publish re-pin runs (recomputeLocked): pins and fingerprints move as they would there.
// It keeps the result as the trunk's next catalog revision of the same source, its call TakeInAgainCall (H-A-212, H-A-213),
// and asks no gate. A repository with no taken key is refused, said in the answer.
func (ix *Indexer) TakeInAgain(ctx context.Context, repository string) TakenIn {
repository = strings.TrimSpace(repository)
r := TakenIn{Module: repository, Repository: repository}
refuse := func(format string, a ...any) TakenIn {
r.Answer = "refused: " + fmt.Sprintf(format, a...)
ix.logf("take-in again of %s: %s", repository, r.Answer)
return r
}
if ix.Taken == nil || ix.Modules == nil {
return refuse("the indexer keeps no taken merges or no modules, so nothing is taken in again")
}
key, err := trunk.RepositoryKey(repository)
if err != nil {
return refuse("%q names no repository as owner/name", repository)
}
value, _, found, err := ix.Taken.Get(ctx, key)
if errors.Is(err, process.ErrNotYet) {
return refuse("the state taken is not on the bus yet")
}
if err != nil {
return refuse("taken.%s cannot be read: %v", key, err)
}
if !found {
return refuse("the indexer keeps no merge of %s (no key taken.%s): an unknown repository, or one never taken in", repository, key)
}
t, err := takein.DecodeTaken(value)
if err != nil || t.Merge == "" {
return refuse("taken.%s names no merge commit", key)
}
ix.pendingMu.Lock() // one take-in step at a time (pending.go)
defer ix.pendingMu.Unlock()
f, err := ix.wholeAt(ctx, repository, t.Merge)
if err != nil {
return refuse("%v", err)
}
f.Branch = TrunkBranch
applied, err := ix.ApplyMessage(ctx, f)
if err != nil {
return refuse("taking %s at %.12s in again: %v", repository, t.Merge, err)
}
// The take-in again keeps its revision as a re-pin does: the branch's next, of the same source, its call naming the
// repository, its movers every module whose fingerprint moved (design 47 H-A-212, H-A-213).
if err := ix.keepRepinned(ctx, ix, TrunkBranch, takeInAgainCall(repository), nil); err != nil {
return refuse("taken in again at %.12s, but its catalog revision is not kept: %v", t.Merge, err)
}
r.Stored, r.Commit = true, t.Merge
r.Answer = fmt.Sprintf("taken in again at %.12s: %d modules written", t.Merge, len(applied.Written))
if len(applied.Written) > 0 {
r.Answer += " (" + strings.Join(applied.Written, ", ") + ")"
}
ix.logf("take-in again of %s: %s", repository, r.Answer)
return r
}
@@ -1,113 +0,0 @@
package index
import (
"context"
"maps"
"slices"
"strings"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
)
// Design 47 H-A-208, H-A-210: a repository taken in before the walk of the module graph (H-A-201) keeps its modules' direct
// requirements only; indexer_take_in_again re-reads it at its kept merge commit and recomputes the modules state, so the
// module's kept requirements and pins gain the transitive package, and keeps the trunk's next catalog revision of the same
// source with the call "take-in again of <repository>" (H-A-212, H-A-213). An unknown repository is refused.
func TestATakeInAgainReachesARepositoryWithNoMergeSince(t *testing.T) {
const conditions = "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
objects := map[string]string{"modules/b/builds": strings.Repeat("b", 40), "modules/c/client": strings.Repeat("c", 40)}
ix, mods, _ := ptpIndexer(t, objects)
ix.Taken, ix.Revisions, ix.Packages = newMem(), newMemObjects(1<<20), newMem()
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
direct := map[string]string{"modules/g": "module g\n\nrequire git.novox.be/novox/mesh-delivery/modules/b/builds" + ptpZero + "\n"}
walked := map[string]string{"modules/c/client": "module git.novox.be/novox/mesh-delivery/modules/c/client\n\nrequire " + conditions + ptpZero + "\n"}
goMods := direct // before the rule: the client's go.mod reads as requiring nothing
ix.GoMod = func(_ context.Context, _, _, dir string) ([]byte, error) {
if c, ok := goMods[dir]; ok {
return []byte(c), nil
}
if dir == "modules/c/client" {
return []byte("module git.novox.be/novox/mesh-delivery/modules/c/client\n"), nil
}
return []byte(ptpGoMods[dir]), nil
}
f := withManifests(mergedMsg("novox/mesh-delivery", map[string]string{"modules/g": "gtree"}), map[string]string{"modules/g": ptpProgram},
map[string]map[string]string{"modules/g": {"go.sum": avahiGoSum}})
var readAt []string
ix.Read = func(_ context.Context, repository, sha, _ string) (RepoRead, error) {
readAt = append(readAt, repository+"@"+sha)
return RepoRead{Root: f.Root, Trees: f.Trees, Manifests: f.Manifests, Locks: f.Locks, Packages: f.Packages}, nil
}
ctx := context.Background()
for p, v := range map[string]string{conditions: "0.1.4", "git.novox.be/novox/mesh-sdk/go": "0.1.27"} {
if _, err := ix.HearVersion(ctx, HeardVersion{Package: p, GoModule: p, Version: v, Published: true}); err != nil {
t.Fatal(err)
}
}
// the modules that hold the two packages, so their published versions pin them (fingerprint.HolderOf)
mods.put("mesh-sdk", mustEncode(t, meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-sdk", Commit: strings.Repeat("1", 40),
Path: meshcatalogclient.RootPath, None2: None2Pending}))
mods.put("meshconditionsclient", mustEncode(t, meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-contracts", Commit: strings.Repeat("2", 40),
Path: "health-and-repair/go/clients/meshconditionsclient", None2: None2Pending}))
if err := ix.Apply(ctx, f); err != nil {
t.Fatal(err)
}
before, _ := moduleOf(t, mods, "g")
if before.Inputs2 == nil || slices.Contains(before.Inputs2.Packages, conditions) || strings.Contains(before.None2, conditions) {
t.Fatalf("before: packages = %+v (%s), want no transitive requirement yet", before.Inputs2, before.None2)
}
entries, err := ix.viewEntries(ctx)
if err != nil {
t.Fatal(err)
}
if err := ix.KeepBranchEntries(ctx, TrunkBranch, 4, entries, nil); err != nil {
t.Fatal(err)
}
if r := ix.TakeInAgain(ctx, "novox/unknown"); r.Stored || !strings.HasPrefix(r.Answer, "refused:") {
t.Errorf("unknown repository: %+v, want refused", r)
}
goMods = maps.Clone(direct)
maps.Copy(goMods, walked)
r := ix.TakeInAgain(ctx, "novox/mesh-delivery")
if !r.Stored || r.Commit != merge {
t.Fatalf("take-in again = %+v, want stored at the kept merge %s", r, merge)
}
if len(readAt) == 0 || readAt[0] != "novox/mesh-delivery@"+merge {
t.Errorf("read at %v, want the kept merge commit", readAt)
}
m, _ := moduleOf(t, mods, "g")
if m.Inputs2 == nil || !slices.Contains(m.Inputs2.Packages, conditions) {
t.Errorf("after: kept requirements = %+v, want %s", m.Inputs2, conditions)
}
// The recompute pins it at its published version, which moves the fingerprint (H-A-167, H-A-204), and the take-in again
// keeps the trunk's next revision of the same source, its call naming the repository, g its mover (H-A-212, H-A-213).
if m.Fp2 == "" || m.Fp2 == before.Fp2 || m.Packages[conditions] != "v0.1.4" {
t.Errorf("after: g = %s (%s) pinned %v, want a new fingerprint pinned to v0.1.4 (was %s)", m.Fp2, m.None2, m.Packages, before.Fp2)
}
rev, _, found, err := readNewestOf(ctx, ix.Revisions, TrunkBranch)
if err != nil || !found || rev.Revision != 2 || rev.Source != 4 || rev.Call != "take-in again of novox/mesh-delivery" ||
rev.CallKind != meshcatalog.CatalogRevisionCallKindTakeInAgain || rev.Repository != "novox/mesh-delivery" || // H-A-215
len(rev.Movers) != 1 || rev.Movers[0].Module != "g" || rev.Movers[0].Fingerprint != m.Fp2 {
t.Errorf("the newest trunk revision = %+v (%v), want revision 2 of source 4 with the take-in again's call and g its mover", rev, err)
}
// H-A-217: a recompute that moves no fingerprint keeps nothing; one that moves one (a toolchain reported) keeps the trunk's
// next revision of the same source, its call-kind recompute, its call naming the reason, the moved modules its movers.
if err := ix.RecomputeKept(ctx, "the assignments of a build module"); err != nil {
t.Fatal(err)
}
if rev, _, _, _ := readNewestOf(ctx, ix.Revisions, TrunkBranch); rev.Revision != 2 {
t.Errorf("a recompute that moved nothing kept revision %d (%s), want none", rev.Revision, rev.Call)
}
reportToolchainRecipe(t, ix, "mesh-build-go", "novox", "go", "go1.27.1", "2") // a recipe change moves a Go module at once
moved, _ := moduleOf(t, mods, "g")
rev, _, found, err = readNewestOf(ctx, ix.Revisions, TrunkBranch)
if err != nil || !found || rev.Revision != 3 || rev.Source != 4 || rev.CallKind != meshcatalog.CatalogRevisionCallKindRecompute ||
rev.Call != RecomputeCall("a toolchain reported") || !slices.ContainsFunc(rev.Movers, func(e meshcatalog.TakeInEntry) bool {
return e.Module == "g" && e.Fingerprint == moved.Fp2 && moved.Fp2 != m.Fp2
}) {
t.Errorf("after a toolchain reported: the newest trunk revision = %+v (%v), want revision 3 of source 4, kind recompute, g its mover at %s (was %s)",
rev, err, moved.Fp2, m.Fp2)
}
}
@@ -46,8 +46,6 @@ type TakeInAsk struct {
HeldSource int64 `json:"held-source,omitempty"`
Skip bool `json:"skip,omitempty"`
NamedMember string `json:"named-member,omitempty"`
// Again names the repository taken in again at its kept merge commit (indexer_take_in_again, take_in_again.go; H-A-208, H-A-210).
Again string `json:"again,omitempty"`
// RunDrop asks one run on request, by its group run-<id>, to be dropped with its candidate (indexer_run_drop, runs_tools.go, A-194).
RunDrop string `json:"run-drop,omitempty"`
// Registry asks one of the registrations' acts (registrations.go): a trusted change of one, the verb registrations,
@@ -2,11 +2,9 @@ package index
import (
"context"
"encoding/json"
"errors"
"fmt"
"maps"
"slices"
"strconv"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
@@ -27,10 +25,9 @@ func TakenID(branch string, source int64) string {
// TakeInSource takes one source revision of a watched branch in, as the gate's source-moved names it (design 47 H §743): for each
// repository it moves, the trees at the named commit through the seat git (wholeAt) and the fingerprints of the modules
// registered for it; its movers (moversOf) against the branch's newest catalog revision; one catalog revision of the branch,
// the branch's next, <branch>.<n> of source <source>, carrying the movers and pinned for the take-in's pipeline when it has one
// (keepBranchEntries, H-A-213); and
// <branch>.<source>, carrying the movers and pinned for the take-in's pipeline when it has one (KeepBranchEntries); and
// take-in.taken once, with branch, source, revision, call and one entry per mover; a branch's first revision (start) has no
// mover (§768). A revision already kept of that source (a
// mover (§768). A revision already kept under that name (a
// restart before the acknowledgement) is said again from its kept movers, recomputing nothing (H-A-92). The first watched
// branch is taken in on the indexer's own view; another on its newest revision's, computed and kept only as that revision.
// An error leaves the source-moved held, written into the state held behind its branch's earlier ones (ErrHeldUnwritten when
@@ -60,13 +57,13 @@ func (ix *Indexer) takeInSource(ctx context.Context, e meshdelivery.SourceMovedE
}
ix.pendingMu.Lock()
defer ix.pendingMu.Unlock()
// A source revision kept already is found by the revision's field source, never by its number (H-A-213 (1)).
if kept, found, err := takenOfSource(ctx, ix.Revisions, e.Branch, e.Source); err != nil {
return taken, err
} else if found {
name := BranchObject(e.Branch, e.Source)
if kept, _, err := ReadRevision(ctx, ix.Revisions, name); err == nil {
taken.Revision, taken.Entries = kept.Revision, kept.Movers
ix.logf("take-in: source revision %d of %s is kept already as %s: take-in.taken said again from its %d movers", e.Source, e.Branch, RevisionObject(kept), len(kept.Movers))
ix.logf("take-in: %s is kept already: take-in.taken said again from its %d movers", name, len(kept.Movers))
return taken, ix.sayTaken(ctx, taken)
} else if !errors.Is(err, ErrNoObject) {
return taken, err
}
_, before, _, err := readNewestOf(ctx, ix.Revisions, e.Branch)
if err != nil {
@@ -109,15 +106,14 @@ func (ix *Indexer) takeInSource(ctx context.Context, e meshdelivery.SourceMovedE
if e.Start {
movers = nil // a branch's first revision moves nothing: its entries are the base later take-ins are held against (§768, H-A-19)
}
number, err := ix.keepBranchEntries(ctx, e.Branch, e.Source, takeInCall(e.Call), entries, movers)
if err != nil {
if err := ix.KeepBranchEntries(ctx, e.Branch, e.Source, entries, movers); err != nil {
if errors.Is(err, ErrObjectsFull) {
ix.heldFull.Store(true) // room is said once the store has room again (RoomAgain), and wakes the take-in (§744)
}
return taken, err
}
taken.Revision, taken.Entries = number, movers
ix.logf("take-in: source revision %d of %s (call %s) taken in as %s, %d repositories, %d movers", e.Source, e.Branch, e.Call, BranchObject(e.Branch, number), len(e.Heads), len(movers))
taken.Revision, taken.Entries = e.Source, movers
ix.logf("take-in: source revision %d of %s (call %s) taken in as %s, %d repositories, %d movers", e.Source, e.Branch, e.Call, name, len(e.Heads), len(movers))
return taken, ix.sayTaken(ctx, taken)
}
@@ -161,57 +157,22 @@ func (ix *Indexer) SourceToTake(ctx context.Context, branch string, a meshdelive
if ix.Revisions == nil {
return e, false, errors.New("the indexer keeps no catalog revisions")
}
// The gate's source a branch is taken in at is its newest revision's field source, never the revision's number (H-A-213 (1)).
newest, found, err := newestOf(ctx, ix.Revisions, branch)
names, err := ix.Revisions.Names(ctx)
if err != nil {
return e, false, fmt.Errorf("the newest catalog revision of %s: %w", branch, err)
return e, false, fmt.Errorf("the objects %s: %w", CatalogRevisionsObjects, err)
}
if found && newest.Source >= a.Revision {
var kept int64
for _, n := range names {
if r, ok := BranchRevisionOf(branch, n); ok && r > kept {
kept = r
}
}
if kept >= a.Revision {
return e, false, nil
}
e.Start = !found
e.Start = kept == 0
for _, r := range a.Repositories {
e.Heads = append(e.Heads, meshdelivery.SourceMovedEventHeadsItem{Repository: r.Repository, Commit: r.Head})
}
return e, true, nil
}
// takenOfSource is the revision a take-in of source src kept on branch: of the branch's revisions whose field source is src,
// the lowest numbered (a re-pin or a take-in again keeps the same source on a later number, H-A-211, H-A-213). The revisions
// are read from the newest down while their source is not below src; none is false.
func takenOfSource(ctx context.Context, o Objects, branch string, src int64) (meshcatalog.CatalogRevision, bool, error) {
names, err := o.Names(ctx)
if err != nil {
return meshcatalog.CatalogRevision{}, false, fmt.Errorf("the objects %s: %w", CatalogRevisionsObjects, err)
}
var numbers []int64
for _, n := range names {
if r, ok := BranchRevisionOf(branch, n); ok {
numbers = append(numbers, r)
}
}
slices.Sort(numbers)
var kept meshcatalog.CatalogRevision
found := false
for i := len(numbers) - 1; i >= 0; i-- {
name := BranchObject(branch, numbers[i])
data, err := o.Get(ctx, name)
if errors.Is(err, ErrNoObject) {
continue // removed under the read by the pin judge
}
if err != nil {
return kept, false, fmt.Errorf("revision %s: %w", name, err)
}
var rev meshcatalog.CatalogRevision
if err := json.Unmarshal(data, &rev); err != nil {
return kept, false, fmt.Errorf("%s: %w", name, err)
}
if rev.Source < src {
break
}
if rev.Source == src {
kept, found = rev, true
}
}
return kept, found, nil
}
@@ -43,8 +43,7 @@ func takeInIndexer(t *testing.T) (*Indexer, *memObjects, *[]meshcatalog.TakeInTa
return ix, objects, said, f
}
// Design 47 H §743, H-A-32, H-A-92, H-A-213 (1): a take-in writes one catalog revision <branch>.<n>, n the branch's newest + 1
// and its source the gate's, carrying its movers, pins it for the
// Design 47 H §743, H-A-32, H-A-92: a take-in writes one catalog revision <branch>.<source> carrying its movers, pins it for the
// take-in's pipeline and says take-in.taken once; a redelivery writes nothing again and says the same; a revision with no mover
// pins nothing and says no entries; the old form is refused.
func TestASourceRevisionIsTakenInAsOneRevision(t *testing.T) {
@@ -57,20 +56,19 @@ func TestASourceRevisionIsTakenInAsOneRevision(t *testing.T) {
t.Fatal(err)
}
if len(taken.Entries) != 1 || taken.Entries[0].Module != "avahi" || taken.Entries[0].Fingerprint == "" || taken.Entries[0].Commit != f.Merge ||
taken.Revision != 1 || taken.Branch != "main" || taken.Source != 7 || taken.Call != moved.Call {
t.Fatalf("taken %+v, want revision 1 (the branch's first) of source 7 of main with avahi as its one mover", taken)
taken.Revision != 7 || taken.Branch != "main" || taken.Source != 7 || taken.Call != moved.Call {
t.Fatalf("taken %+v, want revision 7 of main with avahi as its one mover", taken)
}
rev, _, err := ReadRevision(ctx, objects, "main.1")
if err != nil || !reflect.DeepEqual(rev.Movers, taken.Entries) || rev.Source != 7 ||
rev.Call != moved.Call || rev.CallKind != meshcatalog.CatalogRevisionCallKindTakeIn { // H-A-215
t.Fatalf("main.1 = %+v (%v), want it kept carrying the movers", rev, err)
rev, _, err := ReadRevision(ctx, objects, "main.7")
if err != nil || !reflect.DeepEqual(rev.Movers, taken.Entries) || rev.Source != 7 {
t.Fatalf("main.7 = %+v (%v), want it kept carrying the movers", rev, err)
}
if v, _, found, _ := ix.Pins.Get(ctx, PinKey("main", 1)); !found {
t.Error("main.1 is not pinned for its pipeline")
if v, _, found, _ := ix.Pins.Get(ctx, PinKey("main", 7)); !found {
t.Error("main.7 is not pinned for its pipeline")
} else {
var pv PinValue
if json.Unmarshal(v, &pv) != nil || !reflect.DeepEqual(pv.Records, []string{"pipeline.main.1"}) {
t.Errorf("main.1 is pinned by %s, want pipeline.main.1", v)
if json.Unmarshal(v, &pv) != nil || !reflect.DeepEqual(pv.Records, []string{"pipeline.main.7"}) {
t.Errorf("main.7 is pinned by %s, want pipeline.main.7", v)
}
}
puts := objects.puts
@@ -81,14 +79,14 @@ func TestASourceRevisionIsTakenInAsOneRevision(t *testing.T) {
same := moved
same.Source, same.Call = 8, "fedcba9876543210"
none, err := ix.TakeInSource(ctx, same)
if err != nil || len(none.Entries) != 0 || none.Revision != 2 || none.Source != 8 {
t.Errorf("a revision moving no fingerprint said %+v (%v), want revision 2 of source 8 with no entries", none, err)
if err != nil || len(none.Entries) != 0 || none.Revision != 8 {
t.Errorf("a revision moving no fingerprint said %+v (%v), want revision 8 with no entries", none, err)
}
if _, _, found, _ := ix.Pins.Get(ctx, PinKey("main", 2)); found {
if _, _, found, _ := ix.Pins.Get(ctx, PinKey("main", 8)); found {
t.Error("a take-in with no mover pinned its revision")
}
if _, _, err := ReadRevision(ctx, objects, "main.2"); err != nil {
t.Errorf("main.2 is not kept: %v", err)
if _, _, err := ReadRevision(ctx, objects, "main.8"); err != nil {
t.Errorf("main.8 is not kept: %v", err)
}
if len(*said) != 3 {
t.Errorf("said %d take-in.taken, want 3", len(*said))
@@ -79,22 +79,9 @@ func (ix *Indexer) ApplyToolchain(ctx context.Context, module, key string, value
return ix.ToolchainReported(ctx)
}
// ToolchainReported is what a changed toolchain moves: the modules written again, kept as a revision (RecomputeKept).
// ToolchainReported is what a changed toolchain moves: the modules written again (Recompute).
func (ix *Indexer) ToolchainReported(ctx context.Context) error {
return ix.RecomputeKept(ctx, "a toolchain reported")
}
// RecomputeKept is Recompute kept as a re-pin is (design 47 H-A-217): when it moved at least one fingerprint against the
// trunk's newest revision, it keeps the trunk's next revision of the same source, its movers the modules that moved, its
// call-kind recompute and its call naming reason, pinned for its pipeline, and says catalog-revision; one that moves none
// keeps nothing. Recompute writes the trunk's modules state, so only the trunk's revision is kept.
func (ix *Indexer) RecomputeKept(ctx context.Context, reason string) error {
ix.pendingMu.Lock() // one take-in step at a time, so the trunk's newest stays the one compared against (pending.go)
defer ix.pendingMu.Unlock()
if err := ix.Recompute(ctx); err != nil {
return err
}
return ix.keepRepinned(ctx, ix, TrunkBranch, recomputeCall(reason), nil)
return ix.Recompute(ctx)
}
// agreed is the toolchain of each language every reporting node agrees on, or why there is none.
@@ -311,7 +298,7 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc
r.fp, r.none = f.fp, f.none
if r.fp != "" {
// a pin is a released version, never a commit (H-A-161), and the version line's fingerprint takes it (H-A-167)
r.fp, r.pkgs = pinnedOf(f, series, k.manifest, k.cur.Repository, kept.Packages)
r.fp, r.pkgs = pinnedOf(f, series, k.manifest)
r.tcs = toolchainsUsed(toolchains)
}
}
@@ -530,7 +517,7 @@ func stillComputed(module string, kept meshcatalog.ModulesByModuleValueInputs2,
if f.none != "" || f.fp == "" {
return "", nil, false
}
fp, pkgs := pinnedOf(f, series, k.manifest, k.held.Repository, kept.Packages)
fp, pkgs := pinnedOf(f, series, k.manifest)
return fp, pkgs, fp == k.held.Fp2
}
@@ -557,7 +544,7 @@ func (ix *Indexer) RecomputeAtStart(ctx context.Context) error {
ix.logf("no toolchain is read yet: the new-form fingerprints are computed when one is")
return nil
}
return ix.RecomputeKept(ctx, "the indexer's start")
return ix.Recompute(ctx)
}
// fillPaths gives every module whose key names no directory (a key written before the indexer learned paths) the one its
@@ -611,128 +598,29 @@ func importsOf(pkgs sdkstep.PackageVersions, repository string) bool {
return false
}
// ReleaseCall is the call of the catalog revision a re-pin keeps: the release of a module at a version (design 47 H-A-213 (2)).
func ReleaseCall(module, version string) string { return "release of " + module + " " + version }
// Repin moves the pins of every module whose packages hold a package of module (a held module's directory holds it) to
// module's newest released version, the pass of H-A-161 for those modules alone: a release of module at version starts it
// (design 47 H-A-165), so pins move on the event and never wait for a merge or a hand act. It runs on every watched branch
// holding such a module: the first on the indexer's own modules state, another on its newest revision's picture (as a
// take-in does), and keeps on each whose re-pinned fingerprints moved the branch's next catalog revision, of the same
// source as its newest, with the call ReleaseCall and those modules as its movers, pinned for the release's pipeline, and
// says catalog-revision; no take-in.taken (H-A-211, H-A-212, H-A-213). It answers how many modules it named.
func (ix *Indexer) Repin(ctx context.Context, module, version string) (int, error) {
// module's newest released version, the pass of H-A-161 for those modules alone: a version-released of module starts it
// (design 47 H-A-165), so pins move on the event and never wait for a merge or a hand act. It answers how many it named.
func (ix *Indexer) Repin(ctx context.Context, module string) (int, error) {
if ix.Modules == nil || ix.Manifests == nil {
return 0, nil
}
ix.pendingMu.Lock() // one take-in step at a time, so the branch's newest stays the one re-pinned (pending.go)
defer ix.pendingMu.Unlock()
branches, err := ix.watchedBranches(ctx)
held, err := ix.heldModules(ctx)
if err != nil {
return 0, err
}
if !slices.Contains(branches, TrunkBranch) {
branches = append([]string{TrunkBranch}, branches...) // the modules state is re-pinned whether a revision is kept or not
}
named := 0
for _, branch := range branches {
x := ix
if !isTrunkBranch(branch) {
view, nodes, _, err := BranchView(ctx, ix.Revisions, branch)
if err != nil {
return named, err
}
d := ix.dry()
d.ix.Modules, d.ix.Graph, d.ix.Taken = view, nodes, newOverlay(nil)
d.ix.Logf = func(format string, args ...any) { ix.logf("re-pin of "+branch+": "+format, args...) }
x = d.ix
}
only, err := x.repinned(ctx, module)
if err != nil {
return named, err
}
if len(only) == 0 {
continue
}
named += len(only)
x.recompute.Lock()
err = x.recomputeLocked(ctx, nil, nil, "", only)
x.recompute.Unlock()
if err != nil {
return named, err
}
if err := ix.keepRepinned(ctx, x, branch, releaseCall(module, version), only); err != nil {
return named, err
}
}
return named, nil
}
// repinned are the modules the indexer's modules state holds whose packages name a package of module: by their pins, and
// by their kept requirements, which name a package they hold no pin of yet (design 47 H-A-204, H-A-165, H-A-167).
func (ix *Indexer) repinned(ctx context.Context, module string) (map[string]bool, error) {
held, err := ix.heldModules(ctx)
if err != nil {
return nil, err
}
only := map[string]bool{}
for key, m := range held {
names := slices.Collect(maps.Keys(m.Packages))
if m.Inputs2 != nil {
names = append(names, m.Inputs2.Packages...)
}
for _, p := range names {
for p := range m.Packages {
if moduleOfKey(fingerprint.HolderOf(p, held)) == module {
only[key] = true
}
}
}
return only, nil
}
// keepRepinned keeps x's view, re-pinned for the modules only names (every module for nil), as branch's next catalog revision
// of the source of its newest, with call, when a fingerprint of them moved against that newest; its movers are those that
// moved (H-A-211, H-A-212, H-A-213). A branch with no revision kept yet keeps none: there is no source to keep it of.
func (ix *Indexer) keepRepinned(ctx context.Context, x *Indexer, branch string, call revisionCall, only map[string]bool) error {
if ix.Revisions == nil {
return nil
if len(only) == 0 {
return 0, nil
}
newest, before, found, err := readNewestOf(ctx, ix.Revisions, branch)
if err != nil {
return fmt.Errorf("the newest catalog revision of %s: %w", branch, err)
}
if !found {
ix.logf("re-pin (%s): %s keeps no revision yet, so none is kept for it", call.Call, branch)
return nil
}
entries, err := x.viewEntries(ctx)
if err != nil {
return err
}
was := make(map[string]string, len(before))
for _, e := range before {
was[e.Module] = e.Fp
}
var movers []meshcatalog.TakeInEntry
moved := slices.Sorted(maps.Keys(only))
if only == nil { // a take-in again or a recompute: every module whose fingerprint moved (H-A-212, H-A-217)
moved = slices.Sorted(maps.Keys(entries))
}
for _, key := range moved {
e, ok := entries[key]
if !ok || e.Fp == "" || was[e.Module] == e.Fp {
continue
}
movers = append(movers, meshcatalog.TakeInEntry{Module: e.Module, Commit: e.Commit, Fingerprint: e.Fp, Repository: e.Repository})
}
if len(movers) == 0 {
ix.logf("re-pin (%s): no fingerprint of %s moved, so no revision is kept", call.Call, branch)
return nil
}
number, err := ix.keepBranchEntries(ctx, branch, newest.Source, call, entries, movers)
if err != nil {
return err
}
ix.logf("re-pin (%s): %s kept as revision %d of source %d, %d movers", call.Call, branch, number, newest.Source, len(movers))
return nil
ix.recompute.Lock()
defer ix.recompute.Unlock()
return len(only), ix.recomputeLocked(ctx, nil, nil, "", only)
}
@@ -504,7 +504,7 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { //
}
return first.KeepBranchEntries(context.Background(), TrunkBranch, source, entries, nil)
}
if err := takeIn(6); err != nil || len(firstSaid) != 1 || firstSaid[0].Revision != 1 ||
if err := takeIn(6); err != nil || len(firstSaid) != 1 || firstSaid[0].Revision != 6 ||
firstSaid[0].Entries != int64(len(builderModules(t))) {
t.Fatalf("the take-in said %+v (%v), want trunk revision 1 with every module", firstSaid, err)
}
@@ -519,7 +519,7 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { //
if err := takeIn(7); err != nil || len(firstSaid) != 2 || !reflect.DeepEqual(firstSaid[1].Changed, []string{"avahi"}) {
t.Errorf("the next take-in after a write said %+v (%v), want revision 7 naming avahi", firstSaid, err)
}
if view, _, at, err := TrunkView(context.Background(), first.Revisions); err != nil || at != 2 {
if view, _, at, err := TrunkView(context.Background(), first.Revisions); err != nil || at != 7 {
t.Errorf("the view at a restart is read at %d (%v), want 7", at, err)
} else if raw, _, _, _ := view.Get(context.Background(), viewKey); !strings.Contains(string(raw), "a field the judge refuses") {
t.Errorf("the view at a restart lost the verdict: %s", raw)
@@ -536,21 +536,21 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { //
t.Errorf("the first-view tools %+v serve no modules", FirstViewTools)
}
if a, err := views.FirstView(testSelf, ModulesState, trunkStates.State(ModulesState), trunkStates.State(views.MarksState), "", time.Now()); err != nil ||
a.Revision != 2 || len(a.Values) != len(builderModules(t)) {
t.Errorf("the first view modules answered %d values at %d (%v), want every module at revision 2", len(a.Values), a.Revision, err)
a.Revision != 7 || len(a.Values) != len(builderModules(t)) {
t.Errorf("the first view modules answered %d values at %d (%v), want every module at revision 7", len(a.Values), a.Revision, err)
} else if m, err := meshcatalogclient.DecodeModule(a.Values[viewKey]); err != nil || m.Manifest == "" {
t.Errorf("the first view's avahi = %s (%v), want the module's value naming its manifest", a.Values[viewKey], err)
}
if e, ok, err := trunkStates.State(views.MarksState).Get("graph"); err != nil || !ok || !strings.Contains(string(e.Value), `"revision":2`) {
t.Errorf("the bundle's mark of graph = %+v %v (%v), want revision 2", e, ok, err)
if e, ok, err := trunkStates.State(views.MarksState).Get("graph"); err != nil || !ok || !strings.Contains(string(e.Value), `"revision":7`) {
t.Errorf("the bundle's mark of graph = %+v %v (%v), want revision 7", e, ok, err)
}
// A kept view reads the revision once while it stays the newest, and its first view answers in pages of values, never
// one revision read per key (about 11 s a page live, 2026-11-14).
reads := 0
keptView := TrunkStates{Objects: first.Revisions, Under: func(string) views.Source { return nil }, Kept: &KeptTrunk{Say: func(string, ...any) { reads++ }}}
if a, err := views.FirstView(testSelf, ModulesState, keptView.State(ModulesState), keptView.State(views.MarksState), "", time.Now()); err != nil ||
a.Revision != 2 || len(a.Values) != len(builderModules(t)) || a.Next != "" {
t.Errorf("the kept first view modules answered %d values at %d, next %q (%v), want every module at revision 2 in one page", len(a.Values), a.Revision, a.Next, err)
a.Revision != 7 || len(a.Values) != len(builderModules(t)) || a.Next != "" {
t.Errorf("the kept first view modules answered %d values at %d, next %q (%v), want every module at revision 7 in one page", len(a.Values), a.Revision, a.Next, err)
}
if _, _, err := keptView.State("graph").Get("avahi"); err != nil || reads != 1 {
t.Errorf("the kept view read the revision %d times (%v), want once", reads, err)
@@ -822,7 +822,7 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) {
if _, err := ix.HearVersion(context.Background(), HeardVersion{Package: "mesh-sdk/go", Seat: "go-package-registry", Version: "0.2.0", Commit: strings.Repeat("1", 40), GoModule: sdk, Published: true}); err != nil {
t.Fatal(err)
}
if _, err := ix.Repin(context.Background(), "mesh-sdk", "0.1.27"); err != nil { // the pass for avahi alone: mesh-tools is held stale here
if _, err := ix.Repin(context.Background(), "mesh-sdk"); err != nil { // the pass for avahi alone: mesh-tools is held stale here
t.Fatal(err)
}
if b, _ := moduleOf(t, mods, "avahi"); b.Fp2 != a.Fp2 || b.Packages[sdk] != "v0.1.18" {
@@ -15,7 +15,6 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/keys"
@@ -374,25 +373,13 @@ func isFloor(raw []byte) bool {
// within the series the module accepts, the one its manifest m names or the default (Manifest.AcceptedSeries), whether
// or not that series holds contents yet; a publish in another series pins nothing and moves nothing, and a module's
// release number is never a pin. A package with none published in that series is kept with no version, so its kind-1
// edge stands (H §746) and its build waits for the publish (H-A-165). A module of the module's own repository is never
// pinned, whatever a build reported: it is built from the tree (design 47 H-A-198, H-A-201, H-A-202). nil for none.
//
// required are the mesh packages the module's go.mod graph requires (its kept packages, H-A-201): each of another
// repository is named whether or not the form's versions name it, with no version while none is published, never left
// out, so the builder waits for its publish instead of the build module refusing (design 47 H-A-204, H-A-165).
func releasedPins(versions sdkstep.PackageVersions, series map[string]SeriesPackage, m catalogue.Manifest, repository string, required []string) sdkstep.PackageVersions {
names := slices.Collect(maps.Keys(versions))
if repository != "" {
names = append(names, fingerprint.OtherPackages(required, repository)...)
}
if len(names) == 0 {
// edge stands (H §746) and its build waits for the publish (H-A-165). nil for none.
func releasedPins(versions sdkstep.PackageVersions, series map[string]SeriesPackage, m catalogue.Manifest) sdkstep.PackageVersions {
if len(versions) == 0 {
return nil
}
out := sdkstep.PackageVersions{}
for _, p := range names {
if repo, _, ok := trunk.PackageRepository(p); ok && repository != "" && trunk.SameRepository(repo, repository) {
continue
}
for p := range versions {
out[p] = ""
for _, s := range series {
if s.GoModule != p || s.Newest == nil || s.Series != m.AcceptedSeries(s.Name) {
@@ -411,8 +398,8 @@ func releasedPins(versions sdkstep.PackageVersions, series map[string]SeriesPack
// form's fingerprint takes them (design 47 H-A-167, H-A-172; ADR 0573 §4) and a re-pin moves it, the module a mover
// built again against the release; a fingerprint with no pin is the form's as it is. The held fingerprint of a build that
// stands is kept while its pins are the ones it was built with, else its import form takes the new pins.
func pinnedOf(f form, series map[string]SeriesPackage, m catalogue.Manifest, repository string, required []string) (string, sdkstep.PackageVersions) {
pins := releasedPins(f.versions, series, m, repository, required)
func pinnedOf(f form, series map[string]SeriesPackage, m catalogue.Manifest) (string, sdkstep.PackageVersions) {
pins := releasedPins(f.versions, series, m)
if f.stands {
if maps.Equal(f.built, map[string]string(pins)) {
return f.fp, pins
@@ -451,7 +438,7 @@ func PinsOf(packages map[string]string) sdkstep.PackageVersions {
// repin is Repin of a release of module at version, said in the journal; its failure is the caller's error.
func (ix *Indexer) repin(ctx context.Context, module, version string) error {
n, err := ix.Repin(ctx, module, version)
n, err := ix.Repin(ctx, module)
if err != nil {
return fmt.Errorf("the modules pinned to %s are not re-pinned to %s: %w", module, version, err)
}
@@ -102,7 +102,7 @@ func TestAnEntryNoRevisionListsStaysWhileAWantNamesItAndGoesAfterTheWantMovesOn(
n := NewNamed(newMem(), nil)
ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), EntryIndex: newMem(), NamedSets: n, Now: func() time.Time { return namedBase }}
n.Changed = ix.RemoveUnlisted
if err := ix.KeepBranchEntries(ctx, TrunkBranch, 1, map[string]meshcatalog.CatalogEntry{"avahi": {Module: "avahi", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}}}, nil); err != nil {
if err := ix.keepBranchEntries(ctx, TrunkBranch, 1, map[string]meshcatalog.CatalogEntry{"avahi": {Module: "avahi", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}}}, nil); err != nil {
t.Fatal(err)
}
if err := ix.Revisions.Delete(ctx, "main.1"); err != nil { // the revision that listed it is removed
@@ -213,7 +213,7 @@ func TestNoEntryIsRemovedWhileAStartReadWasEmptyOrAnEventWasNotApplied(t *testin
fp := namedFp("c")
setup := func(n *Named) (*Indexer, func() bool) {
ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), EntryIndex: newMem(), NamedSets: n, Now: func() time.Time { return namedBase }}
if err := ix.KeepBranchEntries(ctx, TrunkBranch, 1, map[string]meshcatalog.CatalogEntry{"m": {Module: "m", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}}}, nil); err != nil {
if err := ix.keepBranchEntries(ctx, TrunkBranch, 1, map[string]meshcatalog.CatalogEntry{"m": {Module: "m", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}}}, nil); err != nil {
t.Fatal(err)
}
if err := ix.Revisions.Delete(ctx, "main.1"); err != nil {
+5 -11
View File
@@ -42,8 +42,6 @@ type artifact struct {
Binary string `json:"binary"`
Loads []string `json:"loads"`
Env map[string]string `json:"env"`
Paths []string `json:"paths"`
Series string `json:"series"`
}
type stream struct {
@@ -205,9 +203,9 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
index.FingerprintsAgainTool, // issue 1966
// milestone 47/G W4.12 (design 47, the indexer's tools): the catalog graph revisions read and steered.
index.CatalogRevisionTool, index.CatalogRevisionsTool, index.CandidateGraphsTool, index.RecomputeTool, index.DropCandidateGraphTool,
index.EntryIndexTool, index.EntryPinsTool, index.NamedDropTool, // design 47 G §675, H-A-204
index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.PinDropTool, index.PendingTool, index.TakeInPendingTool, index.TakeInAgainTool, index.TakeInSkipTool, index.RunsTool, index.RunDropTool) // design 47 G §667, §8, §579, §64 (A-194)
if !reflect.DeepEqual(m.Tools, want) || len(want) != 44 || !slices.Contains(m.Tools, "modules") {
index.EntryIndexTool, index.NamedDropTool, // design 47 G §675
index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.PinDropTool, index.PendingTool, index.TakeInPendingTool, index.TakeInSkipTool, index.RunsTool, index.RunDropTool) // design 47 G §667, §8, §579, §64 (A-194)
if !reflect.DeepEqual(m.Tools, want) || len(want) != 42 || !slices.Contains(m.Tools, "modules") {
t.Errorf("tools = %v, want %v", m.Tools, want)
}
// Design 47 G §2, §9: every version of every registration is kept with no age limit, 64 MiB, on the old streams section.
@@ -346,12 +344,8 @@ func TestManifestProcess(t *testing.T) { // 1238/AC1
if _, bus := p["bus"]; bus {
t.Errorf("the process declares a bus: it runs on its own broker credential")
}
if len(m.Build.Artifacts) != 3 {
t.Fatalf("artifacts = %+v, want the program, the tools and the package takein", m.Build.Artifacts)
}
// H-A-186 (design 47 G §370): takein, which the gate, the view and the watchman import, is a declared package.
if a := m.Build.Artifacts[2]; a.Name != "takein" || a.Kind != "package" || a.Language != "go" || !reflect.DeepEqual(a.Paths, []string{"takein"}) || a.Series != "0.1" {
t.Errorf("package artifact = %+v", a)
if len(m.Build.Artifacts) != 2 {
t.Fatalf("artifacts = %+v, want the program and the tools", m.Build.Artifacts)
}
if a := m.Build.Artifacts[1]; a.Name != "tools" || a.Kind != "bundle" || a.Binary != "mesh-delivery-indexer-tools" || !reflect.DeepEqual(a.Loads, []string{"mesh-delivery-indexer-tools"}) {
t.Errorf("tools artifact = %+v", a) // 1945
-11
View File
@@ -57,7 +57,6 @@
"indexer_recompute",
"indexer_drop_candidate_graph",
"indexer_entries",
"indexer_entry",
"indexer_named_drop",
"indexer_proposals",
"indexer_proposal_drop",
@@ -67,7 +66,6 @@
"indexer_pin_drop",
"indexer_pending",
"indexer_take_in_pending",
"indexer_take_in_again",
"indexer_take_in_skip",
"indexer_runs",
"indexer_run_drop"
@@ -225,15 +223,6 @@
"loads": [
"mesh-delivery-indexer-tools"
]
},
{
"name": "takein",
"kind": "package",
"language": "go",
"paths": [
"takein"
],
"series": "0.1"
}
]
}
@@ -19,7 +19,7 @@ import (
// §549): runner_unlogged lists them, runner_unlogged_append appends them by hand, written in the hand-act log; it is of class
// act and takes §391's default callers.
func UnloggedTools(pipelines stdio.KeptState, log stdio.KeptLog, acts views.HandActs, logf func(string, ...any)) []stdio.Tool {
h := &pipeline.Hearer{Store: &pipeline.MoverStore{Raw: stdioPipelines{pipelines}, Logf: logf}, Logf: logf,
h := &pipeline.Hearer{Store: stdioPipelines{pipelines}, Logf: logf,
Log: func(e seat.LogEntry) error {
_, err := log.Append(keys.EscapeName(e.Pipeline), e)
return err
@@ -67,36 +67,26 @@ func UnloggedTools(pipelines stdio.KeptState, log stdio.KeptLog, acts views.Hand
}
}
// stdioPipelines is the state pipelines as bytes, as the runtime serves it, for the by-hand append: read and written through
// a pipeline.MoverStore, which keeps each mover's steps under its own key (H-A-187).
// stdioPipelines is the state pipelines as the runtime serves it, for the by-hand append.
type stdioPipelines struct{ s stdio.KeptState }
func (s stdioPipelines) Get(key string) ([]byte, uint64, error) {
func (s stdioPipelines) Get(key string) (seat.Pipeline, uint64, error) {
e, err := s.s.Get(key)
if err != nil || e == nil {
return nil, 0, err
return seat.Pipeline{}, 0, errors.Join(err, errors.New("no such pipeline"))
}
return e.Value, e.Revision, nil
var p seat.Pipeline
return p, e.Revision, json.Unmarshal(e.Value, &p)
}
func (s stdioPipelines) Create(key string, value []byte) (uint64, error) {
r, err := s.s.Create(key, json.RawMessage(value))
if err != nil && strings.Contains(err.Error(), "exists") {
return 0, pipeline.ErrExists
}
return r, err
}
func (s stdioPipelines) Update(key string, value []byte, rev uint64) (uint64, error) {
r, err := s.s.Update(key, json.RawMessage(value), rev)
func (s stdioPipelines) Update(key string, p seat.Pipeline, rev uint64) (uint64, error) {
r, err := s.s.Update(key, p, rev)
if err != nil && (strings.Contains(err.Error(), "wrong last sequence") || strings.Contains(err.Error(), "revision")) {
return 0, pipeline.ErrChanged
}
return r, err
}
func (s stdioPipelines) Delete(key string) error { return s.s.Delete(key) }
func (s stdioPipelines) Keys(filter string) ([]string, error) {
all, err := s.s.Keys()
var out []string

Some files were not shown because too many files have changed in this diff Show More