Compare commits
32
Commits
go/v0.1.76
...
go/v0.1.78
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b9cc97470b | ||
|
|
ae91f91e88 | ||
|
|
4664eb1b00 | ||
|
|
d9a3458927 | ||
|
|
b058389d46 | ||
|
|
6c65e82926 | ||
|
|
d40f314e52 | ||
|
|
fc85f7b936 | ||
|
|
37a8f7a83e | ||
|
|
9ef830d224 | ||
|
|
120a0554d6 | ||
|
|
d8a900dac0 | ||
|
|
a62f7b40c1 | ||
|
|
429f03d4cf | ||
|
|
20d59ff4f0 | ||
|
|
12b18721b9 | ||
|
|
3183e0e056 | ||
|
|
30ccad4edc | ||
|
|
4e585f02b1 | ||
|
|
f2d3172eac | ||
|
|
e4781edcd6 | ||
|
|
1db38fcc72 | ||
|
|
eba4309b19 | ||
|
|
a7fc37561e | ||
|
|
e93575ffe7 | ||
|
|
859f88b2f0 | ||
|
|
517a4b3cf1 | ||
|
|
7f88916863 | ||
|
|
5dee4cb263 | ||
|
|
77b683c1c1 | ||
|
|
60e32772a5 | ||
|
|
fd2ef6c7e8 |
@@ -95,15 +95,17 @@ type machineEnd struct {
|
||||
|
||||
// run runs one composition step and answers what it came to: per machine not away it joins or starts the candidate's
|
||||
// composition run (§718), waits for each run's end in its key, and judges each machine from it; failed when a machine does not
|
||||
// compose, could not run when a machine could not be composed or was not judged, else passed with a warning per machine away.
|
||||
// compose, could not run when a machine could not be composed or was not judged, nothing to run when no node places the module on
|
||||
// the branch (H-A-179), else passed with a warning per machine away.
|
||||
func (r *compositionRun) run(ctx context.Context, id string, step meshwants.StepRun) stepOutcome {
|
||||
candidate, err := candidateKeyOf(step)
|
||||
if err != nil {
|
||||
return couldNotRun(err.Error())
|
||||
}
|
||||
nodes := r.inputs.machines(step.Module, step.Branch)
|
||||
if len(nodes) == 0 {
|
||||
return couldNotRun(fmt.Sprintf("no node places %s on the branch %q, so the composition step has no machine to compose", step.Module, step.Branch))
|
||||
if len(nodes) == 0 { // a mover no machine holds on the branch has nothing to judge: nothing to run, passed (H-A-179, ADR 0682 §8)
|
||||
return stepOutcome{Outcome: meshwants.StepEndedOutcomeNothingToRun,
|
||||
Said: fmt.Sprintf("no node places %s on the branch %q, so the composition step has no machine to compose", step.Module, step.Branch)}
|
||||
}
|
||||
reader := readerOf(id, step)
|
||||
prep := &runPrep{r: r, step: step}
|
||||
|
||||
@@ -259,7 +259,11 @@ func TestTheCompositionStepRunsOneAskPerMachine(t *testing.T) {
|
||||
}
|
||||
none := newRunStand(t, "src2:e1", "src2:e1", "d1")
|
||||
none.r.inputs.machines = func(string, string) []string { return machines }
|
||||
if out := none.r.run(context.Background(), "ask-step", step); out.Outcome != meshwants.StepEndedOutcomeCouldNotRun ||
|
||||
outcome := meshwants.StepEndedOutcomeCouldNotRun
|
||||
if len(machines) == 0 { // a mover no machine holds: nothing to run, saying why (H-A-179)
|
||||
outcome = meshwants.StepEndedOutcomeNothingToRun
|
||||
}
|
||||
if out := none.r.run(context.Background(), "ask-step", step); out.Outcome != outcome ||
|
||||
!strings.Contains(out.Said, want) || (len(machines) > 0 && !strings.Contains(out.Said, "shanks is away")) || len(none.asks) != 0 {
|
||||
t.Errorf("%s came to %+v, %d asks", name, out, len(none.asks))
|
||||
}
|
||||
|
||||
@@ -19,6 +19,7 @@ import (
|
||||
"runtime"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -203,11 +204,17 @@ func (w *watchKV) Keys(context.Context, ...jetstream.WatchOpt) ([]string, error)
|
||||
return nil, jetstream.ErrNoKeysFound
|
||||
}
|
||||
|
||||
// putIntoMu holds putInto's writes one at a time: the composition step starts its machines' runs side by side, and each may
|
||||
// raise a finding into the same map.
|
||||
var putIntoMu sync.Mutex
|
||||
|
||||
// putInto is an assign.Findings Put that keeps each finding in into, as the finding writer would, by its key and its words.
|
||||
func putInto(into map[string][]byte) func(mcseat.Finding) error {
|
||||
return func(f mcseat.Finding) error {
|
||||
raw, err := json.Marshal(f)
|
||||
putIntoMu.Lock()
|
||||
into[views.FindingKey(f.Scope, f.ID, f.Kind)] = raw
|
||||
putIntoMu.Unlock()
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -119,6 +119,13 @@ func TestManifest(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
// H-A-186 (design 47 G §370): asks, which the gate, the teller and the view import, is a declared Go package of the
|
||||
// module, so every release publishes it; H-A-188: in the module's version series, 0.3.
|
||||
if !slices.ContainsFunc(m.Build.Artifacts, func(a catalogue.Artifact) bool {
|
||||
return a.Name == "asks" && a.Kind == catalogue.ArtifactPackage && a.Language == "go" && slices.Equal(a.Paths, []string{"asks"}) && a.Series == m.Version && m.Version == "0.3"
|
||||
}) {
|
||||
t.Errorf("artifacts = %+v, want the Go package asks of series 0.3, the module's", m.Build.Artifacts)
|
||||
}
|
||||
// ADR 0635 §5: its views family is its ordinary events, `views.<state>` among its emits, on no stream of its own.
|
||||
// The seat's four events are said on the seat's own subject (the seat's family), so the module keeps no stream for them
|
||||
// and lists none among its emits.
|
||||
|
||||
@@ -370,6 +370,15 @@
|
||||
"loads": [
|
||||
"mesh-delivery-assigner"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "asks",
|
||||
"kind": "package",
|
||||
"language": "go",
|
||||
"paths": [
|
||||
"asks"
|
||||
],
|
||||
"series": "0.3"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -45,7 +45,7 @@ func (m *entryManifests) keep(e meshcatalog.CatalogEntry) error {
|
||||
if m.byFp == nil {
|
||||
m.byFp = map[string]catalogue.Manifest{}
|
||||
}
|
||||
m.byFp[meshcatalog.BareManifestDigest(e.ManifestDigest)] = mf
|
||||
m.byFp[meshcatalogclient.BareManifestDigest(e.ManifestDigest)] = mf
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -228,6 +228,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return err
|
||||
}
|
||||
b.PackageWait = time.Duration(packageWait) * time.Minute
|
||||
// Before it waits for an unpinned package, the build reads what the Go package registry seat holds of it (H-A-194).
|
||||
b.PublishedVersions = publishedVersionsVia(calls.Call)
|
||||
waitsBucket := process.AwaitBucket(ctx, js, conn, builds.StateBucket(cred.Module, build.WaitsState), "the toolchain waits are kept in memory only, and builder_waits shows none", logger.Printf)
|
||||
b.Waits = newWaitsKept(waitsBucket, func(ctx context.Context, event, id string, body any) error {
|
||||
return emitEvent(context.WithoutCancel(ctx), js, cred, event, id, body)
|
||||
@@ -242,9 +244,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return emitSeatEvent(context.WithoutCancel(ctx), js, cred, meshbuilds.SubjectPackagePublished, "package-published-"+p.Module+"-"+p.Version+"-"+p.Artifact, p)
|
||||
}
|
||||
b.Reserved = reserved
|
||||
// A release heard, or the newest the builder's record keeps, pins a package an entry names with none (H-A-165).
|
||||
reserved.heardReleased = b.VersionReleased
|
||||
b.NewestReleased = reserved.r.NewestReleased
|
||||
// The last versions-filled the stream keeps is read at the start (design 47 G §369, §372): a restart does not lose it.
|
||||
reserved.readFilled(ctx, lastKeptOf(js, cred, logger.Printf), sleeps)
|
||||
// A state bound later numbers the builds looked at without it.
|
||||
|
||||
@@ -44,15 +44,12 @@ type reservations struct {
|
||||
// built asks a done build's publish (design 47 G A-457a), released ends its item; nil are none.
|
||||
built func(ctx context.Context, module, version string) error
|
||||
released func(ctx context.Context, module, version string)
|
||||
// heardReleased is told each version-released taken in (design 47 H H-A-165): a step ask waiting for a package of the
|
||||
// module wakes. Nil tells nobody.
|
||||
heardReleased func(module, version string)
|
||||
filled atomic.Bool
|
||||
bounds func() jetstream.KeyValue
|
||||
found *findings
|
||||
bound time.Duration
|
||||
wake func(context.Context)
|
||||
logf func(string, ...any)
|
||||
filled atomic.Bool
|
||||
bounds func() jetstream.KeyValue
|
||||
found *findings
|
||||
bound time.Duration
|
||||
wake func(context.Context)
|
||||
logf func(string, ...any)
|
||||
// call asks a seat's verb: the git seat's tag (design 47 G §370).
|
||||
call toolCaller
|
||||
// say says package-published on the seat mesh-builds; nil says none.
|
||||
@@ -255,9 +252,6 @@ func (rv *reservations) heard(ctx context.Context, m jetstream.Msg) bool {
|
||||
if rv.released != nil {
|
||||
rv.released(ctx, ev.Module, ev.Version)
|
||||
}
|
||||
if rv.heardReleased != nil {
|
||||
rv.heardReleased(ev.Module, ev.Version)
|
||||
}
|
||||
rv.found.clear(ctx, versions.VersionUnreleased, meshconditionsclient.ScopeModule, ev.Module)
|
||||
default:
|
||||
return false
|
||||
|
||||
@@ -134,15 +134,10 @@ func TestTheCatalogsWordsMoveTheReservations(t *testing.T) {
|
||||
if err != nil || !strings.Contains(got.Summary, "m:b unpublished") || strings.Contains(got.Summary, "m:a unpublished") || !strings.Contains(got.Summary, "1.4.3") {
|
||||
t.Fatalf("finding = %+v (%v), want version-unreleased naming m:b of 1.4.3", got, err)
|
||||
}
|
||||
var heardReleases []string // H-A-165: a release heard wakes a step ask waiting for a package of the module
|
||||
rv.heardReleased = func(m, v string) { heardReleases = append(heardReleases, m+" "+v) }
|
||||
released := &wakeMsg{subject: meshcatalog.SubjectVersionReleased, data: []byte(`{"v":1,"module":"m","version":"1.4.3","source":1,"artifacts":[]}`)}
|
||||
if !rv.heard(ctx, released) || !released.acked {
|
||||
t.Fatalf("version-released was not taken in (acked %v)", released.acked)
|
||||
}
|
||||
if strings.Join(heardReleases, ", ") != "m 1.4.3" {
|
||||
t.Errorf("the release heard was told %v, want m 1.4.3 (H-A-165)", heardReleases)
|
||||
}
|
||||
if strings.Join(items, ", ") != "asked m 1.4.3, ended m 1.4.3" {
|
||||
t.Errorf("publish items %v, want the build's end to ask the publish and the release to end it", items)
|
||||
}
|
||||
|
||||
@@ -105,6 +105,10 @@ func (h *seatHearer) heard(ctx context.Context, m jetstream.Msg) bool {
|
||||
if h.res != nil && se.event == versions.HeardPublished {
|
||||
h.res.published(ctx, o)
|
||||
}
|
||||
if se.event == versions.HeardPublished && o.Kind == versions.KindPackage && h.b != nil {
|
||||
// A package registry seat's published pins a package an entry names with none, at the version it carries (H-A-192).
|
||||
h.b.PackagePublished(o.Name, o.Version)
|
||||
}
|
||||
h.ack(m)
|
||||
if len(wake) > 0 {
|
||||
h.logf("the floor of %s is asked again: %s said %s", strings.Join(wake, ", "), se.seat, se.event)
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
gitseat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/gopackageregistry"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
@@ -96,6 +97,28 @@ func (s awaitedVersions) KeysOf(ctx context.Context, filter string) ([]string, e
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// publishedVersionsVia asks the Go package registry seat's verb versions for one package path (design 47 H H-A-194): its
|
||||
// versions, newest first.
|
||||
func publishedVersionsVia(call toolCaller) func(context.Context, string) ([]string, error) {
|
||||
return func(_ context.Context, path string) ([]string, error) {
|
||||
raw, err := call("seat:"+gopackageregistry.Seat+"."+gopackageregistry.VerbVersions, gopackageregistry.VersionsInput{Name: path})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var a gopackageregistry.VersionsAnswer
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return nil, fmt.Errorf("%s's versions answered what is no list of versions: %w", gopackageregistry.Seat, err)
|
||||
}
|
||||
var out []string
|
||||
for _, v := range a.Versions {
|
||||
if v.Version != "" {
|
||||
out = append(out, v.Version)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
}
|
||||
|
||||
// floorVia is the floor's three sources as calls of the builder's own account (design 51 *The floor*): the seat's verb
|
||||
// versions, the seat git's tags and the requirements the catalog half's entries name (A-119). A call that fails is that source
|
||||
// silent.
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
func TestAnEntryOfTheCatalogHalfIsTheFormTheBuilderBuildsBy(t *testing.T) {
|
||||
e := meshcatalog.CatalogEntry{Module: "netcheck", Repository: "novox/mesh-catalog", Path: "modules/netcheck", Commit: "abc", Fp: "src2:f",
|
||||
ManifestDigest: "sha256:m", Bases: []string{"base"}, Packages: map[string]string{"p": "1"}, Needs: []string{"n"},
|
||||
Artifacts: []meshcatalog.Artifact{{Ref: "netcheck:lib", Kind: "package", Language: "go"}},
|
||||
Inputs: &meshcatalog.EntryInputs{Toolchains: map[string]meshcatalog.ToolchainUsed{"go": {Version: "go1", Recipe: "1"}},
|
||||
Paths: map[string][]string{"lib": {"a/b", "c"}}}}
|
||||
got := ModuleOf(e)
|
||||
@@ -30,4 +31,11 @@ func TestAnEntryOfTheCatalogHalfIsTheFormTheBuilderBuildsBy(t *testing.T) {
|
||||
if bare := ModuleOf(meshcatalog.CatalogEntry{Module: "m"}); bare.Inputs2 != nil || bare.Fp2 != "" {
|
||||
t.Errorf("ModuleOf of an entry with none = %+v, want nothing set", bare)
|
||||
}
|
||||
// A module with an artifact that names no paths (a Go program beside its packages) is built from its directory whole
|
||||
// (ADR 0499 clauses 1 and 4): its fetch names no paths, so the program's go.mod and main package stay in the tree.
|
||||
mixed := e
|
||||
mixed.Artifacts = append([]meshcatalog.Artifact{{Ref: "netcheck:program", Kind: "bundle", Language: "go"}}, e.Artifacts...)
|
||||
if got := ModuleOf(mixed); got.Inputs2 == nil || len(got.Inputs2.Paths) != 0 {
|
||||
t.Errorf("ModuleOf of a module with a program naming no paths = %+v, want bases and no named paths", got.Inputs2)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,11 +5,12 @@ import (
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
)
|
||||
|
||||
// **A build waits for its revision; a build with no version is never made** (design 47 H H-A-155, H-A-156; H-A-141): a step
|
||||
@@ -104,13 +105,16 @@ func (b *Builder) awaitRevision(ctx context.Context, run meshbuilds.StepRun) (ou
|
||||
}
|
||||
}
|
||||
|
||||
// **A package with no pin waits for its release** (design 47 H H-A-165, H-A-161): an entry keeps a mesh package none of
|
||||
// whose versions is released with an empty version. Its step ask waits, as for its revision, for the release of the module
|
||||
// that holds the package (the catalog's version-released, VersionReleased), up to PackageWait (the setting
|
||||
// package-wait-minutes), then builds with that release as the package's pin (v<version>); past the bound the step fails,
|
||||
// naming the package, and nothing is built.
|
||||
// **A package with no pin waits for its publish** (design 47 H H-A-192, H-A-165, H-A-183, H-A-184, H-A-185): an entry
|
||||
// keeps a mesh package the registry has not published within the series its module accepts with an empty version. Its
|
||||
// build, a step ask's or a lane's, first reads the versions the registry seat holds of the path (its verb versions,
|
||||
// H-A-194), then waits for the registry seat's published of that package path in that series (the
|
||||
// series the module's manifest names in `packages`, else the default; catalogue.Manifest.AcceptedSeries), up to
|
||||
// PackageWait (the setting package-wait-minutes), and pins the version that event carries; the builder never derives a
|
||||
// pin (no release number of the module that holds the package). Past the bound the build fails, naming the package, and
|
||||
// nothing is built.
|
||||
|
||||
// DefaultPackageWait is the bound of a step ask's wait for a package's release, its setting package-wait-minutes' default.
|
||||
// DefaultPackageWait is the bound of a build's wait for a package's publish, its setting package-wait-minutes' default.
|
||||
const DefaultPackageWait = 10 * time.Minute
|
||||
|
||||
func (b *Builder) packageWait() time.Duration {
|
||||
@@ -120,91 +124,152 @@ func (b *Builder) packageWait() time.Duration {
|
||||
return DefaultPackageWait
|
||||
}
|
||||
|
||||
// VersionReleased takes the catalog's version-released of module in: a step ask waiting for a package of it wakes.
|
||||
func (b *Builder) VersionReleased(module, version string) {
|
||||
// PackagePublished takes the Go package registry seat's published of one package path in (its name and the version the
|
||||
// event carries, H-A-192): the newest published in the version's series is kept, and a build waiting for the package
|
||||
// wakes. A version that is no MAJOR.MINOR.PATCH (a group's tagged version) pins nothing.
|
||||
func (b *Builder) PackagePublished(path, version string) {
|
||||
v := strings.TrimPrefix(version, "v")
|
||||
series, ok := seriesOfVersion(v)
|
||||
if path == "" || !ok {
|
||||
return
|
||||
}
|
||||
b.init()
|
||||
b.mu.Lock()
|
||||
b.releases[module] = version
|
||||
in := b.publishedPins[path]
|
||||
if in == nil {
|
||||
in = map[string]string{}
|
||||
b.publishedPins[path] = in
|
||||
}
|
||||
if old := in[series]; old == "" || compareVersions(v, old) > 0 {
|
||||
in[series] = v
|
||||
}
|
||||
b.mu.Unlock()
|
||||
b.pictureMoved()
|
||||
}
|
||||
|
||||
// releaseOf is the pin a package is built against: the release of the module holding it, as Go names it; "" for none yet.
|
||||
func (b *Builder) releaseOf(ctx context.Context, pkg string) string {
|
||||
if b.Members == nil {
|
||||
return ""
|
||||
}
|
||||
module, _, found, err := b.Members(ctx, pkg)
|
||||
if err != nil || !found {
|
||||
return ""
|
||||
}
|
||||
// publishedPin is the pin the registry seat's published gave the package path in the series, as Go names it; "" for none.
|
||||
func (b *Builder) publishedPin(path, series string) string {
|
||||
b.mu.Lock()
|
||||
v := b.releases[module]
|
||||
v := b.publishedPins[path][series]
|
||||
b.mu.Unlock()
|
||||
if v == "" && b.NewestReleased != nil {
|
||||
if kept, ok, err := b.NewestReleased(ctx, module); err == nil && ok {
|
||||
v = kept
|
||||
}
|
||||
}
|
||||
if v == "" {
|
||||
return ""
|
||||
}
|
||||
return "v" + v
|
||||
}
|
||||
|
||||
// awaitPins fills each package entry names with no pin from its release, waiting for one; no outcome once every package has
|
||||
// a pin, failed past the bound, could not run at the builder's stop.
|
||||
func (b *Builder) awaitPins(ctx context.Context, entry *meshcatalog.CatalogEntry) (outcome, said string) {
|
||||
// awaitPublished fills each package of packages with no pin from the published of its path within the series the manifest
|
||||
// with this digest accepts, waiting for one (H-A-192, H-A-165). It answers the packages with every pin, or an outcome and
|
||||
// its words: failed past the bound naming the package, could not run when the manifest cannot be read or the builder stops.
|
||||
func (b *Builder) awaitPublished(ctx context.Context, module, fp, digest string, packages map[string]string) (map[string]string, string, string) {
|
||||
var unpinned []string
|
||||
for p, v := range entry.Packages {
|
||||
for p, v := range packages {
|
||||
if v == "" {
|
||||
unpinned = append(unpinned, p)
|
||||
}
|
||||
}
|
||||
if len(unpinned) == 0 {
|
||||
return "", ""
|
||||
return packages, "", ""
|
||||
}
|
||||
slices.Sort(unpinned)
|
||||
if b.Manifests == nil {
|
||||
return nil, meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("the series %s accepts of %s cannot be read: no manifests", module, unpinned[0])
|
||||
}
|
||||
man, err := b.Manifests.Manifest(ctx, digest)
|
||||
if err != nil {
|
||||
return nil, meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("the series %s accepts of %s cannot be read: its manifest %s: %v", module, unpinned[0], digest, err)
|
||||
}
|
||||
// What the registry seat holds is read once before the wait (H-A-194): the publishes heard are a cache of it.
|
||||
for _, p := range unpinned {
|
||||
if b.PublishedVersions == nil || b.publishedPin(p, man.AcceptedSeries(p)) != "" {
|
||||
continue
|
||||
}
|
||||
vs, err := b.PublishedVersions(ctx, p)
|
||||
if err != nil {
|
||||
b.logf("%s at %s: the versions the registry seat holds of %s cannot be read, so the build waits for its publish: %v", module, fp, p, err)
|
||||
continue
|
||||
}
|
||||
for _, v := range vs {
|
||||
b.PackagePublished(p, v)
|
||||
}
|
||||
}
|
||||
bound := b.packageWait()
|
||||
wait, stop := context.WithTimeout(ctx, bound) // a context's bound, no timer (ADR 0437 §8)
|
||||
defer stop()
|
||||
pins := map[string]string{}
|
||||
told := false
|
||||
for {
|
||||
b.mu.Lock()
|
||||
moved := b.moved
|
||||
b.mu.Unlock()
|
||||
missing := ""
|
||||
pins := map[string]string{}
|
||||
missing, missingSeries := "", ""
|
||||
for _, p := range unpinned {
|
||||
if pins[p] == "" {
|
||||
pins[p] = b.releaseOf(ctx, p)
|
||||
}
|
||||
series := man.AcceptedSeries(p)
|
||||
pins[p] = b.publishedPin(p, series)
|
||||
if pins[p] == "" && missing == "" {
|
||||
missing = p
|
||||
missing, missingSeries = p, series
|
||||
}
|
||||
}
|
||||
if missing == "" {
|
||||
packages := maps.Clone(entry.Packages)
|
||||
maps.Copy(packages, pins)
|
||||
entry.Packages = packages
|
||||
out := maps.Clone(packages)
|
||||
maps.Copy(out, pins)
|
||||
if told {
|
||||
b.logf("%s at %s: every package it names has a released version: built with %v", entry.Module, entry.Fp, pins)
|
||||
b.logf("%s at %s: every package it names is published: built with %v", module, fp, pins)
|
||||
}
|
||||
return "", ""
|
||||
return out, "", ""
|
||||
}
|
||||
notReleased := fmt.Sprintf("package %s has no released version", missing)
|
||||
notReleased := fmt.Sprintf("package %s has no released version in the series %s", missing, missingSeries)
|
||||
if !told {
|
||||
b.logf("%s at %s: %s yet: the ask waits for its release, at most %s", entry.Module, entry.Fp, notReleased, bound)
|
||||
b.logf("%s at %s: %s yet: the build waits for its publish, at most %s", module, fp, notReleased, bound)
|
||||
told = true
|
||||
}
|
||||
select {
|
||||
case <-moved:
|
||||
case <-wait.Done():
|
||||
if ctx.Err() != nil {
|
||||
return meshbuilds.StepEndedOutcomeCouldNotRun, notReleased + ": the builder stopped before it was"
|
||||
return nil, meshbuilds.StepEndedOutcomeCouldNotRun, notReleased + ": the builder stopped before it was"
|
||||
}
|
||||
b.logf("%s at %s: %s within %s: the step fails, nothing is built", entry.Module, entry.Fp, notReleased, bound)
|
||||
return meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s within %s", notReleased, bound)
|
||||
b.logf("%s at %s: %s within %s: the build fails, nothing is built", module, fp, notReleased, bound)
|
||||
return nil, meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s within %s", notReleased, bound)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// seriesOfVersion is a MAJOR.MINOR.PATCH version's series, MAJOR.MINOR; false for any other form.
|
||||
func seriesOfVersion(v string) (string, bool) {
|
||||
if _, ok := versionParts(v); !ok {
|
||||
return "", false
|
||||
}
|
||||
major, rest, _ := strings.Cut(v, ".")
|
||||
minor, _, _ := strings.Cut(rest, ".")
|
||||
return major + "." + minor, true
|
||||
}
|
||||
|
||||
// versionParts are a MAJOR.MINOR.PATCH version's numbers; false for any other form.
|
||||
func versionParts(v string) ([3]int, bool) {
|
||||
var out [3]int
|
||||
parts := strings.Split(v, ".")
|
||||
if len(parts) != 3 {
|
||||
return out, false
|
||||
}
|
||||
for i, p := range parts {
|
||||
n, err := strconv.Atoi(p)
|
||||
if err != nil || n < 0 {
|
||||
return out, false
|
||||
}
|
||||
out[i] = n
|
||||
}
|
||||
return out, true
|
||||
}
|
||||
|
||||
// compareVersions orders two MAJOR.MINOR.PATCH versions numerically.
|
||||
func compareVersions(a, b string) int {
|
||||
x, _ := versionParts(a)
|
||||
y, _ := versionParts(b)
|
||||
for i := range x {
|
||||
if x[i] != y[i] {
|
||||
return x[i] - y[i]
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
@@ -113,11 +113,13 @@ type Builder struct {
|
||||
// RevisionWait bounds a step ask's wait for its revision, its setting revision-wait-minutes (H-A-156); DefaultRevisionWait
|
||||
// when 0.
|
||||
RevisionWait time.Duration
|
||||
// PackageWait bounds a step ask's wait for the release of a package its entry names with no pin, its setting
|
||||
// package-wait-minutes (H-A-165); DefaultPackageWait when 0. NewestReleased reads the newest version the builder's
|
||||
// record of releases keeps for a module; nil reads none, and only a release heard (VersionReleased) gives a pin.
|
||||
PackageWait time.Duration
|
||||
NewestReleased func(ctx context.Context, module string) (version string, found bool, err error)
|
||||
// PackageWait bounds a build's wait for the publish of a package its entry names with no pin, its setting
|
||||
// package-wait-minutes (H-A-165); DefaultPackageWait when 0. Only a registry seat's published heard (PackagePublished)
|
||||
// gives a pin (H-A-192).
|
||||
PackageWait time.Duration
|
||||
// PublishedVersions asks the Go package registry seat's verb versions for one package path, its versions newest first
|
||||
// (H-A-194): read once before a build waits for that package. Nil reads none.
|
||||
PublishedVersions func(ctx context.Context, path string) ([]string, error)
|
||||
// Forms answers what each node's engine states it reads (ADR 0618 §2, ADR 0624 §1): present, the nodes whose engine
|
||||
// answered when last asked, and counted, every node the assignments names, present or away, a node with no statement
|
||||
// stating nothing (the baseline's first forms). A done key names each artifact in the forms these keep, the oldest in
|
||||
@@ -162,8 +164,9 @@ type Builder struct {
|
||||
builds map[string]context.CancelCauseFunc
|
||||
// moved is closed and made again at each entry the picture takes in or drops: the step asks waiting for a revision wake.
|
||||
moved chan struct{}
|
||||
// releases are the versions heard released, by module (VersionReleased): a package with no pin waits for one.
|
||||
releases map[string]string
|
||||
// publishedPins are the newest version a registry seat said published, by package path and series (PackagePublished):
|
||||
// a package with no pin waits for one (H-A-192).
|
||||
publishedPins map[string]map[string]string
|
||||
}
|
||||
|
||||
type planned struct {
|
||||
@@ -194,7 +197,7 @@ func (b *Builder) init() {
|
||||
b.hands, b.events = map[string]*handWatch{}, map[string]bool{}
|
||||
b.builds = map[string]context.CancelCauseFunc{}
|
||||
b.moved = make(chan struct{})
|
||||
b.releases = map[string]string{}
|
||||
b.publishedPins = map[string]map[string]string{}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -285,16 +288,19 @@ func (b *Builder) take(ctx context.Context, o Origin, key string, m meshcatalog.
|
||||
|
||||
// ModuleOf is a catalog entry in the form the builder builds by: its fingerprint, manifest digest, source (repository,
|
||||
// directory, commit; no source stamp, design 47 H H-A-48), the toolchains its fingerprint was computed with, the bases it is
|
||||
// built against, its packages and needs, and the named paths of its artifacts.
|
||||
// built against, its packages and needs, and the named paths of its artifacts when every artifact names them.
|
||||
func ModuleOf(e meshcatalog.CatalogEntry) meshcatalog.ModulesByModuleValue {
|
||||
m := meshcatalog.ModulesByModuleValue{Repository: e.Repository, Path: e.Path, Commit: e.Commit, Fp2: e.Fp, None2: e.None,
|
||||
Manifest: meshcatalog.BareManifestDigest(e.ManifestDigest), Packages: e.Packages, Needs: e.Needs}
|
||||
Manifest: meshcatalogclient.BareManifestDigest(e.ManifestDigest), Packages: e.Packages, Needs: e.Needs}
|
||||
if len(e.Bases) > 0 {
|
||||
m.Inputs2 = &meshcatalog.ModulesByModuleValueInputs2{Bases: e.Bases}
|
||||
}
|
||||
if in := e.Inputs; in != nil {
|
||||
m.Toolchains, m.Problems = in.Toolchains, in.Problems
|
||||
if len(in.Paths) > 0 {
|
||||
// Only a module whose every artifact names its paths is built from those alone (ADR 0499 clauses 1, 2 and 4): an
|
||||
// artifact without paths, such as a Go program, keeps its module's directory as its build source, so a module
|
||||
// with one names no paths and its fetch keeps the directory whole.
|
||||
if len(in.Paths) > 0 && everyArtifactNamesPaths(e) {
|
||||
if m.Inputs2 == nil {
|
||||
m.Inputs2 = &meshcatalog.ModulesByModuleValueInputs2{}
|
||||
}
|
||||
@@ -309,6 +315,20 @@ func ModuleOf(e meshcatalog.CatalogEntry) meshcatalog.ModulesByModuleValue {
|
||||
return m
|
||||
}
|
||||
|
||||
// everyArtifactNamesPaths is whether the entry names artifacts and the named paths of each, by its reference or its name.
|
||||
func everyArtifactNamesPaths(e meshcatalog.CatalogEntry) bool {
|
||||
if len(e.Artifacts) == 0 || e.Inputs == nil {
|
||||
return false
|
||||
}
|
||||
for _, a := range e.Artifacts {
|
||||
_, name, _ := strings.Cut(a.Ref, ":")
|
||||
if len(e.Inputs.Paths[a.Ref]) == 0 && len(e.Inputs.Paths[name]) == 0 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// originOf is the catalog revision the entry of a build key came from; a key no entry named is of the first branch, as a build
|
||||
// of the trunk's value was before branches were watched.
|
||||
func (b *Builder) originOf(key string) Origin {
|
||||
@@ -1112,9 +1132,15 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m mes
|
||||
ask := sdkstep.StepAsk{ID: sdkstep.NewID(), Kind: kind, Language: language, Asker: b.OwnModule, At: time.Now(), Version: l.version, Build: job}
|
||||
if language == sdkstep.AskGo && job.Fetch != nil && len(m.Packages) > 0 {
|
||||
// The versions of the mesh packages the fingerprint was computed with (novox/hq ADR 0573 §2): the build module writes
|
||||
// them into the workspace, whatever the repository's go.mod names.
|
||||
ask.Packages = maps.Clone(m.Packages)
|
||||
maps.DeleteFunc(ask.Packages, func(_, v string) bool { return v == "" }) // a package with no released version is no pin (H-A-161)
|
||||
// them into the workspace, whatever the repository's go.mod names. A package the entry names with no pin waits for
|
||||
// its publish and is built with the version that publish carries; past the bound the lane fails naming it, and no
|
||||
// ask is put (H-A-192, H-A-165).
|
||||
pinned, outcome, said := b.awaitPublished(ctx, module, m.Fp2, m.Manifest, m.Packages)
|
||||
if outcome != "" {
|
||||
l.err = errors.New(said)
|
||||
return
|
||||
}
|
||||
ask.Packages = maps.Clone(pinned)
|
||||
for _, fb := range req.Builds {
|
||||
delete(ask.Packages, fb.Package) // a package `builds` names is named in neither packages nor package (§465)
|
||||
}
|
||||
|
||||
@@ -333,8 +333,10 @@ func TestATrunkFingerprintIsOneAskOnTheDeliveryStream(t *testing.T) {
|
||||
r := newRig(t, goBundle)
|
||||
r.asks.Answer = nil // no node has pulled it yet
|
||||
// Issue 1585 (ADR 0573 §2): the ask carries the module's pins; a package the entry keeps with no version (none
|
||||
// released, design 47 H-A-161) is no pin.
|
||||
sdk := buildstep.PackageVersions{"git.novox.be/novox/mesh-sdk/go": "v0.1.20"}
|
||||
// published, design 47 H-A-161) is pinned at the version the registry seat's published carries (H-A-192), never
|
||||
// asked without it.
|
||||
r.b.PackagePublished("git.novox.be/novox/mesh-contracts/go/seats/x", "v0.1.7")
|
||||
sdk := buildstep.PackageVersions{"git.novox.be/novox/mesh-sdk/go": "v0.1.20", "git.novox.be/novox/mesh-contracts/go/seats/x": "v0.1.7"}
|
||||
data, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20",
|
||||
Fp2: fp1, Manifest: mdigest('a'), Path: "modules/m", Toolchains: CatalogToolchains(agreedToolchains),
|
||||
Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.20", "git.novox.be/novox/mesh-contracts/go/seats/x": ""}})
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
)
|
||||
@@ -63,10 +64,12 @@ func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry) (outc
|
||||
if entry.Module != e.Module || (e.Commit != "" && entry.Commit != e.Commit) {
|
||||
return meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("the entry of %s at %s is of %s at %s", e.Module, e.Fingerprint, entry.Module, entry.Commit)
|
||||
}
|
||||
// A package the entry names with no pin is built against its release, waited for (H-A-165).
|
||||
if outcome, said := b.awaitPins(ctx, &entry); outcome != "" {
|
||||
// A package the entry names with no pin is built against its publish, waited for (H-A-192, H-A-165).
|
||||
pinned, outcome, said := b.awaitPublished(ctx, entry.Module, entry.Fp, meshcatalogclient.BareManifestDigest(entry.ManifestDigest), entry.Packages)
|
||||
if outcome != "" {
|
||||
return outcome, said
|
||||
}
|
||||
entry.Packages = pinned
|
||||
key, err := builds.BuilderBuildKey(entry.Module, entry.Fp)
|
||||
if err != nil {
|
||||
return meshbuilds.StepEndedOutcomeCouldNotRun, err.Error()
|
||||
@@ -79,8 +82,8 @@ func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry) (outc
|
||||
}
|
||||
return meshbuilds.StepEndedOutcomeFailed, failedWords
|
||||
}
|
||||
m := meshcatalog.ModulesByModuleValue{Repository: entry.Repository, Commit: entry.Commit, Fp2: entry.Fp, Manifest: meshcatalog.BareManifestDigest(entry.ManifestDigest), Path: entry.Path,
|
||||
Toolchains: CatalogToolchains(b.agreedFor(ctx, meshcatalog.BareManifestDigest(entry.ManifestDigest))), Packages: entry.Packages}
|
||||
m := meshcatalog.ModulesByModuleValue{Repository: entry.Repository, Commit: entry.Commit, Fp2: entry.Fp, Manifest: meshcatalogclient.BareManifestDigest(entry.ManifestDigest), Path: entry.Path,
|
||||
Toolchains: CatalogToolchains(b.agreedFor(ctx, meshcatalogclient.BareManifestDigest(entry.ManifestDigest))), Packages: entry.Packages}
|
||||
b.consider(ctx, entry.Module, m)
|
||||
select {
|
||||
case log := <-ended:
|
||||
@@ -111,7 +114,7 @@ func (b *Builder) passedWithItsVersion(ctx context.Context, entry meshcatalog.Ca
|
||||
if !reserving {
|
||||
return meshbuilds.StepEndedOutcomePassed, ""
|
||||
}
|
||||
plan, planErr, _ := b.planOf(ctx, meshcatalog.BareManifestDigest(entry.ManifestDigest))
|
||||
plan, planErr, _ := b.planOf(ctx, meshcatalogclient.BareManifestDigest(entry.ManifestDigest))
|
||||
switch {
|
||||
case planErr != nil:
|
||||
return meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("%s at %s: the plan cannot be read to judge its version: %v", entry.Module, entry.Fp, planErr)
|
||||
|
||||
@@ -3,9 +3,10 @@ package build
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -121,76 +122,155 @@ func TestABuildStepPassesAReservingBuildOnlyWithItsReservedVersion(t *testing.T)
|
||||
t.Run("H-A-162: a revision held that changes no entry wakes the ask", aRevisionHeldWakesTheAsk)
|
||||
t.Run("H-A-165: a package with no pin waits for its release and is built with it", aPackageWithNoPinWaitsForItsRelease)
|
||||
t.Run("H-A-165: past its bound a package with no release fails the step", aPackageWithNoReleaseFailsPastItsBound)
|
||||
t.Run("H-A-192: a lane waits for an unpinned package's publish and fails past its bound", aLaneWaitsForAnUnpinnedPackage)
|
||||
t.Run("H-A-194: what the registry seat holds pins the package before any wait", theSeatsVersionsPinBeforeTheWait)
|
||||
}
|
||||
|
||||
// unpinnedRig is a rig whose step entry m names the package x with no version, built by the module contracts.
|
||||
func unpinnedRig(t *testing.T) (*rig, *fakeReservations) {
|
||||
// unpinnedRig is a rig whose step entry m names the package x with no version. Its manifest names no `packages`, so it
|
||||
// accepts x in the default series 0.1. waiting is told when a build starts its wait for a package's publish.
|
||||
func unpinnedRig(t *testing.T) (*rig, meshcatalog.CatalogEntry, <-chan struct{}) {
|
||||
var held atomic.Int64
|
||||
r, res, entry := revisionRig(t, &held, make(chan struct{}, 1))
|
||||
r, _, entry := revisionRig(t, &held, make(chan struct{}, 1))
|
||||
r.b.Picture = nil // no revision to wait for here
|
||||
entry.Packages = map[string]string{"git.novox.be/novox/mesh-contracts/go/seats/x": "", "git.novox.be/novox/mesh-sdk/go": "v0.1.20"}
|
||||
entry.Packages = map[string]string{unpinnedX: "", "git.novox.be/novox/mesh-sdk/go": "v0.1.20"}
|
||||
r.b.StepEntry = func(context.Context, string) (meshcatalog.CatalogEntry, error) { return entry, nil }
|
||||
r.b.Members = func(_ context.Context, pkg string) (string, string, bool, error) {
|
||||
return "contracts", "contracts:x", pkg == "git.novox.be/novox/mesh-contracts/go/seats/x", nil
|
||||
waiting := make(chan struct{}, 1)
|
||||
r.b.Logf = func(format string, args ...any) {
|
||||
t.Logf(format, args...)
|
||||
if strings.Contains(fmt.Sprintf(format, args...), "waits for its publish") {
|
||||
select {
|
||||
case waiting <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
}
|
||||
return r, res
|
||||
return r, entry, waiting
|
||||
}
|
||||
|
||||
func aPackageWithNoPinWaitsForItsRelease(t *testing.T) {
|
||||
r, _ := unpinnedRig(t)
|
||||
var once sync.Once
|
||||
looked := make(chan struct{})
|
||||
r.b.NewestReleased = func(context.Context, string) (string, bool, error) { // the record holds none yet
|
||||
once.Do(func() { close(looked) })
|
||||
return "", false, nil
|
||||
const unpinnedX = "git.novox.be/novox/mesh-contracts/go/seats/x"
|
||||
|
||||
// awaitWaiting fails the test unless the build started its wait before ended said anything.
|
||||
func awaitWaiting[T any](t *testing.T, waiting <-chan struct{}, ended <-chan T) {
|
||||
t.Helper()
|
||||
select {
|
||||
case <-waiting:
|
||||
case end := <-ended:
|
||||
t.Fatalf("the build ended before the package was published: %+v", end)
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("the build never waited for the package's publish")
|
||||
}
|
||||
}
|
||||
|
||||
// H-A-192: the pin is the version the registry seat's published carries for the path in the accepted series; a publish in
|
||||
// another series and the module's release number pin nothing.
|
||||
func aPackageWithNoPinWaitsForItsRelease(t *testing.T) {
|
||||
r, _, waiting := unpinnedRig(t)
|
||||
r.b.PackagePublished(unpinnedX, "v0.2.9") // another series than the 0.1 the module accepts: no pin
|
||||
r.b.PackagePublished(unpinnedX, "0.1.4-feat-x.2") // a group's tagged version: no pin
|
||||
ended := make(chan meshbuilds.StepEnded, 1)
|
||||
go func() {
|
||||
ended <- r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
|
||||
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
|
||||
}()
|
||||
select {
|
||||
case <-looked:
|
||||
case end := <-ended:
|
||||
t.Fatalf("the ask ended before the package was released: %+v, asks %+v", end, r.asks.Asks())
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("the ask never looked for the package's release")
|
||||
}
|
||||
awaitWaiting(t, waiting, ended)
|
||||
if _, ok := r.st.vals[r.key]; ok {
|
||||
t.Fatal("the build was made before the package had a release")
|
||||
t.Fatal("the build was made before the package was published")
|
||||
}
|
||||
r.b.VersionReleased("contracts", "0.1.3")
|
||||
r.b.PackagePublished(unpinnedX, "v0.1.3")
|
||||
var end meshbuilds.StepEnded
|
||||
select {
|
||||
case end = <-ended:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("the release did not wake the ask")
|
||||
t.Fatal("the publish did not wake the ask")
|
||||
}
|
||||
r.b.Wait()
|
||||
asks := r.asks.Asks()
|
||||
if end.Outcome != meshbuilds.StepEndedOutcomePassed || len(asks) == 0 ||
|
||||
!reflect.DeepEqual(asks[0].Packages, buildstep.PackageVersions{"git.novox.be/novox/mesh-contracts/go/seats/x": "v0.1.3", "git.novox.be/novox/mesh-sdk/go": "v0.1.20"}) {
|
||||
!reflect.DeepEqual(asks[0].Packages, buildstep.PackageVersions{unpinnedX: "v0.1.3", "git.novox.be/novox/mesh-sdk/go": "v0.1.20"}) {
|
||||
t.Errorf("end = %+v, asks %+v; want passed, built with x at v0.1.3", end, asks)
|
||||
}
|
||||
}
|
||||
|
||||
func aPackageWithNoReleaseFailsPastItsBound(t *testing.T) {
|
||||
r, _ := unpinnedRig(t)
|
||||
r, _, _ := unpinnedRig(t)
|
||||
r.b.PackageWait = time.Nanosecond
|
||||
end := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
|
||||
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
|
||||
r.b.Wait()
|
||||
if _, built := r.st.vals[r.key]; end.Outcome != meshbuilds.StepEndedOutcomeFailed || built ||
|
||||
!strings.Contains(end.Said, "package git.novox.be/novox/mesh-contracts/go/seats/x has no released version") {
|
||||
!strings.Contains(end.Said, "package "+unpinnedX+" has no released version in the series 0.1") {
|
||||
t.Errorf("end = %+v, built %v; want failed naming the package, nothing built", end, built)
|
||||
}
|
||||
// A release the builder's own record keeps gives the pin at once.
|
||||
r.b.NewestReleased = func(_ context.Context, module string) (string, bool, error) {
|
||||
return "0.1.2", module == "contracts", nil
|
||||
}
|
||||
// A publish heard before gives the pin at once.
|
||||
r.b.PackagePublished(unpinnedX, "v0.1.2")
|
||||
if end := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
|
||||
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}}); strings.Contains(end.Said, "no released version") {
|
||||
t.Errorf("a release the record keeps did not pin the package: %+v", end)
|
||||
t.Errorf("a publish heard did not pin the package: %+v", end)
|
||||
}
|
||||
}
|
||||
|
||||
// H-A-194: the build reads the registry seat's versions of the path once before it waits: a version in the accepted series
|
||||
// pins the package with no event heard; one only in another series leaves the build waiting for a publish.
|
||||
func theSeatsVersionsPinBeforeTheWait(t *testing.T) {
|
||||
r, _, _ := unpinnedRig(t)
|
||||
r.b.PackageWait = time.Nanosecond // no wait: the seat's answer alone gives the pin
|
||||
var read []string
|
||||
r.b.PublishedVersions = func(_ context.Context, path string) ([]string, error) {
|
||||
read = append(read, path)
|
||||
return []string{"0.2.1", "0.1.9", "0.1.8"}, nil // newest first
|
||||
}
|
||||
end := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
|
||||
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
|
||||
r.b.Wait()
|
||||
asks := r.asks.Asks()
|
||||
if end.Outcome != meshbuilds.StepEndedOutcomePassed || len(asks) == 0 || asks[0].Packages[unpinnedX] != "v0.1.9" || !slices.Equal(read, []string{unpinnedX}) {
|
||||
t.Errorf("end = %+v, asks %+v, read %v; want passed, built with x at v0.1.9 read once from the seat", end, asks, read)
|
||||
}
|
||||
r2, _, waiting := unpinnedRig(t)
|
||||
r2.b.PublishedVersions = func(context.Context, string) ([]string, error) { return []string{"0.2.1"}, nil }
|
||||
ended := make(chan meshbuilds.StepEnded, 1)
|
||||
go func() {
|
||||
ended <- r2.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
|
||||
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
|
||||
}()
|
||||
awaitWaiting(t, waiting, ended)
|
||||
r2.b.PackagePublished(unpinnedX, "0.1.10")
|
||||
if end := <-ended; end.Outcome != meshbuilds.StepEndedOutcomePassed {
|
||||
t.Errorf("end = %+v, want passed once x was published in its series", end)
|
||||
}
|
||||
r2.b.Wait()
|
||||
}
|
||||
|
||||
// H-A-192, H-A-165: the lane of an entry taken in waits the same way, never asking a build without the pin, and fails the
|
||||
// key past the bound naming the package.
|
||||
func aLaneWaitsForAnUnpinnedPackage(t *testing.T) {
|
||||
r, entry, waiting := unpinnedRig(t)
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
r.b.Entry(t.Context(), Origin{Branch: "main", Source: 41, First: true}, entry)
|
||||
r.b.Wait()
|
||||
close(done)
|
||||
}()
|
||||
awaitWaiting(t, waiting, done)
|
||||
if asks := r.asks.Asks(); len(asks) != 0 {
|
||||
t.Fatalf("a build was asked without the package's pin: %+v", asks)
|
||||
}
|
||||
r.b.PackagePublished(unpinnedX, "0.1.5")
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("the publish did not wake the lane")
|
||||
}
|
||||
if asks := r.asks.Asks(); len(asks) == 0 || asks[0].Packages[unpinnedX] != "v0.1.5" {
|
||||
t.Errorf("asks %+v; want one asked with x at v0.1.5", asks)
|
||||
}
|
||||
// Past its bound the key fails naming the package, and nothing is asked.
|
||||
r2, entry2, _ := unpinnedRig(t)
|
||||
r2.b.PackageWait = time.Nanosecond
|
||||
r2.b.Entry(t.Context(), Origin{Branch: "main", Source: 41, First: true}, entry2)
|
||||
r2.b.Wait()
|
||||
if asks := r2.asks.Asks(); len(asks) != 0 || !strings.Contains(string(r2.st.vals[r2.key]), "package "+unpinnedX+" has no released version") {
|
||||
t.Errorf("asks %+v, key %s; want none asked and the key failed naming the package", asks, r2.st.vals[r2.key])
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -295,23 +295,6 @@ func (r *Reserver) ReleasedFor(ctx context.Context, module, fingerprint string)
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// NewestReleased is the newest version the builder's own record of releases keeps for module (design 47 H H-A-165): a
|
||||
// package with no pin is built against it; false when the record holds none.
|
||||
func (r *Reserver) NewestReleased(ctx context.Context, module string) (string, bool, error) {
|
||||
if r.Releases == nil {
|
||||
return "", false, nil
|
||||
}
|
||||
raw, _, found, err := r.Releases.Get(ctx, ReleasedKey(module))
|
||||
if err != nil || !found {
|
||||
return "", false, err
|
||||
}
|
||||
var rec ReleasedRecord
|
||||
if json.Unmarshal(raw, &rec) != nil || len(rec.Kept) == 0 {
|
||||
return "", false, nil
|
||||
}
|
||||
return rec.Kept[len(rec.Kept)-1].Version, true, nil
|
||||
}
|
||||
|
||||
// keepRelease adds a release to the module's record, dropping the oldest past ReleasedKept.
|
||||
func (r *Reserver) keepRelease(ctx context.Context, module, fingerprint, version string) error {
|
||||
n := r.ReleasedKept
|
||||
|
||||
@@ -201,13 +201,11 @@ func TestAReleaseIsRecordedByFingerprintBeforeItsReservationGoes(t *testing.T) {
|
||||
t.Errorf("reservations left: %v", all)
|
||||
}
|
||||
// One key per module holds the last releases, the oldest dropped (A-456b).
|
||||
newest := ""
|
||||
for _, fp := range []string{"src2:b1", "src2:b2"} {
|
||||
n, err := r.Reserve(ctx, "m", "0.1", []string{"m:a"}, Source{Fingerprint: fp})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
newest = n.Version
|
||||
if _, err := r.Released(ctx, "m", n.Version); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -221,13 +219,6 @@ func TestAReleaseIsRecordedByFingerprintBeforeItsReservationGoes(t *testing.T) {
|
||||
if ok, _ := r.ReleasedFor(ctx, "m", "src2:b2"); !ok {
|
||||
t.Error("the newest release is not kept")
|
||||
}
|
||||
// H-A-165: the newest version the record keeps is the pin of a package with none.
|
||||
if v, ok, err := r.NewestReleased(ctx, "m"); !ok || err != nil || v != newest {
|
||||
t.Errorf("newest released of m = %q, %v (%v), want %s", v, ok, err, newest)
|
||||
}
|
||||
if _, ok, _ := r.NewestReleased(ctx, "other"); ok {
|
||||
t.Error("a module never released has a newest release")
|
||||
}
|
||||
}
|
||||
|
||||
// TestARebuildReservesAboveTheCatalogsHighestWhateverItsStandingNumberWas: design 47 G A-490b — a build again of a done key replaces
|
||||
|
||||
@@ -266,10 +266,16 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1
|
||||
if _, bus := p["bus"]; bus {
|
||||
t.Errorf("the process declares a bus: it runs on its own broker credential")
|
||||
}
|
||||
// Issue 1966's bootstrap: the package artifact builds is out of the manifest until the new builder runs; the follow-up
|
||||
// puts it back with its assertion (ADR 0634 §1, ADR 0606 §4: three artifacts, the third the package builds).
|
||||
if len(m.Build.Artifacts) != 2 {
|
||||
t.Fatalf("artifacts = %+v, want the program and the tools", m.Build.Artifacts)
|
||||
// H-A-186, design 47 G §370: a Go package another module imports is a declared package artifact of its module, so every
|
||||
// release publishes it: builds (ADR 0634 §1, ADR 0606 §4), imported by the build modules and the node-engine, and
|
||||
// buildrun, imported by the indexer, the view and the watchman.
|
||||
if len(m.Build.Artifacts) != 4 {
|
||||
t.Fatalf("artifacts = %+v, want the program, the tools and the packages builds and buildrun", m.Build.Artifacts)
|
||||
}
|
||||
for i, name := range []string{"builds", "buildrun"} {
|
||||
if a := m.Build.Artifacts[2+i]; a.Name != name || a.Kind != "package" || a.Language != "go" || !reflect.DeepEqual(a.Paths, []string{name}) || a.Series != "0.1" {
|
||||
t.Errorf("package artifact = %+v, want the Go package %s of series 0.1", a, name)
|
||||
}
|
||||
}
|
||||
if a := m.Build.Artifacts[1]; a.Name != "tools" || a.Kind != "bundle" || a.Binary != "mesh-delivery-builder-tools" || !reflect.DeepEqual(a.Loads, []string{"mesh-delivery-builder-tools"}) {
|
||||
t.Errorf("tools artifact = %+v", a) // 1945
|
||||
|
||||
@@ -334,6 +334,24 @@
|
||||
"loads": [
|
||||
"mesh-delivery-builder-tools"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "builds",
|
||||
"kind": "package",
|
||||
"language": "go",
|
||||
"paths": [
|
||||
"builds"
|
||||
],
|
||||
"series": "0.1"
|
||||
},
|
||||
{
|
||||
"name": "buildrun",
|
||||
"kind": "package",
|
||||
"language": "go",
|
||||
"paths": [
|
||||
"buildrun"
|
||||
],
|
||||
"series": "0.1"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
+4
-1
@@ -1,4 +1,7 @@
|
||||
package meshcatalog
|
||||
package meshcatalogclient
|
||||
|
||||
// The two helpers below are hand-written, so they live in the seat's client, never under its contract's generated
|
||||
// root (design 47 H, H-A-200; ADR 0608 §5).
|
||||
|
||||
import "strings"
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
package meshcatalogclient
|
||||
|
||||
import "testing"
|
||||
|
||||
// An entry's manifest-digest carries sha256: and the module's value and the key of `manifests` the 64 digits bare, one
|
||||
// digest in two forms (design 47 G §578); the two helpers live in the seat's client, outside its contract's generated
|
||||
// root (H-A-200).
|
||||
func TestAManifestDigestIsPrefixedForTheEntryAndBareForTheKey(t *testing.T) {
|
||||
const bare = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
|
||||
for _, tc := range []struct{ in, entry, key string }{
|
||||
{bare, "sha256:" + bare, bare},
|
||||
{"sha256:" + bare, "sha256:" + bare, bare},
|
||||
{"", "", ""},
|
||||
} {
|
||||
if got := EntryManifestDigest(tc.in); got != tc.entry {
|
||||
t.Errorf("EntryManifestDigest(%q) = %q, want %q", tc.in, got, tc.entry)
|
||||
}
|
||||
if got := BareManifestDigest(tc.in); got != tc.key {
|
||||
t.Errorf("BareManifestDigest(%q) = %q, want %q", tc.in, got, tc.key)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -276,6 +276,48 @@ func DecodePipelinesByBranchRunRunValue(raw []byte) (PipelinesByBranchRunRunValu
|
||||
return v, nil
|
||||
}
|
||||
|
||||
// PipelinesByStepsPipelineMoverKey is the key of one value of the state pipelines under the form "steps.<pipeline>.<mover>".
|
||||
func PipelinesByStepsPipelineMoverKey(pipeline, mover string) string {
|
||||
return "steps." + pipeline + "." + mover
|
||||
}
|
||||
|
||||
// ParsePipelinesByStepsPipelineMoverKey reads the parts of a key of the form "steps.<pipeline>.<mover>", in the order of the form; the last result is false for a key of another form.
|
||||
func ParsePipelinesByStepsPipelineMoverKey(key string) (string, string, bool) {
|
||||
rest := key
|
||||
if !strings.HasPrefix(rest, "steps.") {
|
||||
return "", "", false
|
||||
}
|
||||
rest = rest[6:]
|
||||
v1, after1, found1 := strings.Cut(rest, ".")
|
||||
if !found1 || v1 == "" {
|
||||
return "", "", false
|
||||
}
|
||||
rest = after1
|
||||
if rest == "" {
|
||||
return "", "", false
|
||||
}
|
||||
v3 := rest
|
||||
rest = ""
|
||||
if rest != "" {
|
||||
return "", "", false
|
||||
}
|
||||
return v1, v3, true
|
||||
}
|
||||
|
||||
// DecodePipelinesByStepsPipelineMoverValue reads one value of the state pipelines under the form "steps.<pipeline>.<mover>", refusing a field PipelinesByStepsPipelineMoverValue does not know and any text after the value.
|
||||
func DecodePipelinesByStepsPipelineMoverValue(raw []byte) (PipelinesByStepsPipelineMoverValue, error) {
|
||||
var v PipelinesByStepsPipelineMoverValue
|
||||
dec := json.NewDecoder(bytes.NewReader(raw))
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(&v); err != nil {
|
||||
return v, err
|
||||
}
|
||||
if dec.More() {
|
||||
return v, errors.New("text after the value")
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
|
||||
// QueueByQueueNextKey is the key of one value of the state queue under the form "queue.next".
|
||||
func QueueByQueueNextKey() string { return "queue.next" }
|
||||
|
||||
@@ -523,8 +565,10 @@ type LogEntry struct {
|
||||
Artifact string `json:"artifact,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
// the keys of the kept results the step wrote or carried, <level>.<digest>, which the runner ages by their last-read (design 47 G sections 330, 470)
|
||||
Kept []string `json:"kept,omitempty"`
|
||||
Outcome string `json:"outcome,omitempty"`
|
||||
Kept []string `json:"kept,omitempty"`
|
||||
// the module of the mover whose step the entry is of (H-A-190)
|
||||
Module string `json:"module,omitempty"`
|
||||
Outcome string `json:"outcome,omitempty"`
|
||||
// the output's name in step-outputs
|
||||
Output string `json:"output,omitempty"`
|
||||
Pipeline string `json:"pipeline"`
|
||||
@@ -621,6 +665,8 @@ type Pipeline struct {
|
||||
// how often the pipeline has ended: 1 at its first end, one higher at each end again after a passing rerun, so the assigner takes a later end of the same source revision (design 47 G sections 29, 325)
|
||||
EndCount int64 `json:"end-count,omitempty"`
|
||||
Ended *time.Time `json:"ended,omitempty"`
|
||||
// how many movers the pipeline has, set by list alone (H-A-191)
|
||||
MoverCount int64 `json:"mover-count,omitempty"`
|
||||
// the modules the pipeline moves, its take-in's entries (design 47 H §746)
|
||||
Movers []Mover `json:"movers"`
|
||||
Outcome string `json:"outcome"`
|
||||
@@ -637,11 +683,13 @@ type Pipeline struct {
|
||||
// the source revision of the watched branch; the pipeline is keyed by it (ADR 0666 section 9)
|
||||
Source int64 `json:"source"`
|
||||
Started *time.Time `json:"started,omitempty"`
|
||||
// every mover's steps, each naming its mover, approval last
|
||||
// how many steps the pipeline's movers have, set by list alone, which answers the record with its steps empty (H-A-191)
|
||||
StepCount int64 `json:"step-count,omitempty"`
|
||||
// every mover's steps, each naming its mover, approval last; in the state pipelines a record's own are empty, each mover's kept under its own key (H-A-187)
|
||||
Steps []StepResult `json:"steps"`
|
||||
// the modules a newer waiting take-in of the same branch took out of the pipeline, written with the movers by compare-and-set; at its placing the runner writes a mover's build-step why from it (design 47 H §748, H-A-107)
|
||||
Superseded []string `json:"superseded,omitempty"`
|
||||
// the step-log entries of the steps that ended and are not appended to the step log yet, in order; written with the step's end in one write and cleared when appended (design 47 G sections 486, 606)
|
||||
// the step-log entries of the steps that ended and are not appended to the step log yet, in order; written with the step's end in one write and cleared when appended (design 47 G sections 486, 606); in the state pipelines a record's own are empty, each mover's kept under its own key beside its steps (H-A-189)
|
||||
Unlogged []LogEntry `json:"unlogged,omitempty"`
|
||||
// the layout version
|
||||
V int64 `json:"v,omitempty"`
|
||||
@@ -1026,6 +1074,8 @@ type StartAnswer struct {
|
||||
|
||||
// ShowInput is what show takes.
|
||||
type ShowInput struct {
|
||||
// a mover's module: the answer's pipeline then holds that mover's steps alone (H-A-187)
|
||||
Mover string `json:"mover,omitempty"`
|
||||
// the pipeline's key
|
||||
Pipeline string `json:"pipeline"`
|
||||
}
|
||||
@@ -1314,6 +1364,14 @@ type PipelinesByBranchSourcePublishedPackageVersionValue = Pipeline
|
||||
// PipelinesByBranchRunRunValue is one value of the state pipelines under the key <branch>.run.<run>.
|
||||
type PipelinesByBranchRunRunValue = Pipeline
|
||||
|
||||
// PipelinesByStepsPipelineMoverValue is one value of the state pipelines under the key steps.<pipeline>.<mover>.
|
||||
type PipelinesByStepsPipelineMoverValue struct {
|
||||
// the mover's steps
|
||||
Steps []StepResult `json:"steps"`
|
||||
// the step-log entries of the mover's steps that ended and are not appended to the step log yet, in order (H-A-189)
|
||||
Unlogged []LogEntry `json:"unlogged,omitempty"`
|
||||
}
|
||||
|
||||
// QueueByQueueNextValue is one value of the state queue under the key queue.next.
|
||||
type QueueByQueueNextValue = int64
|
||||
|
||||
|
||||
@@ -86,13 +86,17 @@
|
||||
},
|
||||
{
|
||||
"name": "show",
|
||||
"description": "One pipeline whole: its steps, its step log and its outputs, with the stores' fill. Reads only.",
|
||||
"description": "One pipeline whole: its record with every mover's steps, or one mover's steps alone when it names the mover, its step log and its outputs, with the stores' fill. Reads only.",
|
||||
"input": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"pipeline": {
|
||||
"type": "string",
|
||||
"description": "the pipeline's key"
|
||||
},
|
||||
"mover": {
|
||||
"type": "string",
|
||||
"description": "a mover's module: the answer's pipeline then holds that mover's steps alone (H-A-187)"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
@@ -133,7 +137,7 @@
|
||||
},
|
||||
{
|
||||
"name": "list",
|
||||
"description": "The pipelines and runs of a module, a branch or a commit, newest first, with the store's fill. Reads only.",
|
||||
"description": "The pipelines and runs of a module, a branch or a commit, newest first, with the store's fill: each pipeline's record with its movers and steps counted, never its steps (show gives those). Reads only.",
|
||||
"input": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -771,7 +775,7 @@
|
||||
"name": "pipelines",
|
||||
"template": "state",
|
||||
"history": 1,
|
||||
"description": "one key per pipeline, one per take-in, written by compare-and-set on its absence (design 47 H §746, §747)",
|
||||
"description": "one key per pipeline, one per take-in, written by compare-and-set on its absence (design 47 H §746, §747), and one key per mover of a pipeline holding that mover's steps (H-A-187)",
|
||||
"key-forms": [
|
||||
{
|
||||
"key": "<branch>.<source>",
|
||||
@@ -793,6 +797,32 @@
|
||||
"$ref": "#/defs/Pipeline"
|
||||
},
|
||||
"description": "a run on request's pipeline, one per run, its movers the named commits' modules; it delivers nothing (H-A-83)"
|
||||
},
|
||||
{
|
||||
"key": "steps.<pipeline>.<mover>",
|
||||
"value": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"steps": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/defs/StepResult"
|
||||
},
|
||||
"description": "the mover's steps"
|
||||
},
|
||||
"unlogged": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/defs/LogEntry"
|
||||
},
|
||||
"description": "the step-log entries of the mover's steps that ended and are not appended to the step log yet, in order (H-A-189)"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"steps"
|
||||
]
|
||||
},
|
||||
"description": "one mover's steps and unlogged step-log entries in a pipeline, its key the pipeline's and the mover's module escaped as a key's name, written by compare-and-set as the pipeline's record is; the record's own steps stay empty, so a record never grows with the catalog (H-A-187)"
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -1238,7 +1268,7 @@
|
||||
"items": {
|
||||
"$ref": "#/defs/StepResult"
|
||||
},
|
||||
"description": "every mover's steps, each naming its mover, approval last"
|
||||
"description": "every mover's steps, each naming its mover, approval last; in the state pipelines a record's own are empty, each mover's kept under its own key (H-A-187)"
|
||||
},
|
||||
"started": {
|
||||
"type": "string",
|
||||
@@ -1253,7 +1283,15 @@
|
||||
"items": {
|
||||
"$ref": "#/defs/LogEntry"
|
||||
},
|
||||
"description": "the step-log entries of the steps that ended and are not appended to the step log yet, in order; written with the step's end in one write and cleared when appended (design 47 G sections 486, 606)"
|
||||
"description": "the step-log entries of the steps that ended and are not appended to the step log yet, in order; written with the step's end in one write and cleared when appended (design 47 G sections 486, 606); in the state pipelines a record's own are empty, each mover's kept under its own key beside its steps (H-A-189)"
|
||||
},
|
||||
"mover-count": {
|
||||
"type": "integer",
|
||||
"description": "how many movers the pipeline has, set by list alone (H-A-191)"
|
||||
},
|
||||
"step-count": {
|
||||
"type": "integer",
|
||||
"description": "how many steps the pipeline's movers have, set by list alone, which answers the record with its steps empty (H-A-191)"
|
||||
},
|
||||
"published": {
|
||||
"$ref": "#/defs/Published",
|
||||
@@ -1613,6 +1651,10 @@
|
||||
"pipeline": {
|
||||
"type": "string"
|
||||
},
|
||||
"module": {
|
||||
"type": "string",
|
||||
"description": "the module of the mover whose step the entry is of (H-A-190)"
|
||||
},
|
||||
"step": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
|
||||
@@ -112,6 +112,12 @@ export declare function pipelinesByBranchRunRunKey(branch: string, run: string):
|
||||
/** The parts of a key of the form "<branch>.run.<run>", or undefined for a key of another form. */
|
||||
export declare function parsePipelinesByBranchRunRunKey(_key: string): { branch: string; run: string } | undefined;
|
||||
|
||||
/** The key of one value of the state pipelines under the form "steps.<pipeline>.<mover>". */
|
||||
export declare function pipelinesByStepsPipelineMoverKey(pipeline: string, mover: string): string;
|
||||
|
||||
/** The parts of a key of the form "steps.<pipeline>.<mover>", or undefined for a key of another form. */
|
||||
export declare function parsePipelinesByStepsPipelineMoverKey(_key: string): { pipeline: string; mover: string } | undefined;
|
||||
|
||||
/** The key of one value of the state queue under the form "queue.next". */
|
||||
export declare function queueByQueueNextKey(): string;
|
||||
|
||||
@@ -186,6 +192,8 @@ export interface LogEntry {
|
||||
"at": string;
|
||||
/** the keys of the kept results the step wrote or carried, <level>.<digest>, which the runner ages by their last-read (design 47 G sections 330, 470) */
|
||||
"kept"?: string[];
|
||||
/** the module of the mover whose step the entry is of (H-A-190) */
|
||||
"module"?: string;
|
||||
"outcome"?: "running" | "waiting" | "passed" | "failed" | "nothing to run" | "bypassed" | "carried" | "could not run" | "waiting for a build node" | "did not build" | "accepted" | "declined" | "superseded" | "cancelled";
|
||||
/** the output's name in step-outputs */
|
||||
"output"?: string;
|
||||
@@ -248,6 +256,8 @@ export interface Pipeline {
|
||||
/** how often the pipeline has ended: 1 at its first end, one higher at each end again after a passing rerun, so the assigner takes a later end of the same source revision (design 47 G sections 29, 325) */
|
||||
"end-count"?: number;
|
||||
"ended"?: string;
|
||||
/** how many movers the pipeline has, set by list alone (H-A-191) */
|
||||
"mover-count"?: number;
|
||||
/** the modules the pipeline moves, its take-in's entries (design 47 H §746) */
|
||||
"movers": Mover[];
|
||||
"outcome": "waiting" | "running" | "passed" | "failed" | "refused" | "superseded" | "cancelled";
|
||||
@@ -264,11 +274,13 @@ export interface Pipeline {
|
||||
/** the source revision of the watched branch; the pipeline is keyed by it (ADR 0666 section 9) */
|
||||
"source": number;
|
||||
"started"?: string;
|
||||
/** every mover's steps, each naming its mover, approval last */
|
||||
/** how many steps the pipeline's movers have, set by list alone, which answers the record with its steps empty (H-A-191) */
|
||||
"step-count"?: number;
|
||||
/** every mover's steps, each naming its mover, approval last; in the state pipelines a record's own are empty, each mover's kept under its own key (H-A-187) */
|
||||
"steps": StepResult[];
|
||||
/** the modules a newer waiting take-in of the same branch took out of the pipeline, written with the movers by compare-and-set; at its placing the runner writes a mover's build-step why from it (design 47 H §748, H-A-107) */
|
||||
"superseded"?: string[];
|
||||
/** the step-log entries of the steps that ended and are not appended to the step log yet, in order; written with the step's end in one write and cleared when appended (design 47 G sections 486, 606) */
|
||||
/** the step-log entries of the steps that ended and are not appended to the step log yet, in order; written with the step's end in one write and cleared when appended (design 47 G sections 486, 606); in the state pipelines a record's own are empty, each mover's kept under its own key beside its steps (H-A-189) */
|
||||
"unlogged"?: LogEntry[];
|
||||
/** the layout version */
|
||||
"v"?: number;
|
||||
@@ -556,6 +568,8 @@ export interface StartAnswer {
|
||||
|
||||
/** is what show takes. */
|
||||
export interface ShowInput {
|
||||
/** a mover's module: the answer's pipeline then holds that mover's steps alone (H-A-187) */
|
||||
"mover"?: string;
|
||||
/** the pipeline's key */
|
||||
"pipeline": string;
|
||||
}
|
||||
@@ -795,6 +809,14 @@ export type PipelinesByBranchSourcePublishedPackageVersionValue = Pipeline;
|
||||
/** is one value of the state pipelines under the key <branch>.run.<run>. */
|
||||
export type PipelinesByBranchRunRunValue = Pipeline;
|
||||
|
||||
/** is one value of the state pipelines under the key steps.<pipeline>.<mover>. */
|
||||
export interface PipelinesByStepsPipelineMoverValue {
|
||||
/** the mover's steps */
|
||||
"steps": StepResult[];
|
||||
/** the step-log entries of the mover's steps that ended and are not appended to the step log yet, in order (H-A-189) */
|
||||
"unlogged"?: LogEntry[];
|
||||
}
|
||||
|
||||
/** is one value of the state queue under the key queue.next. */
|
||||
export type QueueByQueueNextValue = number;
|
||||
|
||||
|
||||
@@ -173,6 +173,27 @@ export function parsePipelinesByBranchRunRunKey(_key) {
|
||||
return { branch, run };
|
||||
}
|
||||
|
||||
/** The key of one value of the state pipelines under the form "steps.<pipeline>.<mover>". */
|
||||
export function pipelinesByStepsPipelineMoverKey(pipeline, mover) {
|
||||
return "steps." + pipeline + "." + mover;
|
||||
}
|
||||
|
||||
/** The parts of a key of the form "steps.<pipeline>.<mover>", or undefined for a key of another form. */
|
||||
export function parsePipelinesByStepsPipelineMoverKey(_key) {
|
||||
let _rest = _key;
|
||||
if (!_rest.startsWith("steps.")) return undefined;
|
||||
_rest = _rest.slice(6);
|
||||
const _at1 = _rest.indexOf(".");
|
||||
if (_at1 <= 0) return undefined;
|
||||
const pipeline = _rest.slice(0, _at1);
|
||||
_rest = _rest.slice(_at1 + 1);
|
||||
if (_rest === "") return undefined;
|
||||
const mover = _rest;
|
||||
_rest = "";
|
||||
if (_rest !== "") return undefined;
|
||||
return { pipeline, mover };
|
||||
}
|
||||
|
||||
/** The key of one value of the state queue under the form "queue.next". */
|
||||
export function queueByQueueNextKey() {
|
||||
return "queue.next";
|
||||
|
||||
@@ -141,7 +141,7 @@ func main() {
|
||||
tools = append(tools, OwedTools(askState(owedAsks, groups.OwedAsksBucket(served), 25*time.Second, func(v []byte) bool {
|
||||
a, err := groups.DecodeOwedAsk(v)
|
||||
return err == nil && a.Done()
|
||||
}), func() ([]OwedRow, error) { return performedRows(performed) }, func(what, why string) error {
|
||||
}), func(f OwedFilter) ([]OwedRow, error) { return performedRows(performed, f) }, func(what, why string) error {
|
||||
_, err := recorder.Record(meshhandactsclient.Request{What: what, Why: why, Cause: "owed-dropped"})
|
||||
return err
|
||||
}, time.Now)...)
|
||||
|
||||
+43
-5
@@ -94,17 +94,55 @@ func TestTheBridgeTools(t *testing.T) {
|
||||
}
|
||||
return json.Marshal(a)
|
||||
}
|
||||
rows := []OwedRow{{Of: seat.Seat, ID: "a", Outcome: forgemsg.OwedPerformed, At: at}}
|
||||
rows := []OwedRow{
|
||||
{OwedEnded: forgemsg.OwedEnded{Of: seat.Seat, ID: "a", Outcome: forgemsg.OwedPerformed, At: at}, Repository: "novox/mesh-sdk", Number: 178},
|
||||
{OwedEnded: forgemsg.OwedEnded{Of: seat.Seat, ID: "b", Outcome: forgemsg.OwedPerformed, At: at}, Repository: "novox/mesh-sdk", Number: 179}}
|
||||
var filters []OwedFilter
|
||||
performed := func(f OwedFilter) ([]OwedRow, error) {
|
||||
filters = append(filters, f)
|
||||
var out []OwedRow
|
||||
for _, r := range rows {
|
||||
if f.holds(r) {
|
||||
out = append(out, r)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
var recorded []string
|
||||
recordErr := error(nil)
|
||||
record := func(what, why string) error { recorded = append(recorded, what+": "+why); return recordErr }
|
||||
tools := map[string]func(map[string]any) (any, error){}
|
||||
for _, tl := range OwedTools(ask, func() ([]OwedRow, error) { return rows, nil }, record, func() time.Time { return at }) {
|
||||
for _, tl := range OwedTools(ask, performed, record, func() time.Time { return at }) {
|
||||
tools[tl.Name] = tl.Run
|
||||
}
|
||||
out, err := tools[OwedTool](map[string]any{"of": seat.Seat})
|
||||
if m, _ := out.(map[string]any); err != nil || len(m["performed"].([]OwedRow)) != 1 || len(m["backlog"].([]groups.Backlog)) != 1 {
|
||||
t.Errorf("bridge_owed: %v %v", out, err)
|
||||
// bridge_owed reads one act or one pull request's acts, never the whole state (H-A-181).
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
args map[string]any
|
||||
ids []string
|
||||
}{
|
||||
{"one pull request", map[string]any{"repository": "novox/mesh-sdk", "number": float64(179)}, []string{"b"}},
|
||||
{"one act", map[string]any{"of": seat.Seat, "id": "a"}, []string{"a"}},
|
||||
{"the whole state", map[string]any{"of": seat.Seat}, nil},
|
||||
{"a repository with no number", map[string]any{"repository": "novox/mesh-sdk"}, nil},
|
||||
} {
|
||||
n := len(filters)
|
||||
out, err := tools[OwedTool](c.args)
|
||||
if c.ids == nil {
|
||||
if err == nil || len(filters) != n {
|
||||
t.Errorf("bridge_owed of %s: %v %v, read %d; want refused, nothing read", c.name, out, err, len(filters)-n)
|
||||
}
|
||||
continue
|
||||
}
|
||||
m, _ := out.(map[string]any)
|
||||
got, _ := m["performed"].([]OwedRow)
|
||||
var ids []string
|
||||
for _, r := range got {
|
||||
ids = append(ids, r.ID)
|
||||
}
|
||||
if err != nil || !reflect.DeepEqual(ids, c.ids) || len(m["backlog"].([]groups.Backlog)) != 1 {
|
||||
t.Errorf("bridge_owed of %s: %v %v, want %v", c.name, out, err, c.ids)
|
||||
}
|
||||
}
|
||||
if out, err := tools[OwedAgainTool](map[string]any{"of": seat.Seat, "id": "a"}); err != nil || out.(map[string]any)["outcome"] != forgemsg.OwedPerformed ||
|
||||
asked[len(asked)-1].Kind != groups.OwedAgain {
|
||||
|
||||
@@ -21,26 +21,50 @@ const (
|
||||
OwedDropTool = "bridge_owed_drop"
|
||||
)
|
||||
|
||||
// OwedRow is one act the bridge performed, as the state performed keeps it: the outcome said as owed.ended.
|
||||
type OwedRow = forgemsg.OwedEnded
|
||||
// OwedRow is one act the bridge performed, as the state performed keeps it: the outcome said as owed.ended, and the pull
|
||||
// request the act names (groups.Performed).
|
||||
type OwedRow struct {
|
||||
forgemsg.OwedEnded
|
||||
Repository string `json:"repository,omitempty"`
|
||||
Number int `json:"number,omitempty"`
|
||||
}
|
||||
|
||||
// performedRows reads every act the state performed holds; a key that is not an act (a group end's bookkeeping) is left.
|
||||
func performedRows(s stdio.KeptState) ([]OwedRow, error) {
|
||||
keys, err := s.Keys()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the bridge's performed state cannot be listed: %w", err)
|
||||
// OwedFilter is what bridge_owed reads of performed: one act (its id, of narrowing it to one key) or one pull request's
|
||||
// acts (repository and number), never the whole state (novox/hq design 47 H-A-181).
|
||||
type OwedFilter struct {
|
||||
Of, ID, Repository string
|
||||
Number int
|
||||
}
|
||||
|
||||
func (f OwedFilter) holds(r OwedRow) bool {
|
||||
return (f.Of == "" || r.Of == f.Of) && (f.ID == "" || r.ID == f.ID) &&
|
||||
(f.Repository == "" || (r.Repository == f.Repository && r.Number == f.Number))
|
||||
}
|
||||
|
||||
// performedRows reads the acts of performed f names, newest first: one key when f names the seat and the id, else each
|
||||
// key's value held to f. A key that is not an act (a group end's bookkeeping) is left.
|
||||
func performedRows(s stdio.KeptState, f OwedFilter) ([]OwedRow, error) {
|
||||
var keys []string
|
||||
if f.Of != "" && f.ID != "" {
|
||||
keys = []string{groups.PerformedKey(f.Of, f.ID)}
|
||||
} else {
|
||||
all, err := s.Keys()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the bridge's performed state cannot be listed: %w", err)
|
||||
}
|
||||
keys = all
|
||||
}
|
||||
var rows []OwedRow
|
||||
rows := []OwedRow{}
|
||||
for _, k := range keys {
|
||||
e, err := s.Get(k)
|
||||
if err != nil || e == nil {
|
||||
continue
|
||||
}
|
||||
var p struct {
|
||||
Ended OwedRow `json:"ended"`
|
||||
}
|
||||
var p groups.Performed
|
||||
if json.Unmarshal(e.Value, &p) == nil && p.Ended.Of != "" {
|
||||
rows = append(rows, p.Ended)
|
||||
if r := (OwedRow{OwedEnded: p.Ended, Repository: p.Repository, Number: p.Number}); f.holds(r) {
|
||||
rows = append(rows, r)
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Slice(rows, func(i, j int) bool { return rows[i].At.After(rows[j].At) })
|
||||
@@ -50,7 +74,7 @@ func performedRows(s stdio.KeptState) ([]OwedRow, error) {
|
||||
// OwedTools are bridge_owed, bridge_owed_again and bridge_owed_drop: ask writes an owed-asks value and waits for the
|
||||
// bridge's answer; performed reads the state performed; record writes a hand act in the seat mesh-hand-acts' log before
|
||||
// a drop is asked, so a drop is never made without its record.
|
||||
func OwedTools(ask func(key string, value []byte) ([]byte, error), performed func() ([]OwedRow, error), record func(what, why string) error, now func() time.Time) []stdio.Tool {
|
||||
func OwedTools(ask func(key string, value []byte) ([]byte, error), performed func(OwedFilter) ([]OwedRow, error), record func(what, why string) error, now func() time.Time) []stdio.Tool {
|
||||
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
|
||||
act := map[string]any{
|
||||
"of": map[string]any{"type": "string", "description": "the seat that said the act: mesh-delivery (the gate)"},
|
||||
@@ -66,6 +90,12 @@ func OwedTools(ask func(key string, value []byte) ([]byte, error), performed fun
|
||||
}
|
||||
return groups.DecodeOwedAsk(v)
|
||||
}
|
||||
read := map[string]any{
|
||||
"repository": map[string]any{"type": "string", "description": "the pull request's repository, as owner/name; with number"},
|
||||
"number": map[string]any{"type": "integer", "description": "the pull request's number; with repository"}}
|
||||
for k, v := range act {
|
||||
read[k] = v
|
||||
}
|
||||
withWhy := map[string]any{"why": map[string]any{"type": "string", "description": "why it is dropped, in plain words"}}
|
||||
for k, v := range act {
|
||||
withWhy[k] = v
|
||||
@@ -74,22 +104,24 @@ func OwedTools(ask func(key string, value []byte) ([]byte, error), performed fun
|
||||
{Name: OwedTool,
|
||||
Description: "What the forge bridge performed of what the gate and the checker owe the forge (design 47): each act it said as " +
|
||||
"owed.ended, newest first, with its seat, id, pull request, outcome (performed, refused, could not be performed, " +
|
||||
"undecodable, dropped), the forge's words and when, from its state performed (kept 7 days); and the backlog of its " +
|
||||
"undecodable, dropped), the forge's words and when, from its state performed (kept 7 days), of one act (id, of " +
|
||||
"narrowing it) or of one pull request (repository and number), never the whole state; and the backlog of its " +
|
||||
"durable on each seat's stream owed: the last event applied, the events waiting and held, the age of the oldest. " +
|
||||
"of or id narrow it. Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": act},
|
||||
"Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": read},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
rows, err := performed()
|
||||
f := OwedFilter{Of: str(args, "of"), ID: str(args, "id"), Repository: str(args, "repository")}
|
||||
if n, ok := args["number"].(float64); ok {
|
||||
f.Number = int(n)
|
||||
}
|
||||
if f.ID == "" && (f.Repository == "" || f.Number <= 0) {
|
||||
return nil, fmt.Errorf("bridge_owed reads one act (id) or one pull request's acts (repository and number), never the whole state")
|
||||
}
|
||||
rows, err := performed(f)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
kept := []OwedRow{}
|
||||
for _, r := range rows {
|
||||
if (str(args, "of") == "" || r.Of == str(args, "of")) && (str(args, "id") == "" || r.ID == str(args, "id")) {
|
||||
kept = append(kept, r)
|
||||
}
|
||||
}
|
||||
out := map[string]any{"performed": kept}
|
||||
out := map[string]any{"performed": rows}
|
||||
if a, err := asked(groups.OwedBacklog, args); err != nil {
|
||||
out["backlog-error"] = err.Error()
|
||||
} else {
|
||||
@@ -100,8 +132,8 @@ func OwedTools(ask func(key string, value []byte) ([]byte, error), performed fun
|
||||
{Name: OwedAgainTool,
|
||||
Description: "Perform one act owed to the forge again now, by hand, whatever its last outcome (a refusal the forge no longer " +
|
||||
"makes, an act that could not be performed): the bridge calls the seat git's verb of the act under its id (safe to " +
|
||||
"repeat) and says its outcome as owed.ended. The act is the one the bridge kept, or, for the gate's, the seat " +
|
||||
"mesh-delivery's groups. Answers the outcome.",
|
||||
"repeat) and says its outcome as owed.ended. The act is read from the stream owed of the seat that said it, on the " +
|
||||
"subject of the pull request the bridge's state performed names (kept 7 days there). Answers the outcome.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"of", "id"}, "properties": act},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
a, err := asked(groups.OwedAgain, args)
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
)
|
||||
|
||||
// The act bridge_owed_again performs again is read from the stream owed of the seat that said it, on the subject of the
|
||||
// pull request performed names, the stream being the act's one copy (novox/hq design 47 G §162, §163, H-A-181).
|
||||
|
||||
// actReader reads the act the seat of said under id from its stream owed, on the subject of the pull request
|
||||
// (repository and number) the act names, for decode to say each message's id; found false when the stream no longer
|
||||
// holds it.
|
||||
type actReader interface {
|
||||
act(ctx context.Context, of, repository string, number int, id string, idOf func([]byte) string) (data []byte, found bool, err error)
|
||||
}
|
||||
|
||||
// actOf is the act of to perform again, read from its seat's stream owed on the pull request p names.
|
||||
func (o *owing) actOf(ctx context.Context, of string, p Performed, id string) ([]byte, error) {
|
||||
read, err := readerOf(of)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// A value of the form before H-A-182 names no pull request: every act the bridge decodes names its repository (a
|
||||
// status may name no number, said on `.0` as the gate says it), so one with none is the old form, never read again.
|
||||
if p.Repository == "" {
|
||||
return nil, fmt.Errorf("performed names no pull request for the act %s of %s: kept in its old form, it expires with the state's TTL", id, of)
|
||||
}
|
||||
if o.acts == nil {
|
||||
return nil, fmt.Errorf("the bridge reads no stream owed, so the act %s of %s cannot be performed again", id, of)
|
||||
}
|
||||
data, found, err := o.acts.act(ctx, of, p.Repository, p.Number, id, func(m []byte) string { c, _ := read(m); return c.ID })
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the act %s of %s cannot be read from %s's stream owed: %w", id, of, of, err)
|
||||
}
|
||||
if !found {
|
||||
return nil, fmt.Errorf("%s's stream owed no longer holds the act %s of %s#%d (kept 7 days)", of, id, p.Repository, p.Number)
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// streamActs reads an act from the gate's stream owed (seat mesh-delivery's StreamOwed) by an ordered consumer filtered
|
||||
// on the pull request's subject, as the gate says it (SubjectOwed with the repository escaped): every message the
|
||||
// stream keeps there, the newest of the id kept.
|
||||
type streamActs struct{ js jetstream.JetStream }
|
||||
|
||||
func (s streamActs) act(ctx context.Context, of, repository string, number int, id string, idOf func([]byte) string) ([]byte, bool, error) {
|
||||
if of != seat.Seat {
|
||||
return nil, false, fmt.Errorf("the seat %q keeps no stream owed the bridge reads", of)
|
||||
}
|
||||
subject := seat.SubjectOwed(keys.EscapeName(repository), strconv.Itoa(number))
|
||||
st, err := s.js.Stream(ctx, seat.StreamOwed)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
info, err := st.Info(ctx, jetstream.WithSubjectFilter(subject))
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
held := info.State.Subjects[subject]
|
||||
if held == 0 {
|
||||
return nil, false, nil
|
||||
}
|
||||
c, err := st.OrderedConsumer(ctx, jetstream.OrderedConsumerConfig{FilterSubjects: []string{subject}})
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
var found []byte
|
||||
for range held {
|
||||
m, err := c.Next(jetstream.FetchContext(ctx))
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if idOf(m.Data()) == id {
|
||||
found = m.Data()
|
||||
}
|
||||
if md, err := m.Metadata(); err == nil && md.NumPending == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
return found, found != nil, nil
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -38,6 +39,24 @@ func (m memPerformed) Get(_ context.Context, key string) ([]byte, bool, error) {
|
||||
}
|
||||
func (m memPerformed) Put(_ context.Context, key string, v []byte) error { m[key] = v; return nil }
|
||||
|
||||
// memActs is a seat's stream owed by pull request subject (`<repository>#<number>`); asked records what was read.
|
||||
type memActs struct {
|
||||
held map[string][][]byte
|
||||
asked []string
|
||||
}
|
||||
|
||||
func (m *memActs) act(_ context.Context, of, repository string, number int, id string, idOf func([]byte) string) ([]byte, bool, error) {
|
||||
pr := fmt.Sprintf("%s %s#%d", of, repository, number)
|
||||
m.asked = append(m.asked, pr)
|
||||
var found []byte
|
||||
for _, d := range m.held[pr] {
|
||||
if idOf(d) == id {
|
||||
found = d
|
||||
}
|
||||
}
|
||||
return found, found != nil, nil
|
||||
}
|
||||
|
||||
// seatFake answers each seat verb from answers, by its key; one absent is not answered (an error, as a bound passed).
|
||||
type seatFake struct {
|
||||
calls []string
|
||||
@@ -109,9 +128,12 @@ func TestWhatIsOwedIsPerformedSaidAndKept(t *testing.T) {
|
||||
if v, ok := kept[performedKey(c.of, e.ID)]; ok != c.kept {
|
||||
t.Errorf("%s: kept %v (%s), want %v", c.name, ok, v, c.kept)
|
||||
} else if ok {
|
||||
// The outcome and when, and the pull request the act names, nothing more (§163, H-A-181, H-A-182).
|
||||
var p Performed
|
||||
if err := json.Unmarshal(v, &p); err != nil || p.Ended.Outcome != c.outcome || !json.Valid(p.Owed) {
|
||||
t.Errorf("%s: kept %s, want the outcome and the owed event", c.name, v)
|
||||
var fields map[string]json.RawMessage
|
||||
if err := json.Unmarshal(v, &p); err != nil || p.Ended.Outcome != c.outcome || p.Ended.At.IsZero() ||
|
||||
p.Repository != "novox/mesh-sdk" || p.Number != 178 || json.Unmarshal(v, &fields) != nil || len(fields) != 3 {
|
||||
t.Errorf("%s: kept %s, want the outcome, when, and the pull request novox/mesh-sdk#178, nothing more", c.name, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -142,6 +164,9 @@ func TestWhatIsOwedIsPerformedSaidAndKept(t *testing.T) {
|
||||
t.Run("asked again, dropped, and its backlog read by hand", func(t *testing.T) {
|
||||
f := &seatFake{answers: statusOK}
|
||||
o, pub, kept := newOwing(f)
|
||||
// The act is read again from the gate's stream owed on its pull request's subject, never from performed (H-A-181).
|
||||
acts := &memActs{held: map[string][][]byte{"mesh-delivery novox/mesh-sdk#178": {[]byte(`{`), gate("owed is said, a status")}}}
|
||||
o.acts = acts
|
||||
status := "status.novox/mesh-sdk#178.aaaaaaaaaaaa.mesh/merge-gate"
|
||||
if _, err := o.heard(context.Background(), seat.Seat, gate("owed is said, a status")); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -153,11 +178,20 @@ func TestWhatIsOwedIsPerformedSaidAndKept(t *testing.T) {
|
||||
calls, said int
|
||||
outcome, answers string
|
||||
}{
|
||||
{"again, from performed", groups.OwedAsk{Kind: groups.OwedAgain, Of: seat.Seat, ID: status}, 2, 2, forgemsg.OwedPerformed, "performed"},
|
||||
{"again, from the gate's stream owed", groups.OwedAsk{Kind: groups.OwedAgain, Of: seat.Seat, ID: status}, 2, 2, forgemsg.OwedPerformed, "performed"},
|
||||
{"dropped", groups.OwedAsk{Kind: groups.OwedDrop, Of: seat.Seat, ID: status, Why: "set by hand", By: "lead@shanks"}, 2, 3, forgemsg.OwedDropped, "set by hand"},
|
||||
{"again, an act nobody holds", groups.OwedAsk{Kind: groups.OwedAgain, Of: seat.Seat, ID: "x"}, 2, 3, "", "holds no owed act"},
|
||||
{"the backlog", groups.OwedAsk{Kind: groups.OwedBacklog}, 2, 3, "", "backlog"},
|
||||
{"again, the stream no longer holding it", groups.OwedAsk{Kind: groups.OwedAgain, Of: seat.Seat, ID: status}, 2, 3, "", "no longer holds"},
|
||||
{"again, a value of the old form", groups.OwedAsk{Kind: groups.OwedAgain, Of: seat.Seat, ID: "old"}, 2, 3, "", "names no pull request"},
|
||||
} {
|
||||
switch c.name {
|
||||
case "again, the stream no longer holding it":
|
||||
acts.held = nil
|
||||
case "again, a value of the old form":
|
||||
// Kept before H-A-182: the outcome and the whole act, no pull request; the stream is not read for it.
|
||||
kept[performedKey(seat.Seat, "old")] = []byte(`{"ended":{"v":1,"of":"mesh-delivery","id":"old","outcome":"performed","at":"2026-11-09T21:00:00Z"},"owed":{"id":"old"}}`)
|
||||
}
|
||||
a := o.answer(context.Background(), c.ask, backlog)
|
||||
if !a.Done() || a.Outcome != c.outcome || !strings.Contains(a.Answer, c.answers) || len(f.calls) != c.calls || len(pub.data) != c.said {
|
||||
t.Errorf("%s: %+v, called %v, said %d; want %q naming %q, %d calls, %d said", c.name, a, f.calls, len(pub.data), c.outcome, c.answers, c.calls, c.said)
|
||||
@@ -167,8 +201,12 @@ func TestWhatIsOwedIsPerformedSaidAndKept(t *testing.T) {
|
||||
}
|
||||
}
|
||||
var p Performed
|
||||
if err := json.Unmarshal(kept[performedKey(seat.Seat, status)], &p); err != nil || p.Ended.Outcome != forgemsg.OwedDropped {
|
||||
t.Errorf("performed after the drop: %+v %v", p, err)
|
||||
if err := json.Unmarshal(kept[performedKey(seat.Seat, status)], &p); err != nil || p.Ended.Outcome != forgemsg.OwedDropped ||
|
||||
p.Repository != "novox/mesh-sdk" || p.Number != 178 {
|
||||
t.Errorf("performed after the drop: %+v %v; want dropped, its pull request kept", p, err)
|
||||
}
|
||||
if len(acts.asked) != 2 || acts.asked[0] != "mesh-delivery novox/mesh-sdk#178" {
|
||||
t.Errorf("the stream owed was read on %v, want twice on the pull request performed names", acts.asked)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
+22
-3
@@ -22,6 +22,7 @@ import (
|
||||
git "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-forge-bridge/internal/gitfixture"
|
||||
@@ -33,7 +34,8 @@ import (
|
||||
// no others, so a permission the bridge lacks fails a test here and not on the mesh (issue 1279).
|
||||
//
|
||||
// From module.json: the registrations and the gate's source-moved heard through its durable on EVENTS; branch-moved and
|
||||
// what the gate owes heard through its durable on each seat's own stream (SEAT_<seat>_<event>); the seat git's verbs, the
|
||||
// what the gate owes heard through its durable on each seat's own stream (SEAT_<seat>_<event>), and each seat's kept
|
||||
// event also read through an ordered consumer of its own (mesh-grants heardGrants, A-233; bridge_owed_again, H-A-181); the seat git's verbs, the
|
||||
// seat mesh-catalog's registrations, the seat mesh-delivery's source, the seat mesh-broker's show and the seat
|
||||
// mesh-hand-acts' hand-act called on mesh.seat.<seat>.call.<verb>.<token> (or its plain fallback); its own states heads
|
||||
// and owed-asks, and performed under the name its binding gives; its stream forge. `**` is written `>` on the wire. The
|
||||
@@ -82,6 +84,11 @@ accounts {
|
||||
"$JS.ACK.SEAT_mesh-delivery_owed.n1_mesh-delivery-forge-bridge.>"
|
||||
"$JS.API.CONSUMER.INFO.SEAT_mesh-delivery_owed.n1_mesh-delivery-forge-bridge"
|
||||
"$JS.API.CONSUMER.MSG.NEXT.SEAT_mesh-delivery_owed.n1_mesh-delivery-forge-bridge"
|
||||
"$JS.API.CONSUMER.CREATE.SEAT_mesh-delivery_owed"
|
||||
"$JS.API.CONSUMER.CREATE.SEAT_mesh-delivery_owed.>"
|
||||
"$JS.API.CONSUMER.MSG.NEXT.SEAT_mesh-delivery_owed.*"
|
||||
"$JS.API.CONSUMER.INFO.SEAT_mesh-delivery_owed.*"
|
||||
"$JS.API.CONSUMER.DELETE.SEAT_mesh-delivery_owed.*"
|
||||
"mesh.seat.mesh-broker.call.show.n1~mesh-delivery-forge-bridge"
|
||||
"mesh.seat.mesh-broker.tool.show"
|
||||
"mesh.seat.mesh-catalog.call.registrations.n1~mesh-delivery-forge-bridge"
|
||||
@@ -518,7 +525,9 @@ func TestBridgePerformsWhatIsOwedOnce(t *testing.T) {
|
||||
}
|
||||
}
|
||||
gateOwed := gitfixture.Seat(t, seat.Seat, "events", "owed is said, a status", "event:owed")[0]
|
||||
subject := seat.SubjectOwed("novox~mesh-sdk", "178")
|
||||
// On the pull request's subject as the gate says it (its repository escaped as a key part), where bridge_owed_again
|
||||
// reads the act again (H-A-181).
|
||||
subject := seat.SubjectOwed(keys.EscapeName("novox/mesh-sdk"), "178")
|
||||
for range 2 {
|
||||
if _, err := b.js.Publish(b.ctx, subject, gateOwed); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -597,12 +606,22 @@ func TestBridgePerformsWhatIsOwedOnce(t *testing.T) {
|
||||
if a, err := groupkeys.DecodeOwedAsk(e.Value()); err == nil && a.Done() {
|
||||
answered = true
|
||||
if a.Outcome != forgemsg.OwedPerformed || count("git.status") != 3 {
|
||||
t.Errorf("bridge_owed_again: %+v, status asked %d times; want performed again", a, count("git.status"))
|
||||
t.Errorf("bridge_owed_again: %+v, status asked %d times; want performed again, read from the gate's stream owed", a, count("git.status"))
|
||||
}
|
||||
}
|
||||
case <-bound:
|
||||
t.Fatal("bridge_owed_again was not answered within 20 s")
|
||||
}
|
||||
}
|
||||
// performed keeps the outcome, when and the pull request, never the act (§163, H-A-181, H-A-182).
|
||||
kept, err := b.js.KeyValue(b.ctx, boundPerformed)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if e, err := kept.Get(b.ctx, performedKey(seat.Seat, g.ID)); err != nil {
|
||||
t.Errorf("performed of %s: %v", g.ID, err)
|
||||
} else if fields := (map[string]json.RawMessage{}); json.Unmarshal(e.Value(), &fields) != nil || len(fields) != 3 || fields["owed"] != nil {
|
||||
t.Errorf("performed of %s keeps %s, want the outcome, when and the pull request alone", g.ID, e.Value())
|
||||
}
|
||||
refusedRead(t, b, "mesh-delivery-gate_owed")
|
||||
}
|
||||
|
||||
@@ -108,7 +108,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
performedKV := &boundBucket{binding: binding, local: performedLocal, js: js}
|
||||
// What the gate owes the forge, performed through the seat git (design 47 *What the gate owes the forge*).
|
||||
owe := &owing{call: call, pub: js, performed: kvPerformed{performedKV}, waits: binding.Waiting, now: time.Now, logger: logger}
|
||||
owe := &owing{call: call, pub: js, performed: kvPerformed{performedKV}, acts: streamActs{js: js}, waits: binding.Waiting, now: time.Now, logger: logger}
|
||||
keptCall := keptCallOf(ctx, call)
|
||||
reader := &views.ProcessReader{Module: forgeBridgeModule(), Call: keptCall, Logf: logger.Printf, Hub: hub}
|
||||
src := sourcebranches.New(sourcebranches.Call(keptCall))
|
||||
|
||||
@@ -2,7 +2,6 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -22,22 +21,14 @@ import (
|
||||
)
|
||||
|
||||
// The state performed (manifest `requires`, the template state, opened by its binding: bound.go) keeps one key per owed act it
|
||||
// performed, the act's outcome and when, with the owed event as heard (novox/hq design 47 *What the gate owes the
|
||||
// forge*). It is a saving and a read, never what keeps an act from happening twice: every act is safe to repeat at the
|
||||
// forge.
|
||||
// performed, in the form groups.Performed says (novox/hq design 47 G §163, H-A-181, H-A-182). It is a saving and a read,
|
||||
// never what keeps an act from happening twice: every act is safe to repeat at the forge.
|
||||
|
||||
// performedKey is the key of one act in performed: the seat that said it and its id, the id in unpadded base64url, since
|
||||
// an id may hold what a key may not (`#`, `/`).
|
||||
func performedKey(of, id string) string {
|
||||
return of + "." + base64.RawURLEncoding.EncodeToString([]byte(id))
|
||||
}
|
||||
// performedKey is the key of one act in performed (groups.PerformedKey).
|
||||
func performedKey(of, id string) string { return groups.PerformedKey(of, id) }
|
||||
|
||||
// Performed is one value of the state performed: the outcome said as owed.ended, and the owed event it answers, so
|
||||
// bridge_owed_again performs it again from here.
|
||||
type Performed struct {
|
||||
Ended forgemsg.OwedEnded `json:"ended"`
|
||||
Owed json.RawMessage `json:"owed,omitempty"`
|
||||
}
|
||||
// Performed is one value of the state performed.
|
||||
type Performed = groups.Performed
|
||||
|
||||
// performedStore is the state performed as the owing reads and writes it.
|
||||
type performedStore interface {
|
||||
@@ -70,6 +61,8 @@ type owing struct {
|
||||
call caller
|
||||
pub publisher
|
||||
performed performedStore
|
||||
// acts reads an act again from the stream owed of the seat that said it (H-A-181), for bridge_owed_again.
|
||||
acts actReader
|
||||
// waits, when set, says why the state performed cannot be used yet (its binding, design 47 G §577); empty when it can.
|
||||
waits func() string
|
||||
now func() time.Time
|
||||
@@ -90,12 +83,12 @@ func readerOf(of string) (func([]byte) (bridge.OwedCall, error), error) {
|
||||
func (o *owing) heard(ctx context.Context, of string, data []byte) (forgemsg.OwedEnded, error) {
|
||||
read, err := readerOf(of)
|
||||
if err != nil {
|
||||
return o.end(ctx, forgemsg.OwedEnded{Of: of, Outcome: forgemsg.OwedUndecodable, Words: err.Error()}, data)
|
||||
return o.end(ctx, forgemsg.OwedEnded{Of: of, Outcome: forgemsg.OwedUndecodable, Words: err.Error()}, "", 0)
|
||||
}
|
||||
c, err := read(data)
|
||||
if err != nil {
|
||||
return o.end(ctx, forgemsg.OwedEnded{Of: of, ID: c.ID,
|
||||
Outcome: forgemsg.OwedUndecodable, Words: err.Error()}, data)
|
||||
Outcome: forgemsg.OwedUndecodable, Words: err.Error()}, c.Repository, c.Number)
|
||||
}
|
||||
if w := o.waiting(); w != "" {
|
||||
return forgemsg.OwedEnded{Of: c.Of, ID: c.ID}, fmt.Errorf("the bridge %s", w)
|
||||
@@ -104,7 +97,7 @@ func (o *owing) heard(ctx context.Context, of string, data []byte) (forgemsg.Owe
|
||||
// said twice, or said again while its outcome was on its way: performed once, the outcome said once
|
||||
return forgemsg.OwedEnded{Of: c.Of, ID: c.ID}, nil
|
||||
}
|
||||
return o.perform(ctx, c, data)
|
||||
return o.perform(ctx, c)
|
||||
}
|
||||
|
||||
// hear is one message of a seat's stream owed: performed and acknowledged, or, when its outcome could not be said,
|
||||
@@ -127,19 +120,20 @@ func (o *owing) waiting() string {
|
||||
}
|
||||
|
||||
// perform calls the seat git's verb of one act and says what it came to.
|
||||
func (o *owing) perform(ctx context.Context, c bridge.OwedCall, data []byte) (forgemsg.OwedEnded, error) {
|
||||
func (o *owing) perform(ctx context.Context, c bridge.OwedCall) (forgemsg.OwedEnded, error) {
|
||||
answer, callErr := o.call(ctx, "seat:"+git.Seat+"."+c.Verb, c.Input)
|
||||
outcome, words := bridge.Outcome(answer, callErr)
|
||||
if outcome != forgemsg.OwedPerformed {
|
||||
o.logger.Printf("the owed %s %s of %s#%d (said by %s) is %s: %s", c.Act, c.ID, c.Repository, c.Number, c.Of, outcome, words)
|
||||
}
|
||||
return o.end(ctx, forgemsg.OwedEnded{Of: c.Of, ID: c.ID,
|
||||
Outcome: outcome, Words: words}, data)
|
||||
Outcome: outcome, Words: words}, c.Repository, c.Number)
|
||||
}
|
||||
|
||||
// end says an act's outcome as owed.ended on the stream forge, then keeps it in performed. An outcome not kept is
|
||||
// logged: the act may be performed again, which changes nothing at the forge.
|
||||
func (o *owing) end(ctx context.Context, e forgemsg.OwedEnded, owed []byte) (forgemsg.OwedEnded, error) {
|
||||
// end says an act's outcome as owed.ended on the stream forge, then keeps it in performed with the pull request the act
|
||||
// names (repository and number). An outcome not kept is logged: the act may be performed again, which changes nothing at
|
||||
// the forge.
|
||||
func (o *owing) end(ctx context.Context, e forgemsg.OwedEnded, repository string, number int) (forgemsg.OwedEnded, error) {
|
||||
e.At = o.now().UTC()
|
||||
data, err := forgemsg.EncodeOwedEnded(e)
|
||||
if err != nil {
|
||||
@@ -151,10 +145,7 @@ func (o *owing) end(ctx context.Context, e forgemsg.OwedEnded, owed []byte) (for
|
||||
if e.ID == "" {
|
||||
return e, nil // an undecodable act with no id is said, and has no key to keep
|
||||
}
|
||||
if !json.Valid(owed) {
|
||||
owed = nil
|
||||
}
|
||||
v, err := json.Marshal(Performed{Ended: e, Owed: owed})
|
||||
v, err := json.Marshal(Performed{Ended: e, Repository: repository, Number: number})
|
||||
if err == nil {
|
||||
err = o.performed.Put(ctx, performedKey(e.Of, e.ID), v)
|
||||
}
|
||||
@@ -175,19 +166,17 @@ func (o *owing) held(ctx context.Context, of, id string) bool {
|
||||
return found
|
||||
}
|
||||
|
||||
// owedOf is the owed event of an act to perform again, as performed keeps it.
|
||||
func (o *owing) owedOf(ctx context.Context, of, id string) ([]byte, error) {
|
||||
// kept is what performed holds of an act: its outcome and its pull request.
|
||||
func (o *owing) kept(ctx context.Context, of, id string) (Performed, error) {
|
||||
v, found, err := o.performed.Get(ctx, performedKey(of, id))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("performed cannot be read: %w", err)
|
||||
return Performed{}, fmt.Errorf("performed cannot be read: %w", err)
|
||||
}
|
||||
if found {
|
||||
var p Performed
|
||||
if err := json.Unmarshal(v, &p); err == nil && len(p.Owed) > 0 {
|
||||
return p.Owed, nil
|
||||
}
|
||||
var p Performed
|
||||
if !found || json.Unmarshal(v, &p) != nil {
|
||||
return Performed{}, fmt.Errorf("the bridge holds no owed act %s of %s in its state performed", id, of)
|
||||
}
|
||||
return nil, fmt.Errorf("the bridge holds no owed act %s of %s in its state performed", id, of)
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// answer answers one owed ask: an act performed again now, whatever performed holds of it; an act dropped, said as
|
||||
@@ -199,20 +188,22 @@ func (o *owing) answer(ctx context.Context, a groups.OwedAsk, backlog func(conte
|
||||
a.Backlog, a.Answer = backlog(ctx), "the backlog of each seat's stream owed"
|
||||
return a
|
||||
}
|
||||
data, err := o.owedOf(ctx, a.Of, a.ID)
|
||||
p, err := o.kept(ctx, a.Of, a.ID)
|
||||
if err != nil {
|
||||
a.Answer = err.Error()
|
||||
return a
|
||||
}
|
||||
var e forgemsg.OwedEnded
|
||||
if a.Kind == groups.OwedDrop {
|
||||
e, err = o.end(ctx, forgemsg.OwedEnded{Of: a.Of, ID: a.ID, Outcome: forgemsg.OwedDropped, Words: a.Why + " (dropped by " + a.By + ")"}, data)
|
||||
e, err = o.end(ctx, forgemsg.OwedEnded{Of: a.Of, ID: a.ID, Outcome: forgemsg.OwedDropped, Words: a.Why + " (dropped by " + a.By + ")"}, p.Repository, p.Number)
|
||||
} else if read, rerr := readerOf(a.Of); rerr != nil {
|
||||
err = rerr
|
||||
} else if data, rerr := o.actOf(ctx, a.Of, p, a.ID); rerr != nil {
|
||||
err = rerr
|
||||
} else if c, rerr := read(data); rerr != nil {
|
||||
e, err = o.end(ctx, forgemsg.OwedEnded{Of: a.Of, ID: a.ID, Outcome: forgemsg.OwedUndecodable, Words: rerr.Error()}, data)
|
||||
e, err = o.end(ctx, forgemsg.OwedEnded{Of: a.Of, ID: a.ID, Outcome: forgemsg.OwedUndecodable, Words: rerr.Error()}, p.Repository, p.Number)
|
||||
} else {
|
||||
e, err = o.perform(ctx, c, data)
|
||||
e, err = o.perform(ctx, c)
|
||||
}
|
||||
if err != nil {
|
||||
a.Answer = err.Error()
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
package groups
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
)
|
||||
|
||||
// The state performed keeps one key per owed act the bridge performed, its outcome and when, and the pull request the
|
||||
// act names, nothing more (novox/hq design 47 G §163, H-A-181, H-A-182): the act itself stays in the stream owed of the
|
||||
// seat that said it, its one copy, where bridge_owed_again reads it again. The bridge writes it; its tools read it.
|
||||
|
||||
// PerformedKey is the key of one act in performed: the seat that said it and its id, the id in unpadded base64url, since
|
||||
// an id may hold what a key may not (`#`, `/`).
|
||||
func PerformedKey(of, id string) string {
|
||||
return of + "." + base64.RawURLEncoding.EncodeToString([]byte(id))
|
||||
}
|
||||
|
||||
// Performed is one value of the state performed (§163, H-A-182): the outcome said as owed.ended, with when, and the
|
||||
// repository and number of the pull request the act names, so bridge_owed reads one pull request's acts and
|
||||
// bridge_owed_again finds the act on its pull request's subject of the stream owed (H-A-181).
|
||||
type Performed struct {
|
||||
Ended forgemsg.OwedEnded `json:"ended"`
|
||||
Repository string `json:"repository,omitempty"`
|
||||
Number int `json:"number,omitempty"`
|
||||
}
|
||||
@@ -187,10 +187,13 @@ func TestManifestProcess(t *testing.T) { // 1235/AC1
|
||||
if _, bus := p["bus"]; bus {
|
||||
t.Errorf("the process declares a bus: it runs on its own broker credential")
|
||||
}
|
||||
// Issue 1966's bootstrap: the package artifact forgemsg is out of the manifest until the new builder runs; the follow-up
|
||||
// puts it back with its assertion (ADR 0573 §1: three artifacts, the third the package forgemsg).
|
||||
if len(m.Build.Artifacts) != 2 || m.Build.Artifacts[1].Name != "tools" || m.Build.Artifacts[1].From != "cmd/mesh-delivery-forge-bridge-tools" {
|
||||
t.Fatalf("artifacts = %+v, want the program and the tools bundle", m.Build.Artifacts)
|
||||
if len(m.Build.Artifacts) != 3 || m.Build.Artifacts[1].Name != "tools" || m.Build.Artifacts[1].From != "cmd/mesh-delivery-forge-bridge-tools" {
|
||||
t.Fatalf("artifacts = %+v, want the program, the tools bundle and the package forgemsg", m.Build.Artifacts)
|
||||
}
|
||||
// ADR 0573 §1, H-A-186 (design 47 G §370): the gate, the builder and the indexer import forgemsg, so the module declares
|
||||
// it as a package and every release publishes it.
|
||||
if a := m.Build.Artifacts[2]; a.Name != "forgemsg" || a.Kind != "package" || a.Language != "go" || len(a.Paths) != 1 || a.Paths[0] != "forgemsg" || a.Series != "0.1" {
|
||||
t.Errorf("package artifact = %+v", a)
|
||||
}
|
||||
a := m.Build.Artifacts[0]
|
||||
if st, err := os.Stat(a.From); err != nil || !st.IsDir() || a.Binary != "mesh-delivery-forge-bridge" || !strings.HasSuffix(a.From, a.Binary) {
|
||||
|
||||
@@ -101,6 +101,15 @@
|
||||
"loads": [
|
||||
"mesh-delivery-forge-bridge-tools"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "forgemsg",
|
||||
"kind": "package",
|
||||
"language": "go",
|
||||
"paths": [
|
||||
"forgemsg"
|
||||
],
|
||||
"series": "0.1"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"reflect"
|
||||
"slices"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
@@ -30,7 +31,7 @@ const CatalogEntryLayout = 1
|
||||
// ModuleValueOf gives the module's value back from the entry alone. The acting rules are the revision's to add.
|
||||
func CatalogEntryOf(module string, cur meshcatalog.ModulesByModuleValue, m catalogue.Manifest, raw string, n graph.Node) meshcatalog.CatalogEntry {
|
||||
e := meshcatalog.CatalogEntry{V: CatalogEntryLayout, Module: module, Repository: cur.Repository, Path: cur.Path,
|
||||
Commit: cur.Commit, Manifest: manifestWhole(raw), ManifestDigest: meshcatalog.EntryManifestDigest(cur.Manifest), Verdict: cur.Verdict, Fp: n.FP, None: n.None, Bases: n.Bases,
|
||||
Commit: cur.Commit, Manifest: manifestWhole(raw), ManifestDigest: meshcatalogclient.EntryManifestDigest(cur.Manifest), Verdict: cur.Verdict, Fp: n.FP, None: n.None, Bases: n.Bases,
|
||||
Packages: maps.Clone(cur.Packages), Provides: n.Provides, Requires: n.Requires, Awaits: n.Awaits, Invokes: n.Invokes,
|
||||
Serves: n.Serves, LeftOut: n.LeftOut, Main: MainOf(m), Contributes: contributesOf(m)}
|
||||
// requires names every requirement as catalogue.Manifest.Wants lists them, optional ones and those only contributes names
|
||||
@@ -67,7 +68,7 @@ func CatalogEntryOf(module string, cur meshcatalog.ModulesByModuleValue, m catal
|
||||
// ruling on milestone 47/G finding 1; ADR 0656 §6: one form of the fact). The old line's fields (the old-form
|
||||
// fingerprint, the reader needs) are not kept.
|
||||
func ModuleValueOf(e meshcatalog.CatalogEntry) meshcatalog.ModulesByModuleValue {
|
||||
v := meshcatalog.ModulesByModuleValue{V: keys.Version, Repository: e.Repository, Path: e.Path, Commit: e.Commit, Manifest: meshcatalog.BareManifestDigest(e.ManifestDigest),
|
||||
v := meshcatalog.ModulesByModuleValue{V: keys.Version, Repository: e.Repository, Path: e.Path, Commit: e.Commit, Manifest: meshcatalogclient.BareManifestDigest(e.ManifestDigest),
|
||||
Verdict: e.Verdict, Fp2: e.Fp, None2: e.None, Packages: maps.Clone(e.Packages)}
|
||||
if in := e.Inputs; in != nil {
|
||||
v.Toolchains, v.Paths, v.Problems, v.Warnings, v.Unknown = maps.Clone(in.Toolchains), maps.Clone(in.Paths), in.Problems, in.Warnings, in.Unknown
|
||||
|
||||
@@ -544,7 +544,7 @@ func (k *KeptTrunk) reread(ctx context.Context, o Objects) error {
|
||||
}
|
||||
manifests := map[string]string{}
|
||||
for _, e := range entries {
|
||||
if d := meshcatalog.BareManifestDigest(e.ManifestDigest); d != "" && e.Manifest != nil {
|
||||
if d := meshcatalogclient.BareManifestDigest(e.ManifestDigest); d != "" && e.Manifest != nil {
|
||||
if raw, err := json.Marshal(e.Manifest); err == nil {
|
||||
manifests[d] = string(raw)
|
||||
}
|
||||
|
||||
@@ -123,9 +123,21 @@ func TestImportsReportedMoveTheFingerprintOnlyWithAnImportedPackage(t *testing.T
|
||||
moved.Fp2, before.Fp2, moved.Inputs2.ImportForm, moved.Packages)
|
||||
}
|
||||
|
||||
// mesh-sdk releases 0.1.20: the import form's fingerprint takes the pin too and moves, once (H-A-172).
|
||||
// mesh-sdk's module releases 0.1.97 while the registry has published nothing of its Go package: the module's release
|
||||
// number is never the package's pin (H-A-183), so avahi stays unpinned and where it was.
|
||||
ix.Versions = newMem()
|
||||
ix.Versions.(*memStore).put(meshcatalog.VersionsByModuleVersionKey("mesh-sdk", "0.1.20"), []byte(`{}`))
|
||||
ix.Versions.(*memStore).put(meshcatalog.VersionsByModuleVersionKey("mesh-sdk", "0.1.97"), []byte(`{}`))
|
||||
if err := ix.Recompute(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m, _ := moduleOf(t, mods, "avahi"); m.Fp2 != moved.Fp2 || m.Packages[meshSDK] != "" {
|
||||
t.Fatalf("after mesh-sdk's module release alone: fp2 %q (was %q), versions %v; want it unpinned where it was", m.Fp2, moved.Fp2, m.Packages)
|
||||
}
|
||||
// The registry publishes its Go package at 0.1.20, another version than the module's release: the import form's
|
||||
// fingerprint takes that pin and moves, once (H-A-172, H-A-183).
|
||||
if _, err := ix.HearVersion(ctx, HeardVersion{Package: "mesh-sdk/go", Seat: "go-package-registry", Version: "0.1.20", Commit: c3, GoModule: meshSDK, Published: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for range 2 {
|
||||
if err := ix.Recompute(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -283,17 +295,18 @@ func anAcceptedSeriesPinsItsNewest(t *testing.T) {
|
||||
if pinned.Fp2 == before.Fp2 || pinned.Packages[meshSDK] != "v0.1.15" {
|
||||
t.Fatalf("after 0.1.15: fp2 %q (was %q), versions %v; want moved by c3's announce, pinned to v0.1.15", pinned.Fp2, before.Fp2, pinned.Packages)
|
||||
}
|
||||
// A publish in a series avahi does not accept pins nothing and moves nothing (H-A-184).
|
||||
if m := step("0.2.0 published from c1", hear("0.2.0", c1)); m.Fp2 != pinned.Fp2 || m.Packages[meshSDK] != "v0.1.15" {
|
||||
t.Errorf("a publish of a series avahi does not accept moved it: %q %v", m.Fp2, m.Packages)
|
||||
}
|
||||
if m := step("trunk at c2", trunk(c2)); m.Fp2 != pinned.Fp2 || m.Packages[meshSDK] != "v0.1.15" {
|
||||
t.Errorf("a trunk merge moved avahi off its accepted series' newest: %q %v", m.Fp2, m.Packages)
|
||||
}
|
||||
// mesh-sdk releases 0.1.20: an accepted series' older version never stands over a newer release (H-A-167).
|
||||
// mesh-sdk's module releases 0.3.0: a module's release number is never a package's pin (H-A-183).
|
||||
ix.Versions = newMem()
|
||||
ix.Versions.(*memStore).put(meshcatalog.VersionsByModuleVersionKey("mesh-sdk", "0.1.20"), []byte(`{}`))
|
||||
if m := step("0.1.20 released", func() error { return nil }); m.Packages[meshSDK] != "v0.1.20" {
|
||||
t.Errorf("a newer release left avahi on its series' older version: %v", m.Packages)
|
||||
ix.Versions.(*memStore).put(meshcatalog.VersionsByModuleVersionKey("mesh-sdk", "0.3.0"), []byte(`{}`))
|
||||
if m := step("0.3.0 released", func() error { return nil }); m.Packages[meshSDK] != "v0.1.15" {
|
||||
t.Errorf("a module's release pinned its package: %v", m.Packages)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
)
|
||||
@@ -94,7 +95,7 @@ func TestTheKeptTrunkIsReadAtTheStartAndOncePerHeardRevision(t *testing.T) {
|
||||
}
|
||||
// A-487c: the manifests by digest and the verb revision asked of the trunk's newest answer from the view held.
|
||||
listed = objects.n.Load()
|
||||
if raw, err := kept.Manifest(meshcatalog.BareManifestDigest(keptDigest("ca-trust"))); err != nil || raw != `{"module":"ca-trust"}` {
|
||||
if raw, err := kept.Manifest(meshcatalogclient.BareManifestDigest(keptDigest("ca-trust"))); err != nil || raw != `{"module":"ca-trust"}` {
|
||||
t.Errorf("the held manifest = %q (%v), want the entry's manifest", raw, err)
|
||||
}
|
||||
if a, err := Revision(context.Background(), objects, kept, meshcatalog.RevisionInput{}); err != nil || a.Revision.Revision != 2 || len(a.Entries) != 2 {
|
||||
|
||||
@@ -63,6 +63,7 @@ func (ix *Indexer) applyMoving(ctx context.Context, f whole, drop bool, moves ma
|
||||
return w, err
|
||||
}
|
||||
objects, namedErr := ix.namedObjects(ctx, f)
|
||||
trees := ix.goModTree(ctx, f)
|
||||
overlay := map[string]meshcatalog.ModulesByModuleValue{}
|
||||
dirOf := map[string]string{}
|
||||
named, every := map[string]bool{}, true // every module the message's manifests name, and whether each names one
|
||||
@@ -109,7 +110,23 @@ func (ix *Indexer) applyMoving(ctx context.Context, f whole, drop bool, moves ma
|
||||
next.None2 = "the manifest cannot be read: it cannot be kept as a value"
|
||||
} else {
|
||||
packages := ix.goPackages(ctx, f, dir, cur, has)
|
||||
if m, err := judge.ForInputs([]byte(raw)); err == nil && usesGo(languagesOf(m)) {
|
||||
// The entry pins every mesh package of another repository the module's build requires, transitively across
|
||||
// the same-repository modules of its graph (design 47 H-A-201).
|
||||
packages = unionSorted(packages, trees.otherRequired(goModuleDirs(m, dir)))
|
||||
}
|
||||
in := inputsOfNamed(name, raw, f.Trees[dir], f.Locks[dir], objects[dir], packages, ix.baseFP(ctx))
|
||||
if in.Kept != nil {
|
||||
// A Go build resolves the same-repository modules its go.mod files require, transitively, from the tree: their
|
||||
// directories' objects join the tree input of every form, never the entry's paths (design 47 H-A-202, H-A-172).
|
||||
if m, err := judge.ForInputs([]byte(raw)); err == nil && usesGo(languagesOf(m)) {
|
||||
if tree, none := ix.derivedTree(ctx, f, trees, m, dir, in.Kept.Tree); none != "" {
|
||||
in.Kept, in.None2 = nil, none
|
||||
} else {
|
||||
in.Kept.Tree = tree
|
||||
}
|
||||
}
|
||||
}
|
||||
next.Inputs2, next.None2, next.Paths = in.Kept, in.None2, in.ArtifactPaths
|
||||
if in.Kept != nil && has && cur.Inputs2 != nil {
|
||||
// The imports its last trunk build reported stand until its next build reports again (ADR 0593 §2, §4).
|
||||
|
||||
@@ -0,0 +1,196 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
|
||||
"mesh-delivery-indexer/internal/fingerprint"
|
||||
)
|
||||
|
||||
// goModTree reads the go.mod files of one merge's tree, each directory once (Indexer.GoMod), for the walk of the module
|
||||
// graph a Go build resolves from the tree (design 47 H-A-202, H-A-157).
|
||||
type goModTree struct {
|
||||
ix *Indexer
|
||||
ctx context.Context
|
||||
f whole
|
||||
read map[string][]byte // by directory from the repository's root; nil for none
|
||||
}
|
||||
|
||||
func (ix *Indexer) goModTree(ctx context.Context, f whole) *goModTree {
|
||||
return &goModTree{ix: ix, ctx: ctx, f: f, read: map[string][]byte{}}
|
||||
}
|
||||
|
||||
// goMod is the go.mod of the directory dir (from the root, "" for the root) at the merge commit; nil when there is none
|
||||
// or it cannot be read, said.
|
||||
func (t *goModTree) goMod(dir string) []byte {
|
||||
if raw, ok := t.read[dir]; ok {
|
||||
return raw
|
||||
}
|
||||
var raw []byte
|
||||
if t.ix.GoMod != nil {
|
||||
var err error
|
||||
if raw, err = t.ix.GoMod(t.ctx, t.f.Repository, t.f.Merge, dir); err != nil {
|
||||
t.ix.logf("%s at %s: the go.mod of %q cannot be read, so no module of the tree is found there: %v", t.f.Repository, t.f.Merge, dir, err)
|
||||
raw = nil
|
||||
}
|
||||
}
|
||||
t.read[dir] = raw
|
||||
return raw
|
||||
}
|
||||
|
||||
// sameRepositoryDir is the directory of the merge's tree that holds the Go module required: the required path starts with
|
||||
// the repository's module prefix, and the directory it names holds a go.mod declaring that path (H-A-198, H-A-202). false
|
||||
// for a module of another repository, or one the tree holds no go.mod of.
|
||||
func (t *goModTree) sameRepositoryDir(required string) (string, bool) {
|
||||
repo, dir, ok := trunk.PackageRepository(required)
|
||||
if !ok || dir == "" || !trunk.SameRepository(repo, t.f.Repository) {
|
||||
return "", false
|
||||
}
|
||||
if goModModule(t.goMod(dir)) != required {
|
||||
return "", false
|
||||
}
|
||||
return dir, true
|
||||
}
|
||||
|
||||
// sameRepositoryRequired are the directories (from the root, sorted, each once) of every same-repository Go module the
|
||||
// go.mod of each start directory requires, transitively through those modules' own go.mod files (H-A-202, H-A-157). The
|
||||
// start directories themselves are not among them.
|
||||
func (t *goModTree) sameRepositoryRequired(start []string) []string {
|
||||
seen := map[string]bool{}
|
||||
for _, s := range start {
|
||||
seen[s] = true
|
||||
}
|
||||
queue := slices.Clone(start)
|
||||
var out []string
|
||||
for len(queue) > 0 {
|
||||
dir := queue[0]
|
||||
queue = queue[1:]
|
||||
for _, req := range fingerprint.MeshRequiresOf(t.goMod(dir)) {
|
||||
d, ok := t.sameRepositoryDir(req)
|
||||
if !ok || seen[d] {
|
||||
continue
|
||||
}
|
||||
seen[d] = true
|
||||
out = append(out, d)
|
||||
queue = append(queue, d)
|
||||
}
|
||||
}
|
||||
slices.Sort(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// goModuleDirs are the directories (from the root) a module's Go build starts from: its own directory and the directory
|
||||
// of each path of each Go package artifact of its manifest m.
|
||||
func goModuleDirs(m catalogue.Manifest, dir string) []string {
|
||||
out := []string{dir}
|
||||
if m.Build == nil {
|
||||
return out
|
||||
}
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if a.Kind != catalogue.ArtifactPackage || a.Language != "go" {
|
||||
continue
|
||||
}
|
||||
for _, p := range a.Paths {
|
||||
if d := path.Join(dir, strings.TrimSuffix(filepath.ToSlash(p), "/")); !slices.Contains(out, d) {
|
||||
out = append(out, d)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// otherRequired are the mesh packages of other repositories (sorted, each once) that the go.mod of each start directory,
|
||||
// and of every same-repository module those require transitively (sameRepositoryRequired), requires at any version, the
|
||||
// zero version included: every one the module's build resolves, which the entry pins (design 47 H-A-201, H-A-183,
|
||||
// H-A-157). A same-repository module is none: it is built from the tree (H-A-198, H-A-202). nil for none.
|
||||
func (t *goModTree) otherRequired(start []string) []string {
|
||||
var out []string
|
||||
for _, d := range append(slices.Clone(start), t.sameRepositoryRequired(start)...) {
|
||||
out = append(out, fingerprint.OtherPackages(fingerprint.MeshRequiresOf(t.goMod(d)), t.f.Repository)...)
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
return slices.Compact(slices.Sorted(slices.Values(out)))
|
||||
}
|
||||
|
||||
// unionSorted is a and b together, sorted and each once; nil for none.
|
||||
func unionSorted(a, b []string) []string {
|
||||
if len(a)+len(b) == 0 {
|
||||
return nil
|
||||
}
|
||||
return slices.Compact(slices.Sorted(slices.Values(append(slices.Clone(a), b...))))
|
||||
}
|
||||
|
||||
// derivedDirs are the same-repository module directories (from the root, sorted) the module's Go build resolves from the
|
||||
// tree beyond its own tree input (H-A-202): every one its go.mod files require, transitively, that its tree input does
|
||||
// not hold already, which is the module's directory for the directory's tree, and its named paths for the named tree.
|
||||
func (t *goModTree) derivedDirs(m catalogue.Manifest, dir string) []string {
|
||||
var held []string
|
||||
if paths, every := m.NamedPaths(); every {
|
||||
for _, p := range paths {
|
||||
held = append(held, path.Join(dir, filepath.ToSlash(p)))
|
||||
}
|
||||
} else {
|
||||
held = []string{dir}
|
||||
}
|
||||
var out []string
|
||||
for _, d := range t.sameRepositoryRequired(goModuleDirs(m, dir)) {
|
||||
if !slices.ContainsFunc(held, func(h string) bool { return h == "" || h == "." || d == h || strings.HasPrefix(d, h+"/") }) {
|
||||
out = append(out, d)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// DerivedLine opens each line of the tree input naming a derived directory and its git object (H-A-202, H-A-172).
|
||||
const DerivedLine = "derived "
|
||||
|
||||
// derivedTree is the module's tree input with a line per derived directory and its object appended, sorted by directory
|
||||
// (H-A-202, H-A-172: what a build reads moves its fingerprint), so a change of such a directory moves the fingerprint of
|
||||
// every form. The objects are asked of the forge at the merge commit by their paths from the root (Named); an answer that
|
||||
// lacks one gives the reason the module has no inputs, naming them. A module with no derived directory keeps its tree.
|
||||
func (ix *Indexer) derivedTree(ctx context.Context, f whole, t *goModTree, m catalogue.Manifest, dir, tree string) (string, string) {
|
||||
dirs := t.derivedDirs(m, dir)
|
||||
if len(dirs) == 0 {
|
||||
return tree, ""
|
||||
}
|
||||
if ix.Named == nil {
|
||||
return "", "the forge cannot be asked the git objects of the derived directories " + strings.Join(dirs, ", ")
|
||||
}
|
||||
got, err := ix.Named(ctx, f.Repository, f.Merge, map[string][]string{"": dirs})
|
||||
if err != nil {
|
||||
return "", fmt.Sprintf("the git objects of the derived directories %s: %v", strings.Join(dirs, ", "), err)
|
||||
}
|
||||
lines := []string{tree}
|
||||
var missing []string
|
||||
for _, d := range dirs {
|
||||
o := got[""][d]
|
||||
if o == "" {
|
||||
missing = append(missing, d)
|
||||
continue
|
||||
}
|
||||
lines = append(lines, DerivedLine+d+" "+o)
|
||||
}
|
||||
if len(missing) > 0 {
|
||||
return "", "the forge names no git object of the derived directories " + strings.Join(missing, ", ")
|
||||
}
|
||||
return strings.Join(lines, "\n"), ""
|
||||
}
|
||||
|
||||
// goModModule is the module path a go.mod declares; "" for none.
|
||||
func goModModule(raw []byte) string {
|
||||
for _, line := range strings.Split(string(raw), "\n") {
|
||||
if f := strings.Fields(line); len(f) == 2 && f[0] == "module" {
|
||||
return strings.Trim(f[1], `"`)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
|
||||
"mesh-delivery-indexer/internal/fingerprint"
|
||||
)
|
||||
|
||||
// A fixture repository novox/mesh-delivery: the Go package state of modules/g requires the same-repository module
|
||||
// modules/b/builds at the zero version, which requires modules/c/client in turn; mesh-sdk is of another repository, and
|
||||
// modules/b/gone names a directory the tree holds no go.mod of.
|
||||
const ptpZero = " v0.0.0-00010101000000-000000000000"
|
||||
|
||||
var ptpGoMods = map[string]string{
|
||||
"modules/g/state": "module git.novox.be/novox/mesh-delivery/modules/g/state\n\nrequire (\n\tgit.novox.be/novox/mesh-delivery/modules/b/builds" + ptpZero +
|
||||
"\n\tgit.novox.be/novox/mesh-sdk/go" + ptpZero + "\n\tgit.novox.be/novox/mesh-delivery/modules/b/gone" + ptpZero + "\n)\n",
|
||||
"modules/b/builds": "module git.novox.be/novox/mesh-delivery/modules/b/builds\n\nrequire git.novox.be/novox/mesh-delivery/modules/c/client" + ptpZero + " // indirect\n",
|
||||
"modules/c/client": "module git.novox.be/novox/mesh-delivery/modules/c/client\n",
|
||||
}
|
||||
|
||||
// ptpProgram is modules/g's manifest: a program naming no paths beside the package state.
|
||||
const ptpProgram = `{"module":"g","version":"1.0","build":{"artifacts":[{"name":"program","kind":"bundle","language":"go","system":"arch","from":"cmd/g","binary":"g","loads":["g"]},` +
|
||||
`{"name":"state","kind":"package","language":"go","paths":["state"],"series":"0.1"}]}}`
|
||||
|
||||
// ptpIndexer answers the fixture's go.mod files and, for a path asked from the root, the object objects names for it.
|
||||
func ptpIndexer(t *testing.T, objects map[string]string) (*Indexer, *memStore, *[]map[string][]string) {
|
||||
ix, mods, _ := manifestsIndexer(t)
|
||||
ix.GoMod = func(_ context.Context, _, _, dir string) ([]byte, error) {
|
||||
if c, ok := ptpGoMods[dir]; ok {
|
||||
return []byte(c), nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
var asked []map[string][]string
|
||||
ix.Named = func(_ context.Context, _, _ string, paths map[string][]string) (map[string]map[string]string, error) {
|
||||
asked = append(asked, paths)
|
||||
out := map[string]map[string]string{}
|
||||
for dir, ps := range paths {
|
||||
out[dir] = map[string]string{}
|
||||
for _, p := range ps {
|
||||
out[dir][p] = objects[p]
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
return ix, mods, &asked
|
||||
}
|
||||
|
||||
// Design 47 H-A-202, H-A-172: the same-repository modules a Go build resolves from the tree, transitively, join the tree
|
||||
// input of every form with their objects, so a change of one moves the fingerprint of a module whose program names no
|
||||
// paths; the entry's paths stay the manifest's.
|
||||
func TestADerivedDirectoryMovesTheFingerprintOfEveryForm(t *testing.T) {
|
||||
objects := map[string]string{"modules/b/builds": strings.Repeat("b", 40), "modules/c/client": strings.Repeat("c", 40)}
|
||||
tree := func(objects map[string]string) string {
|
||||
ix, mods, asked := ptpIndexer(t, objects)
|
||||
f := withManifests(mergedMsg("novox/mesh-delivery", map[string]string{"modules/g": "gtree"}), map[string]string{"modules/g": ptpProgram}, nil)
|
||||
if err := ix.Apply(context.Background(), f); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, _ := moduleOf(t, mods, "g")
|
||||
if got := strings.Join(m.Paths["state"], ","); got != "state" {
|
||||
t.Errorf("the entry's paths of state are %q, want the manifest's", got)
|
||||
}
|
||||
if len(*asked) != 1 || strings.Join((*asked)[0][""], ",") != "modules/b/builds,modules/c/client" {
|
||||
t.Fatalf("asked the forge %v, want the derived directories from the root once", *asked)
|
||||
}
|
||||
if m.Inputs2 == nil || !strings.HasPrefix(m.Inputs2.Tree, "gtree\n") {
|
||||
t.Fatalf("inputs = %+v (%s), want the directory's tree with the derived lines", m.Inputs2, m.None2)
|
||||
}
|
||||
return m.Inputs2.Tree
|
||||
}
|
||||
before := tree(objects)
|
||||
if want := "gtree\nderived modules/b/builds " + objects["modules/b/builds"] + "\nderived modules/c/client " + objects["modules/c/client"]; before != want {
|
||||
t.Errorf("the tree input is %q, want %q", before, want)
|
||||
}
|
||||
objects["modules/c/client"] = strings.Repeat("d", 40)
|
||||
after := tree(objects)
|
||||
fp := func(tree string) string {
|
||||
v, _ := fingerprint.Of(fingerprint.Inputs{Module: "g", Tree: tree})
|
||||
return v
|
||||
}
|
||||
if fp(before) == fp(after) {
|
||||
t.Errorf("a change of modules/c/client left the fingerprint %s", fp(after))
|
||||
}
|
||||
// The named tree takes them too: a module whose every artifact names its paths.
|
||||
named := `{"module":"g","version":"1.0","build":{"artifacts":[{"name":"state","kind":"package","language":"go","paths":["state"],"series":"0.1"}]}}`
|
||||
ix, mods, _ := ptpIndexer(t, objects)
|
||||
f := withManifests(mergedMsg("novox/mesh-delivery", map[string]string{"modules/g": "gtree"}), map[string]string{"modules/g": named}, nil)
|
||||
f.Paths = map[string]map[string]string{"modules/g": {"state": strings.Repeat("1", 40)}}
|
||||
if err := ix.Apply(context.Background(), f); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m, _ := moduleOf(t, mods, "g"); m.Inputs2 == nil || !strings.HasPrefix(m.Inputs2.Tree, fingerprint.NamedTreePrefix) ||
|
||||
!strings.HasSuffix(m.Inputs2.Tree, "\nderived modules/c/client "+objects["modules/c/client"]) || strings.Join(m.Paths["state"], ",") != "state" {
|
||||
t.Errorf("module = %+v, want the named tree with the derived lines and the manifest's paths", m)
|
||||
}
|
||||
// A forge that names no object of a derived directory gives the module none, naming it.
|
||||
ix, mods, _ = ptpIndexer(t, map[string]string{"modules/b/builds": strings.Repeat("b", 40)})
|
||||
if err := ix.Apply(context.Background(), withManifests(mergedMsg("novox/mesh-delivery", map[string]string{"modules/g": "gtree"}),
|
||||
map[string]string{"modules/g": ptpProgram}, nil)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m, _ := moduleOf(t, mods, "g"); m.Fp2 != "" || !strings.Contains(m.None2, "modules/c/client") {
|
||||
t.Errorf("module = %+v, want none naming the directory the forge gave no object of", m)
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 H-A-201: the entry pins every mesh package of another repository the module's build requires, across the
|
||||
// same-repository modules of its graph: here the one modules/c/client requires, which modules/g's go.mod never names.
|
||||
// A module of the module's own repository is never pinned, whatever a build reported (H-A-198, H-A-202).
|
||||
func TestTheEntryPinsEveryRequirementOfTheModuleGraph(t *testing.T) {
|
||||
const conditions = "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
objects := map[string]string{"modules/b/builds": strings.Repeat("b", 40), "modules/c/client": strings.Repeat("c", 40)}
|
||||
ix, mods, _ := ptpIndexer(t, objects)
|
||||
mod := map[string]string{
|
||||
"modules/g": "module g\n\nrequire git.novox.be/novox/mesh-delivery/modules/b/builds" + ptpZero + "\n",
|
||||
"modules/c/client": "module git.novox.be/novox/mesh-delivery/modules/c/client\n\nrequire " + conditions + ptpZero + "\n",
|
||||
}
|
||||
ix.GoMod = func(_ context.Context, _, _, dir string) ([]byte, error) {
|
||||
if c, ok := mod[dir]; ok {
|
||||
return []byte(c), nil
|
||||
}
|
||||
return []byte(ptpGoMods[dir]), nil
|
||||
}
|
||||
f := withManifests(mergedMsg("novox/mesh-delivery", map[string]string{"modules/g": "gtree"}), map[string]string{"modules/g": ptpProgram},
|
||||
map[string]map[string]string{"modules/g": {"go.sum": avahiGoSum}})
|
||||
if err := ix.Apply(context.Background(), f); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, _ := moduleOf(t, mods, "g")
|
||||
if m.Inputs2 == nil || strings.Join(m.Inputs2.Packages, ",") != conditions+",git.novox.be/novox/mesh-sdk/go" {
|
||||
t.Errorf("packages = %+v, want the client's requirement and state's mesh-sdk, and no module of the repository", m.Inputs2)
|
||||
}
|
||||
builds := "git.novox.be/novox/mesh-delivery/modules/b/builds"
|
||||
pins := releasedPins(sdkstep.PackageVersions{builds: "v0.1.106", conditions: "v0.1.4"}, nil, catalogue.Manifest{}, "novox/mesh-delivery")
|
||||
if _, pinned := pins[builds]; pinned || len(pins) != 1 {
|
||||
t.Errorf("pins = %v, want %s only, never a module of the repository", pins, conditions)
|
||||
}
|
||||
}
|
||||
@@ -218,8 +218,7 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc
|
||||
ix.logf("no fingerprint is written by this pass: %v", err)
|
||||
return fmt.Errorf("no fingerprint is written: %w", err)
|
||||
}
|
||||
series := ix.SeriesAll(ctx) // each package's series, the newest a user accepting it takes (DLV-I6)
|
||||
released := ix.releasedOf(ctx) // each module's newest released version, what a package is pinned to (H-A-161)
|
||||
series := ix.SeriesAll(ctx) // each package's series, the newest a user accepting it takes (DLV-I6)
|
||||
type result struct {
|
||||
fp, none string
|
||||
tcs map[string]meshcatalog.ToolchainUsed // by language, when there is a fingerprint
|
||||
@@ -291,7 +290,7 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc
|
||||
r.none = strings.Join(missing, "; ")
|
||||
case noToolchain != "":
|
||||
r.none = noToolchain
|
||||
if fp, pkgs, ok := stillComputed(module, kept, k, toolchains, held, pub, pins, ix.sourceHeads(), released); ok {
|
||||
if fp, pkgs, ok := stillComputed(module, kept, k, toolchains, held, pub, pins, ix.sourceHeads(), series); ok {
|
||||
r = result{fp: fp, tcs: k.held.Toolchains, pkgs: pkgs}
|
||||
}
|
||||
default:
|
||||
@@ -299,7 +298,7 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc
|
||||
r.fp, r.none = f.fp, f.none
|
||||
if r.fp != "" {
|
||||
// a pin is a released version, never a commit (H-A-161), and the version line's fingerprint takes it (H-A-167)
|
||||
r.fp, r.pkgs = pinnedOf(f, held, released)
|
||||
r.fp, r.pkgs = pinnedOf(f, series, k.manifest, k.cur.Repository)
|
||||
r.tcs = toolchainsUsed(toolchains)
|
||||
}
|
||||
}
|
||||
@@ -498,7 +497,7 @@ func hasContext(m catalogue.Manifest) bool {
|
||||
// with the toolchains it was computed with in place of the languages agreed on none, give it again, in its form. Any
|
||||
// other input that moved (a merge, a base, an agreed toolchain, an imported package) gives another fingerprint, and the
|
||||
// module has none until they agree.
|
||||
func stillComputed(module string, kept meshcatalog.ModulesByModuleValueInputs2, k *known, agreed map[string]meshcatalogclient.Toolchain2, held map[string]meshcatalog.ModulesByModuleValue, published map[string]meshcatalog.PackagesByModuleValue, pins sdkstep.PackageVersions, heads map[string]string, released map[string]string) (string, sdkstep.PackageVersions, bool) {
|
||||
func stillComputed(module string, kept meshcatalog.ModulesByModuleValueInputs2, k *known, agreed map[string]meshcatalogclient.Toolchain2, held map[string]meshcatalog.ModulesByModuleValue, published map[string]meshcatalog.PackagesByModuleValue, pins sdkstep.PackageVersions, heads map[string]string, series map[string]SeriesPackage) (string, sdkstep.PackageVersions, bool) {
|
||||
if k.held.Fp2 == "" {
|
||||
return "", nil, false
|
||||
}
|
||||
@@ -518,7 +517,7 @@ func stillComputed(module string, kept meshcatalog.ModulesByModuleValueInputs2,
|
||||
if f.none != "" || f.fp == "" {
|
||||
return "", nil, false
|
||||
}
|
||||
fp, pkgs := pinnedOf(f, held, released)
|
||||
fp, pkgs := pinnedOf(f, series, k.manifest, k.held.Repository)
|
||||
return fp, pkgs, fp == k.held.Fp2
|
||||
}
|
||||
|
||||
|
||||
@@ -805,17 +805,34 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) {
|
||||
t.Fatal("a release whose re-pin failed was acknowledged")
|
||||
}
|
||||
mods.keysErr = nil
|
||||
// The registry published mesh-sdk's Go package at 0.1.18, not at the module's release number 0.1.20, in the series
|
||||
// avahi accepts (the default 0.1), with no contents read: the pin is that version (H-A-183, H-A-185).
|
||||
ix.Packages = newMem()
|
||||
if _, err := ix.HearVersion(context.Background(), HeardVersion{Package: "mesh-sdk/go", Seat: "go-package-registry", Version: "0.1.18", Commit: strings.Repeat("1", 40), GoModule: sdk, Published: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ix.HearPackagePublished(context.Background(), release); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ix.Revisions = revisions
|
||||
if a, _ := moduleOf(t, mods, "avahi"); a.Fp2 == first.Fp2 || a.Fp2 == "" || !reflect.DeepEqual(a.Packages, map[string]string{sdk: "v0.1.20"}) {
|
||||
t.Fatalf("after mesh-sdk's release: fp2 %q (was %q) packages %v, want a new fingerprint pinned to v0.1.20", a.Fp2, first.Fp2, a.Packages)
|
||||
if a, _ := moduleOf(t, mods, "avahi"); a.Fp2 == first.Fp2 || a.Fp2 == "" || !reflect.DeepEqual(a.Packages, map[string]string{sdk: "v0.1.18"}) {
|
||||
t.Fatalf("after mesh-sdk's release: fp2 %q (was %q) packages %v, want a new fingerprint pinned to v0.1.18", a.Fp2, first.Fp2, a.Packages)
|
||||
} else {
|
||||
// A 0.2.0 publish, of a series avahi does not accept, leaves it alone, contents or none (H-A-184, H-A-185).
|
||||
if _, err := ix.HearVersion(context.Background(), HeardVersion{Package: "mesh-sdk/go", Seat: "go-package-registry", Version: "0.2.0", Commit: strings.Repeat("1", 40), GoModule: sdk, Published: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := ix.Repin(context.Background(), "mesh-sdk"); err != nil { // the pass for avahi alone: mesh-tools is held stale here
|
||||
t.Fatal(err)
|
||||
}
|
||||
if b, _ := moduleOf(t, mods, "avahi"); b.Fp2 != a.Fp2 || b.Packages[sdk] != "v0.1.18" {
|
||||
t.Fatalf("a publish of another series moved avahi: fp2 %q (was %q) packages %v", b.Fp2, a.Fp2, b.Packages)
|
||||
}
|
||||
}
|
||||
if o, _ := moduleOf(t, mods, "mesh-tools"); o.Fp2 != tools.Fp2 {
|
||||
t.Errorf("a re-pin moved mesh-tools, which holds no package of mesh-sdk: %q (was %q)", o.Fp2, tools.Fp2)
|
||||
}
|
||||
second := sdkMerge(strings.Repeat("2", 40), "tree-sdk-2", "v0.1.20")
|
||||
second := sdkMerge(strings.Repeat("2", 40), "tree-sdk-2", "v0.1.18")
|
||||
if second.Fp2 == first.Fp2 {
|
||||
t.Errorf("a mesh-sdk trunk merge kept avahi's fingerprint %q", first.Fp2)
|
||||
}
|
||||
|
||||
@@ -15,7 +15,9 @@ import (
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
|
||||
"mesh-delivery-indexer/internal/fingerprint"
|
||||
@@ -366,51 +368,32 @@ func isFloor(raw []byte) bool {
|
||||
return json.Unmarshal(raw, &v) == nil && len(v.Artifacts) == 0
|
||||
}
|
||||
|
||||
// releasedOf is the newest released version of each module the state versions keeps (§349; design 47 H-A-161), by module;
|
||||
// nil when the indexer keeps no such state or it cannot be listed, said.
|
||||
func (ix *Indexer) releasedOf(ctx context.Context) map[string]string {
|
||||
if ix.Versions == nil {
|
||||
return nil
|
||||
}
|
||||
all, err := ix.Versions.Keys(ctx)
|
||||
if err != nil {
|
||||
ix.logf("versions: the state cannot be listed, so no package is pinned by this pass: %v", err)
|
||||
return nil
|
||||
}
|
||||
out := map[string]string{}
|
||||
for _, k := range all {
|
||||
m, v, ok := meshcatalog.ParseVersionsByModuleVersionKey(k)
|
||||
if _, parts := versionParts(v); !ok || !parts {
|
||||
continue
|
||||
}
|
||||
if cur, has := out[m]; !has || compareVersions(v, cur) > 0 {
|
||||
out[m] = v
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// releasedPins are a module's pins (its catalog entry's packages, which the builder builds it against) from the versions its
|
||||
// fingerprint was computed with (design 47 H-A-161, G §370): a package the held module holding it has released is pinned
|
||||
// to that module's newest released version, as Go names it (v<version>); a version already in that form (an accepted
|
||||
// series' newest, DLV-I6) stands; any other, a commit, is no pin: its package is kept with no version, so its kind-1 edge
|
||||
// stands (H §746) and no build is made against a commit. nil for none.
|
||||
func releasedPins(versions sdkstep.PackageVersions, held map[string]meshcatalog.ModulesByModuleValue, released map[string]string) sdkstep.PackageVersions {
|
||||
// fingerprint was computed with (design 47 H-A-161, H-A-183, H-A-184, H-A-185, G §370): each package is pinned to the
|
||||
// newest version the registry published for its path (the seats' published, kept in the package's series by HearVersion)
|
||||
// within the series the module accepts, the one its manifest m names or the default (Manifest.AcceptedSeries), whether
|
||||
// or not that series holds contents yet; a publish in another series pins nothing and moves nothing, and a module's
|
||||
// release number is never a pin. A package with none published in that series is kept with no version, so its kind-1
|
||||
// edge stands (H §746) and its build waits for the publish (H-A-165). A module of the module's own repository is never
|
||||
// pinned, whatever a build reported: it is built from the tree (design 47 H-A-198, H-A-201, H-A-202). nil for none.
|
||||
func releasedPins(versions sdkstep.PackageVersions, series map[string]SeriesPackage, m catalogue.Manifest, repository string) sdkstep.PackageVersions {
|
||||
if len(versions) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := sdkstep.PackageVersions{}
|
||||
for p, v := range versions {
|
||||
rel := released[moduleOfKey(fingerprint.HolderOf(p, held))]
|
||||
series := sdkstep.PackageSemver.MatchString(v)
|
||||
_, plain := versionParts(strings.TrimPrefix(v, "v"))
|
||||
switch {
|
||||
case rel != "" && (!series || !plain || compareVersions(rel, strings.TrimPrefix(v, "v")) > 0):
|
||||
out[p] = "v" + rel // never an older series' version over a newer release (H-A-167)
|
||||
case series:
|
||||
out[p] = v
|
||||
default:
|
||||
out[p] = ""
|
||||
for p := range versions {
|
||||
if repo, _, ok := trunk.PackageRepository(p); ok && repository != "" && trunk.SameRepository(repo, repository) {
|
||||
continue
|
||||
}
|
||||
out[p] = ""
|
||||
for _, s := range series {
|
||||
if s.GoModule != p || s.Newest == nil || s.Series != m.AcceptedSeries(s.Name) {
|
||||
continue
|
||||
}
|
||||
v := strings.TrimPrefix(s.Newest.Version, "v")
|
||||
if _, ok := versionParts(v); ok && (out[p] == "" || compareVersions(v, strings.TrimPrefix(out[p], "v")) > 0) {
|
||||
out[p] = "v" + v
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
@@ -420,8 +403,8 @@ func releasedPins(versions sdkstep.PackageVersions, held map[string]meshcatalog.
|
||||
// form's fingerprint takes them (design 47 H-A-167, H-A-172; ADR 0573 §4) and a re-pin moves it, the module a mover
|
||||
// built again against the release; a fingerprint with no pin is the form's as it is. The held fingerprint of a build that
|
||||
// stands is kept while its pins are the ones it was built with, else its import form takes the new pins.
|
||||
func pinnedOf(f form, held map[string]meshcatalog.ModulesByModuleValue, released map[string]string) (string, sdkstep.PackageVersions) {
|
||||
pins := releasedPins(f.versions, held, released)
|
||||
func pinnedOf(f form, series map[string]SeriesPackage, m catalogue.Manifest, repository string) (string, sdkstep.PackageVersions) {
|
||||
pins := releasedPins(f.versions, series, m, repository)
|
||||
if f.stands {
|
||||
if maps.Equal(f.built, map[string]string(pins)) {
|
||||
return f.fp, pins
|
||||
|
||||
@@ -42,6 +42,8 @@ type artifact struct {
|
||||
Binary string `json:"binary"`
|
||||
Loads []string `json:"loads"`
|
||||
Env map[string]string `json:"env"`
|
||||
Paths []string `json:"paths"`
|
||||
Series string `json:"series"`
|
||||
}
|
||||
|
||||
type stream struct {
|
||||
@@ -344,8 +346,12 @@ func TestManifestProcess(t *testing.T) { // 1238/AC1
|
||||
if _, bus := p["bus"]; bus {
|
||||
t.Errorf("the process declares a bus: it runs on its own broker credential")
|
||||
}
|
||||
if len(m.Build.Artifacts) != 2 {
|
||||
t.Fatalf("artifacts = %+v, want the program and the tools", m.Build.Artifacts)
|
||||
if len(m.Build.Artifacts) != 3 {
|
||||
t.Fatalf("artifacts = %+v, want the program, the tools and the package takein", m.Build.Artifacts)
|
||||
}
|
||||
// H-A-186 (design 47 G §370): takein, which the gate, the view and the watchman import, is a declared package.
|
||||
if a := m.Build.Artifacts[2]; a.Name != "takein" || a.Kind != "package" || a.Language != "go" || !reflect.DeepEqual(a.Paths, []string{"takein"}) || a.Series != "0.1" {
|
||||
t.Errorf("package artifact = %+v", a)
|
||||
}
|
||||
if a := m.Build.Artifacts[1]; a.Name != "tools" || a.Kind != "bundle" || a.Binary != "mesh-delivery-indexer-tools" || !reflect.DeepEqual(a.Loads, []string{"mesh-delivery-indexer-tools"}) {
|
||||
t.Errorf("tools artifact = %+v", a) // 1945
|
||||
|
||||
@@ -223,6 +223,15 @@
|
||||
"loads": [
|
||||
"mesh-delivery-indexer-tools"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "takein",
|
||||
"kind": "package",
|
||||
"language": "go",
|
||||
"paths": [
|
||||
"takein"
|
||||
],
|
||||
"series": "0.1"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -19,7 +19,7 @@ import (
|
||||
// §549): runner_unlogged lists them, runner_unlogged_append appends them by hand, written in the hand-act log; it is of class
|
||||
// act and takes §391's default callers.
|
||||
func UnloggedTools(pipelines stdio.KeptState, log stdio.KeptLog, acts views.HandActs, logf func(string, ...any)) []stdio.Tool {
|
||||
h := &pipeline.Hearer{Store: stdioPipelines{pipelines}, Logf: logf,
|
||||
h := &pipeline.Hearer{Store: &pipeline.MoverStore{Raw: stdioPipelines{pipelines}, Logf: logf}, Logf: logf,
|
||||
Log: func(e seat.LogEntry) error {
|
||||
_, err := log.Append(keys.EscapeName(e.Pipeline), e)
|
||||
return err
|
||||
@@ -67,26 +67,36 @@ func UnloggedTools(pipelines stdio.KeptState, log stdio.KeptLog, acts views.Hand
|
||||
}
|
||||
}
|
||||
|
||||
// stdioPipelines is the state pipelines as the runtime serves it, for the by-hand append.
|
||||
// stdioPipelines is the state pipelines as bytes, as the runtime serves it, for the by-hand append: read and written through
|
||||
// a pipeline.MoverStore, which keeps each mover's steps under its own key (H-A-187).
|
||||
type stdioPipelines struct{ s stdio.KeptState }
|
||||
|
||||
func (s stdioPipelines) Get(key string) (seat.Pipeline, uint64, error) {
|
||||
func (s stdioPipelines) Get(key string) ([]byte, uint64, error) {
|
||||
e, err := s.s.Get(key)
|
||||
if err != nil || e == nil {
|
||||
return seat.Pipeline{}, 0, errors.Join(err, errors.New("no such pipeline"))
|
||||
return nil, 0, err
|
||||
}
|
||||
var p seat.Pipeline
|
||||
return p, e.Revision, json.Unmarshal(e.Value, &p)
|
||||
return e.Value, e.Revision, nil
|
||||
}
|
||||
|
||||
func (s stdioPipelines) Update(key string, p seat.Pipeline, rev uint64) (uint64, error) {
|
||||
r, err := s.s.Update(key, p, rev)
|
||||
func (s stdioPipelines) Create(key string, value []byte) (uint64, error) {
|
||||
r, err := s.s.Create(key, json.RawMessage(value))
|
||||
if err != nil && strings.Contains(err.Error(), "exists") {
|
||||
return 0, pipeline.ErrExists
|
||||
}
|
||||
return r, err
|
||||
}
|
||||
|
||||
func (s stdioPipelines) Update(key string, value []byte, rev uint64) (uint64, error) {
|
||||
r, err := s.s.Update(key, json.RawMessage(value), rev)
|
||||
if err != nil && (strings.Contains(err.Error(), "wrong last sequence") || strings.Contains(err.Error(), "revision")) {
|
||||
return 0, pipeline.ErrChanged
|
||||
}
|
||||
return r, err
|
||||
}
|
||||
|
||||
func (s stdioPipelines) Delete(key string) error { return s.s.Delete(key) }
|
||||
|
||||
func (s stdioPipelines) Keys(filter string) ([]string, error) {
|
||||
all, err := s.s.Keys()
|
||||
var out []string
|
||||
|
||||
@@ -136,9 +136,9 @@ func (k keptStore) Get(key string) (seat.KeptResult, bool, error) {
|
||||
func (k keptStore) Delete(key string) error { return k.kv.Delete(k.ctx, key) }
|
||||
|
||||
// newAger is the ageing over the runner's own stores.
|
||||
func newAger(ctx context.Context, js jetstream.JetStream, cred process.Account, pipelines, kept, aged jetstream.KeyValue, logf func(string, ...any)) *pipeline.Ager {
|
||||
func newAger(ctx context.Context, js jetstream.JetStream, cred process.Account, pipelines *pipeline.MoverStore, kept, aged jetstream.KeyValue, logf func(string, ...any)) *pipeline.Ager {
|
||||
return &pipeline.Ager{Log: stepLog{ctx: ctx, js: js, module: cred.Module}.Read, Aged: agedState{ctx: ctx, kv: aged},
|
||||
Pipelines: pipelineDeleter{pipelineStore{ctx: ctx, kv: pipelines}}, Kept: keptStore{ctx: ctx, kv: kept}, Logf: logf,
|
||||
Pipelines: pipelines, Kept: keptStore{ctx: ctx, kv: kept}, Logf: logf,
|
||||
Outputs: func(name string) error {
|
||||
o, err := js.ObjectStore(ctx, catalogue.ObjectsBucket(cred.Module, seat.StateStepOutputs))
|
||||
if err != nil {
|
||||
@@ -151,11 +151,6 @@ func newAger(ctx context.Context, js jetstream.JetStream, cred process.Account,
|
||||
}}
|
||||
}
|
||||
|
||||
// pipelineDeleter is the pipelines store with the delete only its owner makes (§330).
|
||||
type pipelineDeleter struct{ pipelineStore }
|
||||
|
||||
func (d pipelineDeleter) Delete(key string) error { return d.kv.Delete(d.ctx, key) }
|
||||
|
||||
// storeFills reads the fill of the runner's stores against their caps: pipelines and kept (their key-value streams), step-outputs
|
||||
// (its object store's) and step-log (its log's), as the bus's own streams say it (design 47 G §366, §486).
|
||||
func storeFills(ctx context.Context, js jetstream.JetStream, cred process.Account, now func() time.Time) func() ([]seat.Fill, error) {
|
||||
|
||||
@@ -128,9 +128,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
return err
|
||||
}
|
||||
hearer := newHearer(ctx, js, cred, pipelines, kept, say, logger.Printf)
|
||||
store := newPipelineStore(ctx, pipelines, logger.Printf)
|
||||
hearer := newHearer(ctx, js, cred, store, kept, say, logger.Printf)
|
||||
hearer.Asker = busAsker{conn: conn, cred: cred}
|
||||
hearer.Ager = newAger(ctx, js, cred, pipelines, kept, aged, logger.Printf)
|
||||
hearer.Ager = newAger(ctx, js, cred, store, kept, aged, logger.Printf)
|
||||
// The stores' fill is said before a store is full, and a full one refuses a start (§366).
|
||||
fills := &pipeline.Fills{Read: storeFills(ctx, js, cred, time.Now), Logf: logger.Printf,
|
||||
Say: func(event string, f seat.Fill) error { return say("mesh.seat."+seat.Seat+".event."+event, f) }}
|
||||
@@ -147,7 +148,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
hearer.Failures = &pipeline.Failures{Findings: rv.Findings, Observer: cred.Module, Author: commitAuthor(calls.Call), Logf: logger.Printf}
|
||||
// Its asking half (§22, §24): a pipeline's steps are placed from the module's registration and each build seat is asked
|
||||
// for its step.
|
||||
starter := &pipeline.Starter{Pipelines: createdKV{ctx: ctx, kv: pipelines}, Logf: logger.Printf, Say: say,
|
||||
starter := &pipeline.Starter{Pipelines: store, Logf: logger.Printf, Say: say,
|
||||
Registration: rv.Registrations.Registration, Steps: busStepAsker{ctx: ctx, js: js, cred: cred}, Name: cred.Module,
|
||||
Kept: keptReader(ctx, kept), Toolchain: toolchainReader(ctx, toolchains),
|
||||
Fills: fills, Recovered: kvRecoveries{ctx, recovered}.Has, Holders: rv.Holders.Holders, Ended: hearer.EndedAtPlacing}
|
||||
@@ -170,10 +171,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return
|
||||
}
|
||||
// A pipeline of the branch whose registration was heard before this half was held gets its steps now (A-327).
|
||||
if n, err := starter.Waiting(pipelineStore{ctx: ctx, kv: pipelines}, branch, rv.Catalog.View); err != nil || n > 0 {
|
||||
if n, err := starter.Waiting(store, branch, rv.Catalog.View); err != nil || n > 0 {
|
||||
logger.Printf("the pipelines of %s that waited on its half for their steps: %d placed (%v)", branch, n, err)
|
||||
}
|
||||
n, err := starter.Resume(pipelineStore{ctx: ctx, kv: pipelines}, branch, half)
|
||||
n, err := starter.Resume(store, branch, half)
|
||||
if err != nil {
|
||||
logger.Printf("the steps placed on %s and not yet asked: %d asked, then %v", branch, n, err)
|
||||
return
|
||||
@@ -184,19 +185,19 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// The view reaching a newer revision places the steps of the pipelines that waited for it, and asks nothing again (A-448, §200).
|
||||
advance := func(branch string) {
|
||||
rv.Holders.refresh()
|
||||
if n, err := starter.Waiting(pipelineStore{ctx: ctx, kv: pipelines}, branch, rv.Catalog.View); err != nil || n > 0 {
|
||||
if n, err := starter.Waiting(store, branch, rv.Catalog.View); err != nil || n > 0 {
|
||||
logger.Printf("the pipelines of %s that waited for a newer revision of its half: %d placed (%v)", branch, n, err)
|
||||
}
|
||||
}
|
||||
advancer.Store(&advance)
|
||||
// A registration-changed places the steps of its module's pipelines started before it, from it (A-327).
|
||||
register := func(reg catalog.Registration) {
|
||||
if n, err := starter.Registered(pipelineStore{ctx: ctx, kv: pipelines}, reg, rv.Catalog.View); err != nil || n > 0 {
|
||||
if n, err := starter.Registered(store, reg, rv.Catalog.View); err != nil || n > 0 {
|
||||
logger.Printf("the pipelines of %s started before its registration: %d placed (%v)", reg.Module, n, err)
|
||||
}
|
||||
}
|
||||
registrar.Store(®ister)
|
||||
starter.Store = pipelineStore{ctx: ctx, kv: pipelines}
|
||||
starter.Store = store
|
||||
// The queue (design 47 H §748): at most pipelines-at-once pipelines have their steps asked, the others wait in order in
|
||||
// the state queue, placed at the start from it and on every end.
|
||||
atOnce, waitingCap, err := queueSettings(getenv)
|
||||
|
||||
@@ -401,6 +401,8 @@ func pipelineKeys(t *testing.T, ctx context.Context, kv jetstream.KeyValue) []st
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A mover's steps are a key of their own (H-A-187), placed with its pipeline: no pipeline of their own.
|
||||
ks = slices.DeleteFunc(ks, pipeline.IsStepsKey)
|
||||
slices.Sort(ks)
|
||||
return ks
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -221,6 +222,34 @@ func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *tes
|
||||
|
||||
waitFor(t, ctx, logs, func() bool { return readPipeline(ctx, pipelines, key, &p) && p.Outcome == seat.PipelineOutcomePassed })
|
||||
|
||||
// H-A-187 over the bus: show composes the pipeline from its record and mover keys, naming a mover its steps alone, and a
|
||||
// record written before it, its steps inline, is read whole and split at its next write.
|
||||
shown, err := pipeline.Reads{Pipelines: kvKept{ctx, pipelines}}.Show(seat.ShowInput{Pipeline: key, Mover: "app"})
|
||||
if err != nil || len(shown.Pipeline.Steps) != len(p.Steps) || shown.Pipeline.Steps[0].Module != "app" {
|
||||
t.Fatalf("show of %s naming app = %d steps (%v), want its %d", key, len(shown.Pipeline.Steps), err, len(p.Steps))
|
||||
}
|
||||
inline := p
|
||||
inline.Pipeline = "main.2"
|
||||
inlineRaw, _ := json.Marshal(inline)
|
||||
if _, err := pipelines.Put(ctx, "main.2", inlineRaw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store := newPipelineStore(ctx, pipelines, func(f string, a ...any) { fmt.Fprintf(logs, f+"\n", a...) })
|
||||
old, oldRev, err := store.Get("main.2")
|
||||
if err != nil || len(old.Steps) != len(p.Steps) {
|
||||
t.Fatalf("the inline record read %d steps (%v), want %d", len(old.Steps), err, len(p.Steps))
|
||||
}
|
||||
if _, err := store.Update("main.2", old, oldRev); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var split seat.Pipeline
|
||||
if !readPipeline(ctx, pipelines, "main.2", &split) || len(split.Steps) != len(p.Steps) {
|
||||
t.Fatalf("the record split reads %d steps, want %d", len(split.Steps), len(p.Steps))
|
||||
}
|
||||
if _, err := pipelines.Get(ctx, pipeline.StepsKey("main.2", "app")); err != nil || !strings.Contains(logs.String(), "split into 1 mover keys") {
|
||||
t.Fatalf("the mover key of main.2's app: %v; the journal says the split: %t", err, strings.Contains(logs.String(), "split into 1 mover keys"))
|
||||
}
|
||||
|
||||
// §470, §330: the step log is the log's stream, read onward by sequence the way the ageing reads it.
|
||||
logged, err := stepLog{ctx: ctx, js: js, module: module}.Read(1, 100)
|
||||
if err != nil || len(logged) == 0 || logged[0].Entry.Pipeline != key {
|
||||
@@ -274,13 +303,20 @@ func stepOf(p seat.Pipeline, step string) seat.StepResult {
|
||||
return seat.StepResult{}
|
||||
}
|
||||
|
||||
// readPipeline reads pipeline key as the runner does, its steps composed from its mover keys, and holds its record to
|
||||
// H-A-187: a record the runner wrote carries no steps of its own, so it never grows with the catalog.
|
||||
func readPipeline(ctx context.Context, kv jetstream.KeyValue, key string, p *seat.Pipeline) bool {
|
||||
e, err := kv.Get(ctx, key)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
*p = seat.Pipeline{}
|
||||
return json.Unmarshal(e.Value(), p) == nil
|
||||
var record seat.Pipeline
|
||||
if json.Unmarshal(e.Value(), &record) != nil || len(record.Steps) > 0 {
|
||||
panic(fmt.Sprintf("the record of %s carries %d steps of its own; each mover's steps are its own key (H-A-187)", key, len(record.Steps)))
|
||||
}
|
||||
got, _, err := newPipelineStore(ctx, kv, nil).Get(key)
|
||||
*p = got
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// waitFor asks cond until it holds or the test's context ends, the module's journal in the failure.
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
@@ -13,20 +11,6 @@ import (
|
||||
"mesh-delivery-pipeline/internal/pipeline"
|
||||
)
|
||||
|
||||
// createdKV is the state pipelines written by compare-and-set on a key's absence (§54).
|
||||
type createdKV struct {
|
||||
ctx context.Context
|
||||
kv jetstream.KeyValue
|
||||
}
|
||||
|
||||
func (c createdKV) Create(key string, value []byte) error {
|
||||
_, err := c.kv.Create(c.ctx, key, value)
|
||||
if errors.Is(err, jetstream.ErrKeyExists) {
|
||||
return pipeline.ErrExists
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// hearPublished hears each package-published (design 47 H §746, §747): in its source's running pipeline, or starting one
|
||||
// pipeline of the package's kind-1 dependents that pipeline does not move. The message is acknowledged once that is
|
||||
// written, on a goroutine of its own, so a slow write holds no other event; the durable consumer delivers a publish heard
|
||||
|
||||
@@ -19,64 +19,74 @@ import (
|
||||
"mesh-delivery-pipeline/internal/pipeline"
|
||||
)
|
||||
|
||||
// pipelineStore is the state pipelines as the hearer reads and writes it: a write by compare-and-set on the revision read (§27).
|
||||
type pipelineStore struct {
|
||||
// kvState is the state pipelines as bytes on the bus, by compare-and-set (§27); the runner reads and writes pipelines
|
||||
// through one pipeline.MoverStore over it, which keeps each mover's steps under its own key (H-A-187).
|
||||
type kvState struct {
|
||||
ctx context.Context
|
||||
kv jetstream.KeyValue
|
||||
}
|
||||
|
||||
func (s pipelineStore) Get(key string) (seat.Pipeline, uint64, error) {
|
||||
e, err := s.kv.Get(s.ctx, key)
|
||||
if err != nil {
|
||||
return seat.Pipeline{}, 0, err
|
||||
}
|
||||
var p seat.Pipeline
|
||||
if err := json.Unmarshal(e.Value(), &p); err != nil {
|
||||
return seat.Pipeline{}, 0, fmt.Errorf("the pipeline %s is no pipeline: %w", key, err)
|
||||
}
|
||||
return p, e.Revision(), nil
|
||||
// newPipelineStore is the process's one store of the state pipelines.
|
||||
func newPipelineStore(ctx context.Context, kv jetstream.KeyValue, logf func(string, ...any)) *pipeline.MoverStore {
|
||||
return &pipeline.MoverStore{Raw: kvState{ctx: ctx, kv: kv}, Logf: logf}
|
||||
}
|
||||
|
||||
func (s pipelineStore) Update(key string, p seat.Pipeline, revision uint64) (uint64, error) {
|
||||
raw, err := json.Marshal(p)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
func (s kvState) Get(key string) ([]byte, uint64, error) {
|
||||
e, err := s.kv.Get(s.ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return nil, 0, nil
|
||||
}
|
||||
rev, err := s.kv.Update(s.ctx, key, raw, revision)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
return e.Value(), e.Revision(), nil
|
||||
}
|
||||
|
||||
func (s kvState) Create(key string, value []byte) (uint64, error) {
|
||||
rev, err := s.kv.Create(s.ctx, key, value)
|
||||
if errors.Is(err, jetstream.ErrKeyExists) {
|
||||
return 0, pipeline.ErrExists
|
||||
}
|
||||
return rev, err
|
||||
}
|
||||
|
||||
func (s kvState) Update(key string, value []byte, revision uint64) (uint64, error) {
|
||||
rev, err := s.kv.Update(s.ctx, key, value, revision)
|
||||
if err != nil && (errors.Is(err, jetstream.ErrKeyExists) || strings.Contains(err.Error(), "wrong last sequence")) {
|
||||
return 0, pipeline.ErrChanged
|
||||
}
|
||||
return rev, err
|
||||
}
|
||||
|
||||
func (s pipelineStore) Keys(filter string) ([]string, error) {
|
||||
all, err := s.kv.Keys(s.ctx)
|
||||
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||
return nil, nil
|
||||
func (s kvState) Delete(key string) error {
|
||||
if err := s.kv.Delete(s.ctx, key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Keys are the keys matching filter, `*` one key part, as the bus filters a key-value's subjects.
|
||||
func (s kvState) Keys(filter string) ([]string, error) {
|
||||
l, err := s.kv.ListKeysFiltered(s.ctx, filter)
|
||||
if err != nil {
|
||||
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
// The lister ends its own watch once the current keys are listed.
|
||||
var out []string
|
||||
want := strings.Split(filter, ".")
|
||||
for _, k := range all {
|
||||
got := strings.Split(k, ".")
|
||||
if len(got) != len(want) {
|
||||
continue
|
||||
}
|
||||
ok := true
|
||||
for i := range want {
|
||||
ok = ok && (want[i] == "*" || want[i] == got[i])
|
||||
}
|
||||
if ok {
|
||||
out = append(out, k)
|
||||
}
|
||||
for k := range l.Keys() {
|
||||
out = append(out, k)
|
||||
}
|
||||
return out, err
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// newHearer is the hearer over the runner's own states: pipelines, kept, the step log on its stream and step-outputs on its
|
||||
// objects (§27). Each store is opened where it is used, so one the grants part has not made yet fails that write loudly.
|
||||
func newHearer(ctx context.Context, js jetstream.JetStream, cred process.Account, pipelines, kept jetstream.KeyValue,
|
||||
func newHearer(ctx context.Context, js jetstream.JetStream, cred process.Account, pipelines pipeline.Updater, kept jetstream.KeyValue,
|
||||
say func(string, any) error, logf func(string, ...any)) *pipeline.Hearer {
|
||||
return &pipeline.Hearer{Store: pipelineStore{ctx: ctx, kv: pipelines}, Say: say, Logf: logf, Name: cred.Module,
|
||||
return &pipeline.Hearer{Store: pipelines, Say: say, Logf: logf, Name: cred.Module,
|
||||
Log: stepLog{ctx: ctx, js: js, module: cred.Module}.Append,
|
||||
Keep: func(key string, k seat.KeptResult) error {
|
||||
raw, err := json.Marshal(k)
|
||||
|
||||
@@ -0,0 +1,396 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
)
|
||||
|
||||
// A pipeline's steps live one key per mover (design 47 H-A-187): the state pipelines keeps the pipeline's record (its
|
||||
// movers, tiers, call, source and outcome) under the pipeline's key with its own steps empty, and each mover's steps under
|
||||
// `steps.<pipeline>.<mover>`, written by compare-and-set as the record is, so a record never grows with the catalog and no
|
||||
// value nears the bus's message limit (ADR 0486). Every read of a pipeline composes its steps from the mover keys it names.
|
||||
|
||||
// StepsKey is the key of one mover's steps in the state pipelines, `steps.<pipeline>.<mover>`, the mover's module escaped
|
||||
// as a key's name is (H-A-187).
|
||||
func StepsKey(pipeline, mover string) string {
|
||||
return seat.PipelinesByStepsPipelineMoverKey(pipeline, keys.EscapeName(mover))
|
||||
}
|
||||
|
||||
// stepsPrefix is the start of every mover key of one pipeline.
|
||||
func stepsPrefix(pipeline string) string { return StepsKey(pipeline, "") }
|
||||
|
||||
// IsStepsKey says whether a key of the state pipelines is a mover's steps rather than a pipeline's record: `steps.` and a
|
||||
// pipeline key of a take-in's (4 parts), a run's (5 parts, its third `run`) or a publish's (7 parts) and one mover.
|
||||
func IsStepsKey(key string) bool {
|
||||
parts := strings.Split(key, ".")
|
||||
if parts[0] != "steps" {
|
||||
return false
|
||||
}
|
||||
switch len(parts) {
|
||||
case 4, 7:
|
||||
return true
|
||||
case 5:
|
||||
return parts[2] == runToken
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// RawState is the state pipelines as bytes: a key's value and revision (nil, 0 and no error when it has none), a create by
|
||||
// compare-and-set on absence (ErrExists), an update on the revision read (ErrChanged), a delete, and the keys matching a
|
||||
// filter, `*` one key part.
|
||||
type RawState interface {
|
||||
Get(key string) ([]byte, uint64, error)
|
||||
Create(key string, value []byte) (uint64, error)
|
||||
Update(key string, value []byte, revision uint64) (uint64, error)
|
||||
Delete(key string) error
|
||||
Keys(filter string) ([]string, error)
|
||||
}
|
||||
|
||||
// ErrNoPipeline is a read of a pipeline key the state pipelines does not hold.
|
||||
var ErrNoPipeline = errors.New("no such pipeline")
|
||||
|
||||
// MoverStore is the state pipelines as the runner reads and writes it (§27, H-A-187): Get composes a pipeline from its
|
||||
// record and its mover keys, Update writes the record and each mover key that changed, each by compare-and-set on the
|
||||
// revision read, and nothing when any of them was written since. One MoverStore serves a process: a write and a read of one
|
||||
// pipeline take its lock, so the process never reads a record and mover keys of two writes.
|
||||
type MoverStore struct {
|
||||
Raw RawState
|
||||
Logf func(string, ...any)
|
||||
|
||||
mu sync.Mutex
|
||||
locks map[string]*sync.Mutex
|
||||
read map[string]readAt
|
||||
inline map[string]bool
|
||||
}
|
||||
|
||||
// readAt is what the last read of a pipeline found: its record's revision and each mover key's revision.
|
||||
type readAt struct {
|
||||
rev uint64
|
||||
movers map[string]uint64
|
||||
}
|
||||
|
||||
func (s *MoverStore) lock(key string) func() {
|
||||
s.mu.Lock()
|
||||
if s.locks == nil {
|
||||
s.locks, s.read, s.inline = map[string]*sync.Mutex{}, map[string]readAt{}, map[string]bool{}
|
||||
}
|
||||
l, ok := s.locks[key]
|
||||
if !ok {
|
||||
l = &sync.Mutex{}
|
||||
s.locks[key] = l
|
||||
}
|
||||
s.mu.Unlock()
|
||||
l.Lock()
|
||||
return l.Unlock
|
||||
}
|
||||
|
||||
func (s *MoverStore) logf(f string, a ...any) {
|
||||
if s.Logf != nil {
|
||||
s.Logf(f, a...)
|
||||
}
|
||||
}
|
||||
|
||||
// moverKeys are the mover keys of pipeline key with their values and revisions.
|
||||
func (s *MoverStore) moverKeys(key string) (map[string][]byte, map[string]uint64, error) {
|
||||
ks, err := s.Raw.Keys(stepsPrefix(key) + "*")
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("the mover keys of %s are not listed: %w", key, err)
|
||||
}
|
||||
vals, revs := map[string][]byte{}, map[string]uint64{}
|
||||
for _, k := range ks {
|
||||
v, rev, err := s.Raw.Get(k)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("%s is not read: %w", k, err)
|
||||
}
|
||||
if v != nil {
|
||||
vals[k], revs[k] = v, rev
|
||||
}
|
||||
}
|
||||
return vals, revs, nil
|
||||
}
|
||||
|
||||
// composeSteps sets p's steps from its mover keys, in the order of p's movers, a key no mover names last in key order, and
|
||||
// its unlogged step-log entries: the record's own (an entry naming no mover, written before H-A-190) and each mover's, in
|
||||
// the order they ended (H-A-189).
|
||||
func composeSteps(p *seat.Pipeline, key string, vals map[string][]byte) error {
|
||||
var names []string
|
||||
for k := range vals {
|
||||
names = append(names, k)
|
||||
}
|
||||
sort.Strings(names)
|
||||
order := make([]string, 0, len(names))
|
||||
for _, m := range p.Movers {
|
||||
if k := StepsKey(key, m.Module); vals[k] != nil && !slices.Contains(order, k) {
|
||||
order = append(order, k)
|
||||
}
|
||||
}
|
||||
for _, k := range names {
|
||||
if !slices.Contains(order, k) {
|
||||
order = append(order, k)
|
||||
}
|
||||
}
|
||||
steps := []seat.StepResult{}
|
||||
unlogged := append([]seat.LogEntry(nil), p.Unlogged...)
|
||||
for _, k := range order {
|
||||
v, err := seat.DecodePipelinesByStepsPipelineMoverValue(vals[k])
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s holds no mover's steps: %w", k, err)
|
||||
}
|
||||
steps = append(steps, v.Steps...)
|
||||
unlogged = append(unlogged, v.Unlogged...)
|
||||
}
|
||||
sort.SliceStable(unlogged, func(i, j int) bool { return unlogged[i].At.Before(unlogged[j].At) })
|
||||
p.Steps, p.Unlogged = steps, unlogged
|
||||
return nil
|
||||
}
|
||||
|
||||
// split is p's record, its own steps and unlogged entries empty, and each mover's steps and unlogged step-log entries by
|
||||
// its mover key (H-A-187, H-A-189). An entry naming no mover (one written before H-A-190) goes under the mover whose step
|
||||
// it alone matches, else it stays in the record.
|
||||
func split(key string, p seat.Pipeline) ([]byte, map[string][]byte, error) {
|
||||
by := map[string]*seat.PipelinesByStepsPipelineMoverValue{}
|
||||
at := func(module string) *seat.PipelinesByStepsPipelineMoverValue {
|
||||
k := StepsKey(key, module)
|
||||
if by[k] == nil {
|
||||
by[k] = &seat.PipelinesByStepsPipelineMoverValue{Steps: []seat.StepResult{}}
|
||||
}
|
||||
return by[k]
|
||||
}
|
||||
for _, st := range p.Steps {
|
||||
if st.Module == "" {
|
||||
return nil, nil, fmt.Errorf("a step %s of %s names no mover, so it has no mover key", st.Step, key)
|
||||
}
|
||||
v := at(st.Module)
|
||||
v.Steps = append(v.Steps, st)
|
||||
}
|
||||
var kept []seat.LogEntry
|
||||
for _, e := range p.Unlogged {
|
||||
module := e.Module
|
||||
if module == "" {
|
||||
module = moverOfEntry(p, e)
|
||||
}
|
||||
if module == "" {
|
||||
kept = append(kept, e)
|
||||
continue
|
||||
}
|
||||
v := at(module)
|
||||
v.Unlogged = append(v.Unlogged, e)
|
||||
}
|
||||
movers := map[string][]byte{}
|
||||
for k, v := range by {
|
||||
raw, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
movers[k] = raw
|
||||
}
|
||||
p.Steps, p.Unlogged = []seat.StepResult{}, kept
|
||||
record, err := json.Marshal(p)
|
||||
return record, movers, err
|
||||
}
|
||||
|
||||
// moverOfEntry is the mover of the one step an entry naming no mover matches by step and artifact, empty when none or
|
||||
// more than one does.
|
||||
func moverOfEntry(p seat.Pipeline, e seat.LogEntry) string {
|
||||
found := ""
|
||||
for _, st := range p.Steps {
|
||||
if st.Step != e.Step || st.Artifact != e.Artifact {
|
||||
continue
|
||||
}
|
||||
if found != "" && found != st.Module {
|
||||
return ""
|
||||
}
|
||||
found = st.Module
|
||||
}
|
||||
return found
|
||||
}
|
||||
|
||||
// Get is pipeline key with every mover's steps and the record's revision. A record that still carries its steps inline (one
|
||||
// written before H-A-187) is read whole, said once in the journal, and split at its next write.
|
||||
func (s *MoverStore) Get(key string) (seat.Pipeline, uint64, error) {
|
||||
defer s.lock(key)()
|
||||
raw, rev, err := s.Raw.Get(key)
|
||||
if err != nil {
|
||||
return seat.Pipeline{}, 0, err
|
||||
}
|
||||
if raw == nil {
|
||||
return seat.Pipeline{}, 0, fmt.Errorf("%w: %s", ErrNoPipeline, key)
|
||||
}
|
||||
var p seat.Pipeline
|
||||
if err := json.Unmarshal(raw, &p); err != nil {
|
||||
return seat.Pipeline{}, 0, fmt.Errorf("the pipeline %s is no pipeline: %w", key, err)
|
||||
}
|
||||
vals, revs, err := s.moverKeys(key)
|
||||
if err != nil {
|
||||
return seat.Pipeline{}, 0, err
|
||||
}
|
||||
s.read[key] = readAt{rev: rev, movers: revs}
|
||||
if len(p.Steps) > 0 {
|
||||
if !s.inline[key] {
|
||||
s.inline[key] = true
|
||||
s.logf("the pipeline %s carries its %d steps inline in its record: they are read whole and split into one key per mover at its next write (H-A-187)", key, len(p.Steps))
|
||||
}
|
||||
return p, rev, nil
|
||||
}
|
||||
if p.Steps == nil {
|
||||
p.Steps = []seat.StepResult{}
|
||||
}
|
||||
return p, rev, composeSteps(&p, key, vals)
|
||||
}
|
||||
|
||||
// Update writes p as pipeline key's record and its mover keys, each by compare-and-set: nothing is written, ErrChanged,
|
||||
// when the record or a mover key was written since the read revision gave. A record carrying its steps inline has its
|
||||
// mover keys written before the record, so its steps are never lost between the two.
|
||||
func (s *MoverStore) Update(key string, p seat.Pipeline, revision uint64) (uint64, error) {
|
||||
defer s.lock(key)()
|
||||
record, want, err := split(key, p)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
raw, cur, err := s.Raw.Get(key)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if raw == nil {
|
||||
return 0, fmt.Errorf("%w: %s", ErrNoPipeline, key)
|
||||
}
|
||||
if cur != revision {
|
||||
return 0, ErrChanged
|
||||
}
|
||||
vals, revs, err := s.moverKeys(key)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
// A mover key written since the read is another write between: the read named its revision.
|
||||
if seen, ok := s.read[key]; ok && seen.rev == revision {
|
||||
for k, r := range revs {
|
||||
if seen.movers[k] != r {
|
||||
return 0, ErrChanged
|
||||
}
|
||||
}
|
||||
for k := range seen.movers {
|
||||
if _, ok := revs[k]; !ok {
|
||||
return 0, ErrChanged
|
||||
}
|
||||
}
|
||||
}
|
||||
var was seat.Pipeline
|
||||
inline := json.Unmarshal(raw, &was) == nil && len(was.Steps) > 0
|
||||
var after map[string]uint64
|
||||
if inline {
|
||||
if after, err = s.writeMovers(key, want, vals, revs); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
rev, err := s.Raw.Update(key, record, revision)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if !inline {
|
||||
if after, err = s.writeMovers(key, want, vals, revs); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
} else {
|
||||
s.logf("the pipeline %s's steps are split into %d mover keys and its record no longer carries them (H-A-187)", key, len(want))
|
||||
delete(s.inline, key)
|
||||
}
|
||||
s.read[key] = readAt{rev: rev, movers: after}
|
||||
return rev, nil
|
||||
}
|
||||
|
||||
// writeMovers writes each mover key whose steps changed and deletes each that p no longer names, by compare-and-set on the
|
||||
// revisions read, and answers every mover key's revision after it.
|
||||
func (s *MoverStore) writeMovers(key string, want, vals map[string][]byte, revs map[string]uint64) (map[string]uint64, error) {
|
||||
after := map[string]uint64{}
|
||||
for _, k := range slices.Sorted(maps.Keys(want)) {
|
||||
v := want[k]
|
||||
old, ok := vals[k]
|
||||
var rev uint64
|
||||
var err error
|
||||
switch {
|
||||
case ok && string(old) == string(v):
|
||||
after[k] = revs[k]
|
||||
continue
|
||||
case ok:
|
||||
rev, err = s.Raw.Update(k, v, revs[k])
|
||||
default:
|
||||
rev, err = s.Raw.Create(k, v)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the steps of %s are not written: %w", k, err)
|
||||
}
|
||||
after[k] = rev
|
||||
}
|
||||
for k := range vals {
|
||||
if _, ok := want[k]; !ok {
|
||||
if err := s.Raw.Delete(k); err != nil {
|
||||
return nil, fmt.Errorf("the steps of %s are not deleted: %w", k, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return after, nil
|
||||
}
|
||||
|
||||
// Create writes a new pipeline, its record by compare-and-set on its absence (ErrExists: the second start writes nothing,
|
||||
// §54), then its mover keys; a mover key an earlier pipeline of the key left is written over, said in the journal.
|
||||
func (s *MoverStore) Create(key string, value []byte) error {
|
||||
var p seat.Pipeline
|
||||
if err := json.Unmarshal(value, &p); err != nil {
|
||||
return fmt.Errorf("the pipeline %s is no pipeline: %w", key, err)
|
||||
}
|
||||
defer s.lock(key)()
|
||||
record, want, err := split(key, p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rev, err := s.Raw.Create(key, record)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
vals, revs, err := s.moverKeys(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for k := range vals {
|
||||
s.logf("%s was left by an earlier pipeline of the key: it is written over (H-A-187)", k)
|
||||
}
|
||||
after, err := s.writeMovers(key, want, vals, revs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.read[key] = readAt{rev: rev, movers: after}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete removes pipeline key's mover keys, then its record (§330): a delete stopped between leaves the record, which the
|
||||
// next ageing deletes again.
|
||||
func (s *MoverStore) Delete(key string) error {
|
||||
unlock := s.lock(key)
|
||||
vals, _, err := s.moverKeys(key)
|
||||
if err == nil {
|
||||
for k := range vals {
|
||||
if err = s.Raw.Delete(k); err != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if err == nil {
|
||||
err = s.Raw.Delete(key)
|
||||
}
|
||||
delete(s.read, key)
|
||||
delete(s.inline, key)
|
||||
unlock()
|
||||
return err
|
||||
}
|
||||
|
||||
// Keys are the keys of the state pipelines matching filter, `*` one key part.
|
||||
func (s *MoverStore) Keys(filter string) ([]string, error) { return s.Raw.Keys(filter) }
|
||||
@@ -0,0 +1,254 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
)
|
||||
|
||||
// memRaw is the state pipelines as bytes, one revision sequence for the bucket as a key-value's stream has.
|
||||
type memRaw struct {
|
||||
vals map[string][]byte
|
||||
revs map[string]uint64
|
||||
seq uint64
|
||||
}
|
||||
|
||||
func newMemRaw() *memRaw { return &memRaw{vals: map[string][]byte{}, revs: map[string]uint64{}} }
|
||||
|
||||
func (m *memRaw) Get(key string) ([]byte, uint64, error) {
|
||||
v, ok := m.vals[key]
|
||||
if !ok {
|
||||
return nil, 0, nil
|
||||
}
|
||||
return append([]byte(nil), v...), m.revs[key], nil
|
||||
}
|
||||
|
||||
func (m *memRaw) put(key string, value []byte) uint64 {
|
||||
m.seq++
|
||||
m.vals[key], m.revs[key] = append([]byte(nil), value...), m.seq
|
||||
return m.seq
|
||||
}
|
||||
|
||||
func (m *memRaw) Create(key string, value []byte) (uint64, error) {
|
||||
if _, ok := m.vals[key]; ok {
|
||||
return 0, ErrExists
|
||||
}
|
||||
return m.put(key, value), nil
|
||||
}
|
||||
|
||||
func (m *memRaw) Update(key string, value []byte, rev uint64) (uint64, error) {
|
||||
if m.revs[key] != rev {
|
||||
return 0, ErrChanged
|
||||
}
|
||||
return m.put(key, value), nil
|
||||
}
|
||||
|
||||
func (m *memRaw) Delete(key string) error {
|
||||
delete(m.vals, key)
|
||||
delete(m.revs, key)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *memRaw) Keys(filter string) ([]string, error) {
|
||||
var out []string
|
||||
for k := range m.vals {
|
||||
if ok, _ := matchKey(filter, k); ok {
|
||||
out = append(out, k)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// recordSteps is the steps the raw record of key carries itself.
|
||||
func recordSteps(t *testing.T, m *memRaw, key string) int {
|
||||
t.Helper()
|
||||
var p seat.Pipeline
|
||||
if err := json.Unmarshal(m.vals[key], &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return len(p.Steps)
|
||||
}
|
||||
|
||||
// TestMoverKeys holds the state pipelines' shape to H-A-187: a pipeline's record keeps its movers and outcome, each mover's
|
||||
// steps their own key written by compare-and-set as the record is, a read composing them, a record carrying its steps inline
|
||||
// read whole and split at its next write, one row per case.
|
||||
func TestMoverKeys(t *testing.T) {
|
||||
const key = "main.7"
|
||||
two := func() seat.Pipeline {
|
||||
return seat.Pipeline{V: 1, Pipeline: key, Branch: "main", Source: 7, Outcome: seat.PipelineOutcomeRunning,
|
||||
Movers: []seat.Mover{{Module: "beta"}, {Module: "alpha"}},
|
||||
Steps: []seat.StepResult{{Module: "alpha", Step: seat.StepResultStepBuild, Outcome: seat.StepResultOutcomeRunning},
|
||||
{Module: "beta", Step: seat.StepResultStepBuild, Outcome: seat.StepResultOutcomeRunning},
|
||||
{Module: "beta", Step: seat.StepResultStepApproval, Outcome: seat.StepResultOutcomeNotAsked}}}
|
||||
}
|
||||
created := func(t *testing.T) (*memRaw, *MoverStore, *[]string) {
|
||||
m, logs := newMemRaw(), &[]string{}
|
||||
s := &MoverStore{Raw: m, Logf: func(f string, a ...any) { *logs = append(*logs, fmt.Sprintf(f, a...)) }}
|
||||
raw, _ := json.Marshal(two())
|
||||
if err := s.Create(key, raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return m, s, logs
|
||||
}
|
||||
steps := func(p seat.Pipeline) string {
|
||||
var out []string
|
||||
for _, st := range p.Steps {
|
||||
out = append(out, st.Module+" "+st.Step+" "+st.Outcome)
|
||||
}
|
||||
return strings.Join(out, "; ")
|
||||
}
|
||||
rows := []struct {
|
||||
name string
|
||||
run func(t *testing.T) string
|
||||
want string
|
||||
}{
|
||||
{"a write keeps the record's steps empty and each mover's under its own key; a read composes them in the movers' order", func(t *testing.T) string {
|
||||
m, s, _ := created(t)
|
||||
p, _, err := s.Get(key)
|
||||
return fmt.Sprintf("record %d, alpha %t, beta %t: %s %v", recordSteps(t, m, key), m.vals["steps.main.7.alpha"] != nil,
|
||||
m.vals["steps.main.7.beta"] != nil, steps(p), err)
|
||||
}, "record 0, alpha true, beta true: beta build running; beta approval not asked; alpha build running <nil>"},
|
||||
{"an update writes the mover whose steps changed and leaves the other's key as it was", func(t *testing.T) string {
|
||||
m, s, _ := created(t)
|
||||
p, rev, _ := s.Get(key)
|
||||
beta := m.revs["steps.main.7.beta"]
|
||||
p.Steps[2].Outcome = seat.StepResultOutcomePassed
|
||||
_, err := s.Update(key, p, rev)
|
||||
q, _, _ := s.Get(key)
|
||||
return fmt.Sprintf("beta kept %t: %s %v", m.revs["steps.main.7.beta"] == beta, steps(q), err)
|
||||
}, "beta kept true: beta build running; beta approval not asked; alpha build passed <nil>"},
|
||||
{"a mover key written since the read refuses the update, nothing written", func(t *testing.T) string {
|
||||
m, s, _ := created(t)
|
||||
p, rev, _ := s.Get(key)
|
||||
m.put("steps.main.7.alpha", m.vals["steps.main.7.alpha"])
|
||||
p.Outcome = seat.PipelineOutcomePassed
|
||||
_, err := s.Update(key, p, rev)
|
||||
return fmt.Sprintf("%t, record at %t", errors.Is(err, ErrChanged), m.revs[key] == rev)
|
||||
}, "true, record at true"},
|
||||
{"a record written since the read refuses the update", func(t *testing.T) string {
|
||||
_, s, _ := created(t)
|
||||
p, rev, _ := s.Get(key)
|
||||
if _, err := s.Update(key, p, rev); err != nil {
|
||||
return err.Error()
|
||||
}
|
||||
_, err := s.Update(key, p, rev)
|
||||
return fmt.Sprint(errors.Is(err, ErrChanged))
|
||||
}, "true"},
|
||||
{"a record carrying its steps inline is read whole, said once, and split at its next write", func(t *testing.T) string {
|
||||
m, logs := newMemRaw(), &[]string{}
|
||||
s := &MoverStore{Raw: m, Logf: func(f string, a ...any) { *logs = append(*logs, fmt.Sprintf(f, a...)) }}
|
||||
raw, _ := json.Marshal(two())
|
||||
m.put(key, raw)
|
||||
p, rev, _ := s.Get(key)
|
||||
_, _, _ = s.Get(key)
|
||||
read := fmt.Sprintf("read %d steps, %d lines", len(p.Steps), len(*logs))
|
||||
_, err := s.Update(key, p, rev)
|
||||
q, _, _ := s.Get(key)
|
||||
return fmt.Sprintf("%s; written: record %d, %s, split said %t %v", read, recordSteps(t, m, key), steps(q),
|
||||
strings.Contains(strings.Join(*logs, "\n"), "split into 2 mover keys"), err)
|
||||
}, "read 3 steps, 1 lines; written: record 0, beta build running; beta approval not asked; alpha build running, split said true <nil>"},
|
||||
{"a second create writes nothing", func(t *testing.T) string {
|
||||
m, s, _ := created(t)
|
||||
p := two()
|
||||
p.Steps[0].Outcome = seat.StepResultOutcomeFailed
|
||||
raw, _ := json.Marshal(p)
|
||||
err := s.Create(key, raw)
|
||||
q, _, _ := s.Get(key)
|
||||
return fmt.Sprintf("%t %d: %s", errors.Is(err, ErrExists), len(m.vals), steps(q))
|
||||
}, "true 3: beta build running; beta approval not asked; alpha build running"},
|
||||
{"a delete removes the mover keys and the record", func(t *testing.T) string {
|
||||
m, s, _ := created(t)
|
||||
err := s.Delete(key)
|
||||
return fmt.Sprintf("%d %v", len(m.vals), err)
|
||||
}, "0 <nil>"},
|
||||
{"a record of 129 movers never grows with their steps (main.26)", func(t *testing.T) string {
|
||||
m, s, _ := created(t)
|
||||
p, rev, _ := s.Get(key)
|
||||
p.Movers, p.Steps = nil, nil
|
||||
for i := range 129 {
|
||||
mod := fmt.Sprintf("module-%03d", i)
|
||||
p.Movers = append(p.Movers, seat.Mover{Module: mod, Commit: strings.Repeat("c", 40), Fingerprint: "fp-" + mod})
|
||||
for _, step := range []string{seat.StepResultStepBuild, seat.StepResultStepLint, seat.StepResultStepUnit,
|
||||
seat.StepResultStepIntegration, seat.StepResultStepValidate, seat.StepResultStepComposition,
|
||||
seat.StepResultStepReview, seat.StepResultStepApproval} {
|
||||
p.Steps = append(p.Steps, seat.StepResult{Module: mod, Step: step, Artifact: mod + ":program",
|
||||
Outcome: seat.StepResultOutcomeFailed, Why: strings.Repeat("why ", 250)})
|
||||
}
|
||||
}
|
||||
if _, err := s.Update(key, p, rev); err != nil {
|
||||
return err.Error()
|
||||
}
|
||||
largest := 0
|
||||
for k, v := range m.vals {
|
||||
if IsStepsKey(k) && len(v) > largest {
|
||||
largest = len(v)
|
||||
}
|
||||
}
|
||||
q, _, _ := s.Get(key)
|
||||
return fmt.Sprintf("record under 64 KiB %t, a mover key under 16 KiB %t, %d steps read", len(m.vals[key]) < 64<<10,
|
||||
largest < 16<<10, len(q.Steps))
|
||||
}, "record under 64 KiB true, a mover key under 16 KiB true, 1032 steps read"},
|
||||
{"a mover's unlogged entries live under its mover key, each filed by the mover it names (H-A-189, H-A-190)", func(t *testing.T) string {
|
||||
m, s, _ := created(t)
|
||||
p, rev, _ := s.Get(key)
|
||||
at := time.Date(2026, 11, 20, 20, 0, 0, 0, time.UTC)
|
||||
// Two movers' approvals end alike at one moment; each entry names its mover.
|
||||
p.Unlogged = []seat.LogEntry{{Pipeline: key, Module: "beta", Step: seat.StepResultStepApproval, What: seat.LogEntryWhatEnded, At: at},
|
||||
{Pipeline: key, Module: "alpha", Step: seat.StepResultStepApproval, What: seat.LogEntryWhatEnded, At: at.Add(time.Second)}}
|
||||
if _, err := s.Update(key, p, rev); err != nil {
|
||||
return err.Error()
|
||||
}
|
||||
var rec seat.Pipeline
|
||||
_ = json.Unmarshal(m.vals[key], &rec)
|
||||
beta, _ := seat.DecodePipelinesByStepsPipelineMoverValue(m.vals["steps.main.7.beta"])
|
||||
q, _, _ := s.Get(key)
|
||||
var got []string
|
||||
for _, e := range q.Unlogged {
|
||||
got = append(got, e.Module)
|
||||
}
|
||||
return fmt.Sprintf("record %d unlogged, beta's key %d (%s), read %s", len(rec.Unlogged), len(beta.Unlogged),
|
||||
beta.Unlogged[0].Module, strings.Join(got, " "))
|
||||
}, "record 0 unlogged, beta's key 1 (beta), read beta alpha"},
|
||||
{"a 129-mover pipeline with 1,000 unlogged entries keeps its record small (H-A-189)", func(t *testing.T) string {
|
||||
m, s, _ := created(t)
|
||||
p, rev, _ := s.Get(key)
|
||||
p.Movers, p.Steps, p.Unlogged = nil, nil, nil
|
||||
at := time.Date(2026, 11, 20, 20, 0, 0, 0, time.UTC)
|
||||
for i := range 129 {
|
||||
mod := fmt.Sprintf("module-%03d", i)
|
||||
p.Movers = append(p.Movers, seat.Mover{Module: mod, Commit: strings.Repeat("c", 40), Fingerprint: "fp-" + mod})
|
||||
p.Steps = append(p.Steps, seat.StepResult{Module: mod, Step: seat.StepResultStepBuild, Outcome: seat.StepResultOutcomeNotAsked})
|
||||
}
|
||||
for i := range 1000 {
|
||||
mod := fmt.Sprintf("module-%03d", i%129)
|
||||
p.Unlogged = append(p.Unlogged, seat.LogEntry{Pipeline: key, Module: mod, Step: seat.StepResultStepBuild, Artifact: mod + ":program",
|
||||
What: seat.LogEntryWhatEnded, Outcome: seat.StepResultOutcomeNotAsked, Output: strings.Repeat("o", 200), At: at.Add(time.Duration(i) * time.Millisecond)})
|
||||
}
|
||||
if _, err := s.Update(key, p, rev); err != nil {
|
||||
return err.Error()
|
||||
}
|
||||
q, _, _ := s.Get(key)
|
||||
return fmt.Sprintf("record under 64 KiB %t, %d unlogged read", len(m.vals[key]) < 64<<10, len(q.Unlogged))
|
||||
}, "record under 64 KiB true, 1000 unlogged read"},
|
||||
{"mover keys are told from records of every key form", func(t *testing.T) string {
|
||||
var out []string
|
||||
for _, k := range []string{"steps.main.7.alpha", "steps.main.run.ab12.alpha", "steps.main.7.published.p.v.alpha",
|
||||
"main.7", "steps.7", "steps.run.ab12", "steps.7.published.p.v"} {
|
||||
out = append(out, fmt.Sprint(IsStepsKey(k)))
|
||||
}
|
||||
return strings.Join(out, " ")
|
||||
}, "true true true false false false false"},
|
||||
}
|
||||
for _, r := range rows {
|
||||
t.Run(r.name, func(t *testing.T) {
|
||||
if got := r.run(t); got != r.want {
|
||||
t.Errorf("got %s\nwant %s", got, r.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -39,7 +39,8 @@ func (r Reads) fills() ([]seat.Fill, error) {
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// Show is one pipeline whole, with its steps and the names of their outputs in step-outputs (§83).
|
||||
// Show is one pipeline whole, with its steps and the names of their outputs in step-outputs (§83), or, naming a mover, the
|
||||
// pipeline's record with that mover's steps alone (H-A-187).
|
||||
func (r Reads) Show(in seat.ShowInput) (seat.ShowAnswer, error) {
|
||||
p, ok, err := r.pipeline(in.Pipeline)
|
||||
if err != nil {
|
||||
@@ -52,6 +53,13 @@ func (r Reads) Show(in seat.ShowInput) (seat.ShowAnswer, error) {
|
||||
if err != nil {
|
||||
return seat.ShowAnswer{}, err
|
||||
}
|
||||
// Named, a mover is answered by its steps alone (H-A-187): one mover's of a pipeline of a hundred movers.
|
||||
if m := strings.TrimSpace(in.Mover); m != "" {
|
||||
if !Moves(p, m) {
|
||||
return seat.ShowAnswer{}, fmt.Errorf("the pipeline %s does not move %s", in.Pipeline, m)
|
||||
}
|
||||
p.Steps = stepsOf(p, m)
|
||||
}
|
||||
a := seat.ShowAnswer{Pipeline: p, Fill: fills}
|
||||
for _, s := range p.Steps {
|
||||
if s.Output != "" {
|
||||
@@ -62,7 +70,7 @@ func (r Reads) Show(in seat.ShowInput) (seat.ShowAnswer, error) {
|
||||
}
|
||||
|
||||
// List is the pipelines and runs of a module, a branch, a commit or a run, newest started first, at most Limit when it
|
||||
// names one (§83).
|
||||
// names one (§83): each one's record with its movers and steps counted, its steps empty (H-A-191).
|
||||
func (r Reads) List(in seat.ListInput) (seat.ListAnswer, error) {
|
||||
all, err := r.pipelines()
|
||||
if err != nil {
|
||||
@@ -72,6 +80,9 @@ func (r Reads) List(in seat.ListInput) (seat.ListAnswer, error) {
|
||||
for _, p := range all {
|
||||
if (in.Module == "" || Moves(p, in.Module)) && (in.Branch == "" || p.Branch == in.Branch) &&
|
||||
(in.Commit == "" || movesCommit(p, in.Commit)) && (in.Run == "" || p.Run == in.Run) {
|
||||
// Each pipeline's record with its movers and steps counted, never its steps: show gives those (H-A-191).
|
||||
p.MoverCount, p.StepCount = int64(len(p.Movers)), int64(len(p.Steps))
|
||||
p.Steps, p.Unlogged = []seat.StepResult{}, nil
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
@@ -178,6 +189,16 @@ func (r Reads) pipeline(key string) (seat.Pipeline, bool, error) {
|
||||
if key == "" {
|
||||
return seat.Pipeline{}, false, fmt.Errorf("name the pipeline's key, <branch>.<revision>.<module>")
|
||||
}
|
||||
all, err := r.Pipelines.Keys()
|
||||
if err != nil {
|
||||
return seat.Pipeline{}, false, fmt.Errorf("%s cannot be read: %w", seat.StatePipelines, err)
|
||||
}
|
||||
return r.pipelineOf(key, all)
|
||||
}
|
||||
|
||||
// pipelineOf is pipeline key, its steps composed from the mover keys among all that it names (H-A-187); a record that
|
||||
// still carries its steps inline is read whole.
|
||||
func (r Reads) pipelineOf(key string, all []string) (seat.Pipeline, bool, error) {
|
||||
raw, err := r.Pipelines.Value(key)
|
||||
if err != nil {
|
||||
return seat.Pipeline{}, false, fmt.Errorf("%s %s cannot be read: %w", seat.StatePipelines, key, err)
|
||||
@@ -192,7 +213,24 @@ func (r Reads) pipeline(key string) (seat.Pipeline, bool, error) {
|
||||
if p.Pipeline == "" {
|
||||
p.Pipeline = key
|
||||
}
|
||||
return p, true, nil
|
||||
if len(p.Steps) > 0 {
|
||||
return p, true, nil
|
||||
}
|
||||
prefix := stepsPrefix(key)
|
||||
vals := map[string][]byte{}
|
||||
for _, k := range all {
|
||||
if !strings.HasPrefix(k, prefix) || strings.Contains(k[len(prefix):], ".") {
|
||||
continue
|
||||
}
|
||||
v, err := r.Pipelines.Value(k)
|
||||
if err != nil {
|
||||
return seat.Pipeline{}, false, fmt.Errorf("%s %s cannot be read: %w", seat.StatePipelines, k, err)
|
||||
}
|
||||
if v != nil {
|
||||
vals[k] = v
|
||||
}
|
||||
}
|
||||
return p, true, composeSteps(&p, key, vals)
|
||||
}
|
||||
|
||||
func (r Reads) pipelines() ([]seat.Pipeline, error) {
|
||||
@@ -203,7 +241,10 @@ func (r Reads) pipelines() ([]seat.Pipeline, error) {
|
||||
sort.Strings(keys)
|
||||
var out []seat.Pipeline
|
||||
for _, k := range keys {
|
||||
p, ok, err := r.pipeline(k)
|
||||
if IsStepsKey(k) {
|
||||
continue
|
||||
}
|
||||
p, ok, err := r.pipelineOf(k, keys)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package pipeline
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -49,6 +50,15 @@ func TestReads(t *testing.T) {
|
||||
Outcome: seat.PipelineOutcomePassed, Started: &t2, Movers: []seat.Mover{{Module: "alpha", Commit: "b3"}}})
|
||||
put(t, pipes, "main.7.gamma", seat.Pipeline{Pipeline: "main.7.gamma", Branch: "main",
|
||||
Outcome: seat.PipelineOutcomeRunning, Started: &t1, Movers: []seat.Mover{{Module: "gamma", Commit: "c7"}}})
|
||||
// A pipeline kept as H-A-187 keeps it: its record's steps empty, each mover's under steps.<pipeline>.<mover>.
|
||||
put(t, pipes, "next.8", seat.Pipeline{Pipeline: "next.8", Branch: "next", Source: 8, Outcome: seat.PipelineOutcomeRunning,
|
||||
Started: &t0, Steps: []seat.StepResult{}, Movers: []seat.Mover{{Module: "delta", Commit: "n8"}, {Module: "epsilon", Commit: "n8"}}})
|
||||
put(t, pipes, "steps.next.8.epsilon", seat.PipelinesByStepsPipelineMoverValue{Steps: []seat.StepResult{{Module: "epsilon",
|
||||
Step: seat.StepResultStepBuild, Status: "required", Outcome: seat.StepResultOutcomePassed, Output: "next.8.build.epsilon:program"}}})
|
||||
put(t, pipes, "steps.next.8.delta", seat.PipelinesByStepsPipelineMoverValue{Steps: []seat.StepResult{{Module: "delta",
|
||||
Step: seat.StepResultStepBuild, Status: "required", Outcome: seat.StepResultOutcomePassed}, {Module: "delta",
|
||||
Step: seat.StepResultStepApproval, Status: "required", Outcome: seat.StepResultOutcomeWaiting, On: seat.WaitOnPerson, Started: &t2}},
|
||||
Unlogged: []seat.LogEntry{{Pipeline: "next.8", Module: "delta", Step: seat.StepResultStepBuild, What: seat.LogEntryWhatEnded, At: t1}}})
|
||||
put(t, kept, "unit.sha256-aa", seat.KeptResult{Level: "unit", Digest: "sha256-aa", Package: "alpha/x"})
|
||||
put(t, kept, "integration.sha256-bb", seat.KeptResult{Level: "integration", Digest: "sha256-bb", Package: "alpha/y"})
|
||||
r := Reads{Pipelines: pipes, Kept: kept}
|
||||
@@ -73,6 +83,35 @@ func TestReads(t *testing.T) {
|
||||
_, err := r.Show(seat.ShowInput{Pipeline: "main.9.alpha"})
|
||||
return "", err
|
||||
}, `error: no pipeline "main.9.alpha" in pipelines`},
|
||||
{"show composes a pipeline's steps from its mover keys, in its movers' order (H-A-187)", func() (string, error) {
|
||||
a, err := r.Show(seat.ShowInput{Pipeline: "next.8"})
|
||||
var s []string
|
||||
for _, st := range a.Pipeline.Steps {
|
||||
s = append(s, st.Module+" "+st.Step)
|
||||
}
|
||||
return strings.Join(s, ", ") + " | " + strings.Join(a.Outputs, ","), err
|
||||
}, "delta build, delta approval, epsilon build | next.8.build.epsilon:program"},
|
||||
{"show naming a mover answers its steps alone", func() (string, error) {
|
||||
a, err := r.Show(seat.ShowInput{Pipeline: "next.8", Mover: "epsilon"})
|
||||
var s []string
|
||||
for _, st := range a.Pipeline.Steps {
|
||||
s = append(s, st.Module+" "+st.Step)
|
||||
}
|
||||
return fmt.Sprintf("%s, %d movers", strings.Join(s, ", "), len(a.Pipeline.Movers)), err
|
||||
}, "epsilon build, 2 movers"},
|
||||
{"show refuses a mover the pipeline does not move", func() (string, error) {
|
||||
_, err := r.Show(seat.ShowInput{Pipeline: "next.8", Mover: "alpha"})
|
||||
return "", err
|
||||
}, "error: the pipeline next.8 does not move alpha"},
|
||||
{"list names a pipeline, never a mover key, its record with its movers and steps counted and no step (H-A-191)", func() (string, error) {
|
||||
a, err := r.List(seat.ListInput{Branch: "next"})
|
||||
var s []string
|
||||
for _, p := range a.Pipelines {
|
||||
s = append(s, fmt.Sprintf("%s %d movers %d steps, %d steps and %d unlogged answered", p.Pipeline, p.MoverCount, p.StepCount,
|
||||
len(p.Steps), len(p.Unlogged)))
|
||||
}
|
||||
return strings.Join(s, "; "), err
|
||||
}, "next.8 2 movers 3 steps, 0 steps and 0 unlogged answered"},
|
||||
{"list by module, newest first", func() (string, error) {
|
||||
a, err := r.List(seat.ListInput{Module: "alpha"})
|
||||
return names(a.Pipelines), err
|
||||
@@ -92,7 +131,7 @@ func TestReads(t *testing.T) {
|
||||
s = append(s, w.Pipeline+" "+w.Step+" "+w.On+" "+w.Since.Format("15:04"))
|
||||
}
|
||||
return strings.Join(s, ";"), err
|
||||
}, "main.7.alpha approval person 10:01"},
|
||||
}, "main.7.alpha approval person 10:01;next.8 approval person 10:02"},
|
||||
{"why-carried names where a carried step came from", func() (string, error) {
|
||||
a, err := r.WhyCarried(seat.WhyCarriedInput{Pipeline: "main.7.alpha", Step: "unit", Artifact: "alpha:program"})
|
||||
return a.CarriedFrom, err
|
||||
|
||||
@@ -441,7 +441,7 @@ func (h *Hearer) said(p seat.Pipeline, asked []int, ended bool) error {
|
||||
|
||||
// endEntry is the step log's entry of a step's end, with the keys of the kept results it wrote or carried.
|
||||
func endEntry(key string, s seat.StepResult, at time.Time, kept ...string) seat.LogEntry {
|
||||
return seat.LogEntry{Pipeline: key, Step: s.Step, Artifact: s.Artifact, What: seat.LogEntryWhatEnded, Outcome: s.Outcome,
|
||||
return seat.LogEntry{Pipeline: key, Module: s.Module, Step: s.Step, Artifact: s.Artifact, What: seat.LogEntryWhatEnded, Outcome: s.Outcome,
|
||||
Output: s.Output, Kept: kept, At: at}
|
||||
}
|
||||
|
||||
|
||||
@@ -7,9 +7,10 @@ import (
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
)
|
||||
|
||||
// sameEntry says whether two step-log entries are one: the same pipeline, step, artifact, end or start and time.
|
||||
// sameEntry says whether two step-log entries are one: the same pipeline, mover (H-A-190), step, artifact, end or start and
|
||||
// time.
|
||||
func sameEntry(a, b seat.LogEntry) bool {
|
||||
return a.Pipeline == b.Pipeline && a.Step == b.Step && a.Artifact == b.Artifact && a.What == b.What && a.At.Equal(b.At)
|
||||
return a.Pipeline == b.Pipeline && a.Module == b.Module && a.Step == b.Step && a.Artifact == b.Artifact && a.What == b.What && a.At.Equal(b.At)
|
||||
}
|
||||
|
||||
// appendUnlogged appends the unlogged entries of pipeline key to the step log in their order and clears each appended one in a
|
||||
|
||||
@@ -146,9 +146,15 @@ func (r FetchRequest) Check() error {
|
||||
if r.Dir != "" && (outside(r.Dir) || strings.HasPrefix(r.Dir, "/")) {
|
||||
return fmt.Errorf("the module directory %q leaves the repository", r.Dir)
|
||||
}
|
||||
// A path is relative to the module's directory; one that leaves it but stays inside the repository (a sibling the
|
||||
// artifact's go.mod requires, "../<module>/<dir>") is accepted and narrowed from the repository's root, and one that
|
||||
// leaves the repository is refused (design 47 H-A-198, H-A-199).
|
||||
for _, p := range r.Paths {
|
||||
if p == "" || strings.HasPrefix(p, "/") || outside(p) || path.Clean(p) == "." {
|
||||
return fmt.Errorf("the path %q leaves the module's directory or is not a path in it", p)
|
||||
if p == "" || strings.HasPrefix(p, "/") || path.Clean(p) == "." {
|
||||
return fmt.Errorf("the path %q is not a path relative to the module's directory", p)
|
||||
}
|
||||
if outside(path.Join(r.Dir, p)) {
|
||||
return fmt.Errorf("the path %q leaves the repository", p)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -37,6 +37,11 @@ func TestAFetchRequestCarriesTheNamedPathsThroughTheToolCall(t *testing.T) {
|
||||
if dirOnly, err := ParseFetchRequest(map[string]any{"repository": "novox/x", "commit": strings.Repeat("b", 40), "dir": "modules/a"}); err != nil || dirOnly.Dir != "modules/a" || dirOnly.Paths != nil {
|
||||
t.Fatalf("a directory with no paths: %+v, %v", dirOnly, err)
|
||||
}
|
||||
// A sibling of the same repository is a path that leaves the module's directory, not the repository (H-A-199).
|
||||
if sib, err := ParseFetchRequest(map[string]any{"repository": "novox/x", "commit": strings.Repeat("b", 40), "dir": "modules/a",
|
||||
"paths": []any{"pkg", "../b/client"}}); err != nil || len(sib.Paths) != 2 {
|
||||
t.Fatalf("a sibling path inside the repository: %+v, %v", sib, err)
|
||||
}
|
||||
for _, c := range []struct {
|
||||
set map[string]any
|
||||
says string
|
||||
@@ -46,7 +51,8 @@ func TestAFetchRequestCarriesTheNamedPathsThroughTheToolCall(t *testing.T) {
|
||||
{map[string]any{"commit": "abc"}, "is not a full commit"},
|
||||
{map[string]any{"base": "abc"}, "is not a trunk commit"},
|
||||
{map[string]any{"paths": []any{"src"}}, "names no module directory"},
|
||||
{map[string]any{"dir": "modules/a", "paths": []any{"../b"}}, "leaves the module's directory"},
|
||||
{map[string]any{"dir": "modules/a", "paths": []any{"../../../b"}}, "leaves the repository"},
|
||||
{map[string]any{"dir": "modules/a", "paths": []any{"/b"}}, "not a path relative to the module's directory"},
|
||||
{map[string]any{"dir": "../a", "paths": []any{"src"}}, "leaves the repository"},
|
||||
{map[string]any{"dir": "modules/a", "paths": []any{3}}, "paths is a list of strings"},
|
||||
{map[string]any{"packages": map[string]any{"git.novox.be/novox/mesh-sdk/go": "main"}}, "neither a full commit nor a Go semantic version"},
|
||||
|
||||
@@ -100,6 +100,14 @@ func TestAContractsModulesGeneratedFilesAreComparedWithTheGenerator(t *testing.T
|
||||
if got, err := OutOfStep(plain, GeneratedRootGo); err != nil || len(got) != 0 {
|
||||
t.Errorf("a module naming no generated-from compared: %v %v", got, err)
|
||||
}
|
||||
// This repository's own contracts module is in step with the generator under both roots: a hand-written file
|
||||
// under a generated root (go/seats/meshcatalog/manifestdigest.go, pipeline main.43) failed every unit step of its
|
||||
// mover; a seat's hand-written helper lives in its client (design 47 H, H-A-200; ADR 0608 §5).
|
||||
for _, root := range []string{GeneratedRootGo, GeneratedRootTS} {
|
||||
if got, err := OutOfStep(filepath.Join("..", "..", "mesh-delivery-contracts"), root); err != nil || len(got) != 0 {
|
||||
t.Errorf("mesh-delivery-contracts, root %s: %v %v, want nothing out of step", root, got, err)
|
||||
}
|
||||
}
|
||||
|
||||
// The build ask answers them beside what it built, and with a failed build too.
|
||||
built := 0
|
||||
|
||||
@@ -25,9 +25,11 @@ func Narrow(tree string, r sdkstep.FetchRequest) error {
|
||||
}
|
||||
keep := map[string]bool{".git": true}
|
||||
for _, p := range r.Paths {
|
||||
// Kept by its path from the repository's root: a path leaving the module's directory but not the repository is
|
||||
// narrowed from the root like any other (design 47 H-A-199).
|
||||
rel := path.Join(r.Dir, p)
|
||||
if strings.HasPrefix(rel, "../") || rel == ".." || path.IsAbs(p) {
|
||||
return fmt.Errorf("the named path %q leaves the module's directory", p)
|
||||
return fmt.Errorf("the named path %q leaves the repository", p)
|
||||
}
|
||||
if _, err := os.Lstat(filepath.Join(tree, filepath.FromSlash(rel))); err != nil {
|
||||
return fmt.Errorf("the named path %q is not in the fetched tree: %w", p, err)
|
||||
|
||||
@@ -68,6 +68,25 @@ func TestNarrowKeepsAModuleDirectorysPathsAndFailsOnAnAbsentOne(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A path leaving the module's directory but not the repository is kept from the root; one leaving the repository is
|
||||
// refused, naming the repository (design 47 H-A-198, H-A-199).
|
||||
func TestNarrowKeepsASiblingOfTheRepositoryAndRefusesOneOutsideIt(t *testing.T) {
|
||||
tree := t.TempDir()
|
||||
for _, f := range []string{"modules/a/module.json", "modules/a/pkg/go.mod", "modules/a/notes.md", "modules/b/client/go.mod",
|
||||
"modules/b/other.go", "root.txt"} {
|
||||
writeTreeFile(t, tree, f, "x")
|
||||
}
|
||||
if err := Narrow(tree, sdkstep.FetchRequest{Dir: "modules/a", Paths: []string{"../../../x"}}); err == nil || !strings.Contains(err.Error(), "leaves the repository") {
|
||||
t.Fatalf("a path leaving the repository gave %v", err)
|
||||
}
|
||||
if err := Narrow(tree, sdkstep.FetchRequest{Dir: "modules/a", Paths: []string{"pkg", "../b/client"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := strings.Join(filesUnder(t, tree), ","); got != "modules/a/module.json,modules/a/pkg/go.mod,modules/b/client/go.mod" {
|
||||
t.Fatalf("the narrowed tree holds %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A build that needs package.json or go.mod names it among its paths, and so keys on it (ADR 0499 clause 4).
|
||||
func TestNarrowKeepsAManifestOfTheLanguageOnlyWhenNamed(t *testing.T) {
|
||||
tree := t.TempDir()
|
||||
|
||||
@@ -38,9 +38,12 @@ type artifact struct {
|
||||
From string `json:"from"`
|
||||
Binary string `json:"binary"`
|
||||
Loads []string `json:"loads"`
|
||||
Paths []string `json:"paths"`
|
||||
Series string `json:"series"`
|
||||
}
|
||||
|
||||
type manifest struct {
|
||||
Packages map[string]string `json:"packages"`
|
||||
Module string `json:"module"`
|
||||
Version string `json:"version"`
|
||||
Slug string `json:"slug"`
|
||||
@@ -176,8 +179,16 @@ func TestManifestProcess(t *testing.T) { // 1240/AC1
|
||||
if _, bus := p["bus"]; bus {
|
||||
t.Errorf("the process declares a bus: it runs on its own broker credential")
|
||||
}
|
||||
if len(m.Build.Artifacts) != 2 {
|
||||
t.Fatalf("artifacts = %+v, want the program and the tools", m.Build.Artifacts)
|
||||
if len(m.Build.Artifacts) != 3 {
|
||||
t.Fatalf("artifacts = %+v, want the program, the tools and the package tellerevents", m.Build.Artifacts)
|
||||
}
|
||||
// H-A-188: the assigner is at 0.3, so its package asks, which this module imports, is accepted in that series.
|
||||
if got := m.Packages["git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"]; got != "0.3" || len(m.Packages) != 1 {
|
||||
t.Errorf("packages = %v, want asks accepted in the assigner's series 0.3", m.Packages)
|
||||
}
|
||||
// H-A-186 (design 47 G §370): tellerevents, which the assigner imports, is a declared package.
|
||||
if a := m.Build.Artifacts[2]; a.Name != "tellerevents" || a.Kind != "package" || a.Language != "go" || !reflect.DeepEqual(a.Paths, []string{"tellerevents"}) || a.Series != "0.1" {
|
||||
t.Errorf("package artifact = %+v", a)
|
||||
}
|
||||
a := m.Build.Artifacts[0]
|
||||
if st, err := os.Stat(a.From); err != nil || !st.IsDir() || a.Binary != "mesh-delivery-teller" || !strings.HasSuffix(a.From, a.Binary) {
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
{
|
||||
"module": "mesh-delivery-teller",
|
||||
"version": "0.1",
|
||||
"packages": {
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks": "0.3"
|
||||
},
|
||||
"slug": "teller",
|
||||
"tools": [
|
||||
"teller_end_ask",
|
||||
@@ -123,6 +126,15 @@
|
||||
"loads": [
|
||||
"mesh-delivery-teller-tools"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "tellerevents",
|
||||
"kind": "package",
|
||||
"language": "go",
|
||||
"paths": [
|
||||
"tellerevents"
|
||||
],
|
||||
"series": "0.1"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -70,6 +70,7 @@ type claim struct {
|
||||
}
|
||||
|
||||
type manifest struct {
|
||||
Packages map[string]string `json:"packages"`
|
||||
Claims []claim `json:"claims"`
|
||||
State []state `json:"state"`
|
||||
Module string `json:"module"`
|
||||
@@ -313,9 +314,12 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7
|
||||
|
||||
func TestManifestIsWholeInItsRepository(t *testing.T) { // 1134/AC7
|
||||
m, _ := load(t)
|
||||
// Issue 1966's bootstrap: the package artifact viewstate is out of the manifest until the new builder runs; the
|
||||
// follow-up puts the count back at 5.
|
||||
if len(m.Build.Artifacts) != 4 {
|
||||
// H-A-188: the assigner is at 0.3, so its package asks, which this module imports, is accepted in that series.
|
||||
if got := m.Packages["git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"]; got != "0.3" || len(m.Packages) != 1 {
|
||||
t.Errorf("packages = %v, want asks accepted in the assigner's series 0.3", m.Packages)
|
||||
}
|
||||
// H-A-186 (design 47 G §370): viewstate, which the gate, the assigner and the teller import, is a declared package.
|
||||
if len(m.Build.Artifacts) != 5 || m.Build.Artifacts[4].Name != "viewstate" {
|
||||
t.Fatalf("artifacts = %+v", m.Build.Artifacts)
|
||||
}
|
||||
for _, a := range m.Build.Artifacts {
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
{
|
||||
"module": "mesh-delivery-view",
|
||||
"version": "0.1",
|
||||
"packages": {
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks": "0.3"
|
||||
},
|
||||
"moved": true,
|
||||
"slug": "view",
|
||||
"tools": [
|
||||
@@ -234,6 +237,15 @@
|
||||
"name": "files",
|
||||
"kind": "archive",
|
||||
"from": "files"
|
||||
},
|
||||
{
|
||||
"name": "viewstate",
|
||||
"kind": "package",
|
||||
"language": "go",
|
||||
"paths": [
|
||||
"viewstate"
|
||||
],
|
||||
"series": "0.1"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -39,6 +39,7 @@ import (
|
||||
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||
@@ -180,8 +181,20 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
Create: creating{ctx: ctx, js: js}, Consumers: creating{ctx: ctx, js: js}, NamedPurge: creating{ctx: ctx, js: js},
|
||||
Objects: sealedObjects{ctx: ctx, js: js}, Purging: ownPurging{ctx: ctx, js: js, module: account.Module}, Logf: logger.Printf,
|
||||
Bus: grants.BusAddress{URL: account.URL, Fingerprint: account.Fingerprint}}
|
||||
// The setting grants-for-revisions-per-branch, read at the start, and the watched branches the part follows: grants-for
|
||||
// keeps those branches' revisions alone, so many to a branch (design 47 H H-A-195).
|
||||
perBranch, err := grants.RevisionsPerBranchSetting(strings.TrimSpace(getenv(grants.EnvRevisionsPerBranch)))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f.Part.RevisionsPerBranch, f.Part.Followed = perBranch, branches.Followed
|
||||
f.Watch = fv
|
||||
f.Part.ReadingRunning(nodeengine.StreamRunning)
|
||||
// The catalog-revision event names each branch's newest revision, so grants-for asks the seat mesh-catalog's revision only
|
||||
// for a revision it does not hold (design 47 H H-A-193), heard on the same reader beside the branch follower.
|
||||
if err := reader.AdoptEvent(meshcatalog.Seat, meshcatalog.EventCatalogRevision, f.Part.HearCatalogRevision); err != nil {
|
||||
return err
|
||||
}
|
||||
f.HearKept = func() error { return hearRunning(ctx, account, js, f, logger.Printf) }
|
||||
// The first view of the bindings (design 25 bindings-first-view): the seat mesh-broker's list, asked at each start and by
|
||||
// hand, with the seat's events held until its answer is applied or its bound passes.
|
||||
|
||||
@@ -83,6 +83,8 @@ type fakeBus struct {
|
||||
refuseCreate bool
|
||||
// refuseUsers refuses the bus module's nats_users: the loaded list cannot be read.
|
||||
refuseUsers bool
|
||||
// revisionNumber, when set, numbers each revision mesh-catalog's revision answers (newest, candidate), one entry a page.
|
||||
revisionNumber map[string]int64
|
||||
// rootFree answers a machine's engine's root-free verb: nil is no answerer at all (no responders).
|
||||
rootFree func(machines []string) (any, error)
|
||||
// consumers are the durable consumers the bus holds, by stream, as nats_streams answers them for one stream; and
|
||||
@@ -379,6 +381,19 @@ func (b *fakeBus) Call(key string, body any) (json.RawMessage, error) {
|
||||
_ = json.Unmarshal([]byte(manifest), &m)
|
||||
out.Entries = append(out.Entries, meshcatalog.CatalogEntry{Module: module, Manifest: m})
|
||||
}
|
||||
if n, numbered := b.revisionNumber[which]; numbered {
|
||||
slices.SortFunc(out.Entries, func(x, y meshcatalog.CatalogEntry) int { return strings.Compare(x.Module, y.Module) })
|
||||
out.Revision = &meshcatalog.CatalogRevision{Branch: in.Branch, Kind: "trunk", Revision: n}
|
||||
if in.Group != "" {
|
||||
out.Revision.Kind, out.Revision.Group, out.Revision.Heads = "candidate", in.Group, "h1"
|
||||
}
|
||||
for len(out.Entries) > 0 && out.Entries[0].Module <= in.After {
|
||||
out.Entries = out.Entries[1:]
|
||||
}
|
||||
if len(out.Entries) > 1 {
|
||||
out.Entries, out.Next = out.Entries[:1], out.Entries[0].Module
|
||||
}
|
||||
}
|
||||
return json.Marshal(out)
|
||||
case grants.CallStreams:
|
||||
if stream, _ := args["stream"].(string); stream != "" {
|
||||
@@ -648,6 +663,17 @@ func TestGrants(t *testing.T) {
|
||||
if _, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || m.SecretsOwner != "mesh-grants" {
|
||||
t.Errorf("own-secrets %v owner %q: a credential of its own, broker", m.OwnSecrets, m.SecretsOwner)
|
||||
}
|
||||
// H-A-186 (design 47 G §370): grantsstate (imported by the assigner, the vault and the node-engine) and busgrants
|
||||
// (imported by the bus module and the watchman) are declared Go packages of the module, so every release publishes them.
|
||||
var packages []string
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if a.Kind == catalogue.ArtifactPackage && a.Language == "go" && len(a.Paths) == 1 && a.Paths[0] == a.Name && a.Series == "0.1" {
|
||||
packages = append(packages, a.Name)
|
||||
}
|
||||
}
|
||||
if !slices.Equal(packages, []string{"grantsstate", "busgrants"}) {
|
||||
t.Errorf("Go packages %v, want grantsstate and busgrants of series 0.1", packages)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("no timer, no update or delete, a publish only of conditions, a create only of declared streams", func(t *testing.T) {
|
||||
@@ -5146,6 +5172,126 @@ func TestGrantsFor(t *testing.T) {
|
||||
if _, err := p.GrantsFor(meshgrants.GrantsForInput{Branch: "main"}); err == nil {
|
||||
t.Error("grants-for with neither a source nor a run answered")
|
||||
}
|
||||
|
||||
// A revision never changes, so one read whole is kept: a second grants-for of the same revisions reads only the first page
|
||||
// of the candidate, which names a revision kept, and none of the newest, whose number the first ask named (H-A-193: it asks
|
||||
// once); once the catalog-revision events name both, a grants-for reads no page.
|
||||
b = fixture()
|
||||
b.revision = map[string]map[string]string{"newest": {"mod-x": xManifest, "mod-y": xManifest}, "candidate": {"mod-x": candidateX, "mod-y": xManifest}}
|
||||
b.revisionNumber = map[string]int64{"newest": 9, "candidate": 8}
|
||||
p = partOn(b)
|
||||
pages := func() int {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
n := 0
|
||||
for _, c := range b.calls {
|
||||
if c == grants.CallRevision {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
first, _ := p.GrantsFor(meshgrants.GrantsForInput{Branch: "main", Source: "7"})
|
||||
read := pages()
|
||||
again, _ := p.GrantsFor(meshgrants.GrantsForInput{Branch: "main", Source: "7"})
|
||||
if read != 4 || pages()-read != 1 {
|
||||
t.Errorf("the revisions were read in %d pages, then %d again, want 4 then 1 (the candidate's first page)", read, pages()-read)
|
||||
}
|
||||
if len(first.Modules) != 1 || !reflect.DeepEqual(first, again) {
|
||||
t.Errorf("grants-for from kept revisions answers %+v, first %+v", again, first)
|
||||
}
|
||||
p = partOn(b)
|
||||
for _, ev := range []string{`{"v":1,"kind":"trunk","branch":"main","revision":8,"source":7}`, `{"v":1,"kind":"trunk","branch":"main","revision":9,"source":8}`} {
|
||||
if err := p.HearCatalogRevision(context.Background(), []byte(ev)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
_, _ = p.GrantsFor(meshgrants.GrantsForInput{Branch: "main", Source: "7"})
|
||||
read = pages()
|
||||
heard, _ := p.GrantsFor(meshgrants.GrantsForInput{Branch: "main", Source: "7"})
|
||||
if pages() != read || !reflect.DeepEqual(first, heard) {
|
||||
t.Errorf("after the events named both revisions held, grants-for read %d pages and answered %+v, want none and %+v", pages()-read, heard, first)
|
||||
}
|
||||
}
|
||||
|
||||
// grants-for keeps what H-A-195 says of the catalog: so many revisions to a branch as the setting
|
||||
// grants-for-revisions-per-branch says, a run's candidate counted within its branch's; only the followed branches', freed when
|
||||
// one leaves them; and an event below the newest held is the numbering started again, which drops what is kept.
|
||||
func TestGrantsForKeepsWhatItMay(t *testing.T) {
|
||||
setting := func(t *testing.T, text string) int {
|
||||
path := filepath.Join(t.TempDir(), "grants-for-revisions-per-branch")
|
||||
if err := os.WriteFile(path, []byte(text), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
n, err := grants.RevisionsPerBranchSetting(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return n
|
||||
}
|
||||
hear := func(t *testing.T, p *grants.Part, revision, source int) {
|
||||
ev := fmt.Sprintf(`{"v":1,"kind":"trunk","branch":"main","revision":%d,"source":%d}`, revision, source)
|
||||
if err := p.HearCatalogRevision(context.Background(), []byte(ev)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
// run asks the candidate by a run, else by the source revision 7; then is done after one grants-for; pages the
|
||||
// revision pages the grants-for after it reads.
|
||||
run bool
|
||||
// perBranch is the part's bound from the start; 0 its default.
|
||||
perBranch int
|
||||
then func(t *testing.T, b *fakeBus, p *grants.Part, followed *[]string)
|
||||
pages int
|
||||
}{
|
||||
{name: "the setting bounds a branch's revisions: with 2, the candidate is dropped once two newer are kept", pages: 2,
|
||||
then: func(t *testing.T, b *fakeBus, p *grants.Part, _ *[]string) {
|
||||
p.RevisionsPerBranch = setting(t, "2\n")
|
||||
hear(t, p, 10, 0)
|
||||
b.revisionNumber["newest"] = 10
|
||||
_, _ = p.GrantsFor(meshgrants.GrantsForInput{Branch: "main", Source: "7"})
|
||||
}},
|
||||
{name: "a run's candidate counts within its branch's bound: with 1, the candidate and the newest each drop the other", run: true,
|
||||
perBranch: 1, pages: 4, then: func(*testing.T, *fakeBus, *grants.Part, *[]string) {}},
|
||||
{name: "a branch that leaves the watched branches frees its revisions", pages: 4,
|
||||
then: func(t *testing.T, _ *fakeBus, p *grants.Part, followed *[]string) {
|
||||
*followed = nil
|
||||
_, _ = p.GrantsFor(meshgrants.GrantsForInput{Branch: "main", Source: "7"})
|
||||
*followed = []string{"main"}
|
||||
}},
|
||||
{name: "an event below the newest held drops what is kept of the branch", pages: 4,
|
||||
then: func(t *testing.T, _ *fakeBus, p *grants.Part, _ *[]string) { hear(t, p, 3, 0) }},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
b := fixture()
|
||||
b.revision = map[string]map[string]string{"newest": {"mod-x": xManifest, "mod-y": xManifest}, "candidate": {"mod-x": xManifest, "mod-y": xManifest}}
|
||||
b.revisionNumber = map[string]int64{"newest": 9, "candidate": 8}
|
||||
followed := []string{"main"}
|
||||
p := partOn(b)
|
||||
p.Followed = func() []string { return followed }
|
||||
p.RevisionsPerBranch = row.perBranch
|
||||
in := meshgrants.GrantsForInput{Branch: "main", Source: "7"}
|
||||
if row.run {
|
||||
in = meshgrants.GrantsForInput{Branch: "main", Run: "r1"}
|
||||
} else {
|
||||
hear(t, p, 8, 7)
|
||||
hear(t, p, 9, 8)
|
||||
}
|
||||
if got, err := p.GrantsFor(in); err != nil || got.Error != "" {
|
||||
t.Fatalf("grants-for: %+v, %v", got, err)
|
||||
}
|
||||
row.then(t, b, p, &followed)
|
||||
before := len(slices.DeleteFunc(slices.Clone(b.calls), func(c string) bool { return c != grants.CallRevision }))
|
||||
_, _ = p.GrantsFor(in)
|
||||
if got := len(slices.DeleteFunc(slices.Clone(b.calls), func(c string) bool { return c != grants.CallRevision })) - before; got != row.pages {
|
||||
t.Errorf("grants-for read %d revision pages, want %d", got, row.pages)
|
||||
}
|
||||
})
|
||||
}
|
||||
if n, err := grants.RevisionsPerBranchSetting(filepath.Join(t.TempDir(), "none")); n != grants.DefaultRevisionsPerBranch || err != nil {
|
||||
t.Errorf("with no file the setting is %d, %v; want its default", n, err)
|
||||
}
|
||||
}
|
||||
|
||||
// heldHalf is an assignment half holding rev, or none.
|
||||
|
||||
@@ -1,18 +1,23 @@
|
||||
package grants
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"maps"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/identity-and-access/go/seats/meshgrants"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
)
|
||||
|
||||
// **The seat mesh-grants' read `grants-for`** (design 47 G §441, §401, A-205): the grants a candidate would compose, per
|
||||
@@ -27,10 +32,240 @@ const RunGroup = "run-"
|
||||
// CallRevision is the seat mesh-catalog's `revision`, as the part calls it.
|
||||
const CallRevision = "seat:" + meshcatalog.Seat + "." + meshcatalog.VerbRevision
|
||||
|
||||
// DefaultRevisionsPerBranch is the setting grants-for-revisions-per-branch's default (design 47 H H-A-195): how many catalog
|
||||
// revisions of one branch grants-for keeps, a run's candidate counted within its branch's, the oldest read dropped first;
|
||||
// twice as many source revisions are kept named by the revision taken in from each.
|
||||
const DefaultRevisionsPerBranch = 4
|
||||
|
||||
// EnvRevisionsPerBranch names the file the manifest writes the setting grants-for-revisions-per-branch into.
|
||||
const EnvRevisionsPerBranch = "MESH_GRANTS_REVISIONS_PER_BRANCH_FILE"
|
||||
|
||||
// RevisionsPerBranchSetting reads the setting grants-for-revisions-per-branch from its file, read at the start: a missing or
|
||||
// empty file is its default, one that holds no positive whole number an error naming the file.
|
||||
func RevisionsPerBranchSetting(path string) (int, error) {
|
||||
if path == "" {
|
||||
return DefaultRevisionsPerBranch, nil
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return DefaultRevisionsPerBranch, nil
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
v := strings.TrimSpace(string(raw))
|
||||
if v == "" {
|
||||
return DefaultRevisionsPerBranch, nil
|
||||
}
|
||||
n, err := strconv.Atoi(v)
|
||||
if err != nil || n < 1 {
|
||||
return 0, fmt.Errorf("%s holds %q, which is not a positive whole number", path, v)
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// revisionsPerBranch is the part's bound of kept revisions per branch: its setting, else the default.
|
||||
func (p *Part) revisionsPerBranch() int {
|
||||
if p.RevisionsPerBranch > 0 {
|
||||
return p.RevisionsPerBranch
|
||||
}
|
||||
return DefaultRevisionsPerBranch
|
||||
}
|
||||
|
||||
// following frees what is kept of every branch the part follows no more, and says whether branch is followed: only a followed
|
||||
// branch's revisions are kept (H-A-195). With no Followed (the tools) every branch is.
|
||||
func (p *Part) following(branch string) bool {
|
||||
if p.Followed == nil {
|
||||
return true
|
||||
}
|
||||
followed := map[string]bool{}
|
||||
for _, b := range p.Followed() {
|
||||
followed[keys.EscapeName(b)] = true
|
||||
}
|
||||
p.revisions.free(followed)
|
||||
return followed[branch]
|
||||
}
|
||||
|
||||
// revisionsKept keeps the catalog revisions grants-for read, whole, as manifests by module (design 47 H H-A-195): a revision
|
||||
// never changes, so one read is read for good (as the runner keeps its revisionCache), a cache of the catalog seat's own truth
|
||||
// and never a state of the mesh. Each is kept by the branch grants-for was asked of (escaped as a key name is), its kind,
|
||||
// group, heads and number; the newest few of each followed branch are kept in memory for the process's life. It also
|
||||
// holds what the catalog-revision event names (H-A-193): each branch's newest trunk revision, and the revision each source
|
||||
// revision was taken in as, so grants-for asks the seat mesh-catalog's revision only for a revision it does not hold, never
|
||||
// to learn the newest.
|
||||
type revisionsKept struct {
|
||||
mu sync.Mutex
|
||||
order map[string][]string
|
||||
kept map[string]map[string]catalogue.Manifest
|
||||
newest map[string]int64
|
||||
sources map[string]map[int64]int64
|
||||
sourced map[string][]int64
|
||||
}
|
||||
|
||||
// revisionKey names a revision of branch; every kind but a candidate's is the branch's own series.
|
||||
func revisionKey(branch string, r meshcatalog.CatalogRevision) string {
|
||||
if r.Kind != "candidate" {
|
||||
r.Kind, r.Group, r.Heads = "trunk", "", ""
|
||||
}
|
||||
return strings.Join([]string{branch, r.Kind, r.Group, r.Heads, strconv.FormatInt(r.Revision, 10)}, "\x00")
|
||||
}
|
||||
|
||||
func (k *revisionsKept) init() {
|
||||
if k.kept == nil {
|
||||
k.kept, k.order, k.newest = map[string]map[string]catalogue.Manifest{}, map[string][]string{}, map[string]int64{}
|
||||
k.sources, k.sourced = map[string]map[int64]int64{}, map[string][]int64{}
|
||||
}
|
||||
}
|
||||
|
||||
func (k *revisionsKept) get(branch string, r meshcatalog.CatalogRevision) (map[string]catalogue.Manifest, bool) {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
m, ok := k.kept[revisionKey(branch, r)]
|
||||
return m, ok
|
||||
}
|
||||
|
||||
func (k *revisionsKept) put(branch string, r meshcatalog.CatalogRevision, manifests map[string]catalogue.Manifest, max int) {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
k.init()
|
||||
key := revisionKey(branch, r)
|
||||
if _, ok := k.kept[key]; !ok {
|
||||
k.order[branch] = append(k.order[branch], key)
|
||||
}
|
||||
k.kept[key] = manifests
|
||||
for len(k.order[branch]) > max {
|
||||
delete(k.kept, k.order[branch][0])
|
||||
k.order[branch] = k.order[branch][1:]
|
||||
}
|
||||
if r.Kind != "candidate" && r.Source != 0 {
|
||||
k.sourceLocked(branch, r.Source, r.Revision, max)
|
||||
}
|
||||
}
|
||||
|
||||
// free drops what is kept of each branch not in followed.
|
||||
func (k *revisionsKept) free(followed map[string]bool) {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
for _, m := range []map[string]bool{boolsOf(k.order), boolsOf(k.newest), boolsOf(k.sources)} {
|
||||
for branch := range m {
|
||||
if !followed[branch] {
|
||||
k.dropLocked(branch)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func boolsOf[V any](m map[string]V) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for k := range m {
|
||||
out[k] = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// dropLocked drops everything kept of branch.
|
||||
func (k *revisionsKept) dropLocked(branch string) {
|
||||
for _, key := range k.order[branch] {
|
||||
delete(k.kept, key)
|
||||
}
|
||||
delete(k.order, branch)
|
||||
delete(k.newest, branch)
|
||||
delete(k.sources, branch)
|
||||
delete(k.sourced, branch)
|
||||
}
|
||||
|
||||
func (k *revisionsKept) sourceLocked(branch string, source, revision int64, max int) {
|
||||
if k.sources[branch] == nil {
|
||||
k.sources[branch] = map[int64]int64{}
|
||||
}
|
||||
if _, ok := k.sources[branch][source]; !ok {
|
||||
k.sourced[branch] = append(k.sourced[branch], source)
|
||||
}
|
||||
k.sources[branch][source] = revision
|
||||
for len(k.sourced[branch]) > 2*max {
|
||||
delete(k.sources[branch], k.sourced[branch][0])
|
||||
k.sourced[branch] = k.sourced[branch][1:]
|
||||
}
|
||||
}
|
||||
|
||||
// heard takes a trunk revision a catalog-revision event names as its branch's newest. One below the newest held is the
|
||||
// numbering started again (design 47 H H-A-141): what is kept of the branch was of the earlier numbering, and is dropped.
|
||||
func (k *revisionsKept) heard(branch string, revision, source int64, max int) {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
k.init()
|
||||
if held, ok := k.newest[branch]; ok && revision < held {
|
||||
k.dropLocked(branch)
|
||||
}
|
||||
k.newest[branch] = revision
|
||||
if source != 0 {
|
||||
k.sourceLocked(branch, source, revision, max)
|
||||
}
|
||||
}
|
||||
|
||||
// asked takes the newest revision an ask answered, when no event named one since the start (H-A-193: it asks once).
|
||||
func (k *revisionsKept) asked(branch string, revision int64) {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
k.init()
|
||||
if _, ok := k.newest[branch]; !ok {
|
||||
k.newest[branch] = revision
|
||||
}
|
||||
}
|
||||
|
||||
// known is the revision in names that the part knows the number of without asking: the branch's newest an event named, or
|
||||
// the revision a source revision was taken in as; false when it knows none.
|
||||
func (k *revisionsKept) known(in meshcatalog.RevisionInput) (int64, bool) {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
switch {
|
||||
case in.Group != "" || in.Heads != "" || in.Revision != 0:
|
||||
return 0, false
|
||||
case in.Source != 0:
|
||||
n, ok := k.sources[in.Branch][in.Source]
|
||||
return n, ok
|
||||
default:
|
||||
n, ok := k.newest[in.Branch]
|
||||
return n, ok
|
||||
}
|
||||
}
|
||||
|
||||
// HearCatalogRevision takes one catalog-revision event of the seat mesh-catalog, as the part's process hears it (design 47 H
|
||||
// H-A-193): a trunk revision is its branch's newest, and the revision its source revision was taken in as; a candidate's moves
|
||||
// nothing here. Data that is no such event is an error.
|
||||
func (p *Part) HearCatalogRevision(_ context.Context, data []byte) error {
|
||||
var ev meshcatalog.CatalogRevisionEvent
|
||||
if err := json.Unmarshal(data, &ev); err != nil || ev.Revision <= 0 {
|
||||
return fmt.Errorf("not a %s event: %.200s", meshcatalog.EventCatalogRevision, data)
|
||||
}
|
||||
if ev.Kind == "candidate" || !p.following(ev.Branch) {
|
||||
return nil
|
||||
}
|
||||
p.revisions.heard(ev.Branch, ev.Revision, ev.Source, p.revisionsPerBranch())
|
||||
return nil
|
||||
}
|
||||
|
||||
// revisionEntries reads a catalog graph revision's entries, page after page, as manifests by module. none says the branch has
|
||||
// no revision yet.
|
||||
// no revision yet. A revision whose number the part knows (the branch's newest an event named, a source revision's) and holds
|
||||
// is answered from what it holds with no read; one it knows but does not hold is read by that number. Otherwise a revision
|
||||
// read whole is kept: once the first page of an answer names a revision that is kept, it is answered from what is kept,
|
||||
// without the pages after it. The branch's newest is read on by the number its first page names, so its pages are of one
|
||||
// revision though a newer one is written between them; a revision whose pages name another is not kept.
|
||||
func (p *Part) revisionEntries(in meshcatalog.RevisionInput) (manifests map[string]catalogue.Manifest, none bool, err error) {
|
||||
branch := in.Branch
|
||||
followed := p.following(branch)
|
||||
newestAsked := in.Group == "" && in.Heads == "" && in.Source == 0 && in.Revision == 0
|
||||
if n, ok := p.revisions.known(in); ok {
|
||||
if m, held := p.revisions.get(branch, meshcatalog.CatalogRevision{Revision: n}); held {
|
||||
return m, false, nil
|
||||
}
|
||||
if newestAsked {
|
||||
in.Revision = n
|
||||
}
|
||||
}
|
||||
manifests = map[string]catalogue.Manifest{}
|
||||
var first *meshcatalog.CatalogRevision
|
||||
keep := true
|
||||
for {
|
||||
raw, err := p.Call.Call(CallRevision, in)
|
||||
if err != nil {
|
||||
@@ -43,6 +278,23 @@ func (p *Part) revisionEntries(in meshcatalog.RevisionInput) (manifests map[stri
|
||||
if a.NoneYet {
|
||||
return nil, true, nil
|
||||
}
|
||||
switch {
|
||||
case a.Revision == nil:
|
||||
keep = false
|
||||
case first == nil:
|
||||
if newestAsked && followed {
|
||||
p.revisions.asked(branch, a.Revision.Revision)
|
||||
}
|
||||
if m, ok := p.revisions.get(branch, *a.Revision); ok {
|
||||
return m, false, nil
|
||||
}
|
||||
first = a.Revision
|
||||
if newestAsked {
|
||||
in.Revision = a.Revision.Revision
|
||||
}
|
||||
case revisionKey(branch, *a.Revision) != revisionKey(branch, *first):
|
||||
keep = false
|
||||
}
|
||||
for _, e := range a.Entries {
|
||||
body, err := json.Marshal(e.Manifest)
|
||||
if err != nil || len(e.Manifest) == 0 {
|
||||
@@ -55,6 +307,9 @@ func (p *Part) revisionEntries(in meshcatalog.RevisionInput) (manifests map[stri
|
||||
manifests[e.Module] = m
|
||||
}
|
||||
if a.Next == "" || a.Next == in.After {
|
||||
if keep && first != nil && followed {
|
||||
p.revisions.put(branch, *first, manifests, p.revisionsPerBranch())
|
||||
}
|
||||
return manifests, false, nil
|
||||
}
|
||||
in.After = a.Next
|
||||
|
||||
@@ -194,11 +194,19 @@ type Part struct {
|
||||
// SeatList is the seat list reader, which gates a module whose seats the list has not named (inputs.go, SeatGate); nil
|
||||
// gates none.
|
||||
SeatList SeatGate
|
||||
// RevisionsPerBranch is the setting grants-for-revisions-per-branch, read at the start (grantsfor.go, design 47 H H-A-195);
|
||||
// 0 is its default.
|
||||
RevisionsPerBranch int
|
||||
// Followed is the watched branches the part follows, as named (the branch follower's): grants-for keeps the revisions
|
||||
// of these alone, and frees a branch's once it leaves them; nil in the tools, where every branch is kept.
|
||||
Followed func() []string
|
||||
|
||||
cache Manifests
|
||||
mu sync.Mutex
|
||||
last Result
|
||||
made creations
|
||||
// revisions are the catalog revisions grants-for read, kept whole by their number (grantsfor.go).
|
||||
revisions revisionsKept
|
||||
mu sync.Mutex
|
||||
last Result
|
||||
made creations
|
||||
// askSeen is the revision of the last ask to compose (KeyAsked) this part acted on.
|
||||
askSeen uint64
|
||||
// namedPurged says the last pass found the build seats' named asks unused and purged them, so the next purges only
|
||||
|
||||
@@ -64,6 +64,16 @@
|
||||
"seat:mesh-broker.list",
|
||||
"seat:mesh-recovery.kept-points"
|
||||
],
|
||||
"config": {
|
||||
"grants-for-revisions-per-branch": {
|
||||
"type": "number",
|
||||
"description": "how many catalog revisions of one watched branch the grants part keeps once read for grants-for, a run's candidate counted within its branch's, the oldest dropped first; each some 1.4 MB (design 47 H H-A-195).",
|
||||
"minimum": 1,
|
||||
"maximum": 32,
|
||||
"trusted": false,
|
||||
"default": 4
|
||||
}
|
||||
},
|
||||
"emits": [
|
||||
"asserted",
|
||||
"grant-written",
|
||||
@@ -156,6 +166,14 @@
|
||||
"place": ".",
|
||||
"unknown": "program"
|
||||
},
|
||||
{
|
||||
"id": "grants-for-revisions-per-branch",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/grants-for-revisions-per-branch",
|
||||
"owner": "mesh-grants",
|
||||
"mode": "0600",
|
||||
"content": "${config:grants-for-revisions-per-branch}"
|
||||
},
|
||||
{
|
||||
"id": "grants",
|
||||
"type": "process",
|
||||
@@ -165,7 +183,8 @@
|
||||
"./mesh-grants"
|
||||
],
|
||||
"env": {
|
||||
"MESH_BROKER_FILE": "${dir:state}/broker"
|
||||
"MESH_BROKER_FILE": "${dir:state}/broker",
|
||||
"MESH_GRANTS_REVISIONS_PER_BRANCH_FILE": "${dir:state}/grants-for-revisions-per-branch"
|
||||
},
|
||||
"user": "mesh-grants",
|
||||
"health": {
|
||||
@@ -185,6 +204,24 @@
|
||||
"loads": [
|
||||
"mesh-grants"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "grantsstate",
|
||||
"kind": "package",
|
||||
"language": "go",
|
||||
"paths": [
|
||||
"grantsstate"
|
||||
],
|
||||
"series": "0.1"
|
||||
},
|
||||
{
|
||||
"name": "busgrants",
|
||||
"kind": "package",
|
||||
"language": "go",
|
||||
"paths": [
|
||||
"busgrants"
|
||||
],
|
||||
"series": "0.1"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user