diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/buildended.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/buildended.go index 98c0465d..f670eb63 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/buildended.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/buildended.go @@ -1,26 +1,25 @@ package main import ( + "context" "encoding/json" "fmt" "sync" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/process" "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "mesh-delivery-assigner/internal/assign" + "mesh-delivery-assigner/internal/assignment" ) // What a want names of a build is heard, never read from the builder's state (design 47 G §379, §526, A-94, A-96; ADR 0596 §2): the // builder's event `build.ended` (a done build's parts by digest, its keeping node and its branch) and the catalog's // `version-released` (a released version and each part's digest, with the source revision it was built from), each kept as heard. -// buildEndedSubjects is the builder's event build.ended on its own stream, as the manifest's consumes names it -// (`mesh-delivery-builder.build.ended`), each module and fingerprint after it. -var buildEndedSubjects = process.ModuleStreamSubject(builds.BuilderModule, builds.BuildEndedEvent+".>") - // builtEnds are the done builds' ends heard, by module and fingerprint (hearBuildEnded). type builtEnds struct { mu sync.Mutex @@ -38,6 +37,9 @@ func (b *builtEnds) put(data []byte) error { if e.Module == "" || e.Fingerprint == "" { return fmt.Errorf("a build's end naming no module or fingerprint: %s", data) } + if e.Outcome == "" { // an end said before A-162 is a done build's + e.Outcome = builds.BuildDone + } b.mu.Lock() defer b.mu.Unlock() if b.ends == nil { @@ -47,6 +49,14 @@ func (b *builtEnds) put(data []byte) error { return nil } +// endOf is the end heard of module's build at fp, done or failed (A-162): none while the build has not ended. +func (b *builtEnds) endOf(module, fp string) (builds.BuildEnded, bool) { + b.mu.Lock() + defer b.mu.Unlock() + e, ok := b.ends[b.key(module, fp)] + return e, ok +} + // partsOf is the parts of module's build at fp as its end names them (each artifact's digest by `:`), with the // node that keeps it, for a kept build or a build of another watched branch; false for a build of the trunk not kept, whose // parts the version's release says (releasedVersions). @@ -104,21 +114,46 @@ func (r *releasedVersions) of(module string, source int64) (assign.KeptBuild, bo return k, ok } -// hearBuildsAndReleases registers the hub's takers of build.ended and version-released, each waking the pass; before the hub starts. -func hearBuildsAndReleases(hub *process.EventHub, ends *builtEnds, released *releasedVersions, wake func(), logf func(string, ...any)) error { - taker := func(put func([]byte) error) func(jetstream.Msg) { - return func(m jetstream.Msg) { - if err := put(m.Data()); err != nil { - logf("%s is not kept: %v", m.Subject(), err) - } - if err := m.Ack(); err != nil { - logf("acknowledging %s: %v", m.Subject(), err) - } - wake() +// hearReleases registers the hub's taker of version-released, waking the pass; before the hub starts. +func hearReleases(hub *process.EventHub, released *releasedVersions, wake func(), logf func(string, ...any)) error { + return hub.Handle(meshcatalog.SubjectVersionReleased, func(m jetstream.Msg) { + if err := released.put(m.Data()); err != nil { + logf("%s is not kept: %v", m.Subject(), err) } + if err := m.Ack(); err != nil { + logf("acknowledging %s: %v", m.Subject(), err) + } + wake() + }) +} + +// buildsStreamName is the builder's stream of build ends as the bus names it: kept last per subject (one end per module and +// fingerprint, its manifest's `last-per-subject`, design 47 G A-181, §703). +var buildsStreamName = catalogue.StreamBusName(builds.BuildsStream) + +// hearBuilds reads the builder's stream of build ends whole at the start and then follows it (design 47 G A-181, §556): what was +// built, and a failed build's error, survive the assigner's restart, and it reads none of the builder's buckets. The read ends before +// it returns, so the first pass knows every build; a stream it cannot hear is said, and no want names a build until the assigner +// starts again. +func hearBuilds(ctx context.Context, js jetstream.JetStream, ends *builtEnds, wake func(), findings assign.Findings, logf func(string, ...any)) { + acct := process.Account{Node: thisNode, Module: assignment.Module} + nh := notHeardWriter(findings, logf) + cc, err := acct.HearStream(ctx, js, buildsStreamName, func(m jetstream.Msg) { + if err := ends.put(m.Data()); err != nil { + logf("%s is not kept: %v", m.Subject(), err) + } + if err := m.Ack(); err != nil { + logf("acknowledging %s: %v", m.Subject(), err) + } + wake() + }, logf) + if err != nil { + nh.NotHeard(buildsStreamName, "what the builder built is not heard until the assigner starts again, so no want names a build's parts or its keeping node and no put goes on", err) + return } - if err := hub.Handle(buildEndedSubjects, taker(ends.put)); err != nil { - return err - } - return hub.Handle(meshcatalog.SubjectVersionReleased, taker(released.put)) + nh.Heard(buildsStreamName) + go func() { + <-ctx.Done() + cc.Stop() + }() } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/buildended_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/buildended_test.go index d73743e2..7a86f9b3 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/buildended_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/buildended_test.go @@ -86,3 +86,25 @@ func TestATrunkEntryTakesItsVersionFromTheReleaseOfTheKeptRevisionsSource(t *tes t.Error("no revision kept names a release") } } + +// A-162: the assigner knows a failed build from its `build.ended`: the error whole; a done end or none heard is not failed. +func TestTheStandingOfABuildIsHeardFromItsEnd(t *testing.T) { + w := newWorld() + say := func(e builds.BuildEnded) { + raw, _ := json.Marshal(e) + if err := w.ends.put(raw); err != nil { + t.Fatal(err) + } + } + if failed, _ := w.buildStanding("web", "fp1"); failed { + t.Error("a build nothing was heard of failed") + } + say(builds.BuildEnded{Module: "web", Fingerprint: "fp1", Outcome: builds.BuildFailed, Error: "transport fault: connection"}) + if failed, why := w.buildStanding("web", "fp1"); !failed || why != "transport fault: connection" { + t.Errorf("failed end: failed %v why %q", failed, why) + } + say(builds.BuildEnded{Module: "web", Fingerprint: "fp1", Outcome: builds.BuildDone}) + if failed, _ := w.buildStanding("web", "fp1"); failed { + t.Error("a later done end leaves the build failed") + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/buildends_helper_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/buildends_helper_test.go new file mode 100644 index 00000000..ce5afcda --- /dev/null +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/buildends_helper_test.go @@ -0,0 +1,24 @@ +package main + +import ( + "encoding/json" + "testing" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" +) + +// endDone is the builder's `build.ended` of a done build of module at fp heard by w, naming the digest of its manifest (design 47 G +// A-181: what was built is heard, never read from the builder's bucket). +func endDone(t testing.TB, w *world, module, fp, manifest string) { + t.Helper() + data, err := json.Marshal(builds.BuildEnded{Module: module, Fingerprint: fp, Outcome: builds.BuildDone, Manifest: manifest}) + if err != nil { + t.Fatal(err) + } + if err := w.ends.put(data); err != nil { + t.Fatal(err) + } +} + +// trunkBranch is the first watched branch the tests give the world: the code names none (design 47 G §507). +const trunkBranch = "main" diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept.go index c013f018..4fc83d69 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept.go @@ -44,7 +44,8 @@ func (p *passer) keepGraph(ctx context.Context) { } in := assigngraph.Inputs{Halves: g.halves(), Placements: p.w.assignments(), Capabilities: p.capabilityDigests()} // With no pipeline ends kept, the wants are the trunk-only form (pass.go): a placement naming no branch runs the trunk's. - in.FirstBranch = func(string) string { return trunkBranch } + first := p.w.trunkName() + in.FirstBranch = func(string) string { return first } if p.ends != nil { in.FirstBranch = func(module string) string { return p.ends.branchOf(module, "") } } @@ -66,7 +67,7 @@ func (p *passer) keepGraph(ctx context.Context) { g.last, g.has, g.kept, g.ok = c.Revision, true, rev, true g.source = 0 for _, cat := range rev.Catalogs { - if cat.Branch == trunkBranch { + if cat.Branch == p.w.trunkName() { g.source = cat.Source } } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/halves.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/halves.go index 8d617071..83ce5f4f 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/halves.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/halves.go @@ -16,11 +16,12 @@ import ( "git.novox.be/novox/mesh-sdk/go/views" ) -// halves follows the catalog half of each watched branch through the seat mesh-catalog (design 47 G §14, §184, §195): one -// meshcatalogclient.Catalog per branch, adopted through the module's views reader (A-26), which asks the first view of the verb -// `revision`, hears each `catalog-revision` of that branch and raises and clears the follower's findings; each entry is taken -// into the world as the value and the graph node the rules read. Of the view the assigner reads only the catalog half, never its -// own output back (§195). +// halves follows the catalog half of each watched branch through the seat mesh-catalog (design 47 G §14, §184, §195, §507, §532): the +// branch follower of meshcatalogclient, adopted on the module's views reader (A-26), follows the branches the registrations name +// and, before the first registrations, the branches the gate's verb `source` answers and its event `source-moved` brings; each +// branch's view asks the verb `revision`, hears each `catalog-revision` of that branch and raises and clears the follower's findings; +// each entry is taken into the world as the value and the graph node the rules read. Of the view the assigner reads only the catalog +// half, never its own output back (§195). type halves struct { w *world call mcc.Call @@ -29,42 +30,70 @@ type halves struct { logf func(string, ...any) ctx context.Context - mu sync.Mutex - cats map[string]*mcc.Catalog - // settled is told once when a branch's first view is answered, for the first pass to wait on (the trunk's alone). + branches *mcc.Branches + mu sync.Mutex + // settled is told once when the first watched branch's first view is answered, for the first pass to wait on. settled chan<- struct{} + told bool } -// follow adopts the catalog of branch through the reader, once; true when it was made now. -func (h *halves) follow(branch string) bool { - h.mu.Lock() - if h.cats[branch] != nil { - h.mu.Unlock() - return false - } - c, _, err := mcc.WireCatalogProcess(h.reader, h.call, branch, func(module string, entry json.RawMessage, deleted bool, _ time.Time) { - h.changed(branch, module, entry, deleted) - }) +// follow makes the branch follower on the reader; the reader's start asks the first views. +func (h *halves) follow() error { + b, err := mcc.FollowBranchesProcess(h.reader, h.call) if err != nil { - h.mu.Unlock() - h.logf("the catalog half of branch %q is not followed: %v", branch, err) - return false + return err } - if h.cats == nil { - h.cats = map[string]*mcc.Catalog{} + b.Logf = h.logf + b.Changed = func(branch, module string, entry json.RawMessage, deleted bool, _ time.Time) { + h.changed(branch, module, entry, deleted) } - h.cats[branch] = c + b.Adopted = h.adopted + h.mu.Lock() + h.branches = b h.mu.Unlock() - if branch == trunkBranch && h.settled != nil { - go func() { - select { - case <-c.Ready(): - h.settled <- struct{}{} - case <-h.ctx.Done(): - } - }() + return nil +} + +// first is the first watched branch followed, the trunk's: "" while none is. +func (h *halves) first() string { + h.mu.Lock() + b := h.branches + h.mu.Unlock() + if b == nil { + return "" } - return true + if f := b.Followed(); len(f) > 0 { + return f[0] + } + return "" +} + +// adopted is told of each branch whose view the follower adopted: the first branch is the trunk's, and the first pass waits for its +// first view. +func (h *halves) adopted(branch string, c *mcc.Catalog) { + first := h.first() + if first == "" { + return + } + h.w.setTrunk(first) + h.wake() + if branch != first || h.settled == nil { + return + } + h.mu.Lock() + told := h.told + h.told = true + h.mu.Unlock() + if told { + return + } + go func() { + select { + case <-c.Ready(): + h.settled <- struct{}{} + case <-h.ctx.Done(): + } + }() } // changed takes one entry of branch's half into the world, or a module taken out of it. @@ -114,11 +143,17 @@ func fromEntry(e meshcatalog.CatalogEntry) (meshcatalog.ModulesByModuleValue, gr // views is the catalog graph revision held now of each followed branch whose first view is answered. func (h *halves) views() map[string]mcc.Half { h.mu.Lock() - defer h.mu.Unlock() - out := make(map[string]mcc.Half, len(h.cats)) - for branch, c := range h.cats { - if half, ok := c.View(); ok { - out[branch] = half + b := h.branches + h.mu.Unlock() + out := map[string]mcc.Half{} + if b == nil { + return out + } + for _, branch := range b.Followed() { + if c, ok := b.Catalog(branch); ok { + if half, ok := c.View(); ok { + out[branch] = half + } } } return out diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go index 80f04ef5..849d0215 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go @@ -44,6 +44,7 @@ import ( occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient" oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" "git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient" + mdseat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/announce" "git.novox.be/novox/mesh-sdk/go/buscompose" @@ -176,7 +177,23 @@ func startBed(t *testing.T, leaveOut string) *bed { Subjects: []string{buscompose.LogSubject(builds.BuilderModule, builds.BuilderLog) + ".>"}, AllowDirect: true}); err != nil { t.Fatal(err) } - if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"}}); err != nil { + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.mesh-catalog.event.>", "mesh.seat.mesh-conditions.event.>", "mesh.seat.mesh-module-assigner.event.>", enginereads.SubjectPrefix + "*"}}); err != nil { + t.Fatal(err) + } + // The node-engine seat's stream of `running`, kept last per subject, which the assigner hears the health of what runs from (§520). + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: nodeengine.StreamRunning, Subjects: []string{nodeengine.SubjectRunningAll}, MaxMsgsPerSubject: 1}); err != nil { + t.Fatal(err) + } + if _, err := js.CreateOrUpdateConsumer(ctx, nodeengine.StreamRunning, jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-assigner", AckPolicy: jetstream.AckExplicitPolicy}); err != nil { + t.Fatal(err) + } + // The builder's stream of build ends, kept last per subject (its manifest's `last-per-subject`, design 47 G A-181), which the assigner + // reads whole at its start and follows through its durable, as the grants part makes it (from the last of each subject). + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buildsStreamName, Subjects: []string{process.ModuleStreamSubject(builds.BuilderModule, builds.BuildEndedEvent+".>")}, MaxMsgsPerSubject: 1}); err != nil { + t.Fatal(err) + } + if _, err := js.CreateOrUpdateConsumer(ctx, buildsStreamName, jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-assigner", AckPolicy: jetstream.AckExplicitPolicy, + FilterSubjects: []string{process.ModuleStreamSubject(builds.BuilderModule, builds.BuildEndedEvent+".>")}, DeliverPolicy: jetstream.DeliverLastPerSubjectPolicy}); err != nil { t.Fatal(err) } // The assigner's own stream of its findings' events (design 47 G §383), which its finding writer says into. @@ -407,11 +424,38 @@ func TestTheAssignerDoesNotStartWithoutABucketItReads(t *testing.T) { } // putJSON writes a value into one of the bed's buckets. +// putBuild says the builder's `build.ended` of the build a builder's key and value (the old form of a done build, as the tests write it) name, +// on the builder's stream (design 47 G A-181): the assigner hears what was built, it reads no bucket of the builder. +func (b *bed) putBuild(t *testing.T, key string, value []byte) { + t.Helper() + module, fp, err := builds.ParseBuilderBuildKey(key) + built, derr := builds.DecodeBuilderBuild(value) + if err != nil || derr != nil { + t.Fatalf("%s: %v %v", key, err, derr) + } + raw, _ := json.Marshal(builds.BuildEnded{Module: module, Fingerprint: fp, Outcome: built.State, Error: built.Error, Manifest: built.Manifest, Needs: built.Needs}) + if _, err := b.js.Publish(b.ctx, process.ModuleStreamSubject(builds.BuilderModule, builds.BuildEndedEventName(module, fp)), raw); err != nil { + t.Fatal(err) + } +} + func (b *bed) putJSON(t *testing.T, bucket, key string, value []byte) { t.Helper() if _, err := b.buckets[bucket].Put(b.ctx, key, value); err != nil { t.Fatalf("%s %s: %v", bucket, key, err) } + if bucket == buckets.Have { // the node-engine says `running` with each have it writes (§520): the assigner reads health from it alone + node, module, isNode, err := nodeengineclient.ParseHaveKey(key) + h, derr := nodeengineclient.DecodeHave(value) + if err != nil || isNode || derr != nil || h.Phase != nodeengineclient.PhaseRunning { + return + } + ok := h.Health == nodeengineclient.HealthOK + raw, _ := json.Marshal(nodeengine.Running{At: time.Now(), Node: node, Module: module, Fingerprint: h.Fp, Healthy: &ok}) + if _, err := b.js.Publish(b.ctx, nodeengine.SubjectRunning(node, module), raw); err != nil { + t.Fatal(err) + } + } } // place adds module to the seat's assignments of node, written as the seat writes it: one node key and a key per module @@ -510,7 +554,7 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) { t.Fatal(err) } bkey, _ := builds.BuilderBuildKey("web", fNew) - b.putJSON(t, buckets.Builds, bkey, built) + b.putBuild(t, bkey, built) for _, fp := range []string{fNew, fOld} { gkey, _ := grantsstate.PartGrantKey("web", fp) grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{}) @@ -660,7 +704,7 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) { for _, fp := range fps { built, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: row.digest}) bkey, _ := builds.BuilderBuildKey(module, fp) - b.putJSON(t, buckets.Builds, bkey, built) + b.putBuild(t, bkey, built) gkey, _ := grantsstate.PartGrantKey(module, fp) grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{}) b.putJSON(t, buckets.Grants, gkey, grant) @@ -736,7 +780,7 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) { t.Fatal(err) } bkey, _ := builds.BuilderBuildKey("forms", fp) - b.putJSON(t, buckets.Builds, bkey, built) + b.putBuild(t, bkey, built) gkey, _ := grantsstate.PartGrantKey("forms", fp) grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{}) b.putJSON(t, buckets.Grants, gkey, grant) @@ -960,7 +1004,7 @@ func TestTheAssignerReplacedResumesWithNothingLostOrRepeated(t *testing.T) { t.Fatal(err) } bkey, _ := builds.BuilderBuildKey(name, fp) - b.putJSON(t, buckets.Builds, bkey, built) + b.putBuild(t, bkey, built) gkey, _ := grantsstate.PartGrantKey(name, fp) grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{}) b.putJSON(t, buckets.Grants, gkey, grant) @@ -1003,7 +1047,7 @@ func (b *bed) fixture(t *testing.T, name, fp, manifest string) { b.putModule(t, name, mod) built, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: digest}) bkey, _ := builds.BuilderBuildKey(name, fp) - b.putJSON(t, buckets.Builds, bkey, built) + b.putBuild(t, bkey, built) gkey, _ := grantsstate.PartGrantKey(name, fp) grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{}) b.putJSON(t, buckets.Grants, gkey, grant) @@ -1911,6 +1955,29 @@ func serveCatalog(t *testing.T, b *bed, nc *nats.Conn) { raw, _ := json.Marshal(map[string]any{"result": meshcatalog.RevisionAnswer{Revision: &rev, Entries: entries}, "node": "n0"}) _ = m.Respond(raw) } + // The gate's verb `source` given no branch (design 47 G §507, §690): the branches it keeps a source revision of, which the assigner + // follows before the first registrations, and the trunk's `main` first. + source := func(m *nats.Msg) { + b.cat.mu.Lock() + defer b.cat.mu.Unlock() + names := map[string]bool{"main": true} + for branch := range b.cat.rev { + names[branch] = true + } + var escaped []string + for _, n := range slices.Sorted(maps.Keys(names)) { + escaped = append(escaped, keys.EscapeName(n)) + } + raw, _ := json.Marshal(map[string]any{"result": mdseat.SourceAnswer{Branches: escaped}, "node": "n0"}) + _ = m.Respond(raw) + } + for _, subject := range []string{"mesh.seat.mesh-delivery.tool.source", "mesh.seat.mesh-delivery.call.source.*", "mesh.seat.mesh-delivery.tool.source.*"} { + sub, err := nc.Subscribe(subject, source) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = sub.Unsubscribe() }) + } for _, subject := range []string{"mesh.seat.mesh-catalog.tool.revision", "mesh.seat.mesh-catalog.call.revision.*", "mesh.seat.mesh-catalog.tool.revision.*"} { sub, err := nc.Subscribe(subject, answer) if err != nil { @@ -1920,3 +1987,52 @@ func serveCatalog(t *testing.T, b *bed, nc *nats.Conn) { } subscribed(t, nc) } + +// A-181: the assigner reads the builder's stream of build ends whole at its start (the last of each fingerprint, a failed one's error +// among them) though its durable is made after them, then follows it: what was built survives its restart, and it reads no builder bucket. +func TestTheAssignerReadsTheBuildersEndsWholeAtItsStart(t *testing.T) { + b := startBed(t, "") + thisNode = "n1" + b.putBuild(t, mustBuildKey(t, "web", "src2:"+strings.Repeat("a", 64)), mustEncodeBuild(t, builds.BuilderBuild{State: builds.BuildFailed, Error: "first try"})) + b.putBuild(t, mustBuildKey(t, "web", "src2:"+strings.Repeat("a", 64)), mustEncodeBuild(t, builds.BuilderBuild{State: builds.BuildDone, Manifest: strings.Repeat("d", 64)})) + b.putBuild(t, mustBuildKey(t, "api", "src2:"+strings.Repeat("b", 64)), mustEncodeBuild(t, builds.BuilderBuild{State: builds.BuildFailed, Error: "transport fault: x"})) + var ends builtEnds + woken := make(chan struct{}, 8) + hearBuilds(b.ctx, b.js, &ends, func() { woken <- struct{}{} }, assign.Findings{}, t.Logf) + if e, ok := ends.endOf("web", "src2:"+strings.Repeat("a", 64)); !ok || e.Outcome != builds.BuildDone || e.Manifest != strings.Repeat("d", 64) { + t.Fatalf("the read of the stream left web's end as %+v, %v: want the last, done", e, ok) + } + if e, ok := ends.endOf("api", "src2:"+strings.Repeat("b", 64)); !ok || e.Outcome != builds.BuildFailed || e.Error != "transport fault: x" { + t.Fatalf("the read of the stream left api's end as %+v, %v", e, ok) + } + b.putBuild(t, mustBuildKey(t, "api", "src2:"+strings.Repeat("c", 64)), mustEncodeBuild(t, builds.BuilderBuild{State: builds.BuildDone, Manifest: strings.Repeat("d", 64)})) + deadline := time.After(10 * time.Second) + for { + if _, ok := ends.endOf("api", "src2:"+strings.Repeat("c", 64)); ok { + return + } + select { + case <-woken: + case <-deadline: + t.Fatal("an end said after the read was not followed") + } + } +} + +func mustBuildKey(t *testing.T, module, fp string) string { + t.Helper() + k, err := builds.BuilderBuildKey(module, fp) + if err != nil { + t.Fatal(err) + } + return k +} + +func mustEncodeBuild(t *testing.T, b builds.BuilderBuild) []byte { + t.Helper() + raw, err := builds.EncodeBuilderBuild(b) + if err != nil { + t.Fatal(err) + } + return raw +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index baa43787..2b579639 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -63,12 +63,9 @@ func main() { var readBuckets = []string{ buckets.Grants, buckets.Answers, - buckets.Builds, buckets.Presence, buckets.Have, buckets.Provisions, - buckets.Source, - buckets.Merges, } // readFamilies are the states kept one bucket per node the assigner reads, each named in its manifest's reads: every @@ -168,12 +165,6 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any logf("the bucket %s cannot be opened (%v): the assigner's conditions are said straight to the router until it starts again", name, err) continue } - if (name == buckets.Source || name == buckets.Merges) && err != nil { - // The gate's source and merges (ADR 0601 §1): without them no release is known to carry a build, so every move of - // a record module is asked of a person, as before. - logf("the bucket %s cannot be opened (%v): no release's Approve answers a record ask until it is bound", name, err) - continue - } if name == engineReadsBucket && err != nil { // New with ADR 0618 §3: without it the statements the verb answers are kept in memory only, and // assigner_engine_reads lists nothing. @@ -215,7 +206,7 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any func startsWithout(name string) bool { switch name { case plc.SeatHoldersBucket, buckets.Provisions, assign.PoliciesBucket, assign.FindingsBucket, - buckets.Puts, buckets.Source, buckets.Merges, engineReadsBucket, retiredBucket, writtenBucket, buckets.Waits, buckets.Inputs: + buckets.Puts, engineReadsBucket, retiredBucket, writtenBucket, buckets.Waits, buckets.Inputs: return true } return slices.Contains(seatBuckets, name) || name == buckets.Grants @@ -326,7 +317,6 @@ func (e emitting) Emit(eventType string, body any) error { return e.Conn.Emit(ev var watched = []string{ buckets.Grants, buckets.Answers, - buckets.Builds, plc.AssignmentsBucket, plc.ProposalsBucket, plc.CandidatesBucket, @@ -340,8 +330,6 @@ var watched = []string{ assign.PoliciesBucket, buckets.Provisions, plc.SeatHoldersBucket, - buckets.Source, - buckets.Merges, engineReadsBucket, writtenBucket, buckets.Waits, @@ -455,8 +443,8 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, } started++ var opts []jetstream.WatchOpt - if name == buckets.Have || name == buckets.Source { - // what each node ran healthy before, for a set-back; and the release behind each source revision (ADR 0601 §1) + if name == buckets.Have { + // what each node ran healthy before, for a set-back opts = append(opts, jetstream.IncludeHistory()) } if err := watch(name, opts...); err != nil { @@ -564,13 +552,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, // The runner's pipeline-ended and group-ended (design 47 G §29): kept in the state written, which the wants follow // once the first registrations are written (§95, §353). p.ends = newPipelineEnds(kvEnds{ctx, kv}, wake, logger.Printf) - if iv != nil && iv.h != nil { - p.ends.onBranches = func(branches []string) { - for _, b := range branches { - iv.h.follow(b) - } - } - } + p.ends.trunk = w.trunkName err := hearEnds(hub, p.ends) if err == nil && bus != nil { err = followRegistrations(ctx, hub, bus.Call, p.ends) @@ -579,9 +561,10 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, logger.Printf("a pipeline's end or a registration is not heard, so the wants keep the trunk-only form until the assigner starts again: %v", err) } } - if err := hearBuildsAndReleases(hub, &w.ends, &w.released, wake, logger.Printf); err != nil { - logger.Printf("a build's end or a version's release is not heard, so no want names a build's parts, its keeping node or its version until the assigner starts again: %v", err) + if err := hearReleases(hub, &w.released, wake, logger.Printf); err != nil { + logger.Printf("a version's release is not heard, so no want names its version until the assigner starts again: %v", err) } + hearBuilds(ctx, js, &w.ends, wake, findings, logger.Printf) hearReadsChanged(ctx, js, hub, reads, findings, gate.EventsLive, logger.Printf) hearRunning(ctx, js, w, wake, findings, logger.Printf) } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/outside.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/outside.go index 1af5f53c..6fc18759 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/outside.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/outside.go @@ -269,7 +269,7 @@ func (p *passer) outsideReview(ctx context.Context, module, fp string, nodes []s dirty := o.dirty[node] cleared := prev != nil && prev.Cause != "" && o.cleared[node+"|"+prev.Cause] o.mu.Unlock() - failing, healthy := ok && h.Fp == fp && assign.Failed(h), ok && assign.Healthy(h, fp) + failing, healthy := ok && h.Fp == fp && assign.Failed(h), n.RunsHealthy(node, module, fp) if !failing && !healthy && prev == nil { continue } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/outside_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/outside_test.go index 4a06236e..3fff5401 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/outside_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/outside_test.go @@ -66,6 +66,8 @@ func (r *outsideRig) review(keeperSelf bool, moved func(string, string) bool) as r.t.Helper() n := assign.Nodes{Online: map[string]bool{"ace": true}, Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{ "ace": {"nats": {Fp: fpTest, Phase: nodeengineclient.PhaseRunning, Health: r.health}}}} + healthy := r.health == nodeengineclient.HealthOK + n.Running = map[string]map[string]nodeengine.Running{"ace": {"nats": {Node: "ace", Module: "nats", Fingerprint: fpTest, Healthy: &healthy}}} if moved == nil { moved = func(string, string) bool { return false } } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go index 71581d0e..8bcfa64d 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go @@ -247,7 +247,7 @@ func (p *passer) wants(ctx context.Context) { views[branch] = v return v } - nodes := viewOf(trunkBranch).nodes + nodes := viewOf(p.w.trunkName()).nodes p.engineReads(nodes) nodesOf := map[string][]string{} for node, modules := range assigned { @@ -319,8 +319,7 @@ func (p *passer) wants(ctx context.Context) { Allowed: func(node, digest string) bool { key, err := asks.RecordAskKey(node, name, digest) return err == nil && p.answeredWith(key, assign.OptionYes) - }, - Release: func(fp string) (string, bool) { return p.w.release(name, fp) }} + }} if fp != "" { held, _ := mwc.HeldKey(name, fp) grant, _ := grantsstate.PartGrantKey(name, fp) @@ -329,6 +328,7 @@ func (p *passer) wants(ctx context.Context) { m.Released = p.answeredWith(hold, "release") m.Kept = p.answeredWith(hold, "keep") m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(buckets.Wants, held) && !m.Released, p.w.has(grants, grant) + m.BuildFailed, m.BuildError = p.w.buildStanding(name, fp) if raw := p.w.value(grants, grant); raw != nil { if g, err := grantsstate.DecodeGrant(raw); err == nil { m.GrantedOn = func(node string) bool { return assign.GrantNamesNode(g, node, name) } @@ -404,7 +404,7 @@ func (p *passer) wants(ctx context.Context) { func (p *passer) movedTo(wanted map[string]map[string]mwc.Want, n assign.Nodes) func(owner, node string) bool { return func(owner, node string) bool { w, ok := wanted[owner][node] - return ok && !w.Removed && !assign.Healthy(n.Haves[node][owner], w.Fp) + return ok && !w.Removed && !n.RunsHealthy(node, owner, w.Fp) } } @@ -1060,9 +1060,8 @@ func (p *passer) backupAsk(ctx context.Context, key, node, module string, want m // yeses writes a placed node's wants naming the build alone, and sees to the moves a node that composes its own judged a // person must allow (ADR 0547 §3): for a have of phase // waiting-yes it raises the record ask once, naming the composition waiting and the node's judgement, and on a yes writes -// the want again naming it; a yes once written is kept while the want stays at its build, so the node is not asked again. A -// move of the build alone (the have says so) to a build an approved release carries is given its yes by that Approve, the -// want naming the release, with no ask (ADR 0601 §1). +// the want again naming it; a yes once written is kept while the want stays at its build, so the node is not asked again. No +// release's Approve is a yes: the Approve per merge goes and the assigner reads no `merges` (design 47 G §431). func (p *passer) yeses(ctx context.Context, module string, m assign.Module, before map[string]mwc.Want, out *assign.Outcome) { for _, node := range m.Nodes { if m.Self == nil || !m.Self(node) { @@ -1091,14 +1090,6 @@ func (p *passer) yeses(ctx context.Context, module string, m assign.Module, befo switch { case m.Allowed != nil && m.Allowed(node, h.Waits): yes, release = h.Waits, "" - case h.BuildAlone && h.Fp != w.Fp && m.Release != nil: - // A move of the build alone, to the build an approved release carries: that Approve is the yes (ADR 0601 §1), - // named in the want. A move that changes a setting or a secret too, or one the node could not tell, is asked. - if r, ok := m.Release(w.Fp); ok { - yes, release = h.Waits, r - break - } - p.recordAsk(ctx, module, w.Fp, assign.RecordNeed{Node: node, Digest: h.Waits, Judgement: h.Judgement, Words: h.Detail}) default: p.recordAsk(ctx, module, w.Fp, assign.RecordNeed{Node: node, Digest: h.Waits, Judgement: h.Judgement, Words: h.Detail}) } @@ -1408,14 +1399,14 @@ type group struct { func (p *passer) groups(module string, nodes []string, placed map[string]string) []group { by := map[string][]string{} for _, node := range nodes { - b := trunkBranch + b := p.w.trunkName() if p.ends.following() { b = p.ends.branchOf(module, placed[node]) } by[b] = append(by[b], node) } if len(by) == 0 { - b := trunkBranch + b := p.w.trunkName() if p.ends.following() { b = p.ends.branchOf(module, "") } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go index e8398ead..67b4464f 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go @@ -18,12 +18,10 @@ import ( "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" wdc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshwatchdogsclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" mdc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient" mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" - mdseat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-teller/tellerevents" "git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate" @@ -70,7 +68,9 @@ func (m *memKV) Delete(_ context.Context, key string, _ ...jetstream.KVDeleteOpt // newPasser is a pass on a fresh world, as after a start, writing to the buckets given, at the clock given. func newPasser(kvs kvSet, at time.Time) *passer { - return &passer{kvs: kvs, w: newWorld(), logf: func(string, ...any) {}, said: map[string]bool{}, answeredAt: map[string]uint64{}, + w := newWorld() + w.setTrunk(trunkBranch) + return &passer{kvs: kvs, w: w, logf: func(string, ...any) {}, said: map[string]bool{}, answeredAt: map[string]uint64{}, now: func() time.Time { return at }} } @@ -560,18 +560,14 @@ func putOfTheBus(t *testing.T) { mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: mdigest, Repository: "novox/mesh-catalog", Commit: strings.Repeat("c", 40)}) putModuleRaw(p.w, trunkBranch, "nats", mod) p.w.manifests[mdigest] = parsed{module: "nats", manifest: catalogue.Manifest{Module: "nats", Provides: []catalogue.Offer{{Name: "mesh-bus"}}}} - bkey, _ := builds.BuilderBuildKey("nats", fp) - built, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: mdigest}) - world(buckets.Builds, bkey, built) + endDone(t, p.w, "nats", fp, mdigest) hkey, _ := nodeengineclient.HaveKey("novox", "nats") before, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}) world(buckets.Have, hkey, before) - buildKV := mem() - buildKV.values[bkey] = built // Issue 1966: newest is asked of the indexer's first view of modules, never read from its bus state. indexer := &fakeIndexer{views: map[string]map[string][]byte{"mesh-delivery-indexer.modules": {"mesh-catalog": mod, "nats": mod}}} - s := newPutTools(memStore{putsState: putsKV, builds.BuilderModule + ".builds": buildKV}, func(key string, body any) (json.RawMessage, error) { + s := newPutTools(memStore{putsState: putsKV}, func(key string, body any) (json.RawMessage, error) { if raw, ok, err := indexer.answer(key, body); ok { return raw, err } @@ -582,10 +578,10 @@ func putOfTheBus(t *testing.T) { answerWithin = 0 defer func() { answerWithin = was }() if _, err := s.doPut(map[string]any{"node": "novox", "module": "nats", "version": "fd497ab8ffff", "why": "x"}, "shanks.node-tools"); err == nil { - t.Error("a prefix of a fingerprint the builder never built was taken") + t.Error("a prefix of a fingerprint was taken: a put names the whole fingerprint (A-181)") } - if got, err := s.version("nats", "fd497ab80000"); err != nil || got != fp { - t.Errorf("the prefix of the fingerprint built reads %q, %v", got, err) + if _, err := s.version("nats", "fd497ab80000"); err == nil { + t.Error("the prefix of the fingerprint built was read: a put names the whole fingerprint (A-181)") } if got, err := s.version("nats", "newest"); err != nil || got != fp || indexer.asks["mesh-delivery-indexer.modules"] != 2 { t.Errorf("newest reads %q, %v, asking %d pages of the indexer's first view (want its 2)", got, err, indexer.asks["mesh-delivery-indexer.modules"]) @@ -811,14 +807,14 @@ func testThePassGathersTheModulesOnNeitherList(t *testing.T) { p := newPasser(kvSet{}, time.Date(2026, 10, 25, 21, 30, 0, 0, time.UTC)) var logged []string p.logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) } - notDone := assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Rule: mwc.RuleBuildDone, Words: "the build of rofi at 4e7b118e is not done"}}} + notDone := assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Rule: mwc.RuleBuildDone, Words: "the build of rofi at 4e7b118e is being built"}}} wanted := assign.Outcome{Wants: map[string]mwc.Want{"shanks": {Fp: fpTest, Composed: "sha256:" + strings.Repeat("c", 64)}}} for i, c := range []struct { before map[string]mwc.Want out assign.Outcome line string // the line said in this pass, "" for none }{ - {nil, notDone, "rofi on shanks gets no want: rule 1: the build of rofi at 4e7b118e is not done"}, + {nil, notDone, "rofi on shanks gets no want: rule 1: the build of rofi at 4e7b118e is being built"}, {nil, notDone, ""}, {map[string]mwc.Want{"shanks": {Removed: true}}, notDone, ""}, {nil, assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Rule: mwc.RuleGrant, Words: "no credential"}}}, "rofi on shanks gets no want: rule 6: no credential"}, @@ -1168,70 +1164,29 @@ func TestARecordAskOfAHeldBackNodeSurvivesPassesAndItsYesMovesTheWant(t *testing aReleasesApproveAnswersTheMoveOfItsBuildAlone(t) } -// ADR 0601 §1: a placed node's have waiting for a yes to a move of the build alone, to a build an approved release carries, is -// given its yes by that Approve, the want naming the release, and no record ask is raised; a move that changes a setting or a -// secret too, one at the build it runs, and one no approved release carries are asked of a person as before. +// Design 47 G §431: no release's Approve is a yes (the assigner reads no `merges`): a placed node's have waiting for a yes to a move +// of the build alone is asked of a person, the want naming no release and no yes. func aReleasesApproveAnswersTheMoveOfItsBuildAlone(t *testing.T) { fpOld := meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64) waiting, was := "sha256:"+strings.Repeat("e", 64), "sha256:"+strings.Repeat("d", 64) commit := strings.Repeat("1", 40) - for _, c := range []struct { - name string - alone bool - at string // the fingerprint the have runs - mergedAt string // the commit the release merged web at; "" for no merge of the release - fpSrc int64 // the source revision at which web's fingerprint last changed - yes, asked bool - }{ - {"the build alone, its release approved", true, fpOld, commit, 7, true, false}, - {"a setting changed too", false, fpOld, commit, 7, false, true}, - {"a setting changed at the build it runs", true, fpTest, commit, 7, false, true}, - {"no approved release carries it", true, fpOld, "", 7, false, true}, - {"the release merged another commit", true, fpOld, strings.Repeat("2", 40), 7, false, true}, - // Its fingerprint last moved at another release's revision, which merged nothing of web: that Approve answers nothing. - {"a fingerprint another release moved", true, fpOld, commit, 8, false, true}, - // Review rounds 1 and 2: moved by a recompute after the release (a toolchain, a package, a base) with no merge, the - // indexer writes fp-src 0, not known, though Src is still the release's: asked of a person. - {"a fingerprint moved by a recompute after the release", true, fpOld, commit, 0, false, true}, - } { - asksKV := &memKV{values: map[string][]byte{}} - p := newPasser(kvSet{buckets.Asks: asksKV}, time.Now()) - p.logf = t.Logf - mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fpTest, Repository: "novox/web", Commit: commit, Src: 7, FpSrc: c.fpSrc, Path: meshcatalogclient.RootPath}) - putModuleRaw(p.w, trunkBranch, "web", mod) - src, _ := mdc.EncodeSource(mdseat.SourceByBranchValue{Group: "feat/web", Heads: map[string]string{"novox/web": commit}}) - p.w.putEntry(buckets.Source, mdc.SourceKey(trunkBranch), src, 7, time.Now()) - later, _ := mdc.EncodeSource(mdseat.SourceByBranchValue{Group: "feat/other", Heads: map[string]string{"novox/web": commit, "novox/x": commit}}) - p.w.putEntry(buckets.Source, mdc.SourceKey(trunkBranch), later, 8, time.Now()) // a later merge leaves revision 7's release - if c.mergedAt != "" { - merge, err := mdc.EncodeMerge(mdseat.MergesByGroupValue{Group: "feat/web", Ask: "merge.feat_web.0123456789ab", Pulls: []mdseat.MergesByGroupValuePullsItem{ - {Repository: "novox/web", Number: 3, Head: strings.Repeat("3", 40), State: string(mdc.MergeMerged), Commit: c.mergedAt}}}) - if err != nil { - t.Fatal(err) - } - p.w.put(buckets.Merges, "feat/web", merge) - other, err := mdc.EncodeMerge(mdseat.MergesByGroupValue{Group: "feat/other", Ask: "merge.feat_other.0123456789ab", Pulls: []mdseat.MergesByGroupValuePullsItem{ - {Repository: "novox/x", Number: 4, Head: strings.Repeat("4", 40), State: string(mdc.MergeMerged), Commit: commit}}}) - if err != nil { - t.Fatal(err) - } - p.w.put(buckets.Merges, "feat/other", other) - } - have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: c.at, Composed: was, Phase: nodeengineclient.PhaseWaitingYes, Health: nodeengineclient.HealthOK, - Judgement: assign.JudgedRecreates, Waits: waiting, Detail: "it recreates web.server", BuildAlone: c.alone}) - p.w.put(buckets.Have, "g14.web", have) - m := assign.Module{Name: "web", FP: fpTest, Nodes: []string{"g14"}, Self: func(string) bool { return true }, - Release: func(fp string) (string, bool) { return p.w.release("web", fp) }} - out := assign.Outcome{Wants: map[string]mwc.Want{"g14": {Fp: fpTest, Self: true}}} - p.yeses(context.Background(), "web", m, map[string]mwc.Want{"g14": {Fp: fpOld, Self: true}}, &out) - w := out.Wants["g14"] - if got := w.Yes == waiting && w.Release == "feat/web"; got != c.yes || (!c.yes && (w.Yes != "" || w.Release != "")) { - t.Errorf("%s: the want is %+v", c.name, w) - } - akey, _ := asks.RecordAskKey("g14", "web", waiting) - if _, there := asksKV.values[akey]; there != c.asked { - t.Errorf("%s: a record ask raised %v, want %v", c.name, there, c.asked) - } + asksKV := &memKV{values: map[string][]byte{}} + p := newPasser(kvSet{buckets.Asks: asksKV}, time.Now()) + p.logf = t.Logf + mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fpTest, Repository: "novox/web", Commit: commit, Src: 7, FpSrc: 7, Path: meshcatalogclient.RootPath}) + putModuleRaw(p.w, trunkBranch, "web", mod) + have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fpOld, Composed: was, Phase: nodeengineclient.PhaseWaitingYes, Health: nodeengineclient.HealthOK, + Judgement: assign.JudgedRecreates, Waits: waiting, Detail: "it recreates web.server", BuildAlone: true}) + p.w.put(buckets.Have, "g14.web", have) + m := assign.Module{Name: "web", FP: fpTest, Nodes: []string{"g14"}, Self: func(string) bool { return true }} + out := assign.Outcome{Wants: map[string]mwc.Want{"g14": {Fp: fpTest, Self: true}}} + p.yeses(context.Background(), "web", m, map[string]mwc.Want{"g14": {Fp: fpOld, Self: true}}, &out) + if w := out.Wants["g14"]; w.Yes != "" || w.Release != "" { + t.Errorf("the want is %+v: no release's Approve is a yes", w) + } + akey, _ := asks.RecordAskKey("g14", "web", waiting) + if _, there := asksKV.values[akey]; !there { + t.Error("no record ask was raised for the move of the build alone") } } @@ -1407,9 +1362,7 @@ func aWantWaitsOnTheVerbEdgesTheGraphKeeps(t *testing.T) { t.Fatal(err) } p.w.manifests[digest] = parsed{module: "server", manifest: parsedServer} - build, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: digest}) - key, _ := builds.BuilderBuildKey("server", fp("f")) - p.w.put(buckets.Builds, key, build) + endDone(t, p.w, "server", fp("f"), digest) running := nodeengine.HaveByNodeModuleValue{Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} at := func(f string) nodeengine.HaveByNodeModuleValue { h := running; h.Fp = f; return h } served := p.served(g, assign.Nodes{Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{ diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go index 63951fc7..8ecc9bec 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go @@ -153,6 +153,8 @@ type pipelineEnds struct { firsts map[string]string // onBranches is told the branches a registration names, for the catalog half of each to be followed (design 47 G §14, §195). onBranches func([]string) + // trunk names the first watched branch, the trunk's (design 47 G §507), the branch of a module registered with none. + trunk func() string } func newPipelineEnds(st endStore, wake func(), logf func(string, ...any)) *pipelineEnds { @@ -225,7 +227,10 @@ func (e *pipelineEnds) branchOf(module, placed string) string { if b := e.firsts[module]; b != "" { return b } - return trunkBranch + if e.trunk != nil { + return e.trunk() + } + return "" } // hearPipeline takes a pipeline-ended. A pipeline of a group writes nothing on its own end, and a run on request delivers diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines_test.go index 19a2d4a4..d0c6fa1c 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines_test.go @@ -20,7 +20,6 @@ import ( oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate" @@ -154,9 +153,7 @@ func TestAPassWritesNoWantWithoutAPassedPipeline(t *testing.T) { mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: mdigest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath}) putModuleRaw(p.w, trunkBranch, "web", mod) p.w.manifests[mdigest] = parsed{module: "web", manifest: catalogue.Manifest{Module: "web"}} - bkey, _ := builds.BuilderBuildKey("web", fp) - built, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: mdigest}) - p.w.put(buckets.Builds, bkey, built) + endDone(t, p.w, "web", fp, mdigest) gkey, _ := grantsstate.PartGrantKey("web", fp) grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{}) p.w.put(buckets.Grants, gkey, grant) @@ -169,6 +166,7 @@ func TestAPassWritesNoWantWithoutAPassedPipeline(t *testing.T) { p.w.put(buckets.Presence, "g14", on) if c.follow { p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf) + p.ends.trunk = p.w.trunkName p.ends.on.Store(true) if c.passed { _ = p.ends.take(pipe("web", trunkBranch, 5, 1, fp, mpr.PipelineEndedOutcomePassed), true) @@ -196,6 +194,7 @@ func TestTheWantsSwitchOnTheFirstRegistration(t *testing.T) { st := newMemEnds() woken := 0 e := newPipelineEnds(st, func() { woken++ }, t.Logf) + e.trunk = func() string { return trunkBranch } _ = e.take(pipe("web", "main", 5, 1, "fp1", mpr.PipelineEndedOutcomeFailed), false) _ = e.take(pipe("web", "beta", 5, 1, "fp1", mpr.PipelineEndedOutcomePassed), true) if e.following() || !e.mayWant("web", "fp1", []string{"main"}, nil) { @@ -225,6 +224,7 @@ func TestACommitOnBetaMovesOnlyTheNodeThatChoseBeta(t *testing.T) { p := newPasser(kvSet{buckets.Wants: wantsKV, buckets.Asks: &memKV{values: map[string][]byte{}}}, time.Now()) p.say = func(oc.Condition) error { return nil } p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf) + p.ends.trunk = p.w.trunkName p.ends.registered(mpcat.Registration{Module: "web", Branches: []string{"main", "beta"}}, false) digest := "sha256:" + strings.Repeat("c", 64) on, _ := presenceValue(true) @@ -251,9 +251,7 @@ func TestACommitOnBetaMovesOnlyTheNodeThatChoseBeta(t *testing.T) { mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: f, Manifest: mdigest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath}) putModuleRaw(p.w, branch, "web", mod) p.w.manifests[mdigest] = parsed{module: "web", manifest: catalogue.Manifest{Module: "web"}} - bkey, _ := builds.BuilderBuildKey("web", f) - built, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: mdigest}) - p.w.put(buckets.Builds, bkey, built) + endDone(t, p.w, "web", f, mdigest) gkey, _ := grantsstate.PartGrantKey("web", f) grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{}) p.w.put(buckets.Grants, gkey, grant) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go index dab41719..27ad5b4a 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go @@ -14,7 +14,6 @@ import ( "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient" mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient" "git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate" @@ -184,14 +183,8 @@ func (p *passer) putWorld(node, module string, put puts.Put, manifest catalogue. w.Assigned = true } } - if bkey, err := builds.BuilderBuildKey(module, put.FP); err == nil { - if raw := p.w.value(buckets.Builds, bkey); raw != nil { - if b, err := builds.DecodeBuilderBuild(raw); err == nil { - w.Build, w.BuildWords = b.State, b.Error - } else { - w.Build = "unreadable: " + err.Error() - } - } + if e, ok := p.w.ends.endOf(module, put.FP); ok { // the build's end heard (A-181): none heard, no state + w.Build, w.BuildWords = e.Outcome, e.Error } w.Composed, w.Granted = p.composedFor(node, module, put.FP) if d := w.Composed; d != nil { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go index 7664c4fb..7a5ad112 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go @@ -17,7 +17,6 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/catalogkey" stdio "git.novox.be/novox/mesh-sdk/go" @@ -72,14 +71,11 @@ func (s *putTools) hear(key string, value []byte) { s.mu.Unlock() } -// fullFP is a fingerprint of the new form, whole; prefix at least its first twelve hex digits. -var ( - fullFP = regexp.MustCompile(`^src2:[0-9a-f]{64}$`) - prefix = regexp.MustCompile(`^(src2:)?[0-9a-f]{12,63}$`) -) +// fullFP is a fingerprint of the new form, whole. +var fullFP = regexp.MustCompile(`^src2:[0-9a-f]{64}$`) -// version reads the version a put names: newest (the indexer's fingerprint at the trunk), a whole fingerprint, or the start -// of one the builder built, which must name one build alone. +// version reads the version a put names: newest (the indexer's fingerprint at the trunk) or a whole fingerprint (design 47 G A-181: +// a put names the whole fingerprint, the builder's builds being no bucket to list). func (s *putTools) version(module, version string) (string, error) { v := strings.ToLower(strings.TrimSpace(version)) switch { @@ -103,29 +99,8 @@ func (s *putTools) version(module, version string) (string, error) { return m.Fp2, nil case fullFP.MatchString(v): return v, nil - case prefix.MatchString(v): - start := strings.TrimPrefix(v, meshcatalogclient.SourcePrefix2) - keys, err := s.st.keys(builds.BuilderModule + ".builds") - if err != nil { - return "", fmt.Errorf("the builder's builds cannot be listed: %w", err) - } - var found []string - for _, k := range keys { - m, fp, err := builds.ParseBuilderBuildKey(k) - if err == nil && m == module && strings.HasPrefix(strings.TrimPrefix(fp, meshcatalogclient.SourcePrefix2), start) { - found = append(found, fp) - } - } - switch len(found) { - case 0: - return "", fmt.Errorf("the builder built no %s whose fingerprint starts %s: give it whole", module, start) - case 1: - return found[0], nil - } - sort.Strings(found) - return "", fmt.Errorf("%s starts %d fingerprints of %s the builder built (%s): give more of it", start, len(found), module, strings.Join(found, ", ")) } - return "", fmt.Errorf("%q is no version: newest, a fingerprint (src2: and 64 hex digits) or at least its first 12 hex digits", version) + return "", fmt.Errorf("%q is no version: newest, a fingerprint (src2: and 64 hex digits) whole", version) } // putAnswer is what assigner_put and assigner_puts answer of one put. diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/verbs.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/verbs.go index ec642a9c..35a3c35f 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/verbs.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/verbs.go @@ -1,7 +1,6 @@ package main import ( - "mesh-delivery-assigner/internal/buckets" "slices" "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" @@ -94,22 +93,14 @@ func (p *passer) servesOf(module, fp string) ([]string, bool) { return graph.ServesOf(m), true } -// manifestOf is the manifest of module's build at fp (the builder's build names its digest in the indexer's manifests), -// and whether it could be read. +// manifestOf is the manifest of module's build at fp (the build's end names its digest, A-181, which the indexer's manifests +// hold), and whether it could be read. func (p *passer) manifestOf(module, fp string) (catalogue.Manifest, bool) { - key, err := builds.BuilderBuildKey(module, fp) - if err != nil { + e, ok := p.w.ends.endOf(module, fp) + if !ok || e.Outcome != builds.BuildDone || e.Manifest == "" { return catalogue.Manifest{}, false } - raw := p.w.value(buckets.Builds, key) - if raw == nil { - return catalogue.Manifest{}, false - } - b, err := builds.DecodeBuilderBuild(raw) - if err != nil || b.Manifest == "" { - return catalogue.Manifest{}, false - } - got := p.w.manifest(b.Manifest) + got := p.w.manifest(e.Manifest) if got.err != "" { return catalogue.Manifest{}, false } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/views.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/views.go index f52e96e7..77d5144b 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/views.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/views.go @@ -136,7 +136,9 @@ func (iv *catalogViews) start(ctx context.Context, w *world, wake func(), settle reader.Findings = func() views.WireWriter { return iv.writer } } iv.h = &halves{w: w, call: iv.call, reader: reader, wake: wake, logf: iv.logf, ctx: ctx, settled: settled} - iv.h.follow(trunkBranch) + if err := iv.h.follow(); err != nil { + iv.logf("the catalog half is not followed, so no module is placed until the assigner starts again: %v", err) + } reader.Start(ctx, iv.js, iv.account, iv.status) return 1 } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go index e591751b..ce13650a 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go @@ -18,7 +18,6 @@ import ( mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk" "git.novox.be/novox/mesh-sdk/go/keys" mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient" @@ -48,9 +47,10 @@ type world struct { // assignments is the seat's state assignments assembled node by node from its keys in either form (mesh-sdk // mmac.Keys; research 072 row 37): the one reader of its keys in the assigner. assignmentKeys *mmac.Keys - // sourceGroups is, by revision of the gate's source.main, the release (group) that moved it, from every revision its - // history keeps (watched with history): a build's source revision names the release that carries it (ADR 0601 §1). - sourceGroups map[uint64]string + // first is the first watched branch the catalog half is followed of, the trunk's: the branch of the trunk-only want, read from the + // branches the registrations name, before the first registrations from those the gate's `source` answers (design 47 G §507, §532), + // and never fixed in code (§7). + first string // running is, by `.`, the last `running` the node-engine seat said of it, kept from its stream by the assigner's // durable consumer (design 47 G §199, §309); runningHeard says that consumer is bound. running map[string]nodeengine.Running @@ -77,7 +77,7 @@ type parsed struct { func newWorld() *world { return &world{values: map[string]map[string][]byte{}, manifests: map[string]parsed{}, healthy: map[string][]string{}, changes: map[string]uint64{}, meta: map[string]entryMeta{}, assignmentKeys: mmac.NewKeys(), - sourceGroups: map[uint64]string{}, running: map[string]nodeengine.Running{}, + running: map[string]nodeengine.Running{}, modules: map[string]map[string]meshcatalog.ModulesByModuleValue{}, graphs: map[string]map[string]graph.Node{}} } @@ -94,44 +94,6 @@ func (w *world) putEntry(bucket, key string, value []byte, revision uint64, at t if bucket == plc.AssignmentsBucket { _, _ = w.assignmentKeys.Put(key, value, revision, false) // put took it already; this is its revision } - if bucket == buckets.Source && key == mdc.SourceKey(trunkBranch) { - if s, err := mdc.DecodeSource(value); err == nil && s.Group != "" { - w.sourceGroups[revision] = s.Group - } - } -} - -// release is the release whose Approve carries module's build fp (ADR 0601 §1), and whether one does: the trunk's value of -// the module names fp, the source revision at which that fingerprint last changed (FPSrc, never the Src of its last merge, -// which a recompute with no merge keeps) names the release that moved it, and the gate's merge of that release, which it -// writes on the operator's Approve, holds the module's own trunk merge among its merged pull requests. Any of them -// unknown: none, and the move is asked of a person. -func (w *world) release(module, fp string) (string, bool) { - t, ok := w.trunk()[module] - if !ok || fp == "" || t.Fp2 != fp || t.FpSrc == 0 || t.Commit == "" { - return "", false - } - w.mu.Lock() - group := w.sourceGroups[uint64(t.FpSrc)] - w.mu.Unlock() - key, err := mdc.MergesKey(group) - if group == "" || err != nil { - return "", false - } - raw := w.value(buckets.Merges, key) - if raw == nil { - return "", false - } - merge, err := mdc.DecodeMerge(raw) - if err != nil { - return "", false - } - for _, pull := range merge.Pulls { - if pull.State == string(mdc.MergeMerged) && pull.Commit == t.Commit && trunk.SameRepository(pull.Repository, t.Repository) { - return group, true - } - } - return "", false } // metaOf is the revision and time of the value a bucket holds at key, as the watch said them. @@ -267,8 +229,8 @@ func (w *world) branches() []string { w.mu.Lock() defer w.mu.Unlock() out := slices.Sorted(maps.Keys(w.modules)) - if i := slices.Index(out, trunkBranch); i > 0 { - out = slices.Insert(slices.Delete(out, i, i+1), 0, trunkBranch) + if i := slices.Index(out, w.first); i > 0 { + out = slices.Insert(slices.Delete(out, i, i+1), 0, w.first) } return out } @@ -459,34 +421,28 @@ func (w *world) has(bucket, key string) bool { return ok } -// built says whether the builder's build of a module at a fingerprint is done. +// built says whether the builder's build of a module at a fingerprint is done, from its `build.ended` heard (design 47 G A-181). func (w *world) built(module, fp string) bool { - key, err := builds.BuilderBuildKey(module, fp) - if err != nil { - return false - } - w.mu.Lock() - value, ok := w.values[buckets.Builds][key] - w.mu.Unlock() - if !ok { - return false - } - b, err := builds.DecodeBuilderBuild(value) - return err == nil && b.State == builds.BuildDone + e, ok := w.ends.endOf(module, fp) + return ok && e.Outcome == builds.BuildDone } -// buildNeeds are the reference forms the builder's done build of module at fp is written in (the build key's needs, ADR -// 0618 §1): nil when it is not done or names none. +// buildStanding says whether the builder's `build.ended` of module at fp said the build failed, and its error whole (design 47 G +// §575, A-162): the words of a hold under rule 1. +func (w *world) buildStanding(module, fp string) (failed bool, words string) { + if e, ok := w.ends.endOf(module, fp); ok && e.Outcome == builds.BuildFailed { + return true, e.Error + } + return false, "" +} + +// buildNeeds are the reference forms the builder's done build of module at fp is written in (the build's end says them, ADR +// 0618 §1, A-181): nil when it is not done or names none. func (w *world) buildNeeds(module, fp string) []string { - key, err := builds.BuilderBuildKey(module, fp) - if err != nil { - return nil + if e, ok := w.ends.endOf(module, fp); ok && e.Outcome == builds.BuildDone { + return e.Needs } - b, err := builds.DecodeBuilderBuild(w.value(buckets.Builds, key)) - if err != nil || b.State != builds.BuildDone { - return nil - } - return b.Needs + return nil } // recordReads is what node's record `.node` states its engine reads (ADR 0614 §4), and whether the assigner holds @@ -582,8 +538,19 @@ func (w *world) unplacedNodes() []string { return out } -// trunkBranch is the branch of the trunk-only want, kept until the first registrations (design 47 G §95, §353). -const trunkBranch = "main" +// trunkName is the first watched branch, the trunk's (design 47 G §507, §353): "" until the first view of the catalog half is adopted. +func (w *world) trunkName() string { + w.mu.Lock() + defer w.mu.Unlock() + return w.first +} + +// setTrunk says which watched branch is the first. +func (w *world) setTrunk(branch string) { + w.mu.Lock() + defer w.mu.Unlock() + w.first = branch +} // placedBranches is, by module and node, the watched branch each placement names ("" when it names none). func (w *world) placedBranches() map[string]map[string]string { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world_test.go index 7cf9ea22..bc013a53 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world_test.go @@ -3,6 +3,8 @@ package main import ( "context" "mesh-delivery-assigner/internal/buckets" + "os" + "path/filepath" "strings" "testing" "time" @@ -81,3 +83,32 @@ func putModuleRaw(w *world, branch, module string, raw []byte) { } w.putModule(branch, module, v, graph.Node{FP: v.Fp2, Repository: v.Repository}, nil) } + +// Design 47 G §507, §532, §7: the first watched branch the catalog half is followed of is the trunk's, whatever its name, and no part of the +// assigner's code names a branch: the world's trunk comes first, and no non-test source holds the string "main". +func TestTheTrunkIsTheFirstWatchedBranchAndNoCodeNamesOne(t *testing.T) { + w := newWorld() + w.setTrunk("develop") + for _, b := range []string{"beta", "develop", "alpha"} { + w.putModule(b, "web", meshcatalog.ModulesByModuleValue{Fp2: "fp"}, graph.Node{FP: "fp"}, nil) + } + if got := w.branches(); strings.Join(got, ",") != "develop,alpha,beta" { + t.Errorf("branches = %v, want the first watched first, then by name", got) + } + files, err := filepath.Glob("*.go") + if err != nil { + t.Fatal(err) + } + for _, f := range files { + if strings.HasSuffix(f, "_test.go") { + continue + } + src, err := os.ReadFile(f) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(src), `"main"`) { + t.Errorf("%s names the branch main in code (design 47 G §507)", f) + } + } +} diff --git a/modules/mesh-delivery-assigner/internal/assign/wants.go b/modules/mesh-delivery-assigner/internal/assign/wants.go index d3cb0742..058770c8 100644 --- a/modules/mesh-delivery-assigner/internal/assign/wants.go +++ b/modules/mesh-delivery-assigner/internal/assign/wants.go @@ -26,11 +26,15 @@ import ( // trunk (the indexer's modules), whether the builder's build of F is done, whether F is held, whether the controller // issued its credential for F, its manifest at F, and the nodes it is assigned to. type Module struct { - Name string - FP string - Built bool - Held bool - Granted bool + Name string + FP string + Built bool + // BuildFailed and BuildError say a build that ended failed, with the error as the builder's `build.ended` carries it whole + // (design 47 G §575, A-162): the words of a hold under rule 1. + BuildFailed bool + BuildError string + Held bool + Granted bool // GrantedAt says whether the controller issued the module's credential for another fingerprint than FP: the first want of a node // that runs an older one names it (issue 1945). Nil: none. GrantedAt func(fp string) bool @@ -69,10 +73,6 @@ type Module struct { // Allowed says whether the operator answered yes to the record ask about the move of this module on node to the composition // with this digest (ADR 0539 §14). Nil: none did. Allowed func(node, digest string) bool - // Release names the release whose Approve carries the build fp, as mesh-delivery's gate holds it for the release's heads, - // read by the build's source revision (ADR 0601 §1): that Approve is the yes to a move of the build alone. False when no - // approved release is known to carry it. Nil: none is read. - Release func(fp string) (string, bool) // Released is a hold on FP the operator released (ADR 0539 §19): FP is not held again by a health check that failed. Released bool // Restored says whether the operator answered "restore" to the restore ask of this data module on node: it is then set back @@ -159,9 +159,6 @@ func healthy(h nodeengine.HaveByNodeModuleValue, fp string) bool { return h.Fp == fp && h.Phase == nodeengineclient.PhaseRunning && h.Health == nodeengineclient.HealthOK } -// Healthy says whether a have runs fp healthy. -func Healthy(h nodeengine.HaveByNodeModuleValue, fp string) bool { return healthy(h, fp) } - // Failed says whether a have's health check failed once the node is done moving. func Failed(h nodeengine.HaveByNodeModuleValue) bool { return failed(h) } @@ -191,7 +188,10 @@ func Holds(m Module, node string, n Nodes) (bool, int, string) { func holds(m Module, node string, n Nodes, ordering bool) (bool, int, string) { switch { case !m.Built: - return false, mwc.RuleBuildDone, fmt.Sprintf("the build of %s at %s is not done", m.Name, short(m.FP)) + if m.BuildFailed { + return false, mwc.RuleBuildDone, fmt.Sprintf("the build of %s at %s failed: %s", m.Name, short(m.FP), m.BuildError) + } + return false, mwc.RuleBuildDone, fmt.Sprintf("the build of %s at %s is being built", m.Name, short(m.FP)) case m.Held: return false, mwc.RuleNotHeld, fmt.Sprintf("%s is held at %s", m.Name, short(m.FP)) case m.MachineHeld != nil && m.MachineHeld(node): @@ -202,7 +202,7 @@ func holds(m Module, node string, n Nodes, ordering bool) (bool, int, string) { switch { case canary == "": return false, mwc.RuleCanary, fmt.Sprintf("none of the nodes of %s is online, so it has no canary", m.Name) - case node != canary && !healthy(n.Haves[canary][m.Name], m.FP): + case node != canary && !n.RunsHealthy(canary, m.Name, m.FP): return false, mwc.RuleCanary, fmt.Sprintf("the canary %s does not run %s healthy yet", canary, short(m.FP)) } } @@ -322,15 +322,19 @@ func (m Module) MissingProvisions(node string) []string { return out } -// runsHealthy says whether module runs healthy on node: from G the node-engine seat's last `running` of it names a build and -// does not say it stopped; with the seat's events not heard, the have's phase and health. +// runsHealthy says whether module runs healthy on node: the node-engine seat's last `running` of it names a build, does not say it +// stopped and does not say `healthy` false (design 47 G §520); nothing heard of it counts as not healthy. func (n Nodes) runsHealthy(node, module string) bool { - if n.Running != nil { - r, ok := n.Running[node][module] - return ok && !r.Stopped && r.Fingerprint != "" - } - h, ok := n.Haves[node][module] - return ok && h.Phase == nodeengineclient.PhaseRunning && h.Health == nodeengineclient.HealthOK + r, ok := n.Running[node][module] + return ok && !r.Stopped && r.Fingerprint != "" && (r.Healthy == nil || *r.Healthy) +} + +// RunsHealthy says whether the build of module at fp runs healthy on node, from the node-engine seat's last `running` of it alone +// (design 47 G §520): a `running` with no `healthy`, said by a node-engine of before G, reads as healthy while it stands, and a +// module of which nothing was heard counts as not healthy. The have is never read for it. +func (n Nodes) RunsHealthy(node, module, fp string) bool { + r, ok := n.Running[node][module] + return ok && n.runsHealthy(node, module) && r.Fingerprint == fp } // providedHealthy is rule 5 for one provision: some module providing it runs healthy on the node, or, assigned @@ -702,7 +706,7 @@ func healthyElsewhere(m Module, n Nodes, node string) []string { for _, other := range slices.Sorted(slices.Values(m.Nodes)) { // A witness is online: a node that sleeps keeps the have it last wrote, and a stale have is no evidence that F runs // healthy now. - if other != node && n.Online[other] && healthy(n.Haves[other][m.Name], m.FP) { + if other != node && n.Online[other] && n.RunsHealthy(other, m.Name, m.FP) { out = append(out, other) } } @@ -750,7 +754,7 @@ func allRunHealthy(m Module, n Nodes) bool { if !ok || (h.Phase != nodeengineclient.PhaseRunning && h.Phase != nodeengineclient.PhaseFailed) { continue } - if !healthy(h, m.FP) { + if !n.RunsHealthy(node, m.Name, m.FP) { return false } ran++ diff --git a/modules/mesh-delivery-assigner/internal/assign/wants_test.go b/modules/mesh-delivery-assigner/internal/assign/wants_test.go index d0f3be9f..28b7f3a0 100644 --- a/modules/mesh-delivery-assigner/internal/assign/wants_test.go +++ b/modules/mesh-delivery-assigner/internal/assign/wants_test.go @@ -60,7 +60,7 @@ func wantedOld() map[string]mwc.Want { // g14 follows at once: step 0, ADR 0528 clause 3). func TestEveryRuleHoldsSoTheWantsMove(t *testing.T) { m, n := ready() - out := Wants(m, n, wantedOld(), nil) + out := Wants(m, heardOfHaves(n), wantedOld(), nil) for _, node := range []string{"ace", "g14"} { if out.Wants[node].Fp != fNew || out.Wants[node].Rule != 0 || out.Wants[node].Composed != dig(fNew) { t.Errorf("%s's want = %+v, want %s", node, out.Wants[node], fNew) @@ -89,7 +89,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) { } { m, n := ready() breakIt(&m, &n) - out := Wants(m, n, wantedOld(), nil) + out := Wants(m, heardOfHaves(n), wantedOld(), nil) if w, ok := out.Wants["g14"]; ok && w.Fp == fNew { t.Errorf("rule %d broken: g14's want moved to F", rule) } @@ -110,7 +110,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) { }, nil} { m, n := ready() m.Provisions = all - if out := Wants(m, n, wantedOld(), nil); out.Wants["g14"].Fp != fNew { + if out := Wants(m, heardOfHaves(n), wantedOld(), nil); out.Wants["g14"].Fp != fNew { t.Errorf("provided provisions held g14's want: %+v (%s)", out.Wants["g14"], out.Rules["g14"].Words) } } @@ -161,7 +161,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) { m, n := ready() m.Name, m.Needs, n.Reads, m.FormNeeds = c.name, c.needs, c.reads, c.forms n.Haves["ace"][c.name], n.Haves["g14"][c.name] = n.Haves["ace"]["mail"], n.Haves["g14"]["mail"] - out := Wants(m, n, wantedOld(), nil) + out := Wants(m, heardOfHaves(n), wantedOld(), nil) if moved := out.Wants["g14"].Fp == fNew; moved == c.held { t.Errorf("%s: g14's want %+v, held by %+v", c.why, out.Wants["g14"], out.Rules["g14"]) } @@ -208,7 +208,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) { if c.holds == graph.BusState { words = graph.BusStateWaitWords("mail", "g14") } - out := Wants(m, n, wantedOld(), nil) + out := Wants(m, heardOfHaves(n), wantedOld(), nil) if c.holds == "" { if out.Wants["g14"].Fp != fNew { t.Errorf("%s: held %+v", c.name, out.Rules["g14"]) @@ -235,7 +235,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) { return map[string]meshprovisionerclient.Provision{node + ".mail.oidc": {Provision: "oidc", State: c.state}, node + ".mail.database": {Provision: "database", State: c.older, Why: "the operator's answer"}} } - out := Wants(m, n, wantedOld(), nil) + out := Wants(m, heardOfHaves(n), wantedOld(), nil) if got := out.Wants["g14"].Fp == fNew; got != c.moves || (!c.moves && (!strings.Contains(out.Rules["g14"].Words, "oidc") || strings.Contains(out.Rules["g14"].Words, "database"))) { t.Errorf("%s: want %+v, held by %q", c.name, out.Wants["g14"], out.Rules["g14"].Words) } @@ -280,7 +280,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) { m.Provisions = func(string) map[string]meshprovisionerclient.Provision { return map[string]meshprovisionerclient.Provision{} } - out := Wants(m, n, wantedOld(), nil) + out := Wants(m, heardOfHaves(n), wantedOld(), nil) if c.missing == "" { if out.Wants["g14"].Fp != fNew || len(m.MissingProvisions("g14")) != 0 { t.Errorf("%s: held %+v", c.name, out.Rules["g14"]) @@ -307,7 +307,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) { m.Provisions = func(string) map[string]meshprovisionerclient.Provision { return map[string]meshprovisionerclient.Provision{} } - out := Wants(m, n, wantedOld(), nil) + out := Wants(m, heardOfHaves(n), wantedOld(), nil) if got := out.Wants["g14"].Fp == fNew; got != c.offered || (!c.offered && (out.Rules["g14"].Rule != mwc.RuleProvisions || !strings.Contains(out.Rules["g14"].Words, "no module offers database yet") || !strings.Contains(out.Rules["g14"].Words, "past its bound"))) { t.Errorf("%s: want %+v, held by %+v", c.name, out.Wants["g14"], out.Rules["g14"]) @@ -342,7 +342,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) { if c.have != nil { n.Haves["novox"][c.module] = *c.have } - out := Wants(m, n, c.before, nil) + out := Wants(m, heardOfHaves(n), c.before, nil) w, written := out.Wants["novox"] if !written { w = c.before["novox"] @@ -431,7 +431,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) { if c.name == "placed" { before["g14"] = mwc.Want{Fp: fNew, Composed: dig(fNew)} } - out := Wants(m, n, before, nil) + out := Wants(m, heardOfHaves(n), before, nil) if got := out.Wants["g14"]; !reflect.DeepEqual(got, c.want) { t.Errorf("%s: g14's want = %+v, want %+v", c.name, got, c.want) } @@ -453,7 +453,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) { m, n := ready() grant := grantsstate.Grant{Accounts: []string{"ace.mail", "ace.mail.send"}} m.GrantedOn = func(node string) bool { return GrantNamesNode(grant, node, "mail") } - out := Wants(m, n, wantedOld(), nil) + out := Wants(m, heardOfHaves(n), wantedOld(), nil) if w := out.Wants["ace"]; w.Fp != fNew { t.Errorf("ace, whose account the grant names, wants %+v", w) } @@ -486,7 +486,7 @@ func TestAnOfflineCanaryIsPassedOver(t *testing.T) { if c := Canary(m.Nodes, n.Online); c != "g14" { t.Fatalf("canary = %q, want g14", c) } - out := Wants(m, n, wantedOld(), nil) + out := Wants(m, heardOfHaves(n), wantedOld(), nil) if out.Wants["g14"].Fp != fNew { t.Errorf("the canary g14's want = %+v", out.Wants["g14"]) } @@ -500,7 +500,7 @@ func TestAFailedHealthHoldsAndSetsBack(t *testing.T) { m, n := ready() n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed, Detail: "connection refused"} m.Now = time.Date(2026, 10, 25, 1, 0, 0, 0, time.UTC) - out := Wants(m, n, map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew)}, "g14": {Fp: fOld, Composed: dig(fOld)}}, map[string]string{"ace": fOld}) + out := Wants(m, heardOfHaves(n), map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew)}, "g14": {Fp: fOld, Composed: dig(fOld)}}, map[string]string{"ace": fOld}) // 1906/H3: the held record keeps its cause in words: which machine (its node), which health, when, and the node's words. if out.Held == nil || out.Held.Node != "ace" || out.Held.Words != "health failed, phase running, seen at 2026-10-25T01:00:00Z: connection refused" { t.Fatalf("held = %+v", out.Held) @@ -516,7 +516,7 @@ func TestAFailedHealthHoldsAndSetsBack(t *testing.T) { // back, the failing node keeps its want, and the outcome names it (on ace a newly assigned module's first check held a version healthy for an hour on three nodes). n.Haves["g14"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} before := map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew)}, "g14": {Fp: fNew, Composed: dig(fNew)}} - out = Wants(m, n, before, map[string]string{"ace": fOld, "g14": fOld}) + out = Wants(m, heardOfHaves(n), before, map[string]string{"ace": fOld, "g14": fOld}) if out.Held != nil || len(out.Wants) != 0 { t.Fatalf("held %+v, wants %+v: want no hold and no want changed", out.Held, out.Wants) } @@ -534,7 +534,7 @@ func TestAFailedHealthHoldsAndSetsBack(t *testing.T) { n.Online["novox"] = true n.Haves["novox"] = map[string]nodeengine.HaveByNodeModuleValue{"mail": {Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}} before["novox"] = mwc.Want{Fp: fOld, Composed: dig(fOld)} - out = Wants(three, n, before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld}) + out = Wants(three, heardOfHaves(n), before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld}) if _, alone := out.NodeFailed["g14"]; !alone { t.Errorf("g14 does not fail alone: %+v", out.NodeFailed) } @@ -547,7 +547,7 @@ func TestAFailedHealthHoldsAndSetsBack(t *testing.T) { // A node that runs F healthy with no want, while g14 fails it alone, gets its want at F and is on no neither list (issue 1945). n.Haves["novox"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} delete(before, "novox") - out = Wants(three, n, before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld}) + out = Wants(three, heardOfHaves(n), before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld}) if w := out.Wants["novox"]; w.Fp != fNew || len(out.Neither) != 0 { t.Errorf("novox running F with no want: want %+v, neither %+v; want its want at F and no neither", w, out.Neither) } @@ -569,7 +569,7 @@ func TestAFailedHealthHoldsAndSetsBack(t *testing.T) { n.Haves["novox"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} } before["novox"] = mwc.Want{Fp: fNew, Composed: dig(fNew)} - out = Wants(three, n, before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld}) + out = Wants(three, heardOfHaves(n), before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld}) if out.Held == nil || out.Held.Node != "ace" || len(out.NodeFailed) != 0 { t.Errorf("%s: held %+v, node failed %+v: want F held at ace", c.name, out.Held, out.NodeFailed) } @@ -584,7 +584,7 @@ func TestADataModuleIsNotSetBack(t *testing.T) { m, n := ready() m.Manifest.Data = &catalogue.Data{Own: []catalogue.DataItem{{ID: "mail", Class: catalogue.ClassIrreplaceable}}} n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed} - out := Wants(m, n, map[string]mwc.Want{"ace": {Fp: fNew}}, map[string]string{"ace": fOld}) + out := Wants(m, heardOfHaves(n), map[string]mwc.Want{"ace": {Fp: fNew}}, map[string]string{"ace": fOld}) if out.Held == nil { t.Fatal("F was not held") } @@ -601,7 +601,7 @@ func TestADataModuleIsNotSetBack(t *testing.T) { func TestAModuleWithoutDataIsNotAskedToRestore(t *testing.T) { m, n := ready() n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed} - out := Wants(m, n, map[string]mwc.Want{"ace": {Fp: fNew}}, map[string]string{"ace": fOld}) + out := Wants(m, heardOfHaves(n), map[string]mwc.Want{"ace": {Fp: fNew}}, map[string]string{"ace": fOld}) if len(out.Restore) != 0 { t.Errorf("restore asks = %v", out.Restore) } @@ -612,7 +612,7 @@ func TestAModuleWithoutDataIsNotAskedToRestore(t *testing.T) { func TestALeavingNodeIsRemovedAndNothingIsWrittenTwice(t *testing.T) { m, n := ready() m.Nodes = []string{"ace"} - out := Wants(m, n, map[string]mwc.Want{"ace": {V: 1, Fp: fNew, Composed: dig(fNew)}, "g14": {Fp: fOld, Composed: dig(fOld)}}, nil) + out := Wants(m, heardOfHaves(n), map[string]mwc.Want{"ace": {V: 1, Fp: fNew, Composed: dig(fNew)}, "g14": {Fp: fOld, Composed: dig(fOld)}}, nil) if w := out.Wants["g14"]; !w.Removed { t.Errorf("g14's want = %+v, want removed", w) } @@ -632,7 +632,7 @@ func TestAWantWaitsForItsCompositionAndNamesItsDigest(t *testing.T) { } return composedEverywhere(node, fp) } - out := Wants(m, n, wantedOld(), nil) + out := Wants(m, heardOfHaves(n), wantedOld(), nil) for node, w := range out.Wants { if w.Fp == fNew { t.Errorf("%s's want moved to F before its composition exists: %+v", node, w) @@ -642,11 +642,11 @@ func TestAWantWaitsForItsCompositionAndNamesItsDigest(t *testing.T) { t.Errorf("ace is held back by %+v, want rule 7 naming the node", r) } // and a node that never had a want gets none - if out := Wants(m, n, nil, nil); len(out.Wants) != 0 { + if out := Wants(m, heardOfHaves(n), nil, nil); len(out.Wants) != 0 { t.Errorf("wants without a composition: %+v", out.Wants) } have["ace|"+fNew] = true - out = Wants(m, n, wantedOld(), nil) + out = Wants(m, heardOfHaves(n), wantedOld(), nil) if w := out.Wants["ace"]; w.Fp != fNew || w.Composed != dig(fNew) { t.Errorf("ace's want = %+v, want F naming the composition's digest", w) } @@ -664,14 +664,14 @@ func TestARevisionAtTheSameFingerprintIsAWantByTheSameRules(t *testing.T) { return ComposedDigest{Digest: revised, Judgement: JudgedNeither}, true } running := map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew)}, "g14": {Fp: fNew, Composed: dig(fNew)}} - out := Wants(m, n, running, nil) + out := Wants(m, heardOfHaves(n), running, nil) for _, node := range []string{"ace", "g14"} { if w := out.Wants[node]; w.Fp != fNew || w.Composed != revised { t.Errorf("%s's want = %+v, want F naming the revised digest", node, w) } } m.Granted = false - out = Wants(m, n, running, nil) + out = Wants(m, heardOfHaves(n), running, nil) if w, ok := out.Wants["ace"]; ok && w.Composed == revised { t.Errorf("a revision was written while a rule holds: %+v", w) } @@ -679,7 +679,7 @@ func TestARevisionAtTheSameFingerprintIsAWantByTheSameRules(t *testing.T) { // A want held back at an earlier fingerprint follows the new revision of that fingerprint's kept composition, which the // controller composed again with today's config (novox/hq ADR 0545: on g14 a capacity of 0 after a rollback), while // the rules still hold F; for a record module whose revision recreates a container, only after the ask. - out = Wants(m, n, wantedOld(), nil) + out = Wants(m, heardOfHaves(n), wantedOld(), nil) if w := out.Wants["g14"]; w.Fp != fOld || w.Composed != revised || w.Newest != fNew { t.Errorf("g14, held back at its earlier fingerprint, wants %+v; want it at that fingerprint naming the revised digest", w) } @@ -687,7 +687,7 @@ func TestARevisionAtTheSameFingerprintIsAWantByTheSameRules(t *testing.T) { return ComposedDigest{Digest: revised, Judgement: JudgedRecreates, Policy: catalogue.PolicyRecord, Against: dig(fOld), Words: "recreates"}, true } - out = Wants(m, n, wantedOld(), nil) + out = Wants(m, heardOfHaves(n), wantedOld(), nil) if w := out.Wants["g14"]; w.Composed == revised { t.Errorf("a recorded module's revision that recreates a container was written with no ask: %+v", w) } @@ -707,12 +707,12 @@ func TestASetBackNamesTheOlderCompositionOrStays(t *testing.T) { } return composedEverywhere(node, fp) } - out := Wants(m, n, running, map[string]string{"ace": fOld}) + out := Wants(m, heardOfHaves(n), running, map[string]string{"ace": fOld}) if w := out.Wants["ace"]; w.Fp == fOld { t.Errorf("set back to a fingerprint whose composition is gone: %+v", w) } m.Composed = composedEverywhere - out = Wants(m, n, running, map[string]string{"ace": fOld}) + out = Wants(m, heardOfHaves(n), running, map[string]string{"ace": fOld}) if w := out.Wants["ace"]; w.Fp != fOld || w.Composed != dig(fOld) { t.Errorf("set-back want = %+v", w) } @@ -741,7 +741,7 @@ func TestARecordModuleWaitsForItsAskWhenTheMoveIsNotJudgedNeither(t *testing.T) for _, j := range []Judgement{JudgedRecreates, JudgedMovesData, JudgedCannot} { m, n, wanted := movingTo(j) recording(&m) - out := Wants(m, n, wanted, nil) + out := Wants(m, heardOfHaves(n), wanted, nil) if w, ok := out.Wants["ace"]; ok { t.Errorf("%s: ace's want was written without the ask's yes: %+v", j, w) } @@ -756,11 +756,11 @@ func TestARecordModuleWaitsForItsAskWhenTheMoveIsNotJudgedNeither(t *testing.T) } // a new revision at the same fingerprint is asked too same := map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fOld)}} - if out := Wants(m, n, same, nil); len(out.Record) == 0 || len(out.Wants) != 0 && out.Wants["ace"].Composed == dig(fNew) { + if out := Wants(m, heardOfHaves(n), same, nil); len(out.Record) == 0 || len(out.Wants) != 0 && out.Wants["ace"].Composed == dig(fNew) { t.Errorf("%s: a new revision at the same fingerprint: %+v", j, out) } m.Allowed = func(node, digest string) bool { return node == "ace" && digest == dig(fNew) } - out = Wants(m, n, wanted, nil) + out = Wants(m, heardOfHaves(n), wanted, nil) if w := out.Wants["ace"]; w.Fp != fNew || w.Composed != dig(fNew) { t.Errorf("%s: after the yes ace's want = %+v", j, w) } @@ -772,25 +772,25 @@ func TestARecordModuleWaitsForItsAskWhenTheMoveIsNotJudgedNeither(t *testing.T) func TestAMoveJudgedNeitherOrNeverRunOrNotRecordedIsWrittenAtOnce(t *testing.T) { m, n, wanted := movingTo(JudgedNeither) recording(&m) - if out := Wants(m, n, wanted, nil); out.Wants["ace"].Fp != fNew || len(out.Record) != 0 { + if out := Wants(m, heardOfHaves(n), wanted, nil); out.Wants["ace"].Fp != fNew || len(out.Record) != 0 { t.Errorf("neither: %+v", out) } m, n, _ = movingTo(JudgedRecreates) recording(&m) m.Nodes = []string{"ace"} // g14 runs it with no want, so its first want is asked (1945/AC19): this case is ace's alone delete(n.Haves["ace"], "mail") - if out := Wants(m, n, nil, nil); out.Wants["ace"].Fp != fNew || len(out.Record) != 0 { + if out := Wants(m, heardOfHaves(n), nil, nil); out.Wants["ace"].Fp != fNew || len(out.Record) != 0 { t.Errorf("a node that never ran it: %+v", out) } // but one that runs an older fingerprint with no want yet is asked like any other (one running F is not: N1 below) m, n, _ = movingTo(JudgedCannot) recording(&m) n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} - if out := Wants(m, n, nil, nil); out.Wants["ace"].Fp == fNew || len(out.Record) == 0 { + if out := Wants(m, heardOfHaves(n), nil, nil); out.Wants["ace"].Fp == fNew || len(out.Record) == 0 { t.Errorf("a node that runs it with no want: %+v", out) } m, n, wanted = movingTo(JudgedRecreates) - if out := Wants(m, n, wanted, nil); out.Wants["ace"].Fp != fNew || len(out.Record) != 0 { + if out := Wants(m, heardOfHaves(n), wanted, nil); out.Wants["ace"].Fp != fNew || len(out.Record) != 0 { t.Errorf("a rolled module: %+v", out) } } @@ -802,11 +802,11 @@ func TestTheBusMovesByTheRulesAndItsUserListRevisionIsWrittenAtOnce(t *testing.T m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}} // ADR 0601 §1, §2: the bus module's want is written by the rules, as any module's, and no snapshot is asked for it: its node // backs its streams up before it applies. - if out := Wants(m, n, wantedOld(), nil); out.Wants["ace"].Fp != fNew || len(out.Record) != 0 || len(out.Neither) != 0 { + if out := Wants(m, heardOfHaves(n), wantedOld(), nil); out.Wants["ace"].Fp != fNew || len(out.Record) != 0 || len(out.Neither) != 0 { t.Errorf("the bus did not move by the rules: %+v", out) } running := map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fOld)}, "g14": {Fp: fNew, Composed: dig(fOld)}} - out := Wants(m, n, running, nil) + out := Wants(m, heardOfHaves(n), running, nil) if w := out.Wants["ace"]; w.Fp != fNew || w.Composed != dig(fNew) { t.Errorf("a user-list revision of the bus: ace's want = %+v", w) } @@ -835,7 +835,7 @@ func TestTheBusMovesByTheRulesAndItsUserListRevisionIsWrittenAtOnce(t *testing.T if c.step == "" { m.Puts = nil } - out := Wants(m, n, c.wanted, nil) + out := Wants(m, heardOfHaves(n), c.wanted, nil) got, wrote := out.Wants["ace"] switch { case c.want == nil && wrote: @@ -852,7 +852,7 @@ func TestTheBusMovesByTheRulesAndItsUserListRevisionIsWrittenAtOnce(t *testing.T held := put held.Step = puts.Holding m.Puts = map[string]puts.Put{"ace": held} - if w := Wants(m, n, map[string]mwc.Want{"ace": PutWant(held)}, nil).Wants["ace"]; w.Composed != userList || !w.Put || !w.Hold { + if w := Wants(m, heardOfHaves(n), map[string]mwc.Want{"ace": PutWant(held)}, nil).Wants["ace"]; w.Composed != userList || !w.Put || !w.Hold { t.Errorf("a held bus did not take its user-list revision as a put: %+v", w) } m.Puts = nil @@ -983,14 +983,14 @@ func stepsOfAPut(t *testing.T) { func TestTheBackupAnswerStaysInTheWantWhileItsTargetStays(t *testing.T) { m, n := ready() running := map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew), Backup: mwc.BackupSkip, Words: BackupAnswerWords("skip", 0)}} - if w, ok := Wants(m, n, running, nil).Wants["ace"]; ok { + if w, ok := Wants(m, heardOfHaves(n), running, nil).Wants["ace"]; ok { t.Errorf("the answer was erased: %+v", w) } // A placed node composes its own: its want names the build alone, and the answer stays in it all the same (the digest its // have names is not in the want, so a want kept for its digest would erase the answer at every pass). m.Self = func(string) bool { return true } placed := map[string]mwc.Want{"ace": {Fp: fNew, Self: true, Backup: mwc.BackupSkip, Words: BackupAnswerWords("skip", 0)}} - if w, ok := Wants(m, n, placed, nil).Wants["ace"]; ok { + if w, ok := Wants(m, heardOfHaves(n), placed, nil).Wants["ace"]; ok { t.Errorf("the answer in a placed node's want was erased: %+v", w) } } @@ -1001,7 +1001,7 @@ func TestAReleasedHoldIsNotHeldAgain(t *testing.T) { m, n := ready() n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed, Detail: "x"} m.Released = true - out := Wants(m, n, map[string]mwc.Want{"ace": {Fp: fOld, Composed: dig(fOld), Newest: fNew, Rule: mwc.RuleNotHeld}}, map[string]string{"ace": fOld}) + out := Wants(m, heardOfHaves(n), map[string]mwc.Want{"ace": {Fp: fOld, Composed: dig(fOld), Newest: fNew, Rule: mwc.RuleNotHeld}}, map[string]string{"ace": fOld}) if out.Held != nil { t.Errorf("a released fingerprint was held again: %+v", out.Held) } @@ -1018,11 +1018,11 @@ func TestARestoreAnswerSetsADataModuleBack(t *testing.T) { n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed} running := map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew)}} m.Restored = func(node string) bool { return false } - if w, ok := Wants(m, n, running, map[string]string{"ace": fOld}).Wants["ace"]; ok && w.Fp == fOld { + if w, ok := Wants(m, heardOfHaves(n), running, map[string]string{"ace": fOld}).Wants["ace"]; ok && w.Fp == fOld { t.Errorf("set back without the operator's restore: %+v", w) } m.Restored = func(node string) bool { return node == "ace" } - if w := Wants(m, n, running, map[string]string{"ace": fOld}).Wants["ace"]; w.Fp != fOld || w.Composed != dig(fOld) { + if w := Wants(m, heardOfHaves(n), running, map[string]string{"ace": fOld}).Wants["ace"]; w.Fp != fOld || w.Composed != dig(fOld) { t.Errorf("after restore ace's want = %+v", w) } } @@ -1040,22 +1040,22 @@ func TestTheRecordPolicyIsTheOneTheControllerHolds(t *testing.T) { } } with(catalogue.PolicyRecord, "person") // a roll module the operator set to record - if out := Wants(m, n, wanted, nil); len(out.Record) == 0 || out.Wants["ace"].Fp == fNew { + if out := Wants(m, heardOfHaves(n), wanted, nil); len(out.Record) == 0 || out.Wants["ace"].Fp == fNew { t.Errorf("the operator's record policy was not honoured: %+v", out) } recording(&m) // the manifest says record, the operator chose roll with("roll", "person") - if out := Wants(m, n, wanted, nil); len(out.Record) != 0 || out.Wants["ace"].Fp != fNew { + if out := Wants(m, heardOfHaves(n), wanted, nil); len(out.Record) != 0 || out.Wants["ace"].Fp != fNew { t.Errorf("the operator's roll policy was not honoured: %+v", out) } with("", "") // made by hand: the manifest decides - if out := Wants(m, n, wanted, nil); len(out.Record) == 0 { + if out := Wants(m, heardOfHaves(n), wanted, nil); len(out.Record) == 0 { t.Errorf("a hand-made composition fell through the manifest's record policy: %+v", out) } // 1953/D4 (ADR 0565 §7): a person's choice at the assigner is over the controller's beside the composition. with(catalogue.PolicyRecord, "person") m.Chosen = &catalogue.PersonPolicy{V: catalogue.PersonPolicyVersion, Module: m.Name, Policy: catalogue.PolicyRoll, Why: "w", By: "op"} - if out := Wants(m, n, wanted, nil); len(out.Record) != 0 || out.Wants["ace"].Fp != fNew { + if out := Wants(m, heardOfHaves(n), wanted, nil); len(out.Record) != 0 || out.Wants["ace"].Fp != fNew { t.Errorf("a person's roll at the assigner was not honoured over the controller's record: %+v", out) } } @@ -1066,11 +1066,11 @@ func TestTheBusWithNoWantIsComparedWithWhatItRuns(t *testing.T) { m, n, _ := movingTo(JudgedNeither) m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}} n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} - if w, ok := Wants(m, n, nil, nil).Wants["ace"]; !ok || w.Fp != fNew { + if w, ok := Wants(m, heardOfHaves(n), nil, nil).Wants["ace"]; !ok || w.Fp != fNew { t.Errorf("the bus runs F with no want and was not given one: %+v %v", w, ok) } n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} - if w, ok := Wants(m, n, nil, nil).Wants["ace"]; !ok || w.Fp != fNew { + if w, ok := Wants(m, heardOfHaves(n), nil, nil).Wants["ace"]; !ok || w.Fp != fNew { t.Errorf("the bus running an older fingerprint with no want was not given its want by the rules: %+v %v", w, ok) } } @@ -1096,7 +1096,7 @@ func TestAUserListOnlyRevisionOfTheBusIsWrittenAtOnceBeforeTheRecordHold(t *test "a want at the fingerprint it runs": {"ace": {Fp: fOld, Composed: dig(fOld)}}, "no readable want": nil, } { - out := Wants(m, n, wanted, nil) + out := Wants(m, heardOfHaves(n), wanted, nil) if w := out.Wants["ace"]; w.Fp != fOld || w.Composed != userList { t.Errorf("%s: ace's want = %+v, want the running fingerprint naming the user-list revision", name, w) } @@ -1106,7 +1106,7 @@ func TestAUserListOnlyRevisionOfTheBusIsWrittenAtOnceBeforeTheRecordHold(t *test } // the same flag on a module that is not the bus changes nothing m.Manifest.Provides = nil - if out := Wants(m, n, map[string]mwc.Want{"ace": {Fp: fOld, Composed: dig(fOld)}}, nil); out.Wants["ace"].Composed == userList { + if out := Wants(m, heardOfHaves(n), map[string]mwc.Want{"ace": {Fp: fOld, Composed: dig(fOld)}}, nil); out.Wants["ace"].Composed == userList { t.Errorf("a module that is not the bus was written at once: %+v", out.Wants["ace"]) } } @@ -1129,7 +1129,7 @@ func TestAHaveFromBeforeTheSwitchGetsItsWantWithNoRecordHold(t *testing.T) { } m.Composed = unchanged n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} // no composition named - out := Wants(m, n, nil, nil) + out := Wants(m, heardOfHaves(n), nil, nil) if w := out.Wants["ace"]; w.Fp != fNew || w.Composed != dig(fNew) { t.Errorf("%s: ace's want = %+v, want F naming its composition", name, w) } @@ -1146,7 +1146,7 @@ func TestAHaveFromBeforeTheSwitchGetsItsWantWithNoRecordHold(t *testing.T) { m2.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}} } n2.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} - out := Wants(m2, n2, nil, nil) + out := Wants(m2, heardOfHaves(n2), nil, nil) if j == JudgedNeither { if out.Wants["ace"].Fp != fNew || len(out.Record) != 0 { t.Errorf("%s, %s: %+v", name, what, out) @@ -1162,12 +1162,12 @@ func TestAHaveFromBeforeTheSwitchGetsItsWantWithNoRecordHold(t *testing.T) { return ComposedDigest{Digest: dig(fp), Judgement: JudgedRecreates, Against: dig(fOld), Words: "judged"}, true } n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Composed: dig(fOld), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} - if out := Wants(m, n, nil, nil); out.Wants["ace"].Fp == fNew && len(out.Record) == 0 { + if out := Wants(m, heardOfHaves(n), nil, nil); out.Wants["ace"].Fp == fNew && len(out.Record) == 0 { t.Errorf("%s: a have naming a composition skipped the hold: %+v", name, out) } // a new fingerprint still asks (the bus is barred instead) n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} - out = Wants(m, n, nil, nil) + out = Wants(m, heardOfHaves(n), nil, nil) if w, ok := out.Wants["ace"]; ok { t.Errorf("%s: a new fingerprint was written with no yes: %+v", name, w) } @@ -1181,7 +1181,7 @@ func TestAHaveFromBeforeTheSwitchGetsItsWantWithNoRecordHold(t *testing.T) { func TestAHoldOfAFailureWithoutWordsSaysSo(t *testing.T) { m, n := ready() n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed} - out := Wants(m, n, map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew)}}, map[string]string{"ace": fOld}) + out := Wants(m, heardOfHaves(n), map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew)}}, map[string]string{"ace": fOld}) if out.Held == nil || out.Held.Words != "health failed, phase running: the node gave no words" { t.Fatalf("held = %+v", out.Held) } @@ -1198,7 +1198,7 @@ func TestAHoldIsLiftedOnceEveryNodeRunsItHealthy(t *testing.T) { return m, n } m, n := everywhere() - out := Wants(m, n, wantedOld(), map[string]string{"ace": fOld, "g14": fOld}) + out := Wants(m, heardOfHaves(n), wantedOld(), map[string]string{"ace": fOld, "g14": fOld}) if !out.Lifted || out.Held != nil { t.Fatalf("lifted = %v, held = %+v: want the hold lifted", out.Lifted, out.Held) } @@ -1210,7 +1210,7 @@ func TestAHoldIsLiftedOnceEveryNodeRunsItHealthy(t *testing.T) { m, n = everywhere() delete(n.Haves["g14"], "mail") - if out := Wants(m, n, wantedOld(), nil); !out.Lifted { + if out := Wants(m, heardOfHaves(n), wantedOld(), nil); !out.Lifted { t.Error("a node that does not run the module kept the hold") } @@ -1227,7 +1227,7 @@ func TestAHoldIsLiftedOnceEveryNodeRunsItHealthy(t *testing.T) { } { m, n := everywhere() change(&m, &n) - if out := Wants(m, n, wantedOld(), map[string]string{"ace": fOld, "g14": fOld}); out.Lifted { + if out := Wants(m, heardOfHaves(n), wantedOld(), map[string]string{"ace": fOld, "g14": fOld}); out.Lifted { t.Errorf("%s: the hold was lifted", name) } } @@ -1258,7 +1258,7 @@ func x11() (Module, Nodes) { // composition, never the newest, and says the newest it is held back from; the canary is never passed over. func TestANodeRunningAModuleWithNoWantGetsItsFirstWantAtWhatItRuns(t *testing.T) { m, n := x11() - out := Wants(m, n, nil, nil) + out := Wants(m, heardOfHaves(n), nil, nil) // 1960: the providers rule holds no want: the canary g14 gets F, its provider's health unknown said if w := out.Wants["g14"]; w.Fp != fNew || out.Unreached["g14"] == "" { t.Errorf("g14's want = %+v (unreached %q), want F, the provider said", w, out.Unreached["g14"]) @@ -1273,13 +1273,13 @@ func TestANodeRunningAModuleWithNoWantGetsItsFirstWantAtWhatItRuns(t *testing.T) m, n = x11() n.Haves["g14"]["xorg"] = nodeengine.HaveByNodeModuleValue{Fp: fpOf('x'), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} n.Haves["shanks"]["xorg"] = n.Haves["g14"]["xorg"] - if w := Wants(m, n, nil, nil).Wants["shanks"]; w.Fp != fOld || w.Rule != mwc.RuleCanary { + if w := Wants(m, heardOfHaves(n), nil, nil).Wants["shanks"]; w.Fp != fOld || w.Rule != mwc.RuleCanary { t.Errorf("shanks held back by the canary with no want: %+v, want at fOld by rule 3", w) } // a node already running F gets its want at F; a removed want is no want either m, n = x11() n.Haves["g14"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthUnknown} - out = Wants(m, n, map[string]mwc.Want{"g14": {Removed: true}}, nil) + out = Wants(m, heardOfHaves(n), map[string]mwc.Want{"g14": {Removed: true}}, nil) if w := out.Wants["g14"]; w.Fp != fNew || w.Composed != dig(fNew) || w.Removed { t.Errorf("g14 running F with a removed want = %+v, want F", w) } @@ -1301,7 +1301,7 @@ func TestNoFirstWantWithoutTheCompositionAndGrantOfWhatItRuns(t *testing.T) { } { m, n := x11() change(&m, &n) - out := Wants(m, n, nil, nil) + out := Wants(m, heardOfHaves(n), nil, nil) if what == "an old-form build" { // the canary g14 does not run F healthy: no want, on neither list by rule 3 (issue 1945) if w, ok := out.Wants["shanks"]; ok { @@ -1322,7 +1322,7 @@ func TestNoFirstWantWithoutTheCompositionAndGrantOfWhatItRuns(t *testing.T) { } m, n := x11() before := map[string]mwc.Want{"g14": {Fp: fOld, Composed: dig(fOld)}, "shanks": {Fp: fOld, Composed: dig(fOld)}} - out := Wants(m, n, before, nil) + out := Wants(m, heardOfHaves(n), before, nil) if len(out.Neither) != 0 { t.Errorf("nodes holding a want named on neither list: %+v", out.Neither) } @@ -1341,7 +1341,7 @@ func TestNoFirstWantWithoutTheCompositionAndGrantOfWhatItRuns(t *testing.T) { func TestANewAssignmentWaitingIsNotOnNeitherList(t *testing.T) { m, n := x11() delete(n.Haves["shanks"], "screen-lock") - out := Wants(m, n, nil, nil) + out := Wants(m, heardOfHaves(n), nil, nil) // 1960: the canary orders moves, never a first want: the new assignment is wanted at F if w := out.Wants["shanks"]; w.Fp != fNew { t.Errorf("a new assignment while the canary does not run F: %+v, want F", w) @@ -1350,7 +1350,7 @@ func TestANewAssignmentWaitingIsNotOnNeitherList(t *testing.T) { t.Errorf("a new assignment is named on neither list: %+v", out.Neither) } m.Built = false - if out := Wants(m, n, nil, nil); len(out.Neither) != 1 || out.Neither["g14"].Rule != mwc.RuleBuildDone { + if out := Wants(m, heardOfHaves(n), nil, nil); len(out.Neither) != 1 || out.Neither["g14"].Rule != mwc.RuleBuildDone { t.Errorf("with no build: neither = %+v, want g14 alone (it runs the module), by rule 1", out.Neither) } // 1945 (the assigner holds a first want back silently): with no build, a node whose want was removed and that does not run @@ -1364,12 +1364,12 @@ func TestANewAssignmentWaitingIsNotOnNeitherList(t *testing.T) { }{ {"no key: nothing written", nil, map[string]mwc.Want{}}, {"a removed want: it says why", map[string]mwc.Want{"shanks": gone}, - map[string]mwc.Want{"shanks": {Removed: true, Words: "no want: rule 1: the build of screen-lock at " + short(m.FP) + " is not done"}}}, - {"a removed want that says why already: nothing written", map[string]mwc.Want{"shanks": {Removed: true, Words: "no want: rule 1: the build of screen-lock at " + short(m.FP) + " is not done"}}, + map[string]mwc.Want{"shanks": {Removed: true, Words: "no want: rule 1: the build of screen-lock at " + short(m.FP) + " is being built"}}}, + {"a removed want that says why already: nothing written", map[string]mwc.Want{"shanks": {Removed: true, Words: "no want: rule 1: the build of screen-lock at " + short(m.FP) + " is being built"}}, map[string]mwc.Want{}}, {"a node that runs it: on neither list, its removed want left", map[string]mwc.Want{"g14": gone}, map[string]mwc.Want{}}, } { - out := Wants(m, n, c.before, nil) + out := Wants(m, heardOfHaves(n), c.before, nil) if len(out.Wants) != len(c.want) { t.Errorf("%s: wants = %+v, want %+v", c.name, out.Wants, c.want) continue @@ -1381,7 +1381,7 @@ func TestANewAssignmentWaitingIsNotOnNeitherList(t *testing.T) { } } m.FP = "" - if out := Wants(m, n, nil, nil); len(out.Neither) != 0 { + if out := Wants(m, heardOfHaves(n), nil, nil); len(out.Neither) != 0 { t.Errorf("a module of the node's own composition named on neither list: %+v", out.Neither) } } @@ -1404,7 +1404,7 @@ func TestAnOldFormNodeGetsItsFirstWantAtFOnlyUnderTheCanary(t *testing.T) { providersHealthy(n) oldForm(n, "shanks") n.Haves["g14"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} - out := Wants(m, n, nil, nil) + out := Wants(m, heardOfHaves(n), nil, nil) if w := out.Wants["shanks"]; w.Fp != fNew || w.Composed != dig(fNew) || w.Newest != "" || w.Rule != 0 { t.Errorf("canary healthy on F: shanks's want = %+v, want F", w) } @@ -1416,7 +1416,7 @@ func TestAnOldFormNodeGetsItsFirstWantAtFOnlyUnderTheCanary(t *testing.T) { providersHealthy(n) oldForm(n, "shanks") n.Haves["g14"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthUnknown} - out = Wants(m, n, nil, nil) + out = Wants(m, heardOfHaves(n), nil, nil) if w, ok := out.Wants["shanks"]; ok { t.Errorf("canary not healthy on F: shanks got a want: %+v", w) } @@ -1427,7 +1427,7 @@ func TestAnOldFormNodeGetsItsFirstWantAtFOnlyUnderTheCanary(t *testing.T) { m, n = x11() oldForm(n, "shanks") n.Haves["g14"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} - out = Wants(m, n, nil, nil) + out = Wants(m, heardOfHaves(n), nil, nil) if w := out.Wants["shanks"]; w.Fp != fNew || out.Unreached["shanks"] == "" { t.Errorf("providers not healthy: shanks's want = %+v (unreached %q), want F, the provider said", w, out.Unreached["shanks"]) } @@ -1440,14 +1440,14 @@ func TestAnOldFormNodeGetsItsFirstWantAtFOnlyUnderTheCanary(t *testing.T) { oldForm(n, "shanks") n.Haves["g14"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} m.Granted = false - if w, ok := Wants(m, n, nil, nil).Wants["shanks"]; ok { + if w, ok := Wants(m, heardOfHaves(n), nil, nil).Wants["shanks"]; ok { t.Errorf("F not granted: shanks got a want: %+v", w) } // the old-form node is the canary itself: F once its providers run healthy m, n = x11() providersHealthy(n) oldForm(n, "g14") - if w := Wants(m, n, nil, nil).Wants["g14"]; w.Fp != fNew || w.Composed != dig(fNew) { + if w := Wants(m, heardOfHaves(n), nil, nil).Wants["g14"]; w.Fp != fNew || w.Composed != dig(fNew) { t.Errorf("old-form canary: g14's want = %+v, want F", w) } } @@ -1462,7 +1462,7 @@ func TestAFailedHealthWhileTheNodeMovesHoldsNothing(t *testing.T) { n := Nodes{Online: map[string]bool{"novox": true}, Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{ "novox": {"mesh-controller": {Fp: fNew, Phase: phase, Health: nodeengineclient.HealthFailed}}}} before := map[string]mwc.Want{"novox": {Fp: fNew, Composed: dig(fNew)}} - if out := Wants(m, n, before, map[string]string{"novox": fOld}); out.Held != nil { + if out := Wants(m, heardOfHaves(n), before, map[string]string{"novox": fOld}); out.Held != nil { t.Errorf("phase %s, health failed: held %+v, want no hold while the node moves", phase, out.Held) } } @@ -1472,7 +1472,7 @@ func TestAFailedHealthWhileTheNodeMovesHoldsNothing(t *testing.T) { n := Nodes{Online: map[string]bool{"novox": true}, Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{ "novox": {"mesh-controller": {Fp: fNew, Phase: phase, Health: nodeengineclient.HealthFailed}}}} before := map[string]mwc.Want{"novox": {Fp: fNew, Composed: dig(fNew)}} - if out := Wants(m, n, before, map[string]string{"novox": fOld}); out.Held == nil || out.Held.Node != "novox" { + if out := Wants(m, heardOfHaves(n), before, map[string]string{"novox": fOld}); out.Held == nil || out.Held.Node != "novox" { t.Errorf("phase %s, health failed: held %+v, want a hold on novox", phase, out.Held) } } @@ -1491,7 +1491,7 @@ func TestAHoldIsLiftedOverHavesThatAreNotRunning(t *testing.T) { } for _, phase := range []string{nodeengineclient.PhaseRemoved, nodeengineclient.PhaseHeld, nodeengineclient.PhaseApplying, nodeengineclient.PhaseBackingUp, nodeengineclient.PhaseChecking} { m, n := setup(phase, fOld, nodeengineclient.HealthUnknown) - if out := Wants(m, n, nil, nil); !out.Lifted { + if out := Wants(m, heardOfHaves(n), nil, nil); !out.Lifted { t.Errorf("g14's have in phase %s kept the hold up", phase) } } @@ -1500,14 +1500,14 @@ func TestAHoldIsLiftedOverHavesThatAreNotRunning(t *testing.T) { "failed": {Fp: fNew, Phase: nodeengineclient.PhaseFailed, Health: nodeengineclient.HealthFailed}, } { m, n := setup(h.Phase, h.Fp, h.Health) - if out := Wants(m, n, nil, nil); out.Lifted { + if out := Wants(m, heardOfHaves(n), nil, nil); out.Lifted { t.Errorf("g14 %s: the hold was lifted", what) } } // a failed health on a removed have holds nothing m, n := setup(nodeengineclient.PhaseRemoved, fNew, nodeengineclient.HealthFailed) m.Held = false - if out := Wants(m, n, nil, nil); out.Held != nil { + if out := Wants(m, heardOfHaves(n), nil, nil); out.Held != nil { t.Errorf("a removed have with a failed health held F: %+v", out.Held) } } @@ -1520,7 +1520,7 @@ func TestAFirstWantAfterARemovedWantGoesThroughTheRecordAsk(t *testing.T) { m.Composed = func(_, fp string) (ComposedDigest, bool) { return ComposedDigest{Digest: dig(fp), Judgement: JudgedRecreates, Policy: catalogue.PolicyRecord}, true } - out := Wants(m, n, map[string]mwc.Want{"shanks": {Removed: true}}, nil) + out := Wants(m, heardOfHaves(n), map[string]mwc.Want{"shanks": {Removed: true}}, nil) if w, ok := out.Wants["shanks"]; ok { t.Errorf("shanks's first want after a removed want was written with no record ask: %+v", w) } @@ -1550,7 +1550,7 @@ func TestAHeldModuleWithNoWantIsNotOnNeitherList(t *testing.T) { } { m, n := x11() change(&m, &n) - out := Wants(m, n, nil, nil) + out := Wants(m, heardOfHaves(n), nil, nil) if len(out.Neither) != 0 { t.Errorf("%s: a held module is on neither list: %+v", what, out.Neither) } @@ -1602,9 +1602,79 @@ func TestAWantNamesAKeptBuildsPartsAndNodeForANodeThatReadsThem(t *testing.T) { m, n := ready() n.Reads = map[string][]string{"ace": row.reads, "g14": row.reads} m.KeptBuilds = row.kept - w := Wants(m, n, wantedOld(), nil).Wants["ace"] + w := Wants(m, heardOfHaves(n), wantedOld(), nil).Wants["ace"] if w.Fp != fNew || (w.KeptOn == "g14") != row.wanted || (w.Version == "1.2.3") != row.wanted || (len(w.Parts) == 1) != row.wanted { t.Errorf("%s: ace's want = %+v", name, w) } } } + +// heardOfHaves gives each have that runs the `running` the node-engine seat says of it (design 47 G §520: health is read from +// `running` alone), for a fixture written as haves; a fixture that names its own `running` is left as it is. +func heardOfHaves(n Nodes) Nodes { + if n.Running != nil { + return n + } + n.Running = map[string]map[string]nodeengine.Running{} + for node, byModule := range n.Haves { + n.Running[node] = map[string]nodeengine.Running{} + for module, h := range byModule { + if h.Phase == nodeengineclient.PhaseRunning && h.Fp != "" { + ok := h.Health == nodeengineclient.HealthOK + n.Running[node][module] = nodeengine.Running{Node: node, Module: module, Fingerprint: h.Fp, Healthy: &ok} + } + } + } + return n +} + +// §520: health is read from the node-engine seat's `running` alone, whatever the have says, and a build of which nothing was heard +// is not healthy. +func TestHealthIsReadFromRunningAlone(t *testing.T) { + yes, no := true, false + say := func(node, module, fp string, healthy *bool) nodeengine.Running { + return nodeengine.Running{Node: node, Module: module, Fingerprint: fp, Healthy: healthy} + } + for _, c := range []struct { + name string + running map[string]map[string]nodeengine.Running + rule int + }{ + {"all healthy by running", map[string]map[string]nodeengine.Running{"ace": {"mail": say("ace", "mail", fNew, &yes), "store": say("ace", "store", fpOf('s'), &yes)}}, 0}, + {"a running with no healthy reads healthy", map[string]map[string]nodeengine.Running{"ace": {"mail": say("ace", "mail", fNew, nil), "store": say("ace", "store", fpOf('s'), nil)}}, 0}, + {"the canary says unhealthy, its have says healthy", map[string]map[string]nodeengine.Running{"ace": {"mail": say("ace", "mail", fNew, &no), "store": say("ace", "store", fpOf('s'), &yes)}}, mwc.RuleCanary}, + {"nothing heard of the canary", map[string]map[string]nodeengine.Running{"ace": {"store": say("ace", "store", fpOf('s'), &yes)}}, mwc.RuleCanary}, + } { + m, n := ready() + n.Running = c.running + if got := Wants(m, n, wantedOld(), nil).Rules["g14"].Rule; got != c.rule { + t.Errorf("%s: g14 held by rule %d, want %d", c.name, got, c.rule) + } + } + // rule 5: a provider that says unhealthy, or that nothing was heard of, holds no want but leaves the consumer unreached + m, n := ready() + n.Running = map[string]map[string]nodeengine.Running{"ace": {"mail": say("ace", "mail", fNew, &yes), "store": say("ace", "store", fpOf('s'), &no)}} + if out := Wants(m, n, wantedOld(), nil); len(out.Unreached) == 0 { + t.Errorf("a provider saying unhealthy: no module unreached, %+v", out) + } +} + +// §575: a hold under rule 1 says how the build stands: failed with the error whole, or being built. +func TestRuleOneSaysHowTheBuildStands(t *testing.T) { + for _, c := range []struct { + name string + failed bool + err string + want string + }{ + {"a build that runs", false, "", "the build of mail at " + short(fNew) + " is being built"}, + {"a failed build", true, "transport fault: no responders at mesh-build-go", "the build of mail at " + short(fNew) + " failed: transport fault: no responders at mesh-build-go"}, + } { + m, n := ready() + m.Built, m.BuildFailed, m.BuildError = false, c.failed, c.err + held, rule, words := Holds(m, "g14", n) + if held || rule != mwc.RuleBuildDone || words != c.want { + t.Errorf("%s: held %v rule %d words %q, want %q", c.name, held, rule, words, c.want) + } + } +} diff --git a/modules/mesh-delivery-assigner/internal/buckets/buckets.go b/modules/mesh-delivery-assigner/internal/buckets/buckets.go index 0cf5be2d..b7c53119 100644 --- a/modules/mesh-delivery-assigner/internal/buckets/buckets.go +++ b/modules/mesh-delivery-assigner/internal/buckets/buckets.go @@ -7,9 +7,7 @@ package buckets import ( "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" watchdogs "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" - delivery "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" wants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate" @@ -58,16 +56,11 @@ const ( Provisions = ProvisionerModule + "_provisions" // Presence holds each part's presence and the node's. Presence = "mesh-delivery-watchman_" + watchdogs.StatePresence - // Source and Merges are the gate's. - Source = "mesh-delivery-gate_" + delivery.StateSource - Merges = "mesh-delivery-gate_" + delivery.StateMerges // TellerModule is the module whose hand the operator's answers pass: the one that holds each ask's answer, and the // name the hand-act log records a warrant under. Held equal to the manifest's reads by its test. TellerModule = "mesh-delivery-teller" // Answers holds the teller's answer to each ask. Answers = TellerModule + "_answers" - // Builds is the builder's. - Builds = builds.BuilderBuildsBucket // Grants is the grants part's. Grants = grantsstate.PartGrantsBucket ) diff --git a/modules/mesh-delivery-assigner/internal/buckets/buckets_test.go b/modules/mesh-delivery-assigner/internal/buckets/buckets_test.go index 0b6de994..e5d8a881 100644 --- a/modules/mesh-delivery-assigner/internal/buckets/buckets_test.go +++ b/modules/mesh-delivery-assigner/internal/buckets/buckets_test.go @@ -8,10 +8,9 @@ func TestBucketsAreTheEarlierNames(t *testing.T) { for name, c := range map[string][2]string{ "Have": {Have, "node-engine_have"}, "ComposedRead": {ComposedRead, "mesh-host.composed"}, "Modules": {Modules, "mesh-delivery-indexer_modules"}, "Manifests": {Manifests, "mesh-delivery-indexer_manifests"}, - "Presence": {Presence, "mesh-delivery-watchman_presence"}, "Source": {Source, "mesh-delivery-gate_source"}, - "Merges": {Merges, "mesh-delivery-gate_merges"}, "Answers": {Answers, "mesh-delivery-teller_answers"}, - "Builds": {Builds, "mesh-delivery-builder_builds"}, "Grants": {Grants, "mesh-grants_grants"}, - "Wants": {Wants, "mesh-delivery-assigner_wants"}, "Differences": {Differences, "mesh-delivery-assigner_differences"}, + "Presence": {Presence, "mesh-delivery-watchman_presence"}, "Answers": {Answers, "mesh-delivery-teller_answers"}, + "Grants": {Grants, "mesh-grants_grants"}, + "Wants": {Wants, "mesh-delivery-assigner_wants"}, "Differences": {Differences, "mesh-delivery-assigner_differences"}, "Asks": {Asks, "mesh-delivery-assigner_asks"}, "Puts": {Puts, "mesh-delivery-assigner_puts"}, "Policies": {Policies, "mesh-delivery-assigner_policies"}, "Provisions": {Provisions, "mesh-provisioner_provisions"}, "TellerModule": {TellerModule, "mesh-delivery-teller"}, } { diff --git a/modules/mesh-delivery-assigner/manifest_test.go b/modules/mesh-delivery-assigner/manifest_test.go index 94ad01ef..61184a25 100644 --- a/modules/mesh-delivery-assigner/manifest_test.go +++ b/modules/mesh-delivery-assigner/manifest_test.go @@ -43,17 +43,18 @@ func TestManifest(t *testing.T) { reads := slices.Clone(m.Reads) slices.Sort(reads) // ADR 0593 §8: it composes nothing, so it reads no candidate, held or resolved value, compose step or build log; it reads - // every node's published compositions to compare them, and the gate's source and merges, for the release that carries a build (ADR 0601 §1). + // every node's published compositions to compare them,. // DLV-F2r (ADR 0615 §1, ADR 0631 §1): the indexer's manifests, modules and module graph (whose verb edges a want waits // on, ADR 0614 §2, §3) are read through its first views and views family, none of the indexer's states by reads. - want := []string{"mesh-delivery-builder.builds", "mesh-delivery-gate.merges", "mesh-delivery-gate.source", - "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-grants.grants", "mesh-host.composed", + // A-181: what was built is heard from the builder's stream of ends, so the builder's builds bucket is no read. + // §431, §507: the gate's source is read through its verb `source` and its event `source-moved`, and no `merges` is read. + want := []string{"mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-grants.grants", "mesh-host.composed", "mesh-provisioner.provisions", "node-engine.have"} if !reflect.DeepEqual(reads, want) { t.Errorf("reads = %v, want exactly %v", reads, want) } // the buckets the program names are the manifest's reads, `.` as `_` - named := []string{buckets.Builds, buckets.Merges, buckets.Source, buckets.Answers, buckets.Presence, buckets.Grants, + named := []string{buckets.Answers, buckets.Presence, buckets.Grants, buckets.ComposedRead, "mesh-provisioner_provisions", buckets.Have} slices.Sort(named) for i, r := range want { @@ -130,12 +131,13 @@ func TestManifest(t *testing.T) { } // ADR 0618 §4: and it hears each node's engine say its reads changed; ADR 0636 §4 (issue 1966): and the seat list's two events; // issue 1966 note 151: and the teller's ask-ended, its subject the one the assigner hears. - if !slices.Equal(m.Consumes, []string{"mesh-conditions.findings-asked", "mesh-conditions.condition-raised", "mesh-conditions.condition-changed", "mesh-conditions.condition-cleared", "mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", "mesh-catalog.version-released", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", builds.BuilderModule + "." + builds.BuildEndedEvent, buckets.TellerModule + "." + tellerevents.AskEnded, "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.group-ended", "node-engine.manifest-reads-changed", "node-engine.running"}) { + if !slices.Equal(m.Consumes, []string{"mesh-conditions.findings-asked", "mesh-conditions.condition-raised", "mesh-conditions.condition-changed", "mesh-conditions.condition-cleared", "mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", "mesh-delivery.source-moved", "mesh-catalog.version-released", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", builds.BuilderModule + "." + builds.BuildsStreamEvents, buckets.TellerModule + "." + tellerevents.AskEnded, "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.group-ended", "node-engine.manifest-reads-changed", "node-engine.running"}) { t.Errorf("consumes = %v, want the catalog half's revisions (design 47 G §195), the teller's ask-ended, the engines' reads changed and what they run", m.Consumes) } if want := []string{"mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "seat:mesh-catalog.registrations", "seat:mesh-catalog.revision", "seat:mesh-catalog.seat-definitions", // ADR 0636 §4: the seat list "seat:mesh-conditions.conditions", // design 47 G §706: the conditions read at each judging of an outside wait + "seat:mesh-delivery.source", // §507: the branches the gate keeps a source revision of, before the first registrations "seat:mesh-hand-acts.hand-act", "seat:mesh-hand-acts.hand-acts", "seat:mesh-hand-acts.warranted", // ADR 0565 §9, §11: the seat alone "seat:node-backup.backed-up", "seat:node-backup.now", "seat:node-engine.manifest-reads", // ADR 0618 §3 diff --git a/modules/mesh-delivery-assigner/module.json b/modules/mesh-delivery-assigner/module.json index fe290015..a494e580 100644 --- a/modules/mesh-delivery-assigner/module.json +++ b/modules/mesh-delivery-assigner/module.json @@ -30,9 +30,6 @@ } ], "reads": [ - "mesh-delivery-builder.builds", - "mesh-delivery-gate.merges", - "mesh-delivery-gate.source", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-grants.grants", @@ -50,10 +47,11 @@ "mesh-conditions.condition-cleared", "mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", + "mesh-delivery.source-moved", "mesh-catalog.version-released", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", - "mesh-delivery-builder.build.ended", + "mesh-delivery-builder.build.ended.**", "mesh-delivery-teller.ask-ended", "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.group-ended", @@ -67,6 +65,7 @@ "seat:mesh-catalog.revision", "seat:mesh-catalog.seat-definitions", "seat:mesh-conditions.conditions", + "seat:mesh-delivery.source", "seat:mesh-hand-acts.hand-act", "seat:mesh-hand-acts.hand-acts", "seat:mesh-hand-acts.warranted", diff --git a/modules/mesh-delivery-builder/builds/ended.go b/modules/mesh-delivery-builder/builds/ended.go index 0f01a3d1..23e99104 100644 --- a/modules/mesh-delivery-builder/builds/ended.go +++ b/modules/mesh-delivery-builder/builds/ended.go @@ -17,11 +17,19 @@ const ( // BuildEnded is the body of the event build.ended: the module, the fingerprint, each artifact made with its digest, the build // node that keeps a kept build (none for one not kept) and the watched branch a build of a candidate is for (none for the trunk). type BuildEnded struct { - Module string `json:"module"` - Fingerprint string `json:"fingerprint"` - Made map[string]string `json:"made,omitempty"` - KeptOn string `json:"kept-on,omitempty"` - Branch string `json:"branch,omitempty"` + Module string `json:"module"` + Fingerprint string `json:"fingerprint"` + // Outcome is done or failed (design 47 G §575, A-162): the event is said for every end that is one of these; Error is why a + // failed build failed, as its key holds it (`transport fault:` included, §568), none for a done one. + Outcome string `json:"outcome"` + Error string `json:"error,omitempty"` + // Manifest is the digest of the manifest built and Needs the reader capabilities its reference forms need (ADR 0618 §1), of a + // done build: what a reader of the end asks of the builder's key otherwise. + Manifest string `json:"manifest,omitempty"` + Needs []string `json:"needs,omitempty"` + Made map[string]string `json:"made,omitempty"` + KeptOn string `json:"kept-on,omitempty"` + Branch string `json:"branch,omitempty"` } // BuildEndedEventName is the event's name for a module and fingerprint: `build.ended..`, each escaped. @@ -29,15 +37,15 @@ func BuildEndedEventName(module, fp string) string { return BuildEndedEvent + "." + keys.EscapeName(module) + "." + keys.EscapeName(fp) } -// BuildEndedOf is the event of a log entry of a build's end: false for any entry but the end of a done build, which says no +// BuildEndedOf is the event of a log entry of a build's end: false for any entry but the end of a done or a failed build. A done end says no // digest of any part. func BuildEndedOf(e BuildLogEntry) (BuildEnded, bool) { - if e.Event != BuildEndedEvent || e.Outcome != BuildDone || e.Module == "" || e.FP == "" { + if e.Event != BuildEndedEvent || (e.Outcome != BuildDone && e.Outcome != BuildFailed) || e.Module == "" || e.FP == "" { return BuildEnded{}, false } made := map[string]string{} for _, a := range e.Made { made[a.Name] = a.Digest } - return BuildEnded{Module: e.Module, Fingerprint: e.FP, Made: made, KeptOn: e.KeptOn, Branch: e.Ref}, true + return BuildEnded{Module: e.Module, Fingerprint: e.FP, Outcome: e.Outcome, Error: e.Error, Manifest: e.Manifest, Needs: e.Needs, Made: made, KeptOn: e.KeptOn, Branch: e.Ref}, true } diff --git a/modules/mesh-delivery-builder/builds/ended_test.go b/modules/mesh-delivery-builder/builds/ended_test.go index 33e3f70f..8572b8cf 100644 --- a/modules/mesh-delivery-builder/builds/ended_test.go +++ b/modules/mesh-delivery-builder/builds/ended_test.go @@ -11,7 +11,7 @@ func TestADoneBuildsEndIsTheEventBuildEnded(t *testing.T) { made := []BuiltArtifact{{Name: "program", Kind: "program", Digest: "sha256:1"}} e := BuildLogEntry{Event: BuildEndedEvent, Outcome: BuildDone, Module: "node-engine", FP: "src2:a", Ref: "cand", KeptOn: "ace", Made: made} got, ok := BuildEndedOf(e) - want := BuildEnded{Module: "node-engine", Fingerprint: "src2:a", Made: map[string]string{"program": "sha256:1"}, KeptOn: "ace", Branch: "cand"} + want := BuildEnded{Module: "node-engine", Fingerprint: "src2:a", Outcome: BuildDone, Made: map[string]string{"program": "sha256:1"}, KeptOn: "ace", Branch: "cand"} if !ok || !reflect.DeepEqual(got, want) { t.Errorf("event = %+v, %v", got, ok) } @@ -20,7 +20,7 @@ func TestADoneBuildsEndIsTheEventBuildEnded(t *testing.T) { } for name, mut := range map[string]func(*BuildLogEntry){ "started": func(e *BuildLogEntry) { e.Event = BuildStartedEvent }, - "failed": func(e *BuildLogEntry) { e.Outcome = BuildFailed }, + "queued": func(e *BuildLogEntry) { e.Outcome = BuildQueued }, "no fp": func(e *BuildLogEntry) { e.FP = "" }, } { c := e @@ -30,3 +30,13 @@ func TestADoneBuildsEndIsTheEventBuildEnded(t *testing.T) { } } } + +// Design 47 G §575, A-162: a failed build's end is said too, with its outcome and its error whole. +func TestAFailedBuildsEndIsSaidWithItsError(t *testing.T) { + e := BuildLogEntry{Event: BuildEndedEvent, Outcome: BuildFailed, Module: "web", FP: "src2:b", Error: "transport fault: no responders at mesh-build-go"} + got, ok := BuildEndedOf(e) + want := BuildEnded{Module: "web", Fingerprint: "src2:b", Outcome: BuildFailed, Error: e.Error, Made: map[string]string{}} + if !ok || !reflect.DeepEqual(got, want) { + t.Errorf("event = %+v, %v", got, ok) + } +} diff --git a/modules/mesh-delivery-builder/builds/log.go b/modules/mesh-delivery-builder/builds/log.go index 3eed958b..3fd4aa85 100644 --- a/modules/mesh-delivery-builder/builds/log.go +++ b/modules/mesh-delivery-builder/builds/log.go @@ -83,6 +83,9 @@ type BuildLogEntry struct { // narration names neither. Error string `json:"error,omitempty"` Step string `json:"step,omitempty"` + // Needs are, at a done build's end, the reader capabilities its reference forms need (BuilderBuild.Needs, ADR 0618 §1): the end + // event carries them (design 47 G §575, A-162). + Needs []string `json:"needs,omitempty"` // Line is a narration's words, Why what it narrates (asked, waiting, stored, refused): a build.said entry's alone. Line string `json:"line,omitempty"` Why string `json:"why,omitempty"` diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go index fe5f9e56..7a264025 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go @@ -594,7 +594,7 @@ func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7 t.Errorf("no build.ended of netcheck's build: %v", err) } else { var ev builds.BuildEnded - if err := json.Unmarshal(raw.Data, &ev); err != nil || ev.Module != "netcheck" || ev.Fingerprint != fp || len(ev.Made) != 2 { + if err := json.Unmarshal(raw.Data, &ev); err != nil || ev.Module != "netcheck" || ev.Fingerprint != fp || ev.Outcome != builds.BuildDone || len(ev.Made) != 2 { t.Errorf("build.ended = %s (%v), want netcheck at its fingerprint with two artifacts' digests", raw.Data, err) } } @@ -656,9 +656,18 @@ func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7 b.fake.hang.Store(&release) third := "src2:" + strings.Repeat("4", 64) b.moduleFP(t, "netcheck", netcheck, "modules/netcheck", bothToolchains, commit, third, 3) - if f := b.awaitFP(t, "netcheck", third, isState(builds.BuildFailed)); !strings.Contains(f.Error, "did not end within") { + f := b.awaitFP(t, "netcheck", third, isState(builds.BuildFailed)) + if !strings.Contains(f.Error, "did not end within") { t.Errorf("the failed build = %+v, want its bound run out", f) } + // A-162: the failed end is said as build.ended too, with its error whole. + if st, err := b.js.Stream(b.ctx, "BUILDER_BUILDS"); err != nil { + t.Fatal(err) + } else if raw, err := st.GetLastMsgForSubject(b.ctx, "mesh.stream.mesh-delivery-builder.event."+builds.BuildEndedEventName("netcheck", third)); err != nil { + t.Errorf("no build.ended of netcheck's failed build: %v", err) + } else if ev := (builds.BuildEnded{}); json.Unmarshal(raw.Data, &ev) != nil || ev.Outcome != builds.BuildFailed || ev.Error != f.Error { + t.Errorf("build.ended = %s, want the failed outcome with the error %q", raw.Data, f.Error) + } failed := meshconditionsclient.Key(meshconditionsclient.ScopeModule, "netcheck", BuildFailed) reg.awaitFinding(t, failed, true) b.fake.hang.Store(nil) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go index 2c5c1ef5..c8d4f0b2 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go @@ -161,9 +161,9 @@ func appendLog(ctx context.Context, js jetstream.JetStream, cred process.Account return nil } -// emitBuildEnded says the builder's event build.ended of a done build, on the builder's own stream (design 47 G §379, A-96): +// emitBuildEnded says the builder's event build.ended of a done or failed build (A-162), on the builder's own stream (design 47 G §379, A-96): // mesh.stream..event.build.ended.., published where the log entry of the end is written. An -// entry that is not a done build's end says none. The id makes a repeated end one event. +// entry that is not a done or failed build's end says none. The id makes a repeated end one event. func emitBuildEnded(ctx context.Context, js jetstream.JetStream, cred process.Account, e builds.BuildLogEntry) error { body, ok := builds.BuildEndedOf(e) if !ok { diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index a6a59251..bce359de 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -1208,6 +1208,7 @@ func LogEnded(module string, m meshcatalog.ModulesByModuleValue, ref, by string, e.KeptOn = result.KeptOn e.Imports = append([]string(nil), result.Imports...) // the indexer keeps them on the module (novox/hq ADR 0593 §2) e.Error, e.Step = result.Error, result.Step // a failed build's reason, not only its outcome + e.Needs = append([]string(nil), result.Needs...) return e } diff --git a/modules/mesh-delivery-builder/manifest_test.go b/modules/mesh-delivery-builder/manifest_test.go index f7498cb2..e09f37f8 100644 --- a/modules/mesh-delivery-builder/manifest_test.go +++ b/modules/mesh-delivery-builder/manifest_test.go @@ -53,10 +53,11 @@ type manifest struct { Consumes []string `json:"consumes"` State []state `json:"state"` Streams []struct { - Name string `json:"name"` - Events []string `json:"events"` - MaxAgeDays int `json:"max-age-days"` - MaxMiB int `json:"max-mib"` + Name string `json:"name"` + Events []string `json:"events"` + MaxAgeDays int `json:"max-age-days"` + MaxMiB int `json:"max-mib"` + LastPerSubject bool `json:"last-per-subject"` } `json:"streams"` Logs []struct { Name string `json:"name"` @@ -177,9 +178,9 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 // Design 47 G §672: the toolchain waits' stream, kept 7 days within 16 MiB, and the bound's setting, 60 minutes. // Design 47 G A-30, §383: and its own findings' events, which its finding writer says. if len(m.Streams) != 3 || m.Streams[0].Name != "waits" || !reflect.DeepEqual(m.Streams[0].Events, []string{"wait-held.**", "wait-ended.**"}) || m.Streams[0].MaxAgeDays != 7 || m.Streams[0].MaxMiB != 16 || - m.Streams[1].Name != builds.BuildsStream || !reflect.DeepEqual(m.Streams[1].Events, []string{builds.BuildsStreamEvents}) || m.Streams[1].MaxAgeDays != 7 || + m.Streams[1].Name != builds.BuildsStream || !reflect.DeepEqual(m.Streams[1].Events, []string{builds.BuildsStreamEvents}) || m.Streams[1].MaxAgeDays != 7 || !m.Streams[1].LastPerSubject || m.Streams[2].Name != "mesh-delivery-builder-findings" || !reflect.DeepEqual(m.Streams[2].Events, []string{"finding-written.*", "finding-cleared.*", "findings-stated.*"}) { - t.Errorf("streams = %+v, want waits with wait-held.** and wait-ended.**, 7 days, 16 MiB, the builds' stream (build.ended.**, 7 days) and the findings' stream", m.Streams) + t.Errorf("streams = %+v, want waits with wait-held.** and wait-ended.**, 7 days, 16 MiB, the builds' stream (build.ended.**, 7 days, kept last per subject: design 47 G A-181) and the findings' stream", m.Streams) } if c, ok := m.Config["toolchain-wait-minutes"]; !ok || c.Default != float64(60) || len(m.Config) != 2 || m.Config["republish-max-versions"].Default != float64(50) { t.Errorf("config = %+v, want toolchain-wait-minutes of 60 and republish-max-versions of 50 (design 47 G §588, §633)", m.Config) diff --git a/modules/mesh-delivery-builder/module.json b/modules/mesh-delivery-builder/module.json index a60b5e69..e27cc4b4 100644 --- a/modules/mesh-delivery-builder/module.json +++ b/modules/mesh-delivery-builder/module.json @@ -130,7 +130,7 @@ ], "streams": [ { "name": "waits", "events": ["wait-held.**", "wait-ended.**"], "max-age-days": 7, "max-mib": 16 }, - { "name": "mesh-delivery-builder-builds", "events": ["build.ended.**"], "max-age-days": 7, "max-mib": 16 }, + { "name": "mesh-delivery-builder-builds", "events": ["build.ended.**"], "max-age-days": 7, "max-mib": 16, "last-per-subject": true }, { "name": "mesh-delivery-builder-findings", "events": ["finding-written.*", "finding-cleared.*", "findings-stated.*"], "max-age-days": 7, "max-mib": 16 } ], "config": { diff --git a/modules/mesh-delivery-contracts/go/clients/meshcatalogclient/branches.go b/modules/mesh-delivery-contracts/go/clients/meshcatalogclient/branches.go index d9a149be..bfe073b1 100644 --- a/modules/mesh-delivery-contracts/go/clients/meshcatalogclient/branches.go +++ b/modules/mesh-delivery-contracts/go/clients/meshcatalogclient/branches.go @@ -68,6 +68,9 @@ type Branches struct { Entries *Entries // Changed, when not nil, is told of each change to a branch's view (module, its entry, deleted), set before Wire. Changed func(branch, module string, entry json.RawMessage, deleted bool, at time.Time) + // Adopted, when not nil, is told on a goroutine of its own of each branch whose view was adopted, with that view (a part that waits for + // its first branch's first view, design 47 G §507), set before Wire. + Adopted func(branch string, c *Catalog) // Logf, when not nil, says each view adopted and dropped. Logf func(format string, args ...any) @@ -250,6 +253,9 @@ func (b *Branches) reconcile() { b.catalogs[e] = c b.order = append(b.order, e) b.logf("meshcatalogclient: the branch %q is followed", want[e]) + if b.Adopted != nil { + go b.Adopted(branch, c) + } } for _, e := range slices.Clone(b.order) { if _, ok := want[e]; ok { diff --git a/modules/mesh-delivery-contracts/go/clients/meshcatalogclient/branches_test.go b/modules/mesh-delivery-contracts/go/clients/meshcatalogclient/branches_test.go index 34c5937b..6e0f733e 100644 --- a/modules/mesh-delivery-contracts/go/clients/meshcatalogclient/branches_test.go +++ b/modules/mesh-delivery-contracts/go/clients/meshcatalogclient/branches_test.go @@ -126,6 +126,12 @@ func TestBranchesFollowSourceThenRegistrations(t *testing.T) { t.Fatal(err) } b.Logf = func(format string, args ...any) { said <- fmt.Sprintf(format, args...) } + adopted := make(chan string, 8) + b.Adopted = func(branch string, c *Catalog) { + if c.Branch == branch { + adopted <- branch + } + } want := func(names ...string) { t.Helper() if got := w.names(); !slices.Equal(got, names) { @@ -148,6 +154,18 @@ func TestBranchesFollowSourceThenRegistrations(t *testing.T) { t.Fatal(err) } waitFor(`"feat/x" is followed`) + told := []string{} // each adopted branch is told, with its view (a part waits for its first one's first view), in no set order + for range 2 { + select { + case got := <-adopted: + told = append(told, got) + case <-ctx.Done(): + t.Fatalf("told of %v, want main and feat/x", told) + } + } + if slices.Sort(told); !slices.Equal(told, []string{"feat/x", "main"}) { + t.Fatalf("told of %v", told) + } if got := b.Followed(); !slices.Equal(got, []string{"main", "feat/x"}) { t.Fatalf("followed = %v", got) } diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main.go index 6a381e25..31ab8967 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main.go @@ -131,7 +131,7 @@ func MergedOf(of []seat.Group, repository, merge string, number int, trees bridg if err != nil { return nil, fmt.Errorf("the forge gives no merged pull request of %s with the commit %s: %w", repository, merge, err) } - msg, err := bridge.MergedMessage(owner, repo, fp.Number, fp.Branch, fp.Head, merge, trees, files) + msg, err := bridge.MergedMessage(owner, repo, fp.Number, fp.Branch, fp.Head, merge, fp.Base, trees, files) if err != nil { return nil, fmt.Errorf("the merged message of %s at %s cannot be built: %w", repository, merge, err) } diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main_test.go index c39fecb5..83d8a758 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main_test.go @@ -135,8 +135,7 @@ func TestTheBridgeToolsReadTheGroupsAndAskTheEndAgain(t *testing.T) { return nil, bridge.ErrNotFound } published := func(repo string, number int, head, branch, merge string) forgemsg.Forge { - body, _ := json.Marshal(git.PullMergedEvent{Repository: "novox/" + repo, Number: int64(number), Branch: branch, Head: head, Base: "main", MergeCommit: merge}) - msg, err := bridge.Convert(bridge.MergedSubject, body, trees, files) + msg, err := bridge.MergedMessage("novox", repo, number, branch, head, merge, "main", trees, files) if err != nil { t.Fatal(err) } @@ -172,8 +171,13 @@ func TestTheBridgeToolsReadTheGroupsAndAskTheEndAgain(t *testing.T) { api: forgeAnswers(t, 9, "", "merged", "fix/x", h2, c1, "main"), fail: "not at " + c2}, {name: "a forge pull request not merged", repository: "novox/a", merge: c2, number: 9, api: forgeAnswers(t, 9, "", "open", "fix/x", h2, "", "main"), fail: "not merged"}, + // design 47 G §714: no filter to the trunk alone; the merged message names the branch it was merged into. {name: "a forge pull request merged into another branch", repository: "novox/a", merge: c2, number: 9, - api: forgeAnswers(t, 9, "", "merged", "fix/x", h2, c2, "dev"), fail: "not the trunk"}, + api: forgeAnswers(t, 9, "", "merged", "fix/x", h2, c2, "dev"), want: func() forgemsg.Forge { + f := published("a", 9, h2, "fix/x", c2) + f.Base = "dev" + return f + }()}, {name: "a forge pull request with no head branch", repository: "novox/a", merge: c2, number: 9, api: forgeAnswers(t, 9, "", "merged", "", h2, c2, "main"), fail: "no head branch"}, {name: "a seat answer that says an error", repository: "novox/a", merge: c2, number: 9, diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/bound.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/bound.go new file mode 100644 index 00000000..ab5b8c77 --- /dev/null +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/bound.go @@ -0,0 +1,100 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "sync" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/busretry" + "git.novox.be/novox/mesh-sdk/go/process" +) + +// brokerSeat is the seat whose verb `show` answers a `requires` entry's bus names, and whose `recreated` and `dropped` +// say it changed (design 47 G §577): the manifest names `seat:mesh-broker.show` in `invokes` and the two events in +// `consumes`. +const brokerSeat = "mesh-broker" + +// performedLocal is the local name of the state performed in the manifest's `requires`. +const performedLocal = "performed" + +// keptStore is the part of a key-value bucket the bridge's stores use. +type keptStore interface { + Get(ctx context.Context, key string) (jetstream.KeyValueEntry, error) + Put(ctx context.Context, key string, value []byte) (uint64, error) + PutString(ctx context.Context, key string, value string) (uint64, error) + Delete(ctx context.Context, key string, opts ...jetstream.KVDeleteOpt) error +} + +// boundBucket is a `requires` entry's bucket opened by its binding, never by an old name (design 47 G §577): each use +// asks the Binding for the bucket's bus name and opens it, again after `show` answers another name (a recreated entry). +// While `show` has not answered, every use fails with the Binding's error naming the entry, so nothing that needs it is +// done. +type boundBucket struct { + binding *process.Binding + local string + js process.BucketOpener + + mu sync.Mutex + name string + kv jetstream.KeyValue +} + +func (s *boundBucket) open(ctx context.Context) (jetstream.KeyValue, error) { + names, err := s.binding.Names(s.local) + if err != nil { + return nil, err + } + if names.Bucket == "" { + return nil, fmt.Errorf("the bus entry %q is bound to no bucket", s.local) + } + s.mu.Lock() + defer s.mu.Unlock() + if s.kv != nil && s.name == names.Bucket { + return s.kv, nil + } + kv, err := s.js.KeyValue(ctx, names.Bucket) + if err != nil { + return nil, fmt.Errorf("the bus entry %q, bucket %s: %w", s.local, names.Bucket, err) + } + s.name, s.kv = names.Bucket, kv + return kv, nil +} + +func (s *boundBucket) Get(ctx context.Context, key string) (jetstream.KeyValueEntry, error) { + kv, err := s.open(ctx) + if err != nil { + return nil, err + } + return kv.Get(ctx, key) +} + +func (s *boundBucket) Put(ctx context.Context, key string, value []byte) (uint64, error) { + kv, err := s.open(ctx) + if err != nil { + return 0, err + } + return kv.Put(ctx, key, value) +} + +func (s *boundBucket) PutString(ctx context.Context, key string, value string) (uint64, error) { + return s.Put(ctx, key, []byte(value)) +} + +func (s *boundBucket) Delete(ctx context.Context, key string, opts ...jetstream.KVDeleteOpt) error { + kv, err := s.open(ctx) + if err != nil { + return err + } + return kv.Delete(ctx, key, opts...) +} + +// keptCallOf is call as a kept call (ADR 0668 §1): asked again through a timeout or no responders until answered or +// busretry.Bound has passed. +func keptCallOf(ctx context.Context, call caller) func(key string, body any) (json.RawMessage, error) { + return func(key string, body any) (json.RawMessage, error) { + return busretry.DoRepeatable("the call "+key, func() (json.RawMessage, error) { return call(ctx, key, body) }, nil) + } +} diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/bound_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/bound_test.go new file mode 100644 index 00000000..ab1c150c --- /dev/null +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/bound_test.go @@ -0,0 +1,61 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + + "github.com/nats-io/nats.go/jetstream" + + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" + "git.novox.be/novox/mesh-sdk/go/process" + + "mesh-delivery-forge-bridge/internal/gitfixture" +) + +// openerFake is a bus that opens no bucket and records each name asked. +type openerFake struct{ asked []string } + +func (o *openerFake) KeyValue(_ context.Context, bucket string) (jetstream.KeyValue, error) { + o.asked = append(o.asked, bucket) + return nil, errors.New("no bucket in a unit test") +} + +// performedIsOpenedByItsBinding: design 47 G §577, unit (forge bridge): the state performed is opened by the bus name +// the seat mesh-broker's show answers for its local name, never by an old name; until show answers no bucket is opened, +// the use names the entry, and nothing owed is performed. +func performedIsOpenedByItsBinding(t *testing.T) { + var shows []string + answered := false + show := func(key string, body any) (json.RawMessage, error) { + b, _ := json.Marshal(body) + shows = append(shows, key+" "+string(b)) + if !answered { + return nil, errors.New("nats: no responders available for request") + } + return json.RawMessage(`{"state":"created","binding":{"local":"performed","storage":"key-value","bucket":"bound-performed"}}`), nil + } + binding := process.NewBinding(brokerSeat, forgeBridgeModule, show, nil, performedLocal) + bus := &openerFake{} + kv := &boundBucket{binding: binding, local: performedLocal, js: bus} + binding.Ask(performedLocal) + if _, err := kv.Get(context.Background(), "k"); err == nil || !strings.Contains(err.Error(), `"performed"`) || len(bus.asked) != 0 { + t.Errorf("unbound: %v, opened %v; want an error naming the entry and no bucket opened", err, bus.asked) + } + f := &seatFake{} + o, pub, kept := newOwing(f) + o.waits = binding.Waiting + gate := gitfixture.Seat(t, seat.Seat, "events", "owed is said, a merge", "event:owed")[0] + if _, err := o.heard(context.Background(), seat.Seat, gate); err == nil || len(f.calls) != 0 || len(pub.data) != 0 || len(kept) != 0 { + t.Errorf("unbound: %v, called %v, said %d; want the act left unperformed and unsaid", err, f.calls, len(pub.data)) + } + answered = true + binding.Ask(performedLocal) + _, _ = kv.Get(context.Background(), "k") + want := `seat:mesh-broker.show {"local-name":"performed","module":"mesh-delivery-forge-bridge"}` + if len(bus.asked) != 1 || bus.asked[0] != "bound-performed" || shows[len(shows)-1] != want { + t.Errorf("bound: opened %v after %v; want bound-performed, asked as %s", bus.asked, shows, want) + } +} diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/gate_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/gate_test.go index 8ab15e0c..20c20e48 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/gate_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/gate_test.go @@ -65,6 +65,7 @@ func newOwing(f *seatFake) (*owing, *fakePub, memPerformed) { // the bridge performs from the seat's groups every act said and not answered. func TestWhatIsOwedIsPerformedSaidAndKept(t *testing.T) { t.Run("the names are the manifest's", theNamesAreTheManifests) + t.Run("performed is opened by its binding", performedIsOpenedByItsBinding) gate := func(name string) []byte { return gitfixture.Seat(t, seat.Seat, "events", name, "event:owed")[0] } answer := func(verb, name string) json.RawMessage { return gitfixture.Given(t, name, "verb:"+verb+".answer") } for _, c := range []struct { diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/held.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/held.go index a0338539..a5b72909 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/held.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/held.go @@ -12,7 +12,7 @@ import ( const headsBucket = forgeBridgeModule + "_heads" // kvHeads is the bucket heads as the bridge's headStore. -type kvHeads struct{ kv jetstream.KeyValue } +type kvHeads struct{ kv keptStore } func (s kvHeads) Get(ctx context.Context, key string) (string, bool, error) { e, err := s.kv.Get(ctx, key) diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/integration_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/integration_test.go index bfb7b416..6f658a15 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/integration_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/integration_test.go @@ -39,7 +39,8 @@ import ( // From module.json: the seat git's events heard through its durable on EVENTS; what the gate and the checker owe heard // through its durable on each seat's own stream owed (SEAT__owed); the seat git's verbs, the seat mesh-delivery's // groups and the seat mesh-hand-acts' hand-act called on mesh.seat..call.. (or its plain fallback); -// its own states heads, performed, inputs, end-asks and owed-asks; its stream forge. It reads no state of the gate's or +// its own states heads, inputs, end-asks and owed-asks, and performed under the name its binding gives (the seat +// mesh-broker's show); its stream forge. It reads no state of the gate's or // the checker's (design 47, acceptance 49 and 53). `**` is written `>` on the wire. The controller's own user keeps all // rights, as it has. const busConf = ` @@ -59,12 +60,12 @@ accounts { "$JS.API.STREAM.INFO.KV_mesh-delivery-forge-bridge_heads" "$JS.FC.KV_mesh-delivery-forge-bridge_heads.>" "$KV.mesh-delivery-forge-bridge_heads.>" - "$JS.API.CONSUMER.CREATE.KV_mesh-delivery-forge-bridge_performed.>" - "$JS.API.CONSUMER.DELETE.KV_mesh-delivery-forge-bridge_performed.>" - "$JS.API.DIRECT.GET.KV_mesh-delivery-forge-bridge_performed.>" - "$JS.API.STREAM.INFO.KV_mesh-delivery-forge-bridge_performed" - "$JS.FC.KV_mesh-delivery-forge-bridge_performed.>" - "$KV.mesh-delivery-forge-bridge_performed.>" + "$JS.API.CONSUMER.CREATE.KV_mod_mesh-delivery-forge-bridge_performed_b1.>" + "$JS.API.CONSUMER.DELETE.KV_mod_mesh-delivery-forge-bridge_performed_b1.>" + "$JS.API.DIRECT.GET.KV_mod_mesh-delivery-forge-bridge_performed_b1.>" + "$JS.API.STREAM.INFO.KV_mod_mesh-delivery-forge-bridge_performed_b1" + "$JS.FC.KV_mod_mesh-delivery-forge-bridge_performed_b1.>" + "$KV.mod_mesh-delivery-forge-bridge_performed_b1.>" "$JS.API.CONSUMER.CREATE.KV_mesh-delivery-forge-bridge_inputs.>" "$JS.API.CONSUMER.DELETE.KV_mesh-delivery-forge-bridge_inputs.>" "$JS.API.DIRECT.GET.KV_mesh-delivery-forge-bridge_inputs.>" @@ -94,6 +95,18 @@ accounts { "$JS.ACK.SEAT_mesh-delivery-checker_owed.n1_mesh-delivery-forge-bridge.>" "$JS.API.CONSUMER.INFO.SEAT_mesh-delivery-checker_owed.n1_mesh-delivery-forge-bridge" "$JS.API.CONSUMER.MSG.NEXT.SEAT_mesh-delivery-checker_owed.n1_mesh-delivery-forge-bridge" + "mesh.seat.mesh-broker.call.show.n1~mesh-delivery-forge-bridge" + "mesh.seat.mesh-broker.tool.show" + "mesh.seat.mesh-catalog.call.registrations.n1~mesh-delivery-forge-bridge" + "mesh.seat.mesh-catalog.tool.registrations" + "mesh.seat.mesh-delivery.call.source.n1~mesh-delivery-forge-bridge" + "mesh.seat.mesh-delivery.tool.source" + "mesh.seat.git.call.branch.n1~mesh-delivery-forge-bridge" + "mesh.seat.git.tool.branch" + "$JS.API.STREAM.INFO.SEAT_git_branch-moved" + "$JS.ACK.SEAT_git_branch-moved.n1_mesh-delivery-forge-bridge.>" + "$JS.API.CONSUMER.INFO.SEAT_git_branch-moved.n1_mesh-delivery-forge-bridge" + "$JS.API.CONSUMER.MSG.NEXT.SEAT_git_branch-moved.n1_mesh-delivery-forge-bridge" "mesh.seat.git.call.status.n1~mesh-delivery-forge-bridge" "mesh.seat.git.tool.status" "mesh.seat.git.call.comment.n1~mesh-delivery-forge-bridge" @@ -201,6 +214,9 @@ func fileAnswer(path, ref, content string, found bool) string { return string(b) } +// boundPerformed is the bus name the stand-in seat mesh-broker's show answers for the state performed. +const boundPerformed = "mod_mesh-delivery-forge-bridge_performed_b1" + // startBed is startBridge with a step between the streams and the consumers: prepare writes what the streams hold before // the bridge's consumers are made, as the grants part makes them, from now (ADR 0443 §1), and reply sets the answer of the // seat git's open-pulls from then on. A stand-in holds the seats the bridge calls: git (every verb, trees r0 by default, @@ -226,14 +242,15 @@ func startBed(t *testing.T, openReply string, prepare func(ctx context.Context, } // Each seat's own stream owed, as its contract keeps it (Stream, SEAT__owed), as the grants part makes it: no // subjects of its own, sourced from EVENTS filtered by the event's subject. - for name, subjects := range map[string]string{seat.StreamOwed: seat.SubjectOwedAll, checker.StreamOwed: checker.SubjectOwedAll} { + for name, subjects := range map[string]string{seat.StreamOwed: seat.SubjectOwedAll, checker.StreamOwed: checker.SubjectOwedAll, git.StreamBranchMoved: git.SubjectBranchMoved} { if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: name, MaxAge: 7 * 24 * time.Hour, Sources: []*jetstream.StreamSource{{Name: "EVENTS", FilterSubject: subjects}}}); err != nil { t.Fatal(err) } } - // The bridge's states, made from the manifest's state entries and its requires before the process starts. - for _, name := range []string{headsBucket, performedBucket, inputsBucket, groupkeys.EndAsksBucket, groupkeys.OwedAsksBucket} { + // The bridge's states, made from the manifest's state entries and its requires before the process starts; the state + // performed under the bus name the seat mesh-broker's show answers for it (boundPerformed), never an old name (§577). + for _, name := range []string{headsBucket, boundPerformed, inputsBucket, groupkeys.EndAsksBucket, groupkeys.OwedAsksBucket} { if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: 1}); err != nil { t.Fatal(err) } @@ -244,11 +261,12 @@ func startBed(t *testing.T, openReply string, prepare func(ctx context.Context, acct := process.Account{Node: "n1", Module: "mesh-delivery-forge-bridge"} if _, err := js.CreateOrUpdateConsumer(ctx, "EVENTS", jetstream.ConsumerConfig{ Durable: acct.EventConsumerName(), AckPolicy: jetstream.AckExplicitPolicy, DeliverPolicy: jetstream.DeliverNewPolicy, - FilterSubjects: []string{git.SubjectPullUpdated, git.SubjectPullMerged, git.SubjectPullClosed, git.SubjectForgeDelete}, + FilterSubjects: []string{git.SubjectPullUpdated, git.SubjectPullMerged, git.SubjectPullClosed, git.SubjectForgeDelete, + "mesh.seat.mesh-catalog.event.registration-changed", "mesh.seat.mesh-delivery.event.source-moved"}, }); err != nil { t.Fatal(err) } - for name, subjects := range map[string]string{seat.StreamOwed: seat.SubjectOwedAll, checker.StreamOwed: checker.SubjectOwedAll} { + for name, subjects := range map[string]string{seat.StreamOwed: seat.SubjectOwedAll, checker.StreamOwed: checker.SubjectOwedAll, git.StreamBranchMoved: git.SubjectBranchMoved} { if _, err := js.CreateOrUpdateConsumer(ctx, name, acct.SeatEventConsumerConfig([]string{subjects}, false)); err != nil { t.Fatal(err) } @@ -259,6 +277,10 @@ func startBed(t *testing.T, openReply string, prepare func(ctx context.Context, answers := map[string]string{ "git.open-pulls": openReply, "git.trees": `{"root":"r0","trees":{"":"t0","modules/a":"t1"}}`, "mesh-delivery.groups": `{"groups":[]}`, "mesh-hand-acts.hand-act": `{"recorded":true}`, + // No registration yet: the watched branches are the gate's source's, main (design 47 G §507). + "mesh-catalog.registrations": `{"registrations":[]}`, "mesh-delivery.source": `{"branches":["main"]}`, + "git.pull": `{"error":"no pull request merged at that commit","error-kind":"not-found"}`, + "mesh-broker.show": `{"state":"created","binding":{"local":"performed","storage":"key-value","template":"state","bucket":"` + boundPerformed + `"}}`, } for _, verb := range []string{"status", "comment", "view", "note", "merge-pull"} { answers["git."+verb] = string(gitfixture.Given(t, verb+" answers", "verb:"+verb+".answer")) @@ -272,6 +294,10 @@ func startBed(t *testing.T, openReply string, prepare func(ctx context.Context, mu.Lock() called = append(called, verb+" "+string(m.Data)) answer, files := answers[verb], filesOf + var in struct{ Repository string } + if json.Unmarshal(m.Data, &in) == nil && answers[verb+" "+in.Repository] != "" { + answer = answers[verb+" "+in.Repository] // a move's branch and pull, set per repository (landedOn) + } mu.Unlock() if verb == "git.file" { var in git.FileInput @@ -318,6 +344,23 @@ func startBed(t *testing.T, openReply string, prepare func(ctx context.Context, }} } +// landedOn sets the stand-in seat git so main of repository is protected at at, which the pull request number on branch +// at prHead merged. +func landedOn(b *bed, repository string, number int, branch, prHead, at string) { + b.answer("git.branch "+repository, `{"repository":"`+repository+`","branch":"main","head":"`+at+`","protected":true}`) + b.answer("git.pull "+repository, fmt.Sprintf(`{"pull":{"repository":%q,"number":%d,"branch":%q,"head":%q,"base":"main","state":"merged","merge-commit":%q}}`, + repository, number, branch, prHead, at)) +} + +// moveMain says main of repository moved, as the seat git says it, naming a head the bridge never takes (§11). +func moveMain(t *testing.T, b *bed, repository string) { + t.Helper() + ev, _ := json.Marshal(git.BranchMovedEvent{Repository: repository, Branch: "main", Head: strings.Repeat("f", 40)}) + if _, err := b.js.Publish(b.ctx, git.SubjectBranchMoved, ev); err != nil { + t.Fatal(err) + } +} + // TestBridgeWritesPullMessages starts the program the way the node does (its entry, run, with MESH_BROKER_FILE). func TestBridgeWritesPullMessages(t *testing.T) { // 1235/AC2 ctl, js, ctx, _ := startBridge(t, `{"pulls":[]}`) @@ -444,12 +487,9 @@ func TestBridgeWritesMergedMessage(t *testing.T) { // 1236/AC4, 1448/AC5 } return "", false }) - // pull.merged as the seat git says it (its contract's PullMergedEvent). - ev, _ := json.Marshal(git.PullMergedEvent{Repository: "novox/mesh-delivery", Number: 12, Title: "t", Head: head, Branch: "feat/x", Base: "main", - MergeCommit: merge, Change: &git.Change{Paths: []string{"modules/a/x.go"}, ModuleDirs: []string{"modules/a"}}}) - if _, err := js.Publish(ctx, git.SubjectPullMerged, ev); err != nil { - t.Fatal(err) - } + // design 47 G §11, §509, §714: main moved, read by the seat's branch, and the forge names #12 merged at its head. + landedOn(bed, "novox/mesh-delivery", 12, "feat/x", head, merge) + moveMain(t, bed, "novox/mesh-delivery") forge, err := js.Stream(ctx, "FORGE") if err != nil { @@ -474,7 +514,7 @@ func TestBridgeWritesMergedMessage(t *testing.T) { // 1236/AC4, 1448/AC5 t.Fatal(err) } want := forgemsg.Forge{V: keys.Version, Kind: "merged", Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, - Merge: merge, Root: "r0", Trees: map[string]string{"": "t0", "modules/a": "t1"}, + Merge: merge, Base: "main", Root: "r0", Trees: map[string]string{"": "t0", "modules/a": "t1"}, Manifests: map[string]string{"": string(manifest), "modules/a": string(manifest)}, // No mesh package: only the indexer computes what a fingerprint takes (ADR 0593 §2). Locks: map[string]map[string]string{"modules/a": {"go.sum": hex.EncodeToString(sha256Sum(sum))}}} @@ -491,7 +531,7 @@ func TestBridgeWritesMergedMessage(t *testing.T) { // 1236/AC4, 1448/AC5 if len(trees) != 1 || trees[0] != "git.trees "+string(in) { t.Errorf("the seat git was asked for trees %v, want once at the merge commit", trees) } - cons, err := js.Consumer(ctx, "EVENTS", "n1_mesh-delivery-forge-bridge") + cons, err := js.Consumer(ctx, git.StreamBranchMoved, "n1_mesh-delivery-forge-bridge") if err != nil { t.Fatal(err) } @@ -717,14 +757,10 @@ func TestBridgeSaysAGroupFromTheSeatsGroups(t *testing.T) { // 1585, ADR 0633 § {Repository: "novox/mesh-delivery", Number: 12, Head: head, State: "merged"}, {Repository: "novox/mesh-sdk", Number: 4, Head: headB, State: "merging"}}}}}) b.answer("mesh-delivery.groups", string(groups)) ch := heard(t, b) - for _, m := range []git.PullMergedEvent{ - {Repository: "novox/mesh-delivery", Number: 12, Head: head, Branch: "feat/x", Base: "main", MergeCommit: merge, Change: &git.Change{Paths: []string{"modules/a/x.go"}, ModuleDirs: []string{"modules/a"}}}, - {Repository: "novox/mesh-sdk", Number: 4, Head: headB, Branch: "feat/x", Base: "main", MergeCommit: mergeB, Change: &git.Change{Paths: []string{"go/x.go"}}}, - } { - ev, _ := json.Marshal(m) - if _, err := b.js.Publish(b.ctx, git.SubjectPullMerged, ev); err != nil { - t.Fatal(err) - } + landedOn(b, "novox/mesh-delivery", 12, "feat/x", head, merge) + landedOn(b, "novox/mesh-sdk", 4, "feat/x", headB, mergeB) + for _, m := range []git.PullMergedEvent{{Repository: "novox/mesh-delivery"}, {Repository: "novox/mesh-sdk"}} { + moveMain(t, b, m.Repository) repo := strings.TrimPrefix(m.Repository, "novox/") f, err := forgemsg.DecodeForge(next(t, ch, "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox."+repo+".merged")) if err != nil || f.Group != "feat/x" || len(f.Members) != 2 || f.Members[1].Number != 4 { diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go index 6a9c63ea..0adb3881 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go @@ -6,6 +6,7 @@ package main import ( "context" + "encoding/json" "errors" "fmt" "io" @@ -17,14 +18,17 @@ import ( "time" "git.novox.be/novox/mesh-sdk/go/process" + "git.novox.be/novox/mesh-sdk/go/views" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" git "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git" + mcseat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" checker "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdeliverychecker" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/sourcebranches" "mesh-delivery-forge-bridge/internal/bridge" groupkeys "mesh-delivery-forge-bridge/internal/groups" @@ -68,17 +72,11 @@ type publisher interface { Publish(ctx context.Context, subject string, payload []byte, opts ...jetstream.PublishOpt) (*jetstream.PubAck, error) } -// handle turns one event into its message on FORGE and acknowledges it. An event it cannot read is terminated, never -// redelivered, and the log names its subject and why. A message it could not write stays unacknowledged: the server -// delivers it again (the server's ack wait is the only clock). -func handle(ctx context.Context, m delivered, pub publisher, trees bridge.Trees, files bridge.Files, kept bridge.Kept, logger *log.Logger) { - handleGrouped(ctx, m, pub, trees, files, kept, nil, logger) -} - -// handleGrouped is handle; a merged message of a delivery group's member names its group (g, ADR 0633 §4; nil names -// none), and the member's merged message written may end the group. -func handleGrouped(ctx context.Context, m delivered, pub publisher, trees bridge.Trees, files bridge.Files, kept bridge.Kept, g *grouper, logger *log.Logger) { - msg, err := bridge.ConvertKept(m.Subject(), m.Data(), trees, files, kept) +// handle turns one pull request event into its message on FORGE and acknowledges it (bridge.Convert). An event it cannot +// read is terminated, never redelivered, and the log names its subject and why. A message it could not write stays +// unacknowledged: the server delivers it again (the server's ack wait is the only clock). +func handle(ctx context.Context, m delivered, pub publisher, files bridge.Files, watched bridge.Watching, logger *log.Logger) { + msg, err := bridge.Convert(m.Subject(), m.Data(), files, watched) switch { case errors.Is(err, bridge.ErrRetry): logger.Printf("%v; heard again in %s", err, retryDelay) @@ -98,18 +96,10 @@ func handleGrouped(ctx context.Context, m delivered, pub publisher, trees bridge if id := m.Headers().Get("x-event-id"); id != "" { opts = append(opts, jetstream.WithMsgID(id)) } - var member forgemsg.Forge - grouped := false - if g != nil { - msg.Data, member, grouped = g.member(ctx, msg.Data) - } if _, err := pub.Publish(ctx, forgeSubject(msg.Event), msg.Data, opts...); err != nil { logger.Printf("%s: %s could not be written on FORGE, left for redelivery: %v", m.Subject(), msg.Event, err) return } - if grouped { - g.published(ctx, member) - } if msg.Said != "" { logger.Printf("%s: %s", m.Subject(), msg.Said) } @@ -147,13 +137,27 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro heads := kvHeads{kv} // Everything written on FORGE, an event's and a sync's, keeps the head in the bucket. rec := &recorder{pub: js, heads: heads, logger: logger} - said := func(ctx context.Context) error { return syncOpen(ctx, call, rec, heads, &turn, logger) } - performedKV, err := process.EssentialBucket(ctx, js, "bridge", performedBucket, "keep what it performed of what is owed to the forge") - if err != nil { + // The branches the bridge watches (design 47 G §4, §507, §532): the registrations' first view and then + // registration-changed; before the first registration, the gate's source's branches and its source-moved. Nothing is + // written before both first views are held (ready). + watched := &bridge.Watched{} + said := func(ctx context.Context) error { return syncOpen(ctx, call, rec, heads, watched, &turn, logger) } + // The module's one pull of its durable consumer on the shared events stream is its event hub (below). + hub := process.NewEventHub(logger.Printf) + // The state performed is a `requires` entry: opened by its binding, asked of the seat mesh-broker's show at this start + // as a kept call and again on its recreated and dropped (design 47 G §577). Until show answers, nothing owed is + // performed and the bridge says it waits, naming the entry. + binding := process.NewBinding(brokerSeat, forgeBridgeModule, keptCallOf(ctx, call), logger.Printf, performedLocal) + if err := binding.Register(hub); err != nil { return err } + binding.Ask(performedLocal) + if w := binding.Waiting(); w != "" { + logger.Printf("the bridge %s: nothing owed to the forge is performed until it is bound", w) + } + performedKV := &boundBucket{binding: binding, local: performedLocal, js: js} // What the gate and the checker owe the forge, performed through the seat git (design 47 *What the gate owes the forge*). - owe := &owing{call: call, pub: js, performed: kvPerformed{performedKV}, now: time.Now, logger: logger} + owe := &owing{call: call, pub: js, performed: kvPerformed{performedKV}, waits: binding.Waiting, now: time.Now, logger: logger} // A delivery group's member names its group, and its end is said once (ADR 0633 §4), from the seat mesh-delivery's groups. groups := &grouper{groups: groupsVia(call), send: sendVia(js), done: kvHeads{performedKV}, logger: logger} // A person says a group's end again with bridge_group_end_again, which writes the state end-asks (ADR 0633 §4): watched, and @@ -166,9 +170,91 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } }() } + keptCall := keptCallOf(ctx, call) + reader := &views.ProcessReader{Module: forgeBridgeModule, Call: keptCall, Logf: logger.Printf, Hub: hub} + src := sourcebranches.New(sourcebranches.Call(keptCall)) + var regs *meshcatalogclient.Registrations + var mv *mover + recompute := func() { + if regs != nil { + watched.Set(registeredOf(regs.All()), unescaped(src.Branches())) + } + } + started := make(chan struct{}) // closed once the watched branches are read and the open pull requests said + regs, _, err = meshcatalogclient.WireRegistrationsProcess(reader, keptCall, func(mcseat.Registration, bool) { + recompute() + select { + case <-started: // a branch registered while the bridge runs is adopted: its head read (§532) + go mv.readHeads(ctx) + default: + } + }) + if err != nil { + return err + } + src.Follower.Changed = func(string, json.RawMessage, bool, time.Time) { recompute() } + if _, err := reader.Adopt(src.Follower, sourcebranches.Seat, sourcebranches.EventSourceMoved, src.Hear); err != nil { + return err + } + reader.Start(ctx, js, cred, nil) + // What a merge read of each module directory, by its tree (issue 1126): a merge reads only the directories it changed. + inputs := newKept(process.OptionalBucket(ctx, js, inputsBucket, "a merge's module directories are kept in memory only, so the first merge after a start reads every one", logger.Printf), logger) + mv = &mover{call: call, pub: rec, heads: heads, watched: watched, inputs: inputs, groups: groups, turn: &turn, logger: logger} + // The hub's handler of the forge's events, for every event its consumes filter lets through. Essential: the forge's + // events are what the bridge carries. The forge's events are the seat git's (novox/hq design 47 *The seat `git`*), + // whichever module holds it. + if err := hub.Handle("mesh.seat."+git.Seat+".event.>", func(m jetstream.Msg) { + select { + case <-started: + case <-ctx.Done(): + return + } + hear(&turn, func() { + // Bounded: the turn is held, and no sync or other event goes on without it. + hctx, cancel := context.WithTimeout(ctx, syncDeadline) + defer cancel() + handle(hctx, m, rec, filesVia(hctx, call), watched, logger) + }) + }); err != nil { + return err + } + if err := hub.Start(ctx, js, cred); err != nil { + return fmt.Errorf("%w: the bridge hears no forge event without it, so it stops; where the bus refused it or did not answer, this module's consumes (made by the controller with its grant) have not reached the bus yet", err) + } + defer hub.Stop() + // Every move of a branch, heard on the seat git's stream branch-moved through this module's durable there (§11), so a + // move said while the bridge was stopped is heard at its start. Essential: every merge and push reaches the stream so. + moves, err := cred.HearSeatEvents(ctx, js, git.StreamBranchMoved, func(m jetstream.Msg) { + select { + case <-started: + case <-ctx.Done(): + return + } + hctx, cancel := context.WithTimeout(ctx, syncDeadline) + defer cancel() + mv.hear(hctx, m) + }, logger.Printf) + if err != nil { + return fmt.Errorf("the seat git's branch-moved cannot be heard, so no merge or push reaches the forge stream: %w", err) + } + defer moves.Stop() + select { + case <-regs.Ready(): + case <-ctx.Done(): + return nil + } + if len(regs.All()) == 0 { + select { + case <-src.Follower.Ready(): + case <-ctx.Done(): + return nil + } + } + recompute() if err := said(ctx); err != nil { return err } + close(started) // What the gate and the checker say they owe, each on its seat's own stream owed through this module's durable there // (made from its consumes). A stream not there yet is said, and the bridge goes on: its other work does not wait on the // gate, and what was said meanwhile is caught up from the groups at the next start. @@ -207,26 +293,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro logger.Printf("%d acts owed to the forge, said while the bridge was away, performed from the groups", n) } go onReconnect(ctx, again, said, logger) - // What a merge read of each module directory, by its tree (issue 1126): a merge reads only the directories it changed. - inputs := newKept(process.OptionalBucket(ctx, js, inputsBucket, "a merge's module directories are kept in memory only, so the first merge after a start reads every one", logger.Printf), logger) - // The module's one pull of its durable consumer on the shared events stream is its event hub; the bridge has the one - // handler, for every event its consumes filter lets through. Essential: the forge's events are what the bridge carries. - hub := process.NewEventHub(logger.Printf) - // The forge's events are the seat git's (novox/hq design 47 *The seat `git`*), whichever module holds it. - if err := hub.Handle("mesh.seat."+git.Seat+".event.>", func(m jetstream.Msg) { - hear(&turn, func() { - // Bounded: the turn is held, and no sync or other event goes on without it. - hctx, cancel := context.WithTimeout(ctx, syncDeadline) - defer cancel() - handleGrouped(hctx, m, rec, treesVia(hctx, call), filesVia(hctx, call), inputs, groups, logger) - }) - }); err != nil { - return err - } - if err := hub.Start(ctx, js, cred); err != nil { - return fmt.Errorf("%w: the bridge hears no forge event without it, so it stops; where the bus refused it or did not answer, this module's consumes (made by the controller with its grant) have not reached the bus yet", err) - } - defer hub.Stop() + // Each watched branch's head, read at the start (§11): a head missed while the bridge was stopped is written now. + mv.readHeads(ctx) <-ctx.Done() return nil } diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go index da888759..b4874634 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go @@ -79,7 +79,7 @@ func TestReadableEventIsWrittenOnForgeAndAcked(t *testing.T) { // 1235/AC2 (unit }) var logs bytes.Buffer m, pub := &fakeMsg{subject: git.SubjectPullUpdated, data: body(nil)}, &fakePub{} - handle(context.Background(), m, pub, nil, someFiles, nil, log.New(&logs, "", 0)) + handle(context.Background(), m, pub, someFiles, watchesMain(), log.New(&logs, "", 0)) if len(pub.subjects) != 1 || pub.subjects[0] != "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.pull" { t.Fatalf("published on %v", pub.subjects) } @@ -95,7 +95,7 @@ func TestReadableEventIsWrittenOnForgeAndAcked(t *testing.T) { // 1235/AC2 (unit func TestUnreadableEventIsTerminatedAndLogged(t *testing.T) { // 1235/AC5 var logs bytes.Buffer m, pub := &fakeMsg{subject: git.SubjectPullUpdated, data: body(map[string]any{"head_sha": "short"})}, &fakePub{} - handle(context.Background(), m, pub, nil, someFiles, nil, log.New(&logs, "", 0)) + handle(context.Background(), m, pub, someFiles, watchesMain(), log.New(&logs, "", 0)) if len(pub.subjects) != 0 { t.Errorf("a message was written: %v", pub.subjects) } @@ -109,7 +109,7 @@ func TestUnreadableEventIsTerminatedAndLogged(t *testing.T) { // 1235/AC5 // from a test of its own, for the test budget. logs.Reset() m, pub = &fakeMsg{subject: git.SubjectPullUpdated, data: body(nil)}, &fakePub{err: errors.New("no stream")} - handle(context.Background(), m, pub, nil, someFiles, nil, log.New(&logs, "", 0)) + handle(context.Background(), m, pub, someFiles, watchesMain(), log.New(&logs, "", 0)) if m.acked != 0 || m.termd != 0 { t.Errorf("acked %d terminated %d, want neither", m.acked, m.termd) } @@ -118,55 +118,93 @@ func TestUnreadableEventIsTerminatedAndLogged(t *testing.T) { // 1235/AC5 } } -// mergedEvent is the seat git's pull.merged. -func mergedEvent() []byte { - b, _ := json.Marshal(map[string]any{"repository": "novox/mesh-delivery", "number": 12, "branch": "feat/x", "head": head, - "base": "main", "merge-commit": "89abcdef0123456789abcdef0123456789abcdef"}) +// watchesMain watches main of every repository, as the gate's source names it before the first registration. +func watchesMain() *bridge.Watched { + w := &bridge.Watched{} + w.Set(nil, []string{"main"}) + return w +} + +// movedEvent is the seat git's branch-moved of main of novox/mesh-delivery, naming a head the bridge never takes. +func movedEvent(branch string) []byte { + b, _ := json.Marshal(map[string]any{"repository": "novox/mesh-delivery", "branch": branch, "head": strings.Repeat("f", 40)}) return b } +const landed = "89abcdef0123456789abcdef0123456789abcdef" + +// moveCall answers the seat git as a forge whose main of novox/mesh-delivery is protected at landed, merged there by #12; +// trees answers trees, every other verb answers err. +func moveCall(trees json.RawMessage, err error, keys *[]string, args *[]any, mu *sync.Mutex) caller { + return func(_ context.Context, key string, body any) (json.RawMessage, error) { + mu.Lock() + *keys, *args = append(*keys, key), append(*args, body) + mu.Unlock() + switch key { + case "seat:git.branch": + return json.RawMessage(`{"repository":"novox/mesh-delivery","branch":"main","head":"` + landed + `","protected":true}`), nil + case "seat:git.pull": + return json.RawMessage(`{"pull":{"repository":"novox/mesh-delivery","number":12,"branch":"feat/x","head":"` + head + `","base":"main","state":"merged","merge-commit":"` + landed + `"}}`), nil + case "seat:git.trees": + if trees != nil { + return trees, nil + } + } + return nil, err + } +} + +func newMover(call caller, pub publisher, heads headStore, logs *bytes.Buffer) *mover { + return &mover{call: call, pub: pub, heads: heads, watched: watchesMain(), turn: &sync.Mutex{}, logger: log.New(logs, "", 0)} +} + func TestTreesRefusedLeavesTheEventForTenSecondsAndLogs(t *testing.T) { // 1236/AC3 var logs bytes.Buffer - m, pub := &fakeMsg{subject: git.SubjectPullMerged, data: mergedEvent()}, &fakePub{} - call := func(context.Context, string, any) (json.RawMessage, error) { - return nil, errors.New("the tree listing of novox/mesh-delivery was cut") - } - handle(context.Background(), m, pub, treesVia(context.Background(), call), filesVia(context.Background(), call), nil, log.New(&logs, "", 0)) - if len(pub.subjects) != 0 { + var keys []string + var args []any + var mu sync.Mutex + m, pub, heads := &fakeMsg{subject: bridge.BranchMovedSubject, data: movedEvent("main")}, &fakePub{}, newHeads() + newMover(moveCall(nil, errors.New("the tree listing of novox/mesh-delivery was cut"), &keys, &args, &mu), pub, heads, &logs).hear(context.Background(), m) + if len(pub.subjects) != 0 || heads.get("branch.novox.mesh-delivery.main") != "" { t.Errorf("a message was written: %v", pub.subjects) } if m.acked != 0 || m.termd != 0 || len(m.nak) != 1 || m.nak[0] < 10*time.Second { t.Errorf("acked %d terminated %d naks %v, want one nak of at least 10 s", m.acked, m.termd, m.nak) } - for _, w := range []string{"novox/mesh-delivery", "89abcdef0123456789abcdef0123456789abcdef", "the tree listing of novox/mesh-delivery was cut"} { + for _, w := range []string{"novox/mesh-delivery", landed, "the tree listing of novox/mesh-delivery was cut"} { if !strings.Contains(logs.String(), w) { t.Errorf("log %q does not name %q", logs.String(), w) } } } -func TestMergeCallsTheToolOnceAndWritesTheMergedMessage(t *testing.T) { // 1236/AC1; design 47 *The seat git* (verbs trees, file) +// 1236/AC1; design 47 *The seat git* (verbs trees, file); design 47 G §11, §20, §509 (FB1, FB10): a move of a watched branch +// is taken by the seat's verb branch, never the event's head, written once as merged with its pull request and base, and +// its head kept, so the same move heard again writes nothing; a move of a branch not watched writes nothing. +func TestMergeCallsTheToolOnceAndWritesTheMergedMessage(t *testing.T) { var logs bytes.Buffer - m, pub := &fakeMsg{subject: git.SubjectPullMerged, data: mergedEvent()}, &fakePub{} var keys []string var args []any var mu sync.Mutex // the files are asked for concurrently (issue 1458) trees := gitfixture.Given(t, "trees answers", "verb:trees.answer") // root ddd…, "" ddd…, contracts eee… + call := moveCall(trees, nil, &keys, &args, &mu) file := gitfixture.Given(t, "file answers", "verb:file.answer") - call := func(_ context.Context, key string, body any) (json.RawMessage, error) { - mu.Lock() - keys, args = append(keys, key), append(args, body) - mu.Unlock() + files := func(ctx context.Context, key string, body any) (json.RawMessage, error) { if key == "seat:git.file" { // the manifests, 1448; the trees are what this test is about + mu.Lock() + keys = append(keys, key) + mu.Unlock() return file, nil } - return trees, nil + return call(ctx, key, body) } - handle(context.Background(), m, pub, treesVia(context.Background(), call), filesVia(context.Background(), call), nil, log.New(&logs, "", 0)) - if len(keys) == 0 || keys[0] != "seat:git.trees" || slices.Index(keys[1:], "seat:git.trees") >= 0 { - t.Fatalf("calls %v, want the trees once and first", keys) + m, pub, heads := &fakeMsg{subject: bridge.BranchMovedSubject, data: movedEvent("main")}, &fakePub{}, newHeads() + mv := newMover(files, pub, heads, &logs) + mv.hear(context.Background(), m) + if len(keys) < 3 || keys[0] != "seat:git.branch" || keys[1] != "seat:git.pull" || keys[2] != "seat:git.trees" || slices.Index(keys[3:], "seat:git.trees") >= 0 { + t.Fatalf("calls %v, want the branch, its pull, then the trees once", keys) } - if got, _ := json.Marshal(args[0]); !sameJSON(got, []byte(`{"repository":"novox/mesh-delivery","commit":"89abcdef0123456789abcdef0123456789abcdef"}`)) { + if got, _ := json.Marshal(args[2]); !sameJSON(got, []byte(`{"repository":"novox/mesh-delivery","commit":"`+landed+`"}`)) { t.Errorf("arguments %s", got) } if len(pub.subjects) != 1 || pub.subjects[0] != "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.merged" { @@ -174,11 +212,17 @@ func TestMergeCallsTheToolOnceAndWritesTheMergedMessage(t *testing.T) { // 1236/ } f, err := forgemsg.DecodeForge(pub.data[0]) d, e := strings.Repeat("d", 40), strings.Repeat("e", 40) - if err != nil || f.Root != d || f.Trees[""] != d || f.Trees["contracts"] != e || f.Merge == "" { + if err != nil || f.Root != d || f.Trees[""] != d || f.Trees["contracts"] != e || f.Merge != landed || f.Base != "main" || f.Number != 12 || f.Branch != "feat/x" { t.Errorf("message %+v err %v", f, err) } - if m.acked != 1 { - t.Errorf("acked %d", m.acked) + if m.acked != 1 || heads.get("branch.novox.mesh-delivery.main") != landed { + t.Errorf("acked %d, head kept %q", m.acked, heads.get("branch.novox.mesh-delivery.main")) + } + again, other := &fakeMsg{subject: bridge.BranchMovedSubject, data: movedEvent("main")}, &fakeMsg{subject: bridge.BranchMovedSubject, data: movedEvent("feat/x")} + mv.hear(context.Background(), again) + mv.hear(context.Background(), other) + if len(pub.subjects) != 1 || again.acked != 1 || other.acked != 1 { + t.Errorf("the same move again and a move of a branch not watched wrote %d messages (acked %d, %d), want none", len(pub.subjects)-1, again.acked, other.acked) } } diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/moves.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/moves.go new file mode 100644 index 00000000..40b6c060 --- /dev/null +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/moves.go @@ -0,0 +1,143 @@ +package main + +import ( + "context" + "errors" + "fmt" + "log" + "strings" + "sync" + + "github.com/nats-io/nats.go/jetstream" + + mcseat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" + "git.novox.be/novox/mesh-sdk/go/keys" + + "mesh-delivery-forge-bridge/internal/bridge" +) + +// mover writes the one message of each move of a watched branch (design 47 G §11, §509, §714): heard on the seat git's +// branch-moved through the bridge's durable on SEAT_git_branch-moved, the head read by the seat's verb branch, never the +// event's; merged with its pull request, or pushed; the head written kept in the state heads under the branch's key +// (A-236), so a move heard twice writes once and the next pushed says its head before. +type mover struct { + call caller + pub publisher + heads headStore + watched *bridge.Watched + inputs bridge.Kept + groups *grouper + turn *sync.Mutex + logger *log.Logger +} + +// hear takes one branch-moved: one it cannot read is terminated, one of a branch not watched is acknowledged with nothing +// written, one the forge could not say about is heard again. +func (mv *mover) hear(ctx context.Context, m delivered) { + moved, err := bridge.MovedOf(m.Data()) + if err != nil { + mv.logger.Printf("%v; the event is dropped", err) + _ = m.Term() + return + } + if !mv.watched.Watches(moved.Repository(), moved.Branch) { + _ = m.Ack() + return + } + mv.turn.Lock() + err = mv.move(ctx, moved) + mv.turn.Unlock() + switch { + case errors.Is(err, bridge.ErrUnreadable): + mv.logger.Printf("%v; the event is dropped", err) + _ = m.Term() + case err != nil: + mv.logger.Printf("%v; heard again in %s", err, retryDelay) + _ = m.NakWithDelay(retryDelay) + default: + _ = m.Ack() + } +} + +// move reads the branch's head and writes its message, unless that head is the one last written for it. +func (mv *mover) move(ctx context.Context, moved bridge.Moved) error { + b, err := bridge.ReadBranch(ctx, bridge.Caller(mv.call), moved) + if err != nil { + return fmt.Errorf("%s of %s: %w", moved.Branch, moved.Repository(), err) + } + before, _, err := mv.heads.Get(ctx, moved.HeadKey()) + if err != nil { + return fmt.Errorf("%s of %s: the head last written cannot be read: %w", moved.Branch, moved.Repository(), err) + } + if before == b.Head { + return nil + } + if !b.Protected { + // A branch whose protection was removed starts nothing: nothing is written (§17, §509). + mv.logger.Printf("%s of %s moved to %s while it is not protected: nothing is written", moved.Branch, moved.Repository(), b.Head) + return nil + } + msg, err := bridge.MoveMessage(ctx, bridge.Caller(mv.call), moved, b.Head, before, treesVia(ctx, mv.call), filesVia(ctx, mv.call), mv.inputs) + if err != nil { + return err + } + data, grouped := msg.Data, false + var member forgemsg.Forge + if mv.groups != nil { + data, member, grouped = mv.groups.member(ctx, msg.Data) + } + // The branch's new head makes a write after a lost acknowledgement the same message to the stream. + id := "moved." + keys.EscapeName(moved.Repository()) + "." + keys.EscapeName(moved.Branch) + "." + b.Head + if _, err := mv.pub.Publish(ctx, forgeSubject(msg.Event), data, jetstream.WithMsgID(id)); err != nil { + return fmt.Errorf("%s could not be written on FORGE: %w", msg.Event, err) + } + if grouped { + mv.groups.published(ctx, member) + } + if err := mv.heads.Put(ctx, moved.HeadKey(), b.Head); err != nil { + mv.logger.Printf("%s of %s: the head %s written is not kept, so the next move says the older head before: %v", moved.Branch, moved.Repository(), b.Head, err) + } + if msg.Said != "" { + mv.logger.Printf("%s", msg.Said) + } + return nil +} + +// readHeads reads each watched branch's head (§11): a head missed while the bridge was stopped is written now. +func (mv *mover) readHeads(ctx context.Context) { + for _, repo := range mv.watched.Repositories() { + owner, name, ok := strings.Cut(repo, "/") + if !ok { + continue + } + for _, branch := range mv.watched.Branches(repo) { + mv.turn.Lock() + err := mv.move(ctx, bridge.Moved{Owner: owner, Repo: name, Branch: branch}) + mv.turn.Unlock() + if err != nil { + mv.logger.Printf("the head of %s of %s is not read at this start, so a move missed meanwhile waits for its next move: %v", branch, repo, err) + } + } + } +} + +// registeredOf is the registrations as the watched branches read them. +func registeredOf(regs []mcseat.Registration) []bridge.Registered { + out := make([]bridge.Registered, 0, len(regs)) + for _, r := range regs { + out = append(out, bridge.Registered{Origin: r.Origin, Branches: r.Branches}) + } + return out +} + +// unescaped is the gate's source's branches by name (they are kept by their escaped name). +func unescaped(branches []string) []string { + out := make([]string, 0, len(branches)) + for _, b := range branches { + if name, err := keys.UnescapeName(b); err == nil { + out = append(out, name) + } + } + return out +} diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/names_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/names_test.go index f2e9e83f..43162be9 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/names_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/names_test.go @@ -44,7 +44,7 @@ func theNamesAreTheManifests(t *testing.T) { own = append(own, forgeBridgeModule+"_"+kept.Name) } } - for _, b := range []string{headsBucket, performedBucket, inputsBucket} { + for _, b := range []string{headsBucket, inputsBucket} { if !slices.Contains(own, b) { t.Errorf("the bridge keeps %s, its manifest's states are %v", b, own) } diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/open.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/open.go index 9441a818..826d5295 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/open.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/open.go @@ -80,7 +80,7 @@ func hear(mu *sync.Mutex, react func()) { // ADR 0443 §5). pub keeps the bucket as it writes (recorder). It holds mu from before it reads the forge until the open // message is written, so no event is heard between, and never past syncDeadline. A bucket that cannot be read is logged // and every open pull request is written. An error names the call and the forge's words. -func syncOpen(ctx context.Context, call caller, pub publisher, heads headStore, mu *sync.Mutex, logger *log.Logger) error { +func syncOpen(ctx context.Context, call caller, pub publisher, heads headStore, watched bridge.Watching, mu *sync.Mutex, logger *log.Logger) error { mu.Lock() defer mu.Unlock() ctx, cancel := context.WithTimeout(ctx, syncDeadline) @@ -89,11 +89,11 @@ func syncOpen(ctx context.Context, call caller, pub publisher, heads headStore, if err != nil { return fmt.Errorf("%s: %w", openPullsTool, err) } - msgs, skipped, err := bridge.OpenPulls(raw) + msgs, skipped, err := bridge.OpenPulls(raw, watched) if err != nil { return fmt.Errorf("%s: %w", openPullsTool, err) } - open, err := bridge.OpenMessage(raw) + open, err := bridge.OpenMessage(raw, watched) if err != nil { return fmt.Errorf("%s: %w", openPullsTool, err) } diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/open_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/open_test.go index 5861b9b7..a10a62f3 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/open_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/open_test.go @@ -90,7 +90,7 @@ func (m *memHeads) get(key string) string { m.mu.Lock(); defer m.mu.Unlock(); re // runSync runs syncOpen with the recorder the program writes through. func runSync(call caller, pub publisher, heads headStore, mu *sync.Mutex, logger *log.Logger) error { - return syncOpen(context.Background(), call, &recorder{pub: pub, heads: heads, logger: logger}, heads, mu, logger) + return syncOpen(context.Background(), call, &recorder{pub: pub, heads: heads, logger: logger}, heads, watchesMain(), mu, logger) } // 1279/AC6: the sync writes a pull message only for a head that differs from what FORGE holds, then one open message diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/owed.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/owed.go index 9384e1c4..0ff09c15 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/owed.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/owed.go @@ -25,11 +25,10 @@ import ( "mesh-delivery-forge-bridge/internal/groups" ) -// performedBucket is the module's state performed (manifest `requires`, the template state): one key per owed act it +// The state performed (manifest `requires`, the template state, opened by its binding: bound.go) keeps one key per owed act it // performed, the act's outcome and when, with the owed event as heard (novox/hq design 47 *What the gate owes the // forge*). It is a saving and a read, never what keeps an act from happening twice: every act is safe to repeat at the // forge. The group ends' bookkeeping (`published.`, `groups.`, groups.go) is in it too, under the same 7 days. -const performedBucket = forgeBridgeModule + "_performed" // performedKey is the key of one act in performed: the seat that said it and its id, the id in unpadded base64url, since // an id may hold what a key may not (`#`, `/`). @@ -50,7 +49,7 @@ type performedStore interface { Put(ctx context.Context, key string, value []byte) error } -type kvPerformed struct{ kv jetstream.KeyValue } +type kvPerformed struct{ kv keptStore } func (s kvPerformed) Get(ctx context.Context, key string) ([]byte, bool, error) { e, err := s.kv.Get(ctx, key) @@ -75,8 +74,10 @@ type owing struct { call caller pub publisher performed performedStore - now func() time.Time - logger *log.Logger + // waits, when set, says why the state performed cannot be used yet (its binding, design 47 G §577); empty when it can. + waits func() string + now func() time.Time + logger *log.Logger } // readerOf is the reading of the seat of's owed event into the seat git's verb. @@ -102,6 +103,9 @@ func (o *owing) heard(ctx context.Context, of string, data []byte) (forgemsg.Owe return o.end(ctx, forgemsg.OwedEnded{Of: of, ID: c.ID, Outcome: forgemsg.OwedUndecodable, Words: err.Error()}, data) } + if w := o.waiting(); w != "" { + return forgemsg.OwedEnded{Of: c.Of, ID: c.ID}, fmt.Errorf("the bridge %s", w) + } if o.held(ctx, c.Of, c.ID) { // said twice, or said again while its outcome was on its way: performed once, the outcome said once return forgemsg.OwedEnded{Of: c.Of, ID: c.ID}, nil @@ -145,6 +149,14 @@ func (o *owing) hear(ctx context.Context, of string, m delivered) { _ = m.Ack() } +// waiting is why the state performed cannot be used yet, or empty. +func (o *owing) waiting() string { + if o.waits == nil { + return "" + } + return o.waits() +} + // perform calls the seat git's verb of one act and says what it came to. func (o *owing) perform(ctx context.Context, c bridge.OwedCall, data []byte) (forgemsg.OwedEnded, error) { answer, callErr := o.call(ctx, "seat:"+git.Seat+"."+c.Verb, c.Input) @@ -197,6 +209,9 @@ func (o *owing) held(ctx context.Context, of, id string) bool { // catchUp reads the seat mesh-delivery's groups and performs every act said there with no recorded outcome that // performed does not hold either (design 47 *The bridge catches up from the seat*). It answers how many it performed. func (o *owing) catchUp(ctx context.Context) (int, error) { + if w := o.waiting(); w != "" { + return 0, fmt.Errorf("the bridge %s", w) + } of, err := groups.Read(ctx, o.call) if err != nil { return 0, err diff --git a/modules/mesh-delivery-forge-bridge/forgemsg/forge.go b/modules/mesh-delivery-forge-bridge/forgemsg/forge.go index ccba891b..027a891c 100644 --- a/modules/mesh-delivery-forge-bridge/forgemsg/forge.go +++ b/modules/mesh-delivery-forge-bridge/forgemsg/forge.go @@ -24,8 +24,11 @@ const ForgeStream = "forge" // The kinds of forge message, the last part of a forge event's name. const ( - ForgePull = "pull" - ForgeMerged = "merged" + ForgePull = "pull" + ForgeMerged = "merged" + // ForgePushed is a move of a watched branch the forge names no pull request's merge for (design 47 G §509): an + // admitted push, taken in as a merged message is. + ForgePushed = "pushed" ForgeRefused = "refused" ForgeOpen = "open" ) @@ -67,10 +70,10 @@ func checkRepository(repository string) error { func checkForgeKind(kind string) error { switch kind { - case ForgePull, ForgeMerged, ForgeRefused, ForgeOpen: + case ForgePull, ForgeMerged, ForgePushed, ForgeRefused, ForgeOpen: return nil } - return fmt.Errorf("delivery: the forge kind %q is not pull, merged, refused or open", kind) + return fmt.Errorf("delivery: the forge kind %q is not pull, merged, pushed, refused or open", kind) } // ForgeEvent is the name of a forge message of the given kind: `forge...`. @@ -111,18 +114,24 @@ func ParseForgeEvent(event string) (repository, kind string, err error) { // DirsKnown, true when the forge could say them: known and empty is DirsKnown with no Dirs, not known is neither // (the forge's module_dirs and module_dirs_said, mesh-catalog modules/gitea/trunk.ts). Other messages carry neither. type Forge struct { - V int `json:"v"` - Kind string `json:"kind"` - Repository string `json:"repository"` - Branch string `json:"branch,omitempty"` - Number int `json:"number,omitempty"` - Head string `json:"head,omitempty"` - Action string `json:"action,omitempty"` - Dirs []string `json:"dirs,omitempty"` - DirsKnown bool `json:"dirs-known,omitempty"` - Merge string `json:"merge,omitempty"` - Trees map[string]string `json:"trees,omitempty"` - Root string `json:"root,omitempty"` + V int `json:"v"` + Kind string `json:"kind"` + Repository string `json:"repository"` + // Branch is a pull or merged message's pull request branch, and a pushed message's branch pushed. + Branch string `json:"branch,omitempty"` + // Base is the watched branch a merged message's commit landed on (design 47 G §714). + Base string `json:"base,omitempty"` + Number int `json:"number,omitempty"` + // Head is a pull or merged message's pull request head commit, and a pushed message's new head of its branch (A-236). + Head string `json:"head,omitempty"` + // Before is a pushed message's head of its branch before, as the forge bridge last wrote it (A-236), empty for none. + Before string `json:"before,omitempty"` + Action string `json:"action,omitempty"` + Dirs []string `json:"dirs,omitempty"` + DirsKnown bool `json:"dirs-known,omitempty"` + Merge string `json:"merge,omitempty"` + Trees map[string]string `json:"trees,omitempty"` + Root string `json:"root,omitempty"` // Manifests are the module.json bytes, exact, of each module directory: of the merge's trees on a merged message, // of Dirs at the head on a pull message. Locks are, of each such directory, its lock files' hex sha256 by path // relative to it (novox/hq ADR 0465 clause 1). Older writers send neither. @@ -203,8 +212,8 @@ func (f Forge) checkPredefined() error { if len(f.Predefined) == 0 && !f.PredefinedRead { return nil } - if f.Kind != ForgeMerged { - return fmt.Errorf("delivery: only a merged message carries the predefined list, not a %s message", f.Kind) + if f.Kind != ForgeMerged && f.Kind != ForgePushed { + return fmt.Errorf("delivery: only a merged or pushed message carries the predefined list, not a %s message", f.Kind) } if len(f.Predefined) > 0 && f.PredefinedRead { return fmt.Errorf("delivery: a merged message carries the predefined list and says the merge holds none (predefined-read)") @@ -262,21 +271,27 @@ func (f Forge) check() error { if err := f.checkInputs(func(d string) bool { return dirs[d] }, "its module directories"); err != nil { return err } - case ForgeMerged: + case ForgeMerged, ForgePushed: if len(f.Dirs) > 0 || f.DirsKnown { - return fmt.Errorf("delivery: a merged message carries no module directories") + return fmt.Errorf("delivery: a %s message carries no module directories", f.Kind) } - if f.Number <= 0 { + if f.Kind == ForgeMerged && f.Number <= 0 { return fmt.Errorf("delivery: a merged message needs its pull request number") } - if f.Head == "" { - return fmt.Errorf("delivery: a merged message needs its head") + if f.Kind == ForgePushed && (f.Number != 0 || f.Merge != "" || f.Base != "") { + return fmt.Errorf("delivery: a pushed message names no pull request, merge commit or base: its branch is the one pushed") } - if f.Merge == "" { + if f.Kind == ForgePushed && f.Branch == "" { + return fmt.Errorf("delivery: a pushed message needs its branch") + } + if f.Head == "" { + return fmt.Errorf("delivery: a %s message needs its head", f.Kind) + } + if f.Kind == ForgeMerged && f.Merge == "" { return fmt.Errorf("delivery: a merged message needs its merge commit") } if f.Root == "" { - return fmt.Errorf("delivery: a merged message needs the repository's root tree") + return fmt.Errorf("delivery: a %s message needs the repository's root tree", f.Kind) } // No trees is a merge that leaves the repository with no module.json anywhere (a documents repository, or one // whose last module was removed): it is written all the same, so the indexer takes the merge in and drops the diff --git a/modules/mesh-delivery-forge-bridge/forgemsg/forge_test.go b/modules/mesh-delivery-forge-bridge/forgemsg/forge_test.go index 41fde65b..c31695e5 100644 --- a/modules/mesh-delivery-forge-bridge/forgemsg/forge_test.go +++ b/modules/mesh-delivery-forge-bridge/forgemsg/forge_test.go @@ -23,6 +23,10 @@ func sampleForges() []Forge { // 1585/T1: a group member's merged message names its group and every approved member (ADR 0633 §4). {Kind: ForgeMerged, Repository: "novox/mesh-sdk", Number: 37, Head: "dddd", Merge: "eeee", Root: "t0", Group: "feat/take-in", Members: []meshdeliveryclient.Head{{Repository: "novox/mesh-sdk", Number: 37, Head: "dddd"}, {Repository: "novox/mesh-delivery", Number: 9, Head: "ffff"}}}, + // design 47 G §509, §714, A-236: a merged message names the watched branch it landed on; a pushed one its branch, new + // head and head before. + {Kind: ForgeMerged, Repository: "novox/mesh-sdk", Number: 38, Branch: "feat/y", Head: "aaaa", Merge: "bbbb", Base: "beta", Root: "t0"}, + {Kind: ForgePushed, Repository: "novox/mesh-sdk", Branch: "beta", Head: "cccc", Before: "bbbb", Root: "t0", Trees: map[string]string{"modules/a": "t1"}}, } } @@ -153,7 +157,7 @@ func TestForge(t *testing.T) { {"assignments read", "mesh-module-assigner", in(func(f *Forge) { f.RetiredRead = true })}, {"predefined not an object", "one JSON object", in(func(f *Forge) { f.Predefined = []byte(`["domain"]`) })}, {"predefined both", "says the merge holds none", in(func(f *Forge) { f.Predefined = []byte(`{}`); f.PredefinedRead = true })}, - {"pull with the predefined list", "only a merged message", Forge{Kind: ForgePull, Repository: "novox/mesh-sdk", Number: 1, Head: "h", Action: PullOpened, PredefinedRead: true}}, + {"pull with the predefined list", "only a merged or pushed message", Forge{Kind: ForgePull, Repository: "novox/mesh-sdk", Number: 1, Head: "h", Action: PullOpened, PredefinedRead: true}}, {"pull manifest outside dirs", `"b"`, pullIn(func(f *Forge) { f.Manifests["b"] = "{}" })}, {"pull locks outside dirs", `"c"`, pullIn(func(f *Forge) { f.Locks["c"] = map[string]string{"go.sum": "h"} })}, {"refused with manifests", "no manifests", Forge{Kind: ForgeRefused, Repository: "a/b", Act: "x", Words: "no", Manifests: map[string]string{"a": "{}"}}}, @@ -167,6 +171,9 @@ func TestForge(t *testing.T) { {"merged with members leaving itself out", "leave out", at(4, func(f *Forge) { f.Members = f.Members[1:] })}, {"merged with a member twice", "twice", at(4, func(f *Forge) { f.Members = append(f.Members, f.Members[0]) })}, {"open with a group", "only the open pull requests", Forge{Kind: ForgeOpen, Group: "g"}}, + // design 47 G §509, A-236: a pushed message names its branch and no pull request, merge commit or base. + {"pushed with a pull request", "names no pull request", Forge{Kind: ForgePushed, Repository: "a/b", Branch: "beta", Head: "h", Root: "r", Number: 3}}, + {"pushed with no branch", "needs its branch", Forge{Kind: ForgePushed, Repository: "a/b", Head: "h", Root: "r"}}, } { if _, err := EncodeForge(c.f); err == nil || !strings.Contains(err.Error(), c.want) { t.Errorf("%s: %v, want a refusal naming %q", c.name, err, c.want) diff --git a/modules/mesh-delivery-forge-bridge/go.mod b/modules/mesh-delivery-forge-bridge/go.mod index e8f3740d..ea862348 100644 --- a/modules/mesh-delivery-forge-bridge/go.mod +++ b/modules/mesh-delivery-forge-bridge/go.mod @@ -4,7 +4,9 @@ go 1.26.0 require ( git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdeliverychecker v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery v0.0.0-00010101000000-000000000000 diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go b/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go index d27bb022..21648ba3 100644 --- a/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go @@ -32,8 +32,10 @@ var ErrNotHandled = errors.New("event not handled") // heard again later. var ErrRetry = errors.New("the forge could not say") -// Trunk is the branch a merged message is written for. -const Trunk = "main" +// Watching says whether the bridge watches a branch of a repository (Watched; design 47 G §507, §532). +type Watching interface { + Watches(repository, branch string) bool +} // Trees asks the forge for the trees of a commit: the root tree and the tree of every directory holding a module.json // (the root directory spelled ""), as the seat git's verb trees answers (issue 1234/AC4). @@ -93,7 +95,7 @@ type BranchDeleted struct { } // branchDeleted turns forge.delete into a branch-deleted message; a deleted tag is not handled. -func branchDeleted(subject string, data []byte) (Message, error) { +func branchDeleted(subject string, data []byte, watched Watching) (Message, error) { var e git.ForgeDeleteEvent if err := json.Unmarshal(data, &e); err != nil { return Message{}, unreadable(subject, "the body is not json: %v", err) @@ -109,8 +111,8 @@ func branchDeleted(subject string, data []byte) (Message, error) { return Message{}, unreadable(subject, "no owner or repo a forge event can name (%q/%q)", d.Owner, d.Repo) case branch == "": return Message{}, unreadable(subject, "no branch") - case branch == Trunk: - return Message{}, fmt.Errorf("%w: %s of the trunk %s", ErrNotHandled, subject, Trunk) + case watched.Watches(d.Owner+"/"+d.Repo, branch): + return Message{}, fmt.Errorf("%w: %s of the watched branch %s", ErrNotHandled, subject, branch) } body, err := json.Marshal(BranchDeleted{V: keys.Version, Kind: ForgeBranchDeleted, Repository: d.Owner + "/" + d.Repo, Branch: branch}) if err != nil { @@ -170,29 +172,20 @@ func unreadable(subject, why string, args ...any) error { return fmt.Errorf("%w: %s: %s", ErrUnreadable, subject, fmt.Sprintf(why, args...)) } -// Convert turns one event of the forge module, by its bus subject and body, into a message. An event it cannot read -// is an ErrUnreadable error naming the subject and why; an event it writes nothing for is ErrNotHandled. A merge into -// the trunk asks trees once and becomes a merged message; a merge into any other branch is a closed pull message, as the -// pull request left its group. A forge that cannot say is ErrRetry. A merge that leaves no module.json anywhere is a merged message with no -// trees all the same (issue 1126): the indexer takes it in and drops the repository's modules, and the view reads the -// repository as reached at it. -func Convert(subject string, data []byte, trees Trees, files Files) (Message, error) { - return ConvertKept(subject, data, trees, files, nil) -} - -// ConvertKept is Convert; with kept, a merge reads the forge only for the module directories whose tree hash kept -// does not hold, and keeps what it read (kept.go). -func ConvertKept(subject string, data []byte, trees Trees, files Files, kept Kept) (Message, error) { +// Convert turns one pull request event of the seat git, by its bus subject and body, into a message. An event it cannot +// read is an ErrUnreadable error naming the subject and why; an event it writes nothing for is ErrNotHandled. A pull +// request into a branch the bridge does not watch is no candidate: written closed. A merge into a watched branch writes +// nothing here: its merged message is the move's, heard on branch-moved (MoveMessage; design 47 G §509, §714); a merge +// into another branch is a closed pull message, as the pull request left its group. A forge that cannot say is ErrRetry. +func Convert(subject string, data []byte, files Files, watched Watching) (Message, error) { var action string switch subject { case UpdatedSubject: action = forgemsg.PullUpdated - case ClosedSubject: + case ClosedSubject, MergedSubject: action = forgemsg.PullClosed - case MergedSubject: - action = forgemsg.PullClosed // unless it merged into the trunk, below case DeletedSubject: - return branchDeleted(subject, data) + return branchDeleted(subject, data, watched) default: return Message{}, fmt.Errorf("%w: %s", ErrNotHandled, subject) } @@ -210,17 +203,17 @@ func ConvertKept(subject string, data []byte, trees Trees, files Files, kept Kep case p.Head == "": return Message{}, unreadable(subject, "no branch") } - if subject == MergedSubject && p.Base == Trunk { - return merged(subject, p, trees, files, kept) + into := p.Base != "" && watched.Watches(p.Owner+"/"+p.Repo, p.Base) + if subject == MergedSubject && into { + return Message{}, fmt.Errorf("%w: %s of %s/%s#%d into the watched branch %s: its merged message is written from the branch's move", ErrNotHandled, subject, p.Owner, p.Repo, p.Number, p.Base) } said := "" - if action == forgemsg.PullUpdated && p.Base != "" && p.Base != Trunk { - // The checker judges pull requests into main only, and source.main names main's head only (novox/hq ADR 0484 §11): a - // pull request into another base is no candidate, as it is none at the start (OpenPulls) and at its merge. It is - // written closed, so a candidate an older head made is taken out, and said (issue 1422). + if action == forgemsg.PullUpdated && p.Base != "" && !into { + // A pull request into a branch no registration watches is no candidate, as it is none at the start (OpenPulls) and at + // its merge. It is written closed, so a candidate an older head made is taken out, and said (issue 1422). action = forgemsg.PullClosed - said = fmt.Sprintf("%s/%s#%d at %s is into %s, not %s: written closed, so it is no candidate; the checker judges pull requests into %s only (novox/hq ADR 0484 §11)", - p.Owner, p.Repo, p.Number, p.HeadSHA[:12], p.Base, Trunk, Trunk) + said = fmt.Sprintf("%s/%s#%d at %s is into %s, which is not watched: written closed, so it is no candidate", + p.Owner, p.Repo, p.Number, p.HeadSHA[:12], p.Base) } f := forgemsg.Forge{Kind: forgemsg.ForgePull, Repository: p.Owner + "/" + p.Repo, Branch: p.Head, Number: p.Number, Head: p.HeadSHA, Action: action} @@ -419,15 +412,17 @@ func fetch(owner, repo, ref string, reads []*read, files Files) { wg.Wait() } -// merged is the merged message of a merge into the trunk: the trees of the merge commit, asked once. -func merged(subject string, p pull, trees Trees, files Files, kept Kept) (Message, error) { - if !sha40.MatchString(p.MergeSHA) { - return Message{}, unreadable(subject, "the merge commit %q is not 40 hex digits", p.MergeSHA) +// landed is the message of a commit that landed on a watched branch, merged or pushed (f says which and what names it): +// the trees of the commit, asked once, and the take-in inputs of every module directory at it (design 47 G §509, A-236). +func landed(what string, f forgemsg.Forge, commit string, trees Trees, files Files, kept Kept) (Message, error) { + if !sha40.MatchString(commit) { + return Message{}, unreadable(what, "the commit %q is not 40 hex digits", commit) } - repository := p.Owner + "/" + p.Repo - root, dirs, err := trees(p.Owner, p.Repo, p.MergeSHA) + repository := f.Repository + owner, repo, _ := strings.Cut(repository, "/") + root, dirs, err := trees(owner, repo, commit) if err != nil { - return Message{}, fmt.Errorf("%w: %s: the trees of %s at the merge commit %s: %v", ErrRetry, subject, repository, p.MergeSHA, err) + return Message{}, fmt.Errorf("%w: %s: the trees of %s at %s: %v", ErrRetry, what, repository, commit, err) } names := make([]string, 0, len(dirs)) for dir := range dirs { @@ -435,10 +430,10 @@ func merged(subject string, p pull, trees Trees, files Files, kept Kept) (Messag } slices.Sort(names) manifests, locks, needs, toRead := keptInputs(names, dirs, kept) - read, readLocks, readNeeds, whole, err := inputsRead(p.Owner, p.Repo, p.MergeSHA, toRead, files) + read, readLocks, readNeeds, whole, err := inputsRead(owner, repo, commit, toRead, files) if err != nil { keep(whole, dirs, read, readLocks, readNeeds, kept) // what was read whole is not asked again at the next try - return Message{}, fmt.Errorf("%w: %s: the manifests of %s at the merge commit %s (%d of %d module directories read from the forge, the others kept by their tree): %v", ErrRetry, subject, repository, p.MergeSHA, len(toRead), len(names), err) + return Message{}, fmt.Errorf("%w: %s: the manifests of %s at %s (%d of %d module directories read from the forge, the others kept by their tree): %v", ErrRetry, what, repository, commit, len(toRead), len(names), err) } keep(toRead, dirs, read, readLocks, readNeeds, kept) maps.Copy(manifests, read) @@ -453,31 +448,30 @@ func merged(subject string, p pull, trees Trees, files Files, kept Kept) (Messag if len(needs) == 0 { needs = nil } - event, err := forgemsg.ForgeEvent(repository, forgemsg.ForgeMerged) + event, err := forgemsg.ForgeEvent(repository, f.Kind) if err != nil { - return Message{}, unreadable(subject, "%v", err) + return Message{}, unreadable(what, "%v", err) } - f := forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: repository, Branch: p.Head, Number: p.Number, - Head: p.HeadSHA, Merge: p.MergeSHA, Root: root, Trees: dirs, Manifests: manifests, Locks: locks, Needs: needs} + f.Root, f.Trees, f.Manifests, f.Locks, f.Needs = root, dirs, manifests, locks, needs if repository == CatalogRepository { - // The catalogue's predefined list at the merge (novox/hq ADR 0565 §2): the indexer keeps it for mesh-config. - raw, err := files(p.Owner, p.Repo, PredefinedFile, p.MergeSHA) + // The catalogue's predefined list at the commit (novox/hq ADR 0565 §2): the indexer keeps it for mesh-config. + raw, err := files(owner, repo, PredefinedFile, commit) switch { case errors.Is(err, ErrNotFound): f.PredefinedRead = true case err != nil: - return Message{}, fmt.Errorf("%w: %s: %s of %s at the merge commit %s: %v", ErrRetry, subject, PredefinedFile, repository, p.MergeSHA, err) + return Message{}, fmt.Errorf("%w: %s: %s of %s at %s: %v", ErrRetry, what, PredefinedFile, repository, commit, err) default: f.Predefined = json.RawMessage(raw) } } body, err := forgemsg.EncodeForge(f) if err != nil { - return Message{}, unreadable(subject, "%v", err) + return Message{}, unreadable(what, "%v", err) } msg := Message{Event: event, Data: body} if len(dirs) == 0 { - msg.Said = fmt.Sprintf("%s at the merge commit %s holds no module.json: a merged message with no module directories", repository, p.MergeSHA) + msg.Said = fmt.Sprintf("%s at %s holds no module.json: a %s message with no module directories", repository, commit, f.Kind) } return msg, nil } @@ -506,16 +500,16 @@ func openPulls(answer []byte) ([]pull, error) { } // OpenPulls turns the seat git's answer of open-pulls into one pull message per open pull request into -// the trunk, action updated and its directories not known (the forge's answer does not say them). A pull request into +// a watched branch, action updated and its directories not known (the forge's answer does not say them). A pull request into // another branch gets none. An entry that cannot be read is named in skipped and gets none; an answer that is not // the contract's, or that says an error, is an error. -func OpenPulls(answer []byte) (msgs []Message, skipped []string, err error) { +func OpenPulls(answer []byte, watched Watching) (msgs []Message, skipped []string, err error) { pulls, err := openPulls(answer) if err != nil { return nil, nil, err } for _, p := range pulls { - if p.Base != Trunk { + if !watched.Watches(p.Owner+"/"+p.Repo, p.Base) { continue } name := fmt.Sprintf("%s/%s#%d", p.Owner, p.Repo, p.Number) @@ -553,17 +547,17 @@ func Track(f forgemsg.Forge) (key, head string, ok bool) { return "", "", false } -// OpenMessage turns the seat git's answer of open-pulls into the open message: every open pull request into the trunk, by +// OpenMessage turns the seat git's answer of open-pulls into the open message: every open pull request into a watched branch, by // repository and number, whether its head reads or not and whether FORGE holds it or not, in repository and number // order. A pull request into another branch is not named. An answer that names none is a message too: nothing is open. -func OpenMessage(answer []byte) (Message, error) { +func OpenMessage(answer []byte, watched Watching) (Message, error) { pulls, err := openPulls(answer) if err != nil { return Message{}, err } open := []forgemsg.OpenPull{} for _, p := range pulls { - if p.Base != Trunk || p.Owner == "" || p.Repo == "" || p.Number <= 0 { + if !watched.Watches(p.Owner+"/"+p.Repo, p.Base) || p.Owner == "" || p.Repo == "" || p.Number <= 0 { continue } open = append(open, forgemsg.OpenPull{Repository: p.Owner + "/" + p.Repo, Number: p.Number}) diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/bridge_test.go b/modules/mesh-delivery-forge-bridge/internal/bridge/bridge_test.go index 5b6b2efb..dc0aaa35 100644 --- a/modules/mesh-delivery-forge-bridge/internal/bridge/bridge_test.go +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/bridge_test.go @@ -68,7 +68,7 @@ const updatedSubject = "mesh.seat.git.event.pull-updated" const closedSubject = "mesh.seat.git.event.pull-closed" func TestUpdatedBecomesAPullMessage(t *testing.T) { // 1235/AC2 (the conversion; the bus is the integration test) - msg, err := Convert(updatedSubject, updated(nil), nil, anyFiles) + msg, err := Convert(updatedSubject, updated(nil), anyFiles, watchesMain) if err != nil { t.Fatal(err) } @@ -86,7 +86,7 @@ func TestUpdatedBecomesAPullMessage(t *testing.T) { // 1235/AC2 (the conversion; } // 1235/AC3, merged into this test for the test budget: dirs not said are not known. - msg, err = Convert(updatedSubject, updated(map[string]any{"module_dirs": nil, "module_dirs_said": nil}), nil, anyFiles) + msg, err = Convert(updatedSubject, updated(map[string]any{"module_dirs": nil, "module_dirs_said": nil}), anyFiles, watchesMain) if err != nil { t.Fatal(err) } @@ -95,7 +95,7 @@ func TestUpdatedBecomesAPullMessage(t *testing.T) { // 1235/AC2 (the conversion; t.Errorf("dirs = %v known = %v, want not known", f.Dirs, f.DirsKnown) } // Said and empty is known and empty, not unknown. - msg, err = Convert(updatedSubject, updated(map[string]any{"module_dirs": []string{}, "module_dirs_said": true}), nil, anyFiles) + msg, err = Convert(updatedSubject, updated(map[string]any{"module_dirs": []string{}, "module_dirs_said": true}), anyFiles, watchesMain) if err != nil { t.Fatal(err) } @@ -104,7 +104,7 @@ func TestUpdatedBecomesAPullMessage(t *testing.T) { // 1235/AC2 (the conversion; t.Errorf("dirs = %v known = %v, want known and empty", f.Dirs, f.DirsKnown) } // Dirs listed without the forge saying it could read them are not trusted. - msg, err = Convert(updatedSubject, updated(map[string]any{"module_dirs_said": nil}), nil, anyFiles) + msg, err = Convert(updatedSubject, updated(map[string]any{"module_dirs_said": nil}), anyFiles, watchesMain) if err != nil { t.Fatal(err) } @@ -114,7 +114,7 @@ func TestUpdatedBecomesAPullMessage(t *testing.T) { // 1235/AC2 (the conversion; } func TestClosedBecomesAClosedPullMessage(t *testing.T) { // 1235/AC4 - msg, err := Convert(closedSubject, closed(), nil, anyFiles) + msg, err := Convert(closedSubject, closed(), anyFiles, watchesMain) if err != nil { t.Fatal(err) } @@ -130,17 +130,17 @@ func TestClosedBecomesAClosedPullMessage(t *testing.T) { // 1235/AC4 if !forgemsg.SameForge(f, want) { t.Errorf("message = %+v, want %+v", f, want) } - // Issue 1422, ADR 0484 §11: a head pushed to a pull request into another base is written closed, read nothing for, and said; - // one into main is no closed message, and an event naming no base is written as before. - msg, err = Convert(updatedSubject, updated(map[string]any{"base": "nox-mesh"}), nil, nil) + // Issue 1422; design 47 G §507: a head pushed to a pull request into a base no registration watches is written closed, + // read nothing for, and said; one into a watched branch is no closed message, and an event naming no base is written as before. + msg, err = Convert(updatedSubject, updated(map[string]any{"base": "nox-mesh"}), nil, watchesMain) if err != nil { t.Fatal(err) } - if f, err = forgemsg.DecodeForge(msg.Data); err != nil || !forgemsg.SameForge(f, want) || !strings.Contains(msg.Said, "is into nox-mesh, not main") { + if f, err = forgemsg.DecodeForge(msg.Data); err != nil || !forgemsg.SameForge(f, want) || !strings.Contains(msg.Said, "is into nox-mesh, which is not watched") { t.Errorf("an update into nox-mesh is %+v (%v), said %q; want it closed and said", f, err, msg.Said) } for _, base := range []any{"main", nil} { - msg, err = Convert(updatedSubject, updated(map[string]any{"base": base}), nil, anyFiles) + msg, err = Convert(updatedSubject, updated(map[string]any{"base": base}), anyFiles, watchesMain) if f, _ = forgemsg.DecodeForge(msg.Data); err != nil || f.Action != forgemsg.PullUpdated || msg.Said != "" { t.Errorf("an update with base %v is %+v (%v), said %q; want it updated", base, f, err, msg.Said) } @@ -165,7 +165,7 @@ func TestUnreadableEventsWriteNothing(t *testing.T) { // 1235/AC5 "closed no head": {closedSubject, mustDrop(closed(), "head"), "head"}, "closed no repo": {closedSubject, mustDrop(closed(), "repository"), "repo"}, } { - msg, err := Convert(c.subject, c.data, nil, anyFiles) + msg, err := Convert(c.subject, c.data, anyFiles, watchesMain) if err == nil { t.Errorf("%s: a message %q was made", name, msg.Event) continue @@ -192,7 +192,7 @@ func mustDrop(body []byte, key string) []byte { func TestOtherSubjectsAreNotHandled(t *testing.T) { // 1235, 1236 for _, s := range []string{"mesh.seat.git.event.something.else", "mesh.mod.gitea.event.pull.updated", "mesh.mod.other.event.forge.delete"} { - if _, err := Convert(s, updated(nil), nil, anyFiles); !errors.Is(err, ErrNotHandled) { + if _, err := Convert(s, updated(nil), anyFiles, watchesMain); !errors.Is(err, ErrNotHandled) { t.Errorf("%s: err = %v, want ErrNotHandled", s, err) } } @@ -202,23 +202,23 @@ func TestOtherSubjectsAreNotHandled(t *testing.T) { // 1235, 1236 b, _ := json.Marshal(map[string]any{"repository": "novox/mesh-sdk", "ref": ref, "ref-type": refType, "sender": "jochen"}) return b } - msg, err := Convert(DeletedSubject, del("branch", "feat/x"), nil, anyFiles) + msg, err := Convert(DeletedSubject, del("branch", "feat/x"), anyFiles, watchesMain) var bd BranchDeleted if err != nil || msg.Event != "forge.novox.mesh-sdk.branch-deleted" || json.Unmarshal(msg.Data, &bd) != nil || bd.Kind != ForgeBranchDeleted || bd.Repository != "novox/mesh-sdk" || bd.Branch != "feat/x" || bd.Head != "" || msg.Said == "" { t.Errorf("forge.delete of a branch = %+v (%s), %v", msg, msg.Data, err) } for _, c := range []struct{ refType, ref string }{{"tag", "go/v0.1.13"}, {"branch", "main"}} { - if _, err := Convert(DeletedSubject, del(c.refType, c.ref), nil, anyFiles); !errors.Is(err, ErrNotHandled) { + if _, err := Convert(DeletedSubject, del(c.refType, c.ref), anyFiles, watchesMain); !errors.Is(err, ErrNotHandled) { t.Errorf("forge.delete of %s %s: err = %v, want ErrNotHandled", c.refType, c.ref, err) } } - if _, err := Convert(DeletedSubject, []byte(`{"ref":"x","ref-type":"branch"}`), nil, anyFiles); !errors.Is(err, ErrUnreadable) { + if _, err := Convert(DeletedSubject, []byte(`{"ref":"x","ref-type":"branch"}`), anyFiles, watchesMain); !errors.Is(err, ErrUnreadable) { t.Errorf("forge.delete naming no repository: err = %v, want ErrUnreadable", err) } // 1236: a merged event without a full merge commit is unreadable, and the forge is not asked. var calls []call - _, err = Convert(mergedSubject, mergedBody(map[string]any{"merge_commit_sha": "abc"}), treesOf(&calls, "r0", map[string]string{"a": "t"}, nil), anyFiles) + _, err = convertMerged(mergedBody(map[string]any{"merge_commit_sha": "abc"}), treesOf(&calls, "r0", map[string]string{"a": "t"}, nil), anyFiles, nil) if !errors.Is(err, ErrUnreadable) || len(calls) != 0 { t.Errorf("merged without a merge commit: err = %v calls = %v", err, calls) } @@ -247,7 +247,7 @@ func treesOf(calls *[]call, root string, trees map[string]string, err error) Tre func TestMergeIntoMainBecomesAMergedMessage(t *testing.T) { // 1236/AC1 var calls []call answer := map[string]string{"modules/a": "t1", "modules/b": "t2"} - msg, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", answer, nil), anyFiles) + msg, err := convertMerged(mergedBody(nil), treesOf(&calls, "r0", answer, nil), anyFiles, nil) if err != nil { t.Fatal(err) } @@ -259,7 +259,7 @@ func TestMergeIntoMainBecomesAMergedMessage(t *testing.T) { // 1236/AC1 t.Fatal(err) } want := forgemsg.Forge{V: keys.Version, Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, - Head: head, Merge: merge, Root: "r0", Trees: answer, Manifests: map[string]string{"modules/a": "{}", "modules/b": "{}"}} + Head: head, Merge: merge, Base: "main", Root: "r0", Trees: answer, Manifests: map[string]string{"modules/a": "{}", "modules/b": "{}"}} if !forgemsg.SameForge(f, want) { t.Errorf("message = %+v, want %+v", f, want) } @@ -268,9 +268,14 @@ func TestMergeIntoMainBecomesAMergedMessage(t *testing.T) { // 1236/AC1 } } -func TestMergeIntoAnotherBranchIsAClosedPull(t *testing.T) { // 1236/AC2 +// 1236/AC2; design 47 G §509, §714: a pull request merged into a branch no registration watches is a closed pull, and one +// merged into a watched branch writes nothing here: its one message is its branch's move (MoveMessage). +func TestMergeIntoAnotherBranchIsAClosedPull(t *testing.T) { var calls []call - msg, err := Convert(mergedSubject, mergedBody(map[string]any{"base": "feat/y"}), treesOf(&calls, "r0", map[string]string{"a": "t"}, nil), anyFiles) + if _, err := Convert(mergedSubject, mergedBody(nil), anyFiles, watchesMain); !errors.Is(err, ErrNotHandled) { + t.Errorf("a merge into the watched main: %v, want nothing written (ErrNotHandled)", err) + } + msg, err := Convert(mergedSubject, mergedBody(map[string]any{"base": "feat/y"}), anyFiles, watchesMain) if err != nil { t.Fatal(err) } @@ -290,7 +295,7 @@ func TestMergeIntoAnotherBranchIsAClosedPull(t *testing.T) { // 1236/AC2 func TestTreesRefusedWritesNothingAndSaysWhy(t *testing.T) { // 1236/AC3 (the conversion; the delay is in main_test) var calls []call - msg, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "", nil, errors.New("the listing for novox/mesh-delivery was cut")), anyFiles) + msg, err := convertMerged(mergedBody(nil), treesOf(&calls, "", nil, errors.New("the listing for novox/mesh-delivery was cut")), anyFiles, nil) if !errors.Is(err, ErrRetry) || msg.Data != nil { t.Fatalf("err = %v data = %s, want ErrRetry and no message", err, msg.Data) } @@ -305,13 +310,13 @@ func TestTreesRefusedWritesNothingAndSaysWhy(t *testing.T) { // 1236/AC3 (the co // message with no trees, so the indexer takes it in and the repository's reach is not held for ever. func TestNoModuleJSONAnywhereWritesAMergedMessageWithNoTrees(t *testing.T) { var calls []call - msg, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", map[string]string{}, nil), anyFiles) + msg, err := convertMerged(mergedBody(nil), treesOf(&calls, "r0", map[string]string{}, nil), anyFiles, nil) if err != nil { t.Fatal(err) } f, err := forgemsg.DecodeForge(msg.Data) want := forgemsg.Forge{V: keys.Version, Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, - Head: head, Merge: merge, Root: "r0"} + Head: head, Merge: merge, Base: "main", Root: "r0"} if err != nil || msg.Event != "forge.novox.mesh-delivery.merged" || !forgemsg.SameForge(f, want) || !strings.Contains(msg.Said, "no module.json") { t.Errorf("event %q message %+v said %q, %v; want a merged message with no trees", msg.Event, f, msg.Said, err) } @@ -330,7 +335,7 @@ func openEntry(repo string, number int, base, branch, sha string) map[string]any func TestOpenPullsAreUpdatedPullMessagesWithUnknownDirs(t *testing.T) { // 1237/AC1 h2 := strings.Repeat("a", 40) - msgs, skipped, err := OpenPulls(openAnswer(openEntry("mesh-delivery", 12, "main", "feat/x", head), openEntry("mesh-catalog", 300, "main", "feat/x", h2))) + msgs, skipped, err := OpenPulls(openAnswer(openEntry("mesh-delivery", 12, "main", "feat/x", head), openEntry("mesh-catalog", 300, "main", "feat/x", h2)), watchesMain) if err != nil || len(skipped) != 0 || len(msgs) != 2 { t.Fatalf("msgs %d skipped %v err %v", len(msgs), skipped, err) } @@ -353,15 +358,15 @@ func TestOpenPullsAreUpdatedPullMessagesWithUnknownDirs(t *testing.T) { // 1237/ func TestOpenPullsUnreadableEntryIsNamedAndTheRestGo(t *testing.T) { // 1237, 1237/AC4 // 1237/AC4: an open pull request into another branch gets no message. - if msgs, skipped, err := OpenPulls(openAnswer(openEntry("mesh-delivery", 12, "feat/y", "feat/x", head))); err != nil || len(msgs) != 0 || len(skipped) != 0 { + if msgs, skipped, err := OpenPulls(openAnswer(openEntry("mesh-delivery", 12, "feat/y", "feat/x", head)), watchesMain); err != nil || len(msgs) != 0 || len(skipped) != 0 { t.Errorf("into another branch: msgs %d skipped %v err %v, want none", len(msgs), skipped, err) } - msgs, skipped, err := OpenPulls(openAnswer(openEntry("a", 1, "main", "b", "short"), openEntry("mesh-delivery", 12, "main", "feat/x", head))) + msgs, skipped, err := OpenPulls(openAnswer(openEntry("a", 1, "main", "b", "short"), openEntry("mesh-delivery", 12, "main", "feat/x", head)), watchesMain) if err != nil || len(msgs) != 1 || len(skipped) != 1 || !strings.Contains(skipped[0], "novox/a#1") { t.Errorf("msgs %d skipped %v err %v", len(msgs), skipped, err) } for _, bad := range []string{`{"pulls":null}`, `{}`, `[]`, `x`, string(gitFixture(t, "open-pulls answers an error, never none", "verb:open-pulls.answer"))} { - if _, _, err := OpenPulls([]byte(bad)); err == nil { + if _, _, err := OpenPulls([]byte(bad), watchesMain); err == nil { t.Errorf("%s was taken as an answer", bad) } } @@ -371,7 +376,7 @@ func TestOpenMessageNamesEveryOpenPullIntoTheTrunk(t *testing.T) { // 1279/AC6 h2 := strings.Repeat("a", 40) answer := openAnswer(openEntry("mesh-delivery", 12, "main", "feat/x", head), openEntry("mesh-catalog", 300, "main", "feat/x", h2), openEntry("mesh-catalog", 301, "feat/y", "feat/z", h2), openEntry("a", 1, "main", "b", "short")) - m, err := OpenMessage(answer) + m, err := OpenMessage(answer, watchesMain) if err != nil || m.Event != "forge.open" { t.Fatalf("message %+v err %v", m, err) } @@ -382,12 +387,12 @@ func TestOpenMessageNamesEveryOpenPullIntoTheTrunk(t *testing.T) { // 1279/AC6 t.Errorf("open message = %+v err %v, want %v", f, err, want) } // Nothing open is a message too: the indexer then drops every candidate. - m, err = OpenMessage([]byte(`{"pulls":[]}`)) + m, err = OpenMessage([]byte(`{"pulls":[]}`), watchesMain) if f, derr := forgemsg.DecodeForge(m.Data); err != nil || derr != nil || f.Kind != forgemsg.ForgeOpen || len(f.Open) != 0 { t.Errorf("empty answer: %+v %v %v", f, err, derr) } for _, bad := range []string{`{"pulls":null}`, `{}`, `x`, `{"error":"the forge answered 502","pulls":[]}`} { - if _, err := OpenMessage([]byte(bad)); err == nil { + if _, err := OpenMessage([]byte(bad), watchesMain); err == nil { t.Errorf("%s was taken as an answer", bad) } } @@ -490,3 +495,25 @@ func slicesEqual(a, b []string) bool { } return true } + +// watchesMain watches the branch main of every repository, as the registrations of a test name it. +var watchesMain = watchedOf("main") + +// watchedOf watches the branches of every repository, before the first registration, as the gate's source names them. +func watchedOf(branches ...string) *Watched { + w := &Watched{} + w.Set(nil, branches) + return w +} + +// convertMerged is the merged message of a pull.merged body as the move of its base writes it (landed, MoveMessage), so the +// tests of what a merge reads keep their bodies. +func convertMerged(data []byte, trees Trees, files Files, kept Kept) (Message, error) { + p, err := pullOf(data) + if err != nil { + return Message{}, unreadable(MergedSubject, "%v", err) + } + f := forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: p.Owner + "/" + p.Repo, Branch: p.Head, Number: p.Number, + Head: p.HeadSHA, Merge: p.MergeSHA, Base: p.Base} + return landed(MergedSubject, f, p.MergeSHA, trees, files, kept) +} diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/call.go b/modules/mesh-delivery-forge-bridge/internal/bridge/call.go index 012fdea5..4c51d447 100644 --- a/modules/mesh-delivery-forge-bridge/internal/bridge/call.go +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/call.go @@ -8,6 +8,7 @@ import ( "strings" git "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" ) // Caller calls a seat's verb by its manifest key (`seat:.`) with a JSON body and returns its answer: the @@ -87,15 +88,17 @@ func FilesVia(ctx context.Context, call Caller) Files { // errNoAnswer is a read whose answer carries nothing the contract says it must. var errNoAnswer = errors.New("the seat git's answer carries nothing") -// MergedMessage is the merged message of a merge into the trunk, asked again: built as the bridge builds the one it writes -// on FORGE (it is that event's body through ConvertKept), from the pull request's owner, repository, number, branch, head -// and merge commit, which the seat git's verb pull answers. It reads the forge for every module directory (no kept inputs). -func MergedMessage(owner, repo string, number int, branch, head, merge string, trees Trees, files Files) (Message, error) { - body, err := json.Marshal(git.PullMergedEvent{Repository: repository(owner, repo), Number: int64(number), Branch: branch, Head: head, Base: Trunk, MergeCommit: merge}) - if err != nil { - return Message{}, err +// MergedMessage is the merged message of a merge into the branch base, asked again: built as the bridge builds the one +// it writes on FORGE for the branch's move (MoveMessage), from the pull request's owner, repository, number, branch, head, +// merge commit and base, which the seat git's verb pull answers. It reads the forge for every module directory (no kept +// inputs). +func MergedMessage(owner, repo string, number int, branch, head, merge, base string, trees Trees, files Files) (Message, error) { + if number <= 0 || branch == "" || base == "" || !sha40.MatchString(head) { + return Message{}, unreadable("bridge_merged_message", "%s#%d names no branch, base or 40-hex head", repository(owner, repo), number) } - return ConvertKept(MergedSubject, body, trees, files, nil) + f := forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: repository(owner, repo), Branch: branch, Number: number, Head: head, + Merge: merge, Base: base} + return landed(repository(owner, repo)+"#"+fmt.Sprint(number), f, merge, trees, files, nil) } // splitRepository is a `/` cut in two, refusing anything else. diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/files_test.go b/modules/mesh-delivery-forge-bridge/internal/bridge/files_test.go index aab23cd9..09b4418e 100644 --- a/modules/mesh-delivery-forge-bridge/internal/bridge/files_test.go +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/files_test.go @@ -66,7 +66,7 @@ func TestMergedCarriesManifestsAndLocks(t *testing.T) { // 1448/AC1 manifest, sum := real(t, "module.json"), real(t, "go.sum") forge := &forgeFiles{have: map[string][]byte{"modules/a/module.json": manifest, "modules/a/go.sum": sum, "modules/b/module.json": []byte(`{"module":"b"}`)}} var calls []call - msg, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", map[string]string{"modules/a": "t1", "modules/b": "t2"}, nil), forge.get) + msg, err := convertMerged(mergedBody(nil), treesOf(&calls, "r0", map[string]string{"modules/a": "t1", "modules/b": "t2"}, nil), forge.get, nil) if err != nil { t.Fatal(err) } @@ -87,7 +87,7 @@ func TestMergedCarriesManifestsAndLocks(t *testing.T) { // 1448/AC1 // ADR 0567: assignments is the seat mesh-module-assigner's: a merge of the catalogue carries no assignments file and asks the // forge for none. forge.calls = nil - msg, err = Convert(mergedSubject, mergedBody(map[string]any{"repo": "mesh-catalog"}), treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get) + msg, err = convertMerged(mergedBody(map[string]any{"repo": "mesh-catalog"}), treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get, nil) if err != nil { t.Fatal(err) } @@ -108,19 +108,19 @@ func TestMergedCarriesManifestsAndLocks(t *testing.T) { // 1448/AC1 t.Errorf("a merge of the catalogue with no predefined.json does not say so: %s", msg.Data) } forge.have[PredefinedFile] = []byte(`{"font-size":{"type":"integer","value":13}}`) - msg, _ = Convert(mergedSubject, mergedBody(map[string]any{"repo": "mesh-catalog"}), treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get) + msg, _ = convertMerged(mergedBody(map[string]any{"repo": "mesh-catalog"}), treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get, nil) if f, err = forgemsg.DecodeForge(msg.Data); err != nil || string(f.Predefined) != string(forge.have[PredefinedFile]) || f.PredefinedRead { t.Errorf("predefined = %s (%v)", f.Predefined, err) } forge.calls = nil - msg, _ = Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get) + msg, _ = convertMerged(mergedBody(nil), treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get, nil) for _, c := range forge.calls { if c.path == PredefinedFile { t.Errorf("another repository's merge asked for %s", c.path) } } forge.errs = map[string]error{PredefinedFile: errors.New("forge down")} - if _, err := Convert(mergedSubject, mergedBody(map[string]any{"repo": "mesh-catalog"}), treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get); !errors.Is(err, ErrRetry) { + if _, err := convertMerged(mergedBody(map[string]any{"repo": "mesh-catalog"}), treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get, nil); !errors.Is(err, ErrRetry) { t.Errorf("a forge that cannot say is %v, want a retry", err) } // ADR 0632 §2: of a manifest naming a template, the template is read at the merge and the module's needs carried @@ -130,7 +130,7 @@ func TestMergedCarriesManifestsAndLocks(t *testing.T) { // 1448/AC1 forge.have["modules/d/module.json"] = []byte(`{"module":"d","version":"1","resources":[{"id":"daemon","type":"file",` + `"path":"/etc/d.json","template":"files/daemon.json"},{"id":"gone","type":"file","path":"/etc/g","template":"files/gone"}]}`) forge.have["modules/d/files/daemon.json"] = []byte(`{"registry":"${seat:container-image-registry:reach}"}`) - msg, err = Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", map[string]string{"modules/b": "t2", "modules/d": "t4"}, nil), forge.get) + msg, err = convertMerged(mergedBody(nil), treesOf(&calls, "r0", map[string]string{"modules/b": "t2", "modules/d": "t4"}, nil), forge.get, nil) if err != nil { t.Fatal(err) } @@ -144,7 +144,7 @@ func TestMergedCarriesManifestsAndLocks(t *testing.T) { // 1448/AC1 func TestPullWithKnownDirsCarriesTheSameAtTheHead(t *testing.T) { // 1448/AC2 manifest := real(t, "module.json") forge := &forgeFiles{have: map[string][]byte{"modules/a/module.json": manifest, "modules/a/go.sum": []byte("git.novox.be/novox/mesh-sdk/go v0.1.7 h1:x=\n")}} - msg, err := Convert(updatedSubject, updated(nil), nil, forge.get) + msg, err := Convert(updatedSubject, updated(nil), forge.get, watchesMain) if err != nil { t.Fatal(err) } @@ -157,7 +157,7 @@ func TestPullWithKnownDirsCarriesTheSameAtTheHead(t *testing.T) { // 1448/AC2 } // Dirs not known: neither field, and the forge is not asked. forge = &forgeFiles{} - msg, err = Convert(updatedSubject, updated(map[string]any{"module_dirs": nil, "module_dirs_said": nil}), nil, forge.get) + msg, err = Convert(updatedSubject, updated(map[string]any{"module_dirs": nil, "module_dirs_said": nil}), forge.get, watchesMain) if err != nil { t.Fatal(err) } @@ -176,17 +176,17 @@ func TestAForgeErrorOrAMissingManifestIsRetryAndALockNotFoundIsLeftOut(t *testin "a lock file error": {have: map[string][]byte{"modules/a/module.json": []byte("{}")}, errs: map[string]error{"modules/a/go.sum": boom}}, "a second lock error": {have: map[string][]byte{"modules/a/module.json": []byte("{}")}, errs: map[string]error{"modules/a/package-lock.json": boom}}, } { - msg, err := Convert(mergedSubject, mergedBody(nil), trees(), forge.get) + msg, err := convertMerged(mergedBody(nil), trees(), forge.get, nil) if !errors.Is(err, ErrRetry) || msg.Data != nil { t.Errorf("%s: err = %v data = %s, want ErrRetry and no message", name, err, msg.Data) } - _, err = Convert(updatedSubject, updated(nil), nil, forge.get) + _, err = Convert(updatedSubject, updated(nil), forge.get, watchesMain) if !errors.Is(err, ErrRetry) { t.Errorf("%s (pull): err = %v, want ErrRetry", name, err) } } forge := &forgeFiles{have: map[string][]byte{"modules/a/module.json": []byte("{}")}} - msg, err := Convert(mergedSubject, mergedBody(nil), trees(), forge.get) + msg, err := convertMerged(mergedBody(nil), trees(), forge.get, nil) if err != nil { t.Fatal(err) } @@ -198,7 +198,7 @@ func TestAForgeErrorOrAMissingManifestIsRetryAndALockNotFoundIsLeftOut(t *testin func TestTheFilesAreAskedAtTheMergeCommitByOwnerRepoPathAndRef(t *testing.T) { // 1448/AC4 (the call's arguments; the manifest is in manifest_test) forge := &forgeFiles{have: map[string][]byte{"modules/a/module.json": []byte("{}")}} var calls []call - if _, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get); err != nil { + if _, err := convertMerged(mergedBody(nil), treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get, nil); err != nil { t.Fatal(err) } want := []fileCall{{"novox", "mesh-delivery", "modules/a/module.json", merge}, {"novox", "mesh-delivery", "modules/a/go.sum", merge}, @@ -335,7 +335,7 @@ func TestOneFailingFileIsRetryAndEveryCallHasReturnedBeforeTheReadDoes(t *testin dirsMap[d] = "t" } slow2 := &slowForge{failOn: "modules/m05/module.json"} - if _, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", dirsMap, nil), slow2.get); !errors.Is(err, ErrRetry) { + if _, err := convertMerged(mergedBody(nil), treesOf(&calls, "r0", dirsMap, nil), slow2.get, nil); !errors.Is(err, ErrRetry) { t.Errorf("err = %v, want ErrRetry", err) } } @@ -353,7 +353,7 @@ func mergeReadsOnlyChangedDirs(t *testing.T) { forge := &forgeFiles{have: map[string][]byte{"modules/a/module.json": manifest, "modules/a/go.sum": sum, "modules/b/module.json": []byte(`{"module":"b"}`)}} kept := memKept{} var calls []call - first, err := ConvertKept(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", map[string]string{"modules/a": "t1", "modules/b": "t2"}, nil), forge.get, kept) + first, err := convertMerged(mergedBody(nil), treesOf(&calls, "r0", map[string]string{"modules/a": "t1", "modules/b": "t2"}, nil), forge.get, kept) if err != nil { t.Fatal(err) } @@ -364,7 +364,7 @@ func mergeReadsOnlyChangedDirs(t *testing.T) { forge.calls = nil forge.have["modules/b/module.json"] = []byte(`{"module":"b","v":2}`) trees := map[string]string{"modules/a": "t1", "modules/b": "t3"} - next, err := ConvertKept(mergedSubject, mergedBody(nil), treesOf(&calls, "r1", trees, nil), forge.get, kept) + next, err := convertMerged(mergedBody(nil), treesOf(&calls, "r1", trees, nil), forge.get, kept) if err != nil { t.Fatal(err) } @@ -376,7 +376,7 @@ func mergeReadsOnlyChangedDirs(t *testing.T) { if len(forge.calls) != 3 { t.Errorf("the second merge read %d files, want 3", len(forge.calls)) } - whole, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r1", trees, nil), forge.get) + whole, err := convertMerged(mergedBody(nil), treesOf(&calls, "r1", trees, nil), forge.get, nil) if err != nil { t.Fatal(err) } @@ -390,7 +390,7 @@ func mergeReadsOnlyChangedDirs(t *testing.T) { // rest: a merge too large for one try's bound converges over its tries. kept2 := memKept{} failing := &forgeFiles{have: forge.have, errs: map[string]error{"modules/b/module.json": errors.New("context deadline exceeded")}} - if _, err := ConvertKept(mergedSubject, mergedBody(nil), treesOf(&calls, "r1", trees, nil), failing.get, kept2); !errors.Is(err, ErrRetry) { + if _, err := convertMerged(mergedBody(nil), treesOf(&calls, "r1", trees, nil), failing.get, kept2); !errors.Is(err, ErrRetry) { t.Fatalf("err = %v, want ErrRetry", err) } if _, ok := kept2["t1"]; !ok || len(kept2) != 1 { // modules/a's files are asked first, before the failure is seen @@ -398,7 +398,7 @@ func mergeReadsOnlyChangedDirs(t *testing.T) { } failing.calls = nil failing.errs = nil - again, err := ConvertKept(mergedSubject, mergedBody(nil), treesOf(&calls, "r1", trees, nil), failing.get, kept2) + again, err := convertMerged(mergedBody(nil), treesOf(&calls, "r1", trees, nil), failing.get, kept2) if err != nil { t.Fatal(err) } diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/forgemade.go b/modules/mesh-delivery-forge-bridge/internal/bridge/forgemade.go index 487fa233..6cc982b7 100644 --- a/modules/mesh-delivery-forge-bridge/internal/bridge/forgemade.go +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/forgemade.go @@ -13,12 +13,13 @@ type ForgePull struct { Number int Branch string // the pull request's head branch Head string // the head commit - Merge string // the merge commit into the trunk + Merge string // the merge commit + Base string // the branch it was merged into } // ForgeMade reads, through the seat `git`'s verb pull, the merged pull request whose merge commit is merge: by number -// when number is given (its merge commit must then equal merge), else by the merge commit. A pull request not merged, -// merged into another branch than the trunk, or merged at another commit is refused. +// when number is given (its merge commit must then equal merge), else by the merge commit. A pull request not merged, or +// merged at another commit, is refused. func ForgeMade(ctx context.Context, call Caller, owner, repo string, number int, merge string) (ForgePull, error) { in := git.PullInput{Repository: repository(owner, repo), MergeCommit: merge} if number > 0 { @@ -36,10 +37,10 @@ func ForgeMade(ctx context.Context, call Caller, owner, repo string, number int, return ForgePull{}, fmt.Errorf("the forge says the pull request %s#%d is %s, not merged", in.Repository, p.Number, p.State) case p.MergeCommit != merge: return ForgePull{}, fmt.Errorf("the forge says %s#%d was merged at %s, not at %s", in.Repository, p.Number, p.MergeCommit, merge) - case p.Base != Trunk: - return ForgePull{}, fmt.Errorf("the forge says %s#%d was merged into %s, not the trunk %s", in.Repository, p.Number, p.Base, Trunk) + case p.Base == "": + return ForgePull{}, fmt.Errorf("the forge gives no base branch for %s#%d", in.Repository, p.Number) case p.Branch == "": return ForgePull{}, fmt.Errorf("the forge gives no head branch for %s#%d", in.Repository, p.Number) } - return ForgePull{Number: int(p.Number), Branch: p.Branch, Head: p.Head, Merge: p.MergeCommit}, nil + return ForgePull{Number: int(p.Number), Branch: p.Branch, Head: p.Head, Merge: p.MergeCommit, Base: p.Base}, nil } diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/moved.go b/modules/mesh-delivery-forge-bridge/internal/bridge/moved.go new file mode 100644 index 00000000..a559662c --- /dev/null +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/moved.go @@ -0,0 +1,95 @@ +package bridge + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + git "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" + "git.novox.be/novox/mesh-sdk/go/keys" +) + +// BranchMovedSubject is the seat git's branch-moved (design 47 G §11): a branch's new head, which a merge and an admitted +// push alike say, kept on the seat's stream SEAT_git_branch-moved and heard there through a durable consumer. +const BranchMovedSubject = git.SubjectBranchMoved + +// Moved is a branch-moved as the bridge reads it: the repository's two halves and the branch. The head the event says is +// never taken: the commit is read from the forge's own record by the seat's verb branch (§11). +type Moved struct { + Owner, Repo, Branch string +} + +// MovedOf reads a branch-moved's body. +func MovedOf(data []byte) (Moved, error) { + var e git.BranchMovedEvent + if err := json.Unmarshal(data, &e); err != nil { + return Moved{}, unreadable(BranchMovedSubject, "the body is not json: %v", err) + } + m := Moved{Branch: e.Branch} + m.Owner, m.Repo, _ = strings.Cut(e.Repository, "/") + if m.Owner == "" || m.Repo == "" || strings.ContainsAny(m.Owner+m.Repo, "./ *>") || m.Branch == "" { + return Moved{}, unreadable(BranchMovedSubject, "no repository / or branch (%q, %q)", e.Repository, e.Branch) + } + return m, nil +} + +// Repository is `/`. +func (m Moved) Repository() string { return repository(m.Owner, m.Repo) } + +// HeadKey is the key of a watched branch's last head written in the state heads: `branch...` (A-236), apart from a pull request's `..`. +func (m Moved) HeadKey() string { + return "branch." + m.Owner + "." + m.Repo + "." + keys.EscapeName(m.Branch) +} + +// Branch is a branch as the seat git's verb branch reads it: its head and whether it is protected. +type Branch struct { + Head string + Protected bool +} + +// ReadBranch reads a branch's head and protection through the seat git's verb branch (§11, §17); an answer that says +// an error, or names no 40-hex head, is an error, never a branch with none. +func ReadBranch(ctx context.Context, call Caller, m Moved) (Branch, error) { + a, err := askGit[git.BranchAnswer](ctx, call, git.VerbBranch, git.BranchInput{Repository: m.Repository(), Branch: m.Branch}) + if err != nil { + return Branch{}, err + } + if !sha40.MatchString(a.Head) { + return Branch{}, fmt.Errorf("the seat git's branch %s of %s names no 40-hex head (%q)", m.Branch, m.Repository(), a.Head) + } + return Branch{Head: a.Head, Protected: a.Protected}, nil +} + +// MoveMessage is the one message of a move of a watched branch to head (design 47 G §509, §714, A-236): merged, with +// the pull request and the watched branch as base, when the seat's verb pull names head a pull request's merge; pushed, +// with the branch, head and before (the head last written for the branch, empty for none), when it names none. Both +// carry the take-in inputs at head. A pull the forge could not say is ErrRetry, never read as none. +func MoveMessage(ctx context.Context, call Caller, m Moved, head, before string, trees Trees, files Files, kept Kept) (Message, error) { + what := fmt.Sprintf("%s of %s at %s", m.Branch, m.Repository(), head) + raw, err := call(ctx, GitKey(git.VerbPull), git.PullInput{Repository: m.Repository(), MergeCommit: head}) + if err != nil { + return Message{}, fmt.Errorf("%w: %s: the seat git did not answer pull: %v", ErrRetry, what, err) + } + var a git.PullAnswer + if err := json.Unmarshal(raw, &a); err != nil { + return Message{}, fmt.Errorf("%w: %s: the seat git's answer of pull is not its contract's: %v", ErrRetry, what, err) + } + f := forgemsg.Forge{Repository: m.Repository()} + switch { + case a.Error != "" && a.ErrorKind == git.ErrorKindNotFound: + f.Kind, f.Branch, f.Head, f.Before = forgemsg.ForgePushed, m.Branch, head, before + case a.Error != "": + return Message{}, fmt.Errorf("%w: %s: the seat git refused pull: %s", ErrRetry, what, a.Error) + case a.Pull == nil || a.Pull.Number <= 0 || a.Pull.MergeCommit != head || a.Pull.State != git.PullStateMerged: + return Message{}, fmt.Errorf("%w: %s: the seat git's pull at that merge commit names no merged pull request merged there", ErrRetry, what) + default: + if !sha40.MatchString(a.Pull.Head) || a.Pull.Branch == "" { + return Message{}, unreadable(BranchMovedSubject, "%s: the pull request %s#%d names no branch or 40-hex head", what, m.Repository(), a.Pull.Number) + } + f.Kind, f.Branch, f.Number, f.Head, f.Merge, f.Base = forgemsg.ForgeMerged, a.Pull.Branch, int(a.Pull.Number), a.Pull.Head, head, m.Branch + } + return landed(what, f, head, trees, files, kept) +} diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/moved_test.go b/modules/mesh-delivery-forge-bridge/internal/bridge/moved_test.go new file mode 100644 index 00000000..17ed5a68 --- /dev/null +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/moved_test.go @@ -0,0 +1,77 @@ +package bridge + +import ( + "context" + "encoding/json" + "errors" + "testing" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" + + "mesh-delivery-forge-bridge/internal/gitfixture" +) + +// Design 47 G §4, §11, §507, §509, §714, A-236 (FB1, FB9, FB10): the watched branches are the registrations' by origin, and +// before the first registration the gate's source's, no branch fixed in code; a move of a watched branch is one message: +// merged with its pull request and base when the seat's pull names the new head a pull request's merge, pushed with +// its branch, head and before when it names none, both carrying the commit's take-in inputs; a pull the forge could not +// say is heard again, never read as a push. +func TestAMoveOfAWatchedBranchIsOneMessage(t *testing.T) { + w := &Watched{} + w.Set([]Registered{{Origin: "novox/mesh-sdk", Branches: []string{"main", "beta"}}}, []string{"main"}) + for _, c := range []struct { + repo, branch string + want bool + }{{"novox/mesh-sdk", "beta", true}, {"novox/mesh-sdk", "feat/x", false}, {"novox/other", "main", false}} { + if got := w.Watches(c.repo, c.branch); got != c.want { + t.Errorf("registered: %s of %s watched %v, want %v", c.branch, c.repo, got, c.want) + } + } + if !watchedOf("main").Watches("novox/other", "main") || watchedOf("main").Watches("novox/other", "beta") { + t.Errorf("before the first registration the gate's source's branches are watched, of every repository") + } + + const landedAt, before = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "cccccccccccccccccccccccccccccccccccccccc" + moved := Moved{Owner: "novox", Repo: "mesh-sdk", Branch: "beta"} + for _, c := range []struct { + name string + pull json.RawMessage + want forgemsg.Forge + retry bool + calls1 int + }{ + {"a merge", gitfixture.Given(t, "pull answers", "verb:pull.answer"), forgemsg.Forge{Kind: forgemsg.ForgeMerged, + Branch: "feat/47-G", Number: 178, Head: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Merge: landedAt, Base: "beta"}, false, 1}, + {"an admitted push", json.RawMessage(`{"error":"no pull request merged at that commit","error-kind":"not-found"}`), + forgemsg.Forge{Kind: forgemsg.ForgePushed, Branch: "beta", Head: landedAt, Before: before}, false, 1}, + {"a pull the forge could not say", gitfixture.Given(t, "pull answers an error, never none", "verb:pull.answer"), + forgemsg.Forge{}, true, 0}, + } { + var asked []string + ask := func(_ context.Context, key string, body any) (json.RawMessage, error) { + b, _ := json.Marshal(body) + asked = append(asked, key+" "+string(b)) + return c.pull, nil + } + var trees []call + msg, err := MoveMessage(context.Background(), ask, moved, landedAt, before, treesOf(&trees, "r0", map[string]string{"modules/a": "t1"}, nil), anyFiles, nil) + if c.retry { + if !errors.Is(err, ErrRetry) || msg.Data != nil { + t.Errorf("%s: %v, %s; want ErrRetry and nothing written", c.name, err, msg.Data) + } + continue + } + if err != nil { + t.Fatalf("%s: %v", c.name, err) + } + f, err := forgemsg.DecodeForge(msg.Data) + c.want.V, c.want.Repository, c.want.Root = forgemsg.Version, "novox/mesh-sdk", "r0" + c.want.Trees, c.want.Manifests = map[string]string{"modules/a": "t1"}, map[string]string{"modules/a": "{}"} + if err != nil || !forgemsg.SameForge(f, c.want) || msg.Event != "forge.novox.mesh-sdk."+c.want.Kind { + t.Errorf("%s: %s %+v (%v), want %+v", c.name, msg.Event, f, err, c.want) + } + if len(trees) != c.calls1 || trees[0].sha != landedAt || asked[0] != `seat:git.pull {"merge-commit":"`+landedAt+`","repository":"novox/mesh-sdk"}` { + t.Errorf("%s: trees asked %v, pull asked %v; want the landed commit's", c.name, trees, asked) + } + } +} diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/paths_test.go b/modules/mesh-delivery-forge-bridge/internal/bridge/paths_test.go index c4ddac59..734224fd 100644 --- a/modules/mesh-delivery-forge-bridge/internal/bridge/paths_test.go +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/paths_test.go @@ -22,7 +22,7 @@ func TestAMergedMessageCarriesNoNamedPathsTheBridgeDerived(t *testing.T) { return nil, ErrNotFound } var calls []call - msg, err := Convert(mergedSubject, mergedBody(map[string]any{"repo": "mesh-sdk", "module_dirs": []string{""}}), treesOf(&calls, "r0", map[string]string{"": "r0"}, nil), files) + msg, err := convertMerged(mergedBody(map[string]any{"repo": "mesh-sdk", "module_dirs": []string{""}}), treesOf(&calls, "r0", map[string]string{"": "r0"}, nil), files, nil) if err != nil { t.Fatal(err) } diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/watched.go b/modules/mesh-delivery-forge-bridge/internal/bridge/watched.go new file mode 100644 index 00000000..27016255 --- /dev/null +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/watched.go @@ -0,0 +1,71 @@ +package bridge + +import ( + "slices" + "sync" +) + +// Watched is the branches the bridge watches, by repository (design 47 G §4, §11, §507, §532): the registrations' watched +// branches by their origin, from a first view of `registrations` and then `registration-changed`; before the first +// registration, the branches the gate's `source` answers, of every repository. No branch is fixed in code (§7). +type Watched struct { + mu sync.Mutex + byRepo map[string][]string + source []string + registered bool +} + +// Registered is one registration as the bridge reads it: its origin `/` and its watched branches. +type Registered struct { + Origin string + Branches []string +} + +// Set takes what the bridge read: the registrations (none before the first) and the gate's source's branches by name. +func (w *Watched) Set(regs []Registered, source []string) { + by := map[string][]string{} + for _, r := range regs { + for _, b := range r.Branches { + if b != "" && !slices.Contains(by[r.Origin], b) { + by[r.Origin] = append(by[r.Origin], b) + } + } + } + w.mu.Lock() + defer w.mu.Unlock() + w.byRepo, w.registered = by, len(regs) > 0 + w.source = slices.Clone(source) +} + +// Watches says whether a move of branch in repository is one the bridge writes. +func (w *Watched) Watches(repository, branch string) bool { + w.mu.Lock() + defer w.mu.Unlock() + if !w.registered { + return slices.Contains(w.source, branch) + } + return slices.Contains(w.byRepo[repository], branch) +} + +// Branches are the watched branches of repository, in the order the registrations name them; before the first +// registration, the gate's source's. +func (w *Watched) Branches(repository string) []string { + w.mu.Lock() + defer w.mu.Unlock() + if !w.registered { + return slices.Clone(w.source) + } + return slices.Clone(w.byRepo[repository]) +} + +// Repositories are the repositories the registrations name, sorted; none before the first registration. +func (w *Watched) Repositories() []string { + w.mu.Lock() + defer w.mu.Unlock() + out := make([]string, 0, len(w.byRepo)) + for r := range w.byRepo { + out = append(out, r) + } + slices.Sort(out) + return out +} diff --git a/modules/mesh-delivery-forge-bridge/manifest_test.go b/modules/mesh-delivery-forge-bridge/manifest_test.go index 034778b3..be2ceab2 100644 --- a/modules/mesh-delivery-forge-bridge/manifest_test.go +++ b/modules/mesh-delivery-forge-bridge/manifest_test.go @@ -101,8 +101,11 @@ func TestManifestDeclarations(t *testing.T) { // 1235/AC1 } // What the gate and the checker owe the forge is each seat's event owed, heard on its own stream (design 47 *What the // gate owes the forge*), never read from their state. The forge's events are the seat git's (design 47 *The seat git*); forge.delete: a branch's end, said as a - // branch-deleted kind (design 51, DLV-FB1). - if want := []string{"git.forge-delete", "git.pull-closed", "git.pull-merged", "git.pull-updated", "mesh-delivery-checker.owed", "mesh-delivery.owed"}; !reflect.DeepEqual(sorted(m.Consumes), want) { + // branch-deleted kind (design 51, DLV-FB1). The state performed's binding is asked again on the seat mesh-broker's + // recreated and dropped, and asked of its show (design 47 G §577). + // Every move of a watched branch is heard on branch-moved, and the watched branches from the registrations, before the + // first the gate's source (design 47 G §4, §11, §507). + if want := []string{"git.branch-moved", "git.forge-delete", "git.pull-closed", "git.pull-merged", "git.pull-updated", "mesh-broker.dropped", "mesh-broker.recreated", "mesh-catalog.registration-changed", "mesh-delivery-checker.owed", "mesh-delivery.owed", "mesh-delivery.source-moved"}; !reflect.DeepEqual(sorted(m.Consumes), want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } // A group is read from the seat mesh-delivery's verb groups (design 47, acceptance 49): the bridge reads no state of @@ -110,8 +113,8 @@ func TestManifestDeclarations(t *testing.T) { // 1235/AC1 if len(m.Reads) != 0 { t.Errorf("reads = %v, want none", m.Reads) } - if want := []string{"seat:git.comment", "seat:git.file", "seat:git.merge-pull", "seat:git.note", "seat:git.open-pulls", - "seat:git.pull", "seat:git.status", "seat:git.tags", "seat:git.trees", "seat:git.view", "seat:mesh-delivery.groups", "seat:mesh-hand-acts.hand-act"}; !reflect.DeepEqual(sorted(m.Invokes), want) { + if want := []string{"seat:git.branch", "seat:git.comment", "seat:git.file", "seat:git.merge-pull", "seat:git.note", "seat:git.open-pulls", + "seat:git.pull", "seat:git.status", "seat:git.tags", "seat:git.trees", "seat:git.view", "seat:mesh-broker.show", "seat:mesh-catalog.registrations", "seat:mesh-delivery.groups", "seat:mesh-delivery.source", "seat:mesh-hand-acts.hand-act"}; !reflect.DeepEqual(sorted(m.Invokes), want) { t.Errorf("invokes = %v, want %v", m.Invokes, want) } if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" { diff --git a/modules/mesh-delivery-forge-bridge/module.json b/modules/mesh-delivery-forge-bridge/module.json index 99fcd8f7..0cbf07f2 100644 --- a/modules/mesh-delivery-forge-bridge/module.json +++ b/modules/mesh-delivery-forge-bridge/module.json @@ -7,8 +7,13 @@ "git.pull-merged", "git.pull-closed", "git.forge-delete", + "git.branch-moved", + "mesh-catalog.registration-changed", + "mesh-delivery.source-moved", "mesh-delivery.owed", - "mesh-delivery-checker.owed" + "mesh-delivery-checker.owed", + "mesh-broker.recreated", + "mesh-broker.dropped" ], "streams": [ { "name": "forge", "events": ["forge.**", "owed.ended"] } @@ -40,7 +45,11 @@ "seat:git.trees", "seat:git.file", "seat:git.pull", - "seat:git.tags" + "seat:git.tags", + "seat:git.branch", + "seat:mesh-catalog.registrations", + "seat:mesh-delivery.source", + "seat:mesh-broker.show" ], "requires": [ { diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index 378ae6c7..f33619f1 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -47,7 +47,7 @@ type Store interface { // trunkLike are the branches that are no group: a pull request from one is nobody's delivery group (the rule of // mesh-catalog modules/mesh-delivery/cmd/mesh-delivery/holder.go:321). -var trunkLike = map[string]bool{"main": true, "master": true, "trunk": true, "": true} +var trunkLike = map[string]bool{TrunkBranch: true, "master": true, "trunk": true, "": true} // writeTries bounds the reads again after a conflict; a message that still conflicts is left to the server to deliver again. const writeTries = 8 diff --git a/modules/mesh-delivery-indexer/internal/index/trunk_named_test.go b/modules/mesh-delivery-indexer/internal/index/trunk_named_test.go new file mode 100644 index 00000000..9ee5cd6c --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/trunk_named_test.go @@ -0,0 +1,37 @@ +package index + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// A-93, design 47 G §7: the trunk `main` is named once in the indexer, as TrunkBranch; no other non-test source of the module holds the +// string, so the first watched branch changes in one place. +func TestTheTrunkIsNamedOnceInTheIndexer(t *testing.T) { + var named []string + for _, dir := range []string{".", "../../cmd/mesh-delivery-indexer", "../judge"} { + files, err := filepath.Glob(filepath.Join(dir, "*.go")) + if err != nil { + t.Fatal(err) + } + for _, f := range files { + if strings.HasSuffix(f, "_test.go") { + continue + } + src, err := os.ReadFile(f) + if err != nil { + t.Fatal(err) + } + for _, line := range strings.Split(string(src), "\n") { + if t := strings.TrimSpace(line); strings.Contains(line, `"main"`) && !strings.HasPrefix(t, "//") { + named = append(named, filepath.Base(f)) + } + } + } + } + if len(named) != 1 { + t.Errorf(`"main" is named %d times in code (%v), want once: TrunkBranch`, len(named), named) + } +} diff --git a/modules/mesh-grants/internal/buscompose/modstreams.go b/modules/mesh-grants/internal/buscompose/modstreams.go index 0c724157..4b32fff5 100644 --- a/modules/mesh-grants/internal/buscompose/modstreams.go +++ b/modules/mesh-grants/internal/buscompose/modstreams.go @@ -60,6 +60,9 @@ type ModuleStream struct { // History says the stream keeps its history (novox/hq ADR 0592 §6, `retention: history`): every message, with no // maximum age, a new one refused when it is full rather than its oldest dropped. History bool + // LastPerSubject says the stream keeps only the last message of each subject (`last-per-subject`, design 47 G §703): a part that + // follows it reads it whole at each start (§556). + LastPerSubject bool } // Subjects are the subjects the stream takes. @@ -104,6 +107,14 @@ func (s ModuleStream) Stream() Stream { out.Why += "; heard only by the consumers its trusted config value grants" } } + if s.LastPerSubject { + out.MaxMsgsPerSubject = 1 + out.Why = fmt.Sprintf("%s's own stream kept last per subject (design 47 G §703): the newest message of each subject, at most %d MiB, "+ + "read whole by the part that follows it at each start (§556)", s.Module, s.MaxMiB) + if s.Restricted { + out.Why += "; heard only by the consumers its trusted config value grants" + } + } if s.WorkQueue { out.Retention = RetentionWorkQueue out.DiscardNew = true @@ -262,8 +273,9 @@ func heardGrants(p Principal, hears []Heard) []string { "$JS.API.CONSUMER.INFO."+h.Stream+"."+durable, "$JS.API.CONSUMER.MSG.NEXT."+h.Stream+"."+durable, "$JS.ACK."+h.Stream+"."+durable+".>") - if strings.HasPrefix(h.Stream, seatdef.KeptStreamPrefix) { - // A seat's kept event is heard through process.HearSeatEvents, which reads the stream's own info to see whether it is + if strings.HasPrefix(h.Stream, seatdef.KeptStreamPrefix) || h.LastPerSubject { + // A seat's kept event, and a module's own stream kept last per subject, are heard through process.HearSeatEvents and + // process.HearStream, which reads the stream's own info to see whether it is // kept last per subject (design 47 G §641, A-19): the info of this stream alone. out = append(out, "$JS.API.STREAM.INFO."+h.Stream) } diff --git a/modules/mesh-grants/internal/buscompose/orderedread_test.go b/modules/mesh-grants/internal/buscompose/orderedread_test.go index 8c7192f0..96fb5fc9 100644 --- a/modules/mesh-grants/internal/buscompose/orderedread_test.go +++ b/modules/mesh-grants/internal/buscompose/orderedread_test.go @@ -30,6 +30,10 @@ func TestAReaderOfASeatsKeptEventIsGrantedItsStreamInfo(t *testing.T) { if !slices.Contains(kept, "$JS.API.STREAM.INFO.SEAT_mesh-delivery_owed") { t.Errorf("a reader of a seat's kept event is not granted its stream's info: %v", kept) } + own := heardGrants(p, []Heard{{Stream: "BUILDER_BUILDS", Filters: []string{"f"}, LastPerSubject: true}}) + if !slices.Contains(own, "$JS.API.STREAM.INFO.BUILDER_BUILDS") { + t.Errorf("a reader of a module's stream kept last per subject is not granted its info: %v", own) + } other := heardGrants(p, []Heard{{Stream: "mesh-x-events", Filters: []string{"f"}}}) if slices.Contains(other, "$JS.API.STREAM.INFO.mesh-x-events") { t.Errorf("a reader of a module's stream is granted its info: %v", other) diff --git a/modules/mesh-grants/internal/grants/defined_test.go b/modules/mesh-grants/internal/grants/defined_test.go index aafa7126..85aa2f3a 100644 --- a/modules/mesh-grants/internal/grants/defined_test.go +++ b/modules/mesh-grants/internal/grants/defined_test.go @@ -25,3 +25,20 @@ func TestAStarEmitterBeforeAFindingEventReachesEachDeclaringStream(t *testing.T) t.Fatalf("with no event defined, heard %+v", hears) } } + +// Design 47 G §703, §556, A-181: a stream a manifest declares kept last per subject is asked of the bus so (one message per subject), +// and what hears it is composed to read it whole. +func TestAStreamKeptLastPerSubjectIsComposedAndHeardSo(t *testing.T) { + shelf := map[string]catalogue.Manifest{ + "builder": {Module: "builder", Streams: []catalogue.StreamDeclaration{{Name: "builder-builds", Events: []string{"build.ended.**"}, LastPerSubject: true}}}, + } + none := func(string, string) []any { return nil } + reader := catalogue.Manifest{Module: "reader", Consumes: []string{"builder.build.ended.**"}} + hears, _, _ := hearsOf(reader, shelf, "ace", none, reader.Consumes, nil) + if len(hears) != 1 || !hears[0].LastPerSubject { + t.Fatalf("heard %+v, want one stream read whole", hears) + } + if got := streamsOf(shelf["builder"]); len(got) != 1 || got[0].Stream().MaxMsgsPerSubject != 1 { + t.Fatalf("composed %+v, want one message per subject", got) + } +} diff --git a/modules/mesh-grants/internal/grants/records.go b/modules/mesh-grants/internal/grants/records.go index 133dc6b4..2e26b50b 100644 --- a/modules/mesh-grants/internal/grants/records.go +++ b/modules/mesh-grants/internal/grants/records.go @@ -434,7 +434,7 @@ func streamsOf(m catalogue.Manifest) []buscompose.ModuleStream { } out = append(out, buscompose.ModuleStream{Module: m.Module, Name: s.BusName(), Events: events, MaxAgeDays: s.Days(), MaxMiB: s.Cap(), Restricted: s.Grant != nil, - WorkQueue: s.Retention == catalogue.StreamRetentionWorkQueue, Duplicates: s.Duplicates, History: s.KeepsHistory()}) + WorkQueue: s.Retention == catalogue.StreamRetentionWorkQueue, Duplicates: s.Duplicates, History: s.KeepsHistory(), LastPerSubject: s.LastPerSubject}) } return out } @@ -510,7 +510,7 @@ func hearsOf(m catalogue.Manifest, declared map[string]catalogue.Manifest, node parts, touches, onEvents := catalogue.Hearing(reading, declared, node, values, defined) var out []buscompose.Heard for _, p := range parts { - h := buscompose.Heard{Stream: p.Stream.BusName(), Emitter: p.Emitter, Restricted: p.Stream.Grant != nil} + h := buscompose.Heard{Stream: p.Stream.BusName(), Emitter: p.Emitter, Restricted: p.Stream.Grant != nil, LastPerSubject: p.Stream.LastPerSubject} for _, pattern := range p.Patterns { h.Filters = append(h.Filters, buscompose.ModuleStreamSubject(p.Emitter, buscompose.WirePattern(pattern))) }