diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go index 19930604..b306039d 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go @@ -36,7 +36,6 @@ import ( "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliverycheckerclient" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/gopackageregistry" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdeliverychecker" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" @@ -1070,49 +1069,22 @@ func TestAMemberNamingAMeshSDKCommitOffTheTrunkIsJudgedAndNotRefused(t *testing. t.Fatalf("verdict = %+v", v) } - // DLV-C3, DLV-C6 (design 51 *Tagged versions*, ADR 0607 §3, §5, ADR 0620): a group whose mesh-sdk member declares a Go - // package, read at its head with no preview yet, says tag-wanted on the checker's stream asks, at the subject and in the body the mesh-delivery-checker - // definition names; once the Go seat's published of the tagged version is heard, the other member is built against it. + // Design 47 G §528: a group whose mesh-sdk member declares a Go package is judged at once, with no tagged version asked + // or waited for, and its other member is built against the group's head. const sdkRepo = "novox/mesh-sdk" b.setTrunk(t, map[string]string{repo: trunk, sdkRepo: trunk}) - // The member carries no preview of its merge yet, so its manifest is read at its head (the package new in the group). b.heads.Store(sdkRepo+" module.json "+head2, `{"module":"mesh-sdk","build":{"artifacts":[{"name":"go","kind":"package","language":"go","paths":["go"]}]}}`) - asks, err := b.js.OrderedConsumer(b.ctx, "ASKS", jetstream.OrderedConsumerConfig{FilterSubjects: []string{meshdeliverychecker.HolderSubjectTagWanted(check.CheckerModule)}}) - if err != nil { - t.Fatal(err) - } sdk := pull(sdkRepo, 12, head2) sdk.Modules, sdk.Fps = []string{"mesh-sdk"}, map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{"mesh-sdk": {Fp2: "src2:" + strings.Repeat("e", 64)}} key = b.candidate(t, "feat/x", pull(repo, 8, head1), sdk) - msg, err := asks.Next(jetstream.FetchMaxWait(20 * time.Second)) - if err != nil { - t.Fatalf("no tag-wanted on the stream asks: %v", err) - } - var wanted meshdeliverychecker.TagWantedHolderEvent - dec := json.NewDecoder(bytes.NewReader(msg.Data())) - dec.DisallowUnknownFields() - if err := dec.Decode(&wanted); err != nil { - t.Fatalf("tag-wanted %s is not the definition's body: %v", msg.Data(), err) - } - if want := (meshdeliverychecker.TagWantedHolderEvent{Repository: sdkRepo, Branch: "feat/x", Head: head2, Module: "mesh-sdk", Package: "go", Pull: 12}); wanted != want { - t.Errorf("tag-wanted = %+v, want %+v", wanted, want) - } - if v, err := b.buckets[check.CheckerVerdictsBucket].Get(b.ctx, key); err == nil && !strings.Contains(string(v.Value()), `"running"`) { - t.Fatalf("the group was judged before its tagged version was published: %s", v.Value()) - } - published, _ := json.Marshal(gopackageregistry.PublishedEvent{Holder: "gitea", Kind: "package", Name: "mesh-sdk/go", Module: check.SDKModulePath, Commit: head2, Tag: "feat-x", Version: "0.1.15-feat-x.1", - Digest: "sha256:" + strings.Repeat("0", 64)}) - if _, err := b.js.Publish(b.ctx, gopackageregistry.SubjectPublished, published); err != nil { - t.Fatal(err) - } if v := b.verdict(t, key, ended); v.State != meshdeliverycheckerclient.VerdictPass { t.Fatalf("the group's verdict = %+v", v) } b.fake.mu.Lock() got := b.fake.packages[repo] b.fake.mu.Unlock() - if got[check.SDKModulePath] != "v0.1.15-feat-x.1" { - t.Errorf("mesh-delivery was built against %v, want mesh-sdk's Go module at the group's tagged version v0.1.15-feat-x.1", got) + if got[check.SDKModulePath] != head2 { + t.Errorf("mesh-delivery was built against %v, want mesh-sdk's Go module at the group's head %s", got, head2) } // DLV-C2, DLV-C6 (design 26 *Out of step*, ADR 0620): each check's end says one verdict per pull request on the stream diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go index 03799ba3..21a4f63b 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go @@ -62,9 +62,7 @@ type judgeDeps struct { nodes *nodeNames // predefined is the indexer's kept list of predefined keys; nil reads none. predefined check.PredefinedList - // tags says a group's tag-wanted and hears its packages' published (design 51 *Tagged versions*); nil asks none. - tags check.TagAsks - logf func(format string, args ...any) + logf func(format string, args ...any) } // keptPredefined reads the indexer's key predefined.list (the seat mesh-catalog's state predefined), bound once its state appears. @@ -234,7 +232,7 @@ func newJudge(d judgeDeps) check.Judge { nodes = d.nodes } ex := &check.Examiner{Nodes: nodes, Compositions: d.compositions, Asks: d.asks, Toolchains: d.tools, - Kept: check.Kept{Store: d.kept}, Previews: d.previews, Candidates: d.candidates, Modules: d.repos, Packages: d.repos, Published: d.published, KnownPackages: d.published, Forge: d.forge, Runs: d.repos, Builds: d.builds, BuildEnds: d.ends, BuildWait: d.buildWait, Announce: d.announce, Trunks: d.trunks, Manifests: d.repos, Read: d.repos, Dirs: d.repos, Predefined: d.predefined, Tags: d.tags, Renames: d.renames, SeatDefinitions: d.seatDefs, SeatList: d.seats, Inputs: d.inputs, BeginBound: d.beginBound, Logf: d.logf} + Kept: check.Kept{Store: d.kept}, Previews: d.previews, Candidates: d.candidates, Modules: d.repos, Packages: d.repos, Published: d.published, KnownPackages: d.published, Forge: d.forge, Runs: d.repos, Builds: d.builds, BuildEnds: d.ends, BuildWait: d.buildWait, Announce: d.announce, Trunks: d.trunks, Manifests: d.repos, Read: d.repos, Dirs: d.repos, Predefined: d.predefined, Renames: d.renames, SeatDefinitions: d.seatDefs, SeatList: d.seats, Inputs: d.inputs, BeginBound: d.beginBound, Logf: d.logf} if d.nodes != nil && d.builds != nil && d.repos != nil { ex.Serving = runningServes{have: d.nodes, builds: d.builds, read: d.repos, logf: d.logf} } diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go index d567a1df..092e36b8 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go @@ -284,26 +284,13 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // once the first views are answered, before any check starts (afterFirstViews). published := &publishedPackages{} compositions.Published = published - ch.Runs = repos // the node-engine a check reads beside at its running commit (ADR 0548 §6) - // A group's tagged versions (design 51 *Tagged versions*): tag-wanted on the checker's stream asks, and the package - // registry seats' published heard on the checker's durable consumer of the shared events stream. Where that consumer is - // not there yet a group's check asks and waits in vain until its bound, said in its verdict. - tags := tagAsks{js: js, node: cred.Node, heard: &check.PublishedHeard{}, logf: logger.Printf, now: time.Now} - ch.Says = tags // a pull request's verdict event, on the same stream asks (design 26 *Out of step*) - for subject := range publishedSeats { - if err := hub.Handle(subject, func(m jetstream.Msg) { - if !tags.heardPublished(m) { - _ = m.Ack() // nothing else the checker consumes rides the shared stream - } - }); err != nil { - return err - } - } + ch.Runs = repos // the node-engine a check reads beside at its running commit (ADR 0548 §6) + ch.Says = verdictSays{js: js, node: cred.Node, now: time.Now} // a pull request's verdict event, on the stream asks (design 26 *Out of step*) ch.Judge = newJudge(judgeDeps{ asks: buildrun.Asker{Publisher: asks, Waiter: waiter, Bound: stepBound, Dropper: buildrun.SeatDrops{Caller: caller}}, tools: tools, trunks: trunks, kept: kvStore{buckets[check.CheckerKeptBucket]}, previews: kvStore{buckets[check.PreviewsBucket]}, candidates: tryCandidates{candidates}, repos: repos, published: published, builds: buildStates, ends: ends, buildWait: set.buildWait, forge: forgeReads{call: caller.Call}, logf: logger.Printf, - announce: announce, compositions: compositions, tags: tags, inputs: runInputs{awaitedKV{inputsKV}}, beginBound: beginBound, + announce: announce, compositions: compositions, inputs: runInputs{awaitedKV{inputsKV}}, beginBound: beginBound, nodes: openNodeNames(ctx, js, logger.Printf), predefined: keptPredefined{predefined}, renames: seatRenames{call: caller.Call}, diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/tags.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/tags.go deleted file mode 100644 index 1f8782ec..00000000 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/tags.go +++ /dev/null @@ -1,97 +0,0 @@ -package main - -import ( - "context" - "crypto/sha256" - "encoding/json" - "fmt" - "strings" - "time" - - "github.com/nats-io/nats.go" - "github.com/nats-io/nats.go/jetstream" - - "mesh-delivery-checker/internal/check" - - "git.novox.be/novox/mesh-delivery/contracts/go/seats/gopackageregistry" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/npmpackageregistry" -) - -// publishedSeats are the two package registry seats' published, by subject (their contracts in mesh-contracts): heard on -// the checker's durable consumer of the shared events stream, made from the manifest's consumes. -var publishedSeats = map[string]string{ - gopackageregistry.SubjectPublished: gopackageregistry.Seat, - npmpackageregistry.SubjectPublished: npmpackageregistry.Seat, -} - -// tagAsks is check.TagAsks and check.VerdictSayer on the bus: tag-wanted and verdict published on the checker's own stream -// asks, and the published heard. -type tagAsks struct { - js jetstream.JetStream - node string - heard *check.PublishedHeard - logf func(string, ...any) - now func() time.Time - nextID func() string -} - -// Want publishes one tag-wanted with the checker's event headers; the message id makes a second publish of the same -// member's head and package within the stream's duplicate window one event. -func (t tagAsks) Want(ctx context.Context, w check.TagWanted) error { - body, err := json.Marshal(w) - if err != nil { - return err - } - msg := nats.NewMsg(eventSubject(check.EventTagWanted)) - id := fmt.Sprintf("tag-wanted.%s.%s.%s.%s", w.Repository, w.Head, w.Module, w.Package) - msg.Header.Set("x-event-id", id) - msg.Header.Set("x-source", check.CheckerModule) - msg.Header.Set("x-node", t.node) - msg.Header.Set("x-time", t.now().UTC().Format(time.RFC3339Nano)) - msg.Header.Set("content-type", "application/json") - msg.Data = body - _, err = t.js.PublishMsg(ctx, msg, jetstream.WithMsgID(id)) - return err -} - -// Say publishes one verdict on the checker's own stream asks with the checker's event headers; the message id makes a -// second publish of the same pull request's head, verdict and out-of-step files within the duplicate window one event. -func (t tagAsks) Say(ctx context.Context, v check.VerdictEvent) error { - body, err := json.Marshal(v) - if err != nil { - return err - } - msg := nats.NewMsg(eventSubject(check.EventVerdict)) - id := fmt.Sprintf("verdict.%s.%d.%s.%s.%x", v.Repository, v.Pull, v.Head, v.Verdict, sha256.Sum256([]byte(strings.Join(v.OutOfStep, "\n")))) - msg.Header.Set("x-event-id", id) - msg.Header.Set("x-source", check.CheckerModule) - msg.Header.Set("x-node", t.node) - msg.Header.Set("x-time", t.now().UTC().Format(time.RFC3339Nano)) - msg.Header.Set("content-type", "application/json") - msg.Data = body - _, err = t.js.PublishMsg(ctx, msg, jetstream.WithMsgID(id)) - return err -} - -func (t tagAsks) Await(ctx context.Context, seat, head string, names []string) (check.TaggedPublished, bool) { - return t.heard.Await(ctx, seat, head, names) -} - -// heardPublished takes a message of the events consumer in when it is a package registry seat's published, and answers -// whether it was: a tagged version wakes the checks waiting on it; any other published is passed over. -func (t tagAsks) heardPublished(m jetstream.Msg) bool { - seat, ok := publishedSeats[m.Subject()] - if !ok { - return false - } - var p check.TaggedPublished - if err := json.Unmarshal(m.Data(), &p); err != nil { - t.logf("%s holds what is no published of %s, left: %v", m.Subject(), seat, err) - } else if t.heard.Heard(seat, p) { - t.logf("%s published %s %s from %s (tag %s)", seat, p.Name, p.Version, p.Commit, p.Tag) - } - if err := m.Ack(); err != nil { - t.logf("%s cannot be acknowledged, so the bus gives it again: %v", m.Subject(), err) - } - return true -} diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/verdictsay.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/verdictsay.go new file mode 100644 index 00000000..9421da5f --- /dev/null +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/verdictsay.go @@ -0,0 +1,41 @@ +package main + +import ( + "context" + "crypto/sha256" + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "mesh-delivery-checker/internal/check" +) + +// verdictSays is check.VerdictSayer on the bus: verdict published on the checker's own stream asks. +type verdictSays struct { + js jetstream.JetStream + node string + now func() time.Time +} + +// Say publishes one verdict on the checker's own stream asks with the checker's event headers; the message id makes a +// second publish of the same pull request's head, verdict and out-of-step files within the duplicate window one event. +func (t verdictSays) Say(ctx context.Context, v check.VerdictEvent) error { + body, err := json.Marshal(v) + if err != nil { + return err + } + msg := nats.NewMsg(eventSubject(check.EventVerdict)) + id := fmt.Sprintf("verdict.%s.%d.%s.%s.%x", v.Repository, v.Pull, v.Head, v.Verdict, sha256.Sum256([]byte(strings.Join(v.OutOfStep, "\n")))) + msg.Header.Set("x-event-id", id) + msg.Header.Set("x-source", check.CheckerModule) + msg.Header.Set("x-node", t.node) + msg.Header.Set("x-time", t.now().UTC().Format(time.RFC3339Nano)) + msg.Header.Set("content-type", "application/json") + msg.Data = body + _, err = t.js.PublishMsg(ctx, msg, jetstream.WithMsgID(id)) + return err +} diff --git a/modules/mesh-delivery-checker/go.mod b/modules/mesh-delivery-checker/go.mod index 8f4a995b..6e83268d 100644 --- a/modules/mesh-delivery-checker/go.mod +++ b/modules/mesh-delivery-checker/go.mod @@ -11,12 +11,10 @@ require ( git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliverycheckerclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient v0.0.0-00010101000000-000000000000 - git.novox.be/novox/mesh-delivery/contracts/go/seats/gopackageregistry v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdeliverychecker v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants v0.0.0-00010101000000-000000000000 - git.novox.be/novox/mesh-delivery/contracts/go/seats/npmpackageregistry v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000 github.com/nats-io/nats-server/v2 v2.11.17 diff --git a/modules/mesh-delivery-checker/internal/check/case.go b/modules/mesh-delivery-checker/internal/check/case.go index e6358bc7..2fbebb87 100644 --- a/modules/mesh-delivery-checker/internal/check/case.go +++ b/modules/mesh-delivery-checker/internal/check/case.go @@ -53,9 +53,6 @@ type Case struct { // Judged are the check's own judge's verdicts on the manifests the change brings, by module, as the seat mesh-catalog's // verdict value (judgeManifests): the composer's request carries them in place of the running indexer's (ADR 0578 §2). Judged map[string]json.RawMessage - // Tagged are the group's tagged versions of its Go packages the check heard (design 51 *Tagged versions*), nil for none: - // a composer built at the node-engine member's head is built against them, as the check's other Go builds are. - Tagged buildstep.PackageVersions notesMu sync.Mutex // warnings are, by pull request, what part of its check did not run (Case.warn). diff --git a/modules/mesh-delivery-checker/internal/check/declared.go b/modules/mesh-delivery-checker/internal/check/declared.go new file mode 100644 index 00000000..274a0ac2 --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/declared.go @@ -0,0 +1,115 @@ +package check + +import ( + "context" + "errors" + "fmt" + "strings" + + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// declaredPackage is one package a group member declares: the module, its directory in the repository (RootPath for its +// root, empty when not known), and the artifact's name, paths and language. +type declaredPackage struct { + pull Pull + module, dir string + name string + paths []string + language string +} + +// manifestAtHead is a touched module's manifest read from the forge at its member's head (the lead's ruling 4 on A2c's +// finding 4): with no preview of the merge, a package new in the group is declared only there, never at the trunk. The +// module's directory is the indexer's when it knows the module, else modules/ or the repository's root, the one +// whose manifest names the module. Found is false when no such file names it. +func (st *state) manifestAtHead(ctx context.Context, p Pull, module string) (m catalogue.Manifest, dir string, found bool, err error) { + var dirs []string + if st.j.Dirs != nil { + if d, ok := st.j.Dirs.ModuleDir(module); ok { + dirs = []string{d} + } + } + if dirs == nil { + dirs = []string{"modules/" + module, meshcatalogclient.RootPath} + } + for _, d := range dirs { + file := "module.json" + if t := strings.Trim(d, "/"); t != "" && t != meshcatalogclient.RootPath { + file = t + "/module.json" + } + text, err := st.j.Forge.File(ctx, p.Repository, file, p.Head) + if errors.Is(err, ErrNotFound) { + continue + } + if err != nil { + return catalogue.Manifest{}, "", false, fmt.Errorf("the manifest of %s at the head of %s cannot be read%s: %w", module, pullKey(p), unheardWords(err), err) + } + m, err := catalogue.Decode([]byte(text)) + if err != nil { + return catalogue.Manifest{}, "", false, fmt.Errorf("the manifest %s of %s at the head of %s is no manifest: %w", file, module, pullKey(p), err) + } + if m.Module == module { + return m, d, true, nil + } + } + return catalogue.Manifest{}, "", false, nil +} + +// declaredPackages are the packages the case's members declare, read from each touched module's manifest at the merge +// preview (the candidate's); with no preview, at the member's head on the forge, so a package new in the group is asked +// for on the first check too (trunk's manifest when no forge is read): a member that declares none is not asked for. +func (st *state) declaredPackages(ctx context.Context) ([]declaredPackage, error) { + if st.j.Read == nil && st.j.Forge == nil { + return nil, nil + } + var trunk map[string]string + if st.j.Manifests != nil { + trunk = st.j.Manifests.ManifestDigests() + } + var out []declaredPackage + for _, p := range st.c.Pulls { + for _, module := range p.Modules { + var m catalogue.Manifest + var dir string + digest := p.Fps[module].Manifest + switch { + case digest != "" && st.j.Read != nil: + var err error + if m, err = st.j.Read.Manifest(ctx, digest); err != nil { + return nil, fmt.Errorf("the manifest of %s (%s) cannot be read, so the packages it declares are not known: %w", module, pullKey(p), err) + } + case st.j.Forge != nil: + var found bool + var err error + if m, dir, found, err = st.manifestAtHead(ctx, p, module); err != nil { + return nil, fmt.Errorf("%v, so the packages it declares are not known", err) + } + if !found { + continue + } + case trunk[module] != "" && st.j.Read != nil: + var err error + if m, err = st.j.Read.Manifest(ctx, trunk[module]); err != nil { + return nil, fmt.Errorf("the manifest of %s (%s) cannot be read, so the packages it declares are not known: %w", module, pullKey(p), err) + } + default: + continue + } + if dir == "" && st.j.Dirs != nil { + dir, _ = st.j.Dirs.ModuleDir(module) + } + if m.Build == nil { + continue + } + for _, a := range m.Build.Artifacts { + if a.Kind != catalogue.ArtifactPackage { + continue + } + out = append(out, declaredPackage{pull: p, module: module, dir: dir, name: a.Name, paths: a.Paths, language: a.Language}) + } + } + } + return out, nil +} diff --git a/modules/mesh-delivery-checker/internal/check/engines.go b/modules/mesh-delivery-checker/internal/check/engines.go index 6cfe453d..5ffdd6c5 100644 --- a/modules/mesh-delivery-checker/internal/check/engines.go +++ b/modules/mesh-delivery-checker/internal/check/engines.go @@ -226,7 +226,7 @@ func (e EngineCompositions) inSandbox(ctx context.Context, c *Case, head, node s return a, err } } - job := buildstep.ComposeJob{Fetch: buildstep.Fetch{Repository: nodeEngineRepo, Commit: head, Packages: packagesFor(e.Published, c, c.Tagged, nodeEngineRepo)}, Dir: meshcatalogclient.RootPath, + job := buildstep.ComposeJob{Fetch: buildstep.Fetch{Repository: nodeEngineRepo, Commit: head, Packages: packagesFor(e.Published, c, nodeEngineRepo)}, Dir: meshcatalogclient.RootPath, Package: composer.Package, Request: raw, Reader: "the node-engine head of " + c.Group} // The ask travels on the bus, whose message limit a whole request passes (1.2 MB for a large node): it goes packed. if err := job.PackRequest(raw); err != nil { diff --git a/modules/mesh-delivery-checker/internal/check/engines_test.go b/modules/mesh-delivery-checker/internal/check/engines_test.go index 1c76571e..31ea8066 100644 --- a/modules/mesh-delivery-checker/internal/check/engines_test.go +++ b/modules/mesh-delivery-checker/internal/check/engines_test.go @@ -243,16 +243,6 @@ func TestTheCompositionIsMadeFromTheEnginesAnswers(t *testing.T) { if pkgs := job.Fetch.Packages; len(pkgs) != 1 || pkgs[trunk.ForgeModulePrefix+"mesh-delivery"] != head1 { t.Errorf("the composer is fetched with the packages %v, want the group's mesh-delivery member at its head %s alone", pkgs, head1) } - // ADR 0607 §3, §5, design 51 *Tagged versions*: once the check heard the group's tagged version of a member's Go - // package, the composer is built against that, not the member's head, as the check's other Go builds are. - const tagged = "v0.1.15-feat-x.1" - host.Tagged = buildstep.PackageVersions{trunk.ForgeModulePrefix + "mesh-delivery": tagged} - if _, err := ec.Compose(context.Background(), host); err != nil { - t.Fatal(err) - } - if pkgs := sb.jobs[len(sb.jobs)-1].Fetch.Packages; len(pkgs) != 1 || pkgs[trunk.ForgeModulePrefix+"mesh-delivery"] != tagged { - t.Errorf("the composer is fetched with the packages %v, want the group's tagged version %s of mesh-delivery", pkgs, tagged) - } if v.State != meshdeliverycheckerclient.CompositionFail || v.Sdk != head2 || !strings.Contains(v.Words, "the new composer refuses it") { t.Errorf("verdict = %+v", v) } diff --git a/modules/mesh-delivery-checker/internal/check/judge.go b/modules/mesh-delivery-checker/internal/check/judge.go index bc6adc0a..1476eae8 100644 --- a/modules/mesh-delivery-checker/internal/check/judge.go +++ b/modules/mesh-delivery-checker/internal/check/judge.go @@ -114,10 +114,6 @@ type Examiner struct { BuildWait time.Duration // Announce says a preview was written under the key (EventPreviewWritten); nil says nothing. Announce Announcer - // Tags says a group's tag-wanted and hears its packages' published (design 51 *Tagged versions*); nil asks none. - Tags TagAsks - // PublishBound is how long a group's check waits for each package's published; DefaultPublishBound when 0. - PublishBound time.Duration // Dirs names a module's directory in its repository, from the indexer's modules, so a manifest is read at a member's // head (tagwanted.go); nil, or a module it does not know, tries modules/ and the repository's root. Dirs interface { @@ -287,8 +283,6 @@ type state struct { // unpreviewed are the members (pullKey) with no preview of a merge: a head that does not merge, or a fetch that did not // answer. The indexer holds no manifest of them, so the assigner's composition would only wait: it is not awaited. unpreviewed []string - // tags are the group's tagged versions its members are built against (ADR 0607 §3), once the check heard them. - tags groupVersions // outOfStep are, by pull request (pullKey), the generated files a level's compare step found out of step (design 26). outOfStep map[string][]string // noCandidate says why the case has no previewed candidate of its heads from the indexer within the bound: for a whole @@ -438,9 +432,6 @@ func (st *state) compose(ctx context.Context) { c.NotRun = late // the builder's lateness, said by name, never the change's refusal (issue 1921) return } - // A composer built at the node-engine member's head is built against the group's tagged versions the check heard, as - // its other Go builds are (design 51 *Tagged versions*); it composes against the heads' seat list (seatList). - st.c.Tagged = st.tags.goPackages comp, err := st.j.Compositions.Compose(ctx, st.c) if err != nil { // Said in the verdict, never left as a composition not yet run (review finding 4). @@ -541,13 +532,6 @@ func (st *state) judgeAll(ctx context.Context) { } return 0 }) - // A group's members are built only after each package they declare is published as the group's tagged version - // (design 51 *Tagged versions*): a package not published within the bound fails the check before any member is built, - // each failure a fact of the check. - if v := st.tagged(ctx); v != nil { - st.failedAll(*v) - return - } var groupPackage *buildstep.FetchPackage met := len(st.facts) for _, p := range pulls { @@ -567,9 +551,7 @@ func (st *state) judgeAll(ctx context.Context) { } args := buildstep.Fetch{Repository: p.Repository, Commit: p.Head, Ref: refOf(p.Number), Base: p.Trunk} if bm == buildrun.NodeBuildModule { - if np, ok := st.tags.nodePackageFor(p.Repository); ok { - args.Package = &buildstep.FetchPackage{Name: np.Name, Version: np.Version} // the group's tagged version (ADR 0607 §3) - } else if groupPackage != nil && p.Repository != SDKRepository { + if groupPackage != nil && p.Repository != SDKRepository { args.Package = groupPackage } } @@ -602,10 +584,8 @@ func (st *state) judgeAll(ctx context.Context) { st.unpreviewed = append(st.unpreviewed, pullKey(p)) } st.judgedByMember(p, merged) - // The old form (ADR 0500 §5): the group's package member packed by the Node fetch, kept only while no tagged - // TypeScript version is heard, the form the running checker hands while the new build modules are delivered. Its - // removal is a subtask of the milestone after this one's delivery (issue 1966). - if p.Repository == SDKRepository && len(st.c.Pulls) > 1 && len(st.tags.node) == 0 && st.j.changesPackage(p, merged) { + // The group's package member packed by the Node fetch (ADR 0500 §5). + if p.Repository == SDKRepository && len(st.c.Pulls) > 1 && st.j.changesPackage(p, merged) { groupPackage = &buildstep.FetchPackage{Repository: p.Repository, Commit: p.Head, Ref: refOf(p.Number)} } } diff --git a/modules/mesh-delivery-checker/internal/check/manifests_judge.go b/modules/mesh-delivery-checker/internal/check/manifests_judge.go index b2880e4e..10fd9fe4 100644 --- a/modules/mesh-delivery-checker/internal/check/manifests_judge.go +++ b/modules/mesh-delivery-checker/internal/check/manifests_judge.go @@ -368,7 +368,7 @@ func (st *state) knownPackages(ctx context.Context) []string { // trunkJudgeVersions are the versions the trunk's manifest validator is built at: the latest published version of each // package, with no group member's head (ADR 0633 clause 6). func (st *state) trunkJudgeVersions() buildstep.PackageVersions { - return packagesFor(st.j.Published, &Case{}, nil, DeliveryRepository) + return packagesFor(st.j.Published, &Case{}, DeliveryRepository) } // judgedVerdicts puts the check's own judge's verdict on each manifest the change brings on the case, as the seat diff --git a/modules/mesh-delivery-checker/internal/check/pins.go b/modules/mesh-delivery-checker/internal/check/pins.go index bf8154db..da1df31e 100644 --- a/modules/mesh-delivery-checker/internal/check/pins.go +++ b/modules/mesh-delivery-checker/internal/check/pins.go @@ -7,7 +7,6 @@ import ( "git.novox.be/novox/mesh-delivery/sdk/buildstep" "git.novox.be/novox/mesh-delivery/sdk/grouppackages" - "git.novox.be/novox/mesh-delivery/sdk/trunk" ) // **A check builds against the group's own package heads, not against what a member names** (novox/hq ADR 0573 §2, §4; @@ -35,44 +34,27 @@ type Forge interface { // packagesOf is the version of each mesh package a Go build of repository (a check ask, the judge, the composer) builds // against (ADR 0573 §2, §4, as ADR 0607 §5 changes §4): packagesFor over the latest published version of each package the -// mesh's modules use (Published), the heads of the case's pull requests and the group's tagged versions the check heard -// (design 51 *Tagged versions*). Nil for none. +// mesh's modules use (Published) and the heads of the case's pull requests. Nil for none. func (st *state) packagesOf(repository string) buildstep.PackageVersions { - return packagesFor(st.j.Published, st.c, st.tags.goPackages, repository) + return packagesFor(st.j.Published, st.c, repository) } -// packagesFor is packagesOf for a case, the published versions and the tagged ones: every producer of a build at a group's -// heads takes its versions from here, so none leaves a member out (issue 1921). grouppackages.Heads names each package at -// its member's head, also a member that publishes no package yet; then, for a package a member of the case publishes, the -// group's tagged version of it once heard stands instead of the head, also for a package new in the group, which has no -// published version yet. A package of repository itself comes with the tree the build fetched, and is left out. +// packagesFor is packagesOf for a case and the published versions: every producer of a build at a group's heads takes its +// versions from here, so none leaves a member out (issue 1921). grouppackages.Heads names each package at its member's +// head, also a member that publishes no package yet. A package of repository itself comes with the tree the build +// fetched, and is left out. func packagesFor(published interface { LatestPackages() buildstep.PackageVersions -}, c *Case, tagged buildstep.PackageVersions, repository string) buildstep.PackageVersions { +}, c *Case, repository string) buildstep.PackageVersions { latest := buildstep.PackageVersions{} if published != nil { maps.Copy(latest, published.LatestPackages()) } - for pkg := range tagged { - if _, named := latest[pkg]; !named { - latest[pkg] = "" - } - } heads := map[string]string{} for _, m := range c.Pulls { heads[m.Repository] = m.Head } versions := grouppackages.Heads(latest, heads, repository) - for pkg, tag := range tagged { - repo, _, ok := trunk.PackageRepository(pkg) - if tag == "" || !ok || trunk.SameRepository(repo, repository) { - continue - } - if versions == nil { - versions = buildstep.PackageVersions{} - } - versions[pkg] = tag - } return versions } diff --git a/modules/mesh-delivery-checker/internal/check/pins_test.go b/modules/mesh-delivery-checker/internal/check/pins_test.go index 0ec9a9ac..e7f6956d 100644 --- a/modules/mesh-delivery-checker/internal/check/pins_test.go +++ b/modules/mesh-delivery-checker/internal/check/pins_test.go @@ -2,14 +2,10 @@ package check import ( "context" - "encoding/json" "maps" - "reflect" "slices" "strings" - "sync" "testing" - "time" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliverycheckerclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" @@ -112,105 +108,23 @@ func TestAGroupMemberIsBuiltAgainstTheSDKMembersHeadWhateverItsGoModNames(t *tes } } - // Design 51 *Tagged versions* (DLV-C3): a group's check says tag-wanted for each package a member declares and builds no - // member before its published; a lone pull request asks nothing; no published within the bound fails naming the package. + // Design 47 G §528: a group's check asks no tagged version and waits on none; it builds its members at once, each Go + // build against the group's heads. sdk := catalogue.Manifest{Module: "mesh-sdk", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{ {Name: "go", Kind: catalogue.ArtifactPackage, Language: "go", Paths: []string{"go"}}, {Name: "lib", Kind: catalogue.ArtifactPackage, Language: "typescript"}}}} - contracts := catalogue.Manifest{Module: "mesh-contracts", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{ - {Name: "go", Kind: catalogue.ArtifactPackage, Language: "go", Paths: []string{"go"}}, - {Name: "go-gopackageregistry", Kind: catalogue.ArtifactPackage, Language: "go", Paths: []string{"go/seats/gopackageregistry"}}}}} - digest, contractsDigest := strings.Repeat("c", 64), strings.Repeat("d", 64) - sdkPull := func() meshcatalog.CandidatesByGroupValuePullsItem { - p := pullOf(repoSDK, 12, head2) - p.Modules = []string{"mesh-sdk"} - p.Fps = map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{"mesh-sdk": {Fp2: "src2:" + strings.Repeat("f", 64), Manifest: digest}} - return p + digest := strings.Repeat("c", 64) + sdkPull := pullOf(repoSDK, 12, head2) + sdkPull.Modules = []string{"mesh-sdk"} + sdkPull.Fps = map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{"mesh-sdk": {Fp2: "src2:" + strings.Repeat("f", 64), Manifest: digest}} + b = newBench(t) + b.fetch.Modules = []string{"modules/m"} + b.ex.Read = fixedManifests{digest: sdk} + b.ex.Modules = langs{repoA: {buildrun.GoBuildModule, buildrun.NodeBuildModule}} + if v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1), sdkPull)).Verdict; v.State != meshdeliverycheckerclient.VerdictPass { + t.Errorf("a group's verdict = %+v", v) } - unpreviewed := func() meshcatalog.CandidatesByGroupValuePullsItem { // a package new in the group: no preview carries its manifest yet - p := sdkPull() - p.Fps = map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{"mesh-sdk": {Fp2: "src2:" + strings.Repeat("f", 64)}} - return p - } - contractsPull := func() meshcatalog.CandidatesByGroupValuePullsItem { - p := pullOf("novox/mesh-catalog", 4, strings.Repeat("3", 40)) - p.Modules = []string{"mesh-contracts"} - p.Fps = map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{"mesh-contracts": {Fp2: "src2:" + strings.Repeat("e", 64), Manifest: contractsDigest}} - return p - } - const sdkGo, contractsGo, seatGo = SDKModulePath, "git.novox.be/novox/mesh-catalog/go", "git.novox.be/novox/mesh-catalog/go/seats/gopackageregistry" - for _, row := range []struct { - name string - pulls []meshcatalog.CandidatesByGroupValuePullsItem - published []string // the seats that publish the members' heads - wants int - fails string - // built is the Go packages mesh-delivery's member is built against (ADR 0607 §3, §5), node the package its Node - // fetch takes from the registry; nil checks none. - built map[string]string - node map[string]any - // noModule: the Go seat's published names no Go module path (ruling 1 on A2d's finding 1: never worked out). - noModule bool - }{ - {name: "a group", pulls: []meshcatalog.CandidatesByGroupValuePullsItem{pullOf(repoA, 7, head1), sdkPull()}, - published: []string{"go-package-registry", "npm-package-registry"}, wants: 2, - built: map[string]string{sdkGo: "v0.1.15-feat-x.1"}, node: map[string]any{"name": "lib", "version": "0.1.15-feat-x.1"}}, - {name: "a package new in the group, read at its head", pulls: []meshcatalog.CandidatesByGroupValuePullsItem{pullOf(repoA, 7, head1), unpreviewed()}, - published: []string{"go-package-registry", "npm-package-registry"}, wants: 2, built: map[string]string{sdkGo: "v0.1.15-feat-x.1"}}, - {name: "one head, two Go packages (as mesh-contracts' head)", pulls: []meshcatalog.CandidatesByGroupValuePullsItem{pullOf(repoA, 7, head1), contractsPull()}, - published: []string{"go-package-registry"}, wants: 2, - built: map[string]string{contractsGo: "v0.1.15-feat-x.1", seatGo: "v0.1.3-feat-x.1"}}, - {name: "a lone pull request", pulls: []meshcatalog.CandidatesByGroupValuePullsItem{sdkPull()}}, - {name: "a Go published naming no module", pulls: []meshcatalog.CandidatesByGroupValuePullsItem{pullOf(repoA, 7, head1), sdkPull()}, - published: []string{"go-package-registry", "npm-package-registry"}, wants: 2, noModule: true, - fails: "no published of the tagged version of go of mesh-sdk (novox/mesh-sdk#12) on go-package-registry within 50ms"}, - {name: "a package never published", pulls: []meshcatalog.CandidatesByGroupValuePullsItem{pullOf(repoA, 7, head1), sdkPull()}, - published: []string{"go-package-registry"}, wants: 2, - fails: "no published of the tagged version of lib of mesh-sdk (novox/mesh-sdk#12) on npm-package-registry within 50ms"}, - } { - b := newBench(t) - b.fetch.Modules = []string{"modules/m"} - b.ex.Read = fixedManifests{digest: sdk, contractsDigest: contracts} - b.ex.Modules = langs{repoA: {buildrun.GoBuildModule, buildrun.NodeBuildModule}} - raw, _ := json.Marshal(sdk) - b.ex.Forge = headFiles{repoSDK + " module.json " + head2: string(raw)} - tags := &recordedTags{heard: &PublishedHeard{}, publish: row.published, noModule: row.noModule} - b.ex.Tags, b.ex.PublishBound = tags, 50*time.Millisecond - b.asks.Answer = func(a buildstep.StepAsk) buildstep.Step { - if len(tags.wanted) != row.wants { - t.Errorf("%s: a member was asked before every tag-wanted was said: %d of %d", row.name, len(tags.wanted), row.wants) - } - return b.answer(a) - } - v := b.judge(t, caseOf(t, row.pulls...)).Verdict - if len(tags.wanted) != row.wants { - t.Errorf("%s: tag-wanted said %d times, want %d: %+v", row.name, len(tags.wanted), row.wants, tags.wanted) - } - for _, w := range tags.wanted { - if !slices.Contains([]string{repoSDK, "novox/mesh-catalog"}, w.Repository) || w.Branch != "feat/x" || w.Head == head1 || w.Module == "" || w.Pull == 7 || w.Tag != "" { - t.Errorf("%s: tag-wanted = %+v", row.name, w) - } - } - switch { - case row.fails == "" && v.State != meshdeliverycheckerclient.VerdictPass: - t.Errorf("%s: verdict = %+v", row.name, v) - case row.fails != "" && (v.State != meshdeliverycheckerclient.VerdictFail || !strings.Contains(v.Failed.Words, row.fails)): - t.Errorf("%s: verdict = %+v, want it to fail saying %q", row.name, v, row.fails) - case row.fails != "" && len(b.steps.tool("fetch")) != 0: - t.Errorf("%s: a member was built though a package was not published", row.name) - } - if row.built != nil { - got := packagesOfFetch(t, b, repoA) - for pkg, version := range row.built { - if got[pkg] != version { - t.Errorf("%s: mesh-delivery is built against %s %q, want the group's tagged version %s (all: %v)", row.name, pkg, got[pkg], version, got) - } - } - } - if row.node != nil { - if got := nodeFetchOf(b, repoA).args["package"]; !reflect.DeepEqual(got, row.node) { - t.Errorf("%s: mesh-delivery's Node fetch takes %v, want the group's tagged version %v from the registry", row.name, got, row.node) - } - } + if got := packagesOfFetch(t, b, repoA); got[SDKModulePath] != head2 { + t.Errorf("mesh-delivery is built against mesh-sdk %q, want the group's head %s (all: %v)", got[SDKModulePath], head2, got) } } @@ -224,40 +138,6 @@ func (h headFiles) File(_ context.Context, repository, path, ref string) (string return "", ErrNotFound } -// recordedTags are the checker's tag asks as a test hears them: each tag-wanted kept, and the seats in publish answering it -// at once with a tagged version, as the builder and a registry seat would. -type recordedTags struct { - heard *PublishedHeard - publish []string - // noModule publishes a Go package naming no Go module path, against go-package-registry's contract. - noModule bool - mu sync.Mutex - wanted []TagWanted -} - -func (r *recordedTags) Want(_ context.Context, w TagWanted) error { - r.mu.Lock() - r.wanted = append(r.wanted, w) - r.mu.Unlock() - for _, seat := range r.publish { - version := "0.1.15-feat-x.1" - if w.Package == "go-gopackageregistry" { - version = "0.1.3-feat-x.1" - } - module := "" // go-package-registry's published names the Go module path (its contract); npm's names none - if seat == "go-package-registry" && !r.noModule { - module = map[string]string{repoSDK + " go": SDKModulePath, "novox/mesh-catalog go": "git.novox.be/novox/mesh-catalog/go", - "novox/mesh-catalog go-gopackageregistry": "git.novox.be/novox/mesh-catalog/go/seats/gopackageregistry"}[w.Repository+" "+w.Package] - } - r.heard.Heard(seat, TaggedPublished{Name: w.Package, Kind: "package", Commit: w.Head, Tag: "feat-x", Version: version, Module: module}) - } - return nil -} - -func (r *recordedTags) Await(ctx context.Context, seat, head string, names []string) (TaggedPublished, bool) { - return r.heard.Await(ctx, seat, head, names) -} - type runsAt map[string]string func (r runsAt) CommitOf(module string) string { return r[module] } diff --git a/modules/mesh-delivery-checker/internal/check/tagwanted.go b/modules/mesh-delivery-checker/internal/check/tagwanted.go deleted file mode 100644 index ec4078af..00000000 --- a/modules/mesh-delivery-checker/internal/check/tagwanted.go +++ /dev/null @@ -1,350 +0,0 @@ -package check - -import ( - "context" - "errors" - "fmt" - "maps" - "slices" - "strings" - "sync" - "time" - - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdeliverychecker" - "git.novox.be/novox/mesh-delivery/sdk/buildstep" - "git.novox.be/novox/mesh-sdk/go/catalogue" -) - -// EventTagWanted is the checker's event that asks the builder for a group member's tagged versions (design 51 *Tagged -// versions*, ADR 0610 §1), on the checker's own family asks (its stream asks, kept seven days), apart from the stream -// checks, so a waiter on a verdict does not wake on it. -const EventTagWanted = meshdeliverychecker.HolderEventTagWanted - -// AsksStream is the checker's stream that carries tag-wanted and verdict, one that keeps its history (ADR 0592 §6): the -// bus names it ASKS. -const AsksStream = "asks" - -// DefaultPublishBound is how long a group's check waits for the published of each package its members declare before it -// fails naming the package (design 51 *Tagged versions*: the publish's bound, 10 minutes). -const DefaultPublishBound = 10 * time.Minute - -// TagWanted is the body of tag-wanted, as the seat definition of mesh-delivery-checker says it. The tag is left to the -// builder, which makes it from the branch, so one part alone names a branch's tag. -type TagWanted = meshdeliverychecker.TagWantedHolderEvent - -// PackageSeats are the registry seats a package of each language is published through (design 51 *The seats*). -var PackageSeats = map[string]string{"go": "go-package-registry", "typescript": "npm-package-registry"} - -// TaggedPublished is what a registry seat's published says of a tagged version, as the checker reads it. -type TaggedPublished struct { - Name string `json:"name"` - Kind string `json:"kind"` - Commit string `json:"commit,omitempty"` - Tag string `json:"tag,omitempty"` - Version string `json:"version,omitempty"` - // Module is the Go module path a Go package is fetched under, as go-package-registry's published names it (its - // contract requires it of a Go package). - Module string `json:"module,omitempty"` -} - -// TagAsks says tag-wanted and hears the registry seats' published: nil asks nothing, and a group's check builds its -// members as it did before tagged versions. -type TagAsks interface { - // Want says one tag-wanted on the checker's family asks. - Want(ctx context.Context, t TagWanted) error - // Await answers the tagged version a seat published from a head under one of names, once heard; false when ctx ends - // first. A head declaring several packages on one seat (mesh-contracts' Go modules) is told apart by the name. - Await(ctx context.Context, seat, head string, names []string) (TaggedPublished, bool) -} - -// PublishedHeard is the registry seats' published of tagged versions as the checker heard them on its durable consumer -// of the shared events stream, by seat and source commit, each name kept: a waiter wakes on the event, never on a timer. -type PublishedHeard struct { - mu sync.Mutex - heard map[string][]TaggedPublished - waiters map[string][]publishedWaiter -} - -type publishedWaiter struct { - names []string - c chan TaggedPublished -} - -func publishedKey(seat, head string) string { return seat + " " + head } - -func (h *PublishedHeard) init() { - if h.heard == nil { - h.heard, h.waiters = map[string][]TaggedPublished{}, map[string][]publishedWaiter{} - } -} - -// Heard takes one published of a seat in; one of no tagged version (no tag, or no commit), and a Go package's that names -// no Go module path, are left out and answer false. A later published of the same name and head replaces the earlier. -func (h *PublishedHeard) Heard(seat string, p TaggedPublished) bool { - if p.Tag == "" || p.Commit == "" || (seat == PackageSeats["go"] && p.Module == "") { - return false - } - h.mu.Lock() - defer h.mu.Unlock() - h.init() - k := publishedKey(seat, p.Commit) - h.heard[k] = append(slices.DeleteFunc(h.heard[k], func(o TaggedPublished) bool { return o.Name == p.Name }), p) - h.waiters[k] = slices.DeleteFunc(h.waiters[k], func(w publishedWaiter) bool { - if !slices.Contains(w.names, p.Name) { - return false - } - w.c <- p - return true - }) - return true -} - -// Await is TagAsks.Await over what was heard. -func (h *PublishedHeard) Await(ctx context.Context, seat, head string, names []string) (TaggedPublished, bool) { - k := publishedKey(seat, head) - h.mu.Lock() - h.init() - for _, p := range h.heard[k] { - if slices.Contains(names, p.Name) { - h.mu.Unlock() - return p, true - } - } - w := publishedWaiter{names: names, c: make(chan TaggedPublished, 1)} - h.waiters[k] = append(h.waiters[k], w) - h.mu.Unlock() - select { - case p := <-w.c: - return p, true - case <-ctx.Done(): - h.mu.Lock() - h.waiters[k] = slices.DeleteFunc(h.waiters[k], func(o publishedWaiter) bool { return o.c == w.c }) - h.mu.Unlock() - select { - case p := <-w.c: // heard as the wait ended - return p, true - default: - } - return TaggedPublished{}, false - } -} - -// declaredPackage is one package a group member declares: the module, its directory in the repository (RootPath for its -// root, empty when not known), the artifact's name and paths, and the seat it is published through. -type declaredPackage struct { - pull Pull - module, dir string - name string - paths []string - language, seat string -} - -// publishedNames are the names a registry seat's published may carry for a declared package: the builder's -// `/` (its versions' name) and the artifact's name. A Go package's module path is the published's own -// `module` (go-package-registry's contract), never worked out here. -func (d declaredPackage) publishedNames() []string { - return []string{d.module + "/" + d.name, d.name} -} - -// manifestAtHead is a touched module's manifest read from the forge at its member's head (the lead's ruling 4 on A2c's -// finding 4): with no preview of the merge, a package new in the group is declared only there, never at the trunk. The -// module's directory is the indexer's when it knows the module, else modules/ or the repository's root, the one -// whose manifest names the module. Found is false when no such file names it. -func (st *state) manifestAtHead(ctx context.Context, p Pull, module string) (m catalogue.Manifest, dir string, found bool, err error) { - var dirs []string - if st.j.Dirs != nil { - if d, ok := st.j.Dirs.ModuleDir(module); ok { - dirs = []string{d} - } - } - if dirs == nil { - dirs = []string{"modules/" + module, meshcatalogclient.RootPath} - } - for _, d := range dirs { - file := "module.json" - if t := strings.Trim(d, "/"); t != "" && t != meshcatalogclient.RootPath { - file = t + "/module.json" - } - text, err := st.j.Forge.File(ctx, p.Repository, file, p.Head) - if errors.Is(err, ErrNotFound) { - continue - } - if err != nil { - return catalogue.Manifest{}, "", false, fmt.Errorf("the manifest of %s at the head of %s cannot be read%s: %w", module, pullKey(p), unheardWords(err), err) - } - m, err := catalogue.Decode([]byte(text)) - if err != nil { - return catalogue.Manifest{}, "", false, fmt.Errorf("the manifest %s of %s at the head of %s is no manifest: %w", file, module, pullKey(p), err) - } - if m.Module == module { - return m, d, true, nil - } - } - return catalogue.Manifest{}, "", false, nil -} - -// declaredPackages are the packages the case's members declare, read from each touched module's manifest at the merge -// preview (the candidate's); with no preview, at the member's head on the forge, so a package new in the group is asked -// for on the first check too (trunk's manifest when no forge is read): a member that declares none is not asked for. -func (st *state) declaredPackages(ctx context.Context) ([]declaredPackage, error) { - if st.j.Read == nil && st.j.Forge == nil { - return nil, nil - } - var trunk map[string]string - if st.j.Manifests != nil { - trunk = st.j.Manifests.ManifestDigests() - } - var out []declaredPackage - for _, p := range st.c.Pulls { - for _, module := range p.Modules { - var m catalogue.Manifest - var dir string - digest := p.Fps[module].Manifest - switch { - case digest != "" && st.j.Read != nil: - var err error - if m, err = st.j.Read.Manifest(ctx, digest); err != nil { - return nil, fmt.Errorf("the manifest of %s (%s) cannot be read, so the packages it declares are not known: %w", module, pullKey(p), err) - } - case st.j.Forge != nil: - var found bool - var err error - if m, dir, found, err = st.manifestAtHead(ctx, p, module); err != nil { - return nil, fmt.Errorf("%v, so the packages it declares are not known", err) - } - if !found { - continue - } - case trunk[module] != "" && st.j.Read != nil: - var err error - if m, err = st.j.Read.Manifest(ctx, trunk[module]); err != nil { - return nil, fmt.Errorf("the manifest of %s (%s) cannot be read, so the packages it declares are not known: %w", module, pullKey(p), err) - } - default: - continue - } - if dir == "" && st.j.Dirs != nil { - dir, _ = st.j.Dirs.ModuleDir(module) - } - if m.Build == nil { - continue - } - for _, a := range m.Build.Artifacts { - if a.Kind != catalogue.ArtifactPackage { - continue - } - seat, ok := PackageSeats[a.Language] - if !ok { - return nil, fmt.Errorf("the package %s of %s (%s) is in %q, which no registry seat publishes", a.Name, module, pullKey(p), a.Language) - } - out = append(out, declaredPackage{pull: p, module: module, dir: dir, name: a.Name, paths: a.Paths, language: a.Language, seat: seat}) - } - } - } - return out, nil -} - -// groupVersions are a group's tagged versions as its check builds its members against them (ADR 0607 §3, §5): the Go -// packages by module path in Go's form (StepAsk.Packages), and the TypeScript packages a member's Node fetch takes from the -// registry, by the repository that publishes them. -type groupVersions struct { - goPackages buildstep.PackageVersions - node map[string]NodePackage -} - -// NodePackage is a TypeScript package a group's check hands the Node build module's fetch (its `package`): the name and the -// tagged version it takes from the registry, its integrity checked against the member's lock (ADR 0607 §3). -type NodePackage struct { - Name string `json:"name"` - Version string `json:"version"` -} - -// nodePackageFor is the TypeScript package a member of repository takes as the group's tagged version: another member's -// (the Node build module places mesh-sdk's alone, so mesh-sdk's first); false when the group published none. -func (g *groupVersions) nodePackageFor(repository string) (NodePackage, bool) { - if np, ok := g.node[SDKRepository]; ok && repository != SDKRepository { - return np, true - } - for _, repo := range slices.Sorted(maps.Keys(g.node)) { - if repo != repository { - return g.node[repo], true - } - } - return NodePackage{}, false -} - -// goVersion is a published version in Go's form: a leading v. -func goVersion(v string) string { - if strings.HasPrefix(v, "v") { - return v - } - return "v" + v -} - -// took keeps a declared package's tagged version as the group's. -func (g *groupVersions) took(d declaredPackage, p TaggedPublished) { - switch d.language { - case "go": - if g.goPackages == nil { - g.goPackages = buildstep.PackageVersions{} - } - g.goPackages[p.Module] = goVersion(p.Version) // Heard took no Go published without its module - case "typescript": - if g.node == nil { - g.node = map[string]NodePackage{} - } - g.node[d.pull.Repository] = NodePackage{Name: p.Name, Version: p.Version} - } -} - -// tagged asks a group's tagged versions before any member is built (design 51 *Tagged versions*): one tag-wanted per -// package a member declares, then each package's published awaited within the publish bound. A lone pull request and a -// try ask nothing. A package not published within the bound fails the check naming it. -func (st *state) tagged(ctx context.Context) *meshdeliverychecker.VerdictsByGroupHeadsValue { - if st.j.Tags == nil || st.c.Try != nil || len(st.c.Pulls) < 2 { - return nil - } - decl, err := st.declaredPackages(ctx) - if err != nil { - return fail(CheckerModule, err.Error()) - } - if len(decl) == 0 { - return nil - } - st.c.Progress.Set("tagged versions", "") - var names []string - for _, d := range decl { - t := TagWanted{Repository: d.pull.Repository, Branch: st.c.Group, Head: d.pull.Head, Module: d.module, Package: d.name, Pull: int64(d.pull.Number)} - if err := st.j.Tags.Want(ctx, t); err != nil { - return fail(CheckerModule, fmt.Sprintf("the tagged version of %s of %s (%s) cannot be asked: %v", d.name, d.module, pullKey(d.pull), err)) - } - names = append(names, d.module+"/"+d.name) - } - bound := st.j.PublishBound - if bound <= 0 { - bound = DefaultPublishBound - } - waits := fmt.Sprintf("waits on the tagged versions of %s before its members are built, at most %s (design 51 *Tagged versions*)", strings.Join(names, ", "), bound) - st.c.note(waits) - st.j.logf("%s %s", st.c.Key, waits) - at, cancel := context.WithTimeout(ctx, bound) - defer cancel() - var late []string - for _, d := range decl { - p, ok := st.j.Tags.Await(at, d.seat, d.pull.Head, d.publishedNames()) - if !ok { - late = append(late, fmt.Sprintf("%s of %s (%s) on %s", d.name, d.module, pullKey(d.pull), d.seat)) - continue - } - st.tags.took(d, p) - st.c.note(fmt.Sprintf("%s of %s is built as %s %s (tag %s)", d.name, d.module, p.Name, p.Version, p.Tag)) - } - if len(late) > 0 { - if ctx.Err() != nil { - return fail(CheckerModule, ctx.Err().Error()) - } - return fail(CheckerModule, fmt.Sprintf("no published of the tagged version of %s within %s, so no member is built (design 51 *Tagged versions*)", strings.Join(late, "; "), bound)) - } - return nil -} diff --git a/modules/mesh-delivery-checker/internal/check/verdictevent.go b/modules/mesh-delivery-checker/internal/check/verdictevent.go index fe558a2a..fa784452 100644 --- a/modules/mesh-delivery-checker/internal/check/verdictevent.go +++ b/modules/mesh-delivery-checker/internal/check/verdictevent.go @@ -38,3 +38,7 @@ func (ch *Checker) sayVerdict(ctx context.Context, cs *Case, v meshdeliverycheck } } } + +// AsksStream is the checker's stream that carries verdict, one that keeps its history (ADR 0592 §6): the bus names it +// ASKS. +const AsksStream = "asks" diff --git a/modules/mesh-delivery-checker/manifest_test.go b/modules/mesh-delivery-checker/manifest_test.go index 78a167bb..c30c2c31 100644 --- a/modules/mesh-delivery-checker/manifest_test.go +++ b/modules/mesh-delivery-checker/manifest_test.go @@ -139,10 +139,9 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1 t.Errorf("uses = %v, want the Go and Node build seats", m.Uses) } // The checker is woken by a build module's report of a toolchain, as the indexer is: the bucket may not exist when it starts. - // And a group's check by the package registry seats' published of its tagged versions (design 51 *Tagged versions*). - if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", + if want := []string{"mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-catalog.catalog-revision", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", "mesh-conditions.findings-asked", "mesh-delivery-assigner.views.assignments", "mesh-delivery-indexer.views.candidates", "mesh-delivery-indexer.views.manifests", "mesh-delivery-indexer.views.modules", - "mesh-delivery-indexer.views.packages", "mesh-delivery-indexer.views.predefined", "mesh-delivery-indexer.views.seat-definitions", "npm-package-registry.published"}; !reflect.DeepEqual(sorted(m.Consumes), want) { + "mesh-delivery-indexer.views.packages", "mesh-delivery-indexer.views.predefined", "mesh-delivery-indexer.views.seat-definitions"}; !reflect.DeepEqual(sorted(m.Consumes), want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" { @@ -152,10 +151,9 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1 t.Errorf("secrets-owner = %q", m.SecretsOwner) } // The checker says each time it wrote a preview or a status, so the indexer's and the forge bridge's watches find the buckets (ADR 0484 clauses 2 and 3). - // Its family asks (design 51 *Tagged versions*, design 26): tag-wanted and verdict, seven days, apart from checks so a - // waiter on a verdict does not wake on a tag ask; a stream that keeps its history, the predefined form of an event family + // Its family asks (design 26): verdict, apart from checks so a waiter on a verdict does not wake on a check; a stream that keeps its history, the predefined form of an event family // (ADR 0592 §6, the manifest validator). - if want := []stream{{Name: "checks", Events: []string{"preview.**", "owed.**", "try.**"}}, {Name: "asks", Events: []string{"verdict", "tag-wanted"}, Retention: "history"}, {Name: "mesh-delivery-checker-findings", Events: []string{"finding-written.*", "finding-cleared.*", "findings-stated.*"}, MaxAgeDays: 7, MaxMiB: 16}}; !reflect.DeepEqual(m.Streams, want) { + if want := []stream{{Name: "checks", Events: []string{"preview.**", "owed.**", "try.**"}}, {Name: "asks", Events: []string{"verdict"}, Retention: "history"}, {Name: "mesh-delivery-checker-findings", Events: []string{"finding-written.*", "finding-cleared.*", "findings-stated.*"}, MaxAgeDays: 7, MaxMiB: 16}}; !reflect.DeepEqual(m.Streams, want) { t.Errorf("streams = %+v, want %+v", m.Streams, want) } // Only the checker's account publishes try.* on its stream checks: the stream is declared by its owner alone, and no other diff --git a/modules/mesh-delivery-checker/module.json b/modules/mesh-delivery-checker/module.json index 3ed9b598..d5fa2154 100644 --- a/modules/mesh-delivery-checker/module.json +++ b/modules/mesh-delivery-checker/module.json @@ -15,7 +15,6 @@ } ], "consumes": [ - "go-package-registry.published", "mesh-catalog.catalog-revision", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", @@ -28,12 +27,11 @@ "mesh-delivery-indexer.views.packages", "mesh-delivery-indexer.views.predefined", "mesh-delivery-indexer.views.seat-definitions", - "npm-package-registry.published", "mesh-conditions.findings-asked" ], "streams": [ { "name": "checks", "events": ["preview.**", "owed.**", "try.**"] }, - { "name": "asks", "events": ["verdict", "tag-wanted"], "retention": "history" }, + { "name": "asks", "events": ["verdict"], "retention": "history" }, { "name": "mesh-delivery-checker-findings", "events": ["finding-written.*", "finding-cleared.*", "findings-stated.*"], "max-age-days": 7, "max-mib": 16 } ], "state": [