Merge branch 'feat/47-G-build' into feat/47-G-build-indexer
This commit is contained in:
@@ -36,7 +36,6 @@ import (
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliverycheckerclient"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/gopackageregistry"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdeliverychecker"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
@@ -1070,49 +1069,22 @@ func TestAMemberNamingAMeshSDKCommitOffTheTrunkIsJudgedAndNotRefused(t *testing.
|
||||
t.Fatalf("verdict = %+v", v)
|
||||
}
|
||||
|
||||
// DLV-C3, DLV-C6 (design 51 *Tagged versions*, ADR 0607 §3, §5, ADR 0620): a group whose mesh-sdk member declares a Go
|
||||
// package, read at its head with no preview yet, says tag-wanted on the checker's stream asks, at the subject and in the body the mesh-delivery-checker
|
||||
// definition names; once the Go seat's published of the tagged version is heard, the other member is built against it.
|
||||
// Design 47 G §528: a group whose mesh-sdk member declares a Go package is judged at once, with no tagged version asked
|
||||
// or waited for, and its other member is built against the group's head.
|
||||
const sdkRepo = "novox/mesh-sdk"
|
||||
b.setTrunk(t, map[string]string{repo: trunk, sdkRepo: trunk})
|
||||
// The member carries no preview of its merge yet, so its manifest is read at its head (the package new in the group).
|
||||
b.heads.Store(sdkRepo+" module.json "+head2, `{"module":"mesh-sdk","build":{"artifacts":[{"name":"go","kind":"package","language":"go","paths":["go"]}]}}`)
|
||||
asks, err := b.js.OrderedConsumer(b.ctx, "ASKS", jetstream.OrderedConsumerConfig{FilterSubjects: []string{meshdeliverychecker.HolderSubjectTagWanted(check.CheckerModule)}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sdk := pull(sdkRepo, 12, head2)
|
||||
sdk.Modules, sdk.Fps = []string{"mesh-sdk"}, map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{"mesh-sdk": {Fp2: "src2:" + strings.Repeat("e", 64)}}
|
||||
key = b.candidate(t, "feat/x", pull(repo, 8, head1), sdk)
|
||||
msg, err := asks.Next(jetstream.FetchMaxWait(20 * time.Second))
|
||||
if err != nil {
|
||||
t.Fatalf("no tag-wanted on the stream asks: %v", err)
|
||||
}
|
||||
var wanted meshdeliverychecker.TagWantedHolderEvent
|
||||
dec := json.NewDecoder(bytes.NewReader(msg.Data()))
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(&wanted); err != nil {
|
||||
t.Fatalf("tag-wanted %s is not the definition's body: %v", msg.Data(), err)
|
||||
}
|
||||
if want := (meshdeliverychecker.TagWantedHolderEvent{Repository: sdkRepo, Branch: "feat/x", Head: head2, Module: "mesh-sdk", Package: "go", Pull: 12}); wanted != want {
|
||||
t.Errorf("tag-wanted = %+v, want %+v", wanted, want)
|
||||
}
|
||||
if v, err := b.buckets[check.CheckerVerdictsBucket].Get(b.ctx, key); err == nil && !strings.Contains(string(v.Value()), `"running"`) {
|
||||
t.Fatalf("the group was judged before its tagged version was published: %s", v.Value())
|
||||
}
|
||||
published, _ := json.Marshal(gopackageregistry.PublishedEvent{Holder: "gitea", Kind: "package", Name: "mesh-sdk/go", Module: check.SDKModulePath, Commit: head2, Tag: "feat-x", Version: "0.1.15-feat-x.1",
|
||||
Digest: "sha256:" + strings.Repeat("0", 64)})
|
||||
if _, err := b.js.Publish(b.ctx, gopackageregistry.SubjectPublished, published); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v := b.verdict(t, key, ended); v.State != meshdeliverycheckerclient.VerdictPass {
|
||||
t.Fatalf("the group's verdict = %+v", v)
|
||||
}
|
||||
b.fake.mu.Lock()
|
||||
got := b.fake.packages[repo]
|
||||
b.fake.mu.Unlock()
|
||||
if got[check.SDKModulePath] != "v0.1.15-feat-x.1" {
|
||||
t.Errorf("mesh-delivery was built against %v, want mesh-sdk's Go module at the group's tagged version v0.1.15-feat-x.1", got)
|
||||
if got[check.SDKModulePath] != head2 {
|
||||
t.Errorf("mesh-delivery was built against %v, want mesh-sdk's Go module at the group's head %s", got, head2)
|
||||
}
|
||||
|
||||
// DLV-C2, DLV-C6 (design 26 *Out of step*, ADR 0620): each check's end says one verdict per pull request on the stream
|
||||
|
||||
@@ -62,9 +62,7 @@ type judgeDeps struct {
|
||||
nodes *nodeNames
|
||||
// predefined is the indexer's kept list of predefined keys; nil reads none.
|
||||
predefined check.PredefinedList
|
||||
// tags says a group's tag-wanted and hears its packages' published (design 51 *Tagged versions*); nil asks none.
|
||||
tags check.TagAsks
|
||||
logf func(format string, args ...any)
|
||||
logf func(format string, args ...any)
|
||||
}
|
||||
|
||||
// keptPredefined reads the indexer's key predefined.list (the seat mesh-catalog's state predefined), bound once its state appears.
|
||||
@@ -234,7 +232,7 @@ func newJudge(d judgeDeps) check.Judge {
|
||||
nodes = d.nodes
|
||||
}
|
||||
ex := &check.Examiner{Nodes: nodes, Compositions: d.compositions, Asks: d.asks, Toolchains: d.tools,
|
||||
Kept: check.Kept{Store: d.kept}, Previews: d.previews, Candidates: d.candidates, Modules: d.repos, Packages: d.repos, Published: d.published, KnownPackages: d.published, Forge: d.forge, Runs: d.repos, Builds: d.builds, BuildEnds: d.ends, BuildWait: d.buildWait, Announce: d.announce, Trunks: d.trunks, Manifests: d.repos, Read: d.repos, Dirs: d.repos, Predefined: d.predefined, Tags: d.tags, Renames: d.renames, SeatDefinitions: d.seatDefs, SeatList: d.seats, Inputs: d.inputs, BeginBound: d.beginBound, Logf: d.logf}
|
||||
Kept: check.Kept{Store: d.kept}, Previews: d.previews, Candidates: d.candidates, Modules: d.repos, Packages: d.repos, Published: d.published, KnownPackages: d.published, Forge: d.forge, Runs: d.repos, Builds: d.builds, BuildEnds: d.ends, BuildWait: d.buildWait, Announce: d.announce, Trunks: d.trunks, Manifests: d.repos, Read: d.repos, Dirs: d.repos, Predefined: d.predefined, Renames: d.renames, SeatDefinitions: d.seatDefs, SeatList: d.seats, Inputs: d.inputs, BeginBound: d.beginBound, Logf: d.logf}
|
||||
if d.nodes != nil && d.builds != nil && d.repos != nil {
|
||||
ex.Serving = runningServes{have: d.nodes, builds: d.builds, read: d.repos, logf: d.logf}
|
||||
}
|
||||
|
||||
@@ -284,26 +284,13 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// once the first views are answered, before any check starts (afterFirstViews).
|
||||
published := &publishedPackages{}
|
||||
compositions.Published = published
|
||||
ch.Runs = repos // the node-engine a check reads beside at its running commit (ADR 0548 §6)
|
||||
// A group's tagged versions (design 51 *Tagged versions*): tag-wanted on the checker's stream asks, and the package
|
||||
// registry seats' published heard on the checker's durable consumer of the shared events stream. Where that consumer is
|
||||
// not there yet a group's check asks and waits in vain until its bound, said in its verdict.
|
||||
tags := tagAsks{js: js, node: cred.Node, heard: &check.PublishedHeard{}, logf: logger.Printf, now: time.Now}
|
||||
ch.Says = tags // a pull request's verdict event, on the same stream asks (design 26 *Out of step*)
|
||||
for subject := range publishedSeats {
|
||||
if err := hub.Handle(subject, func(m jetstream.Msg) {
|
||||
if !tags.heardPublished(m) {
|
||||
_ = m.Ack() // nothing else the checker consumes rides the shared stream
|
||||
}
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
ch.Runs = repos // the node-engine a check reads beside at its running commit (ADR 0548 §6)
|
||||
ch.Says = verdictSays{js: js, node: cred.Node, now: time.Now} // a pull request's verdict event, on the stream asks (design 26 *Out of step*)
|
||||
ch.Judge = newJudge(judgeDeps{
|
||||
asks: buildrun.Asker{Publisher: asks, Waiter: waiter, Bound: stepBound, Dropper: buildrun.SeatDrops{Caller: caller}}, tools: tools, trunks: trunks,
|
||||
kept: kvStore{buckets[check.CheckerKeptBucket]}, previews: kvStore{buckets[check.PreviewsBucket]}, candidates: tryCandidates{candidates},
|
||||
repos: repos, published: published, builds: buildStates, ends: ends, buildWait: set.buildWait, forge: forgeReads{call: caller.Call}, logf: logger.Printf,
|
||||
announce: announce, compositions: compositions, tags: tags, inputs: runInputs{awaitedKV{inputsKV}}, beginBound: beginBound,
|
||||
announce: announce, compositions: compositions, inputs: runInputs{awaitedKV{inputsKV}}, beginBound: beginBound,
|
||||
nodes: openNodeNames(ctx, js, logger.Printf),
|
||||
predefined: keptPredefined{predefined},
|
||||
renames: seatRenames{call: caller.Call},
|
||||
|
||||
@@ -1,97 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"mesh-delivery-checker/internal/check"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/gopackageregistry"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/npmpackageregistry"
|
||||
)
|
||||
|
||||
// publishedSeats are the two package registry seats' published, by subject (their contracts in mesh-contracts): heard on
|
||||
// the checker's durable consumer of the shared events stream, made from the manifest's consumes.
|
||||
var publishedSeats = map[string]string{
|
||||
gopackageregistry.SubjectPublished: gopackageregistry.Seat,
|
||||
npmpackageregistry.SubjectPublished: npmpackageregistry.Seat,
|
||||
}
|
||||
|
||||
// tagAsks is check.TagAsks and check.VerdictSayer on the bus: tag-wanted and verdict published on the checker's own stream
|
||||
// asks, and the published heard.
|
||||
type tagAsks struct {
|
||||
js jetstream.JetStream
|
||||
node string
|
||||
heard *check.PublishedHeard
|
||||
logf func(string, ...any)
|
||||
now func() time.Time
|
||||
nextID func() string
|
||||
}
|
||||
|
||||
// Want publishes one tag-wanted with the checker's event headers; the message id makes a second publish of the same
|
||||
// member's head and package within the stream's duplicate window one event.
|
||||
func (t tagAsks) Want(ctx context.Context, w check.TagWanted) error {
|
||||
body, err := json.Marshal(w)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
msg := nats.NewMsg(eventSubject(check.EventTagWanted))
|
||||
id := fmt.Sprintf("tag-wanted.%s.%s.%s.%s", w.Repository, w.Head, w.Module, w.Package)
|
||||
msg.Header.Set("x-event-id", id)
|
||||
msg.Header.Set("x-source", check.CheckerModule)
|
||||
msg.Header.Set("x-node", t.node)
|
||||
msg.Header.Set("x-time", t.now().UTC().Format(time.RFC3339Nano))
|
||||
msg.Header.Set("content-type", "application/json")
|
||||
msg.Data = body
|
||||
_, err = t.js.PublishMsg(ctx, msg, jetstream.WithMsgID(id))
|
||||
return err
|
||||
}
|
||||
|
||||
// Say publishes one verdict on the checker's own stream asks with the checker's event headers; the message id makes a
|
||||
// second publish of the same pull request's head, verdict and out-of-step files within the duplicate window one event.
|
||||
func (t tagAsks) Say(ctx context.Context, v check.VerdictEvent) error {
|
||||
body, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
msg := nats.NewMsg(eventSubject(check.EventVerdict))
|
||||
id := fmt.Sprintf("verdict.%s.%d.%s.%s.%x", v.Repository, v.Pull, v.Head, v.Verdict, sha256.Sum256([]byte(strings.Join(v.OutOfStep, "\n"))))
|
||||
msg.Header.Set("x-event-id", id)
|
||||
msg.Header.Set("x-source", check.CheckerModule)
|
||||
msg.Header.Set("x-node", t.node)
|
||||
msg.Header.Set("x-time", t.now().UTC().Format(time.RFC3339Nano))
|
||||
msg.Header.Set("content-type", "application/json")
|
||||
msg.Data = body
|
||||
_, err = t.js.PublishMsg(ctx, msg, jetstream.WithMsgID(id))
|
||||
return err
|
||||
}
|
||||
|
||||
func (t tagAsks) Await(ctx context.Context, seat, head string, names []string) (check.TaggedPublished, bool) {
|
||||
return t.heard.Await(ctx, seat, head, names)
|
||||
}
|
||||
|
||||
// heardPublished takes a message of the events consumer in when it is a package registry seat's published, and answers
|
||||
// whether it was: a tagged version wakes the checks waiting on it; any other published is passed over.
|
||||
func (t tagAsks) heardPublished(m jetstream.Msg) bool {
|
||||
seat, ok := publishedSeats[m.Subject()]
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
var p check.TaggedPublished
|
||||
if err := json.Unmarshal(m.Data(), &p); err != nil {
|
||||
t.logf("%s holds what is no published of %s, left: %v", m.Subject(), seat, err)
|
||||
} else if t.heard.Heard(seat, p) {
|
||||
t.logf("%s published %s %s from %s (tag %s)", seat, p.Name, p.Version, p.Commit, p.Tag)
|
||||
}
|
||||
if err := m.Ack(); err != nil {
|
||||
t.logf("%s cannot be acknowledged, so the bus gives it again: %v", m.Subject(), err)
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"mesh-delivery-checker/internal/check"
|
||||
)
|
||||
|
||||
// verdictSays is check.VerdictSayer on the bus: verdict published on the checker's own stream asks.
|
||||
type verdictSays struct {
|
||||
js jetstream.JetStream
|
||||
node string
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
// Say publishes one verdict on the checker's own stream asks with the checker's event headers; the message id makes a
|
||||
// second publish of the same pull request's head, verdict and out-of-step files within the duplicate window one event.
|
||||
func (t verdictSays) Say(ctx context.Context, v check.VerdictEvent) error {
|
||||
body, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
msg := nats.NewMsg(eventSubject(check.EventVerdict))
|
||||
id := fmt.Sprintf("verdict.%s.%d.%s.%s.%x", v.Repository, v.Pull, v.Head, v.Verdict, sha256.Sum256([]byte(strings.Join(v.OutOfStep, "\n"))))
|
||||
msg.Header.Set("x-event-id", id)
|
||||
msg.Header.Set("x-source", check.CheckerModule)
|
||||
msg.Header.Set("x-node", t.node)
|
||||
msg.Header.Set("x-time", t.now().UTC().Format(time.RFC3339Nano))
|
||||
msg.Header.Set("content-type", "application/json")
|
||||
msg.Data = body
|
||||
_, err = t.js.PublishMsg(ctx, msg, jetstream.WithMsgID(id))
|
||||
return err
|
||||
}
|
||||
@@ -11,12 +11,10 @@ require (
|
||||
git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliverycheckerclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/contracts/go/seats/gopackageregistry v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdeliverychecker v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/contracts/go/seats/npmpackageregistry v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
|
||||
@@ -53,9 +53,6 @@ type Case struct {
|
||||
// Judged are the check's own judge's verdicts on the manifests the change brings, by module, as the seat mesh-catalog's
|
||||
// verdict value (judgeManifests): the composer's request carries them in place of the running indexer's (ADR 0578 §2).
|
||||
Judged map[string]json.RawMessage
|
||||
// Tagged are the group's tagged versions of its Go packages the check heard (design 51 *Tagged versions*), nil for none:
|
||||
// a composer built at the node-engine member's head is built against them, as the check's other Go builds are.
|
||||
Tagged buildstep.PackageVersions
|
||||
|
||||
notesMu sync.Mutex
|
||||
// warnings are, by pull request, what part of its check did not run (Case.warn).
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
package check
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// declaredPackage is one package a group member declares: the module, its directory in the repository (RootPath for its
|
||||
// root, empty when not known), and the artifact's name, paths and language.
|
||||
type declaredPackage struct {
|
||||
pull Pull
|
||||
module, dir string
|
||||
name string
|
||||
paths []string
|
||||
language string
|
||||
}
|
||||
|
||||
// manifestAtHead is a touched module's manifest read from the forge at its member's head (the lead's ruling 4 on A2c's
|
||||
// finding 4): with no preview of the merge, a package new in the group is declared only there, never at the trunk. The
|
||||
// module's directory is the indexer's when it knows the module, else modules/<module> or the repository's root, the one
|
||||
// whose manifest names the module. Found is false when no such file names it.
|
||||
func (st *state) manifestAtHead(ctx context.Context, p Pull, module string) (m catalogue.Manifest, dir string, found bool, err error) {
|
||||
var dirs []string
|
||||
if st.j.Dirs != nil {
|
||||
if d, ok := st.j.Dirs.ModuleDir(module); ok {
|
||||
dirs = []string{d}
|
||||
}
|
||||
}
|
||||
if dirs == nil {
|
||||
dirs = []string{"modules/" + module, meshcatalogclient.RootPath}
|
||||
}
|
||||
for _, d := range dirs {
|
||||
file := "module.json"
|
||||
if t := strings.Trim(d, "/"); t != "" && t != meshcatalogclient.RootPath {
|
||||
file = t + "/module.json"
|
||||
}
|
||||
text, err := st.j.Forge.File(ctx, p.Repository, file, p.Head)
|
||||
if errors.Is(err, ErrNotFound) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, "", false, fmt.Errorf("the manifest of %s at the head of %s cannot be read%s: %w", module, pullKey(p), unheardWords(err), err)
|
||||
}
|
||||
m, err := catalogue.Decode([]byte(text))
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, "", false, fmt.Errorf("the manifest %s of %s at the head of %s is no manifest: %w", file, module, pullKey(p), err)
|
||||
}
|
||||
if m.Module == module {
|
||||
return m, d, true, nil
|
||||
}
|
||||
}
|
||||
return catalogue.Manifest{}, "", false, nil
|
||||
}
|
||||
|
||||
// declaredPackages are the packages the case's members declare, read from each touched module's manifest at the merge
|
||||
// preview (the candidate's); with no preview, at the member's head on the forge, so a package new in the group is asked
|
||||
// for on the first check too (trunk's manifest when no forge is read): a member that declares none is not asked for.
|
||||
func (st *state) declaredPackages(ctx context.Context) ([]declaredPackage, error) {
|
||||
if st.j.Read == nil && st.j.Forge == nil {
|
||||
return nil, nil
|
||||
}
|
||||
var trunk map[string]string
|
||||
if st.j.Manifests != nil {
|
||||
trunk = st.j.Manifests.ManifestDigests()
|
||||
}
|
||||
var out []declaredPackage
|
||||
for _, p := range st.c.Pulls {
|
||||
for _, module := range p.Modules {
|
||||
var m catalogue.Manifest
|
||||
var dir string
|
||||
digest := p.Fps[module].Manifest
|
||||
switch {
|
||||
case digest != "" && st.j.Read != nil:
|
||||
var err error
|
||||
if m, err = st.j.Read.Manifest(ctx, digest); err != nil {
|
||||
return nil, fmt.Errorf("the manifest of %s (%s) cannot be read, so the packages it declares are not known: %w", module, pullKey(p), err)
|
||||
}
|
||||
case st.j.Forge != nil:
|
||||
var found bool
|
||||
var err error
|
||||
if m, dir, found, err = st.manifestAtHead(ctx, p, module); err != nil {
|
||||
return nil, fmt.Errorf("%v, so the packages it declares are not known", err)
|
||||
}
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
case trunk[module] != "" && st.j.Read != nil:
|
||||
var err error
|
||||
if m, err = st.j.Read.Manifest(ctx, trunk[module]); err != nil {
|
||||
return nil, fmt.Errorf("the manifest of %s (%s) cannot be read, so the packages it declares are not known: %w", module, pullKey(p), err)
|
||||
}
|
||||
default:
|
||||
continue
|
||||
}
|
||||
if dir == "" && st.j.Dirs != nil {
|
||||
dir, _ = st.j.Dirs.ModuleDir(module)
|
||||
}
|
||||
if m.Build == nil {
|
||||
continue
|
||||
}
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if a.Kind != catalogue.ArtifactPackage {
|
||||
continue
|
||||
}
|
||||
out = append(out, declaredPackage{pull: p, module: module, dir: dir, name: a.Name, paths: a.Paths, language: a.Language})
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -226,7 +226,7 @@ func (e EngineCompositions) inSandbox(ctx context.Context, c *Case, head, node s
|
||||
return a, err
|
||||
}
|
||||
}
|
||||
job := buildstep.ComposeJob{Fetch: buildstep.Fetch{Repository: nodeEngineRepo, Commit: head, Packages: packagesFor(e.Published, c, c.Tagged, nodeEngineRepo)}, Dir: meshcatalogclient.RootPath,
|
||||
job := buildstep.ComposeJob{Fetch: buildstep.Fetch{Repository: nodeEngineRepo, Commit: head, Packages: packagesFor(e.Published, c, nodeEngineRepo)}, Dir: meshcatalogclient.RootPath,
|
||||
Package: composer.Package, Request: raw, Reader: "the node-engine head of " + c.Group}
|
||||
// The ask travels on the bus, whose message limit a whole request passes (1.2 MB for a large node): it goes packed.
|
||||
if err := job.PackRequest(raw); err != nil {
|
||||
|
||||
@@ -243,16 +243,6 @@ func TestTheCompositionIsMadeFromTheEnginesAnswers(t *testing.T) {
|
||||
if pkgs := job.Fetch.Packages; len(pkgs) != 1 || pkgs[trunk.ForgeModulePrefix+"mesh-delivery"] != head1 {
|
||||
t.Errorf("the composer is fetched with the packages %v, want the group's mesh-delivery member at its head %s alone", pkgs, head1)
|
||||
}
|
||||
// ADR 0607 §3, §5, design 51 *Tagged versions*: once the check heard the group's tagged version of a member's Go
|
||||
// package, the composer is built against that, not the member's head, as the check's other Go builds are.
|
||||
const tagged = "v0.1.15-feat-x.1"
|
||||
host.Tagged = buildstep.PackageVersions{trunk.ForgeModulePrefix + "mesh-delivery": tagged}
|
||||
if _, err := ec.Compose(context.Background(), host); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if pkgs := sb.jobs[len(sb.jobs)-1].Fetch.Packages; len(pkgs) != 1 || pkgs[trunk.ForgeModulePrefix+"mesh-delivery"] != tagged {
|
||||
t.Errorf("the composer is fetched with the packages %v, want the group's tagged version %s of mesh-delivery", pkgs, tagged)
|
||||
}
|
||||
if v.State != meshdeliverycheckerclient.CompositionFail || v.Sdk != head2 || !strings.Contains(v.Words, "the new composer refuses it") {
|
||||
t.Errorf("verdict = %+v", v)
|
||||
}
|
||||
|
||||
@@ -114,10 +114,6 @@ type Examiner struct {
|
||||
BuildWait time.Duration
|
||||
// Announce says a preview was written under the key (EventPreviewWritten); nil says nothing.
|
||||
Announce Announcer
|
||||
// Tags says a group's tag-wanted and hears its packages' published (design 51 *Tagged versions*); nil asks none.
|
||||
Tags TagAsks
|
||||
// PublishBound is how long a group's check waits for each package's published; DefaultPublishBound when 0.
|
||||
PublishBound time.Duration
|
||||
// Dirs names a module's directory in its repository, from the indexer's modules, so a manifest is read at a member's
|
||||
// head (tagwanted.go); nil, or a module it does not know, tries modules/<module> and the repository's root.
|
||||
Dirs interface {
|
||||
@@ -287,8 +283,6 @@ type state struct {
|
||||
// unpreviewed are the members (pullKey) with no preview of a merge: a head that does not merge, or a fetch that did not
|
||||
// answer. The indexer holds no manifest of them, so the assigner's composition would only wait: it is not awaited.
|
||||
unpreviewed []string
|
||||
// tags are the group's tagged versions its members are built against (ADR 0607 §3), once the check heard them.
|
||||
tags groupVersions
|
||||
// outOfStep are, by pull request (pullKey), the generated files a level's compare step found out of step (design 26).
|
||||
outOfStep map[string][]string
|
||||
// noCandidate says why the case has no previewed candidate of its heads from the indexer within the bound: for a whole
|
||||
@@ -438,9 +432,6 @@ func (st *state) compose(ctx context.Context) {
|
||||
c.NotRun = late // the builder's lateness, said by name, never the change's refusal (issue 1921)
|
||||
return
|
||||
}
|
||||
// A composer built at the node-engine member's head is built against the group's tagged versions the check heard, as
|
||||
// its other Go builds are (design 51 *Tagged versions*); it composes against the heads' seat list (seatList).
|
||||
st.c.Tagged = st.tags.goPackages
|
||||
comp, err := st.j.Compositions.Compose(ctx, st.c)
|
||||
if err != nil {
|
||||
// Said in the verdict, never left as a composition not yet run (review finding 4).
|
||||
@@ -541,13 +532,6 @@ func (st *state) judgeAll(ctx context.Context) {
|
||||
}
|
||||
return 0
|
||||
})
|
||||
// A group's members are built only after each package they declare is published as the group's tagged version
|
||||
// (design 51 *Tagged versions*): a package not published within the bound fails the check before any member is built,
|
||||
// each failure a fact of the check.
|
||||
if v := st.tagged(ctx); v != nil {
|
||||
st.failedAll(*v)
|
||||
return
|
||||
}
|
||||
var groupPackage *buildstep.FetchPackage
|
||||
met := len(st.facts)
|
||||
for _, p := range pulls {
|
||||
@@ -567,9 +551,7 @@ func (st *state) judgeAll(ctx context.Context) {
|
||||
}
|
||||
args := buildstep.Fetch{Repository: p.Repository, Commit: p.Head, Ref: refOf(p.Number), Base: p.Trunk}
|
||||
if bm == buildrun.NodeBuildModule {
|
||||
if np, ok := st.tags.nodePackageFor(p.Repository); ok {
|
||||
args.Package = &buildstep.FetchPackage{Name: np.Name, Version: np.Version} // the group's tagged version (ADR 0607 §3)
|
||||
} else if groupPackage != nil && p.Repository != SDKRepository {
|
||||
if groupPackage != nil && p.Repository != SDKRepository {
|
||||
args.Package = groupPackage
|
||||
}
|
||||
}
|
||||
@@ -602,10 +584,8 @@ func (st *state) judgeAll(ctx context.Context) {
|
||||
st.unpreviewed = append(st.unpreviewed, pullKey(p))
|
||||
}
|
||||
st.judgedByMember(p, merged)
|
||||
// The old form (ADR 0500 §5): the group's package member packed by the Node fetch, kept only while no tagged
|
||||
// TypeScript version is heard, the form the running checker hands while the new build modules are delivered. Its
|
||||
// removal is a subtask of the milestone after this one's delivery (issue 1966).
|
||||
if p.Repository == SDKRepository && len(st.c.Pulls) > 1 && len(st.tags.node) == 0 && st.j.changesPackage(p, merged) {
|
||||
// The group's package member packed by the Node fetch (ADR 0500 §5).
|
||||
if p.Repository == SDKRepository && len(st.c.Pulls) > 1 && st.j.changesPackage(p, merged) {
|
||||
groupPackage = &buildstep.FetchPackage{Repository: p.Repository, Commit: p.Head, Ref: refOf(p.Number)}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -368,7 +368,7 @@ func (st *state) knownPackages(ctx context.Context) []string {
|
||||
// trunkJudgeVersions are the versions the trunk's manifest validator is built at: the latest published version of each
|
||||
// package, with no group member's head (ADR 0633 clause 6).
|
||||
func (st *state) trunkJudgeVersions() buildstep.PackageVersions {
|
||||
return packagesFor(st.j.Published, &Case{}, nil, DeliveryRepository)
|
||||
return packagesFor(st.j.Published, &Case{}, DeliveryRepository)
|
||||
}
|
||||
|
||||
// judgedVerdicts puts the check's own judge's verdict on each manifest the change brings on the case, as the seat
|
||||
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-delivery/sdk/grouppackages"
|
||||
"git.novox.be/novox/mesh-delivery/sdk/trunk"
|
||||
)
|
||||
|
||||
// **A check builds against the group's own package heads, not against what a member names** (novox/hq ADR 0573 §2, §4;
|
||||
@@ -35,44 +34,27 @@ type Forge interface {
|
||||
|
||||
// packagesOf is the version of each mesh package a Go build of repository (a check ask, the judge, the composer) builds
|
||||
// against (ADR 0573 §2, §4, as ADR 0607 §5 changes §4): packagesFor over the latest published version of each package the
|
||||
// mesh's modules use (Published), the heads of the case's pull requests and the group's tagged versions the check heard
|
||||
// (design 51 *Tagged versions*). Nil for none.
|
||||
// mesh's modules use (Published) and the heads of the case's pull requests. Nil for none.
|
||||
func (st *state) packagesOf(repository string) buildstep.PackageVersions {
|
||||
return packagesFor(st.j.Published, st.c, st.tags.goPackages, repository)
|
||||
return packagesFor(st.j.Published, st.c, repository)
|
||||
}
|
||||
|
||||
// packagesFor is packagesOf for a case, the published versions and the tagged ones: every producer of a build at a group's
|
||||
// heads takes its versions from here, so none leaves a member out (issue 1921). grouppackages.Heads names each package at
|
||||
// its member's head, also a member that publishes no package yet; then, for a package a member of the case publishes, the
|
||||
// group's tagged version of it once heard stands instead of the head, also for a package new in the group, which has no
|
||||
// published version yet. A package of repository itself comes with the tree the build fetched, and is left out.
|
||||
// packagesFor is packagesOf for a case and the published versions: every producer of a build at a group's heads takes its
|
||||
// versions from here, so none leaves a member out (issue 1921). grouppackages.Heads names each package at its member's
|
||||
// head, also a member that publishes no package yet. A package of repository itself comes with the tree the build
|
||||
// fetched, and is left out.
|
||||
func packagesFor(published interface {
|
||||
LatestPackages() buildstep.PackageVersions
|
||||
}, c *Case, tagged buildstep.PackageVersions, repository string) buildstep.PackageVersions {
|
||||
}, c *Case, repository string) buildstep.PackageVersions {
|
||||
latest := buildstep.PackageVersions{}
|
||||
if published != nil {
|
||||
maps.Copy(latest, published.LatestPackages())
|
||||
}
|
||||
for pkg := range tagged {
|
||||
if _, named := latest[pkg]; !named {
|
||||
latest[pkg] = ""
|
||||
}
|
||||
}
|
||||
heads := map[string]string{}
|
||||
for _, m := range c.Pulls {
|
||||
heads[m.Repository] = m.Head
|
||||
}
|
||||
versions := grouppackages.Heads(latest, heads, repository)
|
||||
for pkg, tag := range tagged {
|
||||
repo, _, ok := trunk.PackageRepository(pkg)
|
||||
if tag == "" || !ok || trunk.SameRepository(repo, repository) {
|
||||
continue
|
||||
}
|
||||
if versions == nil {
|
||||
versions = buildstep.PackageVersions{}
|
||||
}
|
||||
versions[pkg] = tag
|
||||
}
|
||||
return versions
|
||||
}
|
||||
|
||||
|
||||
@@ -2,14 +2,10 @@ package check
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"maps"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliverycheckerclient"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
||||
@@ -112,105 +108,23 @@ func TestAGroupMemberIsBuiltAgainstTheSDKMembersHeadWhateverItsGoModNames(t *tes
|
||||
}
|
||||
}
|
||||
|
||||
// Design 51 *Tagged versions* (DLV-C3): a group's check says tag-wanted for each package a member declares and builds no
|
||||
// member before its published; a lone pull request asks nothing; no published within the bound fails naming the package.
|
||||
// Design 47 G §528: a group's check asks no tagged version and waits on none; it builds its members at once, each Go
|
||||
// build against the group's heads.
|
||||
sdk := catalogue.Manifest{Module: "mesh-sdk", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{
|
||||
{Name: "go", Kind: catalogue.ArtifactPackage, Language: "go", Paths: []string{"go"}}, {Name: "lib", Kind: catalogue.ArtifactPackage, Language: "typescript"}}}}
|
||||
contracts := catalogue.Manifest{Module: "mesh-contracts", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{
|
||||
{Name: "go", Kind: catalogue.ArtifactPackage, Language: "go", Paths: []string{"go"}},
|
||||
{Name: "go-gopackageregistry", Kind: catalogue.ArtifactPackage, Language: "go", Paths: []string{"go/seats/gopackageregistry"}}}}}
|
||||
digest, contractsDigest := strings.Repeat("c", 64), strings.Repeat("d", 64)
|
||||
sdkPull := func() meshcatalog.CandidatesByGroupValuePullsItem {
|
||||
p := pullOf(repoSDK, 12, head2)
|
||||
p.Modules = []string{"mesh-sdk"}
|
||||
p.Fps = map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{"mesh-sdk": {Fp2: "src2:" + strings.Repeat("f", 64), Manifest: digest}}
|
||||
return p
|
||||
digest := strings.Repeat("c", 64)
|
||||
sdkPull := pullOf(repoSDK, 12, head2)
|
||||
sdkPull.Modules = []string{"mesh-sdk"}
|
||||
sdkPull.Fps = map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{"mesh-sdk": {Fp2: "src2:" + strings.Repeat("f", 64), Manifest: digest}}
|
||||
b = newBench(t)
|
||||
b.fetch.Modules = []string{"modules/m"}
|
||||
b.ex.Read = fixedManifests{digest: sdk}
|
||||
b.ex.Modules = langs{repoA: {buildrun.GoBuildModule, buildrun.NodeBuildModule}}
|
||||
if v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1), sdkPull)).Verdict; v.State != meshdeliverycheckerclient.VerdictPass {
|
||||
t.Errorf("a group's verdict = %+v", v)
|
||||
}
|
||||
unpreviewed := func() meshcatalog.CandidatesByGroupValuePullsItem { // a package new in the group: no preview carries its manifest yet
|
||||
p := sdkPull()
|
||||
p.Fps = map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{"mesh-sdk": {Fp2: "src2:" + strings.Repeat("f", 64)}}
|
||||
return p
|
||||
}
|
||||
contractsPull := func() meshcatalog.CandidatesByGroupValuePullsItem {
|
||||
p := pullOf("novox/mesh-catalog", 4, strings.Repeat("3", 40))
|
||||
p.Modules = []string{"mesh-contracts"}
|
||||
p.Fps = map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{"mesh-contracts": {Fp2: "src2:" + strings.Repeat("e", 64), Manifest: contractsDigest}}
|
||||
return p
|
||||
}
|
||||
const sdkGo, contractsGo, seatGo = SDKModulePath, "git.novox.be/novox/mesh-catalog/go", "git.novox.be/novox/mesh-catalog/go/seats/gopackageregistry"
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
pulls []meshcatalog.CandidatesByGroupValuePullsItem
|
||||
published []string // the seats that publish the members' heads
|
||||
wants int
|
||||
fails string
|
||||
// built is the Go packages mesh-delivery's member is built against (ADR 0607 §3, §5), node the package its Node
|
||||
// fetch takes from the registry; nil checks none.
|
||||
built map[string]string
|
||||
node map[string]any
|
||||
// noModule: the Go seat's published names no Go module path (ruling 1 on A2d's finding 1: never worked out).
|
||||
noModule bool
|
||||
}{
|
||||
{name: "a group", pulls: []meshcatalog.CandidatesByGroupValuePullsItem{pullOf(repoA, 7, head1), sdkPull()},
|
||||
published: []string{"go-package-registry", "npm-package-registry"}, wants: 2,
|
||||
built: map[string]string{sdkGo: "v0.1.15-feat-x.1"}, node: map[string]any{"name": "lib", "version": "0.1.15-feat-x.1"}},
|
||||
{name: "a package new in the group, read at its head", pulls: []meshcatalog.CandidatesByGroupValuePullsItem{pullOf(repoA, 7, head1), unpreviewed()},
|
||||
published: []string{"go-package-registry", "npm-package-registry"}, wants: 2, built: map[string]string{sdkGo: "v0.1.15-feat-x.1"}},
|
||||
{name: "one head, two Go packages (as mesh-contracts' head)", pulls: []meshcatalog.CandidatesByGroupValuePullsItem{pullOf(repoA, 7, head1), contractsPull()},
|
||||
published: []string{"go-package-registry"}, wants: 2,
|
||||
built: map[string]string{contractsGo: "v0.1.15-feat-x.1", seatGo: "v0.1.3-feat-x.1"}},
|
||||
{name: "a lone pull request", pulls: []meshcatalog.CandidatesByGroupValuePullsItem{sdkPull()}},
|
||||
{name: "a Go published naming no module", pulls: []meshcatalog.CandidatesByGroupValuePullsItem{pullOf(repoA, 7, head1), sdkPull()},
|
||||
published: []string{"go-package-registry", "npm-package-registry"}, wants: 2, noModule: true,
|
||||
fails: "no published of the tagged version of go of mesh-sdk (novox/mesh-sdk#12) on go-package-registry within 50ms"},
|
||||
{name: "a package never published", pulls: []meshcatalog.CandidatesByGroupValuePullsItem{pullOf(repoA, 7, head1), sdkPull()},
|
||||
published: []string{"go-package-registry"}, wants: 2,
|
||||
fails: "no published of the tagged version of lib of mesh-sdk (novox/mesh-sdk#12) on npm-package-registry within 50ms"},
|
||||
} {
|
||||
b := newBench(t)
|
||||
b.fetch.Modules = []string{"modules/m"}
|
||||
b.ex.Read = fixedManifests{digest: sdk, contractsDigest: contracts}
|
||||
b.ex.Modules = langs{repoA: {buildrun.GoBuildModule, buildrun.NodeBuildModule}}
|
||||
raw, _ := json.Marshal(sdk)
|
||||
b.ex.Forge = headFiles{repoSDK + " module.json " + head2: string(raw)}
|
||||
tags := &recordedTags{heard: &PublishedHeard{}, publish: row.published, noModule: row.noModule}
|
||||
b.ex.Tags, b.ex.PublishBound = tags, 50*time.Millisecond
|
||||
b.asks.Answer = func(a buildstep.StepAsk) buildstep.Step {
|
||||
if len(tags.wanted) != row.wants {
|
||||
t.Errorf("%s: a member was asked before every tag-wanted was said: %d of %d", row.name, len(tags.wanted), row.wants)
|
||||
}
|
||||
return b.answer(a)
|
||||
}
|
||||
v := b.judge(t, caseOf(t, row.pulls...)).Verdict
|
||||
if len(tags.wanted) != row.wants {
|
||||
t.Errorf("%s: tag-wanted said %d times, want %d: %+v", row.name, len(tags.wanted), row.wants, tags.wanted)
|
||||
}
|
||||
for _, w := range tags.wanted {
|
||||
if !slices.Contains([]string{repoSDK, "novox/mesh-catalog"}, w.Repository) || w.Branch != "feat/x" || w.Head == head1 || w.Module == "" || w.Pull == 7 || w.Tag != "" {
|
||||
t.Errorf("%s: tag-wanted = %+v", row.name, w)
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case row.fails == "" && v.State != meshdeliverycheckerclient.VerdictPass:
|
||||
t.Errorf("%s: verdict = %+v", row.name, v)
|
||||
case row.fails != "" && (v.State != meshdeliverycheckerclient.VerdictFail || !strings.Contains(v.Failed.Words, row.fails)):
|
||||
t.Errorf("%s: verdict = %+v, want it to fail saying %q", row.name, v, row.fails)
|
||||
case row.fails != "" && len(b.steps.tool("fetch")) != 0:
|
||||
t.Errorf("%s: a member was built though a package was not published", row.name)
|
||||
}
|
||||
if row.built != nil {
|
||||
got := packagesOfFetch(t, b, repoA)
|
||||
for pkg, version := range row.built {
|
||||
if got[pkg] != version {
|
||||
t.Errorf("%s: mesh-delivery is built against %s %q, want the group's tagged version %s (all: %v)", row.name, pkg, got[pkg], version, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
if row.node != nil {
|
||||
if got := nodeFetchOf(b, repoA).args["package"]; !reflect.DeepEqual(got, row.node) {
|
||||
t.Errorf("%s: mesh-delivery's Node fetch takes %v, want the group's tagged version %v from the registry", row.name, got, row.node)
|
||||
}
|
||||
}
|
||||
if got := packagesOfFetch(t, b, repoA); got[SDKModulePath] != head2 {
|
||||
t.Errorf("mesh-delivery is built against mesh-sdk %q, want the group's head %s (all: %v)", got[SDKModulePath], head2, got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -224,40 +138,6 @@ func (h headFiles) File(_ context.Context, repository, path, ref string) (string
|
||||
return "", ErrNotFound
|
||||
}
|
||||
|
||||
// recordedTags are the checker's tag asks as a test hears them: each tag-wanted kept, and the seats in publish answering it
|
||||
// at once with a tagged version, as the builder and a registry seat would.
|
||||
type recordedTags struct {
|
||||
heard *PublishedHeard
|
||||
publish []string
|
||||
// noModule publishes a Go package naming no Go module path, against go-package-registry's contract.
|
||||
noModule bool
|
||||
mu sync.Mutex
|
||||
wanted []TagWanted
|
||||
}
|
||||
|
||||
func (r *recordedTags) Want(_ context.Context, w TagWanted) error {
|
||||
r.mu.Lock()
|
||||
r.wanted = append(r.wanted, w)
|
||||
r.mu.Unlock()
|
||||
for _, seat := range r.publish {
|
||||
version := "0.1.15-feat-x.1"
|
||||
if w.Package == "go-gopackageregistry" {
|
||||
version = "0.1.3-feat-x.1"
|
||||
}
|
||||
module := "" // go-package-registry's published names the Go module path (its contract); npm's names none
|
||||
if seat == "go-package-registry" && !r.noModule {
|
||||
module = map[string]string{repoSDK + " go": SDKModulePath, "novox/mesh-catalog go": "git.novox.be/novox/mesh-catalog/go",
|
||||
"novox/mesh-catalog go-gopackageregistry": "git.novox.be/novox/mesh-catalog/go/seats/gopackageregistry"}[w.Repository+" "+w.Package]
|
||||
}
|
||||
r.heard.Heard(seat, TaggedPublished{Name: w.Package, Kind: "package", Commit: w.Head, Tag: "feat-x", Version: version, Module: module})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *recordedTags) Await(ctx context.Context, seat, head string, names []string) (TaggedPublished, bool) {
|
||||
return r.heard.Await(ctx, seat, head, names)
|
||||
}
|
||||
|
||||
type runsAt map[string]string
|
||||
|
||||
func (r runsAt) CommitOf(module string) string { return r[module] }
|
||||
|
||||
@@ -1,350 +0,0 @@
|
||||
package check
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdeliverychecker"
|
||||
"git.novox.be/novox/mesh-delivery/sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// EventTagWanted is the checker's event that asks the builder for a group member's tagged versions (design 51 *Tagged
|
||||
// versions*, ADR 0610 §1), on the checker's own family asks (its stream asks, kept seven days), apart from the stream
|
||||
// checks, so a waiter on a verdict does not wake on it.
|
||||
const EventTagWanted = meshdeliverychecker.HolderEventTagWanted
|
||||
|
||||
// AsksStream is the checker's stream that carries tag-wanted and verdict, one that keeps its history (ADR 0592 §6): the
|
||||
// bus names it ASKS.
|
||||
const AsksStream = "asks"
|
||||
|
||||
// DefaultPublishBound is how long a group's check waits for the published of each package its members declare before it
|
||||
// fails naming the package (design 51 *Tagged versions*: the publish's bound, 10 minutes).
|
||||
const DefaultPublishBound = 10 * time.Minute
|
||||
|
||||
// TagWanted is the body of tag-wanted, as the seat definition of mesh-delivery-checker says it. The tag is left to the
|
||||
// builder, which makes it from the branch, so one part alone names a branch's tag.
|
||||
type TagWanted = meshdeliverychecker.TagWantedHolderEvent
|
||||
|
||||
// PackageSeats are the registry seats a package of each language is published through (design 51 *The seats*).
|
||||
var PackageSeats = map[string]string{"go": "go-package-registry", "typescript": "npm-package-registry"}
|
||||
|
||||
// TaggedPublished is what a registry seat's published says of a tagged version, as the checker reads it.
|
||||
type TaggedPublished struct {
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
Commit string `json:"commit,omitempty"`
|
||||
Tag string `json:"tag,omitempty"`
|
||||
Version string `json:"version,omitempty"`
|
||||
// Module is the Go module path a Go package is fetched under, as go-package-registry's published names it (its
|
||||
// contract requires it of a Go package).
|
||||
Module string `json:"module,omitempty"`
|
||||
}
|
||||
|
||||
// TagAsks says tag-wanted and hears the registry seats' published: nil asks nothing, and a group's check builds its
|
||||
// members as it did before tagged versions.
|
||||
type TagAsks interface {
|
||||
// Want says one tag-wanted on the checker's family asks.
|
||||
Want(ctx context.Context, t TagWanted) error
|
||||
// Await answers the tagged version a seat published from a head under one of names, once heard; false when ctx ends
|
||||
// first. A head declaring several packages on one seat (mesh-contracts' Go modules) is told apart by the name.
|
||||
Await(ctx context.Context, seat, head string, names []string) (TaggedPublished, bool)
|
||||
}
|
||||
|
||||
// PublishedHeard is the registry seats' published of tagged versions as the checker heard them on its durable consumer
|
||||
// of the shared events stream, by seat and source commit, each name kept: a waiter wakes on the event, never on a timer.
|
||||
type PublishedHeard struct {
|
||||
mu sync.Mutex
|
||||
heard map[string][]TaggedPublished
|
||||
waiters map[string][]publishedWaiter
|
||||
}
|
||||
|
||||
type publishedWaiter struct {
|
||||
names []string
|
||||
c chan TaggedPublished
|
||||
}
|
||||
|
||||
func publishedKey(seat, head string) string { return seat + " " + head }
|
||||
|
||||
func (h *PublishedHeard) init() {
|
||||
if h.heard == nil {
|
||||
h.heard, h.waiters = map[string][]TaggedPublished{}, map[string][]publishedWaiter{}
|
||||
}
|
||||
}
|
||||
|
||||
// Heard takes one published of a seat in; one of no tagged version (no tag, or no commit), and a Go package's that names
|
||||
// no Go module path, are left out and answer false. A later published of the same name and head replaces the earlier.
|
||||
func (h *PublishedHeard) Heard(seat string, p TaggedPublished) bool {
|
||||
if p.Tag == "" || p.Commit == "" || (seat == PackageSeats["go"] && p.Module == "") {
|
||||
return false
|
||||
}
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
h.init()
|
||||
k := publishedKey(seat, p.Commit)
|
||||
h.heard[k] = append(slices.DeleteFunc(h.heard[k], func(o TaggedPublished) bool { return o.Name == p.Name }), p)
|
||||
h.waiters[k] = slices.DeleteFunc(h.waiters[k], func(w publishedWaiter) bool {
|
||||
if !slices.Contains(w.names, p.Name) {
|
||||
return false
|
||||
}
|
||||
w.c <- p
|
||||
return true
|
||||
})
|
||||
return true
|
||||
}
|
||||
|
||||
// Await is TagAsks.Await over what was heard.
|
||||
func (h *PublishedHeard) Await(ctx context.Context, seat, head string, names []string) (TaggedPublished, bool) {
|
||||
k := publishedKey(seat, head)
|
||||
h.mu.Lock()
|
||||
h.init()
|
||||
for _, p := range h.heard[k] {
|
||||
if slices.Contains(names, p.Name) {
|
||||
h.mu.Unlock()
|
||||
return p, true
|
||||
}
|
||||
}
|
||||
w := publishedWaiter{names: names, c: make(chan TaggedPublished, 1)}
|
||||
h.waiters[k] = append(h.waiters[k], w)
|
||||
h.mu.Unlock()
|
||||
select {
|
||||
case p := <-w.c:
|
||||
return p, true
|
||||
case <-ctx.Done():
|
||||
h.mu.Lock()
|
||||
h.waiters[k] = slices.DeleteFunc(h.waiters[k], func(o publishedWaiter) bool { return o.c == w.c })
|
||||
h.mu.Unlock()
|
||||
select {
|
||||
case p := <-w.c: // heard as the wait ended
|
||||
return p, true
|
||||
default:
|
||||
}
|
||||
return TaggedPublished{}, false
|
||||
}
|
||||
}
|
||||
|
||||
// declaredPackage is one package a group member declares: the module, its directory in the repository (RootPath for its
|
||||
// root, empty when not known), the artifact's name and paths, and the seat it is published through.
|
||||
type declaredPackage struct {
|
||||
pull Pull
|
||||
module, dir string
|
||||
name string
|
||||
paths []string
|
||||
language, seat string
|
||||
}
|
||||
|
||||
// publishedNames are the names a registry seat's published may carry for a declared package: the builder's
|
||||
// `<module>/<artifact>` (its versions' name) and the artifact's name. A Go package's module path is the published's own
|
||||
// `module` (go-package-registry's contract), never worked out here.
|
||||
func (d declaredPackage) publishedNames() []string {
|
||||
return []string{d.module + "/" + d.name, d.name}
|
||||
}
|
||||
|
||||
// manifestAtHead is a touched module's manifest read from the forge at its member's head (the lead's ruling 4 on A2c's
|
||||
// finding 4): with no preview of the merge, a package new in the group is declared only there, never at the trunk. The
|
||||
// module's directory is the indexer's when it knows the module, else modules/<module> or the repository's root, the one
|
||||
// whose manifest names the module. Found is false when no such file names it.
|
||||
func (st *state) manifestAtHead(ctx context.Context, p Pull, module string) (m catalogue.Manifest, dir string, found bool, err error) {
|
||||
var dirs []string
|
||||
if st.j.Dirs != nil {
|
||||
if d, ok := st.j.Dirs.ModuleDir(module); ok {
|
||||
dirs = []string{d}
|
||||
}
|
||||
}
|
||||
if dirs == nil {
|
||||
dirs = []string{"modules/" + module, meshcatalogclient.RootPath}
|
||||
}
|
||||
for _, d := range dirs {
|
||||
file := "module.json"
|
||||
if t := strings.Trim(d, "/"); t != "" && t != meshcatalogclient.RootPath {
|
||||
file = t + "/module.json"
|
||||
}
|
||||
text, err := st.j.Forge.File(ctx, p.Repository, file, p.Head)
|
||||
if errors.Is(err, ErrNotFound) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, "", false, fmt.Errorf("the manifest of %s at the head of %s cannot be read%s: %w", module, pullKey(p), unheardWords(err), err)
|
||||
}
|
||||
m, err := catalogue.Decode([]byte(text))
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, "", false, fmt.Errorf("the manifest %s of %s at the head of %s is no manifest: %w", file, module, pullKey(p), err)
|
||||
}
|
||||
if m.Module == module {
|
||||
return m, d, true, nil
|
||||
}
|
||||
}
|
||||
return catalogue.Manifest{}, "", false, nil
|
||||
}
|
||||
|
||||
// declaredPackages are the packages the case's members declare, read from each touched module's manifest at the merge
|
||||
// preview (the candidate's); with no preview, at the member's head on the forge, so a package new in the group is asked
|
||||
// for on the first check too (trunk's manifest when no forge is read): a member that declares none is not asked for.
|
||||
func (st *state) declaredPackages(ctx context.Context) ([]declaredPackage, error) {
|
||||
if st.j.Read == nil && st.j.Forge == nil {
|
||||
return nil, nil
|
||||
}
|
||||
var trunk map[string]string
|
||||
if st.j.Manifests != nil {
|
||||
trunk = st.j.Manifests.ManifestDigests()
|
||||
}
|
||||
var out []declaredPackage
|
||||
for _, p := range st.c.Pulls {
|
||||
for _, module := range p.Modules {
|
||||
var m catalogue.Manifest
|
||||
var dir string
|
||||
digest := p.Fps[module].Manifest
|
||||
switch {
|
||||
case digest != "" && st.j.Read != nil:
|
||||
var err error
|
||||
if m, err = st.j.Read.Manifest(ctx, digest); err != nil {
|
||||
return nil, fmt.Errorf("the manifest of %s (%s) cannot be read, so the packages it declares are not known: %w", module, pullKey(p), err)
|
||||
}
|
||||
case st.j.Forge != nil:
|
||||
var found bool
|
||||
var err error
|
||||
if m, dir, found, err = st.manifestAtHead(ctx, p, module); err != nil {
|
||||
return nil, fmt.Errorf("%v, so the packages it declares are not known", err)
|
||||
}
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
case trunk[module] != "" && st.j.Read != nil:
|
||||
var err error
|
||||
if m, err = st.j.Read.Manifest(ctx, trunk[module]); err != nil {
|
||||
return nil, fmt.Errorf("the manifest of %s (%s) cannot be read, so the packages it declares are not known: %w", module, pullKey(p), err)
|
||||
}
|
||||
default:
|
||||
continue
|
||||
}
|
||||
if dir == "" && st.j.Dirs != nil {
|
||||
dir, _ = st.j.Dirs.ModuleDir(module)
|
||||
}
|
||||
if m.Build == nil {
|
||||
continue
|
||||
}
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if a.Kind != catalogue.ArtifactPackage {
|
||||
continue
|
||||
}
|
||||
seat, ok := PackageSeats[a.Language]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("the package %s of %s (%s) is in %q, which no registry seat publishes", a.Name, module, pullKey(p), a.Language)
|
||||
}
|
||||
out = append(out, declaredPackage{pull: p, module: module, dir: dir, name: a.Name, paths: a.Paths, language: a.Language, seat: seat})
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// groupVersions are a group's tagged versions as its check builds its members against them (ADR 0607 §3, §5): the Go
|
||||
// packages by module path in Go's form (StepAsk.Packages), and the TypeScript packages a member's Node fetch takes from the
|
||||
// registry, by the repository that publishes them.
|
||||
type groupVersions struct {
|
||||
goPackages buildstep.PackageVersions
|
||||
node map[string]NodePackage
|
||||
}
|
||||
|
||||
// NodePackage is a TypeScript package a group's check hands the Node build module's fetch (its `package`): the name and the
|
||||
// tagged version it takes from the registry, its integrity checked against the member's lock (ADR 0607 §3).
|
||||
type NodePackage struct {
|
||||
Name string `json:"name"`
|
||||
Version string `json:"version"`
|
||||
}
|
||||
|
||||
// nodePackageFor is the TypeScript package a member of repository takes as the group's tagged version: another member's
|
||||
// (the Node build module places mesh-sdk's alone, so mesh-sdk's first); false when the group published none.
|
||||
func (g *groupVersions) nodePackageFor(repository string) (NodePackage, bool) {
|
||||
if np, ok := g.node[SDKRepository]; ok && repository != SDKRepository {
|
||||
return np, true
|
||||
}
|
||||
for _, repo := range slices.Sorted(maps.Keys(g.node)) {
|
||||
if repo != repository {
|
||||
return g.node[repo], true
|
||||
}
|
||||
}
|
||||
return NodePackage{}, false
|
||||
}
|
||||
|
||||
// goVersion is a published version in Go's form: a leading v.
|
||||
func goVersion(v string) string {
|
||||
if strings.HasPrefix(v, "v") {
|
||||
return v
|
||||
}
|
||||
return "v" + v
|
||||
}
|
||||
|
||||
// took keeps a declared package's tagged version as the group's.
|
||||
func (g *groupVersions) took(d declaredPackage, p TaggedPublished) {
|
||||
switch d.language {
|
||||
case "go":
|
||||
if g.goPackages == nil {
|
||||
g.goPackages = buildstep.PackageVersions{}
|
||||
}
|
||||
g.goPackages[p.Module] = goVersion(p.Version) // Heard took no Go published without its module
|
||||
case "typescript":
|
||||
if g.node == nil {
|
||||
g.node = map[string]NodePackage{}
|
||||
}
|
||||
g.node[d.pull.Repository] = NodePackage{Name: p.Name, Version: p.Version}
|
||||
}
|
||||
}
|
||||
|
||||
// tagged asks a group's tagged versions before any member is built (design 51 *Tagged versions*): one tag-wanted per
|
||||
// package a member declares, then each package's published awaited within the publish bound. A lone pull request and a
|
||||
// try ask nothing. A package not published within the bound fails the check naming it.
|
||||
func (st *state) tagged(ctx context.Context) *meshdeliverychecker.VerdictsByGroupHeadsValue {
|
||||
if st.j.Tags == nil || st.c.Try != nil || len(st.c.Pulls) < 2 {
|
||||
return nil
|
||||
}
|
||||
decl, err := st.declaredPackages(ctx)
|
||||
if err != nil {
|
||||
return fail(CheckerModule, err.Error())
|
||||
}
|
||||
if len(decl) == 0 {
|
||||
return nil
|
||||
}
|
||||
st.c.Progress.Set("tagged versions", "")
|
||||
var names []string
|
||||
for _, d := range decl {
|
||||
t := TagWanted{Repository: d.pull.Repository, Branch: st.c.Group, Head: d.pull.Head, Module: d.module, Package: d.name, Pull: int64(d.pull.Number)}
|
||||
if err := st.j.Tags.Want(ctx, t); err != nil {
|
||||
return fail(CheckerModule, fmt.Sprintf("the tagged version of %s of %s (%s) cannot be asked: %v", d.name, d.module, pullKey(d.pull), err))
|
||||
}
|
||||
names = append(names, d.module+"/"+d.name)
|
||||
}
|
||||
bound := st.j.PublishBound
|
||||
if bound <= 0 {
|
||||
bound = DefaultPublishBound
|
||||
}
|
||||
waits := fmt.Sprintf("waits on the tagged versions of %s before its members are built, at most %s (design 51 *Tagged versions*)", strings.Join(names, ", "), bound)
|
||||
st.c.note(waits)
|
||||
st.j.logf("%s %s", st.c.Key, waits)
|
||||
at, cancel := context.WithTimeout(ctx, bound)
|
||||
defer cancel()
|
||||
var late []string
|
||||
for _, d := range decl {
|
||||
p, ok := st.j.Tags.Await(at, d.seat, d.pull.Head, d.publishedNames())
|
||||
if !ok {
|
||||
late = append(late, fmt.Sprintf("%s of %s (%s) on %s", d.name, d.module, pullKey(d.pull), d.seat))
|
||||
continue
|
||||
}
|
||||
st.tags.took(d, p)
|
||||
st.c.note(fmt.Sprintf("%s of %s is built as %s %s (tag %s)", d.name, d.module, p.Name, p.Version, p.Tag))
|
||||
}
|
||||
if len(late) > 0 {
|
||||
if ctx.Err() != nil {
|
||||
return fail(CheckerModule, ctx.Err().Error())
|
||||
}
|
||||
return fail(CheckerModule, fmt.Sprintf("no published of the tagged version of %s within %s, so no member is built (design 51 *Tagged versions*)", strings.Join(late, "; "), bound))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -38,3 +38,7 @@ func (ch *Checker) sayVerdict(ctx context.Context, cs *Case, v meshdeliverycheck
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// AsksStream is the checker's stream that carries verdict, one that keeps its history (ADR 0592 §6): the bus names it
|
||||
// ASKS.
|
||||
const AsksStream = "asks"
|
||||
|
||||
@@ -139,10 +139,9 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1
|
||||
t.Errorf("uses = %v, want the Go and Node build seats", m.Uses)
|
||||
}
|
||||
// The checker is woken by a build module's report of a toolchain, as the indexer is: the bucket may not exist when it starts.
|
||||
// And a group's check by the package registry seats' published of its tagged versions (design 51 *Tagged versions*).
|
||||
if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported",
|
||||
if want := []string{"mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported",
|
||||
"mesh-catalog.catalog-revision", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", "mesh-conditions.findings-asked", "mesh-delivery-assigner.views.assignments", "mesh-delivery-indexer.views.candidates", "mesh-delivery-indexer.views.manifests", "mesh-delivery-indexer.views.modules",
|
||||
"mesh-delivery-indexer.views.packages", "mesh-delivery-indexer.views.predefined", "mesh-delivery-indexer.views.seat-definitions", "npm-package-registry.published"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
|
||||
"mesh-delivery-indexer.views.packages", "mesh-delivery-indexer.views.predefined", "mesh-delivery-indexer.views.seat-definitions"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
|
||||
t.Errorf("consumes = %v, want %v", m.Consumes, want)
|
||||
}
|
||||
if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" {
|
||||
@@ -152,10 +151,9 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1
|
||||
t.Errorf("secrets-owner = %q", m.SecretsOwner)
|
||||
}
|
||||
// The checker says each time it wrote a preview or a status, so the indexer's and the forge bridge's watches find the buckets (ADR 0484 clauses 2 and 3).
|
||||
// Its family asks (design 51 *Tagged versions*, design 26): tag-wanted and verdict, seven days, apart from checks so a
|
||||
// waiter on a verdict does not wake on a tag ask; a stream that keeps its history, the predefined form of an event family
|
||||
// Its family asks (design 26): verdict, apart from checks so a waiter on a verdict does not wake on a check; a stream that keeps its history, the predefined form of an event family
|
||||
// (ADR 0592 §6, the manifest validator).
|
||||
if want := []stream{{Name: "checks", Events: []string{"preview.**", "owed.**", "try.**"}}, {Name: "asks", Events: []string{"verdict", "tag-wanted"}, Retention: "history"}, {Name: "mesh-delivery-checker-findings", Events: []string{"finding-written.*", "finding-cleared.*", "findings-stated.*"}, MaxAgeDays: 7, MaxMiB: 16}}; !reflect.DeepEqual(m.Streams, want) {
|
||||
if want := []stream{{Name: "checks", Events: []string{"preview.**", "owed.**", "try.**"}}, {Name: "asks", Events: []string{"verdict"}, Retention: "history"}, {Name: "mesh-delivery-checker-findings", Events: []string{"finding-written.*", "finding-cleared.*", "findings-stated.*"}, MaxAgeDays: 7, MaxMiB: 16}}; !reflect.DeepEqual(m.Streams, want) {
|
||||
t.Errorf("streams = %+v, want %+v", m.Streams, want)
|
||||
}
|
||||
// Only the checker's account publishes try.* on its stream checks: the stream is declared by its owner alone, and no other
|
||||
|
||||
@@ -15,7 +15,6 @@
|
||||
}
|
||||
],
|
||||
"consumes": [
|
||||
"go-package-registry.published",
|
||||
"mesh-catalog.catalog-revision",
|
||||
"mesh-catalog.seat-definition-changed",
|
||||
"mesh-catalog.seat-definition-removed",
|
||||
@@ -28,12 +27,11 @@
|
||||
"mesh-delivery-indexer.views.packages",
|
||||
"mesh-delivery-indexer.views.predefined",
|
||||
"mesh-delivery-indexer.views.seat-definitions",
|
||||
"npm-package-registry.published",
|
||||
"mesh-conditions.findings-asked"
|
||||
],
|
||||
"streams": [
|
||||
{ "name": "checks", "events": ["preview.**", "owed.**", "try.**"] },
|
||||
{ "name": "asks", "events": ["verdict", "tag-wanted"], "retention": "history" },
|
||||
{ "name": "asks", "events": ["verdict"], "retention": "history" },
|
||||
{ "name": "mesh-delivery-checker-findings", "events": ["finding-written.*", "finding-cleared.*", "findings-stated.*"], "max-age-days": 7, "max-mib": 16 }
|
||||
],
|
||||
"state": [
|
||||
|
||||
Reference in New Issue
Block a user