diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index bdfdb4a3..b9f38b52 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -27,6 +27,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/process" "mesh-delivery-builder/internal/build" + "mesh-delivery-builder/internal/versions" ) const ( @@ -109,6 +110,16 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return err } defer calls.Close() + // The builder numbers each trunk build at its claim (novox/hq design 51 *The versions*): its state versions, bound once + // the bus holds it; the floor's three sources asked as its own account; each new number said as version.numbered. + versionsKV := process.AwaitBucket(ctx, js, conn, versions.Bucket, "no trunk build is numbered: each is built unnumbered and published by its digest", logger.Printf) + floorWaits := &versions.FloorWaits{} + numberer := &versions.Numberer{Versions: versions.Versions{Store: awaitedVersions{versionsKV}}, Floor: floorVia(calls.Call), Waits: floorWaits, + Said: func(nb versions.NumberedBody) { + if err := emitNumbered(context.WithoutCancel(ctx), js, cred, nb); err != nil { + logger.Printf("%v", err) + } + }} b := &build.Builder{ Claimer: build.Claimer{Store: kvStore{builds}, By: cred.Node + "." + cred.Module}, Asks: buildrun.Asker{Publisher: asks, Waiter: buildrun.LostAskWaiter{Inner: heard, Dead: dead, Hearing: hearing}, Bound: bound}, @@ -121,7 +132,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro Logf: logger.Printf, Stored: storedVia(calls.Call), Node: cred.Node, + Numbers: build.VersionNumbers{N: numberer}, } + // A state bound later numbers the builds looked at without it. + versionsKV.Ready(func(jetstream.KeyValue) { b.Reconsider(ctx) }) // A candidate's preview is merged onto the trunk head the indexer previewed it onto: source.main at the candidate's base // revision (the gate's source state). The source's revisions are watched with their history (a direct get of one revision // is not what a read grants): a revision heard after the candidate that names it builds that candidate then. Where it diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go index f2b4f371..024528d1 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go @@ -72,21 +72,7 @@ func reAsked(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e jetstre // emitAskedAgain says the builder's event build.asked-again (issue 1945): mesh.mod..event.build.asked-again, // with the SDK's envelope headers and the new ask's id as the bus's dedup id. func emitAskedAgain(ctx context.Context, js jetstream.JetStream, cred process.Account, e build.AskedAgain) error { - data, err := json.Marshal(e) - if err != nil { - return err - } - msg := nats.NewMsg(cred.EventSubject(build.AskedAgainEvent)) - id := "asked-again-" + e.New - msg.Header.Set("x-event-id", id) - msg.Header.Set("x-source", cred.Module) - msg.Header.Set("x-node", cred.Node) - msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano)) - msg.Header.Set("content-type", "application/json") - msg.Data = data - pub, cancel := context.WithTimeout(ctx, 10*time.Second) - defer cancel() - if _, err := js.PublishMsg(pub, msg, jetstream.WithMsgID(id)); err != nil { + if err := emitEvent(ctx, js, cred, build.AskedAgainEvent, "asked-again-"+e.New, e); err != nil { return fmt.Errorf("saying %s for %s: %w", build.AskedAgainEvent, e.Key, err) } return nil diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/versions.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/versions.go new file mode 100644 index 00000000..44a53732 --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/versions.go @@ -0,0 +1,176 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/process" + + "mesh-delivery-builder/internal/versions" +) + +// The floor's two module tools (novox/hq design 51 *The floor*); the seat's is `seat:.versions`. +const ( + bridgeTagsTool = "mesh-delivery-forge-bridge.bridge_tags" + requirementsTool = "mesh-delivery-indexer.indexer_requirements" +) + +// awaitedVersions is the state versions as versions.Store: bound once the bus holds it (the grants part makes it, issue +// 1790); until then every use is an error, so a trunk build is built unnumbered and said (build.Builder.number). +type awaitedVersions struct{ b *process.AwaitedBucket } + +var errVersionsMissing = errors.New("the state " + versions.Bucket + " is not on the bus yet") + +func (s awaitedVersions) kv() (jetstream.KeyValue, error) { + if kv := s.b.KV(); kv != nil { + return kv, nil + } + return nil, errVersionsMissing +} + +func (s awaitedVersions) Get(ctx context.Context, key string) ([]byte, uint64, bool, error) { + kv, err := s.kv() + if err != nil { + return nil, 0, false, err + } + e, err := kv.Get(ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return nil, 0, false, nil + } + if err != nil { + return nil, 0, false, err + } + return e.Value(), e.Revision(), true, nil +} + +func versionsConflict(err error) error { + if errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) { + return versions.ErrConflict + } + return err +} + +func (s awaitedVersions) Create(ctx context.Context, key string, value []byte) (uint64, error) { + kv, err := s.kv() + if err != nil { + return 0, err + } + rev, err := kv.Create(ctx, key, value) + return rev, versionsConflict(err) +} + +func (s awaitedVersions) Update(ctx context.Context, key string, value []byte, revision uint64) (uint64, error) { + kv, err := s.kv() + if err != nil { + return 0, err + } + rev, err := kv.Update(ctx, key, value, revision) + return rev, versionsConflict(err) +} + +func (s awaitedVersions) KeysOf(ctx context.Context, filter string) ([]string, error) { + kv, err := s.kv() + if err != nil { + return nil, err + } + l, err := kv.ListKeysFiltered(ctx, filter) + if err != nil { + return nil, err + } + defer func() { _ = l.Stop() }() + var out []string + for k := range l.Keys() { + out = append(out, k) + } + return out, nil +} + +// floorVia is the floor's three sources as calls of the builder's own account (design 51 *The floor*): the seat's verb +// versions, the forge bridge's bridge_tags and the indexer's indexer_requirements. A call that fails is that source +// silent. +func floorVia(call toolCaller) versions.FloorSources { + return versions.FloorSources{ + Seat: func(_ context.Context, seat, name string) ([]string, error) { + raw, err := call("seat:"+seat+".versions", map[string]any{"name": name}) + if err != nil { + return nil, err + } + var a struct { + Versions []struct { + Version string `json:"version"` + } `json:"versions"` + } + if err := json.Unmarshal(raw, &a); err != nil { + return nil, fmt.Errorf("%s's versions answered what is no list of versions: %w", seat, err) + } + var out []string + for _, v := range a.Versions { + if v.Version != "" { + out = append(out, v.Version) + } + } + return out, nil + }, + Tags: func(_ context.Context, repository, prefix string) ([]string, error) { + raw, err := call(bridgeTagsTool, map[string]any{"repository": repository, "prefix": prefix}) + if err != nil { + return nil, err + } + var a struct { + Tags []string `json:"tags"` + } + if err := json.Unmarshal(raw, &a); err != nil { + return nil, fmt.Errorf("bridge_tags answered what is no list of tags: %w", err) + } + return a.Tags, nil + }, + Requirements: func(_ context.Context, name string) ([]string, error) { + raw, err := call(requirementsTool, map[string]any{"package": name}) + if err != nil { + return nil, err + } + var a struct { + Requirements []string `json:"requirements"` + } + if err := json.Unmarshal(raw, &a); err != nil { + return nil, fmt.Errorf("indexer_requirements answered what is no list of versions: %w", err) + } + return a.Requirements, nil + }, + } +} + +// emitNumbered says the builder's event version.numbered (design 51 *Events of the builder*), on its own family apart +// from its builds' ends; the name and version are the bus's dedup id. +func emitNumbered(ctx context.Context, js jetstream.JetStream, cred process.Account, b versions.NumberedBody) error { + if err := emitEvent(ctx, js, cred, versions.NumberedEvent, "numbered-"+versions.Key(b.Name, b.Version), b); err != nil { + return fmt.Errorf("saying %s for %s@%s: %w", versions.NumberedEvent, b.Name, b.Version, err) + } + return nil +} + +// emitEvent says one of the builder's events, mesh.mod..event., with the SDK's envelope headers and id as +// the bus's dedup id. +func emitEvent(ctx context.Context, js jetstream.JetStream, cred process.Account, event, id string, body any) error { + data, err := json.Marshal(body) + if err != nil { + return err + } + msg := nats.NewMsg(cred.EventSubject(event)) + msg.Header.Set("x-event-id", id) + msg.Header.Set("x-source", cred.Module) + msg.Header.Set("x-node", cred.Node) + msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano)) + msg.Header.Set("content-type", "application/json") + msg.Data = data + pub, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + _, err = js.PublishMsg(pub, msg, jetstream.WithMsgID(id)) + return err +} diff --git a/modules/mesh-delivery-builder/internal/build/numbers.go b/modules/mesh-delivery-builder/internal/build/numbers.go new file mode 100644 index 00000000..22ac25ef --- /dev/null +++ b/modules/mesh-delivery-builder/internal/build/numbers.go @@ -0,0 +1,69 @@ +package build + +import ( + "context" + "fmt" + "strconv" + + "git.novox.be/novox/mesh-sdk/go/delivery" + + "mesh-delivery-builder/internal/versions" +) + +// Numbers numbers a trunk build at its claim (novox/hq design 51 *The versions*, ADR 0606, ADR 0617 §1) and records the +// digests it was built with. Nil numbers nothing. +type Numbers interface { + Number(ctx context.Context, module string, m delivery.Module, plan Plan) (versions.Numbering, error) + Built(ctx context.Context, module string, m delivery.Module, plan Plan, got versions.Numbering, made []delivery.BuiltArtifact) error +} + +// VersionNumbers is Numbers over the builder's state versions. +type VersionNumbers struct{ N *versions.Numberer } + +// ClaimOf is the claim a module value's trunk build is numbered by: a package per source revision (Src), the rest per +// fingerprint, in merge order (Src). +func ClaimOf(module string, m delivery.Module, plan Plan) versions.BuildClaim { + c := versions.BuildClaim{Module: module, Path: m.Path, Repository: delivery.RepositoryName(m.Repository), Fingerprint: m.FP2, + Commit: m.Commit, Merge: m.Src} + if m.Src != 0 { + c.Revision = strconv.FormatUint(m.Src, 10) + } + for _, t := range plan.Tasks { + c.Artifacts = append(c.Artifacts, versions.ClaimArtifact{Artifact: t.Artifact, Kind: t.Kind, Language: t.Language}) + } + return c +} + +func (v VersionNumbers) Number(ctx context.Context, module string, m delivery.Module, plan Plan) (versions.Numbering, error) { + return v.N.Number(ctx, ClaimOf(module, m, plan)) +} + +func (v VersionNumbers) Built(ctx context.Context, module string, m delivery.Module, plan Plan, got versions.Numbering, made []delivery.BuiltArtifact) error { + digests := map[string]string{} + for _, a := range made { + digests[a.Name] = a.Digest + } + return v.N.Built(ctx, ClaimOf(module, m, plan), got, digests) +} + +// number numbers a trunk build before its claim: false when it waits for its floor (a package, design 51 *The floor*), +// and then nothing is claimed; the floor's wake looks at the module again. +func (b *Builder) number(ctx context.Context, module string, m delivery.Module, plan Plan) (versions.Numbering, bool) { + if b.Numbers == nil { + return versions.Numbering{}, true + } + got, err := b.Numbers.Number(ctx, module, m, plan) + for _, s := range got.Said { + b.logf("%s at %s: %s", module, m.FP2, s) + } + if err != nil { + // Said, and built as before: an unnumbered build is published by its digest (ADR 0617 §1), numbered at its next look. + b.logf("%s at %s: %v", module, m.FP2, fmt.Errorf("not numbered, the versions cannot be used: %w", err)) + return versions.Numbering{}, true + } + if got.Wait != "" { + b.logf("%s at %s is not built yet: %s", module, m.FP2, got.Wait) + return got, false + } + return got, true +} diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index 106916cf..49c4a744 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -17,6 +17,8 @@ import ( "git.novox.be/novox/mesh-sdk/go/buildstep" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" + + "mesh-delivery-builder/internal/versions" ) // ToolchainEntries are the nodes' toolchain entries of a language (buildrun.ToolchainSet). @@ -79,6 +81,9 @@ type Builder struct { // time.Now. Node is the node the builder runs on, which asked the build. Now func() time.Time Node string + // Numbers numbers each trunk build at its claim and records what it was built with (design 51 *The versions*); nil + // numbers nothing. + Numbers Numbers once sync.Once sem chan struct{} @@ -430,6 +435,13 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) { first := !b.tried[key] b.tried[key] = true b.mu.Unlock() + var numbering versions.Numbering + if trunk && planErr == nil && !nothing { + var goOn bool + if numbering, goOn = b.number(ctx, module, m, plan); !goOn { + return + } + } cl, err := b.Claimer.Claim(ctx, module, m.FP2, m.Manifest, ClaimOptions{TakeOver: first, ImageBuilder: imageSeat, Trunk: trunk, Published: b.isPublished, Stored: b.storedOnce(key)}) if err != nil { @@ -491,6 +503,11 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) { return } b.logBuild(LogEnded(module, m, b.refOf(module, m.FP2), b.Node, b.now(), result)) + if result.State == delivery.BuildDone && b.Numbers != nil && len(numbering.Versions) > 0 { + if err := b.Numbers.Built(ctx, module, m, plan, numbering, result.Artifacts); err != nil { + b.logf("%s: %v", cl.Key, err) + } + } if result.State == delivery.BuildFailed { b.logf("%s: failed: %s", cl.Key, result.Error) } diff --git a/modules/mesh-delivery-builder/internal/build/run_test.go b/modules/mesh-delivery-builder/internal/build/run_test.go index 3a042d25..54a54568 100644 --- a/modules/mesh-delivery-builder/internal/build/run_test.go +++ b/modules/mesh-delivery-builder/internal/build/run_test.go @@ -17,6 +17,8 @@ import ( stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" + + "mesh-delivery-builder/internal/versions" ) // buildModules simulates the two build modules on the nodes: a long tool answers a step and key at once and writes the @@ -433,14 +435,48 @@ func TestACandidatesImageCallCarriesNoPublishAndItsKeyNoReference(t *testing.T) } } -func TestATrunkImageCallCarriesPublish(t *testing.T) { // 1705b +func TestATrunkImageCallCarriesPublish(t *testing.T) { // 1705b; design 51 *The versions*: numbered at the claim r := newRig(t, imageArt) r.puller = "ace" + nums := &fakeNumbers{wait: "m/img: the package waits for its floor"} + r.b.Numbers = nums + r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) + r.b.Wait() + if _, ok := r.st.vals[r.key]; ok || nums.numbered != 1 { + t.Fatalf("a build waiting for its floor was claimed (numbered %d times)", nums.numbered) + } + nums.wait, nums.versions = "", map[string]string{"img": "0.1.4"} r.consider(t, "modules/m") calls := r.bm.callsOf("build_image_build") if len(calls) != 1 || calls[0].body.(map[string]any)["publish"] != true { t.Errorf("calls = %+v, want publish true", calls) } + if nums.built["img"] != digest { + t.Errorf("the built digest recorded = %v, want %s for img", nums.built, digest) + } +} + +// fakeNumbers is Numbers that answers what it is set to and keeps the digests a done build recorded. +type fakeNumbers struct { + wait string + versions map[string]string + numbered int + built map[string]string +} + +func (f *fakeNumbers) Number(context.Context, string, delivery.Module, Plan) (versions.Numbering, error) { + f.numbered++ + return versions.Numbering{Wait: f.wait, Versions: f.versions}, nil +} + +func (f *fakeNumbers) Built(_ context.Context, _ string, _ delivery.Module, _ Plan, got versions.Numbering, made []delivery.BuiltArtifact) error { + f.built = map[string]string{} + for _, a := range made { + if got.Versions[a.Name] != "" { + f.built[a.Name] = a.Digest + } + } + return nil } func TestAnUnpinnedUpstreamFailsNotPinnedAndIsNeverBuiltAgain(t *testing.T) { // B3, ADR 0498 clause 4 diff --git a/modules/mesh-delivery-builder/internal/versions/claim.go b/modules/mesh-delivery-builder/internal/versions/claim.go new file mode 100644 index 00000000..f16ac6d0 --- /dev/null +++ b/modules/mesh-delivery-builder/internal/versions/claim.go @@ -0,0 +1,181 @@ +package versions + +import ( + "context" + "fmt" + "sort" + "strings" + "time" +) + +// The registry seats a version is published to (design 51 *The seats*), by what is published. +const ( + SeatImages = "container-image-registry" + SeatArtifacts = "oci-artifact-registry" + SeatGoPackages = "go-package-registry" + SeatNPM = "npm-package-registry" +) + +// SeatOf is the registry seat an artifact of kind in language is published to, and the kind the builder numbers it as; +// ok is false for what is not numbered (an upstream copy is mirrored by its pinned digest, design 51 *Publishing*). +func SeatOf(kind, language string) (seat, numberedAs string, ok bool) { + switch kind { + case KindImage: + return SeatImages, KindImage, true + case KindBundle, KindArchive: + return SeatArtifacts, kind, true + case KindPackage: + switch language { + case "go": + return SeatGoPackages, KindPackage, true + case "typescript", "javascript", "node": + return SeatNPM, KindPackage, true + } + } + return "", "", false +} + +// NameOf is the name a module's artifact is numbered under: `/`, the name the seat answers it by +// (ADR 0605 §9's `//@…`). +func NameOf(module, artifact string) string { return module + "/" + artifact } + +// TagPrefix is the git tag prefix of a package in the module's directory path: `go/v` for mesh-sdk's `go`, `v` at the +// root (design 51: the builder tags each Go version `go/v0.1.`). +func TagPrefix(path string) string { + path = strings.Trim(path, "/") + if path == "" || path == "." { + return "v" + } + return path + "/v" +} + +// ClaimArtifact is one artifact of a trunk build to number. +type ClaimArtifact struct { + Artifact, Kind, Language string +} + +// BuildClaim is a trunk build at its claim: what Numberer.Number numbers. +type BuildClaim struct { + Module, Path, Repository, Fingerprint, Commit, Series string + // Revision is the source revision of the merge (a package is numbered per revision); Merge its sequence, for order. + Revision string + Merge uint64 + Artifacts []ClaimArtifact +} + +// Numbering is what the claim got. +type Numbering struct { + // Versions are each artifact's version, by artifact; an artifact built unnumbered (its floor not known) has none. + Versions map[string]string + // Unnumbered are the artifacts built and published by digest, numbered once their floor answers (ADR 0617 §1). + Unnumbered []string + // Wait is why the build waits: a package whose floor is not known (design 51 *The floor*). Nothing is claimed. + Wait string + // Said are the builder's narrations of each decision, for its log. + Said []string +} + +// Numberer numbers trunk builds at their claim (design 51 *The versions*, DLV-B2, B3): it asks a name's floor only +// when versions holds nothing for it, records a silent floor as a wait (FloorWaits, woken by its source's event), and +// says each new number (Said, the event version.numbered). +type Numberer struct { + Versions Versions + Floor FloorSources + Waits *FloorWaits + // Holder is the module holding a seat, whose health condition's clear wakes a silent seat source; nil names none. + Holder func(seat string) string + // Said is told each version newly numbered; nil tells nobody. + Said func(NumberedBody) + Now func() time.Time +} + +// Number numbers every numbered artifact of c. An error is a store that cannot be read or written: the claim is then +// built as before, unnumbered, and said. +func (n *Numberer) Number(ctx context.Context, c BuildClaim) (Numbering, error) { + out := Numbering{Versions: map[string]string{}} + arts := append([]ClaimArtifact(nil), c.Artifacts...) + sort.Slice(arts, func(i, j int) bool { return arts[i].Artifact < arts[j].Artifact }) + for _, a := range arts { + seat, kind, ok := SeatOf(a.Kind, a.Language) + if !ok { + continue + } + name := NameOf(c.Module, a.Artifact) + entries, err := n.Versions.Of(ctx, name) + if err != nil { + return out, err + } + var floor Floor + var why map[string]string + if len(entries) == 0 { + floor, why = n.askFloor(ctx, seat, name, c) + } + d, e, err := n.Versions.Number(ctx, Claim{Name: name, Kind: kind, Series: c.Series, Module: c.Module, Fingerprint: c.Fingerprint, + Commit: c.Commit, Revision: c.Revision, Seat: seat, Merge: c.Merge, Floor: floor}) + if err != nil { + return out, err + } + out.Said = append(out.Said, name+": "+d.Why) + switch { + case d.Wait: + n.wait(name, seat, floor, why) + out.Wait = name + ": " + d.Why + return out, nil + case d.Unnumbered: + n.wait(name, seat, floor, why) + out.Unnumbered = append(out.Unnumbered, a.Artifact) + continue + } + if n.Waits != nil { + n.Waits.Answered(name) + } + out.Versions[a.Artifact] = d.Version + if !d.Keep && n.Said != nil { + n.Said(NumberedOf(e, "")) + } + } + return out, nil +} + +func (n *Numberer) askFloor(ctx context.Context, seat, name string, c BuildClaim) (Floor, map[string]string) { + return n.Floor.Ask(ctx, FloorOf{Seat: seat, Name: name, Repository: c.Repository, Prefix: TagPrefix(c.Path)}) +} + +func (n *Numberer) wait(name, seat string, f Floor, why map[string]string) { + if n.Waits == nil { + return + } + holder := "" + if n.Holder != nil { + holder = n.Holder(seat) + } + now := time.Now() + if n.Now != nil { + now = n.Now() + } + n.Waits.Wait(name, seat, holder, f, why, now) +} + +// Built records the digest each numbered artifact was built with, on an entry that holds none yet: what the holder's +// builder_versions question compares (ADR 0605 §4). +func (n *Numberer) Built(ctx context.Context, c BuildClaim, got Numbering, digests map[string]string) error { + var errs []string + for artifact, version := range got.Versions { + d := digests[artifact] + if d == "" { + continue + } + if _, err := n.Versions.Mark(ctx, NameOf(c.Module, artifact), version, func(e *Entry) { + if e.Digest == "" { + e.Digest = d + } + }); err != nil { + errs = append(errs, err.Error()) + } + } + if len(errs) > 0 { + sort.Strings(errs) + return fmt.Errorf("the built digests could not be recorded: %s", strings.Join(errs, "; ")) + } + return nil +} diff --git a/modules/mesh-delivery-builder/internal/versions/floor.go b/modules/mesh-delivery-builder/internal/versions/floor.go index d4b5dbd7..f88403fc 100644 --- a/modules/mesh-delivery-builder/internal/versions/floor.go +++ b/modules/mesh-delivery-builder/internal/versions/floor.go @@ -15,14 +15,18 @@ import ( type FloorSources struct { // Seat is `seat:.versions` of the registry seat the name is published to. Seat func(ctx context.Context, seat, name string) ([]string, error) - // Tags is the forge bridge's bridge_tags: the package's git tags. - Tags func(ctx context.Context, name string) ([]string, error) + // Tags is the forge bridge's bridge_tags: the git tags of the repository the name is built from, under prefix. + Tags func(ctx context.Context, repository, prefix string) ([]string, error) // Requirements is the indexer's indexer_requirements: the highest pseudo-versions consumers require. Requirements func(ctx context.Context, name string) ([]string, error) } -// Ask asks the three sources of name; a source that errs, or is not wired, is named in Silent with its error in why. -func (s FloorSources) Ask(ctx context.Context, seat, name string) (Floor, map[string]string) { +// FloorOf names what a floor is asked for: the name on its seat, and the repository and tag prefix of its git tags. +type FloorOf struct{ Seat, Name, Repository, Prefix string } + +// Ask asks the three sources of a name; a source that errs, or is not wired, is named in Silent with its error in why. +func (s FloorSources) Ask(ctx context.Context, of FloorOf) (Floor, map[string]string) { + seat, name := of.Seat, of.Name var f Floor why := map[string]string{} ask := func(source string, call func() ([]string, error), into *[]string) { @@ -42,7 +46,7 @@ func (s FloorSources) Ask(ctx context.Context, seat, name string) (Floor, map[st seatCall = func() ([]string, error) { return s.Seat(ctx, seat, name) } } if s.Tags != nil { - tagsCall = func() ([]string, error) { return s.Tags(ctx, name) } + tagsCall = func() ([]string, error) { return s.Tags(ctx, of.Repository, of.Prefix) } } if s.Requirements != nil { reqCall = func() ([]string, error) { return s.Requirements(ctx, name) } diff --git a/modules/mesh-delivery-builder/internal/versions/versions_test.go b/modules/mesh-delivery-builder/internal/versions/versions_test.go index e6a90e8c..712ec98d 100644 --- a/modules/mesh-delivery-builder/internal/versions/versions_test.go +++ b/modules/mesh-delivery-builder/internal/versions/versions_test.go @@ -229,13 +229,12 @@ func TestTheBuildersVersions(t *testing.T) { }) t.Run("the floor wakes on its silent source, never a timer", func(t *testing.T) { ctx := context.Background() - down := func(context.Context, string) ([]string, error) { return nil, errors.New("no answer") } src := FloorSources{ Seat: func(context.Context, string, string) ([]string, error) { return []string{"0.1.3"}, nil }, - Tags: down, + Tags: func(context.Context, string, string) ([]string, error) { return nil, errors.New("no answer") }, Requirements: func(context.Context, string) ([]string, error) { return nil, nil }, } - f, why := src.Ask(ctx, "oci-artifact-registry", "m/program") + f, why := src.Ask(ctx, FloorOf{Seat: "oci-artifact-registry", Name: "m/program"}) if f.Known() || !reflect.DeepEqual(f.Silent, []string{SourceTags}) || why[SourceTags] != "no answer" { t.Fatalf("Ask = %+v, %v; want bridge_tags silent", f, why) } @@ -319,6 +318,54 @@ func TestTheBuildersVersions(t *testing.T) { t.Error("a closed window closes again: old-form-closed would be said twice") } }) + t.Run("at the claim", func(t *testing.T) { + ctx := context.Background() + tagsUp := false + var asked []string + src := FloorSources{ + Seat: func(context.Context, string, string) ([]string, error) { return []string{"0.1.3"}, nil }, + Tags: func(_ context.Context, repo, prefix string) ([]string, error) { + asked = append(asked, repo+" "+prefix) + if !tagsUp { + return nil, errors.New("no answer") + } + return []string{"go/v0.1.13"}, nil + }, + Requirements: func(context.Context, string) ([]string, error) { return nil, nil }, + } + var said []NumberedBody + fw := &FloorWaits{} + n := &Numberer{Versions: Versions{Store: newMem()}, Floor: src, Waits: fw, Said: func(b NumberedBody) { said = append(said, b) }} + c := BuildClaim{Module: "m", Path: "go", Repository: "novox/m", Fingerprint: "fp1", Commit: "c1", Revision: "7", Merge: 7, + Artifacts: []ClaimArtifact{{Artifact: "program", Kind: KindBundle, Language: "go"}, {Artifact: "base", Kind: "upstream"}}} + got, err := n.Number(ctx, c) + if err != nil || len(got.Versions) != 0 || !reflect.DeepEqual(got.Unnumbered, []string{"program"}) || got.Wait != "" { + t.Fatalf("silent floor: %+v, %v; want program unnumbered, the upstream copy left out", got, err) + } + if !reflect.DeepEqual(asked, []string{"novox/m go/v"}) || len(fw.Waiting()) != 1 || len(said) != 0 { + t.Fatalf("asked %v, waits %+v, said %v", asked, fw.Waiting(), said) + } + pkg := c + pkg.Artifacts = []ClaimArtifact{{Artifact: "lib", Kind: KindPackage, Language: "go"}} + if got, _ := n.Number(ctx, pkg); got.Wait == "" || len(got.Versions) != 0 { + t.Fatalf("a package with no floor = %+v; want it to wait", got) + } + tagsUp = true + got, err = n.Number(ctx, c) + if err != nil || got.Versions["program"] != "0.1.14" || len(said) != 1 || said[0].Version != "0.1.14" || len(fw.Waiting()) != 1 { + t.Fatalf("floor answered: %+v, %v, said %v, waits %+v; want 0.1.14 said once, lib still waiting", got, err, said, fw.Waiting()) + } + if err := n.Built(ctx, c, got, map[string]string{"program": "sha256:aa"}); err != nil { + t.Fatal(err) + } + again, _ := n.Number(ctx, c) + if again.Versions["program"] != "0.1.14" || len(said) != 1 || len(asked) != 3 { + t.Fatalf("again: %+v, said %d, asked %v; want the number kept, nothing said, the floor not asked", again, len(said), asked) + } + if _, ok, why, _ := n.Versions.Recorded(ctx, "m/program", "0.1.14", "sha256:bb"); ok { + t.Fatalf("another digest recorded: %s", why) + } + }) t.Run("fill", func(t *testing.T) { if _, full := Fill(Cap*79/100, Cap); full { t.Error("79 % is full") diff --git a/modules/mesh-delivery-builder/manifest_test.go b/modules/mesh-delivery-builder/manifest_test.go index aa4939d2..067e3c7a 100644 --- a/modules/mesh-delivery-builder/manifest_test.go +++ b/modules/mesh-delivery-builder/manifest_test.go @@ -108,7 +108,9 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 // 1937/AC3: the builder asks the controller whether the artifact store still holds a done key's blobs (held), and no // other tool; it holds no capacity. // 1955/L7 (ADR 0571 §2): the seat mesh-artifact-sweep's held, the controller's `held` is no longer asked (issue 1955). - if want := []string{"seat:mesh-artifact-sweep.held"}; !reflect.DeepEqual(sorted(m.Invokes), want) || len(m.Config) != 0 { + // Design 51 *The floor*: the floor's three sources, the registry seats' versions, bridge_tags and indexer_requirements. + if want := []string{"mesh-delivery-forge-bridge.bridge_tags", "mesh-delivery-indexer.indexer_requirements", "seat:container-image-registry.versions", + "seat:go-package-registry.versions", "seat:mesh-artifact-sweep.held", "seat:npm-package-registry.versions", "seat:oci-artifact-registry.versions"}; !reflect.DeepEqual(sorted(m.Invokes), want) || len(m.Config) != 0 { t.Errorf("invokes = %v, config = %v, want %v and no config", m.Invokes, m.Config, want) } // The builder is woken by a build module's report of a toolchain, as the indexer is: the bucket may not exist when it @@ -127,7 +129,8 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 if want := []string{"builder_announce_again", "builder_ask_again", "builder_build_again", "builder_builds_since", "builder_seat_queue", "builder_stuck", "builder_versions"}; !reflect.DeepEqual(sorted(m.Tools), want) { t.Errorf("tools = %v, want %v", m.Tools, want) } - if want := []string{"build.asked-again"}; !reflect.DeepEqual(m.Emits, want) { + // Design 51 *Events of the builder*: numbered and old-form-closed on the family version, apart from the builds' ends. + if want := []string{"build.asked-again", "version.numbered", "version.old-form-closed"}; !reflect.DeepEqual(m.Emits, want) { t.Errorf("emits = %v, want %v", m.Emits, want) } // 1955/L7 (ADR 0571 §1): its module log, each build's start and end, kept 30 days; no bare stream. diff --git a/modules/mesh-delivery-builder/module.json b/modules/mesh-delivery-builder/module.json index e1b9fdea..0aa11ed2 100644 --- a/modules/mesh-delivery-builder/module.json +++ b/modules/mesh-delivery-builder/module.json @@ -12,7 +12,9 @@ "builder_versions" ], "emits": [ - "build.asked-again" + "build.asked-again", + "version.numbered", + "version.old-form-closed" ], "consumes": [ "mesh-artifact-sweep.artifacts-gone", @@ -20,7 +22,13 @@ "mesh-build-node.toolchain.reported" ], "invokes": [ - "seat:mesh-artifact-sweep.held" + "mesh-delivery-forge-bridge.bridge_tags", + "mesh-delivery-indexer.indexer_requirements", + "seat:container-image-registry.versions", + "seat:go-package-registry.versions", + "seat:mesh-artifact-sweep.held", + "seat:npm-package-registry.versions", + "seat:oci-artifact-registry.versions" ], "state": [ { "name": "builds", "history": 5 },