DLV-B2, DLV-B3 wiring: number each trunk build before its claim, record its digest, ask the floor as the builder's own account
A package whose floor is not known waits and nothing is claimed; an image, bundle or archive is built unnumbered. The state versions is bound when the bus holds it, every new number is said as version.numbered, and the manifest invokes the floor's three sources (design 51 The versions, The floor; ADR 0606, ADR 0617 §1). Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig
This commit is contained in:
@@ -27,6 +27,7 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
"mesh-delivery-builder/internal/versions"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -109,6 +110,16 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return err
|
||||
}
|
||||
defer calls.Close()
|
||||
// The builder numbers each trunk build at its claim (novox/hq design 51 *The versions*): its state versions, bound once
|
||||
// the bus holds it; the floor's three sources asked as its own account; each new number said as version.numbered.
|
||||
versionsKV := process.AwaitBucket(ctx, js, conn, versions.Bucket, "no trunk build is numbered: each is built unnumbered and published by its digest", logger.Printf)
|
||||
floorWaits := &versions.FloorWaits{}
|
||||
numberer := &versions.Numberer{Versions: versions.Versions{Store: awaitedVersions{versionsKV}}, Floor: floorVia(calls.Call), Waits: floorWaits,
|
||||
Said: func(nb versions.NumberedBody) {
|
||||
if err := emitNumbered(context.WithoutCancel(ctx), js, cred, nb); err != nil {
|
||||
logger.Printf("%v", err)
|
||||
}
|
||||
}}
|
||||
b := &build.Builder{
|
||||
Claimer: build.Claimer{Store: kvStore{builds}, By: cred.Node + "." + cred.Module},
|
||||
Asks: buildrun.Asker{Publisher: asks, Waiter: buildrun.LostAskWaiter{Inner: heard, Dead: dead, Hearing: hearing}, Bound: bound},
|
||||
@@ -121,7 +132,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
Logf: logger.Printf,
|
||||
Stored: storedVia(calls.Call),
|
||||
Node: cred.Node,
|
||||
Numbers: build.VersionNumbers{N: numberer},
|
||||
}
|
||||
// A state bound later numbers the builds looked at without it.
|
||||
versionsKV.Ready(func(jetstream.KeyValue) { b.Reconsider(ctx) })
|
||||
// A candidate's preview is merged onto the trunk head the indexer previewed it onto: source.main at the candidate's base
|
||||
// revision (the gate's source state). The source's revisions are watched with their history (a direct get of one revision
|
||||
// is not what a read grants): a revision heard after the candidate that names it builds that candidate then. Where it
|
||||
|
||||
@@ -72,21 +72,7 @@ func reAsked(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e jetstre
|
||||
// emitAskedAgain says the builder's event build.asked-again (issue 1945): mesh.mod.<module>.event.build.asked-again,
|
||||
// with the SDK's envelope headers and the new ask's id as the bus's dedup id.
|
||||
func emitAskedAgain(ctx context.Context, js jetstream.JetStream, cred process.Account, e build.AskedAgain) error {
|
||||
data, err := json.Marshal(e)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
msg := nats.NewMsg(cred.EventSubject(build.AskedAgainEvent))
|
||||
id := "asked-again-" + e.New
|
||||
msg.Header.Set("x-event-id", id)
|
||||
msg.Header.Set("x-source", cred.Module)
|
||||
msg.Header.Set("x-node", cred.Node)
|
||||
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
|
||||
msg.Header.Set("content-type", "application/json")
|
||||
msg.Data = data
|
||||
pub, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
if _, err := js.PublishMsg(pub, msg, jetstream.WithMsgID(id)); err != nil {
|
||||
if err := emitEvent(ctx, js, cred, build.AskedAgainEvent, "asked-again-"+e.New, e); err != nil {
|
||||
return fmt.Errorf("saying %s for %s: %w", build.AskedAgainEvent, e.Key, err)
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-builder/internal/versions"
|
||||
)
|
||||
|
||||
// The floor's two module tools (novox/hq design 51 *The floor*); the seat's is `seat:<registry>.versions`.
|
||||
const (
|
||||
bridgeTagsTool = "mesh-delivery-forge-bridge.bridge_tags"
|
||||
requirementsTool = "mesh-delivery-indexer.indexer_requirements"
|
||||
)
|
||||
|
||||
// awaitedVersions is the state versions as versions.Store: bound once the bus holds it (the grants part makes it, issue
|
||||
// 1790); until then every use is an error, so a trunk build is built unnumbered and said (build.Builder.number).
|
||||
type awaitedVersions struct{ b *process.AwaitedBucket }
|
||||
|
||||
var errVersionsMissing = errors.New("the state " + versions.Bucket + " is not on the bus yet")
|
||||
|
||||
func (s awaitedVersions) kv() (jetstream.KeyValue, error) {
|
||||
if kv := s.b.KV(); kv != nil {
|
||||
return kv, nil
|
||||
}
|
||||
return nil, errVersionsMissing
|
||||
}
|
||||
|
||||
func (s awaitedVersions) Get(ctx context.Context, key string) ([]byte, uint64, bool, error) {
|
||||
kv, err := s.kv()
|
||||
if err != nil {
|
||||
return nil, 0, false, err
|
||||
}
|
||||
e, err := kv.Get(ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return nil, 0, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, 0, false, err
|
||||
}
|
||||
return e.Value(), e.Revision(), true, nil
|
||||
}
|
||||
|
||||
func versionsConflict(err error) error {
|
||||
if errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
|
||||
return versions.ErrConflict
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (s awaitedVersions) Create(ctx context.Context, key string, value []byte) (uint64, error) {
|
||||
kv, err := s.kv()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
rev, err := kv.Create(ctx, key, value)
|
||||
return rev, versionsConflict(err)
|
||||
}
|
||||
|
||||
func (s awaitedVersions) Update(ctx context.Context, key string, value []byte, revision uint64) (uint64, error) {
|
||||
kv, err := s.kv()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
rev, err := kv.Update(ctx, key, value, revision)
|
||||
return rev, versionsConflict(err)
|
||||
}
|
||||
|
||||
func (s awaitedVersions) KeysOf(ctx context.Context, filter string) ([]string, error) {
|
||||
kv, err := s.kv()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
l, err := kv.ListKeysFiltered(ctx, filter)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = l.Stop() }()
|
||||
var out []string
|
||||
for k := range l.Keys() {
|
||||
out = append(out, k)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// floorVia is the floor's three sources as calls of the builder's own account (design 51 *The floor*): the seat's verb
|
||||
// versions, the forge bridge's bridge_tags and the indexer's indexer_requirements. A call that fails is that source
|
||||
// silent.
|
||||
func floorVia(call toolCaller) versions.FloorSources {
|
||||
return versions.FloorSources{
|
||||
Seat: func(_ context.Context, seat, name string) ([]string, error) {
|
||||
raw, err := call("seat:"+seat+".versions", map[string]any{"name": name})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var a struct {
|
||||
Versions []struct {
|
||||
Version string `json:"version"`
|
||||
} `json:"versions"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return nil, fmt.Errorf("%s's versions answered what is no list of versions: %w", seat, err)
|
||||
}
|
||||
var out []string
|
||||
for _, v := range a.Versions {
|
||||
if v.Version != "" {
|
||||
out = append(out, v.Version)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
},
|
||||
Tags: func(_ context.Context, repository, prefix string) ([]string, error) {
|
||||
raw, err := call(bridgeTagsTool, map[string]any{"repository": repository, "prefix": prefix})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var a struct {
|
||||
Tags []string `json:"tags"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return nil, fmt.Errorf("bridge_tags answered what is no list of tags: %w", err)
|
||||
}
|
||||
return a.Tags, nil
|
||||
},
|
||||
Requirements: func(_ context.Context, name string) ([]string, error) {
|
||||
raw, err := call(requirementsTool, map[string]any{"package": name})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var a struct {
|
||||
Requirements []string `json:"requirements"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return nil, fmt.Errorf("indexer_requirements answered what is no list of versions: %w", err)
|
||||
}
|
||||
return a.Requirements, nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// emitNumbered says the builder's event version.numbered (design 51 *Events of the builder*), on its own family apart
|
||||
// from its builds' ends; the name and version are the bus's dedup id.
|
||||
func emitNumbered(ctx context.Context, js jetstream.JetStream, cred process.Account, b versions.NumberedBody) error {
|
||||
if err := emitEvent(ctx, js, cred, versions.NumberedEvent, "numbered-"+versions.Key(b.Name, b.Version), b); err != nil {
|
||||
return fmt.Errorf("saying %s for %s@%s: %w", versions.NumberedEvent, b.Name, b.Version, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// emitEvent says one of the builder's events, mesh.mod.<module>.event.<event>, with the SDK's envelope headers and id as
|
||||
// the bus's dedup id.
|
||||
func emitEvent(ctx context.Context, js jetstream.JetStream, cred process.Account, event, id string, body any) error {
|
||||
data, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
msg := nats.NewMsg(cred.EventSubject(event))
|
||||
msg.Header.Set("x-event-id", id)
|
||||
msg.Header.Set("x-source", cred.Module)
|
||||
msg.Header.Set("x-node", cred.Node)
|
||||
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
|
||||
msg.Header.Set("content-type", "application/json")
|
||||
msg.Data = data
|
||||
pub, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
_, err = js.PublishMsg(pub, msg, jetstream.WithMsgID(id))
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-builder/internal/versions"
|
||||
)
|
||||
|
||||
// Numbers numbers a trunk build at its claim (novox/hq design 51 *The versions*, ADR 0606, ADR 0617 §1) and records the
|
||||
// digests it was built with. Nil numbers nothing.
|
||||
type Numbers interface {
|
||||
Number(ctx context.Context, module string, m delivery.Module, plan Plan) (versions.Numbering, error)
|
||||
Built(ctx context.Context, module string, m delivery.Module, plan Plan, got versions.Numbering, made []delivery.BuiltArtifact) error
|
||||
}
|
||||
|
||||
// VersionNumbers is Numbers over the builder's state versions.
|
||||
type VersionNumbers struct{ N *versions.Numberer }
|
||||
|
||||
// ClaimOf is the claim a module value's trunk build is numbered by: a package per source revision (Src), the rest per
|
||||
// fingerprint, in merge order (Src).
|
||||
func ClaimOf(module string, m delivery.Module, plan Plan) versions.BuildClaim {
|
||||
c := versions.BuildClaim{Module: module, Path: m.Path, Repository: delivery.RepositoryName(m.Repository), Fingerprint: m.FP2,
|
||||
Commit: m.Commit, Merge: m.Src}
|
||||
if m.Src != 0 {
|
||||
c.Revision = strconv.FormatUint(m.Src, 10)
|
||||
}
|
||||
for _, t := range plan.Tasks {
|
||||
c.Artifacts = append(c.Artifacts, versions.ClaimArtifact{Artifact: t.Artifact, Kind: t.Kind, Language: t.Language})
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
func (v VersionNumbers) Number(ctx context.Context, module string, m delivery.Module, plan Plan) (versions.Numbering, error) {
|
||||
return v.N.Number(ctx, ClaimOf(module, m, plan))
|
||||
}
|
||||
|
||||
func (v VersionNumbers) Built(ctx context.Context, module string, m delivery.Module, plan Plan, got versions.Numbering, made []delivery.BuiltArtifact) error {
|
||||
digests := map[string]string{}
|
||||
for _, a := range made {
|
||||
digests[a.Name] = a.Digest
|
||||
}
|
||||
return v.N.Built(ctx, ClaimOf(module, m, plan), got, digests)
|
||||
}
|
||||
|
||||
// number numbers a trunk build before its claim: false when it waits for its floor (a package, design 51 *The floor*),
|
||||
// and then nothing is claimed; the floor's wake looks at the module again.
|
||||
func (b *Builder) number(ctx context.Context, module string, m delivery.Module, plan Plan) (versions.Numbering, bool) {
|
||||
if b.Numbers == nil {
|
||||
return versions.Numbering{}, true
|
||||
}
|
||||
got, err := b.Numbers.Number(ctx, module, m, plan)
|
||||
for _, s := range got.Said {
|
||||
b.logf("%s at %s: %s", module, m.FP2, s)
|
||||
}
|
||||
if err != nil {
|
||||
// Said, and built as before: an unnumbered build is published by its digest (ADR 0617 §1), numbered at its next look.
|
||||
b.logf("%s at %s: %v", module, m.FP2, fmt.Errorf("not numbered, the versions cannot be used: %w", err))
|
||||
return versions.Numbering{}, true
|
||||
}
|
||||
if got.Wait != "" {
|
||||
b.logf("%s at %s is not built yet: %s", module, m.FP2, got.Wait)
|
||||
return got, false
|
||||
}
|
||||
return got, true
|
||||
}
|
||||
@@ -17,6 +17,8 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-builder/internal/versions"
|
||||
)
|
||||
|
||||
// ToolchainEntries are the nodes' toolchain entries of a language (buildrun.ToolchainSet).
|
||||
@@ -79,6 +81,9 @@ type Builder struct {
|
||||
// time.Now. Node is the node the builder runs on, which asked the build.
|
||||
Now func() time.Time
|
||||
Node string
|
||||
// Numbers numbers each trunk build at its claim and records what it was built with (design 51 *The versions*); nil
|
||||
// numbers nothing.
|
||||
Numbers Numbers
|
||||
|
||||
once sync.Once
|
||||
sem chan struct{}
|
||||
@@ -430,6 +435,13 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
|
||||
first := !b.tried[key]
|
||||
b.tried[key] = true
|
||||
b.mu.Unlock()
|
||||
var numbering versions.Numbering
|
||||
if trunk && planErr == nil && !nothing {
|
||||
var goOn bool
|
||||
if numbering, goOn = b.number(ctx, module, m, plan); !goOn {
|
||||
return
|
||||
}
|
||||
}
|
||||
cl, err := b.Claimer.Claim(ctx, module, m.FP2, m.Manifest, ClaimOptions{TakeOver: first, ImageBuilder: imageSeat, Trunk: trunk, Published: b.isPublished,
|
||||
Stored: b.storedOnce(key)})
|
||||
if err != nil {
|
||||
@@ -491,6 +503,11 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
|
||||
return
|
||||
}
|
||||
b.logBuild(LogEnded(module, m, b.refOf(module, m.FP2), b.Node, b.now(), result))
|
||||
if result.State == delivery.BuildDone && b.Numbers != nil && len(numbering.Versions) > 0 {
|
||||
if err := b.Numbers.Built(ctx, module, m, plan, numbering, result.Artifacts); err != nil {
|
||||
b.logf("%s: %v", cl.Key, err)
|
||||
}
|
||||
}
|
||||
if result.State == delivery.BuildFailed {
|
||||
b.logf("%s: failed: %s", cl.Key, result.Error)
|
||||
}
|
||||
|
||||
@@ -17,6 +17,8 @@ import (
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-builder/internal/versions"
|
||||
)
|
||||
|
||||
// buildModules simulates the two build modules on the nodes: a long tool answers a step and key at once and writes the
|
||||
@@ -433,14 +435,48 @@ func TestACandidatesImageCallCarriesNoPublishAndItsKeyNoReference(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestATrunkImageCallCarriesPublish(t *testing.T) { // 1705b
|
||||
func TestATrunkImageCallCarriesPublish(t *testing.T) { // 1705b; design 51 *The versions*: numbered at the claim
|
||||
r := newRig(t, imageArt)
|
||||
r.puller = "ace"
|
||||
nums := &fakeNumbers{wait: "m/img: the package waits for its floor"}
|
||||
r.b.Numbers = nums
|
||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
||||
r.b.Wait()
|
||||
if _, ok := r.st.vals[r.key]; ok || nums.numbered != 1 {
|
||||
t.Fatalf("a build waiting for its floor was claimed (numbered %d times)", nums.numbered)
|
||||
}
|
||||
nums.wait, nums.versions = "", map[string]string{"img": "0.1.4"}
|
||||
r.consider(t, "modules/m")
|
||||
calls := r.bm.callsOf("build_image_build")
|
||||
if len(calls) != 1 || calls[0].body.(map[string]any)["publish"] != true {
|
||||
t.Errorf("calls = %+v, want publish true", calls)
|
||||
}
|
||||
if nums.built["img"] != digest {
|
||||
t.Errorf("the built digest recorded = %v, want %s for img", nums.built, digest)
|
||||
}
|
||||
}
|
||||
|
||||
// fakeNumbers is Numbers that answers what it is set to and keeps the digests a done build recorded.
|
||||
type fakeNumbers struct {
|
||||
wait string
|
||||
versions map[string]string
|
||||
numbered int
|
||||
built map[string]string
|
||||
}
|
||||
|
||||
func (f *fakeNumbers) Number(context.Context, string, delivery.Module, Plan) (versions.Numbering, error) {
|
||||
f.numbered++
|
||||
return versions.Numbering{Wait: f.wait, Versions: f.versions}, nil
|
||||
}
|
||||
|
||||
func (f *fakeNumbers) Built(_ context.Context, _ string, _ delivery.Module, _ Plan, got versions.Numbering, made []delivery.BuiltArtifact) error {
|
||||
f.built = map[string]string{}
|
||||
for _, a := range made {
|
||||
if got.Versions[a.Name] != "" {
|
||||
f.built[a.Name] = a.Digest
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestAnUnpinnedUpstreamFailsNotPinnedAndIsNeverBuiltAgain(t *testing.T) { // B3, ADR 0498 clause 4
|
||||
|
||||
@@ -0,0 +1,181 @@
|
||||
package versions
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The registry seats a version is published to (design 51 *The seats*), by what is published.
|
||||
const (
|
||||
SeatImages = "container-image-registry"
|
||||
SeatArtifacts = "oci-artifact-registry"
|
||||
SeatGoPackages = "go-package-registry"
|
||||
SeatNPM = "npm-package-registry"
|
||||
)
|
||||
|
||||
// SeatOf is the registry seat an artifact of kind in language is published to, and the kind the builder numbers it as;
|
||||
// ok is false for what is not numbered (an upstream copy is mirrored by its pinned digest, design 51 *Publishing*).
|
||||
func SeatOf(kind, language string) (seat, numberedAs string, ok bool) {
|
||||
switch kind {
|
||||
case KindImage:
|
||||
return SeatImages, KindImage, true
|
||||
case KindBundle, KindArchive:
|
||||
return SeatArtifacts, kind, true
|
||||
case KindPackage:
|
||||
switch language {
|
||||
case "go":
|
||||
return SeatGoPackages, KindPackage, true
|
||||
case "typescript", "javascript", "node":
|
||||
return SeatNPM, KindPackage, true
|
||||
}
|
||||
}
|
||||
return "", "", false
|
||||
}
|
||||
|
||||
// NameOf is the name a module's artifact is numbered under: `<module>/<artifact>`, the name the seat answers it by
|
||||
// (ADR 0605 §9's `<registry>/<module>/<artifact>@…`).
|
||||
func NameOf(module, artifact string) string { return module + "/" + artifact }
|
||||
|
||||
// TagPrefix is the git tag prefix of a package in the module's directory path: `go/v` for mesh-sdk's `go`, `v` at the
|
||||
// root (design 51: the builder tags each Go version `go/v0.1.<n>`).
|
||||
func TagPrefix(path string) string {
|
||||
path = strings.Trim(path, "/")
|
||||
if path == "" || path == "." {
|
||||
return "v"
|
||||
}
|
||||
return path + "/v"
|
||||
}
|
||||
|
||||
// ClaimArtifact is one artifact of a trunk build to number.
|
||||
type ClaimArtifact struct {
|
||||
Artifact, Kind, Language string
|
||||
}
|
||||
|
||||
// BuildClaim is a trunk build at its claim: what Numberer.Number numbers.
|
||||
type BuildClaim struct {
|
||||
Module, Path, Repository, Fingerprint, Commit, Series string
|
||||
// Revision is the source revision of the merge (a package is numbered per revision); Merge its sequence, for order.
|
||||
Revision string
|
||||
Merge uint64
|
||||
Artifacts []ClaimArtifact
|
||||
}
|
||||
|
||||
// Numbering is what the claim got.
|
||||
type Numbering struct {
|
||||
// Versions are each artifact's version, by artifact; an artifact built unnumbered (its floor not known) has none.
|
||||
Versions map[string]string
|
||||
// Unnumbered are the artifacts built and published by digest, numbered once their floor answers (ADR 0617 §1).
|
||||
Unnumbered []string
|
||||
// Wait is why the build waits: a package whose floor is not known (design 51 *The floor*). Nothing is claimed.
|
||||
Wait string
|
||||
// Said are the builder's narrations of each decision, for its log.
|
||||
Said []string
|
||||
}
|
||||
|
||||
// Numberer numbers trunk builds at their claim (design 51 *The versions*, DLV-B2, B3): it asks a name's floor only
|
||||
// when versions holds nothing for it, records a silent floor as a wait (FloorWaits, woken by its source's event), and
|
||||
// says each new number (Said, the event version.numbered).
|
||||
type Numberer struct {
|
||||
Versions Versions
|
||||
Floor FloorSources
|
||||
Waits *FloorWaits
|
||||
// Holder is the module holding a seat, whose health condition's clear wakes a silent seat source; nil names none.
|
||||
Holder func(seat string) string
|
||||
// Said is told each version newly numbered; nil tells nobody.
|
||||
Said func(NumberedBody)
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
// Number numbers every numbered artifact of c. An error is a store that cannot be read or written: the claim is then
|
||||
// built as before, unnumbered, and said.
|
||||
func (n *Numberer) Number(ctx context.Context, c BuildClaim) (Numbering, error) {
|
||||
out := Numbering{Versions: map[string]string{}}
|
||||
arts := append([]ClaimArtifact(nil), c.Artifacts...)
|
||||
sort.Slice(arts, func(i, j int) bool { return arts[i].Artifact < arts[j].Artifact })
|
||||
for _, a := range arts {
|
||||
seat, kind, ok := SeatOf(a.Kind, a.Language)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
name := NameOf(c.Module, a.Artifact)
|
||||
entries, err := n.Versions.Of(ctx, name)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
var floor Floor
|
||||
var why map[string]string
|
||||
if len(entries) == 0 {
|
||||
floor, why = n.askFloor(ctx, seat, name, c)
|
||||
}
|
||||
d, e, err := n.Versions.Number(ctx, Claim{Name: name, Kind: kind, Series: c.Series, Module: c.Module, Fingerprint: c.Fingerprint,
|
||||
Commit: c.Commit, Revision: c.Revision, Seat: seat, Merge: c.Merge, Floor: floor})
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
out.Said = append(out.Said, name+": "+d.Why)
|
||||
switch {
|
||||
case d.Wait:
|
||||
n.wait(name, seat, floor, why)
|
||||
out.Wait = name + ": " + d.Why
|
||||
return out, nil
|
||||
case d.Unnumbered:
|
||||
n.wait(name, seat, floor, why)
|
||||
out.Unnumbered = append(out.Unnumbered, a.Artifact)
|
||||
continue
|
||||
}
|
||||
if n.Waits != nil {
|
||||
n.Waits.Answered(name)
|
||||
}
|
||||
out.Versions[a.Artifact] = d.Version
|
||||
if !d.Keep && n.Said != nil {
|
||||
n.Said(NumberedOf(e, ""))
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (n *Numberer) askFloor(ctx context.Context, seat, name string, c BuildClaim) (Floor, map[string]string) {
|
||||
return n.Floor.Ask(ctx, FloorOf{Seat: seat, Name: name, Repository: c.Repository, Prefix: TagPrefix(c.Path)})
|
||||
}
|
||||
|
||||
func (n *Numberer) wait(name, seat string, f Floor, why map[string]string) {
|
||||
if n.Waits == nil {
|
||||
return
|
||||
}
|
||||
holder := ""
|
||||
if n.Holder != nil {
|
||||
holder = n.Holder(seat)
|
||||
}
|
||||
now := time.Now()
|
||||
if n.Now != nil {
|
||||
now = n.Now()
|
||||
}
|
||||
n.Waits.Wait(name, seat, holder, f, why, now)
|
||||
}
|
||||
|
||||
// Built records the digest each numbered artifact was built with, on an entry that holds none yet: what the holder's
|
||||
// builder_versions question compares (ADR 0605 §4).
|
||||
func (n *Numberer) Built(ctx context.Context, c BuildClaim, got Numbering, digests map[string]string) error {
|
||||
var errs []string
|
||||
for artifact, version := range got.Versions {
|
||||
d := digests[artifact]
|
||||
if d == "" {
|
||||
continue
|
||||
}
|
||||
if _, err := n.Versions.Mark(ctx, NameOf(c.Module, artifact), version, func(e *Entry) {
|
||||
if e.Digest == "" {
|
||||
e.Digest = d
|
||||
}
|
||||
}); err != nil {
|
||||
errs = append(errs, err.Error())
|
||||
}
|
||||
}
|
||||
if len(errs) > 0 {
|
||||
sort.Strings(errs)
|
||||
return fmt.Errorf("the built digests could not be recorded: %s", strings.Join(errs, "; "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -15,14 +15,18 @@ import (
|
||||
type FloorSources struct {
|
||||
// Seat is `seat:<registry>.versions` of the registry seat the name is published to.
|
||||
Seat func(ctx context.Context, seat, name string) ([]string, error)
|
||||
// Tags is the forge bridge's bridge_tags: the package's git tags.
|
||||
Tags func(ctx context.Context, name string) ([]string, error)
|
||||
// Tags is the forge bridge's bridge_tags: the git tags of the repository the name is built from, under prefix.
|
||||
Tags func(ctx context.Context, repository, prefix string) ([]string, error)
|
||||
// Requirements is the indexer's indexer_requirements: the highest pseudo-versions consumers require.
|
||||
Requirements func(ctx context.Context, name string) ([]string, error)
|
||||
}
|
||||
|
||||
// Ask asks the three sources of name; a source that errs, or is not wired, is named in Silent with its error in why.
|
||||
func (s FloorSources) Ask(ctx context.Context, seat, name string) (Floor, map[string]string) {
|
||||
// FloorOf names what a floor is asked for: the name on its seat, and the repository and tag prefix of its git tags.
|
||||
type FloorOf struct{ Seat, Name, Repository, Prefix string }
|
||||
|
||||
// Ask asks the three sources of a name; a source that errs, or is not wired, is named in Silent with its error in why.
|
||||
func (s FloorSources) Ask(ctx context.Context, of FloorOf) (Floor, map[string]string) {
|
||||
seat, name := of.Seat, of.Name
|
||||
var f Floor
|
||||
why := map[string]string{}
|
||||
ask := func(source string, call func() ([]string, error), into *[]string) {
|
||||
@@ -42,7 +46,7 @@ func (s FloorSources) Ask(ctx context.Context, seat, name string) (Floor, map[st
|
||||
seatCall = func() ([]string, error) { return s.Seat(ctx, seat, name) }
|
||||
}
|
||||
if s.Tags != nil {
|
||||
tagsCall = func() ([]string, error) { return s.Tags(ctx, name) }
|
||||
tagsCall = func() ([]string, error) { return s.Tags(ctx, of.Repository, of.Prefix) }
|
||||
}
|
||||
if s.Requirements != nil {
|
||||
reqCall = func() ([]string, error) { return s.Requirements(ctx, name) }
|
||||
|
||||
@@ -229,13 +229,12 @@ func TestTheBuildersVersions(t *testing.T) {
|
||||
})
|
||||
t.Run("the floor wakes on its silent source, never a timer", func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
down := func(context.Context, string) ([]string, error) { return nil, errors.New("no answer") }
|
||||
src := FloorSources{
|
||||
Seat: func(context.Context, string, string) ([]string, error) { return []string{"0.1.3"}, nil },
|
||||
Tags: down,
|
||||
Tags: func(context.Context, string, string) ([]string, error) { return nil, errors.New("no answer") },
|
||||
Requirements: func(context.Context, string) ([]string, error) { return nil, nil },
|
||||
}
|
||||
f, why := src.Ask(ctx, "oci-artifact-registry", "m/program")
|
||||
f, why := src.Ask(ctx, FloorOf{Seat: "oci-artifact-registry", Name: "m/program"})
|
||||
if f.Known() || !reflect.DeepEqual(f.Silent, []string{SourceTags}) || why[SourceTags] != "no answer" {
|
||||
t.Fatalf("Ask = %+v, %v; want bridge_tags silent", f, why)
|
||||
}
|
||||
@@ -319,6 +318,54 @@ func TestTheBuildersVersions(t *testing.T) {
|
||||
t.Error("a closed window closes again: old-form-closed would be said twice")
|
||||
}
|
||||
})
|
||||
t.Run("at the claim", func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tagsUp := false
|
||||
var asked []string
|
||||
src := FloorSources{
|
||||
Seat: func(context.Context, string, string) ([]string, error) { return []string{"0.1.3"}, nil },
|
||||
Tags: func(_ context.Context, repo, prefix string) ([]string, error) {
|
||||
asked = append(asked, repo+" "+prefix)
|
||||
if !tagsUp {
|
||||
return nil, errors.New("no answer")
|
||||
}
|
||||
return []string{"go/v0.1.13"}, nil
|
||||
},
|
||||
Requirements: func(context.Context, string) ([]string, error) { return nil, nil },
|
||||
}
|
||||
var said []NumberedBody
|
||||
fw := &FloorWaits{}
|
||||
n := &Numberer{Versions: Versions{Store: newMem()}, Floor: src, Waits: fw, Said: func(b NumberedBody) { said = append(said, b) }}
|
||||
c := BuildClaim{Module: "m", Path: "go", Repository: "novox/m", Fingerprint: "fp1", Commit: "c1", Revision: "7", Merge: 7,
|
||||
Artifacts: []ClaimArtifact{{Artifact: "program", Kind: KindBundle, Language: "go"}, {Artifact: "base", Kind: "upstream"}}}
|
||||
got, err := n.Number(ctx, c)
|
||||
if err != nil || len(got.Versions) != 0 || !reflect.DeepEqual(got.Unnumbered, []string{"program"}) || got.Wait != "" {
|
||||
t.Fatalf("silent floor: %+v, %v; want program unnumbered, the upstream copy left out", got, err)
|
||||
}
|
||||
if !reflect.DeepEqual(asked, []string{"novox/m go/v"}) || len(fw.Waiting()) != 1 || len(said) != 0 {
|
||||
t.Fatalf("asked %v, waits %+v, said %v", asked, fw.Waiting(), said)
|
||||
}
|
||||
pkg := c
|
||||
pkg.Artifacts = []ClaimArtifact{{Artifact: "lib", Kind: KindPackage, Language: "go"}}
|
||||
if got, _ := n.Number(ctx, pkg); got.Wait == "" || len(got.Versions) != 0 {
|
||||
t.Fatalf("a package with no floor = %+v; want it to wait", got)
|
||||
}
|
||||
tagsUp = true
|
||||
got, err = n.Number(ctx, c)
|
||||
if err != nil || got.Versions["program"] != "0.1.14" || len(said) != 1 || said[0].Version != "0.1.14" || len(fw.Waiting()) != 1 {
|
||||
t.Fatalf("floor answered: %+v, %v, said %v, waits %+v; want 0.1.14 said once, lib still waiting", got, err, said, fw.Waiting())
|
||||
}
|
||||
if err := n.Built(ctx, c, got, map[string]string{"program": "sha256:aa"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
again, _ := n.Number(ctx, c)
|
||||
if again.Versions["program"] != "0.1.14" || len(said) != 1 || len(asked) != 3 {
|
||||
t.Fatalf("again: %+v, said %d, asked %v; want the number kept, nothing said, the floor not asked", again, len(said), asked)
|
||||
}
|
||||
if _, ok, why, _ := n.Versions.Recorded(ctx, "m/program", "0.1.14", "sha256:bb"); ok {
|
||||
t.Fatalf("another digest recorded: %s", why)
|
||||
}
|
||||
})
|
||||
t.Run("fill", func(t *testing.T) {
|
||||
if _, full := Fill(Cap*79/100, Cap); full {
|
||||
t.Error("79 % is full")
|
||||
|
||||
@@ -108,7 +108,9 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
|
||||
// 1937/AC3: the builder asks the controller whether the artifact store still holds a done key's blobs (held), and no
|
||||
// other tool; it holds no capacity.
|
||||
// 1955/L7 (ADR 0571 §2): the seat mesh-artifact-sweep's held, the controller's `held` is no longer asked (issue 1955).
|
||||
if want := []string{"seat:mesh-artifact-sweep.held"}; !reflect.DeepEqual(sorted(m.Invokes), want) || len(m.Config) != 0 {
|
||||
// Design 51 *The floor*: the floor's three sources, the registry seats' versions, bridge_tags and indexer_requirements.
|
||||
if want := []string{"mesh-delivery-forge-bridge.bridge_tags", "mesh-delivery-indexer.indexer_requirements", "seat:container-image-registry.versions",
|
||||
"seat:go-package-registry.versions", "seat:mesh-artifact-sweep.held", "seat:npm-package-registry.versions", "seat:oci-artifact-registry.versions"}; !reflect.DeepEqual(sorted(m.Invokes), want) || len(m.Config) != 0 {
|
||||
t.Errorf("invokes = %v, config = %v, want %v and no config", m.Invokes, m.Config, want)
|
||||
}
|
||||
// The builder is woken by a build module's report of a toolchain, as the indexer is: the bucket may not exist when it
|
||||
@@ -127,7 +129,8 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
|
||||
if want := []string{"builder_announce_again", "builder_ask_again", "builder_build_again", "builder_builds_since", "builder_seat_queue", "builder_stuck", "builder_versions"}; !reflect.DeepEqual(sorted(m.Tools), want) {
|
||||
t.Errorf("tools = %v, want %v", m.Tools, want)
|
||||
}
|
||||
if want := []string{"build.asked-again"}; !reflect.DeepEqual(m.Emits, want) {
|
||||
// Design 51 *Events of the builder*: numbered and old-form-closed on the family version, apart from the builds' ends.
|
||||
if want := []string{"build.asked-again", "version.numbered", "version.old-form-closed"}; !reflect.DeepEqual(m.Emits, want) {
|
||||
t.Errorf("emits = %v, want %v", m.Emits, want)
|
||||
}
|
||||
// 1955/L7 (ADR 0571 §1): its module log, each build's start and end, kept 30 days; no bare stream.
|
||||
|
||||
@@ -12,7 +12,9 @@
|
||||
"builder_versions"
|
||||
],
|
||||
"emits": [
|
||||
"build.asked-again"
|
||||
"build.asked-again",
|
||||
"version.numbered",
|
||||
"version.old-form-closed"
|
||||
],
|
||||
"consumes": [
|
||||
"mesh-artifact-sweep.artifacts-gone",
|
||||
@@ -20,7 +22,13 @@
|
||||
"mesh-build-node.toolchain.reported"
|
||||
],
|
||||
"invokes": [
|
||||
"seat:mesh-artifact-sweep.held"
|
||||
"mesh-delivery-forge-bridge.bridge_tags",
|
||||
"mesh-delivery-indexer.indexer_requirements",
|
||||
"seat:container-image-registry.versions",
|
||||
"seat:go-package-registry.versions",
|
||||
"seat:mesh-artifact-sweep.held",
|
||||
"seat:npm-package-registry.versions",
|
||||
"seat:oci-artifact-registry.versions"
|
||||
],
|
||||
"state": [
|
||||
{ "name": "builds", "history": 5 },
|
||||
|
||||
Reference in New Issue
Block a user